Hyperparameter Tuning

from sklearn.model_selection import (
    train_test_split, GridSearchCV, RandomizedSearchCV,
    HalvingGridSearchCV, cross_val_score,
)
from sklearn.pipeline import Pipeline
from sklearn.preprocessing import StandardScaler
from sklearn.linear_model import LogisticRegression
from sklearn.ensemble import RandomForestClassifier, GradientBoostingClassifier
from sklearn.datasets import load_breast_cancer
from scipy.stats import loguniform, randint
import numpy as np

# 1) Always pipeline preprocessing + estimator
X, y = load_breast_cancer(return_X_y=True)
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42, stratify=y)

pipe = Pipeline([
    ('scale', StandardScaler()),
    ('clf',    LogisticRegression(max_iter=1000)),
])

# 2) Grid search — exhaustive (only for SMALL spaces)
param_grid = {
    'clf__C':       [0.01, 0.1, 1, 10, 100],
    'clf__penalty': ['l1', 'l2'],
    'clf__solver':  ['liblinear', 'saga'],
}
gs = GridSearchCV(pipe, param_grid, cv=5, scoring='roc_auc', n_jobs=-1, verbose=1)
gs.fit(X_train, y_train)
print(gs.best_params_, gs.best_score_)
# Grid scales BAD: 5 × 2 × 2 × 5 folds = 100 fits. Adding one dim with 5 values → 500 fits.

# 3) Random search — usually wins per-budget
param_dist = {
    'clf__C':       loguniform(1e-3, 1e3),
    'clf__penalty': ['l1', 'l2'],
    'clf__solver':  ['liblinear', 'saga'],
}
rs = RandomizedSearchCV(pipe, param_dist, n_iter=50, cv=5, scoring='roc_auc',
                                                n_jobs=-1, random_state=42, verbose=1)
rs.fit(X_train, y_train)
print(rs.best_params_, rs.best_score_)
# 50 random samples often beats a 500-fit grid because random search hits the productive
# regions sooner — Bergstra & Bengio (2012).

# 4) Halving search — start with small data + many configs, prune the losers
from sklearn.experimental import enable_halving_search_cv   # noqa: F401

hs = HalvingGridSearchCV(pipe, param_grid, factor=3, cv=5, scoring='roc_auc',
                                                  n_jobs=-1, random_state=42, verbose=1)
hs.fit(X_train, y_train)
# Spends compute on promising configs, drops the rest early.

# 5) Score on TEST only after tuning is done
from sklearn.metrics import roc_auc_score
print('test AUC:', roc_auc_score(y_test, rs.predict_proba(X_test)[:, 1]))

# 6) Optuna — Bayesian / TPE, modern default for serious tuning
import optuna

def objective(trial):
    model_name = trial.suggest_categorical('model', ['lr', 'rf', 'gb'])
    if model_name == 'lr':
        clf = LogisticRegression(
            C=trial.suggest_float('lr_C', 1e-3, 1e3, log=True),
            penalty=trial.suggest_categorical('lr_pen', ['l1', 'l2']),
            solver='liblinear', max_iter=1000,
        )
    elif model_name == 'rf':
        clf = RandomForestClassifier(
            n_estimators=trial.suggest_int('rf_n', 100, 800),
            max_depth=trial.suggest_int('rf_d', 3, 16),
            min_samples_split=trial.suggest_int('rf_mss', 2, 10),
            random_state=42, n_jobs=-1,
        )
    else:
        clf = GradientBoostingClassifier(
            n_estimators=trial.suggest_int('gb_n', 100, 600),
            max_depth=trial.suggest_int('gb_d', 2, 6),
            learning_rate=trial.suggest_float('gb_lr', 1e-3, 0.3, log=True),
            random_state=42,
        )
    pipe = Pipeline([('scale', StandardScaler()), ('clf', clf)])
    return cross_val_score(pipe, X_train, y_train, cv=5, scoring='roc_auc', n_jobs=-1).mean()

study = optuna.create_study(direction='maximize', sampler=optuna.samplers.TPESampler(seed=42))
study.optimize(objective, n_trials=200, timeout=60 * 30)
print('best:', study.best_value, study.best_params)

# 7) Optuna with pruning — kill bad trials early
from optuna.integration import OptunaSearchCV
from optuna.pruners import MedianPruner

study = optuna.create_study(pruner=MedianPruner(n_startup_trials=10, n_warmup_steps=20),
                                                      sampler=optuna.samplers.TPESampler(seed=42),
                                                      direction='maximize')
# Useful when each trial has an iterative process you can report progress for (boosters, NNs).

# 8) Define a search space sensibly
# • Real-valued knobs with wide ranges → log scale (learning rate, regularisation strength)
# • Counts (n_estimators, max_depth) → integer range
# • Pick categorical params for genuinely discrete choices (boolean flags, solver name)
# • Don't include parameters that always perform best at a known value — fix them outside the search

# 9) Cross-validation strategy
# • StratifiedKFold for classification with imbalanced classes
# • TimeSeriesSplit for temporal data (no leakage from future)
# • GroupKFold when you have user/group IDs
# • Repeated K-Fold for small datasets where one fold split varies a lot

from sklearn.model_selection import StratifiedKFold, TimeSeriesSplit, GroupKFold

cv = StratifiedKFold(n_splits=5, shuffle=True, random_state=42)
# rs = RandomizedSearchCV(pipe, param_dist, cv=cv, …)

# 10) Multi-metric optimisation
study = optuna.create_study(directions=['maximize', 'minimize'])
# Optimise AUC AND latency simultaneously — Optuna tracks the Pareto front.

# 11) Tune the budget, not just the model
# Decide ahead of time:
#   • Wall-clock budget: 30 minutes / 2 hours / overnight
#   • Trial count target: 100 / 500 / 2000
#   • CV folds: 3 (fast) / 5 (default) / 10 (robust on small data)
#   • Parallelism: n_jobs across cores; distributed: Optuna RDB storage + workers

# 12) Validation curves — eyeball one hyperparameter
from sklearn.model_selection import validation_curve
import matplotlib.pyplot as plt

C_values = np.logspace(-3, 3, 13)
train, test = validation_curve(
    LogisticRegression(max_iter=1000), X_train, y_train,
    param_name='C', param_range=C_values, scoring='roc_auc', cv=5,
)
plt.semilogx(C_values, train.mean(axis=1), label='train')
plt.semilogx(C_values, test.mean(axis=1),  label='val')
plt.xlabel('C'); plt.ylabel('AUC'); plt.legend()

# 13) Learning curves — is more data the answer?
from sklearn.model_selection import learning_curve
sizes, train_scores, test_scores = learning_curve(
    LogisticRegression(C=1, max_iter=1000), X_train, y_train,
    train_sizes=np.linspace(0.1, 1.0, 6), cv=5, scoring='roc_auc',
)
plt.plot(sizes, test_scores.mean(axis=1))
# Plateau → model can't use more data. Still climbing → collect more.

# 14) Avoiding leakage
# • Pipeline so preprocessing fits ONLY on training folds
# • Don't peek at the test set during tuning
# • Group-aware CV when records share an entity (same user across multiple rows)
# • Use a holdout set (train / val / test) for very long tuning runs

# 15) Reproducibility
# • random_state on every model + sampler
# • Pin library versions (requirements.txt with exact pins)
# • Set numpy seed and PYTHONHASHSEED for hash-based ops
# • Log all hyperparameters and seeds with MLflow / W&B / Aim

# 16) When tuning is over-engineering
# • Quick prototype: defaults of a strong model (XGBoost / LightGBM) + light random search
# • New problem: focus on data quality and features before deep tuning
# • Limited data: more folds, fewer hyperparams (over-tuning amplifies CV variance)
# • Production model: re-tune on a schedule with fresh data; don't ship a one-shot manual best

# 17) Common bugs
# • Tuning on the test set → optimistic, not deployable
# • Forgetting to fit preprocessing INSIDE the CV fold → leakage
# • Grid search with 6+ parameters → combinatorial explosion; switch to random / Optuna
# • Picking the search range too narrow → optimum at the edge; expand and rerun
# • Optimising for accuracy on imbalanced data → always shows 'class predicts majority'; use AUC, F1, log loss
# • Stochastic estimators without random_state → different scores every run; can't reproduce best
# • Not budgeting the run → 'I'll just let it cook'; finish at 3am with no actionable result

from sklearn.model_selection import GridSearchCV
grid = GridSearchCV(
    pipe,
    param_grid={'clf__C': [0.01, 0.1, 1, 10]},
    cv=5, scoring='f1_macro',
).fit(X, y)
print(grid.best_params_, grid.best_score_)