GBDT (XGBoost / LightGBM)

# pip install xgboost lightgbm catboost
import pandas as pd
import numpy as np
from sklearn.model_selection import train_test_split, StratifiedKFold, cross_val_score
from sklearn.pipeline import Pipeline
from sklearn.compose import ColumnTransformer
from sklearn.preprocessing import StandardScaler, OneHotEncoder
from sklearn.impute import SimpleImputer
from sklearn.metrics import roc_auc_score, classification_report

from xgboost import XGBClassifier
from lightgbm import LGBMClassifier
from catboost import CatBoostClassifier

# Toy dataset
df = pd.read_csv('churn.csv')
y = df.pop('churned')
X = df

num = X.select_dtypes(include='number').columns.tolist()
cat = X.select_dtypes(exclude='number').columns.tolist()

# 1) Shared preprocessing (less needed for GBDT than linear models, but useful for mixed-type)
pre = ColumnTransformer([
    ('num', SimpleImputer(strategy='median'), num),
    ('cat', Pipeline([
        ('imp', SimpleImputer(strategy='most_frequent')),
        ('ohe', OneHotEncoder(handle_unknown='ignore', sparse_output=False)),
    ]), cat),
])

# 2) XGBoost — most popular, mature, GPU support
xgb = Pipeline([
    ('prep', pre),
    ('clf', XGBClassifier(
        n_estimators=600,
        max_depth=6,
        learning_rate=0.05,
        subsample=0.9,
        colsample_bytree=0.9,
        eval_metric='auc',
        n_jobs=-1,
    )),
])

# 3) LightGBM — fast, low memory, leaf-wise growth
lgb = Pipeline([
    ('prep', pre),
    ('clf', LGBMClassifier(
        n_estimators=600,
        learning_rate=0.05,
        num_leaves=63,
        subsample=0.9,
        colsample_bytree=0.9,
        n_jobs=-1,
    )),
])

# 4) CatBoost — handles categoricals natively (skip OHE for cat columns)
cb = CatBoostClassifier(
    iterations=800, learning_rate=0.05, depth=6,
    cat_features=cat,
    verbose=False,
)

# 5) Cross-validate honestly
X_tr, X_te, y_tr, y_te = train_test_split(X, y, stratify=y, test_size=0.2, random_state=42)
cv = StratifiedKFold(5, shuffle=True, random_state=42)

for name, model in [('xgb', xgb), ('lgb', lgb)]:
    scores = cross_val_score(model, X_tr, y_tr, cv=cv, scoring='roc_auc', n_jobs=-1)
    print(f'{name} cv AUC: {scores.mean():.4f} +/- {scores.std():.4f}')

# CatBoost takes raw frame (no OHE) — different fit path
scores_cb = []
for tr, va in cv.split(X_tr, y_tr):
    cb.fit(X_tr.iloc[tr], y_tr.iloc[tr], eval_set=(X_tr.iloc[va], y_tr.iloc[va]), verbose=False)
    scores_cb.append(roc_auc_score(y_tr.iloc[va], cb.predict_proba(X_tr.iloc[va])[:, 1]))
print(f'cat cv AUC: {np.mean(scores_cb):.4f} +/- {np.std(scores_cb):.4f}')

# 6) Train the winner on all training data, evaluate on test
xgb.fit(X_tr, y_tr)
print('test ROC-AUC:', roc_auc_score(y_te, xgb.predict_proba(X_te)[:, 1]))
print(classification_report(y_te, xgb.predict(X_te)))

# 7) Feature importance
import matplotlib.pyplot as plt
importances = xgb.named_steps['clf'].feature_importances_
feature_names = xgb.named_steps['prep'].get_feature_names_out()
order = np.argsort(importances)[-15:]
plt.barh(np.array(feature_names)[order], importances[order])
plt.title('Top 15 feature importances'); plt.tight_layout()
plt.savefig('importances.png'); plt.close()

# 8) SHAP for individual explanations
# pip install shap
# import shap
# explainer = shap.TreeExplainer(xgb.named_steps['clf'])
# shap_values = explainer.shap_values(xgb.named_steps['prep'].transform(X_te))
# shap.summary_plot(shap_values, xgb.named_steps['prep'].transform(X_te), feature_names=feature_names)

# 9) Hyperparameter tuning — Optuna is the modern default
# pip install optuna
# import optuna
# def objective(trial):
#     params = {
#         'max_depth': trial.suggest_int('max_depth', 3, 10),
#         'learning_rate': trial.suggest_float('learning_rate', 1e-3, 1e-1, log=True),
#         'subsample': trial.suggest_float('subsample', 0.6, 1.0),
#     }
#     model = XGBClassifier(n_estimators=300, n_jobs=-1, **params)
#     # cross_val_score etc.
#     return scores.mean()
# study = optuna.create_study(direction='maximize')
# study.optimize(objective, n_trials=50)

# 10) Decision matrix
# - Tabular data with mixed types       -> XGBoost or LightGBM
# - Many categorical features            -> CatBoost (no OHE needed)
# - GPU available + big data             -> XGBoost or LightGBM with GPU
# - Need explainability                  -> any GBDT + SHAP

# 11) Pitfalls
# - Trusting feature importance without permutation tests -> can lie under correlation
# - Forgetting eval_set + early stopping -> overfit
# - One-hot encoding high-cardinality features for XGBoost -> use CatBoost instead
# - Same train/test seed across all experiments -> overfit the seed

import xgboost as xgb
clf = xgb.XGBClassifier(
    n_estimators=500, max_depth=6, learning_rate=0.05,
    subsample=0.9, colsample_bytree=0.9, eval_metric='logloss',
)
clf.fit(Xtr, ytr, eval_set=[(Xte, yte)], verbose=False)