Random Forest

import numpy as np
from sklearn.datasets import load_wine, fetch_california_housing
from sklearn.model_selection import train_test_split, GridSearchCV, cross_val_score
from sklearn.ensemble import RandomForestClassifier, RandomForestRegressor
from sklearn.metrics import classification_report, mean_absolute_error
import matplotlib.pyplot as plt

# === Classification ===
data = load_wine()
X_train, X_test, y_train, y_test = train_test_split(
    data.data, data.target,
    test_size=0.2, stratify=data.target, random_state=42,
)

clf = RandomForestClassifier(
    n_estimators=500,
    max_depth=None,             # let trees grow; the ensemble controls overfitting
    min_samples_leaf=2,
    n_jobs=-1,                  # use all cores
    random_state=42,
    class_weight='balanced',    # for imbalanced data
)
clf.fit(X_train, y_train)

print(clf.score(X_test, y_test))
print(classification_report(y_test, clf.predict(X_test), target_names=data.target_names))

# === Feature importance ===
import pandas as pd
importances = pd.Series(clf.feature_importances_, index=data.feature_names).sort_values()
importances.tail(10).plot.barh()
plt.title('RF feature importance'); plt.tight_layout(); plt.savefig('rf-importance.png')

# More robust: permutation importance (slower but unbiased toward high-cardinality features)
from sklearn.inspection import permutation_importance
pi = permutation_importance(clf, X_test, y_test, n_repeats=10, random_state=42, n_jobs=-1)

# === Regression ===
hx, hy = fetch_california_housing(return_X_y=True)
rx_tr, rx_te, ry_tr, ry_te = train_test_split(hx, hy, test_size=0.2, random_state=42)
reg = RandomForestRegressor(
    n_estimators=500,
    min_samples_leaf=5,
    n_jobs=-1, random_state=42,
).fit(rx_tr, ry_tr)
print('MAE:', mean_absolute_error(ry_te, reg.predict(rx_te)))

# === Hyperparameter tuning ===
params = {
    'n_estimators':    [200, 500, 1000],
    'max_depth':       [None, 10, 20],
    'min_samples_leaf':[1, 2, 5],
    'max_features':    ['sqrt', 'log2', 0.5],
}
grid = GridSearchCV(
    RandomForestClassifier(n_jobs=-1, random_state=42),
    params, cv=5, n_jobs=-1, scoring='f1_macro',
)
grid.fit(X_train, y_train)
print(grid.best_params_, grid.best_score_)

# === Out-of-bag (OOB) score — free validation ===
clf2 = RandomForestClassifier(n_estimators=500, oob_score=True, bootstrap=True, n_jobs=-1).fit(X_train, y_train)
print('OOB:', clf2.oob_score_)

# === Practical tips ===
#   • Default params are strong; tune n_estimators + max_features first
#   • RF doesn't need feature scaling (tree-based)
#   • Handles missing values via surrogate splits (sklearn: impute first or use HistGradientBoosting)
#   • Slow at predict time for thousands-of-trees forests — use n_estimators=100-200 if latency matters
#   • Replace with GradientBoosting (LightGBM / XGBoost / HistGBDT) when you need top-of-leaderboard accuracy

from sklearn.ensemble import RandomForestClassifier
clf = RandomForestClassifier(n_estimators=300, n_jobs=-1, random_state=0).fit(Xtr, ytr)
print(sorted(zip(clf.feature_importances_, feature_names), reverse=True)[:5])