S091 · Gradient Boosting — XGBoost, LightGBM

# gradient_boosting.py — XGBoost + LightGBM in production-ready form, ~130 lines.
import numpy as np
import pandas as pd
from sklearn.datasets import fetch_california_housing
from sklearn.model_selection import train_test_split
from sklearn.metrics import mean_squared_error, r2_score
import xgboost as xgb
 
# If LightGBM installed, uncomment:
# import lightgbm as lgb
 
np.random.seed(42)
 
# --- Data ---
data = fetch_california_housing(as_frame=True)
X: pd.DataFrame = data.frame.drop(columns=["MedHouseVal"])
y: pd.Series = data.frame["MedHouseVal"]
 
X_trainval, X_test, y_trainval, y_test = train_test_split(X, y, test_size=0.15, random_state=42)
X_train, X_val, y_train, y_val = train_test_split(X_trainval, y_trainval, test_size=0.20, random_state=42)
print(f"train={len(X_train)}  val={len(X_val)}  test={len(X_test)}")
 
# =====================================================
# 1. XGBoost with sane defaults + early stopping
# =====================================================
model = xgb.XGBRegressor(
    n_estimators=2000,           # generous upper bound; early stopping decides
    learning_rate=0.05,
    max_depth=6,
    subsample=0.8,
    colsample_bytree=0.8,
    reg_alpha=0.0,
    reg_lambda=1.0,
    min_child_weight=5,
    random_state=42,
    n_jobs=-1,
    early_stopping_rounds=50,    # stop if val doesn't improve for 50 rounds
    eval_metric="rmse",
)
model.fit(X_train, y_train, eval_set=[(X_val, y_val)], verbose=False)
print(f"[XGBoost]      stopped at round {model.best_iteration}")
print(f"  train RMSE = {mean_squared_error(y_train, model.predict(X_train), squared=False):.4f}")
print(f"  val   RMSE = {mean_squared_error(y_val,   model.predict(X_val),   squared=False):.4f}")
print(f"  test  RMSE = {mean_squared_error(y_test,  model.predict(X_test),  squared=False):.4f}")
 
# =====================================================
# 2. Manual boosting from scratch — for understanding
# =====================================================
from sklearn.tree import DecisionTreeRegressor
 
def gbm_scratch(X_train, y_train, X_test, n_estimators=200, learning_rate=0.05, max_depth=4):
    """Very simple gradient boosting for regression with MSE loss."""
    # Init prediction = mean
    F_train = np.full(len(y_train), y_train.mean())
    F_test = np.full(X_test.shape[0], y_train.mean())
    trees = []
    for m in range(n_estimators):
        residual = y_train - F_train    # for MSE, negative gradient = residual
        tree = DecisionTreeRegressor(max_depth=max_depth, random_state=m)
        tree.fit(X_train, residual)
        F_train += learning_rate * tree.predict(X_train)
        F_test  += learning_rate * tree.predict(X_test)
        trees.append(tree)
    return F_test
 
pred_scratch = gbm_scratch(X_train.values, y_train.values, X_test.values, n_estimators=200)
print(f"\n[scratch GBM]  test RMSE = {mean_squared_error(y_test, pred_scratch, squared=False):.4f}")
 
# =====================================================
# 3. Learning-rate sensitivity (why boosting needs care)
# =====================================================
print("\n=== Learning-rate sweep (XGBoost) ===")
for lr in [0.01, 0.05, 0.1, 0.3, 1.0]:
    m = xgb.XGBRegressor(
        n_estimators=1500, learning_rate=lr, max_depth=6,
        subsample=0.8, colsample_bytree=0.8,
        random_state=42, n_jobs=-1,
        early_stopping_rounds=50, eval_metric="rmse",
    )
    m.fit(X_train, y_train, eval_set=[(X_val, y_val)], verbose=False)
    rmse = mean_squared_error(y_test, m.predict(X_test), squared=False)
    print(f"  lr={lr:>5}: rounds={m.best_iteration:>4}, test RMSE={rmse:.4f}")
 
# =====================================================
# 4. Feature importance (gain-based, XGBoost specific)
# =====================================================
print("\n=== Feature importance (gain) ===")
importances = model.get_booster().get_score(importance_type="gain")
for feat, imp in sorted(importances.items(), key=lambda x: -x[1])[:8]:
    print(f"  {feat:<20} {imp:>10.2f}")
 
# =====================================================
# 5. Uncomment for LightGBM comparison
# =====================================================
# lgb_model = lgb.LGBMRegressor(
#     n_estimators=2000, learning_rate=0.05, num_leaves=31, max_depth=-1,
#     subsample=0.8, colsample_bytree=0.8, min_child_samples=10,
#     random_state=42, n_jobs=-1,
# )
# lgb_model.fit(X_train, y_train,
#               eval_set=[(X_val, y_val)],
#               callbacks=[lgb.early_stopping(stopping_rounds=50)])
# rmse = mean_squared_error(y_test, lgb_model.predict(X_test), squared=False)
# print(f"[LightGBM]     test RMSE = {rmse:.4f}")