Day 43: Random Forest & Gradient Boosting

import numpy as np
rng = np.random.default_rng(0)
true_price = 60.0
one_expert = rng.normal(true_price, 12, 10000)                      # one noisy guess
crowd = rng.normal(true_price, 12, (10000, 50)).mean(axis=1)        # average of 50 independent guesses
print(f"one model  : typical error ±{np.std(one_expert):.1f} lakh")
print(f"50 averaged: typical error ±{np.std(crowd):.1f} lakh")

import pandas as pd
from sklearn.model_selection import train_test_split
from sklearn.tree import DecisionTreeRegressor
from sklearn.ensemble import RandomForestRegressor
from sklearn.metrics import mean_absolute_error
h = pd.read_csv("house_prices.csv")
X = pd.get_dummies(h.drop(columns="price_lakh"), drop_first=True, dtype=int); y = h["price_lakh"]
X_tr, X_te, y_tr, y_te = train_test_split(X, y, test_size=0.25, random_state=42)
tree = DecisionTreeRegressor(random_state=0).fit(X_tr, y_tr)
rf = RandomForestRegressor(n_estimators=300, oob_score=True, random_state=0, n_jobs=-1).fit(X_tr, y_tr)
print(f"single tree   MAE ₹{mean_absolute_error(y_te, tree.predict(X_te)):.2f} lakh")
print(f"random forest MAE ₹{mean_absolute_error(y_te, rf.predict(X_te)):.2f} lakh | OOB R² {rf.oob_score_:.3f}")

import pandas as pd
import matplotlib.pyplot as plt
from sklearn.model_selection import train_test_split
from sklearn.ensemble import RandomForestRegressor
from sklearn.metrics import mean_absolute_error
h = pd.read_csv("house_prices.csv")
X = pd.get_dummies(h.drop(columns="price_lakh"), drop_first=True, dtype=int); y = h["price_lakh"]
X_tr, X_te, y_tr, y_te = train_test_split(X, y, test_size=0.25, random_state=42)
ns = [1, 3, 10, 30, 100, 300]
maes = [mean_absolute_error(y_te, RandomForestRegressor(n_estimators=n, random_state=0, n_jobs=-1).fit(X_tr, y_tr).predict(X_te)) for n in ns]
plt.figure(figsize=(8, 3.6)); plt.plot(ns, maes, marker="o", color="#16a34a"); plt.xscale("log")
plt.xlabel("number of trees"); plt.ylabel("test MAE (₹ lakh)"); plt.title("More trees → lower error, then it levels off")
plt.show()

import numpy as np
import pandas as pd
from sklearn.tree import DecisionTreeRegressor
h = pd.read_csv("house_prices.csv"); p = h[h["city"] == "Pune"]
X, y = p[["area_sqft", "metro_km", "age_years"]], p["price_lakh"].to_numpy()
pred = np.full(len(y), y.mean())                       # start with the average
lr = 0.3
for step in range(1, 6):
    resid = y - pred                                   # what we still get wrong
    t = DecisionTreeRegressor(max_depth=2, random_state=0).fit(X, resid)
    pred = pred + lr * t.predict(X)                    # take a small step
    print(f"step {step}: mean absolute error = {np.abs(y - pred).mean():.2f} lakh")

import pandas as pd
from sklearn.model_selection import train_test_split
from sklearn.ensemble import RandomForestRegressor, HistGradientBoostingRegressor
from sklearn.linear_model import LinearRegression
from sklearn.metrics import mean_absolute_error
h = pd.read_csv("house_prices.csv")
X = pd.get_dummies(h.drop(columns="price_lakh"), drop_first=True, dtype=int); y = h["price_lakh"]
X_tr, X_te, y_tr, y_te = train_test_split(X, y, test_size=0.25, random_state=42)
models = {"Linear regression": LinearRegression(),
          "Random forest": RandomForestRegressor(n_estimators=300, random_state=0, n_jobs=-1),
          "Hist gradient boosting": HistGradientBoostingRegressor(learning_rate=0.05, max_iter=600, early_stopping=True, random_state=0)}
for name, m in models.items():
    m.fit(X_tr, y_tr)
    print(f"{name:24s} MAE ₹{mean_absolute_error(y_te, m.predict(X_te)):.2f} lakh")
print("boosting stopped after", models["Hist gradient boosting"].n_iter_, "trees (early stopping)")

import pandas as pd
from sklearn.model_selection import train_test_split
from sklearn.linear_model import LogisticRegression
from sklearn.tree import DecisionTreeClassifier
from sklearn.ensemble import RandomForestClassifier, HistGradientBoostingClassifier
from sklearn.metrics import roc_auc_score, accuracy_score
l = pd.read_csv("loan_applications.csv")
l["lti"] = l["loan_amount"] / (l["monthly_income"] * 12); l["salaried"] = (l["employment"] == "Salaried").astype(int)
X = l[["monthly_income", "credit_score", "loan_amount", "years_employed", "salaried"]]   # no lti – let models find it
y = l["approved"]
X_tr, X_te, y_tr, y_te = train_test_split(X, y, test_size=0.25, random_state=42, stratify=y)
models = {"Logistic": LogisticRegression(max_iter=5000), "Tree": DecisionTreeClassifier(max_depth=5, random_state=0),
          "Random forest": RandomForestClassifier(n_estimators=400, min_samples_leaf=3, random_state=0, n_jobs=-1),
          "Gradient boosting": HistGradientBoostingClassifier(learning_rate=0.05, max_iter=300, random_state=0)}
for name, m in models.items():
    p = m.fit(X_tr, y_tr).predict_proba(X_te)[:, 1]
    print(f"{name:18s} accuracy={accuracy_score(y_te, p > 0.5):.3f}  AUC={roc_auc_score(y_te, p):.3f}")

import pandas as pd
import matplotlib.pyplot as plt
from sklearn.model_selection import train_test_split
from sklearn.ensemble import RandomForestClassifier
from sklearn.inspection import permutation_importance
l = pd.read_csv("loan_applications.csv"); l["salaried"] = (l["employment"] == "Salaried").astype(int)
X = l[["monthly_income", "credit_score", "loan_amount", "years_employed", "salaried"]]; y = l["approved"]
X_tr, X_te, y_tr, y_te = train_test_split(X, y, test_size=0.25, random_state=42, stratify=y)
rf = RandomForestClassifier(n_estimators=400, min_samples_leaf=3, random_state=0, n_jobs=-1).fit(X_tr, y_tr)
pi = permutation_importance(rf, X_te, y_te, n_repeats=20, random_state=0, scoring="roc_auc")
imp = pd.Series(pi.importances_mean, index=X.columns).sort_values()
imp.plot(kind="barh", figsize=(7.5, 3.4), color="#4f46e5")
plt.title("Permutation importance: AUC drop when a feature is shuffled"); plt.xlabel("AUC drop"); plt.show()
print(imp.round(3).sort_values(ascending=False).to_string())