π² Wisdom of Crowds: Combining Decision Trees to Reduce Variance
Single decision trees are prone to high variance and overfittingβsmall changes in training data can result in completely different tree structures. Bagging (Bootstrap Aggregating) combats this by training multiple base models in parallel on bootstrapped subsets of data and averaging (or voting on) their predictions. Random Forests take bagging a step further by introducing random feature selection at every tree split, decorrelating the trees and producing an exceptionally robust ensemble. This guide covers 10 real-world engineering patterns for Bagging and Random Forests.
import numpy as np
def generate_bootstrap_sample(X, y):
n_samples = len(X)
# Sample indices with replacement
indices = np.random.choice(n_samples, size=n_samples, replace=True)
return X[indices], y[indices], indices
def run_bootstrap():
print("=" * 65)
print("π EXAMPLE 1: BOOTSTRAP SAMPLING WITH REPLACEMENT")
print("=" * 65)
np.random.seed(42)
X = np.arange(10).reshape(-1, 1)
y = np.array([0, 0, 0, 0, 0, 1, 1, 1, 1, 1])
X_boot, y_boot, idx = generate_bootstrap_sample(X, y)
unique_ratio = len(np.unique(idx)) / len(X)
print(f"Original Dataset Size : {len(X)}")
print(f"Bootstrap Indices : {idx}")
print(f"Unique Samples Ratio : {unique_ratio * 100:.1f}% (Expected ~63.2%)")
print("=" * 65)
if __name__ == "__main__":
run_bootstrap()
import numpy as np
from scipy.stats import mode
def aggregate_predictions():
print("=" * 65)
print("π€ EXAMPLE 2: ENSEMBLE PREDICTION AGGREGATION")
print("=" * 65)
# Simulated predictions from 5 independent decision trees for a single test sample
classification_preds = np.array([1, 1, 0, 1, 1])
regression_preds = np.array([120.5, 125.0, 118.2, 122.1, 121.0])
# Classification: Majority Vote
majority_vote = mode(classification_preds, keepdims=True).mode[0]
# Regression: Arithmetic Average
regression_avg = np.mean(regression_preds)
print(f"Classification Tree Predictions : {classification_preds.tolist()}")
print(f"Aggregated Classification Result: {majority_vote} (Spam/Positive)")
print(f"Regression Tree Predictions : {regression_preds.tolist()}")
print(f"Aggregated Regression Result : ${regression_avg:.2f}k")
print("=" * 65)
if __name__ == "__main__":
aggregate_predictions()
import numpy as np
from sklearn.ensemble import RandomForestClassifier
def run_rf_decorrelation():
print("=" * 65)
print("π² EXAMPLE 3: RANDOM FOREST FEATURE RANDOMNESS")
print("=" * 65)
np.random.seed(42)
X = np.random.randn(200, 10) # 10 features
y = np.random.randint(0, 2, 200)
# max_features='sqrt' means at each split, sqrt(10) β 3 features are considered
rf = RandomForestClassifier(n_estimators=50, max_features='sqrt', random_state=42)
rf.fit(X, y)
print(f"Number of Trees in Forest : {len(rf.estimators_)}")
print(f"Max Features per Split : {rf.max_features_}")
print("Explanation: Sub-sampling features decorrelates trees, strengthening ensemble robustness.")
print("=" * 65)
if __name__ == "__main__":
run_rf_decorrelation()
import numpy as np
from sklearn.ensemble import RandomForestClassifier
def run_oob_demo():
print("=" * 65)
print("π EXAMPLE 4: OUT-OF-BAG (OOB) ERROR ESTIMATION")
print("=" * 65)
np.random.seed(42)
X = np.random.randn(150, 4)
y = np.random.randint(0, 2, 150)
# oob_score=True enables internal OOB evaluation
rf = RandomForestClassifier(n_estimators=100, oob_score=True, random_state=42)
rf.fit(X, y)
print(f"OOB Score (Accuracy) : {rf.oob_score_ * 100:.2f}%")
print("Explanation: OOB score provides an unbiased estimate of generalization error without cross-validation.")
print("=" * 65)
if __name__ == "__main__":
run_oob_demo()
import numpy as np
from sklearn.ensemble import RandomForestClassifier
def run_rf_importance():
print("=" * 65)
print("π EXAMPLE 5: RANDOM FOREST FEATURE IMPORTANCE RANKING")
print("=" * 65)
np.random.seed(42)
X = np.random.randn(200, 4)
# Target heavily depends on feature 2 and feature 0
y = (X[:, 2] + 1.5 * X[:, 0] > 0).astype(int)
rf = RandomForestClassifier(n_estimators=100, random_state=42)
rf.fit(X, y)
features = ['Age', 'Income', 'CreditScore', 'DebtRatio']
importances = rf.feature_importances_
sorted_idx = np.argsort(importances)[::-1]
print("Feature Importance Ranking:")
for idx in sorted_idx:
print(f" {features[idx]:11s} : {importances[idx]:.4f}")
print("=" * 65)
if __name__ == "__main__":
run_rf_importance()
import numpy as np
import pandas as pd
from sklearn.ensemble import RandomForestClassifier
def run_rf_missing():
print("=" * 65)
print("π§© EXAMPLE 6: RANDOM FOREST ROBUSTNESS TO MISSING DATA")
print("=" * 65)
df = pd.DataFrame({
'feature_1': [2.5, 3.1, np.nan, 4.2, 5.0],
'feature_2': [100, np.nan, 300, 400, 500],
'target': [0, 0, 1, 1, 1]
})
# Note: Scikit-learn's RF requires imputation for NaN, but tree logic inherently isolates missing branches.
print(f"Dataset with missing values:\n{df}")
print("Explanation: Tree algorithms split missing values down specialized paths or handle via median imputation pipelines.")
print("=" * 65)
if __name__ == "__main__":
run_rf_missing()
n_jobs=-1 utilizes all available CPU cores, dramatically accelerating execution.
import time
import numpy as np
from sklearn.ensemble import RandomForestClassifier
def benchmark_parallel_rf():
print("=" * 65)
print("β‘ EXAMPLE 7: PARALLEL PROCESSING BENCHMARK (n_jobs)")
print("=" * 65)
np.random.seed(42)
X = np.random.randn(2000, 20)
y = np.random.randint(0, 2, 2000)
# 1. Single Core Training (n_jobs=1)
start = time.time()
rf_single = RandomForestClassifier(n_estimators=100, n_jobs=1, random_state=42)
rf_single.fit(X, y)
time_single = time.time() - start
# 2. Multi-Core Training (n_jobs=-1)
start = time.time()
rf_multi = RandomForestClassifier(n_estimators=100, n_jobs=-1, random_state=42)
rf_multi.fit(X, y)
time_multi = time.time() - start
print(f"Single-Core Training Time (n_jobs=1) : {time_single:.4f}s")
print(f"Multi-Core Training Time (n_jobs=-1) : {time_multi:.4f}s")
print(f"Speedup Factor : {time_single / time_multi:.1f}x faster")
print("=" * 65)
if __name__ == "__main__":
benchmark_parallel_rf()
RandomForestRegressor), where final predictions are calculated as the arithmetic average of continuous outputs across all trees.
import numpy as np
from sklearn.ensemble import RandomForestRegressor
from sklearn.metrics import mean_squared_error
def run_rf_regressor():
print("=" * 65)
print("π EXAMPLE 8: RANDOM FOREST REGRESSOR FOR HOUSE PRICES")
print("=" * 65)
np.random.seed(42)
X = np.random.randn(200, 3) # [SqFt, Bedrooms, Age]
y = X[:, 0] * 50.0 + X[:, 1] * 20.0 - X[:, 2] * 5.0 + 300.0 + np.random.normal(0, 5, 200)
rf_reg = RandomForestRegressor(n_estimators=100, random_state=42)
rf_reg.fit(X[:150], y[:150])
predictions = rf_reg.predict(X[150:])
mse = mean_squared_error(y[150:], predictions)
print(f"Test Set MSE : {mse:.2f}")
print(f"Sample Prediction vs True: Pred = ${predictions[0]:.1f}k | True = ${y[150]:.1f}k")
print("=" * 65)
if __name__ == "__main__":
run_rf_regressor()
import numpy as np
from sklearn.ensemble import RandomForestClassifier
from sklearn.model_selection import GridSearchCV
def run_rf_gridsearch():
print("=" * 65)
print("π€ EXAMPLE 9: HYPERPARAMETER TUNING FOR RANDOM FOREST")
print("=" * 65)
np.random.seed(42)
X = np.random.randn(150, 4)
y = np.random.randint(0, 2, 150)
param_grid = {
'n_estimators': [50, 100],
'max_depth': [3, 6],
'min_samples_split': [2, 5]
}
grid = GridSearchCV(
RandomForestClassifier(random_state=42, n_jobs=-1),
param_grid,
cv=3,
scoring='accuracy'
)
grid.fit(X, y)
print(f"Best Hyperparameters Found : {grid.best_params_}")
print(f"Best Cross-Validation Acc : {grid.best_score_ * 100:.2f}%")
print("=" * 65)
if __name__ == "__main__":
run_rf_gridsearch()
BAGGING & RANDOM FORESTS REFERENCE MATRIX:
ββββββββββββββββββββββββββββββββ¬βββββββββββββββββββββββββββββββ¬βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
β Concept / Component β Operational Role β Primary Machine Learning Application β
ββββββββββββββββββββββββββββββββΌβββββββββββββββββββββββββββββββΌβββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ€
β Bootstrap Sampling β Resampling with replacement β Creating diverse training subsets for base estimators β
β Ensemble Aggregation β Majority vote or averaging β Combining predictions to reduce overall variance β
β Feature Randomness β Random subset per split β Decorrelating trees in Random Forests (max_features) β
β Out-of-Bag (OOB) Error β Evaluation on left-out ~36.8%β Unbiased error estimation without cross-validation β
β Feature Importance β Total Gini/Entropy reduction β Ranking predictive features across ensemble trees β
β Parallel Processing (n_jobs) β Multi-core execution β Accelerating training across independent tree estimators β
β Random Forest Regressor β Averaging continuous outputs β Predicting numerical targets (prices, sales, demand) β
β Hyperparameter Tuning β GridSearchCV / Randomized β Optimizing n_estimators, max_depth, and min_samples_splitβ
ββββββββββββββββββββββββββββββββ΄βββββββββββββββββββββββββββββββ΄βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ€
β ENGINEERING BEST PRACTICES: β
β 1. Always set `n_jobs=-1` when training Random Forests to leverage multi-core CPU parallelism. β
β 2. Use `oob_score=True` for quick validation checks without setting aside a separate validation dataset. β
β 3. Tune `max_features` ($\sqrt{p}$ for classification, $p/3$ for regression) to ensure proper tree decorrelation. β
ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ