π Sequential Learning: Turning Weak Learners into High-Accuracy Ensembles
While Bagging (Random Forests) trains multiple models independently in parallel, Boosting trains models sequentially. Each new model is specifically trained to correct the errors (residuals) made by the previous sequence of models. AdaBoost achieves this by dynamically re-weighting misclassified samples, while Gradient Boosting fits successive trees to the gradient residuals of the loss function. This guide covers 10 real-world engineering patterns for boosting algorithms.
import numpy as np
def simulate_adaboost_weights():
print("=" * 65)
print("βοΈ EXAMPLE 1: ADABOOST SAMPLE WEIGHT RE-WEIGHTING")
print("=" * 65)
n_samples = 5
# Initial uniform weights
weights = np.ones(n_samples) / n_samples
y_true = np.array([1, 1, -1, -1, 1])
y_pred = np.array([1, -1, -1, -1, 1]) # Sample at index 1 is misclassified
# Identify incorrect predictions
incorrect = (y_true != y_pred)
# Calculate weighted error rate
error_rate = np.sum(weights[incorrect]) / np.sum(weights)
# Calculate learner weight alpha
alpha = 0.5 * np.log((1.0 - error_rate) / (error_rate + 1e-10))
# Update weights: increase weight for misclassified, decrease for correct
weights = weights * np.exp(-alpha * y_true * y_pred)
weights /= np.sum(weights) # Normalize
print(f"Initial Weights : {[round(w, 4) for w in np.ones(5)/5]}")
print(f"Weighted Error : {error_rate:.4f}")
print(f"Learner Alpha : {alpha:.4f}")
print(f"Updated Weights : {[round(w, 4) for w in weights]}")
print("=" * 65)
if __name__ == "__main__":
simulate_adaboost_weights()
import numpy as np
def simulate_gbm_residuals():
print("=" * 65)
print("π EXAMPLE 2: GRADIENT BOOSTING RESIDUAL FITTING")
print("=" * 65)
X = np.array([1.0, 2.0, 3.0, 4.0, 5.0])
y = np.array([10.0, 20.0, 15.0, 30.0, 35.0])
# Step 1: Initial prediction (mean of y)
y_pred = np.full_like(y, np.mean(y))
# Step 2: Calculate residuals (negative gradient for MSE loss: y - y_pred)
residuals = y - y_pred
print(f"Target Values (y) : {y}")
print(f"Initial Prediction : {y_pred}")
print(f"Calculated Residuals : {residuals}")
print("Explanation: The next tree in the boosting sequence will be trained to predict these residuals.")
print("=" * 65)
if __name__ == "__main__":
simulate_gbm_residuals()
import numpy as np
def simulate_shrinkage():
print("=" * 65)
print("β‘ EXAMPLE 3: LEARNING RATE SHRINKAGE IN GBM")
print("=" * 65)
initial_pred = 10.0
residual_prediction_tree1 = 4.0
learning_rate = 0.1
# Updated prediction with shrinkage
updated_pred = initial_pred + (learning_rate * residual_prediction_tree1)
print(f"Initial Prediction : {initial_pred}")
print(f"Tree 1 Residual Prediction: {residual_prediction_tree1}")
print(f"Learning Rate (eta) : {learning_rate}")
print(f"Updated Ensemble Output : {updated_pred}")
print("=" * 65)
if __name__ == "__main__":
simulate_shrinkage()
import numpy as np
def run_loss_functions():
print("=" * 65)
print("π― EXAMPLE 4: LOSS FUNCTIONS IN GRADIENT BOOSTING")
print("=" * 65)
# Regression Loss: MSE Gradient = -(y - y_pred)
y_reg, pred_reg = 50.0, 45.0
grad_mse = -(y_reg - pred_reg)
# Classification Loss: Log Loss Gradient = p - y
y_cls, prob_cls = 1.0, 0.8
grad_logloss = prob_cls - y_cls
print(f"Regression (MSE) Gradient : {grad_mse}")
print(f"Classification (Log Loss) Grad: {grad_logloss}")
print("=" * 65)
if __name__ == "__main__":
run_loss_functions()
GradientBoostingClassifier and GradientBoostingRegressor.
import numpy as np
from sklearn.ensemble import GradientBoostingClassifier
from sklearn.metrics import accuracy_score
def run_sklearn_gbm():
print("=" * 65)
print("π€ EXAMPLE 5: SCIKIT-LEARN GRADIENT BOOSTING CLASSIFIER")
print("=" * 65)
np.random.seed(42)
X_train = np.random.randn(150, 4)
y_train = np.random.randint(0, 2, 150)
X_test = np.random.randn(50, 4)
y_test = np.random.randint(0, 2, 50)
gbm = GradientBoostingClassifier(n_estimators=100, learning_rate=0.1, max_depth=3, random_state=42)
gbm.fit(X_train, y_train)
preds = gbm.predict(X_test)
print(f"GBM Test Accuracy : {accuracy_score(y_test, preds) * 100:.2f}%")
print("=" * 65)
if __name__ == "__main__":
run_sklearn_gbm()
AdaBoostClassifier trains a sequence of decision stumps (trees with max_depth=1) by default, iteratively adjusting sample weights.
import numpy as np
from sklearn.ensemble import AdaBoostClassifier
from sklearn.metrics import accuracy_score
def run_sklearn_adaboost():
print("=" * 65)
print("π‘οΈ EXAMPLE 6: SCIKIT-LEARN ADABOOST CLASSIFIER")
print("=" * 65)
np.random.seed(42)
X_train = np.random.randn(150, 4)
y_train = np.random.randint(0, 2, 150)
X_test = np.random.randn(50, 4)
y_test = np.random.randint(0, 2, 50)
ada = AdaBoostClassifier(n_estimators=50, learning_rate=1.0, random_state=42)
ada.fit(X_train, y_train)
preds = ada.predict(X_test)
print(f"AdaBoost Test Accuracy : {accuracy_score(y_test, preds) * 100:.2f}%")
print("=" * 65)
if __name__ == "__main__":
run_sklearn_adaboost()
import numpy as np
from sklearn.ensemble import GradientBoostingClassifier
def run_early_stopping():
print("=" * 65)
print("π EXAMPLE 7: EARLY STOPPING IN GRADIENT BOOSTING")
print("=" * 65)
np.random.seed(42)
X = np.random.randn(200, 4)
y = np.random.randint(0, 2, 200)
# n_iter_no_change halts training if validation score doesn't improve for 5 rounds
gbm = GradientBoostingClassifier(
n_estimators=500,
validation_fraction=0.2,
n_iter_no_change=5,
tol=1e-2,
random_state=42
)
gbm.fit(X, y)
print(f"Target Estimators : 500")
print(f"Actual Estimators Built : {gbm.n_estimators_} (Stopped early)")
print("=" * 65)
if __name__ == "__main__":
run_early_stopping()
import numpy as np
from sklearn.ensemble import GradientBoostingClassifier
def run_gbm_importance():
print("=" * 65)
print("π EXAMPLE 8: GBM FEATURE IMPORTANCE")
print("=" * 65)
np.random.seed(42)
X = np.random.randn(150, 4)
y = (X[:, 1] * 2.0 > 0).astype(int)
gbm = GradientBoostingClassifier(n_estimators=50, random_state=42)
gbm.fit(X, y)
features = ['Age', 'Income', 'CreditScore', 'DebtRatio']
importances = gbm.feature_importances_
for feat, imp in zip(features, importances):
print(f" {feat:11s} : {imp:.4f}")
print("=" * 65)
if __name__ == "__main__":
run_gbm_importance()
import numpy as np
from sklearn.ensemble import GradientBoostingClassifier
from sklearn.model_selection import GridSearchCV
def run_gbm_grid():
print("=" * 65)
print("π€ EXAMPLE 9: HYPERPARAMETER TUNING FOR GBM")
print("=" * 65)
np.random.seed(42)
X = np.random.randn(100, 4)
y = np.random.randint(0, 2, 100)
param_grid = {
'n_estimators': [50, 100],
'learning_rate': [0.05, 0.1],
'max_depth': [3, 4]
}
grid = GridSearchCV(
GradientBoostingClassifier(random_state=42),
param_grid,
cv=3,
scoring='accuracy'
)
grid.fit(X, y)
print(f"Best Parameters Found : {grid.best_params_}")
print(f"Best Validation Score : {grid.best_score_ * 100:.2f}%")
print("=" * 65)
if __name__ == "__main__":
run_gbm_grid()
GRADIENT BOOSTING & ADABOOST REFERENCE MATRIX:
ββββββββββββββββββββββββββββββββ¬βββββββββββββββββββββββββββββββ¬ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
β Algorithm / Concept β Core Mechanism β Primary Machine Learning Application β
ββββββββββββββββββββββββββββββββΌβββββββββββββββββββββββββββββββΌββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ€
β AdaBoost β Sequential sample re-weightingβ Adaptive classification via re-weighted training errors β
β Gradient Boosting (GBM) β Fitting trees to residuals β Sequential error correction via gradient descent loss β
β Learning Rate (Ξ·) β Shrinkage parameter (0.01-0.1)β Scaling tree contributions to prevent overfitting β
β Loss Functions β MSE (regression), Log Loss β Optimizing differentiable objective functions β
β Early Stopping β n_iter_no_change β Automatically halting training when validation loss stallsβ
β Feature Importance β Total impurity reduction β Ranking predictive features across sequential trees β
β Decision Stumps β Trees with max_depth=1 β Common weak learners used in AdaBoost β
β Hyperparameter Tuning β GridSearchCV / Randomized β Optimizing n_estimators, learning_rate, and max_depth β
ββββββββββββββββββββββββββββββββ΄βββββββββββββββββββββββββββββββ΄ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ€
β ENGINEERING BEST PRACTICES: β
β 1. Pair lower learning rates ($\eta = 0.01$ or $0.05$) with higher `n_estimators` for superior predictive accuracy. β
β 2. Use early stopping (`n_iter_no_change=10`) to prevent overfitting and save training computation. β
β 3. Keep tree depths shallow (`max_depth = 3` to `6`) for gradient boosting to maintain weak learner performance. β
βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ