🎯 Ensuring Generalization & Optimizing Model Performance
A machine learning model is only as good as its validation strategy. Training a model on data and evaluating it on the exact same dataset results in memorization rather than generalization (overfitting). Model Validation (such as K-Fold Cross-Validation) ensures unbiased performance estimates, while Hyperparameter Tuning systematically explores architectural configurations (e.g., learning rates, tree depths, regularization strengths) to maximize predictive accuracy. This guide covers 10 real-world engineering patterns for validation and tuning.
import numpy as np
from sklearn.model_selection import train_test_split
def run_stratified_split():
print("=" * 65)
print("✂️ EXAMPLE 1: STRATIFIED TRAIN-TEST SPLIT")
print("=" * 65)
np.random.seed(42)
X = np.random.randn(100, 3)
# Imbalanced target: 90 negative (0), 10 positive (1)
y = np.array([0]*90 + [1]*10)
X_train, X_test, y_train, y_test = train_test_split(
X, y, test_size=0.2, stratify=y, random_state=42
)
print(f"Original Positive Class Ratio : {np.mean(y) * 100:.1f}%")
print(f"Training Positive Class Ratio : {np.mean(y_train) * 100:.1f}%")
print(f"Testing Positive Class Ratio : {np.mean(y_test) * 100:.1f}%")
print("=" * 65)
if __name__ == "__main__":
run_stratified_split()
import numpy as np
from sklearn.model_selection import KFold, cross_val_score
from sklearn.ensemble import RandomForestClassifier
def run_kfold():
print("=" * 65)
print("🔄 EXAMPLE 2: K-FOLD CROSS-VALIDATION")
print("=" * 65)
np.random.seed(42)
X = np.random.randn(150, 4)
y = np.random.randint(0, 2, 150)
kf = KFold(n_splits=5, shuffle=True, random_state=42)
model = RandomForestClassifier(random_state=42)
scores = cross_val_score(model, X, y, cv=kf, scoring='accuracy')
print(f"Cross-Validation Accuracy per Fold: {np.round(scores, 4)}")
print(f"Mean CV Accuracy : {np.mean(scores) * 100:.2f}% (+/- {np.std(scores) * 100:.2f}%)")
print("=" * 65)
if __name__ == "__main__":
run_kfold()
import numpy as np
from sklearn.model_selection import StratifiedKFold, cross_val_score
from sklearn.linear_model import LogisticRegression
def run_stratified_kfold():
print("=" * 65)
print("⚖️ EXAMPLE 3: STRATIFIED K-FOLD CROSS-VALIDATION")
print("=" * 65)
np.random.seed(42)
X = np.random.randn(200, 5)
y = np.array([0]*160 + [1]*40) # 80-20 imbalance
skf = StratifiedKFold(n_splits=5, shuffle=True, random_state=42)
model = LogisticRegression()
scores = cross_val_score(model, X, y, cv=skf, scoring='f1')
print(f"Stratified F1-Scores per Fold: {np.round(scores, 4)}")
print(f"Mean Stratified F1-Score : {np.mean(scores):.4f}")
print("=" * 65)
if __name__ == "__main__":
run_stratified_kfold()
import numpy as np
from sklearn.model_selection import learning_curve
from sklearn.tree import DecisionTreeClassifier
def run_learning_curves():
print("=" * 65)
print("📈 EXAMPLE 4: LEARNING CURVES DIAGNOSTICS")
print("=" * 65)
np.random.seed(42)
X = np.random.randn(300, 4)
y = np.random.randint(0, 2, 300)
train_sizes, train_scores, val_scores = learning_curve(
DecisionTreeClassifier(max_depth=8, random_state=42),
X, y, cv=5, train_sizes=np.linspace(0.2, 1.0, 5), scoring='accuracy'
)
print(f"Training Sizes Evaluated : {train_sizes}")
print(f"Mean Training Accuracies : {np.round(np.mean(train_scores, axis=1), 4)}")
print(f"Mean Validation Accuracies: {np.round(np.mean(val_scores, axis=1), 4)}")
print("=" * 65)
if __name__ == "__main__":
run_learning_curves()
GridSearchCV exhaustively searches through a specified subset of hyperparameters, evaluating every combination using cross-validation to identify the optimal configuration.
import numpy as np
from sklearn.model_selection import GridSearchCV
from sklearn.ensemble import RandomForestClassifier
def run_grid_search():
print("=" * 65)
print("🔍 EXAMPLE 5: EXHAUSTIVE GRID SEARCH")
print("=" * 65)
np.random.seed(42)
X = np.random.randn(100, 4)
y = np.random.randint(0, 2, 100)
param_grid = {
'n_estimators': [10, 50],
'max_depth': [3, 5, None]
}
grid = GridSearchCV(
RandomForestClassifier(random_state=42),
param_grid,
cv=3,
scoring='accuracy'
)
grid.fit(X, y)
print(f"Best Hyperparameters Found : {grid.best_params_}")
print(f"Best Cross-Validation Score: {grid.best_score_ * 100:.2f}%")
print("=" * 65)
if __name__ == "__main__":
run_grid_search()
RandomizedSearchCV samples a fixed number of random parameter combinations, significantly speeding up optimization.
import numpy as np
from scipy.stats import randint
from sklearn.model_selection import RandomizedSearchCV
from sklearn.ensemble import RandomForestClassifier
def run_random_search():
print("=" * 65)
print("🎲 EXAMPLE 6: RANDOMIZED SEARCH HYPERPARAMETER TUNING")
print("=" * 65)
np.random.seed(42)
X = np.random.randn(150, 5)
y = np.random.randint(0, 2, 150)
param_dist = {
'n_estimators': randint(10, 200),
'max_depth': randint(2, 10),
'min_samples_split': randint(2, 10)
}
random_search = RandomizedSearchCV(
RandomForestClassifier(random_state=42),
param_distributions=param_dist,
n_iter=10,
cv=3,
random_state=42
)
random_search.fit(X, y)
print(f"Best Parameters Found : {random_search.best_params_}")
print(f"Best Validation Score : {random_search.best_score_ * 100:.2f}%")
print("=" * 65)
if __name__ == "__main__":
run_random_search()
import numpy as np
from sklearn.ensemble import RandomForestClassifier
from sklearn.model_selection import cross_val_score
def simulate_optuna_objective():
print("=" * 65)
print("🔮 EXAMPLE 7: BAYESIAN OPTIMIZATION SIMULATION (OPTUNA)")
print("=" * 65)
np.random.seed(42)
X = np.random.randn(100, 4)
y = np.random.randint(0, 2, 100)
# Simulated Optuna trial evaluation logic
def objective(trial_params):
model = RandomForestClassifier(
n_estimators=trial_params['n_estimators'],
max_depth=trial_params['max_depth'],
random_state=42
)
scores = cross_val_score(model, X, y, cv=3, scoring='accuracy')
return np.mean(scores)
# Simulated best trial found by Bayesian optimization
best_params = {'n_estimators': 85, 'max_depth': 4}
best_score = objective(best_params)
print(f"Optuna Best Parameters : {best_params}")
print(f"Optuna Best CV Score : {best_score * 100:.2f}%")
print("=" * 65)
if __name__ == "__main__":
simulate_optuna_objective()
import numpy as np
from sklearn.pipeline import Pipeline
from sklearn.preprocessing import StandardScaler
from sklearn.model_selection import cross_val_score
from sklearn.linear_model import LogisticRegression
def run_leakage_prevention():
print("=" * 65)
print("🛡️ EXAMPLE 8: PREVENTING DATA LEAKAGE VIA PIPELINES")
print("=" * 65)
np.random.seed(42)
X = np.random.randn(150, 4)
y = np.random.randint(0, 2, 150)
# Pipeline guarantees scaling is computed independently inside each CV fold
pipeline = Pipeline([
('scaler', StandardScaler()),
('classifier', LogisticRegression())
])
scores = cross_val_score(pipeline, X, y, cv=5, scoring='accuracy')
print(f"Pipeline CV Scores : {np.round(scores, 4)}")
print(f"Mean Leak-Free Accuracy: {np.mean(scores) * 100:.2f}%")
print("=" * 65)
if __name__ == "__main__":
run_leakage_prevention()
import numpy as np
from sklearn.model_selection import KFold, GridSearchCV, cross_val_score
from sklearn.ensemble import RandomForestClassifier
def run_nested_cv():
print("=" * 65)
print("🔄 EXAMPLE 9: NESTED CROSS-VALIDATION")
print("=" * 65)
np.random.seed(42)
X = np.random.randn(100, 3)
y = np.random.randint(0, 2, 100)
# Inner loop for hyperparameter tuning
inner_cv = KFold(n_splits=3, shuffle=True, random_state=42)
model = GridSearchCV(
RandomForestClassifier(random_state=42),
param_grid={'n_estimators': [10, 30]},
cv=inner_cv
)
# Outer loop for unbiased performance evaluation
outer_cv = KFold(n_splits=5, shuffle=True, random_state=42)
nested_scores = cross_val_score(model, X, y, cv=outer_cv, scoring='accuracy')
print(f"Nested CV Scores per Outer Fold : {np.round(nested_scores, 4)}")
print(f"Unbiased Nested CV Mean Accuracy: {np.mean(nested_scores) * 100:.2f}%")
print("=" * 65)
if __name__ == "__main__":
run_nested_cv()
MODEL VALIDATION & HYPERPARAMETER TUNING REFERENCE MATRIX:
┌──────────────────────────────┬──────────────────────────────┬──────────────────────────────────────────────────────────┐
│ Technique / Concept │ Scikit-Learn / Library │ Primary Machine Learning Application │
├──────────────────────────────┼──────────────────────────────┼──────────────────────────────────────────────────────────┤
│ Stratified Train-Test Split │ train_test_split(..., strat) │ Preserving class imbalance proportions in train/test sets│
│ K-Fold Cross-Validation │ KFold, cross_val_score │ Unbiased performance estimation across K folds │
│ Stratified K-Fold │ StratifiedKFold │ Ensuring class distribution parity per fold │
│ Learning Curves │ learning_curve │ Diagnosing Bias (underfitting) vs Variance (overfitting) │
│ Exhaustive Grid Search │ GridSearchCV │ Testing every hyperparameter combination in grid │
│ Randomized Search │ RandomizedSearchCV │ Randomly sampling hyperparameter spaces efficiently │
│ Bayesian Optimization │ Optuna / Hyperopt │ Intelligent surrogate-based hyperparameter tuning │
│ Pipeline Leakage Prevention │ Pipeline │ Restricting scaling/imputation strictly to training folds│
│ Nested Cross-Validation │ cross_val_score(GridSearchCV)│ Unbiased evaluation during hyperparameter tuning │
├──────────────────────────────┴──────────────────────────────┴──────────────────────────────────────────────────────────┤
│ ENGINEERING BEST PRACTICES: │
│ 1. Always use Pipelines (`sklearn.pipeline.Pipeline`) inside cross-validation to prevent accidental data leakage. │
│ 2. Prefer RandomizedSearchCV or Optuna over GridSearchCV when optimizing large hyperparameter spaces. │
│ 3. Employ Nested Cross-Validation when reporting final publication-grade model accuracy. │
└────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────┘