📊 What is Model Evaluation in Machine Learning?
Model Evaluation is the systematic process of assessing a trained machine learning model's generalization performance on unseen test data. Simply measuring overall training accuracy leads to dangerous overfitting and failure in production. Reliable model evaluation utilizes specialized scorecards across distinct domains: Regression Metrics (MAE, RMSE, $R^2$), Classification Diagnostics (Confusion Matrix, Precision, Recall, F1, ROC-AUC), Cross-Validation (K-Fold, Stratified, Time-Series), and Validation Curves to balance bias and variance before production deployment.
import numpy as np
from sklearn.metrics import classification_report, confusion_matrix
# Ground truth test defect severity: 0 = Low, 1 = Medium, 2 = Critical Bug
y_actual = np.array([0, 0, 1, 1, 2, 2, 0, 1, 2, 2, 1, 0])
# Classifier automated triage predictions
y_pred = np.array([0, 0, 1, 2, 2, 2, 0, 1, 1, 2, 1, 0])
class_names = ["Low", "Medium", "Critical"]
# 1. Confusion Matrix
matrix = confusion_matrix(y_actual, y_pred)
print("Confusion Matrix:\n", matrix)
# 2. Comprehensive Classification Report
report = classification_report(y_actual, y_pred, target_names=class_names, digits=2)
print("\nClassification Scorecard:\n", report)
import numpy as np
from sklearn.metrics import roc_auc_score, roc_curve
# Binary Ground Truth: 1 = Payment Fraud, 0 = Legitimate Customer
y_true = np.array([0, 0, 0, 0, 1, 0, 1, 1, 0, 1])
# Model output predicted fraud probabilities
y_probabilities = np.array([0.05, 0.12, 0.22, 0.35, 0.78, 0.18, 0.89, 0.95, 0.40, 0.82])
auc_val = roc_auc_score(y_true, y_probabilities)
fpr, tpr, thresholds = roc_curve(y_true, y_probabilities)
print(f"Area Under the ROC Curve (ROC-AUC): {auc_val:.4f}")
print("\nSample Operating Threshold Coordinates:")
for f, t, th in zip(fpr[1:4], tpr[1:4], thresholds[1:4]):
print(f" Threshold >= {th:.2f} -> False Positive Rate: {f:.2f}, True Positive Rate (Recall): {t:.2f}")
import numpy as np
from sklearn.metrics import average_precision_score, precision_recall_curve
# Severe Class Imbalance: 90% Passing Tests (0), 10% Flaky Failures (1)
y_actual_imbalanced = np.array([0, 0, 0, 0, 0, 0, 0, 0, 1, 1])
y_predicted_prob = np.array([0.08, 0.12, 0.05, 0.19, 0.22, 0.15, 0.35, 0.02, 0.75, 0.88])
pr_auc = average_precision_score(y_actual_imbalanced, y_predicted_prob)
precision_vals, recall_vals, thresholds = precision_recall_curve(y_actual_imbalanced, y_predicted_prob)
print(f"Precision-Recall AUC (PR-AUC): {pr_auc:.4f}")
print("Optimal Operating Points for Flaky Detector:")
for p, r, th in zip(precision_vals[:-1], recall_vals[:-1], thresholds):
if r >= 0.5:
print(f" Confidence Cutoff >= {th:.2f} -> Precision: {p:.2f}, Recall: {r:.2f}")
import numpy as np
from sklearn.metrics import (
mean_absolute_error,
mean_squared_error,
r2_score,
mean_absolute_percentage_error
)
# Ground truth cloud cluster bills in USD
y_actual_cost = np.array([240.0, 480.0, 150.0, 890.0, 620.0, 1100.0])
# Model predicted bills
y_pred_cost = np.array([232.0, 495.0, 168.0, 860.0, 640.0, 1070.0])
mae = mean_absolute_error(y_actual_cost, y_pred_cost)
mse = mean_squared_error(y_actual_cost, y_pred_cost)
rmse = np.sqrt(mse)
mape = mean_absolute_percentage_error(y_actual_cost, y_pred_cost)
r2 = r2_score(y_actual_cost, y_pred_cost)
print("Cloud Cost Forecast Scorecard:")
print(f" MAE : ${mae:.2f} (Average dollar error magnitude)")
print(f" RMSE : ${rmse:.2f} (Square-root penalizing larger outliers)")
print(f" MAPE : {mape * 100:.2f}% (Average percentage error across bills)")
print(f" R² : {r2:.4f} (Model explains {r2*100:.2f}% of billing variance)")
import numpy as np
from sklearn.model_selection import StratifiedKFold, cross_val_score
from sklearn.linear_model import LogisticRegression
# Features: [latency_ms, error_rate_percent]
X_service = np.array([
[45, 0.2], [55, 0.3], [120, 2.5], [140, 3.2],
[42, 0.1], [50, 0.4], [130, 2.8], [145, 3.5],
[48, 0.2], [125, 2.7]
])
# Binary health: 0 = Healthy Node, 1 = Degrading Node
y_health = np.array([0, 0, 1, 1, 0, 0, 1, 1, 0, 1])
skf = StratifiedKFold(n_splits=3, shuffle=True, random_state=42)
clf = LogisticRegression()
# Compute F1-score across all 3 stratified folds
fold_f1_scores = cross_val_score(clf, X_service, y_health, cv=skf, scoring='f1')
print("F1-Scores per Fold:", np.round(fold_f1_scores, 4))
print(f"Mean Generalization F1: {np.mean(fold_f1_scores):.4f} (± {np.std(fold_f1_scores):.4f})")
TimeSeriesSplit enforces strict forward-chaining temporal boundaries on sequential data.
import numpy as np
from sklearn.model_selection import TimeSeriesSplit
from sklearn.linear_model import Ridge
from sklearn.metrics import mean_absolute_error
# Sequential weekly sales units: 8 consecutive chronological weeks
time_series_features = np.array([[1], [2], [3], [4], [5], [6], [7], [8]])
weekly_orders = np.array([120, 135, 148, 162, 178, 195, 210, 230])
tscv = TimeSeriesSplit(n_splits=3)
model = Ridge(alpha=1.0)
print("Forward-Chaining Cross-Validation Folds:")
for fold_idx, (train_idx, test_idx) in enumerate(tscv.split(time_series_features)):
X_train, X_test = time_series_features[train_idx], time_series_features[test_idx]
y_train, y_test = weekly_orders[train_idx], weekly_orders[test_idx]
model.fit(X_train, y_train)
y_pred = model.predict(X_test)
error = mean_absolute_error(y_test, y_pred)
print(f" Fold {fold_idx + 1}: Train Weeks {list(X_train.flatten())} -> Test Weeks {list(X_test.flatten())} | MAE: {error:.2f}")
import numpy as np
from sklearn.model_selection import learning_curve
from sklearn.linear_model import Ridge
from sklearn.datasets import make_regression
X, y = make_regression(n_samples=100, n_features=5, noise=5.0, random_state=42)
train_sizes, train_scores, test_scores = learning_curve(
estimator=Ridge(alpha=1.0),
X=X,
y=y,
train_sizes=np.array([0.2, 0.5, 0.8, 1.0]),
cv=5,
scoring='r2',
random_state=42
)
train_mean = np.mean(train_scores, axis=1)
test_mean = np.mean(test_scores, axis=1)
print("Learning Curve Dynamics Across Training Set Sizes:")
for size, t_score, v_score in zip(train_sizes, train_mean, test_mean):
gap = t_score - v_score
print(f" Size: {size:2d} samples -> Train R²: {t_score:.3f} | Val R²: {v_score:.3f} | Gap: {gap:.3f}")
print("Convergence check: Small gap indicates healthy generalization without severe overfitting.")
from sklearn.model_selection import GridSearchCV
from sklearn.svm import SVC
import numpy as np
# Synthetic tabular feature space
X_train = np.array([[1, 2], [2, 3], [3, 1], [8, 9], [9, 8], [8, 7]])
y_train = np.array([0, 0, 0, 1, 1, 1])
param_grid = {
'C': [0.1, 1.0, 10.0],
'kernel': ['linear', 'rbf']
}
grid_search = GridSearchCV(
estimator=SVC(),
param_grid=param_grid,
cv=3,
scoring='accuracy',
return_train_score=False
)
grid_search.fit(X_train, y_train)
print(f"Optimal Hyperparameter Profile: {grid_search.best_params_}")
print(f"Top Cross-Validated Accuracy: {grid_search.best_score_ * 100:.1f}%")
import numpy as np
from sklearn.metrics import brier_score_loss
# Ground truth outcome: 1 = Candidate Converted, 0 = Candidate Left
y_true_outcomes = np.array([1, 1, 0, 0, 1, 0, 1, 0])
# Well-calibrated model predicted probabilities
well_calibrated_probs = np.array([0.90, 0.85, 0.10, 0.15, 0.88, 0.05, 0.92, 0.20])
# Overconfident/poorly calibrated model predictions
poorly_calibrated_probs = np.array([0.60, 0.55, 0.45, 0.48, 0.58, 0.42, 0.62, 0.49])
brier_good = brier_score_loss(y_true_outcomes, well_calibrated_probs)
brier_poor = brier_score_loss(y_true_outcomes, poorly_calibrated_probs)
print(f"Well-Calibrated Model Brier Score : {brier_good:.4f} (Lower is better; close to 0)")
print(f"Poorly-Calibrated Model Brier Score: {brier_poor:.4f} (High error indicates uncertainty)")
import numpy as np
def calculate_psi(expected: np.ndarray, actual: np.ndarray, num_buckets: int = 4) -> float:
"""Computes Population Stability Index (PSI) across binned feature distributions."""
# Define quantile bucket cutoffs based on expected baseline
percentiles = np.linspace(0, 100, num_buckets + 1)
bucket_edges = np.percentile(expected, percentiles)
# Add epsilon to prevent divide-by-zero
eps = 1e-4
expected_counts = np.histogram(expected, bins=bucket_edges)[0] + eps
actual_counts = np.histogram(actual, bins=bucket_edges)[0] + eps
expected_pct = expected_counts / np.sum(expected_counts)
actual_pct = actual_counts / np.sum(actual_counts)
psi_value = np.sum((actual_pct - expected_pct) * np.log(actual_pct / expected_pct))
return float(psi_value)
# Baseline Training Distribution: Standard checkout cart values
baseline_distribution = np.array([250, 450, 600, 850, 1100, 1300, 1500, 1800, 2200, 2500])
# Live Production Stream: Unchanged standard usage
live_normal_traffic = np.array([270, 480, 580, 890, 1050, 1320, 1490, 1850, 2150, 2480])
# Live Production Stream: High-value wholesale drift (Black Friday shift)
live_drifted_traffic = np.array([3200, 4500, 5100, 6200, 7800, 8100, 9500, 11000, 12500, 14000])
psi_normal = calculate_psi(baseline_distribution, live_normal_traffic)
psi_drifted = calculate_psi(baseline_distribution, live_drifted_traffic)
print(f"Normal Stream PSI : {psi_normal:.4f} (PSI < 0.1 -> Stable; No retraining needed)")
print(f"Drifted Stream PSI: {psi_drifted:.4f} (PSI > 0.25 -> Severe drift; Trigger pipeline retraining)")