๐ท๏ธ The Fundamental Supervised Classification Algorithm
While linear regression predicts continuous numerical values, Logistic Regression is the premier supervised learning algorithm for predicting categorical outcomes (e.g., spam vs. not spam, loan default vs. repayment). By passing linear combinations of features through the non-linear Sigmoid function, logistic regression outputs bounded probabilities between 0 and 1. This guide covers 10 real-world engineering patterns for logistic regression and classification workflows.
import numpy as np
def sigmoid(z):
return 1 / (1 + np.exp(-z))
def run_sigmoid_demo():
print("=" * 65)
print("๐ EXAMPLE 1: SIGMOID ACTIVATION FUNCTION")
print("=" * 65)
z_values = np.array([-5.0, -1.0, 0.0, 1.0, 5.0])
probabilities = sigmoid(z_values)
for z, prob in zip(z_values, probabilities):
print(f"Log-odds (z) = {z:5.1f} -> Sigmoid Probability = {prob:.4f}")
print("=" * 65)
if __name__ == "__main__":
run_sigmoid_demo()
import numpy as np
def classify_probabilities(probabilities, threshold=0.5):
return (probabilities >= threshold).astype(int)
def run_threshold_demo():
print("=" * 65)
print("๐ฏ EXAMPLE 2: DECISION BOUNDARY THRESHOLDING")
print("=" * 65)
probs = np.array([0.12, 0.48, 0.51, 0.89, 0.33, 0.75])
preds = classify_probabilities(probs, threshold=0.5)
for p, pred in zip(probs, preds):
label = "Spam / Positive (1)" if pred == 1 else "Ham / Negative (0)"
print(f"Probability: {p:.2f} -> Predicted Class: {label}")
print("=" * 65)
if __name__ == "__main__":
run_threshold_demo()
import numpy as np
def interpret_coefficients():
print("=" * 65)
print("๐ EXAMPLE 3: LOG-ODDS & ODDS RATIO INTERPRETATION")
print("=" * 65)
# Feature: Study Hours -> Passing Exam (1)
# Suppose logistic regression weight w = 0.693 (approx ln(2))
weight = 0.693147
odds_ratio = np.exp(weight)
print(f"Model Coefficient (Log-Odds increase per hour): {weight:.4f}")
print(f"Odds Ratio (e^w) : {odds_ratio:.2f}x")
print("Interpretation: Every additional hour of study doubles the odds of passing the exam.")
print("=" * 65)
if __name__ == "__main__":
interpret_coefficients()
import numpy as np
def compute_log_loss(y_true, y_pred):
epsilon = 1e-15 # Prevent log(0)
y_pred = np.clip(y_pred, epsilon, 1 - epsilon)
loss = -np.mean(y_true * np.log(y_pred) + (1 - y_true) * np.log(1 - y_pred))
return loss
def run_log_loss_demo():
print("=" * 65)
print("๐ EXAMPLE 4: BINARY CROSS-ENTROPY (LOG LOSS)")
print("=" * 65)
y_true = np.array([1, 0, 1, 1, 0])
# Perfect predictions vs Poor predictions
y_pred_good = np.array([0.95, 0.05, 0.90, 0.88, 0.10])
y_pred_poor = np.array([0.20, 0.80, 0.30, 0.40, 0.70])
loss_good = compute_log_loss(y_true, y_pred_good)
loss_poor = compute_log_loss(y_true, y_pred_poor)
print(f"Log Loss (Confident & Correct Predictions) : {loss_good:.4f}")
print(f"Log Loss (Poor & Incorrect Predictions) : {loss_poor:.4f}")
print("=" * 65)
if __name__ == "__main__":
run_log_loss_demo()
import numpy as np
def sigmoid(z):
return 1 / (1 + np.exp(-np.clip(z, -500, 500)))
def run_logistic_gd():
print("=" * 65)
print("๐ EXAMPLE 5: GRADIENT DESCENT FOR LOGISTIC REGRESSION")
print("=" * 65)
np.random.seed(42)
X = np.array([[1.0, 2.0], [2.0, 3.0], [3.0, 1.0], [4.0, 3.0]])
y = np.array([0, 0, 1, 1])
n_samples, n_features = X.shape
weights = np.zeros(n_features)
bias = 0.0
lr = 0.1
for _ in range(500):
model_input = X @ weights + bias
predictions = sigmoid(model_input)
# Gradients
dw = (1 / n_samples) * (X.T @ (predictions - y))
db = (1 / n_samples) * np.sum(predictions - y)
weights -= lr * dw
bias -= lr * db
print(f"Optimized Weights : {np.round(weights, 4)}")
print(f"Optimized Bias : {bias:.4f}")
print("=" * 65)
if __name__ == "__main__":
run_logistic_gd()
import numpy as np
def simulate_ovr():
print("=" * 65)
print("๐ EXAMPLE 6: MULTI-CLASS CLASSIFICATION VIA ONE-VS-REST")
print("=" * 65)
classes = ['Cat', 'Dog', 'Rabbit']
# Simulated probability scores for an incoming image across 3 binary classifiers
probabilities = [0.15, 0.78, 0.07] # Classifier 2 (Dog) has highest probability
best_class_idx = np.argmax(probabilities)
print(f"Class Probabilities -> Cat: {probabilities[0]}, Dog: {probabilities[1]}, Rabbit: {probabilities[2]}")
print(f"Predicted Class (OvRargmax): {classes[best_class_idx]}")
print("=" * 65)
if __name__ == "__main__":
simulate_ovr()
import numpy as np
def evaluate_classification(y_true, y_pred):
tp = np.sum((y_true == 1) & (y_pred == 1))
fp = np.sum((y_true == 0) & (y_pred == 1))
fn = np.sum((y_true == 1) & (y_pred == 0))
tn = np.sum((y_true == 0) & (y_pred == 0))
accuracy = (tp + tn) / len(y_true)
precision = tp / (tp + fp) if (tp + fp) > 0 else 0
recall = tp / (tp + fn) if (tp + fn) > 0 else 0
f1 = 2 * (precision * recall) / (precision + recall) if (precision + recall) > 0 else 0
return accuracy, precision, recall, f1, [[tn, fp], [fn, tp]]
def run_metrics_demo():
print("=" * 65)
print("๐ EXAMPLE 7: CLASSIFICATION EVALUATION METRICS")
print("=" * 65)
y_true = np.array([1, 0, 1, 1, 0, 1, 0, 0, 1, 0])
y_pred = np.array([1, 0, 1, 0, 0, 1, 1, 0, 1, 0])
acc, prec, rec, f1, cm = evaluate_classification(y_true, y_pred)
print(f"Confusion Matrix (TN, FP / FN, TP):\n{cm[0]}\n{cm[1]}")
print(f"Accuracy : {acc:.4f}")
print(f"Precision : {prec:.4f}")
print(f"Recall : {rec:.4f}")
print(f"F1-Score : {f1:.4f}")
print("=" * 65)
if __name__ == "__main__":
run_metrics_demo()
import numpy as np
from sklearn.metrics import roc_auc_score
def run_roc_auc():
print("=" * 65)
print("๐ณ EXAMPLE 8: ROC-AUC SCORE FOR FRAUD DETECTION")
print("=" * 65)
y_true = np.array([0, 0, 1, 0, 1, 1, 0, 0, 1, 0])
# Predicted fraud probabilities from logistic regression
y_probs = np.array([0.05, 0.12, 0.88, 0.22, 0.95, 0.76, 0.08, 0.15, 0.91, 0.30])
auc_score = roc_auc_score(y_true, y_probs)
print(f"Fraud Detection AUC-Score : {auc_score:.4f}")
print("Interpretation: An AUC of 0.97 indicates exceptional model discrimination ability.")
print("=" * 65)
if __name__ == "__main__":
run_roc_auc()
import numpy as np
from sklearn.linear_model import LogisticRegression
def run_balanced_logistic():
print("=" * 65)
print("โ๏ธ EXAMPLE 9: HANDLING CLASS IMBALANCE")
print("=" * 65)
# Imbalanced dataset: 90 negative samples, 10 positive samples
X = np.random.randn(100, 2)
y = np.array([0]*90 + [1]*10)
model_balanced = LogisticRegression(class_weight='balanced', solver='lbfgs')
model_balanced.fit(X, y)
print("Model trained successfully with class_weight='balanced'.")
print(f"Intercept with balancing: {model_balanced.intercept_[0]:.4f}")
print("=" * 65)
if __name__ == "__main__":
run_balanced_logistic()
LOGISTIC REGRESSION & CLASSIFICATION REFERENCE MATRIX:
โโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโฌโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโฌโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโ
โ Concept / Component โ Mathematical Formula โ Primary Machine Learning Application โ
โโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโผโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโผโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโค
โ Sigmoid Function โ ฯ(z) = 1 / (1 + e^-z) โ Squashing continuous log-odds into [0, 1] probabilities โ
โ Logit (Log-Odds) โ z = w*x + b โ Linear combination of features in log-odds space โ
โ Binary Cross-Entropy Loss โ -[y log(ลท) + (1-y)log(1-ลท)] โ Convex cost function optimized via gradient descent โ
โ Decision Threshold โ ลท = 1 if P โฅ 0.5 else 0 โ Converting predicted probabilities into discrete classes โ
โ Confusion Matrix โ [[TN, FP], [FN, TP]] โ Tabulating correct vs incorrect binary predictions โ
โ Precision & Recall โ TP/(TP+FP) & TP/(TP+FN) โ Evaluating false alarms vs missed positive detections โ
โ ROC-AUC Score โ Area under TPR vs FPR curve โ Measuring overall ranking and discrimination capability โ
โ Class Weight Balancing โ w_class = n / (n_classes * n_i)| Correcting bias toward majority classes in imbalanced data |
โโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโดโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโดโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโค
โ ENGINEERING BEST PRACTICES: โ
โ 1. Always scale features using StandardScaler before training logistic regression with regularization (L1/L2). โ
โ 2. Do not rely solely on accuracy for imbalanced datasets; prioritize Precision, Recall, F1-Score, and ROC-AUC. โ
โ 3. Tune classification thresholds (e.g., threshold = 0.3) depending on business costs of False Positives vs False Negatives.โ
โโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโ