๐ The Cornerstone Algorithm of Predictive Modeling
Linear Regression models the relationship between a continuous target variable and one or more independent predictor features by fitting a linear equation. When analytical solutions like Ordinary Least Squares (OLS) become computationally expensive or impossible for massive datasets, Gradient Descent provides an iterative optimization algorithm to navigate the loss landscape and converge upon optimal model weights. This guide covers 10 real-world engineering patterns for linear regression and gradient descent.
import numpy as np
def run_ols():
print("=" * 65)
print("๐ EXAMPLE 1: SIMPLE LINEAR REGRESSION VIA OLS")
print("=" * 65)
# Years of Experience vs. Salary ($k)
X = np.array([1.1, 2.0, 3.5, 4.2, 5.0])
y = np.array([45.0, 52.0, 70.0, 78.0, 90.0])
# Calculate analytical slope (w) and intercept (b)
x_mean = np.mean(X)
y_mean = np.mean(y)
numerator = np.sum((X - x_mean) * (y - y_mean))
denominator = np.sum((X - x_mean) ** 2)
w = numerator / denominator
b = y_mean - (w * x_mean)
print(f"Calculated Slope (Weight w) : {w:.4f}")
print(f"Calculated Intercept (Bias b) : {b:.4f}")
print(f"Prediction for 6.0 Years Exp : ${w * 6.0 + b:.2f}k")
print("=" * 65)
if __name__ == "__main__":
run_ols()
import numpy as np
def run_batch_gd():
print("=" * 65)
print("๐ EXAMPLE 2: HOUSE PRICE PREDICTION VIA BATCH GRADIENT DESCENT")
print("=" * 65)
# House Size (sq ft / 1000) vs Price ($k)
X = np.array([1.0, 1.5, 2.0, 2.5, 3.0])
y = np.array([150.0, 200.0, 250.0, 300.0, 350.0])
n = len(X)
w, b = 0.0, 0.0
lr = 0.05
epochs = 200
for epoch in range(epochs):
y_pred = w * X + b
loss = (1 / n) * np.sum((y_pred - y) ** 2)
# Gradients
dw = (2 / n) * np.sum((y_pred - y) * X)
db = (2 / n) * np.sum(y_pred - y)
# Update weights
w -= lr * dw
b -= lr * db
print(f"Final Optimized Weight (w) : {w:.4f}")
print(f"Final Optimized Bias (b) : {b:.4f}")
print(f"Final MSE Loss : {loss:.6f}")
print("=" * 65)
if __name__ == "__main__":
run_batch_gd()
import numpy as np
def run_multivariate_gd():
print("=" * 65)
print("๐ข EXAMPLE 3: MULTIVARIATE LINEAR REGRESSION VIA MATRIX GD")
print("=" * 65)
np.random.seed(42)
# Features: [Square Feet / 1000, Bedrooms, Age of House]
X_raw = np.array([
[1.5, 3, 10],
[2.0, 4, 5],
[1.2, 2, 20],
[3.0, 5, 2],
[2.5, 4, 8]
])
y = np.array([200.0, 270.0, 160.0, 380.0, 310.0])
# Add bias column of ones to X
X = np.c_[np.ones(X_raw.shape[0]), X_raw]
n_samples, n_features = X.shape
theta = np.zeros(n_features)
lr = 0.01
epochs = 1000
for _ in range(epochs):
predictions = X @ theta
errors = predictions - y
gradient = (2 / n_samples) * (X.T @ errors)
theta -= lr * gradient
print(f"Optimized Parameters (Theta): {np.round(theta, 4)}")
print(f"Prediction for [2.2 sq_ft, 3 beds, 10 yrs] : ${np.dot([1, 2.2, 3, 10], theta):.2f}k")
print("=" * 65)
if __name__ == "__main__":
run_multivariate_gd()
import numpy as np
def test_learning_rates():
print("=" * 65)
print("โก EXAMPLE 4: LEARNING RATE SENSITIVITY ANALYSIS")
print("=" * 65)
X = np.array([1.0, 2.0, 3.0])
y = np.array([2.0, 4.0, 6.0])
learning_rates = [0.001, 0.1, 1.5] # 1.5 is intentionally too large (diverges)
for lr in learning_rates:
w = 0.0
diverged = False
for _ in range(50):
y_pred = w * X
dw = (2 / len(X)) * np.sum((y_pred - y) * X)
w -= lr * dw
if np.isnan(w) or abs(w) > 1e6:
diverged = True
break
status = "DIVERGED (Too High)" if diverged else f"Converged to w = {w:.4f}"
print(f"Learning Rate {lr:5.3f} -> {status}")
print("=" * 65)
if __name__ == "__main__":
test_learning_rates()
import numpy as np
def run_scaling_comparison():
print("=" * 65)
print("โ๏ธ EXAMPLE 5: FEATURE STANDARDIZATION FOR CONVERGENCE")
print("=" * 65)
np.random.seed(42)
# Feature 1: Age (20-60), Feature 2: Income ($30k-$150k)
X = np.column_stack([
np.random.randint(20, 60, 50),
np.random.randint(30000, 150000, 50)
])
y = X[:, 0] * 0.5 + X[:, 1] * 0.002 + np.random.normal(0, 1, 50)
# Standardize X: (X - mean) / std
X_scaled = (X - np.mean(X, axis=0)) / np.std(X, axis=0)
print(f"Unscaled Feature 2 Mean/Std : {np.mean(X[:, 1]):.1f} / {np.std(X[:, 1]):.1f}")
print(f"Scaled Feature 2 Mean/Std : {np.mean(X_scaled[:, 1]):.4f} / {np.std(X_scaled[:, 1]):.4f}")
print("Standardization enables stable gradient descent updates with higher learning rates.")
print("=" * 65)
if __name__ == "__main__":
run_scaling_comparison()
import numpy as np
def run_sgd():
print("=" * 65)
print("๐ฒ EXAMPLE 6: STOCHASTIC GRADIENT DESCENT (SGD)")
print("=" * 65)
np.random.seed(42)
X = np.linspace(0, 10, 100)
y = 3.5 * X + 10.0 + np.random.normal(0, 0.5, 100)
w, b = 0.0, 0.0
lr = 0.01
# Simulate 5 epochs of SGD looping over shuffled individual samples
for epoch in range(5):
indices = np.arange(len(X))
np.random.shuffle(indices)
for i in indices:
xi, yi = X[i], y[i]
y_pred = w * xi + b
dw = 2 * (y_pred - yi) * xi
db = 2 * (y_pred - yi)
w -= lr * dw
b -= lr * db
print(f"SGD Estimated Weight (w) : {w:.4f}")
print(f"SGD Estimated Bias (b) : {b:.4f}")
print("=" * 65)
if __name__ == "__main__":
run_sgd()
import numpy as np
def run_ridge_gd():
print("=" * 65)
print("๐ก๏ธ EXAMPLE 7: RIDGE REGULARIZATION (L2) GRADIENT DESCENT")
print("=" * 65)
X = np.array([[1.0, 2.0], [2.0, 1.0], [3.0, 4.0]])
y = np.array([5.0, 6.0, 11.0])
n_samples = len(X)
w = np.zeros(2)
b = 0.0
lr = 0.01
lam = 0.1 # Regularization strength
for _ in range(500):
y_pred = X @ w + b
# L2 Gradient includes weight decay term: + 2 * lam * w
dw = (2 / n_samples) * (X.T @ (y_pred - y)) + (2 * lam * w)
db = (2 / n_samples) * np.sum(y_pred - y)
w -= lr * dw
b -= lr * db
print(f"Regularized Weights (w) : {np.round(w, 4)}")
print(f"Regularized Bias (b) : {b:.4f}")
print("=" * 65)
if __name__ == "__main__":
run_ridge_gd()
import numpy as np
def evaluate_regression():
print("=" * 65)
print("๐ EXAMPLE 8: REGRESSION PERFORMANCE EVALUATION METRICS")
print("=" * 65)
y_true = np.array([150.0, 200.0, 250.0, 300.0])
y_pred = np.array([155.0, 195.0, 245.0, 310.0])
mse = np.mean((y_true - y_pred) ** 2)
rmse = np.sqrt(mse)
mae = np.mean(np.abs(y_true - y_pred))
ss_total = np.sum((y_true - np.mean(y_true)) ** 2)
ss_residual = np.sum((y_true - y_pred) ** 2)
r2 = 1 - (ss_residual / ss_total)
print(f"Mean Squared Error (MSE) : {mse:.2f}")
print(f"Root MSE (RMSE) : {rmse:.2f}")
print(f"Mean Absolute Error (MAE): {mae:.2f}")
print(f"R-Squared Score (Rยฒ) : {r2:.4f}")
print("=" * 65)
if __name__ == "__main__":
evaluate_regression()
import numpy as np
from sklearn.linear_model import LinearRegression, SGDRegressor
from sklearn.preprocessing import StandardScaler
def run_sklearn_regression():
print("=" * 65)
print("๐ค EXAMPLE 9: SCIKIT-LEARN LINEAR & SGD REGRESSOR")
print("=" * 65)
np.random.seed(42)
X = np.random.randn(100, 2)
y = X[:, 0] * 3.0 + X[:, 1] * 5.0 + 10.0 + np.random.normal(0, 0.1, 100)
# 1. Analytical OLS Model
ols = LinearRegression()
ols.fit(X, y)
# 2. Gradient Descent Model (requires feature scaling)
scaler = StandardScaler()
X_scaled = scaler.fit_transform(X)
sgd = SGDRegressor(max_iter=1000, tol=1e-3, random_state=42)
sgd.fit(X_scaled, y)
print(f"OLS Coefficients : {np.round(ols.coef_, 2)}, Intercept: {ols.intercept_:.2f}")
print(f"SGD Coefficients : {np.round(sgd.coef_, 2)}, Intercept: {sgd.intercept_:.2f}")
print("=" * 65)
if __name__ == "__main__":
run_sklearn_regression()
LINEAR REGRESSION & GRADIENT DESCENT REFERENCE MATRIX:
โโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโฌโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโฌโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโ
โ Concept / Component โ Mathematical Formulation โ Primary Machine Learning Application โ
โโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโผโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโผโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโค
โ Hypothesis Function โ ลท = Xw + b โ Predicting continuous target variables โ
โ Mean Squared Error (MSE) โ L = (1/n) ฮฃ (ลท - y)ยฒ โ Cost function measuring squared prediction error โ
โ Batch Gradient Descent โ w = w - ฮฑ * (2/n)X^T(ลท - y) โ Iterative weight optimization across all samples โ
โ Stochastic Gradient Descent โ w = w - ฮฑ * 2(ลท_i - y_i)x_i โ Fast iterative updates per single sample โ
โ Learning Rate (ฮฑ) โ Step size hyperparameter โ Controlling convergence speed and stability โ
โ Feature Standardization โ z = (x - ฮผ) / ฯ โ Normalizing feature scales for rapid gradient descent โ
โ Ridge Penalty (L2) โ + ฮป ฮฃ wยฒ โ Preventing overfitting via weight magnitude decay โ
โ R-Squared Score (Rยฒ) โ 1 - (SS_res / SS_tot) โ Evaluating model goodness-of-fit โ
โโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโดโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโดโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโค
โ ENGINEERING BEST PRACTICES: โ
โ 1. Always standardize feature matrices before training models with Gradient Descent. โ
โ 2. Monitor loss curves across epochs to tune the learning rate ($\alpha$) and prevent divergence or slow convergence. โ
โ 3. Use analytical OLS for datasets with small-to-medium feature dimensions, and SGD/Mini-Batch for big data. โ
โโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโ