β±οΈ Modeling Time-Series, Speech, and Language with Recurrence
Standard feedforward neural networks assume all inputs are independent, making them inadequate for sequential data where order matters (such as text, audio, and stock prices). Recurrent Neural Networks (RNNs) maintain a hidden state that acts as a memory of past inputs. However, vanilla RNNs suffer from vanishing and exploding gradients over long sequences. LSTMs (Long Short-Term Memory) and GRUs (Gated Recurrent Units) resolve this by introducing sophisticated gating mechanisms (input, forget, output gates) that regulate information flow across extended time horizons. This masterclass covers recurrent architectures across 10 real-world engineering examples.
import numpy as np
def rnn_step_forward(x_t, h_prev, W_xh, W_hh, b_h):
# Vanilla RNN hidden state transition
linear_comb = np.dot(W_xh, x_t) + np.dot(W_hh, h_prev) + b_h
h_next = np.tanh(linear_comb)
return h_next
def simulate_rnn_sequence():
print("=" * 65)
print("π EXAMPLE 1: VANILLA RNN HIDDEN STATE STEP-BY-STEP SIMULATION")
print("=" * 65)
np.random.seed(42)
input_dim = 3
hidden_dim = 4
W_xh = np.random.randn(hidden_dim, input_dim) * 0.1
W_hh = np.random.randn(hidden_dim, hidden_dim) * 0.1
b_h = np.zeros((hidden_dim, 1))
h_prev = np.zeros((hidden_dim, 1))
sequence_input = [np.random.randn(input_dim, 1) for _ in range(3)]
for t, x_t in enumerate(sequence_input, 1):
h_prev = rnn_step_forward(x_t, h_prev, W_xh, W_hh, b_h)
print(f"Time Step {t} Hidden State: {np.round(h_prev.flatten(), 4).tolist()}")
print("=" * 65)
if __name__ == "__main__":
simulate_rnn_sequence()
import torch
import torch.nn as nn
def simulate_stock_lstm():
print("=" * 65)
print("π EXAMPLE 2: PYTORCH LSTM FOR TIME-SERIES STOCK FORECASTING")
print("=" * 65)
# Batch size = 32, Sequence length = 30 days, Input features = 5 (open, high, low, close, volume)
batch_size = 32
seq_length = 30
input_dim = 5
hidden_dim = 64
num_layers = 2
lstm_layer = nn.LSTM(input_size=input_dim, hidden_size=hidden_dim, num_layers=num_layers, batch_first=True)
stock_tensor = torch.randn(batch_size, seq_length, input_dim)
# Initializing hidden and cell states (h_0, c_0)
h_0 = torch.zeros(num_layers, batch_size, hidden_dim)
c_0 = torch.zeros(num_layers, batch_size, hidden_dim)
output, (h_n, c_n) = lstm_layer(stock_tensor, (h_0, c_0))
print(f"Input Stock Tensor Shape: {stock_tensor.shape}")
print(f"LSTM Output Shape : {output.shape}")
print(f"Final Hidden State Shape: {h_n.shape}")
print("=" * 65)
if __name__ == "__main__":
simulate_stock_lstm()
import torch
import torch.nn as nn
def simulate_iot_gru():
print("=" * 65)
print("βοΈ EXAMPLE 3: PYTORCH GRU FOR INDUSTRIAL SENSOR STREAM ANOMALIES")
print("=" * 65)
# Batch = 16 sensors, Time steps = 100, Features = 3 (Vibration, Temp, Pressure)
batch_size = 16
seq_len = 100
input_dim = 3
hidden_dim = 32
gru_layer = nn.GRU(input_size=input_dim, hidden_size=hidden_dim, batch_first=True)
sensor_data = torch.randn(batch_size, seq_len, input_dim)
output, h_n = gru_layer(sensor_data)
print(f"Sensor Stream Shape : {sensor_data.shape}")
print(f"GRU Output Shape : {output.shape}")
print(f"Structural Advantage : Fewer gates than LSTM, enabling faster training on edge devices")
print("=" * 65)
if __name__ == "__main__":
simulate_iot_gru()
import torch
import torch.nn as nn
class CharLSTM(nn.Module):
def __init__(self, vocab_size, embed_dim, hidden_dim):
super(CharLSTM, self).__init__()
self.embedding = nn.Embedding(vocab_size, embed_dim)
self.lstm = nn.LSTM(embed_dim, hidden_dim, batch_first=True)
self.fc = nn.Linear(hidden_dim, vocab_size)
def forward(self, x):
x = self.embedding(x)
out, _ = self.lstm(x)
out = self.fc(out)
return out
def simulate_char_generation():
print("=" * 65)
print("βοΈ EXAMPLE 4: CHARACTER-LEVEL LSTM TEXT GENERATION MODEL")
print("=" * 65)
vocab_size = 80 # ASCII character set
model = CharLSTM(vocab_size=vocab_size, embed_dim=32, hidden_dim=64)
model.eval()
# Batch=1, Sequence of 10 characters tokenized as integer IDs
input_indices = torch.randint(0, vocab_size, (1, 10))
logits = model(input_indices)
print(f"Input Indices Shape : {input_indices.shape}")
print(f"Logits Output Shape : {logits.shape}")
print(f"Generation Objective : Softmax sampling over vocab_size to output next character")
print("=" * 65)
if __name__ == "__main__":
simulate_char_generation()
import torch
import torch.nn as nn
def simulate_weather_lstm():
print("=" * 65)
print("β
EXAMPLE 5: STACKED LSTM FOR METEOROLOGICAL FORECASTING")
print("=" * 65)
# 7 days of hourly data = 168 timesteps, 6 weather features
seq_len = 168
features = 6
lstm = nn.LSTM(input_size=features, hidden_size=128, num_layers=3, batch_first=True)
regressor = nn.Linear(128, 1) # Predict next hour temperature
weather_data = torch.randn(4, seq_len, features) # Batch of 4 weather stations
lstm_out, _ = lstm(weather_data)
next_hour_temp = regressor(lstm_out[:, -1, :]) # Using final hidden state
print(f"Weather Tensor Shape : {weather_data.shape}")
print(f"Temperature Forecast : {next_hour_temp.shape}")
print(f"Forecast Horizon : Continuous multi-variate regression output")
print("=" * 65)
if __name__ == "__main__":
simulate_weather_lstm()
import torch
import torch.nn as nn
def simulate_bilstm_speech():
print("=" * 65)
print("ποΈ EXAMPLE 6: BIDIRECTIONAL LSTM FOR SPEECH TRANSCRIPTION")
print("=" * 65)
# Spectrogram frames: Batch=2, Frames=200, Frequency Bins=80
audio_frames = torch.randn(2, 200, 80)
# Bidirectional LSTM doubles hidden dimension output
hidden_dim = 128
bilstm = nn.LSTM(input_size=80, hidden_size=hidden_dim, bidirectional=True, batch_first=True)
linear_classifier = nn.Linear(hidden_dim * 2, 40) # 40 phoneme classes
lstm_out, _ = bilstm(audio_frames)
phoneme_logits = linear_classifier(lstm_out)
print(f"Audio Frame Tensor : {audio_frames.shape}")
print(f"BiLSTM Output Shape : {lstm_out.shape} (Doubled due to bidirectional context)")
print(f"Phoneme Logits Shape : {phoneme_logits.shape}")
print("=" * 65)
if __name__ == "__main__":
simulate_bilstm_speech()
import torch
import torch.nn as nn
class Seq2SeqEncoderDecoder(nn.Module):
def __init__(self, vocab_size, embed_dim, hidden_dim):
super(Seq2SeqEncoderDecoder, self).__init__()
self.embedding = nn.Embedding(vocab_size, embed_dim)
self.encoder_lstm = nn.LSTM(embed_dim, hidden_dim, batch_first=True)
self.decoder_lstm = nn.LSTM(embed_dim, hidden_dim, batch_first=True)
self.fc = nn.Linear(hidden_dim, vocab_size)
def forward(self, source, target):
# Encoder pass
enc_emb = self.embedding(source)
_, (h_n, c_n) = self.encoder_lstm(enc_emb)
# Decoder pass using encoder final hidden state as initial state
dec_emb = self.embedding(target)
dec_out, _ = self.decoder_lstm(dec_emb, (h_n, c_n))
logits = self.fc(dec_out)
return logits
def simulate_seq2seq():
print("=" * 65)
print("π EXAMPLE 7: SEQ2SEQ LSTM ENCODER-DECODER TRANSLATION")
print("=" * 65)
vocab_size = 1000
model = Seq2SeqEncoderDecoder(vocab_size=vocab_size, embed_dim=64, hidden_dim=128)
model.eval()
source_seq = torch.randint(0, vocab_size, (2, 12)) # Batch=2, Src Len=12
target_seq = torch.randint(0, vocab_size, (2, 15)) # Batch=2, Tgt Len=15
logits = model(source_seq, target_seq)
print(f"Source Sentence Shape: {source_seq.shape}")
print(f"Target Sentence Shape: {target_seq.shape}")
print(f"Translation Logits : {logits.shape}")
print("=" * 65)
if __name__ == "__main__":
simulate_seq2seq()
import torch
import torch.nn as nn
class SentimentLSTM(nn.Module):
def __init__(self, vocab_size, embed_dim, hidden_dim):
super(SentimentLSTM, self).__init__()
self.embedding = nn.Embedding(vocab_size, embed_dim)
self.lstm = nn.LSTM(embed_dim, hidden_dim, batch_first=True)
self.fc = nn.Linear(hidden_dim, 1) # Binary sentiment score
def forward(self, x):
emb = self.embedding(x)
out, (h_n, c_n) = self.lstm(emb)
# Use final layer hidden state for classification
final_hidden = h_n[-1]
score = torch.sigmoid(self.fc(final_hidden))
return score
def simulate_sentiment_model():
print("=" * 65)
print("β€οΈ EXAMPLE 8: LSTM MOVIE REVIEW SENTIMENT CLASSIFIER")
print("=" * 65)
vocab_size = 5000
model = SentimentLSTM(vocab_size=vocab_size, embed_dim=32, hidden_dim=64)
model.eval()
# Batch=4 reviews, each padded to 50 word token IDs
review_batch = torch.randint(0, vocab_size, (4, 50))
probabilities = model(review_batch)
print(f"Review Batch Shape : {review_batch.shape}")
print(f"Sentiment Scores : {probabilities.squeeze().tolist()}")
print(f"Classification Rule : Score > 0.5 = Positive, <= 0.5 = Negative")
print("=" * 65)
if __name__ == "__main__":
simulate_sentiment_model()
import torch
import torch.nn as nn
def simulate_music_generator():
print("=" * 65)
print("π΅ EXAMPLE 9: GRU NETWORK FOR MIDI NOTE SEQUENCE GENERATION")
print("=" * 65)
# 128 MIDI pitch values as vocabulary classes
midi_vocab = 128
gru = nn.GRU(input_size=64, hidden_size=128, num_layers=2, batch_first=True)
note_classifier = nn.Linear(128, midi_vocab)
# Batch=1, 64-dimensional note embedding sequence (length = 32 steps)
note_embeddings = torch.randn(1, 32, 64)
gru_out, _ = gru(note_embeddings)
next_note_logits = note_classifier(gru_out)
print(f"Note Embedding Shape : {note_embeddings.shape}")
print(f"MIDI Logits Output : {next_note_logits.shape}")
print(f"Musical Application : Autoregressive melody generation note by note")
print("=" * 65)
if __name__ == "__main__":
simulate_music_generator()
RNN, LSTM & GRU SEQUENCE MODELS REFERENCE MATRIX:
ββββββββββββββββββββββββββββ¬βββββββββββββββββββββββββββββββ¬βββββββββββββββββββββββββββββββ¬ββββββββββββββββββββββββββββββββββ
β Attribute / Metric β Vanilla RNN β LSTM (Long Short-Term Memory)β GRU (Gated Recurrent Unit) β
ββββββββββββββββββββββββββββΌβββββββββββββββββββββββββββββββΌβββββββββββββββββββββββββββββββΌββββββββββββββββββββββββββββββββββ€
β Gating Mechanism β None (Single tanh state) β 3 Gates (Input, Forget, Out) β 2 Gates (Reset, Update) β
β Gradient Stability β Prone to vanishing/exploding β Excellent over long sequencesβ Excellent over long sequences β
β Parameter Complexity β Lowest β Highest (4 weight matrices) β Moderate (3 weight matrices) β
β Training Speed β Fast per step, slow overall β Slower due to complex gates β Faster than LSTM β
β Ideal Production Domain β Short sequences, baselines β Complex NLP, Long-form audio β Lightweight IoT, fast timeseriesβ
ββββββββββββββββββββββββββββ΄βββββββββββββββββββββββββββββββ΄βββββββββββββββββββββββββββββββ΄ββββββββββββββββββββββββββββββββββ€
β RECURRENT MODEL ENGINEERING BEST PRACTICES: β
β 1. Prefer LSTM or GRU over vanilla RNNs for almost all sequential tasks to prevent vanishing gradient bottlenecks. β
β 2. Use gradient clipping (`torch.nn.utils.clip_grad_norm_`) during training to eliminate exploding gradients. β
β 3. For extremely long text sequences and transformers, consider upgrading from RNN/LSTM to self-attention models. β
ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ