π€ Revolutions in AI: Attention Is All You Need
Introduced in 2017 by Vaswani et al., the Transformer architecture dismantled sequential recurrent bottlenecks (RNNs, LSTMs) by relying entirely on Self-Attention mechanisms. By computing relationships between all words in a sequence simultaneously, Transformers capture long-range dependencies efficiently and enable massive parallelization. Today, Transformers power foundational Large Language Models (GPT-4, LLaMA, Claude), multimodal vision-language models, and protein folding engines. This masterclass covers 10 real-world engineering applications of Transformer attention mechanisms.
import torch
import torch.nn.functional as F
def scaled_dot_product_attention(Q, K, V, mask=None):
d_k = Q.size(-1)
# Step 1: Compute dot product of Queries and Keys
scores = torch.matmul(Q, K.transpose(-2, -1)) / torch.sqrt(torch.tensor(d_k, dtype=torch.float32))
# Step 2: Apply optional mask (e.g., causal mask for autoregressive decoding)
if mask is not None:
scores = scores.masked_fill(mask == 0, -1e9)
# Step 3: Softmax normalization to get attention weights
attention_weights = F.softmax(scores, dim=-1)
# Step 4: Multiply by Values matrix
output = torch.matmul(attention_weights, V)
return output, attention_weights
def simulate_attention():
print("=" * 65)
print("π― EXAMPLE 1: SCALED DOT-PRODUCT ATTENTION CALCULATION")
print("=" * 65)
torch.manual_seed(42)
batch_size, seq_len, d_k = 1, 4, 8
Q = torch.randn(batch_size, seq_len, d_k)
K = torch.randn(batch_size, seq_len, d_k)
V = torch.randn(batch_size, seq_len, d_k)
output, weights = scaled_dot_product_attention(Q, K, V)
print(f"Query Tensor Shape : {Q.shape}")
print(f"Attention Weights Shape : {weights.shape}")
print(f"Output Context Shape : {output.shape}")
print("=" * 65)
if __name__ == "__main__":
simulate_attention()
import torch
import torch.nn as nn
def simulate_multihead_attention():
print("=" * 65)
print("π§ EXAMPLE 2: PYTORCH MULTI-HEAD ATTENTION LAYER")
print("=" * 65)
embed_dim = 512
num_heads = 8
batch_size = 2
seq_len = 16
mha = nn.MultiheadAttention(embed_dim=embed_dim, num_heads=num_heads, batch_first=True)
x = torch.randn(batch_size, seq_len, embed_dim)
# Self-attention: Q, K, V are all derived from input x
attn_output, attn_weights = mha(x, x, x)
print(f"Input Tensor Shape : {x.shape}")
print(f"Multi-Head Output Shape : {attn_output.shape}")
print(f"Attention Matrix Shape : {attn_weights.shape}")
print("=" * 65)
if __name__ == "__main__":
simulate_multihead_attention()
import torch
import math
class PositionalEncoding(torch.nn.Module):
def __init__(self, d_model, max_len=5000):
super().__init__()
pe = torch.zeros(max_len, d_model)
position = torch.arange(0, max_len, dtype=torch.float).unsqueeze(1)
div_term = torch.exp(torch.arange(0, d_model, 2).float() * (-math.log(10000.0) / d_model))
pe[:, 0::2] = torch.sin(position * div_term)
pe[:, 1::2] = torch.cos(position * div_term)
self.register_buffer('pe', pe.unsqueeze(0))
def forward(self, x):
return x + self.pe[:, :x.size(1)]
def simulate_positional_encoding():
print("=" * 65)
print("π EXAMPLE 3: SINUSOIDAL POSITIONAL ENCODING MODULE")
print("=" * 65)
batch_size, seq_len, d_model = 1, 10, 64
pos_encoder = PositionalEncoding(d_model=d_model)
embeddings = torch.randn(batch_size, seq_len, d_model)
encoded_embeddings = pos_encoder(embeddings)
print(f"Raw Embedding Shape : {embeddings.shape}")
print(f"Positional Encoded Shape: {encoded_embeddings.shape}")
print(f"Mechanism : Element-wise addition of token vectors and position tensors")
print("=" * 65)
if __name__ == "__main__":
simulate_positional_encoding()
import torch
def simulate_causal_mask():
print("=" * 65)
print("π EXAMPLE 4: CAUSAL ATTENTION MASK FOR LLM DECODING")
print("=" * 65)
seq_len = 5
# Causal mask: upper triangle filled with -inf (or 0 for multiplicative mask)
causal_mask = torch.triu(torch.ones(seq_len, seq_len), diagonal=1)
print(f"Sequence Length : {seq_len} tokens")
print(f"Causal Mask Matrix:\n{causal_mask}")
print(f"Enforcement Rule : Token at index i cannot attend to index j where j > i")
print("=" * 65)
if __name__ == "__main__":
simulate_causal_mask()
import torch
import torch.nn as nn
class PatchEmbedding(nn.Module):
def __init__(self, img_size=224, patch_size=16, in_channels=3, embed_dim=768):
super().__init__()
self.num_patches = (img_size // patch_size) ** 2
self.proj = nn.Conv2d(in_channels, embed_dim, kernel_size=patch_size, stride=patch_size)
def forward(self, x):
# x shape: [Batch, Channels, Height, Width]
x = self.proj(x) # [Batch, Embed_Dim, Grid_H, Grid_W]
x = x.flatten(2).transpose(1, 2) # [Batch, Num_Patches, Embed_Dim]
return x
def simulate_vit_patches():
print("=" * 65)
print("πΌοΈ EXAMPLE 5: VISION TRANSFORMER (ViT) PATCH EXTRACTION")
print("=" * 65)
patch_embed = PatchEmbedding(img_size=224, patch_size=16, embed_dim=768)
images = torch.randn(2, 3, 224, 224) # Batch of 2 images
patch_tokens = patch_embed(images)
print(f"Input Image Batch Shape : {images.shape}")
print(f"Extracted Patch Tokens : {patch_tokens.shape}")
print(f"Architecture Insight : Treats 196 image patches (14x14) exactly like words in a sentence")
print("=" * 65)
if __name__ == "__main__":
simulate_vit_patches()
import torch
import torch.nn as nn
def simulate_transformer_nmt():
print("=" * 65)
print("π EXAMPLE 6: TRANSFORMER ENCODER-DECODER TRANSLATION PIPELINE")
print("=" * 65)
src_vocab_size = 10000
tgt_vocab_size = 10000
embed_dim = 256
transformer = nn.Transformer(
d_model=embed_dim, nhead=8, num_encoder_layers=3, num_decoder_layers=3, batch_first=True
)
src = torch.randint(0, src_vocab_size, (2, 20)) # Batch=2, Src Len=20
tgt = torch.randint(0, tgt_vocab_size, (2, 15)) # Batch=2, Tgt Len=15
out = transformer(src, tgt)
print(f"Source Sentence Shape : {src.shape}")
print(f"Target Sentence Shape : {tgt.shape}")
print(f"Transformer Output Shape: {out.shape}")
print("=" * 65)
if __name__ == "__main__":
simulate_transformer_nmt()
import torch
import torch.nn as nn
def simulate_cross_attention():
print("=" * 65)
print("ποΈπ¬ EXAMPLE 7: MULTIMODAL CROSS-ATTENTION (IMAGE CAPTIONING)")
print("=" * 65)
embed_dim = 512
batch_size = 2
seq_len_text = 10
num_patches = 196
# Text queries and visual image features (Keys & Values)
text_queries = torch.randn(batch_size, seq_len_text, embed_dim)
visual_keys = torch.randn(batch_size, num_patches, embed_dim)
visual_values = torch.randn(batch_size, num_patches, embed_dim)
cross_attn = nn.MultiheadAttention(embed_dim=embed_dim, num_heads=8, batch_first=True)
caption_features, _ = cross_attn(query=text_queries, key=visual_keys, value=visual_values)
print(f"Text Query Shape : {text_queries.shape}")
print(f"Visual Feature Shape : {visual_keys.shape}")
print(f"Cross-Attention Output : {caption_features.shape}")
print(f"Multimodal Insight : Text generator queries specific image regions to form captions")
print("=" * 65)
if __name__ == "__main__":
simulate_cross_attention()
import torch
import torch.nn as nn
class MockBERTClassifier(nn.Module):
def __init__(self, vocab_size=30522, embed_dim=768, num_classes=2):
super().__init__()
self.embedding = nn.Embedding(vocab_size, embed_dim)
encoder_layer = nn.TransformerEncoderLayer(d_model=embed_dim, nhead=12, batch_first=True)
self.transformer_encoder = nn.TransformerEncoder(encoder_layer, num_layers=2)
self.classifier = nn.Linear(embed_dim, num_classes)
def forward(self, input_ids):
x = self.embedding(input_ids)
x = self.transformer_encoder(x)
# Use [CLS] token representation (index 0) for classification
cls_rep = x[:, 0, :]
return self.classifier(cls_rep)
def simulate_bert_classifier():
print("=" * 65)
print("β€οΈ EXAMPLE 8: BERT ENCODER SENTIMENT CLASSIFIER")
print("=" * 65)
model = MockBERTClassifier()
model.eval()
# Batch=3 tokenized review sentences, length=32
token_ids = torch.randint(0, 30522, (3, 32))
logits = model(token_ids)
print(f"Token IDs Shape : {token_ids.shape}")
print(f"Classification Logits : {logits.shape}")
print(f"Classification Purpose : Polarity detection using contextual [CLS] vector")
print("=" * 65)
if __name__ == "__main__":
simulate_bert_classifier()
import torch
def simulate_flash_attention():
print("=" * 65)
print("β‘ EXAMPLE 9: SCALED DOT-PRODUCT WITH MEMORY TILING (FLASHATTENTION CONCEPT)")
print("=" * 65)
# PyTorch's native scaled_dot_product_attention invokes optimized FlashAttention kernels on CUDA hardware
query = torch.randn(2, 8, 1024, 64) # [Batch, Heads, SeqLen, Dim]
key = torch.randn(2, 8, 1024, 64)
value = torch.randn(2, 8, 1024, 64)
# Fused attention kernel execution
output = torch.nn.functional.scaled_dot_product_attention(query, key, value, is_causal=True)
print(f"Query Tensor Shape : {query.shape}")
print(f"Fused Attention Output : {output.shape}")
print(f"Hardware Optimization : O(N) memory complexity via SRAM block-tiling")
print("=" * 65)
if __name__ == "__main__":
simulate_flash_attention()
TRANSFORMER ARCHITECTURE & ATTENTION REFERENCE MATRIX:
ββββββββββββββββββββββββββββ¬βββββββββββββββββββββββββββββββ¬βββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
β Attention / Architecture β Core Mechanism β Primary Production Application β
ββββββββββββββββββββββββββββΌβββββββββββββββββββββββββββββββΌβββββββββββββββββββββββββββββββββββββββββββββββββββββββββ€
β Scaled Dot-Product β Softmax(QK^T / βd_k)V β Core vector similarity scoring across token pairs β
β Multi-Head Attention β Parallel subspace projectionsβ Capturing diverse syntactic and semantic relationships β
β Causal Self-Attention β Masked upper-triangle QK^T β Autoregressive generative LLMs (GPT, LLaMA) β
β Cross-Attention β Text queries over image K/V β Multimodal fusion (Image captioning, stable diffusion) β
β Positional Encoding β Sinusoidal / Learned offsets β Injecting token order awareness into attention β
β FlashAttention β GPU SRAM tiling & fusing β Ultra-fast, low-memory training for long context LLMs β
ββββββββββββββββββββββββββββ΄βββββββββββββββββββββββββββββββ΄βββββββββββββββββββββββββββββββββββββββββββββββββββββββββ€
β TRANSFORMER ENGINEERING BEST PRACTICES: β
β 1. Use Multi-Head Attention to allow models to focus on different aspects of a sequence simultaneously. β
β 2. Implement causal masking for decoder-only LLMs to prevent information leakage from future tokens. β
β 3. Leverage FlashAttention kernels on CUDA hardware to dramatically reduce GPU memory consumption. β
ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ