π Converting Human Language into Numerical Vectors for Machine Learning
Machine learning algorithms and neural networks cannot process raw strings of text directly; they require structured numerical vectors. Text Preprocessing cleans raw corpora by removing noise, punctuation, and uninformative words. TF-IDF (Term Frequency-Inverse Document Frequency) evaluates statistical keyword importance for information retrieval, while Word2Vec maps words into dense semantic vector spaces where semantic relationships (such as king - man + woman = queen) are preserved. This masterclass covers 10 real-world engineering patterns for NLP feature engineering.
import re
def normalize_and_tokenize(raw_text):
print("=" * 65)
print("π€ EXAMPLE 1: TEXT LOWERCASE NORMALIZATION & TOKENIZATION")
print("=" * 65)
# Convert to lowercase
lower_text = raw_text.lower()
# Remove punctuation using regex
clean_text = re.sub(r'[^\w\s]', '', lower_text)
# Tokenize by splitting on whitespace
tokens = clean_text.split()
print(f"Raw Input Text : {raw_text}")
print(f"Cleaned Tokens : {tokens}")
print("=" * 65)
return tokens
if __name__ == "__main__":
sample = "Natural Language Processing (NLP) enables AI to understand human text!"
normalize_and_tokenize(sample)
def remove_stopwords(tokens):
print("=" * 65)
print("π§Ή EXAMPLE 2: STOPWORD FILTERING & NOISE REDUCTION")
print("=" * 65)
stopwords = {"the", "is", "at", "on", "and", "to", "a", "an", "in", "for", "of", "with"}
filtered_tokens = [word for word in tokens if word not in stopwords]
print(f"Original Tokens : {tokens}")
print(f"Filtered Tokens : {filtered_tokens}")
print("=" * 65)
return filtered_tokens
if __name__ == "__main__":
tokens_input = ['natural', 'language', 'processing', 'is', 'crucial', 'for', 'the', 'development', 'of', 'ai']
remove_stopwords(tokens_input)
def simulate_lemmatization():
print("=" * 65)
print("π± EXAMPLE 3: LEMMATIZATION ROOT FORM REDUCTION")
print("=" * 65)
# Simplified mock lemmatizer dictionary mapping inflections to base lemmas
lemma_dict = {
"running": "run", "runs": "run", "ran": "run",
"better": "good", "best": "good",
"computational": "compute", "computing": "compute",
"analyses": "analysis", "analyzing": "analyze"
}
input_words = ["running", "better", "analyses", "computing"]
lemmatized = [lemma_dict.get(w, w) for w in input_words]
print(f"Input Words : {input_words}")
print(f"Lemmatized Roots : {lemmatized}")
print("=" * 65)
if __name__ == "__main__":
simulate_lemmatization()
import numpy as np
def compute_tfidf_scratch():
print("=" * 65)
print("π EXAMPLE 4: TF-IDF STATISTICAL WEIGHTING CALCULATION")
print("=" * 65)
corpus = [
"machine learning and artificial intelligence",
"deep learning neural networks",
"artificial intelligence and deep learning"
]
# Vocabulary
vocab = ["machine", "learning", "artificial", "intelligence", "deep", "neural", "networks"]
# Document 1: "machine learning and artificial intelligence"
# Term Frequency in Doc 1
tf_doc1 = {"machine": 1, "learning": 1, "artificial": 1, "intelligence": 1, "deep": 0, "neural": 0, "networks": 0}
# IDF calculation: log(Total Docs / Docs containing term)
# 'learning' appears in doc 1, 2, 3 -> IDF = log(3/3) = 0
# 'machine' appears in doc 1 -> IDF = log(3/1) = 1.098
idf = {
"machine": np.log(3 / 1),
"learning": np.log(3 / 3),
"artificial": np.log(3 / 2),
"intelligence": np.log(3 / 2)
}
tfidf_vector = {word: tf_doc1[word] * idf.get(word, 0.0) for word in tf_doc1}
print(f"Corpus Size (N) : 3 documents")
print(f"IDF Weights : { {k: round(v, 3) for k, v in idf.items()} }")
print(f"TF-IDF Vector D1 : { {k: round(v, 3) for k, v in tfidf_vector.items() if v > 0} }")
print("=" * 65)
if __name__ == "__main__":
compute_tfidf_scratch()
from sklearn.feature_extraction.text import TfidfVectorizer
def run_sklearn_tfidf():
print("=" * 65)
print("π EXAMPLE 5: SCIKIT-LEARN TF-IDF SEARCH RETRIEVAL MATRIX")
print("=" * 65)
documents = [
"Python for machine learning and data science",
"Advanced deep learning with PyTorch and TensorFlow",
"Python programming language fundamentals"
]
vectorizer = TfidfVectorizer()
tfidf_matrix = vectorizer.fit_transform(documents)
print(f"Vocabulary Features : {vectorizer.get_feature_names_out().tolist()}")
print(f"TF-IDF Matrix Shape : {tfidf_matrix.shape}")
print(f"Matrix Representation:\n{tfidf_matrix.toarray().round(3)}")
print("=" * 65)
if __name__ == "__main__":
run_sklearn_tfidf()
import numpy as np
def simulate_cbow_prediction():
print("=" * 65)
print("π§ EXAMPLE 6: WORD2VEC CBOW CONTEXT WINDOW PREDICTION")
print("=" * 65)
# Context window: ["the", "cat", "on", "mat"] -> Target: "sat"
context_words = ["the", "cat", "on", "mat"]
target_word = "sat"
# Simulating embedding lookup vectors (dim = 4)
np.random.seed(42)
embedding_dim = 4
mock_embeddings = {
"the": np.array([0.1, 0.2, 0.3, 0.4]),
"cat": np.array([0.5, 0.6, 0.7, 0.8]),
"on": np.array([0.2, 0.1, 0.4, 0.3]),
"mat": np.array([0.6, 0.5, 0.8, 0.7])
}
# CBOW averages context vectors to predict target
context_vectors = [mock_embeddings[w] for w in context_words]
average_context_vector = np.mean(context_vectors, axis=0)
print(f"Context Words : {context_words}")
print(f"Target Word : {target_word}")
print(f"Averaged Context : {average_context_vector.tolist()}")
print(f"Objective : Maximize cosine similarity with embedding of '{target_word}'")
print("=" * 65)
if __name__ == "__main__":
simulate_cbow_prediction()
import numpy as np
def cosine_similarity(vec_a, vec_b):
dot_product = np.dot(vec_a, vec_b)
norm_a = np.linalg.norm(vec_a)
norm_b = np.linalg.norm(vec_b)
return dot_product / (norm_a * norm_b)
def run_skipgram_similarity():
print("=" * 65)
print("π EXAMPLE 7: WORD2VEC COSINE SIMILARITY COMPUTATION")
print("=" * 65)
# Mock 3D Word2Vec embeddings
embeddings = {
"king": np.array([0.8, 0.6, 0.2]),
"queen": np.array([0.79, 0.61, 0.22]),
"apple": np.array([-0.4, 0.1, 0.9])
}
sim_king_queen = cosine_similarity(embeddings["king"], embeddings["queen"])
sim_king_apple = cosine_similarity(embeddings["king"], embeddings["apple"])
print(f"Similarity('king', 'queen') : {sim_king_queen:.4f}")
print(f"Similarity('king', 'apple') : {sim_king_apple:.4f}")
print(f"Semantic Insight : High similarity for semantic peers, low for unrelated entities.")
print("=" * 65)
if __name__ == "__main__":
run_skipgram_similarity()
import numpy as np
def vector_analogy():
print("=" * 65)
print("π EXAMPLE 8: WORD2VEC VECTOR ANALOGY ARITHMETIC")
print("=" * 65)
# Mock 3D embeddings
v_king = np.array([1.0, 0.8, 0.2])
v_man = np.array([0.6, 0.7, 0.1])
v_woman = np.array([0.65, 0.85, 0.15])
v_queen = np.array([1.05, 0.95, 0.25])
# Analogy vector result: vector(king) - vector(man) + vector(woman)
result_vector = v_king - v_man + v_woman
# Compute distance to candidate 'queen'
diff = np.linalg.norm(result_vector - v_queen)
print(f"Vector Result (King - Man + Woman): {np.round(result_vector, 4).tolist()}")
print(f"Target Vector (Queen) : {np.round(v_queen, 4).tolist()}")
print(f"Euclidean Distance to Queen : {diff:.4f}")
print("=" * 65)
if __name__ == "__main__":
vector_analogy()
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.linear_model import LogisticRegression
def simulate_sentiment_pipeline():
print("=" * 65)
print("β€οΈ EXAMPLE 9: END-TO-END SENTIMENT CLASSIFICATION PIPELINE")
print("=" * 65)
corpus = [
"Absolutely fantastic product, loved the quality and fast shipping!",
"Terrible experience, broke after two days and customer support ignored me.",
"Decent item for the price, meets basic expectations."
]
labels = [1, 0, 1] # 1 = Positive, 0 = Negative
vectorizer = TfidfVectorizer(stop_words='english', lowercase=True)
X_train = vectorizer.fit_transform(corpus)
classifier = LogisticRegression()
classifier.fit(X_train, labels)
test_review = ["Horrible product, total waste of money."]
X_test = vectorizer.transform(test_review)
prediction = classifier.predict(X_test)
print(f"Test Review : {test_review[0]}")
print(f"Predicted Sentiment: {'Positive' if prediction[0] == 1 else 'Negative'}")
print(f"Feature Dimensions : {X_train.shape[1]} vocabulary terms")
print("=" * 65)
if __name__ == "__main__":
simulate_sentiment_pipeline()
NLP PREPROCESSING & VECTORIZATION REFERENCE MATRIX:
ββββββββββββββββββββββββββββ¬βββββββββββββββββββββββββββββββ¬βββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
β Technique / Tool β Core Mathematical Operation β Primary Engineering Application β
ββββββββββββββββββββββββββββΌββββββββββββββββββββββββββββββΌββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ€
β Lowercasing & Cleaning β Regex string substitution β Normalizing token variants & removing punctuation β
β Stopword Removal β Frequency-based filtering β Eliminating noise words ('the', 'is') to save dims β
β Lemmatization β Vocabulary morphological rootβ Reducing inflections ('running' -> 'run') β
β TF-IDF Vectorizer β TF(t,d) Γ log(N / DF(t)) β Keyword importance ranking & document search retrieval β
β Word2Vec (CBOW/Skip-gram)β Neural context prediction β Dense semantic embeddings & analogy arithmetic β
β FastText / GloVe β Subword / Global co-occ β Handling out-of-vocabulary words & rich semantics β
ββββββββββββββββββββββββββββ΄βββββββββββββββββββββββββββββββ΄βββββββββββββββββββββββββββββββββββββββββββββββββββββββββ€
β NLP ENGINEERING BEST PRACTICES: β
β 1. Always apply lowercasing, punctuation stripping, and lemmatization before TF-IDF or Word2Vec vectorization. β
β 2. Use TF-IDF for sparse, interpretable information retrieval and lexical keyword matching tasks. β
β 3. Use Word2Vec or Transformer embeddings (BERT, RoBERTa) when capturing deep semantic context is required. β
ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ