Natural Language Processing (NLP) Fundamentals

import re
import nltk
from nltk.tokenize import word_tokenize
from nltk.corpus import stopwords
from nltk.stem import PorterStemmer, WordNetLemmatizer

nltk.download('punkt_tab', quiet=True)
nltk.download('stopwords', quiet=True)
nltk.download('wordnet', quiet=True)

def preprocess_text(text, method='lemmatize'):
    """Complete text preprocessing pipeline."""
    # Step 1: Lowercase
    text = text.lower()
    
    # Step 2: Remove URLs
    text = re.sub(r'http\S+|www\S+', '', text)
    
    # Step 3: Remove HTML tags
    text = re.sub(r'<.*?>', '', text)
    
    # Step 4: Remove punctuation and numbers
    text = re.sub(r'[^a-z\s]', '', text)
    
    # Step 5: Tokenize
    tokens = word_tokenize(text)
    
    # Step 6: Remove stop words
    stop_words = set(stopwords.words('english'))
    tokens = [w for w in tokens if w not in stop_words and len(w) > 2]
    
    # Step 7: Stemming or Lemmatization
    if method == 'stem':
        stemmer = PorterStemmer()
        tokens = [stemmer.stem(w) for w in tokens]
    else:
        lemmatizer = WordNetLemmatizer()
        tokens = [lemmatizer.lemmatize(w) for w in tokens]
    
    return ' '.join(tokens)

# Test on sample texts
texts = [
    "I absolutely LOVED this movie! It was amazing and the acting was brilliant!!!",
    "The customer service was terrible. I waited 3 hours and nobody helped me.",
    "Dr. Sharma presented the findings at the conference in Mumbai on 15th March.",
    "Check out this link: https://example.com for more details <b>NOW</b>!"
]

print(f"{'Original':<60} {'Preprocessed'}")
print("-" * 100)
for text in texts:
    cleaned = preprocess_text(text)
    print(f"{text[:57]+'...' if len(text)>57 else text:<60} {cleaned}")

from sklearn.feature_extraction.text import TfidfVectorizer, CountVectorizer
from sklearn.naive_bayes import MultinomialNB
from sklearn.svm import LinearSVC
from sklearn.linear_model import LogisticRegression
from sklearn.model_selection import train_test_split
from sklearn.metrics import classification_report, accuracy_score
from sklearn.pipeline import Pipeline
import numpy as np

# Sample dataset: product reviews
texts = [
    "This product is absolutely wonderful, love it",
    "Terrible quality, broke after one day",
    "Best purchase I ever made, highly recommend",
    "Waste of money, does not work at all",
    "Great value for the price, works perfectly",
    "Horrible experience, returning immediately",
    "Excellent craftsmanship and fast delivery",
    "Poor design, cheap materials used",
    "Amazing quality, exceeded my expectations",
    "Defective product, customer service unhelpful",
    "Fantastic item, my kids love it",
    "Disappointing, nothing like the description",
    "Superior quality, will buy again",
    "Rubbish, complete waste of time",
    "Delightful product with premium feel",
    "Awful, worst product I have bought"
]
labels = [1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0]  # 1=positive, 0=negative

# Split data
X_train, X_test, y_train, y_test = train_test_split(
    texts, labels, test_size=0.25, random_state=42
)

# Compare BoW vs TF-IDF with different classifiers
pipelines = {
    'BoW + NaiveBayes': Pipeline([
        ('vectorizer', CountVectorizer()),
        ('clf', MultinomialNB())
    ]),
    'TF-IDF + NaiveBayes': Pipeline([
        ('vectorizer', TfidfVectorizer()),
        ('clf', MultinomialNB())
    ]),
    'TF-IDF + SVM': Pipeline([
        ('vectorizer', TfidfVectorizer(ngram_range=(1, 2))),
        ('clf', LinearSVC())
    ]),
    'TF-IDF + LogReg': Pipeline([
        ('vectorizer', TfidfVectorizer(ngram_range=(1, 2))),
        ('clf', LogisticRegression(max_iter=1000))
    ])
}

print(f"{'Pipeline':<30} {'Train Acc':>10} {'Test Acc':>10}")
print("-" * 52)
for name, pipeline in pipelines.items():
    pipeline.fit(X_train, y_train)
    train_acc = accuracy_score(y_train, pipeline.predict(X_train))
    test_acc = accuracy_score(y_test, pipeline.predict(X_test))
    print(f"{name:<30} {train_acc:>10.4f} {test_acc:>10.4f}")

# Detailed report for best model
best = pipelines['TF-IDF + SVM']
print("\nDetailed Classification Report (TF-IDF + SVM):")
print(classification_report(y_test, best.predict(X_test),
                           target_names=['Negative', 'Positive']))

import tensorflow as tf
from tensorflow.keras.layers import Embedding, LSTM, Dense, Dropout, Bidirectional
from tensorflow.keras.preprocessing.text import Tokenizer
from tensorflow.keras.preprocessing.sequence import pad_sequences
from tensorflow.keras.callbacks import EarlyStopping
import numpy as np
from sklearn.model_selection import train_test_split

# Sample SMS messages (ham=0, spam=1)
messages = [
    ("Hey, are you coming to the party tonight?", 0),
    ("WINNER!! You have won a $1000 gift card! Call now!", 1),
    ("Can you pick up milk on your way home?", 0),
    ("FREE entry to win $5000 cash prize! Text WIN to 80085", 1),
    ("Meeting rescheduled to 3pm tomorrow", 0),
    ("Congratulations! You have been selected for a free iPhone", 1),
    ("What time does the movie start?", 0),
    ("URGENT: Your account has been compromised. Click here now", 1),
    ("Happy birthday! Hope you have a great day", 0),
    ("Claim your free vacation package! Limited time offer", 1),
    ("I will be late by 15 minutes, sorry", 0),
    ("You have won a luxury cruise! Reply YES to confirm", 1),
    ("Can you send me the notes from today's lecture?", 0),
    ("EXCLUSIVE deal! Buy 1 get 3 free, hurry limited stock", 1),
    ("Thanks for lunch, it was really nice", 0),
    ("Your loan has been approved! Low interest rates guaranteed", 1),
    ("See you at the gym at 6pm", 0),
    ("Act NOW! Double your money in 24 hours, guaranteed", 1),
    ("Mom asked if you are coming for dinner on Sunday", 0),
    ("Free ringtones! Download now at www.freeringtones.com", 1),
    ("The assignment deadline is next Friday", 0),
    ("PRIZE DRAW! You are the lucky winner, claim now", 1),
    ("Let me know when you reach home", 0),
    ("Make money from home! Earn $500 daily with no effort", 1)
]

texts = [m[0] for m in messages]
labels = np.array([m[1] for m in messages])

print(f"Total messages: {len(texts)}")
print(f"Ham: {sum(labels==0)}, Spam: {sum(labels==1)}")

# Split
X_train, X_test, y_train, y_test = train_test_split(
    texts, labels, test_size=0.25, random_state=42, stratify=labels
)

# Tokenize
VOCAB_SIZE = 2000
MAXLEN = 50

tokenizer = Tokenizer(num_words=VOCAB_SIZE, oov_token='<OOV>')
tokenizer.fit_on_texts(X_train)

X_train_seq = pad_sequences(tokenizer.texts_to_sequences(X_train), maxlen=MAXLEN, padding='post')
X_test_seq = pad_sequences(tokenizer.texts_to_sequences(X_test), maxlen=MAXLEN, padding='post')

print(f"\nVocabulary size: {len(tokenizer.word_index)}")
print(f"Train shape: {X_train_seq.shape}")
print(f"Test shape: {X_test_seq.shape}")

# Build model
model = tf.keras.Sequential([
    Embedding(VOCAB_SIZE, 32, input_length=MAXLEN),
    Bidirectional(LSTM(32, dropout=0.3, recurrent_dropout=0.3)),
    Dense(16, activation='relu'),
    Dropout(0.5),
    Dense(1, activation='sigmoid')
])

model.compile(optimizer='adam', loss='binary_crossentropy', metrics=['accuracy'])
model.summary()

# Train
history = model.fit(
    X_train_seq, y_train,
    epochs=30,
    batch_size=8,
    validation_split=0.2,
    callbacks=[EarlyStopping(monitor='val_loss', patience=5, restore_best_weights=True)],
    verbose=1
)

# Evaluate
test_loss, test_acc = model.evaluate(X_test_seq, y_test, verbose=0)
print(f"\nTest accuracy: {test_acc:.4f}")

# Test on new messages
new_messages = [
    "Hey, want to grab coffee tomorrow?",
    "You have won $10,000! Claim your prize now!",
    "Can you review my code before the meeting?",
    "FREE FREE FREE! Call now for exclusive deal!"
]

new_seq = pad_sequences(tokenizer.texts_to_sequences(new_messages), maxlen=MAXLEN, padding='post')
predictions = model.predict(new_seq)

print("\n--- Spam Detection Results ---")
for msg, pred in zip(new_messages, predictions):
    label = "SPAM" if pred > 0.5 else "HAM"
    confidence = pred[0] if pred > 0.5 else 1 - pred[0]
    print(f"[{label}] ({confidence:.1%}) {msg}")

from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer
import numpy as np

documents = [
    "machine learning is great for data analysis",
    "deep learning neural networks are powerful",
    "data science involves machine learning and statistics",
    "neural networks learn patterns from data"
]

# Bag of Words
bow = CountVectorizer()
bow_matrix = bow.fit_transform(documents)

print("=== Bag of Words ===")
print(f"Vocabulary: {bow.get_feature_names_out()}")
print(f"Matrix shape: {bow_matrix.shape}")
print("\nDocument-Term Matrix (counts):")
for i, doc in enumerate(documents):
    nonzero = [(bow.get_feature_names_out()[j], bow_matrix[i, j])
               for j in bow_matrix[i].nonzero()[1]]
    print(f"  Doc {i+1}: {dict(nonzero)}")

# TF-IDF
tfidf = TfidfVectorizer()
tfidf_matrix = tfidf.fit_transform(documents)

print("\n=== TF-IDF ===")
print("\nDocument-Term Matrix (TF-IDF scores):")
for i, doc in enumerate(documents):
    nonzero = [(tfidf.get_feature_names_out()[j], f"{tfidf_matrix[i, j]:.3f}")
               for j in tfidf_matrix[i].nonzero()[1]]
    sorted_terms = sorted(nonzero, key=lambda x: float(x[1]), reverse=True)
    print(f"  Doc {i+1} (top 4): {dict(sorted_terms[:4])}")

# Key difference: word importance
print("\n=== Key Difference ===")
word = 'data'
bow_idx = list(bow.get_feature_names_out()).index(word)
tfidf_idx = list(tfidf.get_feature_names_out()).index(word)

print(f"Word '{word}' appears in 3/4 documents:")
for i in range(len(documents)):
    print(f"  Doc {i+1}: BoW={bow_matrix[i, bow_idx]}, TF-IDF={tfidf_matrix[i, tfidf_idx]:.3f}")
print(f"\nTF-IDF gives '{word}' LOWER scores because it appears in many documents.")
print("Words unique to a document get HIGHER TF-IDF scores.")

from gensim.models import Word2Vec
import numpy as np

# Training corpus (in practice, use millions of sentences)
corpus = [
    ['machine', 'learning', 'is', 'a', 'subset', 'of', 'artificial', 'intelligence'],
    ['deep', 'learning', 'uses', 'neural', 'networks', 'with', 'many', 'layers'],
    ['natural', 'language', 'processing', 'handles', 'text', 'data'],
    ['computer', 'vision', 'processes', 'image', 'data', 'using', 'cnn'],
    ['python', 'is', 'the', 'most', 'popular', 'language', 'for', 'data', 'science'],
    ['tensorflow', 'and', 'pytorch', 'are', 'deep', 'learning', 'frameworks'],
    ['supervised', 'learning', 'requires', 'labeled', 'training', 'data'],
    ['unsupervised', 'learning', 'finds', 'patterns', 'without', 'labels'],
    ['reinforcement', 'learning', 'uses', 'rewards', 'and', 'penalties'],
    ['neural', 'networks', 'learn', 'features', 'from', 'data', 'automatically']
]

# Train Word2Vec model
model = Word2Vec(
    sentences=corpus,
    vector_size=50,     # Embedding dimension
    window=3,           # Context window size
    min_count=1,        # Include all words
    sg=1,               # 1=Skip-gram, 0=CBOW
    epochs=100          # Training iterations
)

print(f"Vocabulary size: {len(model.wv)}")
print(f"Embedding dimension: {model.wv.vector_size}")

# Get word vector
vector = model.wv['learning']
print(f"\nVector for 'learning' (first 10 dims): {vector[:10].round(3)}")

# Find similar words
print("\nMost similar to 'learning':")
for word, score in model.wv.most_similar('learning', topn=5):
    print(f"  {word}: {score:.4f}")

print("\nMost similar to 'data':")
for word, score in model.wv.most_similar('data', topn=5):
    print(f"  {word}: {score:.4f}")

# Word similarity
pairs = [('deep', 'learning'), ('python', 'language'), ('neural', 'networks')]
print("\nWord pair similarities:")
for w1, w2 in pairs:
    sim = model.wv.similarity(w1, w2)
    print(f"  {w1} <-> {w2}: {sim:.4f}")

print("\nNote: With a larger corpus (millions of sentences),")
print("these similarities would be much more meaningful.")

import tensorflow as tf
from tensorflow.keras.layers import Embedding, LSTM, Dense, Dropout
from tensorflow.keras.preprocessing.text import Tokenizer
from tensorflow.keras.preprocessing.sequence import pad_sequences
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.svm import LinearSVC
from sklearn.metrics import accuracy_score
import numpy as np

# Load IMDB dataset
(X_train_idx, y_train), (X_test_idx, y_test) = tf.keras.datasets.imdb.load_data(num_words=10000)

# Decode back to text for sklearn
word_index = tf.keras.datasets.imdb.get_word_index()
reverse_index = {v+3: k for k, v in word_index.items()}
reverse_index[0] = '<PAD>'
reverse_index[1] = '<START>'
reverse_index[2] = '<UNK>'

def decode_review(encoded):
    return ' '.join([reverse_index.get(i, '?') for i in encoded])

# Prepare text versions for sklearn
X_train_text = [decode_review(x) for x in X_train_idx[:5000]]  # Subset for speed
X_test_text = [decode_review(x) for x in X_test_idx[:2000]]
y_train_sub = y_train[:5000]
y_test_sub = y_test[:2000]

# === Approach 1: TF-IDF + SVM ===
print("=== Approach 1: TF-IDF + SVM ===")
tfidf = TfidfVectorizer(max_features=10000, ngram_range=(1, 2))
X_train_tfidf = tfidf.fit_transform(X_train_text)
X_test_tfidf = tfidf.transform(X_test_text)

svm = LinearSVC(max_iter=2000)
svm.fit(X_train_tfidf, y_train_sub)
svm_acc = accuracy_score(y_test_sub, svm.predict(X_test_tfidf))
print(f"SVM Test Accuracy: {svm_acc:.4f}")

# === Approach 2: Embedding + LSTM ===
print("\n=== Approach 2: Embedding + LSTM ===")
MAXLEN = 200
X_train_pad = pad_sequences(X_train_idx[:5000], maxlen=MAXLEN)
X_test_pad = pad_sequences(X_test_idx[:2000], maxlen=MAXLEN)

lstm_model = tf.keras.Sequential([
    Embedding(10000, 64, input_length=MAXLEN),
    LSTM(64, dropout=0.2),
    Dense(32, activation='relu'),
    Dropout(0.5),
    Dense(1, activation='sigmoid')
])

lstm_model.compile(optimizer='adam', loss='binary_crossentropy', metrics=['accuracy'])
lstm_model.fit(X_train_pad, y_train_sub, epochs=3, batch_size=64,
               validation_split=0.2, verbose=1)

lstm_acc = lstm_model.evaluate(X_test_pad, y_test_sub, verbose=0)[1]
print(f"LSTM Test Accuracy: {lstm_acc:.4f}")

# Summary
print(f"\n{'Method':<25} {'Accuracy':>10}")
print("-" * 37)
print(f"{'TF-IDF + SVM':<25} {svm_acc:>10.4f}")
print(f"{'Embedding + LSTM':<25} {lstm_acc:>10.4f}")
print("\nTF-IDF + SVM: Fast, no GPU needed, good for smaller datasets")
print("Embedding + LSTM: Captures word order, better for complex patterns")