KNN and Naive Bayes Classifiers

import numpy as np
from collections import Counter

class KNNClassifier:
    def __init__(self, k=3):
        self.k = k
    
    def fit(self, X, y):
        """Store training data (no actual training)."""
        self.X_train = np.array(X)
        self.y_train = np.array(y)
    
    def euclidean_distance(self, x1, x2):
        return np.sqrt(np.sum((x1 - x2) ** 2))
    
    def predict_single(self, x):
        # Calculate distances to all training points
        distances = [self.euclidean_distance(x, x_train)
                     for x_train in self.X_train]
        
        # Get K nearest neighbor indices
        k_indices = np.argsort(distances)[:self.k]
        k_labels = self.y_train[k_indices]
        
        # Majority vote
        most_common = Counter(k_labels).most_common(1)
        return most_common[0][0]
    
    def predict(self, X):
        return np.array([self.predict_single(x) for x in X])
    
    def score(self, X, y):
        predictions = self.predict(X)
        return np.mean(predictions == y)

# Generate data
np.random.seed(42)
# Class 0: centered around (2, 2)
X0 = np.random.randn(30, 2) + np.array([2, 2])
# Class 1: centered around (6, 6)  
X1 = np.random.randn(30, 2) + np.array([6, 6])
X = np.vstack([X0, X1])
y = np.array([0]*30 + [1]*30)

# Shuffle
idx = np.random.permutation(60)
X, y = X[idx], y[idx]

# Split
X_train, X_test = X[:48], X[48:]
y_train, y_test = y[:48], y[48:]

# Train and evaluate with different K
for k in [1, 3, 5, 7, 11]:
    knn = KNNClassifier(k=k)
    knn.fit(X_train, y_train)
    train_acc = knn.score(X_train, y_train)
    test_acc = knn.score(X_test, y_test)
    print(f"K={k:2d}: Train={train_acc:.3f}, Test={test_acc:.3f}")

# Predict for a specific point
knn = KNNClassifier(k=3)
knn.fit(X_train, y_train)
point = np.array([4, 4])
pred = knn.predict_single(point)
print(f"\nPrediction for point {point}: Class {pred}")

import numpy as np
import matplotlib.pyplot as plt
from sklearn.neighbors import KNeighborsClassifier
from sklearn.model_selection import train_test_split, cross_val_score
from sklearn.preprocessing import StandardScaler
from sklearn.datasets import load_iris
from sklearn.metrics import classification_report

# Load iris dataset
iris = load_iris()
X, y = iris.data, iris.target

# Split and scale
X_train, X_test, y_train, y_test = train_test_split(
    X, y, test_size=0.2, random_state=42, stratify=y
)
scaler = StandardScaler()
X_train_s = scaler.fit_transform(X_train)
X_test_s = scaler.transform(X_test)

# Find optimal K using cross-validation
k_range = range(1, 31)
cv_scores = []
for k in k_range:
    knn = KNeighborsClassifier(n_neighbors=k)
    scores = cross_val_score(knn, X_train_s, y_train, cv=5)
    cv_scores.append(scores.mean())

best_k = k_range[np.argmax(cv_scores)]
print(f"Best K: {best_k} (CV accuracy: {max(cv_scores):.4f})")

# Plot K vs accuracy
plt.figure(figsize=(10, 5))
plt.plot(k_range, cv_scores, 'bo-', markersize=4)
plt.axvline(x=best_k, color='r', linestyle='--', label=f'Best K={best_k}')
plt.xlabel('K (Number of Neighbors)')
plt.ylabel('Cross-Validation Accuracy')
plt.title('KNN: Finding Optimal K')
plt.legend()
plt.grid(True, alpha=0.3)
plt.show()

# Train final model with best K
knn_best = KNeighborsClassifier(n_neighbors=best_k)
knn_best.fit(X_train_s, y_train)
y_pred = knn_best.predict(X_test_s)

print(f"\nTest Accuracy: {knn_best.score(X_test_s, y_test):.4f}")
print(f"\nClassification Report:")
print(classification_report(y_test, y_pred, target_names=iris.target_names))

import numpy as np
from sklearn.naive_bayes import GaussianNB, MultinomialNB, BernoulliNB
from sklearn.datasets import load_iris, fetch_20newsgroups
from sklearn.model_selection import train_test_split, cross_val_score
from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer
from sklearn.metrics import accuracy_score

# 1. Gaussian Naive Bayes (continuous features)
print("=== Gaussian Naive Bayes ===")
iris = load_iris()
X_train, X_test, y_train, y_test = train_test_split(
    iris.data, iris.target, test_size=0.3, random_state=42
)

gnb = GaussianNB()
gnb.fit(X_train, y_train)
print(f"Iris accuracy: {gnb.score(X_test, y_test):.4f}")
print(f"Class prior probabilities: {np.round(gnb.class_prior_, 3)}")
print(f"Feature means per class:")
for i, name in enumerate(iris.target_names):
    print(f"  {name}: {np.round(gnb.theta_[i], 2)}")

# 2. Multinomial Naive Bayes (word counts / TF-IDF)
print(f"\n=== Multinomial Naive Bayes ===")
texts = [
    "free money win lottery prize",
    "congratulations you won free gift",
    "claim your free reward now",
    "limited offer free discount sale",
    "meeting scheduled for tomorrow",
    "project deadline next week",
    "team lunch plans for friday",
    "quarterly report due monday",
    "free lunch tomorrow everyone invited",
    "submit your expense report free"
]
labels = [1, 1, 1, 1, 0, 0, 0, 0, 0, 0]  # 1=spam, 0=not spam

vectorizer = CountVectorizer()
X_text = vectorizer.fit_transform(texts)

mnb = MultinomialNB(alpha=1.0)  # alpha=1 is Laplace smoothing
mnb.fit(X_text, labels)

# Predict new emails
new_emails = ["free money for you", "meeting tomorrow at office"]
X_new = vectorizer.transform(new_emails)
predictions = mnb.predict(X_new)
for email, pred in zip(new_emails, predictions):
    print(f"  '{email}' -> {'Spam' if pred == 1 else 'Not Spam'}")

# 3. Bernoulli Naive Bayes (binary features)
print(f"\n=== Bernoulli Naive Bayes ===")
# Create binary features (word present or not)
X_binary = (X_text > 0).astype(int)
bnb = BernoulliNB(alpha=1.0)
bnb.fit(X_binary, labels)
print(f"Bernoulli NB accuracy on training: {bnb.score(X_binary, labels):.4f}")

import numpy as np
from sklearn.neighbors import KNeighborsClassifier
from sklearn.naive_bayes import GaussianNB
from sklearn.model_selection import cross_val_score
from sklearn.preprocessing import StandardScaler
from sklearn.datasets import make_classification, make_moons
import time

print("=== Dataset 1: Linear Pattern ===")
X1, y1 = make_classification(n_samples=500, n_features=10,
                              n_informative=5, random_state=42)
scaler = StandardScaler()
X1_s = scaler.fit_transform(X1)

knn = KNeighborsClassifier(n_neighbors=5)
nb = GaussianNB()

knn_scores = cross_val_score(knn, X1_s, y1, cv=5)
nb_scores = cross_val_score(nb, X1, y1, cv=5)  # NB doesn't need scaling

print(f"KNN (K=5): {knn_scores.mean():.3f} +/- {knn_scores.std():.3f}")
print(f"Naive Bayes: {nb_scores.mean():.3f} +/- {nb_scores.std():.3f}")

print(f"\n=== Dataset 2: Non-linear Pattern (Moons) ===")
X2, y2 = make_moons(n_samples=500, noise=0.3, random_state=42)
X2_s = scaler.fit_transform(X2)

knn_scores2 = cross_val_score(knn, X2_s, y2, cv=5)
nb_scores2 = cross_val_score(nb, X2, y2, cv=5)

print(f"KNN (K=5): {knn_scores2.mean():.3f} +/- {knn_scores2.std():.3f}")
print(f"Naive Bayes: {nb_scores2.mean():.3f} +/- {nb_scores2.std():.3f}")

print(f"\n=== Speed Comparison (10,000 samples) ===")
X3, y3 = make_classification(n_samples=10000, n_features=20, random_state=42)
X3_s = scaler.fit_transform(X3)

start = time.time()
knn.fit(X3_s[:8000], y3[:8000])
knn.predict(X3_s[8000:])
t_knn = time.time() - start

start = time.time()
nb.fit(X3[:8000], y3[:8000])
nb.predict(X3[8000:])
t_nb = time.time() - start

print(f"KNN: {t_knn:.4f}s")
print(f"NB:  {t_nb:.4f}s")
print(f"NB is {t_knn/t_nb:.1f}x faster")

print(f"\n=== Summary ===")
print("KNN: Better for non-linear patterns, needs scaling, slow at prediction")
print("NB:  Better for high-dimensional/text data, fast, no scaling needed")

import numpy as np
from sklearn.naive_bayes import MultinomialNB
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.model_selection import train_test_split
from sklearn.metrics import classification_report, confusion_matrix
from sklearn.pipeline import Pipeline

# Simulated email dataset
emails = [
    # Spam emails
    "Congratulations you have won a free iPhone click here",
    "Free money transfer to your account immediately",
    "Win a free vacation package limited time offer",
    "Claim your prize money now free gift card",
    "Earn money from home free registration no experience",
    "Free trial offer click below to claim your reward",
    "You are selected for a free laptop giveaway",
    "Urgent your account will be closed send password",
    "Make money fast guaranteed income work from home",
    "Limited offer discount sale free shipping today only",
    "Free casino bonus play now and win big prizes",
    "Congratulations your email won the lottery jackpot",
    "Double your income with this free investment guide",
    "Free credit score check no obligation required",
    "Get rich quick scheme guaranteed money back",
    # Not spam emails  
    "Meeting scheduled for tomorrow at 3 PM in conference room",
    "Please review the quarterly report and provide feedback",
    "Team lunch at the new restaurant this Friday",
    "Reminder project deadline is next Monday",
    "Can you send me the updated budget spreadsheet",
    "The client meeting has been rescheduled to Thursday",
    "Please find the attached invoice for last month",
    "Happy birthday Priya team celebration at noon",
    "We need to discuss the new feature requirements",
    "The server maintenance is scheduled for this weekend",
    "Updated the documentation please review before release",
    "Great job on the presentation yesterday everyone",
    "Could you help me debug this API endpoint issue",
    "The code review for the payment module is pending",
    "Welcome Rohan to the engineering team orientation Monday",
]
labels = [1]*15 + [0]*15  # 1=spam, 0=not spam

# Split data
X_train, X_test, y_train, y_test = train_test_split(
    emails, labels, test_size=0.3, random_state=42, stratify=labels
)

# Create pipeline: TF-IDF + Multinomial NB
pipeline = Pipeline([
    ('tfidf', TfidfVectorizer(stop_words='english', max_features=500)),
    ('nb', MultinomialNB(alpha=1.0))
])

pipeline.fit(X_train, y_train)
y_pred = pipeline.predict(X_test)

print(f"Accuracy: {pipeline.score(X_test, y_test):.4f}")
print(f"\nClassification Report:")
print(classification_report(y_test, y_pred, target_names=['Not Spam', 'Spam']))
print(f"Confusion Matrix:")
print(confusion_matrix(y_test, y_pred))

# Predict new emails
new_emails = [
    "Free money waiting for you click now",
    "Can we reschedule tomorrow's meeting to 4 PM",
    "Congratulations you won a free trip to Goa",
    "The deployment was successful no issues found"
]
print(f"\nNew email predictions:")
predictions = pipeline.predict(new_emails)
proba = pipeline.predict_proba(new_emails)
for email, pred, prob in zip(new_emails, predictions, proba):
    label = 'SPAM' if pred == 1 else 'NOT SPAM'
    conf = max(prob) * 100
    print(f"  [{label:8s}] ({conf:.0f}% confident) \"{email}\"")

import numpy as np
import matplotlib.pyplot as plt
from sklearn.neighbors import KNeighborsClassifier
from sklearn.datasets import make_classification
from sklearn.model_selection import cross_val_score
from sklearn.preprocessing import StandardScaler

# Custom distance calculation
def euclidean(a, b):
    return np.sqrt(np.sum((a - b) ** 2))

def manhattan(a, b):
    return np.sum(np.abs(a - b))

def chebyshev(a, b):
    return np.max(np.abs(a - b))

# Compare distances for two points
p1 = np.array([1, 2])
p2 = np.array([4, 6])

print("Distance from (1,2) to (4,6):")
print(f"  Euclidean:  {euclidean(p1, p2):.2f}")
print(f"  Manhattan:  {manhattan(p1, p2):.2f}")
print(f"  Chebyshev:  {chebyshev(p1, p2):.2f}")

# Compare metrics on a dataset
X, y = make_classification(n_samples=300, n_features=10,
                            n_informative=5, random_state=42)
scaler = StandardScaler()
X_s = scaler.fit_transform(X)

print(f"\nCross-validation accuracy by distance metric:")
metrics = ['euclidean', 'manhattan', 'chebyshev', 'minkowski']
for metric in metrics:
    knn = KNeighborsClassifier(n_neighbors=5, metric=metric)
    scores = cross_val_score(knn, X_s, y, cv=5)
    print(f"  {metric:12s}: {scores.mean():.3f} +/- {scores.std():.3f}")

# Visualize what K-neighborhoods look like
fig, axes = plt.subplots(1, 3, figsize=(15, 5))
np.random.seed(42)
X_vis = np.random.randn(50, 2)
query = np.array([0, 0])

for ax, metric_name, metric_func in zip(axes,
    ['Euclidean', 'Manhattan', 'Chebyshev'],
    [euclidean, manhattan, chebyshev]):
    
    distances = [metric_func(query, x) for x in X_vis]
    sorted_idx = np.argsort(distances)
    k_nearest = sorted_idx[:5]
    
    ax.scatter(X_vis[:, 0], X_vis[:, 1], c='gray', alpha=0.5, s=30)
    ax.scatter(X_vis[k_nearest, 0], X_vis[k_nearest, 1],
               c='red', s=100, zorder=5, label='5 Nearest')
    ax.scatter(*query, c='blue', s=200, marker='*', zorder=6, label='Query')
    ax.set_title(f'{metric_name} Distance')
    ax.legend()
    ax.grid(True, alpha=0.3)
    ax.set_aspect('equal')

plt.tight_layout()
plt.show()

import numpy as np

def bayes_theorem(prior, likelihood, evidence):
    """Calculate posterior probability using Bayes' theorem."""
    return (likelihood * prior) / evidence

# Example: Medical test for a disease
# - Disease prevalence (prior): 1% of population has the disease
# - Test sensitivity (P(positive|disease)): 99%
# - Test specificity (P(negative|no disease)): 95%

P_disease = 0.01          # Prior: P(Disease)
P_no_disease = 0.99       # Prior: P(No Disease)
P_pos_given_disease = 0.99  # Sensitivity: P(Positive | Disease)
P_neg_given_no_disease = 0.95  # Specificity: P(Negative | No Disease)
P_pos_given_no_disease = 1 - P_neg_given_no_disease  # False positive rate = 0.05

# P(Positive) = P(Pos|Disease)*P(Disease) + P(Pos|No Disease)*P(No Disease)
P_positive = P_pos_given_disease * P_disease + P_pos_given_no_disease * P_no_disease

# Bayes' Theorem: P(Disease | Positive)
P_disease_given_pos = bayes_theorem(P_disease, P_pos_given_disease, P_positive)

print("Medical Diagnosis Example")
print("=" * 50)
print(f"Disease prevalence:     {P_disease:.2%}")
print(f"Test sensitivity:       {P_pos_given_disease:.2%}")
print(f"Test specificity:       {P_neg_given_no_disease:.2%}")
print(f"False positive rate:    {P_pos_given_no_disease:.2%}")
print(f"\nP(Positive test):       {P_positive:.4f} ({P_positive:.2%})")
print(f"P(Disease | Positive):  {P_disease_given_pos:.4f} ({P_disease_given_pos:.2%})")

print(f"\nSurprising result: Even with a 99% accurate test,")
print(f"a positive result only means {P_disease_given_pos:.1%} chance of disease!")
print(f"This is because the disease is rare (1%), so most positives are false positives.")

# What if we test again (independent test)?
P_disease_after_second = bayes_theorem(
    P_disease_given_pos,  # New prior after first test
    P_pos_given_disease,  # Same sensitivity
    P_pos_given_disease * P_disease_given_pos + P_pos_given_no_disease * (1 - P_disease_given_pos)
)
print(f"\nAfter SECOND positive test: {P_disease_after_second:.1%} chance of disease")