Model Evaluation, Cross-Validation, and Hyperparameter Tuning

import numpy as np
from sklearn.metrics import (confusion_matrix, accuracy_score, precision_score,
                             recall_score, f1_score, classification_report)

# Simulate predictions for a disease screening test
np.random.seed(42)
n = 1000
y_true = np.array([1]*50 + [0]*950)  # 5% disease prevalence

# Model predictions (imperfect)
y_pred = y_true.copy()
# Introduce errors
y_pred[45:50] = 0   # 5 false negatives (miss 5 out of 50 sick patients)
y_pred[950:960] = 1  # 10 false positives (flag 10 healthy as sick)

cm = confusion_matrix(y_true, y_pred)
TN, FP, FN, TP = cm.ravel()

print("Confusion Matrix:")
print(f"  TN={TN}  FP={FP}")
print(f"  FN={FN}   TP={TP}")

print(f"\nMetrics (manual calculation):")
print(f"  Accuracy    = (TP+TN)/(Total) = ({TP}+{TN})/{n} = {(TP+TN)/n:.4f}")
print(f"  Precision   = TP/(TP+FP) = {TP}/({TP}+{FP}) = {TP/(TP+FP):.4f}")
print(f"  Recall      = TP/(TP+FN) = {TP}/({TP}+{FN}) = {TP/(TP+FN):.4f}")
print(f"  Specificity = TN/(TN+FP) = {TN}/({TN}+{FP}) = {TN/(TN+FP):.4f}")
f1 = 2*TP/(2*TP+FP+FN)
print(f"  F1 Score    = 2TP/(2TP+FP+FN) = {f1:.4f}")

print(f"\nMetrics (sklearn):")
print(f"  Accuracy:  {accuracy_score(y_true, y_pred):.4f}")
print(f"  Precision: {precision_score(y_true, y_pred):.4f}")
print(f"  Recall:    {recall_score(y_true, y_pred):.4f}")
print(f"  F1:        {f1_score(y_true, y_pred):.4f}")

print(f"\nClassification Report:")
print(classification_report(y_true, y_pred, target_names=['Healthy', 'Sick']))

print("Key Insight: Accuracy is 98.5% but we miss 10% of sick patients!")
print("In medical screening, Recall (90%) is more important than Accuracy.")

import numpy as np
import matplotlib.pyplot as plt
from sklearn.linear_model import LogisticRegression
from sklearn.ensemble import RandomForestClassifier
from sklearn.svm import SVC
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import StandardScaler
from sklearn.metrics import roc_curve, auc, roc_auc_score
from sklearn.datasets import make_classification

# Generate data
X, y = make_classification(n_samples=500, n_features=10,
                            n_informative=5, random_state=42)
X_train, X_test, y_train, y_test = train_test_split(
    X, y, test_size=0.3, random_state=42
)
scaler = StandardScaler()
X_train_s = scaler.fit_transform(X_train)
X_test_s = scaler.transform(X_test)

# Train multiple models
models = {
    'Logistic Regression': LogisticRegression(random_state=42),
    'Random Forest': RandomForestClassifier(n_estimators=100, random_state=42),
    'SVM': SVC(probability=True, random_state=42)
}

plt.figure(figsize=(10, 8))
colors = ['blue', 'green', 'red']

for (name, model), color in zip(models.items(), colors):
    model.fit(X_train_s, y_train)
    y_proba = model.predict_proba(X_test_s)[:, 1]
    
    fpr, tpr, thresholds = roc_curve(y_test, y_proba)
    roc_auc = auc(fpr, tpr)
    
    plt.plot(fpr, tpr, color=color, linewidth=2,
             label=f'{name} (AUC = {roc_auc:.3f})')

# Random classifier line
plt.plot([0, 1], [0, 1], 'k--', linewidth=1, label='Random (AUC = 0.500)')
plt.xlabel('False Positive Rate')
plt.ylabel('True Positive Rate')
plt.title('ROC Curves - Model Comparison')
plt.legend(loc='lower right')
plt.grid(True, alpha=0.3)
plt.show()

# Print AUC scores
print("AUC Scores:")
for name, model in models.items():
    y_proba = model.predict_proba(X_test_s)[:, 1]
    print(f"  {name:25s}: {roc_auc_score(y_test, y_proba):.4f}")

import numpy as np
from sklearn.model_selection import (KFold, StratifiedKFold, cross_val_score,
                                     cross_validate)
from sklearn.ensemble import RandomForestClassifier
from sklearn.datasets import make_classification

# Generate imbalanced data
X, y = make_classification(n_samples=500, n_features=10,
                            n_informative=5, weights=[0.9, 0.1],
                            random_state=42)
print(f"Class distribution: {np.bincount(y)}")
print(f"Class 1 ratio: {np.mean(y):.2%}")

model = RandomForestClassifier(n_estimators=100, random_state=42)

# Regular K-Fold
kf = KFold(n_splits=5, shuffle=True, random_state=42)
kf_scores = cross_val_score(model, X, y, cv=kf, scoring='f1')

# Stratified K-Fold (preserves class proportions)
skf = StratifiedKFold(n_splits=5, shuffle=True, random_state=42)
skf_scores = cross_val_score(model, X, y, cv=skf, scoring='f1')

print(f"\nRegular K-Fold F1 scores:    {np.round(kf_scores, 3)}")
print(f"  Mean: {kf_scores.mean():.4f} +/- {kf_scores.std():.4f}")
print(f"Stratified K-Fold F1 scores: {np.round(skf_scores, 3)}")
print(f"  Mean: {skf_scores.mean():.4f} +/- {skf_scores.std():.4f}")

# Show class distribution per fold
print(f"\nClass 1 count per fold (Stratified):")
for i, (train_idx, test_idx) in enumerate(skf.split(X, y)):
    n_pos = np.sum(y[test_idx])
    n_total = len(test_idx)
    print(f"  Fold {i+1}: {n_pos}/{n_total} positive ({n_pos/n_total:.2%})")

# cross_validate gives train scores too
results = cross_validate(model, X, y, cv=skf,
                         scoring=['accuracy', 'f1', 'roc_auc'],
                         return_train_score=True)

print(f"\nMultiple metrics (Stratified 5-Fold):")
for metric in ['accuracy', 'f1', 'roc_auc']:
    train = results[f'train_{metric}'].mean()
    test = results[f'test_{metric}'].mean()
    print(f"  {metric:10s}: Train={train:.4f}, Test={test:.4f}, Gap={train-test:.4f}")

import numpy as np
from sklearn.ensemble import RandomForestClassifier
from sklearn.model_selection import GridSearchCV, train_test_split
from sklearn.datasets import make_classification
from sklearn.metrics import classification_report
import time

# Generate data
X, y = make_classification(n_samples=1000, n_features=15,
                            n_informative=8, random_state=42)
X_train, X_test, y_train, y_test = train_test_split(
    X, y, test_size=0.2, random_state=42
)

# Define parameter grid
param_grid = {
    'n_estimators': [50, 100, 200],
    'max_depth': [3, 5, 10, None],
    'min_samples_split': [2, 5, 10],
    'min_samples_leaf': [1, 2, 4]
}

total_combinations = 1
for values in param_grid.values():
    total_combinations *= len(values)
print(f"Total combinations to try: {total_combinations}")
print(f"With 5-fold CV: {total_combinations * 5} model fits")

# Run GridSearchCV
start = time.time()
grid = GridSearchCV(
    RandomForestClassifier(random_state=42),
    param_grid,
    cv=5,
    scoring='f1',
    n_jobs=-1,      # Use all CPU cores
    verbose=0,
    return_train_score=True
)
grid.fit(X_train, y_train)
elapsed = time.time() - start

print(f"\nSearch completed in {elapsed:.1f} seconds")
print(f"\nBest parameters: {grid.best_params_}")
print(f"Best CV F1 score: {grid.best_score_:.4f}")

# Evaluate on test set
best_model = grid.best_estimator_
y_pred = best_model.predict(X_test)
print(f"\nTest set results with best model:")
print(classification_report(y_test, y_pred))

# Compare with default parameters
default_model = RandomForestClassifier(random_state=42)
default_model.fit(X_train, y_train)
print(f"Default model test accuracy: {default_model.score(X_test, y_test):.4f}")
print(f"Tuned model test accuracy:   {best_model.score(X_test, y_test):.4f}")
print(f"Improvement: {(best_model.score(X_test, y_test) - default_model.score(X_test, y_test)):.4f}")

import numpy as np
from sklearn.ensemble import RandomForestClassifier
from sklearn.model_selection import RandomizedSearchCV, train_test_split
from sklearn.datasets import make_classification
from scipy.stats import randint, uniform
import time

# Generate data
X, y = make_classification(n_samples=1000, n_features=15,
                            n_informative=8, random_state=42)
X_train, X_test, y_train, y_test = train_test_split(
    X, y, test_size=0.2, random_state=42
)

# Define parameter distributions (not a grid)
param_distributions = {
    'n_estimators': randint(50, 500),       # Random integer between 50 and 500
    'max_depth': [3, 5, 7, 10, 15, 20, None],
    'min_samples_split': randint(2, 20),
    'min_samples_leaf': randint(1, 10),
    'max_features': ['sqrt', 'log2', None],
    'bootstrap': [True, False]
}

# RandomizedSearchCV: sample 50 random combinations
start = time.time()
random_search = RandomizedSearchCV(
    RandomForestClassifier(random_state=42),
    param_distributions,
    n_iter=50,          # Try 50 random combinations
    cv=5,
    scoring='f1',
    n_jobs=-1,
    random_state=42,
    return_train_score=True
)
random_search.fit(X_train, y_train)
elapsed = time.time() - start

print(f"RandomizedSearchCV completed in {elapsed:.1f} seconds")
print(f"Combinations tried: 50")
print(f"\nBest parameters: {random_search.best_params_}")
print(f"Best CV F1 score: {random_search.best_score_:.4f}")
print(f"Test accuracy: {random_search.best_estimator_.score(X_test, y_test):.4f}")

# Show top 5 parameter combinations
import pandas as pd
results = pd.DataFrame(random_search.cv_results_)
top5 = results.nsmallest(5, 'rank_test_score')[[
    'params', 'mean_test_score', 'std_test_score', 'rank_test_score'
]]
print(f"\nTop 5 parameter combinations:")
for _, row in top5.iterrows():
    print(f"  Rank {row['rank_test_score']:.0f}: F1={row['mean_test_score']:.4f} "
          f"+/- {row['std_test_score']:.4f}")

import numpy as np
import matplotlib.pyplot as plt
from sklearn.model_selection import learning_curve, validation_curve
from sklearn.ensemble import RandomForestClassifier
from sklearn.datasets import make_classification

# Generate data
X, y = make_classification(n_samples=1000, n_features=10,
                            n_informative=5, random_state=42)

# 1. Learning Curve: score vs training set size
train_sizes, train_scores, test_scores = learning_curve(
    RandomForestClassifier(n_estimators=100, max_depth=5, random_state=42),
    X, y, cv=5,
    train_sizes=np.linspace(0.1, 1.0, 10),
    scoring='accuracy',
    n_jobs=-1
)

plt.figure(figsize=(14, 5))

plt.subplot(1, 2, 1)
plt.fill_between(train_sizes, train_scores.mean(axis=1) - train_scores.std(axis=1),
                 train_scores.mean(axis=1) + train_scores.std(axis=1), alpha=0.1, color='blue')
plt.fill_between(train_sizes, test_scores.mean(axis=1) - test_scores.std(axis=1),
                 test_scores.mean(axis=1) + test_scores.std(axis=1), alpha=0.1, color='red')
plt.plot(train_sizes, train_scores.mean(axis=1), 'b-o', label='Training Score')
plt.plot(train_sizes, test_scores.mean(axis=1), 'r-o', label='Cross-Validation Score')
plt.xlabel('Training Set Size')
plt.ylabel('Accuracy')
plt.title('Learning Curve')
plt.legend()
plt.grid(True, alpha=0.3)

# 2. Validation Curve: score vs hyperparameter
param_range = [1, 2, 3, 5, 7, 10, 15, 20, None]
train_scores_v, test_scores_v = validation_curve(
    RandomForestClassifier(n_estimators=100, random_state=42),
    X, y,
    param_name='max_depth',
    param_range=param_range[:-1],  # exclude None for plotting
    cv=5,
    scoring='accuracy',
    n_jobs=-1
)

plt.subplot(1, 2, 2)
plt.plot(param_range[:-1], train_scores_v.mean(axis=1), 'b-o', label='Training Score')
plt.plot(param_range[:-1], test_scores_v.mean(axis=1), 'r-o', label='Cross-Validation Score')
plt.xlabel('max_depth')
plt.ylabel('Accuracy')
plt.title('Validation Curve (max_depth)')
plt.legend()
plt.grid(True, alpha=0.3)

plt.tight_layout()
plt.show()

print("Learning Curve Interpretation:")
print("  - If both scores converge high: model is good")
print("  - If gap between train/test is large: overfitting")
print("  - If both scores are low: underfitting")
print("\nValidation Curve Interpretation:")
print("  - Left (low depth): both scores low -> underfitting")
print("  - Right (high depth): train high, test drops -> overfitting")
print("  - Sweet spot: where test score peaks")

import numpy as np
import pandas as pd
from sklearn.ensemble import RandomForestClassifier
from sklearn.model_selection import (GridSearchCV, train_test_split,
                                     StratifiedKFold)
from sklearn.preprocessing import StandardScaler
from sklearn.metrics import (classification_report, roc_auc_score,
                             confusion_matrix)
from sklearn.datasets import make_classification

# Generate realistic dataset
X, y = make_classification(
    n_samples=2000, n_features=20, n_informative=10,
    n_redundant=5, weights=[0.7, 0.3], random_state=42
)

print(f"Dataset: {X.shape[0]} samples, {X.shape[1]} features")
print(f"Class distribution: {np.bincount(y)}")

# Split into train+val and final test
X_trainval, X_test, y_trainval, y_test = train_test_split(
    X, y, test_size=0.15, random_state=42, stratify=y
)

# Step 1: Baseline model
baseline = RandomForestClassifier(random_state=42)
baseline.fit(X_trainval, y_trainval)
print(f"\nBaseline test accuracy: {baseline.score(X_test, y_test):.4f}")
print(f"Baseline test AUC: {roc_auc_score(y_test, baseline.predict_proba(X_test)[:, 1]):.4f}")

# Step 2: GridSearchCV tuning
param_grid = {
    'n_estimators': [100, 200],
    'max_depth': [5, 10, 15],
    'min_samples_split': [2, 5],
    'min_samples_leaf': [1, 2],
    'max_features': ['sqrt', 'log2']
}

cv = StratifiedKFold(n_splits=5, shuffle=True, random_state=42)

grid = GridSearchCV(
    RandomForestClassifier(random_state=42),
    param_grid,
    cv=cv,
    scoring='roc_auc',
    n_jobs=-1,
    return_train_score=True
)
grid.fit(X_trainval, y_trainval)

print(f"\nBest CV AUC: {grid.best_score_:.4f}")
print(f"Best params: {grid.best_params_}")

# Step 3: Final evaluation on test set
best_model = grid.best_estimator_
y_pred = best_model.predict(X_test)
y_proba = best_model.predict_proba(X_test)[:, 1]

print(f"\nFinal Test Results:")
print(f"Accuracy: {best_model.score(X_test, y_test):.4f}")
print(f"AUC: {roc_auc_score(y_test, y_proba):.4f}")
print(f"\nClassification Report:")
print(classification_report(y_test, y_pred))

print(f"\nConfusion Matrix:")
print(confusion_matrix(y_test, y_pred))

# Step 4: Check for overfitting
cv_train = grid.cv_results_['mean_train_score'][grid.best_index_]
cv_test = grid.cv_results_['mean_test_score'][grid.best_index_]
print(f"\nOverfitting check:")
print(f"  CV Train AUC: {cv_train:.4f}")
print(f"  CV Test AUC:  {cv_test:.4f}")
print(f"  Gap: {cv_train - cv_test:.4f} {'(OK)' if cv_train - cv_test < 0.05 else '(might overfit)'}")