S092 · Evaluation Metrics — P/R/F1/ROC/PR/AUC

"""metrics_lab.py — classification metrics from first principles + sklearn.
 
Runs two experiments:
  1. Breast cancer (roughly balanced) — ROC-AUC is fine.
  2. Synthetic fraud (1% positives) — ROC-AUC lies; PR-AUC tells the truth.
"""
from __future__ import annotations
import numpy as np
from sklearn.datasets import load_breast_cancer, make_classification
from sklearn.linear_model import LogisticRegression
from sklearn.model_selection import train_test_split
from sklearn.metrics import (
    confusion_matrix,
    precision_score,
    recall_score,
    f1_score,
    accuracy_score,
    roc_auc_score,
    average_precision_score,
    precision_recall_curve,
    roc_curve,
    classification_report,
)
 
RNG = 42
 
 
def report(name: str, y_true, y_pred, y_prob) -> None:
    """Print a compact metrics report — everything a code review needs."""
    tn, fp, fn, tp = confusion_matrix(y_true, y_pred).ravel()
    print(f"\n=== {name} ===")
    print(f"  n = {len(y_true):>5}  positives = {int(y_true.sum()):>4}"
          f"  base rate = {y_true.mean():.3f}")
    print(f"  TP={tp:>4}  FP={fp:>4}  FN={fn:>4}  TN={tn:>4}")
    print(f"  accuracy   = {accuracy_score(y_true, y_pred):.3f}")
    print(f"  precision  = {precision_score(y_true, y_pred, zero_division=0):.3f}")
    print(f"  recall     = {recall_score(y_true, y_pred, zero_division=0):.3f}")
    print(f"  f1         = {f1_score(y_true, y_pred, zero_division=0):.3f}")
    print(f"  ROC-AUC    = {roc_auc_score(y_true, y_prob):.3f}")
    print(f"  PR-AUC     = {average_precision_score(y_true, y_prob):.3f}")
 
 
def tune_for_recall(y_true, y_prob, target_recall: float = 0.95) -> float:
    """Return the threshold that hits at least target_recall with best precision."""
    prec, rec, thr = precision_recall_curve(y_true, y_prob)
    # last elements of prec/rec correspond to threshold=+inf, no threshold there
    mask = rec[:-1] >= target_recall
    if not mask.any():
        return 0.0
    # among all thresholds that meet recall, pick the one with highest precision
    idx = np.argmax(prec[:-1][mask])
    return float(thr[mask][idx])
 
 
def experiment_balanced() -> None:
    X, y = load_breast_cancer(return_X_y=True)
    Xtr, Xte, ytr, yte = train_test_split(X, y, test_size=0.3, random_state=RNG, stratify=y)
    clf = LogisticRegression(max_iter=5000).fit(Xtr, ytr)
    prob = clf.predict_proba(Xte)[:, 1]
    pred = (prob >= 0.5).astype(int)
    report("Breast cancer — threshold 0.5", yte, pred, prob)
 
 
def experiment_imbalanced() -> None:
    X, y = make_classification(
        n_samples=10_000, n_features=20, weights=[0.99, 0.01],
        random_state=RNG, n_informative=5,
    )
    Xtr, Xte, ytr, yte = train_test_split(X, y, test_size=0.3, random_state=RNG, stratify=y)
    clf = LogisticRegression(max_iter=5000, class_weight=None).fit(Xtr, ytr)
    prob = clf.predict_proba(Xte)[:, 1]
 
    # A: default threshold — will look great on accuracy, awful on recall
    pred_default = (prob >= 0.5).astype(int)
    report("Synthetic fraud — threshold 0.5 (naive)", yte, pred_default, prob)
 
    # B: threshold tuned to reach 95% recall — see precision collapse
    thr = tune_for_recall(yte, prob, target_recall=0.95)
    pred_recall = (prob >= thr).astype(int)
    print(f"\n  → threshold moved to {thr:.3f} to hit recall ≥ 0.95")
    report(f"Synthetic fraud — threshold {thr:.3f} (recall-tuned)", yte, pred_recall, prob)
 
    # Full sklearn text report for the recall-tuned model
    print("\n" + classification_report(yte, pred_recall, digits=3))
 
 
if __name__ == "__main__":
    experiment_balanced()
    experiment_imbalanced()