S096 · Perceptron & Activation Functions

"""perceptron_lab.py — the perceptron from first principles.
 
Trains on:
  1. AND — should reach 100% accuracy quickly.
  2. OR  — same.
  3. XOR — will bounce around near 50% forever. This is the whole point.
 
Then swaps in a variety of activation functions and shows what each output looks like.
"""
from __future__ import annotations
import numpy as np
 
RNG = np.random.default_rng(42)
 
 
class Perceptron:
    """Rosenblatt perceptron. sign activation, perceptron learning rule."""
    def __init__(self, n_features: int, lr: float = 0.1) -> None:
        self.w = RNG.normal(0, 0.5, size=n_features)
        self.b = 0.0
        self.lr = lr
 
    def predict(self, X: np.ndarray) -> np.ndarray:
        z = X @ self.w + self.b
        return np.where(z >= 0, 1, 0)
 
    def fit(self, X: np.ndarray, y: np.ndarray, epochs: int = 20) -> list[float]:
        """Returns accuracy per epoch."""
        history = []
        for _ in range(epochs):
            for xi, yi in zip(X, y, strict=False):
                y_hat = 1 if (xi @ self.w + self.b) >= 0 else 0
                err = yi - y_hat
                self.w += self.lr * err * xi
                self.b += self.lr * err
            acc = float((self.predict(X) == y).mean())
            history.append(acc)
        return history
 
 
def and_data():
    X = np.array([[0, 0], [0, 1], [1, 0], [1, 1]], dtype=float)
    y = np.array([0, 0, 0, 1])
    return X, y
 
 
def or_data():
    X = np.array([[0, 0], [0, 1], [1, 0], [1, 1]], dtype=float)
    y = np.array([0, 1, 1, 1])
    return X, y
 
 
def xor_data():
    X = np.array([[0, 0], [0, 1], [1, 0], [1, 1]], dtype=float)
    y = np.array([0, 1, 1, 0])
    return X, y
 
 
# --- activation function zoo ---
def sigmoid(z):  return 1.0 / (1.0 + np.exp(-z))
def tanh(z):     return np.tanh(z)
def relu(z):     return np.maximum(0.0, z)
def leaky_relu(z, a=0.01): return np.where(z > 0, z, a * z)
def gelu(z):
    # Exact GELU: z * Φ(z), Φ = standard normal CDF
    from math import sqrt
    return 0.5 * z * (1.0 + np.tanh(sqrt(2.0 / np.pi) * (z + 0.044715 * z ** 3)))
def silu(z):     return z * sigmoid(z)
 
 
def demo_activations() -> None:
    z = np.linspace(-3, 3, 7)
    print("\nActivation values on z =", z.round(2).tolist())
    for name, fn in [("sigmoid", sigmoid), ("tanh", tanh), ("relu", relu),
                     ("leaky_relu", leaky_relu), ("gelu", gelu), ("silu", silu)]:
        vals = fn(z)
        print(f"  {name:<10s}: {np.round(vals, 3).tolist()}")
 
 
def train_and_report(name: str, data_fn) -> None:
    X, y = data_fn()
    p = Perceptron(n_features=2, lr=0.1)
    hist = p.fit(X, y, epochs=30)
    print(f"\n{name}: final accuracy = {hist[-1]:.2f}  (final w={p.w.round(2)}, b={p.b:.2f})")
    print(f"  accuracy per epoch (first 10): {[round(h,2) for h in hist[:10]]}")
 
 
if __name__ == "__main__":
    print("=== single-layer perceptron on 3 problems ===")
    train_and_report("AND", and_data)
    train_and_report("OR",  or_data)
    train_and_report("XOR", xor_data)
 
    demo_activations()
 
    print("\n=== dead ReLU illustration ===")
    # A neuron whose pre-activation is very negative for all inputs → gradient = 0 forever
    x = np.array([1.0, 2.0, 3.0])
    w = np.array([-1.0, -1.0, -1.0])
    b = -10.0
    z = x @ w + b
    print(f"  z = {z:.2f}, relu(z) = {relu(z):.2f}, gradient of relu at z<0 = 0")
    print("  If z stays below zero for every input the neuron will never learn — the dead ReLU problem.")