S122 · LLM Evaluation — LLM-as-Judge, RAGAS, Golden Sets

#!/usr/bin/env python3
# eval_judge.py — a calibrated LLM-as-Judge for answer faithfulness.
# pip install openai pydantic
import json
import os
import random
import statistics
from typing import Literal
 
from openai import OpenAI
from pydantic import BaseModel, Field
 
client = OpenAI()
 
# ---------- 1. golden set with human labels ----------
GOLDEN = [
    {
        "context": "The Eiffel Tower was built in 1889 for the World's Fair. It is 330 meters tall.",
        "question": "When was the Eiffel Tower built and how tall is it?",
        "answer": "It was built in 1889 and is 330 meters tall.",
        "human_faithful": True,
    },
    {
        "context": "The Eiffel Tower was built in 1889 for the World's Fair. It is 330 meters tall.",
        "question": "When was the Eiffel Tower built and how tall is it?",
        "answer": "It was built in 1887 by Napoleon and is 400 meters tall.",
        "human_faithful": False,
    },
    {
        "context": "Python 3.12 was released in October 2023 and introduced PEP 695 type parameter syntax.",
        "question": "What did Python 3.12 introduce?",
        "answer": "PEP 695 type parameter syntax, released October 2023.",
        "human_faithful": True,
    },
    {
        "context": "Python 3.12 was released in October 2023 and introduced PEP 695 type parameter syntax.",
        "question": "What did Python 3.12 introduce?",
        "answer": "Match statements and the walrus operator.",
        "human_faithful": False,
    },
    # ...in a real system, 50-500 rows.
]
 
# ---------- 2. judge schema ----------
class Verdict(BaseModel):
    faithful: bool
    unsupported_claims: list[str] = Field(default_factory=list)
    score: Literal[1, 2, 3, 4, 5]
    reasoning: str
 
JUDGE_SYSTEM = """You are a strict fact-checker. You are shown a CONTEXT and an ANSWER.
Decide whether EVERY factual claim in the ANSWER is directly supported by the CONTEXT.
 
Rules:
- Do NOT use outside knowledge. If a claim is true in the real world but not in the CONTEXT, it is UNSUPPORTED.
- Length is not quality. A short answer can be fully faithful; a long answer can be full of hallucinations.
- Return JSON matching the schema exactly."""
 
JUDGE_USER = """CONTEXT:
{context}
 
ANSWER:
{answer}
 
Return JSON:
{{"faithful": bool, "unsupported_claims": [str], "score": 1-5, "reasoning": str}}
Where score=5 means fully supported, score=1 means mostly fabricated."""
 
def judge(context: str, answer: str, model: str = "gpt-4o-mini") -> Verdict:
    resp = client.chat.completions.create(
        model=model,
        temperature=0,  # critical — otherwise judge scores drift across runs
        response_format={"type": "json_object"},
        messages=[
            {"role": "system", "content": JUDGE_SYSTEM},
            {"role": "user", "content": JUDGE_USER.format(context=context, answer=answer)},
        ],
    )
    return Verdict(**json.loads(resp.choices[0].message.content))
 
# ---------- 3. calibration: judge-vs-human agreement ----------
def calibrate(rows, model="gpt-4o-mini"):
    hits = 0
    for row in rows:
        v = judge(row["context"], row["answer"], model=model)
        agree = v.faithful == row["human_faithful"]
        hits += agree
        marker = "✅" if agree else "❌"
        print(f"{marker} judge={v.faithful} human={row['human_faithful']} "
              f"score={v.score} reason={v.reasoning[:80]}")
    agreement = hits / len(rows)
    print(f"\nJudge-human agreement: {agreement:.0%} on {len(rows)} rows")
    return agreement
 
# ---------- 4. bias check: does order matter? ----------
def position_bias_test(row_a, row_b, n=10, model="gpt-4o-mini"):
    """Ask the judge to pick the better of two answers, N times, both orders."""
    wins_a = 0
    for _ in range(n):
        order = random.choice(["ab", "ba"])
        first, second = (row_a, row_b) if order == "ab" else (row_b, row_a)
        # ...judge call with a rubric that picks a winner
        # (elided for brevity — pattern is the same as judge() above)
    # If wins_a >> n/2 regardless of who's actually better → position bias.
 
# ---------- 5. run ----------
if __name__ == "__main__":
    print("=== Judge calibration ===")
    agreement = calibrate(GOLDEN)
    if agreement < 0.85:
        print("⚠️  Agreement <85% — do NOT trust this judge yet.")
        print("   Options: upgrade to gpt-4o, tighten the rubric, or add few-shot examples.")
    else:
        print("✅  Judge is calibrated. Safe to use in CI.")