dongarandClaude Opus 5.5
50b46821d7
CI / checks (push) Skipped
CI / checks (pull_request) Successful in 1m11s
Choosing a local executor model, and tuning its knobs, needs measured success rates across repeated runs, comparable with every model measured before. - Four canned milestones under nisse_dev/bench/tasks: write a function, fix a bug a failing test exposes, thread an option through several files, and find one failure in output far past max_output_chars. Each is an approved plan over a seed project, judged by a hidden check the model never sees; a reference solution proves each is solvable and that its check tells the difference - Runs use the agent's own client wiring, now shared as make_llm_clients, with delegation and ask_expert off: an escalation is a failure of the model under test - Each run is stored as one JSON line in bench/results/, committed, with the model's backing name and declared facts, a note on how it was served, the task's content hash, the agent's commit and tuning, the outcome, turns, misbehaviour counts, context trims, tokens, time and LiteLLM's cost; a fallback answer can never count as a success - The report rebuilds the comparison from every stored run, with Wilson 95% intervals, marking runs against older task versions - scripts/benchmark.py runs and reports; scripts/evaluate_model.sh takes a new model through the context probe, the capability matrix and the benchmark in one command - Clients keep LiteLLM's per-call cost, and BudgetedClient counts the requests it had to trim - The tuning guide gains "Benchmarking models" Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
118 lines
3.9 KiB
Python
118 lines
3.9 KiB
Python
"""Stored benchmark results, and the report that compares them.
|
|
|
|
Every run is one JSON line. The report is rebuilt from all of them each
|
|
time, so a model benchmarked today is read alongside every model measured
|
|
before it, as long as the task is still the same version.
|
|
"""
|
|
|
|
import json
|
|
import math
|
|
import statistics
|
|
from collections import defaultdict
|
|
from collections.abc import Iterable
|
|
from dataclasses import dataclass
|
|
from datetime import UTC, datetime
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
from nisse_dev.bench.runner import RunRecord
|
|
|
|
|
|
def results_file(directory: Path, model: str, when: datetime | None = None) -> Path:
|
|
day = (when or datetime.now(UTC)).strftime("%Y-%m-%d")
|
|
return directory / f"{day}-{model}.jsonl"
|
|
|
|
|
|
def append(path: Path, record: RunRecord) -> None:
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
with path.open("a") as out:
|
|
out.write(json.dumps(record.to_json(), sort_keys=True) + "\n")
|
|
|
|
|
|
def load(directory: Path) -> list[dict[str, Any]]:
|
|
records = []
|
|
for path in sorted(directory.glob("*.jsonl")):
|
|
for line in path.read_text().splitlines():
|
|
if line.strip():
|
|
records.append(json.loads(line))
|
|
return records
|
|
|
|
|
|
def wilson(successes: int, runs: int, z: float = 1.96) -> tuple[float, float]:
|
|
"""A 95% confidence interval for a success rate, sound at small counts.
|
|
|
|
Three successes in five runs and five in five overlap here, which is the
|
|
point: small samples of a noisy model do not separate much."""
|
|
if runs == 0:
|
|
return 0.0, 1.0
|
|
p = successes / runs
|
|
denominator = 1 + z * z / runs
|
|
centre = (p + z * z / (2 * runs)) / denominator
|
|
half = z * math.sqrt(p * (1 - p) / runs + z * z / (4 * runs * runs)) / denominator
|
|
return max(0.0, centre - half), min(1.0, centre + half)
|
|
|
|
|
|
@dataclass
|
|
class Row:
|
|
model: str
|
|
note: str
|
|
task: str
|
|
runs: int
|
|
successes: int
|
|
median_turns: float
|
|
mean_wall_s: float
|
|
cost_per_run: float
|
|
stale: bool
|
|
|
|
@property
|
|
def interval(self) -> tuple[float, float]:
|
|
return wilson(self.successes, self.runs)
|
|
|
|
|
|
def summarise(
|
|
records: Iterable[dict[str, Any]], current_hashes: dict[str, str]
|
|
) -> list[Row]:
|
|
"""One row per (model, how it was served, task, task version)."""
|
|
groups: dict[tuple[str, str, str, str], list[dict[str, Any]]] = defaultdict(list)
|
|
for r in records:
|
|
groups[(r["backing"], r.get("note", ""), r["task"], r["task_hash"])].append(r)
|
|
rows = []
|
|
for (backing, note, task, task_hash), runs in sorted(groups.items()):
|
|
rows.append(
|
|
Row(
|
|
model=backing,
|
|
note=note,
|
|
task=task,
|
|
runs=len(runs),
|
|
successes=sum(1 for r in runs if r["success"]),
|
|
median_turns=statistics.median(r["turns"] for r in runs),
|
|
mean_wall_s=statistics.fmean(r["wall_s"] for r in runs),
|
|
cost_per_run=statistics.fmean(r["cost"] for r in runs),
|
|
stale=current_hashes.get(task) != task_hash,
|
|
)
|
|
)
|
|
return rows
|
|
|
|
|
|
def render(rows: list[Row]) -> str:
|
|
header = (
|
|
f"{'model':<34} {'note':<22} {'task':<15} {'success':>8} {'95% CI':>11} "
|
|
f"{'turns':>6} {'time':>7} {'$/run':>8}"
|
|
)
|
|
lines = [header, "-" * len(header)]
|
|
for row in rows:
|
|
low, high = row.interval
|
|
task = row.task + (" (old)" if row.stale else "")
|
|
lines.append(
|
|
f"{row.model[:34]:<34} {row.note[:22]:<22} {task[:15]:<15} "
|
|
f"{row.successes:>3}/{row.runs:<4} {low:>4.0%}-{high:<5.0%} "
|
|
f"{row.median_turns:>6.0f} {row.mean_wall_s:>6.0f}s {row.cost_per_run:>8.4f}"
|
|
)
|
|
if any(row.stale for row in rows):
|
|
lines.append("")
|
|
lines.append(
|
|
"(old) = measured against an earlier version of the task; "
|
|
"not comparable with current runs."
|
|
)
|
|
return "\n".join(lines)
|