Add an executor benchmark that keeps its results

Choosing a local executor model, and tuning its knobs, needs measured
success rates across repeated runs, comparable with every model
measured before.

- Four canned milestones under nisse_dev/bench/tasks: write a function,
  fix a bug a failing test exposes, thread an option through several
  files, and find one failure in output far past max_output_chars. Each
  is an approved plan over a seed project, judged by a hidden check the
  model never sees; a reference solution proves each is solvable and
  that its check tells the difference
- Runs use the agent's own client wiring, now shared as
  make_llm_clients, with delegation and ask_expert off: an escalation is
  a failure of the model under test
- Each run is stored as one JSON line in bench/results/, committed, with
  the model's backing name and declared facts, a note on how it was
  served, the task's content hash, the agent's commit and tuning, the
  outcome, turns, misbehaviour counts, context trims, tokens, time and
  LiteLLM's cost; a fallback answer can never count as a success
- The report rebuilds the comparison from every stored run, with Wilson
  95% intervals, marking runs against older task versions
- scripts/benchmark.py runs and reports; scripts/evaluate_model.sh takes
  a new model through the context probe, the capability matrix and the
  benchmark in one command
- Clients keep LiteLLM's per-call cost, and BudgetedClient counts the
  requests it had to trim
- The tuning guide gains "Benchmarking models"

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>

nisse-dev/src/nisse_dev/bench/results.py

@@ -0,0 +1,117 @@
+"""Stored benchmark results, and the report that compares them.
+
+Every run is one JSON line. The report is rebuilt from all of them each
+time, so a model benchmarked today is read alongside every model measured
+before it, as long as the task is still the same version.
+"""
+
+import json
+import math
+import statistics
+from collections import defaultdict
+from collections.abc import Iterable
+from dataclasses import dataclass
+from datetime import UTC, datetime
+from pathlib import Path
+from typing import Any
+
+from nisse_dev.bench.runner import RunRecord
+
+
+def results_file(directory: Path, model: str, when: datetime | None = None) -> Path:
+    day = (when or datetime.now(UTC)).strftime("%Y-%m-%d")
+    return directory / f"{day}-{model}.jsonl"
+
+
+def append(path: Path, record: RunRecord) -> None:
+    path.parent.mkdir(parents=True, exist_ok=True)
+    with path.open("a") as out:
+        out.write(json.dumps(record.to_json(), sort_keys=True) + "\n")
+
+
+def load(directory: Path) -> list[dict[str, Any]]:
+    records = []
+    for path in sorted(directory.glob("*.jsonl")):
+        for line in path.read_text().splitlines():
+            if line.strip():
+                records.append(json.loads(line))
+    return records
+
+
+def wilson(successes: int, runs: int, z: float = 1.96) -> tuple[float, float]:
+    """A 95% confidence interval for a success rate, sound at small counts.
+
+    Three successes in five runs and five in five overlap here, which is the
+    point: small samples of a noisy model do not separate much."""
+    if runs == 0:
+        return 0.0, 1.0
+    p = successes / runs
+    denominator = 1 + z * z / runs
+    centre = (p + z * z / (2 * runs)) / denominator
+    half = z * math.sqrt(p * (1 - p) / runs + z * z / (4 * runs * runs)) / denominator
+    return max(0.0, centre - half), min(1.0, centre + half)
+
+
+@dataclass
+class Row:
+    model: str
+    note: str
+    task: str
+    runs: int
+    successes: int
+    median_turns: float
+    mean_wall_s: float
+    cost_per_run: float
+    stale: bool
+
+    @property
+    def interval(self) -> tuple[float, float]:
+        return wilson(self.successes, self.runs)
+
+
+def summarise(
+    records: Iterable[dict[str, Any]], current_hashes: dict[str, str]
+) -> list[Row]:
+    """One row per (model, how it was served, task, task version)."""
+    groups: dict[tuple[str, str, str, str], list[dict[str, Any]]] = defaultdict(list)
+    for r in records:
+        groups[(r["backing"], r.get("note", ""), r["task"], r["task_hash"])].append(r)
+    rows = []
+    for (backing, note, task, task_hash), runs in sorted(groups.items()):
+        rows.append(
+            Row(
+                model=backing,
+                note=note,
+                task=task,
+                runs=len(runs),
+                successes=sum(1 for r in runs if r["success"]),
+                median_turns=statistics.median(r["turns"] for r in runs),
+                mean_wall_s=statistics.fmean(r["wall_s"] for r in runs),
+                cost_per_run=statistics.fmean(r["cost"] for r in runs),
+                stale=current_hashes.get(task) != task_hash,
+            )
+        )
+    return rows
+
+
+def render(rows: list[Row]) -> str:
+    header = (
+        f"{'model':<34} {'note':<22} {'task':<15} {'success':>8} {'95% CI':>11} "
+        f"{'turns':>6} {'time':>7} {'$/run':>8}"
+    )
+    lines = [header, "-" * len(header)]
+    for row in rows:
+        low, high = row.interval
+        task = row.task + (" (old)" if row.stale else "")
+        lines.append(
+            f"{row.model[:34]:<34} {row.note[:22]:<22} {task[:15]:<15} "
+            f"{row.successes:>3}/{row.runs:<4} {low:>4.0%}-{high:<5.0%} "
+            f"{row.median_turns:>6.0f} {row.mean_wall_s:>6.0f}s {row.cost_per_run:>8.4f}"
+        )
+    if any(row.stale for row in rows):
+        lines.append("")
+        lines.append(
+            "(old) = measured against an earlier version of the task; "
+            "not comparable with current runs."
+        )
+    return "\n".join(lines)