Add an executor benchmark that keeps its results

Choosing a local executor model, and tuning its knobs, needs measured
success rates across repeated runs, comparable with every model
measured before.

- Four canned milestones under nisse_dev/bench/tasks: write a function,
  fix a bug a failing test exposes, thread an option through several
  files, and find one failure in output far past max_output_chars. Each
  is an approved plan over a seed project, judged by a hidden check the
  model never sees; a reference solution proves each is solvable and
  that its check tells the difference
- Runs use the agent's own client wiring, now shared as
  make_llm_clients, with delegation and ask_expert off: an escalation is
  a failure of the model under test
- Each run is stored as one JSON line in bench/results/, committed, with
  the model's backing name and declared facts, a note on how it was
  served, the task's content hash, the agent's commit and tuning, the
  outcome, turns, misbehaviour counts, context trims, tokens, time and
  LiteLLM's cost; a fallback answer can never count as a success
- The report rebuilds the comparison from every stored run, with Wilson
  95% intervals, marking runs against older task versions
- scripts/benchmark.py runs and reports; scripts/evaluate_model.sh takes
  a new model through the context probe, the capability matrix and the
  benchmark in one command
- Clients keep LiteLLM's per-call cost, and BudgetedClient counts the
  requests it had to trim
- The tuning guide gains "Benchmarking models"

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>

scripts/benchmark.py

@@ -0,0 +1,124 @@
+"""Benchmark executor models on canned milestones, and compare the results.
+
+    uv run python scripts/benchmark.py run --models A,B [--runs 5] [--tasks T,...]
+        [--note "vLLM 0.29, AWQ, fp8 KV"] [--config PATH]
+    uv run python scripts/benchmark.py report [--models A,B] [--tasks T,...]
+
+`run` appends one JSON line per run to bench/results/, which is committed:
+the report compares every run ever stored, so benchmarking a new model means
+running only that model. See docs/Tuning-Guide.md, "Benchmarking models".
+"""
+
+import argparse
+import os
+import sys
+from pathlib import Path
+
+from nisse_dev.agent import make_llm_clients
+from nisse_dev.bench.results import append, load, render, results_file, summarise
+from nisse_dev.bench.runner import agent_commit, run_task
+from nisse_dev.bench.tasks import load_tasks
+from nisse_dev.catalog import ModelCatalog
+from nisse_dev.config import AgentConfig, ConfigError
+from nisse_dev.envfile import load_env_file
+
+REPO = Path(__file__).resolve().parents[1]
+RESULTS = REPO / "bench" / "results"
+
+
+def _names(raw: str | None) -> list[str] | None:
+    return [n.strip() for n in raw.split(",") if n.strip()] if raw else None
+
+
+def run(args: argparse.Namespace) -> int:
+    load_env_file(Path(os.environ.get("NISSE_ENV", ".env")).expanduser())
+    try:
+        config = (
+            AgentConfig.from_file(Path(args.config))
+            if args.config
+            else AgentConfig.load()
+        )
+    except ConfigError as exc:
+        print(exc, file=sys.stderr)
+        return 2
+    models = _names(args.models) or []
+    tasks = load_tasks(_names(args.tasks))
+    catalog = ModelCatalog.discover(config.llm_base_url, config.llm_token)
+    catalog.require_limits({m: m for m in models})
+    normalized, _passthrough = make_llm_clients(config, catalog)
+    commit = agent_commit(REPO)
+
+    for model in models:
+        facts = catalog.facts(model)
+        out = results_file(args.results, model)
+        for task in tasks:
+            for n in range(args.runs):
+                # The executor always takes the normalized route in the agent,
+                # so it does here too.
+                record = run_task(
+                    task,
+                    facts,
+                    normalized,
+                    config.tuning,
+                    note=args.note,
+                    commit=commit,
+                )
+                append(out, record)
+                verdict = "pass" if record.success else f"FAIL ({record.outcome})"
+                extra = (
+                    f" fallback to {record.fallbacks[0]}!" if record.fallbacks else ""
+                )
+                print(
+                    f"{model} {task.name} run {n + 1}/{args.runs}: {verdict}, "
+                    f"{record.turns} turns, {record.wall_s:.0f}s, "
+                    f"${record.cost:.4f}{extra}",
+                    flush=True,
+                )
+    print()
+    return report(args)
+
+
+def report(args: argparse.Namespace) -> int:
+    records = load(args.results)
+    models = _names(getattr(args, "models", None))
+    tasks = _names(getattr(args, "tasks", None))
+    if models:
+        records = [r for r in records if r["model"] in models or r["backing"] in models]
+    if tasks:
+        records = [r for r in records if r["task"] in tasks]
+    if not records:
+        print("no stored results match", file=sys.stderr)
+        return 1
+    current = {task.name: task.content_hash for task in load_tasks()}
+    print(render(summarise(records, current)))
+    return 0
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0])
+    parser.add_argument("--results", type=Path, default=RESULTS, help=argparse.SUPPRESS)
+    sub = parser.add_subparsers(dest="command", required=True)
+
+    run_p = sub.add_parser("run", help="benchmark models, then report")
+    run_p.add_argument(
+        "--models", required=True, help="LiteLLM aliases, comma separated"
+    )
+    run_p.add_argument("--runs", type=int, default=5, help="runs per task (default 5)")
+    run_p.add_argument("--tasks", help="task names, comma separated (default: all)")
+    run_p.add_argument(
+        "--note", default="", help='how the model was served, e.g. "vLLM 0.29, AWQ"'
+    )
+    run_p.add_argument("--config", help="agent config (default: NISSE_CONFIG)")
+    run_p.set_defaults(handler=run)
+
+    report_p = sub.add_parser("report", help="compare stored results")
+    report_p.add_argument("--models", help="aliases or backing models to include")
+    report_p.add_argument("--tasks", help="tasks to include")
+    report_p.set_defaults(handler=report)
+
+    args = parser.parse_args()
+    return args.handler(args)
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())