Add an executor benchmark that keeps its results

Choosing a local executor model, and tuning its knobs, needs measured
success rates across repeated runs, comparable with every model
measured before.

- Four canned milestones under nisse_dev/bench/tasks: write a function,
  fix a bug a failing test exposes, thread an option through several
  files, and find one failure in output far past max_output_chars. Each
  is an approved plan over a seed project, judged by a hidden check the
  model never sees; a reference solution proves each is solvable and
  that its check tells the difference
- Runs use the agent's own client wiring, now shared as
  make_llm_clients, with delegation and ask_expert off: an escalation is
  a failure of the model under test
- Each run is stored as one JSON line in bench/results/, committed, with
  the model's backing name and declared facts, a note on how it was
  served, the task's content hash, the agent's commit and tuning, the
  outcome, turns, misbehaviour counts, context trims, tokens, time and
  LiteLLM's cost; a fallback answer can never count as a success
- The report rebuilds the comparison from every stored run, with Wilson
  95% intervals, marking runs against older task versions
- scripts/benchmark.py runs and reports; scripts/evaluate_model.sh takes
  a new model through the context probe, the capability matrix and the
  benchmark in one command
- Clients keep LiteLLM's per-call cost, and BudgetedClient counts the
  requests it had to trim
- The tuning guide gains "Benchmarking models"

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>

nisse-dev/src/nisse_dev/bench/tasks/multi-file/hidden/test_hidden.py

@@ -0,0 +1,33 @@
+import subprocess
+import sys
+
+from greeter.cli import main
+from greeter.core import greet
+
+
+def test_default_is_unchanged() -> None:
+    assert greet("Bob") == "Hello, Bob!"
+
+
+def test_shout() -> None:
+    assert greet("Bob", shout=True) == "HELLO, BOB!"
+
+
+def test_cli_default(capsys) -> None:
+    assert main(["Ada"]) == 0
+    assert capsys.readouterr().out == "Hello, Ada!\n"
+
+
+def test_cli_shout(capsys) -> None:
+    assert main(["--shout", "Ada"]) == 0
+    assert capsys.readouterr().out == "HELLO, ADA!\n"
+
+
+def test_module_entry_point() -> None:
+    result = subprocess.run(
+        [sys.executable, "-m", "greeter", "--shout", "Bob"],
+        capture_output=True,
+        text=True,
+        check=True,
+    )
+    assert result.stdout == "HELLO, BOB!\n"

nisse-dev/src/nisse_dev/bench/tasks/multi-file/seed/greeter/__main__.py

@@ -0,0 +1,3 @@
+from greeter.cli import main
+
+raise SystemExit(main())

nisse-dev/src/nisse_dev/bench/tasks/multi-file/seed/greeter/cli.py

@@ -0,0 +1,11 @@
+import argparse
+
+from greeter.core import greet
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = argparse.ArgumentParser(prog="greeter")
+    parser.add_argument("name")
+    args = parser.parse_args(argv)
+    print(greet(args.name))
+    return 0

nisse-dev/src/nisse_dev/bench/tasks/multi-file/seed/greeter/core.py

@@ -0,0 +1,2 @@
+def greet(name: str) -> str:
+    return f"Hello, {name}!"

nisse-dev/src/nisse_dev/bench/tasks/multi-file/seed/tests/test_greeter.py

@@ -0,0 +1,11 @@
+from greeter.cli import main
+from greeter.core import greet
+
+
+def test_greet() -> None:
+    assert greet("Bob") == "Hello, Bob!"
+
+
+def test_cli_prints_the_greeting(capsys) -> None:
+    assert main(["Bob"]) == 0
+    assert capsys.readouterr().out == "Hello, Bob!\n"

nisse-dev/src/nisse_dev/bench/tasks/multi-file/solution/greeter/cli.py

@@ -0,0 +1,12 @@
+import argparse
+
+from greeter.core import greet
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = argparse.ArgumentParser(prog="greeter")
+    parser.add_argument("name")
+    parser.add_argument("--shout", action="store_true")
+    args = parser.parse_args(argv)
+    print(greet(args.name, shout=args.shout))
+    return 0

nisse-dev/src/nisse_dev/bench/tasks/multi-file/solution/greeter/core.py

@@ -0,0 +1,3 @@
+def greet(name: str, shout: bool = False) -> str:
+    greeting = f"Hello, {name}!"
+    return greeting.upper() if shout else greeting

nisse-dev/src/nisse_dev/bench/tasks/multi-file/task.yaml

@@ -0,0 +1,16 @@
+title: Thread a new option through several files
+check: python -m pytest -q .bench_check
+plan: |
+  Milestone 1: add a --shout option to the greeter.
+
+  1. In `greeter/core.py`, give `greet` a keyword argument `shout: bool = False`.
+     When it is true, return the whole greeting in upper case:
+     greet("Bob", shout=True) == "HELLO, BOB!". The default behaviour is unchanged.
+  2. In `greeter/cli.py`, add a `--shout` flag (action="store_true") and pass it
+     to `greet`.
+  3. Add two tests to `tests/test_greeter.py`: one for greet with shout=True, and
+     one that calls `main(["--shout", "Bob"])` and checks the printed output
+     using pytest's capsys fixture.
+  4. Run `python -m pytest -q` until every test passes, and check that
+     `python -m greeter --shout Bob` prints `HELLO, BOB!`.
+  5. Call finish.