Add an executor benchmark that keeps its results

Choosing a local executor model, and tuning its knobs, needs measured
success rates across repeated runs, comparable with every model
measured before.

- Four canned milestones under nisse_dev/bench/tasks: write a function,
  fix a bug a failing test exposes, thread an option through several
  files, and find one failure in output far past max_output_chars. Each
  is an approved plan over a seed project, judged by a hidden check the
  model never sees; a reference solution proves each is solvable and
  that its check tells the difference
- Runs use the agent's own client wiring, now shared as
  make_llm_clients, with delegation and ask_expert off: an escalation is
  a failure of the model under test
- Each run is stored as one JSON line in bench/results/, committed, with
  the model's backing name and declared facts, a note on how it was
  served, the task's content hash, the agent's commit and tuning, the
  outcome, turns, misbehaviour counts, context trims, tokens, time and
  LiteLLM's cost; a fallback answer can never count as a success
- The report rebuilds the comparison from every stored run, with Wilson
  95% intervals, marking runs against older task versions
- scripts/benchmark.py runs and reports; scripts/evaluate_model.sh takes
  a new model through the context probe, the capability matrix and the
  benchmark in one command
- Clients keep LiteLLM's per-call cost, and BudgetedClient counts the
  requests it had to trim
- The tuning guide gains "Benchmarking models"

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>

nisse-dev/src/nisse_dev/agent.py

@@ -113,6 +113,51 @@ class Agent:
         self._stopped.set()
 
 
+def make_llm_clients(
+    config: AgentConfig, catalog: ModelCatalog
+) -> tuple[BudgetedClient, BudgetedClient]:
+    """The two LLM routes, as every model call in the agent uses them.
+
+    Shared with the benchmark, so that what it measures is exactly what the
+    agent would do: the same context budget, the same declared limits, and
+    tool_choice asked only of models declared to enforce it.
+    """
+    tuning = config.tuning
+
+    def limits_for(model: str) -> tuple[int, int | None]:
+        facts = catalog.facts(model)
+        assert facts.max_input_tokens is not None  # require_limits checked
+        return facts.max_input_tokens, facts.max_output_tokens
+
+    def budgeted(client: object) -> BudgetedClient:
+        # Every request, on either route, is fitted to its model's window.
+        return BudgetedClient(
+            client,
+            limits_for,
+            max_output_tokens=config.max_output_tokens,
+            keep_recent_turns=tuning.context.keep_recent_turns,
+            safety_margin=tuning.context.safety_margin,
+            enforces_tool_choice=lambda m: (
+                catalog.facts(m).supports_tool_choice is True
+            ),
+        )
+
+    normalized = budgeted(
+        LiteLLMClient(
+            config.llm_base_url, config.llm_token, max_tokens=config.max_output_tokens
+        )
+    )
+    # Delegations prefer the vendor-native route; fidelity is cheap
+    # elsewhere (A5). Which route a given model gets is decided by
+    # select_client, not by where the caller happens to sit.
+    passthrough = budgeted(
+        AnthropicPassthroughClient(
+            config.llm_base_url, config.llm_token, max_tokens=config.max_output_tokens
+        )
+    )
+    return normalized, passthrough
+
+
 def prepare_workspace(config: AgentConfig, account: str) -> GitRepo:
     """Ensure the workspace is a clone of the project repo.
 
@@ -161,37 +206,7 @@ def build_agent(
         max_output_chars=tuning.tools.max_output_chars,
     )
 
-    def limits_for(model: str) -> tuple[int, int | None]:
-        facts = catalog.facts(model)
-        assert facts.max_input_tokens is not None  # require_limits checked
-        return facts.max_input_tokens, facts.max_output_tokens
-
-    def budgeted(client: object) -> BudgetedClient:
-        # Every request, on either route, is fitted to its model's window.
-        return BudgetedClient(
-            client,
-            limits_for,
-            max_output_tokens=config.max_output_tokens,
-            keep_recent_turns=tuning.context.keep_recent_turns,
-            safety_margin=tuning.context.safety_margin,
-            enforces_tool_choice=lambda m: (
-                catalog.facts(m).supports_tool_choice is True
-            ),
-        )
-
-    normalized = budgeted(
-        LiteLLMClient(
-            config.llm_base_url, config.llm_token, max_tokens=config.max_output_tokens
-        )
-    )
-    # Delegations prefer the vendor-native route; fidelity is cheap
-    # elsewhere (A5). Which route a given model gets is decided by
-    # select_client, not by where the caller happens to sit.
-    passthrough = budgeted(
-        AnthropicPassthroughClient(
-            config.llm_base_url, config.llm_token, max_tokens=config.max_output_tokens
-        )
-    )
+    normalized, passthrough = make_llm_clients(config, catalog)
 
     transport = MatrixTransport(config.matrix)
     announce = notify or transport.send_message