Keep runs with different tuning apart in the report, and raise max_turns to 40

The report grouped runs by model, serving note and task version, so
runs made at different tuning were silently averaged together. Tuning
is now part of the key, and a tuning column shows how a row's settings
differ from today's defaults; it appears only when some row needs it.

tuning.executor.max_turns goes from 24 to 40. On the harder benchmark
tasks Haiku, the calibration ceiling, hit 24 on both refactor-rename
runs and both long-milestone runs, one of which had finished the work.
The cap guards against a model trying forever; it isn't a budget.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>

nisse-dev/src/nisse_dev/bench/results.py

@@ -10,12 +10,13 @@ import math
 import statistics
 from collections import defaultdict
 from collections.abc import Iterable
-from dataclasses import dataclass
+from dataclasses import asdict, dataclass
 from datetime import UTC, datetime
 from pathlib import Path
 from typing import Any
 
 from nisse_dev.bench.runner import RunRecord
+from nisse_dev.config import Tuning
 
 
 def results_file(directory: Path, model: str, when: datetime | None = None) -> Path:
@@ -64,21 +65,47 @@ class Row:
     cost_per_run: float
     stale: bool
     rate_limited: int = 0  # runs that waited out the proxy's rate limit
+    tuning: str = ""  # how the runs' tuning differs from today's defaults
 
     @property
     def interval(self) -> tuple[float, float]:
         return wilson(self.successes, self.runs)
 
 
+def tuning_label(tuning: dict[str, Any]) -> str:
+    """The settings in a run's tuning that differ from today's defaults, as
+    "executor.max_turns=24"; empty when there are none. Relative to today's
+    defaults so that rows at the current settings stay unlabelled."""
+    defaults = asdict(Tuning())
+    return ", ".join(
+        f"{section}.{key}={value}"
+        for section, values in sorted(tuning.items())
+        for key, value in sorted((values or {}).items())
+        if defaults.get(section, {}).get(key) != value
+    )
+
+
 def summarise(
     records: Iterable[dict[str, Any]], current_hashes: dict[str, str]
 ) -> list[Row]:
-    """One row per (model, how it was served, task, task version)."""
-    groups: dict[tuple[str, str, str, str], list[dict[str, Any]]] = defaultdict(list)
+    """One row per (model, how it was served, task, task version, tuning).
+
+    Tuning is part of the key because a knob changes the result: runs capped
+    at 24 turns and runs allowed 40 are not the same measurement."""
+    groups: dict[tuple[str, str, str, str, str], list[dict[str, Any]]] = defaultdict(
+        list
+    )
     for r in records:
-        groups[(r["backing"], r.get("note", ""), r["task"], r["task_hash"])].append(r)
+        key = (
+            r["backing"],
+            r.get("note", ""),
+            r["task"],
+            r["task_hash"],
+            tuning_label(r.get("tuning", {})),
+        )
+        groups[key].append(r)
     rows = []
-    for (backing, note, task, task_hash), runs in sorted(groups.items()):
+    for (backing, note, task, task_hash, tuning), runs in sorted(groups.items()):
         rows.append(
             Row(
                 model=backing,
@@ -93,22 +120,29 @@ def summarise(
                 cost_per_run=statistics.fmean(r["cost"] for r in runs),
                 stale=current_hashes.get(task) != task_hash,
                 rate_limited=sum(1 for r in runs if r.get("rate_limit_wait_s")),
+                tuning=tuning,
             )
         )
     return rows
 
 
 def render(rows: list[Row]) -> str:
+    # A tuning column only when some rows need one, so a report at today's
+    # settings looks as it always has.
+    tuned = any(row.tuning for row in rows)
     header = (
-        f"{'model':<34} {'note':<22} {'task':<15} {'success':>8} {'95% CI':>11} "
+        f"{'model':<34} {'note':<22} "
+        + (f"{'tuning':<24} " if tuned else "")
+        + f"{'task':<15} {'success':>8} {'95% CI':>11} "
         f"{'turns':>6} {'time':>7} {'$/run':>8} {'429':>4}"
     )
     lines = [header, "-" * len(header)]
     for row in rows:
         low, high = row.interval
         task = row.task + (" (old)" if row.stale else "")
+        tuning = f"{row.tuning[:24]:<24} " if tuned else ""
         lines.append(
-            f"{row.model[:34]:<34} {row.note[:22]:<22} {task[:15]:<15} "
+            f"{row.model[:34]:<34} {row.note[:22]:<22} {tuning}{task[:15]:<15} "
             f"{row.successes:>3}/{row.runs:<4} {low:>4.0%}-{high:<5.0%} "
             f"{row.median_turns:>6.0f} {row.mean_wall_s:>6.0f}s {row.cost_per_run:>8.4f} "
             f"{row.rate_limited or '':>4}"
@@ -119,6 +153,9 @@ def render(rows: list[Row]) -> str:
             "429 = runs that waited out the proxy's rate limit; time leaves the "
             "waits out."
         )
+    if tuned:
+        lines.append("")
+        lines.append("tuning = settings that differ from today's defaults.")
     if any(row.stale for row in rows):
         lines.append("")
         lines.append(

nisse-dev/src/nisse_dev/config.py

@@ -64,7 +64,7 @@ class ConfigError(RuntimeError):
 
 @dataclass(frozen=True)
 class ExecutorTuning:
-    max_turns: int = 24
+    max_turns: int = 40
     failure_limit: int = 2