Keep runs with different tuning apart in the report, and raise max_turns to 40

The report grouped runs by model, serving note and task version, so
runs made at different tuning were silently averaged together. Tuning
is now part of the key, and a tuning column shows how a row's settings
differ from today's defaults; it appears only when some row needs it.

tuning.executor.max_turns goes from 24 to 40. On the harder benchmark
tasks Haiku, the calibration ceiling, hit 24 on both refactor-rename
runs and both long-milestone runs, one of which had finished the work.
The cap guards against a model trying forever; it isn't a budget.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>

nisse-dev/tests/test_bench.py

@@ -149,7 +149,7 @@ def test_a_solved_task_is_a_success(tmp_path: Path) -> None:
     assert record.tokens_in == 3000
     assert record.task_hash == task.content_hash
     assert record.model_info["supports_tool_choice"] is True
-    assert record.tuning["executor"]["max_turns"] == 24
+    assert record.tuning["executor"]["max_turns"] == 40
 
 
 def test_saying_finished_is_not_success(tmp_path: Path) -> None:
@@ -317,3 +317,20 @@ def test_results_stored_before_waits_were_recorded_still_load() -> None:
     del old["rate_limit_wait_s"]
     [row] = summarise([old], current)
     assert (row.mean_wall_s, row.rate_limited) == (12.0, 0)
+
+
+def test_runs_with_different_tuning_are_kept_apart() -> None:
+    current = {t.name: t.content_hash for t in TASKS}
+    capped = record_json()
+    capped["tuning"]["executor"]["max_turns"] = 7
+    rows = summarise([record_json(), capped], current)
+    assert sorted(row.tuning for row in rows) == ["", "executor.max_turns=7"]
+    table = render(rows)
+    assert "executor.max_turns=7" in table
+    assert "tuning = settings that differ" in table
+
+
+def test_a_report_at_default_tuning_has_no_tuning_column() -> None:
+    current = {t.name: t.content_hash for t in TASKS}
+    table = render(summarise([record_json()], current))
+    assert "tuning" not in table

nisse-dev/tests/test_config.py

@@ -69,7 +69,7 @@ def test_complete_config_builds_with_secrets_by_reference() -> None:
     assert config.matrix.homeserver == "https://matrix.test"
     assert config.gitea_repo == "don/demo"
     assert config.base_branch == "main"
-    assert config.tuning.executor.max_turns == 24  # absent section: defaults
+    assert config.tuning.executor.max_turns == 40  # absent section: defaults
 
 
 def test_paths_expand_the_home_directory() -> None:
@@ -272,7 +272,7 @@ def test_the_example_file_is_valid() -> None:
         "NISSE_MATRIX_TOKEN": "syt_x",
         "NISSE_GITEA_TOKEN": "gt",
     }
-    assert AgentConfig.from_file(example, env).tuning.executor.max_turns == 24
+    assert AgentConfig.from_file(example, env).tuning.executor.max_turns == 40
 
 
 # --- nothing that carries secrets may sit inside the clone -----------------