Keep runs with different tuning apart in the report, and raise max_turns to 40

The report grouped runs by model, serving note and task version, so
runs made at different tuning were silently averaged together. Tuning
is now part of the key, and a tuning column shows how a row's settings
differ from today's defaults; it appears only when some row needs it.

tuning.executor.max_turns goes from 24 to 40. On the harder benchmark
tasks Haiku, the calibration ceiling, hit 24 on both refactor-rename
runs and both long-milestone runs, one of which had finished the work.
The cap guards against a model trying forever; it isn't a budget.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>

docs/Tuning-Guide.md

@@ -132,7 +132,22 @@ a local model can be held to less without affecting the frontier roles.
   executor. Frontier planning wants the large default. Leave this high, and
   set a local model's smaller ceiling as its `max_output_tokens` in LiteLLM.
 
-### `tuning.executor.max_turns` — default 24
+### `llm.request_timeout_s` — default 600
+
+How long one model request may take before it is abandoned. It applies to
+every request, on both routes.
+
+- **Too low:** a slow model's long answer is cut off, and the executor
+  escalates work that was going well. At 120 s, the old fixed value, a dense
+  27B model on the RTX 4090 timed out writing one module in one turn.
+- **Too high:** a hung connection holds the milestone for that long before
+  anything notices.
+- **Deciding:** this is a safety net, not a budget. `max_output_tokens` and
+  `max_turns` bound the work; set this above the slowest single answer you
+  expect, which is the model's output ceiling divided by its generation
+  speed.
+
+### `tuning.executor.max_turns` — default 40
 
 How many model calls one milestone may make before the executor stops with
 `capped` and asks you what to do.
@@ -146,6 +161,11 @@ How many model calls one milestone may make before the executor stops with
   the cap comfortably above the high end, around 1.5–2× the turns of your
   longer successful runs. A stronger model usually needs fewer turns, not
   more.
+- **Measured:** the default was 24 until the harder benchmark tasks arrived.
+  Haiku, used as the ceiling in calibration, hit 24 on both runs of
+  `refactor-rename` (editing ten files one at a time) and of
+  `long-milestone`, one of which had in fact finished the work. It is a guard
+  against a model trying forever, not a budget, so it was raised to 40.
 
 ### `tuning.executor.failure_limit` — default 2
 

nisse-dev.example.yaml

@@ -18,6 +18,9 @@ llm:                                  # A5: LiteLLM is the only LLM destination
   # Ceiling on each response. The model's own max_output_tokens in LiteLLM
   # caps it further, which is where to hold a local model to less.
   max_output_tokens: 16384
+  # How long one request may take before it is abandoned. A safety net for a
+  # hung connection; slow local models can take minutes over a long answer.
+  request_timeout_s: 600
 
 roles:                                # A11: role -> LiteLLM model alias
   executor: local-fast          # the local tier; the benchmark picks the model
@@ -44,7 +47,7 @@ gitea:                                # B2
 # read at startup, so a change takes effect on the next restart.
 tuning:
   executor:
-    max_turns: 24            # model calls per milestone before it stops as capped
+    max_turns: 40            # model calls per milestone before it stops as capped
     failure_limit: 2         # consecutive failed commands before escalating (A12)
   expert:
     consult_budget: 3        # ask_expert calls per milestone (A14)

nisse-dev/src/nisse_dev/agent.py

@@ -144,7 +144,10 @@ def make_llm_clients(
 
     normalized = budgeted(
         LiteLLMClient(
-            config.llm_base_url, config.llm_token, max_tokens=config.max_output_tokens
+            config.llm_base_url,
+            config.llm_token,
+            max_tokens=config.max_output_tokens,
+            timeout_s=config.request_timeout_s,
         )
     )
     # Delegations prefer the vendor-native route; fidelity is cheap
@@ -152,7 +155,10 @@ def make_llm_clients(
     # select_client, not by where the caller happens to sit.
     passthrough = budgeted(
         AnthropicPassthroughClient(
-            config.llm_base_url, config.llm_token, max_tokens=config.max_output_tokens
+            config.llm_base_url,
+            config.llm_token,
+            max_tokens=config.max_output_tokens,
+            timeout_s=config.request_timeout_s,
         )
     )
     return normalized, passthrough

nisse-dev/src/nisse_dev/bench/results.py

@@ -10,12 +10,13 @@ import math
 import statistics
 from collections import defaultdict
 from collections.abc import Iterable
-from dataclasses import dataclass
+from dataclasses import asdict, dataclass
 from datetime import UTC, datetime
 from pathlib import Path
 from typing import Any
 
 from nisse_dev.bench.runner import RunRecord
+from nisse_dev.config import Tuning
 
 
 def results_file(directory: Path, model: str, when: datetime | None = None) -> Path:
@@ -64,21 +65,47 @@ class Row:
     cost_per_run: float
     stale: bool
     rate_limited: int = 0  # runs that waited out the proxy's rate limit
+    tuning: str = ""  # how the runs' tuning differs from today's defaults
 
     @property
     def interval(self) -> tuple[float, float]:
         return wilson(self.successes, self.runs)
 
 
+def tuning_label(tuning: dict[str, Any]) -> str:
+    """The settings in a run's tuning that differ from today's defaults, as
+    "executor.max_turns=24"; empty when there are none. Relative to today's
+    defaults so that rows at the current settings stay unlabelled."""
+    defaults = asdict(Tuning())
+    return ", ".join(
+        f"{section}.{key}={value}"
+        for section, values in sorted(tuning.items())
+        for key, value in sorted((values or {}).items())
+        if defaults.get(section, {}).get(key) != value
+    )
+
+
 def summarise(
     records: Iterable[dict[str, Any]], current_hashes: dict[str, str]
 ) -> list[Row]:
-    """One row per (model, how it was served, task, task version)."""
-    groups: dict[tuple[str, str, str, str], list[dict[str, Any]]] = defaultdict(list)
+    """One row per (model, how it was served, task, task version, tuning).
+
+    Tuning is part of the key because a knob changes the result: runs capped
+    at 24 turns and runs allowed 40 are not the same measurement."""
+    groups: dict[tuple[str, str, str, str, str], list[dict[str, Any]]] = defaultdict(
+        list
+    )
     for r in records:
-        groups[(r["backing"], r.get("note", ""), r["task"], r["task_hash"])].append(r)
+        key = (
+            r["backing"],
+            r.get("note", ""),
+            r["task"],
+            r["task_hash"],
+            tuning_label(r.get("tuning", {})),
+        )
+        groups[key].append(r)
     rows = []
-    for (backing, note, task, task_hash), runs in sorted(groups.items()):
+    for (backing, note, task, task_hash, tuning), runs in sorted(groups.items()):
         rows.append(
             Row(
                 model=backing,
@@ -93,22 +120,29 @@ def summarise(
                 cost_per_run=statistics.fmean(r["cost"] for r in runs),
                 stale=current_hashes.get(task) != task_hash,
                 rate_limited=sum(1 for r in runs if r.get("rate_limit_wait_s")),
+                tuning=tuning,
             )
         )
     return rows
 
 
 def render(rows: list[Row]) -> str:
+    # A tuning column only when some rows need one, so a report at today's
+    # settings looks as it always has.
+    tuned = any(row.tuning for row in rows)
     header = (
-        f"{'model':<34} {'note':<22} {'task':<15} {'success':>8} {'95% CI':>11} "
+        f"{'model':<34} {'note':<22} "
+        + (f"{'tuning':<24} " if tuned else "")
+        + f"{'task':<15} {'success':>8} {'95% CI':>11} "
         f"{'turns':>6} {'time':>7} {'$/run':>8} {'429':>4}"
     )
     lines = [header, "-" * len(header)]
     for row in rows:
         low, high = row.interval
         task = row.task + (" (old)" if row.stale else "")
+        tuning = f"{row.tuning[:24]:<24} " if tuned else ""
         lines.append(
-            f"{row.model[:34]:<34} {row.note[:22]:<22} {task[:15]:<15} "
+            f"{row.model[:34]:<34} {row.note[:22]:<22} {tuning}{task[:15]:<15} "
             f"{row.successes:>3}/{row.runs:<4} {low:>4.0%}-{high:<5.0%} "
             f"{row.median_turns:>6.0f} {row.mean_wall_s:>6.0f}s {row.cost_per_run:>8.4f} "
             f"{row.rate_limited or '':>4}"
@@ -119,6 +153,9 @@ def render(rows: list[Row]) -> str:
             "429 = runs that waited out the proxy's rate limit; time leaves the "
             "waits out."
         )
+    if tuned:
+        lines.append("")
+        lines.append("tuning = settings that differ from today's defaults.")
     if any(row.stale for row in rows):
         lines.append("")
         lines.append(

nisse-dev/src/nisse_dev/config.py

@@ -23,7 +23,7 @@ from typing import Any
 
 import yaml
 
-from nisse_dev.llm import DEFAULT_MAX_OUTPUT_TOKENS, RoleMap
+from nisse_dev.llm import DEFAULT_MAX_OUTPUT_TOKENS, DEFAULT_REQUEST_TIMEOUT_S, RoleMap
 from nisse_dev.matrix import MatrixConfig
 
 DEFAULT_CONFIG_FILE = "nisse-dev.yaml"
@@ -48,7 +48,7 @@ REQUIRED_KEYS = (
     "gitea.repo",
     "gitea.operators",
 )
-OPTIONAL_KEYS = ("llm.max_output_tokens", "gitea.base_branch")
+OPTIONAL_KEYS = ("llm.max_output_tokens", "llm.request_timeout_s", "gitea.base_branch")
 
 _REFERENCE = re.compile(r"\$\{([A-Za-z_][A-Za-z0-9_]*)\}")
 
@@ -64,7 +64,7 @@ class ConfigError(RuntimeError):
 
 @dataclass(frozen=True)
 class ExecutorTuning:
-    max_turns: int = 24
+    max_turns: int = 40
     failure_limit: int = 2
 
 
@@ -221,6 +221,7 @@ class AgentConfig:
     gitea_operators: frozenset[str]
     base_branch: str = "main"
     max_output_tokens: int = DEFAULT_MAX_OUTPUT_TOKENS
+    request_timeout_s: float = DEFAULT_REQUEST_TIMEOUT_S
     tuning: Tuning = field(default_factory=Tuning)
     config_file: Path | None = None
 
@@ -291,6 +292,14 @@ class AgentConfig:
                 raise ValueError
         except (TypeError, ValueError):
             errors.append("llm.max_output_tokens: must be a positive integer")
+
+        request_timeout = values.get("llm.request_timeout_s", DEFAULT_REQUEST_TIMEOUT_S)
+        try:
+            request_timeout = float(request_timeout)
+            if request_timeout <= 0:
+                raise ValueError
+        except (TypeError, ValueError):
+            errors.append("llm.request_timeout_s: must be a positive number of seconds")
         if errors:
             raise ConfigError("cannot start: " + "; ".join(errors))
 
@@ -319,6 +328,7 @@ class AgentConfig:
             gitea_operators=_people(values["gitea.operators"]),
             base_branch=str(values.get("gitea.base_branch") or "main"),
             max_output_tokens=max_output,
+            request_timeout_s=request_timeout,
             tuning=tuning,
             config_file=config_file,
         )

nisse-dev/src/nisse_dev/llm.py

@@ -46,6 +46,12 @@ RATE_LIMIT_RETRY_S = 20.0
 # was observed at 4,064 against a 4,096 limit.
 DEFAULT_MAX_OUTPUT_TOKENS = 16384
 
+# How long one request may take before it is abandoned (llm.request_timeout_s).
+# A safety net for a hung connection, not a budget: a dense local model
+# writing a whole module in one turn was measured taking over 120 s, and
+# timing it out failed work that was going well.
+DEFAULT_REQUEST_TIMEOUT_S = 600.0
+
 
 class UpstreamUnavailable(LLMError):
     """The proxy or provider could not be reached, or refused for reasons
@@ -124,7 +130,7 @@ class LiteLLMClient:
         base_url: str,
         token: str,
         *,
-        timeout_s: float = 120.0,
+        timeout_s: float = DEFAULT_REQUEST_TIMEOUT_S,
         transport: httpx.BaseTransport | None = None,
         sleep: Any = time.sleep,
         max_tokens: int | None = None,
@@ -250,7 +256,7 @@ class AnthropicPassthroughClient:
         base_url: str,
         token: str,
         *,
-        timeout_s: float = 300.0,
+        timeout_s: float = DEFAULT_REQUEST_TIMEOUT_S,
         max_tokens: int | None = None,
         transport: httpx.BaseTransport | None = None,
         sleep: Any = time.sleep,

nisse-dev/src/nisse_dev/render.py

@@ -60,7 +60,7 @@ class OpenAIChatRenderer:
         kept = effective_events(events)
         answered = {e.event.call_id for e in kept if isinstance(e.event, ToolResult)}
         for stored in kept:
-            messages.append(self._message_for(stored.event))
+            _append_merged(messages, self._message_for(stored.event))
             # Providers reject a tool call with no result following it. One
             # can legitimately be missing: a crash between the call and its
             # result (A3), or a turn-ending gate tool. Render a placeholder so
@@ -128,3 +128,28 @@ class OpenAIChatRenderer:
                 }
             case _:
                 raise ValueError(f"unrenderable event: {event!r}")
+
+
+def _append_merged(messages: list[dict[str, Any]], message: dict[str, Any]) -> None:
+    """Append, folding into the previous message when both are the same user
+    or assistant role.
+
+    One model turn is often several events (some text, then one or more tool
+    calls), and the OpenAI format carries a turn as one assistant message
+    with a tool_calls list. Strict chat templates enforce that: Devstral's
+    (Mistral) rejected a turn rendered as a text message then a tool-call
+    message with a 500, failing runs whenever the model spoke before acting."""
+    previous = messages[-1] if messages else None
+    role = message["role"]
+    if (
+        previous is None
+        or previous["role"] != role
+        or role not in ("user", "assistant")
+    ):
+        messages.append(message)
+        return
+    texts = [t for t in (previous.get("content"), message.get("content")) if t]
+    previous["content"] = "\n\n".join(texts) if texts else None
+    calls = previous.get("tool_calls", []) + message.get("tool_calls", [])
+    if calls:
+        previous["tool_calls"] = calls

nisse-dev/tests/test_bench.py

@@ -149,7 +149,7 @@ def test_a_solved_task_is_a_success(tmp_path: Path) -> None:
     assert record.tokens_in == 3000
     assert record.task_hash == task.content_hash
     assert record.model_info["supports_tool_choice"] is True
-    assert record.tuning["executor"]["max_turns"] == 24
+    assert record.tuning["executor"]["max_turns"] == 40
 
 
 def test_saying_finished_is_not_success(tmp_path: Path) -> None:
@@ -317,3 +317,20 @@ def test_results_stored_before_waits_were_recorded_still_load() -> None:
     del old["rate_limit_wait_s"]
     [row] = summarise([old], current)
     assert (row.mean_wall_s, row.rate_limited) == (12.0, 0)
+
+
+def test_runs_with_different_tuning_are_kept_apart() -> None:
+    current = {t.name: t.content_hash for t in TASKS}
+    capped = record_json()
+    capped["tuning"]["executor"]["max_turns"] = 7
+    rows = summarise([record_json(), capped], current)
+    assert sorted(row.tuning for row in rows) == ["", "executor.max_turns=7"]
+    table = render(rows)
+    assert "executor.max_turns=7" in table
+    assert "tuning = settings that differ" in table
+
+
+def test_a_report_at_default_tuning_has_no_tuning_column() -> None:
+    current = {t.name: t.content_hash for t in TASKS}
+    table = render(summarise([record_json()], current))
+    assert "tuning" not in table

nisse-dev/tests/test_config.py

@@ -5,6 +5,7 @@ from typing import Any
 import pytest
 
 from nisse_dev.config import AgentConfig, ConfigError
+from nisse_dev.llm import DEFAULT_REQUEST_TIMEOUT_S
 
 # A complete config as the YAML file would give it, with secrets by reference.
 COMPLETE: dict[str, Any] = {
@@ -68,7 +69,7 @@ def test_complete_config_builds_with_secrets_by_reference() -> None:
     assert config.matrix.homeserver == "https://matrix.test"
     assert config.gitea_repo == "don/demo"
     assert config.base_branch == "main"
-    assert config.tuning.executor.max_turns == 24  # absent section: defaults
+    assert config.tuning.executor.max_turns == 40  # absent section: defaults
 
 
 def test_paths_expand_the_home_directory() -> None:
@@ -122,6 +123,25 @@ def test_max_output_tokens_is_configurable_and_checked() -> None:
         build(llm__max_output_tokens=0)
 
 
+def test_request_timeout_is_configurable_and_checked() -> None:
+    assert build().request_timeout_s == DEFAULT_REQUEST_TIMEOUT_S
+    assert build(llm__request_timeout_s=900).request_timeout_s == 900.0
+    for bad in (0, -5, "soon"):
+        with pytest.raises(ConfigError, match="llm.request_timeout_s"):
+            build(llm__request_timeout_s=bad)
+
+
+def test_the_request_timeout_reaches_both_clients() -> None:
+    from nisse_dev.agent import make_llm_clients
+    from nisse_dev.catalog import ModelCatalog
+
+    normalized, passthrough = make_llm_clients(
+        build(llm__request_timeout_s=42), ModelCatalog({})
+    )
+    for client in (normalized, passthrough):
+        assert client._inner._http.timeout.read == 42.0
+
+
 def test_redacted_summary_carries_no_secrets() -> None:
     summary = build().redacted()
     flat = " ".join(summary.values())
@@ -252,7 +272,7 @@ def test_the_example_file_is_valid() -> None:
         "NISSE_MATRIX_TOKEN": "syt_x",
         "NISSE_GITEA_TOKEN": "gt",
     }
-    assert AgentConfig.from_file(example, env).tuning.executor.max_turns == 24
+    assert AgentConfig.from_file(example, env).tuning.executor.max_turns == 40
 
 
 # --- nothing that carries secrets may sit inside the clone -----------------

nisse-dev/tests/test_render.py

@@ -1,4 +1,5 @@
 import json
+from itertools import pairwise
 
 from nisse_dev.canonical import (
     AssistantText,
@@ -68,7 +69,8 @@ def test_compaction_replaces_its_range_only() -> None:
     assert [e.id for e in remaining] == [3, 4]
     body = OpenAIChatRenderer().render(events, tools=None, model="q")
     contents = [m["content"] for m in body["messages"]]
-    assert contents == ["kept", "[summary of earlier conversation]\nthe old stuff"]
+    # Both render as user messages, so they share one: roles must alternate.
+    assert contents == ["kept\n\n[summary of earlier conversation]\nthe old stuff"]
 
 
 def test_tool_call_without_a_result_gets_a_placeholder() -> None:
@@ -103,3 +105,47 @@ def test_gate_event_renders_as_context() -> None:
     body = OpenAIChatRenderer().render(events, tools=None, model="q")
     assert body["messages"][0]["role"] == "user"
     assert "arch" in body["messages"][0]["content"]
+
+
+def _roles(body: dict) -> list[str]:
+    return [m["role"] for m in body["messages"]]
+
+
+def test_one_turn_is_one_assistant_message() -> None:
+    # Text and two parallel tool calls from one model turn: the OpenAI format
+    # carries them as a single assistant message with a tool_calls list.
+    events = stored(
+        [
+            UserMessage(text="go", source="m:1"),
+            AssistantText(text="Looking around first.", model="q"),
+            ToolCall(call_id="a", tool="list_dir", arguments={"path": "."}),
+            ToolCall(call_id="b", tool="read_file", arguments={"path": "README.md"}),
+            ToolResult(call_id="a", result="README.md", success=True),
+            ToolResult(call_id="b", result="hello", success=True),
+        ]
+    )
+    body = OpenAIChatRenderer().render(events, tools=None, model="q")
+    assert _roles(body) == ["user", "assistant", "tool", "tool"]
+    turn = body["messages"][1]
+    assert turn["content"] == "Looking around first."
+    assert [c["id"] for c in turn["tool_calls"]] == ["a", "b"]
+
+
+def test_roles_never_repeat_for_strict_templates() -> None:
+    """Mistral's chat template raises on two user or two assistant messages
+    in a row; every mix of events must render without one."""
+    events = stored(
+        [
+            UserMessage(text="one", source="m:1"),
+            UserMessage(text="two", source="m:2"),
+            AssistantText(text="thinking", model="q"),
+            ToolCall(call_id="a", tool="list_dir", arguments={}),
+            ToolResult(call_id="a", result="x", success=True),
+            AssistantText(text="more", model="q"),
+            AssistantText(text="and more", model="q"),
+            ToolCall(call_id="b", tool="finish", arguments={}),
+        ]
+    )
+    roles = _roles(OpenAIChatRenderer().render(events, tools=None, model="q"))
+    for first, second in pairwise(roles):
+        assert not (first == second and first in ("user", "assistant")), roles