M2.8: switch over to the new loop

nisse-dev now runs only the conversational loop. The gated state machine
it was built alongside is gone:

- Removed control.py, planner.py, executor.py, approvals.py, gates.py
  (command_form moves to loop/reader.py), publish.py and poller.py, the
  old assembly in agent.py (its shared helpers move to loop/build.py),
  the store's state-machine view (get_state, transition), their tests,
  the old end-to-end and live planning/milestone/gate tests, and
  specs/orchestrator.qnt with its scenarios.
- Roles are driver, helper, expert and classifier; driver is required.
  loop:, roles.planner, roles.executor and tuning.executor are refused
  with what to use instead; tuning.driver takes the turn cap and failure
  limit.
- The benchmark and the capability matrix measure the driver: the
  loop's own system prompt and tools, the task's plan as the agreed
  restatement. Task plans end with finish_milestone, so earlier runs
  show as (old); stored runs' executor tuning reads as the driver's.
- A delegation on the loop records what its helper's alias resolved to
  (L9), as the old path's did.
- The dump shows a work item's revision and what it waits for.
- Behaviour the old tests covered and the loop shares stays covered:
  driver runs that fail on a model call, stop mid-turn, reset the
  failure count; go-aheads ignoring case and punctuation; a request too
  big to fit is never sent; the assembly tests, now on build_loop_agent.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01PPQ14L3LikyefCMFBqEbKE

nisse-dev/src/nisse_dev/bench/__init__.py

@@ -1,6 +1,6 @@
-"""The executor benchmark: canned milestones, run repeatedly per model.
+"""The driver benchmark: canned milestones, run repeatedly per model.
 
-Choosing an executor model, and later tuning its knobs, should rest on
+Choosing a driver model, and later tuning its knobs, should rest on
 measured success rates, not on one run or on reputation. Each task is a fixed
 approved plan over a seed project, with a hidden check the model never sees;
 success means the check passes, not that the model said it finished. Runs use

nisse-dev/src/nisse_dev/bench/results.py

@@ -74,12 +74,17 @@ class Row:
 
 def tuning_label(tuning: dict[str, Any]) -> str:
     """The settings in a run's tuning that differ from today's defaults, as
-    "executor.max_turns=24"; empty when there are none. Relative to today's
-    defaults so that rows at the current settings stay unlabelled."""
+    "driver.max_turns=24"; empty when there are none. Relative to today's
+    defaults so that rows at the current settings stay unlabelled. Runs from
+    before M2.8 called the driver's section ``executor``."""
     defaults = asdict(Tuning())
+    renamed = {
+        ("driver" if section == "executor" else section): values
+        for section, values in tuning.items()
+    }
     return ", ".join(
         f"{section}.{key}={value}"
-        for section, values in sorted(tuning.items())
+        for section, values in sorted(renamed.items())
         for key, value in sorted((values or {}).items())
         if defaults.get(section, {}).get(key) != value
     )

nisse-dev/src/nisse_dev/bench/runner.py

@@ -1,4 +1,4 @@
-"""Running one task once with one executor model, and recording it."""
+"""Running one task once with one driver model, and recording it."""
 
 import io
 import shlex
@@ -14,7 +14,7 @@ from typing import Any
 from nisse_dev.bench.tasks import Task
 from nisse_dev.canonical import (
     CanonicalEvent,
-    PlanSubmitted,
+    Fact,
     StoredEvent,
     ToolCall,
     ToolResult,
@@ -23,11 +23,37 @@ from nisse_dev.canonical import (
 from nisse_dev.catalog import ModelFacts
 from nisse_dev.config import Tuning
 from nisse_dev.execution import SSHHost
-from nisse_dev.executor import LLMExecutor
-from nisse_dev.llm import RoleMap
+from nisse_dev.loop.agent import SYSTEM
+from nisse_dev.loop.driver import Driver, RunOutcome
+from nisse_dev.loop.tools import (
+    ASK_HUMAN,
+    CLOSE_ITEM,
+    FINISH_MILESTONE,
+    START_MILESTONE,
+)
 from nisse_dev.oplog import OpLog
 from nisse_dev.state import StateStore
-from nisse_dev.tools import Workspace
+from nisse_dev.tools import EXECUTION_TOOLS, Workspace
+
+# What the driver is offered once an item is agreed, less what the benchmark
+# turns off: delegation and ask_expert (an escalation is the model's failure,
+# not a frontier model's success) and review, which has no PR here.
+BENCH_TOOLS = [
+    ASK_HUMAN,
+    START_MILESTONE,
+    FINISH_MILESTONE,
+    CLOSE_ITEM,
+    *EXECUTION_TOOLS,
+]
+
+
+def outcome_of(run: RunOutcome) -> str:
+    """A driver run as the benchmark records it: ``completed`` when the
+    driver says the work is done, ``escalated`` when it asks the operator,
+    otherwise how the run ended (capped, stuck, failed)."""
+    if run.kind == "wait":
+        return "escalated" if run.tool == "ask_human" else "completed"
+    return run.kind
 
 
 class MeteringClient:
@@ -158,11 +184,13 @@ def run_task(
     commit: str = "",
     workdir: Path | None = None,
 ) -> RunRecord:
-    """Run one task once: seed a workspace, hand the executor the approved
-    plan, then judge the result with the hidden check.
+    """Run one task once: seed a workspace, hand the driver an agreed item
+    whose restatement is the task's plan, then judge the result with the
+    hidden check.
 
-    Delegation and ask_expert are off: an escalation is a failure of the
-    model under test, not a frontier model's success."""
+    It is the loop's own driver, system prompt and tools, so what is measured
+    is what the agent would do, without the harness around it: the milestone
+    is started but no branch is cut, and finishing it ends the run."""
     with tempfile.TemporaryDirectory(dir=workdir) as tmp:
         root = Path(tmp)
         # The workspace is on the build VM, like the agent's own (L13).
@@ -174,39 +202,51 @@ def run_task(
         )
         task.seed_workspace(workspace)
         store = StateStore(root / "state.db", oplog=OpLog("nisse-bench", io.StringIO()))
-        # One work item per run, as the agent's own tasks are (L1).
+        # One work item per run, as the agent's own are (L1).
         store.open_work_item(
-            "bench", title=task.name, position="EXECUTING", cause="benchmark run"
+            "bench", title=task.name, position="RUNNING", cause="benchmark run"
         )
-        store.transcript("bench", "planner").append(
-            PlanSubmitted(plan=task.plan, model="benchmark")
+        store.transcript("bench", "orchestrator").append(
+            Fact(text=f"The operator agreed to this restatement:\n\n{task.plan}")
         )
         store.transcript("bench", "operator").append(
-            UserMessage(text="start milestone 1", source="benchmark")
+            UserMessage(text="go", source="benchmark")
         )
 
         meter = MeteringClient(client, facts)
         trims_before = getattr(client, "trimmed_requests", 0)
         model = facts.alias
-        roles = RoleMap(executor=model, planner=model, helper=model, expert=model)
-        executor = LLMExecutor(
+        driver = Driver(
             meter,
-            roles,
-            workspace,
-            max_turns=tuning.executor.max_turns,
-            failure_limit=tuning.executor.failure_limit,
+            model,
+            max_turns=tuning.driver.max_turns,
+            failure_limit=tuning.driver.failure_limit,
+        )
+        system = SYSTEM.format(
+            repo=f"bench/{task.name}",
+            issue=1,
+            title=task.title,
+            labels="",
+            helpers="",
+            workspace=workspace.describe(),
         )
         started = time.monotonic()
         try:
-            outcome = executor.run(store.transcript("bench", "executor"), 1)
-            status, detail = outcome.status, outcome.note
+            run = driver.run(
+                store.transcript("bench", "driver"),
+                system=system,
+                tools=BENCH_TOOLS,
+                workspace=workspace,
+                handlers={"start_milestone": lambda args: ("started", True)},
+            )
+            status, detail = outcome_of(run), run.note
         except Exception as exc:  # noqa: BLE001 -- recorded, the benchmark goes on
             status, detail = "error", repr(exc)
         wall = time.monotonic() - started
 
         success, check_output = task.check_workspace(workspace)
         host.execute(f"rm -rf -- {shlex.quote(str(workspace.root))}", timeout_s=60)
-        failed, invented = _count(store.transcript("bench", "executor").read())
+        failed, invented = _count(store.transcript("bench", "driver").read())
         store.close()
 
     return RunRecord(

nisse-dev/src/nisse_dev/bench/tasks.py

@@ -3,11 +3,11 @@
 Each task is a directory under ``tasks/``:
 
 - ``task.yaml``: ``title``, ``plan`` (the approved milestone plan, written the
-  way the planner is asked to write them), and ``check`` (a shell command run
+  way the driver restates them), and ``check`` (a shell command run
   in the workspace afterwards).
-- ``seed/``: the project the executor starts from.
+- ``seed/``: the project the driver starts from.
 - ``hidden/``: files the check needs, copied in only after the run, into
-  ``.bench_check/``. The executor never sees them.
+  ``.bench_check/``. The driver never sees them.
 - ``solution/``: a reference solution, overlaid on the seed by the tests to
   prove each task can be solved and that its check tells the difference.
 """

nisse-dev/src/nisse_dev/bench/tasks/fix-bug/task.yaml

@@ -9,4 +9,4 @@ plan: |
   3. Fix `total_value` so that each item contributes its unit price times its
      quantity.
   4. Run `python -m pytest -q` until every test passes.
-  5. Call finish.
+  5. Call finish_milestone, with final yes.

nisse-dev/src/nisse_dev/bench/tasks/long-milestone/task.yaml

@@ -37,4 +37,4 @@ plan: |
      then the same with `list --overdue`, which must print
      `[!] 1 Pay rent (due 2026-10-01)`. Delete /tmp/tracker-check.json
      before and after.
-  9. Call finish.
+  9. Call finish_milestone, with final yes.

nisse-dev/src/nisse_dev/bench/tasks/long-output/task.yaml

@@ -9,4 +9,4 @@ plan: |
      (NNN mod 25) / 100. Correct only the failing region's entry; leave every
      other line as it is.
   3. Run `python -m pytest -q` until every test passes.
-  4. Call finish.
+  4. Call finish_milestone, with final yes.

nisse-dev/src/nisse_dev/bench/tasks/multi-file/task.yaml

@@ -13,4 +13,4 @@ plan: |
      using pytest's capsys fixture.
   4. Run `python -m pytest -q` until every test passes, and check that
      `python -m greeter --shout Bob` prints `HELLO, BOB!`.
-  5. Call finish.
+  5. Call finish_milestone, with final yes.

nisse-dev/src/nisse_dev/bench/tasks/refactor-rename/task.yaml

@@ -15,4 +15,4 @@ plan: |
      are different functions and keep their names. Change no behaviour.
   4. Update the tests under `tests/` to use the new name, and run
      `python -m pytest -q` until every test passes.
-  5. Call finish.
+  5. Call finish_milestone, with final yes.

nisse-dev/src/nisse_dev/bench/tasks/spec-parser/task.yaml

@@ -12,4 +12,4 @@ plan: |
      invalid input for each of the eight rules.
   4. Run `python -m pytest -q test_durations.py`, and fix `durations.py`
      until every test passes.
-  5. Call finish.
+  5. Call finish_milestone, with final yes.

nisse-dev/src/nisse_dev/bench/tasks/trace-bug/task.yaml

@@ -18,4 +18,4 @@ plan: |
      `invoicing/currency.py` have up to six decimal places (JPY is 0.006712),
      and no step may round a rate or fraction to fewer places than that.
   4. Run `python -m pytest -q` until every test passes.
-  5. Call finish.
+  5. Call finish_milestone, with final yes.

nisse-dev/src/nisse_dev/bench/tasks/write-function/task.yaml

@@ -10,4 +10,4 @@ plan: |
      "a--b" -> "a-b"; "" -> "".
   3. Create `test_slugify.py` with pytest tests covering the four examples.
   4. Run `python -m pytest -q test_slugify.py`, and fix anything until it passes.
-  5. Call finish.
+  5. Call finish_milestone, with final yes.