M2.8: switch over to the new loop

nisse-dev now runs only the conversational loop. The gated state machine
it was built alongside is gone:

- Removed control.py, planner.py, executor.py, approvals.py, gates.py
  (command_form moves to loop/reader.py), publish.py and poller.py, the
  old assembly in agent.py (its shared helpers move to loop/build.py),
  the store's state-machine view (get_state, transition), their tests,
  the old end-to-end and live planning/milestone/gate tests, and
  specs/orchestrator.qnt with its scenarios.
- Roles are driver, helper, expert and classifier; driver is required.
  loop:, roles.planner, roles.executor and tuning.executor are refused
  with what to use instead; tuning.driver takes the turn cap and failure
  limit.
- The benchmark and the capability matrix measure the driver: the
  loop's own system prompt and tools, the task's plan as the agreed
  restatement. Task plans end with finish_milestone, so earlier runs
  show as (old); stored runs' executor tuning reads as the driver's.
- A delegation on the loop records what its helper's alias resolved to
  (L9), as the old path's did.
- The dump shows a work item's revision and what it waits for.
- Behaviour the old tests covered and the loop shares stays covered:
  driver runs that fail on a model call, stop mid-turn, reset the
  failure count; go-aheads ignoring case and punctuation; a request too
  big to fit is never sent; the assembly tests, now on build_loop_agent.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01PPQ14L3LikyefCMFBqEbKE

nisse-dev/src/nisse_dev/bench/runner.py

@@ -1,4 +1,4 @@
-"""Running one task once with one executor model, and recording it."""
+"""Running one task once with one driver model, and recording it."""
 
 import io
 import shlex
@@ -14,7 +14,7 @@ from typing import Any
 from nisse_dev.bench.tasks import Task
 from nisse_dev.canonical import (
     CanonicalEvent,
-    PlanSubmitted,
+    Fact,
     StoredEvent,
     ToolCall,
     ToolResult,
@@ -23,11 +23,37 @@ from nisse_dev.canonical import (
 from nisse_dev.catalog import ModelFacts
 from nisse_dev.config import Tuning
 from nisse_dev.execution import SSHHost
-from nisse_dev.executor import LLMExecutor
-from nisse_dev.llm import RoleMap
+from nisse_dev.loop.agent import SYSTEM
+from nisse_dev.loop.driver import Driver, RunOutcome
+from nisse_dev.loop.tools import (
+    ASK_HUMAN,
+    CLOSE_ITEM,
+    FINISH_MILESTONE,
+    START_MILESTONE,
+)
 from nisse_dev.oplog import OpLog
 from nisse_dev.state import StateStore
-from nisse_dev.tools import Workspace
+from nisse_dev.tools import EXECUTION_TOOLS, Workspace
+
+# What the driver is offered once an item is agreed, less what the benchmark
+# turns off: delegation and ask_expert (an escalation is the model's failure,
+# not a frontier model's success) and review, which has no PR here.
+BENCH_TOOLS = [
+    ASK_HUMAN,
+    START_MILESTONE,
+    FINISH_MILESTONE,
+    CLOSE_ITEM,
+    *EXECUTION_TOOLS,
+]
+
+
+def outcome_of(run: RunOutcome) -> str:
+    """A driver run as the benchmark records it: ``completed`` when the
+    driver says the work is done, ``escalated`` when it asks the operator,
+    otherwise how the run ended (capped, stuck, failed)."""
+    if run.kind == "wait":
+        return "escalated" if run.tool == "ask_human" else "completed"
+    return run.kind
 
 
 class MeteringClient:
@@ -158,11 +184,13 @@ def run_task(
     commit: str = "",
     workdir: Path | None = None,
 ) -> RunRecord:
-    """Run one task once: seed a workspace, hand the executor the approved
-    plan, then judge the result with the hidden check.
+    """Run one task once: seed a workspace, hand the driver an agreed item
+    whose restatement is the task's plan, then judge the result with the
+    hidden check.
 
-    Delegation and ask_expert are off: an escalation is a failure of the
-    model under test, not a frontier model's success."""
+    It is the loop's own driver, system prompt and tools, so what is measured
+    is what the agent would do, without the harness around it: the milestone
+    is started but no branch is cut, and finishing it ends the run."""
     with tempfile.TemporaryDirectory(dir=workdir) as tmp:
         root = Path(tmp)
         # The workspace is on the build VM, like the agent's own (L13).
@@ -174,39 +202,51 @@ def run_task(
         )
         task.seed_workspace(workspace)
         store = StateStore(root / "state.db", oplog=OpLog("nisse-bench", io.StringIO()))
-        # One work item per run, as the agent's own tasks are (L1).
+        # One work item per run, as the agent's own are (L1).
         store.open_work_item(
-            "bench", title=task.name, position="EXECUTING", cause="benchmark run"
+            "bench", title=task.name, position="RUNNING", cause="benchmark run"
         )
-        store.transcript("bench", "planner").append(
-            PlanSubmitted(plan=task.plan, model="benchmark")
+        store.transcript("bench", "orchestrator").append(
+            Fact(text=f"The operator agreed to this restatement:\n\n{task.plan}")
         )
         store.transcript("bench", "operator").append(
-            UserMessage(text="start milestone 1", source="benchmark")
+            UserMessage(text="go", source="benchmark")
         )
 
         meter = MeteringClient(client, facts)
         trims_before = getattr(client, "trimmed_requests", 0)
         model = facts.alias
-        roles = RoleMap(executor=model, planner=model, helper=model, expert=model)
-        executor = LLMExecutor(
+        driver = Driver(
             meter,
-            roles,
-            workspace,
-            max_turns=tuning.executor.max_turns,
-            failure_limit=tuning.executor.failure_limit,
+            model,
+            max_turns=tuning.driver.max_turns,
+            failure_limit=tuning.driver.failure_limit,
+        )
+        system = SYSTEM.format(
+            repo=f"bench/{task.name}",
+            issue=1,
+            title=task.title,
+            labels="",
+            helpers="",
+            workspace=workspace.describe(),
         )
         started = time.monotonic()
         try:
-            outcome = executor.run(store.transcript("bench", "executor"), 1)
-            status, detail = outcome.status, outcome.note
+            run = driver.run(
+                store.transcript("bench", "driver"),
+                system=system,
+                tools=BENCH_TOOLS,
+                workspace=workspace,
+                handlers={"start_milestone": lambda args: ("started", True)},
+            )
+            status, detail = outcome_of(run), run.note
         except Exception as exc:  # noqa: BLE001 -- recorded, the benchmark goes on
             status, detail = "error", repr(exc)
         wall = time.monotonic() - started
 
         success, check_output = task.check_workspace(workspace)
         host.execute(f"rm -rf -- {shlex.quote(str(workspace.root))}", timeout_s=60)
-        failed, invented = _count(store.transcript("bench", "executor").read())
+        failed, invented = _count(store.transcript("bench", "driver").read())
         store.close()
 
     return RunRecord(