Do not believe a completion claim before any work was done

CI caught the executor recording an empty milestone as a success: the
model answered with prose on its first turn, and the loop reads "no
tool call" as "the plan is done". It had therefore finished a
milestone having touched nothing, and the milestone would have gone
on to publish with no changes.

A completion claim is now challenged once when no tool has been
called yet — the same shape as the planner's nudge — and a model that
persists escalates instead of reporting success. After the first tool
call the claim is taken at face value, as before.

The live milestone test failed roughly one run in five on this; four
consecutive runs pass now.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>

nisse-dev/src/nisse_dev/executor.py

@@ -30,6 +30,15 @@ plain-text summary and no tool call — that ends the milestone."""
 DEFAULT_MAX_TURNS = 24
 DEFAULT_FAILURE_LIMIT = 2
 
+# A turn with no tool call means "the plan is done". Believing that before
+# any tool has been called would record an empty milestone as a success, so
+# the claim is challenged once first.
+_NUDGE = (
+    "\n\nYou have not made any change yet, so the plan cannot be complete. "
+    "Carry out the next step of the plan with a tool call, or run the tests "
+    "to show the work is already done."
+)
+
 
 @dataclass(frozen=True)
 class ExecutorOutcome:
@@ -59,6 +68,8 @@ class LLMExecutor:
         model = self._roles.resolve("executor")
         system = f"{EXECUTOR_SYSTEM}\n\nYou are executing milestone {milestone}."
         consecutive_failures = 0
+        did_work = False
+        challenged = False
 
         for _turn in range(self._max_turns):
             try:
@@ -78,9 +89,23 @@ class LLMExecutor:
                     calls.append(event)
 
             if not calls:
-                return ExecutorOutcome("completed", "executor reported the plan done")
+                if did_work:
+                    return ExecutorOutcome(
+                        "completed", "executor reported the plan done"
+                    )
+                if not challenged:
+                    # Claiming completion without having touched anything is
+                    # not evidence of completion; ask once before believing it.
+                    challenged = True
+                    system += _NUDGE
+                    continue
+                return ExecutorOutcome(
+                    "escalate",
+                    "executor claimed the milestone was done without making any change",
+                )
 
             for call in calls:
+                did_work = True
                 outcome = self._workspace.dispatch(call.tool, call.arguments)
                 self._store.append_event(
                     ToolResult(

nisse-dev/tests/test_executor.py

@@ -88,11 +88,46 @@ def test_uses_executor_role_and_reduced_manifest(
     assert "milestone 2" in client.calls[0]["system"]
 
 
-def test_no_tool_call_ends_the_milestone(store: StateStore, ws: Workspace) -> None:
-    client = ScriptedClient([[AssistantText(text="all steps done", model="m")]])
+def test_no_tool_call_after_work_ends_the_milestone(
+    store: StateStore, ws: Workspace
+) -> None:
+    client = ScriptedClient(
+        [
+            [call("write_file", path="a.py", content="x")],
+            [AssistantText(text="all steps done", model="m")],
+        ]
+    )
     assert build(client, store, ws).run(milestone=1).status == "completed"
 
 
+def test_completion_claimed_without_any_work_is_challenged(
+    store: StateStore, ws: Workspace
+) -> None:
+    # A model that answers with prose on turn one has done nothing; believing
+    # it would record an empty milestone as a success.
+    client = ScriptedClient(
+        [
+            [AssistantText(text="looks done to me", model="m")],
+            [call("write_file", path="a.py", content="x")],
+            [AssistantText(text="now it is done", model="m")],
+        ]
+    )
+    outcome = build(client, store, ws).run(milestone=1)
+    assert outcome.status == "completed"
+    assert (ws.root / "a.py").exists()  # the nudge produced actual work
+    assert "not made any change yet" in client.calls[1]["system"]
+
+
+def test_persisting_in_an_empty_claim_escalates(
+    store: StateStore, ws: Workspace
+) -> None:
+    prose = [AssistantText(text="nothing to do", model="m")]
+    client = ScriptedClient([prose, prose])
+    outcome = build(client, store, ws).run(milestone=1)
+    assert outcome.status == "escalate"
+    assert "without making any change" in outcome.note
+
+
 def test_two_consecutive_command_failures_escalate(
     store: StateStore, ws: Workspace
 ) -> None: