nisse-dev/src/nisse_dev/executor.py

@@ -30,6 +30,15 @@ plain-text summary and no tool call — that ends the milestone."""
 DEFAULT_MAX_TURNS = 24
 DEFAULT_FAILURE_LIMIT = 2
 
+# A turn with no tool call means "the plan is done". Believing that before
+# any tool has been called would record an empty milestone as a success, so
+# the claim is challenged once first.
+_NUDGE = (
+    "\n\nYou have not made any change yet, so the plan cannot be complete. "
+    "Carry out the next step of the plan with a tool call, or run the tests "
+    "to show the work is already done."
+)
+
 
 @dataclass(frozen=True)
 class ExecutorOutcome:
@@ -59,6 +68,8 @@ class LLMExecutor:
         model = self._roles.resolve("executor")
         system = f"{EXECUTOR_SYSTEM}\n\nYou are executing milestone {milestone}."
         consecutive_failures = 0
+        did_work = False
+        challenged = False
 
         for _turn in range(self._max_turns):
             try:
@@ -78,9 +89,23 @@ class LLMExecutor:
                     calls.append(event)
 
             if not calls:
-                return ExecutorOutcome("completed", "executor reported the plan done")
+                if did_work:
+                    return ExecutorOutcome(
+                        "completed", "executor reported the plan done"
+                    )
+                if not challenged:
+                    # Claiming completion without having touched anything is
+                    # not evidence of completion; ask once before believing it.
+                    challenged = True
+                    system += _NUDGE
+                    continue
+                return ExecutorOutcome(
+                    "escalate",
+                    "executor claimed the milestone was done without making any change",
+                )
 
             for call in calls:
+                did_work = True
                 outcome = self._workspace.dispatch(call.tool, call.arguments)
                 self._store.append_event(
                     ToolResult(

nisse-dev/tests/test_executor.py

@@ -88,11 +88,46 @@ def test_uses_executor_role_and_reduced_manifest(
     assert "milestone 2" in client.calls[0]["system"]
 
 
-def test_no_tool_call_ends_the_milestone(store: StateStore, ws: Workspace) -> None:
-    client = ScriptedClient([[AssistantText(text="all steps done", model="m")]])
+def test_no_tool_call_after_work_ends_the_milestone(
+    store: StateStore, ws: Workspace
+) -> None:
+    client = ScriptedClient(
+        [
+            [call("write_file", path="a.py", content="x")],
+            [AssistantText(text="all steps done", model="m")],
+        ]
+    )
     assert build(client, store, ws).run(milestone=1).status == "completed"
 
 
+def test_completion_claimed_without_any_work_is_challenged(
+    store: StateStore, ws: Workspace
+) -> None:
+    # A model that answers with prose on turn one has done nothing; believing
+    # it would record an empty milestone as a success.
+    client = ScriptedClient(
+        [
+            [AssistantText(text="looks done to me", model="m")],
+            [call("write_file", path="a.py", content="x")],
+            [AssistantText(text="now it is done", model="m")],
+        ]
+    )
+    outcome = build(client, store, ws).run(milestone=1)
+    assert outcome.status == "completed"
+    assert (ws.root / "a.py").exists()  # the nudge produced actual work
+    assert "not made any change yet" in client.calls[1]["system"]
+
+
+def test_persisting_in_an_empty_claim_escalates(
+    store: StateStore, ws: Workspace
+) -> None:
+    prose = [AssistantText(text="nothing to do", model="m")]
+    client = ScriptedClient([prose, prose])
+    outcome = build(client, store, ws).run(milestone=1)
+    assert outcome.status == "escalate"
+    assert "without making any change" in outcome.note
+
+
 def test_two_consecutive_command_failures_escalate(
     store: StateStore, ws: Workspace
 ) -> None: