nisse-dev/src/nisse_dev/executor.py

@@ -30,8 +30,13 @@ plain-text summary and no tool call — that ends the milestone."""
 DEFAULT_MAX_TURNS = 24
 DEFAULT_FAILURE_LIMIT = 2
 
+# Tools that change or verify something. Reading a file or listing a
+# directory is the model looking around, not doing the work, so neither
+# counts as evidence that a milestone was carried out.
+MUTATING_TOOLS = frozenset({"write_file", "run_command"})
+
 # A turn with no tool call means "the plan is done". Believing that before
-# any tool has been called would record an empty milestone as a success, so
+# any real work has happened would record an empty milestone as a success, so
 # the claim is challenged once first.
 _NUDGE = (
     "\n\nYou have not made any change yet, so the plan cannot be complete. "
@@ -105,7 +110,8 @@ class LLMExecutor:
                 )
 
             for call in calls:
-                did_work = True
+                if call.tool in MUTATING_TOOLS:
+                    did_work = True
                 outcome = self._workspace.dispatch(call.tool, call.arguments)
                 self._store.append_event(
                     ToolResult(

nisse-dev/tests/test_executor.py

@@ -161,11 +161,14 @@ def test_a_success_resets_the_failure_count(store: StateStore, ws: Workspace) ->
 def test_failed_reads_do_not_count_toward_escalation(
     store: StateStore, ws: Workspace
 ) -> None:
+    # Two failed reads in a row must not trip the failure limit: looking for
+    # a file that is not there is the model orienting itself, not failing.
     client = ScriptedClient(
         [
             [call("read_file", path="missing-a.txt")],
             [call("read_file", path="missing-b.txt")],
-            [AssistantText(text="looked around, done", model="m")],
+            [call("write_file", path="a.py", content="x")],
+            [AssistantText(text="done", model="m")],
         ]
     )
     assert build(client, store, ws, failure_limit=2).run(milestone=1).status == (
@@ -173,6 +176,20 @@ def test_failed_reads_do_not_count_toward_escalation(
     )
 
 
+def test_reading_alone_is_not_doing_the_work(store: StateStore, ws: Workspace) -> None:
+    # Exploration is not evidence a milestone was carried out, so a
+    # completion claim backed only by reads is still challenged.
+    client = ScriptedClient(
+        [
+            [call("list_dir", path=".")],
+            [AssistantText(text="looks done to me", model="m")],
+            [AssistantText(text="really, done", model="m")],
+        ]
+    )
+    outcome = build(client, store, ws).run(milestone=1)
+    assert outcome.status == "escalate"
+
+
 def test_turn_cap_is_enforced(store: StateStore, ws: Workspace) -> None:
     forever = [[call("list_dir", path=".")] for _ in range(50)]
     client = ScriptedClient(forever)