From 886cb80583266c1e15474dcb33097a1e3bf3cbc0 Mon Sep 17 00:00:00 2001 From: Don Garrett Date: Mon, 7 Sep 2026 22:53:20 +0000 Subject: [PATCH] Count only mutating tools as evidence of work MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The previous fix challenged a completion claim made before any tool call, but treated every tool as work — so a model that ran one list_dir and then declared the milestone done was believed, and CI caught the same empty milestone again. Reading a file or listing a directory is the model orienting itself. Only write_file and run_command now count, so a completion claim must be backed by something that changed or verified the tree. Co-Authored-By: Claude Opus 5 --- nisse-dev/src/nisse_dev/executor.py | 10 ++++++++-- nisse-dev/tests/test_executor.py | 19 ++++++++++++++++++- 2 files changed, 26 insertions(+), 3 deletions(-) diff --git a/nisse-dev/src/nisse_dev/executor.py b/nisse-dev/src/nisse_dev/executor.py index c622282..4e647ff 100644 --- a/nisse-dev/src/nisse_dev/executor.py +++ b/nisse-dev/src/nisse_dev/executor.py @@ -30,8 +30,13 @@ plain-text summary and no tool call — that ends the milestone.""" DEFAULT_MAX_TURNS = 24 DEFAULT_FAILURE_LIMIT = 2 +# Tools that change or verify something. Reading a file or listing a +# directory is the model looking around, not doing the work, so neither +# counts as evidence that a milestone was carried out. +MUTATING_TOOLS = frozenset({"write_file", "run_command"}) + # A turn with no tool call means "the plan is done". Believing that before -# any tool has been called would record an empty milestone as a success, so +# any real work has happened would record an empty milestone as a success, so # the claim is challenged once first. _NUDGE = ( "\n\nYou have not made any change yet, so the plan cannot be complete. " @@ -105,7 +110,8 @@ class LLMExecutor: ) for call in calls: - did_work = True + if call.tool in MUTATING_TOOLS: + did_work = True outcome = self._workspace.dispatch(call.tool, call.arguments) self._store.append_event( ToolResult( diff --git a/nisse-dev/tests/test_executor.py b/nisse-dev/tests/test_executor.py index 409160f..5d91924 100644 --- a/nisse-dev/tests/test_executor.py +++ b/nisse-dev/tests/test_executor.py @@ -161,11 +161,14 @@ def test_a_success_resets_the_failure_count(store: StateStore, ws: Workspace) -> def test_failed_reads_do_not_count_toward_escalation( store: StateStore, ws: Workspace ) -> None: + # Two failed reads in a row must not trip the failure limit: looking for + # a file that is not there is the model orienting itself, not failing. client = ScriptedClient( [ [call("read_file", path="missing-a.txt")], [call("read_file", path="missing-b.txt")], - [AssistantText(text="looked around, done", model="m")], + [call("write_file", path="a.py", content="x")], + [AssistantText(text="done", model="m")], ] ) assert build(client, store, ws, failure_limit=2).run(milestone=1).status == ( @@ -173,6 +176,20 @@ def test_failed_reads_do_not_count_toward_escalation( ) +def test_reading_alone_is_not_doing_the_work(store: StateStore, ws: Workspace) -> None: + # Exploration is not evidence a milestone was carried out, so a + # completion claim backed only by reads is still challenged. + client = ScriptedClient( + [ + [call("list_dir", path=".")], + [AssistantText(text="looks done to me", model="m")], + [AssistantText(text="really, done", model="m")], + ] + ) + outcome = build(client, store, ws).run(milestone=1) + assert outcome.status == "escalate" + + def test_turn_cap_is_enforced(store: StateStore, ws: Workspace) -> None: forever = [[call("list_dir", path=".")] for _ in range(50)] client = ScriptedClient(forever)