Count only mutating tools as evidence of work
CI / checks (push) Successful in 38s
CI / checks (pull_request) Successful in 41s

The previous fix challenged a completion claim made before any tool
call, but treated every tool as work — so a model that ran one
list_dir and then declared the milestone done was believed, and CI
caught the same empty milestone again.

Reading a file or listing a directory is the model orienting itself.
Only write_file and run_command now count, so a completion claim must
be backed by something that changed or verified the tree.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
dongarandClaude Opus 5 committed 2026-09-07 22:53:20 +00:00
1 parent 8f365263c4
commit 886cb80583
2 files changed
+26 -3

No files matched your search

+18 -1
View File
@@ -161,11 +161,14 @@ def test_a_success_resets_the_failure_count(store: StateStore, ws: Workspace) ->
def test_failed_reads_do_not_count_toward_escalation(
store: StateStore, ws: Workspace
) -> None:
# Two failed reads in a row must not trip the failure limit: looking for
# a file that is not there is the model orienting itself, not failing.
client = ScriptedClient(
[
[call("read_file", path="missing-a.txt")],
[call("read_file", path="missing-b.txt")],
[AssistantText(text="looked around, done", model="m")],
[call("write_file", path="a.py", content="x")],
[AssistantText(text="done", model="m")],
]
)
assert build(client, store, ws, failure_limit=2).run(milestone=1).status == (
@@ -173,6 +176,20 @@ def test_failed_reads_do_not_count_toward_escalation(
)
def test_reading_alone_is_not_doing_the_work(store: StateStore, ws: Workspace) -> None:
# Exploration is not evidence a milestone was carried out, so a
# completion claim backed only by reads is still challenged.
client = ScriptedClient(
[
[call("list_dir", path=".")],
[AssistantText(text="looks done to me", model="m")],
[AssistantText(text="really, done", model="m")],
]
)
outcome = build(client, store, ws).run(milestone=1)
assert outcome.status == "escalate"
def test_turn_cap_is_enforced(store: StateStore, ws: Workspace) -> None:
forever = [[call("list_dir", path=".")] for _ in range(50)]
client = ScriptedClient(forever)