diff --git a/nisse-dev/src/nisse_dev/executor.py b/nisse-dev/src/nisse_dev/executor.py index 28c78ab..c622282 100644 --- a/nisse-dev/src/nisse_dev/executor.py +++ b/nisse-dev/src/nisse_dev/executor.py @@ -30,6 +30,15 @@ plain-text summary and no tool call — that ends the milestone.""" DEFAULT_MAX_TURNS = 24 DEFAULT_FAILURE_LIMIT = 2 +# A turn with no tool call means "the plan is done". Believing that before +# any tool has been called would record an empty milestone as a success, so +# the claim is challenged once first. +_NUDGE = ( + "\n\nYou have not made any change yet, so the plan cannot be complete. " + "Carry out the next step of the plan with a tool call, or run the tests " + "to show the work is already done." +) + @dataclass(frozen=True) class ExecutorOutcome: @@ -59,6 +68,8 @@ class LLMExecutor: model = self._roles.resolve("executor") system = f"{EXECUTOR_SYSTEM}\n\nYou are executing milestone {milestone}." consecutive_failures = 0 + did_work = False + challenged = False for _turn in range(self._max_turns): try: @@ -78,9 +89,23 @@ class LLMExecutor: calls.append(event) if not calls: - return ExecutorOutcome("completed", "executor reported the plan done") + if did_work: + return ExecutorOutcome( + "completed", "executor reported the plan done" + ) + if not challenged: + # Claiming completion without having touched anything is + # not evidence of completion; ask once before believing it. + challenged = True + system += _NUDGE + continue + return ExecutorOutcome( + "escalate", + "executor claimed the milestone was done without making any change", + ) for call in calls: + did_work = True outcome = self._workspace.dispatch(call.tool, call.arguments) self._store.append_event( ToolResult( diff --git a/nisse-dev/tests/test_executor.py b/nisse-dev/tests/test_executor.py index 70b43f4..409160f 100644 --- a/nisse-dev/tests/test_executor.py +++ b/nisse-dev/tests/test_executor.py @@ -88,11 +88,46 @@ def test_uses_executor_role_and_reduced_manifest( assert "milestone 2" in client.calls[0]["system"] -def test_no_tool_call_ends_the_milestone(store: StateStore, ws: Workspace) -> None: - client = ScriptedClient([[AssistantText(text="all steps done", model="m")]]) +def test_no_tool_call_after_work_ends_the_milestone( + store: StateStore, ws: Workspace +) -> None: + client = ScriptedClient( + [ + [call("write_file", path="a.py", content="x")], + [AssistantText(text="all steps done", model="m")], + ] + ) assert build(client, store, ws).run(milestone=1).status == "completed" +def test_completion_claimed_without_any_work_is_challenged( + store: StateStore, ws: Workspace +) -> None: + # A model that answers with prose on turn one has done nothing; believing + # it would record an empty milestone as a success. + client = ScriptedClient( + [ + [AssistantText(text="looks done to me", model="m")], + [call("write_file", path="a.py", content="x")], + [AssistantText(text="now it is done", model="m")], + ] + ) + outcome = build(client, store, ws).run(milestone=1) + assert outcome.status == "completed" + assert (ws.root / "a.py").exists() # the nudge produced actual work + assert "not made any change yet" in client.calls[1]["system"] + + +def test_persisting_in_an_empty_claim_escalates( + store: StateStore, ws: Workspace +) -> None: + prose = [AssistantText(text="nothing to do", model="m")] + client = ScriptedClient([prose, prose]) + outcome = build(client, store, ws).run(milestone=1) + assert outcome.status == "escalate" + assert "without making any change" in outcome.note + + def test_two_consecutive_command_failures_escalate( store: StateStore, ws: Workspace ) -> None: