Redeliver an interrupted review comment, and close the issue on completion
CI / checks (push) Skipped
CI / checks (pull_request) Successful in 1m16s
CI / checks (push) Skipped
CI / checks (pull_request) Successful in 1m16s
Fixes counterexamples 6 and 7 from specs/orchestrator.qnt, the last two. 6. A crash while the executor answered a review comment lost it: its cursor is saved before the work (at most once), and a restart in PR_OPEN resumed quietly. Now pr.reviewing marks the comment until the executor returns (every way it returns, but not a crash), and boot resumes the watch just before it, with a notice. The mark stays until it is answered, so a second crash cannot lose it either. 7. 'project complete' left the plan issue open. It now closes it with a note, after IDLE is saved, as abandon does (shared helper). The model marks and redelivers, and closes the issue; scenarios 6 and 7 now expect their properties to hold. A mutant without redelivery fails scenario 6. All eight properties held together over 300,000 traces. No xfail reproductions remain. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01PPQ14L3LikyefCMFBqEbKE
This commit is contained in:
1 parent
83a4bd72b5
commit
848bd9f122
4 files changed
+153
-52
No files matched your search
@@ -1,6 +1,6 @@
|
||||
"""The counterexamples found by specs/orchestrator.qnt, replayed against the
|
||||
real orchestrator. Each is marked xfail until its fix lands, and names the
|
||||
scenario in specs/orchestrator_scenarios.qnt it reproduces.
|
||||
real orchestrator. Each names the scenario in specs/orchestrator_scenarios.qnt
|
||||
it reproduces; all seven are fixed, and the tests stay as regressions.
|
||||
|
||||
The model is a hand-written reading of control.py; reproducing each trace
|
||||
here is what shows the flaw is in the design, not in the model.
|
||||
@@ -194,9 +194,6 @@ class Agent:
|
||||
self.transport.receive("don", text)
|
||||
|
||||
|
||||
XFAIL = pytest.mark.xfail(strict=True, reason="counterexample; fix pending")
|
||||
|
||||
|
||||
def test_double_approve_approves_an_unseen_plan(tmp_path: Path) -> None:
|
||||
"""doubleApproveApprovesAnUnseenPlanTest"""
|
||||
worker = QueueingWorker()
|
||||
@@ -275,7 +272,6 @@ def test_crash_after_opening_the_pr_publishes_again(tmp_path: Path) -> None:
|
||||
assert agent.state == PR_OPEN
|
||||
|
||||
|
||||
@XFAIL
|
||||
def test_crash_during_a_review_loses_the_comment(tmp_path: Path) -> None:
|
||||
"""crashDuringAReviewLosesTheCommentTest"""
|
||||
gitea = FakeGitea()
|
||||
@@ -296,17 +292,15 @@ def test_crash_during_a_review_loses_the_comment(tmp_path: Path) -> None:
|
||||
|
||||
agent = Agent(tmp_path, gitea) # restart
|
||||
assert agent.state == PR_OPEN
|
||||
# The comment was taken but never answered: after the restart it must
|
||||
# be handled again, or the operator told it was dropped.
|
||||
delivered = agent.pollers[20].poll_once(20)
|
||||
told = any(
|
||||
"please add a test" in m.text or "review" in m.text
|
||||
for m in agent.transport.outbound[1:]
|
||||
)
|
||||
assert delivered or told
|
||||
# The comment was taken but never answered: after the restart the
|
||||
# operator is told, and it is delivered and answered again.
|
||||
assert "answering it again" in agent.transport.outbound[-1].text
|
||||
[delivered] = agent.pollers[20].poll_once(20)
|
||||
assert delivered.body == "please add a test"
|
||||
assert "pushed" in agent.transport.outbound[-1].text
|
||||
assert "reviewing" not in agent.store.get_state().data["pr"]
|
||||
|
||||
|
||||
@XFAIL
|
||||
def test_project_complete_leaves_the_plan_issue_open(tmp_path: Path) -> None:
|
||||
"""projectCompleteLeavesThePlanIssueOpenTest"""
|
||||
closed: list[int] = []
|
||||
@@ -347,3 +341,60 @@ def test_boot_asks_as_before_when_gitea_cannot_say(
|
||||
agent = Agent(tmp_path, gitea)
|
||||
assert agent.state == AWAIT_INPUT
|
||||
assert "restarted mid-EXECUTING" in agent.transport.outbound[-1].text
|
||||
|
||||
|
||||
class Crash(BaseException):
|
||||
"""The process dying: no handler runs."""
|
||||
|
||||
|
||||
def crash_mid_review(agent: Agent, gitea: FakeGitea) -> None:
|
||||
def dies(milestone: int) -> ExecutorOutcome:
|
||||
raise Crash()
|
||||
|
||||
agent.executor.run = dies # type: ignore[method-assign]
|
||||
with pytest.raises(Crash):
|
||||
agent.pollers[20].poll_once(20)
|
||||
|
||||
|
||||
def test_a_second_crash_while_answering_again_still_redelivers(tmp_path: Path) -> None:
|
||||
gitea = FakeGitea()
|
||||
agent = Agent(tmp_path, gitea)
|
||||
agent.say("use demo", "approve", "approve", "start milestone 1")
|
||||
gitea.add(20, "don", "please add a test")
|
||||
crash_mid_review(agent, gitea)
|
||||
agent = Agent(tmp_path, gitea)
|
||||
crash_mid_review(agent, gitea) # dies again while answering it again
|
||||
agent = Agent(tmp_path, gitea)
|
||||
[delivered] = agent.pollers[20].poll_once(20)
|
||||
assert delivered.body == "please add a test"
|
||||
|
||||
|
||||
def test_an_executor_error_answers_the_comment_so_it_is_not_redelivered(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
gitea = FakeGitea()
|
||||
agent = Agent(tmp_path, gitea)
|
||||
agent.say("use demo", "approve", "approve", "start milestone 1")
|
||||
gitea.add(20, "don", "please add a test")
|
||||
|
||||
def fails(milestone: int) -> ExecutorOutcome:
|
||||
raise RuntimeError("bug in the executor")
|
||||
|
||||
agent.executor.run = fails # type: ignore[method-assign]
|
||||
agent.pollers[20].poll_once(20)
|
||||
assert "could not handle a review comment" in agent.transport.outbound[-1].text
|
||||
agent = Agent(tmp_path, gitea) # restart: nothing to answer again
|
||||
assert "still watching" in agent.transport.outbound[-1].text
|
||||
assert agent.pollers[20].poll_once(20) == []
|
||||
|
||||
|
||||
def test_project_complete_still_completes_when_gitea_is_down(tmp_path: Path) -> None:
|
||||
agent = Agent(tmp_path, FakeGitea())
|
||||
|
||||
def down(issue: int, note: str) -> None:
|
||||
raise GiteaError("gitea is down")
|
||||
|
||||
agent.orch._board.close = down # type: ignore[attr-defined]
|
||||
agent.say("use demo", "approve", "approve", "start milestone 1", "project complete")
|
||||
assert agent.state == IDLE
|
||||
assert "could not close plan issue #10" in agent.transport.outbound[-1].text
|
||||
Reference in new issue
Block a user