// Counterexamples found in orchestrator.qnt, written out as short traces. // Each ends by expecting the property it breaks to be false; once the design // is fixed, the expectation is flipped and the trace kept as a regression. // All fixed: 1-4 (replies bind to the revision up when they arrived), 5 (boot // and publishing adopt an open PR), 6 (redelivered), 7 (issue closed). // // Run: quint test specs/orchestrator_scenarios.qnt (runs every *Test) module orchestrator_scenarios { import orchestrator.* from "./orchestrator" // Architecture plan rev-1 is up for sign-off, posted to Gitea. action toArchSignoff = init .then(send("use")).then(runJob(true)) .then(finishPlanning("submit", true)) // The milestone plan is approved; the next word starts it. action toMilestoneApproved = toArchSignoff .then(send("approve")).then(runJob(true)) .then(finishPlanning("submit", true)) .then(send("approve")).then(runJob(true)) action toPrOpen = toMilestoneApproved .then(send("start")).then(runJob(true)) .then(finishExec("completed")) .then(publishStep("ok")).then(publishStep("ok")) .then(publishStep("ok")).then(publishStep("ok")) // 1. The operator sends "approve" twice, say a double tap or an impatient // repeat, while looking at the architecture plan. Both are queued // before either runs. The first approves the architecture; planning // the milestone runs; the second then approved the milestone plan, // which the operator had not seen. Fixed: it is about rev-1, not // applied. run doubleApproveApprovesAnUnseenPlanTest = toArchSignoff .then(send("approve")).then(send("approve")) .then(runJob(true)) // approves arch rev-1 .then(finishPlanning("submit", true)) // milestone-1 rev-1 is up .then(runJob(true)) // the second "approve" .expect(s.gate.g == "ms1" and not(s.gate.approved) and approval_only_of_what_was_seen) // 2. The same with the revision named: "approve rev-1" meant for the // architecture's rev-1 also matched the milestone's rev-1, because // revisions were numbered per gate. Fixed: numbered across the task, // the milestone plan is rev-2. run namedRevisionApprovesTheWrongGateTest = toArchSignoff .then(send("approve")).then(send("approveRev")) .then(runJob(true)) .then(finishPlanning("submit", true)) .then(runJob(true)) .expect(s.gate.rev == 2 and not(s.gate.approved) and approval_only_of_what_was_seen) // 3. Approval on both channels: the operator approves in Matrix and on // the Gitea issue. The Gitea approval is delivered and queued behind // the Matrix one, which approves the architecture and posts the // milestone plan. Re-attaching the plan watch does not recall a job // already queued, so the Gitea approval approved the milestone plan. // Fixed: it is about the plan posted before it, arch rev-1. run approvalOnBothChannelsApprovesTheNextPlanTest = toArchSignoff .then(send("approve")).then(commentPlan("approve")).then(pollPlan) .then(runJob(true)) .then(finishPlanning("submit", true)) .then(runJob(true)) .expect(s.gate.g == "ms1" and not(s.gate.approved) and approval_only_of_what_was_seen) // 4. An afterthought becomes a rejection: "approve", then "also, keep it // in one file", sent together. The note landed on the milestone plan // as rejection feedback and it was planned again. Fixed: recorded, // but it rejects nothing. run anAfterthoughtRejectsTheNextPlanTest = toArchSignoff .then(send("approve")).then(send("text")) .then(runJob(true)) .then(finishPlanning("submit", true)) .then(runJob(true)) .expect(s.st == "AWAIT_MILESTONE_SIGNOFF" and feedback_only_on_what_was_seen) // 5. A crash after the PR is opened but before PR_OPEN is recorded. // Boot found the intent's PR and marked it done, but the state still // said EXECUTING, so it asked the restart question; 'continue' and // 'start' then published again, which Gitea refused, with no path back // to PR_OPEN. Fixed: boot sees the open PR and adopts it. run crashAfterOpeningThePrPublishesAgainTest = toMilestoneApproved .then(send("start")).then(runJob(true)) .then(finishExec("completed")) .then(publishStep("ok")) // begin_intent .then(publishStep("ok")) // push, open the PR .then(crash(true)) // adopted on boot .expect(s.st == "PR_OPEN" and s.prOn and one_pr_per_milestone) // 5b. Gitea opens the PR but the response is lost; the operator starts // the milestone again. Publishing adopts the open PR. run lostResponseThenRetryAdoptsThePrTest = toMilestoneApproved .then(send("start")).then(runJob(true)) .then(finishExec("completed")) .then(publishStep("ok")).then(publishStep("lost")) .then(send("continue")).then(runJob(true)) // back to the approved plan .then(send("start")).then(runJob(true)) .then(finishExec("completed")) .then(publishStep("ok")).then(publishStep("ok")) .then(publishStep("ok")).then(publishStep("ok")) .expect(s.st == "PR_OPEN" and one_pr_per_milestone) // 6. A crash while the executor answers a review comment. The comment's // cursor was saved before the work started (at most once), and a // restart in PR_OPEN resumed quietly, so it was never answered. Fixed: // the comment is marked while answered, and delivered again. run crashDuringAReviewLosesTheCommentTest = toPrOpen .then(commentPr).then(pollPr).then(runJob(true)) // the executor starts on it .then(crash(true)) .expect(s.st == "PR_OPEN" and review_comments_answered) .then(pollPr).then(runJob(true)).then(finishReview("completed")) .expect(s.answered.size() == 1 and s.reviewing == 0) // 7. 'project complete' left the plan issue open. Fixed: it closes it. run projectCompleteLeavesThePlanIssueOpenTest = toPrOpen .then(send("complete")).then(runJob(true)) .expect(s.st == "IDLE" and no_open_issue_without_a_task) }