nisse/specs/orchestrator_scenarios.qnt

// Counterexamples found in orchestrator.qnt, written out as short traces.
// Each ends by expecting the property it breaks to be false; once the design
// is fixed, the expectation is flipped and the trace kept as a regression.
// All fixed: 1-4 (replies bind to the revision up when they arrived), 5 (boot
// and publishing adopt an open PR), 6 (redelivered), 7 (issue closed).
//
// Run:  quint test specs/orchestrator_scenarios.qnt   (runs every *Test)
module orchestrator_scenarios {
  import orchestrator.* from "./orchestrator"

  // Architecture plan rev-1 is up for sign-off, posted to Gitea.
  action toArchSignoff = init
    .then(send("use")).then(runJob(true))
    .then(finishPlanning("submit", true))

  // The milestone plan is approved; the next word starts it.
  action toMilestoneApproved = toArchSignoff
    .then(send("approve")).then(runJob(true))
    .then(finishPlanning("submit", true))
    .then(send("approve")).then(runJob(true))

  action toPrOpen = toMilestoneApproved
    .then(send("start")).then(runJob(true))
    .then(finishExec("completed"))
    .then(publishStep("ok")).then(publishStep("ok"))
    .then(publishStep("ok")).then(publishStep("ok"))

  // 1. The operator sends "approve" twice, say a double tap or an impatient
  //    repeat, while looking at the architecture plan. Both are queued
  //    before either runs. The first approves the architecture; planning
  //    the milestone runs; the second then approved the milestone plan,
  //    which the operator had not seen. Fixed: it is about rev-1, not
  //    applied.
  run doubleApproveApprovesAnUnseenPlanTest =
    toArchSignoff
      .then(send("approve")).then(send("approve"))
      .then(runJob(true))                    // approves arch rev-1
      .then(finishPlanning("submit", true))  // milestone-1 rev-1 is up
      .then(runJob(true))                    // the second "approve"
      .expect(s.gate.g == "ms1" and not(s.gate.approved) and approval_only_of_what_was_seen)

  // 2. The same with the revision named: "approve rev-1" meant for the
  //    architecture's rev-1 also matched the milestone's rev-1, because
  //    revisions were numbered per gate. Fixed: numbered across the task,
  //    the milestone plan is rev-2.
  run namedRevisionApprovesTheWrongGateTest =
    toArchSignoff
      .then(send("approve")).then(send("approveRev"))
      .then(runJob(true))
      .then(finishPlanning("submit", true))
      .then(runJob(true))
      .expect(s.gate.rev == 2 and not(s.gate.approved) and approval_only_of_what_was_seen)

  // 3. Approval on both channels: the operator approves in Matrix and on
  //    the Gitea issue. The Gitea approval is delivered and queued behind
  //    the Matrix one, which approves the architecture and posts the
  //    milestone plan. Re-attaching the plan watch does not recall a job
  //    already queued, so the Gitea approval approved the milestone plan.
  //    Fixed: it is about the plan posted before it, arch rev-1.
  run approvalOnBothChannelsApprovesTheNextPlanTest =
    toArchSignoff
      .then(send("approve")).then(commentPlan("approve")).then(pollPlan)
      .then(runJob(true))
      .then(finishPlanning("submit", true))
      .then(runJob(true))
      .expect(s.gate.g == "ms1" and not(s.gate.approved) and approval_only_of_what_was_seen)

  // 4. An afterthought becomes a rejection: "approve", then "also, keep it
  //    in one file", sent together. The note landed on the milestone plan
  //    as rejection feedback and it was planned again. Fixed: recorded,
  //    but it rejects nothing.
  run anAfterthoughtRejectsTheNextPlanTest =
    toArchSignoff
      .then(send("approve")).then(send("text"))
      .then(runJob(true))
      .then(finishPlanning("submit", true))
      .then(runJob(true))
      .expect(s.st == "AWAIT_MILESTONE_SIGNOFF" and feedback_only_on_what_was_seen)

  // 5. A crash after the PR is opened but before PR_OPEN is recorded.
  //    Boot found the intent's PR and marked it done, but the state still
  //    said EXECUTING, so it asked the restart question; 'continue' and
  //    'start' then published again, which Gitea refused, with no path back
  //    to PR_OPEN. Fixed: boot sees the open PR and adopts it.
  run crashAfterOpeningThePrPublishesAgainTest =
    toMilestoneApproved
      .then(send("start")).then(runJob(true))
      .then(finishExec("completed"))
      .then(publishStep("ok"))   // begin_intent
      .then(publishStep("ok"))   // push, open the PR
      .then(crash(true))         // adopted on boot
      .expect(s.st == "PR_OPEN" and s.prOn and one_pr_per_milestone)

  // 5b. Gitea opens the PR but the response is lost; the operator starts
  //     the milestone again. Publishing adopts the open PR.
  run lostResponseThenRetryAdoptsThePrTest =
    toMilestoneApproved
      .then(send("start")).then(runJob(true))
      .then(finishExec("completed"))
      .then(publishStep("ok")).then(publishStep("lost"))
      .then(send("continue")).then(runJob(true))   // back to the approved plan
      .then(send("start")).then(runJob(true))
      .then(finishExec("completed"))
      .then(publishStep("ok")).then(publishStep("ok"))
      .then(publishStep("ok")).then(publishStep("ok"))
      .expect(s.st == "PR_OPEN" and one_pr_per_milestone)

  // 6. A crash while the executor answers a review comment. The comment's
  //    cursor was saved before the work started (at most once), and a
  //    restart in PR_OPEN resumed quietly, so it was never answered. Fixed:
  //    the comment is marked while answered, and delivered again.
  run crashDuringAReviewLosesTheCommentTest =
    toPrOpen
      .then(commentPr).then(pollPr).then(runJob(true))  // the executor starts on it
      .then(crash(true))
      .expect(s.st == "PR_OPEN" and review_comments_answered)
      .then(pollPr).then(runJob(true)).then(finishReview("completed"))
      .expect(s.answered.size() == 1 and s.reviewing == 0)

  // 7. 'project complete' left the plan issue open. Fixed: it closes it.
  run projectCompleteLeavesThePlanIssueOpenTest =
    toPrOpen
      .then(send("complete")).then(runJob(true))
      .expect(s.st == "IDLE" and no_open_issue_without_a_task)
}