Bind approvals and feedback to the revision they were about
CI / checks (push) Skipped
CI / checks (pull_request) Successful in 1m16s
CI / checks (push) Skipped
CI / checks (pull_request) Successful in 1m16s
Fixes counterexamples 1-4 from specs/orchestrator.qnt: a repeated 'approve', 'approve rev-1' meant for the architecture, approving on both channels, and 'approve' plus an afterthought all landed on the next plan, one the operator had not seen. - Revisions are numbered across the work item (A4 already required monotonic tags; the code numbered per gate), so rev-N names one plan. - Each reply is stamped on arrival with the revision that was up: in Matrix the one whose approval request had been sent (a plain value the transport thread reads, restored on boot), in Gitea the latest plan comment older than the reply (board.posts). - Gate.evaluate applies a reply that names no revision only if its stamp is the current revision. Otherwise an approval is not applied and feedback rejects nothing (it is still in the transcript), and the agent says which revision is up. 'approve rev-N' binds by its number. - Spec A4 states both rules. The model applies them; scenarios 1-4 now expect the properties to hold, and both held over 200,000 random traces and Apalache's exhaustive check to depth 8. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01PPQ14L3LikyefCMFBqEbKE
This commit is contained in:
1 parent
6c83839f4d
commit
97863c1939
10 files changed
+324
-44
No files matched your search
+15
-9
@@ -55,7 +55,8 @@ module orchestrator {
|
||||
tid: int, // task counter; a task's branches are its own
|
||||
task: bool, // a project is in use
|
||||
gate: Gate,
|
||||
revs: str -> int,
|
||||
revs: str -> int, // latest revision per gate
|
||||
lastRev: int, // revisions run across the task (A4)
|
||||
ms: int,
|
||||
pending: Pending,
|
||||
prOn: bool, prKey: int, prLast: int, // _data["pr"]
|
||||
@@ -120,7 +121,7 @@ module orchestrator {
|
||||
val t = s.tid + 1
|
||||
startPlanning(
|
||||
{ ...s, tid: t, task: true, gate: NO_GATE,
|
||||
revs: Map("arch" -> 0, "ms1" -> 0, "ms2" -> 0), ms: 1, pending: NO_PENDING,
|
||||
revs: Map("arch" -> 0, "ms1" -> 0, "ms2" -> 0), lastRev: 0, ms: 1, pending: NO_PENDING,
|
||||
prOn: false, prKey: 0, prLast: 0,
|
||||
boardOn: giteaUp, boardPosted: false, boardLast: 0,
|
||||
openIssues: if (giteaUp) s.openIssues.union(Set(t)) else s.openIssues,
|
||||
@@ -132,10 +133,10 @@ module orchestrator {
|
||||
// skips what is already there).
|
||||
pure def planSubmitted(s: Sys, giteaUp: bool): Sys =
|
||||
val g = s.workGate
|
||||
val rev = s.revs.get(g) + 1
|
||||
val rev = s.lastRev + 1
|
||||
val posted = s.boardOn and giteaUp
|
||||
val t = endWork({ ...s,
|
||||
revs: s.revs.set(g, rev),
|
||||
revs: s.revs.set(g, rev), lastRev: rev,
|
||||
gate: { on: true, g: g, rev: rev, approved: false },
|
||||
st: if (g == "arch") "AWAIT_ARCH_SIGNOFF" else "AWAIT_MILESTONE_SIGNOFF",
|
||||
shown: { g: g, rev: rev } })
|
||||
@@ -146,19 +147,24 @@ module orchestrator {
|
||||
// _execute_milestone: start
|
||||
pure def startExec(s: Sys): Sys = { ...s, st: "EXECUTING", work: "exec" }
|
||||
|
||||
// _apply_gate_reply, for Matrix and Gitea alike.
|
||||
// _apply_gate_reply and Gate.evaluate, for Matrix and Gitea alike.
|
||||
// A job's `saw` doubles as its arrival stamp: the revision up when it was
|
||||
// sent (Matrix) or the latest plan posted before it (Gitea). A reply that
|
||||
// names no revision applies only if that is the current one.
|
||||
pure def applyGate(s: Sys, j: Job): Sys =
|
||||
val current = { g: s.gate.g, rev: s.gate.rev }
|
||||
if (s.gate.approved)
|
||||
if (j.kind == "start") startExec(s) else s
|
||||
else if (j.kind == "approve" or (j.kind == "approveRev" and j.saw.rev == s.gate.rev))
|
||||
else if (j.kind == "approveRev" and j.saw.rev != s.gate.rev)
|
||||
s // names another revision: ignored with a notice
|
||||
else if (j.kind != "approveRev" and j.saw.rev != s.gate.rev)
|
||||
s // about another revision: not applied, the agent says so
|
||||
else if (j.kind == "approve" or j.kind == "approveRev")
|
||||
val s2 = { ...s, badApproval: s.badApproval or j.saw != current }
|
||||
if (s.gate.g == "arch")
|
||||
startPlanning({ ...s2, gate: NO_GATE }, msGate(s.ms))
|
||||
else
|
||||
{ ...s2, gate: { ...s.gate, approved: true }, st: "AWAIT_MILESTONE_SIGNOFF" }
|
||||
else if (j.kind == "approveRev")
|
||||
s // names another revision: ignored with a notice
|
||||
else
|
||||
// Anything else is rejection feedback, and the gate is planned again.
|
||||
startPlanning({ ...s, staleFeedback: s.staleFeedback or j.saw != current, gate: NO_GATE },
|
||||
@@ -267,7 +273,7 @@ module orchestrator {
|
||||
|
||||
action init = s' = {
|
||||
st: "IDLE", tid: 0, task: false, gate: NO_GATE,
|
||||
revs: Map("arch" -> 0, "ms1" -> 0, "ms2" -> 0), ms: 1, pending: NO_PENDING,
|
||||
revs: Map("arch" -> 0, "ms1" -> 0, "ms2" -> 0), lastRev: 0, ms: 1, pending: NO_PENDING,
|
||||
prOn: false, prKey: 0, prLast: 0,
|
||||
boardOn: false, boardPosted: false, boardLast: 0,
|
||||
intents: Set(),
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
// Counterexamples found in orchestrator.qnt, written out as short traces.
|
||||
// Each ends by expecting the property it breaks to be false; once the design
|
||||
// is fixed, flip the expectation and keep the trace as a regression.
|
||||
// is fixed, the expectation is flipped and the trace kept as a regression.
|
||||
// Fixed: 1-4 (replies bind to the revision up when they arrived).
|
||||
//
|
||||
// Run: quint test specs/orchestrator_scenarios.qnt (runs every *Test)
|
||||
module orchestrator_scenarios {
|
||||
@@ -26,50 +27,54 @@ module orchestrator_scenarios {
|
||||
// 1. The operator sends "approve" twice, say a double tap or an impatient
|
||||
// repeat, while looking at the architecture plan. Both are queued
|
||||
// before either runs. The first approves the architecture; planning
|
||||
// the milestone runs; the second then approves the milestone plan,
|
||||
// which the operator has not seen.
|
||||
// the milestone runs; the second then approved the milestone plan,
|
||||
// which the operator had not seen. Fixed: it is about rev-1, not
|
||||
// applied.
|
||||
run doubleApproveApprovesAnUnseenPlanTest =
|
||||
toArchSignoff
|
||||
.then(send("approve")).then(send("approve"))
|
||||
.then(runJob(true)) // approves arch rev-1
|
||||
.then(finishPlanning("submit", true)) // milestone-1 rev-1 is up
|
||||
.then(runJob(true)) // the second "approve"
|
||||
.expect(s.gate.g == "ms1" and s.gate.approved and not(approval_only_of_what_was_seen))
|
||||
.expect(s.gate.g == "ms1" and not(s.gate.approved) and approval_only_of_what_was_seen)
|
||||
|
||||
// 2. The same with the revision named: "approve rev-1" meant for the
|
||||
// architecture's rev-1 also matches the milestone's rev-1, because a
|
||||
// revision tag carries no gate name.
|
||||
// architecture's rev-1 also matched the milestone's rev-1, because
|
||||
// revisions were numbered per gate. Fixed: numbered across the task,
|
||||
// the milestone plan is rev-2.
|
||||
run namedRevisionApprovesTheWrongGateTest =
|
||||
toArchSignoff
|
||||
.then(send("approve")).then(send("approveRev"))
|
||||
.then(runJob(true))
|
||||
.then(finishPlanning("submit", true))
|
||||
.then(runJob(true))
|
||||
.expect(s.gate.approved and not(approval_only_of_what_was_seen))
|
||||
.expect(s.gate.rev == 2 and not(s.gate.approved) and approval_only_of_what_was_seen)
|
||||
|
||||
// 3. Approval on both channels: the operator approves in Matrix and on
|
||||
// the Gitea issue. The Gitea approval is delivered and queued behind
|
||||
// the Matrix one, which approves the architecture and posts the
|
||||
// milestone plan. Re-attaching the plan watch does not recall a job
|
||||
// already queued, so the Gitea approval approves the milestone plan.
|
||||
// already queued, so the Gitea approval approved the milestone plan.
|
||||
// Fixed: it is about the plan posted before it, arch rev-1.
|
||||
run approvalOnBothChannelsApprovesTheNextPlanTest =
|
||||
toArchSignoff
|
||||
.then(send("approve")).then(commentPlan("approve")).then(pollPlan)
|
||||
.then(runJob(true))
|
||||
.then(finishPlanning("submit", true))
|
||||
.then(runJob(true))
|
||||
.expect(s.gate.g == "ms1" and s.gate.approved and not(approval_only_of_what_was_seen))
|
||||
.expect(s.gate.g == "ms1" and not(s.gate.approved) and approval_only_of_what_was_seen)
|
||||
|
||||
// 4. An afterthought becomes a rejection: "approve", then "also, keep it
|
||||
// in one file", sent together. The note lands on the milestone plan
|
||||
// as rejection feedback and it is planned again.
|
||||
// in one file", sent together. The note landed on the milestone plan
|
||||
// as rejection feedback and it was planned again. Fixed: recorded,
|
||||
// but it rejects nothing.
|
||||
run anAfterthoughtRejectsTheNextPlanTest =
|
||||
toArchSignoff
|
||||
.then(send("approve")).then(send("text"))
|
||||
.then(runJob(true))
|
||||
.then(finishPlanning("submit", true))
|
||||
.then(runJob(true))
|
||||
.expect(s.st == "MILESTONE_PLANNING" and not(feedback_only_on_what_was_seen))
|
||||
.expect(s.st == "AWAIT_MILESTONE_SIGNOFF" and feedback_only_on_what_was_seen)
|
||||
|
||||
// 5. A crash after the PR is opened but before PR_OPEN is recorded.
|
||||
// Boot finds the intent's PR and marks it done, but the state still
|
||||
|
||||
Reference in new issue
Block a user