dongarandClaude Opus 5.5
266b5dbbd7
CI / checks (push) Skipped
test_loop_properties.py walks the new loop with a seeded mix of operator and Gitea actions (go-aheads, stale go-aheads, answers, comments, merges, the comment-then-merge race, closes, abandon, a crash and restart) and an adaptive scripted model, checking every loop.qnt property after every step. It found two bugs, both fixed with regression tests: abandoning an item left its open PR open (the model's fix 1, now in code), and after a review fix became a follow-up PR, the next fix went to the merged branch. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01PPQ14L3LikyefCMFBqEbKE
248 lines
9.8 KiB
Python
248 lines
9.8 KiB
Python
"""specs/loop.qnt's properties, checked against the real loop (M2.1, PR D).
|
|
|
|
The Quint model checks the design; this checks the code. A seeded random
|
|
walk interleaves what the operator and Gitea can do (go-aheads, answers,
|
|
comments, merges, closes, abandon, a crash and restart) with an adaptive
|
|
scripted model, and checks every property after every step. A failure prints
|
|
the seed and the steps, so it replays exactly.
|
|
"""
|
|
|
|
import random
|
|
import subprocess
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
import pytest
|
|
from test_loop_milestones import DON, Forge, call
|
|
from test_loop_milestones import world as world # noqa: PLC0414 -- the fixture
|
|
|
|
from nisse_dev.cancel import Cancel
|
|
from nisse_dev.canonical import Fact, ToolCall
|
|
from nisse_dev.gitea import Comment
|
|
from nisse_dev.loop.agent import LoopAgent
|
|
from nisse_dev.loop.driver import Driver
|
|
from nisse_dev.loop.items import ANSWER, GO_AHEAD, MERGE, WAITING, loop_of
|
|
from nisse_dev.oplog import OpLog
|
|
from nisse_dev.tools import Workspace
|
|
from nisse_dev.transport import InboundMessage, InMemoryTransport
|
|
from nisse_dev.worker import InlineWorker
|
|
|
|
SEEDS = range(32)
|
|
STEPS = 20
|
|
|
|
|
|
class PolicyModel:
|
|
"""Plays a plausible driver: restate first, then a milestone, fixes
|
|
after comments, close after a merge; sometimes asks a question or
|
|
replies in prose. Seeded, so a walk replays exactly."""
|
|
|
|
def __init__(self, rng: random.Random) -> None:
|
|
self.rng = rng
|
|
self.n = 0
|
|
|
|
def complete(self, events, tools, model, system=None) -> list[Any]:
|
|
self.n += 1
|
|
offered = {t["function"]["name"] for t in tools}
|
|
facts = [getattr(e.event, "text", "") for e in events]
|
|
roll = self.rng.random()
|
|
if roll < 0.08:
|
|
return [call("ask_human", question=f"question {self.n}?")]
|
|
if roll < 0.12:
|
|
return [ToolCall(call_id=f"p{self.n}", tool="nonsense", arguments={})]
|
|
if "finish_milestone" not in offered:
|
|
return [call("restate", text=f"restatement {self.n}")]
|
|
merged = any("merged" in t for t in facts[-6:])
|
|
if merged and self.rng.random() < 0.8:
|
|
return [call("close_item", summary=f"done {self.n}")]
|
|
return [
|
|
call("start_milestone", name=f"m{self.n}"),
|
|
call("write_file", path=f"f{self.n}.py", content=f"x = {self.n}\n"),
|
|
call("finish_milestone", summary=f"work {self.n}"),
|
|
]
|
|
|
|
|
|
class Walk:
|
|
def __init__(self, world: dict[str, Any], seed: int) -> None:
|
|
self.world = world
|
|
self.rng = random.Random(seed)
|
|
self.seed = seed
|
|
self.steps: list[str] = []
|
|
self.shown_at_go: list[int] = [] # the revision on screen at each go
|
|
self.agent, self.transport = self._agent()
|
|
|
|
def _agent(self) -> tuple[LoopAgent, InMemoryTransport]:
|
|
transport = InMemoryTransport()
|
|
cancel = Cancel()
|
|
w = self.world
|
|
agent = LoopAgent(
|
|
w["store"],
|
|
transport,
|
|
w["forge"],
|
|
repo="don/sandbox",
|
|
project="sandbox",
|
|
account="nisse-dev",
|
|
operators={DON},
|
|
gitea_operators={"don"},
|
|
driver=Driver(PolicyModel(self.rng), "m", cancel=cancel), # type: ignore[arg-type]
|
|
workspace=Workspace(w["host"], w["repo"].root, timeout_s=30),
|
|
worker=InlineWorker(),
|
|
cancel=cancel,
|
|
oplog=OpLog("nisse-dev", _Null()),
|
|
milestones=w["milestones"],
|
|
)
|
|
transport.on_message(agent.on_message)
|
|
return agent, transport
|
|
|
|
def items(self) -> list[Any]:
|
|
return self.world["store"].work_items()
|
|
|
|
def step(self) -> None:
|
|
forge: Forge = self.world["forge"]
|
|
open_items = [i for i in self.items() if i.status == "open"]
|
|
# Weighted towards what an item can actually do, so review paths
|
|
# are walked often, not drowned out by intake and crashes.
|
|
weights = {"intake": 2 if not open_items else 1, "crash": 1}
|
|
for item in open_items:
|
|
loop = loop_of(item)
|
|
weights.update({"go": 3, "answer": 2, "abandon": 1, "go-stale": 1})
|
|
if loop.pr:
|
|
weights.update(
|
|
{"comment": 4, "merge": 2, "close-pr": 1, "comment-merge": 2}
|
|
)
|
|
action = self.rng.choices(list(weights), list(weights.values()))[0]
|
|
self.steps.append(action)
|
|
item = open_items[0] if open_items else None
|
|
loop = loop_of(item) if item else None
|
|
if action == "intake":
|
|
self.agent.intake()
|
|
elif action == "crash":
|
|
# Volatile state is lost: a new agent over the same store.
|
|
self.agent, self.transport = self._agent()
|
|
self.agent.boot()
|
|
self.agent._schedule()
|
|
elif action in ("go", "answer", "abandon") and loop:
|
|
text = {"go": "go", "answer": "an answer", "abandon": "abandon"}[action]
|
|
if action == "go" and item.position == WAITING and loop.wait == GO_AHEAD:
|
|
self.shown_at_go.append(loop.rev)
|
|
self.transport.receive(DON, text, thread=loop.thread)
|
|
elif action == "go-stale" and loop and loop.rev > 1:
|
|
# A go sent while an older revision was on screen.
|
|
self.agent.handle(
|
|
InboundMessage(DON, "go", thread=loop.thread), about=loop.rev - 1
|
|
)
|
|
elif action == "comment" and loop:
|
|
forge.comments.setdefault(loop.pr, []).append(
|
|
Comment(len(self.steps) * 10, "don", f"comment {len(self.steps)}", "t")
|
|
)
|
|
self.agent.watch()
|
|
self.agent._schedule()
|
|
elif (
|
|
action == "comment-merge" and loop and forge.prs[loop.pr]["state"] == "open"
|
|
):
|
|
# The race: a comment is seen, then the PR merges before the
|
|
# driver's fix lands.
|
|
forge.comments.setdefault(loop.pr, []).append(
|
|
Comment(len(self.steps) * 10, "don", f"comment {len(self.steps)}", "t")
|
|
)
|
|
self.agent.watch()
|
|
forge.merge(loop.pr)
|
|
self.agent._schedule()
|
|
elif action == "merge" and loop and forge.prs[loop.pr]["state"] == "open":
|
|
forge.merge(loop.pr)
|
|
self.agent.watch()
|
|
self.agent._schedule()
|
|
elif action == "close-pr" and loop and forge.prs[loop.pr]["state"] == "open":
|
|
forge.prs[loop.pr]["state"] = "closed"
|
|
self.agent.watch()
|
|
self.agent._schedule()
|
|
|
|
# -- the properties (specs/loop.qnt) -------------------------------------
|
|
|
|
def check(self) -> None:
|
|
where = f"seed {self.seed}, steps {self.steps}"
|
|
store, forge = self.world["store"], self.world["forge"]
|
|
items = self.items()
|
|
issues = [loop_of(i).issue for i in items if loop_of(i).issue]
|
|
assert len(issues) == len(set(issues)), f"one_item_per_issue: {where}"
|
|
for item in items:
|
|
loop = loop_of(item)
|
|
if item.status == "open" and item.position == WAITING:
|
|
explained = (
|
|
(loop.wait == GO_AHEAD and loop.rev > 0)
|
|
or (loop.wait == ANSWER and bool(loop.note))
|
|
or (loop.wait == MERGE and loop.pr > 0)
|
|
)
|
|
assert explained, (
|
|
f"waits_are_explainable: {item.position} {loop}: {where}"
|
|
)
|
|
gos = [
|
|
int(f.event.text.split("rev-")[1].split()[0])
|
|
for f in store.transcript(item.id, "orchestrator").read()
|
|
if isinstance(f.event, Fact)
|
|
and f.event.text.startswith("go-ahead on rev-")
|
|
]
|
|
assert all(rev in self.shown_at_go for rev in gos), (
|
|
f"go_binds: {gos}: {where}"
|
|
)
|
|
mine = [p for p in forge.prs.values() if p["head"].startswith(item.id)]
|
|
assert sum(p["state"] == "open" for p in mine) <= 1, (
|
|
f"milestone_after_merge: {where}"
|
|
)
|
|
if item.status == "closed":
|
|
assert not any(p["state"] == "open" for p in mine), (
|
|
f"closed_items_leave_nothing (PR): {where}"
|
|
)
|
|
assert not [
|
|
i for i in store.pending_intents() if i.work_item == item.id
|
|
], f"closed_items_leave_nothing (intent): {where}"
|
|
heads = [p["head"] for p in forge.prs.values()]
|
|
assert len(heads) == len(set(heads)), f"one_pr_per_milestone: {where}"
|
|
bare = self.world["bare"]
|
|
for n, pr in forge.prs.items():
|
|
if not pr["merged"]:
|
|
continue
|
|
head = _rev(bare, f"refs/heads/{pr['head']}")
|
|
# A fix pushed after the merge must reach the operator as a PR
|
|
# (loop.qnt's lostFix): in main, or in a later PR, whatever they
|
|
# then decide about it.
|
|
landed = _is_ancestor(bare, head, "refs/heads/main") or any(
|
|
_is_ancestor(bare, head, f"refs/heads/{other['head']}")
|
|
for m, other in forge.prs.items()
|
|
if m > n
|
|
)
|
|
assert landed, f"review_fixes_land (PR {n}): {where}"
|
|
|
|
|
|
class _Null:
|
|
def write(self, _: str) -> None: ...
|
|
|
|
def flush(self) -> None: ...
|
|
|
|
|
|
def _rev(bare: Path, ref: str) -> str:
|
|
return subprocess.run(
|
|
["git", "--git-dir", str(bare), "rev-parse", ref],
|
|
capture_output=True,
|
|
text=True,
|
|
check=True,
|
|
).stdout.strip()
|
|
|
|
|
|
def _is_ancestor(bare: Path, commit: str, ref: str) -> bool:
|
|
return (
|
|
subprocess.run(
|
|
["git", "--git-dir", str(bare), "merge-base", "--is-ancestor", commit, ref],
|
|
capture_output=True,
|
|
check=False,
|
|
).returncode
|
|
== 0
|
|
)
|
|
|
|
|
|
@pytest.mark.parametrize("seed", SEEDS)
|
|
def test_the_loop_keeps_its_properties(world: dict[str, Any], seed: int) -> None:
|
|
walk = Walk(world, seed)
|
|
for _ in range(STEPS):
|
|
walk.step()
|
|
walk.check()
|