dongarandClaude Opus 5.5
cceb30c729
CI / checks (push) Skipped
CI / checks (pull_request) Successful in 3m26s
Several work items on the new loop (L1, D4): - Switching parks the item on the workspace: its work is committed and its branch pushed; resuming checks the item's branch back out. Branches of open items are never set aside by a fresh workspace. - Next item: the operator's "do #N next" first, otherwise the oldest. - Issues with open Gitea dependencies are blocked until they close, checked at intake and on every round; status shows blocked items and the line. Tests: queue unit tests, the property walk over several issues with dependencies and a new parked_work_safe property (it catches parking disabled), and an end-to-end test with two issues where B depends on A. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01PPQ14L3LikyefCMFBqEbKE
313 lines
12 KiB
Python
313 lines
12 KiB
Python
"""The end-to-end tests on the new loop (Development Plan M2.1, M2.2): the
|
|
whole slice, from an issue assigned in Gitea to a merged PR and a closed work
|
|
item, and the queue, with an issue blocked on another until it closes; with
|
|
real models, the Gitea sandbox and a real build host.
|
|
|
|
A scripted operator plays the operator's part: in Gitea through its own test
|
|
account (NISSE_OPERATOR_TOKEN; an operator only in nisse-test.yaml), and in
|
|
the item's Matrix thread through the in-memory transport. Run with
|
|
scripts/e2e.sh after each milestone.
|
|
"""
|
|
|
|
import contextlib
|
|
import os
|
|
import time
|
|
import uuid
|
|
from collections.abc import Iterator
|
|
from dataclasses import replace
|
|
from fnmatch import fnmatch
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
import pytest
|
|
from e2e_support import base_branch, dependent_task, operator_gitea, task
|
|
|
|
from nisse_dev.config import AgentConfig
|
|
from nisse_dev.execution import SSHHost
|
|
from nisse_dev.gitea import GiteaClient, GiteaError
|
|
from nisse_dev.loop.build import RunningLoop, build_loop_agent
|
|
from nisse_dev.loop.items import (
|
|
ANSWER,
|
|
BLOCKED,
|
|
GO_AHEAD,
|
|
MERGE,
|
|
WAITING,
|
|
item_for_issue,
|
|
loop_of,
|
|
)
|
|
from nisse_dev.transport import InMemoryTransport
|
|
|
|
pytestmark = [pytest.mark.integration, pytest.mark.e2e]
|
|
|
|
OPERATOR = "@dongar:dongar.net" # an allowlisted Matrix operator in nisse-test.yaml
|
|
DEADLINE_S = 1200
|
|
MAX_ANSWERS = 5 # per wait for something else, before giving up
|
|
|
|
|
|
@pytest.fixture
|
|
def running(
|
|
live_config: AgentConfig,
|
|
ssh_host: SSHHost,
|
|
tmp_path: Path,
|
|
monkeypatch: pytest.MonkeyPatch,
|
|
) -> Iterator[tuple[RunningLoop, InMemoryTransport, str]]:
|
|
from test_end_to_end import build_host
|
|
|
|
run = uuid.uuid4().hex[:8]
|
|
stack = contextlib.ExitStack()
|
|
# Kept outside pytest's temp directories, which pytest prunes, so a
|
|
# failed run's transcript can be read afterwards (nisse_dev.dump).
|
|
keep = Path(os.environ.get("NISSE_E2E_RUNS", "/tmp/nisse-e2e-runs")) / run
|
|
keep.mkdir(parents=True, exist_ok=True)
|
|
print(f"\ne2e run {run}: state store in {keep}")
|
|
operator = operator_gitea(live_config.gitea_url)
|
|
base = stack.enter_context(base_branch(operator, live_config.gitea_repo, run))
|
|
config = replace(
|
|
live_config,
|
|
loop="new",
|
|
base_branch=base,
|
|
state_db=keep / "state.db",
|
|
workspace=Path(f"/tmp/nisse-e2e-{run}/sandbox"),
|
|
build_host=build_host(ssh_host),
|
|
)
|
|
transport = InMemoryTransport()
|
|
monkeypatch.setattr("nisse_dev.loop.build.MatrixTransport", lambda cfg: transport)
|
|
agent = None
|
|
try:
|
|
agent = build_loop_agent(config)
|
|
# The test drives intake and PR watching itself, not on a timer.
|
|
agent.loop._interval = 3600
|
|
agent.start()
|
|
yield agent, transport, run
|
|
finally:
|
|
if agent is not None:
|
|
agent.stop()
|
|
host = SSHHost(
|
|
config.build_host.address,
|
|
key=config.build_host.key,
|
|
user=config.build_host.user,
|
|
port=config.build_host.port,
|
|
state_dir=tmp_path / "ssh",
|
|
)
|
|
host.execute(f"rm -rf /tmp/nisse-e2e-{run}")
|
|
stack.close() # the run's base branch
|
|
|
|
|
|
def is_artifact(path: str) -> bool:
|
|
parts = path.split("/")
|
|
return (
|
|
"__pycache__" in parts
|
|
or ".pytest_cache" in parts
|
|
or fnmatch(parts[-1], "*.py[co]")
|
|
)
|
|
|
|
|
|
def merge(gitea: GiteaClient, repo: str, pr: int, wait_s: float = 60) -> None:
|
|
"""Merge, waiting out Gitea's "Please try again later": it answers that
|
|
while it is still checking a freshly pushed PR can merge."""
|
|
deadline = time.monotonic() + wait_s
|
|
while True:
|
|
try:
|
|
gitea._request(
|
|
"POST", f"/repos/{repo}/pulls/{pr}/merge", json={"Do": "merge"}
|
|
)
|
|
return
|
|
except GiteaError as exc:
|
|
if "try again later" not in str(exc) or time.monotonic() > deadline:
|
|
raise
|
|
time.sleep(3)
|
|
|
|
|
|
class Operator:
|
|
def __init__(self, agent: RunningLoop, transport: InMemoryTransport) -> None:
|
|
self.agent = agent
|
|
self.transport = transport
|
|
|
|
def settle(self) -> None:
|
|
self.agent.worker.call(lambda: None, timeout=DEADLINE_S)
|
|
|
|
def tick(self) -> None:
|
|
"""Intake and PR watching, as the agent's timer would do."""
|
|
self.agent.worker.call(self.agent.loop.intake, timeout=DEADLINE_S)
|
|
|
|
def item(self, issue: int = 0) -> Any:
|
|
"""The work item for ``issue``; by default the first one."""
|
|
if issue:
|
|
store = self.agent.store
|
|
return self.agent.worker.call(lambda: item_for_issue(store, issue))
|
|
items = self.agent.worker.call(self.agent.store.work_items)
|
|
return items[0] if items else None
|
|
|
|
def say(self, text: str, issue: int = 0) -> None:
|
|
thread = loop_of(self.item(issue)).thread
|
|
self.transport.receive(OPERATOR, text, thread=thread)
|
|
self.settle()
|
|
|
|
def drive_until(self, wait: str, issue: int = 0) -> Any:
|
|
"""Answer whatever the agent asks until it waits for ``wait``."""
|
|
deadline = time.monotonic() + DEADLINE_S
|
|
answers = 0
|
|
while time.monotonic() < deadline:
|
|
item = self.item(issue)
|
|
assert item is not None, "no work item"
|
|
loop = loop_of(item)
|
|
if item.position == WAITING and loop.wait == wait:
|
|
return item
|
|
if item.status == "closed":
|
|
raise AssertionError(f"closed early: {self.last()}")
|
|
if item.position == WAITING and loop.wait == GO_AHEAD:
|
|
self.say("go", issue)
|
|
elif item.position == WAITING and loop.wait == ANSWER:
|
|
answers += 1
|
|
# Answering forever is how a stuck agent looks: give up.
|
|
assert answers <= MAX_ANSWERS, f"stuck asking: {self.last()}"
|
|
self.say("Use your best judgement and proceed.", issue)
|
|
else:
|
|
self.settle()
|
|
raise AssertionError(f"never waited for {wait}: {self.last()}")
|
|
|
|
def merge_and_close(self, issue: int) -> None:
|
|
"""Approve and merge the item's PR, and see the agent close the item."""
|
|
gitea, repo = self.agent.gitea, self.agent.config.gitea_repo
|
|
as_operator = operator_gitea(self.agent.config.gitea_url)
|
|
pr = loop_of(self.item(issue)).pr
|
|
with contextlib.suppress(GiteaError): # approval may not be required
|
|
as_operator._request(
|
|
"POST",
|
|
f"/repos/{repo}/pulls/{pr}/reviews",
|
|
json={"event": "APPROVED", "body": "looks good"},
|
|
)
|
|
merge(as_operator, repo, pr)
|
|
self.tick()
|
|
deadline = time.monotonic() + DEADLINE_S
|
|
nudges = 0
|
|
while self.item(issue).status != "closed" and time.monotonic() < deadline:
|
|
if loop_of(self.item(issue)).wait == ANSWER:
|
|
nudges += 1
|
|
assert nudges <= MAX_ANSWERS, f"never closed: {self.last()}"
|
|
self.say("It's merged; please close the item.", issue)
|
|
self.settle()
|
|
assert self.item(issue).outcome == "completed"
|
|
state = gitea._request("GET", f"/repos/{repo}/issues/{issue}")["state"]
|
|
assert state == "closed"
|
|
|
|
def tidy(self, issues: list[int], prs: list[int]) -> None:
|
|
"""Leave the sandbox tidy: whatever is still open gets closed."""
|
|
gitea, repo = self.agent.gitea, self.agent.config.gitea_repo
|
|
as_operator = operator_gitea(self.agent.config.gitea_url)
|
|
for issue in issues:
|
|
item = self.item(issue)
|
|
if item is not None and item.status == "open":
|
|
self.say("abandon", issue)
|
|
with contextlib.suppress(GiteaError):
|
|
as_operator.close_issue(repo, issue)
|
|
for pr in prs:
|
|
with contextlib.suppress(GiteaError):
|
|
head = gitea._request("GET", f"/repos/{repo}/pulls/{pr}")["head"]["ref"]
|
|
as_operator._request("DELETE", f"/repos/{repo}/branches/{head}")
|
|
|
|
def last(self) -> str:
|
|
return self.transport.outbound[-1].text if self.transport.outbound else ""
|
|
|
|
|
|
def test_the_slice_end_to_end(
|
|
running: tuple[RunningLoop, InMemoryTransport, str],
|
|
) -> None:
|
|
agent, transport, run = running
|
|
operator = Operator(agent, transport)
|
|
gitea, repo = agent.gitea, agent.config.gitea_repo
|
|
as_operator = operator_gitea(agent.config.gitea_url)
|
|
issue = None
|
|
pr = 0
|
|
try:
|
|
# --- the operator files an issue and assigns it to the agent (D1) ---
|
|
issue = as_operator.create_issue(repo, f"vowels_{run}", task(run))
|
|
as_operator._request(
|
|
"PATCH",
|
|
f"/repos/{repo}/issues/{issue}",
|
|
json={"assignees": [agent.account]},
|
|
)
|
|
operator.tick()
|
|
item = operator.item()
|
|
assert item is not None and loop_of(item).issue == issue
|
|
|
|
# --- restatement, go-ahead, milestone, PR (L3, D4) ---
|
|
item = operator.drive_until(MERGE)
|
|
pr = loop_of(item).pr
|
|
files = gitea._request("GET", f"/repos/{repo}/pulls/{pr}/files")
|
|
changed = [f["filename"] for f in files]
|
|
assert changed and not [p for p in changed if is_artifact(p)], changed
|
|
body = gitea._request("GET", f"/repos/{repo}/pulls/{pr}")["body"]
|
|
assert f"Closes #{issue}" in body
|
|
|
|
# --- a review comment is answered with a fix (D5) ---
|
|
# A request that can't already be met, so a fix must follow.
|
|
marker = f"reviewed-{uuid.uuid4().hex[:8]}"
|
|
before = gitea._request("GET", f"/repos/{repo}/pulls/{pr}")["head"]["sha"]
|
|
as_operator.post_comment(
|
|
repo, pr, f"Please add the comment line `# {marker}` above vowels_{run}."
|
|
)
|
|
operator.tick()
|
|
operator.drive_until(MERGE)
|
|
after = gitea._request("GET", f"/repos/{repo}/pulls/{pr}")["head"]["sha"]
|
|
assert after != before, "no fix was pushed for the review comment"
|
|
diff = gitea._http.get(f"/repos/{repo}/pulls/{pr}.diff").text
|
|
assert marker in diff, "the pushed fix doesn't do what the comment asked"
|
|
|
|
# --- the operator approves and merges; the agent closes the item ---
|
|
operator.merge_and_close(issue)
|
|
finally:
|
|
operator.tidy([issue] if issue is not None else [], [pr] if pr else [])
|
|
|
|
|
|
def test_an_issue_blocked_on_another_waits_for_it(
|
|
running: tuple[RunningLoop, InMemoryTransport, str],
|
|
) -> None:
|
|
"""M2.2: B depends on A in Gitea. B is blocked until A's PR merges and A
|
|
closes; then B starts from a workspace that has A's work."""
|
|
agent, transport, run = running
|
|
operator = Operator(agent, transport)
|
|
gitea, repo = agent.gitea, agent.config.gitea_repo
|
|
as_operator = operator_gitea(agent.config.gitea_url)
|
|
owner, name = repo.split("/")
|
|
issues: list[int] = []
|
|
prs: list[int] = []
|
|
try:
|
|
a = as_operator.create_issue(repo, f"vowels_{run}", task(run))
|
|
issues.append(a)
|
|
b = as_operator.create_issue(repo, f"consonants_{run}", dependent_task(run))
|
|
issues.append(b)
|
|
as_operator._request(
|
|
"POST",
|
|
f"/repos/{repo}/issues/{b}/dependencies",
|
|
json={"owner": owner, "repo": name, "index": a},
|
|
)
|
|
for n in (b, a): # B first: the dependency, not the order, decides
|
|
as_operator._request(
|
|
"PATCH",
|
|
f"/repos/{repo}/issues/{n}",
|
|
json={"assignees": [agent.account]},
|
|
)
|
|
operator.tick()
|
|
assert operator.item(b).position == BLOCKED
|
|
assert loop_of(operator.item(b)).blocked_by == (f"{repo}#{a}",)
|
|
|
|
# --- A runs to a merged PR; B stays blocked meanwhile ---
|
|
item = operator.drive_until(MERGE, a)
|
|
prs.append(loop_of(item).pr)
|
|
assert operator.item(b).position == BLOCKED
|
|
operator.merge_and_close(a)
|
|
|
|
# --- A closed: B is unblocked and starts from A's work ---
|
|
operator.tick()
|
|
assert operator.item(b).position != BLOCKED
|
|
item = operator.drive_until(MERGE, b)
|
|
prs.append(loop_of(item).pr)
|
|
files = gitea._request("GET", f"/repos/{repo}/pulls/{prs[-1]}/files")
|
|
diff = gitea._http.get(f"/repos/{repo}/pulls/{prs[-1]}.diff").text
|
|
assert f"consonants_{run}" in diff, [f["filename"] for f in files]
|
|
# Built on A's merged work, so A's function isn't added again.
|
|
assert f"\n+def vowels_{run}" not in diff
|
|
finally:
|
|
operator.tidy(issues, prs)
|