"""The end-to-end tests on the new loop (Development Plan M2.1, M2.2): the whole slice, from an issue assigned in Gitea to a merged PR and a closed work item, and the queue, with an issue blocked on another until it closes; with real models, the Gitea sandbox and a real build host. A scripted operator plays the operator's part: in Gitea through its own test account (NISSE_OPERATOR_TOKEN; an operator only in nisse-test.yaml), and in the item's Matrix thread through the in-memory transport. Run with scripts/e2e.sh after each milestone. """ import contextlib import os import time import uuid from collections.abc import Iterator from dataclasses import replace from fnmatch import fnmatch from pathlib import Path from typing import Any import pytest from e2e_support import base_branch, dependent_task, operator_gitea, task from nisse_dev.config import AgentConfig from nisse_dev.execution import SSHHost from nisse_dev.gitea import GiteaClient, GiteaError from nisse_dev.loop.build import RunningLoop, build_loop_agent from nisse_dev.loop.items import ( ANSWER, BLOCKED, GO_AHEAD, MERGE, WAITING, item_for_issue, loop_of, ) from nisse_dev.transport import InMemoryTransport pytestmark = [pytest.mark.integration, pytest.mark.e2e] OPERATOR = "@dongar:dongar.net" # an allowlisted Matrix operator in nisse-test.yaml DEADLINE_S = 1200 MAX_ANSWERS = 5 # per wait for something else, before giving up @pytest.fixture def running( live_config: AgentConfig, ssh_host: SSHHost, tmp_path: Path, monkeypatch: pytest.MonkeyPatch, ) -> Iterator[tuple[RunningLoop, InMemoryTransport, str]]: from test_end_to_end import build_host run = uuid.uuid4().hex[:8] stack = contextlib.ExitStack() # Kept outside pytest's temp directories, which pytest prunes, so a # failed run's transcript can be read afterwards (nisse_dev.dump). keep = Path(os.environ.get("NISSE_E2E_RUNS", "/tmp/nisse-e2e-runs")) / run keep.mkdir(parents=True, exist_ok=True) print(f"\ne2e run {run}: state store in {keep}") operator = operator_gitea(live_config.gitea_url) base = stack.enter_context(base_branch(operator, live_config.gitea_repo, run)) config = replace( live_config, loop="new", base_branch=base, state_db=keep / "state.db", workspace=Path(f"/tmp/nisse-e2e-{run}/sandbox"), build_host=build_host(ssh_host), ) transport = InMemoryTransport() monkeypatch.setattr("nisse_dev.loop.build.MatrixTransport", lambda cfg: transport) agent = None try: agent = build_loop_agent(config) # The test drives intake and PR watching itself, not on a timer. agent.loop._interval = 3600 agent.start() yield agent, transport, run finally: if agent is not None: agent.stop() host = SSHHost( config.build_host.address, key=config.build_host.key, user=config.build_host.user, port=config.build_host.port, state_dir=tmp_path / "ssh", ) host.execute(f"rm -rf /tmp/nisse-e2e-{run}") stack.close() # the run's base branch def is_artifact(path: str) -> bool: parts = path.split("/") return ( "__pycache__" in parts or ".pytest_cache" in parts or fnmatch(parts[-1], "*.py[co]") ) def merge(gitea: GiteaClient, repo: str, pr: int, wait_s: float = 60) -> None: """Merge, waiting out Gitea's "Please try again later": it answers that while it is still checking a freshly pushed PR can merge.""" deadline = time.monotonic() + wait_s while True: try: gitea._request( "POST", f"/repos/{repo}/pulls/{pr}/merge", json={"Do": "merge"} ) return except GiteaError as exc: if "try again later" not in str(exc) or time.monotonic() > deadline: raise time.sleep(3) class Operator: def __init__(self, agent: RunningLoop, transport: InMemoryTransport) -> None: self.agent = agent self.transport = transport def settle(self) -> None: self.agent.worker.call(lambda: None, timeout=DEADLINE_S) def tick(self) -> None: """Intake and PR watching, as the agent's timer would do.""" self.agent.worker.call(self.agent.loop.intake, timeout=DEADLINE_S) def item(self, issue: int = 0) -> Any: """The work item for ``issue``; by default the first one.""" if issue: store = self.agent.store return self.agent.worker.call(lambda: item_for_issue(store, issue)) items = self.agent.worker.call(self.agent.store.work_items) return items[0] if items else None def say(self, text: str, issue: int = 0) -> None: thread = loop_of(self.item(issue)).thread self.transport.receive(OPERATOR, text, thread=thread) self.settle() def drive_until(self, wait: str, issue: int = 0) -> Any: """Answer whatever the agent asks until it waits for ``wait``.""" deadline = time.monotonic() + DEADLINE_S answers = 0 while time.monotonic() < deadline: item = self.item(issue) assert item is not None, "no work item" loop = loop_of(item) if item.position == WAITING and loop.wait == wait: return item if item.status == "closed": raise AssertionError(f"closed early: {self.last()}") if item.position == WAITING and loop.wait == GO_AHEAD: self.say("go", issue) elif item.position == WAITING and loop.wait == ANSWER: answers += 1 # Answering forever is how a stuck agent looks: give up. assert answers <= MAX_ANSWERS, f"stuck asking: {self.last()}" self.say("Use your best judgement and proceed.", issue) else: self.settle() raise AssertionError(f"never waited for {wait}: {self.last()}") def merge_and_close(self, issue: int) -> None: """Approve and merge the item's PR, and see the agent close the item.""" gitea, repo = self.agent.gitea, self.agent.config.gitea_repo as_operator = operator_gitea(self.agent.config.gitea_url) pr = loop_of(self.item(issue)).pr with contextlib.suppress(GiteaError): # approval may not be required as_operator._request( "POST", f"/repos/{repo}/pulls/{pr}/reviews", json={"event": "APPROVED", "body": "looks good"}, ) merge(as_operator, repo, pr) self.tick() deadline = time.monotonic() + DEADLINE_S nudges = 0 while self.item(issue).status != "closed" and time.monotonic() < deadline: if loop_of(self.item(issue)).wait == ANSWER: nudges += 1 assert nudges <= MAX_ANSWERS, f"never closed: {self.last()}" self.say("It's merged; please close the item.", issue) self.settle() assert self.item(issue).outcome == "completed" state = gitea._request("GET", f"/repos/{repo}/issues/{issue}")["state"] assert state == "closed" def tidy(self, issues: list[int], prs: list[int]) -> None: """Leave the sandbox tidy: whatever is still open gets closed.""" gitea, repo = self.agent.gitea, self.agent.config.gitea_repo as_operator = operator_gitea(self.agent.config.gitea_url) for issue in issues: item = self.item(issue) if item is not None and item.status == "open": self.say("abandon", issue) with contextlib.suppress(GiteaError): as_operator.close_issue(repo, issue) for pr in prs: with contextlib.suppress(GiteaError): head = gitea._request("GET", f"/repos/{repo}/pulls/{pr}")["head"]["ref"] as_operator._request("DELETE", f"/repos/{repo}/branches/{head}") def last(self) -> str: return self.transport.outbound[-1].text if self.transport.outbound else "" def test_the_slice_end_to_end( running: tuple[RunningLoop, InMemoryTransport, str], ) -> None: agent, transport, run = running operator = Operator(agent, transport) gitea, repo = agent.gitea, agent.config.gitea_repo as_operator = operator_gitea(agent.config.gitea_url) issue = None pr = 0 try: # --- the operator files an issue and assigns it to the agent (D1) --- issue = as_operator.create_issue(repo, f"vowels_{run}", task(run)) as_operator._request( "PATCH", f"/repos/{repo}/issues/{issue}", json={"assignees": [agent.account]}, ) operator.tick() item = operator.item() assert item is not None and loop_of(item).issue == issue # --- restatement, go-ahead, milestone, PR (L3, D4) --- item = operator.drive_until(MERGE) pr = loop_of(item).pr files = gitea._request("GET", f"/repos/{repo}/pulls/{pr}/files") changed = [f["filename"] for f in files] assert changed and not [p for p in changed if is_artifact(p)], changed body = gitea._request("GET", f"/repos/{repo}/pulls/{pr}")["body"] assert f"Closes #{issue}" in body # --- a review comment is answered with a fix (D5) --- # A request that can't already be met, so a fix must follow. marker = f"reviewed-{uuid.uuid4().hex[:8]}" before = gitea._request("GET", f"/repos/{repo}/pulls/{pr}")["head"]["sha"] as_operator.post_comment( repo, pr, f"Please add the comment line `# {marker}` above vowels_{run}." ) operator.tick() operator.drive_until(MERGE) after = gitea._request("GET", f"/repos/{repo}/pulls/{pr}")["head"]["sha"] assert after != before, "no fix was pushed for the review comment" diff = gitea._http.get(f"/repos/{repo}/pulls/{pr}.diff").text assert marker in diff, "the pushed fix doesn't do what the comment asked" # --- the operator approves and merges; the agent closes the item --- operator.merge_and_close(issue) finally: operator.tidy([issue] if issue is not None else [], [pr] if pr else []) def test_an_issue_blocked_on_another_waits_for_it( running: tuple[RunningLoop, InMemoryTransport, str], ) -> None: """M2.2: B depends on A in Gitea. B is blocked until A's PR merges and A closes; then B starts from a workspace that has A's work.""" agent, transport, run = running operator = Operator(agent, transport) gitea, repo = agent.gitea, agent.config.gitea_repo as_operator = operator_gitea(agent.config.gitea_url) owner, name = repo.split("/") issues: list[int] = [] prs: list[int] = [] try: a = as_operator.create_issue(repo, f"vowels_{run}", task(run)) issues.append(a) b = as_operator.create_issue(repo, f"consonants_{run}", dependent_task(run)) issues.append(b) as_operator._request( "POST", f"/repos/{repo}/issues/{b}/dependencies", json={"owner": owner, "repo": name, "index": a}, ) for n in (b, a): # B first: the dependency, not the order, decides as_operator._request( "PATCH", f"/repos/{repo}/issues/{n}", json={"assignees": [agent.account]}, ) operator.tick() assert operator.item(b).position == BLOCKED assert loop_of(operator.item(b)).blocked_by == (f"{repo}#{a}",) # --- A runs to a merged PR; B stays blocked meanwhile --- item = operator.drive_until(MERGE, a) prs.append(loop_of(item).pr) assert operator.item(b).position == BLOCKED operator.merge_and_close(a) # --- A closed: B is unblocked and starts from A's work --- operator.tick() assert operator.item(b).position != BLOCKED item = operator.drive_until(MERGE, b) prs.append(loop_of(item).pr) files = gitea._request("GET", f"/repos/{repo}/pulls/{prs[-1]}/files") diff = gitea._http.get(f"/repos/{repo}/pulls/{prs[-1]}.diff").text assert f"consonants_{run}" in diff, [f["filename"] for f in files] # Built on A's merged work, so A's function isn't added again. assert f"\n+def vowels_{run}" not in diff finally: operator.tidy(issues, prs)