Files
nisse/tests/integration/e2e_support.py
T
dongarandClaude Opus 5.5 6db80dfb4b
CI / checks (push) Skipped
CI / checks (pull_request) Successful in 5m26s
CI / e2e (pull_request) Successful in 17m27s
End-to-end tests on a local model, and in CI on every PR
- Every e2e role runs on qwen3.8-27b on the RTX 4090 (e2e_roles();
  NISSE_E2E_MODEL overrides, e.g. claude-haiku-4-5). The first full run
  passed 7 of 8 in 26 minutes for $0.0000; the eighth was a real finding,
  fixed with the new-project PR.
- scripts/e2e.sh reports a run's spend on the test key and the day's total
  against its budget: above $0 on the local model means a paid fallback.
- CI runs the suite as its own job on pull requests (not pushes to main),
  on a throwaway sshd (--local), cancelling a run a newer push supersedes,
  so the checks job still reports in minutes.
- The plan says so: e2e on every PR; locally only -k / --lf.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01PPQ14L3LikyefCMFBqEbKE
2026-10-07 15:14:43 +00:00

146 lines
4.9 KiB
Python

"""Shared by the live tests: each run gets a repository of its own, made by
the test operator and deleted afterwards, so runs never depend on one
another or on what earlier runs merged."""
import base64
import contextlib
import os
import subprocess
from collections.abc import Iterator
from dataclasses import dataclass
from pathlib import Path
from nisse_dev.gitea import GiteaClient, GiteaError
from nisse_dev.llm import RoleMap
# The end-to-end tests run every role on a local model, which costs nothing
# per call; NISSE_E2E_MODEL overrides it (e.g. claude-haiku-4-5 for a quick,
# paid run).
E2E_MODEL = "qwen3.8-27b"
def e2e_roles() -> RoleMap:
model = os.environ.get("NISSE_E2E_MODEL", "").strip() or E2E_MODEL
return RoleMap(executor=model, planner=model, helper=model, expert=model)
def task(run: str) -> str:
"""A request no earlier run can have fulfilled."""
return (
f"Add a function vowels_{run}(text) to the sandbox package that returns "
"how many of a, e, i, o, u (either case) the text contains, with tests."
)
def dependent_task(run: str) -> str:
"""A request that builds on task(run): it can only be done after it."""
return (
f"Add a function consonants_{run}(text) to the sandbox package that "
"returns how many letters in the text are not vowels, using "
f"vowels_{run}, with tests."
)
def operator_gitea(url: str) -> GiteaClient:
token = os.environ.get("NISSE_OPERATOR_TOKEN", "")
assert token.strip(), (
"NISSE_OPERATOR_TOKEN is not set: the test operator's Gitea token"
)
return GiteaClient(url, token)
OWNER = "nisse-testing-operator" # the test operator: owns every test repository
SEED = Path(__file__).parent / "seed"
@dataclass(frozen=True)
class FreshRepo:
name: str # owner/repo
push_key: Path # this repository's deploy key, private half
@contextlib.contextmanager
def fresh_repo(
operator: GiteaClient, run: str, keys: Path, *, agent: str
) -> Iterator[FreshRepo]:
"""A repository of its own for one test run, as the operator would make a
project's (D4, D8): seeded with a small Python package, the ``agent``
account a collaborator with write access, the default branch protected (pushes only by the
owner, an approval to merge), and a deploy key of its own, written under
``keys``. Deleted afterwards, unless NISSE_E2E_KEEP_REPOS is set."""
name = f"nisse-e2e-{run}"
full = f"{OWNER}/{name}"
operator._request(
"POST",
"/user/repos",
json={
"name": name,
"private": True,
"auto_init": True,
"default_branch": "main",
},
)
try:
files = [
{
"operation": "create",
"path": str(path.relative_to(SEED)),
"content": base64.b64encode(path.read_bytes()).decode(),
}
for path in sorted(SEED.rglob("*"))
if path.is_file() and path.name != "README.md"
]
operator._request(
"POST",
f"/repos/{full}/contents",
json={"branch": "main", "message": "Seed the test project", "files": files},
)
readme = operator._request("GET", f"/repos/{full}/contents/README.md")
operator._request(
"PUT",
f"/repos/{full}/contents/README.md",
json={
"branch": "main",
"message": "README",
"sha": readme["sha"],
"content": base64.b64encode((SEED / "README.md").read_bytes()).decode(),
},
)
operator._request(
"PUT", f"/repos/{full}/collaborators/{agent}", json={"permission": "write"}
)
keys.mkdir(parents=True, exist_ok=True)
key = keys / f"{name}.key"
key.unlink(missing_ok=True)
subprocess.run(
["ssh-keygen", "-q", "-t", "ed25519", "-N", "", "-C", name, "-f", str(key)],
check=True,
)
operator._request(
"POST",
f"/repos/{full}/keys",
json={
"title": f"{name} push",
"key": key.with_suffix(".key.pub").read_text().strip(),
"read_only": False,
},
)
operator._request(
"POST",
f"/repos/{full}/branch_protections",
# Pushes only by the operator (who sets up fixtures, as an owner
# can); the agent's are refused, and merging needs an approval.
json={
"branch_name": "main",
"enable_push": True,
"enable_push_whitelist": True,
"push_whitelist_usernames": [OWNER],
"required_approvals": 1,
},
)
yield FreshRepo(full, key)
finally:
if not os.environ.get("NISSE_E2E_KEEP_REPOS"):
with contextlib.suppress(GiteaError):
operator._request("DELETE", f"/repos/{full}")