From 6db80dfb4bf2684c986471cbc18346d09aaace42 Mon Sep 17 00:00:00 2001 From: Don Garrett Date: Wed, 7 Oct 2026 15:14:43 +0000 Subject: [PATCH] End-to-end tests on a local model, and in CI on every PR - Every e2e role runs on qwen3.8-27b on the RTX 4090 (e2e_roles(); NISSE_E2E_MODEL overrides, e.g. claude-haiku-4-5). The first full run passed 7 of 8 in 26 minutes for $0.0000; the eighth was a real finding, fixed with the new-project PR. - scripts/e2e.sh reports a run's spend on the test key and the day's total against its budget: above $0 on the local model means a paid fallback. - CI runs the suite as its own job on pull requests (not pushes to main), on a throwaway sshd (--local), cancelling a run a newer push supersedes, so the checks job still reports in minutes. - The plan says so: e2e on every PR; locally only -k / --lf. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01PPQ14L3LikyefCMFBqEbKE --- .gitea/workflows/ci.yml | 41 +++++++++++++++++++++++ docs/Development-Plan.md | 8 +++-- scripts/e2e.sh | 34 ++++++++++++++++--- tests/integration/e2e_support.py | 11 ++++++ tests/integration/test_end_to_end.py | 3 +- tests/integration/test_end_to_end_loop.py | 3 +- 6 files changed, 91 insertions(+), 9 deletions(-) diff --git a/.gitea/workflows/ci.yml b/.gitea/workflows/ci.yml index e261313..0a98acd 100644 --- a/.gitea/workflows/ci.yml +++ b/.gitea/workflows/ci.yml @@ -54,3 +54,44 @@ jobs: # key) and deletes it afterwards. NISSE_OPERATOR_TOKEN: ${{ secrets.NISSE_OPERATOR_TOKEN }} run: uv run pytest -m "integration and not e2e" -rs + + e2e: + # The end-to-end suite (scripts/e2e.sh): the whole agent against real + # services, every role on a local model, so it spends nothing but keeps + # the RTX 4090 busy for ~25 minutes. On PRs only, as its own job so the + # checks above report first; a newer push to the same PR cancels the + # run it supersedes. Locally, run only what you're writing (-k) or + # diagnosing (--lf). + if: github.event_name != 'push' + runs-on: ubuntu-latest + timeout-minutes: 90 + concurrency: + group: e2e-${{ github.head_ref || github.ref }} + cancel-in-progress: true + steps: + - uses: actions/checkout@v4 + + - name: Install uv + run: | + curl -LsSf https://astral.sh/uv/install.sh | sh + echo "$HOME/.local/bin" >> "$GITHUB_PATH" + + - name: Sync workspace + run: uv sync --all-packages + + - name: Install sshd + # The build VM is a throwaway sshd here (scripts/e2e.sh --local). + run: | + SUDO=$(command -v sudo || true) + $SUDO apt-get update -qq + $SUDO apt-get install -y -qq openssh-server openssh-client > /dev/null + $SUDO mkdir -p /run/sshd + + - name: End-to-end tests + env: + NISSE_LLM_TOKEN: ${{ secrets.NISSE_LLM_TOKEN }} + NISSE_MATRIX_TOKEN: ${{ secrets.NISSE_MATRIX_TOKEN }} + NISSE_GITEA_TOKEN: ${{ secrets.NISSE_GITEA_TOKEN }} + NISSE_OPERATOR_TOKEN: ${{ secrets.NISSE_OPERATOR_TOKEN }} + NISSE_E2E_RUNS: /tmp/nisse-e2e-runs + run: scripts/e2e.sh --local -v diff --git a/docs/Development-Plan.md b/docs/Development-Plan.md index cd6dbaa..e7cb803 100644 --- a/docs/Development-Plan.md +++ b/docs/Development-Plan.md @@ -73,9 +73,11 @@ Only `nisse-dev` has real code; `nisse-core` and `nisse-ops` are placeholders. `CLAUDE.md`. A milestone marked **needs discussion** starts with a design discussion first. - **Every change comes with tests,** through `scripts/check.sh`. -- **The end-to-end test follows each milestone** (M1.6): the real agent, real models, - a repository of the run's own and a real build host, driven by a scripted operator. It runs - after each milestone, not on every push, since it spends model tokens. Manual +- **The end-to-end test runs on every PR** (M1.6): the real agent, every role on a + local model (so it costs nothing per call), a repository of the run's own and a + real build host, driven by a scripted operator. CI runs it as its own job on pull + requests, cancelling a run a newer push supersedes. Locally only the tests being + written (`-k`) or diagnosed (`--lf`); `scripts/e2e.sh` reports what a run spent. Manual live runs are for the new loop's first runs and phase ends. Findings are filed as issues and either fixed before moving on or deliberately deferred. - **The Quint models move with the loop:** `orchestrator.qnt` while `control.py` diff --git a/scripts/e2e.sh b/scripts/e2e.sh index a4871f2..f16c1e9 100755 --- a/scripts/e2e.sh +++ b/scripts/e2e.sh @@ -1,13 +1,16 @@ #!/usr/bin/env bash # The end-to-end test (Development Plan M1.6): the whole agent against real -# services, driven by a scripted operator. Run after each milestone; it -# spends model tokens and takes minutes, so it is not part of check.sh or CI. +# services, driven by a scripted operator. CI runs it on every PR (with +# --local); locally, run only the tests you're writing (-k) or diagnosing +# (--lf). # # scripts/e2e.sh on the bring-up build VM (configured below) # scripts/e2e.sh --local on a throwaway local sshd instead # # Uses tests/integration/nisse-test.yaml and its secrets (.env at the repo -# root), like the integration tests. +# root), like the integration tests. Every role runs on a local model +# (tests/integration/e2e_support.py; NISSE_E2E_MODEL overrides it), so tell +# the operator before a full run: it keeps the RTX 4090 busy for a while. set -euo pipefail cd "$(dirname "$0")/.." @@ -19,4 +22,27 @@ else fi # Each run's state store is kept here, for nisse_dev.dump after a failure. export NISSE_E2E_RUNS=${NISSE_E2E_RUNS:-/root/nisse-e2e/test-runs} -exec uv run pytest -m e2e -rs tests/integration/test_end_to_end.py tests/integration/test_end_to_end_loop.py "$@" + +# What the run spent on the test key, read from LiteLLM (spend only). The +# end-to-end tests run on a local model, so anything above $0 means a call +# went to a paid one: an override, or a proxy fallback. +spend() { + (set -a; [ -f .env ] && . ./.env; set +a + uv run python -c ' +import os, httpx +r = httpx.get("https://litellm.dongar.net/key/info", timeout=30, + headers={"Authorization": "Bearer " + os.environ["NISSE_LLM_TOKEN"]}) +i = r.json()["info"] +print(i["spend"], i.get("max_budget"))' 2>/dev/null) || echo "? ?" +} +read -r before budget < <(spend) +status=0 +uv run pytest -m e2e -rs tests/integration/test_end_to_end.py tests/integration/test_end_to_end_loop.py "$@" || status=$? +read -r after _ < <(spend) +python3 -c " +b, a, cap = '$before', '$after', '$budget' +try: + print(f'e2e spend: \${float(a) - float(b):.4f} this run; \${float(a):.2f} of \${float(cap):.2f} today') +except ValueError: + print('e2e spend: unknown (could not read the key)')" +exit "$status" diff --git a/tests/integration/e2e_support.py b/tests/integration/e2e_support.py index f58b57b..bd687da 100644 --- a/tests/integration/e2e_support.py +++ b/tests/integration/e2e_support.py @@ -11,6 +11,17 @@ from dataclasses import dataclass from pathlib import Path from nisse_dev.gitea import GiteaClient, GiteaError +from nisse_dev.llm import RoleMap + +# The end-to-end tests run every role on a local model, which costs nothing +# per call; NISSE_E2E_MODEL overrides it (e.g. claude-haiku-4-5 for a quick, +# paid run). +E2E_MODEL = "qwen3.8-27b" + + +def e2e_roles() -> RoleMap: + model = os.environ.get("NISSE_E2E_MODEL", "").strip() or E2E_MODEL + return RoleMap(executor=model, planner=model, helper=model, expert=model) def task(run: str) -> str: diff --git a/tests/integration/test_end_to_end.py b/tests/integration/test_end_to_end.py index 8254a50..408d90a 100644 --- a/tests/integration/test_end_to_end.py +++ b/tests/integration/test_end_to_end.py @@ -25,7 +25,7 @@ from pathlib import Path from typing import Any import pytest -from e2e_support import fresh_repo, operator_gitea +from e2e_support import e2e_roles, fresh_repo, operator_gitea from e2e_support import task as make_task from nisse_dev.agent import Agent, build_agent @@ -84,6 +84,7 @@ def agent( workspace=Path(f"/tmp/nisse-e2e-{run}/sandbox"), build_host=build_host(ssh_host), ), + roles=e2e_roles(), state_db=tmp_path / "state.db", ) transport = InMemoryTransport() diff --git a/tests/integration/test_end_to_end_loop.py b/tests/integration/test_end_to_end_loop.py index e44e034..1de86f5 100644 --- a/tests/integration/test_end_to_end_loop.py +++ b/tests/integration/test_end_to_end_loop.py @@ -21,7 +21,7 @@ from pathlib import Path from typing import Any import pytest -from e2e_support import dependent_task, fresh_repo, operator_gitea, task +from e2e_support import dependent_task, e2e_roles, fresh_repo, operator_gitea, task from nisse_dev.config import AgentConfig, LoopTuning from nisse_dev.execution import SSHHost @@ -74,6 +74,7 @@ def running( build_host=build_host(ssh_host), ), loop="new", + roles=e2e_roles(), state_db=keep / "state.db", # Short, so a bug's grace period runs out within the test. tuning=replace(live_config.tuning, loop=LoopTuning(bug_grace_s=BUG_GRACE_S)), -- 2.54.0