From e35aea504549f703375fb1a02bb02bedecf82d17 Mon Sep 17 00:00:00 2001 From: Don Garrett Date: Wed, 7 Oct 2026 21:53:31 +0000 Subject: [PATCH 1/3] Test levels: smoke e2e on PRs, the full set on e2e:full and before a release - Every PR: lint, unit tests across the runner's cores (pytest-xdist), 16 property walks from a seed that moves with the run number, integration tests, and the end-to-end smoke tests (the slice, a conversation filed as work, a question; `smoke` marker). - A PR labelled e2e:full, a push to main, or a manual run: all 48 walks and the whole end-to-end suite. Adding e2e:full to a PR runs it; any other label runs nothing. - A published release runs the full set before its image is built. - The walk's seeds come from NISSE_WALK_SEEDS / NISSE_WALK_FIRST (48 from 0 by default). Measured here: the unit tier on 2 workers took 3m49s against 6-8 minutes serially. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01PPQ14L3LikyefCMFBqEbKE --- .gitea/workflows/ci.yml | 42 +++++++++++++++++---- .gitea/workflows/release.yml | 46 ++++++++++++++++++++++- docs/Development-Plan.md | 12 ++++-- nisse-dev/tests/test_loop_properties.py | 11 +++++- pyproject.toml | 2 + tests/integration/test_end_to_end_loop.py | 3 ++ uv.lock | 23 ++++++++++++ 7 files changed, 124 insertions(+), 15 deletions(-) diff --git a/.gitea/workflows/ci.yml b/.gitea/workflows/ci.yml index 9d2fd61..81afecc 100644 --- a/.gitea/workflows/ci.yml +++ b/.gitea/workflows/ci.yml @@ -1,4 +1,10 @@ name: CI +# Three levels of testing: +# - every PR: lint, unit tests (16 property walks), integration tests, and +# the smoke end-to-end tests (`smoke` marker); +# - a PR labelled e2e:full, a push to main and a manual run: all 48 walks +# and the whole end-to-end suite; +# - a published release: the full set again before its image (release.yml). on: # Only main on push. A PR branch is already covered by the pull_request # event, and running both fired two identical suites concurrently — which @@ -7,11 +13,18 @@ on: push: branches: [main] pull_request: + # labeled: adding e2e:full runs the full set; any other label, nothing. + types: [opened, synchronize, reopened, labeled] # Manual re-run, e.g. after rotating a credential or raising a budget cap. workflow_dispatch: +env: + # Full on e2e:full, and outside PRs. + FULL: ${{ github.event_name != 'pull_request' || contains(github.event.pull_request.labels.*.name, 'e2e:full') }} + jobs: checks: + if: github.event.action != 'labeled' || github.event.label.name == 'e2e:full' runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 @@ -40,7 +53,12 @@ jobs: run: uv run ruff format --check . - name: Unit tests - run: uv run pytest -m "not integration and not e2e" + # Across the runner's cores (pytest-xdist). On a PR, 16 property + # walks from a seed that moves with the run number; all 48 otherwise. + env: + NISSE_WALK_SEEDS: ${{ env.FULL == 'true' && '48' || '16' }} + NISSE_WALK_FIRST: ${{ env.FULL == 'true' && '0' || github.run_number }} + run: uv run pytest -n auto -m "not integration and not e2e" - name: Integration tests # Everything but the secrets is in tests/integration/nisse-test.yaml, @@ -58,12 +76,15 @@ jobs: e2e: # The end-to-end suite (scripts/e2e.sh): the whole agent against real # services, every role on a local model, so it spends nothing but keeps - # the RTX 4090 busy for ~25 minutes. On PRs only, and only after the - # checks pass: a PR that's red already doesn't spend the GPU. There is - # one runner and Gitea doesn't honour cancel-in-progress, so a run a - # newer push supersedes holds the runner until it's cancelled by hand. - # Locally, run only what you're writing (-k) or diagnosing (--lf). - if: github.event_name != 'push' + # the RTX 4090 busy: ~10 minutes for the smoke tests, ~25+ for all. On + # PRs and manual runs, and only after the checks pass: a PR that's red + # already doesn't spend the GPU. There is one runner and Gitea doesn't + # honour cancel-in-progress, so a run a newer push supersedes holds the + # runner until it's cancelled by hand. Locally, run only what you're + # writing (-k) or diagnosing (--lf). + if: >- + github.event_name != 'push' + && (github.event.action != 'labeled' || github.event.label.name == 'e2e:full') needs: checks runs-on: ubuntu-latest timeout-minutes: 90 @@ -93,4 +114,9 @@ jobs: NISSE_GITEA_TOKEN: ${{ secrets.NISSE_GITEA_TOKEN }} NISSE_OPERATOR_TOKEN: ${{ secrets.NISSE_OPERATOR_TOKEN }} NISSE_E2E_RUNS: /tmp/nisse-e2e-runs - run: scripts/e2e.sh --local -v + run: | + if [ "$FULL" = true ]; then + scripts/e2e.sh --local -v + else + scripts/e2e.sh --local -v -m "e2e and smoke" + fi diff --git a/.gitea/workflows/release.yml b/.gitea/workflows/release.yml index 000a06d..57c7081 100644 --- a/.gitea/workflows/release.yml +++ b/.gitea/workflows/release.yml @@ -3,7 +3,8 @@ name: Release # registry (Development Plan M1.5, X4): gitea.dongar.net/dev/nisse-dev # tagged with the version (the release tag without its v) and latest. Cut a # release in Gitea (Releases → New release, tag vX.Y.Z); a tag pushed for any -# other reason builds nothing. +# other reason builds nothing. The full test set runs first: no image without +# it passing. # # The registry needs a personal access token with write:package # (REGISTRY_TOKEN, an organization secret of `dev`, belonging to the @@ -23,7 +24,50 @@ env: IMAGE: gitea.dongar.net/dev/nisse-dev jobs: + tests: + # Everything, before an image exists: lint, unit tests with all 48 + # property walks, integration tests and the whole end-to-end suite. + runs-on: ubuntu-latest + timeout-minutes: 150 + env: + NISSE_LLM_TOKEN: ${{ secrets.NISSE_LLM_TOKEN }} + NISSE_MATRIX_TOKEN: ${{ secrets.NISSE_MATRIX_TOKEN }} + NISSE_GITEA_TOKEN: ${{ secrets.NISSE_GITEA_TOKEN }} + NISSE_OPERATOR_TOKEN: ${{ secrets.NISSE_OPERATOR_TOKEN }} + NISSE_E2E_RUNS: /tmp/nisse-e2e-runs + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Install uv + run: | + curl -LsSf https://astral.sh/uv/install.sh | sh + echo "$HOME/.local/bin" >> "$GITHUB_PATH" + + - name: Sync workspace + run: uv sync --all-packages + + - name: Install sshd + run: | + SUDO=$(command -v sudo || true) + $SUDO apt-get update -qq + $SUDO apt-get install -y -qq openssh-server openssh-client > /dev/null + $SUDO mkdir -p /run/sshd + + - name: Lint + run: uv run ruff check . && uv run ruff format --check . + + - name: Unit tests + run: uv run pytest -n auto -m "not integration and not e2e" + + - name: Integration tests + run: uv run pytest -m "integration and not e2e" -rs + + - name: End-to-end tests + run: scripts/e2e.sh --local -v + build-and-push: + needs: tests runs-on: ubuntu-latest permissions: contents: read diff --git a/docs/Development-Plan.md b/docs/Development-Plan.md index 5127628..91b468a 100644 --- a/docs/Development-Plan.md +++ b/docs/Development-Plan.md @@ -73,10 +73,14 @@ Only `nisse-dev` has real code; `nisse-core` and `nisse-ops` are placeholders. `CLAUDE.md`. A milestone marked **needs discussion** starts with a design discussion first. - **Every change comes with tests,** through `scripts/check.sh`. -- **The end-to-end test runs on every PR** (M1.6): the real agent, every role on a - local model (so it costs nothing per call), a repository of the run's own and a - real build host, driven by a scripted operator. CI runs it as its own job on pull - requests, after the checks pass. There is one runner, and a run a newer push +- **Testing has three levels** (M1.6): every PR runs the checks (unit tests across + the runner's cores, 16 property walks from a moving seed, integration) and the + end-to-end **smoke** tests; a PR labelled `e2e:full`, a push to `main` and a + manual run add all 48 walks and the whole end-to-end suite; a published release + runs that full set before its image is built. The end-to-end tests drive the + real agent, every role on a local model (so they cost nothing per call), in a + repository of the run's own, with a scripted operator; CI runs them after the + checks pass. There is one runner, and a run a newer push supersedes has to be cancelled by hand. Locally only the tests being written (`-k`) or diagnosed (`--lf`); `scripts/e2e.sh` reports what a run spent. Manual live runs are for the new loop's first runs and phase ends. Findings are filed as diff --git a/nisse-dev/tests/test_loop_properties.py b/nisse-dev/tests/test_loop_properties.py index 2413276..1debd61 100644 --- a/nisse-dev/tests/test_loop_properties.py +++ b/nisse-dev/tests/test_loop_properties.py @@ -8,6 +8,7 @@ checks every property after every step. A failure prints the seed and the steps, so it replays exactly. """ +import os import random import re import subprocess @@ -46,8 +47,14 @@ from nisse_dev.tools import Workspace from nisse_dev.transport import InboundMessage, InMemoryTransport from nisse_dev.worker import InlineWorker -SEEDS = range(32) -TWO_PROJECT_SEEDS = range(100, 116) +# How many walks, and from which seed: all 48 by default; CI runs fewer on a +# PR, from a seed that moves with the run number, so PRs still cover new +# ground (NISSE_WALK_SEEDS, NISSE_WALK_FIRST). Two thirds walk one project, +# a third two. +_WALKS = int(os.environ.get("NISSE_WALK_SEEDS") or 48) +_FIRST = int(os.environ.get("NISSE_WALK_FIRST") or 0) +SEEDS = range(_FIRST, _FIRST + _WALKS * 2 // 3) +TWO_PROJECT_SEEDS = range(100 + _FIRST, 100 + _FIRST + _WALKS - _WALKS * 2 // 3) STEPS = 30 diff --git a/pyproject.toml b/pyproject.toml index cb1a434..dc8ee20 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,6 +4,7 @@ members = ["nisse-core", "nisse-dev", "nisse-ops"] [dependency-groups] dev = [ "pytest>=8", + "pytest-xdist>=3", "ruff>=0.6", ] @@ -19,6 +20,7 @@ addopts = "--strict-markers --ignore=tests/integration/seed" markers = [ "integration: end-to-end tests that exercise installed entry points or external services", "e2e: the whole agent against real services, run after each milestone (scripts/e2e.sh)", + "smoke: the end-to-end tests every PR runs; the rest on e2e:full and before a release", ] [tool.ruff] diff --git a/tests/integration/test_end_to_end_loop.py b/tests/integration/test_end_to_end_loop.py index a36174a..f1fd533 100644 --- a/tests/integration/test_end_to_end_loop.py +++ b/tests/integration/test_end_to_end_loop.py @@ -239,6 +239,7 @@ class Operator: return self.transport.outbound[-1].text if self.transport.outbound else "" +@pytest.mark.smoke def test_the_slice_end_to_end( running: tuple[RunningLoop, InMemoryTransport, str], ) -> None: @@ -501,6 +502,7 @@ def test_a_two_milestone_feature( operator.tidy(issues, prs) +@pytest.mark.smoke def test_a_conversation_becomes_filed_work( running: tuple[RunningLoop, InMemoryTransport, str], ) -> None: @@ -636,6 +638,7 @@ def test_a_conversation_becomes_a_new_project( assert any(f.startswith("docs/plans/") for f in files), files +@pytest.mark.smoke def test_a_question_leaves_the_restatement_up( running: tuple[RunningLoop, InMemoryTransport, str], ) -> None: diff --git a/uv.lock b/uv.lock index c9cbbdb..3d109c2 100644 --- a/uv.lock +++ b/uv.lock @@ -12,6 +12,7 @@ members = [ [manifest.dependency-groups] dev = [ { name = "pytest", specifier = ">=8" }, + { name = "pytest-xdist", specifier = ">=3" }, { name = "ruff", specifier = ">=0.6" }, ] @@ -46,6 +47,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", size = 25335, upload-time = "2022-10-25T02:36:20.889Z" }, ] +[[package]] +name = "execnet" +version = "2.1.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/bf/89/780e11f9588d9e7128a3f87788354c7946a9cbb1401ad38a48c4db9a4f07/execnet-2.1.2.tar.gz", hash = "sha256:63d83bfdd9a23e35b9c6a3261412324f964c2ec8dcd8d3c6916ee9373e0befcd", size = 166622, upload-time = "2025-11-12T09:56:37.75Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ab/84/02fc1827e8cdded4aa65baef11296a9bbe595c474f0d6d758af082d849fd/execnet-2.1.2-py3-none-any.whl", hash = "sha256:67fba928dd5a544b783f6056f449e5e3931a5c378b128bc18501f7ea79e296ec", size = 40708, upload-time = "2025-11-12T09:56:36.333Z" }, +] + [[package]] name = "h11" version = "0.16.0" @@ -177,6 +187,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/24/25/1de2678b631f5a49215c6c96fff41ba892b0a34df68d6d80292b1b48aa7f/pytest-9.1.1-py3-none-any.whl", hash = "sha256:37a86b45efb9a47a61a36449063e8e18d0cab3161329fc099eb21783169c4f0c", size = 386536, upload-time = "2026-06-19T10:58:31.347Z" }, ] +[[package]] +name = "pytest-xdist" +version = "3.8.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "execnet" }, + { name = "pytest" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/78/b4/439b179d1ff526791eb921115fca8e44e596a13efeda518b9d845a619450/pytest_xdist-3.8.0.tar.gz", hash = "sha256:7e578125ec9bc6050861aa93f2d59f1d8d085595d6551c2c90b6f4fad8d3a9f1", size = 88069, upload-time = "2025-07-01T13:30:59.346Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ca/31/d4e37e9e550c2b92a9cbc2e4d0b7420a27224968580b5a447f420847c975/pytest_xdist-3.8.0-py3-none-any.whl", hash = "sha256:202ca578cfeb7370784a8c33d6d05bc6e13b4f25b5053c30a152269fd10f0b88", size = 46396, upload-time = "2025-07-01T13:30:56.632Z" }, +] + [[package]] name = "pyyaml" version = "6.0.3" From ff5a7a644fa81f7964c07b8be53c32b81ff7f437 Mon Sep 17 00:00:00 2001 From: Don Garrett Date: Wed, 7 Oct 2026 22:00:16 +0000 Subject: [PATCH 2/3] CI: install sshd from a fast mirror, or not at all archive.ubuntu.com has been crawling (a few KB/s), taking the sshd install in every job from 8 seconds to 3.5-9.5 minutes (another agent measured it; the runner and inference-4090 were healthy). scripts/ci-sshd.sh skips the install when the image has sshd, and otherwise points apt at mirrors.edge.kernel.org first (both source layouts, 22.04 and 24.04). Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01PPQ14L3LikyefCMFBqEbKE --- .gitea/workflows/ci.yml | 20 ++++++++------------ .gitea/workflows/release.yml | 6 +----- scripts/ci-sshd.sh | 20 ++++++++++++++++++++ 3 files changed, 29 insertions(+), 17 deletions(-) create mode 100755 scripts/ci-sshd.sh diff --git a/.gitea/workflows/ci.yml b/.gitea/workflows/ci.yml index 81afecc..7379336 100644 --- a/.gitea/workflows/ci.yml +++ b/.gitea/workflows/ci.yml @@ -39,12 +39,10 @@ jobs: - name: Install sshd # The tests reach "the build VM" over real SSH, through a throwaway - # sshd the test fixture starts (nisse_dev/testing.py). - run: | - SUDO=$(command -v sudo || true) - $SUDO apt-get update -qq - $SUDO apt-get install -y -qq openssh-server openssh-client > /dev/null - $SUDO mkdir -p /run/sshd + # sshd the test fixture starts (nisse_dev/testing.py). Skipped when the image has it; otherwise from + # a kernel.org mirror: archive.ubuntu.com has been slow enough to add + # minutes to every job. + run: scripts/ci-sshd.sh - name: Lint run: uv run ruff check . @@ -100,12 +98,10 @@ jobs: run: uv sync --all-packages - name: Install sshd - # The build VM is a throwaway sshd here (scripts/e2e.sh --local). - run: | - SUDO=$(command -v sudo || true) - $SUDO apt-get update -qq - $SUDO apt-get install -y -qq openssh-server openssh-client > /dev/null - $SUDO mkdir -p /run/sshd + # The build VM is a throwaway sshd here (scripts/e2e.sh --local). Skipped when the image has it; otherwise from + # a kernel.org mirror: archive.ubuntu.com has been slow enough to add + # minutes to every job. + run: scripts/ci-sshd.sh - name: End-to-end tests env: diff --git a/.gitea/workflows/release.yml b/.gitea/workflows/release.yml index 57c7081..cbff8ce 100644 --- a/.gitea/workflows/release.yml +++ b/.gitea/workflows/release.yml @@ -48,11 +48,7 @@ jobs: run: uv sync --all-packages - name: Install sshd - run: | - SUDO=$(command -v sudo || true) - $SUDO apt-get update -qq - $SUDO apt-get install -y -qq openssh-server openssh-client > /dev/null - $SUDO mkdir -p /run/sshd + run: scripts/ci-sshd.sh - name: Lint run: uv run ruff check . && uv run ruff format --check . diff --git a/scripts/ci-sshd.sh b/scripts/ci-sshd.sh new file mode 100755 index 0000000..c05b98b --- /dev/null +++ b/scripts/ci-sshd.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash +# The throwaway sshd the tests need (nisse_dev/testing.py), in a CI job. +# Skipped when the job's image already has it. Otherwise apt fetches from a +# kernel.org mirror rather than archive.ubuntu.com, which has been slow +# enough (a few KB/s) to add minutes to every job. +set -euo pipefail +SUDO=$(command -v sudo || true) +if command -v /usr/sbin/sshd > /dev/null && command -v ssh > /dev/null; then + echo "sshd already installed" +else + MIRROR=${APT_MIRROR:-http://mirrors.edge.kernel.org/ubuntu} + # Both layouts: deb822 (ubuntu.sources, 24.04+) and sources.list (older). + for f in /etc/apt/sources.list.d/ubuntu.sources /etc/apt/sources.list; do + [ -f "$f" ] && $SUDO sed -i -E \ + "s#https?://(archive|security)\.ubuntu\.com/ubuntu/?#$MIRROR/#g" "$f" + done + $SUDO apt-get update -qq + $SUDO apt-get install -y -qq openssh-server openssh-client > /dev/null +fi +$SUDO mkdir -p /run/sshd From 28092194fe31bebf305cf7c62e9736a34d2552f9 Mon Sep 17 00:00:00 2001 From: Don Garrett Date: Wed, 7 Oct 2026 22:49:13 +0000 Subject: [PATCH 3/3] E2E on qwen3.8-27b without thinking The e2e roles move to qwen3.8-27b-nothink, a LiteLLM alias for the same model with chat_template_kwargs enable_thinking false. With thinking, a growing share of answers ran to 2-8k tokens (two hit the 8192 cap), which took one test from 3.5 to 20 minutes. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01PPQ14L3LikyefCMFBqEbKE --- tests/integration/e2e_support.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/integration/e2e_support.py b/tests/integration/e2e_support.py index 3a51403..e63950f 100644 --- a/tests/integration/e2e_support.py +++ b/tests/integration/e2e_support.py @@ -15,8 +15,10 @@ from nisse_dev.llm import RoleMap # The end-to-end tests run every role on a local model, which costs nothing # per call; NISSE_E2E_MODEL overrides it (e.g. claude-haiku-4-5 for a quick, -# paid run). -E2E_MODEL = "qwen3.8-27b" +# paid run). The no-thinking alias (LiteLLM: chat_template_kwargs +# enable_thinking false): with thinking, some answers ran to 6-8k tokens, +# minutes each, and stretched a test from 3.5 to 20 minutes. +E2E_MODEL = "qwen3.8-27b-nothink" def e2e_roles() -> RoleMap: