Merge pull request 'Test levels: smoke e2e on PRs, full set on e2e:full and before a release' (#117) from test-tiers into main

.gitea/workflows/ci.yml

@@ -1,4 +1,10 @@
 name: CI
+# Three levels of testing:
+#   - every PR: lint, unit tests (16 property walks), integration tests, and
+#     the smoke end-to-end tests (`smoke` marker);
+#   - a PR labelled e2e:full, a push to main and a manual run: all 48 walks
+#     and the whole end-to-end suite;
+#   - a published release: the full set again before its image (release.yml).
 on:
   # Only main on push. A PR branch is already covered by the pull_request
   # event, and running both fired two identical suites concurrently — which
@@ -7,11 +13,18 @@ on:
   push:
     branches: [main]
   pull_request:
+    # labeled: adding e2e:full runs the full set; any other label, nothing.
+    types: [opened, synchronize, reopened, labeled]
   # Manual re-run, e.g. after rotating a credential or raising a budget cap.
   workflow_dispatch:
 
+env:
+  # Full on e2e:full, and outside PRs.
+  FULL: ${{ github.event_name != 'pull_request' || contains(github.event.pull_request.labels.*.name, 'e2e:full') }}
+
 jobs:
   checks:
+    if: github.event.action != 'labeled' || github.event.label.name == 'e2e:full'
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
@@ -26,12 +39,10 @@ jobs:
 
       - name: Install sshd
         # The tests reach "the build VM" over real SSH, through a throwaway
-        # sshd the test fixture starts (nisse_dev/testing.py).
+        # sshd the test fixture starts (nisse_dev/testing.py). Skipped when the image has it; otherwise from
-        run: |
+        # a kernel.org mirror: archive.ubuntu.com has been slow enough to add
-          SUDO=$(command -v sudo || true)
+        # minutes to every job.
-          $SUDO apt-get update -qq
+        run: scripts/ci-sshd.sh
-          $SUDO apt-get install -y -qq openssh-server openssh-client > /dev/null
-          $SUDO mkdir -p /run/sshd
 
       - name: Lint
         run: uv run ruff check .
@@ -40,7 +51,12 @@ jobs:
         run: uv run ruff format --check .
 
       - name: Unit tests
-        run: uv run pytest -m "not integration and not e2e"
+        # Across the runner's cores (pytest-xdist). On a PR, 16 property
+        # walks from a seed that moves with the run number; all 48 otherwise.
+        env:
+          NISSE_WALK_SEEDS: ${{ env.FULL == 'true' && '48' || '16' }}
+          NISSE_WALK_FIRST: ${{ env.FULL == 'true' && '0' || github.run_number }}
+        run: uv run pytest -n auto -m "not integration and not e2e"
 
       - name: Integration tests
         # Everything but the secrets is in tests/integration/nisse-test.yaml,
@@ -58,12 +74,15 @@ jobs:
   e2e:
     # The end-to-end suite (scripts/e2e.sh): the whole agent against real
     # services, every role on a local model, so it spends nothing but keeps
-    # the RTX 4090 busy for ~25 minutes. On PRs only, and only after the
+    # the RTX 4090 busy: ~10 minutes for the smoke tests, ~25+ for all. On
-    # checks pass: a PR that's red already doesn't spend the GPU. There is
+    # PRs and manual runs, and only after the checks pass: a PR that's red
-    # one runner and Gitea doesn't honour cancel-in-progress, so a run a
+    # already doesn't spend the GPU. There is one runner and Gitea doesn't
-    # newer push supersedes holds the runner until it's cancelled by hand.
+    # honour cancel-in-progress, so a run a newer push supersedes holds the
-    # Locally, run only what you're writing (-k) or diagnosing (--lf).
+    # runner until it's cancelled by hand. Locally, run only what you're
-    if: github.event_name != 'push'
+    # writing (-k) or diagnosing (--lf).
+    if: >-
+      github.event_name != 'push'
+      && (github.event.action != 'labeled' || github.event.label.name == 'e2e:full')
     needs: checks
     runs-on: ubuntu-latest
     timeout-minutes: 90
@@ -79,12 +98,10 @@ jobs:
         run: uv sync --all-packages
 
       - name: Install sshd
-        # The build VM is a throwaway sshd here (scripts/e2e.sh --local).
+        # The build VM is a throwaway sshd here (scripts/e2e.sh --local). Skipped when the image has it; otherwise from
-        run: |
+        # a kernel.org mirror: archive.ubuntu.com has been slow enough to add
-          SUDO=$(command -v sudo || true)
+        # minutes to every job.
-          $SUDO apt-get update -qq
+        run: scripts/ci-sshd.sh
-          $SUDO apt-get install -y -qq openssh-server openssh-client > /dev/null
-          $SUDO mkdir -p /run/sshd
 
       - name: End-to-end tests
         env:
@@ -93,4 +110,9 @@ jobs:
           NISSE_GITEA_TOKEN: ${{ secrets.NISSE_GITEA_TOKEN }}
           NISSE_OPERATOR_TOKEN: ${{ secrets.NISSE_OPERATOR_TOKEN }}
           NISSE_E2E_RUNS: /tmp/nisse-e2e-runs
-        run: scripts/e2e.sh --local -v
+        run: |
+          if [ "$FULL" = true ]; then
+            scripts/e2e.sh --local -v
+          else
+            scripts/e2e.sh --local -v -m "e2e and smoke"
+          fi

.gitea/workflows/release.yml

@@ -3,7 +3,8 @@ name: Release
 # registry (Development Plan M1.5, X4): gitea.dongar.net/dev/nisse-dev
 # tagged with the version (the release tag without its v) and latest. Cut a
 # release in Gitea (Releases → New release, tag vX.Y.Z); a tag pushed for any
-# other reason builds nothing.
+# other reason builds nothing. The full test set runs first: no image without
+# it passing.
 #
 # The registry needs a personal access token with write:package
 # (REGISTRY_TOKEN, an organization secret of `dev`, belonging to the
@@ -23,7 +24,46 @@ env:
   IMAGE: gitea.dongar.net/dev/nisse-dev
 
 jobs:
+  tests:
+    # Everything, before an image exists: lint, unit tests with all 48
+    # property walks, integration tests and the whole end-to-end suite.
+    runs-on: ubuntu-latest
+    timeout-minutes: 150
+    env:
+      NISSE_LLM_TOKEN: ${{ secrets.NISSE_LLM_TOKEN }}
+      NISSE_MATRIX_TOKEN: ${{ secrets.NISSE_MATRIX_TOKEN }}
+      NISSE_GITEA_TOKEN: ${{ secrets.NISSE_GITEA_TOKEN }}
+      NISSE_OPERATOR_TOKEN: ${{ secrets.NISSE_OPERATOR_TOKEN }}
+      NISSE_E2E_RUNS: /tmp/nisse-e2e-runs
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Install uv
+        run: |
+          curl -LsSf https://astral.sh/uv/install.sh | sh
+          echo "$HOME/.local/bin" >> "$GITHUB_PATH"
+
+      - name: Sync workspace
+        run: uv sync --all-packages
+
+      - name: Install sshd
+        run: scripts/ci-sshd.sh
+
+      - name: Lint
+        run: uv run ruff check . && uv run ruff format --check .
+
+      - name: Unit tests
+        run: uv run pytest -n auto -m "not integration and not e2e"
+
+      - name: Integration tests
+        run: uv run pytest -m "integration and not e2e" -rs
+
+      - name: End-to-end tests
+        run: scripts/e2e.sh --local -v
+
   build-and-push:
+    needs: tests
     runs-on: ubuntu-latest
     permissions:
       contents: read

docs/Development-Plan.md

@@ -73,10 +73,14 @@ Only `nisse-dev` has real code; `nisse-core` and `nisse-ops` are placeholders.
   `CLAUDE.md`. A milestone marked **needs discussion** starts with a design
   discussion first.
 - **Every change comes with tests,** through `scripts/check.sh`.
-- **The end-to-end test runs on every PR** (M1.6): the real agent, every role on a
+- **Testing has three levels** (M1.6): every PR runs the checks (unit tests across
-  local model (so it costs nothing per call), a repository of the run's own and a
+  the runner's cores, 16 property walks from a moving seed, integration) and the
-  real build host, driven by a scripted operator. CI runs it as its own job on pull
+  end-to-end **smoke** tests; a PR labelled `e2e:full`, a push to `main` and a
-  requests, after the checks pass. There is one runner, and a run a newer push
+  manual run add all 48 walks and the whole end-to-end suite; a published release
+  runs that full set before its image is built. The end-to-end tests drive the
+  real agent, every role on a local model (so they cost nothing per call), in a
+  repository of the run's own, with a scripted operator; CI runs them after the
+  checks pass. There is one runner, and a run a newer push
   supersedes has to be cancelled by hand. Locally only the tests being written
   (`-k`) or diagnosed (`--lf`); `scripts/e2e.sh` reports what a run spent. Manual
   live runs are for the new loop's first runs and phase ends. Findings are filed as

nisse-dev/tests/test_loop_properties.py

@@ -8,6 +8,7 @@ checks every property after every step. A failure prints
 the seed and the steps, so it replays exactly.
 """
 
+import os
 import random
 import re
 import subprocess
@@ -46,8 +47,14 @@ from nisse_dev.tools import Workspace
 from nisse_dev.transport import InboundMessage, InMemoryTransport
 from nisse_dev.worker import InlineWorker
 
-SEEDS = range(32)
+# How many walks, and from which seed: all 48 by default; CI runs fewer on a
-TWO_PROJECT_SEEDS = range(100, 116)
+# PR, from a seed that moves with the run number, so PRs still cover new
+# ground (NISSE_WALK_SEEDS, NISSE_WALK_FIRST). Two thirds walk one project,
+# a third two.
+_WALKS = int(os.environ.get("NISSE_WALK_SEEDS") or 48)
+_FIRST = int(os.environ.get("NISSE_WALK_FIRST") or 0)
+SEEDS = range(_FIRST, _FIRST + _WALKS * 2 // 3)
+TWO_PROJECT_SEEDS = range(100 + _FIRST, 100 + _FIRST + _WALKS - _WALKS * 2 // 3)
 STEPS = 30
 
 

pyproject.toml

@@ -4,6 +4,7 @@ members = ["nisse-core", "nisse-dev", "nisse-ops"]
 [dependency-groups]
 dev = [
     "pytest>=8",
+    "pytest-xdist>=3",
     "ruff>=0.6",
 ]
 
@@ -19,6 +20,7 @@ addopts = "--strict-markers --ignore=tests/integration/seed"
 markers = [
     "integration: end-to-end tests that exercise installed entry points or external services",
     "e2e: the whole agent against real services, run after each milestone (scripts/e2e.sh)",
+    "smoke: the end-to-end tests every PR runs; the rest on e2e:full and before a release",
 ]
 
 [tool.ruff]

scripts/ci-sshd.sh

@@ -0,0 +1,20 @@
+#!/usr/bin/env bash
+# The throwaway sshd the tests need (nisse_dev/testing.py), in a CI job.
+# Skipped when the job's image already has it. Otherwise apt fetches from a
+# kernel.org mirror rather than archive.ubuntu.com, which has been slow
+# enough (a few KB/s) to add minutes to every job.
+set -euo pipefail
+SUDO=$(command -v sudo || true)
+if command -v /usr/sbin/sshd > /dev/null && command -v ssh > /dev/null; then
+    echo "sshd already installed"
+else
+    MIRROR=${APT_MIRROR:-http://mirrors.edge.kernel.org/ubuntu}
+    # Both layouts: deb822 (ubuntu.sources, 24.04+) and sources.list (older).
+    for f in /etc/apt/sources.list.d/ubuntu.sources /etc/apt/sources.list; do
+        [ -f "$f" ] && $SUDO sed -i -E \
+            "s#https?://(archive|security)\.ubuntu\.com/ubuntu/?#$MIRROR/#g" "$f"
+    done
+    $SUDO apt-get update -qq
+    $SUDO apt-get install -y -qq openssh-server openssh-client > /dev/null
+fi
+$SUDO mkdir -p /run/sshd

tests/integration/e2e_support.py

@@ -15,8 +15,10 @@ from nisse_dev.llm import RoleMap
 
 # The end-to-end tests run every role on a local model, which costs nothing
 # per call; NISSE_E2E_MODEL overrides it (e.g. claude-haiku-4-5 for a quick,
-# paid run).
+# paid run). The no-thinking alias (LiteLLM: chat_template_kwargs
-E2E_MODEL = "qwen3.8-27b"
+# enable_thinking false): with thinking, some answers ran to 6-8k tokens,
+# minutes each, and stretched a test from 3.5 to 20 minutes.
+E2E_MODEL = "qwen3.8-27b-nothink"
 
 
 def e2e_roles() -> RoleMap:

tests/integration/test_end_to_end_loop.py

@@ -239,6 +239,7 @@ class Operator:
         return self.transport.outbound[-1].text if self.transport.outbound else ""
 
 
+@pytest.mark.smoke
 def test_the_slice_end_to_end(
     running: tuple[RunningLoop, InMemoryTransport, str],
 ) -> None:
@@ -501,6 +502,7 @@ def test_a_two_milestone_feature(
         operator.tidy(issues, prs)
 
 
+@pytest.mark.smoke
 def test_a_conversation_becomes_filed_work(
     running: tuple[RunningLoop, InMemoryTransport, str],
 ) -> None:
@@ -636,6 +638,7 @@ def test_a_conversation_becomes_a_new_project(
         assert any(f.startswith("docs/plans/") for f in files), files
 
 
+@pytest.mark.smoke
 def test_a_question_leaves_the_restatement_up(
     running: tuple[RunningLoop, InMemoryTransport, str],
 ) -> None:

uv.lock

@@ -12,6 +12,7 @@ members = [
 [manifest.dependency-groups]
 dev = [
     { name = "pytest", specifier = ">=8" },
+    { name = "pytest-xdist", specifier = ">=3" },
     { name = "ruff", specifier = ">=0.6" },
 ]
 
@@ -46,6 +47,15 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", size = 25335, upload-time = "2022-10-25T02:36:20.889Z" },
 ]
 
+[[package]]
+name = "execnet"
+version = "2.1.2"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/bf/89/780e11f9588d9e7128a3f87788354c7946a9cbb1401ad38a48c4db9a4f07/execnet-2.1.2.tar.gz", hash = "sha256:63d83bfdd9a23e35b9c6a3261412324f964c2ec8dcd8d3c6916ee9373e0befcd", size = 166622, upload-time = "2025-11-12T09:56:37.75Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/ab/84/02fc1827e8cdded4aa65baef11296a9bbe595c474f0d6d758af082d849fd/execnet-2.1.2-py3-none-any.whl", hash = "sha256:67fba928dd5a544b783f6056f449e5e3931a5c378b128bc18501f7ea79e296ec", size = 40708, upload-time = "2025-11-12T09:56:36.333Z" },
+]
+
 [[package]]
 name = "h11"
 version = "0.16.0"
@@ -177,6 +187,19 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/24/25/1de2678b631f5a49215c6c96fff41ba892b0a34df68d6d80292b1b48aa7f/pytest-9.1.1-py3-none-any.whl", hash = "sha256:37a86b45efb9a47a61a36449063e8e18d0cab3161329fc099eb21783169c4f0c", size = 386536, upload-time = "2026-06-19T10:58:31.347Z" },
 ]
 
+[[package]]
+name = "pytest-xdist"
+version = "3.8.0"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "execnet" },
+    { name = "pytest" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/78/b4/439b179d1ff526791eb921115fca8e44e596a13efeda518b9d845a619450/pytest_xdist-3.8.0.tar.gz", hash = "sha256:7e578125ec9bc6050861aa93f2d59f1d8d085595d6551c2c90b6f4fad8d3a9f1", size = 88069, upload-time = "2025-07-01T13:30:59.346Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/ca/31/d4e37e9e550c2b92a9cbc2e4d0b7420a27224968580b5a447f420847c975/pytest_xdist-3.8.0-py3-none-any.whl", hash = "sha256:202ca578cfeb7370784a8c33d6d05bc6e13b4f25b5053c30a152269fd10f0b88", size = 46396, upload-time = "2025-07-01T13:30:56.632Z" },
+]
+
 [[package]]
 name = "pyyaml"
 version = "6.0.3"