Test levels: smoke e2e on PRs, the full set on e2e:full and before a release
- Every PR: lint, unit tests across the runner's cores (pytest-xdist), 16 property walks from a seed that moves with the run number, integration tests, and the end-to-end smoke tests (the slice, a conversation filed as work, a question; `smoke` marker). - A PR labelled e2e:full, a push to main, or a manual run: all 48 walks and the whole end-to-end suite. Adding e2e:full to a PR runs it; any other label runs nothing. - A published release runs the full set before its image is built. - The walk's seeds come from NISSE_WALK_SEEDS / NISSE_WALK_FIRST (48 from 0 by default). Measured here: the unit tier on 2 workers took 3m49s against 6-8 minutes serially. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01PPQ14L3LikyefCMFBqEbKE
This commit is contained in:
dongarandClaude Opus 5.5 committed
1 parent
07a2eb9d10
commit
e35aea5045
7 files changed
+124
-15
No files matched your search
+34
-8
@@ -1,4 +1,10 @@
|
||||
name: CI
|
||||
# Three levels of testing:
|
||||
# - every PR: lint, unit tests (16 property walks), integration tests, and
|
||||
# the smoke end-to-end tests (`smoke` marker);
|
||||
# - a PR labelled e2e:full, a push to main and a manual run: all 48 walks
|
||||
# and the whole end-to-end suite;
|
||||
# - a published release: the full set again before its image (release.yml).
|
||||
on:
|
||||
# Only main on push. A PR branch is already covered by the pull_request
|
||||
# event, and running both fired two identical suites concurrently — which
|
||||
@@ -7,11 +13,18 @@ on:
|
||||
push:
|
||||
branches: [main]
|
||||
pull_request:
|
||||
# labeled: adding e2e:full runs the full set; any other label, nothing.
|
||||
types: [opened, synchronize, reopened, labeled]
|
||||
# Manual re-run, e.g. after rotating a credential or raising a budget cap.
|
||||
workflow_dispatch:
|
||||
|
||||
env:
|
||||
# Full on e2e:full, and outside PRs.
|
||||
FULL: ${{ github.event_name != 'pull_request' || contains(github.event.pull_request.labels.*.name, 'e2e:full') }}
|
||||
|
||||
jobs:
|
||||
checks:
|
||||
if: github.event.action != 'labeled' || github.event.label.name == 'e2e:full'
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
@@ -40,7 +53,12 @@ jobs:
|
||||
run: uv run ruff format --check .
|
||||
|
||||
- name: Unit tests
|
||||
run: uv run pytest -m "not integration and not e2e"
|
||||
# Across the runner's cores (pytest-xdist). On a PR, 16 property
|
||||
# walks from a seed that moves with the run number; all 48 otherwise.
|
||||
env:
|
||||
NISSE_WALK_SEEDS: ${{ env.FULL == 'true' && '48' || '16' }}
|
||||
NISSE_WALK_FIRST: ${{ env.FULL == 'true' && '0' || github.run_number }}
|
||||
run: uv run pytest -n auto -m "not integration and not e2e"
|
||||
|
||||
- name: Integration tests
|
||||
# Everything but the secrets is in tests/integration/nisse-test.yaml,
|
||||
@@ -58,12 +76,15 @@ jobs:
|
||||
e2e:
|
||||
# The end-to-end suite (scripts/e2e.sh): the whole agent against real
|
||||
# services, every role on a local model, so it spends nothing but keeps
|
||||
# the RTX 4090 busy for ~25 minutes. On PRs only, and only after the
|
||||
# checks pass: a PR that's red already doesn't spend the GPU. There is
|
||||
# one runner and Gitea doesn't honour cancel-in-progress, so a run a
|
||||
# newer push supersedes holds the runner until it's cancelled by hand.
|
||||
# Locally, run only what you're writing (-k) or diagnosing (--lf).
|
||||
if: github.event_name != 'push'
|
||||
# the RTX 4090 busy: ~10 minutes for the smoke tests, ~25+ for all. On
|
||||
# PRs and manual runs, and only after the checks pass: a PR that's red
|
||||
# already doesn't spend the GPU. There is one runner and Gitea doesn't
|
||||
# honour cancel-in-progress, so a run a newer push supersedes holds the
|
||||
# runner until it's cancelled by hand. Locally, run only what you're
|
||||
# writing (-k) or diagnosing (--lf).
|
||||
if: >-
|
||||
github.event_name != 'push'
|
||||
&& (github.event.action != 'labeled' || github.event.label.name == 'e2e:full')
|
||||
needs: checks
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 90
|
||||
@@ -93,4 +114,9 @@ jobs:
|
||||
NISSE_GITEA_TOKEN: ${{ secrets.NISSE_GITEA_TOKEN }}
|
||||
NISSE_OPERATOR_TOKEN: ${{ secrets.NISSE_OPERATOR_TOKEN }}
|
||||
NISSE_E2E_RUNS: /tmp/nisse-e2e-runs
|
||||
run: scripts/e2e.sh --local -v
|
||||
run: |
|
||||
if [ "$FULL" = true ]; then
|
||||
scripts/e2e.sh --local -v
|
||||
else
|
||||
scripts/e2e.sh --local -v -m "e2e and smoke"
|
||||
fi
|
||||
@@ -3,7 +3,8 @@ name: Release
|
||||
# registry (Development Plan M1.5, X4): gitea.dongar.net/dev/nisse-dev
|
||||
# tagged with the version (the release tag without its v) and latest. Cut a
|
||||
# release in Gitea (Releases → New release, tag vX.Y.Z); a tag pushed for any
|
||||
# other reason builds nothing.
|
||||
# other reason builds nothing. The full test set runs first: no image without
|
||||
# it passing.
|
||||
#
|
||||
# The registry needs a personal access token with write:package
|
||||
# (REGISTRY_TOKEN, an organization secret of `dev`, belonging to the
|
||||
@@ -23,7 +24,50 @@ env:
|
||||
IMAGE: gitea.dongar.net/dev/nisse-dev
|
||||
|
||||
jobs:
|
||||
tests:
|
||||
# Everything, before an image exists: lint, unit tests with all 48
|
||||
# property walks, integration tests and the whole end-to-end suite.
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 150
|
||||
env:
|
||||
NISSE_LLM_TOKEN: ${{ secrets.NISSE_LLM_TOKEN }}
|
||||
NISSE_MATRIX_TOKEN: ${{ secrets.NISSE_MATRIX_TOKEN }}
|
||||
NISSE_GITEA_TOKEN: ${{ secrets.NISSE_GITEA_TOKEN }}
|
||||
NISSE_OPERATOR_TOKEN: ${{ secrets.NISSE_OPERATOR_TOKEN }}
|
||||
NISSE_E2E_RUNS: /tmp/nisse-e2e-runs
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Install uv
|
||||
run: |
|
||||
curl -LsSf https://astral.sh/uv/install.sh | sh
|
||||
echo "$HOME/.local/bin" >> "$GITHUB_PATH"
|
||||
|
||||
- name: Sync workspace
|
||||
run: uv sync --all-packages
|
||||
|
||||
- name: Install sshd
|
||||
run: |
|
||||
SUDO=$(command -v sudo || true)
|
||||
$SUDO apt-get update -qq
|
||||
$SUDO apt-get install -y -qq openssh-server openssh-client > /dev/null
|
||||
$SUDO mkdir -p /run/sshd
|
||||
|
||||
- name: Lint
|
||||
run: uv run ruff check . && uv run ruff format --check .
|
||||
|
||||
- name: Unit tests
|
||||
run: uv run pytest -n auto -m "not integration and not e2e"
|
||||
|
||||
- name: Integration tests
|
||||
run: uv run pytest -m "integration and not e2e" -rs
|
||||
|
||||
- name: End-to-end tests
|
||||
run: scripts/e2e.sh --local -v
|
||||
|
||||
build-and-push:
|
||||
needs: tests
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
Reference in new issue
Block a user