name: CI # Three levels of testing: # - every PR: lint, unit tests (16 property walks), integration tests, and # the smoke end-to-end tests (`smoke` marker); # - a PR labelled e2e:full, a push to main and a manual run: all 48 walks # and the whole end-to-end suite; # - a published release: the full set again before its image (release.yml). on: # Only main on push. A PR branch is already covered by the pull_request # event, and running both fired two identical suites concurrently — which # doubled the LLM tokens the live tests spend and tripped the proxy's # per-minute limit. push: branches: [main] pull_request: # labeled: adding e2e:full runs the full set; any other label, nothing. types: [opened, synchronize, reopened, labeled] # Manual re-run, e.g. after rotating a credential or raising a budget cap. workflow_dispatch: # A newer run for the same PR or branch cancels the one it supersedes: there # is one runner, so a stale run would hold it (and the GPU) to the end. Adding # a label other than e2e:full starts a run whose jobs all skip; it gets a # group of its own, so it can't cancel the real one. # # Gitea triggers on `labeled` but hands the workflow its own name for the # action, `label_updated` (and `synchronized` for a push), so each check # below accepts both. Nor does it say which label was added # (`github.event.label` is empty), so a label event counts when the PR now # has e2e:full; adding another label to an e2e:full PR restarts its run. concurrency: group: >- ${{ github.workflow }}-${{ github.head_ref || github.ref }}${{ ((github.event.action == 'labeled' || github.event.action == 'label_updated') && !contains(github.event.pull_request.labels.*.name, 'e2e:full')) && format('-label-{0}', github.run_id) || '' }} cancel-in-progress: true env: # Full on e2e:full, and outside PRs. FULL: ${{ github.event_name != 'pull_request' || contains(github.event.pull_request.labels.*.name, 'e2e:full') }} jobs: checks: if: >- (github.event.action != 'labeled' && github.event.action != 'label_updated') || contains(github.event.pull_request.labels.*.name, 'e2e:full') runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - name: Install uv run: | curl -LsSf https://astral.sh/uv/install.sh | sh echo "$HOME/.local/bin" >> "$GITHUB_PATH" - name: Sync workspace run: uv sync --all-packages - name: Install sshd # The tests reach "the build VM" over real SSH, through a throwaway # sshd the test fixture starts (nisse_dev/testing.py). Skipped when the image has it; otherwise from # a kernel.org mirror: archive.ubuntu.com has been slow enough to add # minutes to every job. run: scripts/ci-sshd.sh - name: Lint run: uv run ruff check . - name: Format check run: uv run ruff format --check . - name: Unit tests # Across the runner's cores (pytest-xdist). On a PR, 16 property # walks from a seed that moves with the run number; all 48 otherwise. env: NISSE_WALK_SEEDS: ${{ env.FULL == 'true' && '48' || '16' }} NISSE_WALK_FIRST: ${{ env.FULL == 'true' && '0' || github.run_number }} run: uv run pytest -n auto -m "not integration and not e2e" - name: Integration tests # Everything but the secrets is in tests/integration/nisse-test.yaml, # which refers to these by name. A missing secret fails the live # tests that need it; it is never quietly turned into a skip. env: NISSE_LLM_TOKEN: ${{ secrets.NISSE_LLM_TOKEN }} NISSE_MATRIX_TOKEN: ${{ secrets.NISSE_MATRIX_TOKEN }} NISSE_GITEA_TOKEN: ${{ secrets.NISSE_GITEA_TOKEN }} # The test operator, who makes each run's repository (and its deploy # key) and deletes it afterwards. NISSE_OPERATOR_TOKEN: ${{ secrets.NISSE_OPERATOR_TOKEN }} run: uv run pytest -m "integration and not e2e" -rs e2e: # The end-to-end suite (scripts/e2e.sh): the whole agent against real # services, every role on a local model, so it spends nothing but keeps # the RTX 4090 busy: ~10 minutes for the smoke tests, ~25+ for all. On # PRs and manual runs, and only after the checks pass: a PR that's red # already doesn't spend the GPU. A newer push cancels a run it supersedes # (concurrency, above). Locally, run only what you're writing (-k) or # diagnosing (--lf). if: >- github.event_name != 'push' && ((github.event.action != 'labeled' && github.event.action != 'label_updated') || contains(github.event.pull_request.labels.*.name, 'e2e:full')) needs: checks runs-on: ubuntu-latest timeout-minutes: 90 steps: - uses: actions/checkout@v4 - name: Install uv run: | curl -LsSf https://astral.sh/uv/install.sh | sh echo "$HOME/.local/bin" >> "$GITHUB_PATH" - name: Sync workspace run: uv sync --all-packages - name: Install sshd # The build VM is a throwaway sshd here (scripts/e2e.sh --local). Skipped when the image has it; otherwise from # a kernel.org mirror: archive.ubuntu.com has been slow enough to add # minutes to every job. run: scripts/ci-sshd.sh - name: End-to-end tests env: NISSE_LLM_TOKEN: ${{ secrets.NISSE_LLM_TOKEN }} NISSE_MATRIX_TOKEN: ${{ secrets.NISSE_MATRIX_TOKEN }} NISSE_GITEA_TOKEN: ${{ secrets.NISSE_GITEA_TOKEN }} NISSE_OPERATOR_TOKEN: ${{ secrets.NISSE_OPERATOR_TOKEN }} NISSE_E2E_RUNS: /tmp/nisse-e2e-runs run: | if [ "$FULL" = true ]; then scripts/e2e.sh --local -v else scripts/e2e.sh --local -v -m "e2e and smoke" fi