dongarandClaude Opus 5.5
ff5a7a644f
archive.ubuntu.com has been crawling (a few KB/s), taking the sshd install in every job from 8 seconds to 3.5-9.5 minutes (another agent measured it; the runner and inference-4090 were healthy). scripts/ci-sshd.sh skips the install when the image has sshd, and otherwise points apt at mirrors.edge.kernel.org first (both source layouts, 22.04 and 24.04). Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01PPQ14L3LikyefCMFBqEbKE
119 lines
4.7 KiB
YAML
119 lines
4.7 KiB
YAML
name: CI
|
|
# Three levels of testing:
|
|
# - every PR: lint, unit tests (16 property walks), integration tests, and
|
|
# the smoke end-to-end tests (`smoke` marker);
|
|
# - a PR labelled e2e:full, a push to main and a manual run: all 48 walks
|
|
# and the whole end-to-end suite;
|
|
# - a published release: the full set again before its image (release.yml).
|
|
on:
|
|
# Only main on push. A PR branch is already covered by the pull_request
|
|
# event, and running both fired two identical suites concurrently — which
|
|
# doubled the LLM tokens the live tests spend and tripped the proxy's
|
|
# per-minute limit.
|
|
push:
|
|
branches: [main]
|
|
pull_request:
|
|
# labeled: adding e2e:full runs the full set; any other label, nothing.
|
|
types: [opened, synchronize, reopened, labeled]
|
|
# Manual re-run, e.g. after rotating a credential or raising a budget cap.
|
|
workflow_dispatch:
|
|
|
|
env:
|
|
# Full on e2e:full, and outside PRs.
|
|
FULL: ${{ github.event_name != 'pull_request' || contains(github.event.pull_request.labels.*.name, 'e2e:full') }}
|
|
|
|
jobs:
|
|
checks:
|
|
if: github.event.action != 'labeled' || github.event.label.name == 'e2e:full'
|
|
runs-on: ubuntu-latest
|
|
steps:
|
|
- uses: actions/checkout@v4
|
|
|
|
- name: Install uv
|
|
run: |
|
|
curl -LsSf https://astral.sh/uv/install.sh | sh
|
|
echo "$HOME/.local/bin" >> "$GITHUB_PATH"
|
|
|
|
- name: Sync workspace
|
|
run: uv sync --all-packages
|
|
|
|
- name: Install sshd
|
|
# The tests reach "the build VM" over real SSH, through a throwaway
|
|
# sshd the test fixture starts (nisse_dev/testing.py). Skipped when the image has it; otherwise from
|
|
# a kernel.org mirror: archive.ubuntu.com has been slow enough to add
|
|
# minutes to every job.
|
|
run: scripts/ci-sshd.sh
|
|
|
|
- name: Lint
|
|
run: uv run ruff check .
|
|
|
|
- name: Format check
|
|
run: uv run ruff format --check .
|
|
|
|
- name: Unit tests
|
|
# Across the runner's cores (pytest-xdist). On a PR, 16 property
|
|
# walks from a seed that moves with the run number; all 48 otherwise.
|
|
env:
|
|
NISSE_WALK_SEEDS: ${{ env.FULL == 'true' && '48' || '16' }}
|
|
NISSE_WALK_FIRST: ${{ env.FULL == 'true' && '0' || github.run_number }}
|
|
run: uv run pytest -n auto -m "not integration and not e2e"
|
|
|
|
- name: Integration tests
|
|
# Everything but the secrets is in tests/integration/nisse-test.yaml,
|
|
# which refers to these by name. A missing secret fails the live
|
|
# tests that need it; it is never quietly turned into a skip.
|
|
env:
|
|
NISSE_LLM_TOKEN: ${{ secrets.NISSE_LLM_TOKEN }}
|
|
NISSE_MATRIX_TOKEN: ${{ secrets.NISSE_MATRIX_TOKEN }}
|
|
NISSE_GITEA_TOKEN: ${{ secrets.NISSE_GITEA_TOKEN }}
|
|
# The test operator, who makes each run's repository (and its deploy
|
|
# key) and deletes it afterwards.
|
|
NISSE_OPERATOR_TOKEN: ${{ secrets.NISSE_OPERATOR_TOKEN }}
|
|
run: uv run pytest -m "integration and not e2e" -rs
|
|
|
|
e2e:
|
|
# The end-to-end suite (scripts/e2e.sh): the whole agent against real
|
|
# services, every role on a local model, so it spends nothing but keeps
|
|
# the RTX 4090 busy: ~10 minutes for the smoke tests, ~25+ for all. On
|
|
# PRs and manual runs, and only after the checks pass: a PR that's red
|
|
# already doesn't spend the GPU. There is one runner and Gitea doesn't
|
|
# honour cancel-in-progress, so a run a newer push supersedes holds the
|
|
# runner until it's cancelled by hand. Locally, run only what you're
|
|
# writing (-k) or diagnosing (--lf).
|
|
if: >-
|
|
github.event_name != 'push'
|
|
&& (github.event.action != 'labeled' || github.event.label.name == 'e2e:full')
|
|
needs: checks
|
|
runs-on: ubuntu-latest
|
|
timeout-minutes: 90
|
|
steps:
|
|
- uses: actions/checkout@v4
|
|
|
|
- name: Install uv
|
|
run: |
|
|
curl -LsSf https://astral.sh/uv/install.sh | sh
|
|
echo "$HOME/.local/bin" >> "$GITHUB_PATH"
|
|
|
|
- name: Sync workspace
|
|
run: uv sync --all-packages
|
|
|
|
- name: Install sshd
|
|
# The build VM is a throwaway sshd here (scripts/e2e.sh --local). Skipped when the image has it; otherwise from
|
|
# a kernel.org mirror: archive.ubuntu.com has been slow enough to add
|
|
# minutes to every job.
|
|
run: scripts/ci-sshd.sh
|
|
|
|
- name: End-to-end tests
|
|
env:
|
|
NISSE_LLM_TOKEN: ${{ secrets.NISSE_LLM_TOKEN }}
|
|
NISSE_MATRIX_TOKEN: ${{ secrets.NISSE_MATRIX_TOKEN }}
|
|
NISSE_GITEA_TOKEN: ${{ secrets.NISSE_GITEA_TOKEN }}
|
|
NISSE_OPERATOR_TOKEN: ${{ secrets.NISSE_OPERATOR_TOKEN }}
|
|
NISSE_E2E_RUNS: /tmp/nisse-e2e-runs
|
|
run: |
|
|
if [ "$FULL" = true ]; then
|
|
scripts/e2e.sh --local -v
|
|
else
|
|
scripts/e2e.sh --local -v -m "e2e and smoke"
|
|
fi
|