dongarandClaude Opus 5.5
6db80dfb4b
- Every e2e role runs on qwen3.8-27b on the RTX 4090 (e2e_roles(); NISSE_E2E_MODEL overrides, e.g. claude-haiku-4-5). The first full run passed 7 of 8 in 26 minutes for $0.0000; the eighth was a real finding, fixed with the new-project PR. - scripts/e2e.sh reports a run's spend on the test key and the day's total against its budget: above $0 on the local model means a paid fallback. - CI runs the suite as its own job on pull requests (not pushes to main), on a throwaway sshd (--local), cancelling a run a newer push supersedes, so the checks job still reports in minutes. - The plan says so: e2e on every PR; locally only -k / --lf. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01PPQ14L3LikyefCMFBqEbKE
98 lines
3.5 KiB
YAML
98 lines
3.5 KiB
YAML
name: CI
|
|
on:
|
|
# Only main on push. A PR branch is already covered by the pull_request
|
|
# event, and running both fired two identical suites concurrently — which
|
|
# doubled the LLM tokens the live tests spend and tripped the proxy's
|
|
# per-minute limit.
|
|
push:
|
|
branches: [main]
|
|
pull_request:
|
|
# Manual re-run, e.g. after rotating a credential or raising a budget cap.
|
|
workflow_dispatch:
|
|
|
|
jobs:
|
|
checks:
|
|
runs-on: ubuntu-latest
|
|
steps:
|
|
- uses: actions/checkout@v4
|
|
|
|
- name: Install uv
|
|
run: |
|
|
curl -LsSf https://astral.sh/uv/install.sh | sh
|
|
echo "$HOME/.local/bin" >> "$GITHUB_PATH"
|
|
|
|
- name: Sync workspace
|
|
run: uv sync --all-packages
|
|
|
|
- name: Install sshd
|
|
# The tests reach "the build VM" over real SSH, through a throwaway
|
|
# sshd the test fixture starts (nisse_dev/testing.py).
|
|
run: |
|
|
SUDO=$(command -v sudo || true)
|
|
$SUDO apt-get update -qq
|
|
$SUDO apt-get install -y -qq openssh-server openssh-client > /dev/null
|
|
$SUDO mkdir -p /run/sshd
|
|
|
|
- name: Lint
|
|
run: uv run ruff check .
|
|
|
|
- name: Format check
|
|
run: uv run ruff format --check .
|
|
|
|
- name: Unit tests
|
|
run: uv run pytest -m "not integration and not e2e"
|
|
|
|
- name: Integration tests
|
|
# Everything but the secrets is in tests/integration/nisse-test.yaml,
|
|
# which refers to these by name. A missing secret fails the live
|
|
# tests that need it; it is never quietly turned into a skip.
|
|
env:
|
|
NISSE_LLM_TOKEN: ${{ secrets.NISSE_LLM_TOKEN }}
|
|
NISSE_MATRIX_TOKEN: ${{ secrets.NISSE_MATRIX_TOKEN }}
|
|
NISSE_GITEA_TOKEN: ${{ secrets.NISSE_GITEA_TOKEN }}
|
|
# The test operator, who makes each run's repository (and its deploy
|
|
# key) and deletes it afterwards.
|
|
NISSE_OPERATOR_TOKEN: ${{ secrets.NISSE_OPERATOR_TOKEN }}
|
|
run: uv run pytest -m "integration and not e2e" -rs
|
|
|
|
e2e:
|
|
# The end-to-end suite (scripts/e2e.sh): the whole agent against real
|
|
# services, every role on a local model, so it spends nothing but keeps
|
|
# the RTX 4090 busy for ~25 minutes. On PRs only, as its own job so the
|
|
# checks above report first; a newer push to the same PR cancels the
|
|
# run it supersedes. Locally, run only what you're writing (-k) or
|
|
# diagnosing (--lf).
|
|
if: github.event_name != 'push'
|
|
runs-on: ubuntu-latest
|
|
timeout-minutes: 90
|
|
concurrency:
|
|
group: e2e-${{ github.head_ref || github.ref }}
|
|
cancel-in-progress: true
|
|
steps:
|
|
- uses: actions/checkout@v4
|
|
|
|
- name: Install uv
|
|
run: |
|
|
curl -LsSf https://astral.sh/uv/install.sh | sh
|
|
echo "$HOME/.local/bin" >> "$GITHUB_PATH"
|
|
|
|
- name: Sync workspace
|
|
run: uv sync --all-packages
|
|
|
|
- name: Install sshd
|
|
# The build VM is a throwaway sshd here (scripts/e2e.sh --local).
|
|
run: |
|
|
SUDO=$(command -v sudo || true)
|
|
$SUDO apt-get update -qq
|
|
$SUDO apt-get install -y -qq openssh-server openssh-client > /dev/null
|
|
$SUDO mkdir -p /run/sshd
|
|
|
|
- name: End-to-end tests
|
|
env:
|
|
NISSE_LLM_TOKEN: ${{ secrets.NISSE_LLM_TOKEN }}
|
|
NISSE_MATRIX_TOKEN: ${{ secrets.NISSE_MATRIX_TOKEN }}
|
|
NISSE_GITEA_TOKEN: ${{ secrets.NISSE_GITEA_TOKEN }}
|
|
NISSE_OPERATOR_TOKEN: ${{ secrets.NISSE_OPERATOR_TOKEN }}
|
|
NISSE_E2E_RUNS: /tmp/nisse-e2e-runs
|
|
run: scripts/e2e.sh --local -v
|