From 0660fa3c5189eeb8dc09f136621b6e0f675f8d56 Mon Sep 17 00:00:00 2001 From: Don Garrett Date: Wed, 7 Oct 2026 19:50:59 +0000 Subject: [PATCH] The agent as a container image (M1.5) - Dockerfile: built with uv into /opt/nisse (installed, not editable), run on python:3.12-slim with openssh-client as user nisse; stateless, with state in a /var/lib/nisse-dev volume, config mounted at /etc/nisse-dev/nisse-dev.yaml, secrets as environment variables, keys mounted read-only. 202 MB. Its `test` stage adds the tests and their tools over the installed package (UV_NO_SYNC, so `uv run` never swaps in the source tree); the default target is the agent. - scripts/e2e.sh --image REF runs the suite inside an image's test stage, on its own sshd. - CI builds both images on every PR (nothing pushed); release.yml pushes gitea.dongar.net/dongar/nisse-dev: and :latest on a v* tag, with the REGISTRY_TOKEN secret. - scripts/bringup.sh builds the code under test into an image and runs it in the systemd unit (root, $E2E mounted at its own path); `plan` prints what it would run; the build-VM set-aside reads the projects: config. Checked: the image refuses to start without a config, runs as uid 10001; one e2e test passed from the test stage (2m45s, $0). Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01PPQ14L3LikyefCMFBqEbKE --- .dockerignore | 9 ++++ .gitea/workflows/ci.yml | 14 ++++++ .gitea/workflows/release.yml | 35 +++++++++++++++ Dockerfile | 66 +++++++++++++++++++++++++++++ docs/Development-Plan.md | 11 ++++- scripts/bringup.sh | 82 +++++++++++++++++++++++++----------- scripts/e2e.sh | 14 ++++++ 7 files changed, 205 insertions(+), 26 deletions(-) create mode 100644 .dockerignore create mode 100644 .gitea/workflows/release.yml create mode 100644 Dockerfile diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..4ea817c --- /dev/null +++ b/.dockerignore @@ -0,0 +1,9 @@ +.git +.env +**/.venv +**/__pycache__ +**/.pytest_cache +**/.ruff_cache +_apalache-out +tmp +bench diff --git a/.gitea/workflows/ci.yml b/.gitea/workflows/ci.yml index 9d2fd61..1a19f8b 100644 --- a/.gitea/workflows/ci.yml +++ b/.gitea/workflows/ci.yml @@ -55,6 +55,20 @@ jobs: NISSE_OPERATOR_TOKEN: ${{ secrets.NISSE_OPERATOR_TOKEN }} run: uv run pytest -m "integration and not e2e" -rs + image: + # The agent's image builds (Dockerfile), so a broken one shows before a + # release does. On PRs only; nothing is pushed. + if: github.event_name != 'push' + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Build the agent and test images + run: | + docker build --target agent -t nisse-dev:ci . + docker build --target test -t nisse-dev:ci-test . + docker run --rm nisse-dev:ci 2>&1 | grep -q "cannot start: no config file" + e2e: # The end-to-end suite (scripts/e2e.sh): the whole agent against real # services, every role on a local model, so it spends nothing but keeps diff --git a/.gitea/workflows/release.yml b/.gitea/workflows/release.yml new file mode 100644 index 0000000..51803a2 --- /dev/null +++ b/.gitea/workflows/release.yml @@ -0,0 +1,35 @@ +name: Release +# A tag v* publishes the agent's image to Gitea's registry (Development Plan +# M1.5, X4): gitea.dongar.net/dongar/nisse-dev: and :latest. The token +# is a secret (REGISTRY_TOKEN: write:package for REGISTRY_USER); the +# account is a setting, so it's here. +on: + push: + tags: ["v*"] + +env: + REGISTRY: gitea.dongar.net + REGISTRY_USER: dongar + IMAGE: gitea.dongar.net/dongar/nisse-dev + +jobs: + image: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Build + run: | + docker build --target agent \ + --label org.opencontainers.image.revision="${{ github.sha }}" \ + --label org.opencontainers.image.version="${{ github.ref_name }}" \ + -t "$IMAGE:${{ github.ref_name }}" -t "$IMAGE:latest" . + + - name: Push + env: + REGISTRY_TOKEN: ${{ secrets.REGISTRY_TOKEN }} + run: | + printf '%s' "$REGISTRY_TOKEN" | docker login "$REGISTRY" -u "$REGISTRY_USER" --password-stdin + docker push "$IMAGE:${{ github.ref_name }}" + docker push "$IMAGE:latest" + docker logout "$REGISTRY" diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..500e760 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,66 @@ +# The nisse-dev agent as a container image (Development Plan M1.5, X4). +# +# Stateless: the state store and SSH known-hosts live in a volume +# (/var/lib/nisse-dev); configuration is mounted read-only +# (/etc/nisse-dev/nisse-dev.yaml); secrets come as environment variables +# (docker --env-file); SSH and deploy keys are mounted read-only wherever the +# configuration says. The agent runs no model-chosen command here (L13): +# those go to build VMs over SSH. +# +# docker build -t nisse-dev . the agent +# docker build --target test -t nisse-dev:test . plus the tests, for +# scripts/e2e.sh --image + +FROM python:3.12-slim AS build +COPY --from=ghcr.io/astral-sh/uv:0.12.10 /uv /usr/local/bin/uv +ENV UV_PROJECT_ENVIRONMENT=/opt/nisse UV_COMPILE_BYTECODE=1 UV_LINK_MODE=copy +WORKDIR /src +COPY pyproject.toml uv.lock ./ +COPY nisse-core nisse-core +COPY nisse-dev nisse-dev +COPY nisse-ops nisse-ops +# The agent and what it needs, installed (not editable): the source tree +# isn't part of the image. +RUN uv sync --frozen --no-dev --package nisse-dev --no-editable + +FROM python:3.12-slim AS runtime +RUN apt-get update \ + && apt-get install -y --no-install-recommends openssh-client \ + && rm -rf /var/lib/apt/lists/* \ + && useradd --create-home --uid 10001 nisse \ + && mkdir -p /var/lib/nisse-dev /etc/nisse-dev \ + && chown nisse /var/lib/nisse-dev +COPY --from=build /opt/nisse /opt/nisse +ENV PATH=/opt/nisse/bin:$PATH \ + NISSE_CONFIG=/etc/nisse-dev/nisse-dev.yaml \ + NISSE_ENV=/run/nisse-dev/.env +VOLUME /var/lib/nisse-dev +USER nisse +ENTRYPOINT ["nisse-dev"] + +# The image's own package plus the tests, so the end-to-end suite runs +# against what was built, not the source tree. +FROM runtime AS test +USER root +RUN apt-get update \ + && apt-get install -y --no-install-recommends git openssh-server \ + && rm -rf /var/lib/apt/lists/* \ + && mkdir -p /run/sshd +COPY --from=build /usr/local/bin/uv /usr/local/bin/uv +WORKDIR /src +COPY pyproject.toml uv.lock ./ +COPY nisse-core nisse-core +COPY nisse-dev nisse-dev +COPY nisse-ops nisse-ops +COPY tests tests +COPY scripts scripts +COPY conftest.py ./ +ENV UV_PROJECT_ENVIRONMENT=/opt/nisse +# Test tools too, all installed (not editable), and `uv run` never re-syncs: +# the tests import the package as built, not the source tree beside them. +RUN uv sync --frozen --all-packages --no-editable --inexact +ENV UV_NO_SYNC=1 +ENTRYPOINT [] + +# The default target: the agent itself. +FROM runtime AS agent diff --git a/docs/Development-Plan.md b/docs/Development-Plan.md index 6efe6c8..796a0c0 100644 --- a/docs/Development-Plan.md +++ b/docs/Development-Plan.md @@ -271,11 +271,20 @@ in prose, asking mid-work (L5, #42, #77). - Deferred: config reload without a restart (with M2.5's entry step if wanted); the trust dial's configuration (M3.3). -**M1.5 The agent as a container image.** *Pre-approved.* +**M1.5 The agent as a container image.** *Pre-approved.* **Built** (this PR); +accepted once a tag's image passes the e2e suite. - A Dockerfile for `nisse-dev`, stateless with a volume for state; a Gitea Actions workflow that builds it and pushes to Gitea's registry on each release tag (X4). - `bringup.sh` runs the image instead of a venv. - Acceptance: a tagged release produces an image; the end-to-end test runs from it. +- The image runs as its own user; state in `/var/lib/nisse-dev`, configuration + mounted at `/etc/nisse-dev/nisse-dev.yaml`, secrets as environment variables, + keys mounted read-only. Its `test` stage carries the tests over the installed + package, for `scripts/e2e.sh --image REF`. CI builds both on every PR; + `release.yml` pushes `gitea.dongar.net/dongar/nisse-dev:` and `:latest` + (`REGISTRY_TOKEN`). +- The live bring-up's switch to the image waits for the next live test (it needs + its own test repository too); `bringup.sh plan` shows what it will run. **M2.8 Switch over.** *Pre-approved, after acceptance.* `nisse-dev` runs the new loop; `control.py`, its tests and `orchestrator.qnt` are removed. diff --git a/scripts/bringup.sh b/scripts/bringup.sh index d18123d..01b9233 100755 --- a/scripts/bringup.sh +++ b/scripts/bringup.sh @@ -1,13 +1,19 @@ #!/usr/bin/env bash # Bring-up runs of nisse-dev on this development VM (Development Plan M0.2). # -# The agent runs as root, as it will on its own VM in production (D7): it may -# install what it needs. It touches only test resources (the sandbox repo, the -# test Matrix room, a budget-capped LiteLLM key), from the run's config. It -# runs as a transient systemd unit, so it outlives the shell that started it. +# The agent runs as its container image (M1.5), built from the code under +# test, as a transient systemd unit, so it outlives the shell that started +# it. It touches only test resources (an operator-made test repository, the +# test Matrix room, a budget-capped LiteLLM key), from the run's config. # -# scripts/bringup.sh start [REF] set the last run aside, export REF (default -# HEAD) and start the agent +# $E2E is mounted at the same path inside the container, so the config's +# paths (state_db, keys) mean the same there; for that it runs as root here, +# since the keys are root's. In production the image runs as its own user +# with a state volume (Dockerfile). +# +# scripts/bringup.sh start [REF] set the last run aside, build REF (default +# HEAD) into an image and start the agent +# scripts/bringup.sh plan [REF] print what start would build and run # scripts/bringup.sh stop stop it cleanly (SIGTERM) # scripts/bringup.sh status the unit, and the agent's state # scripts/bringup.sh logs follow the agent's log @@ -16,7 +22,7 @@ # # Layout, under $E2E (default /root/nisse-e2e): # nisse-dev.yaml, .env the run config and its secrets (made by hand once) -# code/ the exported code under test, with its own .venv +# code/ the exported code under test, built into the image # state.db the agent's state (its workspace is on the build VM) # ssh/ the agent's key for the build VM, and push keys # run-info, collected/ what was run, and the output of `collect` @@ -27,11 +33,28 @@ E2E=${NISSE_E2E:-/root/nisse-e2e} SRC=${NISSE_SRC:-/root/nisse} # repository to export code from CODE=$E2E/code UNIT=nisse-bringup -UV=${UV:-$(command -v uv || echo /snap/bin/uv)} +IMAGE=nisse-dev:bringup die() { echo "bringup: $*" >&2; exit 1; } running() { systemctl is-active --quiet "$UNIT"; } +# The container the unit runs: host networking (Matrix, Gitea, LiteLLM as +# here), $E2E at its own path, the config and secrets from it. +docker_run() { + echo docker run --rm --name "$UNIT" --network host --user 0 \ + -v "$E2E:$E2E" \ + -e NISSE_CONFIG="$E2E/nisse-dev.yaml" -e NISSE_ENV="$E2E/.env" \ + "$IMAGE" +} + +cmd_plan() { + local ref=${1:-HEAD} sha + sha=$(git -C "$SRC" rev-parse --verify "$ref^{commit}") || die "unknown ref $ref" + echo "git -C $SRC archive $sha | tar -x -C $CODE" + echo "docker build -t $IMAGE $CODE" + echo "systemd-run --unit=$UNIT ... $(docker_run)" +} + cmd_start() { local ref=${1:-HEAD} sha [ -f "$E2E/nisse-dev.yaml" ] && [ -f "$E2E/.env" ] || die "no config in $E2E" @@ -58,16 +81,16 @@ cmd_start() { rm -rf "$CODE" mkdir -p "$CODE" git -C "$SRC" archive "$sha" | tar -x -C "$CODE" - (cd "$CODE" && "$UV" sync --quiet --all-packages) + docker build --quiet -t "$IMAGE" "$CODE" > /dev/null printf 'ref: %s\ncommit: %s\nstarted: %s\n' "$ref" "$sha" "$(date -Is)" > "$E2E/run-info" systemctl reset-failed "$UNIT" 2>/dev/null || true + # SIGTERM reaches the agent through docker run's signal proxy; it shuts + # down cleanly (main.py). + # shellcheck disable=SC2046 systemd-run --quiet --unit="$UNIT" --collect \ - --property=WorkingDirectory="$CODE" \ --property=KillSignal=SIGTERM --property=TimeoutStopSec=120 \ - --setenv=NISSE_CONFIG="$E2E/nisse-dev.yaml" \ - --setenv=NISSE_ENV="$E2E/.env" \ - "$CODE/.venv/bin/nisse-dev" + $(docker_run) echo "started $UNIT at ${sha:0:8} ($ref)" } @@ -78,15 +101,17 @@ remote_aside() { where=$(python3 - "$E2E/nisse-dev.yaml" <<'PY' import sys, yaml c = yaml.safe_load(open(sys.argv[1])) -b = c["build_host"] -print(b.get("user", "root"), b["address"], b.get("port", 22), b["key"], c["workspace"]) +for p in c["projects"].values(): # each project's workspace, on its build VM (D8) + b = p["build_host"] + print(b.get("user", "root"), b["address"], b.get("port", 22), b["key"], p["workspace"]) PY -) || die "cannot read build_host and workspace from the config" - read -r user address port key workspace <<< "$where" - ssh -i "$key" -p "$port" -o BatchMode=yes -o StrictHostKeyChecking=accept-new \ - -o UserKnownHostsFile="$E2E/ssh/known_hosts" "$user@$address" \ - "if [ -e '$workspace' ]; then mv '$workspace' '$workspace.$stamp' && echo 'build VM: set aside $workspace.$stamp'; fi" \ - || die "cannot reach the build VM $address" +) || die "cannot read the projects' build VMs and workspaces from the config" + while read -r user address port key workspace; do + ssh -n -i "$key" -p "$port" -o BatchMode=yes -o StrictHostKeyChecking=accept-new \ + -o UserKnownHostsFile="$E2E/ssh/known_hosts" "$user@$address" \ + "if [ -e '$workspace' ]; then mv '$workspace' '$workspace.$stamp' && echo 'build VM: set aside $workspace.$stamp'; fi" \ + || die "cannot reach the build VM $address" + done <<< "$where" } cmd_stop() { @@ -98,28 +123,35 @@ cmd_stop() { cmd_status() { if running; then echo "unit: running"; else echo "unit: not running"; fi [ -f "$E2E/run-info" ] && sed 's/^/ /' "$E2E/run-info" - if [ -f "$E2E/state.db" ] && [ -x "$CODE/.venv/bin/python" ]; then - "$CODE/.venv/bin/python" -m nisse_dev.dump "$E2E/state.db" | head -n 3 + if [ -f "$E2E/state.db" ]; then + dump "$E2E/state.db" | head -n 3 fi } cmd_logs() { journalctl -u "$UNIT" -f -o cat; } +# The state store, read with the image's own code. +dump() { + docker run --rm --user 0 -v "$E2E:$E2E" --entrypoint python "$IMAGE" \ + -m nisse_dev.dump "$@" +} + cmd_collect() { [ -f "$E2E/run-info" ] || die "no run to collect" local out=$E2E/collected since since=$(sed -n 's/^started: //p' "$E2E/run-info") mkdir -p "$out" journalctl -u "$UNIT" --since "$since" -o cat > "$out/agent.log" - "$CODE/.venv/bin/python" -m nisse_dev.dump --full "$E2E/state.db" > "$out/transcript.txt" + dump --full "$E2E/state.db" > "$out/transcript.txt" echo "collected into $out" } case "${1:-}" in start) shift; cmd_start "$@" ;; + plan) shift; cmd_plan "$@" ;; stop) cmd_stop ;; status) cmd_status ;; logs) cmd_logs ;; collect) cmd_collect ;; - *) sed -n '2,25p' "$0" | sed 's/^# \{0,1\}//'; exit 2 ;; + *) sed -n '2,30p' "$0" | sed 's/^# \{0,1\}//'; exit 2 ;; esac diff --git a/scripts/e2e.sh b/scripts/e2e.sh index f16c1e9..ddb9068 100755 --- a/scripts/e2e.sh +++ b/scripts/e2e.sh @@ -6,6 +6,8 @@ # # scripts/e2e.sh on the bring-up build VM (configured below) # scripts/e2e.sh --local on a throwaway local sshd instead +# scripts/e2e.sh --image REF inside an image's `test` stage (Dockerfile), +# on its own sshd: checks a release as built # # Uses tests/integration/nisse-test.yaml and its secrets (.env at the repo # root), like the integration tests. Every role runs on a local model @@ -14,6 +16,18 @@ set -euo pipefail cd "$(dirname "$0")/.." +if [ "${1:-}" = "--image" ]; then + image=${2:?--image needs an image reference} + shift 2 + runs=${NISSE_E2E_RUNS:-/root/nisse-e2e/test-runs} + mkdir -p "$runs" + secrets=() + [ -f .env ] && secrets=(--env-file .env) + # Host networking: the suite reaches Matrix, Gitea and LiteLLM as here. + exec docker run --rm --network host "${secrets[@]}" \ + -e NISSE_E2E_RUNS=/runs -v "$runs:/runs" \ + "$image" scripts/e2e.sh --local "$@" +fi if [ "${1:-}" = "--local" ]; then shift else