diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..4ea817c --- /dev/null +++ b/.dockerignore @@ -0,0 +1,9 @@ +.git +.env +**/.venv +**/__pycache__ +**/.pytest_cache +**/.ruff_cache +_apalache-out +tmp +bench diff --git a/.gitea/workflows/release.yml b/.gitea/workflows/release.yml new file mode 100644 index 0000000..000a06d --- /dev/null +++ b/.gitea/workflows/release.yml @@ -0,0 +1,63 @@ +name: Release +# A published release builds the agent's image and pushes it to Gitea's +# registry (Development Plan M1.5, X4): gitea.dongar.net/dev/nisse-dev +# tagged with the version (the release tag without its v) and latest. Cut a +# release in Gitea (Releases → New release, tag vX.Y.Z); a tag pushed for any +# other reason builds nothing. +# +# The registry needs a personal access token with write:package +# (REGISTRY_TOKEN, an organization secret of `dev`, belonging to the +# registry-push bot), never the built-in job token: Gitea's registry +# resolves an Actions job token to an empty package scope (go-gitea/gitea +# #23642 and others). Packages belong to the repository's owner, which is +# why this repository is the `dev` organization's: the bot writes packages +# through the org's registry-push team. Never on pull_request: registry +# credentials stay off that trigger. +on: + release: + types: [published] + +env: + REGISTRY: gitea.dongar.net + REGISTRY_USER: registry-push # the token's account + IMAGE: gitea.dongar.net/dev/nisse-dev + +jobs: + build-and-push: + runs-on: ubuntu-latest + permissions: + contents: read + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Derive the image version from the release's tag + id: vars + env: + TAG: ${{ github.event.release.tag_name }} + run: echo "version=${TAG#v}" >> "$GITHUB_OUTPUT" + + - name: Log in to Gitea's container registry + uses: docker/login-action@v3 + with: + registry: ${{ env.REGISTRY }} + username: ${{ env.REGISTRY_USER }} + password: ${{ secrets.REGISTRY_TOKEN }} + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Build and push the agent + uses: docker/build-push-action@v5 + with: + context: . + target: agent + push: true + build-args: | + VERSION=${{ steps.vars.outputs.version }} + labels: | + org.opencontainers.image.version=${{ steps.vars.outputs.version }} + org.opencontainers.image.revision=${{ github.sha }} + tags: | + ${{ env.IMAGE }}:${{ steps.vars.outputs.version }} + ${{ env.IMAGE }}:latest diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..09df155 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,68 @@ +# The nisse-dev agent as a container image (Development Plan M1.5, X4). +# +# Stateless: the state store and SSH known-hosts live in a volume +# (/var/lib/nisse-dev); configuration is mounted read-only +# (/etc/nisse-dev/nisse-dev.yaml); secrets come as environment variables +# (docker --env-file); SSH and deploy keys are mounted read-only wherever the +# configuration says. The agent runs no model-chosen command here (L13): +# those go to build VMs over SSH. +# +# docker build -t nisse-dev . the agent +# docker build --target test -t nisse-dev:test . plus the tests, for +# scripts/e2e.sh --image + +FROM python:3.12-slim AS build +COPY --from=ghcr.io/astral-sh/uv:0.12.10 /uv /usr/local/bin/uv +ENV UV_PROJECT_ENVIRONMENT=/opt/nisse UV_COMPILE_BYTECODE=1 UV_LINK_MODE=copy +WORKDIR /src +COPY pyproject.toml uv.lock ./ +COPY nisse-core nisse-core +COPY nisse-dev nisse-dev +COPY nisse-ops nisse-ops +# The agent and what it needs, installed (not editable): the source tree +# isn't part of the image. +RUN uv sync --frozen --no-dev --package nisse-dev --no-editable + +FROM python:3.12-slim AS runtime +ARG VERSION=dev +LABEL org.opencontainers.image.title=nisse-dev org.opencontainers.image.version=$VERSION +RUN apt-get update \ + && apt-get install -y --no-install-recommends openssh-client \ + && rm -rf /var/lib/apt/lists/* \ + && useradd --create-home --uid 10001 nisse \ + && mkdir -p /var/lib/nisse-dev /etc/nisse-dev \ + && chown nisse /var/lib/nisse-dev +COPY --from=build /opt/nisse /opt/nisse +ENV PATH=/opt/nisse/bin:$PATH \ + NISSE_CONFIG=/etc/nisse-dev/nisse-dev.yaml \ + NISSE_ENV=/run/nisse-dev/.env +VOLUME /var/lib/nisse-dev +USER nisse +ENTRYPOINT ["nisse-dev"] + +# The image's own package plus the tests, so the end-to-end suite runs +# against what was built, not the source tree. +FROM runtime AS test +USER root +RUN apt-get update \ + && apt-get install -y --no-install-recommends git openssh-server \ + && rm -rf /var/lib/apt/lists/* \ + && mkdir -p /run/sshd +COPY --from=build /usr/local/bin/uv /usr/local/bin/uv +WORKDIR /src +COPY pyproject.toml uv.lock ./ +COPY nisse-core nisse-core +COPY nisse-dev nisse-dev +COPY nisse-ops nisse-ops +COPY tests tests +COPY scripts scripts +COPY conftest.py ./ +ENV UV_PROJECT_ENVIRONMENT=/opt/nisse +# Test tools too, all installed (not editable), and `uv run` never re-syncs: +# the tests import the package as built, not the source tree beside them. +RUN uv sync --frozen --all-packages --no-editable --inexact +ENV UV_NO_SYNC=1 +ENTRYPOINT [] + +# The default target: the agent itself. +FROM runtime AS agent diff --git a/docs/Development-Plan.md b/docs/Development-Plan.md index 6efe6c8..5127628 100644 --- a/docs/Development-Plan.md +++ b/docs/Development-Plan.md @@ -271,11 +271,20 @@ in prose, asking mid-work (L5, #42, #77). - Deferred: config reload without a restart (with M2.5's entry step if wanted); the trust dial's configuration (M3.3). -**M1.5 The agent as a container image.** *Pre-approved.* +**M1.5 The agent as a container image.** *Pre-approved.* **Built** (this PR); +accepted once a tag's image passes the e2e suite. - A Dockerfile for `nisse-dev`, stateless with a volume for state; a Gitea Actions workflow that builds it and pushes to Gitea's registry on each release tag (X4). - `bringup.sh` runs the image instead of a venv. - Acceptance: a tagged release produces an image; the end-to-end test runs from it. +- The image runs as its own user; state in `/var/lib/nisse-dev`, configuration + mounted at `/etc/nisse-dev/nisse-dev.yaml`, secrets as environment variables, + keys mounted read-only. Its `test` stage carries the tests over the installed + package, for `scripts/e2e.sh --image REF`. Images are built only on release: + `release.yml` pushes `gitea.dongar.net/dev/nisse-dev:` and `:latest` + on a published release (`REGISTRY_TOKEN`, the `dev` organization's). +- The live bring-up's switch to the image waits for the next live test (it needs + its own test repository too); `bringup.sh plan` shows what it will run. **M2.8 Switch over.** *Pre-approved, after acceptance.* `nisse-dev` runs the new loop; `control.py`, its tests and `orchestrator.qnt` are removed. diff --git a/scripts/bringup.sh b/scripts/bringup.sh index d18123d..01b9233 100755 --- a/scripts/bringup.sh +++ b/scripts/bringup.sh @@ -1,13 +1,19 @@ #!/usr/bin/env bash # Bring-up runs of nisse-dev on this development VM (Development Plan M0.2). # -# The agent runs as root, as it will on its own VM in production (D7): it may -# install what it needs. It touches only test resources (the sandbox repo, the -# test Matrix room, a budget-capped LiteLLM key), from the run's config. It -# runs as a transient systemd unit, so it outlives the shell that started it. +# The agent runs as its container image (M1.5), built from the code under +# test, as a transient systemd unit, so it outlives the shell that started +# it. It touches only test resources (an operator-made test repository, the +# test Matrix room, a budget-capped LiteLLM key), from the run's config. # -# scripts/bringup.sh start [REF] set the last run aside, export REF (default -# HEAD) and start the agent +# $E2E is mounted at the same path inside the container, so the config's +# paths (state_db, keys) mean the same there; for that it runs as root here, +# since the keys are root's. In production the image runs as its own user +# with a state volume (Dockerfile). +# +# scripts/bringup.sh start [REF] set the last run aside, build REF (default +# HEAD) into an image and start the agent +# scripts/bringup.sh plan [REF] print what start would build and run # scripts/bringup.sh stop stop it cleanly (SIGTERM) # scripts/bringup.sh status the unit, and the agent's state # scripts/bringup.sh logs follow the agent's log @@ -16,7 +22,7 @@ # # Layout, under $E2E (default /root/nisse-e2e): # nisse-dev.yaml, .env the run config and its secrets (made by hand once) -# code/ the exported code under test, with its own .venv +# code/ the exported code under test, built into the image # state.db the agent's state (its workspace is on the build VM) # ssh/ the agent's key for the build VM, and push keys # run-info, collected/ what was run, and the output of `collect` @@ -27,11 +33,28 @@ E2E=${NISSE_E2E:-/root/nisse-e2e} SRC=${NISSE_SRC:-/root/nisse} # repository to export code from CODE=$E2E/code UNIT=nisse-bringup -UV=${UV:-$(command -v uv || echo /snap/bin/uv)} +IMAGE=nisse-dev:bringup die() { echo "bringup: $*" >&2; exit 1; } running() { systemctl is-active --quiet "$UNIT"; } +# The container the unit runs: host networking (Matrix, Gitea, LiteLLM as +# here), $E2E at its own path, the config and secrets from it. +docker_run() { + echo docker run --rm --name "$UNIT" --network host --user 0 \ + -v "$E2E:$E2E" \ + -e NISSE_CONFIG="$E2E/nisse-dev.yaml" -e NISSE_ENV="$E2E/.env" \ + "$IMAGE" +} + +cmd_plan() { + local ref=${1:-HEAD} sha + sha=$(git -C "$SRC" rev-parse --verify "$ref^{commit}") || die "unknown ref $ref" + echo "git -C $SRC archive $sha | tar -x -C $CODE" + echo "docker build -t $IMAGE $CODE" + echo "systemd-run --unit=$UNIT ... $(docker_run)" +} + cmd_start() { local ref=${1:-HEAD} sha [ -f "$E2E/nisse-dev.yaml" ] && [ -f "$E2E/.env" ] || die "no config in $E2E" @@ -58,16 +81,16 @@ cmd_start() { rm -rf "$CODE" mkdir -p "$CODE" git -C "$SRC" archive "$sha" | tar -x -C "$CODE" - (cd "$CODE" && "$UV" sync --quiet --all-packages) + docker build --quiet -t "$IMAGE" "$CODE" > /dev/null printf 'ref: %s\ncommit: %s\nstarted: %s\n' "$ref" "$sha" "$(date -Is)" > "$E2E/run-info" systemctl reset-failed "$UNIT" 2>/dev/null || true + # SIGTERM reaches the agent through docker run's signal proxy; it shuts + # down cleanly (main.py). + # shellcheck disable=SC2046 systemd-run --quiet --unit="$UNIT" --collect \ - --property=WorkingDirectory="$CODE" \ --property=KillSignal=SIGTERM --property=TimeoutStopSec=120 \ - --setenv=NISSE_CONFIG="$E2E/nisse-dev.yaml" \ - --setenv=NISSE_ENV="$E2E/.env" \ - "$CODE/.venv/bin/nisse-dev" + $(docker_run) echo "started $UNIT at ${sha:0:8} ($ref)" } @@ -78,15 +101,17 @@ remote_aside() { where=$(python3 - "$E2E/nisse-dev.yaml" <<'PY' import sys, yaml c = yaml.safe_load(open(sys.argv[1])) -b = c["build_host"] -print(b.get("user", "root"), b["address"], b.get("port", 22), b["key"], c["workspace"]) +for p in c["projects"].values(): # each project's workspace, on its build VM (D8) + b = p["build_host"] + print(b.get("user", "root"), b["address"], b.get("port", 22), b["key"], p["workspace"]) PY -) || die "cannot read build_host and workspace from the config" - read -r user address port key workspace <<< "$where" - ssh -i "$key" -p "$port" -o BatchMode=yes -o StrictHostKeyChecking=accept-new \ - -o UserKnownHostsFile="$E2E/ssh/known_hosts" "$user@$address" \ - "if [ -e '$workspace' ]; then mv '$workspace' '$workspace.$stamp' && echo 'build VM: set aside $workspace.$stamp'; fi" \ - || die "cannot reach the build VM $address" +) || die "cannot read the projects' build VMs and workspaces from the config" + while read -r user address port key workspace; do + ssh -n -i "$key" -p "$port" -o BatchMode=yes -o StrictHostKeyChecking=accept-new \ + -o UserKnownHostsFile="$E2E/ssh/known_hosts" "$user@$address" \ + "if [ -e '$workspace' ]; then mv '$workspace' '$workspace.$stamp' && echo 'build VM: set aside $workspace.$stamp'; fi" \ + || die "cannot reach the build VM $address" + done <<< "$where" } cmd_stop() { @@ -98,28 +123,35 @@ cmd_stop() { cmd_status() { if running; then echo "unit: running"; else echo "unit: not running"; fi [ -f "$E2E/run-info" ] && sed 's/^/ /' "$E2E/run-info" - if [ -f "$E2E/state.db" ] && [ -x "$CODE/.venv/bin/python" ]; then - "$CODE/.venv/bin/python" -m nisse_dev.dump "$E2E/state.db" | head -n 3 + if [ -f "$E2E/state.db" ]; then + dump "$E2E/state.db" | head -n 3 fi } cmd_logs() { journalctl -u "$UNIT" -f -o cat; } +# The state store, read with the image's own code. +dump() { + docker run --rm --user 0 -v "$E2E:$E2E" --entrypoint python "$IMAGE" \ + -m nisse_dev.dump "$@" +} + cmd_collect() { [ -f "$E2E/run-info" ] || die "no run to collect" local out=$E2E/collected since since=$(sed -n 's/^started: //p' "$E2E/run-info") mkdir -p "$out" journalctl -u "$UNIT" --since "$since" -o cat > "$out/agent.log" - "$CODE/.venv/bin/python" -m nisse_dev.dump --full "$E2E/state.db" > "$out/transcript.txt" + dump --full "$E2E/state.db" > "$out/transcript.txt" echo "collected into $out" } case "${1:-}" in start) shift; cmd_start "$@" ;; + plan) shift; cmd_plan "$@" ;; stop) cmd_stop ;; status) cmd_status ;; logs) cmd_logs ;; collect) cmd_collect ;; - *) sed -n '2,25p' "$0" | sed 's/^# \{0,1\}//'; exit 2 ;; + *) sed -n '2,30p' "$0" | sed 's/^# \{0,1\}//'; exit 2 ;; esac diff --git a/scripts/e2e.sh b/scripts/e2e.sh index f16c1e9..ddb9068 100755 --- a/scripts/e2e.sh +++ b/scripts/e2e.sh @@ -6,6 +6,8 @@ # # scripts/e2e.sh on the bring-up build VM (configured below) # scripts/e2e.sh --local on a throwaway local sshd instead +# scripts/e2e.sh --image REF inside an image's `test` stage (Dockerfile), +# on its own sshd: checks a release as built # # Uses tests/integration/nisse-test.yaml and its secrets (.env at the repo # root), like the integration tests. Every role runs on a local model @@ -14,6 +16,18 @@ set -euo pipefail cd "$(dirname "$0")/.." +if [ "${1:-}" = "--image" ]; then + image=${2:?--image needs an image reference} + shift 2 + runs=${NISSE_E2E_RUNS:-/root/nisse-e2e/test-runs} + mkdir -p "$runs" + secrets=() + [ -f .env ] && secrets=(--env-file .env) + # Host networking: the suite reaches Matrix, Gitea and LiteLLM as here. + exec docker run --rm --network host "${secrets[@]}" \ + -e NISSE_E2E_RUNS=/runs -v "$runs:/runs" \ + "$image" scripts/e2e.sh --local "$@" +fi if [ "${1:-}" = "--local" ]; then shift else