Merge pull request 'The agent host's vault and SilverBullet (M5.4)' (#166) from agent-host-vault into main

deploy/agent-host/README.md

@@ -0,0 +1,54 @@
+# The agent host
+
+`nisse.garrett-hall.com`, made by infra/tofu#77 as `nisse-host` and since
+renamed (architecture X1). For now it holds the knowledge vault and
+SilverBullet, the operator's view of it (L10, M5.4). The agents' containers come
+later (M6.1).
+
+## What's here
+
+- `silverbullet/compose.yaml`: SilverBullet, its version pinned, serving the
+  vault as its one space, with the agents' `memory/` read-only. Nothing else of
+  the host. It has no accounts: the operator logs in through Authelia.
+- `setup.sh`: makes the vault's folders, installs the stack under
+  `/opt/stacks/silverbullet` and the firewall, and starts it. Safe to run again.
+- `nisse-firewall`: only the operator's Caddy may reach SilverBullet's port
+  (Docker's `DOCKER-USER` chain, re-applied whenever Docker starts).
+- `check.sh`: run on the host by `scripts/agent-host.sh check`; says "all
+  checks passed" only once every check has run.
+- `caddy/nisse-memory.dongar.net.caddy`: the site for the operator's Caddy on
+  `docker.garrett-hall.com`. HTTPS from Caddy's usual certificate,
+  behind Authelia.
+
+## The vault
+
+`/srv/nisse/vault`, one copy on the host's disk, nothing synced:
+
+- **The operator's:** the root and everything in it but `memory/`, written
+  through SilverBullet. `infrastructure/` holds material for `nisse-ops`, which
+  `nisse-dev` will never be given.
+- **The agents':** `memory/<agent>/`, one writer each, when the agents move in.
+  SilverBullet sees them read-only.
+
+Backed up nightly with the VM (Proxmox; `backup: true` in infra/tofu).
+
+## Installing and checking
+
+From the development VM, with the bring-up key the operator added for `ubuntu`:
+
+```sh
+scripts/agent-host.sh install   # copy this folder over and run setup.sh
+scripts/agent-host.sh check     # running, mounts, firewall
+```
+
+Then **the operator adds the Caddy site** from
+`caddy/nisse-memory.dongar.net.caddy`, behind Authelia, and reloads Caddy.
+There's no setup wizard: SilverBullet starts serving the vault at once.
+
+**Making `memory/` writable for a while** (for example, to upload material that
+will be sorted into the agents' folders): on the host, add
+`/opt/stacks/silverbullet/compose.override.yaml` that replaces the volumes with
+the vault alone (`volumes: !override [/srv/nisse/vault:/data]`), give `memory/`
+to uid 1000, and `docker compose up -d`. `check` fails while it's in place.
+To end it, delete the override and run `install` again, which gives `memory/`
+back to root.

deploy/agent-host/caddy/nisse-memory.dongar.net.caddy

@@ -0,0 +1,12 @@
+# The vault's SilverBullet, for the operator's Caddy on docker.garrett-hall.com
+# (/opt/stacks). HTTPS from Caddy's usual automatic certificate; LAN only.
+# Out of date: the live site is behind Authelia and open beyond the LAN. This
+# copy is updated once the operator shares it.
+nisse-memory.dongar.net {
+	# Anyone outside the LAN gets nothing. Caddy still answers Let's Encrypt's
+	# checks, which it handles before the site's own routes.
+	@outside not remote_ip private_ranges
+	respond @outside 403
+
+	reverse_proxy nisse.garrett-hall.com:3000
+}

deploy/agent-host/check.sh

@@ -0,0 +1,33 @@
+#!/usr/bin/env bash
+# Checks the vault and SilverBullet on the agent host; run there as a user with
+# sudo (scripts/agent-host.sh check sends it). Exits 1 if any check fails, and
+# only says "all checks passed" once every one has run.
+set -u
+stack=${STACK:-/opt/stacks/silverbullet}
+fail=0
+ok() { echo "ok:   $1"; }
+bad() { echo "FAIL: $1"; fail=1; }
+cd "$stack" || { echo "FAIL: no $stack"; exit 1; }
+[ "$(sudo docker compose ps --status running -q | wc -l)" -ge 1 ] \
+  && ok "SilverBullet is running" || bad "SilverBullet isn't running"
+if sudo docker compose logs --no-color silverbullet 2>/dev/null | grep -q 'boot mode: single-space'; then
+  ok "it serves the vault as one space (no setup wizard)"
+else
+  bad "it didn't boot one space: the setup wizard may be open"
+fi
+code=$(curl -s -o /dev/null -w '%{http_code}' http://127.0.0.1:3000/)
+case $code in 2??|3??) ok "it answers locally ($code)" ;; *) bad "it answers $code" ;; esac
+probe=.nisse-probe-$$
+sudo docker compose exec -T -u 1000 silverbullet \
+  sh -c "touch /data/infrastructure/$probe && rm /data/infrastructure/$probe" \
+  && ok "the operator's folders are writable" || bad "the operator's folders aren't writable"
+if sudo docker compose exec -T silverbullet sh -c "touch /data/memory/$probe" 2>/dev/null; then
+  sudo docker compose exec -T silverbullet rm -f "/data/memory/$probe"
+  bad "memory/ is writable from SilverBullet, even as root in it"
+else
+  ok "memory/ is read-only to SilverBullet"
+fi
+sudo iptables -S DOCKER-USER | grep -q -- '--comment nisse-firewall' \
+  && ok "the firewall rule is in place" || bad "no firewall rule"
+[ $fail = 0 ] && echo "all checks passed"
+exit $fail

deploy/agent-host/nisse-firewall

@@ -0,0 +1,12 @@
+#!/bin/sh
+# nisse-firewall ADDRESS PORT: only ADDRESS may reach a port Docker publishes
+# as PORT. Docker publishes ports past the host's own firewall rules, so the
+# limit goes in its DOCKER-USER chain. Installed by setup.sh and run by
+# nisse-firewall.service whenever Docker starts. Its own rules (tagged) are
+# replaced each run, so a changed address leaves no old rule behind.
+set -eu
+address=$1 port=$2
+iptables -S DOCKER-USER | grep -- '--comment nisse-firewall' | sed 's/^-A /-D /' |
+  while read -r old; do eval "iptables $old"; done
+iptables -I DOCKER-USER -p tcp -m conntrack --ctorigdstport "$port" \
+  --ctdir ORIGINAL ! -s "$address" -m comment --comment nisse-firewall -j DROP

deploy/agent-host/setup.sh

@@ -0,0 +1,49 @@
+#!/usr/bin/env bash
+# Set up the agent host's vault and SilverBullet (M5.4). Run as root on the
+# host, from this folder's copy (scripts/agent-host.sh install does that).
+# Safe to run again: it changes only what differs.
+set -euo pipefail
+
+here=$(cd "$(dirname "$0")" && pwd)
+
+# The only address allowed to reach SilverBullet: the operator's Caddy
+# (docker.garrett-hall.com), which serves nisse-memory.dongar.net behind
+# Authelia.
+CADDY_ADDR=192.168.6.13
+SB_PORT=3000
+SB_UID=1000  # the image's PUID/PGID: the operator's side of the vault
+
+# The vault. Its root and everything in it belong to the operator, except
+# memory/, where each agent gets a folder of its own when it moves in. memory/
+# is root's, and SilverBullet mounts it read-only besides.
+install -d -m 0755 /srv/nisse
+install -d -o "$SB_UID" -g "$SB_UID" -m 0755 /srv/nisse/vault
+install -d -o "$SB_UID" -g "$SB_UID" -m 0755 /srv/nisse/vault/infrastructure
+install -d -o root -g root -m 0755 /srv/nisse/vault/memory
+
+# SilverBullet's stack, where the docker role keeps stacks.
+install -d -m 0755 /opt/stacks/silverbullet
+install -m 0644 "$here/silverbullet/compose.yaml" /opt/stacks/silverbullet/compose.yaml
+
+# Only Caddy may reach SilverBullet's port, re-applied whenever Docker starts.
+install -m 0755 "$here/nisse-firewall" /usr/local/sbin/nisse-firewall
+cat > /etc/systemd/system/nisse-firewall.service <<EOF
+[Unit]
+Description=Only the operator's Caddy reaches SilverBullet (deploy/agent-host)
+After=docker.service
+BindsTo=docker.service
+
+[Service]
+Type=oneshot
+RemainAfterExit=yes
+ExecStart=/usr/local/sbin/nisse-firewall $CADDY_ADDR $SB_PORT
+
+[Install]
+WantedBy=docker.service
+EOF
+systemctl daemon-reload
+systemctl enable --now nisse-firewall.service
+systemctl restart nisse-firewall.service  # picks up a changed rule
+
+docker compose -f /opt/stacks/silverbullet/compose.yaml up -d
+echo "SilverBullet is up on port $SB_PORT, reachable from $CADDY_ADDR only"

deploy/agent-host/silverbullet/compose.yaml

@@ -0,0 +1,28 @@
+# SilverBullet on the agent host: the operator's view of the vault (L10, M5.4).
+# Installed to /opt/stacks/silverbullet by ../setup.sh. No secrets here: it
+# has no accounts of its own; the operator logs in through Authelia, in front of
+# it in their Caddy, and only Caddy can reach it (setup.sh's firewall).
+services:
+  silverbullet:
+    # Pinned: the docker role updates stacks' images nightly, and a pinned
+    # version only changes in a PR here.
+    image: ghcr.io/silverbulletmd/silverbullet:2.11.1-slim
+    restart: unless-stopped
+    environment:
+      PUID: "1000"
+      PGID: "1000"
+      # Set on purpose: any SB_* setting boots one space, the vault mounted at
+      # /data, with no accounts or setup wizard. Without one, an empty vault
+      # would boot the wizard instead.
+      SB_NAME: Nisse vault
+    ports:
+      # IPv4 only; setup.sh's firewall lets only the operator's Caddy in.
+      - "0.0.0.0:3000:3000"
+    volumes:
+      # The vault is the space: the operator writes everything in it...
+      - /srv/nisse/vault:/data
+      # ...except the agents' memory, which only the agents write (one writer
+      # per folder, enforced by the kernel).
+      - /srv/nisse/vault/memory:/data/memory:ro
+    logging:
+      driver: journald  # logs go to the host's journal (L12)

docs/Development-Plan.md

@@ -535,19 +535,38 @@ notes are written and read (L6.4).
   items.
 
 **M5.4 The operator's view.** *Pre-approved.* SilverBullet in its own container on
-the vault: the operator's folders read-write, the agents' read-only (L10), behind
+the vault: the operator's folders read-write, the agents' read-only (L10). Done
-Authelia.
+early on the agent host (`deploy/agent-host`), so the operator can keep material
+for `nisse-ops` there now:
+- The vault is `/srv/nisse/vault`. The operator owns its root and everything in
+  it but `memory/`, the agents'; `infrastructure/` is the operator's material
+  for `nisse-ops`. SilverBullet serves the vault as its one space (single-space
+  mode, mounted at `/data`), with `memory/` mounted over it read-only, and sees
+  nothing else of the host.
+- SilverBullet's version is pinned (the docker role updates images nightly).
+  It has no accounts and no setup wizard; `SB_NAME` forces single-space mode,
+  so an empty vault can't boot the wizard.
+- Decided with the operator: behind their Caddy at `nisse-memory.dongar.net` for
+  HTTPS, reachable from anywhere through Authelia, which is the login (agents
+  never go through SilverBullet, so this changes only the operator's access).
+  Its own instance rather than the operator's personal one, so the vault stays
+  one local copy with mounts enforcing its writers. On the host, a
+  `DOCKER-USER` rule lets only Caddy reach it, so nothing gets around Authelia.
 
 ### Phase 6 — Production
 
-*Waiting on infrastructure (2026-10-13):* the agent host doesn't exist yet, and
+*Infrastructure (2026-10-18):* VMs are now made by PR to `infra/tofu`
-Terraform isn't running; the operator is planning both, Terraform with
+(OpenTofu), and the agent host exists (M6.1), holding only the vault and
-`nisse-ops`. Until then M5.4 and M6.1–M6.4 wait, and the build VMs stay made by
+SilverBullet so far (M5.4). Until M6.2, the build VMs stay made by hand, with
-hand (Ubuntu 24 now, 26 soon), with `vm ready` for a rebuilt one's host key.
+`vm ready` for a rebuilt one's host key.
 
 **M6.1 The agent host.** *Needs discussion.* Containers, volumes, snapshots,
 per-agent networks (X1, X2, X4). Each agent runs there from its own published
-image (§3).
+image (§3). The host exists: `nisse` (infra/tofu#77, 2026-10-18, as
+`nisse-host`; 2 cores, 4 GB, 100 GB, on VLAN 1, nightly Proxmox backups),
+requested through nisse-dev's own quota to test that path; the operator has
+since renamed it and moved it out of the quota. It holds only the vault and
+SilverBullet so far (M5.4).
 
 **M6.2 Build VMs by Terraform.** *Needs discussion.* The template, VLAN and egress
 (D7, X2), collection when idle.

scripts/agent-host.sh

@@ -0,0 +1,39 @@
+#!/usr/bin/env bash
+# The agent host's vault and SilverBullet (deploy/agent-host), from this box.
+#   scripts/agent-host.sh install   copy deploy/agent-host over and run setup.sh
+#   scripts/agent-host.sh check     check SilverBullet runs and the mounts and
+#                                   firewall hold
+#   scripts/agent-host.sh ssh ...   run a command on the host
+# Trusts only host keys from a trusted source: bot-ansible's comment
+# (~/bin/tofu-wait.sh saves them) or the operator, read off the console.
+set -euo pipefail
+
+host=${AGENT_HOST:-nisse.garrett-hall.com}
+key=${AGENT_HOST_KEY:-/root/nisse-e2e/ssh/id_ed25519}
+known=${AGENT_HOST_KNOWN:-/root/nisse-e2e/nisse.known_hosts}
+repo=$(cd "$(dirname "$0")/.." && pwd)
+
+on_host() {
+  ssh -i "$key" -o UserKnownHostsFile="$known" -o StrictHostKeyChecking=yes \
+    -o BatchMode=yes -o ConnectTimeout=10 "ubuntu@$host" "$@"
+}
+
+case ${1:-} in
+  install)
+    tar -C "$repo/deploy" -cf - agent-host |
+      on_host 'rm -rf /tmp/agent-host && tar -C /tmp -xf - && sudo bash /tmp/agent-host/setup.sh'
+    ;;
+  check)
+    # As an argument, not on stdin: docker compose exec reads stdin, and would
+    # swallow the rest of a script fed that way.
+    on_host "bash -c $(printf '%q' "$(cat "$repo/deploy/agent-host/check.sh")")"
+    ;;
+  ssh)
+    shift
+    on_host "$@"
+    ;;
+  *)
+    sed -n '2,7p' "$0"
+    exit 2
+    ;;
+esac

tests/deploy/test_agent_host.py

@@ -0,0 +1,212 @@
+"""The agent host's vault and SilverBullet (deploy/agent-host; L10, M5.4):
+the operator writes the vault, never the agents' memory; nothing but the
+vault and SilverBullet's own data is mounted; the image changes only in a
+PR; only the operator's Caddy reaches it."""
+
+import re
+import shutil
+import subprocess
+from pathlib import Path
+from typing import Any
+
+import pytest
+import yaml
+
+HOST = Path(__file__).resolve().parents[2] / "deploy" / "agent-host"
+VAULT = "/srv/nisse/vault"
+
+
+def silverbullet() -> dict[str, Any]:
+    compose = yaml.safe_load((HOST / "silverbullet" / "compose.yaml").read_text())
+    return compose["services"]["silverbullet"]
+
+
+def mounts() -> list[tuple[str, str, str]]:
+    out = []
+    for volume in silverbullet()["volumes"]:
+        source, target, *mode = volume.split(":")
+        out.append((source, target, mode[0] if mode else "rw"))
+    return out
+
+
+def setting(name: str) -> str:
+    match = re.search(rf"^{name}=(\S+)", (HOST / "setup.sh").read_text(), re.MULTILINE)
+    assert match, name
+    return match.group(1)
+
+
+def test_the_agents_memory_is_read_only_and_the_rest_of_the_vault_is_not() -> None:
+    by_source = {source: (target, mode) for source, target, mode in mounts()}
+    vault_target, vault_mode = by_source[VAULT]
+    assert vault_mode == "rw"
+    # memory/ is mounted over the same place in the space, read-only.
+    assert by_source[f"{VAULT}/memory"] == (f"{vault_target}/memory", "ro")
+    # Nothing else of the vault is mounted, so nothing gets around that.
+    others = [s for s in by_source if s.startswith(VAULT)]
+    assert sorted(others) == [VAULT, f"{VAULT}/memory"]
+
+
+def test_it_sees_nothing_of_the_host_but_the_vault() -> None:
+    assert sorted(source for source, _, _ in mounts()) == [VAULT, f"{VAULT}/memory"]
+
+
+def test_the_vault_is_its_space() -> None:
+    # One space, the vault, at the data folder: no accounts or settings of its
+    # own outside the vault.
+    assert (VAULT, "/data", "rw") in mounts()
+
+
+def test_the_image_is_pinned_to_a_release() -> None:
+    # The docker role updates images nightly; a pinned release changes only here.
+    image = silverbullet()["image"]
+    assert re.fullmatch(
+        r"ghcr\.io/silverbulletmd/silverbullet:\d+\.\d+\.\d+(-slim)?", image
+    )
+
+
+def test_no_secrets_in_the_compose_file() -> None:
+    environment = silverbullet().get("environment", {})
+    assert not [k for k in environment if re.search(r"USER|TOKEN|PASS|SECRET|KEY", k)]
+
+
+def test_it_boots_one_space_never_the_setup_wizard() -> None:
+    # Any SB_* setting forces 2.11's single-space mode. Without one, an empty
+    # vault would boot the setup wizard, whose first visitor makes the admin.
+    environment = silverbullet()["environment"]
+    assert any(name.startswith("SB_") for name in environment)
+    # No credentials: the operator logs in through Authelia in front.
+    assert not {"SB_USER", "SB_AUTH"} & set(environment)
+
+
+def test_its_port_caddys_and_the_firewalls_agree() -> None:
+    port = setting("SB_PORT")
+    assert silverbullet()["ports"] == [f"0.0.0.0:{port}:3000"]  # IPv4 only
+    caddy = (HOST / "caddy" / "nisse-memory.dongar.net.caddy").read_text()
+    assert f"reverse_proxy nisse.garrett-hall.com:{port}" in caddy
+    assert "@outside not remote_ip private_ranges" in caddy  # the LAN only
+
+
+def test_logs_go_to_the_hosts_journal() -> None:
+    assert silverbullet()["logging"] == {"driver": "journald"}
+
+
+def _can_use_a_network_namespace() -> bool:
+    if not (shutil.which("unshare") and shutil.which("iptables")):
+        return False
+    probe = subprocess.run(
+        ["unshare", "-n", "iptables", "-S"], capture_output=True, check=False
+    )
+    return probe.returncode == 0
+
+
+@pytest.mark.skipif(
+    not _can_use_a_network_namespace(),
+    reason="needs root, unshare and iptables to try the rule in a namespace",
+)
+def test_the_firewall_lets_only_one_address_in_and_replaces_its_own_rule() -> None:
+    script = HOST / "nisse-firewall"
+    steps = (
+        "iptables -N DOCKER-USER; iptables -A DOCKER-USER -j RETURN; "
+        f"{script} 192.168.6.13 3000; {script} 192.168.6.13 3000; "
+        f"{script} 192.168.6.14 3000; iptables -S DOCKER-USER"
+    )
+    run = subprocess.run(
+        ["unshare", "-n", "sh", "-ec", steps],
+        capture_output=True,
+        text=True,
+        check=True,
+    )
+    rules = [r for r in run.stdout.splitlines() if "nisse-firewall" in r]
+    assert len(rules) == 1, run.stdout  # run again: still one; changed: replaced
+    [rule] = rules
+    assert "! -s 192.168.6.14/32" in rule
+    assert "--ctorigdstport 3000" in rule
+    assert rule.endswith("-j DROP")
+    # Ahead of Docker's own return, so it's checked first.
+    assert run.stdout.splitlines().index(rule) < run.stdout.splitlines().index(
+        "-A DOCKER-USER -j RETURN"
+    )
+
+
+SCRIPT = HOST.parents[1] / "scripts" / "agent-host.sh"
+
+
+def _fake_host(
+    tmp_path: Path,
+    *,
+    memory_writable: bool,
+    boot: str = "single-space (folder is not empty)",
+) -> dict[str, str]:
+    """Commands standing in for the host's: ssh runs the command here, sudo
+    just runs it, and docker reads all of stdin, as docker compose exec does."""
+    bin_ = tmp_path / "bin"
+    bin_.mkdir()
+    stubs = {
+        "ssh": 'for last; do :; done; exec bash -c "$last"',
+        "sudo": 'exec "$@"',
+        "curl": "echo 200",
+        "iptables": "echo '-A DOCKER-USER -m comment --comment nisse-firewall -j DROP'",
+        "docker": (
+            "cat > /dev/null\n"
+            'case "$*" in\n'
+            '  *"ps --status running"*) echo abc123 ;;\n'
+            f'  *" logs "*) echo "boot mode: {boot}" ;;\n'
+            f'  *"/data/memory/"*) exit {0 if memory_writable else 1} ;;\n'
+            "esac"
+        ),
+    }
+    for name, body in stubs.items():
+        path = bin_ / name
+        path.write_text(f"#!/usr/bin/env bash\n{body}\n")
+        path.chmod(0o755)
+    stack = tmp_path / "stack"
+    stack.mkdir()
+    return {"PATH": f"{bin_}:/usr/bin:/bin", "STACK": str(stack), "HOME": str(tmp_path)}
+
+
+def _check(env: dict[str, str]) -> subprocess.CompletedProcess[str]:
+    return subprocess.run(
+        ["bash", str(SCRIPT), "check"],
+        env=env,
+        capture_output=True,
+        text=True,
+        timeout=30,
+        check=False,
+    )
+
+
+def test_the_check_runs_every_step_even_though_docker_reads_stdin(
+    tmp_path: Path,
+) -> None:
+    # Fed to the host on stdin, the first docker compose exec swallowed the
+    # rest of the check, which then passed without the last two steps.
+    result = _check(_fake_host(tmp_path, memory_writable=False))
+    assert result.returncode == 0, result.stdout + result.stderr
+    assert result.stdout.splitlines() == [
+        "ok:   SilverBullet is running",
+        "ok:   it serves the vault as one space (no setup wizard)",
+        "ok:   it answers locally (200)",
+        "ok:   the operator's folders are writable",
+        "ok:   memory/ is read-only to SilverBullet",
+        "ok:   the firewall rule is in place",
+        "all checks passed",
+    ]
+
+
+def test_a_writable_memory_folder_fails_the_check(tmp_path: Path) -> None:
+    result = _check(_fake_host(tmp_path, memory_writable=True))
+    assert result.returncode == 1
+    assert (
+        "FAIL: memory/ is writable from SilverBullet, even as root in it"
+        in result.stdout
+    )
+    assert "ok:   the firewall rule is in place" in result.stdout  # still checked
+    assert "all checks passed" not in result.stdout
+
+
+def test_a_setup_wizard_boot_fails_the_check(tmp_path: Path) -> None:
+    result = _check(_fake_host(tmp_path, memory_writable=False, boot="unconfigured"))
+    assert result.returncode == 1
+    assert (
+        "FAIL: it didn't boot one space: the setup wizard may be open" in result.stdout
+    )