Agent host check: run every step, and say so

The check went to the host on stdin, and the first docker compose exec read
the rest of it: the memory and firewall checks never ran, and it passed.
It is now its own file, sent as an argument, and ends with "all checks
passed" only when every check has run. Tests fake the host with a docker
that reads stdin.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01J5Utyfzw7gF7UhjRDEtjTk

deploy/agent-host/README.md

@@ -14,6 +14,8 @@ later (M6.1).
   `/opt/stacks/silverbullet` and the firewall, and starts it. Safe to run again.
 - `nisse-firewall`: only the operator's Caddy may reach SilverBullet's port
   (Docker's `DOCKER-USER` chain, re-applied whenever Docker starts).
+- `check.sh`: run on the host by `scripts/agent-host.sh check`; says "all
+  checks passed" only once every check has run.
 - `caddy/vault.dongar.net.caddy`: the site for the operator's Caddy on
   `docker.garrett-hall.com`. HTTPS from Caddy's usual certificate, LAN only.
 

deploy/agent-host/check.sh

@@ -0,0 +1,28 @@
+#!/usr/bin/env bash
+# Checks the vault and SilverBullet on the agent host; run there as a user with
+# sudo (scripts/agent-host.sh check sends it). Exits 1 if any check fails, and
+# only says "all checks passed" once every one has run.
+set -u
+stack=${STACK:-/opt/stacks/silverbullet}
+fail=0
+ok() { echo "ok:   $1"; }
+bad() { echo "FAIL: $1"; fail=1; }
+cd "$stack" || { echo "FAIL: no $stack"; exit 1; }
+[ "$(sudo docker compose ps --status running -q | wc -l)" -ge 1 ] \
+  && ok "SilverBullet is running" || bad "SilverBullet isn't running"
+code=$(curl -s -o /dev/null -w '%{http_code}' http://127.0.0.1:3000/)
+case $code in 2??|3??) ok "it answers locally ($code)" ;; *) bad "it answers $code" ;; esac
+probe=.nisse-probe-$$
+sudo docker compose exec -T -u 1000 silverbullet \
+  sh -c "touch /vault/infrastructure/$probe && rm /vault/infrastructure/$probe" \
+  && ok "the operator's folders are writable" || bad "the operator's folders aren't writable"
+if sudo docker compose exec -T silverbullet sh -c "touch /vault/memory/$probe" 2>/dev/null; then
+  sudo docker compose exec -T silverbullet rm -f "/vault/memory/$probe"
+  bad "memory/ is writable from SilverBullet, even as root in it"
+else
+  ok "memory/ is read-only to SilverBullet"
+fi
+sudo iptables -S DOCKER-USER | grep -q -- '--comment nisse-firewall' \
+  && ok "the firewall rule is in place" || bad "no firewall rule"
+[ $fail = 0 ] && echo "all checks passed"
+exit $fail

scripts/agent-host.sh

@@ -4,7 +4,8 @@
 #   scripts/agent-host.sh check     check SilverBullet runs and the mounts and
 #                                   firewall hold
 #   scripts/agent-host.sh ssh ...   run a command on the host
-# Trusts only the host keys bot-ansible posted (~/bin/tofu-wait.sh saved them).
+# Trusts only host keys from a trusted source: bot-ansible's comment
+# (~/bin/tofu-wait.sh saves them) or the operator, read off the console.
 set -euo pipefail
 
 host=${AGENT_HOST:-nisse.garrett-hall.com}
@@ -23,29 +24,9 @@ case ${1:-} in
       on_host 'rm -rf /tmp/agent-host && tar -C /tmp -xf - && sudo bash /tmp/agent-host/setup.sh'
     ;;
   check)
-    on_host 'bash -s' <<'EOF'
-set -u
-fail=0
-ok() { echo "ok:   $1"; }
-bad() { echo "FAIL: $1"; fail=1; }
-cd /opt/stacks/silverbullet
-[ "$(sudo docker compose ps --status running -q | wc -l)" -ge 1 ] \
-  && ok "SilverBullet is running" || bad "SilverBullet isn't running"
-code=$(curl -s -o /dev/null -w '%{http_code}' http://127.0.0.1:3000/)
-case $code in 2??|3??) ok "it answers locally ($code)" ;; *) bad "it answers $code" ;; esac
-probe=.nisse-probe-$$
-sudo docker compose exec -T -u 1000 silverbullet sh -c "touch /vault/infrastructure/$probe && rm /vault/infrastructure/$probe" \
-  && ok "the operator's folders are writable" || bad "the operator's folders aren't writable"
-if sudo docker compose exec -T silverbullet sh -c "touch /vault/memory/$probe" 2>/dev/null; then
-  sudo docker compose exec -T silverbullet rm -f "/vault/memory/$probe"
-  bad "memory/ is writable from SilverBullet, even as root in it"
-else
-  ok "memory/ is read-only to SilverBullet"
-fi
-sudo iptables -S DOCKER-USER | grep -q -- '--comment nisse-firewall' \
-  && ok "the firewall rule is in place" || bad "no firewall rule"
-exit $fail
-EOF
+    # As an argument, not on stdin: docker compose exec reads stdin, and would
+    # swallow the rest of a script fed that way.
+    on_host "bash -c $(printf '%q' "$(cat "$repo/deploy/agent-host/check.sh")")"
     ;;
   ssh)
     shift

tests/deploy/test_agent_host.py

@@ -115,3 +115,72 @@ def test_the_firewall_lets_only_one_address_in_and_replaces_its_own_rule() -> No
     assert run.stdout.splitlines().index(rule) < run.stdout.splitlines().index(
         "-A DOCKER-USER -j RETURN"
     )
+
+
+SCRIPT = HOST.parents[1] / "scripts" / "agent-host.sh"
+
+
+def _fake_host(tmp_path: Path, *, memory_writable: bool) -> dict[str, str]:
+    """Commands standing in for the host's: ssh runs the command here, sudo
+    just runs it, and docker reads all of stdin, as docker compose exec does."""
+    bin_ = tmp_path / "bin"
+    bin_.mkdir()
+    stubs = {
+        "ssh": 'for last; do :; done; exec bash -c "$last"',
+        "sudo": 'exec "$@"',
+        "curl": "echo 200",
+        "iptables": "echo '-A DOCKER-USER -m comment --comment nisse-firewall -j DROP'",
+        "docker": (
+            "cat > /dev/null\n"
+            'case "$*" in\n'
+            '  *"ps --status running"*) echo abc123 ;;\n'
+            f'  *"/vault/memory/"*) exit {0 if memory_writable else 1} ;;\n'
+            "esac"
+        ),
+    }
+    for name, body in stubs.items():
+        path = bin_ / name
+        path.write_text(f"#!/usr/bin/env bash\n{body}\n")
+        path.chmod(0o755)
+    stack = tmp_path / "stack"
+    stack.mkdir()
+    return {"PATH": f"{bin_}:/usr/bin:/bin", "STACK": str(stack), "HOME": str(tmp_path)}
+
+
+def _check(env: dict[str, str]) -> subprocess.CompletedProcess[str]:
+    return subprocess.run(
+        ["bash", str(SCRIPT), "check"],
+        env=env,
+        capture_output=True,
+        text=True,
+        timeout=30,
+        check=False,
+    )
+
+
+def test_the_check_runs_every_step_even_though_docker_reads_stdin(
+    tmp_path: Path,
+) -> None:
+    # Fed to the host on stdin, the first docker compose exec swallowed the
+    # rest of the check, which then passed without the last two steps.
+    result = _check(_fake_host(tmp_path, memory_writable=False))
+    assert result.returncode == 0, result.stdout + result.stderr
+    assert result.stdout.splitlines() == [
+        "ok:   SilverBullet is running",
+        "ok:   it answers locally (200)",
+        "ok:   the operator's folders are writable",
+        "ok:   memory/ is read-only to SilverBullet",
+        "ok:   the firewall rule is in place",
+        "all checks passed",
+    ]
+
+
+def test_a_writable_memory_folder_fails_the_check(tmp_path: Path) -> None:
+    result = _check(_fake_host(tmp_path, memory_writable=True))
+    assert result.returncode == 1
+    assert (
+        "FAIL: memory/ is writable from SilverBullet, even as root in it"
+        in result.stdout
+    )
+    assert "ok:   the firewall rule is in place" in result.stdout  # still checked
+    assert "all checks passed" not in result.stdout