Add the delegation harness (A13) — step 7b
A frontier model receives a scoped sub-task, works autonomously with full tool access, and hands back; the local loop reads the handback note and the file state, never the reasoning. - Branch at delegation start, and abort discards it. That required committing each turn's work onto the branch: without commits, deleting a branch discards nothing, so "abort discards the work" was a comforting phrase rather than a fact. The caller's own in-flight changes are committed to their branch first, so an abort cannot destroy work nobody delegated - Hard caps on turns and wall clock; a breach is a defined failure return, and its result is not usable, so the caller gates rather than resuming on a half-edited tree - Every event is checkpointed as it arrives, so a crash mid-delegation leaves the branch at a known commit and a truthful transcript - Handback is validated locally: blank fields or a non-boolean success flag are refused and the delegation continues rather than accepting a useless note Two failure modes found by running it against a real model: - A success claim with nothing changed was accepted. The handback plus file state is the entire interface back, so a note describing work that does not exist is worse than no note. Success now requires a successful mutating tool call; reporting failure still does not, since "this is impossible" is a legitimate finding - The delegate invented the absolute path /workspace, wrote there, ran its tests there, and handed back truthfully — with the repo empty and git seeing nothing. Writes outside the workspace are now refused with a message naming the real root, and both prompts state where the workspace is instead of leaving the model to guess. This is not sandboxing: work outside the repo is invisible to git, so the milestone would publish an empty PR Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
1 parent
1c8e7e35fc
commit
53ed11f9c4
7 files changed
+824
-7
No files matched your search
@@ -0,0 +1,89 @@
|
||||
"""A real delegation over the Anthropic passthrough: scoped sub-task in,
|
||||
working code and a structured handback out."""
|
||||
|
||||
import io
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from nisse_dev.delegation import Caps, Delegation
|
||||
from nisse_dev.envfile import load_env_file
|
||||
from nisse_dev.git import GitRepo
|
||||
from nisse_dev.llm import AnthropicPassthroughClient
|
||||
from nisse_dev.oplog import OpLog
|
||||
from nisse_dev.state import StateStore
|
||||
from nisse_dev.tools import Workspace
|
||||
|
||||
load_env_file(Path(__file__).resolve().parents[2] / ".env")
|
||||
|
||||
SENTINEL = "NISSE_LIVE_PLANNER_MODEL"
|
||||
|
||||
pytestmark = [
|
||||
pytest.mark.integration,
|
||||
pytest.mark.skipif(
|
||||
not os.environ.get(SENTINEL),
|
||||
reason=f"no live frontier route configured ({SENTINEL} unset)",
|
||||
),
|
||||
]
|
||||
|
||||
BRIEF = """Create `slugify.py` in the workspace root containing a function
|
||||
`slugify(text)` that lowercases the text and replaces every run of
|
||||
non-alphanumeric characters with a single hyphen, with no leading or trailing
|
||||
hyphen. Then create `test_slugify.py` with at least two assertions covering
|
||||
it, and run `python -m pytest test_slugify.py -q` until it passes."""
|
||||
|
||||
|
||||
def build(tmp_path: Path, caps: Caps) -> tuple[Delegation, GitRepo, StateStore]:
|
||||
repo = GitRepo(tmp_path / "work")
|
||||
repo.init()
|
||||
repo.configure_identity("nisse-dev-testing", "n@t.test")
|
||||
(repo.root / "README.md").write_text("sandbox\n")
|
||||
repo.commit_all("seed")
|
||||
store = StateStore(tmp_path / "state.db", oplog=OpLog("nisse-dev", io.StringIO()))
|
||||
delegation = Delegation(
|
||||
AnthropicPassthroughClient.from_env(),
|
||||
store,
|
||||
Workspace(repo.root, timeout_s=60),
|
||||
repo,
|
||||
model=os.environ[SENTINEL],
|
||||
caps=caps,
|
||||
)
|
||||
return delegation, repo, store
|
||||
|
||||
|
||||
def test_delegate_completes_a_scoped_subtask(tmp_path: Path) -> None:
|
||||
delegation, repo, _store = build(tmp_path, Caps(turns=12, wall_clock_s=300.0))
|
||||
|
||||
result = delegation.run(brief=BRIEF, branch="live/delegation-1")
|
||||
|
||||
assert result.usable, f"delegation did not hand back: {result.status} {result.note}"
|
||||
assert (repo.root / "slugify.py").exists()
|
||||
|
||||
# The handback is the entire interface back, so it must actually say
|
||||
# something in each part rather than be structurally present but empty.
|
||||
handback = result.handback
|
||||
assert handback is not None
|
||||
assert len(handback.changed) > 10
|
||||
assert len(handback.now_true) > 10
|
||||
assert len(handback.next) > 5
|
||||
|
||||
# The delegate's own work is on its branch, and it really runs.
|
||||
assert repo.current_branch() == "live/delegation-1"
|
||||
check = Workspace(repo.root, timeout_s=60).run_command(
|
||||
"python -m pytest test_slugify.py -q"
|
||||
)
|
||||
assert check.success, f"delegate's own tests do not pass: {check.result[:400]}"
|
||||
|
||||
|
||||
def test_a_cap_breach_discards_the_work(tmp_path: Path) -> None:
|
||||
# One turn cannot finish the brief, so this exercises the abort path
|
||||
# against a real model rather than a scripted one.
|
||||
delegation, repo, _ = build(tmp_path, Caps(turns=1, wall_clock_s=120.0))
|
||||
|
||||
result = delegation.run(brief=BRIEF, branch="live/delegation-2")
|
||||
|
||||
assert not result.usable
|
||||
assert result.status == "capped"
|
||||
assert repo.current_branch() == "main"
|
||||
assert not (repo.root / "slugify.py").exists()
|
||||
Reference in new issue
Block a user