dongarandClaude Opus 5.5
abb7fa0216
CI / checks (push) Skipped
CI / checks (pull_request) Successful in 49s
`stop` could not have worked by adding the command alone. Planning turns, milestones and review responses ran synchronously on the thread that delivered the message, so the Matrix sync loop read nothing until the work finished. The state store was also opened on the main thread and used from the transport and poller threads, which SQLite refuses with its default same-thread check; the in-memory test transport never showed it. A single worker thread now runs every handler and owns the store: - The worker opens the store, so the same-thread check stays on and fails loudly if anything touches the store from elsewhere - Matrix messages are handed to the worker; pollers hand over and wait, so their cursor never passes an unhandled comment - A message that arrives during long work gets an immediate "busy; say 'stop' to halt" reply and is recorded, not applied later in a state the operator was not answering - Tests keep running inline through InlineWorker, which defers a job submitted from inside another as the real worker would queue it `stop` sets a shared Cancel flag straight from the transport's thread. The executor and delegation loops check it before each model call and between tool calls; the planner checks it before each turn. Calls left unrun get a result saying so, so the transcript still renders. A stopped delegation is discarded like a failed one. The orchestrator moves to AWAIT_INPUT and asks why the work was stopped. The reply is recorded for the model, and a stopped planning turn resumes with it as the answer. `stop` with nothing running says so and changes nothing; at a gate it is no longer taken as rejection feedback. Closes #40 Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
691 lines
23 KiB
Python
691 lines
23 KiB
Python
import io
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
import pytest
|
|
|
|
from nisse_dev.cancel import STOPPED_RESULT, Cancel
|
|
from nisse_dev.canonical import (
|
|
AssistantText,
|
|
CanonicalEvent,
|
|
DelegationCompleted,
|
|
DelegationStarted,
|
|
StoredEvent,
|
|
ToolCall,
|
|
ToolResult,
|
|
)
|
|
from nisse_dev.delegation import (
|
|
DELEGATE_TOOLS,
|
|
DELEGATION_INTENT,
|
|
Caps,
|
|
Delegation,
|
|
delegation_reconciler,
|
|
)
|
|
from nisse_dev.git import GitRepo
|
|
from nisse_dev.llm import BudgetExhausted, LLMError, UpstreamUnavailable
|
|
from nisse_dev.oplog import OpLog
|
|
from nisse_dev.state import StateStore
|
|
from nisse_dev.tools import Workspace
|
|
|
|
|
|
class ScriptedClient:
|
|
def __init__(self, turns: list[list[CanonicalEvent]]) -> None:
|
|
self._turns = turns
|
|
self.calls: list[dict[str, Any]] = []
|
|
|
|
def complete(
|
|
self,
|
|
events: list[StoredEvent],
|
|
tools: list[dict[str, Any]] | None,
|
|
model: str,
|
|
system: str | None = None,
|
|
) -> list[CanonicalEvent]:
|
|
self.calls.append({"tools": tools, "model": model, "system": system})
|
|
if not self._turns:
|
|
return [AssistantText(text="still thinking", model=model)]
|
|
return self._turns.pop(0)
|
|
|
|
|
|
class FailingClient:
|
|
def complete(self, *args: Any, **kwargs: Any) -> list[CanonicalEvent]:
|
|
raise LLMError("frontier unavailable")
|
|
|
|
|
|
def call(tool: str, cid: str = "c1", **arguments: Any) -> ToolCall:
|
|
return ToolCall(call_id=cid, tool=tool, arguments=arguments)
|
|
|
|
|
|
def hand_back(succeeded: bool = True, cid: str = "hb", **overrides: Any) -> ToolCall:
|
|
args = {
|
|
"changed": "added greet.py",
|
|
"now_true": "greet() returns a greeting and its test passes",
|
|
"next": "open the PR",
|
|
"succeeded": succeeded,
|
|
}
|
|
args.update(overrides)
|
|
return ToolCall(call_id=cid, tool="hand_back", arguments=args)
|
|
|
|
|
|
@pytest.fixture
|
|
def store(tmp_path: Path) -> StateStore:
|
|
return StateStore(tmp_path / "state.db", oplog=OpLog("nisse-dev", io.StringIO()))
|
|
|
|
|
|
@pytest.fixture
|
|
def repo(tmp_path: Path) -> GitRepo:
|
|
r = GitRepo(tmp_path / "work")
|
|
r.init()
|
|
r.configure_identity("bot", "bot@example.test")
|
|
(r.root / "seed.txt").write_text("seed")
|
|
r.commit_all("seed")
|
|
return r
|
|
|
|
|
|
@pytest.fixture
|
|
def ws(repo: GitRepo) -> Workspace:
|
|
return Workspace(repo.root, timeout_s=10)
|
|
|
|
|
|
def build(client: Any, store: StateStore, ws: Workspace, repo: GitRepo, **kw: Any):
|
|
return Delegation(client, store, ws, repo, model="claude-haiku-4-5", **kw)
|
|
|
|
|
|
def test_successful_delegation_returns_a_structured_handback(
|
|
store: StateStore, ws: Workspace, repo: GitRepo
|
|
) -> None:
|
|
client = ScriptedClient(
|
|
[
|
|
[call("write_file", path="greet.py", content="def greet(): ...")],
|
|
[hand_back()],
|
|
]
|
|
)
|
|
result = build(client, store, ws, repo).run(brief="add greet", branch="d1")
|
|
|
|
assert result.usable
|
|
assert result.handback is not None
|
|
assert "greet.py" in result.handback.changed
|
|
assert "what to do next" in result.handback.render().lower()
|
|
assert result.artifacts == ["greet.py"]
|
|
|
|
|
|
def test_delegation_works_on_its_own_branch(
|
|
store: StateStore, ws: Workspace, repo: GitRepo
|
|
) -> None:
|
|
client = ScriptedClient(
|
|
[[call("write_file", path="a.py", content="x")], [hand_back()]]
|
|
)
|
|
build(client, store, ws, repo).run(brief="b", branch="deleg-1")
|
|
assert repo.current_branch() == "deleg-1"
|
|
|
|
|
|
def test_abort_discards_the_branch_and_its_work(
|
|
store: StateStore, ws: Workspace, repo: GitRepo
|
|
) -> None:
|
|
# A13: abort must leave nothing to resume on top of.
|
|
client = ScriptedClient(
|
|
[
|
|
[call("write_file", path="half-done.py", content="oops")],
|
|
[hand_back(succeeded=False, changed="could not finish")],
|
|
]
|
|
)
|
|
result = build(client, store, ws, repo).run(brief="b", branch="deleg-2")
|
|
|
|
assert not result.usable
|
|
assert repo.current_branch() == "main"
|
|
assert not (repo.root / "half-done.py").exists()
|
|
|
|
|
|
def test_stop_discards_the_branch_and_settles_the_intent(
|
|
store: StateStore, ws: Workspace, repo: GitRepo
|
|
) -> None:
|
|
# A4 stop is treated like a failure, not like an outage: the operator
|
|
# stopped it for a reason, so nothing is kept for resuming.
|
|
cancel = Cancel()
|
|
client = ScriptedClient(
|
|
[
|
|
[call("write_file", path="half-done.py", content="oops")],
|
|
[call("list_dir", path=".")],
|
|
]
|
|
)
|
|
original = client.complete
|
|
|
|
def complete_then_stop(*args: Any, **kwargs: Any) -> list[CanonicalEvent]:
|
|
produced = original(*args, **kwargs)
|
|
if len(client.calls) == 2:
|
|
cancel.set() # the operator says stop during turn 2's model call
|
|
return produced
|
|
|
|
client.complete = complete_then_stop # type: ignore[method-assign]
|
|
result = build(client, store, ws, repo, cancel=cancel).run(
|
|
brief="b", branch="deleg-stop"
|
|
)
|
|
|
|
assert result.status == "stopped"
|
|
assert not result.usable
|
|
assert repo.current_branch() == "main"
|
|
assert not repo.branch_exists("deleg-stop")
|
|
assert not (repo.root / "half-done.py").exists()
|
|
assert store.pending_intents() == []
|
|
# The unrun call still has a result, so the transcript renders.
|
|
skipped = [
|
|
e.event
|
|
for e in store.read_events()
|
|
if isinstance(e.event, ToolResult) and e.event.result == STOPPED_RESULT
|
|
]
|
|
assert len(skipped) == 1
|
|
|
|
|
|
def test_turn_cap_is_hard(store: StateStore, ws: Workspace, repo: GitRepo) -> None:
|
|
forever = [[call("list_dir", path=".")] for _ in range(20)]
|
|
client = ScriptedClient(forever)
|
|
result = build(client, store, ws, repo, caps=Caps(turns=3)).run(
|
|
brief="b", branch="d"
|
|
)
|
|
|
|
assert result.status == "capped"
|
|
assert "turn cap of 3" in result.note
|
|
assert len(client.calls) == 3
|
|
|
|
|
|
def test_wall_clock_cap_is_hard(
|
|
store: StateStore, ws: Workspace, repo: GitRepo
|
|
) -> None:
|
|
ticks = iter([0.0, 1.0, 99.0, 99.0, 99.0])
|
|
client = ScriptedClient([[call("list_dir", path=".")] for _ in range(10)])
|
|
result = build(
|
|
client,
|
|
store,
|
|
ws,
|
|
repo,
|
|
caps=Caps(turns=10, wall_clock_s=60.0),
|
|
now=lambda: next(ticks),
|
|
).run(brief="b", branch="d")
|
|
|
|
assert result.status == "capped"
|
|
assert "wall-clock cap" in result.note
|
|
|
|
|
|
def test_cap_breach_is_not_usable_and_discards_work(
|
|
store: StateStore, ws: Workspace, repo: GitRepo
|
|
) -> None:
|
|
client = ScriptedClient([[call("write_file", path="partial.py", content="x")]] * 5)
|
|
result = build(client, store, ws, repo, caps=Caps(turns=2)).run(
|
|
brief="b", branch="d"
|
|
)
|
|
|
|
assert not result.usable # the caller must gate, not resume (A13)
|
|
assert not (repo.root / "partial.py").exists()
|
|
|
|
|
|
def test_events_are_checkpointed_as_they_arrive(
|
|
store: StateStore, ws: Workspace, repo: GitRepo
|
|
) -> None:
|
|
client = ScriptedClient(
|
|
[[call("write_file", path="a.py", content="x")], [hand_back()]]
|
|
)
|
|
build(client, store, ws, repo).run(brief="add a", branch="d")
|
|
|
|
kinds = [type(e.event).__name__ for e in store.read_events()]
|
|
assert kinds == [
|
|
"DelegationStarted",
|
|
"ToolCall",
|
|
"ToolResult",
|
|
"ToolCall", # hand_back
|
|
"ToolResult",
|
|
"DelegationCompleted",
|
|
]
|
|
|
|
|
|
def test_composite_events_record_the_brief_caps_and_outcome(
|
|
store: StateStore, ws: Workspace, repo: GitRepo
|
|
) -> None:
|
|
client = ScriptedClient(
|
|
[[call("write_file", path="a.py", content="x")], [hand_back()]]
|
|
)
|
|
caps = Caps(turns=7, tokens=1234, wall_clock_s=42.0)
|
|
build(client, store, ws, repo, caps=caps).run(brief="the brief", branch="d")
|
|
|
|
events = [e.event for e in store.read_events()]
|
|
started = next(e for e in events if isinstance(e, DelegationStarted))
|
|
completed = next(e for e in events if isinstance(e, DelegationCompleted))
|
|
assert started.brief == "the brief"
|
|
assert started.model == "claude-haiku-4-5"
|
|
assert started.caps == {"turns": 7, "tokens": 1234, "wall_clock_s": 42.0}
|
|
# A16: this pair is the eval record, so it must carry the outcome.
|
|
assert completed.exit_status == "handback"
|
|
assert "greet()" in completed.handback
|
|
|
|
|
|
def test_hand_back_tool_is_offered_alongside_the_executor_tools() -> None:
|
|
names = [t["function"]["name"] for t in DELEGATE_TOOLS]
|
|
assert names[-1] == "hand_back"
|
|
assert "run_command" in names # full tool access, not a consult (A13)
|
|
|
|
|
|
def test_malformed_handback_is_rejected_and_the_delegation_continues(
|
|
store: StateStore, ws: Workspace, repo: GitRepo
|
|
) -> None:
|
|
client = ScriptedClient(
|
|
[
|
|
[hand_back(cid="bad", changed=" ")], # blank field
|
|
[call("write_file", path="a.py", content="x")],
|
|
[hand_back(cid="good")],
|
|
]
|
|
)
|
|
result = build(client, store, ws, repo).run(brief="b", branch="d")
|
|
|
|
assert result.usable # recovered rather than accepting a useless note
|
|
assert len(client.calls) == 3
|
|
|
|
|
|
def test_non_boolean_success_flag_is_rejected(
|
|
store: StateStore, ws: Workspace, repo: GitRepo
|
|
) -> None:
|
|
client = ScriptedClient(
|
|
[
|
|
[call("write_file", path="a.py", content="x")],
|
|
[hand_back(succeeded="yes")], # not a bool: unparseable
|
|
[hand_back(cid="ok")],
|
|
]
|
|
)
|
|
result = build(client, store, ws, repo).run(brief="b", branch="d")
|
|
assert result.usable
|
|
assert len(client.calls) == 3
|
|
|
|
|
|
def test_client_failure_is_a_defined_return_not_a_crash(
|
|
store: StateStore, ws: Workspace, repo: GitRepo
|
|
) -> None:
|
|
result = build(FailingClient(), store, ws, repo).run(brief="b", branch="d")
|
|
assert result.status == "failed"
|
|
assert "frontier unavailable" in result.note
|
|
assert not result.usable
|
|
|
|
|
|
def test_a_delegate_that_never_hands_back_is_capped_not_believed(
|
|
store: StateStore, ws: Workspace, repo: GitRepo
|
|
) -> None:
|
|
# Prose forever: the delegate never reports, so its tree state is unknown.
|
|
client = ScriptedClient([])
|
|
result = build(client, store, ws, repo, caps=Caps(turns=3)).run(
|
|
brief="b", branch="d"
|
|
)
|
|
assert result.status == "capped"
|
|
assert not result.usable
|
|
|
|
|
|
def test_success_claimed_without_any_change_is_refused(
|
|
store: StateStore, ws: Workspace, repo: GitRepo
|
|
) -> None:
|
|
# The handback plus file state is the whole interface back, so a note
|
|
# describing work that was never done is worse than no note.
|
|
client = ScriptedClient(
|
|
[
|
|
[hand_back(cid="early")],
|
|
[call("write_file", path="real.py", content="x")],
|
|
[hand_back(cid="earned")],
|
|
]
|
|
)
|
|
result = build(client, store, ws, repo).run(brief="b", branch="d")
|
|
|
|
assert result.usable
|
|
assert len(client.calls) == 3 # the first claim was sent back
|
|
assert (repo.root / "real.py").exists()
|
|
|
|
|
|
def test_failure_may_be_reported_without_doing_work(
|
|
store: StateStore, ws: Workspace, repo: GitRepo
|
|
) -> None:
|
|
# "This sub-task is impossible" is a legitimate finding that requires no
|
|
# changes; only a claim of success needs evidence.
|
|
client = ScriptedClient(
|
|
[[hand_back(succeeded=False, changed="nothing: the API does not exist")]]
|
|
)
|
|
result = build(client, store, ws, repo).run(brief="b", branch="d")
|
|
|
|
assert result.status == "failed"
|
|
assert not result.usable
|
|
assert len(client.calls) == 1 # believed immediately, not challenged
|
|
|
|
|
|
# --- A3 step 5: the resume rule -----------------------------------------
|
|
|
|
|
|
def pending_delegation_intent(store: StateStore, branch: str = "d") -> Any:
|
|
store.begin_intent(
|
|
DELEGATION_INTENT,
|
|
{"branch": branch, "base": "main", "brief": "finish it"},
|
|
cause="test",
|
|
)
|
|
return store.pending_intents()[-1]
|
|
|
|
|
|
def test_intent_is_recorded_and_settled_by_a_completed_delegation(
|
|
store: StateStore, ws: Workspace, repo: GitRepo
|
|
) -> None:
|
|
client = ScriptedClient(
|
|
[[call("write_file", path="a.py", content="x")], [hand_back()]]
|
|
)
|
|
build(client, store, ws, repo).run(brief="b", branch="d")
|
|
# Nothing left in flight: an aborted delegation is a finished one too.
|
|
assert store.pending_intents() == []
|
|
|
|
|
|
def test_aborted_delegation_also_settles_its_intent(
|
|
store: StateStore, ws: Workspace, repo: GitRepo
|
|
) -> None:
|
|
client = ScriptedClient([[hand_back(succeeded=False, changed="cannot")]])
|
|
build(client, store, ws, repo).run(brief="b", branch="d")
|
|
assert store.pending_intents() == []
|
|
|
|
|
|
def test_clean_cut_after_a_tool_result_is_resumable(
|
|
store: StateStore, ws: Workspace, repo: GitRepo
|
|
) -> None:
|
|
repo.checkout_new_branch("d")
|
|
(repo.root / "partial.py").write_text("x")
|
|
repo.commit_all("partial work")
|
|
store.append_event(
|
|
DelegationStarted(brief="b", role="delegate", model="m", caps={})
|
|
)
|
|
store.append_event(ToolCall(call_id="c", tool="write_file", arguments={}))
|
|
store.append_event(ToolResult(call_id="c", result="wrote", success=True))
|
|
intent = pending_delegation_intent(store)
|
|
|
|
assert delegation_reconciler(store, repo)(intent) is True
|
|
|
|
|
|
def test_dangling_tool_call_gates(
|
|
store: StateStore, ws: Workspace, repo: GitRepo
|
|
) -> None:
|
|
# The effect of the call is unknown and not verifiable by inspection.
|
|
repo.checkout_new_branch("d")
|
|
store.append_event(
|
|
DelegationStarted(brief="b", role="delegate", model="m", caps={})
|
|
)
|
|
store.append_event(ToolCall(call_id="c", tool="run_command", arguments={}))
|
|
intent = pending_delegation_intent(store)
|
|
|
|
assert delegation_reconciler(store, repo)(intent) is None
|
|
|
|
|
|
def test_missing_branch_means_nothing_started(
|
|
store: StateStore, ws: Workspace, repo: GitRepo
|
|
) -> None:
|
|
store.append_event(
|
|
DelegationStarted(brief="b", role="delegate", model="m", caps={})
|
|
)
|
|
store.append_event(ToolResult(call_id="c", result="ok", success=True))
|
|
intent = pending_delegation_intent(store, branch="never-created")
|
|
|
|
assert delegation_reconciler(store, repo)(intent) is False
|
|
|
|
|
|
def test_a_finished_delegation_settles_rather_than_resumes(
|
|
store: StateStore, ws: Workspace, repo: GitRepo
|
|
) -> None:
|
|
repo.checkout_new_branch("d")
|
|
store.append_event(
|
|
DelegationStarted(brief="b", role="delegate", model="m", caps={})
|
|
)
|
|
store.append_event(
|
|
DelegationCompleted(
|
|
handback="done",
|
|
artifacts=[],
|
|
tokens=0,
|
|
duration_s=1.0,
|
|
exit_status="handback",
|
|
)
|
|
)
|
|
intent = pending_delegation_intent(store)
|
|
|
|
assert delegation_reconciler(store, repo)(intent) is True
|
|
|
|
|
|
def test_resume_continues_the_same_delegation(
|
|
store: StateStore, ws: Workspace, repo: GitRepo
|
|
) -> None:
|
|
# Interrupt after one turn, then resume: no second delegation_started,
|
|
# because A10 defines a session as the delegation, not the process.
|
|
repo.checkout_new_branch("d")
|
|
store.append_event(
|
|
DelegationStarted(brief="b", role="delegate", model="m", caps={})
|
|
)
|
|
store.append_event(ToolResult(call_id="c", result="wrote a.py", success=True))
|
|
intent = pending_delegation_intent(store)
|
|
|
|
client = ScriptedClient(
|
|
[[call("write_file", path="b.py", content="x")], [hand_back()]]
|
|
)
|
|
result = build(client, store, ws, repo).resume(intent)
|
|
|
|
assert result.usable
|
|
starts = [e for e in store.read_events() if isinstance(e.event, DelegationStarted)]
|
|
assert len(starts) == 1
|
|
assert store.pending_intents() == []
|
|
assert (repo.root / "b.py").exists()
|
|
|
|
|
|
def test_resume_onto_a_missing_branch_fails_cleanly(
|
|
store: StateStore, ws: Workspace, repo: GitRepo
|
|
) -> None:
|
|
intent = pending_delegation_intent(store, branch="gone")
|
|
result = build(ScriptedClient([]), store, ws, repo).resume(intent)
|
|
|
|
assert result.status == "failed"
|
|
assert not result.usable
|
|
assert store.pending_intents() == [] # not left dangling for the next boot
|
|
|
|
|
|
# --- boot recovery sweep ------------------------------------------------
|
|
|
|
|
|
def test_recover_resumes_a_clean_cut_with_a_notice(
|
|
store: StateStore, ws: Workspace, repo: GitRepo
|
|
) -> None:
|
|
repo.checkout_new_branch("d")
|
|
store.append_event(
|
|
DelegationStarted(brief="b", role="delegate", model="m", caps={})
|
|
)
|
|
store.append_event(ToolResult(call_id="c", result="wrote a.py", success=True))
|
|
pending_delegation_intent(store)
|
|
notices: list[str] = []
|
|
|
|
client = ScriptedClient(
|
|
[[call("write_file", path="b.py", content="x")], [hand_back()]]
|
|
)
|
|
result = build(client, store, ws, repo).recover(notices.append)
|
|
|
|
assert result is not None and result.usable
|
|
# A3: a notice, not a question — a clean cut needs no operator decision.
|
|
assert notices and "resuming the delegation" in notices[0]
|
|
|
|
|
|
def test_recover_settles_a_finished_delegation_without_resuming(
|
|
store: StateStore, ws: Workspace, repo: GitRepo
|
|
) -> None:
|
|
repo.checkout_new_branch("d")
|
|
store.append_event(
|
|
DelegationStarted(brief="b", role="delegate", model="m", caps={})
|
|
)
|
|
store.append_event(
|
|
DelegationCompleted(
|
|
handback="done",
|
|
artifacts=[],
|
|
tokens=0,
|
|
duration_s=1.0,
|
|
exit_status="handback",
|
|
)
|
|
)
|
|
pending_delegation_intent(store)
|
|
client = ScriptedClient([])
|
|
|
|
assert build(client, store, ws, repo).recover(lambda _m: None) is None
|
|
assert store.pending_intents() == []
|
|
assert client.calls == [] # the model was never asked
|
|
|
|
|
|
def test_recover_discards_an_intent_whose_branch_never_existed(
|
|
store: StateStore, ws: Workspace, repo: GitRepo
|
|
) -> None:
|
|
store.append_event(ToolResult(call_id="c", result="ok", success=True))
|
|
pending_delegation_intent(store, branch="never-made")
|
|
|
|
assert build(ScriptedClient([]), store, ws, repo).recover(lambda _m: None) is None
|
|
assert store.pending_intents() == []
|
|
|
|
|
|
def test_recover_leaves_an_unclean_cut_for_the_boot_gate(
|
|
store: StateStore, ws: Workspace, repo: GitRepo
|
|
) -> None:
|
|
repo.checkout_new_branch("d")
|
|
store.append_event(
|
|
DelegationStarted(brief="b", role="delegate", model="m", caps={})
|
|
)
|
|
store.append_event(ToolCall(call_id="c", tool="run_command", arguments={}))
|
|
pending_delegation_intent(store)
|
|
|
|
assert build(ScriptedClient([]), store, ws, repo).recover(lambda _m: None) is None
|
|
# Still pending, so reconcile_on_boot gates rather than blind-resuming.
|
|
assert [i.kind for i in store.pending_intents()] == [DELEGATION_INTENT]
|
|
|
|
|
|
# --- the token cap, which A13 calls hard and was decoration -------------
|
|
|
|
|
|
class MeteredClient(ScriptedClient):
|
|
"""A client that reports usage, as the real ones do."""
|
|
|
|
def __init__(self, turns: list, per_call: int = 1000) -> None:
|
|
super().__init__(turns)
|
|
self._per_call = per_call
|
|
self.last_usage: dict[str, int] = {}
|
|
|
|
def complete(self, *args: Any, **kwargs: Any):
|
|
produced = super().complete(*args, **kwargs)
|
|
self.last_usage = {"input": self._per_call, "output": self._per_call}
|
|
return produced
|
|
|
|
|
|
def test_token_cap_stops_a_delegation(
|
|
store: StateStore, ws: Workspace, repo: GitRepo
|
|
) -> None:
|
|
# An unbounded delegation can exhaust a daily budget and take external
|
|
# models away until it resets, so the cap has to be real.
|
|
client = MeteredClient(
|
|
[[call("list_dir", path=".")] for _ in range(20)], per_call=600
|
|
)
|
|
result = build(client, store, ws, repo, caps=Caps(turns=20, tokens=2000)).run(
|
|
brief="b", branch="d"
|
|
)
|
|
|
|
assert result.status == "capped"
|
|
assert "token cap of 2000" in result.note
|
|
# 600 in + 600 out per call: stops on the second turn, not the twentieth.
|
|
assert len(client.calls) == 2
|
|
|
|
|
|
def test_tokens_spent_are_reported_not_left_at_zero(
|
|
store: StateStore, ws: Workspace, repo: GitRepo
|
|
) -> None:
|
|
client = MeteredClient(
|
|
[[call("write_file", path="a.py", content="x")], [hand_back()]], per_call=500
|
|
)
|
|
result = build(client, store, ws, repo).run(brief="b", branch="d")
|
|
|
|
assert result.usable
|
|
assert result.tokens == 2000 # two calls, 1000 each
|
|
|
|
completed = next(
|
|
e.event for e in store.read_events() if isinstance(e.event, DelegationCompleted)
|
|
)
|
|
# A16 evaluates cost against outcome, which needs a real number.
|
|
assert completed.tokens == 2000
|
|
|
|
|
|
def test_a_client_without_usage_still_works(
|
|
store: StateStore, ws: Workspace, repo: GitRepo
|
|
) -> None:
|
|
# Scripted and third-party clients need not report usage; the cap simply
|
|
# does not bind rather than crashing the delegation.
|
|
client = ScriptedClient(
|
|
[[call("write_file", path="a.py", content="x")], [hand_back()]]
|
|
)
|
|
assert build(client, store, ws, repo).run(brief="b", branch="d").usable
|
|
|
|
|
|
def test_budget_exhaustion_preserves_the_work_for_later(
|
|
store: StateStore, ws: Workspace, repo: GitRepo
|
|
) -> None:
|
|
"""An outage is not a bad delegate.
|
|
|
|
The branch may hold perfectly good work, so it is kept and the intent
|
|
left pending — the next boot treats it exactly like a crash mid-delegation
|
|
and resumes if the cut was clean. Discarding would throw away work
|
|
because the budget ran out.
|
|
"""
|
|
|
|
class BrokeClient(ScriptedClient):
|
|
def complete(self, *args: Any, **kwargs: Any):
|
|
if self.calls:
|
|
raise BudgetExhausted("daily budget exhausted")
|
|
return super().complete(*args, **kwargs)
|
|
|
|
client = BrokeClient([[call("write_file", path="progress.py", content="real")]])
|
|
result = build(client, store, ws, repo).run(brief="b", branch="d")
|
|
|
|
assert result.status == "interrupted"
|
|
assert result.interrupted and not result.usable
|
|
# The work survives, on its branch.
|
|
assert repo.current_branch() == "d"
|
|
assert (repo.root / "progress.py").exists()
|
|
# And the intent stays pending, which is what A3's resume rule keys on.
|
|
assert [i.kind for i in store.pending_intents()] == [DELEGATION_INTENT]
|
|
|
|
|
|
def test_an_ordinary_failure_still_discards(
|
|
store: StateStore, ws: Workspace, repo: GitRepo
|
|
) -> None:
|
|
# Only external interruptions are preserved; a delegate that made a mess
|
|
# is still cleaned up (A13).
|
|
client = ScriptedClient([[hand_back(succeeded=False, changed="cannot")]])
|
|
result = build(client, store, ws, repo).run(brief="b", branch="d")
|
|
|
|
assert result.status == "failed"
|
|
assert not result.interrupted
|
|
assert repo.current_branch() == "main"
|
|
assert store.pending_intents() == []
|
|
|
|
|
|
def test_a_network_outage_also_preserves_the_work(
|
|
store: StateStore, ws: Workspace, repo: GitRepo
|
|
) -> None:
|
|
# Same reasoning as an exhausted budget: the branch may be fine, and the
|
|
# outage says nothing about the quality of the work on it.
|
|
class FlakyClient(ScriptedClient):
|
|
def complete(self, *args: Any, **kwargs: Any):
|
|
if self.calls:
|
|
raise UpstreamUnavailable("LLM request failed: connection refused")
|
|
return super().complete(*args, **kwargs)
|
|
|
|
client = FlakyClient([[call("write_file", path="progress.py", content="real")]])
|
|
result = build(client, store, ws, repo).run(brief="b", branch="d")
|
|
|
|
assert result.status == "interrupted"
|
|
assert (repo.root / "progress.py").exists()
|
|
assert [i.kind for i in store.pending_intents()] == [DELEGATION_INTENT]
|
|
|
|
|
|
def test_a_bad_request_is_not_given_the_benefit_of_the_doubt(
|
|
store: StateStore, ws: Workspace, repo: GitRepo
|
|
) -> None:
|
|
# A 4xx that is not a rate limit means we sent something wrong; treating
|
|
# that as an outage would preserve branches and hide a real bug.
|
|
class BadRequestClient(ScriptedClient):
|
|
def complete(self, *args: Any, **kwargs: Any):
|
|
raise LLMError("LLM request returned 400: malformed tool schema")
|
|
|
|
result = build(BadRequestClient([]), store, ws, repo).run(brief="b", branch="d")
|
|
assert result.status == "failed"
|
|
assert not result.interrupted
|