Runs that go in circles end (M7.8)
Driver.run checks each turn with nisse_core.loop.circles and ends the run as "circling" when the same calls get much the same back, one harness correction is given again and again, or the model apologises, for circle_repeats turns in a row (default 3). Near-identical calls (git add a.py, b.py) and scattered apologies stay flags on the status page. A fact in the transcript says why the run ended. nisse-dev calls in a helper as for a stuck run, then asks the operator; nisse-review counts a failed run and posts none of its drafts. Tuning: tuning.driver / tuning.review circle_repeats (0 off) and circle_similar. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01J5Utyfzw7gF7UhjRDEtjTk
This commit is contained in:
1 parent
516a328618
commit
74bc68db1c
20 files changed
+506
-20
No files matched your search
@@ -182,6 +182,23 @@ def test_tuning_is_read() -> None:
|
||||
assert (tuning.build_vm.retry_s, tuning.build_vm.settle_s) == (60.0, 180.0)
|
||||
|
||||
|
||||
def test_circles_are_tuned_and_checked() -> None:
|
||||
# L2, M7.8: when a driver run going in circles ends.
|
||||
assert (
|
||||
build().tuning.driver.circle_repeats,
|
||||
build().tuning.driver.circle_similar,
|
||||
) == (3, 0.9)
|
||||
tuned = build(tuning={"driver": {"circle_repeats": 0, "circle_similar": 0.8}})
|
||||
assert (tuned.tuning.driver.circle_repeats, tuned.tuning.driver.circle_similar) == (
|
||||
0,
|
||||
0.8,
|
||||
)
|
||||
with pytest.raises(ConfigError, match=r"circle_repeats: must be 0 \(off\) or 2"):
|
||||
build(tuning={"driver": {"circle_repeats": 1}})
|
||||
with pytest.raises(ConfigError, match="circle_similar: must be at most 1"):
|
||||
build(tuning={"driver": {"circle_similar": 2}})
|
||||
|
||||
|
||||
def test_a_partly_given_tuning_section_keeps_the_other_defaults() -> None:
|
||||
tuning = build(tuning={"driver": {"max_turns": 40}}).tuning
|
||||
assert tuning.driver.failure_limit == 2
|
||||
|
||||
@@ -386,7 +386,8 @@ def test_an_invented_tool_is_answered_not_run(tmp_path: Path) -> None:
|
||||
|
||||
def test_the_turn_cap_ends_a_run(tmp_path: Path) -> None:
|
||||
model = ScriptedModel(*[[call("read_file", path="a")]] * 3)
|
||||
outcome = Driver(model, "m", max_turns=3).run(
|
||||
# The same call three times is a circle too (M7.8): here, only the cap.
|
||||
outcome = Driver(model, "m", max_turns=3, circle_repeats=0).run(
|
||||
driver_log(tmp_path),
|
||||
system="s",
|
||||
tools=_tools("read_file"),
|
||||
|
||||
@@ -249,6 +249,49 @@ def test_stuck_again_in_the_same_milestone_asks_the_operator(
|
||||
assert len(helper.calls) == 2 # one delegation: its two turns
|
||||
|
||||
|
||||
def test_a_driver_going_in_circles_gets_a_helper_too(world: dict[str, Any]) -> None:
|
||||
# L2, M7.8: the same command, the same output, turn after turn.
|
||||
helper = Spy(
|
||||
[call("write_file", path="fixed.py", content="F = 1\n")],
|
||||
[hand_back()],
|
||||
)
|
||||
model = ScriptedModel(
|
||||
RESTATE,
|
||||
[call("start_milestone", name="words")],
|
||||
*[[call("run_command", command="true")] for _ in range(3)],
|
||||
[call("finish_milestone", summary="fixed with help", final="yes")],
|
||||
)
|
||||
agent, transport = loop_agent(world, model, helpers=helpers(helper))
|
||||
agent.intake()
|
||||
say(transport, world, "go")
|
||||
|
||||
assert any("so a helper is taking over" in m.text for m in transport.outbound)
|
||||
assert "near-identical replies in a row" in str(helper.calls) # in the brief
|
||||
assert loop_of(item(world)).helped_at == 1
|
||||
|
||||
|
||||
def test_circling_again_in_the_same_milestone_asks_the_operator(
|
||||
world: dict[str, Any],
|
||||
) -> None:
|
||||
helper = Spy(
|
||||
[call("write_file", path="tried.py", content="T = 1\n")],
|
||||
[hand_back()],
|
||||
)
|
||||
circling = [[call("run_command", command="true")] for _ in range(3)]
|
||||
model = ScriptedModel(
|
||||
RESTATE,
|
||||
[call("start_milestone", name="words")],
|
||||
*circling,
|
||||
*circling, # after the helper: round again
|
||||
)
|
||||
agent, transport = loop_agent(world, model, helpers=helpers(helper))
|
||||
agent.intake()
|
||||
say(transport, world, "go")
|
||||
loop = loop_of(item(world))
|
||||
assert loop.wait == ANSWER
|
||||
assert "I stopped short: 3 near-identical replies in a row" in loop.note
|
||||
|
||||
|
||||
def test_ask_expert_answers_within_its_budget(world: dict[str, Any]) -> None:
|
||||
answer = {"answer": "use a dict", "confidence": "high", "caveats": ""}
|
||||
expert = Spy(
|
||||
|
||||
@@ -139,8 +139,10 @@ def test_a_pass_that_doesnt_finish_never_holds_up_the_close(
|
||||
|
||||
|
||||
def test_the_pass_is_bounded(world: dict[str, Any], memory: Memory) -> None:
|
||||
looking = [call("list_notes", folder="")]
|
||||
model = Spy(RESTATE, *[looking] * 3)
|
||||
# A different folder each turn: the same call three times would end the
|
||||
# pass as going in circles (M7.8), not at the cap.
|
||||
looking = [[call("list_notes", folder=f)] for f in ("", "a", "b")]
|
||||
model = Spy(RESTATE, *looking)
|
||||
loop, transport = agent(world, model, memory, memory_turns=3)
|
||||
loop.intake()
|
||||
say(transport, world, "abandon")
|
||||
|
||||
Reference in new issue
Block a user