Runs that go in circles end (M7.8)
Driver.run checks each turn with nisse_core.loop.circles and ends the run as "circling" when the same calls get much the same back, one harness correction is given again and again, or the model apologises, for circle_repeats turns in a row (default 3). Near-identical calls (git add a.py, b.py) and scattered apologies stay flags on the status page. A fact in the transcript says why the run ended. nisse-dev calls in a helper as for a stuck run, then asks the operator; nisse-review counts a failed run and posts none of its drafts. Tuning: tuning.driver / tuning.review circle_repeats (0 off) and circle_similar. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01J5Utyfzw7gF7UhjRDEtjTk
This commit is contained in:
1 parent
516a328618
commit
74bc68db1c
20 files changed
+506
-20
No files matched your search
@@ -5,7 +5,13 @@ import io
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from nisse_core.log.canonical import CanonicalEvent, ToolCall, ToolResult
|
||||
from nisse_core.log.canonical import (
|
||||
AssistantText,
|
||||
CanonicalEvent,
|
||||
Fact,
|
||||
ToolCall,
|
||||
ToolResult,
|
||||
)
|
||||
from nisse_core.loop.driver import Driver, ToolOutcome
|
||||
from nisse_core.loop.tools import tool
|
||||
from nisse_core.oplog import OpLog
|
||||
@@ -150,3 +156,156 @@ def test_without_due_a_settled_point_leaves_the_run_going(tmp_path: Path) -> Non
|
||||
handlers={"note": lambda args: ("noted", True)},
|
||||
)
|
||||
assert (ran.tool, ran.arguments) == ("post_review", {"body": "b"})
|
||||
|
||||
|
||||
# --- runs going in circles (L2, M7.8) ------------------------------------------
|
||||
|
||||
COMMENT = tool("post_comment", "Post a comment.", body="The comment.")
|
||||
|
||||
|
||||
def facts(log: Any) -> list[str]:
|
||||
return [e.event.text for e in log.read() if isinstance(e.event, Fact)]
|
||||
|
||||
|
||||
def test_a_run_posting_the_same_comment_ends_as_circling(tmp_path: Path) -> None:
|
||||
# Flash-Next on 2026-10-24: the same apology, numbered, about 20 times
|
||||
# until the turn cap.
|
||||
posted: list[str] = []
|
||||
model = Scripted(
|
||||
*[
|
||||
[call("post_comment", body=f"Duplicate, please disregard ({n}).")]
|
||||
for n in range(20)
|
||||
]
|
||||
)
|
||||
log = transcript(tmp_path)
|
||||
ran = Driver(model, "m", max_turns=30).run(
|
||||
log,
|
||||
system="s",
|
||||
tools=[COMMENT, POST],
|
||||
waiting={"post_review"},
|
||||
workspace=None,
|
||||
handlers={
|
||||
"post_comment": lambda a: (posted.append(a["body"]) or "posted", True)
|
||||
},
|
||||
)
|
||||
assert ran.kind == "circling"
|
||||
assert len(posted) == 3 # ended at the third, not the thirtieth
|
||||
assert ran.note.startswith("3 apologies in a row")
|
||||
assert "Duplicate, please disregard (2)" in ran.note # what it was doing
|
||||
assert facts(log) == [f"The run was ended: it was going in circles ({ran.note})."]
|
||||
|
||||
|
||||
def test_an_invented_tool_told_again_and_again_ends_the_run(tmp_path: Path) -> None:
|
||||
# A different reply each time, but the same correction every time.
|
||||
model = Scripted(
|
||||
*[[call("search_code", query=f"attempt {n} " * n)] for n in range(1, 10)]
|
||||
)
|
||||
ran = Driver(model, "m").run(
|
||||
transcript(tmp_path),
|
||||
system="s",
|
||||
tools=[READ, POST],
|
||||
waiting={"post_review"},
|
||||
workspace=Reader(),
|
||||
)
|
||||
assert ran.kind == "circling"
|
||||
assert ran.note.startswith("told 3 times: no such tool: search_code")
|
||||
|
||||
|
||||
def test_reading_file_after_file_is_progress_not_a_circle(tmp_path: Path) -> None:
|
||||
class Files:
|
||||
def dispatch(self, tool: str, arguments: dict[str, Any]) -> ToolOutcome:
|
||||
# Near-identical contents: still a different file each time.
|
||||
return ToolOutcome(f"{arguments['path']}\n" + "x = 1\n" * 200, True)
|
||||
|
||||
model = Scripted(
|
||||
*[[call("read_file", path=f"src/test_{n}.py")] for n in range(6)],
|
||||
[call("post_review", body="ok")],
|
||||
)
|
||||
ran = Driver(model, "m").run(
|
||||
transcript(tmp_path),
|
||||
system="s",
|
||||
tools=[READ, POST],
|
||||
waiting={"post_review"},
|
||||
workspace=Files(),
|
||||
)
|
||||
assert ran.kind == "wait"
|
||||
|
||||
|
||||
def test_the_same_call_getting_the_same_back_ends_the_run(tmp_path: Path) -> None:
|
||||
model = Scripted(*[[call("read_file", path="a.py")] for _ in range(9)])
|
||||
ran = Driver(model, "m").run(
|
||||
transcript(tmp_path),
|
||||
system="s",
|
||||
tools=[READ, POST],
|
||||
waiting={"post_review"},
|
||||
workspace=Reader(),
|
||||
)
|
||||
assert ran.kind == "circling"
|
||||
assert ran.note.startswith("3 near-identical replies in a row")
|
||||
assert 'read_file({"path": "a.py"})' in ran.note
|
||||
|
||||
|
||||
def test_alike_commands_are_progress_not_a_circle(tmp_path: Path) -> None:
|
||||
# git add a.py, b.py, c.py: alike, each answered the same, each its own.
|
||||
class Git:
|
||||
def dispatch(self, tool: str, arguments: dict[str, Any]) -> ToolOutcome:
|
||||
return ToolOutcome("", True)
|
||||
|
||||
model = Scripted(
|
||||
*[[call("run_command", command=f"git add {f}.py")] for f in "abcdef"],
|
||||
[call("post_review", body="ok")],
|
||||
)
|
||||
ran = Driver(model, "m").run(
|
||||
transcript(tmp_path),
|
||||
system="s",
|
||||
tools=[tool("run_command", "Run.", command="It."), POST],
|
||||
waiting={"post_review"},
|
||||
workspace=Git(),
|
||||
)
|
||||
assert ran.kind == "wait"
|
||||
|
||||
|
||||
def test_circles_can_be_turned_off(tmp_path: Path) -> None:
|
||||
model = Scripted(*[[call("post_comment", body="same")] for _ in range(5)])
|
||||
ran = Driver(model, "m", max_turns=5, circle_repeats=0).run(
|
||||
transcript(tmp_path),
|
||||
system="s",
|
||||
tools=[COMMENT, POST],
|
||||
waiting={"post_review"},
|
||||
workspace=None,
|
||||
handlers={"post_comment": lambda a: ("posted", True)},
|
||||
)
|
||||
assert ran.kind == "capped"
|
||||
|
||||
|
||||
def test_repeats_and_likeness_are_the_agents_to_set(tmp_path: Path) -> None:
|
||||
model = Scripted(*[[call("post_comment", body="same")] for _ in range(5)])
|
||||
ran = Driver(model, "m", circle_repeats=5).run(
|
||||
transcript(tmp_path),
|
||||
system="s",
|
||||
tools=[COMMENT, POST],
|
||||
waiting={"post_review"},
|
||||
workspace=None,
|
||||
handlers={"post_comment": lambda a: ("posted", True)},
|
||||
)
|
||||
assert ran.kind == "circling" and ran.note.startswith("5 near-identical")
|
||||
|
||||
|
||||
def test_apologies_spread_over_a_run_dont_end_it(tmp_path: Path) -> None:
|
||||
# Only flagged on the status page: sorry now and then is not a loop.
|
||||
def turn(n: int, sorry: bool) -> list[CanonicalEvent]:
|
||||
said = "Sorry, wrong file." if sorry else "Reading on."
|
||||
return [AssistantText(text=said, model="m"), call("read_file", path=f"f{n}")]
|
||||
|
||||
model = Scripted(
|
||||
*[turn(n, sorry=n % 2 == 0) for n in range(7)],
|
||||
[call("post_review", body="ok")],
|
||||
)
|
||||
ran = Driver(model, "m").run(
|
||||
transcript(tmp_path),
|
||||
system="s",
|
||||
tools=[READ, POST],
|
||||
waiting={"post_review"},
|
||||
workspace=Reader(),
|
||||
)
|
||||
assert ran.kind == "wait"
|
||||
Reference in new issue
Block a user