Runs that go in circles end (M7.8)
Driver.run checks each turn with nisse_core.loop.circles and ends the run as "circling" when the same calls get much the same back, one harness correction is given again and again, or the model apologises, for circle_repeats turns in a row (default 3). Near-identical calls (git add a.py, b.py) and scattered apologies stay flags on the status page. A fact in the transcript says why the run ended. nisse-dev calls in a helper as for a stuck run, then asks the operator; nisse-review counts a failed run and posts none of its drafts. Tuning: tuning.driver / tuning.review circle_repeats (0 off) and circle_similar. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01J5Utyfzw7gF7UhjRDEtjTk
This commit is contained in:
1 parent
516a328618
commit
74bc68db1c
20 files changed
+506
-20
No files matched your search
@@ -1,6 +1,6 @@
|
||||
"""Runs going in circles (L2, X6), judged from the turns alone."""
|
||||
|
||||
from nisse_core.loop.circles import Turn, find_circles
|
||||
from nisse_core.loop.circles import Turn, apologies_in_a_row, find_circles
|
||||
|
||||
NUDGE = "Your last reply was prose. Every reply must be a tool call."
|
||||
|
||||
@@ -71,3 +71,46 @@ def test_the_limits_are_tuning() -> None:
|
||||
turns = [comment(1), comment(2)]
|
||||
assert [c.kind for c in find_circles(turns, repeats=2)] == ["repeated reply"]
|
||||
assert find_circles([comment(1), comment(22), comment(333)], similar=1.0) == []
|
||||
|
||||
|
||||
def test_alike_replies_that_got_different_results_are_no_circle() -> None:
|
||||
read = [("read_file", '{"path": "src/a.py"}')]
|
||||
turns = [Turn(calls=tuple(read), results=(f"file {n}",)) for n in range(4)]
|
||||
assert find_circles(turns) == []
|
||||
same = [Turn(calls=tuple(read), results=("posted",)) for _ in range(4)]
|
||||
assert [c.kind for c in find_circles(same)] == ["repeated reply"]
|
||||
# The page gives no results: alike replies alone are a circle there.
|
||||
bare = [Turn(calls=tuple(read)) for _ in range(3)]
|
||||
assert [c.kind for c in find_circles(bare)] == ["repeated reply"]
|
||||
|
||||
|
||||
def test_the_driver_counts_only_exactly_repeated_calls() -> None:
|
||||
adds = [
|
||||
Turn(calls=(("run_command", f'{{"command": "git add {f}.py"}}'),))
|
||||
for f in "abc"
|
||||
]
|
||||
assert [c.kind for c in find_circles(adds)] == ["repeated reply"] # flagged
|
||||
assert find_circles(adds, exact_calls=True) == [] # never ended for it
|
||||
same = [Turn(calls=(("read_file", '{"path": "a"}'),))] * 3
|
||||
assert len(find_circles(same, exact_calls=True)) == 1
|
||||
|
||||
|
||||
def test_apologies_end_a_run_only_in_a_row() -> None:
|
||||
sorry, work = Turn(text="Sorry, my mistake."), Turn(text="Reading the file.")
|
||||
assert apologies_in_a_row([work, sorry, sorry, sorry]) is not None
|
||||
assert apologies_in_a_row([sorry, work, sorry, sorry]) is None
|
||||
assert apologies_in_a_row([sorry, sorry]) is None
|
||||
# In what it posted, too: an apology comment is an apology.
|
||||
posted = Turn(calls=(("post_comment", '{"body": "Please disregard (1)."}'),))
|
||||
assert apologies_in_a_row([posted] * 3) is not None
|
||||
|
||||
|
||||
def test_the_same_call_answered_with_a_count_is_still_a_circle() -> None:
|
||||
note = (("comment", '{"line": 2, "text": "Same point."}'),)
|
||||
turns = [Turn(calls=note, results=(f"noted ({n} in the draft)",)) for n in range(3)]
|
||||
assert len(find_circles(turns, exact_calls=True)) == 1
|
||||
files = [
|
||||
Turn(calls=(("read_file", f'{{"path": "{f}"}}'),), results=(f"{f}: x",))
|
||||
for f in "abc"
|
||||
]
|
||||
assert find_circles(files, exact_calls=True) == []
|
||||
@@ -5,7 +5,13 @@ import io
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from nisse_core.log.canonical import CanonicalEvent, ToolCall, ToolResult
|
||||
from nisse_core.log.canonical import (
|
||||
AssistantText,
|
||||
CanonicalEvent,
|
||||
Fact,
|
||||
ToolCall,
|
||||
ToolResult,
|
||||
)
|
||||
from nisse_core.loop.driver import Driver, ToolOutcome
|
||||
from nisse_core.loop.tools import tool
|
||||
from nisse_core.oplog import OpLog
|
||||
@@ -150,3 +156,156 @@ def test_without_due_a_settled_point_leaves_the_run_going(tmp_path: Path) -> Non
|
||||
handlers={"note": lambda args: ("noted", True)},
|
||||
)
|
||||
assert (ran.tool, ran.arguments) == ("post_review", {"body": "b"})
|
||||
|
||||
|
||||
# --- runs going in circles (L2, M7.8) ------------------------------------------
|
||||
|
||||
COMMENT = tool("post_comment", "Post a comment.", body="The comment.")
|
||||
|
||||
|
||||
def facts(log: Any) -> list[str]:
|
||||
return [e.event.text for e in log.read() if isinstance(e.event, Fact)]
|
||||
|
||||
|
||||
def test_a_run_posting_the_same_comment_ends_as_circling(tmp_path: Path) -> None:
|
||||
# Flash-Next on 2026-10-24: the same apology, numbered, about 20 times
|
||||
# until the turn cap.
|
||||
posted: list[str] = []
|
||||
model = Scripted(
|
||||
*[
|
||||
[call("post_comment", body=f"Duplicate, please disregard ({n}).")]
|
||||
for n in range(20)
|
||||
]
|
||||
)
|
||||
log = transcript(tmp_path)
|
||||
ran = Driver(model, "m", max_turns=30).run(
|
||||
log,
|
||||
system="s",
|
||||
tools=[COMMENT, POST],
|
||||
waiting={"post_review"},
|
||||
workspace=None,
|
||||
handlers={
|
||||
"post_comment": lambda a: (posted.append(a["body"]) or "posted", True)
|
||||
},
|
||||
)
|
||||
assert ran.kind == "circling"
|
||||
assert len(posted) == 3 # ended at the third, not the thirtieth
|
||||
assert ran.note.startswith("3 apologies in a row")
|
||||
assert "Duplicate, please disregard (2)" in ran.note # what it was doing
|
||||
assert facts(log) == [f"The run was ended: it was going in circles ({ran.note})."]
|
||||
|
||||
|
||||
def test_an_invented_tool_told_again_and_again_ends_the_run(tmp_path: Path) -> None:
|
||||
# A different reply each time, but the same correction every time.
|
||||
model = Scripted(
|
||||
*[[call("search_code", query=f"attempt {n} " * n)] for n in range(1, 10)]
|
||||
)
|
||||
ran = Driver(model, "m").run(
|
||||
transcript(tmp_path),
|
||||
system="s",
|
||||
tools=[READ, POST],
|
||||
waiting={"post_review"},
|
||||
workspace=Reader(),
|
||||
)
|
||||
assert ran.kind == "circling"
|
||||
assert ran.note.startswith("told 3 times: no such tool: search_code")
|
||||
|
||||
|
||||
def test_reading_file_after_file_is_progress_not_a_circle(tmp_path: Path) -> None:
|
||||
class Files:
|
||||
def dispatch(self, tool: str, arguments: dict[str, Any]) -> ToolOutcome:
|
||||
# Near-identical contents: still a different file each time.
|
||||
return ToolOutcome(f"{arguments['path']}\n" + "x = 1\n" * 200, True)
|
||||
|
||||
model = Scripted(
|
||||
*[[call("read_file", path=f"src/test_{n}.py")] for n in range(6)],
|
||||
[call("post_review", body="ok")],
|
||||
)
|
||||
ran = Driver(model, "m").run(
|
||||
transcript(tmp_path),
|
||||
system="s",
|
||||
tools=[READ, POST],
|
||||
waiting={"post_review"},
|
||||
workspace=Files(),
|
||||
)
|
||||
assert ran.kind == "wait"
|
||||
|
||||
|
||||
def test_the_same_call_getting_the_same_back_ends_the_run(tmp_path: Path) -> None:
|
||||
model = Scripted(*[[call("read_file", path="a.py")] for _ in range(9)])
|
||||
ran = Driver(model, "m").run(
|
||||
transcript(tmp_path),
|
||||
system="s",
|
||||
tools=[READ, POST],
|
||||
waiting={"post_review"},
|
||||
workspace=Reader(),
|
||||
)
|
||||
assert ran.kind == "circling"
|
||||
assert ran.note.startswith("3 near-identical replies in a row")
|
||||
assert 'read_file({"path": "a.py"})' in ran.note
|
||||
|
||||
|
||||
def test_alike_commands_are_progress_not_a_circle(tmp_path: Path) -> None:
|
||||
# git add a.py, b.py, c.py: alike, each answered the same, each its own.
|
||||
class Git:
|
||||
def dispatch(self, tool: str, arguments: dict[str, Any]) -> ToolOutcome:
|
||||
return ToolOutcome("", True)
|
||||
|
||||
model = Scripted(
|
||||
*[[call("run_command", command=f"git add {f}.py")] for f in "abcdef"],
|
||||
[call("post_review", body="ok")],
|
||||
)
|
||||
ran = Driver(model, "m").run(
|
||||
transcript(tmp_path),
|
||||
system="s",
|
||||
tools=[tool("run_command", "Run.", command="It."), POST],
|
||||
waiting={"post_review"},
|
||||
workspace=Git(),
|
||||
)
|
||||
assert ran.kind == "wait"
|
||||
|
||||
|
||||
def test_circles_can_be_turned_off(tmp_path: Path) -> None:
|
||||
model = Scripted(*[[call("post_comment", body="same")] for _ in range(5)])
|
||||
ran = Driver(model, "m", max_turns=5, circle_repeats=0).run(
|
||||
transcript(tmp_path),
|
||||
system="s",
|
||||
tools=[COMMENT, POST],
|
||||
waiting={"post_review"},
|
||||
workspace=None,
|
||||
handlers={"post_comment": lambda a: ("posted", True)},
|
||||
)
|
||||
assert ran.kind == "capped"
|
||||
|
||||
|
||||
def test_repeats_and_likeness_are_the_agents_to_set(tmp_path: Path) -> None:
|
||||
model = Scripted(*[[call("post_comment", body="same")] for _ in range(5)])
|
||||
ran = Driver(model, "m", circle_repeats=5).run(
|
||||
transcript(tmp_path),
|
||||
system="s",
|
||||
tools=[COMMENT, POST],
|
||||
waiting={"post_review"},
|
||||
workspace=None,
|
||||
handlers={"post_comment": lambda a: ("posted", True)},
|
||||
)
|
||||
assert ran.kind == "circling" and ran.note.startswith("5 near-identical")
|
||||
|
||||
|
||||
def test_apologies_spread_over_a_run_dont_end_it(tmp_path: Path) -> None:
|
||||
# Only flagged on the status page: sorry now and then is not a loop.
|
||||
def turn(n: int, sorry: bool) -> list[CanonicalEvent]:
|
||||
said = "Sorry, wrong file." if sorry else "Reading on."
|
||||
return [AssistantText(text=said, model="m"), call("read_file", path=f"f{n}")]
|
||||
|
||||
model = Scripted(
|
||||
*[turn(n, sorry=n % 2 == 0) for n in range(7)],
|
||||
[call("post_review", body="ok")],
|
||||
)
|
||||
ran = Driver(model, "m").run(
|
||||
transcript(tmp_path),
|
||||
system="s",
|
||||
tools=[READ, POST],
|
||||
waiting={"post_review"},
|
||||
workspace=Reader(),
|
||||
)
|
||||
assert ran.kind == "wait"
|
||||
Reference in new issue
Block a user