Runs that go in circles end (M7.8)
CI / checks (push) Skipped
CI / checks (pull_request) Failing after 1m50s
CI / e2e (pull_request) Skipped

Driver.run checks each turn with nisse_core.loop.circles and ends the run
as "circling" when the same calls get much the same back, one harness
correction is given again and again, or the model apologises, for
circle_repeats turns in a row (default 3). Near-identical calls (git add
a.py, b.py) and scattered apologies stay flags on the status page. A
fact in the transcript says why the run ended.

nisse-dev calls in a helper as for a stuck run, then asks the operator;
nisse-review counts a failed run and posts none of its drafts. Tuning:
tuning.driver / tuning.review circle_repeats (0 off) and circle_similar.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01J5Utyfzw7gF7UhjRDEtjTk
This commit is contained in:
dongarandClaude Opus 5.5 committed 2026-10-26 16:30:07 +00:00
1 parent 516a328618
commit 74bc68db1c
20 files changed
+506 -20

No files matched your search

+44 -1
View File
@@ -1,6 +1,6 @@
"""Runs going in circles (L2, X6), judged from the turns alone."""
from nisse_core.loop.circles import Turn, find_circles
from nisse_core.loop.circles import Turn, apologies_in_a_row, find_circles
NUDGE = "Your last reply was prose. Every reply must be a tool call."
@@ -71,3 +71,46 @@ def test_the_limits_are_tuning() -> None:
turns = [comment(1), comment(2)]
assert [c.kind for c in find_circles(turns, repeats=2)] == ["repeated reply"]
assert find_circles([comment(1), comment(22), comment(333)], similar=1.0) == []
def test_alike_replies_that_got_different_results_are_no_circle() -> None:
read = [("read_file", '{"path": "src/a.py"}')]
turns = [Turn(calls=tuple(read), results=(f"file {n}",)) for n in range(4)]
assert find_circles(turns) == []
same = [Turn(calls=tuple(read), results=("posted",)) for _ in range(4)]
assert [c.kind for c in find_circles(same)] == ["repeated reply"]
# The page gives no results: alike replies alone are a circle there.
bare = [Turn(calls=tuple(read)) for _ in range(3)]
assert [c.kind for c in find_circles(bare)] == ["repeated reply"]
def test_the_driver_counts_only_exactly_repeated_calls() -> None:
adds = [
Turn(calls=(("run_command", f'{{"command": "git add {f}.py"}}'),))
for f in "abc"
]
assert [c.kind for c in find_circles(adds)] == ["repeated reply"] # flagged
assert find_circles(adds, exact_calls=True) == [] # never ended for it
same = [Turn(calls=(("read_file", '{"path": "a"}'),))] * 3
assert len(find_circles(same, exact_calls=True)) == 1
def test_apologies_end_a_run_only_in_a_row() -> None:
sorry, work = Turn(text="Sorry, my mistake."), Turn(text="Reading the file.")
assert apologies_in_a_row([work, sorry, sorry, sorry]) is not None
assert apologies_in_a_row([sorry, work, sorry, sorry]) is None
assert apologies_in_a_row([sorry, sorry]) is None
# In what it posted, too: an apology comment is an apology.
posted = Turn(calls=(("post_comment", '{"body": "Please disregard (1)."}'),))
assert apologies_in_a_row([posted] * 3) is not None
def test_the_same_call_answered_with_a_count_is_still_a_circle() -> None:
note = (("comment", '{"line": 2, "text": "Same point."}'),)
turns = [Turn(calls=note, results=(f"noted ({n} in the draft)",)) for n in range(3)]
assert len(find_circles(turns, exact_calls=True)) == 1
files = [
Turn(calls=(("read_file", f'{{"path": "{f}"}}'),), results=(f"{f}: x",))
for f in "abc"
]
assert find_circles(files, exact_calls=True) == []
+160 -1
View File
@@ -5,7 +5,13 @@ import io
from pathlib import Path
from typing import Any
from nisse_core.log.canonical import CanonicalEvent, ToolCall, ToolResult
from nisse_core.log.canonical import (
AssistantText,
CanonicalEvent,
Fact,
ToolCall,
ToolResult,
)
from nisse_core.loop.driver import Driver, ToolOutcome
from nisse_core.loop.tools import tool
from nisse_core.oplog import OpLog
@@ -150,3 +156,156 @@ def test_without_due_a_settled_point_leaves_the_run_going(tmp_path: Path) -> Non
handlers={"note": lambda args: ("noted", True)},
)
assert (ran.tool, ran.arguments) == ("post_review", {"body": "b"})
# --- runs going in circles (L2, M7.8) ------------------------------------------
COMMENT = tool("post_comment", "Post a comment.", body="The comment.")
def facts(log: Any) -> list[str]:
return [e.event.text for e in log.read() if isinstance(e.event, Fact)]
def test_a_run_posting_the_same_comment_ends_as_circling(tmp_path: Path) -> None:
# Flash-Next on 2026-10-24: the same apology, numbered, about 20 times
# until the turn cap.
posted: list[str] = []
model = Scripted(
*[
[call("post_comment", body=f"Duplicate, please disregard ({n}).")]
for n in range(20)
]
)
log = transcript(tmp_path)
ran = Driver(model, "m", max_turns=30).run(
log,
system="s",
tools=[COMMENT, POST],
waiting={"post_review"},
workspace=None,
handlers={
"post_comment": lambda a: (posted.append(a["body"]) or "posted", True)
},
)
assert ran.kind == "circling"
assert len(posted) == 3 # ended at the third, not the thirtieth
assert ran.note.startswith("3 apologies in a row")
assert "Duplicate, please disregard (2)" in ran.note # what it was doing
assert facts(log) == [f"The run was ended: it was going in circles ({ran.note})."]
def test_an_invented_tool_told_again_and_again_ends_the_run(tmp_path: Path) -> None:
# A different reply each time, but the same correction every time.
model = Scripted(
*[[call("search_code", query=f"attempt {n} " * n)] for n in range(1, 10)]
)
ran = Driver(model, "m").run(
transcript(tmp_path),
system="s",
tools=[READ, POST],
waiting={"post_review"},
workspace=Reader(),
)
assert ran.kind == "circling"
assert ran.note.startswith("told 3 times: no such tool: search_code")
def test_reading_file_after_file_is_progress_not_a_circle(tmp_path: Path) -> None:
class Files:
def dispatch(self, tool: str, arguments: dict[str, Any]) -> ToolOutcome:
# Near-identical contents: still a different file each time.
return ToolOutcome(f"{arguments['path']}\n" + "x = 1\n" * 200, True)
model = Scripted(
*[[call("read_file", path=f"src/test_{n}.py")] for n in range(6)],
[call("post_review", body="ok")],
)
ran = Driver(model, "m").run(
transcript(tmp_path),
system="s",
tools=[READ, POST],
waiting={"post_review"},
workspace=Files(),
)
assert ran.kind == "wait"
def test_the_same_call_getting_the_same_back_ends_the_run(tmp_path: Path) -> None:
model = Scripted(*[[call("read_file", path="a.py")] for _ in range(9)])
ran = Driver(model, "m").run(
transcript(tmp_path),
system="s",
tools=[READ, POST],
waiting={"post_review"},
workspace=Reader(),
)
assert ran.kind == "circling"
assert ran.note.startswith("3 near-identical replies in a row")
assert 'read_file({"path": "a.py"})' in ran.note
def test_alike_commands_are_progress_not_a_circle(tmp_path: Path) -> None:
# git add a.py, b.py, c.py: alike, each answered the same, each its own.
class Git:
def dispatch(self, tool: str, arguments: dict[str, Any]) -> ToolOutcome:
return ToolOutcome("", True)
model = Scripted(
*[[call("run_command", command=f"git add {f}.py")] for f in "abcdef"],
[call("post_review", body="ok")],
)
ran = Driver(model, "m").run(
transcript(tmp_path),
system="s",
tools=[tool("run_command", "Run.", command="It."), POST],
waiting={"post_review"},
workspace=Git(),
)
assert ran.kind == "wait"
def test_circles_can_be_turned_off(tmp_path: Path) -> None:
model = Scripted(*[[call("post_comment", body="same")] for _ in range(5)])
ran = Driver(model, "m", max_turns=5, circle_repeats=0).run(
transcript(tmp_path),
system="s",
tools=[COMMENT, POST],
waiting={"post_review"},
workspace=None,
handlers={"post_comment": lambda a: ("posted", True)},
)
assert ran.kind == "capped"
def test_repeats_and_likeness_are_the_agents_to_set(tmp_path: Path) -> None:
model = Scripted(*[[call("post_comment", body="same")] for _ in range(5)])
ran = Driver(model, "m", circle_repeats=5).run(
transcript(tmp_path),
system="s",
tools=[COMMENT, POST],
waiting={"post_review"},
workspace=None,
handlers={"post_comment": lambda a: ("posted", True)},
)
assert ran.kind == "circling" and ran.note.startswith("5 near-identical")
def test_apologies_spread_over_a_run_dont_end_it(tmp_path: Path) -> None:
# Only flagged on the status page: sorry now and then is not a loop.
def turn(n: int, sorry: bool) -> list[CanonicalEvent]:
said = "Sorry, wrong file." if sorry else "Reading on."
return [AssistantText(text=said, model="m"), call("read_file", path=f"f{n}")]
model = Scripted(
*[turn(n, sorry=n % 2 == 0) for n in range(7)],
[call("post_review", body="ok")],
)
ran = Driver(model, "m").run(
transcript(tmp_path),
system="s",
tools=[READ, POST],
waiting={"post_review"},
workspace=Reader(),
)
assert ran.kind == "wait"