diff --git a/docs/Development-Plan.md b/docs/Development-Plan.md index f0b3406..6820b08 100644 --- a/docs/Development-Plan.md +++ b/docs/Development-Plan.md @@ -811,7 +811,12 @@ every agent. Seen 2026-10-24 on Flash-Next: invented tool names met with the prose reminder turn after turn, and a no-thinking run posting the same apology comment about 20 times until the turn cap. Repeated near-identical turns, or the same correction given too often, end the run as escalated, with what it was -doing; the limits are tuning. +doing; the limits are tuning. **Done** (this PR): `RunOutcome("circling")` +after the same calls getting much the same back, one correction, or apologies, +`tuning.*.circle_repeats` turns in a row (3). Near-identical calls only flag on +the page (`git add a.py`, `b.py`). nisse-dev calls in a helper as for a stuck +run, then asks; nisse-review counts a failed run and posts none of its drafts. +The helper's own loop keeps its caps. ### Phase 8 — `nisse-ops` diff --git a/docs/Tuning-Guide.md b/docs/Tuning-Guide.md index 1e11d70..298cb22 100644 --- a/docs/Tuning-Guide.md +++ b/docs/Tuning-Guide.md @@ -193,6 +193,28 @@ looking around, not doing the work. Before M2.8 this was (escalation is doing its job). Second, how often the local model recovers on its own after its second failure (the limit may be too low). +### `tuning.driver.circle_repeats` — default 3, and `circle_similar` — default 0.9 + +When a driver run going in circles ends (architecture L2, M7.8), escalated as +for `failure_limit`: a helper first, then you. A run circles when it makes +the same tool calls this many turns in a row and gets much the same back +each time, when the harness gives one correction this many times ("no such +tool: …"), or when it apologises this many turns in a row. `circle_similar` +is how alike two replies' text and two turns' results must be (0 to 1). 0 +for `circle_repeats` never ends a run this way; 1 isn't allowed, since every +run would end at its first turn. + +- **Why only the same calls:** `git add a.py`, then `b.py`, then `c.py` are + near-identical and each gets the same empty answer, but they are progress. + The status page flags near-identical turns too, and scattered apologies; + the driver ends a run only on what can't be progress. +- **Too low:** a model checking the same thing twice is cut off. 2 is the + least that means anything. +- **Too high:** a looping run spends that many more turns, and context. +- **Measured:** Qwen3.8-Flash-Next (2026-10-24) posted the same apology + comment about 20 times until the turn cap; 3 would have ended it at the + third. + ### `tuning.preflight.max_turns` — default 20 How many model calls one toolchain step may make (D3). The harness starts one @@ -673,6 +695,12 @@ Failed review runs on one PR, at one head, before the reviewer says on the PR that it couldn't review it, and stops trying until the next push or an operator's `review` comment. +### `tuning.review.circle_repeats` — default 3, and `circle_similar` — default 0.9 + +As `tuning.driver.circle_repeats`, for review runs. A review run going in +circles is a failed run (it counts toward `attempts`), and what it drafted is +dropped rather than posted: the drafts are the repeats. + ### `tuning.review.ci_wait_s` — default 3600 How long a new head's review waits while its checks are pending. A review diff --git a/nisse-core/src/nisse_core/config.py b/nisse-core/src/nisse_core/config.py index d17d93f..961c2b3 100644 --- a/nisse-core/src/nisse_core/config.py +++ b/nisse-core/src/nisse_core/config.py @@ -146,6 +146,16 @@ def parse_tuning( return tuning(**sections) +def check_circles(repeats: int, similar: float, where: str, errors: list[str]) -> None: + """A run ends going in circles (L2, M7.8) at ``repeats`` alike turns or + corrections: 0 turns that off, and 1 would end every run at its first + turn. ``similar`` is how alike two replies are, at most 1.""" + if repeats == 1: + errors.append(f"{where}.circle_repeats: must be 0 (off) or 2 or more") + if similar > 1: + errors.append(f"{where}.circle_similar: must be at most 1") + + def llm_numbers(values: Mapping[str, Any], errors: list[str]) -> tuple[int, float]: """`llm.max_output_tokens` and `llm.request_timeout_s`, or their defaults.""" diff --git a/nisse-core/src/nisse_core/loop/circles.py b/nisse-core/src/nisse_core/loop/circles.py index e9a897b..c6f77cb 100644 --- a/nisse-core/src/nisse_core/loop/circles.py +++ b/nisse-core/src/nisse_core/loop/circles.py @@ -1,8 +1,8 @@ """Runs going in circles (L2, X6): turns that repeat near-identically, one correction given again and again, or apologies over and over. -A pure function over a run's turns, so the status page can flag a loop now -and ``Driver.run`` can end one later (M7.8) from the same rules. +A pure function over a run's turns, so the status page flags a loop and +``Driver.run`` ends one (M7.8) from the same rules. """ import re @@ -29,6 +29,10 @@ class Turn: text: str = "" calls: tuple[tuple[str, str], ...] = () # (tool, arguments as sent) told: tuple[str, ...] = () # corrections given since the turn before + # What its calls got back. Two turns alike in their calls but not in + # what they got back, exactly (reading sibling files), are progress, not + # a circle; the driver gives them, the page doesn't yet. + results: tuple[str, ...] = () @dataclass(frozen=True) @@ -39,10 +43,17 @@ class Circle: def find_circles( - turns: Sequence[Turn], *, similar: float = SIMILAR, repeats: int = REPEATS + turns: Sequence[Turn], + *, + similar: float = SIMILAR, + repeats: int = REPEATS, + exact_calls: bool = False, ) -> list[Circle]: - """Every circle in a run, in the order they started.""" - found = _repeated_replies(turns, similar, repeats) + """Every circle in a run, in the order they started. ``exact_calls``: + replies repeat only when their tool calls are exactly the same, as the + driver judges before it ends a run; near-identical is enough for the + page to flag one (``git add a.py`` then ``b.py`` are near-identical).""" + found = _repeated_replies(turns, similar, repeats, exact_calls) found += _repeated_corrections(turns, repeats) apologies = tuple(i for i, t in enumerate(turns) if _APOLOGY.search(t.text)) if len(apologies) >= repeats: @@ -61,6 +72,33 @@ def _signature(turn: Turn) -> str: return f"{turn.text.strip()}\n{calls}"[:_COMPARED] +def _same_turn(a: Turn, b: Turn, similar: float, exact_calls: bool) -> bool: + """The same reply getting the same back. A model repeating itself gets + much the same answer each time ("noted (2 in the draft)"); alike calls + that got different answers are progress (two files of near-identical + contents are still two files).""" + if exact_calls: + joined = ("\n".join(t.results)[:_COMPARED] for t in (a, b)) + return ( + a.calls == b.calls + and _alike(*joined, similar) + and _alike(a.text.strip(), b.text.strip(), similar) + ) + return a.results == b.results and _alike(_signature(a), _signature(b), similar) + + +def apologies_in_a_row(turns: Sequence[Turn], repeats: int = REPEATS) -> Circle | None: + """The last ``repeats`` turns all apologising: a model saying sorry + again and again instead of moving on, as the driver ends a run for. + Apologies spread over a run are only flagged (``find_circles``).""" + last = range(len(turns) - repeats, len(turns)) + if repeats <= 0 or last.start < 0: + return None + if all(_APOLOGY.search(_signature(turns[i])) for i in last): + return Circle("repeated apology", tuple(last), f"{repeats} apologies in a row") + return None + + def _alike(a: str, b: str, similar: float) -> bool: if a == b: return True @@ -69,7 +107,7 @@ def _alike(a: str, b: str, similar: float) -> bool: def _repeated_replies( - turns: Sequence[Turn], similar: float, repeats: int + turns: Sequence[Turn], similar: float, repeats: int, exact_calls: bool ) -> list[Circle]: """Stretches of consecutive turns each alike the one before.""" found: list[Circle] = [] @@ -79,7 +117,7 @@ def _repeated_replies( if ( i < len(turns) and signatures[i].strip() - and _alike(signatures[i - 1], signatures[i], similar) + and _same_turn(turns[i - 1], turns[i], similar, exact_calls) ): continue if i - start >= repeats: diff --git a/nisse-core/src/nisse_core/loop/driver.py b/nisse-core/src/nisse_core/loop/driver.py index 9610474..baca17f 100644 --- a/nisse-core/src/nisse_core/loop/driver.py +++ b/nisse-core/src/nisse_core/loop/driver.py @@ -1,5 +1,5 @@ """A driver run (L2): one model with tools, looping until it calls a waiting -tool, runs out of turns, or is stopped. +tool, runs out of turns, goes in circles, or is stopped. Everything the run does is written to the item's transcript as the driver (L7), and every model call sees the driver's view of it (L6), compacted @@ -9,11 +9,13 @@ here, and a waiting one ends the run, so the scheduler can put it to the operator. Which tools wait is the agent's: each run is told. """ +import json from collections.abc import Callable, Collection, Mapping from dataclasses import dataclass from typing import Any, Literal, Protocol from nisse_core.log.canonical import AssistantText, Fact, ToolCall, ToolResult +from nisse_core.loop import circles from nisse_core.loop.tools import names from nisse_core.models import calls from nisse_core.models.compact import Compactor @@ -47,8 +49,12 @@ class Executor(Protocol): @dataclass(frozen=True) class RunOutcome: # stuck: failed commands in a row hit the failure limit (L9's trigger); - # unreachable: the tool VM went out of reach (#137), noted in ``note`` - kind: Literal["wait", "capped", "stuck", "stopped", "failed", "unreachable"] + # circling: the run repeated itself, or was told the same thing too often + # (L2, M7.8); unreachable: the tool VM went out of reach (#137). What + # happened is in ``note``. + kind: Literal[ + "wait", "capped", "stuck", "circling", "stopped", "failed", "unreachable" + ] tool: str = "" # the waiting tool called arguments: dict[str, Any] | None = None note: str = "" @@ -86,7 +92,11 @@ class Driver: cancel: Cancel | None = None, failure_limit: int = 0, # 0: no limit compactor: Compactor | None = None, + circle_repeats: int = circles.REPEATS, # 0: runs never end as circling + circle_similar: float = circles.SIMILAR, ) -> None: + self._circle_repeats = circle_repeats + self._circle_similar = circle_similar self._client = client self._compactor = compactor self._model = model @@ -140,6 +150,9 @@ class Driver: offered = names(tools) nudged = False cap = max_turns or self._max_turns + # Each turn as circles judge it (L2): the reply, what its calls got + # back, and the corrections the harness gave in answer. + turns: list[circles.Turn] = [] for turn in range(cap): # L2: stop takes effect at the next tool boundary. if self._cancel.requested: @@ -175,7 +188,10 @@ class Driver: # the user's side (models refuse assistant prefill), and the # nudge is then in the log as what it was. transcript.as_actor("orchestrator").append(Fact(text=_PROSE_NUDGE)) + turns.append(circles.Turn(text=said, told=(_PROSE_NUDGE,))) continue + results: list[str] = [] + told: list[str] = [] # results the harness gave, not the tools for index, call in enumerate(calls): if self._cancel.requested: skip_calls(transcript, calls[index:]) @@ -189,6 +205,8 @@ class Driver: transcript.append( ToolResult(call_id=call.call_id, result=problem, success=False) ) + results.append(problem) + told.append(problem) skip_calls( transcript, calls[index + 1 :], "not run: fix the call above" ) @@ -233,6 +251,7 @@ class Driver: + ", ".join(sorted(offered)), False, ) + told.append(result) else: screening = screen(call) if screen else Screening("run") if screening.decision == "ask": @@ -262,6 +281,8 @@ class Driver: success=False, ) ) + results.append(f"not run: {screening.text}") + told.append(f"not run: {screening.text}") skip_calls( transcript, calls[index + 1 :], @@ -281,6 +302,7 @@ class Driver: transcript.append( ToolResult(call_id=call.call_id, result=result, success=ok) ) + results.append(result) if call.tool == "run_command": failures = [] if ok else [*failures, _brief(call, result)] if self._failure_limit and len(failures) >= self._failure_limit: @@ -292,8 +314,53 @@ class Driver: note=f"{len(failures)} commands failed in a row: " + "; ".join(failures), ) + turns.append( + circles.Turn( + text=said, + calls=tuple((c.tool, _args(c)) for c in calls), + told=tuple(told), + results=tuple(results), + ) + ) + circling = self._circling(turns) + if circling is not None: + # In the log as the harness's, for whoever picks it up next. + transcript.as_actor("orchestrator").append( + Fact( + text=f"The run was ended: it was going in circles ({circling})." + ) + ) + return RunOutcome("circling", note=circling) return RunOutcome("capped", note=f"hit the {cap}-turn cap") + def _circling(self, turns: list[circles.Turn]) -> str | None: + """What the run is going in circles on, if it is (L2): the same + calls getting the same back, one correction given again and again, + or apologies in a row. Near-identical calls and scattered apologies + are only flagged, on the status page: ``git add a.py`` then + ``b.py`` is progress.""" + if self._circle_repeats <= 0: + return None + found = [ + c + for c in circles.find_circles( + turns, + similar=self._circle_similar, + repeats=self._circle_repeats, + exact_calls=True, + ) + if c.kind != "repeated apology" + ] + found.append(circles.apologies_in_a_row(turns, self._circle_repeats)) + for circle in found: + if circle is None: + continue + last = turns[circle.turns[-1]] + doing = "; ".join(f"{tool}({args})" for tool, args in last.calls) + doing = doing or last.text + return f"{circle.detail}; the last: {doing[:300]}" + return None + @staticmethod def _unreachable( transcript: Transcript, calls: list[ToolCall], index: int, said: str @@ -309,6 +376,10 @@ class Driver: return RunOutcome("unreachable", note=said) +def _args(call: ToolCall) -> str: + return json.dumps(dict(call.arguments), sort_keys=True, ensure_ascii=False) + + def _brief(call: ToolCall, result: str) -> str: """A failed command and the end of what it said, for a helper's brief.""" command = str(call.arguments.get("command", ""))[:120] diff --git a/nisse-core/tests/test_circles.py b/nisse-core/tests/test_circles.py index c94c8c8..2fdcccf 100644 --- a/nisse-core/tests/test_circles.py +++ b/nisse-core/tests/test_circles.py @@ -1,6 +1,6 @@ """Runs going in circles (L2, X6), judged from the turns alone.""" -from nisse_core.loop.circles import Turn, find_circles +from nisse_core.loop.circles import Turn, apologies_in_a_row, find_circles NUDGE = "Your last reply was prose. Every reply must be a tool call." @@ -71,3 +71,46 @@ def test_the_limits_are_tuning() -> None: turns = [comment(1), comment(2)] assert [c.kind for c in find_circles(turns, repeats=2)] == ["repeated reply"] assert find_circles([comment(1), comment(22), comment(333)], similar=1.0) == [] + + +def test_alike_replies_that_got_different_results_are_no_circle() -> None: + read = [("read_file", '{"path": "src/a.py"}')] + turns = [Turn(calls=tuple(read), results=(f"file {n}",)) for n in range(4)] + assert find_circles(turns) == [] + same = [Turn(calls=tuple(read), results=("posted",)) for _ in range(4)] + assert [c.kind for c in find_circles(same)] == ["repeated reply"] + # The page gives no results: alike replies alone are a circle there. + bare = [Turn(calls=tuple(read)) for _ in range(3)] + assert [c.kind for c in find_circles(bare)] == ["repeated reply"] + + +def test_the_driver_counts_only_exactly_repeated_calls() -> None: + adds = [ + Turn(calls=(("run_command", f'{{"command": "git add {f}.py"}}'),)) + for f in "abc" + ] + assert [c.kind for c in find_circles(adds)] == ["repeated reply"] # flagged + assert find_circles(adds, exact_calls=True) == [] # never ended for it + same = [Turn(calls=(("read_file", '{"path": "a"}'),))] * 3 + assert len(find_circles(same, exact_calls=True)) == 1 + + +def test_apologies_end_a_run_only_in_a_row() -> None: + sorry, work = Turn(text="Sorry, my mistake."), Turn(text="Reading the file.") + assert apologies_in_a_row([work, sorry, sorry, sorry]) is not None + assert apologies_in_a_row([sorry, work, sorry, sorry]) is None + assert apologies_in_a_row([sorry, sorry]) is None + # In what it posted, too: an apology comment is an apology. + posted = Turn(calls=(("post_comment", '{"body": "Please disregard (1)."}'),)) + assert apologies_in_a_row([posted] * 3) is not None + + +def test_the_same_call_answered_with_a_count_is_still_a_circle() -> None: + note = (("comment", '{"line": 2, "text": "Same point."}'),) + turns = [Turn(calls=note, results=(f"noted ({n} in the draft)",)) for n in range(3)] + assert len(find_circles(turns, exact_calls=True)) == 1 + files = [ + Turn(calls=(("read_file", f'{{"path": "{f}"}}'),), results=(f"{f}: x",)) + for f in "abc" + ] + assert find_circles(files, exact_calls=True) == [] diff --git a/nisse-core/tests/test_driver.py b/nisse-core/tests/test_driver.py index e381699..ecb902b 100644 --- a/nisse-core/tests/test_driver.py +++ b/nisse-core/tests/test_driver.py @@ -5,7 +5,13 @@ import io from pathlib import Path from typing import Any -from nisse_core.log.canonical import CanonicalEvent, ToolCall, ToolResult +from nisse_core.log.canonical import ( + AssistantText, + CanonicalEvent, + Fact, + ToolCall, + ToolResult, +) from nisse_core.loop.driver import Driver, ToolOutcome from nisse_core.loop.tools import tool from nisse_core.oplog import OpLog @@ -150,3 +156,156 @@ def test_without_due_a_settled_point_leaves_the_run_going(tmp_path: Path) -> Non handlers={"note": lambda args: ("noted", True)}, ) assert (ran.tool, ran.arguments) == ("post_review", {"body": "b"}) + + +# --- runs going in circles (L2, M7.8) ------------------------------------------ + +COMMENT = tool("post_comment", "Post a comment.", body="The comment.") + + +def facts(log: Any) -> list[str]: + return [e.event.text for e in log.read() if isinstance(e.event, Fact)] + + +def test_a_run_posting_the_same_comment_ends_as_circling(tmp_path: Path) -> None: + # Flash-Next on 2026-10-24: the same apology, numbered, about 20 times + # until the turn cap. + posted: list[str] = [] + model = Scripted( + *[ + [call("post_comment", body=f"Duplicate, please disregard ({n}).")] + for n in range(20) + ] + ) + log = transcript(tmp_path) + ran = Driver(model, "m", max_turns=30).run( + log, + system="s", + tools=[COMMENT, POST], + waiting={"post_review"}, + workspace=None, + handlers={ + "post_comment": lambda a: (posted.append(a["body"]) or "posted", True) + }, + ) + assert ran.kind == "circling" + assert len(posted) == 3 # ended at the third, not the thirtieth + assert ran.note.startswith("3 apologies in a row") + assert "Duplicate, please disregard (2)" in ran.note # what it was doing + assert facts(log) == [f"The run was ended: it was going in circles ({ran.note})."] + + +def test_an_invented_tool_told_again_and_again_ends_the_run(tmp_path: Path) -> None: + # A different reply each time, but the same correction every time. + model = Scripted( + *[[call("search_code", query=f"attempt {n} " * n)] for n in range(1, 10)] + ) + ran = Driver(model, "m").run( + transcript(tmp_path), + system="s", + tools=[READ, POST], + waiting={"post_review"}, + workspace=Reader(), + ) + assert ran.kind == "circling" + assert ran.note.startswith("told 3 times: no such tool: search_code") + + +def test_reading_file_after_file_is_progress_not_a_circle(tmp_path: Path) -> None: + class Files: + def dispatch(self, tool: str, arguments: dict[str, Any]) -> ToolOutcome: + # Near-identical contents: still a different file each time. + return ToolOutcome(f"{arguments['path']}\n" + "x = 1\n" * 200, True) + + model = Scripted( + *[[call("read_file", path=f"src/test_{n}.py")] for n in range(6)], + [call("post_review", body="ok")], + ) + ran = Driver(model, "m").run( + transcript(tmp_path), + system="s", + tools=[READ, POST], + waiting={"post_review"}, + workspace=Files(), + ) + assert ran.kind == "wait" + + +def test_the_same_call_getting_the_same_back_ends_the_run(tmp_path: Path) -> None: + model = Scripted(*[[call("read_file", path="a.py")] for _ in range(9)]) + ran = Driver(model, "m").run( + transcript(tmp_path), + system="s", + tools=[READ, POST], + waiting={"post_review"}, + workspace=Reader(), + ) + assert ran.kind == "circling" + assert ran.note.startswith("3 near-identical replies in a row") + assert 'read_file({"path": "a.py"})' in ran.note + + +def test_alike_commands_are_progress_not_a_circle(tmp_path: Path) -> None: + # git add a.py, b.py, c.py: alike, each answered the same, each its own. + class Git: + def dispatch(self, tool: str, arguments: dict[str, Any]) -> ToolOutcome: + return ToolOutcome("", True) + + model = Scripted( + *[[call("run_command", command=f"git add {f}.py")] for f in "abcdef"], + [call("post_review", body="ok")], + ) + ran = Driver(model, "m").run( + transcript(tmp_path), + system="s", + tools=[tool("run_command", "Run.", command="It."), POST], + waiting={"post_review"}, + workspace=Git(), + ) + assert ran.kind == "wait" + + +def test_circles_can_be_turned_off(tmp_path: Path) -> None: + model = Scripted(*[[call("post_comment", body="same")] for _ in range(5)]) + ran = Driver(model, "m", max_turns=5, circle_repeats=0).run( + transcript(tmp_path), + system="s", + tools=[COMMENT, POST], + waiting={"post_review"}, + workspace=None, + handlers={"post_comment": lambda a: ("posted", True)}, + ) + assert ran.kind == "capped" + + +def test_repeats_and_likeness_are_the_agents_to_set(tmp_path: Path) -> None: + model = Scripted(*[[call("post_comment", body="same")] for _ in range(5)]) + ran = Driver(model, "m", circle_repeats=5).run( + transcript(tmp_path), + system="s", + tools=[COMMENT, POST], + waiting={"post_review"}, + workspace=None, + handlers={"post_comment": lambda a: ("posted", True)}, + ) + assert ran.kind == "circling" and ran.note.startswith("5 near-identical") + + +def test_apologies_spread_over_a_run_dont_end_it(tmp_path: Path) -> None: + # Only flagged on the status page: sorry now and then is not a loop. + def turn(n: int, sorry: bool) -> list[CanonicalEvent]: + said = "Sorry, wrong file." if sorry else "Reading on." + return [AssistantText(text=said, model="m"), call("read_file", path=f"f{n}")] + + model = Scripted( + *[turn(n, sorry=n % 2 == 0) for n in range(7)], + [call("post_review", body="ok")], + ) + ran = Driver(model, "m").run( + transcript(tmp_path), + system="s", + tools=[READ, POST], + waiting={"post_review"}, + workspace=Reader(), + ) + assert ran.kind == "wait" diff --git a/nisse-dev.example.yaml b/nisse-dev.example.yaml index 2ea2a8d..c3161f9 100644 --- a/nisse-dev.example.yaml +++ b/nisse-dev.example.yaml @@ -100,6 +100,8 @@ tuning: driver: max_turns: 40 # model calls per driver run before it stops as capped failure_limit: 2 # consecutive failed commands before escalating (L9) + circle_repeats: 3 # a run repeating itself this often ends, escalated (L2); 0: never + circle_similar: 0.9 # how alike its repeats must be preflight: # the toolchain step (D3) max_turns: 20 # model calls one toolchain step may make command_timeout_s: 900 # one command in it: installs take a while diff --git a/nisse-dev/src/nisse_dev/bench/runner.py b/nisse-dev/src/nisse_dev/bench/runner.py index f76572a..222c81f 100644 --- a/nisse-dev/src/nisse_dev/bench/runner.py +++ b/nisse-dev/src/nisse_dev/bench/runner.py @@ -266,6 +266,8 @@ def run_task( model, max_turns=tuning.driver.max_turns, failure_limit=tuning.driver.failure_limit, + circle_repeats=tuning.driver.circle_repeats, + circle_similar=tuning.driver.circle_similar, # As the agent runs it, so a long task measures the same thing. compactor=Compactor(meter, tuning.context.compaction()), ) diff --git a/nisse-dev/src/nisse_dev/config.py b/nisse-dev/src/nisse_dev/config.py index 4d67003..87dad1b 100644 --- a/nisse-dev/src/nisse_dev/config.py +++ b/nisse-dev/src/nisse_dev/config.py @@ -25,6 +25,7 @@ from nisse_core.config import ( REFERENCE, ConfigError, VaultConfig, + check_circles, flatten, llm_numbers, parse_tuning, @@ -107,6 +108,10 @@ GENERAL = "general" # the scope of conversation not about a project (L1) class DriverTuning: max_turns: int = 40 failure_limit: int = 2 + # A run going in circles ends (L2): this many alike turns in a row, or + # one correction given this often; 0 never ends one. And how alike. + circle_repeats: int = field(default=3, metadata={"zero": True}) + circle_similar: float = 0.9 @dataclass(frozen=True) @@ -199,6 +204,12 @@ def _parse_tuning(raw: Any, errors: list[str]) -> Tuning: if tuning.context.safety_margin >= 1: errors.append("tuning.context.safety_margin: must be less than 1") check_tuning(tuning.context.compaction(), "tuning.context", errors) + check_circles( + tuning.driver.circle_repeats, + tuning.driver.circle_similar, + "tuning.driver", + errors, + ) return tuning diff --git a/nisse-dev/src/nisse_dev/loop/agent.py b/nisse-dev/src/nisse_dev/loop/agent.py index 329e6d9..3834b31 100644 --- a/nisse-dev/src/nisse_dev/loop/agent.py +++ b/nisse-dev/src/nisse_dev/loop/agent.py @@ -2378,11 +2378,12 @@ class LoopAgent: def _help_when_stuck(self, item: WorkItem, loop: Loop, outcome: RunOutcome) -> bool: """L9's observable triggers: the driver stuck (commands failing in a - row) or out of turns mid-milestone. A helper is called in, once per - milestone; True when it finished and the driver goes on.""" + row), going in circles (L2), or out of turns mid-milestone. A helper + is called in, once per milestone; True when it finished and the + driver goes on.""" if ( self._helpers is None - or outcome.kind not in ("stuck", "capped") + or outcome.kind not in ("stuck", "circling", "capped") or not loop.branch or loop.helped_at == loop.milestone ): diff --git a/nisse-dev/src/nisse_dev/loop/build.py b/nisse-dev/src/nisse_dev/loop/build.py index dbab37d..28abbb6 100644 --- a/nisse-dev/src/nisse_dev/loop/build.py +++ b/nisse-dev/src/nisse_dev/loop/build.py @@ -292,6 +292,8 @@ def build_loop_agent( max_turns=config.tuning.driver.max_turns, cancel=cancel, failure_limit=config.tuning.driver.failure_limit, + circle_repeats=config.tuning.driver.circle_repeats, + circle_similar=config.tuning.driver.circle_similar, compactor=Compactor( normalized, config.tuning.context.compaction(), oplog=oplog ), diff --git a/nisse-dev/tests/test_config.py b/nisse-dev/tests/test_config.py index 1dd8ab7..d0c1fb3 100644 --- a/nisse-dev/tests/test_config.py +++ b/nisse-dev/tests/test_config.py @@ -182,6 +182,23 @@ def test_tuning_is_read() -> None: assert (tuning.build_vm.retry_s, tuning.build_vm.settle_s) == (60.0, 180.0) +def test_circles_are_tuned_and_checked() -> None: + # L2, M7.8: when a driver run going in circles ends. + assert ( + build().tuning.driver.circle_repeats, + build().tuning.driver.circle_similar, + ) == (3, 0.9) + tuned = build(tuning={"driver": {"circle_repeats": 0, "circle_similar": 0.8}}) + assert (tuned.tuning.driver.circle_repeats, tuned.tuning.driver.circle_similar) == ( + 0, + 0.8, + ) + with pytest.raises(ConfigError, match=r"circle_repeats: must be 0 \(off\) or 2"): + build(tuning={"driver": {"circle_repeats": 1}}) + with pytest.raises(ConfigError, match="circle_similar: must be at most 1"): + build(tuning={"driver": {"circle_similar": 2}}) + + def test_a_partly_given_tuning_section_keeps_the_other_defaults() -> None: tuning = build(tuning={"driver": {"max_turns": 40}}).tuning assert tuning.driver.failure_limit == 2 diff --git a/nisse-dev/tests/test_loop.py b/nisse-dev/tests/test_loop.py index b339692..c67f669 100644 --- a/nisse-dev/tests/test_loop.py +++ b/nisse-dev/tests/test_loop.py @@ -386,7 +386,8 @@ def test_an_invented_tool_is_answered_not_run(tmp_path: Path) -> None: def test_the_turn_cap_ends_a_run(tmp_path: Path) -> None: model = ScriptedModel(*[[call("read_file", path="a")]] * 3) - outcome = Driver(model, "m", max_turns=3).run( + # The same call three times is a circle too (M7.8): here, only the cap. + outcome = Driver(model, "m", max_turns=3, circle_repeats=0).run( driver_log(tmp_path), system="s", tools=_tools("read_file"), diff --git a/nisse-dev/tests/test_loop_help.py b/nisse-dev/tests/test_loop_help.py index 536f975..1ceab9d 100644 --- a/nisse-dev/tests/test_loop_help.py +++ b/nisse-dev/tests/test_loop_help.py @@ -249,6 +249,49 @@ def test_stuck_again_in_the_same_milestone_asks_the_operator( assert len(helper.calls) == 2 # one delegation: its two turns +def test_a_driver_going_in_circles_gets_a_helper_too(world: dict[str, Any]) -> None: + # L2, M7.8: the same command, the same output, turn after turn. + helper = Spy( + [call("write_file", path="fixed.py", content="F = 1\n")], + [hand_back()], + ) + model = ScriptedModel( + RESTATE, + [call("start_milestone", name="words")], + *[[call("run_command", command="true")] for _ in range(3)], + [call("finish_milestone", summary="fixed with help", final="yes")], + ) + agent, transport = loop_agent(world, model, helpers=helpers(helper)) + agent.intake() + say(transport, world, "go") + + assert any("so a helper is taking over" in m.text for m in transport.outbound) + assert "near-identical replies in a row" in str(helper.calls) # in the brief + assert loop_of(item(world)).helped_at == 1 + + +def test_circling_again_in_the_same_milestone_asks_the_operator( + world: dict[str, Any], +) -> None: + helper = Spy( + [call("write_file", path="tried.py", content="T = 1\n")], + [hand_back()], + ) + circling = [[call("run_command", command="true")] for _ in range(3)] + model = ScriptedModel( + RESTATE, + [call("start_milestone", name="words")], + *circling, + *circling, # after the helper: round again + ) + agent, transport = loop_agent(world, model, helpers=helpers(helper)) + agent.intake() + say(transport, world, "go") + loop = loop_of(item(world)) + assert loop.wait == ANSWER + assert "I stopped short: 3 near-identical replies in a row" in loop.note + + def test_ask_expert_answers_within_its_budget(world: dict[str, Any]) -> None: answer = {"answer": "use a dict", "confidence": "high", "caveats": ""} expert = Spy( diff --git a/nisse-dev/tests/test_loop_memory.py b/nisse-dev/tests/test_loop_memory.py index 14b3e99..ace6f8e 100644 --- a/nisse-dev/tests/test_loop_memory.py +++ b/nisse-dev/tests/test_loop_memory.py @@ -139,8 +139,10 @@ def test_a_pass_that_doesnt_finish_never_holds_up_the_close( def test_the_pass_is_bounded(world: dict[str, Any], memory: Memory) -> None: - looking = [call("list_notes", folder="")] - model = Spy(RESTATE, *[looking] * 3) + # A different folder each turn: the same call three times would end the + # pass as going in circles (M7.8), not at the cap. + looking = [[call("list_notes", folder=f)] for f in ("", "a", "b")] + model = Spy(RESTATE, *looking) loop, transport = agent(world, model, memory, memory_turns=3) loop.intake() say(transport, world, "abandon") diff --git a/nisse-review.example.yaml b/nisse-review.example.yaml index c95db0d..5bc701a 100644 --- a/nisse-review.example.yaml +++ b/nisse-review.example.yaml @@ -36,6 +36,7 @@ gitea: # tuning: # docs/Tuning-Guide.md # review: {poll_s: 60, max_turns: 40, diff_chars: 60000, attempts: 2, -# ci_wait_s: 3600, ci_grace_s: 120, max_rounds: 4} +# ci_wait_s: 3600, ci_grace_s: 120, max_rounds: 4, +# circle_repeats: 3, circle_similar: 0.9} # expert: {consult_budget: 3} # context: {keep_recent_turns: 6, safety_margin: 0.1} diff --git a/nisse-review/src/nisse_review/build.py b/nisse-review/src/nisse_review/build.py index a372c8a..bf56997 100644 --- a/nisse-review/src/nisse_review/build.py +++ b/nisse-review/src/nisse_review/build.py @@ -138,6 +138,8 @@ def build( config.roles.resolve("driver"), cancel=cancel, compactor=Compactor(model, tuning.context.compaction(), oplog=oplog), + circle_repeats=tuning.review.circle_repeats, + circle_similar=tuning.review.circle_similar, ), account=account, gitea_url=config.gitea_url, diff --git a/nisse-review/src/nisse_review/config.py b/nisse-review/src/nisse_review/config.py index 4553d98..583bb87 100644 --- a/nisse-review/src/nisse_review/config.py +++ b/nisse-review/src/nisse_review/config.py @@ -14,6 +14,7 @@ from typing import Any from nisse_core.config import ( ConfigError, VaultConfig, + check_circles, flatten, llm_numbers, parse_tuning, @@ -64,6 +65,10 @@ class ReviewTuning: ci_grace_s: float = field(default=120.0, metadata={"zero": True}) # Reviews and reply sets before it sums up for the operator (R2). max_rounds: int = 4 + # A run going in circles ends (L2): this many alike turns in a row, or + # one correction given this often; 0 never ends one. And how alike. + circle_repeats: int = field(default=3, metadata={"zero": True}) + circle_similar: float = 0.9 @dataclass(frozen=True) @@ -136,6 +141,12 @@ class ReviewConfig: if tuning.context.safety_margin >= 1: errors.append("tuning.context.safety_margin: must be less than 1") check_tuning(tuning.context.compaction(), "tuning.context", errors) + check_circles( + tuning.review.circle_repeats, + tuning.review.circle_similar, + "tuning.review", + errors, + ) vault = parse_vault(tree.get("vault"), errors) flat = flatten({k: v for k, v in tree.items() if k != "vault"}) flat.pop("tuning", None) diff --git a/nisse-review/tests/test_review.py b/nisse-review/tests/test_review.py index a9dc331..1df800e 100644 --- a/nisse-review/tests/test_review.py +++ b/nisse-review/tests/test_review.py @@ -492,6 +492,29 @@ def test_the_ci_waits_may_be_zero(tmp_path: Path) -> None: } review = ReviewConfig.from_tree(tree, {}).tuning.review assert (review.ci_wait_s, review.ci_grace_s) == (0, 0) + assert (review.circle_repeats, review.circle_similar) == (3, 0.9) + + +def test_circles_are_tuned_in_the_config(tmp_path: Path) -> None: + from nisse_core.config import ConfigError + from nisse_review.config import ReviewConfig + + tree = { + "state_db": str(tmp_path / "s.db"), + "llm": {"base_url": "http://llm", "token": "k"}, + "roles": {"reviewer": "local", "expert": "frontier"}, + "gitea": {"url": "https://g", "token": "t", "operators": ["don"]}, + } + off = {**tree, "tuning": {"review": {"circle_repeats": 0}}} + assert ReviewConfig.from_tree(off, {}).tuning.review.circle_repeats == 0 + for bad, message in ( + ({"circle_repeats": 1}, "circle_repeats: must be 0 (off) or 2 or more"), + ({"circle_similar": 1.5}, "circle_similar: must be at most 1"), + ): + with pytest.raises( + ConfigError, match=message.replace("(", r"\(").replace(")", r"\)") + ): + ReviewConfig.from_tree({**tree, "tuning": {"review": bad}}, {}) def test_compaction_is_tuned_in_the_config(tmp_path: Path) -> None: @@ -627,6 +650,20 @@ def test_out_of_turns_with_nothing_drafted_is_a_failed_run(tmp_path: Path) -> No assert (item(w).position, pr_of(item(w)).attempts) == (READY, 1) +def test_a_run_going_in_circles_fails_and_its_repeats_are_not_posted( + tmp_path: Path, +) -> None: + # L2, M7.8: the same comment drafted again and again ends the run as + # circling, a failed run; its drafts are the repeats, so none is posted. + again = [call("comment", path="count.py", line="2", text="Same point.")] + model = Scripted(again, again, again, again, again) + w = world(tmp_path, model, attempts=2) + w["scheduler"].round() + assert w["gitea"].posted == [] + assert (item(w).position, pr_of(item(w)).attempts) == (READY, 1) + assert len(model.seen) == 3 # ended at the third, not at the cap + + def test_code_git_cannot_give_fails_the_run(tmp_path: Path) -> None: from nisse_review.clones import CloneError