Runs that go in circles end (M7.8)
Driver.run checks each turn with nisse_core.loop.circles and ends the run as "circling" when the same calls get much the same back, one harness correction is given again and again, or the model apologises, for circle_repeats turns in a row (default 3). Near-identical calls (git add a.py, b.py) and scattered apologies stay flags on the status page. A fact in the transcript says why the run ended. nisse-dev calls in a helper as for a stuck run, then asks the operator; nisse-review counts a failed run and posts none of its drafts. Tuning: tuning.driver / tuning.review circle_repeats (0 off) and circle_similar. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01J5Utyfzw7gF7UhjRDEtjTk
This commit is contained in:
1 parent
516a328618
commit
74bc68db1c
20 files changed
+506
-20
No files matched your search
@@ -138,6 +138,8 @@ def build(
|
||||
config.roles.resolve("driver"),
|
||||
cancel=cancel,
|
||||
compactor=Compactor(model, tuning.context.compaction(), oplog=oplog),
|
||||
circle_repeats=tuning.review.circle_repeats,
|
||||
circle_similar=tuning.review.circle_similar,
|
||||
),
|
||||
account=account,
|
||||
gitea_url=config.gitea_url,
|
||||
|
||||
@@ -14,6 +14,7 @@ from typing import Any
|
||||
from nisse_core.config import (
|
||||
ConfigError,
|
||||
VaultConfig,
|
||||
check_circles,
|
||||
flatten,
|
||||
llm_numbers,
|
||||
parse_tuning,
|
||||
@@ -64,6 +65,10 @@ class ReviewTuning:
|
||||
ci_grace_s: float = field(default=120.0, metadata={"zero": True})
|
||||
# Reviews and reply sets before it sums up for the operator (R2).
|
||||
max_rounds: int = 4
|
||||
# A run going in circles ends (L2): this many alike turns in a row, or
|
||||
# one correction given this often; 0 never ends one. And how alike.
|
||||
circle_repeats: int = field(default=3, metadata={"zero": True})
|
||||
circle_similar: float = 0.9
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
@@ -136,6 +141,12 @@ class ReviewConfig:
|
||||
if tuning.context.safety_margin >= 1:
|
||||
errors.append("tuning.context.safety_margin: must be less than 1")
|
||||
check_tuning(tuning.context.compaction(), "tuning.context", errors)
|
||||
check_circles(
|
||||
tuning.review.circle_repeats,
|
||||
tuning.review.circle_similar,
|
||||
"tuning.review",
|
||||
errors,
|
||||
)
|
||||
vault = parse_vault(tree.get("vault"), errors)
|
||||
flat = flatten({k: v for k, v in tree.items() if k != "vault"})
|
||||
flat.pop("tuning", None)
|
||||
|
||||
@@ -492,6 +492,29 @@ def test_the_ci_waits_may_be_zero(tmp_path: Path) -> None:
|
||||
}
|
||||
review = ReviewConfig.from_tree(tree, {}).tuning.review
|
||||
assert (review.ci_wait_s, review.ci_grace_s) == (0, 0)
|
||||
assert (review.circle_repeats, review.circle_similar) == (3, 0.9)
|
||||
|
||||
|
||||
def test_circles_are_tuned_in_the_config(tmp_path: Path) -> None:
|
||||
from nisse_core.config import ConfigError
|
||||
from nisse_review.config import ReviewConfig
|
||||
|
||||
tree = {
|
||||
"state_db": str(tmp_path / "s.db"),
|
||||
"llm": {"base_url": "http://llm", "token": "k"},
|
||||
"roles": {"reviewer": "local", "expert": "frontier"},
|
||||
"gitea": {"url": "https://g", "token": "t", "operators": ["don"]},
|
||||
}
|
||||
off = {**tree, "tuning": {"review": {"circle_repeats": 0}}}
|
||||
assert ReviewConfig.from_tree(off, {}).tuning.review.circle_repeats == 0
|
||||
for bad, message in (
|
||||
({"circle_repeats": 1}, "circle_repeats: must be 0 (off) or 2 or more"),
|
||||
({"circle_similar": 1.5}, "circle_similar: must be at most 1"),
|
||||
):
|
||||
with pytest.raises(
|
||||
ConfigError, match=message.replace("(", r"\(").replace(")", r"\)")
|
||||
):
|
||||
ReviewConfig.from_tree({**tree, "tuning": {"review": bad}}, {})
|
||||
|
||||
|
||||
def test_compaction_is_tuned_in_the_config(tmp_path: Path) -> None:
|
||||
@@ -627,6 +650,20 @@ def test_out_of_turns_with_nothing_drafted_is_a_failed_run(tmp_path: Path) -> No
|
||||
assert (item(w).position, pr_of(item(w)).attempts) == (READY, 1)
|
||||
|
||||
|
||||
def test_a_run_going_in_circles_fails_and_its_repeats_are_not_posted(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
# L2, M7.8: the same comment drafted again and again ends the run as
|
||||
# circling, a failed run; its drafts are the repeats, so none is posted.
|
||||
again = [call("comment", path="count.py", line="2", text="Same point.")]
|
||||
model = Scripted(again, again, again, again, again)
|
||||
w = world(tmp_path, model, attempts=2)
|
||||
w["scheduler"].round()
|
||||
assert w["gitea"].posted == []
|
||||
assert (item(w).position, pr_of(item(w)).attempts) == (READY, 1)
|
||||
assert len(model.seen) == 3 # ended at the third, not at the cap
|
||||
|
||||
|
||||
def test_code_git_cannot_give_fails_the_run(tmp_path: Path) -> None:
|
||||
from nisse_review.clones import CloneError
|
||||
|
||||
|
||||
Reference in new issue
Block a user