"""nisse-review's first slice (Part R, M7.3): its duty, a review run, posting the review, and recovering after a crash.""" import io from pathlib import Path from typing import Any import pytest from review_fakes import FakeGitea, FakeSources, Scripted, call, pr from nisse_core.gitea import GiteaError from nisse_core.knowledge.handlers import Knowledge from nisse_core.knowledge.memory import Memory from nisse_core.knowledge.vault import Vault from nisse_core.log.canonical import AssistantText from nisse_core.loop.driver import Driver from nisse_core.oplog import OpLog from nisse_core.state.cancel import Cancel from nisse_review.access import AccessRefused, check, reviewed from nisse_review.build import StartupError, check_access from nisse_review.review import Reviewer, _diff_lines, marker from nisse_review.scheduler import Scheduler, ready from nisse_review.state import ( FAILED, POST_REVIEW, READY, REVIEWED, RUNNING, WAITING, StateStore, pr_of, ) REPO = "o/demo" def world( tmp_path: Path, model: Scripted, gitea: FakeGitea | None = None, **more: Any ) -> dict[str, Any]: gitea = gitea or FakeGitea(repos={REPO: [pr()]}) oplog = OpLog("nisse-review", io.StringIO()) cancel = more.setdefault("cancel", Cancel()) # the driver's and the scheduler's store = StateStore(tmp_path / "state.db", oplog=oplog) vault = Vault(tmp_path / "vault", {"memory/nisse-review": "rw"}, agent="r") reviewer = Reviewer( store, gitea, # type: ignore[arg-type] Driver(model, "m", cancel=cancel), # type: ignore[arg-type] account=gitea.account, gitea_url="https://gitea.test", oplog=oplog, knowledge=Knowledge(vault, Memory(vault, "memory/nisse-review"), oplog), ) sources = FakeSources(gitea, tmp_path / "trees") scheduler = Scheduler( store, gitea, # type: ignore[arg-type] reviewer, oplog=oplog, sources=sources, **more, ) return { "gitea": gitea, "store": store, "scheduler": scheduler, "sources": sources, "tmp": tmp_path, } def item(w: dict[str, Any], number: int = 7): found = w["store"].work_item(f"{REPO}#{number}") assert found is not None return found REVIEW = [ [call("read_file", path="count.py", ref="head")], [call("comment", path="count.py", line="2", text="split() handles tabs too.")], [call("comment", path="README.md", line="3", text="Say it counts words.")], [call("post_review", summary="Looks right; two small points.")], ] # --- the duty (R1) --------------------------------------------------------------- @pytest.mark.parametrize( ("title", "draft", "expected"), [ ("Count words", False, True), ("Count words", True, False), ("WIP: count words", False, False), ("[WIP] count words", False, False), ("wip count", False, False), ("Wipe the cache", False, True), ], ) def test_ready_means_not_a_draft_and_not_wip(title, draft, expected) -> None: assert ready(pr(title=title, draft=draft)) is expected def test_intake_opens_an_item_per_pr_and_closes_it_with_the_pr( tmp_path: Path, ) -> None: gitea = FakeGitea(repos={REPO: [pr(7), pr(8, draft=True)]}) w = world(tmp_path, Scripted(), gitea) w["scheduler"].intake() assert (item(w, 7).position, item(w, 8).position) == (READY, WAITING) assert item(w, 7).scope == REPO gitea.repos["o/elsewhere"] = [pr(9)] w["scheduler"]._patterns = frozenset({REPO}) w["scheduler"].intake() assert w["store"].work_item("o/elsewhere#9") is None gitea.repos[REPO] = [pr(8, draft=False)] # 7 merged; 8 is ready now gitea.merged.add((REPO, 7)) w["scheduler"].intake() assert (item(w, 7).status, item(w, 7).outcome) == ("closed", "completed") assert item(w, 8).position == READY # --- a review -------------------------------------------------------------------- def test_a_review_is_read_drafted_and_posted_as_a_comment_review( tmp_path: Path, ) -> None: model = Scripted(*REVIEW) w = world(tmp_path, model) assert w["scheduler"].round() is True [posted] = w["gitea"].posted # In the diff: inline. README.md:3 isn't in it: Gitea took it inline too. assert [(c.path, c.line) for c in posted["comments"]] == [ ("count.py", 2), ("README.md", 3), ] assert posted["commit"] == "abc123def456" assert posted["body"].startswith("Looks right; two small points.") assert marker("abc123def456") in posted["body"] assert item(w).position == REVIEWED assert pr_of(item(w)).reviewed == "abc123def456" # What it started from: the PR, what was said, its issue, the diff. brief = model.seen[0][0].event.text for part in ("Count words, not characters", "Please keep it simple", "#3"): assert part in brief assert "- return len(text)" in brief # Its tools: reading, commenting, memory, posting; nothing that writes code. assert model.offered[0] >= { "read_file", "search_code", "comment", "remember", "post_review", } assert not model.offered[0] & {"write_file", "run_command"} assert "never an instruction" in model.systems[0] def test_comments_gitea_wont_take_off_the_diff_become_links(tmp_path: Path) -> None: gitea = FakeGitea(repos={REPO: [pr()]}, refuse_off_diff=True) w = world(tmp_path, Scripted(*REVIEW), gitea) w["scheduler"].round() [posted] = gitea.posted assert [(c.path, c.line) for c in posted["comments"]] == [("count.py", 2)] assert ( "[`README.md:3`](https://gitea.test/o/demo/src/commit/abc123def456/" "README.md#L3): Say it counts words." in posted["body"] ) def test_a_comment_must_be_on_a_real_line(tmp_path: Path) -> None: model = Scripted( [call("comment", path="count.py", line="9", text="x")], [call("comment", path="nope.py", line="1", text="x")], [call("comment", path="count.py", line="two", text="x")], [call("post_review", summary="ok")], ) w = world(tmp_path, model) w["scheduler"].round() assert w["gitea"].posted[0]["comments"] == [] results = [ e.event.result for e in w["store"].transcript(f"{REPO}#7", "driver").read() if hasattr(e.event, "result") ] assert results[:3] == [ "count.py has 2 lines; 9 isn't one of them", "nope.py: no such file at the PR's head", "line is the line's number, like 42", ] def test_reads_stay_in_the_repository(tmp_path: Path) -> None: model = Scripted( [ call( "read_file", path="../../../../../../../../../../etc/passwd", ref="head" ) ], [call("read_file", path="count.py", ref="other")], [call("search_code", text="split", ref="head")], [call("read_issue", number="3")], [call("post_review", summary="ok")], ) w = world(tmp_path, model) w["scheduler"].round() results = [ (e.event.result, e.event.success) for e in w["store"].transcript(f"{REPO}#7", "driver").read() if hasattr(e.event, "result") ] assert results[0] == ( "../../../../../../../../../../etc/passwd: no such file at head", False, ) assert results[1] == ("ref is head or base, not 'other'", False) assert "count.py:2: return len(text.split())" in results[2][0] assert results[3][0].startswith("[o/demo#3, by don]") assert w["sources"].opened == [(REPO, 7, "abc123def456", "main")] def test_memory_notes_go_under_the_repositorys_folder(tmp_path: Path) -> None: model = Scripted( [ call( "remember", name="tests-beside-code", description="Tests sit beside the code", fact="o/demo keeps test_x.py next to x.py", scope="project", ) ], [call("post_review", summary="ok")], ) w = world(tmp_path, model) w["scheduler"].round() note = tmp_path / "vault/memory/nisse-review/projects/o-demo/tests-beside-code.md" assert note.is_file() # --- failures and crashes (L8) ---------------------------------------------------- def test_failed_runs_are_retried_then_said_on_the_pr(tmp_path: Path) -> None: prose = [AssistantText(text="I think it's fine.", model="m")] model = Scripted(prose, prose, prose, prose) # never a tool call w = world(tmp_path, model, attempts=2) w["scheduler"].round() assert (item(w).position, pr_of(item(w)).attempts) == (READY, 1) w["scheduler"].round() assert item(w).position == FAILED [(repo, number, said)] = w["gitea"].comments assert (repo, number) == (REPO, 7) and "couldn't review this" in said w["gitea"].repos[REPO] = [pr(head_sha="fff000")] # a new push: tried again w["scheduler"].intake() assert (item(w).position, pr_of(item(w)).attempts) == (READY, 0) def recovering(tmp_path: Path, gitea: FakeGitea) -> dict[str, Any]: """An item a crash left running, with its posting intent pending.""" w = world(tmp_path, Scripted(), gitea) w["scheduler"].intake() store = w["store"] store.update_work_item( f"{REPO}#7", position=RUNNING, data=item(w).data, cause="test" ) store.transcript(f"{REPO}#7", "orchestrator").begin_intent( POST_REVIEW, {"repo": REPO, "pr": 7, "head": "abc123def456"}, cause="test" ) return w def test_a_review_posted_before_a_crash_isnt_posted_again(tmp_path: Path) -> None: gitea = FakeGitea(repos={REPO: [pr()]}) gitea.posted.append( { "repo": REPO, "pr": 7, "body": f"ok\n\n{marker('abc123def456')}", "commit": "abc123def456", "comments": [], } ) w = recovering(tmp_path, gitea) assert w["scheduler"].round() is False # nothing left to review assert item(w).position == REVIEWED and len(gitea.posted) == 1 assert w["store"].pending_intents() == [] def test_a_review_not_posted_before_a_crash_is_done_again(tmp_path: Path) -> None: w = recovering(tmp_path, FakeGitea(repos={REPO: [pr()]})) w["scheduler"].recover() assert item(w).position == READY assert w["store"].pending_intents() == [] def test_with_gitea_down_a_crashed_review_waits(tmp_path: Path) -> None: gitea = FakeGitea(repos={REPO: [pr()]}) w = recovering(tmp_path, gitea) gitea.down = True assert w["scheduler"].round() is False assert item(w).position == RUNNING and len(w["store"].pending_intents()) == 1 # --- access (R3) ------------------------------------------------------------------ def test_an_account_that_can_write_is_refused(tmp_path: Path) -> None: gitea = FakeGitea(repos={REPO: [], "o/other": []}) log = OpLog("t", io.StringIO()) assert check_access(gitea, None, log) == ("nisse-review", [REPO, "o/other"]) # type: ignore[arg-type] gitea.permissions["o/other"] = {"pull": True, "push": True} with pytest.raises(StartupError, match="read access only"): check_access(gitea, None, log) # type: ignore[arg-type] def test_patterns_reach_public_repositories_the_account_was_not_given( tmp_path: Path, ) -> None: # A public repository the account wasn't given isn't in /user/repos: a # pattern finds it all the same (option a: the configuration decides). gitea = FakeGitea( repos={"dev/nisse": [pr(7)], "dev/wordle": [], "infra/tofu": [pr(3)]}, public={"dev/nisse", "dev/wordle", "infra/tofu"}, ) log = OpLog("t", io.StringIO()) assert reviewed(gitea, None, log) == [] # type: ignore[arg-type] assert reviewed(gitea, {"dev/*"}, log) == ["dev/nisse", "dev/wordle"] # type: ignore[arg-type] assert reviewed(gitea, {"*/*"}, log) == ["dev/nisse", "dev/wordle", "infra/tofu"] # type: ignore[arg-type] assert reviewed(gitea, {"dev/n*", "infra/tofu"}, log) == ["dev/nisse", "infra/tofu"] # type: ignore[arg-type] w = world(tmp_path, Scripted(), gitea, patterns=frozenset({"dev/*"})) w["scheduler"].round() assert w["store"].work_item("dev/nisse#7") is not None assert w["store"].work_item("infra/tofu#3") is None def test_a_repository_whose_prs_cannot_be_listed_holds_up_no_other( tmp_path: Path, ) -> None: # Found at bring-up: an empty private repository's /pulls was a 404, # and the round stopped there, so no repository was reviewed. gitea = FakeGitea(repos={"a/empty": [], REPO: [pr()]}, unlisted={"a/empty"}) out = io.StringIO() w = world(tmp_path, Scripted(*REVIEW), gitea) w["scheduler"]._oplog = OpLog("t", out) assert w["scheduler"].round() is True # REPO's PR was reviewed assert item(w).position == REVIEWED assert '"intake.repo_failed"' in out.getvalue() and "a/empty" in out.getvalue() class Clock: def __init__(self) -> None: self.now = 1000.0 def __call__(self) -> float: return self.now def test_a_new_head_waits_for_its_ci_to_finish(tmp_path: Path) -> None: # Follow-up to #154: judged with CI's result, and off CI's GPU. clock = Clock() gitea = FakeGitea(repos={REPO: [pr()]}) head = pr().head_sha gitea.ci[head] = "pending" w = world(tmp_path, Scripted(*REVIEW), gitea, clock=clock) assert w["scheduler"].round() is False # pending: not yet clock.now += 600 assert w["scheduler"].round() is False gitea.ci[head] = "failure" # finished: failing CI is worth a review too assert w["scheduler"].round() is True assert item(w).position == REVIEWED def test_ci_pending_too_long_is_reviewed_anyway(tmp_path: Path) -> None: clock = Clock() gitea = FakeGitea(repos={REPO: [pr()]}) gitea.ci[pr().head_sha] = "pending" w = world(tmp_path, Scripted(*REVIEW), gitea, clock=clock, ci_wait_s=3600) assert w["scheduler"].round() is False clock.now += 3600 assert w["scheduler"].round() is True # a stuck runner blocks nothing def test_no_ci_waits_only_a_grace_for_checks_to_appear(tmp_path: Path) -> None: clock = Clock() gitea = FakeGitea(repos={REPO: [pr()]}) gitea.ci[pr().head_sha] = "none" w = world(tmp_path, Scripted(*REVIEW), gitea, clock=clock, ci_grace_s=120) assert w["scheduler"].round() is False clock.now += 120 assert w["scheduler"].round() is True def test_a_pr_waiting_on_ci_holds_up_no_other(tmp_path: Path) -> None: clock = Clock() gitea = FakeGitea(repos={REPO: [pr(7), pr(8, head_sha="c8")]}) gitea.ci[pr(7).head_sha] = "pending" w = world(tmp_path, Scripted(*REVIEW), gitea, clock=clock) assert w["scheduler"].round() is True # 8, though 7 is older assert (item(w, 7).position, item(w, 8).position) == (READY, REVIEWED) def test_a_failed_review_keeps_its_heads_wait(tmp_path: Path) -> None: # nisse-review on #155: a retry lost seen_at and skipped the wait. clock = Clock() gitea = FakeGitea(repos={REPO: [pr()]}) prose = [AssistantText(text="I think it's fine.", model="m")] w = world(tmp_path, Scripted(prose, prose), gitea, clock=clock, attempts=2) w["scheduler"].round() assert (item(w).position, pr_of(item(w)).seen_at) == (READY, 1000.0) def test_a_head_pushed_during_a_failed_review_waits_for_its_ci( tmp_path: Path, ) -> None: clock = Clock() class Pushed(FakeGitea): def post_review(self, *args: Any, **kwargs: Any) -> int: clock.now += 50 self.repos[REPO] = [pr(head_sha="fff000")] self.ci["fff000"] = "pending" raise GiteaError("422: the head moved") gitea = Pushed(repos={REPO: [pr()]}) w = world(tmp_path, Scripted(*REVIEW), gitea, clock=clock) w["scheduler"].round() assert w["scheduler"].round() is False # the new head's CI is pending def test_checks_gitea_cannot_report_wait_a_round(tmp_path: Path) -> None: class NoChecks(FakeGitea): def checks(self, repo: str, sha: str) -> str: raise GiteaError("502: bad gateway") out = io.StringIO() w = world(tmp_path, Scripted(*REVIEW), NoChecks(repos={REPO: [pr()]})) w["scheduler"]._oplog = OpLog("t", out) assert w["scheduler"].round() is False assert item(w).position == READY assert '"review.checks_unknown"' in out.getvalue() def test_a_configured_repository_it_cannot_read_is_left_out() -> None: gitea = FakeGitea(repos={REPO: []}) out = io.StringIO() log = OpLog("t", out) patterns = {REPO, "o/gone", "nobody/*"} assert check(gitea, reviewed(gitea, patterns, log), log) == [REPO] # type: ignore[arg-type] assert '"access.unreadable"' in out.getvalue() and "o/gone" in out.getvalue() assert '"access.no_match"' in out.getvalue() and "nobody/*" in out.getvalue() def test_write_access_granted_while_running_stops_it(tmp_path: Path) -> None: gitea = FakeGitea(repos={REPO: [pr()]}) w = world(tmp_path, Scripted(), gitea) w["scheduler"].intake() gitea.permissions[REPO] = {"pull": True, "push": True} with pytest.raises(AccessRefused, match="read access only"): w["scheduler"].round() assert item(w).position == READY # nothing reviewed def test_a_repository_pattern_must_be_owner_slash_name(tmp_path: Path) -> None: from nisse_core.config import ConfigError from nisse_review.config import ReviewConfig tree = { "state_db": str(tmp_path / "s.db"), "llm": {"base_url": "http://llm", "token": "k"}, "roles": {"reviewer": "local", "expert": "frontier"}, "gitea": {"url": "https://g", "token": "t", "operators": ["don"]}, } ok = ReviewConfig.from_tree({**tree, "repositories": ["dev/*", "*/*"]}, {}) assert ok.repositories == {"dev/*", "*/*"} for bad in ("dev", "dev/", "/nisse", "a/b/c"): with pytest.raises(ConfigError, match="is not owner/name"): ReviewConfig.from_tree({**tree, "repositories": [bad]}, {}) def test_the_ci_waits_may_be_zero(tmp_path: Path) -> None: # The live test's repository has no CI: it reviews with no grace at all. from nisse_review.config import ReviewConfig tree = { "state_db": str(tmp_path / "s.db"), "llm": {"base_url": "http://llm", "token": "k"}, "roles": {"reviewer": "local", "expert": "frontier"}, "gitea": {"url": "https://g", "token": "t", "operators": ["don"]}, "tuning": {"review": {"ci_wait_s": 0, "ci_grace_s": 0}}, } review = ReviewConfig.from_tree(tree, {}).tuning.review assert (review.ci_wait_s, review.ci_grace_s) == (0, 0) def test_compaction_is_tuned_in_the_config(tmp_path: Path) -> None: from nisse_core.config import ConfigError from nisse_review.config import ReviewConfig tree: dict[str, Any] = { "state_db": str(tmp_path / "s.db"), "llm": {"base_url": "http://llm", "token": "k"}, "roles": {"reviewer": "local", "expert": "frontier"}, "gitea": {"url": "https://g", "token": "t", "operators": ["don"]}, } assert ( ReviewConfig.from_tree(tree, {}).tuning.context.compaction().compact_at == 0.6 ) tuned = { **tree, "tuning": {"context": {"compact_at": 0.7, "compact_keep_turns": 2}}, } compaction = ReviewConfig.from_tree(tuned, {}).tuning.context.compaction() assert (compaction.compact_at, compaction.keep_turns) == (0.7, 2) wrong = {**tree, "tuning": {"context": {"compact_at": 0.2, "compact_to": 0.3}}} with pytest.raises(ConfigError, match="compact_to must be less than compact_at"): ReviewConfig.from_tree(wrong, {}) def test_diff_lines_are_the_heads_lines_a_diff_shows() -> None: diff = ( "--- a/x.py\n+++ b/x.py\n@@ -1,3 +1,4 @@\n a\n-b\n+c\n+d\n e\n" "--- a/gone.py\n+++ /dev/null\n@@ -1 +0,0 @@\n-z\n" ) assert _diff_lines(diff) == {("x.py", 1), ("x.py", 2), ("x.py", 3), ("x.py", 4)} def test_gitea_errors_while_posting_count_as_a_failed_run(tmp_path: Path) -> None: class Refusing(FakeGitea): def post_review(self, *args: Any, **kwargs: Any) -> int: raise GiteaError("403: forbidden") gitea = Refusing(repos={REPO: [pr()]}) w = world(tmp_path, Scripted(*REVIEW), gitea) w["scheduler"].round() assert (item(w).position, pr_of(item(w)).attempts) == (READY, 1) assert w["store"].pending_intents() == [] def test_the_config_names_every_problem(tmp_path: Path) -> None: from nisse_core.config import ConfigError from nisse_review.config import ReviewConfig tree = { "state_db": str(tmp_path / "s.db"), "llm": {"base_url": "http://llm", "token": "${LLM}"}, "roles": {"reviewer": "local", "expert": "frontier"}, "gitea": {"url": "https://g", "token": "${GITEA}", "operators": ["don"]}, "tuning": {"review": {"max_turns": 10}}, } env = {"LLM": "k", "GITEA": "t"} config = ReviewConfig.from_tree(tree, env) assert config.roles.resolve("driver") == "local" assert config.tuning.review.max_turns == 10 assert config.gitea_operators == {"don"} and config.repositories is None only = ReviewConfig.from_tree({**tree, "repositories": ["o/demo"]}, env) assert only.repositories == {"o/demo"} with pytest.raises(ConfigError) as raised: ReviewConfig.from_tree({**tree, "matrix": {"x": 1}}, {"LLM": "k"}) assert "matrix.x: unknown setting" in str(raised.value) assert "references ${GITEA}" in str(raised.value) def test_a_long_file_is_read_in_parts_to_its_end(tmp_path: Path) -> None: # #156: read_file stopped at 20,000 characters with no way on, so the # reviewer hunted the rest a line at a time and ran out of turns. from nisse_review.reader import MAX_FILE_CHARS, Repository body = "".join(f"x{n} = {'y' * 60}\n" for n in range(1, 1001)) gitea = FakeGitea(files={"long.py": body}) trees = FakeSources(gitea, tmp_path).open(REPO, 7, "abc", "main") repo = Repository(gitea, REPO, trees) # type: ignore[arg-type] first = repo.dispatch("read_file", {"path": "long.py"}).result assert len(first) < MAX_FILE_CHARS + 200 assert "lines 1-" in first and "of 1000" in first seen, start = [], 1 while True: part = repo.dispatch("read_file", {"path": "long.py", "start": start}) assert part.success seen += [ int(r.split()[0]) for r in part.result.splitlines()[1:] if r[:5].strip().isdigit() ] if "read on with start=" not in part.result: break start = int(part.result.rsplit("start=", 1)[1].rstrip("]")) assert seen == list(range(1, 1001)) # every line, once bad = repo.dispatch("read_file", {"path": "long.py", "start": "2000"}) assert not bad.success and "1 to 1000" in bad.result def test_a_retry_starts_afresh_from_its_brief(tmp_path: Path) -> None: # #156: the retry carried the failed run's reads, and a call timed out. prose = [AssistantText(text="Reading on...", model="m")] model = Scripted(prose, prose, *REVIEW) # prose twice: the first run fails w = world(tmp_path, model, attempts=2) w["scheduler"].round() assert pr_of(item(w)).attempts == 1 w["scheduler"].round() retry = model.seen[2] # the retry's first call briefs = [e for e in retry if "Review o/demo#7" in str(e.event)] assert len(briefs) == 1 and retry[0] is briefs[0] assert not any("Reading on" in str(e.event) for e in retry) assert item(w).position == REVIEWED def test_out_of_turns_its_drafted_points_are_posted_as_partial( tmp_path: Path, ) -> None: model = Scripted(*REVIEW[:3]) # drafts two comments, never posts w = world(tmp_path, model) w["scheduler"]._reviewer._max_turns = 3 w["scheduler"].round() [posted] = w["gitea"].posted assert len(posted["comments"]) == 2 and "A partial review" in posted["body"] assert item(w).position == REVIEWED def test_out_of_turns_with_nothing_drafted_is_a_failed_run(tmp_path: Path) -> None: model = Scripted(REVIEW[0], REVIEW[0], REVIEW[0]) w = world(tmp_path, model, attempts=2) w["scheduler"]._reviewer._max_turns = 3 w["scheduler"].round() assert w["gitea"].posted == [] assert (item(w).position, pr_of(item(w)).attempts) == (READY, 1) def test_code_git_cannot_give_fails_the_run(tmp_path: Path) -> None: from nisse_review.clones import CloneError class Broken(FakeSources): def open(self, repo: str, number: int, head: str, base: str): # type: ignore[no-untyped-def] trees = super().open(repo, number, head, base) def tree(ref: str) -> Path: raise CloneError("git fetch failed: no route to host") trees.tree = tree # type: ignore[method-assign] return trees w = world(tmp_path, Scripted(*REVIEW)) w["scheduler"]._sources = Broken(w["gitea"], tmp_path / "trees") w["scheduler"].round() assert (item(w).position, pr_of(item(w)).attempts) == (READY, 1) assert w["gitea"].posted == [] def test_the_review_agent_records_its_calls_unless_told_not_to( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: from nisse_core.config import ConfigError from nisse_core.models.catalog import ModelCatalog from nisse_review import build as review_build from nisse_review.config import ReviewConfig tree = { "state_db": str(tmp_path / "s.db"), "llm": {"base_url": "http://llm", "token": "k"}, "roles": {"reviewer": "local", "expert": "frontier"}, "gitea": {"url": "https://g", "token": "t", "operators": ["don"]}, } limits = {"max_input_tokens": 32768, "max_output_tokens": 4096} catalog = ModelCatalog.from_payload( { "data": [ { "model_name": m, "litellm_params": {"model": f"x/{m}"}, "model_info": limits, } for m in ("local", "frontier") ] } ) monkeypatch.setattr(review_build.ModelCatalog, "discover", lambda *a, **k: catalog) on = review_build.llm_client(ReviewConfig.from_tree(tree, {})) assert on.call_recorder is not None assert on.call_recorder.path == tmp_path / "s.calls.db" off = review_build.llm_client( ReviewConfig.from_tree({**tree, "call_record": False}, {}) ) assert off.call_recorder is None with pytest.raises(ConfigError, match="call_record"): ReviewConfig.from_tree({**tree, "call_record": "yes"}, {})