Files
nisse/nisse-review/tests/test_review.py
T
dongarandClaude Opus 5.5 dd060566b2
CI / checks (push) Skipped
CI / checks (pull_request) Successful in 2m54s
CI / e2e (pull_request) Successful in 4m20s
The call record: every model call, for the operator's study (M6.6a)
nisse-core records each model call of every agent into a file beside its
state store (<state>.calls.db), for the status page (L7, X6):

- the request body exactly as sent (never headers), the raw response with
  its reasoning, the model asked and the one that answered, time, tokens
  and llama-server's own prompt reuse (timings.cache_n);
- its work item, role, run and purpose: Driver.run and nisse-dev's
  delegation loop open runs, nested; compaction, expert consults, the
  classifier and the go-ahead reader label their calls;
- provenance, worked out in BudgetedClient by comparing the events at
  each stage: each event sent with its log id and content as sent, which
  a compaction replaced, which fitting shortened (by how much) and
  dropped.

Content is stored once by hash, compressed. It is diagnostic only: the
agent never reads it, a failed write is logged once and recording stops
while the call goes on, and the file is deleted when the agent starts,
so its layout never migrates; CallReader refuses a layout it doesn't
know. While recording, the system prompt ends with one sentence saying
so. `call_record: false` in either agent's YAML turns it off.

The benchmark records each run beside that run's own state store, and
`--keep-runs DIR` keeps both for the page.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01J5Utyfzw7gF7UhjRDEtjTk
2026-10-24 06:28:53 +00:00

688 lines
26 KiB
Python

"""nisse-review's first slice (Part R, M7.3): its duty, a review run, posting
the review, and recovering after a crash."""
import io
from pathlib import Path
from typing import Any
import pytest
from review_fakes import FakeGitea, FakeSources, Scripted, call, pr
from nisse_core.gitea import GiteaError
from nisse_core.knowledge.handlers import Knowledge
from nisse_core.knowledge.memory import Memory
from nisse_core.knowledge.vault import Vault
from nisse_core.log.canonical import AssistantText
from nisse_core.loop.driver import Driver
from nisse_core.oplog import OpLog
from nisse_core.state.cancel import Cancel
from nisse_review.access import AccessRefused, check, reviewed
from nisse_review.build import StartupError, check_access
from nisse_review.review import Reviewer, _diff_lines, marker
from nisse_review.scheduler import Scheduler, ready
from nisse_review.state import (
FAILED,
POST_REVIEW,
READY,
REVIEWED,
RUNNING,
WAITING,
StateStore,
pr_of,
)
REPO = "o/demo"
def world(
tmp_path: Path, model: Scripted, gitea: FakeGitea | None = None, **more: Any
) -> dict[str, Any]:
gitea = gitea or FakeGitea(repos={REPO: [pr()]})
oplog = OpLog("nisse-review", io.StringIO())
cancel = more.setdefault("cancel", Cancel()) # the driver's and the scheduler's
store = StateStore(tmp_path / "state.db", oplog=oplog)
vault = Vault(tmp_path / "vault", {"memory/nisse-review": "rw"}, agent="r")
reviewer = Reviewer(
store,
gitea, # type: ignore[arg-type]
Driver(model, "m", cancel=cancel), # type: ignore[arg-type]
account=gitea.account,
gitea_url="https://gitea.test",
oplog=oplog,
knowledge=Knowledge(vault, Memory(vault, "memory/nisse-review"), oplog),
)
sources = FakeSources(gitea, tmp_path / "trees")
scheduler = Scheduler(
store,
gitea, # type: ignore[arg-type]
reviewer,
oplog=oplog,
sources=sources,
**more,
)
return {
"gitea": gitea,
"store": store,
"scheduler": scheduler,
"sources": sources,
"tmp": tmp_path,
}
def item(w: dict[str, Any], number: int = 7):
found = w["store"].work_item(f"{REPO}#{number}")
assert found is not None
return found
REVIEW = [
[call("read_file", path="count.py", ref="head")],
[call("comment", path="count.py", line="2", text="split() handles tabs too.")],
[call("comment", path="README.md", line="3", text="Say it counts words.")],
[call("post_review", summary="Looks right; two small points.")],
]
# --- the duty (R1) ---------------------------------------------------------------
@pytest.mark.parametrize(
("title", "draft", "expected"),
[
("Count words", False, True),
("Count words", True, False),
("WIP: count words", False, False),
("[WIP] count words", False, False),
("wip count", False, False),
("Wipe the cache", False, True),
],
)
def test_ready_means_not_a_draft_and_not_wip(title, draft, expected) -> None:
assert ready(pr(title=title, draft=draft)) is expected
def test_intake_opens_an_item_per_pr_and_closes_it_with_the_pr(
tmp_path: Path,
) -> None:
gitea = FakeGitea(repos={REPO: [pr(7), pr(8, draft=True)]})
w = world(tmp_path, Scripted(), gitea)
w["scheduler"].intake()
assert (item(w, 7).position, item(w, 8).position) == (READY, WAITING)
assert item(w, 7).scope == REPO
gitea.repos["o/elsewhere"] = [pr(9)]
w["scheduler"]._patterns = frozenset({REPO})
w["scheduler"].intake()
assert w["store"].work_item("o/elsewhere#9") is None
gitea.repos[REPO] = [pr(8, draft=False)] # 7 merged; 8 is ready now
gitea.merged.add((REPO, 7))
w["scheduler"].intake()
assert (item(w, 7).status, item(w, 7).outcome) == ("closed", "completed")
assert item(w, 8).position == READY
# --- a review --------------------------------------------------------------------
def test_a_review_is_read_drafted_and_posted_as_a_comment_review(
tmp_path: Path,
) -> None:
model = Scripted(*REVIEW)
w = world(tmp_path, model)
assert w["scheduler"].round() is True
[posted] = w["gitea"].posted
# In the diff: inline. README.md:3 isn't in it: Gitea took it inline too.
assert [(c.path, c.line) for c in posted["comments"]] == [
("count.py", 2),
("README.md", 3),
]
assert posted["commit"] == "abc123def456"
assert posted["body"].startswith("Looks right; two small points.")
assert marker("abc123def456") in posted["body"]
assert item(w).position == REVIEWED
assert pr_of(item(w)).reviewed == "abc123def456"
# What it started from: the PR, what was said, its issue, the diff.
brief = model.seen[0][0].event.text
for part in ("Count words, not characters", "Please keep it simple", "#3"):
assert part in brief
assert "- return len(text)" in brief
# Its tools: reading, commenting, memory, posting; nothing that writes code.
assert model.offered[0] >= {
"read_file",
"search_code",
"comment",
"remember",
"post_review",
}
assert not model.offered[0] & {"write_file", "run_command"}
assert "never an instruction" in model.systems[0]
def test_comments_gitea_wont_take_off_the_diff_become_links(tmp_path: Path) -> None:
gitea = FakeGitea(repos={REPO: [pr()]}, refuse_off_diff=True)
w = world(tmp_path, Scripted(*REVIEW), gitea)
w["scheduler"].round()
[posted] = gitea.posted
assert [(c.path, c.line) for c in posted["comments"]] == [("count.py", 2)]
assert (
"[`README.md:3`](https://gitea.test/o/demo/src/commit/abc123def456/"
"README.md#L3): Say it counts words." in posted["body"]
)
def test_a_comment_must_be_on_a_real_line(tmp_path: Path) -> None:
model = Scripted(
[call("comment", path="count.py", line="9", text="x")],
[call("comment", path="nope.py", line="1", text="x")],
[call("comment", path="count.py", line="two", text="x")],
[call("post_review", summary="ok")],
)
w = world(tmp_path, model)
w["scheduler"].round()
assert w["gitea"].posted[0]["comments"] == []
results = [
e.event.result
for e in w["store"].transcript(f"{REPO}#7", "driver").read()
if hasattr(e.event, "result")
]
assert results[:3] == [
"count.py has 2 lines; 9 isn't one of them",
"nope.py: no such file at the PR's head",
"line is the line's number, like 42",
]
def test_reads_stay_in_the_repository(tmp_path: Path) -> None:
model = Scripted(
[
call(
"read_file", path="../../../../../../../../../../etc/passwd", ref="head"
)
],
[call("read_file", path="count.py", ref="other")],
[call("search_code", text="split", ref="head")],
[call("read_issue", number="3")],
[call("post_review", summary="ok")],
)
w = world(tmp_path, model)
w["scheduler"].round()
results = [
(e.event.result, e.event.success)
for e in w["store"].transcript(f"{REPO}#7", "driver").read()
if hasattr(e.event, "result")
]
assert results[0] == (
"../../../../../../../../../../etc/passwd: no such file at head",
False,
)
assert results[1] == ("ref is head or base, not 'other'", False)
assert "count.py:2: return len(text.split())" in results[2][0]
assert results[3][0].startswith("[o/demo#3, by don]")
assert w["sources"].opened == [(REPO, 7, "abc123def456", "main")]
def test_memory_notes_go_under_the_repositorys_folder(tmp_path: Path) -> None:
model = Scripted(
[
call(
"remember",
name="tests-beside-code",
description="Tests sit beside the code",
fact="o/demo keeps test_x.py next to x.py",
scope="project",
)
],
[call("post_review", summary="ok")],
)
w = world(tmp_path, model)
w["scheduler"].round()
note = tmp_path / "vault/memory/nisse-review/projects/o-demo/tests-beside-code.md"
assert note.is_file()
# --- failures and crashes (L8) ----------------------------------------------------
def test_failed_runs_are_retried_then_said_on_the_pr(tmp_path: Path) -> None:
prose = [AssistantText(text="I think it's fine.", model="m")]
model = Scripted(prose, prose, prose, prose) # never a tool call
w = world(tmp_path, model, attempts=2)
w["scheduler"].round()
assert (item(w).position, pr_of(item(w)).attempts) == (READY, 1)
w["scheduler"].round()
assert item(w).position == FAILED
[(repo, number, said)] = w["gitea"].comments
assert (repo, number) == (REPO, 7) and "couldn't review this" in said
w["gitea"].repos[REPO] = [pr(head_sha="fff000")] # a new push: tried again
w["scheduler"].intake()
assert (item(w).position, pr_of(item(w)).attempts) == (READY, 0)
def recovering(tmp_path: Path, gitea: FakeGitea) -> dict[str, Any]:
"""An item a crash left running, with its posting intent pending."""
w = world(tmp_path, Scripted(), gitea)
w["scheduler"].intake()
store = w["store"]
store.update_work_item(
f"{REPO}#7", position=RUNNING, data=item(w).data, cause="test"
)
store.transcript(f"{REPO}#7", "orchestrator").begin_intent(
POST_REVIEW, {"repo": REPO, "pr": 7, "head": "abc123def456"}, cause="test"
)
return w
def test_a_review_posted_before_a_crash_isnt_posted_again(tmp_path: Path) -> None:
gitea = FakeGitea(repos={REPO: [pr()]})
gitea.posted.append(
{
"repo": REPO,
"pr": 7,
"body": f"ok\n\n{marker('abc123def456')}",
"commit": "abc123def456",
"comments": [],
}
)
w = recovering(tmp_path, gitea)
assert w["scheduler"].round() is False # nothing left to review
assert item(w).position == REVIEWED and len(gitea.posted) == 1
assert w["store"].pending_intents() == []
def test_a_review_not_posted_before_a_crash_is_done_again(tmp_path: Path) -> None:
w = recovering(tmp_path, FakeGitea(repos={REPO: [pr()]}))
w["scheduler"].recover()
assert item(w).position == READY
assert w["store"].pending_intents() == []
def test_with_gitea_down_a_crashed_review_waits(tmp_path: Path) -> None:
gitea = FakeGitea(repos={REPO: [pr()]})
w = recovering(tmp_path, gitea)
gitea.down = True
assert w["scheduler"].round() is False
assert item(w).position == RUNNING and len(w["store"].pending_intents()) == 1
# --- access (R3) ------------------------------------------------------------------
def test_an_account_that_can_write_is_refused(tmp_path: Path) -> None:
gitea = FakeGitea(repos={REPO: [], "o/other": []})
log = OpLog("t", io.StringIO())
assert check_access(gitea, None, log) == ("nisse-review", [REPO, "o/other"]) # type: ignore[arg-type]
gitea.permissions["o/other"] = {"pull": True, "push": True}
with pytest.raises(StartupError, match="read access only"):
check_access(gitea, None, log) # type: ignore[arg-type]
def test_patterns_reach_public_repositories_the_account_was_not_given(
tmp_path: Path,
) -> None:
# A public repository the account wasn't given isn't in /user/repos: a
# pattern finds it all the same (option a: the configuration decides).
gitea = FakeGitea(
repos={"dev/nisse": [pr(7)], "dev/wordle": [], "infra/tofu": [pr(3)]},
public={"dev/nisse", "dev/wordle", "infra/tofu"},
)
log = OpLog("t", io.StringIO())
assert reviewed(gitea, None, log) == [] # type: ignore[arg-type]
assert reviewed(gitea, {"dev/*"}, log) == ["dev/nisse", "dev/wordle"] # type: ignore[arg-type]
assert reviewed(gitea, {"*/*"}, log) == ["dev/nisse", "dev/wordle", "infra/tofu"] # type: ignore[arg-type]
assert reviewed(gitea, {"dev/n*", "infra/tofu"}, log) == ["dev/nisse", "infra/tofu"] # type: ignore[arg-type]
w = world(tmp_path, Scripted(), gitea, patterns=frozenset({"dev/*"}))
w["scheduler"].round()
assert w["store"].work_item("dev/nisse#7") is not None
assert w["store"].work_item("infra/tofu#3") is None
def test_a_repository_whose_prs_cannot_be_listed_holds_up_no_other(
tmp_path: Path,
) -> None:
# Found at bring-up: an empty private repository's /pulls was a 404,
# and the round stopped there, so no repository was reviewed.
gitea = FakeGitea(repos={"a/empty": [], REPO: [pr()]}, unlisted={"a/empty"})
out = io.StringIO()
w = world(tmp_path, Scripted(*REVIEW), gitea)
w["scheduler"]._oplog = OpLog("t", out)
assert w["scheduler"].round() is True # REPO's PR was reviewed
assert item(w).position == REVIEWED
assert '"intake.repo_failed"' in out.getvalue() and "a/empty" in out.getvalue()
class Clock:
def __init__(self) -> None:
self.now = 1000.0
def __call__(self) -> float:
return self.now
def test_a_new_head_waits_for_its_ci_to_finish(tmp_path: Path) -> None:
# Follow-up to #154: judged with CI's result, and off CI's GPU.
clock = Clock()
gitea = FakeGitea(repos={REPO: [pr()]})
head = pr().head_sha
gitea.ci[head] = "pending"
w = world(tmp_path, Scripted(*REVIEW), gitea, clock=clock)
assert w["scheduler"].round() is False # pending: not yet
clock.now += 600
assert w["scheduler"].round() is False
gitea.ci[head] = "failure" # finished: failing CI is worth a review too
assert w["scheduler"].round() is True
assert item(w).position == REVIEWED
def test_ci_pending_too_long_is_reviewed_anyway(tmp_path: Path) -> None:
clock = Clock()
gitea = FakeGitea(repos={REPO: [pr()]})
gitea.ci[pr().head_sha] = "pending"
w = world(tmp_path, Scripted(*REVIEW), gitea, clock=clock, ci_wait_s=3600)
assert w["scheduler"].round() is False
clock.now += 3600
assert w["scheduler"].round() is True # a stuck runner blocks nothing
def test_no_ci_waits_only_a_grace_for_checks_to_appear(tmp_path: Path) -> None:
clock = Clock()
gitea = FakeGitea(repos={REPO: [pr()]})
gitea.ci[pr().head_sha] = "none"
w = world(tmp_path, Scripted(*REVIEW), gitea, clock=clock, ci_grace_s=120)
assert w["scheduler"].round() is False
clock.now += 120
assert w["scheduler"].round() is True
def test_a_pr_waiting_on_ci_holds_up_no_other(tmp_path: Path) -> None:
clock = Clock()
gitea = FakeGitea(repos={REPO: [pr(7), pr(8, head_sha="c8")]})
gitea.ci[pr(7).head_sha] = "pending"
w = world(tmp_path, Scripted(*REVIEW), gitea, clock=clock)
assert w["scheduler"].round() is True # 8, though 7 is older
assert (item(w, 7).position, item(w, 8).position) == (READY, REVIEWED)
def test_a_failed_review_keeps_its_heads_wait(tmp_path: Path) -> None:
# nisse-review on #155: a retry lost seen_at and skipped the wait.
clock = Clock()
gitea = FakeGitea(repos={REPO: [pr()]})
prose = [AssistantText(text="I think it's fine.", model="m")]
w = world(tmp_path, Scripted(prose, prose), gitea, clock=clock, attempts=2)
w["scheduler"].round()
assert (item(w).position, pr_of(item(w)).seen_at) == (READY, 1000.0)
def test_a_head_pushed_during_a_failed_review_waits_for_its_ci(
tmp_path: Path,
) -> None:
clock = Clock()
class Pushed(FakeGitea):
def post_review(self, *args: Any, **kwargs: Any) -> int:
clock.now += 50
self.repos[REPO] = [pr(head_sha="fff000")]
self.ci["fff000"] = "pending"
raise GiteaError("422: the head moved")
gitea = Pushed(repos={REPO: [pr()]})
w = world(tmp_path, Scripted(*REVIEW), gitea, clock=clock)
w["scheduler"].round()
assert w["scheduler"].round() is False # the new head's CI is pending
def test_checks_gitea_cannot_report_wait_a_round(tmp_path: Path) -> None:
class NoChecks(FakeGitea):
def checks(self, repo: str, sha: str) -> str:
raise GiteaError("502: bad gateway")
out = io.StringIO()
w = world(tmp_path, Scripted(*REVIEW), NoChecks(repos={REPO: [pr()]}))
w["scheduler"]._oplog = OpLog("t", out)
assert w["scheduler"].round() is False
assert item(w).position == READY
assert '"review.checks_unknown"' in out.getvalue()
def test_a_configured_repository_it_cannot_read_is_left_out() -> None:
gitea = FakeGitea(repos={REPO: []})
out = io.StringIO()
log = OpLog("t", out)
patterns = {REPO, "o/gone", "nobody/*"}
assert check(gitea, reviewed(gitea, patterns, log), log) == [REPO] # type: ignore[arg-type]
assert '"access.unreadable"' in out.getvalue() and "o/gone" in out.getvalue()
assert '"access.no_match"' in out.getvalue() and "nobody/*" in out.getvalue()
def test_write_access_granted_while_running_stops_it(tmp_path: Path) -> None:
gitea = FakeGitea(repos={REPO: [pr()]})
w = world(tmp_path, Scripted(), gitea)
w["scheduler"].intake()
gitea.permissions[REPO] = {"pull": True, "push": True}
with pytest.raises(AccessRefused, match="read access only"):
w["scheduler"].round()
assert item(w).position == READY # nothing reviewed
def test_a_repository_pattern_must_be_owner_slash_name(tmp_path: Path) -> None:
from nisse_core.config import ConfigError
from nisse_review.config import ReviewConfig
tree = {
"state_db": str(tmp_path / "s.db"),
"llm": {"base_url": "http://llm", "token": "k"},
"roles": {"reviewer": "local", "expert": "frontier"},
"gitea": {"url": "https://g", "token": "t", "operators": ["don"]},
}
ok = ReviewConfig.from_tree({**tree, "repositories": ["dev/*", "*/*"]}, {})
assert ok.repositories == {"dev/*", "*/*"}
for bad in ("dev", "dev/", "/nisse", "a/b/c"):
with pytest.raises(ConfigError, match="is not owner/name"):
ReviewConfig.from_tree({**tree, "repositories": [bad]}, {})
def test_the_ci_waits_may_be_zero(tmp_path: Path) -> None:
# The live test's repository has no CI: it reviews with no grace at all.
from nisse_review.config import ReviewConfig
tree = {
"state_db": str(tmp_path / "s.db"),
"llm": {"base_url": "http://llm", "token": "k"},
"roles": {"reviewer": "local", "expert": "frontier"},
"gitea": {"url": "https://g", "token": "t", "operators": ["don"]},
"tuning": {"review": {"ci_wait_s": 0, "ci_grace_s": 0}},
}
review = ReviewConfig.from_tree(tree, {}).tuning.review
assert (review.ci_wait_s, review.ci_grace_s) == (0, 0)
def test_compaction_is_tuned_in_the_config(tmp_path: Path) -> None:
from nisse_core.config import ConfigError
from nisse_review.config import ReviewConfig
tree: dict[str, Any] = {
"state_db": str(tmp_path / "s.db"),
"llm": {"base_url": "http://llm", "token": "k"},
"roles": {"reviewer": "local", "expert": "frontier"},
"gitea": {"url": "https://g", "token": "t", "operators": ["don"]},
}
assert (
ReviewConfig.from_tree(tree, {}).tuning.context.compaction().compact_at == 0.6
)
tuned = {
**tree,
"tuning": {"context": {"compact_at": 0.7, "compact_keep_turns": 2}},
}
compaction = ReviewConfig.from_tree(tuned, {}).tuning.context.compaction()
assert (compaction.compact_at, compaction.keep_turns) == (0.7, 2)
wrong = {**tree, "tuning": {"context": {"compact_at": 0.2, "compact_to": 0.3}}}
with pytest.raises(ConfigError, match="compact_to must be less than compact_at"):
ReviewConfig.from_tree(wrong, {})
def test_diff_lines_are_the_heads_lines_a_diff_shows() -> None:
diff = (
"--- a/x.py\n+++ b/x.py\n@@ -1,3 +1,4 @@\n a\n-b\n+c\n+d\n e\n"
"--- a/gone.py\n+++ /dev/null\n@@ -1 +0,0 @@\n-z\n"
)
assert _diff_lines(diff) == {("x.py", 1), ("x.py", 2), ("x.py", 3), ("x.py", 4)}
def test_gitea_errors_while_posting_count_as_a_failed_run(tmp_path: Path) -> None:
class Refusing(FakeGitea):
def post_review(self, *args: Any, **kwargs: Any) -> int:
raise GiteaError("403: forbidden")
gitea = Refusing(repos={REPO: [pr()]})
w = world(tmp_path, Scripted(*REVIEW), gitea)
w["scheduler"].round()
assert (item(w).position, pr_of(item(w)).attempts) == (READY, 1)
assert w["store"].pending_intents() == []
def test_the_config_names_every_problem(tmp_path: Path) -> None:
from nisse_core.config import ConfigError
from nisse_review.config import ReviewConfig
tree = {
"state_db": str(tmp_path / "s.db"),
"llm": {"base_url": "http://llm", "token": "${LLM}"},
"roles": {"reviewer": "local", "expert": "frontier"},
"gitea": {"url": "https://g", "token": "${GITEA}", "operators": ["don"]},
"tuning": {"review": {"max_turns": 10}},
}
env = {"LLM": "k", "GITEA": "t"}
config = ReviewConfig.from_tree(tree, env)
assert config.roles.resolve("driver") == "local"
assert config.tuning.review.max_turns == 10
assert config.gitea_operators == {"don"} and config.repositories is None
only = ReviewConfig.from_tree({**tree, "repositories": ["o/demo"]}, env)
assert only.repositories == {"o/demo"}
with pytest.raises(ConfigError) as raised:
ReviewConfig.from_tree({**tree, "matrix": {"x": 1}}, {"LLM": "k"})
assert "matrix.x: unknown setting" in str(raised.value)
assert "references ${GITEA}" in str(raised.value)
def test_a_long_file_is_read_in_parts_to_its_end(tmp_path: Path) -> None:
# #156: read_file stopped at 20,000 characters with no way on, so the
# reviewer hunted the rest a line at a time and ran out of turns.
from nisse_review.reader import MAX_FILE_CHARS, Repository
body = "".join(f"x{n} = {'y' * 60}\n" for n in range(1, 1001))
gitea = FakeGitea(files={"long.py": body})
trees = FakeSources(gitea, tmp_path).open(REPO, 7, "abc", "main")
repo = Repository(gitea, REPO, trees) # type: ignore[arg-type]
first = repo.dispatch("read_file", {"path": "long.py"}).result
assert len(first) < MAX_FILE_CHARS + 200
assert "lines 1-" in first and "of 1000" in first
seen, start = [], 1
while True:
part = repo.dispatch("read_file", {"path": "long.py", "start": start})
assert part.success
seen += [
int(r.split()[0])
for r in part.result.splitlines()[1:]
if r[:5].strip().isdigit()
]
if "read on with start=" not in part.result:
break
start = int(part.result.rsplit("start=", 1)[1].rstrip("]"))
assert seen == list(range(1, 1001)) # every line, once
bad = repo.dispatch("read_file", {"path": "long.py", "start": "2000"})
assert not bad.success and "1 to 1000" in bad.result
def test_a_retry_starts_afresh_from_its_brief(tmp_path: Path) -> None:
# #156: the retry carried the failed run's reads, and a call timed out.
prose = [AssistantText(text="Reading on...", model="m")]
model = Scripted(prose, prose, *REVIEW) # prose twice: the first run fails
w = world(tmp_path, model, attempts=2)
w["scheduler"].round()
assert pr_of(item(w)).attempts == 1
w["scheduler"].round()
retry = model.seen[2] # the retry's first call
briefs = [e for e in retry if "Review o/demo#7" in str(e.event)]
assert len(briefs) == 1 and retry[0] is briefs[0]
assert not any("Reading on" in str(e.event) for e in retry)
assert item(w).position == REVIEWED
def test_out_of_turns_its_drafted_points_are_posted_as_partial(
tmp_path: Path,
) -> None:
model = Scripted(*REVIEW[:3]) # drafts two comments, never posts
w = world(tmp_path, model)
w["scheduler"]._reviewer._max_turns = 3
w["scheduler"].round()
[posted] = w["gitea"].posted
assert len(posted["comments"]) == 2 and "A partial review" in posted["body"]
assert item(w).position == REVIEWED
def test_out_of_turns_with_nothing_drafted_is_a_failed_run(tmp_path: Path) -> None:
model = Scripted(REVIEW[0], REVIEW[0], REVIEW[0])
w = world(tmp_path, model, attempts=2)
w["scheduler"]._reviewer._max_turns = 3
w["scheduler"].round()
assert w["gitea"].posted == []
assert (item(w).position, pr_of(item(w)).attempts) == (READY, 1)
def test_code_git_cannot_give_fails_the_run(tmp_path: Path) -> None:
from nisse_review.clones import CloneError
class Broken(FakeSources):
def open(self, repo: str, number: int, head: str, base: str): # type: ignore[no-untyped-def]
trees = super().open(repo, number, head, base)
def tree(ref: str) -> Path:
raise CloneError("git fetch failed: no route to host")
trees.tree = tree # type: ignore[method-assign]
return trees
w = world(tmp_path, Scripted(*REVIEW))
w["scheduler"]._sources = Broken(w["gitea"], tmp_path / "trees")
w["scheduler"].round()
assert (item(w).position, pr_of(item(w)).attempts) == (READY, 1)
assert w["gitea"].posted == []
def test_the_review_agent_records_its_calls_unless_told_not_to(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
from nisse_core.config import ConfigError
from nisse_core.models.catalog import ModelCatalog
from nisse_review import build as review_build
from nisse_review.config import ReviewConfig
tree = {
"state_db": str(tmp_path / "s.db"),
"llm": {"base_url": "http://llm", "token": "k"},
"roles": {"reviewer": "local", "expert": "frontier"},
"gitea": {"url": "https://g", "token": "t", "operators": ["don"]},
}
limits = {"max_input_tokens": 32768, "max_output_tokens": 4096}
catalog = ModelCatalog.from_payload(
{
"data": [
{
"model_name": m,
"litellm_params": {"model": f"x/{m}"},
"model_info": limits,
}
for m in ("local", "frontier")
]
}
)
monkeypatch.setattr(review_build.ModelCatalog, "discover", lambda *a, **k: catalog)
on = review_build.llm_client(ReviewConfig.from_tree(tree, {}))
assert on.call_recorder is not None
assert on.call_recorder.path == tmp_path / "s.calls.db"
off = review_build.llm_client(
ReviewConfig.from_tree({**tree, "call_record": False}, {})
)
assert off.call_recorder is None
with pytest.raises(ConfigError, match="call_record"):
ReviewConfig.from_tree({**tree, "call_record": "yes"}, {})