nisse-review: read code from a git clone kept between runs

Archives were downloaded and unpacked afresh for every run, which doesn't
scale to very large repositories. Each repository is now cloned once, with
its whole history, under scratch/repos/<owner>/<name>/ (head/ the clone,
base/ a worktree when read). A run fetches every branch and the PR's
refs/pull/N/head, checks out the head exactly, and fails if that head is
gone. Only the latest repository is kept for now.

Git runs walled off: none of the host's git configuration, links checked
out as plain files, no hooks, submodules or LFS, only the Gitea URL's
protocol, and the token as an http.extraHeader for that URL in git's
environment only. A stop ends a git command; a cut-short first clone is
removed. The reader never shows .git. GiteaClient.archive is gone.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01J5Utyfzw7gF7UhjRDEtjTk

nisse-review/src/nisse_review/build.py

@@ -2,7 +2,6 @@
 before it reviews anything."""
 
 from dataclasses import dataclass
-from pathlib import Path
 
 from nisse_core.gitea import GiteaClient
 from nisse_core.knowledge.handlers import Knowledge
@@ -17,6 +16,7 @@ from nisse_core.models.llm import LiteLLMClient
 from nisse_core.oplog import OpLog
 from nisse_core.state.cancel import Cancel
 from nisse_review.access import AccessRefused, check, reviewed
+from nisse_review.clones import Clones
 from nisse_review.config import ReviewConfig
 from nisse_review.review import Reviewer
 from nisse_review.scheduler import Scheduler
@@ -142,14 +142,20 @@ def build(
         max_turns=tuning.review.max_turns,
         diff_chars=tuning.review.diff_chars,
     )
-    scratch = config.state_db.parent / "scratch"
-    scratch.mkdir(exist_ok=True)
+    # The repositories' clones, kept between runs (R3).
+    clones = Clones(
+        config.state_db.parent / "scratch" / "repos",
+        config.gitea_url,
+        config.gitea_token,
+        oplog=oplog,
+        cancel=cancel,
+    )
     scheduler = Scheduler(
         store,
         gitea,
         reviewer,
         oplog=oplog,
-        scratch=Path(scratch),
+        sources=clones,
         poll_s=tuning.review.poll_s,
         attempts=tuning.review.attempts,
         ci_wait_s=tuning.review.ci_wait_s,

nisse-review/src/nisse_review/clones.py

@@ -0,0 +1,253 @@
+"""The repositories the reviewer reads, as git clones kept between runs (R3).
+
+Each repository is cloned once, with its whole history, under ``root``:
+
+    root/<owner>/<name>/head/   the clone, checked out at the PR's head
+    root/<owner>/<name>/base/   a worktree at the base branch, when read
+
+A run fetches what changed since (every branch, and the PR's own ref, which
+also reaches a PR from a fork), checks out the head it reviews, and removes
+whatever an earlier PR left. Only the most recently used repositories are
+kept (``keep``); the rest are deleted when another is opened.
+
+Git runs on code nobody vetted, so it runs with none of this host's git
+configuration, links checked out as plain files (never pointing out of the
+clone), no hooks, no submodules, no LFS downloads, and only the configured
+Gitea's protocol. The token reaches git in its environment, as a header for
+Gitea's address alone: never on a command line, in a file, or in a URL.
+
+A stop (``cancel``) ends a git command at once: it raises ``Stopped``, and a
+first clone it cut short is deleted, to be made afresh next time.
+"""
+
+import os
+import shutil
+import subprocess
+import time
+from pathlib import Path
+
+from nisse_core.models.interruptible import Stopped
+from nisse_core.oplog import OpLog
+from nisse_core.state.cancel import Cancel
+
+_DONE = "cloned"  # in a repository's folder once its first fetch finished
+_USED = "used"  # touched each time it is opened: which to keep
+
+
+class CloneError(RuntimeError):
+    """Git couldn't give the code: the run fails, and is tried again."""
+
+
+class Clones:
+    def __init__(
+        self,
+        root: Path,
+        gitea_url: str,
+        token: str,
+        *,
+        oplog: OpLog,
+        cancel: Cancel | None = None,
+        keep: int = 1,
+    ) -> None:
+        self._root = root
+        self._url = gitea_url.rstrip("/")
+        self._token = token
+        self._oplog = oplog
+        self._cancel = cancel or Cancel()
+        self._keep = keep
+        root.mkdir(parents=True, exist_ok=True)
+
+    def open(self, repo: str, number: int, head: str, base: str) -> "Checkout":
+        """``repo``'s PR ``number``, at ``head``, merging into ``base``. Git
+        runs only when a tree is first read."""
+        return Checkout(self, repo, number, head, base)
+
+    # -- for Checkout -----------------------------------------------------------
+
+    def folder(self, repo: str) -> Path:
+        owner, _, name = repo.partition("/")
+        return self._root / owner / name
+
+    def prune(self, repo: str) -> None:
+        """Keep ``repo`` and the most recently used others, to ``keep``."""
+        mine = self.folder(repo)
+        mine.mkdir(parents=True, exist_ok=True)
+        (mine / _USED).touch()
+        others = [
+            folder
+            for owner in self._root.iterdir()
+            if owner.is_dir()
+            for folder in owner.iterdir()
+            if folder.is_dir() and folder != mine
+        ]
+        others.sort(key=_used, reverse=True)
+        for folder in others[self._keep - 1 :]:
+            shutil.rmtree(folder, ignore_errors=True)
+            self._oplog.emit(
+                "review.clone_removed", cause="not recent", path=str(folder)
+            )
+        for owner in self._root.iterdir():
+            if owner.is_dir() and not any(owner.iterdir()):
+                owner.rmdir()
+
+    def git(self, *args: str, cwd: Path) -> str:
+        """One git command, in the walled environment; its output."""
+        with subprocess.Popen(
+            ["git", *args],
+            cwd=cwd,
+            env=self._env(),
+            stdin=subprocess.DEVNULL,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
+            text=True,
+        ) as proc:
+            while True:
+                try:
+                    out, err = proc.communicate(timeout=0.5)
+                    break
+                except subprocess.TimeoutExpired:
+                    if self._cancel.requested:
+                        proc.terminate()  # git removes its lock files
+                        proc.communicate()
+                        raise Stopped(f"a stop ended git {args[0]}") from None
+        if proc.returncode != 0:
+            raise CloneError(f"git {args[0]} failed: {err.strip()[-500:]}")
+        return out
+
+    def remote(self, repo: str) -> str:
+        return f"{self._url}/{repo}.git"
+
+    def _env(self) -> dict[str, str]:
+        settings = {
+            "core.symlinks": "false",
+            "core.hooksPath": os.devnull,
+            "core.fsmonitor": "false",
+            "submodule.recurse": "false",
+            "fetch.recurseSubmodules": "false",
+            "advice.detachedHead": "false",
+            "gc.auto": "0",  # no background work left running after a stop
+        }
+        if self._token:
+            settings[f"http.{self._url}/.extraHeader"] = (
+                f"Authorization: token {self._token}"
+            )
+        env = {
+            "PATH": os.environ.get("PATH", "/usr/bin:/bin"),
+            "HOME": str(self._root),  # no ~/.gitconfig of this host's
+            "GIT_CONFIG_NOSYSTEM": "1",
+            "GIT_CONFIG_GLOBAL": os.devnull,
+            "GIT_TERMINAL_PROMPT": "0",
+            # Never another repository above a clone, should one be missing.
+            "GIT_CEILING_DIRECTORIES": str(self._root),
+            "GIT_LFS_SKIP_SMUDGE": "1",
+            "GIT_ALLOW_PROTOCOL": self._url.partition(":")[0],
+            "GIT_CONFIG_COUNT": str(len(settings)),
+        }
+        for n, (key, value) in enumerate(settings.items()):
+            env[f"GIT_CONFIG_KEY_{n}"] = key
+            env[f"GIT_CONFIG_VALUE_{n}"] = value
+        return env
+
+
+class Checkout:
+    """One run's view of a repository: ``tree("head")`` and
+    ``tree("base")``, each a folder of plain files."""
+
+    def __init__(
+        self, clones: Clones, repo: str, number: int, head: str, base: str
+    ) -> None:
+        self._clones = clones
+        self.repo = repo
+        self._number = number
+        self._head = head
+        self._base = base
+        self._trees: dict[str, Path] = {}
+
+    def tree(self, ref: str) -> Path:
+        if ref not in ("head", "base"):
+            raise ValueError(f"ref is head or base, not {ref!r}")
+        if ref not in self._trees:
+            if "head" not in self._trees:
+                self._trees["head"] = self._checkout_head()
+            if ref == "base":
+                self._trees["base"] = self._checkout_base()
+        return self._trees[ref]
+
+    def _checkout_head(self) -> Path:
+        clones, folder = self._clones, self._clones.folder(self.repo)
+        clones.prune(self.repo)
+        head = folder / "head"
+        first = not (folder / _DONE).exists()
+        if first:
+            shutil.rmtree(head, ignore_errors=True)
+            shutil.rmtree(folder / "base", ignore_errors=True)
+            head.mkdir()
+            clones.git("init", "-q", cwd=head)
+            clones.git("remote", "add", "origin", clones.remote(self.repo), cwd=head)
+        started = time.monotonic()
+        try:
+            clones.git(
+                "fetch",
+                "-q",
+                "--prune",
+                "--no-tags",
+                "origin",
+                "+refs/heads/*:refs/remotes/origin/*",
+                f"+refs/pull/{self._number}/head:refs/review/pull/{self._number}",
+                cwd=head,
+            )
+        except BaseException:
+            if first:  # half a clone is no clone
+                shutil.rmtree(folder, ignore_errors=True)
+            raise
+        if first:
+            (folder / _DONE).touch()
+        clones._oplog.emit(
+            "review.clone" if first else "review.fetch",
+            cause=self.repo,
+            seconds=round(time.monotonic() - started, 1),
+            size_kb=_size_kb(clones, head),
+        )
+        try:
+            clones.git("cat-file", "-e", f"{self._head}^{{commit}}", cwd=head)
+        except CloneError:
+            raise CloneError(
+                f"{self._head[:12]} is no longer in {self.repo} (force-pushed?)"
+            ) from None
+        _put(clones, head, self._head)
+        return head
+
+    def _checkout_base(self) -> Path:
+        clones, head = self._clones, self._clones.folder(self.repo) / "head"
+        base = head.parent / "base"
+        commit = f"refs/remotes/origin/{self._base}"
+        if not (base / ".git").exists():
+            shutil.rmtree(base, ignore_errors=True)
+            clones.git("worktree", "prune", cwd=head)
+            clones.git("worktree", "add", "-q", "--detach", str(base), commit, cwd=head)
+            return base
+        _put(clones, base, commit)
+        return base
+
+
+def _put(clones: Clones, tree: Path, commit: str) -> None:
+    """The work tree at ``commit``, exactly: nothing else in it."""
+    git_dir = Path(clones.git("rev-parse", "--absolute-git-dir", cwd=tree).strip())
+    lock = git_dir / "index.lock"
+    if lock.is_file():
+        lock.unlink()  # left by a run killed mid-checkout; it's the only user
+    clones.git("checkout", "-q", "--force", "--detach", commit, cwd=tree)
+    clones.git("clean", "-q", "-ffdx", cwd=tree)
+
+
+def _size_kb(clones: Clones, head: Path) -> int:
+    counted = clones.git("count-objects", "-v", cwd=head)
+    sizes = dict(line.split(": ", 1) for line in counted.splitlines() if ": " in line)
+    return int(sizes.get("size", 0)) + int(sizes.get("size-pack", 0))
+
+
+def _used(folder: Path) -> float:
+    try:
+        return (folder / _USED).stat().st_mtime
+    except OSError:
+        return 0.0

nisse-review/src/nisse_review/reader.py

@@ -1,19 +1,18 @@
 """The repository, read-only, as the reviewer's executor (R3, L13).
 
 Reads are bound to the item's repository by construction: no tool names one.
-Code comes from Gitea's archive of the PR's head and of its base branch, one
-API call each, unpacked into a scratch folder on the agent host and read in
-Python: no git, no shell, nothing a model chose is ever run. Issues come
-from the Gitea API, for the same repository.
+Code comes from a clone kept on the agent host (``clones``), checked out at
+the PR's head and, when asked, its base branch, and is read in Python: git
+runs only the harness's fixed commands, and nothing a model chose is ever
+run. Git's own folder is never shown. Issues come from the Gitea API, for
+the same repository.
 """
 
-import io
+import os
 import posixpath
-import shutil
-import tarfile
 from collections.abc import Mapping
 from pathlib import Path
-from typing import Any
+from typing import Any, Protocol
 
 from nisse_core.gitea import GiteaClient, GiteaError
 from nisse_core.loop.driver import ToolOutcome
@@ -59,37 +58,29 @@ LIST_ISSUES = tool(
 READER_TOOLS = [READ_FILE, LIST_FILES, SEARCH_CODE, READ_ISSUE, LIST_ISSUES]
 
 
-class Repository:
-    """``refs``: what ``head`` and ``base`` name (commits or branches).
-    ``scratch``: a folder of this run's own, removed by ``close``."""
+class Trees(Protocol):
+    """Where a run's code is: ``tree("head")`` or ``tree("base")``, each a
+    folder of plain files, made on first use (``clones.Checkout``)."""
 
-    def __init__(
-        self,
-        gitea: GiteaClient,
-        repo: str,
-        refs: Mapping[str, str],
-        scratch: Path,
-    ) -> None:
+    def tree(self, ref: str) -> Path: ...
+
+
+class Sources(Protocol):
+    """Where runs get their code from (``clones.Clones``)."""
+
+    def open(self, repo: str, number: int, head: str, base: str) -> Trees: ...
+
+
+class Repository:
+    def __init__(self, gitea: GiteaClient, repo: str, trees: Trees) -> None:
         self._gitea = gitea
         self.repo = repo
-        self._refs = dict(refs)
-        self._scratch = scratch
-        self._trees: dict[str, Path] = {}
-
-    def close(self) -> None:
-        shutil.rmtree(self._scratch, ignore_errors=True)
+        self._trees = trees
 
     def tree(self, ref: str) -> Path:
-        """The repository at ``ref``, unpacked on first use."""
-        if ref not in self._refs:
+        if ref not in ("head", "base"):
             raise ValueError(f"ref is head or base, not {ref!r}")
-        if ref not in self._trees:
-            root = self._scratch / ref
-            shutil.rmtree(root, ignore_errors=True)
-            root.mkdir(parents=True)
-            _unpack(self._gitea.archive(self.repo, self._refs[ref]), root)
-            self._trees[ref] = root
-        return self._trees[ref]
+        return self._trees.tree(ref)
 
     def lines(self, path: str, ref: str = "head") -> list[str] | None:
         """A file's lines, or None if there's no such file."""
@@ -162,11 +153,7 @@ class Repository:
         start = root if not folder else root / (_clean(folder) or "\0")
         if not start.is_dir():
             return ToolOutcome(f"{folder}: no such folder at {ref}", False)
-        found = sorted(
-            str(p.relative_to(root))
-            for p in start.rglob("*")
-            if p.is_file() and not p.is_symlink()
-        )
+        found = sorted(str(p.relative_to(root)) for p in _files(start, root))
         shown = found[:MAX_LISTED]
         more = f"\n[{len(found) - len(shown)} more]" if len(found) > len(shown) else ""
         return ToolOutcome(
@@ -184,9 +171,7 @@ class Repository:
         root = self.tree(ref)
         needle = text.lower()
         hits: list[str] = []
-        for path in sorted(root.rglob("*")):
-            if not path.is_file() or path.is_symlink():
-                continue
+        for path in sorted(_files(root, root)):
             try:
                 content = path.read_text()
             except (UnicodeDecodeError, OSError):
@@ -236,29 +221,23 @@ def _clean(path: str) -> str | None:
     if not path or path.startswith("/") or "\\" in path:
         return None
     clean = posixpath.normpath(path)
-    if clean == "." or clean.startswith(".."):
+    if clean == "." or clean.startswith("..") or clean.split("/")[0] == ".git":
         return None
     return clean
 
 
-def _unpack(archive: bytes, root: Path) -> None:
-    """Gitea's archive, without its top folder; only plain files and
-    folders, each inside ``root``."""
-    with tarfile.open(fileobj=io.BytesIO(archive), mode="r:gz") as tar:
-        for member in tar.getmembers():
-            if not (member.isfile() or member.isdir()):
-                continue  # links and devices are never followed
-            parts = member.name.split("/", 1)
-            if len(parts) < 2 or not parts[1]:
-                continue
-            inner = _clean(parts[1])
-            if inner is None:
-                continue
-            target = root / inner
-            if member.isdir():
-                target.mkdir(parents=True, exist_ok=True)
-                continue
-            target.parent.mkdir(parents=True, exist_ok=True)
-            source = tar.extractfile(member)
-            if source is not None:
-                target.write_bytes(source.read())
+def _files(under: Path, root: Path) -> list[Path]:
+    """The plain files under a folder; git's own folder is never walked."""
+    found: list[Path] = []
+    for folder, dirs, files in os.walk(under):
+        here = Path(folder)
+        if here == root:
+            dirs[:] = [d for d in dirs if d != ".git"]
+        found.extend(
+            here / name
+            for name in files
+            if not (here == root and name == ".git")
+            and (here / name).is_file()
+            and not (here / name).is_symlink()
+        )
+    return found

nisse-review/src/nisse_review/scheduler.py

@@ -24,12 +24,10 @@ to ready, with nothing drafted posted and no failure counted.
 """
 
 import re
-import tempfile
 import threading
 import time
 from collections.abc import Callable
 from dataclasses import replace
-from pathlib import Path
 
 from nisse_core.gitea import GiteaClient, GiteaError, PullRequest
 from nisse_core.models.interruptible import Stopped
@@ -37,7 +35,7 @@ from nisse_core.oplog import OpLog
 from nisse_core.state.cancel import Cancel
 from nisse_core.state.store import WorkItem
 from nisse_review.access import check, reviewed
-from nisse_review.reader import Repository
+from nisse_review.reader import Repository, Sources
 from nisse_review.review import Draft, Posted, Reviewer, Said, answered, said_on
 from nisse_review.state import (
     CAPPED,
@@ -92,7 +90,7 @@ class Scheduler:
         reviewer: Reviewer,
         *,
         oplog: OpLog,
-        scratch: Path,
+        sources: Sources,
         poll_s: float = 60.0,
         attempts: int = 2,
         patterns: frozenset[str] | None = None,
@@ -107,7 +105,7 @@ class Scheduler:
         self._gitea = gitea
         self._reviewer = reviewer
         self._oplog = oplog
-        self._scratch = scratch
+        self._sources = sources  # the repositories' code, as clones
         self._poll_s = poll_s
         self._attempts = attempts
         # The repositories reviewed, as patterns; None: every one it was given.
@@ -388,19 +386,11 @@ class Scheduler:
             if kind == REPLIES and not known.fresh:
                 self._settle(item, known, pr, "done", Draft(), posted=None)
                 return
-            scratch = Path(tempfile.mkdtemp(dir=self._scratch, prefix="review-"))
-            repo = Repository(
-                self._gitea,
-                known.repo,
-                {"head": pr.head_sha, "base": pr.base},
-                scratch,
+            trees = self._sources.open(known.repo, pr.number, pr.head_sha, pr.base)
+            repo = Repository(self._gitea, known.repo, trees)
+            outcome, draft = self._reviewer.run(
+                item.id, pr, repo, kind=kind, known=known, said=said
             )
-            try:
-                outcome, draft = self._reviewer.run(
-                    item.id, pr, repo, kind=kind, known=known, said=said
-                )
-            finally:
-                repo.close()
             if outcome.kind == "capped" and draft.comments:
                 # Out of turns with points drafted: they're worth more posted
                 # than lost (#156), as a review that says it's partial.