nisse/nisse-review/src/nisse_review/clones.py

"""The repositories the reviewer reads, as git clones kept between runs (R3).

Each repository is cloned once, with its whole history, under ``root``:

    root/<owner>/<name>/head/   the clone, checked out at the PR's head
    root/<owner>/<name>/base/   a worktree at the base branch, when read

A run fetches what changed since (every branch, and the PR's own ref, which
also reaches a PR from a fork), checks out the head it reviews, and removes
whatever an earlier PR left. Only the most recently used repositories are
kept (``keep``); the rest are deleted when another is opened.

Git runs on code nobody vetted, so it runs with none of this host's git
configuration, links checked out as plain files (never pointing out of the
clone), no hooks, no submodules, no LFS downloads, and only the configured
Gitea's protocol. The token reaches git in its environment, as a header for
Gitea's address alone: never on a command line, in a file, or in a URL.

A stop (``cancel``) ends a git command at once: it raises ``Stopped``, and a
first clone it cut short is deleted, to be made afresh next time.
"""

import os
import shutil
import subprocess
import time
from pathlib import Path

from nisse_core.models.interruptible import Stopped
from nisse_core.oplog import OpLog
from nisse_core.state.cancel import Cancel

_DONE = "cloned"  # in a repository's folder once its first fetch finished
_USED = "used"  # touched each time it is opened: which to keep


class CloneError(RuntimeError):
    """Git couldn't give the code: the run fails, and is tried again."""


class Clones:
    def __init__(
        self,
        root: Path,
        gitea_url: str,
        token: str,
        *,
        oplog: OpLog,
        cancel: Cancel | None = None,
        keep: int = 1,
    ) -> None:
        self._root = root
        self._url = gitea_url.rstrip("/")
        self._token = token
        self._oplog = oplog
        self._cancel = cancel or Cancel()
        self._keep = keep
        root.mkdir(parents=True, exist_ok=True)

    def open(self, repo: str, number: int, head: str, base: str) -> "Checkout":
        """``repo``'s PR ``number``, at ``head``, merging into ``base``. Git
        runs only when a tree is first read."""
        return Checkout(self, repo, number, head, base)

    # -- for Checkout -----------------------------------------------------------

    def folder(self, repo: str) -> Path:
        owner, _, name = repo.partition("/")
        return self._root / owner / name

    def prune(self, repo: str) -> None:
        """Keep ``repo`` and the most recently used others, to ``keep``."""
        mine = self.folder(repo)
        mine.mkdir(parents=True, exist_ok=True)
        (mine / _USED).touch()
        others = [
            folder
            for owner in self._root.iterdir()
            if owner.is_dir()
            for folder in owner.iterdir()
            if folder.is_dir() and folder != mine
        ]
        others.sort(key=_used, reverse=True)
        for folder in others[self._keep - 1 :]:
            shutil.rmtree(folder, ignore_errors=True)
            self._oplog.emit(
                "review.clone_removed", cause="not recent", path=str(folder)
            )
        for owner in self._root.iterdir():
            if owner.is_dir() and not any(owner.iterdir()):
                owner.rmdir()

    def git(self, *args: str, cwd: Path) -> str:
        """One git command, in the walled environment; its output."""
        with subprocess.Popen(
            ["git", *args],
            cwd=cwd,
            env=self._env(),
            stdin=subprocess.DEVNULL,
            stdout=subprocess.PIPE,
            stderr=subprocess.PIPE,
            text=True,
        ) as proc:
            while True:
                try:
                    out, err = proc.communicate(timeout=0.5)
                    break
                except subprocess.TimeoutExpired:
                    if self._cancel.requested:
                        proc.terminate()  # git removes its lock files
                        proc.communicate()
                        raise Stopped(f"a stop ended git {args[0]}") from None
        if proc.returncode != 0:
            raise CloneError(f"git {args[0]} failed: {err.strip()[-500:]}")
        return out

    def remote(self, repo: str) -> str:
        return f"{self._url}/{repo}.git"

    def _env(self) -> dict[str, str]:
        settings = {
            "core.symlinks": "false",
            "core.hooksPath": os.devnull,
            "core.fsmonitor": "false",
            "submodule.recurse": "false",
            "fetch.recurseSubmodules": "false",
            "advice.detachedHead": "false",
            "gc.auto": "0",  # no background work left running after a stop
        }
        if self._token:
            settings[f"http.{self._url}/.extraHeader"] = (
                f"Authorization: token {self._token}"
            )
        env = {
            "PATH": os.environ.get("PATH", "/usr/bin:/bin"),
            "HOME": str(self._root),  # no ~/.gitconfig of this host's
            "GIT_CONFIG_NOSYSTEM": "1",
            "GIT_CONFIG_GLOBAL": os.devnull,
            "GIT_TERMINAL_PROMPT": "0",
            # Never another repository above a clone, should one be missing.
            "GIT_CEILING_DIRECTORIES": str(self._root),
            "GIT_LFS_SKIP_SMUDGE": "1",
            "GIT_ALLOW_PROTOCOL": self._url.partition(":")[0],
            "GIT_CONFIG_COUNT": str(len(settings)),
        }
        for n, (key, value) in enumerate(settings.items()):
            env[f"GIT_CONFIG_KEY_{n}"] = key
            env[f"GIT_CONFIG_VALUE_{n}"] = value
        return env


class Checkout:
    """One run's view of a repository: ``tree("head")`` and
    ``tree("base")``, each a folder of plain files."""

    def __init__(
        self, clones: Clones, repo: str, number: int, head: str, base: str
    ) -> None:
        self._clones = clones
        self.repo = repo
        self._number = number
        self._head = head
        self._base = base
        self._trees: dict[str, Path] = {}

    def tree(self, ref: str) -> Path:
        if ref not in ("head", "base"):
            raise ValueError(f"ref is head or base, not {ref!r}")
        if ref not in self._trees:
            if "head" not in self._trees:
                self._trees["head"] = self._checkout_head()
            if ref == "base":
                self._trees["base"] = self._checkout_base()
        return self._trees[ref]

    def _checkout_head(self) -> Path:
        clones, folder = self._clones, self._clones.folder(self.repo)
        clones.prune(self.repo)
        head = folder / "head"
        first = not (folder / _DONE).exists()
        if first:
            shutil.rmtree(head, ignore_errors=True)
            shutil.rmtree(folder / "base", ignore_errors=True)
            head.mkdir()
            clones.git("init", "-q", cwd=head)
            clones.git("remote", "add", "origin", clones.remote(self.repo), cwd=head)
        started = time.monotonic()
        try:
            clones.git(
                "fetch",
                "-q",
                "--prune",
                "--no-tags",
                "origin",
                "+refs/heads/*:refs/remotes/origin/*",
                f"+refs/pull/{self._number}/head:refs/review/pull/{self._number}",
                cwd=head,
            )
        except BaseException:
            if first:  # half a clone is no clone
                shutil.rmtree(folder, ignore_errors=True)
            raise
        if first:
            (folder / _DONE).touch()
        clones._oplog.emit(
            "review.clone" if first else "review.fetch",
            cause=self.repo,
            seconds=round(time.monotonic() - started, 1),
            size_kb=_size_kb(clones, head),
        )
        try:
            clones.git("cat-file", "-e", f"{self._head}^{{commit}}", cwd=head)
        except CloneError:
            raise CloneError(
                f"{self._head[:12]} is no longer in {self.repo} (force-pushed?)"
            ) from None
        _put(clones, head, self._head)
        return head

    def _checkout_base(self) -> Path:
        clones, head = self._clones, self._clones.folder(self.repo) / "head"
        base = head.parent / "base"
        commit = f"refs/remotes/origin/{self._base}"
        if not (base / ".git").exists():
            shutil.rmtree(base, ignore_errors=True)
            clones.git("worktree", "prune", cwd=head)
            clones.git("worktree", "add", "-q", "--detach", str(base), commit, cwd=head)
            return base
        _put(clones, base, commit)
        return base


def _put(clones: Clones, tree: Path, commit: str) -> None:
    """The work tree at ``commit``, exactly: nothing else in it."""
    git_dir = Path(clones.git("rev-parse", "--absolute-git-dir", cwd=tree).strip())
    lock = git_dir / "index.lock"
    if lock.is_file():
        lock.unlink()  # left by a run killed mid-checkout; it's the only user
    clones.git("checkout", "-q", "--force", "--detach", commit, cwd=tree)
    clones.git("clean", "-q", "-ffdx", cwd=tree)


def _size_kb(clones: Clones, head: Path) -> int:
    counted = clones.git("count-objects", "-v", cwd=head)
    sizes = dict(line.split(": ", 1) for line in counted.splitlines() if ": " in line)
    return int(sizes.get("size", 0)) + int(sizes.get("size-pack", 0))


def _used(folder: Path) -> float:
    try:
        return (folder / _USED).stat().st_mtime
    except OSError:
        return 0.0