"""The repositories the reviewer reads, as git clones kept between runs (R3). Each repository is cloned once, with its whole history, under ``root``: root///head/ the clone, checked out at the PR's head root///base/ a worktree at the base branch, when read A run fetches what changed since (every branch, and the PR's own ref, which also reaches a PR from a fork), checks out the head it reviews, and removes whatever an earlier PR left. Only the most recently used repositories are kept (``keep``); the rest are deleted when another is opened. Git runs on code nobody vetted, so it runs with none of this host's git configuration, links checked out as plain files (never pointing out of the clone), no hooks, no submodules, no LFS downloads, and only the configured Gitea's protocol. The token reaches git in its environment, as a header for Gitea's address alone: never on a command line, in a file, or in a URL. A stop (``cancel``) ends a git command at once: it raises ``Stopped``, and a first clone it cut short is deleted, to be made afresh next time. """ import os import shutil import subprocess import time from pathlib import Path from nisse_core.models.interruptible import Stopped from nisse_core.oplog import OpLog from nisse_core.state.cancel import Cancel _DONE = "cloned" # in a repository's folder once its first fetch finished _USED = "used" # touched each time it is opened: which to keep class CloneError(RuntimeError): """Git couldn't give the code: the run fails, and is tried again.""" class Clones: def __init__( self, root: Path, gitea_url: str, token: str, *, oplog: OpLog, cancel: Cancel | None = None, keep: int = 1, ) -> None: self._root = root self._url = gitea_url.rstrip("/") self._token = token self._oplog = oplog self._cancel = cancel or Cancel() self._keep = keep root.mkdir(parents=True, exist_ok=True) def open(self, repo: str, number: int, head: str, base: str) -> "Checkout": """``repo``'s PR ``number``, at ``head``, merging into ``base``. Git runs only when a tree is first read.""" return Checkout(self, repo, number, head, base) # -- for Checkout ----------------------------------------------------------- def folder(self, repo: str) -> Path: owner, _, name = repo.partition("/") return self._root / owner / name def prune(self, repo: str) -> None: """Keep ``repo`` and the most recently used others, to ``keep``.""" mine = self.folder(repo) mine.mkdir(parents=True, exist_ok=True) (mine / _USED).touch() others = [ folder for owner in self._root.iterdir() if owner.is_dir() for folder in owner.iterdir() if folder.is_dir() and folder != mine ] others.sort(key=_used, reverse=True) for folder in others[self._keep - 1 :]: shutil.rmtree(folder, ignore_errors=True) self._oplog.emit( "review.clone_removed", cause="not recent", path=str(folder) ) for owner in self._root.iterdir(): if owner.is_dir() and not any(owner.iterdir()): owner.rmdir() def git(self, *args: str, cwd: Path) -> str: """One git command, in the walled environment; its output.""" with subprocess.Popen( ["git", *args], cwd=cwd, env=self._env(), stdin=subprocess.DEVNULL, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, ) as proc: while True: try: out, err = proc.communicate(timeout=0.5) break except subprocess.TimeoutExpired: if self._cancel.requested: proc.terminate() # git removes its lock files proc.communicate() raise Stopped(f"a stop ended git {args[0]}") from None if proc.returncode != 0: raise CloneError(f"git {args[0]} failed: {err.strip()[-500:]}") return out def remote(self, repo: str) -> str: return f"{self._url}/{repo}.git" def _env(self) -> dict[str, str]: settings = { "core.symlinks": "false", "core.hooksPath": os.devnull, "core.fsmonitor": "false", "submodule.recurse": "false", "fetch.recurseSubmodules": "false", "advice.detachedHead": "false", "gc.auto": "0", # no background work left running after a stop } if self._token: settings[f"http.{self._url}/.extraHeader"] = ( f"Authorization: token {self._token}" ) env = { "PATH": os.environ.get("PATH", "/usr/bin:/bin"), "HOME": str(self._root), # no ~/.gitconfig of this host's "GIT_CONFIG_NOSYSTEM": "1", "GIT_CONFIG_GLOBAL": os.devnull, "GIT_TERMINAL_PROMPT": "0", # Never another repository above a clone, should one be missing. "GIT_CEILING_DIRECTORIES": str(self._root), "GIT_LFS_SKIP_SMUDGE": "1", "GIT_ALLOW_PROTOCOL": self._url.partition(":")[0], "GIT_CONFIG_COUNT": str(len(settings)), } for n, (key, value) in enumerate(settings.items()): env[f"GIT_CONFIG_KEY_{n}"] = key env[f"GIT_CONFIG_VALUE_{n}"] = value return env class Checkout: """One run's view of a repository: ``tree("head")`` and ``tree("base")``, each a folder of plain files.""" def __init__( self, clones: Clones, repo: str, number: int, head: str, base: str ) -> None: self._clones = clones self.repo = repo self._number = number self._head = head self._base = base self._trees: dict[str, Path] = {} def tree(self, ref: str) -> Path: if ref not in ("head", "base"): raise ValueError(f"ref is head or base, not {ref!r}") if ref not in self._trees: if "head" not in self._trees: self._trees["head"] = self._checkout_head() if ref == "base": self._trees["base"] = self._checkout_base() return self._trees[ref] def _checkout_head(self) -> Path: clones, folder = self._clones, self._clones.folder(self.repo) clones.prune(self.repo) head = folder / "head" first = not (folder / _DONE).exists() if first: shutil.rmtree(head, ignore_errors=True) shutil.rmtree(folder / "base", ignore_errors=True) head.mkdir() clones.git("init", "-q", cwd=head) clones.git("remote", "add", "origin", clones.remote(self.repo), cwd=head) started = time.monotonic() try: clones.git( "fetch", "-q", "--prune", "--no-tags", "origin", "+refs/heads/*:refs/remotes/origin/*", f"+refs/pull/{self._number}/head:refs/review/pull/{self._number}", cwd=head, ) except BaseException: if first: # half a clone is no clone shutil.rmtree(folder, ignore_errors=True) raise if first: (folder / _DONE).touch() clones._oplog.emit( "review.clone" if first else "review.fetch", cause=self.repo, seconds=round(time.monotonic() - started, 1), size_kb=_size_kb(clones, head), ) try: clones.git("cat-file", "-e", f"{self._head}^{{commit}}", cwd=head) except CloneError: raise CloneError( f"{self._head[:12]} is no longer in {self.repo} (force-pushed?)" ) from None _put(clones, head, self._head) return head def _checkout_base(self) -> Path: clones, head = self._clones, self._clones.folder(self.repo) / "head" base = head.parent / "base" commit = f"refs/remotes/origin/{self._base}" if not (base / ".git").exists(): shutil.rmtree(base, ignore_errors=True) clones.git("worktree", "prune", cwd=head) clones.git("worktree", "add", "-q", "--detach", str(base), commit, cwd=head) return base _put(clones, base, commit) return base def _put(clones: Clones, tree: Path, commit: str) -> None: """The work tree at ``commit``, exactly: nothing else in it.""" git_dir = Path(clones.git("rev-parse", "--absolute-git-dir", cwd=tree).strip()) lock = git_dir / "index.lock" if lock.is_file(): lock.unlink() # left by a run killed mid-checkout; it's the only user clones.git("checkout", "-q", "--force", "--detach", commit, cwd=tree) clones.git("clean", "-q", "-ffdx", cwd=tree) def _size_kb(clones: Clones, head: Path) -> int: counted = clones.git("count-objects", "-v", cwd=head) sizes = dict(line.split(": ", 1) for line in counted.splitlines() if ": " in line) return int(sizes.get("size", 0)) + int(sizes.get("size-pack", 0)) def _used(folder: Path) -> float: try: return (folder / _USED).stat().st_mtime except OSError: return 0.0