From ad2d090143c8595d7d5a6cb3c16f8fca280f489d Mon Sep 17 00:00:00 2001 From: Don Garrett Date: Tue, 6 Oct 2026 14:07:37 +0000 Subject: [PATCH] New projects from conversation (M2.5, PR B) - propose_project(name, purpose, decisions) in a conversation goes up for a go-ahead (checked: a slug, not an existing project, no trailing question). - On go, the provisioning checklist (D1): a repository with a README, protection on its default branch, the agent as a collaborator, a build VM, a deploy key, and the project entry as YAML with <...> to fill in. - On "done", the agent reads its configuration again and adds the new project live (build_loop_agent(load=...); only additions), then checks what was provided (D4, verified, not assumed): permissions and protection as it sees them, then on a throwaway branch with an empty commit, a branch push must succeed and a push to the default branch and merging its own PR must be refused. The check's PR is closed and its branch deleted. Failures are said per item; nothing starts. - Checked: the tracking issue (the distilled decisions, carrying the new item's ID), and the project's item, agreed and labelled `design`, whose first milestone is ARCHITECTURE.md and the build plan. A conversation waiting on provisioning doesn't close as idle. - The walk replays exactly from its seed (item IDs too), and event IDs stay unique across simulated restarts, as a server's are. - The test config holds output to 8192 tokens, under the test key's per-request cost cap. Tests: new-project unit tests (checklist and its YAML parse as config, done before the entry, what isn't set up, a checked project's record and first PR, an existing name), protection enforced by a hook on the fake forge, the walk proposing projects and saying done, and an e2e new project in a repository the test operator provisions from the agent's checklist. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01PPQ14L3LikyefCMFBqEbKE --- docs/Development-Plan.md | 18 +- nisse-dev/src/nisse_dev/gitea.py | 13 ++ nisse-dev/src/nisse_dev/loop/agent.py | 179 ++++++++++++++++++- nisse-dev/src/nisse_dev/loop/build.py | 95 ++++++---- nisse-dev/src/nisse_dev/loop/items.py | 3 + nisse-dev/src/nisse_dev/loop/milestones.py | 75 ++++++++ nisse-dev/src/nisse_dev/loop/tools.py | 15 ++ nisse-dev/tests/test_loop_milestones.py | 28 ++- nisse-dev/tests/test_loop_new_projects.py | 193 +++++++++++++++++++++ nisse-dev/tests/test_loop_properties.py | 36 +++- tests/integration/e2e_support.py | 23 ++- tests/integration/nisse-test.yaml | 4 + tests/integration/test_end_to_end_loop.py | 95 +++++++++- 13 files changed, 723 insertions(+), 54 deletions(-) create mode 100644 nisse-dev/tests/test_loop_new_projects.py diff --git a/docs/Development-Plan.md b/docs/Development-Plan.md index e7cb803..3103835 100644 --- a/docs/Development-Plan.md +++ b/docs/Development-Plan.md @@ -192,7 +192,7 @@ between milestones (D2, D4). End-to-end: a two-milestone feature. - Deferred: a milestone with no changes that wasn't expected to change code is still raised with the operator (D4). -**M2.5 Conversations and new projects.** *Needs discussion;* the design was agreed +**M2.5 Conversations and new projects.** **Done** (#106 and this PR). *Needs discussion;* the design was agreed in this PR (`general` scope, L1, L5; the new-project sequence, D1). Work started in conversation, record issues carrying their item's ID, the new-project flow and its project entry (D1, D8, #29). After M1.4. End-to-end: a small new project, in a @@ -207,8 +207,20 @@ deletes afterwards. settles a filing cut off by a crash. - A conversation closes after `tuning.loop.general_idle_s` without a message; a reply in its thread then starts a new one that continues it. -- **PR B, new projects:** the proposal, the checklist, reload-and-verify, and the - tracking issue; the e2e with a per-run repository. +- **PR B, new projects — done (this PR).** + - `propose_project(name, purpose, decisions)` goes up for a go-ahead. + - On go, the checklist: repository, protection, collaborator, build VM, deploy + key, and the project entry as YAML to fill in. + - On "done", the agent reads its configuration again and adds the new project + live (only additions). It then checks the access: the repository's + permissions and protection as the agent sees them, then in practice on a + throwaway branch with an empty commit (a branch push must succeed, a push to + the default branch and merging its own PR must be refused); the branch and PR + are removed. Anything that fails is said, and nothing starts. + - When all is well: the tracking issue with the distilled decisions, and the + project's item, agreed and labelled `design`, whose first milestone is + `ARCHITECTURE.md` and the build plan. A conversation waiting on provisioning + doesn't close as idle. **M2.6 Roles and capability profiles.** *Pre-approved.* `driver`, `helper`, `expert`, `classifier` (L9); escalation triggers; delegation and `ask_expert` on the diff --git a/nisse-dev/src/nisse_dev/gitea.py b/nisse-dev/src/nisse_dev/gitea.py index 04734b6..dd22a8a 100644 --- a/nisse-dev/src/nisse_dev/gitea.py +++ b/nisse-dev/src/nisse_dev/gitea.py @@ -261,6 +261,19 @@ class GiteaClient: who = (event.get("user") or {}).get("login") return who + def repository(self, repo: str) -> dict[str, Any]: + """The repository as this account sees it, permissions included.""" + return dict(self._request("GET", f"/repos/{repo}")) + + def branch(self, repo: str, name: str) -> dict[str, Any]: + """A branch as this account sees it: protected, user_can_push.""" + return dict(self._request("GET", f"/repos/{repo}/branches/{name}")) + + def merge_pull_request(self, repo: str, index: int) -> None: + self._request( + "POST", f"/repos/{repo}/pulls/{index}/merge", json={"Do": "merge"} + ) + def assign_issue(self, repo: str, index: int, assignee: str) -> None: self._request( "PATCH", f"/repos/{repo}/issues/{index}", json={"assignees": [assignee]} diff --git a/nisse-dev/src/nisse_dev/loop/agent.py b/nisse-dev/src/nisse_dev/loop/agent.py index f69c278..39a422d 100644 --- a/nisse-dev/src/nisse_dev/loop/agent.py +++ b/nisse-dev/src/nisse_dev/loop/agent.py @@ -36,6 +36,7 @@ from nisse_dev.loop.items import ( CLOSED, GO_AHEAD, MERGE, + PROVISION, READY, RUNNING, WAITING, @@ -51,6 +52,7 @@ from nisse_dev.loop.tools import ( ASK_HUMAN, CLOSE_ITEM, FINISH_MILESTONE, + PROPOSE_PROJECT, PROPOSE_WORK, REPLY, RESTATE, @@ -133,9 +135,10 @@ is the new issue's body and the work's whole brief. {projects} -Setting up a new project from here isn't available yet: if that's where the -conversation goes, say so, and that the operator can create the repository -and its project entry.""" +If it becomes a new project, use propose_project: its name, what it's for, +and the decisions setting it up depends on. Once agreed, you'll be asked for +what it needs set up; when the operator has it ready and it checks out, its +architecture and build plan are the first piece of work.""" # What an issue's labels add to the driver's instructions (D1). LABELS = { @@ -204,6 +207,9 @@ class LoopAgent: reader: GoAheadReader | None = None, bug_grace_s: float = BUG_GRACE_S, general_idle_s: float = GENERAL_IDLE_S, + # A new project's entry, once the operator has added it (D1, D8): + # the projects configured now that this agent doesn't know yet. + reload: Callable[[set[str]], Sequence[Project]] | None = None, clock: Callable[[], float] = time.time, intake_interval_s: float = INTAKE_INTERVAL_S, ) -> None: @@ -224,6 +230,7 @@ class LoopAgent: self._reader = reader self._grace = bug_grace_s self._idle = general_idle_s + self._reload = reload self._clock = clock self._interval = intake_interval_s # thread root -> revision on screen there, read on the transport's @@ -566,6 +573,7 @@ class LoopAgent: if ( item.scope == GENERAL and item.position == WAITING + and not loop.new_project # setting one up can take days and loop.idle_since and now - loop.idle_since >= self._idle ): @@ -761,6 +769,8 @@ class LoopAgent: item = self._store.work_item(item.id) or item if command == "abandon": self._close(item, loop, "abandoned", f"abandoned by {msg.sender}") + elif command == "done" and loop.new_project: + self._provisioned(item, loop) elif command == "stop": # Stopping a run took effect on arrival; a stop is never an answer. if not loop.note.startswith("stopped ("): @@ -850,6 +860,9 @@ class LoopAgent: def _go(self, item: WorkItem, loop: Loop, by: str) -> None: self._fact(item, f"go-ahead on rev-{loop.rev} from {by}") + if item.scope == GENERAL and loop.proposal_kind == "project": + self._checklist(item, loop) + return if item.scope == GENERAL: self._file_work(item, loop) return @@ -949,6 +962,116 @@ class LoopAgent: "We can keep talking here.", ) + # -- new projects (D1, D8) -------------------------------------------------- + + def _checklist(self, item: WorkItem, loop: Loop) -> None: + """An agreed new project: what it needs, for the operator to provide + (later nisse-ops, O6), with its project entry to add.""" + name = loop.proposal_project + entry = ( + "projects:\n" + f" {name}:\n" + f" repo: /{name}\n" + " base_branch: main\n" + f" workspace: /root/work/{name}\n" + " build_host:\n" + " address: \n" + " key: \n" + " push_key: " + ) + self._fact(item, f"provisioning checklist posted for {name}") + self._wait( + item, + replace( + loop, + wait=PROVISION, + note=f"{name}'s setup: say done when it's ready", + new_project=name, + idle_since=self._clock(), + ), + f"To set up `{name}`, I need:\n" + f"1. A repository for it, e.g. /{name}, with a README so its " + "default branch exists.\n" + "2. Protection on the default branch: none of my pushes, and an " + "approval to merge.\n" + f"3. Me ({self._account}) as a collaborator with write access.\n" + "4. A build VM, with the toolchain from the decisions above.\n" + "5. A deploy key with write access, its private half where I run.\n" + "6. This entry in my configuration, with the <...> parts filled " + f"in:\n\n```yaml\n{entry}\n```\n\n" + "Say done when it's ready, and I'll check each of them.", + ) + + def _provisioned(self, item: WorkItem, loop: Loop) -> None: + """ "done": the entry is read again and the new project checked + (D4: verified, not assumed). What fails is said, per item; when all + is well the tracking issue is opened, and the project's item starts + with its architecture.""" + name = loop.new_project + try: + added = list(self._reload(set(self._projects))) if self._reload else [] + except Exception as exc: # noqa: BLE001 -- said; the operator fixes it + self._transport.post( + f"I couldn't read the new entry: {exc}. Fix it and say done again.", + thread=loop.thread, + ) + return + project = next((p for p in added if p.name == name), None) + if project is None: + self._transport.post( + f"I don't see `{name}` in my configuration yet (item 6). Add its " + "entry and say done again.", + thread=loop.thread, + ) + return + failures = ( + project.milestones.verify_access(self._account) + if project.milestones is not None + else ["the project has no workspace"] + ) + if failures: + self._fact(item, f"{name} checked: " + "; ".join(failures)) + self._transport.post( + "Not ready yet: " + "; ".join(failures) + ". Say done again when " + "they're fixed.", + thread=loop.thread, + ) + return + self._projects[name] = project + self._fact(item, f"{name} set up and checked: {project.repo}") + brief = self._latest_proposal(item) + new_id = f"{name}-c{uuid.uuid4().hex[:6]}" + title = f"Implement the build plan for {name}" + try: + number = self._gitea.create_issue( + project.repo, + title, + f"{brief}\n\nFrom a conversation with the operator, agreed there " + f"as rev-{loop.rev}. First: ARCHITECTURE.md and the build plan." + f"\n\n", + ) + self._gitea.assign_issue(project.repo, number, self._account) + except GiteaError as exc: + self._transport.post( + f"{name} checks out, but I couldn't open its tracking issue: " + f"{exc}. Say done to try again.", + thread=loop.thread, + ) + return + # D1: the architecture is agreed in its PR, as `design` steers. + design = replace( + project, labels=tuple(dict.fromkeys(("design", *project.labels))) + ) + self._start_filed(item, design, new_id, number, title, brief) + loop = loop_of(self._store.work_item(item.id) or item) + save( + self._store, + item.id, + WAITING, + replace(loop, new_project="", proposal_kind=""), + cause=f"{name} provisioned", + ) + def _latest_proposal(self, item: WorkItem) -> str: proposals = [ e.event.plan @@ -1102,15 +1225,30 @@ class LoopAgent: outcome = self._driver.run( self._store.transcript(item.id, "driver"), system=system, - tools=[REPLY, ASK_HUMAN, PROPOSE_WORK], + tools=[REPLY, ASK_HUMAN, PROPOSE_WORK, PROPOSE_PROJECT], workspace=None, - checks={"propose_work": self._proposal_problem}, + checks={ + "propose_work": self._proposal_problem, + "propose_project": self._project_problem, + }, ) finally: self._running = False self._cancel.clear() self._settle(item.id, outcome) + def _project_problem(self, args: dict[str, object]) -> str | None: + name = str(args.get("name", "")).strip() + if not re.fullmatch(r"[a-z0-9][a-z0-9-]*", name) or name == GENERAL: + return "a project's name is lowercase letters, digits and -, not general" + if name in self._projects: + return f"{name} is a project already: propose_work for work on it" + if not str(args.get("purpose", "")).strip(): + return "say what the project is for" + return _restatement_problem( + {"text": f"{args.get('purpose', '')}\n{args.get('decisions', '')}"} + ) + def _proposal_problem(self, args: dict[str, object]) -> str | None: project = str(args.get("project", "")).strip() if project not in self._projects: @@ -1233,6 +1371,7 @@ class LoopAgent: None, ) elif outcome.kind == "wait" and outcome.tool == "propose_work": + loop = replace(loop, proposal_kind="work") project = self._projects[str(args.get("project", "")).strip()] title = str(args.get("title", "")).strip() summary = str(args.get("summary", "")).strip() @@ -1261,6 +1400,36 @@ class LoopAgent: ), None, ) + elif outcome.kind == "wait" and outcome.tool == "propose_project": + name = str(args.get("name", "")).strip() + purpose = str(args.get("purpose", "")).strip() + decisions = str(args.get("decisions", "")).strip() + brief = f"{purpose}\n\nDecisions:\n{decisions}" if decisions else purpose + rev = loop.rev + 1 + self._store.transcript(item.id, "driver").append( + PlanSubmitted(plan=brief, model=self._roles.get("planner", "")) + ) + self._fact(item, f"new project proposed as rev-{rev}: {name}") + self._shown[loop.thread] = rev + self._transport.post( + f"rev-{rev}: a new project, `{name}`. {brief}\n\nSay go and I'll " + "list what it needs set up, or reply with changes.", + thread=loop.thread, + ) + self._wait( + item, + replace( + loop, + rev=rev, + wait=GO_AHEAD, + note="", + proposal_project=name, + proposal_title="", + proposal_kind="project", + idle_since=self._clock(), + ), + None, + ) elif outcome.kind == "wait" and outcome.tool == "ask_human": question = str(args.get("question", "")).strip() self._answer_where_asked(item, loop, question) diff --git a/nisse-dev/src/nisse_dev/loop/build.py b/nisse-dev/src/nisse_dev/loop/build.py index 6542204..634b6c7 100644 --- a/nisse-dev/src/nisse_dev/loop/build.py +++ b/nisse-dev/src/nisse_dev/loop/build.py @@ -2,6 +2,7 @@ LoopAgent in place of the Orchestrator. Chosen with `loop: new` in config.""" import threading +from collections.abc import Callable from dataclasses import dataclass from nisse_dev.agent import ( @@ -14,7 +15,8 @@ from nisse_dev.agent import ( remote_url, ) from nisse_dev.cancel import Cancel -from nisse_dev.config import AgentConfig +from nisse_dev.config import AgentConfig, ConfigError +from nisse_dev.config import Project as ProjectEntry from nisse_dev.gitea import GiteaClient from nisse_dev.loop.agent import LoopAgent, Project from nisse_dev.loop.driver import Driver @@ -64,7 +66,48 @@ class RunningLoop: self._stopped.wait() -def build_loop_agent(config: AgentConfig) -> RunningLoop: +def project_runtime( + config: AgentConfig, + entry: ProjectEntry, + store: StateStore, + gitea: GiteaClient, + account: str, +) -> Project: + """One project's part of the loop: its clone on its build VM, workspace + (with the secrets its commands may see, redacted on return), push + credential and milestones.""" + host = build_host(config, entry) + credential = push_credential(config, entry) + remote = remote_url(gitea, entry.repo, credential) + git = prepare_workspace(host, config, account, remote, credential, entry) + return Project( + name=entry.name, + repo=entry.repo, + labels=entry.labels, + workspace=Workspace( + host, + entry.workspace, + timeout_s=config.tuning.tools.command_timeout_s, + max_output_chars=config.tuning.tools.max_output_chars, + secrets=entry.command_secrets(), + ), + milestones=Milestones( + store, + git, + gitea, + repo=entry.repo, + remote=remote, + credential=credential, + base=entry.base_branch, + ), + ) + + +def build_loop_agent( + config: AgentConfig, load: Callable[[], AgentConfig] | None = None +) -> RunningLoop: + """``load`` reads the configuration again, for a new project's entry + (D8); by default from the file this one came from.""" oplog = OpLog("nisse-dev") worker = SerialWorker( on_error=lambda exc: oplog.emit( @@ -77,36 +120,10 @@ def build_loop_agent(config: AgentConfig) -> RunningLoop: gitea = GiteaClient(config.gitea_url, config.gitea_token) account = gitea.whoami() # D8: each project its own build VM, clone, push credential and milestones. - projects = [] - for entry in config.projects: - host = build_host(config, entry) - credential = push_credential(config, entry) - remote = remote_url(gitea, entry.repo, credential) - git = prepare_workspace(host, config, account, remote, credential, entry) - projects.append( - Project( - name=entry.name, - repo=entry.repo, - labels=entry.labels, - workspace=Workspace( - host, - entry.workspace, - timeout_s=config.tuning.tools.command_timeout_s, - max_output_chars=config.tuning.tools.max_output_chars, - # D7: the secrets its commands may see, redacted on return. - secrets=entry.command_secrets(), - ), - milestones=Milestones( - store, - git, - gitea, - repo=entry.repo, - remote=remote, - credential=credential, - base=entry.base_branch, - ), - ) - ) + projects = [ + project_runtime(config, entry, store, gitea, account) + for entry in config.projects + ] normalized, _passthrough = make_llm_clients(config, catalog) cancel = Cancel() # The driver is the planner's model for now (M2.6 brings the driver role). @@ -133,6 +150,11 @@ def build_loop_agent(config: AgentConfig) -> RunningLoop: # The driver's model reads plain-language go-aheads until the # classifier role arrives (M2.6). reader=GoAheadReader(normalized, config.roles.resolve("planner")), + reload=lambda known: [ + project_runtime(config, entry, store, gitea, account) + for entry in (load or _from_file(config))().projects + if entry.name not in known + ], bug_grace_s=config.tuning.loop.bug_grace_s, general_idle_s=config.tuning.loop.general_idle_s, ) @@ -146,3 +168,12 @@ def build_loop_agent(config: AgentConfig) -> RunningLoop: worker=worker, _stopped=threading.Event(), ) + + +def _from_file(config: AgentConfig) -> Callable[[], AgentConfig]: + def load() -> AgentConfig: + if config.config_file is None: + raise ConfigError("this agent wasn't started from a config file") + return AgentConfig.from_file(config.config_file) + + return load diff --git a/nisse-dev/src/nisse_dev/loop/items.py b/nisse-dev/src/nisse_dev/loop/items.py index 2d520ee..6de75a9 100644 --- a/nisse-dev/src/nisse_dev/loop/items.py +++ b/nisse-dev/src/nisse_dev/loop/items.py @@ -21,6 +21,7 @@ GO_AHEAD = "go-ahead" # a go on the restatement up (L3, D1) ANSWER = "answer" # an answer to a question MERGE = "merge" # the PR's review and merge (D4, D5) CHAT = "conversation" # a conversation waits for the operator to say more (L5) +PROVISION = "provisioning" # a new project waits for what it needs (D1, D8) @dataclass(frozen=True) @@ -52,6 +53,8 @@ class Loop: became: tuple[str, ...] = () proposal_project: str = "" proposal_title: str = "" + proposal_kind: str = "" # "work" on a project, or a new "project" + new_project: str = "" # a new project being provisioned (D1) # Where the operator last spoke (D5: answer where asked): "" for the # Matrix thread, or the PR it came from, with the inline location and a # little of what they said, for quoting. diff --git a/nisse-dev/src/nisse_dev/loop/milestones.py b/nisse-dev/src/nisse_dev/loop/milestones.py index 496cf0a..c88629e 100644 --- a/nisse-dev/src/nisse_dev/loop/milestones.py +++ b/nisse-dev/src/nisse_dev/loop/milestones.py @@ -7,7 +7,9 @@ Publishing is idempotent under an intent (L8): an open PR for the branch is adopted, and a PR the operator closed is an answer, never reopened. """ +import contextlib import time +import uuid from dataclasses import dataclass from typing import Literal @@ -159,6 +161,79 @@ class Milestones: if git.remote_sha(self._remote, parked, credential=self._credential): git.delete_remote_branch(self._remote, parked, credential=self._credential) + # -- a new project's checks (D1, D4) ----------------------------------------- + + def verify_access(self, account: str) -> list[str]: + """What the operator provided, checked rather than assumed: the + agent may push a branch with its push credential, may not push to + the default branch, and may not merge its own PR without approval. + A throwaway branch with an empty commit carries the checks; its PR + is closed and the branch deleted. Returns what failed.""" + git, base, failures = self._git, self._base, [] + try: + repo = self._gitea.repository(self._repo) + if not (repo.get("permissions") or {}).get("push"): + failures.append(f"{account} isn't a collaborator with write access") + branch = self._gitea.branch(self._repo, base) + if not branch.get("protected") or branch.get("user_can_push"): + failures.append(f"{base} isn't protected against my pushes") + except GiteaError as exc: + return [f"I can't see the repository {self._repo}: {exc}"] + check = f"nisse-check-{uuid.uuid4().hex[:6]}" + try: + git.fetch(credential=self._credential) + git.reset_to(check, f"origin/{base}") + git._git("commit", "-q", "--allow-empty", "-m", "nisse-dev: access check") + except GitError as exc: + return [*failures, f"the workspace isn't usable: {exc}"] + try: + git.push(self._remote, check, credential=self._credential) + except GitError as exc: + failures.append(f"my push credential can't push a branch: {exc}") + self._back_to_base(check) + return failures + try: + git.push(self._remote, base, credential=self._credential) + failures.append( + f"I could push to {base}, which protection should refuse; an " + f"empty commit, 'nisse-dev: access check', is on {base} now" + ) + except GitError: + pass # refused, as it should be + pr = None + try: + pr = self._gitea.open_pull_request( + self._repo, + head=check, + base=base, + title="nisse-dev access check (never merged)", + body="Checks that merging needs an approval; closed straight away.", + ) + try: + self._gitea.merge_pull_request(self._repo, pr.number) + failures.append( + f"I could merge my own PR #{pr.number} without an approval" + ) + except GiteaError: + pass # refused, as it should be + except GiteaError as exc: + failures.append(f"I couldn't open a PR to check merging: {exc}") + finally: + if pr is not None: + with contextlib.suppress(GiteaError): + self._gitea.close_pull_request(self._repo, pr.number) + with contextlib.suppress(GitError): + git.delete_remote_branch( + self._remote, check, credential=self._credential + ) + self._back_to_base(check) + return failures + + def _back_to_base(self, branch: str) -> None: + with contextlib.suppress(GitError): + self._git.reset_to(self._base, f"origin/{self._base}") + self._git._git("branch", "-q", "-D", branch) + def _ahead_of_origin(self) -> bool: current = self._git.current_branch() try: diff --git a/nisse-dev/src/nisse_dev/loop/tools.py b/nisse-dev/src/nisse_dev/loop/tools.py index f946f04..44a0136 100644 --- a/nisse-dev/src/nisse_dev/loop/tools.py +++ b/nisse-dev/src/nisse_dev/loop/tools.py @@ -85,6 +85,20 @@ PROPOSE_WORK = _tool( "whole brief.", ) +PROPOSE_PROJECT = _tool( + "propose_project", + "The conversation has become a new project: propose it. The operator " + "agrees to it as to a restatement; then you list what it needs set up " + "(a repository, a build VM, the project's entry), and once it is ready " + "and checked, the project's architecture is your first piece of work. " + "Ends your turn.", + name="The project's name: lowercase letters, digits and -.", + purpose="What it is for, in a paragraph.", + decisions="The decisions setting it up depends on: language and toolchain " + "(which pick the build VM), services, secrets, network access beyond the " + "default; and anything else agreed so far.", +) + WAITING = frozenset( { "restate", @@ -94,6 +108,7 @@ WAITING = frozenset( "wait_for_review", "reply", "propose_work", + "propose_project", } ) diff --git a/nisse-dev/tests/test_loop_milestones.py b/nisse-dev/tests/test_loop_milestones.py index ea03426..f49992a 100644 --- a/nisse-dev/tests/test_loop_milestones.py +++ b/nisse-dev/tests/test_loop_milestones.py @@ -14,7 +14,7 @@ from nisse_dev.cancel import Cancel from nisse_dev.canonical import ToolCall from nisse_dev.execution import SSHHost from nisse_dev.git import GitRepo -from nisse_dev.gitea import Comment, Issue, PullRequest, ReviewNote +from nisse_dev.gitea import Comment, GiteaError, Issue, PullRequest, ReviewNote from nisse_dev.loop.agent import LoopAgent from nisse_dev.loop.driver import Driver from nisse_dev.loop.items import ANSWER, MERGE, WAITING, loop_of @@ -124,6 +124,32 @@ class Forge: def close_pull_request(self, repo: str, index: int) -> None: self.prs[index]["state"] = "closed" + # what a new project's checks read (D4) + protected = False + + def protect(self) -> None: + """Protection as the server enforces it: pushes to main refused by a + hook; merging needs an approval nobody here gives.""" + hook = self.bare / "hooks" / "pre-receive" + hook.write_text( + "#!/bin/sh\nwhile read old new ref; do\n" + ' [ "$ref" = refs/heads/main ] && echo "main is protected" && exit 1\n' + "done\nexit 0\n" + ) + hook.chmod(0o755) + self.protected = True + + def repository(self, repo: str) -> dict[str, Any]: + return {"permissions": {"push": True}} + + def branch(self, repo: str, name: str) -> dict[str, Any]: + return {"protected": self.protected, "user_can_push": not self.protected} + + def merge_pull_request(self, repo: str, index: int) -> None: + if self.protected: + raise GiteaError("gitea POST merge returned 405: approvals needed") + self.merge(index) + # what the operator does in Gitea def merge(self, n: int) -> None: head = f"refs/heads/{self.prs[n]['head']}" diff --git a/nisse-dev/tests/test_loop_new_projects.py b/nisse-dev/tests/test_loop_new_projects.py new file mode 100644 index 0000000..fbbde04 --- /dev/null +++ b/nisse-dev/tests/test_loop_new_projects.py @@ -0,0 +1,193 @@ +"""New projects on the new loop (M2.5, PR B): a conversation proposes one, +the agent lists what it needs, and on "done" reads its configuration again +and checks what was provided (D1, D4, D8) before opening the tracking issue +and starting on the architecture. Real git on the throwaway build host, +with protection enforced by a hook on the fake forge's bare repository.""" + +import io +import re +from pathlib import Path +from typing import Any + +import pytest +import yaml +from test_loop_milestones import DON, Forge, ScriptedModel, call, make_world +from test_loop_projects import Router + +from nisse_dev.cancel import Cancel +from nisse_dev.config import _parse_projects +from nisse_dev.execution import SSHHost +from nisse_dev.loop.agent import LoopAgent, Project +from nisse_dev.loop.driver import Driver +from nisse_dev.loop.items import CHAT, GO_AHEAD, MERGE, PROVISION, loop_of +from nisse_dev.oplog import OpLog +from nisse_dev.tools import Workspace +from nisse_dev.transport import InMemoryTransport +from nisse_dev.worker import InlineWorker + +PROPOSAL = call( + "propose_project", + name="tally", + purpose="A small command-line tally counter.", + decisions="Python 3.12 with pytest; no services, secrets or extra egress.", +) + + +@pytest.fixture +def worlds(ssh_host: SSHHost, tmp_path: Path) -> dict[str, Any]: + sandbox = make_world(ssh_host, tmp_path / "sandbox") + tally = make_world( + ssh_host, tmp_path / "tally", store=sandbox["store"], repo_name="don/tally" + ) + sandbox["forge"].issues.clear() # only the conversation here + tally["forge"].issues.clear() + return {"sandbox": sandbox, "tally": tally} + + +def agent_for( + worlds: dict[str, Any], model: ScriptedModel, *, entry_added: list[bool] +) -> tuple[LoopAgent, InMemoryTransport]: + """An agent that knows only the sandbox; tally's entry appears in its + configuration once ``entry_added[0]`` is set.""" + transport = InMemoryTransport() + cancel = Cancel() + router = Router({w["repo_name"]: w["forge"] for w in worlds.values()}) + for w in worlds.values(): + w["milestones"]._gitea = router + + def runtime(name: str) -> Project: + w = worlds[name] + return Project( + name=name, + repo=w["repo_name"], + workspace=Workspace(w["host"], w["repo"].root, timeout_s=30), + milestones=w["milestones"], + ) + + def reload(known: set[str]) -> list[Project]: + names = ["sandbox", "tally"] if entry_added[0] else ["sandbox"] + return [runtime(n) for n in names if n not in known] + + agent = LoopAgent( + worlds["sandbox"]["store"], + transport, + router, # type: ignore[arg-type] + projects=[runtime("sandbox")], + account="nisse-dev", + operators={DON}, + gitea_operators={"don"}, + driver=Driver(model, "m", cancel=cancel), # type: ignore[arg-type] + worker=InlineWorker(), + cancel=cancel, + oplog=OpLog("nisse-dev", io.StringIO()), + reload=reload, + ) + transport.on_message(agent.on_message) + return agent, transport + + +def conversation(worlds: dict[str, Any]) -> Any: + [chat] = [ + i for i in worlds["sandbox"]["store"].work_items() if i.scope == "general" + ] + return chat + + +def proposed( + worlds: Any, model: ScriptedModel, added: list[bool] +) -> tuple[Any, Any, str]: + agent, transport = agent_for(worlds, model, entry_added=added) + root = transport.receive(DON, "I want a little tally counter, as its own project.") + assert transport.outbound[-1].text.startswith("rev-1: a new project, `tally`.") + assert loop_of(conversation(worlds)).wait == GO_AHEAD + transport.receive(DON, "go", thread=root) + return agent, transport, root + + +def test_an_agreed_project_gets_a_checklist_with_its_entry(worlds: Any) -> None: + _, transport, _ = proposed(worlds, ScriptedModel([PROPOSAL]), [False]) + loop = loop_of(conversation(worlds)) + assert (loop.wait, loop.new_project) == (PROVISION, "tally") + checklist = transport.outbound[-1].text + for need in ("repository", "Protection", "collaborator", "build VM", "deploy key"): + assert need in checklist + # The entry is valid configuration once its placeholders are filled in. + entry = re.search(r"```yaml\n(.*?)\n```", checklist, re.DOTALL) + assert entry is not None + filled = ( + entry.group(1) + .replace("/tally", "don/tally") + .replace("", "build.test") + .replace("", "/k") + .replace("", "/p") + ) + errors: list[str] = [] + [project] = _parse_projects(yaml.safe_load(filled)["projects"], {}, errors) + assert errors == [] and project.name == "tally" and project.repo == "don/tally" + + +def test_done_before_the_entry_is_added_says_so(worlds: Any) -> None: + _, transport, root = proposed(worlds, ScriptedModel([PROPOSAL]), [False]) + transport.receive(DON, "done", thread=root) + assert "I don't see `tally` in my configuration yet" in transport.outbound[-1].text + assert loop_of(conversation(worlds)).new_project == "tally" + + +def test_what_isnt_set_up_is_named_and_nothing_starts(worlds: Any) -> None: + added = [False] + _, transport, root = proposed(worlds, ScriptedModel([PROPOSAL]), added) + added[0] = True # the entry is there, but tally's main isn't protected + transport.receive(DON, "done", thread=root) + said = transport.outbound[-1].text + assert said.startswith("Not ready yet:") + assert "main isn't protected" in said and "could merge my own PR" in said + assert loop_of(conversation(worlds)).new_project == "tally" + assert worlds["tally"]["forge"].issues == [] + # The check's PR was closed and its branch deleted. + assert all(p["state"] == "closed" for p in worlds["tally"]["forge"].prs.values()) + + +def test_a_checked_project_gets_its_record_and_starts_on_its_design( + worlds: Any, +) -> None: + added = [False] + model = ScriptedModel( + [PROPOSAL], + [ # tally's first item: the architecture and the build plan + call("start_milestone", name="architecture"), + call("write_file", path="ARCHITECTURE.md", content="# tally\n"), + call("write_file", path="docs/plans/tally.md", content="# plan\n"), + call("finish_milestone", summary="architecture and plan", final="no"), + ], + ) + agent, transport, root = proposed(worlds, model, added) + forge: Forge = worlds["tally"]["forge"] + forge.protect() + added[0] = True + transport.receive(DON, "done", thread=root) + + assert "tally" in agent._projects + [record] = forge.issues + marker = re.search(r"", record.body) + assert marker and "tally counter" in record.body + work = worlds["sandbox"]["store"].work_item(marker.group(1)) + assert work.scope == "tally" and loop_of(work).started + assert "design" in loop_of(work).labels + assert loop_of(work).wait == MERGE + pr = max(n for n, p in forge.prs.items() if p["state"] == "open") + assert sorted(forge.files(pr)) == ["ARCHITECTURE.md", "docs/plans/tally.md"] + assert f"Part of #{record.number}" in forge.prs[pr]["body"] + chat = loop_of(conversation(worlds)) + assert chat.became == (f"don/tally#{record.number}",) + assert (chat.wait, chat.new_project) == (CHAT, "") + + +def test_a_project_name_that_exists_is_sent_back(worlds: Any) -> None: + model = ScriptedModel( + [call("propose_project", name="sandbox", purpose="again", decisions="x.")], + [call("reply", text="sandbox exists; shall I propose work on it?")], + ) + agent, transport = agent_for(worlds, model, entry_added=[False]) + transport.receive(DON, "a new sandbox?") + assert loop_of(conversation(worlds)).wait == CHAT + assert agent diff --git a/nisse-dev/tests/test_loop_properties.py b/nisse-dev/tests/test_loop_properties.py index 5cfe7be..d246269 100644 --- a/nisse-dev/tests/test_loop_properties.py +++ b/nisse-dev/tests/test_loop_properties.py @@ -11,6 +11,7 @@ the seed and the steps, so it replays exactly. import random import re import subprocess +import uuid from pathlib import Path from typing import Any @@ -32,6 +33,7 @@ from nisse_dev.loop.items import ( CHAT, GO_AHEAD, MERGE, + PROVISION, RUNNING, WAITING, loop_of, @@ -67,6 +69,15 @@ class PolicyModel: if roll < 0.12: return [ToolCall(call_id=f"p{self.n}", tool="nonsense", arguments={})] if "reply" in offered: # a conversation: talk, or propose work + if self.rng.random() < 0.1: + return [ + call( + "propose_project", + name=f"proj{self.n}", + purpose=f"a project {self.n}.", + decisions="Python.", + ) + ] if self.rng.random() < 0.4: return [ call( @@ -125,6 +136,10 @@ class Walk: def _agent(self) -> tuple[LoopAgent, InMemoryTransport]: transport = InMemoryTransport() + # Event IDs are unique across restarts, as a server's are: a new + # transport goes on numbering where the old one stopped. + old = getattr(self, "transport", None) + transport._next_id = old._next_id if old is not None else 0 cancel = Cancel() router = Router({w["repo_name"]: w["forge"] for w in self.worlds.values()}) for w in self.worlds.values(): @@ -193,6 +208,8 @@ class Walk: if item and loop: weights.update({"go": 3, "answer": 2, "abandon": 1, "go-stale": 1}) weights.update({"plain-yes": 2, "wait": 2}) + if loop.new_project: + weights.update({"done": 2}) if item.scope != "general": # conversations aren't named by number weights.update({"first": 1, "named-go": 1}) if loop.pr: @@ -203,6 +220,9 @@ class Walk: self.steps.append(f"{action}{self.name(item)}" if item else action) if action == "intake": self.agent.intake() + elif action == "done" and loop: + # The operator says a new project is ready; no entry was added. + self.transport.receive(DON, "done", thread=loop.thread) elif action == "chat": # A top-level message: a conversation of its own (L1, L5). self.transport.receive(DON, f"thinking aloud {len(self.steps)}") @@ -295,6 +315,7 @@ class Walk: if item.status == "open" and item.position == WAITING: explained = ( (loop.wait == CHAT and item.scope == "general") + or (loop.wait == PROVISION and bool(loop.new_project)) or (loop.wait == GO_AHEAD and loop.rev > 0) or (loop.wait == ANSWER and bool(loop.note)) or (loop.wait == MERGE and loop.pr > 0) @@ -546,8 +567,18 @@ def _is_ancestor(bare: Path, commit: str, ref: str) -> bool: ) +def _seeded_ids(monkeypatch: pytest.MonkeyPatch, seed: int) -> None: + """Item IDs and branch names come from uuid4: seeded too, so a walk + replays exactly from its seed.""" + ids = random.Random(seed) + monkeypatch.setattr(uuid, "uuid4", lambda: uuid.UUID(int=ids.getrandbits(128))) + + @pytest.mark.parametrize("seed", SEEDS) -def test_the_loop_keeps_its_properties(world: dict[str, Any], seed: int) -> None: +def test_the_loop_keeps_its_properties( + world: dict[str, Any], seed: int, monkeypatch: pytest.MonkeyPatch +) -> None: + _seeded_ids(monkeypatch, seed) walk = Walk({"sandbox": world}, seed) for _ in range(STEPS): walk.step() @@ -556,9 +587,10 @@ def test_the_loop_keeps_its_properties(world: dict[str, Any], seed: int) -> None @pytest.mark.parametrize("seed", TWO_PROJECT_SEEDS) def test_two_projects_keep_the_properties( - ssh_host: SSHHost, tmp_path: Path, seed: int + ssh_host: SSHHost, tmp_path: Path, seed: int, monkeypatch: pytest.MonkeyPatch ) -> None: """D8: the same walk over two projects, whose issue numbers collide.""" + _seeded_ids(monkeypatch, seed) sandbox = make_world(ssh_host, tmp_path / "sandbox") other = make_world( ssh_host, tmp_path / "other", store=sandbox["store"], repo_name="don/other" diff --git a/tests/integration/e2e_support.py b/tests/integration/e2e_support.py index bd687da..3a51403 100644 --- a/tests/integration/e2e_support.py +++ b/tests/integration/e2e_support.py @@ -61,7 +61,7 @@ class FreshRepo: @contextlib.contextmanager def fresh_repo( - operator: GiteaClient, run: str, keys: Path, *, agent: str + operator: GiteaClient, run: str, keys: Path, *, agent: str, seed: bool = True ) -> Iterator[FreshRepo]: """A repository of its own for one test run, as the operator would make a project's (D4, D8): seeded with a small Python package, the ``agent`` @@ -88,14 +88,21 @@ def fresh_repo( "content": base64.b64encode(path.read_bytes()).decode(), } for path in sorted(SEED.rglob("*")) - if path.is_file() and path.name != "README.md" + if seed and path.is_file() and path.name != "README.md" ] - operator._request( - "POST", - f"/repos/{full}/contents", - json={"branch": "main", "message": "Seed the test project", "files": files}, - ) + if files: + operator._request( + "POST", + f"/repos/{full}/contents", + json={ + "branch": "main", + "message": "Seed the test project", + "files": files, + }, + ) readme = operator._request("GET", f"/repos/{full}/contents/README.md") + # A new project's README says only what it is; a seeded one, the seed's. + text = (SEED / "README.md").read_bytes() if seed else f"# {name}\n".encode() operator._request( "PUT", f"/repos/{full}/contents/README.md", @@ -103,7 +110,7 @@ def fresh_repo( "branch": "main", "message": "README", "sha": readme["sha"], - "content": base64.b64encode((SEED / "README.md").read_bytes()).decode(), + "content": base64.b64encode(text).decode(), }, ) operator._request( diff --git a/tests/integration/nisse-test.yaml b/tests/integration/nisse-test.yaml index 0601bd0..c450a92 100644 --- a/tests/integration/nisse-test.yaml +++ b/tests/integration/nisse-test.yaml @@ -27,6 +27,10 @@ projects: llm: base_url: https://litellm.dongar.net token: ${NISSE_LLM_TOKEN} + # The test key caps each request's worst-case cost ($0.10): input plus + # this much reserved output. The test agent's replies are short, so half + # the default leaves room for ~40k input tokens on Haiku. + max_output_tokens: 8192 # The live tests exercise the gated loop and the routes over a real proxy, # not local-model competence (measured separately by test_model_capability), diff --git a/tests/integration/test_end_to_end_loop.py b/tests/integration/test_end_to_end_loop.py index 1de86f5..492d3b0 100644 --- a/tests/integration/test_end_to_end_loop.py +++ b/tests/integration/test_end_to_end_loop.py @@ -12,6 +12,7 @@ scripts/e2e.sh after each milestone. import base64 import contextlib import os +import re import time import uuid from collections.abc import Iterator @@ -83,7 +84,11 @@ def running( monkeypatch.setattr("nisse_dev.loop.build.MatrixTransport", lambda cfg: transport) agent = None try: - agent = build_loop_agent(config) + # What the agent reads when told a new project is ready: replaced by + # a test that adds one (D8). + source = [config] + agent = build_loop_agent(config, load=lambda: source[0]) + agent.source = source # type: ignore[attr-defined] # The test drives intake and PR watching itself, not on a timer. agent.loop._interval = 3600 agent.start() @@ -145,11 +150,13 @@ class Operator: """Intake and PR watching, as the agent's timer would do.""" self.agent.worker.call(self.agent.loop.intake, timeout=DEADLINE_S) + project: str | None = None # whose issues ``issue`` numbers, if not the only + def item(self, issue: int = 0) -> Any: """The work item for ``issue``; by default the first one.""" if issue: - store = self.agent.store - return self.agent.worker.call(lambda: item_for_issue(store, issue)) + store, project = self.agent.store, self.project + return self.agent.worker.call(lambda: item_for_issue(store, issue, project)) items = self.agent.worker.call(self.agent.store.work_items) return items[0] if items else None @@ -545,3 +552,85 @@ def test_a_conversation_becomes_filed_work( assert f"Closes #{number}" in pr["body"] finally: operator.tidy(issues, prs) + + +def test_a_conversation_becomes_a_new_project( + running: tuple[RunningLoop, InMemoryTransport, str], +) -> None: + """M2.5: a conversation proposes a new project; the operator provides + what the checklist asks for (here, a repository of its own made the way + fresh_repo makes one, and the entry the agent wrote); "done" has the + agent check it all, open the tracking issue, and start the architecture.""" + import yaml + + from nisse_dev.config import _parse_projects + + agent, transport, run = running + operator = Operator(agent, transport) + as_operator = operator_gitea(agent.config.gitea_url) + name = f"tally-{run}" + + def conversation() -> Any: + items = agent.worker.call(agent.store.work_items) + return next(i for i in items if i.scope == "general") + + root = transport.receive( + OPERATOR, + f"I'd like a new project, named {name}: a tiny command-line tally " + "counter in Python 3 with pytest, no services, secrets or extra network " + "access. Please propose it as a new project.", + ) + operator.settle() + for _ in range(MAX_ANSWERS): + if loop_of(conversation()).wait == GO_AHEAD: + break + transport.receive( + OPERATOR, f"Yes: propose it as a new project named {name}.", thread=root + ) + operator.settle() + assert loop_of(conversation()).proposal_kind == "project", operator.last() + transport.receive(OPERATOR, "go", thread=root) + operator.settle() + checklist = operator.last() + entry = re.search(r"```yaml\n(.*?)\n```", checklist, re.DOTALL) + assert entry is not None, checklist + + # --- the operator provides it all, as the checklist asks --- + keep = agent.config.state_db.parent + with fresh_repo( + as_operator, f"{run}-new", keep / "keys", agent=agent.account, seed=False + ) as repo: + host = agent.config.build_host + filled = ( + entry.group(1) + .replace(f"/{name}", repo.name) + .replace("", host.address) + .replace("", str(host.key)) + .replace("", str(repo.push_key)) + ) + errors: list[str] = [] + [project] = _parse_projects(yaml.safe_load(filled)["projects"], {}, errors) + assert errors == [], errors + # On this run's build host and in its scratch directory. + project = replace( + project, build_host=host, workspace=Path(f"/tmp/nisse-e2e-{run}/{name}") + ) + agent.source[0] = replace( # type: ignore[attr-defined] + agent.config, projects=(*agent.config.projects, project) + ) + transport.receive(OPERATOR, "done", thread=root) + operator.settle() + + became = loop_of(conversation()).became + assert became, f"not set up: {operator.last()}" + number = int(became[0].split("#")[1]) + operator.project = name + item = operator.drive_until(MERGE, number) + assert "design" in loop_of(item).labels + pr = loop_of(item).pr + files = [ + f["filename"] + for f in agent.gitea._request("GET", f"/repos/{repo.name}/pulls/{pr}/files") + ] + assert "ARCHITECTURE.md" in files, files + assert any(f.startswith("docs/plans/") for f in files), files