New projects from conversation (M2.5, PR B)

- propose_project(name, purpose, decisions) in a conversation goes up for
  a go-ahead (checked: a slug, not an existing project, no trailing
  question).
- On go, the provisioning checklist (D1): a repository with a README,
  protection on its default branch, the agent as a collaborator, a build
  VM, a deploy key, and the project entry as YAML with <...> to fill in.
- On "done", the agent reads its configuration again and adds the new
  project live (build_loop_agent(load=...); only additions), then checks
  what was provided (D4, verified, not assumed): permissions and
  protection as it sees them, then on a throwaway branch with an empty
  commit, a branch push must succeed and a push to the default branch and
  merging its own PR must be refused. The check's PR is closed and its
  branch deleted. Failures are said per item; nothing starts.
- Checked: the tracking issue (the distilled decisions, carrying the new
  item's ID), and the project's item, agreed and labelled `design`, whose
  first milestone is ARCHITECTURE.md and the build plan. A conversation
  waiting on provisioning doesn't close as idle.
- The walk replays exactly from its seed (item IDs too), and event IDs
  stay unique across simulated restarts, as a server's are.
- The test config holds output to 8192 tokens, under the test key's
  per-request cost cap.

Tests: new-project unit tests (checklist and its YAML parse as config, done
before the entry, what isn't set up, a checked project's record and first
PR, an existing name), protection enforced by a hook on the fake forge,
the walk proposing projects and saying done, and an e2e new project in a
repository the test operator provisions from the agent's checklist.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01PPQ14L3LikyefCMFBqEbKE

nisse-dev/src/nisse_dev/loop/agent.py

@@ -36,6 +36,7 @@ from nisse_dev.loop.items import (
     CLOSED,
     GO_AHEAD,
     MERGE,
+    PROVISION,
     READY,
     RUNNING,
     WAITING,
@@ -51,6 +52,7 @@ from nisse_dev.loop.tools import (
     ASK_HUMAN,
     CLOSE_ITEM,
     FINISH_MILESTONE,
+    PROPOSE_PROJECT,
     PROPOSE_WORK,
     REPLY,
     RESTATE,
@@ -133,9 +135,10 @@ is the new issue's body and the work's whole brief.
 
 {projects}
 
-Setting up a new project from here isn't available yet: if that's where the
-conversation goes, say so, and that the operator can create the repository
-and its project entry."""
+If it becomes a new project, use propose_project: its name, what it's for,
+and the decisions setting it up depends on. Once agreed, you'll be asked for
+what it needs set up; when the operator has it ready and it checks out, its
+architecture and build plan are the first piece of work."""
 
 # What an issue's labels add to the driver's instructions (D1).
 LABELS = {
@@ -204,6 +207,9 @@ class LoopAgent:
         reader: GoAheadReader | None = None,
         bug_grace_s: float = BUG_GRACE_S,
         general_idle_s: float = GENERAL_IDLE_S,
+        # A new project's entry, once the operator has added it (D1, D8):
+        # the projects configured now that this agent doesn't know yet.
+        reload: Callable[[set[str]], Sequence[Project]] | None = None,
         clock: Callable[[], float] = time.time,
         intake_interval_s: float = INTAKE_INTERVAL_S,
     ) -> None:
@@ -224,6 +230,7 @@ class LoopAgent:
         self._reader = reader
         self._grace = bug_grace_s
         self._idle = general_idle_s
+        self._reload = reload
         self._clock = clock
         self._interval = intake_interval_s
         # thread root -> revision on screen there, read on the transport's
@@ -566,6 +573,7 @@ class LoopAgent:
             if (
                 item.scope == GENERAL
                 and item.position == WAITING
+                and not loop.new_project  # setting one up can take days
                 and loop.idle_since
                 and now - loop.idle_since >= self._idle
             ):
@@ -761,6 +769,8 @@ class LoopAgent:
             item = self._store.work_item(item.id) or item
         if command == "abandon":
             self._close(item, loop, "abandoned", f"abandoned by {msg.sender}")
+        elif command == "done" and loop.new_project:
+            self._provisioned(item, loop)
         elif command == "stop":
             # Stopping a run took effect on arrival; a stop is never an answer.
             if not loop.note.startswith("stopped ("):
@@ -850,6 +860,9 @@ class LoopAgent:
 
     def _go(self, item: WorkItem, loop: Loop, by: str) -> None:
         self._fact(item, f"go-ahead on rev-{loop.rev} from {by}")
+        if item.scope == GENERAL and loop.proposal_kind == "project":
+            self._checklist(item, loop)
+            return
         if item.scope == GENERAL:
             self._file_work(item, loop)
             return
@@ -949,6 +962,116 @@ class LoopAgent:
             "We can keep talking here.",
         )
 
+    # -- new projects (D1, D8) --------------------------------------------------
+
+    def _checklist(self, item: WorkItem, loop: Loop) -> None:
+        """An agreed new project: what it needs, for the operator to provide
+        (later nisse-ops, O6), with its project entry to add."""
+        name = loop.proposal_project
+        entry = (
+            "projects:\n"
+            f"  {name}:\n"
+            f"    repo: <owner>/{name}\n"
+            "    base_branch: main\n"
+            f"    workspace: /root/work/{name}\n"
+            "    build_host:\n"
+            "      address: <the build VM>\n"
+            "      key: <my SSH key for it>\n"
+            "    push_key: <the deploy key's private half>"
+        )
+        self._fact(item, f"provisioning checklist posted for {name}")
+        self._wait(
+            item,
+            replace(
+                loop,
+                wait=PROVISION,
+                note=f"{name}'s setup: say done when it's ready",
+                new_project=name,
+                idle_since=self._clock(),
+            ),
+            f"To set up `{name}`, I need:\n"
+            f"1. A repository for it, e.g. <owner>/{name}, with a README so its "
+            "default branch exists.\n"
+            "2. Protection on the default branch: none of my pushes, and an "
+            "approval to merge.\n"
+            f"3. Me ({self._account}) as a collaborator with write access.\n"
+            "4. A build VM, with the toolchain from the decisions above.\n"
+            "5. A deploy key with write access, its private half where I run.\n"
+            "6. This entry in my configuration, with the <...> parts filled "
+            f"in:\n\n```yaml\n{entry}\n```\n\n"
+            "Say done when it's ready, and I'll check each of them.",
+        )
+
+    def _provisioned(self, item: WorkItem, loop: Loop) -> None:
+        """ "done": the entry is read again and the new project checked
+        (D4: verified, not assumed). What fails is said, per item; when all
+        is well the tracking issue is opened, and the project's item starts
+        with its architecture."""
+        name = loop.new_project
+        try:
+            added = list(self._reload(set(self._projects))) if self._reload else []
+        except Exception as exc:  # noqa: BLE001 -- said; the operator fixes it
+            self._transport.post(
+                f"I couldn't read the new entry: {exc}. Fix it and say done again.",
+                thread=loop.thread,
+            )
+            return
+        project = next((p for p in added if p.name == name), None)
+        if project is None:
+            self._transport.post(
+                f"I don't see `{name}` in my configuration yet (item 6). Add its "
+                "entry and say done again.",
+                thread=loop.thread,
+            )
+            return
+        failures = (
+            project.milestones.verify_access(self._account)
+            if project.milestones is not None
+            else ["the project has no workspace"]
+        )
+        if failures:
+            self._fact(item, f"{name} checked: " + "; ".join(failures))
+            self._transport.post(
+                "Not ready yet: " + "; ".join(failures) + ". Say done again when "
+                "they're fixed.",
+                thread=loop.thread,
+            )
+            return
+        self._projects[name] = project
+        self._fact(item, f"{name} set up and checked: {project.repo}")
+        brief = self._latest_proposal(item)
+        new_id = f"{name}-c{uuid.uuid4().hex[:6]}"
+        title = f"Implement the build plan for {name}"
+        try:
+            number = self._gitea.create_issue(
+                project.repo,
+                title,
+                f"{brief}\n\nFrom a conversation with the operator, agreed there "
+                f"as rev-{loop.rev}. First: ARCHITECTURE.md and the build plan."
+                f"\n\n<!-- nisse-item: {new_id} -->",
+            )
+            self._gitea.assign_issue(project.repo, number, self._account)
+        except GiteaError as exc:
+            self._transport.post(
+                f"{name} checks out, but I couldn't open its tracking issue: "
+                f"{exc}. Say done to try again.",
+                thread=loop.thread,
+            )
+            return
+        # D1: the architecture is agreed in its PR, as `design` steers.
+        design = replace(
+            project, labels=tuple(dict.fromkeys(("design", *project.labels)))
+        )
+        self._start_filed(item, design, new_id, number, title, brief)
+        loop = loop_of(self._store.work_item(item.id) or item)
+        save(
+            self._store,
+            item.id,
+            WAITING,
+            replace(loop, new_project="", proposal_kind=""),
+            cause=f"{name} provisioned",
+        )
+
     def _latest_proposal(self, item: WorkItem) -> str:
         proposals = [
             e.event.plan
@@ -1102,15 +1225,30 @@ class LoopAgent:
             outcome = self._driver.run(
                 self._store.transcript(item.id, "driver"),
                 system=system,
-                tools=[REPLY, ASK_HUMAN, PROPOSE_WORK],
+                tools=[REPLY, ASK_HUMAN, PROPOSE_WORK, PROPOSE_PROJECT],
                 workspace=None,
-                checks={"propose_work": self._proposal_problem},
+                checks={
+                    "propose_work": self._proposal_problem,
+                    "propose_project": self._project_problem,
+                },
             )
         finally:
             self._running = False
             self._cancel.clear()
         self._settle(item.id, outcome)
 
+    def _project_problem(self, args: dict[str, object]) -> str | None:
+        name = str(args.get("name", "")).strip()
+        if not re.fullmatch(r"[a-z0-9][a-z0-9-]*", name) or name == GENERAL:
+            return "a project's name is lowercase letters, digits and -, not general"
+        if name in self._projects:
+            return f"{name} is a project already: propose_work for work on it"
+        if not str(args.get("purpose", "")).strip():
+            return "say what the project is for"
+        return _restatement_problem(
+            {"text": f"{args.get('purpose', '')}\n{args.get('decisions', '')}"}
+        )
+
     def _proposal_problem(self, args: dict[str, object]) -> str | None:
         project = str(args.get("project", "")).strip()
         if project not in self._projects:
@@ -1233,6 +1371,7 @@ class LoopAgent:
                 None,
             )
         elif outcome.kind == "wait" and outcome.tool == "propose_work":
+            loop = replace(loop, proposal_kind="work")
             project = self._projects[str(args.get("project", "")).strip()]
             title = str(args.get("title", "")).strip()
             summary = str(args.get("summary", "")).strip()
@@ -1261,6 +1400,36 @@ class LoopAgent:
                 ),
                 None,
             )
+        elif outcome.kind == "wait" and outcome.tool == "propose_project":
+            name = str(args.get("name", "")).strip()
+            purpose = str(args.get("purpose", "")).strip()
+            decisions = str(args.get("decisions", "")).strip()
+            brief = f"{purpose}\n\nDecisions:\n{decisions}" if decisions else purpose
+            rev = loop.rev + 1
+            self._store.transcript(item.id, "driver").append(
+                PlanSubmitted(plan=brief, model=self._roles.get("planner", ""))
+            )
+            self._fact(item, f"new project proposed as rev-{rev}: {name}")
+            self._shown[loop.thread] = rev
+            self._transport.post(
+                f"rev-{rev}: a new project, `{name}`. {brief}\n\nSay go and I'll "
+                "list what it needs set up, or reply with changes.",
+                thread=loop.thread,
+            )
+            self._wait(
+                item,
+                replace(
+                    loop,
+                    rev=rev,
+                    wait=GO_AHEAD,
+                    note="",
+                    proposal_project=name,
+                    proposal_title="",
+                    proposal_kind="project",
+                    idle_since=self._clock(),
+                ),
+                None,
+            )
         elif outcome.kind == "wait" and outcome.tool == "ask_human":
             question = str(args.get("question", "")).strip()
             self._answer_where_asked(item, loop, question)

nisse-dev/src/nisse_dev/loop/build.py

@@ -2,6 +2,7 @@
 LoopAgent in place of the Orchestrator. Chosen with `loop: new` in config."""
 
 import threading
+from collections.abc import Callable
 from dataclasses import dataclass
 
 from nisse_dev.agent import (
@@ -14,7 +15,8 @@ from nisse_dev.agent import (
     remote_url,
 )
 from nisse_dev.cancel import Cancel
-from nisse_dev.config import AgentConfig
+from nisse_dev.config import AgentConfig, ConfigError
+from nisse_dev.config import Project as ProjectEntry
 from nisse_dev.gitea import GiteaClient
 from nisse_dev.loop.agent import LoopAgent, Project
 from nisse_dev.loop.driver import Driver
@@ -64,7 +66,48 @@ class RunningLoop:
         self._stopped.wait()
 
 
-def build_loop_agent(config: AgentConfig) -> RunningLoop:
+def project_runtime(
+    config: AgentConfig,
+    entry: ProjectEntry,
+    store: StateStore,
+    gitea: GiteaClient,
+    account: str,
+) -> Project:
+    """One project's part of the loop: its clone on its build VM, workspace
+    (with the secrets its commands may see, redacted on return), push
+    credential and milestones."""
+    host = build_host(config, entry)
+    credential = push_credential(config, entry)
+    remote = remote_url(gitea, entry.repo, credential)
+    git = prepare_workspace(host, config, account, remote, credential, entry)
+    return Project(
+        name=entry.name,
+        repo=entry.repo,
+        labels=entry.labels,
+        workspace=Workspace(
+            host,
+            entry.workspace,
+            timeout_s=config.tuning.tools.command_timeout_s,
+            max_output_chars=config.tuning.tools.max_output_chars,
+            secrets=entry.command_secrets(),
+        ),
+        milestones=Milestones(
+            store,
+            git,
+            gitea,
+            repo=entry.repo,
+            remote=remote,
+            credential=credential,
+            base=entry.base_branch,
+        ),
+    )
+
+
+def build_loop_agent(
+    config: AgentConfig, load: Callable[[], AgentConfig] | None = None
+) -> RunningLoop:
+    """``load`` reads the configuration again, for a new project's entry
+    (D8); by default from the file this one came from."""
     oplog = OpLog("nisse-dev")
     worker = SerialWorker(
         on_error=lambda exc: oplog.emit(
@@ -77,36 +120,10 @@ def build_loop_agent(config: AgentConfig) -> RunningLoop:
     gitea = GiteaClient(config.gitea_url, config.gitea_token)
     account = gitea.whoami()
     # D8: each project its own build VM, clone, push credential and milestones.
-    projects = []
-    for entry in config.projects:
-        host = build_host(config, entry)
-        credential = push_credential(config, entry)
-        remote = remote_url(gitea, entry.repo, credential)
-        git = prepare_workspace(host, config, account, remote, credential, entry)
-        projects.append(
-            Project(
-                name=entry.name,
-                repo=entry.repo,
-                labels=entry.labels,
-                workspace=Workspace(
-                    host,
-                    entry.workspace,
-                    timeout_s=config.tuning.tools.command_timeout_s,
-                    max_output_chars=config.tuning.tools.max_output_chars,
-                    # D7: the secrets its commands may see, redacted on return.
-                    secrets=entry.command_secrets(),
-                ),
-                milestones=Milestones(
-                    store,
-                    git,
-                    gitea,
-                    repo=entry.repo,
-                    remote=remote,
-                    credential=credential,
-                    base=entry.base_branch,
-                ),
-            )
-        )
+    projects = [
+        project_runtime(config, entry, store, gitea, account)
+        for entry in config.projects
+    ]
     normalized, _passthrough = make_llm_clients(config, catalog)
     cancel = Cancel()
     # The driver is the planner's model for now (M2.6 brings the driver role).
@@ -133,6 +150,11 @@ def build_loop_agent(config: AgentConfig) -> RunningLoop:
         # The driver's model reads plain-language go-aheads until the
         # classifier role arrives (M2.6).
         reader=GoAheadReader(normalized, config.roles.resolve("planner")),
+        reload=lambda known: [
+            project_runtime(config, entry, store, gitea, account)
+            for entry in (load or _from_file(config))().projects
+            if entry.name not in known
+        ],
         bug_grace_s=config.tuning.loop.bug_grace_s,
         general_idle_s=config.tuning.loop.general_idle_s,
     )
@@ -146,3 +168,12 @@ def build_loop_agent(config: AgentConfig) -> RunningLoop:
         worker=worker,
         _stopped=threading.Event(),
     )
+
+
+def _from_file(config: AgentConfig) -> Callable[[], AgentConfig]:
+    def load() -> AgentConfig:
+        if config.config_file is None:
+            raise ConfigError("this agent wasn't started from a config file")
+        return AgentConfig.from_file(config.config_file)
+
+    return load

nisse-dev/src/nisse_dev/loop/items.py

@@ -21,6 +21,7 @@ GO_AHEAD = "go-ahead"  # a go on the restatement up (L3, D1)
 ANSWER = "answer"  # an answer to a question
 MERGE = "merge"  # the PR's review and merge (D4, D5)
 CHAT = "conversation"  # a conversation waits for the operator to say more (L5)
+PROVISION = "provisioning"  # a new project waits for what it needs (D1, D8)
 
 
 @dataclass(frozen=True)
@@ -52,6 +53,8 @@ class Loop:
     became: tuple[str, ...] = ()
     proposal_project: str = ""
     proposal_title: str = ""
+    proposal_kind: str = ""  # "work" on a project, or a new "project"
+    new_project: str = ""  # a new project being provisioned (D1)
     # Where the operator last spoke (D5: answer where asked): "" for the
     # Matrix thread, or the PR it came from, with the inline location and a
     # little of what they said, for quoting.

nisse-dev/src/nisse_dev/loop/milestones.py

@@ -7,7 +7,9 @@ Publishing is idempotent under an intent (L8): an open PR for the branch is
 adopted, and a PR the operator closed is an answer, never reopened.
 """
 
+import contextlib
 import time
+import uuid
 from dataclasses import dataclass
 from typing import Literal
 
@@ -159,6 +161,79 @@ class Milestones:
         if git.remote_sha(self._remote, parked, credential=self._credential):
             git.delete_remote_branch(self._remote, parked, credential=self._credential)
 
+    # -- a new project's checks (D1, D4) -----------------------------------------
+
+    def verify_access(self, account: str) -> list[str]:
+        """What the operator provided, checked rather than assumed: the
+        agent may push a branch with its push credential, may not push to
+        the default branch, and may not merge its own PR without approval.
+        A throwaway branch with an empty commit carries the checks; its PR
+        is closed and the branch deleted. Returns what failed."""
+        git, base, failures = self._git, self._base, []
+        try:
+            repo = self._gitea.repository(self._repo)
+            if not (repo.get("permissions") or {}).get("push"):
+                failures.append(f"{account} isn't a collaborator with write access")
+            branch = self._gitea.branch(self._repo, base)
+            if not branch.get("protected") or branch.get("user_can_push"):
+                failures.append(f"{base} isn't protected against my pushes")
+        except GiteaError as exc:
+            return [f"I can't see the repository {self._repo}: {exc}"]
+        check = f"nisse-check-{uuid.uuid4().hex[:6]}"
+        try:
+            git.fetch(credential=self._credential)
+            git.reset_to(check, f"origin/{base}")
+            git._git("commit", "-q", "--allow-empty", "-m", "nisse-dev: access check")
+        except GitError as exc:
+            return [*failures, f"the workspace isn't usable: {exc}"]
+        try:
+            git.push(self._remote, check, credential=self._credential)
+        except GitError as exc:
+            failures.append(f"my push credential can't push a branch: {exc}")
+            self._back_to_base(check)
+            return failures
+        try:
+            git.push(self._remote, base, credential=self._credential)
+            failures.append(
+                f"I could push to {base}, which protection should refuse; an "
+                f"empty commit, 'nisse-dev: access check', is on {base} now"
+            )
+        except GitError:
+            pass  # refused, as it should be
+        pr = None
+        try:
+            pr = self._gitea.open_pull_request(
+                self._repo,
+                head=check,
+                base=base,
+                title="nisse-dev access check (never merged)",
+                body="Checks that merging needs an approval; closed straight away.",
+            )
+            try:
+                self._gitea.merge_pull_request(self._repo, pr.number)
+                failures.append(
+                    f"I could merge my own PR #{pr.number} without an approval"
+                )
+            except GiteaError:
+                pass  # refused, as it should be
+        except GiteaError as exc:
+            failures.append(f"I couldn't open a PR to check merging: {exc}")
+        finally:
+            if pr is not None:
+                with contextlib.suppress(GiteaError):
+                    self._gitea.close_pull_request(self._repo, pr.number)
+            with contextlib.suppress(GitError):
+                git.delete_remote_branch(
+                    self._remote, check, credential=self._credential
+                )
+            self._back_to_base(check)
+        return failures
+
+    def _back_to_base(self, branch: str) -> None:
+        with contextlib.suppress(GitError):
+            self._git.reset_to(self._base, f"origin/{self._base}")
+            self._git._git("branch", "-q", "-D", branch)
+
     def _ahead_of_origin(self) -> bool:
         current = self._git.current_branch()
         try:

nisse-dev/src/nisse_dev/loop/tools.py

@@ -85,6 +85,20 @@ PROPOSE_WORK = _tool(
     "whole brief.",
 )
 
+PROPOSE_PROJECT = _tool(
+    "propose_project",
+    "The conversation has become a new project: propose it. The operator "
+    "agrees to it as to a restatement; then you list what it needs set up "
+    "(a repository, a build VM, the project's entry), and once it is ready "
+    "and checked, the project's architecture is your first piece of work. "
+    "Ends your turn.",
+    name="The project's name: lowercase letters, digits and -.",
+    purpose="What it is for, in a paragraph.",
+    decisions="The decisions setting it up depends on: language and toolchain "
+    "(which pick the build VM), services, secrets, network access beyond the "
+    "default; and anything else agreed so far.",
+)
+
 WAITING = frozenset(
     {
         "restate",
@@ -94,6 +108,7 @@ WAITING = frozenset(
         "wait_for_review",
         "reply",
         "propose_work",
+        "propose_project",
     }
 )