Projects as configuration entries (M1.4, D8)
CI / checks (push) Skipped
CI / checks (pull_request) Failing after 5m7s

- Config: `projects: {name: entry}` with repo, base_branch, workspace,
  build_host, push credential, labels (defaults added to every issue),
  secrets by ${VAR} reference with visibility (commands | harness), and
  egress (recorded, not enforced). Every problem in every entry is
  reported at once; an inline secret, a bad visibility, a repeated repo or
  a non-slug name is an error.
- The old loop runs exactly one project (only_project) and refuses more.
- The new loop is per project: each its own clone, workspace (with its
  commands-visible secrets, redacted on return), push credential and
  milestones; intake polls every project's repository; items are scoped
  to their project for every run, publish and PR watch. Issues are named
  project#14 when there is more than one project, and a bare #14 that
  matches two is asked about. Switching projects parks the one worked on
  last (all others after a boot).
- A restatement that ends in a question is sent back to the driver, not
  posted: the e2e driver kept asking "Is this the right interpretation?",
  which a plain yes can't answer.

Tests: config entries and errors, two-project assembly, two-project loop
tests (intake, own workspaces and repos, #14 in both, do-next, label
defaults, parking across projects), the property walk over two projects
with a new own_repository_only property (an item using the first
project's repo fails 15 of 16 seeds), and the restatement check.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01PPQ14L3LikyefCMFBqEbKE
This commit is contained in:
dongarandClaude Opus 5.5 committed 2026-10-06 01:12:57 +00:00
1 parent 4ecff19933
commit 064d9bf554
17 files changed
+1139 -315

No files matched your search

+64 -7
View File
@@ -9,7 +9,7 @@ from typing import Any
import httpx
import pytest
from test_config import SECRETS, tree
from test_config import SECRETS, P, tree
from nisse_dev.agent import build_agent, discover_models, prepare_workspace
from nisse_dev.catalog import ModelCatalog, UnknownModel
@@ -51,11 +51,13 @@ def config_for(tmp_path: Path, **changes: str) -> AgentConfig:
return AgentConfig.from_tree(
tree(
state_db=str(tmp_path / "state.db"),
workspace=str(tmp_path / "work"),
build_host__address=HOST.address,
build_host__user=HOST.user,
build_host__port=str(HOST.port),
build_host__key=str(HOST._key),
**{
f"{P}workspace": str(tmp_path / "work"),
f"{P}build_host__address": HOST.address,
f"{P}build_host__user": HOST.user,
f"{P}build_host__port": str(HOST.port),
f"{P}build_host__key": str(HOST._key),
},
**changes,
),
SECRETS,
@@ -397,8 +399,63 @@ def test_the_new_loop_assembles(tmp_path: Path, origin: Path, monkeypatch) -> No
running = build_loop_agent(config)
try:
assert running.account == "nisse-dev-testing"
assert running.loop._repo == config.gitea_repo
[project] = running.loop._projects.values()
assert project.repo == config.gitea_repo
assert (config.workspace / "README.md").exists() # cloned on the build host
finally:
running.worker.call(running.store.close)
running.worker.stop(timeout=5)
def test_each_project_gets_its_own_clone_and_secrets(
tmp_path: Path, origin: Path, monkeypatch
) -> None:
"""D8: a second project is only a second entry."""
from nisse_dev.loop.build import build_loop_agent
second = tmp_path / "second"
config = config_for(
tmp_path,
loop="new",
projects__other={
"repo": "don/other",
"workspace": str(tmp_path / "other-work"),
"build_host": {
"address": HOST.address,
"user": HOST.user,
"port": str(HOST.port),
"key": str(HOST._key),
},
"labels": ["discuss"],
"secrets": {
"API_KEY": {"value": "${TEST_LLM_TOKEN}", "visible": "commands"}
},
},
)
git("clone", "-q", "--bare", str(origin), str(second))
origins = {"don/demo": str(origin), "don/other": str(second)}
monkeypatch.setattr(
"nisse_dev.loop.build.GiteaClient.whoami", lambda self: "nisse-dev-testing"
)
monkeypatch.setattr(
"nisse_dev.loop.build.GiteaClient.clone_url", lambda self, repo: origins[repo]
)
monkeypatch.setattr(
"nisse_dev.agent.ModelCatalog.discover",
classmethod(lambda cls, *a, **kw: TEST_CATALOG),
)
monkeypatch.setattr(
"nisse_dev.loop.build.MatrixTransport", lambda cfg: FakeTransport()
)
running = build_loop_agent(config)
try:
projects = running.loop._projects
assert set(projects) == {"greet", "other"}
assert (tmp_path / "work" / "README.md").exists()
assert (tmp_path / "other-work" / "README.md").exists()
assert projects["other"].labels == ("discuss",)
assert projects["other"].workspace._secrets == {"API_KEY": "sk-x"}
assert projects["greet"].workspace._secrets == {}
finally:
running.worker.call(running.store.close)
running.worker.stop(timeout=5)
+130 -28
View File
@@ -10,8 +10,13 @@ from nisse_dev.llm import DEFAULT_REQUEST_TIMEOUT_S
# A complete config as the YAML file would give it, with secrets by reference.
COMPLETE: dict[str, Any] = {
"state_db": "~/demo/state.db",
"workspace": "~/demo/work",
"project": "greet",
"projects": {
"greet": {
"repo": "don/demo",
"workspace": "~/demo/work",
"build_host": {"address": "build.test", "key": "~/.ssh/nisse-dev"},
}
},
"llm": {"base_url": "https://litellm.test", "token": "${TEST_LLM_TOKEN}"},
"roles": {
"executor": "claude-haiku-4-5",
@@ -29,11 +34,10 @@ COMPLETE: dict[str, Any] = {
"gitea": {
"url": "https://gitea.test",
"token": "${TEST_GITEA_TOKEN}",
"repo": "don/demo",
"operators": ["don"],
},
"build_host": {"address": "build.test", "key": "~/.ssh/nisse-dev"},
}
P = "projects__greet__" # the one project's keys, for tree()
SECRETS = {
"TEST_LLM_TOKEN": "sk-x",
"TEST_MATRIX_TOKEN": "syt_x",
@@ -83,15 +87,15 @@ def test_every_missing_setting_is_reported_at_once() -> None:
# Failing at first use would surface a missing token twenty minutes into
# a milestone, on the call that mattered.
with pytest.raises(ConfigError) as excinfo:
build(matrix__token=None, gitea__repo=None)
build(matrix__token=None, **{f"{P}repo": None})
message = str(excinfo.value)
assert "matrix.token" in message
assert "gitea.repo" in message
assert "projects.greet: unset or empty — repo" in message
def test_empty_is_treated_as_missing() -> None:
with pytest.raises(ConfigError, match="gitea.repo"):
build(gitea__repo=" ")
with pytest.raises(ConfigError, match="projects.greet: unset or empty — repo"):
build(**{f"{P}repo": " "})
def test_a_reference_to_an_unset_variable_is_an_error() -> None:
@@ -115,7 +119,7 @@ def test_an_operator_list_of_only_blanks_is_missing() -> None:
def test_base_branch_is_configurable() -> None:
assert build(gitea__base_branch="trunk").base_branch == "trunk"
assert build(**{f"{P}base_branch": "trunk"}).base_branch == "trunk"
def test_max_output_tokens_is_configurable_and_checked() -> None:
@@ -148,7 +152,7 @@ def test_redacted_summary_carries_no_secrets() -> None:
flat = " ".join(summary.values())
for secret in SECRETS.values():
assert secret not in flat
assert summary["project"] == "greet"
assert "greet (don/demo, build.test:" in summary["projects"]
def test_tuning_is_read() -> None:
@@ -178,7 +182,7 @@ def test_a_partly_given_tuning_section_keeps_the_other_defaults() -> None:
def test_every_problem_is_reported_at_once() -> None:
# A misspelt knob that was silently ignored would look exactly like a knob
# with no effect, so unknown keys are errors.
bad = tree(project=None, llm_extra=1)
bad = tree(llm_extra=1, **{f"{P}workspace": None})
bad["tuning"] = {
"executor": {"max_turn": 40, "failure_limit": 0},
"expert": {"consult_budget": True},
@@ -188,7 +192,7 @@ def test_every_problem_is_reported_at_once() -> None:
AgentConfig.from_tree(bad, SECRETS)
message = str(excinfo.value)
for expected in (
"project",
"projects.greet: unset or empty — workspace",
"llm_extra: unknown setting",
"tuning.executor.max_turn: unknown setting",
"tuning.executor.failure_limit: must be a positive number",
@@ -207,8 +211,13 @@ def test_a_whole_number_is_required_where_one_is_meant() -> None:
FILE = """
state_db: ~/demo/state.db
workspace: ~/demo/work
project: greet
projects:
greet:
repo: don/demo
workspace: ~/demo/work
build_host:
address: build.test
key: ~/.ssh/nisse-dev
llm: {base_url: "https://litellm.test", token: "${TEST_LLM_TOKEN}"}
roles:
executor: local-fast
@@ -224,11 +233,7 @@ matrix:
gitea:
url: https://gitea.test
token: ${TEST_GITEA_TOKEN}
repo: don/demo
operators: [don]
build_host:
address: build.test
key: ~/.ssh/nisse-dev
"""
@@ -297,7 +302,7 @@ def test_state_db_inside_the_clone_is_refused(tmp_path: Path) -> None:
# own state alongside the project.
work = tmp_path / "project"
with pytest.raises(ConfigError, match="inside the workspace"):
build(workspace=str(work), state_db=str(work / "state.db"))
build(state_db=str(work / "state.db"), **{f"{P}workspace": str(work)})
def test_env_file_inside_the_clone_is_refused(tmp_path: Path) -> None:
@@ -306,7 +311,7 @@ def test_env_file_inside_the_clone_is_refused(tmp_path: Path) -> None:
work = tmp_path / "project"
with pytest.raises(ConfigError, match="NISSE_ENV"):
AgentConfig.from_tree(
tree(workspace=str(work), state_db=str(tmp_path / "state.db")),
tree(state_db=str(tmp_path / "state.db"), **{f"{P}workspace": str(work)}),
{**SECRETS, "NISSE_ENV": str(work / ".env")},
)
@@ -315,13 +320,17 @@ def test_a_config_file_inside_the_clone_is_refused(tmp_path: Path) -> None:
work = tmp_path / "project"
work.mkdir()
text = FILE.replace("workspace: ~/demo/work", f"workspace: {work}")
assert text != FILE
with pytest.raises(ConfigError, match="NISSE_CONFIG"):
AgentConfig.from_file(write(work, text), SECRETS)
def test_sibling_paths_are_fine(tmp_path: Path) -> None:
config = AgentConfig.from_tree(
tree(workspace=str(tmp_path / "project"), state_db=str(tmp_path / "state.db")),
tree(
state_db=str(tmp_path / "state.db"),
**{f"{P}workspace": str(tmp_path / "project")},
),
{**SECRETS, "NISSE_ENV": str(tmp_path / ".env")},
)
assert config.workspace.name == "project"
@@ -342,30 +351,36 @@ def test_the_build_host_is_required_and_defaults_to_root_on_22() -> None:
assert config.build_host.port == 22
assert config.build_host.key.is_absolute()
with pytest.raises(ConfigError, match="build_host.address"):
build(build_host__address=None)
build(**{f"{P}build_host__address": None})
def test_a_bad_build_host_port_is_reported() -> None:
with pytest.raises(ConfigError, match="build_host.port"):
build(build_host__port="ssh")
build(**{f"{P}build_host__port": "ssh"})
def test_the_push_key_is_optional_and_a_path() -> None:
assert build().push_key is None
assert build(gitea__push_key="~/k").push_key == Path("~/k").expanduser()
assert build(**{f"{P}push_key": "~/k"}).push_key == Path("~/k").expanduser()
def test_a_push_token_needs_its_user_and_excludes_a_key() -> None:
config = build(gitea__push_user="pusher", gitea__push_token="${TEST_GITEA_TOKEN}")
token = {f"{P}push_user": "pusher", f"{P}push_token": "${TEST_GITEA_TOKEN}"}
config = build(**token)
assert (config.push_user, config.push_token) == ("pusher", "gt")
with pytest.raises(ConfigError, match="set both or neither"):
build(gitea__push_user="pusher")
build(**{f"{P}push_user": "pusher"})
with pytest.raises(ConfigError, match="not both"):
build(gitea__push_user="p", gitea__push_token="t", gitea__push_key="~/k")
build(**token, **{f"{P}push_key": "~/k"})
def test_no_credential_appears_in_the_config_repr() -> None:
config = build(gitea__push_user="pusher", gitea__push_token="${TEST_GITEA_TOKEN}")
config = build(
**{f"{P}push_user": "pusher", f"{P}push_token": "${TEST_GITEA_TOKEN}"},
**{
f"{P}secrets": {"DB": {"value": "${TEST_LLM_TOKEN}", "visible": "commands"}}
},
)
text = repr(config)
for secret in SECRETS.values():
assert secret not in text
@@ -376,3 +391,90 @@ def test_the_loop_defaults_to_old_and_takes_new() -> None:
assert build(loop="new").loop == "new"
with pytest.raises(ConfigError, match="loop: must be old or new"):
build(loop="newer")
# --- projects (D8) -----------------------------------------------------------
SECOND = {
"repo": "don/other",
"workspace": "~/demo/other",
"build_host": {"address": "build2.test", "key": "~/.ssh/nisse-dev"},
}
def test_a_second_project_is_a_second_entry() -> None:
config = build(projects__other=SECOND)
assert [p.name for p in config.projects] == ["greet", "other"]
other = config.projects[1]
assert (other.repo, other.build_host.address) == ("don/other", "build2.test")
assert other.base_branch == "main" and other.labels == () and other.secrets == {}
def test_the_old_loop_refuses_more_than_one_project() -> None:
config = build(projects__other=SECOND)
with pytest.raises(ConfigError, match="exactly one project"):
config.only_project()
with pytest.raises(ConfigError, match="exactly one project"):
_ = config.gitea_repo
def test_at_least_one_project_is_needed() -> None:
with pytest.raises(ConfigError, match="projects: must name at least one"):
build(projects={})
def test_project_problems_are_named_by_project() -> None:
bad = tree(projects__other={**SECOND, "repo": "don/demo", "colour": "red"})
with pytest.raises(ConfigError) as excinfo:
AgentConfig.from_tree(bad, SECRETS)
message = str(excinfo.value)
assert "projects.other.colour: unknown setting" in message
assert "projects.other.repo: don/demo is also project greet's" in message
def test_a_project_name_is_a_slug() -> None:
with pytest.raises(ConfigError, match="lowercase letters, digits"):
build(projects__Other_Project=SECOND)
def test_labels_and_egress_are_lists_of_names() -> None:
config = build(**{f"{P}labels": ["discuss"], f"{P}egress": ["pypi.org"]})
assert config.only_project().labels == ("discuss",)
assert config.only_project().egress == ("pypi.org",)
with pytest.raises(ConfigError, match="projects.greet.labels: must be a list"):
build(**{f"{P}labels": "discuss"})
def test_secrets_are_by_reference_with_a_visibility() -> None:
config = build(
**{
f"{P}secrets": {
"DB_URL": {"value": "${TEST_LLM_TOKEN}", "visible": "commands"},
"DEPLOY": {"value": "${TEST_GITEA_TOKEN}", "visible": "harness"},
}
}
)
project = config.only_project()
assert project.command_secrets() == {"DB_URL": "sk-x"}
assert project.secrets["DEPLOY"].visible == "harness"
def test_a_secret_written_inline_is_refused() -> None:
with pytest.raises(ConfigError, match="must be a \\$\\{VAR\\} reference"):
build(**{f"{P}secrets": {"DB": {"value": "hunter2", "visible": "commands"}}})
def test_a_secret_needs_a_known_visibility() -> None:
with pytest.raises(ConfigError, match="visible: must be one of commands, harness"):
build(
**{f"{P}secrets": {"DB": {"value": "${TEST_LLM_TOKEN}", "visible": "all"}}}
)
def test_a_workspace_of_any_project_inside_is_refused(tmp_path: Path) -> None:
work = tmp_path / "other"
with pytest.raises(ConfigError, match="project other"):
build(
state_db=str(work / "state.db"),
projects__other={**SECOND, "workspace": str(work)},
)
+29
View File
@@ -280,3 +280,32 @@ def _msg(text: str, thread: str) -> Any:
from nisse_dev.transport import InboundMessage
return InboundMessage(DON, text, thread=thread)
def test_a_restatement_that_asks_is_sent_back(tmp_path: Path) -> None:
"""A restatement is agreed to as it stands; one ending in a question
(models add "Is this right?") never goes up."""
h = Harness(
tmp_path,
ScriptedModel(
[call("restate", text="I'll add count_words. Is this right?")],
[call("restate", text="I'll add count_words to text.py, with tests.")],
),
FakeGitea(issue()),
)
h.agent.intake()
loop = loop_of(h.item())
assert (loop.wait, loop.rev) == (GO_AHEAD, 1)
posted = [p for p in h.posted() if p.startswith("rev-")]
assert posted == [
(
"rev-1: I'll add count_words to text.py, with tests.\n\n"
"Say go to start, or reply with changes."
)
]
results = [
e.event.result
for e in h.store.transcript(h.item().id, "driver").read()
if hasattr(e.event, "result")
]
assert any("a restatement asks no questions" in r for r in results)
+19 -3
View File
@@ -157,6 +157,18 @@ class Forge:
@pytest.fixture
def world(ssh_host: SSHHost, tmp_path: Path) -> dict[str, Any]:
return make_world(ssh_host, tmp_path)
def make_world(
ssh_host: SSHHost,
tmp_path: Path,
store: StateStore | None = None,
repo_name: str = "don/sandbox",
) -> dict[str, Any]:
"""A project's forge (a bare repository and a fake Gitea over it), its
clone and milestones; a second one can share the first's store."""
tmp_path.mkdir(parents=True, exist_ok=True)
bare = tmp_path / "forge.git"
git("init", "-q", "--bare", "-b", "main", str(bare))
seed = tmp_path / "seed"
@@ -179,13 +191,16 @@ def world(ssh_host: SSHHost, tmp_path: Path) -> dict[str, Any]:
repo.clone(str(bare))
repo.configure_identity("nisse-dev", "n@t")
repo.install_excludes()
store = StateStore(tmp_path / "state.db", oplog=OpLog("nisse-dev", io.StringIO()))
if store is None:
store = StateStore(
tmp_path / "state.db", oplog=OpLog("nisse-dev", io.StringIO())
)
forge = Forge(bare)
milestones = Milestones(
store,
repo,
forge,
repo="don/sandbox", # type: ignore[arg-type]
forge, # type: ignore[arg-type]
repo=repo_name,
remote=str(bare),
credential=None,
)
@@ -197,6 +212,7 @@ def world(ssh_host: SSHHost, tmp_path: Path) -> dict[str, Any]:
"host": ssh_host,
"bare": bare,
"seed": seed,
"repo_name": repo_name,
}
+188
View File
@@ -0,0 +1,188 @@
"""Several projects on the new loop (M1.4, D8): intake from each project's
repository, each item working in its own project's workspace and publishing
to its own repository, issue numbers told apart, label defaults, and parking
across projects. Real git on the throwaway build host; fake Gitea per repo."""
import io
from pathlib import Path
from typing import Any
import pytest
from test_loop_milestones import DON, Forge, ScriptedModel, call, git, make_world
from nisse_dev.cancel import Cancel
from nisse_dev.execution import SSHHost
from nisse_dev.loop.agent import LoopAgent, Project
from nisse_dev.loop.driver import Driver
from nisse_dev.loop.items import ANSWER, GO_AHEAD, MERGE, item_for_issue, loop_of
from nisse_dev.oplog import OpLog
from nisse_dev.tools import Workspace
from nisse_dev.transport import InMemoryTransport
from nisse_dev.worker import InlineWorker
REPOS = {"alpha": "don/alpha", "beta": "don/beta"}
class Router:
"""One Gitea for the agent, routing each call to its repository's fake."""
def __init__(self, forges: dict[str, Forge]) -> None:
self.forges = forges
def __getattr__(self, name: str) -> Any:
def call(repo: str, *args: Any, **kwargs: Any) -> Any:
return getattr(self.forges[repo], name)(repo, *args, **kwargs)
return call
@pytest.fixture
def worlds(ssh_host: SSHHost, tmp_path: Path) -> dict[str, dict[str, Any]]:
alpha = make_world(ssh_host, tmp_path / "alpha", repo_name=REPOS["alpha"])
beta = make_world(
ssh_host, tmp_path / "beta", store=alpha["store"], repo_name=REPOS["beta"]
)
return {"alpha": alpha, "beta": beta}
def agent_for(
worlds: dict[str, dict[str, Any]],
model: ScriptedModel,
labels: dict[str, tuple[str, ...]] | None = None,
) -> tuple[LoopAgent, InMemoryTransport]:
transport = InMemoryTransport()
cancel = Cancel()
router = Router({REPOS[n]: w["forge"] for n, w in worlds.items()})
projects = [
Project(
name=n,
repo=REPOS[n],
workspace=Workspace(w["host"], w["repo"].root, timeout_s=30),
milestones=w["milestones"],
labels=(labels or {}).get(n, ()),
)
for n, w in worlds.items()
]
for w in worlds.values():
w["milestones"]._gitea = router # publishing goes through the router too
agent = LoopAgent(
worlds["alpha"]["store"],
transport,
router, # type: ignore[arg-type]
projects=projects,
account="nisse-dev",
operators={DON},
gitea_operators={"don"},
driver=Driver(model, "m", cancel=cancel), # type: ignore[arg-type]
worker=InlineWorker(),
cancel=cancel,
oplog=OpLog("nisse-dev", io.StringIO()),
)
transport.on_message(agent.on_message)
return agent, transport
def item(worlds: dict[str, Any], project: str) -> Any:
found = item_for_issue(worlds["alpha"]["store"], 14, project)
assert found is not None
return found
def say(transport: InMemoryTransport, worlds: Any, project: str, text: str) -> None:
transport.receive(DON, text, thread=loop_of(item(worlds, project)).thread)
def restate(name: str) -> list[Any]:
return [call("restate", text=f"{name}: add count_words")]
def work(path: str) -> list[Any]:
return [
call("start_milestone", name=path),
call("write_file", path=path, content=f"# {path}\n"),
call("finish_milestone", summary=path, final="yes"),
]
def test_both_projects_issues_are_taken_and_told_apart(worlds: Any) -> None:
# Both forges have an issue #14.
agent, transport = agent_for(worlds, ScriptedModel(restate("a"), restate("b")))
agent.intake()
a, b = item(worlds, "alpha"), item(worlds, "beta")
assert (a.scope, b.scope) == ("alpha", "beta")
said = [m.text for m in transport.outbound]
assert any(t.startswith("Queued don/alpha#14") for t in said)
assert any(t.startswith("Queued don/beta#14") for t in said)
status = agent.status()
assert "alpha#14 Add count_words" in status and "beta#14 Add count_words" in status
def test_each_item_works_and_publishes_in_its_own_project(worlds: Any) -> None:
model = ScriptedModel(restate("a"), restate("b"), work("a.py"), work("b.py"))
agent, transport = agent_for(worlds, model)
agent.intake()
say(transport, worlds, "alpha", "go")
say(transport, worlds, "beta", "go")
fa, fb = worlds["alpha"]["forge"], worlds["beta"]["forge"]
[pa], [pb] = fa.prs, fb.prs
assert fa.files(pa) == ["a.py"] and fb.files(pb) == ["b.py"]
assert fa.prs[pa]["head"].startswith(item(worlds, "alpha").id)
assert fb.prs[pb]["head"].startswith(item(worlds, "beta").id)
# Each workspace has only its own project's work.
assert not (worlds["alpha"]["repo"].root / "b.py").exists()
assert loop_of(item(worlds, "beta")).wait == MERGE
def test_a_bare_number_in_two_projects_is_asked_about(worlds: Any) -> None:
model = ScriptedModel(restate("a"), restate("b"), work("b.py"))
agent, transport = agent_for(worlds, model)
agent.intake()
transport.receive(DON, "go #14")
asked = transport.outbound[-1].text
assert "open in more than one project" in asked
assert "go alpha#14" in asked and "go beta#14" in asked
assert not loop_of(item(worlds, "alpha")).started
transport.receive(DON, "go beta#14")
assert loop_of(item(worlds, "beta")).started
assert not loop_of(item(worlds, "alpha")).started
def test_do_next_names_a_project(worlds: Any) -> None:
agent, transport = agent_for(worlds, ScriptedModel(restate("a"), restate("b")))
agent.intake()
transport.receive(DON, "do #14 next")
assert "do alpha#14 next or do beta#14 next" in transport.outbound[-1].text
transport.receive(DON, "do beta#14 next")
assert "beta#14 goes first" in transport.outbound[-1].text
def test_a_projects_label_defaults_steer_its_items(worlds: Any) -> None:
model = ScriptedModel(
[call("ask_human", question="what's it for?")], # alpha: discuss first
restate("b"),
)
agent, _ = agent_for(worlds, model, labels={"alpha": ("discuss",)})
agent.intake()
assert loop_of(item(worlds, "alpha")).labels == ("discuss",)
assert loop_of(item(worlds, "alpha")).wait == ANSWER
assert loop_of(item(worlds, "beta")).wait == GO_AHEAD
def test_switching_projects_parks_the_other_projects_work(worlds: Any) -> None:
model = ScriptedModel(
restate("a"),
restate("b"),
[ # alpha: half done, then a question
call("start_milestone", name="a"),
call("write_file", path="half.py", content="H = 1\n"),
call("ask_human", question="one more thing?"),
],
work("b.py"), # beta runs meanwhile
)
agent, transport = agent_for(worlds, model)
agent.intake()
say(transport, worlds, "alpha", "go")
say(transport, worlds, "beta", "go")
bare = ["--git-dir", str(worlds["alpha"]["bare"])]
parked = f"refs/heads/{item(worlds, 'alpha').id}-parked"
assert git(*bare, "show", f"{parked}^3:half.py") == "H = 1"
+92 -32
View File
@@ -15,13 +15,15 @@ from typing import Any
import pytest
from model_contract import assert_renderable
from test_loop_milestones import DON, Forge, call
from test_loop_milestones import DON, Forge, call, make_world
from test_loop_milestones import world as world # noqa: PLC0414 -- the fixture
from test_loop_projects import Router
from nisse_dev.cancel import Cancel
from nisse_dev.canonical import Fact, PlanSubmitted, ToolCall, UserMessage
from nisse_dev.execution import SSHHost
from nisse_dev.gitea import Comment, Issue
from nisse_dev.loop.agent import LoopAgent
from nisse_dev.loop.agent import LoopAgent, Project
from nisse_dev.loop.driver import Driver
from nisse_dev.loop.items import (
ANSWER,
@@ -38,6 +40,7 @@ from nisse_dev.transport import InboundMessage, InMemoryTransport
from nisse_dev.worker import InlineWorker
SEEDS = range(32)
TWO_PROJECT_SEEDS = range(100, 116)
STEPS = 30
@@ -88,13 +91,17 @@ class PolicyModel:
class Walk:
def __init__(self, world: dict[str, Any], seed: int) -> None:
self.world = world
"""``worlds``: project name -> its world (forge, bare repo, clone,
milestones), all over one store."""
def __init__(self, worlds: dict[str, dict[str, Any]], seed: int) -> None:
self.worlds = worlds
self.world = next(iter(worlds.values())) # the store's home
self.rng = random.Random(seed)
self.seed = seed
self.steps: list[str] = []
self.shown_at_go: list[str] = [] # item:revision on screen at each go
self.next_issue = 15
self.next_issue = dict.fromkeys(worlds, 15) # numbers repeat across projects
self.now = 1000.0
self.agent, self.transport = self._agent()
@@ -106,22 +113,29 @@ class Walk:
def _agent(self) -> tuple[LoopAgent, InMemoryTransport]:
transport = InMemoryTransport()
cancel = Cancel()
w = self.world
router = Router({w["repo_name"]: w["forge"] for w in self.worlds.values()})
for w in self.worlds.values():
w["milestones"]._gitea = router
agent = LoopAgent(
w["store"],
self.world["store"],
transport,
w["forge"],
repo="don/sandbox",
project="sandbox",
router, # type: ignore[arg-type]
projects=[
Project(
name=name,
repo=w["repo_name"],
workspace=Workspace(w["host"], w["repo"].root, timeout_s=30),
milestones=w["milestones"],
)
for name, w in self.worlds.items()
],
account="nisse-dev",
operators={DON},
gitea_operators={"don"},
driver=Driver(PolicyModel(self.rng), "m", cancel=cancel), # type: ignore[arg-type]
workspace=Workspace(w["host"], w["repo"].root, timeout_s=30),
worker=InlineWorker(),
cancel=cancel,
oplog=OpLog("nisse-dev", _Null()),
milestones=w["milestones"],
reader=self, # type: ignore[arg-type]
bug_grace_s=600,
clock=lambda: self.now,
@@ -135,14 +149,21 @@ class Walk:
def items(self) -> list[Any]:
return self.world["store"].work_items()
def name(self, item: Any) -> str:
"""How the operator names an item: by project when there are two."""
issue = loop_of(item).issue
return f"#{issue}" if len(self.worlds) == 1 else f"{item.scope}#{issue}"
def step(self) -> None:
forge: Forge = self.world["forge"]
open_items = [i for i in self.items() if i.status == "open"]
# Weighted towards what an item can actually do, so review paths
# are walked often, not drowned out by intake and crashes.
weights = {"intake": 2 if not open_items else 1, "crash": 1, "issue": 1}
item = self.rng.choice(open_items) if open_items else None
loop = loop_of(item) if item else None
# The forge an item's project lives on; a new issue picks one.
project = item.scope if item else self.rng.choice(list(self.worlds))
forge: Forge = self.worlds[project]["forge"]
if item and loop:
weights.update({"go": 3, "answer": 2, "abandon": 1, "go-stale": 1})
weights.update({"first": 1, "plain-yes": 2, "named-go": 1, "wait": 2})
@@ -151,12 +172,15 @@ class Walk:
{"comment": 4, "merge": 2, "close-pr": 1, "comment-merge": 2}
)
action = self.rng.choices(list(weights), list(weights.values()))[0]
self.steps.append(f"{action}#{loop.issue}" if loop else action)
self.steps.append(f"{action}{self.name(item)}" if item else action)
if action == "intake":
self.agent.intake()
elif action == "issue":
# A new issue, sometimes depending on one that is still open.
n, self.next_issue = self.next_issue, self.next_issue + 1
project = self.rng.choice(list(self.worlds))
forge = self.worlds[project]["forge"]
n = self.next_issue[project]
self.next_issue[project] += 1
labels = self.rng.choice([(), (), ("bug",), ("discuss",), ("design",)])
forge.issues.append(Issue(n, f"task {n}", f"do task {n}", "don", labels))
live = [i.number for i in forge.assigned_issues("", "") if i.number != n]
@@ -171,7 +195,7 @@ class Walk:
elif action == "named-go" and loop:
if item.position == WAITING and loop.wait == GO_AHEAD:
self.shown_at_go.append(f"{item.id}:{loop.rev}")
self.transport.receive(DON, f"go #{loop.issue}")
self.transport.receive(DON, f"go {self.name(item)}")
elif action == "wait":
# Time passes: bug countdowns may run out.
self.now += self.rng.choice([60, 700])
@@ -185,7 +209,7 @@ class Walk:
self.shown_at_go.append(f"{other.id}:{o.rev}")
self.agent.intake()
elif action == "first" and loop:
self.agent.handle(InboundMessage(DON, f"do #{loop.issue} next"), None)
self.agent.handle(InboundMessage(DON, f"do {self.name(item)} next"), None)
elif action == "crash":
# Volatile state is lost: a new agent over the same store.
self.agent, self.transport = self._agent()
@@ -231,12 +255,14 @@ class Walk:
def check(self) -> None:
where = f"seed {self.seed}, steps {self.steps}"
store, forge = self.world["store"], self.world["forge"]
store = self.world["store"]
items = self.items()
issues = [loop_of(i).issue for i in items if loop_of(i).issue]
issues = [(i.scope, loop_of(i).issue) for i in items if loop_of(i).issue]
assert len(issues) == len(set(issues)), f"one_item_per_issue: {where}"
for item in items:
loop = loop_of(item)
forge = self.worlds[item.scope]["forge"]
repo = self.worlds[item.scope]["repo_name"]
if item.status == "open" and item.position == WAITING:
explained = (
(loop.wait == GO_AHEAD and loop.rev > 0)
@@ -257,7 +283,7 @@ class Walk:
)
# Blocked comes from external truth (L1): an item whose issue
# had an open dependency from the start never ran before it closed.
if forge.open_dependencies("don/sandbox", loop.issue):
if forge.open_dependencies(repo, loop.issue):
assert item.position == BLOCKED or item.status == "closed", (
f"blocked_never_runs: #{loop.issue} {item.position}: {where}"
)
@@ -311,11 +337,30 @@ class Walk:
assert not [i for i in items if i.position == RUNNING], (
f"one_active (nothing left running): {where}"
)
self._parked_work_safe(where)
self._milestones_start_fresh(where)
for name, w in self.worlds.items():
self._own_repository_only(name, w, where)
self._parked_work_safe(w, name, where)
self._milestones_start_fresh(w, where)
self._review_fixes_land(w, where)
def _own_repository_only(self, name: str, world: Any, where: str) -> None:
"""D8: an item's branches and PRs reach only its own project's
repository."""
others = [i.id for i in self.items() if i.scope != name]
heads = [p["head"] for p in world["forge"].prs.values()]
branches = _git(
world["bare"], "for-each-ref", "--format=%(refname:short)", "refs/heads"
).split()
for ref in [*heads, *branches]:
assert not any(ref.startswith(f"{i}-") for i in others), (
f"own_repository_only: {ref} in {name}: {where}"
)
def _review_fixes_land(self, world: Any, where: str) -> None:
forge: Forge = world["forge"]
heads = [p["head"] for p in forge.prs.values()]
assert len(heads) == len(set(heads)), f"one_pr_per_milestone: {where}"
bare = self.world["bare"]
bare = world["bare"]
for n, pr in forge.prs.items():
if not pr["merged"]:
continue
@@ -330,11 +375,11 @@ class Walk:
)
assert landed, f"review_fixes_land (PR {n}): {where}"
def _milestones_start_fresh(self, where: str) -> None:
def _milestones_start_fresh(self, world: Any, where: str) -> None:
"""D4: a milestone starts from the latest default branch, so one
redone after a closed PR never carries the rejected work."""
forge: Forge = self.world["forge"]
bare = self.world["bare"]
forge: Forge = world["forge"]
bare = world["bare"]
for item in self.items():
mine = sorted(
(n, p)
@@ -353,13 +398,13 @@ class Walk:
bare, rejected, f"refs/heads/{later['head']}"
), f"milestones_start_fresh: PR {m} carries PR {n}: {where}"
def _parked_work_safe(self, where: str) -> None:
def _parked_work_safe(self, world: Any, name: str, where: str) -> None:
"""L1, D4: nothing of a parked item lives only on the build VM: what
every branch of an open item that isn't on the workspace points at is
in the forge."""
root, bare = self.world["repo"].root, self.world["bare"]
in its project's forge."""
root, bare = world["repo"].root, world["bare"]
current = _git(root, "rev-parse", "--abbrev-ref", "HEAD")
ids = [i.id for i in self.items() if i.status == "open"]
ids = [i.id for i in self.items() if i.status == "open" and i.scope == name]
for line in _git(
root, "for-each-ref", "--format=%(refname:short)", "refs/heads"
).splitlines():
@@ -384,7 +429,7 @@ class Walk:
# And no write is lost: every file an open item's driver wrote is on
# the workspace while it's that item's, or in the forge.
for item in self.items():
if item.status != "open":
if item.status != "open" or item.scope != name:
continue
written = {
str(e.event.arguments.get("path"))
@@ -444,7 +489,22 @@ def _is_ancestor(bare: Path, commit: str, ref: str) -> bool:
@pytest.mark.parametrize("seed", SEEDS)
def test_the_loop_keeps_its_properties(world: dict[str, Any], seed: int) -> None:
walk = Walk(world, seed)
walk = Walk({"sandbox": world}, seed)
for _ in range(STEPS):
walk.step()
walk.check()
@pytest.mark.parametrize("seed", TWO_PROJECT_SEEDS)
def test_two_projects_keep_the_properties(
ssh_host: SSHHost, tmp_path: Path, seed: int
) -> None:
"""D8: the same walk over two projects, whose issue numbers collide."""
sandbox = make_world(ssh_host, tmp_path / "sandbox")
other = make_world(
ssh_host, tmp_path / "other", store=sandbox["store"], repo_name="don/other"
)
walk = Walk({"sandbox": sandbox, "other": other}, seed)
for _ in range(STEPS):
walk.step()
walk.check()