Projects as configuration entries (M1.4, D8)
CI / checks (push) Skipped
CI / checks (pull_request) Failing after 5m7s
CI / checks (push) Skipped
CI / checks (pull_request) Failing after 5m7s
- Config: `projects: {name: entry}` with repo, base_branch, workspace,
build_host, push credential, labels (defaults added to every issue),
secrets by ${VAR} reference with visibility (commands | harness), and
egress (recorded, not enforced). Every problem in every entry is
reported at once; an inline secret, a bad visibility, a repeated repo or
a non-slug name is an error.
- The old loop runs exactly one project (only_project) and refuses more.
- The new loop is per project: each its own clone, workspace (with its
commands-visible secrets, redacted on return), push credential and
milestones; intake polls every project's repository; items are scoped
to their project for every run, publish and PR watch. Issues are named
project#14 when there is more than one project, and a bare #14 that
matches two is asked about. Switching projects parks the one worked on
last (all others after a boot).
- A restatement that ends in a question is sent back to the driver, not
posted: the e2e driver kept asking "Is this the right interpretation?",
which a plain yes can't answer.
Tests: config entries and errors, two-project assembly, two-project loop
tests (intake, own workspaces and repos, #14 in both, do-next, label
defaults, parking across projects), the property walk over two projects
with a new own_repository_only property (an item using the first
project's repo fails 15 of 16 seeds), and the restatement check.
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01PPQ14L3LikyefCMFBqEbKE
This commit is contained in:
1 parent
4ecff19933
commit
064d9bf554
17 files changed
+1139
-315
No files matched your search
@@ -9,7 +9,7 @@ from typing import Any
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
from test_config import SECRETS, tree
|
||||
from test_config import SECRETS, P, tree
|
||||
|
||||
from nisse_dev.agent import build_agent, discover_models, prepare_workspace
|
||||
from nisse_dev.catalog import ModelCatalog, UnknownModel
|
||||
@@ -51,11 +51,13 @@ def config_for(tmp_path: Path, **changes: str) -> AgentConfig:
|
||||
return AgentConfig.from_tree(
|
||||
tree(
|
||||
state_db=str(tmp_path / "state.db"),
|
||||
workspace=str(tmp_path / "work"),
|
||||
build_host__address=HOST.address,
|
||||
build_host__user=HOST.user,
|
||||
build_host__port=str(HOST.port),
|
||||
build_host__key=str(HOST._key),
|
||||
**{
|
||||
f"{P}workspace": str(tmp_path / "work"),
|
||||
f"{P}build_host__address": HOST.address,
|
||||
f"{P}build_host__user": HOST.user,
|
||||
f"{P}build_host__port": str(HOST.port),
|
||||
f"{P}build_host__key": str(HOST._key),
|
||||
},
|
||||
**changes,
|
||||
),
|
||||
SECRETS,
|
||||
@@ -397,8 +399,63 @@ def test_the_new_loop_assembles(tmp_path: Path, origin: Path, monkeypatch) -> No
|
||||
running = build_loop_agent(config)
|
||||
try:
|
||||
assert running.account == "nisse-dev-testing"
|
||||
assert running.loop._repo == config.gitea_repo
|
||||
[project] = running.loop._projects.values()
|
||||
assert project.repo == config.gitea_repo
|
||||
assert (config.workspace / "README.md").exists() # cloned on the build host
|
||||
finally:
|
||||
running.worker.call(running.store.close)
|
||||
running.worker.stop(timeout=5)
|
||||
|
||||
|
||||
def test_each_project_gets_its_own_clone_and_secrets(
|
||||
tmp_path: Path, origin: Path, monkeypatch
|
||||
) -> None:
|
||||
"""D8: a second project is only a second entry."""
|
||||
from nisse_dev.loop.build import build_loop_agent
|
||||
|
||||
second = tmp_path / "second"
|
||||
config = config_for(
|
||||
tmp_path,
|
||||
loop="new",
|
||||
projects__other={
|
||||
"repo": "don/other",
|
||||
"workspace": str(tmp_path / "other-work"),
|
||||
"build_host": {
|
||||
"address": HOST.address,
|
||||
"user": HOST.user,
|
||||
"port": str(HOST.port),
|
||||
"key": str(HOST._key),
|
||||
},
|
||||
"labels": ["discuss"],
|
||||
"secrets": {
|
||||
"API_KEY": {"value": "${TEST_LLM_TOKEN}", "visible": "commands"}
|
||||
},
|
||||
},
|
||||
)
|
||||
git("clone", "-q", "--bare", str(origin), str(second))
|
||||
origins = {"don/demo": str(origin), "don/other": str(second)}
|
||||
monkeypatch.setattr(
|
||||
"nisse_dev.loop.build.GiteaClient.whoami", lambda self: "nisse-dev-testing"
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"nisse_dev.loop.build.GiteaClient.clone_url", lambda self, repo: origins[repo]
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"nisse_dev.agent.ModelCatalog.discover",
|
||||
classmethod(lambda cls, *a, **kw: TEST_CATALOG),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"nisse_dev.loop.build.MatrixTransport", lambda cfg: FakeTransport()
|
||||
)
|
||||
running = build_loop_agent(config)
|
||||
try:
|
||||
projects = running.loop._projects
|
||||
assert set(projects) == {"greet", "other"}
|
||||
assert (tmp_path / "work" / "README.md").exists()
|
||||
assert (tmp_path / "other-work" / "README.md").exists()
|
||||
assert projects["other"].labels == ("discuss",)
|
||||
assert projects["other"].workspace._secrets == {"API_KEY": "sk-x"}
|
||||
assert projects["greet"].workspace._secrets == {}
|
||||
finally:
|
||||
running.worker.call(running.store.close)
|
||||
running.worker.stop(timeout=5)
|
||||
+130
-28
@@ -10,8 +10,13 @@ from nisse_dev.llm import DEFAULT_REQUEST_TIMEOUT_S
|
||||
# A complete config as the YAML file would give it, with secrets by reference.
|
||||
COMPLETE: dict[str, Any] = {
|
||||
"state_db": "~/demo/state.db",
|
||||
"workspace": "~/demo/work",
|
||||
"project": "greet",
|
||||
"projects": {
|
||||
"greet": {
|
||||
"repo": "don/demo",
|
||||
"workspace": "~/demo/work",
|
||||
"build_host": {"address": "build.test", "key": "~/.ssh/nisse-dev"},
|
||||
}
|
||||
},
|
||||
"llm": {"base_url": "https://litellm.test", "token": "${TEST_LLM_TOKEN}"},
|
||||
"roles": {
|
||||
"executor": "claude-haiku-4-5",
|
||||
@@ -29,11 +34,10 @@ COMPLETE: dict[str, Any] = {
|
||||
"gitea": {
|
||||
"url": "https://gitea.test",
|
||||
"token": "${TEST_GITEA_TOKEN}",
|
||||
"repo": "don/demo",
|
||||
"operators": ["don"],
|
||||
},
|
||||
"build_host": {"address": "build.test", "key": "~/.ssh/nisse-dev"},
|
||||
}
|
||||
P = "projects__greet__" # the one project's keys, for tree()
|
||||
SECRETS = {
|
||||
"TEST_LLM_TOKEN": "sk-x",
|
||||
"TEST_MATRIX_TOKEN": "syt_x",
|
||||
@@ -83,15 +87,15 @@ def test_every_missing_setting_is_reported_at_once() -> None:
|
||||
# Failing at first use would surface a missing token twenty minutes into
|
||||
# a milestone, on the call that mattered.
|
||||
with pytest.raises(ConfigError) as excinfo:
|
||||
build(matrix__token=None, gitea__repo=None)
|
||||
build(matrix__token=None, **{f"{P}repo": None})
|
||||
message = str(excinfo.value)
|
||||
assert "matrix.token" in message
|
||||
assert "gitea.repo" in message
|
||||
assert "projects.greet: unset or empty — repo" in message
|
||||
|
||||
|
||||
def test_empty_is_treated_as_missing() -> None:
|
||||
with pytest.raises(ConfigError, match="gitea.repo"):
|
||||
build(gitea__repo=" ")
|
||||
with pytest.raises(ConfigError, match="projects.greet: unset or empty — repo"):
|
||||
build(**{f"{P}repo": " "})
|
||||
|
||||
|
||||
def test_a_reference_to_an_unset_variable_is_an_error() -> None:
|
||||
@@ -115,7 +119,7 @@ def test_an_operator_list_of_only_blanks_is_missing() -> None:
|
||||
|
||||
|
||||
def test_base_branch_is_configurable() -> None:
|
||||
assert build(gitea__base_branch="trunk").base_branch == "trunk"
|
||||
assert build(**{f"{P}base_branch": "trunk"}).base_branch == "trunk"
|
||||
|
||||
|
||||
def test_max_output_tokens_is_configurable_and_checked() -> None:
|
||||
@@ -148,7 +152,7 @@ def test_redacted_summary_carries_no_secrets() -> None:
|
||||
flat = " ".join(summary.values())
|
||||
for secret in SECRETS.values():
|
||||
assert secret not in flat
|
||||
assert summary["project"] == "greet"
|
||||
assert "greet (don/demo, build.test:" in summary["projects"]
|
||||
|
||||
|
||||
def test_tuning_is_read() -> None:
|
||||
@@ -178,7 +182,7 @@ def test_a_partly_given_tuning_section_keeps_the_other_defaults() -> None:
|
||||
def test_every_problem_is_reported_at_once() -> None:
|
||||
# A misspelt knob that was silently ignored would look exactly like a knob
|
||||
# with no effect, so unknown keys are errors.
|
||||
bad = tree(project=None, llm_extra=1)
|
||||
bad = tree(llm_extra=1, **{f"{P}workspace": None})
|
||||
bad["tuning"] = {
|
||||
"executor": {"max_turn": 40, "failure_limit": 0},
|
||||
"expert": {"consult_budget": True},
|
||||
@@ -188,7 +192,7 @@ def test_every_problem_is_reported_at_once() -> None:
|
||||
AgentConfig.from_tree(bad, SECRETS)
|
||||
message = str(excinfo.value)
|
||||
for expected in (
|
||||
"project",
|
||||
"projects.greet: unset or empty — workspace",
|
||||
"llm_extra: unknown setting",
|
||||
"tuning.executor.max_turn: unknown setting",
|
||||
"tuning.executor.failure_limit: must be a positive number",
|
||||
@@ -207,8 +211,13 @@ def test_a_whole_number_is_required_where_one_is_meant() -> None:
|
||||
|
||||
FILE = """
|
||||
state_db: ~/demo/state.db
|
||||
workspace: ~/demo/work
|
||||
project: greet
|
||||
projects:
|
||||
greet:
|
||||
repo: don/demo
|
||||
workspace: ~/demo/work
|
||||
build_host:
|
||||
address: build.test
|
||||
key: ~/.ssh/nisse-dev
|
||||
llm: {base_url: "https://litellm.test", token: "${TEST_LLM_TOKEN}"}
|
||||
roles:
|
||||
executor: local-fast
|
||||
@@ -224,11 +233,7 @@ matrix:
|
||||
gitea:
|
||||
url: https://gitea.test
|
||||
token: ${TEST_GITEA_TOKEN}
|
||||
repo: don/demo
|
||||
operators: [don]
|
||||
build_host:
|
||||
address: build.test
|
||||
key: ~/.ssh/nisse-dev
|
||||
"""
|
||||
|
||||
|
||||
@@ -297,7 +302,7 @@ def test_state_db_inside_the_clone_is_refused(tmp_path: Path) -> None:
|
||||
# own state alongside the project.
|
||||
work = tmp_path / "project"
|
||||
with pytest.raises(ConfigError, match="inside the workspace"):
|
||||
build(workspace=str(work), state_db=str(work / "state.db"))
|
||||
build(state_db=str(work / "state.db"), **{f"{P}workspace": str(work)})
|
||||
|
||||
|
||||
def test_env_file_inside_the_clone_is_refused(tmp_path: Path) -> None:
|
||||
@@ -306,7 +311,7 @@ def test_env_file_inside_the_clone_is_refused(tmp_path: Path) -> None:
|
||||
work = tmp_path / "project"
|
||||
with pytest.raises(ConfigError, match="NISSE_ENV"):
|
||||
AgentConfig.from_tree(
|
||||
tree(workspace=str(work), state_db=str(tmp_path / "state.db")),
|
||||
tree(state_db=str(tmp_path / "state.db"), **{f"{P}workspace": str(work)}),
|
||||
{**SECRETS, "NISSE_ENV": str(work / ".env")},
|
||||
)
|
||||
|
||||
@@ -315,13 +320,17 @@ def test_a_config_file_inside_the_clone_is_refused(tmp_path: Path) -> None:
|
||||
work = tmp_path / "project"
|
||||
work.mkdir()
|
||||
text = FILE.replace("workspace: ~/demo/work", f"workspace: {work}")
|
||||
assert text != FILE
|
||||
with pytest.raises(ConfigError, match="NISSE_CONFIG"):
|
||||
AgentConfig.from_file(write(work, text), SECRETS)
|
||||
|
||||
|
||||
def test_sibling_paths_are_fine(tmp_path: Path) -> None:
|
||||
config = AgentConfig.from_tree(
|
||||
tree(workspace=str(tmp_path / "project"), state_db=str(tmp_path / "state.db")),
|
||||
tree(
|
||||
state_db=str(tmp_path / "state.db"),
|
||||
**{f"{P}workspace": str(tmp_path / "project")},
|
||||
),
|
||||
{**SECRETS, "NISSE_ENV": str(tmp_path / ".env")},
|
||||
)
|
||||
assert config.workspace.name == "project"
|
||||
@@ -342,30 +351,36 @@ def test_the_build_host_is_required_and_defaults_to_root_on_22() -> None:
|
||||
assert config.build_host.port == 22
|
||||
assert config.build_host.key.is_absolute()
|
||||
with pytest.raises(ConfigError, match="build_host.address"):
|
||||
build(build_host__address=None)
|
||||
build(**{f"{P}build_host__address": None})
|
||||
|
||||
|
||||
def test_a_bad_build_host_port_is_reported() -> None:
|
||||
with pytest.raises(ConfigError, match="build_host.port"):
|
||||
build(build_host__port="ssh")
|
||||
build(**{f"{P}build_host__port": "ssh"})
|
||||
|
||||
|
||||
def test_the_push_key_is_optional_and_a_path() -> None:
|
||||
assert build().push_key is None
|
||||
assert build(gitea__push_key="~/k").push_key == Path("~/k").expanduser()
|
||||
assert build(**{f"{P}push_key": "~/k"}).push_key == Path("~/k").expanduser()
|
||||
|
||||
|
||||
def test_a_push_token_needs_its_user_and_excludes_a_key() -> None:
|
||||
config = build(gitea__push_user="pusher", gitea__push_token="${TEST_GITEA_TOKEN}")
|
||||
token = {f"{P}push_user": "pusher", f"{P}push_token": "${TEST_GITEA_TOKEN}"}
|
||||
config = build(**token)
|
||||
assert (config.push_user, config.push_token) == ("pusher", "gt")
|
||||
with pytest.raises(ConfigError, match="set both or neither"):
|
||||
build(gitea__push_user="pusher")
|
||||
build(**{f"{P}push_user": "pusher"})
|
||||
with pytest.raises(ConfigError, match="not both"):
|
||||
build(gitea__push_user="p", gitea__push_token="t", gitea__push_key="~/k")
|
||||
build(**token, **{f"{P}push_key": "~/k"})
|
||||
|
||||
|
||||
def test_no_credential_appears_in_the_config_repr() -> None:
|
||||
config = build(gitea__push_user="pusher", gitea__push_token="${TEST_GITEA_TOKEN}")
|
||||
config = build(
|
||||
**{f"{P}push_user": "pusher", f"{P}push_token": "${TEST_GITEA_TOKEN}"},
|
||||
**{
|
||||
f"{P}secrets": {"DB": {"value": "${TEST_LLM_TOKEN}", "visible": "commands"}}
|
||||
},
|
||||
)
|
||||
text = repr(config)
|
||||
for secret in SECRETS.values():
|
||||
assert secret not in text
|
||||
@@ -376,3 +391,90 @@ def test_the_loop_defaults_to_old_and_takes_new() -> None:
|
||||
assert build(loop="new").loop == "new"
|
||||
with pytest.raises(ConfigError, match="loop: must be old or new"):
|
||||
build(loop="newer")
|
||||
|
||||
|
||||
# --- projects (D8) -----------------------------------------------------------
|
||||
|
||||
SECOND = {
|
||||
"repo": "don/other",
|
||||
"workspace": "~/demo/other",
|
||||
"build_host": {"address": "build2.test", "key": "~/.ssh/nisse-dev"},
|
||||
}
|
||||
|
||||
|
||||
def test_a_second_project_is_a_second_entry() -> None:
|
||||
config = build(projects__other=SECOND)
|
||||
assert [p.name for p in config.projects] == ["greet", "other"]
|
||||
other = config.projects[1]
|
||||
assert (other.repo, other.build_host.address) == ("don/other", "build2.test")
|
||||
assert other.base_branch == "main" and other.labels == () and other.secrets == {}
|
||||
|
||||
|
||||
def test_the_old_loop_refuses_more_than_one_project() -> None:
|
||||
config = build(projects__other=SECOND)
|
||||
with pytest.raises(ConfigError, match="exactly one project"):
|
||||
config.only_project()
|
||||
with pytest.raises(ConfigError, match="exactly one project"):
|
||||
_ = config.gitea_repo
|
||||
|
||||
|
||||
def test_at_least_one_project_is_needed() -> None:
|
||||
with pytest.raises(ConfigError, match="projects: must name at least one"):
|
||||
build(projects={})
|
||||
|
||||
|
||||
def test_project_problems_are_named_by_project() -> None:
|
||||
bad = tree(projects__other={**SECOND, "repo": "don/demo", "colour": "red"})
|
||||
with pytest.raises(ConfigError) as excinfo:
|
||||
AgentConfig.from_tree(bad, SECRETS)
|
||||
message = str(excinfo.value)
|
||||
assert "projects.other.colour: unknown setting" in message
|
||||
assert "projects.other.repo: don/demo is also project greet's" in message
|
||||
|
||||
|
||||
def test_a_project_name_is_a_slug() -> None:
|
||||
with pytest.raises(ConfigError, match="lowercase letters, digits"):
|
||||
build(projects__Other_Project=SECOND)
|
||||
|
||||
|
||||
def test_labels_and_egress_are_lists_of_names() -> None:
|
||||
config = build(**{f"{P}labels": ["discuss"], f"{P}egress": ["pypi.org"]})
|
||||
assert config.only_project().labels == ("discuss",)
|
||||
assert config.only_project().egress == ("pypi.org",)
|
||||
with pytest.raises(ConfigError, match="projects.greet.labels: must be a list"):
|
||||
build(**{f"{P}labels": "discuss"})
|
||||
|
||||
|
||||
def test_secrets_are_by_reference_with_a_visibility() -> None:
|
||||
config = build(
|
||||
**{
|
||||
f"{P}secrets": {
|
||||
"DB_URL": {"value": "${TEST_LLM_TOKEN}", "visible": "commands"},
|
||||
"DEPLOY": {"value": "${TEST_GITEA_TOKEN}", "visible": "harness"},
|
||||
}
|
||||
}
|
||||
)
|
||||
project = config.only_project()
|
||||
assert project.command_secrets() == {"DB_URL": "sk-x"}
|
||||
assert project.secrets["DEPLOY"].visible == "harness"
|
||||
|
||||
|
||||
def test_a_secret_written_inline_is_refused() -> None:
|
||||
with pytest.raises(ConfigError, match="must be a \\$\\{VAR\\} reference"):
|
||||
build(**{f"{P}secrets": {"DB": {"value": "hunter2", "visible": "commands"}}})
|
||||
|
||||
|
||||
def test_a_secret_needs_a_known_visibility() -> None:
|
||||
with pytest.raises(ConfigError, match="visible: must be one of commands, harness"):
|
||||
build(
|
||||
**{f"{P}secrets": {"DB": {"value": "${TEST_LLM_TOKEN}", "visible": "all"}}}
|
||||
)
|
||||
|
||||
|
||||
def test_a_workspace_of_any_project_inside_is_refused(tmp_path: Path) -> None:
|
||||
work = tmp_path / "other"
|
||||
with pytest.raises(ConfigError, match="project other"):
|
||||
build(
|
||||
state_db=str(work / "state.db"),
|
||||
projects__other={**SECOND, "workspace": str(work)},
|
||||
)
|
||||
@@ -280,3 +280,32 @@ def _msg(text: str, thread: str) -> Any:
|
||||
from nisse_dev.transport import InboundMessage
|
||||
|
||||
return InboundMessage(DON, text, thread=thread)
|
||||
|
||||
|
||||
def test_a_restatement_that_asks_is_sent_back(tmp_path: Path) -> None:
|
||||
"""A restatement is agreed to as it stands; one ending in a question
|
||||
(models add "Is this right?") never goes up."""
|
||||
h = Harness(
|
||||
tmp_path,
|
||||
ScriptedModel(
|
||||
[call("restate", text="I'll add count_words. Is this right?")],
|
||||
[call("restate", text="I'll add count_words to text.py, with tests.")],
|
||||
),
|
||||
FakeGitea(issue()),
|
||||
)
|
||||
h.agent.intake()
|
||||
loop = loop_of(h.item())
|
||||
assert (loop.wait, loop.rev) == (GO_AHEAD, 1)
|
||||
posted = [p for p in h.posted() if p.startswith("rev-")]
|
||||
assert posted == [
|
||||
(
|
||||
"rev-1: I'll add count_words to text.py, with tests.\n\n"
|
||||
"Say go to start, or reply with changes."
|
||||
)
|
||||
]
|
||||
results = [
|
||||
e.event.result
|
||||
for e in h.store.transcript(h.item().id, "driver").read()
|
||||
if hasattr(e.event, "result")
|
||||
]
|
||||
assert any("a restatement asks no questions" in r for r in results)
|
||||
@@ -157,6 +157,18 @@ class Forge:
|
||||
|
||||
@pytest.fixture
|
||||
def world(ssh_host: SSHHost, tmp_path: Path) -> dict[str, Any]:
|
||||
return make_world(ssh_host, tmp_path)
|
||||
|
||||
|
||||
def make_world(
|
||||
ssh_host: SSHHost,
|
||||
tmp_path: Path,
|
||||
store: StateStore | None = None,
|
||||
repo_name: str = "don/sandbox",
|
||||
) -> dict[str, Any]:
|
||||
"""A project's forge (a bare repository and a fake Gitea over it), its
|
||||
clone and milestones; a second one can share the first's store."""
|
||||
tmp_path.mkdir(parents=True, exist_ok=True)
|
||||
bare = tmp_path / "forge.git"
|
||||
git("init", "-q", "--bare", "-b", "main", str(bare))
|
||||
seed = tmp_path / "seed"
|
||||
@@ -179,13 +191,16 @@ def world(ssh_host: SSHHost, tmp_path: Path) -> dict[str, Any]:
|
||||
repo.clone(str(bare))
|
||||
repo.configure_identity("nisse-dev", "n@t")
|
||||
repo.install_excludes()
|
||||
store = StateStore(tmp_path / "state.db", oplog=OpLog("nisse-dev", io.StringIO()))
|
||||
if store is None:
|
||||
store = StateStore(
|
||||
tmp_path / "state.db", oplog=OpLog("nisse-dev", io.StringIO())
|
||||
)
|
||||
forge = Forge(bare)
|
||||
milestones = Milestones(
|
||||
store,
|
||||
repo,
|
||||
forge,
|
||||
repo="don/sandbox", # type: ignore[arg-type]
|
||||
forge, # type: ignore[arg-type]
|
||||
repo=repo_name,
|
||||
remote=str(bare),
|
||||
credential=None,
|
||||
)
|
||||
@@ -197,6 +212,7 @@ def world(ssh_host: SSHHost, tmp_path: Path) -> dict[str, Any]:
|
||||
"host": ssh_host,
|
||||
"bare": bare,
|
||||
"seed": seed,
|
||||
"repo_name": repo_name,
|
||||
}
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,188 @@
|
||||
"""Several projects on the new loop (M1.4, D8): intake from each project's
|
||||
repository, each item working in its own project's workspace and publishing
|
||||
to its own repository, issue numbers told apart, label defaults, and parking
|
||||
across projects. Real git on the throwaway build host; fake Gitea per repo."""
|
||||
|
||||
import io
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
from test_loop_milestones import DON, Forge, ScriptedModel, call, git, make_world
|
||||
|
||||
from nisse_dev.cancel import Cancel
|
||||
from nisse_dev.execution import SSHHost
|
||||
from nisse_dev.loop.agent import LoopAgent, Project
|
||||
from nisse_dev.loop.driver import Driver
|
||||
from nisse_dev.loop.items import ANSWER, GO_AHEAD, MERGE, item_for_issue, loop_of
|
||||
from nisse_dev.oplog import OpLog
|
||||
from nisse_dev.tools import Workspace
|
||||
from nisse_dev.transport import InMemoryTransport
|
||||
from nisse_dev.worker import InlineWorker
|
||||
|
||||
REPOS = {"alpha": "don/alpha", "beta": "don/beta"}
|
||||
|
||||
|
||||
class Router:
|
||||
"""One Gitea for the agent, routing each call to its repository's fake."""
|
||||
|
||||
def __init__(self, forges: dict[str, Forge]) -> None:
|
||||
self.forges = forges
|
||||
|
||||
def __getattr__(self, name: str) -> Any:
|
||||
def call(repo: str, *args: Any, **kwargs: Any) -> Any:
|
||||
return getattr(self.forges[repo], name)(repo, *args, **kwargs)
|
||||
|
||||
return call
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def worlds(ssh_host: SSHHost, tmp_path: Path) -> dict[str, dict[str, Any]]:
|
||||
alpha = make_world(ssh_host, tmp_path / "alpha", repo_name=REPOS["alpha"])
|
||||
beta = make_world(
|
||||
ssh_host, tmp_path / "beta", store=alpha["store"], repo_name=REPOS["beta"]
|
||||
)
|
||||
return {"alpha": alpha, "beta": beta}
|
||||
|
||||
|
||||
def agent_for(
|
||||
worlds: dict[str, dict[str, Any]],
|
||||
model: ScriptedModel,
|
||||
labels: dict[str, tuple[str, ...]] | None = None,
|
||||
) -> tuple[LoopAgent, InMemoryTransport]:
|
||||
transport = InMemoryTransport()
|
||||
cancel = Cancel()
|
||||
router = Router({REPOS[n]: w["forge"] for n, w in worlds.items()})
|
||||
projects = [
|
||||
Project(
|
||||
name=n,
|
||||
repo=REPOS[n],
|
||||
workspace=Workspace(w["host"], w["repo"].root, timeout_s=30),
|
||||
milestones=w["milestones"],
|
||||
labels=(labels or {}).get(n, ()),
|
||||
)
|
||||
for n, w in worlds.items()
|
||||
]
|
||||
for w in worlds.values():
|
||||
w["milestones"]._gitea = router # publishing goes through the router too
|
||||
agent = LoopAgent(
|
||||
worlds["alpha"]["store"],
|
||||
transport,
|
||||
router, # type: ignore[arg-type]
|
||||
projects=projects,
|
||||
account="nisse-dev",
|
||||
operators={DON},
|
||||
gitea_operators={"don"},
|
||||
driver=Driver(model, "m", cancel=cancel), # type: ignore[arg-type]
|
||||
worker=InlineWorker(),
|
||||
cancel=cancel,
|
||||
oplog=OpLog("nisse-dev", io.StringIO()),
|
||||
)
|
||||
transport.on_message(agent.on_message)
|
||||
return agent, transport
|
||||
|
||||
|
||||
def item(worlds: dict[str, Any], project: str) -> Any:
|
||||
found = item_for_issue(worlds["alpha"]["store"], 14, project)
|
||||
assert found is not None
|
||||
return found
|
||||
|
||||
|
||||
def say(transport: InMemoryTransport, worlds: Any, project: str, text: str) -> None:
|
||||
transport.receive(DON, text, thread=loop_of(item(worlds, project)).thread)
|
||||
|
||||
|
||||
def restate(name: str) -> list[Any]:
|
||||
return [call("restate", text=f"{name}: add count_words")]
|
||||
|
||||
|
||||
def work(path: str) -> list[Any]:
|
||||
return [
|
||||
call("start_milestone", name=path),
|
||||
call("write_file", path=path, content=f"# {path}\n"),
|
||||
call("finish_milestone", summary=path, final="yes"),
|
||||
]
|
||||
|
||||
|
||||
def test_both_projects_issues_are_taken_and_told_apart(worlds: Any) -> None:
|
||||
# Both forges have an issue #14.
|
||||
agent, transport = agent_for(worlds, ScriptedModel(restate("a"), restate("b")))
|
||||
agent.intake()
|
||||
a, b = item(worlds, "alpha"), item(worlds, "beta")
|
||||
assert (a.scope, b.scope) == ("alpha", "beta")
|
||||
said = [m.text for m in transport.outbound]
|
||||
assert any(t.startswith("Queued don/alpha#14") for t in said)
|
||||
assert any(t.startswith("Queued don/beta#14") for t in said)
|
||||
status = agent.status()
|
||||
assert "alpha#14 Add count_words" in status and "beta#14 Add count_words" in status
|
||||
|
||||
|
||||
def test_each_item_works_and_publishes_in_its_own_project(worlds: Any) -> None:
|
||||
model = ScriptedModel(restate("a"), restate("b"), work("a.py"), work("b.py"))
|
||||
agent, transport = agent_for(worlds, model)
|
||||
agent.intake()
|
||||
say(transport, worlds, "alpha", "go")
|
||||
say(transport, worlds, "beta", "go")
|
||||
fa, fb = worlds["alpha"]["forge"], worlds["beta"]["forge"]
|
||||
[pa], [pb] = fa.prs, fb.prs
|
||||
assert fa.files(pa) == ["a.py"] and fb.files(pb) == ["b.py"]
|
||||
assert fa.prs[pa]["head"].startswith(item(worlds, "alpha").id)
|
||||
assert fb.prs[pb]["head"].startswith(item(worlds, "beta").id)
|
||||
# Each workspace has only its own project's work.
|
||||
assert not (worlds["alpha"]["repo"].root / "b.py").exists()
|
||||
assert loop_of(item(worlds, "beta")).wait == MERGE
|
||||
|
||||
|
||||
def test_a_bare_number_in_two_projects_is_asked_about(worlds: Any) -> None:
|
||||
model = ScriptedModel(restate("a"), restate("b"), work("b.py"))
|
||||
agent, transport = agent_for(worlds, model)
|
||||
agent.intake()
|
||||
transport.receive(DON, "go #14")
|
||||
asked = transport.outbound[-1].text
|
||||
assert "open in more than one project" in asked
|
||||
assert "go alpha#14" in asked and "go beta#14" in asked
|
||||
assert not loop_of(item(worlds, "alpha")).started
|
||||
transport.receive(DON, "go beta#14")
|
||||
assert loop_of(item(worlds, "beta")).started
|
||||
assert not loop_of(item(worlds, "alpha")).started
|
||||
|
||||
|
||||
def test_do_next_names_a_project(worlds: Any) -> None:
|
||||
agent, transport = agent_for(worlds, ScriptedModel(restate("a"), restate("b")))
|
||||
agent.intake()
|
||||
transport.receive(DON, "do #14 next")
|
||||
assert "do alpha#14 next or do beta#14 next" in transport.outbound[-1].text
|
||||
transport.receive(DON, "do beta#14 next")
|
||||
assert "beta#14 goes first" in transport.outbound[-1].text
|
||||
|
||||
|
||||
def test_a_projects_label_defaults_steer_its_items(worlds: Any) -> None:
|
||||
model = ScriptedModel(
|
||||
[call("ask_human", question="what's it for?")], # alpha: discuss first
|
||||
restate("b"),
|
||||
)
|
||||
agent, _ = agent_for(worlds, model, labels={"alpha": ("discuss",)})
|
||||
agent.intake()
|
||||
assert loop_of(item(worlds, "alpha")).labels == ("discuss",)
|
||||
assert loop_of(item(worlds, "alpha")).wait == ANSWER
|
||||
assert loop_of(item(worlds, "beta")).wait == GO_AHEAD
|
||||
|
||||
|
||||
def test_switching_projects_parks_the_other_projects_work(worlds: Any) -> None:
|
||||
model = ScriptedModel(
|
||||
restate("a"),
|
||||
restate("b"),
|
||||
[ # alpha: half done, then a question
|
||||
call("start_milestone", name="a"),
|
||||
call("write_file", path="half.py", content="H = 1\n"),
|
||||
call("ask_human", question="one more thing?"),
|
||||
],
|
||||
work("b.py"), # beta runs meanwhile
|
||||
)
|
||||
agent, transport = agent_for(worlds, model)
|
||||
agent.intake()
|
||||
say(transport, worlds, "alpha", "go")
|
||||
say(transport, worlds, "beta", "go")
|
||||
bare = ["--git-dir", str(worlds["alpha"]["bare"])]
|
||||
parked = f"refs/heads/{item(worlds, 'alpha').id}-parked"
|
||||
assert git(*bare, "show", f"{parked}^3:half.py") == "H = 1"
|
||||
@@ -15,13 +15,15 @@ from typing import Any
|
||||
|
||||
import pytest
|
||||
from model_contract import assert_renderable
|
||||
from test_loop_milestones import DON, Forge, call
|
||||
from test_loop_milestones import DON, Forge, call, make_world
|
||||
from test_loop_milestones import world as world # noqa: PLC0414 -- the fixture
|
||||
from test_loop_projects import Router
|
||||
|
||||
from nisse_dev.cancel import Cancel
|
||||
from nisse_dev.canonical import Fact, PlanSubmitted, ToolCall, UserMessage
|
||||
from nisse_dev.execution import SSHHost
|
||||
from nisse_dev.gitea import Comment, Issue
|
||||
from nisse_dev.loop.agent import LoopAgent
|
||||
from nisse_dev.loop.agent import LoopAgent, Project
|
||||
from nisse_dev.loop.driver import Driver
|
||||
from nisse_dev.loop.items import (
|
||||
ANSWER,
|
||||
@@ -38,6 +40,7 @@ from nisse_dev.transport import InboundMessage, InMemoryTransport
|
||||
from nisse_dev.worker import InlineWorker
|
||||
|
||||
SEEDS = range(32)
|
||||
TWO_PROJECT_SEEDS = range(100, 116)
|
||||
STEPS = 30
|
||||
|
||||
|
||||
@@ -88,13 +91,17 @@ class PolicyModel:
|
||||
|
||||
|
||||
class Walk:
|
||||
def __init__(self, world: dict[str, Any], seed: int) -> None:
|
||||
self.world = world
|
||||
"""``worlds``: project name -> its world (forge, bare repo, clone,
|
||||
milestones), all over one store."""
|
||||
|
||||
def __init__(self, worlds: dict[str, dict[str, Any]], seed: int) -> None:
|
||||
self.worlds = worlds
|
||||
self.world = next(iter(worlds.values())) # the store's home
|
||||
self.rng = random.Random(seed)
|
||||
self.seed = seed
|
||||
self.steps: list[str] = []
|
||||
self.shown_at_go: list[str] = [] # item:revision on screen at each go
|
||||
self.next_issue = 15
|
||||
self.next_issue = dict.fromkeys(worlds, 15) # numbers repeat across projects
|
||||
self.now = 1000.0
|
||||
self.agent, self.transport = self._agent()
|
||||
|
||||
@@ -106,22 +113,29 @@ class Walk:
|
||||
def _agent(self) -> tuple[LoopAgent, InMemoryTransport]:
|
||||
transport = InMemoryTransport()
|
||||
cancel = Cancel()
|
||||
w = self.world
|
||||
router = Router({w["repo_name"]: w["forge"] for w in self.worlds.values()})
|
||||
for w in self.worlds.values():
|
||||
w["milestones"]._gitea = router
|
||||
agent = LoopAgent(
|
||||
w["store"],
|
||||
self.world["store"],
|
||||
transport,
|
||||
w["forge"],
|
||||
repo="don/sandbox",
|
||||
project="sandbox",
|
||||
router, # type: ignore[arg-type]
|
||||
projects=[
|
||||
Project(
|
||||
name=name,
|
||||
repo=w["repo_name"],
|
||||
workspace=Workspace(w["host"], w["repo"].root, timeout_s=30),
|
||||
milestones=w["milestones"],
|
||||
)
|
||||
for name, w in self.worlds.items()
|
||||
],
|
||||
account="nisse-dev",
|
||||
operators={DON},
|
||||
gitea_operators={"don"},
|
||||
driver=Driver(PolicyModel(self.rng), "m", cancel=cancel), # type: ignore[arg-type]
|
||||
workspace=Workspace(w["host"], w["repo"].root, timeout_s=30),
|
||||
worker=InlineWorker(),
|
||||
cancel=cancel,
|
||||
oplog=OpLog("nisse-dev", _Null()),
|
||||
milestones=w["milestones"],
|
||||
reader=self, # type: ignore[arg-type]
|
||||
bug_grace_s=600,
|
||||
clock=lambda: self.now,
|
||||
@@ -135,14 +149,21 @@ class Walk:
|
||||
def items(self) -> list[Any]:
|
||||
return self.world["store"].work_items()
|
||||
|
||||
def name(self, item: Any) -> str:
|
||||
"""How the operator names an item: by project when there are two."""
|
||||
issue = loop_of(item).issue
|
||||
return f"#{issue}" if len(self.worlds) == 1 else f"{item.scope}#{issue}"
|
||||
|
||||
def step(self) -> None:
|
||||
forge: Forge = self.world["forge"]
|
||||
open_items = [i for i in self.items() if i.status == "open"]
|
||||
# Weighted towards what an item can actually do, so review paths
|
||||
# are walked often, not drowned out by intake and crashes.
|
||||
weights = {"intake": 2 if not open_items else 1, "crash": 1, "issue": 1}
|
||||
item = self.rng.choice(open_items) if open_items else None
|
||||
loop = loop_of(item) if item else None
|
||||
# The forge an item's project lives on; a new issue picks one.
|
||||
project = item.scope if item else self.rng.choice(list(self.worlds))
|
||||
forge: Forge = self.worlds[project]["forge"]
|
||||
if item and loop:
|
||||
weights.update({"go": 3, "answer": 2, "abandon": 1, "go-stale": 1})
|
||||
weights.update({"first": 1, "plain-yes": 2, "named-go": 1, "wait": 2})
|
||||
@@ -151,12 +172,15 @@ class Walk:
|
||||
{"comment": 4, "merge": 2, "close-pr": 1, "comment-merge": 2}
|
||||
)
|
||||
action = self.rng.choices(list(weights), list(weights.values()))[0]
|
||||
self.steps.append(f"{action}#{loop.issue}" if loop else action)
|
||||
self.steps.append(f"{action}{self.name(item)}" if item else action)
|
||||
if action == "intake":
|
||||
self.agent.intake()
|
||||
elif action == "issue":
|
||||
# A new issue, sometimes depending on one that is still open.
|
||||
n, self.next_issue = self.next_issue, self.next_issue + 1
|
||||
project = self.rng.choice(list(self.worlds))
|
||||
forge = self.worlds[project]["forge"]
|
||||
n = self.next_issue[project]
|
||||
self.next_issue[project] += 1
|
||||
labels = self.rng.choice([(), (), ("bug",), ("discuss",), ("design",)])
|
||||
forge.issues.append(Issue(n, f"task {n}", f"do task {n}", "don", labels))
|
||||
live = [i.number for i in forge.assigned_issues("", "") if i.number != n]
|
||||
@@ -171,7 +195,7 @@ class Walk:
|
||||
elif action == "named-go" and loop:
|
||||
if item.position == WAITING and loop.wait == GO_AHEAD:
|
||||
self.shown_at_go.append(f"{item.id}:{loop.rev}")
|
||||
self.transport.receive(DON, f"go #{loop.issue}")
|
||||
self.transport.receive(DON, f"go {self.name(item)}")
|
||||
elif action == "wait":
|
||||
# Time passes: bug countdowns may run out.
|
||||
self.now += self.rng.choice([60, 700])
|
||||
@@ -185,7 +209,7 @@ class Walk:
|
||||
self.shown_at_go.append(f"{other.id}:{o.rev}")
|
||||
self.agent.intake()
|
||||
elif action == "first" and loop:
|
||||
self.agent.handle(InboundMessage(DON, f"do #{loop.issue} next"), None)
|
||||
self.agent.handle(InboundMessage(DON, f"do {self.name(item)} next"), None)
|
||||
elif action == "crash":
|
||||
# Volatile state is lost: a new agent over the same store.
|
||||
self.agent, self.transport = self._agent()
|
||||
@@ -231,12 +255,14 @@ class Walk:
|
||||
|
||||
def check(self) -> None:
|
||||
where = f"seed {self.seed}, steps {self.steps}"
|
||||
store, forge = self.world["store"], self.world["forge"]
|
||||
store = self.world["store"]
|
||||
items = self.items()
|
||||
issues = [loop_of(i).issue for i in items if loop_of(i).issue]
|
||||
issues = [(i.scope, loop_of(i).issue) for i in items if loop_of(i).issue]
|
||||
assert len(issues) == len(set(issues)), f"one_item_per_issue: {where}"
|
||||
for item in items:
|
||||
loop = loop_of(item)
|
||||
forge = self.worlds[item.scope]["forge"]
|
||||
repo = self.worlds[item.scope]["repo_name"]
|
||||
if item.status == "open" and item.position == WAITING:
|
||||
explained = (
|
||||
(loop.wait == GO_AHEAD and loop.rev > 0)
|
||||
@@ -257,7 +283,7 @@ class Walk:
|
||||
)
|
||||
# Blocked comes from external truth (L1): an item whose issue
|
||||
# had an open dependency from the start never ran before it closed.
|
||||
if forge.open_dependencies("don/sandbox", loop.issue):
|
||||
if forge.open_dependencies(repo, loop.issue):
|
||||
assert item.position == BLOCKED or item.status == "closed", (
|
||||
f"blocked_never_runs: #{loop.issue} {item.position}: {where}"
|
||||
)
|
||||
@@ -311,11 +337,30 @@ class Walk:
|
||||
assert not [i for i in items if i.position == RUNNING], (
|
||||
f"one_active (nothing left running): {where}"
|
||||
)
|
||||
self._parked_work_safe(where)
|
||||
self._milestones_start_fresh(where)
|
||||
for name, w in self.worlds.items():
|
||||
self._own_repository_only(name, w, where)
|
||||
self._parked_work_safe(w, name, where)
|
||||
self._milestones_start_fresh(w, where)
|
||||
self._review_fixes_land(w, where)
|
||||
|
||||
def _own_repository_only(self, name: str, world: Any, where: str) -> None:
|
||||
"""D8: an item's branches and PRs reach only its own project's
|
||||
repository."""
|
||||
others = [i.id for i in self.items() if i.scope != name]
|
||||
heads = [p["head"] for p in world["forge"].prs.values()]
|
||||
branches = _git(
|
||||
world["bare"], "for-each-ref", "--format=%(refname:short)", "refs/heads"
|
||||
).split()
|
||||
for ref in [*heads, *branches]:
|
||||
assert not any(ref.startswith(f"{i}-") for i in others), (
|
||||
f"own_repository_only: {ref} in {name}: {where}"
|
||||
)
|
||||
|
||||
def _review_fixes_land(self, world: Any, where: str) -> None:
|
||||
forge: Forge = world["forge"]
|
||||
heads = [p["head"] for p in forge.prs.values()]
|
||||
assert len(heads) == len(set(heads)), f"one_pr_per_milestone: {where}"
|
||||
bare = self.world["bare"]
|
||||
bare = world["bare"]
|
||||
for n, pr in forge.prs.items():
|
||||
if not pr["merged"]:
|
||||
continue
|
||||
@@ -330,11 +375,11 @@ class Walk:
|
||||
)
|
||||
assert landed, f"review_fixes_land (PR {n}): {where}"
|
||||
|
||||
def _milestones_start_fresh(self, where: str) -> None:
|
||||
def _milestones_start_fresh(self, world: Any, where: str) -> None:
|
||||
"""D4: a milestone starts from the latest default branch, so one
|
||||
redone after a closed PR never carries the rejected work."""
|
||||
forge: Forge = self.world["forge"]
|
||||
bare = self.world["bare"]
|
||||
forge: Forge = world["forge"]
|
||||
bare = world["bare"]
|
||||
for item in self.items():
|
||||
mine = sorted(
|
||||
(n, p)
|
||||
@@ -353,13 +398,13 @@ class Walk:
|
||||
bare, rejected, f"refs/heads/{later['head']}"
|
||||
), f"milestones_start_fresh: PR {m} carries PR {n}: {where}"
|
||||
|
||||
def _parked_work_safe(self, where: str) -> None:
|
||||
def _parked_work_safe(self, world: Any, name: str, where: str) -> None:
|
||||
"""L1, D4: nothing of a parked item lives only on the build VM: what
|
||||
every branch of an open item that isn't on the workspace points at is
|
||||
in the forge."""
|
||||
root, bare = self.world["repo"].root, self.world["bare"]
|
||||
in its project's forge."""
|
||||
root, bare = world["repo"].root, world["bare"]
|
||||
current = _git(root, "rev-parse", "--abbrev-ref", "HEAD")
|
||||
ids = [i.id for i in self.items() if i.status == "open"]
|
||||
ids = [i.id for i in self.items() if i.status == "open" and i.scope == name]
|
||||
for line in _git(
|
||||
root, "for-each-ref", "--format=%(refname:short)", "refs/heads"
|
||||
).splitlines():
|
||||
@@ -384,7 +429,7 @@ class Walk:
|
||||
# And no write is lost: every file an open item's driver wrote is on
|
||||
# the workspace while it's that item's, or in the forge.
|
||||
for item in self.items():
|
||||
if item.status != "open":
|
||||
if item.status != "open" or item.scope != name:
|
||||
continue
|
||||
written = {
|
||||
str(e.event.arguments.get("path"))
|
||||
@@ -444,7 +489,22 @@ def _is_ancestor(bare: Path, commit: str, ref: str) -> bool:
|
||||
|
||||
@pytest.mark.parametrize("seed", SEEDS)
|
||||
def test_the_loop_keeps_its_properties(world: dict[str, Any], seed: int) -> None:
|
||||
walk = Walk(world, seed)
|
||||
walk = Walk({"sandbox": world}, seed)
|
||||
for _ in range(STEPS):
|
||||
walk.step()
|
||||
walk.check()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("seed", TWO_PROJECT_SEEDS)
|
||||
def test_two_projects_keep_the_properties(
|
||||
ssh_host: SSHHost, tmp_path: Path, seed: int
|
||||
) -> None:
|
||||
"""D8: the same walk over two projects, whose issue numbers collide."""
|
||||
sandbox = make_world(ssh_host, tmp_path / "sandbox")
|
||||
other = make_world(
|
||||
ssh_host, tmp_path / "other", store=sandbox["store"], repo_name="don/other"
|
||||
)
|
||||
walk = Walk({"sandbox": sandbox, "other": other}, seed)
|
||||
for _ in range(STEPS):
|
||||
walk.step()
|
||||
walk.check()
|
||||
Reference in new issue
Block a user