Skip to content

πŸš€ The fast, Pythonic way to build MCP servers and clients.

Let private triage sweeps run without the Sol reviewer (#6229)

nate nowack588297eb53c0
diff --git a/.github/agentic/flows/agentic_stack_controller.py b/.github/agentic/flows/agentic_stack_controller.py
index 68ea45610..9d60401b2 100644
--- a/.github/agentic/flows/agentic_stack_controller.py
+++ b/.github/agentic/flows/agentic_stack_controller.py
@@ -191,7 +191,14 @@ def run_batch(batch: Batch, runtime: Runtime) -> dict:
proof = validate_reproduction_patch(result)
record(owner, proof_key, proof)
result = {**result, "regression_validation": proof}
- if requires_review(destination):
+ if (
+ requires_review(destination)
+ and "triage_review" not in runtime.plans
+ ):
+ # Private sweeps run without the Sol reviewer; anything later
+ # proposed for public action gets its own adversarial review.
+ result = {**result, "adversarial_review": None}
+ elif requires_review(destination):
if final_evidence != evidence:
review_item = {**item, "evidence": final_evidence}
projected = assessment_context(review_item)
diff --git a/.github/agentic/flows/agentic_stack_runtime.py b/.github/agentic/flows/agentic_stack_runtime.py
index fcd5768cf..7da3f0e9b 100644
--- a/.github/agentic/flows/agentic_stack_runtime.py
+++ b/.github/agentic/flows/agentic_stack_runtime.py
@@ -142,7 +142,11 @@ class Runtime(BaseModel):
value is None for value in native
):
raise ValueError("Native triage requires its plan and both deployments")
- if self.triage_plan is not None and "triage_review" not in self.plans:
+ if (
+ self.triage_plan is not None
+ and self.publish_reviews
+ and "triage_review" not in self.plans
+ ):
raise ValueError("Native triage requires adversarial triage review")
return self
diff --git a/.github/agentic/tests/test_agentic_triage_review.py b/.github/agentic/tests/test_agentic_triage_review.py
index 6ce1e2622..80a3ee093 100644
--- a/.github/agentic/tests/test_agentic_triage_review.py
+++ b/.github/agentic/tests/test_agentic_triage_review.py
@@ -82,7 +82,7 @@ def test_operational_failures_are_not_triage_decisions():
assert not requires_review("operational")
-def test_native_triage_cannot_run_without_adversarial_review():
+def test_only_private_sweeps_may_skip_adversarial_review():
plans = {
role: AgentPlan(flow_id=uuid4(), version_id=uuid4())
for role in ("assess", "reproduce")
@@ -93,10 +93,20 @@ def test_native_triage_cannot_run_without_adversarial_review():
"triage_verify_id": uuid4(),
"triage_finalize_id": uuid4(),
}
- with pytest.raises(ValueError, match="adversarial triage review"):
- Runtime(plans=plans, **native)
- plans["triage_review"] = AgentPlan(flow_id=uuid4(), version_id=uuid4())
Runtime(plans=plans, **native)
+ publishing = {
+ **native,
+ "publish_reviews": True,
+ "reviewer_app": {
+ "slug": "reviewer",
+ "app_id": 1,
+ "private_key_block": "reviewer-key",
+ },
+ }
+ with pytest.raises(ValueError, match="independent triage review"):
+ Runtime(plans=plans, **publishing)
+ plans["triage_review"] = AgentPlan(flow_id=uuid4(), version_id=uuid4())
+ Runtime(plans=plans, **publishing)
def test_public_facing_reviewers_run_on_sol():
@@ -124,3 +134,78 @@ def test_adversarial_standards_live_in_an_editable_repository_skill():
"triage_review"
]["objective"]
assert "adversarial-review skill in instructions.files" in objective
+
+
+def test_private_sweep_without_reviewer_skips_sol(monkeypatch):
+ from datetime import UTC, datetime, timedelta
+
+ import flows.agentic_stack_controller as controller
+ from flows.agentic_stack_policy import Batch
+
+ now = datetime.now(UTC)
+ batch = Batch.model_validate(
+ {
+ "id": "sweep",
+ "started_at": now,
+ "expires_at": now + timedelta(hours=1),
+ "groups": [
+ {
+ "name": "g",
+ "rationale": "sweep",
+ "branch": "main",
+ "base_sha": "a" * 40,
+ "items": [{"issue_number": 7}],
+ }
+ ],
+ }
+ )
+ plans = {
+ role: AgentPlan(flow_id=uuid4(), version_id=uuid4())
+ for role in ("assess", "reproduce")
+ }
+ runtime = Runtime(
+ plans=plans,
+ source_sha="b" * 40,
+ triage_plan=AgentPlan(flow_id=uuid4(), version_id=uuid4()),
+ triage_verify_id=uuid4(),
+ triage_finalize_id=uuid4(),
+ )
+ evidence = {"base_sha": "a" * 40}
+ monkeypatch.setattr(controller, "batch_owner", lambda *_: uuid4())
+ monkeypatch.setattr(controller, "saved", lambda *_: None)
+ monkeypatch.setattr(controller, "record", lambda *_: None)
+ monkeypatch.setattr(controller, "owner_alive", lambda *_: None)
+ monkeypatch.setattr(
+ controller,
+ "skills",
+ lambda _: {
+ "existing_instructions": {},
+ "assess_instructions": {},
+ "reproduce_instructions": {},
+ },
+ )
+ monkeypatch.setattr(controller, "collect", lambda *_: evidence)
+ monkeypatch.setattr(
+ controller,
+ "route_intake",
+ lambda _: {"route": "review_existing", "issue_number": 7},
+ )
+ monkeypatch.setattr(controller, "assessment_context", lambda _: {})
+ monkeypatch.setattr(
+ controller,
+ "run_native_triage",
+ lambda *_: ("existing_review", {"issue_number": 7}, [], evidence),
+ )
+
+ def no_sol(*_):
+ raise AssertionError("Sol review ran in a private sweep")
+
+ monkeypatch.setattr(controller, "judge", no_sol)
+ monkeypatch.setattr(controller, "usage_report", lambda _: {})
+ monkeypatch.setattr(controller, "revalidate", lambda result, *_: result)
+
+ result = controller.run_batch(batch, runtime)
+
+ assert result["bins"]["existing_review"] == [
+ {"issue_number": 7, "adversarial_review": None}
+ ]