Skip to content

πŸš€ The fast, Pythonic way to build MCP servers and clients.

Keep posted review receipts on partial delivery and reject differing reviews (#6178)

nate nowack1b243c1cc90e
diff --git a/.github/agentic/flows/agentic_review_delivery.py b/.github/agentic/flows/agentic_review_delivery.py
index fccca8333..669060e5a 100644
--- a/.github/agentic/flows/agentic_review_delivery.py
+++ b/.github/agentic/flows/agentic_review_delivery.py
@@ -2,6 +2,7 @@ from __future__ import annotations
from uuid import UUID
+import httpx
from prefect import task
from flows.agentic_claims import claim_work
@@ -20,6 +21,14 @@ from flows.agentic_stack_runtime import (
)
+class PartialReviewDelivery(RuntimeError):
+ """Delivery failed after some reviews were already public."""
+
+ def __init__(self, reason: str, posted: list[dict]):
+ super().__init__(reason)
+ self.posted = posted
+
+
@task(retries=2, retry_delay_seconds=[2, 5], persist_result=False)
def deliver_reviews(
owner: UUID,
@@ -52,73 +61,91 @@ def deliver_reviews(
receipts = []
skipped = []
with review_client(app, secret_sync(app.private_key_block)) as github:
- for check in verified:
- owner_alive(owner, batch.expires_at)
- pr = originals[check["pr_number"]]
- if pr["state"] != "open" or pr.get("merged") or held(pr):
- skipped.append(
- {
- "pr_number": pr["number"],
- "reason": "Closed, merged, draft or held contribution",
- }
- )
- continue
- if check["status"] != "verification_recorded":
- skipped.append(
- {
+ try:
+ for check in verified:
+ owner_alive(owner, batch.expires_at)
+ pr = originals[check["pr_number"]]
+ if pr["state"] != "open" or pr.get("merged") or held(pr):
+ skipped.append(
+ {
+ "pr_number": pr["number"],
+ "reason": "Closed, merged, draft or held contribution",
+ }
+ )
+ continue
+ if check["status"] != "verification_recorded":
+ skipped.append(
+ {
+ "pr_number": pr["number"],
+ "reason": "Current contribution verification is incomplete",
+ }
+ )
+ continue
+ if (
+ check["status"] != "verification_recorded"
+ or check["head_sha"] != pr["head_sha"]
+ or check["baseline_sha"] != evidence["base_sha"]
+ ):
+ raise ValueError(
+ "Contribution verification is unavailable or stale"
+ )
+ revision = digest([check["head_sha"], check["baseline_sha"], app.slug])[
+ :24
+ ]
+ key = f"{batch.id}-pr-{pr['number']}-review-{revision}"
+ receipt = saved(owner, key)
+ if receipt is not None and any(
+ receipt.get(name) != value
+ for name, value in {
"pr_number": pr["number"],
- "reason": "Current contribution verification is incomplete",
- }
- )
- continue
- if (
- check["status"] != "verification_recorded"
- or check["head_sha"] != pr["head_sha"]
- or check["baseline_sha"] != evidence["base_sha"]
- ):
- raise ValueError("Contribution verification is unavailable or stale")
- revision = digest([check["head_sha"], check["baseline_sha"], app.slug])[:24]
- key = f"{batch.id}-pr-{pr['number']}-review-{revision}"
- receipt = saved(owner, key)
- if receipt is not None and any(
- receipt.get(name) != value
- for name, value in {
- "pr_number": pr["number"],
- "head_sha": check["head_sha"],
- "baseline_sha": check["baseline_sha"],
- "reviewer_login": app.slug + "[bot]",
- }.items()
- ):
- raise ValueError(
- "Retained publication receipt has a different revision"
- )
- if receipt is None:
+ "head_sha": check["head_sha"],
+ "baseline_sha": check["baseline_sha"],
+ "reviewer_login": app.slug + "[bot]",
+ }.items()
+ ):
+ raise ValueError(
+ "Retained publication receipt has a different revision"
+ )
+ if receipt is None:
- def guard() -> None:
- owner_alive(owner, batch.expires_at)
- fresh = collect(number, group.branch)
- if review_revision(
- before_delivery(fresh, receipts, app.slug + "[bot]")
- ) != review_revision(
- before_delivery(evidence, receipts, app.slug + "[bot]")
- ):
- raise ValueError("Issue or contribution changed after review")
+ def guard() -> None:
+ owner_alive(owner, batch.expires_at)
+ fresh = collect(number, group.branch)
+ if review_revision(
+ before_delivery(fresh, receipts, app.slug + "[bot]")
+ ) != review_revision(
+ before_delivery(evidence, receipts, app.slug + "[bot]")
+ ):
+ raise ValueError(
+ "Issue or contribution changed after review"
+ )
- body = review_body(
- pr["number"], check["head_sha"], check["baseline_sha"], outcome
- )
- with claim_work(
- [f"fastmcp-pr-{pr['number']}-review"], owner, batch.expires_at
- ):
- receipt = post_review(
- github,
- pr["number"],
- check["head_sha"],
- check["baseline_sha"],
- app.slug + "[bot]",
- body,
- guard,
+ body = review_body(
+ pr["number"], check["head_sha"], check["baseline_sha"], outcome
)
- record(owner, key, receipt)
- receipts.append(receipt)
+ with claim_work(
+ [f"fastmcp-pr-{pr['number']}-review"], owner, batch.expires_at
+ ):
+ receipt = post_review(
+ github,
+ pr["number"],
+ check["head_sha"],
+ check["baseline_sha"],
+ app.slug + "[bot]",
+ body,
+ guard,
+ )
+ record(owner, key, receipt)
+ receipts.append(receipt)
+ except (
+ ValueError,
+ RuntimeError,
+ TimeoutError,
+ httpx.HTTPError,
+ OSError,
+ ) as error:
+ if not receipts:
+ raise
+ # Keep reviews that are already public visible to the queue.
+ raise PartialReviewDelivery(str(error), receipts) from error
return {"posted": receipts, "skipped": skipped}
diff --git a/.github/agentic/flows/agentic_review_publication.py b/.github/agentic/flows/agentic_review_publication.py
index 3ca34dc4d..480f01fc1 100644
--- a/.github/agentic/flows/agentic_review_publication.py
+++ b/.github/agentic/flows/agentic_review_publication.py
@@ -131,6 +131,11 @@ def post_review(
raise ValueError("Multiple reviews exist for this revision")
if existing:
review = existing[0]
+ # GitHub may normalize line endings; any other difference is new feedback.
+ if (review.get("body") or "").replace("\r\n", "\n") != body.replace(
+ "\r\n", "\n"
+ ):
+ raise ValueError("A different review already exists for this revision")
else:
guard()
current()
diff --git a/.github/agentic/flows/agentic_stack_controller.py b/.github/agentic/flows/agentic_stack_controller.py
index b55ec8982..5aceda3ac 100644
--- a/.github/agentic/flows/agentic_stack_controller.py
+++ b/.github/agentic/flows/agentic_stack_controller.py
@@ -461,7 +461,11 @@ def run_batch(batch: Batch, runtime: Runtime) -> dict:
httpx.HTTPError,
OSError,
) as error:
- delivery = {"status": "failed", "reason": str(error)[:1000]}
+ delivery = {
+ "status": "failed",
+ "reason": str(error)[:1000],
+ "posted": getattr(error, "posted", []),
+ }
bins["operational"].append(
{
"issue_number": number,
diff --git a/.github/agentic/scripts/agentic_stack.sh b/.github/agentic/scripts/agentic_stack.sh
index 94203bf4d..453691fa5 100644
--- a/.github/agentic/scripts/agentic_stack.sh
+++ b/.github/agentic/scripts/agentic_stack.sh
@@ -18,8 +18,11 @@ upsert() {
case "${1:-}" in
validate)
+ scratch=$(mktemp -d)
+ trap 'rm -rf "$scratch"' EXIT
for role in assess reproduce prepare review triage_review; do
- api POST /execution-plans/validate --data "@workflows/fastmcp-stack-$role.plan.json" | jq -e '.valid == true'
+ jq '{plan: .}' "workflows/fastmcp-stack-$role.plan.json" > "$scratch/$role.json"
+ api POST /execution-plans/validate --data "@$scratch/$role.json" | jq -e '.valid == true'
done
;;
register)
diff --git a/.github/agentic/tests/test_agentic_review_publication.py b/.github/agentic/tests/test_agentic_review_publication.py
index 04d6c4b97..3e09142fb 100644
--- a/.github/agentic/tests/test_agentic_review_publication.py
+++ b/.github/agentic/tests/test_agentic_review_publication.py
@@ -1,16 +1,23 @@
import hashlib
import json
import threading
+from contextlib import nullcontext
from copy import deepcopy
+from datetime import UTC, datetime, timedelta
from http.client import RemoteDisconnected
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
from pathlib import Path
+from uuid import uuid4
import jwt
import pytest
from cryptography.hazmat.primitives import serialization
from cryptography.hazmat.primitives.asymmetric import rsa
+from prefect import flow
+from prefect.client.orchestration import get_client
+from prefect.testing.utilities import prefect_test_harness
+from flows import agentic_review_delivery as delivery
from flows.agentic_review_app import ReviewApp, review_client
from flows.agentic_review_publication import (
before_delivery,
@@ -18,7 +25,9 @@ from flows.agentic_review_publication import (
post_review,
review_body,
)
+from flows.agentic_stack_policy import Batch
from flows.agentic_stack_publication import GitHub
+from flows.agentic_stack_runtime import Runtime, record
HEAD = "a" * 40
BASE = "b" * 40
@@ -529,3 +538,106 @@ def test_human_changes_to_posted_bot_review_are_not_hidden(change):
assert before_delivery(original, [receipt], BOT) != before_delivery(
current, [receipt], BOT
)
+
+
+def test_existing_review_with_different_feedback_is_not_adopted(endpoint):
+ state, url, _ = endpoint
+ client = GitHub("test-installation", url)
+ post_review(client, 7, HEAD, BASE, BOT, body(), lambda: None)
+ different = body().replace("now survives rendering", "is still dropped")
+ with pytest.raises(ValueError, match="different review already exists"):
+ post_review(client, 7, HEAD, BASE, BOT, different, lambda: None)
+ assert len(state["reviews"]) == 1
+
+
+def test_existing_review_with_crlf_feedback_is_adopted(endpoint):
+ state, url, _ = endpoint
+ client = GitHub("test-installation", url)
+ crlf = body().replace("now survives rendering.", "now survives\r\nrendering.")
+ post_review(client, 7, HEAD, BASE, BOT, crlf, lambda: None)
+ state["reviews"][0]["body"] = state["reviews"][0]["body"].replace("\r\n", "\n")
+ receipt = post_review(client, 7, HEAD, BASE, BOT, crlf, lambda: None)
+ assert receipt["status"] == "already_posted"
+
+
+def test_partial_delivery_failure_keeps_already_posted_receipts(monkeypatch):
+ @flow
+ def controller():
+ return None
+
+ posted = []
+
+ def post(github, number, head, baseline, reviewer, body, guard):
+ if posted:
+ raise ValueError("Issue or contribution changed after review")
+ posted.append({"pr_number": number, "head_sha": head, "status": "posted"})
+ return posted[-1]
+
+ monkeypatch.setattr(delivery, "review_client", lambda *_: nullcontext())
+ monkeypatch.setattr(delivery, "secret_sync", lambda _: "key")
+ monkeypatch.setattr(delivery, "claim_work", lambda *_: nullcontext())
+ monkeypatch.setattr(delivery, "post_review", post)
+ monkeypatch.setattr(delivery, "owner_alive", lambda *_: None)
+ monkeypatch.setattr(delivery, "review_body", lambda *_: "review")
+ now = datetime.now(UTC)
+ batch = Batch(
+ id="partial",
+ started_at=now,
+ expires_at=now + timedelta(minutes=5),
+ groups=[
+ {
+ "name": "review",
+ "rationale": "Deliver two verified contributions",
+ "base_sha": BASE,
+ "branch": "main",
+ "items": [{"issue_number": 1}],
+ }
+ ],
+ )
+ runtime = Runtime(
+ plans={
+ role: {"flow_id": uuid4(), "version_id": uuid4()}
+ for role in ["assess", "reproduce", "prepare", "review", "triage_review"]
+ },
+ validator_id=uuid4(),
+ source_sha=BASE,
+ publish_reviews=True,
+ reviewer_app={"app_id": 42, "slug": "reviewer", "private_key_block": "k"},
+ )
+ review = {"decision": "review_existing", "reason": "r", "findings": []}
+ prs = [
+ {"number": n, "state": "open", "merged": False, "head_sha": HEAD}
+ for n in (7, 8)
+ ]
+ outcome = {
+ "issue_number": 1,
+ "adversarial_review": review,
+ "triage_review_run_id": "review-run",
+ "verification": [
+ {
+ "pr_number": n,
+ "status": "verification_recorded",
+ "head_sha": HEAD,
+ "baseline_sha": BASE,
+ }
+ for n in (7, 8)
+ ],
+ }
+ with prefect_test_harness(), get_client(sync_client=True) as client:
+ owner = client.create_flow_run(controller).id
+ record.fn(
+ owner,
+ "partial-issue-1-triage-review",
+ {"run_id": "review-run", "result": review},
+ )
+ with pytest.raises((ValueError, RuntimeError)) as failure:
+ delivery.deliver_reviews.fn(
+ owner,
+ batch,
+ batch.groups[0],
+ runtime,
+ outcome,
+ {"prs": prs, "base_sha": BASE},
+ )
+ assert [r["pr_number"] for r in posted] == [7]
+ assert getattr(failure.value, "posted", []) == posted