Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(stamphog): stop failing runs on nul bytes, head races and rate limits (#123276)

Julian Bez7e3eebb7c31f
diff --git a/products/stamphog/backend/temporal/activities.py b/products/stamphog/backend/temporal/activities.py
index 22a044f18ddd..95efeba295a8 100644
--- a/products/stamphog/backend/temporal/activities.py
+++ b/products/stamphog/backend/temporal/activities.py
@@ -51,10 +51,11 @@ from prometheus_client import Counter
from temporalio import activity
from posthog.dataclasses import frozen
+from posthog.egress.github.transport import GitHubEgressBudgetExhausted, GitHubRateLimitError
from posthog.llm.gateway_client import AIGatewayConfig, resolve_ai_gateway_config
from posthog.models import User
from posthog.ph_client import ph_background_capture, ph_scoped_capture
-from posthog.temporal.common.errors import NonReportableError
+from posthog.temporal.common.errors import NonReportableApplicationError, NonReportableError
from posthog.temporal.common.utils import asyncify
from products.stamphog.backend.facade.contracts import StamphogGitHubError
@@ -170,11 +171,26 @@ def _load_run(input: StamphogReviewInput) -> ReviewRun:
)
+def _without_nul(value: Any) -> Any:
+ """Drop NUL characters from every string in ``value``.
+
+ The context holds PR text, and Postgres refuses ``\\u0000`` in jsonb. One NUL in a PR body or
+ comment fails the whole write, and every retry and re-review of that PR fails the same way.
+ """
+ if isinstance(value, str):
+ return value.replace("\x00", "")
+ if isinstance(value, dict):
+ return {_without_nul(key): _without_nul(item) for key, item in value.items()}
+ if isinstance(value, list | tuple):
+ return [_without_nul(item) for item in value]
+ return value
+
+
def _output_merge(updates: dict[str, Any]) -> Func:
"""Postgres ``output || updates``: a top-level key merge that the database performs."""
return Func(
Coalesce(F("output"), Cast(Value("{}"), output_field=JSONField())),
- Cast(Value(json.dumps(updates, cls=DjangoJSONEncoder)), output_field=JSONField()),
+ Cast(Value(json.dumps(_without_nul(updates), cls=DjangoJSONEncoder)), output_field=JSONField()),
template="%(expressions)s",
arg_joiner=" || ",
output_field=JSONField(),
@@ -192,7 +208,7 @@ def _merge_run_output(run: ReviewRun, updates: dict[str, Any]) -> None:
ReviewRun.objects.for_team(run.team_id).using(router.db_for_write(ReviewRun)).filter(id=run.id).update(
output=_output_merge(updates), updated_at=timezone.now()
)
- run.output = {**(run.output or {}), **updates}
+ run.output = {**(run.output or {}), **_without_nul(updates)}
# aio_ continues the series the Action-era runs emitted; the engine blob carries the same word.
@@ -494,12 +510,51 @@ class TransientGitHubError(NonReportableError):
so a review never runs with incomplete context, but only the final attempt reaches error tracking."""
+# The head-fetch command exits with this code when the pull ref names a newer commit. Git does not
+# use it, so a moved head stays apart from a git failure.
+_HEAD_MOVED_EXIT_CODE = 75
+_HEAD_MOVED_OUTPUT_KEY = "head_moved"
+
+
+class PullRequestHeadMovedError(RuntimeError):
+ """The PR head moved after this run was queued."""
+
+
def _is_transient_github_error(error: Exception) -> bool:
if isinstance(error, requests.Timeout):
return True
return isinstance(error, StamphogGitHubError) and error.status_code is not None and error.status_code >= 500
+# GitHub asks for at least a minute before a retry when a limit gives no reset time (a secondary limit).
+# The cap keeps the wait inside the review's time budget.
+_GITHUB_RATE_LIMIT_MIN_RETRY_SECONDS = 60
+_GITHUB_RATE_LIMIT_MAX_RETRY_SECONDS = 5 * 60
+
+
+@contextmanager
+def _wait_out_github_rate_limit() -> Iterator[None]:
+ """Make Temporal's next attempt wait until the GitHub budget allows the call again.
+
+ The budget is GitHub's own rate limit, or the egress limiter that sheds a call before it is sent.
+ The activity retry policy backs off for seconds, and either limit lasts a minute or more, so
+ every attempt fails inside the same window. The last attempt raises the original error, so the
+ failure names the limit and not a retry that will not happen.
+ """
+ try:
+ yield
+ except (GitHubRateLimitError, GitHubEgressBudgetExhausted) as error:
+ if activity.info().attempt >= ACTIVITY_RETRY_POLICY.maximum_attempts:
+ raise
+ retry_after = error.retry_after if isinstance(error, GitHubRateLimitError) else None
+ delay = min(max(retry_after or 0, _GITHUB_RATE_LIMIT_MIN_RETRY_SECONDS), _GITHUB_RATE_LIMIT_MAX_RETRY_SECONDS)
+ raise NonReportableApplicationError(
+ f"GitHub rate limited the call, retrying in {delay}s",
+ type="GitHubRateLimitError",
+ next_retry_delay=timedelta(seconds=delay),
+ ) from error
+
+
@activity.defn
@asyncify
def fetch_review_context(input: StamphogReviewInput) -> dict:
@@ -509,7 +564,8 @@ def fetch_review_context(input: StamphogReviewInput) -> dict:
the PR's author or its file list wait for those two.
"""
try:
- return _fetch_review_context(input)
+ with _wait_out_github_rate_limit():
+ return _fetch_review_context(input)
except (StamphogGitHubError, requests.Timeout) as error:
if _is_transient_github_error(error) and activity.info().attempt < ACTIVITY_RETRY_POLICY.maximum_attempts:
raise TransientGitHubError(str(error)) from error
@@ -829,14 +885,13 @@ def refresh_review_evidence(input: StamphogReviewInput) -> dict:
repo = repo_config.repository
number = run.pull_request.pr_number
client = StamphogGitHubClient(repo_config.installation_id)
- _merge_run_output(
- run,
- {
+ with _wait_out_github_rate_limit():
+ evidence = {
"reviews": client.get_pr_reviews(repo, number),
"discussion": client.get_pr_discussion(repo, number),
"review_threads": client.get_pr_review_threads(repo, number),
- },
- )
+ }
+ _merge_run_output(run, evidence)
return {"refreshed": True}
@@ -1161,6 +1216,8 @@ def _create_review_sandbox(
)
except Exception as exc:
_destroy_sandbox_in_background(sandbox, str(run.id))
+ if isinstance(exc, PullRequestHeadMovedError):
+ _merge_run_output(run, {_HEAD_MOVED_OUTPUT_KEY: True})
raise SandboxPhaseError(f"the sandbox phase failed with {type(exc).__name__}") from exc
return sandbox
@@ -1965,6 +2022,17 @@ def mark_review_failed(input: MarkReviewFailedInput) -> None:
client = StamphogGitHubClient(run.pull_request.repo_config.installation_id)
if run.verdict != ReviewVerdict.APPROVED and run.posted_review_id and run.approval_dismissed_at is None:
_dismiss_orphaned_approval(client, run, input.team_id)
+ # A moved head is a supersession, not a failure. The push that moved it queues the review of the
+ # new head, so the run reports no failure event and posts no failure notice. A notice here would
+ # ask the author to push again for a review that is already queued.
+ if (run.output or {}).get(_HEAD_MOVED_OUTPUT_KEY):
+ now = timezone.now()
+ ReviewRun.objects.for_team(input.team_id).filter(id=run.id).exclude(status__in=TERMINAL_STATUSES).update(
+ status=ReviewRunStatus.SUPERSEDED, completed_at=now, updated_at=now
+ )
+ _remove_own_eyes_reaction(client, run)
+ _sweep_orphan_approvals_at_terminal(client, run, input.team_id)
+ return
# Persist only the first line, truncated: run.error is returned by the serializer to anyone with
# stamphog:read, and raw exception text can embed repository file content (a yaml.YAMLError over
# .stamphog/policy.yml echoes the offending source lines on its continuation lines). Full detail is
@@ -2128,8 +2196,12 @@ def _clone_pr(
result = sandbox.execute(command, timeout_seconds=_step_timeout(deadline, CLONE_STEP_TIMEOUT_SECONDS))
except Exception as exc:
raise RuntimeError(scrub_credentials(str(exc), token, credential.secret)) from None
- if result.exit_code != 0:
- raise RuntimeError(f"{failure_prefix}: {scrub_credentials(result.stderr, token, credential.secret)[:500]}")
+ if result.exit_code == 0:
+ return
+ message = f"{failure_prefix}: {scrub_credentials(result.stderr, token, credential.secret)[:500]}"
+ if result.exit_code == _HEAD_MOVED_EXIT_CODE:
+ raise PullRequestHeadMovedError(message)
+ raise RuntimeError(message)
if step == "head":
fetch_head = (
@@ -2137,7 +2209,7 @@ def _clone_pr(
f"git remote add origin {shlex.quote(repo_url)} && "
f"{auth} fetch --quiet --depth=1 --no-tags origin {shlex.quote(f'pull/{pr_number}/head')} && "
f'fetched=$(git rev-parse FETCH_HEAD) && if [ "$fetched" != {shlex.quote(head_sha)} ]; then '
- f'echo "the PR head is now $fetched" >&2; exit 1; fi'
+ f'echo "the PR head is now $fetched" >&2; exit {_HEAD_MOVED_EXIT_CODE}; fi'
)
_execute_or_raise(fetch_head, f"Failed to fetch the PR head {head_sha}")
diff --git a/products/stamphog/backend/tests/fakes.py b/products/stamphog/backend/tests/fakes.py
index 081ae74e84fc..4a3bb29c8559 100644
--- a/products/stamphog/backend/tests/fakes.py
+++ b/products/stamphog/backend/tests/fakes.py
@@ -643,6 +643,8 @@ def make_fake_sandbox_class(engine_output: str, write_sink: list[tuple[str, byte
executed_commands: list[str] = []
# A test can set this to make the reviewer command fail (sandbox-phase failure coverage).
reviewer_exit_code: int = 0
+ # A test can set this to make the head fetch exit as if the pull ref named a newer commit.
+ head_fetch_exit_code: int = 0
# Every created sandbox by id, so get_by_id hands a later activity the same instance.
instances: dict[str, _FakeSandbox] = {}
@@ -672,6 +674,10 @@ def make_fake_sandbox_class(engine_output: str, write_sink: list[tuple[str, byte
def execute(self, command: str, timeout_seconds: int | None = None) -> FakeExecResult:
type(self).executed_commands.append(command)
+ if "FETCH_HEAD" in command:
+ return FakeExecResult(
+ stdout="", stderr="the PR head is now sha-newer", exit_code=self.head_fetch_exit_code
+ )
if "review_local.py" not in command:
return FakeExecResult(stdout="", stderr="", exit_code=0)
return FakeExecResult(stdout=engine_output, stderr="", exit_code=self.reviewer_exit_code)
diff --git a/products/stamphog/backend/tests/test_integration.py b/products/stamphog/backend/tests/test_integration.py
index d58609ab874f..a8dcc2e804da 100644
--- a/products/stamphog/backend/tests/test_integration.py
+++ b/products/stamphog/backend/tests/test_integration.py
@@ -15,7 +15,7 @@ from django.utils import timezone
import requests
from celery.exceptions import Retry
-from posthog.egress.github.transport import GitHubRateLimitError
+from posthog.egress.github.transport import GitHubEgressBudgetExhausted, GitHubRateLimitError
from posthog.models import OAuthAccessToken, Team
from posthog.models.instance_setting import override_instance_config
from posthog.models.integration import Integration
@@ -517,6 +517,29 @@ def test_failure_once_the_sandbox_exists_is_not_retried(team, stamphog_chain: St
assert SandboxPhaseError.__name__ in (SANDBOX_RETRY_POLICY.non_retryable_error_types or [])
[email protected]_db(databases=PRODUCT_DATABASES)
+def test_a_head_that_moved_before_the_fetch_supersedes_the_run_quietly(team, stamphog_chain: StamphogChain) -> None:
+ # The push that moved the head queues the review of the new head. A failure notice would ask the
+ # author to push again, and a failure event would count a routine race as breakage.
+ _repo_config(team.id)
+ recorder = stamphog_chain.recorder
+ author, head_sha = "devex-dev", "sha119a"
+ recorder.register_pr(REPO, 119, _pr_object(119, author, head_sha), _pr_files())
+ recorder.policy_files[".stamphog/policy.yml"] = "version: 1\n"
+ stamphog_chain.sandbox_class.head_fetch_exit_code = activities._HEAD_MOVED_EXIT_CODE
+
+ capture_fn = MagicMock()
+ with patch("products.stamphog.backend.temporal.activities.ph_scoped_capture") as mock_capture_cm:
+ mock_capture_cm.return_value.__enter__.return_value = capture_fn
+ mock_capture_cm.return_value.__exit__.return_value = False
+ stamphog_chain.post_webhook(_opened_event(119, author, head_sha), delivery_id=str(uuid.uuid4()))
+
+ run = ReviewRun.objects.for_team(team.id).latest("created_at")
+ assert run.status == ReviewRunStatus.SUPERSEDED
+ assert [w for w in recorder.github_writes if w["kind"] == "comment_review"] == []
+ assert "stamphog_review_failed" not in [c.kwargs.get("event") for c in capture_fn.call_args_list]
+
+
@pytest.mark.django_db(databases=PRODUCT_DATABASES)
def test_a_second_attempt_never_provisions_a_second_sandbox(team, stamphog_chain: StamphogChain) -> None:
# Temporal enforces the start-to-close timeout itself and retries a lost worker, and neither path
@@ -1370,8 +1393,9 @@ def test_fetch_review_context_carries_inline_review_threads(team, stamphog_chain
recorder = stamphog_chain.recorder
head_sha = "sha132a"
recorder.register_pr(REPO, 132, _pr_object(132, "devex-dev", head_sha), _pr_files())
+ # The NUL byte stands for PR text that Postgres refuses in jsonb, which used to fail every review of the PR.
recorder.review_threads[(REPO, 132)] = [
- fakes.review_thread_node(path="src/util.py", comments=[("maintainer", "do not merge")], is_resolved=False)
+ fakes.review_thread_node(path="src/util.py", comments=[("maintainer", "do not\x00 merge")], is_resolved=False)
]
pull_request = PullRequest.objects.for_team(team.id).create(
team_id=team.id, repo_config=repo_config, pr_number=132, author_login="devex-dev"
@@ -1397,19 +1421,42 @@ def test_fetch_review_context_carries_inline_review_threads(team, stamphog_chain
@pytest.mark.parametrize(
- "error,attempt,expect_reported",
+ "error,attempt,expect_reported,retry_delay",
[
- pytest.param(StamphogGitHubError("bad gateway", status_code=502), 1, False, id="5xx_first_attempt"),
- pytest.param(requests.Timeout("read timed out"), 2, False, id="timeout_second_attempt"),
- pytest.param(StamphogGitHubError("bad gateway", status_code=502), 3, True, id="5xx_final_attempt"),
- pytest.param(requests.Timeout("read timed out"), 3, True, id="timeout_final_attempt"),
- pytest.param(StamphogGitHubError("not found", status_code=404), 1, True, id="4xx_first_attempt"),
- pytest.param(StamphogGitHubError("truncated list"), 1, True, id="no_status_first_attempt"),
+ pytest.param(StamphogGitHubError("bad gateway", status_code=502), 1, False, None, id="5xx_first_attempt"),
+ pytest.param(requests.Timeout("read timed out"), 2, False, None, id="timeout_second_attempt"),
+ pytest.param(StamphogGitHubError("bad gateway", status_code=502), 3, True, None, id="5xx_final_attempt"),
+ pytest.param(requests.Timeout("read timed out"), 3, True, None, id="timeout_final_attempt"),
+ pytest.param(StamphogGitHubError("not found", status_code=404), 1, True, None, id="4xx_first_attempt"),
+ pytest.param(StamphogGitHubError("truncated list"), 1, True, None, id="no_status_first_attempt"),
+ pytest.param(
+ GitHubRateLimitError("secondary rate limit"), 1, False, timedelta(seconds=60), id="rate_limit_no_reset"
+ ),
+ pytest.param(
+ GitHubRateLimitError("rate limit", retry_after=120),
+ 2,
+ False,
+ timedelta(seconds=120),
+ id="rate_limit_retry_after",
+ ),
+ pytest.param(GitHubRateLimitError("rate limit", retry_after=120), 3, True, None, id="rate_limit_final_attempt"),
+ pytest.param(
+ GitHubEgressBudgetExhausted("shed", scope=INSTALLATION_ID),
+ 1,
+ False,
+ timedelta(seconds=60),
+ id="egress_budget_shed",
+ ),
],
)
@pytest.mark.django_db(databases=PRODUCT_DATABASES)
def test_fetch_review_context_reports_transient_github_errors_only_on_final_attempt(
- team, stamphog_chain: StamphogChain, error: Exception, attempt: int, expect_reported: bool
+ team,
+ stamphog_chain: StamphogChain,
+ error: Exception,
+ attempt: int,
+ expect_reported: bool,
+ retry_delay: timedelta | None,
) -> None:
repo_config = _repo_config(team.id)
head_sha = "sha134a"
@@ -1431,6 +1478,7 @@ def test_fetch_review_context_reports_transient_github_errors_only_on_final_atte
# The activity fails either way, so a review never runs with incomplete context.
assert is_expected_activity_failure(raised.value) is not expect_reported
assert raised.value is error or raised.value.__cause__ is error
+ assert getattr(raised.value, "next_retry_delay", None) == retry_delay
@pytest.mark.parametrize(
diff --git a/products/stamphog/backend/tests/test_policy_defaults.py b/products/stamphog/backend/tests/test_policy_defaults.py
index ac27ad37dabb..069343e4789c 100644
--- a/products/stamphog/backend/tests/test_policy_defaults.py
+++ b/products/stamphog/backend/tests/test_policy_defaults.py
@@ -20,6 +20,7 @@ import policy # noqa: E402
from products.stamphog.backend.logic.engine_pregate import run_engine_pregate # noqa: E402
from products.stamphog.backend.temporal import activities # noqa: E402
from products.stamphog.backend.temporal.activities import ( # noqa: E402
+ PullRequestHeadMovedError,
_clone_pr,
_effective_policy_files,
_prefetch_review_blobs,
@@ -252,7 +253,7 @@ def test_shallow_clone_holds_every_object_the_pr_diff_reads(
if head_moved:
# The pull ref now names a newer commit than the one this run was queued for.
- with pytest.raises(RuntimeError, match="the PR head is now"):
+ with pytest.raises(PullRequestHeadMovedError, match="the PR head is now"):
_clone_pr(sandbox, "acme/widgets", "", base_tip, 7, "tok", deadline, step="head") # type: ignore[arg-type]
return