:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(stamphog): stop failing runs on nul bytes, head races and rate limits (#123276)
Julian Bez7e3eebb7c31f
diff --git a/products/stamphog/backend/temporal/activities.py b/products/stamphog/backend/temporal/activities.pyindex 22a044f18ddd..95efeba295a8 100644--- a/products/stamphog/backend/temporal/activities.py+++ b/products/stamphog/backend/temporal/activities.py@@ -51,10 +51,11 @@ from prometheus_client import Counterfrom temporalio import activityfrom posthog.dataclasses import frozen+from posthog.egress.github.transport import GitHubEgressBudgetExhausted, GitHubRateLimitErrorfrom posthog.llm.gateway_client import AIGatewayConfig, resolve_ai_gateway_configfrom posthog.models import Userfrom posthog.ph_client import ph_background_capture, ph_scoped_capture-from posthog.temporal.common.errors import NonReportableError+from posthog.temporal.common.errors import NonReportableApplicationError, NonReportableErrorfrom posthog.temporal.common.utils import asyncifyfrom products.stamphog.backend.facade.contracts import StamphogGitHubError@@ -170,11 +171,26 @@ def _load_run(input: StamphogReviewInput) -> ReviewRun:)+def _without_nul(value: Any) -> Any:+ """Drop NUL characters from every string in ``value``.++ The context holds PR text, and Postgres refuses ``\\u0000`` in jsonb. One NUL in a PR body or+ comment fails the whole write, and every retry and re-review of that PR fails the same way.+ """+ if isinstance(value, str):+ return value.replace("\x00", "")+ if isinstance(value, dict):+ return {_without_nul(key): _without_nul(item) for key, item in value.items()}+ if isinstance(value, list | tuple):+ return [_without_nul(item) for item in value]+ return value++def _output_merge(updates: dict[str, Any]) -> Func:"""Postgres ``output || updates``: a top-level key merge that the database performs."""return Func(Coalesce(F("output"), Cast(Value("{}"), output_field=JSONField())),- Cast(Value(json.dumps(updates, cls=DjangoJSONEncoder)), output_field=JSONField()),+ Cast(Value(json.dumps(_without_nul(updates), cls=DjangoJSONEncoder)), output_field=JSONField()),template="%(expressions)s",arg_joiner=" || ",output_field=JSONField(),@@ -192,7 +208,7 @@ def _merge_run_output(run: ReviewRun, updates: dict[str, Any]) -> None:ReviewRun.objects.for_team(run.team_id).using(router.db_for_write(ReviewRun)).filter(id=run.id).update(output=_output_merge(updates), updated_at=timezone.now())- run.output = {**(run.output or {}), **updates}+ run.output = {**(run.output or {}), **_without_nul(updates)}# aio_ continues the series the Action-era runs emitted; the engine blob carries the same word.@@ -494,12 +510,51 @@ class TransientGitHubError(NonReportableError):so a review never runs with incomplete context, but only the final attempt reaches error tracking."""+# The head-fetch command exits with this code when the pull ref names a newer commit. Git does not+# use it, so a moved head stays apart from a git failure.+_HEAD_MOVED_EXIT_CODE = 75+_HEAD_MOVED_OUTPUT_KEY = "head_moved"+++class PullRequestHeadMovedError(RuntimeError):+ """The PR head moved after this run was queued."""++def _is_transient_github_error(error: Exception) -> bool:if isinstance(error, requests.Timeout):return Truereturn isinstance(error, StamphogGitHubError) and error.status_code is not None and error.status_code >= 500+# GitHub asks for at least a minute before a retry when a limit gives no reset time (a secondary limit).+# The cap keeps the wait inside the review's time budget.+_GITHUB_RATE_LIMIT_MIN_RETRY_SECONDS = 60+_GITHUB_RATE_LIMIT_MAX_RETRY_SECONDS = 5 * 60+++@contextmanager+def _wait_out_github_rate_limit() -> Iterator[None]:+ """Make Temporal's next attempt wait until the GitHub budget allows the call again.++ The budget is GitHub's own rate limit, or the egress limiter that sheds a call before it is sent.+ The activity retry policy backs off for seconds, and either limit lasts a minute or more, so+ every attempt fails inside the same window. The last attempt raises the original error, so the+ failure names the limit and not a retry that will not happen.+ """+ try:+ yield+ except (GitHubRateLimitError, GitHubEgressBudgetExhausted) as error:+ if activity.info().attempt >= ACTIVITY_RETRY_POLICY.maximum_attempts:+ raise+ retry_after = error.retry_after if isinstance(error, GitHubRateLimitError) else None+ delay = min(max(retry_after or 0, _GITHUB_RATE_LIMIT_MIN_RETRY_SECONDS), _GITHUB_RATE_LIMIT_MAX_RETRY_SECONDS)+ raise NonReportableApplicationError(+ f"GitHub rate limited the call, retrying in {delay}s",+ type="GitHubRateLimitError",+ next_retry_delay=timedelta(seconds=delay),+ ) from error++@activity.defn@asyncifydef fetch_review_context(input: StamphogReviewInput) -> dict:@@ -509,7 +564,8 @@ def fetch_review_context(input: StamphogReviewInput) -> dict:the PR's author or its file list wait for those two."""try:- return _fetch_review_context(input)+ with _wait_out_github_rate_limit():+ return _fetch_review_context(input)except (StamphogGitHubError, requests.Timeout) as error:if _is_transient_github_error(error) and activity.info().attempt < ACTIVITY_RETRY_POLICY.maximum_attempts:raise TransientGitHubError(str(error)) from error@@ -829,14 +885,13 @@ def refresh_review_evidence(input: StamphogReviewInput) -> dict:repo = repo_config.repositorynumber = run.pull_request.pr_numberclient = StamphogGitHubClient(repo_config.installation_id)- _merge_run_output(- run,- {+ with _wait_out_github_rate_limit():+ evidence = {"reviews": client.get_pr_reviews(repo, number),"discussion": client.get_pr_discussion(repo, number),"review_threads": client.get_pr_review_threads(repo, number),- },- )+ }+ _merge_run_output(run, evidence)return {"refreshed": True}@@ -1161,6 +1216,8 @@ def _create_review_sandbox()except Exception as exc:_destroy_sandbox_in_background(sandbox, str(run.id))+ if isinstance(exc, PullRequestHeadMovedError):+ _merge_run_output(run, {_HEAD_MOVED_OUTPUT_KEY: True})raise SandboxPhaseError(f"the sandbox phase failed with {type(exc).__name__}") from excreturn sandbox@@ -1965,6 +2022,17 @@ def mark_review_failed(input: MarkReviewFailedInput) -> None:client = StamphogGitHubClient(run.pull_request.repo_config.installation_id)if run.verdict != ReviewVerdict.APPROVED and run.posted_review_id and run.approval_dismissed_at is None:_dismiss_orphaned_approval(client, run, input.team_id)+ # A moved head is a supersession, not a failure. The push that moved it queues the review of the+ # new head, so the run reports no failure event and posts no failure notice. A notice here would+ # ask the author to push again for a review that is already queued.+ if (run.output or {}).get(_HEAD_MOVED_OUTPUT_KEY):+ now = timezone.now()+ ReviewRun.objects.for_team(input.team_id).filter(id=run.id).exclude(status__in=TERMINAL_STATUSES).update(+ status=ReviewRunStatus.SUPERSEDED, completed_at=now, updated_at=now+ )+ _remove_own_eyes_reaction(client, run)+ _sweep_orphan_approvals_at_terminal(client, run, input.team_id)+ return# Persist only the first line, truncated: run.error is returned by the serializer to anyone with# stamphog:read, and raw exception text can embed repository file content (a yaml.YAMLError over# .stamphog/policy.yml echoes the offending source lines on its continuation lines). Full detail is@@ -2128,8 +2196,12 @@ def _clone_pr(result = sandbox.execute(command, timeout_seconds=_step_timeout(deadline, CLONE_STEP_TIMEOUT_SECONDS))except Exception as exc:raise RuntimeError(scrub_credentials(str(exc), token, credential.secret)) from None- if result.exit_code != 0:- raise RuntimeError(f"{failure_prefix}: {scrub_credentials(result.stderr, token, credential.secret)[:500]}")+ if result.exit_code == 0:+ return+ message = f"{failure_prefix}: {scrub_credentials(result.stderr, token, credential.secret)[:500]}"+ if result.exit_code == _HEAD_MOVED_EXIT_CODE:+ raise PullRequestHeadMovedError(message)+ raise RuntimeError(message)if step == "head":fetch_head = (@@ -2137,7 +2209,7 @@ def _clone_pr(f"git remote add origin {shlex.quote(repo_url)} && "f"{auth} fetch --quiet --depth=1 --no-tags origin {shlex.quote(f'pull/{pr_number}/head')} && "f'fetched=$(git rev-parse FETCH_HEAD) && if [ "$fetched" != {shlex.quote(head_sha)} ]; then '- f'echo "the PR head is now $fetched" >&2; exit 1; fi'+ f'echo "the PR head is now $fetched" >&2; exit {_HEAD_MOVED_EXIT_CODE}; fi')_execute_or_raise(fetch_head, f"Failed to fetch the PR head {head_sha}")diff --git a/products/stamphog/backend/tests/fakes.py b/products/stamphog/backend/tests/fakes.pyindex 081ae74e84fc..4a3bb29c8559 100644--- a/products/stamphog/backend/tests/fakes.py+++ b/products/stamphog/backend/tests/fakes.py@@ -643,6 +643,8 @@ def make_fake_sandbox_class(engine_output: str, write_sink: list[tuple[str, byteexecuted_commands: list[str] = []# A test can set this to make the reviewer command fail (sandbox-phase failure coverage).reviewer_exit_code: int = 0+ # A test can set this to make the head fetch exit as if the pull ref named a newer commit.+ head_fetch_exit_code: int = 0# Every created sandbox by id, so get_by_id hands a later activity the same instance.instances: dict[str, _FakeSandbox] = {}@@ -672,6 +674,10 @@ def make_fake_sandbox_class(engine_output: str, write_sink: list[tuple[str, bytedef execute(self, command: str, timeout_seconds: int | None = None) -> FakeExecResult:type(self).executed_commands.append(command)+ if "FETCH_HEAD" in command:+ return FakeExecResult(+ stdout="", stderr="the PR head is now sha-newer", exit_code=self.head_fetch_exit_code+ )if "review_local.py" not in command:return FakeExecResult(stdout="", stderr="", exit_code=0)return FakeExecResult(stdout=engine_output, stderr="", exit_code=self.reviewer_exit_code)diff --git a/products/stamphog/backend/tests/test_integration.py b/products/stamphog/backend/tests/test_integration.pyindex d58609ab874f..a8dcc2e804da 100644--- a/products/stamphog/backend/tests/test_integration.py+++ b/products/stamphog/backend/tests/test_integration.py@@ -15,7 +15,7 @@ from django.utils import timezoneimport requestsfrom celery.exceptions import Retry-from posthog.egress.github.transport import GitHubRateLimitError+from posthog.egress.github.transport import GitHubEgressBudgetExhausted, GitHubRateLimitErrorfrom posthog.models import OAuthAccessToken, Teamfrom posthog.models.instance_setting import override_instance_configfrom posthog.models.integration import Integration@@ -517,6 +517,29 @@ def test_failure_once_the_sandbox_exists_is_not_retried(team, stamphog_chain: Stassert SandboxPhaseError.__name__ in (SANDBOX_RETRY_POLICY.non_retryable_error_types or [])[email protected]_db(databases=PRODUCT_DATABASES)+def test_a_head_that_moved_before_the_fetch_supersedes_the_run_quietly(team, stamphog_chain: StamphogChain) -> None:+ # The push that moved the head queues the review of the new head. A failure notice would ask the+ # author to push again, and a failure event would count a routine race as breakage.+ _repo_config(team.id)+ recorder = stamphog_chain.recorder+ author, head_sha = "devex-dev", "sha119a"+ recorder.register_pr(REPO, 119, _pr_object(119, author, head_sha), _pr_files())+ recorder.policy_files[".stamphog/policy.yml"] = "version: 1\n"+ stamphog_chain.sandbox_class.head_fetch_exit_code = activities._HEAD_MOVED_EXIT_CODE++ capture_fn = MagicMock()+ with patch("products.stamphog.backend.temporal.activities.ph_scoped_capture") as mock_capture_cm:+ mock_capture_cm.return_value.__enter__.return_value = capture_fn+ mock_capture_cm.return_value.__exit__.return_value = False+ stamphog_chain.post_webhook(_opened_event(119, author, head_sha), delivery_id=str(uuid.uuid4()))++ run = ReviewRun.objects.for_team(team.id).latest("created_at")+ assert run.status == ReviewRunStatus.SUPERSEDED+ assert [w for w in recorder.github_writes if w["kind"] == "comment_review"] == []+ assert "stamphog_review_failed" not in [c.kwargs.get("event") for c in capture_fn.call_args_list]++@pytest.mark.django_db(databases=PRODUCT_DATABASES)def test_a_second_attempt_never_provisions_a_second_sandbox(team, stamphog_chain: StamphogChain) -> None:# Temporal enforces the start-to-close timeout itself and retries a lost worker, and neither path@@ -1370,8 +1393,9 @@ def test_fetch_review_context_carries_inline_review_threads(team, stamphog_chainrecorder = stamphog_chain.recorderhead_sha = "sha132a"recorder.register_pr(REPO, 132, _pr_object(132, "devex-dev", head_sha), _pr_files())+ # The NUL byte stands for PR text that Postgres refuses in jsonb, which used to fail every review of the PR.recorder.review_threads[(REPO, 132)] = [- fakes.review_thread_node(path="src/util.py", comments=[("maintainer", "do not merge")], is_resolved=False)+ fakes.review_thread_node(path="src/util.py", comments=[("maintainer", "do not\x00 merge")], is_resolved=False)]pull_request = PullRequest.objects.for_team(team.id).create(team_id=team.id, repo_config=repo_config, pr_number=132, author_login="devex-dev"@@ -1397,19 +1421,42 @@ def test_fetch_review_context_carries_inline_review_threads(team, stamphog_chain@pytest.mark.parametrize(- "error,attempt,expect_reported",+ "error,attempt,expect_reported,retry_delay",[- pytest.param(StamphogGitHubError("bad gateway", status_code=502), 1, False, id="5xx_first_attempt"),- pytest.param(requests.Timeout("read timed out"), 2, False, id="timeout_second_attempt"),- pytest.param(StamphogGitHubError("bad gateway", status_code=502), 3, True, id="5xx_final_attempt"),- pytest.param(requests.Timeout("read timed out"), 3, True, id="timeout_final_attempt"),- pytest.param(StamphogGitHubError("not found", status_code=404), 1, True, id="4xx_first_attempt"),- pytest.param(StamphogGitHubError("truncated list"), 1, True, id="no_status_first_attempt"),+ pytest.param(StamphogGitHubError("bad gateway", status_code=502), 1, False, None, id="5xx_first_attempt"),+ pytest.param(requests.Timeout("read timed out"), 2, False, None, id="timeout_second_attempt"),+ pytest.param(StamphogGitHubError("bad gateway", status_code=502), 3, True, None, id="5xx_final_attempt"),+ pytest.param(requests.Timeout("read timed out"), 3, True, None, id="timeout_final_attempt"),+ pytest.param(StamphogGitHubError("not found", status_code=404), 1, True, None, id="4xx_first_attempt"),+ pytest.param(StamphogGitHubError("truncated list"), 1, True, None, id="no_status_first_attempt"),+ pytest.param(+ GitHubRateLimitError("secondary rate limit"), 1, False, timedelta(seconds=60), id="rate_limit_no_reset"+ ),+ pytest.param(+ GitHubRateLimitError("rate limit", retry_after=120),+ 2,+ False,+ timedelta(seconds=120),+ id="rate_limit_retry_after",+ ),+ pytest.param(GitHubRateLimitError("rate limit", retry_after=120), 3, True, None, id="rate_limit_final_attempt"),+ pytest.param(+ GitHubEgressBudgetExhausted("shed", scope=INSTALLATION_ID),+ 1,+ False,+ timedelta(seconds=60),+ id="egress_budget_shed",+ ),],)@pytest.mark.django_db(databases=PRODUCT_DATABASES)def test_fetch_review_context_reports_transient_github_errors_only_on_final_attempt(- team, stamphog_chain: StamphogChain, error: Exception, attempt: int, expect_reported: bool+ team,+ stamphog_chain: StamphogChain,+ error: Exception,+ attempt: int,+ expect_reported: bool,+ retry_delay: timedelta | None,) -> None:repo_config = _repo_config(team.id)head_sha = "sha134a"@@ -1431,6 +1478,7 @@ def test_fetch_review_context_reports_transient_github_errors_only_on_final_atte# The activity fails either way, so a review never runs with incomplete context.assert is_expected_activity_failure(raised.value) is not expect_reportedassert raised.value is error or raised.value.__cause__ is error+ assert getattr(raised.value, "next_retry_delay", None) == retry_delay@pytest.mark.parametrize(diff --git a/products/stamphog/backend/tests/test_policy_defaults.py b/products/stamphog/backend/tests/test_policy_defaults.pyindex ac27ad37dabb..069343e4789c 100644--- a/products/stamphog/backend/tests/test_policy_defaults.py+++ b/products/stamphog/backend/tests/test_policy_defaults.py@@ -20,6 +20,7 @@ import policy # noqa: E402from products.stamphog.backend.logic.engine_pregate import run_engine_pregate # noqa: E402from products.stamphog.backend.temporal import activities # noqa: E402from products.stamphog.backend.temporal.activities import ( # noqa: E402+ PullRequestHeadMovedError,_clone_pr,_effective_policy_files,_prefetch_review_blobs,@@ -252,7 +253,7 @@ def test_shallow_clone_holds_every_object_the_pr_diff_reads(if head_moved:# The pull ref now names a newer commit than the one this run was queued for.- with pytest.raises(RuntimeError, match="the PR head is now"):+ with pytest.raises(PullRequestHeadMovedError, match="the PR head is now"):_clone_pr(sandbox, "acme/widgets", "", base_tip, 7, "tok", deadline, step="head") # type: ignore[arg-type]return