:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(tasks): extend clone retry backoff and redact diagnostics (#123904)
Georgiy Tarasov514b2c621bc1
diff --git a/docs/internal/sandboxes-setup-guide.md b/docs/internal/sandboxes-setup-guide.mdindex b9af669f174e..740b7a6b3684 100644--- a/docs/internal/sandboxes-setup-guide.md+++ b/docs/internal/sandboxes-setup-guide.md@@ -112,6 +112,8 @@ orchestrates these activities:The activities live in`products/tasks/backend/temporal/process_task/activities/`.+Repository clone activities allow six attempts in total. Failed git commands, including RPC errors and repository-not-found responses, retry after 1, 2, 16, 32, and 64 seconds. Explicit authentication failures stop immediately. Each attempt keeps the five-minute timeout and reuses the sandbox and credential. Activity timeouts and infrastructure exceptions use Temporal's default exponential backoff within the same six-attempt limit. The `repository_clone_failed` worker log records the run, repository, sandbox, clone mode, exit code, and whether a GitHub token was supplied. Its stdout, stderr, and execution error fields redact the supplied token and sandbox command credentials before truncation to 500 characters each; the same diagnostics are attached to the Temporal error.+The agent's `finish` tool marks a TaskRun terminal and triggers sandbox cleanup.ReviewHog and scout suggestion tasks do not expose it because their callers own session completion.ReviewHog receives and validates each turn's JSON before ending the session, and validation can use multiple turns in the same sandbox.diff --git a/products/tasks/backend/temporal/process_task/activities/provision_sandbox.py b/products/tasks/backend/temporal/process_task/activities/provision_sandbox.pyindex 3ffeae5e54e8..929434d1dffa 100644--- a/products/tasks/backend/temporal/process_task/activities/provision_sandbox.py+++ b/products/tasks/backend/temporal/process_task/activities/provision_sandbox.py@@ -64,6 +64,7 @@ from products.tasks.backend.logic.services.sandbox import (needs_full_history,parse_requested_sandbox_template,pinned_agent_version,+ redact_sandbox_command,sandbox_repo_path,workload_for_origin_product,)@@ -113,6 +114,7 @@ from .get_task_processing_context import TaskProcessingContextlogger = logging.getLogger(__name__)SANDBOX_CREATION_CANCELLATION_WAIT_SECONDS = 10SANDBOX_CREATION_HEARTBEAT_SECONDS = 1+REPOSITORY_CLONE_RETRY_DELAYS_SECONDS = (1, 2, 16, 32, 64)NETWORK_RESTRICTED_AGENT_ENV = {"CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC": "1",@@ -1149,6 +1151,12 @@ async def create_sandbox_for_repository(input: CreateSandboxForRepositoryInput)raise cancellation+def _redact_clone_output(output: str, github_token: str | None) -> str:+ if github_token:+ output = output.replace(github_token, "<redacted>")+ return redact_sandbox_command(output)[:500]++@activity.defn@asyncify@posthoganalytics.scoped(fresh=True, capture_exceptions=False)@@ -1196,31 +1204,48 @@ def clone_repository_in_sandbox(input: CloneRepositoryInSandboxInput) -> CloneReblobless=blobless_clone,)- if _is_repository_access_denied_clone_error(clone_result):- raise RepositoryAccessError(- f"Could not clone {input.repository}: GitHub denied access. Check that the PostHog "- "GitHub app is installed on this repository's account and can read this repository.",- {- "repository": input.repository,- "sandbox_id": input.sandbox_id,- "exit_code": clone_result.exit_code,- "has_github_token": bool(input.github_token),+ if clone_result.exit_code != 0:+ stderr = _redact_clone_output(clone_result.stderr, input.github_token)+ stdout = _redact_clone_output(clone_result.stdout, input.github_token)+ execution_error = (+ _redact_clone_output(clone_result.error, input.github_token) if clone_result.error else None+ )+ error_context = {+ "repository": input.repository,+ "sandbox_id": input.sandbox_id,+ "exit_code": clone_result.exit_code,+ "has_github_token": bool(input.github_token),+ "stderr": stderr,+ "stdout": stdout,+ "error": execution_error,+ }+ logger.warning(+ "repository_clone_failed",+ extra={+ **ctx.to_log_context(),+ **error_context,+ "blobless_clone": blobless_clone,+ "shallow_clone": input.shallow_clone,},)+ if _is_repository_access_denied_clone_error(clone_result):+ raise RepositoryAccessError(+ f"Could not clone {input.repository}: GitHub denied access. Check that the PostHog "+ "GitHub app is installed on this repository's account and can read this repository.",+ error_context,+ )- if clone_result.exit_code != 0:- error_output = clone_result.stderr or clone_result.stdout or clone_result.error or "No output captured"+ error_output = stderr or stdout or execution_error or "No output captured"+ attempt = activity.info().attemptraise RepositoryCloneError(f"Could not clone {input.repository} (git exited with code {clone_result.exit_code})",- {- "repository": input.repository,- "sandbox_id": input.sandbox_id,- "exit_code": clone_result.exit_code,- "stderr": clone_result.stderr[:500],- "stdout": clone_result.stdout[:500],- "error": clone_result.error,- },+ error_context,cause=RuntimeError(error_output[:200]),+ next_retry_delay=(+ timedelta(seconds=REPOSITORY_CLONE_RETRY_DELAYS_SECONDS[attempt - 1])+ if attempt <= len(REPOSITORY_CLONE_RETRY_DELAYS_SECONDS)+ else None+ ),)# A fresh single-repository run checks its requested branch out in the next@@ -1240,11 +1265,8 @@ def _is_repository_access_denied_clone_error(result: ExecutionResult) -> bool:# Git asks for a username when GitHub answers 401, which is how GitHub hides a private# repository from a request without a usable token.output = f"{result.stdout}\n{result.stderr}".casefold()- return (- "could not read username" in output- or "authentication failed for" in output- or "remote: repository not found" in output- )+ # A repository-not-found response can be transient, so leave it to the bounded clone retry policy.+ return "could not read username" in output or "authentication failed for" in outputdef _is_missing_remote_branch_clone_error(result: ExecutionResult) -> bool:diff --git a/products/tasks/backend/temporal/process_task/activities/tests/test_provision_sandbox.py b/products/tasks/backend/temporal/process_task/activities/tests/test_provision_sandbox.pyindex 4d6a64e263de..5a6519536f58 100644--- a/products/tasks/backend/temporal/process_task/activities/tests/test_provision_sandbox.py+++ b/products/tasks/backend/temporal/process_task/activities/tests/test_provision_sandbox.py@@ -4,6 +4,7 @@ import socketimport asyncioimport subprocessimport dataclasses+from datetime import timedeltafrom pathlib import Pathimport pytest@@ -595,6 +596,7 @@ def test_clone_failure_records_failed_latency_and_captures_command_result(mocker"repository": "posthog/posthog","sandbox_id": "sandbox-id","exit_code": 124,+ "has_github_token": True,"stderr": "","stdout": "clone output","error": "execution stopped",@@ -614,25 +616,47 @@ def test_clone_failure_records_failed_latency_and_captures_command_result(mocker@pytest.mark.parametrize(- "stderr, expected_error",+ "stderr, expected_error, attempt, retry_delay",[("Cloning into 'repo'...\nfatal: could not read Username for 'https://github.com': No such device or address\n",RepositoryAccessError,+ 1,+ None,),("Cloning into 'repo'...\nremote: Repository not found.\nfatal: repository 'https://github.com/acme/repo.git/' not found\n",+ RepositoryCloneError,+ 1,+ 1,+ ),+ (+ "fatal: Authentication failed for 'https://github.com/example/repo.git/'\n",RepositoryAccessError,+ 3,+ None,),("Cloning into 'repo'...\nfatal: unable to access 'https://github.com/acme/repo.git/': Could not resolve host: github.com\n",RepositoryCloneError,+ 1,+ 1,),+ *[+ (+ "error: RPC failed; HTTP 404",+ RepositoryCloneError,+ attempt,+ delay,+ )+ for attempt, delay in [(1, 1), (2, 2), (3, 16), (4, 32), (5, 64), (6, None)]+ ],],)def test_clone_failure_stops_retries_only_when_github_denies_access(- mocker, activity_environment, stderr, expected_error+ mocker, activity_environment, stderr, expected_error, attempt, retry_delay):+ activity_environment.info = dataclasses.replace(activity_environment.info, attempt=attempt)context = TaskProcessingContext(task_id="task-id",run_id="run-id",@@ -668,6 +692,73 @@ def test_clone_failure_stops_retries_only_when_github_denies_access(assert error.value.non_retryable is access_deniedassert is_expected_activity_failure(error.value) is access_deniedassert capture_exception.called is not access_denied+ assert error.value.next_retry_delay == (timedelta(seconds=retry_delay) if retry_delay is not None else None)+++ "failure, expected_error",+ [+ ("fatal: Authentication failed for", RepositoryAccessError),+ ("remote: Repository not found.", RepositoryCloneError),+ ],+)+def test_clone_failure_diagnostics_redact_credentials_before_truncation(+ mocker, activity_environment, failure, expected_error+) -> None:+ token = "fake-clone-credential-for-test"+ context = TaskProcessingContext(+ task_id="task-id",+ run_id="run-id",+ team_id=1,+ team_uuid="team-uuid",+ organization_id="organization-id",+ github_integration_id=123,+ repository="example/repo",+ distinct_id="distinct-id",+ state={},+ )+ sandbox = mocker.Mock()+ sandbox.clone_repository.return_value = ExecutionResult(+ stdout="x" * 490 + token,+ stderr=f"{failure} https://x-access-token:{token}@github.com/example/repo.git/",+ exit_code=128,+ error="https://x-access-token:[email protected]/example/repo.git/ " + "x" * 600,+ )+ mocker.patch(+ "products.tasks.backend.temporal.process_task.activities.provision_sandbox.get_sandbox_class_for_sandbox_id",+ return_value=mocker.Mock(get_by_id=mocker.Mock(return_value=sandbox)),+ )+ capture_exception = mocker.patch("products.tasks.backend.exceptions.capture_exception")+ warning = mocker.patch("products.tasks.backend.temporal.process_task.activities.provision_sandbox.logger.warning")++ with pytest.raises(expected_error) as error:+ async_to_sync(activity_environment.run)(+ clone_repository_in_sandbox,+ CloneRepositoryInSandboxInput(+ context=context,+ sandbox_id="sandbox-id",+ repository="example/repo",+ github_token=token,+ shallow_clone=True,+ ),+ )++ assert warning.call_args.args == ("repository_clone_failed",)+ diagnostic = warning.call_args.kwargs["extra"]+ for key in ("stdout", "stderr", "error"):+ assert diagnostic[key] == error.value.context[key]+ assert len(error.value.context[key]) <= 500+ assert diagnostic["run_id"] == "run-id"+ assert diagnostic["shallow_clone"] is True+ assert diagnostic["has_github_token"] is True+ assert error.value.context["stdout"] == "x" * 490 + "<redacted>"+ assert error.value.context["stderr"] == (+ f"{failure} https://x-access-token:<redacted>@github.com/example/repo.git/"+ )+ assert error.value.context["error"].startswith("https://x-access-token:<redacted>@github.com/example/repo.git/ ")+ diagnostics = str(error.value.details) + str(capture_exception.call_args_list) + str(diagnostic)+ assert token not in diagnostics+ assert "fake-other-credential" not in diagnosticsdef _prepared_for_create() -> PrepareSandboxForRepositoryOutput:diff --git a/products/tasks/backend/temporal/process_task/tests/test_workflow.py b/products/tasks/backend/temporal/process_task/tests/test_workflow.pyindex fb2300e8e0f6..ec77b677f699 100644--- a/products/tasks/backend/temporal/process_task/tests/test_workflow.py+++ b/products/tasks/backend/temporal/process_task/tests/test_workflow.py@@ -3285,7 +3285,7 @@ class TestProcessTaskWorkflowUnit:assert cloned == ["posthog/posthog", "posthog/code"]for repository in ("posthog/posthog", "posthog/code"):assert clone_options[repository]["start_to_close_timeout"] == timedelta(minutes=5)- assert clone_options[repository]["retry_policy"].maximum_attempts == 3+ assert clone_options[repository]["retry_policy"].maximum_attempts == 6assert result.clone_ms is Noneasync def test_overlap_releases_agent_after_primary_clone_and_materializes_failed_secondary(self, monkeypatch):@@ -3400,6 +3400,7 @@ class TestProcessTaskWorkflowUnit:if activity_fn is launch_agent_server:return StartAgentServerOutput(sandbox_url=created.sandbox_url)if activity_fn is clone_repository_in_sandbox:+ assert kwargs["retry_policy"].maximum_attempts == 6raise RuntimeError("clone timed out")if activity_fn is mark_repo_ready:ready_inputs.append(args[0])diff --git a/products/tasks/backend/temporal/process_task/workflow.py b/products/tasks/backend/temporal/process_task/workflow.pyindex a7c03ea81c31..d693546da17f 100644--- a/products/tasks/backend/temporal/process_task/workflow.py+++ b/products/tasks/backend/temporal/process_task/workflow.py@@ -79,6 +79,7 @@ from .activities.get_task_processing_context import (from .activities.materialize_context_layer import MaterializeContextLayerInput, materialize_context_layer_in_sandboxfrom .activities.post_slack_update import PostSlackUpdateInput, post_slack_updatefrom .activities.provision_sandbox import (+ REPOSITORY_CLONE_RETRY_DELAYS_SECONDS,CheckoutBranchInSandboxInput,CloneRepositoryInSandboxInput,CloneRepositoryInSandboxOutput,@@ -2098,7 +2099,7 @@ class ProcessTaskWorkflow(PostHogWorkflow):shallow_clone=prepared.shallow_clone,),start_to_close_timeout=timedelta(minutes=5),- retry_policy=RetryPolicy(maximum_attempts=3),+ retry_policy=RetryPolicy(maximum_attempts=len(REPOSITORY_CLONE_RETRY_DELAYS_SECONDS) + 1),)except Exception as error:if _is_dead_sandbox_failure(error):