Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(tasks): cap repository clone retries at five minutes (#126186)

Alessandro Pogliaghi0ddc16941a4f
diff --git a/products/tasks/backend/temporal/process_task/tests/test_workflow.py b/products/tasks/backend/temporal/process_task/tests/test_workflow.py
index 3cbc32ce064f..e91afb85f4ba 100644
--- a/products/tasks/backend/temporal/process_task/tests/test_workflow.py
+++ b/products/tasks/backend/temporal/process_task/tests/test_workflow.py
@@ -22,7 +22,13 @@ from parameterized import parameterized
from temporalio import activity
from temporalio.client import WorkflowFailureError
from temporalio.common import RetryPolicy
-from temporalio.exceptions import ActivityError, ApplicationError, RetryState
+from temporalio.exceptions import (
+ ActivityError,
+ ApplicationError,
+ RetryState,
+ TimeoutError as ActivityTimeoutError,
+ TimeoutType,
+)
from temporalio.testing import WorkflowEnvironment
from temporalio.worker import UnsandboxedWorkflowRunner, Worker
@@ -3321,6 +3327,7 @@ class TestProcessTaskWorkflowUnit:
assert cloned == ["posthog/posthog", "posthog/code"]
for repository in ("posthog/posthog", "posthog/code"):
assert clone_options[repository]["start_to_close_timeout"] == timedelta(minutes=5)
+ assert clone_options[repository]["schedule_to_close_timeout"] == timedelta(minutes=5)
assert clone_options[repository]["retry_policy"].maximum_attempts == 6
assert result.clone_ms is None
@@ -3397,7 +3404,24 @@ class TestProcessTaskWorkflowUnit:
assert ready_inputs[1].release_barrier is False
@pytest.mark.parametrize("fail_on_primary_clone_failure", [True, False])
- async def test_overlap_primary_clone_failure(self, monkeypatch, fail_on_primary_clone_failure):
+ @pytest.mark.parametrize("clone_times_out", [True, False])
+ async def test_overlap_primary_clone_failure(
+ self, monkeypatch: pytest.MonkeyPatch, fail_on_primary_clone_failure: bool, clone_times_out: bool
+ ) -> None:
+ clone_error: Exception = RuntimeError("clone timed out")
+ if clone_times_out:
+ clone_error = ActivityError(
+ "clone timed out",
+ scheduled_event_id=1,
+ started_event_id=2,
+ identity="test-worker",
+ activity_type="clone_repository_in_sandbox",
+ activity_id="clone",
+ retry_state=RetryState.TIMEOUT,
+ )
+ clone_error.__cause__ = ActivityTimeoutError(
+ "clone deadline exceeded", type=TimeoutType.SCHEDULE_TO_CLOSE, last_heartbeat_details=[]
+ )
workflow = ProcessTaskWorkflow()
workflow._context = _build_context(
github_integration_id=123,
@@ -3437,7 +3461,7 @@ class TestProcessTaskWorkflowUnit:
return StartAgentServerOutput(sandbox_url=created.sandbox_url)
if activity_fn is clone_repository_in_sandbox:
assert kwargs["retry_policy"].maximum_attempts == 6
- raise RuntimeError("clone timed out")
+ raise clone_error
if activity_fn is mark_repo_ready:
ready_inputs.append(args[0])
return None
@@ -3458,7 +3482,7 @@ class TestProcessTaskWorkflowUnit:
monkeypatch.setattr(process_task_workflow_module.workflow, "logger", Mock())
if fail_on_primary_clone_failure:
- with pytest.raises(RuntimeError, match="clone timed out"):
+ with pytest.raises(type(clone_error), match="clone timed out"):
await workflow._get_sandbox_for_repository()
assert mark_repo_ready not in calls
assert checkout_branch_in_sandbox not in calls
diff --git a/products/tasks/backend/temporal/process_task/workflow.py b/products/tasks/backend/temporal/process_task/workflow.py
index 21590c85891e..e4a23fc7ad82 100644
--- a/products/tasks/backend/temporal/process_task/workflow.py
+++ b/products/tasks/backend/temporal/process_task/workflow.py
@@ -2095,6 +2095,7 @@ class ProcessTaskWorkflow(PostHogWorkflow):
shallow_clone=prepared.shallow_clone,
),
start_to_close_timeout=timedelta(minutes=5),
+ schedule_to_close_timeout=timedelta(minutes=5),
retry_policy=RetryPolicy(maximum_attempts=len(REPOSITORY_CLONE_RETRY_DELAYS_SECONDS) + 1),
)
except Exception as error: