Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(early-access): stop stage-change task running out of memory (#129019)

Joe Martin4ac5eb5314e4
diff --git a/posthog/tasks/early_access_feature.py b/posthog/tasks/early_access_feature.py
index 30a52fcb9140..bffd6d5508a0 100644
--- a/posthog/tasks/early_access_feature.py
+++ b/posthog/tasks/early_access_feature.py
@@ -12,6 +12,7 @@ from posthog.hogql.constants import MAX_SELECT_RETURNED_ROWS, LimitContext
from posthog.hogql.query import execute_hogql_query
from posthog.cloud_utils import is_cloud
+from posthog.exceptions_capture import capture_exception
from posthog.models import Team
from posthog.scoping_audit import skip_team_scope_audit
@@ -155,25 +156,62 @@ def send_events_for_early_access_feature_stage_change(feature_id: str, from_stag
# Get the unique persons enrolled in the feature along with their distinct ID.
# Property access (rather than JSONExtractString) lets the HogQL printer use
# materialized person-property columns when available.
- response = execute_hogql_query(
- """
- SELECT
- argMax(id, created_at) AS id,
- properties.email AS email,
- argMax(pdi.distinct_id, created_at) as distinct_id
- FROM persons
- WHERE properties[{enrollment_key}] = 'true'
- AND notEmpty(properties.email)
- GROUP BY properties.email
- LIMIT {limit}
- """,
- placeholders={
- "enrollment_key": ast.Constant(value=f"$feature_enrollment/{feature_flag.key}"),
- "limit": ast.Constant(value=MAX_SELECT_RETURNED_ROWS),
- },
- team=instance.team,
- limit_context=LimitContext.QUERY_ASYNC,
- )
+ # Joining persons.pdi would aggregate every distinct ID of the team and run out of memory
+ # on large teams. Prefilter to the enrolled persons' distinct IDs, then resolve ownership
+ # per distinct_id like the person_distinct_ids view so moved or deleted IDs are skipped.
+ try:
+ response = execute_hogql_query(
+ """
+ WITH enrolled AS (
+ SELECT id, created_at, properties.email AS email
+ FROM persons
+ WHERE properties[{enrollment_key}] = 'true'
+ AND notEmpty(properties.email)
+ )
+ SELECT p.id, p.email, d.distinct_id
+ FROM (
+ SELECT argMax(id, created_at) AS id, email
+ FROM enrolled
+ GROUP BY email
+ ) AS p
+ INNER JOIN (
+ SELECT person_id, argMax(distinct_id, max_version) AS distinct_id
+ FROM (
+ SELECT
+ distinct_id,
+ argMax(raw_person_distinct_ids.person_id, version) AS person_id,
+ max(version) AS max_version
+ FROM raw_person_distinct_ids
+ WHERE distinct_id IN (
+ SELECT distinct_id
+ FROM raw_person_distinct_ids
+ WHERE person_id IN (SELECT id FROM enrolled)
+ )
+ GROUP BY distinct_id
+ HAVING argMax(is_deleted, version) = 0
+ )
+ GROUP BY person_id
+ ) AS d ON p.id = d.person_id
+ LIMIT {limit}
+ """,
+ placeholders={
+ "enrollment_key": ast.Constant(value=f"$feature_enrollment/{feature_flag.key}"),
+ "limit": ast.Constant(value=MAX_SELECT_RETURNED_ROWS),
+ },
+ team=instance.team,
+ limit_context=LimitContext.QUERY_ASYNC,
+ )
+ except Exception as e:
+ capture_exception(
+ error=e,
+ additional_properties={
+ "feature_id": str(instance.id),
+ "team_id": team_id,
+ "from_stage": from_stage,
+ "to_stage": to_stage,
+ },
+ )
+ raise
enrolled_persons = response.results
for _id, email, distinct_id in enrolled_persons:
diff --git a/posthog/tasks/test/test_early_access_feature.py b/posthog/tasks/test/test_early_access_feature.py
index 6406f5d952b1..c50b5aa787bb 100644
--- a/posthog/tasks/test/test_early_access_feature.py
+++ b/posthog/tasks/test/test_early_access_feature.py
@@ -3,9 +3,10 @@ from unittest.mock import MagicMock, patch
from posthog.hogql.constants import DEFAULT_RETURNED_ROWS
+from posthog.models.person.util import create_person_distinct_id
from posthog.models.team import Team
from posthog.tasks.early_access_feature import send_events_for_early_access_feature_stage_change
-from posthog.test.persons import create_person
+from posthog.test.persons import add_distinct_id, create_person
from products.early_access_features.backend.models import EarlyAccessFeature
from products.feature_flags.backend.models.feature_flag import FeatureFlag
@@ -29,6 +30,12 @@ class TestSendEventsForEarlyAccessFeatureStageChange(APIBaseTest):
distinct_ids=["abc123"],
properties={f"$feature_enrollment/{feature_flag.key}": True, "email": "[email protected]"},
)
+ create_person(
+ team=team,
+ distinct_ids=["opted_out"],
+ properties={f"$feature_enrollment/{feature_flag.key}": False, "email": "[email protected]"},
+ )
+ create_person(team=team, distinct_ids=["never_enrolled"], properties={"email": "[email protected]"})
send_events_for_early_access_feature_stage_change(feature.id, "concept", "beta")
@@ -45,6 +52,102 @@ class TestSendEventsForEarlyAccessFeatureStageChange(APIBaseTest):
},
)
+ @patch("posthog.tasks.early_access_feature.capture_event")
+ def test_sends_one_event_for_person_with_multiple_distinct_ids(self, mock_capture: MagicMock) -> None:
+ team = Team.objects.create(organization=self.organization)
+ feature_flag = FeatureFlag.objects.create(team=team, key="my-flag", filters={})
+ feature = EarlyAccessFeature.objects.create(
+ team=team,
+ feature_flag=feature_flag,
+ name="Test Feature",
+ description="desc",
+ stage=EarlyAccessFeature.Stage.BETA,
+ )
+
+ create_person(
+ team=team,
+ distinct_ids=["anon123", "abc123"],
+ properties={f"$feature_enrollment/{feature_flag.key}": True, "email": "[email protected]"},
+ )
+
+ send_events_for_early_access_feature_stage_change(feature.id, "concept", "beta")
+
+ mock_capture.assert_called_once()
+ assert mock_capture.call_args.kwargs["distinct_id"] in {"anon123", "abc123"}
+ assert mock_capture.call_args.kwargs["properties"]["user_email"] == "[email protected]"
+
+ @patch("posthog.tasks.early_access_feature.capture_event")
+ def test_skips_distinct_id_reassigned_to_another_person(self, mock_capture: MagicMock) -> None:
+ team = Team.objects.create(organization=self.organization)
+ feature_flag = FeatureFlag.objects.create(team=team, key="my-flag", filters={})
+ feature = EarlyAccessFeature.objects.create(
+ team=team,
+ feature_flag=feature_flag,
+ name="Test Feature",
+ description="desc",
+ stage=EarlyAccessFeature.Stage.BETA,
+ )
+
+ enrolled = create_person(
+ team=team,
+ distinct_ids=["enrolled_id"],
+ properties={f"$feature_enrollment/{feature_flag.key}": True, "email": "[email protected]"},
+ )
+ add_distinct_id(person=enrolled, distinct_id="moved_id", version=1)
+ other = create_person(team=team, distinct_ids=["other_id"], properties={})
+ # The old mapping row to the enrolled person stays in raw_person_distinct_ids.
+ add_distinct_id(person=other, distinct_id="moved_id", version=2)
+
+ send_events_for_early_access_feature_stage_change(feature.id, "concept", "beta")
+
+ mock_capture.assert_called_once()
+ assert mock_capture.call_args.kwargs["distinct_id"] == "enrolled_id"
+ assert mock_capture.call_args.kwargs["properties"]["user_email"] == "[email protected]"
+
+ @patch("posthog.tasks.early_access_feature.capture_event")
+ def test_skips_deleted_distinct_id(self, mock_capture: MagicMock) -> None:
+ team = Team.objects.create(organization=self.organization)
+ feature_flag = FeatureFlag.objects.create(team=team, key="my-flag", filters={})
+ feature = EarlyAccessFeature.objects.create(
+ team=team,
+ feature_flag=feature_flag,
+ name="Test Feature",
+ description="desc",
+ stage=EarlyAccessFeature.Stage.BETA,
+ )
+
+ enrolled = create_person(
+ team=team,
+ distinct_ids=["enrolled_id"],
+ properties={f"$feature_enrollment/{feature_flag.key}": True, "email": "[email protected]"},
+ )
+ add_distinct_id(person=enrolled, distinct_id="gone_id", version=1)
+ create_person_distinct_id(team.pk, "gone_id", str(enrolled.uuid), version=2, is_deleted=True)
+
+ send_events_for_early_access_feature_stage_change(feature.id, "concept", "beta")
+
+ mock_capture.assert_called_once()
+ assert mock_capture.call_args.kwargs["distinct_id"] == "enrolled_id"
+ assert mock_capture.call_args.kwargs["properties"]["user_email"] == "[email protected]"
+
+ @patch("posthog.tasks.early_access_feature.capture_exception")
+ @patch("posthog.tasks.early_access_feature.execute_hogql_query", side_effect=RuntimeError("query failed"))
+ def test_captures_and_reraises_query_error(self, _mock_query: MagicMock, mock_capture_exception: MagicMock) -> None:
+ feature_flag = FeatureFlag.objects.create(team=self.team, key="my-flag", filters={})
+ feature = EarlyAccessFeature.objects.create(
+ team=self.team,
+ feature_flag=feature_flag,
+ name="Test Feature",
+ description="desc",
+ stage=EarlyAccessFeature.Stage.BETA,
+ )
+
+ with self.assertRaises(RuntimeError):
+ send_events_for_early_access_feature_stage_change(feature.id, "concept", "beta")
+
+ mock_capture_exception.assert_called_once()
+ assert mock_capture_exception.call_args.kwargs["additional_properties"]["feature_id"] == str(feature.id)
+
@patch("posthog.tasks.early_access_feature.capture_event")
def test_no_event_for_enrolled_users_on_different_team(self, mock_capture: MagicMock) -> None:
team1 = Team.objects.create(organization=self.organization)