:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
perf(weekly-digest): page organizations by id range instead of offset (#121832)
posthog[bot]35e860d5ba7b
diff --git a/posthog/temporal/tests/weekly_digest/test_activities.py b/posthog/temporal/tests/weekly_digest/test_activities.pyindex d0f1fcda2e95..282c4867acb1 100644--- a/posthog/temporal/tests/weekly_digest/test_activities.py+++ b/posthog/temporal/tests/weekly_digest/test_activities.py@@ -4,7 +4,7 @@ import dataclassesfrom collections.abc import Callable, Iteratorfrom datetime import UTC, datetime, timedeltafrom typing import Any-from uuid import uuid4+from uuid import UUID, uuid4import pytestfrom posthog.test.base import _create_event, flush_persons_and_events@@ -36,6 +36,7 @@ from posthog.temporal.weekly_digest.activities import (NEW_ERROR_ISSUES_PER_TEAM_LIMIT,ActivityCancelled,UploadDeadlineExceeded,+ _cut_organization_id_ranges,_cut_team_id_ranges,_query_team_usage_trends,_redis_url,@@ -56,16 +57,22 @@ from posthog.temporal.weekly_digest.activities import (generate_survey_lookup,generate_usage_trends_lookup,generate_user_notification_lookup,+ list_organization_id_ranges,send_weekly_digest_batch,)from posthog.temporal.weekly_digest.keys import TeamDataKey, UserDataKey, org_digest_key, team_data_key, user_data_key-from posthog.temporal.weekly_digest.queries import query_team_ids_for_digest, query_teams_for_digest+from posthog.temporal.weekly_digest.queries import (+ query_orgs_for_digest,+ query_team_ids_for_digest,+ query_teams_for_digest,+)from posthog.temporal.weekly_digest.types import (DEFAULT_PRODUCT_SUGGESTION_TEXT,CommonInput,Digest,GenerateDigestDataBatchInput,GenerateOrganizationDigestInput,+ OrganizationIdRange,SendWeeklyDigestBatchInput,TeamIdRange,UsageTrends,@@ -82,6 +89,7 @@ from products.surveys.backend.models import Surveyfrom products.warehouse_sources.backend.facade.models import ExternalDataSourceDJANGO_REDIS_URL = "redis://django-cache.example.com:6379"+ALL_ORGANIZATIONS = OrganizationIdRange(start=UUID(int=0), end=None)def run_sync(activity_fn: Callable[..., Any], *args: Any) -> Any:@@ -170,6 +178,37 @@ def test_team_id_ranges_page_every_digest_team_exactly_once(organization, digestassert paged == expected+def test_organization_id_ranges_page_every_digest_organization_exactly_once(+ organization, redis_servers, common_input, digest+):+ for i in range(4):+ Organization.objects.create(name=f"digest org {i}")+ if i == 1:+ # Sits inside a batch range, so a leak here shows up as an extra paged organization.+ Organization.objects.create(name="internal org", for_internal_metrics=True)++ common = common_input.model_copy(update={"batch_size": 2})+ expected = list(query_orgs_for_digest().values_list("id", flat=True))+ # An odd count leaves a last range with no end, which must still reach the newest organization.+ assert len(expected) % common.batch_size == 1++ paged: list[UUID] = []+ for organization_id_range in run_sync(list_organization_id_ranges, common):+ redis_servers.digest.flushall()+ run_sync(+ generate_organization_digest_batch,+ GenerateOrganizationDigestInput(organization_id_range=organization_id_range, digest=digest, common=common),+ )+ organization_ids = sorted(+ UUID(json.loads(redis_servers.digest.get(key))["id"]) for key in redis_servers.digest.keys()+ )+ assert len(organization_ids) <= common.batch_size+ paged.extend(organization_ids)++ assert paged == expected++def _make_dashboards(team: Team, other_team: Team, digest: Digest) -> list[str]:Dashboard.objects.create(team=team, name="New dashboard")Dashboard.objects.create(team=team, name="Generated Dashboard: onboarding")@@ -398,7 +437,7 @@ def _run_error_issue_lookup_without_occurrences(team_range: TeamIdRange, digest:def _run_organization_digest_batch(team_range: TeamIdRange, digest: Digest, common: CommonInput) -> None:run_sync(generate_organization_digest_batch,- GenerateOrganizationDigestInput(batch=(0, Organization.objects.count()), digest=digest, common=common),+ GenerateOrganizationDigestInput(organization_id_range=ALL_ORGANIZATIONS, digest=digest, common=common),)@@ -626,7 +665,7 @@ def test_generate_organization_digest_batch_defaults_missing_team_data()run_sync(generate_organization_digest_batch,- GenerateOrganizationDigestInput(batch=(0, Organization.objects.count()), digest=digest, common=common_input),+ GenerateOrganizationDigestInput(organization_id_range=ALL_ORGANIZATIONS, digest=digest, common=common_input),)# One organization's malformed value skips that organization only.@@ -673,9 +712,9 @@ def _org_digest_json(organization: Organization, team: Team) -> str:def _send_input(organization: Organization, digest: Digest, common: CommonInput, dry_run: bool) -> SendWeeklyDigestBatchInput:- organization_index = list(Organization.objects.order_by("id").values_list("id", flat=True)).index(organization.id)+ organization_ids = list(query_orgs_for_digest().values_list("id", flat=True))return SendWeeklyDigestBatchInput(- batch=(organization_index, organization_index + 1),+ organization_id_range=_cut_organization_id_ranges(organization_ids, 1)[organization_ids.index(organization.id)],dry_run=dry_run,allow_already_sent=False,digest=digest,@@ -813,8 +852,6 @@ def test_send_weekly_digest_batch_resend_reaches_only_listed_organizations(session.post.return_value.status_code = 201resend_input = _send_input(organization, digest, common_input, dry_run=False).model_copy(update={- # A resend batches over the listed organizations only.- "batch": (0, 1),"allow_already_sent": True,"organization_ids": [str(organization.id) if listed else str(uuid4())],"workflows_only": workflows_only,diff --git a/posthog/temporal/tests/weekly_digest/test_workflows.py b/posthog/temporal/tests/weekly_digest/test_workflows.pyindex 41dd7cc1bac8..a6b161c86bd1 100644--- a/posthog/temporal/tests/weekly_digest/test_workflows.py+++ b/posthog/temporal/tests/weekly_digest/test_workflows.py@@ -22,7 +22,6 @@ from temporalio.testing import WorkflowEnvironmentfrom temporalio.worker import Replayer, UnsandboxedWorkflowRunner, Workerfrom posthog.temporal.weekly_digest.activities import (- count_organizations,generate_dashboard_lookup,generate_error_issue_lookup,generate_event_definition_lookup,@@ -37,6 +36,7 @@ from posthog.temporal.weekly_digest.activities import (generate_survey_lookup,generate_usage_trends_lookup,generate_user_notification_lookup,+ list_organization_id_ranges,list_team_id_ranges,)from posthog.temporal.weekly_digest.types import (@@ -45,6 +45,7 @@ from posthog.temporal.weekly_digest.types import (GenerateDigestDataBatchInput,GenerateDigestDataInput,GenerateOrganizationDigestInput,+ OrganizationIdRange,SendWeeklyDigestBatchInput,SendWeeklyDigestInput,TeamIdRange,@@ -131,20 +132,19 @@ class LegacyGenerateDigestDataWorkflow:for team_id_range, generator in itertools.product(team_id_ranges, generators)))- organization_count = await workflow.execute_activity(- count_organizations,+ organization_id_ranges = await workflow.execute_activity(+ list_organization_id_ranges,+ input.common,start_to_close_timeout=timedelta(minutes=5),retry_policy=common.RetryPolicy(maximum_attempts=2, initial_interval=timedelta(minutes=1)),heartbeat_timeout=timedelta(minutes=1),)- batch_size = input.common.batch_size- org_batches = [(start, start + batch_size) for start in range(0, organization_count, batch_size)]await asyncio.gather(*(workflow.execute_activity(generate_organization_digest_batch,GenerateOrganizationDigestInput(- batch=batch,+ organization_id_range=organization_id_range,digest=input.digest,common=input.common,),@@ -152,7 +152,7 @@ class LegacyGenerateDigestDataWorkflow:retry_policy=common.RetryPolicy(maximum_attempts=2, initial_interval=timedelta(minutes=1)),heartbeat_timeout=timedelta(minutes=2),)- for batch in org_batches+ for organization_id_range in organization_id_ranges))@@ -235,22 +235,26 @@ def test_resolve_redis_host(_name: str, env_value: str | None, local_dev: bool,@pytest.mark.asyncioasync def test_generate_digest_data_bounds_pending_activities(patched: bool) -> None:team_ranges = [TeamIdRange(start=i, end=i + 1) for i in range(200)]+ organization_ranges = [+ OrganizationIdRange(start=uuid.UUID(int=0), end=uuid.UUID(int=2)),+ OrganizationIdRange(start=uuid.UUID(int=2), end=None),+ ]pending = 0peak_pending = 0completed: list[tuple[object, int, int]] = []- aggregated: list[tuple[int, int]] = []+ aggregated: list[OrganizationIdRange] = []async def execute_activity(activity_fn: object, activity_input: object = None, **_: object) -> object:nonlocal pending, peak_pendingif activity_fn is list_team_id_ranges:return team_ranges- if activity_fn is count_organizations:+ if activity_fn is list_organization_id_ranges:assert pending == 0assert len(completed) == 13 * len(team_ranges)- return 3+ return organization_rangesif activity_fn is generate_organization_digest_batch:assert isinstance(activity_input, GenerateOrganizationDigestInput)- aggregated.append(activity_input.batch)+ aggregated.append(activity_input.organization_id_range)return Noneassert isinstance(activity_input, GenerateDigestDataBatchInput)@@ -282,7 +286,7 @@ async def test_generate_digest_data_bounds_pending_activities(patched: bool) ->assert Counter((start, end) for _, start, end in completed) == {(team_range.start, team_range.end): 13 for team_range in team_ranges}- assert aggregated == [(0, 2), (2, 4)]+ assert aggregated == organization_rangesif patched:assert peak_pending <= MAX_CONCURRENT_GENERATION_ACTIVITIES < 2000else:@@ -314,9 +318,12 @@ async def test_generate_digest_data_replays_pre_patch_history(caplog: pytest.Logasync def list_team_id_ranges_mocked(_input: CommonInput) -> list[TeamIdRange]:return team_ranges- @activity.defn(name="count-organizations")- async def count_organizations_mocked() -> int:- return 3+ @activity.defn(name="list-organization-id-ranges")+ async def list_organization_id_ranges_mocked(_input: CommonInput) -> list[OrganizationIdRange]:+ return [+ OrganizationIdRange(start=uuid.UUID(int=0), end=uuid.UUID(int=2)),+ OrganizationIdRange(start=uuid.UUID(int=2), end=None),+ ]@activity.defn(name="generate-organization-digest-batch")async def generate_organization_digest_batch_mocked(_input: GenerateOrganizationDigestInput) -> None:@@ -331,7 +338,7 @@ async def test_generate_digest_data_replays_pre_patch_history(caplog: pytest.Logactivities: list[Callable[..., Awaitable[object]]] = [list_team_id_ranges_mocked,- count_organizations_mocked,+ list_organization_id_ranges_mocked,generate_organization_digest_batch_mocked,*(make_generator(name) for name in generator_names),]@@ -391,12 +398,12 @@ async def test_generate_digest_data_replays_pre_patch_history(caplog: pytest.Logasync def test_generate_digest_data_workflow():"""Test the digest data generation workflow with batched activities."""TEST_TEAM_COUNT = 5- TEST_ORG_COUNT = 2+ TEST_ORG_BATCH_COUNT = 2TEST_BATCH_SIZE = 2activity_calls = {"team_id_ranges": 0,- "count_organizations": 0,+ "organization_id_ranges": 0,"dashboard": 0,"event_definition": 0,"experiment_completed": 0,@@ -418,10 +425,10 @@ async def test_generate_digest_data_workflow():activity_calls["team_id_ranges"] += 1return [TeamIdRange(start=i, end=i + TEST_BATCH_SIZE) for i in range(1, TEST_TEAM_COUNT + 1, TEST_BATCH_SIZE)]- @activity.defn(name="count-organizations")- async def count_organizations_mocked() -> int:- activity_calls["count_organizations"] += 1- return TEST_ORG_COUNT+ @activity.defn(name="list-organization-id-ranges")+ async def list_organization_id_ranges_mocked(input) -> list[OrganizationIdRange]:+ activity_calls["organization_id_ranges"] += 1+ return [OrganizationIdRange(start=uuid.UUID(int=i), end=None) for i in range(TEST_ORG_BATCH_COUNT)]@activity.defn(name="generate-dashboard-lookup")async def generate_dashboard_lookup_mocked(input) -> None:@@ -487,7 +494,7 @@ async def test_generate_digest_data_workflow():workflows=[GenerateDigestDataWorkflow],activities=[list_team_id_ranges_mocked,- count_organizations_mocked,+ list_organization_id_ranges_mocked,generate_dashboard_lookup_mocked,generate_event_definition_lookup_mocked,generate_experiment_completed_lookup_mocked,@@ -520,7 +527,7 @@ async def test_generate_digest_data_workflow():)assert activity_calls["team_id_ranges"] == 1- assert activity_calls["count_organizations"] == 1+ assert activity_calls["organization_id_ranges"] == 1# Calculate expected batches for teamsexpected_team_batches = (TEST_TEAM_COUNT + TEST_BATCH_SIZE - 1) // TEST_BATCH_SIZE@@ -540,30 +547,37 @@ async def test_generate_digest_data_workflow():assert activity_calls["error_issue"] == expected_team_batchesassert activity_calls["usage_trends"] == expected_team_batches- # Calculate expected batches for organizations- expected_org_batches = (TEST_ORG_COUNT + TEST_BATCH_SIZE - 1) // TEST_BATCH_SIZE- assert activity_calls["org_digest"] == expected_org_batches+ assert activity_calls["org_digest"] == TEST_ORG_BATCH_COUNT@pytest.mark.asyncioasync def test_send_weekly_digest_workflow():"""Test the digest sending workflow with batched activities."""- TEST_ORG_COUNT = 10TEST_BATCH_SIZE = 3+ organization_ranges = [+ OrganizationIdRange(start=uuid.UUID(int=0), end=uuid.UUID(int=3)),+ OrganizationIdRange(start=uuid.UUID(int=3), end=uuid.UUID(int=6)),+ OrganizationIdRange(start=uuid.UUID(int=6), end=None),+ ]activity_calls = {- "count_organizations": 0,+ "organization_id_ranges": 0,"send_batch": 0,}+ sent_ranges: list[OrganizationIdRange] = []- @activity.defn(name="count-organizations")- async def count_organizations_mocked() -> int:- activity_calls["count_organizations"] += 1- return TEST_ORG_COUNT+ @activity.defn(name="list-organization-id-ranges")+ async def list_organization_id_ranges_mocked(+ input: CommonInput, organization_ids: list[str] | None = None+ ) -> list[OrganizationIdRange]:+ activity_calls["organization_id_ranges"] += 1+ assert input.batch_size == TEST_BATCH_SIZE+ return organization_ranges@activity.defn(name="send-weekly-digest-batch")async def send_weekly_digest_batch_mocked(input: SendWeeklyDigestBatchInput) -> None:activity_calls["send_batch"] += 1+ sent_ranges.append(input.organization_id_range)assert input.dry_run is Trueassert input.digest.key == "test-digest"@@ -574,7 +588,7 @@ async def test_send_weekly_digest_workflow():task_queue=task_queue_name,workflows=[SendWeeklyDigestWorkflow],activities=[- count_organizations_mocked,+ list_organization_id_ranges_mocked,send_weekly_digest_batch_mocked,],workflow_runner=temporalio.worker.UnsandboxedWorkflowRunner(),@@ -595,11 +609,8 @@ async def test_send_weekly_digest_workflow():task_queue=task_queue_name,)- assert activity_calls["count_organizations"] == 1-- # Calculate expected batches- expected_batches = (TEST_ORG_COUNT + TEST_BATCH_SIZE - 1) // TEST_BATCH_SIZE- assert activity_calls["send_batch"] == expected_batches+ assert activity_calls["organization_id_ranges"] == 1+ assert sorted(sent_ranges, key=lambda r: r.start) == organization_ranges@pytest.mark.asynciodiff --git a/posthog/temporal/weekly_digest/README.md b/posthog/temporal/weekly_digest/README.mdindex eee2b0ccb576..1f4ffb2c9fa4 100644--- a/posthog/temporal/weekly_digest/README.md+++ b/posthog/temporal/weekly_digest/README.md@@ -95,7 +95,7 @@ Generated via `user_data_key(digest_key, UserDataKey.*, user_id)`:│ GenerateDigestDataWorkflow │├─────────────────────────────────────────────────────────────────────────────┤│ │-│ 1. Cut teams into id-range batches, count orgs for batching │+│ 1. Cut teams and orgs into id-range batches ││ ││ 2. Generate team-level data (parallel per batch): ││ ├── generate_dashboard_lookup │@@ -123,7 +123,7 @@ Generated via `user_data_key(digest_key, UserDataKey.*, user_id)`:│ SendWeeklyDigestWorkflow │├─────────────────────────────────────────────────────────────────────────────┤│ │-│ For each organization (batched): │+│ For each organization (batched by id range): ││ 1. Load OrganizationDigest from Redis ││ 2. For each org member: ││ a. Load user's notification team set │diff --git a/posthog/temporal/weekly_digest/__init__.py b/posthog/temporal/weekly_digest/__init__.pyindex 686f783933da..d0def072a68b 100644--- a/posthog/temporal/weekly_digest/__init__.py+++ b/posthog/temporal/weekly_digest/__init__.py@@ -1,5 +1,4 @@from posthog.temporal.weekly_digest.activities import (- count_organizations,generate_dashboard_lookup,generate_error_issue_lookup,generate_event_definition_lookup,@@ -14,6 +13,7 @@ from posthog.temporal.weekly_digest.activities import (generate_survey_lookup,generate_usage_trends_lookup,generate_user_notification_lookup,+ list_organization_id_ranges,list_team_id_ranges,send_weekly_digest_batch,)@@ -39,7 +39,7 @@ ACTIVITIES = [generate_feature_flag_lookup,generate_user_notification_lookup,generate_organization_digest_batch,- count_organizations,+ list_organization_id_ranges,list_team_id_ranges,send_weekly_digest_batch,generate_filter_lookup,diff --git a/posthog/temporal/weekly_digest/activities.py b/posthog/temporal/weekly_digest/activities.pyindex 3b0281611c3f..5088744d6957 100644--- a/posthog/temporal/weekly_digest/activities.py+++ b/posthog/temporal/weekly_digest/activities.py@@ -78,6 +78,7 @@ from posthog.temporal.weekly_digest.types import (GenerateDigestDataBatchInput,GenerateOrganizationDigestInput,OrganizationDigest,+ OrganizationIdRange,PlaylistCount,RecordingCount,SendWeeklyDigestBatchInput,@@ -869,18 +870,6 @@ def generate_product_suggestion_lookup(input: GenerateDigestDataBatchInput) -> N_generate_product_suggestion_lookup(input)-def _organizations_to_send(organization_ids: list[str] | None) -> QuerySet:- organizations = query_orgs_for_digest()- return organizations if organization_ids is None else organizations.filter(id__in=organization_ids)--[email protected](name="count-organizations")-@asyncify-def count_organizations(organization_ids: list[str] | None = None) -> int:- with HeartbeaterSync(logger=LOGGER):- return _organizations_to_send(organization_ids).count()--def _cut_team_id_ranges(team_ids: list[int], batch_size: int) -> list[TeamIdRange]:"""Cut ordered team ids into [start, end) ranges of at most `batch_size` teams each."""return [@@ -900,6 +889,54 @@ def list_team_id_ranges(input: CommonInput) -> list[TeamIdRange]:return _cut_team_id_ranges(list(query_team_ids_for_digest()), input.batch_size)+def _cut_organization_id_ranges(organization_ids: list[UUID], batch_size: int) -> list[OrganizationIdRange]:+ """Cut ordered organization ids into [start, end) ranges of at most `batch_size` organizations each."""+ return [+ OrganizationIdRange(+ start=organization_ids[start],+ end=organization_ids[start + batch_size] if start + batch_size < len(organization_ids) else None,+ )+ for start in range(0, len(organization_ids), batch_size)+ ]+++def _organizations_to_send(organization_ids: list[str] | None) -> QuerySet:+ organizations = query_orgs_for_digest()+ return organizations if organization_ids is None else organizations.filter(id__in=organization_ids)++[email protected](name="list-organization-id-ranges")+@asyncify+def list_organization_id_ranges(+ input: CommonInput, organization_ids: list[str] | None = None+) -> list[OrganizationIdRange]:+ """One index scan of the organization ids replaces a LIMIT/OFFSET scan per batch."""+ with HeartbeaterSync(logger=LOGGER):+ ordered_ids = list(_organizations_to_send(organization_ids).values_list("id", flat=True))+ return _cut_organization_id_ranges(ordered_ids, input.batch_size)+++def _organizations_in_range(+ organization_id_range: OrganizationIdRange, organization_ids: list[str] | None = None+) -> QuerySet:+ # A range also spans the unlisted organizations between listed ones, so a resend filters by its list again.+ organizations = _organizations_to_send(organization_ids).filter(id__gte=organization_id_range.start)+ if organization_id_range.end is not None:+ organizations = organizations.filter(id__lt=organization_id_range.end)+ return organizations+++def _bind_organization_batch_logger(+ digest_key: str, organization_id_range: OrganizationIdRange+) -> FilteringBoundLogger:+ bind_contextvars(+ digest_key=digest_key,+ organization_id_start=str(organization_id_range.start),+ organization_id_end=str(organization_id_range.end) if organization_id_range.end is not None else None,+ )+ return LOGGER.bind()++# Positions follow TeamDataKey order and the TeamDigest constructor below.TEAM_DATA_DEFAULTS: list[tuple[TeamDataKey, Any]] = [(TeamDataKey.DASHBOARDS, DashboardList(root=[])),@@ -950,15 +987,13 @@ def _team_digest(team: Team, raw_values: list[str | None]) -> TeamDigest:def _generate_organization_digest_batch(input: GenerateOrganizationDigestInput) -> None:- bind_contextvars(digest_key=input.digest.key, batch_start=input.batch[0], batch_end=input.batch[1])- logger = LOGGER.bind()+ logger = _bind_organization_batch_logger(input.digest.key, input.organization_id_range)logger.info("Generating organization-level digest batch")organization_count = 0team_count = 0- batch_start, batch_end = input.batch- organizations = list(query_orgs_for_digest()[batch_start:batch_end])+ organizations = list(_organizations_in_range(input.organization_id_range))teams_by_org: dict[UUID, list[Team]] = defaultdict(list)for team in query_teams_for_organizations([organization.id for organization in organizations]):teams_by_org[team.organization_id].append(team)@@ -1153,8 +1188,7 @@ def _post_to_workflows(session: requests.Session, body: bytes, idempotency_key:def _send_weekly_digest_batch(input: SendWeeklyDigestBatchInput) -> None:- bind_contextvars(digest_key=input.digest.key, batch_start=input.batch[0], batch_end=input.batch[1])- logger = LOGGER.bind()+ logger = _bind_organization_batch_logger(input.digest.key, input.organization_id_range)logger.info("Sending weekly digest batch")sent_digest_count = 0@@ -1199,8 +1233,7 @@ def _send_weekly_digest_batch(input: SendWeeklyDigestBatchInput) -> None:try:with _digest_redis(input.common) as r:- batch_start, batch_end = input.batch- for organization in _organizations_to_send(input.organization_ids)[batch_start:batch_end]:+ for organization in _organizations_in_range(input.organization_id_range, input.organization_ids):_raise_if_cancelled()partial = Falsetry:diff --git a/posthog/temporal/weekly_digest/types.py b/posthog/temporal/weekly_digest/types.pyindex 7f56346b3dfd..e9482e8e265e 100644--- a/posthog/temporal/weekly_digest/types.py+++ b/posthog/temporal/weekly_digest/types.py@@ -58,8 +58,15 @@ class GenerateDigestDataBatchInput(BaseModel):common: CommonInput+class OrganizationIdRange(BaseModel):+ """A half-open [start, end) range of organization ids. The last range has no end."""++ start: UUID+ end: UUID | None++class GenerateOrganizationDigestInput(BaseModel):- batch: tuple[int, int]+ organization_id_range: OrganizationIdRangedigest: Digestcommon: CommonInput@@ -74,7 +81,7 @@ class SendWeeklyDigestInput(BaseModel):class SendWeeklyDigestBatchInput(BaseModel):- batch: tuple[int, int]+ organization_id_range: OrganizationIdRangedry_run: boolallow_already_sent: booldigest: Digestdiff --git a/posthog/temporal/weekly_digest/workflows.py b/posthog/temporal/weekly_digest/workflows.pyindex e5feae7900a9..0fb6677874e8 100644--- a/posthog/temporal/weekly_digest/workflows.py+++ b/posthog/temporal/weekly_digest/workflows.py@@ -13,7 +13,6 @@ from temporalio.exceptions import ApplicationErrorfrom posthog.temporal.common.base import PostHogWorkflowfrom posthog.temporal.weekly_digest.activities import (- count_organizations,generate_dashboard_lookup,generate_error_issue_lookup,generate_event_definition_lookup,@@ -28,6 +27,7 @@ from posthog.temporal.weekly_digest.activities import (generate_survey_lookup,generate_usage_trends_lookup,generate_user_notification_lookup,+ list_organization_id_ranges,list_team_id_ranges,send_weekly_digest_batch,)@@ -195,8 +195,9 @@ class GenerateDigestDataWorkflow(PostHogWorkflow):))- organization_count = await workflow.execute_activity(- count_organizations,+ organization_id_ranges = await workflow.execute_activity(+ list_organization_id_ranges,+ input.common,start_to_close_timeout=timedelta(minutes=5),retry_policy=common.RetryPolicy(maximum_attempts=2,@@ -205,15 +206,12 @@ class GenerateDigestDataWorkflow(PostHogWorkflow):heartbeat_timeout=timedelta(minutes=1),)- batch_size = input.common.batch_size- org_batches = [(i, i + batch_size) for i in range(0, organization_count, batch_size)]-await asyncio.gather(*[workflow.execute_activity(generate_organization_digest_batch,GenerateOrganizationDigestInput(- batch=batch,+ organization_id_range=organization_id_range,digest=input.digest,common=input.common,),@@ -224,7 +222,7 @@ class GenerateDigestDataWorkflow(PostHogWorkflow):),heartbeat_timeout=timedelta(minutes=2),)- for batch in org_batches+ for organization_id_range in organization_id_ranges])@@ -238,10 +236,9 @@ class SendWeeklyDigestWorkflow(PostHogWorkflow):@workflow.runasync def run(self, input: SendWeeklyDigestInput) -> None:- # Only a resend passes its organizations, so the scheduled run calls the activity exactly as before.- organization_count = await workflow.execute_activity(- count_organizations,- args=[input.organization_ids] if input.organization_ids is not None else [],+ organization_id_ranges = await workflow.execute_activity(+ list_organization_id_ranges,+ args=[input.common, input.organization_ids],start_to_close_timeout=timedelta(minutes=5),retry_policy=common.RetryPolicy(maximum_attempts=2,@@ -250,15 +247,12 @@ class SendWeeklyDigestWorkflow(PostHogWorkflow):heartbeat_timeout=timedelta(minutes=1),)- batch_size = input.common.batch_size- batches = [(i, i + batch_size) for i in range(0, organization_count, batch_size)]-await asyncio.gather(*[workflow.execute_activity(send_weekly_digest_batch,SendWeeklyDigestBatchInput(- batch=batch,+ organization_id_range=organization_id_range,dry_run=input.dry_run,allow_already_sent=input.allow_already_sent,digest=input.digest,@@ -274,6 +268,6 @@ class SendWeeklyDigestWorkflow(PostHogWorkflow):),heartbeat_timeout=timedelta(minutes=2),)- for batch in batches+ for organization_id_range in organization_id_ranges])