Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

perf(weekly-digest): page organizations by id range instead of offset (#121832)

posthog[bot]35e860d5ba7b
diff --git a/posthog/temporal/tests/weekly_digest/test_activities.py b/posthog/temporal/tests/weekly_digest/test_activities.py
index d0f1fcda2e95..282c4867acb1 100644
--- a/posthog/temporal/tests/weekly_digest/test_activities.py
+++ b/posthog/temporal/tests/weekly_digest/test_activities.py
@@ -4,7 +4,7 @@ import dataclasses
from collections.abc import Callable, Iterator
from datetime import UTC, datetime, timedelta
from typing import Any
-from uuid import uuid4
+from uuid import UUID, uuid4
import pytest
from posthog.test.base import _create_event, flush_persons_and_events
@@ -36,6 +36,7 @@ from posthog.temporal.weekly_digest.activities import (
NEW_ERROR_ISSUES_PER_TEAM_LIMIT,
ActivityCancelled,
UploadDeadlineExceeded,
+ _cut_organization_id_ranges,
_cut_team_id_ranges,
_query_team_usage_trends,
_redis_url,
@@ -56,16 +57,22 @@ from posthog.temporal.weekly_digest.activities import (
generate_survey_lookup,
generate_usage_trends_lookup,
generate_user_notification_lookup,
+ list_organization_id_ranges,
send_weekly_digest_batch,
)
from posthog.temporal.weekly_digest.keys import TeamDataKey, UserDataKey, org_digest_key, team_data_key, user_data_key
-from posthog.temporal.weekly_digest.queries import query_team_ids_for_digest, query_teams_for_digest
+from posthog.temporal.weekly_digest.queries import (
+ query_orgs_for_digest,
+ query_team_ids_for_digest,
+ query_teams_for_digest,
+)
from posthog.temporal.weekly_digest.types import (
DEFAULT_PRODUCT_SUGGESTION_TEXT,
CommonInput,
Digest,
GenerateDigestDataBatchInput,
GenerateOrganizationDigestInput,
+ OrganizationIdRange,
SendWeeklyDigestBatchInput,
TeamIdRange,
UsageTrends,
@@ -82,6 +89,7 @@ from products.surveys.backend.models import Survey
from products.warehouse_sources.backend.facade.models import ExternalDataSource
DJANGO_REDIS_URL = "redis://django-cache.example.com:6379"
+ALL_ORGANIZATIONS = OrganizationIdRange(start=UUID(int=0), end=None)
def run_sync(activity_fn: Callable[..., Any], *args: Any) -> Any:
@@ -170,6 +178,37 @@ def test_team_id_ranges_page_every_digest_team_exactly_once(organization, digest
assert paged == expected
+def test_organization_id_ranges_page_every_digest_organization_exactly_once(
+ organization, redis_servers, common_input, digest
+):
+ for i in range(4):
+ Organization.objects.create(name=f"digest org {i}")
+ if i == 1:
+ # Sits inside a batch range, so a leak here shows up as an extra paged organization.
+ Organization.objects.create(name="internal org", for_internal_metrics=True)
+
+ common = common_input.model_copy(update={"batch_size": 2})
+ expected = list(query_orgs_for_digest().values_list("id", flat=True))
+ # An odd count leaves a last range with no end, which must still reach the newest organization.
+ assert len(expected) % common.batch_size == 1
+
+ paged: list[UUID] = []
+ for organization_id_range in run_sync(list_organization_id_ranges, common):
+ redis_servers.digest.flushall()
+ run_sync(
+ generate_organization_digest_batch,
+ GenerateOrganizationDigestInput(organization_id_range=organization_id_range, digest=digest, common=common),
+ )
+ organization_ids = sorted(
+ UUID(json.loads(redis_servers.digest.get(key))["id"]) for key in redis_servers.digest.keys()
+ )
+ assert len(organization_ids) <= common.batch_size
+ paged.extend(organization_ids)
+
+ assert paged == expected
+
+
def _make_dashboards(team: Team, other_team: Team, digest: Digest) -> list[str]:
Dashboard.objects.create(team=team, name="New dashboard")
Dashboard.objects.create(team=team, name="Generated Dashboard: onboarding")
@@ -398,7 +437,7 @@ def _run_error_issue_lookup_without_occurrences(team_range: TeamIdRange, digest:
def _run_organization_digest_batch(team_range: TeamIdRange, digest: Digest, common: CommonInput) -> None:
run_sync(
generate_organization_digest_batch,
- GenerateOrganizationDigestInput(batch=(0, Organization.objects.count()), digest=digest, common=common),
+ GenerateOrganizationDigestInput(organization_id_range=ALL_ORGANIZATIONS, digest=digest, common=common),
)
@@ -626,7 +665,7 @@ def test_generate_organization_digest_batch_defaults_missing_team_data(
)
run_sync(
generate_organization_digest_batch,
- GenerateOrganizationDigestInput(batch=(0, Organization.objects.count()), digest=digest, common=common_input),
+ GenerateOrganizationDigestInput(organization_id_range=ALL_ORGANIZATIONS, digest=digest, common=common_input),
)
# One organization's malformed value skips that organization only.
@@ -673,9 +712,9 @@ def _org_digest_json(organization: Organization, team: Team) -> str:
def _send_input(
organization: Organization, digest: Digest, common: CommonInput, dry_run: bool
) -> SendWeeklyDigestBatchInput:
- organization_index = list(Organization.objects.order_by("id").values_list("id", flat=True)).index(organization.id)
+ organization_ids = list(query_orgs_for_digest().values_list("id", flat=True))
return SendWeeklyDigestBatchInput(
- batch=(organization_index, organization_index + 1),
+ organization_id_range=_cut_organization_id_ranges(organization_ids, 1)[organization_ids.index(organization.id)],
dry_run=dry_run,
allow_already_sent=False,
digest=digest,
@@ -813,8 +852,6 @@ def test_send_weekly_digest_batch_resend_reaches_only_listed_organizations(
session.post.return_value.status_code = 201
resend_input = _send_input(organization, digest, common_input, dry_run=False).model_copy(
update={
- # A resend batches over the listed organizations only.
- "batch": (0, 1),
"allow_already_sent": True,
"organization_ids": [str(organization.id) if listed else str(uuid4())],
"workflows_only": workflows_only,
diff --git a/posthog/temporal/tests/weekly_digest/test_workflows.py b/posthog/temporal/tests/weekly_digest/test_workflows.py
index 41dd7cc1bac8..a6b161c86bd1 100644
--- a/posthog/temporal/tests/weekly_digest/test_workflows.py
+++ b/posthog/temporal/tests/weekly_digest/test_workflows.py
@@ -22,7 +22,6 @@ from temporalio.testing import WorkflowEnvironment
from temporalio.worker import Replayer, UnsandboxedWorkflowRunner, Worker
from posthog.temporal.weekly_digest.activities import (
- count_organizations,
generate_dashboard_lookup,
generate_error_issue_lookup,
generate_event_definition_lookup,
@@ -37,6 +36,7 @@ from posthog.temporal.weekly_digest.activities import (
generate_survey_lookup,
generate_usage_trends_lookup,
generate_user_notification_lookup,
+ list_organization_id_ranges,
list_team_id_ranges,
)
from posthog.temporal.weekly_digest.types import (
@@ -45,6 +45,7 @@ from posthog.temporal.weekly_digest.types import (
GenerateDigestDataBatchInput,
GenerateDigestDataInput,
GenerateOrganizationDigestInput,
+ OrganizationIdRange,
SendWeeklyDigestBatchInput,
SendWeeklyDigestInput,
TeamIdRange,
@@ -131,20 +132,19 @@ class LegacyGenerateDigestDataWorkflow:
for team_id_range, generator in itertools.product(team_id_ranges, generators)
)
)
- organization_count = await workflow.execute_activity(
- count_organizations,
+ organization_id_ranges = await workflow.execute_activity(
+ list_organization_id_ranges,
+ input.common,
start_to_close_timeout=timedelta(minutes=5),
retry_policy=common.RetryPolicy(maximum_attempts=2, initial_interval=timedelta(minutes=1)),
heartbeat_timeout=timedelta(minutes=1),
)
- batch_size = input.common.batch_size
- org_batches = [(start, start + batch_size) for start in range(0, organization_count, batch_size)]
await asyncio.gather(
*(
workflow.execute_activity(
generate_organization_digest_batch,
GenerateOrganizationDigestInput(
- batch=batch,
+ organization_id_range=organization_id_range,
digest=input.digest,
common=input.common,
),
@@ -152,7 +152,7 @@ class LegacyGenerateDigestDataWorkflow:
retry_policy=common.RetryPolicy(maximum_attempts=2, initial_interval=timedelta(minutes=1)),
heartbeat_timeout=timedelta(minutes=2),
)
- for batch in org_batches
+ for organization_id_range in organization_id_ranges
)
)
@@ -235,22 +235,26 @@ def test_resolve_redis_host(_name: str, env_value: str | None, local_dev: bool,
@pytest.mark.asyncio
async def test_generate_digest_data_bounds_pending_activities(patched: bool) -> None:
team_ranges = [TeamIdRange(start=i, end=i + 1) for i in range(200)]
+ organization_ranges = [
+ OrganizationIdRange(start=uuid.UUID(int=0), end=uuid.UUID(int=2)),
+ OrganizationIdRange(start=uuid.UUID(int=2), end=None),
+ ]
pending = 0
peak_pending = 0
completed: list[tuple[object, int, int]] = []
- aggregated: list[tuple[int, int]] = []
+ aggregated: list[OrganizationIdRange] = []
async def execute_activity(activity_fn: object, activity_input: object = None, **_: object) -> object:
nonlocal pending, peak_pending
if activity_fn is list_team_id_ranges:
return team_ranges
- if activity_fn is count_organizations:
+ if activity_fn is list_organization_id_ranges:
assert pending == 0
assert len(completed) == 13 * len(team_ranges)
- return 3
+ return organization_ranges
if activity_fn is generate_organization_digest_batch:
assert isinstance(activity_input, GenerateOrganizationDigestInput)
- aggregated.append(activity_input.batch)
+ aggregated.append(activity_input.organization_id_range)
return None
assert isinstance(activity_input, GenerateDigestDataBatchInput)
@@ -282,7 +286,7 @@ async def test_generate_digest_data_bounds_pending_activities(patched: bool) ->
assert Counter((start, end) for _, start, end in completed) == {
(team_range.start, team_range.end): 13 for team_range in team_ranges
}
- assert aggregated == [(0, 2), (2, 4)]
+ assert aggregated == organization_ranges
if patched:
assert peak_pending <= MAX_CONCURRENT_GENERATION_ACTIVITIES < 2000
else:
@@ -314,9 +318,12 @@ async def test_generate_digest_data_replays_pre_patch_history(caplog: pytest.Log
async def list_team_id_ranges_mocked(_input: CommonInput) -> list[TeamIdRange]:
return team_ranges
- @activity.defn(name="count-organizations")
- async def count_organizations_mocked() -> int:
- return 3
+ @activity.defn(name="list-organization-id-ranges")
+ async def list_organization_id_ranges_mocked(_input: CommonInput) -> list[OrganizationIdRange]:
+ return [
+ OrganizationIdRange(start=uuid.UUID(int=0), end=uuid.UUID(int=2)),
+ OrganizationIdRange(start=uuid.UUID(int=2), end=None),
+ ]
@activity.defn(name="generate-organization-digest-batch")
async def generate_organization_digest_batch_mocked(_input: GenerateOrganizationDigestInput) -> None:
@@ -331,7 +338,7 @@ async def test_generate_digest_data_replays_pre_patch_history(caplog: pytest.Log
activities: list[Callable[..., Awaitable[object]]] = [
list_team_id_ranges_mocked,
- count_organizations_mocked,
+ list_organization_id_ranges_mocked,
generate_organization_digest_batch_mocked,
*(make_generator(name) for name in generator_names),
]
@@ -391,12 +398,12 @@ async def test_generate_digest_data_replays_pre_patch_history(caplog: pytest.Log
async def test_generate_digest_data_workflow():
"""Test the digest data generation workflow with batched activities."""
TEST_TEAM_COUNT = 5
- TEST_ORG_COUNT = 2
+ TEST_ORG_BATCH_COUNT = 2
TEST_BATCH_SIZE = 2
activity_calls = {
"team_id_ranges": 0,
- "count_organizations": 0,
+ "organization_id_ranges": 0,
"dashboard": 0,
"event_definition": 0,
"experiment_completed": 0,
@@ -418,10 +425,10 @@ async def test_generate_digest_data_workflow():
activity_calls["team_id_ranges"] += 1
return [TeamIdRange(start=i, end=i + TEST_BATCH_SIZE) for i in range(1, TEST_TEAM_COUNT + 1, TEST_BATCH_SIZE)]
- @activity.defn(name="count-organizations")
- async def count_organizations_mocked() -> int:
- activity_calls["count_organizations"] += 1
- return TEST_ORG_COUNT
+ @activity.defn(name="list-organization-id-ranges")
+ async def list_organization_id_ranges_mocked(input) -> list[OrganizationIdRange]:
+ activity_calls["organization_id_ranges"] += 1
+ return [OrganizationIdRange(start=uuid.UUID(int=i), end=None) for i in range(TEST_ORG_BATCH_COUNT)]
@activity.defn(name="generate-dashboard-lookup")
async def generate_dashboard_lookup_mocked(input) -> None:
@@ -487,7 +494,7 @@ async def test_generate_digest_data_workflow():
workflows=[GenerateDigestDataWorkflow],
activities=[
list_team_id_ranges_mocked,
- count_organizations_mocked,
+ list_organization_id_ranges_mocked,
generate_dashboard_lookup_mocked,
generate_event_definition_lookup_mocked,
generate_experiment_completed_lookup_mocked,
@@ -520,7 +527,7 @@ async def test_generate_digest_data_workflow():
)
assert activity_calls["team_id_ranges"] == 1
- assert activity_calls["count_organizations"] == 1
+ assert activity_calls["organization_id_ranges"] == 1
# Calculate expected batches for teams
expected_team_batches = (TEST_TEAM_COUNT + TEST_BATCH_SIZE - 1) // TEST_BATCH_SIZE
@@ -540,30 +547,37 @@ async def test_generate_digest_data_workflow():
assert activity_calls["error_issue"] == expected_team_batches
assert activity_calls["usage_trends"] == expected_team_batches
- # Calculate expected batches for organizations
- expected_org_batches = (TEST_ORG_COUNT + TEST_BATCH_SIZE - 1) // TEST_BATCH_SIZE
- assert activity_calls["org_digest"] == expected_org_batches
+ assert activity_calls["org_digest"] == TEST_ORG_BATCH_COUNT
@pytest.mark.asyncio
async def test_send_weekly_digest_workflow():
"""Test the digest sending workflow with batched activities."""
- TEST_ORG_COUNT = 10
TEST_BATCH_SIZE = 3
+ organization_ranges = [
+ OrganizationIdRange(start=uuid.UUID(int=0), end=uuid.UUID(int=3)),
+ OrganizationIdRange(start=uuid.UUID(int=3), end=uuid.UUID(int=6)),
+ OrganizationIdRange(start=uuid.UUID(int=6), end=None),
+ ]
activity_calls = {
- "count_organizations": 0,
+ "organization_id_ranges": 0,
"send_batch": 0,
}
+ sent_ranges: list[OrganizationIdRange] = []
- @activity.defn(name="count-organizations")
- async def count_organizations_mocked() -> int:
- activity_calls["count_organizations"] += 1
- return TEST_ORG_COUNT
+ @activity.defn(name="list-organization-id-ranges")
+ async def list_organization_id_ranges_mocked(
+ input: CommonInput, organization_ids: list[str] | None = None
+ ) -> list[OrganizationIdRange]:
+ activity_calls["organization_id_ranges"] += 1
+ assert input.batch_size == TEST_BATCH_SIZE
+ return organization_ranges
@activity.defn(name="send-weekly-digest-batch")
async def send_weekly_digest_batch_mocked(input: SendWeeklyDigestBatchInput) -> None:
activity_calls["send_batch"] += 1
+ sent_ranges.append(input.organization_id_range)
assert input.dry_run is True
assert input.digest.key == "test-digest"
@@ -574,7 +588,7 @@ async def test_send_weekly_digest_workflow():
task_queue=task_queue_name,
workflows=[SendWeeklyDigestWorkflow],
activities=[
- count_organizations_mocked,
+ list_organization_id_ranges_mocked,
send_weekly_digest_batch_mocked,
],
workflow_runner=temporalio.worker.UnsandboxedWorkflowRunner(),
@@ -595,11 +609,8 @@ async def test_send_weekly_digest_workflow():
task_queue=task_queue_name,
)
- assert activity_calls["count_organizations"] == 1
-
- # Calculate expected batches
- expected_batches = (TEST_ORG_COUNT + TEST_BATCH_SIZE - 1) // TEST_BATCH_SIZE
- assert activity_calls["send_batch"] == expected_batches
+ assert activity_calls["organization_id_ranges"] == 1
+ assert sorted(sent_ranges, key=lambda r: r.start) == organization_ranges
@pytest.mark.asyncio
diff --git a/posthog/temporal/weekly_digest/README.md b/posthog/temporal/weekly_digest/README.md
index eee2b0ccb576..1f4ffb2c9fa4 100644
--- a/posthog/temporal/weekly_digest/README.md
+++ b/posthog/temporal/weekly_digest/README.md
@@ -95,7 +95,7 @@ Generated via `user_data_key(digest_key, UserDataKey.*, user_id)`:
│ GenerateDigestDataWorkflow │
├─────────────────────────────────────────────────────────────────────────────┤
│ │
-│ 1. Cut teams into id-range batches, count orgs for batching │
+│ 1. Cut teams and orgs into id-range batches │
│ │
│ 2. Generate team-level data (parallel per batch): │
│ ├── generate_dashboard_lookup │
@@ -123,7 +123,7 @@ Generated via `user_data_key(digest_key, UserDataKey.*, user_id)`:
│ SendWeeklyDigestWorkflow │
├─────────────────────────────────────────────────────────────────────────────┤
│ │
-│ For each organization (batched): │
+│ For each organization (batched by id range): │
│ 1. Load OrganizationDigest from Redis │
│ 2. For each org member: │
│ a. Load user's notification team set │
diff --git a/posthog/temporal/weekly_digest/__init__.py b/posthog/temporal/weekly_digest/__init__.py
index 686f783933da..d0def072a68b 100644
--- a/posthog/temporal/weekly_digest/__init__.py
+++ b/posthog/temporal/weekly_digest/__init__.py
@@ -1,5 +1,4 @@
from posthog.temporal.weekly_digest.activities import (
- count_organizations,
generate_dashboard_lookup,
generate_error_issue_lookup,
generate_event_definition_lookup,
@@ -14,6 +13,7 @@ from posthog.temporal.weekly_digest.activities import (
generate_survey_lookup,
generate_usage_trends_lookup,
generate_user_notification_lookup,
+ list_organization_id_ranges,
list_team_id_ranges,
send_weekly_digest_batch,
)
@@ -39,7 +39,7 @@ ACTIVITIES = [
generate_feature_flag_lookup,
generate_user_notification_lookup,
generate_organization_digest_batch,
- count_organizations,
+ list_organization_id_ranges,
list_team_id_ranges,
send_weekly_digest_batch,
generate_filter_lookup,
diff --git a/posthog/temporal/weekly_digest/activities.py b/posthog/temporal/weekly_digest/activities.py
index 3b0281611c3f..5088744d6957 100644
--- a/posthog/temporal/weekly_digest/activities.py
+++ b/posthog/temporal/weekly_digest/activities.py
@@ -78,6 +78,7 @@ from posthog.temporal.weekly_digest.types import (
GenerateDigestDataBatchInput,
GenerateOrganizationDigestInput,
OrganizationDigest,
+ OrganizationIdRange,
PlaylistCount,
RecordingCount,
SendWeeklyDigestBatchInput,
@@ -869,18 +870,6 @@ def generate_product_suggestion_lookup(input: GenerateDigestDataBatchInput) -> N
_generate_product_suggestion_lookup(input)
-def _organizations_to_send(organization_ids: list[str] | None) -> QuerySet:
- organizations = query_orgs_for_digest()
- return organizations if organization_ids is None else organizations.filter(id__in=organization_ids)
-
-
[email protected](name="count-organizations")
-@asyncify
-def count_organizations(organization_ids: list[str] | None = None) -> int:
- with HeartbeaterSync(logger=LOGGER):
- return _organizations_to_send(organization_ids).count()
-
-
def _cut_team_id_ranges(team_ids: list[int], batch_size: int) -> list[TeamIdRange]:
"""Cut ordered team ids into [start, end) ranges of at most `batch_size` teams each."""
return [
@@ -900,6 +889,54 @@ def list_team_id_ranges(input: CommonInput) -> list[TeamIdRange]:
return _cut_team_id_ranges(list(query_team_ids_for_digest()), input.batch_size)
+def _cut_organization_id_ranges(organization_ids: list[UUID], batch_size: int) -> list[OrganizationIdRange]:
+ """Cut ordered organization ids into [start, end) ranges of at most `batch_size` organizations each."""
+ return [
+ OrganizationIdRange(
+ start=organization_ids[start],
+ end=organization_ids[start + batch_size] if start + batch_size < len(organization_ids) else None,
+ )
+ for start in range(0, len(organization_ids), batch_size)
+ ]
+
+
+def _organizations_to_send(organization_ids: list[str] | None) -> QuerySet:
+ organizations = query_orgs_for_digest()
+ return organizations if organization_ids is None else organizations.filter(id__in=organization_ids)
+
+
[email protected](name="list-organization-id-ranges")
+@asyncify
+def list_organization_id_ranges(
+ input: CommonInput, organization_ids: list[str] | None = None
+) -> list[OrganizationIdRange]:
+ """One index scan of the organization ids replaces a LIMIT/OFFSET scan per batch."""
+ with HeartbeaterSync(logger=LOGGER):
+ ordered_ids = list(_organizations_to_send(organization_ids).values_list("id", flat=True))
+ return _cut_organization_id_ranges(ordered_ids, input.batch_size)
+
+
+def _organizations_in_range(
+ organization_id_range: OrganizationIdRange, organization_ids: list[str] | None = None
+) -> QuerySet:
+ # A range also spans the unlisted organizations between listed ones, so a resend filters by its list again.
+ organizations = _organizations_to_send(organization_ids).filter(id__gte=organization_id_range.start)
+ if organization_id_range.end is not None:
+ organizations = organizations.filter(id__lt=organization_id_range.end)
+ return organizations
+
+
+def _bind_organization_batch_logger(
+ digest_key: str, organization_id_range: OrganizationIdRange
+) -> FilteringBoundLogger:
+ bind_contextvars(
+ digest_key=digest_key,
+ organization_id_start=str(organization_id_range.start),
+ organization_id_end=str(organization_id_range.end) if organization_id_range.end is not None else None,
+ )
+ return LOGGER.bind()
+
+
# Positions follow TeamDataKey order and the TeamDigest constructor below.
TEAM_DATA_DEFAULTS: list[tuple[TeamDataKey, Any]] = [
(TeamDataKey.DASHBOARDS, DashboardList(root=[])),
@@ -950,15 +987,13 @@ def _team_digest(team: Team, raw_values: list[str | None]) -> TeamDigest:
def _generate_organization_digest_batch(input: GenerateOrganizationDigestInput) -> None:
- bind_contextvars(digest_key=input.digest.key, batch_start=input.batch[0], batch_end=input.batch[1])
- logger = LOGGER.bind()
+ logger = _bind_organization_batch_logger(input.digest.key, input.organization_id_range)
logger.info("Generating organization-level digest batch")
organization_count = 0
team_count = 0
- batch_start, batch_end = input.batch
- organizations = list(query_orgs_for_digest()[batch_start:batch_end])
+ organizations = list(_organizations_in_range(input.organization_id_range))
teams_by_org: dict[UUID, list[Team]] = defaultdict(list)
for team in query_teams_for_organizations([organization.id for organization in organizations]):
teams_by_org[team.organization_id].append(team)
@@ -1153,8 +1188,7 @@ def _post_to_workflows(session: requests.Session, body: bytes, idempotency_key:
def _send_weekly_digest_batch(input: SendWeeklyDigestBatchInput) -> None:
- bind_contextvars(digest_key=input.digest.key, batch_start=input.batch[0], batch_end=input.batch[1])
- logger = LOGGER.bind()
+ logger = _bind_organization_batch_logger(input.digest.key, input.organization_id_range)
logger.info("Sending weekly digest batch")
sent_digest_count = 0
@@ -1199,8 +1233,7 @@ def _send_weekly_digest_batch(input: SendWeeklyDigestBatchInput) -> None:
try:
with _digest_redis(input.common) as r:
- batch_start, batch_end = input.batch
- for organization in _organizations_to_send(input.organization_ids)[batch_start:batch_end]:
+ for organization in _organizations_in_range(input.organization_id_range, input.organization_ids):
_raise_if_cancelled()
partial = False
try:
diff --git a/posthog/temporal/weekly_digest/types.py b/posthog/temporal/weekly_digest/types.py
index 7f56346b3dfd..e9482e8e265e 100644
--- a/posthog/temporal/weekly_digest/types.py
+++ b/posthog/temporal/weekly_digest/types.py
@@ -58,8 +58,15 @@ class GenerateDigestDataBatchInput(BaseModel):
common: CommonInput
+class OrganizationIdRange(BaseModel):
+ """A half-open [start, end) range of organization ids. The last range has no end."""
+
+ start: UUID
+ end: UUID | None
+
+
class GenerateOrganizationDigestInput(BaseModel):
- batch: tuple[int, int]
+ organization_id_range: OrganizationIdRange
digest: Digest
common: CommonInput
@@ -74,7 +81,7 @@ class SendWeeklyDigestInput(BaseModel):
class SendWeeklyDigestBatchInput(BaseModel):
- batch: tuple[int, int]
+ organization_id_range: OrganizationIdRange
dry_run: bool
allow_already_sent: bool
digest: Digest
diff --git a/posthog/temporal/weekly_digest/workflows.py b/posthog/temporal/weekly_digest/workflows.py
index e5feae7900a9..0fb6677874e8 100644
--- a/posthog/temporal/weekly_digest/workflows.py
+++ b/posthog/temporal/weekly_digest/workflows.py
@@ -13,7 +13,6 @@ from temporalio.exceptions import ApplicationError
from posthog.temporal.common.base import PostHogWorkflow
from posthog.temporal.weekly_digest.activities import (
- count_organizations,
generate_dashboard_lookup,
generate_error_issue_lookup,
generate_event_definition_lookup,
@@ -28,6 +27,7 @@ from posthog.temporal.weekly_digest.activities import (
generate_survey_lookup,
generate_usage_trends_lookup,
generate_user_notification_lookup,
+ list_organization_id_ranges,
list_team_id_ranges,
send_weekly_digest_batch,
)
@@ -195,8 +195,9 @@ class GenerateDigestDataWorkflow(PostHogWorkflow):
)
)
- organization_count = await workflow.execute_activity(
- count_organizations,
+ organization_id_ranges = await workflow.execute_activity(
+ list_organization_id_ranges,
+ input.common,
start_to_close_timeout=timedelta(minutes=5),
retry_policy=common.RetryPolicy(
maximum_attempts=2,
@@ -205,15 +206,12 @@ class GenerateDigestDataWorkflow(PostHogWorkflow):
heartbeat_timeout=timedelta(minutes=1),
)
- batch_size = input.common.batch_size
- org_batches = [(i, i + batch_size) for i in range(0, organization_count, batch_size)]
-
await asyncio.gather(
*[
workflow.execute_activity(
generate_organization_digest_batch,
GenerateOrganizationDigestInput(
- batch=batch,
+ organization_id_range=organization_id_range,
digest=input.digest,
common=input.common,
),
@@ -224,7 +222,7 @@ class GenerateDigestDataWorkflow(PostHogWorkflow):
),
heartbeat_timeout=timedelta(minutes=2),
)
- for batch in org_batches
+ for organization_id_range in organization_id_ranges
]
)
@@ -238,10 +236,9 @@ class SendWeeklyDigestWorkflow(PostHogWorkflow):
@workflow.run
async def run(self, input: SendWeeklyDigestInput) -> None:
- # Only a resend passes its organizations, so the scheduled run calls the activity exactly as before.
- organization_count = await workflow.execute_activity(
- count_organizations,
- args=[input.organization_ids] if input.organization_ids is not None else [],
+ organization_id_ranges = await workflow.execute_activity(
+ list_organization_id_ranges,
+ args=[input.common, input.organization_ids],
start_to_close_timeout=timedelta(minutes=5),
retry_policy=common.RetryPolicy(
maximum_attempts=2,
@@ -250,15 +247,12 @@ class SendWeeklyDigestWorkflow(PostHogWorkflow):
heartbeat_timeout=timedelta(minutes=1),
)
- batch_size = input.common.batch_size
- batches = [(i, i + batch_size) for i in range(0, organization_count, batch_size)]
-
await asyncio.gather(
*[
workflow.execute_activity(
send_weekly_digest_batch,
SendWeeklyDigestBatchInput(
- batch=batch,
+ organization_id_range=organization_id_range,
dry_run=input.dry_run,
allow_already_sent=input.allow_already_sent,
digest=input.digest,
@@ -274,6 +268,6 @@ class SendWeeklyDigestWorkflow(PostHogWorkflow):
),
heartbeat_timeout=timedelta(minutes=2),
)
- for batch in batches
+ for organization_id_range in organization_id_ranges
]
)