:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(workflows): send a batch run once its realtime cohort is calculated (#117704)
Harley Alexander7505ecdab222
diff --git a/posthog/jwt.py b/posthog/jwt.pyindex 7909419b1fc4..b7d3b7a600c4 100644--- a/posthog/jwt.py+++ b/posthog/jwt.py@@ -32,6 +32,7 @@ class PosthogJwtAudience(Enum):WORKFLOWS_CANCEL_BATCH = "posthog:workflows:cancel_batch"WORKFLOWS_STEP_RESUME = "posthog:workflows:step_resume"WORKFLOWS_SEND_TRANSACTIONAL_EMAIL = "posthog:workflows:send_transactional_email"+ WORKFLOWS_RESUME_BATCH = "posthog:workflows:resume_batch"CREDENTIALS_SERVICE = "posthog:credentials_service"TASKS_CREATE = "posthog:tasks:create"CUSTOMER_TASKS_CREATE = "posthog:customer-tasks:create"diff --git a/posthog/plugins/plugin_server_api.py b/posthog/plugins/plugin_server_api.pyindex 42c77d0118c2..4176b4f87395 100644--- a/posthog/plugins/plugin_server_api.py+++ b/posthog/plugins/plugin_server_api.py@@ -255,6 +255,15 @@ WORKFLOWS_TRANSACTIONAL_EMAIL_JWT_PURPOSE = ScopedServiceJwtPurpose()+# The workers that wake parked steps also wake parked batch resolvers, so the two share a key. The+# audience keeps a token for one from waking the other.+WORKFLOWS_RESUME_BATCH_JWT_PURPOSE = ScopedServiceJwtPurpose(+ audience=PosthogJwtAudience.WORKFLOWS_RESUME_BATCH,+ settings_name="WORKFLOWS_STEP_RESUME_JWT_SECRETS",+ default_ttl=timedelta(minutes=2),+)++def _mint_transactional_email_jwt(team_id: int, hog_flow_id: str) -> str:"""Short-lived scoped JWT for one transactional email send, pinned to the team and the transactional email'sworkflow so a leaked token can only send that email. Verified in the plugin server's CdpApi."""@@ -274,6 +283,22 @@ def send_transactional_email(team_id: int, hog_flow_id: str, payload: dict) -> r)+def resume_hog_flow_batch_job(team_id: int, hog_flow_id: str, batch_job_id: str) -> requests.Response:+ """Wake a batch run's resolver, parked until its realtime cohort finished calculating. The JSON+ `outcome` is `resumed` or `already_resumed` on success; a 409 means the resolver isn't parked."""+ if not WORKFLOWS_RESUME_BATCH_JWT_PURPOSE.enabled():+ raise RuntimeError("WORKFLOWS_STEP_RESUME_JWT_SECRET is not configured — cannot resume a batch run")+ token = WORKFLOWS_RESUME_BATCH_JWT_PURPOSE.mint(+ {"team_id": team_id, "hog_flow_id": hog_flow_id, "batch_job_id": batch_job_id}+ )+ return internal_requests.post(+ CDP_API_URL + f"/api/projects/{team_id}/hog_flows/{hog_flow_id}/batch_jobs/{batch_job_id}/resume",+ json={},+ headers={"Authorization": f"Bearer {token}"},+ timeout=10,+ )++def cancel_hog_flow_invocations(team_id: int, hog_flow_id: str, payload: dict) -> requests.Response:"""Flag a workflow's in-flight invocations for cancellation. `payload` carries exactly oneselector: {"invocation_ids": [...]} or {"all": true}. The Node side only marks rows and wakes@@ -358,6 +383,8 @@ def create_batch_hog_flow_job_invocation(CDP_API_URL + f"/api/projects/{team_id}/hog_flows/{hog_flow_id}/batch_invocations/{batch_job_id}",json={"max_audience_size": max_audience_size, "filters": filters, "realtime_cohort": realtime_cohort},headers=get_internal_api_headers(),+ # A run waits on this call while it is created, so a stalled CDP API must not hold it open.+ timeout=30,)diff --git a/products/cohorts/backend/backfill/finalize.py b/products/cohorts/backend/backfill/finalize.pyindex 135a8351b7c5..a4395308d8ea 100644--- a/products/cohorts/backend/backfill/finalize.py+++ b/products/cohorts/backend/backfill/finalize.py@@ -34,6 +34,7 @@ from posthog.tasks.utils import CeleryQueuefrom products.cohorts.backend.backfill.allowlist import parse_run_allowlistfrom products.cohorts.backend.backfill.readiness import stamp_events_readiness, stamp_person_properties_readiness+from products.cohorts.backend.facade.ready_signals import realtime_cohort_readyfrom products.cohorts.backend.models.backfill import (READINESS_CHUNKS,CohortBackfillChunk,@@ -44,6 +45,7 @@ from products.cohorts.backend.models.backfill import (CohortBackfillRunStatus,CohortBackfillScope,)+from products.cohorts.backend.models.cohort import Cohortfrom products.cohorts.backend.models.dependencies import invalidate_team_behavioral_cohort_cachelogger = structlog.get_logger(__name__)@@ -192,15 +194,19 @@ def finalize_backfill_runs() -> FinalizerPass:observed.extend(discovered.order_by("reconcile_observed_at").values_list("id", "team_id")[:per_kind])teams_to_invalidate: set[int] = set()+ stamped_cohorts: list[tuple[int, int]] = []for run_id, team_id in observed:+ run_stamped: list[int] = []try:- invalidate_team = _finalize_one_run(run_id, team_id, result)+ invalidate_team = _finalize_one_run(run_id, team_id, result, run_stamped)except Exception:# The transaction rolled back, so the run is still reconciling — count it so the pager# gauge reflects a stuck run rather than reading clean.logger.exception("cohort_backfill_finalizer_run_error", run_id=str(run_id), team_id=team_id)result.errored += 1continue+ # Reached only when the run's transaction committed, so a rolled-back stamp never signals.+ stamped_cohorts.extend((team_id, cohort_id) for cohort_id in run_stamped)if invalidate_team:teams_to_invalidate.add(team_id)@@ -215,6 +221,18 @@ def finalize_backfill_runs() -> FinalizerPass:continueresult.invalidated_teams += 1+ for team_id, cohort_id in stamped_cohorts:+ for _receiver, response in realtime_cohort_ready.send_robust(+ sender=Cohort, team_id=team_id, cohort_id=cohort_id+ ):+ if isinstance(response, Exception):+ logger.error(+ "cohort_backfill_finalizer_ready_receiver_failed",+ team_id=team_id,+ cohort_id=cohort_id,+ error=str(response),+ )+_publish_held_runs(result)return result@@ -226,8 +244,9 @@ def _publish_held_runs(result: FinalizerPass) -> None:HELD_RUNS_GAUGE.labels(reason="not_allowlisted").set(result.not_allowlisted)-def _finalize_one_run(run_id: UUID, team_id: int, result: FinalizerPass) -> bool:- """Terminalize one observed run. Returns whether the team's caches need invalidating."""+def _finalize_one_run(run_id: UUID, team_id: int, result: FinalizerPass, stamped_cohort_ids: list[int]) -> bool:+ """Terminalize one observed run. Returns whether the team's caches need invalidating, and adds+ each cohort it stamps to ``stamped_cohort_ids``."""stamped = 0superseded = 0held = 0@@ -273,6 +292,7 @@ def _finalize_one_run(run_id: UUID, team_id: int, result: FinalizerPass) -> boolif stamp_readiness(run, participation.cohort_id):stamped += 1invalidate_team = True+ stamped_cohort_ids.append(participation.cohort_id)result.stamped_participations += 1READINESS_STAMPS_COUNTER.labels(result="stamped", kind=run.backfill_kind).inc()else:diff --git a/products/cohorts/backend/backfill/test/test_finalize.py b/products/cohorts/backend/backfill/test/test_finalize.pyindex 3d39024f26bb..11ed9ede464c 100644--- a/products/cohorts/backend/backfill/test/test_finalize.py+++ b/products/cohorts/backend/backfill/test/test_finalize.py@@ -3,6 +3,7 @@ from datetime import timedeltafrom posthog.test.base import BaseTestfrom unittest import mock+from unittest.mock import MagicMockfrom django.core.cache import cachefrom django.db import connection@@ -16,6 +17,7 @@ from posthog.tasks.calculate_cohort import finalize_cohort_backfill_runsfrom products.cohorts.backend.backfill.finalize import _STAMP_BY_KIND, FLAGS_CACHE_TASK, finalize_backfill_runsfrom products.cohorts.backend.backfill.readiness import ensure_filters_shape_hash+from products.cohorts.backend.facade.ready_signals import realtime_cohort_readyfrom products.cohorts.backend.models.backfill import (CohortBackfillChunk,CohortBackfillChunkStatus,@@ -170,11 +172,19 @@ class TestBackfillFinalizer(BaseTest):self, _name: str, outcomes: list[str], expected_status: str, stamped_indices: list[int]) -> None:run, cohorts = self._make_run(outcomes)+ ready_receiver = MagicMock()+ realtime_cohort_ready.connect(ready_receiver, dispatch_uid="test_outcome_matrix")+ self.addCleanup(realtime_cohort_ready.disconnect, dispatch_uid="test_outcome_matrix")finalize_backfill_runs()run.refresh_from_db()self.assertEqual(run.status, expected_status)+ # Workflow runs parked on a cohort wake on this signal, so it fires for exactly the stamped cohorts.+ self.assertEqual(+ sorted(call.kwargs["cohort_id"] for call in ready_receiver.call_args_list),+ sorted(cohorts[index].pk for index in stamped_indices),+ )for index, cohort in enumerate(cohorts):cohort.refresh_from_db()participation = CohortBackfillRunCohort.objects.for_team(self.team.id).get(run=run, cohort=cohort)diff --git a/products/cohorts/backend/facade/ready_signals.py b/products/cohorts/backend/facade/ready_signals.pynew file mode 100644index 000000000000..af662c5bb021--- /dev/null+++ b/products/cohorts/backend/facade/ready_signals.py@@ -0,0 +1,14 @@+from collections.abc import Callable++from django.dispatch import Signal++# Sent with `team_id=` and `cohort_id=` once a realtime cohort's backfill readiness stamp commits, so+# other products react to it without the cohorts product importing them. A cohort that needs both+# kinds of backfill can still lack the other stamp, so a receiver checks readiness itself.+realtime_cohort_ready = Signal()+++def connect_realtime_cohort_ready(receiver: Callable[..., object], *, dispatch_uid: str) -> None:+ """``receiver`` gets ``team_id=`` and ``cohort_id=`` after a readiness stamp commits. A receiver+ that raises is logged and never fails the finalizer."""+ realtime_cohort_ready.connect(receiver, dispatch_uid=dispatch_uid)diff --git a/products/workflows/backend/facade/contracts.py b/products/workflows/backend/facade/contracts.pyindex 34986631d870..4578b6ae2184 100644--- a/products/workflows/backend/facade/contracts.py+++ b/products/workflows/backend/facade/contracts.py@@ -86,6 +86,8 @@ class RealtimeAudienceCohort:id: intname: strready: bool+ # False when the realtime pipeline doesn't run for the cohort's project, so it never becomes ready.+ can_finish: bool@frozen@@ -212,6 +214,17 @@ class AccountAudienceProvider(Protocol):def get_account_group_type_name(self, team: "Team") -> str | None: ...+@frozen+class RealtimeCohortStatus:+ """How far a realtime cohort audience is from having its membership calculated."""++ id: int+ name: str+ # A CohortRealtimeState value: "ready", "building", "rebuilding" or "needs_attention".+ state: str+ percent_complete: int | None++@frozenclass AudienceSize:"""How many recipients a batch audience matches, and the most a batch trigger may send to."""@@ -220,6 +233,8 @@ class AudienceSize:total: intlimit: intdedupe_key: str | None+ # Set when the audience is one realtime cohort. A send waits until that cohort is ready.+ realtime_cohort: RealtimeCohortStatus | None = None@frozendiff --git a/products/workflows/backend/facade/enums.py b/products/workflows/backend/facade/enums.pyindex 12ec8b8109a7..c10ba1a2b730 100644--- a/products/workflows/backend/facade/enums.py+++ b/products/workflows/backend/facade/enums.py@@ -126,3 +126,12 @@ class SetupStepKey(LabeledStrEnum):SENDING_RULES = "sending-rules", "Sending rules"EMAIL_TEMPLATE = "email-template", "Email template"FIRST_WORKFLOW = "first-workflow", "First workflow"+++class RealtimeAudienceCohortState(LabeledStrEnum):+ """How far a realtime cohort audience's membership has come, as the audience size response reports it."""++ READY = "ready", "Ready"+ BUILDING = "building", "Building"+ REBUILDING = "rebuilding", "Rebuilding"+ NEEDS_ATTENTION = "needs_attention", "Needs attention"diff --git a/products/workflows/backend/facade/tasks.py b/products/workflows/backend/facade/tasks.pyindex f4258abea162..69f264323ab7 100644--- a/products/workflows/backend/facade/tasks.py+++ b/products/workflows/backend/facade/tasks.py@@ -1,3 +1,4 @@+from products.workflows.backend.tasks.batch_jobs import resume_batch_jobs_waiting_for_realtime_cohortfrom products.workflows.backend.tasks.email_sending_tiers import recompute_workflows_email_sending_tiersfrom products.workflows.backend.tasks.hog_flows import refresh_affected_hog_flowsfrom products.workflows.backend.tasks.ses_account_reputation import (@@ -8,6 +9,7 @@ from products.workflows.backend.tasks.ses_tenant_state import reconcile_ses_tenafrom products.workflows.backend.tasks.workflow_email_health import sweep_workflow_email_deliverability__all__ = [+ "resume_batch_jobs_waiting_for_realtime_cohort","poll_ses_account_enforcement","poll_ses_reputation_findings","recompute_workflows_email_sending_tiers",diff --git a/products/workflows/backend/facade/validation.py b/products/workflows/backend/facade/validation.pyindex c668d94f5ac8..57db6c1958bf 100644--- a/products/workflows/backend/facade/validation.py+++ b/products/workflows/backend/facade/validation.py@@ -18,7 +18,17 @@ def _to_realtime_audience_cohort(cohort: Cohort | None) -> RealtimeAudienceCohorif cohort is None:return Nonereturn RealtimeAudienceCohort(- id=cohort.pk, name=str(cohort.name), ready=batch_audience_cohorts.realtime_cohort_is_ready(cohort)+ id=cohort.pk,+ name=str(cohort.name),+ ready=batch_audience_cohorts.realtime_cohort_is_ready(cohort),+ can_finish=batch_audience_cohorts.realtime_cohort_can_finish(cohort),+ )+++def realtime_cohort_cant_finish_error(cohort: RealtimeAudienceCohort) -> str:+ return (+ f"Cohort '{cohort.name}' can't be calculated in real time for this project, so a send to it would never "+ "start. Use another cohort.")@@ -42,4 +52,5 @@ __all__ = ["is_duration","is_signed_duration","realtime_audience_cohort",+ "realtime_cohort_cant_finish_error",]diff --git a/products/workflows/backend/migrations/0035_hogflowbatchjob_waiting_index.py b/products/workflows/backend/migrations/0035_hogflowbatchjob_waiting_index.pynew file mode 100644index 000000000000..484dee8805e7--- /dev/null+++ b/products/workflows/backend/migrations/0035_hogflowbatchjob_waiting_index.py@@ -0,0 +1,22 @@+from django.db import migrations, models++from posthog.migration_helpers import SafeAddIndexConcurrently+++class Migration(migrations.Migration):+ atomic = False++ dependencies = [+ ("workflows", "0034_teamworkflowsconfig_replay_vision_scan_limits"),+ ]++ operations = [+ SafeAddIndexConcurrently(+ model_name="hogflowbatchjob",+ index=models.Index(+ condition=models.Q(("status", "waiting")),+ fields=["created_at"],+ name="hogflowbatchjob_waiting_idx",+ ),+ ),+ ]diff --git a/products/workflows/backend/migrations/max_migration.txt b/products/workflows/backend/migrations/max_migration.txtindex 852bd44b6c8d..4081a0c3419b 100644--- a/products/workflows/backend/migrations/max_migration.txt+++ b/products/workflows/backend/migrations/max_migration.txt@@ -1 +1 @@-0034_teamworkflowsconfig_replay_vision_scan_limits+0035_hogflowbatchjob_waiting_indexdiff --git a/products/workflows/backend/models/hog_flow_batch_job/hog_flow_batch_job.py b/products/workflows/backend/models/hog_flow_batch_job/hog_flow_batch_job.pyindex 0b6527005fb3..ce1b928570b3 100644--- a/products/workflows/backend/models/hog_flow_batch_job/hog_flow_batch_job.py+++ b/products/workflows/backend/models/hog_flow_batch_job/hog_flow_batch_job.py@@ -1,7 +1,11 @@-from django.db import models+from datetime import timedelta+from typing import TYPE_CHECKING++from django.db import models, transactionfrom django.db.models.signals import post_savefrom django.dispatch import receiver+import requestsimport structlogfrom posthog.models.utils import RootTeamMixin, UUIDTModel@@ -14,6 +18,9 @@ from products.workflows.backend.services.batch_audience_cohorts import ()from products.workflows.backend.utils.batch_trigger_limit import get_hogflow_batch_trigger_limit, hog_flow_sends_email+if TYPE_CHECKING:+ from products.cohorts.backend.models.cohort import Cohort+logger = structlog.get_logger(__name__)@@ -25,6 +32,12 @@ class HogFlowBatchJob(RootTeamMixin, UUIDTModel):class Meta:indexes = [models.Index(fields=["team"]),+ # The minute-by-minute sweep reads only waiting runs, across every team.+ models.Index(+ fields=["created_at"],+ name="hogflowbatchjob_waiting_idx",+ condition=models.Q(status="waiting"),+ ),]State = HogFlowBatchJobState@@ -54,43 +67,72 @@ def batch_job_audience_filters(job: HogFlowBatchJob) -> dict:}+# A run waits this long for its realtime cohort to finish calculating, then fails.+REALTIME_COHORT_WAIT_LIMIT = timedelta(hours=24)+++def dispatch_batch_job(+ job: HogFlowBatchJob, realtime_cohort: "Cohort | None", *, wait_for_cohort: bool = False+) -> "requests.Response":+ cohort_target: dict | None = None+ if realtime_cohort is not None:+ cohort_target = {"id": realtime_cohort.pk, "team_id": realtime_cohort.team_id}+ if wait_for_cohort:+ # The resolver parks until a resume wakes it, and fails the run at this instant otherwise.+ cohort_target["wait_until"] = (job.created_at + REALTIME_COHORT_WAIT_LIMIT).isoformat()+ return create_batch_hog_flow_job_invocation(+ team_id=job.team.id,+ hog_flow_id=job.hog_flow.id,+ batch_job_id=job.id,+ max_audience_size=get_hogflow_batch_trigger_limit(+ job.team.id,+ # The dispatch runs the live config, so the live actions decide the channel.+ # Adding an email step after this write does not re-cap the queued batch; that+ # gap is bounded by the send-time buckets, which cap every email at execution+ # regardless of the audience size dispatched here.+ sends_email=hog_flow_sends_email(job.hog_flow.actions),+ ),+ # The audience snapshot the confirm check validated - the resolver dispatches from+ # this, not the live trigger, so a trigger edit racing the dispatch can't widen it.+ filters=job.filters,+ realtime_cohort=cohort_target,+ )++@receiver(post_save, sender=HogFlowBatchJob)def handle_hog_flow_batch_job_created(sender, instance, created, **kwargs):if created:realtime_cohort = realtime_audience_cohort(instance.team.project_id, batch_job_audience_filters(instance))- if realtime_cohort is not None and not realtime_cohort_is_ready(realtime_cohort):- # An edit to the cohort since the save cleared its readiness. Its membership is partial until- # it finishes calculating again, so the run fails instead of sending to the wrong people.- logger.warning(- "Batch run audience realtime cohort isn't ready", batch_job_id=instance.id, cohort_id=realtime_cohort.pk- )- HogFlowBatchJob.objects.filter(pk=instance.pk).update(status=HogFlowBatchJobState.FAILED)- instance.status = HogFlowBatchJobState.FAILED- return+ # The cohort's membership is partial until it finishes calculating, so its resolver parks until+ # the cohort's ready signal resumes it.+ wait_for_cohort = realtime_cohort is not None and not realtime_cohort_is_ready(realtime_cohort)+ if wait_for_cohort:+ HogFlowBatchJob.objects.filter(pk=instance.pk).update(status=HogFlowBatchJobState.WAITING)+ instance.status = HogFlowBatchJobState.WAITINGtry:- create_batch_hog_flow_job_invocation(- team_id=instance.team.id,- hog_flow_id=instance.hog_flow.id,- batch_job_id=instance.id,- max_audience_size=get_hogflow_batch_trigger_limit(- instance.team.id,- # The dispatch runs the live config, so the live actions decide the channel.- # Adding an email step after this write does not re-cap the queued batch; that- # gap is bounded by the send-time buckets, which cap every email at execution- # regardless of the audience size dispatched here.- sends_email=hog_flow_sends_email(instance.hog_flow.actions),- ),- # The audience snapshot the confirm check validated - the resolver dispatches from- # this, not the live trigger, so a trigger edit racing the dispatch can't widen it.- filters=instance.filters,- realtime_cohort=(- {"id": realtime_cohort.pk, "team_id": realtime_cohort.team_id} if realtime_cohort else None- ),- )+ dispatch_batch_job(instance, realtime_cohort, wait_for_cohort=wait_for_cohort).raise_for_status()except Exception as e:logger.exception("Failed to create batch hogflow job invocation",batch_job_id=instance.id,error=str(e),)+ # A rejected request or one that never connected queued no resolver, so nothing would ever move the+ # run on. Any other failure may have queued one, so the run stays stoppable.+ if isinstance(e, requests.HTTPError | requests.ConnectTimeout):+ HogFlowBatchJob.objects.filter(pk=instance.pk).update(status=HogFlowBatchJobState.FAILED)+ instance.status = HogFlowBatchJobState.FAILEDraise+ if wait_for_cohort and realtime_cohort is not None:+ # The cohort can finish between the check above and the park, and its ready signal then finds+ # no parked resolver. A resume check after the park covers that; it does nothing while the+ # cohort still calculates. A failed publish only logs, because the send is already accepted and a+ # retried request would create a second run.+ from products.workflows.backend.tasks.batch_jobs import ( # noqa: PLC0415 — the task module imports this model+ resume_batch_jobs_waiting_for_realtime_cohort,+ )++ team_id, cohort_id = realtime_cohort.team_id, realtime_cohort.pk+ transaction.on_commit(+ lambda: resume_batch_jobs_waiting_for_realtime_cohort.delay(team_id, cohort_id), robust=True+ )diff --git a/products/workflows/backend/presentation/views/hog_flow.py b/products/workflows/backend/presentation/views/hog_flow.pyindex a1d543ed590e..97c1dca356f1 100644--- a/products/workflows/backend/presentation/views/hog_flow.py+++ b/products/workflows/backend/presentation/views/hog_flow.py@@ -214,6 +214,7 @@ from products.workflows.backend.facade.enums import (HogFlowOriginProduct,HogFlowScheduleStatus,HogFlowState,+ RealtimeAudienceCohortState,WorkflowProposalStatus,)from products.workflows.backend.facade.message_assets import fetch_message_asset_html, fetch_message_assets@@ -278,6 +279,7 @@ from products.workflows.backend.facade.validation import (is_duration,is_signed_duration,realtime_audience_cohort,+ realtime_cohort_cant_finish_error,)from products.workflows.backend.facade.workflows import (BILLABLE_ACTION_TYPES,@@ -1052,6 +1054,21 @@ class BlastRadiusRequestSerializer(serializers.Serializer):)+class RealtimeCohortStatusSerializer(serializers.Serializer):+ id = serializers.IntegerField(help_text="The realtime cohort the audience sends to.")+ name = serializers.CharField(help_text="The cohort's name.")+ state = serializers.ChoiceField(+ choices=RealtimeAudienceCohortState.choices,+ help_text=(+ "'ready' when the cohort's membership is calculated. 'building' or 'rebuilding' while it is "+ "still calculating, when a send waits for it. 'needs_attention' when its calculation is stuck."+ ),+ )+ percent_complete = serializers.IntegerField(+ allow_null=True, help_text="How far the calculation has come, while the cohort is not ready."+ )++class BlastRadiusSerializer(serializers.Serializer):affected = serializers.IntegerField(help_text="Number of users matching the filters")total = serializers.IntegerField(help_text="Total number of users")@@ -1067,6 +1084,14 @@ class BlastRadiusSerializer(serializers.Serializer):"echoing 'affected' to the user. Signs these exact filters; expires in 15 minutes."),)+ realtime_cohort = RealtimeCohortStatusSerializer(+ required=False,+ allow_null=True,+ help_text=(+ "Set when the audience is one realtime cohort. 'affected' is then the cohort's last calculated "+ "size, and a batch run waits until the cohort is ready."+ ),+ )class InternalBlastRadiusPersonsSerializer(serializers.Serializer):@@ -1450,15 +1475,12 @@ class HogFlowActionSerializer(serializers.Serializer):)})- if not realtime_cohort.ready:- raise serializers.ValidationError(- {- "filters": (- f"Cohort '{realtime_cohort.name}' isn't ready yet. You can send to it once it "- "finishes calculating."- )- }- )+ # A stored audience stays editable after its team leaves the realtime pipeline; its runs are refused.+ if not realtime_cohort.can_finish and realtime_cohort.id not in (+ self.context.get("stored_realtime_audience_cohort_ids") or set()+ ):+ raise serializers.ValidationError({"filters": realtime_cohort_cant_finish_error(realtime_cohort)})+ # A cohort that is still calculating is accepted: a batch run waits until it is ready.returncohort_name = find_behavioral_cohort_name(project_id, cohort_ids)@@ -6173,6 +6195,11 @@ class HogFlowViewSet(dedupe_key=dedupe_key,sends_email=params["sends_email"],)+ # The cohort API denies a scoped token without cohort:read, so this response withholds the+ # cohort's name and build state from it too.+ token_scopes = get_authenticator_scopes(request.successful_authenticator)+ reads_cohorts = token_scopes is None or bool({"*", "cohort:read", "cohort:write"} & set(token_scopes))+ realtime_cohort = size.realtime_cohort if reads_cohorts else Nonereturn Response(BlastRadiusSerializer({@@ -6183,6 +6210,16 @@ class HogFlowViewSet("confirm_token": mint_audience_confirm_token(self.team_id, filters, group_type_index, size.dedupe_key),+ "realtime_cohort": (+ {+ "id": realtime_cohort.id,+ "name": realtime_cohort.name,+ "state": realtime_cohort.state,+ "percent_complete": realtime_cohort.percent_complete,+ }+ if realtime_cohort+ else None+ ),}).data)@@ -6721,6 +6758,9 @@ class HogFlowViewSet(serializer = HogFlowBatchJobSerializer(data={**request.data, "hog_flow": hog_flow.id})if not serializer.is_valid():return Response(serializer.errors, status=400)+ audience_cohort = realtime_audience_cohort(self.team.project_id, hog_flow.trigger_filters)+ if audience_cohort is not None and not audience_cohort.can_finish:+ raise exceptions.ValidationError(realtime_cohort_cant_finish_error(audience_cohort))self._require_replay_vision_caller_access(request, self._edit_state().actions)batch_job = create_batch_job(@@ -6780,7 +6820,6 @@ class HogFlowViewSet(}if batch_job.status not in non_terminal:return Response({"status": batch_job.status, "marked": 0, "remaining": 0, "done": True})-# One CDP call flags a bounded chunk of rows; a large batch needs several. Same# shape as cancel_invocations — `done: false` after the cap tells the caller to# request again for the rest.diff --git a/products/workflows/backend/presentation/views/hog_flow_batch_job.py b/products/workflows/backend/presentation/views/hog_flow_batch_job.pyindex f079f2f745f3..f773bc784ebc 100644--- a/products/workflows/backend/presentation/views/hog_flow_batch_job.py+++ b/products/workflows/backend/presentation/views/hog_flow_batch_job.py@@ -28,6 +28,14 @@ class HogFlowBatchJobSerializer(serializers.Serializer):created_by = UserBasicSerializer(read_only=True, help_text="User who started the batch run.")updated_at = serializers.DateTimeField(read_only=True, help_text="When the batch run was last updated.")+ def validate_status(self, value: str) -> str:+ # A waiting run is one PostHog has not dispatched yet, and stopping it skips the workers.+ if value == HogFlowBatchJobState.WAITING:+ raise serializers.ValidationError(+ "PostHog sets 'waiting' itself while the audience's cohort calculates. Leave the status out."+ )+ return value+class HogFlowBatchJobCancelResponseSerializer(serializers.Serializer):"""diff --git a/products/workflows/backend/receivers.py b/products/workflows/backend/receivers.pyindex a3e8e17251f4..20dbbec60c4a 100644--- a/products/workflows/backend/receivers.py+++ b/products/workflows/backend/receivers.py@@ -5,6 +5,7 @@ from django.dispatch import receiverfrom posthog.exceptions_capture import capture_exceptionfrom posthog.models import Team+from products.cohorts.backend.facade.ready_signals import connect_realtime_cohort_readyfrom products.workflows.backend.models import HogFlow, HogFlowOptimization@@ -39,3 +40,14 @@ def workflow_saved(sender: type, instance: HogFlow, **kwargs: object) -> None:and HogFlowOptimization.objects.unscoped().filter(hog_flow_id=instance.id, enabled=True).exists()):_sync_after_commit(instance.team_id)+++def _resume_runs_waiting_for_cohort(sender: object, *, team_id: int, cohort_id: int, **kwargs: object) -> None:+ from products.workflows.backend.tasks.batch_jobs import ( # noqa: PLC0415 — the task module imports the models this module loads+ resume_batch_jobs_waiting_for_realtime_cohort,+ )++ resume_batch_jobs_waiting_for_realtime_cohort.delay(team_id, cohort_id)+++connect_realtime_cohort_ready(_resume_runs_waiting_for_cohort, dispatch_uid="workflows_resume_runs_waiting_for_cohort")diff --git a/products/workflows/backend/services/batch_audience_cohorts.py b/products/workflows/backend/services/batch_audience_cohorts.pyindex 97479a35cf92..969461560555 100644--- a/products/workflows/backend/services/batch_audience_cohorts.py+++ b/products/workflows/backend/services/batch_audience_cohorts.py@@ -1,6 +1,8 @@from products.cohorts.backend.models.cohort import Cohort, CohortTypefrom products.cohorts.backend.models.util import get_all_cohort_dependencies+from products.cohorts.backend.realtime_state import CohortRealtimeState, resolve_realtime_readinessfrom products.cohorts.backend.realtime_teams import is_realtime_cohort_team+from products.workflows.backend.facade.contracts import RealtimeCohortStatusdef find_behavioral_cohort_name(project_id: int, cohort_ids: list[int | str]) -> str | None:@@ -29,8 +31,14 @@ def _filters_on_behavior(cohort: Cohort) -> bool:return False+# The filter kinds the realtime pipeline keys a cohort's membership on. A cohort with none of its own,+# such as one that only references other cohorts, never gets a membership or a readiness stamp.+_REALTIME_LEAF_TYPES = {"behavioral", "person", "person_metadata"}++def find_realtime_behavioral_cohort(project_id: int, cohort_ids: list[int | str]) -> Cohort | None:- """The first realtime cohort among `cohort_ids` that filters on event behavior."""+ """The first realtime cohort among `cohort_ids` that filters on event behavior and that the realtime+ pipeline can calculate."""for cohort_id in cohort_ids:try:cohort = Cohort.objects.get(@@ -38,19 +46,14 @@ def find_realtime_behavioral_cohort(project_id: int, cohort_ids: list[int | str])except (Cohort.DoesNotExist, ValueError, TypeError):continue- if _filters_on_behavior(cohort):+ has_own_leaf = any(p.type in _REALTIME_LEAF_TYPES for p in cohort.properties.flat)+ if has_own_leaf and _filters_on_behavior(cohort):return cohortreturn None-def realtime_audience_cohort(project_id: int, filters: object) -> Cohort | None:- """The realtime cohort an audience sends to from the realtime pipeline's membership, or None.-- Only an audience of exactly one condition, "in" a realtime cohort that filters on event- behavior, takes this path. The membership lives in a Postgres database that holds no person- properties, so any other condition, a negated one, or the test account filter keeps the- ClickHouse audience query.- """+def sole_audience_cohort_id(filters: object) -> int | str | None:+ """The cohort ID of an audience of exactly one condition, "in" a cohort, or None. Reads no rows."""if not isinstance(filters, dict) or filters.get("filter_test_accounts"):return Noneproperties = filters.get("properties")@@ -65,15 +68,57 @@ def realtime_audience_cohort(project_id: int, filters: object) -> Cohort | None:):return Nonecohort_id = condition.get("value")- if not isinstance(cohort_id, int | str):+ return cohort_id if isinstance(cohort_id, int | str) else None+++def realtime_audience_cohort(project_id: int, filters: object) -> Cohort | None:+ """The realtime cohort an audience sends to from the realtime pipeline's membership, or None.++ Only an audience of exactly one condition, "in" a realtime cohort that filters on event+ behavior, takes this path. The membership lives in a Postgres database that holds no person+ properties, so any other condition, a negated one, or the test account filter keeps the+ ClickHouse audience query.+ """+ cohort_id = sole_audience_cohort_id(filters)+ if cohort_id is None:return Nonereturn find_realtime_behavioral_cohort(project_id, [cohort_id])+def realtime_cohort_can_finish(cohort: Cohort) -> bool:+ """Whether the realtime pipeline runs for this cohort's team, so the cohort can become ready at all."""+ return is_realtime_cohort_team(cohort.team_id)++def realtime_cohort_is_ready(cohort: Cohort) -> bool:"""Whether the realtime pipeline holds this cohort's full membership and keeps it current.The backfill stamps stay set when a team leaves the realtime allowlist, but the pipeline thenstops updating the membership, so the team has to be on the allowlist too."""- return cohort.is_flag_compatible and is_realtime_cohort_team(cohort.team_id)+ return cohort.is_flag_compatible and realtime_cohort_can_finish(cohort)+++# The states the size response declares. Any other state of a cohort that isn't ready yet is building.+_REPORTED_PENDING_STATES = {CohortRealtimeState.REBUILDING, CohortRealtimeState.NEEDS_ATTENTION}+++def realtime_cohort_status(cohort: Cohort) -> RealtimeCohortStatus:+ """Whether a realtime cohort audience can send now, and how far its calculation has come."""+ readiness = resolve_realtime_readiness([cohort]).get(cohort.pk)+ if realtime_cohort_is_ready(cohort):+ state: str = CohortRealtimeState.READY+ elif not is_realtime_cohort_team(cohort.team_id):+ # The pipeline doesn't run for this team, so the cohort never finishes calculating.+ state = CohortRealtimeState.NEEDS_ATTENTION+ elif readiness is not None and readiness.state in _REPORTED_PENDING_STATES:+ state = readiness.state+ else:+ state = CohortRealtimeState.BUILDING+ build = readiness.build if readiness is not None else None+ return RealtimeCohortStatus(+ id=cohort.pk,+ name=str(cohort.name),+ state=state,+ percent_complete=build.percent_complete if build is not None and state != CohortRealtimeState.READY else None,+ )diff --git a/products/workflows/backend/services/batch_jobs.py b/products/workflows/backend/services/batch_jobs.pyindex 401cb8c23b5d..07f261384a2d 100644--- a/products/workflows/backend/services/batch_jobs.py+++ b/products/workflows/backend/services/batch_jobs.py@@ -6,11 +6,22 @@ from django.core.exceptions import ValidationError as DjangoValidationErrorfrom django.db.models import Exists, OuterRef, Q, Subqueryfrom django.utils import timezone+import structlog+from prometheus_client import Counter++from posthog.exceptions_capture import capture_exception+from posthog.plugins.plugin_server_api import resume_hog_flow_batch_job++from products.cohorts.backend.models.cohort import Cohortfrom products.workflows.backend.facade.contracts import WorkflowBatchJob, WorkflowBatchJobNotFoundfrom products.workflows.backend.facade.enums import HogFlowBatchJobState, HogFlowScheduleStatusfrom products.workflows.backend.models.hog_flow.hog_flow import HogFlowfrom products.workflows.backend.models.hog_flow_batch_job import HogFlowBatchJob+from products.workflows.backend.models.hog_flow_batch_job.hog_flow_batch_job import batch_job_audience_filtersfrom products.workflows.backend.models.hog_flow_schedule import HogFlowSchedule+from products.workflows.backend.services.batch_audience_cohorts import realtime_cohort_is_ready, sole_audience_cohort_id++logger = structlog.get_logger(__name__)def to_batch_job(job: HogFlowBatchJob) -> WorkflowBatchJob:@@ -119,3 +130,59 @@ def hog_flow_ids_with_broadcast_status(*, team_id: int, statuses: Collection[strfor broadcast_status in statuses:combined |= conditions[broadcast_status]return list(queryset.filter(combined).values_list("id", flat=True))+++REALTIME_COHORT_RESUMES = Counter(+ "workflows_realtime_cohort_resume_total",+ "Batch runs woken after their realtime cohort finished calculating, by the CDP API's outcome.",+ labelnames=["outcome"],+)+++def resume_batch_jobs_waiting_for_cohort(*, team_id: int, cohort_id: int) -> None:+ """Wake the parked runs that wait for this realtime cohort, once it is ready.++ Does nothing while the cohort still calculates. A run the CDP API confirms awake moves to queued.+ Raises after trying every run if any resume failed, so the caller retries; a second resume of the+ same run is a no-op.+ """+ cohort = Cohort.objects.filter(pk=cohort_id, team_id=team_id, deleted=False).select_related("team").first()+ if cohort is None or not realtime_cohort_is_ready(cohort):+ return+ waiting = HogFlowBatchJob.objects.filter(+ status=HogFlowBatchJobState.WAITING, team__project_id=cohort.team.project_id+ ).select_related("team", "hog_flow")+ errors: list[Exception] = []+ for job in waiting.iterator(chunk_size=500):+ # Each ready cohort scans the project's waiting runs, so a run that waits for another cohort is+ # skipped before any cohort query.+ filters = batch_job_audience_filters(job)+ if str(sole_audience_cohort_id(filters)) != str(cohort_id):+ continue+ # A cohort edit during this loop clears the readiness stamps, so each run checks a fresh load. The run+ # parked on this cohort, so an edit that drops its behavioral filters still resumes it once ready.+ fresh_cohort = Cohort.objects.filter(pk=cohort_id, team_id=team_id, deleted=False).first()+ if fresh_cohort is None or not realtime_cohort_is_ready(fresh_cohort):+ continue+ try:+ response = resume_hog_flow_batch_job(+ team_id=job.team_id, hog_flow_id=str(job.hog_flow_id), batch_job_id=str(job.id)+ )+ outcome = str(response.json().get("outcome", f"http_{response.status_code}"))+ except Exception as error:+ REALTIME_COHORT_RESUMES.labels(outcome="error").inc()+ capture_exception(error)+ errors.append(error)+ continue+ REALTIME_COHORT_RESUMES.labels(outcome=outcome).inc()+ if outcome in ("resumed", "already_resumed"):+ HogFlowBatchJob.objects.filter(pk=job.pk, status=HogFlowBatchJobState.WAITING).update(+ status=HogFlowBatchJobState.QUEUED, updated_at=timezone.now()+ )+ elif outcome == "job_running" or response.status_code >= 500:+ # The resolver is busy or the API failed, so a later attempt can still wake the run.+ errors.append(RuntimeError(f"Batch run {job.id} was not resumed: {outcome}"))+ else:+ logger.warning("realtime_cohort_resume_not_delivered", batch_job_id=str(job.id), outcome=outcome)+ if errors:+ raise errors[0]diff --git a/products/workflows/backend/services/blast_radius.py b/products/workflows/backend/services/blast_radius.pyindex 7b35f96895be..edaac0836e93 100644--- a/products/workflows/backend/services/blast_radius.py+++ b/products/workflows/backend/services/blast_radius.py@@ -1,6 +1,8 @@"""Batch audience sizing and paging: the preview counts the editor shows, and the pages the Nodebatch resolver walks when it sends."""+from django.core.cache import cache+from posthog.models.property import GroupTypeIndexfrom posthog.models.team.team import Team@@ -15,9 +17,22 @@ from products.workflows.backend.services.account_audience import ()from products.workflows.backend.services.audience_v2 import get_dedupe_audience_count_v2, get_person_audience_count_v2from products.workflows.backend.services.batch_audience import audience_page_size, get_batch_audience_person_ids-from products.workflows.backend.services.batch_audience_cohorts import realtime_audience_cohort+from products.workflows.backend.services.batch_audience_cohorts import realtime_audience_cohort, realtime_cohort_statusfrom products.workflows.backend.utils.batch_trigger_limit import get_hogflow_batch_trigger_limit+# An open send screen re-sizes a realtime cohort audience every few seconds while the cohort calculates. The+# project's person total only gives the count context, so it is counted once per window, not on every check.+REALTIME_AUDIENCE_TOTAL_CACHE_SECONDS = 600+++def _realtime_audience_total(team: Team) -> int:+ cache_key = f"workflows:realtime_audience_person_total:{team.pk}"+ total = cache.get(cache_key)+ if not isinstance(total, int):+ total = get_person_audience_count_v2(team, {"properties": []}).total+ cache.set(cache_key, total, REALTIME_AUDIENCE_TOTAL_CACHE_SECONDS)+ return total+def get_audience_size(*,@@ -38,12 +53,12 @@ def get_audience_size(# The send reads the realtime pipeline's Postgres table, which this process can't query. The# cohort's count from its periodic calculation stands in as an estimate, and the send skips the# email dedupe.- everyone = get_person_audience_count_v2(team, {"properties": []})return AudienceSize(affected=realtime_cohort.count or 0,- total=everyone.total,+ total=_realtime_audience_total(team),limit=get_hogflow_batch_trigger_limit(team_id, sends_email=sends_email),dedupe_key=None,+ realtime_cohort=realtime_cohort_status(realtime_cohort),)if group_type_index is not None:blast_radius = get_user_blast_radius(team, filters, group_type_index)diff --git a/products/workflows/backend/tasks/batch_jobs.py b/products/workflows/backend/tasks/batch_jobs.pynew file mode 100644index 000000000000..276bc33b5b86--- /dev/null+++ b/products/workflows/backend/tasks/batch_jobs.py@@ -0,0 +1,24 @@+from celery import shared_task++from posthog.scoping_audit import skip_team_scope_audit+from posthog.tasks.utils import CeleryQueue++from products.workflows.backend.services.batch_jobs import resume_batch_jobs_waiting_for_cohort+++@shared_task(+ ignore_result=True,+ queue=CeleryQueue.DEFAULT.value,+ # The ready signal fires once and a resume is idempotent, so a task a dying worker held runs again.+ acks_late=True,+ reject_on_worker_lost=True,+ autoretry_for=(Exception,),+ retry_backoff=True,+ # The ready signal fires once, so the retries span the run's 24-hour wait at up to 10 minutes apart,+ # and a CDP API outage that ends before the deadline still wakes the run.+ retry_backoff_max=600,+ max_retries=150,+)+@skip_team_scope_audit+def resume_batch_jobs_waiting_for_realtime_cohort(team_id: int, cohort_id: int) -> None:+ resume_batch_jobs_waiting_for_cohort(team_id=team_id, cohort_id=cohort_id)diff --git a/products/workflows/backend/test/test_batch_audience.py b/products/workflows/backend/test/test_batch_audience.pyindex f62e12494b6a..7d5308ecb396 100644--- a/products/workflows/backend/test/test_batch_audience.py+++ b/products/workflows/backend/test/test_batch_audience.py@@ -4,7 +4,11 @@ from django.test import override_settingsfrom parameterized import parameterized+from posthog.redis import get_client as get_redis_client++from products.cohorts.backend.models.backfill import CohortBackfillKind, CohortBackfillTriggerfrom products.cohorts.backend.models.cohort import Cohort, CohortType+from products.cohorts.backend.models.dependencies import cohort_backfill_pending_keyfrom products.feature_flags.backend.user_blast_radius import PERSON_BATCH_SIZE, get_user_blast_radius_personsfrom products.workflows.backend.services.batch_audience import audience_page_size, get_batch_audience_person_idsfrom products.workflows.backend.services.blast_radius import get_audience_size@@ -82,38 +86,65 @@ class TestBatchAudience(ClickhouseTestMixin, BaseTest):assert audience_page_size(None) == 7assert audience_page_size(0) == PERSON_BATCH_SIZE- def test_realtime_cohort_audience_size_estimates_from_the_cohort_count(self):+ @parameterized.expand(+ [+ ("build_queued", "behavioral", "all", True, "building"),+ ("nothing_building", "behavioral", "all", False, "needs_attention"),+ # The pipeline doesn't run for the team, so the cohort never finishes calculating.+ ("team_off_realtime_allowlist", "behavioral", "none", True, "needs_attention"),+ # The cohorts product reports this one as person_properties, which the response doesn't declare.+ ("person_property_cohort_on_a_behavioral_cohort", "person_on_behavioral", "all", False, "building"),+ ]+ )+ def test_realtime_cohort_audience_size_estimates_from_the_cohort_count(+ self, _name, shape, allowlist, build_queued, expected_state+ ):# The send reads realtime membership from a Postgres database Django can't reach, so the# preview estimates from the cohort's periodic count and reports no email dedupe.+ behavioral_leaf = {+ "key": "$pageview",+ "event_type": "events",+ "time_value": 30,+ "time_interval": "day",+ "value": "performed_event",+ "type": "behavioral",+ }+ leaves: list[dict] = [behavioral_leaf]+ if shape == "person_on_behavioral":+ inner = Cohort.objects.create(+ team=self.team,+ name="inner",+ cohort_type=CohortType.REALTIME,+ filters={"properties": {"type": "AND", "values": [behavioral_leaf]}},+ )+ leaves = [+ {"key": "email", "type": "person", "value": "x.com", "operator": "icontains"},+ {"key": "id", "type": "cohort", "value": inner.pk},+ ]cohort = Cohort.objects.create(team=self.team,name="c",cohort_type=CohortType.REALTIME,count=2,- filters={- "properties": {- "type": "AND",- "values": [- {- "key": "$pageview",- "event_type": "events",- "time_value": 30,- "time_interval": "day",- "value": "performed_event",- "type": "behavioral",- }- ],- }- },- )-- size = get_audience_size(- team_id=self.team.pk,- filters={"properties": [{"key": "id", "type": "cohort", "value": cohort.pk, "operator": "in"}]},- group_type_index=None,- dedupe_key="email",- sends_email=True,+ filters={"properties": {"type": "AND", "values": leaves}},)+ if build_queued:+ get_redis_client().set(+ cohort_backfill_pending_key(cohort.pk, CohortBackfillKind.BEHAVIORAL),+ CohortBackfillTrigger.COHORT_CREATED,+ ex=300,+ )++ with override_settings(REALTIME_COHORT_TEAM_ALLOWLIST=allowlist):+ size = get_audience_size(+ team_id=self.team.pk,+ filters={"properties": [{"key": "id", "type": "cohort", "value": cohort.pk, "operator": "in"}]},+ group_type_index=None,+ dedupe_key="email",+ sends_email=True,+ )assert (size.affected, size.total, size.dedupe_key) == (2, 3, None)+ assert size.realtime_cohort is not None+ assert (size.realtime_cohort.id, size.realtime_cohort.state) == (cohort.pk, expected_state)diff --git a/products/workflows/backend/test/test_hog_flow_batch_job.py b/products/workflows/backend/test/test_hog_flow_batch_job.pyindex 4b5e2a7ef451..f4e4dc687cff 100644--- a/products/workflows/backend/test/test_hog_flow_batch_job.py+++ b/products/workflows/backend/test/test_hog_flow_batch_job.py@@ -1,20 +1,29 @@+from contextlib import nullcontextfrom datetime import UTC, datetime-from unittest.mock import patch+import pytest+from unittest.mock import MagicMock, patchfrom django.db.models.signals import post_savefrom django.test import TestCase, override_settings+import requests+from celery.exceptions import Retryfrom parameterized import parameterizedfrom posthog.models.user import User+from products.cohorts.backend.facade.ready_signals import realtime_cohort_readyfrom products.cohorts.backend.models.cohort import Cohort, CohortTypefrom products.workflows.backend.models.hog_flow.hog_flow import HogFlowfrom products.workflows.backend.models.hog_flow_batch_job import HogFlowBatchJob-from products.workflows.backend.models.hog_flow_batch_job.hog_flow_batch_job import handle_hog_flow_batch_job_created+from products.workflows.backend.models.hog_flow_batch_job.hog_flow_batch_job import (+ REALTIME_COHORT_WAIT_LIMIT,+ handle_hog_flow_batch_job_created,+)+@override_settings(REALTIME_COHORT_TEAM_ALLOWLIST="all")class TestHogFlowBatchJob(TestCase):def setUp(self):super().setUp()@@ -70,7 +79,6 @@ class TestHogFlowBatchJob(TestCase):("non_realtime_cohort", "not_realtime", None, False),])- @override_settings(REALTIME_COHORT_TEAM_ALLOWLIST="all")@patch("products.workflows.backend.models.hog_flow_batch_job.hog_flow_batch_job.create_batch_hog_flow_job_invocation")@@ -113,27 +121,12 @@ class TestHogFlowBatchJob(TestCase):expected = {"id": cohort.pk, "team_id": self.team.pk} if expect_realtime else Noneassert mock_create_invocation.call_args.kwargs["realtime_cohort"] == expected- @parameterized.expand(- [- # An edit after the save clears the cohort's readiness; its membership is partial until it recalculates.- ("stamps_cleared", False, "all"),- # The stamps stay set, but the pipeline stops updating the membership.- ("team_left_the_realtime_allowlist", True, "none"),- ]- )- @patch(- "products.workflows.backend.models.hog_flow_batch_job.hog_flow_batch_job.create_batch_hog_flow_job_invocation"- )- def test_hog_flow_batch_job_fails_when_its_realtime_cohort_is_no_longer_ready(- self, _name, backfilled, allowlist, mock_create_invocation- ):- backfill_stamp = datetime(2026, 1, 1, tzinfo=UTC) if backfilled else None- cohort = Cohort.objects.create(+ def _realtime_cohort(self, *, ready: bool) -> Cohort:+ stamp = datetime(2026, 1, 1, tzinfo=UTC) if ready else None+ return Cohort.objects.create(team=self.team,name="c",cohort_type=CohortType.REALTIME,- last_backfill_events_at=backfill_stamp,- last_backfill_person_properties_at=backfill_stamp,filters={"properties": {"type": "AND",@@ -149,18 +142,174 @@ class TestHogFlowBatchJob(TestCase):],}},+ last_backfill_events_at=stamp,+ last_backfill_person_properties_at=stamp,)- with override_settings(REALTIME_COHORT_TEAM_ALLOWLIST=allowlist):+ def _realtime_audience(self, cohort: Cohort) -> dict:+ return {"properties": [{"key": "id", "type": "cohort", "value": cohort.pk, "operator": "in"}]}++ @parameterized.expand(+ [+ ("published", None),+ # The send is already accepted, so a broker failure must not fail the request.+ ("publish_fails", ConnectionError("broker unavailable")),+ ]+ )+ @patch("products.workflows.backend.tasks.batch_jobs.resume_batch_jobs_waiting_for_realtime_cohort.delay")+ @patch(+ "products.workflows.backend.models.hog_flow_batch_job.hog_flow_batch_job.create_batch_hog_flow_job_invocation"+ )+ def test_hog_flow_batch_job_parks_its_resolver_while_the_cohort_calculates(+ self, _name, publish_error, mock_create_invocation, mock_resume_task+ ):+ mock_resume_task.side_effect = publish_error+ cohort = self._realtime_cohort(ready=False)++ with self.captureOnCommitCallbacks(execute=True):job = HogFlowBatchJob.objects.create(- team=self.team,- hog_flow=self.hog_flow,- filters={"properties": [{"key": "id", "type": "cohort", "value": cohort.pk, "operator": "in"}]},+ team=self.team, hog_flow=self.hog_flow, filters=self._realtime_audience(cohort)+ )++ job.refresh_from_db()+ assert job.status == HogFlowBatchJob.State.WAITING+ assert mock_create_invocation.call_args.kwargs["realtime_cohort"] == {+ "id": cohort.pk,+ "team_id": self.team.pk,+ "wait_until": (job.created_at + REALTIME_COHORT_WAIT_LIMIT).isoformat(),+ }+ # A cohort that finished between the readiness check and the park still wakes the run.+ mock_resume_task.assert_called_once_with(self.team.pk, cohort.pk)++ @parameterized.expand(+ [+ ("resumed", "calculated", 200, "resumed", "queued", True, False),+ # A second wake of the same run is a no-op on the CDP side.+ ("already_resumed", "calculated", 200, "already_resumed", "queued", True, False),+ ("still_calculating", "calculating", 200, "resumed", "waiting", False, False),+ # The stamps are set, but the pipeline doesn't keep the membership current for this team.+ ("team_off_realtime_allowlist", "off_allowlist", 200, "resumed", "waiting", False, False),+ # A running resolver can still need the wake, so the task retries.+ ("resolver_running", "calculated", 409, "job_running", "waiting", True, True),+ ("cdp_api_error", "calculated", 503, None, "waiting", True, True),+ # The resolver already ran, so there is nothing left to wake.+ ("resolver_finished", "calculated", 200, "job_completed", "waiting", True, False),+ ("another_cohort", "another_cohort", 200, "resumed", "waiting", False, False),+ # The run parked on this cohort, so it resumes even after an edit drops the behavioral filter.+ ("now_property_only", "property_only", 200, "resumed", "queued", True, False),+ ]+ )+ @patch("products.workflows.backend.services.batch_jobs.resume_hog_flow_batch_job")+ @patch(+ "products.workflows.backend.models.hog_flow_batch_job.hog_flow_batch_job.create_batch_hog_flow_job_invocation"+ )+ def test_ready_cohort_resumes_the_runs_parked_on_it(+ self,+ _name,+ cohort_state,+ resume_status,+ resume_outcome,+ expected_status,+ expect_resume,+ expect_retry,+ _mock_create_invocation,+ mock_resume,+ ):+ cohort = self._realtime_cohort(ready=False)+ job = HogFlowBatchJob.objects.create(+ team=self.team, hog_flow=self.hog_flow, filters=self._realtime_audience(cohort)+ )+ if cohort_state == "property_only":+ Cohort.objects.filter(pk=cohort.pk).update(+ filters={+ "properties": {+ "type": "AND",+ "values": [{"key": "plan", "type": "person", "value": "pro", "operator": "exact"}],+ }+ })+ if cohort_state != "calculating":+ Cohort.objects.filter(pk=cohort.pk).update(+ last_backfill_events_at=datetime(2026, 1, 1, tzinfo=UTC),+ last_backfill_person_properties_at=datetime(2026, 1, 1, tzinfo=UTC),+ )+ ready_cohort = self._realtime_cohort(ready=True) if cohort_state == "another_cohort" else cohort+ mock_resume.return_value.status_code = resume_status+ mock_resume.return_value.json.return_value = (+ {"outcome": resume_outcome} if resume_outcome else {"error": "unavailable"}+ )++ allowlist = "none" if cohort_state == "off_allowlist" else "all"+ with (+ override_settings(REALTIME_COHORT_TEAM_ALLOWLIST=allowlist),+ pytest.raises(Retry) if expect_retry else nullcontext(),+ ):+ realtime_cohort_ready.send(sender=Cohort, team_id=self.team.pk, cohort_id=ready_cohort.pk)- mock_create_invocation.assert_not_called()job.refresh_from_db()- assert job.status == HogFlowBatchJob.State.FAILED+ assert job.status == expected_status+ assert mock_resume.called == expect_resume+ if expect_resume:+ assert mock_resume.call_args.kwargs == {+ "team_id": self.team.pk,+ "hog_flow_id": str(self.hog_flow.id),+ "batch_job_id": str(job.id),+ }++ @override_settings(REALTIME_COHORT_TEAM_ALLOWLIST="all")+ @patch("products.workflows.backend.services.batch_jobs.resume_hog_flow_batch_job")+ @patch(+ "products.workflows.backend.models.hog_flow_batch_job.hog_flow_batch_job.create_batch_hog_flow_job_invocation"+ )+ def test_cohort_edited_mid_resume_leaves_the_remaining_runs_waiting(self, _mock_create_invocation, mock_resume):+ cohort = self._realtime_cohort(ready=False)+ jobs = [+ HogFlowBatchJob.objects.create(+ team=self.team, hog_flow=self.hog_flow, filters=self._realtime_audience(cohort)+ )+ for _ in range(2)+ ]+ Cohort.objects.filter(pk=cohort.pk).update(+ last_backfill_events_at=datetime(2026, 1, 1, tzinfo=UTC),+ last_backfill_person_properties_at=datetime(2026, 1, 1, tzinfo=UTC),+ )++ def resume_then_edit(**kwargs):+ Cohort.objects.filter(pk=cohort.pk).update(last_backfill_events_at=None)+ response = MagicMock(status_code=200)+ response.json.return_value = {"outcome": "resumed"}+ return response++ mock_resume.side_effect = resume_then_edit++ realtime_cohort_ready.send(sender=Cohort, team_id=self.team.pk, cohort_id=cohort.pk)++ statuses = sorted(HogFlowBatchJob.objects.filter(pk__in=[j.pk for j in jobs]).values_list("status", flat=True))+ assert statuses == ["queued", "waiting"]+ assert mock_resume.call_count == 1++ @parameterized.expand(+ [+ ("rejected", requests.HTTPError("400 Client Error", response=MagicMock(status_code=400)), "failed"),+ ("never_connected", requests.ConnectTimeout("connect timed out"), "failed"),+ # The request may have reached the CDP API, so a stop must still reach the resolver.+ ("no_response", requests.ReadTimeout("read timed out"), "queued"),+ ]+ )+ @patch(+ "products.workflows.backend.models.hog_flow_batch_job.hog_flow_batch_job.create_batch_hog_flow_job_invocation"+ )+ def test_failed_dispatch_fails_the_run_only_when_no_resolver_was_queued(+ self, _name, error, expected_status, mock_create_invocation+ ):+ mock_create_invocation.return_value.raise_for_status.side_effect = error++ with pytest.raises(type(error)):+ HogFlowBatchJob.objects.create(team=self.team, hog_flow=self.hog_flow, variables=[])++ assert list(HogFlowBatchJob.objects.filter(hog_flow=self.hog_flow).values_list("status", flat=True)) == [+ expected_status+ ]@patch("products.workflows.backend.models.hog_flow_batch_job.hog_flow_batch_job.create_batch_hog_flow_job_invocation"diff --git a/products/workflows/backend/tests/api/test_hog_flow.py b/products/workflows/backend/tests/api/test_hog_flow.pyindex 178fce7bbbe0..74625ddb857a 100644--- a/products/workflows/backend/tests/api/test_hog_flow.py+++ b/products/workflows/backend/tests/api/test_hog_flow.py@@ -35,6 +35,7 @@ from products.cdp.backend.models.hog_functions.hog_function import HogFunctionfrom products.cdp.backend.templates.fixtures import template_slackfrom products.cdp.backend.templates.hog_function_template import sync_template_to_dbfrom products.cohorts.backend.models.cohort import Cohort, CohortType+from products.feature_flags.backend.user_blast_radius import BlastRadiusResultfrom products.replay_vision.backend.models.replay_scanner import ReplayScannerfrom products.replay_vision.backend.tests.helpers import create_experimentfrom products.skills.backend.models.skills import LLMSkill@@ -4131,9 +4132,8 @@ class TestHogFlowAPI(APIBaseTest):("ready_alone", "batch", "ready", None, True, 201, None),# A schedule trigger starts one run with no person, so it never fans out to the members.("schedule_trigger", "schedule", "ready", None, True, 400, "can't evaluate"),- ("not_ready", "batch", "not_ready", None, True, 400, "isn't ready yet"),- # The backfill stamps outlive the team's place on the realtime allowlist; the membership doesn't.- ("team_off_realtime_allowlist", "batch", "ready_off_allowlist", None, True, 400, "isn't ready yet"),+ # A batch run waits for a cohort that is still calculating.+ ("not_ready", "batch", "not_ready", None, True, 201, None),# The send pages the cohort's members from Postgres, which holds no person properties.("with_another_condition", "batch", "ready", "with_property", True, 400, "only be the people in it"),("not_in", "batch", "ready", "not_in", True, 400, "only be the people in it"),@@ -4141,8 +4141,14 @@ class TestHogFlowAPI(APIBaseTest):("test_account_filter", "batch", "ready", "test_accounts", True, 400, "only be the people in it"),("flag_off", "batch", "ready", None, False, 400, "can't evaluate"),("behavioral_not_realtime", "batch", "not_realtime", None, True, 400, "can't evaluate"),+ # The realtime pipeline has no filter of its own to key this cohort's membership on, so it never+ # becomes ready.+ ("realtime_reference_only", "batch", "reference_only", None, True, 400, "can't evaluate"),+ # The realtime pipeline doesn't run for the team, so the cohort never becomes ready.+ ("team_off_realtime_pipeline", "batch", "off_pipeline", None, True, 400, "would never start"),])+ @override_settings(REALTIME_COHORT_TEAM_ALLOWLIST="all")@patch("products.workflows.backend.presentation.views.hog_flow.feature_enabled_or_false")def test_hog_flow_audience_realtime_cohort(self, _name, trigger_type, cohort_kind, shape, flag_on, expected_status, expected_detail, mock_flag@@ -4150,8 +4156,18 @@ class TestHogFlowAPI(APIBaseTest):mock_flag.return_value = flag_onif cohort_kind == "not_realtime":cohort = self._make_cohort(behavioral=True)+ elif cohort_kind == "reference_only":+ referenced = self._create_behavioral_cohort(CohortType.REALTIME, backfilled=True)+ cohort = Cohort.objects.create(+ team=self.team,+ name="refers to power-users",+ cohort_type=CohortType.REALTIME,+ filters={+ "properties": {"type": "AND", "values": [{"key": "id", "type": "cohort", "value": referenced.pk}]}+ },+ )else:- cohort = self._create_behavioral_cohort(CohortType.REALTIME, backfilled=cohort_kind != "not_ready")+ cohort = self._create_behavioral_cohort(CohortType.REALTIME, backfilled=cohort_kind == "ready")condition: dict[str, Any] = {"key": "id", "type": "cohort", "value": cohort.pk, "operator": "in"}if shape == "not_in":condition["operator"] = "not_in"@@ -4176,8 +4192,7 @@ class TestHogFlowAPI(APIBaseTest):],}- allowlist = "none" if cohort_kind == "ready_off_allowlist" else "all"- with override_settings(REALTIME_COHORT_TEAM_ALLOWLIST=allowlist):+ with override_settings(REALTIME_COHORT_TEAM_ALLOWLIST="none" if cohort_kind == "off_pipeline" else "all"):response = self.client.post(f"/api/projects/{self.team.id}/hog_flows", hog_flow)assert response.status_code == expected_status, response.json()@@ -4416,6 +4431,45 @@ class TestHogFlowAPI(APIBaseTest):assert response.status_code == 200, response.json()assert response.json()["limit"] == 50000+ @parameterized.expand(+ [+ ("with_cohort_read", ["hog_flow:read", "person:read", "cohort:read"], True),+ ("without_cohort_read", ["hog_flow:read", "person:read"], False),+ ]+ )+ @override_settings(REALTIME_COHORT_TEAM_ALLOWLIST="all")+ def test_user_blast_radius_shows_the_realtime_cohort_only_to_cohort_readers(self, _name, scopes, shows_cohort):+ cohort = self._create_behavioral_cohort(CohortType.REALTIME, backfilled=False)+ key = generate_random_token_personal()+ PersonalAPIKey.objects.create(label="sizing", user=self.user, secure_value=hash_key_value(key), scopes=scopes)++ response = self.client.post(+ f"/api/projects/{self.team.id}/hog_flows/user_blast_radius",+ {"filters": {"properties": [{"key": "id", "type": "cohort", "value": cohort.pk, "operator": "in"}]}},+ headers={"authorization": f"Bearer {key}"},+ )++ assert response.status_code == 200, response.json()+ realtime_cohort = response.json()["realtime_cohort"]+ assert (realtime_cohort or {}).get("id") == (cohort.pk if shows_cohort else None)++ @override_settings(REALTIME_COHORT_TEAM_ALLOWLIST="all")+ def test_user_blast_radius_counts_the_project_total_once_for_repeated_realtime_checks(self):+ cohort = self._create_behavioral_cohort(CohortType.REALTIME, backfilled=False)+ audience = {"properties": [{"key": "id", "type": "cohort", "value": cohort.pk, "operator": "in"}]}+ with patch(+ "products.workflows.backend.services.blast_radius.get_person_audience_count_v2",+ return_value=BlastRadiusResult(affected=500, total=500),+ ) as mock_count:+ for _ in range(3):+ response = self.client.post(+ f"/api/projects/{self.team.id}/hog_flows/user_blast_radius", {"filters": audience}+ )+ assert response.status_code == 200, response.json()+ assert response.json()["total"] == 500++ assert mock_count.call_count == 1+def test_user_blast_radius_personal_api_key_requires_person_read_scope(self):# Sizing an audience queries person data, so a hog_flow:read-only token must NOT be able to use# this as a person-count oracle — person:read is also required.@@ -5015,6 +5069,41 @@ class TestHogFlowAPI(APIBaseTest):# The per-team audience cap must ride on the invocation so the consumer enforces the team's limit.assert mock_create_invocation.call_args.kwargs["max_audience_size"] == 5000+ @parameterized.expand([("on_realtime_pipeline", "all", 200), ("off_realtime_pipeline", "none", 400)])+ @patch(+ "products.workflows.backend.models.hog_flow_batch_job.hog_flow_batch_job.create_batch_hog_flow_job_invocation"+ )+ def test_post_hog_flow_batch_jobs_refuses_a_realtime_cohort_that_never_finishes(+ self, _name, allowlist, expected_status, mock_create_invocation+ ):+ flow_id = self._create_active_hog_flow()+ cohort = self._create_behavioral_cohort(CohortType.REALTIME, backfilled=False)+ flow = HogFlow.objects.get(pk=flow_id)+ flow.trigger = {+ **flow.trigger,+ "filters": {"properties": [{"key": "id", "type": "cohort", "value": cohort.pk, "operator": "in"}]},+ }+ flow.save(update_fields=["trigger"])++ with override_settings(REALTIME_COHORT_TEAM_ALLOWLIST=allowlist):+ response = self.client.post(f"/api/projects/{self.team.id}/hog_flows/{flow_id}/batch_jobs", {})++ assert response.status_code == expected_status, response.json()+ assert mock_create_invocation.called == (expected_status == 200)++ @patch(+ "products.workflows.backend.models.hog_flow_batch_job.hog_flow_batch_job.create_batch_hog_flow_job_invocation"+ )+ def test_post_hog_flow_batch_jobs_rejects_a_caller_supplied_waiting_status(self, mock_create_invocation):+ flow_id = self._create_active_hog_flow()++ response = self.client.post(+ f"/api/projects/{self.team.id}/hog_flows/{flow_id}/batch_jobs", {"status": "waiting"}+ )++ assert response.status_code == 400, response.json()+ mock_create_invocation.assert_not_called()+@patch("products.workflows.backend.models.hog_flow_batch_job.hog_flow_batch_job.create_batch_hog_flow_job_invocation")diff --git a/products/workflows/backend/tests/api/test_hog_flow_cancel.py b/products/workflows/backend/tests/api/test_hog_flow_cancel.pyindex 241ca1cd8548..abeb856839f6 100644--- a/products/workflows/backend/tests/api/test_hog_flow_cancel.py+++ b/products/workflows/backend/tests/api/test_hog_flow_cancel.py@@ -133,8 +133,10 @@ class TestHogFlowCancelBatchJob(APIBaseTest):f"/api/projects/{self.team.id}/hog_flows/{self.hog_flow.id}/batch_jobs/{batch_job_id}/cancel/")- def test_cancel_active_batch_job_flags_run_and_marks_cancelled(self):- batch_job = self._create_batch_job("active")+ # A waiting run's resolver is parked on its realtime cohort, so stopping it goes through the worker too.+ @parameterized.expand([("active",), ("waiting",)])+ def test_cancel_running_batch_job_flags_run_and_marks_cancelled(self, job_status):+ batch_job = self._create_batch_job(job_status)with patch(CANCEL_BATCH_PROXY, return_value=_cdp_response(marked=5)) as mock_cancel:response = self._cancel(batch_job.id)diff --git a/products/workflows/frontend/generated/api.schemas.ts b/products/workflows/frontend/generated/api.schemas.tsindex 85dd9246e1cd..27a8f7da96f0 100644--- a/products/workflows/frontend/generated/api.schemas.ts+++ b/products/workflows/frontend/generated/api.schemas.ts@@ -2182,6 +2182,41 @@ export interface BlastRadiusRequestApi {sends_email?: boolean}+/**+ * * `ready` - Ready+ * * `building` - Building+ * * `rebuilding` - Rebuilding+ * * `needs_attention` - Needs attention+ */+export type RealtimeAudienceCohortStateEnumApi =+ (typeof RealtimeAudienceCohortStateEnumApi)[keyof typeof RealtimeAudienceCohortStateEnumApi]++export const RealtimeAudienceCohortStateEnumApi = {+ Ready: 'ready',+ Building: 'building',+ Rebuilding: 'rebuilding',+ NeedsAttention: 'needs_attention',+} as const++export interface RealtimeCohortStatusApi {+ /** The realtime cohort the audience sends to. */+ id: number+ /** The cohort's name. */+ name: string+ /** 'ready' when the cohort's membership is calculated. 'building' or 'rebuilding' while it is still calculating, when a send waits for it. 'needs_attention' when its calculation is stuck.+ *+ * * `ready` - Ready+ * * `building` - Building+ * * `rebuilding` - Rebuilding+ * * `needs_attention` - Needs attention */+ state: RealtimeAudienceCohortStateEnumApi+ /**+ * How far the calculation has come, while the cohort is not ready.+ * @nullable+ */+ percent_complete: number | null+}+export interface BlastRadiusApi {/** Number of users matching the filters */affected: number@@ -2195,6 +2230,8 @@ export interface BlastRadiusApi {dedupe_key: DedupeKeyEnumApi | null/** Proof this audience was previewed: pass it to the batch dispatch (confirm_token) after echoing 'affected' to the user. Signs these exact filters; expires in 15 minutes. */confirm_token: string+ /** Set when the audience is one realtime cohort. 'affected' is then the cohort's last calculated size, and a batch run waits until the cohort is ready. */+ realtime_cohort?: RealtimeCohortStatusApi | null}export interface DataSuggestionBuildRequestApi {diff --git a/services/mcp/src/api/generated.ts b/services/mcp/src/api/generated.tsindex 5305abf4621c..d3cb048030d5 100644--- a/services/mcp/src/api/generated.ts+++ b/services/mcp/src/api/generated.ts@@ -18848,6 +18848,41 @@ export namespace Schemas {Email: 'email',} as const;+ /**+ * * `ready` - Ready+ * * `building` - Building+ * * `rebuilding` - Rebuilding+ * * `needs_attention` - Needs attention+ */+ export type RealtimeAudienceCohortStateEnum = typeof RealtimeAudienceCohortStateEnum[keyof typeof RealtimeAudienceCohortStateEnum];+++ export const RealtimeAudienceCohortStateEnum = {+ Ready: 'ready',+ Building: 'building',+ Rebuilding: 'rebuilding',+ NeedsAttention: 'needs_attention',+ } as const;++ export interface RealtimeCohortStatus {+ /** The realtime cohort the audience sends to. */+ id: number;+ /** The cohort's name. */+ name: string;+ /** 'ready' when the cohort's membership is calculated. 'building' or 'rebuilding' while it is still calculating, when a send waits for it. 'needs_attention' when its calculation is stuck.+ *+ * * `ready` - Ready+ * * `building` - Building+ * * `rebuilding` - Rebuilding+ * * `needs_attention` - Needs attention */+ state: RealtimeAudienceCohortStateEnum;+ /**+ * How far the calculation has come, while the cohort is not ready.+ * @nullable+ */+ percent_complete: number | null;+ }+export interface BlastRadius {/** Number of users matching the filters */affected: number;@@ -18861,6 +18896,8 @@ export namespace Schemas {dedupe_key: DedupeKeyEnum | null;/** Proof this audience was previewed: pass it to the batch dispatch (confirm_token) after echoing 'affected' to the user. Signs these exact filters; expires in 15 minutes. */confirm_token: string;+ /** Set when the audience is one realtime cohort. 'affected' is then the cohort's last calculated size, and a batch run waits until the cohort is ready. */+ realtime_cohort?: RealtimeCohortStatus | null;}/**