Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(workflows): show broadcast audience ideas before their counts (#128805)

Daniel Marchuk33bc04d5b417
diff --git a/products/workflows/backend/facade/contracts.py b/products/workflows/backend/facade/contracts.py
index b161fb7f82e1..3ef7f57ba67c 100644
--- a/products/workflows/backend/facade/contracts.py
+++ b/products/workflows/backend/facade/contracts.py
@@ -985,8 +985,8 @@ class AudienceSuggestion:
window_days: int
# How many times the event must happen in the window. Set only for regulars.
min_count: int | None
- # People in the audience with an email address, counted when the suggestion was made.
- reachable_count: int
+ # Roughly how many people in the audience have an email address. None until counted, or when the count failed.
+ reachable_count: int | None
cohort_filters: dict[str, Any]
diff --git a/products/workflows/backend/presentation/views/broadcast_audience_suggestions.py b/products/workflows/backend/presentation/views/broadcast_audience_suggestions.py
index cb3fca1c3cea..9e56e3005de7 100644
--- a/products/workflows/backend/presentation/views/broadcast_audience_suggestions.py
+++ b/products/workflows/backend/presentation/views/broadcast_audience_suggestions.py
@@ -52,7 +52,9 @@ class AudienceSuggestionSerializer(serializers.Serializer):
allow_null=True, help_text="For regulars: how many times the event must happen in the window."
)
reachable_count = serializers.IntegerField(
- help_text="People in the audience with an email address, counted when the suggestion was made."
+ allow_null=True,
+ help_text="Roughly how many people in the audience have an email address, counted when the suggestion was "
+ "made. Null while the count runs, or when it failed.",
)
cohort_filters = serializers.JSONField(
help_text="The audience's cohort filters, in the cohorts endpoint's `filters` format. Saved as a static cohort."
@@ -63,7 +65,7 @@ class AudienceSuggestionsResponseSerializer(serializers.Serializer):
status = serializers.ChoiceField(
choices=DataSuggestionsStatus.choices,
help_text="ready when suggestions were made (the list can still be empty), generating while a background "
- "run makes them (poll current until it ends), ai_not_approved when the organization has not approved AI "
+ "run makes them (poll current until it ends, the list can already hold cards being counted), ai_not_approved when the organization has not approved AI "
"data processing, unavailable when nothing was made or the run failed.",
)
suggestions = AudienceSuggestionSerializer(many=True, help_text="Up to 3 suggested broadcast audiences.")
diff --git a/products/workflows/backend/services/audience_suggestions/audience.py b/products/workflows/backend/services/audience_suggestions/audience.py
index 3753d0730c5c..7fd2fede4429 100644
--- a/products/workflows/backend/services/audience_suggestions/audience.py
+++ b/products/workflows/backend/services/audience_suggestions/audience.py
@@ -1,23 +1,28 @@
"""The audiences a suggestion can describe: the cohort that holds them, and how many of them have an email address."""
+import math
+from collections.abc import Mapping, Sequence
from typing import Any
from posthog.schema import HogQLQueryModifiers, HogQLQueryResponse, PersonsOnEventsMode, ProductKey
from posthog.hogql import ast
from posthog.hogql.constants import HogQLGlobalSettings
-from posthog.hogql.parser import parse_select
+from posthog.hogql.parser import parse_expr, parse_select
from posthog.hogql.query import execute_hogql_query
+from posthog.clickhouse.client.connection import Workload
from posthog.clickhouse.query_tagging import Feature, tag_queries
from posthog.models import Team, User
-from products.workflows.backend.facade.contracts import AudienceKind
+from products.workflows.backend.facade.contracts import AudienceKind, AudienceSuggestion
# A lapsed person did the event within this many days, but not within the suggestion's window.
LAPSED_LOOKBACK_DAYS = 90
_COUNT_TIMEOUT_SECONDS = 30
_COUNT_MAX_BYTES = 4_000_000_000
+# A scan this size answers in about 3 seconds.
+_COUNT_TARGET_ROWS = 10_000_000
# PostHog events that stand for being active in the product.
ACTIVITY_EVENTS = frozenset({"$pageview", "$screen"})
# pinned: broadcast email steps send to {{ person.properties.email }}
@@ -52,72 +57,96 @@ def cohort_filters(
return {"properties": {"type": "AND", "values": [{"type": "AND", "values": [*behavior, email]}]}}
-def reachable_count(
- team: Team,
- *,
- kind: AudienceKind,
- event: str,
- missing_event: str | None,
- window_days: int,
- min_count: int | None,
- user: User | None = None,
-) -> int:
- """How many people match the cohort definition and have an email address. Each window starts where the
- cohort's window starts, so the card and the calculated cohort agree."""
+def sample_denominator(audiences: Sequence[AudienceSuggestion], weekly_counts: Mapping[str, int]) -> int:
+ """Which share of people to count, so the scan stays small however busy the audiences' events are."""
+ weekly_rows = sum(weekly_counts.get(name, 0) for name in _event_names(audiences))
+ return max(1, math.ceil(weekly_rows * _lookback_days(audiences) / 7 / _COUNT_TARGET_ROWS))
+
+
+def reachable_counts(
+ team: Team, audiences: Sequence[AudienceSuggestion], *, denominator: int = 1, user: User | None = None
+) -> list[int]:
+ """Roughly how many people in each audience have an email address, from one scan of the audiences' events. The
+ scan misses an address set after the person's last scanned event, and the sample keeps one distinct ID in
+ `denominator`, so a person with several IDs can look lapsed. Each window starts where the cohort's window starts."""
+ if not audiences:
+ return []
+ matched = parse_select(
+ """
+ SELECT person_id, countIf({email} ILIKE '%@%') > 0 AS has_email
+ FROM events
+ WHERE timestamp >= toStartOfDay(now() - toIntervalDay({lookback})) AND event IN {events}
+ GROUP BY person_id
+ """,
+ placeholders={
+ # The send reads the address from the person, which events copy when they are captured.
+ "email": ast.Field(chain=["person", "properties", EMAIL_PROPERTY]),
+ "lookback": ast.Constant(value=_lookback_days(audiences)),
+ "events": ast.Tuple(exprs=[ast.Constant(value=name) for name in sorted(_event_names(audiences))]),
+ },
+ )
+ assert isinstance(matched, ast.SelectQuery) and matched.select_from is not None
+ if denominator > 1:
+ matched.select_from.sample = ast.SampleExpr(
+ sample_value=ast.RatioExpr(left=ast.Constant(value=1), right=ast.Constant(value=denominator))
+ )
+ aliases = [f"audience_{index}" for index in range(len(audiences))]
+ matched.select.extend(
+ ast.Alias(alias=alias, expr=_matches(audience)) for alias, audience in zip(aliases, audiences)
+ )
+ query = ast.SelectQuery(
+ select=[
+ parse_expr("countIf({matched} AND has_email)", placeholders={"matched": ast.Field(chain=[alias])})
+ for alias in aliases
+ ],
+ select_from=ast.JoinExpr(table=matched),
+ )
+ response = _run(team, query, "workflows_audience_suggestions_size", user)
+ row = (response.results or [[0] * len(audiences)])[0]
+ return [int(count) * denominator for count in row]
+
+
+def _event_names(audiences: Sequence[AudienceSuggestion]) -> set[str]:
+ return {name for audience in audiences for name in (audience.event, audience.missing_event) if name}
+
+
+def _lookback_days(audiences: Sequence[AudienceSuggestion]) -> int:
+ return max(LAPSED_LOOKBACK_DAYS if audience.kind == "lapsed" else audience.window_days for audience in audiences)
+
+
+def _matches(audience: AudienceSuggestion) -> ast.Expr:
placeholders: dict[str, ast.Expr] = {
- "event": ast.Constant(value=event),
- "window": ast.Constant(value=window_days),
- "email": ast.Field(chain=["properties", EMAIL_PROPERTY]),
+ "event": ast.Constant(value=audience.event),
+ "since": parse_expr(
+ "toStartOfDay(now() - toIntervalDay({window}))",
+ placeholders={"window": ast.Constant(value=audience.window_days)},
+ ),
+ "window": ast.Constant(value=audience.window_days),
+ "missing": ast.Constant(value=audience.missing_event or ""),
+ "min_count": ast.Constant(value=audience.min_count or 1),
+ "lookback": ast.Constant(value=LAPSED_LOOKBACK_DAYS),
}
- match kind:
+ match audience.kind:
case "not_yet":
- placeholders["missing"] = ast.Constant(value=missing_event or "")
- inner = """
- SELECT person_id
- FROM events
- WHERE timestamp >= toStartOfDay(now() - toIntervalDay({window})) AND event IN ({event}, {missing})
- GROUP BY person_id
- HAVING countIf(event = {event}) > 0 AND countIf(event = {missing}) = 0
+ template = """
+ countIf(event = {event} AND timestamp >= {since}) > 0
+ AND countIf(event = {missing} AND timestamp >= {since}) = 0
"""
case "regulars":
# The cohort counts repeat events over a rolling window that ends 5 seconds from now, not by the day.
- placeholders["min_count"] = ast.Constant(value=min_count or 1)
- inner = """
- SELECT person_id
- FROM events
- WHERE timestamp >= now() - toIntervalDay({window})
+ template = """
+ countIf(
+ event = {event}
+ AND timestamp >= now() - toIntervalDay({window})
AND timestamp < now() + toIntervalSecond(5)
- AND event = {event}
- GROUP BY person_id
- HAVING count() >= {min_count}
+ ) >= {min_count}
"""
case "lapsed":
- placeholders["lookback"] = ast.Constant(value=LAPSED_LOOKBACK_DAYS)
- inner = """
- SELECT person_id
- FROM events
- WHERE timestamp >= toStartOfDay(now() - toIntervalDay({lookback})) AND event = {event}
- GROUP BY person_id
- HAVING max(timestamp) < toStartOfDay(now() - toIntervalDay({window}))
+ template = """
+ countIf(event = {event} AND timestamp >= toStartOfDay(now() - toIntervalDay({lookback}))) > 0
+ AND countIf(event = {event} AND timestamp >= {since}) = 0
"""
- # The email comes from the person, not the events, because the send reads it from the person. Only the
- # candidates' rows are read: the persons table would copy the events subquery into its own filter and fail.
- # The tuple keeps a newer null email, which a bare argMax skips in favor of an older address.
- query = parse_select(
- """
- SELECT count() FROM (
- SELECT id FROM raw_persons
- WHERE id IN (SELECT person_id FROM {matched})
- GROUP BY id
- HAVING argMax(is_deleted, version) = 0
- AND toString(tupleElement(argMax(tuple({email}), version), 1)) ILIKE '%@%'
- )
- """,
- placeholders={"matched": parse_select(inner, placeholders=placeholders), "email": placeholders["email"]},
- )
- response = _run(team, query, "workflows_audience_suggestions_size", user)
- rows = response.results or []
- return int(rows[0][0]) if rows else 0
+ return parse_expr(template, placeholders=placeholders)
def activity_counts(team: Team, user: User | None = None) -> dict[str, int]:
@@ -151,8 +180,10 @@ def _run(
team=team,
query_type=query_type,
settings=settings,
- # Cohorts resolve merged persons this way, so the counts must too.
- modifiers=HogQLQueryModifiers(personsOnEventsMode=PersonsOnEventsMode.PERSON_ID_OVERRIDE_PROPERTIES_JOINED),
+ # A person waits on the page for these cards.
+ workload=Workload.ONLINE,
+ # Merged persons count once, and the email comes from the events, so counting never reads the persons table.
+ modifiers=HogQLQueryModifiers(personsOnEventsMode=PersonsOnEventsMode.PERSON_ID_OVERRIDE_PROPERTIES_ON_EVENTS),
user=user,
)
diff --git a/products/workflows/backend/services/audience_suggestions/planner.py b/products/workflows/backend/services/audience_suggestions/planner.py
index e5c4abdd2287..1eb23f30ed29 100644
--- a/products/workflows/backend/services/audience_suggestions/planner.py
+++ b/products/workflows/backend/services/audience_suggestions/planner.py
@@ -9,7 +9,7 @@ from products.workflows.backend.services.data_suggestions.llm import invoke_stru
from ee.hogai.utils.untrusted import as_untrusted_data
-IDEAS_MODEL = "claude-sonnet-5-5"
+IDEAS_MODEL = "claude-haiku-5-5"
WindowDays = Literal[7, 14, 30]
MinCount = Literal[3, 5, 10]
@@ -40,7 +40,7 @@ class AudienceIdeas(BaseModel):
_PROMPT = """You suggest audiences for one-off email broadcasts, for a product team using PostHog.
A broadcast is a single email to a group of people, like a feature announcement, a nudge to finish setting up, a
-beta invite, a product update for power users, or a win-back note. Suggest up to 6 audiences built from the project's
+beta invite, a product update for power users, or a win-back note. Suggest up to 4 audiences built from the project's
own events, which are listed with how often they fired in the last 7 days.
Rules:
diff --git a/products/workflows/backend/services/audience_suggestions/service.py b/products/workflows/backend/services/audience_suggestions/service.py
index 6f2e9e75e792..5af9b0b07d2d 100644
--- a/products/workflows/backend/services/audience_suggestions/service.py
+++ b/products/workflows/backend/services/audience_suggestions/service.py
@@ -12,6 +12,7 @@ import structlog
from posthog.cloud_utils import is_cloud
from posthog.errors import CH_TRANSIENT_ERRORS, InternalCHQueryError
from posthog.exceptions import ClickHouseQueryMemoryLimitExceeded, ClickHouseQueryTimeOut
+from posthog.hogql_queries.utils.caller_context import map_in_caller_context
from posthog.models import Team, User
from posthog.ph_client import get_client
from posthog.utils import get_instance_region
@@ -29,7 +30,8 @@ from products.workflows.backend.services.audience_suggestions.audience import (
EMAIL_PROPERTY,
activity_counts,
cohort_filters,
- reachable_count,
+ reachable_counts,
+ sample_denominator,
)
from products.workflows.backend.services.audience_suggestions.planner import AudienceIdea, suggest_audiences
from products.workflows.backend.services.data_suggestions.builder import clean_copy
@@ -44,7 +46,7 @@ CACHE_TTL_SECONDS = 24 * 60 * 60
FAILURE_CACHE_TTL_SECONDS = 5 * 60
_GENERATION_LOCK_SECONDS = 15 * 60
MAX_SUGGESTIONS = 3
-_MAX_IDEAS = 6
+_MAX_IDEAS = 4
# A smaller audience is better written to one by one than broadcast to.
MIN_REACHABLE = 10
# Broadcast audiences often come from feature use, which busy telemetry events push down a volume ranking, so the
@@ -69,7 +71,7 @@ def get_audience_suggestions(*, team: Team) -> AudienceSuggestionsResult:
if unavailable is not None:
return unavailable
if cache.get(_lock_key(team.id)) is not None:
- return _GENERATING
+ return _in_progress(team.id)
return _read_cache(team.id) or _UNAVAILABLE
@@ -88,7 +90,7 @@ def start_audience_suggestions(
return cached
lock_token = str(uuid.uuid4())
if not cache.add(_lock_key(team.id), lock_token, _GENERATION_LOCK_SECONDS):
- return _GENERATING
+ return _in_progress(team.id)
try:
enqueue(lock_token)
except Exception:
@@ -110,7 +112,7 @@ def run_audience_suggestions(*, team_id: int, user_id: int, lock_token: str) ->
previous = _read_cache(team_id)
try:
with _analytics_client() as analytics_client:
- result = _suggest(team=team, user=user, analytics_client=analytics_client)
+ result = _suggest(team=team, user=user, lock_token=lock_token, analytics_client=analytics_client)
except _COUNT_ERRORS:
logger.warning("workflows.audience_suggestions.query_failed", team_id=team_id, exc_info=True)
result = _UNAVAILABLE
@@ -118,10 +120,11 @@ def run_audience_suggestions(*, team_id: int, user_id: int, lock_token: str) ->
# Without a cached failure, the next request would rerun the queries and the model at once.
logger.exception("workflows.audience_suggestions.generation_failed", team_id=team_id)
result = _UNAVAILABLE
+ complete = result.status == "ready" and all(item.reachable_count is not None for item in result.suggestions)
# A failed refresh keeps the working cards, so a click on one still finds it.
- if result.status != "ready" and previous is not None and previous.status == "ready":
+ if not complete and previous is not None and previous.status == "ready":
return
- ttl = CACHE_TTL_SECONDS if result.status == "ready" and result.suggestions else FAILURE_CACHE_TTL_SECONDS
+ ttl = CACHE_TTL_SECONDS if complete and result.suggestions else FAILURE_CACHE_TTL_SECONDS
if cache.get(_lock_key(team_id)) != lock_token:
return
cache.set(_cache_key(team_id), _to_cache(result), ttl)
@@ -137,31 +140,36 @@ def get_audience_suggestion(*, team: Team, suggestion_id: str) -> AudienceSugges
return match
-def _suggest(*, team: Team, user: User, analytics_client: Any) -> AudienceSuggestionsResult:
- counts = {**weekly_counts(team), **_activity_counts_or_empty(team, user)}
+def _suggest(*, team: Team, user: User, lock_token: str, analytics_client: Any) -> AudienceSuggestionsResult:
+ project_counts, activity = map_in_caller_context(
+ lambda count: count(), [lambda: weekly_counts(team), lambda: _activity_counts_or_empty(team, user)]
+ )
+ counts = {**project_counts, **activity}
events = prompt_events(counts)
if not events:
return AudienceSuggestionsResult(status="ready", suggestions=())
ideas = suggest_audiences(team=team, user=user, events=events, analytics_client=analytics_client)
if not ideas:
return _UNAVAILABLE
- sized: list[AudienceSuggestion] = []
- failed = 0
- # Each idea costs a ClickHouse count, and the prompt asks for at most this many.
- for idea in ideas[:_MAX_IDEAS]:
- # One audience too costly to count drops that idea only, not the cards that counted fine.
- try:
- suggestion = _sized(team, user, idea, counts)
- except _COUNT_ERRORS:
- logger.warning(
- "workflows.audience_suggestions.count_failed", team_id=team.id, kind=idea.kind, exc_info=True
- )
- failed += 1
- continue
- if suggestion is not None:
- sized.append(suggestion)
- if failed and not sized:
- return _UNAVAILABLE
+ drafts = [draft for idea in ideas if (draft := _draft(idea, counts)) is not None][:_MAX_IDEAS]
+ if not drafts:
+ return AudienceSuggestionsResult(status="ready", suggestions=())
+ # The cards are worth showing while the count runs. A click on one needs only its rule.
+ if cache.get(_lock_key(team.id)) == lock_token:
+ cache.set(
+ _in_progress_key(team.id),
+ _to_cache(AudienceSuggestionsResult(status="generating", suggestions=_pick(drafts))),
+ _GENERATION_LOCK_SECONDS,
+ )
+ try:
+ sizes = reachable_counts(team, drafts, denominator=sample_denominator(drafts, counts), user=user)
+ except _COUNT_ERRORS:
+ # The cards are still good ideas without a number, and the broadcast counts its audience when it is made.
+ logger.warning("workflows.audience_suggestions.count_failed", team_id=team.id, exc_info=True)
+ return AudienceSuggestionsResult(status="ready", suggestions=_pick(drafts))
+ sized = [
+ dataclasses.replace(draft, reachable_count=size) for draft, size in zip(drafts, sizes) if size >= MIN_REACHABLE
+ ]
return AudienceSuggestionsResult(status="ready", suggestions=_pick(sized))
@@ -191,7 +199,7 @@ def prompt_events(counts: dict[str, int]) -> tuple[tuple[str, int], ...]:
return tuple([*lifecycle, *(item for item in eligible if item[0] not in chosen)][:_PROMPT_EVENTS_LIMIT])
-def _sized(team: Team, user: User, idea: AudienceIdea, counts: dict[str, int]) -> AudienceSuggestion | None:
+def _draft(idea: AudienceIdea, counts: dict[str, int]) -> AudienceSuggestion | None:
# The schema limits the names to the listed events. The checks below catch a plan that does not fit its kind.
if idea.event not in counts:
return None
@@ -206,15 +214,12 @@ def _sized(team: Team, user: User, idea: AudienceIdea, counts: dict[str, int]) -
"window_days": idea.window_days,
"min_count": min_count,
}
- size = reachable_count(team, user=user, **shape)
- if size < MIN_REACHABLE:
- return None
return AudienceSuggestion(
id=str(uuid.uuid4()),
name=clean_copy(idea.name)[:50],
description=clean_copy(idea.description)[:300],
reason=clean_copy(idea.reason)[:200],
- reachable_count=size,
+ reachable_count=None,
cohort_filters=cohort_filters(**shape),
**shape,
)
@@ -261,8 +266,17 @@ def _lock_key(team_id: int) -> str:
return f"workflows:audience_suggestions:lock:{team_id}"
+def _in_progress_key(team_id: int) -> str:
+ return f"workflows:audience_suggestions:in_progress:{team_id}"
+
+
+def _in_progress(team_id: int) -> AudienceSuggestionsResult:
+ return _parse(cache.get(_in_progress_key(team_id))) or _GENERATING
+
+
def _release(team_id: int, lock_token: str) -> None:
if cache.get(_lock_key(team_id)) == lock_token:
+ cache.delete(_in_progress_key(team_id))
cache.delete(_lock_key(team_id))
@@ -271,7 +285,10 @@ def _to_cache(result: AudienceSuggestionsResult) -> dict[str, Any]:
def _read_cache(team_id: int) -> AudienceSuggestionsResult | None:
- raw = cache.get(_cache_key(team_id))
+ return _parse(cache.get(_cache_key(team_id)))
+
+
+def _parse(raw: Any) -> AudienceSuggestionsResult | None:
if not isinstance(raw, dict):
return None
try:
diff --git a/products/workflows/backend/test/test_audience_suggestions.py b/products/workflows/backend/test/test_audience_suggestions.py
index daba4b7ee83f..e9e94c6f8dcf 100644
--- a/products/workflows/backend/test/test_audience_suggestions.py
+++ b/products/workflows/backend/test/test_audience_suggestions.py
@@ -1,8 +1,41 @@
from django.test import SimpleTestCase
+from parameterized import parameterized
+
+from products.workflows.backend.facade.contracts import AudienceKind, AudienceSuggestion
+from products.workflows.backend.services.audience_suggestions.audience import sample_denominator
from products.workflows.backend.services.audience_suggestions.service import _PROMPT_EVENTS_LIMIT, prompt_events
+class TestAudienceSampling(SimpleTestCase):
+ @parameterized.expand(
+ [
+ ("quiet_project_counts_everyone", "regulars", 7, {"used": 500_000}, 1),
+ ("busy_event_over_a_short_window", "regulars", 7, {"used": 100_000_000}, 10),
+ ("lapsed_reads_90_days_whatever_its_window", "lapsed", 7, {"used": 7_000_000}, 9),
+ ("event_without_a_count_adds_nothing", "regulars", 7, {}, 1),
+ ]
+ )
+ def test_keeps_the_scan_small(
+ self, _name: str, kind: AudienceKind, window_days: int, counts: dict[str, int], expected: int
+ ) -> None:
+ audience = AudienceSuggestion(
+ id="1",
+ name="Audience",
+ description="",
+ reason="",
+ kind=kind,
+ event="used",
+ missing_event=None,
+ window_days=window_days,
+ min_count=None,
+ reachable_count=None,
+ cohort_filters={},
+ )
+
+ assert sample_denominator([audience], counts) == expected
+
+
class TestAudiencePromptEvents(SimpleTestCase):
def test_quiet_feature_events_reach_the_prompt_past_busy_telemetry(self) -> None:
counts = {f"telemetry {index}": 1_000_000 - index for index in range(_PROMPT_EVENTS_LIMIT * 2)}
diff --git a/products/workflows/backend/tests/api/test_broadcast_audience_suggestions.py b/products/workflows/backend/tests/api/test_broadcast_audience_suggestions.py
index 91b03fe77622..6e3e9e3acd7a 100644
--- a/products/workflows/backend/tests/api/test_broadcast_audience_suggestions.py
+++ b/products/workflows/backend/tests/api/test_broadcast_audience_suggestions.py
@@ -1,5 +1,6 @@
from datetime import UTC, datetime, timedelta
+import time_machine
from posthog.test.base import APIBaseTest, ClickhouseTestMixin, _create_event, flush_persons_and_events
from unittest.mock import MagicMock, patch
@@ -15,10 +16,16 @@ from posthog.test.persons import create_person
from products.access_control.backend.models.property_access_control import PropertyAccessControl
from products.access_control.backend.property_access_control import PropertyAccessLevel
from products.cohorts.backend.models.cohort import Cohort
-from products.workflows.backend.facade.contracts import AudienceKind
-from products.workflows.backend.services.audience_suggestions.audience import cohort_filters, reachable_count
+from products.workflows.backend.facade.contracts import AudienceKind, AudienceSuggestion
+from products.workflows.backend.services.audience_suggestions.audience import cohort_filters, reachable_counts
from products.workflows.backend.services.audience_suggestions.planner import AudienceIdea
-from products.workflows.backend.services.audience_suggestions.service import _cache_key
+from products.workflows.backend.services.audience_suggestions.service import (
+ FAILURE_CACHE_TTL_SECONDS,
+ _cache_key,
+ _in_progress_key,
+ _lock_key,
+ get_audience_suggestions,
+)
SERVICE = "products.workflows.backend.services.audience_suggestions.service"
@@ -42,10 +49,33 @@ def _idea(
)
+def _audience(
+ kind: AudienceKind,
+ event: str,
+ missing_event: str | None = None,
+ window_days: int = 14,
+ min_count: int | None = None,
+) -> AudienceSuggestion:
+ return AudienceSuggestion(
+ id=f"{kind}-{event}",
+ name="Audience",
+ description="People to tell about the next step.",
+ reason="They are worth a note.",
+ kind=kind,
+ event=event,
+ missing_event=missing_event,
+ window_days=window_days,
+ min_count=min_count,
+ reachable_count=None,
+ cohort_filters={},
+ )
+
+
class TestBroadcastAudienceSuggestions(ClickhouseTestMixin, APIBaseTest):
def setUp(self) -> None:
super().setUp()
- cache.delete(_cache_key(self.team.id))
+ for key in (_cache_key, _in_progress_key, _lock_key):
+ cache.delete(key(self.team.id))
patch(f"{SERVICE}.is_privacy_mode_enabled", return_value=False).start()
self.addCleanup(patch.stopall)
self.organization.is_ai_data_processing_approved = True
@@ -88,9 +118,16 @@ class TestBroadcastAudienceSuggestions(ClickhouseTestMixin, APIBaseTest):
def _person(self, distinct_id: str, events: list[tuple[str, datetime]], email: str | None = "set") -> None:
properties = {} if email is None else {"email": f"{distinct_id}@example.com" if email == "set" else email}
- create_person(team=self.team, distinct_ids=[distinct_id], properties=properties)
+ person = create_person(team=self.team, distinct_ids=[distinct_id], properties=properties)
for event, timestamp in events:
- _create_event(team=self.team, event=event, distinct_id=distinct_id, timestamp=timestamp)
+ _create_event(
+ team=self.team,
+ event=event,
+ distinct_id=distinct_id,
+ timestamp=timestamp,
+ person_id=person.uuid,
+ person_properties=properties,
+ )
def _list(self) -> dict:
url = f"/api/projects/{self.team.id}/broadcast_audience_suggestions/"
@@ -125,12 +162,22 @@ class TestBroadcastAudienceSuggestions(ClickhouseTestMixin, APIBaseTest):
cohort.calculate_people_ch(pending_version=0)
cohort.refresh_from_db()
- count = reachable_count(
- self.team, kind=kind, event=event, missing_event=missing_event, window_days=window_days, min_count=min_count
- )
- assert count == expected
+ counts = reachable_counts(self.team, [_audience(kind, event, missing_event, window_days, min_count)])
+ assert counts == [expected]
assert cohort.count == expected
+ def test_counts_audiences_with_different_windows_in_one_query(self) -> None:
+ counts = reachable_counts(
+ self.team,
+ [
+ _audience("regulars", "report_viewed", window_days=7, min_count=3),
+ _audience("not_yet", "signed_up", "project_created"),
+ _audience("lapsed", "$pageview", window_days=30),
+ ],
+ )
+
+ assert counts == [1, 13, 11]
+
@patch(f"{SERVICE}.suggest_audiences")
def test_keeps_valid_audiences_one_per_kind_and_caches_them(self, suggest: MagicMock) -> None:
suggest.return_value = [
@@ -156,25 +203,40 @@ class TestBroadcastAudienceSuggestions(ClickhouseTestMixin, APIBaseTest):
assert self._list()["suggestions"] == body["suggestions"]
assert suggest.call_count == 1
- @patch(f"{SERVICE}.reachable_count")
+ @patch(f"{SERVICE}.reachable_counts", side_effect=ClickHouseQueryTimeOut())
@patch(f"{SERVICE}.suggest_audiences")
- def test_an_audience_too_costly_to_count_drops_only_that_card(self, suggest: MagicMock, count: MagicMock) -> None:
- suggest.return_value = [
- _idea("lapsed", "$pageview", window_days=30),
- _idea("not_yet", "signed_up", "project_created"),
- ]
+ def test_a_failed_count_keeps_the_cards_without_numbers_and_retries_soon(
+ self, suggest: MagicMock, _count: MagicMock
+ ) -> None:
+ suggest.return_value = [_idea("not_yet", "signed_up", "project_created")]
+
+ body = self._list()
+
+ assert body["status"] == "ready"
+ assert [(s["event"], s["reachable_count"]) for s in body["suggestions"]] == [("signed_up", None)]
+ with time_machine.travel(datetime.now(UTC) + timedelta(seconds=FAILURE_CACHE_TTL_SECONDS + 1)):
+ assert cache.get(_cache_key(self.team.id)) is None
+
+ @patch(f"{SERVICE}.reachable_counts")
+ @patch(f"{SERVICE}.suggest_audiences")
+ def test_cards_show_while_counting_and_keep_their_ids(self, suggest: MagicMock, count: MagicMock) -> None:
+ suggest.return_value = [_idea("not_yet", "signed_up", "project_created")]
+ during_count = []
- def size(team: object, *, kind: AudienceKind, **_: object) -> int:
- if kind == "lapsed":
- raise ClickHouseQueryTimeOut()
- return 13
+ def size(*_: object, **__: object) -> list[int]:
+ during_count.append(get_audience_suggestions(team=self.team))
+ return [13]
count.side_effect = size
body = self._list()
+ assert during_count[0].status == "generating"
+ assert during_count[0].suggestions[0].reachable_count is None
+ assert during_count[0].suggestions[0].id == body["suggestions"][0]["id"]
assert body["status"] == "ready"
- assert [(s["kind"], s["reachable_count"]) for s in body["suggestions"]] == [("not_yet", 13)]
+ assert body["suggestions"][0]["reachable_count"] == 13
+ assert cache.get(_in_progress_key(self.team.id)) is None
@patch(f"{SERVICE}.activity_counts", side_effect=ClickHouseQueryTimeOut())
@patch(f"{SERVICE}.suggest_audiences")
diff --git a/products/workflows/frontend/Broadcasts/audienceSuggestions/BroadcastAudienceSuggestions.tsx b/products/workflows/frontend/Broadcasts/audienceSuggestions/BroadcastAudienceSuggestions.tsx
index 615c4cfa6b46..0e4e66883d75 100644
--- a/products/workflows/frontend/Broadcasts/audienceSuggestions/BroadcastAudienceSuggestions.tsx
+++ b/products/workflows/frontend/Broadcasts/audienceSuggestions/BroadcastAudienceSuggestions.tsx
@@ -5,7 +5,7 @@ import { LemonButton, LemonCard, LemonSkeleton, LemonTag, Spinner, Tooltip } fro
import { TaxonomicFilterGroupType } from 'lib/components/TaxonomicFilter/types'
import { getAccessControlDisabledReason } from 'lib/utils/accessControlUtils'
-import { humanFriendlyNumber } from 'lib/utils/numbers'
+import { compactNumber } from 'lib/utils/numbers'
import { getFilterLabel } from '~/taxonomy/helpers'
import { AccessControlLevel, AccessControlResourceType } from '~/types'
@@ -68,7 +68,7 @@ export function BroadcastAudienceSuggestions(): JSX.Element | null {
</Tooltip>
</div>
<div className="grid grid-cols-[repeat(auto-fill,minmax(16rem,1fr))] gap-2">
- {generating
+ {generating && suggestions.length === 0
? Array.from({ length: LOADING_CARD_COUNT }, (_, index) => (
<LemonSkeleton key={index} className="h-44 rounded" />
))
@@ -76,10 +76,14 @@ export function BroadcastAudienceSuggestions(): JSX.Element | null {
<LemonCard key={suggestion.id} hoverEffect={false} className="flex flex-col gap-2 p-4">
<div className="flex flex-col gap-1">
<h4 className="mb-0 text-base font-semibold">{suggestion.name}</h4>
- <span className="text-xs text-secondary">
- <span translate="no">{humanFriendlyNumber(suggestion.reachable_count)}</span>{' '}
- {suggestion.reachable_count === 1 ? 'person' : 'people'} with an email
- </span>
+ {suggestion.reachable_count !== null ? (
+ <span className="text-xs text-secondary">
+ About <span translate="no">{compactNumber(suggestion.reachable_count)}</span>{' '}
+ people with an email
+ </span>
+ ) : generating ? (
+ <LemonSkeleton className="h-4 w-36" />
+ ) : null}
</div>
<p className="mb-0 text-sm text-secondary">{suggestion.reason}</p>
<p className="mb-0 text-xs text-tertiary">{audienceRule(suggestion)}</p>
diff --git a/products/workflows/frontend/Broadcasts/audienceSuggestions/broadcastAudienceSuggestionsLogic.test.ts b/products/workflows/frontend/Broadcasts/audienceSuggestions/broadcastAudienceSuggestionsLogic.test.ts
index 3d9f1a6eb5ad..1ab572a39d95 100644
--- a/products/workflows/frontend/Broadcasts/audienceSuggestions/broadcastAudienceSuggestionsLogic.test.ts
+++ b/products/workflows/frontend/Broadcasts/audienceSuggestions/broadcastAudienceSuggestionsLogic.test.ts
@@ -2,6 +2,7 @@ import { MOCK_DEFAULT_USER } from 'lib/api.mock'
import { router } from 'kea-router'
import { expectLogic } from 'kea-test-utils'
+import posthog from 'posthog-js'
import { initKeaTests } from '~/test/init'
@@ -66,6 +67,25 @@ describe('broadcastAudienceSuggestionsLogic', () => {
initKeaTests()
})
+ it('shows the cards while their counts run, and records one impression', async () => {
+ const capture = jest.spyOn(posthog, 'capture')
+ const counting = { ...suggestion, reachable_count: null }
+ jest.mocked(broadcastAudienceSuggestionsGenerate).mockResolvedValue({
+ status: 'generating',
+ suggestions: [counting],
+ })
+ const logic = broadcastAudienceSuggestionsLogic()
+ logic.mount()
+
+ await expectLogic(logic).toFinishAllListeners()
+
+ expect(logic.values.suggestions).toEqual([counting])
+ expect(logic.values.generating).toBe(true)
+ logic.actions.pollResultSuccess({ status: 'ready', suggestions: [suggestion] })
+ const shown = capture.mock.calls.filter(([event]) => event === 'broadcast audience suggestions shown')
+ expect(shown).toHaveLength(1)
+ })
+
const minutesAgo = (minutes: number): string => new Date(Date.now() - minutes * 60_000).toISOString()
const snapshot = (overrides: Record<string, any> = {}): Record<string, any> => ({
id: 7,
diff --git a/products/workflows/frontend/Broadcasts/audienceSuggestions/broadcastAudienceSuggestionsLogic.ts b/products/workflows/frontend/Broadcasts/audienceSuggestions/broadcastAudienceSuggestionsLogic.ts
index fe3932b40535..b0d87edeecd6 100644
--- a/products/workflows/frontend/Broadcasts/audienceSuggestions/broadcastAudienceSuggestionsLogic.ts
+++ b/products/workflows/frontend/Broadcasts/audienceSuggestions/broadcastAudienceSuggestionsLogic.ts
@@ -29,7 +29,7 @@ import { broadcastAudienceSuggestionsCurrent, broadcastAudienceSuggestionsGenera
import type { AudienceSuggestionApi, AudienceSuggestionsResponseApi } from '../../generated/api.schemas'
import { urlForNewBroadcastWithAudience } from '../broadcastAudiencePrefill'
-const POLL_INTERVAL_MS = 2000
+const POLL_INTERVAL_MS = 1000
const POLL_DEADLINE_MS = 5 * 60 * 1000
const REQUEST_TIMEOUT_MS = 10_000
// A snapshot older than this may hold people who no longer match the idea.
@@ -201,8 +201,7 @@ export const broadcastAudienceSuggestionsLogic = kea<broadcastAudienceSuggestion
selectors({
suggestions: [
(s) => [s.result],
- (result: AudienceSuggestionsResponseApi | null): AudienceSuggestionApi[] =>
- result?.status === 'ready' ? result.suggestions : [],
+ (result: AudienceSuggestionsResponseApi | null): AudienceSuggestionApi[] => result?.suggestions ?? [],
],
generating: [
(s) => [s.result, s.resultLoading, s.pollingTimedOut],
@@ -213,6 +212,15 @@ export const broadcastAudienceSuggestionsLogic = kea<broadcastAudienceSuggestion
listeners(({ actions, values, cache }) => {
const handleResult = (result: AudienceSuggestionsResponseApi | null): void => {
cache.disposables.dispose('poll')
+ // Cards can show before their counts, and a click can leave the page before the run ends.
+ if (result?.suggestions.length && !cache.shownCaptured) {
+ cache.shownCaptured = true
+ // pinned: analytics event name - renaming breaks dashboards
+ posthog.capture('broadcast audience suggestions shown', {
+ kinds: result.suggestions.map((suggestion) => suggestion.kind),
+ reachable_counts: result.suggestions.map((suggestion) => suggestion.reachable_count),
+ })
+ }
if (result?.status === 'generating') {
if (Date.now() > cache.pollDeadline) {
actions.giveUpPolling()
@@ -224,13 +232,6 @@ export const broadcastAudienceSuggestionsLogic = kea<broadcastAudienceSuggestion
}, 'poll')
return
}
- if (result?.status === 'ready' && result.suggestions.length > 0) {
- // pinned: analytics event name - renaming breaks dashboards
- posthog.capture('broadcast audience suggestions shown', {
- kinds: result.suggestions.map((suggestion) => suggestion.kind),
- reachable_counts: result.suggestions.map((suggestion) => suggestion.reachable_count),
- })
- }
}
return {
loadResultSuccess: ({ result }) => {
diff --git a/products/workflows/frontend/generated/api.schemas.ts b/products/workflows/frontend/generated/api.schemas.ts
index 89ceffdc23c6..0ce1715293ec 100644
--- a/products/workflows/frontend/generated/api.schemas.ts
+++ b/products/workflows/frontend/generated/api.schemas.ts
@@ -66,14 +66,17 @@ export interface AudienceSuggestionApi {
* @nullable
*/
min_count: number | null
- /** People in the audience with an email address, counted when the suggestion was made. */
- reachable_count: number
+ /**
+ * Roughly how many people in the audience have an email address, counted when the suggestion was made. Null while the count runs, or when it failed.
+ * @nullable
+ */
+ reachable_count: number | null
/** The audience's cohort filters, in the cohorts endpoint's `filters` format. Saved as a static cohort. */
cohort_filters: unknown
}
export interface AudienceSuggestionsResponseApi {
- /** ready when suggestions were made (the list can still be empty), generating while a background run makes them (poll current until it ends), ai_not_approved when the organization has not approved AI data processing, unavailable when nothing was made or the run failed.
+ /** ready when suggestions were made (the list can still be empty), generating while a background run makes them (poll current until it ends, the list can already hold cards being counted), ai_not_approved when the organization has not approved AI data processing, unavailable when nothing was made or the run failed.
*
* * `ready` - Ready
* * `generating` - Generating
diff --git a/services/mcp/src/api/generated.ts b/services/mcp/src/api/generated.ts
index 5199a52a6191..6bf3282bca58 100644
--- a/services/mcp/src/api/generated.ts
+++ b/services/mcp/src/api/generated.ts
@@ -15001,8 +15001,11 @@ export namespace Schemas {
* @nullable
*/
min_count: number | null;
- /** People in the audience with an email address, counted when the suggestion was made. */
- reachable_count: number;
+ /**
+ * Roughly how many people in the audience have an email address, counted when the suggestion was made. Null while the count runs, or when it failed.
+ * @nullable
+ */
+ reachable_count: number | null;
/** The audience's cohort filters, in the cohorts endpoint's `filters` format. Saved as a static cohort. */
cohort_filters: unknown;
}
@@ -15024,7 +15027,7 @@ export namespace Schemas {
} as const;
export interface AudienceSuggestionsResponse {
- /** ready when suggestions were made (the list can still be empty), generating while a background run makes them (poll current until it ends), ai_not_approved when the organization has not approved AI data processing, unavailable when nothing was made or the run failed.
+ /** ready when suggestions were made (the list can still be empty), generating while a background run makes them (poll current until it ends, the list can already hold cards being counted), ai_not_approved when the organization has not approved AI data processing, unavailable when nothing was made or the run failed.
*
* * `ready` - Ready
* * `generating` - Generating