Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(workflows): show dry run and sample floor on the self-driving tab (#118283)

Daniel Marchukdda5300a9536
diff --git a/products/signals/backend/facade/api.py b/products/signals/backend/facade/api.py
index f7569ed83097..645a36b01e76 100644
--- a/products/signals/backend/facade/api.py
+++ b/products/signals/backend/facade/api.py
@@ -1685,6 +1685,16 @@ def scout_status_for_product(*, team_id: int, skill_name: str) -> str | None:
)
+def scout_emits_for_product(*, team_id: int, skill_name: str) -> bool | None:
+ """Whether the scout config for the skill writes what it finds, or None when the project has no config for it.
+
+ False is a dry run: the scout runs and logs, but gets no write scopes, so it files nothing.
+ """
+ return (
+ SignalScoutConfig.objects.for_team(team_id).filter(skill_name=skill_name).values_list("emit", flat=True).first()
+ )
+
+
def disable_scout_for_product(*, team_id: int, skill_name: str, source_product: str) -> bool:
"""Remove the config a product created when it switched a scout on.
diff --git a/products/workflows/backend/facade/contracts.py b/products/workflows/backend/facade/contracts.py
index d00664260d76..0da05ca54a1c 100644
--- a/products/workflows/backend/facade/contracts.py
+++ b/products/workflows/backend/facade/contracts.py
@@ -539,6 +539,17 @@ class Workflow:
suggestions_enabled: bool | None = None
+@frozen
+class SuggestionsSample:
+ """Whether the live version has sent enough email for the suggestions scout to judge it."""
+
+ reached: bool
+ # Tracked sends (sent minus untracked) from the live version's busiest email step in the window.
+ tracked_sends: int
+ required_sends: int
+ version_live_since: datetime
+
+
@frozen
class WorkflowRef:
"""A workflow's identity and dispatch settings, for an action that does not return the workflow.
diff --git a/products/workflows/backend/facade/suggestions_scout.py b/products/workflows/backend/facade/suggestions_scout.py
index aefab10550dc..1b7924ee6099 100644
--- a/products/workflows/backend/facade/suggestions_scout.py
+++ b/products/workflows/backend/facade/suggestions_scout.py
@@ -2,9 +2,16 @@
from products.workflows.backend.services.suggestions_scout import (
PROPOSAL_WRITE_SCOPE,
+ suggestions_sample,
suggestions_scout_refusal,
suggestions_scout_status,
sync_suggestions_scout,
)
-__all__ = ["PROPOSAL_WRITE_SCOPE", "suggestions_scout_refusal", "suggestions_scout_status", "sync_suggestions_scout"]
+__all__ = [
+ "PROPOSAL_WRITE_SCOPE",
+ "suggestions_sample",
+ "suggestions_scout_refusal",
+ "suggestions_scout_status",
+ "sync_suggestions_scout",
+]
diff --git a/products/workflows/backend/presentation/views/hog_flow.py b/products/workflows/backend/presentation/views/hog_flow.py
index b910dc3750aa..80ca917cd575 100644
--- a/products/workflows/backend/presentation/views/hog_flow.py
+++ b/products/workflows/backend/presentation/views/hog_flow.py
@@ -266,6 +266,7 @@ from products.workflows.backend.facade.secrets import (
from products.workflows.backend.facade.send_calendar import SEND_CALENDAR_MAX_RANGE_DAYS, list_send_calendar_flows
from products.workflows.backend.facade.suggestions_scout import (
PROPOSAL_WRITE_SCOPE,
+ suggestions_sample,
suggestions_scout_refusal,
suggestions_scout_status,
sync_suggestions_scout,
@@ -4030,16 +4031,43 @@ class SuggestionsScoutStatus(models.TextChoices):
RUNNING = "running"
PAUSED_BY_USER = "paused_by_user"
PAUSED_BY_SYSTEM = "paused_by_system"
+ DRY_RUN = "dry_run"
NOT_RUNNING = "not_running"
+class SuggestionsSampleSerializer(serializers.Serializer):
+ reached = serializers.BooleanField(
+ read_only=True,
+ help_text="Whether an email step of the live version has enough tracked sends, over enough time, for "
+ "PostHog to judge it.",
+ )
+ tracked_sends = serializers.IntegerField(
+ read_only=True,
+ help_text="Tracked sends (sends with open tracking on) from the live version's busiest email step in the "
+ "last 7 days.",
+ )
+ required_sends = serializers.IntegerField(
+ read_only=True, help_text="Tracked sends an email step needs before PostHog judges it."
+ )
+ version_live_since = serializers.DateTimeField(
+ read_only=True, help_text="When the live version was published. PostHog waits 48 hours before judging it."
+ )
+
+
class HogFlowOptimizationSerializer(serializers.Serializer):
enabled = serializers.BooleanField(help_text="Whether PostHog may suggest changes to this workflow.")
scout_status = serializers.ChoiceField(
choices=SuggestionsScoutStatus.choices,
read_only=True,
help_text="Whether the project's suggestions scout runs. A paused scout files no suggestions, even for "
- "workflows that have suggestions on.",
+ "workflows that have suggestions on. A scout in dry run runs but files nothing until someone turns on "
+ "writing to the inbox in its scout settings.",
+ )
+ sample = SuggestionsSampleSerializer(
+ read_only=True,
+ allow_null=True,
+ help_text="How close the live version is to the sample PostHog needs before it suggests anything. Null "
+ "while suggestions are on but the scout is not running, or when the workflow has no email step.",
)
@@ -6075,10 +6103,19 @@ class HogFlowViewSet(
else:
enabled = is_optimization_enabled(instance.id)
+ scout_status = suggestions_scout_status(self.team)
+ sample = None
+ # Off, the sample tells the person what to expect before they turn it on. On, only a running scout is
+ # held back by it, so the metrics read is skipped while the scout is paused or in dry run.
+ if not enabled or scout_status == SuggestionsScoutStatus.RUNNING:
+ tag_queries(product=ProductKey.WORKFLOWS, feature=Feature.QUERY)
+ try:
+ sample = suggestions_sample(team_id=self.team_id, hog_flow_id=instance.id)
+ except Exception as error:
+ # The sample only adds a line of copy, so a failed metrics read must not fail the setting.
+ capture_exception(error)
return Response(
- HogFlowOptimizationSerializer(
- {"enabled": enabled, "scout_status": suggestions_scout_status(self.team)}
- ).data
+ HogFlowOptimizationSerializer({"enabled": enabled, "scout_status": scout_status, "sample": sample}).data
)
def _may_grant_scout(self, request: Request) -> bool:
diff --git a/products/workflows/backend/services/suggestions_scout.py b/products/workflows/backend/services/suggestions_scout.py
index 833a5607a42d..3816ac4bf0b4 100644
--- a/products/workflows/backend/services/suggestions_scout.py
+++ b/products/workflows/backend/services/suggestions_scout.py
@@ -5,25 +5,59 @@ none gains nothing from its runs. The first live opted-in workflow switches the
project, and losing the last one, by opting out, archiving or deleting it, removes it again.
"""
+from datetime import UTC, timedelta
+from uuid import UUID
+
from django.db import connection, transaction
from django.db.models import Q
+from django.utils import timezone
+from posthog.clickhouse.client.execute import sync_execute
from posthog.models import Team, User
from products.signals.backend.facade.api import (
ScoutEnableRefusal,
disable_scout_for_product,
enable_scout_for_product,
+ scout_emits_for_product,
scout_enable_refusal_for_product,
scout_status_for_product,
)
-from products.workflows.backend.models import HogFlow, HogFlowOptimization
+from products.workflows.backend.facade.contracts import SuggestionsSample
+from products.workflows.backend.metrics import (
+ HOG_FLOW_VERSION_APP_SOURCE,
+ MIN_EVIDENCE_SAMPLE,
+ TARGET_SEND_METRIC,
+ TARGET_UNTRACKED_METRIC,
+)
+from products.workflows.backend.models import HogFlow, HogFlowOptimization, HogFlowRevision
SUGGESTIONS_SCOUT = "signals-scout-workflows"
SOURCE_PRODUCT = "workflows"
PROPOSAL_WRITE_SCOPE = "hog_flow_proposal:write"
+# The scout's sample floor: tracked sends per email step, read over this window, from a version live this long.
+SAMPLE_WINDOW = timedelta(days=7)
+SAMPLE_MIN_VERSION_AGE = timedelta(hours=48)
# Arbitrary, fixed namespace for the per-project advisory lock that serializes reconciliation.
_LOCK_NAMESPACE = 72_031
+_SAMPLE_QUERY_TIMEOUT_SECONDS = 5
+# Bounds compare on the hour because the table collapses rows per clock hour and keeps one arbitrary timestamp.
+_STEP_SENDS_QUERY = """
+ SELECT
+ instance_id,
+ sumIf(count, metric_name = %(sent)s AND toStartOfHour(timestamp) >= toDateTime64(%(after)s, 6)
+ AND toStartOfHour(timestamp) < toDateTime64(%(before)s, 6)) AS sent,
+ sumIf(count, metric_name = %(untracked)s AND toStartOfHour(timestamp) >= toDateTime64(%(after)s, 6)
+ AND toStartOfHour(timestamp) < toDateTime64(%(before)s, 6)) AS untracked,
+ min(toStartOfHour(timestamp)) AS first_send
+ FROM app_metrics2
+ WHERE team_id = %(team_id)s
+ AND app_source = %(app_source)s
+ AND app_source_id = %(app_source_id)s
+ AND instance_id IN %(step_ids)s
+ AND metric_name IN (%(sent)s, %(untracked)s)
+ GROUP BY instance_id
+"""
def sync_suggestions_scout(team: Team, *, acting_user: User | None, may_grant: bool) -> None:
@@ -72,9 +106,67 @@ def suggestions_scout_refusal(team: Team, *, acting_user: User, may_grant: bool)
def suggestions_scout_status(team: Team) -> str:
- """`running`, `paused_by_user`, `paused_by_system` or `not_running` for the project's suggestions scout."""
+ """`running`, `dry_run`, `paused_by_user`, `paused_by_system` or `not_running` for the project's suggestions scout."""
project = team.parent_team or team
status = scout_status_for_product(team_id=project.id, skill_name=SUGGESTIONS_SCOUT)
+ if status is None:
+ return "not_running"
if status in ("paused_by_user", "paused_by_system"):
return status
- return "running" if status is not None else "not_running"
+ # A dry run holds no write scopes, so it can never file a suggestion.
+ if scout_emits_for_product(team_id=project.id, skill_name=SUGGESTIONS_SCOUT) is False:
+ return "dry_run"
+ return "running"
+
+
+def suggestions_sample(*, team_id: int, hog_flow_id: UUID) -> SuggestionsSample | None:
+ """Whether any email step of the live version clears the scout's sample floor, or None without an email step."""
+ flow = (
+ HogFlow.objects.filter(team_id=team_id, pk=hog_flow_id).only("id", "version", "actions", "created_at").first()
+ )
+ if flow is None:
+ return None
+ step_ids = [
+ str(action["id"])
+ for action in flow.actions or []
+ if isinstance(action, dict) and action.get("type") == "function_email" and action.get("id")
+ ]
+ if not step_ids:
+ return None
+ now = timezone.now()
+ # The scout reads whole UTC days that have closed, so the window ends at the last midnight.
+ window_end = now.astimezone(UTC).replace(hour=0, minute=0, second=0, microsecond=0)
+ rows = sync_execute(
+ _STEP_SENDS_QUERY,
+ {
+ "team_id": team_id,
+ "app_source": HOG_FLOW_VERSION_APP_SOURCE,
+ "app_source_id": f"{flow.id}/{flow.version}",
+ "step_ids": step_ids,
+ "sent": TARGET_SEND_METRIC,
+ "untracked": TARGET_UNTRACKED_METRIC,
+ "after": (window_end - SAMPLE_WINDOW).strftime("%Y-%m-%dT%H:%M:%S"),
+ "before": window_end.strftime("%Y-%m-%dT%H:%M:%S"),
+ },
+ settings={"max_execution_time": _SAMPLE_QUERY_TIMEOUT_SECONDS},
+ team_id=team_id,
+ )
+ tracked_sends = max((int(sent) - int(untracked) for _, sent, untracked, _ in rows), default=0)
+ first_sends = [first_send for *_, first_send in rows]
+ if first_sends:
+ # Activating a draft does not create a revision, so the version's first send is when it went live.
+ live_since = min(first_sends)
+ else:
+ # A flow with no revision for its version predates revision tracking, so it has been live at least since creation.
+ revision = (
+ HogFlowRevision.objects.for_team(team_id)
+ .filter(hog_flow=flow, version=flow.version)
+ .values_list("created_at", flat=True)
+ )
+ live_since = revision.first() or flow.created_at
+ return SuggestionsSample(
+ reached=tracked_sends >= MIN_EVIDENCE_SAMPLE and now - live_since >= SAMPLE_MIN_VERSION_AGE,
+ tracked_sends=tracked_sends,
+ required_sends=MIN_EVIDENCE_SAMPLE,
+ version_live_since=live_since,
+ )
diff --git a/products/workflows/backend/tests/api/test_workflow_proposals.py b/products/workflows/backend/tests/api/test_workflow_proposals.py
index 6da5822a387f..e2da1bf46bf8 100644
--- a/products/workflows/backend/tests/api/test_workflow_proposals.py
+++ b/products/workflows/backend/tests/api/test_workflow_proposals.py
@@ -1,9 +1,10 @@
import json
from copy import deepcopy
-from datetime import UTC, datetime
+from datetime import UTC, datetime, timedelta
from types import SimpleNamespace
from typing import Any
+import time_machine
from posthog.test.base import APIBaseTest
from unittest.mock import MagicMock, patch
@@ -685,6 +686,7 @@ class TestWorkflowProposals(APIBaseTest):
("a member widening another person's scout", "widen", "running"),
("api key without the proposal scope turning one off", "key_off", "not_running"),
("a scout a person set up", "own", "running"),
+ ("a scout a person set to dry run", "dry_run", "dry_run"),
]
)
def test_suggestions_scout_gains_no_grant(self, _mock_flag, _name: str, case: str, scout_status: str):
@@ -716,9 +718,9 @@ class TestWorkflowProposals(APIBaseTest):
created_by=other,
enabled_by=other,
)
- elif case == "own":
+ elif case in ("own", "dry_run"):
SignalScoutConfig.objects.for_team(self.team.id).create(
- team=self.team, skill_name="signals-scout-workflows", enabled=True
+ team=self.team, skill_name="signals-scout-workflows", enabled=True, emit=case != "dry_run"
)
elif case == "key_off":
key = generate_random_token_personal()
@@ -756,6 +758,62 @@ class TestWorkflowProposals(APIBaseTest):
if case == "system_paused":
assert scout is not None and scout.status == SignalScoutConfig.Status.PAUSED_BY_SYSTEM
+ @parameterized.expand(
+ [
+ ("too few sends", True, (12, 0), 72, 72, False, 12),
+ ("untracked sends do not count", True, (25, 10), 72, 72, False, 15),
+ ("a version live under two days", True, (30, 0), 24, 24, False, 30),
+ ("a draft activated under two days ago", True, (30, 0), 72, 24, False, 30),
+ ("enough sends on a settled version", True, (30, 0), 72, 72, True, 30),
+ ("too few sends before suggestions are on", False, (4, 0), 72, 72, False, 4),
+ ]
+ )
+ def test_optimization_reports_the_sample_floor_of_the_live_version(
+ self,
+ _mock_flag,
+ _name: str,
+ optimize: bool,
+ sends: tuple[int, int],
+ hours_since_created: int,
+ hours_since_first_send: int,
+ reached: bool,
+ tracked_sends: int,
+ ):
+ flow_id = self._create_active_flow(optimize=optimize)
+ flow = HogFlow.objects.get(pk=flow_id)
+ now = timezone.now() + timedelta(hours=hours_since_created)
+ first_send = now - timedelta(hours=hours_since_first_send)
+ with (
+ time_machine.travel(now, tick=False),
+ patch("products.workflows.backend.services.suggestions_scout.sync_execute") as mock_execute,
+ ):
+ mock_execute.return_value = [("email_1", *sends, first_send)]
+ response = self.client.get(f"/api/projects/{self.team.id}/hog_flows/{flow_id}/optimization")
+
+ assert response.status_code == 200, response.json()
+ sample = response.json()["sample"]
+ assert (sample["reached"], sample["tracked_sends"], sample["required_sends"]) == (reached, tracked_sends, 20)
+ args = mock_execute.call_args.args[1]
+ assert (args["app_source_id"], args["step_ids"]) == (f"{flow_id}/{flow.version}", ["email_1"])
+
+ @parameterized.expand(
+ [("scout in dry run", {"emit": False}), ("scout paused", {"enabled": False, "status": "paused_by_user"})]
+ )
+ def test_optimization_skips_the_sample_when_no_suggestion_could_be_filed(
+ self, _mock_flag, _name: str, scout_change: dict
+ ):
+ flow_id = self._create_active_flow()
+ SignalScoutConfig.objects.for_team(self.team.id).filter(skill_name="signals-scout-workflows").update(
+ **scout_change
+ )
+
+ with patch("products.workflows.backend.services.suggestions_scout.sync_execute") as mock_execute:
+ response = self.client.get(f"/api/projects/{self.team.id}/hog_flows/{flow_id}/optimization")
+
+ assert response.status_code == 200, response.json()
+ assert response.json()["sample"] is None
+ mock_execute.assert_not_called()
+
@parameterized.expand(
[
("a project at its enabled-scout limit", "at_cap", "already runs as many scouts"),
diff --git a/products/workflows/frontend/Workflows/suggestions/WorkflowSuggestions.tsx b/products/workflows/frontend/Workflows/suggestions/WorkflowSuggestions.tsx
index df417124ae77..dc2a7b0f5a34 100644
--- a/products/workflows/frontend/Workflows/suggestions/WorkflowSuggestions.tsx
+++ b/products/workflows/frontend/Workflows/suggestions/WorkflowSuggestions.tsx
@@ -42,6 +42,23 @@ function SuggestionsPausedNotice({ byUser }: { byUser: boolean }): JSX.Element {
)
}
+function SuggestionsDryRunNotice(): JSX.Element {
+ return (
+ <LemonBanner
+ type="warning"
+ action={{
+ children: 'Open scout settings',
+ to: urls.inboxScout('signals-scout-workflows'),
+ // pinned: data-attr - autocapture dashboards read it
+ 'data-attr': 'workflow-suggestions-scout-dry-run-open',
+ }}
+ >
+ The workflows scout is in dry run, so it can't file new suggestions. Turn on "Write signals to the inbox" in
+ its settings to get them.
+ </LemonBanner>
+ )
+}
+
function SuggestionsUnreadableList({ onRetry }: { onRetry: () => void }): JSX.Element {
return (
<div className="flex flex-col items-start gap-2">
@@ -121,13 +138,15 @@ export function WorkflowSuggestions({ id }: { id: string }): JSX.Element {
optimizationEnabled && (scoutStatus === 'paused_by_user' || scoutStatus === 'paused_by_system') ? (
<SuggestionsPausedNotice byUser={scoutStatus === 'paused_by_user'} />
) : null
+ const scoutNotice =
+ pausedNotice ?? (optimizationEnabled && scoutStatus === 'dry_run' ? <SuggestionsDryRunNotice /> : null)
// A failed read leaves the setting unknown, so it cannot stand in for "off".
const notice = optimizationUnreadable ? (
<SuggestionsUnreadableNotice />
) : !optimizationEnabled ? (
<SuggestionsOffNotice />
) : (
- pausedNotice
+ scoutNotice
)
if (nothingFiled) {
@@ -137,7 +156,7 @@ export function WorkflowSuggestions({ id }: { id: string }): JSX.Element {
return optimizationUnreadable ? (
<SuggestionsUnreadableNotice />
) : (
- <WorkflowSuggestionsIntroduction id={id} enabled={false} />
+ <WorkflowSuggestionsIntroduction id={id} enabled={false} sample={optimization?.sample} />
)
}
@@ -145,7 +164,12 @@ export function WorkflowSuggestions({ id }: { id: string }): JSX.Element {
return (
<div className="flex flex-col gap-4">
{pausedNotice}
- <WorkflowSuggestionsIntroduction id={id} enabled />
+ <WorkflowSuggestionsIntroduction
+ id={id}
+ enabled
+ scoutStatus={scoutStatus}
+ sample={optimization?.sample}
+ />
</div>
)
}
diff --git a/products/workflows/frontend/Workflows/suggestions/WorkflowSuggestionsIntroduction.tsx b/products/workflows/frontend/Workflows/suggestions/WorkflowSuggestionsIntroduction.tsx
index bbf41cf98b9e..3b5e70d86321 100644
--- a/products/workflows/frontend/Workflows/suggestions/WorkflowSuggestionsIntroduction.tsx
+++ b/products/workflows/frontend/Workflows/suggestions/WorkflowSuggestionsIntroduction.tsx
@@ -1,23 +1,62 @@
import * as research from '@posthog/brand/hoggies/png/research'
import * as stampApproved from '@posthog/brand/hoggies/png/stamp-approved'
+import { LemonButton } from '@posthog/lemon-ui'
import { pngHoggie } from 'lib/brand/hoggies'
import { ProductIntroduction } from 'lib/components/ProductIntroduction/ProductIntroduction'
+import { dayjs } from 'lib/dayjs'
+import { urls } from 'scenes/urls'
+import type { SuggestionsSampleApi, SuggestionsScoutStatusEnumApi } from '../../generated/api.schemas'
import { WorkflowSuggestionsSwitch } from './WorkflowSuggestionsSwitch'
const HedgehogStampApproved = pngHoggie(stampApproved)
const HedgehogResearch = pngHoggie(research)
const DOCS_URL = 'https://posthog.com/docs/workflows/suggestions'
+const MIN_VERSION_AGE_HOURS = 48
-export function WorkflowSuggestionsIntroduction({ id, enabled }: { id: string; enabled: boolean }): JSX.Element {
+function sampleProgress(sample: SuggestionsSampleApi | null | undefined): string | null {
+ if (!sample || sample.reached) {
+ return null
+ }
+ const sends = `${sample.tracked_sends} tracked ${sample.tracked_sends === 1 ? 'send' : 'sends'}`
+ const needs = `PostHog needs ${sample.required_sends} tracked sends on one email step within 7 days, from a version that has been live for at least 2 days.`
+ const liveSince = dayjs(sample.version_live_since)
+ const tooNew = dayjs().diff(liveSince, 'hour') < MIN_VERSION_AGE_HOURS
+ if (sample.tracked_sends >= sample.required_sends) {
+ return `This version went live ${liveSince.fromNow()}. PostHog waits until a version has been live for 2 days before it judges it.`
+ }
+ if (tooNew) {
+ return `This version went live ${liveSince.fromNow()}, and its busiest email step has had ${sends} in the last 7 days. ${needs}`
+ }
+ return `This version's busiest email step has had ${sends} in the last 7 days. ${needs}`
+}
+
+export function WorkflowSuggestionsIntroduction({
+ id,
+ enabled,
+ scoutStatus,
+ sample,
+}: {
+ id: string
+ enabled: boolean
+ scoutStatus?: SuggestionsScoutStatusEnumApi
+ sample?: SuggestionsSampleApi | null
+}): JSX.Element {
+ const progress = sampleProgress(sample)
if (!enabled) {
return (
<ProductIntroduction
thingName="suggestion"
titleOverride="Let PostHog suggest improvements"
- description="PostHog reads how this workflow performs, such as how many people open and click each email, and suggests one concrete change when a step underperforms. Nothing reaches anyone until you approve a suggestion and publish it."
+ description={[
+ 'PostHog reads how this workflow performs, such as how many people open and click each email, and suggests one concrete change when a step underperforms. Nothing reaches anyone until you approve a suggestion and publish it.',
+ progress &&
+ `PostHog can't judge this workflow yet. ${progress} You can turn suggestions on now, and they start once it can.`,
+ ]
+ .filter(Boolean)
+ .join(' ')}
actionElementOverride={<WorkflowSuggestionsSwitch id={id} />}
docsURL={DOCS_URL}
customHog={HedgehogStampApproved}
@@ -27,11 +66,43 @@ export function WorkflowSuggestionsIntroduction({ id, enabled }: { id: string; e
)
}
+ if (scoutStatus === 'dry_run') {
+ return (
+ <ProductIntroduction
+ thingName="suggestion"
+ titleOverride="Suggestions are on hold while the workflows scout is in dry run"
+ description={`The workflows scout reads this workflow, but in dry run it can't file suggestions. To get suggestions, open the scout's settings and turn on "Write signals to the inbox".`}
+ actionElementOverride={
+ <>
+ <LemonButton
+ type="primary"
+ to={urls.inboxScout('signals-scout-workflows')}
+ // pinned: data-attr - autocapture dashboards read it
+ data-attr="workflow-suggestions-scout-dry-run-open"
+ >
+ Open scout settings
+ </LemonButton>
+ <WorkflowSuggestionsSwitch id={id} />
+ </>
+ }
+ docsURL={DOCS_URL}
+ customHog={HedgehogResearch}
+ hogLayout="responsive"
+ useMainContentContainerQueries
+ />
+ )
+ }
+
return (
<ProductIntroduction
thingName="suggestion"
titleOverride="Watching this workflow"
- description="PostHog reads this workflow's metrics once a day and files a suggestion here when it finds a change worth making. It waits for a version to collect a couple of days of opens before judging it, so a fresh publish takes a few days to read."
+ description={[
+ "PostHog reads this workflow's metrics once a day and files a suggestion here when it finds a change worth making. It waits for a version to collect a couple of days of opens before judging it, so a fresh publish takes a few days to read. Transactional emails, like receipts or support replies, are not reviewed.",
+ progress,
+ ]
+ .filter(Boolean)
+ .join(' ')}
actionElementOverride={<WorkflowSuggestionsSwitch id={id} />}
docsURL={DOCS_URL}
customHog={HedgehogResearch}
diff --git a/products/workflows/frontend/generated/api.schemas.ts b/products/workflows/frontend/generated/api.schemas.ts
index 9d9762c86cab..e1086f48d632 100644
--- a/products/workflows/frontend/generated/api.schemas.ts
+++ b/products/workflows/frontend/generated/api.schemas.ts
@@ -1305,6 +1305,7 @@ export interface AppMetricsTotalsResponseApi {
* * `running` - Running
* * `paused_by_user` - Paused By User
* * `paused_by_system` - Paused By System
+ * * `dry_run` - Dry Run
* * `not_running` - Not Running
*/
export type SuggestionsScoutStatusEnumApi =
@@ -1314,19 +1315,34 @@ export const SuggestionsScoutStatusEnumApi = {
Running: 'running',
PausedByUser: 'paused_by_user',
PausedBySystem: 'paused_by_system',
+ DryRun: 'dry_run',
NotRunning: 'not_running',
} as const
+export interface SuggestionsSampleApi {
+ /** Whether an email step of the live version has enough tracked sends, over enough time, for PostHog to judge it. */
+ readonly reached: boolean
+ /** Tracked sends (sends with open tracking on) from the live version's busiest email step in the last 7 days. */
+ readonly tracked_sends: number
+ /** Tracked sends an email step needs before PostHog judges it. */
+ readonly required_sends: number
+ /** When the live version was published. PostHog waits 48 hours before judging it. */
+ readonly version_live_since: string
+}
+
export interface HogFlowOptimizationApi {
/** Whether PostHog may suggest changes to this workflow. */
enabled: boolean
- /** Whether the project's suggestions scout runs. A paused scout files no suggestions, even for workflows that have suggestions on.
+ /** Whether the project's suggestions scout runs. A paused scout files no suggestions, even for workflows that have suggestions on. A scout in dry run runs but files nothing until someone turns on writing to the inbox in its scout settings.
*
* * `running` - Running
* * `paused_by_user` - Paused By User
* * `paused_by_system` - Paused By System
+ * * `dry_run` - Dry Run
* * `not_running` - Not Running */
readonly scout_status: SuggestionsScoutStatusEnumApi
+ /** How close the live version is to the sample PostHog needs before it suggests anything. Null while suggestions are on but the scout is not running, or when the workflow has no email step. */
+ readonly sample: SuggestionsSampleApi | null
}
/**
diff --git a/services/mcp/src/api/generated.ts b/services/mcp/src/api/generated.ts
index 688717666b5b..59c4647a25e4 100644
--- a/services/mcp/src/api/generated.ts
+++ b/services/mcp/src/api/generated.ts
@@ -56313,6 +56313,7 @@ export namespace Schemas {
* * `running` - Running
* * `paused_by_user` - Paused By User
* * `paused_by_system` - Paused By System
+ * * `dry_run` - Dry Run
* * `not_running` - Not Running
*/
export type SuggestionsScoutStatusEnum = typeof SuggestionsScoutStatusEnum[keyof typeof SuggestionsScoutStatusEnum];
@@ -56322,19 +56323,34 @@ export namespace Schemas {
Running: 'running',
PausedByUser: 'paused_by_user',
PausedBySystem: 'paused_by_system',
+ DryRun: 'dry_run',
NotRunning: 'not_running',
} as const;
+ export interface SuggestionsSample {
+ /** Whether an email step of the live version has enough tracked sends, over enough time, for PostHog to judge it. */
+ readonly reached: boolean;
+ /** Tracked sends (sends with open tracking on) from the live version's busiest email step in the last 7 days. */
+ readonly tracked_sends: number;
+ /** Tracked sends an email step needs before PostHog judges it. */
+ readonly required_sends: number;
+ /** When the live version was published. PostHog waits 48 hours before judging it. */
+ readonly version_live_since: string;
+ }
+
export interface HogFlowOptimization {
/** Whether PostHog may suggest changes to this workflow. */
enabled: boolean;
- /** Whether the project's suggestions scout runs. A paused scout files no suggestions, even for workflows that have suggestions on.
+ /** Whether the project's suggestions scout runs. A paused scout files no suggestions, even for workflows that have suggestions on. A scout in dry run runs but files nothing until someone turns on writing to the inbox in its scout settings.
*
* * `running` - Running
* * `paused_by_user` - Paused By User
* * `paused_by_system` - Paused By System
+ * * `dry_run` - Dry Run
* * `not_running` - Not Running */
readonly scout_status: SuggestionsScoutStatusEnum;
+ /** How close the live version is to the sample PostHog needs before it suggests anything. Null while suggestions are on but the scout is not running, or when the workflow has no email step. */
+ readonly sample: SuggestionsSample | null;
}
export interface HogFlowPublishImpactMoveTarget {