Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(workflows): measure a suggestion over the exact window it names (#118578)

Daniel Marchuk815570d22c94
diff --git a/posthog/api/app_metrics2.py b/posthog/api/app_metrics2.py
index bbcc0e22887f..e9723a7c9d8d 100644
--- a/posthog/api/app_metrics2.py
+++ b/posthog/api/app_metrics2.py
@@ -199,6 +199,7 @@ def fetch_app_metric_totals(
instance_id: Optional[str] = None,
name: Optional[list[str]] = None,
kind: Optional[list[str]] = None,
+ before_exclusive: bool = False,
) -> AppMetricsTotalsResponse:
"""
Calculate the totals for the app metrics over the given period.
@@ -220,6 +221,8 @@ def fetch_app_metric_totals(
"kind": kind,
}
+ before_operator = "<" if before_exclusive else "<="
+
clickhouse_query = f"""
SELECT
metric_{breakdown_by} as breakdown,
@@ -229,7 +232,7 @@ def fetch_app_metric_totals(
AND app_source = %(app_source)s
AND app_source_id = %(app_source_id)s
{"AND timestamp >= toDateTime64(%(after)s, 6)" if after else ""}
- {"AND timestamp <= toDateTime64(%(before)s, 6)" if before else ""}
+ {f"AND timestamp {before_operator} toDateTime64(%(before)s, 6)" if before else ""}
{"AND instance_id = %(instance_id)s" if instance_id else ""}
{"AND metric_name IN %(name)s" if name else ""}
{"AND metric_kind IN %(kind)s" if kind else ""}
diff --git a/products/workflows/backend/facade/proposals.py b/products/workflows/backend/facade/proposals.py
index c44995ef2ef6..6f7d1b176ad8 100644
--- a/products/workflows/backend/facade/proposals.py
+++ b/products/workflows/backend/facade/proposals.py
@@ -5,6 +5,7 @@ from products.workflows.backend.services.proposal_approval import approve_propos
from products.workflows.backend.services.proposal_notifications import notify_proposal_filed
from products.workflows.backend.services.workflow_proposals import (
PROPOSAL_MERGE_BY_ID_FIELDS,
+ absolute_window,
as_the_serializer_stores_it,
count_proposals,
create_proposal,
@@ -25,6 +26,7 @@ from products.workflows.backend.services.workflow_proposals import (
)
__all__ = [
+ "absolute_window",
"approve_proposal",
"HOG_FLOW_VERSION_APP_SOURCE",
"PROPOSAL_MERGE_BY_ID_FIELDS",
diff --git a/products/workflows/backend/presentation/views/hog_flow.py b/products/workflows/backend/presentation/views/hog_flow.py
index 1b5aa5efc0dd..b910dc3750aa 100644
--- a/products/workflows/backend/presentation/views/hog_flow.py
+++ b/products/workflows/backend/presentation/views/hog_flow.py
@@ -16,6 +16,7 @@ from django.core.signing import BadSignature, SignatureExpired, TimestampSigner
from django.db import models, transaction
from django.db.models import QuerySet
from django.http import Http404, HttpResponse
+from django.utils import timezone as django_timezone
import requests
import structlog
@@ -222,6 +223,7 @@ from products.workflows.backend.facade.models import HogFlow
from products.workflows.backend.facade.proposals import (
HOG_FLOW_VERSION_APP_SOURCE,
PROPOSAL_MERGE_BY_ID_FIELDS,
+ absolute_window,
approve_proposal,
as_the_serializer_stores_it,
count_proposals,
@@ -4010,10 +4012,11 @@ WORKFLOW_PROPOSAL_EVIDENCE_SCHEMA = {
"{metric, value, n, unit} counter-metrics read over the same window, empty only if none apply. "
"PostHog then reads the step's own metrics at `base_version` when the suggestion is filed and stores "
"them under `measured`; the page shows that reading and flags a disagreement with yours. "
- "Also conventional: target_value, window, query, app_source_id. A rate with no denominator "
- "lets a reviewer mistake noise for a result, a target with no counter-metrics hides a change "
- "that lifts one number by harming another, and a number under a key of your own reads to a "
- "person as no evidence at all."
+ "Also conventional: target_value, window (a relative one such as `-7d`, or an ISO 8601 "
+ "interval `start/end` with offsets, which PostHog reads back exactly), query, app_source_id. "
+ "A rate with no denominator lets a reviewer mistake noise for a result, a target with no "
+ "counter-metrics hides a change that lifts one number by harming another, and a number under a "
+ "key of your own reads to a person as no evidence at all."
),
}
@@ -5967,19 +5970,26 @@ class HogFlowViewSet(
"""PostHog's own reading of the metrics a suggestion is about, taken when it is filed.
Same read as the outcome's "before" side, over the producer's window when it names a
- relative one. A read that fails leaves the suggestion filed with the producer's numbers
- only, which the page labels as unverified."""
+ relative one or an absolute interval. A read that fails leaves the suggestion filed with
+ the producer's numbers only, which the page labels as unverified."""
window = evidence.get("window") if isinstance(evidence.get("window"), str) else None
- if not window or not re.fullmatch(r"-\d+[dh]", window):
+ bounds = absolute_window(window, django_timezone.now())
+ if bounds is None and (not window or not re.fullmatch(r"-\d+[dh]", window)):
window = "-7d"
try:
- after_date, _, _ = relative_date_parse_with_delta_mapping(window, self.team.timezone_info)
+ after_date = (
+ bounds.start
+ if bounds
+ else relative_date_parse_with_delta_mapping(window or "-7d", self.team.timezone_info)[0]
+ )
+ before_date = bounds.end if bounds else None
tag_queries(product=ProductKey.WORKFLOWS, feature=Feature.QUERY)
reading = version_outcome(
team_id=self.team_id,
hog_flow_id=hog_flow.id,
versions=[base_version],
after=after_date,
+ before=before_date,
step_id=step_id,
target=target_metric_in(evidence),
)
diff --git a/products/workflows/backend/services/workflow_proposals.py b/products/workflows/backend/services/workflow_proposals.py
index 45d21de4a6e1..dd42fa56d4c4 100644
--- a/products/workflows/backend/services/workflow_proposals.py
+++ b/products/workflows/backend/services/workflow_proposals.py
@@ -2,7 +2,7 @@ import copy
import json
from collections.abc import Mapping, Sequence
from copy import deepcopy
-from datetime import datetime
+from datetime import datetime, timedelta
from typing import Any, Optional
from uuid import UUID
@@ -11,6 +11,7 @@ from django.db.models import QuerySet
from django.utils import timezone
from posthog.api.app_metrics2 import fetch_app_metric_totals
+from posthog.dataclasses import frozen
from products.workflows.backend.facade.contracts import CreatedWorkflowProposal, ProposalChanges, WorkflowProposalRecord
from products.workflows.backend.facade.enums import WorkflowProposalStatus
@@ -653,12 +654,40 @@ def versions_carrying_change(hog_flow: HogFlow, proposal: WorkflowProposal) -> t
return carrying or [applied], None
+# Longer than any window a reviewer reads as "recent", so a typo cannot ask ClickHouse for years of metrics.
+_MAX_ABSOLUTE_WINDOW = timedelta(days=90)
+
+
+@frozen
+class AbsoluteWindow:
+ start: datetime
+ end: datetime
+
+
+def absolute_window(window: object, now: datetime) -> Optional[AbsoluteWindow]:
+ """The bounds of an ISO 8601 interval such as "2026-10-07T22:00:00Z/2026-10-14T22:00:00Z", or None.
+
+ A producer that reads only closed days names them this way, so PostHog must read the same days back."""
+ if not isinstance(window, str) or window.count("/") != 1:
+ return None
+ try:
+ start, end = (datetime.fromisoformat(part) for part in window.split("/"))
+ except ValueError:
+ return None
+ if start.tzinfo is None or end.tzinfo is None:
+ return None
+ if not start < end <= now or end - start > _MAX_ABSOLUTE_WINDOW:
+ return None
+ return AbsoluteWindow(start=start, end=end)
+
+
def version_outcome(
*,
team_id: int,
hog_flow_id: UUID,
versions: Sequence[Optional[int]],
after: datetime,
+ before: Optional[datetime] = None,
step_id: Optional[str] = None,
target: str = TARGET_OPEN_METRIC,
) -> Optional[dict]:
@@ -674,6 +703,9 @@ def version_outcome(
app_source_id=f"{hog_flow_id}/{version}",
breakdown_by="name",
after=after,
+ # A merged hourly row keeps a sub-second timestamp, so only an exclusive bound keeps the whole last hour.
+ before=before,
+ before_exclusive=True,
instance_id=step_id or None,
name=[
TARGET_SEND_METRIC,
diff --git a/products/workflows/backend/tests/api/test_hog_flow.py b/products/workflows/backend/tests/api/test_hog_flow.py
index 74625ddb857a..b23d81a3ab76 100644
--- a/products/workflows/backend/tests/api/test_hog_flow.py
+++ b/products/workflows/backend/tests/api/test_hog_flow.py
@@ -6333,6 +6333,83 @@ class TestHogFlowVersionedMetrics(ClickhouseTestMixin, APIBaseTest):
}
assert {g["metric"]: g["value"] for g in measured["guardrails"]}["bounce rate"] == 0.02
+ @parameterized.expand(
+ [
+ ("closed days", "{start}/{end}", 100, "{start}/{end}"),
+ ("relative", "-7d", 157, "-7d"),
+ ("no offsets", "{start_naive}/{end_naive}", 157, "-7d"),
+ ("ends in the future", "{start}/{future}", 157, "-7d"),
+ ("reversed", "{end}/{start}", 157, "-7d"),
+ ]
+ )
+ @patch("products.workflows.backend.presentation.views.hog_flow.posthoganalytics.feature_enabled", return_value=True)
+ def test_posthog_measures_the_window_the_suggestion_names(
+ self, _name, window, expected_sends, expected_window, _mock_flag
+ ):
+ HogFlow.objects.filter(id=self.flow.id).update(
+ actions=[{"id": "email_1", "type": "function_email", "name": "Email", "config": {}}], status="active"
+ )
+ self.client.post(
+ f"/api/projects/{self.team.id}/hog_flows/{self.flow.id}/optimization", {"enabled": True}, format="json"
+ )
+ now = datetime.now(UTC).replace(minute=0, second=0, microsecond=0)
+ # The row at the interval's end belongs to the next interval.
+ for timestamp, sent in (
+ (now - timedelta(days=3), 100),
+ (now - timedelta(days=1), 7),
+ (now - timedelta(hours=1), 50),
+ ):
+ create_app_metric2(
+ team_id=self.team.id,
+ app_source="hog_flow_version",
+ app_source_id=f"{self.flow.id}/1",
+ instance_id="email_1",
+ timestamp=timestamp,
+ metric_kind="email",
+ metric_name="email_sent",
+ count=sent,
+ )
+ bounds = {
+ "start": (now - timedelta(days=5)).isoformat().replace("+00:00", "Z"),
+ "end": (now - timedelta(days=1)).isoformat().replace("+00:00", "Z"),
+ "start_naive": (now - timedelta(days=5)).replace(tzinfo=None).isoformat(),
+ "end_naive": (now - timedelta(days=1)).replace(tzinfo=None).isoformat(),
+ "future": (now + timedelta(days=1)).isoformat().replace("+00:00", "Z"),
+ }
+ producer_key = generate_random_token_personal()
+ PersonalAPIKey.objects.create(
+ label="producer",
+ user=self.user,
+ secure_value=hash_key_value(producer_key),
+ scopes=["hog_flow:read", "hog_flow_proposal:write"],
+ )
+
+ response = self.client.post(
+ f"/api/projects/{self.team.id}/hog_flows/{self.flow.id}/proposals/",
+ {
+ "title": "Shorten the subject",
+ "rationale": "Opens are low.",
+ "content": {"exit_condition": "exit_only_at_end"},
+ "step_id": "email_1",
+ "base_version": 1,
+ "evidence": {
+ "metric": "email_opened",
+ "current_value": 0.1,
+ "unit": "rate",
+ "n": 100,
+ "guardrails": [],
+ "window": window.format(**bounds),
+ },
+ },
+ format="json",
+ headers={"authorization": f"Bearer {producer_key}"},
+ )
+
+ assert response.status_code == 201, response.json()
+ measured = response.json()["evidence"]["measured"]
+ assert measured["target"]["n"] == expected_sends
+ assert measured["window"] == expected_window.format(**bounds)
+
@patch("products.workflows.backend.presentation.views.hog_flow.posthoganalytics.feature_enabled", return_value=True)
@patch(
"products.workflows.backend.services.workflow_proposals.fetch_app_metric_totals",
diff --git a/products/workflows/frontend/Workflows/suggestions/suggestionEvidence.test.ts b/products/workflows/frontend/Workflows/suggestions/suggestionEvidence.test.ts
index 51ff8ed6183f..b1d4a1ca4883 100644
--- a/products/workflows/frontend/Workflows/suggestions/suggestionEvidence.test.ts
+++ b/products/workflows/frontend/Workflows/suggestions/suggestionEvidence.test.ts
@@ -1,4 +1,4 @@
-import { evidenceDisagrees, formatValue, readUnit } from './suggestionEvidence'
+import { describeWindow, evidenceDisagrees, formatValue, readUnit } from './suggestionEvidence'
describe('suggestionEvidence', () => {
it.each([
@@ -12,6 +12,14 @@ describe('suggestionEvidence', () => {
expect(formatValue(value, unit)).toBe(expected)
})
+ it.each([
+ ['a relative window', '-7d', 'the last 7 days'],
+ ['an interval', '2026-10-07T22:00:00Z/2026-10-14T22:00:00Z', 'Oct 7, 22:00 to Oct 14, 22:00 UTC'],
+ ['an interval that is not dates', 'last/week', 'last/week'],
+ ])('describes %s', (_name, window, expected) => {
+ expect(describeWindow(window)).toBe(expected)
+ })
+
it('has nothing to show for a value that is not a number', () => {
expect(formatValue(null, 'rate')).toBeNull()
expect(formatValue('8.65%', 'rate')).toBeNull()
diff --git a/products/workflows/frontend/Workflows/suggestions/suggestionEvidence.ts b/products/workflows/frontend/Workflows/suggestions/suggestionEvidence.ts
index 1ed054d2ecb6..2b6ff54a8217 100644
--- a/products/workflows/frontend/Workflows/suggestions/suggestionEvidence.ts
+++ b/products/workflows/frontend/Workflows/suggestions/suggestionEvidence.ts
@@ -1,3 +1,5 @@
+import { dayjs } from 'lib/dayjs'
+
// A rate under this many observations is noise. Mirrors MIN_EVIDENCE_SAMPLE in
// products/workflows/backend/metrics.py.
export const MIN_EVIDENCE_SAMPLE = 20
@@ -55,6 +57,14 @@ export function readMeasured(evidence: Record<string, unknown>): MeasuredEvidenc
}
export function describeWindow(window: string): string {
+ const interval = window.split('/')
+ if (interval.length === 2) {
+ const [start, end] = interval.map((part) => dayjs(part))
+ if (start.isValid() && end.isValid()) {
+ const viewerZone = dayjs.tz.guess()
+ return `${start.tz(viewerZone).format('MMM D, HH:mm')} to ${end.tz(viewerZone).format('MMM D, HH:mm z')}`
+ }
+ }
const match = /^-(\d+)([dh])$/.exec(window)
if (!match) {
return window
diff --git a/products/workflows/frontend/generated/api.schemas.ts b/products/workflows/frontend/generated/api.schemas.ts
index 9cce32d16882..f979f7868a12 100644
--- a/products/workflows/frontend/generated/api.schemas.ts
+++ b/products/workflows/frontend/generated/api.schemas.ts
@@ -1351,7 +1351,7 @@ export const WorkflowProposalStatusEnumApi = {
export type WorkflowProposalApiContent = { [key: string]: unknown }
/**
- * The numbers behind the proposal, read back by name. Five keys are required: `metric`, the metric name; `current_value`, its value as a number (a rate as a fraction, 0.0865, never a string); `unit`, either `rate` or `count`, since 1.0 is either every message or one of them; `n`, the denominator that value was computed over; and `guardrails`, a list of {metric, value, n, unit} counter-metrics read over the same window, empty only if none apply. PostHog then reads the step's own metrics at `base_version` when the suggestion is filed and stores them under `measured`; the page shows that reading and flags a disagreement with yours. Also conventional: target_value, window, query, app_source_id. A rate with no denominator lets a reviewer mistake noise for a result, a target with no counter-metrics hides a change that lifts one number by harming another, and a number under a key of your own reads to a person as no evidence at all.
+ * The numbers behind the proposal, read back by name. Five keys are required: `metric`, the metric name; `current_value`, its value as a number (a rate as a fraction, 0.0865, never a string); `unit`, either `rate` or `count`, since 1.0 is either every message or one of them; `n`, the denominator that value was computed over; and `guardrails`, a list of {metric, value, n, unit} counter-metrics read over the same window, empty only if none apply. PostHog then reads the step's own metrics at `base_version` when the suggestion is filed and stores them under `measured`; the page shows that reading and flags a disagreement with yours. Also conventional: target_value, window (a relative one such as `-7d`, or an ISO 8601 interval `start/end` with offsets, which PostHog reads back exactly), query, app_source_id. A rate with no denominator lets a reviewer mistake noise for a result, a target with no counter-metrics hides a change that lifts one number by harming another, and a number under a key of your own reads to a person as no evidence at all.
*/
export type WorkflowProposalApiEvidence = { [key: string]: unknown }
@@ -1363,7 +1363,7 @@ export interface WorkflowProposalApi {
readonly rationale: string
/** Only the content fields the proposal changes. Valid keys: actions, edges, trigger_masking, conversion, exit_condition, email_sending_rate_limit, variables. Each value has the same shape as on the workflow itself. */
readonly content: WorkflowProposalApiContent
- /** The numbers behind the proposal, read back by name. Five keys are required: `metric`, the metric name; `current_value`, its value as a number (a rate as a fraction, 0.0865, never a string); `unit`, either `rate` or `count`, since 1.0 is either every message or one of them; `n`, the denominator that value was computed over; and `guardrails`, a list of {metric, value, n, unit} counter-metrics read over the same window, empty only if none apply. PostHog then reads the step's own metrics at `base_version` when the suggestion is filed and stores them under `measured`; the page shows that reading and flags a disagreement with yours. Also conventional: target_value, window, query, app_source_id. A rate with no denominator lets a reviewer mistake noise for a result, a target with no counter-metrics hides a change that lifts one number by harming another, and a number under a key of your own reads to a person as no evidence at all. */
+ /** The numbers behind the proposal, read back by name. Five keys are required: `metric`, the metric name; `current_value`, its value as a number (a rate as a fraction, 0.0865, never a string); `unit`, either `rate` or `count`, since 1.0 is either every message or one of them; `n`, the denominator that value was computed over; and `guardrails`, a list of {metric, value, n, unit} counter-metrics read over the same window, empty only if none apply. PostHog then reads the step's own metrics at `base_version` when the suggestion is filed and stores them under `measured`; the page shows that reading and flags a disagreement with yours. Also conventional: target_value, window (a relative one such as `-7d`, or an ISO 8601 interval `start/end` with offsets, which PostHog reads back exactly), query, app_source_id. A rate with no denominator lets a reviewer mistake noise for a result, a target with no counter-metrics hides a change that lifts one number by harming another, and a number under a key of your own reads to a person as no evidence at all. */
readonly evidence: WorkflowProposalApiEvidence
/**
* The workflow step this is about. Set for a change to one step: the evidence and the outcome then read that step's metrics, so a change to one email in a sequence is not measured against the rest. Null only for a change that spans the workflow, such as its exit condition or a step being taken out, which is measured on the workflow's own numbers.
diff --git a/services/mcp/src/api/generated.ts b/services/mcp/src/api/generated.ts
index 35f68d3b666e..41fd3dbdabaa 100644
--- a/services/mcp/src/api/generated.ts
+++ b/services/mcp/src/api/generated.ts
@@ -79428,7 +79428,7 @@ export namespace Schemas {
export type WorkflowProposalContent = { [key: string]: unknown };
/**
- * The numbers behind the proposal, read back by name. Five keys are required: `metric`, the metric name; `current_value`, its value as a number (a rate as a fraction, 0.0865, never a string); `unit`, either `rate` or `count`, since 1.0 is either every message or one of them; `n`, the denominator that value was computed over; and `guardrails`, a list of {metric, value, n, unit} counter-metrics read over the same window, empty only if none apply. PostHog then reads the step's own metrics at `base_version` when the suggestion is filed and stores them under `measured`; the page shows that reading and flags a disagreement with yours. Also conventional: target_value, window, query, app_source_id. A rate with no denominator lets a reviewer mistake noise for a result, a target with no counter-metrics hides a change that lifts one number by harming another, and a number under a key of your own reads to a person as no evidence at all.
+ * The numbers behind the proposal, read back by name. Five keys are required: `metric`, the metric name; `current_value`, its value as a number (a rate as a fraction, 0.0865, never a string); `unit`, either `rate` or `count`, since 1.0 is either every message or one of them; `n`, the denominator that value was computed over; and `guardrails`, a list of {metric, value, n, unit} counter-metrics read over the same window, empty only if none apply. PostHog then reads the step's own metrics at `base_version` when the suggestion is filed and stores them under `measured`; the page shows that reading and flags a disagreement with yours. Also conventional: target_value, window (a relative one such as `-7d`, or an ISO 8601 interval `start/end` with offsets, which PostHog reads back exactly), query, app_source_id. A rate with no denominator lets a reviewer mistake noise for a result, a target with no counter-metrics hides a change that lifts one number by harming another, and a number under a key of your own reads to a person as no evidence at all.
*/
export type WorkflowProposalEvidence = { [key: string]: unknown };
@@ -79440,7 +79440,7 @@ export namespace Schemas {
readonly rationale: string;
/** Only the content fields the proposal changes. Valid keys: actions, edges, trigger_masking, conversion, exit_condition, email_sending_rate_limit, variables. Each value has the same shape as on the workflow itself. */
readonly content: WorkflowProposalContent;
- /** The numbers behind the proposal, read back by name. Five keys are required: `metric`, the metric name; `current_value`, its value as a number (a rate as a fraction, 0.0865, never a string); `unit`, either `rate` or `count`, since 1.0 is either every message or one of them; `n`, the denominator that value was computed over; and `guardrails`, a list of {metric, value, n, unit} counter-metrics read over the same window, empty only if none apply. PostHog then reads the step's own metrics at `base_version` when the suggestion is filed and stores them under `measured`; the page shows that reading and flags a disagreement with yours. Also conventional: target_value, window, query, app_source_id. A rate with no denominator lets a reviewer mistake noise for a result, a target with no counter-metrics hides a change that lifts one number by harming another, and a number under a key of your own reads to a person as no evidence at all. */
+ /** The numbers behind the proposal, read back by name. Five keys are required: `metric`, the metric name; `current_value`, its value as a number (a rate as a fraction, 0.0865, never a string); `unit`, either `rate` or `count`, since 1.0 is either every message or one of them; `n`, the denominator that value was computed over; and `guardrails`, a list of {metric, value, n, unit} counter-metrics read over the same window, empty only if none apply. PostHog then reads the step's own metrics at `base_version` when the suggestion is filed and stores them under `measured`; the page shows that reading and flags a disagreement with yours. Also conventional: target_value, window (a relative one such as `-7d`, or an ISO 8601 interval `start/end` with offsets, which PostHog reads back exactly), query, app_source_id. A rate with no denominator lets a reviewer mistake noise for a result, a target with no counter-metrics hides a change that lifts one number by harming another, and a number under a key of your own reads to a person as no evidence at all. */
readonly evidence: WorkflowProposalEvidence;
/**
* The workflow step this is about. Set for a change to one step: the evidence and the outcome then read that step's metrics, so a change to one email in a sequence is not measured against the rest. Null only for a change that spans the workflow, such as its exit condition or a step being taken out, which is measured on the workflow's own numbers.