Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(subscriptions): skip ai report synthesis when no query returned rows (#121870)

Vasco de Krijger1afa9357c4c9
diff --git a/products/subscriptions/backend/temporal/ai_subscription/report_pipeline.py b/products/subscriptions/backend/temporal/ai_subscription/report_pipeline.py
index aa4bf5ee308d..6885b23ab519 100644
--- a/products/subscriptions/backend/temporal/ai_subscription/report_pipeline.py
+++ b/products/subscriptions/backend/temporal/ai_subscription/report_pipeline.py
@@ -70,6 +70,7 @@ from products.subscriptions.backend.temporal.ai_subscription.spec_generator impo
)
from products.subscriptions.backend.temporal.types import safe_query_error_details
+from ee.hogai.context.insight.format import format_access_control_warnings, format_warehouse_sync_warnings
from ee.hogai.context.insight.query_executor import AssistantQueryExecutor
from ee.hogai.llm import MaxChatOpenAI
from ee.hogai.tool_errors import MaxToolRetryableError
@@ -97,6 +98,7 @@ _MIN_STEP_RESULT_CHARS = _SYNTHESIS_RESULTS_CHAR_BUDGET // MAX_QUERY_PLAN_STEPS
# instead of "no data"; it's injected into that prompt (the {{{failure_marker}}} placeholder) from this
# same constant in `_synthesize`, so the rendered marker and the prompt instruction can't drift apart.
QUERY_FAILED_PREFIX = "Query failed to run"
+QUERY_RETURNED_NO_ROWS = "The query ran but found no data for this period"
# Per-step query-fix budget: the planner occasionally emits HogQL that fails to parse, so we feed the
# error back and ask for a rewrite rather than dropping the step. Worst case per step is one original
@@ -130,6 +132,17 @@ def _all_queries_failed_notice(total_steps: int, *, include_manage_link: bool =
return notice + "\n\n"
+def _no_rows_notice(*, include_manage_link: bool = True) -> str:
+ notice = "> ⚠️ None of the queries in this report found any data for this period, so there is nothing to summarize."
+ if include_manage_link:
+ notice += " Use the Manage subscription link to review the generated queries."
+ return notice + "\n\n"
+
+
+async def _no_rows_report(rendered: Sequence[str], *, include_manage_link: bool) -> str:
+ return _no_rows_notice(include_manage_link=include_manage_link) + "\n\n".join(rendered)
+
+
def _validate_step_chart(
spec: Optional[StepChart],
response: Any,
@@ -183,6 +196,7 @@ class QueryStepDiagnostic:
class StepOutcome:
rendered: str
diagnostic: QueryStepDiagnostic
+ has_rows: bool
chart: Optional[ValidatedChart] = None
source_urls: tuple[str, ...] = ()
@@ -194,6 +208,7 @@ class PlanExecution:
diagnostics: list[QueryStepDiagnostic]
charts: list[ValidatedChart]
source_urls: tuple[str, ...] = ()
+ no_rows_count: int = 0
@frozen
@@ -277,8 +292,13 @@ async def generate_ai_report(
requested=len(charts),
selected=len(selected),
)
+ no_step_has_rows = execution.no_rows_count > 0 and (
+ execution.no_rows_count + failed_count == len(spec.plan.steps)
+ )
synthesis_task = asyncio.ensure_future(
- _synthesize(spec, execution.rendered, team, user, trace_correlation_id, execution.source_urls)
+ _no_rows_report(execution.rendered, include_manage_link=include_manage_link)
+ if no_step_has_rows
+ else _synthesize(spec, execution.rendered, team, user, trace_correlation_id, execution.source_urls)
)
render_task = asyncio.ensure_future(render_charts(selected, team=team, user=user))
try:
@@ -308,6 +328,7 @@ async def generate_ai_report(
failed_steps=failed_count,
query_coverage=(total_steps - failed_count) / total_steps if total_steps else 0.0,
degraded=bool(failed_count),
+ no_rows_steps=execution.no_rows_count,
charts_requested=len(charts),
charts_rendered=len(rendered_charts),
chart_failures=len(chart_failures),
@@ -336,6 +357,7 @@ async def generate_ai_report(
relevant_events=spec.relevant_events,
trace_correlation_id=trace_correlation_id,
chart_failure_count=chart_spec_failures,
+ no_rows_count=execution.no_rows_count,
)
query_plan_status = resolve_ai_query_plan_status(
initial_status=initial_query_plan_status,
@@ -378,6 +400,7 @@ def _plan_to_freeze(
relevant_events: Sequence[str],
trace_correlation_id: Optional[Union[int, str]],
chart_failure_count: int = 0,
+ no_rows_count: int = 0,
) -> Optional[dict]:
# Steps already carry their final HogQL by this point — see the write-back in `run_step`.
# Never freeze a plan the next delivery is better off re-planning: a plan with any failed step would
@@ -397,6 +420,15 @@ def _plan_to_freeze(
total_steps=total_steps,
)
return None
+ # When no step finds data, the plan can filter on an event or value the project never sends. A frozen
+ # copy would then send the no-data notice on every delivery until the prompt changes.
+ if total_steps and no_rows_count == total_steps:
+ logger.warning(
+ "ai_report.plan_found_no_data_not_frozen",
+ trace_correlation_id=trace_correlation_id,
+ total_steps=total_steps,
+ )
+ return None
if chart_failure_count:
logger.warning(
"ai_report.plan_had_chart_failures_not_frozen",
@@ -589,8 +621,6 @@ async def _run_steps(
executor.arun_format_and_capture(query),
timeout=_HOGQL_STEP_TIMEOUT_SECONDS,
)
- # result values are attacker-influenceable (public project tokens) — strip framing markers
- safe_formatted = strip_llm_framing_markers(query_result.formatted, per_step_cap)
# Write the succeeding query (post any fix-LLM rewrite, still window-agnostic) back onto
# the step so `_plan_to_freeze` freezes what actually ran — a no-op unless the fix LLM
# rewrote it. A failed step keeps the planner's original, never a broken rewrite.
@@ -606,8 +636,25 @@ async def _run_steps(
except Exception:
logger.warning("ai_report.chart_validation_error", step_index=step_index, exc_info=True)
chart, chart_dropped_reason = None, ChartFailureReason.VALIDATION_ERROR
+ has_rows = bool(query_result.response.get("results"))
+ if has_rows:
+ # result values are attacker-influenceable (public project tokens) — strip framing markers
+ safe_formatted = strip_llm_framing_markers(query_result.formatted, per_step_cap)
+ rendered = f"### {safe_description}\n\n{safe_formatted}"
+ else:
+ # Access-control filtering or a stale warehouse sync can explain an empty result.
+ query_warnings = strip_llm_framing_markers(
+ format_warehouse_sync_warnings(query_result.response)
+ + format_access_control_warnings(query_result.response),
+ per_step_cap,
+ )
+ rendered = "\n\n".join(
+ part
+ for part in (f"### {safe_description}", query_warnings, f"_{QUERY_RETURNED_NO_ROWS}._")
+ if part
+ )
return StepOutcome(
- rendered=f"### {safe_description}\n\n{safe_formatted}",
+ rendered=rendered,
diagnostic=QueryStepDiagnostic(
description=safe_description,
hogql=executable_hogql,
@@ -616,6 +663,7 @@ async def _run_steps(
chart_dropped_reason=chart_dropped_reason,
),
chart=chart,
+ has_rows=has_rows,
source_urls=tuple(
dict.fromkeys(
value
@@ -679,6 +727,7 @@ async def _run_steps(
error_code=error_details["code"] if error_details else None,
human_readable_error=error_details["message"] if error_details else None,
),
+ has_rows=False,
)
async def run_step_bounded(step: QueryPlanStep, step_index: int) -> StepOutcome:
@@ -694,6 +743,7 @@ async def _run_steps(
diagnostics=diagnostics,
charts=[outcome.chart for outcome in step_results if outcome.chart is not None],
source_urls=tuple(dict.fromkeys(url for outcome in step_results for url in outcome.source_urls))[:100],
+ no_rows_count=sum(1 for outcome in step_results if outcome.diagnostic.ok and not outcome.has_rows),
)
diff --git a/products/subscriptions/backend/temporal/ai_subscription/test/test_report_pipeline.py b/products/subscriptions/backend/temporal/ai_subscription/test/test_report_pipeline.py
index 2f5529ef79a0..5e3a77a880a9 100644
--- a/products/subscriptions/backend/temporal/ai_subscription/test/test_report_pipeline.py
+++ b/products/subscriptions/backend/temporal/ai_subscription/test/test_report_pipeline.py
@@ -25,6 +25,7 @@ from products.subscriptions.backend.temporal.ai_subscription.charts import (
from products.subscriptions.backend.temporal.ai_subscription.report_pipeline import (
_MAX_CONCURRENT_STEPS,
QUERY_FAILED_PREFIX,
+ QUERY_RETURNED_NO_ROWS,
AiReportStageError,
PlanExecution,
QueryStepDiagnostic,
@@ -58,7 +59,8 @@ _SG = "products.subscriptions.backend.temporal.ai_subscription.spec_generator"
_SLO_CAPTURE = "posthog.slo.events.posthoganalytics.capture"
_WINDOW_END = datetime(2026, 6, 29, 16, 0, tzinfo=UTC)
-_RESPONSE: dict = {"results": [], "columns": []}
+_RESPONSE: dict = {"results": [[1]], "columns": ["count()"]}
+_EMPTY_RESPONSE: dict = {"results": [], "columns": ["count()"]}
@pytest.fixture(autouse=True)
@@ -83,6 +85,15 @@ _OK_RUN = PlanExecution(
diagnostics=[QueryStepDiagnostic("s", "SELECT count() FROM events", True, None)],
charts=[],
)
+_NO_ROWS_STEP = f"### s0\n\n_{QUERY_RETURNED_NO_ROWS}._"
+_FAILED_STEP = f"### s1\n\n_{QUERY_FAILED_PREFIX} — metric not computed, not empty data._"
+_NO_ROWS_RUN = PlanExecution(
+ rendered=[_NO_ROWS_STEP],
+ failed_count=0,
+ diagnostics=[QueryStepDiagnostic("s0", "SELECT 1", True, None)],
+ charts=[],
+ no_rows_count=1,
+)
def _spec(steps: int = 1) -> EnrichedPromptSpec:
@@ -409,6 +420,7 @@ async def test_run_steps_non_retryable_error_degrades_to_placeholder(mock_execut
_spec(steps=1), MagicMock(), MagicMock(), _test_window(), None, charts_enabled_for_team=True
)
assert execution.failed_count == 1
+ assert execution.no_rows_count == 0
assert "Query failed to run" in execution.rendered[0]
assert execution.diagnostics[0].ok is False
assert execution.diagnostics[0].error_type == "RuntimeError"
@@ -500,6 +512,118 @@ async def test_synthesis_prompt_carries_the_failure_marker(
assert "{{{" not in system_message # no placeholder left unrendered
+ "response,expected_rendered,expected_no_rows",
+ [
+ pytest.param(_RESPONSE, "### s0\n\nformatted table", 0, id="rows"),
+ pytest.param(_EMPTY_RESPONSE, f"### s0\n\n_{QUERY_RETURNED_NO_ROWS}._", 1, id="no_rows"),
+ pytest.param(
+ {
+ **_EMPTY_RESPONSE,
+ "warnings": [{"type": "access_control", "message": "Results may exclude restricted events."}],
+ },
+ f"### s0\n\n[Access control]\n- Results may exclude restricted events.\n\n_{QUERY_RETURNED_NO_ROWS}._",
+ 1,
+ id="no_rows_keeps_query_warnings",
+ ),
+ ],
+)
+@patch(f"{_RP}.AssistantQueryExecutor")
+async def test_run_steps_marks_a_step_that_returned_no_rows(
+ mock_executor_cls: MagicMock, response: dict, expected_rendered: str, expected_no_rows: int
+) -> None:
+ mock_executor_cls.return_value.arun_format_and_capture = AsyncMock(
+ return_value=FormattedQueryResult(formatted="formatted table", fallback_used=False, response=response)
+ )
+ execution = await _run_steps(
+ _spec(steps=1), MagicMock(), MagicMock(), _test_window(), None, charts_enabled_for_team=True
+ )
+ assert execution.rendered == [expected_rendered]
+ assert execution.no_rows_count == expected_no_rows
+ assert execution.failed_count == 0
+
+
+ "run_result,include_manage_link,expected_report,synthesizes",
+ [
+ pytest.param(
+ _NO_ROWS_RUN,
+ True,
+ "> ⚠️ None of the queries in this report found any data for this period, so there is nothing to "
+ "summarize. Use the Manage subscription link to review the generated queries.\n\n" + _NO_ROWS_STEP,
+ False,
+ id="every_step_empty",
+ ),
+ pytest.param(
+ PlanExecution(
+ rendered=[_NO_ROWS_STEP, _FAILED_STEP],
+ failed_count=1,
+ diagnostics=[
+ QueryStepDiagnostic("s0", "SELECT 1", True, None),
+ QueryStepDiagnostic("s1", "SELECT bad", False, "ExposedHogQLError"),
+ ],
+ charts=[],
+ no_rows_count=1,
+ ),
+ False,
+ "> ⚠️ None of the queries in this report found any data for this period, so there is nothing to "
+ "summarize.\n\n" + _NO_ROWS_STEP + "\n\n" + _FAILED_STEP,
+ False,
+ id="empty_and_failed_without_manage_link",
+ ),
+ pytest.param(
+ PlanExecution(
+ rendered=["### s0\n\nok", _NO_ROWS_STEP],
+ failed_count=0,
+ diagnostics=[
+ QueryStepDiagnostic("s0", "SELECT 1", True, None),
+ QueryStepDiagnostic("s1", "SELECT 2", True, None),
+ ],
+ charts=[],
+ no_rows_count=1,
+ ),
+ True,
+ "# Report",
+ True,
+ id="one_step_with_rows_still_synthesizes",
+ ),
+ ],
+)
+@patch(_SLO_CAPTURE)
+@patch(f"{_RP}.MaxChatOpenAI")
+@patch(f"{_RP}._run_steps", new_callable=AsyncMock)
+@patch(f"{_RP}.build_enriched_prompt")
+async def test_synthesis_runs_only_when_a_step_returned_rows(
+ mock_bep: MagicMock,
+ mock_run: AsyncMock,
+ mock_chat: MagicMock,
+ mock_capture: MagicMock,
+ run_result: PlanExecution,
+ include_manage_link: bool,
+ expected_report: str,
+ synthesizes: bool,
+) -> None:
+ mock_bep.return_value = _spec(steps=len(run_result.rendered))
+ mock_run.return_value = run_result
+ mock_chat.return_value.with_structured_output.return_value.invoke.return_value = ReportDocument(
+ topic=None,
+ sections=[],
+ custom_layout=[ReportText(items=[ReportTextItem(kind="text", text="# Report")])],
+ )
+
+ result = await generate_ai_report(
+ team=MagicMock(),
+ user=MagicMock(),
+ prompt="x",
+ window=_test_window(),
+ include_manage_link=include_manage_link,
+ )
+
+ assert result.markdown == expected_report
+ assert mock_chat.return_value.with_structured_output.return_value.invoke.called is synthesizes
+ assert _slo_completed(mock_capture)["no_rows_steps"] == run_result.no_rows_count
+
+
@patch(f"{_RP}._arequest_hogql_fix", new_callable=AsyncMock)
@patch(f"{_RP}.AssistantQueryExecutor")
async def test_run_steps_retries_then_succeeds(mock_executor_cls: MagicMock, mock_fix: AsyncMock) -> None:
@@ -860,6 +984,7 @@ async def test_run_steps_substitutes_fresh_window_into_placeholder_sql(mock_exec
pytest.param(_spec_with_window_placeholder(), _ALL_FAILED_RUN, id="all_failed"),
# No window placeholder: freezing would cement an unbounded, window-less scan forever.
pytest.param(_spec(steps=1), _OK_RUN, id="missing_window_placeholder"),
+ pytest.param(_spec_with_window_placeholder(), _NO_ROWS_RUN, id="no_step_found_data"),
],
)
@patch(_SLO_CAPTURE)