:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(subscriptions): skip ai report synthesis when no query returned rows (#121870)
Vasco de Krijger1afa9357c4c9
diff --git a/products/subscriptions/backend/temporal/ai_subscription/report_pipeline.py b/products/subscriptions/backend/temporal/ai_subscription/report_pipeline.pyindex aa4bf5ee308d..6885b23ab519 100644--- a/products/subscriptions/backend/temporal/ai_subscription/report_pipeline.py+++ b/products/subscriptions/backend/temporal/ai_subscription/report_pipeline.py@@ -70,6 +70,7 @@ from products.subscriptions.backend.temporal.ai_subscription.spec_generator impo)from products.subscriptions.backend.temporal.types import safe_query_error_details+from ee.hogai.context.insight.format import format_access_control_warnings, format_warehouse_sync_warningsfrom ee.hogai.context.insight.query_executor import AssistantQueryExecutorfrom ee.hogai.llm import MaxChatOpenAIfrom ee.hogai.tool_errors import MaxToolRetryableError@@ -97,6 +98,7 @@ _MIN_STEP_RESULT_CHARS = _SYNTHESIS_RESULTS_CHAR_BUDGET // MAX_QUERY_PLAN_STEPS# instead of "no data"; it's injected into that prompt (the {{{failure_marker}}} placeholder) from this# same constant in `_synthesize`, so the rendered marker and the prompt instruction can't drift apart.QUERY_FAILED_PREFIX = "Query failed to run"+QUERY_RETURNED_NO_ROWS = "The query ran but found no data for this period"# Per-step query-fix budget: the planner occasionally emits HogQL that fails to parse, so we feed the# error back and ask for a rewrite rather than dropping the step. Worst case per step is one original@@ -130,6 +132,17 @@ def _all_queries_failed_notice(total_steps: int, *, include_manage_link: bool =return notice + "\n\n"+def _no_rows_notice(*, include_manage_link: bool = True) -> str:+ notice = "> ⚠️ None of the queries in this report found any data for this period, so there is nothing to summarize."+ if include_manage_link:+ notice += " Use the Manage subscription link to review the generated queries."+ return notice + "\n\n"+++async def _no_rows_report(rendered: Sequence[str], *, include_manage_link: bool) -> str:+ return _no_rows_notice(include_manage_link=include_manage_link) + "\n\n".join(rendered)++def _validate_step_chart(spec: Optional[StepChart],response: Any,@@ -183,6 +196,7 @@ class QueryStepDiagnostic:class StepOutcome:rendered: strdiagnostic: QueryStepDiagnostic+ has_rows: boolchart: Optional[ValidatedChart] = Nonesource_urls: tuple[str, ...] = ()@@ -194,6 +208,7 @@ class PlanExecution:diagnostics: list[QueryStepDiagnostic]charts: list[ValidatedChart]source_urls: tuple[str, ...] = ()+ no_rows_count: int = 0@frozen@@ -277,8 +292,13 @@ async def generate_ai_report(requested=len(charts),selected=len(selected),)+ no_step_has_rows = execution.no_rows_count > 0 and (+ execution.no_rows_count + failed_count == len(spec.plan.steps)+ )synthesis_task = asyncio.ensure_future(- _synthesize(spec, execution.rendered, team, user, trace_correlation_id, execution.source_urls)+ _no_rows_report(execution.rendered, include_manage_link=include_manage_link)+ if no_step_has_rows+ else _synthesize(spec, execution.rendered, team, user, trace_correlation_id, execution.source_urls))render_task = asyncio.ensure_future(render_charts(selected, team=team, user=user))try:@@ -308,6 +328,7 @@ async def generate_ai_report(failed_steps=failed_count,query_coverage=(total_steps - failed_count) / total_steps if total_steps else 0.0,degraded=bool(failed_count),+ no_rows_steps=execution.no_rows_count,charts_requested=len(charts),charts_rendered=len(rendered_charts),chart_failures=len(chart_failures),@@ -336,6 +357,7 @@ async def generate_ai_report(relevant_events=spec.relevant_events,trace_correlation_id=trace_correlation_id,chart_failure_count=chart_spec_failures,+ no_rows_count=execution.no_rows_count,)query_plan_status = resolve_ai_query_plan_status(initial_status=initial_query_plan_status,@@ -378,6 +400,7 @@ def _plan_to_freeze(relevant_events: Sequence[str],trace_correlation_id: Optional[Union[int, str]],chart_failure_count: int = 0,+ no_rows_count: int = 0,) -> Optional[dict]:# Steps already carry their final HogQL by this point — see the write-back in `run_step`.# Never freeze a plan the next delivery is better off re-planning: a plan with any failed step would@@ -397,6 +420,15 @@ def _plan_to_freeze(total_steps=total_steps,)return None+ # When no step finds data, the plan can filter on an event or value the project never sends. A frozen+ # copy would then send the no-data notice on every delivery until the prompt changes.+ if total_steps and no_rows_count == total_steps:+ logger.warning(+ "ai_report.plan_found_no_data_not_frozen",+ trace_correlation_id=trace_correlation_id,+ total_steps=total_steps,+ )+ return Noneif chart_failure_count:logger.warning("ai_report.plan_had_chart_failures_not_frozen",@@ -589,8 +621,6 @@ async def _run_steps(executor.arun_format_and_capture(query),timeout=_HOGQL_STEP_TIMEOUT_SECONDS,)- # result values are attacker-influenceable (public project tokens) — strip framing markers- safe_formatted = strip_llm_framing_markers(query_result.formatted, per_step_cap)# Write the succeeding query (post any fix-LLM rewrite, still window-agnostic) back onto# the step so `_plan_to_freeze` freezes what actually ran — a no-op unless the fix LLM# rewrote it. A failed step keeps the planner's original, never a broken rewrite.@@ -606,8 +636,25 @@ async def _run_steps(except Exception:logger.warning("ai_report.chart_validation_error", step_index=step_index, exc_info=True)chart, chart_dropped_reason = None, ChartFailureReason.VALIDATION_ERROR+ has_rows = bool(query_result.response.get("results"))+ if has_rows:+ # result values are attacker-influenceable (public project tokens) — strip framing markers+ safe_formatted = strip_llm_framing_markers(query_result.formatted, per_step_cap)+ rendered = f"### {safe_description}\n\n{safe_formatted}"+ else:+ # Access-control filtering or a stale warehouse sync can explain an empty result.+ query_warnings = strip_llm_framing_markers(+ format_warehouse_sync_warnings(query_result.response)+ + format_access_control_warnings(query_result.response),+ per_step_cap,+ )+ rendered = "\n\n".join(+ part+ for part in (f"### {safe_description}", query_warnings, f"_{QUERY_RETURNED_NO_ROWS}._")+ if part+ )return StepOutcome(- rendered=f"### {safe_description}\n\n{safe_formatted}",+ rendered=rendered,diagnostic=QueryStepDiagnostic(description=safe_description,hogql=executable_hogql,@@ -616,6 +663,7 @@ async def _run_steps(chart_dropped_reason=chart_dropped_reason,),chart=chart,+ has_rows=has_rows,source_urls=tuple(dict.fromkeys(value@@ -679,6 +727,7 @@ async def _run_steps(error_code=error_details["code"] if error_details else None,human_readable_error=error_details["message"] if error_details else None,),+ has_rows=False,)async def run_step_bounded(step: QueryPlanStep, step_index: int) -> StepOutcome:@@ -694,6 +743,7 @@ async def _run_steps(diagnostics=diagnostics,charts=[outcome.chart for outcome in step_results if outcome.chart is not None],source_urls=tuple(dict.fromkeys(url for outcome in step_results for url in outcome.source_urls))[:100],+ no_rows_count=sum(1 for outcome in step_results if outcome.diagnostic.ok and not outcome.has_rows),)diff --git a/products/subscriptions/backend/temporal/ai_subscription/test/test_report_pipeline.py b/products/subscriptions/backend/temporal/ai_subscription/test/test_report_pipeline.pyindex 2f5529ef79a0..5e3a77a880a9 100644--- a/products/subscriptions/backend/temporal/ai_subscription/test/test_report_pipeline.py+++ b/products/subscriptions/backend/temporal/ai_subscription/test/test_report_pipeline.py@@ -25,6 +25,7 @@ from products.subscriptions.backend.temporal.ai_subscription.charts import (from products.subscriptions.backend.temporal.ai_subscription.report_pipeline import (_MAX_CONCURRENT_STEPS,QUERY_FAILED_PREFIX,+ QUERY_RETURNED_NO_ROWS,AiReportStageError,PlanExecution,QueryStepDiagnostic,@@ -58,7 +59,8 @@ _SG = "products.subscriptions.backend.temporal.ai_subscription.spec_generator"_SLO_CAPTURE = "posthog.slo.events.posthoganalytics.capture"_WINDOW_END = datetime(2026, 6, 29, 16, 0, tzinfo=UTC)-_RESPONSE: dict = {"results": [], "columns": []}+_RESPONSE: dict = {"results": [[1]], "columns": ["count()"]}+_EMPTY_RESPONSE: dict = {"results": [], "columns": ["count()"]}@pytest.fixture(autouse=True)@@ -83,6 +85,15 @@ _OK_RUN = PlanExecution(diagnostics=[QueryStepDiagnostic("s", "SELECT count() FROM events", True, None)],charts=[],)+_NO_ROWS_STEP = f"### s0\n\n_{QUERY_RETURNED_NO_ROWS}._"+_FAILED_STEP = f"### s1\n\n_{QUERY_FAILED_PREFIX} — metric not computed, not empty data._"+_NO_ROWS_RUN = PlanExecution(+ rendered=[_NO_ROWS_STEP],+ failed_count=0,+ diagnostics=[QueryStepDiagnostic("s0", "SELECT 1", True, None)],+ charts=[],+ no_rows_count=1,+)def _spec(steps: int = 1) -> EnrichedPromptSpec:@@ -409,6 +420,7 @@ async def test_run_steps_non_retryable_error_degrades_to_placeholder(mock_execut_spec(steps=1), MagicMock(), MagicMock(), _test_window(), None, charts_enabled_for_team=True)assert execution.failed_count == 1+ assert execution.no_rows_count == 0assert "Query failed to run" in execution.rendered[0]assert execution.diagnostics[0].ok is Falseassert execution.diagnostics[0].error_type == "RuntimeError"@@ -500,6 +512,118 @@ async def test_synthesis_prompt_carries_the_failure_marker(assert "{{{" not in system_message # no placeholder left unrendered+ "response,expected_rendered,expected_no_rows",+ [+ pytest.param(_RESPONSE, "### s0\n\nformatted table", 0, id="rows"),+ pytest.param(_EMPTY_RESPONSE, f"### s0\n\n_{QUERY_RETURNED_NO_ROWS}._", 1, id="no_rows"),+ pytest.param(+ {+ **_EMPTY_RESPONSE,+ "warnings": [{"type": "access_control", "message": "Results may exclude restricted events."}],+ },+ f"### s0\n\n[Access control]\n- Results may exclude restricted events.\n\n_{QUERY_RETURNED_NO_ROWS}._",+ 1,+ id="no_rows_keeps_query_warnings",+ ),+ ],+)+@patch(f"{_RP}.AssistantQueryExecutor")+async def test_run_steps_marks_a_step_that_returned_no_rows(+ mock_executor_cls: MagicMock, response: dict, expected_rendered: str, expected_no_rows: int+) -> None:+ mock_executor_cls.return_value.arun_format_and_capture = AsyncMock(+ return_value=FormattedQueryResult(formatted="formatted table", fallback_used=False, response=response)+ )+ execution = await _run_steps(+ _spec(steps=1), MagicMock(), MagicMock(), _test_window(), None, charts_enabled_for_team=True+ )+ assert execution.rendered == [expected_rendered]+ assert execution.no_rows_count == expected_no_rows+ assert execution.failed_count == 0+++ "run_result,include_manage_link,expected_report,synthesizes",+ [+ pytest.param(+ _NO_ROWS_RUN,+ True,+ "> ⚠️ None of the queries in this report found any data for this period, so there is nothing to "+ "summarize. Use the Manage subscription link to review the generated queries.\n\n" + _NO_ROWS_STEP,+ False,+ id="every_step_empty",+ ),+ pytest.param(+ PlanExecution(+ rendered=[_NO_ROWS_STEP, _FAILED_STEP],+ failed_count=1,+ diagnostics=[+ QueryStepDiagnostic("s0", "SELECT 1", True, None),+ QueryStepDiagnostic("s1", "SELECT bad", False, "ExposedHogQLError"),+ ],+ charts=[],+ no_rows_count=1,+ ),+ False,+ "> ⚠️ None of the queries in this report found any data for this period, so there is nothing to "+ "summarize.\n\n" + _NO_ROWS_STEP + "\n\n" + _FAILED_STEP,+ False,+ id="empty_and_failed_without_manage_link",+ ),+ pytest.param(+ PlanExecution(+ rendered=["### s0\n\nok", _NO_ROWS_STEP],+ failed_count=0,+ diagnostics=[+ QueryStepDiagnostic("s0", "SELECT 1", True, None),+ QueryStepDiagnostic("s1", "SELECT 2", True, None),+ ],+ charts=[],+ no_rows_count=1,+ ),+ True,+ "# Report",+ True,+ id="one_step_with_rows_still_synthesizes",+ ),+ ],+)+@patch(_SLO_CAPTURE)+@patch(f"{_RP}.MaxChatOpenAI")+@patch(f"{_RP}._run_steps", new_callable=AsyncMock)+@patch(f"{_RP}.build_enriched_prompt")+async def test_synthesis_runs_only_when_a_step_returned_rows(+ mock_bep: MagicMock,+ mock_run: AsyncMock,+ mock_chat: MagicMock,+ mock_capture: MagicMock,+ run_result: PlanExecution,+ include_manage_link: bool,+ expected_report: str,+ synthesizes: bool,+) -> None:+ mock_bep.return_value = _spec(steps=len(run_result.rendered))+ mock_run.return_value = run_result+ mock_chat.return_value.with_structured_output.return_value.invoke.return_value = ReportDocument(+ topic=None,+ sections=[],+ custom_layout=[ReportText(items=[ReportTextItem(kind="text", text="# Report")])],+ )++ result = await generate_ai_report(+ team=MagicMock(),+ user=MagicMock(),+ prompt="x",+ window=_test_window(),+ include_manage_link=include_manage_link,+ )++ assert result.markdown == expected_report+ assert mock_chat.return_value.with_structured_output.return_value.invoke.called is synthesizes+ assert _slo_completed(mock_capture)["no_rows_steps"] == run_result.no_rows_count++@patch(f"{_RP}._arequest_hogql_fix", new_callable=AsyncMock)@patch(f"{_RP}.AssistantQueryExecutor")async def test_run_steps_retries_then_succeeds(mock_executor_cls: MagicMock, mock_fix: AsyncMock) -> None:@@ -860,6 +984,7 @@ async def test_run_steps_substitutes_fresh_window_into_placeholder_sql(mock_execpytest.param(_spec_with_window_placeholder(), _ALL_FAILED_RUN, id="all_failed"),# No window placeholder: freezing would cement an unbounded, window-less scan forever.pytest.param(_spec(steps=1), _OK_RUN, id="missing_window_placeholder"),+ pytest.param(_spec_with_window_placeholder(), _NO_ROWS_RUN, id="no_step_found_data"),],)@patch(_SLO_CAPTURE)