:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(subscriptions): show why an ai report query failed (#124981)
Vasco de Krijger7c02056db61c
diff --git a/products/subscriptions/backend/temporal/ai_subscription/report_pipeline.py b/products/subscriptions/backend/temporal/ai_subscription/report_pipeline.pyindex 99c2122edc5c..6159fae3cca9 100644--- a/products/subscriptions/backend/temporal/ai_subscription/report_pipeline.py+++ b/products/subscriptions/backend/temporal/ai_subscription/report_pipeline.py@@ -74,10 +74,15 @@ from products.subscriptions.backend.temporal.ai_subscription.spec_generator imporesolve_ai_query_plan_status,window_token_problems,)-from products.subscriptions.backend.temporal.types import safe_query_error_details+from products.subscriptions.backend.temporal.types import QueryErrorDetails, safe_query_error_details+from ee.hogai.context.insight.clickhouse_rejections import describe_clickhouse_rejectionfrom ee.hogai.context.insight.format import format_access_control_warnings, format_warehouse_sync_warnings-from ee.hogai.context.insight.query_executor import AssistantQueryExecutor+from ee.hogai.context.insight.query_executor import (+ HOGQL_ERROR_CODES_BY_MESSAGE_PREFIX,+ HOGQL_INPUT_ERROR_CODES,+ AssistantQueryExecutor,+)from ee.hogai.llm import MaxChatOpenAIfrom ee.hogai.tool_errors import MaxToolRetryableError@@ -147,6 +152,23 @@ def _reject_unknown_placeholders(hogql: str) -> None:raise QueryError(" ".join([*problems, WINDOW_TOKEN_HINT]))+_HOGQL_INPUT_ERROR_CODES = HOGQL_INPUT_ERROR_CODES | {code for _, code in HOGQL_ERROR_CODES_BY_MESSAGE_PREFIX}+++def _safe_step_error_details(exc: BaseException) -> QueryErrorDetails | None:+ if details := safe_query_error_details(exc):+ return details+ if not isinstance(exc, MaxToolRetryableError) or not exc.error_code:+ return None+ if exc.error_code in _HOGQL_INPUT_ERROR_CODES and exc.error_type == "validation":+ message: str | None = str(exc)+ else:+ message = describe_clickhouse_rejection(exc.error_code)+ if not message:+ return None+ return {"type": type(exc).__name__, "code": exc.error_code, "message": message.replace("\x00", "")}++def _all_queries_failed_notice(total_steps: int, *, include_manage_link: bool = True) -> str:noun = "the query" if total_steps == 1 else f"all {total_steps} queries"notice = f"> ⚠️ This report could not be generated — {noun} the assistant wrote failed to run."@@ -712,7 +734,7 @@ async def _run_steps(max_retries=_MAX_QUERY_FIX_RETRIES,error_type=type(exc).__name__,)- error_details = safe_query_error_details(exc)+ error_details = _safe_step_error_details(exc)fixed = await _arequest_hogql_fix(original_hogql=current_hogql,# Forward explicitly safe detail when available; fall back to the type name.@@ -730,7 +752,7 @@ async def _run_steps(current_hogql = fixedtype_name = type(last_exc).__name__ if last_exc is not None else "UnknownError"- error_details = safe_query_error_details(last_exc) if last_exc is not None else None+ error_details = _safe_step_error_details(last_exc) if last_exc is not None else Nonelogger.warning("ai_report.query_failed",trace_correlation_id=trace_correlation_id,diff --git a/products/subscriptions/backend/temporal/ai_subscription/test/test_report_pipeline.py b/products/subscriptions/backend/temporal/ai_subscription/test/test_report_pipeline.pyindex 9a9c8e09f3ca..bfd3d4d49bd4 100644--- a/products/subscriptions/backend/temporal/ai_subscription/test/test_report_pipeline.py+++ b/products/subscriptions/backend/temporal/ai_subscription/test/test_report_pipeline.py@@ -464,6 +464,85 @@ async def test_run_steps_keeps_wrapped_error_out_of_recipient_copy_and_records_smock_hogql_fix.assert_awaited_once()+ "error,expected_code,expected_message",+ [+ pytest.param(+ MaxToolRetryableError(+ "Unable to resolve field: plan_name", error_type="validation", error_code="hogql_unresolved_field"+ ),+ "hogql_unresolved_field",+ "Unable to resolve field: plan_name",+ id="hogql_input_error",+ ),+ pytest.param(+ MaxToolRetryableError(+ "Cannot parse '[email protected]' as DateTime",+ error_type="validation",+ error_code="cannot_parse_datetime",+ ),+ "cannot_parse_datetime",+ "ClickHouse rejected the query with error CANNOT_PARSE_DATETIME.",+ id="clickhouse_rejection_uses_curated_text",+ ),+ pytest.param(+ MaxToolRetryableError("Code: 999. boom on host ch-1", error_type="internal", error_code="internal_error"),+ None,+ None,+ id="internal_error",+ ),+ pytest.param(MaxToolRetryableError("something failed"), None, None, id="no_error_code"),+ pytest.param(+ MaxToolRetryableError(+ "Code: 6. DB::Exception: Cannot parse '[email protected]' on ch-internal-3",+ error_type="internal",+ error_code="unhandled_dbexception",+ ),+ None,+ None,+ id="raw_unhandled_error_stays_generic",+ ),+ pytest.param(+ MaxToolRetryableError(+ "Unable to resolve field: plan_name", error_type="internal", error_code="hogql_unresolved_field"+ ),+ None,+ None,+ id="input_code_on_an_internal_error_stays_generic",+ ),+ pytest.param(+ MaxToolRetryableError(+ "Unknown table `plans\x00`.", error_type="validation", error_code="hogql_unknown_table"+ ),+ "hogql_unknown_table",+ "Unknown table `plans`.",+ id="nul_is_stripped",+ ),+ ],+)+@patch(f"{_RP}.AssistantQueryExecutor")+@patch(f"{_RP}._arequest_hogql_fix", new_callable=AsyncMock, return_value=None)+async def test_run_steps_records_owner_safe_reason_for_an_async_query_failure(+ mock_hogql_fix: AsyncMock,+ mock_executor_cls: MagicMock,+ error: MaxToolRetryableError,+ expected_code: str | None,+ expected_message: str | None,+) -> None:+ mock_executor_cls.return_value.arun_format_and_capture = AsyncMock(side_effect=error)++ execution = await _run_steps(+ _spec(steps=1), MagicMock(), MagicMock(), _test_window(), None, charts_enabled_for_team=True+ )++ cause = "" if expected_message else " (MaxToolRetryableError)"+ assert execution.rendered[0] == f"### s0\n\n_{QUERY_FAILED_PREFIX}{cause} — metric not computed, not empty data._"+ assert execution.diagnostics[0].error_code == expected_code+ assert execution.diagnostics[0].human_readable_error == expected_message+ assert mock_hogql_fix.await_args is not None+ assert mock_hogql_fix.await_args.kwargs["error_message"] == (expected_message or "MaxToolRetryableError")++@patch(f"{_RP}._arequest_hogql_fix", new_callable=AsyncMock)@patch(f"{_RP}.AssistantQueryExecutor")async def test_run_steps_forwards_exposed_query_error_message_to_fix(