Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(subscriptions): show why an ai report query failed (#124981)

Vasco de Krijger7c02056db61c
diff --git a/products/subscriptions/backend/temporal/ai_subscription/report_pipeline.py b/products/subscriptions/backend/temporal/ai_subscription/report_pipeline.py
index 99c2122edc5c..6159fae3cca9 100644
--- a/products/subscriptions/backend/temporal/ai_subscription/report_pipeline.py
+++ b/products/subscriptions/backend/temporal/ai_subscription/report_pipeline.py
@@ -74,10 +74,15 @@ from products.subscriptions.backend.temporal.ai_subscription.spec_generator impo
resolve_ai_query_plan_status,
window_token_problems,
)
-from products.subscriptions.backend.temporal.types import safe_query_error_details
+from products.subscriptions.backend.temporal.types import QueryErrorDetails, safe_query_error_details
+from ee.hogai.context.insight.clickhouse_rejections import describe_clickhouse_rejection
from ee.hogai.context.insight.format import format_access_control_warnings, format_warehouse_sync_warnings
-from ee.hogai.context.insight.query_executor import AssistantQueryExecutor
+from ee.hogai.context.insight.query_executor import (
+ HOGQL_ERROR_CODES_BY_MESSAGE_PREFIX,
+ HOGQL_INPUT_ERROR_CODES,
+ AssistantQueryExecutor,
+)
from ee.hogai.llm import MaxChatOpenAI
from ee.hogai.tool_errors import MaxToolRetryableError
@@ -147,6 +152,23 @@ def _reject_unknown_placeholders(hogql: str) -> None:
raise QueryError(" ".join([*problems, WINDOW_TOKEN_HINT]))
+_HOGQL_INPUT_ERROR_CODES = HOGQL_INPUT_ERROR_CODES | {code for _, code in HOGQL_ERROR_CODES_BY_MESSAGE_PREFIX}
+
+
+def _safe_step_error_details(exc: BaseException) -> QueryErrorDetails | None:
+ if details := safe_query_error_details(exc):
+ return details
+ if not isinstance(exc, MaxToolRetryableError) or not exc.error_code:
+ return None
+ if exc.error_code in _HOGQL_INPUT_ERROR_CODES and exc.error_type == "validation":
+ message: str | None = str(exc)
+ else:
+ message = describe_clickhouse_rejection(exc.error_code)
+ if not message:
+ return None
+ return {"type": type(exc).__name__, "code": exc.error_code, "message": message.replace("\x00", "")}
+
+
def _all_queries_failed_notice(total_steps: int, *, include_manage_link: bool = True) -> str:
noun = "the query" if total_steps == 1 else f"all {total_steps} queries"
notice = f"> ⚠️ This report could not be generated — {noun} the assistant wrote failed to run."
@@ -712,7 +734,7 @@ async def _run_steps(
max_retries=_MAX_QUERY_FIX_RETRIES,
error_type=type(exc).__name__,
)
- error_details = safe_query_error_details(exc)
+ error_details = _safe_step_error_details(exc)
fixed = await _arequest_hogql_fix(
original_hogql=current_hogql,
# Forward explicitly safe detail when available; fall back to the type name.
@@ -730,7 +752,7 @@ async def _run_steps(
current_hogql = fixed
type_name = type(last_exc).__name__ if last_exc is not None else "UnknownError"
- error_details = safe_query_error_details(last_exc) if last_exc is not None else None
+ error_details = _safe_step_error_details(last_exc) if last_exc is not None else None
logger.warning(
"ai_report.query_failed",
trace_correlation_id=trace_correlation_id,
diff --git a/products/subscriptions/backend/temporal/ai_subscription/test/test_report_pipeline.py b/products/subscriptions/backend/temporal/ai_subscription/test/test_report_pipeline.py
index 9a9c8e09f3ca..bfd3d4d49bd4 100644
--- a/products/subscriptions/backend/temporal/ai_subscription/test/test_report_pipeline.py
+++ b/products/subscriptions/backend/temporal/ai_subscription/test/test_report_pipeline.py
@@ -464,6 +464,85 @@ async def test_run_steps_keeps_wrapped_error_out_of_recipient_copy_and_records_s
mock_hogql_fix.assert_awaited_once()
+ "error,expected_code,expected_message",
+ [
+ pytest.param(
+ MaxToolRetryableError(
+ "Unable to resolve field: plan_name", error_type="validation", error_code="hogql_unresolved_field"
+ ),
+ "hogql_unresolved_field",
+ "Unable to resolve field: plan_name",
+ id="hogql_input_error",
+ ),
+ pytest.param(
+ MaxToolRetryableError(
+ "Cannot parse '[email protected]' as DateTime",
+ error_type="validation",
+ error_code="cannot_parse_datetime",
+ ),
+ "cannot_parse_datetime",
+ "ClickHouse rejected the query with error CANNOT_PARSE_DATETIME.",
+ id="clickhouse_rejection_uses_curated_text",
+ ),
+ pytest.param(
+ MaxToolRetryableError("Code: 999. boom on host ch-1", error_type="internal", error_code="internal_error"),
+ None,
+ None,
+ id="internal_error",
+ ),
+ pytest.param(MaxToolRetryableError("something failed"), None, None, id="no_error_code"),
+ pytest.param(
+ MaxToolRetryableError(
+ "Code: 6. DB::Exception: Cannot parse '[email protected]' on ch-internal-3",
+ error_type="internal",
+ error_code="unhandled_dbexception",
+ ),
+ None,
+ None,
+ id="raw_unhandled_error_stays_generic",
+ ),
+ pytest.param(
+ MaxToolRetryableError(
+ "Unable to resolve field: plan_name", error_type="internal", error_code="hogql_unresolved_field"
+ ),
+ None,
+ None,
+ id="input_code_on_an_internal_error_stays_generic",
+ ),
+ pytest.param(
+ MaxToolRetryableError(
+ "Unknown table `plans\x00`.", error_type="validation", error_code="hogql_unknown_table"
+ ),
+ "hogql_unknown_table",
+ "Unknown table `plans`.",
+ id="nul_is_stripped",
+ ),
+ ],
+)
+@patch(f"{_RP}.AssistantQueryExecutor")
+@patch(f"{_RP}._arequest_hogql_fix", new_callable=AsyncMock, return_value=None)
+async def test_run_steps_records_owner_safe_reason_for_an_async_query_failure(
+ mock_hogql_fix: AsyncMock,
+ mock_executor_cls: MagicMock,
+ error: MaxToolRetryableError,
+ expected_code: str | None,
+ expected_message: str | None,
+) -> None:
+ mock_executor_cls.return_value.arun_format_and_capture = AsyncMock(side_effect=error)
+
+ execution = await _run_steps(
+ _spec(steps=1), MagicMock(), MagicMock(), _test_window(), None, charts_enabled_for_team=True
+ )
+
+ cause = "" if expected_message else " (MaxToolRetryableError)"
+ assert execution.rendered[0] == f"### s0\n\n_{QUERY_FAILED_PREFIX}{cause} — metric not computed, not empty data._"
+ assert execution.diagnostics[0].error_code == expected_code
+ assert execution.diagnostics[0].human_readable_error == expected_message
+ assert mock_hogql_fix.await_args is not None
+ assert mock_hogql_fix.await_args.kwargs["error_message"] == (expected_message or "MaxToolRetryableError")
+
+
@patch(f"{_RP}._arequest_hogql_fix", new_callable=AsyncMock)
@patch(f"{_RP}.AssistantQueryExecutor")
async def test_run_steps_forwards_exposed_query_error_message_to_fix(