Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(mcp): record a code for hogql failures in execute-sql (#119346)

Vasco de Krijgerd1a5dda65fc0
diff --git a/ee/hogai/chat_agent/sql/mixins.py b/ee/hogai/chat_agent/sql/mixins.py
index 700994bae969..459e4f20b383 100644
--- a/ee/hogai/chat_agent/sql/mixins.py
+++ b/ee/hogai/chat_agent/sql/mixins.py
@@ -190,7 +190,7 @@ class HogQLOutputParserMixin(HogQLDatabaseMixin):
except (ExposedHogQLError, HogQLNotImplementedError, QueryError, ResolutionError) as err:
raise PydanticOutputParserException(
llm_output=cleaned_query, validation_message=hogql_validation_message(err, cleaned_query)
- )
+ ) from err
return AssistantHogQLQuery(query=cleaned_query)
diff --git a/ee/hogai/context/insight/query_executor.py b/ee/hogai/context/insight/query_executor.py
index aad17d82b8f4..ad0d0d540d97 100644
--- a/ee/hogai/context/insight/query_executor.py
+++ b/ee/hogai/context/insight/query_executor.py
@@ -37,9 +37,11 @@ from posthog.schema import (
from posthog.hogql.constants import LimitContext
from posthog.hogql.errors import (
+ BaseHogQLError,
ExposedHogQLError,
NotImplementedError as HogQLNotImplementedError,
QueryError,
+ ResolutionError,
SyntaxError as HogQLSyntaxError,
TableAccessDeniedError,
)
@@ -122,6 +124,14 @@ logger = structlog.get_logger(__name__)
TIMING_LOG_PREFIX = "[QUERY_EXECUTOR]"
+# Splits the most common HogQL query errors, which share one `code_name`, so analytics can count each
+# without recording the message. The message can quote the caller's query.
+HOGQL_ERROR_CODES_BY_MESSAGE_PREFIX = (
+ ("Unable to resolve field", "hogql_unresolved_field"),
+ ("Unknown table `", "hogql_unknown_table"),
+ ("Unsupported function call", "hogql_unsupported_function"),
+)
+
def get_clickhouse_error_code(error: BaseException | None) -> str | None:
if not isinstance(error, InternalCHQueryError):
@@ -130,6 +140,23 @@ def get_clickhouse_error_code(error: BaseException | None) -> str | None:
return code if internal_ch_error_user_message(code) else None
+def hogql_error_code(error: BaseException | None) -> str | None:
+ """Return a code for a HogQL error that holds no caller input, or None for any other error."""
+ if not isinstance(error, BaseHogQLError):
+ return None
+ message = str(error)
+ for prefix, code in HOGQL_ERROR_CODES_BY_MESSAGE_PREFIX:
+ if message.startswith(prefix):
+ return code
+ if isinstance(error, ExposedHogQLError):
+ return error.code_name
+ if isinstance(error, HogQLNotImplementedError):
+ return "hogql_not_implemented"
+ if isinstance(error, ResolutionError):
+ return "hogql_resolution_error"
+ return "hogql_internal_error"
+
+
def _hogql_tool_error(error: ExposedHogQLError) -> MaxToolError:
cause: BaseException = error
seen: set[int] = set()
@@ -139,9 +166,10 @@ def _hogql_tool_error(error: ExposedHogQLError) -> MaxToolError:
if isinstance(cause, TableAccessDeniedError):
return MaxToolFatalError(str(error), error_type="permission")
+ if isinstance(cause, (QueryError, HogQLSyntaxError)):
+ return MaxToolRetryableError(str(error), error_type="validation", error_code=hogql_error_code(cause))
# User-safe errors can also describe outages; only known input errors should skip exception capture.
- error_type: MaxToolErrorType = "validation" if isinstance(cause, (QueryError, HogQLSyntaxError)) else "internal"
- return MaxToolRetryableError(str(error), error_type=error_type)
+ return MaxToolRetryableError(str(error), error_type="internal")
@frozen
diff --git a/ee/hogai/context/insight/test/test_query_executor.py b/ee/hogai/context/insight/test/test_query_executor.py
index 235cbd5bb4a4..19bc57077b4e 100644
--- a/ee/hogai/context/insight/test/test_query_executor.py
+++ b/ee/hogai/context/insight/test/test_query_executor.py
@@ -37,7 +37,14 @@ from posthog.schema import (
)
from posthog.hogql.constants import DEFAULT_POSTHOG_AI_RETURNED_ROWS
-from posthog.hogql.errors import ExposedHogQLError
+from posthog.hogql.errors import (
+ ExposedHogQLError,
+ NotImplementedError as HogQLNotImplementedError,
+ ParsingError,
+ QueryError,
+ ResolutionError,
+ SyntaxError as HogQLSyntaxError,
+)
from posthog.clickhouse.client.limit import ConcurrencyLimitExceeded
from posthog.clickhouse.query_tagging import Feature, Product, get_query_tags, tags_context
@@ -51,12 +58,37 @@ from ee.hogai.context.insight.query_executor import (
AssistantQueryExecutor,
execute_and_format_query,
get_example_prompt,
+ hogql_error_code,
is_supported_query,
)
from ee.hogai.tool_errors import MaxToolFatalError, MaxToolRetryableError, MaxToolTransientError
from ee.hogai.utils.query import validate_assistant_query
+def _exposed_from(cause: Exception) -> ExposedHogQLError:
+ error = ExposedHogQLError("HogQL error")
+ error.__cause__ = cause
+ return error
+
+
+class TestHogQLErrorCode(SimpleTestCase):
+ @parameterized.expand(
+ [
+ ("unknown_table", QueryError("Unknown table `evnts`."), "hogql_unknown_table"),
+ (
+ "internal_unknown_table_type",
+ ResolutionError("Unknown table type in LazyTableResolver: X"),
+ "hogql_resolution_error",
+ ),
+ ("not_implemented", HogQLNotImplementedError("No support for X"), "hogql_not_implemented"),
+ ("other_internal", ParsingError("Parsing failed"), "hogql_internal_error"),
+ ("not_hogql", ValueError("Unable to resolve field: x"), None),
+ ]
+ )
+ def test_codes_hold_no_caller_input(self, _name: str, error: Exception, expected_code: str | None) -> None:
+ self.assertEqual(hogql_error_code(error), expected_code)
+
+
class TestSQLResultBudgets(SimpleTestCase):
@parameterized.expand(["table", "fallback", "warnings"])
async def test_sql_result_budget_reaches_formatting_and_fallback(self, kind: str) -> None:
@@ -296,18 +328,31 @@ class TestAssistantQueryExecutor(NonAtomicBaseTest):
self.assertEqual(str(context.exception), "API error message")
+ @parameterized.expand(
+ [
+ ("outage_without_code", ExposedHogQLError("Warehouse connection failed"), "internal", None),
+ ("unresolved_field", QueryError("Unable to resolve field: plan"), "validation", "hogql_unresolved_field"),
+ ("syntax", HogQLSyntaxError("mismatched input 'FORM'"), "validation", "hogql_syntax_error"),
+ (
+ "wrapped_cause",
+ _exposed_from(QueryError("Unable to resolve field: plan")),
+ "validation",
+ "hogql_unresolved_field",
+ ),
+ ]
+ )
@patch("ee.hogai.context.insight.query_executor.process_query_dict")
- async def test_run_and_format_query_handles_exposed_hogql_error(self, mock_process_query):
- """Test handling of ExposedHogQLError"""
-
- mock_process_query.side_effect = ExposedHogQLError("HogQL error")
-
- query = AssistantHogQLQuery(query="SELECT invalid")
+ async def test_run_and_format_query_handles_exposed_hogql_error(
+ self, _name, error, expected_type, expected_code, mock_process_query
+ ):
+ mock_process_query.side_effect = error
with self.assertRaises(MaxToolRetryableError) as context:
- await self.query_runner.arun_and_format_query(query)
+ await self.query_runner.arun_and_format_query(AssistantHogQLQuery(query="SELECT plan FROM events"))
- self.assertIn("HogQL error", str(context.exception))
+ self.assertEqual(str(context.exception), str(error))
+ self.assertEqual(context.exception.error_type, expected_type)
+ self.assertEqual(context.exception.error_code, expected_code)
@parameterized.expand(
[
diff --git a/ee/hogai/tools/execute_sql/mcp_tool.py b/ee/hogai/tools/execute_sql/mcp_tool.py
index 7243bdf3610e..65e06336d1bf 100644
--- a/ee/hogai/tools/execute_sql/mcp_tool.py
+++ b/ee/hogai/tools/execute_sql/mcp_tool.py
@@ -15,6 +15,7 @@ from products.warehouse_sources.backend.facade.models import ExternalDataSource
from ee.hogai.chat_agent.schema_generator.parsers import PydanticOutputParserException
from ee.hogai.chat_agent.sql.mixins import HogQLOutputParserMixin
from ee.hogai.context.insight.context import InsightContext
+from ee.hogai.context.insight.query_executor import hogql_error_code
from ee.hogai.mcp_tool import MCPTool, MCPToolResult, mcp_tool_registry
from ee.hogai.tool_errors import MaxToolRetryableError
from ee.hogai.tools.execute_sql.compatibility_hints import build_compatibility_hint
@@ -90,7 +91,7 @@ class ExecuteSQLMCPTool(HogQLOutputParserMixin, MCPTool[ExecuteSQLMCPToolArgs]):
) or build_compatibility_hint(e.validation_message)
if suggestion:
message = f"{message}\n\n{suggestion}"
- raise MaxToolRetryableError(message)
+ raise MaxToolRetryableError(message, error_code=hogql_error_code(e.__cause__)) from e
variables = await self._abuild_query_variables(validated.query)
query = HogQLQuery(query=validated.query, variables=variables) if variables else validated
diff --git a/ee/hogai/tools/execute_sql/test/test_mcp_tool.py b/ee/hogai/tools/execute_sql/test/test_mcp_tool.py
index 392fc4485e2a..76485ff557f6 100644
--- a/ee/hogai/tools/execute_sql/test/test_mcp_tool.py
+++ b/ee/hogai/tools/execute_sql/test/test_mcp_tool.py
@@ -104,13 +104,21 @@ class TestExecuteSQLMCPTool(ClickhouseTestMixin, NonAtomicBaseTest):
self.assertNotIn("You are given a table with the results of a SQL query", result.content)
self.assertNotIn("Here is the results table", result.content)
- async def test_validation_error_for_invalid_query(self):
+ @parameterized.expand(
+ [
+ ("syntax", "INVALID SQL SYNTAX", "hogql_syntax_error"),
+ ("unresolved_field", "SELECT not_a_field FROM events", "hogql_unresolved_field"),
+ ("unknown_table", "SELECT 1 FROM not_a_table", "hogql_unknown_table"),
+ ("unsupported_function", "SELECT notAFunction(1) FROM events", "hogql_unsupported_function"),
+ ("other_query_error", "SELECT count(count()) FROM events", "hogql_query_error"),
+ ]
+ )
+ async def test_validation_error_for_invalid_query(self, _name: str, query: str, expected_code: str) -> None:
with self.assertRaises(MaxToolRetryableError) as ctx:
- await self.tool.execute(
- ExecuteSQLMCPToolArgs(query="INVALID SQL SYNTAX"),
- )
+ await self.tool.execute(ExecuteSQLMCPToolArgs(query=query))
self.assertIn("validation failed", str(ctx.exception).lower())
+ self.assertEqual(ctx.exception.error_code, expected_code)
@parameterized.expand(
[
diff --git a/products/posthog_ai/backend/api/test_mcp_tools.py b/products/posthog_ai/backend/api/test_mcp_tools.py
index e9c0eefd1112..5cc9229e1400 100644
--- a/products/posthog_ai/backend/api/test_mcp_tools.py
+++ b/products/posthog_ai/backend/api/test_mcp_tools.py
@@ -336,15 +336,17 @@ class TestMCPToolsAPI(APIBaseTest):
"validation",
"Tool failed: MaxToolRetryableError: Invalid query input. You may retry with adjusted inputs.",
),
- (
+ param(
HogQLSyntaxError("Unexpected SELECT"),
"validation",
"Tool failed: MaxToolRetryableError: Unexpected SELECT. You may retry with adjusted inputs.",
+ error_code="hogql_syntax_error",
),
- (
+ param(
QueryError("Unknown field: missing_column"),
"validation",
"Tool failed: MaxToolRetryableError: Unknown field: missing_column. You may retry with adjusted inputs.",
+ error_code="hogql_query_error",
),
(
HogQLNotImplementedError("QueryVisitor has no method visit_select_query"),
@@ -390,13 +392,14 @@ class TestMCPToolsAPI(APIBaseTest):
"permission",
"Tool failed: MaxToolFatalError: Warehouse table access denied.",
),
- (
+ param(
_wrapped_hogql_error(
_wrapped_hogql_error(QueryError("Invalid query input"), "Query validation failed"),
"Warehouse SQL is invalid",
),
"validation",
"Tool failed: MaxToolRetryableError: Warehouse SQL is invalid. You may retry with adjusted inputs.",
+ error_code="hogql_query_error",
),
(
_wrapped_hogql_error(ConnectionError("connection refused"), "Warehouse connection failed"),