:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(mcp): record a code for hogql failures in execute-sql (#119346)
Vasco de Krijgerd1a5dda65fc0
diff --git a/ee/hogai/chat_agent/sql/mixins.py b/ee/hogai/chat_agent/sql/mixins.pyindex 700994bae969..459e4f20b383 100644--- a/ee/hogai/chat_agent/sql/mixins.py+++ b/ee/hogai/chat_agent/sql/mixins.py@@ -190,7 +190,7 @@ class HogQLOutputParserMixin(HogQLDatabaseMixin):except (ExposedHogQLError, HogQLNotImplementedError, QueryError, ResolutionError) as err:raise PydanticOutputParserException(llm_output=cleaned_query, validation_message=hogql_validation_message(err, cleaned_query)- )+ ) from errreturn AssistantHogQLQuery(query=cleaned_query)diff --git a/ee/hogai/context/insight/query_executor.py b/ee/hogai/context/insight/query_executor.pyindex aad17d82b8f4..ad0d0d540d97 100644--- a/ee/hogai/context/insight/query_executor.py+++ b/ee/hogai/context/insight/query_executor.py@@ -37,9 +37,11 @@ from posthog.schema import (from posthog.hogql.constants import LimitContextfrom posthog.hogql.errors import (+ BaseHogQLError,ExposedHogQLError,NotImplementedError as HogQLNotImplementedError,QueryError,+ ResolutionError,SyntaxError as HogQLSyntaxError,TableAccessDeniedError,)@@ -122,6 +124,14 @@ logger = structlog.get_logger(__name__)TIMING_LOG_PREFIX = "[QUERY_EXECUTOR]"+# Splits the most common HogQL query errors, which share one `code_name`, so analytics can count each+# without recording the message. The message can quote the caller's query.+HOGQL_ERROR_CODES_BY_MESSAGE_PREFIX = (+ ("Unable to resolve field", "hogql_unresolved_field"),+ ("Unknown table `", "hogql_unknown_table"),+ ("Unsupported function call", "hogql_unsupported_function"),+)+def get_clickhouse_error_code(error: BaseException | None) -> str | None:if not isinstance(error, InternalCHQueryError):@@ -130,6 +140,23 @@ def get_clickhouse_error_code(error: BaseException | None) -> str | None:return code if internal_ch_error_user_message(code) else None+def hogql_error_code(error: BaseException | None) -> str | None:+ """Return a code for a HogQL error that holds no caller input, or None for any other error."""+ if not isinstance(error, BaseHogQLError):+ return None+ message = str(error)+ for prefix, code in HOGQL_ERROR_CODES_BY_MESSAGE_PREFIX:+ if message.startswith(prefix):+ return code+ if isinstance(error, ExposedHogQLError):+ return error.code_name+ if isinstance(error, HogQLNotImplementedError):+ return "hogql_not_implemented"+ if isinstance(error, ResolutionError):+ return "hogql_resolution_error"+ return "hogql_internal_error"++def _hogql_tool_error(error: ExposedHogQLError) -> MaxToolError:cause: BaseException = errorseen: set[int] = set()@@ -139,9 +166,10 @@ def _hogql_tool_error(error: ExposedHogQLError) -> MaxToolError:if isinstance(cause, TableAccessDeniedError):return MaxToolFatalError(str(error), error_type="permission")+ if isinstance(cause, (QueryError, HogQLSyntaxError)):+ return MaxToolRetryableError(str(error), error_type="validation", error_code=hogql_error_code(cause))# User-safe errors can also describe outages; only known input errors should skip exception capture.- error_type: MaxToolErrorType = "validation" if isinstance(cause, (QueryError, HogQLSyntaxError)) else "internal"- return MaxToolRetryableError(str(error), error_type=error_type)+ return MaxToolRetryableError(str(error), error_type="internal")@frozendiff --git a/ee/hogai/context/insight/test/test_query_executor.py b/ee/hogai/context/insight/test/test_query_executor.pyindex 235cbd5bb4a4..19bc57077b4e 100644--- a/ee/hogai/context/insight/test/test_query_executor.py+++ b/ee/hogai/context/insight/test/test_query_executor.py@@ -37,7 +37,14 @@ from posthog.schema import ()from posthog.hogql.constants import DEFAULT_POSTHOG_AI_RETURNED_ROWS-from posthog.hogql.errors import ExposedHogQLError+from posthog.hogql.errors import (+ ExposedHogQLError,+ NotImplementedError as HogQLNotImplementedError,+ ParsingError,+ QueryError,+ ResolutionError,+ SyntaxError as HogQLSyntaxError,+)from posthog.clickhouse.client.limit import ConcurrencyLimitExceededfrom posthog.clickhouse.query_tagging import Feature, Product, get_query_tags, tags_context@@ -51,12 +58,37 @@ from ee.hogai.context.insight.query_executor import (AssistantQueryExecutor,execute_and_format_query,get_example_prompt,+ hogql_error_code,is_supported_query,)from ee.hogai.tool_errors import MaxToolFatalError, MaxToolRetryableError, MaxToolTransientErrorfrom ee.hogai.utils.query import validate_assistant_query+def _exposed_from(cause: Exception) -> ExposedHogQLError:+ error = ExposedHogQLError("HogQL error")+ error.__cause__ = cause+ return error+++class TestHogQLErrorCode(SimpleTestCase):+ @parameterized.expand(+ [+ ("unknown_table", QueryError("Unknown table `evnts`."), "hogql_unknown_table"),+ (+ "internal_unknown_table_type",+ ResolutionError("Unknown table type in LazyTableResolver: X"),+ "hogql_resolution_error",+ ),+ ("not_implemented", HogQLNotImplementedError("No support for X"), "hogql_not_implemented"),+ ("other_internal", ParsingError("Parsing failed"), "hogql_internal_error"),+ ("not_hogql", ValueError("Unable to resolve field: x"), None),+ ]+ )+ def test_codes_hold_no_caller_input(self, _name: str, error: Exception, expected_code: str | None) -> None:+ self.assertEqual(hogql_error_code(error), expected_code)++class TestSQLResultBudgets(SimpleTestCase):@parameterized.expand(["table", "fallback", "warnings"])async def test_sql_result_budget_reaches_formatting_and_fallback(self, kind: str) -> None:@@ -296,18 +328,31 @@ class TestAssistantQueryExecutor(NonAtomicBaseTest):self.assertEqual(str(context.exception), "API error message")+ @parameterized.expand(+ [+ ("outage_without_code", ExposedHogQLError("Warehouse connection failed"), "internal", None),+ ("unresolved_field", QueryError("Unable to resolve field: plan"), "validation", "hogql_unresolved_field"),+ ("syntax", HogQLSyntaxError("mismatched input 'FORM'"), "validation", "hogql_syntax_error"),+ (+ "wrapped_cause",+ _exposed_from(QueryError("Unable to resolve field: plan")),+ "validation",+ "hogql_unresolved_field",+ ),+ ]+ )@patch("ee.hogai.context.insight.query_executor.process_query_dict")- async def test_run_and_format_query_handles_exposed_hogql_error(self, mock_process_query):- """Test handling of ExposedHogQLError"""-- mock_process_query.side_effect = ExposedHogQLError("HogQL error")-- query = AssistantHogQLQuery(query="SELECT invalid")+ async def test_run_and_format_query_handles_exposed_hogql_error(+ self, _name, error, expected_type, expected_code, mock_process_query+ ):+ mock_process_query.side_effect = errorwith self.assertRaises(MaxToolRetryableError) as context:- await self.query_runner.arun_and_format_query(query)+ await self.query_runner.arun_and_format_query(AssistantHogQLQuery(query="SELECT plan FROM events"))- self.assertIn("HogQL error", str(context.exception))+ self.assertEqual(str(context.exception), str(error))+ self.assertEqual(context.exception.error_type, expected_type)+ self.assertEqual(context.exception.error_code, expected_code)@parameterized.expand([diff --git a/ee/hogai/tools/execute_sql/mcp_tool.py b/ee/hogai/tools/execute_sql/mcp_tool.pyindex 7243bdf3610e..65e06336d1bf 100644--- a/ee/hogai/tools/execute_sql/mcp_tool.py+++ b/ee/hogai/tools/execute_sql/mcp_tool.py@@ -15,6 +15,7 @@ from products.warehouse_sources.backend.facade.models import ExternalDataSourcefrom ee.hogai.chat_agent.schema_generator.parsers import PydanticOutputParserExceptionfrom ee.hogai.chat_agent.sql.mixins import HogQLOutputParserMixinfrom ee.hogai.context.insight.context import InsightContext+from ee.hogai.context.insight.query_executor import hogql_error_codefrom ee.hogai.mcp_tool import MCPTool, MCPToolResult, mcp_tool_registryfrom ee.hogai.tool_errors import MaxToolRetryableErrorfrom ee.hogai.tools.execute_sql.compatibility_hints import build_compatibility_hint@@ -90,7 +91,7 @@ class ExecuteSQLMCPTool(HogQLOutputParserMixin, MCPTool[ExecuteSQLMCPToolArgs]):) or build_compatibility_hint(e.validation_message)if suggestion:message = f"{message}\n\n{suggestion}"- raise MaxToolRetryableError(message)+ raise MaxToolRetryableError(message, error_code=hogql_error_code(e.__cause__)) from evariables = await self._abuild_query_variables(validated.query)query = HogQLQuery(query=validated.query, variables=variables) if variables else validateddiff --git a/ee/hogai/tools/execute_sql/test/test_mcp_tool.py b/ee/hogai/tools/execute_sql/test/test_mcp_tool.pyindex 392fc4485e2a..76485ff557f6 100644--- a/ee/hogai/tools/execute_sql/test/test_mcp_tool.py+++ b/ee/hogai/tools/execute_sql/test/test_mcp_tool.py@@ -104,13 +104,21 @@ class TestExecuteSQLMCPTool(ClickhouseTestMixin, NonAtomicBaseTest):self.assertNotIn("You are given a table with the results of a SQL query", result.content)self.assertNotIn("Here is the results table", result.content)- async def test_validation_error_for_invalid_query(self):+ @parameterized.expand(+ [+ ("syntax", "INVALID SQL SYNTAX", "hogql_syntax_error"),+ ("unresolved_field", "SELECT not_a_field FROM events", "hogql_unresolved_field"),+ ("unknown_table", "SELECT 1 FROM not_a_table", "hogql_unknown_table"),+ ("unsupported_function", "SELECT notAFunction(1) FROM events", "hogql_unsupported_function"),+ ("other_query_error", "SELECT count(count()) FROM events", "hogql_query_error"),+ ]+ )+ async def test_validation_error_for_invalid_query(self, _name: str, query: str, expected_code: str) -> None:with self.assertRaises(MaxToolRetryableError) as ctx:- await self.tool.execute(- ExecuteSQLMCPToolArgs(query="INVALID SQL SYNTAX"),- )+ await self.tool.execute(ExecuteSQLMCPToolArgs(query=query))self.assertIn("validation failed", str(ctx.exception).lower())+ self.assertEqual(ctx.exception.error_code, expected_code)@parameterized.expand([diff --git a/products/posthog_ai/backend/api/test_mcp_tools.py b/products/posthog_ai/backend/api/test_mcp_tools.pyindex e9c0eefd1112..5cc9229e1400 100644--- a/products/posthog_ai/backend/api/test_mcp_tools.py+++ b/products/posthog_ai/backend/api/test_mcp_tools.py@@ -336,15 +336,17 @@ class TestMCPToolsAPI(APIBaseTest):"validation","Tool failed: MaxToolRetryableError: Invalid query input. You may retry with adjusted inputs.",),- (+ param(HogQLSyntaxError("Unexpected SELECT"),"validation","Tool failed: MaxToolRetryableError: Unexpected SELECT. You may retry with adjusted inputs.",+ error_code="hogql_syntax_error",),- (+ param(QueryError("Unknown field: missing_column"),"validation","Tool failed: MaxToolRetryableError: Unknown field: missing_column. You may retry with adjusted inputs.",+ error_code="hogql_query_error",),(HogQLNotImplementedError("QueryVisitor has no method visit_select_query"),@@ -390,13 +392,14 @@ class TestMCPToolsAPI(APIBaseTest):"permission","Tool failed: MaxToolFatalError: Warehouse table access denied.",),- (+ param(_wrapped_hogql_error(_wrapped_hogql_error(QueryError("Invalid query input"), "Query validation failed"),"Warehouse SQL is invalid",),"validation","Tool failed: MaxToolRetryableError: Warehouse SQL is invalid. You may retry with adjusted inputs.",+ error_code="hogql_query_error",),(_wrapped_hogql_error(ConnectionError("connection refused"), "Warehouse connection failed"),