:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(hogql): report raw byte query result errors (#113693)
posthog[bot]7f523ff9cdc6
diff --git a/posthog/hogql/query.py b/posthog/hogql/query.pyindex d454e5167f8b..cb247a898b50 100644--- a/posthog/hogql/query.py+++ b/posthog/hogql/query.py@@ -1,4 +1,5 @@import dataclasses+from collections.abc import Sequencefrom datetime import datetimefrom time import sleepfrom typing import TYPE_CHECKING, Any, ClassVar, Literal, Optional, Union, cast@@ -116,6 +117,49 @@ TRANSIENT_S3_ERROR_RETRY_DELAY_SECONDS = 1.0MAIN_CLUSTER_WORKLOADS = (Workload.DEFAULT, Workload.ONLINE, Workload.OFFLINE)+def _contains_raw_bytes(value: object) -> bool:+ if value is None or isinstance(value, str):+ return False+ if isinstance(value, bytes):+ return True+ if isinstance(value, list):+ return any(_contains_raw_bytes(item) for item in value)+ if isinstance(value, tuple):+ return any(_contains_raw_bytes(item) for item in value)+ if isinstance(value, dict):+ return any(_contains_raw_bytes(key) or _contains_raw_bytes(item) for key, item in value.items())+ return False+++def _raise_for_raw_bytes_in_string_results(+ rows: Sequence[Sequence[object]], types: Sequence[tuple[str, str]], query_type: str+) -> None:+ """Reject raw bytes in String results because query responses and their cache use JSON."""+ columns_with_bytes: list[str] = []+ for column, (column_name, column_type) in enumerate(types):+ if "String" not in column_type:+ continue+ if any(_contains_raw_bytes(row[column]) for row in rows):+ columns_with_bytes.append(column_name)++ if not columns_with_bytes:+ return++ if query_type == "HogQLQuery":+ column_names = ", ".join(repr(column_name) for column_name in columns_with_bytes)+ raise QueryError(+ f"Query result column(s) {column_names} contain raw bytes that cannot be returned as JSON. "+ "Encode the affected values with `hex()` or `base64Encode()` in your query to preserve the original bytes."+ )++ error = QueryError(+ "Some values in these results aren't valid text, so the results can't be shown. "+ "Check the event and person properties used here for values with invalid characters."+ )+ capture_exception(error, {"query_type": query_type, "columns": columns_with_bytes})+ raise error++@frozenclass EmbeddedClickHouseQuery:sql: str@@ -165,6 +209,7 @@ class HogQLQueryExecutor:user_access_control: Optional[UserAccessControl] = Noneresolver_factory: Optional[ResolverFactory] = Nonerefresh_stale_view_columns: bool = True+ returned_row_limit: int | None = None__uninitialized_context: ClassVar[HogQLContext] = HogQLContext()@@ -570,8 +615,8 @@ class HogQLQueryExecutor:if self._references_information_schema(direct_context):return None- from posthog.hogql.transforms.trino.manifest import ( # noqa: PLC0415 -- load the optional backend only for Trino queries- transpile_hogql_to_trino_with_database,+ from posthog.hogql.transforms.trino.manifest import (+ transpile_hogql_to_trino_with_database, # noqa: PLC0415 -- load the optional backend only for Trino queries)transpiled = transpile_hogql_to_trino_with_database(@@ -1023,6 +1068,9 @@ class HogQLQueryExecutor:# Files backing a warehouse table can be replaced mid-read; one retry re-lists themsleep(TRANSIENT_S3_ERROR_RETRY_DELAY_SECONDS)self.results, self.types = run_clickhouse_query()+ _raise_for_raw_bytes_in_string_results(+ self.results[: self.returned_row_limit], self.types, self.query_type+ )except Exception as e:if self.debug:self.results = []diff --git a/posthog/hogql/test/test_query.py b/posthog/hogql/test/test_query.pyindex 04becf0dec36..648f483686c8 100644--- a/posthog/hogql/test/test_query.py+++ b/posthog/hogql/test/test_query.py@@ -2460,6 +2460,61 @@ class TestQuery(ClickhouseTestMixin, APIBaseTest):self.assertEqual(mock_sync_execute.call_count, 2)+ def test_invalid_utf8_string_results_raise_an_actionable_error(self):+ with self.assertRaises(QueryError) as error:+ execute_hogql_query(+ "SELECT unhex('41FF') AS invalid_scalar, [unhex('42FF')] AS invalid_array, "+ "map('key', unhex('43FF')) AS invalid_map",+ team=self.team,+ query_type="HogQLQuery",+ )++ self.assertEqual(+ str(error.exception),+ "Query result column(s) 'invalid_scalar', 'invalid_array', 'invalid_map' contain raw bytes that cannot "+ "be returned as JSON. Encode the affected values with `hex()` or `base64Encode()` in your query to "+ "preserve the original bytes.",+ )++ response = execute_hogql_query(+ "SELECT hex(unhex('41FF')) AS hex_value, base64Encode(unhex('41FF')) AS base64_value", team=self.team+ )++ self.assertEqual(response.results, [("41FF", "Qf8=")])++ @parameterized.expand(+ [+ (+ "sql_query",+ "HogQLQuery",+ "Query result column(s) 'invalid_map_keys' contain raw bytes that cannot be returned as JSON. "+ "Encode the affected values with `hex()` or `base64Encode()` in your query to preserve the original "+ "bytes.",+ False,+ ),+ (+ "built_query",+ "hogql_query",+ "Some values in these results aren't valid text, so the results can't be shown. "+ "Check the event and person properties used here for values with invalid characters.",+ True,+ ),+ ]+ )+ @patch("posthog.hogql.query.capture_exception")+ def test_invalid_utf8_map_keys_raise_an_actionable_error(+ self, _name: str, query_type: str, expected_error: str, captured: bool, mock_capture_exception: mock.MagicMock+ ) -> None:+ with self.assertRaises(QueryError) as error:+ execute_hogql_query(+ "SELECT map(unhex('FF'), 1, unhex('FE'), 2) AS invalid_map_keys",+ team=self.team,+ query_type=query_type,+ )++ self.assertEqual(str(error.exception), expected_error)+ self.assertEqual(mock_capture_exception.called, captured)+def test_non_transient_errors_are_not_retried(self):with (patch(diff --git a/posthog/hogql_queries/paginators.py b/posthog/hogql_queries/paginators.pyindex 69d0161fdff6..2a81845e81e2 100644--- a/posthog/hogql_queries/paginators.py+++ b/posthog/hogql_queries/paginators.py@@ -115,6 +115,7 @@ class HogQLHasMorePaginator:execute_hogql_query(query=self.paginate(query),query_type=query_type,+ returned_row_limit=self.limit,**kwargs if self.limit_context is None else {"limit_context": self.limit_context, **kwargs},),)@@ -328,6 +329,7 @@ class HogQLCursorPaginator:execute_hogql_query(query=self.paginate(query),query_type=query_type,+ returned_row_limit=self.limit,**kwargs if self.limit_context is None else {"limit_context": self.limit_context, **kwargs},),)diff --git a/posthog/hogql_queries/test/test_paginators.py b/posthog/hogql_queries/test/test_paginators.pyindex 1eaeb29a7327..c4c9f87c66b4 100644--- a/posthog/hogql_queries/test/test_paginators.py+++ b/posthog/hogql_queries/test/test_paginators.py@@ -104,6 +104,16 @@ class TestHogQLHasMorePaginator(ClickhouseTestMixin, APIBaseTest):self.assertEqual(len(response.results), 10)self.assertFalse(response.hasMore)+ def test_raw_bytes_in_the_has_more_probe_row_do_not_fail_the_page(self):+ paginator = HogQLHasMorePaginator(limit=1)+ paginator.execute_hogql_query(+ parse_select("SELECT x FROM (SELECT 'a' AS x UNION ALL SELECT unhex('FF') AS x) ORDER BY x"),+ query_type="HogQLQuery",+ team=self.team,+ )+ self.assertEqual(paginator.results, [("a",)])+ self.assertTrue(paginator.has_more())+def test_empty_result_set(self):"""Test behavior when query returns no results."""runner = self._create_runner(