Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(hogql): report raw byte query result errors (#113693)

posthog[bot]7f523ff9cdc6
diff --git a/posthog/hogql/query.py b/posthog/hogql/query.py
index d454e5167f8b..cb247a898b50 100644
--- a/posthog/hogql/query.py
+++ b/posthog/hogql/query.py
@@ -1,4 +1,5 @@
import dataclasses
+from collections.abc import Sequence
from datetime import datetime
from time import sleep
from typing import TYPE_CHECKING, Any, ClassVar, Literal, Optional, Union, cast
@@ -116,6 +117,49 @@ TRANSIENT_S3_ERROR_RETRY_DELAY_SECONDS = 1.0
MAIN_CLUSTER_WORKLOADS = (Workload.DEFAULT, Workload.ONLINE, Workload.OFFLINE)
+def _contains_raw_bytes(value: object) -> bool:
+ if value is None or isinstance(value, str):
+ return False
+ if isinstance(value, bytes):
+ return True
+ if isinstance(value, list):
+ return any(_contains_raw_bytes(item) for item in value)
+ if isinstance(value, tuple):
+ return any(_contains_raw_bytes(item) for item in value)
+ if isinstance(value, dict):
+ return any(_contains_raw_bytes(key) or _contains_raw_bytes(item) for key, item in value.items())
+ return False
+
+
+def _raise_for_raw_bytes_in_string_results(
+ rows: Sequence[Sequence[object]], types: Sequence[tuple[str, str]], query_type: str
+) -> None:
+ """Reject raw bytes in String results because query responses and their cache use JSON."""
+ columns_with_bytes: list[str] = []
+ for column, (column_name, column_type) in enumerate(types):
+ if "String" not in column_type:
+ continue
+ if any(_contains_raw_bytes(row[column]) for row in rows):
+ columns_with_bytes.append(column_name)
+
+ if not columns_with_bytes:
+ return
+
+ if query_type == "HogQLQuery":
+ column_names = ", ".join(repr(column_name) for column_name in columns_with_bytes)
+ raise QueryError(
+ f"Query result column(s) {column_names} contain raw bytes that cannot be returned as JSON. "
+ "Encode the affected values with `hex()` or `base64Encode()` in your query to preserve the original bytes."
+ )
+
+ error = QueryError(
+ "Some values in these results aren't valid text, so the results can't be shown. "
+ "Check the event and person properties used here for values with invalid characters."
+ )
+ capture_exception(error, {"query_type": query_type, "columns": columns_with_bytes})
+ raise error
+
+
@frozen
class EmbeddedClickHouseQuery:
sql: str
@@ -165,6 +209,7 @@ class HogQLQueryExecutor:
user_access_control: Optional[UserAccessControl] = None
resolver_factory: Optional[ResolverFactory] = None
refresh_stale_view_columns: bool = True
+ returned_row_limit: int | None = None
__uninitialized_context: ClassVar[HogQLContext] = HogQLContext()
@@ -570,8 +615,8 @@ class HogQLQueryExecutor:
if self._references_information_schema(direct_context):
return None
- from posthog.hogql.transforms.trino.manifest import ( # noqa: PLC0415 -- load the optional backend only for Trino queries
- transpile_hogql_to_trino_with_database,
+ from posthog.hogql.transforms.trino.manifest import (
+ transpile_hogql_to_trino_with_database, # noqa: PLC0415 -- load the optional backend only for Trino queries
)
transpiled = transpile_hogql_to_trino_with_database(
@@ -1023,6 +1068,9 @@ class HogQLQueryExecutor:
# Files backing a warehouse table can be replaced mid-read; one retry re-lists them
sleep(TRANSIENT_S3_ERROR_RETRY_DELAY_SECONDS)
self.results, self.types = run_clickhouse_query()
+ _raise_for_raw_bytes_in_string_results(
+ self.results[: self.returned_row_limit], self.types, self.query_type
+ )
except Exception as e:
if self.debug:
self.results = []
diff --git a/posthog/hogql/test/test_query.py b/posthog/hogql/test/test_query.py
index 04becf0dec36..648f483686c8 100644
--- a/posthog/hogql/test/test_query.py
+++ b/posthog/hogql/test/test_query.py
@@ -2460,6 +2460,61 @@ class TestQuery(ClickhouseTestMixin, APIBaseTest):
self.assertEqual(mock_sync_execute.call_count, 2)
+ def test_invalid_utf8_string_results_raise_an_actionable_error(self):
+ with self.assertRaises(QueryError) as error:
+ execute_hogql_query(
+ "SELECT unhex('41FF') AS invalid_scalar, [unhex('42FF')] AS invalid_array, "
+ "map('key', unhex('43FF')) AS invalid_map",
+ team=self.team,
+ query_type="HogQLQuery",
+ )
+
+ self.assertEqual(
+ str(error.exception),
+ "Query result column(s) 'invalid_scalar', 'invalid_array', 'invalid_map' contain raw bytes that cannot "
+ "be returned as JSON. Encode the affected values with `hex()` or `base64Encode()` in your query to "
+ "preserve the original bytes.",
+ )
+
+ response = execute_hogql_query(
+ "SELECT hex(unhex('41FF')) AS hex_value, base64Encode(unhex('41FF')) AS base64_value", team=self.team
+ )
+
+ self.assertEqual(response.results, [("41FF", "Qf8=")])
+
+ @parameterized.expand(
+ [
+ (
+ "sql_query",
+ "HogQLQuery",
+ "Query result column(s) 'invalid_map_keys' contain raw bytes that cannot be returned as JSON. "
+ "Encode the affected values with `hex()` or `base64Encode()` in your query to preserve the original "
+ "bytes.",
+ False,
+ ),
+ (
+ "built_query",
+ "hogql_query",
+ "Some values in these results aren't valid text, so the results can't be shown. "
+ "Check the event and person properties used here for values with invalid characters.",
+ True,
+ ),
+ ]
+ )
+ @patch("posthog.hogql.query.capture_exception")
+ def test_invalid_utf8_map_keys_raise_an_actionable_error(
+ self, _name: str, query_type: str, expected_error: str, captured: bool, mock_capture_exception: mock.MagicMock
+ ) -> None:
+ with self.assertRaises(QueryError) as error:
+ execute_hogql_query(
+ "SELECT map(unhex('FF'), 1, unhex('FE'), 2) AS invalid_map_keys",
+ team=self.team,
+ query_type=query_type,
+ )
+
+ self.assertEqual(str(error.exception), expected_error)
+ self.assertEqual(mock_capture_exception.called, captured)
+
def test_non_transient_errors_are_not_retried(self):
with (
patch(
diff --git a/posthog/hogql_queries/paginators.py b/posthog/hogql_queries/paginators.py
index 69d0161fdff6..2a81845e81e2 100644
--- a/posthog/hogql_queries/paginators.py
+++ b/posthog/hogql_queries/paginators.py
@@ -115,6 +115,7 @@ class HogQLHasMorePaginator:
execute_hogql_query(
query=self.paginate(query),
query_type=query_type,
+ returned_row_limit=self.limit,
**kwargs if self.limit_context is None else {"limit_context": self.limit_context, **kwargs},
),
)
@@ -328,6 +329,7 @@ class HogQLCursorPaginator:
execute_hogql_query(
query=self.paginate(query),
query_type=query_type,
+ returned_row_limit=self.limit,
**kwargs if self.limit_context is None else {"limit_context": self.limit_context, **kwargs},
),
)
diff --git a/posthog/hogql_queries/test/test_paginators.py b/posthog/hogql_queries/test/test_paginators.py
index 1eaeb29a7327..c4c9f87c66b4 100644
--- a/posthog/hogql_queries/test/test_paginators.py
+++ b/posthog/hogql_queries/test/test_paginators.py
@@ -104,6 +104,16 @@ class TestHogQLHasMorePaginator(ClickhouseTestMixin, APIBaseTest):
self.assertEqual(len(response.results), 10)
self.assertFalse(response.hasMore)
+ def test_raw_bytes_in_the_has_more_probe_row_do_not_fail_the_page(self):
+ paginator = HogQLHasMorePaginator(limit=1)
+ paginator.execute_hogql_query(
+ parse_select("SELECT x FROM (SELECT 'a' AS x UNION ALL SELECT unhex('FF') AS x) ORDER BY x"),
+ query_type="HogQLQuery",
+ team=self.team,
+ )
+ self.assertEqual(paginator.results, [("a",)])
+ self.assertTrue(paginator.has_more())
+
def test_empty_result_set(self):
"""Test behavior when query returns no results."""
runner = self._create_runner(