:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(hogql): list selectable fields when a field does not resolve (#119347)
Vasco de Krijger210539cafbb4
diff --git a/posthog/hogql/resolver.py b/posthog/hogql/resolver.pyindex 234084074c68..50bae2a91745 100644--- a/posthog/hogql/resolver.py+++ b/posthog/hogql/resolver.py@@ -55,6 +55,7 @@ from posthog.hogql.functions.with_icon import with_iconfrom posthog.hogql.hogqlx import HOGQLX_COMPONENTS, HOGQLX_TAGS, convert_to_hxfrom posthog.hogql.parser import parse_selectfrom posthog.hogql.resolver_utils import (+ available_fields_suffix,expand_hogqlx_query,lookup_field_by_name,lookup_table_by_name,@@ -2517,7 +2518,11 @@ class Resolver(CloningVisitor):return ast.Constant(value=value, type=global_type)suggestions = suggest_field_names(scope, name, self.context)- suggestion_suffix = f". Did you mean: {', '.join(suggestions)}?" if suggestions else ""+ suggestion_suffix = (+ f". Did you mean: {', '.join(suggestions)}?"+ if suggestions+ else available_fields_suffix(scope, node, self.context)+ )# The message lists every close match, but a quick fix can only substitute one, so it# offers the best of them. `get_close_matches` returns them in descending similarity.fix = suggested_field_fix(node, suggestions[0]) if suggestions else Nonediff --git a/posthog/hogql/resolver_utils.py b/posthog/hogql/resolver_utils.pyindex a871c34080c1..951c694b5464 100644--- a/posthog/hogql/resolver_utils.py+++ b/posthog/hogql/resolver_utils.py@@ -1,7 +1,7 @@from __future__ import annotationsimport difflib-from collections.abc import Generator+from collections.abc import Callable, Generator, Iterablefrom typing import Optional, castfrom pydantic import BaseModel@@ -27,7 +27,10 @@ from posthog.hogql.database.models import (UUIDDatabaseField,)from posthog.hogql.errors import QueryError, ResolutionError, SyntaxError-from posthog.hogql.escape_sql import escape_hogql_identifier+from posthog.hogql.escape_sql import escape_hogql_identifier, quote_hogql_identifier++# Bounds the message for wide warehouse tables. A longer list stops helping the reader pick a column.+MAX_LISTED_FIELD_NAMES = 30def lookup_field_by_name(@@ -111,44 +114,89 @@ def _select_set_type_source_name(index: int) -> str:return f"query source {index}"-def _names_on_table_type(table_type: ast.Type, context: HogQLContext) -> set[str]:+def _names_on_table_type(+ table_type: ast.Type, context: HogQLContext, table_names: Callable[[Table], Iterable[str]]+) -> list[str]:+ if isinstance(table_type, ast.ColumnAliasedTableType):+ return list(table_type.alias_to_original)+ if isinstance(table_type, ast.SelectQueryAliasType):+ return _names_on_table_type(table_type.select_query_type, context, table_names)if isinstance(table_type, ast.BaseTableType):try:- table = table_type.resolve_database_table(context)+ return list(table_names(table_type.resolve_database_table(context)))except Exception:- return set()- return set(table.fields.keys())+ return []if isinstance(table_type, (ast.SelectQueryType, ast.SelectSetQueryType)):- return set(getattr(table_type, "columns", {}).keys())- return set()+ return list(getattr(table_type, "columns", {}).keys())+ return []+++def _collect_field_names(+ scope: ast.SelectQueryType | ast.SelectSetQueryType,+ context: HogQLContext,+ table_names: Callable[[Table], Iterable[str]],+) -> list[str]:+ if isinstance(scope, ast.SelectSetQueryType):+ names = [name for inner in scope.types for name in _collect_field_names(inner, context, table_names)]+ else:+ names = list(scope.aliases.keys())+ for table_type in [*scope.tables.values(), *scope.anonymous_tables]:+ names.extend(_names_on_table_type(table_type, context, table_names))+ if scope.parent is not None:+ names.extend(_collect_field_names(scope.parent, context, table_names))+ return list(dict.fromkeys(names))def collect_available_field_names(scope: ast.SelectQueryType | ast.SelectSetQueryType, context: HogQLContext) -> set[str]:- """Gather field names visible in a scope (aliases + joined-table fields)- for use in 'did you mean' suggestions when resolution fails.+ """Every field name visible in a scope, for 'did you mean' suggestions when resolution fails."""+ return set(_collect_field_names(scope, context, lambda table: table.fields.keys()))- Returns a flat set — callers don't need to know whether a name comes from- an alias, a named table, or an anonymous subquery.- """- names: set[str] = set()- if isinstance(scope, ast.SelectSetQueryType):- for inner in scope.types:- names.update(collect_available_field_names(inner, context))- return names+def collect_selectable_field_names(+ scope: ast.SelectQueryType | ast.SelectSetQueryType, context: HogQLContext+) -> list[str]:+ """Names a query in `scope` can select, in declaration order: aliases first, then the columns+ `SELECT *` would return, including explicitly joined tables. Implicit join fields,+ internal sub-tables and hidden fields are left out."""+ return _collect_field_names(scope, context, lambda table: table.get_asterisk().keys())- names.update(scope.aliases.keys())- for table_type in scope.tables.values():- names.update(_names_on_table_type(table_type, context))- for anon in scope.anonymous_tables:- names.update(_names_on_table_type(anon, context))- if scope.parent is not None:- names.update(collect_available_field_names(scope.parent, context))+def _has_one_json_properties_field(scope: ast.SelectQueryType | ast.SelectSetQueryType, context: HogQLContext) -> bool:+ try:+ field = lookup_field_by_name(scope, "properties", context)+ return field is not None and isinstance(field.resolve_constant_type(context), ast.StringJSONType)+ except Exception:+ return False+- return names+def available_fields_suffix(+ scope: ast.SelectQueryType | ast.SelectSetQueryType, node: ast.Field, context: HogQLContext+) -> str:+ """Name the fields `scope` does have, for an unresolved field with no close match.++ Without the list a caller can only guess again. The property hint is worded as a condition because+ a mistyped column read as `properties.x` returns NULL, not an error.+ """+ names = collect_selectable_field_names(scope, context)+ if not names:+ return ""+ listed = ", ".join(names[:MAX_LISTED_FIELD_NAMES])+ remaining = len(names) - MAX_LISTED_FIELD_NAMES+ more = f", and {remaining} more" if remaining > 0 else ""+ name = str(node.chain[0])+ property_hint = (+ f" If {name} is a custom property, read it as properties.{quote_hogql_identifier(name)}."+ if len(node.chain) == 1 and _has_one_json_properties_field(scope, context)+ else ""+ )+ join_hint = (+ " If a field name exists in multiple tables, qualify it with the table name or alias."+ if isinstance(scope, ast.SelectQueryType) and len(scope.tables) + len(scope.anonymous_tables) > 1+ else ""+ )+ return f". Available fields: {listed}{more}.{property_hint}{join_hint}"def suggest_field_names(diff --git a/posthog/hogql/test/test_metadata.py b/posthog/hogql/test/test_metadata.pyindex c24db491ed46..72b96860b895 100644--- a/posthog/hogql/test/test_metadata.py+++ b/posthog/hogql/test/test_metadata.py@@ -294,6 +294,8 @@ class TestMetadata(ClickhouseTestMixin, APIBaseTest):def test_metadata_expr_resolve_error(self):metadata = self._expr("1 + no_field")+ message = metadata.errors[0].message+ self.assertTrue(message.startswith("Unable to resolve field: no_field. Available fields: "), message)self.assertEqual(metadata.dict(),metadata.dict()@@ -302,7 +304,7 @@ class TestMetadata(ClickhouseTestMixin, APIBaseTest):"query": "1 + no_field","errors": [{- "message": "Unable to resolve field: no_field",+ "message": message,"start": 4,"end": 12,"fix": None,diff --git a/posthog/hogql/test/test_resolver.py b/posthog/hogql/test/test_resolver.pyindex 9d7300b18e7d..7a312d168aa1 100644--- a/posthog/hogql/test/test_resolver.py+++ b/posthog/hogql/test/test_resolver.py@@ -450,7 +450,9 @@ class TestResolver(BaseTest):)with self.assertRaises(QueryError) as e:expr = cast(ast.SelectQuery, resolve_types(expr, self.context, dialect="clickhouse"))- self.assertEqual(str(e.exception), "Unable to resolve field: e")+ message = str(e.exception)+ self.assertTrue(message.startswith("Unable to resolve field: e. Available fields: "), message)+ self.assertNotIn("custom property", message)@pytest.mark.usefixtures("unittest_snapshot")def test_resolve_constant_type(self):@@ -527,6 +529,87 @@ class TestResolver(BaseTest):f"expected a plausible suggestion in: {message}",)+ @parameterized.expand(+ [+ (+ "table_columns",+ "SELECT group_type_index FROM groups",+ {"index", "key", "properties"},+ {"team_id", "revenue_analytics"},+ " If group_type_index is a custom property, read it as properties.group_type_index.",+ ),+ (+ "quoted_property_name",+ "SELECT `checkout plan` FROM events",+ {"event", "timestamp", "properties"},+ {"team_id", "pdi", "poe", "goe_0", "override"},+ " If checkout plan is a custom property, read it as properties.`checkout plan`.",+ ),+ ("aliased_subquery", "SELECT x FROM (SELECT 1 AS a, 2 AS b) AS t", {"a", "b"}, set(), None),+ ("non_json_properties", "SELECT zz FROM (SELECT 1 AS properties) AS t", {"properties"}, set(), None),+ (+ "ambiguous_properties",+ "SELECT zz FROM events e JOIN persons p ON e.person_id = p.id",+ {"properties", "event"},+ set(),+ " If a field name exists in multiple tables, qualify it with the table name or alias.",+ ),+ (+ "column_aliased_table",+ "SELECT zz FROM events AS e(a, b)",+ {"a", "b", "properties"},+ {"uuid", "event"},+ " If zz is a custom property, read it as properties.zz.",+ ),+ ]+ )+ def test_unresolved_field_without_close_match_lists_available_fields(+ self, _name: str, query: str, listed: set[str], not_listed: set[str], expected_hint: str | None+ ) -> None:+ with self.assertRaises(QueryError) as ctx:+ resolve_types(self._select(query), self.context, dialect="clickhouse")++ message = str(ctx.exception)+ fields_part, _, hint = message.partition("Available fields: ")[2].partition(".")+ fields = set(fields_part.split(", "))+ self.assertLessEqual(listed, fields, message)+ self.assertFalse(not_listed & fields, message)+ self.assertEqual(hint, expected_hint or "", message)++ @parameterized.expand(+ [+ (+ "aliases_first",+ "SELECT beta AS picked, zz FROM (SELECT 1 AS gamma, 2 AS beta, 3 AS alpha) AS t",+ "picked, gamma, beta, alpha.",+ ),+ (+ "joined_tables",+ "SELECT zz FROM (SELECT 1 AS shared, 2 AS left_only) AS l "+ "JOIN (SELECT 1 AS shared, 2 AS right_only) AS r ON l.shared = r.shared",+ "shared, left_only, right_only."+ " If a field name exists in multiple tables, qualify it with the table name or alias.",+ ),+ ]+ )+ def test_unresolved_field_lists_columns_in_order(self, _name: str, query: str, expected: str) -> None:+ with self.assertRaises(QueryError) as ctx:+ resolve_types(self._select(query), self.context, dialect="clickhouse")++ self.assertEqual(str(ctx.exception), f"Unable to resolve field: zz. Available fields: {expected}")++ @parameterized.expand([(30, ""), (31, ", and 1 more")])+ def test_unresolved_field_caps_available_columns(self, column_count: int, expected_more: str) -> None:+ columns = [f"column_{index:02}" for index in range(column_count, 0, -1)]+ select = ", ".join(f"1 AS {column}" for column in columns)+ with self.assertRaises(QueryError) as ctx:+ resolve_types(self._select(f"SELECT zz FROM (SELECT {select}) AS t"), self.context, dialect="clickhouse")++ expected_fields = ", ".join(columns[:30])+ self.assertEqual(+ str(ctx.exception), f"Unable to resolve field: zz. Available fields: {expected_fields}{expected_more}."+ )+@pytest.mark.usefixtures("unittest_snapshot")def test_resolve_lazy_pdi_person_table(self):expr = self._select("select distinct_id, person.id from person_distinct_ids")diff --git a/products/batch_exports/backend/tests/api/test_create.py b/products/batch_exports/backend/tests/api/test_create.pyindex c6f2c4fda877..fb72cb242a4b 100644--- a/products/batch_exports/backend/tests/api/test_create.py+++ b/products/batch_exports/backend/tests/api/test_create.py@@ -1,3 +1,4 @@+import reimport jsonimport typing as timport datetime as dt@@ -554,7 +555,10 @@ def test_create_batch_export_with_custom_schema([("SELECT", "Failed to parse query"),("SELECT event,, FROM events", "Failed to parse query"),- ("SELECT unknown_field FROM events", "Invalid HogQL query: Unable to resolve field: unknown_field"),+ (+ "SELECT unknown_field FROM events",+ re.compile(r"Invalid HogQL query: Unable to resolve field: unknown_field\. Available fields: .+"),+ ),("SELECT event, persons.id FROM events LEFT JOIN persons ON events.person_id = persons.id","JOINs are not supported",@@ -612,7 +616,11 @@ def test_create_batch_export_fails_with_invalid_query()assert response.status_code == status.HTTP_400_BAD_REQUEST, response.json()- assert response.json()["detail"] == expected_error_message+ detail = response.json()["detail"]+ if isinstance(expected_error_message, re.Pattern):+ assert expected_error_message.fullmatch(detail), detail+ else:+ assert detail == expected_error_message@pytest.mark.usefixtures("hogql_batch_exports_enabled")diff --git a/products/data_warehouse/backend/tests/api/test_view_link.py b/products/data_warehouse/backend/tests/api/test_view_link.pyindex d696ae5ee1a0..99a560971a99 100644--- a/products/data_warehouse/backend/tests/api/test_view_link.py+++ b/products/data_warehouse/backend/tests/api/test_view_link.py@@ -650,7 +650,9 @@ class TestViewLinkValidation(APIBaseTest):data = response.json()self.assertEqual(data["attr"], None)self.assertEqual(data["code"], "QueryError")- self.assertEqual(data["detail"], "Unable to resolve field: nonexistent_field")+ self.assertTrue(+ data["detail"].startswith("Unable to resolve field: nonexistent_field. Available fields: "), data["detail"]+ )self.assertEqual(data["type"], "query_error")self.assertEqual(data["hogql"],