Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(hogql): list selectable fields when a field does not resolve (#119347)

Vasco de Krijger210539cafbb4
diff --git a/posthog/hogql/resolver.py b/posthog/hogql/resolver.py
index 234084074c68..50bae2a91745 100644
--- a/posthog/hogql/resolver.py
+++ b/posthog/hogql/resolver.py
@@ -55,6 +55,7 @@ from posthog.hogql.functions.with_icon import with_icon
from posthog.hogql.hogqlx import HOGQLX_COMPONENTS, HOGQLX_TAGS, convert_to_hx
from posthog.hogql.parser import parse_select
from posthog.hogql.resolver_utils import (
+ available_fields_suffix,
expand_hogqlx_query,
lookup_field_by_name,
lookup_table_by_name,
@@ -2517,7 +2518,11 @@ class Resolver(CloningVisitor):
return ast.Constant(value=value, type=global_type)
suggestions = suggest_field_names(scope, name, self.context)
- suggestion_suffix = f". Did you mean: {', '.join(suggestions)}?" if suggestions else ""
+ suggestion_suffix = (
+ f". Did you mean: {', '.join(suggestions)}?"
+ if suggestions
+ else available_fields_suffix(scope, node, self.context)
+ )
# The message lists every close match, but a quick fix can only substitute one, so it
# offers the best of them. `get_close_matches` returns them in descending similarity.
fix = suggested_field_fix(node, suggestions[0]) if suggestions else None
diff --git a/posthog/hogql/resolver_utils.py b/posthog/hogql/resolver_utils.py
index a871c34080c1..951c694b5464 100644
--- a/posthog/hogql/resolver_utils.py
+++ b/posthog/hogql/resolver_utils.py
@@ -1,7 +1,7 @@
from __future__ import annotations
import difflib
-from collections.abc import Generator
+from collections.abc import Callable, Generator, Iterable
from typing import Optional, cast
from pydantic import BaseModel
@@ -27,7 +27,10 @@ from posthog.hogql.database.models import (
UUIDDatabaseField,
)
from posthog.hogql.errors import QueryError, ResolutionError, SyntaxError
-from posthog.hogql.escape_sql import escape_hogql_identifier
+from posthog.hogql.escape_sql import escape_hogql_identifier, quote_hogql_identifier
+
+# Bounds the message for wide warehouse tables. A longer list stops helping the reader pick a column.
+MAX_LISTED_FIELD_NAMES = 30
def lookup_field_by_name(
@@ -111,44 +114,89 @@ def _select_set_type_source_name(index: int) -> str:
return f"query source {index}"
-def _names_on_table_type(table_type: ast.Type, context: HogQLContext) -> set[str]:
+def _names_on_table_type(
+ table_type: ast.Type, context: HogQLContext, table_names: Callable[[Table], Iterable[str]]
+) -> list[str]:
+ if isinstance(table_type, ast.ColumnAliasedTableType):
+ return list(table_type.alias_to_original)
+ if isinstance(table_type, ast.SelectQueryAliasType):
+ return _names_on_table_type(table_type.select_query_type, context, table_names)
if isinstance(table_type, ast.BaseTableType):
try:
- table = table_type.resolve_database_table(context)
+ return list(table_names(table_type.resolve_database_table(context)))
except Exception:
- return set()
- return set(table.fields.keys())
+ return []
if isinstance(table_type, (ast.SelectQueryType, ast.SelectSetQueryType)):
- return set(getattr(table_type, "columns", {}).keys())
- return set()
+ return list(getattr(table_type, "columns", {}).keys())
+ return []
+
+
+def _collect_field_names(
+ scope: ast.SelectQueryType | ast.SelectSetQueryType,
+ context: HogQLContext,
+ table_names: Callable[[Table], Iterable[str]],
+) -> list[str]:
+ if isinstance(scope, ast.SelectSetQueryType):
+ names = [name for inner in scope.types for name in _collect_field_names(inner, context, table_names)]
+ else:
+ names = list(scope.aliases.keys())
+ for table_type in [*scope.tables.values(), *scope.anonymous_tables]:
+ names.extend(_names_on_table_type(table_type, context, table_names))
+ if scope.parent is not None:
+ names.extend(_collect_field_names(scope.parent, context, table_names))
+ return list(dict.fromkeys(names))
def collect_available_field_names(
scope: ast.SelectQueryType | ast.SelectSetQueryType, context: HogQLContext
) -> set[str]:
- """Gather field names visible in a scope (aliases + joined-table fields)
- for use in 'did you mean' suggestions when resolution fails.
+ """Every field name visible in a scope, for 'did you mean' suggestions when resolution fails."""
+ return set(_collect_field_names(scope, context, lambda table: table.fields.keys()))
- Returns a flat set — callers don't need to know whether a name comes from
- an alias, a named table, or an anonymous subquery.
- """
- names: set[str] = set()
- if isinstance(scope, ast.SelectSetQueryType):
- for inner in scope.types:
- names.update(collect_available_field_names(inner, context))
- return names
+def collect_selectable_field_names(
+ scope: ast.SelectQueryType | ast.SelectSetQueryType, context: HogQLContext
+) -> list[str]:
+ """Names a query in `scope` can select, in declaration order: aliases first, then the columns
+ `SELECT *` would return, including explicitly joined tables. Implicit join fields,
+ internal sub-tables and hidden fields are left out."""
+ return _collect_field_names(scope, context, lambda table: table.get_asterisk().keys())
- names.update(scope.aliases.keys())
- for table_type in scope.tables.values():
- names.update(_names_on_table_type(table_type, context))
- for anon in scope.anonymous_tables:
- names.update(_names_on_table_type(anon, context))
- if scope.parent is not None:
- names.update(collect_available_field_names(scope.parent, context))
+def _has_one_json_properties_field(scope: ast.SelectQueryType | ast.SelectSetQueryType, context: HogQLContext) -> bool:
+ try:
+ field = lookup_field_by_name(scope, "properties", context)
+ return field is not None and isinstance(field.resolve_constant_type(context), ast.StringJSONType)
+ except Exception:
+ return False
+
- return names
+def available_fields_suffix(
+ scope: ast.SelectQueryType | ast.SelectSetQueryType, node: ast.Field, context: HogQLContext
+) -> str:
+ """Name the fields `scope` does have, for an unresolved field with no close match.
+
+ Without the list a caller can only guess again. The property hint is worded as a condition because
+ a mistyped column read as `properties.x` returns NULL, not an error.
+ """
+ names = collect_selectable_field_names(scope, context)
+ if not names:
+ return ""
+ listed = ", ".join(names[:MAX_LISTED_FIELD_NAMES])
+ remaining = len(names) - MAX_LISTED_FIELD_NAMES
+ more = f", and {remaining} more" if remaining > 0 else ""
+ name = str(node.chain[0])
+ property_hint = (
+ f" If {name} is a custom property, read it as properties.{quote_hogql_identifier(name)}."
+ if len(node.chain) == 1 and _has_one_json_properties_field(scope, context)
+ else ""
+ )
+ join_hint = (
+ " If a field name exists in multiple tables, qualify it with the table name or alias."
+ if isinstance(scope, ast.SelectQueryType) and len(scope.tables) + len(scope.anonymous_tables) > 1
+ else ""
+ )
+ return f". Available fields: {listed}{more}.{property_hint}{join_hint}"
def suggest_field_names(
diff --git a/posthog/hogql/test/test_metadata.py b/posthog/hogql/test/test_metadata.py
index c24db491ed46..72b96860b895 100644
--- a/posthog/hogql/test/test_metadata.py
+++ b/posthog/hogql/test/test_metadata.py
@@ -294,6 +294,8 @@ class TestMetadata(ClickhouseTestMixin, APIBaseTest):
def test_metadata_expr_resolve_error(self):
metadata = self._expr("1 + no_field")
+ message = metadata.errors[0].message
+ self.assertTrue(message.startswith("Unable to resolve field: no_field. Available fields: "), message)
self.assertEqual(
metadata.dict(),
metadata.dict()
@@ -302,7 +304,7 @@ class TestMetadata(ClickhouseTestMixin, APIBaseTest):
"query": "1 + no_field",
"errors": [
{
- "message": "Unable to resolve field: no_field",
+ "message": message,
"start": 4,
"end": 12,
"fix": None,
diff --git a/posthog/hogql/test/test_resolver.py b/posthog/hogql/test/test_resolver.py
index 9d7300b18e7d..7a312d168aa1 100644
--- a/posthog/hogql/test/test_resolver.py
+++ b/posthog/hogql/test/test_resolver.py
@@ -450,7 +450,9 @@ class TestResolver(BaseTest):
)
with self.assertRaises(QueryError) as e:
expr = cast(ast.SelectQuery, resolve_types(expr, self.context, dialect="clickhouse"))
- self.assertEqual(str(e.exception), "Unable to resolve field: e")
+ message = str(e.exception)
+ self.assertTrue(message.startswith("Unable to resolve field: e. Available fields: "), message)
+ self.assertNotIn("custom property", message)
@pytest.mark.usefixtures("unittest_snapshot")
def test_resolve_constant_type(self):
@@ -527,6 +529,87 @@ class TestResolver(BaseTest):
f"expected a plausible suggestion in: {message}",
)
+ @parameterized.expand(
+ [
+ (
+ "table_columns",
+ "SELECT group_type_index FROM groups",
+ {"index", "key", "properties"},
+ {"team_id", "revenue_analytics"},
+ " If group_type_index is a custom property, read it as properties.group_type_index.",
+ ),
+ (
+ "quoted_property_name",
+ "SELECT `checkout plan` FROM events",
+ {"event", "timestamp", "properties"},
+ {"team_id", "pdi", "poe", "goe_0", "override"},
+ " If checkout plan is a custom property, read it as properties.`checkout plan`.",
+ ),
+ ("aliased_subquery", "SELECT x FROM (SELECT 1 AS a, 2 AS b) AS t", {"a", "b"}, set(), None),
+ ("non_json_properties", "SELECT zz FROM (SELECT 1 AS properties) AS t", {"properties"}, set(), None),
+ (
+ "ambiguous_properties",
+ "SELECT zz FROM events e JOIN persons p ON e.person_id = p.id",
+ {"properties", "event"},
+ set(),
+ " If a field name exists in multiple tables, qualify it with the table name or alias.",
+ ),
+ (
+ "column_aliased_table",
+ "SELECT zz FROM events AS e(a, b)",
+ {"a", "b", "properties"},
+ {"uuid", "event"},
+ " If zz is a custom property, read it as properties.zz.",
+ ),
+ ]
+ )
+ def test_unresolved_field_without_close_match_lists_available_fields(
+ self, _name: str, query: str, listed: set[str], not_listed: set[str], expected_hint: str | None
+ ) -> None:
+ with self.assertRaises(QueryError) as ctx:
+ resolve_types(self._select(query), self.context, dialect="clickhouse")
+
+ message = str(ctx.exception)
+ fields_part, _, hint = message.partition("Available fields: ")[2].partition(".")
+ fields = set(fields_part.split(", "))
+ self.assertLessEqual(listed, fields, message)
+ self.assertFalse(not_listed & fields, message)
+ self.assertEqual(hint, expected_hint or "", message)
+
+ @parameterized.expand(
+ [
+ (
+ "aliases_first",
+ "SELECT beta AS picked, zz FROM (SELECT 1 AS gamma, 2 AS beta, 3 AS alpha) AS t",
+ "picked, gamma, beta, alpha.",
+ ),
+ (
+ "joined_tables",
+ "SELECT zz FROM (SELECT 1 AS shared, 2 AS left_only) AS l "
+ "JOIN (SELECT 1 AS shared, 2 AS right_only) AS r ON l.shared = r.shared",
+ "shared, left_only, right_only."
+ " If a field name exists in multiple tables, qualify it with the table name or alias.",
+ ),
+ ]
+ )
+ def test_unresolved_field_lists_columns_in_order(self, _name: str, query: str, expected: str) -> None:
+ with self.assertRaises(QueryError) as ctx:
+ resolve_types(self._select(query), self.context, dialect="clickhouse")
+
+ self.assertEqual(str(ctx.exception), f"Unable to resolve field: zz. Available fields: {expected}")
+
+ @parameterized.expand([(30, ""), (31, ", and 1 more")])
+ def test_unresolved_field_caps_available_columns(self, column_count: int, expected_more: str) -> None:
+ columns = [f"column_{index:02}" for index in range(column_count, 0, -1)]
+ select = ", ".join(f"1 AS {column}" for column in columns)
+ with self.assertRaises(QueryError) as ctx:
+ resolve_types(self._select(f"SELECT zz FROM (SELECT {select}) AS t"), self.context, dialect="clickhouse")
+
+ expected_fields = ", ".join(columns[:30])
+ self.assertEqual(
+ str(ctx.exception), f"Unable to resolve field: zz. Available fields: {expected_fields}{expected_more}."
+ )
+
@pytest.mark.usefixtures("unittest_snapshot")
def test_resolve_lazy_pdi_person_table(self):
expr = self._select("select distinct_id, person.id from person_distinct_ids")
diff --git a/products/batch_exports/backend/tests/api/test_create.py b/products/batch_exports/backend/tests/api/test_create.py
index c6f2c4fda877..fb72cb242a4b 100644
--- a/products/batch_exports/backend/tests/api/test_create.py
+++ b/products/batch_exports/backend/tests/api/test_create.py
@@ -1,3 +1,4 @@
+import re
import json
import typing as t
import datetime as dt
@@ -554,7 +555,10 @@ def test_create_batch_export_with_custom_schema(
[
("SELECT", "Failed to parse query"),
("SELECT event,, FROM events", "Failed to parse query"),
- ("SELECT unknown_field FROM events", "Invalid HogQL query: Unable to resolve field: unknown_field"),
+ (
+ "SELECT unknown_field FROM events",
+ re.compile(r"Invalid HogQL query: Unable to resolve field: unknown_field\. Available fields: .+"),
+ ),
(
"SELECT event, persons.id FROM events LEFT JOIN persons ON events.person_id = persons.id",
"JOINs are not supported",
@@ -612,7 +616,11 @@ def test_create_batch_export_fails_with_invalid_query(
)
assert response.status_code == status.HTTP_400_BAD_REQUEST, response.json()
- assert response.json()["detail"] == expected_error_message
+ detail = response.json()["detail"]
+ if isinstance(expected_error_message, re.Pattern):
+ assert expected_error_message.fullmatch(detail), detail
+ else:
+ assert detail == expected_error_message
@pytest.mark.usefixtures("hogql_batch_exports_enabled")
diff --git a/products/data_warehouse/backend/tests/api/test_view_link.py b/products/data_warehouse/backend/tests/api/test_view_link.py
index d696ae5ee1a0..99a560971a99 100644
--- a/products/data_warehouse/backend/tests/api/test_view_link.py
+++ b/products/data_warehouse/backend/tests/api/test_view_link.py
@@ -650,7 +650,9 @@ class TestViewLinkValidation(APIBaseTest):
data = response.json()
self.assertEqual(data["attr"], None)
self.assertEqual(data["code"], "QueryError")
- self.assertEqual(data["detail"], "Unable to resolve field: nonexistent_field")
+ self.assertTrue(
+ data["detail"].startswith("Unable to resolve field: nonexistent_field. Available fields: "), data["detail"]
+ )
self.assertEqual(data["type"], "query_error")
self.assertEqual(
data["hogql"],