:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
"""Request payload validation for endpoint create/update.
These functions validate (and normalize) the ``EndpointRequest`` payload before
it reaches the CRUD service: name format, data freshness buckets, HogQL
parseability, and variable placeholder <-> definition consistency.
"""
import re
import uuid
from typing import Optional
from django.conf import settings
from pydantic import BaseModel
from rest_framework.exceptions import ValidationError
from posthog.schema import EndpointRequest, HogQLQuery, HogQLVariable
from posthog.hogql import ast
from posthog.hogql.context import HogQLContext
from posthog.hogql.errors import ExposedHogQLError, ResolutionError
from posthog.hogql.parser import parse_select
from posthog.hogql.printer import prepare_ast_for_printing
from posthog.hogql.variables import replace_variables
from posthog.exceptions_capture import capture_exception
from posthog.models import Team, User
from products.endpoints.backend.constants import (
DATA_FRESHNESS_BUCKET_SPECS,
ENDPOINT_NAME_REGEX,
VALID_DATA_FRESHNESS_SECONDS,
)
from products.endpoints.backend.logic.strategies import BREAKDOWN_SUPPORTED_QUERY_TYPES
from products.endpoints.backend.materialization_transforms import SUPPORTED_BUCKET_FUNCTIONS, VariablePlaceholderFinder
from products.endpoints.backend.models import (
Endpoint,
EndpointVersion,
_breakdown_property_names,
can_materialize_query,
)
from products.product_analytics.backend.facade.api import insight_variables_by_code_names, insight_variables_by_ids
def validate_data_freshness(data_freshness_seconds: int | None) -> None:
"""Validate data_freshness_seconds is one of the allowed bucket values."""
if data_freshness_seconds is None:
return
if data_freshness_seconds not in VALID_DATA_FRESHNESS_SECONDS:
allowed = sorted(VALID_DATA_FRESHNESS_SECONDS)
human = ", ".join(b.human for b in DATA_FRESHNESS_BUCKET_SPECS)
raise ValidationError(
{"data_freshness_seconds": f"Data freshness must be one of: {allowed} seconds ({human})."}
)
def _is_current_data_freshness(
data_freshness_seconds: int | None, endpoint: Endpoint | None, version_number: int | None
) -> bool:
"""True when the value equals what the API already reports for the version being updated."""
if data_freshness_seconds is None or endpoint is None:
return False
try:
return endpoint.get_version(version_number).data_freshness_seconds == data_freshness_seconds
except EndpointVersion.DoesNotExist:
return False
def validate_bucket_overrides(bucket_overrides: dict[str, str] | None) -> None:
"""Raise ValidationError if any bucket override value is not in SUPPORTED_BUCKET_FUNCTIONS."""
if not bucket_overrides:
return
invalid = {k: v for k, v in bucket_overrides.items() if v not in SUPPORTED_BUCKET_FUNCTIONS}
if invalid:
valid_options = list(SUPPORTED_BUCKET_FUNCTIONS.keys())
raise ValidationError(f"Invalid bucket override values: {invalid}. Valid options: {valid_options}")
DIRECT_CONNECTION_UNSUPPORTED = (
"Endpoints can't query a direct connection. "
"Switch the connection to PostHog (ClickHouse) and query a synced table instead."
)
def validate_hogql_query(query: HogQLQuery, team: Team, user: User) -> None:
"""Validate that a HogQL query parses, its variables are valid, and the author can
access every table/view it references."""
# A direct connection reads the customer's database live, so there is nothing for an
# endpoint to materialize and its table names only exist while that connection is selected.
if query.connectionId:
raise ValidationError({"query": DIRECT_CONNECTION_UNSUPPORTED})
try:
ast_node = parse_select(query.query)
except ExposedHogQLError as e:
raise ValidationError({"query": f"Invalid HogQL query: {e}"}) from e
except ResolutionError as e:
capture_exception(e)
raise ValidationError({"query": "Invalid HogQL query: unable to resolve table or field references."})
except Exception as e:
capture_exception(e)
raise ValidationError({"query": "Unknown error occurred parsing the query."})
validate_variable_placeholders(ast_node, query.variables or {}, team)
_validate_query_access(ast_node, query, team, user)
def _validate_query_access(
ast_node: ast.SelectQuery | ast.SelectSetQuery,
query: HogQLQuery,
team: Team,
user: User,
) -> None:
"""Resolve the query under the author's access control; raise if it hits a denied table/view."""
resolvable_ast: ast.Expr = ast_node
if query.variables:
# Substitute {variables.x} so the AST resolves (endpoints allow variables; views don't).
# Missing definitions were already caught by validate_variable_placeholders above.
resolvable_ast = replace_variables(ast_node, list(query.variables.values()), team)
context = HogQLContext(team_id=team.pk, user=user, enable_select_queries=True)
try:
# Using prepare_ast_for_printing instead of prepare_and_print_ast
# because table/view access is enforced during resolution
prepare_ast_for_printing(node=resolvable_ast, context=context, dialect="clickhouse")
except ExposedHogQLError as err:
# Surfaces "You don't have access to table `X`." (QueryError) and other resolver errors.
raise ValidationError({"query": f"Invalid HogQL query: {err}"}) from err
except Exception as err:
capture_exception(err)
if not settings.DEBUG:
raise ValidationError({"query": f"Unexpected {err.__class__.__name__}"})
raise
def sync_hogql_query_variables(query: HogQLQuery, team: Team) -> None:
"""Sync query variable definitions with the placeholders used in the query string."""
try:
ast_node = parse_select(query.query)
except Exception:
# Validation runs separately and will surface parse errors.
return
finder = VariablePlaceholderFinder()
finder.visit(ast_node)
placeholder_names = {str(p.chain[1]) for p in finder.variable_placeholders if p.chain and len(p.chain) > 1}
if not placeholder_names:
query.variables = None
return
existing_variables = query.variables or {}
existing_variable_ids: list[str] = []
for variable in existing_variables.values():
if not variable.variableId:
continue
try:
uuid.UUID(variable.variableId)
except ValueError:
continue
existing_variable_ids.append(variable.variableId)
team_variables = insight_variables_by_ids(team.pk, existing_variable_ids)
team_variables_by_id = {str(variable.id): variable for variable in team_variables}
synced_variables: dict[str, HogQLVariable] = {}
existing_code_names: set[str] = set()
for variable_id, variable in existing_variables.items():
if variable.code_name and variable.code_name in placeholder_names:
if variable.value is None and variable.isNull is not True:
team_variable = team_variables_by_id.get(variable.variableId)
if team_variable:
variable.value = team_variable.default_value
variable.isNull = team_variable.default_value is None
synced_variables[variable_id] = variable
existing_code_names.add(variable.code_name)
missing_code_names = placeholder_names - existing_code_names
if missing_code_names:
missing_variables = insight_variables_by_code_names(team.pk, missing_code_names)
for definition in missing_variables:
if not definition.code_name:
continue
synced_variables[str(definition.id)] = HogQLVariable(
variableId=str(definition.id),
code_name=definition.code_name,
value=definition.default_value,
isNull=definition.default_value is None,
)
query.variables = synced_variables or None
def validate_variable_placeholders(node: ast.AST, variables: Optional[dict[str, HogQLVariable]], team: Team) -> None:
"""Validate that every {variables.X} placeholder in the query has a matching variable definition."""
finder = VariablePlaceholderFinder()
finder.visit(node)
if not finder.variable_placeholders:
return
placeholder_names = {str(p.chain[1]) for p in finder.variable_placeholders if p.chain and len(p.chain) > 1}
defined_code_names: set[str] = set()
variable_ids: set[str] = set()
if variables:
defined_code_names = {v.code_name for v in variables.values() if v.code_name}
variable_ids = set(variables.keys())
undefined: list[str] = sorted(placeholder_names - defined_code_names)
if undefined:
raise ValidationError(
{
"query": f"Query references undefined variable(s): {', '.join(undefined)}. "
"See https://posthog.com/docs/endpoints/variables for detail."
}
)
if variable_ids:
valid_uuids: set[str] = set()
invalid_uuids: set[str] = set()
invalid_ids: set[str] = set()
for vid in variable_ids:
try:
uuid.UUID(vid)
valid_uuids.add(vid)
except ValueError:
invalid_uuids.add(vid)
if invalid_uuids:
raise ValidationError({"query": f"Variable ID(s) not valid UUIDs: {', '.join(sorted(invalid_uuids))}. "})
if valid_uuids:
existing_ids = {str(variable.id) for variable in insight_variables_by_ids(team.pk, valid_uuids)}
invalid_ids = valid_uuids - existing_ids
if invalid_ids:
raise ValidationError(
{
"query": f"Variable ID(s) not found: {', '.join(sorted(invalid_ids))}. "
"Make sure the variables exist in https://app.posthog.com/data-management/variables."
}
)
def validate_optional_breakdown_properties(
optional_breakdown_properties: list[str] | None,
query: BaseModel | dict | None,
) -> None:
"""Reject optional_breakdown_properties when the query has no breakdownFilter, or when
any name isn't an actual breakdown property in the query.
``query`` is any of ``EndpointRequest.query``'s pydantic models, or a stored version's
query dict — typed by contract so this doesn't chase the growing query-kind union.
"""
if not optional_breakdown_properties:
return
query_dict = query.model_dump() if isinstance(query, BaseModel) else query
query_kind = query_dict.get("kind") if isinstance(query_dict, dict) else None
if query_kind not in BREAKDOWN_SUPPORTED_QUERY_TYPES:
raise ValidationError(
{
"optional_breakdown_properties": (
f"Query kind {query_kind!r} does not support breakdowns. "
f"Supported: {', '.join(sorted(BREAKDOWN_SUPPORTED_QUERY_TYPES))}."
)
}
)
breakdown_filter = (query_dict.get("breakdownFilter") if isinstance(query_dict, dict) else None) or {}
known = set(_breakdown_property_names(breakdown_filter))
unknown = [p for p in optional_breakdown_properties if p not in known]
if unknown:
raise ValidationError(
{
"optional_breakdown_properties": (
f"Unknown breakdown propert{'y' if len(unknown) == 1 else 'ies'}: "
f"{', '.join(repr(p) for p in unknown)}. "
f"Known: {sorted(known) if known else '(none)'}."
)
}
)
def validate_endpoint_request(data: EndpointRequest, team: Team, user: User, strict: bool = True) -> None:
"""Validate a create/update payload. With strict=True, name and query are required."""
query = data.query
if not query and strict:
raise ValidationError({"query": "This field is required."})
name = data.name
if not name:
if name is not None or strict:
raise ValidationError({"name": "This field is required."})
return
if not isinstance(name, str) or not re.fullmatch(ENDPOINT_NAME_REGEX, name):
raise ValidationError(
{
"name": f"Invalid name '{name}'. Must start with a letter, contain only alphanumeric characters, "
"hyphens, or underscores, and be between 1 and 128 characters long."
}
)
if query and isinstance(query, HogQLQuery) and query.query:
sync_hogql_query_variables(query, team)
validate_hogql_query(query, team, user)
validate_data_freshness(data.data_freshness_seconds)
validate_optional_breakdown_properties(data.optional_breakdown_properties, query)
def validate_update_request(
data: EndpointRequest,
team: Team,
user: User,
endpoint: Endpoint | None = None,
version_number: int | None = None,
) -> None:
"""Validate an update payload against the endpoint's resulting state."""
if not _is_current_data_freshness(data.data_freshness_seconds, endpoint, version_number):
validate_data_freshness(data.data_freshness_seconds)
# Determine final states after this request (for validation)
will_be_active = data.is_active if data.is_active is not None else (endpoint.is_active if endpoint else True)
if not will_be_active and data.is_materialized is True:
raise ValidationError({"is_materialized": "Cannot enable materialization on inactive endpoint."})
if data.is_materialized is True:
# Fail fast on queries that can't be materialized. The service re-checks against
# the final version (the authoritative guard); this catches it before any writes.
effective_query = (
data.query.model_dump() if data.query is not None else (endpoint.get_version().query if endpoint else None)
)
if effective_query is not None:
can_materialize, reason = can_materialize_query(effective_query)
if not can_materialize:
raise ValidationError(f"Cannot materialize endpoint. Reason: {reason}")
if data.query and isinstance(data.query, HogQLQuery) and data.query.query:
sync_hogql_query_variables(data.query, team)
validate_hogql_query(data.query, team, user)
# Validate optional_breakdown_properties against the post-update query when one is supplied,
# otherwise against the query of the version the update writes to (targeted or current).
if data.optional_breakdown_properties is not None:
if data.query is not None:
validate_optional_breakdown_properties(data.optional_breakdown_properties, data.query)
elif endpoint is not None:
try:
target_query = endpoint.get_version(version_number).query
except EndpointVersion.DoesNotExist:
target_query = None # a nonexistent targeted version 404s in the update service
if target_query is not None:
validate_optional_breakdown_properties(data.optional_breakdown_properties, target_query)