:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
import re
import json
import logging
import contextlib
from collections.abc import Iterable
from typing import Any, Optional
from django.db import models
from drf_spectacular.utils import extend_schema_field
from rest_framework import serializers
from rest_framework.exceptions import ValidationError
from posthog.hogql import ast
from posthog.hogql.compiler.bytecode import create_bytecode
from posthog.hogql.compiler.javascript import JavaScriptCompiler
from posthog.hogql.context import HogQLContext
from posthog.hogql.parser import parse_program, parse_string_template
from posthog.hogql.visitor import TraversingVisitor
from posthog.helpers.email_utils import email_domain
from posthog.integrations import IntegrationManager
from posthog.models.integration import POSTHOG_CONNECT_KIND, WAREHOUSE_SOURCE_TYPE_CONFIG_KEY
from posthog.models.team.team import Team
from posthog.models.user import User
from products.actions.backend.facade.api import get_actions
from products.cdp.backend.filters import (
DATA_WAREHOUSE_SOURCES,
FILTER_FUNCTIONS,
RUNTIME_CONTRACT,
TEMPLATE_CALLABLES,
TEMPLATE_GLOBALS,
collect_property_cohort_ids,
compile_filters_bytecode,
compile_filters_expr,
filter_action_ids,
filter_cohort_ids,
)
from products.cdp.backend.models.hog_functions.hog_function import (
TYPES_WITH_JAVASCRIPT_SOURCE,
TYPES_WITH_TRANSPILED_FILTERS,
)
from products.cohorts.backend.models.cohort import Cohort, CohortType
from products.cohorts.backend.realtime_state import CohortRealtimeState, has_realtime_state, resolve_realtime_readiness
from common.hogvm.python.stl import STL
from common.hogvm.python.stl.bytecode import BYTECODE_STL
logger = logging.getLogger(__name__)
CORE_SUPPORTED_FUNCTIONS = {"fetch", "postHogCapture"}
MAX_WORKFLOW_EMAIL_SENDERS = 10
# Matches the cap the tasks API puts on a task's repositories.
MAX_TASK_REPOSITORIES = 10
# The mask the UI shows in place of a stored secret. A re-save that did not touch the secret
# sends this back, meaning "keep the stored value". It must never be persisted as a real secret.
MASKED_SECRET_VALUE = "********"
def masked_secret_input_keys(stored_inputs: object) -> list[str]:
"""Input keys whose stored secret is the mask rather than a real credential.
Such an input authenticates against nothing, and the original value is gone, so only the
owner can restore it. The match cannot be a SQL predicate: the storage column is Fernet
encrypted, and Fernet embeds a random IV, so the same plaintext encrypts differently every
write. Callers have to decrypt and inspect.
A row encrypted under a key we no longer hold decrypts to the raw ciphertext string rather
than a dict, because the field swallows the failure, so the shape is checked, not assumed.
"""
if not isinstance(stored_inputs, dict):
return []
return sorted(
key
for key, entry in stored_inputs.items()
if isinstance(entry, dict) and entry.get("value") == MASKED_SECRET_VALUE
)
# Mirrors FROM_OVERRIDE_EMAIL_REGEX in nodejs/src/cdp/services/messaging/email.service.ts, which
# is what the send path enforces after rendering. Keep the two in sync.
FROM_OVERRIDE_EMAIL_REGEX = re.compile(r'^[^\s@"<>,;]+@[^\s@"<>,;]+\.[^\s@"<>,;]+$')
MAX_FROM_OVERRIDE_EMAIL_LENGTH = 254
def _sender_integration_ids(from_value: dict) -> set[int]:
return {
integration_id
for integration_id in [from_value.get("integrationId"), *(from_value.get("integrationIds") or [])]
if isinstance(integration_id, int) and not isinstance(integration_id, bool)
}
def _validate_not_posthog_connection(integration_ids: list[int], context: dict) -> None:
# A PostHog connection acts as the user who created it, so only that user may use it. A function
# runs for the whole team and the runtime never resolves one, so reject it where the author sees why.
get_team = context.get("get_team")
if get_team is None or not integration_ids:
return
# Warehouse source rows must be read too, so the guard below can reject them.
rows = IntegrationManager.any.with_source_rows().get_many(get_team().id, integration_ids).values()
if any(row.kind == POSTHOG_CONNECT_KIND for row in rows):
raise serializers.ValidationError(
{"input": "A PostHog connection can't be used as a function input. Choose a different integration."}
)
# A row that a data warehouse source created holds the source's typed-in credentials (passwords,
# keys). The runtime passes an input's secrets to the function code, so no function may use it.
if any(WAREHOUSE_SOURCE_TYPE_CONFIG_KEY in (row.config or {}) for row in rows):
raise serializers.ValidationError(
{"input": "This integration can't be used as a function input. Choose a different integration."}
)
def _validate_email_sender_override(from_value: dict, context: dict) -> None:
"""Reject a literal custom sender address the send path would refuse.
The runtime only sends from an address on the selected integration's verified domain; an
off-domain override is discarded at send time with a run-log warning the author never sees.
Catching it at save time puts the error in front of the person who can fix it. Only newly
written sender configurations are checked: templated addresses resolve at render time, and a
value already stored on the workflow (live or draft) is grandfathered so legacy placeholder
data (cleaned up by a separate backfill) does not block unrelated edits.
"""
override = (from_value.get("email") or "").strip()
# A brace means a Liquid or Hog template that only resolves at send time.
if not override or "{" in override:
return
existing = context.get("existing_email_from") or []
# Grandfather only a fully unchanged sender configuration: the address AND the selected
# sender set must match a stored variant. Keeping the address while switching senders must
# re-check it against the new sender's domain, or the pair silently falls back at send time.
for stored in existing if isinstance(existing, list) else [existing]:
if not isinstance(stored, dict):
continue
if override == (stored.get("email") or "").strip() and _sender_integration_ids(
from_value
) == _sender_integration_ids(stored):
return
get_team = context.get("get_team")
if get_team is None:
# No request context (internal re-saves, direct construction); the send path still
# enforces the domain.
return
if len(override) > MAX_FROM_OVERRIDE_EMAIL_LENGTH:
raise serializers.ValidationError(
{
"input": "The custom sender address is too long. "
f"Use an address with {MAX_FROM_OVERRIDE_EMAIL_LENGTH} characters or fewer."
}
)
override_domain = email_domain(override.lower())
if override_domain is None or not FROM_OVERRIDE_EMAIL_REGEX.match(override):
raise serializers.ValidationError(
{
"input": f'The custom sender address "{override}" is not a valid email address. '
"Use a single address like [email protected], or a template that resolves to one."
}
)
integration_ids = _sender_integration_ids(from_value)
if not integration_ids:
return
# An empty cached domain means the id resolved to no email integration for this team; the
# save is not blocked on it (there is no domain to compare), matching the uncached behavior.
shared_cache = context.get("email_integration_domain_cache")
domain_cache: dict[int, str] = shared_cache if isinstance(shared_cache, dict) else {}
missing_ids = [integration_id for integration_id in integration_ids if integration_id not in domain_cache]
if missing_ids:
for integration in IntegrationManager.email.credentials.get_many(get_team().id, missing_ids).values():
config = integration.config or {}
domain_cache[integration.id] = (
config.get("domain") or email_domain(config.get("email") or "") or ""
).lower()
for integration_id in missing_ids:
domain_cache.setdefault(integration_id, "")
for integration_id in sorted(integration_ids):
integration_domain = domain_cache.get(integration_id) or ""
if integration_domain and override_domain != integration_domain:
raise serializers.ValidationError(
{
"input": f'The custom sender address "{override}" is not on the verified domain '
f'"{integration_domain}" of the selected sender. Use an address on that domain, '
"or select a different sender."
}
)
PRODUCT_ASYNC_FUNCTIONS: set[str] = set()
def build_html_wrap_design(html: str) -> dict:
"""Build an Unlayer design holding the given html in a single custom HTML block.
Emails authored programmatically (API/MCP) often carry html without a design, which the
visual editor cannot open. Wrapping the html keeps the stored html byte-identical (sends
don't change) while giving the editor a design to load. _meta numbering mirrors what the
Unlayer editor emits so id-addressed design operations keep working. Ids are fixed so the
wrap is deterministic: callers that resend the same html-only value on every save would
otherwise produce a fresh design each time, and the content-equality checks behind
workflow revisions and hog function draft diffing would register a change on every no-op
resave.
"""
return {
"counters": {"u_row": 1, "u_column": 1, "u_content_html": 1},
"schemaVersion": 16,
"body": {
"id": "html-wrap-body",
"headers": [],
"footers": [],
"rows": [
{
"id": "html-wrap-row",
"cells": [1],
"columns": [
{
"id": "html-wrap-column",
"contents": [
{
"id": "html-wrap-content",
"type": "html",
"values": {
"html": html,
"_meta": {"htmlID": "u_content_html_1", "htmlClassNames": "u_content_html"},
},
}
],
"values": {"_meta": {"htmlID": "u_column_1", "htmlClassNames": "u_column"}},
}
],
"values": {"_meta": {"htmlID": "u_row_1", "htmlClassNames": "u_row"}},
}
],
"values": {},
},
}
_UNLAYER_BODY_CLASS = re.compile(r"\bu_body\b")
_UNLAYER_LAYOUT_TAGS = frozenset({"!doctype", "html", "body", "table", "tbody", "tr", "td", "div"})
_TAG_NAME = re.compile(r"</?\s*([a-z!][a-z0-9]*)")
_SKIPPED_ELEMENT_ENDS = {"head": "</head>", "style": "</style>"}
_CONDITIONAL_COMMENT_START = "<!--[if"
_CONDITIONAL_COMMENT_MARKERS = ("<![endif]-->", "<!-->")
# Outlook renders what a conditional comment holds, so the scan skips only its markers.
def _position_after_conditional_marker(html: str, position: int) -> int | None:
if html.startswith(_CONDITIONAL_COMMENT_START, position):
end = html.find("]>", position + len(_CONDITIONAL_COMMENT_START))
return -1 if end == -1 else end + 2
for marker in _CONDITIONAL_COMMENT_MARKERS:
if html.startswith(marker, position):
return position + len(marker)
return None
def _position_after_layout_markup(html: str, position: int) -> int:
after_marker = _position_after_conditional_marker(html, position)
if after_marker is not None:
return after_marker
if html.startswith("<!--", position):
end = html.find("-->", position + 4)
return -1 if end == -1 else end + 3
tag = _TAG_NAME.match(html, position)
end = html.find(">", position)
if tag is None or end == -1:
return -1
closing = _SKIPPED_ELEMENT_ENDS.get(tag.group(1))
if closing is not None:
end = html.find(closing, end)
return -1 if end == -1 else end + len(closing)
return end + 1 if tag.group(1) in _UNLAYER_LAYOUT_TAGS else -1
def _is_only_unlayer_layout(html: str) -> bool:
html = html.lower()
position = 0
while position < len(html):
if html[position] == "<":
position = _position_after_layout_markup(html, position)
if position == -1:
return False
elif html.startswith(" ", position):
position += 6
elif html[position].isspace():
position += 1
else:
return False
return True
def _is_unlayer_render_of(html: str, block_html: str) -> bool:
start = html.find(block_html)
if start == -1:
return False
around_block = html[:start] + html[start + len(block_html) :]
return bool(_UNLAYER_BODY_CLASS.search(around_block)) and _is_only_unlayer_layout(around_block)
def html_wrap_design_needs_refresh(html: str, design: Any) -> bool:
if not design:
return True
try:
wrapped_html = design["body"]["rows"][0]["columns"][0]["contents"][0]["values"]["html"]
except (KeyError, IndexError, TypeError):
return False
return (
isinstance(wrapped_html, str)
and wrapped_html != html
and design == build_html_wrap_design(wrapped_html)
and not _is_unlayer_render_of(html, wrapped_html)
)
def register_supported_function(name: str) -> None:
PRODUCT_ASYNC_FUNCTIONS.add(name)
register_supported_function("postHogGetTicket")
register_supported_function("postHogUpdateTicket")
register_supported_function("postHogSendTicketMessage")
register_supported_function("postHogGetAccount")
register_supported_function("postHogUpdateAccount")
register_supported_function("postHogSetAccountProperties")
# Async functions that put the invocation on a queue the running consumer cannot serve.
#
# The worker's async function registry is global and is not scoped by function type, so any hog
# program that names one of these reaches the real handler. These two handlers set
# `queueParameters` to a bespoke type ('email', 'sendPushNotification') instead of the ordinary
# 'fetch'. Only the messaging consumers process those queues. When a plain destination stages one,
# the cyclotron worker produces to a Kafka topic its cluster does not have. The produce error
# terminates the worker process, and the partition it owned stops draining.
#
# Membership is decided by that failure mode, NOT by "the worker registers it". Most registered
# async functions stage an ordinary 'fetch' (postHogCreateAccount, postHogCreateTask and the
# postHogGet*/postHogUpdate* family), which is the same mechanism CORE_SUPPORTED_FUNCTIONS already
# gives user code, so they are safe here and are deliberately absent. produceToWarehouseWebhooks is
# also absent: it only appends to an in-memory list, and its callers are all
# `warehouse_source_webhook` functions, which HogFunctionSerializer.validate_type refuses anyway.
#
# Before adding a name, check what its handler in nodejs/src/cdp/async-functions/ stages.
RESERVED_ASYNC_FUNCTIONS = {
"sendEmail",
"sendPushNotification",
}
# Globals that the realtime transformer actually populates at runtime.
# Keep in sync with HogTransformerService.createInvocationGlobals
# (nodejs/src/cdp/hog-transformations/hog-transformer.service.ts).
TRANSFORMATION_AVAILABLE_GLOBALS = {"project", "event", "inputs"}
# Globals available to log transformations, which run per log record in the logs
# ingestion pipeline and see a log record instead of an event.
TRANSFORMATION_LOG_AVAILABLE_GLOBALS = {"project", "record", "inputs"}
# Helper functions that the transformer exposes via getTransformationFunctions
# (nodejs/src/cdp/hog-transformations/transformation-functions.ts). These resolve
# via GET_GLOBAL when referenced as a closure rather than called inline.
# postHogCapture is intentionally omitted — it lives in CORE_SUPPORTED_FUNCTIONS.
TRANSFORMATION_RUNTIME_FUNCTIONS = {
"geoipLookup",
"cleanNullValues",
"isKnownBotUserAgent",
"isKnownBotIp",
"parseUserAgent",
}
class InputCollector(TraversingVisitor):
inputs: set[str]
def __init__(self):
super().__init__()
self.inputs = set()
def visit_field(self, node: ast.Field):
super().visit_field(node)
if node.chain[0] == "inputs":
if len(node.chain) > 1:
self.inputs.add(str(node.chain[1]))
class TemplateGlobalsValidator(TraversingVisitor):
"""Reject input templates that reference globals the runtime will not have.
Each function type passes the globals its runtime provides (transformations see `project`,
`event` and `inputs`; everything else the invocation globals). Without this check the bytecode
compiles fine and the failure surfaces only at run time as
"Could not execute bytecode for input field" / "Global variable not found".
"""
invalid_globals: set[str]
# Calls the Node runtime would refuse, as messages: a name it does not have, or the wrong
# argument count for one it does.
invalid_calls: list[str]
def __init__(
self,
available_globals: Optional[set[str]] = None,
runtime_functions: Optional[set[str]] = None,
# The Python tables describe the Python VM. A template that runs in the Node VM passes its
# own callables as runtime_functions and turns these off.
python_stl: bool = True,
):
super().__init__()
self.invalid_globals = set()
self.invalid_calls = []
self._python_stl = python_stl
self._declared: set[str] = set()
self._available_globals = (
available_globals if available_globals is not None else TRANSFORMATION_AVAILABLE_GLOBALS
)
self._runtime_functions = (
runtime_functions if runtime_functions is not None else TRANSFORMATION_RUNTIME_FUNCTIONS
)
def check(self, node: ast.Expr) -> None:
# Lambda parameters and anything a lambda body declares are locals, not globals.
declared = DeclaredNamesCollector()
declared.visit(node)
self._declared = declared.names
self.visit(node)
def visit_field(self, node: ast.Field):
super().visit_field(node)
if not node.chain:
return
root = str(node.chain[0])
if (
root in self._declared
or root in self._available_globals
or root in self._runtime_functions
or root in CORE_SUPPORTED_FUNCTIONS
or root in PRODUCT_ASYNC_FUNCTIONS
or (self._python_stl and (root in STL or root in BYTECODE_STL))
):
return
self.invalid_globals.add(root)
def visit_call(self, node: ast.Call) -> None:
super().visit_call(node)
if self._python_stl or node.name in self._declared:
return
arity = FILTER_FUNCTIONS.get(node.name)
if arity is None:
# The globals check never sees this name, because a call is not a field. Without this the
# template compiles, and the VM refuses the call on every event that reaches it.
# print is left out of the runtime table, but the Node VM has it. The async functions
# (fetch, postHogCapture, product ones) are not exempt: inputs run without them.
if node.name != "print":
self.invalid_calls.append(f"{node.name} is not a function inputs can use")
return
minimum, maximum = arity
count = len(node.args)
if count < minimum:
self.invalid_calls.append(f"{node.name} needs at least {minimum} argument(s), got {count}")
elif maximum is not None and count > maximum:
self.invalid_calls.append(f"{node.name} takes at most {maximum} argument(s), got {count}")
def visit_expr_call(self, node: ast.ExprCall) -> None:
super().visit_expr_call(node)
# `person.properties.email.isNotNull()` compiles to a call on the value at that path. Globals
# hold event data, never a function, so the VM refuses the call on every event.
if isinstance(node.expr, ast.Field) and node.expr.chain and str(node.expr.chain[0]) not in self._declared:
path = ".".join(str(part) for part in node.expr.chain)
self.invalid_calls.append(
f"{path} is a value, not a function. Write the function name first, as in isNotNull(person.properties.email)"
)
class DeclaredNamesCollector(TraversingVisitor):
"""Collect every identifier a hog program declares (variables, functions, params,
loop vars, lambda args) so program-level global validation can ignore them.
Deliberately flat rather than scope-precise: a false negative just defers the
error to runtime, while a false positive would reject valid code.
"""
names: set[str]
def __init__(self):
super().__init__()
self.names = set()
def visit_variable_declaration(self, node: ast.VariableDeclaration):
self.names.add(node.name)
super().visit_variable_declaration(node)
def visit_function(self, node: ast.Function):
self.names.add(node.name)
self.names.update(node.params)
super().visit_function(node)
def visit_lambda(self, node: ast.Lambda):
self.names.update(node.args)
super().visit_lambda(node)
def visit_for_in_statement(self, node: ast.ForInStatement):
if node.keyVar:
self.names.add(node.keyVar)
self.names.add(node.valueVar)
super().visit_for_in_statement(node)
class HyphenatedPropertyDetector(TraversingVisitor):
errors: list[str]
def __init__(self):
super().__init__()
self.errors = []
def visit_arithmetic_operation(self, node: ast.ArithmeticOperation):
super().visit_arithmetic_operation(node)
if node.op == ast.ArithmeticOperationOp.Sub:
if (
isinstance(node.left, ast.Field)
and len(node.left.chain) >= 2
and isinstance(node.right, ast.Field)
and len(node.right.chain) == 1
and self._is_hyphenated(node.left, node.right)
):
right_name = str(node.right.chain[0])
left_last = str(node.left.chain[-1])
parent = ".".join(str(c) for c in node.left.chain[:-1])
self.errors.append(
f"Hyphens are not supported in identifiers and are interpreted as "
f"subtraction. Use bracket notation: "
f"{parent}['{left_last}-{right_name}']"
)
@staticmethod
def _is_hyphenated(left: ast.Field, right: ast.Field) -> bool:
"""Check if the subtraction looks like a hyphenated property name (no spaces around the minus)."""
if left.end is not None and right.start is not None:
return right.start - left.end == 1
return True
class ReservedFunctionDetector(TraversingVisitor):
names: set[str]
def __init__(self) -> None:
super().__init__()
self.names = set()
def visit_call(self, node: ast.Call) -> None:
super().visit_call(node)
if node.name in RESERVED_ASYNC_FUNCTIONS:
self.names.add(node.name)
def visit_field(self, node: ast.Field) -> None:
# A bare reference is caught as well as a direct call. `let f := sendEmail` followed by
# `f()` compiles to the same global dispatch, so the name alone is refused.
super().visit_field(node)
if len(node.chain) == 1 and str(node.chain[0]) in RESERVED_ASYNC_FUNCTIONS:
self.names.add(str(node.chain[0]))
def reserved_functions_used(hog: str) -> set[str]:
"""The reserved async functions that the given hog source names.
Source that does not parse gives an empty set. compile_hog reports the parse error with a
better message, so this must not raise before it runs.
"""
try:
program = parse_program(hog)
except Exception:
return set()
detector = ReservedFunctionDetector()
detector.visit(program)
return detector.names
class RecordAliasRewriter(TraversingVisitor):
"""Rewrite `{record.x}` template references to `{event.properties.x}` for data-warehouse-table
sources. The synced row is delivered under `event.properties` at runtime, so `record` is a
friendlier alias users can write in destination/workflow templates instead of `event.properties`.
"""
def visit_field(self, node: ast.Field):
super().visit_field(node)
if node.chain and str(node.chain[0]) == "record":
node.chain = ["event", "properties", *node.chain[1:]]
def collect_inputs(node: ast.Expr) -> set[str]:
input_collector = InputCollector()
input_collector.visit(node)
return input_collector.inputs
def generate_template_bytecode(
obj: Any,
input_collector: set[str],
function_type: Optional[str] = None,
is_dwh_source: bool = False,
validate_globals: bool = True,
) -> Any:
"""
Clones an object, compiling any string values to bytecode templates
"""
if isinstance(obj, dict):
return {
key: generate_template_bytecode(value, input_collector, function_type, is_dwh_source, validate_globals)
for key, value in obj.items()
}
elif isinstance(obj, list):
return [
generate_template_bytecode(item, input_collector, function_type, is_dwh_source, validate_globals)
for item in obj
]
elif isinstance(obj, str):
node = parse_string_template(obj)
if is_dwh_source:
RecordAliasRewriter().visit(node)
input_collector.update(collect_inputs(node))
detector = HyphenatedPropertyDetector()
detector.visit(node)
if detector.errors:
raise Exception(detector.errors[0])
if function_type == "transformation":
transformation_validator = TemplateGlobalsValidator()
transformation_validator.check(node)
if transformation_validator.invalid_globals:
names = ", ".join(sorted(transformation_validator.invalid_globals))
raise Exception(
f"Variable not available in transformations: {names}. "
f"Transformations only have access to project, event, and inputs."
)
elif function_type == "transformation_log":
log_validator = TemplateGlobalsValidator(
available_globals=TRANSFORMATION_LOG_AVAILABLE_GLOBALS,
runtime_functions=set(),
)
log_validator.check(node)
if log_validator.invalid_globals:
names = ", ".join(sorted(log_validator.invalid_globals))
raise Exception(
f"Variable not available in log transformations: {names}. "
f"Log transformations only have access to project, record, and inputs."
)
elif function_type is not None and validate_globals:
# Every other type resolves its inputs against the invocation globals at run time. A save
# that disables or deletes the function skips this, so a broken function can be turned off.
template_validator = TemplateGlobalsValidator(
available_globals=TEMPLATE_GLOBALS, runtime_functions=TEMPLATE_CALLABLES, python_stl=False
)
template_validator.check(node)
if template_validator.invalid_globals:
names = ", ".join(sorted(template_validator.invalid_globals))
raise Exception(
f"Variable not available in inputs: {names}. "
f"Inputs can read event, person, groups, project, source and inputs, and in a workflow "
f"also variables."
)
if template_validator.invalid_calls:
raise Exception(
"This template would fail on every event: " + "; ".join(template_validator.invalid_calls)
)
return create_bytecode(node).bytecode
else:
return obj
def transpile_template_code(obj: Any, compiler: JavaScriptCompiler, is_dwh_source: bool = False) -> str:
"""
Clones an object, compiling any string values to bytecode templates
"""
if isinstance(obj, dict):
return (
"{"
+ (
", ".join(
[
f"{json.dumps(str(key))}: {transpile_template_code(value, compiler, is_dwh_source)}"
for key, value in obj.items()
]
)
)
+ "}"
)
elif isinstance(obj, list):
return "[" + (", ".join([transpile_template_code(item, compiler, is_dwh_source) for item in obj])) + "]"
elif isinstance(obj, str):
node = parse_string_template(obj)
if is_dwh_source:
RecordAliasRewriter().visit(node)
return compiler.visit(node)
else:
return json.dumps(obj)
def _contains_liquid_style_syntax(value: Any) -> bool:
if isinstance(value, str):
return "{{" in value
if isinstance(value, dict):
return any(_contains_liquid_style_syntax(v) for v in value.values())
if isinstance(value, list):
return any(_contains_liquid_style_syntax(v) for v in value)
return False
@extend_schema_field({"oneOf": [{"type": "boolean"}, {"type": "string", "enum": ["hog", "liquid"]}]})
class _TemplatingChoiceField(serializers.ChoiceField):
"""drf-spectacular 0.29 crashes on sorted() with mixed bool/str choice keys."""
pass
class InputsSchemaItemSerializer(serializers.Serializer):
# nosemgrep: choices-need-a-class -- existing input-type list; this change only adds one value to it
type = serializers.ChoiceField(
choices=[
"string",
"number",
"boolean",
"dictionary",
"choice",
"json",
"integration",
"integration_multi",
"integration_field",
"email",
"native_email",
"posthog_assignee",
"posthog_ticket_tags",
"posthog_business_hours",
"non_failure_status_codes",
"customer_analytics_account_properties",
"customer_analytics_account_relationships",
"task_model",
"task_repository",
"task_mcp_installations",
"signals_scout",
"posthog_survey",
"task_skills",
"replay_vision_scan",
]
)
key = serializers.CharField()
label = serializers.CharField(required=False, allow_blank=True) # type: ignore
choices = serializers.ListField(child=serializers.DictField(), required=False)
# For `choice` inputs: render as a searchable select on the frontend.
searchable = serializers.BooleanField(required=False)
required = serializers.BooleanField(default=False) # type: ignore
default = serializers.JSONField(required=False)
secret = serializers.BooleanField(default=False)
hidden = serializers.BooleanField(default=False)
description = serializers.CharField(required=False)
integration = serializers.CharField(required=False)
integration_key = serializers.CharField(required=False)
requires_field = serializers.CharField(required=False)
integration_field = serializers.CharField(required=False)
requiredScopes = serializers.CharField(required=False)
# Indicates if hog templating should be used for this input
templating = _TemplatingChoiceField(choices=[True, False, "hog", "liquid"], required=False)
# TODO Validate choices if type=choice
DUPLICATE_INPUT_KEYS_ERROR = "Each input key must be unique. Remove duplicate keys."
def duplicate_input_keys(schemas: Any) -> set[str]:
seen: set[str] = set()
duplicates: set[str] = set()
for schema in schemas or []:
if not isinstance(schema, dict) or "key" not in schema:
continue
key = str(schema["key"]).strip()
if key in seen:
duplicates.add(key)
seen.add(key)
return duplicates
def added_duplicate_input_keys(schemas: Any, stored_schemas: Any) -> set[str]:
"""Keys the schema repeats more often than the stored schema already did."""
def counts(items: Any) -> dict[str, int]:
result: dict[str, int] = {}
for schema in items or []:
if isinstance(schema, dict) and "key" in schema:
key = str(schema["key"]).strip()
result[key] = result.get(key, 0) + 1
return result
stored = counts(stored_schemas)
return {key for key, count in counts(schemas).items() if count > 1 and count > stored.get(key, 0)}
class InputsSchemaSerializer(serializers.ListField):
"""A function's input schema.
Turn `unique_keys` off wherever the caller does not always send the schema. The hog function
serializer injects the stored schema into every update, so a row saved before this rule would
fail a request that only disables or deletes it. That serializer checks uniqueness itself and
compares against the stored schema.
"""
child = InputsSchemaItemSerializer()
def __init__(self, *args: Any, unique_keys: bool = True, **kwargs: Any) -> None:
self.unique_keys = unique_keys
super().__init__(*args, **kwargs)
def to_internal_value(self, data: Any) -> list[dict[str, Any]]:
schemas = super().to_internal_value(data)
if self.unique_keys and duplicate_input_keys(schemas):
raise serializers.ValidationError(DUPLICATE_INPUT_KEYS_ERROR)
return schemas
@extend_schema_field({})
class AnyInputField(serializers.Field):
def to_internal_value(self, data):
return data
def to_representation(self, value):
return value
class HogFunctionTemplating(models.TextChoices):
HOG = "hog", "hog"
LIQUID = "liquid", "liquid"
def _is_templated_choice(schema: dict, value: Any) -> bool:
"""A choice value that is not one of the declared choices is a Hog template, not a selection.
A sub-template can prefill a choice input with an expression, for example the PagerDuty
`event_action` that sends `resolve` on the issue-resolved event and `trigger` otherwise. The
destination then rejects the uncompiled text, so the expression has to become bytecode. A value
the user picked from the dropdown is left alone, so it keeps compiling to nothing.
"""
if not isinstance(value, str):
return False
return value not in {choice.get("value") for choice in schema.get("choices") or []}
class InputsItemSerializer(serializers.Serializer):
value = AnyInputField(required=False)
templating = serializers.ChoiceField(choices=HogFunctionTemplating.choices, required=False)
bytecode = serializers.ListField(required=False, read_only=True)
bytecode_contract = serializers.CharField(required=False, read_only=True)
order = serializers.IntegerField(required=False, read_only=True)
transpiled = serializers.JSONField(required=False, read_only=True)
def to_representation(self, value):
# We want to override the way this gets rendered as the underlying serializer is a DictField which does weird things
return {k: v for k, v in value.items() if v is not None}
def validate(self, attrs):
schema = self.context["schema"]
function_type = self.context["function_type"]
is_dwh_source = self.context.get("is_dwh_source", False)
value = attrs.get("value")
item_type = schema["type"]
if schema.get("required") and (value is None or value == ""):
raise serializers.ValidationError({"input": f"This field is required."})
if not value:
return attrs
# Validate each type
if item_type == "string":
if not isinstance(value, str):
raise serializers.ValidationError({"input": f"Value must be a string."})
elif item_type == "number":
if not isinstance(value, int | float):
raise serializers.ValidationError({"input": f"Value must be a number."})
elif item_type == "boolean":
templating_enabled = schema.get("templating", True)
if templating_enabled:
if not isinstance(value, bool) and not isinstance(value, str):
raise serializers.ValidationError({"input": f"Value must be a boolean or a template string."})
# Liquid templating always renders to strings, which bypasses boolean type guarantees.
# Only Hog templating is allowed for boolean fields as it preserves the actual boolean type.
if isinstance(value, str) and attrs.get("templating") == "liquid":
raise serializers.ValidationError(
{"input": "Liquid templating is not supported for boolean fields. Use Hog templating instead."}
)
else:
if not isinstance(value, bool):
raise serializers.ValidationError({"input": f"Value must be a boolean."})
elif item_type in (
"dictionary",
"customer_analytics_account_properties",
"customer_analytics_account_relationships",
):
if not isinstance(value, dict):
raise serializers.ValidationError({"input": f"Value must be a dictionary."})
elif item_type == "integration":
if not isinstance(value, int):
raise serializers.ValidationError({"input": f"Value must be an Integration ID."})
_validate_not_posthog_connection([value], self.context)
elif item_type == "integration_multi":
if not isinstance(value, list) or not all(isinstance(v, int) and not isinstance(v, bool) for v in value):
raise serializers.ValidationError({"input": "Value must be a list of Integration IDs."})
_validate_not_posthog_connection(value, self.context)
elif item_type == "task_repository":
repositories = [value] if isinstance(value, str) else value
if (
not isinstance(repositories, list)
or not all(isinstance(v, str) for v in repositories)
or len(repositories) > MAX_TASK_REPOSITORIES
):
raise serializers.ValidationError(
{
"input": f"Value must be a list of up to {MAX_TASK_REPOSITORIES} repository names like your-org/your-repo."
}
)
elif item_type == "task_model":
# A non-empty value means a model was chosen (an empty value returned above as "use the
# default model"), so it must name a usable model. Otherwise the run-time consumer drops
# the setting and the task silently falls back to the default, which this guard exists to
# prevent for programmatically authored workflows.
model = value.get("model") if isinstance(value, dict) else None
reasoning_effort = value.get("reasoning_effort") if isinstance(value, dict) else None
if (
not isinstance(value, dict)
or not isinstance(model, str)
or not model
or (reasoning_effort is not None and not isinstance(reasoning_effort, str))
):
raise serializers.ValidationError(
{
"input": "Value must be an object with a non-empty 'model' string and an optional 'reasoning_effort' string."
}
)
elif item_type == "task_mcp_installations":
if not isinstance(value, list) or not all(isinstance(v, str) for v in value):
raise serializers.ValidationError({"input": "Value must be a list of MCP connector IDs."})
elif item_type == "signals_scout":
if not isinstance(value, str):
raise serializers.ValidationError({"input": "Value must be a scout skill name."})
elif item_type == "posthog_survey":
if not isinstance(value, str):
raise serializers.ValidationError({"input": "Value must be a survey ID."})
elif item_type == "replay_vision_scan":
if not isinstance(value, dict) or not isinstance(value.get("type"), str):
raise serializers.ValidationError({"input": "Choose what Replay Vision should do."})
# The setup is never templated. Compiled code or a Liquid template from the client could name a different
# scanner than the value the access checks read, so only the literal value reaches the engine.
for key in ("bytecode", "transpiled", "templating"):
attrs.pop(key, None)
elif item_type == "task_skills":
if not isinstance(value, list) or not all(isinstance(v, str) for v in value):
raise serializers.ValidationError({"input": "Value must be a list of skill names."})
elif item_type == "email" or item_type == "native_email":
if not isinstance(value, dict):
raise serializers.ValidationError({"input": f"Value must be an email object."})
# Report every missing key in one error: these objects are typically authored
# programmatically, and a one-at-a-time raise forces a round trip per missing key.
missing = [f"'{key_}'" for key_ in ("from", "to", "subject") if not value.get(key_)]
if not value.get("text") and not value.get("html"):
missing.append("either 'text' or 'html'")
if missing:
label = "value" if len(missing) == 1 else "values"
raise serializers.ValidationError({"input": f"Missing {label} for {', '.join(missing)}."})
# Templated sender overrides on the `from` object. Non-string values would only
# surface as a send-time failure in the runtime's schema parse, so reject them here.
from_value = value.get("from")
if isinstance(from_value, dict):
wrong_types = [
f"'from.{key_}'"
for key_ in ("email", "name")
if from_value.get(key_) is not None and not isinstance(from_value[key_], str)
]
if wrong_types:
label = "value" if len(wrong_types) == 1 else "values"
raise serializers.ValidationError(
{"input": f"Expected string {label} for {', '.join(wrong_types)}."}
)
integration_ids = from_value.get("integrationIds")
if integration_ids is not None:
if not isinstance(integration_ids, list):
raise serializers.ValidationError(
{"input": "Expected 'from.integrationIds' to be a list of Integration IDs."}
)
if len(integration_ids) > MAX_WORKFLOW_EMAIL_SENDERS:
raise serializers.ValidationError(
{"input": f"At most {MAX_WORKFLOW_EMAIL_SENDERS} email senders are allowed."}
)
if not all(
isinstance(integration_id, int) and not isinstance(integration_id, bool)
for integration_id in integration_ids
):
raise serializers.ValidationError(
{"input": "Expected 'from.integrationIds' to be a list of Integration IDs."}
)
_validate_email_sender_override(from_value, self.context)
if (
isinstance(value.get("html"), str)
and value["html"]
and html_wrap_design_needs_refresh(value["html"], value.get("design"))
):
# Programmatically authored emails often supply html without a design, which the
# visual editor can't open. Wrap it so every stored email has an editable design.
value = {**value, "design": build_html_wrap_design(value["html"])}
attrs["value"] = value
elif item_type == "non_failure_status_codes":
if not isinstance(value, list):
raise serializers.ValidationError({"input": "Value must be a list of status codes."})
for entry in value:
if isinstance(entry, bool) or not isinstance(entry, int | str):
raise serializers.ValidationError(
{"input": "Entries must be integers between 400 and 599 or wildcards '4xx' or '5xx'."}
)
if isinstance(entry, int):
if not (400 <= entry <= 599):
raise serializers.ValidationError({"input": "Status code numbers must be between 400 and 599."})
else:
if not re.fullmatch(r"[4-5]xx", entry, re.IGNORECASE):
raise serializers.ValidationError({"input": "Wildcards must be '4xx' or '5xx'."})
try:
if value and schema.get("templating", True):
if attrs.get("templating") == "liquid":
# NOTE: We don't do validaton at this level. The frontend will validate for us
# and we don't care about it being invalid at this stage.
pass
else:
# If we have a value and hog templating is enabled, we need to transpile the value
value_is_transpiled = (
item_type
in [
"string",
"boolean",
"dictionary",
"json",
"email",
"native_email",
"posthog_ticket_tags",
"customer_analytics_account_properties",
"customer_analytics_account_relationships",
]
or (item_type == "boolean" and isinstance(value, str))
or (item_type == "choice" and _is_templated_choice(schema, value))
)
if value_is_transpiled:
if item_type in ("email", "native_email") and isinstance(value, dict):
# We want to exclude the "design" property
value = {key: value[key] for key in value if key != "design"}
if function_type in TYPES_WITH_JAVASCRIPT_SOURCE:
compiler = JavaScriptCompiler()
code = transpile_template_code(value, compiler, is_dwh_source=is_dwh_source)
attrs["transpiled"] = {"lang": "ts", "code": code, "stl": list(compiler.stl_functions)}
if "bytecode" in attrs:
del attrs["bytecode"]
attrs.pop("bytecode_contract", None)
else:
input_collector: set[str] = set()
attrs["bytecode"] = generate_template_bytecode(
value,
input_collector,
function_type=function_type,
is_dwh_source=is_dwh_source,
validate_globals=self.context.get("function_will_be_enabled", True),
)
attrs["input_deps"] = list(input_collector)
attrs["bytecode_contract"] = RUNTIME_CONTRACT
if "transpiled" in attrs:
del attrs["transpiled"]
except Exception as e:
# Liquid-style {{ ... }} in a hog-templated field is the dominant authoring mistake
# behind transpile failures, and the compiler's own message ("Placeholders are not
# allowed in this context") never names it - callers bisect blind without this hint.
if _contains_liquid_style_syntax(value):
raise serializers.ValidationError(
{
"input": (
"Invalid template: this field uses single-curly templating like "
"{person.properties.email}. Liquid-style {{ ... }} syntax is not "
f"supported here. ({str(e)})"
)
}
)
raise serializers.ValidationError({"input": f"Invalid template: {str(e)}"})
return attrs
def exposes_secret_dictionary_keys(schema: dict, schemas: list) -> bool:
"""Only a `secret_<key>` dictionary paired with a plain `<key>` dictionary holds locked rows the editor lists by key."""
key = schema.get("key")
if schema.get("type") != "dictionary" or not isinstance(key, str) or not key.startswith("secret_"):
return False
plain_key = key.removeprefix("secret_")
return any(
isinstance(s, dict) and s.get("key") == plain_key and s.get("type") == "dictionary" and not s.get("secret")
for s in schemas
)
def masked_dictionary_secret(stored: Any) -> dict:
"""Mask a secret dictionary input entry by entry, so the editor can still list its keys."""
entries = stored.get("value") if isinstance(stored, dict) else None
if isinstance(entries, dict):
return {"secret": True, "value": dict.fromkeys(entries, MASKED_SECRET_VALUE)}
return {"secret": True}
def restore_masked_dictionary_entries(value: Any, stored: Any) -> tuple[Any, list[str]]:
"""Put each stored secret back where the editor sent the mask for an entry it did not retype."""
entries = value.get("value") if isinstance(value, dict) else None
if not isinstance(entries, dict) or MASKED_SECRET_VALUE not in entries.values():
return value, []
stored_entries = stored.get("value") if isinstance(stored, dict) else None
stored_entries = stored_entries if isinstance(stored_entries, dict) else {}
missing = [k for k, v in entries.items() if v == MASKED_SECRET_VALUE and k not in stored_entries]
restored = {k: stored_entries.get(k) if v == MASKED_SECRET_VALUE else v for k, v in entries.items()}
stored_metadata = {k: v for k, v in stored.items() if k != "value"} if isinstance(stored, dict) else {}
return {**stored_metadata, **value, "value": restored}, missing
class InputsSerializer(serializers.DictField):
"""
Provides the same typing as the DictField but with custom validation to only include the inputs that are in the schema
"""
child = InputsItemSerializer()
def run_child_validation(self, data):
result = {}
errors: dict[str, Any] = {}
existing_secret_inputs = self.context.get("encrypted_inputs")
# Note this should always be the child of a dict serializer with a sibling 'inputs_schema' field so we can validate against the relevant schema
parent_serializer = self.parent
try:
inputs_schema = parent_serializer.initial_data["inputs_schema"]
except:
raise serializers.ValidationError("Missing inputs_schema.")
assert isinstance(parent_serializer, serializers.Serializer)
inputs_schema = parent_serializer.fields["inputs_schema"].run_validation(inputs_schema)
# Validate each input against the schema
for schema in inputs_schema:
key = str(schema["key"])
value = data.get(key) or {}
if schema.get("secret"):
# A {"secret": true} value the user did not retype is the read-back mask, meaning
# "keep the stored secret". The UI sends it either without a "value" key or with the
# literal mask as the value, so both shapes must count as masked. A different "value"
# is a rotation and must win, so it falls through to normal validation.
is_masked = (
isinstance(value, dict)
and bool(value.get("secret"))
and ("value" not in value or value.get("value") == MASKED_SECRET_VALUE)
)
if not is_masked and schema.get("type") == "dictionary":
value, missing = restore_masked_dictionary_entries(value, (existing_secret_inputs or {}).get(key))
if missing:
errors[key] = f"No value is saved for {', '.join(missing)}. Enter the value again."
continue
if is_masked or value == {}:
existing_value = (existing_secret_inputs or {}).get(key)
if existing_value:
value = existing_value
elif is_masked:
# Nothing stored to keep. Silently dropping the input here has disabled
# webhook auth in production - fail so the caller re-enters the value.
errors[key] = "No value is saved for this secret input. Enter the value again."
continue
if value == {} and schema.get("required") and schema.get("default") is not None:
# The destination editor pre-fills defaults from the template schema, but callers that
# build inputs by hand cannot, so a required input with a default would reject them.
value = {"value": schema["default"]}
self.context["schema"] = schema
# Propagate templating from schema to input item, if set
if "templating" in schema:
templating_val = schema["templating"]
if isinstance(templating_val, bool):
if templating_val:
value["templating"] = "hog"
# If False, do not set templating field
else:
value["templating"] = templating_val
try:
input_value = self.child.run_validation(value)
if "value" not in input_value:
# Indicates no value is provided and no error was thrown which is fine so we can exclude it
continue
if schema.get("secret") and input_value.get("value") == MASKED_SECRET_VALUE:
# The mask reached persistence, so recovery of the stored secret failed. Refuse
# rather than encrypt the mask and silently destroy the real credential.
errors[key] = "This secret input was not updated correctly. Enter the value again."
continue
result[key] = input_value
except ValidationError as e:
# TRICKY: Need to get the nested error message to ensure the structure is correct
if "input" in e.detail and isinstance(e.detail, dict):
errors[key] = e.detail.get("input")
else:
errors[key] = e.detail
if errors:
raise ValidationError(errors)
# We'll topologically sort keys based on their input_deps.
edges = {}
all_keys = list(result.keys())
for k, v in result.items():
deps = v.get("input_deps", [])
deps = [d for d in deps if d in result]
edges[k] = deps
sorted_keys = topological_sort(all_keys, edges)
# Assign order according to topological sort
for i, key in enumerate(sorted_keys):
result[key]["order"] = i
if "input_deps" in result[key]:
del result[key]["input_deps"]
# Rebuild in sorted order
result = {key: result[key] for key in sorted_keys}
return result
# Unlike standard dict validation we are iterating the schema - not the inputs
def _contains_behavioral_property(filters: dict) -> bool:
"""Behavioral ("performed event") property filters compile to a ClickHouse subquery over events
history, which realtime function filters (bytecode per-event, or JS transpiled into the browser)
can never evaluate."""
entities = (filters.get("events") or []) + (filters.get("actions") or []) + (filters.get("data_warehouse") or [])
property_lists = [filters.get("properties") or []] + [
entity.get("properties") or [] for entity in entities if isinstance(entity, dict)
]
return any(
isinstance(prop, dict) and prop.get("type") == "behavioral"
for property_list in property_lists
for prop in property_list
)
def _action_ids_in_filters(filters: dict[str, Any]) -> set[int]:
# Mirrors compile_filters_expr, which compiles every entry typed "actions" in either list.
action_ids: set[int] = set()
for key in ("events", "actions"):
entries = filters.get(key)
for entry in entries if isinstance(entries, list) else []:
if isinstance(entry, dict) and entry.get("type") == "actions":
with contextlib.suppress(KeyError, TypeError, ValueError, OverflowError):
action_id = int(entry["id"])
# Postgres rejects an id outside the bigint range in a query, and no row can match one.
if 0 < action_id < 2**63:
action_ids.add(action_id)
return action_ids
def _filter_action_ids(value: Any) -> set[int]:
if isinstance(value, list):
return set().union(*(_filter_action_ids(item) for item in value))
if not isinstance(value, dict):
return set()
action_ids: set[int] = set()
for key, item in value.items():
if key == "filters" and isinstance(item, dict):
action_ids |= _action_ids_in_filters(item)
elif key != "inputs":
action_ids |= _filter_action_ids(item)
return action_ids
def require_viewer_access_to_added_actions(context: dict[str, Any], submitted: Any, stored: Any) -> None:
user = getattr(context.get("request"), "user", None)
user_access_control = getattr(context.get("view"), "user_access_control", None)
# Synthetic principals (project secret API keys) bypass access control, and callers without a view
# (template installs, plugin migrations) write no references a user picked.
if not isinstance(user, User) or user_access_control is None:
return
added_ids = _filter_action_ids(submitted) - _filter_action_ids(stored)
if not added_ids:
return
actions = get_actions(project_id=context["get_team"]().project_id, action_ids=added_ids)
denied_ids = user_access_control.denied_object_ids(
"action", [(str(action.id), action.team_id, action.created_by_id) for action in actions], "viewer"
)
if denied_ids:
raise serializers.ValidationError(
f"You don't have viewer access to action(s) {', '.join(denied_ids)}. "
"Remove them from the filters, or ask for access."
)
class HogFunctionFiltersSerializer(serializers.Serializer):
source = serializers.ChoiceField(
choices=["events", "internal-events", "person-updates", *DATA_WAREHOUSE_SOURCES],
required=False,
default="events",
) # type: ignore
actions = serializers.ListField(child=serializers.DictField(), required=False)
events = serializers.ListField(child=serializers.DictField(), required=False)
data_warehouse = serializers.ListField(child=serializers.DictField(), required=False)
properties = serializers.ListField(child=serializers.DictField(), required=False)
bytecode = serializers.JSONField(required=False, allow_null=True)
transpiled = serializers.JSONField(required=False)
filter_test_accounts = serializers.BooleanField(required=False)
bytecode_error = serializers.CharField(required=False)
bytecode_contract = serializers.CharField(required=False)
def to_internal_value(self, data):
# Weirdly nested serializers don't get this set...
self.initial_data = data
return super().to_internal_value(data)
def validate(self, data):
function_type = self.context.get("function_type")
team = self.context["get_team"]()
# Ensure data is initialized as an empty dict if it's None
data = data or {}
# The compiler writes the stamp below, so a value a client echoes back is never kept.
data.pop("bytecode_contract", None)
if _contains_behavioral_property(data):
raise serializers.ValidationError(
"Behavioral (performed event) filters can't be evaluated in realtime functions. "
"Use a cohort to filter on past behavior."
)
if function_type == "internal_destination":
data["source"] = "internal-events"
if function_type == "transformation_log":
# Filter bytecode is compiled against event-shaped globals, which log records
# don't have — silently accepting filters would mis-evaluate at ingestion time.
# Log transformations express conditions in Hog code instead.
disallowed = [
key
for key in ("events", "actions", "properties", "data_warehouse", "filter_test_accounts")
if data.get(key)
]
if disallowed:
raise serializers.ValidationError(
f"Filters are not supported for log transformations (got: {', '.join(disallowed)}). "
"Use conditions in the Hog code instead."
)
if data.get("source") == "events":
# Don't allow events or actions for person-updates
data.pop("data_warehouse", None)
if data.get("source") == "internal-events":
disallowed = [key for key in ("actions", "data_warehouse") if data.get(key)]
if disallowed:
raise serializers.ValidationError(
dict.fromkeys(disallowed, "This filter is not supported for internal events.")
)
data.pop("actions", None)
data.pop("data_warehouse", None)
events = data.get("events")
if (
not isinstance(events, list)
or not events
or any(
not isinstance(event, dict) or not isinstance(event.get("id"), str) or not event["id"].strip()
for event in events
)
):
raise serializers.ValidationError(
{"events": "Internal event filters require at least one event with a non-empty id."}
)
if data.get("source") == "person-updates":
# Don't allow events or actions for person-updates
data.pop("events", None)
data.pop("actions", None)
data.pop("data_warehouse", None)
if data.get("source") in DATA_WAREHOUSE_SOURCES:
# Don't allow events or actions for warehouse sources
data.pop("events", None)
data.pop("actions", None)
if "data_warehouse" in data and isinstance(data["data_warehouse"], list):
# A row filter is compiled against its entry's table name, so without one it matches nothing.
# Checked before the placeholder is dropped, or a filter on the placeholder would vanish silently.
if any(
entry.get("properties") and (not entry.get("table_name") or entry.get("name") == "Select a table")
for entry in data["data_warehouse"]
):
raise serializers.ValidationError({"data_warehouse": "Pick a table for each row filter."})
data["data_warehouse"] = [
entry for entry in data["data_warehouse"] if entry.get("name") != "Select a table"
]
# If we have a bytecode, we need to validate the transpiled
if function_type in TYPES_WITH_TRANSPILED_FILTERS:
compiler = JavaScriptCompiler()
code = compiler.visit(compile_filters_expr(data, team))
data["transpiled"] = {"lang": "ts", "code": code, "stl": list(compiler.stl_functions)}
if "bytecode" in data:
del data["bytecode"]
else:
cohort_membership_supported = bool(self.context.get("cohort_membership_supported"))
validated_cohort_ids: Optional[set[int]] = None
if cohort_membership_supported:
validated_cohort_ids = self._validate_realtime_cohorts(
data, team, require_ready=not self.context.get("is_draft")
)
data = compile_filters_bytecode(
data,
team,
cohort_membership_supported=cohort_membership_supported,
allowed_cohort_ids=validated_cohort_ids,
)
# Uncompilable filters are only fatal when the function will run (stay enabled).
# Callers that allow saving anyway (e.g. disabling/deleting a hog function) opt out
# via context; the error stays persisted on the filters for the UI to surface.
if data.get("bytecode_error") and self.context.get("function_will_be_enabled", True):
raise serializers.ValidationError(f"Invalid filter configuration: {data['bytecode_error']}")
return data
def _validate_realtime_cohorts(self, data: dict, team: Team, *, require_ready: bool) -> set[int]:
"""Validate every cohort these filters reference, including inside referenced Actions.
Returns the validated ids so the compiler can reject any IN COHORT operand outside them.
"""
collected = set(filter_cohort_ids(data))
action_ids = filter_action_ids(data)
if action_ids:
for action in get_actions(project_id=team.project_id, action_ids=action_ids, include_deleted=False):
for step in action.steps:
collected |= collect_property_cohort_ids(step.properties or [])
validate_realtime_cohorts(collected, team, require_ready=require_ready)
return collected
COHORT_NOT_READY_CODE = "cohort_not_ready"
def _cohort_not_ready_error(cohort: Cohort) -> serializers.ValidationError:
readiness = resolve_realtime_readiness([cohort]).get(cohort.pk)
if readiness is not None and readiness.state in (CohortRealtimeState.BUILDING, CohortRealtimeState.REBUILDING):
percent = readiness.build.percent_complete if readiness.build is not None else None
progress = f" ({percent}% done)" if percent is not None else ""
return serializers.ValidationError(
f"Cohort {cohort.pk} is still calculating{progress}. Workflows can use it once it finishes.",
code=COHORT_NOT_READY_CODE,
)
return serializers.ValidationError(
f"Cohort {cohort.pk} isn't ready for realtime evaluation. "
f"Workflows can only use realtime cohorts that have finished calculating."
)
def validate_realtime_cohorts(cohort_ids: Iterable[int], team: Team, *, require_ready: bool = True) -> None:
"""Reject any cohort without maintained cohort_membership rows, which would treat everyone as a non-member.
With `require_ready=False` a realtime cohort that is still calculating passes, so a draft can reference
it before its history finishes. A cohort that can never become ready is rejected either way.
Error messages carry the id rather than the name: they surface to callers holding only workflow
scopes, and a name would hand them a cohort-metadata oracle.
"""
ids = sorted(set(cohort_ids))
if not ids:
return
cohorts = {cohort.pk: cohort for cohort in Cohort.objects.filter(pk__in=ids, team=team, deleted=False)}
for cohort_id in ids:
cohort = cohorts.get(cohort_id)
if cohort is None:
raise serializers.ValidationError(f"Cohort {cohort_id} doesn't exist in this environment.")
if cohort.is_static:
raise serializers.ValidationError(
f"Cohort {cohort_id} is a static cohort. Workflows can only use realtime cohorts."
)
if cohort.is_flag_compatible:
continue
can_become_ready = cohort.cohort_type == CohortType.REALTIME and has_realtime_state(cohort)
if require_ready or not can_become_ready:
raise _cohort_not_ready_error(cohort)
class FunctionInputsSerializer(serializers.Serializer):
inputs_schema = InputsSchemaSerializer(required=False)
inputs = InputsSerializer(required=False)
def to_internal_value(self, data):
# Weirdly nested serializers don't get this set...
self.initial_data = data
return super().to_internal_value(data)
class MappingsSerializer(FunctionInputsSerializer):
name = serializers.CharField(required=False)
filters = HogFunctionFiltersSerializer(required=False)
# The hog function serializer compares repeated keys against the stored mappings, because the UI
# resends every mapping on each save and some stored mappings already repeat a key.
inputs_schema = InputsSchemaSerializer(required=False, unique_keys=False)
def to_internal_value(self, data: Any) -> dict[str, Any]:
if isinstance(data, dict) and "inputs_schema" in data:
try:
inputs_schema = self.fields["inputs_schema"].run_validation(data["inputs_schema"])
except ValidationError as error:
raise serializers.ValidationError({"inputs_schema": error.detail}) from error
if any(schema.get("secret") for schema in inputs_schema):
raise serializers.ValidationError(
{
"inputs_schema": "Mappings do not support secret inputs. Set secrets in the destination inputs instead."
}
)
data = {**data, "inputs_schema": inputs_schema}
return super().to_internal_value(data)
def topological_sort(nodes: list[str], edges: dict[str, list[str]]) -> list[str]:
"""
Perform a topological sort on the given graph.
nodes: list of all node identifiers
edges: adjacency list where edges[node] = list of nodes that `node` depends on
Returns: A list of nodes in topologically sorted order (no cycles).
Raises an error if a cycle is detected.
"""
# Build in-degree
in_degree = dict.fromkeys(nodes, 0)
for node, deps in edges.items():
for dep in deps:
if dep in in_degree:
in_degree[node] = in_degree[node] + 1
# Find all nodes with in_degree 0
queue = [n for n, d in in_degree.items() if d == 0]
sorted_list = []
while queue:
current = queue.pop(0)
sorted_list.append(current)
# Decrease in-degree of dependent nodes
for node, deps in edges.items():
if current in deps:
in_degree[node] -= 1
if in_degree[node] == 0:
queue.append(node)
if len(sorted_list) != len(nodes):
raise serializers.ValidationError("Circular dependency detected in input_deps.")
return sorted_list
def compile_hog(
hog: str,
hog_type: str,
in_repl: Optional[bool] = False,
null_safe_comparisons: bool = False,
) -> list[Any]:
# Attempt to compile the hog
try:
program = parse_program(hog)
detector = HyphenatedPropertyDetector()
detector.visit(program)
if detector.errors:
raise serializers.ValidationError({"hog": detector.errors[0]})
supported_functions: set[str] = set()
if hog_type == "destination":
supported_functions = CORE_SUPPORTED_FUNCTIONS | PRODUCT_ASYNC_FUNCTIONS
elif hog_type == "tagger":
# Taggers classify; they must not perform side effects, so we deliberately exclude
# CORE_SUPPORTED_FUNCTIONS (fetch, postHogCapture) and PRODUCT_ASYNC_FUNCTIONS.
# Stated explicitly so a future refactor can't silently widen the surface.
supported_functions = set()
elif hog_type == "transformation_log":
# Log transformations run synchronously per log record in the logs ingestion
# hot path — no async functions (fetch, postHogCapture) can ever be allowed.
# Stated explicitly so a future refactor can't silently widen the surface.
supported_functions = set()
# Validate the code body's globals like input templates are: without this,
# code referencing `event`/`person` compiles fine and only fails per record
# at ingestion time. Declared locals are excluded from the check.
declared = DeclaredNamesCollector()
declared.visit(program)
body_validator = TemplateGlobalsValidator(
available_globals=TRANSFORMATION_LOG_AVAILABLE_GLOBALS | declared.names,
runtime_functions=set(),
)
body_validator.visit(program)
if body_validator.invalid_globals:
names = ", ".join(sorted(body_validator.invalid_globals))
raise serializers.ValidationError(
{
"hog": f"Variable not available in log transformations: {names}. "
f"Log transformations only have access to project, record, and inputs."
}
)
context = HogQLContext(team_id=None)
bytecode = create_bytecode(
program,
supported_functions=supported_functions,
in_repl=in_repl,
null_safe_comparisons=null_safe_comparisons,
context=context,
).bytecode
# The compiler only records unknown-function calls as context errors; the call still
# compiles and fails at runtime. Log transformations run in a synchronous VM with no
# async functions registered, so surface the error at save time instead.
if hog_type == "transformation_log" and context.errors:
raise serializers.ValidationError({"hog": context.errors[0].message})
return bytecode
except serializers.ValidationError:
raise
except Exception as e:
logger.error(f"Failed to compile hog {e}", exc_info=True)
raise serializers.ValidationError({"hog": "Hog code has errors."})