Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(custom): accept literal braces in a custom source query string (#129883)

Tom Owers4ffa02889866
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/rest_source/config_setup.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/rest_source/config_setup.py
index c4b57b444a13..6cbd53d8060c 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/common/rest_source/config_setup.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/common/rest_source/config_setup.py
@@ -291,6 +291,7 @@ def _make_endpoint_resource(resource: str | EndpointResource, default_config: En
def _bind_path_params(resource: EndpointResource) -> None:
path_params: dict[str, Any] = {}
+ deferred: list[str] = []
assert isinstance(resource["endpoint"], dict)
resolve_params = [r.param_name for r in _find_resolved_params(resource["endpoint"])]
path = resource["endpoint"]["path"]
@@ -316,13 +317,28 @@ def _bind_path_params(resource: EndpointResource) -> None:
f"The path {path} defined in resource {resource['name']} tries to bind param {name} with type {param_type}. Paths can only bind 'resource' type params."
)
path_params[name] = "{" + name + "}"
+ deferred.append(name)
if len(resolve_params) > 0:
raise NotImplementedError(
f"Resource {resource['name']} defines resolve params {resolve_params} that are not bound in path {path}. Resolve query params not supported yet."
)
- resource["endpoint"]["path"] = path.format(**path_params)
+ if not deferred:
+ resource["endpoint"]["path"] = path.format(**path_params)
+ return
+
+ # The fan-out formats this path a second time to bind the resolve params, so every
+ # other brace must stay escaped through this first pass.
+ markers = {name: f"\x00{name}\x00" for name in deferred}
+ bound = _escape_braces(path.format(**{**path_params, **markers}))
+ for name, marker in markers.items():
+ bound = bound.replace(marker, "{" + name + "}")
+ resource["endpoint"]["path"] = bound
+
+
+def _escape_braces(text: str) -> str:
+ return text.replace("{", "{{").replace("}", "}}")
def _setup_single_entity_endpoint(endpoint: Endpoint) -> Endpoint:
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/rest_source/tests/test_config_setup.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/rest_source/tests/test_config_setup.py
index 74b179cd30e5..ae7b4a0ddaa2 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/common/rest_source/tests/test_config_setup.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/common/rest_source/tests/test_config_setup.py
@@ -13,8 +13,10 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.res
Incremental,
IncrementalParam,
_merge_resource_endpoints,
+ build_resource_dependency_graph,
create_auth,
create_paginator,
+ process_parent_data_item,
setup_incremental_object,
)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.paginators import (
@@ -214,3 +216,41 @@ class TestAuthSecretValues:
)
def test_extracts_secret_strings(self, auth: Any, expected: tuple[str, ...]) -> None:
assert auth_secret_values(auth) == expected
+
+
+_FORM_ID = {"type": "resolve", "resource": "forms", "field": "id"}
+
+
+class TestBindPathParams:
+ @pytest.mark.parametrize(
+ "path,params,expected",
+ [
+ ("/forms/{form_id}/responses", {"form_id": _FORM_ID}, "/forms/f1/responses"),
+ (
+ "/forms/{form_id}/responses?fields=answers{{value}}",
+ {"form_id": _FORM_ID},
+ "/forms/f1/responses?fields=answers{value}",
+ ),
+ ("/forms/{form_id}/{kind}", {"form_id": _FORM_ID, "kind": "a{b}"}, "/forms/f1/a{b}"),
+ ],
+ )
+ def test_child_path_keeps_literal_braces_through_parent_binding(
+ self, path: str, params: dict[str, Any], expected: str
+ ) -> None:
+ resources: list[str | EndpointResource] = [
+ {"name": "forms", "endpoint": {"path": "/forms"}},
+ {"name": "responses", "endpoint": {"path": path, "params": params}},
+ ]
+ _, resource_map, resolved = build_resource_dependency_graph({}, resources)
+ child_resolved = resolved["responses"]
+ assert child_resolved is not None
+ endpoint = cast(Endpoint, resource_map["responses"]["endpoint"])
+
+ bound_path, _ = process_parent_data_item(cast(str, endpoint["path"]), {"id": "f1"}, child_resolved, [])
+
+ assert bound_path == expected
+
+ def test_top_level_path_unescapes_literal_braces_once(self) -> None:
+ resources: list[str | EndpointResource] = [{"name": "posts", "endpoint": {"path": "/posts?fields=a{{b}}"}}]
+ _, resource_map, _ = build_resource_dependency_graph({}, resources)
+ assert cast(Endpoint, resource_map["posts"]["endpoint"])["path"] == "/posts?fields=a{b}"
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/custom/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/custom/source.py
index ebdf8c9727ab..fc2f5ce36438 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/custom/source.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/custom/source.py
@@ -1,5 +1,7 @@
+import re
import copy
import json
+import string
import hashlib
import inspect
import graphlib
@@ -276,6 +278,74 @@ def validate_manifest_structure(manifest: Any) -> None:
raise ManifestValidationError(_format_validation_errors(exc)) from exc
+_PLACEHOLDER_RE = re.compile(r"\{(\w+)\}")
+
+
+def _declared_params(manifest: dict[str, Any], endpoint: dict[str, Any]) -> set[str]:
+ defaults = manifest.get("resource_defaults")
+ default_endpoint = defaults.get("endpoint") if isinstance(defaults, dict) else None
+ default_params = default_endpoint.get("params") if isinstance(default_endpoint, dict) else None
+ params = endpoint.get("params")
+ return {
+ *(default_params if isinstance(default_params, dict) else {}),
+ *(params if isinstance(params, dict) else {}),
+ }
+
+
+def _unbound_placeholders(path: str, declared: set[str]) -> list[str] | None:
+ """Placeholders in ``path`` that no param fills, or ``None`` when the engine can't parse the path."""
+ try:
+ return [name for _, name, _, _ in string.Formatter().parse(path) if name and name not in declared]
+ except ValueError:
+ return None
+
+
+def _double_braces(text: str) -> str:
+ return text.replace("{", "{{").replace("}", "}}")
+
+
+def _escape_literal_query_braces(manifest: dict[str, Any]) -> None:
+ """Keep braces in a path's query string literal unless a declared param fills them.
+
+ The REST engine formats every path with ``str.format``, so query syntax that uses braces, such
+ as Graph API field expansion (``?fields=attachments{media_type}``), would otherwise be read as
+ a placeholder and rejected. A path the engine already accepts is left untouched.
+ """
+ for resource in manifest["resources"]:
+ endpoint = resource.get("endpoint") if isinstance(resource, dict) else None
+ if not isinstance(endpoint, dict):
+ continue
+ path = endpoint.get("path")
+ if not isinstance(path, str) or "?" not in path:
+ continue
+ declared = _declared_params(manifest, endpoint)
+ if _unbound_placeholders(path, declared) == []:
+ continue
+ base, query = path.split("?", 1)
+ escaped: list[str] = []
+ position = 0
+ for match in _PLACEHOLDER_RE.finditer(query):
+ escaped.append(_double_braces(query[position : match.start()]))
+ escaped.append(match.group(0) if match.group(1) in declared else _double_braces(match.group(0)))
+ position = match.end()
+ escaped.append(_double_braces(query[position:]))
+ endpoint["path"] = f"{base}?{''.join(escaped)}"
+
+
+def _check_path_placeholders(manifest: dict[str, Any]) -> None:
+ # The engine's own error for this echoes the whole path, which can carry account ids.
+ for resource in manifest["resources"]:
+ endpoint = resource.get("endpoint") if isinstance(resource, dict) else None
+ if not isinstance(endpoint, dict) or not isinstance(endpoint.get("path"), str):
+ continue
+ unbound = _unbound_placeholders(endpoint["path"], _declared_params(manifest, endpoint))
+ if unbound:
+ raise ManifestValidationError(
+ f"Table {resource.get('name')!r} has {{{unbound[0]}}} in its path, but no parent field fills it. "
+ "Link the table to a parent table with that field, or remove the braces."
+ )
+
+
def _validate_resource_graph(manifest: dict[str, Any]) -> dict[str, Optional[list[ResolvedParam]]]:
"""Surface parent/child fan-out errors at create-time instead of first sync.
@@ -300,6 +370,7 @@ def _validate_resource_graph(manifest: dict[str, Any]) -> dict[str, Optional[lis
``None`` for top-level resources) so callers that need it — the probe's
child filter — don't rebuild the graph.
"""
+ _check_path_placeholders(manifest)
try:
graph, resource_map, resolved = _build_resource_graph(manifest)
list(graph.static_order())
@@ -1141,6 +1212,7 @@ class CustomSource(SimpleSource[CustomSourceConfig]):
# schemas (graph rules are enforced on the API validation paths in
# `validate_credentials`, and per-schema at sync time in `_fanout_chain`).
validate_manifest_structure(manifest)
+ _escape_literal_query_braces(manifest)
_inject_auth_secrets(manifest, config)
return manifest
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/custom/test_custom_source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/custom/test_custom_source.py
index 78874e690fa4..e255faf4017d 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/custom/test_custom_source.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/custom/test_custom_source.py
@@ -2523,6 +2523,86 @@ class TestCustomSourceFanoutPipeline(SimpleTestCase):
]
+class TestCustomSourceLiteralQueryBraces(SimpleTestCase):
+ @parameterized.expand(
+ [
+ (
+ "field_expansion",
+ "/page/posts?limit=100&fields=id,attachments{media_type},insights.metric(a,b)",
+ None,
+ "fields",
+ "id,attachments{media_type},insights.metric(a,b)",
+ ),
+ (
+ "nested_field_expansion",
+ "/page/posts?fields=insights{values{value}}",
+ None,
+ "fields",
+ "insights{values{value}}",
+ ),
+ ("already_escaped", "/page/posts?fields=a{{b}}", None, "fields", "a{b}"),
+ ("unbalanced_brace", "/page/posts?q=a}", None, "q", "a}"),
+ ("declared_param_still_binds", "/page/posts?since={since}&fields=a{b}", {"since": "2024"}, "since", "2024"),
+ ]
+ )
+ def test_query_braces_reach_request(self, _name, path, params, query_param, expected):
+ manifest = _minimal_manifest()
+ endpoint = manifest["resources"][0]["endpoint"]
+ endpoint.update({"path": path, "paginator": {"type": "single_page"}})
+ if params:
+ endpoint["params"] = params
+
+ result = SourceDriver(CustomSource(), CustomSourceConfig(manifest_json=json.dumps(manifest))).run(
+ "users", [ScriptedResponse(json={"data": [{"id": 1}]})], team_id=999
+ )
+
+ assert result.raised is None
+ assert result.paths == ["/page/posts"]
+ assert result.requests[0].param(query_param) == expected
+
+ def test_fanout_child_query_braces_reach_request(self):
+ manifest = _fanout_manifest()
+ for resource in manifest["resources"]:
+ resource["endpoint"]["paginator"] = {"type": "single_page"}
+ manifest["resources"][1]["endpoint"]["path"] = "/forms/{form_id}/responses?fields=answers{value}"
+
+ result = SourceDriver(CustomSource(), CustomSourceConfig(manifest_json=json.dumps(manifest))).run(
+ "responses",
+ [ScriptedResponse(json={"items": [{"id": "f1"}]}), ScriptedResponse(json={"items": []})],
+ team_id=999,
+ )
+
+ assert result.raised is None
+ assert result.paths == ["/forms", "/forms/f1/responses"]
+ assert result.requests[1].param("fields") == "answers{value}"
+
+ def test_validate_credentials_accepts_query_braces(self):
+ manifest = _minimal_manifest()
+ manifest["resources"][0]["endpoint"]["path"] = "/page/posts?fields=attachments{media_type}"
+ config = CustomSourceConfig(manifest_json=json.dumps(manifest), auth_token="abc")
+ with scripted_network([ScriptedResponse(json={})]):
+ ok, err = CustomSource().validate_credentials(config, team_id=999)
+ assert ok, err
+
+ @parameterized.expand(
+ [
+ ("path_segment", "/accounts/{account_id}/users"),
+ ("path_segment_with_query_braces", "/accounts/{account_id}/users?fields=a{b}"),
+ ]
+ )
+ def test_unbound_path_placeholder_names_table_not_path(self, _name, path):
+ manifest = _minimal_manifest()
+ manifest["resources"][0]["endpoint"]["path"] = path
+ config = CustomSourceConfig(manifest_json=json.dumps(manifest), auth_token="abc")
+
+ ok, err = CustomSource().validate_credentials(config, team_id=999)
+
+ assert ok is False
+ assert err is not None
+ assert "'users'" in err and "{account_id}" in err
+ assert "/accounts/" not in err
+
+
class TestCustomSourceFanoutSchemasAndProbe(SimpleTestCase):
def test_child_resource_is_its_own_schema(self):
manifest = _fanout_manifest()