:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(custom): accept literal braces in a custom source query string (#129883)
Tom Owers4ffa02889866
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/rest_source/config_setup.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/rest_source/config_setup.pyindex c4b57b444a13..6cbd53d8060c 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/common/rest_source/config_setup.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/common/rest_source/config_setup.py@@ -291,6 +291,7 @@ def _make_endpoint_resource(resource: str | EndpointResource, default_config: Endef _bind_path_params(resource: EndpointResource) -> None:path_params: dict[str, Any] = {}+ deferred: list[str] = []assert isinstance(resource["endpoint"], dict)resolve_params = [r.param_name for r in _find_resolved_params(resource["endpoint"])]path = resource["endpoint"]["path"]@@ -316,13 +317,28 @@ def _bind_path_params(resource: EndpointResource) -> None:f"The path {path} defined in resource {resource['name']} tries to bind param {name} with type {param_type}. Paths can only bind 'resource' type params.")path_params[name] = "{" + name + "}"+ deferred.append(name)if len(resolve_params) > 0:raise NotImplementedError(f"Resource {resource['name']} defines resolve params {resolve_params} that are not bound in path {path}. Resolve query params not supported yet.")- resource["endpoint"]["path"] = path.format(**path_params)+ if not deferred:+ resource["endpoint"]["path"] = path.format(**path_params)+ return++ # The fan-out formats this path a second time to bind the resolve params, so every+ # other brace must stay escaped through this first pass.+ markers = {name: f"\x00{name}\x00" for name in deferred}+ bound = _escape_braces(path.format(**{**path_params, **markers}))+ for name, marker in markers.items():+ bound = bound.replace(marker, "{" + name + "}")+ resource["endpoint"]["path"] = bound+++def _escape_braces(text: str) -> str:+ return text.replace("{", "{{").replace("}", "}}")def _setup_single_entity_endpoint(endpoint: Endpoint) -> Endpoint:diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/rest_source/tests/test_config_setup.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/rest_source/tests/test_config_setup.pyindex 74b179cd30e5..ae7b4a0ddaa2 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/common/rest_source/tests/test_config_setup.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/common/rest_source/tests/test_config_setup.py@@ -13,8 +13,10 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.resIncremental,IncrementalParam,_merge_resource_endpoints,+ build_resource_dependency_graph,create_auth,create_paginator,+ process_parent_data_item,setup_incremental_object,)from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.paginators import (@@ -214,3 +216,41 @@ class TestAuthSecretValues:)def test_extracts_secret_strings(self, auth: Any, expected: tuple[str, ...]) -> None:assert auth_secret_values(auth) == expected+++_FORM_ID = {"type": "resolve", "resource": "forms", "field": "id"}+++class TestBindPathParams:+ @pytest.mark.parametrize(+ "path,params,expected",+ [+ ("/forms/{form_id}/responses", {"form_id": _FORM_ID}, "/forms/f1/responses"),+ (+ "/forms/{form_id}/responses?fields=answers{{value}}",+ {"form_id": _FORM_ID},+ "/forms/f1/responses?fields=answers{value}",+ ),+ ("/forms/{form_id}/{kind}", {"form_id": _FORM_ID, "kind": "a{b}"}, "/forms/f1/a{b}"),+ ],+ )+ def test_child_path_keeps_literal_braces_through_parent_binding(+ self, path: str, params: dict[str, Any], expected: str+ ) -> None:+ resources: list[str | EndpointResource] = [+ {"name": "forms", "endpoint": {"path": "/forms"}},+ {"name": "responses", "endpoint": {"path": path, "params": params}},+ ]+ _, resource_map, resolved = build_resource_dependency_graph({}, resources)+ child_resolved = resolved["responses"]+ assert child_resolved is not None+ endpoint = cast(Endpoint, resource_map["responses"]["endpoint"])++ bound_path, _ = process_parent_data_item(cast(str, endpoint["path"]), {"id": "f1"}, child_resolved, [])++ assert bound_path == expected++ def test_top_level_path_unescapes_literal_braces_once(self) -> None:+ resources: list[str | EndpointResource] = [{"name": "posts", "endpoint": {"path": "/posts?fields=a{{b}}"}}]+ _, resource_map, _ = build_resource_dependency_graph({}, resources)+ assert cast(Endpoint, resource_map["posts"]["endpoint"])["path"] == "/posts?fields=a{b}"diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/custom/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/custom/source.pyindex ebdf8c9727ab..fc2f5ce36438 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/custom/source.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/custom/source.py@@ -1,5 +1,7 @@+import reimport copyimport json+import stringimport hashlibimport inspectimport graphlib@@ -276,6 +278,74 @@ def validate_manifest_structure(manifest: Any) -> None:raise ManifestValidationError(_format_validation_errors(exc)) from exc+_PLACEHOLDER_RE = re.compile(r"\{(\w+)\}")+++def _declared_params(manifest: dict[str, Any], endpoint: dict[str, Any]) -> set[str]:+ defaults = manifest.get("resource_defaults")+ default_endpoint = defaults.get("endpoint") if isinstance(defaults, dict) else None+ default_params = default_endpoint.get("params") if isinstance(default_endpoint, dict) else None+ params = endpoint.get("params")+ return {+ *(default_params if isinstance(default_params, dict) else {}),+ *(params if isinstance(params, dict) else {}),+ }+++def _unbound_placeholders(path: str, declared: set[str]) -> list[str] | None:+ """Placeholders in ``path`` that no param fills, or ``None`` when the engine can't parse the path."""+ try:+ return [name for _, name, _, _ in string.Formatter().parse(path) if name and name not in declared]+ except ValueError:+ return None+++def _double_braces(text: str) -> str:+ return text.replace("{", "{{").replace("}", "}}")+++def _escape_literal_query_braces(manifest: dict[str, Any]) -> None:+ """Keep braces in a path's query string literal unless a declared param fills them.++ The REST engine formats every path with ``str.format``, so query syntax that uses braces, such+ as Graph API field expansion (``?fields=attachments{media_type}``), would otherwise be read as+ a placeholder and rejected. A path the engine already accepts is left untouched.+ """+ for resource in manifest["resources"]:+ endpoint = resource.get("endpoint") if isinstance(resource, dict) else None+ if not isinstance(endpoint, dict):+ continue+ path = endpoint.get("path")+ if not isinstance(path, str) or "?" not in path:+ continue+ declared = _declared_params(manifest, endpoint)+ if _unbound_placeholders(path, declared) == []:+ continue+ base, query = path.split("?", 1)+ escaped: list[str] = []+ position = 0+ for match in _PLACEHOLDER_RE.finditer(query):+ escaped.append(_double_braces(query[position : match.start()]))+ escaped.append(match.group(0) if match.group(1) in declared else _double_braces(match.group(0)))+ position = match.end()+ escaped.append(_double_braces(query[position:]))+ endpoint["path"] = f"{base}?{''.join(escaped)}"+++def _check_path_placeholders(manifest: dict[str, Any]) -> None:+ # The engine's own error for this echoes the whole path, which can carry account ids.+ for resource in manifest["resources"]:+ endpoint = resource.get("endpoint") if isinstance(resource, dict) else None+ if not isinstance(endpoint, dict) or not isinstance(endpoint.get("path"), str):+ continue+ unbound = _unbound_placeholders(endpoint["path"], _declared_params(manifest, endpoint))+ if unbound:+ raise ManifestValidationError(+ f"Table {resource.get('name')!r} has {{{unbound[0]}}} in its path, but no parent field fills it. "+ "Link the table to a parent table with that field, or remove the braces."+ )++def _validate_resource_graph(manifest: dict[str, Any]) -> dict[str, Optional[list[ResolvedParam]]]:"""Surface parent/child fan-out errors at create-time instead of first sync.@@ -300,6 +370,7 @@ def _validate_resource_graph(manifest: dict[str, Any]) -> dict[str, Optional[lis``None`` for top-level resources) so callers that need it — the probe'schild filter — don't rebuild the graph."""+ _check_path_placeholders(manifest)try:graph, resource_map, resolved = _build_resource_graph(manifest)list(graph.static_order())@@ -1141,6 +1212,7 @@ class CustomSource(SimpleSource[CustomSourceConfig]):# schemas (graph rules are enforced on the API validation paths in# `validate_credentials`, and per-schema at sync time in `_fanout_chain`).validate_manifest_structure(manifest)+ _escape_literal_query_braces(manifest)_inject_auth_secrets(manifest, config)return manifestdiff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/custom/test_custom_source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/custom/test_custom_source.pyindex 78874e690fa4..e255faf4017d 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/custom/test_custom_source.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/custom/test_custom_source.py@@ -2523,6 +2523,86 @@ class TestCustomSourceFanoutPipeline(SimpleTestCase):]+class TestCustomSourceLiteralQueryBraces(SimpleTestCase):+ @parameterized.expand(+ [+ (+ "field_expansion",+ "/page/posts?limit=100&fields=id,attachments{media_type},insights.metric(a,b)",+ None,+ "fields",+ "id,attachments{media_type},insights.metric(a,b)",+ ),+ (+ "nested_field_expansion",+ "/page/posts?fields=insights{values{value}}",+ None,+ "fields",+ "insights{values{value}}",+ ),+ ("already_escaped", "/page/posts?fields=a{{b}}", None, "fields", "a{b}"),+ ("unbalanced_brace", "/page/posts?q=a}", None, "q", "a}"),+ ("declared_param_still_binds", "/page/posts?since={since}&fields=a{b}", {"since": "2024"}, "since", "2024"),+ ]+ )+ def test_query_braces_reach_request(self, _name, path, params, query_param, expected):+ manifest = _minimal_manifest()+ endpoint = manifest["resources"][0]["endpoint"]+ endpoint.update({"path": path, "paginator": {"type": "single_page"}})+ if params:+ endpoint["params"] = params++ result = SourceDriver(CustomSource(), CustomSourceConfig(manifest_json=json.dumps(manifest))).run(+ "users", [ScriptedResponse(json={"data": [{"id": 1}]})], team_id=999+ )++ assert result.raised is None+ assert result.paths == ["/page/posts"]+ assert result.requests[0].param(query_param) == expected++ def test_fanout_child_query_braces_reach_request(self):+ manifest = _fanout_manifest()+ for resource in manifest["resources"]:+ resource["endpoint"]["paginator"] = {"type": "single_page"}+ manifest["resources"][1]["endpoint"]["path"] = "/forms/{form_id}/responses?fields=answers{value}"++ result = SourceDriver(CustomSource(), CustomSourceConfig(manifest_json=json.dumps(manifest))).run(+ "responses",+ [ScriptedResponse(json={"items": [{"id": "f1"}]}), ScriptedResponse(json={"items": []})],+ team_id=999,+ )++ assert result.raised is None+ assert result.paths == ["/forms", "/forms/f1/responses"]+ assert result.requests[1].param("fields") == "answers{value}"++ def test_validate_credentials_accepts_query_braces(self):+ manifest = _minimal_manifest()+ manifest["resources"][0]["endpoint"]["path"] = "/page/posts?fields=attachments{media_type}"+ config = CustomSourceConfig(manifest_json=json.dumps(manifest), auth_token="abc")+ with scripted_network([ScriptedResponse(json={})]):+ ok, err = CustomSource().validate_credentials(config, team_id=999)+ assert ok, err++ @parameterized.expand(+ [+ ("path_segment", "/accounts/{account_id}/users"),+ ("path_segment_with_query_braces", "/accounts/{account_id}/users?fields=a{b}"),+ ]+ )+ def test_unbound_path_placeholder_names_table_not_path(self, _name, path):+ manifest = _minimal_manifest()+ manifest["resources"][0]["endpoint"]["path"] = path+ config = CustomSourceConfig(manifest_json=json.dumps(manifest), auth_token="abc")++ ok, err = CustomSource().validate_credentials(config, team_id=999)++ assert ok is False+ assert err is not None+ assert "'users'" in err and "{account_id}" in err+ assert "/accounts/" not in err++class TestCustomSourceFanoutSchemasAndProbe(SimpleTestCase):def test_child_resource_is_its_own_schema(self):manifest = _fanout_manifest()