:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(data-warehouse): add 2 missing tables to the Framer source (#123730)
Tom Owers8cc86b71865b
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.mdindex 811202d42fc9..36a3b7aa8358 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md@@ -4912,9 +4912,9 @@ Today (7): `CollectionItems`, `Collections`, `Deployments`, `Locales`, `Pages`,Diffed against: <https://registry.npmjs.org/framer-api/-/framer-api-5.1.0.tgz>-- [ ] `Localization groups (Server API RPC listLocalizationGroups())` — measure translation readiness and locale coverage for pages and CMS items (medium)-- [ ] `Changed page paths (Server API RPC getChangedPaths())` — track pages added, removed, or modified between project versions (medium)-- [ ] `Change contributors (Server API RPC getChangeContributors())` — attribute content changes between project versions to their authors (medium)+- [x] `Localization groups (Server API RPC listLocalizationGroups())` — measure translation readiness and locale coverage for pages and CMS items (medium) → `LocalizationGroups` (full refresh; streamed through the `listLocalizationGroupsNext` cursor on 5.1.0, `getLocalizationGroups` on 0.1.29)+- [x] `Changed page paths (Server API RPC getChangedPaths())` — track pages added, removed, or modified between project versions (medium) → `UnpublishedPageChanges` (full refresh; built on `getUnpublishedPageChanges()`, because framer-api 5.1.0 deprecates `getChangedPaths()` in its favour, and both compare the current draft with the last publish, not two chosen versions)+- [ ] `Change contributors (Server API RPC getChangeContributors())` — attribute content changes between project versions to their authors (medium); skipped: the RPC is alpha, takes version numbers that no other RPC exposes, and returns bare strings with no documented meaningNote: Read the official framer-api 5.1.0 declarations, the shared API reference, and the Server API reference. These collection reads use RPC methods, not REST paths. Collections already include field definitions, and Project already includes publishing information. Excluded canvas editing, plugin state, code files, and publishing writes; version comparisons need explicit version windows.diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/framer/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/framer/canonical_descriptions.pyindex 9b5d23a6960b..cec1ab495e7b 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/framer/canonical_descriptions.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/framer/canonical_descriptions.py@@ -85,4 +85,25 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {"versionUrl": "URL of the versioned deployment, when available.",},},+ "LocalizationGroups": {+ "description": "Groups of translatable content, such as a page or a CMS item, with their source strings and the status of each locale.",+ "docs_url": "https://www.framer.com/developers/localization",+ "columns": {+ "id": "Unique identifier of the localization group.",+ "name": "Name of the localization group.",+ "type": "Kind of content the group represents: collection, collection-item, component, page, settings, or template.",+ "supportsExcludedStatus": "Whether the group can be excluded from a locale.",+ "sources": "Translatable strings in the group, each with its type, default-locale value, and localized values and status by locale id.",+ "statusByLocale": "Status of the group in each locale (excluded or ready), keyed by locale id.",+ },+ },+ "UnpublishedPageChanges": {+ "description": "Web pages that were added, updated, or removed since the project was last published.",+ "docs_url": "https://www.framer.com/developers/server-api-reference",+ "columns": {+ "nodeId": "Stable node identifier of the changed page.",+ "path": "Page path: the current path for added and updated pages, and the last-published path for removed pages.",+ "status": "Change that publishing applies to the page: added, updated, or removed.",+ },+ },}diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/framer/framer.py b/products/warehouse_sources/backend/temporal/data_imports/sources/framer/framer.pyindex 285bca244b3c..3ad2ee6541de 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/framer/framer.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/framer/framer.py@@ -30,6 +30,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.framer impfrom products.warehouse_sources.backend.temporal.data_imports.sources.framer.settings import (DEPLOYMENTS_MAX_PAGES,DEPLOYMENTS_PAGE_SIZE,+ LOCALIZATION_GROUPS_MAX,PRIMARY_KEYS,)@@ -51,6 +52,10 @@ COLLECTION_METHODS_BY_VERSION: dict[str, CollectionMethods] = {FRAMER_SDK_VERSION_5_1_0: {"fields": "getCollectionFields3", "items": "getCollectionItems3"},}+# framer-api 5.x streams localization groups one at a time through a server-side cursor,+# so a large project never arrives as one response. Older versions only have the bulk read.+LOCALIZATION_GROUPS_STREAMED_VERSIONS = {FRAMER_SDK_VERSION_5_1_0}+# The server boots a headless instance of the project on connect, which can take a while# for large projects — the official SDK allows 90s, so we do too.CONNECT_TIMEOUT_SECONDS = 90@@ -493,6 +498,46 @@ def _deployments_rows(client: FramerClient, logger: Optional[FilteringBoundLoggelogger.warning("Framer deployments page cap reached", max_pages=DEPLOYMENTS_MAX_PAGES)+def _streamed_localization_groups_rows(+ client: FramerClient, logger: Optional[FilteringBoundLogger]+) -> Iterator[list[dict[str, Any]]]:+ # The cursor belongs to this connection's session, which closes after the endpoint, so+ # there is no releaseLocalizationGroupsCursor call.+ cursor: Optional[str] = None+ for _group_number in range(LOCALIZATION_GROUPS_MAX):+ result = client.call("listLocalizationGroupsNext", {"cursor": cursor})+ if not isinstance(result, dict) or "group" not in result:+ raise FramerAPIError("Localization group response is missing its group", code="INVALID_RESPONSE")+ group = result["group"]+ if group is None:+ return+ if not isinstance(group, dict):+ raise FramerAPIError("Localization group response has an invalid group", code="INVALID_RESPONSE")+ cursor = result.get("cursor")+ if not cursor:+ raise FramerAPIError("Localization group response is missing its cursor", code="INVALID_RESPONSE")+ yield [group]+ if logger:+ logger.warning("Framer localization groups cap reached", max_groups=LOCALIZATION_GROUPS_MAX)+++def _localization_groups_rows(+ client: FramerClient, logger: Optional[FilteringBoundLogger]+) -> Iterator[list[dict[str, Any]]]:+ if client.protocol_version in LOCALIZATION_GROUPS_STREAMED_VERSIONS:+ yield from _streamed_localization_groups_rows(client, logger)+ return+ rows = _as_dict_rows(client.call("getLocalizationGroups"))+ if rows:+ yield rows+++def _unpublished_page_changes_rows(client: FramerClient) -> Iterator[list[dict[str, Any]]]:+ rows = _as_dict_rows(client.call("getUnpublishedPageChanges"))+ if rows:+ yield rows++def _endpoint_rows(client: FramerClient, endpoint: str, logger: Optional[FilteringBoundLogger]) -> Iterator[list[dict[str, Any]]]:@@ -510,6 +555,10 @@ def _endpoint_rows(yield from _redirects_rows(client)elif endpoint == "Deployments":yield from _deployments_rows(client, logger)+ elif endpoint == "LocalizationGroups":+ yield from _localization_groups_rows(client, logger)+ elif endpoint == "UnpublishedPageChanges":+ yield from _unpublished_page_changes_rows(client)else:raise FramerAPIError(f"Unknown endpoint: {endpoint}", code="INVALID_REQUEST")diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/framer/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/framer/settings.pyindex 1c1192e7a1a0..c329f252d4db 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/framer/settings.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/framer/settings.py@@ -8,6 +8,8 @@ ENDPOINTS = ("Locales","Redirects","Deployments",+ "LocalizationGroups",+ "UnpublishedPageChanges",)# The Server API returns each dataset in full per RPC call, with no server-side time@@ -23,9 +25,16 @@ PRIMARY_KEYS: dict[str, list[str]] = {"Locales": ["id"],"Redirects": ["id"],"Deployments": ["id"],+ "LocalizationGroups": ["id"],+ # A path can repeat when one page is removed and another takes its path, so key on the page node.+ "UnpublishedPageChanges": ["nodeId"],}# listDeployments serves at most 30 per page.DEPLOYMENTS_PAGE_SIZE = 30# Hard cap on deployment pages per sync so a pathological cursor loop can't spin forever.DEPLOYMENTS_MAX_PAGES = 500++# listLocalizationGroupsNext returns one group per call. The cap bounds a cursor loop that+# never reports the end.+LOCALIZATION_GROUPS_MAX = 200_000diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/framer/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/framer/source.pyindex f64d4aeaa1f6..8be0d656b1ef 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/framer/source.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/framer/source.py@@ -120,7 +120,7 @@ class FramerSource(SimpleSource[FramerSourceConfig]):category=DataWarehouseSourceCategory.PRODUCTIVITY,label="Framer",releaseStatus=ReleaseStatus.GA,- caption="""Sync your Framer site's pages, CMS collections and items, locales, redirects, and deployments into the PostHog Data warehouse.+ caption="""Sync your Framer site's pages, CMS collections and items, locales, localization groups, redirects, deployments, and unpublished page changes into the PostHog Data warehouse.Create an API key in your Framer project under **Site Settings → General**, then paste your project URL (https://framer.com/projects/...) and the API key. API keys are project-specific, so add one source per project.""",iconPath="/static/services/framer.png",diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/framer/tests/test_framer.py b/products/warehouse_sources/backend/temporal/data_imports/sources/framer/tests/test_framer.pyindex 517427a81a6e..6ff503e0fa88 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/framer/tests/test_framer.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/framer/tests/test_framer.py@@ -547,13 +547,92 @@ class TestFramer:rows = self._run_endpoint("Deployments", {"listDeployments": list_deployments})assert rows == [{"id": "d1"}, {"id": "d2"}]- def test_locales_and_redirects_rows(self) -> None:- assert self._run_endpoint("Locales", {"getLocales": [{"id": "l1", "code": "en-US"}]}) == [- {"id": "l1", "code": "en-US"}+ @parameterized.expand(+ [+ ("Locales", "getLocales", [{"id": "l1", "code": "en-US"}]),+ ("Redirects", "getRedirects", [{"id": "r1", "from": "/a", "to": "/b"}]),+ (+ "UnpublishedPageChanges",+ "getUnpublishedPageChanges",+ [+ {"nodeId": "n1", "path": "/pricing", "status": "added"},+ {"nodeId": "n2", "path": "/about", "status": "removed"},+ ],+ ),+ ("UnpublishedPageChanges", "getUnpublishedPageChanges", []),+ ]+ )+ def test_single_call_endpoint_rows(self, endpoint: str, method: str, rows: list[dict[str, Any]]) -> None:+ assert self._run_endpoint(endpoint, {method: rows}) == rows++ @parameterized.expand(+ [+ ("streamed", "5.1.0", [None, "cur-1", "cur-2"]),+ ("bulk", "0.1.29", []),+ ]+ )+ def test_localization_groups_rows(+ self, _name: str, protocol_version: str, expected_cursors: list[Optional[str]]+ ) -> None:+ groups = [+ {+ "id": "g1",+ "name": "Home",+ "type": "page",+ "supportsExcludedStatus": False,+ "sources": [+ {+ "id": "s1",+ "type": "string",+ "value": "Welcome",+ "valueByLocale": {"fr": {"status": "done", "value": "Bienvenue"}},+ }+ ],+ "statusByLocale": {"fr": "ready"},+ },+ {+ "id": "g2",+ "name": "Launch post",+ "type": "collection-item",+ "supportsExcludedStatus": True,+ "sources": [],+ "statusByLocale": {"fr": "excluded"},+ },]- assert self._run_endpoint("Redirects", {"getRedirects": [{"id": "r1", "from": "/a", "to": "/b"}]}) == [- {"id": "r1", "from": "/a", "to": "/b"}+ cursors: list[Optional[str]] = []++ def list_next(request: dict[str, Any]) -> dict[str, Any]:+ cursor = request.get("cursor")+ cursors.append(cursor)+ position = int(cursor.removeprefix("cur-")) if cursor else 0+ if position >= len(groups):+ return {"group": None}+ return {"group": groups[position], "cursor": f"cur-{position + 1}"}++ rows = self._run_endpoint(+ "LocalizationGroups",+ {"listLocalizationGroupsNext": list_next, "getLocalizationGroups": groups},+ protocol_version,+ )+ assert rows == groups+ assert cursors == expected_cursors++ @parameterized.expand(+ [+ ("missing_cursor", {"group": {"id": "g1"}}),+ ("missing_group", {}),+ ("non_object_response", None),+ ("invalid_group", {"group": "g1", "cursor": "cur-1"}),]+ )+ def test_streamed_localization_groups_fail_on_malformed_response(self, _name: str, response: Any) -> None:+ with pytest.raises(FramerAPIError) as exc_info:+ self._run_endpoint(+ "LocalizationGroups",+ {"listLocalizationGroupsNext": response},+ "5.1.0",+ )+ assert exc_info.value.code == "INVALID_RESPONSE"def test_source_response_raises_for_invalid_project(self) -> None:response = framer_source("not a project", "key", "Project", protocol_version="0.1.29")