:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(warehouse): add confluence page and blog post permission tables (#119240)
Tom Owers34a37dabc231
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.mdindex 447501d2863a..fa689e51789b 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md@@ -3160,7 +3160,7 @@ Note: Source exposes 12 static endpoints (settings.py CONFIGCAT_ENDPOINTS) with## Confluence — gaps-Today (8): `attachments`, `blogposts`, `footer_comments`, `inline_comments`, `labels`, `pages`, `spaces`, `tasks`+Today (10): `attachments`, `blogpost_permissions`, `blogposts`, `footer_comments`, `inline_comments`, `labels`, `page_permissions`, `pages`, `spaces`, `tasks`Diffed against: <https://dac-static.atlassian.com/cloud/confluence/openapi-v2.v3.json>@@ -3174,6 +3174,7 @@ Diffed against: <https://dac-static.atlassian.com/cloud/confluence/openapi-v2.v3- [ ] `GET /pages/{id}/likes/count and /footer-comments/{id}/likes/count` — engagement counts per page and comment (low)- [ ] `GET /whiteboards/{id}, /databases/{id}, /embeds/{id}, /folders/{id} (+ /descendants)` — the remaining v2 content types; reachable only via space/page descendant walks (low)- [ ] `GET /classification-levels and /pages/{id}/classification-level` — data classification lookup for governance reporting (low)+- [x] `GET /pages/{id}/permissions and /blogposts/{id}/permissions` — direct read/update restrictions per page and blog post; the database, whiteboard and folder variants are not synced because those content types have no list endpoint to fan out from (medium)Note: Diffed against the v2 spec above; the users, groups, analytics and audit gaps come from the v1 spec at https://dac-static.atlassian.com/cloud/confluence/swagger.v3.json, since v2 has no user/analytics endpoints. Content types whiteboards/databases/embeds/folders exist in v2 only as fetch-by-id plus ancestors/descendants - there is no top-level list - so they are harder than the rest.diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/confluence/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/confluence/canonical_descriptions.pyindex 59500b24876e..fd5ad639437d 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/confluence/canonical_descriptions.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/confluence/canonical_descriptions.py@@ -10,6 +10,15 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.canCanonicalDescriptions,)+# Columns of a direct content permission entry, shared by every content type's permissions endpoint.+_PERMISSION_COLUMNS = {+ "operation": "The operation the permission grants: read or update.",+ "principalType": "Type of principal the permission applies to: user or group.",+ "principalId": "ID of the principal. An Atlassian account ID for users, a group ID for groups.",+ "createdAt": "Date and time the permission entry was created.",+ "createdBy": "Atlassian account ID of the user who created the permission entry.",+}+# Fields shared by most Confluence v2 content objects; merged into each entry so we don't repeat them._COMMON_COLUMNS = {"id": "Unique identifier for the object.",@@ -133,4 +142,14 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {"version": "Version metadata for the comment.",},},+ "page_permissions": {+ "description": "A direct read or update restriction on a Confluence page, granted to one user or group.",+ "docs_url": "https://developer.atlassian.com/cloud/confluence/rest/v2/api-group-content-permissions/",+ "columns": {"pageId": "ID of the page the permission is on.", **_PERMISSION_COLUMNS},+ },+ "blogpost_permissions": {+ "description": "A direct read or update restriction on a Confluence blog post, granted to one user or group.",+ "docs_url": "https://developer.atlassian.com/cloud/confluence/rest/v2/api-group-content-permissions/",+ "columns": {"blogPostId": "ID of the blog post the permission is on.", **_PERMISSION_COLUMNS},+ },}diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/confluence/confluence.py b/products/warehouse_sources/backend/temporal/data_imports/sources/confluence/confluence.pyindex fb74ddc3c055..be2ced1673e3 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/confluence/confluence.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/confluence/confluence.py@@ -1,7 +1,7 @@import reimport base64import dataclasses-from typing import Any, Optional+from typing import Any, Optional, castfrom requests import Response@@ -10,13 +10,21 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.resRESTAPIConfig,rest_api_resource,)+from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.fanout import (+ build_dependent_resource,+)from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.paginators import (JSONResponsePaginator,)+from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.resource import Resource+from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.typing import ClientConfigfrom products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManagerfrom products.warehouse_sources.backend.temporal.data_imports.sources.common.source_helpers import validate_via_probefrom products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse-from products.warehouse_sources.backend.temporal.data_imports.sources.confluence.settings import CONFLUENCE_ENDPOINTS+from products.warehouse_sources.backend.temporal.data_imports.sources.confluence.settings import (+ CONFLUENCE_ENDPOINTS,+ ConfluenceEndpointConfig,+)# Confluence Cloud sites always live under <subdomain>.atlassian.net. Building# the host ourselves from a validated subdomain (rather than accepting an@@ -25,7 +33,7 @@ _SUBDOMAIN_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9-]{0,62}$")_ATLASSIAN_HOST_SUFFIX = ".atlassian.net"[email protected](frozen=True)class ConfluenceResumeConfig:next_url: str@@ -76,6 +84,72 @@ class ConfluenceLinkPaginator(JSONResponsePaginator):self._next_url = f"{self._site_origin}{self._next_url}"+def _client_config(subdomain: str, email: str, api_token: str) -> ClientConfig:+ return {+ "base_url": _base_url(subdomain),+ # Auth (HTTP Basic) is supplied via the framework auth config so the token is+ # redacted from logs; only the non-secret Accept header is set here.+ "headers": {"Accept": "application/json"},+ "auth": {"type": "http_basic", "username": email, "password": api_token},+ "paginator": ConfluenceLinkPaginator(_site_origin(subdomain)),+ }+++def _lift_principal(row: dict[str, Any]) -> dict[str, Any]:+ # The primary key needs the principal at the top level.+ principal = row.pop("principal", None) or {}+ row["principalType"] = principal.get("principalType")+ row["principalId"] = principal.get("principalId")+ return row+++def _source_response(config: ConfluenceEndpointConfig, resource: Resource) -> SourceResponse:+ return SourceResponse(+ name=config.name,+ items=lambda: resource,+ primary_keys=config.primary_keys,+ partition_count=1,+ partition_size=1,+ partition_mode="datetime" if config.partition_key else None,+ partition_format="week" if config.partition_key else None,+ partition_keys=[config.partition_key] if config.partition_key else None,+ column_hints=resource.column_hints,+ )+++def _fanout_source(+ config: ConfluenceEndpointConfig,+ subdomain: str,+ email: str,+ api_token: str,+ team_id: int,+ job_id: str,+) -> SourceResponse:+ assert config.fanout is not None++ # The helper types its return loosely, but it is the framework Resource it built.+ resource = cast(+ Resource,+ build_dependent_resource(+ endpoint_configs=CONFLUENCE_ENDPOINTS,+ child_endpoint=config.name,+ fanout=config.fanout,+ client_config=_client_config(subdomain, email, api_token),+ path_format_values={},+ team_id=team_id,+ job_id=job_id,+ # Permission endpoints have no server-side time filter, so the fan-out is always full refresh.+ db_incremental_field_last_value=None,+ parent_endpoint_extra={"data_selector": "results"},+ child_endpoint_extra={"data_selector": "results"},+ # No resume: the framework checkpoints the full list of completed parents after each+ # parent, which grows with every page or blog post on the site.+ ),+ )++ return _source_response(config, resource.add_map(_lift_principal))++def confluence_source(subdomain: str,email: str,@@ -89,15 +163,11 @@ def confluence_source(config = CONFLUENCE_ENDPOINTS[endpoint]subdomain = normalize_subdomain(subdomain)+ if config.fanout is not None:+ return _fanout_source(config, subdomain, email, api_token, team_id, job_id)+rest_config: RESTAPIConfig = {- "client": {- "base_url": _base_url(subdomain),- # Auth (HTTP Basic) is supplied via the framework auth config so the token is- # redacted from logs; only the non-secret Accept header is set here.- "headers": {"Accept": "application/json"},- "auth": {"type": "http_basic", "username": email, "password": api_token},- "paginator": ConfluenceLinkPaginator(_site_origin(subdomain)),- },+ "client": _client_config(subdomain, email, api_token),"resources": [{"name": endpoint,@@ -132,17 +202,7 @@ def confluence_source(initial_paginator_state=initial_paginator_state,)- return SourceResponse(- name=endpoint,- items=lambda: resource,- primary_keys=[config.primary_key],- partition_count=1,- partition_size=1,- partition_mode="datetime" if config.partition_key else None,- partition_format="week" if config.partition_key else None,- partition_keys=[config.partition_key] if config.partition_key else None,- column_hints=resource.column_hints,- )+ return _source_response(config, resource)def validate_credentials(diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/confluence/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/confluence/settings.pyindex 98a1d66d4fbc..0af56c8a1643 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/confluence/settings.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/confluence/settings.py@@ -1,24 +1,53 @@-from dataclasses import dataclass+from dataclasses import dataclass, fieldfrom typing import Optional+from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.fanout import (+ DependentEndpointConfig,+)from products.warehouse_sources.backend.types import IncrementalField-@dataclass+@dataclass(frozen=True)class ConfluenceEndpointConfig:name: strpath: str # Path relative to the v2 API base, e.g. "/pages"- primary_key: str = "id"+ primary_keys: list[str] = field(default_factory=lambda: ["id"])# Stable, immutable datetime field used for partitioning. Confluence's v2# objects expose `createdAt` at the top level (never use a mutable field# like the nested version timestamp). `None` for resources that don't carry# a top-level creation timestamp (labels, comments).partition_key: Optional[str] = Nonelimit: int = 250 # v2 list endpoints accept up to 250 results per page+ # Set for child resources listed per parent content item (e.g. permissions per page).+ fanout: Optional[DependentEndpointConfig] = None+ @property+ def page_size(self) -> int:+ return self.limit-# Confluence Cloud REST API v2 top-level list endpoints. All use cursor-based-# pagination (`_links.next`) and require no parent context to enumerate.+ @property+ def incremental_fields(self) -> list[IncrementalField]:+ return []++ @property+ def default_incremental_field(self) -> Optional[str]:+ return None+++def _permissions_fanout(parent_name: str, parent_id_column: str) -> DependentEndpointConfig:+ return DependentEndpointConfig(+ parent_name=parent_name,+ resolve_param="id",+ resolve_field="id",+ include_from_parent=["id"],+ parent_field_renames={"id": parent_id_column},+ # A page or blog post deleted between the parent listing and its permissions fetch 404s.+ child_response_actions=[{"status_code": 404, "action": "ignore"}],+ )+++# Confluence Cloud REST API v2 list endpoints. All use cursor-based pagination+# (`_links.next`). Endpoints with a `fanout` are listed once per parent content item.## We intentionally ship every endpoint as full-refresh: the v2 list endpoints# expose a `sort` parameter but NO server-side timestamp filter (there is no@@ -66,8 +95,26 @@ CONFLUENCE_ENDPOINTS: dict[str, ConfluenceEndpointConfig] = {name="inline_comments",path="/inline-comments",),+ # Direct read/update restrictions per content item. Databases, whiteboards and folders expose the+ # same endpoint but have no list endpoint to fan out from. Entries carry no id, so the key is the+ # content item plus the (operation, principal) pair.+ "page_permissions": ConfluenceEndpointConfig(+ name="page_permissions",+ path="/pages/{id}/permissions",+ primary_keys=["pageId", "operation", "principalType", "principalId"],+ fanout=_permissions_fanout("pages", "pageId"),+ ),+ "blogpost_permissions": ConfluenceEndpointConfig(+ name="blogpost_permissions",+ path="/blogposts/{id}/permissions",+ primary_keys=["blogPostId", "operation", "principalType", "principalId"],+ fanout=_permissions_fanout("blogposts", "blogPostId"),+ ),}+# Permission tables make one request per page or blog post, so they are opt-in.+SHOULD_SYNC_DEFAULT: dict[str, bool] = {"page_permissions": False, "blogpost_permissions": False}+ENDPOINTS = tuple(CONFLUENCE_ENDPOINTS.keys())# No endpoint supports server-side incremental filtering (see note above), so nodiff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/confluence/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/confluence/source.pyindex 9b4453e4c0c6..66dc7b89c6eb 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/confluence/source.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/confluence/source.py@@ -27,6 +27,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.confluencefrom products.warehouse_sources.backend.temporal.data_imports.sources.confluence.settings import (ENDPOINTS,INCREMENTAL_FIELDS,+ SHOULD_SYNC_DEFAULT,)from products.warehouse_sources.backend.temporal.data_imports.sources.generated_configs.confluence import (ConfluenceSourceConfig,@@ -117,7 +118,7 @@ Only Confluence Cloud sites (`your-domain.atlassian.net`) are supported.""",) -> list[SourceSchema]:# v2 list endpoints have no server-side timestamp filter, so every endpoint is# full refresh only (no incremental fields -> supports_incremental/append False).- return build_endpoint_schemas(ENDPOINTS, INCREMENTAL_FIELDS, names)+ return build_endpoint_schemas(ENDPOINTS, INCREMENTAL_FIELDS, names, should_sync_default=SHOULD_SYNC_DEFAULT)def validate_credentials(self,diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/confluence/tests/test_confluence.py b/products/warehouse_sources/backend/temporal/data_imports/sources/confluence/tests/test_confluence.pyindex 393103efb5ca..52f0f934c68c 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/confluence/tests/test_confluence.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/confluence/tests/test_confluence.py@@ -9,6 +9,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.tesScriptedResponse,SourceDriver,always,+ route,scripted_network,)from products.warehouse_sources.backend.temporal.data_imports.sources.confluence.confluence import (@@ -137,7 +138,7 @@ class TestConfluenceSource:assert response is not Noneconfig = CONFLUENCE_ENDPOINTS[endpoint]assert response.name == endpoint- assert response.primary_keys == [config.primary_key]+ assert response.primary_keys == config.primary_keysif config.partition_key:assert response.partition_mode == "datetime"assert response.partition_keys == [config.partition_key]@@ -226,3 +227,43 @@ class TestPagination:paginator.update_state(response)assert paginator.get_resume_state() == {"next_url": expected_url}+++def _permission(operation: str, principal_type: str, principal_id: str) -> dict[str, Any]:+ return {"operation": operation, "principal": {"principalType": principal_type, "principalId": principal_id}}+++class TestPermissionsFanout:+ @parameterized.expand(+ [+ ("page_permissions", "/wiki/api/v2/pages", "pageId"),+ ("blogpost_permissions", "/wiki/api/v2/blogposts", "blogPostId"),+ ]+ )+ def test_walks_each_parent_and_skips_deleted_ones(+ self, endpoint: str, parent_path: str, parent_column: str+ ) -> None:+ result = _driver().run(+ endpoint,+ route(+ {+ f"{parent_path}/1/permissions": [+ _response(+ [_permission("read", "user", "u1")],+ next_path=f"{parent_path}/1/permissions?cursor=c2",+ ),+ _response([_permission("update", "group", "g1")]),+ ],+ f"{parent_path}/2/permissions": [ScriptedResponse(status=404)],+ f"{parent_path}/3/permissions": [_response([_permission("read", "user", "u3")])],+ parent_path: [_response([{"id": "1"}, {"id": "2"}, {"id": "3"}])],+ }+ ),+ )++ assert result.raised is None+ assert [+ (row[parent_column], row["operation"], row["principalType"], row["principalId"]) for row in result.rows+ ] == [("1", "read", "user", "u1"), ("1", "update", "group", "g1"), ("3", "read", "user", "u3")]+ assert all("principal" not in row for row in result.rows)+ assert result.requests[2].param("cursor") == "c2"