Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(warehouse): add confluence page and blog post permission tables (#119240)

Tom Owers34a37dabc231
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
index 447501d2863a..fa689e51789b 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
@@ -3160,7 +3160,7 @@ Note: Source exposes 12 static endpoints (settings.py CONFIGCAT_ENDPOINTS) with
## Confluence — gaps
-Today (8): `attachments`, `blogposts`, `footer_comments`, `inline_comments`, `labels`, `pages`, `spaces`, `tasks`
+Today (10): `attachments`, `blogpost_permissions`, `blogposts`, `footer_comments`, `inline_comments`, `labels`, `page_permissions`, `pages`, `spaces`, `tasks`
Diffed against: <https://dac-static.atlassian.com/cloud/confluence/openapi-v2.v3.json>
@@ -3174,6 +3174,7 @@ Diffed against: <https://dac-static.atlassian.com/cloud/confluence/openapi-v2.v3
- [ ] `GET /pages/{id}/likes/count and /footer-comments/{id}/likes/count` — engagement counts per page and comment (low)
- [ ] `GET /whiteboards/{id}, /databases/{id}, /embeds/{id}, /folders/{id} (+ /descendants)` — the remaining v2 content types; reachable only via space/page descendant walks (low)
- [ ] `GET /classification-levels and /pages/{id}/classification-level` — data classification lookup for governance reporting (low)
+- [x] `GET /pages/{id}/permissions and /blogposts/{id}/permissions` — direct read/update restrictions per page and blog post; the database, whiteboard and folder variants are not synced because those content types have no list endpoint to fan out from (medium)
Note: Diffed against the v2 spec above; the users, groups, analytics and audit gaps come from the v1 spec at https://dac-static.atlassian.com/cloud/confluence/swagger.v3.json, since v2 has no user/analytics endpoints. Content types whiteboards/databases/embeds/folders exist in v2 only as fetch-by-id plus ancestors/descendants - there is no top-level list - so they are harder than the rest.
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/confluence/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/confluence/canonical_descriptions.py
index 59500b24876e..fd5ad639437d 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/confluence/canonical_descriptions.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/confluence/canonical_descriptions.py
@@ -10,6 +10,15 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.can
CanonicalDescriptions,
)
+# Columns of a direct content permission entry, shared by every content type's permissions endpoint.
+_PERMISSION_COLUMNS = {
+ "operation": "The operation the permission grants: read or update.",
+ "principalType": "Type of principal the permission applies to: user or group.",
+ "principalId": "ID of the principal. An Atlassian account ID for users, a group ID for groups.",
+ "createdAt": "Date and time the permission entry was created.",
+ "createdBy": "Atlassian account ID of the user who created the permission entry.",
+}
+
# Fields shared by most Confluence v2 content objects; merged into each entry so we don't repeat them.
_COMMON_COLUMNS = {
"id": "Unique identifier for the object.",
@@ -133,4 +142,14 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {
"version": "Version metadata for the comment.",
},
},
+ "page_permissions": {
+ "description": "A direct read or update restriction on a Confluence page, granted to one user or group.",
+ "docs_url": "https://developer.atlassian.com/cloud/confluence/rest/v2/api-group-content-permissions/",
+ "columns": {"pageId": "ID of the page the permission is on.", **_PERMISSION_COLUMNS},
+ },
+ "blogpost_permissions": {
+ "description": "A direct read or update restriction on a Confluence blog post, granted to one user or group.",
+ "docs_url": "https://developer.atlassian.com/cloud/confluence/rest/v2/api-group-content-permissions/",
+ "columns": {"blogPostId": "ID of the blog post the permission is on.", **_PERMISSION_COLUMNS},
+ },
}
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/confluence/confluence.py b/products/warehouse_sources/backend/temporal/data_imports/sources/confluence/confluence.py
index fb74ddc3c055..be2ced1673e3 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/confluence/confluence.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/confluence/confluence.py
@@ -1,7 +1,7 @@
import re
import base64
import dataclasses
-from typing import Any, Optional
+from typing import Any, Optional, cast
from requests import Response
@@ -10,13 +10,21 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.res
RESTAPIConfig,
rest_api_resource,
)
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.fanout import (
+ build_dependent_resource,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.paginators import (
JSONResponsePaginator,
)
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.resource import Resource
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.typing import ClientConfig
from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
from products.warehouse_sources.backend.temporal.data_imports.sources.common.source_helpers import validate_via_probe
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
-from products.warehouse_sources.backend.temporal.data_imports.sources.confluence.settings import CONFLUENCE_ENDPOINTS
+from products.warehouse_sources.backend.temporal.data_imports.sources.confluence.settings import (
+ CONFLUENCE_ENDPOINTS,
+ ConfluenceEndpointConfig,
+)
# Confluence Cloud sites always live under <subdomain>.atlassian.net. Building
# the host ourselves from a validated subdomain (rather than accepting an
@@ -25,7 +33,7 @@ _SUBDOMAIN_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9-]{0,62}$")
_ATLASSIAN_HOST_SUFFIX = ".atlassian.net"
[email protected](frozen=True)
class ConfluenceResumeConfig:
next_url: str
@@ -76,6 +84,72 @@ class ConfluenceLinkPaginator(JSONResponsePaginator):
self._next_url = f"{self._site_origin}{self._next_url}"
+def _client_config(subdomain: str, email: str, api_token: str) -> ClientConfig:
+ return {
+ "base_url": _base_url(subdomain),
+ # Auth (HTTP Basic) is supplied via the framework auth config so the token is
+ # redacted from logs; only the non-secret Accept header is set here.
+ "headers": {"Accept": "application/json"},
+ "auth": {"type": "http_basic", "username": email, "password": api_token},
+ "paginator": ConfluenceLinkPaginator(_site_origin(subdomain)),
+ }
+
+
+def _lift_principal(row: dict[str, Any]) -> dict[str, Any]:
+ # The primary key needs the principal at the top level.
+ principal = row.pop("principal", None) or {}
+ row["principalType"] = principal.get("principalType")
+ row["principalId"] = principal.get("principalId")
+ return row
+
+
+def _source_response(config: ConfluenceEndpointConfig, resource: Resource) -> SourceResponse:
+ return SourceResponse(
+ name=config.name,
+ items=lambda: resource,
+ primary_keys=config.primary_keys,
+ partition_count=1,
+ partition_size=1,
+ partition_mode="datetime" if config.partition_key else None,
+ partition_format="week" if config.partition_key else None,
+ partition_keys=[config.partition_key] if config.partition_key else None,
+ column_hints=resource.column_hints,
+ )
+
+
+def _fanout_source(
+ config: ConfluenceEndpointConfig,
+ subdomain: str,
+ email: str,
+ api_token: str,
+ team_id: int,
+ job_id: str,
+) -> SourceResponse:
+ assert config.fanout is not None
+
+ # The helper types its return loosely, but it is the framework Resource it built.
+ resource = cast(
+ Resource,
+ build_dependent_resource(
+ endpoint_configs=CONFLUENCE_ENDPOINTS,
+ child_endpoint=config.name,
+ fanout=config.fanout,
+ client_config=_client_config(subdomain, email, api_token),
+ path_format_values={},
+ team_id=team_id,
+ job_id=job_id,
+ # Permission endpoints have no server-side time filter, so the fan-out is always full refresh.
+ db_incremental_field_last_value=None,
+ parent_endpoint_extra={"data_selector": "results"},
+ child_endpoint_extra={"data_selector": "results"},
+ # No resume: the framework checkpoints the full list of completed parents after each
+ # parent, which grows with every page or blog post on the site.
+ ),
+ )
+
+ return _source_response(config, resource.add_map(_lift_principal))
+
+
def confluence_source(
subdomain: str,
email: str,
@@ -89,15 +163,11 @@ def confluence_source(
config = CONFLUENCE_ENDPOINTS[endpoint]
subdomain = normalize_subdomain(subdomain)
+ if config.fanout is not None:
+ return _fanout_source(config, subdomain, email, api_token, team_id, job_id)
+
rest_config: RESTAPIConfig = {
- "client": {
- "base_url": _base_url(subdomain),
- # Auth (HTTP Basic) is supplied via the framework auth config so the token is
- # redacted from logs; only the non-secret Accept header is set here.
- "headers": {"Accept": "application/json"},
- "auth": {"type": "http_basic", "username": email, "password": api_token},
- "paginator": ConfluenceLinkPaginator(_site_origin(subdomain)),
- },
+ "client": _client_config(subdomain, email, api_token),
"resources": [
{
"name": endpoint,
@@ -132,17 +202,7 @@ def confluence_source(
initial_paginator_state=initial_paginator_state,
)
- return SourceResponse(
- name=endpoint,
- items=lambda: resource,
- primary_keys=[config.primary_key],
- partition_count=1,
- partition_size=1,
- partition_mode="datetime" if config.partition_key else None,
- partition_format="week" if config.partition_key else None,
- partition_keys=[config.partition_key] if config.partition_key else None,
- column_hints=resource.column_hints,
- )
+ return _source_response(config, resource)
def validate_credentials(
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/confluence/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/confluence/settings.py
index 98a1d66d4fbc..0af56c8a1643 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/confluence/settings.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/confluence/settings.py
@@ -1,24 +1,53 @@
-from dataclasses import dataclass
+from dataclasses import dataclass, field
from typing import Optional
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.fanout import (
+ DependentEndpointConfig,
+)
from products.warehouse_sources.backend.types import IncrementalField
-@dataclass
+@dataclass(frozen=True)
class ConfluenceEndpointConfig:
name: str
path: str # Path relative to the v2 API base, e.g. "/pages"
- primary_key: str = "id"
+ primary_keys: list[str] = field(default_factory=lambda: ["id"])
# Stable, immutable datetime field used for partitioning. Confluence's v2
# objects expose `createdAt` at the top level (never use a mutable field
# like the nested version timestamp). `None` for resources that don't carry
# a top-level creation timestamp (labels, comments).
partition_key: Optional[str] = None
limit: int = 250 # v2 list endpoints accept up to 250 results per page
+ # Set for child resources listed per parent content item (e.g. permissions per page).
+ fanout: Optional[DependentEndpointConfig] = None
+ @property
+ def page_size(self) -> int:
+ return self.limit
-# Confluence Cloud REST API v2 top-level list endpoints. All use cursor-based
-# pagination (`_links.next`) and require no parent context to enumerate.
+ @property
+ def incremental_fields(self) -> list[IncrementalField]:
+ return []
+
+ @property
+ def default_incremental_field(self) -> Optional[str]:
+ return None
+
+
+def _permissions_fanout(parent_name: str, parent_id_column: str) -> DependentEndpointConfig:
+ return DependentEndpointConfig(
+ parent_name=parent_name,
+ resolve_param="id",
+ resolve_field="id",
+ include_from_parent=["id"],
+ parent_field_renames={"id": parent_id_column},
+ # A page or blog post deleted between the parent listing and its permissions fetch 404s.
+ child_response_actions=[{"status_code": 404, "action": "ignore"}],
+ )
+
+
+# Confluence Cloud REST API v2 list endpoints. All use cursor-based pagination
+# (`_links.next`). Endpoints with a `fanout` are listed once per parent content item.
#
# We intentionally ship every endpoint as full-refresh: the v2 list endpoints
# expose a `sort` parameter but NO server-side timestamp filter (there is no
@@ -66,8 +95,26 @@ CONFLUENCE_ENDPOINTS: dict[str, ConfluenceEndpointConfig] = {
name="inline_comments",
path="/inline-comments",
),
+ # Direct read/update restrictions per content item. Databases, whiteboards and folders expose the
+ # same endpoint but have no list endpoint to fan out from. Entries carry no id, so the key is the
+ # content item plus the (operation, principal) pair.
+ "page_permissions": ConfluenceEndpointConfig(
+ name="page_permissions",
+ path="/pages/{id}/permissions",
+ primary_keys=["pageId", "operation", "principalType", "principalId"],
+ fanout=_permissions_fanout("pages", "pageId"),
+ ),
+ "blogpost_permissions": ConfluenceEndpointConfig(
+ name="blogpost_permissions",
+ path="/blogposts/{id}/permissions",
+ primary_keys=["blogPostId", "operation", "principalType", "principalId"],
+ fanout=_permissions_fanout("blogposts", "blogPostId"),
+ ),
}
+# Permission tables make one request per page or blog post, so they are opt-in.
+SHOULD_SYNC_DEFAULT: dict[str, bool] = {"page_permissions": False, "blogpost_permissions": False}
+
ENDPOINTS = tuple(CONFLUENCE_ENDPOINTS.keys())
# No endpoint supports server-side incremental filtering (see note above), so no
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/confluence/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/confluence/source.py
index 9b4453e4c0c6..66dc7b89c6eb 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/confluence/source.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/confluence/source.py
@@ -27,6 +27,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.confluence
from products.warehouse_sources.backend.temporal.data_imports.sources.confluence.settings import (
ENDPOINTS,
INCREMENTAL_FIELDS,
+ SHOULD_SYNC_DEFAULT,
)
from products.warehouse_sources.backend.temporal.data_imports.sources.generated_configs.confluence import (
ConfluenceSourceConfig,
@@ -117,7 +118,7 @@ Only Confluence Cloud sites (`your-domain.atlassian.net`) are supported.""",
) -> list[SourceSchema]:
# v2 list endpoints have no server-side timestamp filter, so every endpoint is
# full refresh only (no incremental fields -> supports_incremental/append False).
- return build_endpoint_schemas(ENDPOINTS, INCREMENTAL_FIELDS, names)
+ return build_endpoint_schemas(ENDPOINTS, INCREMENTAL_FIELDS, names, should_sync_default=SHOULD_SYNC_DEFAULT)
def validate_credentials(
self,
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/confluence/tests/test_confluence.py b/products/warehouse_sources/backend/temporal/data_imports/sources/confluence/tests/test_confluence.py
index 393103efb5ca..52f0f934c68c 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/confluence/tests/test_confluence.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/confluence/tests/test_confluence.py
@@ -9,6 +9,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.tes
ScriptedResponse,
SourceDriver,
always,
+ route,
scripted_network,
)
from products.warehouse_sources.backend.temporal.data_imports.sources.confluence.confluence import (
@@ -137,7 +138,7 @@ class TestConfluenceSource:
assert response is not None
config = CONFLUENCE_ENDPOINTS[endpoint]
assert response.name == endpoint
- assert response.primary_keys == [config.primary_key]
+ assert response.primary_keys == config.primary_keys
if config.partition_key:
assert response.partition_mode == "datetime"
assert response.partition_keys == [config.partition_key]
@@ -226,3 +227,43 @@ class TestPagination:
paginator.update_state(response)
assert paginator.get_resume_state() == {"next_url": expected_url}
+
+
+def _permission(operation: str, principal_type: str, principal_id: str) -> dict[str, Any]:
+ return {"operation": operation, "principal": {"principalType": principal_type, "principalId": principal_id}}
+
+
+class TestPermissionsFanout:
+ @parameterized.expand(
+ [
+ ("page_permissions", "/wiki/api/v2/pages", "pageId"),
+ ("blogpost_permissions", "/wiki/api/v2/blogposts", "blogPostId"),
+ ]
+ )
+ def test_walks_each_parent_and_skips_deleted_ones(
+ self, endpoint: str, parent_path: str, parent_column: str
+ ) -> None:
+ result = _driver().run(
+ endpoint,
+ route(
+ {
+ f"{parent_path}/1/permissions": [
+ _response(
+ [_permission("read", "user", "u1")],
+ next_path=f"{parent_path}/1/permissions?cursor=c2",
+ ),
+ _response([_permission("update", "group", "g1")]),
+ ],
+ f"{parent_path}/2/permissions": [ScriptedResponse(status=404)],
+ f"{parent_path}/3/permissions": [_response([_permission("read", "user", "u3")])],
+ parent_path: [_response([{"id": "1"}, {"id": "2"}, {"id": "3"}])],
+ }
+ ),
+ )
+
+ assert result.raised is None
+ assert [
+ (row[parent_column], row["operation"], row["principalType"], row["principalId"]) for row in result.rows
+ ] == [("1", "read", "user", "u1"), ("1", "update", "group", "g1"), ("3", "read", "user", "u3")]
+ assert all("principal" not in row for row in result.rows)
+ assert result.requests[2].param("cursor") == "c2"