Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(warehouse): add omni models, schedule recipients, document permissions, user attributes tables (#123167)

Tom Owers3902836e6f05
diff --git a/posthog/test/repo_invariants/dataclass_frozen_baseline.txt b/posthog/test/repo_invariants/dataclass_frozen_baseline.txt
index ba4d445ed822..286cfbe95fad 100644
--- a/posthog/test/repo_invariants/dataclass_frozen_baseline.txt
+++ b/posthog/test/repo_invariants/dataclass_frozen_baseline.txt
@@ -808,7 +808,6 @@
1 products/warehouse_sources/backend/temporal/data_imports/sources/okendo/settings.py
1 products/warehouse_sources/backend/temporal/data_imports/sources/okta/okta.py
1 products/warehouse_sources/backend/temporal/data_imports/sources/okta/settings.py
-1 products/warehouse_sources/backend/temporal/data_imports/sources/omni/omni.py
1 products/warehouse_sources/backend/temporal/data_imports/sources/omnisend/omnisend.py
1 products/warehouse_sources/backend/temporal/data_imports/sources/omnisend/settings.py
1 products/warehouse_sources/backend/temporal/data_imports/sources/onepagecrm/onepagecrm.py
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
index 182c1e63a205..c45f306a0d73 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
@@ -8608,13 +8608,13 @@ Today (6): `Connections`, `Documents`, `Folders`, `Schedules`, `UserGroups`, `Us
Diffed against: <https://docs.omni.co/sitemap.xml>
-- [ ] `models (GET /api/v1/models — list-models, list-model-schemas)` — Lookup resolving the model id carried by documents, queries and schedules; nothing in the current tables identifies which data model content sits on (high)
-- [ ] `schedule-recipients (list-schedule-recipients)` — Junction between the Schedules table we already sync and the users/groups actually receiving each delivery (high)
-- [ ] `document-permissions (get-document-permissions, all-users-and-groups-with-document-access)` — Who can see which dashboard — the access-audit join across Documents, Users and UserGroups (high)
+- [x] `models (GET /api/v1/models — list-models, list-model-schemas)` — Lookup resolving the model id carried by documents, queries and schedules; nothing in the current tables identifies which data model content sits on (high)
+- [x] `schedule-recipients (list-schedule-recipients)` — Junction between the Schedules table we already sync and the users/groups actually receiving each delivery (high)
+- [x] `document-permissions (get-document-permissions, all-users-and-groups-with-document-access)` — Who can see which dashboard — the access-audit join across Documents, Users and UserGroups (high)
- [ ] `ai conversations (list-ai-conversations, get-conversation)` — AI query usage per user, the closest thing Omni's API offers to consumption analytics (medium)
- [ ] `labels (list-labels)` — Lookup for the labels applied to documents; without it label ids on content are unresolvable (medium)
- [ ] `user-model-roles and user-group-model-roles (retrieve endpoints)` — Per-model permission grants for users and groups — governance reporting over models (medium)
-- [ ] `user-attributes (list-user-attributes)` — Lookup for the attribute keys used in row-level access controls referenced on users (medium)
+- [x] `user-attributes (list-user-attributes)` — Lookup for the attribute keys used in row-level access controls referenced on users (medium)
- [ ] `folder-permissions (get-folder-permissions)` — Folder-level access grants, the inherited layer above document permissions on the Folders table we sync (medium)
- [ ] `document-favorites (list-document-favoriters)` — Per-document favorite counts and favoriters — an engagement signal on content (medium)
- [ ] `embed users and email-only users (list-embed-users, list-email-only-users)` — Two user populations excluded from the main Users table, so seat and delivery counts are understated (medium)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/omni/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/omni/canonical_descriptions.py
index a10ec5a9ddc7..786c90d918e1 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/omni/canonical_descriptions.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/omni/canonical_descriptions.py
@@ -94,4 +94,68 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {
"meta": "Resource metadata, including `created` and `lastModified` timestamps.",
},
},
+ "Models": {
+ "description": "Semantic models in your Omni instance, including schema, shared, workbook, and branch models.",
+ "docs_url": "https://docs.omni.co/api/models/list-models",
+ "columns": {
+ "id": "ID of the model.",
+ "baseModelId": "ID of the base model, if applicable.",
+ "connectionId": "ID of the connection the model is based on.",
+ "modelKind": "The type of model (e.g. `SCHEMA`, `SHARED`, `WORKBOOK`, `BRANCH`).",
+ "name": "The name of the model.",
+ "createdAt": "The time the model was created.",
+ "updatedAt": "The time the model was last updated.",
+ "deletedAt": "The time the model was deleted, if applicable.",
+ },
+ },
+ "ScheduleRecipients": {
+ "description": "Delivery destination and recipients of each schedule, one row per schedule.",
+ "docs_url": "https://docs.omni.co/api/schedule-recipients/list-schedule-recipients",
+ "columns": {
+ "scheduleId": "ID of the schedule the recipients belong to.",
+ "type": "The destination type: `email`, `slack`, `webhook`, `sftp`, or `s3`.",
+ "recipients": "Email recipients (id, name, email, emailOnly) or Slack recipients (recipientType, slackId), depending on the destination type.",
+ "userGroupRecipients": "Email recipients added as members of a user group, grouped by user group.",
+ "url": "The webhook URL, for webhook destinations.",
+ "address": "The SFTP server address, for SFTP destinations.",
+ "port": "The SFTP server port, for SFTP destinations.",
+ "username": "The SFTP username, for SFTP destinations.",
+ "bucketName": "The target S3 bucket, for S3 destinations.",
+ "region": "The AWS region of the S3 bucket, for S3 destinations.",
+ "keyPrefix": "Folder path prefix for the uploaded file, for S3 destinations.",
+ "filename": "Filename template for the uploaded file, for S3 destinations.",
+ },
+ },
+ "DocumentPermissions": {
+ "description": "Users and user groups with access to each document, one row per access grant.",
+ "docs_url": "https://docs.omni.co/api/document-permissions/list-all-users-and-groups-with-document-access",
+ "columns": {
+ "documentId": "Identifier of the document the access applies to.",
+ "principalId": "ID of the user or user group with access (`userId` or `userGroupId`).",
+ "userId": "The ID of the user. Only present when `type` is `user`.",
+ "userGroupId": "The ID of the user group. Only present when `type` is `userGroup`.",
+ "name": "Display name of the user or user group.",
+ "email": "Email address. Only present for users.",
+ "type": "The type of principal: `user` or `userGroup`.",
+ "role": "Permission level: `VIEWER`, `INTERACTOR`, `EDITOR`, `MANAGER`, or `OWNER`.",
+ "accessBoost": "Whether elevated access is enabled for this principal.",
+ "accessSource": "How access was granted: `direct` or inherited from a `folder`.",
+ "isOwner": "Whether this user owns the document. Only present for users.",
+ "folderInfo": "The folder that grants access (id, name, path). Only present when `accessSource` is `folder`.",
+ },
+ },
+ "UserAttributes": {
+ "description": "User attribute definitions in your Omni organization, used for row-level security and personalization.",
+ "docs_url": "https://docs.omni.co/api/user-attributes/list-user-attributes",
+ "columns": {
+ "id": "Unique identifier for custom attributes. Empty for system-defined attributes.",
+ "name": "Reference name used in model SQL and in embed SSO URL parameters. Unique within the organization.",
+ "label": "Display name shown in the Omni UI.",
+ "description": "Description of the attribute and its purpose.",
+ "type": "Data type of the attribute: `String` or `Number`.",
+ "multiple_values": "Whether the attribute accepts an array of values.",
+ "default_value": "Default value applied when no user-specific value is set.",
+ "system": "Whether the attribute is system-defined and read-only.",
+ },
+ },
}
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/omni/omni.py b/products/warehouse_sources/backend/temporal/data_imports/sources/omni/omni.py
index 96b2d1599b85..1163dff087b2 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/omni/omni.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/omni/omni.py
@@ -1,5 +1,6 @@
import re
import dataclasses
+from collections.abc import Callable
from datetime import UTC, date, datetime
from typing import Any, Optional
from urllib.parse import urlparse
@@ -11,12 +12,18 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.mix
from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source import (
RESTAPIConfig,
rest_api_resource,
+ rest_api_resources,
+)
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.fanout import (
+ rename_parent_fields,
)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.paginators import (
BasePaginator,
SinglePagePaginator,
)
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.resource import Resource
from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.typing import (
+ ClientConfig,
Endpoint,
EndpointResource,
)
@@ -25,6 +32,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.sou
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
from products.warehouse_sources.backend.temporal.data_imports.sources.omni.settings import (
DEFAULT_PAGE_SIZE,
+ FANOUT_ENDPOINTS,
PARTITION_KEYS,
PRIMARY_KEYS,
SCIM_ENDPOINTS,
@@ -37,12 +45,14 @@ LOOPBACK_HOSTS = {"localhost", "127.0.0.1", "::1"}
HOST_NOT_ALLOWED_ERROR = "Omni host is not allowed"
[email protected](frozen=True)
class OmniResumeConfig:
# Documents/Folders/Schedules resume via an opaque `nextCursor`; the SCIM Users/UserGroups
- # endpoints resume via a numeric `startIndex`. Only one is ever populated for a given endpoint.
+ # endpoints resume via a numeric `startIndex`; fan-out endpoints resume via the framework's
+ # per-parent state. Only one is ever populated for a given endpoint.
cursor: Optional[str] = None
start_index: Optional[int] = None
+ fanout_state: Optional[dict[str, Any]] = None
def normalize_host(host: str) -> str:
@@ -298,9 +308,120 @@ def get_resource(
"table_format": "delta",
}
+ if name == "Models":
+ endpoint = {
+ "path": "/v1/models",
+ "params": {"sortField": "createdAt", "sortDirection": "asc"},
+ "paginator": OmniPageInfoPaginator(),
+ "data_selector": "records",
+ }
+ return {
+ "name": name,
+ "table_name": "models",
+ "write_disposition": "replace",
+ "endpoint": endpoint,
+ "table_format": "delta",
+ }
+
+ if name == "UserAttributes":
+ endpoint = {
+ "path": "/v1/user-attributes",
+ "paginator": SinglePagePaginator(),
+ "data_selector": "records",
+ }
+ return {
+ "name": name,
+ "table_name": "user_attributes",
+ "write_disposition": "replace",
+ "endpoint": endpoint,
+ "table_format": "delta",
+ }
+
+ if name == "ScheduleRecipients":
+ # The response is one object per schedule describing its destination, not a list.
+ endpoint = {
+ "path": "/v1/schedules/{scheduleId}/recipients",
+ "paginator": SinglePagePaginator(),
+ }
+ return {
+ "name": name,
+ "table_name": "schedule_recipients",
+ "write_disposition": "replace",
+ "endpoint": endpoint,
+ "table_format": "delta",
+ }
+
+ if name == "DocumentPermissions":
+ endpoint = {
+ "path": "/v1/documents/{documentId}/access-list",
+ "params": {"sortField": "name", "sortDirection": "asc"},
+ "paginator": OmniPageInfoPaginator(),
+ "data_selector": "principals",
+ }
+ return {
+ "name": name,
+ "table_name": "document_permissions",
+ "write_disposition": "replace",
+ "endpoint": endpoint,
+ "table_format": "delta",
+ }
+
raise ValueError(f"Unknown Omni endpoint: {name}")
+def _add_principal_id(row: dict[str, Any]) -> dict[str, Any]:
+ # Users carry `userId` and groups carry `userGroupId`; the combined `id` is deprecated.
+ row["principalId"] = row.get("userId") or row.get("userGroupId") or row.get("id")
+ return row
+
+
+def _fanout_resource(
+ endpoint: str,
+ client_config: ClientConfig,
+ team_id: int,
+ job_id: str,
+ resume_hook: Callable[[Optional[dict[str, Any]]], None],
+ initial_paginator_state: Optional[dict[str, Any]],
+) -> Resource:
+ fanout = FANOUT_ENDPOINTS[endpoint]
+
+ parent_resource = get_resource(fanout.parent_name, False, None)
+ parent_endpoint = parent_resource["endpoint"]
+ assert isinstance(parent_endpoint, dict)
+ if fanout.parent_params:
+ parent_endpoint["params"] = {**(parent_endpoint.get("params") or {}), **fanout.parent_params}
+
+ child_resource = get_resource(endpoint, False, None)
+ child_resource["include_from_parent"] = fanout.include_from_parent
+ child_endpoint = child_resource["endpoint"]
+ assert isinstance(child_endpoint, dict)
+ child_endpoint["params"] = {
+ **(child_endpoint.get("params") or {}),
+ fanout.resolve_param: {"type": "resolve", "resource": fanout.parent_name, "field": fanout.resolve_field},
+ }
+ if fanout.child_response_actions:
+ child_endpoint["response_actions"] = fanout.child_response_actions
+
+ config: RESTAPIConfig = {
+ "client": client_config,
+ "resource_defaults": {},
+ "resources": [parent_resource, child_resource],
+ }
+ resources = rest_api_resources(
+ config,
+ team_id,
+ job_id,
+ None,
+ resume_hook=resume_hook,
+ initial_paginator_state=initial_paginator_state,
+ )
+ child = next(r for r in resources if getattr(r, "name", None) == endpoint)
+ child = child.add_map(rename_parent_fields(fanout.parent_name, fanout.parent_field_renames))
+ if endpoint == "DocumentPermissions":
+ child = child.add_map(_add_principal_id)
+ return child
+
+
def validate_credentials(
host: str, api_key: str, team_id: int, schema_name: Optional[str] = None
) -> tuple[bool, str | None]:
@@ -398,36 +519,37 @@ def omni_source(
if not host_ok:
raise ValueError(host_err or HOST_NOT_ALLOWED_ERROR)
- stop_when_older_than = _format_watermark(db_incremental_field_last_value) if should_use_incremental_field else None
- resource_config = get_resource(endpoint, should_use_incremental_field, stop_when_older_than)
-
- config: RESTAPIConfig = {
- "client": {
- "base_url": _base_api_url(host),
- "auth": {"type": "bearer", "token": api_key},
- "headers": {"Accept": "application/json"},
- # SSRF defense in depth: pin every request to the validated host and never follow a
- # 3xx, so a redirect can't bounce the bearer token to a private/metadata address.
- "allowed_hosts": [hostname],
- "allow_redirects": False,
- },
- "resources": [resource_config],
+ client_config: ClientConfig = {
+ "base_url": _base_api_url(host),
+ "auth": {"type": "bearer", "token": api_key},
+ "headers": {"Accept": "application/json"},
+ # SSRF defense in depth: pin every request to the validated host and never follow a
+ # 3xx, so a redirect can't bounce the bearer token to a private/metadata address.
+ "allowed_hosts": [hostname],
+ "allow_redirects": False,
}
+ is_fanout = endpoint in FANOUT_ENDPOINTS
+
initial_paginator_state: Optional[dict[str, Any]] = None
if resumable_source_manager.can_resume():
resume_config = resumable_source_manager.load_state()
if resume_config is not None:
- if resume_config.cursor:
+ if is_fanout:
+ initial_paginator_state = resume_config.fanout_state
+ elif resume_config.cursor:
initial_paginator_state = {"cursor": resume_config.cursor}
elif resume_config.start_index:
initial_paginator_state = {"start_index": resume_config.start_index}
def save_checkpoint(state: Optional[dict[str, Any]]) -> None:
- # Only Documents/Folders/Schedules/Users/UserGroups paginate; Connections uses
- # SinglePagePaginator and never reports a resume state, so this is a no-op for it.
+ # Connections and UserAttributes use SinglePagePaginator and never report a resume state,
+ # so this is a no-op for them.
if not state:
return
+ if is_fanout:
+ resumable_source_manager.save_state(OmniResumeConfig(fanout_state=state))
+ return
cursor = state.get("cursor")
start_index = state.get("start_index")
if cursor:
@@ -435,17 +557,35 @@ def omni_source(
elif start_index:
resumable_source_manager.save_state(OmniResumeConfig(start_index=int(start_index)))
- resource = rest_api_resource(
- config,
- team_id,
- job_id,
- # Incremental filtering is applied via the custom paginator's `stop_when_older_than`
- # above, not the framework's declarative `incremental` param — no endpoint here declares
- # one, so the framework has nothing to inject this into.
- None,
- resume_hook=save_checkpoint,
- initial_paginator_state=initial_paginator_state,
- )
+ resource: Resource
+ if is_fanout:
+ resource = _fanout_resource(
+ endpoint,
+ client_config,
+ team_id,
+ job_id,
+ resume_hook=save_checkpoint,
+ initial_paginator_state=initial_paginator_state,
+ )
+ else:
+ stop_when_older_than = (
+ _format_watermark(db_incremental_field_last_value) if should_use_incremental_field else None
+ )
+ config: RESTAPIConfig = {
+ "client": client_config,
+ "resources": [get_resource(endpoint, should_use_incremental_field, stop_when_older_than)],
+ }
+ resource = rest_api_resource(
+ config,
+ team_id,
+ job_id,
+ # Incremental filtering is applied via the custom paginator's `stop_when_older_than`
+ # above, not the framework's declarative `incremental` param — no endpoint here declares
+ # one, so the framework has nothing to inject this into.
+ None,
+ resume_hook=save_checkpoint,
+ initial_paginator_state=initial_paginator_state,
+ )
partition_key = PARTITION_KEYS.get(endpoint)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/omni/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/omni/settings.py
index a21ead4d0ba4..95e0aabb1a2e 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/omni/settings.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/omni/settings.py
@@ -1,3 +1,6 @@
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.fanout import (
+ DependentEndpointConfig,
+)
from products.warehouse_sources.backend.types import IncrementalField, IncrementalFieldType
ENDPOINTS = (
@@ -7,6 +10,10 @@ ENDPOINTS = (
"Schedules",
"Users",
"UserGroups",
+ "Models",
+ "ScheduleRecipients",
+ "DocumentPermissions",
+ "UserAttributes",
)
# Only Documents exposes anything close to a timestamp cursor (`sortField=updatedAt`, no
@@ -33,6 +40,12 @@ PRIMARY_KEYS: dict[str, list[str]] = {
"Schedules": ["id"],
"Users": ["id"],
"UserGroups": ["id"],
+ "Models": ["id"],
+ "ScheduleRecipients": ["scheduleId"],
+ # A principal appears twice when it has both direct and folder-inherited access to a document.
+ "DocumentPermissions": ["documentId", "principalId", "accessSource"],
+ # System-defined attributes return an empty `id`; `name` is unique within the organization.
+ "UserAttributes": ["name"],
}
# A stable field to partition on — never the incremental cursor (`updatedAt` changes on every
@@ -46,6 +59,10 @@ PARTITION_KEYS: dict[str, str | None] = {
"Schedules": None,
"Users": None,
"UserGroups": None,
+ "Models": "createdAt",
+ "ScheduleRecipients": None,
+ "DocumentPermissions": None,
+ "UserAttributes": None,
}
DEFAULT_PAGE_SIZE = 100
@@ -53,3 +70,31 @@ DEFAULT_PAGE_SIZE = 100
# SCIM users/groups endpoints require an Organization API key (Personal Access Tokens are
# explicitly documented as unsupported for them).
SCIM_ENDPOINTS = frozenset({"Users", "UserGroups"})
+
+# Child endpoints fetched once per parent row. The parent's sort is pinned to a field that rarely
+# changes so cursor pagination doesn't skip rows edited mid-walk (Documents otherwise sorts by
+# `updatedAt`).
+FANOUT_ENDPOINTS: dict[str, DependentEndpointConfig] = {
+ "ScheduleRecipients": DependentEndpointConfig(
+ parent_name="Schedules",
+ resolve_param="scheduleId",
+ resolve_field="id",
+ include_from_parent=["id"],
+ parent_field_renames={"id": "scheduleId"},
+ child_response_actions=[{"status_code": 404, "action": "ignore"}],
+ ),
+ "DocumentPermissions": DependentEndpointConfig(
+ parent_name="Documents",
+ resolve_param="documentId",
+ resolve_field="identifier",
+ include_from_parent=["identifier"],
+ parent_field_renames={"identifier": "documentId"},
+ parent_params={"sortField": "name", "sortDirection": "asc"},
+ # The access list needs Manager permission on each document, so a key that can view a
+ # document but not manage it gets a 403 for that document only.
+ child_response_actions=[
+ {"status_code": 403, "action": "ignore"},
+ {"status_code": 404, "action": "ignore"},
+ ],
+ ),
+}
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/omni/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/omni/source.py
index c193bd30c72a..d64d4118407b 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/omni/source.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/omni/source.py
@@ -112,7 +112,7 @@ class OmniSource(ResumableSource[OmniSourceConfig, OmniResumeConfig]):
category=DataWarehouseSourceCategory.ANALYTICS,
label="Omni Analytics",
releaseStatus=ReleaseStatus.BETA,
- caption="Sync documents, folders, connections, schedules, users, and user groups from your Omni instance.\n\nUse an **Organization API key** for full coverage (**Settings > API access**) — Personal Access Tokens only see content the token owner can access, and can't read the Users or User groups tables at all.",
+ caption="Sync documents, folders, connections, schedules, models, users, user groups, and access settings from your Omni instance.\n\nUse an **Organization API key** for full coverage (**Settings > API access**) — Personal Access Tokens only see content the token owner can access, and can't read the Users or User groups tables at all.",
docsUrl="https://posthog.com/docs/cdp/sources/omni",
iconPath="/static/services/omni.png",
keywords=["bi", "business intelligence"],
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/omni/tests/test_omni.py b/products/warehouse_sources/backend/temporal/data_imports/sources/omni/tests/test_omni.py
index 7a37fe6cf4b1..f4851f241a81 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/omni/tests/test_omni.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/omni/tests/test_omni.py
@@ -10,6 +10,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.tes
ScriptedResponse,
SourceDriver,
always,
+ route,
scripted_network,
)
from products.warehouse_sources.backend.temporal.data_imports.sources.generated_configs.omni import OmniSourceConfig
@@ -24,7 +25,11 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.omni.omni
normalize_host,
validate_credentials,
)
-from products.warehouse_sources.backend.temporal.data_imports.sources.omni.settings import PARTITION_KEYS, PRIMARY_KEYS
+from products.warehouse_sources.backend.temporal.data_imports.sources.omni.settings import (
+ FANOUT_ENDPOINTS,
+ PARTITION_KEYS,
+ PRIMARY_KEYS,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.omni.source import OmniSource
@@ -235,7 +240,115 @@ class TestOmniSourceEndToEnd:
assert result.params("startIndex") == ["1", "2"]
assert result.saved_states == [OmniResumeConfig(start_index=2)]
- @pytest.mark.parametrize("endpoint", list(PRIMARY_KEYS.keys()))
+ def test_schedule_recipients_fans_out_per_schedule(self) -> None:
+ result = self._driver().run(
+ "ScheduleRecipients",
+ route(
+ {
+ "/api/v1/schedules": [
+ ScriptedResponse(
+ json={
+ "pageInfo": {"hasNextPage": False, "nextCursor": None},
+ "records": [{"id": "s1"}, {"id": "s2"}],
+ }
+ )
+ ],
+ "/api/v1/schedules/s1/recipients": [
+ ScriptedResponse(json={"type": "email", "recipients": [{"id": "u1", "email": "[email protected]"}]})
+ ],
+ # A schedule deleted between the listing and its recipients fetch is skipped.
+ "/api/v1/schedules/s2/recipients": [ScriptedResponse(status=404, json={"detail": "not found"})],
+ }
+ ),
+ )
+
+ assert result.raised is None
+ assert result.rows == [
+ {"type": "email", "recipients": [{"id": "u1", "email": "[email protected]"}], "scheduleId": "s1"}
+ ]
+ assert result.response is not None
+ assert result.response.primary_keys == ["scheduleId"]
+
+ def test_document_permissions_paginates_per_document_and_keys_principals(self) -> None:
+ result = self._driver().run(
+ "DocumentPermissions",
+ route(
+ {
+ "/api/v1/documents": [
+ ScriptedResponse(
+ json={
+ "pageInfo": {"hasNextPage": False, "nextCursor": None},
+ "records": [{"identifier": "d1"}, {"identifier": "d2"}],
+ }
+ )
+ ],
+ "/api/v1/documents/d1/access-list": [
+ ScriptedResponse(
+ json={
+ "pageInfo": {"hasNextPage": True, "nextCursor": "p2"},
+ "principals": [{"userId": "u1", "type": "user", "accessSource": "direct"}],
+ }
+ ),
+ ScriptedResponse(
+ json={
+ "pageInfo": {"hasNextPage": False, "nextCursor": None},
+ "principals": [{"userGroupId": "g1", "type": "userGroup", "accessSource": "folder"}],
+ }
+ ),
+ ],
+ # The key lacks Manager permission on this document; it's skipped, not fatal.
+ "/api/v1/documents/d2/access-list": [ScriptedResponse(status=403, json={"status": 403})],
+ }
+ ),
+ )
+
+ assert result.raised is None
+ assert [(row["documentId"], row["principalId"], row["accessSource"]) for row in result.rows] == [
+ ("d1", "u1", "direct"),
+ ("d1", "g1", "folder"),
+ ]
+ # The parent listing sorts on a field that edits don't move, unlike the Documents table's `updatedAt`.
+ assert result.requests[0].path == "/api/v1/documents"
+ assert result.requests[0].param("sortField") == "name"
+ assert result.params("cursor") == [None, None, "p2", None]
+
+ def test_document_permissions_resume_skips_completed_documents(self) -> None:
+ result = self._driver().run(
+ "DocumentPermissions",
+ route(
+ {
+ "/api/v1/documents": [
+ ScriptedResponse(
+ json={
+ "pageInfo": {"hasNextPage": False, "nextCursor": None},
+ "records": [{"identifier": "d1"}, {"identifier": "d2"}],
+ }
+ )
+ ],
+ "/api/v1/documents/d2/access-list": [
+ ScriptedResponse(
+ json={
+ "pageInfo": {"hasNextPage": False, "nextCursor": None},
+ "principals": [{"userId": "u2", "type": "user", "accessSource": "direct"}],
+ }
+ )
+ ],
+ }
+ ),
+ resume_state=OmniResumeConfig(
+ fanout_state={"completed": ["/v1/documents/d1/access-list"], "current": None, "child_state": None}
+ ),
+ )
+
+ assert result.raised is None
+ assert result.paths == ["/api/v1/documents", "/api/v1/documents/d2/access-list"]
+ assert [row["documentId"] for row in result.rows] == ["d2"]
+ assert result.final_state.fanout_state["completed"] == [
+ "/v1/documents/d1/access-list",
+ "/v1/documents/d2/access-list",
+ ]
+
+ @pytest.mark.parametrize("endpoint", [name for name in PRIMARY_KEYS if name not in FANOUT_ENDPOINTS])
def test_response_metadata_per_endpoint(self, endpoint: str) -> None:
selector = {
"Documents": "records",
@@ -244,6 +357,8 @@ class TestOmniSourceEndToEnd:
"Schedules": "records",
"Users": "Resources",
"UserGroups": "Resources",
+ "Models": "records",
+ "UserAttributes": "records",
}[endpoint]
body = {"pageInfo": {"hasNextPage": False, "nextCursor": None}, selector: []}
result = self._driver().run(endpoint, [ScriptedResponse(json=body)])