:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(warehouse-sources): add okta membership, app assignment and factor tables (#123165)
Tom Owers6b6ddd819347
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.mdindex c45f306a0d73..25304803c98c 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md@@ -8583,14 +8583,14 @@ Note: Read the generated Merchant API reference for version 2025-02-01, includin## Okta — gaps-Today (6): `applications`, `group_rules`, `groups`, `logs`, `user_types`, `users`+Today (10): `application_groups`, `application_users`, `applications`, `group_members`, `group_rules`, `groups`, `logs`, `user_factors`, `user_types`, `users`Diffed against: <https://raw.githubusercontent.com/okta/okta-management-openapi-spec/master/dist/2026.07.2/management-minimal.yaml>-- [ ] `groups/{groupId}/users` — The user↔group membership junction; without it the groups and users tables cannot be joined at all (high)-- [ ] `apps/{appId}/users` — Application assignment per user (with scope, status and credentials profile) — the core access-review question (high)-- [ ] `apps/{appId}/groups` — Group-based application assignment, the other half of how access is actually granted (high)-- [ ] `users/{userId}/factors and users/{userId}/authenticator-enrollments` — Enrolled MFA factors per user — the standard security-posture report over an Okta sync (high)+- [x] `groups/{groupId}/users` — The user↔group membership junction; without it the groups and users tables cannot be joined at all (high). Added as `group_members`.+- [x] `apps/{appId}/users` — Application assignment per user (with scope, status and credentials profile) — the core access-review question (high). Added as `application_users`.+- [x] `apps/{appId}/groups` — Group-based application assignment, the other half of how access is actually granted (high). Added as `application_groups`.+- [x] `users/{userId}/factors and users/{userId}/authenticator-enrollments` — Enrolled MFA factors per user — the standard security-posture report over an Okta sync (high). Added `users/{userId}/factors` as `user_factors`; `authenticator-enrollments` is skipped because it is Identity Engine only and would repeat the per-user fan-out for overlapping enrollment data.- [ ] `policies and policies/{policyId}/rules` — Sign-on, password and authenticator policies plus their rules; explains why the logs table shows the auth outcomes it does (medium)- [ ] `devices and devices/{deviceId}/users` — Managed device inventory plus the device↔user junction, needed for device-trust and BYOD analysis (medium)- [ ] `iam/roles, users/{userId}/roles, groups/{groupId}/roles` — Standard and custom admin-role assignments — privileged-access reporting over users and groups we already sync (medium)diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/okta/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/okta/canonical_descriptions.pyindex d55394f2ed75..dea372f85253 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/okta/canonical_descriptions.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/okta/canonical_descriptions.py@@ -98,4 +98,61 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {"lastUpdated": "Time at which the user type was last updated.",},},+ "group_members": {+ "description": "Group membership: one row per user in each Okta group. Joins the users and groups tables.",+ "docs_url": "https://developer.okta.com/docs/api/openapi/okta-management/management/tag/Group/#tag/Group/operation/listGroupUsers",+ "columns": {+ "group_id": "Identifier of the group the user belongs to.",+ "id": "Unique identifier of the member user.",+ "status": "Lifecycle status of the member user.",+ "profile": "Profile attributes of the member user.",+ "created": "Time at which the member user was created.",+ "lastUpdated": "Time at which the member user was last updated.",+ },+ },+ "application_users": {+ "description": "Application assignments: one row per user assigned to each Okta application.",+ "docs_url": "https://developer.okta.com/docs/api/openapi/okta-management/management/tag/ApplicationUsers/#tag/ApplicationUsers/operation/listApplicationUsers",+ "columns": {+ "app_id": "Identifier of the application the user is assigned to.",+ "id": "Unique identifier of the assigned user.",+ "scope": "How the user is assigned: USER (directly) or GROUP (through a group assignment).",+ "status": "Status of the app user (e.g. ACTIVE, PROVISIONED, DEPROVISIONED).",+ "syncState": "Synchronization state of the app user when provisioning is enabled.",+ "credentials": "The user's credentials for the application, such as the app username.",+ "profile": "App-specific profile attributes of the user.",+ "externalId": "Identifier of the user in the application, when provisioning is enabled.",+ "created": "Time at which the user was assigned to the application.",+ "lastUpdated": "Time at which the assignment was last updated.",+ "statusChanged": "Time at which the app user's status last changed.",+ "lastSync": "Time of the last synchronization with the application.",+ "passwordChanged": "Time at which the app password was last changed.",+ },+ },+ "application_groups": {+ "description": "Group-based application assignments: one row per group assigned to each Okta application.",+ "docs_url": "https://developer.okta.com/docs/api/openapi/okta-management/management/tag/ApplicationGroups/#tag/ApplicationGroups/operation/listApplicationGroupAssignments",+ "columns": {+ "app_id": "Identifier of the application the group is assigned to.",+ "id": "Unique identifier of the assigned group.",+ "priority": "Priority of the assignment when a user gets the app through several groups.",+ "profile": "App-specific profile attributes applied to members of the group.",+ "lastUpdated": "Time at which the assignment was last updated.",+ },+ },+ "user_factors": {+ "description": "MFA factors enrolled by each Okta user. Only factors in the user's highest-priority authenticator enrollment policy are listed.",+ "docs_url": "https://developer.okta.com/docs/api/openapi/okta-management/management/tag/UserFactor/#tag/UserFactor/operation/listFactors",+ "columns": {+ "user_id": "Identifier of the user who enrolled the factor.",+ "id": "Unique identifier of the factor.",+ "factorType": "Type of the factor (e.g. push, token:software:totp, sms, webauthn).",+ "provider": "Provider of the factor (e.g. OKTA, GOOGLE, RSA).",+ "vendorName": "Name of the factor vendor.",+ "status": "Status of the factor (e.g. ACTIVE, PENDING_ACTIVATION, NOT_SETUP).",+ "profile": "Factor-specific attributes, such as the phone number or device name.",+ "created": "Time at which the factor was enrolled.",+ "lastUpdated": "Time at which the factor was last updated.",+ },+ },}diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/okta/okta.py b/products/warehouse_sources/backend/temporal/data_imports/sources/okta/okta.pyindex 4b59e55b5d69..75ffe6f443f3 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/okta/okta.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/okta/okta.py@@ -1,8 +1,8 @@import reimport dataclasses-from collections.abc import Iterator+from collections.abc import Callable, Iterable, Iteratorfrom datetime import UTC, date, datetime, timedelta-from typing import Any, Optional+from typing import Any, Optional, castfrom urllib.parse import urlparseimport requests@@ -13,9 +13,17 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.resRESTAPIConfig,rest_api_resource,)+from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.fanout import (+ build_dependent_resource,+)from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.paginators import (HeaderLinkPaginator,)+from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.resource import Resource+from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.typing import (+ ClientConfig,+ Endpoint,+)from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManagerfrom products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponsefrom products.warehouse_sources.backend.temporal.data_imports.sources.okta.settings import (@@ -30,9 +38,12 @@ class OktaHostNotAllowedError(Exception):pass[email protected](frozen=True)class OktaResumeConfig:- next_url: str+ next_url: Optional[str] = None+ # Fan-out tables checkpoint the shape `build_dependent_resource` emits: which parents finished+ # and the child page of the parent in progress.+ fanout_state: Optional[dict[str, Any]] = Nonedef normalize_domain(domain: str) -> str:@@ -102,6 +113,15 @@ def _build_initial_params(return params+def _drop_fields(fields: tuple[str, ...]) -> Callable[[dict[str, Any]], dict[str, Any]]:+ def drop(row: dict[str, Any]) -> dict[str, Any]:+ for name in fields:+ row.pop(name, None)+ return row++ return drop++def _is_same_host(url: str, domain: str) -> bool:"""Whether ``url`` points at the configured Okta org host.@@ -208,54 +228,84 @@ def okta_source() -> SourceResponse:config = OKTA_ENDPOINTS[endpoint]- params = _build_initial_params(- config, should_use_incremental_field, db_incremental_field_last_value, incremental_field- )-- rest_config: RESTAPIConfig = {- "client": {- "base_url": _base_url(domain),- # Auth (the SSWS token) is supplied via the framework auth config so its value is- # redacted from logs and raised error messages; only non-secret headers go here.- "headers": {"Accept": "application/json", "Content-Type": "application/json"},- "auth": {"type": "api_key", "api_key": f"SSWS {api_key}", "name": "Authorization", "location": "header"},- "paginator": OktaLinkPaginator(domain),- # A validated host could 3xx to an internal address; refuse to follow redirects (SSRF).- "allow_redirects": False,- },- "resources": [- {- "name": endpoint,- "endpoint": {- "path": config.path,- "params": params,- },- }- ],+ client_config: ClientConfig = {+ "base_url": _base_url(domain),+ # Auth (the SSWS token) is supplied via the framework auth config so its value is+ # redacted from logs and raised error messages; only non-secret headers go here.+ "headers": {"Accept": "application/json", "Content-Type": "application/json"},+ "auth": {"type": "api_key", "api_key": f"SSWS {api_key}", "name": "Authorization", "location": "header"},+ "paginator": OktaLinkPaginator(domain),+ # A validated host could 3xx to an internal address; refuse to follow redirects (SSRF).+ "allow_redirects": False,}- # A poisoned resume URL (off the org host) must be ignored, falling back to the initial org- # URL rather than being followed (SSRF).- initial_paginator_state: Optional[dict[str, Any]] = None- if resumable_source_manager.can_resume():- resume = resumable_source_manager.load_state()- if resume is not None and _is_same_host(resume.next_url, domain):+ resume = resumable_source_manager.load_state() if resumable_source_manager.can_resume() else None++ resource: Iterable[Any]+ if config.fanout is not None:+ # The saved fan-out state carries child next-page URLs, so pin every request to the org+ # host rather than trusting a stored URL (SSRF).+ client_config["allowed_hosts"] = []++ def save_fanout_checkpoint(state: Optional[dict[str, Any]]) -> None:+ if state:+ resumable_source_manager.save_state(OktaResumeConfig(fanout_state=state))++ child_endpoint_extra: Optional[Endpoint] = None if config.paginated else {"paginator": "single_page"}+ resource = build_dependent_resource(+ endpoint_configs=OKTA_ENDPOINTS,+ child_endpoint=endpoint,+ fanout=config.fanout,+ client_config=client_config,+ path_format_values={},+ team_id=team_id,+ job_id=job_id,+ db_incremental_field_last_value=None,+ child_endpoint_extra=child_endpoint_extra,+ page_size_param="limit" if config.paginated else None,+ resume_hook=save_fanout_checkpoint if config.resumable else None,+ initial_paginator_state=resume.fanout_state if resume and config.resumable else None,+ )+ if config.excluded_fields:+ resource = cast(Resource, resource).add_map(_drop_fields(config.excluded_fields))+ else:+ params = _build_initial_params(+ config, should_use_incremental_field, db_incremental_field_last_value, incremental_field+ )++ rest_config: RESTAPIConfig = {+ "client": client_config,+ "resources": [+ {+ "name": endpoint,+ "endpoint": {+ "path": config.path,+ "params": params,+ },+ }+ ],+ }++ # A poisoned resume URL (off the org host) must be ignored, falling back to the initial org+ # URL rather than being followed (SSRF).+ initial_paginator_state: Optional[dict[str, Any]] = None+ if resume is not None and resume.next_url and _is_same_host(resume.next_url, domain):initial_paginator_state = {"next_url": resume.next_url}- def save_checkpoint(state: Optional[dict[str, Any]]) -> None:- # Persist only when a next page remains; save AFTER a page is yielded so a crash re-fetches- # the next page (merge dedupes) rather than skipping data.- if state and state.get("next_url"):- resumable_source_manager.save_state(OktaResumeConfig(next_url=state["next_url"]))-- resource = rest_api_resource(- rest_config,- team_id,- job_id,- db_incremental_field_last_value,- resume_hook=save_checkpoint,- initial_paginator_state=initial_paginator_state,- )+ def save_checkpoint(state: Optional[dict[str, Any]]) -> None:+ # Persist only when a next page remains; save AFTER a page is yielded so a crash re-fetches+ # the next page (merge dedupes) rather than skipping data.+ if state and state.get("next_url"):+ resumable_source_manager.save_state(OktaResumeConfig(next_url=state["next_url"]))++ resource = rest_api_resource(+ rest_config,+ team_id,+ job_id,+ db_incremental_field_last_value,+ resume_hook=save_checkpoint,+ initial_paginator_state=initial_paginator_state,+ )def items() -> Iterator[list[Any]]:# Re-check at run time (not just at source-create) in case the domain was edited or now@@ -268,7 +318,7 @@ def okta_source(return SourceResponse(name=endpoint,items=items,- primary_keys=[config.primary_key],+ primary_keys=config.primary_keys,# Okta returns the System Log ascending (we request sortOrder=ASCENDING). The filter# endpoints don't guarantee an order, but each sync re-applies `filter=lastUpdated gt# <watermark>` and paginates every page, so completeness doesn't depend on ordering.diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/okta/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/okta/settings.pyindex 7ff87debd8b9..c5f4ba85a26c 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/okta/settings.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/okta/settings.py@@ -1,6 +1,9 @@from dataclasses import dataclass, fieldfrom typing import Literal, Optional+from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.fanout import (+ DependentEndpointConfig,+)from products.warehouse_sources.backend.types import IncrementalField, IncrementalFieldType@@ -13,7 +16,7 @@ def _datetime_incremental_field(name: str) -> IncrementalField:}-@dataclass+@dataclass(frozen=True)class OktaEndpointConfig:name: strpath: str@@ -27,8 +30,18 @@ class OktaEndpointConfig:incremental_param: Optional[Literal["filter", "since"]] = None# Stable, immutable field to partition by. Never use lastUpdated (it mutates).partition_key: Optional[str] = None- primary_key: str = "id"+ primary_keys: list[str] = field(default_factory=lambda: ["id"])page_size: int = 200+ # False for list endpoints that return the whole collection in one response and take no `limit`.+ paginated: bool = True+ # Set for per-parent child endpoints (e.g. a group's members), which Okta only exposes nested+ # under the parent resource.+ fanout: Optional[DependentEndpointConfig] = None+ # False skips fan-out checkpoints: the shared fan-out re-sorts and re-saves every finished parent+ # on each checkpoint, which grows quadratically when the parent is every user in the org.+ resumable: bool = True+ # Top-level row keys never written to the warehouse.+ excluded_fields: tuple[str, ...] = ()# Limit the first sync to the last N days instead of full history. Only used by# endpoints with a true time filter (System Log retention is bounded anyway).default_lookback_days: Optional[int] = None@@ -73,7 +86,7 @@ OKTA_ENDPOINTS: dict[str, OktaEndpointConfig] = {default_incremental_field="published",incremental_param="since",partition_key="published",- primary_key="uuid",+ primary_keys=["uuid"],page_size=1000,default_lookback_days=90,),@@ -91,6 +104,75 @@ OKTA_ENDPOINTS: dict[str, OktaEndpointConfig] = {# Small, unpaginated metadata list. Full refresh only.page_size=200,),+ # The child endpoints below take no server-side time filter, so they are full refresh only.+ # Child ids are only unique within their parent, so the parent id is part of every key.+ "group_members": OktaEndpointConfig(+ name="group_members",+ path="/groups/{groupId}/users",+ partition_key="created",+ primary_keys=["group_id", "id"],+ # Okta's default page size here is very large for historical reasons; 200 is its recommended limit.+ page_size=200,+ fanout=DependentEndpointConfig(+ parent_name="groups",+ resolve_param="groupId",+ resolve_field="id",+ include_from_parent=["id"],+ parent_field_renames={"id": "group_id"},+ # A parent deleted between the parent listing and its child fetch 404s.+ child_response_actions=[{"status_code": 404, "action": "ignore"}],+ ),+ ),+ "application_users": OktaEndpointConfig(+ name="application_users",+ path="/apps/{appId}/users",+ partition_key="created",+ primary_keys=["app_id", "id"],+ page_size=500,+ fanout=DependentEndpointConfig(+ parent_name="applications",+ resolve_param="appId",+ resolve_field="id",+ include_from_parent=["id"],+ parent_field_renames={"id": "app_id"},+ child_response_actions=[{"status_code": 404, "action": "ignore"}],+ ),+ ),+ "application_groups": OktaEndpointConfig(+ name="application_groups",+ path="/apps/{appId}/groups",+ primary_keys=["app_id", "id"],+ page_size=200,+ fanout=DependentEndpointConfig(+ parent_name="applications",+ resolve_param="appId",+ resolve_field="id",+ include_from_parent=["id"],+ parent_field_renames={"id": "app_id"},+ child_response_actions=[{"status_code": 404, "action": "ignore"}],+ ),+ ),+ "user_factors": OktaEndpointConfig(+ name="user_factors",+ path="/users/{userId}/factors",+ partition_key="created",+ primary_keys=["user_id", "id"],+ paginated=False,+ resumable=False,+ # A factor pending activation embeds its enrollment secret (e.g. the TOTP shared secret) and+ # links to its QR code. Neither belongs in queryable warehouse data.+ excluded_fields=("_embedded", "_links"),+ fanout=DependentEndpointConfig(+ parent_name="users",+ resolve_param="userId",+ resolve_field="id",+ include_from_parent=["id"],+ parent_field_renames={"id": "user_id"},+ # The child takes no page size, so the parent's `limit` is set here instead.+ parent_params={"limit": 200},+ child_response_actions=[{"status_code": 404, "action": "ignore"}],+ ),+ ),}ENDPOINTS = tuple(OKTA_ENDPOINTS.keys())diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/okta/tests/test_okta.py b/products/warehouse_sources/backend/temporal/data_imports/sources/okta/tests/test_okta.pyindex cbc0e727d7f9..b230c44fef5c 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/okta/tests/test_okta.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/okta/tests/test_okta.py@@ -223,3 +223,114 @@ class TestOktaPagination:result = self._driver().run("users", [])assert isinstance(result.raised, okta_module.OktaHostNotAllowedError)assert result.requests == []+++class TestOktaFanout:+ @staticmethod+ def _driver() -> SourceDriver:+ return SourceDriver(+ OktaSource(),+ OktaSourceConfig(okta_domain="example.okta.com", api_key="tok"),+ )++ @pytest.mark.parametrize(+ "schema, parent_path, child_path, parent_key, child_limit",+ [+ ("group_members", "/api/v1/groups", "/api/v1/groups/{}/users", "group_id", "200"),+ ("application_users", "/api/v1/apps", "/api/v1/apps/{}/users", "app_id", "500"),+ ("application_groups", "/api/v1/apps", "/api/v1/apps/{}/groups", "app_id", "200"),+ ("user_factors", "/api/v1/users", "/api/v1/users/{}/factors", "user_id", None),+ ],+ )+ def test_fans_out_over_parents_and_skips_deleted_parent(+ self, schema, parent_path, child_path, parent_key, child_limit+ ):+ result = self._driver().run(+ schema,+ [+ ScriptedResponse(json=[{"id": "p1"}, {"id": "p2"}]),+ ScriptedResponse(json=[{"id": "c1"}, {"id": "c2"}]),+ # p2 was deleted after the parent listing.+ ScriptedResponse(status=404, json={"errorCode": "E0000007"}),+ ],+ )++ assert result.raised is None+ assert [(row[parent_key], row["id"]) for row in result.rows] == [("p1", "c1"), ("p1", "c2")]+ assert result.paths == [parent_path, child_path.format("p1"), child_path.format("p2")]+ assert result.requests[0].param("limit") == "200"+ assert result.requests[1].param("limit") == child_limit++ def test_user_factors_drop_activation_secrets(self):+ result = self._driver().run(+ "user_factors",+ [+ ScriptedResponse(json=[{"id": "u1"}]),+ ScriptedResponse(+ json=[+ {+ "id": "f1",+ "factorType": "token:software:totp",+ "status": "PENDING_ACTIVATION",+ "_embedded": {"activation": {"sharedSecret": "FAKESECRET"}},+ "_links": {"qrcode": {"href": "https://example.okta.com/qr/fake"}},+ }+ ]+ ),+ ],+ )++ assert result.raised is None+ assert result.rows == [+ {"id": "f1", "factorType": "token:software:totp", "status": "PENDING_ACTIVATION", "user_id": "u1"}+ ]++ def test_follows_child_link_header_across_pages(self):+ result = self._driver().run(+ "group_members",+ [+ ScriptedResponse(json=[{"id": "g1"}]),+ ScriptedResponse(+ json=[{"id": "u1"}],+ headers={"Link": '<https://example.okta.com/api/v1/groups/g1/users?after=cur>; rel="next"'},+ ),+ ScriptedResponse(json=[{"id": "u2"}]),+ ],+ )++ assert result.raised is None+ assert [(row["group_id"], row["id"]) for row in result.rows] == [("g1", "u1"), ("g1", "u2")]+ assert result.requests[2].path == "/api/v1/groups/g1/users"+ assert result.requests[2].param("after") == "cur"++ def test_resume_skips_completed_parents(self):+ result = self._driver().run(+ "group_members",+ [+ ScriptedResponse(json=[{"id": "g1"}, {"id": "g2"}]),+ ScriptedResponse(json=[{"id": "u3"}]),+ ],+ resume_state=OktaResumeConfig(+ fanout_state={"completed": ["/groups/g1/users"], "current": None, "child_state": None}+ ),+ )++ assert result.raised is None+ assert result.paths == ["/api/v1/groups", "/api/v1/groups/g2/users"]+ assert [(row["group_id"], row["id"]) for row in result.rows] == [("g2", "u3")]++ def test_resumed_child_url_on_foreign_host_is_refused(self):+ result = self._driver().run(+ "group_members",+ [ScriptedResponse(json=[{"id": "g1"}])],+ resume_state=OktaResumeConfig(+ fanout_state={+ "completed": [],+ "current": "/groups/g1/users",+ "child_state": {"next_url": "http://169.254.169.254/latest/meta-data/"},+ }+ ),+ )++ assert isinstance(result.raised, ValueError)+ assert result.paths == ["/api/v1/groups"]