Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(data-warehouse): add 9 missing tables to the Okta source (#128313)

Tom Owers9a3aa80511dc
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
index c8871b8f9561..82e5b668caf6 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
@@ -8632,10 +8632,10 @@ Diffed against: <https://raw.githubusercontent.com/okta/okta-management-openapi-
- [x] `apps/{appId}/users` — Application assignment per user (with scope, status and credentials profile) — the core access-review question (high). Added as `application_users`.
- [x] `apps/{appId}/groups` — Group-based application assignment, the other half of how access is actually granted (high). Added as `application_groups`.
- [x] `users/{userId}/factors and users/{userId}/authenticator-enrollments` — Enrolled MFA factors per user — the standard security-posture report over an Okta sync (high). Added `users/{userId}/factors` as `user_factors`; `authenticator-enrollments` is skipped because it is Identity Engine only and would repeat the per-user fan-out for overlapping enrollment data.
-- [ ] `policies and policies/{policyId}/rules` — Sign-on, password and authenticator policies plus their rules; explains why the logs table shows the auth outcomes it does (medium)
-- [ ] `devices and devices/{deviceId}/users` — Managed device inventory plus the device↔user junction, needed for device-trust and BYOD analysis (medium)
-- [ ] `iam/roles, users/{userId}/roles, groups/{groupId}/roles` — Standard and custom admin-role assignments — privileged-access reporting over users and groups we already sync (medium)
-- [ ] `idps and idps/{idpId}/users` — Identity providers plus the linked external identities, resolving federated logins seen in logs (medium)
+- [x] `policies and policies/{policyId}/rules` — Sign-on, password and authenticator policies plus their rules; explains why the logs table shows the auth outcomes it does (medium). Added as `policies` and `policy_rules`; both read every policy type, because the list endpoint requires `type`.
+- [x] `devices and devices/{deviceId}/users` — Managed device inventory plus the device↔user junction, needed for device-trust and BYOD analysis (medium). Added as `devices` and `device_users` (Identity Engine only).
+- [x] `iam/roles, users/{userId}/roles, groups/{groupId}/roles` — Standard and custom admin-role assignments — privileged-access reporting over users and groups we already sync (medium). Added as `custom_roles`, `user_role_assignments` and `group_role_assignments`.
+- [x] `idps and idps/{idpId}/users` — Identity providers plus the linked external identities, resolving federated logins seen in logs (medium). Added as `identity_providers` (without `protocol.credentials`) and `identity_provider_users`.
- [ ] `authenticators (and /methods)` — Lookup that resolves factor/authenticator type ids appearing in enrollments and policy rules (medium)
- [ ] `authorizationServers with /scopes, /claims and /policies` — Custom auth servers and their scopes/claims, needed to interpret OAuth token events in logs (medium)
- [ ] `users/{userId}/grants and apps/{appId}/grants` — OAuth consent grants per user and per app — who has consented to which scopes (medium)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/okta/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/okta/canonical_descriptions.py
index dea372f85253..9cb554a2e6c9 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/okta/canonical_descriptions.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/okta/canonical_descriptions.py
@@ -155,4 +155,134 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {
"lastUpdated": "Time at which the factor was last updated.",
},
},
+ "policies": {
+ "description": "An Okta policy of any type, such as sign-on, password, MFA enrollment or app sign-in.",
+ "docs_url": "https://developer.okta.com/docs/api/openapi/okta-management/management/tag/Policy/#tag/Policy/operation/listPolicies",
+ "columns": {
+ "id": "Identifier of the policy.",
+ "type": "Type of the policy (e.g. OKTA_SIGN_ON, PASSWORD, MFA_ENROLL, ACCESS_POLICY).",
+ "name": "Name of the policy.",
+ "description": "Description of the policy.",
+ "priority": "Order in which this policy is evaluated in relation to the other policies of its type.",
+ "status": "Whether the policy is ACTIVE or INACTIVE.",
+ "system": "Whether Okta created the policy.",
+ "conditions": "Conditions that decide when the policy applies, such as the groups it covers.",
+ "settings": "Type-specific policy settings.",
+ "created": "Time at which the policy was created.",
+ "lastUpdated": "Time at which the policy was last modified.",
+ },
+ },
+ "policy_rules": {
+ "description": "A rule inside an Okta policy, with the conditions it matches and the actions it applies.",
+ "docs_url": "https://developer.okta.com/docs/api/openapi/okta-management/management/tag/Policy/#tag/Policy/operation/listPolicyRules",
+ "columns": {
+ "policy_id": "Identifier of the policy that contains the rule.",
+ "id": "Identifier of the rule.",
+ "type": "Type of the rule, which matches the type of its policy.",
+ "name": "Name of the rule.",
+ "priority": "Priority of the rule within its policy.",
+ "status": "Whether the rule is ACTIVE or INACTIVE.",
+ "system": "Whether Okta created the rule. System rules can't be deleted.",
+ "conditions": "Conditions the rule matches, such as people, network zones or risk level.",
+ "actions": "Actions the rule applies when its conditions match.",
+ "created": "Time at which the rule was created.",
+ "lastUpdated": "Time at which the rule was last modified.",
+ },
+ },
+ "devices": {
+ "description": "A device registered with Okta. Requires Okta Identity Engine.",
+ "docs_url": "https://developer.okta.com/docs/api/openapi/okta-management/management/tag/Device/#tag/Device/operation/listDevices",
+ "columns": {
+ "id": "Unique identifier of the device.",
+ "status": "Lifecycle status of the device (e.g. ACTIVE, SUSPENDED, DEACTIVATED).",
+ "profile": "Device attributes, such as display name, platform, OS version, serial number and registration state.",
+ "resourceId": "Alternate key for the id.",
+ "resourceDisplayName": "Display name of the device.",
+ "resourceType": "Resource type of the device.",
+ "created": "Time at which the device was created.",
+ "lastUpdated": "Time at which Okta last updated the device record, for example after it collected device signals.",
+ },
+ },
+ "device_users": {
+ "description": "The users linked to each Okta device. Requires Okta Identity Engine.",
+ "docs_url": "https://developer.okta.com/docs/api/openapi/okta-management/management/tag/Device/#tag/Device/operation/listDeviceUsers",
+ "columns": {
+ "device_id": "Identifier of the device.",
+ "user_id": "Identifier of the user linked to the device.",
+ "user": "The linked Okta user.",
+ "managementStatus": "Management status of the device for this user (MANAGED or NOT_MANAGED).",
+ "screenLockType": "Screen lock type of the device (e.g. NONE, PASSCODE, BIOMETRIC).",
+ "created": "Time at which the user was linked to the device.",
+ },
+ },
+ "custom_roles": {
+ "description": "A custom admin role defined in the Okta org.",
+ "docs_url": "https://developer.okta.com/docs/api/openapi/okta-management/management/tag/RoleECustom/#tag/RoleECustom/operation/listRoles",
+ "columns": {
+ "id": "Unique identifier of the role.",
+ "label": "Unique label of the role.",
+ "description": "Description of the role.",
+ "created": "Time at which the role was created.",
+ "lastUpdated": "Time at which the role was last updated.",
+ },
+ },
+ "user_role_assignments": {
+ "description": "Admin roles assigned to each Okta user, directly or through a group.",
+ "docs_url": "https://developer.okta.com/docs/api/openapi/okta-management/management/tag/RoleAssignmentAUser/#tag/RoleAssignmentAUser/operation/listAssignedRolesForUser",
+ "columns": {
+ "user_id": "Identifier of the user who holds the role.",
+ "id": "Identifier of the role assignment.",
+ "type": "Type of the role (e.g. SUPER_ADMIN, ORG_ADMIN, APP_ADMIN, or CUSTOM).",
+ "label": "Label of the role assignment.",
+ "assignmentType": "Whether the role is assigned to the USER directly or through a GROUP.",
+ "status": "Status of the role assignment.",
+ "role": "Identifier of the custom role, for custom role assignments.",
+ "resource-set": "Identifier of the resource set, for custom role assignments.",
+ "created": "Time at which the role was assigned.",
+ "lastUpdated": "Time at which the role assignment was last updated.",
+ },
+ },
+ "group_role_assignments": {
+ "description": "Admin roles assigned to each Okta group.",
+ "docs_url": "https://developer.okta.com/docs/api/openapi/okta-management/management/tag/RoleAssignmentBGroup/#tag/RoleAssignmentBGroup/operation/listGroupAssignedRoles",
+ "columns": {
+ "group_id": "Identifier of the group that holds the role.",
+ "id": "Identifier of the role assignment.",
+ "type": "Type of the role (e.g. SUPER_ADMIN, ORG_ADMIN, APP_ADMIN, or CUSTOM).",
+ "label": "Label of the role assignment.",
+ "assignmentType": "Assignment type of the role, GROUP for this table.",
+ "status": "Status of the role assignment.",
+ "role": "Identifier of the custom role, for custom role assignments.",
+ "resource-set": "Identifier of the resource set, for custom role assignments.",
+ "created": "Time at which the role was assigned.",
+ "lastUpdated": "Time at which the role assignment was last updated.",
+ },
+ },
+ "identity_providers": {
+ "description": "An external identity provider (IdP) that Okta federates sign-in with, such as SAML, OIDC or a social login. Client credentials are not synced.",
+ "docs_url": "https://developer.okta.com/docs/api/openapi/okta-management/management/tag/IdentityProvider/#tag/IdentityProvider/operation/listIdentityProviders",
+ "columns": {
+ "id": "Unique identifier of the IdP.",
+ "type": "Type of the IdP (e.g. SAML2, OIDC, GOOGLE, MICROSOFT, FACEBOOK).",
+ "name": "Unique name of the IdP.",
+ "status": "Whether the IdP is ACTIVE or INACTIVE.",
+ "issuerMode": "Which domain Okta uses as the issuer in requests to the IdP.",
+ "protocol": "Protocol settings for endpoints, bindings and algorithms, without the credentials.",
+ "policy": "Provisioning, account link and subject matching policy of the IdP.",
+ "created": "Time at which the IdP was created.",
+ "lastUpdated": "Time at which the IdP was last updated.",
+ },
+ },
+ "identity_provider_users": {
+ "description": "Okta users linked to an external identity at each identity provider.",
+ "docs_url": "https://developer.okta.com/docs/api/openapi/okta-management/management/tag/IdentityProviderUsers/#tag/IdentityProviderUsers/operation/listIdentityProviderApplicationUsers",
+ "columns": {
+ "idp_id": "Identifier of the identity provider.",
+ "id": "Identifier of the linked Okta user.",
+ "externalId": "Unique IdP-specific identifier of the user.",
+ "profile": "IdP-specific profile of the user.",
+ "created": "Time at which the user was linked to the IdP.",
+ "lastUpdated": "Time at which the link was last updated.",
+ },
+ },
}
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/okta/okta.py b/products/warehouse_sources/backend/temporal/data_imports/sources/okta/okta.py
index 75ffe6f443f3..36ecdeb93cab 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/okta/okta.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/okta/okta.py
@@ -17,22 +17,33 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.res
build_dependent_resource,
)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.paginators import (
+ BaseNextUrlPaginator,
HeaderLinkPaginator,
+ JSONResponsePaginator,
)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.resource import Resource
from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.typing import (
ClientConfig,
Endpoint,
+ ResponseAction,
)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
from products.warehouse_sources.backend.temporal.data_imports.sources.okta.settings import (
OKTA_ENDPOINTS,
+ POLICY_TYPES,
OktaEndpointConfig,
)
HOST_NOT_ALLOWED_ERROR = "Okta domain is not allowed"
+# An org answers 400 for a policy type it does not support, and E0000015 for a type behind a
+# feature it has not enabled. Only `type` is sent, so neither can mean a malformed request.
+UNSUPPORTED_POLICY_TYPE_ACTIONS: list[ResponseAction] = [
+ {"status_code": 400, "action": "ignore"},
+ {"json_field": "errorCode", "json_values": ["E0000015"], "action": "ignore"},
+]
+
class OktaHostNotAllowedError(Exception):
pass
@@ -44,6 +55,8 @@ class OktaResumeConfig:
# Fan-out tables checkpoint the shape `build_dependent_resource` emits: which parents finished
# and the child page of the parent in progress.
fanout_state: Optional[dict[str, Any]] = None
+ # Policy tables: the next policy type to read. Every earlier type is already written.
+ policy_type: Optional[str] = None
def normalize_domain(domain: str) -> str:
@@ -90,7 +103,7 @@ def _build_initial_params(
db_incremental_field_last_value: Any,
incremental_field: str | None,
) -> dict[str, Any]:
- params: dict[str, Any] = {"limit": config.page_size}
+ params: dict[str, Any] = {"limit": config.page_size} if config.send_limit else {}
if config.name == "logs":
# The System Log defaults to the last 7 days when `since` is omitted, so on the
@@ -114,14 +127,40 @@ def _build_initial_params(
def _drop_fields(fields: tuple[str, ...]) -> Callable[[dict[str, Any]], dict[str, Any]]:
+ paths = [name.split(".") for name in fields]
+
def drop(row: dict[str, Any]) -> dict[str, Any]:
- for name in fields:
- row.pop(name, None)
+ for *parents, leaf in paths:
+ target: Any = row
+ for parent in parents:
+ target = target.get(parent) if isinstance(target, dict) else None
+ if isinstance(target, dict):
+ target.pop(leaf, None)
return row
return drop
+def _lift_fields(fields: dict[str, tuple[str, ...]]) -> Callable[[dict[str, Any]], dict[str, Any]]:
+ def lift(row: dict[str, Any]) -> dict[str, Any]:
+ for column, path in fields.items():
+ value: Any = row
+ for key in path:
+ value = value.get(key) if isinstance(value, dict) else None
+ row[column] = value
+ return row
+
+ return lift
+
+
+def _map_rows(resource: Iterable[Any], config: OktaEndpointConfig) -> Iterable[Any]:
+ if config.excluded_fields:
+ resource = cast(Resource, resource).add_map(_drop_fields(config.excluded_fields))
+ if config.lifted_fields:
+ resource = cast(Resource, resource).add_map(_lift_fields(config.lifted_fields))
+ return resource
+
+
def _is_same_host(url: str, domain: str) -> bool:
"""Whether ``url`` points at the configured Okta org host.
@@ -135,6 +174,12 @@ def _is_same_host(url: str, domain: str) -> bool:
return False
+def _stop_off_org_host(paginator: BaseNextUrlPaginator, domain: str) -> None:
+ if paginator._has_next_page and paginator._next_url is not None and not _is_same_host(paginator._next_url, domain):
+ paginator._has_next_page = False
+ paginator._next_url = None
+
+
class OktaLinkPaginator(HeaderLinkPaginator):
"""Okta paginates via the ``Link`` header, with two Okta-specific twists on top of
the framework's ``rel="next"`` following:
@@ -154,9 +199,37 @@ class OktaLinkPaginator(HeaderLinkPaginator):
self._has_next_page = False
return
super().update_state(response, data)
- if self._has_next_page and self._next_url is not None and not _is_same_host(self._next_url, self._domain):
+ _stop_off_org_host(self, self._domain)
+
+
+class OktaBodyLinkPaginator(JSONResponsePaginator):
+ """Follows the ``_links.next.href`` body link of the custom-role endpoints, pinned to the org host."""
+
+ def __init__(self, domain: str) -> None:
+ super().__init__(next_url_path="_links.next.href")
+ self._domain = domain
+
+ def update_state(self, response: requests.Response, data: Optional[list[Any]] = None) -> None:
+ if not data:
self._has_next_page = False
- self._next_url = None
+ return
+ super().update_state(response, data)
+ _stop_off_org_host(self, self._domain)
+
+
+def _iter_policy_types(
+ build_resource: Callable[[str], Iterable[Any]],
+ resumable_source_manager: ResumableSourceManager[OktaResumeConfig],
+ resume: Optional[OktaResumeConfig],
+) -> Iterator[Any]:
+ start = POLICY_TYPES.index(resume.policy_type) if resume and resume.policy_type in POLICY_TYPES else 0
+ for index in range(start, len(POLICY_TYPES)):
+ policy_type = POLICY_TYPES[index]
+ if index > start:
+ # Every row of the earlier types is yielded, so a hand-off here resumes at this type.
+ resumable_source_manager.save_state(OktaResumeConfig(policy_type=policy_type))
+ resumable_source_manager.safe_point()
+ yield from build_resource(policy_type)
def validate_credentials(
@@ -243,6 +316,7 @@ def okta_source(
resource: Iterable[Any]
if config.fanout is not None:
+ fanout = config.fanout
# The saved fan-out state carries child next-page URLs, so pin every request to the org
# host rather than trusting a stored URL (SSRF).
client_config["allowed_hosts"] = []
@@ -252,38 +326,70 @@ def okta_source(
resumable_source_manager.save_state(OktaResumeConfig(fanout_state=state))
child_endpoint_extra: Optional[Endpoint] = None if config.paginated else {"paginator": "single_page"}
- resource = build_dependent_resource(
- endpoint_configs=OKTA_ENDPOINTS,
- child_endpoint=endpoint,
- fanout=config.fanout,
- client_config=client_config,
- path_format_values={},
- team_id=team_id,
- job_id=job_id,
- db_incremental_field_last_value=None,
- child_endpoint_extra=child_endpoint_extra,
- page_size_param="limit" if config.paginated else None,
- resume_hook=save_fanout_checkpoint if config.resumable else None,
- initial_paginator_state=resume.fanout_state if resume and config.resumable else None,
- )
- if config.excluded_fields:
- resource = cast(Resource, resource).add_map(_drop_fields(config.excluded_fields))
+
+ def build_fanout(policy_type: Optional[str] = None) -> Iterable[Any]:
+ child_fanout = fanout
+ parent_endpoint_extra: Optional[Endpoint] = None
+ if policy_type is not None:
+ child_fanout = dataclasses.replace(fanout, parent_params={**fanout.parent_params, "type": policy_type})
+ parent_endpoint_extra = {"response_actions": UNSUPPORTED_POLICY_TYPE_ACTIONS}
+ return _map_rows(
+ build_dependent_resource(
+ endpoint_configs=OKTA_ENDPOINTS,
+ child_endpoint=endpoint,
+ fanout=child_fanout,
+ client_config=client_config,
+ path_format_values={},
+ team_id=team_id,
+ job_id=job_id,
+ db_incremental_field_last_value=None,
+ parent_endpoint_extra=parent_endpoint_extra,
+ child_endpoint_extra=child_endpoint_extra,
+ page_size_param="limit" if config.paginated and config.send_limit else None,
+ resume_hook=save_fanout_checkpoint if config.resumable else None,
+ initial_paginator_state=resume.fanout_state if resume and config.resumable else None,
+ ),
+ config,
+ )
+
+ if config.per_policy_type:
+ resource = _iter_policy_types(build_fanout, resumable_source_manager, resume)
+ else:
+ resource = build_fanout()
+ elif config.per_policy_type:
+ params = _build_initial_params(config, False, None, None)
+
+ def build_policies(policy_type: str) -> Iterable[Any]:
+ policy_config: RESTAPIConfig = {
+ "client": client_config,
+ "resources": [
+ {
+ "name": endpoint,
+ "endpoint": {
+ "path": config.path,
+ "params": {**params, "type": policy_type},
+ "response_actions": UNSUPPORTED_POLICY_TYPE_ACTIONS,
+ },
+ }
+ ],
+ }
+ return _map_rows(rest_api_resource(policy_config, team_id, job_id, None), config)
+
+ resource = _iter_policy_types(build_policies, resumable_source_manager, resume)
else:
params = _build_initial_params(
config, should_use_incremental_field, db_incremental_field_last_value, incremental_field
)
+ endpoint_config: Endpoint = {"path": config.path, "params": params}
+ if config.data_selector:
+ endpoint_config["data_selector"] = config.data_selector
+ if config.next_link_in_body:
+ endpoint_config["paginator"] = OktaBodyLinkPaginator(domain)
+
rest_config: RESTAPIConfig = {
"client": client_config,
- "resources": [
- {
- "name": endpoint,
- "endpoint": {
- "path": config.path,
- "params": params,
- },
- }
- ],
+ "resources": [{"name": endpoint, "endpoint": endpoint_config}],
}
# A poisoned resume URL (off the org host) must be ignored, falling back to the initial org
@@ -306,6 +412,7 @@ def okta_source(
resume_hook=save_checkpoint,
initial_paginator_state=initial_paginator_state,
)
+ resource = _map_rows(resource, config)
def items() -> Iterator[list[Any]]:
# Re-check at run time (not just at source-create) in case the domain was edited or now
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/okta/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/okta/settings.py
index c5f4ba85a26c..0e9720974934 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/okta/settings.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/okta/settings.py
@@ -40,8 +40,18 @@ class OktaEndpointConfig:
# False skips fan-out checkpoints: the shared fan-out re-sorts and re-saves every finished parent
# on each checkpoint, which grows quadratically when the parent is every user in the org.
resumable: bool = True
- # Top-level row keys never written to the warehouse.
+ # Row keys never written to the warehouse. A dotted name drops a nested key.
excluded_fields: tuple[str, ...] = ()
+ # False for list endpoints that paginate by Link header but whose `limit` maximum is undocumented.
+ send_limit: bool = True
+ # The custom-role endpoints wrap rows in an object and put the next page in `_links.next.href`.
+ data_selector: Optional[str] = None
+ next_link_in_body: bool = False
+ # Policy list endpoints require a `type`, so the table reads every policy type in turn.
+ per_policy_type: bool = False
+ # Copies a nested value to a top-level column, for keys Okta only returns inside an object.
+ lifted_fields: dict[str, tuple[str, ...]] = field(default_factory=dict)
+ description: Optional[str] = None
# Limit the first sync to the last N days instead of full history. Only used by
# endpoints with a true time filter (System Log retention is bounded anyway).
default_lookback_days: Optional[int] = None
@@ -89,6 +99,7 @@ OKTA_ENDPOINTS: dict[str, OktaEndpointConfig] = {
primary_keys=["uuid"],
page_size=1000,
default_lookback_days=90,
+ description="Only syncs the last 90 days on initial sync",
),
"group_rules": OktaEndpointConfig(
name="group_rules",
@@ -173,8 +184,141 @@ OKTA_ENDPOINTS: dict[str, OktaEndpointConfig] = {
child_response_actions=[{"status_code": 404, "action": "ignore"}],
),
),
+ "policies": OktaEndpointConfig(
+ name="policies",
+ path="/policies",
+ partition_key="created",
+ send_limit=False,
+ per_policy_type=True,
+ ),
+ "policy_rules": OktaEndpointConfig(
+ name="policy_rules",
+ path="/policies/{policyId}/rules",
+ partition_key="created",
+ primary_keys=["policy_id", "id"],
+ send_limit=False,
+ per_policy_type=True,
+ # The policy type loop checkpoints between types instead.
+ resumable=False,
+ fanout=DependentEndpointConfig(
+ parent_name="policies",
+ resolve_param="policyId",
+ resolve_field="id",
+ include_from_parent=["id"],
+ parent_field_renames={"id": "policy_id"},
+ child_response_actions=[{"status_code": 404, "action": "ignore"}],
+ ),
+ ),
+ "devices": OktaEndpointConfig(
+ name="devices",
+ path="/devices",
+ # The `search` filter reads an eventually consistent index, so it cannot carry a watermark.
+ partition_key="created",
+ page_size=200,
+ description="Requires Okta Identity Engine",
+ ),
+ "device_users": OktaEndpointConfig(
+ name="device_users",
+ path="/devices/{deviceId}/users",
+ partition_key="created",
+ primary_keys=["device_id", "user_id"],
+ paginated=False,
+ resumable=False,
+ lifted_fields={"user_id": ("user", "id")},
+ description="Requires Okta Identity Engine",
+ fanout=DependentEndpointConfig(
+ parent_name="devices",
+ resolve_param="deviceId",
+ resolve_field="id",
+ include_from_parent=["id"],
+ parent_field_renames={"id": "device_id"},
+ parent_params={"limit": 200},
+ child_response_actions=[{"status_code": 404, "action": "ignore"}],
+ ),
+ ),
+ "custom_roles": OktaEndpointConfig(
+ name="custom_roles",
+ path="/iam/roles",
+ partition_key="created",
+ send_limit=False,
+ data_selector="roles",
+ next_link_in_body=True,
+ ),
+ "user_role_assignments": OktaEndpointConfig(
+ name="user_role_assignments",
+ path="/users/{userId}/roles",
+ partition_key="created",
+ primary_keys=["user_id", "id"],
+ paginated=False,
+ resumable=False,
+ fanout=DependentEndpointConfig(
+ parent_name="users",
+ resolve_param="userId",
+ resolve_field="id",
+ include_from_parent=["id"],
+ parent_field_renames={"id": "user_id"},
+ parent_params={"limit": 200},
+ child_response_actions=[{"status_code": 404, "action": "ignore"}],
+ ),
+ ),
+ "group_role_assignments": OktaEndpointConfig(
+ name="group_role_assignments",
+ path="/groups/{groupId}/roles",
+ partition_key="created",
+ primary_keys=["group_id", "id"],
+ paginated=False,
+ fanout=DependentEndpointConfig(
+ parent_name="groups",
+ resolve_param="groupId",
+ resolve_field="id",
+ include_from_parent=["id"],
+ parent_field_renames={"id": "group_id"},
+ parent_params={"limit": 200},
+ child_response_actions=[{"status_code": 404, "action": "ignore"}],
+ ),
+ ),
+ "identity_providers": OktaEndpointConfig(
+ name="identity_providers",
+ path="/idps",
+ partition_key="created",
+ page_size=200,
+ # OIDC and social IdPs return their client secret here.
+ excluded_fields=("protocol.credentials",),
+ ),
+ "identity_provider_users": OktaEndpointConfig(
+ name="identity_provider_users",
+ path="/idps/{idpId}/users",
+ partition_key="created",
+ primary_keys=["idp_id", "id"],
+ page_size=200,
+ fanout=DependentEndpointConfig(
+ parent_name="identity_providers",
+ resolve_param="idpId",
+ resolve_field="id",
+ include_from_parent=["id"],
+ parent_field_renames={"id": "idp_id"},
+ child_response_actions=[{"status_code": 404, "action": "ignore"}],
+ ),
+ ),
}
+# Every value of the PolicyType enum in the Management API spec. An org rejects a type it does not
+# have (for example ACCESS_POLICY on Classic Engine), and the source skips that type.
+POLICY_TYPES: tuple[str, ...] = (
+ "OKTA_SIGN_ON",
+ "PASSWORD",
+ "MFA_ENROLL",
+ "IDP_DISCOVERY",
+ "ACCESS_POLICY",
+ "PROFILE_ENROLLMENT",
+ "POST_AUTH_SESSION",
+ "ENTITY_RISK",
+ "DEVICE_SIGNAL_COLLECTION",
+ "SESSION_VIOLATION_DETECTION",
+ "CLIENT_UPDATE",
+ "IDENTITY_CLAIM_SOURCING",
+)
+
ENDPOINTS = tuple(OKTA_ENDPOINTS.keys())
INCREMENTAL_FIELDS: dict[str, list[IncrementalField]] = {
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/okta/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/okta/source.py
index 07314b11dc58..738db5b380ba 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/okta/source.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/okta/source.py
@@ -23,7 +23,11 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.okta.okta
okta_source,
validate_credentials as validate_okta_credentials,
)
-from products.warehouse_sources.backend.temporal.data_imports.sources.okta.settings import ENDPOINTS, INCREMENTAL_FIELDS
+from products.warehouse_sources.backend.temporal.data_imports.sources.okta.settings import (
+ ENDPOINTS,
+ INCREMENTAL_FIELDS,
+ OKTA_ENDPOINTS,
+)
from products.warehouse_sources.backend.types import ExternalDataSourceType
@@ -156,6 +160,10 @@ The token's user should have read access to the resources you want to sync, for
- `okta.groups.read`
- `okta.apps.read`
- `okta.logs.read`
+- `okta.policies.read`
+- `okta.devices.read`
+- `okta.roles.read`
+- `okta.idps.read`
""",
iconPath="/static/services/okta.png",
docsUrl="https://posthog.com/docs/cdp/sources/okta",
@@ -211,7 +219,7 @@ The token's user should have read access to the resources you want to sync, for
supports_incremental=bool(INCREMENTAL_FIELDS.get(endpoint)),
supports_append=bool(INCREMENTAL_FIELDS.get(endpoint)),
incremental_fields=INCREMENTAL_FIELDS.get(endpoint, []),
- description="Only syncs the last 90 days on initial sync" if endpoint == "logs" else None,
+ description=OKTA_ENDPOINTS[endpoint].description,
)
for endpoint in ENDPOINTS
]
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/okta/tests/test_okta.py b/products/warehouse_sources/backend/temporal/data_imports/sources/okta/tests/test_okta.py
index b230c44fef5c..3b84e9c73797 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/okta/tests/test_okta.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/okta/tests/test_okta.py
@@ -16,7 +16,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.okta.okta
_format_incremental_value,
validate_credentials,
)
-from products.warehouse_sources.backend.temporal.data_imports.sources.okta.settings import OKTA_ENDPOINTS
+from products.warehouse_sources.backend.temporal.data_imports.sources.okta.settings import OKTA_ENDPOINTS, POLICY_TYPES
from products.warehouse_sources.backend.temporal.data_imports.sources.okta.source import OktaSource
@@ -240,6 +240,9 @@ class TestOktaFanout:
("application_users", "/api/v1/apps", "/api/v1/apps/{}/users", "app_id", "500"),
("application_groups", "/api/v1/apps", "/api/v1/apps/{}/groups", "app_id", "200"),
("user_factors", "/api/v1/users", "/api/v1/users/{}/factors", "user_id", None),
+ ("user_role_assignments", "/api/v1/users", "/api/v1/users/{}/roles", "user_id", None),
+ ("group_role_assignments", "/api/v1/groups", "/api/v1/groups/{}/roles", "group_id", None),
+ ("identity_provider_users", "/api/v1/idps", "/api/v1/idps/{}/users", "idp_id", "200"),
],
)
def test_fans_out_over_parents_and_skips_deleted_parent(
@@ -261,29 +264,75 @@ class TestOktaFanout:
assert result.requests[0].param("limit") == "200"
assert result.requests[1].param("limit") == child_limit
- def test_user_factors_drop_activation_secrets(self):
+ @pytest.mark.parametrize(
+ "schema, script, expected_rows",
+ [
+ (
+ "user_factors",
+ [
+ ScriptedResponse(json=[{"id": "u1"}]),
+ ScriptedResponse(
+ json=[
+ {
+ "id": "f1",
+ "factorType": "token:software:totp",
+ "status": "PENDING_ACTIVATION",
+ "_embedded": {"activation": {"sharedSecret": "FAKESECRET"}},
+ "_links": {"qrcode": {"href": "https://example.okta.com/qr/fake"}},
+ }
+ ]
+ ),
+ ],
+ [{"id": "f1", "factorType": "token:software:totp", "status": "PENDING_ACTIVATION", "user_id": "u1"}],
+ ),
+ (
+ "identity_providers",
+ [
+ ScriptedResponse(
+ json=[
+ {
+ "id": "idp1",
+ "type": "OIDC",
+ "protocol": {
+ "type": "OIDC",
+ "scopes": ["openid"],
+ "credentials": {"client": {"client_id": "fake-id", "client_secret": "FAKESECRET"}},
+ },
+ },
+ {"id": "idp2", "type": "SAML2", "protocol": None},
+ ]
+ )
+ ],
+ [
+ {"id": "idp1", "type": "OIDC", "protocol": {"type": "OIDC", "scopes": ["openid"]}},
+ {"id": "idp2", "type": "SAML2", "protocol": None},
+ ],
+ ),
+ ],
+ )
+ def test_drops_secret_fields(self, schema, script, expected_rows):
+ result = self._driver().run(schema, script)
+
+ assert result.raised is None
+ assert result.rows == expected_rows
+
+ @pytest.mark.parametrize(
+ "device_user, expected_user_id",
+ [
+ ({"created": "2026-01-01T00:00:00.000Z", "user": {"id": "u1", "status": "ACTIVE"}}, "u1"),
+ ({"created": "2026-01-01T00:00:00.000Z", "user": None}, None),
+ ],
+ )
+ def test_device_users_lift_the_nested_user_id(self, device_user, expected_user_id):
result = self._driver().run(
- "user_factors",
- [
- ScriptedResponse(json=[{"id": "u1"}]),
- ScriptedResponse(
- json=[
- {
- "id": "f1",
- "factorType": "token:software:totp",
- "status": "PENDING_ACTIVATION",
- "_embedded": {"activation": {"sharedSecret": "FAKESECRET"}},
- "_links": {"qrcode": {"href": "https://example.okta.com/qr/fake"}},
- }
- ]
- ),
- ],
+ "device_users",
+ [ScriptedResponse(json=[{"id": "d1"}]), ScriptedResponse(json=[device_user])],
)
assert result.raised is None
- assert result.rows == [
- {"id": "f1", "factorType": "token:software:totp", "status": "PENDING_ACTIVATION", "user_id": "u1"}
- ]
+ assert [(row["device_id"], row["user_id"]) for row in result.rows] == [("d1", expected_user_id)]
+ assert result.paths == ["/api/v1/devices", "/api/v1/devices/d1/users"]
+ assert result.params("limit") == ["200", None]
def test_follows_child_link_header_across_pages(self):
result = self._driver().run(
@@ -334,3 +383,95 @@ class TestOktaFanout:
assert isinstance(result.raised, ValueError)
assert result.paths == ["/api/v1/groups"]
+
+
+class TestOktaCustomRoles:
+ @pytest.mark.parametrize(
+ "next_href, expected_ids, expected_requests",
+ [
+ ("https://example.okta.com/api/v1/iam/roles?after=cr1", ["cr1", "cr2"], 2),
+ ("http://169.254.169.254/latest/meta-data/", ["cr1"], 1),
+ ],
+ )
+ def test_follows_body_link_on_org_host_only(self, next_href, expected_ids, expected_requests):
+ script = [
+ ScriptedResponse(json={"roles": [{"id": "cr1"}], "_links": {"next": {"href": next_href}}}),
+ ScriptedResponse(json={"roles": [{"id": "cr2"}], "_links": {}}),
+ ]
+ result = SourceDriver(OktaSource(), OktaSourceConfig(okta_domain="example.okta.com", api_key="tok")).run(
+ "custom_roles", script[:expected_requests]
+ )
+
+ assert result.raised is None
+ assert [row["id"] for row in result.rows] == expected_ids
+ assert result.paths == ["/api/v1/iam/roles"] * expected_requests
+ assert result.requests[0].param("limit") is None
+
+
+class TestOktaPolicies:
+ @staticmethod
+ def _driver() -> SourceDriver:
+ return SourceDriver(
+ OktaSource(),
+ OktaSourceConfig(okta_domain="example.okta.com", api_key="tok"),
+ )
+
+ @staticmethod
+ def _policies_script(request):
+ policy_type = request.param("type")
+ if policy_type == "ACCESS_POLICY":
+ return ScriptedResponse(status=400, json={"errorCode": "E0000001", "errorSummary": "Api validation failed"})
+ if policy_type == "ENTITY_RISK":
+ return ScriptedResponse(status=401, json={"errorCode": "E0000015", "errorSummary": "Feature not enabled"})
+ if request.path == "/api/v1/policies":
+ return ScriptedResponse(json=[{"id": f"pol-{policy_type}", "type": policy_type}])
+ return ScriptedResponse(json=[{"id": "rule1"}])
+
+ def test_reads_every_policy_type_and_skips_rejected_types(self):
+ result = self._driver().run("policies", self._policies_script)
+
+ assert result.raised is None
+ assert result.params("type") == list(POLICY_TYPES)
+ assert result.params("limit") == [None] * len(POLICY_TYPES)
+ assert [row["type"] for row in result.rows] == [
+ t for t in POLICY_TYPES if t not in ("ACCESS_POLICY", "ENTITY_RISK")
+ ]
+ assert result.committed_states[-1] == OktaResumeConfig(policy_type=POLICY_TYPES[-1])
+
+ def test_policy_rules_fan_out_per_policy_type(self):
+ result = self._driver().run("policy_rules", self._policies_script)
+
+ assert result.raised is None
+ assert result.paths[:3] == [
+ "/api/v1/policies",
+ "/api/v1/policies/pol-OKTA_SIGN_ON/rules",
+ "/api/v1/policies",
+ ]
+ assert [request.param("type") for request in result.requests if request.path == "/api/v1/policies"] == list(
+ POLICY_TYPES
+ )
+ assert ("pol-PASSWORD", "rule1") in [(row["policy_id"], row["id"]) for row in result.rows]
+ assert not any(row["policy_id"] in ("pol-ACCESS_POLICY", "pol-ENTITY_RISK") for row in result.rows)
+
+ @pytest.mark.parametrize("schema", ["policies", "policy_rules"])
+ def test_resumes_at_saved_policy_type(self, schema):
+ result = self._driver().run(
+ schema,
+ self._policies_script,
+ resume_state=OktaResumeConfig(policy_type="CLIENT_UPDATE"),
+ )
+
+ assert result.raised is None
+ assert [r.param("type") for r in result.requests if r.path == "/api/v1/policies"] == [
+ "CLIENT_UPDATE",
+ "IDENTITY_CLAIM_SOURCING",
+ ]
+
+ def test_other_errors_still_fail_the_sync(self):
+ result = self._driver().run(
+ "policies",
+ [ScriptedResponse(status=401, json={"errorCode": "E0000011", "errorSummary": "Invalid token provided"})],
+ )
+
+ assert result.raised is not None
+ assert result.params("type") == ["OKTA_SIGN_ON"]