:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(data-warehouse): add auth0 role, membership, and session tables (#116967)
Tom Owers1268ba1f88e7
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.mdindex bf9aec3e390f..1434a1a5473b 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md@@ -977,14 +977,14 @@ Note: This is a webhook-ingestion source (the directory contains webhook_templat## Auth0 — gaps-Today (9): `actions`, `clients`, `connections`, `log_streams`, `logs`, `organizations`, `resource_servers`, `roles`, `users`+Today (13): `actions`, `clients`, `connections`, `log_streams`, `logs`, `organization_member_roles`, `organization_members`, `organizations`, `resource_servers`, `roles`, `user_roles`, `user_sessions`, `users`Diffed against: <https://raw.githubusercontent.com/auth0/node-auth0/master/reference.md>-- [ ] `Organization members (GET /api/v2/organizations/{id}/members)` — connects existing users and organizations for customer membership and account adoption analysis (high)-- [ ] `User roles (GET /api/v2/users/{id}/roles)` — connects existing users to role IDs for access and user segment analysis (high)-- [ ] `Organization member roles (GET /api/v2/organizations/{id}/members/{user_id}/roles)` — shows which roles each user holds within each customer organization (high)-- [ ] `User sessions (GET /api/v2/users/{user_id}/sessions)` — measures active sessions and connects authentication activity to devices and application clients (high)+- [x] `Organization members (GET /api/v2/organizations/{id}/members)` — connects existing users and organizations for customer membership and account adoption analysis (high)+- [x] `User roles (GET /api/v2/users/{id}/roles)` — connects existing users to role IDs for access and user segment analysis (high)+- [x] `Organization member roles (GET /api/v2/organizations/{id}/members/{user_id}/roles)` — shows which roles each user holds within each customer organization (high)+- [x] `User sessions (GET /api/v2/users/{user_id}/sessions)` — measures active sessions and connects authentication activity to devices and application clients (high)- [ ] `User permissions (GET /api/v2/users/{id}/permissions)` — tracks direct permission assignments and their resource servers for access analysis (medium)- [ ] `Role permissions (GET /api/v2/roles/{id}/permissions)` — explains the permissions granted by each existing role (medium)- [ ] `Organization invitations (GET /api/v2/organizations/{id}/invitations)` — measures pending and expired invitations during customer organization onboarding (medium)diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/auth0/auth0.py b/products/warehouse_sources/backend/temporal/data_imports/sources/auth0/auth0.pyindex ccdc62e03aba..1060dd845c82 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/auth0/auth0.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/auth0/auth0.py@@ -3,16 +3,17 @@ import jsonimport timeimport threadingimport dataclasses-from collections.abc import Iterator+from collections.abc import Callable, Iteratorfrom datetime import UTC, date, datetimefrom typing import Any, Optional-from urllib.parse import urlencode+from urllib.parse import quote, urlencodeimport requestsfrom structlog.types import FilteringBoundLoggerfrom products.warehouse_sources.backend.temporal.data_imports.sources.auth0.settings import (AUTH0_ENDPOINTS,+ CHECKPOINT_PAGE_SIZE,MAX_PAGE_SIZE,SEARCH_RESULT_CAP,Auth0EndpointConfig,@@ -81,6 +82,21 @@ class Auth0ResumeConfig:# Lower bound of the Lucene date window currently being paged, or None while the run is# still inside the first (unfiltered) window.window_start: Optional[str] = None+ # Fan-out children only: the last parent on this page whose children were all handed to the+ # pipeline. A resumed run skips the parents up to and including it.+ last_parent_id: Optional[str] = None+ # Fan-out children only: the `from` token of the next unread page of the parent after+ # `last_parent_id`, and of the next parent page when the parent is checkpoint-paged.+ child_checkpoint: Optional[str] = None+ parent_checkpoint: Optional[str] = None++[email protected](frozen=True)+class _Page:+ rows: list[dict[str, Any]]+ # Where to resume to re-read this page, and to read past it (None after the last page).+ position: Auth0ResumeConfig+ next_position: Optional[Auth0ResumeConfig]def normalize_domain(domain: str) -> str:@@ -280,6 +296,7 @@ def _build_params(page: int,window_field: Optional[str],window_start: Optional[str],+ fields: Optional[str] = None,) -> dict[str, Any]:if not config.paginated:return {}@@ -289,6 +306,9 @@ def _build_params(params["include_totals"] = "true"if config.search_engine:params["search_engine"] = config.search_engine+ if fields:+ params["fields"] = fields+ params["include_fields"] = "true"if window_field:# `:1` is Auth0's ascending sort order. Rows must arrive oldest-first for the pipeline's# watermark to checkpoint per batch and for the window to slide forward.@@ -300,8 +320,16 @@ def _build_params(return params-def _build_url(domain: str, config: Auth0EndpointConfig, api_version: str, params: dict[str, Any]) -> str:- url = f"{_base_url(domain)}{config.path_template.format(version=api_version)}"+def _build_url(+ domain: str,+ config: Auth0EndpointConfig,+ api_version: str,+ params: dict[str, Any],+ parent_id: Optional[str] = None,+) -> str:+ # User ids carry the connection strategy (`auth0|123`), so the id is escaped as one segment.+ path = config.path_template.format(version=api_version, parent_id=quote(parent_id or "", safe=""))+ url = f"{_base_url(domain)}{path}"if not params:return urlreturn f"{url}?{urlencode(params, doseq=True)}"@@ -344,9 +372,11 @@ CREDENTIAL_FIELDS_BY_ENDPOINT: dict[str, tuple[str, ...]] = {# Credential fields nested inside each item of an array field, keyed by endpoint then array# field name. `read:user_idp_tokens` returns the linked identity provider's own access and# refresh tokens inside every `users` row's `identities` entries, letting a reader of the synced-# table call those providers' APIs as the user without the source's own credentials.+# table call those providers' APIs as the user without the source's own credentials. Role members+# are user rows too, so they get the same treatment.NESTED_CREDENTIAL_FIELDS_BY_ENDPOINT: dict[str, dict[str, tuple[str, ...]]] = {"users": {"identities": ("access_token", "access_token_secret", "refresh_token")},+ "user_roles": {"identities": ("access_token", "access_token_secret", "refresh_token")},}@@ -374,6 +404,19 @@ def _strip_credential_fields(rows: list[dict[str, Any]], endpoint: str) -> list[return rows+def _shape_child_rows(rows: list[dict[str, Any]], config: Auth0EndpointConfig, parent_id: str) -> list[dict[str, Any]]:+ if config.explode_field:+ explode_field = config.explode_field+ rows = [+ {**{key: value for key, value in row.items() if key != explode_field}, **item}+ for row in rows+ for item in (row.get(explode_field) or [])+ if isinstance(item, dict)+ ]+ assert config.parent_id_column is not None+ return [{**row, config.parent_id_column: parent_id} for row in rows]++def _max_window_value(rows: list[dict[str, Any]], window_field: str) -> Optional[str]:"""Newest window-field value on a page.@@ -385,6 +428,37 @@ def _max_window_value(rows: list[dict[str, Any]], window_field: str) -> Optionalreturn max(values) if values else None+def _probe(session: requests.Session, token: str, url: str, read_body: bool) -> tuple[int, Any]:+ """Request one probe URL and return its status, plus the parsed body when asked for it.++ Without ``read_body`` the response is streamed and closed untouched: a customer-controlled+ server must not get to buffer an unbounded body at source-create.+ """+ response = session.get(+ url,+ headers={"Authorization": f"Bearer {token}", "Accept": "application/json"},+ timeout=10,+ allow_redirects=False,+ stream=True,+ hooks={"response": _reject_redirect_response},+ )+ try:+ if response.is_redirect or response.is_permanent_redirect:+ raise Auth0HostNotAllowedError(+ f"Auth0 returned an unexpected redirect (status={response.status_code}); refusing to follow it"+ )+ body = _read_capped_json(response) if read_body and response.status_code == 200 else None+ return response.status_code, body+ finally:+ response.close()+++def _probe_failure(status_code: int, config: Auth0EndpointConfig, schema_name: str) -> tuple[bool, str | None]:+ if status_code in (401, 403):+ return False, f"Your Auth0 application is missing the `{config.required_scope}` scope for {schema_name}"+ return False, f"Auth0 returned status {status_code} for {schema_name}"++def validate_credentials(domain: str,client_id: str,@@ -430,38 +504,239 @@ def validate_credentials(return True, Noneconfig = AUTH0_ENDPOINTS[schema_name]- probe_params = {"page": 0, "per_page": 1} if config.paginated else {}try:- # Stream and close without touching the body — the probe only needs the status, and a- # customer-controlled server must not get to buffer an unbounded body at source-create.- response = session.get(- _build_url(normalized, config, api_version, probe_params),- headers={"Authorization": f"Bearer {token}", "Accept": "application/json"},- timeout=10,- allow_redirects=False,- stream=True,- hooks={"response": _reject_redirect_response},- )- try:- is_redirect = response.is_redirect or response.is_permanent_redirect- status_code = response.status_code- finally:- response.close()+ if config.parent is None:+ probe_params = {"page": 0, "per_page": 1} if config.paginated else {}+ status_code, _ = _probe(+ session, token, _build_url(normalized, config, api_version, probe_params), read_body=False+ )+ return (True, None) if status_code == 200 else _probe_failure(status_code, config, schema_name)++ # A child collection is only reachable through a real parent id, so take one from the+ # parent collection, which the child sync needs access to as well.+ parent_config = AUTH0_ENDPOINTS[config.parent]+ parent_url = _build_url(normalized, parent_config, api_version, {"page": 0, "per_page": 1})+ status_code, body = _probe(session, token, parent_url, read_body=True)+ if status_code != 200:+ return _probe_failure(status_code, parent_config, schema_name)+ parents = _extract_rows(body, parent_config)+ parent_id = parents[0].get(config.parent_id_field) if parents and isinstance(parents[0], dict) else None+ if not parent_id:+ return True, None++ child_params = {"take": 1, **config.extra_params}+ child_url = _build_url(normalized, config, api_version, child_params, parent_id=str(parent_id))+ status_code, _ = _probe(session, token, child_url, read_body=False)except Auth0HostNotAllowedError:return False, HOST_NOT_ALLOWED_ERROR- except requests.exceptions.RequestException as e:+ except (+ Auth0ResponseTooLargeError,+ Auth0ResponseTooSlowError,+ ValueError,+ requests.exceptions.RequestException,+ ) as e:return False, str(e)- if is_redirect:- return False, HOST_NOT_ALLOWED_ERROR+ return (True, None) if status_code == 200 else _probe_failure(status_code, config, schema_name)- if status_code == 200:- return True, None- if status_code in (401, 403):- return False, f"Your Auth0 application is missing the `{config.required_scope}` scope for {schema_name}"+def _iter_pages(+ fetch: Callable[..., Any],+ domain: str,+ config: Auth0EndpointConfig,+ api_version: str,+ window_field: Optional[str],+ position: Auth0ResumeConfig,+ fields: Optional[str] = None,+) -> Iterator[_Page]:+ """Walk an offset-paged collection from ``position``, sliding the window past the result cap."""+ page = position.page+ window_start = position.window_start+ pages_fetched = 0+ while True:+ if pages_fetched >= MAX_PAGES:+ raise Auth0PaginationLimitError(+ f"Auth0 pagination for {config.name} exceeded {MAX_PAGES} pages without terminating"+ )- return False, f"Auth0 returned status {status_code} for {schema_name}"+ params = _build_params(config, page, window_field, window_start, fields)+ data = fetch(_build_url(domain, config, api_version, params))+ pages_fetched += 1++ rows = _extract_rows(data, config)+ if not rows:+ return++ next_page = page + 1+ total = data.get("total") if isinstance(data, dict) else None+ is_last_page = (isinstance(total, int) and next_page * config.page_size >= total) or (+ len(rows) < config.page_size+ )++ next_window_start = window_start+ if not is_last_page and window_field and next_page * config.page_size >= SEARCH_RESULT_CAP:+ # Auth0 refuses to page past the first 1000 matches, so restart at page 0 with the+ # window's lower bound moved up to the newest row seen.+ newest = _max_window_value(rows, window_field)+ if newest is None:+ raise Auth0PaginationStalledError(+ f"{PAGINATION_STALLED_ERROR}: {config.name} rows carry no `{window_field}` value to slide on"+ )+ if newest == window_start:+ raise Auth0PaginationStalledError(+ f"{PAGINATION_STALLED_ERROR}: more than {SEARCH_RESULT_CAP} {config.name} rows share "+ f"`{window_field}` = {newest}"+ )+ next_window_start = newest+ next_page = 0++ yield _Page(+ rows=rows,+ position=Auth0ResumeConfig(page=page, window_start=window_start),+ next_position=None if is_last_page else Auth0ResumeConfig(page=next_page, window_start=next_window_start),+ )++ if is_last_page:+ return++ page = next_page+ window_start = next_window_start+++def _iter_checkpoint_pages(+ fetch: Callable[..., Any],+ domain: str,+ config: Auth0EndpointConfig,+ api_version: str,+ start: Optional[str],+ parent_id: Optional[str] = None,+) -> Iterator[tuple[list[dict[str, Any]], Optional[str]]]:+ """Each page of a `from`/`take` collection, with the checkpoint after it (None after the last).++ A collection without rows still yields one empty last page, so the caller stages a position+ for every parent.+ """+ checkpoint = start+ for _ in range(MAX_PAGES):+ # Without `include_totals` Auth0 answers with a bare array that carries no `next` token.+ params: dict[str, Any] = {"take": CHECKPOINT_PAGE_SIZE, "include_totals": "true", **config.extra_params}+ if checkpoint:+ params["from"] = checkpoint+ # A parent deleted after the parent page was read returns 404; it has no children left.+ data = fetch(+ _build_url(domain, config, api_version, params, parent_id=parent_id), not_found_ok=parent_id is not None+ )+ rows = _extract_rows(data, config)+ next_checkpoint = data.get("next") if isinstance(data, dict) else None+ if not rows or not isinstance(next_checkpoint, str) or not next_checkpoint:+ yield rows, None+ return+ yield rows, next_checkpoint+ checkpoint = next_checkpoint+ raise Auth0PaginationLimitError(+ f"Auth0 pagination for {config.name} exceeded {MAX_PAGES} pages without terminating"+ )+++def _iter_parent_pages(+ fetch: Callable[..., Any],+ domain: str,+ parent_config: Auth0EndpointConfig,+ api_version: str,+ position: Auth0ResumeConfig,+ fields: Optional[str],+) -> Iterator[_Page]:+ if not parent_config.checkpoint_parent:+ yield from _iter_pages(+ fetch, domain, parent_config, api_version, parent_config.full_refresh_window_field, position, fields+ )+ return++ checkpoint = position.parent_checkpoint+ for rows, next_checkpoint in _iter_checkpoint_pages(fetch, domain, parent_config, api_version, checkpoint):+ if not rows:+ return+ yield _Page(+ rows=rows,+ position=Auth0ResumeConfig(page=0, parent_checkpoint=checkpoint),+ next_position=None+ if next_checkpoint is None+ else Auth0ResumeConfig(page=0, parent_checkpoint=next_checkpoint),+ )+ checkpoint = next_checkpoint+++def _iter_fan_out_rows(+ fetch: Callable[..., Any],+ domain: str,+ config: Auth0EndpointConfig,+ api_version: str,+ resumable_source_manager: ResumableSourceManager[Auth0ResumeConfig],+ logger: FilteringBoundLogger,+) -> Iterator[list[dict[str, Any]]]:+ assert config.parent is not None+ parent_config = AUTH0_ENDPOINTS[config.parent]++ resume_config = resumable_source_manager.load_state() if resumable_source_manager.can_resume() else None+ position = resume_config or Auth0ResumeConfig(page=0)+ resume_after = position.last_parent_id+ # Applies only to the first parent the resumed run reads.+ child_start = position.child_checkpoint+ if resume_config is not None:+ logger.debug(f"Auth0: resuming {config.name} from {config.parent} page {position.page} after {resume_after}")++ for parent_page in _iter_parent_pages(+ fetch,+ domain,+ parent_config,+ api_version,+ dataclasses.replace(position, last_parent_id=None, child_checkpoint=None),+ config.parent_fields,+ ):+ parent_ids = [+ str(row[config.parent_id_field])+ for row in parent_page.rows+ if isinstance(row, dict) and row.get(config.parent_id_field)+ ]+ last_done: Optional[str] = None+ if resume_after is not None:+ # If the resume parent left the page, re-read the whole page: duplicates merge away,+ # a skipped parent would not.+ if resume_after in parent_ids:+ parent_ids = parent_ids[parent_ids.index(resume_after) + 1 :]+ last_done = resume_after+ else:+ child_start = None+ resume_after = None++ for index, parent_id in enumerate(parent_ids):+ # Staged before each yield: a page that ends this parent moves the cursor past it, any+ # earlier page moves the cursor to this parent's next child page.+ done = (+ parent_page.next_position+ if index == len(parent_ids) - 1+ else dataclasses.replace(parent_page.position, last_parent_id=parent_id)+ )+ for rows, next_checkpoint in _iter_checkpoint_pages(+ fetch, domain, config, api_version, child_start, parent_id=parent_id+ ):+ state = (+ done+ if next_checkpoint is None+ else dataclasses.replace(+ parent_page.position, last_parent_id=last_done, child_checkpoint=next_checkpoint+ )+ )+ if state is not None:+ resumable_source_manager.save_state(state)+ shaped = _strip_credential_fields(_shape_child_rows(rows, config, parent_id), config.name)+ if shaped:+ yield shaped+ else:+ resumable_source_manager.safe_point()+ child_start = None+ last_done = parent_id+ child_start = Nonedef get_rows(@@ -497,7 +772,7 @@ def get_rows()session = make_tracked_session(capture=config.capture_samples, redact_values=(client_secret,))- def fetch(url: str) -> Any:+ def fetch(url: str, not_found_ok: bool = False) -> Any:# The tracked session already retries 429 and transient 5xx honoring Retry-After, which# is what Auth0's rate limiter returns — no second retry layer here.headers = {"Authorization": f"Bearer {token_manager.get_token()}", "Accept": "application/json"}@@ -514,6 +789,8 @@ def get_rows(raise Auth0HostNotAllowedError(f"Auth0 returned an unexpected redirect (status={response.status_code}); refusing to follow it")+ if not_found_ok and response.status_code == 404:+ return Noneif not response.ok:logger.error(f"Auth0 API error: status={response.status_code}, body={_read_body_preview(response)}, url={url}"@@ -523,6 +800,10 @@ def get_rows(finally:response.close()+ if config.parent is not None:+ yield from _iter_fan_out_rows(fetch, domain, config, api_version, resumable_source_manager, logger)+ return+if not config.paginated:data = fetch(_build_url(domain, config, api_version, {}))rows = _strip_credential_fields(_extract_rows(data, config), endpoint)@@ -531,67 +812,23 @@ def get_rows(returnwindow_field = resolve_window_field(config, should_use_incremental_field, incremental_field)- window_start: Optional[str] = None+ position = Auth0ResumeConfig(page=0)if window_field and should_use_incremental_field and db_incremental_field_last_value:- window_start = _format_window_value(db_incremental_field_last_value)+ position = Auth0ResumeConfig(page=0, window_start=_format_window_value(db_incremental_field_last_value))resume_config = resumable_source_manager.load_state() if resumable_source_manager.can_resume() else None- page = 0if resume_config is not None:- page = resume_config.page- window_start = resume_config.window_start- logger.debug(f"Auth0: resuming {endpoint} from page {page} of window {window_start}")+ position = resume_config+ logger.debug(f"Auth0: resuming {endpoint} from page {position.page} of window {position.window_start}")- pages_fetched = 0- while True:- if pages_fetched >= MAX_PAGES:- raise Auth0PaginationLimitError(- f"Auth0 pagination for {endpoint} exceeded {MAX_PAGES} pages without terminating"- )-- data = fetch(_build_url(domain, config, api_version, _build_params(config, page, window_field, window_start)))- pages_fetched += 1-- rows = _strip_credential_fields(_extract_rows(data, config), endpoint)- if not rows:- break-- next_page = page + 1- total = data.get("total") if isinstance(data, dict) else None- is_last_page = (isinstance(total, int) and next_page * config.page_size >= total) or (- len(rows) < config.page_size- )-- next_window_start = window_start- if not is_last_page and window_field and next_page * config.page_size >= SEARCH_RESULT_CAP:- # Auth0 refuses to page past the first 1000 matches, so restart at page 0 with the- # window's lower bound moved up to the newest row seen.- newest = _max_window_value(rows, window_field)- if newest is None:- raise Auth0PaginationStalledError(- f"{PAGINATION_STALLED_ERROR}: {endpoint} rows carry no `{window_field}` value to slide on"- )- if newest == window_start:- raise Auth0PaginationStalledError(- f"{PAGINATION_STALLED_ERROR}: more than {SEARCH_RESULT_CAP} {endpoint} rows share "- f"`{window_field}` = {newest}"- )- next_window_start = newest- next_page = 0-- if not is_last_page:+ for page in _iter_pages(fetch, domain, config, api_version, window_field, position):+ if page.next_position is not None:# Save BEFORE yielding so the state already covers this page once the pipeline commits# it; a worker shutdown right after the yield then resumes from the next page instead# of silently re-reading (and, for a full refresh, duplicating) this one.- resumable_source_manager.save_state(Auth0ResumeConfig(page=next_page, window_start=next_window_start))+ resumable_source_manager.save_state(page.next_position)- yield rows-- if is_last_page:- break-- page = next_page- window_start = next_window_start+ yield _strip_credential_fields(page.rows, endpoint)def auth0_source(@@ -624,7 +861,7 @@ def auth0_source(incremental_field=incremental_field,db_incremental_field_last_value=db_incremental_field_last_value,),- primary_keys=[config.primary_key],+ primary_keys=list(config.primary_keys),partition_keys=[config.partition_key] if config.partition_key else None,partition_mode="datetime" if config.partition_key else None,# Every windowed collection is requested with `sort=<field>:1`, so rows arrive oldestdiff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/auth0/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/auth0/canonical_descriptions.pyindex d17415103592..dcf2650e57ca 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/auth0/canonical_descriptions.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/auth0/canonical_descriptions.py@@ -154,4 +154,55 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {"filters": "Event categories the stream is limited to, when filtering is configured.",},},+ "user_roles": {+ "description": "Roles assigned directly to each user at the tenant level, one row per role and user. Role assignments made inside organizations are in organization_member_roles.",+ "docs_url": "https://auth0.com/docs/api/management/v2/roles/get-role-user",+ "columns": {+ "role_id": "Identifier of the assigned role.",+ "user_id": "Identifier of the user the role is assigned to.",+ "email": "Email address of the user.",+ "name": "Full name of the user.",+ "picture": "URL of the user's profile picture.",+ },+ },+ "organization_members": {+ "description": "Members of each organization, one row per organization and user.",+ "docs_url": "https://auth0.com/docs/api/management/v2/organizations/get-organization-members",+ "columns": {+ "organization_id": "Identifier of the organization.",+ "user_id": "Identifier of the member user.",+ "email": "Email address of the member.",+ "name": "Full name of the member.",+ "picture": "URL of the member's profile picture.",+ },+ },+ "organization_member_roles": {+ "description": "Roles assigned directly to each member within an organization, one row per organization, member, and role.",+ "docs_url": "https://auth0.com/docs/api/management/v2/organizations/get-organization-member-roles",+ "columns": {+ "organization_id": "Identifier of the organization.",+ "user_id": "Identifier of the member user.",+ "id": "Identifier of the assigned role.",+ "name": "Name of the assigned role.",+ },+ },+ "user_sessions": {+ "description": "Sessions of each user that have not yet expired or been revoked.",+ "docs_url": "https://auth0.com/docs/api/management/v2/users/get-sessions-for-user",+ "columns": {+ "id": "Unique identifier of the session.",+ "user_id": "Identifier of the user the session belongs to.",+ "created_at": "Timestamp of when the session was created.",+ "updated_at": "Timestamp of the most recent change to the session.",+ "authenticated_at": "Timestamp of the user's most recent authentication in this session.",+ "last_interacted_at": "Timestamp of the user's most recent interaction with the session.",+ "idle_expires_at": "Timestamp of when the session expires if the user stays inactive.",+ "expires_at": "Timestamp of when the session expires regardless of activity.",+ "device": "Initial and most recent user agent, IP address, and ASN seen for the session.",+ "clients": "Applications the session has been used with, by client_id.",+ "authentication": "Authentication methods completed in the session, with their timestamps.",+ "cookie": "Cookie mode of the session (persistent or non-persistent).",+ "session_metadata": "Custom metadata attached to the session.",+ },+ },}diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/auth0/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/auth0/settings.pyindex 1cea57caf448..6d888c961dd4 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/auth0/settings.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/auth0/settings.py@@ -11,6 +11,10 @@ SEARCH_RESULT_CAP = 1000DEFAULT_PAGE_SIZE = 100+# `take` defaults to 50 on the checkpoint-paginated collections, and no larger maximum is+# documented for every one of them.+CHECKPOINT_PAGE_SIZE = 50+# The Management API caps `per_page` at 100 on every collection that accepts it.MAX_PAGE_SIZE = 100@@ -24,7 +28,7 @@ class Auth0EndpointConfig:# Key the rows sit under in the response envelope. None means the endpoint returns a bare# JSON array and takes no pagination parameters.data_key: Optional[str] = None- primary_key: str = "id"+ primary_keys: tuple[str, ...] = ("id",)page_size: int = DEFAULT_PAGE_SIZE# Only the collections that document `include_totals` accept it; the Actions API always# returns its own totals envelope instead.@@ -46,6 +50,22 @@ class Auth0EndpointConfig:capture_samples: bool = False# Management API scope the token must carry, surfaced when a scoped probe is denied.required_scope: str = ""+ # Fan-out children: the collection walked for parent ids (`{parent_id}` in the path), the+ # parent row field that holds the id, and the column the id is written to on each child row.+ # Children are paged with `from`/`take` and a `next` token, because Auth0 refuses offset+ # paging past 1000 rows on these collections and offers no window to slide on.+ parent: Optional[str] = None+ parent_id_field: str = "id"+ parent_id_column: Optional[str] = None+ # Fields requested from the parent collection to keep its pages small. Must include the+ # parent's window field so the parent walk can slide past the search result cap.+ parent_fields: Optional[str] = None+ extra_params: dict[str, str] = field(default_factory=dict)+ # Nested array each response row is expanded into, one output row per item.+ explode_field: Optional[str] = None+ # Walked with `from`/`take` when it is a fan-out parent: offset paging stops at 1000 rows and+ # the collection has no window to slide on.+ checkpoint_parent: bool = False@propertydef paginated(self) -> bool:@@ -57,7 +77,7 @@ AUTH0_ENDPOINTS: dict[str, Auth0EndpointConfig] = {name="users",path_template="/api/{version}/users",data_key="users",- primary_key="user_id",+ primary_keys=("user_id",),window_field="updated_at",full_refresh_window_field="created_at",search_engine="v3",@@ -69,7 +89,7 @@ AUTH0_ENDPOINTS: dict[str, Auth0EndpointConfig] = {name="logs",path_template="/api/{version}/logs",data_key="logs",- primary_key="log_id",+ primary_keys=("log_id",),window_field="date",full_refresh_window_field="date",incremental_fields=[incremental_field("date")],@@ -80,7 +100,7 @@ AUTH0_ENDPOINTS: dict[str, Auth0EndpointConfig] = {name="clients",path_template="/api/{version}/clients",data_key="clients",- primary_key="client_id",+ primary_keys=("client_id",),required_scope="read:clients",),"connections": Auth0EndpointConfig(@@ -102,6 +122,7 @@ AUTH0_ENDPOINTS: dict[str, Auth0EndpointConfig] = {data_key="organizations",capture_samples=True,required_scope="read:organizations",+ checkpoint_parent=True,),"resource_servers": Auth0EndpointConfig(name="resource_servers",@@ -122,6 +143,51 @@ AUTH0_ENDPOINTS: dict[str, Auth0EndpointConfig] = {path_template="/api/{version}/log-streams",required_scope="read:log_streams",),+ # Walks role -> users rather than user -> roles: tenants hold far fewer roles than users, so+ # this needs one request per role instead of one per user for the same assignments.+ "user_roles": Auth0EndpointConfig(+ name="user_roles",+ path_template="/api/{version}/roles/{parent_id}/users",+ data_key="users",+ primary_keys=("role_id", "user_id"),+ required_scope="read:users",+ parent="roles",+ parent_id_column="role_id",+ ),+ "organization_members": Auth0EndpointConfig(+ name="organization_members",+ path_template="/api/{version}/organizations/{parent_id}/members",+ data_key="members",+ primary_keys=("organization_id", "user_id"),+ required_scope="read:organization_members",+ parent="organizations",+ parent_id_column="organization_id",+ ),+ # The members collection returns each member's directly assigned roles when asked for them,+ # which replaces one `members/{user_id}/roles` request per member with one page per 50.+ "organization_member_roles": Auth0EndpointConfig(+ name="organization_member_roles",+ path_template="/api/{version}/organizations/{parent_id}/members",+ data_key="members",+ primary_keys=("organization_id", "user_id", "id"),+ extra_params={"fields": "user_id,roles", "include_fields": "true"},+ explode_field="roles",+ required_scope="read:organization_member_roles",+ parent="organizations",+ parent_id_column="organization_id",+ ),+ # Auth0 has no tenant-wide session listing, so this needs one request per user.+ "user_sessions": Auth0EndpointConfig(+ name="user_sessions",+ path_template="/api/{version}/users/{parent_id}/sessions",+ data_key="sessions",+ primary_keys=("user_id", "id"),+ required_scope="read:sessions",+ parent="users",+ parent_id_field="user_id",+ parent_id_column="user_id",+ parent_fields="user_id,created_at",+ ),}ENDPOINTS = tuple(AUTH0_ENDPOINTS.keys())diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/auth0/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/auth0/source.pyindex f82dbddb79f3..65c17e4e37f2 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/auth0/source.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/auth0/source.py@@ -193,7 +193,7 @@ class Auth0Source(ResumableSource[Auth0SourceConfig, Auth0ResumeConfig]):keywords=["okta", "identity", "sso", "authentication", "ciam"],caption="""Enter your Auth0 tenant domain and a machine-to-machine application's credentials to pull your users, tenant logs, and tenant configuration into the PostHog Data warehouse.-In the Auth0 dashboard go to **Applications > Applications**, create a **Machine to Machine** application, and authorize it for the **Auth0 Management API**. Grant the read scopes for the tables you want to sync: `read:users`, `read:logs`, `read:clients`, `read:connections`, `read:roles`, `read:organizations`, `read:resource_servers`, `read:actions`, `read:log_streams`.+In the Auth0 dashboard go to **Applications > Applications**, create a **Machine to Machine** application, and authorize it for the **Auth0 Management API**. Grant the read scopes for the tables you want to sync: `read:users`, `read:logs`, `read:clients`, `read:connections`, `read:roles`, `read:organizations`, `read:resource_servers`, `read:actions`, `read:log_streams`, `read:organization_members`, `read:organization_member_roles`, `read:sessions`.Use your canonical tenant domain (for example `your-tenant.us.auth0.com`), not a custom domain. Auth0 only issues Management API tokens for the canonical one.""",iconPath="/static/services/auth0.png",diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/auth0/tests/test_auth0.py b/products/warehouse_sources/backend/temporal/data_imports/sources/auth0/tests/test_auth0.pyindex d701a4b5c4c8..4d3b16022965 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/auth0/tests/test_auth0.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/auth0/tests/test_auth0.py@@ -370,6 +370,114 @@ class TestGetRows:assert isinstance(result.raised, Auth0HostNotAllowedError)assert result.requests == []+ def test_fan_out_walks_each_parent_through_its_checkpoint_pages(self) -> None:+ orgs = {"organizations": [{"id": "org_1"}, {"id": "org_2"}, {"id": "org_3"}]}+ members_page_1 = {+ "members": [{"user_id": "auth0|1", "roles": [{"id": "rol_a", "name": "Admin"}]}],+ "next": "cp1",+ }+ members_page_2 = {+ "members": [+ {"user_id": "auth0|2", "roles": [{"id": "rol_a", "name": "Admin"}, {"id": "rol_b", "name": "Viewer"}]},+ {"user_id": "auth0|3", "roles": []},+ ]+ }+ result = self._run(+ "organization_member_roles",+ [+ ScriptedResponse(json=orgs),+ ScriptedResponse(json=members_page_1),+ ScriptedResponse(json=members_page_2),+ # Deleted after the organization page was read.+ ScriptedResponse(status=404),+ ScriptedResponse(json={"members": []}),+ ],+ )++ assert result.raised is None+ assert result.rows == [+ {"user_id": "auth0|1", "id": "rol_a", "name": "Admin", "organization_id": "org_1"},+ {"user_id": "auth0|2", "id": "rol_a", "name": "Admin", "organization_id": "org_1"},+ {"user_id": "auth0|2", "id": "rol_b", "name": "Viewer", "organization_id": "org_1"},+ ]+ assert result.paths[2:] == [+ "/api/v2/organizations/org_1/members",+ "/api/v2/organizations/org_1/members",+ "/api/v2/organizations/org_2/members",+ "/api/v2/organizations/org_3/members",+ ]+ assert result.params("from")[2:] == [None, "cp1", None, None]+ assert result.requests[2].param("fields") == "user_id,roles"+ # Without it Auth0 returns a bare array with no `next` token.+ assert set(result.params("include_totals")[1:]) == {"true"}+ assert result.saved_states == [+ Auth0ResumeConfig(page=0, last_parent_id=None, child_checkpoint="cp1"),+ Auth0ResumeConfig(page=0, last_parent_id="org_1"),+ Auth0ResumeConfig(page=0, last_parent_id="org_2"),+ ]++ def test_fan_out_over_users_escapes_the_user_id_and_slides_the_parent_window(self) -> None:+ users = {"users": [{"user_id": "auth0|1", "created_at": "2024-01-01T00:00:00.000Z"}], "total": 1}+ session = {"id": "sess_1", "user_id": "auth0|1", "created_at": "2024-02-01T00:00:00.000Z"}+ result = self._run(+ "user_sessions", [ScriptedResponse(json=users), ScriptedResponse(json={"sessions": [session]})]+ )++ assert result.raised is None+ assert result.rows == [session]+ assert result.requests[1].param("fields") == "user_id,created_at"+ assert result.requests[1].param("sort") == "created_at:1"+ assert result.paths[2] == "/api/v2/users/auth0%7C1/sessions"++ @pytest.mark.parametrize(+ "last_parent_id, child_checkpoint, expected_member_requests",+ [+ ("org_1", None, [("/api/v2/organizations/org_2/members", None)]),+ ("org_1", "cp9", [("/api/v2/organizations/org_2/members", "cp9")]),+ (+ None,+ "cp9",+ [("/api/v2/organizations/org_1/members", "cp9"), ("/api/v2/organizations/org_2/members", None)],+ ),+ # The resume parent left the page, so the whole page is read again rather than skipped.+ (+ "org_gone",+ "cp9",+ [("/api/v2/organizations/org_1/members", None), ("/api/v2/organizations/org_2/members", None)],+ ),+ ],+ )+ def test_fan_out_resumes_at_the_next_unread_child_page(+ self,+ last_parent_id: Optional[str],+ child_checkpoint: Optional[str],+ expected_member_requests: list[tuple[str, Optional[str]]],+ ) -> None:+ orgs = {"organizations": [{"id": "org_1"}, {"id": "org_2"}]}+ result = self._run(+ "organization_members",+ [ScriptedResponse(json=orgs), *[ScriptedResponse(json={"members": []})] * len(expected_member_requests)],+ resume_state=Auth0ResumeConfig(+ page=0, parent_checkpoint="cp_orgs", last_parent_id=last_parent_id, child_checkpoint=child_checkpoint+ ),+ )++ assert result.raised is None+ # Organizations are walked by checkpoint, so tenants with over 1000 of them are covered.+ assert result.requests[1].param("from") == "cp_orgs"+ assert result.requests[1].param("page") is None+ assert list(zip(result.paths[2:], result.params("from")[2:])) == expected_member_requests++ def test_role_members_omit_identity_provider_tokens(self) -> None:+ roles = {"roles": [{"id": "rol_1"}], "total": 1}+ member = {"user_id": "auth0|1", "identities": [{"provider": "google-oauth2", "access_token": "shh"}]}+ result = self._run("user_roles", [ScriptedResponse(json=roles), ScriptedResponse(json={"users": [member]})])++ assert result.raised is None+ assert result.rows == [+ {"user_id": "auth0|1", "identities": [{"provider": "google-oauth2"}], "role_id": "rol_1"}+ ]+@pytest.mark.parametrize("status_code", [400, 401, 403, 404])def test_client_errors_propagate(self, status_code: int) -> None:result = self._run("roles", [ScriptedResponse(status=status_code)])@@ -437,6 +545,47 @@ class TestValidateCredentials:assert error is not Noneassert scope in error+ @pytest.mark.parametrize(+ "parent_response, child_response, expected",+ [+ (ScriptedResponse(json={"organizations": [], "total": 0}), None, (True, None)),+ (+ ScriptedResponse(status=403),+ None,+ (+ False,+ "Your Auth0 application is missing the `read:organizations` scope for organization_members",+ ),+ ),+ (+ ScriptedResponse(json={"organizations": [{"id": "org_1"}], "total": 1}),+ ScriptedResponse(status=403),+ (+ False,+ "Your Auth0 application is missing the `read:organization_members` scope for organization_members",+ ),+ ),+ (+ ScriptedResponse(json={"organizations": [{"id": "org_1"}], "total": 1}),+ ScriptedResponse(json={"members": []}),+ (True, None),+ ),+ ],+ )+ def test_scoped_probe_of_a_child_collection_uses_a_real_parent(+ self,+ parent_response: ScriptedResponse,+ child_response: ScriptedResponse | None,+ expected: tuple[bool, str | None],+ ) -> None:+ script = [ScriptedResponse(json=TOKEN_JSON), parent_response, *([child_response] if child_response else [])]+ with scripted_network(script) as network:+ result = validate_credentials(DOMAIN, "cid", "secret", "v2", schema_name="organization_members", team_id=1)++ assert result == expected+ if child_response is not None:+ assert network.requests_log[-1].path == "/api/v2/organizations/org_1/members"+def test_scoped_probe_refuses_a_redirect(self) -> None:ok, error = self._validate(post_response=ScriptedResponse(json=TOKEN_JSON),