:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(data-warehouse): add 8 missing tables to the FusionAuth source (#123608)
Tom Owers64f649c88858
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.mdindex 1fb53e7a680a..1765c645b23c 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md@@ -5106,10 +5106,10 @@ Diffed against: <https://raw.githubusercontent.com/FusionAuth/fusionauth-openapi- [x] `tenant (POST /api/tenant/search)` — lookup resolving the tenantId on users and login records — the top-level isolation dimension (high)- [x] `group (POST /api/group/search)` — lookup naming the groups referenced by user memberships (high)- [x] `group/member (POST /api/group/member/search)` — the user-to-group membership table; group-level access analysis is impossible without it (high)-- [ ] `user/registration (GET /api/user/registration/{userId}/{applicationId})` — which users are registered to which applications, with roles and registration dates (medium)-- [ ] `user/consent (GET /api/user/consent, POST /api/consent/search)` — consent grants per user plus the consent definition lookup — compliance reporting (medium)-- [ ] `entity + entity/grant (POST /api/entity/search, /api/entity/grant/search)` — non-user entities and the grants linking them to users — the machine-to-machine authorization graph (medium)-- [ ] `user/action + user-action + user-action-reason` — moderation action history against users, plus the action and reason lookups that decode it (medium)+- [x] `user/registration (GET /api/user/registration/{userId}/{applicationId})` — which users are registered to which applications, with roles and registration dates (medium)+- [x] `user/consent (GET /api/user/consent, POST /api/consent/search)` — consent grants per user plus the consent definition lookup — compliance reporting (medium)+- [x] `entity + entity/grant (POST /api/entity/search, /api/entity/grant/search)` — non-user entities and the grants linking them to users — the machine-to-machine authorization graph (medium)+- [x] `user/action + user-action + user-action-reason` — moderation action history against users, plus the action and reason lookups that decode it (medium)- [ ] `jwt/refresh (GET /api/jwt/refresh)` — active refresh tokens per user — session longevity and device analysis (medium)- [ ] `report/daily-active-user and report/monthly-active-user` — FusionAuth's headline DAU/MAU metrics, precomputed by the vendor (medium)- [ ] `identity-provider/link (GET /api/identity-provider/link)` — which users are linked to which social/SSO provider — signup source breakdown (medium)diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/fusionauth/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/fusionauth/canonical_descriptions.pyindex 68aa3bfe88bd..63c534c1da18 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/fusionauth/canonical_descriptions.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/fusionauth/canonical_descriptions.py@@ -102,4 +102,114 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {"insertInstant": "The instant (epoch milliseconds) the user was added to the group.",},},+ "Consents": {+ "description": "Consent types defined in FusionAuth, such as marketing or terms-of-service consent.",+ "docs_url": "https://fusionauth.io/docs/apis/consents",+ "columns": {+ "id": "Unique identifier for the consent.",+ "name": "The name of the consent.",+ "values": "The values a user can choose from when granting the consent.",+ "multipleValuesAllowed": "Whether a user can grant more than one value of the consent.",+ "defaultMinimumAgeForSelfConsent": "The minimum age at which a user can grant the consent for themselves.",+ "insertInstant": "The instant (epoch milliseconds) the consent was created.",+ "lastUpdateInstant": "The instant (epoch milliseconds) the consent was last updated.",+ },+ },+ "UserConsents": {+ "description": "Consents granted by or for each user, with their status and chosen values.",+ "docs_url": "https://fusionauth.io/docs/apis/consents",+ "columns": {+ "id": "Unique identifier for the user consent.",+ "userId": "The unique identifier of the user the consent applies to.",+ "giverUserId": "The unique identifier of the user who gave the consent, such as a parent.",+ "consentId": "The unique identifier of the consent type.",+ "consent": "The consent type that was granted.",+ "status": "The status of the user consent (Active or Revoked).",+ "values": "The consent values the user chose.",+ "insertInstant": "The instant (epoch milliseconds) the consent was granted.",+ "lastUpdateInstant": "The instant (epoch milliseconds) the user consent was last updated.",+ },+ },+ "Entities": {+ "description": "Non-user entities, such as services or devices, used for machine-to-machine authorization.",+ "docs_url": "https://fusionauth.io/docs/apis/entities/entities",+ "columns": {+ "id": "Unique identifier for the entity.",+ "name": "The name of the entity.",+ "clientId": "The OAuth client id of the entity.",+ "type": "The entity type, which defines the permissions that can be granted on the entity.",+ "tenantId": "The unique identifier of the tenant the entity belongs to.",+ "parentId": "The unique identifier of the parent entity, if any.",+ "insertInstant": "The instant (epoch milliseconds) the entity was created.",+ "lastUpdateInstant": "The instant (epoch milliseconds) the entity was last updated.",+ },+ },+ "EntityGrants": {+ "description": "Grants that give a user or another entity a set of permissions on an entity.",+ "docs_url": "https://fusionauth.io/docs/apis/entities/grants",+ "columns": {+ "id": "Unique identifier for the grant.",+ "entity": "The entity the grant gives access to.",+ "userId": "The unique identifier of the user who receives the grant, if any.",+ "recipientEntityId": "The unique identifier of the entity that receives the grant, if any.",+ "permissions": "The permissions granted on the entity.",+ "insertInstant": "The instant (epoch milliseconds) the grant was created.",+ "lastUpdateInstant": "The instant (epoch milliseconds) the grant was last updated.",+ },+ },+ "UserRegistrations": {+ "description": "Registrations of users to applications, with the roles each user has in each application.",+ "docs_url": "https://fusionauth.io/docs/apis/registrations",+ "columns": {+ "id": "Unique identifier for the registration.",+ "userId": "The unique identifier of the registered user.",+ "applicationId": "The unique identifier of the application the user is registered to.",+ "roles": "The application roles granted to the user.",+ "username": "The user's username for this application, if any.",+ "verified": "Whether the registration is verified.",+ "insertInstant": "The instant (epoch milliseconds) the registration was created.",+ "lastLoginInstant": "The instant (epoch milliseconds) the user last logged in to the application.",+ "lastUpdateInstant": "The instant (epoch milliseconds) the registration was last updated.",+ },+ },+ "UserActionLogs": {+ "description": "Actions taken on users, such as account locks or warnings, with their reasons and expiry.",+ "docs_url": "https://fusionauth.io/docs/apis/actioning-users",+ "columns": {+ "id": "Unique identifier for the action log entry.",+ "actioneeUserId": "The unique identifier of the user the action was taken on.",+ "actionerUserId": "The unique identifier of the user who took the action.",+ "userActionId": "The unique identifier of the user action that was taken.",+ "name": "The name of the user action.",+ "reason": "The reason for the action.",+ "reasonCode": "The code of the reason for the action.",+ "comment": "A comment the actioner added to the action.",+ "expiry": "The instant (epoch milliseconds) a time-based action expires.",+ "insertInstant": "The instant (epoch milliseconds) the action was taken.",+ },+ },+ "UserActions": {+ "description": "User actions that can be taken on users, such as locking an account or sending a warning.",+ "docs_url": "https://fusionauth.io/docs/apis/user-actions",+ "columns": {+ "id": "Unique identifier for the user action.",+ "name": "The name of the user action.",+ "active": "Whether the user action is active.",+ "preventLogin": "Whether the action prevents the user from logging in while it is active.",+ "temporal": "Whether the action is time based and has an expiry.",+ "insertInstant": "The instant (epoch milliseconds) the user action was created.",+ "lastUpdateInstant": "The instant (epoch milliseconds) the user action was last updated.",+ },+ },+ "UserActionReasons": {+ "description": "Reasons that can be given when an action is taken on a user.",+ "docs_url": "https://fusionauth.io/docs/apis/user-action-reasons",+ "columns": {+ "id": "Unique identifier for the user action reason.",+ "code": "The short code of the reason.",+ "text": "The text of the reason.",+ "insertInstant": "The instant (epoch milliseconds) the reason was created.",+ "lastUpdateInstant": "The instant (epoch milliseconds) the reason was last updated.",+ },+ },}diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/fusionauth/fusionauth.py b/products/warehouse_sources/backend/temporal/data_imports/sources/fusionauth/fusionauth.pyindex 388b6d56facf..2411d2121698 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/fusionauth/fusionauth.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/fusionauth/fusionauth.py@@ -1,7 +1,7 @@import threadingimport dataclasses-from collections.abc import Iterator-from typing import Any, Optional+from collections.abc import Callable, Iterable, Iterator+from typing import Any, Optional, castfrom urllib.parse import urlparseimport requests@@ -13,7 +13,12 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.resRESTAPIConfig,rest_api_resource,)+from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.fanout import (+ build_dependent_resource,+)from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.paginators import BasePaginator+from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.resource import Resource+from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.typing import ClientConfigfrom products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManagerfrom products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponsefrom products.warehouse_sources.backend.temporal.data_imports.sources.fusionauth.settings import (@@ -21,6 +26,8 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.fusionauthFusionAuthEndpointConfig,)+USERS_ENDPOINT = "Users"+HOST_NOT_ALLOWED_ERROR = "FusionAuth base URL is not allowed"HTTP_NOT_ALLOWED_ERROR = "FusionAuth base URL must use HTTPS"@@ -235,8 +242,8 @@ class FusionAuthOffsetPaginator(BasePaginator):def _build_search_body(config: FusionAuthEndpointConfig, search_extra: dict[str, Any]) -> dict[str, Any]:search: dict[str, Any] = {}- if config.name == "Users":- # `queryString: "*"` matches every user; sortFields keeps ordering stable across pages.+ if config.query_string_search:+ # `queryString: "*"` matches every row; sortFields keeps ordering stable across pages.search["queryString"] = "*"search["sortFields"] = [{"name": "insertInstant", "order": "asc"}]elif config.sort_mode == "asc":@@ -260,6 +267,29 @@ def _drop_fields(row: Any, paths: tuple[str, ...]) -> Any:return row+def _explode_rows(row: Any, field: str, exclude_fields: tuple[str, ...]) -> list[dict[str, Any]]:+ # Exploded rows come from the user search, so each child row carries its user's id.+ children = row.get(field) if isinstance(row, dict) else None+ if not isinstance(children, list):+ return []+ return [+ _drop_fields({**child, "userId": row.get("id")}, exclude_fields)+ for child in children+ if isinstance(child, dict)+ ]+++def _check_host(normalized_url: str, team_id: int) -> None:+ # Re-check at run time (not just at source-create) in case the base URL was edited to+ # now resolve to an internal address (SSRF / DNS rebinding). Only enforced on cloud. Refuse+ # plaintext HTTP before the key is used. Both raise before any request leaves the process.+ host_ok, host_err = _is_host_safe(_hostname(normalized_url), team_id)+ if not host_ok:+ raise FusionAuthHostNotAllowedError(host_err or HOST_NOT_ALLOWED_ERROR)+ if not _is_https(normalized_url):+ raise FusionAuthHostNotAllowedError(HTTP_NOT_ALLOWED_ERROR)++def _get_headers() -> dict[str, str]:return {"Accept": "application/json", "Content-Type": "application/json"}@@ -336,22 +366,42 @@ def fusionauth_source(config = FUSIONAUTH_ENDPOINTS[endpoint]normalized_url = normalize_base_url(base_url)+ def make_client_config(maximum_offset: Optional[int]) -> ClientConfig:+ return {+ "base_url": normalized_url,+ "headers": _get_headers(),+ # Auth rides on the framework auth config so it's redacted from logs/errors;+ # FusionAuth expects the raw key value, no "Bearer"/scheme prefix.+ "auth": {"type": "api_key", "api_key": api_key, "name": "Authorization", "location": "header"},+ "paginator": FusionAuthOffsetPaginator(limit=config.page_size, maximum_offset=maximum_offset),+ # A validated host could still 3xx to an internal address; refuse to follow+ # redirects so the Authorization header never replays to an unexpected host (SSRF).+ "allow_redirects": False,+ # Bound the customer-controlled host: pin a connect/read timeout and read every body+ # under a decoded-byte cap and wall-clock deadline so it can't hang a worker or OOM it.+ "session": _make_bounded_session(api_key),+ }++ def make_get_config(params: dict[str, Any]) -> RESTAPIConfig:+ return {+ "client": make_client_config(None),+ "resources": [+ {+ "name": endpoint,+ "endpoint": {+ "method": "GET",+ "path": config.path,+ "params": params,+ "data_selector": config.data_selector,+ "paginator": "single_page",+ },+ }+ ],+ }+def make_rest_config(search_extra: dict[str, Any], maximum_offset: Optional[int]) -> RESTAPIConfig:return {- "client": {- "base_url": normalized_url,- "headers": _get_headers(),- # Auth rides on the framework auth config so it's redacted from logs/errors;- # FusionAuth expects the raw key value, no "Bearer"/scheme prefix.- "auth": {"type": "api_key", "api_key": api_key, "name": "Authorization", "location": "header"},- "paginator": FusionAuthOffsetPaginator(limit=config.page_size, maximum_offset=maximum_offset),- # A validated host could still 3xx to an internal address; refuse to follow- # redirects so the Authorization header never replays to an unexpected host (SSRF).- "allow_redirects": False,- # Bound the customer-controlled host: pin a connect/read timeout and read every body- # under a decoded-byte cap and wall-clock deadline so it can't hang a worker or OOM it.- "session": _make_bounded_session(api_key),- },+ "client": make_client_config(maximum_offset),"resources": [{"name": endpoint,@@ -371,15 +421,70 @@ def fusionauth_source(if state and state.get("offset") is not None:resumable_source_manager.save_state(FusionAuthResumeConfig(offset=int(state["offset"])))+ def load_initial_state() -> Optional[dict[str, Any]]:+ if resumable_source_manager.can_resume():+ resume = resumable_source_manager.load_state()+ if resume is not None:+ return {"offset": resume.offset}+ return None++ def fanout_resource() -> Resource:+ # Per-user children have no list API, so they fan out over the user search. The pipeline+ # iterates this resource directly, so the framework reaches a safe point after each request.+ _check_host(normalized_url, team_id)+ assert config.fanout is not None+ users = FUSIONAUTH_ENDPOINTS[USERS_ENDPOINT]+ return cast(+ Resource,+ build_dependent_resource(+ endpoint_configs=FUSIONAUTH_ENDPOINTS,+ child_endpoint=endpoint,+ fanout=config.fanout,+ client_config=make_client_config(None),+ path_format_values={},+ team_id=team_id,+ job_id=job_id,+ db_incremental_field_last_value=None,+ page_size_param=None,+ parent_endpoint_extra={+ "method": "POST",+ "json": _build_search_body(users, {}),+ "data_selector": users.data_selector,+ "paginator": FusionAuthOffsetPaginator(limit=users.page_size, maximum_offset=users.maximum_offset),+ },+ child_endpoint_extra={"data_selector": config.data_selector, "paginator": "single_page"},+ ),+ )++ def exploded_resource() -> Resource:+ _check_host(normalized_url, team_id)+ assert config.explode_field is not None+ explode_field = config.explode_field+ resource = rest_api_resource(+ make_rest_config({}, config.maximum_offset),+ team_id,+ job_id,+ None,+ resume_hook=save_checkpoint,+ initial_paginator_state=load_initial_state(),+ )+ return resource.add_map(lambda row: _explode_rows(row, explode_field, config.exclude_fields))+def pages() -> Iterator[list[Any]]:- # Re-check at run time (not just at source-create) in case the base URL was edited to- # now resolve to an internal address (SSRF / DNS rebinding). Only enforced on cloud. Refuse- # plaintext HTTP before the key is used. Both raise before any request leaves the process.- host_ok, host_err = _is_host_safe(_hostname(normalized_url), team_id)- if not host_ok:- raise FusionAuthHostNotAllowedError(host_err or HOST_NOT_ALLOWED_ERROR)- if not _is_https(normalized_url):- raise FusionAuthHostNotAllowedError(HTTP_NOT_ALLOWED_ERROR)+ _check_host(normalized_url, team_id)++ if config.method == "GET":+ seen_ids: set[Any] = set()+ param_sets: list[dict[str, Any]] = [{}]+ if config.inactive_param:+ param_sets.append({config.inactive_param: "true"})+ for params in param_sets:+ for page in rest_api_resource(make_get_config(params), team_id, job_id, None):+ rows = [row for row in page if row.get("id") not in seen_ids]+ seen_ids.update(row.get("id") for row in rows)+ if rows:+ yield rows+ returnif config.sort_mode == "desc":# LoginRecords has no documented `orderBy`, so we can't assert ascending order.@@ -393,18 +498,13 @@ def fusionauth_source(or db_incremental_field_last_value is Noneand db_incremental_field_earliest_value is None):- initial_state = None- if resumable_source_manager.can_resume():- resume = resumable_source_manager.load_state()- if resume is not None:- initial_state = {"offset": resume.offset}resource = rest_api_resource(make_rest_config({}, config.maximum_offset),team_id,job_id,None,resume_hook=save_checkpoint,- initial_paginator_state=initial_state,+ initial_paginator_state=load_initial_state(),)yield from resourcereturn@@ -428,19 +528,13 @@ def fusionauth_source(if should_use_incremental_field and db_incremental_field_last_value is not None and config.incremental_fields:search_extra["start"] = db_incremental_field_last_value- initial_state = None- if resumable_source_manager.can_resume():- resume = resumable_source_manager.load_state()- if resume is not None:- initial_state = {"offset": resume.offset}-resource = rest_api_resource(make_rest_config(search_extra, config.maximum_offset),team_id,job_id,None,resume_hook=save_checkpoint,- initial_paginator_state=initial_state,+ initial_paginator_state=load_initial_state(),)yield from resource@@ -450,9 +544,15 @@ def fusionauth_source(page = [_drop_fields(row, config.exclude_fields) for row in page]yield page+ items_fn: Callable[[], Iterable[Any]] = items+ if config.fanout is not None:+ items_fn = fanout_resource+ elif config.explode_field is not None:+ items_fn = exploded_resource+return SourceResponse(name=endpoint,- items=items,+ items=items_fn,primary_keys=config.primary_keys,sort_mode=config.sort_mode,partition_count=1,diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/fusionauth/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/fusionauth/settings.pyindex 5f813a1ac26b..f19e048a860b 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/fusionauth/settings.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/fusionauth/settings.py@@ -1,6 +1,9 @@from dataclasses import dataclassfrom typing import Literal, Optional+from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.fanout import (+ DependentEndpointConfig,+)from products.warehouse_sources.backend.types import IncrementalField, IncrementalFieldType@@ -37,6 +40,18 @@ class FusionAuthEndpointConfig:order_by: str = "insertInstant ASC"# Dotted paths removed from every row before it is written, for secrets the API returns.exclude_fields: tuple[str, ...] = ()+ # "GET" endpoints are unpaginated lookups that return the whole collection in one response.+ method: Literal["POST", "GET"] = "POST"+ # GET only: also request the deactivated rows with `<param>=true`, deduplicated on `id`.+ inactive_param: Optional[str] = None+ # Elasticsearch-style searches (users, entities) take a `queryString` and `sortFields`+ # instead of the database searches' `orderBy`.+ query_string_search: bool = False+ # Emit one row per element of this list field of each parent row, instead of the row itself.+ explode_field: Optional[str] = None+ # Per-user child endpoints: one GET per user returned by the user search.+ fanout: Optional[DependentEndpointConfig] = None+ default_incremental_field: Optional[str] = None# Search endpoints only: FusionAuth also has non-search read APIs (e.g. GET /api/user/{id})@@ -56,6 +71,7 @@ FUSIONAUTH_ENDPOINTS: dict[str, FusionAuthEndpointConfig] = {# Standard search window caps around 10k results (search_after/nextResults token# pagination for deeper scans is not implemented here).maximum_offset=9_900,+ query_string_search=True,),"AuditLogs": FusionAuthEndpointConfig(name="AuditLogs",@@ -131,6 +147,105 @@ FUSIONAUTH_ENDPOINTS: dict[str, FusionAuthEndpointConfig] = {partition_key="insertInstant",order_by="id ASC",),+ "Consents": FusionAuthEndpointConfig(+ name="Consents",+ path="/api/consent/search",+ data_selector="consents",+ primary_keys=["id"],+ incremental_fields=[],+ partition_key="insertInstant",+ order_by="id ASC",+ ),+ "Entities": FusionAuthEndpointConfig(+ name="Entities",+ path="/api/entity/search",+ data_selector="entities",+ primary_keys=["id"],+ incremental_fields=[],+ partition_key="insertInstant",+ maximum_offset=9_900,+ query_string_search=True,+ exclude_fields=("clientSecret",),+ ),+ "EntityGrants": FusionAuthEndpointConfig(+ name="EntityGrants",+ path="/api/entity/grant/search",+ data_selector="grants",+ primary_keys=["id"],+ incremental_fields=[],+ partition_key="insertInstant",+ # Grant search only orders by `insertInstant` or `name`.+ order_by="insertInstant ASC",+ exclude_fields=("entity.clientSecret",),+ ),+ # Registrations have no list API (only GET /api/user/registration/{userId}/{applicationId}), but+ # every user search result embeds the user's registrations, so they are read from there.+ "UserRegistrations": FusionAuthEndpointConfig(+ name="UserRegistrations",+ path="/api/user/search",+ data_selector="users",+ # A user has at most one registration per application.+ primary_keys=["userId", "applicationId"],+ incremental_fields=[],+ partition_key="insertInstant",+ maximum_offset=9_900,+ query_string_search=True,+ explode_field="registrations",+ exclude_fields=("tokens", "authenticationToken"),+ ),+ "UserConsents": FusionAuthEndpointConfig(+ name="UserConsents",+ path="/api/user/consent?userId={user_id}",+ data_selector="userConsents",+ primary_keys=["id"],+ incremental_fields=[],+ partition_key="insertInstant",+ method="GET",+ fanout=DependentEndpointConfig(+ parent_name="Users",+ resolve_param="user_id",+ resolve_field="id",+ include_from_parent=[],+ # A user deleted between the user search and this request.+ child_response_actions=[{"status_code": 404, "action": "ignore"}],+ ),+ ),+ "UserActionLogs": FusionAuthEndpointConfig(+ name="UserActionLogs",+ path="/api/user/action?userId={user_id}",+ data_selector="actions",+ primary_keys=["id"],+ incremental_fields=[],+ partition_key="insertInstant",+ method="GET",+ fanout=DependentEndpointConfig(+ parent_name="Users",+ resolve_param="user_id",+ resolve_field="id",+ include_from_parent=[],+ child_response_actions=[{"status_code": 404, "action": "ignore"}],+ ),+ ),+ "UserActions": FusionAuthEndpointConfig(+ name="UserActions",+ path="/api/user-action",+ data_selector="userActions",+ primary_keys=["id"],+ incremental_fields=[],+ partition_key="insertInstant",+ method="GET",+ # Deactivated actions are listed separately, and past action logs can still reference them.+ inactive_param="inactive",+ ),+ "UserActionReasons": FusionAuthEndpointConfig(+ name="UserActionReasons",+ path="/api/user-action-reason",+ data_selector="userActionReasons",+ primary_keys=["id"],+ incremental_fields=[],+ partition_key="insertInstant",+ method="GET",+ ),}ENDPOINTS = tuple(FUSIONAUTH_ENDPOINTS.keys())diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/fusionauth/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/fusionauth/source.pyindex db01f20530a0..0084d111b1f4 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/fusionauth/source.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/fusionauth/source.py@@ -32,6 +32,17 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.generated_)from products.warehouse_sources.backend.types import ExternalDataSourceType+_SEARCH_WINDOW_DESCRIPTION = "Full sync only; the search API's result window is capped at ~10,000 rows."+_PER_USER_DESCRIPTION = "Full sync only; read for the first ~10,000 users the user search returns."++SCHEMA_DESCRIPTIONS: dict[str, str] = {+ "Users": _SEARCH_WINDOW_DESCRIPTION,+ "Entities": _SEARCH_WINDOW_DESCRIPTION,+ "UserRegistrations": _PER_USER_DESCRIPTION,+ "UserConsents": _PER_USER_DESCRIPTION,+ "UserActionLogs": _PER_USER_DESCRIPTION,+}+@SourceRegistry.registerclass FusionAuthSource(ResumableSource[FusionAuthSourceConfig, FusionAuthResumeConfig]):@@ -144,9 +155,7 @@ Audit logs, event logs, and login records require the Elasticsearch search enginsupports_incremental=bool(FUSIONAUTH_ENDPOINTS[endpoint].incremental_fields),supports_append=bool(FUSIONAUTH_ENDPOINTS[endpoint].incremental_fields),incremental_fields=FUSIONAUTH_ENDPOINTS[endpoint].incremental_fields,- description="Full sync only; the search API's result window is capped at ~10,000 rows."- if endpoint == "Users"- else None,+ description=SCHEMA_DESCRIPTIONS.get(endpoint),)for endpoint in ENDPOINTS]diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/fusionauth/tests/test_fusionauth.py b/products/warehouse_sources/backend/temporal/data_imports/sources/fusionauth/tests/test_fusionauth.pyindex 1937062448cd..0446d49dc071 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/fusionauth/tests/test_fusionauth.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/fusionauth/tests/test_fusionauth.py@@ -61,7 +61,14 @@ def _wire(session: mock.MagicMock, responses: list[Response]) -> list[dict[str,snapshots: list[dict[str, Any]] = []def _prepare(request: Any) -> mock.MagicMock:- snapshots.append({"url": request.url, "json": json.loads(json.dumps(request.json or {}))})+ snapshots.append(+ {+ "method": request.method,+ "url": request.url,+ "params": dict(request.params or {}),+ "json": json.loads(json.dumps(request.json or {})),+ }+ )return mock.MagicMock()session.prepare_request.side_effect = _prepare@@ -74,10 +81,12 @@ def _rows(source_response: Any) -> list[dict[str, Any]]:class TestBuildSearchBody:- def test_users_has_query_string_and_sort_fields(self):- body = _build_search_body(FUSIONAUTH_ENDPOINTS["Users"], {})+ @pytest.mark.parametrize("endpoint", ["Users", "Entities", "UserRegistrations"])+ def test_query_string_searches_have_query_string_and_sort_fields(self, endpoint):+ body = _build_search_body(FUSIONAUTH_ENDPOINTS[endpoint], {})assert body["search"]["queryString"] == "*"assert body["search"]["sortFields"] == [{"name": "insertInstant", "order": "asc"}]+ assert "orderBy" not in body["search"]class TestFusionAuthOffsetPaginator:@@ -206,6 +215,18 @@ parameterized_secret_cases = pytest.mark.parametrize(},),("Tenants", "tenants", {"id": "t2", "emailConfiguration": None}, {"id": "t2", "emailConfiguration": None}),+ (+ "Entities",+ "entities",+ {"id": "e1", "clientId": "e1", "clientSecret": "fake-secret"},+ {"id": "e1", "clientId": "e1"},+ ),+ (+ "EntityGrants",+ "grants",+ {"id": "g1", "userId": "u1", "entity": {"id": "e1", "clientSecret": "fake-secret"}},+ {"id": "g1", "userId": "u1", "entity": {"id": "e1"}},+ ),],)@@ -376,3 +397,171 @@ class TestReadBounded:with pytest.raises(FusionAuthResponseTimeoutError):_read_bounded(cast(requests.Response, response), max_bytes=100, max_seconds=0)assert response.closed is True+++class TestFusionAuthUserRegistrations:+ @pytest.mark.parametrize(+ "users, expected",+ [+ (+ [+ {+ "id": "u1",+ "registrations": [+ {+ "id": "r1",+ "applicationId": "a1",+ "roles": ["admin"],+ "tokens": {"idp": "fake-token"},+ "authenticationToken": "fake-auth-token",+ },+ {"id": "r2", "applicationId": "a2", "roles": []},+ ],+ },+ {"id": "u2", "registrations": [{"id": "r3", "applicationId": "a1"}]},+ ],+ [+ {"id": "r1", "applicationId": "a1", "roles": ["admin"], "userId": "u1"},+ {"id": "r2", "applicationId": "a2", "roles": [], "userId": "u1"},+ {"id": "r3", "applicationId": "a1", "userId": "u2"},+ ],+ ),+ ([{"id": "u1"}, {"id": "u2", "registrations": []}, {"id": "u3", "registrations": None}], []),+ (+ [{"id": "u1", "registrations": ["not-an-object", {"id": "r1", "applicationId": "a1"}]}],+ [{"id": "r1", "applicationId": "a1", "userId": "u1"}],+ ),+ ],+ )+ @mock.patch(CLIENT_SESSION_PATCH)+ def test_rows_are_one_per_registration(self, MockSession, users, expected):+ session = MockSession.return_value+ snaps = _wire(session, [_response({"users": users})])+ source = fusionauth_source(+ base_url="https://auth.example.com",+ api_key="tok",+ endpoint="UserRegistrations",+ team_id=1,+ job_id="j",+ resumable_source_manager=_make_manager(),+ )+ assert _rows(source) == expected+ assert snaps[0]["url"].endswith("/api/user/search")+ assert snaps[0]["json"]["search"]["queryString"] == "*"++ @mock.patch(CLIENT_SESSION_PATCH)+ def test_resumes_user_search_from_saved_offset(self, MockSession):+ session = MockSession.return_value+ snaps = _wire(session, [_response({"users": [{"id": "u9", "registrations": [{"applicationId": "a1"}]}]})])+ source = fusionauth_source(+ base_url="https://auth.example.com",+ api_key="tok",+ endpoint="UserRegistrations",+ team_id=1,+ job_id="j",+ resumable_source_manager=_make_manager(FusionAuthResumeConfig(offset=300)),+ )+ assert _rows(source) == [{"applicationId": "a1", "userId": "u9"}]+ assert snaps[0]["json"]["search"]["startRow"] == 300+++class TestFusionAuthPerUserFanout:+ def _source(self, endpoint):+ return fusionauth_source(+ base_url="https://auth.example.com",+ api_key="tok",+ endpoint=endpoint,+ team_id=1,+ job_id="j",+ resumable_source_manager=_make_manager(),+ )++ @pytest.mark.parametrize(+ "endpoint, child_path, data_selector",+ [+ ("UserConsents", "/api/user/consent", "userConsents"),+ ("UserActionLogs", "/api/user/action", "actions"),+ ],+ )+ @mock.patch(CLIENT_SESSION_PATCH)+ def test_requests_each_user_and_skips_missing_users(self, MockSession, endpoint, child_path, data_selector):+ session = MockSession.return_value+ snaps = _wire(+ session,+ [+ _response({"users": [{"id": "u1"}, {"id": "u2"}, {"id": "u3"}]}),+ _response({data_selector: [{"id": "c1"}, {"id": "c2"}]}),+ _response({"generalErrors": [{"message": "not found"}]}, status_code=404),+ _response({data_selector: []}),+ ],+ )++ assert [row["id"] for row in _rows(self._source(endpoint))] == ["c1", "c2"]+ assert snaps[0]["method"] == "POST"+ assert snaps[0]["url"].endswith("/api/user/search")+ assert snaps[0]["json"]["search"]["queryString"] == "*"+ assert [snap["url"].removeprefix("https://auth.example.com") for snap in snaps[1:]] == [+ f"{child_path}?userId=u1",+ f"{child_path}?userId=u2",+ f"{child_path}?userId=u3",+ ]+ assert all(snap["method"] == "GET" for snap in snaps[1:])++ @mock.patch(CLIENT_SESSION_PATCH)+ def test_pages_through_users(self, MockSession):+ session = MockSession.return_value+ snaps = _wire(+ session,+ [+ _response({"users": [{"id": f"u{i}"} for i in range(100)]}),+ *[_response({"userConsents": [{"id": f"c{i}"}]}) for i in range(100)],+ _response({"users": [{"id": "u100"}]}),+ _response({"userConsents": [{"id": "c100"}]}),+ ],+ )++ assert len(_rows(self._source("UserConsents"))) == 101+ user_searches = [snap for snap in snaps if snap["method"] == "POST"]+ assert [snap["json"]["search"]["startRow"] for snap in user_searches] == [0, 100]++ @mock.patch(CLIENT_SESSION_PATCH)+ def test_runtime_host_check_blocks_unsafe_domain(self, MockSession):+ session = MockSession.return_value+ _wire(session, [_response({"users": [{"id": "u1"}]})])+ with mock.patch.object(fusionauth_module, "_is_host_safe", return_value=(False, "internal address")):+ with pytest.raises(fusionauth_module.FusionAuthHostNotAllowedError):+ _rows(self._source("UserConsents"))+ session.send.assert_not_called()+++class TestFusionAuthLookups:+ @pytest.mark.parametrize(+ "endpoint, data_selector, responses, expected_ids, expected_params",+ [+ (+ "UserActions",+ "userActions",+ [[{"id": "a1"}, {"id": "a2"}], [{"id": "a2"}, {"id": "a3", "active": False}]],+ ["a1", "a2", "a3"],+ [{}, {"inactive": "true"}],+ ),+ ("UserActions", "userActions", [[], []], [], [{}, {"inactive": "true"}]),+ ("UserActionReasons", "userActionReasons", [[{"id": "r1"}, {"id": "r2"}]], ["r1", "r2"], [{}]),+ ],+ )+ @mock.patch(CLIENT_SESSION_PATCH)+ def test_single_page_lookups(self, MockSession, endpoint, data_selector, responses, expected_ids, expected_params):+ session = MockSession.return_value+ snaps = _wire(session, [_response({data_selector: rows}) for rows in responses])+ source = fusionauth_source(+ base_url="https://auth.example.com",+ api_key="tok",+ endpoint=endpoint,+ team_id=1,+ job_id="j",+ resumable_source_manager=_make_manager(),+ )++ assert [row["id"] for row in _rows(source)] == expected_ids+ assert [snap["params"] for snap in snaps] == expected_params+ assert all(snap["method"] == "GET" and snap["json"] == {} for snap in snaps)