Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(data-warehouse): add 8 missing tables to the FusionAuth source (#123608)

Tom Owers64f649c88858
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
index 1fb53e7a680a..1765c645b23c 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
@@ -5106,10 +5106,10 @@ Diffed against: <https://raw.githubusercontent.com/FusionAuth/fusionauth-openapi
- [x] `tenant (POST /api/tenant/search)` — lookup resolving the tenantId on users and login records — the top-level isolation dimension (high)
- [x] `group (POST /api/group/search)` — lookup naming the groups referenced by user memberships (high)
- [x] `group/member (POST /api/group/member/search)` — the user-to-group membership table; group-level access analysis is impossible without it (high)
-- [ ] `user/registration (GET /api/user/registration/{userId}/{applicationId})` — which users are registered to which applications, with roles and registration dates (medium)
-- [ ] `user/consent (GET /api/user/consent, POST /api/consent/search)` — consent grants per user plus the consent definition lookup — compliance reporting (medium)
-- [ ] `entity + entity/grant (POST /api/entity/search, /api/entity/grant/search)` — non-user entities and the grants linking them to users — the machine-to-machine authorization graph (medium)
-- [ ] `user/action + user-action + user-action-reason` — moderation action history against users, plus the action and reason lookups that decode it (medium)
+- [x] `user/registration (GET /api/user/registration/{userId}/{applicationId})` — which users are registered to which applications, with roles and registration dates (medium)
+- [x] `user/consent (GET /api/user/consent, POST /api/consent/search)` — consent grants per user plus the consent definition lookup — compliance reporting (medium)
+- [x] `entity + entity/grant (POST /api/entity/search, /api/entity/grant/search)` — non-user entities and the grants linking them to users — the machine-to-machine authorization graph (medium)
+- [x] `user/action + user-action + user-action-reason` — moderation action history against users, plus the action and reason lookups that decode it (medium)
- [ ] `jwt/refresh (GET /api/jwt/refresh)` — active refresh tokens per user — session longevity and device analysis (medium)
- [ ] `report/daily-active-user and report/monthly-active-user` — FusionAuth's headline DAU/MAU metrics, precomputed by the vendor (medium)
- [ ] `identity-provider/link (GET /api/identity-provider/link)` — which users are linked to which social/SSO provider — signup source breakdown (medium)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/fusionauth/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/fusionauth/canonical_descriptions.py
index 68aa3bfe88bd..63c534c1da18 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/fusionauth/canonical_descriptions.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/fusionauth/canonical_descriptions.py
@@ -102,4 +102,114 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {
"insertInstant": "The instant (epoch milliseconds) the user was added to the group.",
},
},
+ "Consents": {
+ "description": "Consent types defined in FusionAuth, such as marketing or terms-of-service consent.",
+ "docs_url": "https://fusionauth.io/docs/apis/consents",
+ "columns": {
+ "id": "Unique identifier for the consent.",
+ "name": "The name of the consent.",
+ "values": "The values a user can choose from when granting the consent.",
+ "multipleValuesAllowed": "Whether a user can grant more than one value of the consent.",
+ "defaultMinimumAgeForSelfConsent": "The minimum age at which a user can grant the consent for themselves.",
+ "insertInstant": "The instant (epoch milliseconds) the consent was created.",
+ "lastUpdateInstant": "The instant (epoch milliseconds) the consent was last updated.",
+ },
+ },
+ "UserConsents": {
+ "description": "Consents granted by or for each user, with their status and chosen values.",
+ "docs_url": "https://fusionauth.io/docs/apis/consents",
+ "columns": {
+ "id": "Unique identifier for the user consent.",
+ "userId": "The unique identifier of the user the consent applies to.",
+ "giverUserId": "The unique identifier of the user who gave the consent, such as a parent.",
+ "consentId": "The unique identifier of the consent type.",
+ "consent": "The consent type that was granted.",
+ "status": "The status of the user consent (Active or Revoked).",
+ "values": "The consent values the user chose.",
+ "insertInstant": "The instant (epoch milliseconds) the consent was granted.",
+ "lastUpdateInstant": "The instant (epoch milliseconds) the user consent was last updated.",
+ },
+ },
+ "Entities": {
+ "description": "Non-user entities, such as services or devices, used for machine-to-machine authorization.",
+ "docs_url": "https://fusionauth.io/docs/apis/entities/entities",
+ "columns": {
+ "id": "Unique identifier for the entity.",
+ "name": "The name of the entity.",
+ "clientId": "The OAuth client id of the entity.",
+ "type": "The entity type, which defines the permissions that can be granted on the entity.",
+ "tenantId": "The unique identifier of the tenant the entity belongs to.",
+ "parentId": "The unique identifier of the parent entity, if any.",
+ "insertInstant": "The instant (epoch milliseconds) the entity was created.",
+ "lastUpdateInstant": "The instant (epoch milliseconds) the entity was last updated.",
+ },
+ },
+ "EntityGrants": {
+ "description": "Grants that give a user or another entity a set of permissions on an entity.",
+ "docs_url": "https://fusionauth.io/docs/apis/entities/grants",
+ "columns": {
+ "id": "Unique identifier for the grant.",
+ "entity": "The entity the grant gives access to.",
+ "userId": "The unique identifier of the user who receives the grant, if any.",
+ "recipientEntityId": "The unique identifier of the entity that receives the grant, if any.",
+ "permissions": "The permissions granted on the entity.",
+ "insertInstant": "The instant (epoch milliseconds) the grant was created.",
+ "lastUpdateInstant": "The instant (epoch milliseconds) the grant was last updated.",
+ },
+ },
+ "UserRegistrations": {
+ "description": "Registrations of users to applications, with the roles each user has in each application.",
+ "docs_url": "https://fusionauth.io/docs/apis/registrations",
+ "columns": {
+ "id": "Unique identifier for the registration.",
+ "userId": "The unique identifier of the registered user.",
+ "applicationId": "The unique identifier of the application the user is registered to.",
+ "roles": "The application roles granted to the user.",
+ "username": "The user's username for this application, if any.",
+ "verified": "Whether the registration is verified.",
+ "insertInstant": "The instant (epoch milliseconds) the registration was created.",
+ "lastLoginInstant": "The instant (epoch milliseconds) the user last logged in to the application.",
+ "lastUpdateInstant": "The instant (epoch milliseconds) the registration was last updated.",
+ },
+ },
+ "UserActionLogs": {
+ "description": "Actions taken on users, such as account locks or warnings, with their reasons and expiry.",
+ "docs_url": "https://fusionauth.io/docs/apis/actioning-users",
+ "columns": {
+ "id": "Unique identifier for the action log entry.",
+ "actioneeUserId": "The unique identifier of the user the action was taken on.",
+ "actionerUserId": "The unique identifier of the user who took the action.",
+ "userActionId": "The unique identifier of the user action that was taken.",
+ "name": "The name of the user action.",
+ "reason": "The reason for the action.",
+ "reasonCode": "The code of the reason for the action.",
+ "comment": "A comment the actioner added to the action.",
+ "expiry": "The instant (epoch milliseconds) a time-based action expires.",
+ "insertInstant": "The instant (epoch milliseconds) the action was taken.",
+ },
+ },
+ "UserActions": {
+ "description": "User actions that can be taken on users, such as locking an account or sending a warning.",
+ "docs_url": "https://fusionauth.io/docs/apis/user-actions",
+ "columns": {
+ "id": "Unique identifier for the user action.",
+ "name": "The name of the user action.",
+ "active": "Whether the user action is active.",
+ "preventLogin": "Whether the action prevents the user from logging in while it is active.",
+ "temporal": "Whether the action is time based and has an expiry.",
+ "insertInstant": "The instant (epoch milliseconds) the user action was created.",
+ "lastUpdateInstant": "The instant (epoch milliseconds) the user action was last updated.",
+ },
+ },
+ "UserActionReasons": {
+ "description": "Reasons that can be given when an action is taken on a user.",
+ "docs_url": "https://fusionauth.io/docs/apis/user-action-reasons",
+ "columns": {
+ "id": "Unique identifier for the user action reason.",
+ "code": "The short code of the reason.",
+ "text": "The text of the reason.",
+ "insertInstant": "The instant (epoch milliseconds) the reason was created.",
+ "lastUpdateInstant": "The instant (epoch milliseconds) the reason was last updated.",
+ },
+ },
}
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/fusionauth/fusionauth.py b/products/warehouse_sources/backend/temporal/data_imports/sources/fusionauth/fusionauth.py
index 388b6d56facf..2411d2121698 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/fusionauth/fusionauth.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/fusionauth/fusionauth.py
@@ -1,7 +1,7 @@
import threading
import dataclasses
-from collections.abc import Iterator
-from typing import Any, Optional
+from collections.abc import Callable, Iterable, Iterator
+from typing import Any, Optional, cast
from urllib.parse import urlparse
import requests
@@ -13,7 +13,12 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.res
RESTAPIConfig,
rest_api_resource,
)
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.fanout import (
+ build_dependent_resource,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.paginators import BasePaginator
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.resource import Resource
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.typing import ClientConfig
from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
from products.warehouse_sources.backend.temporal.data_imports.sources.fusionauth.settings import (
@@ -21,6 +26,8 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.fusionauth
FusionAuthEndpointConfig,
)
+USERS_ENDPOINT = "Users"
+
HOST_NOT_ALLOWED_ERROR = "FusionAuth base URL is not allowed"
HTTP_NOT_ALLOWED_ERROR = "FusionAuth base URL must use HTTPS"
@@ -235,8 +242,8 @@ class FusionAuthOffsetPaginator(BasePaginator):
def _build_search_body(config: FusionAuthEndpointConfig, search_extra: dict[str, Any]) -> dict[str, Any]:
search: dict[str, Any] = {}
- if config.name == "Users":
- # `queryString: "*"` matches every user; sortFields keeps ordering stable across pages.
+ if config.query_string_search:
+ # `queryString: "*"` matches every row; sortFields keeps ordering stable across pages.
search["queryString"] = "*"
search["sortFields"] = [{"name": "insertInstant", "order": "asc"}]
elif config.sort_mode == "asc":
@@ -260,6 +267,29 @@ def _drop_fields(row: Any, paths: tuple[str, ...]) -> Any:
return row
+def _explode_rows(row: Any, field: str, exclude_fields: tuple[str, ...]) -> list[dict[str, Any]]:
+ # Exploded rows come from the user search, so each child row carries its user's id.
+ children = row.get(field) if isinstance(row, dict) else None
+ if not isinstance(children, list):
+ return []
+ return [
+ _drop_fields({**child, "userId": row.get("id")}, exclude_fields)
+ for child in children
+ if isinstance(child, dict)
+ ]
+
+
+def _check_host(normalized_url: str, team_id: int) -> None:
+ # Re-check at run time (not just at source-create) in case the base URL was edited to
+ # now resolve to an internal address (SSRF / DNS rebinding). Only enforced on cloud. Refuse
+ # plaintext HTTP before the key is used. Both raise before any request leaves the process.
+ host_ok, host_err = _is_host_safe(_hostname(normalized_url), team_id)
+ if not host_ok:
+ raise FusionAuthHostNotAllowedError(host_err or HOST_NOT_ALLOWED_ERROR)
+ if not _is_https(normalized_url):
+ raise FusionAuthHostNotAllowedError(HTTP_NOT_ALLOWED_ERROR)
+
+
def _get_headers() -> dict[str, str]:
return {"Accept": "application/json", "Content-Type": "application/json"}
@@ -336,22 +366,42 @@ def fusionauth_source(
config = FUSIONAUTH_ENDPOINTS[endpoint]
normalized_url = normalize_base_url(base_url)
+ def make_client_config(maximum_offset: Optional[int]) -> ClientConfig:
+ return {
+ "base_url": normalized_url,
+ "headers": _get_headers(),
+ # Auth rides on the framework auth config so it's redacted from logs/errors;
+ # FusionAuth expects the raw key value, no "Bearer"/scheme prefix.
+ "auth": {"type": "api_key", "api_key": api_key, "name": "Authorization", "location": "header"},
+ "paginator": FusionAuthOffsetPaginator(limit=config.page_size, maximum_offset=maximum_offset),
+ # A validated host could still 3xx to an internal address; refuse to follow
+ # redirects so the Authorization header never replays to an unexpected host (SSRF).
+ "allow_redirects": False,
+ # Bound the customer-controlled host: pin a connect/read timeout and read every body
+ # under a decoded-byte cap and wall-clock deadline so it can't hang a worker or OOM it.
+ "session": _make_bounded_session(api_key),
+ }
+
+ def make_get_config(params: dict[str, Any]) -> RESTAPIConfig:
+ return {
+ "client": make_client_config(None),
+ "resources": [
+ {
+ "name": endpoint,
+ "endpoint": {
+ "method": "GET",
+ "path": config.path,
+ "params": params,
+ "data_selector": config.data_selector,
+ "paginator": "single_page",
+ },
+ }
+ ],
+ }
+
def make_rest_config(search_extra: dict[str, Any], maximum_offset: Optional[int]) -> RESTAPIConfig:
return {
- "client": {
- "base_url": normalized_url,
- "headers": _get_headers(),
- # Auth rides on the framework auth config so it's redacted from logs/errors;
- # FusionAuth expects the raw key value, no "Bearer"/scheme prefix.
- "auth": {"type": "api_key", "api_key": api_key, "name": "Authorization", "location": "header"},
- "paginator": FusionAuthOffsetPaginator(limit=config.page_size, maximum_offset=maximum_offset),
- # A validated host could still 3xx to an internal address; refuse to follow
- # redirects so the Authorization header never replays to an unexpected host (SSRF).
- "allow_redirects": False,
- # Bound the customer-controlled host: pin a connect/read timeout and read every body
- # under a decoded-byte cap and wall-clock deadline so it can't hang a worker or OOM it.
- "session": _make_bounded_session(api_key),
- },
+ "client": make_client_config(maximum_offset),
"resources": [
{
"name": endpoint,
@@ -371,15 +421,70 @@ def fusionauth_source(
if state and state.get("offset") is not None:
resumable_source_manager.save_state(FusionAuthResumeConfig(offset=int(state["offset"])))
+ def load_initial_state() -> Optional[dict[str, Any]]:
+ if resumable_source_manager.can_resume():
+ resume = resumable_source_manager.load_state()
+ if resume is not None:
+ return {"offset": resume.offset}
+ return None
+
+ def fanout_resource() -> Resource:
+ # Per-user children have no list API, so they fan out over the user search. The pipeline
+ # iterates this resource directly, so the framework reaches a safe point after each request.
+ _check_host(normalized_url, team_id)
+ assert config.fanout is not None
+ users = FUSIONAUTH_ENDPOINTS[USERS_ENDPOINT]
+ return cast(
+ Resource,
+ build_dependent_resource(
+ endpoint_configs=FUSIONAUTH_ENDPOINTS,
+ child_endpoint=endpoint,
+ fanout=config.fanout,
+ client_config=make_client_config(None),
+ path_format_values={},
+ team_id=team_id,
+ job_id=job_id,
+ db_incremental_field_last_value=None,
+ page_size_param=None,
+ parent_endpoint_extra={
+ "method": "POST",
+ "json": _build_search_body(users, {}),
+ "data_selector": users.data_selector,
+ "paginator": FusionAuthOffsetPaginator(limit=users.page_size, maximum_offset=users.maximum_offset),
+ },
+ child_endpoint_extra={"data_selector": config.data_selector, "paginator": "single_page"},
+ ),
+ )
+
+ def exploded_resource() -> Resource:
+ _check_host(normalized_url, team_id)
+ assert config.explode_field is not None
+ explode_field = config.explode_field
+ resource = rest_api_resource(
+ make_rest_config({}, config.maximum_offset),
+ team_id,
+ job_id,
+ None,
+ resume_hook=save_checkpoint,
+ initial_paginator_state=load_initial_state(),
+ )
+ return resource.add_map(lambda row: _explode_rows(row, explode_field, config.exclude_fields))
+
def pages() -> Iterator[list[Any]]:
- # Re-check at run time (not just at source-create) in case the base URL was edited to
- # now resolve to an internal address (SSRF / DNS rebinding). Only enforced on cloud. Refuse
- # plaintext HTTP before the key is used. Both raise before any request leaves the process.
- host_ok, host_err = _is_host_safe(_hostname(normalized_url), team_id)
- if not host_ok:
- raise FusionAuthHostNotAllowedError(host_err or HOST_NOT_ALLOWED_ERROR)
- if not _is_https(normalized_url):
- raise FusionAuthHostNotAllowedError(HTTP_NOT_ALLOWED_ERROR)
+ _check_host(normalized_url, team_id)
+
+ if config.method == "GET":
+ seen_ids: set[Any] = set()
+ param_sets: list[dict[str, Any]] = [{}]
+ if config.inactive_param:
+ param_sets.append({config.inactive_param: "true"})
+ for params in param_sets:
+ for page in rest_api_resource(make_get_config(params), team_id, job_id, None):
+ rows = [row for row in page if row.get("id") not in seen_ids]
+ seen_ids.update(row.get("id") for row in rows)
+ if rows:
+ yield rows
+ return
if config.sort_mode == "desc":
# LoginRecords has no documented `orderBy`, so we can't assert ascending order.
@@ -393,18 +498,13 @@ def fusionauth_source(
or db_incremental_field_last_value is None
and db_incremental_field_earliest_value is None
):
- initial_state = None
- if resumable_source_manager.can_resume():
- resume = resumable_source_manager.load_state()
- if resume is not None:
- initial_state = {"offset": resume.offset}
resource = rest_api_resource(
make_rest_config({}, config.maximum_offset),
team_id,
job_id,
None,
resume_hook=save_checkpoint,
- initial_paginator_state=initial_state,
+ initial_paginator_state=load_initial_state(),
)
yield from resource
return
@@ -428,19 +528,13 @@ def fusionauth_source(
if should_use_incremental_field and db_incremental_field_last_value is not None and config.incremental_fields:
search_extra["start"] = db_incremental_field_last_value
- initial_state = None
- if resumable_source_manager.can_resume():
- resume = resumable_source_manager.load_state()
- if resume is not None:
- initial_state = {"offset": resume.offset}
-
resource = rest_api_resource(
make_rest_config(search_extra, config.maximum_offset),
team_id,
job_id,
None,
resume_hook=save_checkpoint,
- initial_paginator_state=initial_state,
+ initial_paginator_state=load_initial_state(),
)
yield from resource
@@ -450,9 +544,15 @@ def fusionauth_source(
page = [_drop_fields(row, config.exclude_fields) for row in page]
yield page
+ items_fn: Callable[[], Iterable[Any]] = items
+ if config.fanout is not None:
+ items_fn = fanout_resource
+ elif config.explode_field is not None:
+ items_fn = exploded_resource
+
return SourceResponse(
name=endpoint,
- items=items,
+ items=items_fn,
primary_keys=config.primary_keys,
sort_mode=config.sort_mode,
partition_count=1,
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/fusionauth/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/fusionauth/settings.py
index 5f813a1ac26b..f19e048a860b 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/fusionauth/settings.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/fusionauth/settings.py
@@ -1,6 +1,9 @@
from dataclasses import dataclass
from typing import Literal, Optional
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.fanout import (
+ DependentEndpointConfig,
+)
from products.warehouse_sources.backend.types import IncrementalField, IncrementalFieldType
@@ -37,6 +40,18 @@ class FusionAuthEndpointConfig:
order_by: str = "insertInstant ASC"
# Dotted paths removed from every row before it is written, for secrets the API returns.
exclude_fields: tuple[str, ...] = ()
+ # "GET" endpoints are unpaginated lookups that return the whole collection in one response.
+ method: Literal["POST", "GET"] = "POST"
+ # GET only: also request the deactivated rows with `<param>=true`, deduplicated on `id`.
+ inactive_param: Optional[str] = None
+ # Elasticsearch-style searches (users, entities) take a `queryString` and `sortFields`
+ # instead of the database searches' `orderBy`.
+ query_string_search: bool = False
+ # Emit one row per element of this list field of each parent row, instead of the row itself.
+ explode_field: Optional[str] = None
+ # Per-user child endpoints: one GET per user returned by the user search.
+ fanout: Optional[DependentEndpointConfig] = None
+ default_incremental_field: Optional[str] = None
# Search endpoints only: FusionAuth also has non-search read APIs (e.g. GET /api/user/{id})
@@ -56,6 +71,7 @@ FUSIONAUTH_ENDPOINTS: dict[str, FusionAuthEndpointConfig] = {
# Standard search window caps around 10k results (search_after/nextResults token
# pagination for deeper scans is not implemented here).
maximum_offset=9_900,
+ query_string_search=True,
),
"AuditLogs": FusionAuthEndpointConfig(
name="AuditLogs",
@@ -131,6 +147,105 @@ FUSIONAUTH_ENDPOINTS: dict[str, FusionAuthEndpointConfig] = {
partition_key="insertInstant",
order_by="id ASC",
),
+ "Consents": FusionAuthEndpointConfig(
+ name="Consents",
+ path="/api/consent/search",
+ data_selector="consents",
+ primary_keys=["id"],
+ incremental_fields=[],
+ partition_key="insertInstant",
+ order_by="id ASC",
+ ),
+ "Entities": FusionAuthEndpointConfig(
+ name="Entities",
+ path="/api/entity/search",
+ data_selector="entities",
+ primary_keys=["id"],
+ incremental_fields=[],
+ partition_key="insertInstant",
+ maximum_offset=9_900,
+ query_string_search=True,
+ exclude_fields=("clientSecret",),
+ ),
+ "EntityGrants": FusionAuthEndpointConfig(
+ name="EntityGrants",
+ path="/api/entity/grant/search",
+ data_selector="grants",
+ primary_keys=["id"],
+ incremental_fields=[],
+ partition_key="insertInstant",
+ # Grant search only orders by `insertInstant` or `name`.
+ order_by="insertInstant ASC",
+ exclude_fields=("entity.clientSecret",),
+ ),
+ # Registrations have no list API (only GET /api/user/registration/{userId}/{applicationId}), but
+ # every user search result embeds the user's registrations, so they are read from there.
+ "UserRegistrations": FusionAuthEndpointConfig(
+ name="UserRegistrations",
+ path="/api/user/search",
+ data_selector="users",
+ # A user has at most one registration per application.
+ primary_keys=["userId", "applicationId"],
+ incremental_fields=[],
+ partition_key="insertInstant",
+ maximum_offset=9_900,
+ query_string_search=True,
+ explode_field="registrations",
+ exclude_fields=("tokens", "authenticationToken"),
+ ),
+ "UserConsents": FusionAuthEndpointConfig(
+ name="UserConsents",
+ path="/api/user/consent?userId={user_id}",
+ data_selector="userConsents",
+ primary_keys=["id"],
+ incremental_fields=[],
+ partition_key="insertInstant",
+ method="GET",
+ fanout=DependentEndpointConfig(
+ parent_name="Users",
+ resolve_param="user_id",
+ resolve_field="id",
+ include_from_parent=[],
+ # A user deleted between the user search and this request.
+ child_response_actions=[{"status_code": 404, "action": "ignore"}],
+ ),
+ ),
+ "UserActionLogs": FusionAuthEndpointConfig(
+ name="UserActionLogs",
+ path="/api/user/action?userId={user_id}",
+ data_selector="actions",
+ primary_keys=["id"],
+ incremental_fields=[],
+ partition_key="insertInstant",
+ method="GET",
+ fanout=DependentEndpointConfig(
+ parent_name="Users",
+ resolve_param="user_id",
+ resolve_field="id",
+ include_from_parent=[],
+ child_response_actions=[{"status_code": 404, "action": "ignore"}],
+ ),
+ ),
+ "UserActions": FusionAuthEndpointConfig(
+ name="UserActions",
+ path="/api/user-action",
+ data_selector="userActions",
+ primary_keys=["id"],
+ incremental_fields=[],
+ partition_key="insertInstant",
+ method="GET",
+ # Deactivated actions are listed separately, and past action logs can still reference them.
+ inactive_param="inactive",
+ ),
+ "UserActionReasons": FusionAuthEndpointConfig(
+ name="UserActionReasons",
+ path="/api/user-action-reason",
+ data_selector="userActionReasons",
+ primary_keys=["id"],
+ incremental_fields=[],
+ partition_key="insertInstant",
+ method="GET",
+ ),
}
ENDPOINTS = tuple(FUSIONAUTH_ENDPOINTS.keys())
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/fusionauth/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/fusionauth/source.py
index db01f20530a0..0084d111b1f4 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/fusionauth/source.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/fusionauth/source.py
@@ -32,6 +32,17 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.generated_
)
from products.warehouse_sources.backend.types import ExternalDataSourceType
+_SEARCH_WINDOW_DESCRIPTION = "Full sync only; the search API's result window is capped at ~10,000 rows."
+_PER_USER_DESCRIPTION = "Full sync only; read for the first ~10,000 users the user search returns."
+
+SCHEMA_DESCRIPTIONS: dict[str, str] = {
+ "Users": _SEARCH_WINDOW_DESCRIPTION,
+ "Entities": _SEARCH_WINDOW_DESCRIPTION,
+ "UserRegistrations": _PER_USER_DESCRIPTION,
+ "UserConsents": _PER_USER_DESCRIPTION,
+ "UserActionLogs": _PER_USER_DESCRIPTION,
+}
+
@SourceRegistry.register
class FusionAuthSource(ResumableSource[FusionAuthSourceConfig, FusionAuthResumeConfig]):
@@ -144,9 +155,7 @@ Audit logs, event logs, and login records require the Elasticsearch search engin
supports_incremental=bool(FUSIONAUTH_ENDPOINTS[endpoint].incremental_fields),
supports_append=bool(FUSIONAUTH_ENDPOINTS[endpoint].incremental_fields),
incremental_fields=FUSIONAUTH_ENDPOINTS[endpoint].incremental_fields,
- description="Full sync only; the search API's result window is capped at ~10,000 rows."
- if endpoint == "Users"
- else None,
+ description=SCHEMA_DESCRIPTIONS.get(endpoint),
)
for endpoint in ENDPOINTS
]
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/fusionauth/tests/test_fusionauth.py b/products/warehouse_sources/backend/temporal/data_imports/sources/fusionauth/tests/test_fusionauth.py
index 1937062448cd..0446d49dc071 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/fusionauth/tests/test_fusionauth.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/fusionauth/tests/test_fusionauth.py
@@ -61,7 +61,14 @@ def _wire(session: mock.MagicMock, responses: list[Response]) -> list[dict[str,
snapshots: list[dict[str, Any]] = []
def _prepare(request: Any) -> mock.MagicMock:
- snapshots.append({"url": request.url, "json": json.loads(json.dumps(request.json or {}))})
+ snapshots.append(
+ {
+ "method": request.method,
+ "url": request.url,
+ "params": dict(request.params or {}),
+ "json": json.loads(json.dumps(request.json or {})),
+ }
+ )
return mock.MagicMock()
session.prepare_request.side_effect = _prepare
@@ -74,10 +81,12 @@ def _rows(source_response: Any) -> list[dict[str, Any]]:
class TestBuildSearchBody:
- def test_users_has_query_string_and_sort_fields(self):
- body = _build_search_body(FUSIONAUTH_ENDPOINTS["Users"], {})
+ @pytest.mark.parametrize("endpoint", ["Users", "Entities", "UserRegistrations"])
+ def test_query_string_searches_have_query_string_and_sort_fields(self, endpoint):
+ body = _build_search_body(FUSIONAUTH_ENDPOINTS[endpoint], {})
assert body["search"]["queryString"] == "*"
assert body["search"]["sortFields"] == [{"name": "insertInstant", "order": "asc"}]
+ assert "orderBy" not in body["search"]
class TestFusionAuthOffsetPaginator:
@@ -206,6 +215,18 @@ parameterized_secret_cases = pytest.mark.parametrize(
},
),
("Tenants", "tenants", {"id": "t2", "emailConfiguration": None}, {"id": "t2", "emailConfiguration": None}),
+ (
+ "Entities",
+ "entities",
+ {"id": "e1", "clientId": "e1", "clientSecret": "fake-secret"},
+ {"id": "e1", "clientId": "e1"},
+ ),
+ (
+ "EntityGrants",
+ "grants",
+ {"id": "g1", "userId": "u1", "entity": {"id": "e1", "clientSecret": "fake-secret"}},
+ {"id": "g1", "userId": "u1", "entity": {"id": "e1"}},
+ ),
],
)
@@ -376,3 +397,171 @@ class TestReadBounded:
with pytest.raises(FusionAuthResponseTimeoutError):
_read_bounded(cast(requests.Response, response), max_bytes=100, max_seconds=0)
assert response.closed is True
+
+
+class TestFusionAuthUserRegistrations:
+ @pytest.mark.parametrize(
+ "users, expected",
+ [
+ (
+ [
+ {
+ "id": "u1",
+ "registrations": [
+ {
+ "id": "r1",
+ "applicationId": "a1",
+ "roles": ["admin"],
+ "tokens": {"idp": "fake-token"},
+ "authenticationToken": "fake-auth-token",
+ },
+ {"id": "r2", "applicationId": "a2", "roles": []},
+ ],
+ },
+ {"id": "u2", "registrations": [{"id": "r3", "applicationId": "a1"}]},
+ ],
+ [
+ {"id": "r1", "applicationId": "a1", "roles": ["admin"], "userId": "u1"},
+ {"id": "r2", "applicationId": "a2", "roles": [], "userId": "u1"},
+ {"id": "r3", "applicationId": "a1", "userId": "u2"},
+ ],
+ ),
+ ([{"id": "u1"}, {"id": "u2", "registrations": []}, {"id": "u3", "registrations": None}], []),
+ (
+ [{"id": "u1", "registrations": ["not-an-object", {"id": "r1", "applicationId": "a1"}]}],
+ [{"id": "r1", "applicationId": "a1", "userId": "u1"}],
+ ),
+ ],
+ )
+ @mock.patch(CLIENT_SESSION_PATCH)
+ def test_rows_are_one_per_registration(self, MockSession, users, expected):
+ session = MockSession.return_value
+ snaps = _wire(session, [_response({"users": users})])
+ source = fusionauth_source(
+ base_url="https://auth.example.com",
+ api_key="tok",
+ endpoint="UserRegistrations",
+ team_id=1,
+ job_id="j",
+ resumable_source_manager=_make_manager(),
+ )
+ assert _rows(source) == expected
+ assert snaps[0]["url"].endswith("/api/user/search")
+ assert snaps[0]["json"]["search"]["queryString"] == "*"
+
+ @mock.patch(CLIENT_SESSION_PATCH)
+ def test_resumes_user_search_from_saved_offset(self, MockSession):
+ session = MockSession.return_value
+ snaps = _wire(session, [_response({"users": [{"id": "u9", "registrations": [{"applicationId": "a1"}]}]})])
+ source = fusionauth_source(
+ base_url="https://auth.example.com",
+ api_key="tok",
+ endpoint="UserRegistrations",
+ team_id=1,
+ job_id="j",
+ resumable_source_manager=_make_manager(FusionAuthResumeConfig(offset=300)),
+ )
+ assert _rows(source) == [{"applicationId": "a1", "userId": "u9"}]
+ assert snaps[0]["json"]["search"]["startRow"] == 300
+
+
+class TestFusionAuthPerUserFanout:
+ def _source(self, endpoint):
+ return fusionauth_source(
+ base_url="https://auth.example.com",
+ api_key="tok",
+ endpoint=endpoint,
+ team_id=1,
+ job_id="j",
+ resumable_source_manager=_make_manager(),
+ )
+
+ @pytest.mark.parametrize(
+ "endpoint, child_path, data_selector",
+ [
+ ("UserConsents", "/api/user/consent", "userConsents"),
+ ("UserActionLogs", "/api/user/action", "actions"),
+ ],
+ )
+ @mock.patch(CLIENT_SESSION_PATCH)
+ def test_requests_each_user_and_skips_missing_users(self, MockSession, endpoint, child_path, data_selector):
+ session = MockSession.return_value
+ snaps = _wire(
+ session,
+ [
+ _response({"users": [{"id": "u1"}, {"id": "u2"}, {"id": "u3"}]}),
+ _response({data_selector: [{"id": "c1"}, {"id": "c2"}]}),
+ _response({"generalErrors": [{"message": "not found"}]}, status_code=404),
+ _response({data_selector: []}),
+ ],
+ )
+
+ assert [row["id"] for row in _rows(self._source(endpoint))] == ["c1", "c2"]
+ assert snaps[0]["method"] == "POST"
+ assert snaps[0]["url"].endswith("/api/user/search")
+ assert snaps[0]["json"]["search"]["queryString"] == "*"
+ assert [snap["url"].removeprefix("https://auth.example.com") for snap in snaps[1:]] == [
+ f"{child_path}?userId=u1",
+ f"{child_path}?userId=u2",
+ f"{child_path}?userId=u3",
+ ]
+ assert all(snap["method"] == "GET" for snap in snaps[1:])
+
+ @mock.patch(CLIENT_SESSION_PATCH)
+ def test_pages_through_users(self, MockSession):
+ session = MockSession.return_value
+ snaps = _wire(
+ session,
+ [
+ _response({"users": [{"id": f"u{i}"} for i in range(100)]}),
+ *[_response({"userConsents": [{"id": f"c{i}"}]}) for i in range(100)],
+ _response({"users": [{"id": "u100"}]}),
+ _response({"userConsents": [{"id": "c100"}]}),
+ ],
+ )
+
+ assert len(_rows(self._source("UserConsents"))) == 101
+ user_searches = [snap for snap in snaps if snap["method"] == "POST"]
+ assert [snap["json"]["search"]["startRow"] for snap in user_searches] == [0, 100]
+
+ @mock.patch(CLIENT_SESSION_PATCH)
+ def test_runtime_host_check_blocks_unsafe_domain(self, MockSession):
+ session = MockSession.return_value
+ _wire(session, [_response({"users": [{"id": "u1"}]})])
+ with mock.patch.object(fusionauth_module, "_is_host_safe", return_value=(False, "internal address")):
+ with pytest.raises(fusionauth_module.FusionAuthHostNotAllowedError):
+ _rows(self._source("UserConsents"))
+ session.send.assert_not_called()
+
+
+class TestFusionAuthLookups:
+ @pytest.mark.parametrize(
+ "endpoint, data_selector, responses, expected_ids, expected_params",
+ [
+ (
+ "UserActions",
+ "userActions",
+ [[{"id": "a1"}, {"id": "a2"}], [{"id": "a2"}, {"id": "a3", "active": False}]],
+ ["a1", "a2", "a3"],
+ [{}, {"inactive": "true"}],
+ ),
+ ("UserActions", "userActions", [[], []], [], [{}, {"inactive": "true"}]),
+ ("UserActionReasons", "userActionReasons", [[{"id": "r1"}, {"id": "r2"}]], ["r1", "r2"], [{}]),
+ ],
+ )
+ @mock.patch(CLIENT_SESSION_PATCH)
+ def test_single_page_lookups(self, MockSession, endpoint, data_selector, responses, expected_ids, expected_params):
+ session = MockSession.return_value
+ snaps = _wire(session, [_response({data_selector: rows}) for rows in responses])
+ source = fusionauth_source(
+ base_url="https://auth.example.com",
+ api_key="tok",
+ endpoint=endpoint,
+ team_id=1,
+ job_id="j",
+ resumable_source_manager=_make_manager(),
+ )
+
+ assert [row["id"] for row in _rows(source)] == expected_ids
+ assert [snap["params"] for snap in snaps] == expected_params
+ assert all(snap["method"] == "GET" and snap["json"] == {} for snap in snaps)