Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(data-warehouse): add 5 missing tables to the FusionAuth source (#128570)

Tom Owers36fa2b5a142b
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
index ca1e20c09630..e6be73423a6d 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
@@ -5136,10 +5136,10 @@ Diffed against: <https://raw.githubusercontent.com/FusionAuth/fusionauth-openapi
- [x] `user/consent (GET /api/user/consent, POST /api/consent/search)` — consent grants per user plus the consent definition lookup — compliance reporting (medium)
- [x] `entity + entity/grant (POST /api/entity/search, /api/entity/grant/search)` — non-user entities and the grants linking them to users — the machine-to-machine authorization graph (medium)
- [x] `user/action + user-action + user-action-reason` — moderation action history against users, plus the action and reason lookups that decode it (medium)
-- [ ] `jwt/refresh (GET /api/jwt/refresh)` — active refresh tokens per user — session longevity and device analysis (medium)
-- [ ] `report/daily-active-user and report/monthly-active-user` — FusionAuth's headline DAU/MAU metrics, precomputed by the vendor (medium)
-- [ ] `identity-provider/link (GET /api/identity-provider/link)` — which users are linked to which social/SSO provider — signup source breakdown (medium)
-- [ ] `user/comment (POST /api/user/comment/search)` — admin notes recorded against users during support or moderation (low)
+- [x] `jwt/refresh (GET /api/jwt/refresh)` — active refresh tokens per user — session longevity and device analysis (medium)
+- [x] `report/daily-active-user and report/monthly-active-user` — FusionAuth's headline DAU/MAU metrics, precomputed by the vendor (medium)
+- [x] `identity-provider/link (GET /api/identity-provider/link)` — which users are linked to which social/SSO provider — signup source breakdown (medium)
+- [x] `user/comment (POST /api/user/comment/search)` — admin notes recorded against users during support or moderation (low)
Note: Diffed against FusionAuth's official OpenAPI 3.0.3 spec (v1.68.0, 176 paths). Many listable resources are POST /search rather than GET, so a path-only scan understates the API. Config-heavy resources (keys, lambdas, themes, email templates, connectors, IP ACLs, webhooks) are correctly left out.
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/fusionauth/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/fusionauth/canonical_descriptions.py
index 63c534c1da18..90b447b99be5 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/fusionauth/canonical_descriptions.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/fusionauth/canonical_descriptions.py
@@ -212,4 +212,59 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {
"lastUpdateInstant": "The instant (epoch milliseconds) the reason was last updated.",
},
},
+ "UserComments": {
+ "description": "Comments that admins wrote on users, for example during support or moderation.",
+ "docs_url": "https://fusionauth.io/docs/apis/user-comments/search-for-user-comments",
+ "columns": {
+ "id": "Unique identifier for the user comment.",
+ "comment": "The text of the user comment.",
+ "commenterId": "The unique identifier of the user who wrote the comment.",
+ "userId": "The unique identifier of the user the comment was written for.",
+ "insertInstant": "The instant (epoch milliseconds) the comment was written.",
+ },
+ },
+ "RefreshTokens": {
+ "description": "Refresh tokens issued to users, which represent their active sessions. The token values are not synced.",
+ "docs_url": "https://fusionauth.io/docs/apis/jwt/retrieve-refresh-tokens",
+ "columns": {
+ "id": "Unique identifier for the refresh token.",
+ "userId": "The unique identifier of the user the token was issued to.",
+ "applicationId": "The unique identifier of the application the token was issued for.",
+ "tenantId": "The unique identifier of the tenant of the user.",
+ "insertInstant": "The instant (epoch milliseconds) the token was created.",
+ "startInstant": "The instant (epoch milliseconds) the session of the token started.",
+ "metaData": "Metadata about the token, such as the device it was issued to and its scopes.",
+ },
+ },
+ "IdentityProviderLinks": {
+ "description": "Links between users and their accounts at identity providers, such as social or SSO logins. The identity provider tokens are not synced.",
+ "docs_url": "https://fusionauth.io/docs/apis/identity-providers/links",
+ "columns": {
+ "userId": "The unique identifier of the linked user.",
+ "identityProviderId": "The unique identifier of the identity provider.",
+ "identityProviderName": "The name of the identity provider.",
+ "identityProviderType": "The type of the identity provider, for example Google or OpenIDConnect.",
+ "identityProviderUserId": "The unique identifier of the user at the identity provider.",
+ "displayName": "A display name for the linked account, for example an email address or username.",
+ "tenantId": "The unique identifier of the tenant of the user.",
+ "insertInstant": "The instant (epoch milliseconds) the link was created.",
+ "lastLoginInstant": "The instant (epoch milliseconds) the user last logged in through this link.",
+ },
+ },
+ "DailyActiveUsers": {
+ "description": "The number of daily active users across all applications, one row per day.",
+ "docs_url": "https://fusionauth.io/docs/apis/reports/retrieve-daily-active-users-report",
+ "columns": {
+ "interval": "The number of days since the epoch, in the report timezone of the instance.",
+ "count": "The number of daily active users for the day.",
+ },
+ },
+ "MonthlyActiveUsers": {
+ "description": "The number of monthly active users across all applications, one row per month.",
+ "docs_url": "https://fusionauth.io/docs/apis/reports/retrieve-monthly-active-users-report",
+ "columns": {
+ "interval": "The number of months since the epoch, in the report timezone of the instance.",
+ "count": "The number of monthly active users for the month.",
+ },
+ },
}
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/fusionauth/fusionauth.py b/products/warehouse_sources/backend/temporal/data_imports/sources/fusionauth/fusionauth.py
index 2411d2121698..5e0de6bee4fc 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/fusionauth/fusionauth.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/fusionauth/fusionauth.py
@@ -1,3 +1,4 @@
+import time
import threading
import dataclasses
from collections.abc import Callable, Iterable, Iterator
@@ -290,6 +291,10 @@ def _check_host(normalized_url: str, team_id: int) -> None:
raise FusionAuthHostNotAllowedError(HTTP_NOT_ALLOWED_ERROR)
+def _now_millis() -> int:
+ return int(time.time() * 1000)
+
+
def _get_headers() -> dict[str, str]:
return {"Accept": "application/json", "Content-Type": "application/json"}
@@ -434,7 +439,7 @@ def fusionauth_source(
_check_host(normalized_url, team_id)
assert config.fanout is not None
users = FUSIONAUTH_ENDPOINTS[USERS_ENDPOINT]
- return cast(
+ resource = cast(
Resource,
build_dependent_resource(
endpoint_configs=FUSIONAUTH_ENDPOINTS,
@@ -455,6 +460,10 @@ def fusionauth_source(
child_endpoint_extra={"data_selector": config.data_selector, "paginator": "single_page"},
),
)
+ if config.exclude_fields:
+ exclude_fields = config.exclude_fields
+ resource.add_map(lambda row: _drop_fields(row, exclude_fields))
+ return resource
def exploded_resource() -> Resource:
_check_host(normalized_url, team_id)
@@ -474,14 +483,21 @@ def fusionauth_source(
_check_host(normalized_url, team_id)
if config.method == "GET":
- seen_ids: set[Any] = set()
- param_sets: list[dict[str, Any]] = [{}]
+ base_params: dict[str, Any] = {}
+ if config.report_range:
+ base_params = {"start": 0, "end": _now_millis()}
+ seen_keys: set[tuple[Any, ...]] = set()
+ param_sets: list[dict[str, Any]] = [base_params]
if config.inactive_param:
- param_sets.append({config.inactive_param: "true"})
+ param_sets.append({**base_params, config.inactive_param: "true"})
for params in param_sets:
for page in rest_api_resource(make_get_config(params), team_id, job_id, None):
- rows = [row for row in page if row.get("id") not in seen_ids]
- seen_ids.update(row.get("id") for row in rows)
+ rows = []
+ for row in page:
+ key = tuple(row.get(field) for field in config.primary_keys)
+ if key not in seen_keys:
+ seen_keys.add(key)
+ rows.append(row)
if rows:
yield rows
return
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/fusionauth/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/fusionauth/settings.py
index f19e048a860b..a196d9313fa6 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/fusionauth/settings.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/fusionauth/settings.py
@@ -51,6 +51,8 @@ class FusionAuthEndpointConfig:
explode_field: Optional[str] = None
# Per-user child endpoints: one GET per user returned by the user search.
fanout: Optional[DependentEndpointConfig] = None
+ # GET reports that require a `start`/`end` range; we request the whole history up to now.
+ report_range: bool = False
default_incremental_field: Optional[str] = None
@@ -246,6 +248,73 @@ FUSIONAUTH_ENDPOINTS: dict[str, FusionAuthEndpointConfig] = {
partition_key="insertInstant",
method="GET",
),
+ "UserComments": FusionAuthEndpointConfig(
+ name="UserComments",
+ path="/api/user/comment/search",
+ data_selector="userComments",
+ primary_keys=["id"],
+ incremental_fields=[],
+ partition_key="insertInstant",
+ order_by="id ASC",
+ ),
+ "RefreshTokens": FusionAuthEndpointConfig(
+ name="RefreshTokens",
+ path="/api/jwt/refresh?userId={user_id}",
+ data_selector="refreshTokens",
+ primary_keys=["id"],
+ incremental_fields=[],
+ partition_key="insertInstant",
+ method="GET",
+ # The token value itself is a live credential.
+ exclude_fields=("token",),
+ fanout=DependentEndpointConfig(
+ parent_name="Users",
+ resolve_param="user_id",
+ resolve_field="id",
+ include_from_parent=[],
+ child_response_actions=[{"status_code": 404, "action": "ignore"}],
+ ),
+ ),
+ "IdentityProviderLinks": FusionAuthEndpointConfig(
+ name="IdentityProviderLinks",
+ path="/api/identity-provider/link?userId={user_id}",
+ data_selector="identityProviderLinks",
+ # One identity provider account links to one user, but a user can link several accounts
+ # of the same provider.
+ primary_keys=["userId", "identityProviderId", "identityProviderUserId"],
+ incremental_fields=[],
+ partition_key="insertInstant",
+ method="GET",
+ # The access or refresh token the identity provider returned at login.
+ exclude_fields=("token",),
+ fanout=DependentEndpointConfig(
+ parent_name="Users",
+ resolve_param="user_id",
+ resolve_field="id",
+ include_from_parent=[],
+ child_response_actions=[{"status_code": 404, "action": "ignore"}],
+ ),
+ ),
+ # Global reports across all applications. `interval` is the number of days (or months) since
+ # the epoch in the report timezone, so it is unique within a report.
+ "DailyActiveUsers": FusionAuthEndpointConfig(
+ name="DailyActiveUsers",
+ path="/api/report/daily-active-user",
+ data_selector="dailyActiveUsers",
+ primary_keys=["interval"],
+ incremental_fields=[],
+ method="GET",
+ report_range=True,
+ ),
+ "MonthlyActiveUsers": FusionAuthEndpointConfig(
+ name="MonthlyActiveUsers",
+ path="/api/report/monthly-active-user",
+ data_selector="monthlyActiveUsers",
+ primary_keys=["interval"],
+ incremental_fields=[],
+ method="GET",
+ report_range=True,
+ ),
}
ENDPOINTS = tuple(FUSIONAUTH_ENDPOINTS.keys())
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/fusionauth/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/fusionauth/source.py
index 0084d111b1f4..e1eda8bfd836 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/fusionauth/source.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/fusionauth/source.py
@@ -34,6 +34,7 @@ from products.warehouse_sources.backend.types import ExternalDataSourceType
_SEARCH_WINDOW_DESCRIPTION = "Full sync only; the search API's result window is capped at ~10,000 rows."
_PER_USER_DESCRIPTION = "Full sync only; read for the first ~10,000 users the user search returns."
+_REPORT_DESCRIPTION = "Full sync only; covers all applications and needs an API key that is not scoped to a tenant."
SCHEMA_DESCRIPTIONS: dict[str, str] = {
"Users": _SEARCH_WINDOW_DESCRIPTION,
@@ -41,6 +42,10 @@ SCHEMA_DESCRIPTIONS: dict[str, str] = {
"UserRegistrations": _PER_USER_DESCRIPTION,
"UserConsents": _PER_USER_DESCRIPTION,
"UserActionLogs": _PER_USER_DESCRIPTION,
+ "RefreshTokens": _PER_USER_DESCRIPTION,
+ "IdentityProviderLinks": _PER_USER_DESCRIPTION,
+ "DailyActiveUsers": _REPORT_DESCRIPTION,
+ "MonthlyActiveUsers": _REPORT_DESCRIPTION,
}
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/fusionauth/tests/test_fusionauth.py b/products/warehouse_sources/backend/temporal/data_imports/sources/fusionauth/tests/test_fusionauth.py
index 0446d49dc071..f39c0b042127 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/fusionauth/tests/test_fusionauth.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/fusionauth/tests/test_fusionauth.py
@@ -266,6 +266,29 @@ class TestFusionAuthAscendingPagination:
assert snaps[0]["json"]["search"]["start"] == 1700000000000
assert snaps[0]["json"]["search"]["orderBy"] == "insertInstant ASC"
+ @pytest.mark.parametrize(
+ "endpoint, path, data_selector, order_by",
+ [
+ ("Groups", "/api/group/search", "groups", "id ASC"),
+ ("UserComments", "/api/user/comment/search", "userComments", "id ASC"),
+ ("EntityGrants", "/api/entity/grant/search", "grants", "insertInstant ASC"),
+ ],
+ )
+ @mock.patch(CLIENT_SESSION_PATCH)
+ def test_database_searches_page_in_stable_order(self, MockSession, endpoint, path, data_selector, order_by):
+ session = MockSession.return_value
+ snaps = _wire(
+ session,
+ [
+ _response({data_selector: [{"id": f"r{i}"} for i in range(100)]}),
+ _response({data_selector: [{"id": "r100"}]}),
+ ],
+ )
+ assert len(_rows(self._source(endpoint=endpoint))) == 101
+ assert [snap["url"].removeprefix("https://auth.example.com") for snap in snaps] == [path, path]
+ assert [snap["json"]["search"]["startRow"] for snap in snaps] == [0, 100]
+ assert all(snap["json"]["search"]["orderBy"] == order_by for snap in snaps)
+
@parameterized_secret_cases
@mock.patch(CLIENT_SESSION_PATCH)
def test_secret_fields_are_dropped(self, MockSession, endpoint, data_selector, row, expected):
@@ -477,26 +500,76 @@ class TestFusionAuthPerUserFanout:
)
@pytest.mark.parametrize(
- "endpoint, child_path, data_selector",
+ "endpoint, child_path, data_selector, child_rows, expected_rows",
[
- ("UserConsents", "/api/user/consent", "userConsents"),
- ("UserActionLogs", "/api/user/action", "actions"),
+ (
+ "UserConsents",
+ "/api/user/consent",
+ "userConsents",
+ [{"id": "c1"}, {"id": "c2"}],
+ [{"id": "c1"}, {"id": "c2"}],
+ ),
+ (
+ "UserActionLogs",
+ "/api/user/action",
+ "actions",
+ [{"id": "c1"}, {"id": "c2"}],
+ [{"id": "c1"}, {"id": "c2"}],
+ ),
+ (
+ "RefreshTokens",
+ "/api/jwt/refresh",
+ "refreshTokens",
+ [
+ {
+ "id": "t1",
+ "userId": "u1",
+ "token": "fake-refresh-token",
+ "metaData": {"device": {"type": "BROWSER"}},
+ },
+ {"id": "t2", "userId": "u1"},
+ ],
+ [
+ {"id": "t1", "userId": "u1", "metaData": {"device": {"type": "BROWSER"}}},
+ {"id": "t2", "userId": "u1"},
+ ],
+ ),
+ (
+ "IdentityProviderLinks",
+ "/api/identity-provider/link",
+ "identityProviderLinks",
+ [
+ {
+ "userId": "u1",
+ "identityProviderId": "idp1",
+ "identityProviderUserId": "ext-1",
+ "token": "fake-idp-token",
+ },
+ {"userId": "u1", "identityProviderId": "idp1", "identityProviderUserId": "ext-2", "token": None},
+ ],
+ [
+ {"userId": "u1", "identityProviderId": "idp1", "identityProviderUserId": "ext-1"},
+ {"userId": "u1", "identityProviderId": "idp1", "identityProviderUserId": "ext-2"},
+ ],
+ ),
],
)
@mock.patch(CLIENT_SESSION_PATCH)
- def test_requests_each_user_and_skips_missing_users(self, MockSession, endpoint, child_path, data_selector):
+ def test_requests_each_user_and_skips_missing_users(
+ self, MockSession, endpoint, child_path, data_selector, child_rows, expected_rows
+ ):
session = MockSession.return_value
snaps = _wire(
session,
[
_response({"users": [{"id": "u1"}, {"id": "u2"}, {"id": "u3"}]}),
- _response({data_selector: [{"id": "c1"}, {"id": "c2"}]}),
+ _response({data_selector: child_rows}),
_response({"generalErrors": [{"message": "not found"}]}, status_code=404),
_response({data_selector: []}),
],
)
- assert [row["id"] for row in _rows(self._source(endpoint))] == ["c1", "c2"]
+ assert _rows(self._source(endpoint)) == expected_rows
assert snaps[0]["method"] == "POST"
assert snaps[0]["url"].endswith("/api/user/search")
assert snaps[0]["json"]["search"]["queryString"] == "*"
@@ -534,23 +607,50 @@ class TestFusionAuthPerUserFanout:
session.send.assert_not_called()
+NOW_MILLIS = 1_750_000_000_000
+
+
class TestFusionAuthLookups:
@pytest.mark.parametrize(
- "endpoint, data_selector, responses, expected_ids, expected_params",
+ "endpoint, data_selector, responses, expected_rows, expected_params",
[
(
"UserActions",
"userActions",
[[{"id": "a1"}, {"id": "a2"}], [{"id": "a2"}, {"id": "a3", "active": False}]],
- ["a1", "a2", "a3"],
+ [{"id": "a1"}, {"id": "a2"}, {"id": "a3", "active": False}],
[{}, {"inactive": "true"}],
),
("UserActions", "userActions", [[], []], [], [{}, {"inactive": "true"}]),
- ("UserActionReasons", "userActionReasons", [[{"id": "r1"}, {"id": "r2"}]], ["r1", "r2"], [{}]),
+ (
+ "UserActionReasons",
+ "userActionReasons",
+ [[{"id": "r1"}, {"id": "r2"}]],
+ [{"id": "r1"}, {"id": "r2"}],
+ [{}],
+ ),
+ (
+ "DailyActiveUsers",
+ "dailyActiveUsers",
+ [[{"interval": 20000, "count": 12}, {"interval": 20001, "count": 15}]],
+ [{"interval": 20000, "count": 12}, {"interval": 20001, "count": 15}],
+ [{"start": 0, "end": NOW_MILLIS}],
+ ),
+ (
+ "MonthlyActiveUsers",
+ "monthlyActiveUsers",
+ [[{"interval": 660, "count": 40}, {"interval": 661, "count": 0}]],
+ [{"interval": 660, "count": 40}, {"interval": 661, "count": 0}],
+ [{"start": 0, "end": NOW_MILLIS}],
+ ),
+ ("MonthlyActiveUsers", "monthlyActiveUsers", [[]], [], [{"start": 0, "end": NOW_MILLIS}]),
],
)
+ @mock.patch.object(fusionauth_module, "_now_millis", return_value=NOW_MILLIS)
@mock.patch(CLIENT_SESSION_PATCH)
- def test_single_page_lookups(self, MockSession, endpoint, data_selector, responses, expected_ids, expected_params):
+ def test_single_page_lookups(
+ self, MockSession, _mock_now, endpoint, data_selector, responses, expected_rows, expected_params
+ ):
session = MockSession.return_value
snaps = _wire(session, [_response({data_selector: rows}) for rows in responses])
source = fusionauth_source(
@@ -562,6 +662,6 @@ class TestFusionAuthLookups:
resumable_source_manager=_make_manager(),
)
- assert [row["id"] for row in _rows(source)] == expected_ids
+ assert _rows(source) == expected_rows
assert [snap["params"] for snap in snaps] == expected_params
assert all(snap["method"] == "GET" and snap["json"] == {} for snap in snaps)