Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(data-warehouse): add 4 missing tables to the Stytch source (#125068)

Tom Owersc4dcbd8c1226
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
index a0b67d8a2c75..9e3fd55df0cb 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
@@ -11799,10 +11799,10 @@ Today (4): `members`, `organizations`, `sessions`, `users`
Diffed against: <https://stytch.com/docs/sitemap.xml>
-- [ ] `b2b/api/rbac/get-policy (and consumer/api/rbac/get-rbac-policy)` — the roles, resources and scopes lookup that resolves the role IDs carried on every synced member (high)
-- [ ] `b2b/api/sso/shared/get-sso-connections` — per-organization SAML/OIDC/external connections; lookup resolving the connection IDs in member.sso_registrations (medium)
-- [ ] `b2b/api/connected-apps/consent-management/get-connected-apps-member` — which apps each member has authorized - a membership/consent table over users we already sync (medium)
-- [ ] `b2b/api/connected-apps/consent-management/get-connected-apps-organization` — org-level view of authorized connected apps (medium)
+- [x] `b2b/api/rbac/get-policy (and consumer/api/rbac/get-rbac-policy)` — the roles, resources and scopes lookup that resolves the role IDs carried on every synced member (high)
+- [x] `b2b/api/sso/shared/get-sso-connections` — per-organization SAML/OIDC/external connections; lookup resolving the connection IDs in member.sso_registrations (medium)
+- [x] `b2b/api/connected-apps/consent-management/get-connected-apps-member` — which apps each member has authorized - a membership/consent table over users we already sync (medium)
+- [x] `b2b/api/connected-apps/consent-management/get-connected-apps-organization` — org-level view of authorized connected apps (medium)
- [ ] `consumer/api/connected-apps/consent-management/get-connected-apps-user` — same consent mapping on the consumer side (medium)
- [ ] `b2b/api/scim/connection-management/get-scim-connection + scim/scim-groups/get-scim-connection-groups` — SCIM groups and their implicit role assignments, which explain how members got their roles (medium)
- [ ] `b2b/api/connected-apps/application-management/search-connected-apps` — registry of connected apps, the lookup the consent tables point at (medium)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/stytch/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/stytch/canonical_descriptions.py
index e68e174c2d42..35a2326ec737 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/stytch/canonical_descriptions.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/stytch/canonical_descriptions.py
@@ -89,4 +89,63 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {
"external_id": "An application-supplied identifier that can be used in most API calls in place of the member ID.",
},
},
+ "rbac_policy": {
+ "description": "The RBAC policy of your Stytch project, with one row per role, resource and scope. `entry_type` says which kind each row is.",
+ "docs_url": "https://stytch.com/docs/api-reference/b2b/api/rbac/get-policy",
+ "columns": {
+ "entry_type": "The kind of policy entry: `role`, `resource` or `scope`.",
+ "entry_id": "The identifier of the entry: the `role_id`, `resource_id` or `scope`, depending on `entry_type`.",
+ "role_id": "The unique identifier of the role, set for role rows. Matches the role IDs carried on members and users.",
+ "resource_id": "The unique identifier of the resource, set for resource rows.",
+ "scope": "The unique identifier of the scope, set for scope rows.",
+ "description": "The description of the role, resource or scope.",
+ "permissions": "For roles and scopes: a list of permissions, each linking a resource to the actions allowed on it.",
+ "actions": "For resources: every action that is possible on the resource.",
+ },
+ },
+ "sso_connections": {
+ "description": "A SAML, OIDC or external SSO connection owned by an organization in your Stytch B2B project. OIDC client secrets and SAML encryption private keys are not synced.",
+ "docs_url": "https://stytch.com/docs/api-reference/b2b/api/sso/shared/get-sso-connections",
+ "columns": {
+ "connection_id": "Globally unique ID of the SSO connection. Matches the connection IDs in a member's `sso_registrations`.",
+ "connection_type": "The kind of connection: `saml`, `oidc` or `external`.",
+ "organization_id": "The ID of the organization that owns the connection.",
+ "status": "The status of the connection, `pending` or `active`.",
+ "display_name": "A human-readable display name for the connection.",
+ "identity_provider": "The identity provider of the connection, for SAML and OIDC connections.",
+ "idp_entity_id": "A globally unique name for the SAML identity provider.",
+ "idp_sso_url": "The URL to which SAML login requests are sent.",
+ "issuer": "The issuer of the OIDC identity provider.",
+ "client_id": "The OAuth 2.0 client ID of an OIDC connection.",
+ "external_organization_id": "For external connections, the ID of the organization that owns the source connection.",
+ "external_connection_id": "For external connections, the ID of the source connection.",
+ "attribute_mapping": "How identity provider attributes map to member fields.",
+ },
+ },
+ "organization_connected_apps": {
+ "description": "A connected app that members of an organization in your Stytch B2B project have authorized and not revoked.",
+ "docs_url": "https://stytch.com/docs/api-reference/b2b/api/connected-apps/consent-management/get-connected-apps-organization",
+ "columns": {
+ "organization_id": "The ID of the organization.",
+ "connected_app_id": "The ID of the connected app.",
+ "name": "The name of the connected app.",
+ "description": "A description of the connected app.",
+ "client_type": "The type of connected app: `first_party`, `first_party_public`, `third_party` or `third_party_public`.",
+ "logo_url": "The logo URL of the connected app, if any.",
+ },
+ },
+ "member_connected_apps": {
+ "description": "A connected app that a member of your Stytch B2B project completed an authorization flow with and has not revoked.",
+ "docs_url": "https://stytch.com/docs/api-reference/b2b/api/connected-apps/consent-management/get-connected-apps-member",
+ "columns": {
+ "member_id": "The ID of the member who authorized the app.",
+ "organization_id": "The ID of the member's organization.",
+ "connected_app_id": "The ID of the connected app.",
+ "name": "The name of the connected app.",
+ "description": "A description of the connected app.",
+ "client_type": "The type of connected app: `first_party`, `first_party_public`, `third_party` or `third_party_public`.",
+ "scopes_granted": "The scopes granted to the app at the end of the last authorization flow.",
+ "logo_url": "The logo URL of the connected app, if any.",
+ },
+ },
}
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/stytch/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/stytch/settings.py
index 9ab00c5db00a..5aeeb34105da 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/stytch/settings.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/stytch/settings.py
@@ -3,16 +3,17 @@ from typing import Literal, Optional
from products.warehouse_sources.backend.types import IncrementalField, IncrementalFieldType
-StytchFanOut = Literal["users", "organizations"]
+StytchFanOut = Literal["users", "organizations", "organization", "member"]
-@dataclass
+@dataclass(frozen=True)
class StytchEndpointConfig:
name: str
path: str
- # Response field holding the rows (Stytch wraps them per endpoint: results/sessions/organizations/members).
- data_key: str
primary_keys: list[str]
+ # Response field holding the rows (Stytch wraps them per endpoint: results/sessions/organizations/members).
+ # Empty for rbac_policy and sso_connections, whose rows span several response fields.
+ data_key: str = ""
incremental_fields: list[IncrementalField] = field(default_factory=list)
# Stable creation-time field for datetime partitioning; never an updated-at style field.
partition_key: Optional[str] = None
@@ -20,9 +21,13 @@ class StytchEndpointConfig:
description: Optional[str] = None
# "users": one GET per user enumerated via the users search (sessions).
# "organizations": member search fanned out over organization_id chunks (members).
+ # "organization": one GET per organization, with `{organization_id}` in the path.
+ # "member": one GET per member, enumerated via the chunked member search.
fan_out: Optional[StytchFanOut] = None
# B2B-only endpoints error on consumer (B2C) projects, so they must not be default-on.
b2b_only: bool = False
+ # Served by both consumer and B2B projects, so the schema picker never flags it.
+ any_product_line: bool = False
STYTCH_ENDPOINTS: dict[str, StytchEndpointConfig] = {
@@ -77,6 +82,58 @@ STYTCH_ENDPOINTS: dict[str, StytchEndpointConfig] = {
b2b_only=True,
description="Members across all organizations in a Stytch B2B project. Errors on consumer (B2C) projects",
),
+ "rbac_policy": StytchEndpointConfig(
+ name="rbac_policy",
+ # Consumer projects answer the B2B path with invalid_b2b_endpoint; the transport then reads /v1/rbac/policy.
+ path="/v1/b2b/rbac/policy",
+ primary_keys=["entry_type", "entry_id"],
+ should_sync_default=False,
+ any_product_line=True,
+ description=(
+ "The project's RBAC policy, one row per role, resource and scope (entry_type tells them apart). "
+ "Resolves the role IDs on members and the scopes granted to connected apps"
+ ),
+ ),
+ "sso_connections": StytchEndpointConfig(
+ name="sso_connections",
+ path="/v1/b2b/sso/{organization_id}",
+ primary_keys=["connection_id"],
+ fan_out="organization",
+ should_sync_default=False,
+ b2b_only=True,
+ description=(
+ "SAML, OIDC and external SSO connections of every organization, fetched one request per "
+ "organization. OIDC client secrets and SAML encryption private keys are not synced. "
+ "Errors on consumer (B2C) projects"
+ ),
+ ),
+ "organization_connected_apps": StytchEndpointConfig(
+ name="organization_connected_apps",
+ path="/v1/b2b/organizations/{organization_id}/connected_apps",
+ data_key="connected_apps",
+ primary_keys=["organization_id", "connected_app_id"],
+ fan_out="organization",
+ should_sync_default=False,
+ b2b_only=True,
+ description=(
+ "Connected apps that members of each organization have authorized, fetched one request per "
+ "organization. Errors on consumer (B2C) projects"
+ ),
+ ),
+ "member_connected_apps": StytchEndpointConfig(
+ name="member_connected_apps",
+ path="/v1/b2b/organizations/{organization_id}/members/{member_id}/connected_apps",
+ data_key="connected_apps",
+ primary_keys=["member_id", "connected_app_id"],
+ fan_out="member",
+ should_sync_default=False,
+ b2b_only=True,
+ description=(
+ "Connected apps each member has authorized, with the scopes granted. Fetched one request per "
+ "member, so it is off by default because of the API cost on large projects. Errors on consumer "
+ "(B2C) projects"
+ ),
+ ),
}
ENDPOINTS = tuple(STYTCH_ENDPOINTS.keys())
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/stytch/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/stytch/source.py
index afb7c6458ee3..0b4b0e552f50 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/stytch/source.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/stytch/source.py
@@ -69,7 +69,7 @@ class StytchSource(ResumableSource[StytchSourceConfig, StytchResumeConfig]):
You can find both under [API keys](https://stytch.com/dashboard/api-keys) in your Stytch dashboard. Live credentials (`project-live-...`) sync from `api.stytch.com`; test credentials (`project-test-...`) sync your test environment's data from `test.stytch.com`.
-Every Stytch project is either a consumer project or a B2B project, and the two have separate tables. The `users` and `sessions` tables are only available for consumer projects, and the `organizations` and `members` tables only for B2B projects.""",
+Every Stytch project is either a consumer project or a B2B project, and the two have separate tables. The `users` and `sessions` tables are only available for consumer projects. The `organizations`, `members`, `sso_connections`, `organization_connected_apps` and `member_connected_apps` tables are only available for B2B projects. The `rbac_policy` table works for both.""",
iconPath="/static/services/stytch.png",
docsUrl="https://posthog.com/docs/cdp/sources/stytch",
releaseStatus=ReleaseStatus.GA,
@@ -163,14 +163,17 @@ Every Stytch project is either a consumer project or a B2B project, and the two
# B2C and B2B projects have disjoint API surfaces; probe each requested surface once so
# the schema picker can flag the tables the connected project can't serve.
probe_paths = {False: "/v1/users/search", True: "/v1/b2b/organizations/search"}
- surfaces_needed = {
- STYTCH_ENDPOINTS[endpoint].b2b_only for endpoint in endpoints if endpoint in STYTCH_ENDPOINTS
- }
+ probed = [
+ endpoint
+ for endpoint in endpoints
+ if endpoint in STYTCH_ENDPOINTS and not STYTCH_ENDPOINTS[endpoint].any_product_line
+ ]
+ surfaces_needed = {STYTCH_ENDPOINTS[endpoint].b2b_only for endpoint in probed}
access_by_surface = {
b2b: check_endpoint_access(config.project_id, config.secret, probe_paths[b2b]) for b2b in surfaces_needed
}
return {
- endpoint: access_by_surface[STYTCH_ENDPOINTS[endpoint].b2b_only] if endpoint in STYTCH_ENDPOINTS else None
+ endpoint: access_by_surface[STYTCH_ENDPOINTS[endpoint].b2b_only] if endpoint in probed else None
for endpoint in endpoints
}
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/stytch/stytch.py b/products/warehouse_sources/backend/temporal/data_imports/sources/stytch/stytch.py
index 1d6a2cb993cd..c7e175e1a79e 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/stytch/stytch.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/stytch/stytch.py
@@ -1,3 +1,4 @@
+import bisect
import dataclasses
from collections.abc import Iterator
from datetime import UTC, date, datetime
@@ -37,8 +38,11 @@ TRANSIENT_ERROR_TYPES = {"search_timeout"}
# error types on every request, so they can never sync against the connected project.
PRODUCT_LINE_MISMATCH_MESSAGES: dict[str, str] = {
"invalid_consumer_endpoint": "The users and sessions tables only exist for Stytch consumer projects, and this is a B2B project. Sync the organizations and members tables instead.",
- "invalid_b2b_endpoint": "The organizations and members tables only exist for Stytch B2B projects, and this is a consumer project. Sync the users and sessions tables instead.",
+ "invalid_b2b_endpoint": "The organizations, members, sso_connections, organization_connected_apps and member_connected_apps tables only exist for Stytch B2B projects, and this is a consumer project. Sync the users and sessions tables instead.",
}
+CONSUMER_RBAC_POLICY_PATH = "/v1/rbac/policy"
+# Response field -> connection_type value for the rows of the sso_connections table.
+SSO_CONNECTION_FIELDS = {"saml_connections": "saml", "oidc_connections": "oidc", "external_connections": "external"}
class StytchRetryableError(Exception):
@@ -49,8 +53,13 @@ class StytchAPIError(Exception):
"""Non-2xx Stytch response. The message carries Stytch's `error_type` so permanent
credential failures can be matched by `get_non_retryable_errors`."""
+ def __init__(self, message: str, status_code: int | None = None, error_type: str | None = None) -> None:
+ super().__init__(message)
+ self.status_code = status_code
+ self.error_type = error_type
+
[email protected](frozen=True)
class StytchResumeConfig:
# Search cursor for the page chain currently being walked. For the sessions fan-out this is the
# users-search cursor of the next page of users to fan out over (the interrupted page's sessions
@@ -59,6 +68,8 @@ class StytchResumeConfig:
# Members fan-out bookmark: first organization_id of the chunk currently being processed.
# Organization ids are sorted before chunking, so chunks are deterministic across resumes.
org_bookmark: str | None = None
+ # member_connected_apps bookmark: next unread member on the member-search page at `cursor`.
+ member_bookmark: str | None = None
def base_url_for_project(project_id: str) -> str:
@@ -147,7 +158,11 @@ def _request(
)
logger.error(f"Stytch API error: status={response.status_code}, error_type={error_type}, url={url}")
- raise StytchAPIError(f"Stytch API error: status={response.status_code}, error_type={error_type}, url={url}")
+ raise StytchAPIError(
+ f"Stytch API error: status={response.status_code}, error_type={error_type}, url={url}",
+ status_code=response.status_code,
+ error_type=error_type,
+ )
return response.json()
@@ -287,33 +302,44 @@ def _collect_organization_ids(
return sorted(org_ids)
-def _get_member_rows(
+def _remaining_member_search_chunks(
session: requests.Session,
base_url: str,
auth: tuple[str, str],
- config: StytchEndpointConfig,
logger: FilteringBoundLogger,
resumable_source_manager: ResumableSourceManager[StytchResumeConfig],
-) -> Iterator[list[dict[str, Any]]]:
- """Fan the member search out over chunks of organization_ids (the endpoint requires at least
- one organization_id, so members can't be listed project-wide in one call)."""
+) -> tuple[list[list[str]], str | None]:
+ """Chunk the sorted organization ids for the member search, and drop the chunks a resumed run
+ already finished. Returns the chunks left and the member-search cursor for the first of them."""
org_ids = _collect_organization_ids(session, base_url, auth, logger)
chunks = [
org_ids[i : i + MEMBER_SEARCH_ORG_CHUNK_SIZE] for i in range(0, len(org_ids), MEMBER_SEARCH_ORG_CHUNK_SIZE)
]
# Resolve the saved chunk bookmark. If the bookmarked organization no longer exists (deleted
- # between attempts), start over — merge dedupes re-pulled rows on member_id.
+ # between attempts), start over — merge dedupes re-pulled rows on the primary key.
resume = resumable_source_manager.load_state() if resumable_source_manager.can_resume() else None
- remaining = chunks
- resume_cursor: str | None = None
if resume is not None and resume.org_bookmark is not None:
for index, chunk in enumerate(chunks):
if chunk and chunk[0] == resume.org_bookmark:
- remaining = chunks[index:]
- resume_cursor = resume.cursor
- logger.debug(f"Stytch: resuming members fan-out from organization chunk starting {chunk[0]}")
- break
+ logger.debug(f"Stytch: resuming member fan-out from organization chunk starting {chunk[0]}")
+ return chunks[index:], resume.cursor
+ return chunks, None
+
+
+def _get_member_rows(
+ session: requests.Session,
+ base_url: str,
+ auth: tuple[str, str],
+ config: StytchEndpointConfig,
+ logger: FilteringBoundLogger,
+ resumable_source_manager: ResumableSourceManager[StytchResumeConfig],
+) -> Iterator[list[dict[str, Any]]]:
+ """Fan the member search out over chunks of organization_ids (the endpoint requires at least
+ one organization_id, so members can't be listed project-wide in one call)."""
+ remaining, resume_cursor = _remaining_member_search_chunks(
+ session, base_url, auth, logger, resumable_source_manager
+ )
for index, chunk in enumerate(remaining):
body = {"limit": PAGE_SIZE, "organization_ids": chunk}
@@ -332,6 +358,173 @@ def _get_member_rows(
resumable_source_manager.save_state(StytchResumeConfig(cursor=None, org_bookmark=remaining[index + 1][0]))
+def _rbac_policy_rows(policy: dict[str, Any]) -> list[dict[str, Any]]:
+ rows: list[dict[str, Any]] = []
+ for entry_type, list_key, id_key in (
+ ("role", "roles", "role_id"),
+ ("resource", "resources", "resource_id"),
+ ("scope", "scopes", "scope"),
+ ):
+ for entry in policy.get(list_key) or []:
+ rows.append({"entry_type": entry_type, "entry_id": entry.get(id_key), **entry})
+ return rows
+
+
+def _get_rbac_policy_rows(
+ session: requests.Session,
+ base_url: str,
+ auth: tuple[str, str],
+ config: StytchEndpointConfig,
+ logger: FilteringBoundLogger,
+) -> Iterator[list[dict[str, Any]]]:
+ """Consumer and B2B projects keep their RBAC policy under different paths. Try the B2B one and
+ fall back to the consumer one when Stytch says this is not a B2B project."""
+ try:
+ payload = _request(session, "GET", f"{base_url}{config.path}", auth, logger)
+ except StytchAPIError as error:
+ if error.error_type != "invalid_b2b_endpoint":
+ raise
+ payload = _request(session, "GET", f"{base_url}{CONSUMER_RBAC_POLICY_PATH}", auth, logger)
+
+ rows = _rbac_policy_rows(payload.get("policy") or {})
+ if rows:
+ yield rows
+
+
+def _sso_connection_rows(payload: dict[str, Any]) -> list[dict[str, Any]]:
+ rows: list[dict[str, Any]] = []
+ for field_name, connection_type in SSO_CONNECTION_FIELDS.items():
+ for connection in payload.get(field_name) or []:
+ row = {**connection, "connection_type": connection_type}
+ # IdP credentials must never land in the warehouse.
+ row.pop("client_secret", None)
+ if row.get("encryption_private_keys"):
+ row["encryption_private_keys"] = [
+ {key: value for key, value in private_key.items() if key != "private_key"}
+ for private_key in row["encryption_private_keys"]
+ ]
+ rows.append(row)
+ return rows
+
+
+def _get_or_skip_deleted(
+ session: requests.Session, url: str, auth: tuple[str, str], logger: FilteringBoundLogger
+) -> dict[str, Any]:
+ # An organization or member deleted after it was enumerated answers 404; it has no rows left.
+ try:
+ return _request(session, "GET", url, auth, logger)
+ except StytchAPIError as error:
+ if error.status_code == 404:
+ return {}
+ raise
+
+
+def _get_organization_fan_out_rows(
+ session: requests.Session,
+ base_url: str,
+ auth: tuple[str, str],
+ config: StytchEndpointConfig,
+ logger: FilteringBoundLogger,
+ resumable_source_manager: ResumableSourceManager[StytchResumeConfig],
+) -> Iterator[list[dict[str, Any]]]:
+ """One GET per organization. The bookmark is the next organization to fetch, staged before the
+ current organization's rows are yielded."""
+ org_ids = _collect_organization_ids(session, base_url, auth, logger)
+
+ resume = resumable_source_manager.load_state() if resumable_source_manager.can_resume() else None
+ start = 0
+ if resume is not None and resume.org_bookmark is not None:
+ # Ids are sorted, so a bookmarked organization deleted since the last attempt resumes at the next one.
+ start = bisect.bisect_left(org_ids, resume.org_bookmark)
+ logger.debug(f"Stytch: resuming {config.name} fan-out from organization {resume.org_bookmark}")
+ remaining = org_ids[start:]
+
+ for index, org_id in enumerate(remaining):
+ payload = _get_or_skip_deleted(session, f"{base_url}{config.path.format(organization_id=org_id)}", auth, logger)
+ if config.name == "sso_connections":
+ rows = _sso_connection_rows(payload)
+ else:
+ rows = [{**item, "organization_id": org_id} for item in payload.get(config.data_key) or []]
+
+ if index + 1 < len(remaining):
+ resumable_source_manager.save_state(StytchResumeConfig(org_bookmark=remaining[index + 1]))
+ if rows:
+ yield rows
+ else:
+ resumable_source_manager.safe_point()
+
+
+def _get_member_fan_out_rows(
+ session: requests.Session,
+ base_url: str,
+ auth: tuple[str, str],
+ config: StytchEndpointConfig,
+ logger: FilteringBoundLogger,
+ resumable_source_manager: ResumableSourceManager[StytchResumeConfig],
+) -> Iterator[list[dict[str, Any]]]:
+ """One GET per member, over the same chunked member search as the members table. The staged
+ state points at the next unread member, so a resume skips the members already written."""
+ remaining, cursor = _remaining_member_search_chunks(session, base_url, auth, logger, resumable_source_manager)
+ resume = resumable_source_manager.load_state() if resumable_source_manager.can_resume() else None
+ skip_to_member = (
+ resume.member_bookmark if resume is not None and remaining and resume.org_bookmark == remaining[0][0] else None
+ )
+
+ for chunk_index, chunk in enumerate(remaining):
+ while True:
+ body: dict[str, Any] = {"limit": PAGE_SIZE, "organization_ids": chunk}
+ if cursor:
+ body["cursor"] = cursor
+ page = _request(
+ session, "POST", f"{base_url}/v1/b2b/organizations/members/search", auth, logger, json_body=body
+ )
+ members = page.get("members", []) or []
+ next_cursor = (page.get("results_metadata") or {}).get("next_cursor")
+
+ if skip_to_member is not None:
+ # A bookmarked member deleted since the last attempt is not on the page; re-read the whole page.
+ member_ids = [member["member_id"] for member in members]
+ if skip_to_member in member_ids:
+ members = members[member_ids.index(skip_to_member) :]
+ skip_to_member = None
+
+ if next_cursor:
+ after_page: StytchResumeConfig | None = StytchResumeConfig(cursor=next_cursor, org_bookmark=chunk[0])
+ elif chunk_index + 1 < len(remaining):
+ after_page = StytchResumeConfig(org_bookmark=remaining[chunk_index + 1][0])
+ else:
+ after_page = None
+
+ if not members:
+ if after_page is not None:
+ resumable_source_manager.save_state(after_page)
+ resumable_source_manager.safe_point()
+
+ for index, member in enumerate(members):
+ path = config.path.format(organization_id=member["organization_id"], member_id=member["member_id"])
+ payload = _get_or_skip_deleted(session, f"{base_url}{path}", auth, logger)
+ rows = [
+ {**item, "organization_id": member["organization_id"], "member_id": member["member_id"]}
+ for item in payload.get(config.data_key) or []
+ ]
+ if index + 1 < len(members):
+ resumable_source_manager.save_state(
+ StytchResumeConfig(
+ cursor=cursor, org_bookmark=chunk[0], member_bookmark=members[index + 1]["member_id"]
+ )
+ )
+ elif after_page is not None:
+ resumable_source_manager.save_state(after_page)
+ if rows:
+ yield rows
+ else:
+ resumable_source_manager.safe_point()
+
+ cursor = next_cursor
+ if not cursor:
+ break
+
+
def get_rows(
project_id: str,
secret: str,
@@ -347,10 +540,16 @@ def get_rows(
# One session reused across every page so urllib3 keeps the connection alive.
session = _make_stytch_session(secret)
- if config.fan_out == "users":
+ if endpoint == "rbac_policy":
+ yield from _get_rbac_policy_rows(session, base_url, auth, config, logger)
+ elif config.fan_out == "users":
yield from _get_session_rows(session, base_url, auth, config, logger, resumable_source_manager)
elif config.fan_out == "organizations":
yield from _get_member_rows(session, base_url, auth, config, logger, resumable_source_manager)
+ elif config.fan_out == "organization":
+ yield from _get_organization_fan_out_rows(session, base_url, auth, config, logger, resumable_source_manager)
+ elif config.fan_out == "member":
+ yield from _get_member_fan_out_rows(session, base_url, auth, config, logger, resumable_source_manager)
else:
body = (
build_users_search_body(should_use_incremental_field, db_incremental_field_last_value)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/stytch/tests/test_stytch.py b/products/warehouse_sources/backend/temporal/data_imports/sources/stytch/tests/test_stytch.py
index 3ad06260c126..117bee64c0c5 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/stytch/tests/test_stytch.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/stytch/tests/test_stytch.py
@@ -61,7 +61,7 @@ class TestCheckEndpointAccess:
),
(
"invalid_b2b_endpoint",
- "The organizations and members tables only exist for Stytch B2B projects, and this is a consumer project. Sync the users and sessions tables instead.",
+ "The organizations, members, sso_connections, organization_connected_apps and member_connected_apps tables only exist for Stytch B2B projects, and this is a consumer project. Sync the users and sessions tables instead.",
),
("some_other_denial", "Not available for this Stytch project (some_other_denial)"),
],
@@ -161,6 +161,198 @@ class TestGetRowsMembers:
assert member_calls[0].json()["cursor"] == "member-cursor-3"
+class TestGetRowsRbacPolicy:
+ POLICY = {
+ "policy": {
+ "roles": [{"role_id": "stytch_admin", "description": "Admin", "permissions": []}],
+ "resources": [{"resource_id": "documents", "description": "Docs", "actions": ["read"]}],
+ "scopes": [{"scope": "read:documents", "description": "Read docs", "permissions": []}],
+ }
+ }
+
+ @pytest.mark.parametrize(
+ "script, expected_paths",
+ [
+ ([POLICY], ["/v1/b2b/rbac/policy"]),
+ (
+ [{"error_type": "invalid_b2b_endpoint"}, POLICY],
+ ["/v1/b2b/rbac/policy", "/v1/rbac/policy"],
+ ),
+ ],
+ ids=["b2b_project", "consumer_project_falls_back"],
+ )
+ def test_reads_the_policy_of_either_product_line(self, script, expected_paths):
+ responses = [_response(payload, 400 if "error_type" in payload else 200) for payload in script]
+ result = _driver().run("rbac_policy", responses)
+
+ assert result.raised is None
+ assert [request.path for request in result.requests] == expected_paths
+ assert [(row["entry_type"], row["entry_id"]) for row in result.rows] == [
+ ("role", "stytch_admin"),
+ ("resource", "documents"),
+ ("scope", "read:documents"),
+ ]
+
+ def test_other_errors_do_not_fall_back(self):
+ result = _driver().run("rbac_policy", [_response({"error_type": "unauthorized_credentials"}, 401)])
+
+ assert isinstance(result.raised, StytchAPIError)
+ assert len(result.requests) == 1
+
+
+def _org_search(org_ids: list[str]) -> ScriptedResponse:
+ return _response(_search_page("organizations", [{"organization_id": org_id} for org_id in org_ids], None))
+
+
+class TestGetRowsOrganizationFanOut:
+ def test_sso_connections_drop_idp_secrets_and_tag_the_protocol(self):
+ connections = {
+ "saml_connections": [
+ {
+ "connection_id": "saml-1",
+ "organization_id": "org-1",
+ "encryption_private_keys": [{"private_key_id": "key-1", "private_key": "fake-pem"}],
+ }
+ ],
+ "oidc_connections": [{"connection_id": "oidc-1", "organization_id": "org-1", "client_secret": "fake"}],
+ "external_connections": [{"connection_id": "ext-1", "organization_id": "org-1"}],
+ }
+ result = _driver().run(
+ "sso_connections",
+ route(
+ {
+ "/v1/b2b/organizations/search": [_org_search(["org-1"])],
+ "/v1/b2b/sso/org-1": [_response(connections)],
+ }
+ ),
+ )
+
+ assert [(row["connection_id"], row["connection_type"]) for row in result.rows] == [
+ ("saml-1", "saml"),
+ ("oidc-1", "oidc"),
+ ("ext-1", "external"),
+ ]
+ assert "client_secret" not in result.rows[1]
+ assert result.rows[0]["encryption_private_keys"] == [{"private_key_id": "key-1"}]
+
+ @pytest.mark.parametrize(
+ "resume_bookmark, expected_org_paths",
+ [
+ (None, ["org-a", "org-b", "org-c"]),
+ ("org-b", ["org-b", "org-c"]),
+ # The bookmarked organization was deleted between attempts: resume at the next one.
+ ("org-bb", ["org-c"]),
+ ],
+ )
+ def test_walks_organizations_from_the_bookmark_and_skips_deleted_ones(self, resume_bookmark, expected_org_paths):
+ apps = {"connected_apps": [{"connected_app_id": "app-1"}]}
+ result = _driver().run(
+ "organization_connected_apps",
+ route(
+ {
+ "/v1/b2b/organizations/search": [_org_search(["org-c", "org-a", "org-b"])],
+ "/v1/b2b/organizations/org-a/connected_apps": [_response(apps)],
+ # Deleted after the organization search ran.
+ "/v1/b2b/organizations/org-b/connected_apps": [
+ _response({"error_type": "organization_not_found"}, 404)
+ ],
+ "/v1/b2b/organizations/org-c/connected_apps": [_response(apps)],
+ }
+ ),
+ resume_state=StytchResumeConfig(org_bookmark=resume_bookmark) if resume_bookmark else None,
+ )
+
+ assert result.raised is None
+ fetched = [
+ request.path.split("/")[4] for request in result.requests if request.path.endswith("/connected_apps")
+ ]
+ assert fetched == expected_org_paths
+ assert [(row["organization_id"], row["connected_app_id"]) for row in result.rows] == [
+ (org_id, "app-1") for org_id in expected_org_paths if org_id != "org-b"
+ ]
+
+ def test_stages_the_next_organization_before_yielding(self):
+ apps = {"connected_apps": [{"connected_app_id": "app-1"}]}
+ result = _driver().run(
+ "organization_connected_apps",
+ route(
+ {
+ "/v1/b2b/organizations/search": [_org_search(["org-a", "org-b"])],
+ "/v1/b2b/organizations/org-a/connected_apps": [_response(apps)],
+ "/v1/b2b/organizations/org-b/connected_apps": [_response(apps)],
+ }
+ ),
+ )
+
+ assert [state.org_bookmark for state in result.saved_states] == ["org-b"]
+
+
+class TestGetRowsMemberConnectedApps:
+ def test_fetches_each_member_across_member_search_pages(self):
+ result = _driver().run(
+ "member_connected_apps",
+ route(
+ {
+ "/v1/b2b/organizations/search": [_org_search(["org-1"])],
+ "/v1/b2b/organizations/members/search": [
+ _response(
+ _search_page("members", [{"member_id": "m-1", "organization_id": "org-1"}], "page-2")
+ ),
+ _response(_search_page("members", [{"member_id": "m-2", "organization_id": "org-1"}], None)),
+ ],
+ "/v1/b2b/organizations/org-1/members/m-1/connected_apps": [
+ _response({"connected_apps": [{"connected_app_id": "app-1", "scopes_granted": "openid"}]})
+ ],
+ "/v1/b2b/organizations/org-1/members/m-2/connected_apps": [_response({"connected_apps": []})],
+ }
+ ),
+ )
+
+ assert result.raised is None
+ assert result.rows == [
+ {"connected_app_id": "app-1", "scopes_granted": "openid", "organization_id": "org-1", "member_id": "m-1"}
+ ]
+ member_searches = [request.json() for request in result.requests if request.path.endswith("/members/search")]
+ assert [body.get("cursor") for body in member_searches] == [None, "page-2"]
+ assert [(state.org_bookmark, state.cursor) for state in result.saved_states] == [("org-1", "page-2")]
+
+ @pytest.mark.parametrize(
+ "member_bookmark, expected_members",
+ [
+ (None, ["m-1", "m-2", "m-3"]),
+ ("m-2", ["m-2", "m-3"]),
+ # The bookmarked member was deleted between attempts: re-read the whole page.
+ ("m-gone", ["m-1", "m-2", "m-3"]),
+ ],
+ )
+ def test_resumes_at_the_bookmarked_member(self, member_bookmark, expected_members):
+ members = [{"member_id": member_id, "organization_id": "org-1"} for member_id in ["m-1", "m-2", "m-3"]]
+ apps = {"connected_apps": [{"connected_app_id": "app-1"}]}
+ result = _driver().run(
+ "member_connected_apps",
+ route(
+ {
+ "/v1/b2b/organizations/search": [_org_search(["org-1"])],
+ "/v1/b2b/organizations/members/search": [_response(_search_page("members", members, None))],
+ **{
+ f"/v1/b2b/organizations/org-1/members/{member_id}/connected_apps": [_response(apps)]
+ for member_id in ["m-1", "m-2", "m-3"]
+ },
+ }
+ ),
+ resume_state=StytchResumeConfig(cursor="page-2", org_bookmark="org-1", member_bookmark=member_bookmark),
+ )
+
+ assert result.raised is None
+ assert [row["member_id"] for row in result.rows] == expected_members
+ member_searches = [request.json() for request in result.requests if request.path.endswith("/members/search")]
+ assert [body.get("cursor") for body in member_searches] == ["page-2"]
+ # Each yield stages the next unread member on the same page.
+ assert [(state.cursor, state.member_bookmark) for state in result.saved_states] == [
+ ("page-2", member_id) for member_id in expected_members[1:]
+ ]
+
+
class TestStytchSourceResponse:
@pytest.mark.parametrize("endpoint", list(ENDPOINTS))
def test_response_metadata_per_endpoint(self, endpoint):
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/stytch/tests/test_stytch_source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/stytch/tests/test_stytch_source.py
index b3dbd08d7d16..56beb0723cb6 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/stytch/tests/test_stytch_source.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/stytch/tests/test_stytch_source.py
@@ -74,12 +74,17 @@ class TestStytchSource:
)
permissions = self.source.get_endpoint_permissions(
- self.config, self.team_id, ["users", "sessions", "organizations", "members"]
+ self.config,
+ self.team_id,
+ ["users", "sessions", "organizations", "members", "sso_connections", "rbac_policy"],
)
assert permissions["users"] is None
assert permissions["sessions"] is None
assert permissions["organizations"] is not None
assert permissions["members"] is not None
+ assert permissions["sso_connections"] is not None
+ # Both product lines serve the RBAC policy, so it is never flagged.
+ assert permissions["rbac_policy"] is None
# One probe per surface, not per endpoint.
assert mock_check.call_count == 2