:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(data-warehouse): add 4 missing tables to the Stytch source (#125068)
Tom Owersc4dcbd8c1226
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.mdindex a0b67d8a2c75..9e3fd55df0cb 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md@@ -11799,10 +11799,10 @@ Today (4): `members`, `organizations`, `sessions`, `users`Diffed against: <https://stytch.com/docs/sitemap.xml>-- [ ] `b2b/api/rbac/get-policy (and consumer/api/rbac/get-rbac-policy)` — the roles, resources and scopes lookup that resolves the role IDs carried on every synced member (high)-- [ ] `b2b/api/sso/shared/get-sso-connections` — per-organization SAML/OIDC/external connections; lookup resolving the connection IDs in member.sso_registrations (medium)-- [ ] `b2b/api/connected-apps/consent-management/get-connected-apps-member` — which apps each member has authorized - a membership/consent table over users we already sync (medium)-- [ ] `b2b/api/connected-apps/consent-management/get-connected-apps-organization` — org-level view of authorized connected apps (medium)+- [x] `b2b/api/rbac/get-policy (and consumer/api/rbac/get-rbac-policy)` — the roles, resources and scopes lookup that resolves the role IDs carried on every synced member (high)+- [x] `b2b/api/sso/shared/get-sso-connections` — per-organization SAML/OIDC/external connections; lookup resolving the connection IDs in member.sso_registrations (medium)+- [x] `b2b/api/connected-apps/consent-management/get-connected-apps-member` — which apps each member has authorized - a membership/consent table over users we already sync (medium)+- [x] `b2b/api/connected-apps/consent-management/get-connected-apps-organization` — org-level view of authorized connected apps (medium)- [ ] `consumer/api/connected-apps/consent-management/get-connected-apps-user` — same consent mapping on the consumer side (medium)- [ ] `b2b/api/scim/connection-management/get-scim-connection + scim/scim-groups/get-scim-connection-groups` — SCIM groups and their implicit role assignments, which explain how members got their roles (medium)- [ ] `b2b/api/connected-apps/application-management/search-connected-apps` — registry of connected apps, the lookup the consent tables point at (medium)diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/stytch/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/stytch/canonical_descriptions.pyindex e68e174c2d42..35a2326ec737 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/stytch/canonical_descriptions.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/stytch/canonical_descriptions.py@@ -89,4 +89,63 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {"external_id": "An application-supplied identifier that can be used in most API calls in place of the member ID.",},},+ "rbac_policy": {+ "description": "The RBAC policy of your Stytch project, with one row per role, resource and scope. `entry_type` says which kind each row is.",+ "docs_url": "https://stytch.com/docs/api-reference/b2b/api/rbac/get-policy",+ "columns": {+ "entry_type": "The kind of policy entry: `role`, `resource` or `scope`.",+ "entry_id": "The identifier of the entry: the `role_id`, `resource_id` or `scope`, depending on `entry_type`.",+ "role_id": "The unique identifier of the role, set for role rows. Matches the role IDs carried on members and users.",+ "resource_id": "The unique identifier of the resource, set for resource rows.",+ "scope": "The unique identifier of the scope, set for scope rows.",+ "description": "The description of the role, resource or scope.",+ "permissions": "For roles and scopes: a list of permissions, each linking a resource to the actions allowed on it.",+ "actions": "For resources: every action that is possible on the resource.",+ },+ },+ "sso_connections": {+ "description": "A SAML, OIDC or external SSO connection owned by an organization in your Stytch B2B project. OIDC client secrets and SAML encryption private keys are not synced.",+ "docs_url": "https://stytch.com/docs/api-reference/b2b/api/sso/shared/get-sso-connections",+ "columns": {+ "connection_id": "Globally unique ID of the SSO connection. Matches the connection IDs in a member's `sso_registrations`.",+ "connection_type": "The kind of connection: `saml`, `oidc` or `external`.",+ "organization_id": "The ID of the organization that owns the connection.",+ "status": "The status of the connection, `pending` or `active`.",+ "display_name": "A human-readable display name for the connection.",+ "identity_provider": "The identity provider of the connection, for SAML and OIDC connections.",+ "idp_entity_id": "A globally unique name for the SAML identity provider.",+ "idp_sso_url": "The URL to which SAML login requests are sent.",+ "issuer": "The issuer of the OIDC identity provider.",+ "client_id": "The OAuth 2.0 client ID of an OIDC connection.",+ "external_organization_id": "For external connections, the ID of the organization that owns the source connection.",+ "external_connection_id": "For external connections, the ID of the source connection.",+ "attribute_mapping": "How identity provider attributes map to member fields.",+ },+ },+ "organization_connected_apps": {+ "description": "A connected app that members of an organization in your Stytch B2B project have authorized and not revoked.",+ "docs_url": "https://stytch.com/docs/api-reference/b2b/api/connected-apps/consent-management/get-connected-apps-organization",+ "columns": {+ "organization_id": "The ID of the organization.",+ "connected_app_id": "The ID of the connected app.",+ "name": "The name of the connected app.",+ "description": "A description of the connected app.",+ "client_type": "The type of connected app: `first_party`, `first_party_public`, `third_party` or `third_party_public`.",+ "logo_url": "The logo URL of the connected app, if any.",+ },+ },+ "member_connected_apps": {+ "description": "A connected app that a member of your Stytch B2B project completed an authorization flow with and has not revoked.",+ "docs_url": "https://stytch.com/docs/api-reference/b2b/api/connected-apps/consent-management/get-connected-apps-member",+ "columns": {+ "member_id": "The ID of the member who authorized the app.",+ "organization_id": "The ID of the member's organization.",+ "connected_app_id": "The ID of the connected app.",+ "name": "The name of the connected app.",+ "description": "A description of the connected app.",+ "client_type": "The type of connected app: `first_party`, `first_party_public`, `third_party` or `third_party_public`.",+ "scopes_granted": "The scopes granted to the app at the end of the last authorization flow.",+ "logo_url": "The logo URL of the connected app, if any.",+ },+ },}diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/stytch/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/stytch/settings.pyindex 9ab00c5db00a..5aeeb34105da 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/stytch/settings.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/stytch/settings.py@@ -3,16 +3,17 @@ from typing import Literal, Optionalfrom products.warehouse_sources.backend.types import IncrementalField, IncrementalFieldType-StytchFanOut = Literal["users", "organizations"]+StytchFanOut = Literal["users", "organizations", "organization", "member"]-@dataclass+@dataclass(frozen=True)class StytchEndpointConfig:name: strpath: str- # Response field holding the rows (Stytch wraps them per endpoint: results/sessions/organizations/members).- data_key: strprimary_keys: list[str]+ # Response field holding the rows (Stytch wraps them per endpoint: results/sessions/organizations/members).+ # Empty for rbac_policy and sso_connections, whose rows span several response fields.+ data_key: str = ""incremental_fields: list[IncrementalField] = field(default_factory=list)# Stable creation-time field for datetime partitioning; never an updated-at style field.partition_key: Optional[str] = None@@ -20,9 +21,13 @@ class StytchEndpointConfig:description: Optional[str] = None# "users": one GET per user enumerated via the users search (sessions).# "organizations": member search fanned out over organization_id chunks (members).+ # "organization": one GET per organization, with `{organization_id}` in the path.+ # "member": one GET per member, enumerated via the chunked member search.fan_out: Optional[StytchFanOut] = None# B2B-only endpoints error on consumer (B2C) projects, so they must not be default-on.b2b_only: bool = False+ # Served by both consumer and B2B projects, so the schema picker never flags it.+ any_product_line: bool = FalseSTYTCH_ENDPOINTS: dict[str, StytchEndpointConfig] = {@@ -77,6 +82,58 @@ STYTCH_ENDPOINTS: dict[str, StytchEndpointConfig] = {b2b_only=True,description="Members across all organizations in a Stytch B2B project. Errors on consumer (B2C) projects",),+ "rbac_policy": StytchEndpointConfig(+ name="rbac_policy",+ # Consumer projects answer the B2B path with invalid_b2b_endpoint; the transport then reads /v1/rbac/policy.+ path="/v1/b2b/rbac/policy",+ primary_keys=["entry_type", "entry_id"],+ should_sync_default=False,+ any_product_line=True,+ description=(+ "The project's RBAC policy, one row per role, resource and scope (entry_type tells them apart). "+ "Resolves the role IDs on members and the scopes granted to connected apps"+ ),+ ),+ "sso_connections": StytchEndpointConfig(+ name="sso_connections",+ path="/v1/b2b/sso/{organization_id}",+ primary_keys=["connection_id"],+ fan_out="organization",+ should_sync_default=False,+ b2b_only=True,+ description=(+ "SAML, OIDC and external SSO connections of every organization, fetched one request per "+ "organization. OIDC client secrets and SAML encryption private keys are not synced. "+ "Errors on consumer (B2C) projects"+ ),+ ),+ "organization_connected_apps": StytchEndpointConfig(+ name="organization_connected_apps",+ path="/v1/b2b/organizations/{organization_id}/connected_apps",+ data_key="connected_apps",+ primary_keys=["organization_id", "connected_app_id"],+ fan_out="organization",+ should_sync_default=False,+ b2b_only=True,+ description=(+ "Connected apps that members of each organization have authorized, fetched one request per "+ "organization. Errors on consumer (B2C) projects"+ ),+ ),+ "member_connected_apps": StytchEndpointConfig(+ name="member_connected_apps",+ path="/v1/b2b/organizations/{organization_id}/members/{member_id}/connected_apps",+ data_key="connected_apps",+ primary_keys=["member_id", "connected_app_id"],+ fan_out="member",+ should_sync_default=False,+ b2b_only=True,+ description=(+ "Connected apps each member has authorized, with the scopes granted. Fetched one request per "+ "member, so it is off by default because of the API cost on large projects. Errors on consumer "+ "(B2C) projects"+ ),+ ),}ENDPOINTS = tuple(STYTCH_ENDPOINTS.keys())diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/stytch/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/stytch/source.pyindex afb7c6458ee3..0b4b0e552f50 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/stytch/source.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/stytch/source.py@@ -69,7 +69,7 @@ class StytchSource(ResumableSource[StytchSourceConfig, StytchResumeConfig]):You can find both under [API keys](https://stytch.com/dashboard/api-keys) in your Stytch dashboard. Live credentials (`project-live-...`) sync from `api.stytch.com`; test credentials (`project-test-...`) sync your test environment's data from `test.stytch.com`.-Every Stytch project is either a consumer project or a B2B project, and the two have separate tables. The `users` and `sessions` tables are only available for consumer projects, and the `organizations` and `members` tables only for B2B projects.""",+Every Stytch project is either a consumer project or a B2B project, and the two have separate tables. The `users` and `sessions` tables are only available for consumer projects. The `organizations`, `members`, `sso_connections`, `organization_connected_apps` and `member_connected_apps` tables are only available for B2B projects. The `rbac_policy` table works for both.""",iconPath="/static/services/stytch.png",docsUrl="https://posthog.com/docs/cdp/sources/stytch",releaseStatus=ReleaseStatus.GA,@@ -163,14 +163,17 @@ Every Stytch project is either a consumer project or a B2B project, and the two# B2C and B2B projects have disjoint API surfaces; probe each requested surface once so# the schema picker can flag the tables the connected project can't serve.probe_paths = {False: "/v1/users/search", True: "/v1/b2b/organizations/search"}- surfaces_needed = {- STYTCH_ENDPOINTS[endpoint].b2b_only for endpoint in endpoints if endpoint in STYTCH_ENDPOINTS- }+ probed = [+ endpoint+ for endpoint in endpoints+ if endpoint in STYTCH_ENDPOINTS and not STYTCH_ENDPOINTS[endpoint].any_product_line+ ]+ surfaces_needed = {STYTCH_ENDPOINTS[endpoint].b2b_only for endpoint in probed}access_by_surface = {b2b: check_endpoint_access(config.project_id, config.secret, probe_paths[b2b]) for b2b in surfaces_needed}return {- endpoint: access_by_surface[STYTCH_ENDPOINTS[endpoint].b2b_only] if endpoint in STYTCH_ENDPOINTS else None+ endpoint: access_by_surface[STYTCH_ENDPOINTS[endpoint].b2b_only] if endpoint in probed else Nonefor endpoint in endpoints}diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/stytch/stytch.py b/products/warehouse_sources/backend/temporal/data_imports/sources/stytch/stytch.pyindex 1d6a2cb993cd..c7e175e1a79e 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/stytch/stytch.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/stytch/stytch.py@@ -1,3 +1,4 @@+import bisectimport dataclassesfrom collections.abc import Iteratorfrom datetime import UTC, date, datetime@@ -37,8 +38,11 @@ TRANSIENT_ERROR_TYPES = {"search_timeout"}# error types on every request, so they can never sync against the connected project.PRODUCT_LINE_MISMATCH_MESSAGES: dict[str, str] = {"invalid_consumer_endpoint": "The users and sessions tables only exist for Stytch consumer projects, and this is a B2B project. Sync the organizations and members tables instead.",- "invalid_b2b_endpoint": "The organizations and members tables only exist for Stytch B2B projects, and this is a consumer project. Sync the users and sessions tables instead.",+ "invalid_b2b_endpoint": "The organizations, members, sso_connections, organization_connected_apps and member_connected_apps tables only exist for Stytch B2B projects, and this is a consumer project. Sync the users and sessions tables instead.",}+CONSUMER_RBAC_POLICY_PATH = "/v1/rbac/policy"+# Response field -> connection_type value for the rows of the sso_connections table.+SSO_CONNECTION_FIELDS = {"saml_connections": "saml", "oidc_connections": "oidc", "external_connections": "external"}class StytchRetryableError(Exception):@@ -49,8 +53,13 @@ class StytchAPIError(Exception):"""Non-2xx Stytch response. The message carries Stytch's `error_type` so permanentcredential failures can be matched by `get_non_retryable_errors`."""+ def __init__(self, message: str, status_code: int | None = None, error_type: str | None = None) -> None:+ super().__init__(message)+ self.status_code = status_code+ self.error_type = error_type+[email protected](frozen=True)class StytchResumeConfig:# Search cursor for the page chain currently being walked. For the sessions fan-out this is the# users-search cursor of the next page of users to fan out over (the interrupted page's sessions@@ -59,6 +68,8 @@ class StytchResumeConfig:# Members fan-out bookmark: first organization_id of the chunk currently being processed.# Organization ids are sorted before chunking, so chunks are deterministic across resumes.org_bookmark: str | None = None+ # member_connected_apps bookmark: next unread member on the member-search page at `cursor`.+ member_bookmark: str | None = Nonedef base_url_for_project(project_id: str) -> str:@@ -147,7 +158,11 @@ def _request()logger.error(f"Stytch API error: status={response.status_code}, error_type={error_type}, url={url}")- raise StytchAPIError(f"Stytch API error: status={response.status_code}, error_type={error_type}, url={url}")+ raise StytchAPIError(+ f"Stytch API error: status={response.status_code}, error_type={error_type}, url={url}",+ status_code=response.status_code,+ error_type=error_type,+ )return response.json()@@ -287,33 +302,44 @@ def _collect_organization_ids(return sorted(org_ids)-def _get_member_rows(+def _remaining_member_search_chunks(session: requests.Session,base_url: str,auth: tuple[str, str],- config: StytchEndpointConfig,logger: FilteringBoundLogger,resumable_source_manager: ResumableSourceManager[StytchResumeConfig],-) -> Iterator[list[dict[str, Any]]]:- """Fan the member search out over chunks of organization_ids (the endpoint requires at least- one organization_id, so members can't be listed project-wide in one call)."""+) -> tuple[list[list[str]], str | None]:+ """Chunk the sorted organization ids for the member search, and drop the chunks a resumed run+ already finished. Returns the chunks left and the member-search cursor for the first of them."""org_ids = _collect_organization_ids(session, base_url, auth, logger)chunks = [org_ids[i : i + MEMBER_SEARCH_ORG_CHUNK_SIZE] for i in range(0, len(org_ids), MEMBER_SEARCH_ORG_CHUNK_SIZE)]# Resolve the saved chunk bookmark. If the bookmarked organization no longer exists (deleted- # between attempts), start over — merge dedupes re-pulled rows on member_id.+ # between attempts), start over — merge dedupes re-pulled rows on the primary key.resume = resumable_source_manager.load_state() if resumable_source_manager.can_resume() else None- remaining = chunks- resume_cursor: str | None = Noneif resume is not None and resume.org_bookmark is not None:for index, chunk in enumerate(chunks):if chunk and chunk[0] == resume.org_bookmark:- remaining = chunks[index:]- resume_cursor = resume.cursor- logger.debug(f"Stytch: resuming members fan-out from organization chunk starting {chunk[0]}")- break+ logger.debug(f"Stytch: resuming member fan-out from organization chunk starting {chunk[0]}")+ return chunks[index:], resume.cursor+ return chunks, None+++def _get_member_rows(+ session: requests.Session,+ base_url: str,+ auth: tuple[str, str],+ config: StytchEndpointConfig,+ logger: FilteringBoundLogger,+ resumable_source_manager: ResumableSourceManager[StytchResumeConfig],+) -> Iterator[list[dict[str, Any]]]:+ """Fan the member search out over chunks of organization_ids (the endpoint requires at least+ one organization_id, so members can't be listed project-wide in one call)."""+ remaining, resume_cursor = _remaining_member_search_chunks(+ session, base_url, auth, logger, resumable_source_manager+ )for index, chunk in enumerate(remaining):body = {"limit": PAGE_SIZE, "organization_ids": chunk}@@ -332,6 +358,173 @@ def _get_member_rows(resumable_source_manager.save_state(StytchResumeConfig(cursor=None, org_bookmark=remaining[index + 1][0]))+def _rbac_policy_rows(policy: dict[str, Any]) -> list[dict[str, Any]]:+ rows: list[dict[str, Any]] = []+ for entry_type, list_key, id_key in (+ ("role", "roles", "role_id"),+ ("resource", "resources", "resource_id"),+ ("scope", "scopes", "scope"),+ ):+ for entry in policy.get(list_key) or []:+ rows.append({"entry_type": entry_type, "entry_id": entry.get(id_key), **entry})+ return rows+++def _get_rbac_policy_rows(+ session: requests.Session,+ base_url: str,+ auth: tuple[str, str],+ config: StytchEndpointConfig,+ logger: FilteringBoundLogger,+) -> Iterator[list[dict[str, Any]]]:+ """Consumer and B2B projects keep their RBAC policy under different paths. Try the B2B one and+ fall back to the consumer one when Stytch says this is not a B2B project."""+ try:+ payload = _request(session, "GET", f"{base_url}{config.path}", auth, logger)+ except StytchAPIError as error:+ if error.error_type != "invalid_b2b_endpoint":+ raise+ payload = _request(session, "GET", f"{base_url}{CONSUMER_RBAC_POLICY_PATH}", auth, logger)++ rows = _rbac_policy_rows(payload.get("policy") or {})+ if rows:+ yield rows+++def _sso_connection_rows(payload: dict[str, Any]) -> list[dict[str, Any]]:+ rows: list[dict[str, Any]] = []+ for field_name, connection_type in SSO_CONNECTION_FIELDS.items():+ for connection in payload.get(field_name) or []:+ row = {**connection, "connection_type": connection_type}+ # IdP credentials must never land in the warehouse.+ row.pop("client_secret", None)+ if row.get("encryption_private_keys"):+ row["encryption_private_keys"] = [+ {key: value for key, value in private_key.items() if key != "private_key"}+ for private_key in row["encryption_private_keys"]+ ]+ rows.append(row)+ return rows+++def _get_or_skip_deleted(+ session: requests.Session, url: str, auth: tuple[str, str], logger: FilteringBoundLogger+) -> dict[str, Any]:+ # An organization or member deleted after it was enumerated answers 404; it has no rows left.+ try:+ return _request(session, "GET", url, auth, logger)+ except StytchAPIError as error:+ if error.status_code == 404:+ return {}+ raise+++def _get_organization_fan_out_rows(+ session: requests.Session,+ base_url: str,+ auth: tuple[str, str],+ config: StytchEndpointConfig,+ logger: FilteringBoundLogger,+ resumable_source_manager: ResumableSourceManager[StytchResumeConfig],+) -> Iterator[list[dict[str, Any]]]:+ """One GET per organization. The bookmark is the next organization to fetch, staged before the+ current organization's rows are yielded."""+ org_ids = _collect_organization_ids(session, base_url, auth, logger)++ resume = resumable_source_manager.load_state() if resumable_source_manager.can_resume() else None+ start = 0+ if resume is not None and resume.org_bookmark is not None:+ # Ids are sorted, so a bookmarked organization deleted since the last attempt resumes at the next one.+ start = bisect.bisect_left(org_ids, resume.org_bookmark)+ logger.debug(f"Stytch: resuming {config.name} fan-out from organization {resume.org_bookmark}")+ remaining = org_ids[start:]++ for index, org_id in enumerate(remaining):+ payload = _get_or_skip_deleted(session, f"{base_url}{config.path.format(organization_id=org_id)}", auth, logger)+ if config.name == "sso_connections":+ rows = _sso_connection_rows(payload)+ else:+ rows = [{**item, "organization_id": org_id} for item in payload.get(config.data_key) or []]++ if index + 1 < len(remaining):+ resumable_source_manager.save_state(StytchResumeConfig(org_bookmark=remaining[index + 1]))+ if rows:+ yield rows+ else:+ resumable_source_manager.safe_point()+++def _get_member_fan_out_rows(+ session: requests.Session,+ base_url: str,+ auth: tuple[str, str],+ config: StytchEndpointConfig,+ logger: FilteringBoundLogger,+ resumable_source_manager: ResumableSourceManager[StytchResumeConfig],+) -> Iterator[list[dict[str, Any]]]:+ """One GET per member, over the same chunked member search as the members table. The staged+ state points at the next unread member, so a resume skips the members already written."""+ remaining, cursor = _remaining_member_search_chunks(session, base_url, auth, logger, resumable_source_manager)+ resume = resumable_source_manager.load_state() if resumable_source_manager.can_resume() else None+ skip_to_member = (+ resume.member_bookmark if resume is not None and remaining and resume.org_bookmark == remaining[0][0] else None+ )++ for chunk_index, chunk in enumerate(remaining):+ while True:+ body: dict[str, Any] = {"limit": PAGE_SIZE, "organization_ids": chunk}+ if cursor:+ body["cursor"] = cursor+ page = _request(+ session, "POST", f"{base_url}/v1/b2b/organizations/members/search", auth, logger, json_body=body+ )+ members = page.get("members", []) or []+ next_cursor = (page.get("results_metadata") or {}).get("next_cursor")++ if skip_to_member is not None:+ # A bookmarked member deleted since the last attempt is not on the page; re-read the whole page.+ member_ids = [member["member_id"] for member in members]+ if skip_to_member in member_ids:+ members = members[member_ids.index(skip_to_member) :]+ skip_to_member = None++ if next_cursor:+ after_page: StytchResumeConfig | None = StytchResumeConfig(cursor=next_cursor, org_bookmark=chunk[0])+ elif chunk_index + 1 < len(remaining):+ after_page = StytchResumeConfig(org_bookmark=remaining[chunk_index + 1][0])+ else:+ after_page = None++ if not members:+ if after_page is not None:+ resumable_source_manager.save_state(after_page)+ resumable_source_manager.safe_point()++ for index, member in enumerate(members):+ path = config.path.format(organization_id=member["organization_id"], member_id=member["member_id"])+ payload = _get_or_skip_deleted(session, f"{base_url}{path}", auth, logger)+ rows = [+ {**item, "organization_id": member["organization_id"], "member_id": member["member_id"]}+ for item in payload.get(config.data_key) or []+ ]+ if index + 1 < len(members):+ resumable_source_manager.save_state(+ StytchResumeConfig(+ cursor=cursor, org_bookmark=chunk[0], member_bookmark=members[index + 1]["member_id"]+ )+ )+ elif after_page is not None:+ resumable_source_manager.save_state(after_page)+ if rows:+ yield rows+ else:+ resumable_source_manager.safe_point()++ cursor = next_cursor+ if not cursor:+ break++def get_rows(project_id: str,secret: str,@@ -347,10 +540,16 @@ def get_rows(# One session reused across every page so urllib3 keeps the connection alive.session = _make_stytch_session(secret)- if config.fan_out == "users":+ if endpoint == "rbac_policy":+ yield from _get_rbac_policy_rows(session, base_url, auth, config, logger)+ elif config.fan_out == "users":yield from _get_session_rows(session, base_url, auth, config, logger, resumable_source_manager)elif config.fan_out == "organizations":yield from _get_member_rows(session, base_url, auth, config, logger, resumable_source_manager)+ elif config.fan_out == "organization":+ yield from _get_organization_fan_out_rows(session, base_url, auth, config, logger, resumable_source_manager)+ elif config.fan_out == "member":+ yield from _get_member_fan_out_rows(session, base_url, auth, config, logger, resumable_source_manager)else:body = (build_users_search_body(should_use_incremental_field, db_incremental_field_last_value)diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/stytch/tests/test_stytch.py b/products/warehouse_sources/backend/temporal/data_imports/sources/stytch/tests/test_stytch.pyindex 3ad06260c126..117bee64c0c5 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/stytch/tests/test_stytch.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/stytch/tests/test_stytch.py@@ -61,7 +61,7 @@ class TestCheckEndpointAccess:),("invalid_b2b_endpoint",- "The organizations and members tables only exist for Stytch B2B projects, and this is a consumer project. Sync the users and sessions tables instead.",+ "The organizations, members, sso_connections, organization_connected_apps and member_connected_apps tables only exist for Stytch B2B projects, and this is a consumer project. Sync the users and sessions tables instead.",),("some_other_denial", "Not available for this Stytch project (some_other_denial)"),],@@ -161,6 +161,198 @@ class TestGetRowsMembers:assert member_calls[0].json()["cursor"] == "member-cursor-3"+class TestGetRowsRbacPolicy:+ POLICY = {+ "policy": {+ "roles": [{"role_id": "stytch_admin", "description": "Admin", "permissions": []}],+ "resources": [{"resource_id": "documents", "description": "Docs", "actions": ["read"]}],+ "scopes": [{"scope": "read:documents", "description": "Read docs", "permissions": []}],+ }+ }++ @pytest.mark.parametrize(+ "script, expected_paths",+ [+ ([POLICY], ["/v1/b2b/rbac/policy"]),+ (+ [{"error_type": "invalid_b2b_endpoint"}, POLICY],+ ["/v1/b2b/rbac/policy", "/v1/rbac/policy"],+ ),+ ],+ ids=["b2b_project", "consumer_project_falls_back"],+ )+ def test_reads_the_policy_of_either_product_line(self, script, expected_paths):+ responses = [_response(payload, 400 if "error_type" in payload else 200) for payload in script]+ result = _driver().run("rbac_policy", responses)++ assert result.raised is None+ assert [request.path for request in result.requests] == expected_paths+ assert [(row["entry_type"], row["entry_id"]) for row in result.rows] == [+ ("role", "stytch_admin"),+ ("resource", "documents"),+ ("scope", "read:documents"),+ ]++ def test_other_errors_do_not_fall_back(self):+ result = _driver().run("rbac_policy", [_response({"error_type": "unauthorized_credentials"}, 401)])++ assert isinstance(result.raised, StytchAPIError)+ assert len(result.requests) == 1+++def _org_search(org_ids: list[str]) -> ScriptedResponse:+ return _response(_search_page("organizations", [{"organization_id": org_id} for org_id in org_ids], None))+++class TestGetRowsOrganizationFanOut:+ def test_sso_connections_drop_idp_secrets_and_tag_the_protocol(self):+ connections = {+ "saml_connections": [+ {+ "connection_id": "saml-1",+ "organization_id": "org-1",+ "encryption_private_keys": [{"private_key_id": "key-1", "private_key": "fake-pem"}],+ }+ ],+ "oidc_connections": [{"connection_id": "oidc-1", "organization_id": "org-1", "client_secret": "fake"}],+ "external_connections": [{"connection_id": "ext-1", "organization_id": "org-1"}],+ }+ result = _driver().run(+ "sso_connections",+ route(+ {+ "/v1/b2b/organizations/search": [_org_search(["org-1"])],+ "/v1/b2b/sso/org-1": [_response(connections)],+ }+ ),+ )++ assert [(row["connection_id"], row["connection_type"]) for row in result.rows] == [+ ("saml-1", "saml"),+ ("oidc-1", "oidc"),+ ("ext-1", "external"),+ ]+ assert "client_secret" not in result.rows[1]+ assert result.rows[0]["encryption_private_keys"] == [{"private_key_id": "key-1"}]++ @pytest.mark.parametrize(+ "resume_bookmark, expected_org_paths",+ [+ (None, ["org-a", "org-b", "org-c"]),+ ("org-b", ["org-b", "org-c"]),+ # The bookmarked organization was deleted between attempts: resume at the next one.+ ("org-bb", ["org-c"]),+ ],+ )+ def test_walks_organizations_from_the_bookmark_and_skips_deleted_ones(self, resume_bookmark, expected_org_paths):+ apps = {"connected_apps": [{"connected_app_id": "app-1"}]}+ result = _driver().run(+ "organization_connected_apps",+ route(+ {+ "/v1/b2b/organizations/search": [_org_search(["org-c", "org-a", "org-b"])],+ "/v1/b2b/organizations/org-a/connected_apps": [_response(apps)],+ # Deleted after the organization search ran.+ "/v1/b2b/organizations/org-b/connected_apps": [+ _response({"error_type": "organization_not_found"}, 404)+ ],+ "/v1/b2b/organizations/org-c/connected_apps": [_response(apps)],+ }+ ),+ resume_state=StytchResumeConfig(org_bookmark=resume_bookmark) if resume_bookmark else None,+ )++ assert result.raised is None+ fetched = [+ request.path.split("/")[4] for request in result.requests if request.path.endswith("/connected_apps")+ ]+ assert fetched == expected_org_paths+ assert [(row["organization_id"], row["connected_app_id"]) for row in result.rows] == [+ (org_id, "app-1") for org_id in expected_org_paths if org_id != "org-b"+ ]++ def test_stages_the_next_organization_before_yielding(self):+ apps = {"connected_apps": [{"connected_app_id": "app-1"}]}+ result = _driver().run(+ "organization_connected_apps",+ route(+ {+ "/v1/b2b/organizations/search": [_org_search(["org-a", "org-b"])],+ "/v1/b2b/organizations/org-a/connected_apps": [_response(apps)],+ "/v1/b2b/organizations/org-b/connected_apps": [_response(apps)],+ }+ ),+ )++ assert [state.org_bookmark for state in result.saved_states] == ["org-b"]+++class TestGetRowsMemberConnectedApps:+ def test_fetches_each_member_across_member_search_pages(self):+ result = _driver().run(+ "member_connected_apps",+ route(+ {+ "/v1/b2b/organizations/search": [_org_search(["org-1"])],+ "/v1/b2b/organizations/members/search": [+ _response(+ _search_page("members", [{"member_id": "m-1", "organization_id": "org-1"}], "page-2")+ ),+ _response(_search_page("members", [{"member_id": "m-2", "organization_id": "org-1"}], None)),+ ],+ "/v1/b2b/organizations/org-1/members/m-1/connected_apps": [+ _response({"connected_apps": [{"connected_app_id": "app-1", "scopes_granted": "openid"}]})+ ],+ "/v1/b2b/organizations/org-1/members/m-2/connected_apps": [_response({"connected_apps": []})],+ }+ ),+ )++ assert result.raised is None+ assert result.rows == [+ {"connected_app_id": "app-1", "scopes_granted": "openid", "organization_id": "org-1", "member_id": "m-1"}+ ]+ member_searches = [request.json() for request in result.requests if request.path.endswith("/members/search")]+ assert [body.get("cursor") for body in member_searches] == [None, "page-2"]+ assert [(state.org_bookmark, state.cursor) for state in result.saved_states] == [("org-1", "page-2")]++ @pytest.mark.parametrize(+ "member_bookmark, expected_members",+ [+ (None, ["m-1", "m-2", "m-3"]),+ ("m-2", ["m-2", "m-3"]),+ # The bookmarked member was deleted between attempts: re-read the whole page.+ ("m-gone", ["m-1", "m-2", "m-3"]),+ ],+ )+ def test_resumes_at_the_bookmarked_member(self, member_bookmark, expected_members):+ members = [{"member_id": member_id, "organization_id": "org-1"} for member_id in ["m-1", "m-2", "m-3"]]+ apps = {"connected_apps": [{"connected_app_id": "app-1"}]}+ result = _driver().run(+ "member_connected_apps",+ route(+ {+ "/v1/b2b/organizations/search": [_org_search(["org-1"])],+ "/v1/b2b/organizations/members/search": [_response(_search_page("members", members, None))],+ **{+ f"/v1/b2b/organizations/org-1/members/{member_id}/connected_apps": [_response(apps)]+ for member_id in ["m-1", "m-2", "m-3"]+ },+ }+ ),+ resume_state=StytchResumeConfig(cursor="page-2", org_bookmark="org-1", member_bookmark=member_bookmark),+ )++ assert result.raised is None+ assert [row["member_id"] for row in result.rows] == expected_members+ member_searches = [request.json() for request in result.requests if request.path.endswith("/members/search")]+ assert [body.get("cursor") for body in member_searches] == ["page-2"]+ # Each yield stages the next unread member on the same page.+ assert [(state.cursor, state.member_bookmark) for state in result.saved_states] == [+ ("page-2", member_id) for member_id in expected_members[1:]+ ]++class TestStytchSourceResponse:@pytest.mark.parametrize("endpoint", list(ENDPOINTS))def test_response_metadata_per_endpoint(self, endpoint):diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/stytch/tests/test_stytch_source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/stytch/tests/test_stytch_source.pyindex b3dbd08d7d16..56beb0723cb6 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/stytch/tests/test_stytch_source.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/stytch/tests/test_stytch_source.py@@ -74,12 +74,17 @@ class TestStytchSource:)permissions = self.source.get_endpoint_permissions(- self.config, self.team_id, ["users", "sessions", "organizations", "members"]+ self.config,+ self.team_id,+ ["users", "sessions", "organizations", "members", "sso_connections", "rbac_policy"],)assert permissions["users"] is Noneassert permissions["sessions"] is Noneassert permissions["organizations"] is not Noneassert permissions["members"] is not None+ assert permissions["sso_connections"] is not None+ # Both product lines serve the RBAC policy, so it is never flagged.+ assert permissions["rbac_policy"] is None# One probe per surface, not per endpoint.assert mock_check.call_count == 2