:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(data-warehouse): add platform.sh deployments, invoices, usage and regions tables (#126277)
Tom Owers1764336904a9
diff --git a/posthog/test/repo_invariants/dataclass_frozen_baseline.txt b/posthog/test/repo_invariants/dataclass_frozen_baseline.txtindex 5634a3cfb8c7..89d7494eae9f 100644--- a/posthog/test/repo_invariants/dataclass_frozen_baseline.txt+++ b/posthog/test/repo_invariants/dataclass_frozen_baseline.txt@@ -857,8 +857,6 @@1 products/warehouse_sources/backend/temporal/data_imports/sources/plaid/settings.py1 products/warehouse_sources/backend/temporal/data_imports/sources/planhat/planhat.py1 products/warehouse_sources/backend/temporal/data_imports/sources/planhat/settings.py-1 products/warehouse_sources/backend/temporal/data_imports/sources/platform_sh/platform_sh.py-1 products/warehouse_sources/backend/temporal/data_imports/sources/platform_sh/settings.py1 products/warehouse_sources/backend/temporal/data_imports/sources/plausible/plausible.py1 products/warehouse_sources/backend/temporal/data_imports/sources/plausible/settings.py1 products/warehouse_sources/backend/temporal/data_imports/sources/plivo/plivo.pydiff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.mdindex 0182b5b2c252..0710066e35a8 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md@@ -9558,10 +9558,10 @@ Today (6): `activities`, `environments`, `members`, `organizations`, `projects`,Diffed against: <https://meta.upsun.com/openapi-spec>-- [ ] `/projects/{projectId}/environments/{environmentId}/deployments` — what is actually running per environment — services, workers, crons, resource sizes (high)-- [ ] `/organizations/{organization_id}/records/usage` — consumption records; the only way to attribute spend to projects and environments (high)-- [ ] `/organizations/{organization_id}/invoices` — billed amounts per period, the headline financial object (high)-- [ ] `/regions` — lookup resolving the region IDs already carried on synced projects and subscriptions (high)+- [x] `/projects/{projectId}/environments/{environmentId}/deployments` — what is actually running per environment — services, workers, crons, resource sizes (high)+- [x] `/organizations/{organization_id}/records/usage` — consumption records; the only way to attribute spend to projects and environments (high)+- [x] `/organizations/{organization_id}/invoices` — billed amounts per period, the headline financial object (high)+- [x] `/regions` — lookup resolving the region IDs already carried on synced projects and subscriptions (high)- [ ] `/projects/{projectId}/environments/{environmentId}/backups` — backup history for recovery-posture and retention reporting (medium)- [ ] `/projects/{projectId}/domains` — domains attached to a project, needed to map projects to sites (medium)- [ ] `/projects/{projectId}/environments/{environmentId}/routes` — route table per environment, resolves which URL serves which app (medium)diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/platform_sh/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/platform_sh/canonical_descriptions.pyindex 9cd6828325f3..6b2e8bb38e0b 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/platform_sh/canonical_descriptions.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/platform_sh/canonical_descriptions.py@@ -119,4 +119,76 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {"updated_at": "When the membership was last updated.",},},+ "deployments": {+ "description": "The read-only configuration currently deployed to an environment: apps, workers, services, routes, crons and container sizes.",+ "docs_url": "https://api.platform.sh/docs/#tag/Deployment",+ "columns": {+ "id": "Identifier of the deployment, unique within its environment.",+ "project_id": "ID of the project the deployment belongs to (injected from the parent project).",+ "environment_id": "ID of the environment the deployment belongs to (injected from the parent environment).",+ "created_at": "When the deployment was created.",+ "updated_at": "When the deployment was last updated.",+ "fingerprint": "Fingerprint of the deployment configuration.",+ "cluster_name": "Name of the cluster the deployment runs on.",+ "deployment_target": "Deployment target the environment runs on.",+ "services": "Managed services (databases, caches, queues) in the deployment, with their type and size.",+ "routes": "Routes that map URLs to the deployment's apps.",+ "webapps": "Web applications in the deployment, with their runtime, crons and resources.",+ "workers": "Worker processes in the deployment.",+ "container_profiles": "Container size profiles available to the deployment.",+ "tasks": "Sizing applied to tasks when they run.",+ },+ },+ "invoices": {+ "description": "An invoice or credit memo billed to an organization.",+ "docs_url": "https://api.platform.sh/docs/#tag/Invoices",+ "columns": {+ "id": "Unique identifier for the invoice.",+ "organization_id": "ID of the organization the invoice belongs to (injected from the parent organization).",+ "invoice_number": "Invoice number.",+ "type": "Invoice type: invoice for a standard invoice, credit_memo for a refund or credit.",+ "order_id": "ID of the order the invoice belongs to.",+ "related_invoice_id": "For a credit memo, the ID of the invoice it refunds.",+ "status": "Invoice status (e.g. paid, pending, refunded, canceled).",+ "owner": "ULID of the invoice owner.",+ "invoice_date": "Date of the invoice.",+ "invoice_due": "Due date of the invoice.",+ "created": "When the invoice was created.",+ "changed": "When the invoice was last changed.",+ "company": "Company name on the invoice, if any.",+ "total": "Invoice total.",+ "address": "Billing address on the invoice.",+ "notes": "Notes on the invoice.",+ },+ },+ "usage_records": {+ "description": "A usage record of a subscription for one usage group, such as storage, environments or user licenses.",+ "docs_url": "https://api.platform.sh/docs/#tag/Records",+ "columns": {+ "id": "Unique identifier for the usage record.",+ "organization_id": "ID of the organization the record belongs to (injected from the parent organization).",+ "subscription_id": "ID of the subscription the usage belongs to.",+ "usage_group": "Type of usage the record measures (storage, environments, user_licenses).",+ "quantity": "Quantity used.",+ "start": "When the usage record starts.",+ },+ },+ "regions": {+ "description": "A region that projects can be hosted in.",+ "docs_url": "https://api.platform.sh/docs/#tag/Regions",+ "columns": {+ "id": "Unique identifier for the region, as stored on projects and subscriptions.",+ "label": "Human-readable name of the region.",+ "zone": "Geographical zone of the region.",+ "selection_label": "Label shown when choosing a region for a new project.",+ "project_label": "Label shown on existing projects in the region.",+ "timezone": "Default timezone of the region.",+ "available": "Whether the region can be selected for new projects.",+ "private": "Whether the region is for private use only.",+ "endpoint": "URL of the region API endpoint.",+ "provider": "Information about the region's infrastructure provider.",+ "datacenter": "Information about the provider's data center.",+ "environmental_impact": "Information about the provider's environmental impact.",+ },+ },}diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/platform_sh/platform_sh.py b/products/warehouse_sources/backend/temporal/data_imports/sources/platform_sh/platform_sh.pyindex c77691ee9700..44f3747d7339 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/platform_sh/platform_sh.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/platform_sh/platform_sh.py@@ -15,9 +15,10 @@ activities feed is newest-first and pages backwards through history via `count`(params absent from the OpenAPI spec but used by the official Platform.sh PHP client and CLI, whichpage by setting `starts_at` to the oldest `created_at` seen and dedupe by id).-Fan-out: organizations -> projects/subscriptions/members, and organizations -> projects ->-environments/activities. Resume state checkpoints the current organizations page URL after all its-children are processed; a resumed run re-fans that page and merge dedupes on the primary key.+Fan-out: organizations -> projects/subscriptions/members/invoices/usage records, organizations ->+projects -> environments/activities, and organizations -> projects -> environments -> deployments.+Resume state checkpoints the current organizations page URL after all its children are processed;+a resumed run re-fans that page and merge dedupes on the primary key."""import time@@ -25,7 +26,7 @@ import dataclassesfrom collections.abc import Iteratorfrom datetime import UTC, date, datetimefrom typing import Any-from urllib.parse import urlencode, urljoin, urlparse+from urllib.parse import parse_qsl, quote, urlencode, urljoin, urlparse, urlunparseimport requestsfrom structlog.types import FilteringBoundLogger@@ -59,10 +60,10 @@ REQUEST_TIMEOUT_SECONDS = 60AUTH_FAILED_MESSAGE = "Platform.sh authentication failed"[email protected](frozen=True)class PlatformShResumeConfig:- # Next URL to fetch. For the top-level organizations table it's the next page URL; for fan-out- # tables it's the current organizations page URL (resume re-fans that page and its children,+ # Next URL to fetch. For top-level tables (organizations, regions) it's the next page URL; for+ # fan-out tables it's the current organizations page URL (resume re-fans that page and its children,# merge dedupes on the primary key).next_url: str | None = None@@ -81,6 +82,12 @@ class PlatformShUntrustedURLError(Exception):token to every request, so following an off-host URL would leak it; refuse instead."""+class PlatformShPaginationError(Exception):+ """Raised when a page-numbered list repeats a page it already returned. The server is ignoring+ the page param, so the rows past the first page can't be fetched; fail rather than sync a short+ table."""++class PlatformShPageCapExceededError(Exception):"""Raised when a fan-out parent exceeds the per-parent page cap. Failing loudly beats silentlywriting an incomplete table that later runs would never backfill."""@@ -100,9 +107,9 @@ class PlatformShClient:self._logger = loggerself._access_token: str | None = Noneself._token_deadline = 0.0- # capture=False: environment responses carry `http_access.basic_auth` and activity- # responses carry raw `log` output — secrets we strip client-side in `_clean_rows`, which- # runs after HTTP sample capture would have recorded the raw body. Requests stay metered+ # capture=False: environment and deployment responses carry `http_access.basic_auth` and+ # activity responses carry raw `log` output — secrets we strip client-side in `_clean_rows`,+ # which runs after HTTP sample capture would have recorded the raw body. Requests stay metered# and logged; only the opt-in body capture is excluded.self._session = make_tracked_session(redact_values=(api_token,), capture=False)@@ -221,27 +228,56 @@ def _next_link(body: dict[str, Any], current_url: str, client: PlatformShClient)return client.validate_url(urljoin(current_url, href))+def _with_page_number(url: str, param: str, page: int) -> str:+ parsed = urlparse(url)+ query = {**dict(parse_qsl(parsed.query)), param: str(page)}+ return urlunparse(parsed._replace(query=urlencode(query)))++def _iter_envelope_pages(client: PlatformShClient,url: str,max_pages: int | None = None,page_cap_context: dict[str, Any] | None = None,+ items_key: str = "items",+ page_number_param: str | None = None,) -> Iterator[tuple[list[dict[str, Any]], str]]:"""Yield (items, page_url) for each page of an `{"items": [...], "_links": {...}}` list,- following the `_links.next` cursor. When `max_pages` is set and more pages remain, raise- rather than truncate: a silently short table would stay incomplete on every later run."""+ following the `_links.next` cursor, or incrementing `page_number_param` when a page has no+ next link. When `max_pages` is set and more pages remain, raise rather than truncate: a+ silently short table would stay incomplete on every later run."""page_count = 0+ seen_ids: set[Any] = set()while True:body = client.get(url).json()- items = body.get("items") if isinstance(body, dict) else None+ items = body.get(items_key) if isinstance(body, dict) else Noneif not items:return+ if max_pages is not None and page_count >= max_pages:+ # Only a guessed page-number URL is fetched past the cap: an empty probe means the list+ # ended exactly at the cap, anything else means rows remain.+ raise PlatformShPageCapExceededError(+ f"Platform.sh: page cap of {max_pages} reached with more pages remaining; "+ f"raise max_pages_per_parent to sync this parent fully. context={page_cap_context or {}}"+ )next_url = _next_link(body, url, client)+ guessed_next = False+ if page_number_param is not None:+ page_ids = {item.get("id") for item in items}+ if page_ids <= seen_ids:+ raise PlatformShPaginationError(+ f"Platform.sh: {url} repeated a page it already returned; the server ignored "+ f"`{page_number_param}`, so the full list can't be fetched. context={page_cap_context or {}}"+ )+ seen_ids |= page_ids+ if next_url is None:+ next_url = _with_page_number(url, page_number_param, page_count + 2)+ guessed_next = Trueyield items, urlpage_count += 1if not next_url:return- if max_pages is not None and page_count >= max_pages:+ if max_pages is not None and page_count >= max_pages and not guessed_next:raise PlatformShPageCapExceededError(f"Platform.sh: page cap of {max_pages} reached with more pages remaining; "f"raise max_pages_per_parent to sync this parent fully. context={page_cap_context or {}}"@@ -354,6 +390,31 @@ def _iter_project_activities(starts_at = min(row["created_at"] for row in new_rows)+def _iter_environment_deployments(+ client: PlatformShClient,+ environment: dict[str, Any],+ logger: FilteringBoundLogger,+ config: PlatformShEndpointConfig,+) -> Iterator[list[dict[str, Any]]]:+ # Inactive environments run nothing, so there is no deployment to report.+ if environment.get("status") == "inactive":+ return+ # Environment ids are Git branch names and can contain "/".+ url = client.api_base + config.path.format(+ project_id=environment["project_id"], environment_id=quote(environment["id"], safe="")+ )+ try:+ data = client.get(url).json()+ except requests.HTTPError as e:+ # The environment was deleted after the environments list was read.+ if e.response is not None and e.response.status_code == 404:+ logger.debug(f"Platform.sh: no deployments for environment {environment['id']}, skipping")+ return+ raise+ if isinstance(data, list) and data:+ yield _clean_rows(_inject_parent_fields(data, environment, config.include_parent_fields), config)++def _iter_project_children(client: PlatformShClient,project: dict[str, Any],@@ -362,6 +423,13 @@ def _iter_project_children(cutoff: datetime | None,) -> Iterator[list[dict[str, Any]]]:project_id = project["id"]+ if config.fan_out_parent == "environments":+ environments_config = PLATFORM_SH_ENDPOINTS["environments"]+ for environments in _iter_project_children(client, project, logger, environments_config, cutoff=None):+ for environment in environments:+ yield from _iter_environment_deployments(client, environment, logger, config)+ return+if config.name == "activities":raw_batches: Iterator[list[dict[str, Any]]] = _iter_project_activities(client, project_id, logger, config, cutoff@@ -402,30 +470,32 @@ def get_rows(logger.debug(f"Platform.sh: syncing {endpoint} incrementally from {cutoff.isoformat()}")resume = resumable_source_manager.load_state() if resumable_source_manager.can_resume() else None- orgs_url = _build_url(client.api_base, "/organizations", PLATFORM_SH_ENDPOINTS["organizations"].page_size)+ # Top-level tables page their own path; fan-out tables page the organizations list.+ list_config = config if config.fan_out_parent is None else PLATFORM_SH_ENDPOINTS["organizations"]+ list_url = _build_url(client.api_base, list_config.path, list_config.page_size)if resume is not None and resume.next_url:- orgs_url = client.validate_url(resume.next_url)- logger.debug(f"Platform.sh: resuming {endpoint} from organizations page: {orgs_url}")+ list_url = client.validate_url(resume.next_url)+ logger.debug(f"Platform.sh: resuming {endpoint} from page: {list_url}")- if endpoint == "organizations":+ if config.fan_out_parent is None:while True:- body = client.get(orgs_url).json()- items = body.get("items") if isinstance(body, dict) else None+ body = client.get(list_url).json()+ items = body.get(config.items_key) if isinstance(body, dict) else Noneif not items:return- next_url = _next_link(body, orgs_url, client)+ next_url = _next_link(body, list_url, client)yield _clean_rows(items, config)if not next_url:return# Save AFTER yielding so a crash re-yields the last page rather than skipping it# (merge dedupes on the primary key).resumable_source_manager.save_state(PlatformShResumeConfig(next_url=next_url))- orgs_url = next_url+ list_url = next_url# Fan-out tables: walk the organizations list and emit child rows per org (and, for# project-scoped tables, per project). Checkpoint the org page URL after all of its children# are processed; a resumed run re-fans that page and merge dedupes.- for orgs, org_page_url in _iter_envelope_pages(client, orgs_url):+ for orgs, org_page_url in _iter_envelope_pages(client, list_url):for org in orgs:if config.fan_out_parent == "organizations":child_url = _build_url(client.api_base, config.path.format(organization_id=org["id"]), config.page_size)@@ -434,9 +504,11 @@ def get_rows(child_url,max_pages=config.max_pages_per_parent,page_cap_context={"organization_id": org["id"], "endpoint": endpoint},+ items_key=config.items_key,+ page_number_param=config.page_number_param,):yield _clean_rows(_inject_parent_fields(child_items, org, config.include_parent_fields), config)- else: # children of projects (environments, activities)+ else: # children of projects (environments, activities, deployments)projects_config = PLATFORM_SH_ENDPOINTS["projects"]projects_url = _build_url(client.api_base,diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/platform_sh/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/platform_sh/settings.pyindex d5892a0eb34f..41964f9b25f3 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/platform_sh/settings.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/platform_sh/settings.py@@ -7,11 +7,14 @@ from products.warehouse_sources.backend.types import IncrementalField, Increment## - Organization-level lists (/organizations, /organizations/{id}/projects, .../members,# .../subscriptions) return an `{"items": [...], "_links": {...}}` envelope with cursor-# pagination (`page[size]` 1-100, `_links.next.href`).-# - Project-scoped lists (/projects/{id}/environments, /projects/{id}/activities) return a bare-# JSON array. Environments are unpaginated; activities page backwards through history with the-# `count` + `starts_at` params (undocumented in the OpenAPI spec but used by the official-# Platform.sh PHP client and CLI).+# pagination (`page[size]` 1-100, `_links.next.href`). /regions uses the same envelope but names+# its list `regions`. The billing lists (invoices, usage records) page with a `page` number, and+# the spec documents no `_links` on invoices, so they fall back to incrementing `page`.+# - Project-scoped lists (/projects/{id}/environments, /projects/{id}/activities,+# /projects/{id}/environments/{id}/deployments) return a bare JSON array. Environments and+# deployments are unpaginated; activities page backwards through history with the `count` ++# `starts_at` params (undocumented in the OpenAPI spec but used by the official Platform.sh PHP+# client and CLI).## Only `activities` syncs incrementally: it's the one unbounded, append-heavy table, and the API# prunes expired activities server-side, so a full refresh would drop history the API no longer@@ -21,7 +24,7 @@ from products.warehouse_sources.backend.types import IncrementalField, Increment# until that's confirmed.-@dataclass+@dataclass(frozen=True)class PlatformShEndpointConfig:name: str# Path template appended to the API base. Fan-out children carry a placeholder filled from the@@ -40,11 +43,16 @@ class PlatformShEndpointConfig:# order, but they're all full refresh so no watermark ever depends on it.sort_mode: Literal["asc", "desc"] = "asc"should_sync_default: bool = True- # Fan-out: the endpoint whose rows seed this child's path ("organizations" or "projects").+ # Fan-out: the endpoint whose rows seed this child's path ("organizations", "projects" or+ # "environments").fan_out_parent: Optional[str] = None# True when the response is the `{"items": [...], "_links": {...}}` envelope with cursor# pagination; False for project-scoped endpoints that return a bare JSON array.response_envelope: bool = True+ # Envelope key that holds the rows.+ items_key: str = "items"+ # Page-number query param to increment when a page carries no `_links.next`.+ page_number_param: Optional[str] = None# Send `page[size]` on envelope requests / `count` on the activities feed. None for# unpaginated bare lists (environments).page_size: Optional[int] = 100@@ -136,6 +144,58 @@ PLATFORM_SH_ENDPOINTS: dict[str, PlatformShEndpointConfig] = {primary_keys=["organization_id", "id"],fan_out_parent="organizations",),+ "deployments": PlatformShEndpointConfig(+ name="deployments",+ # The resolved configuration running in each environment: apps, workers, services, crons,+ # routes and container sizes. The id is only unique within its environment (usually+ # "current"), so key on the injected project and environment ids too.+ path="/projects/{project_id}/environments/{environment_id}/deployments",+ primary_keys=["project_id", "environment_id", "id"],+ # The spec doesn't say whether `created_at` moves when the environment redeploys.+ partition_key=None,+ fan_out_parent="environments",+ response_envelope=False,+ page_size=None,+ include_parent_fields={"project_id": "project_id", "id": "environment_id"},+ # Variable values can hold secrets even when the variable isn't flagged sensitive, and they+ # appear at the top level and under each app and worker.+ strip_keys_recursive=["basic_auth", "variables"],+ ),+ "invoices": PlatformShEndpointConfig(+ name="invoices",+ path="/organizations/{organization_id}/invoices",+ primary_keys=["organization_id", "id"],+ partition_key="created",+ fan_out_parent="organizations",+ page_size=None,+ page_number_param="page",+ include_parent_fields={"id": "organization_id"},+ # A download link for the PDF, not invoice data.+ drop_keys=["invoice_pdf"],+ # Billing lists need the organization billing permission, which many token users lack.+ should_sync_default=False,+ ),+ "usage_records": PlatformShEndpointConfig(+ name="usage_records",+ # Usage per subscription and usage group (storage, environments, user licenses). The spec+ # documents `filter[started_at]`, but it isn't verified against the live API, so this+ # ships as full refresh like projects.+ path="/organizations/{organization_id}/records/usage",+ primary_keys=["organization_id", "id"],+ partition_key="start",+ fan_out_parent="organizations",+ page_size=None,+ page_number_param="page",+ include_parent_fields={"id": "organization_id"},+ max_pages_per_parent=1000,+ should_sync_default=False,+ ),+ "regions": PlatformShEndpointConfig(+ name="regions",+ path="/regions",+ partition_key=None,+ items_key="regions",+ ),}ENDPOINTS = tuple(PLATFORM_SH_ENDPOINTS.keys())diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/platform_sh/tests/test_platform_sh.py b/products/warehouse_sources/backend/temporal/data_imports/sources/platform_sh/tests/test_platform_sh.pyindex 4d37a4ecd319..2d6224fb144b 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/platform_sh/tests/test_platform_sh.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/platform_sh/tests/test_platform_sh.py@@ -1,3 +1,4 @@+import dataclassesfrom typing import Anyimport pytest@@ -19,6 +20,8 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.platform_sAUTH_FAILED_MESSAGE,PlatformShAuthenticationError,PlatformShClient,+ PlatformShPageCapExceededError,+ PlatformShPaginationError,PlatformShResumeConfig,PlatformShUntrustedURLError,platform_sh_source,@@ -107,21 +110,28 @@ class TestPlatformShClientAuth:class TestGetRowsOrganizations:- def test_follows_relative_next_link_and_checkpoints_after_each_page(self) -> None:+ @parameterized.expand([("organizations", "items"), ("regions", "regions")])+ def test_follows_relative_next_link_and_checkpoints_after_each_page(self, endpoint: str, items_key: str) -> None:# `_links.next.href` is a relative HAL link; failing to resolve it against the request URL- # would silently sync only the first page.+ # would silently sync only the first page. /regions names its list `regions`, so reading+ # `items` there would silently sync nothing.result = _driver().run(- "organizations",+ endpoint,[_token(),- ScriptedResponse(json=_envelope([{"id": "org-1"}], next_href="/organizations?page%5Bafter%5D=org-1")),- ScriptedResponse(json=_envelope([{"id": "org-2"}])),+ ScriptedResponse(+ json={+ items_key: [{"id": "row-1"}],+ "_links": {"next": {"href": f"/{endpoint}?page%5Bafter%5D=row-1"}},+ }+ ),+ ScriptedResponse(json={items_key: [{"id": "row-2"}]}),],)assert result.raised is None- assert result.items == [[{"id": "org-1"}], [{"id": "org-2"}]]- second_url = f"{API}/organizations?page%5Bafter%5D=org-1"+ assert result.items == [[{"id": "row-1"}], [{"id": "row-2"}]]+ second_url = f"{API}/{endpoint}?page%5Bafter%5D=row-1"assert result.urls[-1] == second_url# Saved once, after yielding the first page, pointing at the second.assert result.saved_states == [PlatformShResumeConfig(next_url=second_url)]@@ -203,6 +213,116 @@ class TestGetRowsOrgFanOut:]]+ @parameterized.expand(+ [+ ("stops_on_empty_page", None, [[{"id": "inv-3"}], []], ["inv-1", "inv-2", "inv-3"], [None, "2", "3"], None),+ (+ "raises_when_page_param_is_ignored",+ None,+ [[{"id": "inv-1"}, {"id": "inv-2"}]],+ ["inv-1", "inv-2"],+ [None, "2"],+ PlatformShPaginationError,+ ),+ ("empty_probe_at_cap_completes", 1, [[]], ["inv-1", "inv-2"], [None, "2"], None),+ (+ "rows_past_cap_raise",+ 1,+ [[{"id": "inv-3"}]],+ ["inv-1", "inv-2"],+ [None, "2"],+ PlatformShPageCapExceededError,+ ),+ ]+ )+ def test_invoices_fall_back_to_page_numbers_without_next_link(+ self,+ _name: str,+ max_pages: int | None,+ later_pages: list[list[dict[str, Any]]],+ expected_ids: list[str],+ expected_page_params: list[str | None],+ expected_error: type[Exception] | None,+ ) -> None:+ # Invoices document only a `page` param and no `_links`. Stopping after the first page, or+ # treating a repeated page as the end, would silently sync a short table; a list that ends+ # exactly at the page cap must still complete.+ first_page = [+ {"id": "inv-1", "total": 10.0, "invoice_pdf": {"url": "https://example.com/inv-1.pdf"}},+ {"id": "inv-2", "total": 20.0},+ ]+ invoices_config = PLATFORM_SH_ENDPOINTS["invoices"]+ if max_pages is not None:+ invoices_config = dataclasses.replace(invoices_config, max_pages_per_parent=max_pages)+ with mock.patch.dict(PLATFORM_SH_ENDPOINTS, {"invoices": invoices_config}):+ result = _driver().run(+ "invoices",+ [+ _token(),+ ScriptedResponse(json=_envelope([{"id": "org-1"}])),+ ScriptedResponse(json=_envelope(first_page)),+ *[ScriptedResponse(json=_envelope(page)) for page in later_pages],+ ],+ )++ if expected_error is None:+ assert result.raised is None+ else:+ assert isinstance(result.raised, expected_error)+ assert [row["id"] for row in result.rows] == expected_ids+ assert all(row["organization_id"] == "org-1" and "invoice_pdf" not in row for row in result.rows)+ invoice_requests = [request for request in result.requests if request.path.endswith("/invoices")]+ assert [request.param("page") for request in invoice_requests] == expected_page_params++ def test_deployments_fan_out_through_environments(self) -> None:+ # Deployment ids ("current") repeat in every environment, so both parent ids must be+ # injected for the composite key. Branch names with "/" must be escaped in the path, an+ # inactive environment has nothing deployed, an environment deleted mid-sync must not fail+ # the whole table, and app-level variables can hold secrets.+ result = _driver().run(+ "deployments",+ [+ _token(),+ ScriptedResponse(json=_envelope([{"id": "org-1"}])),+ ScriptedResponse(json=_envelope([{"id": "proj-1"}])),+ ScriptedResponse(+ json=[+ {"id": "old", "status": "inactive"},+ {"id": "feature/x", "status": "active"},+ {"id": "gone", "status": "active"},+ ]+ ),+ ScriptedResponse(+ json=[+ {+ "id": "current",+ "variables": [{"name": "API_KEY", "value": "hunter2"}],+ "http_access": {"is_enabled": True, "basic_auth": {"admin": "hunter2"}},+ "webapps": {"app": {"type": "python:3.12", "variables": {"env": {"API_KEY": "hunter2"}}}},+ }+ ]+ ),+ ScriptedResponse(status=404, json={}),+ ],+ )++ assert result.raised is None+ assert result.items == [+ [+ {+ "id": "current",+ "http_access": {"is_enabled": True},+ "webapps": {"app": {"type": "python:3.12"}},+ "project_id": "proj-1",+ "environment_id": "feature/x",+ }+ ]+ ]+ assert result.urls[-2:] == [+ f"{API}/projects/proj-1/environments/feature%2Fx/deployments",+ f"{API}/projects/proj-1/environments/gone/deployments",+ ]+class TestGetRowsActivities:def _activity(self, activity_id: str, created_at: str) -> dict[str, Any]:@@ -295,5 +415,8 @@ class TestPlatformShSourceResponse:assert response.name == endpointassert response.primary_keys == config.primary_keysassert response.sort_mode == config.sort_mode- assert response.partition_mode == "datetime"- assert response.partition_keys == [config.partition_key]+ if config.partition_key:+ assert response.partition_mode == "datetime"+ assert response.partition_keys == [config.partition_key]+ else:+ assert response.partition_mode is None