Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(data-warehouse): add platform.sh deployments, invoices, usage and regions tables (#126277)

Tom Owers1764336904a9
diff --git a/posthog/test/repo_invariants/dataclass_frozen_baseline.txt b/posthog/test/repo_invariants/dataclass_frozen_baseline.txt
index 5634a3cfb8c7..89d7494eae9f 100644
--- a/posthog/test/repo_invariants/dataclass_frozen_baseline.txt
+++ b/posthog/test/repo_invariants/dataclass_frozen_baseline.txt
@@ -857,8 +857,6 @@
1 products/warehouse_sources/backend/temporal/data_imports/sources/plaid/settings.py
1 products/warehouse_sources/backend/temporal/data_imports/sources/planhat/planhat.py
1 products/warehouse_sources/backend/temporal/data_imports/sources/planhat/settings.py
-1 products/warehouse_sources/backend/temporal/data_imports/sources/platform_sh/platform_sh.py
-1 products/warehouse_sources/backend/temporal/data_imports/sources/platform_sh/settings.py
1 products/warehouse_sources/backend/temporal/data_imports/sources/plausible/plausible.py
1 products/warehouse_sources/backend/temporal/data_imports/sources/plausible/settings.py
1 products/warehouse_sources/backend/temporal/data_imports/sources/plivo/plivo.py
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
index 0182b5b2c252..0710066e35a8 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
@@ -9558,10 +9558,10 @@ Today (6): `activities`, `environments`, `members`, `organizations`, `projects`,
Diffed against: <https://meta.upsun.com/openapi-spec>
-- [ ] `/projects/{projectId}/environments/{environmentId}/deployments` — what is actually running per environment — services, workers, crons, resource sizes (high)
-- [ ] `/organizations/{organization_id}/records/usage` — consumption records; the only way to attribute spend to projects and environments (high)
-- [ ] `/organizations/{organization_id}/invoices` — billed amounts per period, the headline financial object (high)
-- [ ] `/regions` — lookup resolving the region IDs already carried on synced projects and subscriptions (high)
+- [x] `/projects/{projectId}/environments/{environmentId}/deployments` — what is actually running per environment — services, workers, crons, resource sizes (high)
+- [x] `/organizations/{organization_id}/records/usage` — consumption records; the only way to attribute spend to projects and environments (high)
+- [x] `/organizations/{organization_id}/invoices` — billed amounts per period, the headline financial object (high)
+- [x] `/regions` — lookup resolving the region IDs already carried on synced projects and subscriptions (high)
- [ ] `/projects/{projectId}/environments/{environmentId}/backups` — backup history for recovery-posture and retention reporting (medium)
- [ ] `/projects/{projectId}/domains` — domains attached to a project, needed to map projects to sites (medium)
- [ ] `/projects/{projectId}/environments/{environmentId}/routes` — route table per environment, resolves which URL serves which app (medium)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/platform_sh/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/platform_sh/canonical_descriptions.py
index 9cd6828325f3..6b2e8bb38e0b 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/platform_sh/canonical_descriptions.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/platform_sh/canonical_descriptions.py
@@ -119,4 +119,76 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {
"updated_at": "When the membership was last updated.",
},
},
+ "deployments": {
+ "description": "The read-only configuration currently deployed to an environment: apps, workers, services, routes, crons and container sizes.",
+ "docs_url": "https://api.platform.sh/docs/#tag/Deployment",
+ "columns": {
+ "id": "Identifier of the deployment, unique within its environment.",
+ "project_id": "ID of the project the deployment belongs to (injected from the parent project).",
+ "environment_id": "ID of the environment the deployment belongs to (injected from the parent environment).",
+ "created_at": "When the deployment was created.",
+ "updated_at": "When the deployment was last updated.",
+ "fingerprint": "Fingerprint of the deployment configuration.",
+ "cluster_name": "Name of the cluster the deployment runs on.",
+ "deployment_target": "Deployment target the environment runs on.",
+ "services": "Managed services (databases, caches, queues) in the deployment, with their type and size.",
+ "routes": "Routes that map URLs to the deployment's apps.",
+ "webapps": "Web applications in the deployment, with their runtime, crons and resources.",
+ "workers": "Worker processes in the deployment.",
+ "container_profiles": "Container size profiles available to the deployment.",
+ "tasks": "Sizing applied to tasks when they run.",
+ },
+ },
+ "invoices": {
+ "description": "An invoice or credit memo billed to an organization.",
+ "docs_url": "https://api.platform.sh/docs/#tag/Invoices",
+ "columns": {
+ "id": "Unique identifier for the invoice.",
+ "organization_id": "ID of the organization the invoice belongs to (injected from the parent organization).",
+ "invoice_number": "Invoice number.",
+ "type": "Invoice type: invoice for a standard invoice, credit_memo for a refund or credit.",
+ "order_id": "ID of the order the invoice belongs to.",
+ "related_invoice_id": "For a credit memo, the ID of the invoice it refunds.",
+ "status": "Invoice status (e.g. paid, pending, refunded, canceled).",
+ "owner": "ULID of the invoice owner.",
+ "invoice_date": "Date of the invoice.",
+ "invoice_due": "Due date of the invoice.",
+ "created": "When the invoice was created.",
+ "changed": "When the invoice was last changed.",
+ "company": "Company name on the invoice, if any.",
+ "total": "Invoice total.",
+ "address": "Billing address on the invoice.",
+ "notes": "Notes on the invoice.",
+ },
+ },
+ "usage_records": {
+ "description": "A usage record of a subscription for one usage group, such as storage, environments or user licenses.",
+ "docs_url": "https://api.platform.sh/docs/#tag/Records",
+ "columns": {
+ "id": "Unique identifier for the usage record.",
+ "organization_id": "ID of the organization the record belongs to (injected from the parent organization).",
+ "subscription_id": "ID of the subscription the usage belongs to.",
+ "usage_group": "Type of usage the record measures (storage, environments, user_licenses).",
+ "quantity": "Quantity used.",
+ "start": "When the usage record starts.",
+ },
+ },
+ "regions": {
+ "description": "A region that projects can be hosted in.",
+ "docs_url": "https://api.platform.sh/docs/#tag/Regions",
+ "columns": {
+ "id": "Unique identifier for the region, as stored on projects and subscriptions.",
+ "label": "Human-readable name of the region.",
+ "zone": "Geographical zone of the region.",
+ "selection_label": "Label shown when choosing a region for a new project.",
+ "project_label": "Label shown on existing projects in the region.",
+ "timezone": "Default timezone of the region.",
+ "available": "Whether the region can be selected for new projects.",
+ "private": "Whether the region is for private use only.",
+ "endpoint": "URL of the region API endpoint.",
+ "provider": "Information about the region's infrastructure provider.",
+ "datacenter": "Information about the provider's data center.",
+ "environmental_impact": "Information about the provider's environmental impact.",
+ },
+ },
}
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/platform_sh/platform_sh.py b/products/warehouse_sources/backend/temporal/data_imports/sources/platform_sh/platform_sh.py
index c77691ee9700..44f3747d7339 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/platform_sh/platform_sh.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/platform_sh/platform_sh.py
@@ -15,9 +15,10 @@ activities feed is newest-first and pages backwards through history via `count`
(params absent from the OpenAPI spec but used by the official Platform.sh PHP client and CLI, which
page by setting `starts_at` to the oldest `created_at` seen and dedupe by id).
-Fan-out: organizations -> projects/subscriptions/members, and organizations -> projects ->
-environments/activities. Resume state checkpoints the current organizations page URL after all its
-children are processed; a resumed run re-fans that page and merge dedupes on the primary key.
+Fan-out: organizations -> projects/subscriptions/members/invoices/usage records, organizations ->
+projects -> environments/activities, and organizations -> projects -> environments -> deployments.
+Resume state checkpoints the current organizations page URL after all its children are processed;
+a resumed run re-fans that page and merge dedupes on the primary key.
"""
import time
@@ -25,7 +26,7 @@ import dataclasses
from collections.abc import Iterator
from datetime import UTC, date, datetime
from typing import Any
-from urllib.parse import urlencode, urljoin, urlparse
+from urllib.parse import parse_qsl, quote, urlencode, urljoin, urlparse, urlunparse
import requests
from structlog.types import FilteringBoundLogger
@@ -59,10 +60,10 @@ REQUEST_TIMEOUT_SECONDS = 60
AUTH_FAILED_MESSAGE = "Platform.sh authentication failed"
[email protected](frozen=True)
class PlatformShResumeConfig:
- # Next URL to fetch. For the top-level organizations table it's the next page URL; for fan-out
- # tables it's the current organizations page URL (resume re-fans that page and its children,
+ # Next URL to fetch. For top-level tables (organizations, regions) it's the next page URL; for
+ # fan-out tables it's the current organizations page URL (resume re-fans that page and its children,
# merge dedupes on the primary key).
next_url: str | None = None
@@ -81,6 +82,12 @@ class PlatformShUntrustedURLError(Exception):
token to every request, so following an off-host URL would leak it; refuse instead."""
+class PlatformShPaginationError(Exception):
+ """Raised when a page-numbered list repeats a page it already returned. The server is ignoring
+ the page param, so the rows past the first page can't be fetched; fail rather than sync a short
+ table."""
+
+
class PlatformShPageCapExceededError(Exception):
"""Raised when a fan-out parent exceeds the per-parent page cap. Failing loudly beats silently
writing an incomplete table that later runs would never backfill."""
@@ -100,9 +107,9 @@ class PlatformShClient:
self._logger = logger
self._access_token: str | None = None
self._token_deadline = 0.0
- # capture=False: environment responses carry `http_access.basic_auth` and activity
- # responses carry raw `log` output — secrets we strip client-side in `_clean_rows`, which
- # runs after HTTP sample capture would have recorded the raw body. Requests stay metered
+ # capture=False: environment and deployment responses carry `http_access.basic_auth` and
+ # activity responses carry raw `log` output — secrets we strip client-side in `_clean_rows`,
+ # which runs after HTTP sample capture would have recorded the raw body. Requests stay metered
# and logged; only the opt-in body capture is excluded.
self._session = make_tracked_session(redact_values=(api_token,), capture=False)
@@ -221,27 +228,56 @@ def _next_link(body: dict[str, Any], current_url: str, client: PlatformShClient)
return client.validate_url(urljoin(current_url, href))
+def _with_page_number(url: str, param: str, page: int) -> str:
+ parsed = urlparse(url)
+ query = {**dict(parse_qsl(parsed.query)), param: str(page)}
+ return urlunparse(parsed._replace(query=urlencode(query)))
+
+
def _iter_envelope_pages(
client: PlatformShClient,
url: str,
max_pages: int | None = None,
page_cap_context: dict[str, Any] | None = None,
+ items_key: str = "items",
+ page_number_param: str | None = None,
) -> Iterator[tuple[list[dict[str, Any]], str]]:
"""Yield (items, page_url) for each page of an `{"items": [...], "_links": {...}}` list,
- following the `_links.next` cursor. When `max_pages` is set and more pages remain, raise
- rather than truncate: a silently short table would stay incomplete on every later run."""
+ following the `_links.next` cursor, or incrementing `page_number_param` when a page has no
+ next link. When `max_pages` is set and more pages remain, raise rather than truncate: a
+ silently short table would stay incomplete on every later run."""
page_count = 0
+ seen_ids: set[Any] = set()
while True:
body = client.get(url).json()
- items = body.get("items") if isinstance(body, dict) else None
+ items = body.get(items_key) if isinstance(body, dict) else None
if not items:
return
+ if max_pages is not None and page_count >= max_pages:
+ # Only a guessed page-number URL is fetched past the cap: an empty probe means the list
+ # ended exactly at the cap, anything else means rows remain.
+ raise PlatformShPageCapExceededError(
+ f"Platform.sh: page cap of {max_pages} reached with more pages remaining; "
+ f"raise max_pages_per_parent to sync this parent fully. context={page_cap_context or {}}"
+ )
next_url = _next_link(body, url, client)
+ guessed_next = False
+ if page_number_param is not None:
+ page_ids = {item.get("id") for item in items}
+ if page_ids <= seen_ids:
+ raise PlatformShPaginationError(
+ f"Platform.sh: {url} repeated a page it already returned; the server ignored "
+ f"`{page_number_param}`, so the full list can't be fetched. context={page_cap_context or {}}"
+ )
+ seen_ids |= page_ids
+ if next_url is None:
+ next_url = _with_page_number(url, page_number_param, page_count + 2)
+ guessed_next = True
yield items, url
page_count += 1
if not next_url:
return
- if max_pages is not None and page_count >= max_pages:
+ if max_pages is not None and page_count >= max_pages and not guessed_next:
raise PlatformShPageCapExceededError(
f"Platform.sh: page cap of {max_pages} reached with more pages remaining; "
f"raise max_pages_per_parent to sync this parent fully. context={page_cap_context or {}}"
@@ -354,6 +390,31 @@ def _iter_project_activities(
starts_at = min(row["created_at"] for row in new_rows)
+def _iter_environment_deployments(
+ client: PlatformShClient,
+ environment: dict[str, Any],
+ logger: FilteringBoundLogger,
+ config: PlatformShEndpointConfig,
+) -> Iterator[list[dict[str, Any]]]:
+ # Inactive environments run nothing, so there is no deployment to report.
+ if environment.get("status") == "inactive":
+ return
+ # Environment ids are Git branch names and can contain "/".
+ url = client.api_base + config.path.format(
+ project_id=environment["project_id"], environment_id=quote(environment["id"], safe="")
+ )
+ try:
+ data = client.get(url).json()
+ except requests.HTTPError as e:
+ # The environment was deleted after the environments list was read.
+ if e.response is not None and e.response.status_code == 404:
+ logger.debug(f"Platform.sh: no deployments for environment {environment['id']}, skipping")
+ return
+ raise
+ if isinstance(data, list) and data:
+ yield _clean_rows(_inject_parent_fields(data, environment, config.include_parent_fields), config)
+
+
def _iter_project_children(
client: PlatformShClient,
project: dict[str, Any],
@@ -362,6 +423,13 @@ def _iter_project_children(
cutoff: datetime | None,
) -> Iterator[list[dict[str, Any]]]:
project_id = project["id"]
+ if config.fan_out_parent == "environments":
+ environments_config = PLATFORM_SH_ENDPOINTS["environments"]
+ for environments in _iter_project_children(client, project, logger, environments_config, cutoff=None):
+ for environment in environments:
+ yield from _iter_environment_deployments(client, environment, logger, config)
+ return
+
if config.name == "activities":
raw_batches: Iterator[list[dict[str, Any]]] = _iter_project_activities(
client, project_id, logger, config, cutoff
@@ -402,30 +470,32 @@ def get_rows(
logger.debug(f"Platform.sh: syncing {endpoint} incrementally from {cutoff.isoformat()}")
resume = resumable_source_manager.load_state() if resumable_source_manager.can_resume() else None
- orgs_url = _build_url(client.api_base, "/organizations", PLATFORM_SH_ENDPOINTS["organizations"].page_size)
+ # Top-level tables page their own path; fan-out tables page the organizations list.
+ list_config = config if config.fan_out_parent is None else PLATFORM_SH_ENDPOINTS["organizations"]
+ list_url = _build_url(client.api_base, list_config.path, list_config.page_size)
if resume is not None and resume.next_url:
- orgs_url = client.validate_url(resume.next_url)
- logger.debug(f"Platform.sh: resuming {endpoint} from organizations page: {orgs_url}")
+ list_url = client.validate_url(resume.next_url)
+ logger.debug(f"Platform.sh: resuming {endpoint} from page: {list_url}")
- if endpoint == "organizations":
+ if config.fan_out_parent is None:
while True:
- body = client.get(orgs_url).json()
- items = body.get("items") if isinstance(body, dict) else None
+ body = client.get(list_url).json()
+ items = body.get(config.items_key) if isinstance(body, dict) else None
if not items:
return
- next_url = _next_link(body, orgs_url, client)
+ next_url = _next_link(body, list_url, client)
yield _clean_rows(items, config)
if not next_url:
return
# Save AFTER yielding so a crash re-yields the last page rather than skipping it
# (merge dedupes on the primary key).
resumable_source_manager.save_state(PlatformShResumeConfig(next_url=next_url))
- orgs_url = next_url
+ list_url = next_url
# Fan-out tables: walk the organizations list and emit child rows per org (and, for
# project-scoped tables, per project). Checkpoint the org page URL after all of its children
# are processed; a resumed run re-fans that page and merge dedupes.
- for orgs, org_page_url in _iter_envelope_pages(client, orgs_url):
+ for orgs, org_page_url in _iter_envelope_pages(client, list_url):
for org in orgs:
if config.fan_out_parent == "organizations":
child_url = _build_url(client.api_base, config.path.format(organization_id=org["id"]), config.page_size)
@@ -434,9 +504,11 @@ def get_rows(
child_url,
max_pages=config.max_pages_per_parent,
page_cap_context={"organization_id": org["id"], "endpoint": endpoint},
+ items_key=config.items_key,
+ page_number_param=config.page_number_param,
):
yield _clean_rows(_inject_parent_fields(child_items, org, config.include_parent_fields), config)
- else: # children of projects (environments, activities)
+ else: # children of projects (environments, activities, deployments)
projects_config = PLATFORM_SH_ENDPOINTS["projects"]
projects_url = _build_url(
client.api_base,
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/platform_sh/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/platform_sh/settings.py
index d5892a0eb34f..41964f9b25f3 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/platform_sh/settings.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/platform_sh/settings.py
@@ -7,11 +7,14 @@ from products.warehouse_sources.backend.types import IncrementalField, Increment
#
# - Organization-level lists (/organizations, /organizations/{id}/projects, .../members,
# .../subscriptions) return an `{"items": [...], "_links": {...}}` envelope with cursor
-# pagination (`page[size]` 1-100, `_links.next.href`).
-# - Project-scoped lists (/projects/{id}/environments, /projects/{id}/activities) return a bare
-# JSON array. Environments are unpaginated; activities page backwards through history with the
-# `count` + `starts_at` params (undocumented in the OpenAPI spec but used by the official
-# Platform.sh PHP client and CLI).
+# pagination (`page[size]` 1-100, `_links.next.href`). /regions uses the same envelope but names
+# its list `regions`. The billing lists (invoices, usage records) page with a `page` number, and
+# the spec documents no `_links` on invoices, so they fall back to incrementing `page`.
+# - Project-scoped lists (/projects/{id}/environments, /projects/{id}/activities,
+# /projects/{id}/environments/{id}/deployments) return a bare JSON array. Environments and
+# deployments are unpaginated; activities page backwards through history with the `count` +
+# `starts_at` params (undocumented in the OpenAPI spec but used by the official Platform.sh PHP
+# client and CLI).
#
# Only `activities` syncs incrementally: it's the one unbounded, append-heavy table, and the API
# prunes expired activities server-side, so a full refresh would drop history the API no longer
@@ -21,7 +24,7 @@ from products.warehouse_sources.backend.types import IncrementalField, Increment
# until that's confirmed.
-@dataclass
+@dataclass(frozen=True)
class PlatformShEndpointConfig:
name: str
# Path template appended to the API base. Fan-out children carry a placeholder filled from the
@@ -40,11 +43,16 @@ class PlatformShEndpointConfig:
# order, but they're all full refresh so no watermark ever depends on it.
sort_mode: Literal["asc", "desc"] = "asc"
should_sync_default: bool = True
- # Fan-out: the endpoint whose rows seed this child's path ("organizations" or "projects").
+ # Fan-out: the endpoint whose rows seed this child's path ("organizations", "projects" or
+ # "environments").
fan_out_parent: Optional[str] = None
# True when the response is the `{"items": [...], "_links": {...}}` envelope with cursor
# pagination; False for project-scoped endpoints that return a bare JSON array.
response_envelope: bool = True
+ # Envelope key that holds the rows.
+ items_key: str = "items"
+ # Page-number query param to increment when a page carries no `_links.next`.
+ page_number_param: Optional[str] = None
# Send `page[size]` on envelope requests / `count` on the activities feed. None for
# unpaginated bare lists (environments).
page_size: Optional[int] = 100
@@ -136,6 +144,58 @@ PLATFORM_SH_ENDPOINTS: dict[str, PlatformShEndpointConfig] = {
primary_keys=["organization_id", "id"],
fan_out_parent="organizations",
),
+ "deployments": PlatformShEndpointConfig(
+ name="deployments",
+ # The resolved configuration running in each environment: apps, workers, services, crons,
+ # routes and container sizes. The id is only unique within its environment (usually
+ # "current"), so key on the injected project and environment ids too.
+ path="/projects/{project_id}/environments/{environment_id}/deployments",
+ primary_keys=["project_id", "environment_id", "id"],
+ # The spec doesn't say whether `created_at` moves when the environment redeploys.
+ partition_key=None,
+ fan_out_parent="environments",
+ response_envelope=False,
+ page_size=None,
+ include_parent_fields={"project_id": "project_id", "id": "environment_id"},
+ # Variable values can hold secrets even when the variable isn't flagged sensitive, and they
+ # appear at the top level and under each app and worker.
+ strip_keys_recursive=["basic_auth", "variables"],
+ ),
+ "invoices": PlatformShEndpointConfig(
+ name="invoices",
+ path="/organizations/{organization_id}/invoices",
+ primary_keys=["organization_id", "id"],
+ partition_key="created",
+ fan_out_parent="organizations",
+ page_size=None,
+ page_number_param="page",
+ include_parent_fields={"id": "organization_id"},
+ # A download link for the PDF, not invoice data.
+ drop_keys=["invoice_pdf"],
+ # Billing lists need the organization billing permission, which many token users lack.
+ should_sync_default=False,
+ ),
+ "usage_records": PlatformShEndpointConfig(
+ name="usage_records",
+ # Usage per subscription and usage group (storage, environments, user licenses). The spec
+ # documents `filter[started_at]`, but it isn't verified against the live API, so this
+ # ships as full refresh like projects.
+ path="/organizations/{organization_id}/records/usage",
+ primary_keys=["organization_id", "id"],
+ partition_key="start",
+ fan_out_parent="organizations",
+ page_size=None,
+ page_number_param="page",
+ include_parent_fields={"id": "organization_id"},
+ max_pages_per_parent=1000,
+ should_sync_default=False,
+ ),
+ "regions": PlatformShEndpointConfig(
+ name="regions",
+ path="/regions",
+ partition_key=None,
+ items_key="regions",
+ ),
}
ENDPOINTS = tuple(PLATFORM_SH_ENDPOINTS.keys())
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/platform_sh/tests/test_platform_sh.py b/products/warehouse_sources/backend/temporal/data_imports/sources/platform_sh/tests/test_platform_sh.py
index 4d37a4ecd319..2d6224fb144b 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/platform_sh/tests/test_platform_sh.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/platform_sh/tests/test_platform_sh.py
@@ -1,3 +1,4 @@
+import dataclasses
from typing import Any
import pytest
@@ -19,6 +20,8 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.platform_s
AUTH_FAILED_MESSAGE,
PlatformShAuthenticationError,
PlatformShClient,
+ PlatformShPageCapExceededError,
+ PlatformShPaginationError,
PlatformShResumeConfig,
PlatformShUntrustedURLError,
platform_sh_source,
@@ -107,21 +110,28 @@ class TestPlatformShClientAuth:
class TestGetRowsOrganizations:
- def test_follows_relative_next_link_and_checkpoints_after_each_page(self) -> None:
+ @parameterized.expand([("organizations", "items"), ("regions", "regions")])
+ def test_follows_relative_next_link_and_checkpoints_after_each_page(self, endpoint: str, items_key: str) -> None:
# `_links.next.href` is a relative HAL link; failing to resolve it against the request URL
- # would silently sync only the first page.
+ # would silently sync only the first page. /regions names its list `regions`, so reading
+ # `items` there would silently sync nothing.
result = _driver().run(
- "organizations",
+ endpoint,
[
_token(),
- ScriptedResponse(json=_envelope([{"id": "org-1"}], next_href="/organizations?page%5Bafter%5D=org-1")),
- ScriptedResponse(json=_envelope([{"id": "org-2"}])),
+ ScriptedResponse(
+ json={
+ items_key: [{"id": "row-1"}],
+ "_links": {"next": {"href": f"/{endpoint}?page%5Bafter%5D=row-1"}},
+ }
+ ),
+ ScriptedResponse(json={items_key: [{"id": "row-2"}]}),
],
)
assert result.raised is None
- assert result.items == [[{"id": "org-1"}], [{"id": "org-2"}]]
- second_url = f"{API}/organizations?page%5Bafter%5D=org-1"
+ assert result.items == [[{"id": "row-1"}], [{"id": "row-2"}]]
+ second_url = f"{API}/{endpoint}?page%5Bafter%5D=row-1"
assert result.urls[-1] == second_url
# Saved once, after yielding the first page, pointing at the second.
assert result.saved_states == [PlatformShResumeConfig(next_url=second_url)]
@@ -203,6 +213,116 @@ class TestGetRowsOrgFanOut:
]
]
+ @parameterized.expand(
+ [
+ ("stops_on_empty_page", None, [[{"id": "inv-3"}], []], ["inv-1", "inv-2", "inv-3"], [None, "2", "3"], None),
+ (
+ "raises_when_page_param_is_ignored",
+ None,
+ [[{"id": "inv-1"}, {"id": "inv-2"}]],
+ ["inv-1", "inv-2"],
+ [None, "2"],
+ PlatformShPaginationError,
+ ),
+ ("empty_probe_at_cap_completes", 1, [[]], ["inv-1", "inv-2"], [None, "2"], None),
+ (
+ "rows_past_cap_raise",
+ 1,
+ [[{"id": "inv-3"}]],
+ ["inv-1", "inv-2"],
+ [None, "2"],
+ PlatformShPageCapExceededError,
+ ),
+ ]
+ )
+ def test_invoices_fall_back_to_page_numbers_without_next_link(
+ self,
+ _name: str,
+ max_pages: int | None,
+ later_pages: list[list[dict[str, Any]]],
+ expected_ids: list[str],
+ expected_page_params: list[str | None],
+ expected_error: type[Exception] | None,
+ ) -> None:
+ # Invoices document only a `page` param and no `_links`. Stopping after the first page, or
+ # treating a repeated page as the end, would silently sync a short table; a list that ends
+ # exactly at the page cap must still complete.
+ first_page = [
+ {"id": "inv-1", "total": 10.0, "invoice_pdf": {"url": "https://example.com/inv-1.pdf"}},
+ {"id": "inv-2", "total": 20.0},
+ ]
+ invoices_config = PLATFORM_SH_ENDPOINTS["invoices"]
+ if max_pages is not None:
+ invoices_config = dataclasses.replace(invoices_config, max_pages_per_parent=max_pages)
+ with mock.patch.dict(PLATFORM_SH_ENDPOINTS, {"invoices": invoices_config}):
+ result = _driver().run(
+ "invoices",
+ [
+ _token(),
+ ScriptedResponse(json=_envelope([{"id": "org-1"}])),
+ ScriptedResponse(json=_envelope(first_page)),
+ *[ScriptedResponse(json=_envelope(page)) for page in later_pages],
+ ],
+ )
+
+ if expected_error is None:
+ assert result.raised is None
+ else:
+ assert isinstance(result.raised, expected_error)
+ assert [row["id"] for row in result.rows] == expected_ids
+ assert all(row["organization_id"] == "org-1" and "invoice_pdf" not in row for row in result.rows)
+ invoice_requests = [request for request in result.requests if request.path.endswith("/invoices")]
+ assert [request.param("page") for request in invoice_requests] == expected_page_params
+
+ def test_deployments_fan_out_through_environments(self) -> None:
+ # Deployment ids ("current") repeat in every environment, so both parent ids must be
+ # injected for the composite key. Branch names with "/" must be escaped in the path, an
+ # inactive environment has nothing deployed, an environment deleted mid-sync must not fail
+ # the whole table, and app-level variables can hold secrets.
+ result = _driver().run(
+ "deployments",
+ [
+ _token(),
+ ScriptedResponse(json=_envelope([{"id": "org-1"}])),
+ ScriptedResponse(json=_envelope([{"id": "proj-1"}])),
+ ScriptedResponse(
+ json=[
+ {"id": "old", "status": "inactive"},
+ {"id": "feature/x", "status": "active"},
+ {"id": "gone", "status": "active"},
+ ]
+ ),
+ ScriptedResponse(
+ json=[
+ {
+ "id": "current",
+ "variables": [{"name": "API_KEY", "value": "hunter2"}],
+ "http_access": {"is_enabled": True, "basic_auth": {"admin": "hunter2"}},
+ "webapps": {"app": {"type": "python:3.12", "variables": {"env": {"API_KEY": "hunter2"}}}},
+ }
+ ]
+ ),
+ ScriptedResponse(status=404, json={}),
+ ],
+ )
+
+ assert result.raised is None
+ assert result.items == [
+ [
+ {
+ "id": "current",
+ "http_access": {"is_enabled": True},
+ "webapps": {"app": {"type": "python:3.12"}},
+ "project_id": "proj-1",
+ "environment_id": "feature/x",
+ }
+ ]
+ ]
+ assert result.urls[-2:] == [
+ f"{API}/projects/proj-1/environments/feature%2Fx/deployments",
+ f"{API}/projects/proj-1/environments/gone/deployments",
+ ]
+
class TestGetRowsActivities:
def _activity(self, activity_id: str, created_at: str) -> dict[str, Any]:
@@ -295,5 +415,8 @@ class TestPlatformShSourceResponse:
assert response.name == endpoint
assert response.primary_keys == config.primary_keys
assert response.sort_mode == config.sort_mode
- assert response.partition_mode == "datetime"
- assert response.partition_keys == [config.partition_key]
+ if config.partition_key:
+ assert response.partition_mode == "datetime"
+ assert response.partition_keys == [config.partition_key]
+ else:
+ assert response.partition_mode is None