:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(warehouse-sources): add railway usage, metrics and event tables (#126984)
Tom Owers5c16e09a56ce
diff --git a/posthog/test/repo_invariants/dataclass_frozen_baseline.txt b/posthog/test/repo_invariants/dataclass_frozen_baseline.txtindex 9df86fc192a3..52a9a36e93ee 100644--- a/posthog/test/repo_invariants/dataclass_frozen_baseline.txt+++ b/posthog/test/repo_invariants/dataclass_frozen_baseline.txt@@ -882,8 +882,6 @@1 products/warehouse_sources/backend/temporal/data_imports/sources/qualaroo/qualaroo.py1 products/warehouse_sources/backend/temporal/data_imports/sources/qualaroo/settings.py1 products/warehouse_sources/backend/temporal/data_imports/sources/qualys_vmdr/qualys_vmdr.py-1 products/warehouse_sources/backend/temporal/data_imports/sources/railway/railway.py-1 products/warehouse_sources/backend/temporal/data_imports/sources/railway/settings.py1 products/warehouse_sources/backend/temporal/data_imports/sources/ramp/ramp.py1 products/warehouse_sources/backend/temporal/data_imports/sources/ramp/settings.py1 products/warehouse_sources/backend/temporal/data_imports/sources/rapid7_insightvm/rapid7_insightvm.pydiff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.mdindex d0d2668e4add..0be057813c84 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md@@ -9990,10 +9990,10 @@ Today (6): `deployments`, `environments`, `project_members`, `projects`, `servicDiffed against: <https://backboard.railway.com/graphql/v2>-- [ ] `usage / projectServiceUsage / estimatedUsage` — Railway's headline metric - cost and resource usage per project/service/workspace; currently no spend data at all (high)-- [ ] `events` — Project event/activity stream - the audit-style timeline of what changed and when (high)-- [ ] `metrics` — CPU/memory/network/disk metrics per project, environment and service - core observability data (high)-- [ ] `deploymentEvents` — State-transition history for each deployment we already sync, enabling deploy-duration and failure analysis (high)+- [x] `usage / projectServiceUsage / estimatedUsage` — Railway's headline metric - cost and resource usage per project/service/workspace; currently no spend data at all (high)+- [x] `events` — Project event/activity stream - the audit-style timeline of what changed and when (high)+- [x] `metrics` — CPU/memory/network/disk metrics per project, environment and service - core observability data (high)+- [x] `deploymentEvents` — State-transition history for each deployment we already sync, enabling deploy-duration and failure analysis (high)- [ ] `serviceInstance` — Lookup resolving the service x environment pair that deployments already carry IDs for (high)- [ ] `auditLogs` — Workspace-level audit trail of who did what, for compliance and change analysis (medium)- [ ] `httpMetrics / httpMetricsGroupedByStatus / httpDurationMetrics` — Request volume, status-code breakdown and latency percentiles per service (medium)diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/railway/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/railway/canonical_descriptions.pyindex 7419c29b7e35..47316ae022dc 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/railway/canonical_descriptions.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/railway/canonical_descriptions.py@@ -93,4 +93,71 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {"createdAt": "When the volume was created.",},},+ "events": {+ "description": "An entry in a Railway project's activity feed, recording an action taken on an object in the project.",+ "docs_url": "https://docs.railway.com/integrations/api",+ "columns": {+ "id": "Unique identifier for the event.",+ "action": "The action that happened, such as a create, update, or deploy.",+ "object": "The kind of object the action applied to.",+ "severity": "Severity of the event (INFO, NOTICE, WARNING, or CRITICAL).",+ "projectId": "Identifier of the project the event belongs to.",+ "environmentId": "Identifier of the environment the event belongs to, if any.",+ "payload": "Details of the event as JSON.",+ "createdAt": "When the event happened.",+ },+ },+ "deployment_events": {+ "description": "A step in the lifecycle of a Railway deployment, such as building the image or running the healthcheck.",+ "docs_url": "https://docs.railway.com/integrations/api",+ "columns": {+ "deployment_id": "Identifier of the deployment the event belongs to.",+ "project_id": "Identifier of the project the deployment belongs to.",+ "service_id": "Identifier of the service the deployment belongs to.",+ "environment_id": "Identifier of the environment the deployment belongs to.",+ "id": "Identifier of the event.",+ "step": "The deployment step, such as BUILD_IMAGE, CREATE_CONTAINER, or HEALTHCHECK.",+ "payload": "Step details: attempt and max attempts, duration in milliseconds, whether the step was skipped, and any reason, detail, or error.",+ "createdAt": "When the step started.",+ "completedAt": "When the step finished, if it has.",+ },+ },+ "usage": {+ "description": "Resource usage of a Railway service for one day, aggregated per measurement.",+ "docs_url": "https://docs.railway.com/reference/pricing",+ "columns": {+ "id": "Synthesized key built from the day, project, service, environment, and measurement.",+ "date": "Start of the UTC day the usage covers.",+ "workspace_id": "Identifier of the workspace that owns the project.",+ "projectId": "Identifier of the project the usage belongs to.",+ "serviceId": "Identifier of the service the usage belongs to, if any.",+ "environmentId": "Identifier of the environment the usage belongs to, if any.",+ "measurement": "What was measured, such as CPU_USAGE, MEMORY_USAGE_GB, NETWORK_TX_GB, DISK_USAGE_GB, or BACKUP_USAGE_GB.",+ "value": "The aggregated value of the measurement over the day.",+ },+ },+ "estimated_usage": {+ "description": "Railway's estimate of a project's usage at the end of the current billing period.",+ "docs_url": "https://docs.railway.com/reference/pricing",+ "columns": {+ "workspace_id": "Identifier of the workspace that owns the project.",+ "projectId": "Identifier of the project the estimate belongs to.",+ "measurement": "What was estimated, such as CPU_USAGE or MEMORY_USAGE_GB.",+ "estimatedValue": "The estimated value of the measurement at the end of the billing period.",+ },+ },+ "metrics": {+ "description": "Hourly resource metric samples for a Railway service in an environment.",+ "docs_url": "https://docs.railway.com/reference/metrics",+ "columns": {+ "id": "Synthesized key built from the project, service, environment, measurement, and sample time.",+ "project_id": "Identifier of the project the sample belongs to.",+ "serviceId": "Identifier of the service the sample belongs to.",+ "environmentId": "Identifier of the environment the sample belongs to.",+ "measurement": "What was measured, such as CPU_USAGE, MEMORY_USAGE_GB, or NETWORK_RX_GB.",+ "ts": "Sample time as seconds since the Unix epoch.",+ "timestamp": "Sample time as a UTC datetime.",+ "value": "The value of the sample.",+ },+ },}diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/railway/railway.py b/products/warehouse_sources/backend/temporal/data_imports/sources/railway/railway.pyindex 2daaea1fb302..53d0a1684528 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/railway/railway.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/railway/railway.py@@ -1,12 +1,14 @@import dataclasses-from collections.abc import Iterator-from datetime import UTC, date, datetime+from collections.abc import Callable, Iterator+from datetime import UTC, date, datetime, timedeltafrom typing import Any, Optionalimport requestsfrom structlog.types import FilteringBoundLoggerfrom tenacity import retry_if_exception_type, stop_after_attempt, wait_exponential_jitter+from posthog.dataclasses import frozen+from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import (make_tracked_session,retry_with_deadline,@@ -14,9 +16,17 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.httfrom products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManagerfrom products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponsefrom products.warehouse_sources.backend.temporal.data_imports.sources.railway.settings import (+ CREATED_AT,+ METRICS_INITIAL_LOOKBACK_DAYS,+ METRICS_MEASUREMENTS,+ METRICS_SAMPLE_RATE_SECONDS,+ METRICS_WINDOW_DAYS,RAILWAY_API_URL,RAILWAY_ENDPOINTS,RAILWAY_PAGE_SIZE,+ USAGE_INITIAL_LOOKBACK_DAYS,+ USAGE_MEASUREMENTS,+ USAGE_PAGE_SIZE,VALIDATION_QUERY,RailwayEndpointConfig,)@@ -28,7 +38,7 @@ class RailwayRetryableError(Exception):pass[email protected](frozen=True)class RailwayResumeConfig:# Relay cursor (`pageInfo.endCursor`) to resume the current connection from. None means# "start the connection at its first page".@@ -37,6 +47,16 @@ class RailwayResumeConfig:# (not a positional index) so projects created/deleted between a crash and the retry can't# resume us into the wrong project. None for the top-level `projects` endpoint.project_id: str | None = None+ # ISO start of the time window to resume from, for the windowed `usage` and `metrics` endpoints.+ window_start: str | None = None+ # ISO end of the interrupted run's last window. A retry ends there too, so every parent of a+ # partial window covers the same interval. The next incremental sync reads what came after.+ window_end: str | None = None+ # The parent to resume at inside the current page or window: a deployment for `deployment_events`,+ # a workspace or project for the windowed endpoints.+ parent_id: str | None = None+ # Cursor of the parent's own connection, such as the deployment's events.+ child_cursor: str | None = Nonedef _get_headers(api_token: str) -> dict[str, str]:@@ -186,12 +206,28 @@ def _iter_connection_pages(cursor = next_cursor-def _iter_project_ids(session: requests.Session, headers: dict[str, str], logger: FilteringBoundLogger) -> list[str]:+def _list_projects(+ session: requests.Session, headers: dict[str, str], logger: FilteringBoundLogger+) -> list[dict[str, Any]]:projects_config = RAILWAY_ENDPOINTS["projects"]- ids: list[str] = []+ projects: list[dict[str, Any]] = []for rows, _ in _iter_connection_pages(session, headers, logger, projects_config, {}, None):- ids.extend(row["id"] for row in rows)- return ids+ projects.extend(rows)+ return projects+++def _iter_project_ids(session: requests.Session, headers: dict[str, str], logger: FilteringBoundLogger) -> list[str]:+ return [project["id"] for project in _list_projects(session, headers, logger)]+++def _iter_workspace_ids(session: requests.Session, headers: dict[str, str], logger: FilteringBoundLogger) -> list[str]:+ # Usage is reported per workspace. A token can reach several, so take every workspace that owns+ # a visible project.+ workspace_ids: dict[str, None] = {}+ for project in _list_projects(session, headers, logger):+ if project.get("workspaceId"):+ workspace_ids[project["workspaceId"]] = None+ return list(workspace_ids)def _get_top_level_rows(@@ -275,6 +311,314 @@ def _get_fan_out_rows(resumable_source_manager.save_state(RailwayResumeConfig(cursor=None, project_id=remaining[index + 1]))+def _get_deployment_event_rows(+ session: requests.Session,+ headers: dict[str, str],+ logger: FilteringBoundLogger,+ endpoint_config: RailwayEndpointConfig,+ resumable_source_manager: ResumableSourceManager[RailwayResumeConfig],+ watermark: datetime | None,+) -> Iterator[list[dict[str, Any]]]:+ """Walk each project's deployments newest-first and read the events of each deployment.++ Before each batch, the resume state moves to the position after it, so a resumed run reads+ no batch twice. A full refresh appends, so a replayed batch would duplicate rows.+ """+ deployments_config = RAILWAY_ENDPOINTS["deployments"]+ project_ids = _iter_project_ids(session, headers, logger)++ resume = resumable_source_manager.load_state() if resumable_source_manager.can_resume() else None+ remaining = project_ids+ if resume is not None and resume.project_id is not None and resume.project_id in project_ids:+ remaining = project_ids[project_ids.index(resume.project_id) :]+ else:+ resume = None++ for project_index, project_id in enumerate(remaining):+ next_project_id = remaining[project_index + 1] if project_index + 1 < len(remaining) else None+ page_cursor = resume.cursor if resume is not None else None+ seek_id = resume.parent_id if resume is not None else None+ seek_events_cursor = resume.child_cursor if resume is not None else None+ resume = None+ while True:+ for deployments, next_page_cursor in _iter_connection_pages(+ session, headers, logger, deployments_config, {"projectId": project_id}, page_cursor+ ):+ page_predates_watermark = (+ watermark is not None+ and bool(deployments)+ and _page_predates_watermark(deployments, CREATED_AT, watermark)+ )+ start_index, events_cursor = 0, None+ if seek_id is not None:+ deployment_ids = [deployment["id"] for deployment in deployments]+ if seek_id not in deployment_ids:+ # Deployments created since the interrupted run push the saved one onto a+ # later page.+ if page_predates_watermark:+ seek_id = None+ break+ page_cursor = next_page_cursor+ continue+ start_index, events_cursor = deployment_ids.index(seek_id), seek_events_cursor+ seek_id = None++ for index in range(start_index, len(deployments)):+ deployment = deployments[index]+ created_at = _as_utc_datetime(deployment.get(CREATED_AT))+ # An event is never older than its deployment. The 24h incremental lookback covers+ # events that a deployment created before the watermark still emits later.+ if watermark is not None and created_at is not None and created_at < watermark:+ continue+ context = {+ "deployment_id": deployment["id"],+ "project_id": project_id,+ "service_id": deployment.get("serviceId"),+ "environment_id": deployment.get("environmentId"),+ }+ for events, next_events_cursor in _iter_connection_pages(+ session, headers, logger, endpoint_config, {"deploymentId": deployment["id"]}, events_cursor+ ):+ if next_events_cursor is not None:+ position: RailwayResumeConfig | None = RailwayResumeConfig(+ cursor=page_cursor,+ project_id=project_id,+ parent_id=deployment["id"],+ child_cursor=next_events_cursor,+ )+ elif index + 1 < len(deployments):+ position = RailwayResumeConfig(+ cursor=page_cursor, project_id=project_id, parent_id=deployments[index + 1]["id"]+ )+ elif next_page_cursor is not None:+ position = RailwayResumeConfig(cursor=next_page_cursor, project_id=project_id)+ elif next_project_id is not None:+ position = RailwayResumeConfig(project_id=next_project_id)+ else:+ # The last page of the run. A position without a project would restart the walk.+ position = None+ if position is not None:+ resumable_source_manager.save_state(position)+ if events:+ yield [{**context, **event} for event in events]+ else:+ resumable_source_manager.safe_point()+ events_cursor = None+ if page_predates_watermark:+ break+ page_cursor = next_page_cursor+ if seek_id is None:+ break+ logger.debug(f"Railway: saved deployment {seek_id} not found in project {project_id}, rereading it")+ page_cursor, seek_id, seek_events_cursor = None, None, None+++@frozen+class _Window:+ start: datetime+ end: datetime+++def _iter_windows(start: datetime, end: datetime, size: timedelta) -> Iterator[_Window]:+ while start < end:+ yield _Window(start=start, end=min(start + size, end))+ start += size+++def _get_windowed_rows(+ resumable_source_manager: ResumableSourceManager[RailwayResumeConfig],+ start: datetime,+ end: datetime,+ window_size: timedelta,+ parent_ids: list[str],+ fetch_page: Callable[[str, datetime, datetime, str | None], Iterator[tuple[list[dict[str, Any]], str | None]]],+) -> Iterator[list[dict[str, Any]]]:+ """Read every parent for each time window, oldest window first.++ Before each page, the resume state moves to the position after it. A resumed run rebuilds+ its windows from the saved window start and end, because both depend on the current time+ and would shift between attempts.+ """+ resume = resumable_source_manager.load_state() if resumable_source_manager.can_resume() else None+ resume_from = _as_utc_datetime(resume.window_start) if resume is not None and resume.window_start else None+ if resume is not None and resume_from is not None:+ start = resume_from+ resume_end = _as_utc_datetime(resume.window_end) if resume.window_end else None+ if resume_end is not None:+ end = resume_end+ else:+ resume = None++ run_end = end.isoformat()+ windows = list(_iter_windows(start, end, window_size))+ for window_index, window in enumerate(windows):+ window_start, window_end = window.start, window.end+ parents, cursor = parent_ids, None+ if resume is not None:+ if resume.parent_id in parent_ids:+ parents, cursor = parent_ids[parent_ids.index(resume.parent_id) :], resume.cursor+ resume = None++ for parent_index, parent_id in enumerate(parents):+ for rows, next_cursor in fetch_page(parent_id, window_start, window_end, cursor):+ if next_cursor is not None:+ position: RailwayResumeConfig | None = RailwayResumeConfig(+ window_start=window_start.isoformat(),+ window_end=run_end,+ parent_id=parent_id,+ cursor=next_cursor,+ )+ elif parent_index + 1 < len(parents):+ position = RailwayResumeConfig(+ window_start=window_start.isoformat(), window_end=run_end, parent_id=parents[parent_index + 1]+ )+ elif window_index + 1 < len(windows):+ position = RailwayResumeConfig(+ window_start=windows[window_index + 1].start.isoformat(), window_end=run_end+ )+ else:+ # The last page of the run. Keep the staged position: resuming past the end+ # would start a window that is not aligned to the bucket size.+ position = None+ if position is not None:+ resumable_source_manager.save_state(position)+ if rows:+ yield rows+ else:+ resumable_source_manager.safe_point()+ cursor = None+++def _usage_rows(+ session: requests.Session,+ headers: dict[str, str],+ logger: FilteringBoundLogger,+ endpoint_config: RailwayEndpointConfig,+ workspace_id: str,+ window_start: datetime,+ window_end: datetime,+ cursor: str | None,+) -> Iterator[tuple[list[dict[str, Any]], str | None]]:+ while True:+ data = _execute(+ session,+ headers,+ endpoint_config.query,+ {+ "workspaceId": workspace_id,+ "startDate": window_start.isoformat(),+ "endDate": window_end.isoformat(),+ "measurements": USAGE_MEASUREMENTS,+ "first": USAGE_PAGE_SIZE,+ "after": cursor,+ },+ logger,+ )+ page = _dig(data, endpoint_config.data_path) or {}+ rows = []+ for item in page.get("usage") or []:+ tags = item.get("tags") or {}+ project_id, service_id, environment_id = (+ tags.get("projectId"),+ tags.get("serviceId"),+ tags.get("environmentId"),+ )+ key_parts = (window_start.date().isoformat(), project_id, service_id, environment_id, item["measurement"])+ rows.append(+ {+ # Tags are null when usage is not attributed to a service, so build a non-null key.+ "id": ":".join(part or "" for part in key_parts),+ "date": window_start,+ "workspace_id": workspace_id,+ "projectId": project_id,+ "serviceId": service_id,+ "environmentId": environment_id,+ "measurement": item["measurement"],+ "value": item.get("value"),+ }+ )+ page_info = page.get("pageInfo") or {}+ cursor = page_info.get("endCursor") if page_info.get("hasNextPage") else None+ yield rows, cursor+ if cursor is None:+ return+++def _metrics_rows(+ session: requests.Session,+ headers: dict[str, str],+ logger: FilteringBoundLogger,+ endpoint_config: RailwayEndpointConfig,+ project_id: str,+ window_start: datetime,+ window_end: datetime,+) -> Iterator[tuple[list[dict[str, Any]], str | None]]:+ data = _execute(+ session,+ headers,+ endpoint_config.query,+ {+ "projectId": project_id,+ "startDate": window_start.isoformat(),+ "endDate": window_end.isoformat(),+ "measurements": METRICS_MEASUREMENTS,+ "sampleRateSeconds": METRICS_SAMPLE_RATE_SECONDS,+ },+ logger,+ )+ rows = []+ for result in _dig(data, endpoint_config.data_path) or []:+ tags = result.get("tags") or {}+ service_id, environment_id = tags.get("serviceId"), tags.get("environmentId")+ for sample in result.get("values") or []:+ ts = sample["ts"]+ # Keep the window end exclusive, so a sample on the boundary is not read twice in one run.+ if ts >= window_end.timestamp():+ continue+ rows.append(+ {+ "id": f"{project_id}:{service_id or ''}:{environment_id or ''}:{result['measurement']}:{ts}",+ "project_id": project_id,+ "serviceId": service_id,+ "environmentId": environment_id,+ "measurement": result["measurement"],+ "ts": ts,+ "timestamp": datetime.fromtimestamp(ts, tz=UTC),+ "value": sample.get("value"),+ }+ )+ yield rows, None+++def _get_estimated_usage_rows(+ session: requests.Session,+ headers: dict[str, str],+ logger: FilteringBoundLogger,+ endpoint_config: RailwayEndpointConfig,+ resumable_source_manager: ResumableSourceManager[RailwayResumeConfig],+) -> Iterator[list[dict[str, Any]]]:+ for workspace_id in _iter_workspace_ids(session, headers, logger):+ data = _execute(+ session,+ headers,+ endpoint_config.query,+ {"workspaceId": workspace_id, "measurements": USAGE_MEASUREMENTS},+ logger,+ )+ rows = [{"workspace_id": workspace_id, **item} for item in _dig(data, endpoint_config.data_path) or []]+ if rows:+ yield rows+ else:+ resumable_source_manager.safe_point()+++def _window_start(watermark: datetime | None, now: datetime, initial_lookback: timedelta, unit: timedelta) -> datetime:+ start = watermark if watermark is not None else now - initial_lookback+ # Align to the bucket size so re-reading the last, partial bucket reproduces the same row ids.+ epoch = datetime(1970, 1, 1, tzinfo=UTC)+ return epoch + ((start - epoch) // unit) * unit++def get_rows(api_token: str,endpoint: str,@@ -293,7 +637,45 @@ def get_rows(watermark = _as_utc_datetime(db_incremental_field_last_value)try:- if endpoint_config.fan_out_over_projects:+ if endpoint == "deployment_events":+ yield from _get_deployment_event_rows(+ session, headers, logger, endpoint_config, resumable_source_manager, watermark+ )+ elif endpoint == "estimated_usage":+ yield from _get_estimated_usage_rows(session, headers, logger, endpoint_config, resumable_source_manager)+ elif endpoint == "usage":+ now = datetime.now(UTC)+ day = timedelta(days=1)+ start = _window_start(watermark, now, timedelta(days=USAGE_INITIAL_LOOKBACK_DAYS), day)+ yield from _get_windowed_rows(+ resumable_source_manager,+ start,+ now,+ day,+ _iter_workspace_ids(session, headers, logger),+ lambda workspace_id, window_start, window_end, cursor: _usage_rows(+ session, headers, logger, endpoint_config, workspace_id, window_start, window_end, cursor+ ),+ )+ elif endpoint == "metrics":+ now = datetime.now(UTC)+ start = _window_start(+ watermark,+ now,+ timedelta(days=METRICS_INITIAL_LOOKBACK_DAYS),+ timedelta(seconds=METRICS_SAMPLE_RATE_SECONDS),+ )+ yield from _get_windowed_rows(+ resumable_source_manager,+ start,+ now,+ timedelta(days=METRICS_WINDOW_DAYS),+ _iter_project_ids(session, headers, logger),+ lambda project_id, window_start, window_end, _cursor: _metrics_rows(+ session, headers, logger, endpoint_config, project_id, window_start, window_end+ ),+ )+ elif endpoint_config.fan_out_over_projects:yield from _get_fan_out_rows(session,headers,@@ -335,7 +717,7 @@ def railway_source(# Railway's connections return newest-first and have no server-side time filter, so the# incremental watermark must persist only at successful job end — a partial run's max# says nothing about older pages (or other projects) it never reached.- sort_mode="desc",+ sort_mode=endpoint_config.sort_mode,partition_count=1 if endpoint_config.partition_keys else None,partition_size=1 if endpoint_config.partition_keys else None,partition_mode="datetime" if endpoint_config.partition_keys else None,diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/railway/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/railway/settings.pyindex 9b15fd253c1f..508f45b21a52 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/railway/settings.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/railway/settings.py@@ -1,4 +1,5 @@from dataclasses import dataclass, field+from typing import Literalfrom products.warehouse_sources.backend.types import IncrementalField, IncrementalFieldType@@ -7,6 +8,9 @@ RAILWAY_API_URL = "https://backboard.railway.com/graphql/v2"RAILWAY_PAGE_SIZE = 100CREATED_AT = "createdAt"+# Synthesized on rows of the windowed `usage` and `metrics` tables, which carry no timestamp of their own.+DATE = "date"+TIMESTAMP = "timestamp"INCREMENTAL_CREATED_AT: list[IncrementalField] = [{@@ -17,6 +21,45 @@ INCREMENTAL_CREATED_AT: list[IncrementalField] = [},]+INCREMENTAL_DATE: list[IncrementalField] = [+ {+ "label": DATE,+ "type": IncrementalFieldType.DateTime,+ "field": DATE,+ "field_type": IncrementalFieldType.DateTime,+ },+]++INCREMENTAL_TIMESTAMP: list[IncrementalField] = [+ {+ "label": TIMESTAMP,+ "type": IncrementalFieldType.DateTime,+ "field": TIMESTAMP,+ "field_type": IncrementalFieldType.DateTime,+ },+]++# `projectServiceUsage` caps `first` at 50 projects per page.+USAGE_PAGE_SIZE = 50+USAGE_INITIAL_LOOKBACK_DAYS = 90+# Railway keeps about 30 days of metrics, so an older start date returns nothing.+METRICS_INITIAL_LOOKBACK_DAYS = 30+METRICS_SAMPLE_RATE_SECONDS = 3600+METRICS_WINDOW_DAYS = 7++# The measurements Railway bills for: vCPU, memory, egress, volume storage and backups.+USAGE_MEASUREMENTS = ["CPU_USAGE", "MEMORY_USAGE_GB", "NETWORK_TX_GB", "DISK_USAGE_GB", "BACKUP_USAGE_GB"]+METRICS_MEASUREMENTS = [+ "CPU_USAGE",+ "CPU_LIMIT",+ "MEMORY_USAGE_GB",+ "MEMORY_LIMIT_GB",+ "NETWORK_RX_GB",+ "NETWORK_TX_GB",+ "DISK_USAGE_GB",+ "EPHEMERAL_DISK_USAGE_GB",+]+_PAGE_INFO = "pageInfo { hasNextPage endCursor }"PROJECTS_QUERY = f"""@@ -147,6 +190,122 @@ query ProjectVolumes($projectId: String!, $first: Int!, $after: String) {{}}"""+EVENTS_QUERY = f"""+query ProjectEvents($projectId: String!, $first: Int!, $after: String) {{+ events(projectId: $projectId, first: $first, after: $after) {{+ edges {{+ node {{+ id+ action+ object+ severity+ projectId+ environmentId+ payload+ createdAt+ }}+ }}+ {_PAGE_INFO}+ }}+}}+"""++DEPLOYMENT_EVENTS_QUERY = f"""+query DeploymentEvents($deploymentId: String!, $first: Int!, $after: String) {{+ deploymentEvents(id: $deploymentId, first: $first, after: $after) {{+ edges {{+ node {{+ id+ step+ payload {{+ attempt+ maxAttempts+ durationMs+ skipped+ reason+ detail+ error+ }}+ createdAt+ completedAt+ }}+ }}+ {_PAGE_INFO}+ }}+}}+"""++USAGE_QUERY = """+query ProjectServiceUsage(+ $workspaceId: String!+ $startDate: DateTime!+ $endDate: DateTime!+ $measurements: [MetricMeasurement!]!+ $first: Int!+ $after: String+) {+ projectServiceUsage(+ workspaceId: $workspaceId+ startDate: $startDate+ endDate: $endDate+ measurements: $measurements+ includeDeleted: true+ first: $first+ after: $after+ ) {+ usage {+ measurement+ value+ tags {+ projectId+ serviceId+ environmentId+ }+ }+ pageInfo { hasNextPage endCursor }+ }+}+"""++ESTIMATED_USAGE_QUERY = """+query EstimatedUsage($workspaceId: String!, $measurements: [MetricMeasurement!]!) {+ estimatedUsage(workspaceId: $workspaceId, measurements: $measurements, includeDeleted: true) {+ projectId+ measurement+ estimatedValue+ }+}+"""++METRICS_QUERY = """+query ProjectMetrics(+ $projectId: String!+ $startDate: DateTime!+ $endDate: DateTime!+ $measurements: [MetricMeasurement!]!+ $sampleRateSeconds: Int!+) {+ metrics(+ projectId: $projectId+ startDate: $startDate+ endDate: $endDate+ measurements: $measurements+ groupBy: [SERVICE_ID, ENVIRONMENT_ID]+ sampleRateSeconds: $sampleRateSeconds+ ) {+ measurement+ tags {+ serviceId+ environmentId+ }+ values {+ ts+ value+ }+ }+}+"""+# Cheapest authenticated probe that works for both account and workspace tokens# (workspace/team tokens cannot call `me`, so probe `projects` instead).VALIDATION_QUERY = """@@ -162,7 +321,7 @@ query ValidateToken {"""-@dataclass+@dataclass(frozen=True)class RailwayEndpointConfig:name: strquery: str@@ -175,6 +334,9 @@ class RailwayEndpointConfig:paginated: bool = True# Whether the endpoint runs once per project (projects are enumerated first).fan_out_over_projects: bool = True+ sort_mode: Literal["asc", "desc"] | None = "desc"+ # Overlap re-read on incremental syncs, for rows that keep changing after they are created.+ default_incremental_lookback_seconds: int | None = NoneRAILWAY_ENDPOINTS: dict[str, RailwayEndpointConfig] = {@@ -200,6 +362,7 @@ RAILWAY_ENDPOINTS: dict[str, RailwayEndpointConfig] = {data_path=("deployments",),incremental_fields=INCREMENTAL_CREATED_AT,partition_keys=[CREATED_AT],+ default_incremental_lookback_seconds=86400,),"project_members": RailwayEndpointConfig(name="project_members",@@ -214,6 +377,49 @@ RAILWAY_ENDPOINTS: dict[str, RailwayEndpointConfig] = {query=VOLUMES_QUERY,data_path=("project", "volumes"),),+ "events": RailwayEndpointConfig(+ name="events",+ query=EVENTS_QUERY,+ data_path=("events",),+ partition_keys=[CREATED_AT],+ ),+ "deployment_events": RailwayEndpointConfig(+ name="deployment_events",+ query=DEPLOYMENT_EVENTS_QUERY,+ data_path=("deploymentEvents",),+ incremental_fields=INCREMENTAL_CREATED_AT,+ # Deployment events carry no deployment id, so the transport adds one. Event ids are not+ # documented as globally unique, so the deployment id is part of the key.+ primary_keys=["deployment_id", "id"],+ partition_keys=[CREATED_AT],+ sort_mode=None,+ default_incremental_lookback_seconds=86400,+ ),+ "usage": RailwayEndpointConfig(+ name="usage",+ query=USAGE_QUERY,+ data_path=("projectServiceUsage",),+ incremental_fields=INCREMENTAL_DATE,+ partition_keys=[DATE],+ sort_mode=None,+ ),+ "estimated_usage": RailwayEndpointConfig(+ name="estimated_usage",+ query=ESTIMATED_USAGE_QUERY,+ data_path=("estimatedUsage",),+ paginated=False,+ primary_keys=["projectId", "measurement"],+ sort_mode=None,+ ),+ "metrics": RailwayEndpointConfig(+ name="metrics",+ query=METRICS_QUERY,+ data_path=("metrics",),+ paginated=False,+ incremental_fields=INCREMENTAL_TIMESTAMP,+ partition_keys=[TIMESTAMP],+ sort_mode=None,+ ),}ENDPOINTS = tuple(RAILWAY_ENDPOINTS.keys())diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/railway/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/railway/source.pyindex 7e1b3a9e5a5e..b9d6a8a8e1bd 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/railway/source.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/railway/source.py@@ -27,9 +27,25 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.railway.rafrom products.warehouse_sources.backend.temporal.data_imports.sources.railway.settings import (ENDPOINTS,INCREMENTAL_FIELDS,+ RAILWAY_ENDPOINTS,)from products.warehouse_sources.backend.types import ExternalDataSourceType+_SCHEMA_DESCRIPTIONS: dict[str, str] = {+ "deployments": (+ "Incremental syncs page newest-first and stop at the last-seen createdAt. A 24h "+ "lookback re-pulls recent deployments so late status changes are picked up; older "+ "status changes only refresh on a full refresh."+ ),+ "deployment_events": (+ "Incremental syncs read events only for deployments created since the last-seen createdAt, "+ "with a 24h lookback for deployments that were still in progress."+ ),+ "usage": "Daily resource usage per project and service, over the last 90 days on the first sync.",+ "metrics": "Hourly resource metrics per service and environment. Railway keeps about 30 days of metrics.",+ "estimated_usage": "Estimated usage per project at the end of the current billing period.",+}+@SourceRegistry.registerclass RailwaySource(ResumableSource[RailwaySourceConfig, RailwayResumeConfig]):@@ -48,7 +64,7 @@ class RailwaySource(ResumableSource[RailwaySourceConfig, RailwayResumeConfig]):category=DataWarehouseSourceCategory.ENGINEERING___MONITORING,label="Railway",releaseStatus=ReleaseStatus.GA,- caption="""Enter your Railway API token to pull your Railway projects, services, environments, deployments, members, and volumes into the PostHog Data warehouse.+ caption="""Enter your Railway API token to pull your Railway projects, services, environments, deployments, deployment events, project events, members, volumes, usage, and metrics into the PostHog Data warehouse.Create an account or workspace token in your [Railway account settings](https://railway.com/account/tokens). Project tokens are not supported — they are scoped to a single environment and cannot list your projects.@@ -113,16 +129,10 @@ Note that Railway rate limits API requests per plan (as low as 100 requests/hour# safe write mode — append would materialize stale duplicates.supports_append=False,incremental_fields=INCREMENTAL_FIELDS.get(endpoint, []),- description=(- "Incremental syncs page newest-first and stop at the last-seen createdAt. A 24h "- "lookback re-pulls recent deployments so late status changes are picked up; older "- "status changes only refresh on a full refresh."- if endpoint == "deployments"- else None- ),- # Deployments still building/deploying keep changing status after creation; re-read a- # trailing day each incremental run so those rows settle without a full refresh.- default_incremental_lookback_seconds=86400 if endpoint == "deployments" else None,+ description=_SCHEMA_DESCRIPTIONS.get(endpoint),+ # Deployments still building/deploying keep changing after creation; re-read a trailing+ # window each incremental run so those rows settle without a full refresh.+ default_incremental_lookback_seconds=RAILWAY_ENDPOINTS[endpoint].default_incremental_lookback_seconds,)schemas = [_build_schema(endpoint) for endpoint in ENDPOINTS]diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/railway/tests/test_railway.py b/products/warehouse_sources/backend/temporal/data_imports/sources/railway/tests/test_railway.pyindex 2ddbac18e002..cc23117241e6 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/railway/tests/test_railway.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/railway/tests/test_railway.py@@ -2,6 +2,7 @@ from datetime import UTC, datetimefrom typing import Anyimport pytest+import time_machinefrom parameterized import parameterized@@ -38,6 +39,17 @@ def _projects_page(ids: list[str], end_cursor: str | None = None) -> dict[str, Areturn _connection(("projects",), [{"id": project_id} for project_id in ids], end_cursor)+def _usage_page(usage: list[dict[str, Any]], end_cursor: str | None = None) -> dict[str, Any]:+ return {+ "data": {+ "projectServiceUsage": {+ "usage": usage,+ "pageInfo": {"hasNextPage": end_cursor is not None, "endCursor": end_cursor},+ }+ }+ }++def _run(endpoint: str, responses: list[dict[str, Any]], **inputs: Any) -> DriveResult:script: list[ScriptedResponse | BaseException] = [ScriptedResponse(json=payload) for payload in responses]return SourceDriver(RailwaySource(), RailwaySourceConfig(api_token="token")).run(endpoint, script, **inputs)@@ -150,6 +162,178 @@ class TestRailway:assert _batch_ids(result) == [["d1", "d2"], ["d3", "d4"]]assert len(result.requests) == 3+ def test_deployment_events_carry_deployment_context_and_skip_deployments_before_watermark(self):+ result = _run(+ "deployment_events",+ [+ _projects_page(["p1"]),+ _connection(+ ("deployments",),+ [+ {"id": "d1", "serviceId": "s1", "environmentId": "e1", "createdAt": "2024-07-01T00:00:00Z"},+ ],+ end_cursor="c1",+ ),+ _connection(+ ("deploymentEvents",),+ [{"id": "ev1", "step": "BUILD_IMAGE", "createdAt": "2024-07-01T00:01:00Z"}],+ ),+ _connection(("deployments",), [{"id": "d2", "createdAt": "2024-05-01T00:00:00Z"}], end_cursor="c2"),+ # d2 predates the watermark: no events request for it, and the page after it is never+ # requested.+ ],+ incremental_field="createdAt",+ db_incremental_field_last_value=datetime(2024, 6, 1, tzinfo=UTC),+ )++ assert result.raised is None+ assert result.items == [+ [+ {+ "deployment_id": "d1",+ "project_id": "p1",+ "service_id": "s1",+ "environment_id": "e1",+ "id": "ev1",+ "step": "BUILD_IMAGE",+ "createdAt": "2024-07-01T00:01:00Z",+ }+ ]+ ]+ assert len(result.requests) == 4+ assert _request_variables(result, 2)["deploymentId"] == "d1"++ def test_deployment_events_resume_at_the_saved_deployment_and_stage_the_next_position(self):+ result = _run(+ "deployment_events",+ [+ _projects_page(["p1", "p2"]),+ _connection(("deployments",), [{"id": "d1"}, {"id": "d2"}, {"id": "d3"}]),+ _connection(("deploymentEvents",), [{"id": "ev2"}]),+ _connection(("deploymentEvents",), [{"id": "ev3"}]),+ ],+ resume_state=RailwayResumeConfig(cursor="c4", project_id="p2", parent_id="d2", child_cursor="e1"),+ )++ assert _request_variables(result, 1) == {"projectId": "p2", "first": 100, "after": "c4"}+ assert _request_variables(result, 2) == {"deploymentId": "d2", "first": 100, "after": "e1"}+ assert _request_variables(result, 3)["after"] is None+ assert _batch_ids(result) == [["ev2"], ["ev3"]]+ # The state staged before each batch points past it, so a resumed full refresh appends no+ # batch twice. The last batch keeps the earlier position rather than restarting the walk.+ assert [(state.cursor, state.project_id, state.parent_id) for state in result.saved_states] == [+ ("c4", "p2", "d3")+ ]++ def test_deployment_events_resume_pages_forward_to_a_saved_deployment_pushed_off_the_first_page(self):+ result = _run(+ "deployment_events",+ [+ _projects_page(["p1"]),+ # New deployments now fill the first page, and d1 and d2 were already read.+ _connection(("deployments",), [{"id": "d9"}, {"id": "d1"}, {"id": "d2"}], end_cursor="c1"),+ _connection(("deployments",), [{"id": "d3"}]),+ _connection(("deploymentEvents",), [{"id": "ev3"}]),+ ],+ resume_state=RailwayResumeConfig(project_id="p1", parent_id="d3"),+ )++ assert result.raised is None+ assert _request_variables(result, 2)["after"] == "c1"+ assert _request_variables(result, 3)["deploymentId"] == "d3"+ assert _batch_ids(result) == [["ev3"]]+ assert len(result.requests) == 4++ @time_machine.travel("2024-07-03T12:00:00Z", tick=False)+ def test_usage_reads_one_day_window_per_workspace_from_the_watermark_day(self):+ result = _run(+ "usage",+ [+ _connection(+ ("projects",),+ [{"id": "p1", "workspaceId": "w1"}, {"id": "p2", "workspaceId": "w1"}],+ ),+ _usage_page(+ [{"measurement": "CPU_USAGE", "value": 1.5, "tags": {"projectId": "p1", "serviceId": None}}],+ end_cursor="p1",+ ),+ _usage_page([{"measurement": "CPU_USAGE", "value": 2.0, "tags": {"projectId": "p2"}}]),+ _usage_page([]),+ ],+ incremental_field="date",+ db_incremental_field_last_value=datetime(2024, 7, 2, 0, 0, tzinfo=UTC),+ )++ assert result.raised is None+ windows = [+ (variables["workspaceId"], variables["startDate"], variables["endDate"], variables["after"])+ for variables in (_request_variables(result, index) for index in range(1, len(result.requests)))+ ]+ assert windows == [+ ("w1", "2024-07-02T00:00:00+00:00", "2024-07-03T00:00:00+00:00", None),+ ("w1", "2024-07-02T00:00:00+00:00", "2024-07-03T00:00:00+00:00", "p1"),+ ("w1", "2024-07-03T00:00:00+00:00", "2024-07-03T12:00:00+00:00", None),+ ]+ # Null tags must still give a non-null, stable key, or merges seed duplicate rows.+ assert _batch_ids(result) == [["2024-07-02:p1:::CPU_USAGE"], ["2024-07-02:p2:::CPU_USAGE"]]+ assert result.items[0][0]["date"] == datetime(2024, 7, 2, tzinfo=UTC)+ assert [(state.window_start, state.parent_id, state.cursor) for state in result.saved_states] == [+ ("2024-07-02T00:00:00+00:00", "w1", "p1"),+ ("2024-07-03T00:00:00+00:00", None, None),+ ]++ @time_machine.travel("2024-07-10T01:30:00Z", tick=False)+ def test_metrics_retry_resumes_at_the_saved_window_and_drops_samples_on_the_window_end(self):+ window_end = int(datetime(2024, 7, 10, 1, 30, tzinfo=UTC).timestamp())+ result = _run(+ "metrics",+ [+ _projects_page(["p1"]),+ {+ "data": {+ "metrics": [+ {+ "measurement": "CPU_USAGE",+ "tags": {"serviceId": "s1", "environmentId": "e1"},+ "values": [{"ts": window_end - 1800, "value": 0.5}, {"ts": window_end, "value": 0.7}],+ }+ ]+ }+ },+ ],+ # A first sync that an earlier attempt, an hour and a half ago, left in its 07-08 00:00 window.+ # The grid this attempt derives from the current time starts at 01:00, which would skip an hour.+ resume_state=RailwayResumeConfig(window_start="2024-07-08T00:00:00+00:00"),+ )++ assert result.raised is None+ variables = _request_variables(result, 1)+ assert (variables["startDate"], variables["endDate"]) == (+ "2024-07-08T00:00:00+00:00",+ "2024-07-10T01:30:00+00:00",+ )+ assert _batch_ids(result) == [[f"p1:s1:e1:CPU_USAGE:{window_end - 1800}"]]++ @time_machine.travel("2024-07-10T15:00:00Z", tick=False)+ def test_metrics_retry_keeps_the_saved_window_end_for_the_remaining_projects(self):+ result = _run(+ "metrics",+ [_projects_page(["p1", "p2"]), {"data": {"metrics": []}}],+ # p1 finished the 07-10 window, which ended at noon, before the earlier attempt stopped.+ resume_state=RailwayResumeConfig(+ window_start="2024-07-10T00:00:00+00:00", window_end="2024-07-10T12:00:00+00:00", parent_id="p2"+ ),+ )++ assert result.raised is None+ assert len(result.requests) == 2+ variables = _request_variables(result, 1)+ assert (variables["projectId"], variables["startDate"], variables["endDate"]) == (+ "p2",+ "2024-07-10T00:00:00+00:00",+ "2024-07-10T12:00:00+00:00",+ )+def test_not_authorized_error_matches_non_retryable_catalog(self):result = _run("projects", [{"errors": [{"message": "Not Authorized"}], "data": None}])@@ -188,6 +372,10 @@ class TestRailway:assert members.primary_keys == ["project_id", "id"]assert members.partition_mode is None+ deployment_events = _run("deployment_events", [_projects_page([])]).response+ assert deployment_events is not None+ assert deployment_events.primary_keys == ["deployment_id", "id"]+@parameterized.expand([({"data": {"projects": {"edges": []}}}, True, None),