Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(warehouse-sources): add railway instances, triggers, workspaces (#127355)

Tom Owers2eb85207e2f2
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
index 6e2244c4bbc4..f518f6973029 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
@@ -10021,12 +10021,12 @@ Diffed against: <https://backboard.railway.com/graphql/v2>
- [x] `events` — Project event/activity stream - the audit-style timeline of what changed and when (high)
- [x] `metrics` — CPU/memory/network/disk metrics per project, environment and service - core observability data (high)
- [x] `deploymentEvents` — State-transition history for each deployment we already sync, enabling deploy-duration and failure analysis (high)
-- [ ] `serviceInstance` — Lookup resolving the service x environment pair that deployments already carry IDs for (high)
+- [x] `serviceInstance` — Lookup resolving the service x environment pair that deployments already carry IDs for (high)
- [ ] `auditLogs` — Workspace-level audit trail of who did what, for compliance and change analysis (medium)
- [ ] `httpMetrics / httpMetricsGroupedByStatus / httpDurationMetrics` — Request volume, status-code breakdown and latency percentiles per service (medium)
-- [ ] `volumeInstance / volumeInstanceBackupList` — Per-environment volume instances and backups that resolve the volume IDs already synced (medium)
-- [ ] `workspace` — Lookup table resolving the workspace/team that every project belongs to (medium)
-- [ ] `deploymentTriggers` — Explains what caused each deployment (branch, PR, check suite) (medium)
+- [x] `volumeInstance / volumeInstanceBackupList` — Per-environment volume instances and backups that resolve the volume IDs already synced (medium)
+- [x] `workspace` — Lookup table resolving the workspace/team that every project belongs to (medium)
+- [x] `deploymentTriggers` — Explains what caused each deployment (branch, PR, check suite) (medium)
- [ ] `httpLogs / deploymentLogs` — Request and deployment log streams for error and traffic analysis (low)
- [ ] `regions` — Small lookup resolving region codes on services and volumes (low)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/railway/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/railway/canonical_descriptions.py
index 47316ae022dc..791b46b26004 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/railway/canonical_descriptions.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/railway/canonical_descriptions.py
@@ -160,4 +160,111 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {
"value": "The value of the sample.",
},
},
+ "service_instances": {
+ "description": "The configuration of a Railway service in one environment: how it builds, starts, scales, and restarts.",
+ "docs_url": "https://docs.railway.com/reference/services",
+ "columns": {
+ "id": "Unique identifier for the service instance.",
+ "project_id": "Identifier of the project the service belongs to.",
+ "serviceId": "Identifier of the service.",
+ "serviceName": "Display name of the service.",
+ "environmentId": "Identifier of the environment the instance runs in.",
+ "region": "Region the instance deploys to.",
+ "numReplicas": "Number of replicas the instance runs.",
+ "builder": "Builder used for the instance, such as NIXPACKS, RAILPACK, or HEROKU.",
+ "buildCommand": "Custom build command, if any.",
+ "startCommand": "Custom start command, if any.",
+ "preDeployCommand": "Command run before each deployment, if any.",
+ "rootDirectory": "Directory of the repository the service builds from.",
+ "dockerfilePath": "Path of the Dockerfile used for the build, if any.",
+ "railwayConfigFile": "Path of the Railway config-as-code file, if any.",
+ "watchPatterns": "File patterns that trigger a deployment when they change.",
+ "healthcheckPath": "HTTP path Railway checks before it routes traffic to a new deployment.",
+ "healthcheckTimeout": "Seconds Railway waits for the health check to pass.",
+ "cronSchedule": "Cron schedule of the instance, if it runs as a cron job.",
+ "nextCronRunAt": "When the cron job runs next, if it is a cron job.",
+ "restartPolicyType": "When Railway restarts the instance: ALWAYS, NEVER, or ON_FAILURE.",
+ "restartPolicyMaxRetries": "Maximum number of restarts for the ON_FAILURE policy.",
+ "sleepApplication": "Whether the instance sleeps when it receives no traffic.",
+ "upstreamUrl": "Upstream repository URL of the service, if any.",
+ "source": "Where the service deploys from: a Docker image or a repository.",
+ "createdAt": "When the service instance was created.",
+ "updatedAt": "When the service instance was last updated.",
+ "deletedAt": "When the service instance was deleted, if it has been.",
+ },
+ },
+ "volume_instances": {
+ "description": "A Railway volume in one environment, mounted on a service.",
+ "docs_url": "https://docs.railway.com/reference/volumes",
+ "columns": {
+ "id": "Unique identifier for the volume instance.",
+ "project_id": "Identifier of the project the volume belongs to.",
+ "volumeId": "Identifier of the volume.",
+ "serviceId": "Identifier of the service the volume is mounted on, if any.",
+ "environmentId": "Identifier of the environment the instance belongs to.",
+ "mountPath": "Path the volume is mounted at inside the service.",
+ "region": "Region the volume instance is in.",
+ "state": "Current state of the volume instance, such as READY, MIGRATING, or ERROR.",
+ "sizeMB": "Allocated size of the volume instance, in megabytes.",
+ "currentSizeMB": "Space the volume instance uses, in megabytes.",
+ "externalId": "Railway's internal storage identifier for the volume instance.",
+ "isPendingDeletion": "Whether the volume instance is scheduled for deletion.",
+ "createdAt": "When the volume instance was created.",
+ "deletedAt": "When the volume instance was deleted, if it has been.",
+ },
+ },
+ "volume_instance_backups": {
+ "description": "A backup of a Railway volume instance.",
+ "docs_url": "https://docs.railway.com/volumes/backups",
+ "columns": {
+ "id": "Identifier of the backup.",
+ "project_id": "Identifier of the project the volume belongs to.",
+ "environment_id": "Identifier of the environment the volume instance belongs to.",
+ "volume_instance_id": "Identifier of the volume instance the backup was taken from.",
+ "volume_id": "Identifier of the volume the backup was taken from.",
+ "service_id": "Identifier of the service the volume is mounted on, if any.",
+ "name": "Name of the backup, if any.",
+ "externalId": "Railway's internal storage identifier for the backup.",
+ "creatorId": "Identifier of the user who created the backup, if it was created manually.",
+ "scheduleId": "Identifier of the backup schedule that created the backup, if any.",
+ "usedMB": "Space the backup uses, in megabytes.",
+ "referencedMB": "Data the backup references, in megabytes.",
+ "volumeInstanceSizeMB": "Size of the volume instance when the backup was taken, in megabytes.",
+ "createdAt": "When the backup was created.",
+ "expiresAt": "When the backup expires, if it does.",
+ },
+ },
+ "deployment_triggers": {
+ "description": "A rule that deploys a Railway service when a connected repository branch changes.",
+ "docs_url": "https://docs.railway.com/deployments/github-autodeploys",
+ "columns": {
+ "id": "Unique identifier for the deployment trigger.",
+ "projectId": "Identifier of the project the trigger belongs to.",
+ "environmentId": "Identifier of the environment the trigger deploys to.",
+ "serviceId": "Identifier of the service the trigger deploys, if any.",
+ "provider": "Source control provider of the repository, such as github.",
+ "repository": "Repository the trigger watches.",
+ "branch": "Branch the trigger watches.",
+ "checkSuites": "Whether the trigger waits for the repository's CI check suites to pass before it deploys.",
+ "validCheckSuites": "Number of check suites that must pass before a deployment.",
+ "baseEnvironmentOverrideId": "Identifier of the environment PR environments fork from instead of the project default, if any.",
+ },
+ },
+ "workspaces": {
+ "description": "A Railway workspace — the team or account that owns projects and is billed for them.",
+ "docs_url": "https://docs.railway.com/projects/workspaces",
+ "columns": {
+ "id": "Unique identifier for the workspace.",
+ "name": "Display name of the workspace.",
+ "avatar": "URL of the workspace avatar, if any.",
+ "plan": "Subscription plan of the workspace: FREE, HOBBY, or PRO.",
+ "preferredRegion": "Default region for new services in the workspace.",
+ "hasSAML": "Whether SAML single sign-on is set up for the workspace.",
+ "has2FAEnforcement": "Whether the workspace requires two-factor authentication.",
+ "hasHipaaBAA": "Whether the workspace has a HIPAA business associate agreement.",
+ "restrictProjectVisibilityToGroups": "Whether only members of granted groups can see projects.",
+ "createdAt": "When the workspace was created.",
+ "updatedAt": "When the workspace was last updated.",
+ },
+ },
}
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/railway/railway.py b/products/warehouse_sources/backend/temporal/data_imports/sources/railway/railway.py
index 53d0a1684528..a46215b2678c 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/railway/railway.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/railway/railway.py
@@ -57,6 +57,15 @@ class RailwayResumeConfig:
parent_id: str | None = None
# Cursor of the parent's own connection, such as the deployment's events.
child_cursor: str | None = None
+ # The child to resume at inside the parent's current page, such as a volume instance for
+ # `volume_instance_backups`.
+ child_id: str | None = None
+ # Listing positions of project_id, parent_id and child_id, for a retry that finds the item deleted.
+ project_index: int | None = None
+ parent_index: int | None = None
+ child_index: int | None = None
+ # The run read every row, so a retry of the same run reads nothing.
+ completed: bool = False
def _get_headers(api_token: str) -> dict[str, str]:
@@ -413,6 +422,200 @@ def _get_deployment_event_rows(
page_cursor, seek_id, seek_events_cursor = None, None, None
+def _stage_and_emit(
+ resumable_source_manager: ResumableSourceManager[RailwayResumeConfig],
+ position: RailwayResumeConfig,
+ rows: list[dict[str, Any]],
+) -> Iterator[list[dict[str, Any]]]:
+ resumable_source_manager.save_state(position)
+ if rows:
+ yield rows
+ else:
+ resumable_source_manager.safe_point()
+
+
+def _resume_index(ids: list[str], saved_id: str | None, saved_index: int | None) -> int | None:
+ """Where a retry continues in `ids`: at the saved item, or at its saved position when it is gone.
+
+ The walk relies on a stable listing order, so the item that now holds a deleted item's position
+ is the first one the earlier attempt did not read.
+ """
+ if saved_id is None:
+ return None
+ if saved_id in ids:
+ return ids.index(saved_id)
+ if saved_index is not None:
+ return min(saved_index, len(ids))
+ return None
+
+
+# Gets the project id, the environment id, the position of the environment, the saved position when
+# the run resumes inside the environment, and the position after the environment.
+ReadEnvironment = Callable[
+ [str, str, RailwayResumeConfig, RailwayResumeConfig | None, RailwayResumeConfig],
+ Iterator[list[dict[str, Any]]],
+]
+
+
+def _get_environment_fan_out_rows(
+ session: requests.Session,
+ headers: dict[str, str],
+ logger: FilteringBoundLogger,
+ resumable_source_manager: ResumableSourceManager[RailwayResumeConfig],
+ read_environment: ReadEnvironment,
+) -> Iterator[list[dict[str, Any]]]:
+ """Walk every environment of every project and read it with `read_environment`.
+
+ Before each batch, the resume state moves to the position after it, and before the last batch
+ to a completed position. A resumed full refresh appends, so it must read no batch twice.
+ """
+ resume = resumable_source_manager.load_state() if resumable_source_manager.can_resume() else None
+ if resume is not None and resume.completed:
+ return
+
+ environments_config = RAILWAY_ENDPOINTS["environments"]
+ project_ids = _iter_project_ids(session, headers, logger)
+
+ project_start = None
+ if resume is not None:
+ project_start = _resume_index(project_ids, resume.project_id, resume.project_index)
+ if (
+ project_start is None
+ or project_start >= len(project_ids)
+ or project_ids[project_start] != resume.project_id
+ ):
+ # The saved project is gone, so its environment bookmark points nowhere.
+ resume = None
+
+ for project_index in range(project_start or 0, len(project_ids)):
+ project_id = project_ids[project_index]
+ next_project = (
+ RailwayResumeConfig(project_id=project_ids[project_index + 1], project_index=project_index + 1)
+ if project_index + 1 < len(project_ids)
+ else RailwayResumeConfig(completed=True)
+ )
+ environment_ids = [
+ environment["id"]
+ for environments, _ in _iter_connection_pages(
+ session, headers, logger, environments_config, {"projectId": project_id}, None
+ )
+ for environment in environments
+ ]
+ resumable_source_manager.safe_point()
+
+ environment_start = _resume_index(environment_ids, resume.parent_id, resume.parent_index) if resume else None
+ environment_resume = (
+ resume
+ if resume is not None
+ and environment_start is not None
+ and environment_start < len(environment_ids)
+ and environment_ids[environment_start] == resume.parent_id
+ else None
+ )
+ resume = None
+
+ for environment_index in range(environment_start or 0, len(environment_ids)):
+ environment_id = environment_ids[environment_index]
+ here = RailwayResumeConfig(
+ project_id=project_id,
+ project_index=project_index,
+ parent_id=environment_id,
+ parent_index=environment_index,
+ )
+ after = (
+ dataclasses.replace(
+ here, parent_id=environment_ids[environment_index + 1], parent_index=environment_index + 1
+ )
+ if environment_index + 1 < len(environment_ids)
+ else next_project
+ )
+ yield from read_environment(project_id, environment_id, here, environment_resume, after)
+ environment_resume = None
+
+
+def _environment_connection_rows(
+ session: requests.Session,
+ headers: dict[str, str],
+ logger: FilteringBoundLogger,
+ endpoint_config: RailwayEndpointConfig,
+ resumable_source_manager: ResumableSourceManager[RailwayResumeConfig],
+ project_id: str,
+ environment_id: str,
+ here: RailwayResumeConfig,
+ resume: RailwayResumeConfig | None,
+ after: RailwayResumeConfig,
+) -> Iterator[list[dict[str, Any]]]:
+ start_cursor = resume.cursor if resume is not None else None
+ for rows, next_cursor in _iter_connection_pages(
+ session, headers, logger, endpoint_config, {"environmentId": environment_id}, start_cursor
+ ):
+ position = dataclasses.replace(here, cursor=next_cursor) if next_cursor is not None else after
+ # Service and volume instances carry no project id of their own.
+ rows = [row if "projectId" in row else {"project_id": project_id, **row} for row in rows]
+ yield from _stage_and_emit(resumable_source_manager, position, rows)
+
+
+def _volume_instance_backup_rows(
+ session: requests.Session,
+ headers: dict[str, str],
+ logger: FilteringBoundLogger,
+ endpoint_config: RailwayEndpointConfig,
+ resumable_source_manager: ResumableSourceManager[RailwayResumeConfig],
+ project_id: str,
+ environment_id: str,
+ here: RailwayResumeConfig,
+ resume: RailwayResumeConfig | None,
+ after: RailwayResumeConfig,
+) -> Iterator[list[dict[str, Any]]]:
+ volume_instances_config = RAILWAY_ENDPOINTS["volume_instances"]
+ page_cursor = resume.cursor if resume is not None else None
+ for instances, next_page_cursor in _iter_connection_pages(
+ session, headers, logger, volume_instances_config, {"environmentId": environment_id}, page_cursor
+ ):
+ if not instances:
+ resumable_source_manager.safe_point()
+ instance_ids = [instance["id"] for instance in instances]
+ start_index = (_resume_index(instance_ids, resume.child_id, resume.child_index) if resume else None) or 0
+ resume = None
+ for index in range(start_index, len(instances)):
+ instance = instances[index]
+ data = _execute(session, headers, endpoint_config.query, {"volumeInstanceId": instance["id"]}, logger)
+ if index + 1 < len(instances):
+ position = dataclasses.replace(
+ here, cursor=page_cursor, child_id=instance_ids[index + 1], child_index=index + 1
+ )
+ elif next_page_cursor is not None:
+ position = dataclasses.replace(here, cursor=next_page_cursor)
+ else:
+ position = after
+ context = {
+ "project_id": project_id,
+ "environment_id": environment_id,
+ "volume_instance_id": instance["id"],
+ "volume_id": instance.get("volumeId"),
+ "service_id": instance.get("serviceId"),
+ }
+ backups = [{**context, **backup} for backup in _dig(data, endpoint_config.data_path) or []]
+ yield from _stage_and_emit(resumable_source_manager, position, backups)
+ page_cursor = next_page_cursor
+
+
+def _get_workspace_rows(
+ session: requests.Session,
+ headers: dict[str, str],
+ logger: FilteringBoundLogger,
+ endpoint_config: RailwayEndpointConfig,
+ resumable_source_manager: ResumableSourceManager[RailwayResumeConfig],
+) -> Iterator[list[dict[str, Any]]]:
+ for workspace_id in _iter_workspace_ids(session, headers, logger):
+ data = _execute(session, headers, endpoint_config.query, {"workspaceId": workspace_id}, logger)
+ workspace = _dig(data, endpoint_config.data_path)
+ if workspace:
+ yield [workspace]
+ else:
+ resumable_source_manager.safe_point()
+
+
@frozen
class _Window:
start: datetime
@@ -643,6 +846,46 @@ def get_rows(
)
elif endpoint == "estimated_usage":
yield from _get_estimated_usage_rows(session, headers, logger, endpoint_config, resumable_source_manager)
+ elif endpoint == "workspaces":
+ yield from _get_workspace_rows(session, headers, logger, endpoint_config, resumable_source_manager)
+ elif endpoint == "volume_instance_backups":
+ yield from _get_environment_fan_out_rows(
+ session,
+ headers,
+ logger,
+ resumable_source_manager,
+ lambda project_id, environment_id, here, resume, after: _volume_instance_backup_rows(
+ session,
+ headers,
+ logger,
+ endpoint_config,
+ resumable_source_manager,
+ project_id,
+ environment_id,
+ here,
+ resume,
+ after,
+ ),
+ )
+ elif endpoint_config.fan_out_over_environments:
+ yield from _get_environment_fan_out_rows(
+ session,
+ headers,
+ logger,
+ resumable_source_manager,
+ lambda project_id, environment_id, here, resume, after: _environment_connection_rows(
+ session,
+ headers,
+ logger,
+ endpoint_config,
+ resumable_source_manager,
+ project_id,
+ environment_id,
+ here,
+ resume,
+ after,
+ ),
+ )
elif endpoint == "usage":
now = datetime.now(UTC)
day = timedelta(days=1)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/railway/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/railway/settings.py
index 508f45b21a52..58eca4afe5e4 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/railway/settings.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/railway/settings.py
@@ -235,6 +235,135 @@ query DeploymentEvents($deploymentId: String!, $first: Int!, $after: String) {{
}}
"""
+SERVICE_INSTANCES_QUERY = f"""
+query EnvironmentServiceInstances($environmentId: String!, $first: Int!, $after: String) {{
+ environment(id: $environmentId) {{
+ serviceInstances(first: $first, after: $after) {{
+ edges {{
+ node {{
+ id
+ serviceId
+ serviceName
+ environmentId
+ region
+ numReplicas
+ builder
+ buildCommand
+ startCommand
+ preDeployCommand
+ rootDirectory
+ dockerfilePath
+ railwayConfigFile
+ watchPatterns
+ healthcheckPath
+ healthcheckTimeout
+ cronSchedule
+ nextCronRunAt
+ restartPolicyType
+ restartPolicyMaxRetries
+ sleepApplication
+ upstreamUrl
+ source {{
+ image
+ repo
+ }}
+ createdAt
+ updatedAt
+ deletedAt
+ }}
+ }}
+ {_PAGE_INFO}
+ }}
+ }}
+}}
+"""
+
+VOLUME_INSTANCES_QUERY = f"""
+query EnvironmentVolumeInstances($environmentId: String!, $first: Int!, $after: String) {{
+ environment(id: $environmentId) {{
+ volumeInstances(first: $first, after: $after) {{
+ edges {{
+ node {{
+ id
+ volumeId
+ serviceId
+ environmentId
+ mountPath
+ region
+ state
+ sizeMB
+ currentSizeMB
+ externalId
+ isPendingDeletion
+ createdAt
+ deletedAt
+ }}
+ }}
+ {_PAGE_INFO}
+ }}
+ }}
+}}
+"""
+
+VOLUME_INSTANCE_BACKUPS_QUERY = """
+query VolumeInstanceBackups($volumeInstanceId: String!) {
+ volumeInstanceBackupList(volumeInstanceId: $volumeInstanceId) {
+ id
+ name
+ externalId
+ creatorId
+ scheduleId
+ usedMB
+ referencedMB
+ volumeInstanceSizeMB
+ createdAt
+ expiresAt
+ }
+}
+"""
+
+DEPLOYMENT_TRIGGERS_QUERY = f"""
+query EnvironmentDeploymentTriggers($environmentId: String!, $first: Int!, $after: String) {{
+ environment(id: $environmentId) {{
+ deploymentTriggers(first: $first, after: $after) {{
+ edges {{
+ node {{
+ id
+ projectId
+ environmentId
+ serviceId
+ provider
+ repository
+ branch
+ checkSuites
+ validCheckSuites
+ baseEnvironmentOverrideId
+ }}
+ }}
+ {_PAGE_INFO}
+ }}
+ }}
+}}
+"""
+
+WORKSPACE_QUERY = """
+query Workspace($workspaceId: String!) {
+ workspace(workspaceId: $workspaceId) {
+ id
+ name
+ avatar
+ plan
+ preferredRegion
+ hasSAML
+ has2FAEnforcement
+ hasHipaaBAA
+ restrictProjectVisibilityToGroups
+ createdAt
+ updatedAt
+ }
+}
+"""
+
USAGE_QUERY = """
query ProjectServiceUsage(
$workspaceId: String!
@@ -334,6 +463,8 @@ class RailwayEndpointConfig:
paginated: bool = True
# Whether the endpoint runs once per project (projects are enumerated first).
fan_out_over_projects: bool = True
+ # Whether the endpoint runs once per environment of each project.
+ fan_out_over_environments: bool = False
sort_mode: Literal["asc", "desc"] | None = "desc"
# Overlap re-read on incremental syncs, for rows that keep changing after they are created.
default_incremental_lookback_seconds: int | None = None
@@ -420,6 +551,46 @@ RAILWAY_ENDPOINTS: dict[str, RailwayEndpointConfig] = {
partition_keys=[TIMESTAMP],
sort_mode=None,
),
+ "service_instances": RailwayEndpointConfig(
+ name="service_instances",
+ query=SERVICE_INSTANCES_QUERY,
+ data_path=("environment", "serviceInstances"),
+ fan_out_over_environments=True,
+ sort_mode=None,
+ ),
+ "volume_instances": RailwayEndpointConfig(
+ name="volume_instances",
+ query=VOLUME_INSTANCES_QUERY,
+ data_path=("environment", "volumeInstances"),
+ fan_out_over_environments=True,
+ sort_mode=None,
+ ),
+ "volume_instance_backups": RailwayEndpointConfig(
+ name="volume_instance_backups",
+ query=VOLUME_INSTANCE_BACKUPS_QUERY,
+ data_path=("volumeInstanceBackupList",),
+ paginated=False,
+ fan_out_over_environments=True,
+ # Backup ids are not documented as globally unique, so the volume instance id is part of the key.
+ primary_keys=["volume_instance_id", "id"],
+ partition_keys=[CREATED_AT],
+ sort_mode=None,
+ ),
+ "deployment_triggers": RailwayEndpointConfig(
+ name="deployment_triggers",
+ query=DEPLOYMENT_TRIGGERS_QUERY,
+ data_path=("environment", "deploymentTriggers"),
+ fan_out_over_environments=True,
+ sort_mode=None,
+ ),
+ "workspaces": RailwayEndpointConfig(
+ name="workspaces",
+ query=WORKSPACE_QUERY,
+ data_path=("workspace",),
+ paginated=False,
+ fan_out_over_projects=False,
+ sort_mode=None,
+ ),
}
ENDPOINTS = tuple(RAILWAY_ENDPOINTS.keys())
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/railway/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/railway/source.py
index b9d6a8a8e1bd..20a5b12128b7 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/railway/source.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/railway/source.py
@@ -64,7 +64,7 @@ class RailwaySource(ResumableSource[RailwaySourceConfig, RailwayResumeConfig]):
category=DataWarehouseSourceCategory.ENGINEERING___MONITORING,
label="Railway",
releaseStatus=ReleaseStatus.GA,
- caption="""Enter your Railway API token to pull your Railway projects, services, environments, deployments, deployment events, project events, members, volumes, usage, and metrics into the PostHog Data warehouse.
+ caption="""Enter your Railway API token to pull your Railway workspaces, projects, services, service instances, environments, deployments, deployment events, deployment triggers, project events, members, volumes, volume instances, volume backups, usage, and metrics into the PostHog Data warehouse.
Create an account or workspace token in your [Railway account settings](https://railway.com/account/tokens). Project tokens are not supported — they are scoped to a single environment and cannot list your projects.
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/railway/tests/test_railway.py b/products/warehouse_sources/backend/temporal/data_imports/sources/railway/tests/test_railway.py
index cc23117241e6..4b9465fee81e 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/railway/tests/test_railway.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/railway/tests/test_railway.py
@@ -244,6 +244,114 @@ class TestRailway:
assert _batch_ids(result) == [["ev3"]]
assert len(result.requests) == 4
+ def test_service_instances_resume_at_the_saved_environment_and_stage_the_next_position(self):
+ result = _run(
+ "service_instances",
+ [
+ _projects_page(["p1", "p2"]),
+ _connection(("environments",), [{"id": "e1"}, {"id": "e2"}]),
+ _connection(("environment", "serviceInstances"), [{"id": "si2", "serviceId": "s1"}], end_cursor="c4"),
+ _connection(("environment", "serviceInstances"), [{"id": "si3"}]),
+ _connection(("environments",), [{"id": "e3"}]),
+ _connection(("environment", "serviceInstances"), []),
+ ],
+ resume_state=RailwayResumeConfig(project_id="p1", parent_id="e2", cursor="c3"),
+ )
+
+ assert result.raised is None
+ assert _request_variables(result, 2) == {"environmentId": "e2", "first": 100, "after": "c3"}
+ assert _request_variables(result, 3)["after"] == "c4"
+ assert _request_variables(result, 5) == {"environmentId": "e3", "first": 100, "after": None}
+ assert result.items == [
+ [{"project_id": "p1", "id": "si2", "serviceId": "s1"}],
+ [{"project_id": "p1", "id": "si3"}],
+ ]
+ # The last batch stages a completed position, so a retry after it is written reads nothing.
+ assert [
+ (state.project_id, state.parent_id, state.cursor, state.completed) for state in result.saved_states
+ ] == [
+ ("p1", "e2", "c4", False),
+ ("p2", None, None, False),
+ (None, None, None, True),
+ ]
+
+ @parameterized.expand(
+ [
+ ("completed_run", RailwayResumeConfig(completed=True), [], []),
+ (
+ "deleted_environment",
+ RailwayResumeConfig(
+ project_id="p1", project_index=0, parent_id="e-deleted", parent_index=1, cursor="c3"
+ ),
+ [_projects_page(["p1"]), _connection(("environments",), [{"id": "e1"}, {"id": "e2"}])],
+ [("e2", None)],
+ ),
+ (
+ "deleted_project",
+ RailwayResumeConfig(project_id="p-deleted", project_index=1, parent_id="e2", parent_index=1),
+ [_projects_page(["p1", "p3"]), _connection(("environments",), [{"id": "e3"}, {"id": "e4"}])],
+ [("e3", None), ("e4", None)],
+ ),
+ ]
+ )
+ def test_environment_walk_retry_reads_no_completed_environment(
+ self, _name, resume_state, responses, expected_reads
+ ):
+ result = _run(
+ "service_instances",
+ [*responses, *[_connection(("environment", "serviceInstances"), [])] * len(expected_reads)],
+ resume_state=resume_state,
+ )
+
+ assert result.raised is None
+ reads = [
+ (variables["environmentId"], variables["after"])
+ for variables in (_request_variables(result, index) for index in range(len(result.requests)))
+ if "environmentId" in variables
+ ]
+ assert reads == expected_reads
+
+ @parameterized.expand([("saved_instance", "vi2", None), ("deleted_instance", "vi-deleted", 1)])
+ def test_volume_instance_backups_resume_at_the_saved_volume_instance(self, _name, child_id, child_index):
+ result = _run(
+ "volume_instance_backups",
+ [
+ _projects_page(["p1"]),
+ _connection(("environments",), [{"id": "e1"}]),
+ _connection(
+ ("environment", "volumeInstances"),
+ [{"id": "vi1"}, {"id": "vi2", "volumeId": "v2", "serviceId": "s2"}, {"id": "vi3"}],
+ ),
+ {"data": {"volumeInstanceBackupList": [{"id": "b1"}]}},
+ {"data": {"volumeInstanceBackupList": []}},
+ ],
+ resume_state=RailwayResumeConfig(
+ project_id="p1", parent_id="e1", child_id=child_id, child_index=child_index
+ ),
+ )
+
+ assert result.raised is None
+ assert [_request_variables(result, index) for index in (3, 4)] == [
+ {"volumeInstanceId": "vi2"},
+ {"volumeInstanceId": "vi3"},
+ ]
+ assert result.items == [
+ [
+ {
+ "project_id": "p1",
+ "environment_id": "e1",
+ "volume_instance_id": "vi2",
+ "volume_id": "v2",
+ "service_id": "s2",
+ "id": "b1",
+ }
+ ]
+ ]
+ assert [(state.child_id, state.child_index, state.completed) for state in result.saved_states] == [
+ ("vi3", 2, False),
+ (None, None, True),
+ ]
+
@time_machine.travel("2024-07-03T12:00:00Z", tick=False)
def test_usage_reads_one_day_window_per_workspace_from_the_watermark_day(self):
result = _run(
@@ -376,6 +484,10 @@ class TestRailway:
assert deployment_events is not None
assert deployment_events.primary_keys == ["deployment_id", "id"]
+ backups = _run("volume_instance_backups", [_projects_page([])]).response
+ assert backups is not None
+ assert backups.primary_keys == ["volume_instance_id", "id"]
+
@parameterized.expand(
[
({"data": {"projects": {"edges": []}}}, True, None),