Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(data-warehouse): add 4 missing tables to the Spacelift source (#118918)

Tom Owersc799ede0f5e4
diff --git a/posthog/test/repo_invariants/dataclass_frozen_baseline.txt b/posthog/test/repo_invariants/dataclass_frozen_baseline.txt
index c8bed2d451b7..e5a9e9532aa8 100644
--- a/posthog/test/repo_invariants/dataclass_frozen_baseline.txt
+++ b/posthog/test/repo_invariants/dataclass_frozen_baseline.txt
@@ -1069,8 +1069,6 @@
1 products/warehouse_sources/backend/temporal/data_imports/sources/sonatype_nexus/sonatype_nexus.py
1 products/warehouse_sources/backend/temporal/data_imports/sources/sourcegraph/settings.py
1 products/warehouse_sources/backend/temporal/data_imports/sources/sourcegraph/sourcegraph.py
-1 products/warehouse_sources/backend/temporal/data_imports/sources/spacelift/settings.py
-1 products/warehouse_sources/backend/temporal/data_imports/sources/spacelift/spacelift.py
1 products/warehouse_sources/backend/temporal/data_imports/sources/sparkpost/settings.py
1 products/warehouse_sources/backend/temporal/data_imports/sources/sparkpost/sparkpost.py
1 products/warehouse_sources/backend/temporal/data_imports/sources/split_io/settings.py
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
index edbda4711798..cb4c86482fe1 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
@@ -11451,10 +11451,10 @@ Today (8): `contexts`, `managed_entities`, `modules`, `policies`, `runs`, `space
Diffed against: <https://docs.spacelift.io/integrations/api>
-- [ ] `searchAuditTrailEntries` — full who-did-what-when history across stacks, runs and policies; the state-transition table for everything already synced (high)
-- [ ] `run.changesV (per-run resource changes)` — add/change/delete deltas per resource for each run - the core IaC change metric; runs are synced but their change breakdown is not (high)
-- [ ] `searchEvaluationRecords` — policy evaluation records per run; joins to the policies table we already sync and shows which decisions actually fired (high)
-- [ ] `searchModuleVersions` — module registry version history with test results and consumers; modules are synced but their versions are not (high)
+- [x] `searchAuditTrailEntries` — full who-did-what-when history across stacks, runs and policies; the state-transition table for everything already synced (high)
+- [x] `run.changesV (per-run resource changes)` — add/change/delete deltas per resource for each run - the core IaC change metric; runs are synced but their change breakdown is not (high)
+- [x] `searchEvaluationRecords` — policy evaluation records per run; joins to the policies table we already sync and shows which decisions actually fired (high)
+- [x] `searchModuleVersions` — module registry version history with test results and consumers; modules are synced but their versions are not (high)
- [ ] `workerPool.workers (and busyWorkers)` — individual worker state and utilization behind the worker_pools already synced; capacity and queueing analysis (medium)
- [ ] `stack.dependsOn / isDependedOnBy` — stack dependency graph edges; needed to model blast radius and deployment ordering (medium)
- [ ] `stack.tasks` — one-off tasks executed against a stack, a run class not covered by the runs table (medium)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/spacelift/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/spacelift/canonical_descriptions.py
index 52153a5319b7..4e928595ea91 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/spacelift/canonical_descriptions.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/spacelift/canonical_descriptions.py
@@ -151,4 +151,71 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {
"updater": "Run that last updated the resource.",
},
},
+ "audit_trail_entries": {
+ "description": "Audit trail of changes made in the Spacelift account: who created, updated, or deleted which stack, policy, run, or other resource, and when.",
+ "docs_url": "https://docs.spacelift.io/integrations/audit-trail",
+ "columns": {
+ "id": "Unique identifier of the audit trail entry.",
+ "action": "Action that was performed.",
+ "event_type": "Kind of change: CREATE, UPDATE, DELETE, NONE, or UNKNOWN.",
+ "body": "JSON body of the action, with the details of the change.",
+ "created_at": "Unix timestamp (seconds) when the action was performed.",
+ "updated_at": "Unix timestamp (seconds) when the entry was last updated.",
+ "remote_ip": "IP address the action came from.",
+ "identity_ulid": "ULID of the identity that performed the action.",
+ "actor": "User or run that performed the action.",
+ "affected_resource": "Type and ID of the resource the action changed.",
+ "related_resource": "Type and ID of a resource related to the action, if any.",
+ },
+ },
+ "run_changes": {
+ "description": "Per-resource changes planned or applied by each run: one row per resource address, phase, and change type.",
+ "docs_url": "https://docs.spacelift.io/concepts/run",
+ "columns": {
+ "run_id": "ID of the run that planned or applied the change.",
+ "stack_id": "ID of the stack (or module) the run belongs to.",
+ "is_module": "Whether the run belongs to a module instead of a stack.",
+ "run_created_at": "Unix timestamp (seconds) when the run was created.",
+ "phase": "Run phase the change belongs to: DRIFT, PLAN, or APPLY.",
+ "change_type": "Kind of change to the resource (e.g. ADD, CHANGE, DELETE, REPLACE_CREATE_BEFORE_DESTROY, NOOP).",
+ "address": "Address of the changed resource (e.g. aws_instance.web).",
+ "entity_id": "ID of the managed entity that changed.",
+ "entity_name": "Name of the changed resource.",
+ "entity_type": "Resource type (e.g. aws_instance).",
+ "parent": "Address of the parent module the resource belongs to.",
+ },
+ },
+ "policy_evaluation_records": {
+ "description": "Records of policy evaluations, one row per evaluation of each policy, with its outcome.",
+ "docs_url": "https://docs.spacelift.io/concepts/policy",
+ "columns": {
+ "policy_id": "ID of the policy that was evaluated.",
+ "key": "Key of the policy input sample for this evaluation.",
+ "outcome": "Outcome of evaluating the policy.",
+ "timestamp": "Unix timestamp (seconds) of the evaluation.",
+ },
+ },
+ "module_versions": {
+ "description": "Versions published for each module in the Spacelift module registry, with their test state and consumers.",
+ "docs_url": "https://docs.spacelift.io/vendors/terraform/module-registry",
+ "columns": {
+ "id": "Unique identifier of the module version.",
+ "module_id": "ID of the module the version belongs to.",
+ "number": "Semantic version number (e.g. 1.2.3).",
+ "state": "State of the version: ACTIVE, PENDING (tests running), or FAILED.",
+ "created_at": "Unix timestamp (seconds) when the version was created.",
+ "last_pulled_at": "Unix timestamp (seconds) when the version was last downloaded.",
+ "triggered_by": "User or process that triggered the version.",
+ "source_url": "URL of the version's source code.",
+ "download_link": "Link to download the version, if available.",
+ "notes": "Release notes for the version.",
+ "yanked": "Whether the version was yanked from the registry.",
+ "yanked_by": "User who yanked the version.",
+ "yank_note": "Reason given for yanking the version.",
+ "consumer_count": "Number of stacks that use this version.",
+ "commit": "VCS commit the version was built from.",
+ "module": "ID, name, and owner of the module.",
+ "consumers": "Stacks that use this version.",
+ },
+ },
}
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/spacelift/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/spacelift/settings.py
index c03d16105cac..74f200e90751 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/spacelift/settings.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/spacelift/settings.py
@@ -18,6 +18,12 @@ CREATED_AT = "createdAt"
# state changes onto the primary key. Runs typically settle within hours; a week
# covers long-lived approvals and drift-detection reruns without re-reading history.
RUNS_INCREMENTAL_LOOKBACK_SECONDS = 7 * 24 * 60 * 60
+# Audit entries never change, so the overlap only covers entries written in the same
+# second as the last synced one while the previous sync was reading.
+AUDIT_TRAIL_INCREMENTAL_LOOKBACK_SECONDS = 60 * 60
+
+RUN_CREATED_AT = "runCreatedAt"
+TIMESTAMP = "timestamp"
INCREMENTAL_CREATED_AT_FIELDS: list[IncrementalField] = [
{
@@ -28,8 +34,28 @@ INCREMENTAL_CREATED_AT_FIELDS: list[IncrementalField] = [
},
]
+INCREMENTAL_RUN_CREATED_AT_FIELDS: list[IncrementalField] = [
+ {
+ "label": RUN_CREATED_AT,
+ "type": IncrementalFieldType.Integer,
+ "field": RUN_CREATED_AT,
+ "field_type": IncrementalFieldType.Integer,
+ },
+]
+
+
+@dataclass(frozen=True)
+class SpaceliftParentConfig:
+ # Connection that lists the parents, e.g. "searchModules".
+ search_field: str
+ # Single-object query field that scopes the child connection, e.g. "module".
+ lookup_field: str
+ # Row column the parent's id is written to.
+ id_column: str
+ page_size: int = SPACELIFT_PAGE_SIZE
-@dataclass
+
+@dataclass(frozen=True)
class SpaceliftEndpointConfig:
name: str
# Top-level GraphQL field, e.g. "searchStacks". Fields prefixed "search" are
@@ -42,8 +68,17 @@ class SpaceliftEndpointConfig:
# searchRuns nodes are `RunWithStack { run {...} stack {...} }` rather than a
# flat object; the transport flattens them into run rows with stack context.
flatten_run_with_stack: bool = False
+ # Child connections (e.g. a module's versions) are queried once per parent row.
+ parent: SpaceliftParentConfig | None = None
+ # Per-resource changes are selected inside searchRuns and flattened to one row per change.
+ flatten_run_changes: bool = False
+ order_by: dict[str, str] | None = None
+ # API field the incremental predicate filters on, when the row column has another name.
+ incremental_predicate_field: str | None = None
+ incremental_lookback_seconds: int = RUNS_INCREMENTAL_LOOKBACK_SECONDS
primary_keys: list[str] = field(default_factory=lambda: ["id"])
- # Rows per page. Most connections accept 100; searchModules rejects anything over 50.
+ # Rows per page. Most connections accept 100; searchModules, searchModuleVersions,
+ # searchEvaluationRecords and searchAuditTrailEntries reject anything over 50.
page_size: int = SPACELIFT_PAGE_SIZE
incremental_fields: list[IncrementalField] = field(default_factory=list)
partition_key: str | None = None
@@ -195,6 +230,88 @@ updater {
}
"""
+AUDIT_TRAIL_ENTRY_SELECTION = """
+id
+action
+eventType
+body
+createdAt
+updatedAt
+remoteIP
+identityUlid
+actor {
+ username
+ run {
+ id
+ stackId
+ }
+}
+affectedResource {
+ resourceType
+ resourceId
+}
+relatedResource {
+ resourceType
+ resourceId
+}
+"""
+
+# Selected under each searchRuns node as `run { id createdAt changesV2(...) { ... } }`.
+# changesV2 works for every vendor; changesV3 and searchEntityChanges error outside Terraform.
+RUN_CHANGE_SELECTION = """
+type
+phase
+entity {
+ id
+ address
+ name
+ type
+ parent
+}
+"""
+RUN_CHANGES_PAGE_SIZE = 50
+
+POLICY_EVALUATION_RECORD_SELECTION = """
+key
+outcome
+timestamp
+"""
+
+MODULE_VERSION_SELECTION = """
+id
+number
+state
+createdAt
+lastPulledAt
+triggeredBy
+sourceURL
+downloadLink
+notes
+yanked
+yankedBy
+yankNote
+consumerCount
+versionCount
+commit {
+ authorLogin
+ authorName
+ hash
+ message
+ timestamp
+ url
+}
+module {
+ id
+ name
+ owner
+}
+consumers {
+ id
+}
+"""
+
+CREATED_AT_DESC = {"field": CREATED_AT, "direction": "DESC"}
+
SPACELIFT_ENDPOINTS: dict[str, SpaceliftEndpointConfig] = {
"stacks": SpaceliftEndpointConfig(
name="stacks",
@@ -251,6 +368,61 @@ SPACELIFT_ENDPOINTS: dict[str, SpaceliftEndpointConfig] = {
node_selection=MANAGED_ENTITY_SELECTION,
primary_keys=["stackId", "id"],
),
+ "audit_trail_entries": SpaceliftEndpointConfig(
+ name="audit_trail_entries",
+ graphql_field="searchAuditTrailEntries",
+ node_selection=AUDIT_TRAIL_ENTRY_SELECTION,
+ page_size=50,
+ order_by=CREATED_AT_DESC,
+ incremental_fields=INCREMENTAL_CREATED_AT_FIELDS,
+ incremental_lookback_seconds=AUDIT_TRAIL_INCREMENTAL_LOOKBACK_SECONDS,
+ partition_key=CREATED_AT,
+ partition_mode="datetime",
+ partition_format="month",
+ sort_mode="desc",
+ ),
+ # One row per resource change of each run. Changes appear once a run plans or applies,
+ # so incremental syncs re-read the same trailing window of runs as the runs table.
+ "run_changes": SpaceliftEndpointConfig(
+ name="run_changes",
+ graphql_field="searchRuns",
+ node_selection=RUN_CHANGE_SELECTION,
+ flatten_run_changes=True,
+ # Each run in the page carries up to RUN_CHANGES_PAGE_SIZE nested changes.
+ page_size=20,
+ primary_keys=["runId", "phase", "address"],
+ incremental_fields=INCREMENTAL_RUN_CREATED_AT_FIELDS,
+ incremental_predicate_field=CREATED_AT,
+ partition_key=RUN_CREATED_AT,
+ partition_mode="datetime",
+ partition_format="month",
+ sort_mode="desc",
+ ),
+ "policy_evaluation_records": SpaceliftEndpointConfig(
+ name="policy_evaluation_records",
+ graphql_field="searchEvaluationRecords",
+ node_selection=POLICY_EVALUATION_RECORD_SELECTION,
+ parent=SpaceliftParentConfig(search_field="searchPolicies", lookup_field="policy", id_column="policyId"),
+ page_size=50,
+ primary_keys=["policyId", "key"],
+ partition_key=TIMESTAMP,
+ partition_mode="datetime",
+ partition_format="month",
+ ),
+ "module_versions": SpaceliftEndpointConfig(
+ name="module_versions",
+ graphql_field="searchModuleVersions",
+ node_selection=MODULE_VERSION_SELECTION,
+ parent=SpaceliftParentConfig(
+ search_field="searchModules", lookup_field="module", id_column="moduleId", page_size=50
+ ),
+ page_size=50,
+ order_by=CREATED_AT_DESC,
+ primary_keys=["moduleId", "id"],
+ partition_key=CREATED_AT,
+ partition_mode="datetime",
+ partition_format="month",
+ ),
}
ENDPOINTS = tuple(SPACELIFT_ENDPOINTS.keys())
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/spacelift/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/spacelift/source.py
index a15549e4c4f4..ff9b82a69473 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/spacelift/source.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/spacelift/source.py
@@ -122,6 +122,11 @@ Your account name is the subdomain you use to access Spacelift (e.g. `my-company
"All runs across every stack and module. Incremental syncs re-pull the last "
f"{RUNS_INCREMENTAL_LOOKBACK_SECONDS // 86400} days so state changes on recent runs are captured"
)
+ if endpoint == "run_changes":
+ return (
+ "Per-resource changes of each run. Incremental syncs re-pull the runs of the last "
+ f"{RUNS_INCREMENTAL_LOOKBACK_SECONDS // 86400} days so changes from runs that finish later are captured"
+ )
if endpoint == "managed_entities":
return "Terraform/OpenTofu resources managed across your stacks, including drift status"
return None
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/spacelift/spacelift.py b/products/warehouse_sources/backend/temporal/data_imports/sources/spacelift/spacelift.py
index 9a416f0e85a5..f7207637ef2f 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/spacelift/spacelift.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/spacelift/spacelift.py
@@ -13,10 +13,13 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.htt
from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
from products.warehouse_sources.backend.temporal.data_imports.sources.spacelift.settings import (
+ RUN_CHANGE_SELECTION,
+ RUN_CHANGES_PAGE_SIZE,
RUNS_INCREMENTAL_LOOKBACK_SECONDS,
SPACELIFT_ENDPOINTS,
SPACELIFT_HOST_TEMPLATE,
SpaceliftEndpointConfig,
+ SpaceliftParentConfig,
)
# Spacelift publishes no rate limits; ride out transient 429/5xx bursts in-process
@@ -65,10 +68,16 @@ class SpaceliftPermissionError(Exception):
pass
[email protected](frozen=True)
class SpaceliftResumeConfig:
- # Relay `after` cursor of the next page to fetch for the current endpoint.
+ # Relay `after` cursor of the next page to fetch for the current endpoint. For
+ # per-parent endpoints it is the cursor of the parent page being walked.
cursor: str
+ # Parents on that parent page whose child rows are all synced.
+ completed_parent_ids: list[str] = dataclasses.field(default_factory=list)
+ # Parent whose child rows are partly synced, and the cursor of its next child page.
+ child_parent_id: str | None = None
+ child_cursor: str | None = None
def normalize_account_name(account_name: str) -> str:
@@ -113,6 +122,79 @@ query Search($input: SearchInput!) {{
"""
+def _connection_query(operation: str, field_call: str, node_selection: str) -> str:
+ return f"""
+query {operation} {{
+ {field_call} {{
+ edges {{
+ cursor
+ node {{ {node_selection} }}
+ }}
+ pageInfo {{
+ endCursor
+ hasNextPage
+ }}
+ }}
+}}
+"""
+
+
+def build_run_changes_query() -> str:
+ node_selection = f"""
+isModule
+stack {{ id }}
+run {{
+ id
+ createdAt
+ changesV2(input: $changesInput) {{
+ edges {{ node {{ {RUN_CHANGE_SELECTION} }} }}
+ pageInfo {{ endCursor hasNextPage }}
+ }}
+}}
+"""
+ return _connection_query(
+ "SearchRunChanges($input: SearchInput!, $changesInput: EntityChangeCollectionInputV2!)",
+ "searchRuns(input: $input)",
+ node_selection,
+ )
+
+
+RUN_CHANGES_OVERFLOW_QUERY = f"""
+query RunChanges($id: ID!, $changesInput: EntityChangeCollectionInputV2!) {{
+ run(id: $id) {{
+ changesV2(input: $changesInput) {{
+ edges {{ node {{ {RUN_CHANGE_SELECTION} }} }}
+ pageInfo {{ endCursor hasNextPage }}
+ }}
+ }}
+}}
+"""
+
+
+def build_parent_query(parent: SpaceliftParentConfig) -> str:
+ return _connection_query("SearchParents($input: SearchInput!)", f"{parent.search_field}(input: $input)", "id")
+
+
+def build_child_query(config: SpaceliftEndpointConfig) -> str:
+ assert config.parent is not None
+ return f"""
+query SearchChildren($id: ID!, $input: SearchInput!) {{
+ {config.parent.lookup_field}(id: $id) {{
+ {config.graphql_field}(input: $input) {{
+ edges {{
+ cursor
+ node {{ {config.node_selection} }}
+ }}
+ pageInfo {{
+ endCursor
+ hasNextPage
+ }}
+ }}
+ }}
+}}
+"""
+
+
def to_unix_seconds(value: Any) -> int | None:
"""Coerce an incremental watermark into the Unix-seconds Int Spacelift timestamps use."""
if value is None:
@@ -138,7 +220,9 @@ def to_unix_seconds(value: Any) -> int | None:
return None
-def build_incremental_predicates(incremental_field: str, last_value: Any) -> list[dict[str, Any]] | None:
+def build_incremental_predicates(
+ incremental_field: str, last_value: Any, lookback_seconds: int = RUNS_INCREMENTAL_LOOKBACK_SECONDS
+) -> list[dict[str, Any]] | None:
"""Build the server-side `timeInRange` predicate for an incremental sync.
Mirrors the filter the Spacelift UI itself sends to `searchRuns`. The watermark is
@@ -150,7 +234,7 @@ def build_incremental_predicates(incremental_field: str, last_value: Any) -> lis
start = to_unix_seconds(last_value)
if start is None:
return None
- start = max(0, start - RUNS_INCREMENTAL_LOOKBACK_SECONDS)
+ start = max(0, start - lookback_seconds)
return [{"field": incremental_field, "constraint": {"timeInRange": {"start": start}}}]
@@ -276,6 +360,203 @@ def _normalize_run_node(node: dict[str, Any]) -> dict[str, Any]:
return row
+def _next_cursor(page_info: dict[str, Any], name: str) -> str | None:
+ if not page_info.get("hasNextPage"):
+ return None
+ end_cursor = page_info.get("endCursor")
+ if not end_cursor:
+ # A hasNextPage=True page with no cursor would loop forever on itself;
+ # fail loudly instead of silently returning partial results.
+ raise Exception(f"Spacelift: hasNextPage=True but endCursor is empty for {name}")
+ return end_cursor
+
+
+def _build_search_input(
+ endpoint_config: SpaceliftEndpointConfig,
+ logger: FilteringBoundLogger | None = None,
+ should_use_incremental_field: bool = False,
+ db_incremental_field_last_value: Any = None,
+ incremental_field: str | None = None,
+) -> dict[str, Any]:
+ search_input: dict[str, Any] = {"first": endpoint_config.page_size, "after": None}
+ if endpoint_config.order_by:
+ search_input["orderBy"] = endpoint_config.order_by
+
+ if should_use_incremental_field and incremental_field:
+ predicates = build_incremental_predicates(
+ endpoint_config.incremental_predicate_field or incremental_field,
+ db_incremental_field_last_value,
+ endpoint_config.incremental_lookback_seconds,
+ )
+ if predicates:
+ search_input["predicates"] = predicates
+ if logger is not None:
+ logger.debug(f"Spacelift: incremental sync for {endpoint_config.name} with predicates {predicates}")
+
+ return search_input
+
+
+def _normalize_run_change(run: dict[str, Any], stack_id: Any, is_module: Any, change: dict[str, Any]) -> dict[str, Any]:
+ entity = change.get("entity") or {}
+ return {
+ "runId": run.get("id"),
+ "stackId": stack_id,
+ "isModule": is_module,
+ "runCreatedAt": run.get("createdAt"),
+ "phase": change.get("phase"),
+ "changeType": change.get("type"),
+ "address": entity.get("address"),
+ "entityId": entity.get("id"),
+ "entityName": entity.get("name"),
+ "entityType": entity.get("type"),
+ "parent": entity.get("parent"),
+ }
+
+
+def _collect_run_changes(client: SpaceliftClient, node: dict[str, Any]) -> list[dict[str, Any]]:
+ run = node.get("run") or {}
+ stack_id = (node.get("stack") or {}).get("id")
+ changes = run.get("changesV2") or {}
+ rows: list[dict[str, Any]] = []
+
+ while True:
+ for edge in changes.get("edges") or []:
+ change = edge.get("node")
+ if change is not None:
+ rows.append(_normalize_run_change(run, stack_id, node.get("isModule"), change))
+
+ # Runs with more changes than fit in the nested page are finished with per-run queries.
+ after = _next_cursor(changes.get("pageInfo") or {}, f"run_changes of run {run.get('id')}")
+ if after is None:
+ return rows
+ data = client.execute(
+ RUN_CHANGES_OVERFLOW_QUERY,
+ {"id": run.get("id"), "changesInput": {"first": RUN_CHANGES_PAGE_SIZE, "after": after}},
+ )
+ changes = (data.get("run") or {}).get("changesV2") or {}
+
+
+def _get_run_change_rows(
+ client: SpaceliftClient,
+ endpoint_config: SpaceliftEndpointConfig,
+ resumable_source_manager: ResumableSourceManager[SpaceliftResumeConfig],
+ search_input: dict[str, Any],
+) -> Iterator[list[dict[str, Any]]]:
+ query = build_run_changes_query()
+
+ resume_config = resumable_source_manager.load_state() if resumable_source_manager.can_resume() else None
+ if resume_config is not None and resume_config.cursor:
+ search_input["after"] = resume_config.cursor
+
+ while True:
+ data = client.execute(query, {"input": search_input, "changesInput": {"first": RUN_CHANGES_PAGE_SIZE}})
+ connection = data.get(endpoint_config.graphql_field) or {}
+
+ rows: list[dict[str, Any]] = []
+ for edge in connection.get("edges") or []:
+ rows.extend(_collect_run_changes(client, edge.get("node") or {}))
+
+ end_cursor = _next_cursor(connection.get("pageInfo") or {}, endpoint_config.name)
+ if end_cursor is not None:
+ resumable_source_manager.save_state(SpaceliftResumeConfig(cursor=end_cursor))
+
+ if rows:
+ yield rows
+ else:
+ # Tasks and failed runs have no changes, so whole pages can come back empty.
+ resumable_source_manager.safe_point()
+
+ if end_cursor is None:
+ break
+ search_input["after"] = end_cursor
+
+
+def _iter_parent_pages(
+ client: SpaceliftClient, parent: SpaceliftParentConfig, after: str | None, name: str
+) -> Iterator[tuple[str | None, list[str]]]:
+ """Yield `(cursor used to fetch the page, parent ids on the page)` for each parent page."""
+ query = build_parent_query(parent)
+ while True:
+ data = client.execute(query, {"input": {"first": parent.page_size, "after": after}})
+ connection = data.get(parent.search_field) or {}
+ parent_ids = [(edge.get("node") or {}).get("id") for edge in connection.get("edges") or [] if edge.get("node")]
+ yield after, [parent_id for parent_id in parent_ids if parent_id]
+
+ after = _next_cursor(connection.get("pageInfo") or {}, f"{name} parents")
+ if after is None:
+ return
+
+
+def _iter_child_pages(
+ client: SpaceliftClient, endpoint_config: SpaceliftEndpointConfig, parent_id: str, after: str | None
+) -> Iterator[tuple[list[dict[str, Any]], str | None]]:
+ """Yield `(rows, cursor of the next child page or None)` for each child page of one parent."""
+ assert endpoint_config.parent is not None
+ query = build_child_query(endpoint_config)
+ search_input = _build_search_input(endpoint_config)
+ if after:
+ search_input["after"] = after
+
+ while True:
+ data = client.execute(query, {"id": parent_id, "input": search_input})
+ # A parent deleted after it was listed resolves to null.
+ connection = (data.get(endpoint_config.parent.lookup_field) or {}).get(endpoint_config.graphql_field) or {}
+ rows = [
+ {**edge["node"], endpoint_config.parent.id_column: parent_id}
+ for edge in connection.get("edges") or []
+ if edge.get("node") is not None
+ ]
+
+ after = _next_cursor(connection.get("pageInfo") or {}, endpoint_config.name)
+ yield rows, after
+ if after is None:
+ return
+ search_input["after"] = after
+
+
+def _get_child_rows(
+ client: SpaceliftClient,
+ endpoint_config: SpaceliftEndpointConfig,
+ resumable_source_manager: ResumableSourceManager[SpaceliftResumeConfig],
+) -> Iterator[list[dict[str, Any]]]:
+ assert endpoint_config.parent is not None
+
+ resume_config = resumable_source_manager.load_state() if resumable_source_manager.can_resume() else None
+ completed_parent_ids = list(resume_config.completed_parent_ids) if resume_config is not None else []
+ child_parent_id = resume_config.child_parent_id if resume_config is not None else None
+ child_cursor = resume_config.child_cursor if resume_config is not None else None
+ start_cursor = resume_config.cursor if resume_config is not None and resume_config.cursor else None
+
+ for page_cursor, parent_ids in _iter_parent_pages(
+ client, endpoint_config.parent, start_cursor, endpoint_config.name
+ ):
+ for parent_id in parent_ids:
+ if parent_id in completed_parent_ids:
+ continue
+
+ after = child_cursor if parent_id == child_parent_id else None
+ for rows, next_after in _iter_child_pages(client, endpoint_config, parent_id, after):
+ if next_after is None:
+ completed_parent_ids.append(parent_id)
+ resumable_source_manager.save_state(
+ SpaceliftResumeConfig(
+ cursor=page_cursor or "",
+ completed_parent_ids=list(completed_parent_ids),
+ child_parent_id=parent_id if next_after is not None else None,
+ child_cursor=next_after,
+ )
+ )
+ if rows:
+ yield rows
+ else:
+ resumable_source_manager.safe_point()
+
+ # The saved progress only applies to the parent page it was recorded on.
+ completed_parent_ids = []
+ child_parent_id = None
+ child_cursor = None
+
+
def _get_rows(
client: SpaceliftClient,
endpoint_config: SpaceliftEndpointConfig,
@@ -295,13 +576,17 @@ def _get_rows(
yield rows
return
- search_input: dict[str, Any] = {"first": endpoint_config.page_size, "after": None}
+ if endpoint_config.parent is not None:
+ yield from _get_child_rows(client, endpoint_config, resumable_source_manager)
+ return
- if should_use_incremental_field and incremental_field:
- predicates = build_incremental_predicates(incremental_field, db_incremental_field_last_value)
- if predicates:
- search_input["predicates"] = predicates
- logger.debug(f"Spacelift: incremental sync for {endpoint_config.name} with predicates {predicates}")
+ search_input = _build_search_input(
+ endpoint_config, logger, should_use_incremental_field, db_incremental_field_last_value, incremental_field
+ )
+
+ if endpoint_config.flatten_run_changes:
+ yield from _get_run_change_rows(client, endpoint_config, resumable_source_manager, search_input)
+ return
resume_config = resumable_source_manager.load_state() if resumable_source_manager.can_resume() else None
if resume_config is not None and resume_config.cursor:
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/spacelift/tests/test_spacelift.py b/products/warehouse_sources/backend/temporal/data_imports/sources/spacelift/tests/test_spacelift.py
index af5e33954511..cc9b4b2d1e5e 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/spacelift/tests/test_spacelift.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/spacelift/tests/test_spacelift.py
@@ -14,6 +14,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.generated_
SpaceliftSourceConfig,
)
from products.warehouse_sources.backend.temporal.data_imports.sources.spacelift.settings import (
+ AUDIT_TRAIL_INCREMENTAL_LOOKBACK_SECONDS,
RUNS_INCREMENTAL_LOOKBACK_SECONDS,
)
from products.warehouse_sources.backend.temporal.data_imports.sources.spacelift.source import SpaceliftSource
@@ -63,6 +64,61 @@ def _run(
return SourceDriver(SpaceliftSource(), config).run(endpoint, script, resume_state=resume_state, **inputs)
+def _module_page(module_ids: list[str], end_cursor: str = "", has_next: bool = False) -> ScriptedResponse:
+ return _search_page("searchModules", [{"id": module_id} for module_id in module_ids], end_cursor, has_next)
+
+
+def _versions_page(version_ids: list[str] | None, end_cursor: str = "", has_next: bool = False) -> ScriptedResponse:
+ if version_ids is None:
+ # The module was deleted between listing it and querying its versions.
+ return _response({"data": {"module": None}})
+ return _response(
+ {
+ "data": {
+ "module": {
+ "searchModuleVersions": {
+ "edges": [{"cursor": v, "node": {"id": v}} for v in version_ids],
+ "pageInfo": {"endCursor": end_cursor, "hasNextPage": has_next},
+ }
+ }
+ }
+ }
+ )
+
+
+def _changes(addresses: list[str], end_cursor: str = "", has_next: bool = False) -> dict[str, Any]:
+ return {
+ "edges": [
+ {"node": {"type": "ADD", "phase": "PLAN", "entity": {"id": f"ent-{a}", "address": a}}} for a in addresses
+ ],
+ "pageInfo": {"endCursor": end_cursor, "hasNextPage": has_next},
+ }
+
+
+def _run_node(run_id: str, changes: dict[str, Any] | None) -> dict[str, Any]:
+ return {
+ "isModule": False,
+ "stack": {"id": "stack-1"},
+ "run": {"id": run_id, "createdAt": 1700000100, "changesV2": changes},
+ }
+
+
+def _change_row(run_id: str, address: str) -> dict[str, Any]:
+ return {
+ "runId": run_id,
+ "stackId": "stack-1",
+ "isModule": False,
+ "runCreatedAt": 1700000100,
+ "phase": "PLAN",
+ "changeType": "ADD",
+ "address": address,
+ "entityId": f"ent-{address}",
+ "entityName": None,
+ "entityType": None,
+ "parent": None,
+ }
+
+
def _query_requests(result: DriveResult) -> list[RecordedRequest]:
# Skip the initial apiKeyUser token exchange, leaving the data queries.
return result.requests[1:]
@@ -180,22 +236,155 @@ class TestSpacelift:
]
]
- def test_incremental_sync_sends_time_range_predicate(self):
+ @pytest.mark.parametrize(
+ "endpoint, graphql_field, incremental_field, lookback, order_by",
+ [
+ ("runs", "searchRuns", "createdAt", RUNS_INCREMENTAL_LOOKBACK_SECONDS, None),
+ (
+ "audit_trail_entries",
+ "searchAuditTrailEntries",
+ "createdAt",
+ AUDIT_TRAIL_INCREMENTAL_LOOKBACK_SECONDS,
+ {"field": "createdAt", "direction": "DESC"},
+ ),
+ # The row column is runCreatedAt, but searchRuns only filters on createdAt.
+ ("run_changes", "searchRuns", "runCreatedAt", RUNS_INCREMENTAL_LOOKBACK_SECONDS, None),
+ ],
+ )
+ def test_incremental_sync_sends_time_range_predicate(
+ self, endpoint, graphql_field, incremental_field, lookback, order_by
+ ):
result = _run(
- "runs",
- [_response(TOKEN_PAYLOAD), _search_page("searchRuns", [])],
+ endpoint,
+ [_response(TOKEN_PAYLOAD), _search_page(graphql_field, [])],
should_use_incremental_field=True,
db_incremental_field_last_value=1700000000,
- incremental_field="createdAt",
+ incremental_field=incremental_field,
)
+ assert result.raised is None
sent_input = _query_requests(result)[0].json()["variables"]["input"]
assert sent_input["predicates"] == [
- {
- "field": "createdAt",
- "constraint": {"timeInRange": {"start": 1700000000 - RUNS_INCREMENTAL_LOOKBACK_SECONDS}},
- }
+ {"field": "createdAt", "constraint": {"timeInRange": {"start": 1700000000 - lookback}}}
+ ]
+ assert sent_input.get("orderBy") == order_by
+
+ def test_run_changes_flatten_nested_changes_and_fetch_overflow_per_run(self):
+ result = _run(
+ "run_changes",
+ [
+ _response(TOKEN_PAYLOAD),
+ _search_page(
+ "searchRuns",
+ [
+ _run_node("run-1", _changes(["aws_s3_bucket.a"], end_cursor="ch-1", has_next=True)),
+ _run_node("run-2", None),
+ ],
+ end_cursor="runs-1",
+ has_next=True,
+ ),
+ _response({"data": {"run": {"changesV2": _changes(["aws_s3_bucket.b"])}}}),
+ # A page of runs with no changes yields nothing but the walk continues.
+ _search_page("searchRuns", [_run_node("run-3", _changes([]))], end_cursor="runs-2", has_next=True),
+ _search_page("searchRuns", [_run_node("run-4", _changes(["aws_iam_role.c"]))]),
+ ],
+ )
+
+ assert result.raised is None
+ assert result.items == [
+ [_change_row("run-1", "aws_s3_bucket.a"), _change_row("run-1", "aws_s3_bucket.b")],
+ [_change_row("run-4", "aws_iam_role.c")],
+ ]
+ requests = _query_requests(result)
+ overflow = requests[1].json()["variables"]
+ assert overflow["id"] == "run-1"
+ assert overflow["changesInput"]["after"] == "ch-1"
+ assert [r.json()["variables"].get("input", {}).get("after") for r in requests] == [
+ None,
+ None,
+ "runs-1",
+ "runs-2",
+ ]
+ assert result.saved_states == [
+ SpaceliftResumeConfig(cursor="runs-1"),
+ SpaceliftResumeConfig(cursor="runs-2"),
+ ]
+
+ def test_child_endpoint_walks_every_parent_and_child_page(self):
+ result = _run(
+ "module_versions",
+ [
+ _response(TOKEN_PAYLOAD),
+ _module_page(["mod-a", "mod-b"], end_cursor="mods-1", has_next=True),
+ _versions_page(["v1"], end_cursor="v-1", has_next=True),
+ _versions_page(["v2"]),
+ _versions_page(None),
+ _module_page(["mod-c"]),
+ _versions_page(["v3"]),
+ ],
+ )
+
+ assert result.raised is None
+ assert result.items == [
+ [{"id": "v1", "moduleId": "mod-a"}],
+ [{"id": "v2", "moduleId": "mod-a"}],
+ [{"id": "v3", "moduleId": "mod-c"}],
]
+ requests = _query_requests(result)
+ assert [r.json()["variables"].get("id") for r in requests] == [
+ None,
+ "mod-a",
+ "mod-a",
+ "mod-b",
+ None,
+ "mod-c",
+ ]
+ assert requests[2].json()["variables"]["input"]["after"] == "v-1"
+ assert requests[4].json()["variables"]["input"]["after"] == "mods-1"
+ assert result.saved_states == [
+ SpaceliftResumeConfig(cursor="", child_parent_id="mod-a", child_cursor="v-1"),
+ SpaceliftResumeConfig(cursor="", completed_parent_ids=["mod-a"]),
+ SpaceliftResumeConfig(cursor="", completed_parent_ids=["mod-a", "mod-b"]),
+ SpaceliftResumeConfig(cursor="mods-1", completed_parent_ids=["mod-c"]),
+ ]
+
+ @pytest.mark.parametrize(
+ "resume_state, expected_requests",
+ [
+ (
+ SpaceliftResumeConfig(cursor="mods-1", completed_parent_ids=["mod-a"]),
+ [("mod-b", None)],
+ ),
+ # Parents are skipped by id, so a completed parent that no longer exists changes nothing.
+ (
+ SpaceliftResumeConfig(cursor="mods-1", completed_parent_ids=["mod-a", "mod-deleted"]),
+ [("mod-b", None)],
+ ),
+ # The parent that was partly synced continues from its saved child cursor.
+ (
+ SpaceliftResumeConfig(cursor="mods-1", child_parent_id="mod-a", child_cursor="v-1"),
+ [("mod-a", "v-1"), ("mod-b", None)],
+ ),
+ ],
+ )
+ def test_child_endpoint_resumes_without_repeating_finished_work(self, resume_state, expected_requests):
+ result = _run(
+ "module_versions",
+ [
+ _response(TOKEN_PAYLOAD),
+ _module_page(["mod-a", "mod-b"]),
+ *[_versions_page([f"v-{parent}"]) for parent, _ in expected_requests],
+ ],
+ resume_state=resume_state,
+ )
+
+ assert result.raised is None
+ requests = _query_requests(result)
+ assert requests[0].json()["variables"]["input"]["after"] == "mods-1"
+ assert [
+ (r.json()["variables"]["id"], r.json()["variables"]["input"].get("after")) for r in requests[1:]
+ ] == expected_requests
+ assert [row["moduleId"] for row in result.rows] == [parent for parent, _ in expected_requests]
def test_plain_list_endpoint_yields_rows_without_pagination(self):
result = _run(