:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(data-warehouse): add 4 missing tables to the Spacelift source (#118918)
Tom Owersc799ede0f5e4
diff --git a/posthog/test/repo_invariants/dataclass_frozen_baseline.txt b/posthog/test/repo_invariants/dataclass_frozen_baseline.txtindex c8bed2d451b7..e5a9e9532aa8 100644--- a/posthog/test/repo_invariants/dataclass_frozen_baseline.txt+++ b/posthog/test/repo_invariants/dataclass_frozen_baseline.txt@@ -1069,8 +1069,6 @@1 products/warehouse_sources/backend/temporal/data_imports/sources/sonatype_nexus/sonatype_nexus.py1 products/warehouse_sources/backend/temporal/data_imports/sources/sourcegraph/settings.py1 products/warehouse_sources/backend/temporal/data_imports/sources/sourcegraph/sourcegraph.py-1 products/warehouse_sources/backend/temporal/data_imports/sources/spacelift/settings.py-1 products/warehouse_sources/backend/temporal/data_imports/sources/spacelift/spacelift.py1 products/warehouse_sources/backend/temporal/data_imports/sources/sparkpost/settings.py1 products/warehouse_sources/backend/temporal/data_imports/sources/sparkpost/sparkpost.py1 products/warehouse_sources/backend/temporal/data_imports/sources/split_io/settings.pydiff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.mdindex edbda4711798..cb4c86482fe1 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md@@ -11451,10 +11451,10 @@ Today (8): `contexts`, `managed_entities`, `modules`, `policies`, `runs`, `spaceDiffed against: <https://docs.spacelift.io/integrations/api>-- [ ] `searchAuditTrailEntries` — full who-did-what-when history across stacks, runs and policies; the state-transition table for everything already synced (high)-- [ ] `run.changesV (per-run resource changes)` — add/change/delete deltas per resource for each run - the core IaC change metric; runs are synced but their change breakdown is not (high)-- [ ] `searchEvaluationRecords` — policy evaluation records per run; joins to the policies table we already sync and shows which decisions actually fired (high)-- [ ] `searchModuleVersions` — module registry version history with test results and consumers; modules are synced but their versions are not (high)+- [x] `searchAuditTrailEntries` — full who-did-what-when history across stacks, runs and policies; the state-transition table for everything already synced (high)+- [x] `run.changesV (per-run resource changes)` — add/change/delete deltas per resource for each run - the core IaC change metric; runs are synced but their change breakdown is not (high)+- [x] `searchEvaluationRecords` — policy evaluation records per run; joins to the policies table we already sync and shows which decisions actually fired (high)+- [x] `searchModuleVersions` — module registry version history with test results and consumers; modules are synced but their versions are not (high)- [ ] `workerPool.workers (and busyWorkers)` — individual worker state and utilization behind the worker_pools already synced; capacity and queueing analysis (medium)- [ ] `stack.dependsOn / isDependedOnBy` — stack dependency graph edges; needed to model blast radius and deployment ordering (medium)- [ ] `stack.tasks` — one-off tasks executed against a stack, a run class not covered by the runs table (medium)diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/spacelift/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/spacelift/canonical_descriptions.pyindex 52153a5319b7..4e928595ea91 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/spacelift/canonical_descriptions.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/spacelift/canonical_descriptions.py@@ -151,4 +151,71 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {"updater": "Run that last updated the resource.",},},+ "audit_trail_entries": {+ "description": "Audit trail of changes made in the Spacelift account: who created, updated, or deleted which stack, policy, run, or other resource, and when.",+ "docs_url": "https://docs.spacelift.io/integrations/audit-trail",+ "columns": {+ "id": "Unique identifier of the audit trail entry.",+ "action": "Action that was performed.",+ "event_type": "Kind of change: CREATE, UPDATE, DELETE, NONE, or UNKNOWN.",+ "body": "JSON body of the action, with the details of the change.",+ "created_at": "Unix timestamp (seconds) when the action was performed.",+ "updated_at": "Unix timestamp (seconds) when the entry was last updated.",+ "remote_ip": "IP address the action came from.",+ "identity_ulid": "ULID of the identity that performed the action.",+ "actor": "User or run that performed the action.",+ "affected_resource": "Type and ID of the resource the action changed.",+ "related_resource": "Type and ID of a resource related to the action, if any.",+ },+ },+ "run_changes": {+ "description": "Per-resource changes planned or applied by each run: one row per resource address, phase, and change type.",+ "docs_url": "https://docs.spacelift.io/concepts/run",+ "columns": {+ "run_id": "ID of the run that planned or applied the change.",+ "stack_id": "ID of the stack (or module) the run belongs to.",+ "is_module": "Whether the run belongs to a module instead of a stack.",+ "run_created_at": "Unix timestamp (seconds) when the run was created.",+ "phase": "Run phase the change belongs to: DRIFT, PLAN, or APPLY.",+ "change_type": "Kind of change to the resource (e.g. ADD, CHANGE, DELETE, REPLACE_CREATE_BEFORE_DESTROY, NOOP).",+ "address": "Address of the changed resource (e.g. aws_instance.web).",+ "entity_id": "ID of the managed entity that changed.",+ "entity_name": "Name of the changed resource.",+ "entity_type": "Resource type (e.g. aws_instance).",+ "parent": "Address of the parent module the resource belongs to.",+ },+ },+ "policy_evaluation_records": {+ "description": "Records of policy evaluations, one row per evaluation of each policy, with its outcome.",+ "docs_url": "https://docs.spacelift.io/concepts/policy",+ "columns": {+ "policy_id": "ID of the policy that was evaluated.",+ "key": "Key of the policy input sample for this evaluation.",+ "outcome": "Outcome of evaluating the policy.",+ "timestamp": "Unix timestamp (seconds) of the evaluation.",+ },+ },+ "module_versions": {+ "description": "Versions published for each module in the Spacelift module registry, with their test state and consumers.",+ "docs_url": "https://docs.spacelift.io/vendors/terraform/module-registry",+ "columns": {+ "id": "Unique identifier of the module version.",+ "module_id": "ID of the module the version belongs to.",+ "number": "Semantic version number (e.g. 1.2.3).",+ "state": "State of the version: ACTIVE, PENDING (tests running), or FAILED.",+ "created_at": "Unix timestamp (seconds) when the version was created.",+ "last_pulled_at": "Unix timestamp (seconds) when the version was last downloaded.",+ "triggered_by": "User or process that triggered the version.",+ "source_url": "URL of the version's source code.",+ "download_link": "Link to download the version, if available.",+ "notes": "Release notes for the version.",+ "yanked": "Whether the version was yanked from the registry.",+ "yanked_by": "User who yanked the version.",+ "yank_note": "Reason given for yanking the version.",+ "consumer_count": "Number of stacks that use this version.",+ "commit": "VCS commit the version was built from.",+ "module": "ID, name, and owner of the module.",+ "consumers": "Stacks that use this version.",+ },+ },}diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/spacelift/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/spacelift/settings.pyindex c03d16105cac..74f200e90751 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/spacelift/settings.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/spacelift/settings.py@@ -18,6 +18,12 @@ CREATED_AT = "createdAt"# state changes onto the primary key. Runs typically settle within hours; a week# covers long-lived approvals and drift-detection reruns without re-reading history.RUNS_INCREMENTAL_LOOKBACK_SECONDS = 7 * 24 * 60 * 60+# Audit entries never change, so the overlap only covers entries written in the same+# second as the last synced one while the previous sync was reading.+AUDIT_TRAIL_INCREMENTAL_LOOKBACK_SECONDS = 60 * 60++RUN_CREATED_AT = "runCreatedAt"+TIMESTAMP = "timestamp"INCREMENTAL_CREATED_AT_FIELDS: list[IncrementalField] = [{@@ -28,8 +34,28 @@ INCREMENTAL_CREATED_AT_FIELDS: list[IncrementalField] = [},]+INCREMENTAL_RUN_CREATED_AT_FIELDS: list[IncrementalField] = [+ {+ "label": RUN_CREATED_AT,+ "type": IncrementalFieldType.Integer,+ "field": RUN_CREATED_AT,+ "field_type": IncrementalFieldType.Integer,+ },+]+++@dataclass(frozen=True)+class SpaceliftParentConfig:+ # Connection that lists the parents, e.g. "searchModules".+ search_field: str+ # Single-object query field that scopes the child connection, e.g. "module".+ lookup_field: str+ # Row column the parent's id is written to.+ id_column: str+ page_size: int = SPACELIFT_PAGE_SIZE-@dataclass++@dataclass(frozen=True)class SpaceliftEndpointConfig:name: str# Top-level GraphQL field, e.g. "searchStacks". Fields prefixed "search" are@@ -42,8 +68,17 @@ class SpaceliftEndpointConfig:# searchRuns nodes are `RunWithStack { run {...} stack {...} }` rather than a# flat object; the transport flattens them into run rows with stack context.flatten_run_with_stack: bool = False+ # Child connections (e.g. a module's versions) are queried once per parent row.+ parent: SpaceliftParentConfig | None = None+ # Per-resource changes are selected inside searchRuns and flattened to one row per change.+ flatten_run_changes: bool = False+ order_by: dict[str, str] | None = None+ # API field the incremental predicate filters on, when the row column has another name.+ incremental_predicate_field: str | None = None+ incremental_lookback_seconds: int = RUNS_INCREMENTAL_LOOKBACK_SECONDSprimary_keys: list[str] = field(default_factory=lambda: ["id"])- # Rows per page. Most connections accept 100; searchModules rejects anything over 50.+ # Rows per page. Most connections accept 100; searchModules, searchModuleVersions,+ # searchEvaluationRecords and searchAuditTrailEntries reject anything over 50.page_size: int = SPACELIFT_PAGE_SIZEincremental_fields: list[IncrementalField] = field(default_factory=list)partition_key: str | None = None@@ -195,6 +230,88 @@ updater {}"""+AUDIT_TRAIL_ENTRY_SELECTION = """+id+action+eventType+body+createdAt+updatedAt+remoteIP+identityUlid+actor {+ username+ run {+ id+ stackId+ }+}+affectedResource {+ resourceType+ resourceId+}+relatedResource {+ resourceType+ resourceId+}+"""++# Selected under each searchRuns node as `run { id createdAt changesV2(...) { ... } }`.+# changesV2 works for every vendor; changesV3 and searchEntityChanges error outside Terraform.+RUN_CHANGE_SELECTION = """+type+phase+entity {+ id+ address+ name+ type+ parent+}+"""+RUN_CHANGES_PAGE_SIZE = 50++POLICY_EVALUATION_RECORD_SELECTION = """+key+outcome+timestamp+"""++MODULE_VERSION_SELECTION = """+id+number+state+createdAt+lastPulledAt+triggeredBy+sourceURL+downloadLink+notes+yanked+yankedBy+yankNote+consumerCount+versionCount+commit {+ authorLogin+ authorName+ hash+ message+ timestamp+ url+}+module {+ id+ name+ owner+}+consumers {+ id+}+"""++CREATED_AT_DESC = {"field": CREATED_AT, "direction": "DESC"}+SPACELIFT_ENDPOINTS: dict[str, SpaceliftEndpointConfig] = {"stacks": SpaceliftEndpointConfig(name="stacks",@@ -251,6 +368,61 @@ SPACELIFT_ENDPOINTS: dict[str, SpaceliftEndpointConfig] = {node_selection=MANAGED_ENTITY_SELECTION,primary_keys=["stackId", "id"],),+ "audit_trail_entries": SpaceliftEndpointConfig(+ name="audit_trail_entries",+ graphql_field="searchAuditTrailEntries",+ node_selection=AUDIT_TRAIL_ENTRY_SELECTION,+ page_size=50,+ order_by=CREATED_AT_DESC,+ incremental_fields=INCREMENTAL_CREATED_AT_FIELDS,+ incremental_lookback_seconds=AUDIT_TRAIL_INCREMENTAL_LOOKBACK_SECONDS,+ partition_key=CREATED_AT,+ partition_mode="datetime",+ partition_format="month",+ sort_mode="desc",+ ),+ # One row per resource change of each run. Changes appear once a run plans or applies,+ # so incremental syncs re-read the same trailing window of runs as the runs table.+ "run_changes": SpaceliftEndpointConfig(+ name="run_changes",+ graphql_field="searchRuns",+ node_selection=RUN_CHANGE_SELECTION,+ flatten_run_changes=True,+ # Each run in the page carries up to RUN_CHANGES_PAGE_SIZE nested changes.+ page_size=20,+ primary_keys=["runId", "phase", "address"],+ incremental_fields=INCREMENTAL_RUN_CREATED_AT_FIELDS,+ incremental_predicate_field=CREATED_AT,+ partition_key=RUN_CREATED_AT,+ partition_mode="datetime",+ partition_format="month",+ sort_mode="desc",+ ),+ "policy_evaluation_records": SpaceliftEndpointConfig(+ name="policy_evaluation_records",+ graphql_field="searchEvaluationRecords",+ node_selection=POLICY_EVALUATION_RECORD_SELECTION,+ parent=SpaceliftParentConfig(search_field="searchPolicies", lookup_field="policy", id_column="policyId"),+ page_size=50,+ primary_keys=["policyId", "key"],+ partition_key=TIMESTAMP,+ partition_mode="datetime",+ partition_format="month",+ ),+ "module_versions": SpaceliftEndpointConfig(+ name="module_versions",+ graphql_field="searchModuleVersions",+ node_selection=MODULE_VERSION_SELECTION,+ parent=SpaceliftParentConfig(+ search_field="searchModules", lookup_field="module", id_column="moduleId", page_size=50+ ),+ page_size=50,+ order_by=CREATED_AT_DESC,+ primary_keys=["moduleId", "id"],+ partition_key=CREATED_AT,+ partition_mode="datetime",+ partition_format="month",+ ),}ENDPOINTS = tuple(SPACELIFT_ENDPOINTS.keys())diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/spacelift/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/spacelift/source.pyindex a15549e4c4f4..ff9b82a69473 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/spacelift/source.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/spacelift/source.py@@ -122,6 +122,11 @@ Your account name is the subdomain you use to access Spacelift (e.g. `my-company"All runs across every stack and module. Incremental syncs re-pull the last "f"{RUNS_INCREMENTAL_LOOKBACK_SECONDS // 86400} days so state changes on recent runs are captured")+ if endpoint == "run_changes":+ return (+ "Per-resource changes of each run. Incremental syncs re-pull the runs of the last "+ f"{RUNS_INCREMENTAL_LOOKBACK_SECONDS // 86400} days so changes from runs that finish later are captured"+ )if endpoint == "managed_entities":return "Terraform/OpenTofu resources managed across your stacks, including drift status"return Nonediff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/spacelift/spacelift.py b/products/warehouse_sources/backend/temporal/data_imports/sources/spacelift/spacelift.pyindex 9a416f0e85a5..f7207637ef2f 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/spacelift/spacelift.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/spacelift/spacelift.py@@ -13,10 +13,13 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.httfrom products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManagerfrom products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponsefrom products.warehouse_sources.backend.temporal.data_imports.sources.spacelift.settings import (+ RUN_CHANGE_SELECTION,+ RUN_CHANGES_PAGE_SIZE,RUNS_INCREMENTAL_LOOKBACK_SECONDS,SPACELIFT_ENDPOINTS,SPACELIFT_HOST_TEMPLATE,SpaceliftEndpointConfig,+ SpaceliftParentConfig,)# Spacelift publishes no rate limits; ride out transient 429/5xx bursts in-process@@ -65,10 +68,16 @@ class SpaceliftPermissionError(Exception):pass[email protected](frozen=True)class SpaceliftResumeConfig:- # Relay `after` cursor of the next page to fetch for the current endpoint.+ # Relay `after` cursor of the next page to fetch for the current endpoint. For+ # per-parent endpoints it is the cursor of the parent page being walked.cursor: str+ # Parents on that parent page whose child rows are all synced.+ completed_parent_ids: list[str] = dataclasses.field(default_factory=list)+ # Parent whose child rows are partly synced, and the cursor of its next child page.+ child_parent_id: str | None = None+ child_cursor: str | None = Nonedef normalize_account_name(account_name: str) -> str:@@ -113,6 +122,79 @@ query Search($input: SearchInput!) {{"""+def _connection_query(operation: str, field_call: str, node_selection: str) -> str:+ return f"""+query {operation} {{+ {field_call} {{+ edges {{+ cursor+ node {{ {node_selection} }}+ }}+ pageInfo {{+ endCursor+ hasNextPage+ }}+ }}+}}+"""+++def build_run_changes_query() -> str:+ node_selection = f"""+isModule+stack {{ id }}+run {{+ id+ createdAt+ changesV2(input: $changesInput) {{+ edges {{ node {{ {RUN_CHANGE_SELECTION} }} }}+ pageInfo {{ endCursor hasNextPage }}+ }}+}}+"""+ return _connection_query(+ "SearchRunChanges($input: SearchInput!, $changesInput: EntityChangeCollectionInputV2!)",+ "searchRuns(input: $input)",+ node_selection,+ )+++RUN_CHANGES_OVERFLOW_QUERY = f"""+query RunChanges($id: ID!, $changesInput: EntityChangeCollectionInputV2!) {{+ run(id: $id) {{+ changesV2(input: $changesInput) {{+ edges {{ node {{ {RUN_CHANGE_SELECTION} }} }}+ pageInfo {{ endCursor hasNextPage }}+ }}+ }}+}}+"""+++def build_parent_query(parent: SpaceliftParentConfig) -> str:+ return _connection_query("SearchParents($input: SearchInput!)", f"{parent.search_field}(input: $input)", "id")+++def build_child_query(config: SpaceliftEndpointConfig) -> str:+ assert config.parent is not None+ return f"""+query SearchChildren($id: ID!, $input: SearchInput!) {{+ {config.parent.lookup_field}(id: $id) {{+ {config.graphql_field}(input: $input) {{+ edges {{+ cursor+ node {{ {config.node_selection} }}+ }}+ pageInfo {{+ endCursor+ hasNextPage+ }}+ }}+ }}+}}+"""++def to_unix_seconds(value: Any) -> int | None:"""Coerce an incremental watermark into the Unix-seconds Int Spacelift timestamps use."""if value is None:@@ -138,7 +220,9 @@ def to_unix_seconds(value: Any) -> int | None:return None-def build_incremental_predicates(incremental_field: str, last_value: Any) -> list[dict[str, Any]] | None:+def build_incremental_predicates(+ incremental_field: str, last_value: Any, lookback_seconds: int = RUNS_INCREMENTAL_LOOKBACK_SECONDS+) -> list[dict[str, Any]] | None:"""Build the server-side `timeInRange` predicate for an incremental sync.Mirrors the filter the Spacelift UI itself sends to `searchRuns`. The watermark is@@ -150,7 +234,7 @@ def build_incremental_predicates(incremental_field: str, last_value: Any) -> lisstart = to_unix_seconds(last_value)if start is None:return None- start = max(0, start - RUNS_INCREMENTAL_LOOKBACK_SECONDS)+ start = max(0, start - lookback_seconds)return [{"field": incremental_field, "constraint": {"timeInRange": {"start": start}}}]@@ -276,6 +360,203 @@ def _normalize_run_node(node: dict[str, Any]) -> dict[str, Any]:return row+def _next_cursor(page_info: dict[str, Any], name: str) -> str | None:+ if not page_info.get("hasNextPage"):+ return None+ end_cursor = page_info.get("endCursor")+ if not end_cursor:+ # A hasNextPage=True page with no cursor would loop forever on itself;+ # fail loudly instead of silently returning partial results.+ raise Exception(f"Spacelift: hasNextPage=True but endCursor is empty for {name}")+ return end_cursor+++def _build_search_input(+ endpoint_config: SpaceliftEndpointConfig,+ logger: FilteringBoundLogger | None = None,+ should_use_incremental_field: bool = False,+ db_incremental_field_last_value: Any = None,+ incremental_field: str | None = None,+) -> dict[str, Any]:+ search_input: dict[str, Any] = {"first": endpoint_config.page_size, "after": None}+ if endpoint_config.order_by:+ search_input["orderBy"] = endpoint_config.order_by++ if should_use_incremental_field and incremental_field:+ predicates = build_incremental_predicates(+ endpoint_config.incremental_predicate_field or incremental_field,+ db_incremental_field_last_value,+ endpoint_config.incremental_lookback_seconds,+ )+ if predicates:+ search_input["predicates"] = predicates+ if logger is not None:+ logger.debug(f"Spacelift: incremental sync for {endpoint_config.name} with predicates {predicates}")++ return search_input+++def _normalize_run_change(run: dict[str, Any], stack_id: Any, is_module: Any, change: dict[str, Any]) -> dict[str, Any]:+ entity = change.get("entity") or {}+ return {+ "runId": run.get("id"),+ "stackId": stack_id,+ "isModule": is_module,+ "runCreatedAt": run.get("createdAt"),+ "phase": change.get("phase"),+ "changeType": change.get("type"),+ "address": entity.get("address"),+ "entityId": entity.get("id"),+ "entityName": entity.get("name"),+ "entityType": entity.get("type"),+ "parent": entity.get("parent"),+ }+++def _collect_run_changes(client: SpaceliftClient, node: dict[str, Any]) -> list[dict[str, Any]]:+ run = node.get("run") or {}+ stack_id = (node.get("stack") or {}).get("id")+ changes = run.get("changesV2") or {}+ rows: list[dict[str, Any]] = []++ while True:+ for edge in changes.get("edges") or []:+ change = edge.get("node")+ if change is not None:+ rows.append(_normalize_run_change(run, stack_id, node.get("isModule"), change))++ # Runs with more changes than fit in the nested page are finished with per-run queries.+ after = _next_cursor(changes.get("pageInfo") or {}, f"run_changes of run {run.get('id')}")+ if after is None:+ return rows+ data = client.execute(+ RUN_CHANGES_OVERFLOW_QUERY,+ {"id": run.get("id"), "changesInput": {"first": RUN_CHANGES_PAGE_SIZE, "after": after}},+ )+ changes = (data.get("run") or {}).get("changesV2") or {}+++def _get_run_change_rows(+ client: SpaceliftClient,+ endpoint_config: SpaceliftEndpointConfig,+ resumable_source_manager: ResumableSourceManager[SpaceliftResumeConfig],+ search_input: dict[str, Any],+) -> Iterator[list[dict[str, Any]]]:+ query = build_run_changes_query()++ resume_config = resumable_source_manager.load_state() if resumable_source_manager.can_resume() else None+ if resume_config is not None and resume_config.cursor:+ search_input["after"] = resume_config.cursor++ while True:+ data = client.execute(query, {"input": search_input, "changesInput": {"first": RUN_CHANGES_PAGE_SIZE}})+ connection = data.get(endpoint_config.graphql_field) or {}++ rows: list[dict[str, Any]] = []+ for edge in connection.get("edges") or []:+ rows.extend(_collect_run_changes(client, edge.get("node") or {}))++ end_cursor = _next_cursor(connection.get("pageInfo") or {}, endpoint_config.name)+ if end_cursor is not None:+ resumable_source_manager.save_state(SpaceliftResumeConfig(cursor=end_cursor))++ if rows:+ yield rows+ else:+ # Tasks and failed runs have no changes, so whole pages can come back empty.+ resumable_source_manager.safe_point()++ if end_cursor is None:+ break+ search_input["after"] = end_cursor+++def _iter_parent_pages(+ client: SpaceliftClient, parent: SpaceliftParentConfig, after: str | None, name: str+) -> Iterator[tuple[str | None, list[str]]]:+ """Yield `(cursor used to fetch the page, parent ids on the page)` for each parent page."""+ query = build_parent_query(parent)+ while True:+ data = client.execute(query, {"input": {"first": parent.page_size, "after": after}})+ connection = data.get(parent.search_field) or {}+ parent_ids = [(edge.get("node") or {}).get("id") for edge in connection.get("edges") or [] if edge.get("node")]+ yield after, [parent_id for parent_id in parent_ids if parent_id]++ after = _next_cursor(connection.get("pageInfo") or {}, f"{name} parents")+ if after is None:+ return+++def _iter_child_pages(+ client: SpaceliftClient, endpoint_config: SpaceliftEndpointConfig, parent_id: str, after: str | None+) -> Iterator[tuple[list[dict[str, Any]], str | None]]:+ """Yield `(rows, cursor of the next child page or None)` for each child page of one parent."""+ assert endpoint_config.parent is not None+ query = build_child_query(endpoint_config)+ search_input = _build_search_input(endpoint_config)+ if after:+ search_input["after"] = after++ while True:+ data = client.execute(query, {"id": parent_id, "input": search_input})+ # A parent deleted after it was listed resolves to null.+ connection = (data.get(endpoint_config.parent.lookup_field) or {}).get(endpoint_config.graphql_field) or {}+ rows = [+ {**edge["node"], endpoint_config.parent.id_column: parent_id}+ for edge in connection.get("edges") or []+ if edge.get("node") is not None+ ]++ after = _next_cursor(connection.get("pageInfo") or {}, endpoint_config.name)+ yield rows, after+ if after is None:+ return+ search_input["after"] = after+++def _get_child_rows(+ client: SpaceliftClient,+ endpoint_config: SpaceliftEndpointConfig,+ resumable_source_manager: ResumableSourceManager[SpaceliftResumeConfig],+) -> Iterator[list[dict[str, Any]]]:+ assert endpoint_config.parent is not None++ resume_config = resumable_source_manager.load_state() if resumable_source_manager.can_resume() else None+ completed_parent_ids = list(resume_config.completed_parent_ids) if resume_config is not None else []+ child_parent_id = resume_config.child_parent_id if resume_config is not None else None+ child_cursor = resume_config.child_cursor if resume_config is not None else None+ start_cursor = resume_config.cursor if resume_config is not None and resume_config.cursor else None++ for page_cursor, parent_ids in _iter_parent_pages(+ client, endpoint_config.parent, start_cursor, endpoint_config.name+ ):+ for parent_id in parent_ids:+ if parent_id in completed_parent_ids:+ continue++ after = child_cursor if parent_id == child_parent_id else None+ for rows, next_after in _iter_child_pages(client, endpoint_config, parent_id, after):+ if next_after is None:+ completed_parent_ids.append(parent_id)+ resumable_source_manager.save_state(+ SpaceliftResumeConfig(+ cursor=page_cursor or "",+ completed_parent_ids=list(completed_parent_ids),+ child_parent_id=parent_id if next_after is not None else None,+ child_cursor=next_after,+ )+ )+ if rows:+ yield rows+ else:+ resumable_source_manager.safe_point()++ # The saved progress only applies to the parent page it was recorded on.+ completed_parent_ids = []+ child_parent_id = None+ child_cursor = None++def _get_rows(client: SpaceliftClient,endpoint_config: SpaceliftEndpointConfig,@@ -295,13 +576,17 @@ def _get_rows(yield rowsreturn- search_input: dict[str, Any] = {"first": endpoint_config.page_size, "after": None}+ if endpoint_config.parent is not None:+ yield from _get_child_rows(client, endpoint_config, resumable_source_manager)+ return- if should_use_incremental_field and incremental_field:- predicates = build_incremental_predicates(incremental_field, db_incremental_field_last_value)- if predicates:- search_input["predicates"] = predicates- logger.debug(f"Spacelift: incremental sync for {endpoint_config.name} with predicates {predicates}")+ search_input = _build_search_input(+ endpoint_config, logger, should_use_incremental_field, db_incremental_field_last_value, incremental_field+ )++ if endpoint_config.flatten_run_changes:+ yield from _get_run_change_rows(client, endpoint_config, resumable_source_manager, search_input)+ returnresume_config = resumable_source_manager.load_state() if resumable_source_manager.can_resume() else Noneif resume_config is not None and resume_config.cursor:diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/spacelift/tests/test_spacelift.py b/products/warehouse_sources/backend/temporal/data_imports/sources/spacelift/tests/test_spacelift.pyindex af5e33954511..cc9b4b2d1e5e 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/spacelift/tests/test_spacelift.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/spacelift/tests/test_spacelift.py@@ -14,6 +14,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.generated_SpaceliftSourceConfig,)from products.warehouse_sources.backend.temporal.data_imports.sources.spacelift.settings import (+ AUDIT_TRAIL_INCREMENTAL_LOOKBACK_SECONDS,RUNS_INCREMENTAL_LOOKBACK_SECONDS,)from products.warehouse_sources.backend.temporal.data_imports.sources.spacelift.source import SpaceliftSource@@ -63,6 +64,61 @@ def _run(return SourceDriver(SpaceliftSource(), config).run(endpoint, script, resume_state=resume_state, **inputs)+def _module_page(module_ids: list[str], end_cursor: str = "", has_next: bool = False) -> ScriptedResponse:+ return _search_page("searchModules", [{"id": module_id} for module_id in module_ids], end_cursor, has_next)+++def _versions_page(version_ids: list[str] | None, end_cursor: str = "", has_next: bool = False) -> ScriptedResponse:+ if version_ids is None:+ # The module was deleted between listing it and querying its versions.+ return _response({"data": {"module": None}})+ return _response(+ {+ "data": {+ "module": {+ "searchModuleVersions": {+ "edges": [{"cursor": v, "node": {"id": v}} for v in version_ids],+ "pageInfo": {"endCursor": end_cursor, "hasNextPage": has_next},+ }+ }+ }+ }+ )+++def _changes(addresses: list[str], end_cursor: str = "", has_next: bool = False) -> dict[str, Any]:+ return {+ "edges": [+ {"node": {"type": "ADD", "phase": "PLAN", "entity": {"id": f"ent-{a}", "address": a}}} for a in addresses+ ],+ "pageInfo": {"endCursor": end_cursor, "hasNextPage": has_next},+ }+++def _run_node(run_id: str, changes: dict[str, Any] | None) -> dict[str, Any]:+ return {+ "isModule": False,+ "stack": {"id": "stack-1"},+ "run": {"id": run_id, "createdAt": 1700000100, "changesV2": changes},+ }+++def _change_row(run_id: str, address: str) -> dict[str, Any]:+ return {+ "runId": run_id,+ "stackId": "stack-1",+ "isModule": False,+ "runCreatedAt": 1700000100,+ "phase": "PLAN",+ "changeType": "ADD",+ "address": address,+ "entityId": f"ent-{address}",+ "entityName": None,+ "entityType": None,+ "parent": None,+ }++def _query_requests(result: DriveResult) -> list[RecordedRequest]:# Skip the initial apiKeyUser token exchange, leaving the data queries.return result.requests[1:]@@ -180,22 +236,155 @@ class TestSpacelift:]]- def test_incremental_sync_sends_time_range_predicate(self):+ @pytest.mark.parametrize(+ "endpoint, graphql_field, incremental_field, lookback, order_by",+ [+ ("runs", "searchRuns", "createdAt", RUNS_INCREMENTAL_LOOKBACK_SECONDS, None),+ (+ "audit_trail_entries",+ "searchAuditTrailEntries",+ "createdAt",+ AUDIT_TRAIL_INCREMENTAL_LOOKBACK_SECONDS,+ {"field": "createdAt", "direction": "DESC"},+ ),+ # The row column is runCreatedAt, but searchRuns only filters on createdAt.+ ("run_changes", "searchRuns", "runCreatedAt", RUNS_INCREMENTAL_LOOKBACK_SECONDS, None),+ ],+ )+ def test_incremental_sync_sends_time_range_predicate(+ self, endpoint, graphql_field, incremental_field, lookback, order_by+ ):result = _run(- "runs",- [_response(TOKEN_PAYLOAD), _search_page("searchRuns", [])],+ endpoint,+ [_response(TOKEN_PAYLOAD), _search_page(graphql_field, [])],should_use_incremental_field=True,db_incremental_field_last_value=1700000000,- incremental_field="createdAt",+ incremental_field=incremental_field,)+ assert result.raised is Nonesent_input = _query_requests(result)[0].json()["variables"]["input"]assert sent_input["predicates"] == [- {- "field": "createdAt",- "constraint": {"timeInRange": {"start": 1700000000 - RUNS_INCREMENTAL_LOOKBACK_SECONDS}},- }+ {"field": "createdAt", "constraint": {"timeInRange": {"start": 1700000000 - lookback}}}+ ]+ assert sent_input.get("orderBy") == order_by++ def test_run_changes_flatten_nested_changes_and_fetch_overflow_per_run(self):+ result = _run(+ "run_changes",+ [+ _response(TOKEN_PAYLOAD),+ _search_page(+ "searchRuns",+ [+ _run_node("run-1", _changes(["aws_s3_bucket.a"], end_cursor="ch-1", has_next=True)),+ _run_node("run-2", None),+ ],+ end_cursor="runs-1",+ has_next=True,+ ),+ _response({"data": {"run": {"changesV2": _changes(["aws_s3_bucket.b"])}}}),+ # A page of runs with no changes yields nothing but the walk continues.+ _search_page("searchRuns", [_run_node("run-3", _changes([]))], end_cursor="runs-2", has_next=True),+ _search_page("searchRuns", [_run_node("run-4", _changes(["aws_iam_role.c"]))]),+ ],+ )++ assert result.raised is None+ assert result.items == [+ [_change_row("run-1", "aws_s3_bucket.a"), _change_row("run-1", "aws_s3_bucket.b")],+ [_change_row("run-4", "aws_iam_role.c")],+ ]+ requests = _query_requests(result)+ overflow = requests[1].json()["variables"]+ assert overflow["id"] == "run-1"+ assert overflow["changesInput"]["after"] == "ch-1"+ assert [r.json()["variables"].get("input", {}).get("after") for r in requests] == [+ None,+ None,+ "runs-1",+ "runs-2",+ ]+ assert result.saved_states == [+ SpaceliftResumeConfig(cursor="runs-1"),+ SpaceliftResumeConfig(cursor="runs-2"),+ ]++ def test_child_endpoint_walks_every_parent_and_child_page(self):+ result = _run(+ "module_versions",+ [+ _response(TOKEN_PAYLOAD),+ _module_page(["mod-a", "mod-b"], end_cursor="mods-1", has_next=True),+ _versions_page(["v1"], end_cursor="v-1", has_next=True),+ _versions_page(["v2"]),+ _versions_page(None),+ _module_page(["mod-c"]),+ _versions_page(["v3"]),+ ],+ )++ assert result.raised is None+ assert result.items == [+ [{"id": "v1", "moduleId": "mod-a"}],+ [{"id": "v2", "moduleId": "mod-a"}],+ [{"id": "v3", "moduleId": "mod-c"}],]+ requests = _query_requests(result)+ assert [r.json()["variables"].get("id") for r in requests] == [+ None,+ "mod-a",+ "mod-a",+ "mod-b",+ None,+ "mod-c",+ ]+ assert requests[2].json()["variables"]["input"]["after"] == "v-1"+ assert requests[4].json()["variables"]["input"]["after"] == "mods-1"+ assert result.saved_states == [+ SpaceliftResumeConfig(cursor="", child_parent_id="mod-a", child_cursor="v-1"),+ SpaceliftResumeConfig(cursor="", completed_parent_ids=["mod-a"]),+ SpaceliftResumeConfig(cursor="", completed_parent_ids=["mod-a", "mod-b"]),+ SpaceliftResumeConfig(cursor="mods-1", completed_parent_ids=["mod-c"]),+ ]++ @pytest.mark.parametrize(+ "resume_state, expected_requests",+ [+ (+ SpaceliftResumeConfig(cursor="mods-1", completed_parent_ids=["mod-a"]),+ [("mod-b", None)],+ ),+ # Parents are skipped by id, so a completed parent that no longer exists changes nothing.+ (+ SpaceliftResumeConfig(cursor="mods-1", completed_parent_ids=["mod-a", "mod-deleted"]),+ [("mod-b", None)],+ ),+ # The parent that was partly synced continues from its saved child cursor.+ (+ SpaceliftResumeConfig(cursor="mods-1", child_parent_id="mod-a", child_cursor="v-1"),+ [("mod-a", "v-1"), ("mod-b", None)],+ ),+ ],+ )+ def test_child_endpoint_resumes_without_repeating_finished_work(self, resume_state, expected_requests):+ result = _run(+ "module_versions",+ [+ _response(TOKEN_PAYLOAD),+ _module_page(["mod-a", "mod-b"]),+ *[_versions_page([f"v-{parent}"]) for parent, _ in expected_requests],+ ],+ resume_state=resume_state,+ )++ assert result.raised is None+ requests = _query_requests(result)+ assert requests[0].json()["variables"]["input"]["after"] == "mods-1"+ assert [+ (r.json()["variables"]["id"], r.json()["variables"]["input"].get("after")) for r in requests[1:]+ ] == expected_requests+ assert [row["moduleId"] for row in result.rows] == [parent for parent, _ in expected_requests]def test_plain_list_endpoint_yields_rows_without_pagination(self):result = _run(