Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(warehouse): add humanitec deltas, resource and pipeline run tables (#122033)

Tom Owers84e2f7e97a43
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
index cc07c02931ff..88fc9a2ecb6a 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
@@ -6111,10 +6111,10 @@ Today (6): `active_resources`, `applications`, `deployments`, `environment_types
Diffed against: <https://api-docs.humanitec.com/openapi.json>
-- [ ] `Resource Definitions (GET /orgs/{orgId}/resources/defs)` — resolve resource definition IDs on active resources and explain how infrastructure was provisioned (high)
-- [ ] `Resource Types (GET /orgs/{orgId}/resources/types)` — resolve active resource types to their definitions for infrastructure inventory analysis (high)
-- [ ] `Deployment Deltas (GET /orgs/{orgId}/apps/{appId}/deltas)` — resolve delta_id on deployments and analyze the changes applied during each release (high)
-- [ ] `Pipeline Runs (GET /orgs/{orgId}/pipeline-runs)` — resolve deployment run references and measure pipeline success rates and execution time (high)
+- [x] `Resource Definitions (GET /orgs/{orgId}/resources/defs)` — resolve resource definition IDs on active resources and explain how infrastructure was provisioned (high)
+- [x] `Resource Types (GET /orgs/{orgId}/resources/types)` — resolve active resource types to their definitions for infrastructure inventory analysis (high)
+- [x] `Deployment Deltas (GET /orgs/{orgId}/apps/{appId}/deltas)` — resolve delta_id on deployments and analyze the changes applied during each release (high)
+- [x] `Pipeline Runs (GET /orgs/{orgId}/pipeline-runs)` — resolve deployment run references and measure pipeline success rates and execution time (high)
- [ ] `Pipeline run jobs (GET /orgs/{orgId}/apps/{appId}/pipelines/{pipelineId}/runs/{runId}/jobs)` — identify failed or slow jobs within each pipeline run (medium)
- [ ] `Deployment errors (GET /orgs/{orgId}/apps/{appId}/envs/{envId}/deploys/{deployId}/errors)` — classify deployment failures by error code, scope, and affected workload (medium)
- [ ] `Audit logs (GET /orgs/{orgId}/audit-logs)` — analyze successful changes by actor, operation, and time across the organization (medium)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/humanitec/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/humanitec/canonical_descriptions.py
index 322f32fbc57c..e2d092f475bc 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/humanitec/canonical_descriptions.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/humanitec/canonical_descriptions.py
@@ -80,4 +80,75 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {
"trigger_types": "Trigger types in the current pipeline schema.",
},
},
+ "deployment_deltas": {
+ "description": "Deployment deltas describe a set of changes to apply to an environment in a deployment.",
+ "docs_url": API_DOCS_URL,
+ "columns": {
+ "id": "Unique ID of the delta.",
+ "app_id": "Application ID from the parent application.",
+ "metadata": "Delta metadata such as its name, contributors, environment, and archive state.",
+ "modules": "Module operations that the delta adds, removes, or updates.",
+ "workloads": "Workload operations that the delta adds, removes, or updates.",
+ "shared": "Operations on shared values and resources.",
+ },
+ },
+ "resource_definitions": {
+ "description": "Resource definitions tell the Platform Orchestrator how to provision a resource type.",
+ "docs_url": API_DOCS_URL,
+ "columns": {
+ "id": "Resource definition ID.",
+ "org_id": "Organization that contains the resource definition.",
+ "name": "Display name.",
+ "type": "Resource type that the definition provisions.",
+ "driver_type": "Driver used to create the resource.",
+ "driver_account": "Security account that the driver uses.",
+ "driver_inputs": "Input data passed to the driver. Secrets appear as secret store references.",
+ "criteria": "Matching criteria used to select the definition during a deployment.",
+ "provision": "Co-provisioned resources.",
+ "active_version_id": "Active resource definition version ID.",
+ "is_default": "True for built-in definitions that Humanitec provides.",
+ "is_deleted": "True when the definition is marked for deletion.",
+ "created_at": "Time when the definition was created.",
+ "created_by": "User who created the definition.",
+ "updated_at": "Time when the definition was last updated.",
+ "updated_by": "User who last updated the definition.",
+ },
+ },
+ "resource_types": {
+ "description": "Resource types are the kinds of infrastructure resources a workload can depend on.",
+ "docs_url": API_DOCS_URL,
+ "columns": {
+ "type": "Unique resource type identifier across all organizations.",
+ "name": "Display name.",
+ "category": "Category that groups similar resource types.",
+ "use": "Dependency kind between the resource and a workload: direct, indirect, or implicit.",
+ "inputs_schema": "JSON schema of the driver inputs for this resource type.",
+ "outputs_schema": "JSON schema of the data passed to the deployment.",
+ },
+ },
+ "pipeline_runs": {
+ "description": "Pipeline runs are executions of an application pipeline.",
+ "docs_url": API_DOCS_URL,
+ "columns": {
+ "id": "Run ID.",
+ "org_id": "Organization that contains the run.",
+ "app_id": "Application that contains the run.",
+ "pipeline_id": "Pipeline that the run executes.",
+ "pipeline_version": "Pipeline version that the run executes.",
+ "env_ids": "Environments linked to the run through inputs or step executions.",
+ "status": "Current run status.",
+ "status_message": "Reason for the current status.",
+ "trigger": "Trigger type that started the run.",
+ "inputs": "Inputs provided to the run.",
+ "created_at": "Time when the run was created.",
+ "created_by": "User who created or triggered the run.",
+ "executing_at": "Time when the run started to execute.",
+ "completed_at": "Time when the run succeeded, failed, or was cancelled.",
+ "cancellation_requested_at": "Time when cancellation of the run was requested.",
+ "timeout_seconds": "Timeout for the run.",
+ "run_as": "User that the run acts as when it calls Humanitec APIs.",
+ "concurrency_group": "Optional concurrency group for the run within the application.",
+ "waiting_for": "Events that the run's jobs wait for.",
+ },
+ },
}
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/humanitec/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/humanitec/settings.py
index 46af6ea4ffef..b412950f4120 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/humanitec/settings.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/humanitec/settings.py
@@ -31,6 +31,15 @@ ENDPOINTS: dict[str, HumanitecEndpoint] = {
"pipelines": HumanitecEndpoint(
path="apps/{app_id}/pipelines", primary_keys=("app_id", "id"), parent="applications", paginated=True
),
+ "deployment_deltas": HumanitecEndpoint(
+ path="apps/{app_id}/deltas", primary_keys=("app_id", "id"), parent="applications"
+ ),
+ "resource_definitions": HumanitecEndpoint(path="resources/defs", primary_keys=("id",)),
+ "resource_types": HumanitecEndpoint(path="resources/types", primary_keys=("type",)),
+ # Runs change status after creation, so a created_after watermark would freeze in-flight runs; sync in full.
+ "pipeline_runs": HumanitecEndpoint(
+ path="pipeline-runs", primary_keys=("app_id", "pipeline_id", "id"), paginated=True
+ ),
}
AUTH_ERROR = "Humanitec rejected the API token. Check the token and its expiration date."
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/humanitec/tests/test_humanitec.py b/products/warehouse_sources/backend/temporal/data_imports/sources/humanitec/tests/test_humanitec.py
index 999ee1779d2a..0104eb386b6a 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/humanitec/tests/test_humanitec.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/humanitec/tests/test_humanitec.py
@@ -58,6 +58,42 @@ def test_environment_children_keep_parent_keys(endpoint: str, suffix: str, key:
]
+def test_deployment_deltas_keep_application_key() -> None:
+ result = SourceDriver(HumanitecSource(), CONFIG).run(
+ "deployment_deltas",
+ [
+ ScriptedResponse(json=[{"id": "app-a", "envs": []}, {"id": "app-b", "envs": []}]),
+ ScriptedResponse(json=[{"id": "same-delta"}]),
+ ScriptedResponse(json=[{"id": "same-delta"}]),
+ ],
+ )
+
+ assert result.raised is None
+ assert result.rows == [{"id": "same-delta", "app_id": "app-a"}, {"id": "same-delta", "app_id": "app-b"}]
+ assert result.urls == [f"{BASE}/apps", f"{BASE}/apps/app-a/deltas", f"{BASE}/apps/app-b/deltas"]
+
+
+def test_pipeline_runs_follow_link_pages() -> None:
+ run = {"app_id": "app-a", "pipeline_id": "deploy"}
+ result = SourceDriver(HumanitecSource(), CONFIG).run(
+ "pipeline_runs",
+ [
+ ScriptedResponse(
+ json=[{**run, "id": "run-2"}],
+ headers={"Link": f'<{BASE}/pipeline-runs?per_page=100&page=next-token>; rel="next"'},
+ ),
+ ScriptedResponse(json=[{**run, "id": "run-1"}]),
+ ],
+ )
+
+ assert result.raised is None
+ assert [row["id"] for row in result.rows] == ["run-2", "run-1"]
+ assert result.urls == [
+ f"{BASE}/pipeline-runs?per_page=100",
+ f"{BASE}/pipeline-runs?per_page=100&page=next-token",
+ ]
+
+
@pytest.mark.parametrize("status,expected", [(404, "could not find")])
def test_credential_probe(status: int, expected: str | None) -> None:
body = {"id": "example-org"} if status == 200 else {"error": f"HTTP-{status}", "message": HTTPStatus(status).phrase}