Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(data-warehouse): add 4 missing tables to the Northflank source (#123792)

Tom Owers1438a338631d
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
index 38e69e1ea712..2d0d9ecb5248 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
@@ -8454,10 +8454,10 @@ Diffed against: <https://northflank.com/docs/sitemap.xml>
- [x] `GET /v1/projects/{projectId}/jobs/{jobId}/runs` — job run history with success/failure and durations; the jobs table alone has no execution record (high)
- [x] `GET /v1/billing/usage` — hourly resource usage entries - the basis for cost attribution per project/service (high)
- [x] `GET /v1/plans` — lookup table resolving the plan ids carried on every service, job, addon and volume already synced (vCPU, RAM, hourly cost) (high)
-- [ ] `GET /v1/projects/{projectId}/jobs/{jobId}/build` — build history for jobs, the job-side counterpart to service builds (medium)
-- [ ] `GET /v1/projects/{projectId}/workflows/{workflowId}/runs` — workflow execution history, the release automation audit trail (medium)
-- [ ] `GET /v1/projects/{projectId}/pipelines` — pipeline definitions that services and release flows belong to - the missing grouping dimension (medium)
-- [ ] `GET /v1/projects/{projectId}/pipelines/{pipelineId}/release-flows/{id}/runs` — release flow run history: what shipped, when, and whether it succeeded (medium)
+- [x] `GET /v1/projects/{projectId}/jobs/{jobId}/build` — build history for jobs, the job-side counterpart to service builds (medium)
+- [x] `GET /v1/projects/{projectId}/workflows/{workflowId}/runs` — workflow execution history, the release automation audit trail (medium)
+- [x] `GET /v1/projects/{projectId}/pipelines` — pipeline definitions that services and release flows belong to - the missing grouping dimension (medium)
+- [x] `GET /v1/projects/{projectId}/pipelines/{pipelineId}/release-flows/{id}/runs` — release flow run history: what shipped, when, and whether it succeeded (medium)
- [ ] `GET /v1/projects/{projectId}/preview-environments` — preview environment inventory and lifetimes, a major driver of ephemeral spend (medium)
- [ ] `GET /v1/tags` — lookup table resolving the tag ids attached to projects, services and addons (medium)
- [ ] `GET /v1/clusters and /v1/clusters/{id}/nodes` — BYOC cluster and node inventory - the infrastructure dimension services are scheduled onto (medium)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/northflank/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/northflank/canonical_descriptions.py
index 583eb0e661e5..3b846cb06f85 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/northflank/canonical_descriptions.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/northflank/canonical_descriptions.py
@@ -114,6 +114,72 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {
"concludedAt": "When the run concluded.",
},
},
+ "pipelines": {
+ "description": "A pipeline that groups services, jobs, and addons into development, staging, and production stages.",
+ "docs_url": "https://northflank.com/docs/v1/api/project/pipelines/list-pipelines",
+ "columns": {
+ "id": "Identifier for the pipeline.",
+ "projectId": "Identifier of the project the pipeline belongs to.",
+ "name": "The name of the pipeline.",
+ "description": "A description of the pipeline.",
+ "createdAt": "When the pipeline was created.",
+ "updatedAt": "When the pipeline was last updated.",
+ },
+ },
+ "job_builds": {
+ "description": "A build of a job, one row per build run with its commit, status, and timings.",
+ "docs_url": "https://northflank.com/docs/v1/api/project/jobs/list-job-builds",
+ "columns": {
+ "id": "Identifier for the build.",
+ "projectId": "Identifier of the project the job belongs to.",
+ "jobId": "Identifier of the job that was built.",
+ "branch": "Name of the branch the built commit belongs to.",
+ "pullRequestId": "Identifier of the pull request the build was triggered from, if any.",
+ "status": "The status of the build, such as QUEUED, BUILDING, SUCCESS, or FAILURE.",
+ "sha": "The sha of the built commit.",
+ "registry": "Registry details, including the URI to pull the built image from.",
+ "concluded": "Whether the build has finished.",
+ "createdAt": "When the build started.",
+ "success": "Whether the build was successful.",
+ "message": "Description of the build status.",
+ "buildConcludedAt": "When the build concluded, as a Unix timestamp.",
+ },
+ },
+ "workflow_runs": {
+ "description": "A single run of a workflow, with its status and start and end times.",
+ "docs_url": "https://northflank.com/docs/v1/api/project/workflows/list-workflow-runs",
+ "columns": {
+ "id": "Identifier for the workflow run.",
+ "projectId": "Identifier of the project the workflow belongs to.",
+ "workflowId": "Identifier of the workflow that ran.",
+ "name": "Optional name for the workflow run.",
+ "description": "Optional description for the workflow run.",
+ "status": "The status of the run, such as running, success, failure, or aborted.",
+ "concluded": "Whether the run has concluded (aborted, succeeded, or failed).",
+ "startedAt": "When the run started.",
+ "concludedAt": "When the run concluded.",
+ "createdAt": "When the run was created.",
+ "updatedAt": "When the run was last updated.",
+ },
+ },
+ "release_flow_runs": {
+ "description": "A single run of a pipeline stage's release flow, with its status and start and end times.",
+ "docs_url": "https://northflank.com/docs/v1/api/project/pipelines/list-release-flow-runs",
+ "columns": {
+ "id": "Identifier for the release flow run.",
+ "projectId": "Identifier of the project the pipeline belongs to.",
+ "pipelineId": "Identifier of the pipeline the release flow belongs to.",
+ "stage": "The pipeline stage of the release flow: development, staging, or production.",
+ "name": "Optional name for the release flow run.",
+ "description": "Optional description for the release flow run.",
+ "status": "The status of the run, such as running, success, failure, or aborted.",
+ "concluded": "Whether the run has concluded (aborted, succeeded, or failed).",
+ "startedAt": "When the run started.",
+ "concludedAt": "When the run concluded.",
+ "createdAt": "When the run was created.",
+ "updatedAt": "When the run was last updated.",
+ },
+ },
"plans": {
"description": "A Northflank compute plan, which sets the CPU, memory, and price of a workload.",
"docs_url": "https://northflank.com/docs/v1/api/miscellaneous/miscellaneous/list-plans",
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/northflank/northflank.py b/products/warehouse_sources/backend/temporal/data_imports/sources/northflank/northflank.py
index a1db1b46a1e1..0d370d92b1c7 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/northflank/northflank.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/northflank/northflank.py
@@ -17,6 +17,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.res
from products.warehouse_sources.backend.temporal.data_imports.sources.common.safe_point import reach_safe_point
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
from products.warehouse_sources.backend.temporal.data_imports.sources.northflank.settings import (
+ FAN_OUT_PARENTS,
NORTHFLANK_ENDPOINTS,
NorthflankEndpointConfig,
)
@@ -38,9 +39,11 @@ MAX_RATE_LIMIT_SLEEP_SECONDS = 120
class NorthflankResumeConfig:
# Next page of the list in progress. None means that list is done.
cursor: str | None = None
- # Fan-out position. `parent_id` is set only for endpoints that fan out over a second level.
+ # Fan-out position. `parent_id` is set only for endpoints that fan out over a second level, and
+ # `stage` only for endpoints queried once per pipeline stage.
project_id: str | None = None
parent_id: str | None = None
+ stage: str | None = None
class NorthflankRetryableError(Exception):
@@ -169,7 +172,8 @@ def _make_fetch_page(api_token: str, logger: FilteringBoundLogger) -> FetchPageF
)
if not response.ok:
- logger.error(f"Northflank API error: status={response.status_code}, body={response.text}, url={url}")
+ if response.status_code != 404:
+ logger.error(f"Northflank API error: status={response.status_code}, body={response.text}, url={url}")
response.raise_for_status()
return response.json()
@@ -189,6 +193,7 @@ def _iter_pages(
paginated: bool = True,
start_cursor: str | None = None,
checkpoint: Callable[[str | None], None] | None = None,
+ not_found_is_empty: bool = False,
) -> Iterator[list[dict[str, Any]]]:
"""Yield each page's rows for a cursor-paginated list endpoint.
@@ -202,7 +207,12 @@ def _iter_pages(
page_params = dict(params or {})
if paginated:
page_params.update({page_size_param: PAGE_SIZE, "cursor": cursor})
- body = fetch_page(_build_url(path, page_params))
+ try:
+ body = fetch_page(_build_url(path, page_params))
+ except requests.HTTPError as e:
+ if not (not_found_is_empty and e.response is not None and e.response.status_code == 404):
+ raise
+ body = {}
pages_fetched += 1
rows = _extract_rows(body, data_key)
@@ -269,11 +279,13 @@ def get_rows(
manager = resumable_source_manager
resume = manager.load_state() if manager is not None and manager.can_resume() else None
- def checkpoint_at(project_id: str | None = None, parent_id: str | None = None) -> Callable[[str | None], None]:
+ def checkpoint_at(
+ project_id: str | None = None, parent_id: str | None = None, stage: str | None = None
+ ) -> Callable[[str | None], None]:
def checkpoint(next_cursor: str | None) -> None:
if manager is not None:
manager.save_state(
- NorthflankResumeConfig(cursor=next_cursor, project_id=project_id, parent_id=parent_id)
+ NorthflankResumeConfig(cursor=next_cursor, project_id=project_id, parent_id=parent_id, stage=stage)
)
return checkpoint
@@ -339,11 +351,11 @@ def _iter_parent_children(
project_id: str,
logger: FilteringBoundLogger,
resume: NorthflankResumeConfig | None,
- checkpoint_at: Callable[[str | None, str | None], Callable[[str | None], None]],
+ checkpoint_at: Callable[[str | None, str | None, str | None], Callable[[str | None], None]],
) -> Iterator[list[dict[str, Any]]]:
if not config.parent or not config.parent_id_key:
raise ValueError(f"Northflank endpoint {config.name} has no parent to fan out over")
- parent_config = NORTHFLANK_ENDPOINTS[config.parent]
+ parent_config = FAN_OUT_PARENTS[config.parent]
parent_id_key = config.parent_id_key
parents = _iter_ids(
@@ -358,25 +370,33 @@ def _iter_parent_children(
config.parent,
)
resume_parent_id = resume.parent_id if resume else None
+ all_stages: tuple[str | None, ...] = config.stages or (None,)
for parent_id, parent in _skip_to(parents, resume_parent_id, logger, config.parent):
if config.include_parent and not config.include_parent(parent):
continue
- start_cursor: str | None = None
- if resume is not None and parent_id == resume_parent_id:
- if resume.cursor is None:
- continue
- start_cursor = resume.cursor
- for rows in _iter_pages(
- fetch_page,
- config.path.format(project_id=project_id, parent_id=parent_id),
- logger,
- max_pages=MAX_CHILD_PAGES_PER_PROJECT,
- resource=f"{config.name} of {config.parent} {parent_id} in project {project_id}",
- data_key=config.data_key,
- start_cursor=start_cursor,
- checkpoint=checkpoint_at(project_id, parent_id),
- ):
- yield [{**row, "projectId": project_id, parent_id_key: parent_id} for row in rows]
+ resume_in_parent = resume if resume is not None and parent_id == resume_parent_id else None
+ stages = all_stages
+ if resume_in_parent is not None and resume_in_parent.stage in all_stages:
+ stages = all_stages[all_stages.index(resume_in_parent.stage) :]
+ for stage in stages:
+ start_cursor: str | None = None
+ if resume_in_parent is not None and stage == resume_in_parent.stage:
+ if resume_in_parent.cursor is None:
+ continue
+ start_cursor = resume_in_parent.cursor
+ stage_column = {"stage": stage} if stage is not None else {}
+ for rows in _iter_pages(
+ fetch_page,
+ config.path.format(project_id=project_id, parent_id=parent_id, stage=stage),
+ logger,
+ max_pages=MAX_CHILD_PAGES_PER_PROJECT,
+ resource=f"{config.name} of {config.parent} {parent_id} in project {project_id}",
+ data_key=config.data_key,
+ start_cursor=start_cursor,
+ checkpoint=checkpoint_at(project_id, parent_id, stage),
+ not_found_is_empty=config.not_found_is_empty,
+ ):
+ yield [{**row, "projectId": project_id, parent_id_key: parent_id, **stage_column} for row in rows]
def northflank_source(
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/northflank/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/northflank/settings.py
index 6886150a1485..369a9600ecbc 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/northflank/settings.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/northflank/settings.py
@@ -27,6 +27,10 @@ class NorthflankEndpointConfig:
parent_id_key: Optional[str] = None
# Skips parents the child endpoint does not apply to.
include_parent: Optional[Callable[[dict[str, Any]], bool]] = None
+ # Values for a `{stage}` placeholder in the path. Each parent is queried once per stage.
+ stages: tuple[str, ...] = ()
+ # Treat a 404 on a child list as "this parent has none" instead of failing the sync.
+ not_found_is_empty: bool = False
paginated: bool = True
page_size_param: str = "per_page"
params: dict[str, Any] = field(default_factory=dict)
@@ -39,6 +43,9 @@ def _service_has_builds(service: dict[str, Any]) -> bool:
return service.get("serviceType") != "deployment"
+# Every pipeline has these three stages, and each stage can have its own release flow.
+PIPELINE_STAGES = ("development", "staging", "production")
+
# Only the billing usage endpoint documents a server-side timestamp filter (`startTime`); every other
# stream is full refresh. Most resources are project-scoped, so the fan-out streams first list
# projects then query the child endpoint once per project.
@@ -78,6 +85,13 @@ NORTHFLANK_ENDPOINTS: dict[str, NorthflankEndpointConfig] = {
primary_keys=["projectId", "id"],
partition_key="createdAt",
),
+ "pipelines": NorthflankEndpointConfig(
+ name="pipelines",
+ path="/v1/projects/{project_id}/pipelines",
+ data_key="pipelines",
+ fan_out_over_projects=True,
+ primary_keys=["projectId", "id"],
+ ),
"service_builds": NorthflankEndpointConfig(
name="service_builds",
path="/v1/projects/{project_id}/services/{parent_id}/build",
@@ -99,6 +113,41 @@ NORTHFLANK_ENDPOINTS: dict[str, NorthflankEndpointConfig] = {
primary_keys=["projectId", "jobId", "id"],
partition_key="startedAt",
),
+ "job_builds": NorthflankEndpointConfig(
+ name="job_builds",
+ path="/v1/projects/{project_id}/jobs/{parent_id}/build",
+ data_key="builds",
+ fan_out_over_projects=True,
+ parent="jobs",
+ parent_id_key="jobId",
+ primary_keys=["projectId", "jobId", "id"],
+ partition_key="createdAt",
+ # The job list does not say which jobs run an external image and so have no builds.
+ not_found_is_empty=True,
+ ),
+ "workflow_runs": NorthflankEndpointConfig(
+ name="workflow_runs",
+ path="/v1/projects/{project_id}/workflows/{parent_id}/runs",
+ data_key="runs",
+ fan_out_over_projects=True,
+ parent="workflows",
+ parent_id_key="workflowId",
+ primary_keys=["projectId", "workflowId", "id"],
+ partition_key="createdAt",
+ ),
+ "release_flow_runs": NorthflankEndpointConfig(
+ name="release_flow_runs",
+ path="/v1/projects/{project_id}/pipelines/{parent_id}/release-flows/{stage}/runs",
+ data_key="runs",
+ fan_out_over_projects=True,
+ parent="pipelines",
+ parent_id_key="pipelineId",
+ stages=PIPELINE_STAGES,
+ # A stage without a release flow has no runs to list.
+ not_found_is_empty=True,
+ primary_keys=["projectId", "pipelineId", "stage", "id"],
+ partition_key="createdAt",
+ ),
"plans": NorthflankEndpointConfig(
name="plans",
path="/v1/plans",
@@ -121,6 +170,19 @@ NORTHFLANK_ENDPOINTS: dict[str, NorthflankEndpointConfig] = {
ENDPOINTS = tuple(NORTHFLANK_ENDPOINTS.keys())
+# Fan-out parents, including ones that are not synced as tables themselves.
+FAN_OUT_PARENTS: dict[str, NorthflankEndpointConfig] = {
+ **NORTHFLANK_ENDPOINTS,
+ "workflows": NorthflankEndpointConfig(
+ name="workflows",
+ path="/v1/projects/{project_id}/workflows",
+ # The API returns workflows under `data.pipelines`.
+ data_key="pipelines",
+ fan_out_over_projects=True,
+ primary_keys=["projectId", "id"],
+ ),
+}
+
INCREMENTAL_FIELDS: dict[str, list[IncrementalField]] = {
"usage": [
{
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/northflank/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/northflank/source.py
index 780ba3b2142d..c12d23438615 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/northflank/source.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/northflank/source.py
@@ -122,7 +122,7 @@ class NorthflankSource(ResumableSource[NorthflankSourceConfig, NorthflankResumeC
category=DataWarehouseSourceCategory.ENGINEERING___MONITORING,
label="Northflank",
releaseStatus=ReleaseStatus.GA,
- caption="""Enter your Northflank API token to pull your Northflank projects, services, jobs, addons, volumes, builds, job runs, plans, and billing usage into the PostHog Data warehouse.
+ caption="""Enter your Northflank API token to pull your Northflank projects, services, jobs, addons, volumes, pipelines, builds, job runs, workflow runs, release flow runs, plans, and billing usage into the PostHog Data warehouse.
You can create a personal or team API token in your [Northflank account settings](https://app.northflank.com/). Grant it read access to the resources you want to sync. The usage table also needs billing read access.""",
iconPath="/static/services/northflank.svg",
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/northflank/tests/test_northflank.py b/products/warehouse_sources/backend/temporal/data_imports/sources/northflank/tests/test_northflank.py
index 26fb0c16eaa5..a09da1f6c362 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/northflank/tests/test_northflank.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/northflank/tests/test_northflank.py
@@ -271,6 +271,92 @@ class TestFanOut:
assert result.paths == ["/v1/projects", "/v1/projects/proj-b/jobs", *child_paths]
assert result.params("cursor")[2] == resume_state.cursor
+ @parameterized.expand(
+ [
+ ("full_run", None, ["development", "staging", "production"], ["d1", "p1"]),
+ (
+ "resume_mid_stage",
+ NorthflankResumeConfig(
+ cursor="dev-cursor-2", project_id="proj-a", parent_id="pl-1", stage="development"
+ ),
+ ["development", "staging", "production"],
+ ["d1", "p1"],
+ ),
+ (
+ "resume_after_stage",
+ NorthflankResumeConfig(cursor=None, project_id="proj-a", parent_id="pl-1", stage="staging"),
+ ["production"],
+ ["p1"],
+ ),
+ ]
+ )
+ def test_release_flow_runs_walk_each_stage(self, _name, resume_state, stages_requested, expected_ids):
+ stage_path = "/v1/projects/proj-a/pipelines/pl-1/release-flows/{}/runs"
+ result = _driver().run(
+ "release_flow_runs",
+ route(
+ {
+ "/v1/projects": [ScriptedResponse(json=_page("projects", [{"id": "proj-a"}], None))],
+ "/v1/projects/proj-a/pipelines": [
+ ScriptedResponse(json=_page("pipelines", [{"id": "pl-1"}], None))
+ ],
+ stage_path.format("development"): [ScriptedResponse(json=_page("runs", [{"id": "d1"}], None))],
+ # A stage without a release flow 404s.
+ stage_path.format("staging"): [ScriptedResponse(status=404, json={})],
+ stage_path.format("production"): [ScriptedResponse(json=_page("runs", [{"id": "p1"}], None))],
+ }
+ ),
+ resume_state=resume_state,
+ )
+
+ assert result.raised is None
+ assert [row["id"] for row in result.rows] == expected_ids
+ assert all(row["projectId"] == "proj-a" and row["pipelineId"] == "pl-1" for row in result.rows)
+ stage_by_id = {"d1": "development", "p1": "production"}
+ assert [row["stage"] for row in result.rows] == [stage_by_id[row_id] for row_id in expected_ids]
+ assert result.paths[2:] == [stage_path.format(stage) for stage in stages_requested]
+ assert result.params("cursor")[2] == (resume_state.cursor if resume_state else None)
+ assert result.saved_states[-1] == NorthflankResumeConfig(
+ cursor=None, project_id="proj-a", parent_id="pl-1", stage="production"
+ )
+
+ @parameterized.expand(
+ [
+ ("job_builds_skips_missing", "job_builds", "jobs", "jobs", "/v1/projects/proj-a/jobs/j1/build", False),
+ ("job_runs_raises", "job_runs", "jobs", "jobs", "/v1/projects/proj-a/jobs/j1/runs", True),
+ # The workflow list nests its rows under `data.pipelines`.
+ (
+ "workflow_runs_raises",
+ "workflow_runs",
+ "workflows",
+ "pipelines",
+ "/v1/projects/proj-a/workflows/j1/runs",
+ True,
+ ),
+ ]
+ )
+ def test_child_not_found(self, _name, endpoint, parent_resource, parent_data_key, child_path, raises):
+ result = _driver().run(
+ endpoint,
+ route(
+ {
+ "/v1/projects": [ScriptedResponse(json=_page("projects", [{"id": "proj-a"}], None))],
+ f"/v1/projects/proj-a/{parent_resource}": [
+ ScriptedResponse(json=_page(parent_data_key, [{"id": "j1"}], None))
+ ],
+ child_path: [ScriptedResponse(status=404, json={})],
+ }
+ ),
+ )
+
+ assert result.paths == ["/v1/projects", f"/v1/projects/proj-a/{parent_resource}", child_path]
+ assert result.rows == []
+ if raises:
+ assert isinstance(result.raised, requests.HTTPError)
+ assert "404 Client Error" in str(result.raised)
+ else:
+ assert result.raised is None
+
def test_project_without_id_raises(self):
# A project missing its id would silently drop all of its nested resources; fail loudly instead.
result = _driver().run("services", [ScriptedResponse(json=_page("projects", [{"name": "no id"}], None))])