Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(warehouse-sources): add northflank builds, job runs, plans and usage tables (#123164)

Tom Owersf9a0223e3fd6
diff --git a/posthog/test/repo_invariants/dataclass_frozen_baseline.txt b/posthog/test/repo_invariants/dataclass_frozen_baseline.txt
index 286cfbe95fad..94996d749eea 100644
--- a/posthog/test/repo_invariants/dataclass_frozen_baseline.txt
+++ b/posthog/test/repo_invariants/dataclass_frozen_baseline.txt
@@ -791,7 +791,6 @@
1 products/warehouse_sources/backend/temporal/data_imports/sources/newsdata/settings.py
1 products/warehouse_sources/backend/temporal/data_imports/sources/nocrm/nocrm.py
1 products/warehouse_sources/backend/temporal/data_imports/sources/nocrm/settings.py
-1 products/warehouse_sources/backend/temporal/data_imports/sources/northflank/settings.py
1 products/warehouse_sources/backend/temporal/data_imports/sources/northpass_lms/northpass_lms.py
1 products/warehouse_sources/backend/temporal/data_imports/sources/northpass_lms/settings.py
1 products/warehouse_sources/backend/temporal/data_imports/sources/notion/notion.py
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
index 8548914fa45d..8fafd642ab72 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
@@ -8446,14 +8446,14 @@ Note: The API reference is a single large HTML page; extracted all /api/v2/\* pa
## Northflank — gaps
-Today (5): `addons`, `jobs`, `projects`, `services`, `volumes`
+Today (9): `addons`, `job_runs`, `jobs`, `plans`, `projects`, `service_builds`, `services`, `usage`, `volumes`
Diffed against: <https://northflank.com/docs/sitemap.xml>
-- [ ] `GET /v1/projects/{projectId}/services/{serviceId}/build` — build history per service (status, branch, sha, timings) - deployment frequency and build-failure analysis (high)
-- [ ] `GET /v1/projects/{projectId}/jobs/{jobId}/runs` — job run history with success/failure and durations; the jobs table alone has no execution record (high)
-- [ ] `GET /v1/billing/usage` — hourly resource usage entries - the basis for cost attribution per project/service (high)
-- [ ] `GET /v1/plans` — lookup table resolving the plan ids carried on every service, job, addon and volume already synced (vCPU, RAM, hourly cost) (high)
+- [x] `GET /v1/projects/{projectId}/services/{serviceId}/build` — build history per service (status, branch, sha, timings) - deployment frequency and build-failure analysis (high)
+- [x] `GET /v1/projects/{projectId}/jobs/{jobId}/runs` — job run history with success/failure and durations; the jobs table alone has no execution record (high)
+- [x] `GET /v1/billing/usage` — hourly resource usage entries - the basis for cost attribution per project/service (high)
+- [x] `GET /v1/plans` — lookup table resolving the plan ids carried on every service, job, addon and volume already synced (vCPU, RAM, hourly cost) (high)
- [ ] `GET /v1/projects/{projectId}/jobs/{jobId}/build` — build history for jobs, the job-side counterpart to service builds (medium)
- [ ] `GET /v1/projects/{projectId}/workflows/{workflowId}/runs` — workflow execution history, the release automation audit trail (medium)
- [ ] `GET /v1/projects/{projectId}/pipelines` — pipeline definitions that services and release flows belong to - the missing grouping dimension (medium)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/northflank/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/northflank/canonical_descriptions.py
index c52f2467a0e3..583eb0e661e5 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/northflank/canonical_descriptions.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/northflank/canonical_descriptions.py
@@ -76,4 +76,68 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {
"updatedAt": "When the volume was last updated.",
},
},
+ "service_builds": {
+ "description": "A build of a service, one row per build run with its commit, status, and timings.",
+ "docs_url": "https://northflank.com/docs/v1/api/project/services/list-service-builds",
+ "columns": {
+ "id": "Identifier for the build.",
+ "projectId": "Identifier of the project the service belongs to.",
+ "serviceId": "Identifier of the service that was built.",
+ "branch": "Name of the branch the built commit belongs to.",
+ "pullRequestId": "Identifier of the pull request the build was triggered from, if any.",
+ "status": "The status of the build, such as QUEUED, BUILDING, SUCCESS, or FAILURE.",
+ "sha": "The sha of the built commit.",
+ "registry": "Registry details, including the URI to pull the built image from.",
+ "concluded": "Whether the build has finished.",
+ "createdAt": "When the build started.",
+ "success": "Whether the build was successful.",
+ "message": "Description of the build status.",
+ "buildConcludedAt": "When the build concluded, as a Unix timestamp.",
+ },
+ },
+ "job_runs": {
+ "description": "A single run of a job, with its outcome and start and end times.",
+ "docs_url": "https://northflank.com/docs/v1/api/project/jobs/get-job-runs",
+ "columns": {
+ "id": "Identifier for the job run.",
+ "projectId": "Identifier of the project the job belongs to.",
+ "jobId": "Identifier of the job that ran.",
+ "runName": "The name of the job run.",
+ "status": "The status of the run: SUCCESS, RUNNING, or FAILED.",
+ "active": "1 while the run is in progress, otherwise 0.",
+ "concluded": "Whether the run has finished.",
+ "succeeded": "1 if the run completed successfully, otherwise 0.",
+ "failed": "1 if the run failed, otherwise 0.",
+ "backoffLimit": "Number of retry attempts allowed before the run is marked as failed.",
+ "completions": "Number of times the run concluded, successfully or with an error.",
+ "startedAt": "When the run started.",
+ "concludedAt": "When the run concluded.",
+ },
+ },
+ "plans": {
+ "description": "A Northflank compute plan, which sets the CPU, memory, and price of a workload.",
+ "docs_url": "https://northflank.com/docs/v1/api/miscellaneous/miscellaneous/list-plans",
+ "columns": {
+ "id": "Identifier for the plan, as referenced by services, jobs, and addons.",
+ "name": "The name of the plan.",
+ "currency": "Currency code of the plan prices.",
+ "amountPerMonth": "Approximate cost of the plan for 30 days.",
+ "amountPerHour": "Hourly cost of the plan.",
+ "cpuResource": "CPU resource of the plan, in vCPUs.",
+ "ramResource": "Memory resource of the plan, in megabytes.",
+ },
+ },
+ "usage": {
+ "description": "Hourly billing usage and spend for the account, one row per billing hour.",
+ "docs_url": "https://northflank.com/docs/v1/api/team/billing/list-usage",
+ "columns": {
+ "timestamp": "Start of the billing hour, as a Unix timestamp.",
+ "currency": "Currency code of the amounts.",
+ "total": "Total spend for the hour across every category.",
+ "paas": "Price and usage (vCPU, memory, storage, GPU hours) of Northflank-managed compute.",
+ "byoc": "Price and usage of bring-your-own-cloud clusters.",
+ "egressIp": "Price and hours of egress IPs.",
+ "loadBalancer": "Price and hours of load balancers.",
+ },
+ },
}
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/northflank/northflank.py b/products/warehouse_sources/backend/temporal/data_imports/sources/northflank/northflank.py
index 42cda34fc1ee..a1db1b46a1e1 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/northflank/northflank.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/northflank/northflank.py
@@ -7,12 +7,19 @@ from structlog.types import FilteringBoundLogger
from tenacity import RetryCallState, retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
from urllib3.util.retry import Retry
+from posthog.dataclasses import frozen
+
from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import (
make_tracked_session,
retry_with_deadline,
)
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.safe_point import reach_safe_point
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
-from products.warehouse_sources.backend.temporal.data_imports.sources.northflank.settings import NORTHFLANK_ENDPOINTS
+from products.warehouse_sources.backend.temporal.data_imports.sources.northflank.settings import (
+ NORTHFLANK_ENDPOINTS,
+ NorthflankEndpointConfig,
+)
NORTHFLANK_BASE_URL = "https://api.northflank.com"
REQUEST_TIMEOUT_SECONDS = 60
@@ -27,6 +34,15 @@ MAX_CHILD_PAGES_PER_PROJECT = 500
MAX_RATE_LIMIT_SLEEP_SECONDS = 120
+@frozen
+class NorthflankResumeConfig:
+ # Next page of the list in progress. None means that list is done.
+ cursor: str | None = None
+ # Fan-out position. `parent_id` is set only for endpoints that fan out over a second level.
+ project_id: str | None = None
+ parent_id: str | None = None
+
+
class NorthflankRetryableError(Exception):
def __init__(self, message: str, retry_after: int = 0) -> None:
super().__init__(message)
@@ -168,56 +184,108 @@ def _iter_pages(
max_pages: int,
resource: str,
data_key: str,
+ params: dict[str, Any] | None = None,
+ page_size_param: str = "per_page",
+ paginated: bool = True,
+ start_cursor: str | None = None,
+ checkpoint: Callable[[str | None], None] | None = None,
) -> Iterator[list[dict[str, Any]]]:
- """Yield each page's rows for a cursor-paginated list endpoint."""
- cursor: str | None = None
+ """Yield each page's rows for a cursor-paginated list endpoint.
+
+ `checkpoint` receives the cursor of the next page (None after the last page) before the page's
+ rows are yielded.
+ """
+ cursor = start_cursor
pages_fetched = 0
while True:
- params = {"per_page": PAGE_SIZE, "cursor": cursor}
- body = fetch_page(_build_url(path, params))
+ page_params = dict(params or {})
+ if paginated:
+ page_params.update({page_size_param: PAGE_SIZE, "cursor": cursor})
+ body = fetch_page(_build_url(path, page_params))
pages_fetched += 1
rows = _extract_rows(body, data_key)
- if rows:
- yield rows
-
- cursor = _next_cursor(body)
- if not cursor:
- return
-
- if pages_fetched >= max_pages:
+ next_cursor = _next_cursor(body) if paginated else None
+ if next_cursor and pages_fetched >= max_pages:
logger.warning(
f"Northflank: page cap reached for {resource}, stopping pagination. max_pages={max_pages}, path={path}"
)
- return
+ next_cursor = None
+ if checkpoint:
+ checkpoint(next_cursor)
+ if rows:
+ yield rows
+ else:
+ # Fan-out over parents with no children can otherwise make long runs of empty requests.
+ reach_safe_point()
-def _iter_project_ids(fetch_page: FetchPageFn, logger: FilteringBoundLogger) -> Iterator[str]:
- for projects in _iter_pages(
- fetch_page, "/v1/projects", logger, max_pages=MAX_PROJECT_PAGES, resource="projects", data_key="projects"
- ):
- for project in projects:
- project_id = project.get("id")
- if not project_id:
- # A project without an id would silently drop all of its nested resources
- # (services, jobs, addons, volumes), leaving a partial import. Fail loudly instead.
- raise ValueError(f"Northflank: project is missing a required 'id' field: {project!r}")
- yield project_id
+ if not next_cursor:
+ return
+ cursor = next_cursor
+
+
+def _iter_ids(pages: Iterator[list[dict[str, Any]]], resource: str) -> Iterator[tuple[str, dict[str, Any]]]:
+ for page in pages:
+ for row in page:
+ row_id = row.get("id")
+ if not row_id:
+ # A parent without an id would silently drop all of its nested resources, leaving a
+ # partial import. Fail loudly instead.
+ raise ValueError(f"Northflank: {resource} is missing a required 'id' field: {row!r}")
+ yield row_id, row
+
+
+def _skip_to(
+ ids: Iterator[tuple[str, dict[str, Any]]], target: str | None, logger: FilteringBoundLogger, resource: str
+) -> Iterator[tuple[str, dict[str, Any]]]:
+ """Drop ids before `target`, the position a resumed run continues from."""
+ if target is None:
+ yield from ids
+ return
+ found = False
+ for row_id, row in ids:
+ if not found and row_id != target:
+ continue
+ found = True
+ yield row_id, row
+ if not found:
+ logger.warning(f"Northflank: resume position {resource} {target} no longer exists, nothing left to sync")
def get_rows(
api_token: str,
endpoint: str,
logger: FilteringBoundLogger,
+ incremental_start: Optional[int] = None,
+ resumable_source_manager: Optional[ResumableSourceManager[NorthflankResumeConfig]] = None,
) -> Iterator[list[dict[str, Any]]]:
if endpoint not in NORTHFLANK_ENDPOINTS:
raise ValueError(f"Unknown Northflank endpoint: {endpoint}")
config = NORTHFLANK_ENDPOINTS[endpoint]
fetch_page = _make_fetch_page(api_token, logger)
+ manager = resumable_source_manager
+ resume = manager.load_state() if manager is not None and manager.can_resume() else None
+
+ def checkpoint_at(project_id: str | None = None, parent_id: str | None = None) -> Callable[[str | None], None]:
+ def checkpoint(next_cursor: str | None) -> None:
+ if manager is not None:
+ manager.save_state(
+ NorthflankResumeConfig(cursor=next_cursor, project_id=project_id, parent_id=parent_id)
+ )
+
+ return checkpoint
if not config.fan_out_over_projects:
+ if resume is not None and resume.cursor is None:
+ return
+ params = dict(config.params)
+ if config.incremental_param:
+ # Without a start time the API does not document how far back hourly entries go, so a
+ # full refresh asks for everything since the epoch.
+ params[config.incremental_param] = incremental_start if incremental_start is not None else 0
yield from _iter_pages(
fetch_page,
config.path,
@@ -225,35 +293,111 @@ def get_rows(
max_pages=MAX_PROJECT_PAGES,
resource=endpoint,
data_key=config.data_key,
+ params=params,
+ page_size_param=config.page_size_param,
+ paginated=config.paginated,
+ start_cursor=resume.cursor if resume else None,
+ checkpoint=checkpoint_at(),
)
return
- for project_id in _iter_project_ids(fetch_page, logger):
- path = config.path.format(project_id=project_id)
+ projects = _iter_ids(
+ _iter_pages(
+ fetch_page, "/v1/projects", logger, max_pages=MAX_PROJECT_PAGES, resource="projects", data_key="projects"
+ ),
+ "project",
+ )
+ for project_id, _ in _skip_to(projects, resume.project_id if resume else None, logger, "project"):
+ resuming_project = resume is not None and resume.project_id == project_id
+
+ if not config.parent:
+ if resuming_project and resume is not None and resume.cursor is None:
+ continue
+ for rows in _iter_pages(
+ fetch_page,
+ config.path.format(project_id=project_id),
+ logger,
+ max_pages=MAX_CHILD_PAGES_PER_PROJECT,
+ resource=f"{endpoint} of project {project_id}",
+ data_key=config.data_key,
+ start_cursor=resume.cursor if resuming_project and resume is not None else None,
+ checkpoint=checkpoint_at(project_id),
+ ):
+ # Inject the parent project id so the composite primary key is present even on child
+ # objects that don't carry it natively (addons, volumes).
+ yield [{**row, "projectId": project_id} for row in rows]
+ continue
+
+ yield from _iter_parent_children(
+ fetch_page, config, project_id, logger, resume if resuming_project else None, checkpoint_at
+ )
+
+
+def _iter_parent_children(
+ fetch_page: FetchPageFn,
+ config: NorthflankEndpointConfig,
+ project_id: str,
+ logger: FilteringBoundLogger,
+ resume: NorthflankResumeConfig | None,
+ checkpoint_at: Callable[[str | None, str | None], Callable[[str | None], None]],
+) -> Iterator[list[dict[str, Any]]]:
+ if not config.parent or not config.parent_id_key:
+ raise ValueError(f"Northflank endpoint {config.name} has no parent to fan out over")
+ parent_config = NORTHFLANK_ENDPOINTS[config.parent]
+ parent_id_key = config.parent_id_key
+
+ parents = _iter_ids(
+ _iter_pages(
+ fetch_page,
+ parent_config.path.format(project_id=project_id),
+ logger,
+ max_pages=MAX_CHILD_PAGES_PER_PROJECT,
+ resource=f"{config.parent} of project {project_id}",
+ data_key=parent_config.data_key,
+ ),
+ config.parent,
+ )
+ resume_parent_id = resume.parent_id if resume else None
+ for parent_id, parent in _skip_to(parents, resume_parent_id, logger, config.parent):
+ if config.include_parent and not config.include_parent(parent):
+ continue
+ start_cursor: str | None = None
+ if resume is not None and parent_id == resume_parent_id:
+ if resume.cursor is None:
+ continue
+ start_cursor = resume.cursor
for rows in _iter_pages(
fetch_page,
- path,
+ config.path.format(project_id=project_id, parent_id=parent_id),
logger,
max_pages=MAX_CHILD_PAGES_PER_PROJECT,
- resource=f"{endpoint} of project {project_id}",
+ resource=f"{config.name} of {config.parent} {parent_id} in project {project_id}",
data_key=config.data_key,
+ start_cursor=start_cursor,
+ checkpoint=checkpoint_at(project_id, parent_id),
):
- # Inject the parent project id so the composite primary key is present even on child
- # objects that don't carry it natively (addons, volumes).
- yield [{**row, "projectId": project_id} for row in rows]
+ yield [{**row, "projectId": project_id, parent_id_key: parent_id} for row in rows]
def northflank_source(
api_token: str,
endpoint: str,
logger: FilteringBoundLogger,
+ incremental_start: Optional[int] = None,
+ resumable_source_manager: Optional[ResumableSourceManager[NorthflankResumeConfig]] = None,
) -> SourceResponse:
config = NORTHFLANK_ENDPOINTS[endpoint]
partition_key: Optional[str] = config.partition_key
return SourceResponse(
name=endpoint,
- items=lambda: get_rows(api_token=api_token, endpoint=endpoint, logger=logger),
+ items=lambda: get_rows(
+ api_token=api_token,
+ endpoint=endpoint,
+ logger=logger,
+ incremental_start=incremental_start,
+ resumable_source_manager=resumable_source_manager,
+ ),
primary_keys=config.primary_keys,
partition_count=1,
partition_size=1,
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/northflank/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/northflank/settings.py
index 2bc00f8f7f30..6886150a1485 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/northflank/settings.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/northflank/settings.py
@@ -1,10 +1,13 @@
-from dataclasses import dataclass, field
-from typing import Optional
+from collections.abc import Callable
+from dataclasses import field
+from typing import Any, Optional
-from products.warehouse_sources.backend.types import IncrementalField
+from posthog.dataclasses import frozen
+from products.warehouse_sources.backend.types import IncrementalField, IncrementalFieldType
-@dataclass
+
+@frozen
class NorthflankEndpointConfig:
name: str
# Path relative to the API base. Fan-out children use a `{project_id}` placeholder.
@@ -14,14 +17,31 @@ class NorthflankEndpointConfig:
# Whether the endpoint is queried once per project (fan-out) rather than once globally.
fan_out_over_projects: bool = False
primary_keys: list[str] = field(default_factory=lambda: ["id"])
- # Stable creation-time field for datetime partitioning. Only volumes document one; the other
- # list objects expose no timestamps, so they ship unpartitioned.
+ # Stable creation-time field for datetime partitioning. Inventory objects other than volumes
+ # expose no timestamps, so they ship unpartitioned.
partition_key: Optional[str] = None
+ # Project-scoped endpoint to fan out over a second time (e.g. a job's runs). The path then also
+ # carries a `{parent_id}` placeholder, filled from each parent row's `id`.
+ parent: Optional[str] = None
+ # Column the parent id is injected under on child rows.
+ parent_id_key: Optional[str] = None
+ # Skips parents the child endpoint does not apply to.
+ include_parent: Optional[Callable[[dict[str, Any]], bool]] = None
+ paginated: bool = True
+ page_size_param: str = "per_page"
+ params: dict[str, Any] = field(default_factory=dict)
+ # Query param that takes the incremental watermark as a Unix timestamp. Full refresh sends 0.
+ incremental_param: Optional[str] = None
+
+
+def _service_has_builds(service: dict[str, Any]) -> bool:
+ # Deployment services run an image built elsewhere, so they have no builds of their own.
+ return service.get("serviceType") != "deployment"
-# Northflank list endpoints document no server-side timestamp filter (no updated-since/created-after
-# param), so every stream is full refresh. Most resources are project-scoped, so the fan-out streams
-# first list projects then query the child endpoint once per project.
+# Only the billing usage endpoint documents a server-side timestamp filter (`startTime`); every other
+# stream is full refresh. Most resources are project-scoped, so the fan-out streams first list
+# projects then query the child endpoint once per project.
NORTHFLANK_ENDPOINTS: dict[str, NorthflankEndpointConfig] = {
"projects": NorthflankEndpointConfig(
name="projects",
@@ -58,9 +78,56 @@ NORTHFLANK_ENDPOINTS: dict[str, NorthflankEndpointConfig] = {
primary_keys=["projectId", "id"],
partition_key="createdAt",
),
+ "service_builds": NorthflankEndpointConfig(
+ name="service_builds",
+ path="/v1/projects/{project_id}/services/{parent_id}/build",
+ data_key="builds",
+ fan_out_over_projects=True,
+ parent="services",
+ parent_id_key="serviceId",
+ include_parent=_service_has_builds,
+ primary_keys=["projectId", "serviceId", "id"],
+ partition_key="createdAt",
+ ),
+ "job_runs": NorthflankEndpointConfig(
+ name="job_runs",
+ path="/v1/projects/{project_id}/jobs/{parent_id}/runs",
+ data_key="runs",
+ fan_out_over_projects=True,
+ parent="jobs",
+ parent_id_key="jobId",
+ primary_keys=["projectId", "jobId", "id"],
+ partition_key="startedAt",
+ ),
+ "plans": NorthflankEndpointConfig(
+ name="plans",
+ path="/v1/plans",
+ data_key="plans",
+ # A small static catalog that documents no pagination params.
+ paginated=False,
+ ),
+ "usage": NorthflankEndpointConfig(
+ name="usage",
+ path="/v1/billing/usage",
+ data_key="usage",
+ # The default hourly granularity returns one entry per billing hour.
+ primary_keys=["timestamp"],
+ page_size_param="perPage",
+ # Accounts still on the legacy billing contract otherwise get a different row shape.
+ params={"removeLegacyFields": "true"},
+ incremental_param="startTime",
+ ),
}
ENDPOINTS = tuple(NORTHFLANK_ENDPOINTS.keys())
-# No endpoint exposes a server-side timestamp filter, so there are no advertised incremental fields.
-INCREMENTAL_FIELDS: dict[str, list[IncrementalField]] = {}
+INCREMENTAL_FIELDS: dict[str, list[IncrementalField]] = {
+ "usage": [
+ {
+ "label": "timestamp",
+ "type": IncrementalFieldType.DateTime,
+ "field": "timestamp",
+ "field_type": IncrementalFieldType.Integer,
+ },
+ ],
+}
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/northflank/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/northflank/source.py
index e9b846ddd04d..780ba3b2142d 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/northflank/source.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/northflank/source.py
@@ -8,17 +8,19 @@ from products.warehouse_sources.backend.facade.source_config import (
SourceFieldInputConfig,
SourceFieldInputConfigType,
)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.base import FieldType, SimpleSource
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.base import FieldType, ResumableSource
from products.warehouse_sources.backend.temporal.data_imports.sources.common.canonical_descriptions import (
CanonicalDescriptions,
)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.registry import SourceRegistry
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
from products.warehouse_sources.backend.temporal.data_imports.sources.common.schema import SourceSchema
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceInputs, SourceResponse
from products.warehouse_sources.backend.temporal.data_imports.sources.generated_configs.northflank import (
NorthflankSourceConfig,
)
from products.warehouse_sources.backend.temporal.data_imports.sources.northflank.northflank import (
+ NorthflankResumeConfig,
northflank_source,
validate_credentials as validate_northflank_credentials,
)
@@ -30,7 +32,7 @@ from products.warehouse_sources.backend.types import ExternalDataSourceType
@SourceRegistry.register
-class NorthflankSource(SimpleSource[NorthflankSourceConfig]):
+class NorthflankSource(ResumableSource[NorthflankSourceConfig, NorthflankResumeConfig]):
codebase_detection = detect(env=("^NORTHFLANK_",), npm=("@northflank/js-client", "@northflank/cli"))
lists_tables_without_credentials = True # static endpoint catalog — safe for public docs
@@ -64,12 +66,13 @@ class NorthflankSource(SimpleSource[NorthflankSourceConfig]):
force_refresh: bool = False,
api_version: str | None = None,
) -> list[SourceSchema]:
- # No Northflank list endpoint exposes a server-side timestamp filter, so every table is
- # full refresh only.
+ # Only endpoints with a server-side timestamp filter advertise incremental fields; the rest
+ # are full refresh only.
schemas = [
SourceSchema(
name=endpoint,
- supports_incremental=False,
+ supports_incremental=bool(INCREMENTAL_FIELDS.get(endpoint)),
+ # `startTime` is inclusive, so append would duplicate the boundary hour each sync.
supports_append=False,
incremental_fields=INCREMENTAL_FIELDS.get(endpoint, []),
)
@@ -91,11 +94,25 @@ class NorthflankSource(SimpleSource[NorthflankSourceConfig]):
) -> tuple[bool, str | None]:
return validate_northflank_credentials(config.api_token)
- def source_for_pipeline(self, config: NorthflankSourceConfig, inputs: SourceInputs) -> SourceResponse:
+ def get_resumable_source_manager(self, inputs: SourceInputs) -> ResumableSourceManager[NorthflankResumeConfig]:
+ return ResumableSourceManager[NorthflankResumeConfig](inputs, NorthflankResumeConfig)
+
+ def source_for_pipeline(
+ self,
+ config: NorthflankSourceConfig,
+ resumable_source_manager: ResumableSourceManager[NorthflankResumeConfig],
+ inputs: SourceInputs,
+ ) -> SourceResponse:
+ incremental_start: Optional[int] = None
+ if inputs.should_use_incremental_field and inputs.db_incremental_field_last_value is not None:
+ incremental_start = int(inputs.db_incremental_field_last_value)
+
return northflank_source(
api_token=config.api_token,
endpoint=inputs.schema_name,
logger=inputs.logger,
+ incremental_start=incremental_start,
+ resumable_source_manager=resumable_source_manager,
)
@property
@@ -105,9 +122,9 @@ class NorthflankSource(SimpleSource[NorthflankSourceConfig]):
category=DataWarehouseSourceCategory.ENGINEERING___MONITORING,
label="Northflank",
releaseStatus=ReleaseStatus.GA,
- caption="""Enter your Northflank API token to pull your Northflank projects, services, jobs, addons, and volumes into the PostHog Data warehouse.
+ caption="""Enter your Northflank API token to pull your Northflank projects, services, jobs, addons, volumes, builds, job runs, plans, and billing usage into the PostHog Data warehouse.
-You can create a personal or team API token in your [Northflank account settings](https://app.northflank.com/). Grant it read access to the resources you want to sync.""",
+You can create a personal or team API token in your [Northflank account settings](https://app.northflank.com/). Grant it read access to the resources you want to sync. The usage table also needs billing read access.""",
iconPath="/static/services/northflank.svg",
docsUrl="https://posthog.com/docs/cdp/sources/northflank",
keywords=["deployment", "containers", "paas", "devops"],
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/northflank/tests/test_northflank.py b/products/warehouse_sources/backend/temporal/data_imports/sources/northflank/tests/test_northflank.py
index 7bd06c407cc8..26fb0c16eaa5 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/northflank/tests/test_northflank.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/northflank/tests/test_northflank.py
@@ -17,6 +17,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.generated_
NorthflankSourceConfig,
)
from products.warehouse_sources.backend.temporal.data_imports.sources.northflank.northflank import (
+ NorthflankResumeConfig,
NorthflankRetryableError,
_build_url,
_extract_rows,
@@ -188,6 +189,88 @@ class TestFanOut:
assert result.paths == ["/v1/projects", "/v1/projects/proj-a/jobs", "/v1/projects/proj-a/jobs"]
assert result.params("cursor") == [None, None, "jobs-cursor-2"]
+ def test_nested_fan_out_skips_deployment_services_and_injects_parent_ids(self):
+ result = _driver().run(
+ "service_builds",
+ route(
+ {
+ "/v1/projects": [ScriptedResponse(json=_page("projects", [{"id": "proj-a"}], None))],
+ "/v1/projects/proj-a/services": [
+ ScriptedResponse(
+ json=_page(
+ "services",
+ [
+ {"id": "svc-deploy", "serviceType": "deployment"},
+ {"id": "svc-build", "serviceType": "combined"},
+ ],
+ None,
+ )
+ )
+ ],
+ "/v1/projects/proj-a/services/svc-build/build": [
+ ScriptedResponse(json=_page("builds", [{"id": "b1"}], "builds-cursor-2")),
+ ScriptedResponse(json=_page("builds", [{"id": "b2"}], None)),
+ ],
+ }
+ ),
+ )
+
+ assert result.raised is None
+ assert result.rows == [
+ {"id": "b1", "projectId": "proj-a", "serviceId": "svc-build"},
+ {"id": "b2", "projectId": "proj-a", "serviceId": "svc-build"},
+ ]
+ assert result.paths == [
+ "/v1/projects",
+ "/v1/projects/proj-a/services",
+ "/v1/projects/proj-a/services/svc-build/build",
+ "/v1/projects/proj-a/services/svc-build/build",
+ ]
+ assert result.params("cursor") == [None, None, None, "builds-cursor-2"]
+ assert result.saved_states == [
+ NorthflankResumeConfig(cursor="builds-cursor-2", project_id="proj-a", parent_id="svc-build"),
+ NorthflankResumeConfig(cursor=None, project_id="proj-a", parent_id="svc-build"),
+ ]
+
+ @parameterized.expand(
+ [
+ (
+ "mid_parent",
+ NorthflankResumeConfig(cursor="runs-cursor-2", project_id="proj-b", parent_id="job-1"),
+ ["/v1/projects/proj-b/jobs/job-1/runs", "/v1/projects/proj-b/jobs/job-2/runs"],
+ ["r2", "r3"],
+ ),
+ (
+ "parent_done",
+ NorthflankResumeConfig(cursor=None, project_id="proj-b", parent_id="job-1"),
+ ["/v1/projects/proj-b/jobs/job-2/runs"],
+ ["r3"],
+ ),
+ ]
+ )
+ def test_nested_fan_out_resumes_from_saved_position(self, _name, resume_state, child_paths, expected_ids):
+ result = _driver().run(
+ "job_runs",
+ route(
+ {
+ "/v1/projects": [
+ ScriptedResponse(json=_page("projects", [{"id": "proj-a"}, {"id": "proj-b"}], None))
+ ],
+ "/v1/projects/proj-b/jobs": [
+ ScriptedResponse(json=_page("jobs", [{"id": "job-0"}, {"id": "job-1"}, {"id": "job-2"}], None))
+ ],
+ "/v1/projects/proj-b/jobs/job-1/runs": [ScriptedResponse(json=_page("runs", [{"id": "r2"}], None))],
+ "/v1/projects/proj-b/jobs/job-2/runs": [ScriptedResponse(json=_page("runs", [{"id": "r3"}], None))],
+ }
+ ),
+ resume_state=resume_state,
+ )
+
+ assert result.raised is None
+ assert [row["id"] for row in result.rows] == expected_ids
+ assert result.paths == ["/v1/projects", "/v1/projects/proj-b/jobs", *child_paths]
+ assert result.params("cursor")[2] == resume_state.cursor
+
def test_project_without_id_raises(self):
# A project missing its id would silently drop all of its nested resources; fail loudly instead.
result = _driver().run("services", [ScriptedResponse(json=_page("projects", [{"name": "no id"}], None))])
@@ -197,6 +280,36 @@ class TestFanOut:
assert result.paths == ["/v1/projects"]
+class TestTopLevelRequestShaping:
+ @parameterized.expand(
+ [
+ (
+ "usage_full_refresh",
+ "usage",
+ {},
+ {"perPage": "100", "removeLegacyFields": "true", "startTime": "0", "per_page": None},
+ ),
+ (
+ "usage_incremental",
+ "usage",
+ {"incremental_field": "timestamp", "db_incremental_field_last_value": 1767225600.0},
+ {"perPage": "100", "removeLegacyFields": "true", "startTime": "1767225600", "per_page": None},
+ ),
+ ("plans_unpaginated", "plans", {}, {"per_page": None, "cursor": None}),
+ ]
+ )
+ def test_request_params(self, _name, endpoint, inputs, expected_params):
+ result = _driver().run(
+ endpoint, [ScriptedResponse(json={"data": {endpoint: [{"timestamp": 1767225600, "id": "x"}]}})], **inputs
+ )
+
+ assert result.raised is None
+ assert len(result.rows) == 1
+ assert len(result.requests) == 1
+ for name, value in expected_params.items():
+ assert result.params(name) == [value]
+
+
class TestRetryBehavior:
@mock.patch("tenacity.nap.time.sleep")
@mock.patch(PATCH_SESSION)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/northflank/tests/test_northflank_source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/northflank/tests/test_northflank_source.py
index 3bdb51044d33..457509e50b8e 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/northflank/tests/test_northflank_source.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/northflank/tests/test_northflank_source.py
@@ -23,9 +23,9 @@ class TestNorthflankSource:
assert not any(key in other_error for key in non_retryable_errors)
@parameterized.expand([(endpoint,) for endpoint in ENDPOINTS])
- def test_no_endpoint_advertises_incremental(self, endpoint):
- # Northflank exposes no server-side timestamp filter, so every table is full refresh.
+ def test_only_usage_advertises_incremental(self, endpoint):
+ # Only billing usage has a server-side timestamp filter; every other table is full refresh.
schema = next(s for s in self.source.get_schemas(self.config, self.team_id) if s.name == endpoint)
- assert schema.supports_incremental is False
+ assert schema.supports_incremental is (endpoint == "usage")
assert schema.supports_append is False
- assert schema.incremental_fields == []
+ assert bool(schema.incremental_fields) is (endpoint == "usage")