:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(warehouse-sources): add northflank builds, job runs, plans and usage tables (#123164)
Tom Owersf9a0223e3fd6
diff --git a/posthog/test/repo_invariants/dataclass_frozen_baseline.txt b/posthog/test/repo_invariants/dataclass_frozen_baseline.txtindex 286cfbe95fad..94996d749eea 100644--- a/posthog/test/repo_invariants/dataclass_frozen_baseline.txt+++ b/posthog/test/repo_invariants/dataclass_frozen_baseline.txt@@ -791,7 +791,6 @@1 products/warehouse_sources/backend/temporal/data_imports/sources/newsdata/settings.py1 products/warehouse_sources/backend/temporal/data_imports/sources/nocrm/nocrm.py1 products/warehouse_sources/backend/temporal/data_imports/sources/nocrm/settings.py-1 products/warehouse_sources/backend/temporal/data_imports/sources/northflank/settings.py1 products/warehouse_sources/backend/temporal/data_imports/sources/northpass_lms/northpass_lms.py1 products/warehouse_sources/backend/temporal/data_imports/sources/northpass_lms/settings.py1 products/warehouse_sources/backend/temporal/data_imports/sources/notion/notion.pydiff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.mdindex 8548914fa45d..8fafd642ab72 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md@@ -8446,14 +8446,14 @@ Note: The API reference is a single large HTML page; extracted all /api/v2/\* pa## Northflank — gaps-Today (5): `addons`, `jobs`, `projects`, `services`, `volumes`+Today (9): `addons`, `job_runs`, `jobs`, `plans`, `projects`, `service_builds`, `services`, `usage`, `volumes`Diffed against: <https://northflank.com/docs/sitemap.xml>-- [ ] `GET /v1/projects/{projectId}/services/{serviceId}/build` — build history per service (status, branch, sha, timings) - deployment frequency and build-failure analysis (high)-- [ ] `GET /v1/projects/{projectId}/jobs/{jobId}/runs` — job run history with success/failure and durations; the jobs table alone has no execution record (high)-- [ ] `GET /v1/billing/usage` — hourly resource usage entries - the basis for cost attribution per project/service (high)-- [ ] `GET /v1/plans` — lookup table resolving the plan ids carried on every service, job, addon and volume already synced (vCPU, RAM, hourly cost) (high)+- [x] `GET /v1/projects/{projectId}/services/{serviceId}/build` — build history per service (status, branch, sha, timings) - deployment frequency and build-failure analysis (high)+- [x] `GET /v1/projects/{projectId}/jobs/{jobId}/runs` — job run history with success/failure and durations; the jobs table alone has no execution record (high)+- [x] `GET /v1/billing/usage` — hourly resource usage entries - the basis for cost attribution per project/service (high)+- [x] `GET /v1/plans` — lookup table resolving the plan ids carried on every service, job, addon and volume already synced (vCPU, RAM, hourly cost) (high)- [ ] `GET /v1/projects/{projectId}/jobs/{jobId}/build` — build history for jobs, the job-side counterpart to service builds (medium)- [ ] `GET /v1/projects/{projectId}/workflows/{workflowId}/runs` — workflow execution history, the release automation audit trail (medium)- [ ] `GET /v1/projects/{projectId}/pipelines` — pipeline definitions that services and release flows belong to - the missing grouping dimension (medium)diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/northflank/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/northflank/canonical_descriptions.pyindex c52f2467a0e3..583eb0e661e5 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/northflank/canonical_descriptions.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/northflank/canonical_descriptions.py@@ -76,4 +76,68 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {"updatedAt": "When the volume was last updated.",},},+ "service_builds": {+ "description": "A build of a service, one row per build run with its commit, status, and timings.",+ "docs_url": "https://northflank.com/docs/v1/api/project/services/list-service-builds",+ "columns": {+ "id": "Identifier for the build.",+ "projectId": "Identifier of the project the service belongs to.",+ "serviceId": "Identifier of the service that was built.",+ "branch": "Name of the branch the built commit belongs to.",+ "pullRequestId": "Identifier of the pull request the build was triggered from, if any.",+ "status": "The status of the build, such as QUEUED, BUILDING, SUCCESS, or FAILURE.",+ "sha": "The sha of the built commit.",+ "registry": "Registry details, including the URI to pull the built image from.",+ "concluded": "Whether the build has finished.",+ "createdAt": "When the build started.",+ "success": "Whether the build was successful.",+ "message": "Description of the build status.",+ "buildConcludedAt": "When the build concluded, as a Unix timestamp.",+ },+ },+ "job_runs": {+ "description": "A single run of a job, with its outcome and start and end times.",+ "docs_url": "https://northflank.com/docs/v1/api/project/jobs/get-job-runs",+ "columns": {+ "id": "Identifier for the job run.",+ "projectId": "Identifier of the project the job belongs to.",+ "jobId": "Identifier of the job that ran.",+ "runName": "The name of the job run.",+ "status": "The status of the run: SUCCESS, RUNNING, or FAILED.",+ "active": "1 while the run is in progress, otherwise 0.",+ "concluded": "Whether the run has finished.",+ "succeeded": "1 if the run completed successfully, otherwise 0.",+ "failed": "1 if the run failed, otherwise 0.",+ "backoffLimit": "Number of retry attempts allowed before the run is marked as failed.",+ "completions": "Number of times the run concluded, successfully or with an error.",+ "startedAt": "When the run started.",+ "concludedAt": "When the run concluded.",+ },+ },+ "plans": {+ "description": "A Northflank compute plan, which sets the CPU, memory, and price of a workload.",+ "docs_url": "https://northflank.com/docs/v1/api/miscellaneous/miscellaneous/list-plans",+ "columns": {+ "id": "Identifier for the plan, as referenced by services, jobs, and addons.",+ "name": "The name of the plan.",+ "currency": "Currency code of the plan prices.",+ "amountPerMonth": "Approximate cost of the plan for 30 days.",+ "amountPerHour": "Hourly cost of the plan.",+ "cpuResource": "CPU resource of the plan, in vCPUs.",+ "ramResource": "Memory resource of the plan, in megabytes.",+ },+ },+ "usage": {+ "description": "Hourly billing usage and spend for the account, one row per billing hour.",+ "docs_url": "https://northflank.com/docs/v1/api/team/billing/list-usage",+ "columns": {+ "timestamp": "Start of the billing hour, as a Unix timestamp.",+ "currency": "Currency code of the amounts.",+ "total": "Total spend for the hour across every category.",+ "paas": "Price and usage (vCPU, memory, storage, GPU hours) of Northflank-managed compute.",+ "byoc": "Price and usage of bring-your-own-cloud clusters.",+ "egressIp": "Price and hours of egress IPs.",+ "loadBalancer": "Price and hours of load balancers.",+ },+ },}diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/northflank/northflank.py b/products/warehouse_sources/backend/temporal/data_imports/sources/northflank/northflank.pyindex 42cda34fc1ee..a1db1b46a1e1 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/northflank/northflank.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/northflank/northflank.py@@ -7,12 +7,19 @@ from structlog.types import FilteringBoundLoggerfrom tenacity import RetryCallState, retry_if_exception_type, stop_after_attempt, wait_exponential_jitterfrom urllib3.util.retry import Retry+from posthog.dataclasses import frozen+from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import (make_tracked_session,retry_with_deadline,)+from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager+from products.warehouse_sources.backend.temporal.data_imports.sources.common.safe_point import reach_safe_pointfrom products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse-from products.warehouse_sources.backend.temporal.data_imports.sources.northflank.settings import NORTHFLANK_ENDPOINTS+from products.warehouse_sources.backend.temporal.data_imports.sources.northflank.settings import (+ NORTHFLANK_ENDPOINTS,+ NorthflankEndpointConfig,+)NORTHFLANK_BASE_URL = "https://api.northflank.com"REQUEST_TIMEOUT_SECONDS = 60@@ -27,6 +34,15 @@ MAX_CHILD_PAGES_PER_PROJECT = 500MAX_RATE_LIMIT_SLEEP_SECONDS = 120+@frozen+class NorthflankResumeConfig:+ # Next page of the list in progress. None means that list is done.+ cursor: str | None = None+ # Fan-out position. `parent_id` is set only for endpoints that fan out over a second level.+ project_id: str | None = None+ parent_id: str | None = None++class NorthflankRetryableError(Exception):def __init__(self, message: str, retry_after: int = 0) -> None:super().__init__(message)@@ -168,56 +184,108 @@ def _iter_pages(max_pages: int,resource: str,data_key: str,+ params: dict[str, Any] | None = None,+ page_size_param: str = "per_page",+ paginated: bool = True,+ start_cursor: str | None = None,+ checkpoint: Callable[[str | None], None] | None = None,) -> Iterator[list[dict[str, Any]]]:- """Yield each page's rows for a cursor-paginated list endpoint."""- cursor: str | None = None+ """Yield each page's rows for a cursor-paginated list endpoint.++ `checkpoint` receives the cursor of the next page (None after the last page) before the page's+ rows are yielded.+ """+ cursor = start_cursorpages_fetched = 0while True:- params = {"per_page": PAGE_SIZE, "cursor": cursor}- body = fetch_page(_build_url(path, params))+ page_params = dict(params or {})+ if paginated:+ page_params.update({page_size_param: PAGE_SIZE, "cursor": cursor})+ body = fetch_page(_build_url(path, page_params))pages_fetched += 1rows = _extract_rows(body, data_key)- if rows:- yield rows-- cursor = _next_cursor(body)- if not cursor:- return-- if pages_fetched >= max_pages:+ next_cursor = _next_cursor(body) if paginated else None+ if next_cursor and pages_fetched >= max_pages:logger.warning(f"Northflank: page cap reached for {resource}, stopping pagination. max_pages={max_pages}, path={path}")- return+ next_cursor = None+ if checkpoint:+ checkpoint(next_cursor)+ if rows:+ yield rows+ else:+ # Fan-out over parents with no children can otherwise make long runs of empty requests.+ reach_safe_point()-def _iter_project_ids(fetch_page: FetchPageFn, logger: FilteringBoundLogger) -> Iterator[str]:- for projects in _iter_pages(- fetch_page, "/v1/projects", logger, max_pages=MAX_PROJECT_PAGES, resource="projects", data_key="projects"- ):- for project in projects:- project_id = project.get("id")- if not project_id:- # A project without an id would silently drop all of its nested resources- # (services, jobs, addons, volumes), leaving a partial import. Fail loudly instead.- raise ValueError(f"Northflank: project is missing a required 'id' field: {project!r}")- yield project_id+ if not next_cursor:+ return+ cursor = next_cursor+++def _iter_ids(pages: Iterator[list[dict[str, Any]]], resource: str) -> Iterator[tuple[str, dict[str, Any]]]:+ for page in pages:+ for row in page:+ row_id = row.get("id")+ if not row_id:+ # A parent without an id would silently drop all of its nested resources, leaving a+ # partial import. Fail loudly instead.+ raise ValueError(f"Northflank: {resource} is missing a required 'id' field: {row!r}")+ yield row_id, row+++def _skip_to(+ ids: Iterator[tuple[str, dict[str, Any]]], target: str | None, logger: FilteringBoundLogger, resource: str+) -> Iterator[tuple[str, dict[str, Any]]]:+ """Drop ids before `target`, the position a resumed run continues from."""+ if target is None:+ yield from ids+ return+ found = False+ for row_id, row in ids:+ if not found and row_id != target:+ continue+ found = True+ yield row_id, row+ if not found:+ logger.warning(f"Northflank: resume position {resource} {target} no longer exists, nothing left to sync")def get_rows(api_token: str,endpoint: str,logger: FilteringBoundLogger,+ incremental_start: Optional[int] = None,+ resumable_source_manager: Optional[ResumableSourceManager[NorthflankResumeConfig]] = None,) -> Iterator[list[dict[str, Any]]]:if endpoint not in NORTHFLANK_ENDPOINTS:raise ValueError(f"Unknown Northflank endpoint: {endpoint}")config = NORTHFLANK_ENDPOINTS[endpoint]fetch_page = _make_fetch_page(api_token, logger)+ manager = resumable_source_manager+ resume = manager.load_state() if manager is not None and manager.can_resume() else None++ def checkpoint_at(project_id: str | None = None, parent_id: str | None = None) -> Callable[[str | None], None]:+ def checkpoint(next_cursor: str | None) -> None:+ if manager is not None:+ manager.save_state(+ NorthflankResumeConfig(cursor=next_cursor, project_id=project_id, parent_id=parent_id)+ )++ return checkpointif not config.fan_out_over_projects:+ if resume is not None and resume.cursor is None:+ return+ params = dict(config.params)+ if config.incremental_param:+ # Without a start time the API does not document how far back hourly entries go, so a+ # full refresh asks for everything since the epoch.+ params[config.incremental_param] = incremental_start if incremental_start is not None else 0yield from _iter_pages(fetch_page,config.path,@@ -225,35 +293,111 @@ def get_rows(max_pages=MAX_PROJECT_PAGES,resource=endpoint,data_key=config.data_key,+ params=params,+ page_size_param=config.page_size_param,+ paginated=config.paginated,+ start_cursor=resume.cursor if resume else None,+ checkpoint=checkpoint_at(),)return- for project_id in _iter_project_ids(fetch_page, logger):- path = config.path.format(project_id=project_id)+ projects = _iter_ids(+ _iter_pages(+ fetch_page, "/v1/projects", logger, max_pages=MAX_PROJECT_PAGES, resource="projects", data_key="projects"+ ),+ "project",+ )+ for project_id, _ in _skip_to(projects, resume.project_id if resume else None, logger, "project"):+ resuming_project = resume is not None and resume.project_id == project_id++ if not config.parent:+ if resuming_project and resume is not None and resume.cursor is None:+ continue+ for rows in _iter_pages(+ fetch_page,+ config.path.format(project_id=project_id),+ logger,+ max_pages=MAX_CHILD_PAGES_PER_PROJECT,+ resource=f"{endpoint} of project {project_id}",+ data_key=config.data_key,+ start_cursor=resume.cursor if resuming_project and resume is not None else None,+ checkpoint=checkpoint_at(project_id),+ ):+ # Inject the parent project id so the composite primary key is present even on child+ # objects that don't carry it natively (addons, volumes).+ yield [{**row, "projectId": project_id} for row in rows]+ continue++ yield from _iter_parent_children(+ fetch_page, config, project_id, logger, resume if resuming_project else None, checkpoint_at+ )+++def _iter_parent_children(+ fetch_page: FetchPageFn,+ config: NorthflankEndpointConfig,+ project_id: str,+ logger: FilteringBoundLogger,+ resume: NorthflankResumeConfig | None,+ checkpoint_at: Callable[[str | None, str | None], Callable[[str | None], None]],+) -> Iterator[list[dict[str, Any]]]:+ if not config.parent or not config.parent_id_key:+ raise ValueError(f"Northflank endpoint {config.name} has no parent to fan out over")+ parent_config = NORTHFLANK_ENDPOINTS[config.parent]+ parent_id_key = config.parent_id_key++ parents = _iter_ids(+ _iter_pages(+ fetch_page,+ parent_config.path.format(project_id=project_id),+ logger,+ max_pages=MAX_CHILD_PAGES_PER_PROJECT,+ resource=f"{config.parent} of project {project_id}",+ data_key=parent_config.data_key,+ ),+ config.parent,+ )+ resume_parent_id = resume.parent_id if resume else None+ for parent_id, parent in _skip_to(parents, resume_parent_id, logger, config.parent):+ if config.include_parent and not config.include_parent(parent):+ continue+ start_cursor: str | None = None+ if resume is not None and parent_id == resume_parent_id:+ if resume.cursor is None:+ continue+ start_cursor = resume.cursorfor rows in _iter_pages(fetch_page,- path,+ config.path.format(project_id=project_id, parent_id=parent_id),logger,max_pages=MAX_CHILD_PAGES_PER_PROJECT,- resource=f"{endpoint} of project {project_id}",+ resource=f"{config.name} of {config.parent} {parent_id} in project {project_id}",data_key=config.data_key,+ start_cursor=start_cursor,+ checkpoint=checkpoint_at(project_id, parent_id),):- # Inject the parent project id so the composite primary key is present even on child- # objects that don't carry it natively (addons, volumes).- yield [{**row, "projectId": project_id} for row in rows]+ yield [{**row, "projectId": project_id, parent_id_key: parent_id} for row in rows]def northflank_source(api_token: str,endpoint: str,logger: FilteringBoundLogger,+ incremental_start: Optional[int] = None,+ resumable_source_manager: Optional[ResumableSourceManager[NorthflankResumeConfig]] = None,) -> SourceResponse:config = NORTHFLANK_ENDPOINTS[endpoint]partition_key: Optional[str] = config.partition_keyreturn SourceResponse(name=endpoint,- items=lambda: get_rows(api_token=api_token, endpoint=endpoint, logger=logger),+ items=lambda: get_rows(+ api_token=api_token,+ endpoint=endpoint,+ logger=logger,+ incremental_start=incremental_start,+ resumable_source_manager=resumable_source_manager,+ ),primary_keys=config.primary_keys,partition_count=1,partition_size=1,diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/northflank/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/northflank/settings.pyindex 2bc00f8f7f30..6886150a1485 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/northflank/settings.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/northflank/settings.py@@ -1,10 +1,13 @@-from dataclasses import dataclass, field-from typing import Optional+from collections.abc import Callable+from dataclasses import field+from typing import Any, Optional-from products.warehouse_sources.backend.types import IncrementalField+from posthog.dataclasses import frozen+from products.warehouse_sources.backend.types import IncrementalField, IncrementalFieldType-@dataclass++@frozenclass NorthflankEndpointConfig:name: str# Path relative to the API base. Fan-out children use a `{project_id}` placeholder.@@ -14,14 +17,31 @@ class NorthflankEndpointConfig:# Whether the endpoint is queried once per project (fan-out) rather than once globally.fan_out_over_projects: bool = Falseprimary_keys: list[str] = field(default_factory=lambda: ["id"])- # Stable creation-time field for datetime partitioning. Only volumes document one; the other- # list objects expose no timestamps, so they ship unpartitioned.+ # Stable creation-time field for datetime partitioning. Inventory objects other than volumes+ # expose no timestamps, so they ship unpartitioned.partition_key: Optional[str] = None+ # Project-scoped endpoint to fan out over a second time (e.g. a job's runs). The path then also+ # carries a `{parent_id}` placeholder, filled from each parent row's `id`.+ parent: Optional[str] = None+ # Column the parent id is injected under on child rows.+ parent_id_key: Optional[str] = None+ # Skips parents the child endpoint does not apply to.+ include_parent: Optional[Callable[[dict[str, Any]], bool]] = None+ paginated: bool = True+ page_size_param: str = "per_page"+ params: dict[str, Any] = field(default_factory=dict)+ # Query param that takes the incremental watermark as a Unix timestamp. Full refresh sends 0.+ incremental_param: Optional[str] = None+++def _service_has_builds(service: dict[str, Any]) -> bool:+ # Deployment services run an image built elsewhere, so they have no builds of their own.+ return service.get("serviceType") != "deployment"-# Northflank list endpoints document no server-side timestamp filter (no updated-since/created-after-# param), so every stream is full refresh. Most resources are project-scoped, so the fan-out streams-# first list projects then query the child endpoint once per project.+# Only the billing usage endpoint documents a server-side timestamp filter (`startTime`); every other+# stream is full refresh. Most resources are project-scoped, so the fan-out streams first list+# projects then query the child endpoint once per project.NORTHFLANK_ENDPOINTS: dict[str, NorthflankEndpointConfig] = {"projects": NorthflankEndpointConfig(name="projects",@@ -58,9 +78,56 @@ NORTHFLANK_ENDPOINTS: dict[str, NorthflankEndpointConfig] = {primary_keys=["projectId", "id"],partition_key="createdAt",),+ "service_builds": NorthflankEndpointConfig(+ name="service_builds",+ path="/v1/projects/{project_id}/services/{parent_id}/build",+ data_key="builds",+ fan_out_over_projects=True,+ parent="services",+ parent_id_key="serviceId",+ include_parent=_service_has_builds,+ primary_keys=["projectId", "serviceId", "id"],+ partition_key="createdAt",+ ),+ "job_runs": NorthflankEndpointConfig(+ name="job_runs",+ path="/v1/projects/{project_id}/jobs/{parent_id}/runs",+ data_key="runs",+ fan_out_over_projects=True,+ parent="jobs",+ parent_id_key="jobId",+ primary_keys=["projectId", "jobId", "id"],+ partition_key="startedAt",+ ),+ "plans": NorthflankEndpointConfig(+ name="plans",+ path="/v1/plans",+ data_key="plans",+ # A small static catalog that documents no pagination params.+ paginated=False,+ ),+ "usage": NorthflankEndpointConfig(+ name="usage",+ path="/v1/billing/usage",+ data_key="usage",+ # The default hourly granularity returns one entry per billing hour.+ primary_keys=["timestamp"],+ page_size_param="perPage",+ # Accounts still on the legacy billing contract otherwise get a different row shape.+ params={"removeLegacyFields": "true"},+ incremental_param="startTime",+ ),}ENDPOINTS = tuple(NORTHFLANK_ENDPOINTS.keys())-# No endpoint exposes a server-side timestamp filter, so there are no advertised incremental fields.-INCREMENTAL_FIELDS: dict[str, list[IncrementalField]] = {}+INCREMENTAL_FIELDS: dict[str, list[IncrementalField]] = {+ "usage": [+ {+ "label": "timestamp",+ "type": IncrementalFieldType.DateTime,+ "field": "timestamp",+ "field_type": IncrementalFieldType.Integer,+ },+ ],+}diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/northflank/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/northflank/source.pyindex e9b846ddd04d..780ba3b2142d 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/northflank/source.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/northflank/source.py@@ -8,17 +8,19 @@ from products.warehouse_sources.backend.facade.source_config import (SourceFieldInputConfig,SourceFieldInputConfigType,)-from products.warehouse_sources.backend.temporal.data_imports.sources.common.base import FieldType, SimpleSource+from products.warehouse_sources.backend.temporal.data_imports.sources.common.base import FieldType, ResumableSourcefrom products.warehouse_sources.backend.temporal.data_imports.sources.common.canonical_descriptions import (CanonicalDescriptions,)from products.warehouse_sources.backend.temporal.data_imports.sources.common.registry import SourceRegistry+from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManagerfrom products.warehouse_sources.backend.temporal.data_imports.sources.common.schema import SourceSchemafrom products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceInputs, SourceResponsefrom products.warehouse_sources.backend.temporal.data_imports.sources.generated_configs.northflank import (NorthflankSourceConfig,)from products.warehouse_sources.backend.temporal.data_imports.sources.northflank.northflank import (+ NorthflankResumeConfig,northflank_source,validate_credentials as validate_northflank_credentials,)@@ -30,7 +32,7 @@ from products.warehouse_sources.backend.types import ExternalDataSourceType@SourceRegistry.register-class NorthflankSource(SimpleSource[NorthflankSourceConfig]):+class NorthflankSource(ResumableSource[NorthflankSourceConfig, NorthflankResumeConfig]):codebase_detection = detect(env=("^NORTHFLANK_",), npm=("@northflank/js-client", "@northflank/cli"))lists_tables_without_credentials = True # static endpoint catalog — safe for public docs@@ -64,12 +66,13 @@ class NorthflankSource(SimpleSource[NorthflankSourceConfig]):force_refresh: bool = False,api_version: str | None = None,) -> list[SourceSchema]:- # No Northflank list endpoint exposes a server-side timestamp filter, so every table is- # full refresh only.+ # Only endpoints with a server-side timestamp filter advertise incremental fields; the rest+ # are full refresh only.schemas = [SourceSchema(name=endpoint,- supports_incremental=False,+ supports_incremental=bool(INCREMENTAL_FIELDS.get(endpoint)),+ # `startTime` is inclusive, so append would duplicate the boundary hour each sync.supports_append=False,incremental_fields=INCREMENTAL_FIELDS.get(endpoint, []),)@@ -91,11 +94,25 @@ class NorthflankSource(SimpleSource[NorthflankSourceConfig]):) -> tuple[bool, str | None]:return validate_northflank_credentials(config.api_token)- def source_for_pipeline(self, config: NorthflankSourceConfig, inputs: SourceInputs) -> SourceResponse:+ def get_resumable_source_manager(self, inputs: SourceInputs) -> ResumableSourceManager[NorthflankResumeConfig]:+ return ResumableSourceManager[NorthflankResumeConfig](inputs, NorthflankResumeConfig)++ def source_for_pipeline(+ self,+ config: NorthflankSourceConfig,+ resumable_source_manager: ResumableSourceManager[NorthflankResumeConfig],+ inputs: SourceInputs,+ ) -> SourceResponse:+ incremental_start: Optional[int] = None+ if inputs.should_use_incremental_field and inputs.db_incremental_field_last_value is not None:+ incremental_start = int(inputs.db_incremental_field_last_value)+return northflank_source(api_token=config.api_token,endpoint=inputs.schema_name,logger=inputs.logger,+ incremental_start=incremental_start,+ resumable_source_manager=resumable_source_manager,)@property@@ -105,9 +122,9 @@ class NorthflankSource(SimpleSource[NorthflankSourceConfig]):category=DataWarehouseSourceCategory.ENGINEERING___MONITORING,label="Northflank",releaseStatus=ReleaseStatus.GA,- caption="""Enter your Northflank API token to pull your Northflank projects, services, jobs, addons, and volumes into the PostHog Data warehouse.+ caption="""Enter your Northflank API token to pull your Northflank projects, services, jobs, addons, volumes, builds, job runs, plans, and billing usage into the PostHog Data warehouse.-You can create a personal or team API token in your [Northflank account settings](https://app.northflank.com/). Grant it read access to the resources you want to sync.""",+You can create a personal or team API token in your [Northflank account settings](https://app.northflank.com/). Grant it read access to the resources you want to sync. The usage table also needs billing read access.""",iconPath="/static/services/northflank.svg",docsUrl="https://posthog.com/docs/cdp/sources/northflank",keywords=["deployment", "containers", "paas", "devops"],diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/northflank/tests/test_northflank.py b/products/warehouse_sources/backend/temporal/data_imports/sources/northflank/tests/test_northflank.pyindex 7bd06c407cc8..26fb0c16eaa5 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/northflank/tests/test_northflank.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/northflank/tests/test_northflank.py@@ -17,6 +17,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.generated_NorthflankSourceConfig,)from products.warehouse_sources.backend.temporal.data_imports.sources.northflank.northflank import (+ NorthflankResumeConfig,NorthflankRetryableError,_build_url,_extract_rows,@@ -188,6 +189,88 @@ class TestFanOut:assert result.paths == ["/v1/projects", "/v1/projects/proj-a/jobs", "/v1/projects/proj-a/jobs"]assert result.params("cursor") == [None, None, "jobs-cursor-2"]+ def test_nested_fan_out_skips_deployment_services_and_injects_parent_ids(self):+ result = _driver().run(+ "service_builds",+ route(+ {+ "/v1/projects": [ScriptedResponse(json=_page("projects", [{"id": "proj-a"}], None))],+ "/v1/projects/proj-a/services": [+ ScriptedResponse(+ json=_page(+ "services",+ [+ {"id": "svc-deploy", "serviceType": "deployment"},+ {"id": "svc-build", "serviceType": "combined"},+ ],+ None,+ )+ )+ ],+ "/v1/projects/proj-a/services/svc-build/build": [+ ScriptedResponse(json=_page("builds", [{"id": "b1"}], "builds-cursor-2")),+ ScriptedResponse(json=_page("builds", [{"id": "b2"}], None)),+ ],+ }+ ),+ )++ assert result.raised is None+ assert result.rows == [+ {"id": "b1", "projectId": "proj-a", "serviceId": "svc-build"},+ {"id": "b2", "projectId": "proj-a", "serviceId": "svc-build"},+ ]+ assert result.paths == [+ "/v1/projects",+ "/v1/projects/proj-a/services",+ "/v1/projects/proj-a/services/svc-build/build",+ "/v1/projects/proj-a/services/svc-build/build",+ ]+ assert result.params("cursor") == [None, None, None, "builds-cursor-2"]+ assert result.saved_states == [+ NorthflankResumeConfig(cursor="builds-cursor-2", project_id="proj-a", parent_id="svc-build"),+ NorthflankResumeConfig(cursor=None, project_id="proj-a", parent_id="svc-build"),+ ]++ @parameterized.expand(+ [+ (+ "mid_parent",+ NorthflankResumeConfig(cursor="runs-cursor-2", project_id="proj-b", parent_id="job-1"),+ ["/v1/projects/proj-b/jobs/job-1/runs", "/v1/projects/proj-b/jobs/job-2/runs"],+ ["r2", "r3"],+ ),+ (+ "parent_done",+ NorthflankResumeConfig(cursor=None, project_id="proj-b", parent_id="job-1"),+ ["/v1/projects/proj-b/jobs/job-2/runs"],+ ["r3"],+ ),+ ]+ )+ def test_nested_fan_out_resumes_from_saved_position(self, _name, resume_state, child_paths, expected_ids):+ result = _driver().run(+ "job_runs",+ route(+ {+ "/v1/projects": [+ ScriptedResponse(json=_page("projects", [{"id": "proj-a"}, {"id": "proj-b"}], None))+ ],+ "/v1/projects/proj-b/jobs": [+ ScriptedResponse(json=_page("jobs", [{"id": "job-0"}, {"id": "job-1"}, {"id": "job-2"}], None))+ ],+ "/v1/projects/proj-b/jobs/job-1/runs": [ScriptedResponse(json=_page("runs", [{"id": "r2"}], None))],+ "/v1/projects/proj-b/jobs/job-2/runs": [ScriptedResponse(json=_page("runs", [{"id": "r3"}], None))],+ }+ ),+ resume_state=resume_state,+ )++ assert result.raised is None+ assert [row["id"] for row in result.rows] == expected_ids+ assert result.paths == ["/v1/projects", "/v1/projects/proj-b/jobs", *child_paths]+ assert result.params("cursor")[2] == resume_state.cursor+def test_project_without_id_raises(self):# A project missing its id would silently drop all of its nested resources; fail loudly instead.result = _driver().run("services", [ScriptedResponse(json=_page("projects", [{"name": "no id"}], None))])@@ -197,6 +280,36 @@ class TestFanOut:assert result.paths == ["/v1/projects"]+class TestTopLevelRequestShaping:+ @parameterized.expand(+ [+ (+ "usage_full_refresh",+ "usage",+ {},+ {"perPage": "100", "removeLegacyFields": "true", "startTime": "0", "per_page": None},+ ),+ (+ "usage_incremental",+ "usage",+ {"incremental_field": "timestamp", "db_incremental_field_last_value": 1767225600.0},+ {"perPage": "100", "removeLegacyFields": "true", "startTime": "1767225600", "per_page": None},+ ),+ ("plans_unpaginated", "plans", {}, {"per_page": None, "cursor": None}),+ ]+ )+ def test_request_params(self, _name, endpoint, inputs, expected_params):+ result = _driver().run(+ endpoint, [ScriptedResponse(json={"data": {endpoint: [{"timestamp": 1767225600, "id": "x"}]}})], **inputs+ )++ assert result.raised is None+ assert len(result.rows) == 1+ assert len(result.requests) == 1+ for name, value in expected_params.items():+ assert result.params(name) == [value]++class TestRetryBehavior:@mock.patch("tenacity.nap.time.sleep")@mock.patch(PATCH_SESSION)diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/northflank/tests/test_northflank_source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/northflank/tests/test_northflank_source.pyindex 3bdb51044d33..457509e50b8e 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/northflank/tests/test_northflank_source.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/northflank/tests/test_northflank_source.py@@ -23,9 +23,9 @@ class TestNorthflankSource:assert not any(key in other_error for key in non_retryable_errors)@parameterized.expand([(endpoint,) for endpoint in ENDPOINTS])- def test_no_endpoint_advertises_incremental(self, endpoint):- # Northflank exposes no server-side timestamp filter, so every table is full refresh.+ def test_only_usage_advertises_incremental(self, endpoint):+ # Only billing usage has a server-side timestamp filter; every other table is full refresh.schema = next(s for s in self.source.get_schemas(self.config, self.team_id) if s.name == endpoint)- assert schema.supports_incremental is False+ assert schema.supports_incremental is (endpoint == "usage")assert schema.supports_append is False- assert schema.incremental_fields == []+ assert bool(schema.incremental_fields) is (endpoint == "usage")