Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(data-warehouse): add 4 missing tables to the qdrant source (#126781)

Tom Owers7d0e17d2ac42
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
index c55d93e5297e..cebf4c57e9dc 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
@@ -9892,10 +9892,10 @@ Today (4): `backup_restores`, `backup_schedules`, `backups`, `clusters`
Diffed against: <https://raw.githubusercontent.com/qdrant/qdrant-cloud-public-api/main/gen/openapiv2/qdrant/cloud/monitoring/v1/monitoring.swagger.json>
- [ ] `Packages (GET /api/booking/v1/accounts/{accountId}/packages)` — resolve cluster package identifiers into resource sizes and pricing dimensions (high)
-- [ ] `Cloud providers (GET /api/platform/v1/accounts/{accountId}/cloud-providers)` — resolve cloud provider identifiers stored on clusters (high)
-- [ ] `Cloud provider regions (GET /api/platform/v1/accounts/{accountId}/cloud-providers/{cloudProviderId}/regions)` — resolve cluster region identifiers for geographic and capacity analysis (high)
-- [ ] `Cluster usage metrics (GET /api/monitoring/v1/accounts/{accountId}/cluster/{clusterId}/usage-metrics)` — measure resource utilization over time and identify capacity bottlenecks (high)
-- [ ] `Metering entries (GET /api/metering/v1/accounts/{accountId}/meterings/{year}/{month})` — analyze monthly usage charges and attribute consumption to resources (high)
+- [x] `Cloud providers (GET /api/platform/v1/accounts/{accountId}/cloud-providers)` — resolve cloud provider identifiers stored on clusters (high)
+- [x] `Cloud provider regions (GET /api/platform/v1/accounts/{accountId}/cloud-providers/{cloudProviderId}/regions)` — resolve cluster region identifiers for geographic and capacity analysis (high)
+- [x] `Cluster usage metrics (GET /api/monitoring/v1/accounts/{accountId}/cluster/{clusterId}/usage-metrics)` — measure resource utilization over time and identify capacity bottlenecks (high)
+- [x] `Metering entries (GET /api/metering/v1/accounts/{accountId}/meterings/{year}/{month})` — analyze monthly usage charges and attribute consumption to resources (high)
- [ ] `Invoices (GET /api/billing/v1/accounts/{accountId}/invoices)` — track billed spending and reconcile charges with usage (medium)
- [ ] `Cluster alerts (GET /api/monitoring/v1/accounts/{accountId}/cluster/{clusterId}/alerts)` — analyze alert frequency, severity, and recent incidents across clusters (medium)
- [ ] `Cluster inference metrics (GET /api/monitoring/v1/accounts/{accountId}/cluster/{clusterId}/metrics/inference)` — track model token usage for cluster inference workloads (medium)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/qdrant/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/qdrant/canonical_descriptions.py
index d5019a1268eb..1311f021bfb6 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/qdrant/canonical_descriptions.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/qdrant/canonical_descriptions.py
@@ -57,4 +57,62 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {
"status": "Restore status.",
},
},
+ "cloud_providers": {
+ "description": "Cloud providers available to the selected Qdrant Cloud account.",
+ "docs_url": "https://github.com/qdrant/qdrant-cloud-public-api/blob/main/proto/qdrant/cloud/platform/v1/platform.proto",
+ "columns": {
+ "id": "Cloud provider identifier, for example aws, gcp, azure, or hybrid.",
+ "name": "Human-readable cloud provider name.",
+ "free_tier": "Whether the cloud provider is part of the free-tier offering.",
+ "available": "Whether the cloud provider is available to the account.",
+ },
+ },
+ "cloud_provider_regions": {
+ "description": "Regions of each cloud provider available to the selected Qdrant Cloud account, including hybrid cloud environments.",
+ "docs_url": "https://github.com/qdrant/qdrant-cloud-public-api/blob/main/proto/qdrant/cloud/platform/v1/platform.proto",
+ "columns": {
+ "cloud_provider_id": "Identifier of the cloud provider that was listed to get this region.",
+ "id": "Region identifier, for example us-west-1. Hybrid cloud regions use a UUID.",
+ "name": "Human-readable region name.",
+ "free_tier": "Whether the region is part of the free-tier offering.",
+ "available": "Whether the region is available, or whether the hybrid cloud region is ready for clusters.",
+ "provider": "Identifier of the underlying cloud provider, for example aws or openshift.",
+ "country_iso_code": "ISO 3166-1 alpha-2 country code of the region.",
+ "geographical_sub_region": "Geographic grouping of the region, for example Europe.",
+ "namespace": "Kubernetes namespace of a hybrid cloud environment.",
+ "capabilities": "Storage capabilities available in the region.",
+ },
+ },
+ "metering_entries": {
+ "description": "Detailed metering items for every month with usage in the selected Qdrant Cloud account.",
+ "docs_url": "https://github.com/qdrant/qdrant-cloud-public-api/blob/main/proto/qdrant/cloud/metering/v1/metering.proto",
+ "columns": {
+ "account_id": "Identifier of the account that the metering item belongs to.",
+ "cluster_id": "Identifier of the cluster associated with the usage.",
+ "cluster_name": "Name of the cluster associated with the usage.",
+ "start_time": "Start of the metering window, in UTC.",
+ "end_time": "End of the metering window, in UTC.",
+ "billable_entity_id": "Identifier of the billable entity, such as a cluster booking.",
+ "billable_entity_reference_name": "Reference name of the billable entity.",
+ "billable_entity_type": "Type of the billable entity, for example cluster_booking, resource_option, backup, or inference_model.",
+ "price_per_hour": "Unit price in millicents per hour.",
+ "usage_hours": "Usage duration in decimal hours.",
+ "amount_millicents": "Total charge for the item, in millicents.",
+ "discount_amount_millicents": "Total discount applied to the item, in millicents.",
+ "discount_amount_percent": "Discount applied to the item, as a percentage.",
+ "currency": "ISO 4217 currency of the amounts.",
+ "cluster_labels": "Cluster labels at the time of metering.",
+ },
+ },
+ "cluster_usage_metrics": {
+ "description": "Resource usage time series for each cluster and its nodes, one row per metric data point.",
+ "docs_url": "https://github.com/qdrant/qdrant-cloud-public-api/blob/main/proto/qdrant/cloud/monitoring/v1/monitoring.proto",
+ "columns": {
+ "cluster_id": "Identifier of the cluster that the data point belongs to.",
+ "node_id": "Identifier of the node. Empty for cluster-wide series.",
+ "metric": "Name of the time series, for example cpu, ram, ramCache, ramRss, ramQdrantRss, disk, rps, latency, gpu, or gpuRam.",
+ "timestamp": "Time of the data point.",
+ "value": "Value of the metric at the timestamp.",
+ },
+ },
}
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/qdrant/qdrant.py b/products/warehouse_sources/backend/temporal/data_imports/sources/qdrant/qdrant.py
index db0fd4774554..f97b7bb23498 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/qdrant/qdrant.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/qdrant/qdrant.py
@@ -1,3 +1,4 @@
+from datetime import UTC, datetime
from typing import Any
from uuid import UUID
@@ -5,27 +6,36 @@ from requests.exceptions import HTTPError
from posthog.dataclasses import frozen
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.datetime_utils import parse_datetime_value
from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source import (
RESTAPIConfig,
rest_api_resource,
+ rest_api_resources,
)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.auth import APIKeyAuth
from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.paginators import (
SinglePagePaginator,
)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.rest_client import RESTClient
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.typing import (
+ ClientConfig,
+ Endpoint,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
from products.warehouse_sources.backend.temporal.data_imports.sources.generated_configs.qdrant import QdrantSourceConfig
from products.warehouse_sources.backend.temporal.data_imports.sources.qdrant.settings import (
ACCOUNT_ERROR,
+ ACCOUNTS_PATH,
AUTH_ERROR,
BASE_URL,
ENDPOINTS,
PAGE_SIZE,
- PARTITION_KEY,
PERMISSION_ERROR,
- PRIMARY_KEYS,
+ USAGE_METRICS_LOOKBACK,
+ USAGE_METRICS_OVERLAP,
+ QdrantEndpoint,
+ QdrantParent,
)
@@ -41,10 +51,14 @@ def account_id(config: QdrantSourceConfig) -> str:
raise ValueError(ACCOUNT_ERROR) from error
-def endpoint_path(endpoint: str, account: str) -> str:
+def endpoint_config(endpoint: str) -> QdrantEndpoint:
if endpoint not in ENDPOINTS:
raise ValueError(f"Unknown Qdrant table: {endpoint}")
- return ENDPOINTS[endpoint].format(account_id=account)
+ return ENDPOINTS[endpoint]
+
+
+def _account_path(path: str, account: str) -> str:
+ return path.replace("{account_id}", account)
def validate_credentials(config: QdrantSourceConfig, schema_name: str | None = None) -> tuple[bool, str | None]:
@@ -56,7 +70,13 @@ def validate_credentials(config: QdrantSourceConfig, schema_name: str | None = N
if not config.api_key.strip() or not config.api_key.isascii() or any(c in config.api_key for c in "\r\n"):
return False, AUTH_ERROR
- path = endpoint_path(schema_name, account) if schema_name else "/api/account/v1/accounts"
+ path, params = ACCOUNTS_PATH, {}
+ if schema_name:
+ endpoint = endpoint_config(schema_name)
+ # A fan-out table needs the same permission as its parent listing, which has no path params to fill.
+ probe: QdrantEndpoint | QdrantParent = endpoint.parent or endpoint
+ path = _account_path(probe.path, account)
+ params = {"pageSize": 1} if probe.paginated else {}
client = RESTClient(
base_url=BASE_URL,
auth=APIKeyAuth(api_key=f"apikey {config.api_key}"),
@@ -66,7 +86,7 @@ def validate_credentials(config: QdrantSourceConfig, schema_name: str | None = N
allowed_hosts=[],
)
try:
- rows = next(client.paginate(path=path, params={"pageSize": 1} if schema_name else {}, data_selector="items"))
+ rows = next(client.paginate(path=path, params=params, data_selector="items"))
except HTTPError as error:
status = error.response.status_code if error.response is not None else None
if status == 401:
@@ -82,66 +102,188 @@ def validate_credentials(config: QdrantSourceConfig, schema_name: str | None = N
return True, None
-def qdrant_source(
- config: QdrantSourceConfig,
+def _list_endpoint(path: str, paginated: bool) -> Endpoint:
+ if not paginated:
+ return {"path": path, "data_selector": "items", "paginator": "single_page"}
+ return {
+ "path": path,
+ "data_selector": "items",
+ "params": {"pageSize": PAGE_SIZE},
+ "paginator": {
+ "type": "cursor",
+ "cursor_path": "nextPageToken",
+ "cursor_param": "pageToken",
+ "raise_on_repeated_cursor": True,
+ },
+ }
+
+
+def usage_metrics_window(should_use_incremental_field: bool, db_incremental_field_last_value: Any) -> dict[str, str]:
+ # One `until` for every cluster, so the table-wide watermark never passes a cluster's unread points.
+ until = datetime.now(UTC)
+ since = until - USAGE_METRICS_LOOKBACK
+ last_value = parse_datetime_value(db_incremental_field_last_value) if should_use_incremental_field else None
+ if last_value is not None and last_value - USAGE_METRICS_OVERLAP > since:
+ since = last_value - USAGE_METRICS_OVERLAP
+ return {"since": _rfc3339(since), "until": _rfc3339(until)}
+
+
+def _rfc3339(value: datetime) -> str:
+ return value.isoformat().replace("+00:00", "Z")
+
+
+def usage_metric_rows(response: dict[str, Any]) -> list[dict[str, Any]]:
+ cluster_id = response.get("cluster_id")
+ rows: list[dict[str, Any]] = []
+
+ def add_series(node_id: str, series: dict[str, Any]) -> None:
+ for metric, points in series.items():
+ if metric == "nodes" or not isinstance(points, list):
+ continue
+ for point in points:
+ rows.append(
+ {
+ "cluster_id": cluster_id,
+ # Cluster-wide series have no node; an empty id keeps the primary key non-null.
+ "node_id": node_id,
+ "metric": metric,
+ "timestamp": point.get("timestamp"),
+ "value": point.get("value"),
+ }
+ )
+
+ add_series("", response)
+ for node in response.get("nodes") or []:
+ add_series(node.get("nodeId") or "", node)
+ return rows
+
+
+def _fanout_resource(
+ client_config: ClientConfig,
endpoint: str,
+ config: QdrantEndpoint,
+ parent: QdrantParent,
+ account: str,
+ child_params: dict[str, Any],
+ incremental: bool,
team_id: int,
job_id: str,
- resumable_source_manager: ResumableSourceManager[QdrantResumeConfig],
-) -> SourceResponse:
- path = endpoint_path(endpoint, account_id(config))
- rest_config: RESTAPIConfig = {
- "client": {
- "base_url": BASE_URL,
- "auth": {"type": "api_key", "name": "Authorization", "api_key": f"apikey {config.api_key}"},
- "request_timeout": 30,
- "allow_redirects": False,
- "allowed_hosts": [],
+) -> Any:
+ child_endpoint = _list_endpoint(_account_path(config.path, account), config.paginated)
+ if endpoint == "cluster_usage_metrics":
+ child_endpoint["data_selector"] = "$"
+ child_endpoint["params"] = {
+ **(child_endpoint.get("params") or {}),
+ **child_params,
+ **{
+ param: {"type": "resolve", "resource": parent.name, "field": field}
+ for param, field in parent.path_params.items()
},
+ }
+ if parent.ignored_statuses:
+ child_endpoint["response_actions"] = [
+ {"status_code": status, "action": "ignore"} for status in parent.ignored_statuses
+ ]
+
+ def child_row_map(row: dict[str, Any]) -> dict[str, Any] | list[dict[str, Any]]:
+ for field, column in parent.include.items():
+ row[column] = row.pop(f"_{parent.name}_{field}", None)
+ return usage_metric_rows(row) if endpoint == "cluster_usage_metrics" else row
+
+ rest_config: RESTAPIConfig = {
+ "client": client_config,
"resources": [
+ {"name": parent.name, "endpoint": _list_endpoint(_account_path(parent.path, account), parent.paginated)},
{
"name": endpoint,
"table_name": endpoint,
- "write_disposition": "replace",
- "endpoint": {
- "path": path,
- "data_selector": "items",
- "params": {"pageSize": PAGE_SIZE},
- "paginator": {
- "type": "cursor",
- "cursor_path": "nextPageToken",
- "cursor_param": "pageToken",
- "raise_on_repeated_cursor": True,
- },
- },
- }
+ "write_disposition": {"disposition": "merge", "strategy": "upsert"} if incremental else "replace",
+ "endpoint": child_endpoint,
+ "include_from_parent": list(parent.include),
+ "data_map": child_row_map,
+ },
],
}
- initial_state = None
- if resumable_source_manager.can_resume():
- resume = resumable_source_manager.load_state()
- if resume is not None:
- initial_state = {"cursor": resume.cursor}
-
- def save_checkpoint(state: dict[str, Any] | None) -> None:
- if state and state.get("cursor"):
- resumable_source_manager.save_state(QdrantResumeConfig(cursor=str(state["cursor"])))
-
- resource = rest_api_resource(
- rest_config,
- team_id,
- job_id,
- None,
- resume_hook=save_checkpoint,
- initial_paginator_state=initial_state,
- )
+ resources = rest_api_resources(rest_config, team_id, job_id, None)
+ return next(resource for resource in resources if resource.name == endpoint)
+
+
+def qdrant_source(
+ config: QdrantSourceConfig,
+ endpoint: str,
+ team_id: int,
+ job_id: str,
+ resumable_source_manager: ResumableSourceManager[QdrantResumeConfig],
+ should_use_incremental_field: bool = False,
+ db_incremental_field_last_value: Any = None,
+) -> SourceResponse:
+ settings = endpoint_config(endpoint)
+ account = account_id(config)
+ client_config: ClientConfig = {
+ "base_url": BASE_URL,
+ "auth": {"type": "api_key", "name": "Authorization", "api_key": f"apikey {config.api_key}"},
+ "request_timeout": 30,
+ "allow_redirects": False,
+ "allowed_hosts": [],
+ }
+
+ if settings.parent is not None:
+ child_params = (
+ usage_metrics_window(should_use_incremental_field, db_incremental_field_last_value)
+ if endpoint == "cluster_usage_metrics"
+ else {}
+ )
+ resource = _fanout_resource(
+ client_config,
+ endpoint,
+ settings,
+ settings.parent,
+ account,
+ child_params,
+ should_use_incremental_field,
+ team_id,
+ job_id,
+ )
+ else:
+ rest_config: RESTAPIConfig = {
+ "client": client_config,
+ "resources": [
+ {
+ "name": endpoint,
+ "table_name": endpoint,
+ "write_disposition": "replace",
+ "endpoint": _list_endpoint(_account_path(settings.path, account), settings.paginated),
+ }
+ ],
+ }
+ initial_state = None
+ if resumable_source_manager.can_resume():
+ resume = resumable_source_manager.load_state()
+ if resume is not None:
+ initial_state = {"cursor": resume.cursor}
+
+ def save_checkpoint(state: dict[str, Any] | None) -> None:
+ if state and state.get("cursor"):
+ resumable_source_manager.save_state(QdrantResumeConfig(cursor=str(state["cursor"])))
+
+ resource = rest_api_resource(
+ rest_config,
+ team_id,
+ job_id,
+ None,
+ resume_hook=save_checkpoint,
+ initial_paginator_state=initial_state,
+ )
+
return SourceResponse(
name=endpoint,
on_complete=resumable_source_manager.clear_state,
items=lambda: resource,
- primary_keys=PRIMARY_KEYS,
- partition_keys=[PARTITION_KEY],
- partition_mode="datetime",
- partition_format="month",
+ primary_keys=list(settings.primary_keys),
+ partition_keys=[settings.partition_key] if settings.partition_key else None,
+ partition_mode="datetime" if settings.partition_key else None,
+ partition_format="month" if settings.partition_key else None,
sort_mode=None,
+ # Fan-out tables save no checkpoint, so a handoff would replay every parent.
+ supports_resume=settings.parent is None,
)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/qdrant/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/qdrant/settings.py
index 5825f360c62e..6f1857895c46 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/qdrant/settings.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/qdrant/settings.py
@@ -1,17 +1,110 @@
+from datetime import timedelta
+
+from posthog.dataclasses import frozen
+
+from products.warehouse_sources.backend.types import IncrementalField, IncrementalFieldType
+
BASE_URL = "https://api.cloud.qdrant.io"
API_VERSION = "v1"
PAGE_SIZE = 100
-ENDPOINTS = {
- "clusters": "/api/cluster/v1/accounts/{account_id}/clusters",
- "backups": "/api/cluster/backup/v1/accounts/{account_id}/backups",
- "backup_schedules": "/api/cluster/backup/v1/accounts/{account_id}/backup_schedules",
- "backup_restores": "/api/cluster/backup/v1/accounts/{account_id}/backup_restores",
+ACCOUNTS_PATH = "/api/account/v1/accounts"
+CLUSTERS_PATH = "/api/cluster/v1/accounts/{account_id}/clusters"
+CLOUD_PROVIDERS_PATH = "/api/platform/v1/accounts/{account_id}/cloud-providers"
+MONTHLY_METERINGS_PATH = "/api/metering/v1/accounts/{account_id}/meterings"
+
+# The usage metrics API defaults to the last hour. Full refreshes and long-paused incremental
+# syncs read at most this far back.
+USAGE_METRICS_LOOKBACK = timedelta(days=7)
+# Incremental syncs re-read this much before the watermark, for points Qdrant records late.
+USAGE_METRICS_OVERLAP = timedelta(hours=1)
+
+
+@frozen
+class QdrantParent:
+ name: str
+ path: str
+ paginated: bool
+ # Child path param -> parent row field.
+ path_params: dict[str, str]
+ # Parent row field -> child row column.
+ include: dict[str, str]
+ # Child responses that skip one parent instead of failing the table.
+ ignored_statuses: tuple[int, ...] = ()
+
+
+@frozen
+class QdrantEndpoint:
+ path: str
+ paginated: bool = True
+ primary_keys: tuple[str, ...] = ("id",)
+ partition_key: str | None = "createdAt"
+ parent: QdrantParent | None = None
+
+
+ENDPOINTS: dict[str, QdrantEndpoint] = {
+ "clusters": QdrantEndpoint(path=CLUSTERS_PATH),
+ "backups": QdrantEndpoint(path="/api/cluster/backup/v1/accounts/{account_id}/backups"),
+ "backup_schedules": QdrantEndpoint(path="/api/cluster/backup/v1/accounts/{account_id}/backup_schedules"),
+ "backup_restores": QdrantEndpoint(path="/api/cluster/backup/v1/accounts/{account_id}/backup_restores"),
+ "cloud_providers": QdrantEndpoint(path=CLOUD_PROVIDERS_PATH, paginated=False, partition_key=None),
+ "cloud_provider_regions": QdrantEndpoint(
+ path=CLOUD_PROVIDERS_PATH + "/{cloudProviderId}/regions",
+ paginated=False,
+ primary_keys=("cloud_provider_id", "id"),
+ partition_key=None,
+ parent=QdrantParent(
+ name="cloud_providers",
+ path=CLOUD_PROVIDERS_PATH,
+ paginated=False,
+ path_params={"cloudProviderId": "id"},
+ include={"id": "cloud_provider_id"},
+ # The "hybrid" provider needs read:clusters or read:hybrid_cloud_environments; the others need none.
+ ignored_statuses=(403,),
+ ),
+ ),
+ "metering_entries": QdrantEndpoint(
+ path=MONTHLY_METERINGS_PATH + "/{year}/{month}",
+ paginated=False,
+ # Metering items carry no id of their own: one row is one billable entity over one window.
+ primary_keys=("billableEntityId", "startTime", "endTime"),
+ partition_key="startTime",
+ parent=QdrantParent(
+ name="monthly_meterings",
+ path=MONTHLY_METERINGS_PATH,
+ paginated=False,
+ path_params={"year": "year", "month": "month"},
+ include={},
+ ),
+ ),
+ "cluster_usage_metrics": QdrantEndpoint(
+ path="/api/monitoring/v1/accounts/{account_id}/cluster/{clusterId}/usage-metrics",
+ paginated=False,
+ primary_keys=("cluster_id", "node_id", "metric", "timestamp"),
+ partition_key="timestamp",
+ parent=QdrantParent(
+ name="clusters",
+ path=CLUSTERS_PATH,
+ paginated=True,
+ path_params={"clusterId": "id"},
+ include={"id": "cluster_id"},
+ # A cluster deleted between the listing and its metrics request.
+ ignored_statuses=(404,),
+ ),
+ ),
}
-PRIMARY_KEYS = ["id"]
-PARTITION_KEY = "createdAt"
+INCREMENTAL_FIELDS: dict[str, list[IncrementalField]] = {
+ "cluster_usage_metrics": [
+ {
+ "label": "timestamp",
+ "type": IncrementalFieldType.DateTime,
+ "field": "timestamp",
+ "field_type": IncrementalFieldType.DateTime,
+ }
+ ],
+}
AUTH_ERROR = "Qdrant rejected the Cloud Management Key. Check the key in Access Management > Cloud Management Keys."
-PERMISSION_ERROR = "The Qdrant key lacks permission. Grant read:clusters, read:backups, or read:backup_schedules for the selected tables."
+PERMISSION_ERROR = "The Qdrant key lacks permission. Grant read:clusters, read:backups, read:backup_schedules, or read:payment_information for the selected tables."
ACCOUNT_ERROR = "Enter a valid Qdrant account ID from the Cloud Console."
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/qdrant/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/qdrant/source.py
index a8a83ee15895..1bc2c895e9b6 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/qdrant/source.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/qdrant/source.py
@@ -32,6 +32,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.qdrant.set
API_VERSION,
AUTH_ERROR,
ENDPOINTS,
+ INCREMENTAL_FIELDS,
PERMISSION_ERROR,
)
from products.warehouse_sources.backend.types import ExternalDataSourceType
@@ -77,7 +78,8 @@ class QdrantSource(ResumableSource[QdrantSourceConfig, QdrantResumeConfig]):
force_refresh: bool = False,
api_version: str | None = None,
) -> list[SourceSchema]:
- return build_endpoint_schemas(ENDPOINTS, {}, names)
+ # The metrics window re-reads points before the watermark, so append would duplicate them.
+ return build_endpoint_schemas(ENDPOINTS, INCREMENTAL_FIELDS, names, merge_only=INCREMENTAL_FIELDS.keys())
def validate_credentials(
self,
@@ -97,7 +99,15 @@ class QdrantSource(ResumableSource[QdrantSourceConfig, QdrantResumeConfig]):
resumable_source_manager: ResumableSourceManager[QdrantResumeConfig],
inputs: SourceInputs,
) -> SourceResponse:
- return qdrant_source(config, inputs.schema_name, inputs.team_id, inputs.job_id, resumable_source_manager)
+ return qdrant_source(
+ config,
+ inputs.schema_name,
+ inputs.team_id,
+ inputs.job_id,
+ resumable_source_manager,
+ should_use_incremental_field=inputs.should_use_incremental_field,
+ db_incremental_field_last_value=inputs.db_incremental_field_last_value,
+ )
@property
def get_source_config(self) -> SourceConfig:
@@ -110,7 +120,7 @@ class QdrantSource(ResumableSource[QdrantSourceConfig, QdrantResumeConfig]):
caption=(
"Import Qdrant Cloud infrastructure metadata. "
"Create a Cloud Management Key in the Cloud Console under Access Management > Cloud Management Keys. "
- "Grant `read:clusters`, `read:backups`, and `read:backup_schedules` for the tables you select. "
+ "Grant `read:clusters`, `read:backups`, `read:backup_schedules`, and `read:payment_information` for the tables you select. "
"Copy your account ID from the Cloud Console. Vector data and collections are not included."
),
fields=cast(
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/qdrant/tests/test_qdrant.py b/products/warehouse_sources/backend/temporal/data_imports/sources/qdrant/tests/test_qdrant.py
index 7bc4f001b110..e032ebbe43c1 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/qdrant/tests/test_qdrant.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/qdrant/tests/test_qdrant.py
@@ -2,6 +2,7 @@ from typing import Any
from urllib.parse import parse_qs, urlsplit
import pytest
+import time_machine
from unittest.mock import patch
import requests_mock
@@ -27,6 +28,10 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.qdrant.sou
ACCOUNT_ID = "00000000-0000-4000-8000-000000000001"
ACCOUNTS_URL = "https://api.cloud.qdrant.io/api/account/v1/accounts"
CLUSTERS_URL = f"https://api.cloud.qdrant.io/api/cluster/v1/accounts/{ACCOUNT_ID}/clusters"
+PROVIDERS_PATH = f"/api/platform/v1/accounts/{ACCOUNT_ID}/cloud-providers"
+METERINGS_PATH = f"/api/metering/v1/accounts/{ACCOUNT_ID}/meterings"
+CLUSTER_A = "00000000-0000-4000-8000-00000000000a"
+CLUSTER_B = "00000000-0000-4000-8000-00000000000b"
@pytest.fixture
@@ -82,23 +87,123 @@ def test_repeated_cursor_fails(config: QdrantSourceConfig) -> None:
assert len(result.requests) == 1
[email protected]("schema_name", [None, "clusters", "backups", "backup_schedules", "backup_restores"])
-def test_credentials_make_one_probe(config: QdrantSourceConfig, schema_name: str | None) -> None:
+ "schema_name,url,paginated",
+ [
+ (None, ACCOUNTS_URL, False),
+ ("clusters", CLUSTERS_URL, True),
+ ("backups", f"https://api.cloud.qdrant.io/api/cluster/backup/v1/accounts/{ACCOUNT_ID}/backups", True),
+ (
+ "backup_schedules",
+ f"https://api.cloud.qdrant.io/api/cluster/backup/v1/accounts/{ACCOUNT_ID}/backup_schedules",
+ True,
+ ),
+ (
+ "backup_restores",
+ f"https://api.cloud.qdrant.io/api/cluster/backup/v1/accounts/{ACCOUNT_ID}/backup_restores",
+ True,
+ ),
+ ("cloud_providers", f"https://api.cloud.qdrant.io{PROVIDERS_PATH}", False),
+ ("cloud_provider_regions", f"https://api.cloud.qdrant.io{PROVIDERS_PATH}", False),
+ ("metering_entries", f"https://api.cloud.qdrant.io{METERINGS_PATH}", False),
+ ("cluster_usage_metrics", CLUSTERS_URL, True),
+ ],
+)
+def test_credentials_make_one_probe(
+ config: QdrantSourceConfig, schema_name: str | None, url: str, paginated: bool
+) -> None:
with requests_mock.Mocker() as http:
- body: dict[str, Any]
- if schema_name is None:
- url = ACCOUNTS_URL
- body = {"items": [{"id": ACCOUNT_ID}]}
- else:
- service = "cluster" if schema_name == "clusters" else "cluster/backup"
- url = f"https://api.cloud.qdrant.io/api/{service}/v1/accounts/{ACCOUNT_ID}/{schema_name}"
- body = {"items": [], "nextPageToken": "unused"}
+ body: dict[str, Any] = {"items": [{"id": ACCOUNT_ID}]} if schema_name is None else {"items": []}
http.get(url, json=body)
assert validate_credentials(config, schema_name) == (True, None)
assert http.call_count == 1
request = http.request_history[0]
assert request.headers["Authorization"] == "apikey fake-management-key"
- assert parse_qs(urlsplit(request.url).query) == ({"pageSize": ["1"]} if schema_name else {})
+ assert parse_qs(urlsplit(request.url).query) == ({"pageSize": ["1"]} if paginated else {})
+
+
+ "schema_name,script,expected_paths,expected_rows",
+ [
+ (
+ "cloud_provider_regions",
+ [
+ ScriptedResponse(json={"items": [{"id": "aws"}, {"id": "hybrid"}]}),
+ ScriptedResponse(json={"items": [{"id": "us-west-1", "provider": "aws"}]}),
+ ScriptedResponse(status=403, json={"code": 7, "message": "permission denied"}),
+ ],
+ [PROVIDERS_PATH, f"{PROVIDERS_PATH}/aws/regions", f"{PROVIDERS_PATH}/hybrid/regions"],
+ [{"id": "us-west-1", "provider": "aws", "cloud_provider_id": "aws"}],
+ ),
+ (
+ "metering_entries",
+ [
+ ScriptedResponse(json={"items": [{"year": 2026, "month": 9}, {"year": 2026, "month": 10}]}),
+ ScriptedResponse(json={"items": [{"billableEntityId": "booking-1", "amountMillicents": "10"}]}),
+ ScriptedResponse(json={"items": []}),
+ ],
+ [METERINGS_PATH, f"{METERINGS_PATH}/2026/9", f"{METERINGS_PATH}/2026/10"],
+ [{"billableEntityId": "booking-1", "amountMillicents": "10"}],
+ ),
+ ],
+)
+def test_fanout_binds_parent_rows_into_child_paths(
+ config: QdrantSourceConfig,
+ schema_name: str,
+ script: Script,
+ expected_paths: list[str],
+ expected_rows: list[dict[str, Any]],
+) -> None:
+ result = drive(config, schema_name, script)
+
+ assert result.raised is None
+ assert result.paths == expected_paths
+ assert result.rows == expected_rows
+
+
+@time_machine.travel("2026-10-31T12:00:00Z", tick=False)
+ "inputs,expected_since",
+ [
+ ({}, "2026-10-24T12:00:00Z"),
+ (
+ {"incremental_field": "timestamp", "db_incremental_field_last_value": "2026-10-31T10:00:00Z"},
+ "2026-10-31T09:00:00Z",
+ ),
+ (
+ {"incremental_field": "timestamp", "db_incremental_field_last_value": "2026-01-01T00:00:00Z"},
+ "2026-10-24T12:00:00Z",
+ ),
+ ],
+)
+def test_cluster_usage_metrics_explode_per_cluster(
+ config: QdrantSourceConfig, inputs: dict[str, Any], expected_since: str
+) -> None:
+ point = {"timestamp": "2026-10-31T11:00:00Z", "value": 0.5}
+ result = drive(
+ config,
+ "cluster_usage_metrics",
+ [
+ ScriptedResponse(json={"items": [{"id": CLUSTER_A}, {"id": CLUSTER_B}]}),
+ ScriptedResponse(json={"cpu": [point], "rps": [], "nodes": [{"nodeId": "node-0", "ram": [point]}]}),
+ ScriptedResponse(status=404, json={"code": 5, "message": "cluster not found"}),
+ ],
+ **inputs,
+ )
+
+ assert result.raised is None
+ assert result.paths == [
+ urlsplit(CLUSTERS_URL).path,
+ f"/api/monitoring/v1/accounts/{ACCOUNT_ID}/cluster/{CLUSTER_A}/usage-metrics",
+ f"/api/monitoring/v1/accounts/{ACCOUNT_ID}/cluster/{CLUSTER_B}/usage-metrics",
+ ]
+ assert result.params("since") == [None, expected_since, expected_since]
+ assert result.params("until") == [None, "2026-10-31T12:00:00Z", "2026-10-31T12:00:00Z"]
+ assert result.response is not None and result.response.supports_resume is False
+ assert result.rows == [
+ {"cluster_id": CLUSTER_A, "node_id": "", "metric": "cpu", **point},
+ {"cluster_id": CLUSTER_A, "node_id": "node-0", "metric": "ram", **point},
+ ]
@pytest.mark.parametrize("body", [{"items": []}, {"items": [{"id": "another-account"}]}])