Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(data-warehouse): add 7 missing tables to the Vantage source (#124557)

Tom Owersae03a4c6fe55
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
index 098949680c6d..8cd28bfed7ba 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
@@ -12942,10 +12942,10 @@ Diffed against: <https://api.vantage.sh/v2/swagger.json>
- [x] `GET /resources` — the resource inventory returned by a resource report; today only the report definition is synced (high)
- [x] `GET /financial_commitments` — actual RI/savings-plan commitments; we sync only financial_commitment_reports definitions (high)
- [x] `GET /tags and GET /tags/{key}/values` — lookup for the cost-allocation tag keys/values every cost breakdown groups by (high)
-- [ ] `GET /cost_providers, GET /cost_provider_accounts, GET /cost_services` — lookup tables resolving the provider, account and service ids that appear in cost data (high)
-- [ ] `GET /unit_costs` — unit economics per cost report (cost per business unit), a headline FinOps metric (medium)
-- [ ] `GET /business_metrics and GET /business_metrics/{token}/values` — the metric series that unit costs divide by; needed to reproduce unit-cost math (medium)
-- [ ] `GET /network_flow_logs` — the aggregated flow data behind network_flow_reports, which we sync definitions of only (medium)
+- [x] `GET /cost_providers, GET /cost_provider_accounts, GET /cost_services` — lookup tables resolving the provider, account and service ids that appear in cost data (high)
+- [x] `GET /unit_costs` — unit economics per cost report (cost per business unit), a headline FinOps metric (medium)
+- [x] `GET /business_metrics and GET /business_metrics/{token}/values` — the metric series that unit costs divide by; needed to reproduce unit-cost math (medium)
+- [x] `GET /network_flow_logs` — the aggregated flow data behind network_flow_reports, which we sync definitions of only (medium)
- [ ] `GET /cost_alerts/{cost_alert_token}/events` — firing history for the cost alerts we already sync — the state-transition table (medium)
- [ ] `GET /cost_reports/{cost_report_token}/forecasted_costs` — forecast series per report, used for budget-vs-forecast analysis (medium)
- [ ] `GET /recommendations/{recommendation_token}/resources` — the specific resources each savings recommendation targets (medium)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/vantage/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/vantage/canonical_descriptions.py
index 33c82f83363a..0c873ef44d9c 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/vantage/canonical_descriptions.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/vantage/canonical_descriptions.py
@@ -415,4 +415,88 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {
"tags": "Key-value pairs of tags associated with the resource.",
},
},
+ "business_metrics": {
+ "description": "A business KPI, such as requests, users, or revenue, that cost reports divide costs by to compute unit costs.",
+ "docs_url": "https://vantage.readme.io/reference/getbusinessmetrics",
+ "columns": {
+ "token": "The token of the BusinessMetric.",
+ "title": "The title of the BusinessMetric.",
+ "created_by_token": "The token of the Creator of the BusinessMetric.",
+ "cost_report_tokens_with_metadata": "The tokens for any CostReports that use the BusinessMetric, the unit scale, and label filter.",
+ "import_type": "The type of import for the BusinessMetric.",
+ "integration_token": "The Integration token used to import the BusinessMetric.",
+ "cloudwatch_fields": "The fields used to generate the cloudwatch metrics BusinessMetric.",
+ "datadog_metric_fields": "The fields used to generate the Datadog metrics for BusinessMetric.",
+ "gcp_bigquery_metric_fields": "The fields used to generate the GCP BigQuery metrics for BusinessMetric.",
+ "snowflake_metric_fields": "The fields used to generate the Snowflake metrics for BusinessMetric.",
+ "metronome_metric_fields": "The fields used to import Metronome metrics for the BusinessMetric.",
+ "clickhouse_metric_fields": "The fields used to import ClickHouse metrics for the BusinessMetric.",
+ },
+ },
+ "business_metric_values": {
+ "description": "A dated value of a business metric, the denominator that unit costs divide by.",
+ "docs_url": "https://vantage.readme.io/reference/getbusinessmetricvalues",
+ "columns": {
+ "business_metric_token": "The token of the BusinessMetric that this value belongs to.",
+ "date": "The date of the Business Metric Value. ISO 8601 formatted.",
+ "amount": "The amount of the Business Metric Value as a string to ensure precision.",
+ "label": "The label of the Business Metric Value.",
+ },
+ },
+ "cost_providers": {
+ "description": "A cost provider (such as AWS, Azure, or a custom provider) available to query in a workspace.",
+ "docs_url": "https://vantage.readme.io/reference/getcostproviders",
+ "columns": {
+ "workspace_token": "The token of the Workspace that the CostProvider is available in.",
+ "name": "The name of the CostProvider. For Custom Providers, this is the customer-defined name.",
+ "key": "The key of the CostProvider, useful for filtering Costs. Custom Providers use the form custom_provider:<token>.",
+ },
+ },
+ "cost_provider_accounts": {
+ "description": "A provider account (such as an AWS account or Azure subscription) available in a workspace.",
+ "docs_url": "https://vantage.readme.io/reference/getcostprovideraccounts",
+ "columns": {
+ "workspace_token": "The token of the Workspace that the provider account is available in.",
+ "title": "The display name of the provider account.",
+ "account_id": "The provider account identifier (e.g., AWS account ID, Azure subscription ID).",
+ "provider_uuid": "The provider-specific unique identifier.",
+ "provider": "The provider type (aws, azure, gcp, etc.).",
+ },
+ },
+ "cost_services": {
+ "description": "A cost service of a cost provider, available to query in a workspace.",
+ "docs_url": "https://vantage.readme.io/reference/getcostservices",
+ "columns": {
+ "workspace_token": "The token of the Workspace that the CostService is available in.",
+ "name": "The name of the CostService.",
+ "provider": "The key value of the CostProvider.",
+ },
+ },
+ "unit_costs": {
+ "description": "The cost per unit of a business metric for a cost report, for each date in the report's date range.",
+ "docs_url": "https://vantage.readme.io/reference/getunitcosts",
+ "columns": {
+ "cost_report_token": "The token of the CostReport that returned this unit cost.",
+ "business_metric_token": "The token of the BusinessMetric for which the unit cost was calculated.",
+ "business_metric_title": "The title of the BusinessMetric for which the unit cost was calculated.",
+ "calculation_type": "The calculation type applied to produce this result.",
+ "unit_cost_amount": "The amount of the unit cost. For raw_business_metric types, this equals the business_metric_amount.",
+ "business_metric_amount": "The amount of the business metric.",
+ "scale": "The scale of the BusinessMetric's values within a particular CostReport.",
+ "date": "The date for which the unit cost was calculated. ISO 8601 Formatted.",
+ },
+ },
+ "network_flow_logs": {
+ "description": "An aggregated row of network flow log traffic returned by a network flow report.",
+ "docs_url": "https://vantage.readme.io/reference/getnetworkflowlogs",
+ "columns": {
+ "network_flow_report_token": "The token of the NetworkFlowReport that returned this row.",
+ "groupings": "The grouping values for this aggregated Network Flow Log row.",
+ "bytes": "The sampling-adjusted estimated byte count.",
+ "estimated_cost": "The sampling-adjusted estimated cost.",
+ "currency": "The ISO 4217 currency code for estimated costs.",
+ "sampled_bytes": "The observed byte count when sampling applies.",
+ "sampled_estimated_cost": "The observed estimated cost when sampling applies.",
+ },
+ },
}
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/vantage/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/vantage/settings.py
index f40cf8d9bc64..cecd3628876b 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/vantage/settings.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/vantage/settings.py
@@ -12,8 +12,9 @@ from products.warehouse_sources.backend.types import IncrementalField
# paginated requests (and thus stay well under the ~1,000 requests/hour and ~20 requests/minute
# per-key rate limits).
PAGE_SIZE = 1000
-# `/costs` documents a higher cap than the other list endpoints.
+# `/costs` and `/business_metrics/{token}/values` document a higher cap than the other list endpoints.
COSTS_PAGE_SIZE = 5000
+BUSINESS_METRIC_VALUES_PAGE_SIZE = 5000
@frozen
@@ -37,6 +38,8 @@ class VantageEndpointConfig:
incremental_fields: list[IncrementalField] = field(default_factory=list)
default_incremental_field: Optional[str] = None
page_size: int = PAGE_SIZE
+ # None for endpoints that document no page-size param at all.
+ page_size_param: Optional[str] = "limit"
# Set for endpoints that need a parent row (a report token or a tag key) to build each request.
fanout: Optional[DependentEndpointConfig] = None
# Child-request params that are not page size, e.g. a fixed sort order.
@@ -119,6 +122,56 @@ VANTAGE_ENDPOINTS: dict[str, VantageEndpointConfig] = {
primary_keys=None,
partition_key="created_at",
),
+ "business_metrics": VantageEndpointConfig(
+ name="business_metrics", path="/business_metrics", data_key="business_metrics"
+ ),
+ # These lookups take `workspace_token`, which Vantage requires when the API token can see more
+ # than one workspace, so they fan out over workspaces. An account has few workspaces, so they
+ # stay on by default.
+ "cost_providers": VantageEndpointConfig(
+ name="cost_providers",
+ path="/cost_providers?workspace_token={workspace_token}",
+ data_key="cost_providers",
+ primary_keys=["workspace_token", "key"],
+ fanout=DependentEndpointConfig(
+ parent_name="workspaces",
+ resolve_param="workspace_token",
+ resolve_field="token",
+ include_from_parent=["token"],
+ parent_field_renames={"token": "workspace_token"},
+ child_response_actions=[{"status_code": 404, "action": "ignore"}],
+ ),
+ ),
+ "cost_provider_accounts": VantageEndpointConfig(
+ name="cost_provider_accounts",
+ path="/cost_provider_accounts?workspace_token={workspace_token}",
+ data_key="cost_provider_accounts",
+ primary_keys=["workspace_token", "provider", "provider_uuid"],
+ page_size_param=None,
+ fanout=DependentEndpointConfig(
+ parent_name="workspaces",
+ resolve_param="workspace_token",
+ resolve_field="token",
+ include_from_parent=["token"],
+ parent_field_renames={"token": "workspace_token"},
+ child_response_actions=[{"status_code": 404, "action": "ignore"}],
+ ),
+ ),
+ "cost_services": VantageEndpointConfig(
+ name="cost_services",
+ path="/cost_services?workspace_token={workspace_token}",
+ data_key="cost_services",
+ primary_keys=["workspace_token", "provider", "name"],
+ page_size_param=None,
+ fanout=DependentEndpointConfig(
+ parent_name="workspaces",
+ resolve_param="workspace_token",
+ resolve_field="token",
+ include_from_parent=["token"],
+ parent_field_renames={"token": "workspace_token"},
+ child_response_actions=[{"status_code": 404, "action": "ignore"}],
+ ),
+ ),
# The fan-out tables below make one request series per parent row. They are off by default
# because a large account can have many reports or tag keys, and the cost endpoints have
# stricter rate limits than the config endpoints.
@@ -175,6 +228,59 @@ VANTAGE_ENDPOINTS: dict[str, VantageEndpointConfig] = {
child_response_actions=[{"status_code": 404, "action": "ignore"}],
),
),
+ # Unit cost rows have no identifier, and their dates follow each report's saved date range and
+ # date bin, so the table is replaced on every sync.
+ "unit_costs": VantageEndpointConfig(
+ name="unit_costs",
+ path="/unit_costs?cost_report_token={cost_report_token}",
+ data_key="unit_costs",
+ primary_keys=None,
+ should_sync_default=False,
+ params={"order": "asc"},
+ fanout=DependentEndpointConfig(
+ parent_name="cost_reports",
+ resolve_param="cost_report_token",
+ resolve_field="token",
+ include_from_parent=["token"],
+ parent_field_renames={"token": "cost_report_token"},
+ child_response_actions=[{"status_code": 404, "action": "ignore"}],
+ ),
+ ),
+ # Values can carry a null label and are re-imported by the metric's integration, so the table
+ # is replaced on every sync. No `date_bin` is sent, so the values keep their imported timestamps.
+ "business_metric_values": VantageEndpointConfig(
+ name="business_metric_values",
+ path="/business_metrics/{business_metric_token}/values",
+ data_key="values",
+ primary_keys=None,
+ should_sync_default=False,
+ page_size=BUSINESS_METRIC_VALUES_PAGE_SIZE,
+ fanout=DependentEndpointConfig(
+ parent_name="business_metrics",
+ resolve_param="business_metric_token",
+ resolve_field="token",
+ include_from_parent=["token"],
+ parent_field_renames={"token": "business_metric_token"},
+ child_response_actions=[{"status_code": 404, "action": "ignore"}],
+ ),
+ ),
+ # Flow log rows are aggregates over each report's saved date interval and groupings, with no
+ # identifier, so the table is replaced on every sync.
+ "network_flow_logs": VantageEndpointConfig(
+ name="network_flow_logs",
+ path="/network_flow_logs?network_flow_report_token={network_flow_report_token}",
+ data_key="network_flow_logs",
+ primary_keys=None,
+ should_sync_default=False,
+ fanout=DependentEndpointConfig(
+ parent_name="network_flow_reports",
+ resolve_param="network_flow_report_token",
+ resolve_field="token",
+ include_from_parent=["token"],
+ parent_field_renames={"token": "network_flow_report_token"},
+ child_response_actions=[{"status_code": 404, "action": "ignore"}],
+ ),
+ ),
}
ENDPOINTS = tuple(VANTAGE_ENDPOINTS.keys())
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/vantage/tests/test_vantage.py b/products/warehouse_sources/backend/temporal/data_imports/sources/vantage/tests/test_vantage.py
index 7cd2efa07ff2..5a2bc8c5552f 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/vantage/tests/test_vantage.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/vantage/tests/test_vantage.py
@@ -15,6 +15,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.tes
from products.warehouse_sources.backend.temporal.data_imports.sources.generated_configs.vantage import (
VantageSourceConfig,
)
+from products.warehouse_sources.backend.temporal.data_imports.sources.vantage.settings import VANTAGE_ENDPOINTS
from products.warehouse_sources.backend.temporal.data_imports.sources.vantage.source import VantageSource
from products.warehouse_sources.backend.temporal.data_imports.sources.vantage.vantage import (
VANTAGE_BASE_URL,
@@ -145,6 +146,13 @@ def _by_url(answers: dict[str, ScriptedResponse]) -> Script:
_COSTS_CHILD = f"{VANTAGE_BASE_URL}/costs?cost_report_token={{token}}&limit=5000&order=asc"
_RESOURCES_CHILD = f"{VANTAGE_BASE_URL}/resources?resource_report_token={{token}}&limit=1000"
_TAG_VALUES_CHILD = f"{VANTAGE_BASE_URL}/tags/{{token}}/values?limit=1000"
+_UNIT_COSTS_CHILD = f"{VANTAGE_BASE_URL}/unit_costs?cost_report_token={{token}}&limit=1000&order=asc"
+_BUSINESS_METRIC_VALUES_CHILD = f"{VANTAGE_BASE_URL}/business_metrics/{{token}}/values?limit=5000"
+_NETWORK_FLOW_LOGS_CHILD = f"{VANTAGE_BASE_URL}/network_flow_logs?network_flow_report_token={{token}}&limit=1000"
+_COST_PROVIDERS_CHILD = f"{VANTAGE_BASE_URL}/cost_providers?workspace_token={{token}}&limit=1000"
+# These two endpoints document no page-size param, so neither they nor their parent send `limit`.
+_COST_PROVIDER_ACCOUNTS_CHILD = f"{VANTAGE_BASE_URL}/cost_provider_accounts?workspace_token={{token}}"
+_COST_SERVICES_CHILD = f"{VANTAGE_BASE_URL}/cost_services?workspace_token={{token}}"
class TestFanOut:
@@ -153,6 +161,7 @@ class TestFanOut:
(
"costs",
"cost_reports",
+ "?limit=1000",
[{"token": "rprt_a"}, {"token": "rprt_b"}],
_COSTS_CHILD,
["rprt_a", "rprt_b"],
@@ -162,6 +171,7 @@ class TestFanOut:
(
"resources",
"resource_reports",
+ "?limit=1000",
[{"token": "rsrc_rprt_a"}, {"token": "rsrc_rprt_b"}],
_RESOURCES_CHILD,
["rsrc_rprt_a", "rsrc_rprt_b"],
@@ -172,18 +182,85 @@ class TestFanOut:
# A `/` in a tag key would otherwise split the path into a different route.
"tag_values",
"tags",
+ "?limit=1000",
[{"tag_key": "team"}, {"tag_key": "kubernetes.io/name"}],
_TAG_VALUES_CHILD,
["team", "kubernetes.io%2Fname"],
"tag_key",
{"tag_value": "web"},
),
+ (
+ "unit_costs",
+ "cost_reports",
+ "?limit=1000",
+ [{"token": "rprt_a"}, {"token": "rprt_b"}],
+ _UNIT_COSTS_CHILD,
+ ["rprt_a", "rprt_b"],
+ "cost_report_token",
+ {"business_metric_token": "bsnss_mtrc_1", "unit_cost_amount": "0.42", "date": "2025-01-01"},
+ ),
+ (
+ "business_metric_values",
+ "business_metrics",
+ "?limit=1000",
+ [{"token": "bsnss_mtrc_a"}, {"token": "bsnss_mtrc_b"}],
+ _BUSINESS_METRIC_VALUES_CHILD,
+ ["bsnss_mtrc_a", "bsnss_mtrc_b"],
+ "business_metric_token",
+ {"date": "2025-01-01T00:00:00Z", "amount": "120.0", "label": None},
+ ),
+ (
+ "network_flow_logs",
+ "network_flow_reports",
+ "?limit=1000",
+ [{"token": "ntflw_lg_rprt_a"}, {"token": "ntflw_lg_rprt_b"}],
+ _NETWORK_FLOW_LOGS_CHILD,
+ ["ntflw_lg_rprt_a", "ntflw_lg_rprt_b"],
+ "network_flow_report_token",
+ {"groupings": {"region": {"value": "us-east-1"}}, "bytes": 2048.0, "estimated_cost": "0.10"},
+ ),
+ (
+ "cost_providers",
+ "workspaces",
+ "?limit=1000",
+ [{"token": "wrkspc_a"}, {"token": "wrkspc_b"}],
+ _COST_PROVIDERS_CHILD,
+ ["wrkspc_a", "wrkspc_b"],
+ "workspace_token",
+ {"name": "AWS", "key": "aws"},
+ ),
+ (
+ "cost_provider_accounts",
+ "workspaces",
+ "",
+ [{"token": "wrkspc_a"}, {"token": "wrkspc_b"}],
+ _COST_PROVIDER_ACCOUNTS_CHILD,
+ ["wrkspc_a", "wrkspc_b"],
+ "workspace_token",
+ {
+ "title": "Example account",
+ "account_id": "000000000000",
+ "provider_uuid": "uuid-1",
+ "provider": "aws",
+ },
+ ),
+ (
+ "cost_services",
+ "workspaces",
+ "",
+ [{"token": "wrkspc_a"}, {"token": "wrkspc_b"}],
+ _COST_SERVICES_CHILD,
+ ["wrkspc_a", "wrkspc_b"],
+ "workspace_token",
+ {"name": "Amazon S3", "provider": "aws"},
+ ),
]
)
def test_walks_every_parent_and_child_page_and_tags_rows_with_parent(
self,
endpoint: str,
parent_key: str,
+ parent_query: str,
parents: list[dict[str, Any]],
child_url: str,
child_path_values: list[str],
@@ -195,13 +272,15 @@ class TestFanOut:
second_child = child_url.format(token=child_path_values[1])
parent_values = [next(iter(parent.values())) for parent in parents]
+ data_key = VANTAGE_ENDPOINTS[endpoint].data_key
+
result = _run(
_by_url(
{
- f"{VANTAGE_BASE_URL}/{parent_key}?limit=1000": _response(parents, data_key=parent_key),
- first_child: _response([dict(child_row)], data_key=endpoint, next_url=second_page),
- second_page: _response([dict(child_row)], data_key=endpoint),
- second_child: _response([dict(child_row)], data_key=endpoint),
+ f"{VANTAGE_BASE_URL}/{parent_key}{parent_query}": _response(parents, data_key=parent_key),
+ first_child: _response([dict(child_row)], data_key=data_key, next_url=second_page),
+ second_page: _response([dict(child_row)], data_key=data_key),
+ second_child: _response([dict(child_row)], data_key=data_key),
}
),
endpoint=endpoint,
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/vantage/tests/test_vantage_source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/vantage/tests/test_vantage_source.py
index a41f3c57471c..5706b7c58e2a 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/vantage/tests/test_vantage_source.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/vantage/tests/test_vantage_source.py
@@ -56,5 +56,12 @@ class TestPublicDocsCatalog:
class TestSchemaDefaults:
def test_fan_out_tables_are_off_by_default(self) -> None:
schemas = {schema.name: schema for schema in VantageSource().get_schemas(MagicMock(), team_id=1)}
- off_by_default = {"costs", "resources", "tag_values"}
+ off_by_default = {
+ "costs",
+ "resources",
+ "tag_values",
+ "unit_costs",
+ "business_metric_values",
+ "network_flow_logs",
+ }
assert {name for name, schema in schemas.items() if not schema.should_sync_default} == off_by_default
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/vantage/vantage.py b/products/warehouse_sources/backend/temporal/data_imports/sources/vantage/vantage.py
index 0b435b8e4cde..2e233f953e22 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/vantage/vantage.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/vantage/vantage.py
@@ -124,6 +124,10 @@ def _client_config(api_key: str) -> ClientConfig:
}
+def _page_size_params(config: VantageEndpointConfig) -> dict[str, Any]:
+ return {config.page_size_param: config.page_size} if config.page_size_param else {}
+
+
def _top_level_resource(
config: VantageEndpointConfig,
api_key: str,
@@ -140,7 +144,7 @@ def _top_level_resource(
"endpoint": {
"path": config.path,
# Cap page size on the first request; `links.next` carries limit/page thereafter.
- "params": {"limit": config.page_size, **config.params},
+ "params": {**_page_size_params(config), **config.params},
# Rows nest under the endpoint's own key (e.g. "cost_reports"). A missing key on a
# 200 yields zero rows (matching the prior `data.get(key, [])`), not a hard error.
"data_selector": config.data_key,
@@ -203,6 +207,7 @@ def _fanout_resource(
job_id=job_id,
db_incremental_field_last_value=None,
child_params_extra=config.params or None,
+ page_size_param=config.page_size_param,
parent_endpoint_extra={"data_selector": parent_config.data_key},
child_endpoint_extra={
"data_selector": config.data_key,