:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(data-warehouse): add linode volume quota tables (#119525)
Tom Owers9532c6bc66fe
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.mdindex 883e641d62f6..2a835f579fdc 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md@@ -7384,6 +7384,7 @@ Diffed against: <https://raw.githubusercontent.com/linode/linode-api-openapi/mai- [ ] `object-storage/buckets (+ object-storage/transfer, quotas usage)` — per-bucket size and object count plus transfer usage for storage cost attribution (medium)- [ ] `support/tickets (+ /{ticketId}/replies)` — support ticket volume and response history (medium)- [ ] `account/maintenance` — scheduled and completed host maintenance events per entity - explains reboots seen in the events table (medium)+- [x] `volumes/quotas (+ /volumes/quotas/{quotaId}/usage)` — Block Storage quota limits and current usage per region (medium). Added as `volume_quotas` and `volume_quota_usage` (fan-out over `volume_quotas`, full refresh). Quotas without usage data return 404 and get no usage row.Note: The Akamai/Linode techdocs site serves no spec directly; the authoritative OpenAPI is linode/linode-api-openapi on GitHub (openapi.json, spec version 4.229.1, 249 GET paths). Also unsynced but lower value: account/logins, tags, placement/groups, vpcs/subnets, networking/firewalls, longview, managed services and stackscripts.diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/linode/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/linode/canonical_descriptions.pyindex 3e52dc760b18..1b5fa3c01d33 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/linode/canonical_descriptions.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/linode/canonical_descriptions.py@@ -265,4 +265,28 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {"bytes_total": "The total public network traffic sent and received by the Linode in the month, in bytes.",},},+ "volume_quotas": {+ "description": "Block Storage (Volumes) quotas applied to the account, with the maximum allowed per region or globally.",+ "docs_url": "https://techdocs.akamai.com/linode-api/reference/get-volume-quotas",+ "columns": {+ "quota_id": "The id of the quota, formatted as vol-{quota_type}-{region}, for example vol-capacity-us-east.",+ "quota_name": "The human-readable name of the quota.",+ "quota_type": "The type of the quota: vol-capacity (storage capacity), vol-volumes (number of volumes) or vol-attachments (number of attached volumes).",+ "description": "A description of what the quota controls.",+ "quota_limit": "The maximum quantity of the resource_metric that the quota allows.",+ "resource_metric": "The resource the quota measures: gigabyte, volume or attachment.",+ "region": "The region the quota applies to. Joins to the id column on regions. Null for global quotas.",+ "scope": "Whether the quota applies per region or globally.",+ "has_usage": "Whether usage data is available for the quota in volume_quota_usage.",+ },+ },+ "volume_quota_usage": {+ "description": "Current usage of each Block Storage (Volumes) quota on the account. Quotas without usage data have no row.",+ "docs_url": "https://techdocs.akamai.com/linode-api/reference/get-volume-quota-usage",+ "columns": {+ "quota_id": "The id of the quota. Joins to the quota_id column on volume_quotas.",+ "quota_limit": "The maximum quantity of the resource that the quota allows.",+ "usage": "The quantity of the resource in use now.",+ },+ },}diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/linode/linode.py b/products/warehouse_sources/backend/temporal/data_imports/sources/linode/linode.pyindex 43cb35f7e7a6..cb07bbd3c15c 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/linode/linode.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/linode/linode.py@@ -179,8 +179,11 @@ def _fanout_source(job_id: str,) -> SourceResponse:assert config.fanout is not None- # No resume state: the parent lists (invoices, LKE clusters) are small, so a restarted run- # re-walks them cheaply.+ child_endpoint_extra: Endpoint = (+ {"data_selector": "$", "paginator": "single_page"} if config.single_object else {"data_selector": "data"}+ )+ # No resume state: the parent lists (invoices, LKE clusters, volume quotas) are small, so a+ # restarted run re-walks them cheaply.dependent_resource = cast(Iterable[Any],build_dependent_resource(@@ -193,8 +196,8 @@ def _fanout_source(job_id=job_id,db_incremental_field_last_value=None,parent_endpoint_extra={"data_selector": "data"},- child_endpoint_extra={"data_selector": "data"},- page_size_param="page_size",+ child_endpoint_extra=child_endpoint_extra,+ page_size_param=None if config.single_object else "page_size",),)return _make_source_response(config, endpoint, lambda: dependent_resource)diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/linode/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/linode/settings.pyindex b0a48b500260..a69124a65d9f 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/linode/settings.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/linode/settings.py@@ -215,6 +215,32 @@ LINODE_ENDPOINTS: dict[str, LinodeEndpointConfig] = {},],),+ "volume_quotas": LinodeEndpointConfig(+ name="volume_quotas",+ path="/volumes/quotas",+ # Quotas key on a string `quota_id` (e.g. `vol-capacity-us-east`) and carry no timestamp.+ primary_keys=["quota_id"],+ incremental_fields=[],+ ),+ "volume_quota_usage": LinodeEndpointConfig(+ name="volume_quota_usage",+ path="/volumes/quotas/{quota_id}/usage",+ # One row per quota: the current usage against its limit, replaced on every sync.+ primary_keys=["quota_id"],+ single_object=True,+ incremental_fields=[],+ fanout=DependentEndpointConfig(+ parent_name="volume_quotas",+ resolve_param="quota_id",+ resolve_field="quota_id",+ include_from_parent=["quota_id"],+ parent_field_renames={"quota_id": "quota_id"},+ # The usage request takes no page params, so the listing's page size is set here instead.+ parent_params={"page_size": PAGE_SIZE},+ # Quotas with `has_usage` false return 404 for their usage.+ child_response_actions=[{"status_code": 404, "action": "ignore", "message": None}],+ ),+ ),}ENDPOINTS = tuple(LINODE_ENDPOINTS.keys())diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/linode/tests/test_linode.py b/products/warehouse_sources/backend/temporal/data_imports/sources/linode/tests/test_linode.pyindex 10100c8d20e4..f9815b04bd52 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/linode/tests/test_linode.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/linode/tests/test_linode.py@@ -182,6 +182,26 @@ class TestPagination:assert result.response is not Noneassert result.response.partition_keys == ["invoice_date"]+ def test_volume_quota_usage_fans_out_per_quota_and_skips_quotas_without_usage(self) -> None:+ def respond(request: RecordedRequest) -> ScriptedResponse:+ if request.path == "/v4/volumes/quotas":+ return _page([{"quota_id": "vol-capacity-us-east"}, {"quota_id": "vol-attachments-us-east"}])+ if request.path == "/v4/volumes/quotas/vol-attachments-us-east/usage":+ return ScriptedResponse(status=404, json={"errors": [{"reason": "Not found"}]})+ return ScriptedResponse(json={"quota_limit": 51200, "usage": 12800})++ result = _driver().run("volume_quota_usage", respond)++ assert result.raised is None+ assert result.paths == [+ "/v4/volumes/quotas",+ "/v4/volumes/quotas/vol-capacity-us-east/usage",+ "/v4/volumes/quotas/vol-attachments-us-east/usage",+ ]+ assert result.requests[0].param("page_size") == "500"+ assert result.requests[1].param("page_size") is None+ assert result.rows == [{"quota_id": "vol-capacity-us-east", "quota_limit": 51200, "usage": 12800}]+def test_account_transfer_yields_the_bare_object_as_one_row(self) -> None:transfer = {"quota": 4000, "used": 120, "billable": 0, "region_transfers": []}result = _driver().run("account_transfer", [ScriptedResponse(json=transfer)])