Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(data-warehouse): add linode volume quota tables (#119525)

Tom Owers9532c6bc66fe
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
index 883e641d62f6..2a835f579fdc 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
@@ -7384,6 +7384,7 @@ Diffed against: <https://raw.githubusercontent.com/linode/linode-api-openapi/mai
- [ ] `object-storage/buckets (+ object-storage/transfer, quotas usage)` — per-bucket size and object count plus transfer usage for storage cost attribution (medium)
- [ ] `support/tickets (+ /{ticketId}/replies)` — support ticket volume and response history (medium)
- [ ] `account/maintenance` — scheduled and completed host maintenance events per entity - explains reboots seen in the events table (medium)
+- [x] `volumes/quotas (+ /volumes/quotas/{quotaId}/usage)` — Block Storage quota limits and current usage per region (medium). Added as `volume_quotas` and `volume_quota_usage` (fan-out over `volume_quotas`, full refresh). Quotas without usage data return 404 and get no usage row.
Note: The Akamai/Linode techdocs site serves no spec directly; the authoritative OpenAPI is linode/linode-api-openapi on GitHub (openapi.json, spec version 4.229.1, 249 GET paths). Also unsynced but lower value: account/logins, tags, placement/groups, vpcs/subnets, networking/firewalls, longview, managed services and stackscripts.
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/linode/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/linode/canonical_descriptions.py
index 3e52dc760b18..1b5fa3c01d33 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/linode/canonical_descriptions.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/linode/canonical_descriptions.py
@@ -265,4 +265,28 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {
"bytes_total": "The total public network traffic sent and received by the Linode in the month, in bytes.",
},
},
+ "volume_quotas": {
+ "description": "Block Storage (Volumes) quotas applied to the account, with the maximum allowed per region or globally.",
+ "docs_url": "https://techdocs.akamai.com/linode-api/reference/get-volume-quotas",
+ "columns": {
+ "quota_id": "The id of the quota, formatted as vol-{quota_type}-{region}, for example vol-capacity-us-east.",
+ "quota_name": "The human-readable name of the quota.",
+ "quota_type": "The type of the quota: vol-capacity (storage capacity), vol-volumes (number of volumes) or vol-attachments (number of attached volumes).",
+ "description": "A description of what the quota controls.",
+ "quota_limit": "The maximum quantity of the resource_metric that the quota allows.",
+ "resource_metric": "The resource the quota measures: gigabyte, volume or attachment.",
+ "region": "The region the quota applies to. Joins to the id column on regions. Null for global quotas.",
+ "scope": "Whether the quota applies per region or globally.",
+ "has_usage": "Whether usage data is available for the quota in volume_quota_usage.",
+ },
+ },
+ "volume_quota_usage": {
+ "description": "Current usage of each Block Storage (Volumes) quota on the account. Quotas without usage data have no row.",
+ "docs_url": "https://techdocs.akamai.com/linode-api/reference/get-volume-quota-usage",
+ "columns": {
+ "quota_id": "The id of the quota. Joins to the quota_id column on volume_quotas.",
+ "quota_limit": "The maximum quantity of the resource that the quota allows.",
+ "usage": "The quantity of the resource in use now.",
+ },
+ },
}
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/linode/linode.py b/products/warehouse_sources/backend/temporal/data_imports/sources/linode/linode.py
index 43cb35f7e7a6..cb07bbd3c15c 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/linode/linode.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/linode/linode.py
@@ -179,8 +179,11 @@ def _fanout_source(
job_id: str,
) -> SourceResponse:
assert config.fanout is not None
- # No resume state: the parent lists (invoices, LKE clusters) are small, so a restarted run
- # re-walks them cheaply.
+ child_endpoint_extra: Endpoint = (
+ {"data_selector": "$", "paginator": "single_page"} if config.single_object else {"data_selector": "data"}
+ )
+ # No resume state: the parent lists (invoices, LKE clusters, volume quotas) are small, so a
+ # restarted run re-walks them cheaply.
dependent_resource = cast(
Iterable[Any],
build_dependent_resource(
@@ -193,8 +196,8 @@ def _fanout_source(
job_id=job_id,
db_incremental_field_last_value=None,
parent_endpoint_extra={"data_selector": "data"},
- child_endpoint_extra={"data_selector": "data"},
- page_size_param="page_size",
+ child_endpoint_extra=child_endpoint_extra,
+ page_size_param=None if config.single_object else "page_size",
),
)
return _make_source_response(config, endpoint, lambda: dependent_resource)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/linode/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/linode/settings.py
index b0a48b500260..a69124a65d9f 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/linode/settings.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/linode/settings.py
@@ -215,6 +215,32 @@ LINODE_ENDPOINTS: dict[str, LinodeEndpointConfig] = {
},
],
),
+ "volume_quotas": LinodeEndpointConfig(
+ name="volume_quotas",
+ path="/volumes/quotas",
+ # Quotas key on a string `quota_id` (e.g. `vol-capacity-us-east`) and carry no timestamp.
+ primary_keys=["quota_id"],
+ incremental_fields=[],
+ ),
+ "volume_quota_usage": LinodeEndpointConfig(
+ name="volume_quota_usage",
+ path="/volumes/quotas/{quota_id}/usage",
+ # One row per quota: the current usage against its limit, replaced on every sync.
+ primary_keys=["quota_id"],
+ single_object=True,
+ incremental_fields=[],
+ fanout=DependentEndpointConfig(
+ parent_name="volume_quotas",
+ resolve_param="quota_id",
+ resolve_field="quota_id",
+ include_from_parent=["quota_id"],
+ parent_field_renames={"quota_id": "quota_id"},
+ # The usage request takes no page params, so the listing's page size is set here instead.
+ parent_params={"page_size": PAGE_SIZE},
+ # Quotas with `has_usage` false return 404 for their usage.
+ child_response_actions=[{"status_code": 404, "action": "ignore", "message": None}],
+ ),
+ ),
}
ENDPOINTS = tuple(LINODE_ENDPOINTS.keys())
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/linode/tests/test_linode.py b/products/warehouse_sources/backend/temporal/data_imports/sources/linode/tests/test_linode.py
index 10100c8d20e4..f9815b04bd52 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/linode/tests/test_linode.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/linode/tests/test_linode.py
@@ -182,6 +182,26 @@ class TestPagination:
assert result.response is not None
assert result.response.partition_keys == ["invoice_date"]
+ def test_volume_quota_usage_fans_out_per_quota_and_skips_quotas_without_usage(self) -> None:
+ def respond(request: RecordedRequest) -> ScriptedResponse:
+ if request.path == "/v4/volumes/quotas":
+ return _page([{"quota_id": "vol-capacity-us-east"}, {"quota_id": "vol-attachments-us-east"}])
+ if request.path == "/v4/volumes/quotas/vol-attachments-us-east/usage":
+ return ScriptedResponse(status=404, json={"errors": [{"reason": "Not found"}]})
+ return ScriptedResponse(json={"quota_limit": 51200, "usage": 12800})
+
+ result = _driver().run("volume_quota_usage", respond)
+
+ assert result.raised is None
+ assert result.paths == [
+ "/v4/volumes/quotas",
+ "/v4/volumes/quotas/vol-capacity-us-east/usage",
+ "/v4/volumes/quotas/vol-attachments-us-east/usage",
+ ]
+ assert result.requests[0].param("page_size") == "500"
+ assert result.requests[1].param("page_size") is None
+ assert result.rows == [{"quota_id": "vol-capacity-us-east", "quota_limit": 51200, "usage": 12800}]
+
def test_account_transfer_yields_the_bare_object_as_one_row(self) -> None:
transfer = {"quota": 4000, "used": 120, "billable": 0, "region_transfers": []}
result = _driver().run("account_transfer", [ScriptedResponse(json=transfer)])