Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(scaleway): add instance volumes, images, snapshots and ips tables (#129116)

Tom Owers5aa5594b397e
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
index e6be73423a6d..3c3ad0cc35d7 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
@@ -10629,10 +10629,10 @@ Diffed against: <https://www.scaleway.com/en/developers/api/billing/v2beta1/sche
- [x] `kubernetes clusters (GET /k8s/v1/regions/{region}/clusters)` — major compute inventory missing entirely from the resource picture (high)
- [x] `kubernetes pools and nodes (/k8s/v1/regions/{region}/pools, /nodes)` — node-level inventory that attributes k8s cost and capacity (high)
- [x] `managed database instances (GET /rdb/v1/regions/{region}/instances)` — managed Postgres/MySQL fleet, a large recurring cost line (high)
-- [ ] `instance volumes (GET /instance/v1/zones/{zone}/volumes)` — storage attached to the servers already synced; needed to attribute storage spend (high)
-- [ ] `instance images (GET /instance/v1/zones/{zone}/images)` — lookup resolving the image id carried on every synced server (medium)
-- [ ] `instance snapshots (GET /instance/v1/zones/{zone}/snapshots)` — snapshot inventory, a common source of untracked storage cost (medium)
-- [ ] `instance IPs (GET /instance/v1/zones/{zone}/ips)` — flexible IP inventory and attachment state, billed per unattached IP (medium)
+- [x] `instance volumes (GET /instance/v1/zones/{zone}/volumes)` — storage attached to the servers already synced; needed to attribute storage spend (high)
+- [x] `instance images (GET /instance/v1/zones/{zone}/images)` — lookup resolving the image id carried on every synced server (medium)
+- [x] `instance snapshots (GET /instance/v1/zones/{zone}/snapshots)` — snapshot inventory, a common source of untracked storage cost (medium)
+- [x] `instance IPs (GET /instance/v1/zones/{zone}/ips)` — flexible IP inventory and attachment state, billed per unattached IP (medium)
- [ ] `billing discounts (GET /billing/v2beta1/discounts)` — discounts applied to invoices; without it invoiced totals cannot be reconciled (medium)
- [ ] `rdb backups and snapshots (/rdb/v1/regions/{region}/backups, /snapshots)` — database backup inventory and retention footprint (medium)
- [ ] `instance security groups (GET /instance/v1/zones/{zone}/security_groups)` — lookup resolving the security group referenced by each server, useful for posture reporting (low)
diff --git a/products/warehouse_sources/sources/scaleway/canonical_descriptions.py b/products/warehouse_sources/sources/scaleway/canonical_descriptions.py
index d0bb959c041e..a3626e3ecbb0 100644
--- a/products/warehouse_sources/sources/scaleway/canonical_descriptions.py
+++ b/products/warehouse_sources/sources/scaleway/canonical_descriptions.py
@@ -149,6 +149,81 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {
"modification_date": "Date the server was last modified.",
},
},
+ "instance_volumes": {
+ "description": "Instance volumes across all zones the organization operates in.",
+ "docs_url": "https://www.scaleway.com/en/developers/api/instance/#path-volumes-list-volumes",
+ "columns": {
+ "id": "Unique identifier of the volume.",
+ "name": "Name of the volume.",
+ "organization": "ID of the organization the volume belongs to.",
+ "project": "ID of the project the volume belongs to.",
+ "size": "Disk size of the volume, in bytes.",
+ "volume_type": "Type of the volume.",
+ "state": "Current state of the volume.",
+ "server": "Instance the volume is attached to, if any.",
+ "tags": "Tags associated with the volume.",
+ "zone": "Availability zone the volume is located in.",
+ "creation_date": "Date the volume was created.",
+ "modification_date": "Date the volume was last modified.",
+ },
+ },
+ "instance_images": {
+ "description": "Instance images owned by the organization, across all zones.",
+ "docs_url": "https://www.scaleway.com/en/developers/api/instance/#path-images-list-instance-images",
+ "columns": {
+ "id": "Unique identifier of the image.",
+ "name": "Name of the image.",
+ "organization": "ID of the organization the image belongs to.",
+ "project": "ID of the project the image belongs to.",
+ "arch": "CPU architecture of the image.",
+ "public": "Whether the image is public.",
+ "state": "Current state of the image.",
+ "root_volume": "Root volume of the image.",
+ "extra_volumes": "Additional volumes of the image.",
+ "from_server": "ID of the Instance the image was created from.",
+ "tags": "Tags associated with the image.",
+ "zone": "Availability zone the image is located in.",
+ "creation_date": "Date the image was created.",
+ "modification_date": "Date the image was last modified.",
+ },
+ },
+ "instance_snapshots": {
+ "description": "Instance volume snapshots across all zones the organization operates in.",
+ "docs_url": "https://www.scaleway.com/en/developers/api/instance/#path-snapshots-list-snapshots",
+ "columns": {
+ "id": "Unique identifier of the snapshot.",
+ "name": "Name of the snapshot.",
+ "organization": "ID of the organization the snapshot belongs to.",
+ "project": "ID of the project the snapshot belongs to.",
+ "size": "Size of the snapshot, in bytes.",
+ "volume_type": "Volume type of the snapshot.",
+ "state": "Current state of the snapshot.",
+ "base_volume": "Volume the snapshot was taken from.",
+ "error_reason": "Reason the snapshot import failed, if it did.",
+ "tags": "Tags associated with the snapshot.",
+ "zone": "Availability zone the snapshot is located in.",
+ "creation_date": "Date the snapshot was created.",
+ "modification_date": "Date the snapshot was last modified.",
+ },
+ },
+ "instance_ips": {
+ "description": "Flexible IP addresses across all zones the organization operates in.",
+ "docs_url": "https://www.scaleway.com/en/developers/api/instance/#path-ips-list-all-flexible-ips",
+ "columns": {
+ "id": "Unique identifier of the flexible IP.",
+ "address": "IP address.",
+ "prefix": "IP network of the flexible IP.",
+ "type": "Type of the flexible IP.",
+ "state": "Current state of the flexible IP.",
+ "server": "Instance the flexible IP is attached to. Empty when the IP is detached.",
+ "reverse": "Reverse DNS of the flexible IP.",
+ "ipam_id": "ID of the IP in Scaleway IPAM.",
+ "organization": "ID of the organization the flexible IP belongs to.",
+ "project": "ID of the project the flexible IP belongs to.",
+ "tags": "Tags associated with the flexible IP.",
+ "zone": "Availability zone the flexible IP is reserved in.",
+ },
+ },
"audit_trail_events": {
"description": "Audit trail events recording actions taken on the organization's resources.",
"docs_url": "https://www.scaleway.com/en/developers/api/audit-trail/",
diff --git a/products/warehouse_sources/sources/scaleway/docs/scaleway.md b/products/warehouse_sources/sources/scaleway/docs/scaleway.md
index 3707e3ae4c53..21a8c84934e6 100644
--- a/products/warehouse_sources/sources/scaleway/docs/scaleway.md
+++ b/products/warehouse_sources/sources/scaleway/docs/scaleway.md
@@ -10,7 +10,7 @@ import SourceSetupIntro from "../\_snippets/source-setup-intro.mdx"
import SyncModes from "../\_snippets/sync-modes.mdx"
import TroubleshootingLink from "../\_snippets/dw-troubleshooting-link.mdx"
-Sync your [Scaleway](https://www.scaleway.com) organization data — billing invoices and consumption, IAM identities, projects, audit trail events, compute instances, Kubernetes clusters, and managed databases — into PostHog to build cost, security, and infrastructure reporting alongside your product data.
+Sync your [Scaleway](https://www.scaleway.com) organization data — billing invoices and consumption, IAM identities, projects, audit trail events, compute instances and their volumes, images, snapshots and IPs, Kubernetes clusters, and managed databases — into PostHog to build cost, security, and infrastructure reporting alongside your product data.
## Prerequisites
@@ -35,7 +35,7 @@ Grant the API key the read permission sets that match the tables you want to syn
- `ProjectReadOnly` — projects
- `BillingReadOnly` — invoices, billing consumptions
- `AuditTrailReadOnly` — audit trail events
-- `InstancesReadOnly` — instance servers
+- `InstancesReadOnly` — instance servers, volumes, images, snapshots, IPs
- `KubernetesReadOnly` — Kubernetes clusters, pools, nodes
- `RelationalDatabasesReadOnly` — database instances
@@ -59,6 +59,6 @@ All Scaleway tables sync via full refresh. Resource inventories (IAM, projects,
**"Your API key is missing the read permission set required to sync ..."**: the API key is valid but lacks the permission set for that table. Add the matching read permission (see the list above) to the key in the Scaleway console, then reconnect.
-**No rows for a table**: some resources are region- or zone-scoped. Instance servers are pulled from every zone your organization can use, audit trail events from the `fr-par` and `nl-ams` regions, Kubernetes clusters, pools and nodes from `fr-par`, `nl-ams`, `pl-waw` and `it-mil`, and database instances from `fr-par`, `nl-ams` and `pl-waw` — a table can legitimately be empty if you have no resources there.
+**No rows for a table**: some resources are region- or zone-scoped. Instance servers, volumes, images, snapshots and IPs are pulled from every zone your organization can use, audit trail events from the `fr-par` and `nl-ams` regions, Kubernetes clusters, pools and nodes from `fr-par`, `nl-ams`, `pl-waw` and `it-mil`, and database instances from `fr-par`, `nl-ams` and `pl-waw` — a table can legitimately be empty if you have no resources there.
<TroubleshootingLink />
diff --git a/products/warehouse_sources/sources/scaleway/settings.py b/products/warehouse_sources/sources/scaleway/settings.py
index 0c2b206bc054..5339e3da0896 100644
--- a/products/warehouse_sources/sources/scaleway/settings.py
+++ b/products/warehouse_sources/sources/scaleway/settings.py
@@ -183,6 +183,48 @@ SCALEWAY_ENDPOINTS: dict[str, ScalewayEndpointConfig] = {
order_param="order",
order_value="creation_date_asc",
),
+ # The other Instance list endpoints take no sort param, so they page in the API's default order.
+ "instance_volumes": ScalewayEndpointConfig(
+ name="instance_volumes",
+ path="/instance/v1/zones/{zone}/volumes",
+ data_key="volumes",
+ pagination="per_page",
+ partition_key="creation_date",
+ scope="zone",
+ scope_values=INSTANCE_ZONES,
+ org_param="organization",
+ ),
+ "instance_images": ScalewayEndpointConfig(
+ name="instance_images",
+ path="/instance/v1/zones/{zone}/images",
+ data_key="images",
+ pagination="per_page",
+ partition_key="creation_date",
+ scope="zone",
+ scope_values=INSTANCE_ZONES,
+ # The organization filter keeps the table to the organization's own images, not the public catalog.
+ org_param="organization",
+ ),
+ "instance_snapshots": ScalewayEndpointConfig(
+ name="instance_snapshots",
+ path="/instance/v1/zones/{zone}/snapshots",
+ data_key="snapshots",
+ pagination="per_page",
+ partition_key="creation_date",
+ scope="zone",
+ scope_values=INSTANCE_ZONES,
+ org_param="organization",
+ ),
+ "instance_ips": ScalewayEndpointConfig(
+ name="instance_ips",
+ path="/instance/v1/zones/{zone}/ips",
+ data_key="ips",
+ pagination="per_page",
+ # Flexible IPs carry no creation date, so this table is not partitioned.
+ scope="zone",
+ scope_values=INSTANCE_ZONES,
+ org_param="organization",
+ ),
"audit_trail_events": ScalewayEndpointConfig(
name="audit_trail_events",
path="/audit-trail/v1alpha1/regions/{region}/events",
diff --git a/products/warehouse_sources/sources/scaleway/source.py b/products/warehouse_sources/sources/scaleway/source.py
index 8b523b493869..55e66db2fa66 100644
--- a/products/warehouse_sources/sources/scaleway/source.py
+++ b/products/warehouse_sources/sources/scaleway/source.py
@@ -80,7 +80,7 @@ Grant the key the read permission sets for the data you want to sync, for exampl
- `ProjectReadOnly` for projects
- `BillingReadOnly` for invoices and billing consumptions
- `AuditTrailReadOnly` for audit trail events
-- `InstancesReadOnly` for instance servers
+- `InstancesReadOnly` for instance servers, volumes, images, snapshots and IPs
- `KubernetesReadOnly` for Kubernetes clusters, pools and nodes
- `RelationalDatabasesReadOnly` for database instances
""",
diff --git a/products/warehouse_sources/sources/scaleway/tests/test_scaleway.py b/products/warehouse_sources/sources/scaleway/tests/test_scaleway.py
index b8aeaa950d99..1d1950e93382 100644
--- a/products/warehouse_sources/sources/scaleway/tests/test_scaleway.py
+++ b/products/warehouse_sources/sources/scaleway/tests/test_scaleway.py
@@ -48,6 +48,11 @@ class TestBaseParams:
("projects", "page_size", "organization_id"),
("invoices", "page_size", "organization_id"),
("instance_servers", "per_page", "organization"),
+ ("instance_volumes", "per_page", "organization"),
+ # Without the organization filter, images would return Scaleway's whole public catalog.
+ ("instance_images", "per_page", "organization"),
+ ("instance_snapshots", "per_page", "organization"),
+ ("instance_ips", "per_page", "organization"),
("audit_trail_events", "page_size", "organization_id"),
("billing_consumptions", "page_size", "organization_id"),
("kubernetes_clusters", "page_size", "organization_id"),
@@ -84,6 +89,10 @@ class TestResolvePath:
[
("users", "fr-par", "/iam/v1alpha1/users"),
("instance_servers", "nl-ams-1", "/instance/v1/zones/nl-ams-1/servers"),
+ ("instance_volumes", "fr-par-2", "/instance/v1/zones/fr-par-2/volumes"),
+ ("instance_images", "pl-waw-3", "/instance/v1/zones/pl-waw-3/images"),
+ ("instance_snapshots", "it-mil-1", "/instance/v1/zones/it-mil-1/snapshots"),
+ ("instance_ips", "nl-ams-2", "/instance/v1/zones/nl-ams-2/ips"),
("audit_trail_events", "fr-par", "/audit-trail/v1alpha1/regions/fr-par/events"),
("kubernetes_clusters", "it-mil", "/k8s/v1/regions/it-mil/clusters"),
("database_instances", "pl-waw", "/rdb/v1/regions/pl-waw/instances"),
@@ -141,15 +150,23 @@ class TestTokenPagination:
class TestFanOut:
- def test_iterates_every_zone_and_advances_scope_between_them(self) -> None:
+ @parameterized.expand(
+ [
+ ("instance_servers", "servers"),
+ ("instance_volumes", "volumes"),
+ ("instance_images", "images"),
+ ("instance_snapshots", "snapshots"),
+ ("instance_ips", "ips"),
+ ]
+ )
+ def test_iterates_every_zone_and_advances_scope_between_them(self, endpoint: str, data_key: str) -> None:
# Each zone returns one short page, so the connector visits every zone once.
- responses = [_response({"servers": [{"id": f"srv-{z}"}]}) for z in INSTANCE_ZONES]
+ responses = [_response({data_key: [{"id": f"{data_key}-{z}"}]}) for z in INSTANCE_ZONES]
- result = _drive("instance_servers", responses)
+ result = _drive(endpoint, responses)
- assert len(result.requests) == len(INSTANCE_ZONES)
- for zone in INSTANCE_ZONES:
- assert any(f"/zones/{zone}/servers" in path for path in result.paths)
+ assert result.paths == [f"/instance/v1/zones/{zone}/{data_key}" for zone in INSTANCE_ZONES]
+ assert all(query["organization"] == ("org-123",) for query in result.queries)
# Between scopes the bookmark advances so a crash resumes at the next zone, not the first.
assert [state.scope_index for state in result.saved_states] == list(range(1, len(INSTANCE_ZONES)))
assert len(result.rows) == len(INSTANCE_ZONES)