Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(data-warehouse): add fastly ai usage metrics table (#119248)

Tom Owers4bcdd202fcde
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
index 9f19a3047446..4224c2d42b70 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
@@ -4465,7 +4465,7 @@ Note: Developer docs, reference routes, sitemap, and spec candidates returned Cl
## Fastly — **thin**
-Today (15): `acl_entries`, `billing_usage_metrics`, `current_user`, `dictionary_items`, `historical_stats`, `invoices`, `origin_inspector`, `pops`, `service_acls`, `service_authorizations`, `service_backends`, `service_dictionaries`, `service_domains`, `service_versions`, `services`
+Today (16): `acl_entries`, `ai_usage_metrics`, `billing_usage_metrics`, `current_user`, `dictionary_items`, `historical_stats`, `invoices`, `origin_inspector`, `pops`, `service_acls`, `service_authorizations`, `service_backends`, `service_dictionaries`, `service_domains`, `service_versions`, `services`
Diffed against: <https://www.fastly.com/documentation/reference/api/>
@@ -4480,6 +4480,7 @@ Diffed against: <https://www.fastly.com/documentation/reference/api/>
- [ ] `account/customer (and its users list)` — lookup resolving customer ids on services and users; today only current_user is synced (medium)
- [x] `account/service-authorization` — user-to-service permission membership table (medium)
- [x] `utils/pops` — lookup resolving POP/datacenter codes that appear in stats and inspector breakdowns (medium)
+- [x] `ai-runtime-control/usage-metrics` — AI gateway request, session and token usage per virtual key, provider and model (medium)
Note: Coverage is service configuration objects, the account/billing family, and the historical stats and Origin Inspector halves of metrics-stats; the domain inspector and bots halves are not exposed. Fastly's own OpenAPI YAML is no longer served at the old developer.fastly.com path (404) - the category tree was read from the live documentation reference index and its per-category pages. Logging endpoint types (~25 of them), TLS, purging and VCL objects are config/plumbing and deliberately excluded.
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/fastly/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/fastly/canonical_descriptions.py
index 368d1db313ba..8090db69c66f 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/fastly/canonical_descriptions.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/fastly/canonical_descriptions.py
@@ -236,4 +236,20 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {
"end_time": "End of the billing cycle the usage covers.",
},
},
+ "ai_usage_metrics": {
+ "description": "Daily AI Runtime Control request, session and token usage per virtual key, provider and model.",
+ "docs_url": "https://www.fastly.com/documentation/reference/api/ai-runtime-control/usage-metrics/",
+ "columns": {
+ "date": "The date (or hour) the usage was measured.",
+ "start_time": "Start of the day the usage was measured, as a Unix timestamp.",
+ "usage_type": "The type of usage being measured, such as requests, sessions, input_tokens or output_tokens.",
+ "quantity": "The quantity of the usage type for this date.",
+ "virtual_key_id": "The ID of the virtual key this usage is attributed to.",
+ "virtual_key_name": "The display name of the virtual key this usage is attributed to.",
+ "provider": "The name of the provider connection that ultimately served the request.",
+ "provider_connection_id": "The ID of the provider connection that ultimately served the request.",
+ "primary_provider": "The ID of the provider connection originally requested, before any failover occurred.",
+ "model": "The AI model identifier used.",
+ },
+ },
}
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/fastly/fastly.py b/products/warehouse_sources/backend/temporal/data_imports/sources/fastly/fastly.py
index 1343707bfd58..e8c37738e2ff 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/fastly/fastly.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/fastly/fastly.py
@@ -54,6 +54,9 @@ BUCKET_SECONDS: dict[str, int] = {"minute": 60, "hour": 60 * 60, "day": 24 * 60
# than failing every other service's rows with it.
ORIGIN_INSPECTOR_UNAVAILABLE_STATUSES = frozenset({403, 404})
+# AI Runtime Control aggregates usage by `daily` or `hourly`; a day matches the other metrics tables.
+AI_USAGE_METRICS_GRANULARITY = "daily"
+
class FastlyRetryableError(Exception):
pass
@@ -257,6 +260,20 @@ def _as_epoch_seconds(value: Any) -> int | None:
return None
+def _date_to_epoch_seconds(value: Any) -> int | None:
+ """Parse an AI Runtime Control `date` into Unix seconds. The docs give `YYYY-MM-DD` while live
+ responses carry a full UTC timestamp, so accept any ISO form and read a zoneless one as UTC."""
+ if not isinstance(value, str) or not value:
+ return None
+ try:
+ parsed = date_parser.isoparse(value)
+ except (ValueError, OverflowError):
+ return None
+ if parsed.tzinfo is None:
+ parsed = parsed.replace(tzinfo=UTC)
+ return int(parsed.timestamp())
+
+
def _flatten_json_api(item: dict[str, Any]) -> dict[str, Any]:
"""Lift a JSON:API resource's `attributes` into the row root and each `relationships` member down
to a `<name>_id` column, so the row reads like every other Fastly table."""
@@ -510,6 +527,40 @@ def _get_origin_inspector_rows(
cursor = next_cursor
+def _get_ai_usage_metrics_rows(
+ session: requests.Session,
+ config: FastlyEndpointConfig,
+ headers: dict[str, str],
+ logger: FilteringBoundLogger,
+ incremental_start: int | None,
+) -> Iterator[list[dict[str, Any]]]:
+ """Read the AI Runtime Control usage report, which answers in one unpaginated response.
+
+ With no `from`, Fastly applies its own default window, so a full refresh takes that and an
+ incremental run takes everything since its watermark."""
+ params: dict[str, Any] = {"granularity": AI_USAGE_METRICS_GRANULARITY, "sort": "date"}
+ if incremental_start is not None:
+ params["from"] = datetime.fromtimestamp(incremental_start, tz=UTC).strftime("%Y-%m-%dT%H:%M:%SZ")
+
+ response = _fetch(session, _build_url(f"{FASTLY_BASE_URL}{config.path}", params), headers, logger)
+ payload = response.json()
+ data = payload.get("data") if isinstance(payload, dict) else None
+ if not isinstance(data, list):
+ return
+
+ rows: list[dict[str, Any]] = []
+ for row in data:
+ if not isinstance(row, dict):
+ continue
+ start_time = _date_to_epoch_seconds(row.get("date"))
+ if start_time is None:
+ logger.warning(f"Fastly: skipping AI usage metric with an unreadable date: {row.get('date')!r}")
+ continue
+ rows.append({**row, "start_time": start_time})
+ if rows:
+ yield rows
+
+
def _get_service_list_rows(
session: requests.Session,
headers: dict[str, str],
@@ -730,6 +781,8 @@ def get_rows(
yield from _get_origin_inspector_rows(
session, config, headers, logger, resumable_source_manager, incremental_start
)
+ elif config.kind == "ai_usage_metrics":
+ yield from _get_ai_usage_metrics_rows(session, config, headers, logger, incremental_start)
elif config.kind == "service_list":
yield from _get_service_list_rows(session, headers, logger, resumable_source_manager)
elif config.kind == "version_resource_child":
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/fastly/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/fastly/settings.py
index 4c952839a8e9..44d7bee69ae4 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/fastly/settings.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/fastly/settings.py
@@ -22,6 +22,7 @@ from products.warehouse_sources.backend.types import IncrementalField, Increment
# across `attributes` and `relationships`, and the next page is a body link.
# - "stats_list": GET /stats returns one array of time buckets per service, keyed by service id.
# - "origin_inspector": fan out over every service, reading its origin metrics timeseries.
+# - "ai_usage_metrics": GET an unpaginated AI Runtime Control usage report; rows sit under `data`.
FastlyEndpointKind = Literal[
"object",
"service_list",
@@ -34,11 +35,12 @@ FastlyEndpointKind = Literal[
"json_api_list",
"stats_list",
"origin_inspector",
+ "ai_usage_metrics",
]
-# Fastly reports both metrics families in whole time buckets keyed by their start. The cursor is the
-# bucket start as a Unix timestamp, which is also what the `from` / `start` request params take, so a
-# watermark feeds straight back into the next sync with no reformatting.
+# Fastly reports every metrics family in whole time buckets keyed by their start. The cursor is the
+# bucket start as a Unix timestamp, which is also what the `/stats` and Origin Inspector `from` / `start`
+# request params take, so a watermark feeds straight back into the next sync with no reformatting.
_BUCKET_START_INCREMENTAL_FIELDS: list[IncrementalField] = [
{
"label": "Bucket start",
@@ -175,6 +177,18 @@ FASTLY_ENDPOINTS: dict[str, FastlyEndpointConfig] = {
should_sync_default=False,
description="Daily origin latency, status code and byte counts per origin host, for services with Origin Inspector enabled.",
),
+ "ai_usage_metrics": FastlyEndpointConfig(
+ name="ai_usage_metrics",
+ path="/ai-runtime-control/v1/usage-metrics",
+ kind="ai_usage_metrics",
+ partition_key="start_time",
+ primary_keys=["start_time", "usage_type", "virtual_key_id", "provider", "model"],
+ incremental_fields=_BUCKET_START_INCREMENTAL_FIELDS,
+ incremental_lookback_seconds=METRICS_LOOKBACK_SECONDS,
+ # AI Runtime Control is a separate product, so most accounts would sync an empty table.
+ should_sync_default=False,
+ description="Daily AI Runtime Control request, session and token usage per virtual key, provider and model.",
+ ),
"service_authorizations": FastlyEndpointConfig(
name="service_authorizations",
path="/service-authorizations",
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/fastly/tests/test_fastly.py b/products/warehouse_sources/backend/temporal/data_imports/sources/fastly/tests/test_fastly.py
index 3932a16ce479..00b296dc3ab0 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/fastly/tests/test_fastly.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/fastly/tests/test_fastly.py
@@ -617,6 +617,48 @@ class TestGetRowsStatsList:
assert result.rows == []
+class TestGetRowsAiUsageMetrics:
+ URL = f"{FASTLY_BASE_URL}/ai-runtime-control/v1/usage-metrics?granularity=daily&sort=date"
+
+ @parameterized.expand(
+ [
+ ("full_refresh", False, None),
+ ("incremental", True, "2023-11-14T22:13:20Z"),
+ ]
+ )
+ def test_watermark_is_sent_as_an_rfc3339_from(self, _name: str, incremental: bool, expected: str | None) -> None:
+ url = f"{self.URL}&from={expected}" if expected else self.URL
+ result = _drive(
+ "ai_usage_metrics",
+ {url: {"data": [], "meta": {"sort": "date", "total": 0}}},
+ should_use_incremental_field=incremental,
+ incremental_field="start_time",
+ db_incremental_field_last_value=1_700_000_000,
+ )
+ assert result.raised is None
+ assert result.params("from") == [expected]
+
+ def test_each_row_gets_its_day_as_start_time(self) -> None:
+ # The docs give `YYYY-MM-DD` while live responses carry a full timestamp; a row without a
+ # readable date has no partition or key and is dropped.
+ payload = {
+ "data": [
+ {"date": "2026-09-01T00:00:00Z", "usage_type": "requests", "quantity": 3},
+ {"date": "2026-09-02", "usage_type": "input_tokens", "quantity": 40},
+ {"date": "2026-09-03 00:00:00", "usage_type": "output_tokens", "quantity": 50},
+ {"date": None, "usage_type": "sessions", "quantity": 1},
+ ],
+ "meta": {"sort": "date", "total": 4},
+ }
+ result = _drive("ai_usage_metrics", {self.URL: payload})
+ assert result.raised is None
+ assert [(row["usage_type"], row["start_time"]) for row in result.rows] == [
+ ("requests", 1_788_220_800),
+ ("input_tokens", 1_788_307_200),
+ ("output_tokens", 1_788_393_600),
+ ]
+
+
class TestBucketTimeline:
@parameterized.expand(
[