Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(warehouse-sources): add openaq latest, flags and yearly tables (#123358)

Tom Owersd0389a2a8352
diff --git a/posthog/test/repo_invariants/dataclass_frozen_baseline.txt b/posthog/test/repo_invariants/dataclass_frozen_baseline.txt
index 850befe461a9..1efc084eccad 100644
--- a/posthog/test/repo_invariants/dataclass_frozen_baseline.txt
+++ b/posthog/test/repo_invariants/dataclass_frozen_baseline.txt
@@ -825,7 +825,6 @@
1 products/warehouse_sources/backend/temporal/data_imports/sources/openai_ads/settings.py
1 products/warehouse_sources/backend/temporal/data_imports/sources/openalex/openalex.py
1 products/warehouse_sources/backend/temporal/data_imports/sources/openaq/openaq.py
-1 products/warehouse_sources/backend/temporal/data_imports/sources/openaq/settings.py
1 products/warehouse_sources/backend/temporal/data_imports/sources/openfda/openfda.py
1 products/warehouse_sources/backend/temporal/data_imports/sources/openweather/settings.py
1 products/warehouse_sources/backend/temporal/data_imports/sources/opinion_stage/opinion_stage.py
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
index e3703660e98c..8e86e99fc77d 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
@@ -8761,15 +8761,15 @@ Note: Parsed the current endpoint index and entity references, including the col
## OpenAQ — gaps
-Today (12): `countries`, `instruments`, `licenses`, `locations`, `manufacturers`, `measurements`, `measurements_daily`, `measurements_hourly`, `owners`, `parameters`, `providers`, `sensors`
+Today (16): `countries`, `instruments`, `licenses`, `location_flags`, `location_latest`, `locations`, `manufacturers`, `measurements`, `measurements_daily`, `measurements_hourly`, `measurements_yearly`, `owners`, `parameters`, `providers`, `sensor_flags`, `sensors`
Diffed against: <https://api.openaq.org/openapi.json>
-- [ ] `/v3/locations/{id}/latest` — most recent value per location - the headline 'current air quality' read (high)
-- [ ] `/v3/sensors/{id}/flags` — data-quality flags needed to exclude bad readings from measurement aggregates (medium)
-- [ ] `/v3/locations/{id}/flags` — location-level data-quality flags for the same filtering (medium)
+- [x] `/v3/locations/{id}/latest` — most recent value per location - the headline 'current air quality' read (high)
+- [x] `/v3/sensors/{id}/flags` — data-quality flags needed to exclude bad readings from measurement aggregates (medium)
+- [x] `/v3/locations/{id}/flags` — location-level data-quality flags for the same filtering (medium)
- [ ] `/v3/parameters/{id}/latest` — latest reading per pollutant across the network (medium)
-- [ ] `/v3/sensors/{id}/years` — yearly rollups, cheaper than aggregating hourly data for long-range trends (medium)
+- [x] `/v3/sensors/{id}/years` — yearly rollups, cheaper than aggregating hourly data for long-range trends (medium)
- [ ] `/v3/manufacturers/{id}/instruments` — join table linking manufacturers to instruments we already sync (low)
- [ ] `/v3/sensors/{id}/hours/hourofday and /days/dayofweek, /days/monthofyear` — seasonal/diurnal aggregations the API precomputes (low)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/openaq/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/openaq/canonical_descriptions.py
index 3dba9080a92a..382faf2a6353 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/openaq/canonical_descriptions.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/openaq/canonical_descriptions.py
@@ -168,4 +168,62 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {
"flag_info": "Quality/flag information for the aggregated measurement.",
},
},
+ "measurements_yearly": {
+ "description": "Yearly aggregated measurements per sensor.",
+ "docs_url": "https://docs.openaq.org/resources/measurements",
+ "columns": {
+ "sensor_id": "Identifier of the sensor that produced the measurement.",
+ "datetime_from": "UTC start of the yearly period.",
+ "datetime_to": "UTC end of the yearly period.",
+ "value": "Aggregated value for the year, in the parameter's units.",
+ "parameter_id": "Identifier of the measured parameter.",
+ "parameter_name": "Short code of the measured parameter.",
+ "parameter_units": "Unit the value is reported in.",
+ "coordinates": "Latitude/longitude of the sensor's location.",
+ "summary": "Statistical summary of the readings aggregated into the year.",
+ "coverage": "Data-coverage summary for the year.",
+ "flag_info": "Quality/flag information for the aggregated measurement.",
+ },
+ },
+ "location_latest": {
+ "description": "Most recent reading of each sensor at each location.",
+ "docs_url": "https://docs.openaq.org/resources/latest",
+ "columns": {
+ "location_id": "Identifier of the location.",
+ "sensor_id": "Identifier of the sensor that produced the reading.",
+ "datetime": "UTC timestamp of the reading.",
+ "datetime_local": "Timestamp of the reading in the location's local timezone.",
+ "value": "Most recent measured value, in the parameter's units.",
+ "coordinates": "Latitude/longitude of the reading.",
+ },
+ },
+ "location_flags": {
+ "description": "Data-quality flags raised on a location for a period of time.",
+ "docs_url": "https://docs.openaq.org/api/operations/location_flags_get_v3_locations__locations_id__flags_get",
+ "columns": {
+ "location_id": "Identifier of the location the flag applies to.",
+ "flag_type_id": "Identifier of the flag type.",
+ "flag_type_label": "Human-readable label of the flag type.",
+ "flag_level": "Severity level of the flag.",
+ "datetime_from": "UTC start of the flagged period.",
+ "datetime_to": "UTC end of the flagged period.",
+ "sensor_ids": "Sensors at the location the flag covers.",
+ "note": "Free-text note explaining the flag.",
+ },
+ },
+ "sensor_flags": {
+ "description": "Data-quality flags that cover a sensor for a period of time.",
+ "docs_url": "https://docs.openaq.org/api/operations/sensor_flags_get_v3_sensors__sensor_id__flags_get",
+ "columns": {
+ "sensor_id": "Identifier of the sensor the flag was fetched for.",
+ "location_id": "Identifier of the location the flag applies to.",
+ "flag_type_id": "Identifier of the flag type.",
+ "flag_type_label": "Human-readable label of the flag type.",
+ "flag_level": "Severity level of the flag.",
+ "datetime_from": "UTC start of the flagged period.",
+ "datetime_to": "UTC end of the flagged period.",
+ "sensor_ids": "Sensors at the location the flag covers.",
+ "note": "Free-text note explaining the flag.",
+ },
+ },
}
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/openaq/openaq.py b/products/warehouse_sources/backend/temporal/data_imports/sources/openaq/openaq.py
index 5ead994957d5..e6128056c8c8 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/openaq/openaq.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/openaq/openaq.py
@@ -1,4 +1,5 @@
import dataclasses
+from collections.abc import Callable
from datetime import UTC, date, datetime
from functools import partial
from typing import Any, Optional
@@ -41,10 +42,12 @@ MAX_PAGES: int = 100
# pagination for the full-refresh reference tables.
_LIST_SORT_PARAMS = {"order_by": "id", "sort_order": "asc"}
-# Name of the internal parent resource that enumerates sensor ids for the measurement fan-out.
+# Names of the internal parent resources that enumerate sensor / location ids for the fan-outs.
_SENSOR_IDS_RESOURCE = "sensor_ids"
-# Key the framework injects a parent field under: `_{parent_resource}_{field}`.
+_LOCATION_IDS_RESOURCE = "location_ids"
+# Keys the framework injects a parent field under: `_{parent_resource}_{field}`.
_PARENT_SENSOR_ID_KEY = f"_{_SENSOR_IDS_RESOURCE}_id"
+_PARENT_LOCATION_ID_KEY = f"_{_LOCATION_IDS_RESOURCE}_id"
@dataclasses.dataclass
@@ -158,6 +161,47 @@ def _reshape_measurement(item: dict[str, Any]) -> dict[str, Any]:
return _flatten_measurement(item, item[_PARENT_SENSOR_ID_KEY])
+def _flatten_flag(item: dict[str, Any]) -> dict[str, Any]:
+ flag_type = item.get("flagType") or {}
+ return {
+ "location_id": item.get("locationId"),
+ "flag_type_id": flag_type.get("id"),
+ "flag_type_label": flag_type.get("label"),
+ "flag_level": flag_type.get("level"),
+ "datetime_from": (item.get("datetimeFrom") or {}).get("utc"),
+ "datetime_to": (item.get("datetimeTo") or {}).get("utc"),
+ "sensor_ids": item.get("sensorIds"),
+ "note": item.get("note"),
+ }
+
+
+def _reshape_location_flag(item: dict[str, Any]) -> dict[str, Any]:
+ return {**_flatten_flag(item), "location_id": item[_PARENT_LOCATION_ID_KEY]}
+
+
+def _reshape_sensor_flag(item: dict[str, Any]) -> dict[str, Any]:
+ return {"sensor_id": item[_PARENT_SENSOR_ID_KEY], **_flatten_flag(item)}
+
+
+def _reshape_latest(item: dict[str, Any]) -> dict[str, Any]:
+ reading_time = item.get("datetime") or {}
+ return {
+ "location_id": item[_PARENT_LOCATION_ID_KEY],
+ "sensor_id": item["sensorsId"],
+ "datetime": reading_time.get("utc"),
+ "datetime_local": reading_time.get("local"),
+ "value": item.get("value"),
+ "coordinates": item.get("coordinates"),
+ }
+
+
+_FANOUT_ROW_MAPPERS: dict[str, Callable[[dict[str, Any]], dict[str, Any]]] = {
+ "location_latest": _reshape_latest,
+ "location_flags": _reshape_location_flag,
+ "sensor_flags": _reshape_sensor_flag,
+}
+
+
def _client_config(api_key: str) -> ClientConfig:
return {
"base_url": OPENAQ_BASE_URL,
@@ -186,7 +230,7 @@ def _list_resource(endpoint: str, config: OpenAQEndpointConfig, page_size: int)
return resource
-def _measurement_resources(
+def _fanout_resources(
endpoint: str,
config: OpenAQEndpointConfig,
page_size: int,
@@ -201,13 +245,18 @@ def _measurement_resources(
"data_selector": "results",
"paginator": OpenAQPageNumberPaginator(page_size, MAX_PAGES),
},
- "data_map": _explode_location_sensor_ids,
}
+ if config.kind == "location_fanout":
+ parent["name"] = _LOCATION_IDS_RESOURCE
+ resolve_param = "locations_id"
+ else:
+ parent["data_map"] = _explode_location_sensor_ids
+ resolve_param = "sensors_id"
child_endpoint: Endpoint = {
"path": config.path,
"params": {
- "sensors_id": {"type": "resolve", "resource": _SENSOR_IDS_RESOURCE, "field": "id"},
+ resolve_param: {"type": "resolve", "resource": parent["name"], "field": "id"},
"limit": page_size,
},
"data_selector": "results",
@@ -227,7 +276,7 @@ def _measurement_resources(
"name": endpoint,
"endpoint": child_endpoint,
"include_from_parent": ["id"],
- "data_map": _reshape_measurement,
+ "data_map": _reshape_measurement if config.kind == "measurement" else _FANOUT_ROW_MAPPERS[endpoint],
}
return [parent, child]
@@ -247,11 +296,11 @@ def openaq_source(
resume = resumable_source_manager.load_state() if resumable_source_manager.can_resume() else None
- if config.kind == "measurement":
+ if config.kind in ("measurement", "location_fanout", "sensor_fanout"):
rest_config: RESTAPIConfig = {
"client": client_config,
"resource_defaults": {},
- "resources": _measurement_resources(
+ "resources": _fanout_resources(
endpoint, config, page_size, should_use_incremental_field, db_incremental_field_last_value
),
}
@@ -308,7 +357,7 @@ def openaq_source(
# incremental watermark relies on. Unverified against the live API (no key available); noted
# in the PR. If this proves wrong, the datetime_from server filter still bounds each fetch, so
# completeness holds — only mid-sync watermark checkpointing would need sort_mode="desc".
- sort_mode="asc",
+ sort_mode=config.sort_mode,
column_hints=resource.column_hints,
)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/openaq/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/openaq/settings.py
index 82c72c4367d9..eadebf0aa32e 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/openaq/settings.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/openaq/settings.py
@@ -9,7 +9,9 @@ from products.warehouse_sources.backend.types import IncrementalField, Increment
# `sensors` array into one row per sensor. Full refresh (single locations walk, cheap).
# - "measurement": fanned out per sensor (/v3/sensors/{sensors_id}/...). Incremental via the
# API's server-side datetime filter. Request-heavy (one paginated fetch per sensor), so opt-in.
-EndpointKind = Literal["list", "sensors", "measurement"]
+# - "location_fanout" / "sensor_fanout": fanned out per location (/v3/locations/{locations_id}/...)
+# or per sensor (/v3/sensors/{sensors_id}/...). Full refresh. Request-heavy, so opt-in.
+EndpointKind = Literal["list", "sensors", "measurement", "location_fanout", "sensor_fanout"]
def _datetime_incremental_field(name: str) -> IncrementalField:
@@ -21,11 +23,11 @@ def _datetime_incremental_field(name: str) -> IncrementalField:
}
-@dataclass
+@dataclass(frozen=True)
class OpenAQEndpointConfig:
name: str
kind: EndpointKind
- # API path. For "measurement" endpoints this is a template with a `{sensors_id}` placeholder.
+ # API path. Fan-out endpoints use a `{sensors_id}` or `{locations_id}` placeholder.
path: str
primary_keys: list[str] = field(default_factory=lambda: ["id"])
partition_key: Optional[str] = None
@@ -34,6 +36,7 @@ class OpenAQEndpointConfig:
# `datetime_from`/`datetime_to`; the daily + yearly aggregates use `date_from`/`date_to`.
time_param_prefix: Optional[str] = None
should_sync_default: bool = True
+ sort_mode: Optional[Literal["asc", "desc"]] = "asc"
OPENAQ_ENDPOINTS: dict[str, OpenAQEndpointConfig] = {
@@ -81,6 +84,49 @@ OPENAQ_ENDPOINTS: dict[str, OpenAQEndpointConfig] = {
time_param_prefix="date",
should_sync_default=False,
),
+ "measurements_yearly": OpenAQEndpointConfig(
+ name="measurements_yearly",
+ kind="measurement",
+ path="/v3/sensors/{sensors_id}/years",
+ primary_keys=["sensor_id", "datetime_from"],
+ partition_key="datetime_from",
+ incremental_fields=[_datetime_incremental_field("datetime_from")],
+ time_param_prefix="date",
+ should_sync_default=False,
+ # Rows are paged per sensor, so the run is not one ascending series across the table.
+ sort_mode=None,
+ ),
+ # One row per sensor holding its most recent reading. The reading time moves on every sync, so the
+ # table is not partitioned.
+ "location_latest": OpenAQEndpointConfig(
+ name="location_latest",
+ kind="location_fanout",
+ path="/v3/locations/{locations_id}/latest",
+ primary_keys=["location_id", "sensor_id"],
+ should_sync_default=False,
+ sort_mode=None,
+ ),
+ # Flags carry no id. A flag is a period of a given type, so (parent, flag type, period start) keys it.
+ # Full refresh only: an open flag's end moves forward, and the API's datetime filter applies to the
+ # flag period, so an incremental read on the start would miss those updates.
+ "location_flags": OpenAQEndpointConfig(
+ name="location_flags",
+ kind="location_fanout",
+ path="/v3/locations/{locations_id}/flags",
+ primary_keys=["location_id", "flag_type_id", "datetime_from"],
+ partition_key="datetime_from",
+ should_sync_default=False,
+ sort_mode=None,
+ ),
+ "sensor_flags": OpenAQEndpointConfig(
+ name="sensor_flags",
+ kind="sensor_fanout",
+ path="/v3/sensors/{sensors_id}/flags",
+ primary_keys=["sensor_id", "flag_type_id", "datetime_from"],
+ partition_key="datetime_from",
+ should_sync_default=False,
+ sort_mode=None,
+ ),
}
ENDPOINTS = tuple(OPENAQ_ENDPOINTS.keys())
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/openaq/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/openaq/source.py
index b2dfd2455de5..b11d5696233e 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/openaq/source.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/openaq/source.py
@@ -100,8 +100,11 @@ The free tier is limited to 60 requests per minute. The measurement tables fetch
def _description(endpoint: str) -> str | None:
if endpoint == "sensors":
return "Sensors materialized from each location's embedded sensor list. Full refresh only"
- if OPENAQ_ENDPOINTS[endpoint].kind == "measurement":
+ kind = OPENAQ_ENDPOINTS[endpoint].kind
+ if kind in ("measurement", "sensor_fanout"):
return "Fetched per sensor across all locations — request-heavy, so off by default"
+ if kind == "location_fanout":
+ return "Fetched per location — request-heavy, so off by default"
return None
def _build_schema(endpoint: str) -> SourceSchema:
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/openaq/tests/test_openaq.py b/products/warehouse_sources/backend/temporal/data_imports/sources/openaq/tests/test_openaq.py
index a945bd21fa34..242a26f5767e 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/openaq/tests/test_openaq.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/openaq/tests/test_openaq.py
@@ -121,9 +121,17 @@ class TestMeasurementFanOut:
def _measurement(self, dt: str, value: float) -> dict[str, Any]:
return {"value": value, "parameter": {"name": "pm25"}, "period": {"datetimeFrom": {"utc": dt}}}
- def test_applies_incremental_datetime_filter(self) -> None:
+ @parameterized.expand(
+ [
+ ("raw", "measurements", "/v3/sensors/10/measurements", "datetime_from", "2026-01-05T00:00:00Z"),
+ ("yearly", "measurements_yearly", "/v3/sensors/10/years", "date_from", "2026-01-05"),
+ ]
+ )
+ def test_applies_incremental_datetime_filter(
+ self, _name: str, endpoint: str, sensor_path: str, filter_param: str, filter_value: str
+ ) -> None:
result = _driver().run(
- "measurements",
+ endpoint,
[
_page([{"id": 1, "sensors": [{"id": 10}]}]),
_page([self._measurement("2026-01-06T00:00:00Z", 7.0)]),
@@ -133,6 +141,54 @@ class TestMeasurementFanOut:
)
assert result.raised is None
assert [r["datetime_from"] for r in result.rows] == ["2026-01-06T00:00:00Z"]
- # The per-sensor request carried the server-side datetime filter.
- sensor_req = next(request for request in result.requests if "/sensors/10/measurements" in request.path)
- assert sensor_req.param("datetime_from") == "2026-01-05T00:00:00Z"
+ # The per-sensor request carried the server-side time filter in the format the endpoint takes.
+ sensor_req = next(request for request in result.requests if request.path == sensor_path)
+ assert sensor_req.param(filter_param) == filter_value
+
+
+class TestDetailFanOut:
+ _FLAG = {
+ "locationId": 1,
+ "flagType": {"id": 3, "label": "Calibration", "level": "warning"},
+ "datetimeFrom": {"utc": "2026-01-01T00:00:00Z", "local": "2026-01-01T01:00:00+01:00"},
+ "datetimeTo": {"utc": "2026-01-02T00:00:00Z", "local": "2026-01-02T01:00:00+01:00"},
+ "sensorIds": [10],
+ "note": None,
+ }
+
+ @parameterized.expand(
+ [
+ (
+ "location_latest",
+ {
+ "sensorsId": 10,
+ "locationsId": 1,
+ "value": 12.5,
+ "datetime": {"utc": "2026-01-06T00:00:00Z", "local": "2026-01-06T01:00:00+01:00"},
+ },
+ "/v3/locations/1/latest",
+ {"location_id": 1, "sensor_id": 10, "datetime": "2026-01-06T00:00:00Z", "value": 12.5},
+ ),
+ (
+ "location_flags",
+ _FLAG,
+ "/v3/locations/1/flags",
+ {"location_id": 1, "flag_type_id": 3, "datetime_from": "2026-01-01T00:00:00Z"},
+ ),
+ (
+ "sensor_flags",
+ _FLAG,
+ "/v3/sensors/10/flags",
+ {"sensor_id": 10, "flag_type_id": 3, "datetime_from": "2026-01-01T00:00:00Z"},
+ ),
+ ]
+ )
+ def test_fans_out_per_parent_and_keys_rows_by_parent(
+ self, endpoint: str, item: dict[str, Any], child_path: str, expected_key: dict[str, Any]
+ ) -> None:
+ result = _driver().run(endpoint, [_page([{"id": 1, "sensors": [{"id": 10}]}]), _page([item])])
+
+ assert result.raised is None
+ assert result.paths == ["/v3/locations", child_path]
+ assert len(result.rows) == 1
+ assert {key: result.rows[0][key] for key in expected_key} == expected_key
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/openaq/tests/test_openaq_source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/openaq/tests/test_openaq_source.py
index 8e99c4c5fbcf..67b7210a7b8e 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/openaq/tests/test_openaq_source.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/openaq/tests/test_openaq_source.py
@@ -29,6 +29,10 @@ class TestOpenAQSchemas:
("measurements", True, False),
("measurements_hourly", True, False),
("measurements_daily", True, False),
+ ("measurements_yearly", True, False),
+ ("location_latest", False, False),
+ ("location_flags", False, False),
+ ("sensor_flags", False, False),
("locations", False, True),
("parameters", False, True),
("sensors", False, True),
@@ -38,7 +42,7 @@ class TestOpenAQSchemas:
self, endpoint: str, expected_incremental: bool, expected_default_sync: bool
) -> None:
# Only the per-sensor measurement streams have a server-side datetime filter, so only they are
- # incremental; and because they're request-heavy they must be off by default.
+ # incremental; and because fan-out tables are request-heavy they must be off by default.
schema = {s.name: s for s in OpenAQSource().get_schemas(_config(), team_id=1)}[endpoint]
assert schema.supports_incremental is expected_incremental
assert schema.should_sync_default is expected_default_sync