:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(warehouse-sources): add openaq latest, flags and yearly tables (#123358)
Tom Owersd0389a2a8352
diff --git a/posthog/test/repo_invariants/dataclass_frozen_baseline.txt b/posthog/test/repo_invariants/dataclass_frozen_baseline.txtindex 850befe461a9..1efc084eccad 100644--- a/posthog/test/repo_invariants/dataclass_frozen_baseline.txt+++ b/posthog/test/repo_invariants/dataclass_frozen_baseline.txt@@ -825,7 +825,6 @@1 products/warehouse_sources/backend/temporal/data_imports/sources/openai_ads/settings.py1 products/warehouse_sources/backend/temporal/data_imports/sources/openalex/openalex.py1 products/warehouse_sources/backend/temporal/data_imports/sources/openaq/openaq.py-1 products/warehouse_sources/backend/temporal/data_imports/sources/openaq/settings.py1 products/warehouse_sources/backend/temporal/data_imports/sources/openfda/openfda.py1 products/warehouse_sources/backend/temporal/data_imports/sources/openweather/settings.py1 products/warehouse_sources/backend/temporal/data_imports/sources/opinion_stage/opinion_stage.pydiff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.mdindex e3703660e98c..8e86e99fc77d 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md@@ -8761,15 +8761,15 @@ Note: Parsed the current endpoint index and entity references, including the col## OpenAQ — gaps-Today (12): `countries`, `instruments`, `licenses`, `locations`, `manufacturers`, `measurements`, `measurements_daily`, `measurements_hourly`, `owners`, `parameters`, `providers`, `sensors`+Today (16): `countries`, `instruments`, `licenses`, `location_flags`, `location_latest`, `locations`, `manufacturers`, `measurements`, `measurements_daily`, `measurements_hourly`, `measurements_yearly`, `owners`, `parameters`, `providers`, `sensor_flags`, `sensors`Diffed against: <https://api.openaq.org/openapi.json>-- [ ] `/v3/locations/{id}/latest` — most recent value per location - the headline 'current air quality' read (high)-- [ ] `/v3/sensors/{id}/flags` — data-quality flags needed to exclude bad readings from measurement aggregates (medium)-- [ ] `/v3/locations/{id}/flags` — location-level data-quality flags for the same filtering (medium)+- [x] `/v3/locations/{id}/latest` — most recent value per location - the headline 'current air quality' read (high)+- [x] `/v3/sensors/{id}/flags` — data-quality flags needed to exclude bad readings from measurement aggregates (medium)+- [x] `/v3/locations/{id}/flags` — location-level data-quality flags for the same filtering (medium)- [ ] `/v3/parameters/{id}/latest` — latest reading per pollutant across the network (medium)-- [ ] `/v3/sensors/{id}/years` — yearly rollups, cheaper than aggregating hourly data for long-range trends (medium)+- [x] `/v3/sensors/{id}/years` — yearly rollups, cheaper than aggregating hourly data for long-range trends (medium)- [ ] `/v3/manufacturers/{id}/instruments` — join table linking manufacturers to instruments we already sync (low)- [ ] `/v3/sensors/{id}/hours/hourofday and /days/dayofweek, /days/monthofyear` — seasonal/diurnal aggregations the API precomputes (low)diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/openaq/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/openaq/canonical_descriptions.pyindex 3dba9080a92a..382faf2a6353 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/openaq/canonical_descriptions.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/openaq/canonical_descriptions.py@@ -168,4 +168,62 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {"flag_info": "Quality/flag information for the aggregated measurement.",},},+ "measurements_yearly": {+ "description": "Yearly aggregated measurements per sensor.",+ "docs_url": "https://docs.openaq.org/resources/measurements",+ "columns": {+ "sensor_id": "Identifier of the sensor that produced the measurement.",+ "datetime_from": "UTC start of the yearly period.",+ "datetime_to": "UTC end of the yearly period.",+ "value": "Aggregated value for the year, in the parameter's units.",+ "parameter_id": "Identifier of the measured parameter.",+ "parameter_name": "Short code of the measured parameter.",+ "parameter_units": "Unit the value is reported in.",+ "coordinates": "Latitude/longitude of the sensor's location.",+ "summary": "Statistical summary of the readings aggregated into the year.",+ "coverage": "Data-coverage summary for the year.",+ "flag_info": "Quality/flag information for the aggregated measurement.",+ },+ },+ "location_latest": {+ "description": "Most recent reading of each sensor at each location.",+ "docs_url": "https://docs.openaq.org/resources/latest",+ "columns": {+ "location_id": "Identifier of the location.",+ "sensor_id": "Identifier of the sensor that produced the reading.",+ "datetime": "UTC timestamp of the reading.",+ "datetime_local": "Timestamp of the reading in the location's local timezone.",+ "value": "Most recent measured value, in the parameter's units.",+ "coordinates": "Latitude/longitude of the reading.",+ },+ },+ "location_flags": {+ "description": "Data-quality flags raised on a location for a period of time.",+ "docs_url": "https://docs.openaq.org/api/operations/location_flags_get_v3_locations__locations_id__flags_get",+ "columns": {+ "location_id": "Identifier of the location the flag applies to.",+ "flag_type_id": "Identifier of the flag type.",+ "flag_type_label": "Human-readable label of the flag type.",+ "flag_level": "Severity level of the flag.",+ "datetime_from": "UTC start of the flagged period.",+ "datetime_to": "UTC end of the flagged period.",+ "sensor_ids": "Sensors at the location the flag covers.",+ "note": "Free-text note explaining the flag.",+ },+ },+ "sensor_flags": {+ "description": "Data-quality flags that cover a sensor for a period of time.",+ "docs_url": "https://docs.openaq.org/api/operations/sensor_flags_get_v3_sensors__sensor_id__flags_get",+ "columns": {+ "sensor_id": "Identifier of the sensor the flag was fetched for.",+ "location_id": "Identifier of the location the flag applies to.",+ "flag_type_id": "Identifier of the flag type.",+ "flag_type_label": "Human-readable label of the flag type.",+ "flag_level": "Severity level of the flag.",+ "datetime_from": "UTC start of the flagged period.",+ "datetime_to": "UTC end of the flagged period.",+ "sensor_ids": "Sensors at the location the flag covers.",+ "note": "Free-text note explaining the flag.",+ },+ },}diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/openaq/openaq.py b/products/warehouse_sources/backend/temporal/data_imports/sources/openaq/openaq.pyindex 5ead994957d5..e6128056c8c8 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/openaq/openaq.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/openaq/openaq.py@@ -1,4 +1,5 @@import dataclasses+from collections.abc import Callablefrom datetime import UTC, date, datetimefrom functools import partialfrom typing import Any, Optional@@ -41,10 +42,12 @@ MAX_PAGES: int = 100# pagination for the full-refresh reference tables._LIST_SORT_PARAMS = {"order_by": "id", "sort_order": "asc"}-# Name of the internal parent resource that enumerates sensor ids for the measurement fan-out.+# Names of the internal parent resources that enumerate sensor / location ids for the fan-outs._SENSOR_IDS_RESOURCE = "sensor_ids"-# Key the framework injects a parent field under: `_{parent_resource}_{field}`.+_LOCATION_IDS_RESOURCE = "location_ids"+# Keys the framework injects a parent field under: `_{parent_resource}_{field}`._PARENT_SENSOR_ID_KEY = f"_{_SENSOR_IDS_RESOURCE}_id"+_PARENT_LOCATION_ID_KEY = f"_{_LOCATION_IDS_RESOURCE}_id"@dataclasses.dataclass@@ -158,6 +161,47 @@ def _reshape_measurement(item: dict[str, Any]) -> dict[str, Any]:return _flatten_measurement(item, item[_PARENT_SENSOR_ID_KEY])+def _flatten_flag(item: dict[str, Any]) -> dict[str, Any]:+ flag_type = item.get("flagType") or {}+ return {+ "location_id": item.get("locationId"),+ "flag_type_id": flag_type.get("id"),+ "flag_type_label": flag_type.get("label"),+ "flag_level": flag_type.get("level"),+ "datetime_from": (item.get("datetimeFrom") or {}).get("utc"),+ "datetime_to": (item.get("datetimeTo") or {}).get("utc"),+ "sensor_ids": item.get("sensorIds"),+ "note": item.get("note"),+ }+++def _reshape_location_flag(item: dict[str, Any]) -> dict[str, Any]:+ return {**_flatten_flag(item), "location_id": item[_PARENT_LOCATION_ID_KEY]}+++def _reshape_sensor_flag(item: dict[str, Any]) -> dict[str, Any]:+ return {"sensor_id": item[_PARENT_SENSOR_ID_KEY], **_flatten_flag(item)}+++def _reshape_latest(item: dict[str, Any]) -> dict[str, Any]:+ reading_time = item.get("datetime") or {}+ return {+ "location_id": item[_PARENT_LOCATION_ID_KEY],+ "sensor_id": item["sensorsId"],+ "datetime": reading_time.get("utc"),+ "datetime_local": reading_time.get("local"),+ "value": item.get("value"),+ "coordinates": item.get("coordinates"),+ }+++_FANOUT_ROW_MAPPERS: dict[str, Callable[[dict[str, Any]], dict[str, Any]]] = {+ "location_latest": _reshape_latest,+ "location_flags": _reshape_location_flag,+ "sensor_flags": _reshape_sensor_flag,+}++def _client_config(api_key: str) -> ClientConfig:return {"base_url": OPENAQ_BASE_URL,@@ -186,7 +230,7 @@ def _list_resource(endpoint: str, config: OpenAQEndpointConfig, page_size: int)return resource-def _measurement_resources(+def _fanout_resources(endpoint: str,config: OpenAQEndpointConfig,page_size: int,@@ -201,13 +245,18 @@ def _measurement_resources("data_selector": "results","paginator": OpenAQPageNumberPaginator(page_size, MAX_PAGES),},- "data_map": _explode_location_sensor_ids,}+ if config.kind == "location_fanout":+ parent["name"] = _LOCATION_IDS_RESOURCE+ resolve_param = "locations_id"+ else:+ parent["data_map"] = _explode_location_sensor_ids+ resolve_param = "sensors_id"child_endpoint: Endpoint = {"path": config.path,"params": {- "sensors_id": {"type": "resolve", "resource": _SENSOR_IDS_RESOURCE, "field": "id"},+ resolve_param: {"type": "resolve", "resource": parent["name"], "field": "id"},"limit": page_size,},"data_selector": "results",@@ -227,7 +276,7 @@ def _measurement_resources("name": endpoint,"endpoint": child_endpoint,"include_from_parent": ["id"],- "data_map": _reshape_measurement,+ "data_map": _reshape_measurement if config.kind == "measurement" else _FANOUT_ROW_MAPPERS[endpoint],}return [parent, child]@@ -247,11 +296,11 @@ def openaq_source(resume = resumable_source_manager.load_state() if resumable_source_manager.can_resume() else None- if config.kind == "measurement":+ if config.kind in ("measurement", "location_fanout", "sensor_fanout"):rest_config: RESTAPIConfig = {"client": client_config,"resource_defaults": {},- "resources": _measurement_resources(+ "resources": _fanout_resources(endpoint, config, page_size, should_use_incremental_field, db_incremental_field_last_value),}@@ -308,7 +357,7 @@ def openaq_source(# incremental watermark relies on. Unverified against the live API (no key available); noted# in the PR. If this proves wrong, the datetime_from server filter still bounds each fetch, so# completeness holds — only mid-sync watermark checkpointing would need sort_mode="desc".- sort_mode="asc",+ sort_mode=config.sort_mode,column_hints=resource.column_hints,)diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/openaq/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/openaq/settings.pyindex 82c72c4367d9..eadebf0aa32e 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/openaq/settings.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/openaq/settings.py@@ -9,7 +9,9 @@ from products.warehouse_sources.backend.types import IncrementalField, Increment# `sensors` array into one row per sensor. Full refresh (single locations walk, cheap).# - "measurement": fanned out per sensor (/v3/sensors/{sensors_id}/...). Incremental via the# API's server-side datetime filter. Request-heavy (one paginated fetch per sensor), so opt-in.-EndpointKind = Literal["list", "sensors", "measurement"]+# - "location_fanout" / "sensor_fanout": fanned out per location (/v3/locations/{locations_id}/...)+# or per sensor (/v3/sensors/{sensors_id}/...). Full refresh. Request-heavy, so opt-in.+EndpointKind = Literal["list", "sensors", "measurement", "location_fanout", "sensor_fanout"]def _datetime_incremental_field(name: str) -> IncrementalField:@@ -21,11 +23,11 @@ def _datetime_incremental_field(name: str) -> IncrementalField:}-@dataclass+@dataclass(frozen=True)class OpenAQEndpointConfig:name: strkind: EndpointKind- # API path. For "measurement" endpoints this is a template with a `{sensors_id}` placeholder.+ # API path. Fan-out endpoints use a `{sensors_id}` or `{locations_id}` placeholder.path: strprimary_keys: list[str] = field(default_factory=lambda: ["id"])partition_key: Optional[str] = None@@ -34,6 +36,7 @@ class OpenAQEndpointConfig:# `datetime_from`/`datetime_to`; the daily + yearly aggregates use `date_from`/`date_to`.time_param_prefix: Optional[str] = Noneshould_sync_default: bool = True+ sort_mode: Optional[Literal["asc", "desc"]] = "asc"OPENAQ_ENDPOINTS: dict[str, OpenAQEndpointConfig] = {@@ -81,6 +84,49 @@ OPENAQ_ENDPOINTS: dict[str, OpenAQEndpointConfig] = {time_param_prefix="date",should_sync_default=False,),+ "measurements_yearly": OpenAQEndpointConfig(+ name="measurements_yearly",+ kind="measurement",+ path="/v3/sensors/{sensors_id}/years",+ primary_keys=["sensor_id", "datetime_from"],+ partition_key="datetime_from",+ incremental_fields=[_datetime_incremental_field("datetime_from")],+ time_param_prefix="date",+ should_sync_default=False,+ # Rows are paged per sensor, so the run is not one ascending series across the table.+ sort_mode=None,+ ),+ # One row per sensor holding its most recent reading. The reading time moves on every sync, so the+ # table is not partitioned.+ "location_latest": OpenAQEndpointConfig(+ name="location_latest",+ kind="location_fanout",+ path="/v3/locations/{locations_id}/latest",+ primary_keys=["location_id", "sensor_id"],+ should_sync_default=False,+ sort_mode=None,+ ),+ # Flags carry no id. A flag is a period of a given type, so (parent, flag type, period start) keys it.+ # Full refresh only: an open flag's end moves forward, and the API's datetime filter applies to the+ # flag period, so an incremental read on the start would miss those updates.+ "location_flags": OpenAQEndpointConfig(+ name="location_flags",+ kind="location_fanout",+ path="/v3/locations/{locations_id}/flags",+ primary_keys=["location_id", "flag_type_id", "datetime_from"],+ partition_key="datetime_from",+ should_sync_default=False,+ sort_mode=None,+ ),+ "sensor_flags": OpenAQEndpointConfig(+ name="sensor_flags",+ kind="sensor_fanout",+ path="/v3/sensors/{sensors_id}/flags",+ primary_keys=["sensor_id", "flag_type_id", "datetime_from"],+ partition_key="datetime_from",+ should_sync_default=False,+ sort_mode=None,+ ),}ENDPOINTS = tuple(OPENAQ_ENDPOINTS.keys())diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/openaq/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/openaq/source.pyindex b2dfd2455de5..b11d5696233e 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/openaq/source.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/openaq/source.py@@ -100,8 +100,11 @@ The free tier is limited to 60 requests per minute. The measurement tables fetchdef _description(endpoint: str) -> str | None:if endpoint == "sensors":return "Sensors materialized from each location's embedded sensor list. Full refresh only"- if OPENAQ_ENDPOINTS[endpoint].kind == "measurement":+ kind = OPENAQ_ENDPOINTS[endpoint].kind+ if kind in ("measurement", "sensor_fanout"):return "Fetched per sensor across all locations — request-heavy, so off by default"+ if kind == "location_fanout":+ return "Fetched per location — request-heavy, so off by default"return Nonedef _build_schema(endpoint: str) -> SourceSchema:diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/openaq/tests/test_openaq.py b/products/warehouse_sources/backend/temporal/data_imports/sources/openaq/tests/test_openaq.pyindex a945bd21fa34..242a26f5767e 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/openaq/tests/test_openaq.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/openaq/tests/test_openaq.py@@ -121,9 +121,17 @@ class TestMeasurementFanOut:def _measurement(self, dt: str, value: float) -> dict[str, Any]:return {"value": value, "parameter": {"name": "pm25"}, "period": {"datetimeFrom": {"utc": dt}}}- def test_applies_incremental_datetime_filter(self) -> None:+ @parameterized.expand(+ [+ ("raw", "measurements", "/v3/sensors/10/measurements", "datetime_from", "2026-01-05T00:00:00Z"),+ ("yearly", "measurements_yearly", "/v3/sensors/10/years", "date_from", "2026-01-05"),+ ]+ )+ def test_applies_incremental_datetime_filter(+ self, _name: str, endpoint: str, sensor_path: str, filter_param: str, filter_value: str+ ) -> None:result = _driver().run(- "measurements",+ endpoint,[_page([{"id": 1, "sensors": [{"id": 10}]}]),_page([self._measurement("2026-01-06T00:00:00Z", 7.0)]),@@ -133,6 +141,54 @@ class TestMeasurementFanOut:)assert result.raised is Noneassert [r["datetime_from"] for r in result.rows] == ["2026-01-06T00:00:00Z"]- # The per-sensor request carried the server-side datetime filter.- sensor_req = next(request for request in result.requests if "/sensors/10/measurements" in request.path)- assert sensor_req.param("datetime_from") == "2026-01-05T00:00:00Z"+ # The per-sensor request carried the server-side time filter in the format the endpoint takes.+ sensor_req = next(request for request in result.requests if request.path == sensor_path)+ assert sensor_req.param(filter_param) == filter_value+++class TestDetailFanOut:+ _FLAG = {+ "locationId": 1,+ "flagType": {"id": 3, "label": "Calibration", "level": "warning"},+ "datetimeFrom": {"utc": "2026-01-01T00:00:00Z", "local": "2026-01-01T01:00:00+01:00"},+ "datetimeTo": {"utc": "2026-01-02T00:00:00Z", "local": "2026-01-02T01:00:00+01:00"},+ "sensorIds": [10],+ "note": None,+ }++ @parameterized.expand(+ [+ (+ "location_latest",+ {+ "sensorsId": 10,+ "locationsId": 1,+ "value": 12.5,+ "datetime": {"utc": "2026-01-06T00:00:00Z", "local": "2026-01-06T01:00:00+01:00"},+ },+ "/v3/locations/1/latest",+ {"location_id": 1, "sensor_id": 10, "datetime": "2026-01-06T00:00:00Z", "value": 12.5},+ ),+ (+ "location_flags",+ _FLAG,+ "/v3/locations/1/flags",+ {"location_id": 1, "flag_type_id": 3, "datetime_from": "2026-01-01T00:00:00Z"},+ ),+ (+ "sensor_flags",+ _FLAG,+ "/v3/sensors/10/flags",+ {"sensor_id": 10, "flag_type_id": 3, "datetime_from": "2026-01-01T00:00:00Z"},+ ),+ ]+ )+ def test_fans_out_per_parent_and_keys_rows_by_parent(+ self, endpoint: str, item: dict[str, Any], child_path: str, expected_key: dict[str, Any]+ ) -> None:+ result = _driver().run(endpoint, [_page([{"id": 1, "sensors": [{"id": 10}]}]), _page([item])])++ assert result.raised is None+ assert result.paths == ["/v3/locations", child_path]+ assert len(result.rows) == 1+ assert {key: result.rows[0][key] for key in expected_key} == expected_keydiff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/openaq/tests/test_openaq_source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/openaq/tests/test_openaq_source.pyindex 8e99c4c5fbcf..67b7210a7b8e 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/openaq/tests/test_openaq_source.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/openaq/tests/test_openaq_source.py@@ -29,6 +29,10 @@ class TestOpenAQSchemas:("measurements", True, False),("measurements_hourly", True, False),("measurements_daily", True, False),+ ("measurements_yearly", True, False),+ ("location_latest", False, False),+ ("location_flags", False, False),+ ("sensor_flags", False, False),("locations", False, True),("parameters", False, True),("sensors", False, True),@@ -38,7 +42,7 @@ class TestOpenAQSchemas:self, endpoint: str, expected_incremental: bool, expected_default_sync: bool) -> None:# Only the per-sensor measurement streams have a server-side datetime filter, so only they are- # incremental; and because they're request-heavy they must be off by default.+ # incremental; and because fan-out tables are request-heavy they must be off by default.schema = {s.name: s for s in OpenAQSource().get_schemas(_config(), team_id=1)}[endpoint]assert schema.supports_incremental is expected_incrementalassert schema.should_sync_default is expected_default_sync