:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(warehouse_sources): add openweather history, geocoding and alert tables (#124002)
Tom Owers335e80838b9e
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.mdindex c43de21b2f19..f1733e76f020 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md@@ -8850,10 +8850,10 @@ Today (4): `air_pollution`, `air_pollution_forecast`, `current_weather`, `forecaDiffed against: <https://openweathermap.org/api>-- [ ] `/data/2.5/air_pollution/history` — historical air quality back to Nov 2020 — the only way to trend AQI; today only current + forecast are synced (high)-- [ ] `Geocoding API (/geo/1.0/direct, /geo/1.0/reverse, /geo/1.0/zip)` — lookup table resolving city/state/country/ZIP to the lat/lon that every synced row is keyed on (high)-- [ ] `One Call API 3.0/4.0 (/data/3.0/onecall)` — unified current + minutely + hourly + daily + government weather alerts in one row set; the vendor's headline product (high)-- [ ] `Weather History API (/data/3.0/onecall/timemachine, /data/2.5/history/city)` — hourly historical observations — required for any backward-looking weather correlation (high)+- [x] `/data/2.5/air_pollution/history` — historical air quality back to Nov 2020 — the only way to trend AQI; today only current + forecast are synced (high)+- [x] `Geocoding API (/geo/1.0/direct, /geo/1.0/reverse, /geo/1.0/zip)` — lookup table resolving city/state/country/ZIP to the lat/lon that every synced row is keyed on (high)+- [x] `One Call API 3.0/4.0 (/data/3.0/onecall)` — unified current + minutely + hourly + daily + government weather alerts in one row set; the vendor's headline product (high)+- [x] `Weather History API (/data/3.0/onecall/timemachine, /data/2.5/history/city)` — hourly historical observations — required for any backward-looking weather correlation (high)- [ ] `One Call day summary (/data/3.0/onecall/day_summary)` — pre-aggregated daily weather rollup, the natural grain for joining against daily business metrics (medium)- [ ] `Daily Forecast 16 days (/data/2.5/forecast/daily)` — extends forecast horizon past the 5-day/3-hour table already synced (medium)- [ ] `Hourly Forecast 4 days (/data/2.5/forecast/hourly)` — finer-grained forecast than the 3-hour table (medium)diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/openweather/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/openweather/canonical_descriptions.pyindex 80176e23d377..4cb47cc573e6 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/openweather/canonical_descriptions.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/openweather/canonical_descriptions.py@@ -64,6 +64,45 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {"components": "Pollutant concentrations in μg/m³: co, no, no2, o3, so2, pm2_5, pm10, nh3.",},},+ "air_pollution_history": {+ "description": "Hourly historical air quality for a configured location, from 27 November 2020.",+ "docs_url": "https://openweathermap.org/api/air-pollution",+ "columns": {+ **_INJECTED_COLUMNS,+ "dt": "Time of the observed air-quality hour, Unix UTC timestamp.",+ "main": "Air Quality Index: `aqi` from 1 (Good) to 5 (Very Poor).",+ "components": "Pollutant concentrations in μg/m³: co, no, no2, o3, so2, pm2_5, pm10, nh3.",+ },+ },+ "weather_history": {+ "description": "Hourly historical weather observations for a configured location, from the History API.",+ "docs_url": "https://openweathermap.org/history",+ "columns": {+ **_INJECTED_COLUMNS,+ "dt": "Time of the observed hour, Unix UTC timestamp.",+ "main": "Main measurements: temp, feels_like, pressure, humidity, temp_min, temp_max, sea_level, grnd_level.",+ "wind": "Wind data: speed, deg.",+ "clouds": "Cloudiness: `all`, as a percentage.",+ "rain": "Rain volume for the last 1h/3h in mm, where available.",+ "snow": "Snow volume for the last 1h/3h in mm, where available.",+ "weather": "List of weather condition objects (id, main, description, icon).",+ },+ },+ "geocoding": {+ "description": "Place name, state and country for a configured location, from the OpenWeather reverse Geocoding API.",+ "docs_url": "https://openweathermap.org/api/geocoding-api",+ "columns": {+ "lat": _INJECTED_COLUMNS["lat"],+ "lon": _INJECTED_COLUMNS["lon"],+ "location_label": _INJECTED_COLUMNS["location_label"],+ "name": "Name of the place found at the location.",+ "local_names": "Name of the place in other languages, keyed by ISO 639 language code.",+ "country": "Country code (ISO 3166).",+ "state": "State, where available.",+ "place_lat": "Latitude of the place found.",+ "place_lon": "Longitude of the place found.",+ },+ },"air_pollution_forecast": {"description": "Hourly air-quality forecast for a configured location.","docs_url": "https://openweathermap.org/api/air-pollution",@@ -109,6 +148,22 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {"alerts": _ALERTS_COLUMN,},},+ "alerts": {+ "description": "Government weather alert covering a configured location, from the One Call API 3.0.",+ "docs_url": "https://openweathermap.org/api/one-call-3",+ "columns": {+ "lat": _INJECTED_COLUMNS["lat"],+ "lon": _INJECTED_COLUMNS["lon"],+ "location_label": _INJECTED_COLUMNS["location_label"],+ "sender_name": "Name of the agency that issued the alert.",+ "event": "Alert event name.",+ "start": "Start of the alert, Unix UTC timestamp.",+ "start_iso": "ISO 8601 UTC timestamp derived from `start`; used as the partition key.",+ "end": "End of the alert, Unix UTC timestamp.",+ "description": "Description of the alert.",+ "tags": "Types of severe weather the alert covers.",+ },+ },"quarter_hourly": {"description": "15 minute weather forecast timeline for a configured location; one row per 15 minute slot.","docs_url": "https://openweathermap.org/api/one-call-4",diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/openweather/openweather.py b/products/warehouse_sources/backend/temporal/data_imports/sources/openweather/openweather.pyindex 166f3fdd2899..aa257deca70c 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/openweather/openweather.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/openweather/openweather.py@@ -9,10 +9,17 @@ import requestsfrom structlog.types import FilteringBoundLoggerfrom tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_exponential_jitter+from posthog.dataclasses import frozen+from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session+from products.warehouse_sources.backend.temporal.data_imports.sources.common.interruptible_wait import (+ interruptible_wait,+)+from products.warehouse_sources.backend.temporal.data_imports.sources.common.safe_point import reach_safe_pointfrom products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponsefrom products.warehouse_sources.backend.temporal.data_imports.sources.openweather.settings import (PROBE_ENDPOINT_BY_VERSION,+ HistoryWindow,OpenWeatherEndpointConfig,endpoints_for_version,)@@ -26,6 +33,9 @@ MAX_RETRY_ATTEMPTS = 5# worker time and outbound fan-out — a malformed/abusive config can't tie up the pipeline.MAX_LOCATIONS = 100+# A history backfill makes many calls per location, and the free tier allows 60 calls a minute.+HISTORY_REQUEST_INTERVAL_SECONDS = 1.0+class OpenWeatherRetryableError(Exception):pass@@ -83,8 +93,37 @@ def parse_locations(raw: str | None) -> list[Location]:return locations-def _build_url(path: str, params: dict[str, Any]) -> str:- return f"{OPENWEATHER_BASE_URL}{path}?{urlencode(params)}"+def _build_url(path: str, params: dict[str, Any], base_url: str | None = None) -> str:+ return f"{base_url or OPENWEATHER_BASE_URL}{path}?{urlencode(params)}"+++@frozen+class TimeWindow:+ start: int+ end: int+++def history_windows(window: HistoryWindow, start_from: int | None, end: int) -> Iterator[TimeWindow]:+ """Split ``[start, end]`` into consecutive inclusive Unix ranges of at most ``window.size``.++ ``start_from`` is the incremental watermark. Without one the read starts at the window's+ ``earliest_start``, or ``lookback`` before ``end``.+ """+ if start_from is not None:+ start = int(start_from)+ elif window.earliest_start is not None:+ start = int(window.earliest_start.timestamp())+ elif window.lookback is not None:+ start = end - int(window.lookback.total_seconds())+ else:+ raise ValueError("A history window needs an earliest_start or a lookback.")++ size = int(window.size.total_seconds())+ while start <= end:+ window_end = min(start + size, end)+ yield TimeWindow(start=start, end=window_end)+ # Ranges are inclusive, so the next one starts a second later to not read a boundary row twice.+ start = window_end + 1# The API key is passed as the `appid` query param, so it ends up in `response.url`. `raise_for_status()`@@ -102,7 +141,7 @@ def _redact_appid(text: str) -> str:wait=wait_exponential_jitter(initial=1, max=30),reraise=True,)-def _fetch(session: requests.Session, url: str, logger: FilteringBoundLogger) -> dict[str, Any]:+def _fetch(session: requests.Session, url: str, logger: FilteringBoundLogger) -> Any:response = session.get(url, timeout=REQUEST_TIMEOUT_SECONDS)# 429 (free tier is 60 req/min) and transient 5xx are retryable; back off and try again.@@ -128,16 +167,17 @@ def _dt_to_iso(dt: Any) -> str | None:return datetime.fromtimestamp(int(dt), tz=UTC).isoformat()-def _normalize_rows(- config: OpenWeatherEndpointConfig, response: dict[str, Any], location: Location-) -> list[dict[str, Any]]:+def _normalize_rows(config: OpenWeatherEndpointConfig, response: Any, location: Location) -> list[dict[str, Any]]:"""Turn an API response into flat rows, stamping each with the requested coordinates.We inject the *requested* lat/lon (not the response's echoed ``coord``, which can snap to thenearest station and drift between syncs) so the ``[lat, lon, dt]`` primary key stays stable."""if config.data_key is None:- items: list[dict[str, Any]] = [dict(response)]+ if isinstance(response, list):+ items: list[dict[str, Any]] = [dict(item) for item in response if isinstance(item, dict)]+ else:+ items = [dict(response)]else:section = response.get(config.data_key)# A section is either a list of rows (forecast slots, One Call hourly/daily) or a single@@ -156,12 +196,16 @@ def _normalize_rows(item.setdefault("city", city)for item in items:+ if config.has_place_coords:+ item["place_lat"] = item.pop("lat", None)+ item["place_lon"] = item.pop("lon", None)item["lat"] = location.latitem["lon"] = location.lonitem["location_label"] = location.label- # `dt` is part of the primary key, so a row without it is degenerate — read it directly- # so a missing field fails loudly rather than flowing a null partition key into the merge.- item["dt_iso"] = _dt_to_iso(item["dt"])+ if config.timestamp_field is not None:+ # The timestamp is part of the primary key, so a row without it is degenerate — read it+ # directly so a missing field fails loudly rather than flowing a null partition key into the merge.+ item[f"{config.timestamp_field}_iso"] = _dt_to_iso(item[config.timestamp_field])return items@@ -208,17 +252,35 @@ def get_rows(locations: list[Location],logger: FilteringBoundLogger,api_version: str,+ start_from: int | None = None,) -> Iterator[list[dict[str, Any]]]:config = endpoints_for_version(api_version)[endpoint]# One session reused across every location so urllib3 keeps the connection alive.session = make_tracked_session()+ window = config.history_window+ end = int(datetime.now(UTC).timestamp())for location in locations:- url = _build_url(config.path, {"lat": location.lat, "lon": location.lon, "appid": api_key})- response = _fetch(session, url, logger)- rows = _normalize_rows(config, response, location)- if rows:- yield rows+ params = {"lat": location.lat, "lon": location.lon, **config.extra_params, "appid": api_key}+ if window is None:+ response = _fetch(session, _build_url(config.path, params, config.base_url), logger)+ rows = _normalize_rows(config, response, location)+ if rows:+ yield rows+ continue++ for index, time_window in enumerate(history_windows(window, start_from, end)):+ if index > 0:+ interruptible_wait(HISTORY_REQUEST_INTERVAL_SECONDS, safe_point=reach_safe_point)+ url = _build_url(+ config.path, {**params, "start": time_window.start, "end": time_window.end}, config.base_url+ )+ rows = _normalize_rows(config, _fetch(session, url, logger), location)+ if rows:+ yield rows+ else:+ # Windows before the location's data begins come back empty.+ reach_safe_point()def openweather_source(@@ -227,15 +289,28 @@ def openweather_source(locations_raw: str | None,logger: FilteringBoundLogger,api_version: str,+ db_incremental_field_last_value: Any = None,) -> SourceResponse:config = endpoints_for_version(api_version)[endpoint]locations = parse_locations(locations_raw)+ start_from = int(db_incremental_field_last_value) if db_incremental_field_last_value is not None else None++ def items() -> Iterator[list[dict[str, Any]]]:+ return get_rows(+ api_key=api_key,+ endpoint=endpoint,+ locations=locations,+ logger=logger,+ api_version=api_version,+ start_from=start_from,+ )++ if config.partition_key is None:+ return SourceResponse(name=endpoint, items=items, primary_keys=config.primary_keys)return SourceResponse(name=endpoint,- items=lambda: get_rows(- api_key=api_key, endpoint=endpoint, locations=locations, logger=logger, api_version=api_version- ),+ items=items,primary_keys=config.primary_keys,partition_count=1,partition_size=1,diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/openweather/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/openweather/settings.pyindex c89eec461d6c..938cf10a7dbf 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/openweather/settings.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/openweather/settings.py@@ -1,5 +1,6 @@from dataclasses import dataclass, field-from typing import Optional+from datetime import UTC, datetime, timedelta+from typing import Any, Optionalfrom products.warehouse_sources.backend.types import IncrementalField, IncrementalFieldType@@ -17,22 +18,50 @@ _DT_INCREMENTAL_FIELDS: list[IncrementalField] = []-@dataclass+@dataclass(frozen=True)+class HistoryWindow:+ """How a history endpoint that takes a ``start``/``end`` Unix range is read."""++ # Longest range one request covers. Each window is its own billed call.+ size: timedelta+ # Where a sync with no watermark starts: the first date the vendor has data for, or a lookback+ # from the sync time when how far back the data goes depends on the customer's plan.+ earliest_start: Optional[datetime] = None+ lookback: Optional[timedelta] = None+++@dataclass(frozen=True)class OpenWeatherEndpointConfig:name: strpath: str# Where rows live in the JSON response. ``None`` means the response *is* the row- # (current weather), ``"list"`` means rows are the elements of the ``list`` array- # (forecast, air pollution).+ # (current weather) or a list of rows (geocoding), ``"list"`` means rows are the elements+ # of the ``list`` array (forecast, air pollution).data_key: Optional[str]incremental_fields: list[IncrementalField] = field(default_factory=lambda: list(_DT_INCREMENTAL_FIELDS))# ``dt`` alone is not unique table-wide because rows aggregate across every configured# location, so the requested coordinates are part of the key.primary_keys: list[str] = field(default_factory=lambda: ["lat", "lon", "dt"])- # Stable datetime column used for partitioning (derived from ``dt``).- partition_key: str = "dt_iso"+ # Unix timestamp column each row is stamped with an ISO 8601 copy of (``<field>_iso``).+ timestamp_field: Optional[str] = "dt"+ # Stable datetime column used for partitioning (derived from ``timestamp_field``).+ partition_key: Optional[str] = "dt_iso"should_sync_default: bool = Truedescription: Optional[str] = None+ # Host serving the path, when it is not the main API host.+ base_url: Optional[str] = None+ extra_params: dict[str, Any] = field(default_factory=dict)+ # Set for endpoints that take a ``start``/``end`` range; these filter server-side on ``dt``,+ # so they support true incremental sync.+ history_window: Optional[HistoryWindow] = None+ supports_append: bool = True+ # The row carries the coordinates of the place it describes (geocoding). They move to+ # ``place_lat``/``place_lon`` so the requested ``lat``/``lon`` key stays stable.+ has_place_coords: bool = False++ @property+ def supports_incremental(self) -> bool:+ return self.history_window is not None# Opaque vendor version labels (never parsed or ordered). The Current Weather, 5 Day / 3 Hour@@ -42,7 +71,27 @@ API_VERSION_2_5 = "2.5"API_VERSION_3_0 = "3.0"API_VERSION_4_0 = "4.0"-# The 2.5 general-purpose APIs. Each endpoint is its own request per location per sync.+# The Geocoding API is versioned on its own (`1.0`) and does not depend on the pinned product, so+# every version exposes it as a lookup table over the configured locations.+_SHARED_ENDPOINTS: dict[str, OpenWeatherEndpointConfig] = {+ "geocoding": OpenWeatherEndpointConfig(+ name="geocoding",+ path="/geo/1.0/reverse",+ data_key=None,+ extra_params={"limit": 1},+ incremental_fields=[],+ primary_keys=["lat", "lon"],+ timestamp_field=None,+ partition_key=None,+ supports_append=False,+ has_place_coords=True,+ description="Place name, state and country for each configured location, from reverse geocoding. "+ "One row per location.",+ ),+}++# The 2.5 general-purpose APIs. Each endpoint is its own request per location per sync, except the+# history endpoints, which make one request per window per location.OPENWEATHER_ENDPOINTS: dict[str, OpenWeatherEndpointConfig] = {"current_weather": OpenWeatherEndpointConfig(name="current_weather",@@ -69,6 +118,30 @@ OPENWEATHER_ENDPOINTS: dict[str, OpenWeatherEndpointConfig] = {data_key="list",description="Hourly air-quality forecast for each configured location.",),+ "air_pollution_history": OpenWeatherEndpointConfig(+ name="air_pollution_history",+ path="/data/2.5/air_pollution/history",+ data_key="list",+ # The API documents no range limit per call; 30 days keeps each response a few hundred rows.+ history_window=HistoryWindow(size=timedelta(days=30), earliest_start=datetime(2020, 11, 27, tzinfo=UTC)),+ description="Hourly historical air quality for each configured location, from 27 November 2020. "+ "One row per hour per location.",+ ),+ "weather_history": OpenWeatherEndpointConfig(+ name="weather_history",+ path="/data/2.5/history/city",+ base_url="https://history.openweathermap.org",+ data_key="list",+ extra_params={"type": "hour"},+ # One week is the most the API returns per call. How far back data goes depends on the+ # subscription, so the first sync reads only the last week and later syncs extend it.+ history_window=HistoryWindow(size=timedelta(days=7), lookback=timedelta(days=7)),+ # Sold separately, so a key that passes validation can still get a 401 here.+ should_sync_default=False,+ description="Hourly historical weather observations for each configured location. The first sync reads "+ "the last 7 days and each sync after it adds the hours since. Needs a History API subscription.",+ ),+ **_SHARED_ENDPOINTS,}# The One Call API 3.0 serves current weather plus forecasts from a single `/data/3.0/onecall`@@ -83,6 +156,12 @@ OPENWEATHER_ENDPOINTS_3_0: dict[str, OpenWeatherEndpointConfig] = {description="Current weather snapshot for each configured location, from the One Call API 3.0. ""One row per location per sync; use append sync to accumulate a time series.",),+ "minutely": OpenWeatherEndpointConfig(+ name="minutely",+ path=_ONECALL_PATH,+ data_key="minutely",+ description="Minute-by-minute precipitation forecast for the next hour. One row per minute per location.",+ ),"hourly": OpenWeatherEndpointConfig(name="hourly",path=_ONECALL_PATH,@@ -95,6 +174,21 @@ OPENWEATHER_ENDPOINTS_3_0: dict[str, OpenWeatherEndpointConfig] = {data_key="daily",description="Daily weather forecast (8 days) for each configured location. One row per day.",),+ "alerts": OpenWeatherEndpointConfig(+ name="alerts",+ path=_ONECALL_PATH,+ data_key="alerts",+ # Alerts carry no `dt` or id; an issuer's alert is identified by its event and start time.+ # Full refresh only: an alert stays in the response while active, so append would insert it again+ # on every sync.+ incremental_fields=[],+ primary_keys=["lat", "lon", "sender_name", "event", "start"],+ timestamp_field="start",+ partition_key="start_iso",+ supports_append=False,+ description="Government weather alerts active at sync time for each configured location. One row per alert.",+ ),+ **_SHARED_ENDPOINTS,}# The One Call API 4.0 splits the 3.0 response into one endpoint per timeline step, so each table@@ -139,6 +233,7 @@ OPENWEATHER_ENDPOINTS_4_0: dict[str, OpenWeatherEndpointConfig] = {data_key="data",description="Daily weather forecast timeline. One row per day, up to 10 days ahead of the sync time.",),+ **_SHARED_ENDPOINTS,}OPENWEATHER_ENDPOINTS_BY_VERSION: dict[str, dict[str, OpenWeatherEndpointConfig]] = {diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/openweather/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/openweather/source.pyindex 43003d93b525..b8ed888e41f8 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/openweather/source.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/openweather/source.py@@ -100,6 +100,7 @@ Each sync polls every location once. To accumulate a history of point-in-time sn# A missing/invalid key (or a key not subscribed to the requested product) surfaces as a# 401 when `_fetch` calls `raise_for_status()`. Retrying can never satisfy it."401 Client Error: Unauthorized for url: https://api.openweathermap.org": "Your OpenWeather API key is invalid, not yet activated, or not subscribed to this product. Check the key in your OpenWeather account, then reconnect.",+ "401 Client Error: Unauthorized for url: https://history.openweathermap.org": "Your OpenWeather API key is not subscribed to the History API. Disable the weather_history table or upgrade your OpenWeather plan.",}def get_schemas(@@ -115,11 +116,11 @@ Each sync polls every location once. To accumulate a history of point-in-time snschemas = [SourceSchema(name=endpoint,- # No server-side timestamp filter exists, so this is not true incremental sync.- # Append is supported: each sync re-polls the (cheap) snapshot and merge dedupes on- # `[lat, lon, dt]`, accumulating a time series across runs.- supports_incremental=False,- supports_append=True,+ # Only the history endpoints filter server-side (`start`/`end`), so only they are+ # true incremental. Append on the others re-polls the (cheap) snapshot each sync,+ # accumulating a time series across runs.+ supports_incremental=config.supports_incremental,+ supports_append=config.supports_append,incremental_fields=config.incremental_fields,should_sync_default=config.should_sync_default,description=config.description,@@ -147,4 +148,7 @@ Each sync polls every location once. To accumulate a history of point-in-time snlocations_raw=config.locations,logger=inputs.logger,api_version=self.resolve_api_version(inputs.api_version),+ db_incremental_field_last_value=(+ inputs.db_incremental_field_last_value if inputs.should_use_incremental_field else None+ ),)diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/openweather/tests/test_openweather.py b/products/warehouse_sources/backend/temporal/data_imports/sources/openweather/tests/test_openweather.pyindex 05c05ff6a21f..d6ea746f005a 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/openweather/tests/test_openweather.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/openweather/tests/test_openweather.py@@ -1,7 +1,9 @@import json+from datetime import UTC, datetime, timedeltafrom typing import Any, Optionalimport pytest+import time_machinefrom unittest import mockimport requests@@ -21,10 +23,12 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.openweatheOPENWEATHER_BASE_URL,Location,OpenWeatherRetryableError,+ TimeWindow,_dt_to_iso,_fetch,_normalize_rows,_redact_appid,+ history_windows,openweather_source,parse_locations,validate_credentials,@@ -34,6 +38,8 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.openweatheAPI_VERSION_3_0,API_VERSION_4_0,OPENWEATHER_ENDPOINTS,+ OPENWEATHER_ENDPOINTS_3_0,+ HistoryWindow,endpoints_for_version,)from products.warehouse_sources.backend.temporal.data_imports.sources.openweather.source import OpenWeatherSource@@ -136,6 +142,31 @@ class TestNormalizeRows:assert all(row["city"] == {"id": 1, "name": "London"} for row in rows)assert all(row["lat"] == 51.5 and row["lon"] == -0.12 for row in rows)+ def test_geocoding_keeps_place_coords_apart_from_requested_coords(self):+ response = [{"name": "London", "lat": 51.507, "lon": -0.127, "country": "GB", "state": "England"}]+ rows = _normalize_rows(OPENWEATHER_ENDPOINTS["geocoding"], response, Location(51.5, -0.12, "London"))++ assert rows == [+ {+ "name": "London",+ "country": "GB",+ "state": "England",+ "place_lat": 51.507,+ "place_lon": -0.127,+ "lat": 51.5,+ "lon": -0.12,+ "location_label": "London",+ }+ ]++ def test_alerts_partition_on_start(self):+ response = {"alerts": [{"sender_name": "NWS", "event": "Flood Watch", "start": 1719158400, "end": 1719169200}]}+ rows = _normalize_rows(OPENWEATHER_ENDPOINTS_3_0["alerts"], response, Location(51.5, -0.12, None))++ assert len(rows) == 1+ assert rows[0]["start_iso"] == "2024-06-23T16:00:00+00:00"+ assert "dt_iso" not in rows[0]+def test_row_without_dt_raises(self):# `dt` is part of the primary key; a row missing it must fail loudly, not yield a null key.response = {"main": {"temp": 280}}@@ -143,6 +174,44 @@ class TestNormalizeRows:_normalize_rows(OPENWEATHER_ENDPOINTS["current_weather"], response, Location(51.5, -0.12, None))+_DAY = 86400+++class TestHistoryWindows:+ @pytest.mark.parametrize(+ "window, start_from, end, expected",+ [+ # A watermark wins over the backfill start.+ (+ HistoryWindow(size=timedelta(days=7), lookback=timedelta(days=7)),+ 10 * _DAY,+ 12 * _DAY,+ [TimeWindow(start=10 * _DAY, end=12 * _DAY)],+ ),+ (+ HistoryWindow(size=timedelta(days=7), lookback=timedelta(days=7)),+ None,+ 30 * _DAY,+ [TimeWindow(start=23 * _DAY, end=30 * _DAY)],+ ),+ # Consecutive windows never share a boundary second, so no row is read twice.+ (+ HistoryWindow(size=timedelta(days=2), earliest_start=datetime.fromtimestamp(0, tz=UTC)),+ None,+ 5 * _DAY,+ [+ TimeWindow(start=0, end=2 * _DAY),+ TimeWindow(start=2 * _DAY + 1, end=4 * _DAY + 1),+ TimeWindow(start=4 * _DAY + 2, end=5 * _DAY),+ ],+ ),+ (HistoryWindow(size=timedelta(days=7), lookback=timedelta(days=7)), 12 * _DAY + 1, 12 * _DAY, []),+ ],+ )+ def test_windows(self, window, start_from, end, expected):+ assert list(history_windows(window, start_from, end)) == expected++# The undecorated `_fetch` (tenacity exposes the original via `__wrapped__`) so the status# classification can be asserted without waiting through retry backoff._fetch_once = _fetch.__wrapped__ # type: ignore[attr-defined]@@ -253,16 +322,73 @@ class TestGetRows:assert result.requests[0].param("appid") == "test-key"assert result.items[0][0]["dt"] == 1+ @pytest.mark.parametrize(+ "endpoint, should_use_incremental_field, expected_base, expected_starts, expected_type",+ [+ # Incremental: one window from the watermark to now, on the History API's own host.+ ("weather_history", True, "https://history.openweathermap.org", [1782000000], "hour"),+ # A full refresh ignores a stored watermark and reads the last week.+ ("weather_history", False, "https://history.openweathermap.org", [1782518400 - 7 * _DAY], "hour"),+ ("air_pollution_history", True, OPENWEATHER_BASE_URL, [1782000000], None),+ ],+ )+ @time_machine.travel(datetime.fromtimestamp(1782518400, tz=UTC), tick=False)+ def test_history_request_shaping(+ self,+ endpoint: str,+ should_use_incremental_field: bool,+ expected_base: str,+ expected_starts: list[int],+ expected_type: str | None,+ ) -> None:+ driver = SourceDriver(OpenWeatherSource(), OpenWeatherSourceConfig(api_key="test-key", locations="51.5,-0.12"))+ result = driver.run(+ endpoint,+ [ScriptedResponse(json={"list": [{"dt": 1782003600, "main": {"temp": 280}}]})],+ api_version=API_VERSION_2_5,+ incremental_field="dt",+ should_use_incremental_field=should_use_incremental_field,+ db_incremental_field_last_value=1782000000,+ )++ assert result.raised is None+ assert all(url.startswith(f"{expected_base}/data/2.5/") for url in result.urls)+ assert [request.param("start") for request in result.requests] == [str(start) for start in expected_starts]+ assert result.requests[-1].param("end") == "1782518400"+ assert result.requests[0].param("type") == expected_type++ @time_machine.travel(datetime(2021, 1, 1, tzinfo=UTC), tick=False)+ def test_air_pollution_history_full_refresh_walks_back_to_first_data(self) -> None:+ driver = SourceDriver(OpenWeatherSource(), OpenWeatherSourceConfig(api_key="test-key", locations="51.5,-0.12"))+ result = driver.run(+ "air_pollution_history",+ always(ScriptedResponse(json={"list": [{"dt": 1609459200}]})),+ api_version=API_VERSION_2_5,+ )++ assert result.raised is None+ # 27 Nov 2020 to 1 Jan 2021 is two 30-day windows.+ assert [request.param("start") for request in result.requests] == [+ "1606435200",+ str(1606435200 + 30 * _DAY + 1),+ ]+ assert result.requests[-1].param("end") == "1609459200"+class TestOpenWeatherSource:@pytest.mark.parametrize("api_version, endpoint", _ALL_VERSIONED_ENDPOINTS)def test_source_response_shape(self, api_version, endpoint):+ config = endpoints_for_version(api_version)[endpoint]response = openweather_source("test-key", endpoint, "51.5,-0.12,London", structlog.get_logger(), api_version)assert response.name == endpoint- assert response.primary_keys == ["lat", "lon", "dt"]- assert response.partition_mode == "datetime"- assert response.partition_keys == ["dt_iso"]+ assert set(config.primary_keys) >= {"lat", "lon"}+ if config.partition_key is None:+ assert response.partition_mode is None+ else:+ assert response.partition_mode == "datetime"+ assert response.partition_keys == [f"{config.timestamp_field}_iso"]+ # Rows arrive per location, so no read is one ordered series.assert response.sort_mode is Nonedef test_invalid_locations_raise(self):