Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(warehouse_sources): add openweather history, geocoding and alert tables (#124002)

Tom Owers335e80838b9e
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
index c43de21b2f19..f1733e76f020 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
@@ -8850,10 +8850,10 @@ Today (4): `air_pollution`, `air_pollution_forecast`, `current_weather`, `foreca
Diffed against: <https://openweathermap.org/api>
-- [ ] `/data/2.5/air_pollution/history` — historical air quality back to Nov 2020 — the only way to trend AQI; today only current + forecast are synced (high)
-- [ ] `Geocoding API (/geo/1.0/direct, /geo/1.0/reverse, /geo/1.0/zip)` — lookup table resolving city/state/country/ZIP to the lat/lon that every synced row is keyed on (high)
-- [ ] `One Call API 3.0/4.0 (/data/3.0/onecall)` — unified current + minutely + hourly + daily + government weather alerts in one row set; the vendor's headline product (high)
-- [ ] `Weather History API (/data/3.0/onecall/timemachine, /data/2.5/history/city)` — hourly historical observations — required for any backward-looking weather correlation (high)
+- [x] `/data/2.5/air_pollution/history` — historical air quality back to Nov 2020 — the only way to trend AQI; today only current + forecast are synced (high)
+- [x] `Geocoding API (/geo/1.0/direct, /geo/1.0/reverse, /geo/1.0/zip)` — lookup table resolving city/state/country/ZIP to the lat/lon that every synced row is keyed on (high)
+- [x] `One Call API 3.0/4.0 (/data/3.0/onecall)` — unified current + minutely + hourly + daily + government weather alerts in one row set; the vendor's headline product (high)
+- [x] `Weather History API (/data/3.0/onecall/timemachine, /data/2.5/history/city)` — hourly historical observations — required for any backward-looking weather correlation (high)
- [ ] `One Call day summary (/data/3.0/onecall/day_summary)` — pre-aggregated daily weather rollup, the natural grain for joining against daily business metrics (medium)
- [ ] `Daily Forecast 16 days (/data/2.5/forecast/daily)` — extends forecast horizon past the 5-day/3-hour table already synced (medium)
- [ ] `Hourly Forecast 4 days (/data/2.5/forecast/hourly)` — finer-grained forecast than the 3-hour table (medium)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/openweather/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/openweather/canonical_descriptions.py
index 80176e23d377..4cb47cc573e6 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/openweather/canonical_descriptions.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/openweather/canonical_descriptions.py
@@ -64,6 +64,45 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {
"components": "Pollutant concentrations in μg/m³: co, no, no2, o3, so2, pm2_5, pm10, nh3.",
},
},
+ "air_pollution_history": {
+ "description": "Hourly historical air quality for a configured location, from 27 November 2020.",
+ "docs_url": "https://openweathermap.org/api/air-pollution",
+ "columns": {
+ **_INJECTED_COLUMNS,
+ "dt": "Time of the observed air-quality hour, Unix UTC timestamp.",
+ "main": "Air Quality Index: `aqi` from 1 (Good) to 5 (Very Poor).",
+ "components": "Pollutant concentrations in μg/m³: co, no, no2, o3, so2, pm2_5, pm10, nh3.",
+ },
+ },
+ "weather_history": {
+ "description": "Hourly historical weather observations for a configured location, from the History API.",
+ "docs_url": "https://openweathermap.org/history",
+ "columns": {
+ **_INJECTED_COLUMNS,
+ "dt": "Time of the observed hour, Unix UTC timestamp.",
+ "main": "Main measurements: temp, feels_like, pressure, humidity, temp_min, temp_max, sea_level, grnd_level.",
+ "wind": "Wind data: speed, deg.",
+ "clouds": "Cloudiness: `all`, as a percentage.",
+ "rain": "Rain volume for the last 1h/3h in mm, where available.",
+ "snow": "Snow volume for the last 1h/3h in mm, where available.",
+ "weather": "List of weather condition objects (id, main, description, icon).",
+ },
+ },
+ "geocoding": {
+ "description": "Place name, state and country for a configured location, from the OpenWeather reverse Geocoding API.",
+ "docs_url": "https://openweathermap.org/api/geocoding-api",
+ "columns": {
+ "lat": _INJECTED_COLUMNS["lat"],
+ "lon": _INJECTED_COLUMNS["lon"],
+ "location_label": _INJECTED_COLUMNS["location_label"],
+ "name": "Name of the place found at the location.",
+ "local_names": "Name of the place in other languages, keyed by ISO 639 language code.",
+ "country": "Country code (ISO 3166).",
+ "state": "State, where available.",
+ "place_lat": "Latitude of the place found.",
+ "place_lon": "Longitude of the place found.",
+ },
+ },
"air_pollution_forecast": {
"description": "Hourly air-quality forecast for a configured location.",
"docs_url": "https://openweathermap.org/api/air-pollution",
@@ -109,6 +148,22 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {
"alerts": _ALERTS_COLUMN,
},
},
+ "alerts": {
+ "description": "Government weather alert covering a configured location, from the One Call API 3.0.",
+ "docs_url": "https://openweathermap.org/api/one-call-3",
+ "columns": {
+ "lat": _INJECTED_COLUMNS["lat"],
+ "lon": _INJECTED_COLUMNS["lon"],
+ "location_label": _INJECTED_COLUMNS["location_label"],
+ "sender_name": "Name of the agency that issued the alert.",
+ "event": "Alert event name.",
+ "start": "Start of the alert, Unix UTC timestamp.",
+ "start_iso": "ISO 8601 UTC timestamp derived from `start`; used as the partition key.",
+ "end": "End of the alert, Unix UTC timestamp.",
+ "description": "Description of the alert.",
+ "tags": "Types of severe weather the alert covers.",
+ },
+ },
"quarter_hourly": {
"description": "15 minute weather forecast timeline for a configured location; one row per 15 minute slot.",
"docs_url": "https://openweathermap.org/api/one-call-4",
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/openweather/openweather.py b/products/warehouse_sources/backend/temporal/data_imports/sources/openweather/openweather.py
index 166f3fdd2899..aa257deca70c 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/openweather/openweather.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/openweather/openweather.py
@@ -9,10 +9,17 @@ import requests
from structlog.types import FilteringBoundLogger
from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
+from posthog.dataclasses import frozen
+
from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.interruptible_wait import (
+ interruptible_wait,
+)
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.safe_point import reach_safe_point
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
from products.warehouse_sources.backend.temporal.data_imports.sources.openweather.settings import (
PROBE_ENDPOINT_BY_VERSION,
+ HistoryWindow,
OpenWeatherEndpointConfig,
endpoints_for_version,
)
@@ -26,6 +33,9 @@ MAX_RETRY_ATTEMPTS = 5
# worker time and outbound fan-out — a malformed/abusive config can't tie up the pipeline.
MAX_LOCATIONS = 100
+# A history backfill makes many calls per location, and the free tier allows 60 calls a minute.
+HISTORY_REQUEST_INTERVAL_SECONDS = 1.0
+
class OpenWeatherRetryableError(Exception):
pass
@@ -83,8 +93,37 @@ def parse_locations(raw: str | None) -> list[Location]:
return locations
-def _build_url(path: str, params: dict[str, Any]) -> str:
- return f"{OPENWEATHER_BASE_URL}{path}?{urlencode(params)}"
+def _build_url(path: str, params: dict[str, Any], base_url: str | None = None) -> str:
+ return f"{base_url or OPENWEATHER_BASE_URL}{path}?{urlencode(params)}"
+
+
+@frozen
+class TimeWindow:
+ start: int
+ end: int
+
+
+def history_windows(window: HistoryWindow, start_from: int | None, end: int) -> Iterator[TimeWindow]:
+ """Split ``[start, end]`` into consecutive inclusive Unix ranges of at most ``window.size``.
+
+ ``start_from`` is the incremental watermark. Without one the read starts at the window's
+ ``earliest_start``, or ``lookback`` before ``end``.
+ """
+ if start_from is not None:
+ start = int(start_from)
+ elif window.earliest_start is not None:
+ start = int(window.earliest_start.timestamp())
+ elif window.lookback is not None:
+ start = end - int(window.lookback.total_seconds())
+ else:
+ raise ValueError("A history window needs an earliest_start or a lookback.")
+
+ size = int(window.size.total_seconds())
+ while start <= end:
+ window_end = min(start + size, end)
+ yield TimeWindow(start=start, end=window_end)
+ # Ranges are inclusive, so the next one starts a second later to not read a boundary row twice.
+ start = window_end + 1
# The API key is passed as the `appid` query param, so it ends up in `response.url`. `raise_for_status()`
@@ -102,7 +141,7 @@ def _redact_appid(text: str) -> str:
wait=wait_exponential_jitter(initial=1, max=30),
reraise=True,
)
-def _fetch(session: requests.Session, url: str, logger: FilteringBoundLogger) -> dict[str, Any]:
+def _fetch(session: requests.Session, url: str, logger: FilteringBoundLogger) -> Any:
response = session.get(url, timeout=REQUEST_TIMEOUT_SECONDS)
# 429 (free tier is 60 req/min) and transient 5xx are retryable; back off and try again.
@@ -128,16 +167,17 @@ def _dt_to_iso(dt: Any) -> str | None:
return datetime.fromtimestamp(int(dt), tz=UTC).isoformat()
-def _normalize_rows(
- config: OpenWeatherEndpointConfig, response: dict[str, Any], location: Location
-) -> list[dict[str, Any]]:
+def _normalize_rows(config: OpenWeatherEndpointConfig, response: Any, location: Location) -> list[dict[str, Any]]:
"""Turn an API response into flat rows, stamping each with the requested coordinates.
We inject the *requested* lat/lon (not the response's echoed ``coord``, which can snap to the
nearest station and drift between syncs) so the ``[lat, lon, dt]`` primary key stays stable.
"""
if config.data_key is None:
- items: list[dict[str, Any]] = [dict(response)]
+ if isinstance(response, list):
+ items: list[dict[str, Any]] = [dict(item) for item in response if isinstance(item, dict)]
+ else:
+ items = [dict(response)]
else:
section = response.get(config.data_key)
# A section is either a list of rows (forecast slots, One Call hourly/daily) or a single
@@ -156,12 +196,16 @@ def _normalize_rows(
item.setdefault("city", city)
for item in items:
+ if config.has_place_coords:
+ item["place_lat"] = item.pop("lat", None)
+ item["place_lon"] = item.pop("lon", None)
item["lat"] = location.lat
item["lon"] = location.lon
item["location_label"] = location.label
- # `dt` is part of the primary key, so a row without it is degenerate — read it directly
- # so a missing field fails loudly rather than flowing a null partition key into the merge.
- item["dt_iso"] = _dt_to_iso(item["dt"])
+ if config.timestamp_field is not None:
+ # The timestamp is part of the primary key, so a row without it is degenerate — read it
+ # directly so a missing field fails loudly rather than flowing a null partition key into the merge.
+ item[f"{config.timestamp_field}_iso"] = _dt_to_iso(item[config.timestamp_field])
return items
@@ -208,17 +252,35 @@ def get_rows(
locations: list[Location],
logger: FilteringBoundLogger,
api_version: str,
+ start_from: int | None = None,
) -> Iterator[list[dict[str, Any]]]:
config = endpoints_for_version(api_version)[endpoint]
# One session reused across every location so urllib3 keeps the connection alive.
session = make_tracked_session()
+ window = config.history_window
+ end = int(datetime.now(UTC).timestamp())
for location in locations:
- url = _build_url(config.path, {"lat": location.lat, "lon": location.lon, "appid": api_key})
- response = _fetch(session, url, logger)
- rows = _normalize_rows(config, response, location)
- if rows:
- yield rows
+ params = {"lat": location.lat, "lon": location.lon, **config.extra_params, "appid": api_key}
+ if window is None:
+ response = _fetch(session, _build_url(config.path, params, config.base_url), logger)
+ rows = _normalize_rows(config, response, location)
+ if rows:
+ yield rows
+ continue
+
+ for index, time_window in enumerate(history_windows(window, start_from, end)):
+ if index > 0:
+ interruptible_wait(HISTORY_REQUEST_INTERVAL_SECONDS, safe_point=reach_safe_point)
+ url = _build_url(
+ config.path, {**params, "start": time_window.start, "end": time_window.end}, config.base_url
+ )
+ rows = _normalize_rows(config, _fetch(session, url, logger), location)
+ if rows:
+ yield rows
+ else:
+ # Windows before the location's data begins come back empty.
+ reach_safe_point()
def openweather_source(
@@ -227,15 +289,28 @@ def openweather_source(
locations_raw: str | None,
logger: FilteringBoundLogger,
api_version: str,
+ db_incremental_field_last_value: Any = None,
) -> SourceResponse:
config = endpoints_for_version(api_version)[endpoint]
locations = parse_locations(locations_raw)
+ start_from = int(db_incremental_field_last_value) if db_incremental_field_last_value is not None else None
+
+ def items() -> Iterator[list[dict[str, Any]]]:
+ return get_rows(
+ api_key=api_key,
+ endpoint=endpoint,
+ locations=locations,
+ logger=logger,
+ api_version=api_version,
+ start_from=start_from,
+ )
+
+ if config.partition_key is None:
+ return SourceResponse(name=endpoint, items=items, primary_keys=config.primary_keys)
return SourceResponse(
name=endpoint,
- items=lambda: get_rows(
- api_key=api_key, endpoint=endpoint, locations=locations, logger=logger, api_version=api_version
- ),
+ items=items,
primary_keys=config.primary_keys,
partition_count=1,
partition_size=1,
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/openweather/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/openweather/settings.py
index c89eec461d6c..938cf10a7dbf 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/openweather/settings.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/openweather/settings.py
@@ -1,5 +1,6 @@
from dataclasses import dataclass, field
-from typing import Optional
+from datetime import UTC, datetime, timedelta
+from typing import Any, Optional
from products.warehouse_sources.backend.types import IncrementalField, IncrementalFieldType
@@ -17,22 +18,50 @@ _DT_INCREMENTAL_FIELDS: list[IncrementalField] = [
]
-@dataclass
+@dataclass(frozen=True)
+class HistoryWindow:
+ """How a history endpoint that takes a ``start``/``end`` Unix range is read."""
+
+ # Longest range one request covers. Each window is its own billed call.
+ size: timedelta
+ # Where a sync with no watermark starts: the first date the vendor has data for, or a lookback
+ # from the sync time when how far back the data goes depends on the customer's plan.
+ earliest_start: Optional[datetime] = None
+ lookback: Optional[timedelta] = None
+
+
+@dataclass(frozen=True)
class OpenWeatherEndpointConfig:
name: str
path: str
# Where rows live in the JSON response. ``None`` means the response *is* the row
- # (current weather), ``"list"`` means rows are the elements of the ``list`` array
- # (forecast, air pollution).
+ # (current weather) or a list of rows (geocoding), ``"list"`` means rows are the elements
+ # of the ``list`` array (forecast, air pollution).
data_key: Optional[str]
incremental_fields: list[IncrementalField] = field(default_factory=lambda: list(_DT_INCREMENTAL_FIELDS))
# ``dt`` alone is not unique table-wide because rows aggregate across every configured
# location, so the requested coordinates are part of the key.
primary_keys: list[str] = field(default_factory=lambda: ["lat", "lon", "dt"])
- # Stable datetime column used for partitioning (derived from ``dt``).
- partition_key: str = "dt_iso"
+ # Unix timestamp column each row is stamped with an ISO 8601 copy of (``<field>_iso``).
+ timestamp_field: Optional[str] = "dt"
+ # Stable datetime column used for partitioning (derived from ``timestamp_field``).
+ partition_key: Optional[str] = "dt_iso"
should_sync_default: bool = True
description: Optional[str] = None
+ # Host serving the path, when it is not the main API host.
+ base_url: Optional[str] = None
+ extra_params: dict[str, Any] = field(default_factory=dict)
+ # Set for endpoints that take a ``start``/``end`` range; these filter server-side on ``dt``,
+ # so they support true incremental sync.
+ history_window: Optional[HistoryWindow] = None
+ supports_append: bool = True
+ # The row carries the coordinates of the place it describes (geocoding). They move to
+ # ``place_lat``/``place_lon`` so the requested ``lat``/``lon`` key stays stable.
+ has_place_coords: bool = False
+
+ @property
+ def supports_incremental(self) -> bool:
+ return self.history_window is not None
# Opaque vendor version labels (never parsed or ordered). The Current Weather, 5 Day / 3 Hour
@@ -42,7 +71,27 @@ API_VERSION_2_5 = "2.5"
API_VERSION_3_0 = "3.0"
API_VERSION_4_0 = "4.0"
-# The 2.5 general-purpose APIs. Each endpoint is its own request per location per sync.
+# The Geocoding API is versioned on its own (`1.0`) and does not depend on the pinned product, so
+# every version exposes it as a lookup table over the configured locations.
+_SHARED_ENDPOINTS: dict[str, OpenWeatherEndpointConfig] = {
+ "geocoding": OpenWeatherEndpointConfig(
+ name="geocoding",
+ path="/geo/1.0/reverse",
+ data_key=None,
+ extra_params={"limit": 1},
+ incremental_fields=[],
+ primary_keys=["lat", "lon"],
+ timestamp_field=None,
+ partition_key=None,
+ supports_append=False,
+ has_place_coords=True,
+ description="Place name, state and country for each configured location, from reverse geocoding. "
+ "One row per location.",
+ ),
+}
+
+# The 2.5 general-purpose APIs. Each endpoint is its own request per location per sync, except the
+# history endpoints, which make one request per window per location.
OPENWEATHER_ENDPOINTS: dict[str, OpenWeatherEndpointConfig] = {
"current_weather": OpenWeatherEndpointConfig(
name="current_weather",
@@ -69,6 +118,30 @@ OPENWEATHER_ENDPOINTS: dict[str, OpenWeatherEndpointConfig] = {
data_key="list",
description="Hourly air-quality forecast for each configured location.",
),
+ "air_pollution_history": OpenWeatherEndpointConfig(
+ name="air_pollution_history",
+ path="/data/2.5/air_pollution/history",
+ data_key="list",
+ # The API documents no range limit per call; 30 days keeps each response a few hundred rows.
+ history_window=HistoryWindow(size=timedelta(days=30), earliest_start=datetime(2020, 11, 27, tzinfo=UTC)),
+ description="Hourly historical air quality for each configured location, from 27 November 2020. "
+ "One row per hour per location.",
+ ),
+ "weather_history": OpenWeatherEndpointConfig(
+ name="weather_history",
+ path="/data/2.5/history/city",
+ base_url="https://history.openweathermap.org",
+ data_key="list",
+ extra_params={"type": "hour"},
+ # One week is the most the API returns per call. How far back data goes depends on the
+ # subscription, so the first sync reads only the last week and later syncs extend it.
+ history_window=HistoryWindow(size=timedelta(days=7), lookback=timedelta(days=7)),
+ # Sold separately, so a key that passes validation can still get a 401 here.
+ should_sync_default=False,
+ description="Hourly historical weather observations for each configured location. The first sync reads "
+ "the last 7 days and each sync after it adds the hours since. Needs a History API subscription.",
+ ),
+ **_SHARED_ENDPOINTS,
}
# The One Call API 3.0 serves current weather plus forecasts from a single `/data/3.0/onecall`
@@ -83,6 +156,12 @@ OPENWEATHER_ENDPOINTS_3_0: dict[str, OpenWeatherEndpointConfig] = {
description="Current weather snapshot for each configured location, from the One Call API 3.0. "
"One row per location per sync; use append sync to accumulate a time series.",
),
+ "minutely": OpenWeatherEndpointConfig(
+ name="minutely",
+ path=_ONECALL_PATH,
+ data_key="minutely",
+ description="Minute-by-minute precipitation forecast for the next hour. One row per minute per location.",
+ ),
"hourly": OpenWeatherEndpointConfig(
name="hourly",
path=_ONECALL_PATH,
@@ -95,6 +174,21 @@ OPENWEATHER_ENDPOINTS_3_0: dict[str, OpenWeatherEndpointConfig] = {
data_key="daily",
description="Daily weather forecast (8 days) for each configured location. One row per day.",
),
+ "alerts": OpenWeatherEndpointConfig(
+ name="alerts",
+ path=_ONECALL_PATH,
+ data_key="alerts",
+ # Alerts carry no `dt` or id; an issuer's alert is identified by its event and start time.
+ # Full refresh only: an alert stays in the response while active, so append would insert it again
+ # on every sync.
+ incremental_fields=[],
+ primary_keys=["lat", "lon", "sender_name", "event", "start"],
+ timestamp_field="start",
+ partition_key="start_iso",
+ supports_append=False,
+ description="Government weather alerts active at sync time for each configured location. One row per alert.",
+ ),
+ **_SHARED_ENDPOINTS,
}
# The One Call API 4.0 splits the 3.0 response into one endpoint per timeline step, so each table
@@ -139,6 +233,7 @@ OPENWEATHER_ENDPOINTS_4_0: dict[str, OpenWeatherEndpointConfig] = {
data_key="data",
description="Daily weather forecast timeline. One row per day, up to 10 days ahead of the sync time.",
),
+ **_SHARED_ENDPOINTS,
}
OPENWEATHER_ENDPOINTS_BY_VERSION: dict[str, dict[str, OpenWeatherEndpointConfig]] = {
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/openweather/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/openweather/source.py
index 43003d93b525..b8ed888e41f8 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/openweather/source.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/openweather/source.py
@@ -100,6 +100,7 @@ Each sync polls every location once. To accumulate a history of point-in-time sn
# A missing/invalid key (or a key not subscribed to the requested product) surfaces as a
# 401 when `_fetch` calls `raise_for_status()`. Retrying can never satisfy it.
"401 Client Error: Unauthorized for url: https://api.openweathermap.org": "Your OpenWeather API key is invalid, not yet activated, or not subscribed to this product. Check the key in your OpenWeather account, then reconnect.",
+ "401 Client Error: Unauthorized for url: https://history.openweathermap.org": "Your OpenWeather API key is not subscribed to the History API. Disable the weather_history table or upgrade your OpenWeather plan.",
}
def get_schemas(
@@ -115,11 +116,11 @@ Each sync polls every location once. To accumulate a history of point-in-time sn
schemas = [
SourceSchema(
name=endpoint,
- # No server-side timestamp filter exists, so this is not true incremental sync.
- # Append is supported: each sync re-polls the (cheap) snapshot and merge dedupes on
- # `[lat, lon, dt]`, accumulating a time series across runs.
- supports_incremental=False,
- supports_append=True,
+ # Only the history endpoints filter server-side (`start`/`end`), so only they are
+ # true incremental. Append on the others re-polls the (cheap) snapshot each sync,
+ # accumulating a time series across runs.
+ supports_incremental=config.supports_incremental,
+ supports_append=config.supports_append,
incremental_fields=config.incremental_fields,
should_sync_default=config.should_sync_default,
description=config.description,
@@ -147,4 +148,7 @@ Each sync polls every location once. To accumulate a history of point-in-time sn
locations_raw=config.locations,
logger=inputs.logger,
api_version=self.resolve_api_version(inputs.api_version),
+ db_incremental_field_last_value=(
+ inputs.db_incremental_field_last_value if inputs.should_use_incremental_field else None
+ ),
)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/openweather/tests/test_openweather.py b/products/warehouse_sources/backend/temporal/data_imports/sources/openweather/tests/test_openweather.py
index 05c05ff6a21f..d6ea746f005a 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/openweather/tests/test_openweather.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/openweather/tests/test_openweather.py
@@ -1,7 +1,9 @@
import json
+from datetime import UTC, datetime, timedelta
from typing import Any, Optional
import pytest
+import time_machine
from unittest import mock
import requests
@@ -21,10 +23,12 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.openweathe
OPENWEATHER_BASE_URL,
Location,
OpenWeatherRetryableError,
+ TimeWindow,
_dt_to_iso,
_fetch,
_normalize_rows,
_redact_appid,
+ history_windows,
openweather_source,
parse_locations,
validate_credentials,
@@ -34,6 +38,8 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.openweathe
API_VERSION_3_0,
API_VERSION_4_0,
OPENWEATHER_ENDPOINTS,
+ OPENWEATHER_ENDPOINTS_3_0,
+ HistoryWindow,
endpoints_for_version,
)
from products.warehouse_sources.backend.temporal.data_imports.sources.openweather.source import OpenWeatherSource
@@ -136,6 +142,31 @@ class TestNormalizeRows:
assert all(row["city"] == {"id": 1, "name": "London"} for row in rows)
assert all(row["lat"] == 51.5 and row["lon"] == -0.12 for row in rows)
+ def test_geocoding_keeps_place_coords_apart_from_requested_coords(self):
+ response = [{"name": "London", "lat": 51.507, "lon": -0.127, "country": "GB", "state": "England"}]
+ rows = _normalize_rows(OPENWEATHER_ENDPOINTS["geocoding"], response, Location(51.5, -0.12, "London"))
+
+ assert rows == [
+ {
+ "name": "London",
+ "country": "GB",
+ "state": "England",
+ "place_lat": 51.507,
+ "place_lon": -0.127,
+ "lat": 51.5,
+ "lon": -0.12,
+ "location_label": "London",
+ }
+ ]
+
+ def test_alerts_partition_on_start(self):
+ response = {"alerts": [{"sender_name": "NWS", "event": "Flood Watch", "start": 1719158400, "end": 1719169200}]}
+ rows = _normalize_rows(OPENWEATHER_ENDPOINTS_3_0["alerts"], response, Location(51.5, -0.12, None))
+
+ assert len(rows) == 1
+ assert rows[0]["start_iso"] == "2024-06-23T16:00:00+00:00"
+ assert "dt_iso" not in rows[0]
+
def test_row_without_dt_raises(self):
# `dt` is part of the primary key; a row missing it must fail loudly, not yield a null key.
response = {"main": {"temp": 280}}
@@ -143,6 +174,44 @@ class TestNormalizeRows:
_normalize_rows(OPENWEATHER_ENDPOINTS["current_weather"], response, Location(51.5, -0.12, None))
+_DAY = 86400
+
+
+class TestHistoryWindows:
+ @pytest.mark.parametrize(
+ "window, start_from, end, expected",
+ [
+ # A watermark wins over the backfill start.
+ (
+ HistoryWindow(size=timedelta(days=7), lookback=timedelta(days=7)),
+ 10 * _DAY,
+ 12 * _DAY,
+ [TimeWindow(start=10 * _DAY, end=12 * _DAY)],
+ ),
+ (
+ HistoryWindow(size=timedelta(days=7), lookback=timedelta(days=7)),
+ None,
+ 30 * _DAY,
+ [TimeWindow(start=23 * _DAY, end=30 * _DAY)],
+ ),
+ # Consecutive windows never share a boundary second, so no row is read twice.
+ (
+ HistoryWindow(size=timedelta(days=2), earliest_start=datetime.fromtimestamp(0, tz=UTC)),
+ None,
+ 5 * _DAY,
+ [
+ TimeWindow(start=0, end=2 * _DAY),
+ TimeWindow(start=2 * _DAY + 1, end=4 * _DAY + 1),
+ TimeWindow(start=4 * _DAY + 2, end=5 * _DAY),
+ ],
+ ),
+ (HistoryWindow(size=timedelta(days=7), lookback=timedelta(days=7)), 12 * _DAY + 1, 12 * _DAY, []),
+ ],
+ )
+ def test_windows(self, window, start_from, end, expected):
+ assert list(history_windows(window, start_from, end)) == expected
+
+
# The undecorated `_fetch` (tenacity exposes the original via `__wrapped__`) so the status
# classification can be asserted without waiting through retry backoff.
_fetch_once = _fetch.__wrapped__ # type: ignore[attr-defined]
@@ -253,16 +322,73 @@ class TestGetRows:
assert result.requests[0].param("appid") == "test-key"
assert result.items[0][0]["dt"] == 1
+ @pytest.mark.parametrize(
+ "endpoint, should_use_incremental_field, expected_base, expected_starts, expected_type",
+ [
+ # Incremental: one window from the watermark to now, on the History API's own host.
+ ("weather_history", True, "https://history.openweathermap.org", [1782000000], "hour"),
+ # A full refresh ignores a stored watermark and reads the last week.
+ ("weather_history", False, "https://history.openweathermap.org", [1782518400 - 7 * _DAY], "hour"),
+ ("air_pollution_history", True, OPENWEATHER_BASE_URL, [1782000000], None),
+ ],
+ )
+ @time_machine.travel(datetime.fromtimestamp(1782518400, tz=UTC), tick=False)
+ def test_history_request_shaping(
+ self,
+ endpoint: str,
+ should_use_incremental_field: bool,
+ expected_base: str,
+ expected_starts: list[int],
+ expected_type: str | None,
+ ) -> None:
+ driver = SourceDriver(OpenWeatherSource(), OpenWeatherSourceConfig(api_key="test-key", locations="51.5,-0.12"))
+ result = driver.run(
+ endpoint,
+ [ScriptedResponse(json={"list": [{"dt": 1782003600, "main": {"temp": 280}}]})],
+ api_version=API_VERSION_2_5,
+ incremental_field="dt",
+ should_use_incremental_field=should_use_incremental_field,
+ db_incremental_field_last_value=1782000000,
+ )
+
+ assert result.raised is None
+ assert all(url.startswith(f"{expected_base}/data/2.5/") for url in result.urls)
+ assert [request.param("start") for request in result.requests] == [str(start) for start in expected_starts]
+ assert result.requests[-1].param("end") == "1782518400"
+ assert result.requests[0].param("type") == expected_type
+
+ @time_machine.travel(datetime(2021, 1, 1, tzinfo=UTC), tick=False)
+ def test_air_pollution_history_full_refresh_walks_back_to_first_data(self) -> None:
+ driver = SourceDriver(OpenWeatherSource(), OpenWeatherSourceConfig(api_key="test-key", locations="51.5,-0.12"))
+ result = driver.run(
+ "air_pollution_history",
+ always(ScriptedResponse(json={"list": [{"dt": 1609459200}]})),
+ api_version=API_VERSION_2_5,
+ )
+
+ assert result.raised is None
+ # 27 Nov 2020 to 1 Jan 2021 is two 30-day windows.
+ assert [request.param("start") for request in result.requests] == [
+ "1606435200",
+ str(1606435200 + 30 * _DAY + 1),
+ ]
+ assert result.requests[-1].param("end") == "1609459200"
+
class TestOpenWeatherSource:
@pytest.mark.parametrize("api_version, endpoint", _ALL_VERSIONED_ENDPOINTS)
def test_source_response_shape(self, api_version, endpoint):
+ config = endpoints_for_version(api_version)[endpoint]
response = openweather_source("test-key", endpoint, "51.5,-0.12,London", structlog.get_logger(), api_version)
assert response.name == endpoint
- assert response.primary_keys == ["lat", "lon", "dt"]
- assert response.partition_mode == "datetime"
- assert response.partition_keys == ["dt_iso"]
+ assert set(config.primary_keys) >= {"lat", "lon"}
+ if config.partition_key is None:
+ assert response.partition_mode is None
+ else:
+ assert response.partition_mode == "datetime"
+ assert response.partition_keys == [f"{config.timestamp_field}_iso"]
+ # Rows arrive per location, so no read is one ordered series.
assert response.sort_mode is None
def test_invalid_locations_raise(self):