Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(data-warehouse): add 6 missing tables to the Meteostat source (#119145)

Tom Owers46e4025c8ea4
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
index 26b770181ecb..ac9910d038e0 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
@@ -7813,10 +7813,10 @@ Today (3): `Daily`, `Hourly`, `Monthly`
Diffed against: <https://dev.meteostat.net/api/stations>
-- [ ] `Meta Data (GET /stations/meta)` — resolve station_id values into station names, coordinates, elevation, and country (high)
-- [ ] `Climate Normals (GET /stations/normals)` — compare observed weather with long-term monthly averages to measure climate anomalies (medium)
-- [ ] `Point Data (GET /point/hourly; GET /point/daily; GET /point/monthly)` — join weather to business locations through coordinates, without selecting individual weather stations (medium)
-- [ ] `Point Climate Normals (GET /point/normals)` — compare weather at business locations with normal monthly conditions (medium)
+- [x] `Meta Data (GET /stations/meta)` — resolve station_id values into station names, coordinates, elevation, and country (high)
+- [x] `Climate Normals (GET /stations/normals)` — compare observed weather with long-term monthly averages to measure climate anomalies (medium)
+- [x] `Point Data (GET /point/hourly; GET /point/daily; GET /point/monthly)` — join weather to business locations through coordinates, without selecting individual weather stations (medium)
+- [x] `Point Climate Normals (GET /point/normals)` — compare weather at business locations with normal monthly conditions (medium)
Note: The official station and point reference pages, including their GitHub Markdown sources, provided the GET operation list. Existing tables call only station observation endpoints for configured station IDs. Station details need one request per station; point tables need coordinate inputs. Nearby station discovery and the MCP interface are excluded.
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/meteostat.py b/products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/meteostat.py
index 1107191a2dc9..fcc221257c79 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/meteostat.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/meteostat.py
@@ -10,5 +10,6 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common imp
class MeteostatSourceConfig(config.Config):
api_key: str
station_ids: str
+ locations: str | None = None
units: Literal["metric", "imperial", "scientific"] = config.value(default="metric")
start_date: str | None = None
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/meteostat/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/meteostat/canonical_descriptions.py
index 2e87fef1be03..cb8ed9102db2 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/meteostat/canonical_descriptions.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/meteostat/canonical_descriptions.py
@@ -11,6 +11,69 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.can
_STATION_ID_COLUMN = "Meteostat weather station ID this record belongs to, as configured in the source settings."
+_POINT_COLUMNS = {
+ "latitude": "Latitude of the geographic location, as configured in the source settings.",
+ "longitude": "Longitude of the geographic location, as configured in the source settings.",
+ "altitude": "Elevation in meters of the geographic location, as configured in the source settings. Empty "
+ "when not set, in which case Meteostat estimates the elevation.",
+}
+
+_HOURLY_COLUMNS = {
+ "time": "Time of observation (YYYY-MM-DD hh:mm:ss).",
+ "temp": "The air temperature in °C.",
+ "dwpt": "The dew point in °C.",
+ "rhum": "The relative humidity in percent (%).",
+ "prcp": "The one hour precipitation total in mm.",
+ "snow": "The snow depth in mm.",
+ "wdir": "The wind direction in degrees (°).",
+ "wspd": "The average wind speed in km/h.",
+ "wpgt": "The peak wind gust in km/h.",
+ "pres": "The sea-level air pressure in hPa.",
+ "tsun": "The one hour sunshine total in minutes (m).",
+ "coco": "The weather condition code.",
+}
+
+_DAILY_COLUMNS = {
+ "date": "The date (YYYY-MM-DD).",
+ "tavg": "The average air temperature in °C.",
+ "tmin": "The minimum air temperature in °C.",
+ "tmax": "The maximum air temperature in °C.",
+ "prcp": "The daily precipitation total in mm.",
+ "snow": "The maximum snow depth in mm.",
+ "wdir": "The average wind direction in degrees (°).",
+ "wspd": "The average wind speed in km/h.",
+ "wpgt": "The peak wind gust in km/h.",
+ "pres": "The average sea-level air pressure in hPa.",
+ "tsun": "The daily sunshine total in minutes (m).",
+}
+
+_MONTHLY_COLUMNS = {
+ "date": "The first date (YYYY-MM-DD) of the month.",
+ "tavg": "The average daily air temperature in °C.",
+ "tmin": "The average daily minimum air temperature in °C.",
+ "tmax": "The average daily maximum air temperature in °C.",
+ "prcp": "The monthly precipitation total in mm.",
+ "snow": "The maximum snow depth in mm.",
+ "wdir": "The average wind direction in degrees (°).",
+ "wspd": "The average wind speed in km/h.",
+ "wpgt": "The peak wind gust in km/h.",
+ "pres": "The average sea-level air pressure in hPa.",
+ "tsun": "The monthly sunshine total in minutes (m).",
+}
+
+_NORMALS_COLUMNS = {
+ "start": "The first year (YYYY) of the reference period.",
+ "end": "The last year (YYYY) of the reference period.",
+ "month": "The month (1 to 12).",
+ "tavg": "The average daily air temperature in °C.",
+ "tmin": "The average daily minimum air temperature in °C.",
+ "tmax": "The average daily maximum air temperature in °C.",
+ "prcp": "The monthly precipitation total in mm.",
+ "wspd": "The average wind speed in km/h.",
+ "pres": "The average sea-level air pressure in hPa.",
+ "tsun": "The monthly sunshine total in minutes (m).",
+}
+
CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {
"Hourly": {
"description": "Historical hourly weather observations for a station, with gaps optionally filled "
@@ -18,18 +81,7 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {
"docs_url": "https://dev.meteostat.net/api/stations/hourly",
"columns": {
"station_id": _STATION_ID_COLUMN,
- "time": "Time of observation (YYYY-MM-DD hh:mm:ss).",
- "temp": "The air temperature in °C.",
- "dwpt": "The dew point in °C.",
- "rhum": "The relative humidity in percent (%).",
- "prcp": "The one hour precipitation total in mm.",
- "snow": "The snow depth in mm.",
- "wdir": "The wind direction in degrees (°).",
- "wspd": "The average wind speed in km/h.",
- "wpgt": "The peak wind gust in km/h.",
- "pres": "The sea-level air pressure in hPa.",
- "tsun": "The one hour sunshine total in minutes (m).",
- "coco": "The weather condition code.",
+ **_HOURLY_COLUMNS,
},
},
"Daily": {
@@ -38,17 +90,7 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {
"docs_url": "https://dev.meteostat.net/api/stations/daily",
"columns": {
"station_id": _STATION_ID_COLUMN,
- "date": "The date (YYYY-MM-DD).",
- "tavg": "The average air temperature in °C.",
- "tmin": "The minimum air temperature in °C.",
- "tmax": "The maximum air temperature in °C.",
- "prcp": "The daily precipitation total in mm.",
- "snow": "The maximum snow depth in mm.",
- "wdir": "The average wind direction in degrees (°).",
- "wspd": "The average wind speed in km/h.",
- "wpgt": "The peak wind gust in km/h.",
- "pres": "The average sea-level air pressure in hPa.",
- "tsun": "The daily sunshine total in minutes (m).",
+ **_DAILY_COLUMNS,
},
},
"Monthly": {
@@ -57,17 +99,57 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {
"docs_url": "https://dev.meteostat.net/api/stations/monthly",
"columns": {
"station_id": _STATION_ID_COLUMN,
- "date": "The first date (YYYY-MM-DD) of the month.",
- "tavg": "The average daily air temperature in °C.",
- "tmin": "The average daily minimum air temperature in °C.",
- "tmax": "The average daily maximum air temperature in °C.",
- "prcp": "The monthly precipitation total in mm.",
- "snow": "The maximum snow depth in mm.",
- "wdir": "The average wind direction in degrees (°).",
- "wspd": "The average wind speed in km/h.",
- "wpgt": "The peak wind gust in km/h.",
- "pres": "The average sea-level air pressure in hPa.",
- "tsun": "The monthly sunshine total in minutes (m).",
+ **_MONTHLY_COLUMNS,
+ },
+ },
+ "Stations": {
+ "description": "Meta data for each configured weather station, including its names, country, region, "
+ "identifiers, location, time zone, and data inventory.",
+ "docs_url": "https://dev.meteostat.net/api/stations/meta",
+ "columns": {
+ "station_id": _STATION_ID_COLUMN,
+ "id": "Meteostat ID of the weather station.",
+ "name": "Name of the weather station in different languages, keyed by language code.",
+ "country": "ISO 3166-1 alpha-2 country code, e.g. CA for Canada.",
+ "region": "ISO 3166-2 state or region code, e.g. TX for Texas.",
+ "identifier": "Other identifiers of the weather station: national, WMO, and ICAO IDs.",
+ "location": "Geographic location of the weather station: latitude, longitude, and elevation in meters.",
+ "timezone": "Time zone of the weather station.",
+ "inventory": "First and last day or year of the available model, hourly, daily, monthly, and climate "
+ "normals data.",
},
},
+ "Normals": {
+ "description": "Climate normals for a station: long-term monthly averages of weather observations, "
+ "with twelve rows (one per month) for each reference period.",
+ "docs_url": "https://dev.meteostat.net/api/stations/normals",
+ "columns": {
+ "station_id": _STATION_ID_COLUMN,
+ **_NORMALS_COLUMNS,
+ },
+ },
+ "PointHourly": {
+ "description": "Historical hourly weather observations for a geographic location, interpolated from "
+ "nearby weather stations, with gaps optionally filled by model data.",
+ "docs_url": "https://dev.meteostat.net/api/point/hourly",
+ "columns": {**_POINT_COLUMNS, **_HOURLY_COLUMNS},
+ },
+ "PointDaily": {
+ "description": "Historical daily weather statistics for a geographic location, interpolated from nearby "
+ "weather stations.",
+ "docs_url": "https://dev.meteostat.net/api/point/daily",
+ "columns": {**_POINT_COLUMNS, **_DAILY_COLUMNS},
+ },
+ "PointMonthly": {
+ "description": "Historical monthly weather statistics for a geographic location, interpolated from "
+ "nearby weather stations.",
+ "docs_url": "https://dev.meteostat.net/api/point/monthly",
+ "columns": {**_POINT_COLUMNS, **_MONTHLY_COLUMNS},
+ },
+ "PointNormals": {
+ "description": "Climate normals for a geographic location: long-term monthly averages, with twelve rows "
+ "(one per month) for each reference period.",
+ "docs_url": "https://dev.meteostat.net/api/point/normals",
+ "columns": {**_POINT_COLUMNS, **_NORMALS_COLUMNS},
+ },
}
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/meteostat/meteostat.py b/products/warehouse_sources/backend/temporal/data_imports/sources/meteostat/meteostat.py
index 5470430c0bc9..080cc6bcf085 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/meteostat/meteostat.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/meteostat/meteostat.py
@@ -1,3 +1,4 @@
+import re
import dataclasses
from collections.abc import Iterator
from datetime import UTC, date, datetime, timedelta
@@ -15,27 +16,57 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.meteostat.
DEFAULT_START_DATE,
DEFAULT_UNITS,
INCREMENTAL_OVERLAP_DAYS,
+ MAX_LOCATIONS,
MAX_STATIONS,
METEOSTAT_ENDPOINTS,
+ METEOSTAT_SNAPSHOT_ENDPOINTS,
MINIMUM_START_DATE,
MeteostatEndpointConfig,
+ MeteostatSnapshotEndpointConfig,
+ MeteostatTarget,
)
REQUEST_TIMEOUT_SECONDS = 60
NO_STATIONS_ERROR = "No weather station IDs configured"
+NO_LOCATIONS_ERROR = "No locations configured"
+INVALID_LOCATIONS_ERROR = "Invalid locations setting"
_TIME_FORMATS = ("%Y-%m-%d %H:%M:%S", "%Y-%m-%d")
@dataclasses.dataclass(frozen=True)
class MeteostatResumeConfig:
- # Index into the configured station list of the station currently being fetched.
+ # Index into the configured station (or, for point tables, location) list of the target
+ # currently being fetched.
station_index: int
- # ISO date (YYYY-MM-DD) of the first day the next window should fetch for that station.
+ # ISO date (YYYY-MM-DD) of the first day the next window should fetch for that target. Empty
+ # for snapshot tables, which fetch each target with one request.
next_start: str
[email protected](frozen=True)
+class MeteostatLocation:
+ latitude: float
+ longitude: float
+ altitude: Optional[int] = None
+
+ def request_params(self) -> dict[str, str]:
+ params = {"lat": str(self.latitude), "lon": str(self.longitude)}
+ if self.altitude is not None:
+ params["alt"] = str(self.altitude)
+ return params
+
+ def row_columns(self) -> dict[str, Any]:
+ return {"latitude": self.latitude, "longitude": self.longitude, "altitude": self.altitude}
+
+
[email protected](frozen=True)
+class _Target:
+ params: dict[str, str]
+ columns: dict[str, Any]
+
+
def _parse_station_ids(station_ids: Optional[str]) -> list[str]:
if not station_ids:
return []
@@ -52,6 +83,74 @@ def _parse_station_ids(station_ids: Optional[str]) -> list[str]:
return stations
+def _parse_location(entry: str) -> MeteostatLocation:
+ parts = [part.strip() for part in entry.split(",")]
+ invalid = ValueError(
+ f"{INVALID_LOCATIONS_ERROR}: '{entry[:50]}' is not in the form latitude,longitude or "
+ "latitude,longitude,elevation."
+ )
+ if len(parts) not in (2, 3):
+ raise invalid
+ try:
+ latitude = float(parts[0])
+ longitude = float(parts[1])
+ altitude = round(float(parts[2])) if len(parts) == 3 and parts[2] else None
+ except (ValueError, OverflowError):
+ raise invalid
+ # NaN fails both comparisons, so it is rejected here too.
+ if not (-90 <= latitude <= 90 and -180 <= longitude <= 180):
+ raise ValueError(
+ f"{INVALID_LOCATIONS_ERROR}: '{entry[:50]}' is out of range. Latitude must be between -90 and 90, "
+ "and longitude between -180 and 180."
+ )
+ return MeteostatLocation(latitude=latitude, longitude=longitude, altitude=altitude)
+
+
+def parse_locations(locations: Optional[str]) -> list[MeteostatLocation]:
+ """Parse `latitude,longitude[,elevation]` entries separated by semicolons or new lines.
+
+ Raises a `ValueError` with a user-facing message for a malformed entry or too many entries.
+ """
+ if not locations:
+ return []
+ parsed: list[MeteostatLocation] = []
+ seen: set[tuple[float, float]] = set()
+ for match in re.finditer(r"[^;\n]+", locations):
+ entry = match.group().strip()
+ if not entry:
+ continue
+ location = _parse_location(entry)
+ if (location.latitude, location.longitude) in seen:
+ continue
+ if len(parsed) == MAX_LOCATIONS:
+ raise ValueError(f"{INVALID_LOCATIONS_ERROR}: list at most {MAX_LOCATIONS} locations.")
+ seen.add((location.latitude, location.longitude))
+ parsed.append(location)
+ return parsed
+
+
+def _resolve_targets(
+ target: MeteostatTarget,
+ station_param: str,
+ station_ids: Optional[str],
+ locations: Optional[str],
+ logger: FilteringBoundLogger,
+) -> list[_Target]:
+ if target == "point":
+ parsed_locations = parse_locations(locations)
+ if not parsed_locations:
+ raise ValueError(NO_LOCATIONS_ERROR)
+ return [_Target(params=loc.request_params(), columns=loc.row_columns()) for loc in parsed_locations]
+
+ stations = _parse_station_ids(station_ids)
+ if not stations:
+ raise ValueError(NO_STATIONS_ERROR)
+ if len(stations) > MAX_STATIONS:
+ logger.warning(f"Meteostat: {len(stations)} station IDs configured, syncing only the first {MAX_STATIONS}")
+ stations = stations[:MAX_STATIONS]
+ return [_Target(params={station_param: station}, columns={"station_id": station}) for station in stations]
+
+
def _coerce_date(value: Any) -> Optional[date]:
if isinstance(value, datetime):
return value.date()
@@ -99,33 +198,46 @@ def _request_headers(api_key: str) -> dict[str, str]:
return {"x-rapidapi-key": api_key, "x-rapidapi-host": "meteostat.p.rapidapi.com"}
+def _fetch_data(
+ session: requests.Session,
+ headers: dict[str, str],
+ path: str,
+ params: dict[str, str],
+) -> list[dict[str, Any]]:
+ url = f"{BASE_URL}{path}?{urlencode(params)}"
+ response = session.get(url, headers=headers, timeout=REQUEST_TIMEOUT_SECONDS)
+ response.raise_for_status()
+
+ data = response.json().get("data")
+ # `/stations/meta` returns one object instead of a list.
+ if isinstance(data, dict):
+ return [data]
+ return data if isinstance(data, list) else []
+
+
def _fetch_window(
session: requests.Session,
headers: dict[str, str],
endpoint: MeteostatEndpointConfig,
- station_id: str,
+ target: _Target,
window_start: date,
window_end: date,
units: str,
) -> list[dict[str, Any]]:
params: dict[str, str] = {
- "station": station_id,
+ **target.params,
"start": window_start.isoformat(),
"end": window_end.isoformat(),
}
if units != DEFAULT_UNITS:
params["units"] = units
- url = f"{BASE_URL}{endpoint.path}?{urlencode(params)}"
- response = session.get(url, headers=headers, timeout=REQUEST_TIMEOUT_SECONDS)
- response.raise_for_status()
-
- data = response.json().get("data")
- return data if isinstance(data, list) else []
+ return _fetch_data(session, headers, endpoint.path, params)
def _get_rows(
api_key: str,
station_ids: str,
+ locations: Optional[str],
endpoint: MeteostatEndpointConfig,
units: str,
start_date: Optional[str],
@@ -134,12 +246,7 @@ def _get_rows(
should_use_incremental_field: bool,
db_incremental_field_last_value: Any,
) -> Iterator[list[dict[str, Any]]]:
- stations = _parse_station_ids(station_ids)
- if not stations:
- raise ValueError(NO_STATIONS_ERROR)
- if len(stations) > MAX_STATIONS:
- logger.warning(f"Meteostat: {len(stations)} station IDs configured, syncing only the first {MAX_STATIONS}")
- stations = stations[:MAX_STATIONS]
+ targets = _resolve_targets(endpoint.target, "station", station_ids, locations, logger)
session = make_tracked_session(redact_values=(api_key,))
headers = _request_headers(api_key)
@@ -159,25 +266,25 @@ def _get_rows(
resume = resumable_source_manager.load_state() if resumable_source_manager.can_resume() else None
start_index = resume.station_index if resume is not None else 0
- for index in range(start_index, len(stations)):
- station_id = stations[index]
+ for index in range(start_index, len(targets)):
+ target = targets[index]
cursor = base_start
if resume is not None and index == start_index:
resumed_start = _coerce_date(resume.next_start)
if resumed_start is not None:
cursor = resumed_start
- logger.debug(f"Meteostat: resuming {endpoint.name} for station {station_id} from {cursor.isoformat()}")
+ logger.debug(f"Meteostat: resuming {endpoint.name} for target {index} from {cursor.isoformat()}")
while cursor <= end_boundary:
window_end = min(cursor + timedelta(days=endpoint.window_days - 1), end_boundary)
- data = _fetch_window(session, headers, endpoint, station_id, cursor, window_end, units)
+ data = _fetch_window(session, headers, endpoint, target, cursor, window_end, units)
if data:
rows = []
for row in data:
row = dict(row)
- row["station_id"] = station_id
+ row.update(target.columns)
row[endpoint.date_field] = _parse_timestamp(row.get(endpoint.date_field))
rows.append(row)
yield rows
@@ -190,6 +297,37 @@ def _get_rows(
)
+def _get_snapshot_rows(
+ api_key: str,
+ station_ids: str,
+ locations: Optional[str],
+ endpoint: MeteostatSnapshotEndpointConfig,
+ units: str,
+ logger: FilteringBoundLogger,
+ resumable_source_manager: ResumableSourceManager[MeteostatResumeConfig],
+) -> Iterator[list[dict[str, Any]]]:
+ targets = _resolve_targets(endpoint.target, endpoint.station_param, station_ids, locations, logger)
+
+ session = make_tracked_session(redact_values=(api_key,))
+ headers = _request_headers(api_key)
+
+ resume = resumable_source_manager.load_state() if resumable_source_manager.can_resume() else None
+ start_index = resume.station_index if resume is not None else 0
+
+ for index in range(start_index, len(targets)):
+ target = targets[index]
+ params = dict(target.params)
+ if endpoint.supports_units and units != DEFAULT_UNITS:
+ params["units"] = units
+ rows = [{**row, **target.columns} for row in _fetch_data(session, headers, endpoint.path, params)]
+
+ resumable_source_manager.save_state(MeteostatResumeConfig(station_index=index + 1, next_start=""))
+ if rows:
+ yield rows
+ else:
+ resumable_source_manager.safe_point()
+
+
def meteostat_source(
api_key: str,
station_ids: str,
@@ -200,7 +338,24 @@ def meteostat_source(
resumable_source_manager: ResumableSourceManager[MeteostatResumeConfig],
should_use_incremental_field: bool = False,
db_incremental_field_last_value: Optional[Any] = None,
+ locations: Optional[str] = None,
) -> SourceResponse:
+ if endpoint_name in METEOSTAT_SNAPSHOT_ENDPOINTS:
+ snapshot = METEOSTAT_SNAPSHOT_ENDPOINTS[endpoint_name]
+ return SourceResponse(
+ name=snapshot.name,
+ items=lambda: _get_snapshot_rows(
+ api_key=api_key,
+ station_ids=station_ids,
+ locations=locations,
+ endpoint=snapshot,
+ units=units,
+ logger=logger,
+ resumable_source_manager=resumable_source_manager,
+ ),
+ primary_keys=snapshot.primary_keys,
+ )
+
endpoint = METEOSTAT_ENDPOINTS[endpoint_name]
return SourceResponse(
@@ -208,6 +363,7 @@ def meteostat_source(
items=lambda: _get_rows(
api_key=api_key,
station_ids=station_ids,
+ locations=locations,
endpoint=endpoint,
units=units,
start_date=start_date,
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/meteostat/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/meteostat/settings.py
index ce37a00226c4..e8cd3537a21e 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/meteostat/settings.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/meteostat/settings.py
@@ -1,5 +1,6 @@
from dataclasses import dataclass
from datetime import date
+from typing import Literal
from products.warehouse_sources.backend.temporal.data_imports.sources.common.schema import incremental_field
from products.warehouse_sources.backend.types import IncrementalField
@@ -10,12 +11,22 @@ API_HOST = "meteostat.p.rapidapi.com"
HOURLY_ENDPOINT = "Hourly"
DAILY_ENDPOINT = "Daily"
MONTHLY_ENDPOINT = "Monthly"
+POINT_HOURLY_ENDPOINT = "PointHourly"
+POINT_DAILY_ENDPOINT = "PointDaily"
+POINT_MONTHLY_ENDPOINT = "PointMonthly"
+STATIONS_ENDPOINT = "Stations"
+NORMALS_ENDPOINT = "Normals"
+POINT_NORMALS_ENDPOINT = "PointNormals"
+
+# Station endpoints take a configured station ID; point endpoints take configured coordinates.
+MeteostatTarget = Literal["station", "point"]
@dataclass(frozen=True)
class MeteostatEndpointConfig:
name: str
path: str
+ target: MeteostatTarget
date_field: str
# Per-request date-range cap the vendor enforces, expressed as a chunk size so a
# multi-year backfill is split into requests the API will actually accept.
@@ -28,6 +39,7 @@ METEOSTAT_ENDPOINTS: dict[str, MeteostatEndpointConfig] = {
HOURLY_ENDPOINT: MeteostatEndpointConfig(
name=HOURLY_ENDPOINT,
path="/stations/hourly",
+ target="station",
date_field="time",
# Vendor docs: "Hourly data can be queried for a maximum of 30 days per request."
window_days=30,
@@ -37,6 +49,7 @@ METEOSTAT_ENDPOINTS: dict[str, MeteostatEndpointConfig] = {
DAILY_ENDPOINT: MeteostatEndpointConfig(
name=DAILY_ENDPOINT,
path="/stations/daily",
+ target="station",
date_field="date",
# Vendor docs: "Daily data can be queried for a maximum of 10 years per request."
window_days=365 * 10,
@@ -46,6 +59,7 @@ METEOSTAT_ENDPOINTS: dict[str, MeteostatEndpointConfig] = {
MONTHLY_ENDPOINT: MeteostatEndpointConfig(
name=MONTHLY_ENDPOINT,
path="/stations/monthly",
+ target="station",
date_field="date",
# The vendor's monthly docs don't publish an explicit per-request range cap; reuse the
# documented daily cap as a conservative window size rather than guessing a larger one.
@@ -53,9 +67,100 @@ METEOSTAT_ENDPOINTS: dict[str, MeteostatEndpointConfig] = {
primary_keys=["station_id", "date"],
description="Historical monthly weather statistics for a station, aggregated from observations, daily records, and model data.",
),
+ POINT_HOURLY_ENDPOINT: MeteostatEndpointConfig(
+ name=POINT_HOURLY_ENDPOINT,
+ path="/point/hourly",
+ target="point",
+ date_field="time",
+ # Vendor docs: "Hourly data can be queried for a maximum of 30 days per request."
+ window_days=30,
+ primary_keys=["latitude", "longitude", "time"],
+ description="Historical hourly weather observations for a geographic location, interpolated from nearby stations.",
+ ),
+ POINT_DAILY_ENDPOINT: MeteostatEndpointConfig(
+ name=POINT_DAILY_ENDPOINT,
+ path="/point/daily",
+ target="point",
+ date_field="date",
+ # Vendor docs: "Daily data can be queried for a maximum of 10 years per request."
+ window_days=365 * 10,
+ primary_keys=["latitude", "longitude", "date"],
+ description="Historical daily weather statistics for a geographic location, interpolated from nearby stations.",
+ ),
+ POINT_MONTHLY_ENDPOINT: MeteostatEndpointConfig(
+ name=POINT_MONTHLY_ENDPOINT,
+ path="/point/monthly",
+ target="point",
+ date_field="date",
+ # No documented cap, as for the station monthly endpoint.
+ window_days=365 * 10,
+ primary_keys=["latitude", "longitude", "date"],
+ description="Historical monthly weather statistics for a geographic location, interpolated from nearby stations.",
+ ),
}
-ENDPOINTS = tuple(METEOSTAT_ENDPOINTS.keys())
+
+@dataclass(frozen=True)
+class MeteostatSnapshotEndpointConfig:
+ """An endpoint that returns the full dataset for a target in one request, with no date range."""
+
+ name: str
+ path: str
+ target: MeteostatTarget
+ # Query parameter that carries the station ID. Unused for point endpoints.
+ station_param: str
+ supports_units: bool
+ primary_keys: list[str]
+ description: str
+
+
+METEOSTAT_SNAPSHOT_ENDPOINTS: dict[str, MeteostatSnapshotEndpointConfig] = {
+ STATIONS_ENDPOINT: MeteostatSnapshotEndpointConfig(
+ name=STATIONS_ENDPOINT,
+ path="/stations/meta",
+ target="station",
+ station_param="id",
+ supports_units=False,
+ primary_keys=["station_id"],
+ description="Meta data for each configured weather station: name, country, region, identifiers, location, time zone, and data inventory.",
+ ),
+ NORMALS_ENDPOINT: MeteostatSnapshotEndpointConfig(
+ name=NORMALS_ENDPOINT,
+ path="/stations/normals",
+ target="station",
+ station_param="station",
+ supports_units=True,
+ # Without a start and end year the API returns twelve monthly rows for every reference period.
+ primary_keys=["station_id", "start", "end", "month"],
+ description="Climate normals for a station: long-term monthly averages for each reference period.",
+ ),
+ POINT_NORMALS_ENDPOINT: MeteostatSnapshotEndpointConfig(
+ name=POINT_NORMALS_ENDPOINT,
+ path="/point/normals",
+ target="point",
+ station_param="station",
+ supports_units=True,
+ primary_keys=["latitude", "longitude", "start", "end", "month"],
+ description="Climate normals for a geographic location: long-term monthly averages for each reference period.",
+ ),
+}
+
+ENDPOINTS = (
+ HOURLY_ENDPOINT,
+ DAILY_ENDPOINT,
+ MONTHLY_ENDPOINT,
+ STATIONS_ENDPOINT,
+ NORMALS_ENDPOINT,
+ POINT_HOURLY_ENDPOINT,
+ POINT_DAILY_ENDPOINT,
+ POINT_MONTHLY_ENDPOINT,
+ POINT_NORMALS_ENDPOINT,
+)
+
+POINT_ENDPOINTS = frozenset(
+ [name for name, config in METEOSTAT_ENDPOINTS.items() if config.target == "point"]
+ + [name for name, config in METEOSTAT_SNAPSHOT_ENDPOINTS.items() if config.target == "point"]
+)
INCREMENTAL_FIELDS: dict[str, list[IncrementalField]] = {
name: [incremental_field(config.date_field)] for name, config in METEOSTAT_ENDPOINTS.items()
@@ -90,3 +195,4 @@ INCREMENTAL_OVERLAP_DAYS = 7
# Each station costs at least one request per date window; an unbounded list can silently burn
# a whole month's free-tier quota in a single sync.
MAX_STATIONS = 25
+MAX_LOCATIONS = 25
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/meteostat/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/meteostat/source.py
index e0f65a13afcb..fe50db802474 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/meteostat/source.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/meteostat/source.py
@@ -25,17 +25,22 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.generated_
MeteostatSourceConfig,
)
from products.warehouse_sources.backend.temporal.data_imports.sources.meteostat.meteostat import (
+ INVALID_LOCATIONS_ERROR,
+ NO_LOCATIONS_ERROR,
NO_STATIONS_ERROR,
MeteostatResumeConfig,
_parse_station_ids,
meteostat_source,
+ parse_locations,
start_date_error,
validate_station,
)
from products.warehouse_sources.backend.temporal.data_imports.sources.meteostat.settings import (
ENDPOINTS,
INCREMENTAL_FIELDS,
+ MAX_LOCATIONS,
MAX_STATIONS,
+ POINT_ENDPOINTS,
UNITS_OPTIONS,
)
from products.warehouse_sources.backend.types import ExternalDataSourceType
@@ -61,6 +66,8 @@ class MeteostatSource(ResumableSource[MeteostatSourceConfig, MeteostatResumeConf
"This RapidAPI key isn't subscribed to the Meteostat API. Subscribe on RapidAPI and reconnect."
),
NO_STATIONS_ERROR: "Add at least one weather station ID in the source settings to sync this table.",
+ NO_LOCATIONS_ERROR: "Add at least one location in the source settings to sync this table.",
+ INVALID_LOCATIONS_ERROR: "Fix the locations in the source settings to sync this table.",
}
def get_canonical_descriptions(self) -> CanonicalDescriptions:
@@ -79,7 +86,14 @@ class MeteostatSource(ResumableSource[MeteostatSourceConfig, MeteostatResumeConf
force_refresh: bool = False,
api_version: str | None = None,
) -> list[SourceSchema]:
- return build_endpoint_schemas(ENDPOINTS, INCREMENTAL_FIELDS, names)
+ # Point tables fail without coordinates, so only select them by default once locations are set.
+ has_locations = bool(config.locations and config.locations.strip())
+ return build_endpoint_schemas(
+ ENDPOINTS,
+ INCREMENTAL_FIELDS,
+ names,
+ should_sync_default=dict.fromkeys(POINT_ENDPOINTS, has_locations),
+ )
def validate_credentials(
self,
@@ -98,6 +112,11 @@ class MeteostatSource(ResumableSource[MeteostatSourceConfig, MeteostatResumeConf
if error is not None:
return False, error
+ try:
+ parse_locations(config.locations)
+ except ValueError as e:
+ return False, str(e)
+
return validate_station(config.api_key, stations[0])
def get_resumable_source_manager(self, inputs: SourceInputs) -> ResumableSourceManager[MeteostatResumeConfig]:
@@ -112,6 +131,7 @@ class MeteostatSource(ResumableSource[MeteostatSourceConfig, MeteostatResumeConf
return meteostat_source(
api_key=config.api_key,
station_ids=config.station_ids,
+ locations=config.locations,
units=config.units,
start_date=config.start_date,
endpoint_name=inputs.schema_name,
@@ -136,7 +156,8 @@ class MeteostatSource(ResumableSource[MeteostatSourceConfig, MeteostatResumeConf
"[Meteostat API listing](https://rapidapi.com/meteostat/api/meteostat/) on RapidAPI — the free "
"plan includes 500 requests per month.\n\n"
"Meteostat has no account-scoped list of stations, so list the "
- "[weather station IDs](https://meteostat.net) you want to sync."
+ "[weather station IDs](https://meteostat.net) you want to sync. To sync the point tables, "
+ "which interpolate weather for any coordinates, also list your locations."
),
docsUrl="https://posthog.com/docs/cdp/sources/meteostat",
iconPath="/static/services/meteostat.png",
@@ -160,6 +181,19 @@ class MeteostatSource(ResumableSource[MeteostatSourceConfig, MeteostatResumeConf
secret=False,
caption=f"Comma-separated list of Meteostat station IDs. Up to {MAX_STATIONS} stations.",
),
+ SourceFieldInputConfig(
+ name="locations",
+ label="Locations",
+ type=SourceFieldInputConfigType.TEXTAREA,
+ required=False,
+ placeholder="43.6667,-79.4,113; 59.9127,10.7461",
+ secret=False,
+ caption=(
+ "Optional. Locations for the point tables, separated by semicolons or new lines. "
+ "Write each one as latitude,longitude with an optional elevation in meters, "
+ f"for example 43.6667,-79.4,113. Up to {MAX_LOCATIONS} locations."
+ ),
+ ),
SourceFieldSelectConfig(
name="units",
label="Unit system",
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/meteostat/tests/test_meteostat.py b/products/warehouse_sources/backend/temporal/data_imports/sources/meteostat/tests/test_meteostat.py
index 5c9dfa7a962c..e5ffded9a2b3 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/meteostat/tests/test_meteostat.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/meteostat/tests/test_meteostat.py
@@ -1,6 +1,7 @@
from datetime import UTC, date, datetime
from typing import Literal
+import pytest
import time_machine
from unittest import mock
@@ -17,21 +18,30 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.generated_
MeteostatSourceConfig,
)
from products.warehouse_sources.backend.temporal.data_imports.sources.meteostat.meteostat import (
+ INVALID_LOCATIONS_ERROR,
+ NO_LOCATIONS_ERROR,
NO_STATIONS_ERROR,
+ MeteostatLocation,
MeteostatResumeConfig,
_coerce_date,
_parse_station_ids,
_parse_timestamp,
+ parse_locations,
start_date_error,
validate_station,
)
from products.warehouse_sources.backend.temporal.data_imports.sources.meteostat.settings import (
DAILY_ENDPOINT,
HOURLY_ENDPOINT,
+ MAX_LOCATIONS,
MAX_STATIONS,
METEOSTAT_ENDPOINTS,
MINIMUM_START_DATE,
MONTHLY_ENDPOINT,
+ NORMALS_ENDPOINT,
+ POINT_DAILY_ENDPOINT,
+ POINT_NORMALS_ENDPOINT,
+ STATIONS_ENDPOINT,
)
from products.warehouse_sources.backend.temporal.data_imports.sources.meteostat.source import MeteostatSource
@@ -40,10 +50,13 @@ def _driver(
station_ids: str = "10637",
units: Literal["metric", "imperial", "scientific"] = "metric",
start_date: str | None = None,
+ locations: str | None = None,
) -> SourceDriver:
return SourceDriver(
MeteostatSource(),
- MeteostatSourceConfig(api_key="key-123", station_ids=station_ids, units=units, start_date=start_date),
+ MeteostatSourceConfig(
+ api_key="key-123", station_ids=station_ids, units=units, start_date=start_date, locations=locations
+ ),
)
@@ -100,6 +113,48 @@ class TestHelpers:
def test_parse_timestamp(self, _name, value, expected):
assert _parse_timestamp(value) == expected
+ @parameterized.expand(
+ [
+ ("none", None, []),
+ ("empty", "", []),
+ ("pair", "43.6667,-79.4", [MeteostatLocation(43.6667, -79.4)]),
+ ("triple_with_spaces", " 43.6667 , -79.4 , 113 ", [MeteostatLocation(43.6667, -79.4, 113)]),
+ (
+ "semicolons_new_lines_and_blank_entries",
+ "1,2; 3,4\n\n5,6,7;",
+ [MeteostatLocation(1, 2), MeteostatLocation(3, 4), MeteostatLocation(5, 6, 7)],
+ ),
+ ("dedupes_on_coordinates", "1,2,10;1,2,20", [MeteostatLocation(1, 2, 10)]),
+ ("range_boundaries", "-90,-180;90,180", [MeteostatLocation(-90, -180), MeteostatLocation(90, 180)]),
+ ("rounds_elevation", "1,2,113.6", [MeteostatLocation(1, 2, 114)]),
+ ("empty_elevation", "1,2,", [MeteostatLocation(1, 2)]),
+ (
+ "duplicates_do_not_count_toward_the_cap",
+ ";".join(f"{i},{i}" for i in range(MAX_LOCATIONS)) + ";0,0",
+ [MeteostatLocation(i, i) for i in range(MAX_LOCATIONS)],
+ ),
+ ]
+ )
+ def test_parse_locations(self, _name, value, expected):
+ assert parse_locations(value) == expected
+
+ @parameterized.expand(
+ [
+ ("one_value", "43.6"),
+ ("four_values", "1,2,3,4"),
+ ("not_a_number", "north,west"),
+ ("latitude_out_of_range", "91,0"),
+ ("longitude_out_of_range", "0,-181"),
+ ("nan_latitude", "nan,0"),
+ ("infinite_elevation", "1,2,inf"),
+ ("one_bad_entry_among_good_ones", "1,2;oops;3,4"),
+ ("too_many", ";".join(f"{i},{i}" for i in range(MAX_LOCATIONS + 1))),
+ ]
+ )
+ def test_parse_locations_rejects(self, _name, value):
+ with pytest.raises(ValueError, match=INVALID_LOCATIONS_ERROR):
+ parse_locations(value)
+
class TestGetRows:
@time_machine.travel("2026-07-21", tick=False)
@@ -144,13 +199,40 @@ class TestGetRows:
assert result.params("start") == ["2026-07-19"]
assert result.params("end") == ["2026-07-21"]
- def test_raises_without_configured_stations(self):
- result = _driver(station_ids=" , ").run(DAILY_ENDPOINT, [])
+ @parameterized.expand(
+ [
+ ("station_window", DAILY_ENDPOINT, NO_STATIONS_ERROR),
+ ("station_snapshot", STATIONS_ENDPOINT, NO_STATIONS_ERROR),
+ ("point_window", POINT_DAILY_ENDPOINT, NO_LOCATIONS_ERROR),
+ ("point_snapshot", POINT_NORMALS_ENDPOINT, NO_LOCATIONS_ERROR),
+ ]
+ )
+ def test_raises_without_configured_targets(self, _name, endpoint_name, error):
+ result = _driver(station_ids=" , ", locations=" ; ").run(endpoint_name, [])
assert isinstance(result.raised, ValueError)
- assert NO_STATIONS_ERROR in str(result.raised)
+ assert error in str(result.raised)
assert result.requests == []
+ @time_machine.travel("2026-07-21", tick=False)
+ def test_point_tables_query_each_location_by_coordinates(self):
+ result = _driver(locations="43.6667,-79.4,113; 59.9127,10.7461", start_date="2026-07-18").run(
+ POINT_DAILY_ENDPOINT,
+ [ScriptedResponse(json={"data": [{"date": "2026-07-18", "tavg": 5.0}]})] * 2,
+ )
+
+ assert result.raised is None
+ assert set(result.paths) == {"/point/daily"}
+ assert result.params("lat") == ["43.6667", "59.9127"]
+ assert result.params("lon") == ["-79.4", "10.7461"]
+ assert result.params("alt") == ["113", None]
+ assert result.params("station") == [None, None]
+ assert result.params("start") == ["2026-07-18"] * 2
+ assert result.rows == [
+ {"date": datetime(2026, 7, 18), "tavg": 5.0, "latitude": 43.6667, "longitude": -79.4, "altitude": 113},
+ {"date": datetime(2026, 7, 18), "tavg": 5.0, "latitude": 59.9127, "longitude": 10.7461, "altitude": None},
+ ]
+
@time_machine.travel("2026-07-21", tick=False)
def test_station_count_is_capped_at_runtime(self):
station_ids = ",".join(str(i) for i in range(MAX_STATIONS + 5))
@@ -191,6 +273,66 @@ class TestGetRows:
assert set(result.paths) == {"/stations/daily"}
+class TestSnapshotRows:
+ @parameterized.expand(
+ [
+ (
+ "station_meta_object_without_units",
+ STATIONS_ENDPOINT,
+ {"id": "10637", "country": "DE", "location": {"latitude": 50.05}},
+ "/stations/meta",
+ {"id": "10637", "station": None, "units": None},
+ {"id": "10637", "country": "DE", "location": {"latitude": 50.05}, "station_id": "10637"},
+ ),
+ (
+ "station_normals_with_units",
+ NORMALS_ENDPOINT,
+ [{"start": 1961, "end": 1990, "month": 1, "tavg": 33.0}],
+ "/stations/normals",
+ {"station": "10637", "id": None, "units": "imperial"},
+ {"start": 1961, "end": 1990, "month": 1, "tavg": 33.0, "station_id": "10637"},
+ ),
+ (
+ "point_normals_with_units",
+ POINT_NORMALS_ENDPOINT,
+ [{"start": 1991, "end": 2020, "month": 7, "tavg": 64.0}],
+ "/point/normals",
+ {"lat": "59.9127", "lon": "10.7461", "alt": "26", "station": None, "units": "imperial"},
+ {
+ "start": 1991,
+ "end": 2020,
+ "month": 7,
+ "tavg": 64.0,
+ "latitude": 59.9127,
+ "longitude": 10.7461,
+ "altitude": 26,
+ },
+ ),
+ ]
+ )
+ def test_request_and_row_shape(self, _name, endpoint_name, data, expected_path, expected_params, expected_row):
+ result = _driver(units="imperial", locations="59.9127,10.7461,26").run(
+ endpoint_name, [ScriptedResponse(json={"data": data})]
+ )
+
+ assert result.raised is None
+ assert result.paths == [expected_path]
+ assert {name: result.requests[0].param(name) for name in expected_params} == expected_params
+ assert result.rows == [expected_row]
+
+ def test_resume_skips_fetched_targets_and_tolerates_an_empty_response(self):
+ result = _driver(station_ids="10637,71508,D1234").run(
+ NORMALS_ENDPOINT,
+ [ScriptedResponse(json={"data": []}), ScriptedResponse(json={"data": [{"month": 1}]})],
+ resume_state=MeteostatResumeConfig(station_index=1, next_start=""),
+ )
+
+ assert result.raised is None
+ assert result.params("station") == ["71508", "D1234"]
+ assert result.rows == [{"month": 1, "station_id": "D1234"}]
+ assert result.saved_states[-1] == MeteostatResumeConfig(station_index=3, next_start="")
+
+
class TestMeteostatSourceResponse:
@parameterized.expand([(name,) for name in (HOURLY_ENDPOINT, DAILY_ENDPOINT, MONTHLY_ENDPOINT)])
@time_machine.travel("2026-07-21", tick=False)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/meteostat/tests/test_meteostat_source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/meteostat/tests/test_meteostat_source.py
index a4629d547822..1d105506320e 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/meteostat/tests/test_meteostat_source.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/meteostat/tests/test_meteostat_source.py
@@ -1,3 +1,5 @@
+from parameterized import parameterized
+
from products.warehouse_sources.backend.temporal.data_imports.sources.common.testing import (
ScriptedResponse,
scripted_network,
@@ -5,7 +7,10 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.tes
from products.warehouse_sources.backend.temporal.data_imports.sources.generated_configs.meteostat import (
MeteostatSourceConfig,
)
-from products.warehouse_sources.backend.temporal.data_imports.sources.meteostat.settings import MAX_STATIONS
+from products.warehouse_sources.backend.temporal.data_imports.sources.meteostat.settings import (
+ MAX_STATIONS,
+ POINT_ENDPOINTS,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.meteostat.source import MeteostatSource
@@ -34,6 +39,32 @@ class TestMeteostatSource:
assert is_valid is False
assert message is not None and "start date" in message.lower()
+ def test_validate_credentials_rejects_invalid_locations(self):
+ config = MeteostatSourceConfig(api_key="key-123", station_ids="10637", locations="43.6,-79.4;north,west")
+
+ with scripted_network([]) as network:
+ is_valid, message = self.source.validate_credentials(config, self.team_id)
+
+ assert is_valid is False
+ assert message is not None and "north,west" in message
+ assert network.requests_log == []
+
+ @parameterized.expand(
+ [
+ ("no_locations", None, False),
+ ("blank_locations", " ", False),
+ ("with_locations", "43.6667,-79.4", True),
+ ]
+ )
+ def test_point_tables_are_selected_by_default_only_with_locations(self, _name, locations, expected):
+ config = MeteostatSourceConfig(api_key="key-123", station_ids="10637", locations=locations)
+
+ defaults = {schema.name: schema.should_sync_default for schema in self.source.get_schemas(config, self.team_id)}
+
+ assert {name for name, selected in defaults.items() if not selected} == (
+ set() if expected else set(POINT_ENDPOINTS)
+ )
+
def test_validate_credentials_probes_the_first_configured_station(self):
config = MeteostatSourceConfig(api_key="key-123", station_ids="10637, 71508")