Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(warehouse_sources): add electricity maps forecast, zones and price tables (#119545)

Tom Owersf05eac289b6e
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
index 8bd083a36d5d..f5fe97bcf127 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
@@ -4220,10 +4220,10 @@ Today (2): `carbon_intensity`, `power_breakdown`
Diffed against: <https://api.electricitymap.org/v3/openapi.json>
-- [ ] `Zones (GET /v3/zones)` — resolve zone identifiers into names, countries, and parent regions for existing carbon and power records (high)
-- [ ] `Carbon intensity forecast (GET /v3/carbon-intensity/forecast)` — compare predicted emissions with actual emissions and choose cleaner times for electricity use (high)
-- [ ] `Day-ahead price, past range (GET /v3/price-day-ahead/past-range)` — join electricity prices to carbon intensity for cost and emissions analysis (high)
-- [ ] `Day-ahead locational marginal price, past range (GET /v3/locational-marginal-price-day-ahead/past-range)` — measure electricity costs at United States pricing nodes alongside regional emissions (high)
+- [x] `Zones (GET /v3/zones)` — resolve zone identifiers into names, countries, and parent regions for existing carbon and power records (high)
+- [x] `Carbon intensity forecast (GET /v3/carbon-intensity/forecast)` — compare predicted emissions with actual emissions and choose cleaner times for electricity use (high)
+- [x] `Day-ahead price, past range (GET /v3/price-day-ahead/past-range)` — join electricity prices to carbon intensity for cost and emissions analysis (high)
+- [x] `Day-ahead locational marginal price, past range (GET /v3/locational-marginal-price-day-ahead/past-range)` — measure electricity costs at United States pricing nodes alongside regional emissions (high)
- [ ] `Power breakdown forecast (GET /v3/power-breakdown/forecast)` — compare predicted generation and consumption mixes with actual power records (medium)
- [ ] `Fossil-only carbon intensity, past range (GET /v3/carbon-intensity-fossil-only/past-range)` — analyze emissions intensity from fossil electricity separately from the complete grid mix (medium)
- [ ] `Total reported load, past range (GET /v3/total-reported-load/past-range)` — compare reported electricity demand with consumption estimates and identify demand trends (medium)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/electricity_maps/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/electricity_maps/canonical_descriptions.py
index 3e38426a1d04..bcb22b04675f 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/electricity_maps/canonical_descriptions.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/electricity_maps/canonical_descriptions.py
@@ -48,4 +48,67 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {
"estimationMethod": "Model used to produce the estimate, or null for measured data.",
},
},
+ "price_day_ahead": {
+ "description": "Hourly day-ahead electricity market price in a zone, in the currency and unit "
+ "given on each row (for example EUR/MWh).",
+ "docs_url": "https://app.electricitymaps.com/docs/api",
+ "columns": {
+ "zone": "Identifier of the electricity grid zone, e.g. `DE` or `DK-DK1`.",
+ "datetime": "UTC start of the hour this price applies to.",
+ "updatedAt": "When Electricity Maps last updated this data point.",
+ "createdAt": "When Electricity Maps first created this data point.",
+ "value": "Day-ahead price for this hour, in the currency and unit given by `unit`.",
+ "unit": "Currency and unit of the price, e.g. `EUR/MWh`.",
+ "source": "Where Electricity Maps obtained the price.",
+ "temporalGranularity": "Time resolution of the data point, e.g. `hourly`.",
+ },
+ },
+ "locational_marginal_price_day_ahead": {
+ "description": "Hourly day-ahead locational marginal price (LMP) at a United States pricing node, "
+ "with its energy, congestion, loss and greenhouse gas components.",
+ "docs_url": "https://app.electricitymaps.com/docs/api",
+ "columns": {
+ "entityName": "Pricing node identifier, e.g. `CORTINA_1_N106`.",
+ "entityType": "Type of the priced entity, e.g. `PNODE`.",
+ "datetime": "UTC start of the hour this price applies to.",
+ "updatedAt": "When Electricity Maps last updated this data point.",
+ "createdAt": "When Electricity Maps first created this data point.",
+ "lmp": "Locational marginal price for this hour, in the currency and unit given by `unit`.",
+ "unit": "Currency and unit of the price, e.g. `USD/MWh`.",
+ "energy": "Energy component of the locational marginal price.",
+ "congestion": "Congestion component of the locational marginal price.",
+ "loss": "Loss component of the locational marginal price.",
+ "greenhouseGas": "Greenhouse gas component of the locational marginal price.",
+ "source": "Where Electricity Maps obtained the price.",
+ "temporalGranularity": "Time resolution of the data point, e.g. `hourly`.",
+ },
+ },
+ "carbon_intensity_forecast": {
+ "description": "Latest hourly carbon intensity forecast for a zone, in gCO2eq/kWh. Each sync "
+ "replaces the table with the forecast Electricity Maps has most recently published.",
+ "docs_url": "https://app.electricitymaps.com/docs/api",
+ "columns": {
+ "zone": "Identifier of the electricity grid zone, e.g. `DE` or `DK-DK1`.",
+ "datetime": "UTC start of the future hour this forecast applies to.",
+ "carbonIntensity": "Forecasted carbon intensity of consumed electricity for this hour, in gCO2eq/kWh.",
+ "updatedAt": "When Electricity Maps last updated the forecast.",
+ },
+ },
+ "zones": {
+ "description": "Electricity Maps zones, with names, countries and parent zones, for resolving "
+ "the zone identifiers on other Electricity Maps tables.",
+ "docs_url": "https://app.electricitymaps.com/docs/api",
+ "columns": {
+ "zoneKey": "Identifier of the electricity grid zone, e.g. `DK-DK2`.",
+ "zoneName": "Human-readable zone name.",
+ "displayName": "Display name for the zone.",
+ "countryCode": "ISO country code of the zone.",
+ "countryName": "Country name of the zone.",
+ "zoneParentKey": "Key of the parent zone when this zone is a subzone, otherwise null.",
+ "subZoneKeys": "Keys of the subzones of this zone.",
+ "isCommerciallyAvailable": "Whether the zone is commercially available.",
+ "tier": "Data tier of the zone.",
+ "access": "Endpoints that the API token can reach for this zone.",
+ },
+ },
}
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/electricity_maps/electricity_maps.py b/products/warehouse_sources/backend/temporal/data_imports/sources/electricity_maps/electricity_maps.py
index 0ab4bc590126..6d89e214b6ad 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/electricity_maps/electricity_maps.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/electricity_maps/electricity_maps.py
@@ -1,9 +1,10 @@
import re
+from collections.abc import Iterator
from datetime import UTC, datetime, timedelta
from typing import Any, Optional
from dateutil import parser as dateutil_parser
-from requests import Request, Response
+from requests import Request, Response, Session
from requests.exceptions import RequestException
from posthog.dataclasses import frozen
@@ -19,10 +20,15 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.res
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
from products.warehouse_sources.backend.temporal.data_imports.sources.electricity_maps.settings import (
BASE_URL,
+ CARBON_INTENSITY_FORECAST,
+ CARBON_INTENSITY_FORECAST_PATH,
DEFAULT_HISTORY_DAYS,
- ENDPOINT_PATHS,
+ NODE_ENDPOINTS,
+ RANGE_ENDPOINT_PATHS,
REQUEST_TIMEOUT_SECONDS,
WINDOW_DAYS,
+ ZONES,
+ ZONES_PATH,
)
# Zone identifiers are short uppercase codes like DE, DK-DK1, US-CAL-CISO. The value goes into a
@@ -46,6 +52,16 @@ def parse_zones(zones: str) -> list[str]:
return parsed
+def parse_pricing_nodes(pricing_nodes: Optional[str]) -> list[str]:
+ # Node identifiers are vendor-defined (e.g. CORTINA_1_N106), so keep their case as entered.
+ parsed: list[str] = []
+ for node in (pricing_nodes or "").split(","):
+ normalized = node.strip()
+ if normalized and normalized not in parsed:
+ parsed.append(normalized)
+ return parsed
+
+
def invalid_zones(zones: list[str]) -> list[str]:
return [zone for zone in zones if not _ZONE_RE.match(zone)]
@@ -76,7 +92,7 @@ def initial_window_start(db_incremental_field_last_value: Optional[Any], history
class ElectricityMapsRangePaginator(BasePaginator):
- """Walks fixed datetime windows across every configured zone.
+ """Walks fixed datetime windows across every configured zone (or pricing node).
Iteration is window-major (every zone for a window, then the next window) rather than
zone-major. The pipeline checkpoints the incremental watermark per batch assuming ascending
@@ -85,9 +101,17 @@ class ElectricityMapsRangePaginator(BasePaginator):
Window-major bounds that loss to a single window.
"""
- def __init__(self, zones: list[str], initial_start: datetime, until: datetime, window: timedelta) -> None:
+ def __init__(
+ self,
+ zones: list[str],
+ initial_start: datetime,
+ until: datetime,
+ window: timedelta,
+ zone_param: str = "zone",
+ ) -> None:
super().__init__()
self._zones = zones
+ self._zone_param = zone_param
self._until = until
self._window = window
self._window_start = self._clamp_start(initial_start)
@@ -104,7 +128,7 @@ class ElectricityMapsRangePaginator(BasePaginator):
def _set_params(self, request: Request) -> None:
if request.params is None:
request.params = {}
- request.params["zone"] = self._zones[self._zone_index]
+ request.params[self._zone_param] = self._zones[self._zone_index]
request.params["start"] = _format_datetime(self._window_start)
request.params["end"] = _format_datetime(self._window_end())
@@ -149,7 +173,7 @@ def _get_resource(endpoint: str, should_use_incremental_field: bool) -> Endpoint
if should_use_incremental_field
else "replace",
"endpoint": {
- "path": ENDPOINT_PATHS[endpoint],
+ "path": RANGE_ENDPOINT_PATHS[endpoint],
"data_selector": "data",
"data_selector_required": True,
"params": {},
@@ -163,6 +187,56 @@ def _get_resource(endpoint: str, should_use_incremental_field: bool) -> Endpoint
}
+def _make_session(api_token: str) -> Session:
+ # The auth-token header must never leave the API host, so redirects are refused.
+ return make_tracked_session(headers={"auth-token": api_token}, redact_values=(api_token,), allow_redirects=False)
+
+
+def _get_json(session: Session, path: str, params: Optional[dict[str, str]] = None) -> Any:
+ response = session.get(f"{BASE_URL}{path}", params=params, timeout=REQUEST_TIMEOUT_SECONDS)
+ response.raise_for_status()
+ return response.json()
+
+
+def _iter_carbon_intensity_forecast(
+ api_token: str,
+ zones: list[str],
+ resumable_source_manager: ResumableSourceManager[ElectricityMapsResumeConfig],
+) -> Iterator[list[dict[str, Any]]]:
+ session = _make_session(api_token)
+ for zone in zones:
+ body = _get_json(session, CARBON_INTENSITY_FORECAST_PATH, {"zone": zone})
+ forecast = body.get("forecast")
+ # A full refresh replaces the table, so a malformed response must fail the sync rather than
+ # silently drop this zone's rows.
+ if not isinstance(forecast, list):
+ raise ValueError(f"Electricity Maps returned a forecast response without a forecast list for zone {zone}")
+ # Forecast points carry only the target hour and value; the zone and the forecast's issue
+ # time live on the envelope.
+ updated_at = _coerce_datetime(body.get("updatedAt"))
+ rows = [
+ {
+ **point,
+ "datetime": _coerce_datetime(point.get("datetime")),
+ "zone": body.get("zone") or zone,
+ "updatedAt": updated_at,
+ }
+ for point in forecast
+ ]
+ if rows:
+ yield rows
+ else:
+ resumable_source_manager.safe_point()
+
+
+def _iter_zones(api_token: str) -> Iterator[list[dict[str, Any]]]:
+ body = _get_json(_make_session(api_token), ZONES_PATH)
+ # The response is an object keyed by zone key rather than a list.
+ rows = [{**zone, "zoneKey": zone.get("zoneKey") or zone_key} for zone_key, zone in body.items()]
+ if rows:
+ yield rows
+
+
def electricity_maps_source(
api_token: str,
zones: list[str],
@@ -173,11 +247,28 @@ def electricity_maps_source(
db_incremental_field_last_value: Optional[Any],
should_use_incremental_field: bool = False,
history_days: Optional[int] = None,
+ pricing_nodes: Optional[list[str]] = None,
) -> SourceResponse:
- if endpoint not in ENDPOINT_PATHS:
+ if endpoint not in RANGE_ENDPOINT_PATHS and endpoint not in (CARBON_INTENSITY_FORECAST, ZONES):
raise ValueError(f"Unknown Electricity Maps endpoint: {endpoint}")
- if not zones:
- raise ValueError("No Electricity Maps zones configured")
+
+ if endpoint == ZONES:
+ return SourceResponse(name=endpoint, items=lambda: _iter_zones(api_token), primary_keys=["zoneKey"])
+
+ is_node_endpoint = endpoint in NODE_ENDPOINTS
+ entities = (pricing_nodes or []) if is_node_endpoint else zones
+ if not entities:
+ raise ValueError(f"No Electricity Maps {'pricing nodes' if is_node_endpoint else 'zones'} configured")
+
+ if endpoint == CARBON_INTENSITY_FORECAST:
+ return SourceResponse(
+ name=endpoint,
+ items=lambda: _iter_carbon_intensity_forecast(api_token, zones, resumable_source_manager),
+ primary_keys=["zone", "datetime"],
+ partition_keys=["datetime"],
+ partition_mode="datetime",
+ partition_format="month",
+ )
if history_days is None or history_days <= 0:
history_days = DEFAULT_HISTORY_DAYS
@@ -187,7 +278,11 @@ def electricity_maps_source(
db_incremental_field_last_value if should_use_incremental_field else None, history_days, now
)
paginator = ElectricityMapsRangePaginator(
- zones=zones, initial_start=start, until=now, window=timedelta(days=WINDOW_DAYS)
+ zones=entities,
+ initial_start=start,
+ until=now,
+ window=timedelta(days=WINDOW_DAYS),
+ zone_param="node" if is_node_endpoint else "zone",
)
config: RESTAPIConfig = {
@@ -239,7 +334,7 @@ def electricity_maps_source(
return SourceResponse(
name=endpoint,
items=lambda: resource,
- primary_keys=["zone", "datetime"],
+ primary_keys=["entityName", "datetime"] if is_node_endpoint else ["zone", "datetime"],
column_hints=resource.column_hints,
partition_keys=["datetime"],
partition_mode="datetime",
@@ -249,9 +344,7 @@ def electricity_maps_source(
def validate_credentials(api_token: str, zones: list[str]) -> tuple[bool, str | None]:
- # Same redirect boundary as the extraction client: the auth-token header must never leave
- # the API host, so this probe pins to BASE_URL rather than following the session default.
- session = make_tracked_session(headers={"auth-token": api_token}, redact_values=(api_token,), allow_redirects=False)
+ session = _make_session(api_token)
# `/carbon-intensity/latest` is the cheapest authenticated call and is available on every
# plan, so it both proves the token and surfaces per-zone entitlement before the first sync.
for zone in zones:
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/electricity_maps/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/electricity_maps/settings.py
index 7933bc9e8179..9e8a4f9f66fa 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/electricity_maps/settings.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/electricity_maps/settings.py
@@ -4,14 +4,34 @@ BASE_URL = "https://api.electricitymaps.com/v3"
CARBON_INTENSITY = "carbon_intensity"
POWER_BREAKDOWN = "power_breakdown"
-
-ENDPOINTS = (CARBON_INTENSITY, POWER_BREAKDOWN)
-
-ENDPOINT_PATHS: dict[str, str] = {
+PRICE_DAY_AHEAD = "price_day_ahead"
+LOCATIONAL_MARGINAL_PRICE_DAY_AHEAD = "locational_marginal_price_day_ahead"
+CARBON_INTENSITY_FORECAST = "carbon_intensity_forecast"
+ZONES = "zones"
+
+ENDPOINTS = (
+ CARBON_INTENSITY,
+ POWER_BREAKDOWN,
+ PRICE_DAY_AHEAD,
+ LOCATIONAL_MARGINAL_PRICE_DAY_AHEAD,
+ CARBON_INTENSITY_FORECAST,
+ ZONES,
+)
+
+RANGE_ENDPOINT_PATHS: dict[str, str] = {
CARBON_INTENSITY: "/carbon-intensity/past-range",
POWER_BREAKDOWN: "/power-breakdown/past-range",
+ PRICE_DAY_AHEAD: "/price-day-ahead/past-range",
+ LOCATIONAL_MARGINAL_PRICE_DAY_AHEAD: "/locational-marginal-price-day-ahead/past-range",
}
+# Locational marginal prices are published per United States pricing node rather than per zone,
+# and their rows name the node in `entityName` instead of carrying a `zone`.
+NODE_ENDPOINTS = frozenset({LOCATIONAL_MARGINAL_PRICE_DAY_AHEAD})
+
+CARBON_INTENSITY_FORECAST_PATH = "/carbon-intensity/forecast"
+ZONES_PATH = "/zones"
+
# The past-range endpoints have no pagination: one request returns every hourly point in the
# requested window, and the API rejects windows longer than 10 days. Walk the range in windows
# comfortably under that cap.
@@ -35,4 +55,6 @@ _DATETIME_FIELD: IncrementalField = {
INCREMENTAL_FIELDS: dict[str, list[IncrementalField]] = {
CARBON_INTENSITY: [_DATETIME_FIELD],
POWER_BREAKDOWN: [_DATETIME_FIELD],
+ PRICE_DAY_AHEAD: [_DATETIME_FIELD],
+ LOCATIONAL_MARGINAL_PRICE_DAY_AHEAD: [_DATETIME_FIELD],
}
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/electricity_maps/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/electricity_maps/source.py
index dc5c3aae3529..294d660da7ad 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/electricity_maps/source.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/electricity_maps/source.py
@@ -23,12 +23,15 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.electricit
ElectricityMapsResumeConfig,
electricity_maps_source,
invalid_zones,
+ parse_pricing_nodes,
parse_zones,
validate_credentials as validate_electricity_maps_credentials,
)
from products.warehouse_sources.backend.temporal.data_imports.sources.electricity_maps.settings import (
ENDPOINTS,
INCREMENTAL_FIELDS,
+ LOCATIONAL_MARGINAL_PRICE_DAY_AHEAD,
+ NODE_ENDPOINTS,
)
from products.warehouse_sources.backend.temporal.data_imports.sources.generated_configs.electricitymaps import (
ElectricityMapsSourceConfig,
@@ -54,6 +57,7 @@ class ElectricityMapsSource(ResumableSource[ElectricityMapsSourceConfig, Electri
"401 Client Error": "Electricity Maps rejected your API token. Check the token in the Electricity Maps portal and reconnect the source.",
"403 Client Error": "Your Electricity Maps token does not have access to a configured zone or the requested history range. Check your plan in the Electricity Maps portal.",
"404 Client Error": "Electricity Maps does not recognize one of the configured zones. Check the zone identifiers in the source settings.",
+ "No Electricity Maps pricing nodes configured": "Add at least one pricing node in the source settings to sync day-ahead locational marginal prices.",
}
def get_canonical_descriptions(self) -> CanonicalDescriptions:
@@ -72,7 +76,13 @@ class ElectricityMapsSource(ResumableSource[ElectricityMapsSourceConfig, Electri
force_refresh: bool = False,
api_version: str | None = None,
) -> list[SourceSchema]:
- return build_endpoint_schemas(ENDPOINTS, INCREMENTAL_FIELDS, names)
+ # Locational marginal prices need pricing nodes, which most users won't have configured.
+ return build_endpoint_schemas(
+ ENDPOINTS,
+ INCREMENTAL_FIELDS,
+ names,
+ should_sync_default={LOCATIONAL_MARGINAL_PRICE_DAY_AHEAD: False},
+ )
def validate_credentials(
self,
@@ -93,11 +103,29 @@ class ElectricityMapsSource(ResumableSource[ElectricityMapsSourceConfig, Electri
"Enter comma-separated zone codes, like DE, DK-DK1, US-CAL-CISO.",
)
+ if schema_name in NODE_ENDPOINTS and not parse_pricing_nodes(config.pricing_nodes):
+ return False, "Enter at least one pricing node, like CORTINA_1_N106, to sync locational marginal prices."
+
return validate_electricity_maps_credentials(config.api_token, zones)
+ def get_endpoint_permissions(
+ self,
+ config: ElectricityMapsSourceConfig,
+ team_id: int,
+ endpoints: list[str],
+ api_version: str | None = None,
+ ) -> dict[str, str | None]:
+ has_pricing_nodes = bool(parse_pricing_nodes(config.pricing_nodes))
+ return {
+ endpoint: "Add at least one pricing node in the source settings to sync this table."
+ if endpoint in NODE_ENDPOINTS and not has_pricing_nodes
+ else None
+ for endpoint in endpoints
+ }
+
def get_resumable_source_manager(self, inputs: SourceInputs) -> ResumableSourceManager[ElectricityMapsResumeConfig]:
- # carbon_intensity and power_breakdown each keep their own (window, zone) cursor, so a
- # retry that switches endpoints can't load the other endpoint's resume state.
+ # Each endpoint keeps its own (window, zone) cursor, so a retry that switches
+ # endpoints can't load another endpoint's resume state.
return ResumableSourceManager[ElectricityMapsResumeConfig](inputs, ElectricityMapsResumeConfig).with_namespace(
inputs.schema_name
)
@@ -120,6 +148,7 @@ class ElectricityMapsSource(ResumableSource[ElectricityMapsSourceConfig, Electri
if inputs.should_use_incremental_field
else None,
history_days=config.history_days,
+ pricing_nodes=parse_pricing_nodes(config.pricing_nodes),
)
@property
@@ -131,7 +160,9 @@ class ElectricityMapsSource(ResumableSource[ElectricityMapsSourceConfig, Electri
caption="Enter your Electricity Maps API token, created in the "
"[Electricity Maps portal](https://portal.electricitymaps.com/).\n\n"
"Zones is a comma-separated list of zone identifiers to sync, like `DE, DK-DK1, US-CAL-CISO`. "
- "Zone access and history depth depend on your Electricity Maps plan.",
+ "Zone access and history depth depend on your Electricity Maps plan.\n\n"
+ "Pricing nodes is an optional comma-separated list of United States pricing node identifiers, "
+ "like `CORTINA_1_N106`. It's only needed for the day-ahead locational marginal price table.",
keywords=["carbon intensity", "grid", "electricity map"],
docsUrl="https://posthog.com/docs/cdp/sources/electricity-maps",
iconPath="/static/services/electricity_maps.png",
@@ -155,6 +186,14 @@ class ElectricityMapsSource(ResumableSource[ElectricityMapsSourceConfig, Electri
placeholder="DE, DK-DK1, US-CAL-CISO",
secret=False,
),
+ SourceFieldInputConfig(
+ name="pricing_nodes",
+ label="Pricing nodes",
+ type=SourceFieldInputConfigType.TEXT,
+ required=False,
+ placeholder="CORTINA_1_N106",
+ secret=False,
+ ),
SourceFieldInputConfig(
name="history_days",
label="Days of history to sync initially",
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/electricity_maps/tests/test_electricity_maps.py b/products/warehouse_sources/backend/temporal/data_imports/sources/electricity_maps/tests/test_electricity_maps.py
index 88634b28f83e..9ecf0d5d4b7f 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/electricity_maps/tests/test_electricity_maps.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/electricity_maps/tests/test_electricity_maps.py
@@ -34,10 +34,12 @@ def _page(zone: str) -> ScriptedResponse:
)
-def _driver(zones: str = "DE", history_days: int | None = None) -> SourceDriver:
+def _driver(zones: str = "DE", history_days: int | None = None, pricing_nodes: str | None = None) -> SourceDriver:
return SourceDriver(
ElectricityMapsSource(),
- ElectricityMapsSourceConfig(api_token="test-token", zones=zones, history_days=history_days),
+ ElectricityMapsSourceConfig(
+ api_token="test-token", zones=zones, history_days=history_days, pricing_nodes=pricing_nodes
+ ),
)
@@ -87,9 +89,19 @@ class TestWindowWalk:
]
@time_machine.travel(_NOW, tick=False)
- def test_watermark_at_or_past_now_still_requests_one_valid_window(self) -> None:
- result = _driver().run(
- "carbon_intensity",
+ @pytest.mark.parametrize(
+ ("endpoint", "entity_param", "entity"),
+ [
+ ("carbon_intensity", "zone", "DE"),
+ # Locational marginal prices are addressed by pricing node; the API rejects a zone.
+ ("locational_marginal_price_day_ahead", "node", "CORTINA_1_N106"),
+ ],
+ )
+ def test_watermark_at_or_past_now_still_requests_one_valid_window(
+ self, endpoint: str, entity_param: str, entity: str
+ ) -> None:
+ result = _driver(pricing_nodes="CORTINA_1_N106").run(
+ endpoint,
[_page("DE")],
should_use_incremental_field=True,
db_incremental_field_last_value=datetime(2025, 6, 15, 13, 0, 0, tzinfo=UTC),
@@ -97,7 +109,7 @@ class TestWindowWalk:
assert result.raised is None
assert result.queries == [
- {"zone": ("DE",), "start": ("2025-06-15T11:00:00Z",), "end": ("2025-06-15T12:00:00Z",)}
+ {entity_param: (entity,), "start": ("2025-06-15T11:00:00Z",), "end": ("2025-06-15T12:00:00Z",)}
]
assert result.saved_states == []
@@ -156,11 +168,87 @@ class TestWindowWalk:
assert isinstance(result.raised, ValueError)
assert "Unknown Electricity Maps endpoint" in str(result.raised)
- def test_no_zones_raises(self) -> None:
- result = _driver(zones="").run("carbon_intensity", [])
+ @pytest.mark.parametrize(
+ ("endpoint", "zones", "pricing_nodes", "expected_fragment"),
+ [
+ ("carbon_intensity", "", "CORTINA_1_N106", "zones"),
+ ("carbon_intensity_forecast", "", None, "zones"),
+ ("locational_marginal_price_day_ahead", "DE", " , ", "pricing nodes"),
+ ],
+ )
+ def test_missing_zones_or_nodes_raises(
+ self, endpoint: str, zones: str, pricing_nodes: str | None, expected_fragment: str
+ ) -> None:
+ result = _driver(zones=zones, pricing_nodes=pricing_nodes).run(endpoint, [])
assert isinstance(result.raised, ValueError)
- assert "zones" in str(result.raised)
+ assert expected_fragment in str(result.raised)
+ assert result.requests == []
+
+
+class TestSnapshotEndpoints:
+ def test_forecast_rows_carry_zone_and_issue_time_from_envelope(self) -> None:
+ result = _driver("DE, SE").run(
+ "carbon_intensity_forecast",
+ [
+ ScriptedResponse(
+ json={
+ "zone": "DE",
+ "updatedAt": "2025-06-15T11:00:00.000Z",
+ "forecast": [
+ {"carbonIntensity": 250, "datetime": "2025-06-15T12:00:00.000Z"},
+ {"carbonIntensity": 240, "datetime": "2025-06-15T13:00:00.000Z"},
+ ],
+ }
+ ),
+ ScriptedResponse(json={"zone": "SE", "updatedAt": "2025-06-15T11:00:00.000Z", "forecast": []}),
+ ],
+ )
+
+ assert result.raised is None
+ assert result.paths == ["/v3/carbon-intensity/forecast", "/v3/carbon-intensity/forecast"]
+ assert result.params("zone") == ["DE", "SE"]
+ assert result.rows == [
+ {
+ "carbonIntensity": 250,
+ "datetime": datetime(2025, 6, 15, 12, 0, 0, tzinfo=UTC),
+ "zone": "DE",
+ "updatedAt": datetime(2025, 6, 15, 11, 0, 0, tzinfo=UTC),
+ },
+ {
+ "carbonIntensity": 240,
+ "datetime": datetime(2025, 6, 15, 13, 0, 0, tzinfo=UTC),
+ "zone": "DE",
+ "updatedAt": datetime(2025, 6, 15, 11, 0, 0, tzinfo=UTC),
+ },
+ ]
+
+ @pytest.mark.parametrize("body", [{"zone": "DE"}, {"zone": "DE", "forecast": None}])
+ def test_forecast_response_without_forecast_list_fails_the_sync(self, body: dict[str, Any]) -> None:
+ result = _driver().run("carbon_intensity_forecast", [ScriptedResponse(json=body)])
+
+ assert isinstance(result.raised, ValueError)
+ assert "forecast list for zone DE" in str(result.raised)
+
+ def test_zones_object_is_flattened_to_one_row_per_zone(self) -> None:
+ result = _driver(zones="").run(
+ "zones",
+ [
+ ScriptedResponse(
+ json={
+ "DK-DK2": {"zoneKey": "DK-DK2", "zoneName": "East Denmark", "zoneParentKey": "DK"},
+ "DE": {"zoneName": "Germany", "zoneParentKey": None},
+ }
+ )
+ ],
+ )
+
+ assert result.raised is None
+ assert result.paths == ["/v3/zones"]
+ assert result.rows == [
+ {"zoneKey": "DK-DK2", "zoneName": "East Denmark", "zoneParentKey": "DK"},
+ {"zoneKey": "DE", "zoneName": "Germany", "zoneParentKey": None},
+ ]
class TestValidateCredentials:
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/electricity_maps/tests/test_electricity_maps_source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/electricity_maps/tests/test_electricity_maps_source.py
index d83008100537..8b098e47ab28 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/electricity_maps/tests/test_electricity_maps_source.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/electricity_maps/tests/test_electricity_maps_source.py
@@ -39,6 +39,31 @@ class TestElectricityMapsCredentialValidation:
assert message is not None and malformed in message
assert network.requests_log == []
+ @pytest.mark.parametrize("pricing_nodes", [None, " , "])
+ def test_rejects_locational_marginal_price_schema_without_pricing_nodes(self, pricing_nodes: str | None) -> None:
+ config = ElectricityMapsSourceConfig(api_token="token", zones="DE", pricing_nodes=pricing_nodes)
+ with scripted_network([]) as network:
+ is_valid, message = ElectricityMapsSource().validate_credentials(
+ config, team_id=1, schema_name="locational_marginal_price_day_ahead"
+ )
+
+ assert is_valid is False
+ assert message is not None and "pricing node" in message
+ assert network.requests_log == []
+
+ @pytest.mark.parametrize(("pricing_nodes", "expected_blocked"), [(None, True), ("CORTINA_1_N106", False)])
+ def test_schema_picker_flags_locational_marginal_price_without_pricing_nodes(
+ self, pricing_nodes: str | None, expected_blocked: bool
+ ) -> None:
+ config = ElectricityMapsSourceConfig(api_token="token", zones="DE", pricing_nodes=pricing_nodes)
+
+ permissions = ElectricityMapsSource().get_endpoint_permissions(
+ config, team_id=1, endpoints=["carbon_intensity", "locational_marginal_price_day_ahead"]
+ )
+
+ assert permissions["carbon_intensity"] is None
+ assert (permissions["locational_marginal_price_day_ahead"] is not None) is expected_blocked
+
def test_delegates_normalized_zones_to_the_api_probe(self) -> None:
config = ElectricityMapsSourceConfig(api_token="token", zones="de, dk-dk1")
with scripted_network([ScriptedResponse(status=200), ScriptedResponse(status=200)]) as network:
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/electricitymaps.py b/products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/electricitymaps.py
index 2e4db4276034..6b73351cd817 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/electricitymaps.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/electricitymaps.py
@@ -9,3 +9,4 @@ class ElectricityMapsSourceConfig(config.Config):
api_token: str
zones: str
history_days: int | None = config.value(converter=config.str_to_optional_int, default_factory=lambda: None)
+ pricing_nodes: str | None = None