Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(warehouse): add open exchange rates time series and ohlc tables (#123355)

Tom Owers79475db324db
diff --git a/posthog/test/repo_invariants/dataclass_frozen_baseline.txt b/posthog/test/repo_invariants/dataclass_frozen_baseline.txt
index 94996d749eea..bcc0f7137bfb 100644
--- a/posthog/test/repo_invariants/dataclass_frozen_baseline.txt
+++ b/posthog/test/repo_invariants/dataclass_frozen_baseline.txt
@@ -812,8 +812,6 @@
1 products/warehouse_sources/backend/temporal/data_imports/sources/onepagecrm/onepagecrm.py
1 products/warehouse_sources/backend/temporal/data_imports/sources/onepassword/onepassword.py
1 products/warehouse_sources/backend/temporal/data_imports/sources/onepassword/settings.py
-1 products/warehouse_sources/backend/temporal/data_imports/sources/open_exchange_rates/open_exchange_rates.py
-1 products/warehouse_sources/backend/temporal/data_imports/sources/open_exchange_rates/settings.py
1 products/warehouse_sources/backend/temporal/data_imports/sources/open_meteo/open_meteo.py
1 products/warehouse_sources/backend/temporal/data_imports/sources/openai/openai.py
1 products/warehouse_sources/backend/temporal/data_imports/sources/openai/settings.py
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
index db9c02320663..28bbde2700f8 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
@@ -8795,12 +8795,12 @@ Note: Parsed every GET operation in the official API reference, including nested
## OpenExchangeRates — gaps
-Today (4): `currencies`, `historical`, `latest`, `usage`
+Today (6): `currencies`, `historical`, `latest`, `ohlc`, `time_series`, `usage`
Diffed against: <https://docs.openexchangerates.org/reference/api-introduction>
-- [ ] `time-series.json` — pulls a whole date range in one call instead of day-by-day historical requests (high)
-- [ ] `ohlc.json` — open/high/low/close/average per period - the vendor's analytical rate summary (medium)
+- [x] `time-series.json` — pulls a whole date range in one call instead of day-by-day historical requests (high)
+- [x] `ohlc.json` — open/high/low/close/average per period - the vendor's analytical rate summary (medium)
Note: Small API. Remaining endpoints are convert (a single-value calculator, not a table), plus bid-ask and alternative-currencies which are request options on latest/historical rather than separate resources.
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/open_exchange_rates/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/open_exchange_rates/canonical_descriptions.py
index ddf673c8c4b6..88274372b980 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/open_exchange_rates/canonical_descriptions.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/open_exchange_rates/canonical_descriptions.py
@@ -35,6 +35,32 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {
"timestamp": "Unix timestamp of when the rates for that day were published.",
},
},
+ "time_series": {
+ "description": "Daily historical exchange rates per currency, one row per (base, currency, date). Fetched from the /time-series.json endpoint one month at a time. Requires an Enterprise or Unlimited plan.",
+ "docs_url": "https://docs.openexchangerates.org/reference/time-series-json",
+ "columns": {
+ "base": "Base currency the rates are quoted against.",
+ "currency": "Three-letter ISO 4217 code of the quoted currency.",
+ "rate": "Exchange rate for that day: units of `currency` per one unit of `base`.",
+ "date": "Value date of the rate (YYYY-MM-DD).",
+ },
+ },
+ "ohlc": {
+ "description": "Daily open, high, low, close and average exchange rates per currency, one row per (base, currency, date). Fetched from the /ohlc.json endpoint with a one-day period. Requires a VIP Platinum plan.",
+ "docs_url": "https://docs.openexchangerates.org/reference/ohlc-json",
+ "columns": {
+ "base": "Base currency the rates are quoted against.",
+ "currency": "Three-letter ISO 4217 code of the quoted currency.",
+ "date": "Day the period covers (YYYY-MM-DD, UTC).",
+ "start_time": "ISO 8601 UTC timestamp of the start of the period, which the open rate is recorded at.",
+ "end_time": "ISO 8601 UTC timestamp of the end of the period (inclusive), which the close rate is recorded at.",
+ "open": "Midpoint rate recorded at the start of the period.",
+ "high": "Highest midpoint rate recorded during the period.",
+ "low": "Lowest midpoint rate recorded during the period.",
+ "close": "Midpoint rate recorded at the end of the period.",
+ "average": "Time-weighted average midpoint rate for the whole period.",
+ },
+ },
"usage": {
"description": "Your Open Exchange Rates account plan and current-period request usage, as a single row.",
"docs_url": "https://docs.openexchangerates.org/reference/usage-json",
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/open_exchange_rates/open_exchange_rates.py b/products/warehouse_sources/backend/temporal/data_imports/sources/open_exchange_rates/open_exchange_rates.py
index a9cb79b1858c..ea1808b48550 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/open_exchange_rates/open_exchange_rates.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/open_exchange_rates/open_exchange_rates.py
@@ -8,6 +8,7 @@ import requests
from structlog.types import FilteringBoundLogger
from tenacity import retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
+from posthog.dataclasses import frozen
from posthog.exceptions_capture import capture_exception
from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import (
@@ -29,6 +30,11 @@ MAX_RETRY_ATTEMPTS = 6
# stays deliberately small to avoid burning a free-tier monthly request quota (1000/month) in one run.
DEFAULT_LOOKBACK_DAYS = 30
+# The OHLC endpoint has no data before this day and rejects an earlier `start_time`.
+OHLC_EARLIEST_DATE = date(2016, 12, 19)
+# Historical rates go back to this day; time-series.json answers 400 `not_available` for earlier windows.
+TIME_SERIES_EARLIEST_DATE = date(1999, 1, 1)
+
class OpenExchangeRatesRetryableError(Exception):
"""Raised for transient upstream failures (5xx) that are worth retrying."""
@@ -36,11 +42,11 @@ class OpenExchangeRatesRetryableError(Exception):
pass
[email protected](frozen=False)
class OpenExchangeRatesResumeConfig:
- # ISO date (YYYY-MM-DD) of the next `historical` day to fetch. Lets a multi-day backfill resume
- # mid-sync after a heartbeat timeout instead of restarting from the first day. Unused for the
- # single-response currencies/latest/usage endpoints.
+ # ISO date (YYYY-MM-DD) of the next `historical`/`ohlc` day or `time_series` window to fetch. Lets
+ # a multi-day backfill resume mid-sync after a heartbeat timeout instead of restarting from the
+ # first day. Unused for the single-response currencies/latest/usage endpoints.
next_date: str | None = None
@@ -143,6 +149,41 @@ def _iter_rates(data: dict[str, Any], value_date: str) -> list[dict[str, Any]]:
]
+def _iter_time_series(data: dict[str, Any]) -> list[dict[str, Any]]:
+ # time-series.json nests {date: {currency: rate}} under `rates` and has no timestamp. Sort by
+ # date so rows reach the pipeline in ascending order of the incremental field.
+ base = data["base"]
+ rates_by_date = data.get("rates") or {}
+ return [
+ {"base": base, "currency": currency, "rate": rate, "date": value_date}
+ for value_date in sorted(rates_by_date)
+ for currency, rate in (rates_by_date[value_date] or {}).items()
+ ]
+
+
+def _iter_ohlc(data: dict[str, Any], value_date: str) -> list[dict[str, Any]]:
+ base = data["base"]
+ start_time = data.get("start_time")
+ end_time = data.get("end_time")
+ rates = data.get("rates") or {}
+ return [
+ {
+ "base": base,
+ "currency": currency,
+ "date": value_date,
+ "start_time": start_time,
+ "end_time": end_time,
+ "open": values.get("open"),
+ "high": values.get("high"),
+ "low": values.get("low"),
+ "close": values.get("close"),
+ "average": values.get("average"),
+ }
+ for currency, values in rates.items()
+ if values
+ ]
+
+
def _iter_usage(data: dict[str, Any]) -> list[dict[str, Any]]:
# usage.json wraps the account details under `data`; flatten the plan + usage blocks into one row.
payload = data.get("data") or {}
@@ -197,6 +238,23 @@ def _date_range(start: date, end: date) -> list[date]:
return [start + timedelta(days=offset) for offset in range((end - start).days + 1)]
+@frozen
+class _DateWindow:
+ start: date
+ end: date
+
+
+def _month_windows(start: date, end: date) -> list[_DateWindow]:
+ windows: list[_DateWindow] = []
+ current = start
+ while current <= end:
+ next_month = (current.replace(day=1) + timedelta(days=32)).replace(day=1)
+ window_end = min(next_month - timedelta(days=1), end)
+ windows.append(_DateWindow(start=current, end=window_end))
+ current = next_month
+ return windows
+
+
def _resolve_historical_start(
should_use_incremental_field: bool,
db_incremental_field_last_value: Any,
@@ -288,6 +346,67 @@ def get_rows(
)
return
+ if endpoint == "time_series":
+ today = datetime.now(tz=UTC).date()
+ end = today - timedelta(days=1)
+ start = max(
+ _resolve_historical_start(should_use_incremental_field, db_incremental_field_last_value, start_date, today),
+ TIME_SERIES_EARLIEST_DATE,
+ )
+ resume = resumable_source_manager.load_state() if resumable_source_manager.can_resume() else None
+ resume_date = _to_date(resume.next_date) if resume is not None else None
+ if resume_date is not None and resume_date > start:
+ start = resume_date
+ logger.debug(f"Open Exchange Rates: resuming time_series from {resume_date.isoformat()}")
+
+ windows = _month_windows(start, end)
+ for index, window in enumerate(windows):
+ data = fetch(
+ "time-series.json",
+ {"start": window.start.isoformat(), "end": window.end.isoformat(), "base": base},
+ )
+ rows = _iter_time_series(data)
+ if index + 1 < len(windows):
+ resumable_source_manager.save_state(
+ OpenExchangeRatesResumeConfig(next_date=windows[index + 1].start.isoformat())
+ )
+ if rows:
+ yield rows
+ else:
+ resumable_source_manager.safe_point()
+ return
+
+ if endpoint == "ohlc":
+ today = datetime.now(tz=UTC).date()
+ # A period may not end in the future, so the last complete daily period is yesterday.
+ end = today - timedelta(days=1)
+ start = max(
+ _resolve_historical_start(should_use_incremental_field, db_incremental_field_last_value, start_date, today),
+ OHLC_EARLIEST_DATE,
+ )
+ days = _date_range(start, end)
+
+ resume = resumable_source_manager.load_state() if resumable_source_manager.can_resume() else None
+ if resume is not None and resume.next_date:
+ days = [day for day in days if day.isoformat() >= resume.next_date]
+ logger.debug(f"Open Exchange Rates: resuming ohlc from {resume.next_date}")
+
+ for index, value_date in enumerate(days):
+ data = fetch(
+ "ohlc.json",
+ {"start_time": f"{value_date.isoformat()}T00:00:00Z", "period": "1d", "base": base},
+ )
+ rows = _iter_ohlc(data, value_date.isoformat())
+ if index + 1 < len(days):
+ resumable_source_manager.save_state(
+ OpenExchangeRatesResumeConfig(next_date=days[index + 1].isoformat())
+ )
+ if rows:
+ yield rows
+ else:
+ resumable_source_manager.safe_point()
+ return
+
raise ValueError(f"Unknown Open Exchange Rates endpoint: {endpoint}")
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/open_exchange_rates/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/open_exchange_rates/settings.py
index d297bc4e8ac4..fdd44258c537 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/open_exchange_rates/settings.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/open_exchange_rates/settings.py
@@ -7,7 +7,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.typ
from products.warehouse_sources.backend.types import IncrementalField, IncrementalFieldType
-@dataclass
+@dataclass(frozen=False)
class OpenExchangeRatesEndpointConfig:
name: str
# Fields the rows are merged on. The rate endpoints carry one row per (base, currency, date), so
@@ -15,8 +15,8 @@ class OpenExchangeRatesEndpointConfig:
# currencies.
primary_keys: list[str]
incremental_fields: list[IncrementalField] = field(default_factory=list)
- # Only set on `historical`, which selects a single value date server-side (via the URL path), so
- # an incremental sync only fetches days newer than the watermark rather than the whole history.
+ # Only set on the dated rate endpoints, which select their value dates server-side, so an
+ # incremental sync only fetches days newer than the watermark rather than the whole history.
supports_incremental: bool = False
# `date` is the rate's value date — historical rates never change, so it's a stable partition key.
partition_keys: list[str] | None = None
@@ -66,6 +66,35 @@ OPEN_EXCHANGE_RATES_ENDPOINTS: dict[str, OpenExchangeRatesEndpointConfig] = {
partition_format="month",
description="Daily historical rates per currency, one request per day (walked from the start date). Supports incremental sync on the value date.",
),
+ # /time-series.json — daily rates for a date range in one request, normalized to the same
+ # (base, currency, date) rows as `historical`. The vendor caps a request at one month, so the walk
+ # requests one calendar month at a time. Each day still counts as one request toward the quota.
+ # Only available on the Enterprise and Unlimited plans, so it is off by default.
+ "time_series": OpenExchangeRatesEndpointConfig(
+ name="time_series",
+ primary_keys=["base", "currency", "date"],
+ incremental_fields=[_DATE_INCREMENTAL_FIELD],
+ supports_incremental=True,
+ partition_keys=["date"],
+ partition_mode="datetime",
+ partition_format="month",
+ should_sync_default=False,
+ description="Daily historical rates per currency, fetched one month per request. Needs an Enterprise or Unlimited plan. Supports incremental sync on the value date.",
+ ),
+ # /ohlc.json — open/high/low/close/average rates for one period. Walked one request per day with
+ # `period=1d`, from the start date (or the incremental watermark) up to yesterday, because a
+ # period may not end in the future. Only available on the VIP Platinum plan, so it is off by default.
+ "ohlc": OpenExchangeRatesEndpointConfig(
+ name="ohlc",
+ primary_keys=["base", "currency", "date"],
+ incremental_fields=[_DATE_INCREMENTAL_FIELD],
+ supports_incremental=True,
+ partition_keys=["date"],
+ partition_mode="datetime",
+ partition_format="month",
+ should_sync_default=False,
+ description="Daily open, high, low, close and average rates per currency, one request per day. Needs a VIP Platinum plan. Supports incremental sync on the value date.",
+ ),
# /usage.json — the account's plan and current-period request usage as a single flattened row.
# Free and does not count toward the monthly request quota. Keyed on a constant synthetic id
# (`id`) rather than the account's app_id, which is the API credential and must not be persisted.
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/open_exchange_rates/tests/test_open_exchange_rates.py b/products/warehouse_sources/backend/temporal/data_imports/sources/open_exchange_rates/tests/test_open_exchange_rates.py
index 16eb28bc880e..f9680788d972 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/open_exchange_rates/tests/test_open_exchange_rates.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/open_exchange_rates/tests/test_open_exchange_rates.py
@@ -274,6 +274,80 @@ class TestGetRows:
assert result.urls == [f"{BASE_URL}/historical/{days[1].isoformat()}.json?base=USD"]
assert [batch[0]["date"] for batch in result.items] == [days[1].isoformat()]
+ def test_time_series_walks_calendar_months_in_date_order(self) -> None:
+ yesterday = datetime.now(tz=UTC).date() - timedelta(days=1)
+ current_month = yesterday.replace(day=1)
+ previous_month = (current_month - timedelta(days=1)).replace(day=1)
+ previous_month_end = current_month - timedelta(days=1)
+ body1 = {
+ "base": "USD",
+ "rates": {
+ previous_month_end.isoformat(): {"EUR": 0.91},
+ previous_month.isoformat(): {"EUR": 0.9, "GBP": 0.8},
+ },
+ }
+ body2 = {"base": "USD", "rates": {yesterday.isoformat(): {"EUR": 0.95}}}
+ result = self._driver(previous_month.isoformat()).run(
+ "time_series", [ScriptedResponse(json=body1), ScriptedResponse(json=body2)]
+ )
+ assert result.raised is None
+ assert result.paths == ["/api/time-series.json", "/api/time-series.json"]
+ assert result.params("start") == [previous_month.isoformat(), current_month.isoformat()]
+ assert result.params("end") == [previous_month_end.isoformat(), yesterday.isoformat()]
+ assert result.items == [
+ [
+ {"base": "USD", "currency": "EUR", "rate": 0.9, "date": previous_month.isoformat()},
+ {"base": "USD", "currency": "GBP", "rate": 0.8, "date": previous_month.isoformat()},
+ {"base": "USD", "currency": "EUR", "rate": 0.91, "date": previous_month_end.isoformat()},
+ ],
+ [{"base": "USD", "currency": "EUR", "rate": 0.95, "date": yesterday.isoformat()}],
+ ]
+ assert result.saved_states == [OpenExchangeRatesResumeConfig(next_date=current_month.isoformat())]
+
+ def test_time_series_resumes_from_saved_window(self) -> None:
+ yesterday = datetime.now(tz=UTC).date() - timedelta(days=1)
+ current_month = yesterday.replace(day=1)
+ previous_month = (current_month - timedelta(days=1)).replace(day=1)
+ body = {"base": "USD", "rates": {yesterday.isoformat(): {"EUR": 0.95}}}
+ result = self._driver(previous_month.isoformat()).run(
+ "time_series",
+ [ScriptedResponse(json=body)],
+ resume_state=OpenExchangeRatesResumeConfig(next_date=current_month.isoformat()),
+ )
+ assert result.raised is None
+ assert result.params("start") == [current_month.isoformat()]
+ assert [batch[0]["date"] for batch in result.items] == [yesterday.isoformat()]
+
+ def test_ohlc_walks_each_day_with_a_daily_period(self) -> None:
+ yesterday = datetime.now(tz=UTC).date() - timedelta(days=1)
+ days = [yesterday - timedelta(days=1), yesterday]
+ values = {"open": 0.9, "high": 0.92, "low": 0.89, "close": 0.91, "average": 0.905}
+ bodies = [
+ {
+ "base": "USD",
+ "start_time": f"{day.isoformat()}T00:00:00Z",
+ "end_time": f"{day.isoformat()}T23:59:59Z",
+ "rates": {"EUR": values, "GBP": None},
+ }
+ for day in days
+ ]
+ result = self._driver(days[0].isoformat()).run("ohlc", [ScriptedResponse(json=body) for body in bodies])
+ assert result.raised is None
+ assert result.params("start_time") == [f"{day.isoformat()}T00:00:00Z" for day in days]
+ assert result.params("period") == ["1d", "1d"]
+ assert result.items[0] == [
+ {
+ "base": "USD",
+ "currency": "EUR",
+ "date": days[0].isoformat(),
+ "start_time": f"{days[0].isoformat()}T00:00:00Z",
+ "end_time": f"{days[0].isoformat()}T23:59:59Z",
+ **values,
+ }
+ ]
+ assert [batch[0]["date"] for batch in result.items] == [day.isoformat() for day in days]
+ assert result.saved_states == [OpenExchangeRatesResumeConfig(next_date=days[1].isoformat())]
+
def test_unknown_endpoint_raises(self) -> None:
with pytest.raises(ValueError):
self._run("nope", [], _manager())
@@ -292,6 +366,10 @@ class TestOpenExchangeRatesSourceResponse:
return ScriptedResponse(json={"USD": "US Dollar"})
if endpoint == "usage":
return ScriptedResponse(json={"data": {}})
+ if endpoint == "time_series":
+ return ScriptedResponse(json={"base": "USD", "rates": {"2024-01-02": {"EUR": 0.9}}})
+ if endpoint == "ohlc":
+ return ScriptedResponse(json={"base": "USD", "rates": {"EUR": {"open": 0.9}}})
return ScriptedResponse(json={"base": "USD", "timestamp": 1704153600, "rates": {"EUR": 0.9}})
@parameterized.expand(
@@ -299,6 +377,8 @@ class TestOpenExchangeRatesSourceResponse:
("currencies", ["code"]),
("latest", ["base", "currency", "date"]),
("historical", ["base", "currency", "date"]),
+ ("time_series", ["base", "currency", "date"]),
+ ("ohlc", ["base", "currency", "date"]),
("usage", ["id"]),
]
)
@@ -311,7 +391,7 @@ class TestOpenExchangeRatesSourceResponse:
assert response.primary_keys == expected_keys
assert response.sort_mode == "asc"
- @parameterized.expand([("latest",), ("historical",)])
+ @parameterized.expand([("latest",), ("historical",), ("time_series",), ("ohlc",)])
def test_rate_endpoints_partition_on_stable_date(self, endpoint: str) -> None:
result = self._run(endpoint).run(endpoint, always(self._answer(endpoint)))
assert result.raised is None