:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(warehouse): add open exchange rates time series and ohlc tables (#123355)
Tom Owers79475db324db
diff --git a/posthog/test/repo_invariants/dataclass_frozen_baseline.txt b/posthog/test/repo_invariants/dataclass_frozen_baseline.txtindex 94996d749eea..bcc0f7137bfb 100644--- a/posthog/test/repo_invariants/dataclass_frozen_baseline.txt+++ b/posthog/test/repo_invariants/dataclass_frozen_baseline.txt@@ -812,8 +812,6 @@1 products/warehouse_sources/backend/temporal/data_imports/sources/onepagecrm/onepagecrm.py1 products/warehouse_sources/backend/temporal/data_imports/sources/onepassword/onepassword.py1 products/warehouse_sources/backend/temporal/data_imports/sources/onepassword/settings.py-1 products/warehouse_sources/backend/temporal/data_imports/sources/open_exchange_rates/open_exchange_rates.py-1 products/warehouse_sources/backend/temporal/data_imports/sources/open_exchange_rates/settings.py1 products/warehouse_sources/backend/temporal/data_imports/sources/open_meteo/open_meteo.py1 products/warehouse_sources/backend/temporal/data_imports/sources/openai/openai.py1 products/warehouse_sources/backend/temporal/data_imports/sources/openai/settings.pydiff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.mdindex db9c02320663..28bbde2700f8 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md@@ -8795,12 +8795,12 @@ Note: Parsed every GET operation in the official API reference, including nested## OpenExchangeRates — gaps-Today (4): `currencies`, `historical`, `latest`, `usage`+Today (6): `currencies`, `historical`, `latest`, `ohlc`, `time_series`, `usage`Diffed against: <https://docs.openexchangerates.org/reference/api-introduction>-- [ ] `time-series.json` — pulls a whole date range in one call instead of day-by-day historical requests (high)-- [ ] `ohlc.json` — open/high/low/close/average per period - the vendor's analytical rate summary (medium)+- [x] `time-series.json` — pulls a whole date range in one call instead of day-by-day historical requests (high)+- [x] `ohlc.json` — open/high/low/close/average per period - the vendor's analytical rate summary (medium)Note: Small API. Remaining endpoints are convert (a single-value calculator, not a table), plus bid-ask and alternative-currencies which are request options on latest/historical rather than separate resources.diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/open_exchange_rates/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/open_exchange_rates/canonical_descriptions.pyindex ddf673c8c4b6..88274372b980 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/open_exchange_rates/canonical_descriptions.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/open_exchange_rates/canonical_descriptions.py@@ -35,6 +35,32 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {"timestamp": "Unix timestamp of when the rates for that day were published.",},},+ "time_series": {+ "description": "Daily historical exchange rates per currency, one row per (base, currency, date). Fetched from the /time-series.json endpoint one month at a time. Requires an Enterprise or Unlimited plan.",+ "docs_url": "https://docs.openexchangerates.org/reference/time-series-json",+ "columns": {+ "base": "Base currency the rates are quoted against.",+ "currency": "Three-letter ISO 4217 code of the quoted currency.",+ "rate": "Exchange rate for that day: units of `currency` per one unit of `base`.",+ "date": "Value date of the rate (YYYY-MM-DD).",+ },+ },+ "ohlc": {+ "description": "Daily open, high, low, close and average exchange rates per currency, one row per (base, currency, date). Fetched from the /ohlc.json endpoint with a one-day period. Requires a VIP Platinum plan.",+ "docs_url": "https://docs.openexchangerates.org/reference/ohlc-json",+ "columns": {+ "base": "Base currency the rates are quoted against.",+ "currency": "Three-letter ISO 4217 code of the quoted currency.",+ "date": "Day the period covers (YYYY-MM-DD, UTC).",+ "start_time": "ISO 8601 UTC timestamp of the start of the period, which the open rate is recorded at.",+ "end_time": "ISO 8601 UTC timestamp of the end of the period (inclusive), which the close rate is recorded at.",+ "open": "Midpoint rate recorded at the start of the period.",+ "high": "Highest midpoint rate recorded during the period.",+ "low": "Lowest midpoint rate recorded during the period.",+ "close": "Midpoint rate recorded at the end of the period.",+ "average": "Time-weighted average midpoint rate for the whole period.",+ },+ },"usage": {"description": "Your Open Exchange Rates account plan and current-period request usage, as a single row.","docs_url": "https://docs.openexchangerates.org/reference/usage-json",diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/open_exchange_rates/open_exchange_rates.py b/products/warehouse_sources/backend/temporal/data_imports/sources/open_exchange_rates/open_exchange_rates.pyindex a9cb79b1858c..ea1808b48550 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/open_exchange_rates/open_exchange_rates.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/open_exchange_rates/open_exchange_rates.py@@ -8,6 +8,7 @@ import requestsfrom structlog.types import FilteringBoundLoggerfrom tenacity import retry_if_exception_type, stop_after_attempt, wait_exponential_jitter+from posthog.dataclasses import frozenfrom posthog.exceptions_capture import capture_exceptionfrom products.warehouse_sources.backend.temporal.data_imports.sources.common.http import (@@ -29,6 +30,11 @@ MAX_RETRY_ATTEMPTS = 6# stays deliberately small to avoid burning a free-tier monthly request quota (1000/month) in one run.DEFAULT_LOOKBACK_DAYS = 30+# The OHLC endpoint has no data before this day and rejects an earlier `start_time`.+OHLC_EARLIEST_DATE = date(2016, 12, 19)+# Historical rates go back to this day; time-series.json answers 400 `not_available` for earlier windows.+TIME_SERIES_EARLIEST_DATE = date(1999, 1, 1)+class OpenExchangeRatesRetryableError(Exception):"""Raised for transient upstream failures (5xx) that are worth retrying."""@@ -36,11 +42,11 @@ class OpenExchangeRatesRetryableError(Exception):pass[email protected](frozen=False)class OpenExchangeRatesResumeConfig:- # ISO date (YYYY-MM-DD) of the next `historical` day to fetch. Lets a multi-day backfill resume- # mid-sync after a heartbeat timeout instead of restarting from the first day. Unused for the- # single-response currencies/latest/usage endpoints.+ # ISO date (YYYY-MM-DD) of the next `historical`/`ohlc` day or `time_series` window to fetch. Lets+ # a multi-day backfill resume mid-sync after a heartbeat timeout instead of restarting from the+ # first day. Unused for the single-response currencies/latest/usage endpoints.next_date: str | None = None@@ -143,6 +149,41 @@ def _iter_rates(data: dict[str, Any], value_date: str) -> list[dict[str, Any]]:]+def _iter_time_series(data: dict[str, Any]) -> list[dict[str, Any]]:+ # time-series.json nests {date: {currency: rate}} under `rates` and has no timestamp. Sort by+ # date so rows reach the pipeline in ascending order of the incremental field.+ base = data["base"]+ rates_by_date = data.get("rates") or {}+ return [+ {"base": base, "currency": currency, "rate": rate, "date": value_date}+ for value_date in sorted(rates_by_date)+ for currency, rate in (rates_by_date[value_date] or {}).items()+ ]+++def _iter_ohlc(data: dict[str, Any], value_date: str) -> list[dict[str, Any]]:+ base = data["base"]+ start_time = data.get("start_time")+ end_time = data.get("end_time")+ rates = data.get("rates") or {}+ return [+ {+ "base": base,+ "currency": currency,+ "date": value_date,+ "start_time": start_time,+ "end_time": end_time,+ "open": values.get("open"),+ "high": values.get("high"),+ "low": values.get("low"),+ "close": values.get("close"),+ "average": values.get("average"),+ }+ for currency, values in rates.items()+ if values+ ]++def _iter_usage(data: dict[str, Any]) -> list[dict[str, Any]]:# usage.json wraps the account details under `data`; flatten the plan + usage blocks into one row.payload = data.get("data") or {}@@ -197,6 +238,23 @@ def _date_range(start: date, end: date) -> list[date]:return [start + timedelta(days=offset) for offset in range((end - start).days + 1)]+@frozen+class _DateWindow:+ start: date+ end: date+++def _month_windows(start: date, end: date) -> list[_DateWindow]:+ windows: list[_DateWindow] = []+ current = start+ while current <= end:+ next_month = (current.replace(day=1) + timedelta(days=32)).replace(day=1)+ window_end = min(next_month - timedelta(days=1), end)+ windows.append(_DateWindow(start=current, end=window_end))+ current = next_month+ return windows++def _resolve_historical_start(should_use_incremental_field: bool,db_incremental_field_last_value: Any,@@ -288,6 +346,67 @@ def get_rows()return+ if endpoint == "time_series":+ today = datetime.now(tz=UTC).date()+ end = today - timedelta(days=1)+ start = max(+ _resolve_historical_start(should_use_incremental_field, db_incremental_field_last_value, start_date, today),+ TIME_SERIES_EARLIEST_DATE,+ )+ resume = resumable_source_manager.load_state() if resumable_source_manager.can_resume() else None+ resume_date = _to_date(resume.next_date) if resume is not None else None+ if resume_date is not None and resume_date > start:+ start = resume_date+ logger.debug(f"Open Exchange Rates: resuming time_series from {resume_date.isoformat()}")++ windows = _month_windows(start, end)+ for index, window in enumerate(windows):+ data = fetch(+ "time-series.json",+ {"start": window.start.isoformat(), "end": window.end.isoformat(), "base": base},+ )+ rows = _iter_time_series(data)+ if index + 1 < len(windows):+ resumable_source_manager.save_state(+ OpenExchangeRatesResumeConfig(next_date=windows[index + 1].start.isoformat())+ )+ if rows:+ yield rows+ else:+ resumable_source_manager.safe_point()+ return++ if endpoint == "ohlc":+ today = datetime.now(tz=UTC).date()+ # A period may not end in the future, so the last complete daily period is yesterday.+ end = today - timedelta(days=1)+ start = max(+ _resolve_historical_start(should_use_incremental_field, db_incremental_field_last_value, start_date, today),+ OHLC_EARLIEST_DATE,+ )+ days = _date_range(start, end)++ resume = resumable_source_manager.load_state() if resumable_source_manager.can_resume() else None+ if resume is not None and resume.next_date:+ days = [day for day in days if day.isoformat() >= resume.next_date]+ logger.debug(f"Open Exchange Rates: resuming ohlc from {resume.next_date}")++ for index, value_date in enumerate(days):+ data = fetch(+ "ohlc.json",+ {"start_time": f"{value_date.isoformat()}T00:00:00Z", "period": "1d", "base": base},+ )+ rows = _iter_ohlc(data, value_date.isoformat())+ if index + 1 < len(days):+ resumable_source_manager.save_state(+ OpenExchangeRatesResumeConfig(next_date=days[index + 1].isoformat())+ )+ if rows:+ yield rows+ else:+ resumable_source_manager.safe_point()+ return+raise ValueError(f"Unknown Open Exchange Rates endpoint: {endpoint}")diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/open_exchange_rates/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/open_exchange_rates/settings.pyindex d297bc4e8ac4..fdd44258c537 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/open_exchange_rates/settings.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/open_exchange_rates/settings.py@@ -7,7 +7,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.typfrom products.warehouse_sources.backend.types import IncrementalField, IncrementalFieldType-@dataclass+@dataclass(frozen=False)class OpenExchangeRatesEndpointConfig:name: str# Fields the rows are merged on. The rate endpoints carry one row per (base, currency, date), so@@ -15,8 +15,8 @@ class OpenExchangeRatesEndpointConfig:# currencies.primary_keys: list[str]incremental_fields: list[IncrementalField] = field(default_factory=list)- # Only set on `historical`, which selects a single value date server-side (via the URL path), so- # an incremental sync only fetches days newer than the watermark rather than the whole history.+ # Only set on the dated rate endpoints, which select their value dates server-side, so an+ # incremental sync only fetches days newer than the watermark rather than the whole history.supports_incremental: bool = False# `date` is the rate's value date — historical rates never change, so it's a stable partition key.partition_keys: list[str] | None = None@@ -66,6 +66,35 @@ OPEN_EXCHANGE_RATES_ENDPOINTS: dict[str, OpenExchangeRatesEndpointConfig] = {partition_format="month",description="Daily historical rates per currency, one request per day (walked from the start date). Supports incremental sync on the value date.",),+ # /time-series.json — daily rates for a date range in one request, normalized to the same+ # (base, currency, date) rows as `historical`. The vendor caps a request at one month, so the walk+ # requests one calendar month at a time. Each day still counts as one request toward the quota.+ # Only available on the Enterprise and Unlimited plans, so it is off by default.+ "time_series": OpenExchangeRatesEndpointConfig(+ name="time_series",+ primary_keys=["base", "currency", "date"],+ incremental_fields=[_DATE_INCREMENTAL_FIELD],+ supports_incremental=True,+ partition_keys=["date"],+ partition_mode="datetime",+ partition_format="month",+ should_sync_default=False,+ description="Daily historical rates per currency, fetched one month per request. Needs an Enterprise or Unlimited plan. Supports incremental sync on the value date.",+ ),+ # /ohlc.json — open/high/low/close/average rates for one period. Walked one request per day with+ # `period=1d`, from the start date (or the incremental watermark) up to yesterday, because a+ # period may not end in the future. Only available on the VIP Platinum plan, so it is off by default.+ "ohlc": OpenExchangeRatesEndpointConfig(+ name="ohlc",+ primary_keys=["base", "currency", "date"],+ incremental_fields=[_DATE_INCREMENTAL_FIELD],+ supports_incremental=True,+ partition_keys=["date"],+ partition_mode="datetime",+ partition_format="month",+ should_sync_default=False,+ description="Daily open, high, low, close and average rates per currency, one request per day. Needs a VIP Platinum plan. Supports incremental sync on the value date.",+ ),# /usage.json — the account's plan and current-period request usage as a single flattened row.# Free and does not count toward the monthly request quota. Keyed on a constant synthetic id# (`id`) rather than the account's app_id, which is the API credential and must not be persisted.diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/open_exchange_rates/tests/test_open_exchange_rates.py b/products/warehouse_sources/backend/temporal/data_imports/sources/open_exchange_rates/tests/test_open_exchange_rates.pyindex 16eb28bc880e..f9680788d972 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/open_exchange_rates/tests/test_open_exchange_rates.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/open_exchange_rates/tests/test_open_exchange_rates.py@@ -274,6 +274,80 @@ class TestGetRows:assert result.urls == [f"{BASE_URL}/historical/{days[1].isoformat()}.json?base=USD"]assert [batch[0]["date"] for batch in result.items] == [days[1].isoformat()]+ def test_time_series_walks_calendar_months_in_date_order(self) -> None:+ yesterday = datetime.now(tz=UTC).date() - timedelta(days=1)+ current_month = yesterday.replace(day=1)+ previous_month = (current_month - timedelta(days=1)).replace(day=1)+ previous_month_end = current_month - timedelta(days=1)+ body1 = {+ "base": "USD",+ "rates": {+ previous_month_end.isoformat(): {"EUR": 0.91},+ previous_month.isoformat(): {"EUR": 0.9, "GBP": 0.8},+ },+ }+ body2 = {"base": "USD", "rates": {yesterday.isoformat(): {"EUR": 0.95}}}+ result = self._driver(previous_month.isoformat()).run(+ "time_series", [ScriptedResponse(json=body1), ScriptedResponse(json=body2)]+ )+ assert result.raised is None+ assert result.paths == ["/api/time-series.json", "/api/time-series.json"]+ assert result.params("start") == [previous_month.isoformat(), current_month.isoformat()]+ assert result.params("end") == [previous_month_end.isoformat(), yesterday.isoformat()]+ assert result.items == [+ [+ {"base": "USD", "currency": "EUR", "rate": 0.9, "date": previous_month.isoformat()},+ {"base": "USD", "currency": "GBP", "rate": 0.8, "date": previous_month.isoformat()},+ {"base": "USD", "currency": "EUR", "rate": 0.91, "date": previous_month_end.isoformat()},+ ],+ [{"base": "USD", "currency": "EUR", "rate": 0.95, "date": yesterday.isoformat()}],+ ]+ assert result.saved_states == [OpenExchangeRatesResumeConfig(next_date=current_month.isoformat())]++ def test_time_series_resumes_from_saved_window(self) -> None:+ yesterday = datetime.now(tz=UTC).date() - timedelta(days=1)+ current_month = yesterday.replace(day=1)+ previous_month = (current_month - timedelta(days=1)).replace(day=1)+ body = {"base": "USD", "rates": {yesterday.isoformat(): {"EUR": 0.95}}}+ result = self._driver(previous_month.isoformat()).run(+ "time_series",+ [ScriptedResponse(json=body)],+ resume_state=OpenExchangeRatesResumeConfig(next_date=current_month.isoformat()),+ )+ assert result.raised is None+ assert result.params("start") == [current_month.isoformat()]+ assert [batch[0]["date"] for batch in result.items] == [yesterday.isoformat()]++ def test_ohlc_walks_each_day_with_a_daily_period(self) -> None:+ yesterday = datetime.now(tz=UTC).date() - timedelta(days=1)+ days = [yesterday - timedelta(days=1), yesterday]+ values = {"open": 0.9, "high": 0.92, "low": 0.89, "close": 0.91, "average": 0.905}+ bodies = [+ {+ "base": "USD",+ "start_time": f"{day.isoformat()}T00:00:00Z",+ "end_time": f"{day.isoformat()}T23:59:59Z",+ "rates": {"EUR": values, "GBP": None},+ }+ for day in days+ ]+ result = self._driver(days[0].isoformat()).run("ohlc", [ScriptedResponse(json=body) for body in bodies])+ assert result.raised is None+ assert result.params("start_time") == [f"{day.isoformat()}T00:00:00Z" for day in days]+ assert result.params("period") == ["1d", "1d"]+ assert result.items[0] == [+ {+ "base": "USD",+ "currency": "EUR",+ "date": days[0].isoformat(),+ "start_time": f"{days[0].isoformat()}T00:00:00Z",+ "end_time": f"{days[0].isoformat()}T23:59:59Z",+ **values,+ }+ ]+ assert [batch[0]["date"] for batch in result.items] == [day.isoformat() for day in days]+ assert result.saved_states == [OpenExchangeRatesResumeConfig(next_date=days[1].isoformat())]+def test_unknown_endpoint_raises(self) -> None:with pytest.raises(ValueError):self._run("nope", [], _manager())@@ -292,6 +366,10 @@ class TestOpenExchangeRatesSourceResponse:return ScriptedResponse(json={"USD": "US Dollar"})if endpoint == "usage":return ScriptedResponse(json={"data": {}})+ if endpoint == "time_series":+ return ScriptedResponse(json={"base": "USD", "rates": {"2024-01-02": {"EUR": 0.9}}})+ if endpoint == "ohlc":+ return ScriptedResponse(json={"base": "USD", "rates": {"EUR": {"open": 0.9}}})return ScriptedResponse(json={"base": "USD", "timestamp": 1704153600, "rates": {"EUR": 0.9}})@parameterized.expand(@@ -299,6 +377,8 @@ class TestOpenExchangeRatesSourceResponse:("currencies", ["code"]),("latest", ["base", "currency", "date"]),("historical", ["base", "currency", "date"]),+ ("time_series", ["base", "currency", "date"]),+ ("ohlc", ["base", "currency", "date"]),("usage", ["id"]),])@@ -311,7 +391,7 @@ class TestOpenExchangeRatesSourceResponse:assert response.primary_keys == expected_keysassert response.sort_mode == "asc"- @parameterized.expand([("latest",), ("historical",)])+ @parameterized.expand([("latest",), ("historical",), ("time_series",), ("ohlc",)])def test_rate_endpoints_partition_on_stable_date(self, endpoint: str) -> None:result = self._run(endpoint).run(endpoint, always(self._answer(endpoint)))assert result.raised is None