:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(warehouse): add coingecko fear and greed chart table (#126932)
Tom Owersd1c7a846805e
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.mdindex ca09f9677603..ab30eee85c14 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md@@ -3057,7 +3057,7 @@ Note: docs.coinapi.io is behind a Cloudflare interstitial and returns 403 to cur## CoinGecko — gaps-Today (20): `asset_platforms`, `coins_categories`, `coins_categories_list`, `coins_list`, `coins_market_chart`, `coins_markets`, `coins_ohlc`, `coins_tickers`, `derivatives_exchanges`, `derivatives_tickers`, `exchange_rates`, `exchanges`, `exchanges_list`, `global_market_cap_chart`, `global_market_data`, `insights`, `nfts_markets`, `rwas_issuers_list`, `rwas_list`, `rwas_markets`+Today (21): `asset_platforms`, `coins_categories`, `coins_categories_list`, `coins_list`, `coins_market_chart`, `coins_markets`, `coins_ohlc`, `coins_tickers`, `derivatives_exchanges`, `derivatives_tickers`, `exchange_rates`, `exchanges`, `exchanges_list`, `global_fear_and_greed_chart`, `global_market_cap_chart`, `global_market_data`, `insights`, `nfts_markets`, `rwas_issuers_list`, `rwas_list`, `rwas_markets`Diffed against: <https://docs.coingecko.com/llms.txt>@@ -3065,6 +3065,7 @@ Diffed against: <https://docs.coingecko.com/llms.txt>- [x] `/coins/{id}/ohlc/range` — OHLC candles per coin over an arbitrary range, needed for any price analysis (high)- [x] `/global` — total crypto market cap, active coins and BTC dominance - CoinGecko's flagship market-wide metric (high)- [x] `/global/market_cap_chart` — historical global market cap and volume, the time series behind the above (high)+- [x] `/global/fear_and_greed_chart` — daily Fear & Greed index history from 2024-08-14 onwards (Pro, Analyst plan and above) (medium)- [x] `/coins/{id}/tickers` — market pairs per coin on CEX and DEX - the join that resolves coins_list against exchanges_list, which we sync but cannot currently link (high)- [ ] `/exchanges/{id}/tickers` — trading pairs per exchange; the other half of the coin-to-exchange lookup (medium)- [ ] `/exchanges/{id}/volume_chart/range` — historical exchange volume for ranking exchanges over time (medium)diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/coingecko/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/coingecko/canonical_descriptions.pyindex 6ef97a4f9a6c..9a1f9fe4519c 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/coingecko/canonical_descriptions.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/coingecko/canonical_descriptions.py@@ -177,6 +177,15 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {"volume": "Total trading volume over the 24 hours to that point, in USD.",},},+ "global_fear_and_greed_chart": {+ "description": "Daily history of the crypto Fear & Greed index, from 14 August 2024 onwards. The current value is on global_market_data. Requires a Pro API key on the Analyst plan or above.",+ "docs_url": "https://docs.coingecko.com/reference/global-fear-and-greed-chart",+ "columns": {+ "timestamp": "Day of the data point, at 00:00 UTC.",+ "value": "Fear & Greed index value from 0 (extreme fear) to 100 (extreme greed).",+ "sentiment": "Sentiment band for the value: `extreme_fear` (0-30), `fear` (31-45), `neutral` (46-55), `greed` (56-70), or `extreme_greed` (71-100).",+ },+ },"exchange_rates": {"description": "Exchange rate of 1 BTC against every currency CoinGecko tracks, covering fiat, commodities and other crypto. Use it to convert the BTC-denominated columns on the derivatives and exchange tables into another currency.","docs_url": "https://docs.coingecko.com/reference/exchange-rates",diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/coingecko/coingecko.py b/products/warehouse_sources/backend/temporal/data_imports/sources/coingecko/coingecko.pyindex e9586ecfdf77..7ec8f6b7959f 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/coingecko/coingecko.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/coingecko/coingecko.py@@ -11,6 +11,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.coingecko.COINGECKO_ENDPOINTS,DEFAULT_HISTORY_DAYS,EXCHANGE_RATES_ENDPOINT,+ FEAR_AND_GREED_CHART_ENDPOINT,GLOBAL_CHART_DAYS_OPTIONS,GLOBAL_MARKET_CAP_CHART_ENDPOINT,MARKET_CHART_ENDPOINT,@@ -267,6 +268,23 @@ def _global_market_cap_chart_rows(page: list[Any]) -> list[dict[str, Any]]:return _zip_series(body, _GLOBAL_CHART_SERIES, {})+def _fear_and_greed_rows(page: list[Any]) -> list[dict[str, Any]]:+ """Map each ``[timestamp, value, sentiment]`` point onto a row.++ The ``fear_and_greed`` envelope holds a bare array of points, which is the page itself."""+ rows: list[dict[str, Any]] = []+ for point in page:+ if not isinstance(point, list):+ raise ValueError("Required a fear and greed array of points. The API response shape may have changed.")+ if len(point) < 3:+ continue+ timestamp = _from_millis(point[0])+ if timestamp is None:+ continue+ rows.append({"timestamp": timestamp, "value": point[1], "sentiment": point[2]})+ return sorted(rows, key=lambda row: row["timestamp"])++def _exchange_rate_rows(page: list[Any]) -> list[dict[str, Any]]:"""Flatten the ``rates`` object, which is keyed by currency code rather than being a row list,into a row per currency with the code as its id."""@@ -316,6 +334,7 @@ _CHART_ROW_BUILDERS: dict[str, Callable[[list[Any], str], list[dict[str, Any]]]]# that reshapes it. Membership also routes them away from the declarative top-level path._TOP_LEVEL_ROW_BUILDERS: dict[str, Callable[[list[Any]], list[dict[str, Any]]]] = {GLOBAL_MARKET_CAP_CHART_ENDPOINT: _global_market_cap_chart_rows,+ FEAR_AND_GREED_CHART_ENDPOINT: _fear_and_greed_rows,EXCHANGE_RATES_ENDPOINT: _exchange_rate_rows,}diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/coingecko/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/coingecko/settings.pyindex ea87e18d5fa9..40808a7dd06f 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/coingecko/settings.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/coingecko/settings.py@@ -17,10 +17,12 @@ OHLC_ENDPOINT = "coins_ohlc"# string.EXCHANGE_RATES_ENDPOINT = "exchange_rates"GLOBAL_MARKET_CAP_CHART_ENDPOINT = "global_market_cap_chart"+FEAR_AND_GREED_CHART_ENDPOINT = "global_fear_and_greed_chart"# /global/market_cap_chart takes a relative `days` window from a fixed enum rather than a from/to# range. `days=1` is served at hourly granularity and every larger value at daily, so 1 is left out:# a shorter window would return hourly points that don't line up with the daily rows already synced.+# /global/fear_and_greed_chart takes the same enum and is always daily, so it reuses these options.GLOBAL_CHART_DAYS_OPTIONS = ("7", "14", "30", "90", "180", "365", "max")# /coins/{id}/tickers is fixed at 100 items per page, unlike the 250 the other list endpoints take.@@ -193,6 +195,17 @@ COINGECKO_ENDPOINTS: dict[str, CoinGeckoEndpointConfig] = {extra_params={"vs_currency": "usd"},pro_only=True,),+ # Daily Fear & Greed index history, from 2024-08-14 onwards. A bare array of+ # [timestamp, value, sentiment] points under a `fear_and_greed` envelope, mapped onto a row per+ # day. Requires a Pro key on the Analyst plan or above.+ FEAR_AND_GREED_CHART_ENDPOINT: CoinGeckoEndpointConfig(+ name=FEAR_AND_GREED_CHART_ENDPOINT,+ path="/global/fear_and_greed_chart",+ data_selector="fear_and_greed",+ date_field="timestamp",+ primary_keys=["timestamp"],+ pro_only=True,+ ),# BTC-to-currency rates, which normalise the BTC-denominated columns on the derivatives and# exchanges tables into any other currency. One object keyed by currency code under a `rates`# envelope, flattened into a row per currency.diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/coingecko/tests/test_coingecko.py b/products/warehouse_sources/backend/temporal/data_imports/sources/coingecko/tests/test_coingecko.pyindex fd12837f7022..5babd5cf1dd7 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/coingecko/tests/test_coingecko.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/coingecko/tests/test_coingecko.py@@ -390,6 +390,31 @@ class TestGlobalMarketCapChart:assert result.params("days") == [expected_days]+class TestFearAndGreedChart:+ def test_maps_each_point_onto_a_row_in_ascending_order(self) -> None:+ result = _driver(plan=PLAN_PRO).run(+ "global_fear_and_greed_chart",+ [+ ScriptedResponse(+ json={+ "fear_and_greed": [+ [1792368000000, 72, "extreme_greed"],+ [1792281600000, 44, "fear"],+ ["not-a-timestamp", 50, "neutral"],+ ]+ }+ )+ ],+ )++ assert result.raised is None+ assert result.rows == [+ {"timestamp": datetime.fromtimestamp(1792281600, tz=UTC), "value": 44, "sentiment": "fear"},+ {"timestamp": datetime.fromtimestamp(1792368000, tz=UTC), "value": 72, "sentiment": "extreme_greed"},+ ]+ assert result.params("days") == ["max"]++class TestReshapedEndpointsFailLoud:@parameterized.expand([@@ -402,6 +427,12 @@ class TestReshapedEndpointsFailLoud:{"market_cap_chart": []},"global market cap chart object",),+ (+ "fear and greed, wrong inner type",+ "global_fear_and_greed_chart",+ {"fear_and_greed": {"unexpected": "object"}},+ "fear and greed array",+ ),])def test_unexpected_shape_fails_loud(self, _name: str, endpoint: str, body: Any, expected_message: str) -> None: