:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(warehouse): add coinmarketcap cmc ai tables (#126937)
Tom Owers2ac0c080d746
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.mdindex 763d73aa3cab..ac3b581c310f 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md@@ -3072,7 +3072,7 @@ Note: docs.coingecko.com/reference/\* pages are client-rendered and unparseable## CoinMarketCap — gaps-Today (15): `categories`, `cryptocurrency_info`, `cryptocurrency_map`, `exchange_info`, `exchange_listings_latest`, `exchange_map`, `exchange_quotes_historical`, `fiat_map`, `global_metrics_quotes_historical`, `liquidations_by_cryptocurrency_latest`, `liquidations_by_exchange_latest`, `listings_historical`, `listings_latest`, `ohlcv_historical`, `quotes_historical`+Today (17): `categories`, `cmc_ai_coins_latest`, `cmc_ai_latest`, `cryptocurrency_info`, `cryptocurrency_map`, `exchange_info`, `exchange_listings_latest`, `exchange_map`, `exchange_quotes_historical`, `fiat_map`, `global_metrics_quotes_historical`, `liquidations_by_cryptocurrency_latest`, `liquidations_by_exchange_latest`, `listings_historical`, `listings_latest`, `ohlcv_historical`, `quotes_historical`Diffed against: <https://pro.coinmarketcap.com/api/documentation/pro-api-reference/cryptocurrency.md>@@ -3090,6 +3090,8 @@ Diffed against: <https://pro.coinmarketcap.com/api/documentation/pro-api-referen- [ ] `GET /v3/fear-and-greed/historical` — CMC's proprietary sentiment index over time, a commonly requested signal (medium)- [x] `GET /v5/derivatives/liquidations/cryptocurrency/list/latest` — latest long/short liquidations per coin over rolling 1h/4h/24h windows (medium)- [x] `GET /v5/derivatives/liquidations/exchange/list/latest` — latest long/short liquidations per derivatives exchange over rolling 1h/4h/24h windows (medium)+- [x] `GET /v5/cmc-ai/latest` — market-wide CMC AI questions, answers and top news from the homepage (low)+- [x] `GET /v5/cmc-ai/coins/latest` — per-coin CMC AI questions and answers from Coin Detail Pages (low)Note: coinmarketcap.com/api/documentation is a client-rendered zudoku app, but every page is served as markdown by appending .md, and pro.coinmarketcap.com/llms.txt indexes the whole reference by family (Cryptocurrency 19, Exchange 7, Global Metrics 6, DEX/Token 16, Holder 5, Derivatives 3, RWA 7, CMC Index 4...). The cryptocurrency and global-metrics historical families are now covered, as is the exchange family apart from market pairs, assets and proof-of-reserves. The DEX (Token/Pool/Holder/OHLCV) and Real World Assets families are also entirely uncovered but are treated as separate products rather than enumerated here.diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/coinmarketcap/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/coinmarketcap/canonical_descriptions.pyindex 3c5ac3d69345..fdb32ada12be 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/coinmarketcap/canonical_descriptions.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/coinmarketcap/canonical_descriptions.py@@ -233,4 +233,38 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {"quotes": "One entry per requested currency conversion (default USD only): crypto_id and symbol of the conversion currency, total_liquidations, long_liquidations and short_liquidations for each of the rolling 1h, 4h and 24h windows, and last_updated.",},},+ "cmc_ai_latest": {+ "description": "Current market-wide CMC AI questions, answers and top news shown on the CoinMarketCap homepage. Answers are regenerated about every 30 minutes. Requires an Enterprise plan.",+ "docs_url": "https://coinmarketcap.com/api/documentation/pro-api-reference/cmc-ai/market-feed-latest.md",+ "columns": {+ "type": "Insight type: fixed_question (a stable question template), trending_question (a market-dependent question) or top_news (a headline and recap).",+ "title": "Display text: the question for fixed_question, the headline for trending_question and top_news.",+ "answer_id": "24-character hex id of the generated answer. Changes each time the answer is regenerated.",+ "question_key": "Stable semantic key of the question template for fixed_question, null for trending_question and top_news.",+ "answer": "Generated answer as markdown, split into a tldr section and a body section.",+ "sources": "Source URLs cited by the answer, capped at 100.",+ "sources_count": "Total number of sources for this insight before the cap is applied.",+ "sources_truncated": "True if sources has fewer entries than sources_count.",+ "generated_at": "Timestamp (ISO 8601, UTC) of when the answer was generated.",+ },+ },+ "cmc_ai_coins_latest": {+ "description": "Current CMC AI questions and answers per cryptocurrency, as shown on Coin Detail Pages, one row per insight. Covers the coins CMC AI writes content for, currently the top 100 by market cap. Requires an Enterprise plan.",+ "docs_url": "https://coinmarketcap.com/api/documentation/pro-api-reference/cmc-ai/coin-insights-latest.md",+ "columns": {+ "crypto_id": "The unique CoinMarketCap ID for this cryptocurrency.",+ "name": "The name of this cryptocurrency.",+ "symbol": "The ticker symbol for this cryptocurrency.",+ "slug": "The web URL friendly shorthand version of this cryptocurrency name.",+ "type": "Insight type: fixed_question (a stable question template), trending_question (a market-dependent question) or top_news (a headline and recap).",+ "title": "Display text: the question for fixed_question, the headline for trending_question and top_news.",+ "answer_id": "24-character hex id of the generated answer. Changes each time the answer is regenerated.",+ "question_key": "Stable semantic key of the question template for fixed_question, null for trending_question and top_news.",+ "answer": "Generated answer as markdown, split into a tldr section and a body section.",+ "sources": "Source URLs cited by the answer, capped at 100.",+ "sources_count": "Total number of sources for this insight before the cap is applied.",+ "sources_truncated": "True if sources has fewer entries than sources_count.",+ "generated_at": "Timestamp (ISO 8601, UTC) of when the answer was generated.",+ },+ },}diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/coinmarketcap/coinmarketcap.py b/products/warehouse_sources/backend/temporal/data_imports/sources/coinmarketcap/coinmarketcap.pyindex a1477fb8e1e4..4067f3a8fe63 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/coinmarketcap/coinmarketcap.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/coinmarketcap/coinmarketcap.py@@ -8,6 +8,7 @@ from structlog.types import FilteringBoundLoggerfrom posthog.dataclasses import frozenfrom products.warehouse_sources.backend.temporal.data_imports.sources.coinmarketcap.settings import (+ CMC_AI_PAGE_SIZE,COINMARKETCAP_BATCH_ENDPOINTS,COINMARKETCAP_ENDPOINTS,COINMARKETCAP_SNAPSHOT_ENDPOINTS,@@ -61,9 +62,9 @@ class CoinMarketCapPaginator(OffsetPaginator):condition (`total_path=None`)."""- def __init__(self) -> None:+ def __init__(self, page_size: int = PAGE_SIZE) -> None:super().__init__(- limit=PAGE_SIZE,+ limit=page_size,offset=1, # `start` is 1-based; start=0 is rejected with a 400.offset_param="start",limit_param="limit",@@ -113,17 +114,25 @@ def _get(session: requests.Session, path: str, params: dict[str, Any]) -> dict[sreturn body if isinstance(body, dict) else {}-def _ids_from_map(session: requests.Session, path: str) -> list[int]:+def _ids_from_map(+ session: requests.Session,+ path: str,+ id_field: str = "id",+ page_size: int = PAGE_SIZE,+ data_key: Optional[str] = None,+) -> list[int]:ids: list[int] = []start = 1while True:- page = _get(session, path, {"start": start, "limit": PAGE_SIZE, "sort": "id"})- rows = page.get("data") or []- ids.extend(int(row["id"]) for row in rows if isinstance(row, dict) and row.get("id") is not None)+ page = _get(session, path, {"start": start, "limit": page_size, "sort": id_field})+ data = page.get("data")+ rows = (data.get(data_key) if isinstance(data, dict) else None) if data_key else data+ rows = rows or []+ ids.extend(int(row[id_field]) for row in rows if isinstance(row, dict) and row.get(id_field) is not None)# Counted before filtering: an unusable row on a full page must not end the walk.- if len(rows) < PAGE_SIZE:+ if len(rows) < page_size:return ids- start += PAGE_SIZE+ start += page_sizedef _top_ids_by_rank(session: requests.Session, path: str, sort: str, limit: int) -> list[int]:@@ -139,6 +148,13 @@ def _universe_ids(session: requests.Session, universe: CoinUniverse) -> list[intreturn _ids_from_map(session, "/v1/cryptocurrency/map")if universe == "exchange_map":return _ids_from_map(session, "/v1/exchange/map")+ if universe == "cmc_ai_coverage":+ # Sorted here too because the map documents no `sort_dir`, and resume relies on ascending ids.+ return sorted(+ _ids_from_map(+ session, "/v5/cmc-ai/coins/map", id_field="crypto_id", page_size=CMC_AI_PAGE_SIZE, data_key="coins"+ )+ )if universe == "top_by_market_cap":return _top_ids_by_rank(session, "/v1/cryptocurrency/listings/latest", "market_cap", HISTORICAL_COIN_LIMIT)return _top_ids_by_rank(session, "/v1/exchange/listings/latest", "volume_24h", HISTORICAL_EXCHANGE_LIMIT)@@ -166,6 +182,18 @@ def _rows_from(row_shape: RowShape, data: Any) -> list[dict[str, Any]]:quotes = data.get("quotes") if isinstance(data, dict) else Nonereturn [quote for quote in (quotes or []) if isinstance(quote, dict)]+ if row_shape == "per_coin_insights":+ coins = data.get("coins") if isinstance(data, dict) else None+ insight_rows: list[dict[str, Any]] = []+ for coin in coins or []:+ if not isinstance(coin, dict):+ continue+ identity = {key: value for key, value in coin.items() if key != "insights"}+ insight_rows.extend(+ {**identity, **insight} for insight in (coin.get("insights") or []) if isinstance(insight, dict)+ )+ return insight_rows+if row_shape == "per_coin_quotes":rows: list[dict[str, Any]] = []for entry in _coin_entries(data):@@ -214,7 +242,7 @@ def get_batch_rows(for index in range(0, len(coin_ids), config.batch_size):batch = coin_ids[index : index + config.batch_size]- body = _get(session, config.path, {**params, "id": ",".join(str(coin_id) for coin_id in batch)})+ body = _get(session, config.path, {**params, config.id_param: ",".join(str(coin_id) for coin_id in batch)})rows = _rows_from(config.row_shape, body.get("data"))if rows:yield rows@@ -385,7 +413,7 @@ def coinmarketcap_source("name": API_KEY_HEADER,"location": "header",},- "paginator": CoinMarketCapPaginator(),+ "paginator": CoinMarketCapPaginator(page_size=endpoint_config.page_size),# CoinMarketCap's API responds directly, so there's no legitimate redirect# to follow; pinning it off keeps traffic on the validated host."session": make_tracked_session(redact_values=(api_key,), allow_redirects=False),diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/coinmarketcap/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/coinmarketcap/settings.pyindex cfaa18b780c8..4c2d0e7eb857 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/coinmarketcap/settings.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/coinmarketcap/settings.py@@ -10,6 +10,9 @@ from products.warehouse_sources.backend.types import IncrementalField, Increment# the monthly credit cost is per data point, so it's unaffected by page size.PAGE_SIZE = 5000+# The v5 CMC AI endpoints cap `limit` at 250 rather than 5000.+CMC_AI_PAGE_SIZE = 250+# Coins per `id=` request on the endpoints that take an explicit id list. CoinMarketCap bills# metadata at 1 credit per 250 records returned, so batching on that boundary is the cheapest# whole multiple. The historical endpoints bill per data point instead, where a smaller batch@@ -17,6 +20,10 @@ PAGE_SIZE = 5000METADATA_BATCH_SIZE = 250HISTORICAL_BATCH_SIZE = 20+# CMC AI bills per coin returned, so the batch size doesn't change the cost. Each coin carries several+# markdown answers, so a smaller batch keeps one response from growing large.+CMC_AI_BATCH_SIZE = 50+# Per-coin history is billed per data point, so the coin universe has to be bounded: asking for# a year of daily candles across every listed coin would spend a month of credits in one sync.# The highest-ranked coins by market cap are what these tables get used for.@@ -57,6 +64,7 @@ class CoinMarketCapEndpointConfig:# Extra query params merged into every request for this endpoint (e.g. a# stable `sort` field for deterministic offset pagination).extra_params: dict[str, str] = field(default_factory=dict)+ page_size: int = PAGE_SIZE# CoinMarketCap's REST API wraps each collection under a top-level "data" key and@@ -129,6 +137,17 @@ COINMARKETCAP_ENDPOINTS: dict[str, CoinMarketCapEndpointConfig] = {primary_key="exchange_id",extra_params={"sort": "total_liquidations_24h", "sort_dir": "desc"},),+ # Market-wide CMC AI questions, answers and top news from the CoinMarketCap homepage.+ "cmc_ai_latest": CoinMarketCapEndpointConfig(+ name="cmc_ai_latest",+ path="/v5/cmc-ai/latest",+ data_selector="data.insights",+ # Changes each time an answer is regenerated, so it is unique within the current feed.+ primary_key="answer_id",+ page_size=CMC_AI_PAGE_SIZE,+ # The endpoint takes no `sort`. The feed is a few dozen insights, so it fits on one page.+ extra_params={"sources_limit": "100"},+ ),}# How a response body turns into rows.@@ -136,7 +155,9 @@ COINMARKETCAP_ENDPOINTS: dict[str, CoinMarketCapEndpointConfig] = {# "per_coin_quotes": `data` maps an id to a record holding a `quotes` array; each quote becomes# a row carrying the coin's identity fields.# "quote_list": `data` is a single object whose `quotes` array is the whole table.-RowShape = Literal["object_map", "per_coin_quotes", "quote_list"]+# "per_coin_insights": `data.coins` holds one record per coin with an `insights` array; each+# insight becomes a row carrying the coin's identity fields.+RowShape = Literal["object_map", "per_coin_quotes", "quote_list", "per_coin_insights"]# Where the ids for an `id=` param come from.# "map": every id in /v1/cryptocurrency/map.@@ -144,7 +165,8 @@ RowShape = Literal["object_map", "per_coin_quotes", "quote_list"]# "exchange_map": every id in /v1/exchange/map.# "top_exchanges_by_volume": the HISTORICAL_EXCHANGE_LIMIT highest-volume exchanges in# exchange/listings/latest.-CoinUniverse = Literal["map", "top_by_market_cap", "exchange_map", "top_exchanges_by_volume"]+# "cmc_ai_coverage": every id in /v5/cmc-ai/coins/map, the coins CMC AI writes content for.+CoinUniverse = Literal["map", "top_by_market_cap", "exchange_map", "top_exchanges_by_volume", "cmc_ai_coverage"]@frozen@@ -166,6 +188,8 @@ class CoinMarketCapBatchEndpointConfig:windowed: bool = Falsepartition_key: Optional[str] = Noneextra_params: dict[str, str] = field(default_factory=dict)+ # Query param that carries the id list.+ id_param: str = "id"COINMARKETCAP_BATCH_ENDPOINTS: dict[str, CoinMarketCapBatchEndpointConfig] = {@@ -262,6 +286,25 @@ COINMARKETCAP_BATCH_ENDPOINTS: dict[str, CoinMarketCapBatchEndpointConfig] = {),},),+ # Per-coin CMC AI questions and answers from the Coin Detail Pages.+ "cmc_ai_coins_latest": CoinMarketCapBatchEndpointConfig(+ name="cmc_ai_coins_latest",+ path="/v5/cmc-ai/coins/latest",+ # The docs don't promise `answer_id` is unique across coins, so the coin id is part of the key.+ primary_keys=["crypto_id", "answer_id"],+ row_shape="per_coin_insights",+ universe="cmc_ai_coverage",+ batch_size=CMC_AI_BATCH_SIZE,+ id_param="crypto_id",+ extra_params={+ # Pagination runs over coins, so one page has to hold the whole batch.+ "limit": str(CMC_AI_BATCH_SIZE),+ "sources_limit": "100",+ # A coin that rotates out of coverage between the map walk and this request would+ # otherwise fail the whole batch.+ "skip_invalid": "true",+ },+ ),}diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/coinmarketcap/tests/test_coinmarketcap.py b/products/warehouse_sources/backend/temporal/data_imports/sources/coinmarketcap/tests/test_coinmarketcap.pyindex 7c40d3c42b9e..5bcc58dcf8ad 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/coinmarketcap/tests/test_coinmarketcap.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/coinmarketcap/tests/test_coinmarketcap.py@@ -14,6 +14,8 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.coinmarketvalidate_credentials,)from products.warehouse_sources.backend.temporal.data_imports.sources.coinmarketcap.settings import (+ CMC_AI_BATCH_SIZE,+ CMC_AI_PAGE_SIZE,HISTORICAL_BACKFILL_DAYS,HISTORICAL_BATCH_SIZE,HISTORICAL_EXCHANGE_LIMIT,@@ -31,10 +33,6 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.generated_)-def _full_page() -> list[dict[str, Any]]:- return [{"id": i} for i in range(PAGE_SIZE)]--class TestCoinMarketCapPaginator:def test_get_resume_state_none_on_terminal_page(self) -> None:paginator = CoinMarketCapPaginator()@@ -56,27 +54,33 @@ class TestCoinMarketCapSourceResumeBehavior:return {"data": items, "status": {"error_code": 0, "error_message": None}}@pytest.mark.parametrize(- ("endpoint", "list_key"),+ ("endpoint", "list_key", "page_size"),[- ("cryptocurrency_map", None),- ("listings_latest", None),- ("fiat_map", None),- ("liquidations_by_cryptocurrency_latest", "cryptocurrencies"),- ("liquidations_by_exchange_latest", "exchanges"),+ ("cryptocurrency_map", None, PAGE_SIZE),+ ("listings_latest", None, PAGE_SIZE),+ ("fiat_map", None, PAGE_SIZE),+ ("liquidations_by_cryptocurrency_latest", "cryptocurrencies", PAGE_SIZE),+ ("liquidations_by_exchange_latest", "exchanges", PAGE_SIZE),+ # A 5000-row `limit` here would be rejected, and a 250-row page would read as the last one.+ ("cmc_ai_latest", "insights", CMC_AI_PAGE_SIZE),],)- def test_fresh_run_saves_start_after_each_non_terminal_page(self, endpoint: str, list_key: str | None) -> None:+ def test_fresh_run_saves_start_after_each_non_terminal_page(+ self, endpoint: str, list_key: str | None, page_size: int+ ) -> None:def body(items: list[dict[str, Any]]) -> dict[str, Any]:if list_key is None:return self._page_body(items)- return {"data": {list_key: items, "total_size": PAGE_SIZE + 1, "has_more": False}}+ return {"data": {list_key: items, "total_size": page_size + 1, "has_more": False}}- result = _driver().run(endpoint, [_response(body(_full_page())), _response(body([{"id": "x"}]))])+ full_page = [{"id": i} for i in range(page_size)]+ result = _driver().run(endpoint, [_response(body(full_page)), _response(body([{"id": "x"}]))])# First request starts at 1; the second carries the advanced start.- assert result.params("start") == ["1", str(1 + PAGE_SIZE)]- assert result.saved_states == [CoinMarketCapResumeConfig(start=1 + PAGE_SIZE)]- assert len(result.rows) == PAGE_SIZE + 1+ assert result.params("start") == ["1", str(1 + page_size)]+ assert result.params("limit") == [str(page_size), str(page_size)]+ assert result.saved_states == [CoinMarketCapResumeConfig(start=1 + page_size)]+ assert len(result.rows) == page_size + 1assert result.raised is Nonedef test_resume_seeds_paginator_with_saved_start(self) -> None:@@ -156,6 +160,30 @@ class TestRowsFrom:rows = _rows_from("quote_list", {"quotes": [{"timestamp": "2026-01-01T00:00:00.000Z", "btc_dominance": 50.0}]})assert rows == [{"timestamp": "2026-01-01T00:00:00.000Z", "btc_dominance": 50.0}]+ def test_per_coin_insights_stamps_coin_identity_on_each_insight(self) -> None:+ rows = _rows_from(+ "per_coin_insights",+ {+ "coins": [+ {+ "crypto_id": 1,+ "symbol": "BTC",+ "insights": [+ {"type": "fixed_question", "answer_id": "a1", "question_key": "overview"},+ {"type": "top_news", "answer_id": "a2", "question_key": None},+ ],+ },+ # Covered but nothing generated yet: the coin contributes no rows.+ {"crypto_id": 1027, "symbol": "ETH", "insights": []},+ ],+ "total_size": 2,+ "has_more": False,+ },+ )++ assert [(row["crypto_id"], row["answer_id"]) for row in rows] == [(1, "a1"), (1, "a2")]+ assert all(row["symbol"] == "BTC" and "insights" not in row for row in rows)+@pytest.mark.parametrize("data", [None, {}, {"quotes": None}])def test_quote_list_tolerates_an_empty_body(self, data: Any) -> None:assert _rows_from("quote_list", data) == []@@ -264,6 +292,30 @@ class TestGetBatchRows:assert first_batch.param("time_start") == _history_start(None)assert result.raised is None+ def test_cmc_ai_coins_batches_the_coverage_map_by_crypto_id(self) -> None:+ coverage = [{"crypto_id": coin_id} for coin_id in range(CMC_AI_BATCH_SIZE + 1, 0, -1)]+ result = _driver().run(+ "cmc_ai_coins_latest",+ [+ _response({"data": {"coins": coverage, "total_size": len(coverage), "has_more": False}}),+ _response({"data": {"coins": [{"crypto_id": 1, "insights": [{"answer_id": "a1"}]}]}}),+ _response({"data": {"coins": []}}),+ ],+ )++ map_call, first_batch, second_batch = result.requests+ assert map_call.path == "/v5/cmc-ai/coins/map"+ assert map_call.param("limit") == str(CMC_AI_PAGE_SIZE)+ # The coin list goes in `crypto_id`, not `id`, in ascending order so resume holds.+ assert first_batch.path == "/v5/cmc-ai/coins/latest"+ assert first_batch.param("id") is None+ assert first_batch.param("crypto_id") == ",".join(str(i) for i in range(1, CMC_AI_BATCH_SIZE + 1))+ assert first_batch.param("limit") == str(CMC_AI_BATCH_SIZE)+ assert second_batch.param("crypto_id") == str(CMC_AI_BATCH_SIZE + 1)+ assert result.rows == [{"crypto_id": 1, "answer_id": "a1"}]+ assert result.saved_states[-1] == CoinMarketCapResumeConfig(last_coin_id=CMC_AI_BATCH_SIZE + 1)+ assert result.raised is None+class TestLastPublishedSnapshotDate:@pytest.mark.parametrize(