:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(data-warehouse): add 4 missing tables to the StockData source (#123814)
Tom Owersfd8e59e8bbd3
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.mdindex 12f46d738b10..748c9d999052 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md@@ -11745,10 +11745,10 @@ Today (6): `dividends`, `eod`, `intraday`, `news`, `quote`, `splits`Diffed against: <https://www.stockdata.org/documentation>-- [ ] `/v1/entity/search` — ticker/entity lookup that resolves the symbols carried on every eod, intraday, quote, dividend and split row (name, exchange, industry, type) (high)-- [ ] `/v1/news/stats/aggregation` — per-entity news sentiment and volume aggregates - the headline output of StockData's news analysis (medium)-- [ ] `/v1/news/sources` — lookup resolving the source identifiers on synced news rows (medium)-- [ ] `/v1/news/stats/intraday` — intraday time series of entity news volume and sentiment (medium)+- [x] `/v1/entity/search` — ticker/entity lookup that resolves the symbols carried on every eod, intraday, quote, dividend and split row (name, exchange, industry, type) (high)+- [x] `/v1/news/stats/aggregation` — per-entity news sentiment and volume aggregates - the headline output of StockData's news analysis (medium)+- [x] `/v1/news/sources` — lookup resolving the source identifiers on synced news rows (medium)+- [x] `/v1/news/stats/intraday` — intraday time series of entity news volume and sentiment (medium)- [ ] `/v1/news/stats/trending` — trending entities by news activity over a window (medium)- [ ] `/v1/entity/industry/list` — industry lookup for grouping tickers in breakdowns (medium)- [ ] `/v1/data/intraday/adjusted` — split/dividend-adjusted intraday bars; the synced intraday table uses the unadjusted /data/intraday endpoint, so long-range series are discontinuous (medium)diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/stockdata/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/stockdata/canonical_descriptions.pyindex 2ccdb72dc978..bce02e18e53b 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/stockdata/canonical_descriptions.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/stockdata/canonical_descriptions.py@@ -103,4 +103,46 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {"ratio": 'Split ratio as a string (e.g. "2:1").',},},+ "entity_search": {+ "description": "Entity metadata (name, type, industry, exchange) for the configured symbols.",+ "docs_url": _DOCS_URL,+ "columns": {+ "symbol": "Unique entity symbol (or ticker).",+ "name": "Entity name.",+ "type": "Entity type (e.g. equity, index, etf, mutualfund, currency, cryptocurrency).",+ "industry": "Industry of the entity.",+ "exchange": "Exchange identifier.",+ "exchange_long": "Exchange name.",+ "mic_code": "ISO 10383 market identifier code of the exchange.",+ "country": "ISO 3166-1 two-letter country code of the exchange locale.",+ },+ },+ "news_sources": {+ "description": "News source feeds that StockData.org collects articles from.",+ "docs_url": _DOCS_URL,+ "columns": {+ "source_id": "Unique ID of the source feed. A domain can have many source IDs.",+ "domain": "Domain of the source.",+ "language": "Language of the source.",+ },+ },+ "news_stats_intraday": {+ "description": "Daily time series of news volume and average sentiment per symbol.",+ "docs_url": _DOCS_URL,+ "columns": {+ "date": "Start of the daily time series bucket (UTC, ISO 8601).",+ "key": "Symbol the stats are grouped by.",+ "total_documents": "Total number of articles that identified the symbol in the bucket.",+ "sentiment_avg": "Average sentiment of the symbol across the articles in the bucket.",+ },+ },+ "news_stats_aggregation": {+ "description": "News volume and average sentiment per symbol, aggregated over one time frame.",+ "docs_url": _DOCS_URL,+ "columns": {+ "key": "Symbol the stats are grouped by.",+ "total_documents": "Total number of articles that identified the symbol.",+ "sentiment_avg": "Average sentiment of the symbol across the articles.",+ },+ },}diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/stockdata/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/stockdata/settings.pyindex 9cb89908f182..e88a6eeecbb5 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/stockdata/settings.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/stockdata/settings.py@@ -3,9 +3,9 @@ from typing import Any, Literalfrom products.warehouse_sources.backend.types import IncrementalField, IncrementalFieldType-# StockData.org windows the price feeds server-side with `date_from` and the news feed with-# `published_after`, so those endpoints sync incrementally. Dividends and splits document no date-# filter, so they are full refresh only.+# StockData.org windows the price feeds server-side with `date_from` and the news feed and news stats+# time series with `published_after`, so those endpoints sync incrementally. The other endpoints+# document no date filter, so they are full refresh only._DATE_INCREMENTAL_FIELDS: list[IncrementalField] = [{"label": "date",@@ -24,12 +24,18 @@ _PUBLISHED_AT_INCREMENTAL_FIELDS: list[IncrementalField] = []+# StockData.org caps any paginated news result set at 20,000 records (limit × page); requesting+# beyond it can't return more rows, so pagination stops there and logs the truncation.+MAX_NEWS_RESULTS = 20_000++[email protected](frozen=False)class StockDataEndpointConfig:name: strpath: strprimary_keys: list[str]- # Price endpoints require a `symbols` query param; news accepts it as an optional filter.+ # Price endpoints require a `symbols` query param; news accepts it as an optional filter. Entity+ # search also requires it, because an unfiltered search pages through every listed entity.requires_symbols: bool = Falseaccepts_symbols: bool = False# Empty list = full refresh only (no server-side date filter).@@ -42,10 +48,16 @@ class StockDataEndpointConfig:params: dict[str, Any] = dataclasses.field(default_factory=dict)# EOD/intraday rows nest OHLCV under item["data"]; the transport flattens it into the row root.flatten_data: bool = False- # Only the news feed paginates (page param, 20,000-result cap); price endpoints return one page.+ # News, entity search, and news sources paginate (page param, `meta.found`); the rest return one page.paginated: bool = False+ # Documented result-set cap for paginated endpoints; None when the vendor documents no cap.+ max_results: int | None = None+ # News stats time series rows arrive as `{date, data: [{key, ...}]}` buckets; the transport+ # explodes each bucket into one row per key.+ explode_buckets: bool = False# The order rows actually arrive in. News is newest-first and can't be flipped (sort_order is- # only valid with the entity-score sorts); price feeds are requested with sort=asc.+ # only valid with the entity-score sorts); price feeds are requested with sort=asc and the news+ # stats time series with date_order=asc.sort_mode: Literal["asc", "desc"] = "asc"description: str | None = None@@ -61,6 +73,7 @@ STOCKDATA_ENDPOINTS: dict[str, StockDataEndpointConfig] = {partition_key="published_at",params={"sort": "published_on"},paginated=True,+ max_results=MAX_NEWS_RESULTS,sort_mode="desc",description="Financial and market news articles with per-entity sentiment scores. Optionally filtered to the configured symbols. Incremental on published_at.",),@@ -113,6 +126,41 @@ STOCKDATA_ENDPOINTS: dict[str, StockDataEndpointConfig] = {partition_key="date",description="Historical stock splits per symbol. Requires symbols and a Standard or higher StockData.org plan. Full refresh.",),+ "entity_search": StockDataEndpointConfig(+ name="entity_search",+ path="/entity/search",+ primary_keys=["symbol"],+ requires_symbols=True,+ paginated=True,+ description="Entity metadata (name, type, industry, exchange, country) for the configured symbols. Requires symbols. Full refresh.",+ ),+ "news_sources": StockDataEndpointConfig(+ name="news_sources",+ path="/news/sources",+ primary_keys=["source_id"],+ paginated=True,+ description="News source feeds (source ID, domain, language) that the news table draws from. The API returns 50 sources per request, so a full sync takes about one request per 50 sources. Full refresh.",+ ),+ "news_stats_intraday": StockDataEndpointConfig(+ name="news_stats_intraday",+ path="/news/stats/intraday",+ primary_keys=["date", "key"],+ accepts_symbols=True,+ incremental_fields=_DATE_INCREMENTAL_FIELDS,+ incremental_param="published_after",+ partition_key="date",+ params={"interval": "day", "group_by": "symbol", "date_order": "asc"},+ explode_buckets=True,+ description="Daily news volume and average sentiment per symbol. Optionally filtered to the configured symbols. Requires a Standard or higher StockData.org plan. Incremental on date.",+ ),+ "news_stats_aggregation": StockDataEndpointConfig(+ name="news_stats_aggregation",+ path="/news/stats/aggregation",+ primary_keys=["key"],+ accepts_symbols=True,+ params={"group_by": "symbol"},+ description="News volume and average sentiment per symbol, aggregated over one time frame. Optionally filtered to the configured symbols. Requires a Standard or higher StockData.org plan. Full refresh.",+ ),}ENDPOINTS = tuple(STOCKDATA_ENDPOINTS.keys())diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/stockdata/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/stockdata/source.pyindex 5975e58f8120..1202988b0410 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/stockdata/source.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/stockdata/source.py@@ -45,10 +45,10 @@ class StockDataSource(ResumableSource[StockDataSourceConfig, StockDataResumeConfreturn {"401 Client Error: Unauthorized for url: https://api.stockdata.org": "Your StockData.org API token is invalid. Generate a new token in your StockData.org dashboard, then reconnect.","402 Client Error: Payment Required for url: https://api.stockdata.org": "Your StockData.org plan's usage limit has been reached. Upgrade your plan or wait for the quota to reset, then resync.",- "403 Client Error: Forbidden for url: https://api.stockdata.org": "Your StockData.org plan does not grant access to this data (e.g. dividends and splits require a Standard or higher plan). Upgrade your plan or deselect the restricted tables, then resync.",- # The price tables need at least one symbol; without one the sync can never make+ "403 Client Error: Forbidden for url: https://api.stockdata.org": "Your StockData.org plan does not grant access to this data (e.g. dividends, splits, and the news stats tables require a Standard or higher plan). Upgrade your plan or deselect the restricted tables, then resync.",+ # The price and entity search tables need at least one symbol; without one the sync can never make# progress, so fail permanently with a fix-it message rather than retrying.- "StockData.org API error [missing_symbols]": "One or more of the selected tables (quote, EOD, intraday, dividends, splits) requires symbols. Add symbols to the source configuration, then resync.",+ "StockData.org API error [missing_symbols]": "One or more of the selected tables (quote, EOD, intraday, dividends, splits, entity search) requires symbols. Add symbols to the source configuration, then resync.",}def get_canonical_descriptions(self) -> CanonicalDescriptions:@@ -70,8 +70,8 @@ class StockDataSource(ResumableSource[StockDataSourceConfig, StockDataResumeConfschemas = [SourceSchema(name=endpoint.name,- # Only news (published_after) and the EOD/intraday price feeds (date_from) expose a- # server-side date filter; quote, dividends, and splits are full refresh only.+ # Only news and the news stats time series (published_after) and the EOD/intraday+ # price feeds (date_from) expose a server-side date filter; the rest are full refresh only.supports_incremental=bool(endpoint.incremental_fields),supports_append=bool(endpoint.incremental_fields),incremental_fields=list(endpoint.incremental_fields),@@ -123,13 +123,13 @@ class StockDataSource(ResumableSource[StockDataSourceConfig, StockDataResumeConfcategory=DataWarehouseSourceCategory.FINANCE___ACCOUNTING,label="StockData.org",releaseStatus=ReleaseStatus.BETA,- caption="""Enter your StockData.org API token to pull market news with sentiment plus quote, end-of-day, intraday, dividend, and split price data into the PostHog Data warehouse.+ caption="""Enter your StockData.org API token to pull market news with sentiment, news stats, entity and news source metadata, plus quote, end-of-day, intraday, dividend, and split price data into the PostHog Data warehouse.You can find your API token in your [StockData.org dashboard](https://www.stockdata.org/dashboard).-Add one or more comma-separated **symbols** (e.g. `AAPL,MSFT,TSLA`) to sync the price tables (quote, EOD, intraday, dividends, splits) — those endpoints require at least one symbol. The news table works without symbols (all market news) and is filtered to your symbols when they are set.+Add one or more comma-separated **symbols** (e.g. `AAPL,MSFT,TSLA`) to sync the price tables (quote, EOD, intraday, dividends, splits) and the entity search table — those endpoints require at least one symbol. The news and news stats tables work without symbols (all market news) and are filtered to your symbols when they are set.-Note: StockData.org plans are limited by daily request quotas (100 requests/day on the free plan), and some tables (dividends, splits) require a Standard or higher plan.""",+Note: StockData.org plans are limited by daily request quotas (100 requests/day on the free plan), and some tables (dividends, splits, news stats) require a Standard or higher plan.""",iconPath="/static/services/stockdata.png",docsUrl="https://posthog.com/docs/cdp/sources/stockdata",fields=cast(diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/stockdata/stockdata.py b/products/warehouse_sources/backend/temporal/data_imports/sources/stockdata/stockdata.pyindex 12b38a3059f4..1b6441740245 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/stockdata/stockdata.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/stockdata/stockdata.py@@ -21,14 +21,11 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.stockdata.logger = logging.getLogger(__name__)STOCKDATA_BASE_URL = "https://api.stockdata.org/v1"-# StockData.org caps any paginated news result set at 20,000 records (limit × page); requesting-# beyond it can't return more rows, so pagination stops there and logs the truncation.-MAX_NEWS_RESULTS = 20_000[email protected](frozen=False)class StockDataResumeConfig:- # Number of the next news page to fetch — StockData.org uses page-number pagination.+ # Number of the next page to fetch — StockData.org uses page-number pagination.next_page: int@@ -62,16 +59,28 @@ def _flatten_ohlcv(row: dict[str, Any]) -> dict[str, Any]:return flat-class StockDataNewsPaginator(PageNumberPaginator):- """Page-number paginator that stops on the documented 20,000-result cap.+def _explode_stats_bucket(row: dict[str, Any]) -> list[dict[str, Any]]:+ """News stats time series rows arrive as `{date, data: [{key, total_documents, sentiment_avg}]}`+ — emit one row per key, carrying the bucket date."""+ entries = row.get("data")+ # A malformed bucket must fail the sync: dropping it would let later buckets advance the+ # incremental watermark past its date, so that day would never be re-fetched.+ if not isinstance(entries, list) or not all(isinstance(entry, dict) for entry in entries):+ raise ValueError(f"StockData.org returned a malformed news stats bucket for date {row.get('date')!r}")+ return [{"date": row.get("date"), **entry} for entry in entries]- The news response's `meta` block carries `found` (total matches), `limit` (page size), and++class StockDataPagePaginator(PageNumberPaginator):+ """Page-number paginator that stops on an endpoint's result-set cap, when it has one.++ The paginated responses' `meta` block carries `found` (total matches), `limit` (page size), and`page`, so the paginator stops after the last real page instead of paying an extra empty-pagerequest, and terminates cleanly at the result-set cap instead of erroring past it."""- def __init__(self) -> None:+ def __init__(self, max_results: int | None = None) -> None:super().__init__(base_page=1, page_param="page", stop_after_empty_page=True)+ self.max_results = max_resultsdef update_state(self, response: Response, data: Optional[list[Any]] = None) -> None:if data is None or len(data) == 0:@@ -100,11 +109,11 @@ class StockDataNewsPaginator(PageNumberPaginator):if fetched >= found:self._has_next_page = Falsereturn- if fetched >= MAX_NEWS_RESULTS:+ if self.max_results is not None and fetched >= self.max_results:logger.info(- "StockData.org news pagination reached the API's %s-result cap; %s matching "- "articles were not fetched. Older articles sync as the incremental watermark advances.",- MAX_NEWS_RESULTS,+ "StockData.org pagination reached the API's %s-result cap; %s matching "+ "records were not fetched. Older news articles sync as the incremental watermark advances.",+ self.max_results,found - fetched,)self._has_next_page = False@@ -155,6 +164,8 @@ def stockdata_source(}if config.flatten_data:resource["data_map"] = _flatten_ohlcv+ elif config.explode_buckets:+ resource["data_map"] = _explode_stats_bucketrest_config: RESTAPIConfig = {"client": {@@ -162,7 +173,7 @@ def stockdata_source(# api_token rides in the query string (the API's only auth method); the framework auth# redacts its value from every logged URL, captured sample, and raised error message."auth": {"type": "api_key", "api_key": api_token, "name": "api_token", "location": "query"},- "paginator": StockDataNewsPaginator() if config.paginated else "single_page",+ "paginator": StockDataPagePaginator(config.max_results) if config.paginated else "single_page",},"resources": [resource],}@@ -205,8 +216,9 @@ def stockdata_source(name=endpoint,items=lambda: rest_resource,primary_keys=config.primary_keys,- # Price feeds are requested with sort=asc; news is served newest-first and its sort order- # can't be flipped, so it declares desc and the watermark checkpoints at end of sync.+ # Price feeds and the news stats time series are requested ascending; news is served+ # newest-first and its sort order can't be flipped, so it declares desc and the watermark+ # checkpoints at end of sync.sort_mode=config.sort_mode,**partition_kwargs,)diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/stockdata/tests/test_stockdata.py b/products/warehouse_sources/backend/temporal/data_imports/sources/stockdata/tests/test_stockdata.pyindex a76068f2f147..6f2168f395f0 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/stockdata/tests/test_stockdata.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/stockdata/tests/test_stockdata.py@@ -79,6 +79,15 @@ class TestNewsPagination:assert len(result.rows) == 6assert len(result.requests) == 2+ def test_news_sources_is_not_held_to_the_news_result_cap(self) -> None:+ pages = [[{"source_id": f"p{page}"}] for page in (1, 2, 3)]+ result = _driver().run(+ "news_sources",+ [_news_page(rows, found=30_000, limit=10_000, page=page) for page, rows in enumerate(pages, start=1)],+ )++ assert [request.param("page") for request in result.requests] == ["1", "2", "3"]+def test_stops_on_empty_first_page(self) -> None:result = _driver().run("news", [_news_page([], found=0, limit=100, page=1)])@@ -95,6 +104,22 @@ class TestNewsPagination:# The first request must start from the persisted page, not page 1.assert result.requests[0].param("page") == "3"+ @parameterized.expand(+ [+ ("news_sources", "source_id", None),+ ("entity_search", "symbol", "AAPL,MSFT"),+ ]+ )+ def test_metadata_endpoints_walk_every_page(self, endpoint: str, key: str, symbols: str | None) -> None:+ pages = [[{key: f"p{page}-{i}"} for i in range(2)] for page in (1, 2, 3)]+ result = _driver(symbols=symbols).run(+ endpoint,+ [_news_page(rows, found=5, limit=2, page=page) for page, rows in enumerate(pages, start=1)],+ )++ assert [row[key] for row in result.rows] == [row[key] for rows in pages for row in rows]+ assert [request.param("page") for request in result.requests] == ["1", "2", "3"]+def test_missing_data_key_raises_loudly(self) -> None:result = _driver().run("news", [_data_page(None, drop_data=True)])@@ -115,6 +140,44 @@ class TestRequestParams:assert result.requests[0].param("published_after") == "2021-04-09T12:30:05"+ @parameterized.expand(+ [+ ("incremental", datetime(2026, 10, 1), "2026-10-01T00:00:00"),+ ("full_refresh", None, None),+ ]+ )+ def test_news_stats_intraday_requests_ascending_daily_buckets(+ self, _name: str, watermark: datetime | None, expected_published_after: str | None+ ) -> None:+ result = _driver(symbols="tsla, amzn").run(+ "news_stats_intraday",+ [_data_page([])],+ incremental_field="date" if watermark else None,+ db_incremental_field_last_value=watermark,+ )++ request = result.requests[0]+ assert request.param("interval") == "day"+ assert request.param("group_by") == "symbol"+ assert request.param("date_order") == "asc"+ assert request.param("symbols") == "TSLA,AMZN"+ assert request.param("published_after") == expected_published_after++ @parameterized.expand(+ [+ ("news_stats_aggregation_without_symbols", "news_stats_aggregation", None, None),+ ("news_stats_aggregation_with_symbols", "news_stats_aggregation", "TSLA", "TSLA"),+ ("news_sources_ignores_symbols", "news_sources", "TSLA", None),+ ]+ )+ def test_optional_symbols_filter(+ self, _name: str, endpoint: str, symbols: str | None, expected_symbols: str | None+ ) -> None:+ result = _driver(symbols=symbols).run(endpoint, [_news_page([], found=0, limit=50, page=1)])++ assert result.raised is None+ assert result.requests[0].param("symbols") == expected_symbols+def test_eod_requests_ascending_day_interval_with_date_from_watermark(self) -> None:result = _driver(symbols="AAPL").run("eod",@@ -130,8 +193,66 @@ class TestRequestParams:assert request.param("date_from") == "2021-04-09"+class TestNewsStatsRowShaping:+ @parameterized.expand(+ [+ (+ "one_row_per_key_with_bucket_date",+ [+ {+ "date": "2026-10-02T00:00:00.000Z",+ "data": [+ {"key": "TSLA", "total_documents": 4, "sentiment_avg": 0.25},+ {"key": "AMZN", "total_documents": 2, "sentiment_avg": -0.5},+ ],+ },+ {+ "date": "2026-10-01T00:00:00.000Z",+ "data": [{"key": "TSLA", "total_documents": 1, "sentiment_avg": 0.0}],+ },+ ],+ [+ {"date": "2026-10-02T00:00:00.000Z", "key": "TSLA", "total_documents": 4, "sentiment_avg": 0.25},+ {"date": "2026-10-02T00:00:00.000Z", "key": "AMZN", "total_documents": 2, "sentiment_avg": -0.5},+ {"date": "2026-10-01T00:00:00.000Z", "key": "TSLA", "total_documents": 1, "sentiment_avg": 0.0},+ ],+ ),+ ("empty_bucket_yields_nothing", [{"date": "2026-10-02T00:00:00.000Z", "data": []}], []),+ ("empty_object_result", {}, []),+ ]+ )+ def test_intraday_buckets_explode_into_rows(+ self, _name: str, data: Any, expected_rows: list[dict[str, Any]]+ ) -> None:+ result = _driver().run("news_stats_intraday", [_data_page(data)])++ assert result.raised is None+ assert result.rows == expected_rows++ @parameterized.expand(+ [+ ("missing_data", {"date": "2026-10-03T00:00:00.000Z"}),+ ("non_list_data", {"date": "2026-10-03T00:00:00.000Z", "data": None}),+ ("non_dict_entry", {"date": "2026-10-04T00:00:00.000Z", "data": ["not-a-dict"]}),+ ]+ )+ def test_malformed_bucket_fails_the_sync(self, _name: str, bucket: dict[str, Any]) -> None:+ valid = {"date": "2026-10-05T00:00:00.000Z", "data": [{"key": "TSLA", "total_documents": 1}]}+ result = _driver().run("news_stats_intraday", [_data_page([bucket, valid])])++ # Dropping the bucket would let the later one advance the watermark past its date.+ assert isinstance(result.raised, ValueError)+ assert "malformed news stats bucket" in str(result.raised)++ def test_aggregation_rows_pass_through_unchanged(self) -> None:+ rows = [{"key": "TSLA", "total_documents": 55, "sentiment_avg": 0.33}]+ result = _driver().run("news_stats_aggregation", [_data_page(rows)])++ assert result.rows == rows++class TestRequiresSymbols:- @parameterized.expand([("quote",), ("eod",), ("intraday",), ("dividends",), ("splits",)])+ @parameterized.expand([("quote",), ("eod",), ("intraday",), ("dividends",), ("splits",), ("entity_search",)])def test_price_endpoints_require_symbols(self, endpoint: str) -> None:# Selecting a price table with no symbols is a permanent misconfiguration.result = _driver(symbols=None).run(endpoint, [])@@ -173,6 +294,10 @@ class TestSourceResponse:("intraday", ["ticker", "date"], "date", "asc"),("dividends", ["ticker", "date"], "date", "asc"),("splits", ["ticker", "date"], "date", "asc"),+ ("entity_search", ["symbol"], None, "asc"),+ ("news_sources", ["source_id"], None, "asc"),+ ("news_stats_intraday", ["date", "key"], "date", "asc"),+ ("news_stats_aggregation", ["key"], None, "asc"),])def test_source_response_keys_partitioning_and_sort(