Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(data-warehouse): add 4 missing tables to the StockData source (#123814)

Tom Owersfd8e59e8bbd3
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
index 12f46d738b10..748c9d999052 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
@@ -11745,10 +11745,10 @@ Today (6): `dividends`, `eod`, `intraday`, `news`, `quote`, `splits`
Diffed against: <https://www.stockdata.org/documentation>
-- [ ] `/v1/entity/search` — ticker/entity lookup that resolves the symbols carried on every eod, intraday, quote, dividend and split row (name, exchange, industry, type) (high)
-- [ ] `/v1/news/stats/aggregation` — per-entity news sentiment and volume aggregates - the headline output of StockData's news analysis (medium)
-- [ ] `/v1/news/sources` — lookup resolving the source identifiers on synced news rows (medium)
-- [ ] `/v1/news/stats/intraday` — intraday time series of entity news volume and sentiment (medium)
+- [x] `/v1/entity/search` — ticker/entity lookup that resolves the symbols carried on every eod, intraday, quote, dividend and split row (name, exchange, industry, type) (high)
+- [x] `/v1/news/stats/aggregation` — per-entity news sentiment and volume aggregates - the headline output of StockData's news analysis (medium)
+- [x] `/v1/news/sources` — lookup resolving the source identifiers on synced news rows (medium)
+- [x] `/v1/news/stats/intraday` — intraday time series of entity news volume and sentiment (medium)
- [ ] `/v1/news/stats/trending` — trending entities by news activity over a window (medium)
- [ ] `/v1/entity/industry/list` — industry lookup for grouping tickers in breakdowns (medium)
- [ ] `/v1/data/intraday/adjusted` — split/dividend-adjusted intraday bars; the synced intraday table uses the unadjusted /data/intraday endpoint, so long-range series are discontinuous (medium)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/stockdata/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/stockdata/canonical_descriptions.py
index 2ccdb72dc978..bce02e18e53b 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/stockdata/canonical_descriptions.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/stockdata/canonical_descriptions.py
@@ -103,4 +103,46 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {
"ratio": 'Split ratio as a string (e.g. "2:1").',
},
},
+ "entity_search": {
+ "description": "Entity metadata (name, type, industry, exchange) for the configured symbols.",
+ "docs_url": _DOCS_URL,
+ "columns": {
+ "symbol": "Unique entity symbol (or ticker).",
+ "name": "Entity name.",
+ "type": "Entity type (e.g. equity, index, etf, mutualfund, currency, cryptocurrency).",
+ "industry": "Industry of the entity.",
+ "exchange": "Exchange identifier.",
+ "exchange_long": "Exchange name.",
+ "mic_code": "ISO 10383 market identifier code of the exchange.",
+ "country": "ISO 3166-1 two-letter country code of the exchange locale.",
+ },
+ },
+ "news_sources": {
+ "description": "News source feeds that StockData.org collects articles from.",
+ "docs_url": _DOCS_URL,
+ "columns": {
+ "source_id": "Unique ID of the source feed. A domain can have many source IDs.",
+ "domain": "Domain of the source.",
+ "language": "Language of the source.",
+ },
+ },
+ "news_stats_intraday": {
+ "description": "Daily time series of news volume and average sentiment per symbol.",
+ "docs_url": _DOCS_URL,
+ "columns": {
+ "date": "Start of the daily time series bucket (UTC, ISO 8601).",
+ "key": "Symbol the stats are grouped by.",
+ "total_documents": "Total number of articles that identified the symbol in the bucket.",
+ "sentiment_avg": "Average sentiment of the symbol across the articles in the bucket.",
+ },
+ },
+ "news_stats_aggregation": {
+ "description": "News volume and average sentiment per symbol, aggregated over one time frame.",
+ "docs_url": _DOCS_URL,
+ "columns": {
+ "key": "Symbol the stats are grouped by.",
+ "total_documents": "Total number of articles that identified the symbol.",
+ "sentiment_avg": "Average sentiment of the symbol across the articles.",
+ },
+ },
}
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/stockdata/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/stockdata/settings.py
index 9cb89908f182..e88a6eeecbb5 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/stockdata/settings.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/stockdata/settings.py
@@ -3,9 +3,9 @@ from typing import Any, Literal
from products.warehouse_sources.backend.types import IncrementalField, IncrementalFieldType
-# StockData.org windows the price feeds server-side with `date_from` and the news feed with
-# `published_after`, so those endpoints sync incrementally. Dividends and splits document no date
-# filter, so they are full refresh only.
+# StockData.org windows the price feeds server-side with `date_from` and the news feed and news stats
+# time series with `published_after`, so those endpoints sync incrementally. The other endpoints
+# document no date filter, so they are full refresh only.
_DATE_INCREMENTAL_FIELDS: list[IncrementalField] = [
{
"label": "date",
@@ -24,12 +24,18 @@ _PUBLISHED_AT_INCREMENTAL_FIELDS: list[IncrementalField] = [
]
+# StockData.org caps any paginated news result set at 20,000 records (limit × page); requesting
+# beyond it can't return more rows, so pagination stops there and logs the truncation.
+MAX_NEWS_RESULTS = 20_000
+
+
[email protected](frozen=False)
class StockDataEndpointConfig:
name: str
path: str
primary_keys: list[str]
- # Price endpoints require a `symbols` query param; news accepts it as an optional filter.
+ # Price endpoints require a `symbols` query param; news accepts it as an optional filter. Entity
+ # search also requires it, because an unfiltered search pages through every listed entity.
requires_symbols: bool = False
accepts_symbols: bool = False
# Empty list = full refresh only (no server-side date filter).
@@ -42,10 +48,16 @@ class StockDataEndpointConfig:
params: dict[str, Any] = dataclasses.field(default_factory=dict)
# EOD/intraday rows nest OHLCV under item["data"]; the transport flattens it into the row root.
flatten_data: bool = False
- # Only the news feed paginates (page param, 20,000-result cap); price endpoints return one page.
+ # News, entity search, and news sources paginate (page param, `meta.found`); the rest return one page.
paginated: bool = False
+ # Documented result-set cap for paginated endpoints; None when the vendor documents no cap.
+ max_results: int | None = None
+ # News stats time series rows arrive as `{date, data: [{key, ...}]}` buckets; the transport
+ # explodes each bucket into one row per key.
+ explode_buckets: bool = False
# The order rows actually arrive in. News is newest-first and can't be flipped (sort_order is
- # only valid with the entity-score sorts); price feeds are requested with sort=asc.
+ # only valid with the entity-score sorts); price feeds are requested with sort=asc and the news
+ # stats time series with date_order=asc.
sort_mode: Literal["asc", "desc"] = "asc"
description: str | None = None
@@ -61,6 +73,7 @@ STOCKDATA_ENDPOINTS: dict[str, StockDataEndpointConfig] = {
partition_key="published_at",
params={"sort": "published_on"},
paginated=True,
+ max_results=MAX_NEWS_RESULTS,
sort_mode="desc",
description="Financial and market news articles with per-entity sentiment scores. Optionally filtered to the configured symbols. Incremental on published_at.",
),
@@ -113,6 +126,41 @@ STOCKDATA_ENDPOINTS: dict[str, StockDataEndpointConfig] = {
partition_key="date",
description="Historical stock splits per symbol. Requires symbols and a Standard or higher StockData.org plan. Full refresh.",
),
+ "entity_search": StockDataEndpointConfig(
+ name="entity_search",
+ path="/entity/search",
+ primary_keys=["symbol"],
+ requires_symbols=True,
+ paginated=True,
+ description="Entity metadata (name, type, industry, exchange, country) for the configured symbols. Requires symbols. Full refresh.",
+ ),
+ "news_sources": StockDataEndpointConfig(
+ name="news_sources",
+ path="/news/sources",
+ primary_keys=["source_id"],
+ paginated=True,
+ description="News source feeds (source ID, domain, language) that the news table draws from. The API returns 50 sources per request, so a full sync takes about one request per 50 sources. Full refresh.",
+ ),
+ "news_stats_intraday": StockDataEndpointConfig(
+ name="news_stats_intraday",
+ path="/news/stats/intraday",
+ primary_keys=["date", "key"],
+ accepts_symbols=True,
+ incremental_fields=_DATE_INCREMENTAL_FIELDS,
+ incremental_param="published_after",
+ partition_key="date",
+ params={"interval": "day", "group_by": "symbol", "date_order": "asc"},
+ explode_buckets=True,
+ description="Daily news volume and average sentiment per symbol. Optionally filtered to the configured symbols. Requires a Standard or higher StockData.org plan. Incremental on date.",
+ ),
+ "news_stats_aggregation": StockDataEndpointConfig(
+ name="news_stats_aggregation",
+ path="/news/stats/aggregation",
+ primary_keys=["key"],
+ accepts_symbols=True,
+ params={"group_by": "symbol"},
+ description="News volume and average sentiment per symbol, aggregated over one time frame. Optionally filtered to the configured symbols. Requires a Standard or higher StockData.org plan. Full refresh.",
+ ),
}
ENDPOINTS = tuple(STOCKDATA_ENDPOINTS.keys())
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/stockdata/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/stockdata/source.py
index 5975e58f8120..1202988b0410 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/stockdata/source.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/stockdata/source.py
@@ -45,10 +45,10 @@ class StockDataSource(ResumableSource[StockDataSourceConfig, StockDataResumeConf
return {
"401 Client Error: Unauthorized for url: https://api.stockdata.org": "Your StockData.org API token is invalid. Generate a new token in your StockData.org dashboard, then reconnect.",
"402 Client Error: Payment Required for url: https://api.stockdata.org": "Your StockData.org plan's usage limit has been reached. Upgrade your plan or wait for the quota to reset, then resync.",
- "403 Client Error: Forbidden for url: https://api.stockdata.org": "Your StockData.org plan does not grant access to this data (e.g. dividends and splits require a Standard or higher plan). Upgrade your plan or deselect the restricted tables, then resync.",
- # The price tables need at least one symbol; without one the sync can never make
+ "403 Client Error: Forbidden for url: https://api.stockdata.org": "Your StockData.org plan does not grant access to this data (e.g. dividends, splits, and the news stats tables require a Standard or higher plan). Upgrade your plan or deselect the restricted tables, then resync.",
+ # The price and entity search tables need at least one symbol; without one the sync can never make
# progress, so fail permanently with a fix-it message rather than retrying.
- "StockData.org API error [missing_symbols]": "One or more of the selected tables (quote, EOD, intraday, dividends, splits) requires symbols. Add symbols to the source configuration, then resync.",
+ "StockData.org API error [missing_symbols]": "One or more of the selected tables (quote, EOD, intraday, dividends, splits, entity search) requires symbols. Add symbols to the source configuration, then resync.",
}
def get_canonical_descriptions(self) -> CanonicalDescriptions:
@@ -70,8 +70,8 @@ class StockDataSource(ResumableSource[StockDataSourceConfig, StockDataResumeConf
schemas = [
SourceSchema(
name=endpoint.name,
- # Only news (published_after) and the EOD/intraday price feeds (date_from) expose a
- # server-side date filter; quote, dividends, and splits are full refresh only.
+ # Only news and the news stats time series (published_after) and the EOD/intraday
+ # price feeds (date_from) expose a server-side date filter; the rest are full refresh only.
supports_incremental=bool(endpoint.incremental_fields),
supports_append=bool(endpoint.incremental_fields),
incremental_fields=list(endpoint.incremental_fields),
@@ -123,13 +123,13 @@ class StockDataSource(ResumableSource[StockDataSourceConfig, StockDataResumeConf
category=DataWarehouseSourceCategory.FINANCE___ACCOUNTING,
label="StockData.org",
releaseStatus=ReleaseStatus.BETA,
- caption="""Enter your StockData.org API token to pull market news with sentiment plus quote, end-of-day, intraday, dividend, and split price data into the PostHog Data warehouse.
+ caption="""Enter your StockData.org API token to pull market news with sentiment, news stats, entity and news source metadata, plus quote, end-of-day, intraday, dividend, and split price data into the PostHog Data warehouse.
You can find your API token in your [StockData.org dashboard](https://www.stockdata.org/dashboard).
-Add one or more comma-separated **symbols** (e.g. `AAPL,MSFT,TSLA`) to sync the price tables (quote, EOD, intraday, dividends, splits) — those endpoints require at least one symbol. The news table works without symbols (all market news) and is filtered to your symbols when they are set.
+Add one or more comma-separated **symbols** (e.g. `AAPL,MSFT,TSLA`) to sync the price tables (quote, EOD, intraday, dividends, splits) and the entity search table — those endpoints require at least one symbol. The news and news stats tables work without symbols (all market news) and are filtered to your symbols when they are set.
-Note: StockData.org plans are limited by daily request quotas (100 requests/day on the free plan), and some tables (dividends, splits) require a Standard or higher plan.""",
+Note: StockData.org plans are limited by daily request quotas (100 requests/day on the free plan), and some tables (dividends, splits, news stats) require a Standard or higher plan.""",
iconPath="/static/services/stockdata.png",
docsUrl="https://posthog.com/docs/cdp/sources/stockdata",
fields=cast(
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/stockdata/stockdata.py b/products/warehouse_sources/backend/temporal/data_imports/sources/stockdata/stockdata.py
index 12b38a3059f4..1b6441740245 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/stockdata/stockdata.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/stockdata/stockdata.py
@@ -21,14 +21,11 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.stockdata.
logger = logging.getLogger(__name__)
STOCKDATA_BASE_URL = "https://api.stockdata.org/v1"
-# StockData.org caps any paginated news result set at 20,000 records (limit × page); requesting
-# beyond it can't return more rows, so pagination stops there and logs the truncation.
-MAX_NEWS_RESULTS = 20_000
[email protected](frozen=False)
class StockDataResumeConfig:
- # Number of the next news page to fetch — StockData.org uses page-number pagination.
+ # Number of the next page to fetch — StockData.org uses page-number pagination.
next_page: int
@@ -62,16 +59,28 @@ def _flatten_ohlcv(row: dict[str, Any]) -> dict[str, Any]:
return flat
-class StockDataNewsPaginator(PageNumberPaginator):
- """Page-number paginator that stops on the documented 20,000-result cap.
+def _explode_stats_bucket(row: dict[str, Any]) -> list[dict[str, Any]]:
+ """News stats time series rows arrive as `{date, data: [{key, total_documents, sentiment_avg}]}`
+ — emit one row per key, carrying the bucket date."""
+ entries = row.get("data")
+ # A malformed bucket must fail the sync: dropping it would let later buckets advance the
+ # incremental watermark past its date, so that day would never be re-fetched.
+ if not isinstance(entries, list) or not all(isinstance(entry, dict) for entry in entries):
+ raise ValueError(f"StockData.org returned a malformed news stats bucket for date {row.get('date')!r}")
+ return [{"date": row.get("date"), **entry} for entry in entries]
- The news response's `meta` block carries `found` (total matches), `limit` (page size), and
+
+class StockDataPagePaginator(PageNumberPaginator):
+ """Page-number paginator that stops on an endpoint's result-set cap, when it has one.
+
+ The paginated responses' `meta` block carries `found` (total matches), `limit` (page size), and
`page`, so the paginator stops after the last real page instead of paying an extra empty-page
request, and terminates cleanly at the result-set cap instead of erroring past it.
"""
- def __init__(self) -> None:
+ def __init__(self, max_results: int | None = None) -> None:
super().__init__(base_page=1, page_param="page", stop_after_empty_page=True)
+ self.max_results = max_results
def update_state(self, response: Response, data: Optional[list[Any]] = None) -> None:
if data is None or len(data) == 0:
@@ -100,11 +109,11 @@ class StockDataNewsPaginator(PageNumberPaginator):
if fetched >= found:
self._has_next_page = False
return
- if fetched >= MAX_NEWS_RESULTS:
+ if self.max_results is not None and fetched >= self.max_results:
logger.info(
- "StockData.org news pagination reached the API's %s-result cap; %s matching "
- "articles were not fetched. Older articles sync as the incremental watermark advances.",
- MAX_NEWS_RESULTS,
+ "StockData.org pagination reached the API's %s-result cap; %s matching "
+ "records were not fetched. Older news articles sync as the incremental watermark advances.",
+ self.max_results,
found - fetched,
)
self._has_next_page = False
@@ -155,6 +164,8 @@ def stockdata_source(
}
if config.flatten_data:
resource["data_map"] = _flatten_ohlcv
+ elif config.explode_buckets:
+ resource["data_map"] = _explode_stats_bucket
rest_config: RESTAPIConfig = {
"client": {
@@ -162,7 +173,7 @@ def stockdata_source(
# api_token rides in the query string (the API's only auth method); the framework auth
# redacts its value from every logged URL, captured sample, and raised error message.
"auth": {"type": "api_key", "api_key": api_token, "name": "api_token", "location": "query"},
- "paginator": StockDataNewsPaginator() if config.paginated else "single_page",
+ "paginator": StockDataPagePaginator(config.max_results) if config.paginated else "single_page",
},
"resources": [resource],
}
@@ -205,8 +216,9 @@ def stockdata_source(
name=endpoint,
items=lambda: rest_resource,
primary_keys=config.primary_keys,
- # Price feeds are requested with sort=asc; news is served newest-first and its sort order
- # can't be flipped, so it declares desc and the watermark checkpoints at end of sync.
+ # Price feeds and the news stats time series are requested ascending; news is served
+ # newest-first and its sort order can't be flipped, so it declares desc and the watermark
+ # checkpoints at end of sync.
sort_mode=config.sort_mode,
**partition_kwargs,
)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/stockdata/tests/test_stockdata.py b/products/warehouse_sources/backend/temporal/data_imports/sources/stockdata/tests/test_stockdata.py
index a76068f2f147..6f2168f395f0 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/stockdata/tests/test_stockdata.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/stockdata/tests/test_stockdata.py
@@ -79,6 +79,15 @@ class TestNewsPagination:
assert len(result.rows) == 6
assert len(result.requests) == 2
+ def test_news_sources_is_not_held_to_the_news_result_cap(self) -> None:
+ pages = [[{"source_id": f"p{page}"}] for page in (1, 2, 3)]
+ result = _driver().run(
+ "news_sources",
+ [_news_page(rows, found=30_000, limit=10_000, page=page) for page, rows in enumerate(pages, start=1)],
+ )
+
+ assert [request.param("page") for request in result.requests] == ["1", "2", "3"]
+
def test_stops_on_empty_first_page(self) -> None:
result = _driver().run("news", [_news_page([], found=0, limit=100, page=1)])
@@ -95,6 +104,22 @@ class TestNewsPagination:
# The first request must start from the persisted page, not page 1.
assert result.requests[0].param("page") == "3"
+ @parameterized.expand(
+ [
+ ("news_sources", "source_id", None),
+ ("entity_search", "symbol", "AAPL,MSFT"),
+ ]
+ )
+ def test_metadata_endpoints_walk_every_page(self, endpoint: str, key: str, symbols: str | None) -> None:
+ pages = [[{key: f"p{page}-{i}"} for i in range(2)] for page in (1, 2, 3)]
+ result = _driver(symbols=symbols).run(
+ endpoint,
+ [_news_page(rows, found=5, limit=2, page=page) for page, rows in enumerate(pages, start=1)],
+ )
+
+ assert [row[key] for row in result.rows] == [row[key] for rows in pages for row in rows]
+ assert [request.param("page") for request in result.requests] == ["1", "2", "3"]
+
def test_missing_data_key_raises_loudly(self) -> None:
result = _driver().run("news", [_data_page(None, drop_data=True)])
@@ -115,6 +140,44 @@ class TestRequestParams:
assert result.requests[0].param("published_after") == "2021-04-09T12:30:05"
+ @parameterized.expand(
+ [
+ ("incremental", datetime(2026, 10, 1), "2026-10-01T00:00:00"),
+ ("full_refresh", None, None),
+ ]
+ )
+ def test_news_stats_intraday_requests_ascending_daily_buckets(
+ self, _name: str, watermark: datetime | None, expected_published_after: str | None
+ ) -> None:
+ result = _driver(symbols="tsla, amzn").run(
+ "news_stats_intraday",
+ [_data_page([])],
+ incremental_field="date" if watermark else None,
+ db_incremental_field_last_value=watermark,
+ )
+
+ request = result.requests[0]
+ assert request.param("interval") == "day"
+ assert request.param("group_by") == "symbol"
+ assert request.param("date_order") == "asc"
+ assert request.param("symbols") == "TSLA,AMZN"
+ assert request.param("published_after") == expected_published_after
+
+ @parameterized.expand(
+ [
+ ("news_stats_aggregation_without_symbols", "news_stats_aggregation", None, None),
+ ("news_stats_aggregation_with_symbols", "news_stats_aggregation", "TSLA", "TSLA"),
+ ("news_sources_ignores_symbols", "news_sources", "TSLA", None),
+ ]
+ )
+ def test_optional_symbols_filter(
+ self, _name: str, endpoint: str, symbols: str | None, expected_symbols: str | None
+ ) -> None:
+ result = _driver(symbols=symbols).run(endpoint, [_news_page([], found=0, limit=50, page=1)])
+
+ assert result.raised is None
+ assert result.requests[0].param("symbols") == expected_symbols
+
def test_eod_requests_ascending_day_interval_with_date_from_watermark(self) -> None:
result = _driver(symbols="AAPL").run(
"eod",
@@ -130,8 +193,66 @@ class TestRequestParams:
assert request.param("date_from") == "2021-04-09"
+class TestNewsStatsRowShaping:
+ @parameterized.expand(
+ [
+ (
+ "one_row_per_key_with_bucket_date",
+ [
+ {
+ "date": "2026-10-02T00:00:00.000Z",
+ "data": [
+ {"key": "TSLA", "total_documents": 4, "sentiment_avg": 0.25},
+ {"key": "AMZN", "total_documents": 2, "sentiment_avg": -0.5},
+ ],
+ },
+ {
+ "date": "2026-10-01T00:00:00.000Z",
+ "data": [{"key": "TSLA", "total_documents": 1, "sentiment_avg": 0.0}],
+ },
+ ],
+ [
+ {"date": "2026-10-02T00:00:00.000Z", "key": "TSLA", "total_documents": 4, "sentiment_avg": 0.25},
+ {"date": "2026-10-02T00:00:00.000Z", "key": "AMZN", "total_documents": 2, "sentiment_avg": -0.5},
+ {"date": "2026-10-01T00:00:00.000Z", "key": "TSLA", "total_documents": 1, "sentiment_avg": 0.0},
+ ],
+ ),
+ ("empty_bucket_yields_nothing", [{"date": "2026-10-02T00:00:00.000Z", "data": []}], []),
+ ("empty_object_result", {}, []),
+ ]
+ )
+ def test_intraday_buckets_explode_into_rows(
+ self, _name: str, data: Any, expected_rows: list[dict[str, Any]]
+ ) -> None:
+ result = _driver().run("news_stats_intraday", [_data_page(data)])
+
+ assert result.raised is None
+ assert result.rows == expected_rows
+
+ @parameterized.expand(
+ [
+ ("missing_data", {"date": "2026-10-03T00:00:00.000Z"}),
+ ("non_list_data", {"date": "2026-10-03T00:00:00.000Z", "data": None}),
+ ("non_dict_entry", {"date": "2026-10-04T00:00:00.000Z", "data": ["not-a-dict"]}),
+ ]
+ )
+ def test_malformed_bucket_fails_the_sync(self, _name: str, bucket: dict[str, Any]) -> None:
+ valid = {"date": "2026-10-05T00:00:00.000Z", "data": [{"key": "TSLA", "total_documents": 1}]}
+ result = _driver().run("news_stats_intraday", [_data_page([bucket, valid])])
+
+ # Dropping the bucket would let the later one advance the watermark past its date.
+ assert isinstance(result.raised, ValueError)
+ assert "malformed news stats bucket" in str(result.raised)
+
+ def test_aggregation_rows_pass_through_unchanged(self) -> None:
+ rows = [{"key": "TSLA", "total_documents": 55, "sentiment_avg": 0.33}]
+ result = _driver().run("news_stats_aggregation", [_data_page(rows)])
+
+ assert result.rows == rows
+
+
class TestRequiresSymbols:
- @parameterized.expand([("quote",), ("eod",), ("intraday",), ("dividends",), ("splits",)])
+ @parameterized.expand([("quote",), ("eod",), ("intraday",), ("dividends",), ("splits",), ("entity_search",)])
def test_price_endpoints_require_symbols(self, endpoint: str) -> None:
# Selecting a price table with no symbols is a permanent misconfiguration.
result = _driver(symbols=None).run(endpoint, [])
@@ -173,6 +294,10 @@ class TestSourceResponse:
("intraday", ["ticker", "date"], "date", "asc"),
("dividends", ["ticker", "date"], "date", "asc"),
("splits", ["ticker", "date"], "date", "asc"),
+ ("entity_search", ["symbol"], None, "asc"),
+ ("news_sources", ["source_id"], None, "asc"),
+ ("news_stats_intraday", ["date", "key"], "date", "asc"),
+ ("news_stats_aggregation", ["key"], None, "asc"),
]
)
def test_source_response_keys_partitioning_and_sort(