Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(data-warehouse): add 1 missing table to the NewsData source (#119981)

Tom Owers6a1a903b10b6
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
index 4e26e96a2e95..c5324702a9dc 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
@@ -8388,7 +8388,7 @@ Today (4): `archive`, `crypto`, `latest`, `sources`
Diffed against: <https://newsdata.io/llms.txt>
-- [ ] `/api/1/market` — stock-market and financial news with ticker-symbol filtering - a distinct corpus from latest/crypto and the obvious peer of the crypto table already synced (high)
+- [x] `/api/1/market` — stock-market and financial news with ticker-symbol filtering - a distinct corpus from latest/crypto and the obvious peer of the crypto table already synced (high)
- [ ] `/api/1/count` — pre-aggregated article counts per hour/day for a query, useful for coverage/share-of-voice trends without paging every article (low)
- [ ] `/api/1/crypto/count` — same aggregation over the crypto corpus already synced (low)
- [ ] `/api/1/market/count` — same aggregation over the market corpus (low)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/newsdata/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/newsdata/canonical_descriptions.py
index 484dc6ac81fb..af468f93d0d6 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/newsdata/canonical_descriptions.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/newsdata/canonical_descriptions.py
@@ -9,7 +9,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.can
CanonicalDescriptions,
)
-# The latest, archive, and crypto endpoints all return the same news-article shape.
+# The latest, archive, crypto, and market endpoints all return the same news-article shape.
_ARTICLE_COLUMNS = {
"article_id": "Unique identifier for the news article.",
"title": "The article's headline.",
@@ -41,6 +41,7 @@ _ARTICLE_COLUMNS = {
_LATEST_DOCS = "https://newsdata.io/documentation/#latest-news"
_ARCHIVE_DOCS = "https://newsdata.io/documentation/#news-archive"
_CRYPTO_DOCS = "https://newsdata.io/documentation/#crypto-news"
+_MARKET_DOCS = "https://newsdata.io/documentation"
CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {
"latest": {
@@ -61,6 +62,11 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {
"coin": "Cryptocurrency coins the article relates to (e.g. btc, eth).",
},
},
+ "market": {
+ "description": "Stock-market and financial news articles filterable by publish date and stock ticker.",
+ "docs_url": _MARKET_DOCS,
+ "columns": _ARTICLE_COLUMNS,
+ },
"sources": {
"description": "Catalog of news sources available through NewsData.io.",
"docs_url": "https://newsdata.io/documentation/#news-sources",
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/newsdata/newsdata.py b/products/warehouse_sources/backend/temporal/data_imports/sources/newsdata/newsdata.py
index 751219b87d9d..526805fb6a59 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/newsdata/newsdata.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/newsdata/newsdata.py
@@ -121,7 +121,7 @@ def get_rows(
],
}
- # Only date-filter endpoints (archive, crypto) accept from_date; on the first incremental sync
+ # Only date-filter endpoints (archive, crypto, market) accept from_date; on the first incremental sync
# the lookback floor seeds it, on later syncs the stored watermark advances it.
if config.supports_date_filter and should_use_incremental_field:
incremental: IncrementalConfig = {
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/newsdata/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/newsdata/settings.py
index 31b7f1175ea2..b46154d6f062 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/newsdata/settings.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/newsdata/settings.py
@@ -70,6 +70,16 @@ NEWSDATA_ENDPOINTS: dict[str, NewsDataEndpointConfig] = {
sort_mode="desc",
incremental_fields=_pub_date_incremental_fields(),
),
+ # Stock-market and financial news (labelled beta by the vendor). Same shape as crypto.
+ "market": NewsDataEndpointConfig(
+ name="market",
+ path="/market",
+ partition_key="pubDate",
+ supports_date_filter=True,
+ default_lookback_days=30,
+ sort_mode="desc",
+ incremental_fields=_pub_date_incremental_fields(),
+ ),
# Catalog of available news sources. Single-response, no pagination or date filter.
"sources": NewsDataEndpointConfig(
name="sources",
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/newsdata/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/newsdata/source.py
index cd808293d1b2..17763da5c144 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/newsdata/source.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/newsdata/source.py
@@ -106,7 +106,7 @@ Page size and daily request credits are determined by your NewsData.io plan.""",
) -> list[SourceSchema]:
def _build_schema(endpoint: str) -> SourceSchema:
endpoint_config = NEWSDATA_ENDPOINTS[endpoint]
- # Only date-filter endpoints (archive, crypto) have a genuine server-side timestamp filter.
+ # Only date-filter endpoints (archive, crypto, market) have a genuine server-side timestamp filter.
has_incremental = endpoint_config.supports_date_filter and bool(INCREMENTAL_FIELDS.get(endpoint))
return SourceSchema(
name=endpoint,
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/newsdata/tests/test_newsdata.py b/products/warehouse_sources/backend/temporal/data_imports/sources/newsdata/tests/test_newsdata.py
index e415720ab390..768dd26320ab 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/newsdata/tests/test_newsdata.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/newsdata/tests/test_newsdata.py
@@ -109,7 +109,7 @@ class TestIncrementalFromDate:
assert result.raised is None
assert result.params("from_date") == [None]
- @parameterized.expand([("archive",), ("crypto",)])
+ @parameterized.expand([("archive",), ("crypto",), ("market",)])
def test_incremental_uses_watermark_date(self, endpoint: str) -> None:
result = _driver().run(
endpoint,
@@ -119,9 +119,10 @@ class TestIncrementalFromDate:
)
assert result.raised is None
+ assert result.paths == [f"/api/1/{endpoint}"]
assert result.params("from_date") == ["2024-03-04"]
- @parameterized.expand([("archive",), ("crypto",)])
+ @parameterized.expand([("archive",), ("crypto",), ("market",)])
@time_machine.travel("2026-06-15T12:00:00Z", tick=False)
def test_first_sync_applies_lookback_floor(self, endpoint: str) -> None:
# Without a watermark the first sync must floor at the trailing lookback window instead of
@@ -180,6 +181,7 @@ class TestSourceResponseMetadata:
("latest", ["article_id"], "pubDate", "asc"),
("archive", ["article_id"], "pubDate", "desc"),
("crypto", ["article_id"], "pubDate", "desc"),
+ ("market", ["article_id"], "pubDate", "desc"),
]
)
def test_partitioned_endpoints(
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/newsdata/tests/test_newsdata_source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/newsdata/tests/test_newsdata_source.py
index 992e6cea32e8..cb9bb164e180 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/newsdata/tests/test_newsdata_source.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/newsdata/tests/test_newsdata_source.py
@@ -23,6 +23,7 @@ class TestGetSchemas:
("latest", False),
("archive", True),
("crypto", True),
+ ("market", True),
("sources", False),
]
)