:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(data-warehouse): add 1 missing table to the NewsData source (#119981)
Tom Owers6a1a903b10b6
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.mdindex 4e26e96a2e95..c5324702a9dc 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md@@ -8388,7 +8388,7 @@ Today (4): `archive`, `crypto`, `latest`, `sources`Diffed against: <https://newsdata.io/llms.txt>-- [ ] `/api/1/market` — stock-market and financial news with ticker-symbol filtering - a distinct corpus from latest/crypto and the obvious peer of the crypto table already synced (high)+- [x] `/api/1/market` — stock-market and financial news with ticker-symbol filtering - a distinct corpus from latest/crypto and the obvious peer of the crypto table already synced (high)- [ ] `/api/1/count` — pre-aggregated article counts per hour/day for a query, useful for coverage/share-of-voice trends without paging every article (low)- [ ] `/api/1/crypto/count` — same aggregation over the crypto corpus already synced (low)- [ ] `/api/1/market/count` — same aggregation over the market corpus (low)diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/newsdata/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/newsdata/canonical_descriptions.pyindex 484dc6ac81fb..af468f93d0d6 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/newsdata/canonical_descriptions.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/newsdata/canonical_descriptions.py@@ -9,7 +9,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.canCanonicalDescriptions,)-# The latest, archive, and crypto endpoints all return the same news-article shape.+# The latest, archive, crypto, and market endpoints all return the same news-article shape._ARTICLE_COLUMNS = {"article_id": "Unique identifier for the news article.","title": "The article's headline.",@@ -41,6 +41,7 @@ _ARTICLE_COLUMNS = {_LATEST_DOCS = "https://newsdata.io/documentation/#latest-news"_ARCHIVE_DOCS = "https://newsdata.io/documentation/#news-archive"_CRYPTO_DOCS = "https://newsdata.io/documentation/#crypto-news"+_MARKET_DOCS = "https://newsdata.io/documentation"CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {"latest": {@@ -61,6 +62,11 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {"coin": "Cryptocurrency coins the article relates to (e.g. btc, eth).",},},+ "market": {+ "description": "Stock-market and financial news articles filterable by publish date and stock ticker.",+ "docs_url": _MARKET_DOCS,+ "columns": _ARTICLE_COLUMNS,+ },"sources": {"description": "Catalog of news sources available through NewsData.io.","docs_url": "https://newsdata.io/documentation/#news-sources",diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/newsdata/newsdata.py b/products/warehouse_sources/backend/temporal/data_imports/sources/newsdata/newsdata.pyindex 751219b87d9d..526805fb6a59 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/newsdata/newsdata.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/newsdata/newsdata.py@@ -121,7 +121,7 @@ def get_rows(],}- # Only date-filter endpoints (archive, crypto) accept from_date; on the first incremental sync+ # Only date-filter endpoints (archive, crypto, market) accept from_date; on the first incremental sync# the lookback floor seeds it, on later syncs the stored watermark advances it.if config.supports_date_filter and should_use_incremental_field:incremental: IncrementalConfig = {diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/newsdata/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/newsdata/settings.pyindex 31b7f1175ea2..b46154d6f062 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/newsdata/settings.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/newsdata/settings.py@@ -70,6 +70,16 @@ NEWSDATA_ENDPOINTS: dict[str, NewsDataEndpointConfig] = {sort_mode="desc",incremental_fields=_pub_date_incremental_fields(),),+ # Stock-market and financial news (labelled beta by the vendor). Same shape as crypto.+ "market": NewsDataEndpointConfig(+ name="market",+ path="/market",+ partition_key="pubDate",+ supports_date_filter=True,+ default_lookback_days=30,+ sort_mode="desc",+ incremental_fields=_pub_date_incremental_fields(),+ ),# Catalog of available news sources. Single-response, no pagination or date filter."sources": NewsDataEndpointConfig(name="sources",diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/newsdata/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/newsdata/source.pyindex cd808293d1b2..17763da5c144 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/newsdata/source.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/newsdata/source.py@@ -106,7 +106,7 @@ Page size and daily request credits are determined by your NewsData.io plan.""",) -> list[SourceSchema]:def _build_schema(endpoint: str) -> SourceSchema:endpoint_config = NEWSDATA_ENDPOINTS[endpoint]- # Only date-filter endpoints (archive, crypto) have a genuine server-side timestamp filter.+ # Only date-filter endpoints (archive, crypto, market) have a genuine server-side timestamp filter.has_incremental = endpoint_config.supports_date_filter and bool(INCREMENTAL_FIELDS.get(endpoint))return SourceSchema(name=endpoint,diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/newsdata/tests/test_newsdata.py b/products/warehouse_sources/backend/temporal/data_imports/sources/newsdata/tests/test_newsdata.pyindex e415720ab390..768dd26320ab 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/newsdata/tests/test_newsdata.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/newsdata/tests/test_newsdata.py@@ -109,7 +109,7 @@ class TestIncrementalFromDate:assert result.raised is Noneassert result.params("from_date") == [None]- @parameterized.expand([("archive",), ("crypto",)])+ @parameterized.expand([("archive",), ("crypto",), ("market",)])def test_incremental_uses_watermark_date(self, endpoint: str) -> None:result = _driver().run(endpoint,@@ -119,9 +119,10 @@ class TestIncrementalFromDate:)assert result.raised is None+ assert result.paths == [f"/api/1/{endpoint}"]assert result.params("from_date") == ["2024-03-04"]- @parameterized.expand([("archive",), ("crypto",)])+ @parameterized.expand([("archive",), ("crypto",), ("market",)])@time_machine.travel("2026-06-15T12:00:00Z", tick=False)def test_first_sync_applies_lookback_floor(self, endpoint: str) -> None:# Without a watermark the first sync must floor at the trailing lookback window instead of@@ -180,6 +181,7 @@ class TestSourceResponseMetadata:("latest", ["article_id"], "pubDate", "asc"),("archive", ["article_id"], "pubDate", "desc"),("crypto", ["article_id"], "pubDate", "desc"),+ ("market", ["article_id"], "pubDate", "desc"),])def test_partitioned_endpoints(diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/newsdata/tests/test_newsdata_source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/newsdata/tests/test_newsdata_source.pyindex 992e6cea32e8..cb9bb164e180 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/newsdata/tests/test_newsdata_source.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/newsdata/tests/test_newsdata_source.py@@ -23,6 +23,7 @@ class TestGetSchemas:("latest", False),("archive", True),("crypto", True),+ ("market", True),("sources", False),])