:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(data-imports): skip invalid pages in bing webmaster sync (#119733)
Tom Owerse65164d307fd
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/bing_webmaster_tools/bing_webmaster_tools.py b/products/warehouse_sources/backend/temporal/data_imports/sources/bing_webmaster_tools/bing_webmaster_tools.pyindex 099e942c9d38..853620e9ca56 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/bing_webmaster_tools/bing_webmaster_tools.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/bing_webmaster_tools/bing_webmaster_tools.py@@ -282,7 +282,19 @@ def _page_query_stats_batches(pages = list(dict.fromkeys(page for item in page_stats if isinstance(page := item.get("Query"), str) and page))for page in pages:# The JSON endpoint documents `page` as a quoted JSON string literal, unlike `siteUrl`.- items = _request(session, api_key, "GetPageQueryStats", {"siteUrl": site_url, "page": json.dumps(page)})+ try:+ items = _request(session, api_key, "GetPageQueryStats", {"siteUrl": site_url, "page": json.dumps(page)})+ except BingWebmasterToolsError as exc:+ # GetPageStats and GetPageQueryStats can disagree about which of a site's pages are+ # valid: Bing rejects some pages GetPageStats itself just listed. Skip that one page+ # rather than losing every other page's stats to a page Bing won't serve.+ # `_request_payload` appends the fault message verbatim (observed as the API's own+ # "ERROR!!! InvalidUrl" wording), so anchor on the suffix rather than a bare substring+ # match that could also catch an unrelated fault mentioning a URL.+ if not str(exc).endswith("InvalidUrl"):+ raise+ logger.warning("Bing Webmaster Tools rejected a page as invalid", site_url=site_url, page=page)+ continuerows = _stats_rows(items, site_url, "page_query_stats", logger)for row in rows:row["page"] = pagediff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/bing_webmaster_tools/tests/test_bing_webmaster_tools.py b/products/warehouse_sources/backend/temporal/data_imports/sources/bing_webmaster_tools/tests/test_bing_webmaster_tools.pyindex fe2cbf81ad89..5ca116d05103 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/bing_webmaster_tools/tests/test_bing_webmaster_tools.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/bing_webmaster_tools/tests/test_bing_webmaster_tools.py@@ -286,6 +286,29 @@ class TestGetRows:("https://example.com/a", "query", "https://example.com/")]+ def test_page_query_stats_skips_a_page_bing_rejects_as_invalid(self):+ # GetPageStats and GetPageQueryStats can disagree about which pages are valid: Bing can+ # reject a page GetPageStats itself just listed. That page's stats should be skipped+ # without losing the other pages' data.+ page_stats = [+ {**_QUERY_STATS[0], "Query": "https://example.com/a"},+ {**_QUERY_STATS[0], "Query": "https://example.com/b"},+ ]+ result = self._driver().run(+ "page_query_stats",+ [+ ScriptedResponse(json={"d": _SITES}),+ ScriptedResponse(json={"d": page_stats}),+ ScriptedResponse(status=400, json={"ErrorCode": 38, "Message": "ERROR!!! InvalidUrl"}),+ ScriptedResponse(json={"d": _QUERY_STATS}),+ ],+ )++ assert result.raised is None+ assert [(row["page"], row["query"], row["site_url"]) for row in result.rows] == [+ ("https://example.com/b", "query", "https://example.com/")+ ]+def test_link_counts_walk_every_page(self):def links_page(url: str) -> ScriptedResponse:return ScriptedResponse(