:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(bing_webmaster_tools): skip a non-json GetPageQueryStats reply (#129419)
Tom Owersca2744366fdf
diff --git a/products/warehouse_sources/sources/bing_webmaster_tools/bing_webmaster_tools.py b/products/warehouse_sources/sources/bing_webmaster_tools/bing_webmaster_tools.pyindex 04eea0349591..21bf22603416 100644--- a/products/warehouse_sources/sources/bing_webmaster_tools/bing_webmaster_tools.py+++ b/products/warehouse_sources/sources/bing_webmaster_tools/bing_webmaster_tools.py@@ -282,14 +282,20 @@ def _page_query_stats_batches(items = _request(session, api_key, "GetPageQueryStats", {"siteUrl": site_url, "page": json.dumps(page)})except BingWebmasterToolsError as exc:# GetPageStats and GetPageQueryStats can disagree about which of a site's pages are- # valid: Bing rejects some pages GetPageStats itself just listed. Skip that one page+ # valid: Bing rejects some pages GetPageStats itself just listed, either with an+ # ApiFault or with a non-JSON (often empty) body for that one page. Skip that one page# rather than losing every other page's stats to a page Bing won't serve.# `_request_payload` appends the fault message verbatim (observed as the API's own# "ERROR!!! InvalidUrl" wording), so anchor on the suffix rather than a bare substring- # match that could also catch an unrelated fault mentioning a URL.- if not str(exc).endswith("InvalidUrl"):+ # match that could also catch an unrelated fault mentioning a URL. The non-JSON message+ # names the method literally, so an exact match is just as safe.+ message = str(exc)+ if not (+ message.endswith("InvalidUrl")+ or message == "Bing Webmaster Tools GetPageQueryStats returned a non-JSON response"+ ):raise- logger.warning("Bing Webmaster Tools rejected a page as invalid", site_url=site_url, page=page)+ logger.warning("Bing Webmaster Tools rejected a page", site_url=site_url, page=page, reason=message)continuerows = _stats_rows(items, site_url, "page_query_stats", logger)for row in rows:diff --git a/products/warehouse_sources/sources/bing_webmaster_tools/tests/test_bing_webmaster_tools.py b/products/warehouse_sources/sources/bing_webmaster_tools/tests/test_bing_webmaster_tools.pyindex b2f0bd551871..54bbffa73b62 100644--- a/products/warehouse_sources/sources/bing_webmaster_tools/tests/test_bing_webmaster_tools.py+++ b/products/warehouse_sources/sources/bing_webmaster_tools/tests/test_bing_webmaster_tools.py@@ -277,7 +277,16 @@ class TestGetRows:("https://example.com/a", "query", "https://example.com/")]- def test_page_query_stats_skips_a_page_bing_rejects_as_invalid(self):+ @pytest.mark.parametrize(+ "rejected_page_response",+ [+ # Bing rejects a page GetPageStats itself just listed with an ApiFault.+ ScriptedResponse(status=400, json={"ErrorCode": 38, "Message": "ERROR!!! InvalidUrl"}),+ # Bing can also reject a page with a non-JSON (here, empty) body instead of a fault.+ ScriptedResponse(status=200, body=b""),+ ],+ )+ def test_page_query_stats_skips_a_page_bing_rejects(self, rejected_page_response):# GetPageStats and GetPageQueryStats can disagree about which pages are valid: Bing can# reject a page GetPageStats itself just listed. That page's stats should be skipped# without losing the other pages' data.@@ -290,7 +299,7 @@ class TestGetRows:[ScriptedResponse(json={"d": _SITES}),ScriptedResponse(json={"d": page_stats}),- ScriptedResponse(status=400, json={"ErrorCode": 38, "Message": "ERROR!!! InvalidUrl"}),+ rejected_page_response,ScriptedResponse(json={"d": _QUERY_STATS}),],)