Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(bing_webmaster_tools): skip a non-json GetPageQueryStats reply (#129419)

Tom Owersca2744366fdf
diff --git a/products/warehouse_sources/sources/bing_webmaster_tools/bing_webmaster_tools.py b/products/warehouse_sources/sources/bing_webmaster_tools/bing_webmaster_tools.py
index 04eea0349591..21bf22603416 100644
--- a/products/warehouse_sources/sources/bing_webmaster_tools/bing_webmaster_tools.py
+++ b/products/warehouse_sources/sources/bing_webmaster_tools/bing_webmaster_tools.py
@@ -282,14 +282,20 @@ def _page_query_stats_batches(
items = _request(session, api_key, "GetPageQueryStats", {"siteUrl": site_url, "page": json.dumps(page)})
except BingWebmasterToolsError as exc:
# GetPageStats and GetPageQueryStats can disagree about which of a site's pages are
- # valid: Bing rejects some pages GetPageStats itself just listed. Skip that one page
+ # valid: Bing rejects some pages GetPageStats itself just listed, either with an
+ # ApiFault or with a non-JSON (often empty) body for that one page. Skip that one page
# rather than losing every other page's stats to a page Bing won't serve.
# `_request_payload` appends the fault message verbatim (observed as the API's own
# "ERROR!!! InvalidUrl" wording), so anchor on the suffix rather than a bare substring
- # match that could also catch an unrelated fault mentioning a URL.
- if not str(exc).endswith("InvalidUrl"):
+ # match that could also catch an unrelated fault mentioning a URL. The non-JSON message
+ # names the method literally, so an exact match is just as safe.
+ message = str(exc)
+ if not (
+ message.endswith("InvalidUrl")
+ or message == "Bing Webmaster Tools GetPageQueryStats returned a non-JSON response"
+ ):
raise
- logger.warning("Bing Webmaster Tools rejected a page as invalid", site_url=site_url, page=page)
+ logger.warning("Bing Webmaster Tools rejected a page", site_url=site_url, page=page, reason=message)
continue
rows = _stats_rows(items, site_url, "page_query_stats", logger)
for row in rows:
diff --git a/products/warehouse_sources/sources/bing_webmaster_tools/tests/test_bing_webmaster_tools.py b/products/warehouse_sources/sources/bing_webmaster_tools/tests/test_bing_webmaster_tools.py
index b2f0bd551871..54bbffa73b62 100644
--- a/products/warehouse_sources/sources/bing_webmaster_tools/tests/test_bing_webmaster_tools.py
+++ b/products/warehouse_sources/sources/bing_webmaster_tools/tests/test_bing_webmaster_tools.py
@@ -277,7 +277,16 @@ class TestGetRows:
("https://example.com/a", "query", "https://example.com/")
]
- def test_page_query_stats_skips_a_page_bing_rejects_as_invalid(self):
+ @pytest.mark.parametrize(
+ "rejected_page_response",
+ [
+ # Bing rejects a page GetPageStats itself just listed with an ApiFault.
+ ScriptedResponse(status=400, json={"ErrorCode": 38, "Message": "ERROR!!! InvalidUrl"}),
+ # Bing can also reject a page with a non-JSON (here, empty) body instead of a fault.
+ ScriptedResponse(status=200, body=b""),
+ ],
+ )
+ def test_page_query_stats_skips_a_page_bing_rejects(self, rejected_page_response):
# GetPageStats and GetPageQueryStats can disagree about which pages are valid: Bing can
# reject a page GetPageStats itself just listed. That page's stats should be skipped
# without losing the other pages' data.
@@ -290,7 +299,7 @@ class TestGetRows:
[
ScriptedResponse(json={"d": _SITES}),
ScriptedResponse(json={"d": page_stats}),
- ScriptedResponse(status=400, json={"ErrorCode": 38, "Message": "ERROR!!! InvalidUrl"}),
+ rejected_page_response,
ScriptedResponse(json={"d": _QUERY_STATS}),
],
)