Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(data-imports): skip invalid pages in bing webmaster sync (#119733)

Tom Owerse65164d307fd
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/bing_webmaster_tools/bing_webmaster_tools.py b/products/warehouse_sources/backend/temporal/data_imports/sources/bing_webmaster_tools/bing_webmaster_tools.py
index 099e942c9d38..853620e9ca56 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/bing_webmaster_tools/bing_webmaster_tools.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/bing_webmaster_tools/bing_webmaster_tools.py
@@ -282,7 +282,19 @@ def _page_query_stats_batches(
pages = list(dict.fromkeys(page for item in page_stats if isinstance(page := item.get("Query"), str) and page))
for page in pages:
# The JSON endpoint documents `page` as a quoted JSON string literal, unlike `siteUrl`.
- items = _request(session, api_key, "GetPageQueryStats", {"siteUrl": site_url, "page": json.dumps(page)})
+ try:
+ items = _request(session, api_key, "GetPageQueryStats", {"siteUrl": site_url, "page": json.dumps(page)})
+ except BingWebmasterToolsError as exc:
+ # GetPageStats and GetPageQueryStats can disagree about which of a site's pages are
+ # valid: Bing rejects some pages GetPageStats itself just listed. Skip that one page
+ # rather than losing every other page's stats to a page Bing won't serve.
+ # `_request_payload` appends the fault message verbatim (observed as the API's own
+ # "ERROR!!! InvalidUrl" wording), so anchor on the suffix rather than a bare substring
+ # match that could also catch an unrelated fault mentioning a URL.
+ if not str(exc).endswith("InvalidUrl"):
+ raise
+ logger.warning("Bing Webmaster Tools rejected a page as invalid", site_url=site_url, page=page)
+ continue
rows = _stats_rows(items, site_url, "page_query_stats", logger)
for row in rows:
row["page"] = page
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/bing_webmaster_tools/tests/test_bing_webmaster_tools.py b/products/warehouse_sources/backend/temporal/data_imports/sources/bing_webmaster_tools/tests/test_bing_webmaster_tools.py
index fe2cbf81ad89..5ca116d05103 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/bing_webmaster_tools/tests/test_bing_webmaster_tools.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/bing_webmaster_tools/tests/test_bing_webmaster_tools.py
@@ -286,6 +286,29 @@ class TestGetRows:
("https://example.com/a", "query", "https://example.com/")
]
+ def test_page_query_stats_skips_a_page_bing_rejects_as_invalid(self):
+ # GetPageStats and GetPageQueryStats can disagree about which pages are valid: Bing can
+ # reject a page GetPageStats itself just listed. That page's stats should be skipped
+ # without losing the other pages' data.
+ page_stats = [
+ {**_QUERY_STATS[0], "Query": "https://example.com/a"},
+ {**_QUERY_STATS[0], "Query": "https://example.com/b"},
+ ]
+ result = self._driver().run(
+ "page_query_stats",
+ [
+ ScriptedResponse(json={"d": _SITES}),
+ ScriptedResponse(json={"d": page_stats}),
+ ScriptedResponse(status=400, json={"ErrorCode": 38, "Message": "ERROR!!! InvalidUrl"}),
+ ScriptedResponse(json={"d": _QUERY_STATS}),
+ ],
+ )
+
+ assert result.raised is None
+ assert [(row["page"], row["query"], row["site_url"]) for row in result.rows] == [
+ ("https://example.com/b", "query", "https://example.com/")
+ ]
+
def test_link_counts_walk_every_page(self):
def links_page(url: str) -> ScriptedResponse:
return ScriptedResponse(