Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(data-warehouse): add 3 missing tables to the bing webmaster tools source (#118994)

Tom Owerscc8e42a7d348
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
index b16eb992c6a4..4f0e0a1033c4 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
@@ -1640,14 +1640,14 @@ Note: The official API index and individual reference pages supplied explicit GE
## Bing Webmaster Tools — gaps
-Today (5): `crawl_stats`, `page_stats`, `query_stats`, `rank_and_traffic_stats`, `sites`
+Today (8): `crawl_issues`, `crawl_stats`, `link_counts`, `page_query_stats`, `page_stats`, `query_stats`, `rank_and_traffic_stats`, `sites`
Diffed against: <https://learn.microsoft.com/en-us/dotnet/api/microsoft.bing.webmaster.api.interfaces.iwebmasterapi>
-- [ ] `GetPageQueryStats (GET /webmaster/api.svc/json/GetPageQueryStats)` — connect search queries to landing pages, which separate page and query totals cannot show (high)
-- [ ] `GetQueryPageDetailStats (GET /webmaster/api.svc/json/GetQueryPageDetailStats)` — analyze clicks and impressions by ranking position for each query and page (medium)
-- [ ] `GetCrawlIssues (GET /webmaster/api.svc/json/GetCrawlIssues)` — identify URLs with crawl failures and compare those failures with traffic changes (medium)
-- [ ] `GetLinkCounts (GET /webmaster/api.svc/json/GetLinkCounts)` — measure inbound link counts for each site page (medium)
+- [x] `GetPageQueryStats (GET /webmaster/api.svc/json/GetPageQueryStats)` — connect search queries to landing pages, which separate page and query totals cannot show (high). Added as `page_query_stats` (fan-out over the top pages GetPageStats returns; off by default because it makes one call per page).
+- ~~`GetQueryPageDetailStats (GET /webmaster/api.svc/json/GetQueryPageDetailStats)`~~ — skipped: it takes one query and one page per call, so a table needs one request per query-page pair on top of the `page_query_stats` fan-out
+- [x] `GetCrawlIssues (GET /webmaster/api.svc/json/GetCrawlIssues)` — identify URLs with crawl failures and compare those failures with traffic changes (medium). Added as `crawl_issues` (full refresh).
+- [x] `GetLinkCounts (GET /webmaster/api.svc/json/GetLinkCounts)` — measure inbound link counts for each site page (medium). Added as `link_counts` (full refresh).
- [ ] `GetUrlLinks (GET /webmaster/api.svc/json/GetUrlLinks)` — analyze referring pages and anchor text for each linked page (medium)
Note: IWebmasterApi lists methods; individual method pages confirm GET paths through JSON request examples. Current stats cover site, page, and query totals but omit the intersection of queries and pages. Detailed rankings and backlinks need additional calls for each page or query. Crawl settings, URL submissions, quotas, and health operations are excluded.
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/bing_webmaster_tools/bing_webmaster_tools.py b/products/warehouse_sources/backend/temporal/data_imports/sources/bing_webmaster_tools/bing_webmaster_tools.py
index 43925bc21ce7..099e942c9d38 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/bing_webmaster_tools/bing_webmaster_tools.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/bing_webmaster_tools/bing_webmaster_tools.py
@@ -1,4 +1,5 @@
import re
+import json
import datetime as dt
from collections.abc import Iterator
from typing import Any
@@ -78,13 +79,13 @@ def _api_fault_message(response: requests.Response) -> str | None:
return None
-def _request(
+def _request_payload(
session: requests.Session,
api_key: str,
method: str,
params: dict[str, str] | None = None,
timeout: int = REQUEST_TIMEOUT_SECONDS,
-) -> list[dict[str, Any]]:
+) -> Any:
try:
response = session.get(
f"{BASE_URL}/{method}",
@@ -115,8 +116,18 @@ def _request(
except ValueError:
raise BingWebmasterToolsError(f"Bing Webmaster Tools {method} returned a non-JSON response")
- # Successful responses wrap the result list in WCF's `d` envelope: `{"d": [...]}`.
- data = payload.get("d") if isinstance(payload, dict) else None
+ # Successful responses wrap the result in WCF's `d` envelope: `{"d": [...]}`.
+ return payload.get("d") if isinstance(payload, dict) else None
+
+
+def _request(
+ session: requests.Session,
+ api_key: str,
+ method: str,
+ params: dict[str, str] | None = None,
+ timeout: int = REQUEST_TIMEOUT_SECONDS,
+) -> list[dict[str, Any]]:
+ data = _request_payload(session, api_key, method, params, timeout)
if not isinstance(data, list):
raise BingWebmasterToolsError(f"Bing Webmaster Tools {method} returned an unexpected response shape")
return [item for item in data if isinstance(item, dict)]
@@ -214,6 +225,12 @@ def _site_row(site: dict[str, Any]) -> dict[str, Any]:
return {"url": site.get("Url"), "is_verified": bool(site.get("IsVerified", False))}
+def _normalized_row(item: dict[str, Any], site_url: str) -> dict[str, Any]:
+ row = {NamingConvention.normalize_identifier(key): value for key, value in item.items() if key != "__type"}
+ row["site_url"] = site_url
+ return row
+
+
def _stats_row(item: dict[str, Any], site_url: str) -> dict[str, Any] | None:
"""Turn one API stats object into a row: snake_case keys, parsed date, stamped site URL.
@@ -223,15 +240,73 @@ def _stats_row(item: dict[str, Any], site_url: str) -> dict[str, Any] | None:
should fail loudly, while a present-but-malformed value skips the row so a null key never flows
into the merge.
"""
- row = {NamingConvention.normalize_identifier(key): value for key, value in item.items() if key != "__type"}
+ row = _normalized_row(item, site_url)
date = parse_wcf_date(row["date"])
if date is None:
return None
row["date"] = date
- row["site_url"] = site_url
return row
+def _stats_rows(
+ items: list[dict[str, Any]], site_url: str, endpoint: str, logger: FilteringBoundLogger
+) -> list[dict[str, Any]]:
+ rows = []
+ dropped = 0
+ for item in items:
+ row = _stats_row(item, site_url)
+ if row is None:
+ dropped += 1
+ else:
+ rows.append(row)
+ if dropped:
+ logger.warning(
+ "Dropped Bing Webmaster Tools rows with unparseable dates",
+ endpoint=endpoint,
+ site_url=site_url,
+ dropped=dropped,
+ )
+ return rows
+
+
+def _url_rows(items: list[dict[str, Any]], site_url: str) -> list[dict[str, Any]]:
+ # `url` is part of the primary key, so entries without one are skipped.
+ return [_normalized_row(item, site_url) for item in items if item.get("Url")]
+
+
+def _page_query_stats_batches(
+ session: requests.Session, api_key: str, site_url: str, logger: FilteringBoundLogger
+) -> Iterator[list[dict[str, Any]]]:
+ """Fan out GetPageQueryStats over the site's top pages, as listed by GetPageStats."""
+ page_stats = _request(session, api_key, "GetPageStats", {"siteUrl": site_url})
+ pages = list(dict.fromkeys(page for item in page_stats if isinstance(page := item.get("Query"), str) and page))
+ for page in pages:
+ # The JSON endpoint documents `page` as a quoted JSON string literal, unlike `siteUrl`.
+ items = _request(session, api_key, "GetPageQueryStats", {"siteUrl": site_url, "page": json.dumps(page)})
+ rows = _stats_rows(items, site_url, "page_query_stats", logger)
+ for row in rows:
+ row["page"] = page
+ if rows:
+ yield rows
+
+
+def _link_count_batches(session: requests.Session, api_key: str, site_url: str) -> Iterator[list[dict[str, Any]]]:
+ """Walk GetLinkCounts, which pages through `{"Links": [...], "TotalPages": n}` with a zero-based `page`."""
+ page = 0
+ while True:
+ data = _request_payload(session, api_key, "GetLinkCounts", {"siteUrl": site_url, "page": str(page)})
+ links = data.get("Links") if isinstance(data, dict) else None
+ if not isinstance(links, list):
+ raise BingWebmasterToolsError("Bing Webmaster Tools GetLinkCounts returned an unexpected response shape")
+ rows = _url_rows([link for link in links if isinstance(link, dict)], site_url)
+ if rows:
+ yield rows
+ page += 1
+ total_pages = data.get("TotalPages")
+ if not links or not isinstance(total_pages, int) or page >= total_pages:
+ return
+
+
def get_rows(
api_key: str,
endpoint: str,
@@ -251,22 +326,18 @@ def get_rows(
return
for site_url in select_site_urls(sites, parse_site_urls(site_urls_raw)):
+ if config.method == "GetPageQueryStats":
+ yield from _page_query_stats_batches(session, api_key, site_url, logger)
+ continue
+ if config.method == "GetLinkCounts":
+ yield from _link_count_batches(session, api_key, site_url)
+ continue
+
items = _request(session, api_key, config.method, {"siteUrl": site_url})
- rows = []
- dropped = 0
- for item in items:
- row = _stats_row(item, site_url)
- if row is None:
- dropped += 1
- else:
- rows.append(row)
- if dropped:
- logger.warning(
- "Dropped Bing Webmaster Tools rows with unparseable dates",
- endpoint=endpoint,
- site_url=site_url,
- dropped=dropped,
- )
+ if config.incremental_fields:
+ rows = _stats_rows(items, site_url, endpoint, logger)
+ else:
+ rows = _url_rows(items, site_url)
if rows:
yield rows
@@ -316,7 +387,7 @@ def bing_webmaster_tools_source(
) -> SourceResponse:
config = ENDPOINT_CONFIGS[endpoint]
- if not config.per_site:
+ if not config.incremental_fields:
return SourceResponse(
name=endpoint,
items=lambda: get_rows(api_key=api_key, endpoint=endpoint, site_urls_raw=site_urls_raw, logger=logger),
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/bing_webmaster_tools/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/bing_webmaster_tools/canonical_descriptions.py
index 1f802eed8a20..aee7eb111261 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/bing_webmaster_tools/canonical_descriptions.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/bing_webmaster_tools/canonical_descriptions.py
@@ -90,4 +90,42 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {
**_SITE_URL_COLUMN,
},
},
+ "page_query_stats": {
+ "description": (
+ "Traffic statistics for each search query that led to each of the site's top pages on Bing. "
+ "Bing updates this data weekly and retains about six months."
+ ),
+ "docs_url": f"{_API_DOCS_BASE}.getpagequerystats",
+ "columns": {
+ "page": "The page URL the statistics are for, taken from the site's top pages.",
+ **_DATE_COLUMN,
+ "query": "The search query users typed before they saw or clicked the page.",
+ **_QUERY_STATS_METRIC_COLUMNS,
+ **_SITE_URL_COLUMN,
+ },
+ },
+ "crawl_issues": {
+ "description": "The site's URLs that currently have crawl issues in Bing.",
+ "docs_url": f"{_API_DOCS_BASE}.getcrawlissues",
+ "columns": {
+ "url": "The URL that has crawl issues.",
+ "http_code": "The HTTP status code Bingbot received for the URL.",
+ "issues": (
+ "Bit flags for the issue types found: 1 = HTTP 301, 2 = HTTP 302, 4 = HTTP 4xx, "
+ "8 = HTTP 5xx, 16 = blocked by robots.txt, 32 = contains malware, 64 = important URL "
+ "blocked by robots.txt, 128 = DNS errors, 256 = timeout errors."
+ ),
+ "in_links": "Number of inbound links Bing found to the URL.",
+ **_SITE_URL_COLUMN,
+ },
+ },
+ "link_counts": {
+ "description": "The site's pages that have inbound links, with the number of inbound links for each.",
+ "docs_url": f"{_API_DOCS_BASE}.getlinkcounts",
+ "columns": {
+ "url": "The page URL on the site.",
+ "count": "Number of inbound links Bing found to the page.",
+ **_SITE_URL_COLUMN,
+ },
+ },
}
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/bing_webmaster_tools/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/bing_webmaster_tools/settings.py
index 36c8fe4caadf..009aa09d5f6a 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/bing_webmaster_tools/settings.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/bing_webmaster_tools/settings.py
@@ -72,6 +72,32 @@ ENDPOINT_CONFIGS: dict[str, BingWebmasterToolsEndpoint] = {
"counts, and pages in the index. Bing retains about six months."
),
),
+ "page_query_stats": BingWebmasterToolsEndpoint(
+ method="GetPageQueryStats",
+ primary_keys=["site_url", "page", "date", "query"],
+ # One API call per top page of every site, so it stays opt-in.
+ should_sync_default=False,
+ description=(
+ "Traffic statistics for each search query that led to each of the site's top pages: "
+ "impressions, clicks, and average positions. Bing updates this data weekly and retains "
+ "about six months."
+ ),
+ ),
+ "crawl_issues": BingWebmasterToolsEndpoint(
+ method="GetCrawlIssues",
+ primary_keys=["site_url", "url"],
+ incremental_fields=[],
+ description=(
+ "The site's URLs that currently have crawl issues, with the HTTP status Bingbot received "
+ "and the issue types found. Fixed issues can take a few days to drop off."
+ ),
+ ),
+ "link_counts": BingWebmasterToolsEndpoint(
+ method="GetLinkCounts",
+ primary_keys=["site_url", "url"],
+ incremental_fields=[],
+ description="The site's pages that have inbound links, with the number of inbound links Bing found for each.",
+ ),
}
ENDPOINTS = tuple(ENDPOINT_CONFIGS.keys())
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/bing_webmaster_tools/tests/test_bing_webmaster_tools.py b/products/warehouse_sources/backend/temporal/data_imports/sources/bing_webmaster_tools/tests/test_bing_webmaster_tools.py
index 22ca1dfe5fbd..fe2cbf81ad89 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/bing_webmaster_tools/tests/test_bing_webmaster_tools.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/bing_webmaster_tools/tests/test_bing_webmaster_tools.py
@@ -258,6 +258,94 @@ class TestGetRows:
]
]
+ def test_page_query_stats_fan_out_over_distinct_top_pages(self):
+ page_stats = [
+ {**_QUERY_STATS[0], "Query": "https://example.com/a"},
+ {**_QUERY_STATS[0], "Query": "https://example.com/a", "Date": "/Date(1316761200000-0700)/"},
+ {**_QUERY_STATS[0], "Query": "https://example.com/b"},
+ ]
+ result = self._driver().run(
+ "page_query_stats",
+ [
+ ScriptedResponse(json={"d": _SITES}),
+ ScriptedResponse(json={"d": page_stats}),
+ ScriptedResponse(json={"d": _QUERY_STATS}),
+ ScriptedResponse(json={"d": []}),
+ ],
+ )
+
+ assert result.raised is None
+ assert result.paths == [
+ "/webmaster/api.svc/json/GetUserSites",
+ "/webmaster/api.svc/json/GetPageStats",
+ "/webmaster/api.svc/json/GetPageQueryStats",
+ "/webmaster/api.svc/json/GetPageQueryStats",
+ ]
+ assert [r.param("page") for r in result.requests[2:]] == ['"https://example.com/a"', '"https://example.com/b"']
+ assert [(row["page"], row["query"], row["site_url"]) for row in result.rows] == [
+ ("https://example.com/a", "query", "https://example.com/")
+ ]
+
+ def test_link_counts_walk_every_page(self):
+ def links_page(url: str) -> ScriptedResponse:
+ return ScriptedResponse(
+ json={
+ "d": {
+ "__type": "LinkCounts:#Microsoft.Bing.Webmaster.Api",
+ "Links": [{"__type": "LinkCount:#Microsoft.Bing.Webmaster.Api", "Count": 14, "Url": url}],
+ "TotalPages": 2,
+ }
+ }
+ )
+
+ result = self._driver().run(
+ "link_counts",
+ [
+ ScriptedResponse(json={"d": _SITES}),
+ links_page("https://example.com/a"),
+ links_page("https://example.com/b"),
+ ],
+ )
+
+ assert result.raised is None
+ assert [r.param("page") for r in result.requests[1:]] == ["0", "1"]
+ assert result.rows == [
+ {"count": 14, "url": "https://example.com/a", "site_url": "https://example.com/"},
+ {"count": 14, "url": "https://example.com/b", "site_url": "https://example.com/"},
+ ]
+
+ def test_crawl_issues_rows_skip_entries_without_url(self):
+ issues = [
+ {
+ "__type": "UrlWithCrawlIssues:#Microsoft.Bing.Webmaster.Api",
+ "HttpCode": 200,
+ "Issues": 32,
+ "Url": "https://example.com/a",
+ "InLinks": 10,
+ },
+ {
+ "__type": "UrlWithCrawlIssues:#Microsoft.Bing.Webmaster.Api",
+ "HttpCode": 404,
+ "Issues": 4,
+ "Url": None,
+ "InLinks": 0,
+ },
+ ]
+ result = self._driver().run(
+ "crawl_issues", [ScriptedResponse(json={"d": _SITES}), ScriptedResponse(json={"d": issues})]
+ )
+
+ assert result.raised is None
+ assert result.rows == [
+ {
+ "http_code": 200,
+ "issues": 32,
+ "url": "https://example.com/a",
+ "in_links": 10,
+ "site_url": "https://example.com/",
+ }
+ ]
+
def test_rows_with_unparseable_dates_are_dropped(self):
bad_row = {**_QUERY_STATS[0], "Date": "garbage", "Query": "other"}
result = self._driver().run(