Universal SEO skill for Claude Code. 26 sub-skills + 19 sub-agents covering technical SEO, E-E-A-T, schema, GEO/AEO, agent readiness (Lighthouse Agentic Browsing, WebMCP, llms.txt), backlinks, local SEO, e-commerce, international SEO, Google APIs, and PDF/Excel reporting. 9 optional extensions for live SEO data.
fix(url_safety): send browser-like default headers on safe fetches
Jaibharath484ce21501b2
diff --git a/scripts/url_safety.py b/scripts/url_safety.pyindex 21c3189..077bc07 100644--- a/scripts/url_safety.py+++ b/scripts/url_safety.py@@ -553,6 +553,34 @@ def _pin_dns(_dns_patch_lock.release()+# Default request headers, mirroring the browser-like defaults fetch_page.py+# has used since v1.2.1 (issue #9). Without them ``requests`` announces itself+# as ``User-Agent: python-requests/x.y.z`` with no Accept-Language, which many+# managed WAFs and CDNs answer with 403/406, and which SSR frameworks answer+# with the empty client-side shell. Callers do not see an exception in that+# case, they analyse the error page or the shell as if it were the real+# document. Any header here can be overridden by passing ``headers=``.+DEFAULT_REQUEST_HEADERS = {+ "User-Agent": (+ "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 "+ "(KHTML, like Gecko) Chrome/150.0.7871.115 Safari/537.36 ClaudeSEO/2.0"+ ),+ "Accept": (+ "text/html,application/xhtml+xml,application/xml;q=0.9,"+ "image/avif,image/webp,*/*;q=0.8"+ ),+ "Accept-Language": "en-US,en;q=0.9",+}+++def _with_default_headers(kwargs: dict) -> dict:+ """Fill in DEFAULT_REQUEST_HEADERS for header keys the caller did not set."""+ headers = dict(DEFAULT_REQUEST_HEADERS)+ headers.update(kwargs.get("headers") or {})+ kwargs["headers"] = headers+ return kwargs++def safe_requests_get(url: str,*,@@ -563,12 +591,15 @@ def safe_requests_get(``requests.get`` with DNS-rebinding protection.The request's hostname is pinned to a pre-validated IP for the- duration of the call. Standard ``requests`` semantics otherwise.+ duration of the call. Standard ``requests`` semantics otherwise,+ except that browser-like default headers are supplied for any header+ the caller did not set (see ``DEFAULT_REQUEST_HEADERS``)."""norm_url, pinned_ip = validate_url_strict(url)parsed = urlparse(norm_url)port = parsed.port or (443 if parsed.scheme == "https" else 80)assert parsed.hostname is not None # validate_url_strict guarantees this+ kwargs = _with_default_headers(kwargs)exempt = _validated_proxy_hosts(norm_url, kwargs.get("proxies"))with _pin_dns(parsed.hostname, pinned_ip, port, exempt_hosts=exempt):return requests.get(norm_url, timeout=timeout, **kwargs)@@ -584,12 +615,15 @@ def safe_requests_head(``requests.head`` with DNS-rebinding protection.The request's hostname is pinned to a pre-validated IP for the- duration of the call. Standard ``requests`` semantics otherwise.+ duration of the call. Standard ``requests`` semantics otherwise,+ except that browser-like default headers are supplied for any header+ the caller did not set (see ``DEFAULT_REQUEST_HEADERS``)."""norm_url, pinned_ip = validate_url_strict(url)parsed = urlparse(norm_url)port = parsed.port or (443 if parsed.scheme == "https" else 80)assert parsed.hostname is not None+ kwargs = _with_default_headers(kwargs)exempt = _validated_proxy_hosts(norm_url, kwargs.get("proxies"))with _pin_dns(parsed.hostname, pinned_ip, port, exempt_hosts=exempt):return requests.head(norm_url, timeout=timeout, **kwargs)diff --git a/tests/test_url_safety.py b/tests/test_url_safety.pyindex 0906704..46a2d2e 100644--- a/tests/test_url_safety.py+++ b/tests/test_url_safety.py@@ -350,11 +350,49 @@ def test_safe_requests_head_uses_strict_validation_and_dns_pin() -> None:"https://safe.example/path",timeout=7,allow_redirects=True,+ headers=url_safety.DEFAULT_REQUEST_HEADERS,)assert captured["pin"] == ("safe.example", "1.1.1.1", 443)assert result is response+def test_default_headers_are_browser_like() -> None:+ headers = url_safety.DEFAULT_REQUEST_HEADERS+ assert "python-requests" not in headers["User-Agent"]+ assert headers["User-Agent"].startswith("Mozilla/5.0")+ assert "Accept-Language" in headers+++def test_with_default_headers_fills_unset_headers() -> None:+ assert url_safety._with_default_headers({})["headers"] == (+ url_safety.DEFAULT_REQUEST_HEADERS+ )+ assert url_safety._with_default_headers({"headers": None})["headers"] == (+ url_safety.DEFAULT_REQUEST_HEADERS+ )+++def test_with_default_headers_lets_caller_override() -> None:+ # fetch_page.py --user-agent (Googlebot cloaking checks) must still win.+ merged = url_safety._with_default_headers(+ {"headers": {"User-Agent": "Googlebot/2.1"}}+ )["headers"]+ assert merged["User-Agent"] == "Googlebot/2.1"+ # Headers the caller did not set are still filled in.+ assert merged["Accept-Language"] == (+ url_safety.DEFAULT_REQUEST_HEADERS["Accept-Language"]+ )+++def test_with_default_headers_preserves_other_kwargs_and_constant() -> None:+ before = dict(url_safety.DEFAULT_REQUEST_HEADERS)+ kwargs = url_safety._with_default_headers({"stream": True, "allow_redirects": False})+ assert kwargs["stream"] is True+ assert kwargs["allow_redirects"] is False+ url_safety._with_default_headers({"headers": {"User-Agent": "mutating/1.0"}})+ assert url_safety.DEFAULT_REQUEST_HEADERS == before++def test_pin_dns_restores_getaddrinfo_on_exception() -> None:before = socket.getaddrinfowith pytest.raises(RuntimeError):