Skip to content

Universal SEO skill for Claude Code. 26 sub-skills + 19 sub-agents covering technical SEO, E-E-A-T, schema, GEO/AEO, agent readiness (Lighthouse Agentic Browsing, WebMCP, llms.txt), backlinks, local SEO, e-commerce, international SEO, Google APIs, and PDF/Excel reporting. 9 optional extensions for live SEO data.

fix(url_safety): send browser-like default headers on safe fetches

Jaibharath484ce21501b2
diff --git a/scripts/url_safety.py b/scripts/url_safety.py
index 21c3189..077bc07 100644
--- a/scripts/url_safety.py
+++ b/scripts/url_safety.py
@@ -553,6 +553,34 @@ def _pin_dns(
_dns_patch_lock.release()
+# Default request headers, mirroring the browser-like defaults fetch_page.py
+# has used since v1.2.1 (issue #9). Without them ``requests`` announces itself
+# as ``User-Agent: python-requests/x.y.z`` with no Accept-Language, which many
+# managed WAFs and CDNs answer with 403/406, and which SSR frameworks answer
+# with the empty client-side shell. Callers do not see an exception in that
+# case, they analyse the error page or the shell as if it were the real
+# document. Any header here can be overridden by passing ``headers=``.
+DEFAULT_REQUEST_HEADERS = {
+ "User-Agent": (
+ "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 "
+ "(KHTML, like Gecko) Chrome/150.0.7871.115 Safari/537.36 ClaudeSEO/2.0"
+ ),
+ "Accept": (
+ "text/html,application/xhtml+xml,application/xml;q=0.9,"
+ "image/avif,image/webp,*/*;q=0.8"
+ ),
+ "Accept-Language": "en-US,en;q=0.9",
+}
+
+
+def _with_default_headers(kwargs: dict) -> dict:
+ """Fill in DEFAULT_REQUEST_HEADERS for header keys the caller did not set."""
+ headers = dict(DEFAULT_REQUEST_HEADERS)
+ headers.update(kwargs.get("headers") or {})
+ kwargs["headers"] = headers
+ return kwargs
+
+
def safe_requests_get(
url: str,
*,
@@ -563,12 +591,15 @@ def safe_requests_get(
``requests.get`` with DNS-rebinding protection.
The request's hostname is pinned to a pre-validated IP for the
- duration of the call. Standard ``requests`` semantics otherwise.
+ duration of the call. Standard ``requests`` semantics otherwise,
+ except that browser-like default headers are supplied for any header
+ the caller did not set (see ``DEFAULT_REQUEST_HEADERS``).
"""
norm_url, pinned_ip = validate_url_strict(url)
parsed = urlparse(norm_url)
port = parsed.port or (443 if parsed.scheme == "https" else 80)
assert parsed.hostname is not None # validate_url_strict guarantees this
+ kwargs = _with_default_headers(kwargs)
exempt = _validated_proxy_hosts(norm_url, kwargs.get("proxies"))
with _pin_dns(parsed.hostname, pinned_ip, port, exempt_hosts=exempt):
return requests.get(norm_url, timeout=timeout, **kwargs)
@@ -584,12 +615,15 @@ def safe_requests_head(
``requests.head`` with DNS-rebinding protection.
The request's hostname is pinned to a pre-validated IP for the
- duration of the call. Standard ``requests`` semantics otherwise.
+ duration of the call. Standard ``requests`` semantics otherwise,
+ except that browser-like default headers are supplied for any header
+ the caller did not set (see ``DEFAULT_REQUEST_HEADERS``).
"""
norm_url, pinned_ip = validate_url_strict(url)
parsed = urlparse(norm_url)
port = parsed.port or (443 if parsed.scheme == "https" else 80)
assert parsed.hostname is not None
+ kwargs = _with_default_headers(kwargs)
exempt = _validated_proxy_hosts(norm_url, kwargs.get("proxies"))
with _pin_dns(parsed.hostname, pinned_ip, port, exempt_hosts=exempt):
return requests.head(norm_url, timeout=timeout, **kwargs)
diff --git a/tests/test_url_safety.py b/tests/test_url_safety.py
index 0906704..46a2d2e 100644
--- a/tests/test_url_safety.py
+++ b/tests/test_url_safety.py
@@ -350,11 +350,49 @@ def test_safe_requests_head_uses_strict_validation_and_dns_pin() -> None:
"https://safe.example/path",
timeout=7,
allow_redirects=True,
+ headers=url_safety.DEFAULT_REQUEST_HEADERS,
)
assert captured["pin"] == ("safe.example", "1.1.1.1", 443)
assert result is response
+def test_default_headers_are_browser_like() -> None:
+ headers = url_safety.DEFAULT_REQUEST_HEADERS
+ assert "python-requests" not in headers["User-Agent"]
+ assert headers["User-Agent"].startswith("Mozilla/5.0")
+ assert "Accept-Language" in headers
+
+
+def test_with_default_headers_fills_unset_headers() -> None:
+ assert url_safety._with_default_headers({})["headers"] == (
+ url_safety.DEFAULT_REQUEST_HEADERS
+ )
+ assert url_safety._with_default_headers({"headers": None})["headers"] == (
+ url_safety.DEFAULT_REQUEST_HEADERS
+ )
+
+
+def test_with_default_headers_lets_caller_override() -> None:
+ # fetch_page.py --user-agent (Googlebot cloaking checks) must still win.
+ merged = url_safety._with_default_headers(
+ {"headers": {"User-Agent": "Googlebot/2.1"}}
+ )["headers"]
+ assert merged["User-Agent"] == "Googlebot/2.1"
+ # Headers the caller did not set are still filled in.
+ assert merged["Accept-Language"] == (
+ url_safety.DEFAULT_REQUEST_HEADERS["Accept-Language"]
+ )
+
+
+def test_with_default_headers_preserves_other_kwargs_and_constant() -> None:
+ before = dict(url_safety.DEFAULT_REQUEST_HEADERS)
+ kwargs = url_safety._with_default_headers({"stream": True, "allow_redirects": False})
+ assert kwargs["stream"] is True
+ assert kwargs["allow_redirects"] is False
+ url_safety._with_default_headers({"headers": {"User-Agent": "mutating/1.0"}})
+ assert url_safety.DEFAULT_REQUEST_HEADERS == before
+
+
def test_pin_dns_restores_getaddrinfo_on_exception() -> None:
before = socket.getaddrinfo
with pytest.raises(RuntimeError):