Skip to content

Universal SEO skill for Claude Code. 26 sub-skills + 19 sub-agents covering technical SEO, E-E-A-T, schema, GEO/AEO, agent readiness (Lighthouse Agentic Browsing, WebMCP, llms.txt), backlinks, local SEO, e-commerce, international SEO, Google APIs, and PDF/Excel reporting. 9 optional extensions for live SEO data.

feat(v2): foundation - shared SSRF-safe URL layer + headless renderer

Danielef35dd60c246
diff --git a/requirements.txt b/requirements.txt
index badce38..e8655dc 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,6 +1,6 @@
# Claude SEO - Python Dependencies
# Bounded version pinning with security-conscious minimums
-# Last updated: February 7, 2026
+# Last updated: May 17, 2026 (v2.0.0)
beautifulsoup4>=4.12.0,<5.0.0 # No known CVEs
requests>=2.32.4,<3.0.0 # CVE-2024-47081, CVE-2024-35195 fixes
@@ -10,6 +10,11 @@ Pillow>=12.1.0,<13.0.0 # CVE-2025-48379 fix
urllib3>=2.6.3,<3.0.0 # CRITICAL: CVE-2026-21441 (CVSS 8.9), CVE-2025-66418
validators>=0.22.0,<1.0.0 # No known CVEs
+# v2.0.0 Phase A: headless rendering across all agents
+trafilatura>=2.0.0,<3.0.0 # Boilerplate-free content extraction (SPA-safe)
+htmldate>=1.9.0,<2.0.0 # Publication-date extraction for freshness signals
+courlan>=1.3.0,<2.0.0 # trafilatura URL helper; explicit pin avoids transitive drift
+
# Report generation (for seo-google PDF/Excel reports)
matplotlib>=3.8.0,<4.0.0 # No known CVEs
weasyprint>=68.1,<70.0 # No known CVEs
diff --git a/scripts/backlinks_auth.py b/scripts/backlinks_auth.py
index 3cc63d1..c9d96d8 100644
--- a/scripts/backlinks_auth.py
+++ b/scripts/backlinks_auth.py
@@ -20,25 +20,22 @@ import os
import sys
from typing import Optional
-# Import SSRF protection from google_auth (reuse, don't duplicate)
+# Import SSRF protection from the canonical url_safety module.
+# google_auth.validate_url is a back-compat wrapper around the same function.
_SCRIPTS_DIR = os.path.dirname(os.path.abspath(__file__))
-sys.path.insert(0, _SCRIPTS_DIR)
+if _SCRIPTS_DIR not in sys.path:
+ sys.path.insert(0, _SCRIPTS_DIR)
try:
- from google_auth import validate_url
-except ImportError:
- # Fallback: basic URL validation if google_auth not available
- def validate_url(url: str) -> bool:
- from urllib.parse import urlparse
- parsed = urlparse(url)
- if parsed.scheme not in ("http", "https"):
- return False
- if not parsed.hostname:
- return False
- blocked = ["localhost", "127.0.0.1", "0.0.0.0", "::1",
- "metadata.google.internal"]
- if parsed.hostname in blocked:
- return False
- return True
+ from url_safety import validate_url
+except ImportError as _import_exc:
+ # Hard fail: a private-IP/loopback fallback that omits SSRF checks is
+ # worse than no validation at all. The previous fallback shipped in
+ # v1.9.0 silently allowed private IP literals and was flagged in the
+ # cybersecurity audit (v2.0.0 Phase H). Refuse to run instead.
+ raise RuntimeError(
+ "scripts/url_safety.py is required for SSRF protection. "
+ "Install with: pip install -r requirements.txt"
+ ) from _import_exc
CONFIG_PATH = os.path.expanduser("~/.config/claude-seo/backlinks-api.json")
CACHE_DIR = os.path.expanduser("~/.cache/claude-seo/commoncrawl")
diff --git a/scripts/google_auth.py b/scripts/google_auth.py
index 565e0ed..9e27c40 100644
--- a/scripts/google_auth.py
+++ b/scripts/google_auth.py
@@ -422,29 +422,22 @@ def validate_url(url: str) -> bool:
Returns:
True if the URL is a valid public http/https URL, False otherwise.
+
+ Note:
+ Back-compat wrapper around :func:`url_safety.validate_url`. The shared
+ module is the canonical implementation and adds DNS-rebinding-safe
+ helpers (``validate_url_strict``, ``safe_requests_get``) that
+ ``fetch_page.py`` and ``render_page.py`` use before opening sockets.
"""
- from urllib.parse import urlparse
-
- parsed = urlparse(url)
- if parsed.scheme not in ("http", "https"):
- return False
- if not parsed.hostname:
- return False
- blocked = [
- "localhost", "127.0.0.1", "0.0.0.0", "::1",
- "metadata.google.internal",
- ]
- if parsed.hostname in blocked:
- return False
- # Block private IP ranges (10.x, 172.16-31.x, 192.168.x)
- try:
- import ipaddress
- ip = ipaddress.ip_address(parsed.hostname)
- if ip.is_private or ip.is_loopback or ip.is_link_local:
- return False
- except ValueError:
- pass # Not an IP address (hostname), which is fine
- return True
+ # Lazy import: avoids a hard requirement on url_safety for callers that
+ # only need google_auth's other helpers (e.g. token refresh) and keeps
+ # the import graph one-directional.
+ _scripts_dir = os.path.dirname(os.path.abspath(__file__))
+ if _scripts_dir not in sys.path:
+ sys.path.insert(0, _scripts_dir)
+ from url_safety import validate_url as _validate_url
+
+ return _validate_url(url)
def get_api_key() -> Optional[str]:
diff --git a/scripts/render_page.py b/scripts/render_page.py
new file mode 100644
index 0000000..957fa70
--- /dev/null
+++ b/scripts/render_page.py
@@ -0,0 +1,451 @@
+#!/usr/bin/env python3
+"""
+Shared headless renderer for claude-seo.
+
+Every subagent that fetches HTML for analysis (technical, content, schema,
+geo, local, ecommerce, hreflang, images) calls this module instead of
+``fetch_page.py`` whenever JS execution might change what an audit can see.
+Built on Playwright Chromium with trafilatura for boilerplate-free content
+extraction and htmldate for publication-date detection.
+
+Why
+===
+Before v2.0.0 only ``seo-visual`` used Playwright. Every other agent
+fetched raw HTML, which produces false negatives on SPAs (empty
+``<div id="root">``, no schema in source, no content in source). The
+gap analysis (see ``compass_artifact_*.md``) ranks "headless rendering
+across all subagents" as the single highest-impact v2 change. This module
+delivers it as a shared subsystem so the change is one foundation, not
+eight retrofits.
+
+Modes
+=====
+- ``auto`` : raw fetch first; render only when an SPA shell is detected
+ (see ``_is_spa``). Default. Cheapest correct behaviour.
+- ``always`` : always render with Playwright, even for static HTML.
+- ``never`` : raw HTML only. Equivalent to legacy ``fetch_page.py``.
+
+Result shape
+============
+A dict with::
+
+ url final URL after redirects
+ status_code HTTP status of the main document
+ content HTML after JS execution (post-render DOM)
+ raw_content HTML before JS execution (server response)
+ is_spa True iff raw_content looks like a hydration shell
+ extracted_text trafilatura main-content extraction (or None)
+ publication_date htmldate ISO 8601 string (or None)
+ headers response headers from the main document
+ redirect_chain list of {url, status_code}
+ console_errors list of browser console error strings
+ render_engine 'playwright-chromium' or None
+ render_ms elapsed wall-clock for the render step
+ mode_used 'rendered' or 'raw'
+ error str or None
+
+SSRF
+====
+The URL is validated via :func:`url_safety.validate_url_strict` before
+Playwright sees it. Inside Playwright a ``route()`` handler intercepts
+every subresource and aborts requests whose hostname resolves to a
+non-public IP. This is defence in depth against DNS rebinding inside
+Chromium's resolver. The residual rebinding risk for browser fetches
+is documented in SECURITY.md.
+
+CLI
+===
+ python render_page.py https://nuxt.com --mode always
+ python render_page.py https://example.com --mode auto --json
+ python render_page.py https://store.example.com --block image --block font
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import re
+import socket
+import sys
+import time
+from typing import Optional
+from urllib.parse import urlparse
+
+# Optional native dependencies. Each is checked lazily so callers that
+# only need raw-mode (mode='never') don't pay the import cost.
+try:
+ from playwright.sync_api import (
+ sync_playwright,
+ TimeoutError as PlaywrightTimeout,
+ )
+except ImportError: # pragma: no cover - exercised in environments without playwright
+ sync_playwright = None
+ PlaywrightTimeout = Exception # type: ignore[assignment,misc]
+
+try:
+ import trafilatura
+except ImportError: # pragma: no cover
+ trafilatura = None
+
+try:
+ from htmldate import find_date
+except ImportError: # pragma: no cover
+ find_date = None
+
+# Reuse the canonical safety module.
+_SCRIPTS_DIR = os.path.dirname(os.path.abspath(__file__))
+if _SCRIPTS_DIR not in sys.path:
+ sys.path.insert(0, _SCRIPTS_DIR)
+from url_safety import ( # noqa: E402 (sys.path massage above is intentional)
+ URLSafetyError,
+ is_safe_ip,
+ safe_requests_get,
+ validate_url_strict,
+)
+
+
+VIEWPORTS: dict[str, dict[str, int]] = {
+ "desktop": {"width": 1920, "height": 1080, "device_scale": 1},
+ "laptop": {"width": 1366, "height": 768, "device_scale": 1},
+ "tablet": {"width": 768, "height": 1024, "device_scale": 1},
+ "mobile": {"width": 375, "height": 812, "device_scale": 2},
+}
+
+USER_AGENT = (
+ "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 "
+ "(KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36 ClaudeSEO/2.0"
+)
+
+# Hydration-shell signatures. Any single match flips is_spa to True. These
+# cover the dominant SPA frameworks: React (CRA, Vite, Remix), Next.js,
+# Vue, Nuxt, Svelte, Astro islands, and the "JS required" noscript pattern.
+_SPA_SHELL_PATTERNS = (
+ '<div id="root"></div>',
+ '<div id="__next">',
+ '<div id="app"></div>',
+ '<div id="__nuxt">',
+ 'data-svelte-h=',
+ '<astro-island ',
+ 'you need to enable javascript',
+ 'please enable javascript',
+)
+
+_TAG_STRIP = re.compile(r"<[^>]+>")
+_WHITESPACE = re.compile(r"\s+")
+
+
+def _is_spa(raw_html: Optional[str]) -> bool:
+ """Heuristic SPA detector. Conservative: any positive signal flips True."""
+ if not raw_html:
+ return True
+ lc = raw_html.lower()
+ if any(pattern in lc for pattern in _SPA_SHELL_PATTERNS):
+ return True
+ # Very thin <body> suggests JS-rendered content even without a shell.
+ # Threshold (100 chars) sits between typical SPA shells (0-50 chars of
+ # body text) and minimal informational pages like example.com (~125
+ # chars). Tuned conservatively to avoid false positives that would
+ # force a redundant Playwright render in auto mode.
+ body_start = lc.find("<body")
+ body_end = lc.rfind("</body>")
+ if body_start != -1 and body_end > body_start:
+ body = lc[body_start:body_end]
+ text = _WHITESPACE.sub(" ", _TAG_STRIP.sub("", body)).strip()
+ if len(text) < 100:
+ return True
+ return False
+
+
+def _make_route_handler(blocked_resource_types: set[str]):
+ """
+ Build a Playwright route() callback that:
+ (1) refuses subresource requests resolving to non-public IPs
+ (defence in depth against DNS rebinding inside Chromium),
+ (2) blocks resource types in ``blocked_resource_types`` for faster
+ renders when the caller doesn't need images/fonts/etc.
+ """
+
+ def handler(route, request): # type: ignore[no-untyped-def]
+ try:
+ if blocked_resource_types and request.resource_type in blocked_resource_types:
+ route.abort()
+ return
+
+ parsed = urlparse(request.url)
+ if parsed.scheme not in ("http", "https"):
+ # data:, blob:, chrome-extension:, etc. — no DNS, allow.
+ route.continue_()
+ return
+ host = parsed.hostname
+ if not host:
+ route.abort()
+ return
+
+ try:
+ addrinfo = socket.getaddrinfo(
+ host, None, family=socket.AF_INET, type=socket.SOCK_STREAM
+ )
+ except socket.gaierror:
+ route.abort()
+ return
+ ips = {info[4][0] for info in addrinfo}
+ if any(not is_safe_ip(ip) for ip in ips):
+ route.abort()
+ return
+ route.continue_()
+ except Exception: # pragma: no cover - fail-closed on any handler error
+ try:
+ route.abort()
+ except Exception:
+ pass
+
+ return handler
+
+
+def render_page(
+ url: str,
+ *,
+ mode: str = "auto",
+ viewport: str = "desktop",
+ timeout_ms: int = 15000,
+ block_resources: Optional[list[str]] = None,
+ extract_content: bool = True,
+ user_agent: Optional[str] = None,
+) -> dict:
+ """Render or fetch ``url`` per the chosen mode. See module docstring."""
+ result: dict = {
+ "url": url,
+ "status_code": None,
+ "content": None,
+ "raw_content": None,
+ "is_spa": None,
+ "extracted_text": None,
+ "publication_date": None,
+ "headers": {},
+ "redirect_chain": [],
+ "console_errors": [],
+ "render_engine": None,
+ "render_ms": None,
+ "mode_used": None,
+ "error": None,
+ }
+
+ if mode not in ("auto", "always", "never"):
+ result["error"] = f"Invalid mode: {mode!r}"
+ return result
+ if viewport not in VIEWPORTS:
+ result["error"] = f"Invalid viewport: {viewport!r}"
+ return result
+
+ # Pre-flight SSRF check.
+ try:
+ norm_url, _pinned_ip = validate_url_strict(url)
+ result["url"] = norm_url
+ except URLSafetyError as exc:
+ result["error"] = f"url_safety: {exc}"
+ return result
+
+ # Step 1 — raw fetch (always; needed for SPA detection and as a baseline).
+ try:
+ resp = safe_requests_get(norm_url, timeout=30, allow_redirects=True)
+ result["raw_content"] = resp.text
+ if resp.history:
+ result["redirect_chain"] = [
+ {"url": r.url, "status_code": r.status_code} for r in resp.history
+ ]
+ raw_status = resp.status_code
+ raw_headers = dict(resp.headers)
+ final_raw_url = resp.url
+ except Exception as exc:
+ result["error"] = f"raw fetch failed: {exc}"
+ return result
+
+ result["is_spa"] = _is_spa(result["raw_content"])
+ should_render = mode == "always" or (mode == "auto" and result["is_spa"])
+
+ if not should_render:
+ result["mode_used"] = "raw"
+ result["url"] = final_raw_url
+ result["status_code"] = raw_status
+ result["headers"] = raw_headers
+ result["content"] = result["raw_content"]
+ else:
+ result["mode_used"] = "rendered"
+ if sync_playwright is None:
+ result["error"] = (
+ "playwright is required for rendered mode. "
+ "Install: pip install -r requirements.txt "
+ "&& playwright install chromium"
+ )
+ return result
+
+ vp = VIEWPORTS[viewport]
+ blocked = set(block_resources or [])
+ route_handler = _make_route_handler(blocked)
+ start = time.monotonic()
+
+ try:
+ with sync_playwright() as p:
+ browser = p.chromium.launch(headless=True)
+ context = browser.new_context(
+ viewport={"width": vp["width"], "height": vp["height"]},
+ device_scale_factor=vp["device_scale"],
+ user_agent=user_agent or USER_AGENT,
+ )
+ page = context.new_page()
+
+ def _on_console(msg): # type: ignore[no-untyped-def]
+ if msg.type == "error":
+ result["console_errors"].append(msg.text)
+
+ page.on("console", _on_console)
+ page.route("**/*", route_handler)
+
+ response = page.goto(
+ norm_url, wait_until="networkidle", timeout=timeout_ms
+ )
+ # Allow late hydration (deferred islands, useEffect chains).
+ page.wait_for_timeout(500)
+
+ result["url"] = page.url
+ result["content"] = page.content()
+ result["status_code"] = response.status if response else raw_status
+ result["headers"] = (
+ dict(response.all_headers()) if response else raw_headers
+ )
+ result["render_engine"] = "playwright-chromium"
+
+ browser.close()
+ except PlaywrightTimeout:
+ result["error"] = f"playwright navigation timed out after {timeout_ms}ms"
+ return result
+ except Exception as exc:
+ result["error"] = f"playwright error: {exc}"
+ return result
+ finally:
+ result["render_ms"] = (time.monotonic() - start) * 1000.0
+
+ # Step 2 — content extraction (works on either raw or rendered HTML).
+ if extract_content and result["content"]:
+ if trafilatura is not None:
+ try:
+ result["extracted_text"] = trafilatura.extract(
+ result["content"],
+ include_comments=False,
+ include_tables=True,
+ favor_recall=False,
+ )
+ except Exception:
+ # Extraction is best-effort; never block the audit on it.
+ pass
+ if find_date is not None:
+ try:
+ result["publication_date"] = find_date(result["content"])
+ except Exception:
+ pass
+
+ return result
+
+
+def _cli() -> None:
+ parser = argparse.ArgumentParser(
+ description="claude-seo shared headless renderer (Playwright + trafilatura)"
+ )
+ parser.add_argument("url", help="URL to render")
+ parser.add_argument(
+ "--mode",
+ choices=("auto", "always", "never"),
+ default="auto",
+ help="auto: render only when SPA detected; always: always render; "
+ "never: raw HTML only (default: auto)",
+ )
+ parser.add_argument(
+ "--viewport", choices=list(VIEWPORTS), default="desktop"
+ )
+ parser.add_argument(
+ "--timeout-ms",
+ type=int,
+ default=15000,
+ help="Playwright navigation timeout in ms (default: 15000)",
+ )
+ parser.add_argument(
+ "--block",
+ action="append",
+ default=[],
+ choices=("image", "media", "font", "stylesheet"),
+ help="resource types to block during render (faster, less accurate)",
+ )
+ parser.add_argument(
+ "--no-extract",
+ action="store_true",
+ help="skip trafilatura and htmldate post-processing",
+ )
+ parser.add_argument(
+ "--json",
+ action="store_true",
+ help="emit a JSON summary (truncates content fields)",
+ )
+ parser.add_argument("--output", "-o", help="write HTML content to file")
+ args = parser.parse_args()
+
+ res = render_page(
+ args.url,
+ mode=args.mode,
+ viewport=args.viewport,
+ timeout_ms=args.timeout_ms,
+ block_resources=args.block or None,
+ extract_content=not args.no_extract,
+ )
+
+ if args.json:
+ summary = dict(res)
+ # JSON-safe truncation so the CLI is usable from agents without
+ # piping megabytes of HTML across stdio.
+ for field, limit in (
+ ("content", 500),
+ ("raw_content", 200),
+ ("extracted_text", 500),
+ ):
+ if summary.get(field):
+ value = summary[field]
+ summary[field] = (
+ value[:limit] + "..." if len(value) > limit else value
+ )
+ print(json.dumps(summary, indent=2, default=str))
+ sys.exit(1 if res["error"] else 0)
+
+ if res["error"]:
+ print(f"Error: {res['error']}", file=sys.stderr)
+ sys.exit(1)
+
+ if args.output:
+ with open(args.output, "w", encoding="utf-8") as fh:
+ fh.write(res["content"] or "")
+ print(f"saved to {args.output}", file=sys.stderr)
+ else:
+ print(res["content"])
+
+ print(
+ f"\nFinal URL: {res['url']}\n"
+ f"Status: {res['status_code']} | mode={res['mode_used']} | "
+ f"is_spa={res['is_spa']}",
+ file=sys.stderr,
+ )
+ if res["render_ms"]:
+ print(
+ f"Render: {res['render_ms']:.0f}ms via {res['render_engine']}",
+ file=sys.stderr,
+ )
+ if res["publication_date"]:
+ print(f"Publication date: {res['publication_date']}", file=sys.stderr)
+ if res["console_errors"]:
+ print(
+ f"Console errors ({len(res['console_errors'])}):", file=sys.stderr
+ )
+ for err in res["console_errors"][:5]:
+ print(f" - {err}", file=sys.stderr)
+
+
+if __name__ == "__main__":
+ _cli()
diff --git a/scripts/url_safety.py b/scripts/url_safety.py
new file mode 100644
index 0000000..4316506
--- /dev/null
+++ b/scripts/url_safety.py
@@ -0,0 +1,384 @@
+#!/usr/bin/env python3
+"""
+Canonical URL safety module for claude-seo.
+
+Centralizes SSRF protection, DNS rebinding mitigation, and DNS-pinned HTTP
+fetching. Every script in this repository that accepts a user-supplied URL
+MUST validate it through this module before issuing any network request.
+
+Public API
+==========
+
+validate_url(url) -> bool
+ Back-compat boolean check. Rejects non-http(s) schemes, missing
+ hostnames, hard-blocked hostnames (localhost, cloud metadata
+ endpoints), and IP literals that fall inside private/loopback/reserved
+ ranges. Does NOT resolve DNS. Preserves the v1.9.9 contract used by
+ google_auth.py.
+
+validate_url_strict(url) -> tuple[str, str]
+ Resolves the hostname via socket.getaddrinfo, validates every returned
+ A record against the safety predicate, and returns
+ ``(normalized_url, pinned_ipv4)``. Raises ``URLSafetyError`` if any
+ resolved IP is non-public. Use this whenever the caller is about to
+ open a network connection so DNS rebinding cannot swap a public IP
+ for a private one between checks.
+
+safe_requests_get(url, *, timeout=30, **kwargs) -> requests.Response
+ ``requests.get(...)`` wrapped in a DNS-pinning context manager so the
+ OS-level resolver only ever sees the pre-validated IP for the duration
+ of the call. The original hostname is preserved in the HTTP Host
+ header and TLS SNI; only the connect() target is forced to the pinned
+ address.
+
+safe_requests_session(url) -> context manager yielding requests.Session
+ Same protection as ``safe_requests_get`` for callers that need a
+ session (cookies, redirect chains, multiple requests to one host).
+
+is_safe_ip(ip_str) -> bool
+ True iff the address parses as IPv4/IPv6 and is none of:
+ private, loopback, reserved, link-local, multicast, unspecified.
+
+URLSafetyError
+ ValueError subclass raised by the strict validator and pinning helpers.
+
+Threading
+=========
+The DNS pinning helper is a critical section guarded by a non-blocking
+``threading.Lock``. Two concurrent pinned fetches on the same process will
+raise rather than corrupt the global ``socket.getaddrinfo`` reference.
+claude-seo scripts are intentionally single-threaded; parallelism is
+delegated to the agent-process layer.
+
+Limitations
+===========
+Playwright/Chromium-based fetches (``render_page.py``,
+``capture_screenshot.py``) perform their own DNS resolution inside
+Chromium and therefore cannot be DNS-pinned at the Python layer. Those
+scripts must:
+
+ 1. Call ``validate_url_strict()`` as a pre-flight check, AND
+ 2. Attach a Playwright ``route()`` handler that re-validates each
+ resolved request IP and aborts subresource fetches to private
+ ranges.
+
+The residual DNS-rebinding risk for browser-based fetches is documented
+in SECURITY.md.
+"""
+
+from __future__ import annotations
+
+import ipaddress
+import socket
+import threading
+from contextlib import contextmanager
+from typing import Iterator, Optional
+from urllib.parse import urlparse
+
+try:
+ import requests
+except ImportError as exc: # pragma: no cover - hard dependency
+ raise RuntimeError(
+ "scripts/url_safety.py requires the 'requests' package. "
+ "Install with: pip install -r requirements.txt"
+ ) from exc
+
+
+__all__ = [
+ "URLSafetyError",
+ "is_safe_ip",
+ "validate_url",
+ "validate_url_strict",
+ "safe_requests_get",
+ "safe_requests_session",
+]
+
+
+# Hard-blocked hostnames. Anything here is refused even before DNS resolution.
+# Cloud metadata endpoints are the most common SSRF target; we list every
+# documented address across AWS, Azure, GCP, Oracle, and Alibaba so a single
+# typo (e.g., metadata.google.internal vs metadata.googleapis.internal)
+# cannot slip through.
+_BLOCKED_HOSTNAMES: frozenset[str] = frozenset(
+ {
+ "localhost",
+ "ip6-localhost",
+ "ip6-loopback",
+ "metadata.google.internal",
+ "metadata.goog",
+ "metadata",
+ "metadata.azure.com",
+ "metadata.ec2.internal",
+ "metadata.oraclecloud.com",
+ # Numeric metadata endpoints (also caught by IP literal check, listed
+ # explicitly for defence-in-depth and clearer error messages).
+ "127.0.0.1",
+ "0.0.0.0",
+ "::1",
+ "169.254.169.254", # AWS, Azure, GCP, Oracle, Alibaba metadata IPv4
+ "fd00:ec2::254", # AWS IMDS IPv6
+ }
+)
+
+
+class URLSafetyError(ValueError):
+ """Raised when a URL fails SSRF safety checks."""
+
+
+def is_safe_ip(ip_str: str) -> bool:
+ """Return True iff ``ip_str`` is a public unicast address."""
+ try:
+ ip = ipaddress.ip_address(ip_str)
+ except ValueError:
+ return False
+ return not (
+ ip.is_private
+ or ip.is_loopback
+ or ip.is_reserved
+ or ip.is_link_local
+ or ip.is_multicast
+ or ip.is_unspecified
+ )
+
+
+def validate_url(url: str) -> bool:
+ """
+ Back-compat boolean validator. Does not resolve DNS.
+
+ Returns False when:
+ - Scheme is not http or https
+ - Hostname is missing
+ - Hostname is in the hard-block list
+ - Hostname is an IP literal that fails ``is_safe_ip``
+ Returns True for any other well-formed http(s) URL with a public-looking
+ hostname. Use ``validate_url_strict`` if the caller will open a socket
+ using the hostname (DNS rebinding cannot be defeated at parse time).
+ """
+ parsed = urlparse(url)
+ if parsed.scheme not in ("http", "https"):
+ return False
+ if not parsed.hostname:
+ return False
+ hostname = parsed.hostname.lower()
+ if hostname in _BLOCKED_HOSTNAMES:
+ return False
+ try:
+ ipaddress.ip_address(hostname)
+ except ValueError:
+ return True # Hostname is a name, not a literal — OK at parse time.
+ return is_safe_ip(hostname)
+
+
+def validate_url_strict(url: str) -> tuple[str, str]:
+ """
+ Resolve and validate the URL's hostname.
+
+ Returns ``(url, pinned_ipv4)`` on success. Raises ``URLSafetyError`` if:
+ - Scheme is invalid or hostname is missing.
+ - Hostname is hard-blocked.
+ - DNS resolution fails.
+ - Any A record resolves to a non-public IP (DNS rebinding refused).
+
+ Multi-A-record handling: every returned record must be public. A
+ hostname with one public and one private A record is refused so an
+ attacker cannot race the resolver between validate and connect.
+ """
+ parsed = urlparse(url)
+ if parsed.scheme not in ("http", "https"):
+ raise URLSafetyError(f"Invalid URL scheme: {parsed.scheme!r}")
+ if not parsed.hostname:
+ raise URLSafetyError("URL has no hostname")
+
+ hostname = parsed.hostname.lower()
+ if hostname in _BLOCKED_HOSTNAMES:
+ raise URLSafetyError(f"Blocked hostname: {hostname}")
+
+ # If the hostname is an IP literal, validate it directly without DNS.
+ try:
+ literal = ipaddress.ip_address(hostname)
+ except ValueError:
+ literal = None
+
+ if literal is not None:
+ if not is_safe_ip(hostname):
+ raise URLSafetyError(f"Blocked IP literal: {hostname}")
+ return url, str(literal)
+
+ port = parsed.port or (443 if parsed.scheme == "https" else 80)
+ try:
+ addrinfo = socket.getaddrinfo(
+ hostname,
+ port,
+ family=socket.AF_INET,
+ type=socket.SOCK_STREAM,
+ )
+ except socket.gaierror as exc:
+ raise URLSafetyError(f"DNS resolution failed for {hostname}: {exc}") from exc
+
+ resolved_ips = sorted({info[4][0] for info in addrinfo})
+ if not resolved_ips:
+ raise URLSafetyError(f"No A records for {hostname}")
+
+ for ip_str in resolved_ips:
+ if not is_safe_ip(ip_str):
+ raise URLSafetyError(
+ f"DNS rebinding refused: {hostname} resolves to "
+ f"non-public IP {ip_str}"
+ )
+
+ pinned = resolved_ips[0]
+ return url, pinned
+
+
+# A single non-blocking lock guards the global getaddrinfo monkey-patch.
+# This is a deliberate choice: claude-seo scripts run one URL fetch at a
+# time, and we'd rather raise loudly than silently corrupt resolver state
+# if a caller ever introduces threading.
+_dns_patch_lock = threading.Lock()
+
+
+@contextmanager
+def _pin_dns(hostname: str, pinned_ip: str, port: int) -> Iterator[None]:
+ """
+ Temporarily override ``socket.getaddrinfo`` so the named host resolves
+ only to ``pinned_ip``. Other hostnames are forwarded to the real
+ resolver. Restores the original function on exit, even on exception.
+ """
+ if not _dns_patch_lock.acquire(blocking=False):
+ raise URLSafetyError(
+ "DNS-pinned fetch already in progress on another thread; "
+ "claude-seo url_safety is not thread-safe by design."
+ )
+
+ original_getaddrinfo = socket.getaddrinfo
+ target = hostname.lower()
+ pinned_record = (
+ socket.AF_INET,
+ socket.SOCK_STREAM,
+ socket.IPPROTO_TCP,
+ "",
+ (pinned_ip, port),
+ )
+
+ def patched(host, requested_port, *args, **kwargs):
+ if host and host.lower() == target:
+ family = kwargs.get("family", args[0] if args else 0)
+ if family in (0, socket.AF_UNSPEC, socket.AF_INET):
+ # Reuse the pinned record regardless of which port the
+ # caller asked for. urllib3 always asks for the same port
+ # the URL specified, which is what we validated.
+ return [(
+ pinned_record[0],
+ pinned_record[1],
+ pinned_record[2],
+ pinned_record[3],
+ (pinned_ip, requested_port or port),
+ )]
+ raise socket.gaierror(
+ socket.EAI_FAIL,
+ f"url_safety: address family {family} refused for pinned "
+ f"IPv4 host {host}",
+ )
+ return original_getaddrinfo(host, requested_port, *args, **kwargs)
+
+ socket.getaddrinfo = patched # type: ignore[assignment]
+ try:
+ yield
+ finally:
+ socket.getaddrinfo = original_getaddrinfo # type: ignore[assignment]
+ _dns_patch_lock.release()
+
+
+def safe_requests_get(
+ url: str,
+ *,
+ timeout: int = 30,
+ **kwargs,
+) -> requests.Response:
+ """
+ ``requests.get`` with DNS-rebinding protection.
+
+ The request's hostname is pinned to a pre-validated IP for the
+ duration of the call. Standard ``requests`` semantics otherwise.
+ """
+ norm_url, pinned_ip = validate_url_strict(url)
+ parsed = urlparse(norm_url)
+ port = parsed.port or (443 if parsed.scheme == "https" else 80)
+ assert parsed.hostname is not None # validate_url_strict guarantees this
+ with _pin_dns(parsed.hostname, pinned_ip, port):
+ return requests.get(norm_url, timeout=timeout, **kwargs)
+
+
+@contextmanager
+def safe_requests_session(url: str) -> Iterator[requests.Session]:
+ """
+ Yield a ``requests.Session`` whose connections to ``url``'s hostname
+ are DNS-pinned. Callers may make multiple requests to that host
+ within the ``with`` block without re-resolving.
+ """
+ norm_url, pinned_ip = validate_url_strict(url)
+ parsed = urlparse(norm_url)
+ port = parsed.port or (443 if parsed.scheme == "https" else 80)
+ assert parsed.hostname is not None
+ session = requests.Session()
+ with _pin_dns(parsed.hostname, pinned_ip, port):
+ try:
+ yield session
+ finally:
+ session.close()
+
+
+def _cli() -> None:
+ """Tiny CLI for manual SSRF-policy checks. Not used by other scripts."""
+ import argparse
+ import json
+ import sys
+
+ parser = argparse.ArgumentParser(
+ description="Validate a URL against claude-seo's SSRF policy."
+ )
+ parser.add_argument("url", help="URL to validate")
+ parser.add_argument(
+ "--strict",
+ action="store_true",
+ help="Run DNS resolution and refuse on any non-public A record.",
+ )
+ parser.add_argument(
+ "--json",
+ action="store_true",
+ help="Emit a JSON object instead of a one-line summary.",
+ )
+ args = parser.parse_args()
+
+ result: dict[str, Optional[str]] = {
+ "url": args.url,
+ "mode": "strict" if args.strict else "parse",
+ "ok": None,
+ "pinned_ip": None,
+ "error": None,
+ }
+
+ try:
+ if args.strict:
+ _, ip = validate_url_strict(args.url)
+ result["ok"] = "true"
+ result["pinned_ip"] = ip
+ else:
+ result["ok"] = "true" if validate_url(args.url) else "false"
+ except URLSafetyError as exc:
+ result["ok"] = "false"
+ result["error"] = str(exc)
+
+ if args.json:
+ print(json.dumps(result, indent=2))
+ else:
+ if result["ok"] == "true":
+ extra = f" -> {result['pinned_ip']}" if result["pinned_ip"] else ""
+ print(f"OK: {args.url}{extra}")
+ else:
+ print(f"BLOCKED: {args.url} ({result['error'] or 'parse-time reject'})")
+ sys.exit(2)
+
+
+if __name__ == "__main__":
+ _cli()