Universal SEO skill for Claude Code. 26 sub-skills + 19 sub-agents covering technical SEO, E-E-A-T, schema, GEO/AEO, agent readiness (Lighthouse Agentic Browsing, WebMCP, llms.txt), backlinks, local SEO, e-commerce, international SEO, Google APIs, and PDF/Excel reporting. 9 optional extensions for live SEO data.
feat(v2): foundation - shared SSRF-safe URL layer + headless renderer
Danielef35dd60c246
diff --git a/requirements.txt b/requirements.txtindex badce38..e8655dc 100644--- a/requirements.txt+++ b/requirements.txt@@ -1,6 +1,6 @@# Claude SEO - Python Dependencies# Bounded version pinning with security-conscious minimums-# Last updated: February 7, 2026+# Last updated: May 17, 2026 (v2.0.0)beautifulsoup4>=4.12.0,<5.0.0 # No known CVEsrequests>=2.32.4,<3.0.0 # CVE-2024-47081, CVE-2024-35195 fixes@@ -10,6 +10,11 @@ Pillow>=12.1.0,<13.0.0 # CVE-2025-48379 fixurllib3>=2.6.3,<3.0.0 # CRITICAL: CVE-2026-21441 (CVSS 8.9), CVE-2025-66418validators>=0.22.0,<1.0.0 # No known CVEs+# v2.0.0 Phase A: headless rendering across all agents+trafilatura>=2.0.0,<3.0.0 # Boilerplate-free content extraction (SPA-safe)+htmldate>=1.9.0,<2.0.0 # Publication-date extraction for freshness signals+courlan>=1.3.0,<2.0.0 # trafilatura URL helper; explicit pin avoids transitive drift+# Report generation (for seo-google PDF/Excel reports)matplotlib>=3.8.0,<4.0.0 # No known CVEsweasyprint>=68.1,<70.0 # No known CVEsdiff --git a/scripts/backlinks_auth.py b/scripts/backlinks_auth.pyindex 3cc63d1..c9d96d8 100644--- a/scripts/backlinks_auth.py+++ b/scripts/backlinks_auth.py@@ -20,25 +20,22 @@ import osimport sysfrom typing import Optional-# Import SSRF protection from google_auth (reuse, don't duplicate)+# Import SSRF protection from the canonical url_safety module.+# google_auth.validate_url is a back-compat wrapper around the same function._SCRIPTS_DIR = os.path.dirname(os.path.abspath(__file__))-sys.path.insert(0, _SCRIPTS_DIR)+if _SCRIPTS_DIR not in sys.path:+ sys.path.insert(0, _SCRIPTS_DIR)try:- from google_auth import validate_url-except ImportError:- # Fallback: basic URL validation if google_auth not available- def validate_url(url: str) -> bool:- from urllib.parse import urlparse- parsed = urlparse(url)- if parsed.scheme not in ("http", "https"):- return False- if not parsed.hostname:- return False- blocked = ["localhost", "127.0.0.1", "0.0.0.0", "::1",- "metadata.google.internal"]- if parsed.hostname in blocked:- return False- return True+ from url_safety import validate_url+except ImportError as _import_exc:+ # Hard fail: a private-IP/loopback fallback that omits SSRF checks is+ # worse than no validation at all. The previous fallback shipped in+ # v1.9.0 silently allowed private IP literals and was flagged in the+ # cybersecurity audit (v2.0.0 Phase H). Refuse to run instead.+ raise RuntimeError(+ "scripts/url_safety.py is required for SSRF protection. "+ "Install with: pip install -r requirements.txt"+ ) from _import_excCONFIG_PATH = os.path.expanduser("~/.config/claude-seo/backlinks-api.json")CACHE_DIR = os.path.expanduser("~/.cache/claude-seo/commoncrawl")diff --git a/scripts/google_auth.py b/scripts/google_auth.pyindex 565e0ed..9e27c40 100644--- a/scripts/google_auth.py+++ b/scripts/google_auth.py@@ -422,29 +422,22 @@ def validate_url(url: str) -> bool:Returns:True if the URL is a valid public http/https URL, False otherwise.++ Note:+ Back-compat wrapper around :func:`url_safety.validate_url`. The shared+ module is the canonical implementation and adds DNS-rebinding-safe+ helpers (``validate_url_strict``, ``safe_requests_get``) that+ ``fetch_page.py`` and ``render_page.py`` use before opening sockets."""- from urllib.parse import urlparse-- parsed = urlparse(url)- if parsed.scheme not in ("http", "https"):- return False- if not parsed.hostname:- return False- blocked = [- "localhost", "127.0.0.1", "0.0.0.0", "::1",- "metadata.google.internal",- ]- if parsed.hostname in blocked:- return False- # Block private IP ranges (10.x, 172.16-31.x, 192.168.x)- try:- import ipaddress- ip = ipaddress.ip_address(parsed.hostname)- if ip.is_private or ip.is_loopback or ip.is_link_local:- return False- except ValueError:- pass # Not an IP address (hostname), which is fine- return True+ # Lazy import: avoids a hard requirement on url_safety for callers that+ # only need google_auth's other helpers (e.g. token refresh) and keeps+ # the import graph one-directional.+ _scripts_dir = os.path.dirname(os.path.abspath(__file__))+ if _scripts_dir not in sys.path:+ sys.path.insert(0, _scripts_dir)+ from url_safety import validate_url as _validate_url++ return _validate_url(url)def get_api_key() -> Optional[str]:diff --git a/scripts/render_page.py b/scripts/render_page.pynew file mode 100644index 0000000..957fa70--- /dev/null+++ b/scripts/render_page.py@@ -0,0 +1,451 @@+#!/usr/bin/env python3+"""+Shared headless renderer for claude-seo.++Every subagent that fetches HTML for analysis (technical, content, schema,+geo, local, ecommerce, hreflang, images) calls this module instead of+``fetch_page.py`` whenever JS execution might change what an audit can see.+Built on Playwright Chromium with trafilatura for boilerplate-free content+extraction and htmldate for publication-date detection.++Why+===+Before v2.0.0 only ``seo-visual`` used Playwright. Every other agent+fetched raw HTML, which produces false negatives on SPAs (empty+``<div id="root">``, no schema in source, no content in source). The+gap analysis (see ``compass_artifact_*.md``) ranks "headless rendering+across all subagents" as the single highest-impact v2 change. This module+delivers it as a shared subsystem so the change is one foundation, not+eight retrofits.++Modes+=====+- ``auto`` : raw fetch first; render only when an SPA shell is detected+ (see ``_is_spa``). Default. Cheapest correct behaviour.+- ``always`` : always render with Playwright, even for static HTML.+- ``never`` : raw HTML only. Equivalent to legacy ``fetch_page.py``.++Result shape+============+A dict with::++ url final URL after redirects+ status_code HTTP status of the main document+ content HTML after JS execution (post-render DOM)+ raw_content HTML before JS execution (server response)+ is_spa True iff raw_content looks like a hydration shell+ extracted_text trafilatura main-content extraction (or None)+ publication_date htmldate ISO 8601 string (or None)+ headers response headers from the main document+ redirect_chain list of {url, status_code}+ console_errors list of browser console error strings+ render_engine 'playwright-chromium' or None+ render_ms elapsed wall-clock for the render step+ mode_used 'rendered' or 'raw'+ error str or None++SSRF+====+The URL is validated via :func:`url_safety.validate_url_strict` before+Playwright sees it. Inside Playwright a ``route()`` handler intercepts+every subresource and aborts requests whose hostname resolves to a+non-public IP. This is defence in depth against DNS rebinding inside+Chromium's resolver. The residual rebinding risk for browser fetches+is documented in SECURITY.md.++CLI+===+ python render_page.py https://nuxt.com --mode always+ python render_page.py https://example.com --mode auto --json+ python render_page.py https://store.example.com --block image --block font+"""++from __future__ import annotations++import argparse+import json+import os+import re+import socket+import sys+import time+from typing import Optional+from urllib.parse import urlparse++# Optional native dependencies. Each is checked lazily so callers that+# only need raw-mode (mode='never') don't pay the import cost.+try:+ from playwright.sync_api import (+ sync_playwright,+ TimeoutError as PlaywrightTimeout,+ )+except ImportError: # pragma: no cover - exercised in environments without playwright+ sync_playwright = None+ PlaywrightTimeout = Exception # type: ignore[assignment,misc]++try:+ import trafilatura+except ImportError: # pragma: no cover+ trafilatura = None++try:+ from htmldate import find_date+except ImportError: # pragma: no cover+ find_date = None++# Reuse the canonical safety module.+_SCRIPTS_DIR = os.path.dirname(os.path.abspath(__file__))+if _SCRIPTS_DIR not in sys.path:+ sys.path.insert(0, _SCRIPTS_DIR)+from url_safety import ( # noqa: E402 (sys.path massage above is intentional)+ URLSafetyError,+ is_safe_ip,+ safe_requests_get,+ validate_url_strict,+)+++VIEWPORTS: dict[str, dict[str, int]] = {+ "desktop": {"width": 1920, "height": 1080, "device_scale": 1},+ "laptop": {"width": 1366, "height": 768, "device_scale": 1},+ "tablet": {"width": 768, "height": 1024, "device_scale": 1},+ "mobile": {"width": 375, "height": 812, "device_scale": 2},+}++USER_AGENT = (+ "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 "+ "(KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36 ClaudeSEO/2.0"+)++# Hydration-shell signatures. Any single match flips is_spa to True. These+# cover the dominant SPA frameworks: React (CRA, Vite, Remix), Next.js,+# Vue, Nuxt, Svelte, Astro islands, and the "JS required" noscript pattern.+_SPA_SHELL_PATTERNS = (+ '<div id="root"></div>',+ '<div id="__next">',+ '<div id="app"></div>',+ '<div id="__nuxt">',+ 'data-svelte-h=',+ '<astro-island ',+ 'you need to enable javascript',+ 'please enable javascript',+)++_TAG_STRIP = re.compile(r"<[^>]+>")+_WHITESPACE = re.compile(r"\s+")+++def _is_spa(raw_html: Optional[str]) -> bool:+ """Heuristic SPA detector. Conservative: any positive signal flips True."""+ if not raw_html:+ return True+ lc = raw_html.lower()+ if any(pattern in lc for pattern in _SPA_SHELL_PATTERNS):+ return True+ # Very thin <body> suggests JS-rendered content even without a shell.+ # Threshold (100 chars) sits between typical SPA shells (0-50 chars of+ # body text) and minimal informational pages like example.com (~125+ # chars). Tuned conservatively to avoid false positives that would+ # force a redundant Playwright render in auto mode.+ body_start = lc.find("<body")+ body_end = lc.rfind("</body>")+ if body_start != -1 and body_end > body_start:+ body = lc[body_start:body_end]+ text = _WHITESPACE.sub(" ", _TAG_STRIP.sub("", body)).strip()+ if len(text) < 100:+ return True+ return False+++def _make_route_handler(blocked_resource_types: set[str]):+ """+ Build a Playwright route() callback that:+ (1) refuses subresource requests resolving to non-public IPs+ (defence in depth against DNS rebinding inside Chromium),+ (2) blocks resource types in ``blocked_resource_types`` for faster+ renders when the caller doesn't need images/fonts/etc.+ """++ def handler(route, request): # type: ignore[no-untyped-def]+ try:+ if blocked_resource_types and request.resource_type in blocked_resource_types:+ route.abort()+ return++ parsed = urlparse(request.url)+ if parsed.scheme not in ("http", "https"):+ # data:, blob:, chrome-extension:, etc. — no DNS, allow.+ route.continue_()+ return+ host = parsed.hostname+ if not host:+ route.abort()+ return++ try:+ addrinfo = socket.getaddrinfo(+ host, None, family=socket.AF_INET, type=socket.SOCK_STREAM+ )+ except socket.gaierror:+ route.abort()+ return+ ips = {info[4][0] for info in addrinfo}+ if any(not is_safe_ip(ip) for ip in ips):+ route.abort()+ return+ route.continue_()+ except Exception: # pragma: no cover - fail-closed on any handler error+ try:+ route.abort()+ except Exception:+ pass++ return handler+++def render_page(+ url: str,+ *,+ mode: str = "auto",+ viewport: str = "desktop",+ timeout_ms: int = 15000,+ block_resources: Optional[list[str]] = None,+ extract_content: bool = True,+ user_agent: Optional[str] = None,+) -> dict:+ """Render or fetch ``url`` per the chosen mode. See module docstring."""+ result: dict = {+ "url": url,+ "status_code": None,+ "content": None,+ "raw_content": None,+ "is_spa": None,+ "extracted_text": None,+ "publication_date": None,+ "headers": {},+ "redirect_chain": [],+ "console_errors": [],+ "render_engine": None,+ "render_ms": None,+ "mode_used": None,+ "error": None,+ }++ if mode not in ("auto", "always", "never"):+ result["error"] = f"Invalid mode: {mode!r}"+ return result+ if viewport not in VIEWPORTS:+ result["error"] = f"Invalid viewport: {viewport!r}"+ return result++ # Pre-flight SSRF check.+ try:+ norm_url, _pinned_ip = validate_url_strict(url)+ result["url"] = norm_url+ except URLSafetyError as exc:+ result["error"] = f"url_safety: {exc}"+ return result++ # Step 1 — raw fetch (always; needed for SPA detection and as a baseline).+ try:+ resp = safe_requests_get(norm_url, timeout=30, allow_redirects=True)+ result["raw_content"] = resp.text+ if resp.history:+ result["redirect_chain"] = [+ {"url": r.url, "status_code": r.status_code} for r in resp.history+ ]+ raw_status = resp.status_code+ raw_headers = dict(resp.headers)+ final_raw_url = resp.url+ except Exception as exc:+ result["error"] = f"raw fetch failed: {exc}"+ return result++ result["is_spa"] = _is_spa(result["raw_content"])+ should_render = mode == "always" or (mode == "auto" and result["is_spa"])++ if not should_render:+ result["mode_used"] = "raw"+ result["url"] = final_raw_url+ result["status_code"] = raw_status+ result["headers"] = raw_headers+ result["content"] = result["raw_content"]+ else:+ result["mode_used"] = "rendered"+ if sync_playwright is None:+ result["error"] = (+ "playwright is required for rendered mode. "+ "Install: pip install -r requirements.txt "+ "&& playwright install chromium"+ )+ return result++ vp = VIEWPORTS[viewport]+ blocked = set(block_resources or [])+ route_handler = _make_route_handler(blocked)+ start = time.monotonic()++ try:+ with sync_playwright() as p:+ browser = p.chromium.launch(headless=True)+ context = browser.new_context(+ viewport={"width": vp["width"], "height": vp["height"]},+ device_scale_factor=vp["device_scale"],+ user_agent=user_agent or USER_AGENT,+ )+ page = context.new_page()++ def _on_console(msg): # type: ignore[no-untyped-def]+ if msg.type == "error":+ result["console_errors"].append(msg.text)++ page.on("console", _on_console)+ page.route("**/*", route_handler)++ response = page.goto(+ norm_url, wait_until="networkidle", timeout=timeout_ms+ )+ # Allow late hydration (deferred islands, useEffect chains).+ page.wait_for_timeout(500)++ result["url"] = page.url+ result["content"] = page.content()+ result["status_code"] = response.status if response else raw_status+ result["headers"] = (+ dict(response.all_headers()) if response else raw_headers+ )+ result["render_engine"] = "playwright-chromium"++ browser.close()+ except PlaywrightTimeout:+ result["error"] = f"playwright navigation timed out after {timeout_ms}ms"+ return result+ except Exception as exc:+ result["error"] = f"playwright error: {exc}"+ return result+ finally:+ result["render_ms"] = (time.monotonic() - start) * 1000.0++ # Step 2 — content extraction (works on either raw or rendered HTML).+ if extract_content and result["content"]:+ if trafilatura is not None:+ try:+ result["extracted_text"] = trafilatura.extract(+ result["content"],+ include_comments=False,+ include_tables=True,+ favor_recall=False,+ )+ except Exception:+ # Extraction is best-effort; never block the audit on it.+ pass+ if find_date is not None:+ try:+ result["publication_date"] = find_date(result["content"])+ except Exception:+ pass++ return result+++def _cli() -> None:+ parser = argparse.ArgumentParser(+ description="claude-seo shared headless renderer (Playwright + trafilatura)"+ )+ parser.add_argument("url", help="URL to render")+ parser.add_argument(+ "--mode",+ choices=("auto", "always", "never"),+ default="auto",+ help="auto: render only when SPA detected; always: always render; "+ "never: raw HTML only (default: auto)",+ )+ parser.add_argument(+ "--viewport", choices=list(VIEWPORTS), default="desktop"+ )+ parser.add_argument(+ "--timeout-ms",+ type=int,+ default=15000,+ help="Playwright navigation timeout in ms (default: 15000)",+ )+ parser.add_argument(+ "--block",+ action="append",+ default=[],+ choices=("image", "media", "font", "stylesheet"),+ help="resource types to block during render (faster, less accurate)",+ )+ parser.add_argument(+ "--no-extract",+ action="store_true",+ help="skip trafilatura and htmldate post-processing",+ )+ parser.add_argument(+ "--json",+ action="store_true",+ help="emit a JSON summary (truncates content fields)",+ )+ parser.add_argument("--output", "-o", help="write HTML content to file")+ args = parser.parse_args()++ res = render_page(+ args.url,+ mode=args.mode,+ viewport=args.viewport,+ timeout_ms=args.timeout_ms,+ block_resources=args.block or None,+ extract_content=not args.no_extract,+ )++ if args.json:+ summary = dict(res)+ # JSON-safe truncation so the CLI is usable from agents without+ # piping megabytes of HTML across stdio.+ for field, limit in (+ ("content", 500),+ ("raw_content", 200),+ ("extracted_text", 500),+ ):+ if summary.get(field):+ value = summary[field]+ summary[field] = (+ value[:limit] + "..." if len(value) > limit else value+ )+ print(json.dumps(summary, indent=2, default=str))+ sys.exit(1 if res["error"] else 0)++ if res["error"]:+ print(f"Error: {res['error']}", file=sys.stderr)+ sys.exit(1)++ if args.output:+ with open(args.output, "w", encoding="utf-8") as fh:+ fh.write(res["content"] or "")+ print(f"saved to {args.output}", file=sys.stderr)+ else:+ print(res["content"])++ print(+ f"\nFinal URL: {res['url']}\n"+ f"Status: {res['status_code']} | mode={res['mode_used']} | "+ f"is_spa={res['is_spa']}",+ file=sys.stderr,+ )+ if res["render_ms"]:+ print(+ f"Render: {res['render_ms']:.0f}ms via {res['render_engine']}",+ file=sys.stderr,+ )+ if res["publication_date"]:+ print(f"Publication date: {res['publication_date']}", file=sys.stderr)+ if res["console_errors"]:+ print(+ f"Console errors ({len(res['console_errors'])}):", file=sys.stderr+ )+ for err in res["console_errors"][:5]:+ print(f" - {err}", file=sys.stderr)+++if __name__ == "__main__":+ _cli()diff --git a/scripts/url_safety.py b/scripts/url_safety.pynew file mode 100644index 0000000..4316506--- /dev/null+++ b/scripts/url_safety.py@@ -0,0 +1,384 @@+#!/usr/bin/env python3+"""+Canonical URL safety module for claude-seo.++Centralizes SSRF protection, DNS rebinding mitigation, and DNS-pinned HTTP+fetching. Every script in this repository that accepts a user-supplied URL+MUST validate it through this module before issuing any network request.++Public API+==========++validate_url(url) -> bool+ Back-compat boolean check. Rejects non-http(s) schemes, missing+ hostnames, hard-blocked hostnames (localhost, cloud metadata+ endpoints), and IP literals that fall inside private/loopback/reserved+ ranges. Does NOT resolve DNS. Preserves the v1.9.9 contract used by+ google_auth.py.++validate_url_strict(url) -> tuple[str, str]+ Resolves the hostname via socket.getaddrinfo, validates every returned+ A record against the safety predicate, and returns+ ``(normalized_url, pinned_ipv4)``. Raises ``URLSafetyError`` if any+ resolved IP is non-public. Use this whenever the caller is about to+ open a network connection so DNS rebinding cannot swap a public IP+ for a private one between checks.++safe_requests_get(url, *, timeout=30, **kwargs) -> requests.Response+ ``requests.get(...)`` wrapped in a DNS-pinning context manager so the+ OS-level resolver only ever sees the pre-validated IP for the duration+ of the call. The original hostname is preserved in the HTTP Host+ header and TLS SNI; only the connect() target is forced to the pinned+ address.++safe_requests_session(url) -> context manager yielding requests.Session+ Same protection as ``safe_requests_get`` for callers that need a+ session (cookies, redirect chains, multiple requests to one host).++is_safe_ip(ip_str) -> bool+ True iff the address parses as IPv4/IPv6 and is none of:+ private, loopback, reserved, link-local, multicast, unspecified.++URLSafetyError+ ValueError subclass raised by the strict validator and pinning helpers.++Threading+=========+The DNS pinning helper is a critical section guarded by a non-blocking+``threading.Lock``. Two concurrent pinned fetches on the same process will+raise rather than corrupt the global ``socket.getaddrinfo`` reference.+claude-seo scripts are intentionally single-threaded; parallelism is+delegated to the agent-process layer.++Limitations+===========+Playwright/Chromium-based fetches (``render_page.py``,+``capture_screenshot.py``) perform their own DNS resolution inside+Chromium and therefore cannot be DNS-pinned at the Python layer. Those+scripts must:++ 1. Call ``validate_url_strict()`` as a pre-flight check, AND+ 2. Attach a Playwright ``route()`` handler that re-validates each+ resolved request IP and aborts subresource fetches to private+ ranges.++The residual DNS-rebinding risk for browser-based fetches is documented+in SECURITY.md.+"""++from __future__ import annotations++import ipaddress+import socket+import threading+from contextlib import contextmanager+from typing import Iterator, Optional+from urllib.parse import urlparse++try:+ import requests+except ImportError as exc: # pragma: no cover - hard dependency+ raise RuntimeError(+ "scripts/url_safety.py requires the 'requests' package. "+ "Install with: pip install -r requirements.txt"+ ) from exc+++__all__ = [+ "URLSafetyError",+ "is_safe_ip",+ "validate_url",+ "validate_url_strict",+ "safe_requests_get",+ "safe_requests_session",+]+++# Hard-blocked hostnames. Anything here is refused even before DNS resolution.+# Cloud metadata endpoints are the most common SSRF target; we list every+# documented address across AWS, Azure, GCP, Oracle, and Alibaba so a single+# typo (e.g., metadata.google.internal vs metadata.googleapis.internal)+# cannot slip through.+_BLOCKED_HOSTNAMES: frozenset[str] = frozenset(+ {+ "localhost",+ "ip6-localhost",+ "ip6-loopback",+ "metadata.google.internal",+ "metadata.goog",+ "metadata",+ "metadata.azure.com",+ "metadata.ec2.internal",+ "metadata.oraclecloud.com",+ # Numeric metadata endpoints (also caught by IP literal check, listed+ # explicitly for defence-in-depth and clearer error messages).+ "127.0.0.1",+ "0.0.0.0",+ "::1",+ "169.254.169.254", # AWS, Azure, GCP, Oracle, Alibaba metadata IPv4+ "fd00:ec2::254", # AWS IMDS IPv6+ }+)+++class URLSafetyError(ValueError):+ """Raised when a URL fails SSRF safety checks."""+++def is_safe_ip(ip_str: str) -> bool:+ """Return True iff ``ip_str`` is a public unicast address."""+ try:+ ip = ipaddress.ip_address(ip_str)+ except ValueError:+ return False+ return not (+ ip.is_private+ or ip.is_loopback+ or ip.is_reserved+ or ip.is_link_local+ or ip.is_multicast+ or ip.is_unspecified+ )+++def validate_url(url: str) -> bool:+ """+ Back-compat boolean validator. Does not resolve DNS.++ Returns False when:+ - Scheme is not http or https+ - Hostname is missing+ - Hostname is in the hard-block list+ - Hostname is an IP literal that fails ``is_safe_ip``+ Returns True for any other well-formed http(s) URL with a public-looking+ hostname. Use ``validate_url_strict`` if the caller will open a socket+ using the hostname (DNS rebinding cannot be defeated at parse time).+ """+ parsed = urlparse(url)+ if parsed.scheme not in ("http", "https"):+ return False+ if not parsed.hostname:+ return False+ hostname = parsed.hostname.lower()+ if hostname in _BLOCKED_HOSTNAMES:+ return False+ try:+ ipaddress.ip_address(hostname)+ except ValueError:+ return True # Hostname is a name, not a literal — OK at parse time.+ return is_safe_ip(hostname)+++def validate_url_strict(url: str) -> tuple[str, str]:+ """+ Resolve and validate the URL's hostname.++ Returns ``(url, pinned_ipv4)`` on success. Raises ``URLSafetyError`` if:+ - Scheme is invalid or hostname is missing.+ - Hostname is hard-blocked.+ - DNS resolution fails.+ - Any A record resolves to a non-public IP (DNS rebinding refused).++ Multi-A-record handling: every returned record must be public. A+ hostname with one public and one private A record is refused so an+ attacker cannot race the resolver between validate and connect.+ """+ parsed = urlparse(url)+ if parsed.scheme not in ("http", "https"):+ raise URLSafetyError(f"Invalid URL scheme: {parsed.scheme!r}")+ if not parsed.hostname:+ raise URLSafetyError("URL has no hostname")++ hostname = parsed.hostname.lower()+ if hostname in _BLOCKED_HOSTNAMES:+ raise URLSafetyError(f"Blocked hostname: {hostname}")++ # If the hostname is an IP literal, validate it directly without DNS.+ try:+ literal = ipaddress.ip_address(hostname)+ except ValueError:+ literal = None++ if literal is not None:+ if not is_safe_ip(hostname):+ raise URLSafetyError(f"Blocked IP literal: {hostname}")+ return url, str(literal)++ port = parsed.port or (443 if parsed.scheme == "https" else 80)+ try:+ addrinfo = socket.getaddrinfo(+ hostname,+ port,+ family=socket.AF_INET,+ type=socket.SOCK_STREAM,+ )+ except socket.gaierror as exc:+ raise URLSafetyError(f"DNS resolution failed for {hostname}: {exc}") from exc++ resolved_ips = sorted({info[4][0] for info in addrinfo})+ if not resolved_ips:+ raise URLSafetyError(f"No A records for {hostname}")++ for ip_str in resolved_ips:+ if not is_safe_ip(ip_str):+ raise URLSafetyError(+ f"DNS rebinding refused: {hostname} resolves to "+ f"non-public IP {ip_str}"+ )++ pinned = resolved_ips[0]+ return url, pinned+++# A single non-blocking lock guards the global getaddrinfo monkey-patch.+# This is a deliberate choice: claude-seo scripts run one URL fetch at a+# time, and we'd rather raise loudly than silently corrupt resolver state+# if a caller ever introduces threading.+_dns_patch_lock = threading.Lock()+++@contextmanager+def _pin_dns(hostname: str, pinned_ip: str, port: int) -> Iterator[None]:+ """+ Temporarily override ``socket.getaddrinfo`` so the named host resolves+ only to ``pinned_ip``. Other hostnames are forwarded to the real+ resolver. Restores the original function on exit, even on exception.+ """+ if not _dns_patch_lock.acquire(blocking=False):+ raise URLSafetyError(+ "DNS-pinned fetch already in progress on another thread; "+ "claude-seo url_safety is not thread-safe by design."+ )++ original_getaddrinfo = socket.getaddrinfo+ target = hostname.lower()+ pinned_record = (+ socket.AF_INET,+ socket.SOCK_STREAM,+ socket.IPPROTO_TCP,+ "",+ (pinned_ip, port),+ )++ def patched(host, requested_port, *args, **kwargs):+ if host and host.lower() == target:+ family = kwargs.get("family", args[0] if args else 0)+ if family in (0, socket.AF_UNSPEC, socket.AF_INET):+ # Reuse the pinned record regardless of which port the+ # caller asked for. urllib3 always asks for the same port+ # the URL specified, which is what we validated.+ return [(+ pinned_record[0],+ pinned_record[1],+ pinned_record[2],+ pinned_record[3],+ (pinned_ip, requested_port or port),+ )]+ raise socket.gaierror(+ socket.EAI_FAIL,+ f"url_safety: address family {family} refused for pinned "+ f"IPv4 host {host}",+ )+ return original_getaddrinfo(host, requested_port, *args, **kwargs)++ socket.getaddrinfo = patched # type: ignore[assignment]+ try:+ yield+ finally:+ socket.getaddrinfo = original_getaddrinfo # type: ignore[assignment]+ _dns_patch_lock.release()+++def safe_requests_get(+ url: str,+ *,+ timeout: int = 30,+ **kwargs,+) -> requests.Response:+ """+ ``requests.get`` with DNS-rebinding protection.++ The request's hostname is pinned to a pre-validated IP for the+ duration of the call. Standard ``requests`` semantics otherwise.+ """+ norm_url, pinned_ip = validate_url_strict(url)+ parsed = urlparse(norm_url)+ port = parsed.port or (443 if parsed.scheme == "https" else 80)+ assert parsed.hostname is not None # validate_url_strict guarantees this+ with _pin_dns(parsed.hostname, pinned_ip, port):+ return requests.get(norm_url, timeout=timeout, **kwargs)+++@contextmanager+def safe_requests_session(url: str) -> Iterator[requests.Session]:+ """+ Yield a ``requests.Session`` whose connections to ``url``'s hostname+ are DNS-pinned. Callers may make multiple requests to that host+ within the ``with`` block without re-resolving.+ """+ norm_url, pinned_ip = validate_url_strict(url)+ parsed = urlparse(norm_url)+ port = parsed.port or (443 if parsed.scheme == "https" else 80)+ assert parsed.hostname is not None+ session = requests.Session()+ with _pin_dns(parsed.hostname, pinned_ip, port):+ try:+ yield session+ finally:+ session.close()+++def _cli() -> None:+ """Tiny CLI for manual SSRF-policy checks. Not used by other scripts."""+ import argparse+ import json+ import sys++ parser = argparse.ArgumentParser(+ description="Validate a URL against claude-seo's SSRF policy."+ )+ parser.add_argument("url", help="URL to validate")+ parser.add_argument(+ "--strict",+ action="store_true",+ help="Run DNS resolution and refuse on any non-public A record.",+ )+ parser.add_argument(+ "--json",+ action="store_true",+ help="Emit a JSON object instead of a one-line summary.",+ )+ args = parser.parse_args()++ result: dict[str, Optional[str]] = {+ "url": args.url,+ "mode": "strict" if args.strict else "parse",+ "ok": None,+ "pinned_ip": None,+ "error": None,+ }++ try:+ if args.strict:+ _, ip = validate_url_strict(args.url)+ result["ok"] = "true"+ result["pinned_ip"] = ip+ else:+ result["ok"] = "true" if validate_url(args.url) else "false"+ except URLSafetyError as exc:+ result["ok"] = "false"+ result["error"] = str(exc)++ if args.json:+ print(json.dumps(result, indent=2))+ else:+ if result["ok"] == "true":+ extra = f" -> {result['pinned_ip']}" if result["pinned_ip"] else ""+ print(f"OK: {args.url}{extra}")+ else:+ print(f"BLOCKED: {args.url} ({result['error'] or 'parse-time reject'})")+ sys.exit(2)+++if __name__ == "__main__":+ _cli()