Skip to content

Universal SEO skill for Claude Code. 26 sub-skills + 19 sub-agents covering technical SEO, E-E-A-T, schema, GEO/AEO, agent readiness (Lighthouse Agentic Browsing, WebMCP, llms.txt), backlinks, local SEO, e-commerce, international SEO, Google APIs, and PDF/Excel reporting. 9 optional extensions for live SEO data.

36.1 KB
Raw Blame
#!/usr/bin/env python3
"""Discover sitemaps through robots.txt and common locations safely.

Every network request uses the repository's DNS-pinned URL safety layer. The
helper accepts cross-host Sitemap directives because the sitemap protocol
allows them, but validates each target independently before connecting.
A gzip sitemap (.xml.gz, detected by its magic bytes) is inflated under the
50 MiB uncompressed cap and classified by its content; its entry carries
"gzip": true.

Usage:
    sitemap_discovery.py <url> --json
    sitemap_discovery.py <url> --expand --json
    sitemap_discovery.py <url> --expand --max-children 200 --sample 5 --json

--expand also reads every sitemap that was found: it follows a sitemap index
one level down to its child sitemaps and adds an "expanded" object with
per-file statistics (URL count, lastmod quality, off-host entries, protocol
limit findings) and totals across files. Every URL is counted; only a bounded
sample is printed.

Sitemap file URLs are printed without their query string ("query_redacted").
"sample_urls" are page URLs meant to be requested, so they keep their query;
a page URL with a credential-like parameter (token, session, signature) is
never sampled, and is counted and shown with the value replaced in
"redacted_url_count" and "redacted_url_examples" instead.

Duplicates are counted per entry, after normalisation. In one file, the first
entry for a URL that an earlier file already listed is one cross-file
duplicate ("duplicates_across_files"); every further entry for a URL the same
file already listed is one within-file duplicate ("duplicate_count",
"duplicates_within_files"). Each entry is counted once, so for the page
sitemaps: URLs = unique + within-file + cross-file duplicates + invalid locs.

A malformed URL in robots.txt or in a sitemap is recorded and skipped.
"""

from __future__ import annotations

import argparse
import gzip
import hashlib
import io
import json
import os
import re
import sys
import zlib
from collections import Counter
from collections.abc import Iterable, Iterator
from datetime import datetime, timedelta, timezone
from urllib.parse import unquote, urlparse, urlunparse

import requests
from lxml import etree

_SCRIPTS = os.path.dirname(os.path.abspath(__file__))
if _SCRIPTS not in sys.path:
    sys.path.insert(0, _SCRIPTS)

from url_safety import URLSafetyError, read_bounded, safe_requests_session, validate_url  # noqa: E402

COMMON_PATHS = (
    "/sitemap.xml",
    "/sitemap_index.xml",
    "/sitemap-index.xml",
    "/wp-sitemap.xml",
)
MAX_DECLARED = 16
MAX_ROBOTS_BYTES = 1024 * 1024
MAX_SITEMAP_BYTES = 50 * 1024 * 1024
MAX_SITEMAP_URLS = 50000
# Protocol limit: a <loc> value must be shorter than 2,048 characters.
MAX_LOC_CHARS = 2048
DEFAULT_MAX_CHILDREN = 50
DEFAULT_SAMPLE = 20
MAX_EXAMPLES = 5
# Distinct URLs remembered for duplicate detection (about 100 bytes each).
MAX_TRACKED_URLS = 2_000_000
USER_AGENT = "ClaudeSEO/2.2 SitemapDiscovery"
_SITEMAP_LINE = re.compile(r"^\s*sitemap\s*:\s*(\S+)\s*$", re.IGNORECASE)
_TOO_LARGE = "Sitemap exceeds the 50 MiB uncompressed protocol limit"
_GZIP_MAGIC = b"\x1f\x8b"
_INFLATE_CHUNK = 1024 * 1024
_TRACKING_KEYS = frozenset({"fbclid", "gclid", "mc_cid", "mc_eid", "_ga", "ref"})
# W3C Datetime: YYYY, YYYY-MM, YYYY-MM-DD, or a date with a time and a zone.
_W3C_DATETIME = re.compile(
    r"(\d{4})(?:-(\d{2})(?:-(\d{2})"
    r"(?:T(\d{2}):(\d{2})(?::(\d{2})(?:\.\d{1,9})?)?(Z|[+-]\d{2}:\d{2}))?)?)?",
    re.ASCII,
)
# Query parameters whose value is a credential, not part of the page address.
_SECRET_PARAM = re.compile(
    r"token|secret|passw|signature|credential|session|sessid|api[-_]?key|access[-_]?key"
    r"|^(?:key|sig|auth|pwd|sid|jwt)$"
)
_MALFORMED = "Malformed sitemap URL"
# Element that holds one entry, per sitemap kind.
_ENTRY_TAG = {"urlset": "url", "sitemapindex": "sitemap", "rss": "item", "feed": "entry"}


def _display_url(url: str) -> tuple[str, bool]:
    """Return a safe-to-display URL with userinfo, query, and fragment removed.

    Never raises: text that does not parse as a URL is cut the same way by hand.
    """
    try:
        parsed = urlparse(url)
    except ValueError:
        text = re.split(r"[?#]", url, maxsplit=1)[0]
        return re.sub(r"(?<=//)[^/]*@", "", text), True
    host = parsed.hostname or ""
    try:
        port = parsed.port
    except ValueError:
        port = None
        redacted = True
    else:
        redacted = False
    if port:
        host = f"{host}:{port}"
    clean = urlunparse((parsed.scheme, host, parsed.path or "/", "", "", ""))
    redacted = redacted or bool(parsed.username or parsed.password or parsed.query or parsed.fragment)
    return clean, redacted


def _origin(url: str) -> str:
    if "://" not in url:
        url = f"https://{url}"
    if not validate_url(url):
        raise ValueError("Target must be a public HTTP or HTTPS URL")
    parsed = urlparse(url)
    netloc = parsed.hostname or ""
    try:
        port = parsed.port
    except ValueError as exc:
        raise ValueError("Target URL contains an invalid port") from exc
    if port:
        netloc = f"{netloc}:{port}"
    return urlunparse((parsed.scheme, netloc, "", "", "", ""))


def _bounded_fetch(url: str, max_bytes: int) -> dict:
    """Fetch at most max_bytes after decompression without exposing response text."""
    result = {
        "status_code": None,
        "content": b"",
        "content_type": "",
        "final_url": url,
        "too_large": False,
        "error": None,
    }
    try:
        with safe_requests_session(url) as session:
            response = session.get(
                url,
                headers={"User-Agent": USER_AGENT, "Accept": "application/xml,text/xml,text/plain,*/*"},
                timeout=30,
                allow_redirects=True,
                stream=True,
            )
            result["status_code"] = response.status_code
            result["content_type"] = response.headers.get("Content-Type", "")
            result["final_url"] = response.url
            result["content"], result["too_large"] = read_bounded(response, max_bytes)
            response.close()
    except URLSafetyError:
        result["error"] = "URL safety validation failed"
    except requests.exceptions.Timeout:
        result["error"] = "Request timed out"
    except requests.exceptions.RequestException:
        result["error"] = "Request failed"
    except ValueError:
        # Text that is not a URL at all, such as an unclosed IPv6 bracket.
        result["error"] = _MALFORMED
    return result


def _robots_sitemaps(content: bytes) -> list[str]:
    """Return Sitemap URLs declared in a robots.txt body.

    Google's robots.txt syntax is ``field:value`` with an optional ``#``
    comment through the end of the line. The comment is not part of the URL.
    A leading UTF-8 BOM is an encoding signature, not part of the first field.
    """
    text = content.decode("utf-8", errors="replace")
    if text.startswith("\ufeff"):
        text = text[1:]
    found = []
    for line in text.splitlines():
        line = line.split("#", 1)[0]
        match = _SITEMAP_LINE.match(line)
        if match and match.group(1) not in found:
            found.append(match.group(1))
    return found


def _valid_sitemap_url_syntax(value: str) -> bool:
    """Check a text-sitemap entry without resolving or connecting to its host."""
    if not value or any(char.isspace() for char in value) or "\\" in value:
        return False
    try:
        parsed = urlparse(value)
        _ = parsed.port
    except ValueError:
        return False
    return (
        parsed.scheme.lower() in {"http", "https"}
        and bool(parsed.hostname)
        and parsed.username is None
        and parsed.password is None
    )


def _classify(content: bytes, content_type: str, url: str) -> tuple[str | None, str | None, object, list[str] | None]:
    """Classify a sitemap body. Returns (kind, error, XML root, text lines).

    The root is set for an accepted XML sitemap and the lines for a text
    sitemap, including one refused for passing the 50,000 URL limit, so
    --expand can count entries without parsing the body a second time.
    """
    if not content.strip():
        return None, "empty response", None, None
    if b"<!DOCTYPE" in content.upper():
        return None, "DOCTYPE is not allowed in sitemap XML", None, None

    stripped = content.removeprefix(b"\xef\xbb\xbf").lstrip()
    if stripped.startswith(b"<"):
        try:
            parser = etree.XMLParser(
                resolve_entities=False,
                no_network=True,
                load_dtd=False,
                recover=False,
                huge_tree=False,
            )
            root = etree.fromstring(content, parser=parser)
        except (etree.XMLSyntaxError, ValueError):
            return None, "invalid XML", None, None
        local = etree.QName(root).localname.lower()
        if local in {"urlset", "sitemapindex"}:
            return local, None, root, None
        if local in {"rss", "feed"}:
            return local, None, root, None
        return None, "unsupported XML root", None, None

    if "text/plain" in content_type.lower() or _url_path(url).lower().endswith(".txt"):
        # Decode the BOM-free bytes: a leading UTF-8 BOM is an encoding
        # signature, not part of the first URL.
        lines = [line.strip() for line in stripped.decode("utf-8", errors="replace").splitlines() if line.strip()]
        if len(lines) > MAX_SITEMAP_URLS:
            return None, "text sitemap exceeds the 50,000 URL protocol limit", None, lines
        # Entries are reported only as a sitemap classification, not fetched.
        # Avoid up to 50,000 DNS lookups here; connection-time validation still
        # applies to every sitemap candidate that discovery fetches.
        if lines and all(_valid_sitemap_url_syntax(line) for line in lines):
            return "text", None, None, lines
        return None, "invalid text sitemap", None, None
    return None, "response is not a supported sitemap format", None, None


def _sitemap_kind(content: bytes, content_type: str, url: str) -> tuple[str | None, str | None]:
    kind, error, _root, _lines = _classify(content, content_type, url)
    return kind, error


def _gunzip_bounded(raw: bytes, max_bytes: int) -> tuple[bytes, bool]:
    """Inflate gzip data in chunks, stopping once the output passes max_bytes.

    Returns (data, too_large). The output is never held past max_bytes plus
    one chunk, so a gzip bomb cannot exhaust memory. Raises ValueError when
    the data is not a complete gzip stream.
    """
    chunks = []
    total = 0
    try:
        with gzip.GzipFile(fileobj=io.BytesIO(raw)) as stream:
            while True:
                chunk = stream.read(_INFLATE_CHUNK)
                if not chunk:
                    break
                total += len(chunk)
                if total > max_bytes:
                    return b"", True
                chunks.append(chunk)
    except (OSError, EOFError, zlib.error) as exc:
        raise ValueError("invalid gzip data") from exc
    return b"".join(chunks), False


def _read_body(fetched: dict) -> dict:
    """Classify a fetched body. A gzip file is inflated first, under the size cap."""
    body = {
        "kind": None,
        "error": None,
        "root": None,
        "lines": None,
        "gzip": False,
        "decompressed_bytes": None,
        "too_large": False,
    }
    content = fetched["content"]
    url = fetched["final_url"]
    if content[:2] == _GZIP_MAGIC:
        body["gzip"] = True
        try:
            content, body["too_large"] = _gunzip_bounded(content, MAX_SITEMAP_BYTES)
        except ValueError:
            body["error"] = "invalid gzip data"
            return body
        if body["too_large"]:
            body["error"] = _TOO_LARGE
            return body
        body["decompressed_bytes"] = len(content)
        try:
            parsed = urlparse(url)
            if parsed.path.lower().endswith(".gz"):
                # sitemap.txt.gz is a text sitemap: classify it by the inner name.
                url = urlunparse(parsed._replace(path=parsed.path[:-3]))
        except ValueError:
            pass
    body["kind"], body["error"], body["root"], body["lines"] = _classify(content, fetched["content_type"], url)
    return body


def _fetch_sitemap(url: str) -> tuple[dict, dict | None, str | None]:
    """Fetch one sitemap candidate through the bounded, validated fetch.

    Returns (fetched, body, error). The body is None when nothing usable came
    back: a refused or failed request, an oversized body, or a non-2xx status.
    """
    fetched = _bounded_fetch(url, MAX_SITEMAP_BYTES)
    if fetched["error"]:
        return fetched, None, fetched["error"]
    if fetched["too_large"]:
        return fetched, None, _TOO_LARGE
    status = fetched["status_code"]
    if status is None or not 200 <= status < 300:
        return fetched, None, f"HTTP {status}" if status is not None else "No response"
    body = _read_body(fetched)
    return fetched, body, body["error"]


def _parses(url: str) -> bool:
    try:
        urlparse(url)
    except ValueError:
        return False
    return True


def _url_path(url: str) -> str:
    try:
        return urlparse(url).path
    except ValueError:
        return ""


def _host(url: str) -> str:
    """Lower-case host of a URL; empty when the text does not parse."""
    try:
        return (urlparse(url).hostname or "").lower().rstrip(".")
    except ValueError:
        return ""


def _sample_url(loc: str) -> tuple[str, bool]:
    """Return a valid page URL as printed, and whether a value was replaced.

    A sample is there to be requested, so the query is kept. The fragment is
    dropped (it is never sent). The value of a credential-like parameter is
    replaced, which makes that URL useless for a status check.
    """
    if "?" not in loc:
        return loc.split("#", 1)[0], False
    parsed = urlparse(loc)
    pairs = []
    replaced = False
    for pair in parsed.query.split("&"):
        key, equals, _value = pair.partition("=")
        if equals and _SECRET_PARAM.search(unquote(key).lower()):
            pair = f"{key}=REDACTED"
            replaced = True
        pairs.append(pair)
    if not replaced:
        return loc.split("#", 1)[0], False
    netloc = parsed.netloc.rpartition("@")[2]
    return urlunparse((parsed.scheme, netloc, parsed.path, parsed.params, "&".join(pairs), "")), True


def _local(element) -> str:
    """Lower-case tag name without its namespace; empty for comments and PIs."""
    tag = element.tag
    if not isinstance(tag, str):
        return ""
    return tag.rsplit("}", 1)[-1].lower()


def _xml_entries(kind: str, root) -> Iterator[tuple[str | None, str | None]]:
    """Yield (loc, lastmod) for each entry of a parsed sitemap, index, or feed."""
    feed = kind in {"rss", "feed"}
    for node in root.iter() if feed else root:
        if _local(node) != _ENTRY_TAG[kind]:
            continue
        loc = lastmod = None
        for child in node:
            tag = _local(child)
            text = (child.text or "").strip()
            if kind == "feed":
                if tag == "link" and loc is None and child.get("rel", "alternate") == "alternate":
                    loc = (child.get("href") or "").strip()
                elif tag == "updated" and lastmod is None:
                    lastmod = text
            elif kind == "rss":
                if tag == "link" and loc is None:
                    loc = text
            elif tag == "loc" and loc is None:
                loc = text
            elif tag == "lastmod" and lastmod is None:
                lastmod = text
        yield loc or None, lastmod or None


def _parse_w3c_datetime(value: str) -> datetime | None:
    """Return a comparable UTC time for a W3C Datetime value, or None."""
    match = _W3C_DATETIME.fullmatch(value)
    if not match:
        return None
    year, month, day, hour, minute, second, zone = match.groups()
    try:
        moment = datetime(
            int(year), int(month or 1), int(day or 1),
            int(hour or 0), int(minute or 0), int(second or 0),
            tzinfo=timezone.utc,
        )
        if zone and zone != "Z":
            hours, minutes = int(zone[1:3]), int(zone[4:6])
            if hours > 23 or minutes > 59:
                return None
            offset = timedelta(hours=hours, minutes=minutes)
            moment = moment - offset if zone[0] == "+" else moment + offset
    except (ValueError, OverflowError):
        return None
    return moment


def _normalize_url(url: str) -> str | None:
    """Normalise a URL for duplicate detection only; never fetch or print it.

    Lower-cases scheme and host, drops a default port, the fragment, and
    tracking parameters, and collapses repeated slashes. The trailing slash is
    kept: /about and /about/ are different addresses.
    """
    try:
        parsed = urlparse(url)
        port = parsed.port
    except ValueError:
        return None
    scheme = parsed.scheme.lower()
    if scheme not in {"http", "https"} or not parsed.hostname:
        return None
    host = parsed.hostname.lower().rstrip(".")
    if ":" in host:
        host = f"[{host}]"
    netloc = f"{host}:{port}" if port and port != (443 if scheme == "https" else 80) else host
    path = re.sub(r"/{2,}", "/", parsed.path or "/")
    kept = []
    for pair in parsed.query.split("&"):
        key = unquote(pair.split("=", 1)[0]).lower()
        if pair and not key.startswith("utm_") and key not in _TRACKING_KEYS:
            kept.append(pair)
    return urlunparse((scheme, netloc, path, parsed.params, "&".join(kept), ""))


class _Expansion:
    """Per-file statistics and cross-file totals for --expand.

    Duplicates are counted per entry on the normalised URL. In one file, the
    first entry for a URL that an earlier file listed is a cross-file
    duplicate; any further entry for a URL that file already listed is a
    within-file duplicate. No entry is counted both ways.
    """

    def __init__(self, sample_size: int) -> None:
        self.sample_size = sample_size
        self.files: list[dict] = []
        # Fetched and final URL of every sitemap request so far, with its kind.
        self.kinds: dict[str, str | None] = {}
        # Final URLs of the valid files already reported.
        self.analysed: set[str] = set()
        # Top-level indexes waiting for their children: (report, final URL, child URLs).
        self.pending: list[tuple[dict, str, list[str]]] = []
        # Digest of a normalised URL -> number of the last file that listed it.
        self.seen: dict[bytes, int] = {}
        self.tracking_complete = True
        self.unique = 0

    def add_top_level(self, candidate: str, entry: dict, fetched: dict, body: dict | None) -> None:
        """Record a candidate from robots.txt or a common path."""
        final_url = fetched["final_url"]
        self.kinds[candidate] = entry["kind"]
        if entry["valid"] and final_url in self.analysed:
            return
        self.kinds[final_url] = entry["kind"]
        report, children = self._report(entry, fetched, body)
        if entry["valid"]:
            self.analysed.add(final_url)
            if entry["kind"] == "sitemapindex":
                self.pending.append((report, final_url, children))
        if entry["valid"] or report["findings"]:
            self.files.append(report)

    def follow_children(self, max_children: int, warnings: list[str]) -> dict:
        """Fetch the child sitemaps of every top-level index, one level deep.

        Child URLs come from fetched content, so each one takes the same
        validated fetch path as any other candidate. A child that is itself an
        index is reported, never followed.
        """
        listed = fetched_count = skipped = 0
        scheduled: set[str] = set()
        for parent, parent_url, children in self.pending:
            for loc in children:
                if loc in self.kinds:
                    if self.kinds[loc] == "sitemapindex":
                        self._nested(parent, _display_url(loc)[0], warnings)
                    continue
                if loc in scheduled:
                    continue
                scheduled.add(loc)
                listed += 1
                if fetched_count >= max_children:
                    skipped += 1
                    continue
                fetched_count += 1
                fetched, body, error = _fetch_sitemap(loc)
                kind = body["kind"] if body else None
                final_url = fetched["final_url"]
                self.kinds[loc] = kind
                if kind is not None and final_url in self.analysed:
                    # Redirects to a file that is already reported: count it once.
                    if self.kinds.get(final_url) == "sitemapindex":
                        self._nested(parent, _display_url(final_url)[0], warnings)
                    continue
                self.kinds[final_url] = kind
                display, redacted = _display_url(final_url if kind is not None else loc)
                entry = {
                    "url": display,
                    "query_redacted": redacted or _display_url(loc)[1],
                    "source": "sitemapindex",
                    "parent": parent["url"],
                    "status_code": fetched["status_code"],
                    "kind": kind,
                    "valid": kind is not None,
                    "error": error,
                }
                if _host(loc) != _host(parent_url):
                    entry["cross_host"] = True
                report, _grandchildren = self._report(entry, fetched, body)
                if kind is not None:
                    self.analysed.add(final_url)
                self.files.append(report)
                if kind == "sitemapindex":
                    self._nested(parent, display, warnings)
        if skipped:
            warnings.append(
                f"Child sitemap limit reached ({max_children}); {skipped} child sitemaps were not fetched"
            )
        return {"limit": max_children, "listed": listed, "fetched": fetched_count, "skipped": skipped}

    def totals(self, warnings: list[str]) -> dict:
        valid = [item for item in self.files if item["valid"]]
        totals = {
            "sitemap_files": len(valid),
            "total_urls": sum(item["url_count"] for item in valid),
            "unique_urls": self.unique,
            "duplicates_within_files": sum(
                item["duplicate_count"] or 0 for item in valid if item["kind"] != "sitemapindex"
            ),
            "duplicates_across_files": sum(item["_cross_file_duplicates"] for item in valid),
            "invalid_locs": sum(item["invalid_loc_count"] for item in valid),
        }
        if not self.tracking_complete:
            totals["unique_urls"] = None
            totals["duplicates_within_files"] = None
            totals["duplicates_across_files"] = None
            warnings.append(
                f"More than {MAX_TRACKED_URLS} distinct URLs; unique and duplicate totals are not reported"
            )
        for item in self.files:
            item.pop("_cross_file_duplicates", None)
        return totals

    @staticmethod
    def _nested(parent: dict, display: str, warnings: list[str]) -> None:
        if "nested_sitemap_index" not in parent["findings"]:
            parent["findings"].append("nested_sitemap_index")
        warnings.append(f"Nested sitemap index was not followed: {display} (listed in {parent['url']})")

    def _report(self, entry: dict, fetched: dict, body: dict | None) -> tuple[dict, list[str]]:
        """Build the report for one file. Returns it with the child URLs of an index."""
        report = dict(entry)
        report.update({
            "gzip": bool(body and body["gzip"]),
            "bytes": None if fetched["too_large"] else len(fetched["content"]),
            "decompressed_bytes": body["decompressed_bytes"] if body else None,
            "url_count": None,
            "child_count": None,
            "invalid_loc_count": None,
            "duplicate_count": None,
            "lastmod_count": None,
            "lastmod_invalid_count": None,
            "lastmod_oldest": None,
            "lastmod_newest": None,
            "lastmod_most_common": None,
            "lastmod_identical_share": None,
            "cross_host_url_count": None,
            "cross_host_url_examples": [],
            "redacted_url_count": None,
            "redacted_url_examples": [],
            "findings": [],
            "sample_urls": [],
            "_cross_file_duplicates": 0,
        })
        if fetched["too_large"] or (body and body["too_large"]):
            report["findings"].append("over_50mib_uncompressed")
        if body is None:
            return report, []
        if body["kind"] is None:
            if body["lines"] is not None:
                # A text sitemap refused for its size: the count is still known.
                report["url_count"] = len(body["lines"])
                report["findings"].append("over_50000_urls")
            return report, []
        if body["root"] is not None:
            entries: Iterable[tuple[str | None, str | None]] = _xml_entries(body["kind"], body["root"])
        else:
            entries = ((line, None) for line in body["lines"])
        return report, self._summarise(report, body["kind"], entries, fetched["final_url"])

    def _summarise(self, report: dict, kind: str, entries: Iterable, final_url: str) -> list[str]:
        """Count every entry of one file into its report. Returns an index's child URLs."""
        index = kind == "sitemapindex"
        file_number = len(self.files)
        host = _host(final_url)
        count = invalid = with_lastmod = bad_lastmod = cross_host = repeats = cross_file = secret = 0
        oldest: tuple[datetime, str] | None = None
        newest: tuple[datetime, str] | None = None
        lastmods: Counter[str] = Counter()
        examples: list[str] = report["cross_host_url_examples"]
        secret_examples: list[str] = report["redacted_url_examples"]
        sampled: set[str] = set()
        children: list[str] = []
        child_set: set[str] = set()
        for loc, lastmod in entries:
            count += 1
            if lastmod:
                with_lastmod += 1
                moment = _parse_w3c_datetime(lastmod)
                if moment is None:
                    bad_lastmod += 1
                else:
                    lastmods[lastmod] += 1
                    if oldest is None or moment < oldest[0]:
                        oldest = (moment, lastmod)
                    if newest is None or moment > newest[0]:
                        newest = (moment, lastmod)
            if not loc or len(loc) >= MAX_LOC_CHARS or not _valid_sitemap_url_syntax(loc):
                invalid += 1
                continue
            if _host(loc) != host:
                cross_host += 1
                shown = _display_url(loc)[0]
                if len(examples) < MAX_EXAMPLES and shown not in examples:
                    examples.append(shown)
            if index:
                if loc in child_set:
                    repeats += 1
                else:
                    child_set.add(loc)
                    children.append(loc)
                continue
            shown, replaced = _sample_url(loc)
            if replaced:
                secret += 1
                if len(secret_examples) < MAX_EXAMPLES and shown not in secret_examples:
                    secret_examples.append(shown)
            elif len(sampled) < self.sample_size and shown not in sampled:
                sampled.add(shown)
                report["sample_urls"].append(shown)
            seen_in = self._track(loc, file_number)
            if seen_in == "same_file":
                repeats += 1
            elif seen_in == "earlier_file":
                cross_file += 1

        report.update({
            "url_count": 0 if index else count,
            "child_count": count if index else 0,
            "invalid_loc_count": invalid,
            "duplicate_count": repeats if index or self.tracking_complete else None,
            "lastmod_count": with_lastmod,
            "lastmod_invalid_count": bad_lastmod,
            "lastmod_oldest": oldest[1] if oldest else None,
            "lastmod_newest": newest[1] if newest else None,
            "cross_host_url_count": cross_host,
            "redacted_url_count": secret,
            "_cross_file_duplicates": cross_file,
        })
        if lastmods:
            value, uses = lastmods.most_common(1)[0]
            report["lastmod_most_common"] = value
            report["lastmod_identical_share"] = round(uses / with_lastmod, 4)
        if count > MAX_SITEMAP_URLS:
            report["findings"].append("over_50000_sitemaps" if index else "over_50000_urls")
        return children

    def _track(self, loc: str, file_number: int) -> str | None:
        """Remember one entry and say where its URL was listed before.

        Returns "new", "same_file" (this file already listed it: a within-file
        duplicate), "earlier_file" (first entry in this file for a URL that an
        earlier file listed: a cross-file duplicate), or None when the URL
        cannot be tracked. The table then points at this file, so a second
        entry here is a within-file duplicate, not another cross-file one.
        """
        key = _normalize_url(loc)
        if key is None:
            return None
        digest = hashlib.blake2b(key.encode("utf-8", "replace"), digest_size=8).digest()
        last_file = self.seen.get(digest)
        if last_file is None:
            if len(self.seen) < MAX_TRACKED_URLS:
                self.seen[digest] = file_number
                self.unique += 1
                return "new"
            self.tracking_complete = False
            return None
        if last_file == file_number:
            return "same_file"
        self.seen[digest] = file_number
        return "earlier_file"


def discover_sitemaps(
    target_url: str,
    expand: bool = False,
    max_children: int = DEFAULT_MAX_CHILDREN,
    sample_size: int = DEFAULT_SAMPLE,
) -> dict:
    result = {
        "target": None,
        "robots_url": None,
        "declared": [],
        "found": [],
        "checked": [],
        "warnings": [],
        "error": None,
    }
    try:
        origin = _origin(target_url)
    except ValueError as exc:
        result["error"] = str(exc)
        return result

    result["target"] = origin
    robots_url = f"{origin}/robots.txt"
    result["robots_url"] = robots_url
    robots = _bounded_fetch(robots_url, MAX_ROBOTS_BYTES)
    declared_raw = []
    if robots["error"]:
        result["warnings"].append("robots.txt could not be fetched safely")
    elif robots["too_large"]:
        result["warnings"].append("robots.txt exceeded the 1 MiB discovery limit")
    elif robots["status_code"] == 200:
        declared_raw = _robots_sitemaps(robots["content"])
    elif robots["status_code"] is not None:
        result["warnings"].append(f"robots.txt returned HTTP {robots['status_code']}")

    if len(declared_raw) > MAX_DECLARED:
        result["warnings"].append(
            f"robots.txt declared more than {MAX_DECLARED} sitemaps; extra entries were not fetched"
        )
        declared_raw = declared_raw[:MAX_DECLARED]

    for item in declared_raw:
        display, redacted = _display_url(item)
        result["declared"].append({"url": display, "query_redacted": redacted})

    candidates = list(declared_raw)
    candidates.extend(f"{origin}{path}" for path in COMMON_PATHS)
    deduped = []
    seen = set()
    for candidate in candidates:
        if candidate not in seen:
            seen.add(candidate)
            deduped.append(candidate)

    expansion = _Expansion(sample_size) if expand else None
    target_host = urlparse(origin).hostname
    # Several paths can redirect to one sitemap (/sitemap.xml to
    # /sitemap_index.xml): list each file once, by its final URL.
    listed = set()
    for candidate in deduped:
        display, query_redacted = _display_url(candidate)
        entry = {
            "url": display,
            "query_redacted": query_redacted,
            "source": "robots.txt" if candidate in declared_raw else "common_path",
            "status_code": None,
            "kind": None,
            "valid": False,
            "error": None,
        }
        if not _parses(candidate):
            # robots.txt is page-controlled text: record the line, fetch nothing.
            entry["error"] = _MALFORMED
            result["checked"].append(entry)
            continue
        if urlparse(candidate).hostname != target_host and candidate in declared_raw:
            entry["cross_host"] = True

        fetched, body, entry["error"] = _fetch_sitemap(candidate)
        entry["status_code"] = fetched["status_code"]
        if body is not None:
            if body["gzip"]:
                entry["gzip"] = True
            entry["kind"] = body["kind"]
            entry["valid"] = body["kind"] is not None
            if entry["valid"]:
                final_display, final_redacted = _display_url(fetched["final_url"])
                entry["url"] = final_display
                entry["query_redacted"] = query_redacted or final_redacted
                if fetched["final_url"] not in listed:
                    listed.add(fetched["final_url"])
                    result["found"].append(dict(entry))
        result["checked"].append(entry)
        if expansion is not None:
            expansion.add_top_level(candidate, entry, fetched, body)

    if expansion is not None:
        # Release the last parsed tree before the child sitemaps are read.
        fetched = body = None
        children = expansion.follow_children(max_children, result["warnings"])
        result["expanded"] = {
            "files": expansion.files,
            "totals": expansion.totals(result["warnings"]),
            "children": children,
            "truncated": children["skipped"] > 0,
            "sample_size": sample_size,
        }
    return result


def _bounded_int(low: int, high: int):
    def parse(value: str) -> int:
        number = int(value)
        if not low <= number <= high:
            raise argparse.ArgumentTypeError(f"must be between {low} and {high}")
        return number
    return parse


def main() -> int:
    parser = argparse.ArgumentParser(description="Discover sitemaps safely")
    parser.add_argument("url", help="Public site URL")
    parser.add_argument("--json", action="store_true", help="Output structured JSON")
    parser.add_argument(
        "--expand",
        action="store_true",
        help="Read each sitemap found, follow a sitemap index to its children, and report per-file statistics",
    )
    parser.add_argument(
        "--max-children",
        type=_bounded_int(0, MAX_SITEMAP_URLS),
        default=DEFAULT_MAX_CHILDREN,
        metavar="N",
        help=f"With --expand: most child sitemaps to fetch across all indexes (default {DEFAULT_MAX_CHILDREN})",
    )
    parser.add_argument(
        "--sample",
        type=_bounded_int(0, 1000),
        default=DEFAULT_SAMPLE,
        metavar="N",
        help=f"With --expand: URLs kept per file in sample_urls; all are counted (default {DEFAULT_SAMPLE})",
    )
    args = parser.parse_args()
    result = discover_sitemaps(args.url, expand=args.expand, max_children=args.max_children, sample_size=args.sample)
    if args.json:
        print(json.dumps(result, indent=2))
    elif result["error"]:
        print(f"Error: {result['error']}", file=sys.stderr)
    elif args.expand and result["expanded"]["files"]:
        for item in result["expanded"]["files"]:
            if not item["valid"]:
                detail = item["error"]
            elif item["kind"] == "sitemapindex":
                detail = f"{item['child_count']} child sitemaps"
            else:
                detail = f"{item['url_count']} URLs"
            flags = f" [{', '.join(item['findings'])}]" if item["findings"] else ""
            print(f"{item['url']} ({item['kind'] or 'invalid'}, {detail}){flags}")
        totals = result["expanded"]["totals"]
        print(f"Total: {totals['total_urls']} URLs in {totals['sitemap_files']} files, {totals['unique_urls']} unique")
        for warning in result["warnings"]:
            print(f"Warning: {warning}", file=sys.stderr)
    elif result["found"]:
        for item in result["found"]:
            print(f"{item['url']} ({item['kind']})")
    else:
        print("No valid sitemap found")
    return 1 if result["error"] else 0


if __name__ == "__main__":
    raise SystemExit(main())