Skip to content

Universal SEO skill for Claude Code. 26 sub-skills + 19 sub-agents covering technical SEO, E-E-A-T, schema, GEO/AEO, agent readiness (Lighthouse Agentic Browsing, WebMCP, llms.txt), backlinks, local SEO, e-commerce, international SEO, Google APIs, and PDF/Excel reporting. 9 optional extensions for live SEO data.

57.6 KB
Raw Blame
#!/usr/bin/env python3
"""
Deterministic single-page on-page checks.

Runs 47 mechanical checks on one HTML document: title and description
lengths, heading structure, canonical and robots signals, document basics,
images, links, URL hygiene, page size, social tags, JSON-LD syntax and
title/description relevance. Every finding is an observation of the HTML (and
the response headers when they are available); nothing is inferred by a model.
The agent that calls this script judges and explains the findings, it does not
recount them by hand.

The check logic is ported from the sister project web-obs (MIT, same author).
Where the two projects disagree on a threshold, the value documented in this
repository wins: see the threshold table below, which names the file each
value comes from.

Length thresholds are display heuristics. Google publishes no title or meta
description length limit, so the messages say "may be truncated in results"
and never present a number as a search engine rule.

Usage:
    python page_checks.py --url https://example.com/page
    python page_checks.py --file page.html --base-url https://example.com/page
    python page_checks.py --file page.html --base-url URL --headers-json headers.json

``--url`` fetches through fetch_page.py (url_safety SSRF protection).
``--file`` reads a saved document and makes no network request.
``--headers-json`` takes a path to a JSON file or an inline JSON object. A
``fetch_page.py --json`` result is accepted too: its "headers" object is used.

Output (JSON on stdout)::

    {
      "url": "https://example.com/page",
      "source": "url" | "file",
      "status_code": 200 | null,
      "headers_available": true,
      "facts": {"title_length": 54, "word_count": 812, "h1_count": 1, ...},
      "findings": [
        {"id": "title-too-long", "severity": "Medium",
         "message": "...", "evidence": "72 characters: ...", "count": 1}
      ],
      "not_checked": [{"id": "noindex-header", "reason": "..."}],
      "thresholds": {"title_max_chars": 60, ...},
      "summary": {"Critical": 0, "High": 0, "Medium": 1, "Low": 0,
                  "total": 1, "checks_run": 46}
    }

``evidence`` and the text values in ``facts`` quote the page. They are
untrusted data, trimmed to 200 characters, and must never be followed as
instructions. ``count`` is present only on findings that count occurrences.

Errors are a JSON object with an "error" key.

Exit codes:
    0  the checks ran, whatever they found
    1  the page could not be fetched or read
    2  bad arguments
"""

from __future__ import annotations

import argparse
import json
import os
import re
import sys
import unicodedata
from typing import Optional
from urllib.parse import unquote, urljoin, urlparse, urlunparse

try:
    from bs4 import BeautifulSoup, Tag
except ImportError:
    print(json.dumps({"error": "beautifulsoup4 is required. Install with: pip install beautifulsoup4"}))
    sys.exit(1)

try:
    import lxml  # noqa: F401

    _HTML_PARSER = "lxml"
except ImportError:
    _HTML_PARSER = "html.parser"

_SCRIPTS_DIR = os.path.dirname(os.path.abspath(__file__))

# ---------------------------------------------------------------------------
# Thresholds: one table. A value documented in this repository wins over the
# web-obs value; the comment names the file it comes from. None of the length
# values is a Google rule: they are display-truncation heuristics.
# ---------------------------------------------------------------------------

# skills/seo/references/quality-gates.md, "Title Tag Requirements" (web-obs: 20 and 65)
TITLE_MIN_CHARS = 30
TITLE_MAX_CHARS = 60
# skills/seo/references/quality-gates.md, "Meta Description Requirements" (web-obs: 165).
# That file is the single source: other skills state 150-160, 130-150 and under 155.
META_DESCRIPTION_MIN_CHARS = 120  # reported as a fact only, no finding id exists for it
META_DESCRIPTION_MAX_CHARS = 160
# skills/seo-technical/SKILL.md, "URL Structure": "flag >100 characters" (web-obs: 115)
URL_MAX_CHARS = 100
# skills/seo-technical/SKILL.md, "Crawlability": Googlebot fetches the first 2MB of HTML
HTML_MAX_BYTES = 2_000_000
# web-obs value. This repository documents minimums per page type only
# (skills/seo/references/quality-gates.md, "Minimum Word Counts by Page Type"), and a
# script cannot know the page type, so this is a type-blind floor. Compare the
# word_count fact with that table for the real gate.
THIN_CONTENT_MIN_WORDS = 200
# Lighthouse DOM size guidance, as in web-obs. Not documented in this repository.
DOM_MAX_DEPTH = 32
DOM_MAX_CHILDREN = 60
# web-obs values: share of significant title/description terms found in the body.
TITLE_RELEVANCE_MIN = 0.3
DESCRIPTION_RELEVANCE_MIN = 0.2
RELEVANCE_MIN_WORDS = 50
# Text made of more than this share of unspaced-script letters is not word counted.
UNSPACED_SCRIPT_RATIO = 0.3
# Page text is untrusted: every evidence string is cut to this length.
EVIDENCE_MAX_CHARS = 200
# Largest file read with --file, the same cap fetch_page.py applies to a response.
MAX_FILE_BYTES = 10 * 1024 * 1024

THRESHOLDS = {
    "title_min_chars": TITLE_MIN_CHARS,
    "title_max_chars": TITLE_MAX_CHARS,
    "meta_description_min_chars": META_DESCRIPTION_MIN_CHARS,
    "meta_description_max_chars": META_DESCRIPTION_MAX_CHARS,
    "url_max_chars": URL_MAX_CHARS,
    "html_max_bytes": HTML_MAX_BYTES,
    "thin_content_min_words": THIN_CONTENT_MIN_WORDS,
    "dom_max_depth": DOM_MAX_DEPTH,
    "dom_max_children": DOM_MAX_CHILDREN,
    "title_relevance_min": TITLE_RELEVANCE_MIN,
    "description_relevance_min": DESCRIPTION_RELEVANCE_MIN,
    "relevance_min_words": RELEVANCE_MIN_WORDS,
    "evidence_max_chars": EVIDENCE_MAX_CHARS,
}

# ---------------------------------------------------------------------------
# Severity: the one place web-obs tones become this repository's words.
# Vocabulary from skills/seo-audit/SKILL.md, "Priority Definitions":
#   Critical  blocks indexing            High  significantly impacts rankings
#   Medium    optimization opportunity   Low   nice to have
# ---------------------------------------------------------------------------

SEVERITY_ORDER = ("Critical", "High", "Medium", "Low")
SEVERITY_BY_TONE = {"attention": "High", "review": "Medium"}
SEVERITY_OVERRIDES = {
    # Asks search engines not to index the page: "blocks indexing".
    "noindex": "Critical",
    "noindex-header": "Critical",
    # Asks search engines to index a different URL instead of this one.
    "canonical-points-elsewhere": "High",
    # Conventions and hygiene with no direct effect on Search: "nice to have".
    "multiple-h1": "Low",
    "heading-level-skip": "Low",
    "og-url-differs-from-canonical": "Low",
    "missing-og-image": "Low",
    "missing-doctype": "Low",
    "deprecated-html-tags": "Low",
    "generic-anchor-text": "Low",
    "links-to-http": "Low",
    "url-too-long": "Low",
    "url-uppercase": "Low",
    "url-underscores": "Low",
    "url-non-ascii": "Low",
    "url-double-slash": "Low",
    "deep-dom": "Low",
    "wide-dom-node": "Low",
}

_HEURISTIC = "This is a display heuristic, not a published search engine limit."

# id: (web-obs tone, message). Ids and tones follow the web-obs issue catalogue.
CHECKS: dict[str, tuple[str, str]] = {
    "missing-title": ("attention", "The page has no title tag. Search engines then build the result title from other text, such as the main heading."),
    "title-too-long": ("review", f"The title is longer than {TITLE_MAX_CHARS} characters and may be truncated in results. {_HEURISTIC}"),
    "title-too-short": ("review", f"The title is shorter than {TITLE_MIN_CHARS} characters. A very short title rarely describes the page well enough to earn a click. This is a heuristic, not a search engine rule."),
    "missing-meta-description": ("review", "The page has no meta description. Search engines write their own snippet when none is given."),
    "meta-description-too-long": ("review", f"The meta description is longer than {META_DESCRIPTION_MAX_CHARS} characters and may be truncated in results. {_HEURISTIC}"),
    "duplicate-meta-tags": ("review", "The same title, meta tag or canonical appears more than once, so it is unclear which one search engines and social platforms read."),
    "missing-h1": ("review", "No H1 heading was found on the page."),
    "multiple-h1": ("review", "The page has more than one H1. Google says there is no ideal number of headings, so this does not affect Search on its own. Check that the most prominent heading names the page."),
    "heading-level-skip": ("review", "Heading levels skip inside the main content, for example from H2 to H4. Google Search does not mind heading order, but screen reader users move through a page by heading level."),
    "missing-canonical": ("review", "The page has no canonical link. Search engines then choose the preferred URL themselves."),
    "canonical-points-elsewhere": ("review", "The canonical names a different URL, so this page asks search engines to index that URL instead. Confirm this is intended."),
    "og-url-differs-from-canonical": ("review", "og:url differs from the canonical, so social shares may point at a different URL than search engines use."),
    "noindex": ("review", "A robots meta tag asks search engines not to index this page. Confirm this is intended."),
    "noindex-header": ("review", "The X-Robots-Tag response header asks search engines not to index this page. Confirm this is intended."),
    "nofollow": ("review", "The page asks search engines not to follow any of its links. Confirm this is intended."),
    "meta-refresh-redirect": ("review", "The page redirects with a meta refresh. Google follows it, but a server side 301 or 308 redirect is the more reliable choice."),
    "hreflang-missing-self": ("review", "The page has hreflang links but none of them points to the page itself. Hreflang sets should include a self reference."),
    "missing-doctype": ("review", "The document does not start with <!doctype html>, so browsers fall back to quirks mode."),
    "missing-charset": ("review", "No character encoding is declared for the document."),
    "charset-mismatch": ("review", "The Content-Type response header and the meta tag declare different character encodings."),
    "missing-lang": ("review", "The html element does not declare the page language with a lang attribute."),
    "missing-viewport": ("review", "The page has no viewport meta tag, so phones render it zoomed out."),
    "uses-frames": ("attention", "The page uses frames. Content inside frames is hard for search engines to attribute to this URL."),
    "deprecated-html-tags": ("review", "The page uses HTML tags that are obsolete, such as <center> or <font>."),
    "placeholder-text": ("attention", "Placeholder text such as lorem ipsum was found in the visible content."),
    "images-missing-alt": ("review", "Some images have no alt attribute. Screen readers and image search rely on alt text. An empty alt is correct for decorative images."),
    "images-missing-dimensions": ("review", "Some images have no width and height attributes, which lets the layout shift while they load."),
    "first-image-lazy-loaded": ("review", "The first image in the main content is lazy loaded. It is often the largest paint, and lazy loading delays it."),
    "links-without-text": ("review", "Some links have no accessible text: no link text, no label and no image alt text."),
    "generic-anchor-text": ("review", "Some links in the content use generic text such as \"click here\", which says nothing about where they lead."),
    "links-to-http": ("review", "This HTTPS page links to plain http:// addresses."),
    "mixed-content": ("attention", "This HTTPS page loads resources over plain HTTP. Browsers block or warn about these."),
    "url-too-long": ("review", f"The URL is longer than {URL_MAX_CHARS} characters. Long URLs are harder to share and may be truncated in results. This is a heuristic, not a search engine limit."),
    "url-has-parameters": ("review", "The URL has query parameters other than tracking parameters. Parameters can create duplicate versions of the same page."),
    "url-uppercase": ("review", "The URL path has uppercase letters. URLs are case sensitive, so mixed case invites duplicates and broken links."),
    "url-underscores": ("review", "The URL path uses underscores. Hyphens are the usual word separator in URLs."),
    "url-non-ascii": ("review", "The URL has unencoded non-ASCII characters. Words in the audience's language are fine in URLs; links should use the percent encoded form so the address survives copying."),
    "url-double-slash": ("review", "The URL path has a double slash, which often creates accidental duplicate URLs."),
    "html-over-2mb": ("attention", "The HTML is larger than 2 MB. Googlebot reads only the first 2 MB of an HTML file, measured before compression, so content after that point is not considered for indexing."),
    "deep-dom": ("review", f"The DOM is nested more than {DOM_MAX_DEPTH} levels deep, which slows rendering (Lighthouse guidance)."),
    "wide-dom-node": ("review", f"One element has more than {DOM_MAX_CHILDREN} direct children, which is slow to lay out (Lighthouse guidance)."),
    "missing-open-graph": ("review", "The page has neither og:title nor og:image, so shared links show no title or preview image."),
    "missing-og-image": ("review", "The page has no og:image, so shared links show no preview image."),
    "invalid-jsonld": ("review", "A JSON-LD block is not valid JSON, so search engines cannot read any of it and the page loses the rich result that block was meant to support."),
    "title-not-reflected-in-content": ("review", "Most significant words of the title do not appear in the visible page text."),
    "description-not-reflected-in-content": ("review", "Most significant words of the meta description do not appear in the visible page text."),
    "thin-content": ("review", f"The page has fewer than {THIN_CONTENT_MIN_WORDS} words of visible body text. Google has no minimum word count, so this is not a ranking rule. Check that the page fully answers what a visitor came for."),
}


def severity_for(check_id: str) -> str:
    """Severity word for a check id: the override if one exists, else the tone mapping."""
    return SEVERITY_OVERRIDES.get(check_id) or SEVERITY_BY_TONE[CHECKS[check_id][0]]


# ---------------------------------------------------------------------------
# Vocabulary
# ---------------------------------------------------------------------------

STOPWORDS = frozenset("""
a about above after again against all also am an and any are as at be because been before being below between both
but by can could did do does doing down during each few for from further get got had has have having he her here hers
herself him himself his how i if in into is it its itself just let me more most my myself no nor not now of off on once
only or other our ours ourselves out over own same she should so some such than that the their theirs them themselves
then there these they this those through to too under until up us very was we were what when where which while who whom
why will with would you your yours yourself yourselves new one two use using used like make made way may might must
need via per yet etc www http https com
""".split())

DEPRECATED_TAGS = ("acronym", "applet", "basefont", "big", "blink", "center", "dir", "font", "frame", "frameset",
                   "isindex", "marquee", "nobr", "noframes", "spacer", "strike", "tt")
GENERIC_ANCHORS = frozenset({"click here", "read more", "learn more", "here", "more", "link", "this page",
                             "details", "view", "website", "continue", "go"})
# Link text longer than this is not compared with GENERIC_ANCHORS. The longest generic
# phrase is 10 characters, so only a link padded with punctuation could be missed, and the
# cap keeps the work per link fixed however long the page makes its link text.
GENERIC_ANCHOR_MAX_CHARS = 200
_CHROME_TAGS = ("nav", "header", "footer", "aside")
_CHROME_ROLES = ("navigation", "banner", "contentinfo", "complementary", "menubar")
_CHROME_MARKERS = ("navbar", "topbar", "top-bar", "site-header", "site-footer", "main-nav", "footer-nav",
                   "mega-menu", "breadcrumb", "cookie-banner", "sidebar-nav")
_LAZY_SRC_ATTRS = ("data-perfmatters-src", "data-ewww-src", "data-src", "data-lazy-src", "data-original",
                   "data-full-url")
_MIXED_CONTENT_TAGS = ("img", "script", "iframe", "source", "video", "audio", "embed")
_SKIPPED_LINK_SCHEMES = ("#", "javascript:", "data:", "mailto:", "tel:", "sms:")
_ROBOTS_META_NAMES = ("robots", "googlebot")
_ROBOTS_VALUED_DIRECTIVES = ("max-snippet", "max-image-preview", "max-video-preview", "unavailable_after")
# User agents whose rules decide Google Search indexing: none named (every crawler) or Googlebot.
_ROBOTS_GENERAL_AGENTS = ("", "googlebot")
_TRACKING_KEYS = frozenset({"fbclid", "gclid", "mc_cid", "mc_eid", "_ga", "ref"})

_PLACEHOLDER_RE = re.compile(r"\blorem ipsum\b|\bdolor sit amet\b|\bconsectetur adipiscing\b", re.I)
_WORD_RE = re.compile(r"[^\W\d_][\w'-]*", re.UNICODE)
_TRACKING_PARAM_RE = re.compile(r"^(utm_|fbclid$|gclid$|msclkid$|mc_)", re.I)
_PERCENT_ESCAPE_RE = re.compile(r"%([0-9A-Fa-f]{2})")
_NON_ASCII_BYTES_RE = re.compile(rb"[\x80-\xff]+")
# RFC 3986 unreserved characters: their percent-encoded and literal forms are the same URL.
_URL_UNRESERVED = frozenset("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-._~")
# A DNS name is at most 253 characters. A longer host is compared as written, not IDNA encoded.
HOST_MAX_CHARS = 253
# A crawler name in "googlebot: noindex" is one token, unlike the pieces of a comma-split
# date in "unavailable_after: Wed, 03 Nov 2026 15:00:00 GMT".
_CRAWLER_NAME_RE = re.compile(r"[a-z][a-z0-9_.-]*")
# Rules for other named crawlers are listed as a fact, at most this many.
ROBOTS_OTHER_AGENTS_MAX = 10
_UTF8_ALIASES = {"utf8": "utf-8", "utf-8": "utf-8", "unicode-1-1-utf-8": "utf-8"}
_CONTROL_RE = re.compile(r"[\x00-\x1f\x7f-\x9f​-‏
-‮⁦-⁩]")
# Scripts written without spaces between words: Han, Hiragana, Katakana, Thai, Lao, Myanmar
# and Khmer. Korean (Hangul) does use spaces, so it is counted normally.
_UNSPACED_SCRIPT_RE = re.compile(
    r"[぀-ヿ㐀-䶿一-鿿豈-﫿฀-໿က-႟ក-៿]"
)
_UNSPACED_LANGS = ("zh", "ja", "th", "lo", "km", "my")


# ---------------------------------------------------------------------------
# Small helpers
# ---------------------------------------------------------------------------


def _trim(value: object, limit: int = EVIDENCE_MAX_CHARS) -> str:
    """One line of untrusted text: control characters removed, cut to ``limit``."""
    text = re.sub(r"\s+", " ", _CONTROL_RE.sub(" ", str(value))).strip()
    return text if len(text) <= limit else text[: limit - 3].rstrip() + "..."


def _attr(el, name: str) -> str:
    """Attribute as a stripped string, whether bs4 holds it as a string or a list."""
    value = el.get(name)
    if value is None:
        return ""
    if isinstance(value, (list, tuple)):
        value = " ".join(str(v) for v in value)
    return str(value).strip()


def _rels(el) -> list[str]:
    return _attr(el, "rel").lower().split()


def _text(el) -> str:
    return re.sub(r"\s+", " ", el.get_text(" ", strip=True)).strip()


def _join(base: str, href: str) -> str:
    try:
        return urljoin(base, href)
    except ValueError:
        return ""


def _sample(values: list[str], each: int = 60) -> str:
    """The first few values as one string. ``_trim`` applies the final cut."""
    shown = ", ".join(_trim(v, each) for v in values[:5])
    return shown + (f" (+{len(values) - 5} more)" if len(values) > 5 else "")


def _is_generic_anchor(anchor: str) -> bool:
    """Whether link text is a generic phrase once surrounding punctuation is dropped.

    The ends are walked character by character: a regex with an unbounded run
    before ``$`` rescans the run from every position, which is quadratic on a
    long run of punctuation.
    """
    if len(anchor) > GENERIC_ANCHOR_MAX_CHARS:
        return False
    start, end = 0, len(anchor)
    while start < end and not anchor[start].isalnum():
        start += 1
    while end > start and not anchor[end - 1].isalnum():
        end -= 1
    return anchor[start:end].lower() in GENERIC_ANCHORS


def tokenize(text: str) -> list[str]:
    text = (text or "").replace("’", "'").replace("‘", "'")
    return [w.lower().strip("'-") for w in _WORD_RE.findall(text) if w.strip("'-")]


def significant_terms(text: str) -> list[str]:
    return [t for t in tokenize(text) if len(t) >= 3 and t not in STOPWORDS]


def words_measurable(text: str, lang: str = "") -> bool:
    """Whether a whitespace word count means anything for this text.

    Chinese, Japanese, Thai, Lao, Khmer and Burmese put no spaces between words,
    so splitting on spaces finds a handful of "words" in a long article. For
    those, the word-based checks are not measured.
    """
    if (lang or "").lower().split("-")[0] in _UNSPACED_LANGS:
        return False
    letters = sum(1 for ch in text if ch.isalpha())
    return not letters or len(_UNSPACED_SCRIPT_RE.findall(text)) / letters < UNSPACED_SCRIPT_RATIO


def _one_escape_spelling(part: str) -> str:
    """One spelling for the percent-encoding of a path or query (RFC 3986, section 6.2.2).

    "%c3%a9", "%C3%A9" and a literal "é" all become "%C3%A9", and the escape of
    an unreserved character becomes the character ("%7E" is "~"). Escapes of
    reserved characters keep their meaning: "%2F" stays apart from "/", "%26"
    from "&" and "%20" from "+". Each escape is read once, so "%2541" is not "A".
    """
    def spell(match: re.Match) -> str:
        char = chr(int(match.group(1), 16))
        return char if char in _URL_UNRESERVED else match.group(0).upper()

    part = _PERCENT_ESCAPE_RE.sub(spell, part)
    if part.isascii():
        return part
    raw = part.encode("utf-8", "surrogatepass")
    return _NON_ASCII_BYTES_RE.sub(lambda m: b"%" + m.group(0).hex("%").upper().encode("ascii"), raw).decode("ascii")


def _idna_host(host: str) -> str:
    """Host in the ASCII form browsers send: lower case, Unicode labels as Punycode.

    "bücher.de" and "BÜCHER.DE" become "xn--bcher-kva.de". The mapping is the
    UTS 46 one browsers use (NFKC and lower case, then Punycode), not the IDNA
    2003 codec in the standard library, which would turn "straße.de" into the
    separate domain "strasse.de". A host or label that cannot be encoded is
    returned lower-cased as written, so the comparison never raises.
    """
    host = host.lower()
    if host.isascii() or len(host) > HOST_MAX_CHARS:
        return host
    try:
        mapped = unicodedata.normalize("NFKC", unicodedata.normalize("NFKC", host).lower())
        labels = [label if label.isascii() else "xn--" + label.encode("punycode").decode("ascii")
                  for label in mapped.replace("。", ".").split(".")]  # the ideographic full stop
    except (UnicodeError, ValueError):
        return host
    return ".".join(labels)


def canonicalize_url(url: str) -> Optional[str]:
    """Comparison form of a URL: IDNA lower-case host, no fragment, no tracking parameters,
    one spelling for percent-encoding. Returns None for anything that is not an http(s) URL."""
    if not url or any(c.isspace() for c in url):
        return None
    try:
        parsed = urlparse(url.strip())
        port = parsed.port
    except ValueError:
        return None
    scheme = parsed.scheme.lower()
    if scheme not in ("http", "https") or not parsed.hostname:
        return None
    host = _idna_host(parsed.hostname).rstrip(".")
    if ":" in host:
        host = f"[{host}]"
    default_port = 443 if scheme == "https" else 80
    netloc = f"{host}:{port}" if port and port != default_port else host
    path = _one_escape_spelling(re.sub(r"/{2,}", "/", parsed.path or "/"))
    kept = []
    for pair in parsed.query.split("&"):
        if not pair:
            continue
        key = unquote(pair.split("=", 1)[0]).lower()
        if key.startswith("utm_") or key in _TRACKING_KEYS:
            continue
        kept.append(_one_escape_spelling(pair))
    return urlunparse((scheme, netloc, path, _one_escape_spelling(parsed.params), "&".join(kept), ""))


def url_match_key(url: str) -> str:
    """Key for deciding that two URL forms name the same page: trailing slash ignored."""
    canonical = canonicalize_url(url)
    if canonical is None:
        return url
    parsed = urlparse(canonical)
    path = parsed.path
    if path != "/" and path.endswith("/"):
        path = path.rstrip("/") or "/"
    return urlunparse((parsed.scheme, parsed.netloc, path, parsed.params, parsed.query, ""))


def _norm_charset(value: str) -> str:
    v = (value or "").strip().lower()
    return _UTF8_ALIASES.get(v, v)


def _charset_of(content_type: str) -> str:
    m = re.search(r"charset\s*=\s*['\"]?([^;,'\"\s]+)", content_type or "", re.I)
    return _norm_charset(m.group(1)) if m else ""


def _robots_rules(value: str) -> list[tuple[str, str]]:
    """(user agent, directive) pairs of one robots meta or X-Robots-Tag value.

    The grammar is an optional user agent, then comma-separated directives,
    some of which carry a value::

        [googlebot:] noindex, max-snippet: 50, unavailable_after: <date>

    The user agent is "" when the rule names none. A named one holds until the
    next, because a server joins several headers into one with commas. A valued
    directive is dropped with its value, so the "none" in "max-image-preview:
    none" is never read as the "none" directive, with or without a user agent
    in front. A piece that is neither (the rest of a date that has commas) is
    dropped too.
    """
    rules: list[tuple[str, str]] = []
    agent = ""
    for part in (value or "").lower().split(","):
        name, sep, rest = (piece.strip() for piece in part.partition(":"))
        if sep and name not in _ROBOTS_VALUED_DIRECTIVES and _CRAWLER_NAME_RE.fullmatch(name):
            agent = name
            name, sep, rest = (piece.strip() for piece in rest.partition(":"))
        if not sep:
            rules.extend((agent, word) for word in name.split())
    return rules


def _robots_directives(value: str) -> set[str]:
    """Directives of a robots value that apply to Google Search.

    "googlebot: noindex, nofollow" and "noindex, nofollow" both yield
    {"noindex", "nofollow"}. A rule for another named crawler ("otherbot:
    noindex") yields nothing: it does not keep the page out of Google, so it is
    never reported as a noindex or nofollow finding. ``run_checks`` lists such
    rules in the ``x_robots_other_agents`` fact instead, with no severity.
    """
    return {directive for agent, directive in _robots_rules(value) if agent in _ROBOTS_GENERAL_AGENTS}


def _document_base(soup, page_url: str) -> str:
    """The first <base href> when it resolves to an http(s) URL, else the page URL."""
    for base in soup.find_all("base"):
        if base.get("href") is None:
            continue
        resolved = _join(page_url, _attr(base, "href"))
        try:
            parsed = urlparse(resolved)
            usable = bool(_attr(base, "href")) and parsed.scheme in ("http", "https") and bool(parsed.hostname)
        except ValueError:
            usable = False
        return resolved if usable else page_url
    return page_url


def _is_decorative(img) -> bool:
    """True when the image or an ancestor is hidden from assistive technology or presentational."""
    node = img
    while node is not None and getattr(node, "attrs", None) is not None:
        if _attr(node, "aria-hidden").lower() == "true":
            return True
        if _attr(node, "role").lower() in ("presentation", "none"):
            return True
        node = node.parent
    return False


def _in_chrome_zone(tag) -> bool:
    """True inside site-wide navigation, header, footer, breadcrumb or aside."""
    curr = tag.parent
    while isinstance(curr, Tag):
        if (curr.name or "").lower() in _CHROME_TAGS or _attr(curr, "role").lower() in _CHROME_ROLES:
            return True
        marker = f"{_attr(curr, 'class')} {_attr(curr, 'id')}".lower()
        if any(tok in marker for tok in _CHROME_MARKERS):
            return True
        curr = curr.parent
    return False


def _image_src(img) -> str:
    """The real source of an image, looking past lazy-loading placeholders."""
    for attr in (*_LAZY_SRC_ATTRS, "src"):
        value = _attr(img, attr)
        if value and not value.startswith("data:"):
            return value
    for attr in ("data-srcset", "srcset"):
        for candidate in _attr(img, attr).split(","):
            url = candidate.strip().split(" ")[0]
            if url and not url.startswith(("data:", "blob:")):
                return url
    return ""


def _visible_text(html: str) -> str:
    """Body text without scripts, styles and site chrome (nav, footer, site header)."""
    soup = BeautifulSoup(html, _HTML_PARSER)
    for tag in soup(["head", "script", "style", "noscript", "nav", "footer"]):
        tag.decompose()
    # A site header is chrome, but <article><header><h1> holds the page's own heading.
    for tag in soup.find_all("header"):
        if not tag.decomposed and tag.find_parent(["article", "main"]) is None:
            tag.decompose()
    return re.sub(r"\s+", " ", soup.get_text(" ", strip=True)).strip()


def _dom_stats(soup) -> dict:
    root = soup.find("html") or soup
    nodes = max_depth = max_children = 0
    widest = ""
    stack = [(root, 1)]
    while stack:
        el, depth = stack.pop()
        nodes += 1
        max_depth = max(max_depth, depth)
        children = [c for c in el.children if isinstance(c, Tag)]
        if len(children) > max_children:
            max_children = len(children)
            widest = el.name or ""
        stack.extend((c, depth + 1) for c in children)
    return {"nodes": nodes, "max_depth": max_depth, "max_children": max_children, "widest_element": widest}


def _heading_skips(soup) -> tuple[list[str], int, int]:
    """H1 texts, level skips inside the main content, and skips in template regions.

    Skips are judged inside the main content when the page has one. Headings in
    site-wide regions (header, nav, footer, aside) are counted apart, so a
    footer widget cannot flag every page.
    """
    h1: list[str] = []
    content_skips = template_skips = 0
    main_root = soup.find("main") or soup.find(attrs={"role": "main"}) or soup.find("article")
    prev_level: Optional[int] = None
    prev_template_level: Optional[int] = None
    for h in soup.find_all(["h1", "h2", "h3", "h4", "h5", "h6"]):
        if _attr(h, "aria-hidden").lower() == "true":
            continue
        text = _text(h)
        if not text:
            continue
        level = int(h.name[1])
        in_template = h.find_parent(list(_CHROME_TAGS)) is not None
        in_main = main_root is not None and any(p is main_root for p in h.parents)
        if main_root is not None and not in_main:
            if prev_template_level is not None and level > prev_template_level + 1:
                template_skips += 1
            prev_template_level = level
        else:
            if prev_level is not None and level > prev_level + 1:
                if in_template:
                    template_skips += 1
                else:
                    content_skips += 1
            prev_level = level
        if level == 1:
            h1.append(text)
    return h1, content_skips, template_skips


def _relevance(source: str, body_terms: set[str]) -> tuple[Optional[float], list[str]]:
    terms = set(significant_terms(source))
    if not terms:
        return None, []
    return round(len(terms & body_terms) / len(terms), 2), sorted(terms - body_terms)


# ---------------------------------------------------------------------------
# The checks
# ---------------------------------------------------------------------------


def run_checks(html: str, page_url: str, headers: Optional[dict] = None,
               html_bytes: Optional[int] = None) -> dict:
    """Run every check on one document.

    ``headers`` is the response header mapping, or None when the headers are not
    known (a saved file): header-based checks are then listed in ``not_checked``.
    ``html_bytes`` is the size of the document as served; it defaults to the
    UTF-8 size of ``html``.
    """
    headers_available = headers is not None
    hdrs = {str(k).lower(): ", ".join(map(str, v)) if isinstance(v, (list, tuple)) else str(v)
            for k, v in (headers or {}).items()}
    soup = BeautifulSoup(html, _HTML_PARSER)
    base_url = _document_base(soup, page_url)
    is_https = page_url.lower().startswith("https://")
    if html_bytes is None:
        html_bytes = len(html.encode("utf-8", errors="replace"))

    findings: list[dict] = []
    not_checked: list[dict] = []

    def add(check_id: str, evidence: object, count: Optional[int] = None) -> None:
        item = {"id": check_id, "severity": severity_for(check_id), "message": CHECKS[check_id][1],
                "evidence": _trim(evidence)}
        if count is not None:
            item["count"] = count
        findings.append(item)

    def skip(check_id: str, reason: str) -> None:
        not_checked.append({"id": check_id, "reason": reason})

    html_tag = soup.find("html")
    lang = _attr(html_tag, "lang") if isinstance(html_tag, Tag) else ""
    visible_text = _visible_text(html)
    word_count = len(visible_text.split())
    measurable = words_measurable(visible_text, lang)
    unspaced = "the page is written in a script without spaces between words, so words are not counted"

    # --- Title and meta description -------------------------------------
    # An SVG <title> names a graphic, not the document.
    title_tags = [t for t in soup.find_all("title") if t.find_parent("svg") is None]
    title = _text(title_tags[0]) if title_tags else ""
    if not title:
        add("missing-title", "no <title> element" if not title_tags else "the <title> element is empty")
    else:
        if len(title) > TITLE_MAX_CHARS:
            add("title-too-long", f"{len(title)} characters: {title}")
        if len(title) < TITLE_MIN_CHARS:
            if words_measurable(title, lang):
                add("title-too-short", f"{len(title)} characters: {title}")
            else:
                # Thirty characters is a few English words but a full title in Chinese or Japanese.
                skip("title-too-short", unspaced)

    meta_tags: list[tuple[str, str, str]] = []
    for meta in soup.find_all("meta"):
        content = _attr(meta, "content")
        if content:
            meta_tags.append((_attr(meta, "name").lower(), _attr(meta, "property").lower(), content))
    description = next((c for name, _prop, c in meta_tags if name == "description"), "")
    if not description:
        add("missing-meta-description", "no <meta name=\"description\"> with content")
    elif len(description) > META_DESCRIPTION_MAX_CHARS:
        add("meta-description-too-long", f"{len(description)} characters: {description}")

    canonical_links = [link for link in soup.find_all("link") if "canonical" in _rels(link)]
    duplicates = []
    if len(title_tags) > 1:
        duplicates.append(f"title x{len(title_tags)}")
    for wanted, index in ((("description", "robots", "viewport", "title"), 0),
                          (("og:title", "og:description", "og:url", "og:image"), 1)):
        for key in wanted:
            n = sum(1 for tag in meta_tags if tag[index] == key)
            if n > 1:
                duplicates.append(f"meta {key} x{n}")
    if len(canonical_links) > 1:
        duplicates.append(f"canonical x{len(canonical_links)}")
    if duplicates:
        add("duplicate-meta-tags", ", ".join(duplicates), len(duplicates))

    # --- Headings -------------------------------------------------------
    h1, heading_skips, template_skips = _heading_skips(soup)
    if not h1:
        add("missing-h1", "no H1 element with text")
    elif len(h1) > 1:
        add("multiple-h1", f"{len(h1)} H1 headings: {_sample(h1)}", len(h1))
    if heading_skips:
        add("heading-level-skip", f"{heading_skips} skipped heading level(s) in the main content", heading_skips)

    # --- Canonical, og:url ---------------------------------------------
    self_key = url_match_key(page_url)
    canonical = next((u for u in (_join(base_url, _attr(link, "href")) for link in canonical_links
                                  if _attr(link, "href")) if u), "")
    canonical_note = None
    if not canonical:
        canonical_status = "missing"
        add("missing-canonical", "no <link rel=\"canonical\"> with an href")
    elif url_match_key(canonical) == self_key:
        canonical_status = "self"
        if canonical != page_url and canonical.rstrip("/") == page_url.rstrip("/"):
            canonical_note = "differs from the page URL only by a trailing slash, treated as self-referencing"
    else:
        canonical_status = "other"
        add("canonical-points-elsewhere",
            f"canonical {_trim(canonical, 60)} vs page {_trim(page_url, 60)} "
            "(a trailing slash difference alone is tolerated)")

    og: dict[str, str] = {}
    for name, prop, content in meta_tags:
        key = prop or name
        if key.startswith("og:") and key not in og:
            og[key] = content
    og_url = _join(base_url, og["og:url"]) if og.get("og:url") else ""
    if og_url and canonical and url_match_key(og_url) != url_match_key(canonical):
        add("og-url-differs-from-canonical",
            f"og:url {_trim(og_url, 60)} vs canonical {_trim(canonical, 60)} "
            "(a trailing slash difference alone is tolerated)")

    # --- Robots, meta refresh, hreflang ---------------------------------
    robots_metas = [(name, content) for name, _prop, content in meta_tags if name in _ROBOTS_META_NAMES]
    meta_directives: set[str] = set()
    for _name, content in robots_metas:
        meta_directives |= _robots_directives(content)
    meta_robots = "; ".join(f"{name}: {content}" for name, content in robots_metas)
    # Each X-Robots-Tag header is read on its own, so a user agent named in one does not
    # reach into the next. Headers a server or client has already joined with commas cannot
    # be told apart again: what follows a named crawler is read as that crawler's.
    x_robots_values: list[str] = []
    for key, value in (headers or {}).items():
        if str(key).lower() == "x-robots-tag":
            x_robots_values.extend(map(str, value) if isinstance(value, (list, tuple)) else [str(value)])
    x_robots = ", ".join(x_robots_values).strip()
    header_rules = [rule for value in x_robots_values for rule in _robots_rules(value)]
    header_directives = {directive for agent, directive in header_rules if agent in _ROBOTS_GENERAL_AGENTS}
    # A rule for another named crawler is a fact, not a finding: it does not block Google Search.
    other_agent_rules = list(dict.fromkeys(f"{agent}: {directive}" for agent, directive in header_rules
                                           if agent not in _ROBOTS_GENERAL_AGENTS))
    if meta_directives & {"noindex", "none"}:
        add("noindex", f"meta {meta_robots}")
    if header_directives & {"noindex", "none"}:
        add("noindex-header", f"X-Robots-Tag: {x_robots}")
    elif not headers_available:
        skip("noindex-header", "no response headers were supplied")
    nofollow_sources = []
    if meta_directives & {"nofollow", "none"}:
        nofollow_sources.append(f"meta {meta_robots}")
    if header_directives & {"nofollow", "none"}:
        nofollow_sources.append(f"X-Robots-Tag: {x_robots}")
    if nofollow_sources:
        add("nofollow", "; ".join(nofollow_sources))

    meta_refresh = next((_attr(m, "content") for m in soup.find_all("meta")
                         if _attr(m, "http-equiv").lower() == "refresh"), "")
    if meta_refresh and "url=" in meta_refresh.lower():
        add("meta-refresh-redirect", f"content=\"{meta_refresh}\"")

    hreflang = [(_attr(link, "hreflang"), _join(base_url, _attr(link, "href")))
                for link in soup.find_all("link")
                if "alternate" in _rels(link) and _attr(link, "hreflang") and _attr(link, "href")]
    if hreflang and not any(url_match_key(url) == self_key for _lang, url in hreflang):
        add("hreflang-missing-self",
            f"{len(hreflang)} hreflang link(s) ({_sample([code for code, _url in hreflang], 12)}), "
            f"none points to {_trim(page_url, 80)}")

    # --- Document basics ------------------------------------------------
    head_text = re.sub(r"^(?:<!--.*?-->\s*)+", "", html.lstrip().lstrip("").lstrip(), flags=re.S)
    has_doctype = head_text[:15].lower().startswith("<!doctype html")
    if not has_doctype:
        add("missing-doctype", f"document starts with: {head_text[:40]}" if head_text else "empty document")
    meta_charset = ""
    for m in soup.find_all("meta"):
        if _attr(m, "charset"):
            meta_charset = _norm_charset(_attr(m, "charset"))
            break
        if _attr(m, "http-equiv").lower() == "content-type":
            meta_charset = _charset_of(_attr(m, "content"))
            if meta_charset:
                break
    header_charset = _charset_of(hdrs.get("content-type", ""))
    if not meta_charset and not header_charset:
        add("missing-charset", "no meta charset and no charset in the Content-Type header" if headers_available
            else "no meta charset; response headers were not supplied, so the Content-Type header was not checked")
    if meta_charset and header_charset and meta_charset != header_charset:
        add("charset-mismatch", f"meta charset {meta_charset}, Content-Type header charset {header_charset}")
    elif not headers_available:
        skip("charset-mismatch", "no response headers were supplied")
    if not lang:
        add("missing-lang", "<html> has no lang attribute")
    has_viewport = any(_attr(m, "name").lower() == "viewport" for m in soup.find_all("meta"))
    if not has_viewport:
        add("missing-viewport", "no <meta name=\"viewport\">")
    frames = soup.find_all(["frameset", "frame"])
    if frames:
        add("uses-frames", f"{len(frames)} <frameset> or <frame> element(s)", len(frames))
    deprecated = sorted({t.name for t in soup.find_all(list(DEPRECATED_TAGS))})
    if deprecated:
        add("deprecated-html-tags", ", ".join(f"<{name}>" for name in deprecated), len(deprecated))
    placeholder = _PLACEHOLDER_RE.search(visible_text)
    if placeholder:
        start = max(0, placeholder.start() - 40)
        add("placeholder-text", visible_text[start:placeholder.end() + 60])

    # --- Images ---------------------------------------------------------
    images_total = images_empty_alt = 0
    missing_alt: list[str] = []
    missing_dimensions: list[str] = []
    for img in soup.find_all("img"):
        if _attr(img, "aria-hidden").lower() == "true":
            continue
        src = _image_src(img)
        if not src:
            continue
        images_total += 1
        alt = img.get("alt")
        if alt is None:
            if not _is_decorative(img):
                missing_alt.append(src)
        elif not _attr(img, "alt") and not _is_decorative(img):
            # An empty alt is valid for decorative images. Counted as a fact, never a finding.
            images_empty_alt += 1
        if not _attr(img, "width") or not _attr(img, "height"):
            missing_dimensions.append(src)
    if missing_alt:
        add("images-missing-alt", f"{len(missing_alt)} of {images_total} images: {_sample(missing_alt)}",
            len(missing_alt))
    if missing_dimensions:
        add("images-missing-dimensions",
            f"{len(missing_dimensions)} of {images_total} images: {_sample(missing_dimensions)}",
            len(missing_dimensions))
    scope = soup.find("main") or soup.body or soup
    first_img = scope.find("img")
    first_image_lazy = first_img is not None and _attr(first_img, "loading").lower() == "lazy"
    if first_image_lazy:
        add("first-image-lazy-loaded", f"loading=\"lazy\" on {_image_src(first_img) or 'the first <img>'}")

    # --- Links ----------------------------------------------------------
    links_total = 0
    links_without_text: list[str] = []
    generic_anchors: list[str] = []
    insecure_links: list[str] = []
    for a in soup.find_all("a", href=True):
        href = _attr(a, "href")
        anchor = _text(a)
        named = bool(anchor or _attr(a, "aria-label") or _attr(a, "aria-labelledby") or _attr(a, "title")
                     or any(_attr(img, "alt") for img in a.find_all("img"))
                     or any(svg.find("title") is not None for svg in a.find_all("svg")))
        if not named:
            links_without_text.append(href or "(empty href)")
        if not href or href.lower().startswith(_SKIPPED_LINK_SCHEMES):
            continue
        full = _join(base_url, href)
        try:
            parsed = urlparse(full)
            scheme, host = parsed.scheme.lower(), parsed.hostname
        except ValueError:
            continue
        if scheme not in ("http", "https") or not host:
            continue
        links_total += 1
        if is_https and scheme == "http":
            insecure_links.append(full)
        if _is_generic_anchor(anchor) and not _in_chrome_zone(a):
            generic_anchors.append(f"\"{anchor}\" to {full}")
    if links_without_text:
        add("links-without-text", f"{len(links_without_text)} link(s): {_sample(links_without_text)}",
            len(links_without_text))
    if generic_anchors:
        add("generic-anchor-text", f"{len(generic_anchors)} link(s): {_sample(generic_anchors)}",
            len(generic_anchors))
    if insecure_links:
        add("links-to-http", f"{len(insecure_links)} link(s): {_sample(insecure_links)}", len(insecure_links))

    mixed: list[str] = []
    if is_https:
        for el in soup.find_all(list(_MIXED_CONTENT_TAGS)):
            if _attr(el, "src").lower().startswith("http://"):
                mixed.append(_attr(el, "src"))
        for link in soup.find_all("link", href=True):
            rels = _rels(link)
            if ("stylesheet" in rels or "preload" in rels) and _attr(link, "href").lower().startswith("http://"):
                mixed.append(_attr(link, "href"))
    if mixed:
        add("mixed-content", f"{len(mixed)} resource(s): {_sample(mixed)}", len(mixed))

    # --- URL hygiene ----------------------------------------------------
    parsed_url = urlparse(page_url)
    path = parsed_url.path or "/"
    params = [p.split("=", 1)[0] for p in parsed_url.query.split("&") if p]
    content_params = [p for p in params if not _TRACKING_PARAM_RE.match(p)]
    if len(page_url) > URL_MAX_CHARS:
        add("url-too-long", f"{len(page_url)} characters: {page_url}")
    if content_params:
        add("url-has-parameters", f"parameters: {_sample(content_params, 30)} (tracking parameters are ignored)",
            len(content_params))
    # Percent escapes such as %C3%A9 are hex digits, not uppercase letters in the address.
    unescaped_path = re.sub(r"%[0-9A-Fa-f]{2}", "", path)
    if unescaped_path != unescaped_path.lower():
        add("url-uppercase", f"path {path}")
    if "_" in path:
        add("url-underscores", f"path {path}")
    if any(ord(ch) > 127 for ch in page_url):
        add("url-non-ascii", page_url)
    if "//" in path:
        add("url-double-slash", f"path {path}")

    # --- Size -----------------------------------------------------------
    if html_bytes > HTML_MAX_BYTES:
        add("html-over-2mb", f"{html_bytes} bytes of HTML")
    dom = _dom_stats(soup)
    if dom["max_depth"] > DOM_MAX_DEPTH:
        add("deep-dom", f"{dom['max_depth']} levels deep")
    if dom["max_children"] > DOM_MAX_CHILDREN:
        add("wide-dom-node", f"<{dom['widest_element']}> has {dom['max_children']} direct children")

    # --- Social ---------------------------------------------------------
    if not og.get("og:title") and not og.get("og:image"):
        add("missing-open-graph", "no og:title and no og:image")
    elif not og.get("og:image"):
        add("missing-og-image", "og:title is present, og:image is not")

    # --- JSON-LD --------------------------------------------------------
    jsonld_blocks = 0
    jsonld_errors: list[str] = []
    for script in soup.find_all("script", attrs={"type": re.compile(r"ld\+json", re.I)}):
        raw = script.string or script.get_text()
        if not raw or not raw.strip():
            continue
        jsonld_blocks += 1
        try:
            json.loads(raw)
        except RecursionError:
            jsonld_errors.append(f"block {jsonld_blocks}: nested too deeply to parse")
        except ValueError as exc:
            jsonld_errors.append(f"block {jsonld_blocks}: {exc}")
    if jsonld_errors:
        add("invalid-jsonld", f"{len(jsonld_errors)} of {jsonld_blocks} JSON-LD block(s): {'; '.join(jsonld_errors)}",
            len(jsonld_errors))

    # --- Content --------------------------------------------------------
    body_terms = set(significant_terms(visible_text))
    title_relevance, title_missing = _relevance(title, body_terms)
    description_relevance, description_missing = _relevance(description, body_terms)
    if not measurable:
        for check_id in ("thin-content", "title-not-reflected-in-content", "description-not-reflected-in-content"):
            skip(check_id, unspaced)
    else:
        if word_count < THIN_CONTENT_MIN_WORDS:
            add("thin-content", f"{word_count} words of visible body text")
        if word_count >= RELEVANCE_MIN_WORDS:
            if title_relevance is not None and title_relevance < TITLE_RELEVANCE_MIN:
                add("title-not-reflected-in-content",
                    f"{title_relevance:.0%} of title terms found in the body; missing: {', '.join(title_missing)}")
            if description_relevance is not None and description_relevance < DESCRIPTION_RELEVANCE_MIN:
                add("description-not-reflected-in-content",
                    f"{description_relevance:.0%} of description terms found in the body; "
                    f"missing: {', '.join(description_missing)}")

    facts = {
        "title": _trim(title),
        "title_length": len(title),
        "title_count": len(title_tags),
        "meta_description": _trim(description),
        "meta_description_length": len(description),
        "h1": [_trim(text, 120) for text in h1[:10]],
        "h1_count": len(h1),
        "heading_level_skips": heading_skips,
        "template_heading_level_skips": template_skips,
        "canonical": _trim(canonical) or None,
        "canonical_status": canonical_status,
        "canonical_note": canonical_note,
        "meta_robots": _trim(meta_robots) or None,
        "x_robots_tag": _trim(x_robots) or None,
        "x_robots_other_agents": [_trim(rule, 100) for rule in other_agent_rules[:ROBOTS_OTHER_AGENTS_MAX]],
        "hreflang_count": len(hreflang),
        "lang": _trim(lang, 35) or None,
        "has_doctype": has_doctype,
        "meta_charset": _trim(meta_charset, 40) or None,
        "header_charset": _trim(header_charset, 40) or None,
        "has_viewport": has_viewport,
        "images": {"total": images_total, "missing_alt": len(missing_alt), "empty_alt": images_empty_alt,
                   "missing_dimensions": len(missing_dimensions), "first_image_lazy": first_image_lazy},
        "links": {"total": links_total, "without_text": len(links_without_text),
                  "generic_anchor": len(generic_anchors), "http_on_https": len(insecure_links)},
        "mixed_content": len(mixed),
        "url": {"length": len(page_url), "query_params": [_trim(p, 40) for p in params[:20]]},
        "html_bytes": html_bytes,
        "dom": dom,
        "open_graph": {_trim(k, 40): _trim(v) for k, v in list(og.items())[:20]},
        "jsonld_blocks": jsonld_blocks,
        "jsonld_invalid": len(jsonld_errors),
        "word_count": word_count if measurable else None,
        "words_measured": measurable,
        "title_relevance": title_relevance if measurable else None,
        "description_relevance": description_relevance if measurable else None,
    }
    findings.sort(key=lambda item: SEVERITY_ORDER.index(item["severity"]))
    summary = {severity: sum(1 for f in findings if f["severity"] == severity) for severity in SEVERITY_ORDER}
    summary["total"] = len(findings)
    summary["checks_run"] = len(CHECKS) - len(not_checked)
    return {
        "url": page_url,
        "headers_available": headers_available,
        "facts": facts,
        "findings": findings,
        "not_checked": not_checked,
        "thresholds": dict(THRESHOLDS),
        "summary": summary,
    }


# ---------------------------------------------------------------------------
# CLI
# ---------------------------------------------------------------------------


class _InputError(Exception):
    """A problem with the arguments or the input. Carries the exit code."""

    def __init__(self, message: str, code: int = 1):
        super().__init__(message)
        self.code = code


def _decode(raw: bytes) -> str:
    """Decode a saved HTML file: UTF-8 first, then the charset it declares, then cp1252."""
    try:
        return raw.decode("utf-8-sig")
    except UnicodeDecodeError:
        pass
    declared = re.search(rb"charset\s*=\s*['\"]?([A-Za-z0-9_.:-]+)", raw[:4096], re.I)
    if declared:
        try:
            return raw.decode(declared.group(1).decode("ascii"))
        except (LookupError, UnicodeDecodeError):
            pass
    return raw.decode("cp1252", errors="replace")


def _load_headers(value: str) -> dict:
    """Headers from an inline JSON object or a JSON file. Accepts a fetch_page.py --json result."""
    text = value
    if not value.lstrip().startswith(("{", "[")):
        try:
            with open(value, encoding="utf-8-sig") as handle:
                text = handle.read(MAX_FILE_BYTES)
        except OSError as exc:
            raise _InputError(f"cannot read --headers-json file: {exc}") from exc
    try:
        data = json.loads(text)
    except (ValueError, RecursionError) as exc:
        raise _InputError(f"--headers-json is not valid JSON: {exc}", 2) from exc
    if isinstance(data, dict) and isinstance(data.get("headers"), dict):
        data = data["headers"]
    if not isinstance(data, dict):
        raise _InputError("--headers-json must be a JSON object of header names and values", 2)
    return data


def _http_url(value: str, option: str) -> str:
    try:
        parsed = urlparse(value.strip())
        valid = parsed.scheme.lower() in ("http", "https") and bool(parsed.hostname)
    except ValueError:
        valid = False
    if not valid:
        raise _InputError(f"{option} must be an absolute http or https URL", 2)
    return value.strip()


def _from_file(path: str, base_url: Optional[str], headers_json: Optional[str]) -> dict:
    if not base_url:
        raise _InputError("--file needs --base-url, the address the page is published at", 2)
    page_url = _http_url(base_url, "--base-url")
    headers = _load_headers(headers_json) if headers_json else None
    try:
        with open(path, "rb") as handle:
            raw = handle.read(MAX_FILE_BYTES + 1)
    except OSError as exc:
        raise _InputError(f"cannot read --file: {exc}") from exc
    if len(raw) > MAX_FILE_BYTES:
        raise _InputError(f"--file is larger than {MAX_FILE_BYTES} bytes")
    result = run_checks(_decode(raw), page_url, headers, html_bytes=len(raw))
    return {"url": result.pop("url"), "source": "file", "status_code": None, **result}


def _from_url(url: str, timeout: int) -> dict:
    if _SCRIPTS_DIR not in sys.path:
        sys.path.insert(0, _SCRIPTS_DIR)
    from fetch_page import fetch_page  # SSRF-safe fetch: url_safety validates and pins DNS

    page = fetch_page(url, timeout=timeout)
    if page.get("error"):
        raise _InputError(str(page["error"]))
    status = page.get("status_code")
    if page.get("content") is None or (isinstance(status, int) and status >= 400):
        raise _InputError(f"HTTP {status} from {page.get('url')}: no page to check")
    # The size as served: the raw body bytes fetch_page.py read. The UTF-8 size of the
    # decoded text, the fallback inside run_checks, doubles a single-byte Cyrillic page.
    content_bytes = page.get("content_bytes")
    served = content_bytes if isinstance(content_bytes, int) and not isinstance(content_bytes, bool) else None
    result = run_checks(page["content"], page["url"], page.get("headers") or {}, html_bytes=served)
    out = {"url": result.pop("url"), "source": "url", "status_code": status, **result}
    if page.get("url") != url or page.get("redirect_chain"):
        out["facts"]["requested_url"] = _trim(url)
        out["facts"]["redirects"] = len(page.get("redirect_chain") or [])
    if page.get("too_large"):
        out["facts"]["body_truncated"] = True
    return out


def main(argv: Optional[list[str]] = None) -> int:
    parser = argparse.ArgumentParser(
        description="Deterministic single-page on-page checks. JSON on stdout."
    )
    source = parser.add_mutually_exclusive_group(required=True)
    source.add_argument("--url", help="Page to fetch and check (fetched through fetch_page.py).")
    source.add_argument("--file", help="Saved HTML file to check. No network request is made.")
    parser.add_argument("--base-url", help="With --file: the URL the page is published at.")
    parser.add_argument(
        "--headers-json",
        help="With --file: response headers, as a path to a JSON file or an inline JSON object.",
    )
    parser.add_argument("--timeout", type=int, default=30, help="With --url: timeout in seconds (default 30).")
    parser.add_argument("--json", action="store_true", help="Accepted for consistency. Output is always JSON.")
    args = parser.parse_args(argv)

    try:
        if args.file:
            result = _from_file(args.file, args.base_url, args.headers_json)
        else:
            if args.base_url or args.headers_json:
                raise _InputError("--base-url and --headers-json apply to --file only", 2)
            result = _from_url(args.url, args.timeout)
    except _InputError as exc:
        print(json.dumps({"error": str(exc)}, indent=2))
        return exc.code
    except RecursionError:
        print(json.dumps({"error": "the document is nested too deeply to parse"}, indent=2))
        return 1
    print(json.dumps(result, indent=2))
    return 0


if __name__ == "__main__":
    sys.exit(main())