Skip to content

Universal SEO skill for Claude Code. 26 sub-skills + 19 sub-agents covering technical SEO, E-E-A-T, schema, GEO/AEO, agent readiness (Lighthouse Agentic Browsing, WebMCP, llms.txt), backlinks, local SEO, e-commerce, international SEO, Google APIs, and PDF/Excel reporting. 9 optional extensions for live SEO data.

65.1 KB
Raw Blame
#!/usr/bin/env python3
"""
Agent-readiness HTTP auditor.

Checks the parts of agent readiness that live in HTTP responses and raw HTML,
so they can be verified without a browser:

1. robots.txt groups for AI crawlers and user-triggered agents, evaluated with
   RFC 9309 group selection (a named group replaces ``*``; nothing inherits),
   plus ``Content-Signal`` lines and the ``Agentmap:`` directive. User-agent
   lines are matched by product token (case-insensitive, compared whole). A
   429 on robots.txt is reported as inconclusive, not as "no robots.txt".
2. ``/llms.txt`` against the Lighthouse ``llms-txt`` rules (the headline
   verdict), plus line-level format notes against the llmstxt.org proposal.
3. Markdown delivery: ``Accept: text/markdown`` negotiation (``Vary: Accept``),
   ``rel="alternate" type="text/markdown"`` links, and ``.md`` siblings.
4. Agentic Resource Discovery: ``ai-catalog.json`` discovery in the same order
   Lighthouse uses (robots ``Agentmap``, ``<link rel="ai-catalog">``, HTTP
   ``Link``, ``/.well-known/ai-catalog.json``) and the ARD conformance rules.
5. ``/.well-known`` discovery documents: RFC 9727 API Catalog, RFC 9728 and
   RFC 8414 OAuth metadata, and the A2A agent card. MCP Server Cards are found
   through ``ai-catalog.json`` (step 4), per the SEP-2127 proposal.
6. Server-rendered content: visible words in the raw HTML and JS-shell markers.
7. WebMCP hints in markup: declarative form attributes and imperative
   ``modelContext.registerTool`` calls in same-origin scripts.
8. Optional (``--ua-matrix``): how the site answers requests that carry each
   AI agent's user-agent token, compared with a browser user agent.

Audit posture
=============
Findings carry a priority (P0 to P3) and a status (pass, warn, fail, info, na).
Items built on drafts or proposals (Content-Signal, WebMCP, ARD) are labelled with their standards status. The script
does not compute a 0-100 score. For the Lighthouse "Agentic Browsing" fraction
use ``lighthouse_agentic.py``; for the accessibility tree use
``agent_ux_check.py``.

The user-agent matrix sends unverified requests. A WAF that challenges them is
often behaving correctly, because real agents are verified by IP range or Web
Bot Auth signature, not by the user-agent string. Read that section as
"observed behaviour for unverified traffic", never as proof that the real
agent is blocked.

SSRF
====
Every request goes through ``url_safety.safe_requests_get`` or
``url_safety.validate_url_strict``. Redirect targets are re-validated by the
DNS-pinning layer.

CLI
===
    python agentic_check.py https://example.com --json
    python agentic_check.py https://example.com/pricing --json --ua-matrix
"""

from __future__ import annotations

import argparse
import datetime as _dt
import json
import os
import re
import sys
from typing import Optional
from urllib.parse import urljoin, urlparse

_SCRIPTS_DIR = os.path.dirname(os.path.abspath(__file__))
if _SCRIPTS_DIR not in sys.path:
    sys.path.insert(0, _SCRIPTS_DIR)
from url_safety import (  # noqa: E402
    MAX_JSON_DEPTH,
    URLSafetyError,
    decode_body,
    json_depth_exceeds,
    safe_requests_get,
)

CHECKED_ON = "2026-09-23"
MAX_BODY = 2_000_000
MAX_SCRIPTS = 8
BROWSER_UA = ("Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) "
              "Chrome/153.0.0.0 Safari/537.36")

# token, vendor, role, robots behaviour documented by the vendor.
# role: training | search | user (user-triggered fetch or agent) | control (robots token only)
AI_AGENTS = [
    ("GPTBot", "OpenAI", "training", "honours"),
    ("OAI-SearchBot", "OpenAI", "search", "honours"),
    ("ChatGPT-User", "OpenAI", "user", "may not apply"),
    ("OAI-AdsBot", "OpenAI", "ads", "see vendor documentation"),
    ("ClaudeBot", "Anthropic", "training", "honours"),
    ("Claude-SearchBot", "Anthropic", "search", "honours"),
    ("Claude-User", "Anthropic", "user", "honours"),
    ("PerplexityBot", "Perplexity", "search", "honours"),
    ("Perplexity-User", "Perplexity", "user", "generally ignores"),
    ("Google-Extended", "Google", "control", "honours"),
    ("Google-Agent", "Google", "user", "generally ignores"),
    ("Applebot-Extended", "Apple", "control", "honours"),
    ("CCBot", "Common Crawl", "training", "honours"),
]
ROLE_MEANING = {
    "training": "model training",
    "search": "AI search indexing and citation",
    "user": "fetches and actions a person asked for",
    "control": "a robots.txt control token (no separate crawler)",
    "ads": "ad landing-page checks",
}
# Representative user-agent strings containing each documented product token.
# Only the token matters for robots.txt matching and most WAF rules.
UA_STRINGS = {
    "GPTBot": "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; GPTBot/1.3; +https://openai.com/gptbot)",
    "OAI-SearchBot": "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; OAI-SearchBot/1.3; +https://openai.com/searchbot)",
    "ChatGPT-User": "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; ChatGPT-User/1.0; +https://openai.com/bot)",
    "ClaudeBot": "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; ClaudeBot/1.0; [email protected])",
    "Claude-SearchBot": "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Claude-SearchBot/1.0; [email protected])",
    "Claude-User": "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Claude-User/1.0; [email protected])",
    "PerplexityBot": "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; PerplexityBot/1.0; +https://perplexity.ai/perplexitybot)",
    "Perplexity-User": "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Perplexity-User/1.0; +https://perplexity.ai/perplexity-user)",
}
CONTENT_SIGNAL_KEYS = {"search", "ai-input", "ai-train"}
# Cloudflare is testing a fourth key; accept it without treating it as standard.
CONTENT_SIGNAL_EXPERIMENTAL = {"use": {"immediate", "reference", "full"}}
# Vendor interstitial fingerprints. Generic words such as "captcha" or "access
# denied" appear on ordinary pages, so they only count with a 4xx/5xx status.
CHALLENGE_MARKERS = (
    "just a moment...", "cf-chl", "cf_chl", "px-captcha", "_incapsula_resource",
    "datadome", "verify you are human",
)
ARD_MEDIA_TYPES = {
    "application/ai-catalog+json", "application/agent-card+json",
    "application/a2a-agent-card+json", "application/mcp-server-card+json",
    "application/agent-skills+zip", "application/agent-skills+gzip",
    'text/markdown; profile="urn:air:agent-skills"', "application/ai-registry",
    "application/ai-registry+json",
}
ARD_URN = re.compile(r"^urn:air:([a-zA-Z0-9.-]+)(?::([a-zA-Z0-9._:-]+))?:([a-zA-Z0-9._-]+)$")
WELL_KNOWN = [
    # path, label, standards status, priority, expected content-type fragment
    ("/.well-known/api-catalog", "API Catalog", "RFC 9727", "P2", "linkset+json"),
    ("/.well-known/oauth-protected-resource", "OAuth Protected Resource Metadata",
     "RFC 9728", "P2", "json"),
    ("/.well-known/oauth-authorization-server", "OAuth Authorization Server Metadata",
     "RFC 8414", "P2", "json"),
    ("/.well-known/agent-card.json", "A2A agent card", "A2A protocol", "P3", "json"),
    ("/.well-known/ucp", "UCP commerce profile", "UCP (Google + Shopify open spec); "
     "depth in seo-ecommerce ucp_check.py", "P3", "json"),
]
# MCP Server Cards (SEP-2127, unmerged) are discovered through ai-catalog.json, which
# audit_ard() covers. The older /.well-known/mcp.json proposal (SEP-1649) is closed.


# --------------------------------------------------------------------------- fetch


def _site_root(url: str) -> str:
    parsed = urlparse(url if "://" in url else "https://" + url)
    return f"{parsed.scheme}://{parsed.netloc}"


def fetch(url: str, *, headers: Optional[dict] = None, timeout: int = 15) -> dict:
    """GET ``url`` through url_safety and return a small, JSON-safe record."""
    record = {"url": url, "status": None, "headers": {}, "text": "", "final_url": None,
              "error": None}
    try:
        resp = safe_requests_get(url, timeout=timeout, headers=headers or {},
                                 allow_redirects=True, stream=True)
        body = resp.raw.read(MAX_BODY + 1, decode_content=True) or b""
        resp.close()
    except URLSafetyError as exc:
        record["error"] = f"blocked by url_safety: {exc}"
        return record
    except Exception as exc:  # network errors are findings, not crashes
        record["error"] = f"{type(exc).__name__}: {exc}"[:300]
        return record
    record["status"] = resp.status_code
    record["final_url"] = resp.url
    record["headers"] = {k.lower(): v for k, v in resp.headers.items()}
    record["truncated"] = len(body) > MAX_BODY
    record["text"] = _decode(body[:MAX_BODY], record["headers"].get("content-type", ""))
    return record


def _decode(body: bytes, content_type: str) -> str:
    """Decode streamed bytes with url_safety's shared rules (#314), minus any BOM."""
    return decode_body(body, content_type).lstrip("\ufeff")


def _ctype(rec: dict) -> str:
    return (rec.get("headers") or {}).get("content-type", "").lower()


def _check(cid, title, priority, status, standard, evidence=None, fix=None) -> dict:
    return {"id": cid, "title": title, "priority": priority, "status": status,
            "standard": standard, "evidence": evidence or {}, "fix": fix}


# --------------------------------------------------------------------------- robots.txt


_LINE_BREAK = re.compile(r"\r\n|\n|\r")


def robots_lines(text: str) -> list:
    """Split robots.txt on CR, LF or CRLF only (RFC 9309).

    ``str.splitlines`` also breaks on U+2028 and other Unicode separators, which
    would turn text inside a comment into a live rule.
    """
    lines = _LINE_BREAK.split(text.lstrip("\ufeff"))
    if lines and lines[-1] == "" and _LINE_BREAK.search(text[-2:] if text else ""):
        lines.pop()
    return lines


def parse_robots(text: str) -> dict:
    """Parse robots.txt into RFC 9309 groups plus global lines.

    A group is one or more consecutive ``user-agent`` lines followed by rules.
    ``content-signal`` and other non-global lines attach to the current group.
    """
    groups: list = []
    global_lines = {"sitemap": [], "agentmap": [], "orphan_content_signal": []}
    current = None
    last_was_ua = False
    for raw in robots_lines(text):
        line = raw.split("#", 1)[0].strip()
        if not line or ":" not in line:
            continue
        field, value = (part.strip() for part in line.split(":", 1))
        field = field.lower()
        if field == "user-agent":
            if current is None or not last_was_ua:
                current = {"agents": [], "rules": [], "content_signal": [], "other": []}
                groups.append(current)
            current["agents"].append(value)
            last_was_ua = True
            continue
        last_was_ua = False
        if field == "sitemap":
            global_lines["sitemap"].append(value)
        elif field == "agentmap":
            global_lines["agentmap"].append(value)
        elif current is None:
            if field == "content-signal":
                global_lines["orphan_content_signal"].append(value)
        elif field in ("allow", "disallow"):
            current["rules"].append((field, value))
        elif field == "content-signal":
            current["content_signal"].append(value)
        else:
            current["other"].append((field, value))
    return {"groups": groups, **global_lines}


_PRODUCT_TOKEN = re.compile(r"[A-Za-z_-]+")


def product_token(value: str) -> str:
    """Lowercased product token of one ``User-agent:`` value: ``*``, a name, or ``""``.

    RFC 9309 section 2.2 allows only ``product-token = identifier / "*"`` with
    ``identifier = 1*(%x2D / %x41-5A / %x5F / %x61-7A)`` (letters, "_", "-"), and
    section 2.2.1 says "Crawlers MUST use case-insensitive matching to find the
    group that matches the product token"
    (https://www.rfc-editor.org/rfc/rfc9309.html, read 2026-11-10).

    The RFC is silent on values outside that grammar, such as ``GPTBot/1.1`` or
    ``ClaudeBot*``. This follows Google's documented reading: "All non-matching
    text is ignored (for example, both googlebot/1.2 and googlebot* are
    equivalent to googlebot)"
    (https://developers.google.com/search/docs/crawling-indexing/robots/robots_txt,
    read 2026-11-10), which Google's open-source parser implements by keeping
    the leading run of ``[a-zA-Z_-]``. Other crawlers may be stricter, so
    ``audit_robots`` lists such lines as ``nonstandard_user_agent_lines``.

    A value with no leading token character ("008", "*bot") gives ``""``. It
    still starts a group, one that matches no crawler.
    """
    value = (value or "").strip()
    if value == "*":
        return "*"
    match = _PRODUCT_TOKEN.match(value)
    return match.group(0).lower() if match else ""


def _group_names(group: dict, token_l: str) -> bool:
    return any(product_token(agent) == token_l for agent in group["agents"])


def select_group(parsed: dict, token: str) -> dict:
    """RFC 9309: combine every group naming ``token``; else every ``*`` group.

    Tokens are compared whole, so ``GPTBot`` does not select a group written
    for ``GPTBot-Image`` (the RFC's examples merge groups "that match the same
    product token exactly"). See ``product_token`` for versioned values.
    """
    token_l = token.strip().lower()
    named = [g for g in parsed["groups"] if _group_names(g, token_l)]
    chosen = named or [g for g in parsed["groups"] if _group_names(g, "*")]
    return {
        "matched": "named" if named else ("star" if chosen else "none"),
        "rules": [r for g in chosen for r in g["rules"]],
        "content_signal": [c for g in chosen for c in g["content_signal"]],
    }


_STAR_RUN = re.compile(r"\*{2,}")


def _pattern_matches(pattern: str, path: str) -> bool:
    """RFC 9309 path match: ``*`` is any run of characters, a final ``$`` the end.

    Uses ``str.find``, not a regex built from the rule. The rule is untrusted,
    and one such as ``****x`` became ``^.*.*.*.*x``, which backtracks without
    limit. ``*`` is the only wildcard, so placing each literal piece at its
    first possible position is enough.
    """
    anchored = pattern.endswith("$")
    body = pattern[:-1] if anchored else pattern
    if "*" not in body:
        return path == body if anchored else path.startswith(body)
    first, *middle, last = _STAR_RUN.sub("*", body).split("*")
    if not path.startswith(first):
        return False
    pos = len(first)
    for piece in middle:
        found = path.find(piece, pos)
        if found < 0:
            return False
        pos = found + len(piece)
    if anchored:
        return len(path) - len(last) >= pos and path.endswith(last)
    return path.find(last, pos) >= 0


def is_allowed(rules: list, path: str = "/") -> bool:
    """Longest-match evaluation; allow wins a tie; empty disallow allows all."""
    best_len, allowed = -1, True
    for field, value in rules:
        if field == "disallow" and value == "":
            continue
        if _pattern_matches(value, path):
            length = len(value)
            if length > best_len or (length == best_len and field == "allow"):
                best_len, allowed = length, field == "allow"
    return allowed


def parse_content_signal(value: str) -> dict:
    """Parse ``search=yes, ai-input=yes, ai-train=no`` into a dict plus issues."""
    signals, issues = {}, []
    for part in value.split(","):
        part = part.strip()
        if not part:
            continue
        if "=" not in part:
            issues.append(f"malformed entry '{part}'")
            continue
        key, val = (s.strip().lower() for s in part.split("=", 1))
        signals[key] = val
        if key in CONTENT_SIGNAL_EXPERIMENTAL:
            if val not in CONTENT_SIGNAL_EXPERIMENTAL[key]:
                issues.append(f"value for '{key}' must be one of "
                              f"{sorted(CONTENT_SIGNAL_EXPERIMENTAL[key])}")
            continue
        if key not in CONTENT_SIGNAL_KEYS:
            issues.append(f"unknown key '{key}'")
        elif val not in ("yes", "no"):
            issues.append(f"value for '{key}' must be yes or no")
    return {"signals": signals, "issues": issues}


def audit_robots(root: str) -> tuple[list, dict]:
    rec = fetch(root + "/robots.txt")
    checks, data = [], {"status": rec["status"], "error": rec["error"]}
    if rec["error"] or rec["status"] is None:
        checks.append(_check("robots-reachable", "robots.txt reachable", "P0", "warn",
                             "RFC 9309", {"error": rec["error"]},
                             "Serve /robots.txt so agents can read your policy."))
        return checks, data
    if rec["status"] == 429:
        # RFC 9309 section 2.3.1.3 gives "status codes ... in the 400-499 range"
        # as its example of an unavailable file and does not single out 429.
        # Google does: "Google's crawlers treat all 4xx errors, except 429, as
        # if a valid robots.txt file didn't exist"
        # (https://developers.google.com/search/docs/crawling-indexing/robots/robots_txt)
        # and "Google's crawlers treat the 429 status code as a signal that the
        # server is overloaded, and it's considered a server error"
        # (https://developers.google.com/search/docs/crawling-indexing/http-network-errors).
        # Both read 2026-11-10. A rate-limited fetch also says nothing about the
        # file's contents, so the result is neither "all allowed" nor "all
        # disallowed": it is inconclusive.
        data["inconclusive"] = True
        retry_after = (rec.get("headers") or {}).get("retry-after")
        checks.append(_check("robots-reachable", "robots.txt reachable", "P0", "warn",
                             "RFC 9309; Google robots.txt documentation (429)",
                             {"status": 429, "inconclusive": True,
                              "retry_after": retry_after[:80] if retry_after else None},
                             "Inconclusive: robots.txt answered 429 (rate limited), so its "
                             "rules were not read. This is not a missing file. Google treats "
                             "a 429 on robots.txt as a server error, not as 'no "
                             "restrictions'. Rerun later, and make sure the server or WAF "
                             "does not rate-limit /robots.txt for crawlers."))
        return checks, data
    if rec["status"] >= 500:
        checks.append(_check("robots-reachable", "robots.txt reachable", "P0", "fail",
                             "RFC 9309", {"status": rec["status"]},
                             "A 5xx robots.txt tells compliant crawlers to treat the "
                             "whole site as disallowed. Fix the server error."))
        return checks, data
    if rec["status"] >= 400:
        checks.append(_check("robots-reachable", "robots.txt present", "P0", "warn",
                             "RFC 9309", {"status": rec["status"]},
                             "No robots.txt means everything is allowed and no "
                             "AI-usage preference is declared. Publish one deliberately."))
        data["parsed"] = parse_robots("")
        return checks, data

    parsed = parse_robots(rec["text"])
    data["parsed"] = {k: v for k, v in parsed.items() if k != "groups"}
    data["group_count"] = len(parsed["groups"])
    checks.append(_check("robots-reachable", "robots.txt reachable", "P0", "pass",
                         "RFC 9309", {"status": rec["status"], "groups": len(parsed["groups"])}))

    per_agent = []
    for token, vendor, role, behaviour in AI_AGENTS:
        sel = select_group(parsed, token)
        per_agent.append({
            "token": token, "vendor": vendor, "role": role,
            "governs": ROLE_MEANING[role], "robots_behaviour": behaviour,
            "group": sel["matched"], "root_allowed": is_allowed(sel["rules"], "/"),
            "content_signal": sel["content_signal"],
        })
    data["agents"] = per_agent

    # Values outside the RFC 9309 product-token grammar (a version, a trailing
    # "*", digits). They are matched leniently here; a stricter crawler may not.
    nonstandard = list(dict.fromkeys(
        agent[:80] for g in parsed["groups"] for agent in g["agents"]
        if agent != "*" and not _PRODUCT_TOKEN.fullmatch(agent)))[:20]
    data["nonstandard_user_agent_lines"] = nonstandard

    explicit = [a["token"] for a in per_agent if a["group"] == "named"]
    blocked_search = [a["token"] for a in per_agent if a["role"] == "search" and not a["root_allowed"]]
    checks.append(_check(
        "robots-ai-groups", "Deliberate robots.txt groups for AI user agents", "P0",
        "warn" if blocked_search else ("pass" if explicit else "info"), "RFC 9309",
        {"named_groups": explicit, "search_crawlers_blocked_at_root": blocked_search,
         "nonstandard_user_agent_lines": nonstandard},
        ("Blocking an AI search crawler removes the site from that engine's answers. "
         "Confirm this is intended." if blocked_search else
         None if explicit else
         "Every AI agent falls through to the * group. Add named groups if your "
         "policy differs by purpose (training vs search vs user fetches).")))

    user_agents = [a for a in per_agent if a["role"] == "user" and not a["root_allowed"]]
    checks.append(_check(
        "robots-user-agents", "User-triggered agents and robots.txt", "P1",
        "info" if user_agents else "pass", "vendor documentation",
        {"blocked_at_root": [a["token"] for a in user_agents],
         "documented_behaviour": {a["token"]: a["robots_behaviour"] for a in per_agent
                                  if a["role"] == "user"}},
        "Several user-triggered agents do not treat robots.txt as binding. Protect "
        "private paths with authentication, not robots.txt." if user_agents else None))

    all_signals = [s for g in parsed["groups"] for s in g["content_signal"]]
    signal_issues = []
    for value in all_signals + parsed["orphan_content_signal"]:
        signal_issues += parse_content_signal(value)["issues"]
    gap = [a["token"] for a in per_agent
           if a["group"] == "named" and not a["content_signal"]
           and any(g["content_signal"] for g in parsed["groups"] if _group_names(g, "*"))]
    if not all_signals and not parsed["orphan_content_signal"]:
        status, fix = "info", ("No Content-Signal line. Optional: add one (for example "
                               "'Content-Signal: search=yes, ai-input=yes, ai-train=no') "
                               "to each group whose policy you want to state.")
    elif signal_issues or parsed["orphan_content_signal"] or gap:
        status = "warn"
        fix = ("A crawler that matches a named group reads only that group (RFC 9309), "
               "and no normative text says Content-Signal escapes that rule. Repeat the "
               "line inside every named group to be safe." if gap else
               "Place Content-Signal inside a user-agent group and use yes/no values "
               "for search, ai-input and ai-train.")
    else:
        status, fix = "pass", None
    checks.append(_check(
        "content-signal", "Content-Signal preference declared", "P1", status,
        "Cloudflare Content Signals Policy (CC0) + IETF draft; a preference, not "
        f"enforcement; Google says it does not act on it (checked {CHECKED_ON})",
        {"lines": all_signals, "outside_any_group": parsed["orphan_content_signal"],
         "issues": signal_issues, "named_groups_without_signal": gap}, fix))
    data["agentmap"] = parsed["agentmap"]
    return checks, data


# --------------------------------------------------------------------------- llms.txt


# Line-level format notes, adapted from the WebObs llms.txt validator (same
# author, MIT). Source for every rule: https://llmstxt.org/ ("The /llms.txt
# file, v2", modified 2026-08-10, read 2026-11-10) and its reference parser
# (AnswerDotAI/llms-txt, llms_txt/core.py), which reads a file list item as
# ``-\s*\[title\]\(url\)`` plus an optional ``:\s*notes``.
#
# llmstxt.org describes itself as "A proposal to standardise on using an
# /llms.txt file". It is a community proposal, not a ratified standard, so the
# levels are relative to that text and nothing here is a ranking factor:
#   error  the one part the proposal requires is missing ("An H1 with the name
#          of the project or site. This is the only required section")
#   warn   the line departs from the proposed format, so a parser may misread it
#   info   an optional part, the proposal's writing guidance, or a claude-seo
#          heuristic
LLMS_FORMAT_BASIS = ("llmstxt.org community proposal (v2), not a ratified standard. "
                     "Format hygiene only: no note is a ranking or citation factor.")
LLMS_REVIEW_BYTES = 100 * 1024  # claude-seo review threshold, not a limit in the proposal
LLMS_NOTES_PER_ID = 5
LLMS_FORMAT_RULES = {
    "missing-h1-title": ("error", "The file does not start with an H1 holding the project or "
                         "site name (`# Name`). It is the only part the proposal requires."),
    "multiple-h1": ("warn", "More than one H1. The proposed format has one H1 title; a second "
                    "one is outside the format."),
    "summary-not-after-title": ("warn", "A blockquote appears later in the file, not directly "
                                "after the H1, so parsers do not read it as the summary."),
    "heading-in-details": ("warn", "A heading before the first H2 section. The proposal allows "
                           "any Markdown there except headings."),
    "malformed-link": ("warn", "This item looks meant as a link but is not written as "
                       "`[name](url)`, so a parser cannot read the address."),
    "bullet-not-dash": ("warn", "The item uses `*`, `+` or a number. The proposal's examples "
                        "use `-`, and the reference parser only matches `-` items."),
    "notes-without-colon": ("warn", "Text follows the link without the `:` separator, so it is "
                            "not read as the link's notes."),
    "invalid-url": ("warn", "The link is not an http or https address."),
    "no-summary-blockquote": ("info", "No blockquote summary (`> One sentence about the site`) "
                              "after the H1. The proposal lists it; only the H1 is required."),
    "item-without-link": ("info", "A list item without a link inside an H2 section. The "
                          "proposal's file list items carry a `[name](url)` link; general "
                          "notes belong before the first H2."),
    "link-not-at-start": ("info", "The link sits inside a sentence. The reference parser only "
                          "reads items that start with the link."),
    "empty-section": ("info", "An H2 section without links. In the proposal, H2 sections hold "
                      "file lists; general text belongs before the first H2."),
    "duplicate-link": ("info", "The same address is listed more than once."),
    "relative-url": ("info", "A relative link. It resolves only against the file's own "
                     "address; the proposal's examples use full URLs."),
    "optional-heading-variant": ("info", "This looks meant as the Optional section, but the "
                                 "proposal names it exactly `## Optional`."),
    "no-link-list": ("info", "No file lists. The proposal allows this (zero or more H2 "
                     "sections), but links are how agents reach the detail."),
    "links-without-notes": ("info", "Most links have no notes. The proposal's guidance: 'When "
                            "linking to resources, include brief, informative descriptions.'"),
    "oversize": ("info", "Over 100 KB (a claude-seo review threshold). The proposal says the "
                 "file 'stays small enough to fit in context', with the detail behind links."),
    "content-type": ("info", "Not served as text/plain or text/markdown. The proposal names no "
                     "content type; this is a claude-seo heuristic."),
}
_LLMS_ITEM = re.compile(r"^-\s*\[(?P<title>[^\]]+)\]\((?P<url><[^>\n]+>|(?:[^()\s<]|\([^()\s]*\))+)"
                        r"(?:\s+\"[^\"]*\")?\)(?P<rest>.*)$")
_LLMS_BULLET = re.compile(r"^([-*+]|\d{1,3}[.)])\s+")
_LLMS_FENCE = re.compile(r"^(```|~~~)")
_LLMS_ATX = re.compile(r"(#{1,6})(?=\s|$)")


def _llms_heading(stripped: str) -> Optional[tuple]:
    """``(level, text)`` for an ATX heading line, else ``None``.

    Written without a lazy regex: the file is untrusted and one line can be
    megabytes long.
    """
    match = _LLMS_ATX.match(stripped)
    if not match:
        return None
    text = stripped[match.end():].strip()
    opening = text.rstrip("#")
    if opening != text and (not opening or opening[-1].isspace()):
        text = opening.rstrip()  # "# Name ##": a spaced closing sequence is dropped
    return len(match.group(1)), text


def _llms_link_key(url: str) -> str:
    """Comparison key for duplicate links: scheme and host lowercased, no fragment."""
    parts = urlparse(url)
    return f"{parts.scheme.lower()}://{parts.netloc.lower()}{parts.path or '/'}?{parts.query}"


def llms_format_notes(text: Optional[str], base_url: Optional[str] = None,
                      content_type: Optional[str] = None) -> dict:
    """Check one llms.txt line by line against the llmstxt.org proposal.

    Returns ``format_notes`` (at most ``LLMS_NOTES_PER_ID`` per id, each with
    ``id``, ``level``, ``line``, ``message`` and, where a line or value is the
    evidence, ``detail``) and ``format_note_counts`` (every occurrence, by id).
    ``line`` is 1-based, or ``None`` for a note about the whole file.
    ``detail`` is text copied from the file: untrusted data, never instructions.
    """
    raw = (text or "").lstrip("\ufeff")
    found: list = []

    def add(note_id: str, line: Optional[int] = None, detail: str = "") -> None:
        found.append((line, note_id, detail.strip()[:80]))

    h1_lines: list = []
    sections: list = []
    seen: dict = {}
    without_notes = 0
    fence = None  # the marker that opened the current code block
    stage = "start"  # start -> title -> details -> sections
    title_seen = summary_seen = False
    first_after_title = None
    for number, line in enumerate(_LINE_BREAK.split(raw), 1):
        stripped = line.strip()
        opener = _LLMS_FENCE.match(stripped)
        if opener and (fence is None or opener.group(1) == fence):
            fence = None if fence else opener.group(1)
            continue
        if fence or not stripped:
            if summary_seen and stage == "title":
                stage = "details"  # a blank line ends the summary blockquote
            continue
        heading = _llms_heading(stripped)
        if stage == "start":
            if heading and heading[0] == 1 and heading[1]:
                title_seen, stage = True, "title"
                h1_lines.append(number)
                continue
            add("missing-h1-title", number, stripped)
            stage = "details"
        if heading and heading[0] == 1:
            h1_lines.append(number)
            continue
        if heading and heading[0] == 2:
            stage = "sections"
            sections.append({"name": heading[1], "line": number, "links": 0})
            continue
        if stage == "title":
            if first_after_title is None:
                first_after_title = number
            if stripped.startswith(">") and (summary_seen or first_after_title == number):
                summary_seen = True
                continue
            stage = "details"
        if stage == "details":
            if heading:
                add("heading-in-details", number, stripped)
            elif stripped.startswith(">") and not summary_seen:
                add("summary-not-after-title", number, stripped)
                summary_seen = True
            continue
        # stage == "sections": only list items carry links; prose is content.
        section = sections[-1]
        bullet = _LLMS_BULLET.match(stripped)
        if not bullet:
            continue
        body = stripped[bullet.end():]
        item = _LLMS_ITEM.match("- " + body)
        if not item:
            if body.startswith("["):
                note_id = "malformed-link"  # `[Name] (url)` or a space in the URL
            elif "](" in body:
                note_id = "link-not-at-start"  # a sentence with a link in it
            elif re.search(r"https?://", body):
                note_id = "malformed-link"  # a bare address, not written as a link
            else:
                note_id = "item-without-link"
            add(note_id, number, stripped)
            continue
        if bullet.group(1) != "-":
            add("bullet-not-dash", number, stripped)
        rest = item.group("rest").strip()
        if rest and not rest.startswith(":"):
            add("notes-without-colon", number, stripped)
        href = item.group("url").strip()
        if href.startswith("<") and href.endswith(">"):
            href = href[1:-1].strip()
        try:
            scheme = urlparse(href).scheme
            absolute = href if scheme else (urljoin(base_url, href) if base_url else "")
            key = _llms_link_key(absolute) if absolute else ""
        except ValueError:  # for example an unclosed IPv6 bracket
            scheme, key = "invalid", ""
        if scheme and scheme not in ("http", "https"):
            add("invalid-url", number, href)
            continue
        if not scheme:
            add("relative-url", number, href)
        section["links"] += 1
        without_notes += not rest.lstrip(":").strip()
        if key and key in seen:
            add("duplicate-link", number, f"repeats line {seen[key]}: {href}")
        elif key:
            seen[key] = number

    if stage == "start":
        add("missing-h1-title")  # empty file, or nothing outside a code block
    elif title_seen and not summary_seen:
        add("no-summary-blockquote", h1_lines[0])
    if len(h1_lines) > 1:
        add("multiple-h1", h1_lines[1], "H1 on lines " + ", ".join(map(str, h1_lines[:5])))
    for section in sections:
        if not section["links"]:
            add("empty-section", section["line"], "## " + section["name"])
        if section["name"] != "Optional" and re.match(r"optional\b", section["name"], re.I):
            add("optional-heading-variant", section["line"], "## " + section["name"])
    total_links = sum(section["links"] for section in sections)
    if not total_links:
        add("no-link-list")
    elif without_notes > total_links / 2:
        add("links-without-notes", None, f"{without_notes} of {total_links} links")
    size = len(raw.encode("utf-8", errors="replace"))
    if size > LLMS_REVIEW_BYTES:
        add("oversize", None, f"{size // 1024} KB")
    kind = (content_type or "").split(";")[0].strip().lower()
    if kind and kind not in ("text/plain", "text/markdown", "text/x-markdown"):
        add("content-type", None, kind)

    found.sort(key=lambda note: (note[0] is None, note[0] or 0, note[1]))
    counts: dict = {}
    notes = []
    for line, note_id, detail in found:
        counts[note_id] = counts.get(note_id, 0) + 1
        if counts[note_id] > LLMS_NOTES_PER_ID:
            continue  # a long file can repeat one pattern hundreds of times
        level, message = LLMS_FORMAT_RULES[note_id]
        note = {"id": note_id, "level": level, "line": line, "message": message}
        if detail:
            note["detail"] = detail
        notes.append(note)
    return {"format_notes": notes, "format_note_counts": counts}


def evaluate_llms_txt(status: Optional[int], content: Optional[str], *,
                      base_url: Optional[str] = None,
                      content_type: Optional[str] = None) -> dict:
    """Mirror Lighthouse 13.5.0 ``llms-txt`` and add llmstxt.org structure notes.

    ``lighthouse``, ``errors`` and ``notes`` are the headline verdict and are
    not affected by ``format_notes``, the line-level list from
    ``llms_format_notes``. That list is empty when there is no file to read.
    """
    result = _lighthouse_llms_txt(status, content)
    served_html = any("soft 404" in note for note in result["notes"])
    has_file = status is not None and status < 400 and not served_html
    fmt = (llms_format_notes(content, base_url, content_type) if has_file
           else {"format_notes": [], "format_note_counts": {}})
    return {**result, **fmt, "format_basis": LLMS_FORMAT_BASIS}


# Lighthouse 13.5.0 tests the file with ``/^\s*#\s+.+/m``, ``/\[.+\]\(.+\)/``
# and the like. Run as written they are quadratic: ``^\s*`` rescans the rest of
# a run of blank lines from every line start (40,000 newlines took 1.8 s, a
# 2 MB file over an hour), and ``\[.+\]\(.+\)`` rescans the line from every
# ``[``. The patterns below give the same answer for every input in one pass.
# ``^\s*X`` matches exactly when X follows only whitespace on its own line, so
# the leading run is limited to that line (``[^\S\n]`` is ``\s`` minus ``\n``).
_LLMS_LH_H1 = re.compile(r"^[^\S\n]*#\s+.+", re.M)
_LLMS_LH_SUMMARY = re.compile(r"^[^\S\n]*>\s+\S", re.M)
_LLMS_LH_H2 = re.compile(r"^[^\S\n]*##\s+\S", re.M)


def _llms_has_markdown_link(content: str) -> bool:
    """Same answer as ``re.search(r"\\[.+\\]\\(.+\\)", content)``, in linear time.

    ``.`` stops at a newline, so a match lies within one line: a ``[``, at
    least one character, ``](``, at least one character, ``)``. Taking the
    first ``[`` and then the first ``](`` leaves the most room for the rest, so
    one ``str.find`` each settles the line.
    """
    if "](" not in content:
        return False
    for line in content.split("\n"):
        opening = line.find("[")
        if opening < 0:
            continue
        middle = line.find("](", opening + 2)
        if middle >= 0 and line.find(")", middle + 3) >= 0:
            return True
    return False


def _lighthouse_llms_txt(status: Optional[int], content: Optional[str]) -> dict:
    if status is None:
        return {"lighthouse": "fail", "errors": ["fetch failed"], "notes": []}
    if status >= 500:
        return {"lighthouse": "fail", "errors": [f"HTTP {status}"], "notes": []}
    if status >= 400:
        return {"lighthouse": "not-applicable", "errors": [], "notes": [f"HTTP {status}"]}
    content = (content or "").lstrip("\ufeff")  # JavaScript's \s matches a BOM
    errors = []
    if not _LLMS_LH_H1.search(content):
        errors.append('missing an H1 header ("# Title")')
    if not _llms_has_markdown_link(content):
        errors.append("contains no Markdown links")
    if len(content) < 50:
        errors.append("shorter than 50 characters")
    notes = []
    lines = [ln for ln in content.splitlines() if ln.strip()]
    if lines and not lines[0].lstrip().startswith("# "):
        notes.append("llmstxt.org expects the H1 to be the first line")
    if not _LLMS_LH_SUMMARY.search(content):
        notes.append("no blockquote summary ('> ...') as llmstxt.org recommends")
    if not _LLMS_LH_H2.search(content):
        notes.append("no H2 sections grouping the links")
    if re.search(r"^\s*<(!doctype|html)", content, re.I):
        notes.append("served HTML, not Markdown (likely a soft 404); not a Lighthouse rule")
    return {"lighthouse": "pass" if not errors else "fail", "errors": errors, "notes": notes}


def audit_llms(root: str) -> tuple[list, dict]:
    rec = fetch(root + "/llms.txt")
    result = evaluate_llms_txt(rec["status"], rec["text"] if rec["status"] else None,
                               base_url=rec.get("final_url") or root + "/llms.txt",
                               content_type=_ctype(rec))
    full = fetch(root + "/llms-full.txt")
    data = {"status": rec["status"], "content_type": _ctype(rec), "bytes": len(rec["text"]),
            "llms_full_status": full["status"], **result}
    status = {"pass": "pass", "fail": "fail", "not-applicable": "info"}[result["lighthouse"]]
    fix = None
    if result["lighthouse"] == "not-applicable":
        fix = ("No /llms.txt. Lighthouse drops the audit from the fraction; publishing a "
               "valid one adds a counted pass. Include an H1, a '>' summary and Markdown links.")
    elif result["lighthouse"] == "fail" and any("soft 404" in n for n in result["notes"]):
        fix = ("The host answers /llms.txt with an HTML page and status 200 (a catch-all). "
               "Lighthouse counts that as a failed audit. Return a real 404 (the audit "
               "becomes N/A) or publish a real llms.txt.")
    elif result["lighthouse"] == "fail":
        fix = "Fix: " + "; ".join(result["errors"])
    checks = [_check("llms-txt", "llms.txt follows the Lighthouse rules", "P1", status,
                     "community spec (llmstxt.org); Lighthouse checks it, Google Search "
                     "ignores it", data, fix)]
    return checks, data


# --------------------------------------------------------------------------- HTML page


def _soup(html: str):
    from bs4 import BeautifulSoup
    return BeautifulSoup(html, "lxml")


def visible_words(html: str) -> int:
    soup = _soup(html)
    for tag in soup(["script", "style", "noscript", "template", "svg"]):
        tag.decompose()
    return len(soup.get_text(" ", strip=True).split())


# The rule, kept for importers. Do not run it on fetched HTML: it is quadratic
# (every ``<div`` rescans to the next ``>``, and so does every ``id=`` inside
# one tag). ``js_shell_marker`` gives the same answer in one pass.
JS_SHELL = re.compile(
    r'<div[^>]+id=["\'](root|app|__next|__nuxt)["\'][^>]*>\s*</div>|'
    r"enable javascript|you need to enable javascript", re.I)
_SHELL_PHRASE = re.compile(r"enable javascript", re.I)
_SHELL_DIV = re.compile(r"<div", re.I)
_SHELL_ID = re.compile(r'id=["\'](root|app|__next|__nuxt)["\']', re.I)
_SHELL_EMPTY = re.compile(r"\s*</div>", re.I)


def js_shell_marker(html: str) -> bool:
    """Same answer as ``JS_SHELL.search(html)``, in linear time.

    A match is a ``<div``, the first ``>`` after it, a shell id between the
    two and only whitespace before ``</div>``. Every ``<div`` before the same
    ``>`` shares that tag end and the first one has the most room for the id,
    so each tag end is looked at once.
    """
    if _SHELL_PHRASE.search(html):
        return True
    pos = 0
    while True:
        div = _SHELL_DIV.search(html, pos)
        if not div:
            return False
        close = html.find(">", div.end())
        if close < 0:
            return False
        if (_SHELL_EMPTY.match(html, close + 1)
                and _SHELL_ID.search(html, div.end() + 1, close)):
            return True
        pos = close + 1


def audit_page(url: str) -> tuple[list, dict]:
    rec = fetch(url)
    checks, data = [], {"status": rec["status"], "error": rec["error"]}
    if rec["error"] or not rec["status"] or rec["status"] >= 400:
        checks.append(_check("page-reachable", "Page reachable for a plain HTTP client",
                             "P0", "fail", "HTTP", {"status": rec["status"],
                                                   "error": rec["error"]},
                             "Agents and fetchers that do not run a browser cannot read it."))
        return checks, data
    html = rec["text"]
    words = visible_words(html)
    shell = js_shell_marker(html)
    data.update({"words_without_js": words, "js_shell_marker": shell,
                 "final_url": rec["final_url"], "headers": {
                     k: rec["headers"].get(k) for k in ("link", "vary", "content-signal",
                                                        "x-robots-tag") if rec["headers"].get(k)}})
    status = "fail" if words < 50 and shell else "warn" if words < 150 else "pass"
    checks.append(_check(
        "server-rendered", "Primary content present without JavaScript", "P0", status,
        "practice (all fetchers)", {"words_without_js": words, "js_shell_marker": shell},
        None if status == "pass" else
        "Server-render or pre-render the main content. Several agent fetchers do not "
        "execute complex JavaScript. Compare with render_page.py --mode always."))

    soup = _soup(html)
    data["soup_links"] = _link_signals(soup, rec["headers"].get("link", ""), rec["final_url"])
    checks += _webmcp_markup(soup, rec["final_url"], data)
    return checks, data


def _link_signals(soup, link_header: str, base: str) -> dict:
    md_alt = [urljoin(base, tag.get("href", "")) for tag in soup.find_all("link")
              if "alternate" in [r.lower() for r in (tag.get("rel") or [])]
              and "text/markdown" in (tag.get("type") or "").lower() and tag.get("href")]
    ai_catalog = [urljoin(base, tag.get("href", "")) for tag in soup.find_all("link")
                  if "ai-catalog" in [r.lower() for r in (tag.get("rel") or [])]
                  and tag.get("href")]
    header_links = parse_link_header(link_header)
    for entry in header_links:
        rels = entry["rel"]
        if "alternate" in rels and "text/markdown" in entry.get("type", ""):
            md_alt.append(urljoin(base, entry["uri"]))
        if "ai-catalog" in rels:
            ai_catalog.append(urljoin(base, entry["uri"]))
    # Built once: rebuilding it for every catalog link was quadratic.
    header_urls = {urljoin(base, e["uri"]) for e in header_links} if ai_catalog else set()
    return {"markdown_alternates": sorted(set(md_alt)), "ai_catalog_links": ai_catalog,
            "ai_catalog_html": [u for u in ai_catalog if u not in header_urls]}


def parse_link_header(value: str) -> list:
    """Minimal RFC 8288 parser: ``<uri>; rel="a b"; type="x"``.

    The URI may not hold a ``<``. A URI never does, and allowing one made
    every ``<`` in a header without a ``>`` rescan the rest of it.
    """
    out = []
    for match in re.finditer(r"<([^<>]*)>([^,<]*)", value or ""):
        params = {}
        for p in re.finditer(r';\s*([a-zA-Z*-]+)\s*=\s*("([^"]*)"|[^;,\s]+)', match.group(2)):
            params[p.group(1).lower()] = p.group(3) if p.group(3) is not None else p.group(2)
        out.append({"uri": match.group(1),
                    "rel": [r.lower() for r in params.get("rel", "").split()],
                    "type": params.get("type", "").lower()})
    return out


def _webmcp_markup(soup, base: str, data: dict) -> list:
    forms = soup.find_all("form")
    annotated = [f for f in forms if f.get("toolname") or f.get("tooldescription")]
    unannotated = [(f.get("id") or f.get("name") or f.get("action") or "form") for f in forms
                   if not (f.get("toolname") or f.get("tooldescription"))]
    sources = [s.get_text() for s in soup.find_all("script") if not s.get("src")]
    origin = urlparse(base).netloc
    fetched = 0
    for tag in soup.find_all("script", src=True):
        if fetched >= MAX_SCRIPTS:
            break
        src = urljoin(base, tag["src"])
        if urlparse(src).netloc != origin:
            continue
        rec = fetch(src)
        fetched += 1
        if rec["status"] == 200:
            sources.append(rec["text"])
    blob = "\n".join(sources)
    registers = len(re.findall(r"\bregisterTool\s*\(", blob))
    uses_document = "document.modelContext" in blob
    uses_navigator = "navigator.modelContext" in blob
    data["webmcp"] = {"forms": len(forms), "forms_annotated": len(annotated),
                      "forms_unannotated": unannotated[:20], "registerTool_call_sites": registers,
                      "document_modelContext": uses_document,
                      "navigator_modelContext": uses_navigator,
                      "same_origin_scripts_scanned": fetched}
    checks = []
    has_tools = registers > 0 or annotated
    checks.append(_check(
        "webmcp-tools", "WebMCP tools for forms and transactions", "P2",
        "pass" if has_tools else "info",
        f"W3C Community Group draft, not a standard; WebKit opposes, Mozilla neutral "
        f"(checked {CHECKED_ON})",
        {"registerTool_call_sites": registers, "forms": len(forms),
         "forms_annotated": len(annotated),
         "note": "Static count of registerTool( call sites in inline and same-origin "
                 "scripts, not the number of tools (a loop registers many). Use "
                 "lighthouse_agentic.py webmcp-registered-tools for the real list."},
        None if has_tools else
        "Optional. If the site has search, booking, checkout or lead forms, register "
        "imperative tools bound to the same handlers the UI uses. Static scan only; "
        "confirm at runtime with lighthouse_agentic.py (webmcp-registered-tools)."))
    legacy_only = uses_navigator and not uses_document
    checks.append(_check(
        "webmcp-entry-point", "WebMCP registered on the current entry point", "P2",
        "warn" if legacy_only else ("pass" if uses_document else "na"),
        "W3C Community Group draft",
        {"navigator_modelContext": uses_navigator, "document_modelContext": uses_document},
        "Feature-detect both: 'const mc = document.modelContext ?? "
        "navigator.modelContext'. Check the current spec before shipping."
        if legacy_only else None))
    if not forms:
        form_status = "na"
    elif not unannotated:
        form_status = "pass"
    else:
        form_status = "info"
    checks.append(_check(
        "webmcp-form-annotations", "Forms annotated for declarative WebMCP (static scan)", "P3",
        form_status, "W3C Community Group draft (Chrome only)",
        {"forms": len(forms), "unannotated": unannotated[:20]},
        "Lighthouse's webmcp-form-coverage audit counts only when every form has toolname "
        "or tooldescription. Annotate only forms that are safe for an agent to submit."
        if form_status == "info" else None))
    return checks


# --------------------------------------------------------------------------- markdown


def audit_markdown(url: str, page: dict) -> tuple[list, dict]:
    neg = fetch(url, headers={"Accept": "text/markdown"})
    neg_ct = _ctype(neg)
    vary = (neg.get("headers") or {}).get("vary", "").lower()
    negotiated = neg["status"] == 200 and "text/markdown" in neg_ct
    data = {"negotiation": {"status": neg["status"], "content_type": neg_ct,
                            "vary_accept": "accept" in vary,
                            "x_markdown_tokens": (neg.get("headers") or {}).get("x-markdown-tokens"),
                            "content_signal_header": (neg.get("headers") or {}).get("content-signal")}}
    alternates = []
    for alt in (page.get("soup_links") or {}).get("markdown_alternates", [])[:3]:
        rec = fetch(alt)
        alternates.append({"url": alt, "status": rec["status"], "content_type": _ctype(rec),
                           "ok": rec["status"] == 200 and not _ctype(rec).startswith("text/html")})
    data["alternates"] = alternates
    parsed = urlparse(url)
    sibling = (f"{parsed.scheme}://{parsed.netloc}"
               + (parsed.path.rstrip("/") + ".md" if parsed.path.strip("/") else "/index.md"))
    sib = fetch(sibling)
    data["md_sibling"] = {"url": sibling, "status": sib["status"], "content_type": _ctype(sib),
                          "ok": sib["status"] == 200 and not _ctype(sib).startswith("text/html")}
    ok = negotiated or any(a["ok"] for a in alternates) or data["md_sibling"]["ok"]
    issues = []
    if negotiated and "accept" not in vary:
        issues.append("Markdown negotiation without 'Vary: Accept' lets caches serve "
                      "Markdown to browsers or HTML to agents.")
    status = "pass" if ok and not issues else ("warn" if ok else "info")
    checks = [_check(
        "markdown-delivery", "Markdown version of the page", "P1", status,
        "HTTP content negotiation (RFC 9110) + community practice; no consumer agent is "
        f"confirmed to request it (checked {CHECKED_ON})", data,
        "; ".join(issues) if issues else None if ok else
        "Optional. Offer a content-equivalent Markdown version via "
        "'Accept: text/markdown' (with 'Vary: Accept'), a rel=\"alternate\" "
        "type=\"text/markdown\" link, or a .md URL.")]
    return checks, data


# --------------------------------------------------------------------------- ARD


def validate_ai_catalog(raw: str) -> dict:
    """Port of the ARD semantic checks Lighthouse 13.5.0 runs (third-party/ard).

    The JSON Schema pass is not reproduced; ``lighthouse_agentic.py`` reports it.
    """
    errors, warnings = [], []
    if json_depth_exceeds(raw):
        return {"errors": [f"malformed JSON: nested deeper than {MAX_JSON_DEPTH} levels"],
                "warnings": [], "entries": 0}
    try:
        data = json.loads(raw)
    except (ValueError, RecursionError) as exc:  # RecursionError: absurdly deep nesting
        return {"errors": [f"malformed JSON: {type(exc).__name__}: {str(exc)[:200]}"],
                "warnings": [], "entries": 0}
    if not isinstance(data, dict):
        return {"errors": ["root must be a JSON object"], "warnings": [], "entries": 0}
    spec = data.get("specVersion")
    if not spec:
        errors.append("missing required 'specVersion'")
    elif spec != "1.0":
        warnings.append(f"unrecognised specVersion {spec!r}, expected '1.0'")
    entries = data.get("entries")
    if entries is None:
        errors.append("missing required 'entries' array")
        return {"errors": errors, "warnings": warnings, "entries": 0}
    if not isinstance(entries, list):
        errors.append("'entries' must be an array")
        return {"errors": errors, "warnings": warnings, "entries": 0}
    for idx, entry in enumerate(entries):
        if not isinstance(entry, dict):
            errors.append(f"entry #{idx} is not an object")
            continue
        label = entry.get("displayName") or entry.get("identifier") or f"entry #{idx}"
        ident = entry.get("identifier")
        if not ident:
            errors.append(f"{label}: missing 'identifier'")
        elif not ARD_URN.match(str(ident)):
            errors.append(f"{label}: identifier must match "
                          "'urn:air:<publisher>[:<namespace>]:<name>'")
        if not entry.get("displayName"):
            errors.append(f"{label}: missing 'displayName'")
        media = entry.get("type")
        if not media:
            errors.append(f"{label}: missing 'type'")
        elif not isinstance(media, str):
            errors.append(f"{label}: 'type' must be a string")
        elif media not in ARD_MEDIA_TYPES:
            warnings.append(f"{label}: non-standard type {media!r}")
        has_url, has_data = "url" in entry, "data" in entry
        if has_url == has_data:
            errors.append(f"{label}: provide exactly one of 'url' or 'data'")
        queries = entry.get("representativeQueries")
        if queries is None:
            warnings.append(f"{label}: add 2 to 5 'representativeQueries'")
        elif not isinstance(queries, list):
            errors.append(f"{label}: 'representativeQueries' must be an array")
        else:
            if not 2 <= len(queries) <= 5:
                warnings.append(f"{label}: {len(queries)} representativeQueries, 2 to 5 recommended")
            if any(not isinstance(q, str) for q in queries):
                errors.append(f"{label}: every representative query must be a string")
        trust = entry.get("trustManifest")
        if trust is not None:
            if not isinstance(trust, dict):
                errors.append(f"{label}: 'trustManifest' must be an object")
            elif not trust.get("identity"):
                errors.append(f"{label}: 'trustManifest' needs 'identity'")
    if "collections" in data:
        errors.append("top-level 'collections' was removed (ADR-0003); nest catalogs in entries")
    return {"errors": errors, "warnings": warnings, "entries": len(entries)}


def audit_ard(root: str, agentmap: list, page: dict) -> tuple[list, dict]:
    links = page.get("soup_links") or {}
    candidates = ([urljoin(root + "/", a) for a in agentmap]
                  + links.get("ai_catalog_links", [])
                  + [root + "/.well-known/ai-catalog.json"])
    signalled = bool(agentmap or links.get("ai_catalog_links"))
    url = candidates[0]
    rec = fetch(url)
    data = {"catalog_url": url, "signalled": signalled, "status": rec["status"],
            "content_type": _ctype(rec)}
    standard = (f"Agentic Resource Discovery (ARD) spec 1.0; checked by Lighthouse 13.5 "
                f"ard-schema (checked {CHECKED_ON})")
    if not signalled and rec["status"] == 200 and _ctype(rec).startswith("text/html"):
        data["soft_404"] = True
        return [_check("ard-catalog", "ai-catalog.json (Agentic Resource Discovery)", "P1",
                       "fail", standard, data,
                       "/.well-known/ai-catalog.json returns an HTML page with status 200 (a "
                       "catch-all). Lighthouse treats any 200 as a catalog and counts a "
                       "failed ard-schema audit. Return a real 404 for unknown paths.")], data
    if not signalled and rec["status"] != 200:
        return [_check("ard-catalog", "ai-catalog.json (Agentic Resource Discovery)", "P3", "na",
                       standard, data,
                       "Only publish one if you have agent resources to list (MCP server, "
                       "A2A agent, skills). Lighthouse treats its absence as not applicable.")], data
    if rec["status"] != 200:
        return [_check("ard-catalog", "ai-catalog.json (Agentic Resource Discovery)", "P1",
                       "fail", standard, data,
                       "A catalog is signalled but does not load. Lighthouse counts this "
                       "as a failure. Fix the URL or remove the signal.")], data
    result = validate_ai_catalog(rec["text"])
    data.update(result)
    status = "fail" if result["errors"] else "warn" if result["warnings"] else "pass"
    return [_check("ard-catalog", "ai-catalog.json (Agentic Resource Discovery)",
                   "P1" if status == "fail" else "P3", status, standard, data,
                   None if status == "pass" else "Fix: " + "; ".join(
                       (result["errors"] + result["warnings"])[:6]))], data


# --------------------------------------------------------------------------- well-known


def audit_well_known(root: str) -> tuple[list, dict]:
    checks, data = [], {}
    for path, label, standard, priority, want in WELL_KNOWN:
        rec = fetch(root + path)
        ctype = _ctype(rec)
        row = {"status": rec["status"], "content_type": ctype}
        if rec["status"] == 200 and ctype.startswith("text/html"):
            row["soft_404"] = True
            status, fix = "warn", ("Returns HTML with 200. Serve a real 404 or the JSON "
                                   "document; soft 404s mislead discovery clients.")
        elif rec["status"] == 200 and json_depth_exceeds(rec["text"]):
            row["valid_json"] = False
            status, fix = "fail", f"Document is nested deeper than {MAX_JSON_DEPTH} levels."
        elif rec["status"] == 200:
            try:
                json.loads(rec["text"])
                row["valid_json"] = True
                status = "pass" if want in ctype else "warn"
                fix = None if status == "pass" else f"Serve with a content type containing '{want}'."
            except (ValueError, RecursionError):
                row["valid_json"] = False
                status, fix = "fail", "Document is not valid JSON."
        else:
            status, fix = "na", f"Only needed if you operate the matching service ({label})."
        data[path] = row
        checks.append(_check(f"well-known:{path.split('/.well-known/')[1]}", label, priority,
                             status, standard, row, fix))
    return checks, data


# --------------------------------------------------------------------------- UA matrix


def _challenged(rec: dict) -> bool:
    headers = rec.get("headers") or {}
    if headers.get("cf-mitigated") == "challenge":
        return True
    if rec.get("status") in (401, 403, 429, 503):
        return True
    # Challenge interstitials are small pages; a marker inside a full content
    # page is not a challenge.
    text = rec.get("text") or ""
    return len(text) < 30000 and any(marker in text.lower() for marker in CHALLENGE_MARKERS)


def audit_ua_matrix(url: str) -> tuple[list, dict]:
    baseline = fetch(url, headers={"User-Agent": BROWSER_UA})
    rows = {"browser": {"status": baseline["status"], "challenged": _challenged(baseline),
                        "bytes": len(baseline["text"])}}
    for token, ua in UA_STRINGS.items():
        rec = fetch(url, headers={"User-Agent": ua})
        rows[token] = {"status": rec["status"], "challenged": _challenged(rec),
                       "bytes": len(rec["text"]), "error": rec["error"]}
    if rows["browser"]["challenged"] or not rows["browser"]["status"]:
        return [_check(
            "waf-ua-matrix", "Responses to AI agent user agents (unverified requests)", "P0",
            "info", "observation only; real agents are verified by IP or Web Bot Auth",
            {"rows": rows}, "Inconclusive: the browser baseline itself was challenged or "
            "failed, so agent rows cannot be compared. Check WAF logs instead.")], rows
    differing = [t for t, r in rows.items() if t != "browser"
                 and (r["status"] != rows["browser"]["status"] or r["challenged"])]
    status = "info" if differing else "pass"
    return [_check(
        "waf-ua-matrix", "Responses to AI agent user agents (unverified requests)", "P0",
        status, "observation only; real agents are verified by IP or Web Bot Auth",
        {"rows": rows, "differs_from_browser": differing},
        None if not differing else
        "Unverified requests using these tokens were treated differently. That is correct "
        "if your WAF verifies bots. Confirm in your WAF logs that verified bots and signed "
        "agents (Web Bot Auth) are allowed, and keep CAPTCHAs off content pages.")], rows


# --------------------------------------------------------------------------- driver


def audit_not_found(root: str) -> tuple[list, dict]:
    """Does an unknown URL return a real 404? Catch-all 200s break discovery files."""
    probe = f"{root}/claude-seo-404-probe-{os.urandom(4).hex()}"
    rec = fetch(probe)
    data = {"probe_url": probe, "status": rec["status"], "content_type": _ctype(rec)}
    catch_all = rec["status"] == 200
    data["catch_all_200"] = catch_all
    return [_check(
        "http-404", "Unknown URLs return a real 404", "P1",
        "warn" if catch_all else ("pass" if rec["status"] and rec["status"] >= 400 else "info"),
        "HTTP semantics (RFC 9110); Google soft-404 guidance", data,
        "Unknown paths return 200. Crawlers see soft 404s, and Lighthouse fails llms-txt and "
        "ard-schema on the catch-all page. Serve a real 404 for paths that do not exist."
        if catch_all else None)], data


def summarize(checks: list) -> dict:
    summary: dict = {}
    for c in checks:
        bucket = summary.setdefault(c["priority"], {})
        bucket[c["status"]] = bucket.get(c["status"], 0) + 1
    return dict(sorted(summary.items()))


def audit(url: str, *, ua_matrix: bool = False) -> dict:
    if "://" not in url:
        url = "https://" + url
    root = _site_root(url)
    report = {"url": url, "site_root": root,
              "checked_at": _dt.datetime.now(_dt.timezone.utc).isoformat(timespec="seconds"),
              "facts_checked_on": CHECKED_ON, "checks": [], "data": {}}
    page_checks, page = audit_page(url)
    robots_checks, robots = audit_robots(root)
    llms_checks, llms = audit_llms(root)
    md_checks, md = audit_markdown(url, page)
    ard_checks, ard = audit_ard(root, robots.get("agentmap", []), page)
    wk_checks, wk = audit_well_known(root)
    nf_checks, nf = audit_not_found(root)
    report["checks"] = (page_checks + robots_checks + llms_checks + md_checks + ard_checks
                        + wk_checks + nf_checks)
    page.pop("soup_links", None)
    report["data"] = {"page": page, "robots": robots, "llms": llms, "markdown": md,
                      "ard": ard, "well_known": wk, "not_found": nf}
    if ua_matrix:
        ua_checks, rows = audit_ua_matrix(url)
        report["checks"] += ua_checks
        report["data"]["ua_matrix"] = rows
    order = {"P0": 0, "P1": 1, "P2": 2, "P3": 3}
    rank = {"fail": 0, "warn": 1, "info": 2, "pass": 3, "na": 4}
    report["checks"].sort(key=lambda c: (order[c["priority"]], rank[c["status"]]))
    report["summary"] = summarize(report["checks"])
    return report


def _print_text(report: dict) -> None:
    print(f"Agent readiness: {report['url']}")
    for c in report["checks"]:
        print(f"  {c['priority']} [{c['status']:>4}] {c['title']}")
        if c.get("fix") and c["status"] in ("fail", "warn"):
            print(f"      -> {c['fix']}")
        format_notes = sum((c["evidence"].get("format_note_counts") or {}).values())
        if c["id"] == "llms-txt" and format_notes:
            print(f"      {format_notes} format note(s) against the llmstxt.org proposal "
                  "(format hygiene, not a ranking factor); see --json")
    print(f"Summary: {json.dumps(report['summary'])}")


def main() -> None:
    parser = argparse.ArgumentParser(description="Agent-readiness HTTP auditor")
    parser.add_argument("url", help="Page URL (the site root is derived from it)")
    parser.add_argument("--ua-matrix", action="store_true",
                        help="Also request the page with AI agent user agents "
                             "(use only on sites you control or are authorized to test)")
    parser.add_argument("--json", action="store_true", help="JSON output")
    args = parser.parse_args()
    report = audit(args.url, ua_matrix=args.ua_matrix)
    if args.json:
        print(json.dumps(report, indent=2, default=str))
    else:
        _print_text(report)
    failed_page = any(c["id"] == "page-reachable" for c in report["checks"])
    sys.exit(1 if failed_page else 0)


if __name__ == "__main__":
    main()