Skip to content

Universal SEO skill for Claude Code. 26 sub-skills + 19 sub-agents covering technical SEO, E-E-A-T, schema, GEO/AEO, agent readiness (Lighthouse Agentic Browsing, WebMCP, llms.txt), backlinks, local SEO, e-commerce, international SEO, Google APIs, and PDF/Excel reporting. 9 optional extensions for live SEO data.

fix(agentic,parse): linear-time parsing of hostile llms.txt, robots rules and Link headers; deep JSON-LD is invalid, not a crash

Agrici Danielaca4fa93cd65
diff --git a/scripts/agentic_check.py b/scripts/agentic_check.py
index fd30ffe..4555821 100644
--- a/scripts/agentic_check.py
+++ b/scripts/agentic_check.py
@@ -303,12 +303,33 @@ def select_group(parsed: dict, token: str) -> dict:
}
+_STAR_RUN = re.compile(r"\*{2,}")
+
+
def _pattern_matches(pattern: str, path: str) -> bool:
+ """RFC 9309 path match: ``*`` is any run of characters, a final ``$`` the end.
+
+ Uses ``str.find``, not a regex built from the rule. The rule is untrusted,
+ and one such as ``****x`` became ``^.*.*.*.*x``, which backtracks without
+ limit. ``*`` is the only wildcard, so placing each literal piece at its
+ first possible position is enough.
+ """
anchored = pattern.endswith("$")
body = pattern[:-1] if anchored else pattern
- regex = "^" + ".*".join(re.escape(part) for part in body.split("*"))
- regex += "$" if anchored else ""
- return re.match(regex, path) is not None
+ if "*" not in body:
+ return path == body if anchored else path.startswith(body)
+ first, *middle, last = _STAR_RUN.sub("*", body).split("*")
+ if not path.startswith(first):
+ return False
+ pos = len(first)
+ for piece in middle:
+ found = path.find(piece, pos)
+ if found < 0:
+ return False
+ pos = found + len(piece)
+ if anchored:
+ return len(path) - len(last) >= pos and path.endswith(last)
+ return path.find(last, pos) >= 0
def is_allowed(rules: list, path: str = "/") -> bool:
@@ -722,6 +743,38 @@ def evaluate_llms_txt(status: Optional[int], content: Optional[str], *,
return {**result, **fmt, "format_basis": LLMS_FORMAT_BASIS}
+# Lighthouse 13.5.0 tests the file with ``/^\s*#\s+.+/m``, ``/\[.+\]\(.+\)/``
+# and the like. Run as written they are quadratic: ``^\s*`` rescans the rest of
+# a run of blank lines from every line start (40,000 newlines took 1.8 s, a
+# 2 MB file over an hour), and ``\[.+\]\(.+\)`` rescans the line from every
+# ``[``. The patterns below give the same answer for every input in one pass.
+# ``^\s*X`` matches exactly when X follows only whitespace on its own line, so
+# the leading run is limited to that line (``[^\S\n]`` is ``\s`` minus ``\n``).
+_LLMS_LH_H1 = re.compile(r"^[^\S\n]*#\s+.+", re.M)
+_LLMS_LH_SUMMARY = re.compile(r"^[^\S\n]*>\s+\S", re.M)
+_LLMS_LH_H2 = re.compile(r"^[^\S\n]*##\s+\S", re.M)
+
+
+def _llms_has_markdown_link(content: str) -> bool:
+ """Same answer as ``re.search(r"\\[.+\\]\\(.+\\)", content)``, in linear time.
+
+ ``.`` stops at a newline, so a match lies within one line: a ``[``, at
+ least one character, ``](``, at least one character, ``)``. Taking the
+ first ``[`` and then the first ``](`` leaves the most room for the rest, so
+ one ``str.find`` each settles the line.
+ """
+ if "](" not in content:
+ return False
+ for line in content.split("\n"):
+ opening = line.find("[")
+ if opening < 0:
+ continue
+ middle = line.find("](", opening + 2)
+ if middle >= 0 and line.find(")", middle + 3) >= 0:
+ return True
+ return False
+
+
def _lighthouse_llms_txt(status: Optional[int], content: Optional[str]) -> dict:
if status is None:
return {"lighthouse": "fail", "errors": ["fetch failed"], "notes": []}
@@ -731,9 +784,9 @@ def _lighthouse_llms_txt(status: Optional[int], content: Optional[str]) -> dict:
return {"lighthouse": "not-applicable", "errors": [], "notes": [f"HTTP {status}"]}
content = (content or "").lstrip("\ufeff") # JavaScript's \s matches a BOM
errors = []
- if not re.search(r"^\s*#\s+.+", content, re.M):
+ if not _LLMS_LH_H1.search(content):
errors.append('missing an H1 header ("# Title")')
- if not re.search(r"\[.+\]\(.+\)", content):
+ if not _llms_has_markdown_link(content):
errors.append("contains no Markdown links")
if len(content) < 50:
errors.append("shorter than 50 characters")
@@ -741,9 +794,9 @@ def _lighthouse_llms_txt(status: Optional[int], content: Optional[str]) -> dict:
lines = [ln for ln in content.splitlines() if ln.strip()]
if lines and not lines[0].lstrip().startswith("# "):
notes.append("llmstxt.org expects the H1 to be the first line")
- if not re.search(r"^\s*>\s+\S", content, re.M):
+ if not _LLMS_LH_SUMMARY.search(content):
notes.append("no blockquote summary ('> ...') as llmstxt.org recommends")
- if not re.search(r"^\s*##\s+\S", content, re.M):
+ if not _LLMS_LH_H2.search(content):
notes.append("no H2 sections grouping the links")
if re.search(r"^\s*<(!doctype|html)", content, re.I):
notes.append("served HTML, not Markdown (likely a soft 404); not a Lighthouse rule")
@@ -790,9 +843,40 @@ def visible_words(html: str) -> int:
return len(soup.get_text(" ", strip=True).split())
+# The rule, kept for importers. Do not run it on fetched HTML: it is quadratic
+# (every ``<div`` rescans to the next ``>``, and so does every ``id=`` inside
+# one tag). ``js_shell_marker`` gives the same answer in one pass.
JS_SHELL = re.compile(
r'<div[^>]+id=["\'](root|app|__next|__nuxt)["\'][^>]*>\s*</div>|'
r"enable javascript|you need to enable javascript", re.I)
+_SHELL_PHRASE = re.compile(r"enable javascript", re.I)
+_SHELL_DIV = re.compile(r"<div", re.I)
+_SHELL_ID = re.compile(r'id=["\'](root|app|__next|__nuxt)["\']', re.I)
+_SHELL_EMPTY = re.compile(r"\s*</div>", re.I)
+
+
+def js_shell_marker(html: str) -> bool:
+ """Same answer as ``JS_SHELL.search(html)``, in linear time.
+
+ A match is a ``<div``, the first ``>`` after it, a shell id between the
+ two and only whitespace before ``</div>``. Every ``<div`` before the same
+ ``>`` shares that tag end and the first one has the most room for the id,
+ so each tag end is looked at once.
+ """
+ if _SHELL_PHRASE.search(html):
+ return True
+ pos = 0
+ while True:
+ div = _SHELL_DIV.search(html, pos)
+ if not div:
+ return False
+ close = html.find(">", div.end())
+ if close < 0:
+ return False
+ if (_SHELL_EMPTY.match(html, close + 1)
+ and _SHELL_ID.search(html, div.end() + 1, close)):
+ return True
+ pos = close + 1
def audit_page(url: str) -> tuple[list, dict]:
@@ -806,7 +890,7 @@ def audit_page(url: str) -> tuple[list, dict]:
return checks, data
html = rec["text"]
words = visible_words(html)
- shell = bool(JS_SHELL.search(html))
+ shell = js_shell_marker(html)
data.update({"words_without_js": words, "js_shell_marker": shell,
"final_url": rec["final_url"], "headers": {
k: rec["headers"].get(k) for k in ("link", "vary", "content-signal",
@@ -839,15 +923,20 @@ def _link_signals(soup, link_header: str, base: str) -> dict:
md_alt.append(urljoin(base, entry["uri"]))
if "ai-catalog" in rels:
ai_catalog.append(urljoin(base, entry["uri"]))
+ # Built once: rebuilding it for every catalog link was quadratic.
+ header_urls = {urljoin(base, e["uri"]) for e in header_links} if ai_catalog else set()
return {"markdown_alternates": sorted(set(md_alt)), "ai_catalog_links": ai_catalog,
- "ai_catalog_html": [u for u in ai_catalog if u not in
- [urljoin(base, e["uri"]) for e in header_links]]}
+ "ai_catalog_html": [u for u in ai_catalog if u not in header_urls]}
def parse_link_header(value: str) -> list:
- """Minimal RFC 8288 parser: ``<uri>; rel="a b"; type="x"``."""
+ """Minimal RFC 8288 parser: ``<uri>; rel="a b"; type="x"``.
+
+ The URI may not hold a ``<``. A URI never does, and allowing one made
+ every ``<`` in a header without a ``>`` rescan the rest of it.
+ """
out = []
- for match in re.finditer(r"<([^>]*)>([^,<]*)", value or ""):
+ for match in re.finditer(r"<([^<>]*)>([^,<]*)", value or ""):
params = {}
for p in re.finditer(r';\s*([a-zA-Z*-]+)\s*=\s*("([^"]*)"|[^;,\s]+)', match.group(2)):
params[p.group(1).lower()] = p.group(3) if p.group(3) is not None else p.group(2)
diff --git a/scripts/parse_html.py b/scripts/parse_html.py
index c5fa160..b444baf 100755
--- a/scripts/parse_html.py
+++ b/scripts/parse_html.py
@@ -29,7 +29,10 @@ Output keys (JSON):
every other subdomain is external
schema parsed JSON-LD entities, @graph flattened
schema_invalid {count, samples}: JSON-LD blocks that could not be
- used, with the first 80 characters of up to 10
+ used, with the first 80 characters of up to 10.
+ A block nested deeper than 64 levels is invalid too
+ and adds too_deep: {count, reason}, a key that is
+ only present when there is such a block
open_graph, twitter_card, hreflang, word_count
hidden_text {chars, by_source, collapsed_chars, samples}: text
that is in the HTML but not shown (see below)
@@ -79,7 +82,12 @@ _SCRIPTS_DIR = os.path.dirname(os.path.abspath(__file__))
if _SCRIPTS_DIR not in sys.path:
sys.path.insert(0, _SCRIPTS_DIR)
import requests # noqa: E402 (url_safety already hard-requires it)
-from url_safety import URLSafetyError, decode_response_text, safe_requests_get # noqa: E402, I001
+from url_safety import ( # noqa: E402, I001
+ URLSafetyError,
+ decode_response_text,
+ json_depth_exceeds,
+ safe_requests_get,
+)
# Lazy-loader detection: covers native + the major JS lazy-loaders found on
# WordPress/WooCommerce sites (Perfmatters, EWWW Image Optimizer, generic
@@ -115,6 +123,14 @@ _MAX_OUTLINE_HEADINGS = 200
_MAX_HEADING_CHARS = 300
_MAX_INVALID_SCHEMA_SAMPLES = 10
_INVALID_SCHEMA_SAMPLE_CHARS = 80
+# Deepest nesting of arrays and objects a JSON-LD block may have. Real markup
+# stays under about ten levels. Python's JSON encoder recurses once per level,
+# so a block nested past the interpreter's recursion limit (1,000 by default)
+# parsed fine and then raised RecursionError when the result was printed. A
+# deeper block is counted in `schema_invalid` and never reaches `schema`, which
+# keeps every consumer of `schema` (this script's JSON output and the scripts
+# that import it) clear of the limit.
+_MAX_JSON_LD_DEPTH = 64
_MAX_HIDDEN_SAMPLES = 5
_HIDDEN_SAMPLE_CHARS = 200
@@ -800,7 +816,13 @@ def parse_html(html: str, base_url: Optional[str] = None) -> dict:
for script in soup.find_all("script"):
if not _is_json_ld(script):
continue
+ too_deep = False
try:
+ # A linear scan of the text, with no recursion, before anything
+ # is built from it.
+ too_deep = json_depth_exceeds(script.string or "", _MAX_JSON_LD_DEPTH)
+ if too_deep:
+ raise ValueError("JSON-LD nested too deep")
schema_data = json.loads(script.string)
# Flatten @graph containers so each @type is a separate entry.
# The container is not an entity, so it carries no @type of its
@@ -816,12 +838,16 @@ def parse_html(html: str, base_url: Optional[str] = None) -> dict:
result["schema"].append(item)
else:
result["schema"].append(schema_data)
- except (ValueError, TypeError, RecursionError):
+ except (ValueError, TypeError, RecursionError) as exc:
# Not valid JSON, empty, nested too deep, or a @graph that cannot
# be walked. Count the block and keep a short sample instead of
- # dropping it without a trace.
+ # dropping it without a trace. RecursionError is the backstop for
+ # the depth scan above.
invalid = result["schema_invalid"]
invalid["count"] += 1
+ if too_deep or isinstance(exc, RecursionError):
+ reason = f"nested deeper than {_MAX_JSON_LD_DEPTH} levels"
+ invalid.setdefault("too_deep", {"count": 0, "reason": reason})["count"] += 1
if len(invalid["samples"]) < _MAX_INVALID_SCHEMA_SAMPLES:
raw = str(script.string or "")[: _INVALID_SCHEMA_SAMPLE_CHARS * 5]
invalid["samples"].append(" ".join(raw.split())[:_INVALID_SCHEMA_SAMPLE_CHARS])
diff --git a/tests/test_agentic.py b/tests/test_agentic.py
index 1b719bb..dc84a22 100644
--- a/tests/test_agentic.py
+++ b/tests/test_agentic.py
@@ -15,7 +15,10 @@ from __future__ import annotations
import copy
import json
import os
+import random
+import re
import sys
+import time
from pathlib import Path
from unittest.mock import patch
@@ -644,3 +647,100 @@ def test_fix_robots_refuses_to_draft_from_nothing_on_429(monkeypatch):
with pytest.raises(SystemExit) as exc:
af.main()
assert "Could not read robots.txt" in str(exc.value)
+
+
+# --------------------------------------------------------------------------- hostile input
+#
+# Fetched text is hostile and one body is at most ac.MAX_BODY (2,000,000 bytes).
+# Each case below was quadratic or worse before the parser was made linear.
+
+HOSTILE_CHARS = 2_000_000
+# Headers are not bodies: http.client stops a header line at 65,536 bytes, and
+# repeated Link headers are joined, so three full lines stand in for a long one.
+HOSTILE_HEADER_CHARS = 200_000
+
+
+def _seconds(fn, *args) -> float:
+ started = time.monotonic()
+ fn(*args)
+ return time.monotonic() - started
+
+
+def _repeat(unit: str, chars: int = HOSTILE_CHARS) -> str:
+ return unit * (chars // len(unit))
+
+
[email protected]("pattern", [
+ _repeat("*") + "x", "/" + _repeat("*") + "x$", _repeat("*a"), _repeat("*/") + "x",
+ _repeat("/*") + "$", _repeat("a"), _repeat("$"),
+])
+def test_a_hostile_robots_wildcard_rule_is_matched_in_linear_time(pattern):
+ """The rule used to become a regex such as ``^.*.*.*x``."""
+ assert _seconds(ac.is_allowed, [("disallow", pattern)], "/" * 64) < 2
+ robots = f"User-agent: *\nDisallow: {pattern}\nAllow: {pattern}\n"
+ with patch.object(ac, "fetch", return_value=_rec(200, robots)):
+ assert _seconds(ac.audit_robots, "https://example.com") < 2
+
+
+def _robots_regex_reference(pattern: str, path: str) -> bool:
+ """The wildcard match as first written: a regex built from the rule."""
+ anchored = pattern.endswith("$")
+ body = pattern[:-1] if anchored else pattern
+ regex = "^" + ".*".join(re.escape(part) for part in body.split("*"))
+ regex += "$" if anchored else ""
+ return re.match(regex, path) is not None
+
+
+def test_robots_wildcard_match_agrees_with_the_original_regex():
+ rng = random.Random(260)
+ for _ in range(40_000):
+ pattern = "".join(rng.choice("/ab*$.") for _ in range(rng.randint(0, 7)))
+ path = "".join(rng.choice("/ab.$") for _ in range(rng.randint(0, 9)))
+ assert ac._pattern_matches(pattern, path) is _robots_regex_reference(pattern, path), (
+ pattern, path)
+
+
+ "<div ", "<div", '<div id="root" ', "<div>", "></div>", '<div a id="app">\n',
+ 'id="root"', "<div x> ",
+])
+def test_a_hostile_page_is_checked_for_a_js_shell_in_linear_time(unit):
+ for html in (_repeat(unit), _repeat(unit) + ">", '<div id="root" ' + _repeat(unit) + "></span>"):
+ assert _seconds(ac.js_shell_marker, html) < 2
+ assert ac.js_shell_marker('<div id="root" ' + _repeat("<div ") + "> </div>")
+ assert not ac.js_shell_marker('<div id="root" ' + _repeat("<div ") + "> x</div>")
+
+
+def test_js_shell_marker_agrees_with_the_original_regex():
+ pieces = ["<div", "<DIV", " ", "\n", ">", "</div>", "</DIV>", 'id="root"', "id='app'",
+ 'id="__next"', 'ID="__NUXT"', 'id="roots"', 'data-id="app"', "x", "<", "<p>",
+ "Enable JavaScript", "you need to enable javascript", 'class="a"', "<divider"]
+ rng = random.Random(260)
+ for _ in range(40_000):
+ html = "".join(rng.choice(pieces) for _ in range(rng.randint(0, 10)))
+ assert ac.js_shell_marker(html) is bool(ac.JS_SHELL.search(html)), html
+ assert ac.js_shell_marker('<div class="app" id="root" data-x="1">\n </div>')
+ assert not ac.js_shell_marker('<div id="root"><p>Rendered</p></div>')
+
+
[email protected]("unit", ["<", "<a", "<a>", "<a>;", '<a>; rel="', "<a>;a ", ">", ";a="])
+def test_a_hostile_link_header_is_parsed_in_linear_time(unit):
+ header = _repeat(unit, HOSTILE_HEADER_CHARS)
+ assert _seconds(ac.parse_link_header, header) < 2
+ assert _seconds(ac.parse_link_header, '</c.json>; rel="ai-catalog", ' + header) < 2
+
+
+def test_many_catalog_links_do_not_stall_the_link_signals():
+ """Each catalog link used to rebuild the list of every header link."""
+ count = 5_000
+ header = ", ".join(f'<https://e.test/c{n}.json>; rel="ai-catalog"' for n in range(count))
+ html = "".join(f'<link rel="ai-catalog" href="/h{n}.json">' for n in range(count))
+ started = time.monotonic()
+ signals = ac._link_signals(ac._soup(html), header, "https://e.test/")
+ assert time.monotonic() - started < 2
+ assert len(signals["ai_catalog_links"]) == 2 * count
+ assert signals["ai_catalog_html"] == [f"https://e.test/h{n}.json" for n in range(count)]
+
+
+def test_the_hostile_cases_are_full_size():
+ assert HOSTILE_CHARS >= ac.MAX_BODY
diff --git a/tests/test_llms_txt_validation.py b/tests/test_llms_txt_validation.py
index 4a7b3db..c4029e7 100644
--- a/tests/test_llms_txt_validation.py
+++ b/tests/test_llms_txt_validation.py
@@ -6,6 +6,8 @@ Lighthouse-style headline verdict. No live network: fetches are patched.
from __future__ import annotations
+import random
+import re
import sys
import time
from pathlib import Path
@@ -246,6 +248,76 @@ def test_hostile_lines_do_not_crash_or_stall():
assert all(len(note.get("detail", "")) <= 80 for note in result["format_notes"])
+# One fetched file is at most agentic_check.MAX_BODY (2,000,000 bytes). The
+# Lighthouse-style regexes used to be quadratic: 40,000 blank lines took 1.8 s,
+# so a full-size file of them would have run for over an hour.
+HOSTILE_CHARS = 2_000_000
+HOSTILE_FILES = [
+ ("", "\n"), ("", "#"), ("", "- ["), ("", " "),
+ ("", "["), ("", "\r"), ("", " \n"), ("", "\t\n#"), ("", "[x]("), ("", "](x"),
+ ("#", "\n"), (">", "\n"), ("##", "\n"), ("# ", " "), ("# T\n\n", "["),
+ ("# T\n\n> S\n\n## Docs\n", "- ["), ("# T\n## Docs\n", "- [x]("),
+ ("# T\n## Docs\n- [t](", "(a"), ("# T\n## Docs\n", "- [t](u \""),
+ ("# T\n## Docs\n", "\n"), ("# T\n", "## S\n"), ("# T\n## Docs\n", "- x\n"),
+]
+
+
[email protected](("prefix", "unit"), HOSTILE_FILES)
+def test_a_full_size_hostile_file_is_read_in_linear_time(prefix, unit):
+ text = prefix + unit * (HOSTILE_CHARS // len(unit))
+
+ started = time.monotonic()
+ verdict = ac._lighthouse_llms_txt(200, text)
+ verdict_seconds = time.monotonic() - started
+ started = time.monotonic()
+ fmt = ac.llms_format_notes(text)
+ notes_seconds = time.monotonic() - started
+
+ assert verdict_seconds < 2, f"Lighthouse-style verdict took {verdict_seconds:.1f} s"
+ assert notes_seconds < 2, f"llms_format_notes took {notes_seconds:.1f} s"
+ assert verdict["lighthouse"] == "fail"
+ assert len(fmt["format_notes"]) <= len(ac.LLMS_FORMAT_RULES) * ac.LLMS_NOTES_PER_ID
+ assert HOSTILE_CHARS >= ac.MAX_BODY
+
+
+def _lighthouse_reference(content):
+ """The verdict as first written, with the Lighthouse 13.5.0 regexes verbatim.
+
+ Quadratic on hostile input, so only ever given short strings here.
+ """
+ errors = []
+ if not re.search(r"^\s*#\s+.+", content, re.M):
+ errors.append('missing an H1 header ("# Title")')
+ if not re.search(r"\[.+\]\(.+\)", content):
+ errors.append("contains no Markdown links")
+ if len(content) < 50:
+ errors.append("shorter than 50 characters")
+ notes_ = []
+ lines = [ln for ln in content.splitlines() if ln.strip()]
+ if lines and not lines[0].lstrip().startswith("# "):
+ notes_.append("llmstxt.org expects the H1 to be the first line")
+ if not re.search(r"^\s*>\s+\S", content, re.M):
+ notes_.append("no blockquote summary ('> ...') as llmstxt.org recommends")
+ if not re.search(r"^\s*##\s+\S", content, re.M):
+ notes_.append("no H2 sections grouping the links")
+ if re.search(r"^\s*<(!doctype|html)", content, re.I):
+ notes_.append("served HTML, not Markdown (likely a soft 404); not a Lighthouse rule")
+ return {"lighthouse": "pass" if not errors else "fail", "errors": errors, "notes": notes_}
+
+
+def test_linear_verdict_agrees_with_the_original_regexes_on_random_files():
+ pieces = ["#", "##", " ", " ", "\n", "\n\n", "\r", "\r\n", "\t", "\x0b", "\x0c", "\x1c",
+ "\x85", "\xa0", LINE_SEPARATOR, "[", "]", "(", ")", "](", ">", "<", "a", "Title",
+ "<!DOCTYPE", "<html", "- ", "x" * 50]
+ rng = random.Random(260)
+ for _ in range(30_000):
+ text = "".join(rng.choice(pieces) for _ in range(rng.randint(0, 14)))
+ assert ac._lighthouse_llms_txt(200, text) == _lighthouse_reference(text), repr(text)
+ for text in (SPEC_SHAPED, MALFORMED, "", "#", "# ", "# ", "# \n", "#\n\nx", " # x", "x # y",
+ "[a](b)", "[](b)", "[a]()", "[a]\n(b)", "[a](b\n)", "x[[a](b))", "\n\n<html>"):
+ assert ac._lighthouse_llms_txt(200, text) == _lighthouse_reference(text), repr(text)
+
+
def test_every_rule_has_a_level_and_a_plain_message():
for note_id, (level, message) in ac.LLMS_FORMAT_RULES.items():
assert level in ("error", "warn", "info"), note_id
diff --git a/tests/test_parse_html_extraction.py b/tests/test_parse_html_extraction.py
index 8f6b8c1..39a5f8a 100644
--- a/tests/test_parse_html_extraction.py
+++ b/tests/test_parse_html_extraction.py
@@ -344,6 +344,86 @@ def test_deeply_nested_json_ld_does_not_abort_the_parse():
assert result["schema_invalid"]["samples"] == ["[" * 80]
+def _nested(depth: int, kind: str) -> str:
+ """Valid JSON nested ``depth`` containers deep."""
+ if kind == "objects":
+ return '{"a":' * depth + "1" + "}" * depth
+ if kind == "arrays":
+ return "[" * depth + "]" * depth
+ return '[{"a":' * (depth // 2) + "1" + "}]" * (depth // 2)
+
+
[email protected]("depth", [1_500, 100_000])
[email protected]("kind", ["objects", "arrays", "mixed"])
+def test_json_ld_nested_too_deep_is_invalid_and_the_result_still_serialises(depth, kind):
+ """Depth 1,500 used to parse and then raise RecursionError in json.dumps."""
+ block = _nested(depth, kind)
+ html = (
+ f'<h1>Title</h1><script type="application/ld+json">{ORG}</script>'
+ f'<script type="application/ld+json">{block}</script>'
+ )
+ result = parse_html(html)
+
+ assert result["h1"] == ["Title"]
+ assert result["schema"] == [json.loads(ORG)]
+ assert result["schema_invalid"] == {
+ "count": 1,
+ "samples": [block[:80]],
+ "too_deep": {"count": 1, "reason": "nested deeper than 64 levels"},
+ }
+ json.dumps(result)
+ json.dumps(result, indent=2)
+
+
+def test_json_ld_depth_limit_is_64_levels_and_normal_markup_is_untouched():
+ limit = parse_html_module._MAX_JSON_LD_DEPTH
+ assert limit == 64
+
+ ten_deep = parse_html(f'<script type="application/ld+json">{_nested(10, "objects")}</script>')
+ assert ten_deep["schema"] == [json.loads(_nested(10, "objects"))]
+ assert ten_deep["schema_invalid"] == {"count": 0, "samples": []}
+
+ at_limit = parse_html(f'<script type="application/ld+json">{_nested(limit, "objects")}</script>')
+ assert len(at_limit["schema"]) == 1 and at_limit["schema_invalid"]["count"] == 0
+ json.dumps(at_limit, indent=2)
+
+ over = parse_html(f'<script type="application/ld+json">{_nested(limit + 1, "objects")}</script>')
+ assert over["schema"] == []
+ assert over["schema_invalid"]["too_deep"]["count"] == 1
+
+
+def test_json_ld_depth_ignores_brackets_inside_strings_and_counts_graph_entries():
+ quoted = json.dumps({"@type": "Thing", "name": "[{" * 500, "note": 'a "[[[" b'})
+ in_string = parse_html(f'<script type="application/ld+json">{quoted}</script>')
+ assert in_string["schema"] == [json.loads(quoted)]
+ assert in_string["schema_invalid"] == {"count": 0, "samples": []}
+
+ deep_entry = _nested(1_500, "arrays")
+ graph = '{"@context": "https://schema.org", "@graph": [{"@type": "Thing", "x": %s}]}' % deep_entry
+ in_graph = parse_html(f'<script type="application/ld+json">{graph}</script>')
+ assert in_graph["schema"] == []
+ assert in_graph["schema_invalid"]["count"] == 1
+ assert in_graph["schema_invalid"]["too_deep"]["count"] == 1
+ json.dumps(in_graph, indent=2)
+
+
+def test_cli_json_output_survives_a_deep_json_ld_block(tmp_path, monkeypatch, capsys):
+ """The crash site: ``print(json.dumps(result, indent=2))`` in ``main``."""
+ page = tmp_path / "page.html"
+ page.write_text(
+ "<title>T</title>"
+ f'<script type="application/ld+json">{_nested(1_500, "objects")}</script>',
+ encoding="utf-8",
+ )
+ monkeypatch.setattr(sys, "argv", ["parse_html.py", str(page), "--json"])
+ parse_html_module.main()
+
+ printed = json.loads(capsys.readouterr().out)
+ assert printed["title"] == "T"
+ assert printed["schema"] == []
+ assert printed["schema_invalid"]["count"] == 1
+
+
# ---------------------------------------------------------------------------
# 4. Heading outline
# ---------------------------------------------------------------------------