Skip to content

Universal SEO skill for Claude Code. 26 sub-skills + 19 sub-agents covering technical SEO, E-E-A-T, schema, GEO/AEO, agent readiness (Lighthouse Agentic Browsing, WebMCP, llms.txt), backlinks, local SEO, e-commerce, international SEO, Google APIs, and PDF/Excel reporting. 9 optional extensions for live SEO data.

15.3 KB
Raw Blame
"""render_page.js_gap: what a page only has after JavaScript ran.

js_gap is a pure function over two HTML strings (raw as served, rendered
DOM), so every test here runs without a browser or the network. The first
test is ported from web-obs (scripts/verify-browser-audit.py), where the
comparison was first written.
"""

from __future__ import annotations

import json
import os
import sys

import pytest

_SCRIPTS = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "scripts")
if _SCRIPTS not in sys.path:
    sys.path.insert(0, _SCRIPTS)

pytest.importorskip("requests")
import render_page  # noqa: E402
from render_page import js_gap, js_gap_reasons  # noqa: E402

# Ceiling for the whole js_gap object as JSON when a page pushes every field to
# its limit on both sides. The limits add up to about 22,200 characters, most of
# it the 12 H1 texts of 300 characters kept for raw, rendered and the diff.
MAX_GAP_JSON_CHARS = 24_000

SHELL = "<html><head><title>App</title></head><body><p>one two</p></body></html>"
HYDRATED = (
    "<html><head><title>App</title>"
    '<script type="application/ld+json">{"@type": ["Organization", "Brand"]}</script></head>'
    "<body><h1>Title here</h1>"
    + "<p>word word word word word</p>" * 10
    + '<a href="/a">a</a>' * 6
    + "</body></html>"
)


def _page(head: str = "", body: str = "") -> str:
    filler = "<p>" + "steady server rendered copy " * 20 + "</p>"
    return f"<html><head>{head}</head><body>{filler}{body}</body></html>"


def test_js_gap_measures_words_links_schema_and_h1() -> None:
    gap = js_gap(SHELL, HYDRATED)
    assert gap["raw"]["words"] == 2
    assert gap["rendered"]["words"] == 58
    assert gap["rendered"]["links"] == 6
    assert gap["rendered"]["schema_types"] == ["Brand", "Organization"]
    diff = gap["diff"]
    assert diff["words_only_rendered"] == 58
    assert diff["words_only_rendered_share"] == 1.0
    assert diff["links_only_rendered"] == 6
    assert diff["links_only_rendered_share"] == 1.0
    assert diff["schema_types_only_rendered"] == ["Brand", "Organization"]
    assert diff["h1_only_rendered"] == ["Title here"]
    assert diff["title_changed"] is False
    assert gap["significant"] is True
    assert len(gap["reasons"]) == 4, gap["reasons"]
    assert gap["reasons"][0].startswith("100% of visible words appear only after JavaScript")
    assert gap["reasons"] == js_gap_reasons(gap)
    json.dumps(gap)  # the object goes straight into the CLI JSON


def test_identical_html_has_no_gap_and_no_reasons() -> None:
    gap = js_gap(HYDRATED, HYDRATED)
    assert gap["diff"]["words_only_rendered"] == 0
    assert gap["diff"]["words_only_rendered_share"] == 0.0
    assert gap["diff"]["links_only_rendered"] == 0
    assert gap["reasons"] == []
    assert gap["significant"] is False


def test_missing_side_means_no_render_and_returns_none() -> None:
    assert js_gap(None, HYDRATED) is None
    assert js_gap(SHELL, None) is None
    assert js_gap_reasons(None) == []


def test_empty_raw_body_is_compared_not_skipped() -> None:
    gap = js_gap("", HYDRATED)
    assert gap["raw"] == {
        "words": 0, "links": 0, "schema_types": [], "h1": [], "title": None,
        "canonical": None, "canonical_all": [], "meta_robots": None,
    }
    assert gap["diff"]["words_only_rendered_share"] == 1.0
    assert gap["diff"]["title_changed"] is True


def test_title_canonical_and_robots_changes_are_flagged() -> None:
    raw = _page(
        "<title>Server title</title>"
        '<link rel="canonical" href="https://shop.example/a">'
        '<meta name="robots" content="noindex">'
    )
    rendered = _page(
        "<title>Client title</title>"
        '<link rel="canonical" href="https://shop.example/b">'
        '<meta name="robots" content="index, follow">'
    )
    gap = js_gap(raw, rendered)
    assert gap["raw"]["canonical"] == "https://shop.example/a"
    assert gap["rendered"]["canonical"] == "https://shop.example/b"
    assert gap["raw"]["meta_robots"] == "noindex"
    assert gap["rendered"]["meta_robots"] == "index, follow"
    assert gap["diff"]["title_changed"] is True
    assert gap["diff"]["canonical_changed"] is True
    assert gap["diff"]["meta_robots_changed"] is True
    # Same words and links on both sides: only the three head tags differ.
    assert gap["diff"]["words_only_rendered"] == 0
    assert gap["significant"] is True
    assert gap["reasons"] == [
        "the title is not the same before and after JavaScript "
        "(raw: 'Server title', rendered: 'Client title')",
        "the canonical URL is not the same before and after JavaScript "
        "(raw: 'https://shop.example/a', rendered: 'https://shop.example/b')",
        "the meta robots value is not the same before and after JavaScript "
        "(raw: 'noindex', rendered: 'index, follow')",
    ]


def test_head_tags_added_only_by_javascript_count_as_changed() -> None:
    rendered = _page(
        "<title>Client title</title>"
        '<link rel="canonical" href="https://shop.example/b">'
        '<meta name="robots" content="noindex">'
    )
    gap = js_gap(_page(), rendered)
    assert gap["diff"]["title_changed"] is True
    assert gap["diff"]["canonical_changed"] is True
    assert gap["diff"]["meta_robots_changed"] is True
    assert "(raw: none, rendered: 'noindex')" in gap["reasons"][-1]


def test_second_canonical_injected_by_javascript_is_a_change() -> None:
    first = '<link rel="canonical" href="https://shop.example/a">'
    gap = js_gap(
        _page(first),
        _page(first + '<link href="https://shop.example/b" rel="alternate CANONICAL">'),
    )
    assert gap["rendered"]["canonical"] == "https://shop.example/a"
    assert gap["rendered"]["canonical_all"] == [
        "https://shop.example/a", "https://shop.example/b",
    ]
    assert gap["diff"]["canonical_changed"] is True
    assert "'https://shop.example/a | https://shop.example/b'" in gap["reasons"][0]


def test_robots_directive_order_case_and_split_tags_are_not_a_change() -> None:
    raw = _page('<meta name="ROBOTS" content="NOINDEX,nofollow">')
    rendered = _page(
        '<meta name="robots" content="nofollow"><meta name="robots" content=" noindex ">'
    )
    gap = js_gap(raw, rendered)
    assert gap["raw"]["meta_robots"] == "noindex, nofollow"
    assert gap["rendered"]["meta_robots"] == "nofollow, noindex"
    assert gap["diff"]["meta_robots_changed"] is False
    assert gap["reasons"] == []


def test_replaced_copy_counts_even_when_the_word_total_is_equal() -> None:
    raw = "<html><body><p>" + "loading " * 40 + "</p></body></html>"
    rendered = "<html><body><p>" + "product " * 40 + "</p></body></html>"
    gap = js_gap(raw, rendered)
    assert gap["raw"]["words"] == gap["rendered"]["words"] == 40
    assert gap["diff"]["words_only_rendered"] == 40
    assert gap["significant"] is True


def test_links_are_matched_by_href_and_non_links_are_ignored() -> None:
    raw = _page(body='<a href="/a">a</a><a href="/b">b</a>')
    rendered = _page(
        body='<a href="/a">a</a><a href="/b">b</a><a href="/c">c</a>'
        '<a href="#top">top</a><a href="javascript:void(0)">x</a>'
        '<a href="mailto:[email protected]">m</a><a>no href</a>'
    )
    gap = js_gap(raw, rendered)
    assert gap["raw"]["links"] == 2
    assert gap["rendered"]["links"] == 3
    assert gap["diff"]["links_only_rendered"] == 1
    assert gap["diff"]["links_only_rendered_share"] == 0.333


def test_significance_thresholds_are_the_named_constants(monkeypatch) -> None:
    raw = "<html><body><p>" + "base " * 100 + "</p></body></html>"
    rendered = (
        "<html><body><p>" + "base " * 100 + "extra " * 20 + "</p>"
        '<a href="/1">1</a><a href="/2">2</a></body></html>'
    )
    gap = js_gap(raw, rendered)
    # 22 of 122 words (18%) and 2 links: under every default limit.
    assert gap["diff"]["words_only_rendered"] == 22
    assert gap["reasons"] == []

    monkeypatch.setattr(render_page, "JS_GAP_WORDS_SHARE", 0.10)
    monkeypatch.setattr(render_page, "JS_GAP_WORDS_MIN", 10)
    monkeypatch.setattr(render_page, "JS_GAP_LINKS_MIN", 2)
    reasons = js_gap_reasons(gap)
    assert len(reasons) == 2
    assert reasons[0].startswith("18% of visible words")
    assert reasons[1].startswith("2 of 2 links appear only after JavaScript")


def test_a_few_new_words_on_a_tiny_page_are_not_significant() -> None:
    gap = js_gap(
        "<html><body><p>hello</p></body></html>",
        "<html><body><p>hello there dear visitor</p></body></html>",
    )
    assert gap["diff"]["words_only_rendered_share"] == 0.75
    assert gap["significant"] is False


def test_non_visible_markup_is_not_counted_as_words_links_or_title() -> None:
    html = (
        "<html><head><title>Real title</title><style>.a{color:red}</style></head><body>"
        "<svg><title>Icon name</title></svg>"
        "<script>var hidden = 'one two three';</script>"
        '<noscript><h1>No script</h1><a href="/ns">enable js</a></noscript>'
        '<template><h1>Tpl</h1><a href="/tpl">tpl</a></template>'
        "<h1>Seen</h1><p>four visible words here</p></body></html>"
    )
    signals = js_gap(html, html)["raw"]
    assert signals["title"] == "Real title"
    assert signals["h1"] == ["Seen"]
    assert signals["links"] == 0
    assert signals["words"] == 5


def test_svg_title_alone_is_not_a_document_title() -> None:
    signals = js_gap("<body><svg><title>Icon</title></svg></body>", "")["raw"]
    assert signals["title"] is None


def test_h1_change_and_removed_tags_are_reported() -> None:
    gap = js_gap(
        _page("<title>T</title>", "<h1>Old heading</h1>"),
        _page("", "<h1>New   heading</h1>"),
    )
    assert gap["diff"]["h1_only_rendered"] == ["New heading"]
    assert gap["reasons"] == [
        "the H1 changes after JavaScript",
        "the title is not the same before and after JavaScript (raw: 'T', rendered: none)",
    ]


def test_output_is_bounded_for_hostile_pages() -> None:
    hostile = (
        "<html><head><title>" + "t" * 5000 + "</title>"
        + "".join(f'<link rel="canonical" href="https://x.example/{i}">' for i in range(40))
        + '<meta name="robots" content="' + ",".join(f"d{i}" for i in range(500)) + '">'
        + '<script type="application/ld+json">['
        + ",".join('{"@type":"T%d"}' % i for i in range(60))
        + "]</script></head><body>"
        + "".join(f"<h1>Heading {i}</h1>" for i in range(60))
        + "</body></html>"
    )
    gap = js_gap("", hostile, raw_truncated=True)
    rendered = gap["rendered"]
    assert len(rendered["title"]) == render_page.JS_GAP_MAX_TEXT
    assert len(rendered["h1"]) == render_page.JS_GAP_MAX_ITEMS
    assert len(rendered["schema_types"]) == render_page.JS_GAP_MAX_ITEMS
    assert len(rendered["canonical_all"]) == render_page.JS_GAP_MAX_CANONICALS
    assert len(rendered["meta_robots"].split(", ")) == render_page.JS_GAP_MAX_ITEMS
    assert gap["raw_truncated"] is True
    assert all(len(reason) < 400 for reason in gap["reasons"])


def _hostile(tag: str) -> str:
    """A page that pushes every js_gap field to its limit; ``tag`` keeps two apart."""
    big = tag * 2_000
    return (
        f"<html><head><title>{big}</title>"
        + "".join(f'<link rel="canonical" href="https://x.example/{i}/{big}">' for i in range(40))
        + '<meta name="robots" content="' + ",".join(f"{i}{big}" for i in range(40)) + '">'
        + '<script type="application/ld+json">['
        + ",".join('{"@type":"%d%s"}' % (i, big) for i in range(60))
        + "]</script>"
        + "".join(
            f'<script type="application/ld+json">{{"@type": "{i}{tag * 200_000}"}}</script>'
            for i in range(3)
        )
        + "</head><body>"
        + "".join(f"<h1>{i} {big}</h1>" for i in range(40))
        + "".join(f'<a href="/{tag}/{i}/{big}">{tag}{i}</a>' for i in range(40))
        + "</body></html>"
    )


def test_huge_schema_type_names_cannot_inflate_the_output() -> None:
    # Found in review: five blocks with 200,000-character @type values made
    # 3 MB of js_gap, and --max-text does not cut js_gap.
    big = "A" * 200_000
    blocks = "".join(
        f'<script type="application/ld+json">{{"@type": "{big}{i}"}}</script>' for i in range(5)
    )
    gap = js_gap(
        "<html><body>x</body></html>", f"<html><head>{blocks}</head><body>x</body></html>"
    )
    names = gap["rendered"]["schema_types"]
    assert names and all(len(name) <= render_page.JS_GAP_MAX_NAME for name in names)
    assert gap["diff"]["schema_types_only_rendered"] == names
    assert all(len(reason) < 400 for reason in gap["reasons"])
    assert len(json.dumps(gap)) < 2_000


def test_schema_type_names_are_cleaned_like_the_other_page_text() -> None:
    block = '<script type="application/ld+json">{"@type": ["  Local\\n\\t Business ", " ", ""]}</script>'
    gap = js_gap(_page(), _page(block))
    assert gap["rendered"]["schema_types"] == ["Local Business"]
    assert gap["reasons"] == ["structured data appears only after JavaScript: 'Local Business'"]


def test_reason_lists_a_bounded_number_of_quoted_type_names() -> None:
    types = ",".join(f'"Type{i:02d}"' for i in range(30))
    block = f'<script type="application/ld+json">{{"@type": [{types}]}}</script>'
    gap = js_gap(_page(), _page(block))
    assert len(gap["rendered"]["schema_types"]) == render_page.JS_GAP_MAX_ITEMS
    assert gap["reasons"] == [
        "structured data appears only after JavaScript: "
        "'Type00', 'Type01', 'Type02', 'Type03', 'Type04'"
    ]
    # js_gap_reasons is public: it bounds what it quotes even for a gap it did not build.
    gap["diff"]["schema_types_only_rendered"] = ["Z" * 100_000] * 50
    (reason,) = js_gap_reasons(gap)
    assert len(reason) < 600


def test_whole_js_gap_json_stays_small_when_every_field_is_pushed_to_its_limit() -> None:
    gap = js_gap(_hostile("a"), _hostile("b"), raw_truncated=True)
    # Both versions are at their limits, and they differ everywhere.
    for side in ("raw", "rendered"):
        signals = gap[side]
        assert len(signals["title"]) == render_page.JS_GAP_MAX_TEXT
        assert len(signals["h1"]) == render_page.JS_GAP_MAX_ITEMS
        assert len(signals["schema_types"]) == render_page.JS_GAP_MAX_ITEMS
        assert len(signals["canonical_all"]) == render_page.JS_GAP_MAX_CANONICALS
        assert len(signals["meta_robots"].split(", ")) == render_page.JS_GAP_MAX_ITEMS
        assert all(len(name) <= render_page.JS_GAP_MAX_NAME for name in signals["schema_types"])
        assert all(
            len(part) <= render_page.JS_GAP_MAX_NAME
            for part in signals["meta_robots"].split(", ")
        )
    assert len(gap["diff"]["schema_types_only_rendered"]) == render_page.JS_GAP_MAX_ITEMS
    assert len(gap["diff"]["h1_only_rendered"]) == render_page.JS_GAP_MAX_ITEMS
    assert len(gap["reasons"]) == 7  # every reason js_gap can give
    assert all(len(reason) < 600 for reason in gap["reasons"])
    assert len(json.dumps(gap)) < MAX_GAP_JSON_CHARS, len(json.dumps(gap))
    assert len(json.dumps(gap, indent=2)) < MAX_GAP_JSON_CHARS, len(json.dumps(gap, indent=2))


def test_malformed_json_ld_and_markup_do_not_raise() -> None:
    broken = (
        '<html><head><script type="application/ld+json">{"@type": </script>'
        "<link rel=canonical><meta name=robots><title></title></head>"
        "<body><h1></h1><a href=''>x</a><p>unclosed"
    )
    gap = js_gap(broken, broken)
    assert gap["raw"]["schema_types"] == []
    assert gap["raw"]["canonical"] is None
    assert gap["raw"]["meta_robots"] is None
    assert gap["raw"]["title"] is None
    assert gap["reasons"] == []