Skip to content

Universal SEO skill for Claude Code. 26 sub-skills + 19 sub-agents covering technical SEO, E-E-A-T, schema, GEO/AEO, agent readiness (Lighthouse Agentic Browsing, WebMCP, llms.txt), backlinks, local SEO, e-commerce, international SEO, Google APIs, and PDF/Excel reporting. 9 optional extensions for live SEO data.

6.4 KB
Raw Blame
"""Deterministic response decoding for fetch_page.py."""

from __future__ import annotations

import sys
from pathlib import Path

import pytest

REPO_ROOT = Path(__file__).resolve().parents[1]
SCRIPTS_DIR = REPO_ROOT / "scripts"
if str(SCRIPTS_DIR) not in sys.path:
    sys.path.insert(0, str(SCRIPTS_DIR))

pytest.importorskip("requests")
import fetch_page  # noqa: E402


class FakeResponse:
    def __init__(self, content: bytes, content_type: str = "") -> None:
        self.content = content
        self.headers = {"Content-Type": content_type} if content_type else {}


def test_no_charset_defaults_to_utf8_with_replacement() -> None:
    response = FakeResponse("Café".encode("utf-8"))
    assert fetch_page._decode_response_content(response) == "Café"


def test_explicit_charset_from_content_type_wins() -> None:
    response = FakeResponse("Café".encode("iso-8859-1"), "text/html; charset=iso-8859-1")
    assert fetch_page._decode_response_content(response) == "Café"


def test_meta_charset_is_used_when_header_has_no_charset() -> None:
    html = '<meta charset="windows-1252"><p>Smart quote: “</p>'.encode("windows-1252")
    response = FakeResponse(html, "text/html")
    assert "Smart quote: “" in fetch_page._decode_response_content(response)


def test_og_image_query_does_not_override_a_later_charset_attribute() -> None:
    """charset= inside another attribute is not the document encoding.

    HTML's prescan honors a charset attribute, or a content attribute only
    when http-equiv is content-type. An og:image query must not win over a
    later <meta charset="utf-8">, or the canonical URL is decoded wrong.
    """
    import parse_html

    html = (
        '<meta property="og:image" content="https://cdn.example/a.jpg?charset=iso-8859-1">'
        '<meta charset="utf-8">'
        '<link rel="canonical" href="https://example.com/café">'
    )
    decoded = fetch_page._decode_response_content(FakeResponse(html.encode("utf-8"), "text/html"))
    assert parse_html.parse_html(decoded)["canonical"] == "https://example.com/café"


def test_charset_attribute_wins_when_it_is_the_encoding() -> None:
    html = (
        '<meta property="og:image" content="https://cdn.example/a.jpg?charset=utf-8">'
        '<meta charset="windows-1252"><p>Smart quote: “</p>'
    ).encode("windows-1252")
    decoded = fetch_page._decode_response_content(FakeResponse(html, "text/html"))
    assert "Smart quote: “" in decoded


@pytest.mark.parametrize(
    "meta",
    [
        "<meta http-equiv=content-type content=text/html;charset=windows-1252>",
        "<meta content=text/html;charset=windows-1252 http-equiv=Content-Type>",
        "<meta http-equiv=content-type content=text/html;charset=windows-1252/>",
    ],
)
def test_unquoted_content_type_pragma_is_detected(meta: str) -> None:
    """An unquoted value runs to whitespace or ">", so the whole
    ``text/html;charset=...`` is the content attribute."""
    html = (meta + "<p>Smart quote: “</p>").encode("windows-1252")
    decoded = fetch_page._decode_response_content(FakeResponse(html, "text/html"))
    assert "Smart quote: “" in decoded


@pytest.mark.parametrize(
    "meta",
    ["<meta charset=windows-1252/>", "<meta charset=windows-1252 />", "<meta charset=windows-1252>"],
)
def test_self_closing_unquoted_charset_is_detected(meta: str) -> None:
    html = (meta + "<p>Smart quote: “</p>").encode("windows-1252")
    decoded = fetch_page._decode_response_content(FakeResponse(html, "text/html"))
    assert "Smart quote: “" in decoded


def test_charset_inside_an_unquoted_url_is_not_the_encoding() -> None:
    """The og:image case again, with the URL unquoted and the text after a "/"."""
    html = (
        "<meta property=og:image content=https://cdn.example/charset=iso-8859-1>"
        "<meta name=description content=text/html;charset=iso-8859-1>"
        "<p>Café</p>"
    )
    decoded = fetch_page._decode_response_content(FakeResponse(html.encode("utf-8"), "text/html"))
    assert "Café" in decoded


def test_meta_scan_stays_fast_on_a_hostile_head() -> None:
    """A 4096-byte run with no "=" must not be rescanned from every offset."""
    import time

    from url_safety import _extract_meta_charset

    for filler in (b"a" * 4096, b"\"'" * 2048, b'a="' + b"x" * 4093):
        raw = (b"<meta " + filler)[:4095] + b">"
        best = float("inf")
        for _ in range(3):
            start = time.perf_counter()
            assert _extract_meta_charset(raw) is None
            best = min(best, time.perf_counter() - start)
        # Linear is well under a millisecond; the quadratic scan took 90 ms or more.
        assert best < 0.02, filler[:8]


def test_invalid_bytes_are_replaced_not_dropped() -> None:
    response = FakeResponse(b"valid utf8 then invalid: \x8f")
    decoded = fetch_page._decode_response_content(response)
    assert "valid utf8" in decoded
    assert "\ufffd" in decoded


def _real_response(body: bytes, content_type: str):
    """A requests.Response built the way the HTTP adapter builds one."""
    import io

    import requests

    resp = requests.Response()
    resp.status_code = 200
    resp.headers["Content-Type"] = content_type
    resp.raw = io.BytesIO(body)
    resp.encoding = requests.utils.get_encoding_from_headers(resp.headers)
    return resp


def test_issue_314_bare_text_html_utf8_is_not_decoded_as_latin1():
    from url_safety import decode_response_text

    body = "<html><head><title>Spécialité</title></head></html>".encode()
    assert "Spécialité" in _real_response(body, "text/html").text  # the bug
    assert "Spécialité" in decode_response_text(_real_response(body, "text/html"))


def test_issue_314_explicit_charset_is_still_honoured():
    from url_safety import decode_response_text

    body = "Café".encode("iso-8859-1")
    assert decode_response_text(_real_response(body, "text/html; charset=iso-8859-1")) == "Café"


def test_issue_314_fetching_scripts_use_the_shared_decoder():
    """Scripts that fetch pages must not fall back to response.text."""
    from pathlib import Path

    scripts = Path(__file__).resolve().parents[1] / "scripts"
    for name in ("render_page.py", "parse_html.py", "nlp_analyze.py", "preload_check.py",
                 "parasite_risk.py", "ucp_check.py", "gbp_deprecation_lint.py"):
        text = (scripts / name).read_text(encoding="utf-8")
        # decode_body is the same shared decoder, for streamed bodies.
        assert "decode_response_text(" in text or "decode_body(" in text, name
        assert "resp.text" not in text, name