Universal SEO skill for Claude Code. 26 sub-skills + 19 sub-agents covering technical SEO, E-E-A-T, schema, GEO/AEO, agent readiness (Lighthouse Agentic Browsing, WebMCP, llms.txt), backlinks, local SEO, e-commerce, international SEO, Google APIs, and PDF/Excel reporting. 9 optional extensions for live SEO data.
"""Deterministic response decoding for fetch_page.py."""
from __future__ import annotations
import sys
from pathlib import Path
import pytest
REPO_ROOT = Path(__file__).resolve().parents[1]
SCRIPTS_DIR = REPO_ROOT / "scripts"
if str(SCRIPTS_DIR) not in sys.path:
sys.path.insert(0, str(SCRIPTS_DIR))
pytest.importorskip("requests")
import fetch_page # noqa: E402
class FakeResponse:
def __init__(self, content: bytes, content_type: str = "") -> None:
self.content = content
self.headers = {"Content-Type": content_type} if content_type else {}
def test_no_charset_defaults_to_utf8_with_replacement() -> None:
response = FakeResponse("Café".encode("utf-8"))
assert fetch_page._decode_response_content(response) == "Café"
def test_explicit_charset_from_content_type_wins() -> None:
response = FakeResponse("Café".encode("iso-8859-1"), "text/html; charset=iso-8859-1")
assert fetch_page._decode_response_content(response) == "Café"
def test_meta_charset_is_used_when_header_has_no_charset() -> None:
html = '<meta charset="windows-1252"><p>Smart quote: “</p>'.encode("windows-1252")
response = FakeResponse(html, "text/html")
assert "Smart quote: “" in fetch_page._decode_response_content(response)
def test_og_image_query_does_not_override_a_later_charset_attribute() -> None:
"""charset= inside another attribute is not the document encoding.
HTML's prescan honors a charset attribute, or a content attribute only
when http-equiv is content-type. An og:image query must not win over a
later <meta charset="utf-8">, or the canonical URL is decoded wrong.
"""
import parse_html
html = (
'<meta property="og:image" content="https://cdn.example/a.jpg?charset=iso-8859-1">'
'<meta charset="utf-8">'
'<link rel="canonical" href="https://example.com/café">'
)
decoded = fetch_page._decode_response_content(FakeResponse(html.encode("utf-8"), "text/html"))
assert parse_html.parse_html(decoded)["canonical"] == "https://example.com/café"
def test_charset_attribute_wins_when_it_is_the_encoding() -> None:
html = (
'<meta property="og:image" content="https://cdn.example/a.jpg?charset=utf-8">'
'<meta charset="windows-1252"><p>Smart quote: “</p>'
).encode("windows-1252")
decoded = fetch_page._decode_response_content(FakeResponse(html, "text/html"))
assert "Smart quote: “" in decoded
@pytest.mark.parametrize(
"meta",
[
"<meta http-equiv=content-type content=text/html;charset=windows-1252>",
"<meta content=text/html;charset=windows-1252 http-equiv=Content-Type>",
"<meta http-equiv=content-type content=text/html;charset=windows-1252/>",
],
)
def test_unquoted_content_type_pragma_is_detected(meta: str) -> None:
"""An unquoted value runs to whitespace or ">", so the whole
``text/html;charset=...`` is the content attribute."""
html = (meta + "<p>Smart quote: “</p>").encode("windows-1252")
decoded = fetch_page._decode_response_content(FakeResponse(html, "text/html"))
assert "Smart quote: “" in decoded
@pytest.mark.parametrize(
"meta",
["<meta charset=windows-1252/>", "<meta charset=windows-1252 />", "<meta charset=windows-1252>"],
)
def test_self_closing_unquoted_charset_is_detected(meta: str) -> None:
html = (meta + "<p>Smart quote: “</p>").encode("windows-1252")
decoded = fetch_page._decode_response_content(FakeResponse(html, "text/html"))
assert "Smart quote: “" in decoded
def test_charset_inside_an_unquoted_url_is_not_the_encoding() -> None:
"""The og:image case again, with the URL unquoted and the text after a "/"."""
html = (
"<meta property=og:image content=https://cdn.example/charset=iso-8859-1>"
"<meta name=description content=text/html;charset=iso-8859-1>"
"<p>Café</p>"
)
decoded = fetch_page._decode_response_content(FakeResponse(html.encode("utf-8"), "text/html"))
assert "Café" in decoded
def test_meta_scan_stays_fast_on_a_hostile_head() -> None:
"""A 4096-byte run with no "=" must not be rescanned from every offset."""
import time
from url_safety import _extract_meta_charset
for filler in (b"a" * 4096, b"\"'" * 2048, b'a="' + b"x" * 4093):
raw = (b"<meta " + filler)[:4095] + b">"
best = float("inf")
for _ in range(3):
start = time.perf_counter()
assert _extract_meta_charset(raw) is None
best = min(best, time.perf_counter() - start)
# Linear is well under a millisecond; the quadratic scan took 90 ms or more.
assert best < 0.02, filler[:8]
def test_invalid_bytes_are_replaced_not_dropped() -> None:
response = FakeResponse(b"valid utf8 then invalid: \x8f")
decoded = fetch_page._decode_response_content(response)
assert "valid utf8" in decoded
assert "\ufffd" in decoded
def _real_response(body: bytes, content_type: str):
"""A requests.Response built the way the HTTP adapter builds one."""
import io
import requests
resp = requests.Response()
resp.status_code = 200
resp.headers["Content-Type"] = content_type
resp.raw = io.BytesIO(body)
resp.encoding = requests.utils.get_encoding_from_headers(resp.headers)
return resp
def test_issue_314_bare_text_html_utf8_is_not_decoded_as_latin1():
from url_safety import decode_response_text
body = "<html><head><title>Spécialité</title></head></html>".encode()
assert "Spécialité" in _real_response(body, "text/html").text # the bug
assert "Spécialité" in decode_response_text(_real_response(body, "text/html"))
def test_issue_314_explicit_charset_is_still_honoured():
from url_safety import decode_response_text
body = "Café".encode("iso-8859-1")
assert decode_response_text(_real_response(body, "text/html; charset=iso-8859-1")) == "Café"
def test_issue_314_fetching_scripts_use_the_shared_decoder():
"""Scripts that fetch pages must not fall back to response.text."""
from pathlib import Path
scripts = Path(__file__).resolve().parents[1] / "scripts"
for name in ("render_page.py", "parse_html.py", "nlp_analyze.py", "preload_check.py",
"parasite_risk.py", "ucp_check.py", "gbp_deprecation_lint.py"):
text = (scripts / name).read_text(encoding="utf-8")
# decode_body is the same shared decoder, for streamed bodies.
assert "decode_response_text(" in text or "decode_body(" in text, name
assert "resp.text" not in text, name