Skip to content

~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.

7.2 KB
Raw Blame
from __future__ import annotations

import subprocess
import sys
from pathlib import Path

import pytest

HOOK_SCRIPT = (
    Path(__file__).resolve().parents[2]
    / ".pre-commit-hooks"
    / "check-zero-width-unicode.py"
)

# Contract pin: exactly the set the hook rejects — all Bidi_Control codepoints
# (U+061C, U+200E-U+200F, U+202A-U+202E, U+2066-U+2069), the tag block
# U+E0000-E007F, the variation-selector supplement U+E0100-E01EF, the hook's
# ZERO_WIDTH_CODEPOINTS constant (U+061C, U+180E, U+200B-U+200D, U+200E-U+200F,
# U+2060, U+FEFF), and the invisible operators / deprecated format controls
# U+2061-U+2064 and U+206A-U+206F. Keep in sync with AGENTS.md's "Hidden
# Unicode" invariant.
REJECTED_CODEPOINTS = (
    [0x061C, 0x180E, 0x200B, 0x200C, 0x200D, 0x200E, 0x200F, 0x2060, 0xFEFF]
    + list(range(0x202A, 0x202F))
    + list(range(0x2061, 0x2065))
    + list(range(0x2066, 0x206A))
    + list(range(0x206A, 0x2070))
    + list(range(0xE0000, 0xE0080))
    + list(range(0xE0100, 0xE01F0))
)

# Contract pin: variation selectors are NOT rejected — they are permitted
# unconditionally (U+FE00-U+FE0F commonly form part of emoji sequences;
# U+180B-U+180D are Mongolian free variation selectors).
PERMITTED_CODEPOINTS = (
    list(range(0xFE00, 0xFE10)) + list(range(0x180B, 0x180E)) + [0x00AD]
)


@pytest.mark.parametrize("codepoint", REJECTED_CODEPOINTS)
def test_trapdoor_codepoint_rejected(tmp_path: Path, codepoint: int) -> None:
    path = tmp_path / "payload.txt"
    _ = path.write_text(f"safe\nunsafe{chr(codepoint)}text\n", encoding="utf-8")

    result = subprocess.run(
        [sys.executable, str(HOOK_SCRIPT), str(path)],
        capture_output=True,
        text=True,
        check=False,
    )

    assert result.returncode == 1
    assert f"{path}:2" in result.stdout
    assert f"U+{codepoint:04X}" in result.stdout


def test_each_finding_reported(tmp_path: Path) -> None:
    path = tmp_path / "multiple.txt"
    _ = path.write_text("first\u200b\nsecond\u202e\n", encoding="utf-8")

    result = subprocess.run(
        [sys.executable, str(HOOK_SCRIPT), str(path)],
        capture_output=True,
        text=True,
        check=False,
    )

    assert result.returncode == 1
    assert f"{path}:1" in result.stdout
    assert "U+200B" in result.stdout
    assert f"{path}:2" in result.stdout
    assert "U+202E" in result.stdout


def test_line_numbers_ignore_non_newline_separators(tmp_path: Path) -> None:
    path = tmp_path / "separators.txt"
    _ = path.write_text("a\x1cb\x85c\u2028d\nunsafe\u200b\n", encoding="utf-8")

    result = subprocess.run(
        [sys.executable, str(HOOK_SCRIPT), str(path)],
        capture_output=True,
        text=True,
        check=False,
    )

    assert result.returncode == 1
    assert f"{path}:2:" in result.stdout


def test_clean_text_passes(tmp_path: Path) -> None:
    path = tmp_path / "clean.txt"
    _ = path.write_text("ordinary text\n", encoding="utf-8")

    result = subprocess.run(
        [sys.executable, str(HOOK_SCRIPT), str(path)], check=False
    )

    assert result.returncode == 0


def test_binary_file_fails_closed(tmp_path: Path) -> None:
    path = tmp_path / "binary.dat"
    _ = path.write_bytes(b"\xff\xfe\x00\x01")

    result = subprocess.run(
        [sys.executable, str(HOOK_SCRIPT), str(path)],
        capture_output=True,
        text=True,
        check=False,
    )

    # Undecodable input is refused loudly (fail closed), never skipped.
    assert result.returncode == 1
    assert "fail closed" in result.stdout


def test_undecodable_input_with_hidden_codepoint_rejected(
    tmp_path: Path,
) -> None:
    path = tmp_path / "stray.txt"
    # Valid UTF-8 U+202E next to one stray 0xFF byte: must NOT slip through.
    _ = path.write_bytes("safe\nunsafe\u202etext\n".encode("utf-8") + b"\xff")

    result = subprocess.run(
        [sys.executable, str(HOOK_SCRIPT), str(path)],
        capture_output=True,
        text=True,
        check=False,
    )

    assert result.returncode == 1
    assert "U+202E" in result.stdout


def test_undecodable_input_without_hidden_codepoint_refused(
    tmp_path: Path,
) -> None:
    path = tmp_path / "stray.txt"
    _ = path.write_bytes("hello\n".encode("utf-8") + b"\xff")

    result = subprocess.run(
        [sys.executable, str(HOOK_SCRIPT), str(path)],
        capture_output=True,
        text=True,
        check=False,
    )

    assert result.returncode == 1
    assert "fail closed" in result.stdout


def test_utf16_bom_input_refused(tmp_path: Path) -> None:
    path = tmp_path / "utf16.txt"
    _ = path.write_bytes("harmless\u202etext\n".encode("utf-16"))

    result = subprocess.run(
        [sys.executable, str(HOOK_SCRIPT), str(path)],
        capture_output=True,
        text=True,
        check=False,
    )

    assert result.returncode == 1
    assert "UTF-16/UTF-32" in result.stdout


# Contract pin: input handling fails closed. BOM-less UTF-16/UTF-32 text
# carries NUL bytes and must be refused outright, not silently accepted;
# the NUL rule fires on any NUL byte, not only UTF-16/32-shaped input.
@pytest.mark.parametrize(
    "encoding", ["utf-16-le", "utf-16-be", "utf-32-le", "utf-32-be"]
)
@pytest.mark.parametrize("codepoint", [0x202E, 0x200B])
def test_bomless_utf16_utf32_input_refused(
    tmp_path: Path, encoding: str, codepoint: int
) -> None:
    path = tmp_path / "bomless.txt"
    _ = path.write_bytes(f"safe\nunsafe{chr(codepoint)}text\n".encode(encoding))

    result = subprocess.run(
        [sys.executable, str(HOOK_SCRIPT), str(path)],
        capture_output=True,
        text=True,
        check=False,
    )

    assert result.returncode == 1
    assert "UTF-16/UTF-32" in result.stdout


def test_bomless_utf16_without_rejected_codepoint_still_refused(
    tmp_path: Path,
) -> None:
    path = tmp_path / "clean_utf16.txt"
    _ = path.write_bytes("harmless text\n".encode("utf-16-le"))

    result = subprocess.run(
        [sys.executable, str(HOOK_SCRIPT), str(path)],
        capture_output=True,
        text=True,
        check=False,
    )

    assert result.returncode == 1
    assert "UTF-16/UTF-32" in result.stdout


def test_nul_byte_in_utf8_text_refused(tmp_path: Path) -> None:
    path = tmp_path / "nul.txt"
    _ = path.write_bytes(b"hello\x00world\n")

    result = subprocess.run(
        [sys.executable, str(HOOK_SCRIPT), str(path)],
        capture_output=True,
        text=True,
        check=False,
    )

    assert result.returncode == 1
    assert "fail closed" in result.stdout


@pytest.mark.parametrize("codepoint", PERMITTED_CODEPOINTS)
def test_variation_selector_permitted(tmp_path: Path, codepoint: int) -> None:
    path = tmp_path / "permitted.txt"
    _ = path.write_text(f"safe {chr(codepoint)} text\n", encoding="utf-8")

    result = subprocess.run(
        [sys.executable, str(HOOK_SCRIPT), str(path)],
        capture_output=True,
        text=True,
        check=False,
    )

    assert result.returncode == 0


def test_emoji_variation_selector_sequence_permitted(tmp_path: Path) -> None:
    path = tmp_path / "emoji.txt"
    _ = path.write_text("results are ready \u2764\ufe0f\n", encoding="utf-8")

    result = subprocess.run(
        [sys.executable, str(HOOK_SCRIPT), str(path)],
        capture_output=True,
        text=True,
        check=False,
    )

    assert result.returncode == 0