Skip to content

~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.

3.9 KB
Raw Blame
#!/usr/bin/env python3
# allow: no-sut-import

from __future__ import annotations

import sys
from pathlib import Path
from typing import Final

ZERO_WIDTH_CODEPOINTS: Final = frozenset(
    {0x061C, 0x180E, 0x200B, 0x200C, 0x200D, 0x200E, 0x200F, 0x2060, 0xFEFF}
)


def is_trapdoor_codepoint(codepoint: int) -> bool:
    return (
        codepoint in ZERO_WIDTH_CODEPOINTS
        or 0x202A <= codepoint <= 0x202E
        or 0x2061 <= codepoint <= 0x2064
        or 0x2066 <= codepoint <= 0x2069
        or 0x206A <= codepoint <= 0x206F
        or 0xE0000 <= codepoint <= 0xE007F
        or 0xE0100 <= codepoint <= 0xE01EF
    )


def all_trapdoor_codepoints() -> tuple[int, ...]:
    """Every codepoint ``is_trapdoor_codepoint`` rejects, as an iterable."""
    return (
        tuple(sorted(ZERO_WIDTH_CODEPOINTS))
        + tuple(range(0x202A, 0x202F))
        + tuple(range(0x2061, 0x2065))
        + tuple(range(0x2066, 0x2070))
        + tuple(range(0xE0000, 0xE0080))
        + tuple(range(0xE0100, 0xE01F0))
    )


def main() -> int:
    has_violations = False
    for path_arg in sys.argv[1:]:
        path = Path(path_arg)
        try:
            # The local repository checker reads caller-selected source files.
            # bearer:disable python_lang_path_using_user_input
            text = path.read_text(encoding="utf-8")
        except UnicodeDecodeError:
            # Fail closed: undecodable bytes can hide trapdoor codepoints
            # (e.g. a valid UTF-8 U+202E next to a stray 0xFF byte), so scan
            # the raw bytes for the UTF-8 encodings of every rejected
            # codepoint and refuse the file outright instead of skipping it.
            # Inspect the same caller-selected file when decoding fails.
            # bearer:disable python_lang_path_using_user_input
            raw = path.read_bytes()
            for codepoint in all_trapdoor_codepoints():
                if chr(codepoint).encode("utf-8") in raw:
                    message = (
                        f"{path}:1: disallowed zero-width Unicode "
                        + f"U+{codepoint:04X} (raw bytes in undecodable input)"
                    )
                    print(message)
                    has_violations = True
            if raw.startswith(
                (
                    b"\xff\xfe",
                    b"\xfe\xff",
                    b"\xff\xfe\x00\x00",
                    b"\x00\x00\xfe\xff",
                )
            ):
                print(
                    f"{path}:1: UTF-16/UTF-32 input refused; convert to UTF-8"
                )
                has_violations = True
            print(
                f"{path}:1: undecodable as UTF-8; refusing file (fail closed)"
            )
            has_violations = True
            continue

        # Fail closed on NUL bytes: they mark binary or UTF-16/UTF-32 input.
        # BOM-less UTF-16/32 with ASCII content decodes as UTF-8 with embedded
        # NULs, so a U+202E hidden in it would otherwise pass undetected;
        # refuse the file outright instead of scanning it.
        if "\x00" in text:
            print(
                f"{path}:1: NUL byte in input (UTF-16/UTF-32 or binary); "
                + "refusing file (fail closed); convert to UTF-8"
            )
            has_violations = True
            continue

        # split("\n"), not splitlines(): splitlines() also breaks on \x1c,
        # \x85, U+2028 etc., which skews the reported line numbers.
        for line_number, line in enumerate(text.split("\n"), start=1):
            for character in line:
                codepoint = ord(character)
                if is_trapdoor_codepoint(codepoint):
                    message = (
                        f"{path}:{line_number}: disallowed zero-width Unicode "
                        + f"U+{codepoint:04X}"
                    )
                    print(message)
                    has_violations = True

    return int(has_violations)


if __name__ == "__main__":
    sys.exit(main())