~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.
Python academiaanthropicarxivbravedeep-researchencryptionhome-automationhomeserverlocallocal-deep-researchlocal-llmmistralollamaopenaipubmedresearchresearch-toolretrieval-augmented-generationsearxngself-hosted
#!/usr/bin/env python3
# allow: no-sut-import
from __future__ import annotations
import sys
from pathlib import Path
from typing import Final
ZERO_WIDTH_CODEPOINTS: Final = frozenset(
{0x061C, 0x180E, 0x200B, 0x200C, 0x200D, 0x200E, 0x200F, 0x2060, 0xFEFF}
)
def is_trapdoor_codepoint(codepoint: int) -> bool:
return (
codepoint in ZERO_WIDTH_CODEPOINTS
or 0x202A <= codepoint <= 0x202E
or 0x2061 <= codepoint <= 0x2064
or 0x2066 <= codepoint <= 0x2069
or 0x206A <= codepoint <= 0x206F
or 0xE0000 <= codepoint <= 0xE007F
or 0xE0100 <= codepoint <= 0xE01EF
)
def all_trapdoor_codepoints() -> tuple[int, ...]:
"""Every codepoint ``is_trapdoor_codepoint`` rejects, as an iterable."""
return (
tuple(sorted(ZERO_WIDTH_CODEPOINTS))
+ tuple(range(0x202A, 0x202F))
+ tuple(range(0x2061, 0x2065))
+ tuple(range(0x2066, 0x2070))
+ tuple(range(0xE0000, 0xE0080))
+ tuple(range(0xE0100, 0xE01F0))
)
def main() -> int:
has_violations = False
for path_arg in sys.argv[1:]:
path = Path(path_arg)
try:
# The local repository checker reads caller-selected source files.
# bearer:disable python_lang_path_using_user_input
text = path.read_text(encoding="utf-8")
except UnicodeDecodeError:
# Fail closed: undecodable bytes can hide trapdoor codepoints
# (e.g. a valid UTF-8 U+202E next to a stray 0xFF byte), so scan
# the raw bytes for the UTF-8 encodings of every rejected
# codepoint and refuse the file outright instead of skipping it.
# Inspect the same caller-selected file when decoding fails.
# bearer:disable python_lang_path_using_user_input
raw = path.read_bytes()
for codepoint in all_trapdoor_codepoints():
if chr(codepoint).encode("utf-8") in raw:
message = (
f"{path}:1: disallowed zero-width Unicode "
+ f"U+{codepoint:04X} (raw bytes in undecodable input)"
)
print(message)
has_violations = True
if raw.startswith(
(
b"\xff\xfe",
b"\xfe\xff",
b"\xff\xfe\x00\x00",
b"\x00\x00\xfe\xff",
)
):
print(
f"{path}:1: UTF-16/UTF-32 input refused; convert to UTF-8"
)
has_violations = True
print(
f"{path}:1: undecodable as UTF-8; refusing file (fail closed)"
)
has_violations = True
continue
# Fail closed on NUL bytes: they mark binary or UTF-16/UTF-32 input.
# BOM-less UTF-16/32 with ASCII content decodes as UTF-8 with embedded
# NULs, so a U+202E hidden in it would otherwise pass undetected;
# refuse the file outright instead of scanning it.
if "\x00" in text:
print(
f"{path}:1: NUL byte in input (UTF-16/UTF-32 or binary); "
+ "refusing file (fail closed); convert to UTF-8"
)
has_violations = True
continue
# split("\n"), not splitlines(): splitlines() also breaks on \x1c,
# \x85, U+2028 etc., which skews the reported line numbers.
for line_number, line in enumerate(text.split("\n"), start=1):
for character in line:
codepoint = ord(character)
if is_trapdoor_codepoint(codepoint):
message = (
f"{path}:{line_number}: disallowed zero-width Unicode "
+ f"U+{codepoint:04X}"
)
print(message)
has_violations = True
return int(has_violations)
if __name__ == "__main__":
sys.exit(main())