~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.
Python academiaanthropicarxivbravedeep-researchencryptionhome-automationhomeserverlocallocal-deep-researchlocal-llmmistralollamaopenaipubmedresearchresearch-toolretrieval-augmented-generationsearxngself-hosted
"""Install the NLTK data the upload parser needs into the Docker image.
The bounded upload parser child (security/isolated_loader.py) never
downloads NLTK data, so the image ships the packages that unstructured's
text classification loads (``unstructured/nlp/tokenize.py``): the
``punkt_tab`` sentence tokenizer and the ``averaged_perceptron_tagger_eng``
part-of-speech tagger.
NLTK's own downloader fetches whatever its index on the mutable
``gh-pages`` branch lists and checks nothing it could not also have been
served. This script instead fetches each archive from one pinned commit of
github.com/nltk/nltk_data, refuses it unless its size and SHA-256 match the
values below (the same values as that commit's ``index.xml``), and extracts
only regular files under the package's own directory.
Usage: python3 -I install_nltk_data.py <destination directory>
To update, pick a newer nltk_data commit and copy each package's ``size``
and ``sha256_checksum`` from ``index.xml`` at that commit.
"""
from __future__ import annotations
import hashlib
import io
import os
import stat
import sys
import time
import urllib.request
import zipfile
from pathlib import Path, PurePosixPath
from typing import Callable, NamedTuple
# Public revision of the upstream corpus repository.
NLTK_DATA_COMMIT = (
"550b6625bcef1f2abff2ff770a5a0d272c9c6b2a" # DevSkim: ignore DS173237
)
URL_TEMPLATE = (
"https://raw.githubusercontent.com/nltk/nltk_data/"
"{commit}/packages/{subdir}/{name}.zip"
)
# Extracted bytes per package; both are about 11 MB or less.
MAX_UNZIPPED_BYTES = 64 * 1024 * 1024
class Package(NamedTuple):
subdir: str
name: str
size: int
sha256: str
# Keep the names equal to isolated_loader.NLTK_DATA_PACKAGES.
PACKAGES = (
Package(
"tokenizers",
"punkt_tab",
4_319_076,
"e57f64187974277726a3417ca6f181ec5403676c717672eef6a748a7b20e0106", # DevSkim: ignore DS173237
),
Package(
"taggers",
"averaged_perceptron_tagger_eng",
1_539_115,
"6025f530624335c67d6547d44757b357b4e79bae030a0383e9887a92c1718f0b", # DevSkim: ignore DS173237
),
)
def package_url(package: Package) -> str:
return URL_TEMPLATE.format(
commit=NLTK_DATA_COMMIT, subdir=package.subdir, name=package.name
)
def fetch(url: str, limit: int) -> bytes:
"""Return at most *limit* + 1 bytes of *url*, retrying transient errors."""
if not url.startswith("https://"):
raise ValueError(f"refusing non-HTTPS URL {url}")
for attempt in range(1, 4):
try:
with urllib.request.urlopen(url, timeout=60) as response: # noqa: S310 https only, checked above
return response.read(limit + 1)
except OSError as exc:
if attempt == 3:
raise
print(f"download of {url} failed ({exc}); retrying", flush=True)
time.sleep(5 * attempt)
raise AssertionError("unreachable")
def verify(package: Package, archive: bytes) -> None:
if len(archive) != package.size:
raise ValueError(
f"{package.name}: expected {package.size} bytes, got {len(archive)}"
)
digest = hashlib.sha256(archive).hexdigest()
if digest != package.sha256:
raise ValueError(
f"{package.name}: SHA-256 mismatch: expected {package.sha256}, "
f"got {digest}"
)
def _make_readable(path: Path, *, directory: bool) -> None:
# World-readable by design: the image's unprivileged service account
# must read this data. Nothing but the owner (root) can write it.
# bearer:disable python_lang_file_permissions
os.chmod(path, 0o755 if directory else 0o644) # noqa: S103 read-only data for every user
def extract(package: Package, archive: bytes, destination: Path) -> None:
"""Extract *archive* into ``destination/<subdir>/<name>/``.
Every member must be a regular file or directory whose path stays under
``<name>/``; anything else refuses the whole archive before any file is
written.
"""
target = destination / package.subdir
with zipfile.ZipFile(io.BytesIO(archive)) as bundle:
members = bundle.infolist()
total = 0
for member in members:
path = PurePosixPath(member.filename)
# Unix file type bits, when the archive records them; a member
# without them is a plain file or directory.
file_type = stat.S_IFMT(member.external_attr >> 16)
if (
"\\" in member.filename
or path.is_absolute()
or ".." in path.parts
or not path.parts
or path.parts[0] != package.name
or file_type not in (0, stat.S_IFREG, stat.S_IFDIR)
):
raise ValueError(
f"{package.name}: unsafe archive member {member.filename!r}"
)
total += member.file_size
if total > MAX_UNZIPPED_BYTES:
raise ValueError(f"{package.name}: archive expands too far")
for member in members:
out = target.joinpath(*PurePosixPath(member.filename).parts)
if member.is_dir():
out.mkdir(parents=True, exist_ok=True)
continue
out.parent.mkdir(parents=True, exist_ok=True)
with bundle.open(member) as source, open(out, "wb") as sink:
sink.write(source.read())
# Readable by every user, writable by none but the owner (root), so the
# unprivileged service account can load the data but not replace it.
for root, dirs, files in (target / package.name).walk():
_make_readable(root, directory=True)
for name in dirs:
_make_readable(root / name, directory=True)
for name in files:
_make_readable(root / name, directory=False)
def install(
destination: Path,
fetcher: Callable[[str, int], bytes] = fetch,
) -> None:
for package in PACKAGES:
archive = fetcher(package_url(package), package.size)
verify(package, archive)
extract(package, archive, destination)
print(f"installed NLTK {package.subdir}/{package.name}", flush=True)
_make_readable(destination, directory=True)
for package in PACKAGES:
_make_readable(destination / package.subdir, directory=True)
def main(argv: list[str]) -> int:
if len(argv) != 2:
print(__doc__, file=sys.stderr)
return 2
destination = Path(argv[1])
destination.mkdir(parents=True, exist_ok=True)
install(destination)
return 0
if __name__ == "__main__":
raise SystemExit(main(sys.argv))