Skip to content

~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.

6.5 KB
Raw Blame
"""Install the NLTK data the upload parser needs into the Docker image.

The bounded upload parser child (security/isolated_loader.py) never
downloads NLTK data, so the image ships the packages that unstructured's
text classification loads (``unstructured/nlp/tokenize.py``): the
``punkt_tab`` sentence tokenizer and the ``averaged_perceptron_tagger_eng``
part-of-speech tagger.

NLTK's own downloader fetches whatever its index on the mutable
``gh-pages`` branch lists and checks nothing it could not also have been
served. This script instead fetches each archive from one pinned commit of
github.com/nltk/nltk_data, refuses it unless its size and SHA-256 match the
values below (the same values as that commit's ``index.xml``), and extracts
only regular files under the package's own directory.

Usage: python3 -I install_nltk_data.py <destination directory>

To update, pick a newer nltk_data commit and copy each package's ``size``
and ``sha256_checksum`` from ``index.xml`` at that commit.
"""

from __future__ import annotations

import hashlib
import io
import os
import stat
import sys
import time
import urllib.request
import zipfile
from pathlib import Path, PurePosixPath
from typing import Callable, NamedTuple

# Public revision of the upstream corpus repository.
NLTK_DATA_COMMIT = (
    "550b6625bcef1f2abff2ff770a5a0d272c9c6b2a"  # DevSkim: ignore DS173237
)
URL_TEMPLATE = (
    "https://raw.githubusercontent.com/nltk/nltk_data/"
    "{commit}/packages/{subdir}/{name}.zip"
)
# Extracted bytes per package; both are about 11 MB or less.
MAX_UNZIPPED_BYTES = 64 * 1024 * 1024


class Package(NamedTuple):
    subdir: str
    name: str
    size: int
    sha256: str


# Keep the names equal to isolated_loader.NLTK_DATA_PACKAGES.
PACKAGES = (
    Package(
        "tokenizers",
        "punkt_tab",
        4_319_076,
        "e57f64187974277726a3417ca6f181ec5403676c717672eef6a748a7b20e0106",  # DevSkim: ignore DS173237
    ),
    Package(
        "taggers",
        "averaged_perceptron_tagger_eng",
        1_539_115,
        "6025f530624335c67d6547d44757b357b4e79bae030a0383e9887a92c1718f0b",  # DevSkim: ignore DS173237
    ),
)


def package_url(package: Package) -> str:
    return URL_TEMPLATE.format(
        commit=NLTK_DATA_COMMIT, subdir=package.subdir, name=package.name
    )


def fetch(url: str, limit: int) -> bytes:
    """Return at most *limit* + 1 bytes of *url*, retrying transient errors."""
    if not url.startswith("https://"):
        raise ValueError(f"refusing non-HTTPS URL {url}")
    for attempt in range(1, 4):
        try:
            with urllib.request.urlopen(url, timeout=60) as response:  # noqa: S310 https only, checked above
                return response.read(limit + 1)
        except OSError as exc:
            if attempt == 3:
                raise
            print(f"download of {url} failed ({exc}); retrying", flush=True)
            time.sleep(5 * attempt)
    raise AssertionError("unreachable")


def verify(package: Package, archive: bytes) -> None:
    if len(archive) != package.size:
        raise ValueError(
            f"{package.name}: expected {package.size} bytes, got {len(archive)}"
        )
    digest = hashlib.sha256(archive).hexdigest()
    if digest != package.sha256:
        raise ValueError(
            f"{package.name}: SHA-256 mismatch: expected {package.sha256}, "
            f"got {digest}"
        )


def _make_readable(path: Path, *, directory: bool) -> None:
    # World-readable by design: the image's unprivileged service account
    # must read this data. Nothing but the owner (root) can write it.
    # bearer:disable python_lang_file_permissions
    os.chmod(path, 0o755 if directory else 0o644)  # noqa: S103 read-only data for every user


def extract(package: Package, archive: bytes, destination: Path) -> None:
    """Extract *archive* into ``destination/<subdir>/<name>/``.

    Every member must be a regular file or directory whose path stays under
    ``<name>/``; anything else refuses the whole archive before any file is
    written.
    """
    target = destination / package.subdir
    with zipfile.ZipFile(io.BytesIO(archive)) as bundle:
        members = bundle.infolist()
        total = 0
        for member in members:
            path = PurePosixPath(member.filename)
            # Unix file type bits, when the archive records them; a member
            # without them is a plain file or directory.
            file_type = stat.S_IFMT(member.external_attr >> 16)
            if (
                "\\" in member.filename
                or path.is_absolute()
                or ".." in path.parts
                or not path.parts
                or path.parts[0] != package.name
                or file_type not in (0, stat.S_IFREG, stat.S_IFDIR)
            ):
                raise ValueError(
                    f"{package.name}: unsafe archive member {member.filename!r}"
                )
            total += member.file_size
        if total > MAX_UNZIPPED_BYTES:
            raise ValueError(f"{package.name}: archive expands too far")
        for member in members:
            out = target.joinpath(*PurePosixPath(member.filename).parts)
            if member.is_dir():
                out.mkdir(parents=True, exist_ok=True)
                continue
            out.parent.mkdir(parents=True, exist_ok=True)
            with bundle.open(member) as source, open(out, "wb") as sink:
                sink.write(source.read())
    # Readable by every user, writable by none but the owner (root), so the
    # unprivileged service account can load the data but not replace it.
    for root, dirs, files in (target / package.name).walk():
        _make_readable(root, directory=True)
        for name in dirs:
            _make_readable(root / name, directory=True)
        for name in files:
            _make_readable(root / name, directory=False)


def install(
    destination: Path,
    fetcher: Callable[[str, int], bytes] = fetch,
) -> None:
    for package in PACKAGES:
        archive = fetcher(package_url(package), package.size)
        verify(package, archive)
        extract(package, archive, destination)
        print(f"installed NLTK {package.subdir}/{package.name}", flush=True)
    _make_readable(destination, directory=True)
    for package in PACKAGES:
        _make_readable(destination / package.subdir, directory=True)


def main(argv: list[str]) -> int:
    if len(argv) != 2:
        print(__doc__, file=sys.stderr)
        return 2
    destination = Path(argv[1])
    destination.mkdir(parents=True, exist_ok=True)
    install(destination)
    return 0


if __name__ == "__main__":
    raise SystemExit(main(sys.argv))