Skip to content

A community-supported supercharged document management system: scan, index and archive all your documents

3.5 KB
Raw Blame
from __future__ import annotations

import dataclasses
from typing import TYPE_CHECKING

from whoosh_compat import FieldKind
from whoosh_compat import FieldRegistry

from documents.search._fields import PUBLIC_FIELDS
from documents.search._tokenizer import ascii_fold
from documents.search._tokenizer import paperless_text_analyzer
from documents.search._tokenizer import stem_pattern_text

if TYPE_CHECKING:
    from whoosh_compat import PatternNormalizer

_registry_cache: dict[str | None, FieldRegistry] = {}


def _identity_analyzer(text: str) -> list[str]:
    """Analyzer for KEYWORD fields indexed with the raw tokenizer (no splitting)."""
    return [text]


def _fold_normalizer(text: str) -> str:
    """Wildcard/regex literal-run normalizer for fields indexed without stemming."""
    return ascii_fold(text.lower())


def _make_pattern_normalizer(language: str | None) -> PatternNormalizer:
    """Build the wildcard/regex literal-run normalizer for a search language."""

    def _pattern_normalizer(text: str) -> tuple[str, ...]:
        """Normalize a literal run into the forms a term may match.

        TEXT index terms go through lowercase -> ascii_fold -> stem, so a
        pattern that skips stemming can never match one: "invoice*" would look
        for a term starting with "invoice" while the index holds "invoic". The
        run is therefore offered stemmed as well. KEYWORD fields are indexed
        raw and get _fold_normalizer instead, so their patterns stay literal.

        Both forms are returned, as alternatives, because neither is a prefix
        of the other in general: English stemming substitutes as well as
        truncates ("copy" -> "copi"), so the stem alone loses the compounds
        the typed run reaches ("copyright") while the typed run alone loses
        the inflections the stem reaches ("copies"). whoosh-compat ORs the
        alternatives per literal run and deduplicates them, so a run the
        stemmer leaves alone costs exactly the one branch it did before.

        Inside a bracket class the emitter calls this once per character and
        uses the answer only if it is a single one-character form; two forms
        there leave the character as typed. A stemmer does not change a lone
        character, so the two forms deduplicate to one and the class body is
        folded as before.
        """
        folded = ascii_fold(text.lower())
        stemmed = stem_pattern_text(folded, language)
        return (folded, stemmed)

    return _pattern_normalizer


def get_field_registry(language: str | None) -> FieldRegistry:
    """Build (or return the cached) FieldRegistry for the given search language.

    Cached keyed by language, rebuilt on the same trigger register_tokenizers()
    uses (settings.SEARCH_LANGUAGE change). A fresh call with a new language
    builds and caches a new registry rather than mutating the old one.
    """
    if language in _registry_cache:
        return _registry_cache[language]

    text_analyzer = paperless_text_analyzer(language).analyze
    pattern_normalizer = _make_pattern_normalizer(language)

    specs = [
        dataclasses.replace(
            field,
            analyzer=_identity_analyzer
            if field.kind is FieldKind.KEYWORD
            else text_analyzer,
            pattern_normalizer=_fold_normalizer
            if field.kind is FieldKind.KEYWORD
            else pattern_normalizer,
        )
        for field in PUBLIC_FIELDS
    ]

    registry = FieldRegistry(specs)
    _registry_cache[language] = registry
    return registry