Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

15.1 KB
Raw Blame
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0

"""RAG config; every value is env-overridable."""

from __future__ import annotations

import os
import re

DEFAULT_EMBEDDING_MODEL = "unsloth/bge-small-en-v1.5"
EMBEDDING_MODEL = os.environ.get("RAG_EMBEDDING_MODEL", DEFAULT_EMBEDDING_MODEL)
# Keep <= embedder_max - ~12: bge's 512 limit plus 2 special tokens (llama-server 500s on overflow, ST truncates).
CHUNK_TOKENS = int(os.environ.get("RAG_CHUNK_TOKENS", "500"))
CHUNK_OVERLAP = int(os.environ.get("RAG_CHUNK_OVERLAP", "64"))
TOP_K_LEXICAL = int(os.environ.get("RAG_TOP_K_LEXICAL", "30"))
TOP_K_DENSE = int(os.environ.get("RAG_TOP_K_DENSE", "30"))
TOP_K_HYBRID = int(os.environ.get("RAG_TOP_K_HYBRID", "10"))
RRF_K = int(os.environ.get("RAG_RRF_K", "60"))

# Whole-document context: a file under this budget is injected in full instead of top-K retrieval.
THREAD_WHOLE_DOC = os.environ.get("RAG_THREAD_WHOLE_DOC", "1") == "1"
WHOLE_DOC_MAX_TOKENS = int(os.environ.get("RAG_WHOLE_DOC_MAX_TOKENS", "6000"))

# Off: evicted turns are dropped, no recall reserve taken. Applies once the window evicts (opt-in: context_overflow truncate_oldest).
CONVERSATION_ARCHIVE = os.environ.get("RAG_CONVERSATION_ARCHIVE", "1") == "1"
CONVERSATION_ARCHIVE_TOP_K = int(os.environ.get("RAG_CONVERSATION_ARCHIVE_TOP_K", "4"))
# Sized to CONVERSATION_ARCHIVE_TOP_K * CHUNK_TOKENS with slack for the wrapper text.
CONVERSATION_RECALL_RESERVE_TOKENS = int(
    os.environ.get("RAG_CONVERSATION_RECALL_RESERVE_TOKENS", "2048")
)
# Off restores the plain OR-of-every-token query other scopes use.
CONVERSATION_QUERY_FOCUS = os.environ.get("RAG_CONVERSATION_QUERY_FOCUS", "1") == "1"
# Presentation only: neither ordering changes which turns are selected.
CONVERSATION_RECALL_ORDER = os.environ.get("RAG_CONVERSATION_RECALL_ORDER", "chronological")
# Automatic recall only; 0.0 since a weak match is often still the right turn.
CONVERSATION_FORCED_MIN_SCORE = float(os.environ.get("RAG_CONVERSATION_FORCED_MIN_SCORE", "0.0"))

# Types parsers.parse handles; frontend and Rust parity tests read this literal.
SUPPORTED_UPLOAD_EXTS = {".pdf", ".txt", ".md", ".markdown", ".docx", ".html", ".htm"}
# Source and config files, decoded like .txt; the chat's TEXT_ATTACHMENT_EXTENSIONS minus the above (parity test).
SOURCE_TEXT_EXTS = frozenset(
    """
    .text .log .mdx .rst .adoc .asciidoc .org .textile .wiki .tex .latex .sty .cls .bib .rmd .qmd
    .srt .vtt .sbv .ass .ssa .sub .lrc
    .csv .tsv .psv .json .jsonl .ndjson .jsonc .json5 .geojson .kml .gpx .har .avsc .xml .yaml .yml .toml .ini
    .cfg .conf .cnf .env .properties .plist .edn .ron .cue .lock .mod .sum .reg .desktop .service
    .po .pot .strings .resx .xliff .xlf .ics .vcf .eml .mbox .m3u8 .pls
    .css .scss .sass .less .styl .svg .vue .svelte .astro .pug .jade .haml .slim .ejs .erb .hbs
    .handlebars .mustache .njk .jinja .jinja2 .j2 .twig .liquid .cshtml .razor .aspx .jsp .tpl .qml
    .js .jsx .mjs .cjs .ts .tsx .mts .cts .py .pyi .pyx .pxd .ipynb
    .java .kt .kts .scala .groovy .gradle .sbt .clj .cljs .cljc
    .c .h .cc .cpp .hpp .cxx .hxx .hh .ipp .inl .cu .cuh .rs .go .zig .odin .nim .nims .nimble .cr .d
    .v .sv .svh .vhd .vhdl .asm .s
    .cs .vb .vbs .fs .fsi .fsx .csproj .vbproj .fsproj .sln .props .targets
    .m .mm .swift .applescript .metal
    .rb .rake .gemspec .podspec .php .pl .pm .r .jl .lua .tcl .dart .hx .hs .lhs .ml .mli .ex .exs
    .erl .hrl .rkt .scm .ss .lisp .lsp .cl .el .pas .pp .ada .adb .ads .cob .cbl .f .for .f90 .f95
    .f03 .sas .awk .sed .m4 .sol .move .cairo .mojo .gd .sqf
    .sh .bash .zsh .fish .ksh .csh .tcsh .nu .ps1 .psm1 .psd1 .bat .cmd
    .sql .psql .plsql .hql .cql .graphql .gql .proto .thrift .capnp .prisma
    .tf .tfvars .tfstate .hcl .nix .dhall .bicep .dockerfile .containerfile .makefile .mk .mak .cmake
    .ninja .bzl .bazel .star .starlark .gn .gni .pro .pri .cabal .opam
    .glsl .frag .vert .geom .comp .hlsl .wgsl .shader
    .mmd .mermaid .puml .plantuml .dot .gv .feature .robot .http .rest .diff .patch
    """.split()
)
DOCUMENT_UPLOAD_EXTS = frozenset(
    """
    .doc .docm .dotx .dotm .xls .xlt .xlsx .xlsm .xltx .xltm .ppt .pps .pptx .pptm .potx .potm .ppsx .ppsm
    .odt .ods .odp .ott .ots .otp .msg .eml .mht .mhtml .rtf .epub .xhtml .xht
    """.split()
)
_PARSEABLE_EXTS = SUPPORTED_UPLOAD_EXTS | SOURCE_TEXT_EXTS | DOCUMENT_UPLOAD_EXTS
# RAG_UPLOAD_EXTS (e.g. ".md,.markdown") can only narrow: a type without a parser would fail every ingest.
_requested_exts = {
    "." + ext.strip().lstrip(".").lower()
    for ext in os.environ.get("RAG_UPLOAD_EXTS", "").split(",")
    if ext.strip().lstrip(".")
}
UPLOAD_EXTS = (_PARSEABLE_EXTS & _requested_exts) or set(_PARSEABLE_EXTS)
# 0 disables the cap; bounds parse + vision work at ingest.
MAX_UPLOAD_BYTES = int(os.environ.get("RAG_MAX_UPLOAD_BYTES", str(200 * 1024 * 1024)))

# Caps prevent an accidentally broad linked folder from becoming an unbounded ingestion queue.
FOLDER_SYNC_INTERVAL_S = float(os.environ.get("RAG_FOLDER_SYNC_INTERVAL_S", "30"))
FOLDER_MAX_FILES = int(os.environ.get("RAG_FOLDER_MAX_FILES", "10000"))
# Linked-folder plain-text files above this are logs/data dumps, not documents; 0 disables.
FOLDER_MAX_TEXT_BYTES = int(os.environ.get("RAG_FOLDER_MAX_TEXT_BYTES", str(20 * 1024 * 1024)))
FOLDER_JOB_HISTORY_LIMIT = int(os.environ.get("RAG_FOLDER_JOB_HISTORY_LIMIT", "200"))
# Linked-folder documents ingested concurrently, clamped to 1..4.
FOLDER_INGEST_WORKERS = int(os.environ.get("RAG_FOLDER_INGEST_WORKERS", "2"))
# Linked-folder indexing waits while a local chat generates, else it starves decode (#16992).
FOLDER_YIELD_TO_CHAT = os.environ.get("RAG_FOLDER_YIELD_TO_CHAT", "1") != "0"
# Small linked-folder files in flight, clamped to workers..64; past the workers they only wait to embed.
FOLDER_INGEST_IN_FLIGHT = int(os.environ.get("RAG_FOLDER_INGEST_IN_FLIGHT", "16"))
FOLDER_INGEST_BATCH_FILE_BYTES = int(
    os.environ.get("RAG_FOLDER_INGEST_BATCH_FILE_BYTES", str(1024 * 1024))
)

# Falls back to plain PyMuPDF text when off, when pymupdf4llm is missing, or when extraction fails.
PDF_MARKDOWN = os.environ.get("RAG_PDF_MARKDOWN", "1") == "1"

# Figure descriptions are opt-in; the chat toggle overrides this default.
CAPTION_IMAGES = os.environ.get("RAG_CAPTION_IMAGES", "0") == "1"
# Total per-document tile budget (figure-bearing pages are tiled, see below).
CAPTION_MAX_IMAGES = int(os.environ.get("RAG_CAPTION_MAX_IMAGES", "24"))
CAPTION_TIMEOUT_S = float(os.environ.get("RAG_CAPTION_TIMEOUT_S", "60"))
# Captions transcribe every label, and FIGURE_DPI keeps small box/axis labels legible when tiled.
CAPTION_MAX_TOKENS = int(os.environ.get("RAG_CAPTION_MAX_TOKENS", "768"))
FIGURE_DPI = int(os.environ.get("RAG_FIGURE_DPI", "200"))
# Overlapping tile grid covers sub-figures and small labels without exact region detection.
FIGURE_TILE_ROWS = int(os.environ.get("RAG_FIGURE_TILE_ROWS", "2"))
FIGURE_TILE_COLS = int(os.environ.get("RAG_FIGURE_TILE_COLS", "2"))
FIGURE_TILE_OVERLAP = float(os.environ.get("RAG_FIGURE_TILE_OVERLAP", "0.12"))
FIGURE_FULLPAGE = os.environ.get("RAG_FIGURE_FULLPAGE", "1") == "1"
CAPTION_MAX_PAGES = int(os.environ.get("RAG_CAPTION_MAX_PAGES", "4"))

# OCR uses the loaded vision model, else local Tesseract data; MIN_CHARS: pages under it get OCR.
OCR_SCANNED = os.environ.get("RAG_OCR_SCANNED", "1") == "1"
OCR_MIN_CHARS = int(os.environ.get("RAG_OCR_MIN_CHARS", "16"))
OCR_MAX_PAGES = int(os.environ.get("RAG_OCR_MAX_PAGES", "20"))
OCR_DPI = int(os.environ.get("RAG_OCR_DPI", "150"))
OCR_TIMEOUT_S = float(os.environ.get("RAG_OCR_TIMEOUT_S", "60"))
OCR_MAX_TOKENS = int(os.environ.get("RAG_OCR_MAX_TOKENS", "2048"))

# Switching backends changes the vectors, so the index must be rebuilt.
EMBED_BACKEND = os.environ.get("RAG_EMBED_BACKEND", "auto")

# The model name alone is not the embedding space: llama-server embeds through the GGUF companion and pools its own way.
EMBEDDING_IDENTITY_TAGS = ("sentence-transformers", "llama-server")


def _escape_identity_segment(value: str) -> str:
    """Colons separate the segments, and a model can be a local path that contains one
    (``C:\\models\\bge``), which would otherwise read back as ``C``. A repo id contains
    neither character, so the identity of a normal model is unchanged."""
    return value.replace("%", "%25").replace(":", "%3A")


def _unescape_identity_segment(value: str) -> str:
    return value.replace("%3A", ":").replace("%25", "%")


def embedding_identity(
    backend: str,
    model: str,
    *,
    gguf_repo: str | None = None,
    pooling: str | None = None,
) -> str:
    """The model name comes first, so untagged rows written before the tag still compare equal on it."""
    model = _escape_identity_segment(model)
    if gguf_repo is None:
        return f"{backend}:{model}"
    identity = f"{backend}:{model}:{_escape_identity_segment(gguf_repo)}"
    return identity if pooling is None else f"{identity}:{pooling}"


def embedding_identity_model(identity: str | None) -> str | None:
    """The configured model inside a tagged identity, or None when untagged."""
    for tag in EMBEDDING_IDENTITY_TAGS:
        if identity and identity.startswith(f"{tag}:"):
            return _unescape_identity_segment(identity[len(tag) + 1 :].split(":", 1)[0])
    return None


def embedding_identity_matches(stored: str | None, current: str) -> bool:
    """Untagged rows match on the model name alone; dropping them would empty dense search. They are reported instead (store.count_untagged_documents)."""
    if stored is None:
        return True
    if embedding_identity_model(stored) is not None:
        return stored == current
    return stored == (embedding_identity_model(current) or current)


def effective_embedding_model(fresh: bool = False) -> str:
    """The embedding model actually in use: the persisted Settings override when
    one is stored, else ``EMBEDDING_MODEL`` (env/default). Read at call time so a
    Settings change applies without a restart; ``fresh`` also skips the short cache and
    raises rather than fall back."""
    try:
        from utils.embedding_model_settings import get_rag_embedding_model
        return get_rag_embedding_model(fresh = fresh)
    except Exception:  # noqa: BLE001 - settings store unavailable (tests, early boot)
        if fresh:
            raise
        return EMBEDDING_MODEL


def _names_gguf(model: str) -> bool:
    """True when `gguf` is a whole name segment; substrings like `bigguf` do not count."""
    return "gguf" in re.split(r"[^a-z0-9]+", model.lower())


# Suffixes unsloth puts on an unquantized re-upload; the GGUF sits on the base name.
_QUANT_SUFFIX_RE = re.compile(r"(?:-qat)?(?:-q\d+_\d+[a-z]*)?-unquantized$", re.I)


def gguf_repo_candidates(model: str) -> list[str]:
    """Repos that may hold the GGUF, in preference order; loader and settings must agree."""
    if "RAG_EMBED_GGUF_REPO" in os.environ:
        return [EMBED_GGUF_REPO]
    if _names_gguf(model):
        return [model]
    owner, _, name = model.rpartition("/")
    out = [f"{model}-GGUF"]
    base = _QUANT_SUFFIX_RE.sub("", name)
    if base != name:
        out.append(f"{owner}/{base}-GGUF" if owner else f"{base}-GGUF")
    out.append(model)
    return list(dict.fromkeys(out))


def gguf_repo_is_explicit() -> bool:
    """Whether one GGUF repository was explicitly pinned for every embedder."""
    return "RAG_EMBED_GGUF_REPO" in os.environ


def gguf_repo_for_embedding_model(model: str) -> str:
    """GGUF repo for ``model``, honoring an explicit companion override."""
    if "RAG_EMBED_GGUF_REPO" in os.environ:
        return EMBED_GGUF_REPO
    if model == DEFAULT_EMBEDDING_MODEL:
        return EMBED_GGUF_REPO
    if _names_gguf(model):
        return model
    return f"{model}-GGUF"


def default_gguf_repo() -> str:
    """GGUF companion for the env/default embedding model."""
    return gguf_repo_for_embedding_model(EMBEDDING_MODEL)


def effective_gguf_repo() -> str:
    """RAG_EMBED_GGUF_REPO wins, then the repo stored for this model, then the -GGUF convention."""
    return effective_gguf_repo_for_embedding_model(effective_embedding_model())


def effective_gguf_repo_for_embedding_model(model: str) -> str:
    """Resolved repo is part of vector identity: two conversions of one model must not mix vectors."""
    if "RAG_EMBED_GGUF_REPO" in os.environ:
        return EMBED_GGUF_REPO
    try:
        from utils.embedding_model_settings import get_stored_gguf_repo, remembered_gguf_repo
        stored = get_stored_gguf_repo(model)
        if stored is None:
            # One stored record: saving another model would move a pinned job's identity mid-run.
            stored = remembered_gguf_repo(model)
    except Exception:  # noqa: BLE001 - store unavailable: fall back to the convention
        stored = None
    return stored or gguf_repo_for_embedding_model(model)


# F16 over Q8_0: faster (no per-block dequant at this size) and exact, for ~30MB more on disk.
EMBED_GGUF_REPO = os.environ.get("RAG_EMBED_GGUF_REPO", "unsloth/bge-small-en-v1.5-GGUF")
EMBED_GGUF_VARIANT = os.environ.get("RAG_EMBED_GGUF_VARIANT", "F16")
# "auto": llama-server offloads in its subprocess; sentence-transformers stays on CPU (CUDA ctx cost).
EMBED_DEVICE = os.environ.get("RAG_EMBED_DEVICE", "auto")


def _env_embed_device() -> str:
    """``EMBED_DEVICE`` normalized to exactly ``gpu``, ``cpu`` or ``auto``.

    One reader for both backends, because they used to disagree about the same
    string: the llama path compared a bare ``.lower()``, so ``" gpu "`` fell through
    to auto, and an Intel user writing the accelerator's own name (``xpu``) got CPU
    from a setting that named their device. Anything that is not recognizably a
    request for CPU or for an accelerator is ``auto``, so a typo degrades to each
    backend's default rather than to silence.
    """
    value = (EMBED_DEVICE or "").strip().lower()
    if value in ("gpu", "cuda", "rocm", "hip", "xpu", "mps", "metal"):
        return "gpu"
    if value == "cpu":
        return "cpu"
    return "auto"


def embed_device_locked() -> bool:
    """Whether ``RAG_EMBED_DEVICE`` names a device, which Settings then cannot override."""
    return _env_embed_device() != "auto"


def embed_device_preference() -> str:
    """``gpu``, ``cpu`` or ``auto``: ``RAG_EMBED_DEVICE`` when it names a device, else the Settings choice."""
    env = _env_embed_device()
    if env != "auto":
        return env
    try:
        from utils.embedding_model_settings import get_stored_embedding_device
        return get_stored_embedding_device()
    except Exception:  # noqa: BLE001 - store unavailable: each backend's default
        return "auto"


def embed_device_requires_gpu() -> bool:
    """Only literal gpu makes a failed GPU start raise, not retry on CPU. Newer spellings like xpu opt in but do not insist, else RAG is lost on those hosts."""
    return (EMBED_DEVICE or "").lower() == "gpu"


EMBED_HOST = os.environ.get("RAG_EMBED_HOST", "127.0.0.1")
EMBED_PORT = int(os.environ.get("RAG_EMBED_PORT", "0"))
EMBED_BATCH = int(os.environ.get("RAG_EMBED_BATCH", "64"))
EMBED_STARTUP_TIMEOUT_S = float(os.environ.get("RAG_EMBED_STARTUP_TIMEOUT_S", "120"))
EMBED_REQUEST_TIMEOUT_S = float(os.environ.get("RAG_EMBED_REQUEST_TIMEOUT_S", "60"))