Skip to content

~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.

4.4 KB
Raw Blame
"""Limits checked before tomllib parses TOML that a user supplied.

tomllib is pure Python. It rebuilds a key's tuple for every part and, on a
dotted key/value line, also builds and keeps every prefix of the key, so
one long key costs time (and, on a key/value line, memory) that grows with
the square of its length. Uploaded TOML documents and the raw configuration
endpoint both parse through ``loads_toml_within_limits``.
"""

import re
import tomllib
from itertools import islice
from typing import Any

# Largest TOML document parsed, in UTF-8 bytes. TOML files are mostly
# configuration: a full pyproject.toml is about 30 KB, and gitleaks' rule
# set, one of the larger common ones, about 100 KB, so this leaves fivefold
# headroom. tomllib is pure Python, and hostile but valid documents were
# measured holding several hundred bytes of parser memory per input byte;
# neither the general upload limit (3 GB by default) nor the default 16 MB
# JSON body limit bounds that.
MAX_TOML_BYTES = 512 * 1024

# Most dotted parts allowed in one key or table header ("a.b.c" has three).
# The deepest key in 252 sampled TOML files had five parts. Under this cap
# the total work stays proportional to the document size. A number such as
# 1.5 also scans as a two-part run, so keep this at least 2.
MAX_TOML_KEY_PARTS = 32

# One key part: bare, or a one-line basic or literal string that ends
# where tomllib ends it.
_KEY_PART = (
    r"(?:[A-Za-z0-9_-]++"
    r"|\x22(?:[^\x22\\\n]++|\\[^\n])*+\x22"
    r"|\x27[^\x27\n]*+\x27)"
)
_KEY_PART_RE = re.compile(_KEY_PART)

# Some alternative matches at every position, so the tokens cover the text
# in order and, up to where tomllib would reject the document, string and
# comment contents are skipped exactly where tomllib skips them. A "key"
# token is a run of dot-separated key parts.
_TOKEN_RE = re.compile(
    rf"""
    (?P<string>
        \x22{{3}} (?: [^\x22\\]++ | \\[\s\S] | \x22(?!\x22{{2}}) )*+ \x22{{3,5}}
      | \x27{{3}} (?: [^\x27]++ | \x27(?!\x27{{2}}) )*+ \x27{{3,5}}
    )
  | (?P<key>
        (?!\x22{{3}}|\x27{{3}})
        {_KEY_PART} (?: [ \t]*+ \. [ \t]*+ {_KEY_PART} )*+
    )
  | (?P<comment> \# [^\n]*+ )
  | (?P<other> [^A-Za-z0-9_\-\x22\x27\#]++ )
  | (?P<unterminated> [\x22\x27] )
    """,
    re.VERBOSE,
)


class TOMLLimitError(ValueError):
    """TOML input exceeds a limit that is checked before tomllib runs."""


def _has_overlong_key(text: str, max_parts: int) -> bool:
    """Return whether a key or table header has more than *max_parts* parts.

    The scan stops at an unterminated string, where tomllib rejects the
    document, so it stays linear in the length of *text*.
    """
    for token in _TOKEN_RE.finditer(text):
        kind = token.lastgroup
        if kind == "unterminated":
            return False
        start, end = token.span()
        if kind == "key" and text.find(".", start, end) != -1:
            parts = _KEY_PART_RE.finditer(text, start, end)
            if sum(1 for _ in islice(parts, max_parts + 1)) > max_parts:
                return True
    return False


def check_toml_size(size: int) -> None:
    """Raise TOMLLimitError when a *size*-byte document is over the limit."""
    if size > MAX_TOML_BYTES:
        raise TOMLLimitError(
            f"TOML document is larger than {MAX_TOML_BYTES} bytes"
        )


def loads_toml_within_limits(text: str) -> dict[str, Any]:
    """Parse *text* with tomllib unless it exceeds the limits.

    Raises TOMLLimitError, before tomllib runs, when the UTF-8 form of
    *text* is larger than MAX_TOML_BYTES or a key or table header has more
    than MAX_TOML_KEY_PARTS dotted parts, and TypeError when *text* is not
    a string. Otherwise it returns or raises what
    tomllib.loads does, such as TOMLDecodeError for invalid TOML.
    """
    if not isinstance(text, str):
        raise TypeError(f"Expected str object, not '{type(text).__qualname__}'")
    # A character takes at least one UTF-8 byte, so the length check refuses
    # most oversized text before its encoded size is measured.
    check_toml_size(len(text))
    check_toml_size(len(text.encode("utf-8", "surrogatepass")))
    # tomllib.loads folds CRLF before it parses; scan that same text.
    text = text.replace("\r\n", "\n")
    if _has_overlong_key(text, MAX_TOML_KEY_PARTS):
        raise TOMLLimitError(
            "TOML key or table header has more than "
            f"{MAX_TOML_KEY_PARTS} dotted parts"
        )
    return tomllib.loads(text)