~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.
Python academiaanthropicarxivbravedeep-researchencryptionhome-automationhomeserverlocallocal-deep-researchlocal-llmmistralollamaopenaipubmedresearchresearch-toolretrieval-augmented-generationsearxngself-hosted
"""Limits checked before tomllib parses TOML that a user supplied.
tomllib is pure Python. It rebuilds a key's tuple for every part and, on a
dotted key/value line, also builds and keeps every prefix of the key, so
one long key costs time (and, on a key/value line, memory) that grows with
the square of its length. Uploaded TOML documents and the raw configuration
endpoint both parse through ``loads_toml_within_limits``.
"""
import re
import tomllib
from itertools import islice
from typing import Any
# Largest TOML document parsed, in UTF-8 bytes. TOML files are mostly
# configuration: a full pyproject.toml is about 30 KB, and gitleaks' rule
# set, one of the larger common ones, about 100 KB, so this leaves fivefold
# headroom. tomllib is pure Python, and hostile but valid documents were
# measured holding several hundred bytes of parser memory per input byte;
# neither the general upload limit (3 GB by default) nor the default 16 MB
# JSON body limit bounds that.
MAX_TOML_BYTES = 512 * 1024
# Most dotted parts allowed in one key or table header ("a.b.c" has three).
# The deepest key in 252 sampled TOML files had five parts. Under this cap
# the total work stays proportional to the document size. A number such as
# 1.5 also scans as a two-part run, so keep this at least 2.
MAX_TOML_KEY_PARTS = 32
# One key part: bare, or a one-line basic or literal string that ends
# where tomllib ends it.
_KEY_PART = (
r"(?:[A-Za-z0-9_-]++"
r"|\x22(?:[^\x22\\\n]++|\\[^\n])*+\x22"
r"|\x27[^\x27\n]*+\x27)"
)
_KEY_PART_RE = re.compile(_KEY_PART)
# Some alternative matches at every position, so the tokens cover the text
# in order and, up to where tomllib would reject the document, string and
# comment contents are skipped exactly where tomllib skips them. A "key"
# token is a run of dot-separated key parts.
_TOKEN_RE = re.compile(
rf"""
(?P<string>
\x22{{3}} (?: [^\x22\\]++ | \\[\s\S] | \x22(?!\x22{{2}}) )*+ \x22{{3,5}}
| \x27{{3}} (?: [^\x27]++ | \x27(?!\x27{{2}}) )*+ \x27{{3,5}}
)
| (?P<key>
(?!\x22{{3}}|\x27{{3}})
{_KEY_PART} (?: [ \t]*+ \. [ \t]*+ {_KEY_PART} )*+
)
| (?P<comment> \# [^\n]*+ )
| (?P<other> [^A-Za-z0-9_\-\x22\x27\#]++ )
| (?P<unterminated> [\x22\x27] )
""",
re.VERBOSE,
)
class TOMLLimitError(ValueError):
"""TOML input exceeds a limit that is checked before tomllib runs."""
def _has_overlong_key(text: str, max_parts: int) -> bool:
"""Return whether a key or table header has more than *max_parts* parts.
The scan stops at an unterminated string, where tomllib rejects the
document, so it stays linear in the length of *text*.
"""
for token in _TOKEN_RE.finditer(text):
kind = token.lastgroup
if kind == "unterminated":
return False
start, end = token.span()
if kind == "key" and text.find(".", start, end) != -1:
parts = _KEY_PART_RE.finditer(text, start, end)
if sum(1 for _ in islice(parts, max_parts + 1)) > max_parts:
return True
return False
def check_toml_size(size: int) -> None:
"""Raise TOMLLimitError when a *size*-byte document is over the limit."""
if size > MAX_TOML_BYTES:
raise TOMLLimitError(
f"TOML document is larger than {MAX_TOML_BYTES} bytes"
)
def loads_toml_within_limits(text: str) -> dict[str, Any]:
"""Parse *text* with tomllib unless it exceeds the limits.
Raises TOMLLimitError, before tomllib runs, when the UTF-8 form of
*text* is larger than MAX_TOML_BYTES or a key or table header has more
than MAX_TOML_KEY_PARTS dotted parts, and TypeError when *text* is not
a string. Otherwise it returns or raises what
tomllib.loads does, such as TOMLDecodeError for invalid TOML.
"""
if not isinstance(text, str):
raise TypeError(f"Expected str object, not '{type(text).__qualname__}'")
# A character takes at least one UTF-8 byte, so the length check refuses
# most oversized text before its encoded size is measured.
check_toml_size(len(text))
check_toml_size(len(text.encode("utf-8", "surrogatepass")))
# tomllib.loads folds CRLF before it parses; scan that same text.
text = text.replace("\r\n", "\n")
if _has_overlong_key(text, MAX_TOML_KEY_PARTS):
raise TOMLLimitError(
"TOML key or table header has more than "
f"{MAX_TOML_KEY_PARTS} dotted parts"
)
return tomllib.loads(text)