~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.
Python academiaanthropicarxivbravedeep-researchencryptionhome-automationhomeserverlocallocal-deep-researchlocal-llmmistralollamaopenaipubmedresearchresearch-toolretrieval-augmented-generationsearxngself-hosted
"""unstructured's import-time usage request must stay off in LDR processes."""
import json
import os
import subprocess
import sys
from local_deep_research.config.unstructured_telemetry import (
opt_out_of_unstructured_telemetry,
)
TELEMETRY_URL_PREFIX = "https://packages.unstructured.io/python-telemetry?"
# Runs in a fresh interpreter, where unstructured has not been imported yet.
# requests.get only records, so nothing is sent whichever way this goes.
CHILD_SCRIPT = """
import json, os, sys
import requests
calls = []
requests.get = lambda *args, **kwargs: calls.append(
args[0] if args else kwargs.get("url")
)
report_path, import_ldr_first = sys.argv[1], sys.argv[2] == "yes"
if import_ldr_first:
import local_deep_research
import unstructured.logger
with open(report_path, "w") as handle:
json.dump(
{"calls": calls, "opt_out": os.environ.get("SCARF_NO_ANALYTICS")},
handle,
)
"""
def _import_unstructured_logger(tmp_path, *, import_ldr_first):
# Importing LDR in this process already set the opt-out, so the child
# must not inherit it: it starts with neither opt-out variable set.
report = tmp_path / "report.json"
env = {
name: value
for name, value in os.environ.items()
if name not in ("SCARF_NO_ANALYTICS", "DO_NOT_TRACK")
}
result = subprocess.run(
[
sys.executable,
"-c",
CHILD_SCRIPT,
str(report),
"yes" if import_ldr_first else "no",
],
env=env,
check=False,
capture_output=True,
timeout=120,
)
assert result.returncode == 0, result.stderr[-2000:]
return json.loads(report.read_text())
def test_opt_out_overwrites_a_value_unstructured_does_not_honour():
# unstructured opts out only on exactly "true", so "1" would still send
# the request. DO_NOT_TRACK is not touched: other libraries read it.
env = {"SCARF_NO_ANALYTICS": "1", "DO_NOT_TRACK": "1"}
opt_out_of_unstructured_telemetry(env)
assert env == {"SCARF_NO_ANALYTICS": "true", "DO_NOT_TRACK": "1"}
def test_importing_ldr_opts_out_before_unstructured_is_imported(tmp_path):
# The server, or any program that imports LDR, imports unstructured
# lazily, at its first unstructured-backed parse. A fresh interpreter
# that imports LDR first and unstructured.logger second stands in for
# one.
assert _import_unstructured_logger(tmp_path, import_ldr_first=True) == {
"calls": [],
"opt_out": "true",
}
def test_unstructured_sends_the_request_without_the_opt_out(tmp_path):
# Control for the test above: without LDR, the same recording sees the
# request. If unstructured stopped sending it through requests.get, the
# test above would pass without proving anything; this one fails then.
report = _import_unstructured_logger(tmp_path, import_ldr_first=False)
assert report["opt_out"] is None
assert len(report["calls"]) == 1
assert report["calls"][0].startswith(TELEMETRY_URL_PREFIX)