Skip to content

~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.

3.0 KB
Raw Blame
"""unstructured's import-time usage request must stay off in LDR processes."""

import json
import os
import subprocess
import sys

from local_deep_research.config.unstructured_telemetry import (
    opt_out_of_unstructured_telemetry,
)

TELEMETRY_URL_PREFIX = "https://packages.unstructured.io/python-telemetry?"

# Runs in a fresh interpreter, where unstructured has not been imported yet.
# requests.get only records, so nothing is sent whichever way this goes.
CHILD_SCRIPT = """
import json, os, sys

import requests

calls = []
requests.get = lambda *args, **kwargs: calls.append(
    args[0] if args else kwargs.get("url")
)

report_path, import_ldr_first = sys.argv[1], sys.argv[2] == "yes"
if import_ldr_first:
    import local_deep_research
import unstructured.logger

with open(report_path, "w") as handle:
    json.dump(
        {"calls": calls, "opt_out": os.environ.get("SCARF_NO_ANALYTICS")},
        handle,
    )
"""


def _import_unstructured_logger(tmp_path, *, import_ldr_first):
    # Importing LDR in this process already set the opt-out, so the child
    # must not inherit it: it starts with neither opt-out variable set.
    report = tmp_path / "report.json"
    env = {
        name: value
        for name, value in os.environ.items()
        if name not in ("SCARF_NO_ANALYTICS", "DO_NOT_TRACK")
    }
    result = subprocess.run(
        [
            sys.executable,
            "-c",
            CHILD_SCRIPT,
            str(report),
            "yes" if import_ldr_first else "no",
        ],
        env=env,
        check=False,
        capture_output=True,
        timeout=120,
    )
    assert result.returncode == 0, result.stderr[-2000:]
    return json.loads(report.read_text())


def test_opt_out_overwrites_a_value_unstructured_does_not_honour():
    # unstructured opts out only on exactly "true", so "1" would still send
    # the request. DO_NOT_TRACK is not touched: other libraries read it.
    env = {"SCARF_NO_ANALYTICS": "1", "DO_NOT_TRACK": "1"}
    opt_out_of_unstructured_telemetry(env)
    assert env == {"SCARF_NO_ANALYTICS": "true", "DO_NOT_TRACK": "1"}


def test_importing_ldr_opts_out_before_unstructured_is_imported(tmp_path):
    # The server, or any program that imports LDR, imports unstructured
    # lazily, at its first unstructured-backed parse. A fresh interpreter
    # that imports LDR first and unstructured.logger second stands in for
    # one.
    assert _import_unstructured_logger(tmp_path, import_ldr_first=True) == {
        "calls": [],
        "opt_out": "true",
    }


def test_unstructured_sends_the_request_without_the_opt_out(tmp_path):
    # Control for the test above: without LDR, the same recording sees the
    # request. If unstructured stopped sending it through requests.get, the
    # test above would pass without proving anything; this one fails then.
    report = _import_unstructured_logger(tmp_path, import_ldr_first=False)
    assert report["opt_out"] is None
    assert len(report["calls"]) == 1
    assert report["calls"][0].startswith(TELEMETRY_URL_PREFIX)