~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.
Python academiaanthropicarxivbravedeep-researchencryptionhome-automationhomeserverlocallocal-deep-researchlocal-llmmistralollamaopenaipubmedresearchresearch-toolretrieval-augmented-generationsearxngself-hosted
"""Archive provenance survives live and persisted Sources reconstruction."""
from copy import deepcopy
from types import SimpleNamespace
from unittest.mock import Mock
import pytest
from local_deep_research.advanced_search_system.strategies.base_strategy import (
BaseSearchStrategy,
)
from local_deep_research.advanced_search_system.strategies.langgraph_agent_strategy import (
SearchResultsCollector,
)
from local_deep_research.advanced_search_system.tools.fetch import (
_register_in_collector,
)
from local_deep_research.database.models.research import (
ResearchHistory,
ResearchResource,
)
from local_deep_research.utilities.search_utilities import (
extract_links_from_search_results,
)
from local_deep_research.utilities.url_utils import FETCHED_TITLE_KEY
from local_deep_research.web.services.report_assembly_service import (
assemble_full_report,
)
from local_deep_research.web_search_engines.wayback_client import (
validate_capture_fetch_url,
)
CAPTURE = "20240203112233"
DATE = "2024-02-03 11:22:33 UTC"
TARGET = (
"https://example.com/owner's/history/"
"?utm_source=first&fbclid=one&next=https://nested.example/path/#part"
)
REPLAY = f"https://web.archive.org/web/{CAPTURE}/{TARGET}"
ARCHIVE_METADATA = {
"source_type": "archive",
"original_url": TARGET,
"requested_url": TARGET.replace(
"https://example.com", "http://www.example.com"
),
"timestamp": CAPTURE,
"formatted_date": DATE,
}
CITATION_CASES = [
pytest.param(True, 1, id="same-excerpt-upgraded-index"),
pytest.param(False, 1, id="different-excerpt-original-index"),
pytest.param(False, 2, id="different-excerpt-archive-index"),
]
def _archived_results(same_excerpt, cited_index, reset):
collector = SearchResultsCollector([])
assert (
_register_in_collector(
collector, REPLAY, "First fetched title", "Original excerpt"
)
== 1
)
archive_index = _register_in_collector(
collector,
REPLAY,
"Historical page",
"Original excerpt" if same_excerpt else "Different archived excerpt",
ARCHIVE_METADATA,
)
assert archive_index == (1 if same_excerpt else 2)
assert collector.find_by_index(1)["snippet"] == "Original excerpt"
assert collector.find_by_index(archive_index)[FETCHED_TITLE_KEY] == (
"First fetched title"
)
if reset:
tracked = collector.find_by_index(cited_index)
collector.reset()
assert (
collector.find_or_add_result(
{
"link": REPLAY,
"title": "Later generic title",
"snippet": tracked["snippet"],
}
)
== cited_index
)
return collector.results
def _assert_archive_sources(output, cited_index):
assert output.count("## Sources") == 1
sources = output.split("## Sources\n\n", 1)[1]
assert sources.startswith(
f"[{cited_index}] First fetched title (archived {DATE}) "
)
assert sources.count(f"(archived {DATE})") == 1
displayed = [
line.removeprefix(" URL: ")
for line in sources.splitlines()
if line.startswith(" URL: ")
]
assert displayed == [REPLAY]
validate_capture_fetch_url(displayed[0], CAPTURE, TARGET)
@pytest.mark.parametrize("same_excerpt,cited_index", CITATION_CASES)
@pytest.mark.parametrize(
"reset", [False, True], ids=["same-section", "after-reset"]
)
def test_live_sources_preserve_archive_date_after_metadata_rebuild(
same_excerpt, cited_index, reset
):
results = _archived_results(same_excerpt, cited_index, reset)
before = deepcopy(results)
settings_owner = SimpleNamespace(get_setting=Mock(return_value="strict"))
body = f"Historical finding [{cited_index}]."
output = BaseSearchStrategy._format_citations(settings_owner, body, results)
_assert_archive_sources(output, cited_index)
settings_owner.get_setting.assert_called_once_with(
"report.uncited_sources_mode"
)
assert results == before
@pytest.mark.parametrize("same_excerpt,cited_index", CITATION_CASES)
@pytest.mark.parametrize(
"reset", [False, True], ids=["same-section", "after-reset"]
)
def test_saved_report_preserves_archive_date_after_json_round_trip(
report_db, same_excerpt, cited_index, reset
):
results = _archived_results(same_excerpt, cited_index, reset)
research = report_db.get(ResearchHistory, "report-errors")
body = f"Historical finding [{cited_index}]."
research.report_content = body
research.research_meta = {
"settings_snapshot": {"report.uncited_sources_mode": "strict"}
}
for result in results:
report_db.add(
ResearchResource(
research_id=research.id,
title=result["title"],
url=result["link"],
source_type=result["source_type"],
resource_metadata={"original_data": dict(result)},
created_at="2026-10-11T00:00:00+00:00",
)
)
report_db.commit()
report_db.expire_all()
output = assemble_full_report(research, report_db)
_assert_archive_sources(output, cited_index)
assert research.report_content == body
resources = report_db.query(ResearchResource).all()
assert all(
resource.resource_metadata["original_data"]["formatted_date"] == DATE
for resource in resources
)
def test_live_link_extraction_preserves_complete_archive_provenance():
results = _archived_results(False, 2, False)
before = deepcopy(results)
extracted = extract_links_from_search_results(results)
assert [link["index"] for link in extracted] == ["1", "2"]
assert all(link["url"] == REPLAY for link in extracted)
for link in extracted:
assert all(
link[field] == value for field, value in ARCHIVE_METADATA.items()
)
assert results == before
def test_live_sources_keep_existing_url_normalization():
result = {
"link": "HTTPS://EXAMPLE.COM:443/current/?utm_source=ignored&v=1#part",
"title": "Current page",
"index": "1",
}
settings_owner = SimpleNamespace(get_setting=lambda key: "strict")
output = BaseSearchStrategy._format_citations(
settings_owner, "Current finding [1].", [result]
)
assert " URL: https://example.com/current?v=1\n" in output
assert "archived" not in output
REPLAY_TARGETS = [
"http://[::ffff:8.8.8.8]:80/owner's/history/"
"?utm_source=first&next=https://other.example/a/#part",
"https://example.com/history/[Part]/"
"?next=https://[2001:4860:4860::8888]:8443/a/&q[range]=[alpha]#part",
"https://example.org/a[b[c]]?values=[x[y]]",
"https://[2001:4860:4860::8888]:8443/a[b[c]]"
"?values=[x[y]]&next=https://[2606:4700:4700::1111]/a/#part",
]
REPLAY_TEMPLATES = [
"Historical finding {url}. See [4].",
"Historical finding [History]({url}). See [4].",
"Historical finding [History](<{url}>). See [4].",
"Historical finding {url}[4].",
]
def _url_cited_archive_results(target):
replay = f"https://web.archive.org/web/{CAPTURE}/{target}"
archive = {
"link": replay,
"title": "Historical page",
"index": "7",
FETCHED_TITLE_KEY: "First fetched title",
"source_type": "archive",
"original_url": target,
"requested_url": target,
"timestamp": CAPTURE,
"formatted_date": DATE,
}
later = dict(
archive,
link=f"https://web.archive.org/web/20250304123456/{target}",
index="8",
timestamp="20250304123456",
formatted_date="2025-03-04 12:34:56 UTC",
)
other = {
"link": "https://other.example/four",
"title": "Other source",
"source_type": "web",
"index": "4",
}
return [archive, later, other]
def _assert_url_cited_archive_sources(output, results):
sources = output.split("## Sources\n\n", 1)[1]
assert f"[7] First fetched title (archived {DATE}) " in sources
assert "[4] Other source " in sources
assert "[8]" not in sources
assert "2025-03-04" not in sources
assert sources.count(f"(archived {DATE})") == 1
displayed = [
line.removeprefix(" URL: ")
for line in sources.splitlines()
if line.startswith(" URL: ")
]
assert displayed == [results[0]["link"], results[2]["link"]]
validate_capture_fetch_url(
displayed[0], CAPTURE, results[0]["original_url"]
)
@pytest.mark.parametrize("target", REPLAY_TARGETS)
@pytest.mark.parametrize("template", REPLAY_TEMPLATES)
def test_live_sources_keep_url_cited_replay_with_ipv6_and_other_index(
target, template
):
results = _url_cited_archive_results(target)
before = deepcopy(results)
body = template.format(url=results[0]["link"])
owner = SimpleNamespace(get_setting=lambda key: "strict")
output = BaseSearchStrategy._format_citations(owner, body, results)
_assert_url_cited_archive_sources(output, results)
assert results == before
@pytest.mark.parametrize("target", REPLAY_TARGETS)
@pytest.mark.parametrize("template", REPLAY_TEMPLATES)
def test_saved_sources_keep_url_cited_replay_with_ipv6_and_other_index(
report_db, target, template
):
results = _url_cited_archive_results(target)
research = report_db.get(ResearchHistory, "report-errors")
body = template.format(url=results[0]["link"])
research.report_content = body
research.research_meta = {
"settings_snapshot": {"report.uncited_sources_mode": "strict"}
}
for result in results:
report_db.add(
ResearchResource(
research_id=research.id,
title=result["title"],
url=result["link"],
source_type=result["source_type"],
resource_metadata={"original_data": dict(result)},
created_at="2026-10-11T00:00:00+00:00",
)
)
report_db.commit()
report_db.expire_all()
output = assemble_full_report(research, report_db)
_assert_url_cited_archive_sources(output, results)
assert research.report_content == body
def _assert_adjacent_capture_sources(output, results):
sources = output.split("## Sources\n\n", 1)[1]
assert f"[7] First fetched title (archived {DATE}) " in sources
assert (
"[8] First fetched title (archived 2025-03-04 12:34:56 UTC) " in sources
)
assert "[4] Other source " in sources
displayed = [
line.removeprefix(" URL: ")
for line in sources.splitlines()
if line.startswith(" URL: ")
]
assert displayed == [result["link"] for result in results]
for result in results[:2]:
validate_capture_fetch_url(
result["link"], result["timestamp"], result["original_url"]
)
def test_live_sources_keep_adjacent_markdown_capture_links():
target = "https://[2001:4860:4860::8888]:8443/a_(history)[b[c]]?values=[x[y]]&doc=(a)"
results = _url_cited_archive_results(target)
body = f"[old]({results[0]['link']})[new]({results[1]['link']}). See [4]."
owner = SimpleNamespace(get_setting=lambda key: "strict")
output = BaseSearchStrategy._format_citations(owner, body, results)
_assert_adjacent_capture_sources(output, results)
def test_saved_sources_keep_adjacent_markdown_capture_links(report_db):
target = "https://[2001:4860:4860::8888]:8443/a_(history)[b[c]]?values=[x[y]]&doc=(a)"
results = _url_cited_archive_results(target)
research = report_db.get(ResearchHistory, "report-errors")
body = f"[old]({results[0]['link']})[new]({results[1]['link']}). See [4]."
research.report_content = body
research.research_meta = {
"settings_snapshot": {"report.uncited_sources_mode": "strict"}
}
for result in results:
report_db.add(
ResearchResource(
research_id=research.id,
title=result["title"],
url=result["link"],
source_type=result["source_type"],
resource_metadata={"original_data": dict(result)},
created_at="2026-10-11T00:00:00+00:00",
)
)
report_db.commit()
report_db.expire_all()
output = assemble_full_report(research, report_db)
_assert_adjacent_capture_sources(output, results)
assert research.report_content == body