~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.
Python academiaanthropicarxivbravedeep-researchencryptionhome-automationhomeserverlocallocal-deep-researchlocal-llmmistralollamaopenaipubmedresearchresearch-toolretrieval-augmented-generationsearxngself-hosted
"""Tests for research-related database models."""
import json
import uuid
from datetime import datetime, timezone
import pytest
from sqlalchemy import create_engine, text
from sqlalchemy.orm import sessionmaker
from local_deep_research.database.models import (
Base,
Research,
ResearchHistory,
ResearchMode,
ResearchResource,
ResearchStatus,
ResearchStrategy,
ResearchTask,
SearchQuery,
SearchResult,
)
class TestResearchModels:
"""Test suite for research-related models."""
@pytest.fixture
def engine(self):
"""Create an in-memory SQLite database for testing."""
engine = create_engine("sqlite:///:memory:")
Base.metadata.create_all(engine)
yield engine
engine.dispose()
@pytest.fixture
def session(self, engine):
"""Create a database session for testing."""
Session = sessionmaker(bind=engine)
session = Session()
yield session
session.close()
def test_research_history_creation(self, session):
"""Test creating a ResearchHistory record."""
research = ResearchHistory(
id=str(uuid.uuid4()),
query="What is quantum computing?",
mode="comprehensive",
status="completed",
created_at="2024-01-01T12:00:00",
completed_at="2024-01-01T12:05:00",
duration_seconds=300,
progress=100,
report_path="/reports/quantum_computing.md",
title="Quantum Computing Research",
research_meta={"sources": 10, "quality": "high"},
)
session.add(research)
session.commit()
# Verify the record
saved = session.query(ResearchHistory).first()
assert saved is not None
assert saved.query == "What is quantum computing?"
assert saved.mode == "comprehensive"
assert saved.status == "completed"
assert saved.duration_seconds == 300
assert saved.progress == 100
assert saved.research_meta["sources"] == 10
def test_research_status_enum(self, session):
"""Test Research model with ResearchStatus enum."""
# Create Research with enum status
research = Research(
query="Test query",
status=ResearchStatus.PENDING,
mode=ResearchMode.QUICK,
)
session.add(research)
session.commit()
# Test status transitions
research.status = ResearchStatus.IN_PROGRESS
session.commit()
assert research.status == ResearchStatus.IN_PROGRESS
# Test all status values
for status in ResearchStatus:
r = Research(
query=f"Test {status.value}",
status=status,
mode=ResearchMode.QUICK,
)
session.add(r)
session.commit()
# Verify all statuses
all_research = session.query(Research).all()
assert len(all_research) >= len(ResearchStatus)
def test_research_progress_log(self, session):
"""Test progress log JSON field in ResearchHistory."""
progress_log = {
"steps": [
{"time": "2024-01-01T12:00:00", "message": "Starting research"},
{"time": "2024-01-01T12:01:00", "message": "Searching sources"},
{"time": "2024-01-01T12:03:00", "message": "Analyzing results"},
{"time": "2024-01-01T12:05:00", "message": "Generating report"},
],
"current_step": 4,
"total_steps": 4,
}
research = ResearchHistory(
id=str(uuid.uuid4()),
query="AI research",
mode="normal",
status="completed",
created_at="2024-01-01T12:00:00",
progress_log=progress_log,
progress=100,
)
session.add(research)
session.commit()
# Verify progress log
saved = session.query(ResearchHistory).first()
assert saved.progress_log is not None
assert len(saved.progress_log["steps"]) == 4
assert saved.progress_log["current_step"] == 4
@pytest.mark.parametrize(
"stored_log",
[
"{'steps': []}", # str(dict) repr, never valid JSON
"", # empty TEXT from an interrupted legacy write
"{truncated", # truncated json.dumps output
],
ids=["str-repr", "empty-string", "truncated"],
)
def test_progress_log_unparseable_legacy_row_loads_as_absent(
self, session, stored_log
):
"""A legacy TEXT row that is not valid JSON must not break the load.
The stock ``JSON`` result processor raised ``JSONDecodeError``
while the row was being loaded, above every handler ``try``, so
``GET /history/status/{research_id}`` answered 500 for such a row
instead of degrading to an empty log (#6533). The column now reads
the value as ``None`` — the "no log" value every reader already
handles — and the load succeeds.
"""
research_id = f"legacy-{stored_log[:6] or 'empty'}"
session.execute(
text(
"INSERT INTO research_history "
"(id, query, mode, status, created_at, progress_log) "
"VALUES (:id, :query, :mode, :status, :created_at, :log)"
),
{
"id": research_id,
"query": "legacy row",
"mode": "detailed",
"status": "completed",
"created_at": "2024-01-01T12:00:00",
"log": stored_log,
},
)
session.commit()
loaded = (
session.query(ResearchHistory).filter_by(id=research_id).first()
)
assert loaded is not None
assert loaded.progress_log is None
# What the status endpoint does with the value: an empty log.
assert json.loads(loaded.progress_log or "[]") == []
@pytest.mark.parametrize(
"stored_meta",
["{'subscription': ", "", "{truncated"],
ids=["str-repr", "empty-string", "truncated"],
)
def test_research_meta_unparseable_row_does_not_hide_history(
self, session, stored_meta
):
"""One corrupt legacy metadata cell must not break row or list loads."""
bad_id = f"legacy-meta-{stored_meta[:6] or 'empty'}"
session.execute(
text(
"INSERT INTO research_history "
"(id, query, mode, status, created_at, research_meta) "
"VALUES (:id, :query, :mode, :status, :created_at, :meta)"
),
{
"id": bad_id,
"query": "corrupt legacy row",
"mode": "detailed",
"status": "completed",
"created_at": "2024-01-01T12:00:00",
"meta": stored_meta,
},
)
good = ResearchHistory(
id="good-neighbor",
query="intact row",
mode="detailed",
status="completed",
created_at="2024-01-02T12:00:00",
research_meta={"headline": "still visible"},
)
session.add(good)
session.commit()
bad = session.query(ResearchHistory).filter_by(id=bad_id).first()
assert bad is not None
assert bad.research_meta is None
# The history endpoint selects this column for every listed row.
rows = session.query(
ResearchHistory.id, ResearchHistory.research_meta
).all()
assert {row.id: row.research_meta for row in rows} == {
bad_id: None,
"good-neighbor": {"headline": "still visible"},
}
def test_research_meta_subscription_like_filter_keeps_stored_format(
self, session
):
"""The news subscription lookup still matches JSON's TEXT form."""
session.add(
ResearchHistory(
id="subscription-match",
query="subscription run",
mode="detailed",
status="completed",
created_at="2024-01-01T12:00:00",
research_meta={"subscription_id": "sub-42"},
)
)
session.commit()
matched = (
session.query(ResearchHistory.id)
.filter(
ResearchHistory.research_meta.like(
'%"subscription_id": "sub-42"%', escape="\\"
)
)
.all()
)
assert [row.id for row in matched] == ["subscription-match"]
indexed = (
session.query(ResearchHistory.id)
.filter(
ResearchHistory.research_meta["subscription_id"].as_string()
== "sub-42"
)
.all()
)
assert [row.id for row in indexed] == ["subscription-match"]
def test_lenient_json_keeps_sqlite_column_types(self, session):
"""The read decorator must not need a schema migration."""
types = {
row.name: row.type
for row in session.execute(
text("PRAGMA table_info(research_history)")
)
}
assert types["progress_log"] == "JSON"
assert types["research_meta"] == "JSON"
@pytest.mark.parametrize(
"stored_scalar",
[5, 0, True, False],
ids=["int", "zero", "true", "false"],
)
def test_lenient_json_passes_a_native_scalar_carrier_through(
self, session, stored_scalar
):
"""A scalar the driver already decoded is returned untouched.
``progress_log`` is ``JSON``-declared, so an ORM write of a scalar
binds ``json.dumps(scalar)`` and SQLite's NUMERIC affinity stores
``5`` / ``0`` as integers, handing the result processor a real
``int``. Feeding that to ``json.loads`` raises ``TypeError``, which
the first cut of this gate caught and turned into ``None`` — a
perfectly valid value silently replaced by the absent one. The gate
now decodes only TEXT/BLOB carriers and passes everything else
through.
No writer stores a scalar (all four write sites assign a list), so
the malformed-row cases above cannot reach this branch: they pass
equally on the old code, and this case does not.
"""
research_id = f"scalar-{stored_scalar!r}"
session.add(
ResearchHistory(
id=research_id,
query="scalar carrier",
mode="detailed",
status="completed",
created_at="2024-01-01T12:00:00",
progress_log=stored_scalar,
)
)
session.commit()
loaded = (
session.query(ResearchHistory).filter_by(id=research_id).first()
)
assert loaded is not None
assert loaded.progress_log == stored_scalar
assert loaded.progress_log is not None
@pytest.mark.parametrize(
"carrier",
[b"[]", bytearray(b"[]")],
ids=["bytes", "bytearray"],
)
def test_lenient_json_decodes_binary_carriers(self, carrier):
"""Every binary TEXT carrier is decoded, not passed through raw.
The gate's job on these is the same as on ``str``: SQLite hands
back ``bytes`` for a BLOB-stored value, and that value still has to
reach the reader as the parsed log.
"""
from local_deep_research.database.models.research import LenientJSON
process = LenientJSON().result_processor(None, None)
assert process(carrier) == []
def test_lenient_json_passes_a_decoded_object_through_untouched(self):
"""Anything that is not a TEXT carrier is returned exactly as given.
A backend with a native JSON type hands the processor a parsed
object (list, dict) or a scalar (``5``, ``True``); ``memoryview``
is a value no SQLite driver emits for a JSON column but is the same
case — an already-decoded object the gate must not reinterpret.
Feeding any of them to ``json.loads`` raises ``TypeError``, which
the first cut of this gate caught and turned into ``None``.
"""
from local_deep_research.database.models.research import LenientJSON
process = LenientJSON().result_processor(None, None)
assert process([{"time": "t"}]) == [{"time": "t"}]
assert process({"a": 1}) == {"a": 1}
assert process(None) is None
assert process(5) == 5
assert process(True) is True
assert bytes(process(memoryview(b"..."))) == b"..."
def test_research_task_creation(self, session):
"""Test ResearchTask model."""
task = ResearchTask(
title="Machine Learning Research",
description="Research current ML trends and applications",
status="in_progress",
priority=5,
tags=["ml", "ai", "trends"],
research_metadata={
"estimated_time": "2 hours",
"complexity": "medium",
},
)
session.add(task)
session.commit()
# Verify task
saved = session.query(ResearchTask).first()
assert saved.title == "Machine Learning Research"
assert saved.priority == 5
assert "ml" in saved.tags
assert saved.research_metadata["complexity"] == "medium"
def test_search_query_and_results(self, session):
"""Test SearchQuery and SearchResult models."""
# Create research task first
task = ResearchTask(
title="Test Research", description="Test", status="pending"
)
session.add(task)
session.commit()
# Create search query
query = SearchQuery(
research_task_id=task.id,
query="quantum computing applications",
search_engine="google",
search_type="web",
parameters={"num_results": 10, "time_range": "past_year"},
status="completed",
)
session.add(query)
session.commit()
# Add search results
results = [
SearchResult(
research_task_id=task.id,
search_query_id=query.id,
title="Quantum Computing in 2024",
url="https://example.com/quantum-2024",
snippet="Latest developments in quantum computing...",
relevance_score=0.95,
content="Full article content here...",
content_type="article",
position=1,
domain="example.com",
language="en",
author="Dr. Smith",
fetch_status="fetched",
),
SearchResult(
research_task_id=task.id,
search_query_id=query.id,
title="Practical Quantum Applications",
url="https://example.com/quantum-apps",
snippet="Real-world applications of quantum tech...",
relevance_score=0.87,
position=2,
domain="example.com",
fetch_status="pending",
),
]
session.add_all(results)
session.commit()
# Verify relationships
assert len(query.results) == 2
assert query.results[0].relevance_score == 0.95
assert task.searches[0].query == "quantum computing applications"
def test_research_strategy(self, session):
"""Test ResearchStrategy model.
ResearchStrategy.research_id is a String(36) FK to research_history.id —
the live UUID-keyed table. The legacy Integer FK to the dormant
``research`` table caused FOREIGN KEY constraint failed once
PRAGMA foreign_keys was enabled in v1.6.0; migration 0008 retargets it.
"""
history_id = str(uuid.uuid4())
history = ResearchHistory(
id=history_id,
query="Climate change solutions",
mode="detailed",
status="completed",
created_at="2026-01-01T10:00:00",
)
session.add(history)
session.commit()
strategy = ResearchStrategy(
research_id=history_id, strategy_name="comprehensive_search"
)
session.add(strategy)
session.commit()
saved = session.query(ResearchStrategy).first()
assert saved.strategy_name == "comprehensive_search"
assert saved.research_id == history_id
def test_research_relationships(self, session):
"""Test relationships between research models."""
# Create research history
history = ResearchHistory(
id=str(uuid.uuid4()),
query="AI Ethics",
mode="comprehensive",
status="completed",
created_at="2024-01-01T10:00:00",
)
session.add(history)
session.commit()
# Add resources
resources = [
ResearchResource(
research_id=history.id,
title="AI Ethics Guidelines",
url="https://example.com/ai-ethics",
content_preview="Guidelines for ethical AI development...",
source_type="article",
resource_metadata={"credibility": "high"},
created_at="2024-01-01T10:30:00",
),
ResearchResource(
research_id=history.id,
title="Ethics in Machine Learning",
url="https://example.com/ml-ethics",
content_preview="Exploring ethical considerations in ML...",
source_type="research_paper",
resource_metadata={"peer_reviewed": True},
created_at="2024-01-01T10:45:00",
),
]
session.add_all(resources)
session.commit()
# Verify relationships
assert len(history.resources) == 2
assert history.resources[0].title == "AI Ethics Guidelines"
# Test cascade delete
session.delete(history)
session.commit()
# Resources should be deleted too
remaining_resources = session.query(ResearchResource).count()
assert remaining_resources == 0
def test_research_metadata_handling(self, session):
"""Test JSON metadata fields across models."""
# ResearchHistory with complex metadata
history = ResearchHistory(
id=str(uuid.uuid4()),
query="Complex research",
mode="normal",
status="completed",
created_at="2024-01-01T00:00:00",
research_meta={
"sources": {"academic": 5, "news": 10, "blogs": 3},
"quality_metrics": {
"relevance": 0.85,
"credibility": 0.9,
"recency": 0.95,
},
"search_iterations": 3,
"total_sources_examined": 50,
},
)
session.add(history)
session.commit()
# Verify complex metadata
saved = session.query(ResearchHistory).first()
assert saved.research_meta["sources"]["academic"] == 5
assert saved.research_meta["quality_metrics"]["relevance"] == 0.85
def test_search_result_content(self, session):
"""Test SearchResult content storage and retrieval."""
# Create parent objects
task = ResearchTask(
title="Content Test",
description="Testing content storage",
status="pending",
)
session.add(task)
session.commit()
query = SearchQuery(
research_task_id=task.id,
query="test query",
search_engine="google",
status="completed",
)
session.add(query)
session.commit()
# Large content
large_content = "x" * 10000 # 10KB of content
result = SearchResult(
research_task_id=task.id,
search_query_id=query.id,
title="Large Article",
url="https://example.com/large",
snippet="Beginning of large article...",
content=large_content,
content_type="article",
relevance_score=0.8,
position=1,
domain="example.com",
language="en",
fetch_status="fetched",
fetched_at=datetime.now(timezone.utc),
)
session.add(result)
session.commit()
# Verify content
saved = session.query(SearchResult).first()
assert len(saved.content) == 10000
assert saved.fetch_status == "fetched"
assert saved.domain == "example.com"
def test_research_error_handling(self, session):
"""Test error tracking in research models."""
# Failed research
failed_research = ResearchHistory(
id=str(uuid.uuid4()),
query="This will fail",
mode="quick",
status="failed",
created_at="2024-01-01T00:00:00",
research_meta={
"error": "NetworkError",
"error_message": "Connection timeout",
"retry_count": 3,
"last_error_timestamp": "2024-01-01T00:05:00",
},
)
session.add(failed_research)
session.commit()
# Failed search query
task = ResearchTask(
title="Error Test", description="Test", status="failed"
)
session.add(task)
session.commit()
failed_query = SearchQuery(
research_task_id=task.id,
query="problematic query",
search_engine="bing",
status="failed",
error_message="Rate limit exceeded",
retry_count=5,
)
session.add(failed_query)
session.commit()
# Verify error tracking
assert failed_research.research_meta["error"] == "NetworkError"
assert failed_query.error_message == "Rate limit exceeded"
assert failed_query.retry_count == 5