~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.
Python academiaanthropicarxivbravedeep-researchencryptionhome-automationhomeserverlocallocal-deep-researchlocal-llmmistralollamaopenaipubmedresearchresearch-toolretrieval-augmented-generationsearxngself-hosted
fix(db): emit CreateIndex for fresh user DBs; repair download_tracker FK targets (#3697) (#3708)
LearningCircuit4db93eb2b2d0
diff --git a/src/local_deep_research/database/encrypted_db.py b/src/local_deep_research/database/encrypted_db.pyindex a8d29d1604..6f83cd703f 100644--- a/src/local_deep_research/database/encrypted_db.py+++ b/src/local_deep_research/database/encrypted_db.py@@ -382,21 +382,31 @@ class DatabaseManager:try:# Get the CREATE TABLE statements from SQLAlchemy modelsfrom sqlalchemy.dialects import sqlite- from sqlalchemy.schema import CreateTable+ from sqlalchemy.schema import CreateIndex, CreateTablefrom .models import Base- # Create tables one by one+ # Indexes must be emitted explicitly — SQLAlchemy compiles+ # `index=True`/`unique=True` and `Index(...)` to separate+ # CREATE [UNIQUE] INDEX statements, not inline.sqlite_dialect = sqlite.dialect()for table in Base.metadata.sorted_tables:- if table.name != "users":- create_sql = str(- CreateTable(table).compile(+ if table.name == "users":+ continue+ create_sql = str(+ CreateTable(table, if_not_exists=True).compile(+ dialect=sqlite_dialect+ )+ )+ logger.debug(f"Creating table {table.name}")+ conn.execute(create_sql)+ for index in table.indexes:+ index_sql = str(+ CreateIndex(index, if_not_exists=True).compile(dialect=sqlite_dialect))- logger.debug(f"Creating table {table.name}")- conn.execute(create_sql)+ conn.execute(index_sql)conn.commit()finally:diff --git a/src/local_deep_research/database/migrations/versions/0007_backfill_missing_indexes.py b/src/local_deep_research/database/migrations/versions/0007_backfill_missing_indexes.pynew file mode 100644index 0000000000..f639b72bb2--- /dev/null+++ b/src/local_deep_research/database/migrations/versions/0007_backfill_missing_indexes.py@@ -0,0 +1,187 @@+"""Backfill model-declared indexes that were never emitted by the+encrypted-DB create-tables loop, and repair download_tracker FK targets.++Background+==========+``encrypted_db.create_user_database()`` previously emitted only+``CreateTable(...)`` for each model in ``Base.metadata.sorted_tables``,+never ``CreateIndex(...)``. As a result, every model-declared index+(``index=True``, ``unique=True``, ``Index(...)`` in ``__table_args__``)+was missing from per-user encrypted databases.++This was harmless until v1.6.0 enabled ``PRAGMA foreign_keys = ON``,+which then started raising ``foreign key mismatch`` whenever+``download_attempts`` / ``download_duplicates`` cascade deletes ran —+because their FK target ``download_tracker.url_hash`` had no UNIQUE+backing visible to SQLCipher.++What this migration does+========================+1. Removes duplicate ``url_hash`` rows in ``download_tracker`` so the+ subsequent UNIQUE INDEX creation does not fail. Survivor: smallest+ ``id`` per ``url_hash``.+2. Removes orphan child rows in ``download_attempts`` /+ ``download_duplicates`` whose ``url_hash`` no longer exists in+ ``download_tracker`` (cascade was inert before v1.6.0, so orphans+ may have accumulated).+3. Creates the canonical UNIQUE index on ``download_tracker.url_hash``+ (named to match the new ``UniqueConstraint`` in the model).+4. Iterates every table in ``Base.metadata`` and creates any model-+ declared index that is missing in this database. Idempotent:+ indexes that already exist (created by earlier migrations or by+ the now-fixed encrypted_db loop) are skipped.++Revision ID: 0007+Revises: 0006+Create Date: 2026-04-27+"""++from alembic import op+from loguru import logger+from sqlalchemy import inspect, text++revision = "0007"+down_revision = "0006"+branch_labels = None+depends_on = None+++_DOWNLOAD_TRACKER_UNIQUE_INDEX = "uq_download_tracker_url_hash"+++def _scrub_download_tracker_duplicates(bind) -> None:+ """Keep the smallest-id row per url_hash; drop the rest."""+ if not inspect(bind).has_table("download_tracker"):+ return++ duplicates = bind.execute(+ text(+ "SELECT url_hash, COUNT(*) AS c FROM download_tracker "+ "GROUP BY url_hash HAVING c > 1"+ )+ ).all()+ if not duplicates:+ return++ total_removed = 0+ for url_hash, _ in duplicates:+ result = bind.execute(+ text(+ "DELETE FROM download_tracker "+ "WHERE url_hash = :h AND id NOT IN ("+ " SELECT MIN(id) FROM download_tracker WHERE url_hash = :h"+ ")"+ ),+ {"h": url_hash},+ )+ total_removed += result.rowcount or 0+ logger.warning(+ f"0007: removed {total_removed} duplicate download_tracker row(s) "+ f"across {len(duplicates)} url_hash group(s) before adding UNIQUE index"+ )+++_ORPHAN_SCRUB_QUERIES = {+ "download_attempts": text(+ "DELETE FROM download_attempts "+ "WHERE url_hash NOT IN (SELECT url_hash FROM download_tracker)"+ ),+ "download_duplicates": text(+ "DELETE FROM download_duplicates "+ "WHERE url_hash NOT IN (SELECT url_hash FROM download_tracker)"+ ),+}+++def _scrub_orphan_children(bind) -> None:+ """Drop child rows referencing url_hash values that no longer exist."""+ inspector = inspect(bind)+ for child_table, stmt in _ORPHAN_SCRUB_QUERIES.items():+ if not inspector.has_table(child_table):+ continue+ result = bind.execute(stmt)+ if result.rowcount:+ logger.warning(+ f"0007: removed {result.rowcount} orphan row(s) from {child_table}"+ )+++def _index_exists(inspector, index_name: str, table_name: str) -> bool:+ if not inspector.has_table(table_name):+ return False+ return any(+ idx["name"] == index_name for idx in inspector.get_indexes(table_name)+ )+++def _backfill_model_indexes(bind) -> None:+ """Create any model-declared index that's missing in this database."""+ from local_deep_research.database.models import Base++ inspector = inspect(bind)+ created = 0+ for table in Base.metadata.sorted_tables:+ if table.name == "users":+ continue+ if not inspector.has_table(table.name):+ continue+ for index in table.indexes:+ if not index.name:+ continue+ if _index_exists(inspector, index.name, table.name):+ continue+ try:+ op.create_index(+ index.name,+ table.name,+ [c.name for c in index.columns],+ unique=index.unique,+ if_not_exists=True,+ )+ created += 1+ except Exception:+ logger.exception(+ f"0007: failed to create index {index.name} on {table.name}"+ )+ if created:+ logger.info(f"0007: created {created} missing model-declared index(es)")+++def upgrade() -> None:+ bind = op.get_bind()++ _scrub_download_tracker_duplicates(bind)+ _scrub_orphan_children(bind)++ inspector = inspect(bind)+ if inspector.has_table("download_tracker") and not _index_exists(+ inspector, _DOWNLOAD_TRACKER_UNIQUE_INDEX, "download_tracker"+ ):+ op.create_index(+ _DOWNLOAD_TRACKER_UNIQUE_INDEX,+ "download_tracker",+ ["url_hash"],+ unique=True,+ if_not_exists=True,+ )++ _backfill_model_indexes(bind)+++def downgrade() -> None:+ """Drop only the canonical url_hash index this migration introduced.++ The defensive backfill of model-declared indexes is forward-only —+ those indexes are part of the model definitions and would be re-emitted+ on the next user-DB open via the fixed encrypted_db.create_user_database+ code path.+ """+ bind = op.get_bind()+ inspector = inspect(bind)+ if _index_exists(+ inspector, _DOWNLOAD_TRACKER_UNIQUE_INDEX, "download_tracker"+ ):+ op.drop_index(+ _DOWNLOAD_TRACKER_UNIQUE_INDEX,+ table_name="download_tracker",+ )diff --git a/src/local_deep_research/database/models/download_tracker.py b/src/local_deep_research/database/models/download_tracker.pyindex df0a4f9212..ebb1179177 100644--- a/src/local_deep_research/database/models/download_tracker.py+++ b/src/local_deep_research/database/models/download_tracker.py@@ -31,7 +31,7 @@ class DownloadTracker(Base):# URL trackingurl = Column(Text, nullable=False) # Original URLurl_hash = Column(- String(64), nullable=False, unique=True, index=True+ String(64), nullable=False, index=True) # SHA256 of normalized URL# Resource tracking (can be multiple resources with same URL)@@ -61,13 +61,22 @@ class DownloadTracker(Base):downloaded_at = Column(UtcDateTime, nullable=True)last_checked = Column(UtcDateTime, default=utcnow(), nullable=False)- # Link to full document record if it exists+ # Link to full document record if it exists. documents.id is String(36) UUID;+ # the previous Integer declaration produced an FK type mismatch that SQLite+ # silently coerced as TEXT due to type-affinity rules.library_document_id = Column(- Integer,+ String(36),ForeignKey("documents.id", ondelete="SET NULL"),nullable=True,)+ # Inline UNIQUE so it lands in CREATE TABLE; column-level unique=True+ # produces a separate CREATE UNIQUE INDEX which SQLCipher does not accept+ # as an FK target for url_hash references.+ __table_args__ = (+ UniqueConstraint("url_hash", name="uq_download_tracker_url_hash"),+ )+def __repr__(self):status = "downloaded" if self.is_downloaded else "not downloaded"return f"<DownloadTracker(url_hash={self.url_hash[:8]}..., status={status})>"diff --git a/tests/database/test_alembic_migrations.py b/tests/database/test_alembic_migrations.pyindex 6fac0bb27d..eb38d63187 100644--- a/tests/database/test_alembic_migrations.py+++ b/tests/database/test_alembic_migrations.py@@ -640,7 +640,7 @@ class TestAlembicVersionTable:# Verify it's at headassert revision == get_head_revision()- assert revision == "0006" # Current head+ assert revision == "0007" # Current headfinally:engine.dispose()@@ -658,6 +658,8 @@ class TestMigrationChain:"0003_add_research_indexes.py","0004_migrate_legacy_app_settings.py","0005_add_resource_document_id.py",+ "0006_journal_quality_system.py",+ "0007_backfill_missing_indexes.py",]for filename in expected_files:@@ -682,9 +684,15 @@ class TestMigrationChain:assert len(revisions) >= 3# Head should be the latest migration- assert script.get_current_head() == "0006"+ assert script.get_current_head() == "0007"++ # Verify chain: 0007 -> 0006 -> 0005 -> 0004 -> 0003 -> 0002 -> 0001 -> None+ rev_0007 = script.get_revision("0007")+ assert rev_0007.down_revision == "0006"++ rev_0006 = script.get_revision("0006")+ assert rev_0006.down_revision == "0005"- # Verify chain: 0005 -> 0004 -> 0003 -> 0002 -> 0001 -> Nonerev_0005 = script.get_revision("0005")assert rev_0005.down_revision == "0004"@@ -718,7 +726,7 @@ class TestMigrationChain:# Run to headrun_migrations(engine, target="head") # raises on failure- assert get_current_revision(engine) == "0006"+ assert get_current_revision(engine) == "0007"finally:engine.dispose()@@ -1172,7 +1180,7 @@ class TestStampDatabaseBehavior:# Stamp at headstamp_database(engine, "head")- assert get_current_revision(engine) == "0006"+ assert get_current_revision(engine) == "0007"finally:engine.dispose()@@ -1299,7 +1307,7 @@ class TestMigrationRobustness:# Reconnect and continueengine = create_engine(f"sqlite:///{db_path}")run_migrations(engine) # raises on failure- assert get_current_revision(engine) == "0006"+ assert get_current_revision(engine) == "0007"finally:engine.dispose()@@ -1706,7 +1714,7 @@ class TestDowngradeMigrations:try:# First upgrade to headrun_migrations(engine)- assert get_current_revision(engine) == "0006"+ assert get_current_revision(engine) == "0007"# Downgrade to 0001config = get_alembic_config(engine)@@ -1827,7 +1835,7 @@ class TestDowngradeMigrations:# Re-upgrade to headrun_migrations(engine)- assert get_current_revision(engine) == "0006"+ assert get_current_revision(engine) == "0007"# Verify data still existswith engine.connect() as conn:@@ -2234,7 +2242,7 @@ class TestMigrationRecovery:# Should be able to continuerun_migrations(engine) # raises on failure- assert get_current_revision(engine) == "0006"+ assert get_current_revision(engine) == "0007"finally:engine.dispose()@@ -2498,8 +2506,8 @@ class TestMigrationVersionConsistency:head = script.get_current_head()# Current head should be the latest migration- assert head == "0006", (- f"Head revision mismatch: expected 0006, got {head}"+ assert head == "0007", (+ f"Head revision mismatch: expected 0007, got {head}")def test_all_revision_files_have_valid_structure(self):@@ -2561,7 +2569,7 @@ class TestMigrationVersionConsistency:# Stamp at headstamp_database(engine, "head")- assert get_current_revision(engine) == "0006"+ assert get_current_revision(engine) == "0007"with engine.connect() as conn:count = conn.execute(@@ -4460,7 +4468,7 @@ class TestPreAlembicDatabaseUpgrade:# Step 3: Upgrade to head (0005)run_migrations(engine, target="head") # raises on failure- assert get_current_revision(engine) == "0006"+ assert get_current_revision(engine) == "0007"# Step 4: Verify data survivedwith engine.connect() as conn:@@ -4509,7 +4517,7 @@ class TestPreAlembicDatabaseUpgrade:# Step 6: Another run_migrations should be no-oprun_migrations(engine) # raises on failure- assert get_current_revision(engine) == "0006"+ assert get_current_revision(engine) == "0007"finally:engine.dispose()@@ -5399,7 +5407,7 @@ class TestMigrationSafetyGuards:row = conn.execute(text("SELECT version_num FROM alembic_version")).fetchone()- assert row[0] == "0006"+ assert row[0] == "0007"engine.dispose()diff --git a/tests/database/test_fk_targets_resolve.py b/tests/database/test_fk_targets_resolve.pynew file mode 100644index 0000000000..31ceab784d--- /dev/null+++ b/tests/database/test_fk_targets_resolve.py@@ -0,0 +1,41 @@+"""Static check: every FK in Base.metadata points at a real table+column.++Catches the R4 class of bug — ``ResearchStrategy.research_id`` declared+``ForeignKey("research.id")`` while the live code wrote UUID strings from+``research_history``. PRAGMA-OFF hid this for ~10 months; PRAGMA-ON in+v1.6.0 made every save fail.+"""++from __future__ import annotations++from local_deep_research.database.models import Base+++def test_every_fk_target_table_and_column_exists():+ tables = {t.name: t for t in Base.metadata.sorted_tables}+ problems: list[str] = []+ for table in Base.metadata.sorted_tables:+ for fk in table.foreign_keys:+ target_table_name = fk.column.table.name+ target_col_name = fk.column.name+ target = tables.get(target_table_name)+ if target is None:+ problems.append(+ f"{table.name}.{fk.parent.name} -> "+ f"{target_table_name}.{target_col_name}: target table missing"+ )+ continue+ if target_col_name not in target.columns:+ problems.append(+ f"{table.name}.{fk.parent.name} -> "+ f"{target_table_name}.{target_col_name}: target column missing"+ )+ continue+ target_col = target.columns[target_col_name]+ if str(fk.parent.type) != str(target_col.type):+ problems.append(+ f"{table.name}.{fk.parent.name} ({fk.parent.type}) -> "+ f"{target_table_name}.{target_col_name} ({target_col.type}): "+ "type mismatch"+ )+ assert not problems, "FK declaration problems:\n " + "\n ".join(problems)diff --git a/tests/database/test_migration_0003_indexes.py b/tests/database/test_migration_0003_indexes.pyindex fb24a73937..cdbb907f14 100644--- a/tests/database/test_migration_0003_indexes.py+++ b/tests/database/test_migration_0003_indexes.py@@ -216,13 +216,13 @@ class TestMigration0003FromFreshDatabase:assert indexes[idx_name] == idx_columnsdef test_head_revision_is_current(self):- """get_head_revision() should return the head revision '0006'."""- assert get_head_revision() == "0006"+ """get_head_revision() should return the head revision '0007'."""+ assert get_head_revision() == "0007"def test_current_revision_is_head_after_migrate(self, fresh_engine):- """After full migration, current revision should be 0006."""+ """After full migration, current revision should be 0007."""run_migrations(fresh_engine)- assert get_current_revision(fresh_engine) == "0006"+ assert get_current_revision(fresh_engine) == "0007"def test_needs_migration_false_after_full_upgrade(self, fresh_engine):"""After full migration, needs_migration() should return False."""@@ -579,7 +579,7 @@ class TestMigration0003EdgeCases:try:run_migrations(engine)- assert get_current_revision(engine) == "0006"+ assert get_current_revision(engine) == "0007"for table_name, expected_indexes in [("research_tasks", RESEARCH_TASKS_INDEXES),diff --git a/tests/database/test_migration_0004_app_settings.py b/tests/database/test_migration_0004_app_settings.pyindex 4b1cbc1685..c6bd90f328 100644--- a/tests/database/test_migration_0004_app_settings.py+++ b/tests/database/test_migration_0004_app_settings.py@@ -359,7 +359,7 @@ class TestMigration0004Idempotency:def test_full_migration_from_fresh_db(self, fresh_engine):"""Full migration chain on empty DB should work (no settings to migrate)."""run_migrations(fresh_engine)- assert get_current_revision(fresh_engine) == "0006"+ assert get_current_revision(fresh_engine) == "0007"class TestMigration0004Downgrade:diff --git a/tests/database/test_schema_index_completeness.py b/tests/database/test_schema_index_completeness.pynew file mode 100644index 0000000000..47cc8ca684--- /dev/null+++ b/tests/database/test_schema_index_completeness.py@@ -0,0 +1,112 @@+"""Regression guard for the encrypted-DB CreateIndex defect.++Until v1.6.x, ``encrypted_db.create_user_database()`` only emitted+``CreateTable`` statements and never ``CreateIndex``. As a result every+model-declared index (``index=True``, ``unique=True``, explicit+``Index(...)`` in ``__table_args__``) was missing in user databases.++This test asserts that for every table in ``Base.metadata``, every named+index declared by the model is present in a freshly created encrypted+database — both column-level and table-level.+"""++from __future__ import annotations++import sys+import tempfile+from pathlib import Path++import pytest+from sqlalchemy import inspect++pytest.importorskip(+ "sqlcipher3",+ reason="SQLCipher is required to test the encrypted-DB schema path.",+)++sys.path.insert(+ 0,+ str(Path(__file__).parent.parent.parent.resolve()),+)++from local_deep_research.database.encrypted_db import DatabaseManager+from local_deep_research.database.models import Base+++def temp_data_dir(monkeypatch):+ with tempfile.TemporaryDirectory() as tmpdir:+ path = Path(tmpdir)+ monkeypatch.setattr(+ "local_deep_research.database.encrypted_db.get_data_directory",+ lambda: path,+ )+ yield path+++def db_manager(temp_data_dir):+ m = DatabaseManager()+ yield m+ for username in list(m.connections.keys()):+ m.close_user_database(username)+++def test_every_model_declared_index_exists_in_fresh_db(db_manager):+ """Iterate Base.metadata; assert every named index is present."""+ assert db_manager.has_encryption, (+ "sqlcipher3 imports but DatabaseManager reported has_encryption=False"+ )+ username = "indexcheck"+ password = "StrongPassword1!"+ db_manager.create_user_database(username, password)++ engine = db_manager.connections[username]+ inspector = inspect(engine)++ missing: list[tuple[str, str]] = []+ for table in Base.metadata.sorted_tables:+ if table.name == "users":+ continue+ if not inspector.has_table(table.name):+ continue+ existing = {idx["name"] for idx in inspector.get_indexes(table.name)}+ for index in table.indexes:+ if not index.name:+ continue+ if index.name not in existing:+ missing.append((table.name, index.name))++ assert not missing, (+ f"Missing model-declared indexes in fresh user DB: {missing}. "+ f"This usually means encrypted_db.create_user_database() is not "+ f"emitting CreateIndex for declared indexes."+ )+++def test_download_tracker_url_hash_has_unique_backing(db_manager):+ """The exact regression that caused #3697 — url_hash must be UNIQUE.++ Without UNIQUE backing on the FK target, SQLCipher raises+ "foreign key mismatch — download_attempts referencing download_tracker"+ on cascade delete.+ """+ assert db_manager.has_encryption+ username = "trackercheck"+ password = "StrongPassword1!"+ db_manager.create_user_database(username, password)++ engine = db_manager.connections[username]+ inspector = inspect(engine)++ indexes = inspector.get_indexes("download_tracker")+ unique_on_url_hash = [+ idx+ for idx in indexes+ if idx.get("unique") and idx.get("column_names") == ["url_hash"]+ ]+ assert unique_on_url_hash, (+ "download_tracker.url_hash must have a UNIQUE backing index for "+ "FK references from download_attempts/download_duplicates to "+ f"resolve. Indexes present: {indexes}"+ )