Skip to content

~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.

fix(db): emit CreateIndex for fresh user DBs; repair download_tracker FK targets (#3697) (#3708)

LearningCircuit4db93eb2b2d0
diff --git a/src/local_deep_research/database/encrypted_db.py b/src/local_deep_research/database/encrypted_db.py
index a8d29d1604..6f83cd703f 100644
--- a/src/local_deep_research/database/encrypted_db.py
+++ b/src/local_deep_research/database/encrypted_db.py
@@ -382,21 +382,31 @@ class DatabaseManager:
try:
# Get the CREATE TABLE statements from SQLAlchemy models
from sqlalchemy.dialects import sqlite
- from sqlalchemy.schema import CreateTable
+ from sqlalchemy.schema import CreateIndex, CreateTable
from .models import Base
- # Create tables one by one
+ # Indexes must be emitted explicitly — SQLAlchemy compiles
+ # `index=True`/`unique=True` and `Index(...)` to separate
+ # CREATE [UNIQUE] INDEX statements, not inline.
sqlite_dialect = sqlite.dialect()
for table in Base.metadata.sorted_tables:
- if table.name != "users":
- create_sql = str(
- CreateTable(table).compile(
+ if table.name == "users":
+ continue
+ create_sql = str(
+ CreateTable(table, if_not_exists=True).compile(
+ dialect=sqlite_dialect
+ )
+ )
+ logger.debug(f"Creating table {table.name}")
+ conn.execute(create_sql)
+ for index in table.indexes:
+ index_sql = str(
+ CreateIndex(index, if_not_exists=True).compile(
dialect=sqlite_dialect
)
)
- logger.debug(f"Creating table {table.name}")
- conn.execute(create_sql)
+ conn.execute(index_sql)
conn.commit()
finally:
diff --git a/src/local_deep_research/database/migrations/versions/0007_backfill_missing_indexes.py b/src/local_deep_research/database/migrations/versions/0007_backfill_missing_indexes.py
new file mode 100644
index 0000000000..f639b72bb2
--- /dev/null
+++ b/src/local_deep_research/database/migrations/versions/0007_backfill_missing_indexes.py
@@ -0,0 +1,187 @@
+"""Backfill model-declared indexes that were never emitted by the
+encrypted-DB create-tables loop, and repair download_tracker FK targets.
+
+Background
+==========
+``encrypted_db.create_user_database()`` previously emitted only
+``CreateTable(...)`` for each model in ``Base.metadata.sorted_tables``,
+never ``CreateIndex(...)``. As a result, every model-declared index
+(``index=True``, ``unique=True``, ``Index(...)`` in ``__table_args__``)
+was missing from per-user encrypted databases.
+
+This was harmless until v1.6.0 enabled ``PRAGMA foreign_keys = ON``,
+which then started raising ``foreign key mismatch`` whenever
+``download_attempts`` / ``download_duplicates`` cascade deletes ran —
+because their FK target ``download_tracker.url_hash`` had no UNIQUE
+backing visible to SQLCipher.
+
+What this migration does
+========================
+1. Removes duplicate ``url_hash`` rows in ``download_tracker`` so the
+ subsequent UNIQUE INDEX creation does not fail. Survivor: smallest
+ ``id`` per ``url_hash``.
+2. Removes orphan child rows in ``download_attempts`` /
+ ``download_duplicates`` whose ``url_hash`` no longer exists in
+ ``download_tracker`` (cascade was inert before v1.6.0, so orphans
+ may have accumulated).
+3. Creates the canonical UNIQUE index on ``download_tracker.url_hash``
+ (named to match the new ``UniqueConstraint`` in the model).
+4. Iterates every table in ``Base.metadata`` and creates any model-
+ declared index that is missing in this database. Idempotent:
+ indexes that already exist (created by earlier migrations or by
+ the now-fixed encrypted_db loop) are skipped.
+
+Revision ID: 0007
+Revises: 0006
+Create Date: 2026-04-27
+"""
+
+from alembic import op
+from loguru import logger
+from sqlalchemy import inspect, text
+
+revision = "0007"
+down_revision = "0006"
+branch_labels = None
+depends_on = None
+
+
+_DOWNLOAD_TRACKER_UNIQUE_INDEX = "uq_download_tracker_url_hash"
+
+
+def _scrub_download_tracker_duplicates(bind) -> None:
+ """Keep the smallest-id row per url_hash; drop the rest."""
+ if not inspect(bind).has_table("download_tracker"):
+ return
+
+ duplicates = bind.execute(
+ text(
+ "SELECT url_hash, COUNT(*) AS c FROM download_tracker "
+ "GROUP BY url_hash HAVING c > 1"
+ )
+ ).all()
+ if not duplicates:
+ return
+
+ total_removed = 0
+ for url_hash, _ in duplicates:
+ result = bind.execute(
+ text(
+ "DELETE FROM download_tracker "
+ "WHERE url_hash = :h AND id NOT IN ("
+ " SELECT MIN(id) FROM download_tracker WHERE url_hash = :h"
+ ")"
+ ),
+ {"h": url_hash},
+ )
+ total_removed += result.rowcount or 0
+ logger.warning(
+ f"0007: removed {total_removed} duplicate download_tracker row(s) "
+ f"across {len(duplicates)} url_hash group(s) before adding UNIQUE index"
+ )
+
+
+_ORPHAN_SCRUB_QUERIES = {
+ "download_attempts": text(
+ "DELETE FROM download_attempts "
+ "WHERE url_hash NOT IN (SELECT url_hash FROM download_tracker)"
+ ),
+ "download_duplicates": text(
+ "DELETE FROM download_duplicates "
+ "WHERE url_hash NOT IN (SELECT url_hash FROM download_tracker)"
+ ),
+}
+
+
+def _scrub_orphan_children(bind) -> None:
+ """Drop child rows referencing url_hash values that no longer exist."""
+ inspector = inspect(bind)
+ for child_table, stmt in _ORPHAN_SCRUB_QUERIES.items():
+ if not inspector.has_table(child_table):
+ continue
+ result = bind.execute(stmt)
+ if result.rowcount:
+ logger.warning(
+ f"0007: removed {result.rowcount} orphan row(s) from {child_table}"
+ )
+
+
+def _index_exists(inspector, index_name: str, table_name: str) -> bool:
+ if not inspector.has_table(table_name):
+ return False
+ return any(
+ idx["name"] == index_name for idx in inspector.get_indexes(table_name)
+ )
+
+
+def _backfill_model_indexes(bind) -> None:
+ """Create any model-declared index that's missing in this database."""
+ from local_deep_research.database.models import Base
+
+ inspector = inspect(bind)
+ created = 0
+ for table in Base.metadata.sorted_tables:
+ if table.name == "users":
+ continue
+ if not inspector.has_table(table.name):
+ continue
+ for index in table.indexes:
+ if not index.name:
+ continue
+ if _index_exists(inspector, index.name, table.name):
+ continue
+ try:
+ op.create_index(
+ index.name,
+ table.name,
+ [c.name for c in index.columns],
+ unique=index.unique,
+ if_not_exists=True,
+ )
+ created += 1
+ except Exception:
+ logger.exception(
+ f"0007: failed to create index {index.name} on {table.name}"
+ )
+ if created:
+ logger.info(f"0007: created {created} missing model-declared index(es)")
+
+
+def upgrade() -> None:
+ bind = op.get_bind()
+
+ _scrub_download_tracker_duplicates(bind)
+ _scrub_orphan_children(bind)
+
+ inspector = inspect(bind)
+ if inspector.has_table("download_tracker") and not _index_exists(
+ inspector, _DOWNLOAD_TRACKER_UNIQUE_INDEX, "download_tracker"
+ ):
+ op.create_index(
+ _DOWNLOAD_TRACKER_UNIQUE_INDEX,
+ "download_tracker",
+ ["url_hash"],
+ unique=True,
+ if_not_exists=True,
+ )
+
+ _backfill_model_indexes(bind)
+
+
+def downgrade() -> None:
+ """Drop only the canonical url_hash index this migration introduced.
+
+ The defensive backfill of model-declared indexes is forward-only —
+ those indexes are part of the model definitions and would be re-emitted
+ on the next user-DB open via the fixed encrypted_db.create_user_database
+ code path.
+ """
+ bind = op.get_bind()
+ inspector = inspect(bind)
+ if _index_exists(
+ inspector, _DOWNLOAD_TRACKER_UNIQUE_INDEX, "download_tracker"
+ ):
+ op.drop_index(
+ _DOWNLOAD_TRACKER_UNIQUE_INDEX,
+ table_name="download_tracker",
+ )
diff --git a/src/local_deep_research/database/models/download_tracker.py b/src/local_deep_research/database/models/download_tracker.py
index df0a4f9212..ebb1179177 100644
--- a/src/local_deep_research/database/models/download_tracker.py
+++ b/src/local_deep_research/database/models/download_tracker.py
@@ -31,7 +31,7 @@ class DownloadTracker(Base):
# URL tracking
url = Column(Text, nullable=False) # Original URL
url_hash = Column(
- String(64), nullable=False, unique=True, index=True
+ String(64), nullable=False, index=True
) # SHA256 of normalized URL
# Resource tracking (can be multiple resources with same URL)
@@ -61,13 +61,22 @@ class DownloadTracker(Base):
downloaded_at = Column(UtcDateTime, nullable=True)
last_checked = Column(UtcDateTime, default=utcnow(), nullable=False)
- # Link to full document record if it exists
+ # Link to full document record if it exists. documents.id is String(36) UUID;
+ # the previous Integer declaration produced an FK type mismatch that SQLite
+ # silently coerced as TEXT due to type-affinity rules.
library_document_id = Column(
- Integer,
+ String(36),
ForeignKey("documents.id", ondelete="SET NULL"),
nullable=True,
)
+ # Inline UNIQUE so it lands in CREATE TABLE; column-level unique=True
+ # produces a separate CREATE UNIQUE INDEX which SQLCipher does not accept
+ # as an FK target for url_hash references.
+ __table_args__ = (
+ UniqueConstraint("url_hash", name="uq_download_tracker_url_hash"),
+ )
+
def __repr__(self):
status = "downloaded" if self.is_downloaded else "not downloaded"
return f"<DownloadTracker(url_hash={self.url_hash[:8]}..., status={status})>"
diff --git a/tests/database/test_alembic_migrations.py b/tests/database/test_alembic_migrations.py
index 6fac0bb27d..eb38d63187 100644
--- a/tests/database/test_alembic_migrations.py
+++ b/tests/database/test_alembic_migrations.py
@@ -640,7 +640,7 @@ class TestAlembicVersionTable:
# Verify it's at head
assert revision == get_head_revision()
- assert revision == "0006" # Current head
+ assert revision == "0007" # Current head
finally:
engine.dispose()
@@ -658,6 +658,8 @@ class TestMigrationChain:
"0003_add_research_indexes.py",
"0004_migrate_legacy_app_settings.py",
"0005_add_resource_document_id.py",
+ "0006_journal_quality_system.py",
+ "0007_backfill_missing_indexes.py",
]
for filename in expected_files:
@@ -682,9 +684,15 @@ class TestMigrationChain:
assert len(revisions) >= 3
# Head should be the latest migration
- assert script.get_current_head() == "0006"
+ assert script.get_current_head() == "0007"
+
+ # Verify chain: 0007 -> 0006 -> 0005 -> 0004 -> 0003 -> 0002 -> 0001 -> None
+ rev_0007 = script.get_revision("0007")
+ assert rev_0007.down_revision == "0006"
+
+ rev_0006 = script.get_revision("0006")
+ assert rev_0006.down_revision == "0005"
- # Verify chain: 0005 -> 0004 -> 0003 -> 0002 -> 0001 -> None
rev_0005 = script.get_revision("0005")
assert rev_0005.down_revision == "0004"
@@ -718,7 +726,7 @@ class TestMigrationChain:
# Run to head
run_migrations(engine, target="head") # raises on failure
- assert get_current_revision(engine) == "0006"
+ assert get_current_revision(engine) == "0007"
finally:
engine.dispose()
@@ -1172,7 +1180,7 @@ class TestStampDatabaseBehavior:
# Stamp at head
stamp_database(engine, "head")
- assert get_current_revision(engine) == "0006"
+ assert get_current_revision(engine) == "0007"
finally:
engine.dispose()
@@ -1299,7 +1307,7 @@ class TestMigrationRobustness:
# Reconnect and continue
engine = create_engine(f"sqlite:///{db_path}")
run_migrations(engine) # raises on failure
- assert get_current_revision(engine) == "0006"
+ assert get_current_revision(engine) == "0007"
finally:
engine.dispose()
@@ -1706,7 +1714,7 @@ class TestDowngradeMigrations:
try:
# First upgrade to head
run_migrations(engine)
- assert get_current_revision(engine) == "0006"
+ assert get_current_revision(engine) == "0007"
# Downgrade to 0001
config = get_alembic_config(engine)
@@ -1827,7 +1835,7 @@ class TestDowngradeMigrations:
# Re-upgrade to head
run_migrations(engine)
- assert get_current_revision(engine) == "0006"
+ assert get_current_revision(engine) == "0007"
# Verify data still exists
with engine.connect() as conn:
@@ -2234,7 +2242,7 @@ class TestMigrationRecovery:
# Should be able to continue
run_migrations(engine) # raises on failure
- assert get_current_revision(engine) == "0006"
+ assert get_current_revision(engine) == "0007"
finally:
engine.dispose()
@@ -2498,8 +2506,8 @@ class TestMigrationVersionConsistency:
head = script.get_current_head()
# Current head should be the latest migration
- assert head == "0006", (
- f"Head revision mismatch: expected 0006, got {head}"
+ assert head == "0007", (
+ f"Head revision mismatch: expected 0007, got {head}"
)
def test_all_revision_files_have_valid_structure(self):
@@ -2561,7 +2569,7 @@ class TestMigrationVersionConsistency:
# Stamp at head
stamp_database(engine, "head")
- assert get_current_revision(engine) == "0006"
+ assert get_current_revision(engine) == "0007"
with engine.connect() as conn:
count = conn.execute(
@@ -4460,7 +4468,7 @@ class TestPreAlembicDatabaseUpgrade:
# Step 3: Upgrade to head (0005)
run_migrations(engine, target="head") # raises on failure
- assert get_current_revision(engine) == "0006"
+ assert get_current_revision(engine) == "0007"
# Step 4: Verify data survived
with engine.connect() as conn:
@@ -4509,7 +4517,7 @@ class TestPreAlembicDatabaseUpgrade:
# Step 6: Another run_migrations should be no-op
run_migrations(engine) # raises on failure
- assert get_current_revision(engine) == "0006"
+ assert get_current_revision(engine) == "0007"
finally:
engine.dispose()
@@ -5399,7 +5407,7 @@ class TestMigrationSafetyGuards:
row = conn.execute(
text("SELECT version_num FROM alembic_version")
).fetchone()
- assert row[0] == "0006"
+ assert row[0] == "0007"
engine.dispose()
diff --git a/tests/database/test_fk_targets_resolve.py b/tests/database/test_fk_targets_resolve.py
new file mode 100644
index 0000000000..31ceab784d
--- /dev/null
+++ b/tests/database/test_fk_targets_resolve.py
@@ -0,0 +1,41 @@
+"""Static check: every FK in Base.metadata points at a real table+column.
+
+Catches the R4 class of bug — ``ResearchStrategy.research_id`` declared
+``ForeignKey("research.id")`` while the live code wrote UUID strings from
+``research_history``. PRAGMA-OFF hid this for ~10 months; PRAGMA-ON in
+v1.6.0 made every save fail.
+"""
+
+from __future__ import annotations
+
+from local_deep_research.database.models import Base
+
+
+def test_every_fk_target_table_and_column_exists():
+ tables = {t.name: t for t in Base.metadata.sorted_tables}
+ problems: list[str] = []
+ for table in Base.metadata.sorted_tables:
+ for fk in table.foreign_keys:
+ target_table_name = fk.column.table.name
+ target_col_name = fk.column.name
+ target = tables.get(target_table_name)
+ if target is None:
+ problems.append(
+ f"{table.name}.{fk.parent.name} -> "
+ f"{target_table_name}.{target_col_name}: target table missing"
+ )
+ continue
+ if target_col_name not in target.columns:
+ problems.append(
+ f"{table.name}.{fk.parent.name} -> "
+ f"{target_table_name}.{target_col_name}: target column missing"
+ )
+ continue
+ target_col = target.columns[target_col_name]
+ if str(fk.parent.type) != str(target_col.type):
+ problems.append(
+ f"{table.name}.{fk.parent.name} ({fk.parent.type}) -> "
+ f"{target_table_name}.{target_col_name} ({target_col.type}): "
+ "type mismatch"
+ )
+ assert not problems, "FK declaration problems:\n " + "\n ".join(problems)
diff --git a/tests/database/test_migration_0003_indexes.py b/tests/database/test_migration_0003_indexes.py
index fb24a73937..cdbb907f14 100644
--- a/tests/database/test_migration_0003_indexes.py
+++ b/tests/database/test_migration_0003_indexes.py
@@ -216,13 +216,13 @@ class TestMigration0003FromFreshDatabase:
assert indexes[idx_name] == idx_columns
def test_head_revision_is_current(self):
- """get_head_revision() should return the head revision '0006'."""
- assert get_head_revision() == "0006"
+ """get_head_revision() should return the head revision '0007'."""
+ assert get_head_revision() == "0007"
def test_current_revision_is_head_after_migrate(self, fresh_engine):
- """After full migration, current revision should be 0006."""
+ """After full migration, current revision should be 0007."""
run_migrations(fresh_engine)
- assert get_current_revision(fresh_engine) == "0006"
+ assert get_current_revision(fresh_engine) == "0007"
def test_needs_migration_false_after_full_upgrade(self, fresh_engine):
"""After full migration, needs_migration() should return False."""
@@ -579,7 +579,7 @@ class TestMigration0003EdgeCases:
try:
run_migrations(engine)
- assert get_current_revision(engine) == "0006"
+ assert get_current_revision(engine) == "0007"
for table_name, expected_indexes in [
("research_tasks", RESEARCH_TASKS_INDEXES),
diff --git a/tests/database/test_migration_0004_app_settings.py b/tests/database/test_migration_0004_app_settings.py
index 4b1cbc1685..c6bd90f328 100644
--- a/tests/database/test_migration_0004_app_settings.py
+++ b/tests/database/test_migration_0004_app_settings.py
@@ -359,7 +359,7 @@ class TestMigration0004Idempotency:
def test_full_migration_from_fresh_db(self, fresh_engine):
"""Full migration chain on empty DB should work (no settings to migrate)."""
run_migrations(fresh_engine)
- assert get_current_revision(fresh_engine) == "0006"
+ assert get_current_revision(fresh_engine) == "0007"
class TestMigration0004Downgrade:
diff --git a/tests/database/test_schema_index_completeness.py b/tests/database/test_schema_index_completeness.py
new file mode 100644
index 0000000000..47cc8ca684
--- /dev/null
+++ b/tests/database/test_schema_index_completeness.py
@@ -0,0 +1,112 @@
+"""Regression guard for the encrypted-DB CreateIndex defect.
+
+Until v1.6.x, ``encrypted_db.create_user_database()`` only emitted
+``CreateTable`` statements and never ``CreateIndex``. As a result every
+model-declared index (``index=True``, ``unique=True``, explicit
+``Index(...)`` in ``__table_args__``) was missing in user databases.
+
+This test asserts that for every table in ``Base.metadata``, every named
+index declared by the model is present in a freshly created encrypted
+database — both column-level and table-level.
+"""
+
+from __future__ import annotations
+
+import sys
+import tempfile
+from pathlib import Path
+
+import pytest
+from sqlalchemy import inspect
+
+pytest.importorskip(
+ "sqlcipher3",
+ reason="SQLCipher is required to test the encrypted-DB schema path.",
+)
+
+sys.path.insert(
+ 0,
+ str(Path(__file__).parent.parent.parent.resolve()),
+)
+
+from local_deep_research.database.encrypted_db import DatabaseManager
+from local_deep_research.database.models import Base
+
+
+def temp_data_dir(monkeypatch):
+ with tempfile.TemporaryDirectory() as tmpdir:
+ path = Path(tmpdir)
+ monkeypatch.setattr(
+ "local_deep_research.database.encrypted_db.get_data_directory",
+ lambda: path,
+ )
+ yield path
+
+
+def db_manager(temp_data_dir):
+ m = DatabaseManager()
+ yield m
+ for username in list(m.connections.keys()):
+ m.close_user_database(username)
+
+
+def test_every_model_declared_index_exists_in_fresh_db(db_manager):
+ """Iterate Base.metadata; assert every named index is present."""
+ assert db_manager.has_encryption, (
+ "sqlcipher3 imports but DatabaseManager reported has_encryption=False"
+ )
+ username = "indexcheck"
+ password = "StrongPassword1!"
+ db_manager.create_user_database(username, password)
+
+ engine = db_manager.connections[username]
+ inspector = inspect(engine)
+
+ missing: list[tuple[str, str]] = []
+ for table in Base.metadata.sorted_tables:
+ if table.name == "users":
+ continue
+ if not inspector.has_table(table.name):
+ continue
+ existing = {idx["name"] for idx in inspector.get_indexes(table.name)}
+ for index in table.indexes:
+ if not index.name:
+ continue
+ if index.name not in existing:
+ missing.append((table.name, index.name))
+
+ assert not missing, (
+ f"Missing model-declared indexes in fresh user DB: {missing}. "
+ f"This usually means encrypted_db.create_user_database() is not "
+ f"emitting CreateIndex for declared indexes."
+ )
+
+
+def test_download_tracker_url_hash_has_unique_backing(db_manager):
+ """The exact regression that caused #3697 — url_hash must be UNIQUE.
+
+ Without UNIQUE backing on the FK target, SQLCipher raises
+ "foreign key mismatch — download_attempts referencing download_tracker"
+ on cascade delete.
+ """
+ assert db_manager.has_encryption
+ username = "trackercheck"
+ password = "StrongPassword1!"
+ db_manager.create_user_database(username, password)
+
+ engine = db_manager.connections[username]
+ inspector = inspect(engine)
+
+ indexes = inspector.get_indexes("download_tracker")
+ unique_on_url_hash = [
+ idx
+ for idx in indexes
+ if idx.get("unique") and idx.get("column_names") == ["url_hash"]
+ ]
+ assert unique_on_url_hash, (
+ "download_tracker.url_hash must have a UNIQUE backing index for "
+ "FK references from download_attempts/download_duplicates to "
+ f"resolve. Indexes present: {indexes}"
+ )