Skip to content

~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.

fix(db): restore missing legacy collection columns (#7078)

LearningCircuit2a0dd72cb28b
diff --git a/changelog.d/6144.bugfix.md b/changelog.d/6144.bugfix.md
new file mode 100644
index 0000000000..a536a39a88
--- /dev/null
+++ b/changelog.d/6144.bugfix.md
@@ -0,0 +1 @@
+Fix upgrades from databases created before Alembic. Model tables those databases never had are now created. Columns added or renamed in the models after a table already existed are now added, keeping existing rows: the advanced embedding settings on collections and RAG indexes, the PDF storage mode on documents, the context-overflow fields on token usage, the environment-variable field on settings, the stored report on research history, and the refresh interval in minutes on news subscriptions. The refresh interval is converted from the old hourly value, or set to 24 hours when no value exists. The PDF storage mode is inferred from where each document's PDF is stored. Both values are also filled when an interrupted upgrade added the column but did not fill it. A document whose storage mode is still unset is now handled by the same inference: deleting it removes its PDF file when that file is in the user's own library directory (a file still in the shared root used before per-user isolation is left in place, as for every deletion), a PDF stored in the database for it is shown as available, and uploading a PDF for it no longer replaces the path of a PDF already stored on disk. Legacy `app.*` settings now migrate even when the settings table has no environment-variable field, and setting types stored in a form the application cannot read are repaired, so the settings table loads again.
diff --git a/src/local_deep_research/database/alembic_runner.py b/src/local_deep_research/database/alembic_runner.py
index c3f6031224..a554ed3e4c 100644
--- a/src/local_deep_research/database/alembic_runner.py
+++ b/src/local_deep_research/database/alembic_runner.py
@@ -16,7 +16,7 @@ from alembic.config import Config
from alembic.runtime.migration import MigrationContext
from alembic.script import ScriptDirectory
from loguru import logger
-from sqlalchemy import Connection, Engine, inspect
+from sqlalchemy import Connection, Engine, inspect, text
from sqlalchemy.exc import IntegrityError, OperationalError
@@ -354,6 +354,33 @@ def _restore_fk_after_migration(conn: Connection) -> None:
)
+# Revisions a database can be at before 0004 runs. A pre-Alembic database is
+# stamped at 0001, and one whose earlier upgrade failed at 0004 stays below it.
+_REVISIONS_BEFORE_0004 = frozenset({"0001", "0002", "0003"})
+
+
+def _add_settings_env_var_before_0004(
+ conn: Connection, current: Optional[str]
+) -> None:
+ """Give a pre-0004 ``settings`` table the ``env_var`` column 0004 copies.
+
+ Revision 0004 names ``settings.env_var`` when it re-scopes legacy
+ ``app.*`` keys, and some pre-Alembic settings tables predate that column,
+ so the upgrade failed there. 0004 has shipped and cannot change, and 0032
+ adds the column only after 0004 runs, so add it here first. The column
+ is nullable, matching the model; databases that have it are untouched.
+ """
+ if current not in _REVISIONS_BEFORE_0004:
+ return
+ inspector = inspect(conn)
+ if not inspector.has_table("settings"):
+ return
+ if "env_var" in {c["name"] for c in inspector.get_columns("settings")}:
+ return
+ conn.execute(text("ALTER TABLE settings ADD COLUMN env_var VARCHAR(255)"))
+ logger.info("Added settings.env_var before revision 0004")
+
+
def run_migrations(engine: Engine, target: str = "head") -> None:
"""
Run pending migrations on a database.
@@ -493,6 +520,7 @@ def run_migrations(engine: Engine, target: str = "head") -> None:
# raise after SQLite has already accepted PRAGMA ... = OFF.
_disable_fk_for_migration(conn)
with conn.begin():
+ _add_settings_env_var_before_0004(conn, current)
config.attributes["connection"] = conn
command.upgrade(config, target)
except BaseException:
diff --git a/src/local_deep_research/database/migrations/versions/0032_repair_pre_alembic_databases.py b/src/local_deep_research/database/migrations/versions/0032_repair_pre_alembic_databases.py
new file mode 100644
index 0000000000..8e287a69ea
--- /dev/null
+++ b/src/local_deep_research/database/migrations/versions/0032_repair_pre_alembic_databases.py
@@ -0,0 +1,235 @@
+"""Repair pre-Alembic databases that the current ORM cannot read.
+
+Before Alembic, every user database was built by
+``Base.metadata.create_all()``. A pre-Alembic database is stamped at 0001, so
+the baseline's ``create_all()`` never runs for it. This revision repairs what
+that leaves behind.
+
+1. Create model tables the database does not have.
+
+The old application ran ``create_all()`` at every start, so a database had
+every table of the last version that opened it. Tables added to the models
+after that version and before 0001 have no migration of their own, so
+upgrading such a database (for example from v1.2 or v1.3) never created them.
+This step creates exactly the missing tables, as 0001 does.
+
+2. Add ORM columns that pre-Alembic databases can still be missing.
+
+``create_all()`` creates missing tables but never alters an existing one, so
+a column added to a model after its table already existed never reached that
+database unless something added it explicitly. The pre-Alembic
+``_run_migrations()`` helper only did that for ``task_metadata`` (now 0002)
+and ``research_resources.document_id`` (now 0005). The remaining columns,
+with the commits that added or renamed them:
+
+- ``collections`` and ``rag_indices``: the five advanced embedding settings
+ (4438d23c92, #1054).
+- ``documents.storage_mode`` (8d7bb38d1a, encrypted PDF storage).
+- ``token_usage``: the ten context-overflow and raw Ollama fields
+ (fe415bb30f, #645).
+- ``settings.env_var`` (6784ea9747). ``run_migrations()`` adds it before
+ 0004 when 0004 is still pending, because 0004 copies it by name.
+- ``research_history.report_content`` (62928db777).
+- ``news_subscriptions.refresh_interval_minutes``, renamed in the model from
+ ``refresh_interval_hours`` (6b5bf6d220). The old column is left in place.
+
+ORM queries on these models select every mapped column, so they fail with
+"no such column" until the columns exist. Only add what is missing: newer
+databases already have these columns and may hold user data in them. Every
+column is added nullable without a server default, so SQLite's ADD COLUMN
+keeps existing rows. Two columns are backfilled, because code uses them
+without a NULL fallback:
+
+- ``refresh_interval_minutes`` becomes the old hour interval times 60, or
+ the model default of 24 hours when no hour value exists.
+- ``storage_mode`` is inferred as ``PDFStorageManager`` does: a blob means
+ ``database``, a real ``file_path`` means ``filesystem``, otherwise
+ ``none``.
+
+Each backfill only fills NULL rows and runs whenever the column exists, not
+only when this step adds it. On SQLite and SQLCipher an ADD COLUMN issued
+before any DML commits on its own, so a failed upgrade can leave the column
+added but not backfilled; the retry then fills it.
+
+3. Rewrite ``settings.type`` values to SettingType member names.
+
+``Setting.type`` is ``Enum(SettingType)``, which stores member names such as
+``SEARCH``. Revision 0004 writes the lowercase values (``app``, ``search``,
+``llm``) for re-scoped legacy ``app.*`` keys, and any ORM read that loads such
+a row fails with "is not among the defined enum values". 0004 has shipped and
+cannot change, so this step maps each value-form type to its name, both for
+databases that ran 0004 earlier and for those that run it on the way here.
+Rows already holding a name never match, so the step is idempotent.
+
+Revision ID: 0032
+Revises: 0031
+"""
+
+import sqlalchemy as sa
+from alembic import op
+from sqlalchemy import inspect
+
+revision = "0032"
+down_revision = "0031"
+branch_labels = None
+depends_on = None
+
+_ADVANCED_EMBEDDING_COLUMNS = (
+ ("splitter_type", sa.String(50)),
+ ("text_separators", sa.JSON()),
+ ("distance_metric", sa.String(50)),
+ ("normalize_vectors", sa.Boolean()),
+ ("index_type", sa.String(50)),
+)
+
+# Types match the current ORM definitions.
+_PRE_ALEMBIC_COLUMNS = {
+ "collections": _ADVANCED_EMBEDDING_COLUMNS,
+ "rag_indices": _ADVANCED_EMBEDDING_COLUMNS,
+ "documents": (("storage_mode", sa.String(20)),),
+ "token_usage": (
+ ("context_limit", sa.Integer()),
+ ("context_truncated", sa.Boolean()),
+ ("tokens_truncated", sa.Integer()),
+ ("truncation_ratio", sa.Float()),
+ ("ollama_prompt_eval_count", sa.Integer()),
+ ("ollama_eval_count", sa.Integer()),
+ ("ollama_total_duration", sa.Integer()),
+ ("ollama_load_duration", sa.Integer()),
+ ("ollama_prompt_eval_duration", sa.Integer()),
+ ("ollama_eval_duration", sa.Integer()),
+ ),
+ "settings": (("env_var", sa.String(255)),),
+ "research_history": (("report_content", sa.Text()),),
+ "news_subscriptions": (("refresh_interval_minutes", sa.Integer()),),
+}
+
+# Mirrors FILE_PATH_SENTINELS in constants.py: file_path markers that do not
+# name a stored file. Kept as literals so this revision cannot drift.
+_FILE_PATH_SENTINELS = ("metadata_only", "text_only_not_stored", "blob_deleted")
+
+# SettingType value -> member name, as Enum(SettingType) stores it. Kept as
+# literals so this revision does not change if the model enum does.
+_SETTING_TYPE_NAMES = {
+ "app": "APP",
+ "chat": "CHAT",
+ "llm": "LLM",
+ "search": "SEARCH",
+ "report": "REPORT",
+ "database": "DATABASE",
+}
+
+
+def _create_missing_tables(conn) -> None:
+ # Same import and skip as 0001: the users table lives in the auth DB.
+ from local_deep_research.database.models import Base
+
+ existing = set(inspect(conn).get_table_names())
+ missing = [
+ table
+ for table in Base.metadata.sorted_tables
+ if table.name != "users" and table.name not in existing
+ ]
+ if missing:
+ Base.metadata.create_all(conn, tables=missing)
+
+
+# One literal statement per combination of legacy hour columns present, so no
+# SQL is assembled at run time; the keys are the columns in this fixed order.
+_REFRESH_MINUTES_BACKFILL = {
+ ("refresh_interval_hours", "refresh_hours"): (
+ "UPDATE news_subscriptions SET refresh_interval_minutes = "
+ "COALESCE(refresh_interval_hours, refresh_hours, 24) * 60 "
+ "WHERE refresh_interval_minutes IS NULL"
+ ),
+ ("refresh_interval_hours",): (
+ "UPDATE news_subscriptions SET refresh_interval_minutes = "
+ "COALESCE(refresh_interval_hours, 24) * 60 "
+ "WHERE refresh_interval_minutes IS NULL"
+ ),
+ ("refresh_hours",): (
+ "UPDATE news_subscriptions SET refresh_interval_minutes = "
+ "COALESCE(refresh_hours, 24) * 60 "
+ "WHERE refresh_interval_minutes IS NULL"
+ ),
+ (): (
+ "UPDATE news_subscriptions SET refresh_interval_minutes = 24 * 60 "
+ "WHERE refresh_interval_minutes IS NULL"
+ ),
+}
+
+
+def _backfill_refresh_interval_minutes(conn, existing: set[str]) -> None:
+ hour_columns = tuple(
+ name
+ for name in ("refresh_interval_hours", "refresh_hours")
+ if name in existing
+ )
+ conn.execute(sa.text(_REFRESH_MINUTES_BACKFILL[hour_columns]))
+
+
+def _backfill_storage_mode(conn, existing: set[str]) -> None:
+ # document_blobs exists here: _create_missing_tables() ran first.
+ conn.execute(
+ sa.text(
+ "UPDATE documents SET storage_mode = CASE "
+ "WHEN EXISTS (SELECT 1 FROM document_blobs "
+ "WHERE document_blobs.document_id = documents.id) "
+ "THEN 'database' "
+ "WHEN file_path IS NOT NULL AND file_path != '' "
+ "AND file_path NOT IN (:s0, :s1, :s2) THEN 'filesystem' "
+ "ELSE 'none' END "
+ "WHERE storage_mode IS NULL"
+ ),
+ dict(zip(("s0", "s1", "s2"), _FILE_PATH_SENTINELS)),
+ )
+
+
+# Columns that code reads without a NULL fallback. Each backfill fills only
+# NULL rows, so it is safe to run whenever the column exists.
+_BACKFILLS = {
+ ("news_subscriptions", "refresh_interval_minutes"): (
+ _backfill_refresh_interval_minutes
+ ),
+ ("documents", "storage_mode"): _backfill_storage_mode,
+}
+
+
+def upgrade() -> None:
+ conn = op.get_bind()
+ _create_missing_tables(conn)
+
+ inspector = inspect(conn)
+ for table, columns in _PRE_ALEMBIC_COLUMNS.items():
+ if not inspector.has_table(table):
+ continue
+
+ existing = {column["name"] for column in inspector.get_columns(table)}
+ for name, column_type in columns:
+ if name not in existing:
+ # Nullable ADD COLUMN works directly on SQLite and preserves
+ # existing rows without rebuilding the table.
+ op.add_column(
+ table, sa.Column(name, column_type, nullable=True)
+ )
+ # Also when the column already existed: an earlier attempt may
+ # have added it (the DDL commits on its own) and then failed
+ # before its backfill committed.
+ backfill = _BACKFILLS.get((table, name))
+ if backfill is not None:
+ backfill(conn, existing)
+
+ if inspector.has_table("settings"):
+ for value, name in _SETTING_TYPE_NAMES.items():
+ conn.execute(
+ sa.text("UPDATE settings SET type = :name WHERE type = :value"),
+ {"name": name, "value": value},
+ )
+
+
+def downgrade() -> None:
+ """Keep tables, columns, backfilled values and names.
+
+ Upgrade cannot tell pre-existing tables or columns from added ones, and
+ restoring value-form setting types would make those rows unreadable again.
+ """
diff --git a/src/local_deep_research/research_library/deletion/services/document_deletion.py b/src/local_deep_research/research_library/deletion/services/document_deletion.py
index 5760fee59a..3b0b8ef6d5 100644
--- a/src/local_deep_research/research_library/deletion/services/document_deletion.py
+++ b/src/local_deep_research/research_library/deletion/services/document_deletion.py
@@ -25,6 +25,7 @@ from ....database.models.library import (
from ....database.session_context import get_user_db_session
from ....database.session_passwords import capture_request_db_password
from ..utils.cascade_helper import CascadeHelper
+from ...services.pdf_storage_manager import effective_storage_mode
# Document deletion is a read-then-delete workflow with filesystem and FAISS
@@ -229,7 +230,10 @@ class DocumentDeletionService:
result["blob_deleted"] = result["blob_size"] > 0
# 4. Delete filesystem file if exists
- if document.storage_mode == "filesystem" and document.file_path:
+ if (
+ effective_storage_mode(document) == "filesystem"
+ and document.file_path
+ ):
from ...utils import get_absolute_path_from_settings
try:
@@ -447,14 +451,16 @@ class DocumentDeletionService:
"storage_mode_updated": False,
}
- # Handle based on storage mode
- if document.storage_mode == "database":
+ # Handle based on storage mode. Documents that predate the
+ # column have it NULL, so infer it like PDFStorageManager.
+ storage_mode = effective_storage_mode(document)
+ if storage_mode == "database":
# Delete blob from database
result["bytes_freed"] = CascadeHelper.delete_document_blob(
session, document_id
)
- elif document.storage_mode == "filesystem":
+ elif storage_mode == "filesystem":
# Delete filesystem file
from ...utils import get_absolute_path_from_settings
@@ -711,7 +717,7 @@ class DocumentDeletionService:
# Delete filesystem file if applicable
if (
- document.storage_mode == "filesystem"
+ effective_storage_mode(document) == "filesystem"
and document.file_path
):
from ...utils import get_absolute_path_from_settings
diff --git a/src/local_deep_research/research_library/services/library_service.py b/src/local_deep_research/research_library/services/library_service.py
index 09be4153cd..b4734a52b7 100644
--- a/src/local_deep_research/research_library/services/library_service.py
+++ b/src/local_deep_research/research_library/services/library_service.py
@@ -682,7 +682,9 @@ class LibraryService:
# Check if PDF is available (filesystem OR database)
has_pdf = bool(file_absolute_path)
- if not has_pdf and doc.storage_mode == "database":
+ # NULL storage_mode predates the column: a blob then
+ # means database storage, as infer_storage_mode() says.
+ if not has_pdf and doc.storage_mode in (None, "database"):
has_pdf = doc.id in blob_ids
has_text_db = bool(doc.text_content) # Text now in Document
@@ -1205,7 +1207,9 @@ class LibraryService:
has_pdf = bool(
doc.file_path and doc.file_path not in FILE_PATH_SENTINELS
)
- if not has_pdf and doc.storage_mode == "database":
+ # NULL storage_mode predates the column: a blob then means
+ # database storage, as infer_storage_mode() says.
+ if not has_pdf and doc.storage_mode in (None, "database"):
has_pdf = self._has_blob_in_db(session, doc.id)
return {
diff --git a/src/local_deep_research/research_library/services/pdf_storage_manager.py b/src/local_deep_research/research_library/services/pdf_storage_manager.py
index 4666d9605c..795f791231 100644
--- a/src/local_deep_research/research_library/services/pdf_storage_manager.py
+++ b/src/local_deep_research/research_library/services/pdf_storage_manager.py
@@ -82,6 +82,28 @@ def resolve_pdf_storage_mode(mode):
return mode
+def infer_storage_mode(document: Document) -> str:
+ """Infer where a document's PDF lives when ``storage_mode`` is unset.
+
+ Documents written before ``storage_mode`` existed have it NULL: a blob
+ means database storage, a real ``file_path`` means filesystem storage,
+ and anything else means no stored PDF.
+ """
+ # If there's a blob, it's database storage
+ if hasattr(document, "blob") and document.blob:
+ return "database"
+ # If there's a file_path (and not a sentinel), it's filesystem
+ if document.file_path and document.file_path not in FILE_PATH_SENTINELS:
+ return "filesystem"
+ # Otherwise no storage
+ return "none"
+
+
+def effective_storage_mode(document: Document) -> str:
+ """Return ``storage_mode``, inferring it when the column is NULL."""
+ return document.storage_mode or infer_storage_mode(document)
+
+
# Keep the historical paper-derived basename; filesystem writes add the
# content hash so different versions cannot overwrite one another.
_ARXIV_VERSION_SUFFIX = re.compile(r"v\d+$")
@@ -299,14 +321,7 @@ class PDFStorageManager:
Infer storage mode for documents without explicit mode set.
Used for backward compatibility with existing documents.
"""
- # If there's a blob, it's database storage
- if hasattr(document, "blob") and document.blob:
- return "database"
- # If there's a file_path (and not a sentinel), it's filesystem
- if document.file_path and document.file_path not in FILE_PATH_SENTINELS:
- return "filesystem"
- # Otherwise no storage
- return "none"
+ return infer_storage_mode(document)
def _save_to_filesystem(
self,
@@ -477,9 +492,7 @@ class PDFStorageManager:
Returns:
True if deletion succeeded
"""
- storage_mode = document.storage_mode or self._infer_storage_mode(
- document
- )
+ storage_mode = effective_storage_mode(document)
try:
if storage_mode == "filesystem":
@@ -542,10 +555,14 @@ class PDFStorageManager:
Returns:
True if PDF was added, False if already had PDF or failed
"""
- # Only upgrade if document is currently text-only
- if document.storage_mode not in (None, "none"):
+ # Only upgrade if document is currently text-only. A NULL
+ # storage_mode predates the column, so infer it: a legacy row whose
+ # file_path names a stored PDF is not text-only, and replacing its
+ # file_path would orphan that file.
+ storage_mode = effective_storage_mode(document)
+ if storage_mode != "none":
logger.debug(
- f"Document {document.id} already has storage_mode={document.storage_mode}"
+ f"Document {document.id} already has storage_mode={storage_mode}"
)
return False
diff --git a/tests/database/test_alembic_migrations.py b/tests/database/test_alembic_migrations.py
index 15ad7256ac..0b37afa4f1 100644
--- a/tests/database/test_alembic_migrations.py
+++ b/tests/database/test_alembic_migrations.py
@@ -701,6 +701,7 @@ class TestMigrationChain:
"0029_gate_filesystem_pdf_storage.py",
"0030_default_time_period_all.py",
"0031_reconcile_corrupted_settings.py",
+ "0032_repair_pre_alembic_databases.py",
]
for filename in expected_files:
@@ -4258,6 +4259,443 @@ class TestPreAlembicDatabaseUpgrade:
yield engine
engine.dispose()
+ @pytest.fixture
+ def pre_1054_collections_db(self, tmp_path):
+ """A real pre-#1054 collections shape, not today's ORM schema."""
+ db_path = tmp_path / "pre_1054_collections.db"
+ engine = create_engine(f"sqlite:///{db_path}")
+ with engine.begin() as conn:
+ conn.execute(
+ text("""
+ CREATE TABLE collections (
+ id VARCHAR(36) NOT NULL PRIMARY KEY,
+ name VARCHAR(255) NOT NULL,
+ description TEXT,
+ collection_type VARCHAR(50),
+ is_default BOOLEAN,
+ embedding_model VARCHAR(100),
+ embedding_model_type VARCHAR(20),
+ embedding_dimension INTEGER,
+ chunk_size INTEGER,
+ chunk_overlap INTEGER,
+ created_at DATETIME NOT NULL,
+ updated_at DATETIME NOT NULL
+ )
+ """)
+ )
+ conn.execute(
+ text("""
+ INSERT INTO collections (id, name, created_at, updated_at)
+ VALUES ('legacy-collection', 'Existing library',
+ '2025-10-01 00:00:00', '2025-10-01 00:00:00')
+ """)
+ )
+
+ # The old application created other tables normally, but create_all
+ # cannot add the five #1054 columns to this already-existing table.
+ Base.metadata.create_all(engine, checkfirst=True)
+ yield engine
+ engine.dispose()
+
+ def test_pre_1054_collections_gain_missing_columns_and_remain_readable(
+ self, pre_1054_collections_db
+ ):
+ """Upgrading an existing collection must support current ORM reads."""
+ from local_deep_research.database.models.library import Collection
+
+ engine = pre_1054_collections_db
+ missing_columns = {
+ "splitter_type",
+ "text_separators",
+ "distance_metric",
+ "normalize_vectors",
+ "index_type",
+ }
+ before = {c["name"] for c in inspect(engine).get_columns("collections")}
+ assert missing_columns.isdisjoint(before)
+
+ run_migrations(engine)
+
+ after = {c["name"] for c in inspect(engine).get_columns("collections")}
+ assert missing_columns <= after
+ assert get_current_revision(engine) == get_head_revision()
+ with sessionmaker(bind=engine)() as session:
+ collection = session.get(Collection, "legacy-collection")
+ assert collection is not None
+ assert collection.name == "Existing library"
+ assert collection.splitter_type is None
+ assert collection.text_separators is None
+
+ # Columns that create_all() could not add to tables that already existed
+ # before Alembic, with no pre-Alembic repair (see 0032's docstring).
+ PRE_ALEMBIC_MISSING_COLUMNS = {
+ "rag_indices": (
+ "splitter_type",
+ "text_separators",
+ "distance_metric",
+ "normalize_vectors",
+ "index_type",
+ ),
+ "documents": ("storage_mode",),
+ "token_usage": (
+ "context_limit",
+ "context_truncated",
+ "tokens_truncated",
+ "truncation_ratio",
+ "ollama_prompt_eval_count",
+ "ollama_eval_count",
+ "ollama_total_duration",
+ "ollama_load_duration",
+ "ollama_prompt_eval_duration",
+ "ollama_eval_duration",
+ ),
+ "settings": ("env_var",),
+ "research_history": ("report_content",),
+ # Renamed in the model from refresh_interval_hours (6b5bf6d220).
+ "news_subscriptions": ("refresh_interval_minutes",),
+ }
+
+ # A model table with no migration of its own, absent from the old DB.
+ PRE_ALEMBIC_MISSING_TABLE = "library_statistics"
+
+ @pytest.fixture
+ def pre_alembic_missing_columns_db(self, tmp_path):
+ """Old tables that predate columns the ORM added later."""
+ engine = create_engine(f"sqlite:///{tmp_path / 'pre_alembic_cols.db'}")
+ Base.metadata.create_all(engine, checkfirst=True)
+ with engine.begin() as conn:
+ for table, columns in self.PRE_ALEMBIC_MISSING_COLUMNS.items():
+ for column in columns:
+ conn.execute(
+ text(f"ALTER TABLE {table} DROP COLUMN {column}")
+ )
+ conn.execute(text(f"DROP TABLE {self.PRE_ALEMBIC_MISSING_TABLE}"))
+ legacy_ts = "2025-08-01 00:00:00"
+ conn.execute(
+ text(
+ "ALTER TABLE news_subscriptions "
+ "ADD COLUMN refresh_interval_hours INTEGER"
+ )
+ )
+ for sub_id, hours in (("every-6h", 6), ("no-hours", None)):
+ conn.execute(
+ text(
+ "INSERT INTO news_subscriptions (id, "
+ "subscription_type, query_or_topic, "
+ "refresh_interval_hours) VALUES "
+ "(:id, 'search', 'q', :hours)"
+ ),
+ {"id": sub_id, "hours": hours},
+ )
+ conn.execute(
+ text(
+ "INSERT INTO source_types (id, name, display_name, "
+ "created_at) VALUES ('st', 'pdf', 'PDF', :ts)"
+ ),
+ {"ts": legacy_ts},
+ )
+ for doc_id, file_path in (
+ ("on-disk", "pdfs/legacy.pdf"),
+ ("in-db", None),
+ ("metadata-only", "metadata_only"),
+ ("nothing", None),
+ ):
+ conn.execute(
+ text(
+ "INSERT INTO documents (id, source_type_id, "
+ "document_hash, file_size, file_type, file_path, "
+ "status, processed_at, created_at, updated_at) "
+ "VALUES (:id, 'st', :id, 1, 'pdf', :path, "
+ "'completed', :ts, :ts, :ts)"
+ ),
+ {"id": doc_id, "path": file_path, "ts": legacy_ts},
+ )
+ conn.execute(
+ text(
+ "INSERT INTO document_blobs (document_id, pdf_binary, "
+ "stored_at) VALUES ('in-db', X'25504446', :ts)"
+ ),
+ {"ts": legacy_ts},
+ )
+ # A legacy app.* key makes 0004 copy settings rows, naming
+ # env_var explicitly and writing the type in value form.
+ for key, setting_type in (
+ ("app.iterations", "APP"),
+ ("general.legacy_marker", "APP"),
+ ):
+ conn.execute(
+ text(
+ "INSERT INTO settings (key, value, type, name, "
+ "ui_element, visible, editable, created_at, "
+ "updated_at) VALUES (:key, '3', :type, 'Legacy', "
+ "'text', 1, 1, '2025-08-01 00:00:00', "
+ "'2025-08-01 00:00:00')"
+ ),
+ {"key": key, "type": setting_type},
+ )
+ yield engine
+ engine.dispose()
+
+ def test_pre_alembic_tables_gain_missing_columns_and_remain_readable(
+ self, pre_alembic_missing_columns_db
+ ):
+ """Every table missing a later ORM column must be readable at head."""
+ from local_deep_research.database.models.library import (
+ Document,
+ RAGIndex,
+ )
+ from local_deep_research.database.models.metrics import TokenUsage
+ from local_deep_research.database.models.news import NewsSubscription
+ from local_deep_research.database.models.research import (
+ ResearchHistory,
+ )
+ from local_deep_research.database.models.settings import (
+ Setting,
+ SettingType,
+ )
+
+ engine = pre_alembic_missing_columns_db
+ assert not inspect(engine).has_table(self.PRE_ALEMBIC_MISSING_TABLE)
+ for table, columns in self.PRE_ALEMBIC_MISSING_COLUMNS.items():
+ before = {c["name"] for c in inspect(engine).get_columns(table)}
+ assert set(columns).isdisjoint(before), table
+
+ run_migrations(engine)
+
+ assert get_current_revision(engine) == get_head_revision()
+ for table, columns in self.PRE_ALEMBIC_MISSING_COLUMNS.items():
+ after = {c["name"] for c in inspect(engine).get_columns(table)}
+ assert set(columns) <= after, table
+
+ # Every model table exists and selects every mapped column.
+ with engine.connect() as conn:
+ for table in Base.metadata.sorted_tables:
+ if table.name != "users":
+ conn.execute(table.select().limit(1)).fetchall()
+
+ with sessionmaker(bind=engine)() as session:
+ assert session.query(RAGIndex).all() == []
+ assert session.query(TokenUsage).all() == []
+ assert session.query(ResearchHistory).all() == []
+ modes = {d.id: d.storage_mode for d in session.query(Document)}
+ assert modes == {
+ "on-disk": "filesystem",
+ "in-db": "database",
+ "metadata-only": "none",
+ "nothing": "none",
+ }
+ minutes = {
+ s.id: s.refresh_interval_minutes
+ for s in session.query(NewsSubscription)
+ }
+ assert minutes == {"every-6h": 360, "no-hours": 1440}
+ # Loads every row, including the one 0004 re-scoped.
+ settings = {s.key: s for s in session.query(Setting).all()}
+ assert settings["general.legacy_marker"].env_var is None
+ migrated = settings["search.iterations"]
+ assert migrated.type is SettingType.SEARCH
+ assert migrated.name == "Legacy"
+
+ def test_0032_repairs_value_form_setting_types(self, tmp_path):
+ """Rows 0004 wrote as SettingType values must become names."""
+ from alembic import command
+
+ from local_deep_research.database.models.settings import (
+ Setting,
+ SettingType,
+ )
+
+ value_forms = [member.value for member in SettingType]
+
+ engine = create_engine(f"sqlite:///{tmp_path / 'value_types.db'}")
+ try:
+ run_migrations(engine, target="0031")
+ rows = [(f"legacy.{value}", value) for value in value_forms]
+ rows.append(("already.named", "SEARCH"))
+ with engine.begin() as conn:
+ for key, setting_type in rows:
+ conn.execute(
+ text(
+ "INSERT INTO settings (key, value, type, name, "
+ "ui_element, visible, editable) VALUES "
+ "(:key, '1', :type, :key, 'text', 1, 1)"
+ ),
+ {"key": key, "type": setting_type},
+ )
+
+ run_migrations(engine)
+
+ expected = {key: setting_type.upper() for key, setting_type in rows}
+ with engine.connect() as conn:
+ stored = dict(
+ conn.execute(
+ text("SELECT key, type FROM settings")
+ ).fetchall()
+ )
+ assert {k: stored[k] for k in expected} == expected
+ with sessionmaker(bind=engine)() as session:
+ loaded = session.query(Setting).all()
+ assert {s.key for s in loaded} >= set(expected)
+
+ # Idempotent: re-running 0032 changes nothing.
+ config = get_alembic_config(engine)
+ with engine.begin() as conn:
+ config.attributes["connection"] = conn
+ command.downgrade(config, "0031")
+ command.upgrade(config, "0032")
+ with engine.connect() as conn:
+ again = dict(
+ conn.execute(
+ text("SELECT key, type FROM settings")
+ ).fetchall()
+ )
+ assert again == stored
+ finally:
+ engine.dispose()
+
+ def test_0032_backfills_columns_an_interrupted_upgrade_left_null(
+ self, tmp_path
+ ):
+ """A retry must backfill columns that already exist but hold NULL.
+
+ On SQLite an ADD COLUMN issued before any DML commits on its own, so
+ a failed 0032 can leave the column added but not backfilled.
+ """
+ engine = create_engine(f"sqlite:///{tmp_path / 'interrupted.db'}")
+ try:
+ run_migrations(engine, target="0031")
+ ts = "2025-08-01 00:00:00"
+ with engine.begin() as conn:
+ conn.execute(
+ text(
+ "ALTER TABLE news_subscriptions "
+ "ADD COLUMN refresh_interval_hours INTEGER"
+ )
+ )
+ for sub_id, hours in (("every-6h", 6), ("no-hours", None)):
+ conn.execute(
+ text(
+ "INSERT INTO news_subscriptions (id, "
+ "subscription_type, query_or_topic, "
+ "refresh_interval_hours, "
+ "refresh_interval_minutes) VALUES "
+ "(:id, 'search', 'q', :hours, NULL)"
+ ),
+ {"id": sub_id, "hours": hours},
+ )
+ conn.execute(
+ text(
+ "INSERT INTO news_subscriptions (id, "
+ "subscription_type, query_or_topic, "
+ "refresh_interval_hours, refresh_interval_minutes) "
+ "VALUES ('kept', 'search', 'q', 6, 90)"
+ )
+ )
+ conn.execute(
+ text(
+ "INSERT INTO source_types (id, name, display_name, "
+ "created_at) VALUES ('st', 'pdf', 'PDF', :ts)"
+ ),
+ {"ts": ts},
+ )
+ for doc_id, file_path, mode in (
+ ("on-disk", "pdfs/legacy.pdf", None),
+ ("in-db", None, None),
+ ("metadata-only", "metadata_only", None),
+ ("kept", "pdfs/kept.pdf", "database"),
+ ):
+ conn.execute(
+ text(
+ "INSERT INTO documents (id, source_type_id, "
+ "document_hash, file_size, file_type, file_path, "
+ "storage_mode, status, processed_at, created_at, "
+ "updated_at) VALUES (:id, 'st', :id, 1, 'pdf', "
+ ":path, :mode, 'completed', :ts, :ts, :ts)"
+ ),
+ {
+ "id": doc_id,
+ "path": file_path,
+ "mode": mode,
+ "ts": ts,
+ },
+ )
+ conn.execute(
+ text(
+ "INSERT INTO document_blobs (document_id, pdf_binary, "
+ "stored_at) VALUES ('in-db', X'25504446', :ts)"
+ ),
+ {"ts": ts},
+ )
+
+ run_migrations(engine)
+
+ assert get_current_revision(engine) == get_head_revision()
+ with engine.connect() as conn:
+ modes = dict(
+ conn.execute(
+ text("SELECT id, storage_mode FROM documents")
+ ).fetchall()
+ )
+ minutes = dict(
+ conn.execute(
+ text(
+ "SELECT id, refresh_interval_minutes "
+ "FROM news_subscriptions"
+ )
+ ).fetchall()
+ )
+ assert modes == {
+ "on-disk": "filesystem",
+ "in-db": "database",
+ "metadata-only": "none",
+ "kept": "database",
+ }
+ assert minutes == {"every-6h": 360, "no-hours": 1440, "kept": 90}
+ finally:
+ engine.dispose()
+
+ def test_0032_downgrade_preserves_columns_that_predated_migration(
+ self, tmp_path
+ ):
+ """A fresh database's embedding settings must survive rollback."""
+ from alembic import command
+
+ engine = create_engine(f"sqlite:///{tmp_path / 'fresh_collections.db'}")
+ try:
+ run_migrations(engine)
+ with engine.begin() as conn:
+ conn.execute(
+ text("""
+ INSERT INTO collections
+ (id, name, created_at, updated_at, splitter_type)
+ VALUES
+ ('configured', 'Configured collection',
+ datetime('now'), datetime('now'), 'token')
+ """)
+ )
+
+ config = get_alembic_config(engine)
+ with engine.begin() as conn:
+ config.attributes["connection"] = conn
+ command.downgrade(config, "0031")
+
+ columns = {
+ c["name"] for c in inspect(engine).get_columns("collections")
+ }
+ assert "splitter_type" in columns
+ with engine.connect() as conn:
+ assert (
+ conn.scalar(
+ text(
+ "SELECT splitter_type FROM collections "
+ "WHERE id = 'configured'"
+ )
+ )
+ == "token"
+ )
+ finally:
+ engine.dispose()
+
@pytest.fixture
def pre_alembic_db_old(self, tmp_path):
"""
@@ -6185,6 +6623,11 @@ class TestMigrationSafetyGuards:
# 0031: restoring duplicate rows or legacy corruption sentinels would
# recreate invalid settings, with no reliable way to recover prior rows.
"0031_reconcile_corrupted_settings.py",
+ # 0032 only repairs missing columns and value-form setting types.
+ # Other databases had these fields before Alembic, so a downgrade
+ # cannot safely identify which to drop, and value-form types are
+ # unreadable by the ORM.
+ "0032_repair_pre_alembic_databases.py",
}
# Migrations whose downgrade is intentionally NotImplementedError
diff --git a/tests/database/test_migration_chain_integrity.py b/tests/database/test_migration_chain_integrity.py
index 134f326950..d1a4ef4c28 100644
--- a/tests/database/test_migration_chain_integrity.py
+++ b/tests/database/test_migration_chain_integrity.py
@@ -1566,3 +1566,74 @@ def test_no_migration_opens_a_database_of_its_own():
f"{offenders}. On a SQLCipher database these either fail to open "
"the file or write an unencrypted one. Use op.get_bind()."
)
+
+
+def _declared_revisions(versions_dir):
+ """Map each ``revision`` id declared under *versions_dir* to its files.
+
+ Read with ``ast`` rather than through Alembic's ``ScriptDirectory``.
+ When two files declare the same revision id, Alembic only emits a
+ warning and keeps whichever file it loaded last, so a chain walked
+ through Alembic shows one revision and the other file's migration is
+ silently never run. A file whose ``revision`` is missing or is not a
+ string literal maps to ``None``, so it cannot slip past the check.
+ """
+ declared = {}
+ for path in sorted(Path(versions_dir).glob("*.py")):
+ if path.name.startswith("__"):
+ continue
+ tree = ast.parse(path.read_text(encoding="utf-8"), filename=str(path))
+ values = []
+ for node in tree.body:
+ if isinstance(node, ast.Assign):
+ targets, value = node.targets, node.value
+ elif isinstance(node, ast.AnnAssign) and node.value is not None:
+ targets, value = [node.target], node.value
+ else:
+ continue
+ if any(
+ isinstance(t, ast.Name) and t.id == "revision" for t in targets
+ ):
+ values.append(
+ value.value
+ if isinstance(value, ast.Constant)
+ and isinstance(value.value, str)
+ else None
+ )
+ revision = values[-1] if len(values) == 1 else None
+ declared.setdefault(revision, []).append(path.name)
+ return declared
+
+
+def test_no_two_migration_files_declare_the_same_revision_id():
+ """Every file under ``versions/`` declares its own revision id.
+
+ Two branches that each add the next-numbered migration (for example
+ two PRs that both ship revision ``"0032"`` on ``down_revision
+ "0031"``) merge without a textual conflict because the filenames
+ differ. Alembic then warns about the duplicate and loads only one of
+ them, so the other migration never runs and every chain-shape check
+ that walks Alembic's script directory still passes. Parsing the files
+ directly makes the collision a test failure.
+ """
+ declared = _declared_revisions(get_migrations_dir() / "versions")
+
+ assert sum(len(files) for files in declared.values()) >= MIN_REVISIONS, (
+ f"only {sum(len(f) for f in declared.values())} revision file(s) "
+ "parsed -- the versions directory did not resolve, so the check "
+ "below would pass on nothing"
+ )
+ assert None not in declared, (
+ "migration file(s) without exactly one string-literal "
+ f"'revision = \"...\"' assignment: {declared.get(None)}"
+ )
+ duplicates = {
+ revision: files
+ for revision, files in declared.items()
+ if len(files) > 1
+ }
+ assert duplicates == {}, (
+ "migration files that share a revision id; Alembic keeps only one "
+ f"of each and silently drops the rest: {duplicates}. Renumber the "
+ "later one onto the current head."
+ )
diff --git a/tests/research_library/deletion/services/test_document_deletion_coverage.py b/tests/research_library/deletion/services/test_document_deletion_coverage.py
index 30895fa028..52ce295d29 100644
--- a/tests/research_library/deletion/services/test_document_deletion_coverage.py
+++ b/tests/research_library/deletion/services/test_document_deletion_coverage.py
@@ -78,6 +78,36 @@ class TestDeleteDocumentFilesystem:
assert result["file_deleted"] is True
assert result["blob_deleted"] is False
+ def test_null_storage_mode_with_file_path_deletes_the_file(self):
+ """A document that predates storage_mode must not orphan its PDF."""
+ service = DocumentDeletionService(username="testuser")
+ mock_session, mock_cm = _make_session_context()
+
+ with patch(f"{MODULE_PATH}.get_user_db_session", return_value=mock_cm):
+ mock_doc = MagicMock()
+ mock_doc.id = "doc-12345678"
+ mock_doc.title = "Legacy FS Document"
+ mock_doc.storage_mode = None
+ mock_doc.blob = None
+ mock_doc.file_path = "pdfs/legacy.pdf"
+ mock_session.get.return_value = mock_doc
+
+ with patch(f"{MODULE_PATH}.CascadeHelper") as mock_helper:
+ mock_helper.get_document_collections.return_value = []
+ mock_helper.get_document_blob_size.return_value = 0
+ mock_helper.delete_filesystem_file.return_value = True
+ mock_helper.delete_document_completely.return_value = True
+
+ with patch(
+ f"{UTILS_PATH}.get_absolute_path_from_settings",
+ return_value=MagicMock(),
+ ):
+ result = service.delete_document("doc-12345678")
+
+ assert result["deleted"] is True
+ assert result["file_deleted"] is True
+ mock_helper.delete_filesystem_file.assert_called_once()
+
def test_filesystem_file_path_none_returns_no_deletion(self):
"""Should not attempt file deletion when get_absolute_path_from_settings returns None."""
service = DocumentDeletionService(username="testuser")
@@ -336,6 +366,34 @@ class TestDeleteBlobOnlyFilesystem:
assert mock_doc.storage_mode == "none"
mock_helper.delete_filesystem_file.assert_called_once()
+ def test_null_storage_mode_with_file_path_deletes_the_file(self):
+ """delete_blob_only must infer filesystem storage for legacy rows."""
+ service = DocumentDeletionService(username="testuser")
+ mock_session, mock_cm = _make_session_context()
+
+ with patch(f"{MODULE_PATH}.get_user_db_session", return_value=mock_cm):
+ mock_doc = MagicMock()
+ mock_doc.id = "doc-12345678"
+ mock_doc.storage_mode = None
+ mock_doc.blob = None
+ mock_doc.file_path = "pdfs/legacy.pdf"
+ mock_session.get.return_value = mock_doc
+
+ mock_file_path = MagicMock()
+ mock_file_path.is_file.return_value = True
+ mock_file_path.stat.return_value.st_size = 2048
+
+ with patch(f"{MODULE_PATH}.CascadeHelper") as mock_helper:
+ with patch(
+ f"{UTILS_PATH}.get_absolute_path_from_settings",
+ return_value=mock_file_path,
+ ):
+ result = service.delete_blob_only("doc-12345678")
+
+ assert result["deleted"] is True
+ assert result["bytes_freed"] == 2048
+ mock_helper.delete_filesystem_file.assert_called_once()
+
def test_filesystem_blob_no_file_path(self):
"""Should handle filesystem mode with no file_path."""
service = DocumentDeletionService(username="testuser")
diff --git a/tests/research_library/services/test_library_service_coverage.py b/tests/research_library/services/test_library_service_coverage.py
index 1b9f363084..2f1bc3d1dd 100644
--- a/tests/research_library/services/test_library_service_coverage.py
+++ b/tests/research_library/services/test_library_service_coverage.py
@@ -14,6 +14,8 @@ Focuses narrowly on uncovered branches:
from contextlib import contextmanager
from unittest.mock import Mock, MagicMock, patch
+import pytest
+
from local_deep_research.database.models.library import DocumentStatus
MODULE = "local_deep_research.research_library.services.library_service"
@@ -742,12 +744,15 @@ class TestGetDocumentsResourceCoalesce:
assert len(results) == 1
assert results[0]["id"] == "doc-dup"
- def test_batch_blob_check_for_database_storage(self, mocker):
- """Documents with storage_mode=database use batch blob check for has_pdf."""
+ # None: a document that predates the storage_mode column, whose blob
+ # still means database storage (infer_storage_mode).
+ @pytest.mark.parametrize("storage_mode", ["database", None])
+ def test_batch_blob_check_for_database_storage(self, mocker, storage_mode):
+ """Database-stored (or NULL-mode) documents use the batch blob check."""
service = _make_service()
mock_session = MagicMock()
- doc = self._make_doc(doc_id="doc-blob", storage_mode="database")
+ doc = self._make_doc(doc_id="doc-blob", storage_mode=storage_mode)
doc.file_path = None # No filesystem path
doc.filename = "paper.pdf"
diff --git a/tests/research_library/services/test_library_service_crud_deep.py b/tests/research_library/services/test_library_service_crud_deep.py
index 34eb8dbe54..88732788c9 100644
--- a/tests/research_library/services/test_library_service_crud_deep.py
+++ b/tests/research_library/services/test_library_service_crud_deep.py
@@ -12,6 +12,8 @@ Covers:
from contextlib import contextmanager
from unittest.mock import Mock, MagicMock, patch
+import pytest
+
from local_deep_research.constants import (
FILE_PATH_METADATA_ONLY,
FILE_PATH_TEXT_ONLY,
@@ -406,10 +408,13 @@ class TestGetDocumentByIdHappyPaths:
assert result["word_count"] == 0
- def test_has_pdf_via_database_storage_mode(self, mocker):
- """storage_mode='database' + no file_path checks _has_blob_in_db."""
+ # None: a document that predates the storage_mode column, whose blob
+ # still means database storage (infer_storage_mode).
+ @pytest.mark.parametrize("storage_mode", ["database", None])
+ def test_has_pdf_via_database_storage_mode(self, mocker, storage_mode):
+ """Database (or NULL) storage_mode + no file_path checks the blob."""
service, mock_doc, mock_session = self._setup_found_doc(
- mocker, file_path=FILE_PATH_METADATA_ONLY, storage_mode="database"
+ mocker, file_path=FILE_PATH_METADATA_ONLY, storage_mode=storage_mode
)
# Mock _has_blob_in_db to return True
mocker.patch.object(service, "_has_blob_in_db", return_value=True)
diff --git a/tests/research_library/services/test_pdf_storage_manager_edge_cases.py b/tests/research_library/services/test_pdf_storage_manager_edge_cases.py
index 229299a489..6d7677f47b 100644
--- a/tests/research_library/services/test_pdf_storage_manager_edge_cases.py
+++ b/tests/research_library/services/test_pdf_storage_manager_edge_cases.py
@@ -185,6 +185,7 @@ class TestUpgradeToPdf:
document.id = 10
document.storage_mode = None
document.file_path = None
+ document.blob = None # a text-only legacy row has no blob
session = MagicMock()
# No existing blob
@@ -201,6 +202,34 @@ class TestUpgradeToPdf:
mock_save.assert_called_once_with(pdf_content, document, session)
assert document.storage_mode == "database"
+ def test_upgrade_to_pdf_refuses_null_mode_with_stored_file(self):
+ """A NULL-mode legacy row whose file_path names a PDF is not text-only.
+
+ Upgrading it would store a blob and clear file_path, orphaning the
+ PDF on disk.
+ """
+ manager = _make_manager()
+ document = Mock()
+ document.id = 12
+ document.storage_mode = None
+ document.file_path = "pdfs/legacy.pdf"
+ document.blob = None
+
+ session = MagicMock()
+ session.query.return_value.filter_by.return_value.first.return_value = (
+ None
+ )
+
+ with patch.object(manager, "_save_to_database") as mock_save:
+ result = manager.upgrade_to_pdf(
+ document, b"%PDF-1.4 small content", session
+ )
+
+ assert result is False
+ mock_save.assert_not_called()
+ assert document.file_path == "pdfs/legacy.pdf"
+ assert document.storage_mode is None
+
def test_upgrade_to_pdf_with_string_none_rejected(self):
"""String 'none' storage_mode is also accepted for upgrade."""
manager = _make_manager()
diff --git a/tests/security/test_injection_and_template_safety.py b/tests/security/test_injection_and_template_safety.py
index ce4df4b985..076e517bbb 100644
--- a/tests/security/test_injection_and_template_safety.py
+++ b/tests/security/test_injection_and_template_safety.py
@@ -178,6 +178,11 @@ REVIEWED_RAW_SQL_MODULES = {
"database/migrations/versions/0013_remove_meta_search_engines.py",
"database/migrations/versions/0021_add_note_tables.py",
"database/migrations/versions/0025_split_context_warning_dismissals.py",
+ # The refresh_interval_minutes backfill interpolates only column names
+ # from its literal tuple ("refresh_interval_hours", "refresh_hours"),
+ # kept when the inspector lists them: the catalog only filters, and no
+ # name read from it reaches the SQL. Sentinels and setting types bind.
+ "database/migrations/versions/0032_repair_pre_alembic_databases.py",
}
# Layers that serve HTTP requests. Raw SQL here would sit one variable