A community-supported supercharged document management system: scan, index and archive all your documents
Python aiangulararchivingdjangodmsdocument-managementdocument-management-systemllmmachine-learningocroptical-character-recognitionpdf
Fix: rebuild the search index automatically when it is corrupted, instead of hard-failing (#14180)
Trenton Hd5425a893d6c
diff --git a/src/documents/search/_schema.py b/src/documents/search/_schema.pyindex 21889b12f..3430356e0 100644--- a/src/documents/search/_schema.py+++ b/src/documents/search/_schema.py@@ -337,4 +337,13 @@ def open_or_rebuild_index(index_dir: Path | None = None) -> tantivy.Index:idx = tantivy.Index(build_schema(), path=str(index_dir))_write_sentinels(index_dir)return idx- return tantivy.Index.open(str(index_dir))+ try:+ return tantivy.Index.open(str(index_dir))+ except ValueError:+ logger.exception(+ "Search index is corrupted or incomplete - rebuilding from scratch.",+ )+ wipe_index(index_dir)+ idx = tantivy.Index(build_schema(), path=str(index_dir))+ _write_sentinels(index_dir)+ return idxdiff --git a/src/documents/tests/search/test_schema_version.py b/src/documents/tests/search/test_schema_version.pyindex a9f9d60fd..3826f69c0 100644--- a/src/documents/tests/search/test_schema_version.py+++ b/src/documents/tests/search/test_schema_version.py@@ -21,9 +21,11 @@ import pytestimport tantivyfrom django.conf import settings as django_settings+from documents.search._schema import SCHEMA_VERSIONfrom documents.search._schema import build_schemafrom documents.search._schema import needs_rebuildfrom documents.search._schema import open_or_rebuild_index+from documents.search._schema import schema_fingerprintif TYPE_CHECKING:from pathlib import Path@@ -176,3 +178,50 @@ class TestUpgradeFromReleasedV1Index:open_or_rebuild_index(released_v1_index)assert needs_rebuild(released_v1_index) is False+++class TestCorruptedIndex:+ @pytest.fixture+ def current_index_missing_meta_json(self, tmp_path: Path) -> Path:+ """A current-schema index directory whose meta.json has gone missing.++ The sentinel matches the current schema, so needs_rebuild() reports+ False and open_or_rebuild_index() takes the plain Index.open() path.+ """+ index_dir = tmp_path / "index"+ index_dir.mkdir()+ tantivy.Index(build_schema(), path=str(index_dir))+ (index_dir / ".index_settings.json").write_text(+ json.dumps(+ {+ "schema_version": SCHEMA_VERSION,+ "language": django_settings.SEARCH_LANGUAGE,+ "schema_fingerprint": schema_fingerprint(),+ },+ ),+ )+ (index_dir / "meta.json").unlink()+ return index_dir++ def test_missing_meta_json_is_rebuilt_instead_of_raising(+ self,+ current_index_missing_meta_json: Path,+ ) -> None:+ """+ GIVEN:+ - An index directory with a sentinel matching the current+ schema, but a missing/corrupted meta.json (e.g. an+ interrupted write left the directory in a torn state)+ WHEN:+ - open_or_rebuild_index() is called against it+ THEN:+ - It rebuilds the index instead of letting tantivy's+ ValueError propagate; before this, a corrupted index left+ every read, write, and `document_index reindex --if-needed`+ hard-failing indefinitely, since the sentinel already+ matched and reported the index as up to date+ """+ open_or_rebuild_index(current_index_missing_meta_json)++ assert (current_index_missing_meta_json / "meta.json").exists()+ tantivy.Index.open(str(current_index_missing_meta_json))