Skip to content

DeepTutor: Lifelong Personalized Tutoring. https://deeptutor.info/.

fix(workspace): surface unreadable migration journals and skipped corrupt sources (#1708)

evan188199-techf94f9869a836
diff --git a/deeptutor/api/routers/workspace.py b/deeptutor/api/routers/workspace.py
index b87e1f35a..67f50a7ad 100644
--- a/deeptutor/api/routers/workspace.py
+++ b/deeptutor/api/routers/workspace.py
@@ -173,7 +173,7 @@ async def _migrate_workspaces(path: str, workspace_id: str | None = None) -> dic
from deeptutor.services.workspace.data_migration import assert_no_pending_recovery
with data_activity(exclusive=True):
- assert_no_pending_recovery()
+ assert_no_pending_recovery(reject_unreadable=True)
return await migrate_location()
async def migrate_location() -> dict:
@@ -259,7 +259,7 @@ async def update_registered_workspace(workspace_id: str, payload: UpdateWorkspac
from deeptutor.services.workspace.data_migration import assert_no_pending_recovery
with data_activity(exclusive=True):
- assert_no_pending_recovery()
+ assert_no_pending_recovery(reject_unreadable=True)
workspace = get_content_workspace_service().update_workspace(
workspace_id, **payload.model_dump(exclude_unset=True)
)
diff --git a/deeptutor/services/workspace/data_migration.py b/deeptutor/services/workspace/data_migration.py
index 197d403f0..1cdd7dd80 100644
--- a/deeptutor/services/workspace/data_migration.py
+++ b/deeptutor/services/workspace/data_migration.py
@@ -11,6 +11,7 @@ from __future__ import annotations
from datetime import datetime, timezone
import hashlib
import json
+import logging
import os
from pathlib import Path
import shutil
@@ -25,6 +26,8 @@ from deeptutor.services.workspace.activity import data_activity
from deeptutor.services.workspace.context import workspace_context
from deeptutor.services.workspace.models import WorkspaceError
+logger = logging.getLogger(__name__)
+
FEATURES = {
"chat": "Conversations",
"book": "Books",
@@ -315,8 +318,9 @@ def preview(
sessions = _sessions(source)
from deeptutor.services.workspace.dependencies import dependency_closure
+ closure_warnings: list[str] = []
selected, session_ids = dependency_closure(
- source, sessions, features, session_ids=session_ids
+ source, sessions, features, session_ids=session_ids, warnings=closure_warnings
)
# Legacy capability artifacts may be keyed by turn IDs. Preserve a
# copy of the chat/output trees while transferring only selected rows.
@@ -419,6 +423,7 @@ def preview(
"files": sum(row["files"] for row in rows),
"bytes": sum(row["bytes"] for row in rows),
"blockers": blockers,
+ "warnings": closure_warnings,
"rows": rows,
"artifact_files": artifacts,
}
@@ -504,7 +509,7 @@ def export_data(source_id: str, features: list[str], *, include_historical: bool
root = _journal_root() / operation_id
root.mkdir()
with data_activity(exclusive=True), workspace_context(source_id):
- assert_no_pending_recovery()
+ assert_no_pending_recovery(reject_unreadable=True)
plan = preview(source_id, "", features)
paths = get_path_service()
manifests = {}
@@ -555,7 +560,7 @@ def migrate_data(
from deeptutor.services.workspace.session_transfer import transfer_sessions
with data_activity(exclusive=True):
- assert_no_pending_recovery()
+ assert_no_pending_recovery(reject_unreadable=True)
if "chat" in features or "attachments" in features:
from deeptutor.services.storage.attachment_store import (
LocalDiskAttachmentStore,
@@ -893,17 +898,58 @@ def _restore_sessions(root: Path, source, target) -> None:
Path(str(target.get_chat_history_db()) + suffix).unlink(missing_ok=True)
-def operations() -> list[dict]:
- rows = []
+def _journal_rows() -> list[tuple[Path, dict | None]]:
+ """Read every migration journal. Corrupt entries yield ``(path, None)``.
+
+ Journals are only written through :func:`atomic_write_json`, so an
+ unreadable journal means real corruption rather than a torn write. This
+ helper runs inside the per-request recovery precheck, so it stays silent;
+ ``operations`` and the strict prechecks do the reporting.
+ """
+ rows: list[tuple[Path, dict | None]] = []
for path in _journal_root().glob("*/operation.json"):
try:
- rows.append(json.loads(path.read_text()))
+ row = json.loads(path.read_text())
+ if not isinstance(row, dict):
+ raise ValueError("journal is not a JSON object")
except (OSError, ValueError):
- continue
+ row = None
+ rows.append((path, row))
+ return rows
+
+
+def operations() -> list[dict]:
+ rows = []
+ for path, row in _journal_rows():
+ if row is None:
+ logger.warning("Listing unreadable migration journal %s for recovery.", path)
+ try:
+ created_at = datetime.fromtimestamp(
+ path.stat().st_mtime, tz=timezone.utc
+ ).isoformat()
+ except OSError:
+ created_at = ""
+ rows.append(
+ {
+ "id": path.parent.name,
+ "status": "unreadable",
+ "created_at": created_at,
+ "error": f"Migration journal is unreadable: {path}",
+ }
+ )
+ else:
+ rows.append(row)
return sorted(rows, key=lambda row: row.get("created_at", ""), reverse=True)
-def assert_no_pending_recovery() -> None:
+def assert_no_pending_recovery(*, reject_unreadable: bool = False) -> None:
+ """Fail when a recovery may be pending.
+
+ Readable journals with a pending status always fail. An unreadable
+ journal cannot prove a pending recovery, and this precheck runs on every
+ request, so only migration-class callers pass ``reject_unreadable``;
+ failing every request would lock the whole app behind one corrupt file.
+ """
pending = {
"preparing",
"copying",
@@ -912,7 +958,15 @@ def assert_no_pending_recovery() -> None:
"cleanup_required",
"recovery_required",
}
- for row in operations():
+ for path, row in _journal_rows():
+ if row is None:
+ if reject_unreadable:
+ raise WorkspaceError(
+ "A migration journal is unreadable, so a pending recovery cannot be "
+ f"ruled out: {path}. Use Recover migration in Settings → Data migration to "
+ "quarantine it before changing learning data."
+ )
+ continue
if row.get("status") in pending:
raise WorkspaceError(
"A data migration needs recovery. Open Settings → Data migration before changing learning data."
@@ -928,7 +982,24 @@ def recover_operation(operation_id: str) -> dict:
if not journal.exists():
raise WorkspaceError("Migration not found.")
with data_activity(exclusive=True):
- result = json.loads(journal.read_text())
+ try:
+ result = json.loads(journal.read_text())
+ if not isinstance(result, dict):
+ raise ValueError("journal is not a JSON object")
+ except (OSError, ValueError) as exc:
+ # The plan and status are unrecoverable, so finish- or
+ # rollback-recovery is impossible. Quarantine the journal (the
+ # snapshots stay for inspection) so prechecks pass again.
+ quarantine = root / "operation.json.corrupt"
+ quarantine.unlink(missing_ok=True)
+ journal.rename(quarantine)
+ logger.warning("Quarantined unreadable migration journal %s: %s", journal, exc)
+ return {
+ "id": operation_id,
+ "status": "recovered",
+ "created_at": datetime.now(timezone.utc).isoformat(),
+ "recovery_path": str(root),
+ }
if result["status"] in {"completed", "exported", "recovered"}:
return result
plan = result["plan"]
diff --git a/deeptutor/services/workspace/dependencies.py b/deeptutor/services/workspace/dependencies.py
index f171f6252..89883e23e 100644
--- a/deeptutor/services/workspace/dependencies.py
+++ b/deeptutor/services/workspace/dependencies.py
@@ -3,8 +3,11 @@
from __future__ import annotations
import json
+import logging
import sqlite3
+_LOGGER = logging.getLogger(__name__)
+
MODES = {
"mastery_path": "learning",
"immersive_reading": "reading",
@@ -12,6 +15,15 @@ MODES = {
}
+def _warn(warnings: list[str] | None, message: str) -> None:
+ """Unreadable sources stay skipped, but each skip is surfaced exactly once."""
+ if warnings is not None and message in warnings:
+ return
+ _LOGGER.warning("%s", message)
+ if warnings is not None:
+ warnings.append(message)
+
+
def _preferences(row):
prefs = json.loads(row.get("preferences_json") or "{}")
for key, values in prefs.get("workspace_dependencies", {}).items():
@@ -24,13 +36,20 @@ def _preferences(row):
def dependency_closure(
- paths, sessions: list[dict], features: list[str], *, session_ids: set[str] | None = None
+ paths,
+ sessions: list[dict],
+ features: list[str],
+ *,
+ session_ids: set[str] | None = None,
+ warnings: list[str] | None = None,
) -> tuple[set[str], set[str]]:
"""Move connected stores together so sources left behind keep valid links."""
- sessions = _with_historical_references(paths, sessions)
- selected, ids = _forward_closure(paths, sessions, features, session_ids=session_ids)
+ sessions = _with_historical_references(paths, sessions, warnings)
+ selected, ids = _forward_closure(
+ paths, sessions, features, session_ids=session_ids, warnings=warnings
+ )
groups = [
- _forward_closure(paths, sessions, [feature])
+ _forward_closure(paths, sessions, [feature], warnings=warnings)
for feature in (
"book",
"learning",
@@ -78,7 +97,11 @@ def dependency_closure(
reverse_ids.add(sid)
if reverse_ids - ids:
related, related_ids = _forward_closure(
- paths, sessions, list(selected - {"chat"}), session_ids=ids | reverse_ids
+ paths,
+ sessions,
+ list(selected - {"chat"}),
+ session_ids=ids | reverse_ids,
+ warnings=warnings,
)
selected |= related
ids |= related_ids
@@ -87,7 +110,12 @@ def dependency_closure(
def _forward_closure(
- paths, sessions: list[dict], features: list[str], *, session_ids: set[str] | None = None
+ paths,
+ sessions: list[dict],
+ features: list[str],
+ *,
+ session_ids: set[str] | None = None,
+ warnings: list[str] | None = None,
) -> tuple[set[str], set[str]]:
selected = set(features)
all_chats = "chat" in selected and session_ids is None
@@ -102,7 +130,8 @@ def _forward_closure(
continue
try:
book = json.loads(path.read_text())
- except (OSError, ValueError):
+ except (OSError, ValueError) as exc:
+ _warn(warnings, f"Skipping unreadable book manifest {path}: {exc}")
continue
books.append(book)
if book.get("chat_session_id"):
@@ -211,7 +240,8 @@ def _forward_closure(
for path in (paths.get_workspace_dir() / feature).rglob("*.json"):
try:
document = json.loads(path.read_text())
- except (OSError, ValueError):
+ except (OSError, ValueError) as exc:
+ _warn(warnings, f"Skipping unreadable {feature} document {path}: {exc}")
continue
_collect_references(document, prefs, ids, selected)
if previous == (frozenset(selected), frozenset(ids)):
@@ -283,7 +313,9 @@ def _question_sessions(paths, entry_ids=None) -> set[str]:
}
-def _with_historical_references(paths, sessions: list[dict]) -> list[dict]:
+def _with_historical_references(
+ paths, sessions: list[dict], warnings: list[str] | None = None
+) -> list[dict]:
"""Old request snapshots retain references no longer selected by the last turn."""
prefs = {row["id"]: _preferences(row) for row in sessions}
aliases = {
@@ -324,7 +356,11 @@ def _with_historical_references(paths, sessions: list[dict]) -> list[dict]:
if sid in prefs and raw:
try:
collect(json.loads(raw), prefs[sid])
- except (ValueError, TypeError):
+ except (ValueError, TypeError) as exc:
+ _warn(
+ warnings,
+ f"Skipping unreadable message metadata for session {sid}: {exc}",
+ )
continue
for sid, followup in conn.execute(
"SELECT session_id,followup_session_id FROM notebook_entries"
@@ -342,7 +378,12 @@ def _with_historical_references(paths, sessions: list[dict]) -> list[dict]:
if isinstance(value, str):
try:
value = json.loads(value)
- except ValueError:
+ except ValueError as exc:
+ sid = row.get("session_id") or "unknown"
+ _warn(
+ warnings,
+ f"Skipping unreadable PocketBase message metadata for session {sid}: {exc}",
+ )
continue
if row.get("session_id") in prefs:
collect(value, prefs[row["session_id"]])
diff --git a/deeptutor/services/workspace/kb_move.py b/deeptutor/services/workspace/kb_move.py
index d6af4ae2d..536e08392 100644
--- a/deeptutor/services/workspace/kb_move.py
+++ b/deeptutor/services/workspace/kb_move.py
@@ -326,7 +326,7 @@ def _rebase_llamaindex_paths(
def move_kb(source_id: str, target_workspace_id: str) -> dict:
"""Publish one verified KB copy and preserve qualified saved references."""
with data_activity(exclusive=True):
- assert_no_pending_recovery()
+ assert_no_pending_recovery(reject_unreadable=True)
plan = preview_kb_move(source_id, target_workspace_id)
if plan["blockers"]:
raise WorkspaceError(" ".join(plan["blockers"]))
diff --git a/deeptutor/services/workspace/session_move.py b/deeptutor/services/workspace/session_move.py
index 795a59f22..8d2ad8e5f 100644
--- a/deeptutor/services/workspace/session_move.py
+++ b/deeptutor/services/workspace/session_move.py
@@ -67,7 +67,7 @@ def migrate_legacy_bindings() -> int:
with data_activity(exclusive=True), workspace_context():
from deeptutor.services.workspace.data_migration import assert_no_pending_recovery
- assert_no_pending_recovery()
+ assert_no_pending_recovery(reject_unreadable=True)
if marker.exists():
return 0
paths = get_path_service()
diff --git a/tests/services/workspace/test_data_migration.py b/tests/services/workspace/test_data_migration.py
index b31a5443f..4a466f204 100644
--- a/tests/services/workspace/test_data_migration.py
+++ b/tests/services/workspace/test_data_migration.py
@@ -2,6 +2,7 @@ from __future__ import annotations
import json
from pathlib import Path
+import re
import sqlite3
import zipfile
@@ -229,6 +230,47 @@ async def test_crash_after_session_commit_can_restore_both_stores(account, monke
assert_no_pending_recovery()
+def test_unreadable_journal_blocks_only_migration_paths(account):
+ from deeptutor.services.workspace.activity import acquire_activity
+ from deeptutor.services.workspace.data_migration import (
+ _journal_root,
+ assert_no_pending_recovery,
+ operations,
+ recover_operation,
+ )
+
+ valid_id = "1" * 32
+ valid_dir = _journal_root() / valid_id
+ valid_dir.mkdir(parents=True, exist_ok=True)
+ (valid_dir / "operation.json").write_text(
+ json.dumps({"id": valid_id, "status": "completed", "created_at": "2026-01-01T00:00:00Z"})
+ )
+ corrupt_id = "0" * 32
+ corrupt_dir = _journal_root() / corrupt_id
+ corrupt_dir.mkdir(parents=True, exist_ok=True)
+ corrupt_journal = corrupt_dir / "operation.json"
+ corrupt_journal.write_text("{ truncated journal")
+
+ # Settings → Data migration lists the unreadable journal instead of hiding it.
+ listed = {row["id"]: row for row in operations()}
+ assert listed[corrupt_id]["status"] == "unreadable"
+ assert str(corrupt_journal) in listed[corrupt_id]["error"]
+ # The per-request precheck (and therefore normal requests) keeps working.
+ assert_no_pending_recovery()
+ handle = acquire_activity()
+ handle.close()
+ # Migration-class prechecks reject the journal and name its full path.
+ with pytest.raises(WorkspaceError, match=re.escape(str(corrupt_journal))):
+ assert_no_pending_recovery(reject_unreadable=True)
+ with pytest.raises(WorkspaceError, match=re.escape(str(corrupt_journal))):
+ migrate_data("", account.create_workspace("Destination")["workspace_id"], ["chat"])
+ # The settings recover entry quarantines the journal, not a 500.
+ assert recover_operation(corrupt_id)["status"] == "recovered"
+ assert not corrupt_journal.exists()
+ assert (corrupt_dir / "operation.json.corrupt").is_file()
+ assert_no_pending_recovery(reject_unreadable=True)
+
+
def test_crash_during_copy_tracks_partial_destination(account, monkeypatch):
import shutil
@@ -432,3 +474,94 @@ async def test_old_request_snapshot_keeps_cited_material(account):
)
assert "reading" in features
assert row["id"] in ids
+
+
+async def test_corrupt_book_manifest_and_feature_json_are_skipped_with_warnings(account, caplog):
+ from deeptutor.services.workspace.data_migration import _sessions
+ from deeptutor.services.workspace.dependencies import dependency_closure
+
+ with workspace_context():
+ store = get_sqlite_session_store()
+ cited = await store.create_session("Cited by good book")
+ notebook_chat = await store.create_session("Cited by notebook document")
+ books = get_path_service().get_book_dir()
+ corrupt_book = books / "book_corrupt"
+ corrupt_book.mkdir(parents=True, exist_ok=True)
+ (corrupt_book / "manifest.json").write_text("{corrupt manifest payload")
+ good_book = books / "book_good"
+ good_book.mkdir(parents=True, exist_ok=True)
+ (good_book / "inputs.json").write_text(
+ json.dumps(
+ {
+ "chat_selections": [{"session_id": cited["id"]}],
+ "notebook_refs": ["notebook-one"],
+ }
+ )
+ )
+ notebook_dir = get_path_service().get_workspace_dir() / "notebook"
+ notebook_dir.mkdir(parents=True, exist_ok=True)
+ (notebook_dir / "corrupt.json").write_text("[corrupt notebook payload")
+ (notebook_dir / "refs.json").write_text(
+ json.dumps({"entries": [{"kind": "chat", "ref_id": notebook_chat["id"]}]})
+ )
+
+ warnings: list[str] = []
+ with caplog.at_level("WARNING", logger="deeptutor.services.workspace.dependencies"):
+ features, ids = dependency_closure(
+ get_path_service(),
+ _sessions(get_path_service()),
+ ["chat"],
+ session_ids={cited["id"]},
+ warnings=warnings,
+ )
+ assert "book" in features
+ assert "notebook" in features
+ assert cited["id"] in ids
+ assert notebook_chat["id"] in ids
+ assert any("book_corrupt/manifest.json" in message for message in warnings)
+ assert any(str(notebook_dir / "corrupt.json") in message for message in warnings)
+ assert not any("payload" in message for message in warnings)
+ logged = [record.getMessage() for record in caplog.records]
+ assert any("book_corrupt/manifest.json" in message for message in logged)
+ assert any(str(notebook_dir / "corrupt.json") in message for message in logged)
+
+
+async def test_corrupt_message_metadata_is_skipped_with_session_warning(account):
+ from deeptutor.services.workspace.data_migration import _sessions
+ from deeptutor.services.workspace.dependencies import dependency_closure
+
+ with workspace_context():
+ store = get_sqlite_session_store()
+ row = await store.create_session("Damaged metadata")
+ await store.add_message(row["id"], "user", "hello")
+ with sqlite3.connect(get_path_service().get_chat_history_db()) as conn:
+ conn.execute(
+ "UPDATE messages SET metadata_json='{corrupt' WHERE session_id=?",
+ (row["id"],),
+ )
+
+ warnings: list[str] = []
+ features, ids = dependency_closure(
+ get_path_service(),
+ _sessions(get_path_service()),
+ ["chat"],
+ session_ids={row["id"]},
+ warnings=warnings,
+ )
+ assert row["id"] in ids
+ assert any(row["id"] in message and "metadata" in message for message in warnings)
+ assert not any("corrupt" in message for message in warnings)
+
+
+def test_migration_preview_returns_skip_warnings(account):
+ target = account.create_workspace("Destination")["workspace_id"]
+ with workspace_context():
+ books = get_path_service().get_book_dir()
+ corrupt_book = books / "book_corrupt"
+ corrupt_book.mkdir(parents=True, exist_ok=True)
+ (corrupt_book / "manifest.json").write_text("{corrupt")
+ plan = preview("", target, ["book"])
+ assert "warnings" in plan
+ assert any("book_corrupt/manifest.json" in message for message in plan["warnings"])