DeepTutor: Lifelong Personalized Tutoring. https://deeptutor.info/.
Python ai-agentsai-tutorclawbotcli-tooldeepresearchinteractive-learninglearningmulti-agent-systemsrag
fix(workspace): surface unreadable migration journals and skipped corrupt sources (#1708)
evan188199-techf94f9869a836
diff --git a/deeptutor/api/routers/workspace.py b/deeptutor/api/routers/workspace.pyindex b87e1f35a..67f50a7ad 100644--- a/deeptutor/api/routers/workspace.py+++ b/deeptutor/api/routers/workspace.py@@ -173,7 +173,7 @@ async def _migrate_workspaces(path: str, workspace_id: str | None = None) -> dicfrom deeptutor.services.workspace.data_migration import assert_no_pending_recoverywith data_activity(exclusive=True):- assert_no_pending_recovery()+ assert_no_pending_recovery(reject_unreadable=True)return await migrate_location()async def migrate_location() -> dict:@@ -259,7 +259,7 @@ async def update_registered_workspace(workspace_id: str, payload: UpdateWorkspacfrom deeptutor.services.workspace.data_migration import assert_no_pending_recoverywith data_activity(exclusive=True):- assert_no_pending_recovery()+ assert_no_pending_recovery(reject_unreadable=True)workspace = get_content_workspace_service().update_workspace(workspace_id, **payload.model_dump(exclude_unset=True))diff --git a/deeptutor/services/workspace/data_migration.py b/deeptutor/services/workspace/data_migration.pyindex 197d403f0..1cdd7dd80 100644--- a/deeptutor/services/workspace/data_migration.py+++ b/deeptutor/services/workspace/data_migration.py@@ -11,6 +11,7 @@ from __future__ import annotationsfrom datetime import datetime, timezoneimport hashlibimport json+import loggingimport osfrom pathlib import Pathimport shutil@@ -25,6 +26,8 @@ from deeptutor.services.workspace.activity import data_activityfrom deeptutor.services.workspace.context import workspace_contextfrom deeptutor.services.workspace.models import WorkspaceError+logger = logging.getLogger(__name__)+FEATURES = {"chat": "Conversations","book": "Books",@@ -315,8 +318,9 @@ def preview(sessions = _sessions(source)from deeptutor.services.workspace.dependencies import dependency_closure+ closure_warnings: list[str] = []selected, session_ids = dependency_closure(- source, sessions, features, session_ids=session_ids+ source, sessions, features, session_ids=session_ids, warnings=closure_warnings)# Legacy capability artifacts may be keyed by turn IDs. Preserve a# copy of the chat/output trees while transferring only selected rows.@@ -419,6 +423,7 @@ def preview("files": sum(row["files"] for row in rows),"bytes": sum(row["bytes"] for row in rows),"blockers": blockers,+ "warnings": closure_warnings,"rows": rows,"artifact_files": artifacts,}@@ -504,7 +509,7 @@ def export_data(source_id: str, features: list[str], *, include_historical: boolroot = _journal_root() / operation_idroot.mkdir()with data_activity(exclusive=True), workspace_context(source_id):- assert_no_pending_recovery()+ assert_no_pending_recovery(reject_unreadable=True)plan = preview(source_id, "", features)paths = get_path_service()manifests = {}@@ -555,7 +560,7 @@ def migrate_data(from deeptutor.services.workspace.session_transfer import transfer_sessionswith data_activity(exclusive=True):- assert_no_pending_recovery()+ assert_no_pending_recovery(reject_unreadable=True)if "chat" in features or "attachments" in features:from deeptutor.services.storage.attachment_store import (LocalDiskAttachmentStore,@@ -893,17 +898,58 @@ def _restore_sessions(root: Path, source, target) -> None:Path(str(target.get_chat_history_db()) + suffix).unlink(missing_ok=True)-def operations() -> list[dict]:- rows = []+def _journal_rows() -> list[tuple[Path, dict | None]]:+ """Read every migration journal. Corrupt entries yield ``(path, None)``.++ Journals are only written through :func:`atomic_write_json`, so an+ unreadable journal means real corruption rather than a torn write. This+ helper runs inside the per-request recovery precheck, so it stays silent;+ ``operations`` and the strict prechecks do the reporting.+ """+ rows: list[tuple[Path, dict | None]] = []for path in _journal_root().glob("*/operation.json"):try:- rows.append(json.loads(path.read_text()))+ row = json.loads(path.read_text())+ if not isinstance(row, dict):+ raise ValueError("journal is not a JSON object")except (OSError, ValueError):- continue+ row = None+ rows.append((path, row))+ return rows+++def operations() -> list[dict]:+ rows = []+ for path, row in _journal_rows():+ if row is None:+ logger.warning("Listing unreadable migration journal %s for recovery.", path)+ try:+ created_at = datetime.fromtimestamp(+ path.stat().st_mtime, tz=timezone.utc+ ).isoformat()+ except OSError:+ created_at = ""+ rows.append(+ {+ "id": path.parent.name,+ "status": "unreadable",+ "created_at": created_at,+ "error": f"Migration journal is unreadable: {path}",+ }+ )+ else:+ rows.append(row)return sorted(rows, key=lambda row: row.get("created_at", ""), reverse=True)-def assert_no_pending_recovery() -> None:+def assert_no_pending_recovery(*, reject_unreadable: bool = False) -> None:+ """Fail when a recovery may be pending.++ Readable journals with a pending status always fail. An unreadable+ journal cannot prove a pending recovery, and this precheck runs on every+ request, so only migration-class callers pass ``reject_unreadable``;+ failing every request would lock the whole app behind one corrupt file.+ """pending = {"preparing","copying",@@ -912,7 +958,15 @@ def assert_no_pending_recovery() -> None:"cleanup_required","recovery_required",}- for row in operations():+ for path, row in _journal_rows():+ if row is None:+ if reject_unreadable:+ raise WorkspaceError(+ "A migration journal is unreadable, so a pending recovery cannot be "+ f"ruled out: {path}. Use Recover migration in Settings → Data migration to "+ "quarantine it before changing learning data."+ )+ continueif row.get("status") in pending:raise WorkspaceError("A data migration needs recovery. Open Settings → Data migration before changing learning data."@@ -928,7 +982,24 @@ def recover_operation(operation_id: str) -> dict:if not journal.exists():raise WorkspaceError("Migration not found.")with data_activity(exclusive=True):- result = json.loads(journal.read_text())+ try:+ result = json.loads(journal.read_text())+ if not isinstance(result, dict):+ raise ValueError("journal is not a JSON object")+ except (OSError, ValueError) as exc:+ # The plan and status are unrecoverable, so finish- or+ # rollback-recovery is impossible. Quarantine the journal (the+ # snapshots stay for inspection) so prechecks pass again.+ quarantine = root / "operation.json.corrupt"+ quarantine.unlink(missing_ok=True)+ journal.rename(quarantine)+ logger.warning("Quarantined unreadable migration journal %s: %s", journal, exc)+ return {+ "id": operation_id,+ "status": "recovered",+ "created_at": datetime.now(timezone.utc).isoformat(),+ "recovery_path": str(root),+ }if result["status"] in {"completed", "exported", "recovered"}:return resultplan = result["plan"]diff --git a/deeptutor/services/workspace/dependencies.py b/deeptutor/services/workspace/dependencies.pyindex f171f6252..89883e23e 100644--- a/deeptutor/services/workspace/dependencies.py+++ b/deeptutor/services/workspace/dependencies.py@@ -3,8 +3,11 @@from __future__ import annotationsimport json+import loggingimport sqlite3+_LOGGER = logging.getLogger(__name__)+MODES = {"mastery_path": "learning","immersive_reading": "reading",@@ -12,6 +15,15 @@ MODES = {}+def _warn(warnings: list[str] | None, message: str) -> None:+ """Unreadable sources stay skipped, but each skip is surfaced exactly once."""+ if warnings is not None and message in warnings:+ return+ _LOGGER.warning("%s", message)+ if warnings is not None:+ warnings.append(message)++def _preferences(row):prefs = json.loads(row.get("preferences_json") or "{}")for key, values in prefs.get("workspace_dependencies", {}).items():@@ -24,13 +36,20 @@ def _preferences(row):def dependency_closure(- paths, sessions: list[dict], features: list[str], *, session_ids: set[str] | None = None+ paths,+ sessions: list[dict],+ features: list[str],+ *,+ session_ids: set[str] | None = None,+ warnings: list[str] | None = None,) -> tuple[set[str], set[str]]:"""Move connected stores together so sources left behind keep valid links."""- sessions = _with_historical_references(paths, sessions)- selected, ids = _forward_closure(paths, sessions, features, session_ids=session_ids)+ sessions = _with_historical_references(paths, sessions, warnings)+ selected, ids = _forward_closure(+ paths, sessions, features, session_ids=session_ids, warnings=warnings+ )groups = [- _forward_closure(paths, sessions, [feature])+ _forward_closure(paths, sessions, [feature], warnings=warnings)for feature in ("book","learning",@@ -78,7 +97,11 @@ def dependency_closure(reverse_ids.add(sid)if reverse_ids - ids:related, related_ids = _forward_closure(- paths, sessions, list(selected - {"chat"}), session_ids=ids | reverse_ids+ paths,+ sessions,+ list(selected - {"chat"}),+ session_ids=ids | reverse_ids,+ warnings=warnings,)selected |= relatedids |= related_ids@@ -87,7 +110,12 @@ def dependency_closure(def _forward_closure(- paths, sessions: list[dict], features: list[str], *, session_ids: set[str] | None = None+ paths,+ sessions: list[dict],+ features: list[str],+ *,+ session_ids: set[str] | None = None,+ warnings: list[str] | None = None,) -> tuple[set[str], set[str]]:selected = set(features)all_chats = "chat" in selected and session_ids is None@@ -102,7 +130,8 @@ def _forward_closure(continuetry:book = json.loads(path.read_text())- except (OSError, ValueError):+ except (OSError, ValueError) as exc:+ _warn(warnings, f"Skipping unreadable book manifest {path}: {exc}")continuebooks.append(book)if book.get("chat_session_id"):@@ -211,7 +240,8 @@ def _forward_closure(for path in (paths.get_workspace_dir() / feature).rglob("*.json"):try:document = json.loads(path.read_text())- except (OSError, ValueError):+ except (OSError, ValueError) as exc:+ _warn(warnings, f"Skipping unreadable {feature} document {path}: {exc}")continue_collect_references(document, prefs, ids, selected)if previous == (frozenset(selected), frozenset(ids)):@@ -283,7 +313,9 @@ def _question_sessions(paths, entry_ids=None) -> set[str]:}-def _with_historical_references(paths, sessions: list[dict]) -> list[dict]:+def _with_historical_references(+ paths, sessions: list[dict], warnings: list[str] | None = None+) -> list[dict]:"""Old request snapshots retain references no longer selected by the last turn."""prefs = {row["id"]: _preferences(row) for row in sessions}aliases = {@@ -324,7 +356,11 @@ def _with_historical_references(paths, sessions: list[dict]) -> list[dict]:if sid in prefs and raw:try:collect(json.loads(raw), prefs[sid])- except (ValueError, TypeError):+ except (ValueError, TypeError) as exc:+ _warn(+ warnings,+ f"Skipping unreadable message metadata for session {sid}: {exc}",+ )continuefor sid, followup in conn.execute("SELECT session_id,followup_session_id FROM notebook_entries"@@ -342,7 +378,12 @@ def _with_historical_references(paths, sessions: list[dict]) -> list[dict]:if isinstance(value, str):try:value = json.loads(value)- except ValueError:+ except ValueError as exc:+ sid = row.get("session_id") or "unknown"+ _warn(+ warnings,+ f"Skipping unreadable PocketBase message metadata for session {sid}: {exc}",+ )continueif row.get("session_id") in prefs:collect(value, prefs[row["session_id"]])diff --git a/deeptutor/services/workspace/kb_move.py b/deeptutor/services/workspace/kb_move.pyindex d6af4ae2d..536e08392 100644--- a/deeptutor/services/workspace/kb_move.py+++ b/deeptutor/services/workspace/kb_move.py@@ -326,7 +326,7 @@ def _rebase_llamaindex_paths(def move_kb(source_id: str, target_workspace_id: str) -> dict:"""Publish one verified KB copy and preserve qualified saved references."""with data_activity(exclusive=True):- assert_no_pending_recovery()+ assert_no_pending_recovery(reject_unreadable=True)plan = preview_kb_move(source_id, target_workspace_id)if plan["blockers"]:raise WorkspaceError(" ".join(plan["blockers"]))diff --git a/deeptutor/services/workspace/session_move.py b/deeptutor/services/workspace/session_move.pyindex 795a59f22..8d2ad8e5f 100644--- a/deeptutor/services/workspace/session_move.py+++ b/deeptutor/services/workspace/session_move.py@@ -67,7 +67,7 @@ def migrate_legacy_bindings() -> int:with data_activity(exclusive=True), workspace_context():from deeptutor.services.workspace.data_migration import assert_no_pending_recovery- assert_no_pending_recovery()+ assert_no_pending_recovery(reject_unreadable=True)if marker.exists():return 0paths = get_path_service()diff --git a/tests/services/workspace/test_data_migration.py b/tests/services/workspace/test_data_migration.pyindex b31a5443f..4a466f204 100644--- a/tests/services/workspace/test_data_migration.py+++ b/tests/services/workspace/test_data_migration.py@@ -2,6 +2,7 @@ from __future__ import annotationsimport jsonfrom pathlib import Path+import reimport sqlite3import zipfile@@ -229,6 +230,47 @@ async def test_crash_after_session_commit_can_restore_both_stores(account, monkeassert_no_pending_recovery()+def test_unreadable_journal_blocks_only_migration_paths(account):+ from deeptutor.services.workspace.activity import acquire_activity+ from deeptutor.services.workspace.data_migration import (+ _journal_root,+ assert_no_pending_recovery,+ operations,+ recover_operation,+ )++ valid_id = "1" * 32+ valid_dir = _journal_root() / valid_id+ valid_dir.mkdir(parents=True, exist_ok=True)+ (valid_dir / "operation.json").write_text(+ json.dumps({"id": valid_id, "status": "completed", "created_at": "2026-01-01T00:00:00Z"})+ )+ corrupt_id = "0" * 32+ corrupt_dir = _journal_root() / corrupt_id+ corrupt_dir.mkdir(parents=True, exist_ok=True)+ corrupt_journal = corrupt_dir / "operation.json"+ corrupt_journal.write_text("{ truncated journal")++ # Settings → Data migration lists the unreadable journal instead of hiding it.+ listed = {row["id"]: row for row in operations()}+ assert listed[corrupt_id]["status"] == "unreadable"+ assert str(corrupt_journal) in listed[corrupt_id]["error"]+ # The per-request precheck (and therefore normal requests) keeps working.+ assert_no_pending_recovery()+ handle = acquire_activity()+ handle.close()+ # Migration-class prechecks reject the journal and name its full path.+ with pytest.raises(WorkspaceError, match=re.escape(str(corrupt_journal))):+ assert_no_pending_recovery(reject_unreadable=True)+ with pytest.raises(WorkspaceError, match=re.escape(str(corrupt_journal))):+ migrate_data("", account.create_workspace("Destination")["workspace_id"], ["chat"])+ # The settings recover entry quarantines the journal, not a 500.+ assert recover_operation(corrupt_id)["status"] == "recovered"+ assert not corrupt_journal.exists()+ assert (corrupt_dir / "operation.json.corrupt").is_file()+ assert_no_pending_recovery(reject_unreadable=True)++def test_crash_during_copy_tracks_partial_destination(account, monkeypatch):import shutil@@ -432,3 +474,94 @@ async def test_old_request_snapshot_keeps_cited_material(account):)assert "reading" in featuresassert row["id"] in ids+++async def test_corrupt_book_manifest_and_feature_json_are_skipped_with_warnings(account, caplog):+ from deeptutor.services.workspace.data_migration import _sessions+ from deeptutor.services.workspace.dependencies import dependency_closure++ with workspace_context():+ store = get_sqlite_session_store()+ cited = await store.create_session("Cited by good book")+ notebook_chat = await store.create_session("Cited by notebook document")+ books = get_path_service().get_book_dir()+ corrupt_book = books / "book_corrupt"+ corrupt_book.mkdir(parents=True, exist_ok=True)+ (corrupt_book / "manifest.json").write_text("{corrupt manifest payload")+ good_book = books / "book_good"+ good_book.mkdir(parents=True, exist_ok=True)+ (good_book / "inputs.json").write_text(+ json.dumps(+ {+ "chat_selections": [{"session_id": cited["id"]}],+ "notebook_refs": ["notebook-one"],+ }+ )+ )+ notebook_dir = get_path_service().get_workspace_dir() / "notebook"+ notebook_dir.mkdir(parents=True, exist_ok=True)+ (notebook_dir / "corrupt.json").write_text("[corrupt notebook payload")+ (notebook_dir / "refs.json").write_text(+ json.dumps({"entries": [{"kind": "chat", "ref_id": notebook_chat["id"]}]})+ )++ warnings: list[str] = []+ with caplog.at_level("WARNING", logger="deeptutor.services.workspace.dependencies"):+ features, ids = dependency_closure(+ get_path_service(),+ _sessions(get_path_service()),+ ["chat"],+ session_ids={cited["id"]},+ warnings=warnings,+ )+ assert "book" in features+ assert "notebook" in features+ assert cited["id"] in ids+ assert notebook_chat["id"] in ids+ assert any("book_corrupt/manifest.json" in message for message in warnings)+ assert any(str(notebook_dir / "corrupt.json") in message for message in warnings)+ assert not any("payload" in message for message in warnings)+ logged = [record.getMessage() for record in caplog.records]+ assert any("book_corrupt/manifest.json" in message for message in logged)+ assert any(str(notebook_dir / "corrupt.json") in message for message in logged)+++async def test_corrupt_message_metadata_is_skipped_with_session_warning(account):+ from deeptutor.services.workspace.data_migration import _sessions+ from deeptutor.services.workspace.dependencies import dependency_closure++ with workspace_context():+ store = get_sqlite_session_store()+ row = await store.create_session("Damaged metadata")+ await store.add_message(row["id"], "user", "hello")+ with sqlite3.connect(get_path_service().get_chat_history_db()) as conn:+ conn.execute(+ "UPDATE messages SET metadata_json='{corrupt' WHERE session_id=?",+ (row["id"],),+ )++ warnings: list[str] = []+ features, ids = dependency_closure(+ get_path_service(),+ _sessions(get_path_service()),+ ["chat"],+ session_ids={row["id"]},+ warnings=warnings,+ )+ assert row["id"] in ids+ assert any(row["id"] in message and "metadata" in message for message in warnings)+ assert not any("corrupt" in message for message in warnings)+++def test_migration_preview_returns_skip_warnings(account):+ target = account.create_workspace("Destination")["workspace_id"]+ with workspace_context():+ books = get_path_service().get_book_dir()+ corrupt_book = books / "book_corrupt"+ corrupt_book.mkdir(parents=True, exist_ok=True)+ (corrupt_book / "manifest.json").write_text("{corrupt")+ plan = preview("", target, ["book"])+ assert "warnings" in plan+ assert any("book_corrupt/manifest.json" in message for message in plan["warnings"])