Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Preserve external chat import recovery across checkpoint failures (#16458)

Daniel Hanf9cecc99dd26
diff --git a/studio/backend/core/external_import/__init__.py b/studio/backend/core/external_import/__init__.py
index cf5ede7250b..46bd261e8ac 100644
--- a/studio/backend/core/external_import/__init__.py
+++ b/studio/backend/core/external_import/__init__.py
@@ -287,18 +287,28 @@ def _import_session(source: Source, path: Path, project_id: str, summary: Import
existing = studio_db.get_chat_thread(thread_id) or {}
try:
- studio_db.upsert_chat_thread(_thread_row(transcript, existing, project_id))
+ studio_db.upsert_chat_thread(
+ _thread_row(transcript, existing, project_id),
+ external_import_session = (source.key, session_id) if not existing else None,
+ )
except studio_db.ChatThreadDeletedError:
summary.skipped += 1
return False
pending = _pending_messages(source, transcript, bool(existing))
if existing:
pending = _late_tool_results(transcript) + pending
- if pending:
- studio_db.sync_chat_messages(thread_id, pending, prune_missing = False)
- studio_db.record_external_import_mark(
- source.key, session_id, len(transcript.messages), transcript.revision
+ mark = studio_db.ExternalImportMark(
+ source = source.key,
+ session_id = session_id,
+ turns = len(transcript.messages),
+ revision = transcript.revision,
)
+ if pending:
+ studio_db.sync_chat_messages(
+ thread_id, pending, prune_missing = False, external_import_mark = mark
+ )
+ else:
+ studio_db.record_external_import_mark(*mark)
summary.new_chats += 0 if existing else 1
summary.messages += len(pending)
return True
diff --git a/studio/backend/storage/studio_db.py b/studio/backend/storage/studio_db.py
index ef9f370683c..63e5f9e9038 100644
--- a/studio/backend/storage/studio_db.py
+++ b/studio/backend/storage/studio_db.py
@@ -25,7 +25,7 @@ from datetime import datetime, timezone
from pathlib import Path
logger = logging.getLogger(__name__)
-from typing import Any, Callable, Iterable, Iterator, Mapping, Optional
+from typing import Any, Callable, Iterable, Iterator, Mapping, NamedTuple, Optional
from core import chat_originals
@@ -2219,11 +2219,19 @@ def _raise_if_chat_thread_deleted(conn: sqlite3.Connection, thread_id: str) -> N
raise ChatThreadDeletedError(thread_id)
-def upsert_chat_thread(thread: dict) -> dict:
+def upsert_chat_thread(
+ thread: dict, *, external_import_session: Optional[tuple[str, str]] = None
+) -> dict:
conn = get_connection(_CONTENDED_BUSY_TIMEOUT_SECONDS)
try:
conn.execute("BEGIN IMMEDIATE")
_raise_if_chat_thread_deleted(conn, thread["id"])
+ if external_import_session is not None:
+ conn.execute(
+ "DELETE FROM external_import_sessions WHERE source = ? AND session_id = ? "
+ "AND NOT EXISTS (SELECT 1 FROM chat_threads WHERE id = ?)",
+ (*external_import_session, thread["id"]),
+ )
conn.execute(
"""
INSERT INTO chat_threads
@@ -4346,6 +4354,13 @@ def upsert_chat_message(
conn.close()
+class ExternalImportMark(NamedTuple):
+ source: str
+ session_id: str
+ turns: int
+ revision: str = ""
+
+
def sync_chat_messages(
thread_id: str,
messages: list[dict],
@@ -4353,6 +4368,7 @@ def sync_chat_messages(
*,
allow_research_update: bool = False,
deleted_message_ids: Iterable[str] = (),
+ external_import_mark: Optional[ExternalImportMark] = None,
) -> list[dict]:
conn = get_connection()
try:
@@ -4510,6 +4526,8 @@ def sync_chat_messages(
max(int(m["createdAt"]) for m in reconciled_messages),
)
_mark_chat_attachment_inventory_clean(conn)
+ if external_import_mark is not None:
+ _record_external_import_mark(conn, external_import_mark)
conn.commit()
return list_chat_messages(thread_id)
except (ChatMessageConflictError, ChatMessageProtectedError):
@@ -6072,6 +6090,21 @@ def get_external_import_mark(source: str, session_id: str) -> Optional[tuple[int
conn.close()
+def _record_external_import_mark(conn: sqlite3.Connection, mark: ExternalImportMark) -> None:
+ conn.execute(
+ """
+ INSERT INTO external_import_sessions (source, session_id, turns_imported, revision)
+ VALUES (?, ?, ?, ?)
+ ON CONFLICT(source, session_id) DO UPDATE SET
+ turns_imported = CASE WHEN excluded.revision = external_import_sessions.revision
+ THEN MAX(excluded.turns_imported, external_import_sessions.turns_imported)
+ ELSE excluded.turns_imported END,
+ revision = excluded.revision
+ """,
+ (mark.source, mark.session_id, int(mark.turns), mark.revision),
+ )
+
+
def record_external_import_mark(
source: str,
session_id: str,
@@ -6080,18 +6113,7 @@ def record_external_import_mark(
) -> None:
conn = get_connection()
try:
- conn.execute(
- """
- INSERT INTO external_import_sessions (source, session_id, turns_imported, revision)
- VALUES (?, ?, ?, ?)
- ON CONFLICT(source, session_id) DO UPDATE SET
- turns_imported = CASE WHEN excluded.revision = external_import_sessions.revision
- THEN MAX(excluded.turns_imported, external_import_sessions.turns_imported)
- ELSE excluded.turns_imported END,
- revision = excluded.revision
- """,
- (source, session_id, int(turns), revision),
- )
+ _record_external_import_mark(conn, ExternalImportMark(source, session_id, turns, revision))
conn.commit()
finally:
conn.close()
diff --git a/studio/backend/tests/test_external_import_atomic.py b/studio/backend/tests/test_external_import_atomic.py
new file mode 100644
index 00000000000..59e5cc90c79
--- /dev/null
+++ b/studio/backend/tests/test_external_import_atomic.py
@@ -0,0 +1,188 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+import json
+import sqlite3
+from contextlib import contextmanager
+
+import pytest
+
+from core.external_import import claude, run_import, thread_id_for
+from storage import studio_db
+
+
+def _turn(
+ role,
+ uuid,
+ text,
+ parent = None,
+ second = 0,
+):
+ return {
+ "type": role,
+ "uuid": uuid,
+ "parentUuid": parent,
+ "timestamp": f"2026-11-01T10:00:{second:02d}Z",
+ "message": {
+ "role": role,
+ "content": text if role == "user" else [{"type": "text", "text": text}],
+ },
+ }
+
+
+def _write(
+ path,
+ records,
+ mode = "w",
+):
+ path.parent.mkdir(parents = True, exist_ok = True)
+ with path.open(mode, encoding = "utf-8") as handle:
+ handle.write("".join(json.dumps(record) + "\n" for record in records))
+
+
+def _execute(sql):
+ conn = studio_db.get_connection()
+ try:
+ conn.execute(sql)
+ conn.commit()
+ finally:
+ conn.close()
+
+
+@contextmanager
+def _reject_checkpoint():
+ _execute(
+ "CREATE TRIGGER reject_external_import_checkpoint "
+ "BEFORE INSERT ON external_import_sessions "
+ "BEGIN SELECT RAISE(ABORT, 'checkpoint write failed'); END"
+ )
+ try:
+ yield
+ finally:
+ _execute("DROP TRIGGER reject_external_import_checkpoint")
+
+
+def source_path(tmp_path, monkeypatch):
+ home = tmp_path / "claude"
+ monkeypatch.setenv(claude.SOURCE.home_env, str(home))
+ path = home / "projects" / "-Users-me-app" / "s1.jsonl"
+ _write(path, [_turn("user", "u1", "first")])
+ return path
+
+
+def _messages():
+ return studio_db.list_chat_messages(thread_id_for(claude.SOURCE, "s1"))
+
+
+def _mark():
+ return studio_db.get_external_import_mark(claude.SOURCE.key, "s1")
+
+
+def _texts(messages):
+ return [message["content"][0]["text"] for message in messages]
+
+
+def test_first_import_rolls_back_messages_when_checkpoint_fails(source_path):
+ with (
+ _reject_checkpoint(),
+ pytest.raises(sqlite3.IntegrityError, match = "checkpoint write failed"),
+ ):
+ run_import(claude.SOURCE)
+
+ assert _messages() == []
+ assert _mark() is None
+ _write(source_path, [_turn("user", "u2", "second", parent = "u1", second = 1)], mode = "a")
+
+ assert run_import(claude.SOURCE).messages == 2
+ rows = _messages()
+ assert _texts(rows) == ["first", "second"]
+ assert rows[1]["parentId"] == rows[0]["id"]
+ assert _mark() == (2, "")
+ assert run_import(claude.SOURCE).messages == 0
+ assert _messages() == rows
+
+
+def test_incremental_import_rolls_back_and_preserves_studio_changes(source_path):
+ _write(source_path, [_turn("assistant", "a1", "reply", parent = "u1", second = 1)], mode = "a")
+ assert run_import(claude.SOURCE).messages == 2
+ first, reply = _messages()
+ studio_db.sync_chat_messages(
+ thread_id_for(claude.SOURCE, "s1"),
+ [{**first, "content": [{"type": "text", "text": "edited in Studio"}]}],
+ prune_missing = True,
+ )
+ before = _messages()
+ assert _texts(before) == ["edited in Studio"]
+ assert _mark() == (2, "")
+ _write(source_path, [_turn("user", "u2", "second", parent = "a1", second = 2)], mode = "a")
+
+ with (
+ _reject_checkpoint(),
+ pytest.raises(sqlite3.IntegrityError, match = "checkpoint write failed"),
+ ):
+ run_import(claude.SOURCE)
+
+ assert _messages() == before
+ assert _mark() == (2, "")
+ _write(source_path, [_turn("user", "u3", "third", parent = "u2", second = 3)], mode = "a")
+
+ assert run_import(claude.SOURCE).messages == 2
+ rows = _messages()
+ assert _texts(rows) == ["edited in Studio", "second", "third"]
+ assert reply["id"] not in {message["id"] for message in rows}
+ assert rows[1]["parentId"] == rows[0]["id"]
+ assert rows[2]["parentId"] == rows[1]["id"]
+ assert _mark() == (4, "")
+ assert run_import(claude.SOURCE).messages == 0
+ assert _messages() == rows
+
+
+def test_checkpoint_only_failure_keeps_existing_import_intact(source_path):
+ assert run_import(claude.SOURCE).messages == 1
+ before = _messages()
+ assert _texts(before) == ["first"]
+ assert _mark() == (1, "")
+
+ with (
+ _reject_checkpoint(),
+ pytest.raises(sqlite3.IntegrityError, match = "checkpoint write failed"),
+ ):
+ run_import(claude.SOURCE)
+
+ assert _messages() == before
+ assert _mark() == (1, "")
+ assert run_import(claude.SOURCE).messages == 0
+ assert _messages() == before
+ assert _mark() == (1, "")
+
+
+def test_recreated_chat_drops_its_deleted_incarnations_checkpoint(source_path):
+ assert run_import(claude.SOURCE).messages == 1
+ studio_db.delete_chat_threads([thread_id_for(claude.SOURCE, "s1")])
+ assert _mark() == (1, "")
+
+ with (
+ _reject_checkpoint(),
+ pytest.raises(sqlite3.IntegrityError, match = "checkpoint write failed"),
+ ):
+ run_import(claude.SOURCE)
+
+ assert _messages() == []
+ assert _mark() is None
+ _write(source_path, [_turn("user", "u2", "second", parent = "u1", second = 1)], mode = "a")
+ assert run_import(claude.SOURCE).messages == 2
+ rows = _messages()
+ assert _texts(rows) == ["first", "second"]
+ assert rows[1]["parentId"] == rows[0]["id"]
+ assert _mark() == (2, "")
+ assert run_import(claude.SOURCE).messages == 0
+ assert _messages() == rows
+
+
+def test_empty_existing_chat_keeps_intentional_message_deletions(source_path):
+ assert run_import(claude.SOURCE).messages == 1
+ studio_db.sync_chat_messages(thread_id_for(claude.SOURCE, "s1"), [], prune_missing = True)
+ assert run_import(claude.SOURCE).messages == 0
+ assert _messages() == []
+ assert _mark() == (1, "")