LLM inference in C/C++
C++ ggml
ui: detect the conversation import format from file contents (#26121)
Pascal55b7d6c4c7e5
diff --git a/tools/ui/src/lib/components/app/settings/SettingsChat/SettingsChatImportExportTab.svelte b/tools/ui/src/lib/components/app/settings/SettingsChat/SettingsChatImportExportTab.svelteindex a86d68584c..57dbba30bc 100644--- a/tools/ui/src/lib/components/app/settings/SettingsChat/SettingsChatImportExportTab.svelte+++ b/tools/ui/src/lib/components/app/settings/SettingsChat/SettingsChatImportExportTab.svelte@@ -159,8 +159,10 @@try {const input = document.createElement('input');+ // No `accept` filter: iOS resolves each entry to a UTI and has none for+ // `.jsonl`, which greys out exported conversations in the file picker.+ // `parseImportFile` detects the format from the file contents instead.input.type = HtmlInputType.FILE;- input.accept = `${FileExtensionText.JSON},${FileExtensionText.JSONL},${FileExtensionText.ZIP}`;input.onchange = async (e) => {const file = (e.target as HTMLInputElement)?.files?.[0];@@ -199,9 +201,17 @@.snapshot(fullImportData).filter((item) => selectedIds.has(item.conv.id));- await conversationsStore.importConversationsData(selectedData);+ const { imported, skipped } = await conversationsStore.importConversationsData(selectedData);- importedConversations = selectedConversations;+ // A conversation already in the database is left untouched, so the summary+ // lists what was written and the toast accounts for the rest.+ if (skipped.length > 0) {+ toast.info(+ `Skipped ${skipped.length} conversation${skipped.length === 1 ? '' : 's'} already in your library`+ );+ }++ importedConversations = imported;showImportSummary = true;showExportSummary = false;showImportDialog = false;diff --git a/tools/ui/src/lib/constants/conversation-import.ts b/tools/ui/src/lib/constants/conversation-import.tsnew file mode 100644index 0000000000..ed500440a4--- /dev/null+++ b/tools/ui/src/lib/constants/conversation-import.ts@@ -0,0 +1,3 @@+// First bytes of every ZIP local file header ("PK"). Import detects an archive+// from these bytes rather than from the filename, which the OS may not preserve.+export const ZIP_MAGIC = [0x50, 0x4b];diff --git a/tools/ui/src/lib/constants/index.ts b/tools/ui/src/lib/constants/index.tsindex a80e0cb633..100432c18c 100644--- a/tools/ui/src/lib/constants/index.ts+++ b/tools/ui/src/lib/constants/index.ts@@ -13,6 +13,7 @@ export * from './storage';export * from './attachment-menu';export * from './auto-scroll';export * from './context-gauge-popup';+export * from './conversation-import';export * from './binary-detection';export * from './built-in-tools';export * from './cache';diff --git a/tools/ui/src/lib/constants/message-export.ts b/tools/ui/src/lib/constants/message-export.tsindex 79fa36f914..fc4dbe259c 100644--- a/tools/ui/src/lib/constants/message-export.ts+++ b/tools/ui/src/lib/constants/message-export.ts@@ -7,6 +7,9 @@ export const EXPORT_CONV_NAME_SUFFIX_MAX_LENGTH = 20;// Characters to keep in the ISO timestamp. 19 keeps 2026-01-01T00:00:00export const ISO_TIMESTAMP_SLICE_LENGTH = 19;+// Producer marker carried by the session record of a JSONL export+export const SESSION_HARNESS = 'llama.app';+// Replacements for making the conversation title filename-friendlyexport const NON_ALPHANUMERIC_REGEX = /[^a-z0-9]/gi;export const EXPORT_CONV_NONALNUM_REPLACEMENT = '_';diff --git a/tools/ui/src/lib/enums/conversation-import.enums.ts b/tools/ui/src/lib/enums/conversation-import.enums.tsnew file mode 100644index 0000000000..eef47c5cc1--- /dev/null+++ b/tools/ui/src/lib/enums/conversation-import.enums.ts@@ -0,0 +1,9 @@+/**+ * Discriminator of a record line in the JSONL conversation format. A session+ * record opens a conversation and carries its properties; every following+ * message record belongs to it.+ */+export enum SessionRecordType {+ SESSION = 'session',+ MESSAGE = 'message'+}diff --git a/tools/ui/src/lib/enums/index.ts b/tools/ui/src/lib/enums/index.tsindex 2f70e063d4..ee14293fc9 100644--- a/tools/ui/src/lib/enums/index.ts+++ b/tools/ui/src/lib/enums/index.ts@@ -27,6 +27,8 @@ export {ReasoningFormat} from './chat.enums';+export { SessionRecordType } from './conversation-import.enums';+export { ReasoningEffort } from './reasoning-effort.enums';export {diff --git a/tools/ui/src/lib/services/database.service.ts b/tools/ui/src/lib/services/database.service.tsindex 4fb70e29af..bc65caaca3 100644--- a/tools/ui/src/lib/services/database.service.ts+++ b/tools/ui/src/lib/services/database.service.ts@@ -554,12 +554,13 @@ export class DatabaseService {* Skips conversations that already exist.** @param data - Array of { conv, messages } objects+ * @returns The conversations written to the database and the ones skipped*/static async importConversations(data: { conv: DatabaseConversation; messages: DatabaseMessage[] }[]- ): Promise<{ imported: number; skipped: number }> {- let importedCount = 0;- let skippedCount = 0;+ ): Promise<{ imported: DatabaseConversation[]; skipped: DatabaseConversation[] }> {+ const imported: DatabaseConversation[] = [];+ const skipped: DatabaseConversation[] = [];return await db.transaction('rw',@@ -570,8 +571,7 @@ export class DatabaseService {const existing = await db[IDXDB_TABLES.conversations].get(conv.id);if (existing) {- console.warn(`Conversation "${conv.name}" already exists, skipping...`);- skippedCount++;+ skipped.push(conv);continue;}@@ -580,10 +580,10 @@ export class DatabaseService {await db[IDXDB_TABLES.messages].put(msg);}- importedCount++;+ imported.push(conv);}- return { imported: importedCount, skipped: skippedCount };+ return { imported, skipped };});}diff --git a/tools/ui/src/lib/stores/conversations.svelte.ts b/tools/ui/src/lib/stores/conversations.svelte.tsindex bc2feefd3d..e467c8fad9 100644--- a/tools/ui/src/lib/stores/conversations.svelte.ts+++ b/tools/ui/src/lib/stores/conversations.svelte.ts@@ -30,11 +30,11 @@ import type { McpServerOverride } from '$lib/types/database';import { zipSync, unzipSync, strToU8, strFromU8 } from 'fflate';import {MessageRole,- HtmlInputType,FileExtensionText,MimeTypeText,MimeTypeApplication,- ReasoningEffort+ ReasoningEffort,+ SessionRecordType} from '$lib/enums';import {ISO_DATE_TIME_SEPARATOR,@@ -47,7 +47,10 @@ import {ISO_TIME_SEPARATOR_REPLACEMENT,NON_ALPHANUMERIC_REGEX,MULTIPLE_UNDERSCORE_REGEX,- REASONING_EFFORT_DEFAULT_LOCALSTORAGE_KEY+ REASONING_EFFORT_DEFAULT_LOCALSTORAGE_KEY,+ NEWLINE,+ SESSION_HARNESS,+ ZIP_MAGIC} from '$lib/constants';import { ROUTES } from '$lib/constants/routes';@@ -914,30 +917,35 @@ class ConversationsStore {/*** Serializes a session (a conversation with its messages) as JSONL.- * The first line is the session header (a `type: 'session'` record carrying the- * conversation properties); each subsequent line is a single message.+ * The first line is the session header (a `SessionRecordType.SESSION` record+ * carrying the conversation properties); each subsequent line is a single message.* @param data - The exported conversation payload* @returns The JSONL string (one record per line)*/serializeSessionToJsonl(data: ExportedConversation): string {const { conv, messages } = data;- const sessionLine = JSON.stringify({ type: 'session', harness: 'llama.app', ...conv });+ const sessionLine = JSON.stringify({+ type: SessionRecordType.SESSION,+ harness: SESSION_HARNESS,+ ...conv+ });const messageLines = messages.map((message: DatabaseMessage) => {// `toolCalls` is stored as a JSON string; drop it when empty, otherwise parse it.const { toolCalls, ...rest } = message;const normalized = toolCalls ? { ...rest, toolCalls: JSON.parse(toolCalls) } : rest;- return JSON.stringify({ type: 'message', message: normalized });+ return JSON.stringify({ type: SessionRecordType.MESSAGE, message: normalized });});- return [sessionLine, ...messageLines].join('\n');+ return [sessionLine, ...messageLines].join(NEWLINE);}/*** Parses the JSONL session format produced by {@link serializeSessionToJsonl}.- * A `type: 'session'` line starts a new session; following `type: 'message'`- * lines are appended to it. Supports multiple sessions in a single file.+ * A `SessionRecordType.SESSION` line starts a new session; following+ * `SessionRecordType.MESSAGE` lines are appended to it. Supports multiple+ * sessions in a single file.* @param text - The JSONL file contents* @returns The parsed conversations with their messages*/@@ -945,20 +953,20 @@ class ConversationsStore {const sessions: ExportedConversation[] = [];let current: ExportedConversation | null = null;- for (const line of text.split('\n')) {+ for (const line of text.split(NEWLINE)) {const trimmed = line.trim();if (!trimmed) continue;const record = JSON.parse(trimmed);- if (record.type === 'session') {+ if (record.type === SessionRecordType.SESSION) {// Drop the discriminator and harness marker; the rest is the conversation.const conv = { ...record };delete conv.type;delete conv.harness;current = { conv: conv as DatabaseConversation, messages: [] };sessions.push(current);- } else if (record.type === 'message') {+ } else if (record.type === SessionRecordType.MESSAGE) {if (!current) {throw new Error('Invalid JSONL: message record before any session record');}@@ -977,27 +985,47 @@ class ConversationsStore {}/**- * Parses an import file into conversations, accepting the current `.jsonl` and- * `.zip` formats as well as the legacy `.json` format.+ * Reports whether the text is the JSONL session format, whose first non-empty+ * line is a `SessionRecordType.SESSION` record. A legacy JSON export starts+ * with an array or an object that has no such discriminator.+ * @param text - The file contents+ */+ private isSessionsJsonl(text: string): boolean {+ const trimmed = text.trimStart();+ const lineEnd = trimmed.indexOf(NEWLINE);+ const firstLine = lineEnd === -1 ? trimmed : trimmed.slice(0, lineEnd);++ try {+ return JSON.parse(firstLine).type === SessionRecordType.SESSION;+ } catch {+ // Not a standalone JSON record, so not the JSONL format.+ return false;+ }+ }++ /**+ * Parses an import file into conversations, accepting the current JSONL and+ * ZIP formats as well as the legacy JSON format. The format comes from the+ * contents, so an import works whatever the file is named.* @param file - The user-selected file* @returns The parsed conversations with their messages*/async parseImportFile(file: File): Promise<ExportedConversation[]> {- const name = file.name.toLowerCase();+ const bytes = new Uint8Array(await file.arrayBuffer());- if (name.endsWith(FileExtensionText.ZIP)) {- const entries = unzipSync(new Uint8Array(await file.arrayBuffer()));+ if (ZIP_MAGIC.every((byte, index) => bytes[index] === byte)) {+ const entries = unzipSync(bytes);const sessions: ExportedConversation[] = [];- for (const [entryName, bytes] of Object.entries(entries)) {+ for (const [entryName, entryBytes] of Object.entries(entries)) {if (!entryName.toLowerCase().endsWith(FileExtensionText.JSONL)) continue;- sessions.push(...this.parseSessionsJsonl(strFromU8(bytes)));+ sessions.push(...this.parseSessionsJsonl(strFromU8(entryBytes)));}return sessions;}- const text = await file.text();+ const text = strFromU8(bytes);- if (name.endsWith(FileExtensionText.JSONL)) {+ if (this.isSessionsJsonl(text)) {return this.parseSessionsJsonl(text);}@@ -1103,73 +1131,14 @@ class ConversationsStore {this.downloadConversationFile({ conv: conversation, messages });}- /**- * Imports conversations from a JSON file- * Opens file picker and processes the selected file- * @returns The list of imported conversations- */- async importConversations(): Promise<DatabaseConversation[]> {- return new Promise((resolve, reject) => {- const input = document.createElement('input');- input.type = HtmlInputType.FILE;- input.accept = FileExtensionText.JSON;-- input.onchange = async (e) => {- const file = (e.target as HTMLInputElement)?.files?.[0];-- if (!file) {- reject(new Error('No file selected'));- return;- }-- try {- const text = await file.text();- const parsedData = JSON.parse(text);- let importedData: ExportedConversations;-- if (Array.isArray(parsedData)) {- importedData = parsedData;- } else if (- parsedData &&- typeof parsedData === 'object' &&- 'conv' in parsedData &&- 'messages' in parsedData- ) {- importedData = [parsedData];- } else {- throw new Error('Invalid file format');- }-- const result = await DatabaseService.importConversations(importedData);- toast.success(`Imported ${result.imported} conversation(s), skipped ${result.skipped}`);-- await this.loadConversations();-- const importedConversations = (- Array.isArray(importedData) ? importedData : [importedData]- ).map((item) => item.conv);-- resolve(importedConversations);- } catch (err: unknown) {- const message = err instanceof Error ? err.message : 'Unknown error';- console.error('Failed to import conversations:', err);- toast.error('Import failed', { description: message });- reject(new Error(`Import failed: ${message}`));- }- };-- input.click();- });- }-/*** Imports conversations from provided data (without file picker)* @param data - Array of conversation data with messages- * @returns Import result with counts+ * @returns The conversations written to the database and the ones skipped*/async importConversationsData(data: ExportedConversations- ): Promise<{ imported: number; skipped: number }> {+ ): Promise<{ imported: DatabaseConversation[]; skipped: DatabaseConversation[] }> {const result = await DatabaseService.importConversations(data);await this.loadConversations();return result;diff --git a/tools/ui/src/lib/utils/modality-file-validation.ts b/tools/ui/src/lib/utils/modality-file-validation.tsindex bf78a70088..bfdee75ce3 100644--- a/tools/ui/src/lib/utils/modality-file-validation.ts+++ b/tools/ui/src/lib/utils/modality-file-validation.ts@@ -161,9 +161,3 @@ export function generateModalityErrorMessage(return message;}--/**- * Generate file input accept string based on model modalities- * @param capabilities - The modality capabilities to check against- * @returns Accept string for HTML file input element- */diff --git a/tools/ui/tests/client/conversation-import-db.svelte.test.ts b/tools/ui/tests/client/conversation-import-db.svelte.test.tsnew file mode 100644index 0000000000..2a27be9b0c--- /dev/null+++ b/tools/ui/tests/client/conversation-import-db.svelte.test.ts@@ -0,0 +1,68 @@+import { afterEach, describe, expect, it } from 'vitest';+import { DatabaseService } from '$lib/services/database.service';+import { MessageRole, MessageType } from '$lib/enums';+import type { ExportedConversation } from '$lib/types/database';++function makeSession(id: string): ExportedConversation {+ return {+ conv: { id, currNode: `${id}-msg`, lastModified: 0, name: `Chat ${id}` },+ messages: [+ {+ id: `${id}-msg`,+ convId: id,+ type: MessageType.TEXT,+ timestamp: 0,+ role: MessageRole.USER,+ content: `hello from ${id}`,+ parent: null,+ children: []+ }+ ]+ } as unknown as ExportedConversation;+}++afterEach(async () => {+ const conversations = await DatabaseService.getAllConversations();+ await DatabaseService.bulkDeleteConversations(conversations.map((conv) => conv.id));+});++/**+ * An import leaves a conversation already in the database untouched, so the+ * caller needs to know what was written to report it instead of echoing the+ * selection back at the user.+ */+describe('DatabaseService.importConversations', () => {+ it('reports the conversations it wrote', async () => {+ const { imported, skipped } = await DatabaseService.importConversations([+ makeSession('a'),+ makeSession('b')+ ]);++ expect(imported.map((conv) => conv.id)).toEqual(['a', 'b']);+ expect(skipped).toEqual([]);+ expect(await DatabaseService.getConversationMessages('a')).toHaveLength(1);+ });++ it('reports an existing conversation as skipped and leaves it untouched', async () => {+ await DatabaseService.importConversations([makeSession('a')]);+ await DatabaseService.updateConversation('a', { name: 'Renamed locally' });++ const { imported, skipped } = await DatabaseService.importConversations([makeSession('a')]);++ expect(imported).toEqual([]);+ expect(skipped.map((conv) => conv.id)).toEqual(['a']);+ expect((await DatabaseService.getConversation('a'))?.name).toBe('Renamed locally');+ });++ it('imports the new conversations of a partially known selection', async () => {+ await DatabaseService.importConversations([makeSession('a')]);++ const { imported, skipped } = await DatabaseService.importConversations([+ makeSession('a'),+ makeSession('b')+ ]);++ expect(imported.map((conv) => conv.id)).toEqual(['b']);+ expect(skipped.map((conv) => conv.id)).toEqual(['a']);+ });+});diff --git a/tools/ui/tests/unit/conversation-import.test.ts b/tools/ui/tests/unit/conversation-import.test.tsnew file mode 100644index 0000000000..4565ed5b5d--- /dev/null+++ b/tools/ui/tests/unit/conversation-import.test.ts@@ -0,0 +1,112 @@+import { beforeAll, describe, expect, it } from 'vitest';+import { zipSync, strToU8 } from 'fflate';+import { MessageRole, MessageType } from '$lib/enums';+import { NEWLINE } from '$lib/constants';+import type { ExportedConversation } from '$lib/types/database';++let conversationsStore: typeof import('$lib/stores/conversations.svelte').conversationsStore;++// node env unit project has no DOM, install a minimal localStorage backed by a+// Map before the store module reads it. Transforming the store takes seconds,+// so import it once for the whole file.+beforeAll(async () => {+ const store = new Map<string, string>();+ const polyfill: Storage = {+ get length() {+ return store.size;+ },+ clear: () => store.clear(),+ getItem: (k) => (store.has(k) ? store.get(k)! : null),+ key: (i) => Array.from(store.keys())[i] ?? null,+ removeItem: (k) => {+ store.delete(k);+ },+ setItem: (k, v) => {+ store.set(k, String(v));+ }+ };+ (globalThis as unknown as { localStorage: Storage }).localStorage = polyfill;++ ({ conversationsStore } = await import('$lib/stores/conversations.svelte'));+}, 30000);++function makeSession(id: string): ExportedConversation {+ return {+ conv: { id, currNode: `${id}-msg`, lastModified: 0, name: `Chat ${id}` },+ messages: [+ {+ id: `${id}-msg`,+ convId: id,+ type: MessageType.TEXT,+ timestamp: 0,+ role: MessageRole.USER,+ content: `hello from ${id}`,+ parent: null,+ children: []+ }+ ]+ } as unknown as ExportedConversation;+}++/**+ * `parseImportFile` detects the format from the file contents. iOS has no UTI+ * for `.jsonl`, so the picker cannot filter on it and the filename carries no+ * guarantee: a JSONL export must import under any name.+ */+describe('conversationsStore.parseImportFile', () => {+ it('imports a JSONL export whose name has no meaningful extension', async () => {+ const jsonl = conversationsStore.serializeSessionToJsonl(makeSession('a'));++ const sessions = await conversationsStore.parseImportFile(new File([jsonl], 'export'));++ expect(sessions).toHaveLength(1);+ expect(sessions[0].conv.id).toBe('a');+ expect(sessions[0].messages[0].content).toBe('hello from a');+ });++ it('imports several sessions from one JSONL file', async () => {+ const jsonl = [makeSession('a'), makeSession('b')]+ .map((session) => conversationsStore.serializeSessionToJsonl(session))+ .join(NEWLINE);++ const sessions = await conversationsStore.parseImportFile(new File([jsonl], 'export.txt'));++ expect(sessions.map((session) => session.conv.id)).toEqual(['a', 'b']);+ });++ it('imports a ZIP archive whose name has no meaningful extension', async () => {+ const zipped = zipSync({+ 'a.jsonl': strToU8(conversationsStore.serializeSessionToJsonl(makeSession('a'))),+ 'b.jsonl': strToU8(conversationsStore.serializeSessionToJsonl(makeSession('b'))),+ 'notes.txt': strToU8('ignored')+ });++ const sessions = await conversationsStore.parseImportFile(new File([zipped], 'archive'));++ expect(sessions.map((session) => session.conv.id).sort()).toEqual(['a', 'b']);+ });++ it('imports the legacy JSON array format', async () => {+ const json = JSON.stringify([makeSession('a')], null, 2);++ const sessions = await conversationsStore.parseImportFile(new File([json], 'export.jsonl'));++ expect(sessions).toHaveLength(1);+ expect(sessions[0].conv.id).toBe('a');+ });++ it('imports the legacy JSON single object format', async () => {+ const json = JSON.stringify(makeSession('a'));++ const sessions = await conversationsStore.parseImportFile(new File([json], 'export'));++ expect(sessions).toHaveLength(1);+ expect(sessions[0].conv.id).toBe('a');+ });++ it('rejects a file that holds neither format', async () => {+ await expect(+ conversationsStore.parseImportFile(new File(['not an export'], 'export.jsonl'))+ ).rejects.toThrow();+ });+});