Skip to content

LLM inference in C/C++

C++ ggml

webui: export conversations as jsonl (#24688)

Julien Chaumond8086439a4cea
diff --git a/tools/ui/package-lock.json b/tools/ui/package-lock.json
index 61fa529d20..9d0cdfea6c 100644
--- a/tools/ui/package-lock.json
+++ b/tools/ui/package-lock.json
@@ -40,6 +40,7 @@
"eslint-config-prettier": "10.1.8",
"eslint-plugin-storybook": "10.4.2",
"eslint-plugin-svelte": "3.19.0",
+ "fflate": "0.8.3",
"globals": "16.5.0",
"highlight.js": "11.11.1",
"http-server": "14.1.1",
@@ -9454,6 +9455,13 @@
}
}
},
+ "node_modules/fflate": {
+ "version": "0.8.3",
+ "resolved": "https://registry.npmjs.org/fflate/-/fflate-0.8.3.tgz",
+ "integrity": "sha512-tbZNuJrLwGUp3zshBtdy4W+ORxZuIh8a5ilyIEQDC5rY1f3U20JMry0Ll3WBzU58EZKsEuJFXhb5gwv8CsPvgA==",
+ "dev": true,
+ "license": "MIT"
+ },
"node_modules/file-entry-cache": {
"version": "8.0.0",
"resolved": "https://registry.npmjs.org/file-entry-cache/-/file-entry-cache-8.0.0.tgz",
diff --git a/tools/ui/package.json b/tools/ui/package.json
index 2f47992e1e..4803922889 100644
--- a/tools/ui/package.json
+++ b/tools/ui/package.json
@@ -59,6 +59,7 @@
"eslint-config-prettier": "10.1.8",
"eslint-plugin-storybook": "10.4.2",
"eslint-plugin-svelte": "3.19.0",
+ "fflate": "0.8.3",
"globals": "16.5.0",
"highlight.js": "11.11.1",
"http-server": "14.1.1",
diff --git a/tools/ui/src/lib/components/app/settings/SettingsChat/SettingsChatImportExportTab.svelte b/tools/ui/src/lib/components/app/settings/SettingsChat/SettingsChatImportExportTab.svelte
index b7b91d65b6..a86d68584c 100644
--- a/tools/ui/src/lib/components/app/settings/SettingsChat/SettingsChatImportExportTab.svelte
+++ b/tools/ui/src/lib/components/app/settings/SettingsChat/SettingsChatImportExportTab.svelte
@@ -132,14 +132,18 @@
async function handleExportConfirm(selectedConversations: DatabaseConversation[]) {
try {
- const allData: ExportedConversations = await Promise.all(
+ const allData: ExportedConversation[] = await Promise.all(
selectedConversations.map(async (conv) => {
const messages = await conversationsStore.getConversationMessages(conv.id);
return { conv: $state.snapshot(conv), messages: $state.snapshot(messages) };
})
);
- conversationsStore.downloadConversationFile(allData);
+ if (allData.length === 1) {
+ conversationsStore.downloadConversationFile(allData[0]);
+ } else {
+ conversationsStore.downloadConversationsArchive(allData);
+ }
exportedConversations = selectedConversations;
showExportSummary = true;
@@ -156,37 +160,21 @@
const input = document.createElement('input');
input.type = HtmlInputType.FILE;
- input.accept = FileExtensionText.JSON;
+ input.accept = `${FileExtensionText.JSON},${FileExtensionText.JSONL},${FileExtensionText.ZIP}`;
input.onchange = async (e) => {
const file = (e.target as HTMLInputElement)?.files?.[0];
if (!file) return;
try {
- const text = await file.text();
- const parsedData = JSON.parse(text);
- let importedData: ExportedConversations;
-
- if (Array.isArray(parsedData)) {
- importedData = parsedData;
- } else if (
- parsedData &&
- typeof parsedData === 'object' &&
- 'conv' in parsedData &&
- 'messages' in parsedData
- ) {
- // Single conversation object
- importedData = [parsedData];
- } else {
- throw new Error(
- 'Invalid file format: expected array of conversations or single conversation object'
- );
+ const importedData = await conversationsStore.parseImportFile(file);
+
+ if (importedData.length === 0) {
+ throw new Error('No conversations found in file');
}
fullImportData = importedData;
- availableConversations = importedData.map(
- (item: { conv: DatabaseConversation; messages: DatabaseMessage[] }) => item.conv
- );
+ availableConversations = importedData.map((item) => item.conv);
messageCountMap = createMessageCountMap(importedData);
showImportDialog = true;
} catch (err: unknown) {
@@ -258,7 +246,7 @@
<SettingsGroup title="Conversations">
<SettingsChatImportExportSection
title="Export"
- description="Download your conversations as a JSON file. This includes all messages, attachments, and conversation history."
+ description="Download your conversations as a ZIP of JSONL files. This includes all messages, attachments, and conversation history."
IconComponent={Download}
buttonText="Export conversations"
onclick={handleExportClick}
@@ -267,7 +255,7 @@
<SettingsChatImportExportSection
title="Import"
- description="Import one or more conversations from a previously exported JSON file. This will merge with your existing conversations."
+ description="Import one or more conversations from a previously exported ZIP or JSONL file. This will merge with your existing conversations."
IconComponent={Upload}
buttonText="Import conversations"
onclick={handleImportClick}
diff --git a/tools/ui/src/lib/enums/files.enums.ts b/tools/ui/src/lib/enums/files.enums.ts
index dbe07ea586..a908b89965 100644
--- a/tools/ui/src/lib/enums/files.enums.ts
+++ b/tools/ui/src/lib/enums/files.enums.ts
@@ -123,6 +123,8 @@ export enum FileExtensionText {
HTML = '.html',
HTM = '.htm',
JSON = '.json',
+ JSONL = '.jsonl',
+ ZIP = '.zip',
XML = '.xml',
YAML = '.yaml',
YML = '.yml',
@@ -179,7 +181,8 @@ export enum UriPattern {
// MIME type enums
export enum MimeTypeApplication {
PDF = 'application/pdf',
- OCTET_STREAM = 'application/octet-stream'
+ OCTET_STREAM = 'application/octet-stream',
+ ZIP = 'application/zip'
}
export enum MimeTypeAudio {
@@ -226,6 +229,7 @@ export enum MimeTypeText {
CSS = 'text/css',
HTML = 'text/html',
JSON = 'application/json',
+ JSONL = 'application/jsonl',
XML_TEXT = 'text/xml',
XML_APP = 'application/xml',
YAML_TEXT = 'text/yaml',
diff --git a/tools/ui/src/lib/stores/conversations.svelte.ts b/tools/ui/src/lib/stores/conversations.svelte.ts
index bba5191ed5..235015213d 100644
--- a/tools/ui/src/lib/stores/conversations.svelte.ts
+++ b/tools/ui/src/lib/stores/conversations.svelte.ts
@@ -26,7 +26,15 @@ import { MigrationService } from '$lib/services/migration.service';
import { config } from '$lib/stores/settings.svelte';
import { filterByLeafNodeId, findLeafNode, generateConversationTitle } from '$lib/utils';
import type { McpServerOverride } from '$lib/types/database';
-import { MessageRole, HtmlInputType, FileExtensionText, ReasoningEffort } from '$lib/enums';
+import { zipSync, unzipSync, strToU8, strFromU8 } from 'fflate';
+import {
+ MessageRole,
+ HtmlInputType,
+ FileExtensionText,
+ MimeTypeText,
+ MimeTypeApplication,
+ ReasoningEffort
+} from '$lib/enums';
import {
ISO_DATE_TIME_SEPARATOR,
ISO_DATE_TIME_SEPARATOR_REPLACEMENT,
@@ -934,41 +942,177 @@ class ConversationsStore {
.replace(ISO_DATE_TIME_SEPARATOR, ISO_DATE_TIME_SEPARATOR_REPLACEMENT)
.replaceAll(ISO_TIME_SEPARATOR, ISO_TIME_SEPARATOR_REPLACEMENT);
const trimmedConvId = conversation.id?.slice(0, EXPORT_CONV_ID_TRIM_LENGTH) ?? '';
- return `${formattedDate}_conv_${trimmedConvId}_${sanitizedName}.json`;
+ return `${formattedDate}_conv_${trimmedConvId}_${sanitizedName}${FileExtensionText.JSONL}`;
+ }
+
+ /**
+ * Serializes a session (a conversation with its messages) as JSONL.
+ * The first line is the session header (a `type: 'session'` record carrying the
+ * conversation properties); each subsequent line is a single message.
+ * @param data - The exported conversation payload
+ * @returns The JSONL string (one record per line)
+ */
+ serializeSessionToJsonl(data: ExportedConversation): string {
+ const { conv, messages } = data;
+
+ const sessionLine = JSON.stringify({ type: 'session', harness: 'llama.app', ...conv });
+ const messageLines = messages.map((message: DatabaseMessage) => {
+ // `toolCalls` is stored as a JSON string; drop it when empty, otherwise parse it.
+ const { toolCalls, ...rest } = message;
+ const normalized = toolCalls ? { ...rest, toolCalls: JSON.parse(toolCalls) } : rest;
+
+ return JSON.stringify({ type: 'message', message: normalized });
+ });
+
+ return [sessionLine, ...messageLines].join('\n');
+ }
+
+ /**
+ * Parses the JSONL session format produced by {@link serializeSessionToJsonl}.
+ * A `type: 'session'` line starts a new session; following `type: 'message'`
+ * lines are appended to it. Supports multiple sessions in a single file.
+ * @param text - The JSONL file contents
+ * @returns The parsed conversations with their messages
+ */
+ parseSessionsJsonl(text: string): ExportedConversation[] {
+ const sessions: ExportedConversation[] = [];
+ let current: ExportedConversation | null = null;
+
+ for (const line of text.split('\n')) {
+ const trimmed = line.trim();
+ if (!trimmed) continue;
+
+ const record = JSON.parse(trimmed);
+
+ if (record.type === 'session') {
+ // Drop the discriminator and harness marker; the rest is the conversation.
+ const conv = { ...record };
+ delete conv.type;
+ delete conv.harness;
+ current = { conv: conv as DatabaseConversation, messages: [] };
+ sessions.push(current);
+ } else if (record.type === 'message') {
+ if (!current) {
+ throw new Error('Invalid JSONL: message record before any session record');
+ }
+
+ const message = record.message as DatabaseMessage;
+ // `toolCalls` is parsed to an array on export; the DB stores it as a string.
+ if (message.toolCalls !== undefined && typeof message.toolCalls !== 'string') {
+ message.toolCalls = JSON.stringify(message.toolCalls);
+ }
+ current.messages.push(message);
+ }
+ // Ignore unknown record types for forward compatibility.
+ }
+
+ return sessions;
+ }
+
+ /**
+ * Parses an import file into conversations, accepting the current `.jsonl` and
+ * `.zip` formats as well as the legacy `.json` format.
+ * @param file - The user-selected file
+ * @returns The parsed conversations with their messages
+ */
+ async parseImportFile(file: File): Promise<ExportedConversation[]> {
+ const name = file.name.toLowerCase();
+
+ if (name.endsWith(FileExtensionText.ZIP)) {
+ const entries = unzipSync(new Uint8Array(await file.arrayBuffer()));
+ const sessions: ExportedConversation[] = [];
+ for (const [entryName, bytes] of Object.entries(entries)) {
+ if (!entryName.toLowerCase().endsWith(FileExtensionText.JSONL)) continue;
+ sessions.push(...this.parseSessionsJsonl(strFromU8(bytes)));
+ }
+ return sessions;
+ }
+
+ const text = await file.text();
+
+ if (name.endsWith(FileExtensionText.JSONL)) {
+ return this.parseSessionsJsonl(text);
+ }
+
+ // Legacy JSON format: an array of conversations or a single conversation object.
+ const parsed = JSON.parse(text);
+ if (Array.isArray(parsed)) {
+ return parsed;
+ }
+ if (parsed && typeof parsed === 'object' && 'conv' in parsed && 'messages' in parsed) {
+ return [parsed];
+ }
+ throw new Error(
+ 'Invalid file format: expected array of conversations or single conversation object'
+ );
}
/**
* Triggers a browser download of the provided exported conversation data
- * @param data - The exported conversation payload (either a single conversation or array of them)
+ * @param data - The exported conversation payload (a single conversation with its messages)
* @param filename - Filename; if omitted, a deterministic name is generated
*/
- downloadConversationFile(data: ExportedConversations, filename?: string): void {
- // Choose the first conversation or message
- const conversation =
- 'conv' in data ? data.conv : Array.isArray(data) ? data[0]?.conv : undefined;
- const msgs =
- 'messages' in data ? data.messages : Array.isArray(data) ? data[0]?.messages : undefined;
+ downloadConversationFile(data: ExportedConversation, filename?: string): void {
+ const { conv: conversation, messages: msgs } = data;
if (!conversation) {
console.error('Invalid data: missing conversation');
return;
}
- let downloadFilename: string;
+ const downloadFilename = filename ?? this.generateConversationFilename(conversation, msgs);
- if (filename) {
- downloadFilename = filename;
- } else if (Array.isArray(data) && data.length > 1) {
- downloadFilename = `${new Date().toISOString().split(ISO_DATE_TIME_SEPARATOR)[0]}_conversations.json`;
- } else {
- downloadFilename = this.generateConversationFilename(conversation, msgs);
+ const jsonl = this.serializeSessionToJsonl(data);
+ const blob = new Blob([jsonl], { type: MimeTypeText.JSONL });
+ this.triggerDownload(blob, downloadFilename);
+ }
+
+ /**
+ * Triggers a browser download of multiple conversations as a `.zip`, one
+ * `.jsonl` file per conversation.
+ * @param data - The conversations to export
+ */
+ downloadConversationsArchive(data: ExportedConversation[]): void {
+ if (data.length === 0) {
+ console.error('Invalid data: no conversations to export');
+ return;
}
- const blob = new Blob([JSON.stringify(data, null, 2)], { type: 'application/json' });
+ const usedNames = new SvelteSet<string>();
+ const files: Record<string, Uint8Array> = {};
+
+ for (const session of data) {
+ const baseName = this.generateConversationFilename(session.conv, session.messages);
+
+ // Disambiguate any duplicate filenames within the archive.
+ let entryName = baseName;
+ let suffix = 1;
+ while (usedNames.has(entryName)) {
+ entryName = baseName.replace(
+ new RegExp(`${FileExtensionText.JSONL}$`),
+ `_${suffix++}${FileExtensionText.JSONL}`
+ );
+ }
+ usedNames.add(entryName);
+
+ files[entryName] = strToU8(this.serializeSessionToJsonl(session));
+ }
+
+ const archiveName = `${new Date().toISOString().split(ISO_DATE_TIME_SEPARATOR)[0]}_conversations${FileExtensionText.ZIP}`;
+
+ const zipped = zipSync(files);
+ const blob = new Blob([zipped], { type: MimeTypeApplication.ZIP });
+ this.triggerDownload(blob, archiveName);
+ }
+
+ /**
+ * Triggers a browser download of a blob under the given filename.
+ */
+ private triggerDownload(blob: Blob, filename: string): void {
const url = URL.createObjectURL(blob);
const a = document.createElement('a');
a.href = url;
- a.download = downloadFilename;
+ a.download = filename;
document.body.appendChild(a);
a.click();
document.body.removeChild(a);