Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
/**
* ChatGPT Import Service
*
* Parses ChatGPT data exports and imports them as Brain Memory entries. Full
* conversation transcripts are also archived to
* `data/brain/imports/chatgpt/<conversation-id>.json` so the original structure
* (mapping tree, model, citations) and a rich markdown transcript are preserved
* for the in-app conversation viewer.
*
* Two upload shapes are supported:
* - The legacy single `conversations.json` (parsed in the browser, POSTed as
* JSON). No assets — image/audio parts render as `[image]` / `[audio]`.
* - The modern multi-file ZIP export (streamed up whole, extracted server-
* side by `chatgptZipImport.js`). That path passes an `assetResolver` so
* image/audio/file parts render as inline markdown (``,
* `[🔊 audio](url)`, `[📎 file](url)`) pointing at the extracted assets.
*
* `parseExport` / `summarizeConversation` / `extractMessages` all accept an
* optional `{ assetResolver }`; when omitted they behave exactly as before.
*/
import { join } from 'path';
import { stripChatgptCitations } from '../lib/chatgptText.js';
import { unlink } from 'fs/promises';
import { withBackupAssetPublication } from '../lib/backupSnapshotBoundary.js';
import { atomicWrite, ensureDir, PATHS, tryReadFile, safeJSONParse, safeDate } from '../lib/fileUtils.js';
import { createMemoryEntry, updateMemoryEntry, query as queryBrain } from './brainStorage.js';
const MAX_MEMORY_CONTENT = 9800;
const MAX_TITLE_LEN = 200;
const MAX_TAG_LEN = 50;
const ROLE_LABEL = { user: 'You', assistant: 'ChatGPT', system: 'System', tool: 'Tool' };
// Resolve lazily — computing it at module load reads PATHS.brain, which is
// undefined in suites that mock fileUtils with a partial PATHS (e.g.
// brain.test.js) and would crash the import. Each call is a cheap path join.
const importRoot = () => join(PATHS.brain, 'imports', 'chatgpt');
const sanitizeTag = (s) => String(s || '')
.toLowerCase()
.replace(/[^a-z0-9-_]+/g, '-')
.replace(/^-+|-+$/g, '')
.slice(0, MAX_TAG_LEN);
const cleanTitle = (s, fallback = 'Untitled conversation') => {
const trimmed = String(s || '').trim();
if (!trimmed) return fallback;
return trimmed.length > MAX_TITLE_LEN ? `${trimmed.slice(0, MAX_TITLE_LEN - 1)}…` : trimmed;
};
/**
* Normalize a ChatGPT asset pointer to its bare file id. Pointers appear as
* `file-service://file-XXX`, `sediment://file_HASH`, or (in `attachments`) a
* bare `file-XXX` id. The matching on-disk asset is `<id>.dat`.
*/
export const assetPointerId = (pointer) => String(pointer || '')
.replace(/^file-service:\/\//, '')
.replace(/^sediment:\/\//, '')
.trim();
const escapeMd = (s) => String(s || '').replace(/([\\`*_[\]()])/g, '\\$1');
/**
* Render one message part to markdown text. `assetResolver(pointer)` returns
* `{ url, name, mime } | null` for an extracted asset (ZIP path), or is null
* (legacy JSON path) — in which case asset parts degrade to `[image]` etc.
* `renderedIds` collects the asset ids we inlined so `extractMessages` can
* avoid double-listing them in the message's attachment footer.
*/
const partToText = (part, assetResolver, renderedIds) => {
if (typeof part === 'string') return part;
if (!part || typeof part !== 'object') return '';
const ct = part.content_type;
// Spoken-turn transcript text — render the words, not a placeholder.
if (ct === 'audio_transcription' && typeof part.text === 'string') return part.text;
if (ct === 'image_asset_pointer') {
const resolved = assetResolver?.(part.asset_pointer);
if (resolved?.url) {
if (renderedIds) renderedIds.add(assetPointerId(part.asset_pointer));
return `\n\n\n`;
}
return '[image]';
}
if (ct === 'audio_asset_pointer') {
const resolved = assetResolver?.(part.asset_pointer);
if (resolved?.url) {
if (renderedIds) renderedIds.add(assetPointerId(part.asset_pointer));
return `[🔊 ${escapeMd(resolved.name || 'audio')}](${resolved.url})`;
}
return '[audio]';
}
if (ct === 'real_time_user_audio_video_asset_pointer') {
const resolved = assetResolver?.(part.asset_pointer);
if (resolved?.url) {
if (renderedIds) renderedIds.add(assetPointerId(part.asset_pointer));
return `[🎙️ ${escapeMd(resolved.name || 'voice message')}](${resolved.url})`;
}
return '[voice message]';
}
if (typeof part.text === 'string') return part.text;
if (ct) return `[${ct}]`;
return '';
};
/**
* Reduce a single message's `content.parts` array into a plain text / markdown
* string. Returns `{ text, renderedIds }` so the caller can dedupe attachments.
*/
const renderParts = (parts, assetResolver) => {
const renderedIds = new Set();
if (!Array.isArray(parts)) return { text: '', renderedIds };
const text = parts
.map((part) => partToText(part, assetResolver, renderedIds))
.filter(Boolean)
.join('\n');
return { text, renderedIds };
};
// Kept for back-compat with the existing unit tests (`__test.partsToText`).
const partsToText = (parts) => renderParts(parts, null).text;
/**
* Render a message's `metadata.attachments` (files the user attached) as a
* markdown footer of links — skipping any already inlined as image parts.
* Non-image attachments (PDFs, docs, spreadsheets) only appear here, never as
* parts, so this is the only place they surface.
*/
const renderAttachments = (msg, assetResolver, renderedIds) => {
const attachments = msg?.metadata?.attachments;
if (!Array.isArray(attachments) || attachments.length === 0) return '';
const links = [];
for (const att of attachments) {
const id = att?.id;
if (!id || renderedIds.has(id)) continue;
const resolved = assetResolver?.(id);
if (resolved?.url) {
const icon = (resolved.mime || att.mime_type || '').startsWith('image/') ? '🖼️' : '📎';
links.push(`[${icon} ${escapeMd(resolved.name || att.name || 'file')}](${resolved.url})`);
}
}
return links.length ? `\n\n${links.join('\n')}` : '';
};
/**
* Walk the `mapping` tree of a conversation from `current_node` back to the
* root, then reverse — that path is the visible conversation thread (ChatGPT
* mappings can include alternate branches from edits/regenerations).
*/
export function extractMessages(conversation, { assetResolver = null } = {}) {
if (!conversation || typeof conversation !== 'object') return [];
const mapping = conversation.mapping || {};
const seen = new Set();
const path = [];
let nodeId = conversation.current_node;
while (nodeId && mapping[nodeId] && !seen.has(nodeId)) {
seen.add(nodeId);
path.push(mapping[nodeId]);
nodeId = mapping[nodeId].parent;
}
path.reverse();
const messages = [];
for (const node of path) {
const msg = node.message;
if (!msg) continue;
const role = msg.author?.role;
if (!role || role === 'system') continue;
const { text: partsText, renderedIds } = renderParts(msg.content?.parts, assetResolver);
const attachmentText = renderAttachments(msg, assetResolver, renderedIds);
const text = stripChatgptCitations(`${partsText}${attachmentText}`);
if (!text.trim()) continue;
messages.push({
id: msg.id || node.id,
role,
text,
createTime: typeof msg.create_time === 'number' ? msg.create_time : null
});
}
return messages;
}
/**
* Render an array of {role,text} messages as a markdown-ish transcript.
*/
export function formatTranscript(messages) {
return messages
.map((m) => `**${ROLE_LABEL[m.role] || m.role}**:\n${m.text}`)
.join('\n\n---\n\n');
}
const epochToISO = (epoch) => {
if (typeof epoch !== 'number' || !isFinite(epoch) || epoch <= 0) return null;
return new Date(epoch * 1000).toISOString();
};
/**
* Build a lightweight summary record for a parsed conversation.
*/
export function summarizeConversation(conversation, { assetResolver = null } = {}) {
const messages = extractMessages(conversation, { assetResolver });
const userMessages = messages.filter((m) => m.role === 'user').length;
const assistantMessages = messages.filter((m) => m.role === 'assistant').length;
const transcript = formatTranscript(messages);
// Count inlined assets (markdown image embeds + asset link icons) for the
// preview/summary so the wizard can report "N conversations, M assets".
const assetCount = (transcript.match(/!\[[^\]]*\]\([^)]+\)|\[(?:🔊|🎙️|🖼️|📎)[^\]]*\]\([^)]+\)/g) || []).length;
return {
id: conversation.id || conversation.conversation_id || null,
title: cleanTitle(conversation.title),
createTime: epochToISO(conversation.create_time),
updateTime: epochToISO(conversation.update_time),
messageCount: messages.length,
userMessages,
assistantMessages,
charCount: transcript.length,
assetCount,
gizmoId: conversation.gizmo_id || null,
messages,
transcript
};
}
/**
* Parse a raw conversations payload into analysis + per-conversation summaries.
* Accepts: a top-level array of conversations, an object with a `conversations`
* array (legacy single-file `conversations.json`), OR an object with a
* `conversationFiles` array of already-parsed JSON payloads (the multi-file ZIP
* path, where each `conversations-NNN.json` is its own array/object).
*/
export function parseExport(raw, { assetResolver = null } = {}) {
let conversations;
if (Array.isArray(raw)) {
conversations = raw;
} else if (raw && Array.isArray(raw.conversations)) {
conversations = raw.conversations;
} else if (raw && Array.isArray(raw.conversationFiles)) {
// Multi-file export: flatten every shard into one conversation list.
conversations = raw.conversationFiles.flatMap((shard) =>
Array.isArray(shard) ? shard : (Array.isArray(shard?.conversations) ? shard.conversations : []));
} else {
return { ok: false, error: 'Expected an array of conversations or an object with a "conversations" array.' };
}
if (conversations.length === 0) {
return { ok: false, error: 'No conversations found in the upload.' };
}
const summaries = [];
let totalMessages = 0;
let totalChars = 0;
let totalAssets = 0;
let earliest = null;
let latest = null;
const gizmos = new Set();
for (const c of conversations) {
const s = summarizeConversation(c, { assetResolver });
summaries.push(s);
totalMessages += s.messageCount;
totalChars += s.charCount;
totalAssets += s.assetCount;
if (s.gizmoId) gizmos.add(s.gizmoId);
if (s.createTime && (!earliest || s.createTime < earliest)) earliest = s.createTime;
if (s.updateTime && (!latest || s.updateTime > latest)) latest = s.updateTime;
}
return {
ok: true,
summary: {
totalConversations: summaries.length,
totalMessages,
totalChars,
totalAssets,
earliest,
latest,
gizmoCount: gizmos.size
},
conversations: summaries
};
}
/**
* Strip the heavy transcript/messages fields so the preview payload sent to
* the client stays small for large exports.
*/
export function stripPreview(parsed) {
if (!parsed?.ok) return parsed;
return {
ok: true,
summary: parsed.summary,
conversations: parsed.conversations.map(({ messages, transcript, ...rest }) => rest)
};
}
const safeFilename = (s) => String(s || 'conversation')
.replace(/[^a-zA-Z0-9-_]+/g, '-')
.replace(/^-+|-+$/g, '')
.slice(0, 80) || 'conversation';
/**
* Read one archived conversation transcript by its archive filename (the
* `sourceRef` stored on a chatgpt-import Memory entry). Returns null if the
* name escapes the archive dir or the file is missing — the conversation
* viewer falls back to the truncated Memory `content` in that case.
*/
export async function readArchivedConversation(archiveName) {
const name = String(archiveName || '');
// Reject path traversal — archive names are flat `<safe-id>.json`.
if (!/^[a-zA-Z0-9-_]+\.json$/.test(name)) return null;
const raw = await tryReadFile(join(importRoot(), name));
if (!raw) return null;
return safeJSONParse(raw, null, { allowArray: false });
}
// Served-asset URL prefix every imported image/audio/file link points at (see
// chatgptZipImport.js — assets land at `/data/brain-imports/<assetId><ext>`).
const ASSET_URL_PREFIX = '/data/brain-imports/';
/**
* Extract the bare asset filenames an imported memory's markdown `content`
* references (`/data/brain-imports/<file>`). Returns a Set of filenames (no
* directory, no query/hash). The leading-bytes restriction in the asset
* filename charset means a simple bounded match is safe against path traversal —
* we additionally reject any name that isn't a flat basename below.
*/
export const extractAssetFileNames = (content) => {
const names = new Set();
if (typeof content !== 'string') return names;
// Match the served prefix followed by a flat filename (no slash) up to the
// markdown link close `)`, whitespace, or end — then strip any trailing
// punctuation a transcript might place right after the URL.
const re = new RegExp(`${ASSET_URL_PREFIX.replace(/\//g, '\\/')}([^)\\s]+)`, 'g');
let m;
while ((m = re.exec(content)) !== null) {
const name = m[1].replace(/[).,]+$/, '');
// Served asset names are `<assetId><ext>` (see chatgptZipImport.js) and are
// flat in practice. These guards are the actual safety boundary regardless
// of how the producer names files: the charset anchor rejects `/`, `\`,
// null bytes, and stray query-string/punctuation tails, and the explicit
// `..` reject keeps it a strict superset of the prior path-traversal guard
// (the charset alone would admit a literal `..`, which `join` resolves to
// the parent dir). Together a crafted link can't produce a name we'd hand
// to unlink() that escapes — or names — the assets dir.
if (/^[a-zA-Z0-9._-]+$/.test(name) && !name.includes('..')) {
names.add(name);
}
}
return names;
};
/**
* Every asset filename an import memory references — across BOTH its (possibly
* truncated) `content` AND its full archived transcript. The memory `content`
* is capped at MAX_MEMORY_CONTENT, so an asset linked only past the truncation
* point is invisible in `content` but present in the archive; unioning the two
* catches it. Reads the archive only when the memory carries a `sourceRef`.
*/
const assetNamesForMemory = async (record) => {
const names = extractAssetFileNames(record?.content);
if (record?.sourceRef) {
const archive = await readArchivedConversation(record.sourceRef);
if (typeof archive?.transcript === 'string') {
for (const name of extractAssetFileNames(archive.transcript)) names.add(name);
}
}
return names;
};
/**
* Delete the on-disk assets + archived transcript that belonged to a deleted
* `chatgpt-import` memory, leaving no orphans under `data/brain/imports/`.
*
* `survivors` is every OTHER live import memory. Neither the archive nor an
* asset is unlinked while a survivor still references it:
* - `importConversations` now upserts by conversation id, so re-importing
* the same export can no longer CREATE a second memory sharing an
* archive. Installs that imported the same export twice before that fix
* shipped can still hold such a pair, so the survivor check stays: keep
* the archive while any survivor shares its `sourceRef`.
* - An asset id can likewise recur across conversations (ChatGPT reuses a
* `file-service://` id), so an asset is only unlinked when NO survivor
* still references it — compared via the FULL archived transcript, not the
* truncated `content`, so an asset linked only in a survivor's truncated
* tail isn't wrongly pulled out from under it.
*
* Best-effort: a missing file is a no-op (already gone), and any unlink failure
* is logged but never thrown — cleanup must not turn a successful delete into a
* request error.
*/
export async function deleteMemoryAssets(record, survivors = []) {
if (!record || record.source !== 'chatgpt-import') return;
// Resolve the deleted record's full asset set BEFORE unlinking its transcript
// below (the transcript is one of the sources we read asset links from).
const referenced = await assetNamesForMemory(record);
// 1. Archived transcript — remove only when no surviving import shares it (a
// re-import of the same conversation reuses the same `<id>.json` archive).
const ref = String(record.sourceRef || '');
const refStillUsed = survivors.some((m) => m?.sourceRef === ref);
if (/^[a-zA-Z0-9-_]+\.json$/.test(ref) && !refStillUsed) {
await unlink(join(importRoot(), ref)).catch((err) => {
if (err?.code !== 'ENOENT') console.error(`⚠️ Failed to remove archived transcript ${ref}: ${err.message}`);
});
}
// 2. Served assets — only those no surviving import memory still references
// (in its content OR its full archive). Short-circuit when this memory had no
// assets so an asset-free conversation never reads a single survivor archive.
if (referenced.size === 0) return;
const stillUsed = new Set();
for (const m of survivors) {
for (const name of await assetNamesForMemory(m)) stillUsed.add(name);
}
const orphaned = [...referenced].filter((name) => !stillUsed.has(name));
await Promise.all(orphaned.map((name) =>
unlink(join(PATHS.brainImportAssets, name)).catch((err) => {
if (err?.code !== 'ENOENT') console.error(`⚠️ Failed to remove import asset ${name}: ${err.message}`);
})
));
if (orphaned.length) console.log(`🧠 Removed ${orphaned.length} orphaned import asset(s) for memory ${record.id}`);
}
/**
* The archive filename (and therefore the `sourceRef` a Memory entry carries)
* for one conversation summary. Deterministic when `summary.id` is present —
* that's the ChatGPT-assigned conversation id, so re-importing the same
* export (or a newer export that overlaps it) resolves to the same name and
* `importConversations` can upsert instead of duplicating. A summary with no
* id (malformed export row) falls back to a timestamp, which is NOT stable
* across calls — such a conversation always imports as new.
*/
const conversationArchiveName = (summary) => `${safeFilename(summary.id || `conv-${Date.now()}`)}.json`;
/**
* Persist one conversation's full transcript + structured messages to the
* import archive directory.
*/
async function archiveConversation(summary, fname = conversationArchiveName(summary)) {
await ensureDir(importRoot());
const filePath = join(importRoot(), fname);
const payload = {
id: summary.id || `conv-${Date.now()}`,
title: summary.title,
createTime: summary.createTime,
updateTime: summary.updateTime,
messageCount: summary.messageCount,
assetCount: summary.assetCount,
gizmoId: summary.gizmoId,
messages: summary.messages,
transcript: summary.transcript,
importedAt: new Date().toISOString()
};
await atomicWrite(filePath, payload);
return fname;
}
const buildContent = (summary) => {
const header = [
`Source: ChatGPT export`,
summary.createTime ? `Started: ${summary.createTime}` : null,
summary.updateTime ? `Updated: ${summary.updateTime}` : null,
`Messages: ${summary.messageCount}`,
summary.assetCount ? `Assets: ${summary.assetCount}` : null
].filter(Boolean).join('\n');
const body = summary.transcript;
const combined = `${header}\n\n${body}`;
if (combined.length <= MAX_MEMORY_CONTENT) return combined;
const truncated = combined.slice(0, MAX_MEMORY_CONTENT - 80).trimEnd();
return `${truncated}\n\n…(transcript truncated — open the full conversation to see everything)`;
};
/**
* Create a Brain Memory entry for each summarised conversation. Conversations
* with zero messages are skipped (they appear in ChatGPT exports as empty
* shells when the user starts a chat but never sends a message).
*
* Returns counts and per-conversation result records so the wizard can show
* which entries were skipped/imported.
*/
export async function importConversations(parsed, options = {}) {
if (!parsed?.ok) return { ok: false, error: parsed?.error || 'Invalid parsed payload' };
const baseTags = (options.tags || ['chatgpt-import'])
.map(sanitizeTag)
.filter(Boolean);
const skipEmpty = options.skipEmpty !== false;
await ensureDir(importRoot());
// Upsert by conversation id (sourceRef == archive filename), keyed BEFORE the
// loop so a re-import of the same export — or a newer export overlapping
// it — updates the existing memory instead of creating a duplicate. See
// `conversationArchiveName` for why a summary with no id can't be matched.
const existingBySourceRef = new Map();
for (const memory of await queryBrain('memories', { source: 'chatgpt-import' })) {
if (memory.sourceRef) existingBySourceRef.set(memory.sourceRef, memory);
}
const results = [];
let imported = 0;
let updated = 0;
let unchanged = 0;
let skipped = 0;
let archived = 0;
for (const summary of parsed.conversations) {
if (skipEmpty && summary.messageCount === 0) {
results.push({ id: summary.id, title: summary.title, status: 'skipped', reason: 'empty' });
skipped += 1;
continue;
}
const archiveName = conversationArchiveName(summary);
const existingMemory = existingBySourceRef.get(archiveName);
const isNewer = existingMemory
&& safeDate(summary.updateTime) > safeDate(existingMemory.sourceUpdatedAt);
if (existingMemory && !isNewer) {
results.push({
id: summary.id,
memoryId: existingMemory.id,
title: summary.title,
messageCount: summary.messageCount,
assetCount: summary.assetCount,
archiveName,
status: 'unchanged'
});
unchanged += 1;
continue;
}
archived += 1;
const tags = [
...baseTags,
summary.gizmoId ? sanitizeTag(`gizmo-${summary.gizmoId}`) : null
].filter(Boolean);
const memoryData = {
title: summary.title,
content: buildContent(summary),
tags,
source: 'chatgpt-import',
sourceRef: archiveName,
// Both source clocks are persisted so the memory list can order imports by
// the original conversation recency (updateTime preferred) rather than the
// shared bulk-import timestamp. See `memoryRecencyMs` in brainStorage.js.
sourceCreatedAt: summary.createTime || null,
sourceUpdatedAt: summary.updateTime || null
};
// The archived transcript and the memory row that names it are one backup
// workflow; the assets were extracted earlier and are named only by this row.
const commit = (write) => withBackupAssetPublication(async () => {
await archiveConversation(summary, archiveName);
return write();
});
if (existingMemory) {
const entry = await commit(() => updateMemoryEntry(existingMemory.id, memoryData));
results.push({
id: summary.id,
memoryId: entry.id,
title: summary.title,
messageCount: summary.messageCount,
assetCount: summary.assetCount,
archiveName,
status: 'updated'
});
updated += 1;
continue;
}
const entry = await commit(() => createMemoryEntry(memoryData));
results.push({
id: summary.id,
memoryId: entry.id,
title: summary.title,
messageCount: summary.messageCount,
assetCount: summary.assetCount,
archiveName,
status: 'imported'
});
imported += 1;
}
return {
ok: true,
imported,
updated,
unchanged,
skipped,
archived,
archiveDir: importRoot(),
results
};
}
export const __test = { sanitizeTag, cleanTitle, partsToText, buildContent, assetPointerId, extractAssetFileNames };