Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

5.8 KB
Raw Blame
/**
 * Brain Import Routes
 *
 * Guided workflows for pulling content from third-party sources into the
 * digital brain. Currently supports ChatGPT data exports — both the legacy
 * single `conversations.json` (parsed in the browser, POSTed as JSON) and the
 * modern multi-file ZIP export (streamed up whole and extracted server-side,
 * which also ingests image / voice-audio / PDF assets).
 *
 * Endpoints:
 *   GET  /api/brain/import/sources            List available import sources
 *   POST /api/brain/import/chatgpt/preview    Validate + summarize a parsed JSON payload
 *   POST /api/brain/import/chatgpt            Run a JSON-payload import
 *   POST /api/brain/import/chatgpt/zip        Stream-upload + import a full export ZIP
 *   GET  /api/brain/import/chatgpt/archive/:name  Fetch one archived conversation transcript
 */

import { markdownImages } from '../lib/markdownImages.js';
import { Router } from 'express';
import { z } from 'zod';
import { unlink } from 'fs/promises';
import { asyncHandler, ServerError } from '../lib/errorHandler.js';
import { validateRequest } from '../lib/validation.js';
import { uploadSingle } from '../lib/multipart.js';
import { parseExport, stripPreview, importConversations, readArchivedConversation } from '../services/chatgptImport.js';
import { importChatgptZip } from '../services/chatgptZipImport.js';

const router = Router();

const SOURCES = [
  {
    id: 'chatgpt',
    label: 'ChatGPT',
    status: 'available',
    description: 'Import every conversation — and the images, voice clips, and files — from your ChatGPT data export.',
    fileExpected: 'the export .zip (or conversations.json)',
    helpUrl: 'https://help.openai.com/en/articles/7260999-how-do-i-export-my-chatgpt-history-and-data',
    instructions: [
      'Open chatgpt.com → Settings → Data Controls → Export data → Confirm export.',
      'OpenAI emails you a download link within a few minutes (link expires in 24h).',
      'Download the ZIP.',
      'Upload the whole .zip here — PortOS extracts your conversations and their images/voice/files for you.'
    ]
  }
];

router.get('/sources', asyncHandler(async (_req, res) => {
  res.json({ sources: SOURCES });
}));

const previewSchema = z.object({
  data: z.unknown()
});

const importSchema = z.object({
  data: z.unknown(),
  tags: z.array(z.string().min(1).max(50)).max(10).optional(),
  skipEmpty: z.boolean().optional()
});

router.post('/chatgpt/preview', asyncHandler(async (req, res) => {
  const { data } = validateRequest(previewSchema, req.body);
  const parsed = parseExport(data);
  if (!parsed.ok) {
    throw new ServerError(parsed.error, { status: 400, code: 'INVALID_CHATGPT_EXPORT' });
  }
  console.log(`📥 ChatGPT import preview: ${parsed.summary.totalConversations} conversations, ${parsed.summary.totalMessages} messages`);
  res.json(stripPreview(parsed));
}));

router.post('/chatgpt', asyncHandler(async (req, res) => {
  const { data, tags, skipEmpty } = validateRequest(importSchema, req.body);
  const parsed = parseExport(data);
  if (!parsed.ok) {
    throw new ServerError(parsed.error, { status: 400, code: 'INVALID_CHATGPT_EXPORT' });
  }
  console.log(`📥 ChatGPT import start: ${parsed.summary.totalConversations} conversations`);
  const result = await importConversations(parsed, { tags, skipEmpty });
  console.log(`✅ ChatGPT import complete: imported=${result.imported} updated=${result.updated} unchanged=${result.unchanged} skipped=${result.skipped} archived=${result.archived}`);
  res.json(result);
}));

// Stream the whole export ZIP up (no JSON-body size cap — mirrors the Apple
// Health importer). 2 GB ceiling; the file is written to a temp path by the
// streaming multipart parser and removed once the import finishes.
const uploadZip = uploadSingle('file', {
  limits: { fileSize: 2 * 1024 * 1024 * 1024 },
  fileFilter: (req, file, cb) => {
    const ok = file.mimetype === 'application/zip'
      || file.mimetype === 'application/x-zip-compressed'
      || file.originalname.toLowerCase().endsWith('.zip');
    if (ok) cb(null, true);
    else cb(new ServerError('Upload the ChatGPT export .zip file.', { status: 400, code: 'BAD_REQUEST' }));
  }
});

router.post('/chatgpt/zip', uploadZip, asyncHandler(async (req, res) => {
  const filePath = req.file?.path;
  if (!filePath) throw new ServerError('No file uploaded', { status: 400, code: 'BAD_REQUEST' });

  // `tags` arrives as a multipart text field — a comma-separated string.
  const tags = String(req.body?.tags || '')
    .split(',')
    .map((t) => t.trim())
    .filter(Boolean)
    .slice(0, 10);
  const skipEmpty = req.body?.skipEmpty !== 'false';

  console.log(`📥 ChatGPT ZIP import start: ${req.file.originalname} (${req.file.size} bytes)`);
  try {
    const result = await importChatgptZip(filePath, { tags: tags.length ? tags : undefined, skipEmpty });
    if (!result.ok) {
      throw new ServerError(result.error, { status: 400, code: 'INVALID_CHATGPT_EXPORT' });
    }
    console.log(`✅ ChatGPT ZIP import complete: imported=${result.imported} updated=${result.updated} unchanged=${result.unchanged} skipped=${result.skipped} assets=${result.assetStats?.assetCount}`);
    res.json(result);
  } finally {
    await unlink(filePath).catch(() => {});
  }
}));

// Fetch one archived conversation's full transcript + structured messages for
// the Memory conversation viewer (the Memory record itself stores a truncated
// preview; the archive holds the complete thread).
router.get('/chatgpt/archive/:name', asyncHandler(async (req, res) => {
  const archived = await readArchivedConversation(req.params.name);
  if (!archived) {
    throw new ServerError('Archived conversation not found', { status: 404, code: 'NOT_FOUND' });
  }
  res.json(req.query.preview === 'images' ? { images: markdownImages(archived.transcript) } : archived);
}));

export default router;