Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

23.4 KB
Raw Blame
/**
 * ChatGPT ZIP Import Service
 *
 * Ingests the modern multi-file ChatGPT data export (a `.zip`, streamed up
 * whole — no in-memory cap) end to end:
 *
 *   1. Stream-extract the ZIP (`parseZip`) — never decompress the whole archive
 *      into one buffer; members are handled one at a time as they arrive.
 *      - `conversations*.json` + `conversation_asset_file_names.json` are
 *        buffered in memory (small relative to the assets).
 *      - `*.dat` asset files (images / voice audio / PDFs) are **streamed
 *        straight to a temp file** in the served assets dir
 *        (`PATHS.brainImportAssets`) as they arrive — only the leading bytes are
 *        held to sniff the real extension, so peak RAM stays bounded to one chunk
 *        no matter how large the export is. After the stream closes (the
 *        asset-name map may arrive after the `.dat` members) each temp file is
 *        renamed to its final `<assetId><ext>` served name, keyed by the global
 *        asset id and falling back to the friendly name in the asset-name map
 *        for the extension when magic-byte sniffing comes up empty. `chat.html`
 *        and other bulky members are drained without buffering.
 *   2. Build a `pointer -> { url, name, mime }` resolver so transcript rendering
 *      can inline `![](url)` images and `[🔊 audio](url)` / `[📎 file](url)`
 *      links pointing at the extracted assets.
 *   3. Parse + import via the shared `chatgptImport` service, passing the
 *      resolver so every conversation's transcript embeds its assets.
 *
 * Both the legacy single-`conversations.json` JSON path (browser-parsed) and
 * this ZIP path converge on the same `parseExport`/`importConversations` — the
 * only difference is the `assetResolver`.
 */

import { createReadStream } from 'fs';
import { rename, lstat } from 'fs/promises';
import { join } from 'path';
import { Writable } from 'stream';
import { PATHS, getMimeType, ensureDir, createWriteStreamGuarded, unlinkGuarded } from '../lib/fileUtils.js';
import { isTopLevelEntryName } from '../lib/pathSafety.js';
import { parseZip, collectZipEntry, createZipBudget, MAX_ZIP_MEMBER_BYTES } from '../lib/zipStream.js';
import { parseExport, importConversations, assetPointerId } from './chatgptImport.js';
import { ServerError } from '../lib/errorHandler.js';

// Cap an individual member (JSON shard buffered in memory, or `.dat` asset
// streamed to disk). The largest conversation shard in a real export is ~6 MB
// and assets run KB–few MB; 100 MB is a generous per-member backstop against a
// malicious zip claiming one member is enormous. Assets stream straight to disk
// (peak RAM is one chunk), so there is no aggregate in-memory ceiling to keep —
// a legitimately large multi-GB export imports without an arbitrary budget.
const MAX_MEMBER_BYTES = MAX_ZIP_MEMBER_BYTES;

// Leading bytes captured from each streamed asset to sniff its magic-byte type
// (the widest sniff — WAV/WebP — reads bytes 8–11). Everything past this streams
// straight to the temp file without being held.
const SNIFF_BYTES = 12;

// Magic-byte sniffers — `.dat` files carry no extension, so we detect the real
// type from the leading bytes and give the served file a correct extension.
// Order matters (WebP's RIFF prefix is also used by WAV, so check the form id).
const sniffExtension = (buf) => {
  if (!buf || buf.length < 12) return null;
  if (buf[0] === 0x89 && buf[1] === 0x50 && buf[2] === 0x4e && buf[3] === 0x47) return '.png';
  if (buf[0] === 0xff && buf[1] === 0xd8 && buf[2] === 0xff) return '.jpg';
  if (buf[0] === 0x47 && buf[1] === 0x49 && buf[2] === 0x46) return '.gif';
  if (buf[0] === 0x25 && buf[1] === 0x50 && buf[2] === 0x44 && buf[3] === 0x46) return '.pdf';
  if (buf.slice(0, 4).toString('ascii') === 'RIFF') {
    const form = buf.slice(8, 12).toString('ascii');
    if (form === 'WEBP') return '.webp';
    if (form === 'WAVE') return '.wav';
    return null;
  }
  // ID3-tagged or raw MPEG audio.
  if (buf.slice(0, 3).toString('ascii') === 'ID3') return '.mp3';
  if (buf[0] === 0xff && (buf[1] & 0xe0) === 0xe0) return '.mp3';
  return null;
};

// Extensions we'll trust from the export-controlled friendly name when magic-
// byte sniffing fails. These are inert document/media types the static mount
// serves harmlessly. Active-content types (.html/.svg/.xml/.js/...) are
// DELIBERATELY excluded: an attachment named `x.html` would otherwise be
// written under /data/brain-imports/ and served inline with an executable
// content type, and the viewer links to those same-origin assets — clicking
// one would run script in PortOS's origin. Anything not on this list falls
// back to `.bin` (served as application/octet-stream — download, never inline).
const INERT_FALLBACK_EXTS = new Set([
  '.png', '.jpg', '.jpeg', '.gif', '.webp', '.bmp', '.ico',
  '.pdf', '.wav', '.mp3', '.m4a', '.ogg', '.mp4', '.webm', '.mov',
  '.txt', '.md', '.csv', '.json',
]);

// Friendly-name → extension fallback when magic-byte sniffing comes up empty.
// Returns null for anything not on the inert allowlist (caller uses `.bin`),
// so an export can never steer the served extension to an active-content type.
const extFromName = (name) => {
  const m = String(name || '').match(/\.([a-z0-9]{1,5})$/i);
  if (!m) return null;
  const ext = `.${m[1].toLowerCase()}`;
  return INERT_FALLBACK_EXTS.has(ext) ? ext : null;
};

const IS_DAT = (path) => path.endsWith('.dat');
const IS_CONVO_JSON = (path) => /(?:^|\/)conversations(?:-\d+)?\.json$/i.test(path);
const IS_ASSET_NAME_MAP = (path) => /(?:^|\/)conversation_asset_file_names\.json$/i.test(path);

// A `.dat` member's asset id is its basename without extension. Conversations
// reference assets as `file-service://file-XXX` / `sediment://file_HASH`, both
// of which `assetPointerId()` reduces to that same bare id.
//
// Splits on BOTH separators so the id is always a real basename on every
// platform: a hostile archive can name a member `a\..\..\evil.dat`, and a
// forward-slash-only strip would leave the backslashes in the id — on Windows
// `${assetDir}/${id}` would then escape the asset dir (Zip Slip).
const datAssetId = (path) => path.split(/[\\/]/).pop().replace(/\.dat$/i, '');

// Windows resolves these basenames to devices whatever the extension, so a
// member named `CON.dat` would open the console instead of `CON.part` and the
// later rename would fail — aborting the whole import. Real exports only ever
// use `file-…` ids, so they are refused on every platform for one consistent
// outcome rather than a Windows-only failure.
const WINDOWS_RESERVED_NAME = /^(?:con|prn|aux|nul|com[1-9]|lpt[1-9])$/i;

// A `.dat` member is extractable only when BOTH its ZIP-spec name and the asset
// id it reduces to are plain filenames.
//
// The ZIP spec mandates `/` as the only separator, so everything after the last
// `/` is one filename — a `\` or `..` surviving there means a hostile or
// malformed member, not a nested directory, and it is skipped rather than
// silently normalized into an extracted file. The second half catches ids that
// degenerate once `.dat` is dropped (`.dat` -> ``, `..dat` -> `.`), which would
// otherwise write a `.png` / `..png` dotfile into the served dir.
//
// Today `zipStream.js` sanitizes separators and `.`/`..` components as it decodes
// each header, so a `\` never actually reaches this predicate — the flattened
// basename is what arrives. That makes this guard the SECOND layer, not the
// first, and it is deliberately kept anyway: the parser's sanitizing is its own
// private policy (its header comment describes it as a faithful sequential
// reader), while this is the code that actually opens the file. Validating at
// the write site means a future refactor that makes the parser truly faithful
// cannot silently reintroduce the escape.
const isSafeDatMember = (path) => {
  const assetId = datAssetId(path);
  return isTopLevelEntryName(path.replace(/^.*\//, ''))
    && isTopLevelEntryName(assetId)
    && !WINDOWS_RESERVED_NAME.test(assetId);
};

// Single-sourced so both guards below log identically. Deliberately carries NO
// member path — the name is untrusted third-party text.
const UNSAFE_MEMBER_WARNING = '⚠️ ChatGPT import: skipped a ZIP member with an unsafe asset name';

// Stream a `.dat` entry straight to `filePath`, holding only the leading
// `SNIFF_BYTES` so the asset's real extension can be sniffed without buffering
// the whole file (peak RAM is one chunk). Enforces the per-member size cap.
// Resolves with the sniffed extension (or null when the magic bytes aren't
// recognized — the caller then falls back to the friendly name / `.bin`).
const streamAssetToFile = (entry, filePath, max, budget) => new Promise((resolve, reject) => {
  const outPromise = createWriteStreamGuarded(filePath);
  const head = [];
  let headLen = 0;
  let size = 0;
  let done = false;
  // Reject only after the write stream has CLOSED. The caller unlinks the
  // `.part` once this task settles; rejecting first lets the async open land
  // after that unlink (re-creating the file) or leaves the fd open so Windows
  // still lists the file (#12701).
  const fail = (err) => {
    if (done) return;
    done = true;
    outPromise
      .then((stream) => {
        if (stream.closed) return undefined;
        return new Promise((closed) => { stream.once('close', closed); stream.destroy(); });
      })
      .catch(() => {})
      .then(() => reject(err));
  };
  outPromise.then((stream) => {
    if (done) return;
    stream.on('error', fail);
    stream.on('finish', () => { if (done) return; done = true; resolve(sniffExtension(Buffer.concat(head))); });
  }).catch(fail);
  const sink = new Writable({
    async write(chunk, _enc, cb) {
      size += chunk.length;
      if (size > max) { cb(new ServerError(`ZIP member exceeds ${max} byte limit`, { status: 413 })); return; }
      try { budget.addBytes(chunk.length); } catch (err) { cb(err); return; }
      if (headLen < SNIFF_BYTES) {
        // Keep only the bytes still needed to reach SNIFF_BYTES, so `head` holds
        // at most SNIFF_BYTES regardless of how the producer chunks the asset.
        const slice = chunk.subarray(0, SNIFF_BYTES - headLen);
        head.push(slice);
        headLen += slice.length;
      }
      try {
        const stream = await outPromise;
        if (stream.write(chunk)) cb();
        else stream.once('drain', cb);
      } catch (err) {
        cb(err);
      }
    },
    async final(cb) {
      try {
        const stream = await outPromise;
        stream.end(cb);
      } catch (err) {
        cb(err);
      }
    },
  });
  sink.on('error', fail);
  entry.pipe(sink);
});

/**
 * Stream-extract a ChatGPT export ZIP at `zipPath`.
 *
 * Returns:
 *   {
 *     conversationFiles: Array<parsed JSON>,   // each conversations*.json
 *     assets: Map<assetId, { url, name, mime, file }>,
 *     assetNameMap: Record<datFilename, friendlyName>,
 *     stats: { assetCount, conversationFileCount }
 *   }
 *
 * Assets are written to `data/brain/imports/assets/<assetId><ext>` and surfaced
 * at `/data/brain-imports/<assetId><ext>`.
 */
export async function extractChatgptZip(zipPath, { assetDir = PATHS.brainImportAssets, maxTotalBytes, maxEntries, maxAssetBytes = 20 * 1024 ** 3 } = {}) {
  await ensureDir(assetDir);

  const assetBudget = createZipBudget({ maxTotalBytes: maxAssetBytes });
  const jsonBudget = createZipBudget({ maxTotalBytes, maxEntries });
  const archiveBudget = createZipBudget({ maxEntries });
  const convoBuffers = [];          // { path, buffer }
  let assetNameMap = {};
  // assetId -> { datPath, tempPath, sniffedExt } recorded as each `.dat` streams
  // to a temp file; we rename to the final `<assetId><ext>` name after the
  // stream closes, because the name map may arrive after the .dat members
  // (member order in the ZIP isn't guaranteed). `tempPaths` tracks every temp
  // file written so a mid-stream failure can clean partial writes off disk.
  const pendingAssets = new Map();
  const tempPaths = new Set();
  // Every per-member task (asset writes, JSON collects). Hoisted out of the
  // Promise executor so the rejection handler below can wait for in-flight asset
  // writes to fully settle (their write streams close) before unlinking temp
  // files — otherwise a write that finishes AFTER the unlink re-creates the
  // `.part` file and orphans it (a filesystem-timing race under heavy I/O).
  const inFlight = [];

  await new Promise((resolve, reject) => {
    let settled = false;
    const src = createReadStream(zipPath);
    const parser = parseZip();
    // On any error/settle, tear down both streams. Rejecting the Promise alone
    // doesn't stop ingestion — the read stream keeps flowing into the parser and
    // entries keep streaming to disk, so a per-member overflow would otherwise
    // keep writing *after* the failure already fired. Temp files left behind are
    // cleaned by `tempPaths` below.
    const settle = (fn) => (arg) => {
      if (settled) return;
      settled = true;
      if (fn === reject) { src.destroy(); parser.destroy?.(); }
      fn(arg);
    };
    const onErr = settle(reject);

    src
      .on('error', onErr)
      .pipe(parser)
      .on('entry', (entry) => {
        // An entry the parser emits after the failure would start a task the
        // cleanup's `allSettled(inFlight)` snapshot never waits for.
        if (settled) { entry.autodrain(); return; }
        const { path } = entry;
        if (IS_ASSET_NAME_MAP(path) || IS_CONVO_JSON(path) || (IS_DAT(path) && isSafeDatMember(path))) {
          try { archiveBudget.addEntry(); } catch (err) { onErr(err); return; }
        }
        if (IS_ASSET_NAME_MAP(path)) {
          inFlight.push(collectZipEntry(entry, { maxBytes: MAX_MEMBER_BYTES, budget: jsonBudget })
            .then((buf) => { assetNameMap = JSON.parse(buf.toString('utf8')); })
            .catch(onErr));
        } else if (IS_CONVO_JSON(path)) {
          inFlight.push(collectZipEntry(entry, { maxBytes: MAX_MEMBER_BYTES, budget: jsonBudget })
            .then((buf) => { convoBuffers.push({ path, buffer: buf }); })
            .catch(onErr));
        } else if (IS_DAT(path)) {
          // Stream the asset straight to a temp file, holding only its leading
          // bytes to sniff the extension. Peak RAM is one chunk regardless of
          // asset size. A separate aggregate disk ceiling permits multi-GB
          // exports while bounding expansion. Renamed to its served name below.
          // Refuse any member whose name isn't a plain filename BEFORE the temp
          // file is opened. Skip the member rather than failing the whole
          // import: one malformed member in a multi-GB export shouldn't cost the
          // user the entire ingest, and a conversation referencing it simply
          // renders without its asset (the resolver already returns null).
          if (!isSafeDatMember(path)) {
            console.warn(UNSAFE_MEMBER_WARNING);
            entry.autodrain();
            return;
          }
          const assetId = datAssetId(path);
          const tempPath = join(assetDir, `${assetId}.part`);
          tempPaths.add(tempPath);
          inFlight.push(streamAssetToFile(entry, tempPath, MAX_MEMBER_BYTES, assetBudget)
            .then((sniffedExt) => { pendingAssets.set(assetId, { datPath: path, tempPath, sniffedExt }); })
            .catch(onErr));
        } else {
          // chat.html, export_manifest.json, library_files.json, user.json, etc.
          // — not needed for the transcript+asset import. Drain without buffering.
          entry.autodrain();
        }
      })
      .on('close', () => { Promise.all(inFlight).then(settle(resolve), onErr); })
      .on('error', onErr);
  }).catch(async (err) => {
    // Streaming failed (size cap, parser error, …) — assets already streamed to
    // temp files would otherwise be orphaned on disk. Remove them before
    // re-throwing so a bad upload can't accumulate `.part` files.
    //
    // Wait for every in-flight member task to settle FIRST. `settle(reject)`
    // rejects this Promise immediately and destroys the source/parser, but a
    // `streamAssetToFile` whose write stream hasn't finished is still mid-flight:
    // `createWriteStream` opens its `.part` file asynchronously, so unlinking now
    // can race the open — the unlink no-ops (file not created yet), then the open
    // lands and orphans the `.part` (the intermittent full-suite failure). The
    // parser's teardown ends the in-flight entry stream (see zipStream.js
    // `destroy`), so each task settles deterministically — once `allSettled`
    // resolves, every write handle is closed and the file's on-disk existence is
    // settled, making the unlink below final rather than racing a pending open.
    await Promise.allSettled(inFlight);
    await cleanupTempFiles(tempPaths);
    throw err;
  });

  // Rename each streamed temp file to its final served name, resolving the
  // extension from the sniffed magic bytes, then the friendly-name fallback,
  // then `.bin`. On any failure, clean up both the renamed assets and any
  // still-unfinalized temp files so nothing is orphaned.
  const assets = new Map();
  try {
    for (const [assetId, { datPath, tempPath, sniffedExt }] of pendingAssets) {
      const friendlyName = assetNameMap[`${assetId}.dat`] || assetNameMap[datPath] || null;
      const ext = sniffedExt || extFromName(friendlyName) || '.bin';
      const fileName = `${assetId}${ext}`;
      // Defense in depth: `assetId` was already vetted above, but `ext` comes
      // from `extFromName`/the sniffer — re-check the composed name so a future
      // change there can't reintroduce a traversal.
      if (!isTopLevelEntryName(fileName)) {
        console.warn(UNSAFE_MEMBER_WARNING);
        // eslint-disable-next-line no-await-in-loop -- same sequential loop
        await unlinkGuarded(tempPath).catch(() => {});
        continue;
      }
      const filePath = join(assetDir, fileName);
      // Asset ids are content ids (ChatGPT reuses the same file-service id for
      // the same bytes), so a file already at this name belongs to an earlier
      // successful import — keep it rather than overwriting, and mark
      // `created: false` so a later failure in THIS run's cleanup path never
      // deletes a file THIS run didn't write.
      // eslint-disable-next-line no-await-in-loop -- same sequential loop
      const alreadyExists = await lstat(filePath).then(() => true, () => false);
      if (alreadyExists) {
        // eslint-disable-next-line no-await-in-loop -- same sequential loop
        await unlinkGuarded(tempPath).catch(() => {});
        assets.set(assetId, {
          url: `/data/brain-imports/${fileName}`,
          name: friendlyName || fileName,
          mime: getMimeType(ext),
          file: fileName,
          created: false,
        });
        continue;
      }
      // eslint-disable-next-line no-await-in-loop -- sequential rename keeps peak
      // disk/IO bounded; an export has a few hundred assets, not millions.
      await rename(tempPath, filePath);
      assets.set(assetId, {
        url: `/data/brain-imports/${fileName}`,
        name: friendlyName || fileName,
        mime: getMimeType(ext),
        file: fileName,
        created: true,
      });
    }
  } catch (err) {
    await cleanupExtractedAssets(assets, assetDir);
    await cleanupTempFiles(tempPaths);
    throw err;
  }

  const conversationFiles = [];
  for (const { path, buffer } of convoBuffers) {
    // A truncated/corrupt export (realistic for a partial download) makes
    // JSON.parse throw HERE — after the asset files were already written to the
    // served dir. Clean those orphans up before surfacing a clean 400, otherwise
    // the throw escapes extractChatgptZip and importChatgptZip never reaches its
    // cleanupExtractedAssets() path.
    try {
      conversationFiles.push(JSON.parse(buffer.toString('utf8')));
    } catch {
      await cleanupExtractedAssets(assets, assetDir);
      throw new ServerError(`${path} is not valid JSON — the ChatGPT export looks truncated or corrupt.`, {
        status: 400,
        code: 'INVALID_CHATGPT_EXPORT',
      });
    }
  }

  return {
    conversationFiles,
    assets,
    assetNameMap,
    stats: {
      assetCount: assets.size,
      conversationFileCount: conversationFiles.length,
    },
  };
}

/**
 * Build an `assetResolver(pointer)` closure over an extracted-assets Map.
 * Resolves `file-service://`, `sediment://`, and bare-id pointers to the
 * served asset descriptor, or null when the pointer's asset wasn't in the
 * export (ChatGPT omits expired/server-side-only assets — normal, not an error).
 */
export function makeAssetResolver(assets) {
  return (pointer) => {
    if (!pointer) return null;
    const id = assetPointerId(pointer);
    return assets.get(id) || null;
  };
}

// Remove the asset files THIS extraction run wrote to the served dir. Called on
// any FAILED import path: extraction writes assets to `/data/brain-imports/`
// before conversations are validated, so a ZIP with assets but no valid
// conversation shards (corrupt/wrong upload) would otherwise leave orphaned
// served files behind, and repeated bad uploads could fill the disk.
//
// Only unlinks entries with `created: true`. An asset id can recur across
// separate imports (ChatGPT reuses a `file-service://` id for the same
// bytes) — when the rename step above found the file already on disk, it
// kept the earlier import's copy and marked `created: false`; deleting it
// here on a LATER failure in this run would strand that earlier import's
// memory with a broken image/audio link, which is exactly the data loss this
// guards against. `created` defaults true for callers/fixtures built before
// this distinction existed.
async function cleanupExtractedAssets(assets, assetDir = PATHS.brainImportAssets) {
  await Promise.all(
    [...assets.values()]
      .filter((a) => a.created !== false)
      .map((a) => unlinkGuarded(`${assetDir}/${a.file}`).catch(() => {}))
  );
}

// Remove the `.part` temp files written while streaming assets to disk. Called
// when extraction fails before (or during) the rename-to-final step so a
// partial/aborted import doesn't leave orphaned temp files behind. Already-
// renamed temps no longer exist — the unlink simply no-ops on them.
async function cleanupTempFiles(tempPaths) {
  await Promise.all([...tempPaths].map((p) => unlinkGuarded(p).catch(() => {})));
}

/**
 * Full ZIP-import pipeline: extract → resolve assets → parse → import.
 * `zipPath` is a temp file (the streamed multipart upload); the caller owns
 * unlinking it. Returns the `importConversations` result augmented with asset
 * stats, plus the parsed preview summary. On any failure the extracted assets
 * are removed so a bad upload doesn't leave orphaned files under the served dir.
 */
export async function importChatgptZip(zipPath, { tags, skipEmpty } = {}) {
  const { conversationFiles, assets, stats } = await extractChatgptZip(zipPath);

  if (conversationFiles.length === 0) {
    await cleanupExtractedAssets(assets);
    return { ok: false, error: 'No conversations*.json files found in the ZIP. Is this a ChatGPT data export?' };
  }

  const assetResolver = makeAssetResolver(assets);
  const parsed = parseExport({ conversationFiles }, { assetResolver });
  if (!parsed.ok) {
    await cleanupExtractedAssets(assets);
    return parsed;
  }

  const result = await importConversations(parsed, { tags, skipEmpty });
  return {
    ...result,
    summary: parsed.summary,
    assetStats: stats,
  };
}

export const __test = { sniffExtension, extFromName, datAssetId, isSafeDatMember, makeAssetResolver, cleanupExtractedAssets };