Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

21.1 KB
Raw Blame
import { describe, it, expect, beforeAll, afterAll } from 'vitest';
import { Readable, Writable } from 'stream';
import { deflateRawSync } from 'zlib';
import { writeFile, rm, mkdtemp } from 'fs/promises';
import { join } from 'path';
import { tmpdir } from 'os';
import { parseZip, extractZipEntryToBuffer, collectZipEntry, isZipUpload, collectZipEntries } from './zipStream.js';

const LOCAL_SIG = 0x04034b50;
const CENTRAL_SIG = 0x02014b50;

// Build a minimal local file header for one entry — 30-byte fixed prefix + name + data.
function buildEntry(name, payload, { method = 0 } = {}) {
  const nameBuf = Buffer.from(name, 'utf-8');
  const header = Buffer.alloc(30);
  header.writeUInt32LE(LOCAL_SIG, 0);
  header.writeUInt16LE(20, 4);          // version needed
  header.writeUInt16LE(0, 6);           // flags (no data descriptor)
  header.writeUInt16LE(method, 8);      // 0 = stored, 8 = deflate
  header.writeUInt16LE(0, 10);          // mod time
  header.writeUInt16LE(0, 12);          // mod date
  header.writeUInt32LE(0, 14);          // crc32 (unused by reader)
  header.writeUInt32LE(payload.length, 18); // compressed size
  header.writeUInt32LE(payload.length, 22); // uncompressed size
  header.writeUInt16LE(nameBuf.length, 26);
  header.writeUInt16LE(0, 28);          // extra length
  return Buffer.concat([header, nameBuf, payload]);
}

function buildEocd() {
  const buf = Buffer.alloc(4);
  buf.writeUInt32LE(CENTRAL_SIG, 0);
  return buf;
}

function collectEntries(zipBuf) {
  return new Promise((resolve, reject) => {
    const entryPromises = [];
    const parser = parseZip();
    parser.on('entry', (entry) => {
      entryPromises.push(new Promise((res) => {
        const chunks = [];
        const sink = new Writable({
          write(chunk, _, cb) { chunks.push(chunk); cb(); }
        });
        sink.on('finish', () => res({ path: entry.path, data: Buffer.concat(chunks) }));
        entry.pipe(sink);
      }));
    });
    parser.on('close', () => Promise.all(entryPromises).then(resolve, reject));
    parser.on('error', reject);
    Readable.from([zipBuf]).pipe(parser);
  });
}

describe('parseZip', () => {
  it.each([false, true])('bounds upstream reads with a blocked consumer (descriptor=%s)', async (descriptor) => {
    const zip = buildEntry('slow.bin', Buffer.alloc(2 * 1024 * 1024, 65));
    let archive = zip;
    if (descriptor) {
      zip.writeUInt16LE(8, 6);
      const footer = Buffer.alloc(16);
      footer.writeUInt32LE(0x08074b50);
      archive = Buffer.concat([zip, footer, buildEocd()]);
    }
    let produced = 0;
    let release;
    let blocked;
    const firstWrite = new Promise(resolve => { blocked = resolve; });
    const destination = new Writable({
      highWaterMark: 1024,
      write(_chunk, _enc, cb) {
        if (!release) { release = cb; blocked(); } else cb();
      }
    });
    const parser = parseZip();
    parser.on('entry', entry => entry.pipe(destination));
    const done = new Promise((resolve, reject) => {
      parser.on('close', resolve);
      parser.on('error', reject);
    });
    const source = new Readable({
      highWaterMark: 16384,
      read() {
        if (produced === archive.length) { this.push(null); return; }
        const end = Math.min(produced + 16384, archive.length);
        const chunk = archive.subarray(produced, end);
        produced = end;
        this.push(chunk);
      }
    });
    source.pipe(parser);
    await firstWrite;
    // Flush scheduled stream work without relying on elapsed-time sleeps.
    await new Promise(resolve => setImmediate(resolve));
    expect(produced).toBeLessThan(512 * 1024);
    release();
    await done;
    expect(produced).toBe(archive.length);
  });

  it('parses a single stored entry', async () => {
    const data = Buffer.from('hello world');
    const zip = Buffer.concat([buildEntry('hello.txt', data), buildEocd()]);
    const entries = await collectEntries(zip);
    expect(entries).toHaveLength(1);
    expect(entries[0].path).toBe('hello.txt');
    expect(entries[0].data.toString()).toBe('hello world');
  });

  it('parses a deflated entry by decompressing the stream', async () => {
    const original = Buffer.from('Compressed payload that should round-trip cleanly');
    const compressed = deflateRawSync(original);
    const zip = Buffer.concat([buildEntry('comp.txt', compressed, { method: 8 }), buildEocd()]);
    const entries = await collectEntries(zip);
    expect(entries).toHaveLength(1);
    expect(entries[0].data.toString()).toBe(original.toString());
  });

  it('parses multiple entries in order', async () => {
    const a = buildEntry('a.txt', Buffer.from('A'));
    const b = buildEntry('b.txt', Buffer.from('BB'));
    const c = buildEntry('c.txt', Buffer.from('CCC'));
    const zip = Buffer.concat([a, b, c, buildEocd()]);
    const entries = await collectEntries(zip);
    expect(entries.map(e => e.path)).toEqual(['a.txt', 'b.txt', 'c.txt']);
    expect(entries[2].data.toString()).toBe('CCC');
  });

  it('sanitizes path traversal segments in entry names', async () => {
    const data = Buffer.from('payload');
    const zip = Buffer.concat([buildEntry('../../etc/passwd', data), buildEocd()]);
    const entries = await collectEntries(zip);
    expect(entries[0].path).toBe('etc/passwd');
  });

  it('normalizes Windows-style backslash separators to forward slashes', async () => {
    const data = Buffer.from('x');
    const zip = Buffer.concat([buildEntry('dir\\sub\\file.txt', data), buildEocd()]);
    const entries = await collectEntries(zip);
    expect(entries[0].path).toBe('dir/sub/file.txt');
  });

  it('autodrains entries whose consumer never pipes', async () => {
    const data = Buffer.from('ignored');
    const zip = Buffer.concat([
      buildEntry('skip.txt', data),
      buildEntry('keep.txt', Buffer.from('kept')),
      buildEocd()
    ]);

    return new Promise((resolve, reject) => {
      const entryPromises = [];
      const parser = parseZip();
      parser.on('entry', (entry) => {
        if (entry.path === 'skip.txt') return;
        entryPromises.push(new Promise((res) => {
          const chunks = [];
          const sink = new Writable({
            write(c, _, cb) { chunks.push(c); cb(); }
          });
          sink.on('finish', () => res({ path: entry.path, data: Buffer.concat(chunks) }));
          entry.pipe(sink);
        }));
      });
      parser.on('close', () => Promise.all(entryPromises).then((out) => {
        try {
          expect(out).toHaveLength(1);
          expect(out[0].path).toBe('keep.txt');
          expect(out[0].data.toString()).toBe('kept');
          resolve();
        } catch (err) { reject(err); }
      }, reject));
      parser.on('error', reject);
      Readable.from([zip]).pipe(parser);
    });
  });

  it('handles multi-chunk arrival of the input buffer', async () => {
    const data = Buffer.from('split across chunks');
    const zip = Buffer.concat([buildEntry('chunked.txt', data), buildEocd()]);
    const pieces = [];
    for (let i = 0; i < zip.length; i += 5) {
      pieces.push(zip.slice(i, Math.min(i + 5, zip.length)));
    }
    const entries = await collectEntries(Buffer.concat(pieces));
    expect(entries[0].data.toString()).toBe(data.toString());
  });

  it('ends the in-flight entry stream when the parser is destroyed mid-entry (no hang)', async () => {
    // A consumer that aborts ingestion (parser.destroy()) while an entry is
    // still streaming must not strand that entry's stream: anything piping it
    // and awaiting completion would otherwise hang forever. Feed a stored entry
    // whose declared size (1000) exceeds the bytes we actually supply (100), so
    // the entry is emitted and piped but left mid-stream when we abort.
    const payload = Buffer.alloc(1000, 0x41);
    const fullEntry = buildEntry('big.dat', payload);
    const headerLen = fullEntry.length - payload.length;
    const partial = fullEntry.subarray(0, headerLen + 100);

    const parser = parseZip();
    const sinkFinished = new Promise((resolve) => {
      parser.on('entry', (entry) => {
        const chunks = [];
        const sink = new Writable({ write(c, _e, cb) { chunks.push(c); cb(); } });
        // Resolves only if the entry stream is allowed to finish — i.e. the
        // parser's teardown ended it rather than abandoning it mid-flight.
        sink.on('finish', () => resolve(Buffer.concat(chunks)));
        entry.pipe(sink);
      });
    });

    // Await the write callback so processBuffer has emitted+piped the entry and
    // buffered its 100 bytes before we abort.
    await new Promise((res) => parser.write(partial, res));
    parser.destroy();

    // Without the destroy→end teardown this never resolves and the test times out.
    const data = await sinkFinished;
    expect(data.length).toBe(100);
  });
});

describe('collectZipEntry', () => {
  it('collects a stored entry into one Buffer', async () => {
    const data = Buffer.from('clinical record json payload');
    const zip = Buffer.concat([buildEntry('clinical_records/a.json', data), buildEocd()]);
    const out = await new Promise((resolve, reject) => {
      const parser = parseZip();
      let read;
      parser.on('entry', (entry) => { read = collectZipEntry(entry); });
      parser.on('close', () => read.then(resolve, reject));
      parser.on('error', reject);
      Readable.from([zip]).pipe(parser);
    });
    expect(out.toString()).toBe('clinical record json payload');
  });

  it('round-trips a deflated entry', async () => {
    const original = Buffer.from('{"resourceType":"Observation","value":42}');
    const zip = Buffer.concat([
      buildEntry('clinical_records/obs.json', deflateRawSync(original), { method: 8 }),
      buildEocd(),
    ]);
    const out = await new Promise((resolve, reject) => {
      const parser = parseZip();
      let read;
      parser.on('entry', (entry) => { read = collectZipEntry(entry); });
      parser.on('close', () => read.then(resolve, reject));
      parser.on('error', reject);
      Readable.from([zip]).pipe(parser);
    });
    expect(out.toString()).toBe(original.toString());
  });

  it('rejects when the member exceeds maxBytes', async () => {
    const data = Buffer.alloc(64, 0x41);
    const zip = Buffer.concat([buildEntry('big.json', data), buildEocd()]);
    const collected = new Promise((resolve, reject) => {
      const parser = parseZip();
      let read;
      parser.on('entry', (entry) => { read = collectZipEntry(entry, 16); });
      parser.on('close', () => read.then(resolve, reject));
      parser.on('error', reject);
      Readable.from([zip]).pipe(parser);
    });
    await expect(collected).rejects.toThrow(/exceeds 16 byte limit/);
  });

  it('rejects (does not hang) when a deflated member is corrupt', async () => {
    // method:8 declares deflate, but the payload is not a valid raw-deflate
    // stream — the inflate pipeline must error and reject the collect, not hang.
    const garbage = Buffer.from('not a valid deflate stream at all');
    const zip = Buffer.concat([buildEntry('clinical_records/bad.json', garbage, { method: 8 }), buildEocd()]);
    const collected = new Promise((resolve, reject) => {
      const parser = parseZip();
      let read;
      parser.on('entry', (entry) => { read = collectZipEntry(entry); });
      parser.on('close', () => read.then(resolve, reject));
      parser.on('error', reject);
      Readable.from([zip]).pipe(parser);
    });
    await expect(collected).rejects.toThrow();
  });

  it('collects multiple JSON members alongside a drained member (appleHealth pattern)', async () => {
    // Mirrors server/routes/appleHealth.js: drain export.xml, collect every
    // clinical_records/*.json into buffers, await them all on 'close'.
    const zip = Buffer.concat([
      buildEntry('apple_health_export/export.xml', Buffer.from('<HealthData/>')),
      buildEntry('clinical_records/r1.json', deflateRawSync(Buffer.from('{"id":1}')), { method: 8 }),
      buildEntry('clinical_records/r2.json', Buffer.from('{"id":2}')),
      buildEocd(),
    ]);
    const jsons = await new Promise((resolve, reject) => {
      const reads = [];
      const collected = [];
      const parser = parseZip();
      parser.on('entry', (entry) => {
        if (entry.path.includes('clinical_records/') && entry.path.endsWith('.json')) {
          reads.push(collectZipEntry(entry).then((buf) => collected.push(buf.toString('utf-8'))));
        } else {
          entry.autodrain();
        }
      });
      parser.on('close', () => Promise.all(reads).then(() => resolve(collected), reject));
      parser.on('error', reject);
      Readable.from([zip]).pipe(parser);
    });
    expect(jsons.sort()).toEqual(['{"id":1}', '{"id":2}']);
  });
});

describe('isZipUpload', () => {
  it('accepts application/zip mimetype', () => {
    expect(isZipUpload({ mimetype: 'application/zip' })).toBe(true);
  });

  it('accepts the Windows x-zip-compressed mimetype', () => {
    expect(isZipUpload({ mimetype: 'application/x-zip-compressed' })).toBe(true);
  });

  it('accepts a .zip filename regardless of mimetype (case-insensitive)', () => {
    expect(isZipUpload({ mimetype: 'application/octet-stream', originalname: 'export.ZIP' })).toBe(true);
    expect(isZipUpload({ originalname: 'takeout.zip' })).toBe(true);
  });

  it('rejects a non-zip upload', () => {
    expect(isZipUpload({ mimetype: 'application/json', originalname: 'history.json' })).toBe(false);
    expect(isZipUpload({ originalname: '_chat.txt' })).toBe(false);
  });

  it('is nullish-safe (missing file / fields do not throw)', () => {
    expect(isZipUpload(null)).toBe(false);
    expect(isZipUpload(undefined)).toBe(false);
    expect(isZipUpload({})).toBe(false);
  });
});

describe('collectZipEntries', () => {
  let dir;
  beforeAll(async () => { dir = await mkdtemp(join(tmpdir(), 'zipentries-')); });
  afterAll(async () => { await rm(dir, { recursive: true, force: true }); });

  async function writeZip(name, buf) {
    const p = join(dir, name);
    await writeFile(p, buf);
    return p;
  }

  it('hands each matching member to onMatch and drains the rest', async () => {
    const zip = Buffer.concat([
      buildEntry('data/keep-a.json', Buffer.from('{"a":1}')),
      buildEntry('data/skip.bin', Buffer.alloc(32, 9)),
      buildEntry('data/keep-b.json', deflateRawSync(Buffer.from('{"b":2}')), { method: 8 }),
      buildEocd(),
    ]);
    const zipPath = await writeZip('mixed.zip', zip);
    const collected = [];
    await collectZipEntries(zipPath, {
      match: (p) => p.endsWith('.json'),
      onMatch: (buf, p) => { collected.push(`${p}:${buf.toString('utf-8')}`); },
    });
    expect(collected.sort()).toEqual(['data/keep-a.json:{"a":1}', 'data/keep-b.json:{"b":2}']);
  });

  it('awaits async onMatch work before resolving (close does not beat the reads)', async () => {
    const zip = Buffer.concat([
      buildEntry('a.json', Buffer.from('1')),
      buildEntry('b.json', Buffer.from('2')),
      buildEocd(),
    ]);
    const zipPath = await writeZip('async.zip', zip);
    const seen = [];
    await collectZipEntries(zipPath, {
      match: '.json',
      onMatch: async (buf) => {
        await new Promise((r) => setTimeout(r, 5));
        seen.push(buf.toString('utf-8'));
      },
    });
    expect(seen.sort()).toEqual(['1', '2']);
  });

  it('supports a match closure over caller state (whatsapp first-fallback pattern)', async () => {
    const zip = Buffer.concat([
      buildEntry('WhatsApp Chat with Bob.txt', Buffer.from('fallback body')),
      buildEntry('extra.txt', Buffer.from('second txt')),
      buildEocd(),
    ]);
    const zipPath = await writeZip('whatsapp.zip', zip);
    let preferred = null;
    let fallback = null;
    await collectZipEntries(zipPath, {
      match: (p) => /_chat\.txt$/i.test(p) || (/\.txt$/i.test(p) && fallback === null),
      onMatch: (buf, p) => {
        if (/_chat\.txt$/i.test(p)) preferred = buf.toString('utf-8');
        else if (fallback === null) fallback = buf.toString('utf-8');
      },
    });
    expect(preferred).toBeNull();
    expect(fallback).toBe('fallback body');
  });

  it('rejects (does not hang) when a matching member fails to parse in onMatch', async () => {
    const zip = Buffer.concat([
      buildEntry('good.json', Buffer.from('{"ok":true}')),
      buildEntry('bad.json', Buffer.from('{not valid json')),
      buildEocd(),
    ]);
    const zipPath = await writeZip('badmember.zip', zip);
    await expect(
      collectZipEntries(zipPath, {
        match: '.json',
        onMatch: (buf) => { JSON.parse(buf.toString('utf-8')); },
      }),
    ).rejects.toThrow();
  });

  it('rejects (does not hang) when a matching deflated member is corrupt', async () => {
    // method:8 declares deflate but the payload is garbage — the inflate pipeline
    // errors; the collector must reject rather than hang awaiting the entry.
    const zip = Buffer.concat([
      buildEntry('corrupt.json', Buffer.from('definitely not a deflate stream'), { method: 8 }),
      buildEocd(),
    ]);
    const zipPath = await writeZip('corrupt.zip', zip);
    await expect(
      collectZipEntries(zipPath, { match: '.json', onMatch: () => {} }),
    ).rejects.toThrow();
  });

  it('rejects (does not hang) when a matching member exceeds maxBytes', async () => {
    const zip = Buffer.concat([
      buildEntry('big.json', Buffer.alloc(64, 0x41)),
      buildEocd(),
    ]);
    const zipPath = await writeZip('oversize.zip', zip);
    await expect(
      collectZipEntries(zipPath, { match: '.json', onMatch: () => {}, maxBytes: 16 }),
    ).rejects.toThrow(/exceeds 16 byte limit/);
  });

  it('rejects when the source file is missing', async () => {
    await expect(
      collectZipEntries(join(dir, 'does-not-exist.zip'), { match: '.json', onMatch: () => {} }),
    ).rejects.toThrow();
  });

  it.each([
    { maxTotalBytes: 5 },
    { maxEntries: 1 }
  ])('rejects many individually valid members over the archive budget: %j', async (limits) => {
    const zipPath = await writeZip('aggregate.zip', Buffer.concat([
      buildEntry('a.json', Buffer.from('{} ')),
      buildEntry('b.json', Buffer.from('{} ')), buildEocd()
    ]));
    await expect(collectZipEntries(zipPath, {
      match: '.json', onMatch: () => {}, ...limits
    })).rejects.toMatchObject({ status: 413 });
  });

  it('allows more than 10000 ignored members without spending the selected-entry budget', async () => {
    const entries = Array.from({ length: 10001 }, (_, i) => buildEntry(`skip/${i}`, Buffer.alloc(0)));
    const zipPath = await writeZip('skipped.zip', Buffer.concat([
      ...entries, buildEntry('keep.json', Buffer.from('{}')), buildEocd()
    ]));
    const found = [];
    await collectZipEntries(zipPath, { match: '.json', maxEntries: 1, onMatch: buf => found.push(buf.toString()) });
    expect(found).toEqual(['{}']);
  });

  it('resolves with no matches on a truncated archive whose central directory is cut off', async () => {
    // A truncated ZIP (upload interrupted): a complete stored entry followed by a
    // sliced-off second header and no EOCD. The parser reads the whole first entry
    // and simply reaches stream end — no hang, no throw, matches processed.
    const full = Buffer.concat([
      buildEntry('first.json', Buffer.from('{"n":1}')),
      buildEntry('second.json', Buffer.from('{"n":2}')),
    ]);
    // Cut mid-way through the second entry's header so it never completes.
    const truncated = full.subarray(0, full.length - 20);
    const zipPath = await writeZip('truncated.zip', truncated);
    const collected = [];
    await collectZipEntries(zipPath, {
      match: '.json',
      onMatch: (buf, p) => { collected.push(p); },
    });
    // The first entry is intact and processed; the second is never fully emitted.
    expect(collected).toContain('first.json');
    expect(collected).not.toContain('second.json');
  });
});

describe('extractZipEntryToBuffer', () => {
  let dir;
  let zipPath;
  // A two-entry zip mirroring an mflux checkpoint: a big stored "optimizer"
  // FIRST (must be skipped without inflation), then a deflated "adapter".
  const adapterPayload = Buffer.from('adapter weights — round-trip me cleanly');

  beforeAll(async () => {
    dir = await mkdtemp(join(tmpdir(), 'zipextract-'));
    zipPath = join(dir, 'ckpt.zip');
    const optimizer = buildEntry('0001_optimizer.safetensors', Buffer.alloc(4096, 7));
    const adapter = buildEntry('0001_adapter.safetensors', deflateRawSync(adapterPayload), { method: 8 });
    await writeFile(zipPath, Buffer.concat([optimizer, adapter, buildEocd()]));
  });

  afterAll(async () => { await rm(dir, { recursive: true, force: true }); });

  it('extracts a later deflated entry past an earlier one via predicate', async () => {
    const out = await extractZipEntryToBuffer(
      zipPath,
      (name) => name.endsWith('_adapter.safetensors') && !name.includes('optimizer'),
    );
    expect(out).not.toBeNull();
    expect(out.toString()).toBe(adapterPayload.toString());
  });

  it('matches by substring shorthand', async () => {
    const out = await extractZipEntryToBuffer(zipPath, '_adapter.safetensors');
    expect(out.toString()).toBe(adapterPayload.toString());
  });

  it('resolves null when nothing matches', async () => {
    expect(await extractZipEntryToBuffer(zipPath, 'no-such-member.bin')).toBeNull();
  });

  it('rejects on a missing file', async () => {
    await expect(extractZipEntryToBuffer(join(dir, 'nope.zip'), 'x')).rejects.toThrow();
  });
});