Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

19.2 KB
Raw Blame
import { describe, it, expect, beforeEach, afterEach, afterAll, vi } from 'vitest';
import { mkdtempSync, rmSync, mkdirSync } from 'fs';
import { tmpdir } from 'os';
import { join } from 'path';
import { mockNoPeerSync, mockNoPeers, mockTestIdentity } from '../../lib/mockPathsDataRoot.js';

// Universe builder evaluates PATHS.data at module top, so allocate a stable
// temp dir before any service imports. beforeEach wipes + reseeds inside.
const tempData = mkdtempSync(join(tmpdir(), 'portos-subs-test-data-'));
let tempBucket;
// Lets a test freeze the next export mid-flight (the exporter resolves the
// instance id before it writes the manifest).
const gate = vi.hoisted(() => ({ hold: null, held: 0, calls: 0 }));

vi.mock('../../lib/fileUtils.js', async () => {
  const actual = await vi.importActual('../../lib/fileUtils.js');
  return new Proxy(actual, {
    get(target, prop) {
      if (prop === 'PATHS') return {
        ...actual.PATHS,
        data: tempData,
        images: join(tempData, 'images'),
        videos: join(tempData, 'videos'),
      };
      return target[prop];
    },
  });
});
vi.mock('../instances.js', () => mockNoPeers());
vi.mock('../instanceIdentity.js', () => mockTestIdentity({
  getInstanceId: async () => {
    gate.calls++;
    const hold = gate.hold;
    if (hold) { gate.hold = null; gate.held++; await hold; }
    return 'inst-test';
  },
}));
vi.mock('./peerSync.js', () => mockNoPeerSync());
vi.mock('../mediaJobQueue/index.js', () => ({ getJob: () => null }));

const buckets = await import('./buckets.js');
const subs = await import('./subscriptions.js');
const universeBuilder = await import('../universeBuilder.js');
const series = await import('../pipeline/series.js');

describe('sharing/subscriptions', () => {
  beforeEach(() => {
    rmSync(tempData, { recursive: true, force: true });
    mkdirSync(tempData, { recursive: true });
    mkdirSync(join(tempData, 'images'), { recursive: true });
    tempBucket = mkdtempSync(join(tmpdir(), 'portos-subs-test-bucket-'));
    subs.__resetForTests();
    gate.hold = null; gate.held = 0; gate.calls = 0;
  });
  afterEach(async () => {
    vi.useRealTimers();
    // Debounce timers and listener-triggered exports outlive the test body and
    // would recreate the bucket after the rmSync below (#12288).
    await subs.__settleForTests();
    if (tempBucket) rmSync(tempBucket, { recursive: true, force: true });
  });

  // tempData is minted once (module scope) and reused across every test in
  // this file — beforeEach only wipes-and-recreates it, so the LAST test's
  // root is never removed. Clean it once here (#9032).
  afterAll(() => {
    rmSync(tempData, { recursive: true, force: true });
  });

  it('subscribe creates a subscription and writes a deterministic-named manifest', async () => {
    const bucket = await buckets.createBucket({ name: 'B', path: tempBucket, mode: 'inbox' });
    const u = await universeBuilder.createUniverse({ name: 'U1' });

    const sub = await subs.subscribe({
      bucketId: bucket.id, recordKind: 'universe', recordId: u.id,
    });
    expect(sub.id).toMatch(/^sub-universe-/);
    expect(sub.bucketId).toBe(bucket.id);
    expect(sub.lastManifestId).toBeTruthy();

    const fs = await import('fs');
    const filename = subs.subscriptionFilename({ recordKind: 'universe', recordId: u.id, senderInstanceId: 'inst-test' });
    expect(filename).toBe(`sub-universe-${u.id}-inst-test.json`);
    expect(fs.existsSync(join(tempBucket, 'manifests', filename))).toBe(true);
  });

  it('subscriptionFilename falls back to "unknown" sender when missing', async () => {
    const filename = subs.subscriptionFilename({ recordKind: 'series', recordId: 'ser-x' });
    expect(filename).toBe('sub-series-ser-x-unknown.json');
  });

  it('subscribe is idempotent — second call re-exports onto the same file with a new manifestId', async () => {
    const bucket = await buckets.createBucket({ name: 'B', path: tempBucket });
    const u = await universeBuilder.createUniverse({ name: 'U2' });
    const first = await subs.subscribe({ bucketId: bucket.id, recordKind: 'universe', recordId: u.id });
    const second = await subs.subscribe({ bucketId: bucket.id, recordKind: 'universe', recordId: u.id });
    expect(first.id).toBe(second.id);
    expect(second.lastManifestId).not.toBe(first.lastManifestId);

    const all = await subs.listSubscriptions();
    expect(all.filter((s) => s.recordId === u.id)).toHaveLength(1);
  });

  it('unsubscribe removes the subscription and deletes the bucket file', async () => {
    const bucket = await buckets.createBucket({ name: 'B', path: tempBucket });
    const u = await universeBuilder.createUniverse({ name: 'U3' });
    const sub = await subs.subscribe({ bucketId: bucket.id, recordKind: 'universe', recordId: u.id });
    const filename = subs.subscriptionFilename({ ...sub, senderInstanceId: 'inst-test' });
    const filePath = join(tempBucket, 'manifests', filename);
    const fs = await import('fs');
    expect(fs.existsSync(filePath)).toBe(true);

    await subs.unsubscribe(sub.id);
    expect(fs.existsSync(filePath)).toBe(false);
    expect(await subs.listSubscriptions()).toEqual([]);
  });

  it('unsubscribe also cleans up the pre-sharing-v2 legacy filename when authored by this instance', async () => {
    const bucket = await buckets.createBucket({ name: 'B', path: tempBucket });
    const u = await universeBuilder.createUniverse({ name: 'U-legacy' });
    const fs = await import('fs');

    const sub = await subs.subscribe({ bucketId: bucket.id, recordKind: 'universe', recordId: u.id });
    const legacyPath = join(tempBucket, 'manifests', `sub-universe-${u.id}.json`);
    fs.writeFileSync(legacyPath, JSON.stringify({ senderInstanceId: 'inst-test' }));

    await subs.unsubscribe(sub.id);
    expect(fs.existsSync(legacyPath)).toBe(false);
  });

  it('unsubscribe preserves a legacy filename authored by a DIFFERENT peer', async () => {
    const bucket = await buckets.createBucket({ name: 'B', path: tempBucket });
    const u = await universeBuilder.createUniverse({ name: 'U-legacy-other' });
    const fs = await import('fs');

    const sub = await subs.subscribe({ bucketId: bucket.id, recordKind: 'universe', recordId: u.id });
    const legacyPath = join(tempBucket, 'manifests', `sub-universe-${u.id}.json`);
    fs.writeFileSync(legacyPath, JSON.stringify({ senderInstanceId: 'some-other-peer' }));

    await subs.unsubscribe(sub.id);
    expect(fs.existsSync(legacyPath)).toBe(true);
  });

  it('unsubscribe preserves a legacy filename with no senderInstanceId (ownership unverifiable)', async () => {
    // A pre-v2 or malformed legacy file with no `senderInstanceId` could
    // belong to any peer; deleting it on unsubscribe risks stomping a
    // peer's share. The strict-equality gate must leave it in place.
    const bucket = await buckets.createBucket({ name: 'B', path: tempBucket });
    const u = await universeBuilder.createUniverse({ name: 'U-legacy-anon' });
    const fs = await import('fs');

    const sub = await subs.subscribe({ bucketId: bucket.id, recordKind: 'universe', recordId: u.id });
    const legacyPath = join(tempBucket, 'manifests', `sub-universe-${u.id}.json`);
    fs.writeFileSync(legacyPath, JSON.stringify({ /* senderInstanceId absent */ }));

    await subs.unsubscribe(sub.id);
    expect(fs.existsSync(legacyPath)).toBe(true);
  });

  it('listSubscriptions filters by bucketId / recordKind / recordId', async () => {
    const bucket1 = await buckets.createBucket({ name: 'A', path: tempBucket });
    const bucket2Path = mkdtempSync(join(tmpdir(), 'portos-subs-b2-'));
    const bucket2 = await buckets.createBucket({ name: 'B', path: bucket2Path });
    const u = await universeBuilder.createUniverse({ name: 'U4' });
    const s = await series.createSeries({ name: 'S1' });

    await subs.subscribe({ bucketId: bucket1.id, recordKind: 'universe', recordId: u.id });
    await subs.subscribe({ bucketId: bucket2.id, recordKind: 'universe', recordId: u.id });
    await subs.subscribe({ bucketId: bucket1.id, recordKind: 'series', recordId: s.id });

    expect((await subs.listSubscriptions({ bucketId: bucket1.id }))).toHaveLength(2);
    expect((await subs.listSubscriptions({ recordKind: 'universe' }))).toHaveLength(2);
    expect((await subs.listSubscriptions({ recordKind: 'series', recordId: s.id }))).toHaveLength(1);

    rmSync(bucket2Path, { recursive: true, force: true });
  });

  it('subscribe rejects unsubscribable kinds', async () => {
    const bucket = await buckets.createBucket({ name: 'B', path: tempBucket });
    await expect(subs.subscribe({ bucketId: bucket.id, recordKind: 'media', recordId: 'x' }))
      .rejects.toMatchObject({ code: subs.ERR_VALIDATION });
  });

  it('unsubscribe throws ERR_NOT_FOUND for unknown id', async () => {
    await expect(subs.unsubscribe('does-not-exist'))
      .rejects.toMatchObject({ code: subs.ERR_NOT_FOUND });
  });

  it('findSubscription returns null when none exists', async () => {
    const bucket = await buckets.createBucket({ name: 'B', path: tempBucket });
    expect(await subs.findSubscription(bucket.id, 'universe', 'no-such-id')).toBeNull();
  });

  it('unsubscribeAllForRecord drops every matching subscription', async () => {
    const bucket1 = await buckets.createBucket({ name: 'A', path: tempBucket });
    const bucket2Path = mkdtempSync(join(tmpdir(), 'portos-subs-b2-'));
    const bucket2 = await buckets.createBucket({ name: 'B', path: bucket2Path });
    const u = await universeBuilder.createUniverse({ name: 'shared-uni' });
    await subs.subscribe({ bucketId: bucket1.id, recordKind: 'universe', recordId: u.id });
    await subs.subscribe({ bucketId: bucket2.id, recordKind: 'universe', recordId: u.id });
    expect(await subs.listSubscriptions({ recordId: u.id })).toHaveLength(2);

    const result = await subs.unsubscribeAllForRecord('universe', u.id);
    expect(result.removed).toHaveLength(2);
    expect(await subs.listSubscriptions({ recordId: u.id })).toEqual([]);
    rmSync(bucket2Path, { recursive: true, force: true });
  });

  it('adoptImportedSubscription does not flip a deliberately-subscribed record to adoptedFromImport on re-adopt', async () => {
    const bucket = await buckets.createBucket({ name: 'B', path: tempBucket });
    const u = await universeBuilder.createUniverse({ name: 'U-adopt' });
    const original = await subs.subscribe({ bucketId: bucket.id, recordKind: 'universe', recordId: u.id });
    expect(original.adoptedFromImport).toBeUndefined();

    const adopted = await subs.adoptImportedSubscription({
      bucketId: bucket.id, recordKind: 'universe', recordId: u.id, lastManifestId: 'inbound-1',
    });
    expect(adopted.id).toBe(original.id);
    expect(adopted.adoptedFromImport).toBeUndefined();
    expect(adopted.lastManifestId).toBe('inbound-1');
  });

  it('deleting the local record auto-unsubscribes via the deleted recordEvent', async () => {
    subs.installSubscriptionListener();
    const bucket = await buckets.createBucket({ name: 'D', path: tempBucket });
    const u = await universeBuilder.createUniverse({ name: 'to-delete' });
    await subs.subscribe({ bucketId: bucket.id, recordKind: 'universe', recordId: u.id });
    expect((await subs.listSubscriptions({ recordId: u.id }))).toHaveLength(1);

    await universeBuilder.deleteUniverse(u.id);
    await vi.waitFor(async () => {
      expect(await subs.listSubscriptions({ recordId: u.id })).toEqual([]);
    }, { timeout: 2_000, interval: 20 });
  });

  // vi.waitFor polls on setTimeout, which these tests fake; poll on setImmediate.
  async function until(check) {
    // Bound by wall-clock, not iterations: the export awaits real fs I/O, which a loaded CI box can stretch.
    const deadline = Date.now() + 10000;
    while (Date.now() < deadline) {
      if (await check()) return;
      await new Promise((r) => setImmediate(r));
    }
    throw new Error('until: condition never became true');
  }

  // The tests drive the 3s re-export debounce by hand, but the export itself can
  // also sleep: on Windows atomicWrite retries a locked rename (EPERM from AV or
  // an open handle) after a setTimeout. A frozen fake clock never fires that
  // retry, so the export (and an unsubscribe waiting on it) hangs. Let the fake
  // clock follow real time so incidental sleeps still elapse.
  function fakeDebounceTimers() {
    vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout'], shouldAdvanceTime: true });
  }

  // Give a released export (which would write its manifest late) real time to finish.
  async function settleExports() {
    vi.useRealTimers();
    await new Promise((r) => setTimeout(r, 250));
  }

  // Schedule a debounced re-export of the universe and freeze it mid-export.
  async function holdNextExport(recordId) {
    let release;
    gate.hold = new Promise((r) => { release = r; });
    await subs.reexportSubscribedRecord('universe', recordId);
    await vi.advanceTimersByTimeAsync(3000);
    await until(() => gate.held === 1);
    return release;
  }

  it('Stop sharing during an in-flight re-export does not resurrect the manifest or stamp state', async () => {
    fakeDebounceTimers();
    const bucket = await buckets.createBucket({ name: 'B', path: tempBucket });
    const u = await universeBuilder.createUniverse({ name: 'U-race' });
    const sub = await subs.subscribe({ bucketId: bucket.id, recordKind: 'universe', recordId: u.id });
    const fs = await import('fs');
    const filePath = join(tempBucket, 'manifests', subs.subscriptionFilename({ ...sub, senderInstanceId: 'inst-test' }));

    const release = await holdNextExport(u.id);

    const unsub = subs.unsubscribe(sub.id);
    release();
    await unsub;
    await settleExports();

    expect(fs.existsSync(filePath)).toBe(false);
    expect(await subs.listSubscriptions()).toEqual([]);
  });

  it('deleting the record during an in-flight re-export leaves no manifest in the bucket', async () => {
    fakeDebounceTimers();
    subs.installSubscriptionListener();
    const bucket = await buckets.createBucket({ name: 'B', path: tempBucket });
    const u = await universeBuilder.createUniverse({ name: 'U-del-race' });
    const sub = await subs.subscribe({ bucketId: bucket.id, recordKind: 'universe', recordId: u.id });
    const fs = await import('fs');
    const filePath = join(tempBucket, 'manifests', subs.subscriptionFilename({ ...sub, senderInstanceId: 'inst-test' }));

    const release = await holdNextExport(u.id);

    await universeBuilder.deleteUniverse(u.id);
    await until(async () => (await subs.listSubscriptions({ recordId: u.id })).length === 0);
    release();
    vi.useRealTimers();
    // unsubscribe waits out the export, then deletes.
    await until(() => !fs.existsSync(filePath));
    await settleExports();
    expect(fs.existsSync(filePath)).toBe(false);
  });

  it('overlapping re-exports of one subscription run one after another', async () => {
    fakeDebounceTimers();
    const bucket = await buckets.createBucket({ name: 'B', path: tempBucket });
    const u = await universeBuilder.createUniverse({ name: 'U-order' });
    const sub = await subs.subscribe({ bucketId: bucket.id, recordKind: 'universe', recordId: u.id });

    const release = await holdNextExport(u.id);
    const callsWhileFirstHeld = gate.calls;

    await subs.reexportSubscribedRecord('universe', u.id);
    await vi.advanceTimersByTimeAsync(3000);
    for (let i = 0; i < 20; i++) await new Promise((r) => setImmediate(r));
    expect(gate.calls).toBe(callsWhileFirstHeld); // second export has not started

    release();
    // After both exports finish the stamped manifest id is the one on disk
    // (the second export wrote last).
    const fs = await import('fs');
    const filePath = join(tempBucket, 'manifests', subs.subscriptionFilename({ ...sub, senderInstanceId: 'inst-test' }));
    await until(async () => {
      const [live] = await subs.listSubscriptions({ recordId: u.id });
      if (gate.calls <= callsWhileFirstHeld || live.lastManifestId === sub.lastManifestId) return false;
      const onDisk = JSON.parse(fs.readFileSync(filePath, 'utf8'));
      return onDisk.id === live.lastManifestId;
    });
  });

  describe('reconcileSubscriptionsOnBoot (#11750)', () => {
    // Rewrite persisted lastExportedAt to simulate "exported before the edit"
    // (past) or "exported after the last edit" (future) without sleeping.
    async function setLastExportedAt(subId, iso) {
      const fs = await import('fs');
      const path = join(tempData, 'sharing', 'subscriptions.json');
      const state = JSON.parse(fs.readFileSync(path, 'utf8'));
      state.subscriptions.find((x) => x.id === subId).lastExportedAt = iso;
      fs.writeFileSync(path, JSON.stringify(state));
    }

    it('re-exports a share whose record changed after lastExportedAt exactly once, and leaves an unchanged one alone', async () => {
      const bucket = await buckets.createBucket({ name: 'B', path: tempBucket });
      const stale = await universeBuilder.createUniverse({ name: 'U-stale' });
      const fresh = await universeBuilder.createUniverse({ name: 'U-fresh' });
      const staleSub = await subs.subscribe({ bucketId: bucket.id, recordKind: 'universe', recordId: stale.id });
      const freshSub = await subs.subscribe({ bucketId: bucket.id, recordKind: 'universe', recordId: fresh.id });
      await setLastExportedAt(staleSub.id, '2000-01-01T00:00:00.000Z');
      await setLastExportedAt(freshSub.id, '2999-01-01T00:00:00.000Z');

      const first = await subs.reconcileSubscriptionsOnBoot();
      expect(first.reexported).toBe(1);
      const [afterStale] = await subs.listSubscriptions({ recordId: stale.id });
      const [afterFresh] = await subs.listSubscriptions({ recordId: fresh.id });
      expect(afterStale.lastManifestId).not.toBe(staleSub.lastManifestId);
      expect(afterFresh.lastManifestId).toBe(freshSub.lastManifestId);

      // The stamp moved lastExportedAt past the record, so a second pass is a no-op.
      expect((await subs.reconcileSubscriptionsOnBoot()).reexported).toBe(0);
    });

    it('exports a never-exported subscription, but not an import-adopted one whose record is unchanged', async () => {
      const bucket = await buckets.createBucket({ name: 'B', path: tempBucket });
      const never = await universeBuilder.createUniverse({ name: 'U-never' });
      const adopted = await universeBuilder.createUniverse({ name: 'U-adopted' });
      const neverSub = await subs.subscribe({ bucketId: bucket.id, recordKind: 'universe', recordId: never.id });
      await setLastExportedAt(neverSub.id, null);
      await subs.adoptImportedSubscription({ bucketId: bucket.id, recordKind: 'universe', recordId: adopted.id });

      const result = await subs.reconcileSubscriptionsOnBoot();
      expect(result.reexported).toBe(1);
      const [reexported] = await subs.listSubscriptions({ recordId: never.id });
      expect(reexported.lastExportedAt).toBeTruthy();
      const [stillAdopted] = await subs.listSubscriptions({ recordId: adopted.id });
      expect(stillAdopted.lastExportedAt).toBeNull();
    });

    it('skips a subscription whose record no longer exists', async () => {
      const bucket = await buckets.createBucket({ name: 'B', path: tempBucket });
      await subs.adoptImportedSubscription({ bucketId: bucket.id, recordKind: 'universe', recordId: 'missing-universe' });
      await expect(subs.reconcileSubscriptionsOnBoot()).resolves.toMatchObject({ reexported: 0 });
    });
  });
});