Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

21.4 KB
Raw Blame
/**
 * Share Bucket — manifest & cursor helpers.
 *
 * A manifest is one JSON file in `<bucket>/manifests/` per share action. It
 * names the records + assets bundled in that share, plus identifying
 * metadata (sender, source display name, timestamp). The cursor tracks which
 * manifest filenames the local PortOS has already processed so the watcher
 * doesn't replay them on restart.
 *
 * Manifest filename convention:
 *   <iso-ts>-<sender-id>-<manifest-id>.json
 *
 * Using an ISO-prefixed timestamp gives chokidar's lexicographic event order a
 * stable replay order if multiple manifests land in the same poll cycle.
 */

import { randomUUID } from 'crypto';
import { join } from 'path';
import { readdir, rename } from 'fs/promises';
import { PATHS, atomicWrite, readJSONFile, ensureDir } from '../../lib/fileUtils.js';
import { createKeyCachedQueue } from '../../lib/createKeyCachedQueue.js';
import { isPlainObject } from '../../lib/objects.js';
import { SHARING_SCHEMA_VERSION, getProducedByVersion } from './version.js';
import { universeCollectionNameFor, seriesCollectionNameFor } from '../mediaCollections.js';
import { UNKNOWN_INSTANCE_ID } from '../instanceIdentity.js';
import { isStr } from '../../lib/textUtils.js';

export const MANIFEST_KIND = Object.freeze(['series', 'universe', 'media', 'media-annotations']);

/** Deterministic filename for an instance's annotation manifest in a bucket.
 *  One file per (bucket, instanceId) — re-exports overwrite in place so peers
 *  see updates via chokidar `change` events, same pattern as subscriptions. */
export function annotationManifestFilename(senderInstanceId) {
  const safeId = String(senderInstanceId || UNKNOWN_INSTANCE_ID).replace(/[^A-Za-z0-9_-]/g, '_').slice(0, 80);
  return `annotations-${safeId}.json`;
}

/** @deprecated Use SHARING_SCHEMA_VERSION from ./version.js. Kept exported for back-compat. */
export const MANIFEST_SCHEMA_VERSION = SHARING_SCHEMA_VERSION;

const queueCursorWrite = createKeyCachedQueue();

const cursorPath = (bucketId) => join(PATHS.data, 'sharing', 'cursors', `${bucketId}.json`);

/**
 * Cursor shape:
 *   {
 *     processedById: { '<filename>': '<lastSeenManifestId>' },
 *     processed: ['<filename>', ...],            // legacy: pre-content-aware cursor entries
 *     refusedById: { '<filename>': { manifestId, localSchema } },
 *     lastProcessedAt: ISO
 *   }
 *
 * The `processedById` map is the content-aware path used for subscription
 * manifests: a fresh `manifestId` inside the same filename means the file's
 * contents changed (the subscription re-exported) and we need to re-import.
 * Legacy `processed[]` entries from buckets created on v1.0..v1.2 still
 * suppress re-import — those were one-shot manifests with random filenames,
 * so they're never going to be re-processed by content anyway.
 */
export async function readCursor(bucketId) {
  await ensureDir(join(PATHS.data, 'sharing', 'cursors'));
  const raw = await readJSONFile(cursorPath(bucketId), { processedById: {}, processed: [] }, { logError: false, strict: true });
  return {
    processedById: (raw.processedById && typeof raw.processedById === 'object') ? raw.processedById : {},
    processed: Array.isArray(raw.processed) ? raw.processed : [],
    refusedById: isPlainObject(raw.refusedById) ? raw.refusedById : {},
    lastProcessedAt: raw.lastProcessedAt || null,
  };
}

export async function writeCursor(bucketId, cursor) {
  await ensureDir(join(PATHS.data, 'sharing', 'cursors'));
  await atomicWrite(cursorPath(bucketId), cursor);
}

/**
 * Mark a manifest filename as processed at the given manifestId. Caps the
 * cursor at 5000 entries so a runaway peer can't grow the file unbounded.
 */
export async function markProcessed(bucketId, manifestFilename, manifestId = null) {
  return queueCursorWrite(bucketId, async () => {
    const cursor = await readCursor(bucketId);
    cursor.processedById[manifestFilename] = manifestId || cursor.processedById[manifestFilename] || '';
    // Drop legacy entry if it was tracked there; the map is now authoritative.
    if (cursor.processed.includes(manifestFilename)) {
      cursor.processed = cursor.processed.filter((f) => f !== manifestFilename);
    }
    delete cursor.refusedById[manifestFilename];
    const keys = Object.keys(cursor.processedById);
    if (keys.length > 5000) {
      // Drop the lexicographically-smallest 1000 — for timestamp-prefixed
      // one-shot manifests this is the oldest entries. Subscription filenames
      // (sub-…) sort after timestamp-prefixed ones, so they survive a prune
      // even when the cursor is full of legacy one-shot history.
      const drop = new Set(keys.sort().slice(0, keys.length - 5000));
      for (const k of drop) delete cursor.processedById[k];
    }
    cursor.lastProcessedAt = new Date().toISOString();
    await writeCursor(bucketId, cursor);
    return cursor;
  });
}

/**
 * Remove a filename from the cursor entirely (used on `unlink` so a future
 * re-share of the same record under the same filename re-imports cleanly).
 * No-op short-circuit when the filename isn't tracked — chokidar fires
 * `unlink` for arbitrary files in the watched dir, not just our manifests.
 */
export async function forgetProcessed(bucketId, manifestFilename) {
  return queueCursorWrite(bucketId, async () => {
    const cursor = await readCursor(bucketId);
    const inMap = cursor.processedById && manifestFilename in cursor.processedById;
    const inLegacy = cursor.processed.includes(manifestFilename);
    const inRefused = manifestFilename in cursor.refusedById;
    if (!inMap && !inLegacy && !inRefused) return cursor;
    if (inMap) delete cursor.processedById[manifestFilename];
    if (inLegacy) cursor.processed = cursor.processed.filter((f) => f !== manifestFilename);
    if (inRefused) delete cursor.refusedById[manifestFilename];
    await writeCursor(bucketId, cursor);
    return cursor;
  });
}

/**
 * Record a manifest this install refused because its own schema is behind the
 * sender's. Unlike `markProcessed`, the entry is bound to `localSchema` — a
 * fingerprint of the versions that refused it — so it suppresses the watcher's
 * replays only until this install's schema changes. The first backlog walk
 * after an upgrade re-evaluates the manifest and imports it.
 */
export async function markRefused(bucketId, manifestFilename, manifestId, localSchema) {
  return queueCursorWrite(bucketId, async () => {
    const cursor = await readCursor(bucketId);
    cursor.refusedById[manifestFilename] = { manifestId: manifestId || '', localSchema };
    await writeCursor(bucketId, cursor);
    return cursor;
  });
}

/** True when this exact manifest was already refused under the current local schema. */
export function isRefusalCurrent(cursor, manifestFilename, manifestId, localSchema) {
  const refusal = cursor?.refusedById?.[manifestFilename];
  return Boolean(refusal) && refusal.manifestId === (manifestId || '') && refusal.localSchema === localSchema;
}

/**
 * Returns true when we've already imported THIS specific version of the
 * manifest. For subscription manifests, content-aware: a new manifestId on
 * the same filename re-processes. For legacy entries (processed[] array),
 * any subsequent reference to that filename is suppressed.
 */
export function hasBeenProcessed(cursor, manifestFilename, manifestId = null) {
  if (cursor?.processedById && manifestFilename in cursor.processedById) {
    const stored = cursor.processedById[manifestFilename];
    if (!stored) return true; // unknown id → trust filename match (idempotent)
    return manifestId ? stored === manifestId : true;
  }
  return Array.isArray(cursor?.processed) && cursor.processed.includes(manifestFilename);
}

/**
 * Build a fresh manifest. Caller writes it after every record + asset is in
 * place. `producedByVersion` is the PortOS app version; resolve via
 * `getProducedByVersion()` (from ./version.js) and pass in — kept as a
 * parameter so this stays pure + synchronous.
 *
 * `schemaVersion` is the wire-protocol version (the only one importer compat
 * is gated on). `sharingSchemaVersion` is its more descriptive alias kept for
 * forward-compat readability — both carry the same value.
 *
 * `subscription`, when provided as `{ recordKind, recordId }`, marks this
 * manifest as a persistent subscription rather than a one-shot share. The
 * filename is then deterministic per (recordKind, recordId), so re-exports
 * overwrite in place and recipients see updates via chokidar `change`
 * events. Importer cursor keys on the manifest id (fresh per export) so
 * content-changes re-process correctly.
 */
export function buildManifest({
  kind, senderInstanceId, source, sourceBio, recordIds, assetRefs, reviewRefs, outlineRefs,
  bucketId, bucketName, note, producedByVersion, subscription, collection,
  portosSchemaVersions,
}) {
  if (!MANIFEST_KIND.includes(kind)) throw new Error(`buildManifest: invalid kind '${kind}'`);
  return {
    id: randomUUID(),
    schemaVersion: SHARING_SCHEMA_VERSION,
    sharingSchemaVersion: SHARING_SCHEMA_VERSION,
    producedByVersion: producedByVersion || 'unknown',
    // Per-category storage layout versions the sender's code expected at
    // export time. The importer compares against its own
    // `PORTOS_SCHEMA_VERSIONS` and refuses to apply a manifest that's
    // ahead — the user is told to upgrade PortOS before the import can
    // proceed. Optional on the wire so legacy manifests (no field) still
    // validate; absent treated as "no contract" by the comparator.
    // Plain-object only: `typeof === 'object'` alone would accept arrays, and
    // `{ ...[] }` spreads into numeric keys, producing a malformed
    // `{ "0": ... }` map the importer's comparator can't read. `isPlainObject`
    // rejects arrays + null (the realistic junk); exotic inputs like a Date
    // spread to a harmless `{}` the comparator treats as "no contract".
    portosSchemaVersions: isPlainObject(portosSchemaVersions)
      ? { ...portosSchemaVersions }
      : null,
    createdAt: new Date().toISOString(),
    kind,                              // 'series' | 'universe' | 'media'
    subscription: subscription && subscription.recordKind && subscription.recordId
      ? { recordKind: subscription.recordKind, recordId: subscription.recordId }
      : null,
    senderInstanceId: senderInstanceId || UNKNOWN_INSTANCE_ID,
    source: source || 'unknown',        // human display name
    sourceBio: sourceBio || null,
    bucketId,
    bucketName: bucketName || bucketId,
    recordIds: Array.isArray(recordIds) ? recordIds : [],
    assetRefs: Array.isArray(assetRefs) ? assetRefs : [],
    // Series ids whose manuscript-review sibling doc (records/reviews/<id>.json)
    // is bundled in this manifest. Declared separately from `recordIds` because
    // the review is keyed by seriesId, not a record id of its own — so the
    // importer can WAIT for the review file to land before markProcessed
    // (cloud-relayed buckets deliver files out of order). Absent on legacy
    // manifests → importer treats as "no declared reviews".
    reviewRefs: Array.isArray(reviewRefs) ? reviewRefs : [],
    // Series ids whose reverse-outline sibling doc (records/outlines/<id>.json)
    // is bundled (#1348). Same out-of-band-by-seriesId rationale as reviewRefs:
    // the outline has no record id of its own, so the importer waits for the
    // declared file before markProcessed (cloud-relayed buckets deliver out of
    // order). Absent on legacy manifests → importer treats as "no outlines".
    outlineRefs: Array.isArray(outlineRefs) ? outlineRefs : [],
    // Optional payload for universe/series shares — the linked media
    // collection so recipients gain the same set of generated images and
    // the link is restored on their side. Either `universeId` (universe-
    // owned auto-collection, or a universe-linked series' bundle) OR
    // `seriesId` (per-series auto-collection for universeless series) is
    // present, never both.
    collection: collection && Array.isArray(collection.items)
      && (collection.universeId || collection.seriesId)
      ? {
        name: collection.name
          || (collection.universeId
            ? universeCollectionNameFor(collection.universeId)
            : seriesCollectionNameFor(collection.seriesId)),
        ...(collection.universeId ? { universeId: collection.universeId } : {}),
        ...(collection.seriesId && !collection.universeId ? { seriesId: collection.seriesId } : {}),
        description: collection.description || '',
        items: collection.items.map((it) => ({ kind: it.kind, ref: it.ref, addedAt: it.addedAt || null })),
      }
      : null,
    note: isStr(note) ? note.slice(0, 1000) : null,
  };
}

/** Filename prefix shared by every subscription manifest, regardless of
 *  schema version. The prune filter at `pruneBucketManifests` uses this same
 *  prefix to exempt subscription manifests from archive cleanup. */
export const SUBSCRIPTION_FILE_PREFIX = 'sub-';

/** Filename for a subscription manifest. Used by both the writer (exporter)
 *  and the reader (subscriptions service, watcher) so the formula stays in
 *  one place. The `senderInstanceId` segment keeps two peers' shares of the
 *  same record from colliding; absent senderInstanceId falls back to the
 *  UNKNOWN_INSTANCE_ID sentinel so the filename is always valid. The sender
 *  segment is sanitized with the same allowlist/length cap used by
 *  `annotationManifestFilename` so a malformed persisted instance id
 *  containing path separators can't escape the `manifests` directory. */
export function subscriptionFilename({ recordKind, recordId, senderInstanceId }) {
  const raw = (isStr(senderInstanceId) ? senderInstanceId.trim() : '') || UNKNOWN_INSTANCE_ID;
  const sender = raw.replace(/[^A-Za-z0-9_-]/g, '_').slice(0, 80) || UNKNOWN_INSTANCE_ID;
  return `${SUBSCRIPTION_FILE_PREFIX}${recordKind}-${recordId}-${sender}.json`;
}

/** Legacy pre-sharing-v2 subscription filename — kept so we can clean up the
 *  old file when a peer upgrades and switches to the per-sender name. */
export function legacySubscriptionFilename({ recordKind, recordId }) {
  return `${SUBSCRIPTION_FILE_PREFIX}${recordKind}-${recordId}.json`;
}

/**
 * Filename a manifest lives under inside `<bucket>/manifests/`.
 *
 * For subscription manifests we use a deterministic name keyed on the
 * (recordKind, recordId, senderInstanceId) tuple so re-exports overwrite the
 * same file — bucket recipients see updates via chokidar `change` events, not
 * as accumulating new files. Including the senderInstanceId means two peers
 * each subscribed to the same record write to distinct files instead of
 * clobbering each other. For one-shot manifests we use the legacy
 * `<iso-ts>-<source>-<uuid>.json` naming so the watcher's add-order remains
 * lexicographically deterministic.
 */

export function manifestFilename(manifest) {
  if (manifest.kind === 'media-annotations') {
    return annotationManifestFilename(manifest.senderInstanceId);
  }
  if (manifest.subscription?.recordKind && manifest.subscription?.recordId) {
    return subscriptionFilename({
      recordKind: manifest.subscription.recordKind,
      recordId: manifest.subscription.recordId,
      senderInstanceId: manifest.senderInstanceId,
    });
  }
  const ts = manifest.createdAt.replace(/[:.]/g, '-');
  const senderSlug = (manifest.source || manifest.senderInstanceId || UNKNOWN_INSTANCE_ID)
    .toLowerCase().replace(/[^a-z0-9-]+/g, '-').replace(/^-+|-+$/g, '').slice(0, 40)
    || UNKNOWN_INSTANCE_ID;
  return `${ts}-${senderSlug}-${manifest.id}.json`;
}

export async function writeManifest(bucketPath, manifest) {
  await ensureDir(join(bucketPath, 'manifests'));
  const filename = manifestFilename(manifest);
  await atomicWrite(join(bucketPath, 'manifests', filename), manifest);
  return filename;
}

export async function readManifest(bucketPath, filename) {
  // Input-only transport document: unreadable manifests are skipped/deferred,
  // never merged back here. Exports rebuild manifests from authoritative records.
  return readJSONFile(join(bucketPath, 'manifests', filename), null, { logError: false });
}

/** List every manifest filename in a bucket, newest first by ISO-prefixed name. */
export async function listManifestFilenames(bucketPath) {
  const dir = join(bucketPath, 'manifests');
  const entries = await readdir(dir).catch(() => []);
  return entries.filter((f) => f.endsWith('.json')).sort().reverse();
}

/**
 * Default retention cap per bucket per local instance. A long-lived bucket
 * accumulates one manifest per share action, so capping owned manifests at
 * 500 keeps disk + cloud-sync chatter bounded while preserving enough recent
 * history for "did I share this?" forensics in `/sharing/:id/activity`.
 */
export const DEFAULT_MANIFEST_RETENTION = 500;

/**
 * Filenames currently being moved into `<bucket>/.archive/manifests/` by the
 * pruner. The watcher consults this set so its `unlink` handler can skip
 * `handleUnshare` for our own archive moves — those would otherwise reset
 * cursor entries and emit a misleading "peer unshared" socket event.
 *
 * Held for ~5s after each rename to outlast chokidar's awaitWriteFinish
 * debounce; the entry is unref'd from the event loop so it never blocks
 * shutdown.
 */
const pruningInFlight = new Map();

export function isManifestPruning(bucketId, filename) {
  return pruningInFlight.get(bucketId)?.has(filename) || false;
}

function markPruning(bucketId, filename) {
  let set = pruningInFlight.get(bucketId);
  if (!set) {
    set = new Set();
    pruningInFlight.set(bucketId, set);
  }
  set.add(filename);
}

function scheduleUnmark(bucketId, filename, delayMs = 5000) {
  const t = setTimeout(() => {
    const set = pruningInFlight.get(bucketId);
    if (!set) return;
    set.delete(filename);
    if (set.size === 0) pruningInFlight.delete(bucketId);
  }, delayMs);
  t.unref?.();
}

// Cap concurrent readManifest fd usage during classification — long-lived
// buckets can hold thousands of manifests, and `EMFILE: too many open files`
// is real on macOS defaults if we Promise.all the whole list.
const CLASSIFY_CONCURRENCY = 32;

/**
 * Archive older one-shot manifests authored by `localInstanceId` so a bucket
 * that lives long enough does not accumulate manifests indefinitely.
 *
 * Rules:
 *   - Subscription manifests (`sub-*.json`) are never archived — they own a
 *     deterministic filename per (recordKind, recordId) so older ones don't
 *     accumulate, and removing one would replicate as an unshare to peers.
 *   - Manifests authored by other peers are never archived — only the author
 *     of a manifest knows whether it's been superseded, and a cross-peer
 *     delete would replicate as an unshare on the originating peer.
 *   - Owned one-shot manifests in excess of `maxManifests` are moved (oldest
 *     first by ISO-prefixed filename) into `<bucket>/.archive/manifests/`.
 *
 * Returns `{ archived, kept, ownedTotal, skippedForeign, skippedReason }`.
 * When `localInstanceId` is missing or UNKNOWN_INSTANCE_ID, returns a noop
 * result with `skippedReason` set — we never blanket-archive without an
 * author check.
 */
export async function pruneBucketManifests(bucket, opts = {}) {
  const maxManifests = Number.isFinite(opts.maxManifests)
    ? Math.max(0, opts.maxManifests)
    : DEFAULT_MANIFEST_RETENTION;
  const localInstanceId = opts.localInstanceId;
  if (!localInstanceId || localInstanceId === UNKNOWN_INSTANCE_ID) {
    return { archived: 0, kept: 0, ownedTotal: 0, skippedForeign: 0, skippedReason: 'no-local-instance-id' };
  }
  const manifestsDir = join(bucket.path, 'manifests');
  const entries = await readdir(manifestsDir).catch(() => []);
  const candidates = entries
    .filter((f) => f.endsWith('.json') && !f.startsWith(SUBSCRIPTION_FILE_PREFIX) && !f.startsWith('annotations-'))
    .sort();
  // Fast path: if the total candidate count is at or below the cap, no read
  // of any manifest is necessary — owned count cannot exceed total.
  if (candidates.length <= maxManifests) {
    return { archived: 0, kept: candidates.length, ownedTotal: candidates.length, skippedForeign: 0, skippedReason: null };
  }
  // Classify in bounded-concurrency batches so a bucket with thousands of
  // manifests doesn't exhaust file descriptors.
  const owned = [];
  let skippedForeign = 0;
  for (let i = 0; i < candidates.length; i += CLASSIFY_CONCURRENCY) {
    const batch = candidates.slice(i, i + CLASSIFY_CONCURRENCY);
    const reads = await Promise.all(batch.map((filename) => readManifest(bucket.path, filename).catch(() => null)));
    for (let j = 0; j < batch.length; j++) {
      const m = reads[j];
      if (!m) continue;
      if (m.senderInstanceId === localInstanceId) owned.push(batch[j]);
      else skippedForeign += 1;
    }
  }
  if (owned.length <= maxManifests) {
    return { archived: 0, kept: owned.length, ownedTotal: owned.length, skippedForeign, skippedReason: null };
  }
  const archiveDir = join(bucket.path, '.archive', 'manifests');
  await ensureDir(archiveDir);
  const toArchive = owned.slice(0, owned.length - maxManifests);
  for (const filename of toArchive) markPruning(bucket.id, filename);
  const renames = await Promise.all(toArchive.map(async (filename) => {
    try {
      await rename(join(manifestsDir, filename), join(archiveDir, filename));
      return true;
    } catch (err) {
      console.log(`⚠️ sharing.manifest: archive failed for ${filename}: ${err.message}`);
      return false;
    } finally {
      scheduleUnmark(bucket.id, filename);
    }
  }));
  const archived = renames.filter(Boolean).length;
  return { archived, kept: maxManifests, ownedTotal: owned.length, skippedForeign, skippedReason: null };
}