Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

14.0 KB
Raw Blame
/**
 * Outgoing annotation manifest writer. One file per (bucket, instance),
 * rewritten on a 2s debounce after any local annotation change. Inbox-mode
 * buckets are skipped (not a fit for constant pings). Per-bucket scoping: an
 * annotation is only written into a bucket that already has the underlying
 * asset, so a private note can't leak into an unrelated share.
 */

import { join } from 'path';
import { readdir, stat } from 'fs/promises';
import { existsSync } from 'fs';
import { ensureDir, atomicWrite, readJSONFile } from '../../lib/fileUtils.js';
import { listBuckets } from './buckets.js';
import { buildManifest, writeManifest, annotationManifestFilename, readManifest, listManifestFilenames } from './manifest.js';
import { getProducedByVersion } from './version.js';
import { PORTOS_SCHEMA_VERSIONS } from '../../lib/schemaVersions.js';
import { getInstanceId, UNKNOWN_INSTANCE_ID } from '../instanceIdentity.js';
import { resolveBucketSourceName } from './annotationIdentity.js';
import { onLocalAnnotationChange, listLocalAuthorAnnotations } from '../mediaAnnotations.js';

const DEBOUNCE_MS = 2000;
// Retry delays after a failed bucket export (unmounted/unreachable folder);
// the last entry repeats until an export succeeds or a local change resets it.
const RETRY_BACKOFF_MS = [60_000, 300_000, 900_000];
let pendingTimer = null;
let retryTimer = null;
let retryAttempt = 0;
let installed = false;

/**
 * Cache of manifest-derived asset keys per bucket, invalidated by the
 * `manifests/` directory's mtime. Every manifest write goes through
 * `atomicWrite` (temp + rename), and every archive sweep uses `rename`, both
 * of which bump the directory mtime — so the mtime is the cheapest available
 * signal that "the set of manifests in this bucket has changed."
 *
 * Without the cache, every 2s annotation flush re-reads + re-parses every
 * manifest in every auto-merge bucket — fan-out is O(buckets × manifests)
 * per flush. With the cache the steady-state cost drops to one `stat` per
 * bucket.
 *
 * @type {Map<string, { manifestsMtimeMs: number | null, manifestKeys: Set<string> }>}
 */
const manifestKeysCache = new Map();

/**
 * Set of `${kind}:${filename}` keys for every asset referenced by any manifest
 * in the bucket. v2's content-addressed `assets/blobs/<hash>` paths don't
 * carry filenames, so the manifests are the only source-of-truth for which
 * user-facing filenames the bucket holds; the legacy `assets/{images,videos}/`
 * scan covers v1 buckets that never wrote a manifest's `hash` field.
 *
 * The manifests-side scan is cached per (bucket, manifests-dir mtime); the
 * legacy v1 dir scan is small and runs uncached. Callers must treat the
 * returned Set as read-only — the cache may hand back the same Set instance
 * to subsequent callers when the legacy fall-through adds nothing (the
 * common v2-only path), so mutating it would poison the next cache hit.
 */
async function listBucketAssetKeys(bucketPath) {
  const manifestsDir = join(bucketPath, 'manifests');
  // ENOENT (dir not created yet) is normal; null is a stable cache key.
  // Other errors (EACCES, EIO) are surfaced via a warning + uncached scan so
  // a real permission/IO problem is observable instead of being swallowed
  // into a phantom `unknown` mtime.
  let manifestsMtimeMs = null;
  let cacheable = true;
  const dirStat = await stat(manifestsDir).catch((err) => err);
  if (dirStat instanceof Error) {
    if (dirStat.code !== 'ENOENT') {
      console.error(`⚠️ sharing.annotations: stat(${manifestsDir}) failed code=${dirStat.code}: ${dirStat.message}`);
      cacheable = false;
    }
  } else {
    manifestsMtimeMs = dirStat.mtimeMs;
  }

  let manifestKeys;
  const cached = cacheable ? manifestKeysCache.get(bucketPath) : null;
  if (cached && cached.manifestsMtimeMs === manifestsMtimeMs) {
    manifestKeys = cached.manifestKeys;
  } else {
    manifestKeys = new Set();
    const filenames = await listManifestFilenames(bucketPath);
    await Promise.all(filenames.map(async (filename) => {
      const m = await readManifest(bucketPath, filename).catch(() => null);
      for (const ref of m?.assetRefs || []) {
        if (!ref?.ref || typeof ref.ref !== 'string') continue;
        const kind = ref.kind === 'video' ? 'video' : 'image';
        manifestKeys.add(`${kind}:${ref.ref}`);
      }
      for (const item of m?.collection?.items || []) {
        if (!item?.ref || typeof item.ref !== 'string') continue;
        const kind = item.kind === 'video' ? 'video' : 'image';
        manifestKeys.add(`${kind}:${item.ref}`);
      }
    }));
    if (cacheable) manifestKeysCache.set(bucketPath, { manifestsMtimeMs, manifestKeys });
  }

  // Layer on the legacy v1 dir scan (small; runs every call). The common
  // v2-native bucket has no `assets/{images,videos}/` so we return the
  // cached Set unmodified and avoid the per-call copy entirely.
  let legacyKeys = null;
  for (const [subdir, kind] of [['images', 'image'], ['videos', 'video']]) {
    const dir = join(bucketPath, 'assets', subdir);
    if (!existsSync(dir)) continue;
    const files = await readdir(dir).catch(() => []);
    for (const f of files) {
      if (f.endsWith('.metadata.json')) continue;
      if (!legacyKeys) legacyKeys = new Set();
      legacyKeys.add(`${kind}:${f}`);
    }
  }
  if (!legacyKeys) return manifestKeys;
  return new Set([...manifestKeys, ...legacyKeys]);
}

/** Test hook — drop listener-installed state and pending timers. */
export function __resetAnnotationsSyncForTests() {
  installed = false;
  if (pendingTimer) clearTimeout(pendingTimer);
  pendingTimer = null;
  clearRetry();
}

/** Test hook — reset the per-bucket manifest-keys cache between cases. */
export function __resetBucketAssetKeysCache() {
  manifestKeysCache.clear();
}

function isSameAnnotationSet(a, b) {
  const keysA = Object.keys(a).sort();
  const keysB = Object.keys(b).sort();
  if (keysA.length !== keysB.length) return false;
  return keysA.every((key, i) => key === keysB[i] && JSON.stringify(sortedEntry(a[key])) === JSON.stringify(sortedEntry(b[key])));
}

function sortedEntry(entry) {
  return entry && typeof entry === 'object'
    ? Object.fromEntries(Object.entries(entry).sort(([x], [y]) => (x < y ? -1 : 1)))
    : entry;
}

export async function exportAnnotationsToBucket(bucket, localAnnotations, senderInstanceId) {
  if (bucket.mode !== 'auto-merge') return { skipped: true, reason: 'not-auto-merge' };
  const recordDir = join(bucket.path, 'records', 'media-annotations');
  const recordId = senderInstanceId;
  const recordPath = join(recordDir, `${recordId}.json`);
  // Cheap early-out: no local annotations AND no prior record means this
  // bucket has nothing to publish and never did. Skip the manifest+asset-dir
  // scan that `listBucketAssetKeys` does — for a fresh install the listener
  // fires for every set/clear on every auto-merge bucket, and most of them
  // hit this branch.
  if (Object.keys(localAnnotations).length === 0 && !existsSync(recordPath)) {
    return { skipped: true, reason: 'no-annotations-for-bucket' };
  }
  const assetKeys = await listBucketAssetKeys(bucket.path);
  if (assetKeys.size === 0) return { skipped: true, reason: 'no-bucket-assets' };
  const filtered = {};
  for (const [key, entry] of Object.entries(localAnnotations)) {
    if (assetKeys.has(key)) filtered[key] = entry;
  }
  const sourceName = await resolveBucketSourceName(bucket);
  await ensureDir(recordDir);

  // Tombstone synthesis: peers only learn about deletions by seeing them. The
  // import-side merge (`mergePeerAnnotations`) iterates incoming keys and
  // treats `sanitizeAuthorEntry` returning null as "delete the peer's entry."
  // So we diff the previously-written record (for this bucket+instance) for
  // keys we used to publish but no longer do, and emit an explicit empty
  // tombstone for each. Without this, a local delete shows up as "the key is
  // simply absent from the next snapshot" — and peers retain the stale note
  // forever. Scoped per-bucket because each bucket's prior record is its own
  // file, so a key dropped from bucket A doesn't tombstone the same key on
  // bucket B (it may still be valid there).
  const tombstoneTs = new Date().toISOString();
  const prior = await readJSONFile(recordPath, null, { logError: false, strict: true });
  const priorKeys = prior && prior.annotations && typeof prior.annotations === 'object'
    ? Object.keys(prior.annotations)
    : [];
  for (const key of priorKeys) {
    if (filtered[key]) continue;
    // Keep an already-published tombstone verbatim so an unchanged state
    // compares equal below instead of being rewritten with a fresh timestamp.
    const priorEntry = prior.annotations[key];
    if (priorEntry && priorEntry.starred === false && priorEntry.note === '' && assetKeys.has(key)) {
      filtered[key] = priorEntry;
      continue;
    }
    // Only tombstone keys whose assets are still in this bucket — pruning a
    // key whose asset has been removed from the bucket would tell peers to
    // delete the note even though they may still hold the asset.
    if (!assetKeys.has(key)) continue;
    filtered[key] = { starred: false, note: '', updatedAt: tombstoneTs };
  }

  // Nothing to publish and nothing was ever published for this (bucket,
  // instance) — skip the record + manifest write so a local annotation edit
  // doesn't fan an empty record into every unrelated auto-merge bucket.
  if (Object.keys(filtered).length === 0 && priorKeys.length === 0) {
    return { skipped: true, reason: 'no-annotations-for-bucket' };
  }

  // Idempotent re-export: boot and retry flushes must not rewrite a manifest
  // whose content peers already hold. The record is written before the
  // manifest, so a missing manifest means a previous export died midway.
  if (prior && prior.annotations && isSameAnnotationSet(prior.annotations, filtered)
    && existsSync(join(bucket.path, 'manifests', annotationManifestFilename(senderInstanceId)))) {
    return { skipped: true, reason: 'unchanged' };
  }

  const record = {
    id: recordId,
    instanceId: senderInstanceId,
    authorName: sourceName,
    updatedAt: tombstoneTs,
    annotations: filtered,
  };
  await atomicWrite(recordPath, record);
  const producedByVersion = await getProducedByVersion();
  const manifest = buildManifest({
    kind: 'media-annotations',
    senderInstanceId,
    source: sourceName,
    sourceBio: null,
    recordIds: [recordId],
    assetRefs: [],
    bucketId: bucket.id,
    bucketName: bucket.name,
    producedByVersion,
    portosSchemaVersions: PORTOS_SCHEMA_VERSIONS,
  });
  const filename = await writeManifest(bucket.path, manifest);
  // The annotation manifest we just wrote bumps the manifests-dir mtime, but
  // its `assetRefs: []` means the bucket's asset-key set is unchanged. Roll
  // the cached mtime forward so the next flush still hits this bucket's
  // cache instead of re-parsing every other manifest in the dir.
  const cached = manifestKeysCache.get(bucket.path);
  if (cached) {
    const dirStat = await stat(join(bucket.path, 'manifests')).catch(() => null);
    if (dirStat) cached.manifestsMtimeMs = dirStat.mtimeMs;
  }
  return { skipped: false, filename, entryCount: Object.keys(filtered).length };
}

async function flushAll() {
  // pendingTimer is already cleared by scheduleFlush's setTimeout callback.
  // Each bucket resolves its own sourceName inside exportAnnotationsToBucket
  // so per-bucket displayNameOverride is honored (otherwise every bucket
  // would carry the global sharing display name in its annotation envelope).
  const [buckets, senderInstanceId, localAnnotations] = await Promise.all([
    listBuckets().catch(() => []),
    getInstanceId().catch(() => null),
    // A failed source read is not a deletion: abort before tombstone synthesis.
    listLocalAuthorAnnotations(),
  ]);
  if (!senderInstanceId || senderInstanceId === UNKNOWN_INSTANCE_ID) return true;
  const autoMerge = buckets.filter((b) => b.mode === 'auto-merge');
  if (autoMerge.length === 0) return true;
  let allOk = true;
  await Promise.all(autoMerge.map(async (bucket) => {
    const res = await exportAnnotationsToBucket(bucket, localAnnotations, senderInstanceId).catch((err) => {
      console.error(`⚠️ sharing.annotations: export to bucket=${bucket.name} failed: ${err.message}`);
      allOk = false;
      return null;
    });
    if (res && !res.skipped) {
      console.log(`📤 sharing.annotations: bucket=${bucket.name} wrote ${res.entryCount} annotation(s) (${res.filename})`);
    }
  }));
  return allOk;
}

function clearRetry() {
  if (retryTimer) clearTimeout(retryTimer);
  retryTimer = null;
  retryAttempt = 0;
}

function scheduleRetry() {
  if (retryTimer) return;
  const delay = RETRY_BACKOFF_MS[Math.min(retryAttempt, RETRY_BACKOFF_MS.length - 1)];
  retryAttempt += 1;
  retryTimer = setTimeout(() => {
    retryTimer = null;
    runFlush();
  }, delay);
  retryTimer.unref?.();
}

function runFlush() {
  return flushAll().then((ok) => {
    if (ok) clearRetry();
    else scheduleRetry();
  }).catch((err) => {
    console.error(`⚠️ sharing.annotations: flushAll failed: ${err.message}`);
    scheduleRetry();
  });
}

function scheduleFlush() {
  // A new local change supersedes any pending retry and resets its backoff.
  clearRetry();
  // True debounce: restart the timer on every call so a burst of rapid edits
  // collapses into one flush 2s after the LAST change. The previous
  // "if (pendingTimer) return" was a throttle — the first edit started the
  // timer and subsequent edits within the window were absorbed but the
  // flush still fired mid-burst (and the trailing edits then triggered
  // another flush a moment later).
  if (pendingTimer) clearTimeout(pendingTimer);
  pendingTimer = setTimeout(() => {
    pendingTimer = null;
    runFlush();
  }, DEBOUNCE_MS);
  pendingTimer.unref?.();
}

/** Install the local-annotation-change listener. Idempotent. Called once on boot. */
export function initAnnotationsSync() {
  if (installed) return;
  installed = true;
  onLocalAnnotationChange(() => scheduleFlush());
  // An edit made just before a restart (or during a failed export) never
  // reached the bucket; the export is idempotent so a boot flush is cheap.
  scheduleFlush();
}