Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
/**
* Media Collections
*
* User-named buckets for image filenames + video ids. Many-to-many: the same
* media item can live in any number of collections at once. Persisted via the
* per-record `createCollectionStore` layout under data/media-collections/ —
* one `{id}/index.json` per collection plus a type-level `index.json` stamping
* the storage `schemaVersion`. (Migrated from the legacy monolithic
* data/media-collections.json by scripts/migrations/059-split-media-collections.js.)
*
* An item is identified by `{ kind: 'image'|'video', ref: <filename|videoId> }`.
* A collection's `coverKey` is `null` (auto: newest item) or `"<kind>:<ref>"`
* to pin a specific item as the cover thumbnail.
*
* **Concurrency.** Every public mutator routes its read-modify-write through
* the store's per-record queue (`store().queueRecordWrite(id, fn)`), so two
* writes against the SAME collection serialize while writes against DIFFERENT
* collections run in parallel. This replaces the legacy single-tail file queue
* that serialized every unrelated write and rewrote the whole ~200 KB document
* per item. The hot render-filing paths (pipeline cover filer, Universe Builder
* completion hook, image-gen auto-file) all target deterministic `uc-`/`sc-`
* ids, so concurrent `addItem`s to the same bucket still serialize correctly.
*/
import { join } from 'path';
import { randomUUID } from 'crypto';
import { PATHS } from '../lib/fileUtils.js';
import { createCollectionStore } from '../lib/collectionStore.js';
import { ITEM_KIND, REF_MAX_LENGTH, itemKey, parseKey } from '../lib/mediaItemKey.js';
import { sanitizeOrigin } from '../lib/sharingOrigin.js';
import { mapWithConcurrency } from '../lib/mapWithConcurrency.js';
import { compareNewerWins, compareEarlierWins, parseTsMs } from '../lib/lwwTimestamp.js';
import { emitRecordUpdated, emitRecordDeleted, autoSubscribeRecordToAllPeers } from './sharing/recordEvents.js';
import {
maybeJournalBeforeOverwrite, setSyncBaseHash, contentHashForRecord, flushBaseHashes, deleteSyncBaseHash,
withBaseHashFlushBatch,
} from '../lib/conflictJournal.js';
import { codedError } from '../lib/codedError.js';
export const ERR_NOT_FOUND = 'NOT_FOUND';
export const ERR_DUPLICATE = 'DUPLICATE';
export const ERR_VALIDATION = 'VALIDATION_ERROR';
export const NAME_MAX_LENGTH = 80;
export const DESCRIPTION_MAX_LENGTH = 500;
export const ITEMS_MAX = 5000;
// Bounded fan-out for the multi-record snapshot-merge / GC-sweep loops. Distinct
// record ids ride distinct per-record write queues, so these are safe to run
// concurrently; the cap keeps a large snapshot from opening hundreds of file
// descriptors at once. (Same-id writes still serialize inside queueRecordWrite.)
const COLLECTION_WRITE_CONCURRENCY = 8;
// A collection id is a filesystem path segment (data/media-collections/<id>/).
// The store's idPattern allowlist rejects anything else, so a record whose id
// can't round-trip to its own dir is unrepresentable — `sanitizeCollection`
// drops it here so a malformed/peer-supplied id is skipped at the boundary
// rather than thrown deep inside `queueRecordWrite` (which would abort a whole
// sync batch). Single source of truth: also passed to `createCollectionStore`.
const COLLECTION_ID_PATTERN = /^[A-Za-z0-9_-]{1,128}$/;
// A malformed/overlong id can't name any stored collection (ids are the fixed
// path-segment allowlist above), so map it to NOT_FOUND — the same result the
// pre-split `listCollections` scan gave for an unknown id. Without this, the
// id-accepting write mutators would hand a bad id to `queueRecordWrite`, which
// throws a raw (uncoded) store error that the route mapper surfaces as a 500
// instead of a clean 404. Read paths (`loadOne`) already return null for a bad
// id, so they don't need this; the write paths throw before `loadOne` runs.
const assertCollectionId = (id) => {
if (typeof id !== 'string' || !COLLECTION_ID_PATTERN.test(id)) {
throw codedError(`Collection not found: ${id}`, ERR_NOT_FOUND);
}
};
export { REF_MAX_LENGTH, itemKey };
const sanitizeItem = (raw) => {
if (!raw || typeof raw !== 'object') return null;
if (!ITEM_KIND.has(raw.kind)) return null;
if (typeof raw.ref !== 'string') return null;
const ref = raw.ref.trim();
if (!ref || ref.length > REF_MAX_LENGTH) return null;
// The DELETE-item route key is `<kind>:<ref>` split on the first `:`. A
// ref that contains `:` would be unambiguously persisted but ambiguously
// addressable via the REST surface — reject it here so persisted data
// always round-trips.
if (ref.includes(':')) return null;
// Path-traversal defense in depth. Refs persist to wire-syncable state
// (peer pushes carry collections via `linkedCollection`, snapshot sync
// carries the whole file), and downstream code joins the ref onto a
// PATHS dir to hash / pull the asset. A peer-supplied ref like
// `../etc/passwd` would otherwise let one peer make another peer hash
// (and leak the hash of) arbitrary local files.
//
// Reject separators and parent-directory PATH SEGMENTS (exact `.` or
// `..` between slashes, or as the whole basename). Do NOT reject the
// substring `..` in general — gallery filenames like `my..render.png`
// are valid basenames that the rest of the system stores and serves,
// and dropping them would silently disappear existing collection items.
if (ref.includes('/') || ref.includes('\\')) return null;
if (ref === '.' || ref === '..') return null;
// A hand-edited or corrupted `addedAt` would feed NaN into the cover
// resolver's Date sort — replace anything unparseable with now().
const parsed = typeof raw.addedAt === 'string' ? Date.parse(raw.addedAt) : NaN;
const addedAt = Number.isFinite(parsed) ? raw.addedAt : new Date().toISOString();
const out = { kind: raw.kind, ref, addedAt };
const origin = sanitizeOrigin(raw.origin);
if (origin) out.origin = origin;
return out;
};
// Per-item removal records (#12433): `removedItems: { "<kind>:<ref>": removedAt }`.
// Item lists union on merge, so without a marker any peer still holding a removed
// item re-adds it. A copy of an item survives only when its `addedAt` is strictly
// newer than the removal for its key; a re-add supersedes the removal by carrying
// a newer `addedAt`, and the record itself is KEPT (a stale peer copy with the
// original `addedAt` must still lose). Records are pruned only by tombstone GC
// once every subscribed peer has had the chance to apply them
// (`pruneCollectionItemRemovals`). Omitted from the record when empty.
const sanitizeRemovedItems = (raw) => {
if (!raw || typeof raw !== 'object' || Array.isArray(raw)) return {};
const entries = [];
for (const [key, removedAt] of Object.entries(raw)) {
if (parseTsMs(removedAt) === null) continue;
const parsed = parseKey(key);
const item = parsed && sanitizeItem({ ...parsed, addedAt: removedAt });
if (!item || itemKey(item) !== key) continue;
entries.push([key, removedAt]);
}
// Deliberately uncapped: dropping an unacknowledged record would let a stale
// copy back in. There is at most one record per distinct key, and tombstone
// GC is what bounds them. Sorted keys keep the wire form and snapshot
// checksum byte-stable.
return Object.fromEntries(entries.sort(([a], [b]) => a.localeCompare(b)));
};
const isSupersededByRemoval = (item, removedItems) => {
const removedMs = parseTsMs(removedItems[itemKey(item)]);
return removedMs !== null && !(parseTsMs(item.addedAt) > removedMs);
};
// A local add/remove must order strictly after the opposite event it replaces,
// even when that event came from a peer whose clock runs ahead of ours.
const stampAfter = (ts) => new Date(Math.max(Date.now(), (parseTsMs(ts) ?? -Infinity) + 1)).toISOString();
const UNIVERSE_ID_MAX = 80;
const SERIES_ID_MAX = 80;
// Provenance stamp (#3311). `'auto'` = minted by a machine flow (Creative
// Director project, Writers Room work, universe/series render bucket);
// `'user'` = the person created it through the API. The grid uses it to sort
// auto-generated empties last and to badge the card.
//
// **LOCAL-ONLY.** `sanitizeRecordForWire` strips it, so provenance never
// crosses to a peer — see the rationale there (an emitted field would leave an
// upgraded peer permanently checksum-mismatched against an older one, and
// gating it behind a `schemaVersions` bump would pause all collection sync).
// Every install derives its own: the creators below stamp at mint time and
// `scripts/migrations/220-backfill-media-collection-source.js` classifies
// whatever was already on disk when the install upgraded.
//
// **Absent is a THIRD state, not a synonym for `'user'`.** A record written
// before this field shipped, or received from a peer after this install's
// migration ran, carries no stamp at all — the client falls back to its marker
// heuristic for those rather than mislabeling them user-made. So the sanitizer
// OMITS the key entirely rather than defaulting it (same posture as
// `sanitizeItem`'s optional `origin`).
export const COLLECTION_SOURCES = Object.freeze(['auto', 'user']);
const sanitizeSource = (raw) => (COLLECTION_SOURCES.includes(raw) ? raw : null);
const sanitizeCollection = (raw) => {
if (!raw || typeof raw !== 'object') return null;
// Reject ids the store can't persist (path-segment allowlist) so a peer-
// supplied / hand-edited bogus id is dropped here instead of throwing inside
// queueRecordWrite and aborting the rest of a merge batch.
if (typeof raw.id !== 'string' || !COLLECTION_ID_PATTERN.test(raw.id)) return null;
if (typeof raw.name !== 'string') return null;
const name = raw.name.trim().slice(0, NAME_MAX_LENGTH);
if (!name) return null;
const description = typeof raw.description === 'string'
? raw.description.trim().slice(0, DESCRIPTION_MAX_LENGTH)
: '';
const removedItems = sanitizeRemovedItems(raw.removedItems);
const seen = new Set();
const items = [];
if (Array.isArray(raw.items)) {
for (const it of raw.items) {
const s = sanitizeItem(it);
if (!s || isSupersededByRemoval(s, removedItems)) continue;
const key = itemKey(s);
if (seen.has(key)) continue;
seen.add(key);
items.push(s);
if (items.length >= ITEMS_MAX) break;
}
}
// Cover key is only meaningful when it points at an item that's in the
// collection; otherwise drop it back to auto so the UI doesn't render a
// dangling thumbnail reference.
const coverKey = typeof raw.coverKey === 'string' && seen.has(raw.coverKey)
? raw.coverKey
: null;
// Optional link to a universe — used by share-bucket subscriptions to
// know which universe-record's subscription should re-export when this
// collection's items change.
const universeId = typeof raw.universeId === 'string' && raw.universeId
? raw.universeId.slice(0, UNIVERSE_ID_MAX)
: null;
// Mutually exclusive with universeId on read — a hand-edited record
// carrying both is resolved to the universe link (older, established
// relationship) so a single collection can't drive two subscriptions.
const seriesId = !universeId && typeof raw.seriesId === 'string' && raw.seriesId
? raw.seriesId.slice(0, SERIES_ID_MAX)
: null;
const createdAt = typeof raw.createdAt === 'string' ? raw.createdAt : new Date().toISOString();
const updatedAt = typeof raw.updatedAt === 'string' ? raw.updatedAt : createdAt;
const deleted = raw.deleted === true;
// When a tombstone is missing its explicit deletedAt, fall back to updatedAt
// (the most-recent timestamp we have) rather than createdAt — the deletion
// happened at or after the last edit, so createdAt would make the tombstone
// look far older than it is and skew LWW merges + the GC cutoff window.
const deletedAt = deleted && typeof raw.deletedAt === 'string' ? raw.deletedAt : (deleted ? updatedAt : null);
const source = sanitizeSource(raw.source);
return {
id: raw.id,
name,
description,
coverKey,
universeId,
seriesId,
...(source ? { source } : {}),
items,
...(Object.keys(removedItems).length ? { removedItems } : {}),
createdAt,
updatedAt,
deleted,
deletedAt,
};
};
// Storage-layout (type-level) schema version stamped on
// data/media-collections/index.json. Distinct from the wire schemaVersion in
// server/lib/schemaVersions.js (the on-disk split doesn't change the wire
// payload shape, so that stays at 1). Bump only on a future layout change.
const TYPE_SCHEMA_VERSION = 1;
// Lazy store getter — PATHS.data may not be available at module-load time
// (tests swap it through a Proxy mock so different cases get different temp
// roots). Mirrors universeBuilder.js's `store()` pattern: re-create when the
// resolved dir changes so a remapped data root takes effect.
let _store = null;
const store = () => {
const dir = join(PATHS.data, 'media-collections');
if (_store && _store.dir === dir) return _store;
_store = createCollectionStore({
dir,
type: 'mediaCollections',
schemaVersion: TYPE_SCHEMA_VERSION,
// Same allowlist `sanitizeCollection` enforces, so the store and the
// sanitizer agree on which ids are representable (no drift).
idPattern: COLLECTION_ID_PATTERN,
// Runs on every loadOne/loadAll. Tombstones (deleted:true) survive
// sanitization so the mutators and merge path can see them.
sanitizeRecord: sanitizeCollection,
});
return _store;
};
// Exposed for the boot-time verifier in server/index.js.
export const mediaCollectionStore = () => store();
export async function listCollections({ includeDeleted = false } = {}) {
const all = await store().loadAll();
if (includeDeleted) return all;
return all.filter((c) => c.deleted !== true);
}
/**
* Every collection id on disk (live AND tombstoned) — a directory listing, no
* record hydration. Backs tombstoneGc's ALL_ID_LISTERS.mediaCollection (the
* orphan peer-subscription sweep); LIVE_ID_LISTERS keeps `listCollections()`
* because `deleted` lives inside the record file, so there is no cheaper live
* projection for this file-backed store the way the DB-backed kinds have one.
*/
export async function listCollectionIds() {
return store().listIds();
}
export async function getCollection(id, { includeDeleted = false } = {}) {
const c = await store().loadOne(id);
if (!c) throw codedError(`Collection not found: ${id}`, ERR_NOT_FOUND);
if (c.deleted === true && !includeDeleted) throw codedError(`Collection not found: ${id}`, ERR_NOT_FOUND);
return c;
}
// Announce a newly-created collection to the per-record peer-sync pipeline:
// emit the 'updated' event so any existing subscription pushes it, AND
// auto-subscribe every mediaCollections-enabled peer so brand-new collections
// (and their later tombstones) propagate even when that peer has universe/series
// sync disabled. Goes through the recordEvents subscription adapter — peerSync
// imports mergeMediaCollectionsFromSync from here, so importing it back (even
// dynamically) would close a cycle. Call ONLY when a brand-new record was
// persisted — never on a find-existing hit, or every render would re-announce
// and churn the pipeline.
const announceNewCollection = (id) => {
emitRecordUpdated('mediaCollection', id);
autoSubscribeRecordToAllPeers('mediaCollection', id).catch(() => {});
};
// `source` defaults to 'user' because the HTTP POST route is the only caller
// that doesn't pass it — a person clicking "New collection". Machine flows
// (Creative Director, Writers Room) pass `source: 'auto'` explicitly. The route
// schema doesn't accept `source`, so a client can't claim either stamp.
export async function createCollection({ name, description = '', source = 'user' }) {
// Service-layer guards mirror sanitizeCollection so a direct caller
// (tests, future internal usage) can't persist a record that the next
// listCollections() read would silently drop.
const trimmedName = typeof name === 'string' ? name.trim() : '';
if (!trimmedName || trimmedName.length > NAME_MAX_LENGTH) {
throw codedError('Collection name is required (1..' + NAME_MAX_LENGTH + ' chars)', ERR_VALIDATION);
}
if (!COLLECTION_SOURCES.includes(source)) {
throw codedError(`source must be one of ${COLLECTION_SOURCES.join('|')}`, ERR_VALIDATION);
}
const trimmedDescription = typeof description === 'string'
? description.trim().slice(0, DESCRIPTION_MAX_LENGTH)
: '';
const id = randomUUID();
const created = await store().queueRecordWrite(id, async () => {
const now = new Date().toISOString();
const next = {
id,
name: trimmedName,
description: trimmedDescription,
coverKey: null,
source,
items: [],
createdAt: now,
updatedAt: now,
};
await store().saveOneNow(id, next);
return next;
});
announceNewCollection(created.id);
return created;
}
// Find an existing collection by case-insensitive trimmed name, else create
// a fresh one. The non-universe-aware upsert — used by user-driven flows
// where the bucket is identified by visible name (e.g. tests, ad-hoc CLI
// scripts).
//
// **Universe-owned paths must use `findOrCreateUniverseCollection`.**
// This helper is name-first and will return any same-name collection,
// including one already linked to a different universe, so it can route
// renders into the wrong bucket when two universes share a display name.
// The legacy `universeId` parameter remains for callers that need the
// best-effort backfill of an unlinked legacy bucket, but new code should
// prefer the universeId-first helper.
//
// Records this mints are stamped `source: 'auto'` (#3311) like the other
// provisioners: reaching this helper means a script or hook asked for a bucket,
// which is not the same as a person creating one in the UI (that path is
// `createCollection`, which defaults to `'user'`).
export async function findOrCreateCollectionByName({ name, description = '', universeId = null }) {
const trimmed = typeof name === 'string' ? name.trim() : '';
if (!trimmed || trimmed.length > NAME_MAX_LENGTH) {
throw codedError('Collection name is required (1..' + NAME_MAX_LENGTH + ' chars)', ERR_VALIDATION);
}
const trimmedDescription = typeof description === 'string'
? description.trim().slice(0, DESCRIPTION_MAX_LENGTH)
: '';
// Normalize the universe id (trim whitespace, then cap length) so the
// presence guard and the stored/derived value operate on the SAME string —
// a padded id can't yield `uc- u1 ` while the guard saw a non-empty value.
// Whitespace-only is treated as "no universe". Keeps the derived deterministic
// id under the peer-sync id length cap.
const normalizedUniverseId = typeof universeId === 'string' && universeId.trim()
? universeId.trim().slice(0, UNIVERSE_ID_MAX)
: null;
// Deterministic id when universe-linked (cross-machine convergence); a fresh
// random id otherwise. Queue on whichever id this call would create/adopt so
// two concurrent calls for the same universe serialize on the same record.
// NOTE: the no-universe branch queues on a random id, so two concurrent calls
// for the same NAME do NOT serialize and could both create a same-named
// bucket. Acceptable: this name-first helper has no production callers (it's
// the ad-hoc/CLI path); universe/series filing uses the deterministic-id
// helpers, which converge. A hot caller should derive a stable queue key.
const canonId = linkedCollectionId({ universeId: normalizedUniverseId });
const queueId = canonId || randomUUID();
let createdId = null;
const result = await store().queueRecordWrite(queueId, async () => {
const all = await store().loadAll();
const needle = trimmed.toLowerCase();
// Do not match (or reuse) tombstoned records — a deleted collection
// should not be resurrected by name.
const existing = all.find((c) => !c.deleted && c.name.toLowerCase() === needle);
if (existing) {
if (normalizedUniverseId && !existing.universeId) {
// Lazy backfill so legacy "Universe: <name>" collections gain the link
// the first time a universe-builder render references them. Adopt the
// DETERMINISTIC id at the same time — keeping the old random id would
// leave a universe-linked collection that can't converge across peers
// (the very bug deterministic ids fix).
//
// If the universe ALREADY has its canonical collection, don't promote a
// same-named orphan over it (that would clobber the canonical record's
// items) — return the canonical one; the orphan is a duplicate the merge
// / migration 038 reconciles.
const liveCanonical = all.find((c) => c.id === canonId && !c.deleted);
if (liveCanonical) return liveCanonical;
// No live canonical — rename the orphan to the deterministic id,
// reclaiming a tombstone at that id if one exists (saveOneNow overwrites
// the canonId record), then drop the old random-id record.
const linked = { ...existing, id: canonId, universeId: normalizedUniverseId, updatedAt: new Date().toISOString() };
await store().saveOneNow(canonId, linked);
if (canonId !== existing.id) {
await store().deleteOneNow(existing.id);
createdId = canonId; // announce the newly-linked record
}
return linked;
}
return existing;
}
const now = new Date().toISOString();
const next = {
id: queueId,
name: trimmed,
description: trimmedDescription,
coverKey: null,
universeId: normalizedUniverseId,
// Programmatic provisioner — see the docstring above (#3311).
source: 'auto',
items: [],
createdAt: now,
updatedAt: now,
};
createdId = next.id;
// saveOneNow overwrites any tombstone already at this deterministic id
// (re-creating a previously-deleted universe/series collection).
await store().saveOneNow(next.id, next);
return next;
});
if (createdId) announceNewCollection(createdId);
return result;
}
// Naming convention for the auto-managed universe collection. Single source
// of truth so the upsert path, the rename cascade (updateUniverse), and any
// caller that needs to display the name stay aligned. Truncated to
// NAME_MAX_LENGTH because long universe names would otherwise overflow
// sanitizeCollection's slice and silently mismatch.
export const universeCollectionNameFor = (universeName) =>
`Universe: ${typeof universeName === 'string' ? universeName : ''}`.slice(0, NAME_MAX_LENGTH);
// Deterministic id for a universe/series-LINKED collection. Every federated
// machine derives the SAME id from the same universe/series id — so per-record
// collection sync (which keys on `id`) treats both machines' copies as ONE
// record and converges, instead of each minting a random UUID and duplicating
// the collection on every peer. Standalone (unlinked) collections keep a random
// id — they have no stable cross-machine identity. Migration 038 canonicalizes
// existing linked-collection ids to this scheme. Mirror any change here in that
// migration. `uc-`/`sc-` prefixes keep linked ids visually distinct from the
// random UUIDs of standalone collections. The owner id is sliced to the same cap
// storage uses so the derived id is bounded by construction — callers can pass a
// raw owner id without first slicing it and still converge on the canonical id
// (and stay under the peer-sync id length cap).
export const linkedCollectionId = ({ universeId = null, seriesId = null } = {}) => {
if (universeId) return `uc-${String(universeId).slice(0, UNIVERSE_ID_MAX)}`;
if (seriesId) return `sc-${String(seriesId).slice(0, SERIES_ID_MAX)}`;
return null;
};
// Resolve the live collection linked to `rawValue` on `field` ('universeId' or
// 'seriesId'). Field-parameterized like clearOwnerLink / renameOwnerLinked so
// the universe and series variants can't drift.
//
// Fast path: any collection provisioned after migration 038 lives at the
// deterministic `uc-`/`sc-<id>` id, so one loadOne resolves the warm case in
// O(1) — the hot path peerSync drives on every outbound push. Only fall
// through to the full scan when that record is absent, tombstoned, or its
// stamp doesn't match — that's a legacy/unconverged collection carrying the
// link at a random id. Tombstones are excluded so "deleted" reads as gone
// (matching listCollections and the upsert provisioners' adopt rule). Returns
// null when nothing is linked; callers provision via findOrCreate*.
async function findLinkedCollection(field, rawValue, maxLen) {
if (typeof rawValue !== 'string' || !rawValue) return null;
const needle = rawValue.slice(0, maxLen);
const canon = await store().loadOne(linkedCollectionId({ [field]: needle }));
if (canon && !canon.deleted && canon[field] === needle) return canon;
const all = await store().loadAll();
return all.find((c) => !c.deleted && c[field] === needle) || null;
}
// Look up an existing collection by its universeId stamp. Returns null if no
// collection has ever been linked to this universe — callers fall back to
// `findOrCreateUniverseCollection` (the universeId-first upsert) to provision
// on first use. Do NOT fall back to `findOrCreateCollectionByName` here —
// that path is name-first and would adopt a same-named foreign-universe
// bucket. The argument is sliced to the same limit storage uses so passing
// an overlong id (e.g. from a malformed share manifest) matches whatever
// the upsert helper would have persisted.
export async function findCollectionByUniverseId(universeId) {
return findLinkedCollection('universeId', universeId, UNIVERSE_ID_MAX);
}
/**
* Atomic universeId-first upsert for a universe's auto-managed collection.
*
* Resolution order (serialized on the deterministic id's per-record queue):
* 1. universeId stamp wins. Returned as-is — the caller's `universeName`
* can be stale (it was a snapshot taken before this call entered the
* queue), so reconciling the name here could ping-pong a fresh
* cascade rename back to an old name. `renameCollectionForUniverse`
* is the canonical path for name changes; if its best-effort cascade
* fails, the surviving stale name is acceptable until the user
* retriggers it (the lock blocks `updateCollection` but the cascade
* itself is unrestricted).
* 2. Otherwise a fresh collection is created and stamped with `universeId`,
* even when the canonical name is already taken by another collection
* (a different universe's bucket, OR a previously-unlinked orphan from
* a deleted universe). Adopting either would silently mix renders
* across universes; two same-named collections is the user-correctable
* case.
*
* Callers (pipeline cover filer, Universe Builder render route) all funnel
* through this so the universe → collection identity stays consistent across
* both auto-filing and explicit-render paths.
*/
export async function findOrCreateUniverseCollection({ universeId, universeName, description = '' }) {
if (!universeId || typeof universeId !== 'string') {
throw codedError('universeId is required', ERR_VALIDATION);
}
if (typeof universeName !== 'string' || !universeName.trim()) {
throw codedError('universeName is required', ERR_VALIDATION);
}
// Normalize the universeId once so lookup and storage both key on the
// same string. Without this, an overlong id (e.g. from a malformed
// share manifest) would not match the row it just created — the lookup
// compares the raw value but persistence sliced — and retries would
// pile up duplicate stamped collections.
const normalizedUniverseId = universeId.slice(0, UNIVERSE_ID_MAX);
const desiredName = universeCollectionNameFor(universeName);
const trimmedDescription = typeof description === 'string'
? description.trim().slice(0, DESCRIPTION_MAX_LENGTH)
: '';
// Deterministic id so this universe's collection has the SAME id on every
// peer (per-record sync keys on id) — see linkedCollectionId. Queue on it so
// two concurrent provisions for the same universe serialize.
const id = linkedCollectionId({ universeId: normalizedUniverseId });
let createdId = null;
const result = await store().queueRecordWrite(id, async () => {
// Resolve any existing linked collection via the deterministic-id fast path
// (warm) with a full-scan fallback for a legacy unconverged record at a
// random id — never adopting a tombstone (deleted means gone, re-create
// below). Same lookup the read-only finder uses, so they can't drift.
const linked = await findLinkedCollection('universeId', normalizedUniverseId, UNIVERSE_ID_MAX);
if (linked) return linked;
// No universeId match — always create fresh. The runtime intentionally
// does NOT adopt a same-named unlinked collection here: it can't tell
// a true pre-link legacy bucket apart from a post-`deleteUniverse`
// orphan, and adopting the orphan would silently mix the deleted
// universe's renders into the new same-named universe.
//
// Upgrade path for pre-link installs: migration 021
// (`scripts/migrations/021-link-orphan-universe-collections.js`) does
// a one-shot name-match link at boot — safe because at install/upgrade
// time the orphan case doesn't exist yet. New collisions after the
// migration belong to "ambiguous, leave it" rather than "auto-adopt."
const now = new Date().toISOString();
const next = {
id,
name: desiredName,
description: trimmedDescription,
coverKey: null,
universeId: normalizedUniverseId,
// Machine-provisioned render bucket — never a user-made collection (#3311).
source: 'auto',
items: [],
createdAt: now,
updatedAt: now,
};
createdId = next.id;
// saveOneNow overwrites any tombstone already at this deterministic id
// (re-creating a previously-deleted universe collection).
await store().saveOneNow(next.id, next);
return next;
});
// Announce only when a NEW record was persisted (not a find-existing hit), so
// a mediaCollections-enabled peer receives universe-linked collections even
// with universe sync off.
if (createdId) announceNewCollection(createdId);
return result;
}
// Series-side mirror of the universe collection helpers above
// (`universeCollectionNameFor`, `findCollectionByUniverseId`,
// `findOrCreateUniverseCollection`, `unlinkCollectionsForUniverse`,
// `renameCollectionForUniverse`). Same resolution-order + orphan-avoidance
// rules — see those functions' docstrings for the rationale.
export const seriesCollectionNameFor = (seriesName) =>
`Series: ${typeof seriesName === 'string' ? seriesName : ''}`.slice(0, NAME_MAX_LENGTH);
export async function findCollectionBySeriesId(seriesId) {
return findLinkedCollection('seriesId', seriesId, SERIES_ID_MAX);
}
export async function findOrCreateSeriesCollection({ seriesId, seriesName, description = '' }) {
if (!seriesId || typeof seriesId !== 'string') {
throw codedError('seriesId is required', ERR_VALIDATION);
}
if (typeof seriesName !== 'string' || !seriesName.trim()) {
throw codedError('seriesName is required', ERR_VALIDATION);
}
const normalizedSeriesId = seriesId.slice(0, SERIES_ID_MAX);
const desiredName = seriesCollectionNameFor(seriesName);
const trimmedDescription = typeof description === 'string'
? description.trim().slice(0, DESCRIPTION_MAX_LENGTH)
: '';
// Deterministic id — same series collection id on every peer (see linkedCollectionId).
const id = linkedCollectionId({ seriesId: normalizedSeriesId });
let createdId = null;
const result = await store().queueRecordWrite(id, async () => {
// Deterministic-id fast path + full-scan fallback for a legacy unconverged
// record; never adopts a tombstone. Shared with the read-only finder.
const linked = await findLinkedCollection('seriesId', normalizedSeriesId, SERIES_ID_MAX);
if (linked) return linked;
const now = new Date().toISOString();
const next = {
id,
name: desiredName,
description: trimmedDescription,
coverKey: null,
seriesId: normalizedSeriesId,
// Machine-provisioned render bucket — never a user-made collection (#3311).
source: 'auto',
items: [],
createdAt: now,
updatedAt: now,
};
createdId = next.id;
// saveOneNow overwrites any tombstone already at this deterministic id
// (re-creating a previously-deleted series collection).
await store().saveOneNow(next.id, next);
return next;
});
if (createdId) announceNewCollection(createdId);
return result;
}
// A music video's own collection (one per project): every still and clip
// generated for it, kept findable after a shot drops them. The id is derived
// from the project id, so the same project converges on one collection on
// every peer and no owner stamp is needed.
const MUSIC_VIDEO_ID_MAX = 100;
export const musicVideoCollectionId = (projectId) => `mvc-${String(projectId).slice(0, MUSIC_VIDEO_ID_MAX)}`;
export const musicVideoCollectionNameFor = (projectName) =>
`Music video: ${typeof projectName === 'string' && projectName.trim() ? projectName.trim() : 'Untitled'}`.slice(0, NAME_MAX_LENGTH);
export async function findOrCreateMusicVideoCollection({ projectId, projectName }) {
if (!projectId || typeof projectId !== 'string') throw codedError('projectId is required', ERR_VALIDATION);
const id = musicVideoCollectionId(projectId);
assertCollectionId(id);
let createdId = null;
const result = await store().queueRecordWrite(id, async () => {
const current = await store().loadOne(id);
if (current && !current.deleted) return current;
const now = new Date().toISOString();
const next = {
id,
name: musicVideoCollectionNameFor(projectName),
description: 'Every still and clip generated for this music video.',
coverKey: null,
source: 'auto',
items: [],
createdAt: now,
updatedAt: now,
};
createdId = id;
await store().saveOneNow(id, next);
return next;
});
if (createdId) announceNewCollection(createdId);
return result;
}
// Both cascade families below — unlink (clear the owner link) and rename
// (cascade the owner's display name) — share the same shape: scan all records
// by their owner stamp, then per-match queue a load→re-check→save. The
// re-check inside the queue (`cur[field] !== needle`) is the concurrency-safety
// invariant now that there's no single file lock spanning the whole sweep —
// keep it in these two helpers so the universe/series variants can't drift.
// Clear an owner-link field (`universeId` / `seriesId`) on every collection
// stamped with `needle`. Returns the cleared ids. Deliberately does NOT bump
// `updatedAt` — see `unlinkCollectionsForUniverse`'s note.
async function clearOwnerLink(field, rawValue, maxLen) {
if (typeof rawValue !== 'string' || !rawValue) return [];
const needle = rawValue.slice(0, maxLen);
const matches = (await store().loadAll()).filter((c) => c[field] === needle);
const clearedIds = [];
for (const c of matches) {
await store().queueRecordWrite(c.id, async () => {
const cur = await store().loadOne(c.id);
if (!cur || cur[field] !== needle) return;
await store().saveOneNow(c.id, { ...cur, [field]: null });
});
clearedIds.push(c.id);
}
return clearedIds;
}
// Cascade an owner rename onto EVERY collection stamped with `needle` (not just
// the first match — hand-edited state or a pre-serialization race could leave
// duplicate linked rows that would otherwise stay rename-locked under the stale
// name). Returns the first updated collection (back-compat).
async function renameOwnerLinked(field, rawValue, maxLen, nameFor, newOwnerName) {
if (typeof rawValue !== 'string' || !rawValue) return null;
const needle = rawValue.slice(0, maxLen);
const matches = (await store().loadAll()).filter((c) => c[field] === needle);
if (!matches.length) return null;
const desired = nameFor(newOwnerName);
if (!desired) return matches[0];
const now = new Date().toISOString();
let first = null;
for (const c of matches) {
const updated = await store().queueRecordWrite(c.id, async () => {
const cur = await store().loadOne(c.id);
if (!cur || cur[field] !== needle || cur.name === desired) return cur;
const nextRec = { ...cur, name: desired, updatedAt: now };
await store().saveOneNow(c.id, nextRec);
return nextRec;
});
if (first === null) first = updated;
}
return first;
}
// Series mirror of `unlinkCollectionsForUniverse` — same cascade semantics,
// same reason for preserving `updatedAt`. A series merge tombstones the loser
// series-collection and the loser series; the series-tombstone cascade must not
// bump the collection past its own tombstone.
export const unlinkCollectionsForSeries = (seriesId) =>
clearOwnerLink('seriesId', seriesId, SERIES_ID_MAX);
export const renameCollectionForSeries = (seriesId, newSeriesName) =>
renameOwnerLinked('seriesId', seriesId, SERIES_ID_MAX, seriesCollectionNameFor, newSeriesName);
// Clear the `universeId` link on any collection bound to this universe.
// Used by deleteUniverse to release the rename-lock so the orphaned bucket
// becomes a normal user-owned collection (renamable, deletable, etc.). The
// items themselves are preserved — the user may still want the renders.
// Returns the list of unlinked collection ids (empty when none matched).
//
// Deliberately does NOT bump `updatedAt`. This is a CASCADED side-effect of
// the owner universe's deletion, not a user edit — severing the link doesn't
// change the collection's content (items are untouched). Advancing the LWW
// clock here would make the unlinked bucket look "freshly edited" and out-race
// the collection's OWN tombstone on a peer: a universe merge tombstones the
// loser auto-collection (older `updatedAt`) AND tombstones the universe, whose
// cascade lands first on the receiver. A bump here would defeat the (older)
// collection tombstone under pure LWW and strand a live duplicate on the peer.
// Items always union on merge (`mergeCollectionItems`), so preserving the
// timestamp can't lose renders.
export const unlinkCollectionsForUniverse = (universeId) =>
clearOwnerLink('universeId', universeId, UNIVERSE_ID_MAX);
// Cascade a universe rename to its linked collection(s). Skips the rename-lock
// guard in updateCollection by writing the name directly — the lock exists to
// block user-driven renames, not system-driven cascades from the universe
// rename itself.
export const renameCollectionForUniverse = (universeId, newUniverseName) =>
renameOwnerLinked('universeId', universeId, UNIVERSE_ID_MAX, universeCollectionNameFor, newUniverseName);
// The `updateCollection` patch keys that peers can actually observe. Anything
// outside this list is local-only and must not advance `updatedAt` (the LWW
// clock every peer compares against) — see the `source` note below.
const WIRE_VISIBLE_PATCH_FIELDS = ['name', 'description', 'coverKey'];
export async function updateCollection(id, patch) {
assertCollectionId(id);
const merged = await store().queueRecordWrite(id, async () => {
const cur = await store().loadOne(id);
if (!cur) throw codedError(`Collection not found: ${id}`, ERR_NOT_FOUND);
if (cur.deleted === true) throw codedError(`Collection not found: ${id}`, ERR_NOT_FOUND);
// Product/UI constraint: universe-linked collections own their visible
// name. The name is the user-facing identity of a universe's bucket, so
// renaming it independent of the universe is confusing — the supported
// workflow is renaming the universe (which cascades down via
// renameCollectionForUniverse). Routing is by `universeId` regardless;
// this lock exists to keep what the user sees consistent with the
// universe's name, not to prevent routing forks.
if ('name' in patch && cur.universeId && patch.name !== cur.name) {
throw codedError(
'This collection is linked to a Universe — rename the universe to rename it.',
ERR_VALIDATION,
);
}
if ('name' in patch && cur.seriesId && patch.name !== cur.name) {
throw codedError(
'This collection is linked to a Series — rename the series to rename it.',
ERR_VALIDATION,
);
}
// Cover key validation is deferred to sanitizeCollection on read, but we
// also reject up-front so the API gives a clear error rather than silently
// dropping the cover.
if (patch.coverKey != null && !cur.items.find((it) => itemKey(it) === patch.coverKey)) {
throw codedError('coverKey must reference an item in this collection', ERR_VALIDATION);
}
const merged = {
...cur,
...('name' in patch ? { name: patch.name } : {}),
...('description' in patch ? { description: patch.description } : {}),
...('coverKey' in patch ? { coverKey: patch.coverKey } : {}),
// Provenance is correctable (#3311). The migration classifies legacy
// records from markers a user could in principle have typed themselves
// (a description starting `Auto-created for project …`), so "the stamp is
// wrong and I can't fix it" must not be a dead end. Local-only like the
// rest of the field — this edit never leaves the install.
...('source' in patch ? { source: patch.source } : {}),
// A source-only edit changes nothing a peer can see (the field is
// stripped from the wire), so it must NOT advance the shared LWW clock —
// same reasoning as migration 220 leaving `updatedAt` alone. Otherwise
// correcting a badge locally would out-race a peer's real edit.
updatedAt: WIRE_VISIBLE_PATCH_FIELDS.some((k) => k in patch)
? new Date().toISOString()
: cur.updatedAt,
};
await store().saveOneNow(id, merged);
return merged;
});
// A standalone collection (no universe/series link) reaches peers ONLY via a
// direct per-record mediaCollection subscription — without this emit a
// rename/description/cover edit never propagates. For linked collections the
// universe/series emit nudges the share-bucket re-export. Emit outside the
// serialized critical section so subscribers' own reads don't deadlock the tail.
emitRecordUpdated('mediaCollection', merged.id);
if (merged.universeId) emitRecordUpdated('universe', merged.universeId);
if (merged.seriesId) emitRecordUpdated('series', merged.seriesId);
return merged;
}
export async function deleteCollection(id) {
assertCollectionId(id);
const { universeId: deletedUniverseId, seriesId: deletedSeriesId, alreadyDeleted } = await store().queueRecordWrite(id, async () => {
const target = await store().loadOne(id);
if (!target) throw codedError(`Collection not found: ${id}`, ERR_NOT_FOUND);
// Idempotent on an already-tombstoned record: re-stamping deletedAt/
// updatedAt would make an old tombstone look "newer" to a peer's LWW (and
// re-emitting churns the sync pipeline). Return without rewriting or
// re-emitting.
if (target.deleted === true) return { universeId: null, seriesId: null, alreadyDeleted: true };
const now = new Date().toISOString();
// Soft-delete: keep the record on disk as a tombstone (NOT deleteOne) so it
// still syncs to peers as a deletion. Clear coverKey/items too — with items
// emptied the cover would otherwise dangle and leak into the wire payload.
// Item removal records go too: a tombstone holds no items to protect.
const { removedItems: _removedItems, ...live } = target;
const tombstone = { ...live, deleted: true, deletedAt: now, updatedAt: now, items: [], coverKey: null, universeId: null, seriesId: null };
await store().saveOneNow(id, tombstone);
return { universeId: target.universeId || null, seriesId: target.seriesId || null, alreadyDeleted: false };
});
if (alreadyDeleted) return { id };
// Mirror addItem/removeItem/bulkUpdateCollectionItems — universe-linked
// shares need to know membership changed so the subscriber doesn't keep
// publishing the deleted collection's contents until an unrelated edit
// fires. Emit outside the serialized critical section so subscribers'
// own reads don't deadlock the tail.
if (deletedUniverseId) emitRecordUpdated('universe', deletedUniverseId);
if (deletedSeriesId) emitRecordUpdated('series', deletedSeriesId);
emitRecordDeleted('mediaCollection', id);
return { id };
}
// Mirror sanitizeItem so direct callers (addItem + bulkUpdateCollectionItems)
// can't write a record that the next listCollections() would drop (silent add).
// Returns a normalized `{ kind, ref }` so callers don't re-trim.
const validateItemInput = (item) => {
if (!item || !ITEM_KIND.has(item.kind)) {
throw codedError('item.kind must be "image" or "video"', ERR_VALIDATION);
}
const ref = typeof item.ref === 'string' ? item.ref.trim() : '';
if (!ref || ref.length > REF_MAX_LENGTH || ref.includes(':')) {
throw codedError('item.ref invalid (empty, too long, or contains ":")', ERR_VALIDATION);
}
// Mirror sanitizeItem's path-traversal rejection so the write path can't
// persist a ref that the next listCollections() read would silently drop
// (which would also churn coverKey/updatedAt for the dangling-cover guard).
// Reject separators and exact `.`/`..` segments only — `..` inside a
// basename (`my..render.png`) is a legitimate gallery filename. Must
// mirror sanitizeItem exactly or local addItem rejects refs that peer
// sync would accept (and vice versa).
if (ref.includes('/') || ref.includes('\\')) {
throw codedError('item.ref invalid (contains path separators)', ERR_VALIDATION);
}
if (ref === '.' || ref === '..') {
throw codedError('item.ref invalid (parent-directory segment)', ERR_VALIDATION);
}
return { kind: item.kind, ref };
};
export async function addItem(id, item) {
const { kind, ref } = validateItemInput(item);
assertCollectionId(id);
const merged = await store().queueRecordWrite(id, async () => {
const cur = await store().loadOne(id);
if (!cur) throw codedError(`Collection not found: ${id}`, ERR_NOT_FOUND);
// A soft-deleted collection behaves as not-found for mutations (matches
// updateCollection) — never resurrect a tombstone or churn its timestamps.
if (cur.deleted === true) throw codedError(`Collection not found: ${id}`, ERR_NOT_FOUND);
const key = `${kind}:${ref}`;
if (cur.items.find((it) => itemKey(it) === key)) {
throw codedError(`Item already in collection: ${key}`, ERR_DUPLICATE);
}
if (cur.items.length >= ITEMS_MAX) {
throw codedError(`Collection full (limit ${ITEMS_MAX})`, ERR_VALIDATION);
}
const updated = {
...cur,
items: [...cur.items, { kind, ref, addedAt: stampAfter(cur.removedItems?.[key]) }],
updatedAt: new Date().toISOString(),
};
await store().saveOneNow(id, updated);
return updated;
});
// Emit outside the serialized critical section — subscribers may issue
// their own collection reads and we don't want to deadlock the tail.
// mediaCollection covers standalone direct subscriptions; universe/series
// nudge the bundled share-bucket re-export for linked collections.
emitRecordUpdated('mediaCollection', merged.id);
if (merged.universeId) emitRecordUpdated('universe', merged.universeId);
if (merged.seriesId) emitRecordUpdated('series', merged.seriesId);
return merged;
}
/**
* Bulk add/remove items in a single read-modify-write. Halves wall-clock for
* UI "Move N items" / "select all" flows that would otherwise issue 2N
* round-trips (one DELETE per source-collection item + one POST per
* destination-collection item) and dodges the race window where an in-flight
* AddItem can collide with a parallel RemoveItem on the same collection.
*
* `add` items use sanitizeItem rules (kind + ref); duplicates of items
* already present are skipped silently (idempotent), matching the
* UI's expectation that "add what isn't already there." `remove` keys
* are `<kind>:<ref>` strings; unknown keys are silently ignored so a
* stale UI selection can't 404 the whole batch.
*
* Validation rules mirror `addItem`: any invalid `add` entry throws before
* any mutation lands. `remove` keys that aren't strings throw too.
*
* Returns the post-write collection plus counts: `{ collection, added, removed }`.
*/
export async function bulkUpdateCollectionItems(id, { add = [], remove = [] } = {}) {
if (!Array.isArray(add)) throw codedError('add must be an array', ERR_VALIDATION);
if (!Array.isArray(remove)) throw codedError('remove must be an array', ERR_VALIDATION);
// Validate every `add` entry up front so a single bad item doesn't leave
// a partially-applied batch behind.
const cleanAdd = add.map(validateItemInput);
for (const key of remove) {
if (typeof key !== 'string' || !key) {
throw codedError('remove keys must be non-empty strings of the form "<kind>:<ref>"', ERR_VALIDATION);
}
}
assertCollectionId(id);
const result = await store().queueRecordWrite(id, async () => {
const cur = await store().loadOne(id);
if (!cur) throw codedError(`Collection not found: ${id}`, ERR_NOT_FOUND);
// A soft-deleted collection behaves as not-found for mutations (matches
// updateCollection) — never resurrect a tombstone or churn its timestamps.
if (cur.deleted === true) throw codedError(`Collection not found: ${id}`, ERR_NOT_FOUND);
const removeSet = new Set(remove);
const removedItems = { ...cur.removedItems };
const remainingItems = cur.items.filter((it) => {
if (!removeSet.has(itemKey(it))) return true;
removedItems[itemKey(it)] = stampAfter(it.addedAt);
return false;
});
const removed = cur.items.length - remainingItems.length;
// Dedupe `add` against itself AND against items that survived the removal.
// This is intentionally tolerant — re-adding an existing item is a no-op
// rather than an error, so a multi-select Move that overlaps with prior
// membership stays idempotent.
const presentKeys = new Set(remainingItems.map(itemKey));
const now = new Date().toISOString();
const additions = [];
for (const { kind, ref } of cleanAdd) {
const key = `${kind}:${ref}`;
if (presentKeys.has(key)) continue;
presentKeys.add(key);
// A key removed earlier in this same batch keeps its removal record, so
// the re-add must postdate it to survive.
additions.push({ kind, ref, addedAt: removedItems[key] ? stampAfter(removedItems[key]) : now });
}
const nextItems = [...remainingItems, ...additions];
if (nextItems.length > ITEMS_MAX) {
throw codedError(`Collection full (limit ${ITEMS_MAX})`, ERR_VALIDATION);
}
// Drop a cover that pointed at a now-removed item — matches the
// single-item removeItem path so the cover never dangles after a batch.
const coverKey = cur.coverKey && removeSet.has(cur.coverKey) ? null : cur.coverKey;
const merged = {
...cur,
items: nextItems,
...(Object.keys(removedItems).length ? { removedItems: sanitizeRemovedItems(removedItems) } : {}),
coverKey,
updatedAt: now,
};
await store().saveOneNow(id, merged);
return { collection: merged, added: additions.length, removed };
});
if (result.added || result.removed) {
emitRecordUpdated('mediaCollection', result.collection.id);
if (result.collection.universeId) emitRecordUpdated('universe', result.collection.universeId);
if (result.collection.seriesId) emitRecordUpdated('series', result.collection.seriesId);
}
return result;
}
/**
* Drop one media item from every live collection that holds it — the sweep a
* deleted asset needs so no collection keeps a dangling row. Goes through
* `bulkUpdateCollectionItems`, so each touched collection bumps `updatedAt`
* (the LWW stamp peers sync on) and clears a cover that pointed at the item.
* Returns the number of collections changed.
*/
export async function removeItemFromAllCollections(kind, ref) {
const key = `${kind}:${ref}`;
const holders = (await listCollections()).filter((c) => c.items.some((it) => itemKey(it) === key));
let changed = 0;
for (const c of holders) {
// A collection tombstoned between the listing and the write is not a failure.
const res = await bulkUpdateCollectionItems(c.id, { remove: [key] }).catch((err) => {
if (err?.code === ERR_NOT_FOUND) return null;
throw err;
});
if (res?.removed) changed += 1;
}
return changed;
}
export async function removeItem(id, key) {
assertCollectionId(id);
const merged = await store().queueRecordWrite(id, async () => {
const cur = await store().loadOne(id);
if (!cur) throw codedError(`Collection not found: ${id}`, ERR_NOT_FOUND);
// A soft-deleted collection behaves as not-found for mutations (matches
// updateCollection) — never resurrect a tombstone or churn its timestamps.
if (cur.deleted === true) throw codedError(`Collection not found: ${id}`, ERR_NOT_FOUND);
const target = cur.items.find((it) => itemKey(it) === key);
if (!target) throw codedError(`Item not in collection: ${key}`, ERR_NOT_FOUND);
const updated = {
...cur,
items: cur.items.filter((it) => it !== target),
removedItems: sanitizeRemovedItems({ ...cur.removedItems, [key]: stampAfter(target.addedAt) }),
// Drop the cover if it pointed at the removed item — sanitizeCollection
// would do this on the next read, but doing it here keeps the in-memory
// return consistent with what's persisted.
coverKey: cur.coverKey === key ? null : cur.coverKey,
updatedAt: new Date().toISOString(),
};
await store().saveOneNow(id, updated);
return updated;
});
emitRecordUpdated('mediaCollection', merged.id);
if (merged.universeId) emitRecordUpdated('universe', merged.universeId);
if (merged.seriesId) emitRecordUpdated('series', merged.seriesId);
return merged;
}
// Hard-remove tombstoned collections whose deletedAt is older than the cutoff.
// Called by tombstoneGc once every subscribed peer has acked the deletion.
export async function pruneTombstonedCollections(olderThanMs) {
if (!Number.isFinite(olderThanMs)) return { pruned: 0 };
const all = await store().loadAll();
const candidates = all.filter((c) => {
if (c.deleted !== true) return false;
const ms = Date.parse(c.deletedAt || '');
return Number.isFinite(ms) && ms < olderThanMs;
});
// Candidates have distinct ids → distinct per-record queues, so prune them
// with a bounded fan-out instead of one-at-a-time. Each returns whether it
// actually pruned; sum after (no shared mutable counter across the workers).
// Each worker catches its own error so mapWithConcurrency never rejects mid-
// flight (which would let the OTHER in-flight workers keep writing in the
// background past our return); we rethrow the first error after every worker
// has settled, preserving the throw-on-failure contract without leaking
// background writes.
let firstError = null;
const outcomes = await withBaseHashFlushBatch(() =>
mapWithConcurrency(candidates, COLLECTION_WRITE_CONCURRENCY, (c) =>
store().queueRecordWrite(c.id, async () => {
// Re-check inside the queue (deleteOneNow spans the load→delete boundary)
// so a concurrent re-create at the same id isn't blown away.
const cur = await store().loadOne(c.id);
if (!cur || cur.deleted !== true) return false;
const ms = Date.parse(cur.deletedAt || '');
if (!(Number.isFinite(ms) && ms < olderThanMs)) return false;
await store().deleteOneNow(c.id);
// Evict the conflict-journal base hash so the side store doesn't grow
// dead keys (mirrors pruneTombstonedUniverses / pruneTombstonedSeries).
await deleteSyncBaseHash('mediaCollection', c.id);
return true;
}).catch((err) => { if (!firstError) firstError = err; return false; })));
if (firstError) throw firstError;
return { pruned: outcomes.filter(Boolean).length };
}
// Drop item removal records older than the cutoff from live collections.
// Called by tombstoneGc with a cutoff every subscribed peer has acknowledged,
// so no peer still holds a pre-removal copy the record would have to defeat.
// Housekeeping, not an edit: `updatedAt` is left alone so the prune can't win
// LWW over a peer's real scalar change. Returns the number of records dropped.
export async function pruneCollectionItemRemovals(olderThanMs) {
if (!Number.isFinite(olderThanMs)) return { pruned: 0 };
const isExpired = (removedAt) => parseTsMs(removedAt) < olderThanMs;
const candidates = (await store().loadAll()).filter((c) =>
c.deleted !== true && Object.values(c.removedItems || {}).some(isExpired));
let firstError = null;
const outcomes = await mapWithConcurrency(candidates, COLLECTION_WRITE_CONCURRENCY, (c) =>
store().queueRecordWrite(c.id, async () => {
const cur = await store().loadOne(c.id);
if (!cur || cur.deleted === true || !cur.removedItems) return 0;
const kept = Object.entries(cur.removedItems).filter(([, removedAt]) => !isExpired(removedAt));
const dropped = Object.keys(cur.removedItems).length - kept.length;
if (dropped === 0) return 0;
const { removedItems: _removedItems, ...rest } = cur;
await store().saveOneNow(c.id, kept.length ? { ...rest, removedItems: Object.fromEntries(kept) } : rest);
return dropped;
}).catch((err) => { if (!firstError) firstError = err; return 0; }));
if (firstError) throw firstError;
return { pruned: outcomes.reduce((sum, n) => sum + n, 0) };
}
/**
* Merge an incoming list of collections from a peer (snapshot sync OR the
* per-record push payload's `linkedCollection` field). Per-collection
* semantics:
*
* - New (unseen id): inserted verbatim after sanitization.
* - Existing id: top-level scalars (name, description, coverKey, universeId,
* seriesId) follow LWW on `updatedAt`. Items[] is **union by `kind:ref`**
* minus copies a newer per-item removal record supersedes (see
* `mergeCollectionItems`) — a divergence (peer-A added image-X while
* peer-B added image-Y to the same collection on the same day) retains
* both rather than discarding whichever has the older collection-level
* updatedAt, while a deliberate removal sticks against a peer that still
* holds the item.
*
* Each remote collection's merge runs inside the store's per-record queue
* (`queueRecordWrite(id)`), so a concurrent local `addItem` and a remote apply
* against the SAME collection can't interleave, while merges of DIFFERENT
* collections proceed in parallel.
*/
export async function mergeMediaCollectionsFromSync(remoteCollections, { source = { via: 'sync', peerId: null } } = {}) {
if (!Array.isArray(remoteCollections)) return { applied: false, count: 0 };
// Distinct collection ids ride distinct per-record queues, so apply the
// snapshot batch with a bounded fan-out rather than one-at-a-time. (A duplicate
// id within the batch still serializes on the same queue, and the merge is
// LWW so its outcome is order-independent.) The base-hash mutations each
// record makes accumulate in memory and are persisted by the single
// `flushBaseHashes()` after the fan-out settles — unchanged from the
// sequential version.
//
// Each worker catches its own error so mapWithConcurrency never rejects mid-
// flight: an early reject would let the other in-flight workers keep writing
// (and mutating the in-memory base-hash map) in the background past our return
// AND skip the flush below, stranding completed writes' base hashes unsaved.
// We let every worker settle, ALWAYS flush, then rethrow the first error.
let firstError = null;
const outcomes = await mapWithConcurrency(remoteCollections, COLLECTION_WRITE_CONCURRENCY, (remote) => {
const sanitized = sanitizeCollection(remote);
if (!sanitized) return false;
return store().queueRecordWrite(sanitized.id, async () => {
const local = await store().loadOne(sanitized.id);
if (!local) {
await store().saveOneNow(sanitized.id, sanitized);
// No local counterpart to lose — nothing to journal, but seed the base
// hash so a FUTURE scalar divergence on this collection is detected.
await setSyncBaseHash('mediaCollection', sanitized.id, contentHashForRecord('mediaCollection', sanitized));
return true;
}
const { items: mergedItems, removedItems: mergedRemovals } = mergeCollectionItems(local, sanitized);
// LWW on collection-level scalars. Parse to epoch ms (not lexicographic
// compare on the raw strings): `sanitizeCollection` accepts any
// Date.parse-able string for `updatedAt`, so a hand-edit or older peer
// writing a non-ISO format could otherwise lose to a "newer" string
// that's actually an older moment in time. Unparseable side LOSES (a
// corrupted timestamp can't claim to be newer); both unparseable
// breaks to local-wins (no signal to override). `localTs`/`remoteTs`
// keep the original strings so the winning side's `updatedAt`
// round-trips to disk verbatim — only the comparison is numeric.
const localTs = local.updatedAt || '';
const remoteTs = sanitized.updatedAt || '';
const remoteWins = compareNewerWins(remoteTs, localTs);
if (remoteWins) {
// Remote's scalars are about to overwrite local's. Non-blocking conflict
// journal: archive the about-to-be-lost local version when BOTH sides
// diverged from the last synced base — but only over the SCALAR SUBSET
// (contentHashForRecord narrows mediaCollection to its overwritable
// scalars, so an item-only divergence — items and removal records merge
// per key, never overwritten — does NOT false-positive here). Always
// advances the base hash (clean or conflict) so the next snapshot cycle
// doesn't re-journal the same divergence. Never throws into the merge.
// Skipped when local wins: local scalars are KEPT, so there's nothing to
// lose (and the base stays put — it advances only on an accepted
// overwrite, mirroring universe).
await maybeJournalBeforeOverwrite({ kind: 'mediaCollection', id: sanitized.id, local, remote: sanitized, source });
}
const scalarSource = remoteWins ? sanitized : local;
// Cover key: only adopt the scalar source's coverKey if it points at an
// item that survives the union — otherwise sanitizeCollection on next
// read would drop it back to null and we'd churn updatedAt forever.
const scalarDeleted = scalarSource.deleted === true;
const presentKeys = new Set(mergedItems.map(itemKey));
const coverKey = !scalarDeleted && scalarSource.coverKey && presentKeys.has(scalarSource.coverKey)
? scalarSource.coverKey
: null;
// NOTE — `source` (#3311) is deliberately absent from the scalar list
// below. It is a LOCAL-ONLY provenance stamp that `sanitizeRecordForWire`
// strips before sending, so a remote record never carries one and a
// remote-wins overwrite must not be able to clear ours. The `...local`
// spread preserves it; a brand-new record inserted above keeps whatever
// the sanitizer accepted (an offline export/import bundle can legitimately
// carry it). Do NOT add it to the LWW scalars without also un-stripping it
// on the wire — and that needs a schemaVersions bump.
// Re-sanitized so a record that newly gains `removedItems` lands in the
// canonical key order the equality check and the wire checksum rely on.
const next = sanitizeCollection({
...local,
name: scalarSource.name,
description: scalarSource.description,
coverKey: scalarDeleted ? null : coverKey,
universeId: scalarDeleted ? null : scalarSource.universeId,
seriesId: scalarDeleted ? null : scalarSource.seriesId,
items: scalarDeleted ? [] : mergedItems,
removedItems: scalarDeleted ? undefined : mergedRemovals,
updatedAt: remoteWins ? remoteTs : localTs,
deleted: scalarDeleted,
deletedAt: scalarDeleted ? (scalarSource.deletedAt || (remoteWins ? remoteTs : localTs)) : null,
});
if (collectionsEqual(local, next)) return false;
await store().saveOneNow(sanitized.id, next);
return true;
}).catch((err) => { if (!firstError) firstError = err; return false; });
});
const changed = outcomes.filter(Boolean).length;
// Persist the batched conflict-journal base-hash updates accumulated above in
// one write (seeds on insert + advances on accepted overwrite). Always runs —
// even if a worker threw — so completed writes' base hashes aren't stranded.
await flushBaseHashes();
if (firstError) throw firstError;
if (changed === 0) return { applied: false, count: 0 };
return { applied: true, count: changed };
}
// Union of both sides' items, minus any copy a removal record supersedes. The
// removal maps merge latest-wins per key, and an item is kept when EITHER side
// holds a copy added after the merged removal — so concurrent adds are never
// lost, and a removal wins only over copies that predate it.
function mergeCollectionItems(local, remote) {
const removedItems = { ...local.removedItems };
for (const [k, removedAt] of Object.entries(remote.removedItems || {})) {
if (compareNewerWins(removedAt, removedItems[k])) removedItems[k] = removedAt;
}
const byKey = new Map();
for (const it of [...(local.items || []), ...(remote.items || [])]) {
if (isSupersededByRemoval(it, removedItems)) continue;
const k = itemKey(it);
const existing = byKey.get(k);
if (!existing) {
byKey.set(k, it);
continue;
}
// Earliest addedAt wins — a sync replay shouldn't bump the addedAt of
// an item already in the collection. Parse to epoch ms instead of
// lexicographic compare: `sanitizeItem` keeps any Date.parse-able
// string (not strictly ISO-8601), so two parseable-but-different-format
// timestamps could compare incorrectly as strings. Tie → existing wins
// (matches the previous `<=` behavior — sync replay shouldn't churn).
const cmp = compareEarlierWins(existing.addedAt, it.addedAt);
const earlier = cmp <= 0 ? existing : it;
byKey.set(k, earlier);
}
// Sort by canonical itemKey so the merged output is deterministic
// regardless of insertion order. Without this, the same set of items
// arriving in different orders from a peer (or replayed from snapshot
// vs. push) produces a different array order — and the downstream
// collectionsEqual() JSON.stringify equality check then sees a diff
// and triggers a redundant write + checksum churn. The dataSync
// snapshot path already sorts collections by id and items by key for
// wire stability; aligning the in-memory merge keeps reads and writes
// self-consistent.
const items = Array.from(byKey.values()).sort((a, b) => itemKey(a).localeCompare(itemKey(b)));
return { items, removedItems };
}
function collectionsEqual(a, b) {
// Both sides come through sanitizeCollection so key order is canonical and
// JSON.stringify is sufficient for "did anything actually move".
return JSON.stringify(a) === JSON.stringify(b);
}