Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

fix: stop a running share export from resurrecting an unshared or deleted record's manifest (#11762)

[._.]/ Adam Eivy5ac6194cd941
diff --git a/server/services/sharing/exporter.js b/server/services/sharing/exporter.js
index b053faf488..98a1468582 100644
--- a/server/services/sharing/exporter.js
+++ b/server/services/sharing/exporter.js
@@ -366,6 +366,11 @@ function stampOrigin(record, { bucket, source, sourceBio, manifestId }) {
* deterministic (`sub-series-<id>-<senderInstanceId>.json`) so re-exports
* overwrite in place instead of accumulating, and two peers sharing the same
* series don't collide on the bucket file. Omit for one-shot legacy shares.
+ *
+ * `opts.isLive` (optional async predicate): consulted right before the
+ * manifest is written. When it resolves false (the subscription was removed
+ * mid-export) the manifest write is skipped and the export returns `null`;
+ * already-copied records/blobs are left for blob GC.
*/
export async function exportSeries(seriesId, bucketId, opts = {}) {
const bucket = await getBucket(bucketId);
@@ -487,6 +492,7 @@ export async function exportSeries(seriesId, bucketId, opts = {}) {
});
const manifest = { ...manifestStub, recordIds, assetRefs, reviewRefs, outlineRefs };
+ if (opts.isLive && !(await opts.isLive())) return null;
const filename = await writeManifest(bucket.path, manifest);
await pruneAfterExport(bucket, senderInstanceId);
return { manifestId, filename, recordCount: recordIds.length, assetCount: assetRefs.length };
@@ -549,6 +555,7 @@ export async function exportUniverse(universeId, bucketId, opts = {}) {
});
const manifest = { ...manifestStub, recordIds: [universe.id], assetRefs };
+ if (opts.isLive && !(await opts.isLive())) return null;
const filename = await writeManifest(bucket.path, manifest);
await pruneAfterExport(bucket, senderInstanceId);
return { manifestId, filename, recordCount: 1, assetCount: assetRefs.length };
diff --git a/server/services/sharing/stateConcurrency.test.js b/server/services/sharing/stateConcurrency.test.js
index ae47a8a807..0efd12661f 100644
--- a/server/services/sharing/stateConcurrency.test.js
+++ b/server/services/sharing/stateConcurrency.test.js
@@ -56,8 +56,10 @@ it('does not restore a subscription removed during its own export', async () =>
const subscribing = subscriptions.subscribe({ bucketId: bucket.id, recordKind: 'universe', recordId: 'a' });
await started.promise;
const [sub] = await subscriptions.listSubscriptions();
- await subscriptions.unsubscribe(sub.id);
+ // unsubscribe waits for the running export so its manifest write can't land after the delete.
+ const unsubscribing = subscriptions.unsubscribe(sub.id);
exported.resolve({ manifestId: 'a-export' });
+ await unsubscribing;
expect(await subscribing).toBeNull();
expect(await subscriptions.listSubscriptions()).toEqual([]);
});
diff --git a/server/services/sharing/subscriptions.js b/server/services/sharing/subscriptions.js
index 4f50e853c5..8b98781df6 100644
--- a/server/services/sharing/subscriptions.js
+++ b/server/services/sharing/subscriptions.js
@@ -156,15 +156,38 @@ export async function subscribe({ bucketId, recordKind, recordId }) {
return live;
});
- // Export outside the state queue, then stamp only a still-live row.
- const exp = await runExport({ bucketId, recordKind, recordId });
- if (exp) return stampExport(sub, exp);
- return sub;
+ // Export outside the state queue (serialized per subscription), then stamp
+ // only a still-live row.
+ return enqueueExport(id, async () => {
+ const exp = await runExport(sub);
+ return exp ? stampExport(sub, exp) : sub;
+ });
+}
+
+// subId → tail of that subscription's export chain. Exports of one
+// subscription run strictly one after another so the last manifest written
+// is always the newest content, and unsubscribe can wait out a running one.
+const inFlightExports = new Map();
+
+function enqueueExport(id, task) {
+ const prev = inFlightExports.get(id) || Promise.resolve();
+ const run = prev.then(task);
+ const tail = run.catch(() => {});
+ inFlightExports.set(id, tail);
+ tail.then(() => {
+ if (inFlightExports.get(id) === tail) inFlightExports.delete(id);
+ });
+ return run;
+}
+
+async function isSubscriptionLive(id) {
+ const { subscriptions } = await queueStateWrite(readState);
+ return subscriptions.some((s) => s.id === id);
}
/** Internal: dispatch to the right exporter based on recordKind. */
-async function runExport({ bucketId, recordKind, recordId }) {
- const opts = { subscription: { recordKind, recordId } };
+async function runExport({ id, bucketId, recordKind, recordId }) {
+ const opts = { subscription: { recordKind, recordId }, isLive: () => isSubscriptionLive(id) };
if (recordKind === 'series') return exportSeries(recordId, bucketId, opts);
if (recordKind === 'universe') return exportUniverse(recordId, bucketId, opts);
return null;
@@ -191,6 +214,11 @@ export async function unsubscribe(id) {
pendingTimers.delete(sub.id);
}
+ // An export already running would otherwise write its manifest after the
+ // delete below and resurrect the share. Its liveness check now fails, but
+ // wait for it (and any queued follow-ups) so the delete always lands last.
+ await inFlightExports.get(sub.id);
+
// Delete the deterministic file from the bucket. Best-effort: the bucket
// path may be unmounted; the subscription record removal still wins so the
// user's "I want to stop sharing" intent is honored regardless. Importers
@@ -256,13 +284,14 @@ export async function unsubscribeAllForRecord(recordKind, recordId) {
const DEBOUNCE_MS = 3000;
const pendingTimers = new Map(); // subId → Timeout
-async function reexportNow(sub) {
- const exp = await runExport(sub).catch((err) => {
- console.log(`⚠️ sharing.subscriptions: re-export failed for ${sub.id}: ${err.message}`);
- return null;
+function reexportNow(sub) {
+ return enqueueExport(sub.id, async () => {
+ const exp = await runExport(sub).catch((err) => {
+ console.log(`⚠️ sharing.subscriptions: re-export failed for ${sub.id}: ${err.message}`);
+ return null;
+ });
+ if (exp) await stampExport(sub, exp);
});
- if (!exp) return;
- await stampExport(sub, exp);
}
function stampExport(sub, exp) {
diff --git a/server/services/sharing/subscriptions.test.js b/server/services/sharing/subscriptions.test.js
index 0329d0851e..d4ab69118e 100644
--- a/server/services/sharing/subscriptions.test.js
+++ b/server/services/sharing/subscriptions.test.js
@@ -8,6 +8,9 @@ import { mockNoPeerSync, mockNoPeers, mockTestIdentity } from '../../lib/mockPat
// temp dir before any service imports. beforeEach wipes + reseeds inside.
const tempData = mkdtempSync(join(tmpdir(), 'portos-subs-test-data-'));
let tempBucket;
+// Lets a test freeze the next export mid-flight (the exporter resolves the
+// instance id before it writes the manifest).
+const gate = vi.hoisted(() => ({ hold: null, held: 0, calls: 0 }));
vi.mock('../../lib/fileUtils.js', async () => {
const actual = await vi.importActual('../../lib/fileUtils.js');
@@ -25,7 +28,12 @@ vi.mock('../../lib/fileUtils.js', async () => {
});
vi.mock('../instances.js', () => mockNoPeers());
vi.mock('../instanceIdentity.js', () => mockTestIdentity({
- getInstanceId: () => Promise.resolve('inst-test'),
+ getInstanceId: async () => {
+ gate.calls++;
+ const hold = gate.hold;
+ if (hold) { gate.hold = null; gate.held++; await hold; }
+ return 'inst-test';
+ },
}));
vi.mock('./peerSync.js', () => mockNoPeerSync());
vi.mock('../mediaJobQueue/index.js', () => ({ getJob: () => null }));
@@ -42,8 +50,10 @@ describe('sharing/subscriptions', () => {
mkdirSync(join(tempData, 'images'), { recursive: true });
tempBucket = mkdtempSync(join(tmpdir(), 'portos-subs-test-bucket-'));
subs.__resetForTests();
+ gate.hold = null; gate.held = 0; gate.calls = 0;
});
afterEach(() => {
+ vi.useRealTimers();
if (tempBucket) rmSync(tempBucket, { recursive: true, force: true });
});
@@ -219,4 +229,96 @@ describe('sharing/subscriptions', () => {
expect(await subs.listSubscriptions({ recordId: u.id })).toEqual([]);
}, { timeout: 2_000, interval: 20 });
});
+
+ // vi.waitFor polls on setTimeout, which these tests fake; poll on setImmediate.
+ async function until(check) {
+ for (let i = 0; i < 2000; i++) {
+ if (await check()) return;
+ await new Promise((r) => setImmediate(r));
+ }
+ throw new Error('until: condition never became true');
+ }
+
+ // Give a released export (which would write its manifest late) real time to finish.
+ async function settleExports() {
+ vi.useRealTimers();
+ await new Promise((r) => setTimeout(r, 250));
+ }
+
+ // Schedule a debounced re-export of the universe and freeze it mid-export.
+ async function holdNextExport(recordId) {
+ let release;
+ gate.hold = new Promise((r) => { release = r; });
+ await subs.reexportSubscribedRecord('universe', recordId);
+ await vi.advanceTimersByTimeAsync(3000);
+ await until(() => gate.held === 1);
+ return release;
+ }
+
+ it('Stop sharing during an in-flight re-export does not resurrect the manifest or stamp state', async () => {
+ vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout'] });
+ const bucket = await buckets.createBucket({ name: 'B', path: tempBucket });
+ const u = await universeBuilder.createUniverse({ name: 'U-race' });
+ const sub = await subs.subscribe({ bucketId: bucket.id, recordKind: 'universe', recordId: u.id });
+ const fs = await import('fs');
+ const filePath = join(tempBucket, 'manifests', subs.subscriptionFilename({ ...sub, senderInstanceId: 'inst-test' }));
+
+ const release = await holdNextExport(u.id);
+
+ const unsub = subs.unsubscribe(sub.id);
+ release();
+ await unsub;
+ await settleExports();
+
+ expect(fs.existsSync(filePath)).toBe(false);
+ expect(await subs.listSubscriptions()).toEqual([]);
+ });
+
+ it('deleting the record during an in-flight re-export leaves no manifest in the bucket', async () => {
+ vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout'] });
+ subs.installSubscriptionListener();
+ const bucket = await buckets.createBucket({ name: 'B', path: tempBucket });
+ const u = await universeBuilder.createUniverse({ name: 'U-del-race' });
+ const sub = await subs.subscribe({ bucketId: bucket.id, recordKind: 'universe', recordId: u.id });
+ const fs = await import('fs');
+ const filePath = join(tempBucket, 'manifests', subs.subscriptionFilename({ ...sub, senderInstanceId: 'inst-test' }));
+
+ const release = await holdNextExport(u.id);
+
+ await universeBuilder.deleteUniverse(u.id);
+ await until(async () => (await subs.listSubscriptions({ recordId: u.id })).length === 0);
+ release();
+ vi.useRealTimers();
+ // unsubscribe waits out the export, then deletes.
+ await until(() => !fs.existsSync(filePath));
+ await settleExports();
+ expect(fs.existsSync(filePath)).toBe(false);
+ });
+
+ it('overlapping re-exports of one subscription run one after another', async () => {
+ vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout'] });
+ const bucket = await buckets.createBucket({ name: 'B', path: tempBucket });
+ const u = await universeBuilder.createUniverse({ name: 'U-order' });
+ const sub = await subs.subscribe({ bucketId: bucket.id, recordKind: 'universe', recordId: u.id });
+
+ const release = await holdNextExport(u.id);
+ const callsWhileFirstHeld = gate.calls;
+
+ await subs.reexportSubscribedRecord('universe', u.id);
+ await vi.advanceTimersByTimeAsync(3000);
+ for (let i = 0; i < 20; i++) await new Promise((r) => setImmediate(r));
+ expect(gate.calls).toBe(callsWhileFirstHeld); // second export has not started
+
+ release();
+ // After both exports finish the stamped manifest id is the one on disk
+ // (the second export wrote last).
+ const fs = await import('fs');
+ const filePath = join(tempBucket, 'manifests', subs.subscriptionFilename({ ...sub, senderInstanceId: 'inst-test' }));
+ await until(async () => {
+ const [live] = await subs.listSubscriptions({ recordId: u.id });
+ if (gate.calls <= callsWhileFirstHeld || live.lastManifestId === sub.lastManifestId) return false;
+ const onDisk = JSON.parse(fs.readFileSync(filePath, 'utf8'));
+ return onDisk.id === live.lastManifestId;
+ });
+ });
});