Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

fix: retry peer media index repairs after landed byte sync (#10676)

Adam Eivye0390c77a901
diff --git a/server/services/sharing/peerMediaLibrarySync.js b/server/services/sharing/peerMediaLibrarySync.js
index 6dadb9b5a6..f8684ad7f1 100644
--- a/server/services/sharing/peerMediaLibrarySync.js
+++ b/server/services/sharing/peerMediaLibrarySync.js
@@ -198,6 +198,9 @@ export async function buildMediaLibraryManifest() {
// post-boot sweep just re-confirms disk, cheap because the diff finds everything
// present and pulls nothing).
const lastLibraryManifestHash = new Map(); // peerInstanceId → manifestHash
+// Landed bytes stay durable when the derived index is unavailable. Keep repair
+// pending independently of the byte diff so an empty diff can retry it.
+const libraryIndexRepairPending = new Set(); // peerInstanceId
// Consecutive unchanged-manifest ticks per peer; after FORCE_REVALIDATE_EVERY we
// force a full re-diff even though the remote manifest is unchanged, so a local
// file loss self-heals without waiting for a restart or a remote library change.
@@ -226,12 +229,14 @@ function reconcileMediaLibraryIndex() {
// graph (it no-ops under the file/test backend). image+video rows are rebuilt
// from disk; audio/music aren't indexed (served from disk directly).
const mod = await import('../mediaAssetIndex/index.js').catch(() => null);
- if (!mod?.reconcileMediaAssets) return;
+ if (!mod?.reconcileMediaAssets) return false;
// The pull wrote bytes outside any lease; indexing them takes one, so a cut
// that copied the media directories first never dumps a row for a file it missed.
- await withBackupAssetPublication(() => mod.reconcileMediaAssets()).catch((err) => {
- console.log(`⚠️ peerSync: media_assets reconcile after library sweep failed: ${err.message}`);
+ const result = await withBackupAssetPublication(() => mod.reconcileMediaAssets({ requireComplete: true })).catch((err) => {
+ console.error(`❌ peerSync: media_assets reconcile after library sweep failed: ${err.message}`);
+ return null;
});
+ return result?.ok === true && Array.isArray(result.skippedPrune) && result.skippedPrune.length === 0;
});
return reconcilePending;
}
@@ -299,7 +304,7 @@ export async function syncMediaLibraryFromPeer(peer) {
// corruption self-heals even while the REMOTE manifest stays put. (The
// recorded hash is in-memory, so a process restart also re-diffs; this covers
// the mid-session window between restarts.)
- if (lastLibraryManifestHash.get(peer.instanceId) === manifest.manifestHash) {
+ if (!libraryIndexRepairPending.has(peer.instanceId) && lastLibraryManifestHash.get(peer.instanceId) === manifest.manifestHash) {
const skips = (libraryUnchangedSkips.get(peer.instanceId) || 0) + 1;
if (skips < FORCE_REVALIDATE_EVERY) {
libraryUnchangedSkips.set(peer.instanceId, skips);
@@ -308,33 +313,30 @@ export async function syncMediaLibraryFromPeer(peer) {
libraryUnchangedSkips.set(peer.instanceId, 0); // periodic forced re-diff — fall through
}
const missing = await diffAssetManifestAgainstLocal(manifest.assets);
- if (missing.length === 0) {
- lastLibraryManifestHash.set(peer.instanceId, manifest.manifestHash);
- return { pulled: 0 };
- }
const requested = missing.length;
- // Reuse the per-record pull worker (in-flight dedup, image-sidecar fetch,
- // video-thumbnail regen).
- await pullMissingAssetsFromPeer(peer.instanceId, missing);
- // `pullMissingAssetsFromPeer` swallows per-asset failures (peer drops
- // mid-sweep, 404, size-cap reject) and always resolves — so a resolved pull
- // does NOT mean every byte landed. Re-diff against disk to see what actually
- // arrived; this is the authoritative signal, not the pull's resolution.
- const stillMissing = await diffAssetManifestAgainstLocal(manifest.assets);
+ let stillMissing = missing;
+ if (requested > 0) {
+ // Reuse the per-record pull worker (in-flight dedup, image-sidecar fetch,
+ // video-thumbnail regen). It swallows per-asset failures, so re-diff disk
+ // rather than treating the pull's resolution as evidence bytes landed.
+ await pullMissingAssetsFromPeer(peer.instanceId, missing);
+ stillMissing = await diffAssetManifestAgainstLocal(manifest.assets);
+ }
const pulled = requested - stillMissing.length;
- // Rebuild the derived media_assets index when any image/video bytes landed so
- // the gallery/Media tab reflects them. Idempotent; best-effort.
- if (pulled > 0) await reconcileMediaLibraryIndex();
- if (stillMissing.length === 0) {
- // Full sweep — safe to short-circuit future ticks on this manifestHash.
+ if (pulled > 0) libraryIndexRepairPending.add(peer.instanceId);
+ if (libraryIndexRepairPending.has(peer.instanceId) && await reconcileMediaLibraryIndex()) {
+ libraryIndexRepairPending.delete(peer.instanceId);
+ }
+ if (stillMissing.length === 0 && !libraryIndexRepairPending.has(peer.instanceId)) {
+ // Full sweep, including index repair — safe to short-circuit future ticks on this manifestHash.
lastLibraryManifestHash.set(peer.instanceId, manifest.manifestHash);
- console.log(`📥 peerSync: media-library sweep from ${peer.name || peer.instanceId} — pulled ${pulled} asset(s)`);
- } else {
+ if (pulled > 0) console.log(`📥 peerSync: media-library sweep from ${peer.name || peer.instanceId} — pulled ${pulled} asset(s)`);
+ } else if (stillMissing.length > 0) {
// Partial pull — do NOT record the hash, so the next tick re-diffs and
// retries the still-missing assets instead of being marked done.
console.log(`⚠️ peerSync: media-library sweep from ${peer.name || peer.instanceId} — pulled ${pulled}/${requested}, ${stillMissing.length} still missing; retrying next tick`);
}
- return { pulled, missing: stillMissing.length };
+ return requested > 0 ? { pulled, missing: stillMissing.length } : { pulled };
} finally {
librarySweepInFlight.delete(peer.instanceId);
}
diff --git a/server/services/sharing/peerMediaLibrarySync.test.js b/server/services/sharing/peerMediaLibrarySync.test.js
new file mode 100644
index 0000000000..3129be3183
--- /dev/null
+++ b/server/services/sharing/peerMediaLibrarySync.test.js
@@ -0,0 +1,157 @@
+import { afterAll, afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
+import { mkdir, readFile, readdir, rm } from 'node:fs/promises';
+import { join } from 'node:path';
+import { createHash } from 'node:crypto';
+import { PORTOS_SCHEMA_VERSIONS } from '../../lib/schemaVersions.js';
+import { createTempDataRoot, makePathsProxy } from '../../lib/mockPathsDataRoot.js';
+
+let tempRoot;
+vi.mock('../../lib/fileUtils.js', async importOriginal => makePathsProxy(await importOriginal(), { dataRoot: () => tempRoot }));
+vi.mock('../../lib/paths.js', async importOriginal => makePathsProxy(await importOriginal(), { dataRoot: () => tempRoot }));
+vi.mock('../instances.js', () => ({ getPeers: vi.fn(async () => []) }));
+vi.mock('../../lib/peerHttpClient.js', async importOriginal => ({ ...(await importOriginal()), peerFetch: vi.fn() }));
+vi.mock('./peerSyncAssets.js', async importOriginal => {
+ const actual = await importOriginal();
+ return { ...actual, diffAssetManifestAgainstLocal: vi.fn(actual.diffAssetManifestAgainstLocal) };
+});
+vi.mock('../../lib/databaseMaintenanceJournal.js', async importOriginal => ({ ...(await importOriginal()), assertDatabaseAdmission: () => {} }));
+
+const indexAdapter = vi.hoisted(() => ({ reconcile: vi.fn(), indexImage: vi.fn() }));
+vi.mock('../mediaAssetIndex/index.js', () => ({ reconcileMediaAssets: indexAdapter.reconcile, indexImage: indexAdapter.indexImage }));
+
+// Load after the temp redirect exists; no production peers, files or DB are used.
+tempRoot = createTempDataRoot('portos-library-repair-');
+const { PATHS } = await import('../../lib/fileUtils.js');
+const { getPeers } = await import('../instances.js');
+const { peerFetch } = await import('../../lib/peerHttpClient.js');
+const { syncMediaLibraryFromPeer } = await import('./peerMediaLibrarySync.js');
+const { diffAssetManifestAgainstLocal } = await import('./peerSyncAssets.js');
+const { acquireBackupSnapshotCut } = await import('../../lib/backupSnapshotBoundary.js');
+
+const rows = new Map();
+const bytes = Buffer.from('synthetic-peer-image');
+let sequence = 0;
+const peer = () => ({ instanceId: `repair-peer-${++sequence}`, address: '192.0.2.10', port: 5555, fullSync: true });
+const deferred = () => {
+ let resolve;
+ const promise = new Promise(r => { resolve = r; });
+ return { promise, resolve };
+};
+async function repairIndex() {
+ for (const filename of await readdir(PATHS.images)) {
+ if (filename.endsWith('.png')) rows.set(filename, await readFile(join(PATHS.images, filename)));
+ }
+ return { ok: true, skippedPrune: [] };
+}
+function advertise(peers, filenames = ['repair.png']) {
+ vi.mocked(getPeers).mockResolvedValue(peers);
+ const manifest = {
+ schemaVersion: PORTOS_SCHEMA_VERSIONS.mediaLibrary, manifestHash: 'a'.repeat(64),
+ assets: filenames.map(filename => ({ kind: 'image', filename, sha256: createHash('sha256').update(bytes).digest('hex') })),
+ };
+ vi.mocked(peerFetch).mockImplementation(async url => {
+ if (String(url).endsWith('/library-manifest')) return { ok: true, json: async () => manifest };
+ if (String(url).endsWith('.png')) return { ok: true, headers: new Headers({ 'content-length': String(bytes.length) }), arrayBuffer: async () => bytes };
+ return { ok: false }; // no optional metadata sidecar
+ });
+}
+const imageDownloads = () => vi.mocked(peerFetch).mock.calls.filter(([url]) => String(url).endsWith('.png')).length;
+
+beforeEach(async () => {
+ rows.clear();
+ await mkdir(PATHS.images, { recursive: true });
+ vi.mocked(peerFetch).mockReset();
+ vi.mocked(diffAssetManifestAgainstLocal).mockClear();
+ indexAdapter.indexImage.mockReset().mockRejectedValue(new Error('synthetic publication index outage'));
+ indexAdapter.reconcile.mockReset().mockImplementation(repairIndex);
+ vi.doMock('../mediaAssetIndex/index.js', () => ({ reconcileMediaAssets: indexAdapter.reconcile, indexImage: indexAdapter.indexImage }));
+});
+afterEach(async () => {
+ await rm(PATHS.images, { recursive: true, force: true });
+});
+afterAll(async () => { await rm(tempRoot, { recursive: true, force: true }); });
+
+describe('peer library index recovery', () => {
+ it.each(['write rejection', 'incomplete authoritative read'])('repairs landed images on the identical next sweep after %s, then resumes unchanged shortcuts', async failure => {
+ const source = peer();
+ advertise([source]);
+ if (failure === 'write rejection') indexAdapter.reconcile.mockRejectedValueOnce(new Error('synthetic index write outage'));
+ else indexAdapter.reconcile.mockResolvedValueOnce({ ok: true, indexed: 0, skippedPrune: ['images'] });
+
+ expect(await syncMediaLibraryFromPeer(source)).toEqual({ pulled: 1, missing: 0 });
+ expect(await readFile(join(PATHS.images, 'repair.png'))).toEqual(bytes);
+ expect(rows.size).toBe(0);
+
+ expect(await syncMediaLibraryFromPeer(source)).toEqual({ pulled: 0 });
+ expect(rows.get('repair.png')).toEqual(bytes);
+ expect(imageDownloads()).toBe(1);
+ expect(indexAdapter.reconcile).toHaveBeenCalledTimes(2);
+ expect(indexAdapter.reconcile).toHaveBeenLastCalledWith({ requireComplete: true });
+ for (let tick = 0; tick < 3; tick++) expect((await syncMediaLibraryFromPeer(source)).skipped).toBe('unchanged');
+ expect(indexAdapter.reconcile).toHaveBeenCalledTimes(2);
+ });
+
+ it('recovers when the index module cannot load, without losing pending repair or downloading again', async () => {
+ const source = peer();
+ advertise([source]);
+ vi.doMock('../mediaAssetIndex/index.js', () => { throw new Error('synthetic unavailable index module'); });
+ expect(await syncMediaLibraryFromPeer(source)).toEqual({ pulled: 1, missing: 0 });
+ expect(rows.size).toBe(0);
+ vi.doMock('../mediaAssetIndex/index.js', () => ({ reconcileMediaAssets: indexAdapter.reconcile, indexImage: indexAdapter.indexImage }));
+ expect(await syncMediaLibraryFromPeer(source)).toEqual({ pulled: 0 });
+ expect(rows.get('repair.png')).toEqual(bytes);
+ expect(imageDownloads()).toBe(1);
+ expect((await syncMediaLibraryFromPeer(source)).skipped).toBe('unchanged');
+ });
+
+ it('waits for backup admission before repairing already landed bytes', async () => {
+ const source = peer();
+ advertise([source]);
+ indexAdapter.reconcile.mockRejectedValueOnce(new Error('synthetic outage'));
+ await syncMediaLibraryFromPeer(source);
+ const release = await acquireBackupSnapshotCut();
+ let retry;
+ try {
+ retry = syncMediaLibraryFromPeer(source);
+ await vi.waitFor(() => expect(peerFetch).toHaveBeenCalledTimes(4));
+ // Drain the request/import continuation; its index write is awaiting the cut.
+ await new Promise(resolve => setImmediate(resolve));
+ expect(indexAdapter.reconcile).toHaveBeenCalledTimes(1);
+ expect(rows.size).toBe(0);
+ } finally { release(); }
+ await retry;
+ expect(rows.get('repair.png')).toEqual(bytes);
+ expect(indexAdapter.reconcile).toHaveBeenCalledTimes(2);
+ expect(imageDownloads()).toBe(1);
+ });
+
+ it('serializes active repairs and coalesces queued peers while retaining failure-to-retry state', async () => {
+ const sources = [peer(), peer(), peer()];
+ // Distinct files ensure each peer has landed bytes and owns a pending repair.
+ indexAdapter.reconcile.mockRejectedValue(new Error('synthetic outage'));
+ for (let i = 0; i < sources.length; i++) {
+ advertise(sources, [`repair-${i}.png`]);
+ await syncMediaLibraryFromPeer(sources[i]);
+ }
+ advertise(sources, ['repair-0.png', 'repair-1.png', 'repair-2.png']);
+ const firstStarted = deferred();
+ const firstRelease = deferred();
+ indexAdapter.reconcile.mockReset().mockImplementationOnce(async () => {
+ firstStarted.resolve();
+ await firstRelease.promise;
+ return repairIndex();
+ }).mockImplementation(repairIndex);
+ const first = syncMediaLibraryFromPeer(sources[0]);
+ await firstStarted.promise;
+ const queued = sources.slice(1).map(syncMediaLibraryFromPeer);
+ await vi.waitFor(() => expect(vi.mocked(diffAssetManifestAgainstLocal).mock.settledResults.filter(result => result.type === 'fulfilled')).toHaveLength(9));
+ await new Promise(resolve => setImmediate(resolve));
+ expect(indexAdapter.reconcile).toHaveBeenCalledTimes(1);
+ firstRelease.resolve();
+ await Promise.all([first, ...queued]);
+ expect(indexAdapter.reconcile).toHaveBeenCalledTimes(2);
+ expect([...rows.keys()].sort()).toEqual(['repair-0.png', 'repair-1.png', 'repair-2.png']);
+ expect(imageDownloads()).toBe(3);
+ for (const source of sources) expect((await syncMediaLibraryFromPeer(source)).skipped).toBe('unchanged');
+ });
+});
diff --git a/server/services/sharing/peerSync.test.js b/server/services/sharing/peerSync.test.js
index 5d0bfec690..f0c50d0a22 100644
--- a/server/services/sharing/peerSync.test.js
+++ b/server/services/sharing/peerSync.test.js
@@ -81,7 +81,7 @@ vi.mock('../mediaCollections.js', async () => ({
// media_assets index after bytes land. Mock it to a no-op spy so the sweep tests
// don't touch Postgres and can assert the reconcile fired.
vi.mock('../mediaAssetIndex/index.js', () => ({
- reconcileMediaAssets: vi.fn().mockResolvedValue(undefined),
+ reconcileMediaAssets: vi.fn().mockResolvedValue({ ok: true, skippedPrune: [] }),
}));
vi.mock('../artists/index.js', async () => {