Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
fix: retry peer media index repairs after landed byte sync (#10676)
Adam Eivye0390c77a901
diff --git a/server/services/sharing/peerMediaLibrarySync.js b/server/services/sharing/peerMediaLibrarySync.jsindex 6dadb9b5a6..f8684ad7f1 100644--- a/server/services/sharing/peerMediaLibrarySync.js+++ b/server/services/sharing/peerMediaLibrarySync.js@@ -198,6 +198,9 @@ export async function buildMediaLibraryManifest() {// post-boot sweep just re-confirms disk, cheap because the diff finds everything// present and pulls nothing).const lastLibraryManifestHash = new Map(); // peerInstanceId → manifestHash+// Landed bytes stay durable when the derived index is unavailable. Keep repair+// pending independently of the byte diff so an empty diff can retry it.+const libraryIndexRepairPending = new Set(); // peerInstanceId// Consecutive unchanged-manifest ticks per peer; after FORCE_REVALIDATE_EVERY we// force a full re-diff even though the remote manifest is unchanged, so a local// file loss self-heals without waiting for a restart or a remote library change.@@ -226,12 +229,14 @@ function reconcileMediaLibraryIndex() {// graph (it no-ops under the file/test backend). image+video rows are rebuilt// from disk; audio/music aren't indexed (served from disk directly).const mod = await import('../mediaAssetIndex/index.js').catch(() => null);- if (!mod?.reconcileMediaAssets) return;+ if (!mod?.reconcileMediaAssets) return false;// The pull wrote bytes outside any lease; indexing them takes one, so a cut// that copied the media directories first never dumps a row for a file it missed.- await withBackupAssetPublication(() => mod.reconcileMediaAssets()).catch((err) => {- console.log(`⚠️ peerSync: media_assets reconcile after library sweep failed: ${err.message}`);+ const result = await withBackupAssetPublication(() => mod.reconcileMediaAssets({ requireComplete: true })).catch((err) => {+ console.error(`❌ peerSync: media_assets reconcile after library sweep failed: ${err.message}`);+ return null;});+ return result?.ok === true && Array.isArray(result.skippedPrune) && result.skippedPrune.length === 0;});return reconcilePending;}@@ -299,7 +304,7 @@ export async function syncMediaLibraryFromPeer(peer) {// corruption self-heals even while the REMOTE manifest stays put. (The// recorded hash is in-memory, so a process restart also re-diffs; this covers// the mid-session window between restarts.)- if (lastLibraryManifestHash.get(peer.instanceId) === manifest.manifestHash) {+ if (!libraryIndexRepairPending.has(peer.instanceId) && lastLibraryManifestHash.get(peer.instanceId) === manifest.manifestHash) {const skips = (libraryUnchangedSkips.get(peer.instanceId) || 0) + 1;if (skips < FORCE_REVALIDATE_EVERY) {libraryUnchangedSkips.set(peer.instanceId, skips);@@ -308,33 +313,30 @@ export async function syncMediaLibraryFromPeer(peer) {libraryUnchangedSkips.set(peer.instanceId, 0); // periodic forced re-diff — fall through}const missing = await diffAssetManifestAgainstLocal(manifest.assets);- if (missing.length === 0) {- lastLibraryManifestHash.set(peer.instanceId, manifest.manifestHash);- return { pulled: 0 };- }const requested = missing.length;- // Reuse the per-record pull worker (in-flight dedup, image-sidecar fetch,- // video-thumbnail regen).- await pullMissingAssetsFromPeer(peer.instanceId, missing);- // `pullMissingAssetsFromPeer` swallows per-asset failures (peer drops- // mid-sweep, 404, size-cap reject) and always resolves — so a resolved pull- // does NOT mean every byte landed. Re-diff against disk to see what actually- // arrived; this is the authoritative signal, not the pull's resolution.- const stillMissing = await diffAssetManifestAgainstLocal(manifest.assets);+ let stillMissing = missing;+ if (requested > 0) {+ // Reuse the per-record pull worker (in-flight dedup, image-sidecar fetch,+ // video-thumbnail regen). It swallows per-asset failures, so re-diff disk+ // rather than treating the pull's resolution as evidence bytes landed.+ await pullMissingAssetsFromPeer(peer.instanceId, missing);+ stillMissing = await diffAssetManifestAgainstLocal(manifest.assets);+ }const pulled = requested - stillMissing.length;- // Rebuild the derived media_assets index when any image/video bytes landed so- // the gallery/Media tab reflects them. Idempotent; best-effort.- if (pulled > 0) await reconcileMediaLibraryIndex();- if (stillMissing.length === 0) {- // Full sweep — safe to short-circuit future ticks on this manifestHash.+ if (pulled > 0) libraryIndexRepairPending.add(peer.instanceId);+ if (libraryIndexRepairPending.has(peer.instanceId) && await reconcileMediaLibraryIndex()) {+ libraryIndexRepairPending.delete(peer.instanceId);+ }+ if (stillMissing.length === 0 && !libraryIndexRepairPending.has(peer.instanceId)) {+ // Full sweep, including index repair — safe to short-circuit future ticks on this manifestHash.lastLibraryManifestHash.set(peer.instanceId, manifest.manifestHash);- console.log(`📥 peerSync: media-library sweep from ${peer.name || peer.instanceId} — pulled ${pulled} asset(s)`);- } else {+ if (pulled > 0) console.log(`📥 peerSync: media-library sweep from ${peer.name || peer.instanceId} — pulled ${pulled} asset(s)`);+ } else if (stillMissing.length > 0) {// Partial pull — do NOT record the hash, so the next tick re-diffs and// retries the still-missing assets instead of being marked done.console.log(`⚠️ peerSync: media-library sweep from ${peer.name || peer.instanceId} — pulled ${pulled}/${requested}, ${stillMissing.length} still missing; retrying next tick`);}- return { pulled, missing: stillMissing.length };+ return requested > 0 ? { pulled, missing: stillMissing.length } : { pulled };} finally {librarySweepInFlight.delete(peer.instanceId);}diff --git a/server/services/sharing/peerMediaLibrarySync.test.js b/server/services/sharing/peerMediaLibrarySync.test.jsnew file mode 100644index 0000000000..3129be3183--- /dev/null+++ b/server/services/sharing/peerMediaLibrarySync.test.js@@ -0,0 +1,157 @@+import { afterAll, afterEach, beforeEach, describe, expect, it, vi } from 'vitest';+import { mkdir, readFile, readdir, rm } from 'node:fs/promises';+import { join } from 'node:path';+import { createHash } from 'node:crypto';+import { PORTOS_SCHEMA_VERSIONS } from '../../lib/schemaVersions.js';+import { createTempDataRoot, makePathsProxy } from '../../lib/mockPathsDataRoot.js';++let tempRoot;+vi.mock('../../lib/fileUtils.js', async importOriginal => makePathsProxy(await importOriginal(), { dataRoot: () => tempRoot }));+vi.mock('../../lib/paths.js', async importOriginal => makePathsProxy(await importOriginal(), { dataRoot: () => tempRoot }));+vi.mock('../instances.js', () => ({ getPeers: vi.fn(async () => []) }));+vi.mock('../../lib/peerHttpClient.js', async importOriginal => ({ ...(await importOriginal()), peerFetch: vi.fn() }));+vi.mock('./peerSyncAssets.js', async importOriginal => {+ const actual = await importOriginal();+ return { ...actual, diffAssetManifestAgainstLocal: vi.fn(actual.diffAssetManifestAgainstLocal) };+});+vi.mock('../../lib/databaseMaintenanceJournal.js', async importOriginal => ({ ...(await importOriginal()), assertDatabaseAdmission: () => {} }));++const indexAdapter = vi.hoisted(() => ({ reconcile: vi.fn(), indexImage: vi.fn() }));+vi.mock('../mediaAssetIndex/index.js', () => ({ reconcileMediaAssets: indexAdapter.reconcile, indexImage: indexAdapter.indexImage }));++// Load after the temp redirect exists; no production peers, files or DB are used.+tempRoot = createTempDataRoot('portos-library-repair-');+const { PATHS } = await import('../../lib/fileUtils.js');+const { getPeers } = await import('../instances.js');+const { peerFetch } = await import('../../lib/peerHttpClient.js');+const { syncMediaLibraryFromPeer } = await import('./peerMediaLibrarySync.js');+const { diffAssetManifestAgainstLocal } = await import('./peerSyncAssets.js');+const { acquireBackupSnapshotCut } = await import('../../lib/backupSnapshotBoundary.js');++const rows = new Map();+const bytes = Buffer.from('synthetic-peer-image');+let sequence = 0;+const peer = () => ({ instanceId: `repair-peer-${++sequence}`, address: '192.0.2.10', port: 5555, fullSync: true });+const deferred = () => {+ let resolve;+ const promise = new Promise(r => { resolve = r; });+ return { promise, resolve };+};+async function repairIndex() {+ for (const filename of await readdir(PATHS.images)) {+ if (filename.endsWith('.png')) rows.set(filename, await readFile(join(PATHS.images, filename)));+ }+ return { ok: true, skippedPrune: [] };+}+function advertise(peers, filenames = ['repair.png']) {+ vi.mocked(getPeers).mockResolvedValue(peers);+ const manifest = {+ schemaVersion: PORTOS_SCHEMA_VERSIONS.mediaLibrary, manifestHash: 'a'.repeat(64),+ assets: filenames.map(filename => ({ kind: 'image', filename, sha256: createHash('sha256').update(bytes).digest('hex') })),+ };+ vi.mocked(peerFetch).mockImplementation(async url => {+ if (String(url).endsWith('/library-manifest')) return { ok: true, json: async () => manifest };+ if (String(url).endsWith('.png')) return { ok: true, headers: new Headers({ 'content-length': String(bytes.length) }), arrayBuffer: async () => bytes };+ return { ok: false }; // no optional metadata sidecar+ });+}+const imageDownloads = () => vi.mocked(peerFetch).mock.calls.filter(([url]) => String(url).endsWith('.png')).length;++beforeEach(async () => {+ rows.clear();+ await mkdir(PATHS.images, { recursive: true });+ vi.mocked(peerFetch).mockReset();+ vi.mocked(diffAssetManifestAgainstLocal).mockClear();+ indexAdapter.indexImage.mockReset().mockRejectedValue(new Error('synthetic publication index outage'));+ indexAdapter.reconcile.mockReset().mockImplementation(repairIndex);+ vi.doMock('../mediaAssetIndex/index.js', () => ({ reconcileMediaAssets: indexAdapter.reconcile, indexImage: indexAdapter.indexImage }));+});+afterEach(async () => {+ await rm(PATHS.images, { recursive: true, force: true });+});+afterAll(async () => { await rm(tempRoot, { recursive: true, force: true }); });++describe('peer library index recovery', () => {+ it.each(['write rejection', 'incomplete authoritative read'])('repairs landed images on the identical next sweep after %s, then resumes unchanged shortcuts', async failure => {+ const source = peer();+ advertise([source]);+ if (failure === 'write rejection') indexAdapter.reconcile.mockRejectedValueOnce(new Error('synthetic index write outage'));+ else indexAdapter.reconcile.mockResolvedValueOnce({ ok: true, indexed: 0, skippedPrune: ['images'] });++ expect(await syncMediaLibraryFromPeer(source)).toEqual({ pulled: 1, missing: 0 });+ expect(await readFile(join(PATHS.images, 'repair.png'))).toEqual(bytes);+ expect(rows.size).toBe(0);++ expect(await syncMediaLibraryFromPeer(source)).toEqual({ pulled: 0 });+ expect(rows.get('repair.png')).toEqual(bytes);+ expect(imageDownloads()).toBe(1);+ expect(indexAdapter.reconcile).toHaveBeenCalledTimes(2);+ expect(indexAdapter.reconcile).toHaveBeenLastCalledWith({ requireComplete: true });+ for (let tick = 0; tick < 3; tick++) expect((await syncMediaLibraryFromPeer(source)).skipped).toBe('unchanged');+ expect(indexAdapter.reconcile).toHaveBeenCalledTimes(2);+ });++ it('recovers when the index module cannot load, without losing pending repair or downloading again', async () => {+ const source = peer();+ advertise([source]);+ vi.doMock('../mediaAssetIndex/index.js', () => { throw new Error('synthetic unavailable index module'); });+ expect(await syncMediaLibraryFromPeer(source)).toEqual({ pulled: 1, missing: 0 });+ expect(rows.size).toBe(0);+ vi.doMock('../mediaAssetIndex/index.js', () => ({ reconcileMediaAssets: indexAdapter.reconcile, indexImage: indexAdapter.indexImage }));+ expect(await syncMediaLibraryFromPeer(source)).toEqual({ pulled: 0 });+ expect(rows.get('repair.png')).toEqual(bytes);+ expect(imageDownloads()).toBe(1);+ expect((await syncMediaLibraryFromPeer(source)).skipped).toBe('unchanged');+ });++ it('waits for backup admission before repairing already landed bytes', async () => {+ const source = peer();+ advertise([source]);+ indexAdapter.reconcile.mockRejectedValueOnce(new Error('synthetic outage'));+ await syncMediaLibraryFromPeer(source);+ const release = await acquireBackupSnapshotCut();+ let retry;+ try {+ retry = syncMediaLibraryFromPeer(source);+ await vi.waitFor(() => expect(peerFetch).toHaveBeenCalledTimes(4));+ // Drain the request/import continuation; its index write is awaiting the cut.+ await new Promise(resolve => setImmediate(resolve));+ expect(indexAdapter.reconcile).toHaveBeenCalledTimes(1);+ expect(rows.size).toBe(0);+ } finally { release(); }+ await retry;+ expect(rows.get('repair.png')).toEqual(bytes);+ expect(indexAdapter.reconcile).toHaveBeenCalledTimes(2);+ expect(imageDownloads()).toBe(1);+ });++ it('serializes active repairs and coalesces queued peers while retaining failure-to-retry state', async () => {+ const sources = [peer(), peer(), peer()];+ // Distinct files ensure each peer has landed bytes and owns a pending repair.+ indexAdapter.reconcile.mockRejectedValue(new Error('synthetic outage'));+ for (let i = 0; i < sources.length; i++) {+ advertise(sources, [`repair-${i}.png`]);+ await syncMediaLibraryFromPeer(sources[i]);+ }+ advertise(sources, ['repair-0.png', 'repair-1.png', 'repair-2.png']);+ const firstStarted = deferred();+ const firstRelease = deferred();+ indexAdapter.reconcile.mockReset().mockImplementationOnce(async () => {+ firstStarted.resolve();+ await firstRelease.promise;+ return repairIndex();+ }).mockImplementation(repairIndex);+ const first = syncMediaLibraryFromPeer(sources[0]);+ await firstStarted.promise;+ const queued = sources.slice(1).map(syncMediaLibraryFromPeer);+ await vi.waitFor(() => expect(vi.mocked(diffAssetManifestAgainstLocal).mock.settledResults.filter(result => result.type === 'fulfilled')).toHaveLength(9));+ await new Promise(resolve => setImmediate(resolve));+ expect(indexAdapter.reconcile).toHaveBeenCalledTimes(1);+ firstRelease.resolve();+ await Promise.all([first, ...queued]);+ expect(indexAdapter.reconcile).toHaveBeenCalledTimes(2);+ expect([...rows.keys()].sort()).toEqual(['repair-0.png', 'repair-1.png', 'repair-2.png']);+ expect(imageDownloads()).toBe(3);+ for (const source of sources) expect((await syncMediaLibraryFromPeer(source)).skipped).toBe('unchanged');+ });+});diff --git a/server/services/sharing/peerSync.test.js b/server/services/sharing/peerSync.test.jsindex 5d0bfec690..f0c50d0a22 100644--- a/server/services/sharing/peerSync.test.js+++ b/server/services/sharing/peerSync.test.js@@ -81,7 +81,7 @@ vi.mock('../mediaCollections.js', async () => ({// media_assets index after bytes land. Mock it to a no-op spy so the sweep tests// don't touch Postgres and can assert the reconcile fired.vi.mock('../mediaAssetIndex/index.js', () => ({- reconcileMediaAssets: vi.fn().mockResolvedValue(undefined),+ reconcileMediaAssets: vi.fn().mockResolvedValue({ ok: true, skippedPrune: [] }),}));vi.mock('../artists/index.js', async () => {