Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
fix: re-export share-bucket annotations at boot and retry failed exports (#12623)
[._.]/ Adam Eivy3ffe915fe280
diff --git a/server/services/sharing/annotationsSync.js b/server/services/sharing/annotationsSync.jsindex 7c9438070f..57d6b5b956 100644--- a/server/services/sharing/annotationsSync.js+++ b/server/services/sharing/annotationsSync.js@@ -19,7 +19,12 @@ import { resolveBucketSourceName } from './annotationIdentity.js';import { onLocalAnnotationChange, listLocalAuthorAnnotations } from '../mediaAnnotations.js';const DEBOUNCE_MS = 2000;+// Retry delays after a failed bucket export (unmounted/unreachable folder);+// the last entry repeats until an export succeeds or a local change resets it.+const RETRY_BACKOFF_MS = [60_000, 300_000, 900_000];let pendingTimer = null;+let retryTimer = null;+let retryAttempt = 0;let installed = false;/**@@ -110,11 +115,32 @@ async function listBucketAssetKeys(bucketPath) {return new Set([...manifestKeys, ...legacyKeys]);}+/** Test hook — drop listener-installed state and pending timers. */+export function __resetAnnotationsSyncForTests() {+ installed = false;+ if (pendingTimer) clearTimeout(pendingTimer);+ pendingTimer = null;+ clearRetry();+}+/** Test hook — reset the per-bucket manifest-keys cache between cases. */export function __resetBucketAssetKeysCache() {manifestKeysCache.clear();}+function isSameAnnotationSet(a, b) {+ const keysA = Object.keys(a).sort();+ const keysB = Object.keys(b).sort();+ if (keysA.length !== keysB.length) return false;+ return keysA.every((key, i) => key === keysB[i] && JSON.stringify(sortedEntry(a[key])) === JSON.stringify(sortedEntry(b[key])));+}++function sortedEntry(entry) {+ return entry && typeof entry === 'object'+ ? Object.fromEntries(Object.entries(entry).sort(([x], [y]) => (x < y ? -1 : 1)))+ : entry;+}+export async function exportAnnotationsToBucket(bucket, localAnnotations, senderInstanceId) {if (bucket.mode !== 'auto-merge') return { skipped: true, reason: 'not-auto-merge' };const recordDir = join(bucket.path, 'records', 'media-annotations');@@ -154,6 +180,13 @@ export async function exportAnnotationsToBucket(bucket, localAnnotations, sender: [];for (const key of priorKeys) {if (filtered[key]) continue;+ // Keep an already-published tombstone verbatim so an unchanged state+ // compares equal below instead of being rewritten with a fresh timestamp.+ const priorEntry = prior.annotations[key];+ if (priorEntry && priorEntry.starred === false && priorEntry.note === '' && assetKeys.has(key)) {+ filtered[key] = priorEntry;+ continue;+ }// Only tombstone keys whose assets are still in this bucket — pruning a// key whose asset has been removed from the bucket would tell peers to// delete the note even though they may still hold the asset.@@ -168,6 +201,14 @@ export async function exportAnnotationsToBucket(bucket, localAnnotations, senderreturn { skipped: true, reason: 'no-annotations-for-bucket' };}+ // Idempotent re-export: boot and retry flushes must not rewrite a manifest+ // whose content peers already hold. The record is written before the+ // manifest, so a missing manifest means a previous export died midway.+ if (prior && prior.annotations && isSameAnnotationSet(prior.annotations, filtered)+ && existsSync(join(bucket.path, 'manifests', annotationManifestFilename(senderInstanceId)))) {+ return { skipped: true, reason: 'unchanged' };+ }+const record = {id: recordId,instanceId: senderInstanceId,@@ -213,21 +254,53 @@ async function flushAll() {// A failed source read is not a deletion: abort before tombstone synthesis.listLocalAuthorAnnotations(),]);- if (!senderInstanceId || senderInstanceId === UNKNOWN_INSTANCE_ID) return;+ if (!senderInstanceId || senderInstanceId === UNKNOWN_INSTANCE_ID) return true;const autoMerge = buckets.filter((b) => b.mode === 'auto-merge');- if (autoMerge.length === 0) return;+ if (autoMerge.length === 0) return true;+ let allOk = true;await Promise.all(autoMerge.map(async (bucket) => {const res = await exportAnnotationsToBucket(bucket, localAnnotations, senderInstanceId).catch((err) => {console.error(`⚠️ sharing.annotations: export to bucket=${bucket.name} failed: ${err.message}`);+ allOk = false;return null;});if (res && !res.skipped) {console.log(`📤 sharing.annotations: bucket=${bucket.name} wrote ${res.entryCount} annotation(s) (${res.filename})`);}}));+ return allOk;+}++function clearRetry() {+ if (retryTimer) clearTimeout(retryTimer);+ retryTimer = null;+ retryAttempt = 0;+}++function scheduleRetry() {+ if (retryTimer) return;+ const delay = RETRY_BACKOFF_MS[Math.min(retryAttempt, RETRY_BACKOFF_MS.length - 1)];+ retryAttempt += 1;+ retryTimer = setTimeout(() => {+ retryTimer = null;+ runFlush();+ }, delay);+ retryTimer.unref?.();+}++function runFlush() {+ return flushAll().then((ok) => {+ if (ok) clearRetry();+ else scheduleRetry();+ }).catch((err) => {+ console.error(`⚠️ sharing.annotations: flushAll failed: ${err.message}`);+ scheduleRetry();+ });}function scheduleFlush() {+ // A new local change supersedes any pending retry and resets its backoff.+ clearRetry();// True debounce: restart the timer on every call so a burst of rapid edits// collapses into one flush 2s after the LAST change. The previous// "if (pendingTimer) return" was a throttle — the first edit started the@@ -237,9 +310,7 @@ function scheduleFlush() {if (pendingTimer) clearTimeout(pendingTimer);pendingTimer = setTimeout(() => {pendingTimer = null;- flushAll().catch((err) => {- console.error(`⚠️ sharing.annotations: flushAll failed: ${err.message}`);- });+ runFlush();}, DEBOUNCE_MS);pendingTimer.unref?.();}@@ -249,4 +320,7 @@ export function initAnnotationsSync() {if (installed) return;installed = true;onLocalAnnotationChange(() => scheduleFlush());+ // An edit made just before a restart (or during a failed export) never+ // reached the bucket; the export is idempotent so a boot flush is cheap.+ scheduleFlush();}diff --git a/server/services/sharing/annotationsSync.test.js b/server/services/sharing/annotationsSync.test.jsindex 1bc946b65d..68ea55cf86 100644--- a/server/services/sharing/annotationsSync.test.js+++ b/server/services/sharing/annotationsSync.test.js@@ -23,7 +23,7 @@ vi.mock('./manifest.js', () => ({writeManifest: writeManifestMock,readManifest: vi.fn(),listManifestFilenames: vi.fn(async () => []),- annotationManifestFilename: vi.fn(),+ annotationManifestFilename: vi.fn(() => 'annotations-local-instance.json'),}));vi.mock('./version.js', () => ({ getProducedByVersion: vi.fn(async () => '1.0.0-test') }));@@ -312,7 +312,7 @@ describe('sharing/annotationsSync.listBucketAssetKeys cache', () => {const ann = { 'image:a.png': { starred: true, note: 'hi', updatedAt: '2026-01-01T00:00:00.000Z' } };const r1 = await svc.exportAnnotationsToBucket(bucket(), ann, 'local-instance');- const r2 = await svc.exportAnnotationsToBucket(bucket(), ann, 'local-instance');+ const r2 = await svc.exportAnnotationsToBucket(bucket(), { 'image:a.png': { ...ann['image:a.png'], note: 'edited' } }, 'local-instance');fs.existsSync.mockReturnValue(false);expect(r1.skipped).toBe(false);expect(r2.skipped).toBe(false);@@ -436,4 +436,59 @@ describe('sharing/annotationsSync event flush', () => {errorLog.mockRestore();}});++ it('flushes once at boot, skips an unchanged manifest, and retries a failed export on backoff until it succeeds', async () => {+ const { listLocalAuthorAnnotations } = await import('../mediaAnnotations.js');+ const { listBuckets } = await import('./buckets.js');+ const { listManifestFilenames, readManifest } = await import('./manifest.js');+ const { existsSync } = await import('fs');+ const recordPath = '/mock/bucket/records/media-annotations/local-instance.json';+ const local = { 'image:kept.png': { starred: true, note: 'Boot', updatedAt: '2026-01-01T00:00:00.000Z' } };+ fileStore.clear();+ writeCalls.length = 0;+ writeManifestMock.mockClear();+ svc.__resetBucketAssetKeysCache();+ listBuckets.mockResolvedValue([bucket()]);+ listManifestFilenames.mockResolvedValue(['asset.json']);+ readManifest.mockResolvedValue({ assetRefs: [{ kind: 'image', ref: 'kept.png' }] });+ // The manifest file only "exists" once writeManifest has succeeded.+ let manifestOnDisk = false;+ writeManifestMock.mockImplementation(async (bucketPath, manifest) => {+ manifestOnDisk = true;+ return `${manifest.kind}.manifest.json`;+ });+ existsSync.mockImplementation((p) => !toPosix(String(p)).endsWith('annotations-local-instance.json') || manifestOnDisk);+ listLocalAuthorAnnotations.mockResolvedValue(local);+ const errorLog = vi.spyOn(console, 'error').mockImplementation(() => {});+ const logSpy = vi.spyOn(console, 'log').mockImplementation(() => {});+ vi.useFakeTimers();+ try {+ // First export fails (unreachable bucket), then recovers on the 1 min retry.+ writeManifestMock.mockImplementationOnce(async () => { throw new Error('bucket offline'); });+ svc.__resetAnnotationsSyncForTests();+ svc.initAnnotationsSync();+ await vi.advanceTimersByTimeAsync(2000);+ expect(errorLog).toHaveBeenCalledWith(expect.stringContaining('bucket offline'));+ expect(writeManifestMock).toHaveBeenCalledTimes(1);++ await vi.advanceTimersByTimeAsync(59_000);+ expect(writeManifestMock).toHaveBeenCalledTimes(1);+ await vi.advanceTimersByTimeAsync(1000);+ expect(writeManifestMock).toHaveBeenCalledTimes(2);+ expect(fileStore.get(recordPath).annotations['image:kept.png'].note).toBe('Boot');++ // Success stops retries; a later boot with identical content writes nothing.+ await vi.advanceTimersByTimeAsync(20 * 60_000);+ expect(writeManifestMock).toHaveBeenCalledTimes(2);+ svc.__resetAnnotationsSyncForTests();+ svc.initAnnotationsSync();+ await vi.advanceTimersByTimeAsync(2000);+ expect(writeManifestMock).toHaveBeenCalledTimes(2);+ } finally {+ svc.__resetAnnotationsSyncForTests();+ vi.useRealTimers();+ errorLog.mockRestore();+ logSpy.mockRestore();+ }+ });});