Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

fix: re-export share-bucket annotations at boot and retry failed exports (#12623)

[._.]/ Adam Eivy3ffe915fe280
diff --git a/server/services/sharing/annotationsSync.js b/server/services/sharing/annotationsSync.js
index 7c9438070f..57d6b5b956 100644
--- a/server/services/sharing/annotationsSync.js
+++ b/server/services/sharing/annotationsSync.js
@@ -19,7 +19,12 @@ import { resolveBucketSourceName } from './annotationIdentity.js';
import { onLocalAnnotationChange, listLocalAuthorAnnotations } from '../mediaAnnotations.js';
const DEBOUNCE_MS = 2000;
+// Retry delays after a failed bucket export (unmounted/unreachable folder);
+// the last entry repeats until an export succeeds or a local change resets it.
+const RETRY_BACKOFF_MS = [60_000, 300_000, 900_000];
let pendingTimer = null;
+let retryTimer = null;
+let retryAttempt = 0;
let installed = false;
/**
@@ -110,11 +115,32 @@ async function listBucketAssetKeys(bucketPath) {
return new Set([...manifestKeys, ...legacyKeys]);
}
+/** Test hook — drop listener-installed state and pending timers. */
+export function __resetAnnotationsSyncForTests() {
+ installed = false;
+ if (pendingTimer) clearTimeout(pendingTimer);
+ pendingTimer = null;
+ clearRetry();
+}
+
/** Test hook — reset the per-bucket manifest-keys cache between cases. */
export function __resetBucketAssetKeysCache() {
manifestKeysCache.clear();
}
+function isSameAnnotationSet(a, b) {
+ const keysA = Object.keys(a).sort();
+ const keysB = Object.keys(b).sort();
+ if (keysA.length !== keysB.length) return false;
+ return keysA.every((key, i) => key === keysB[i] && JSON.stringify(sortedEntry(a[key])) === JSON.stringify(sortedEntry(b[key])));
+}
+
+function sortedEntry(entry) {
+ return entry && typeof entry === 'object'
+ ? Object.fromEntries(Object.entries(entry).sort(([x], [y]) => (x < y ? -1 : 1)))
+ : entry;
+}
+
export async function exportAnnotationsToBucket(bucket, localAnnotations, senderInstanceId) {
if (bucket.mode !== 'auto-merge') return { skipped: true, reason: 'not-auto-merge' };
const recordDir = join(bucket.path, 'records', 'media-annotations');
@@ -154,6 +180,13 @@ export async function exportAnnotationsToBucket(bucket, localAnnotations, sender
: [];
for (const key of priorKeys) {
if (filtered[key]) continue;
+ // Keep an already-published tombstone verbatim so an unchanged state
+ // compares equal below instead of being rewritten with a fresh timestamp.
+ const priorEntry = prior.annotations[key];
+ if (priorEntry && priorEntry.starred === false && priorEntry.note === '' && assetKeys.has(key)) {
+ filtered[key] = priorEntry;
+ continue;
+ }
// Only tombstone keys whose assets are still in this bucket — pruning a
// key whose asset has been removed from the bucket would tell peers to
// delete the note even though they may still hold the asset.
@@ -168,6 +201,14 @@ export async function exportAnnotationsToBucket(bucket, localAnnotations, sender
return { skipped: true, reason: 'no-annotations-for-bucket' };
}
+ // Idempotent re-export: boot and retry flushes must not rewrite a manifest
+ // whose content peers already hold. The record is written before the
+ // manifest, so a missing manifest means a previous export died midway.
+ if (prior && prior.annotations && isSameAnnotationSet(prior.annotations, filtered)
+ && existsSync(join(bucket.path, 'manifests', annotationManifestFilename(senderInstanceId)))) {
+ return { skipped: true, reason: 'unchanged' };
+ }
+
const record = {
id: recordId,
instanceId: senderInstanceId,
@@ -213,21 +254,53 @@ async function flushAll() {
// A failed source read is not a deletion: abort before tombstone synthesis.
listLocalAuthorAnnotations(),
]);
- if (!senderInstanceId || senderInstanceId === UNKNOWN_INSTANCE_ID) return;
+ if (!senderInstanceId || senderInstanceId === UNKNOWN_INSTANCE_ID) return true;
const autoMerge = buckets.filter((b) => b.mode === 'auto-merge');
- if (autoMerge.length === 0) return;
+ if (autoMerge.length === 0) return true;
+ let allOk = true;
await Promise.all(autoMerge.map(async (bucket) => {
const res = await exportAnnotationsToBucket(bucket, localAnnotations, senderInstanceId).catch((err) => {
console.error(`⚠️ sharing.annotations: export to bucket=${bucket.name} failed: ${err.message}`);
+ allOk = false;
return null;
});
if (res && !res.skipped) {
console.log(`📤 sharing.annotations: bucket=${bucket.name} wrote ${res.entryCount} annotation(s) (${res.filename})`);
}
}));
+ return allOk;
+}
+
+function clearRetry() {
+ if (retryTimer) clearTimeout(retryTimer);
+ retryTimer = null;
+ retryAttempt = 0;
+}
+
+function scheduleRetry() {
+ if (retryTimer) return;
+ const delay = RETRY_BACKOFF_MS[Math.min(retryAttempt, RETRY_BACKOFF_MS.length - 1)];
+ retryAttempt += 1;
+ retryTimer = setTimeout(() => {
+ retryTimer = null;
+ runFlush();
+ }, delay);
+ retryTimer.unref?.();
+}
+
+function runFlush() {
+ return flushAll().then((ok) => {
+ if (ok) clearRetry();
+ else scheduleRetry();
+ }).catch((err) => {
+ console.error(`⚠️ sharing.annotations: flushAll failed: ${err.message}`);
+ scheduleRetry();
+ });
}
function scheduleFlush() {
+ // A new local change supersedes any pending retry and resets its backoff.
+ clearRetry();
// True debounce: restart the timer on every call so a burst of rapid edits
// collapses into one flush 2s after the LAST change. The previous
// "if (pendingTimer) return" was a throttle — the first edit started the
@@ -237,9 +310,7 @@ function scheduleFlush() {
if (pendingTimer) clearTimeout(pendingTimer);
pendingTimer = setTimeout(() => {
pendingTimer = null;
- flushAll().catch((err) => {
- console.error(`⚠️ sharing.annotations: flushAll failed: ${err.message}`);
- });
+ runFlush();
}, DEBOUNCE_MS);
pendingTimer.unref?.();
}
@@ -249,4 +320,7 @@ export function initAnnotationsSync() {
if (installed) return;
installed = true;
onLocalAnnotationChange(() => scheduleFlush());
+ // An edit made just before a restart (or during a failed export) never
+ // reached the bucket; the export is idempotent so a boot flush is cheap.
+ scheduleFlush();
}
diff --git a/server/services/sharing/annotationsSync.test.js b/server/services/sharing/annotationsSync.test.js
index 1bc946b65d..68ea55cf86 100644
--- a/server/services/sharing/annotationsSync.test.js
+++ b/server/services/sharing/annotationsSync.test.js
@@ -23,7 +23,7 @@ vi.mock('./manifest.js', () => ({
writeManifest: writeManifestMock,
readManifest: vi.fn(),
listManifestFilenames: vi.fn(async () => []),
- annotationManifestFilename: vi.fn(),
+ annotationManifestFilename: vi.fn(() => 'annotations-local-instance.json'),
}));
vi.mock('./version.js', () => ({ getProducedByVersion: vi.fn(async () => '1.0.0-test') }));
@@ -312,7 +312,7 @@ describe('sharing/annotationsSync.listBucketAssetKeys cache', () => {
const ann = { 'image:a.png': { starred: true, note: 'hi', updatedAt: '2026-01-01T00:00:00.000Z' } };
const r1 = await svc.exportAnnotationsToBucket(bucket(), ann, 'local-instance');
- const r2 = await svc.exportAnnotationsToBucket(bucket(), ann, 'local-instance');
+ const r2 = await svc.exportAnnotationsToBucket(bucket(), { 'image:a.png': { ...ann['image:a.png'], note: 'edited' } }, 'local-instance');
fs.existsSync.mockReturnValue(false);
expect(r1.skipped).toBe(false);
expect(r2.skipped).toBe(false);
@@ -436,4 +436,59 @@ describe('sharing/annotationsSync event flush', () => {
errorLog.mockRestore();
}
});
+
+ it('flushes once at boot, skips an unchanged manifest, and retries a failed export on backoff until it succeeds', async () => {
+ const { listLocalAuthorAnnotations } = await import('../mediaAnnotations.js');
+ const { listBuckets } = await import('./buckets.js');
+ const { listManifestFilenames, readManifest } = await import('./manifest.js');
+ const { existsSync } = await import('fs');
+ const recordPath = '/mock/bucket/records/media-annotations/local-instance.json';
+ const local = { 'image:kept.png': { starred: true, note: 'Boot', updatedAt: '2026-01-01T00:00:00.000Z' } };
+ fileStore.clear();
+ writeCalls.length = 0;
+ writeManifestMock.mockClear();
+ svc.__resetBucketAssetKeysCache();
+ listBuckets.mockResolvedValue([bucket()]);
+ listManifestFilenames.mockResolvedValue(['asset.json']);
+ readManifest.mockResolvedValue({ assetRefs: [{ kind: 'image', ref: 'kept.png' }] });
+ // The manifest file only "exists" once writeManifest has succeeded.
+ let manifestOnDisk = false;
+ writeManifestMock.mockImplementation(async (bucketPath, manifest) => {
+ manifestOnDisk = true;
+ return `${manifest.kind}.manifest.json`;
+ });
+ existsSync.mockImplementation((p) => !toPosix(String(p)).endsWith('annotations-local-instance.json') || manifestOnDisk);
+ listLocalAuthorAnnotations.mockResolvedValue(local);
+ const errorLog = vi.spyOn(console, 'error').mockImplementation(() => {});
+ const logSpy = vi.spyOn(console, 'log').mockImplementation(() => {});
+ vi.useFakeTimers();
+ try {
+ // First export fails (unreachable bucket), then recovers on the 1 min retry.
+ writeManifestMock.mockImplementationOnce(async () => { throw new Error('bucket offline'); });
+ svc.__resetAnnotationsSyncForTests();
+ svc.initAnnotationsSync();
+ await vi.advanceTimersByTimeAsync(2000);
+ expect(errorLog).toHaveBeenCalledWith(expect.stringContaining('bucket offline'));
+ expect(writeManifestMock).toHaveBeenCalledTimes(1);
+
+ await vi.advanceTimersByTimeAsync(59_000);
+ expect(writeManifestMock).toHaveBeenCalledTimes(1);
+ await vi.advanceTimersByTimeAsync(1000);
+ expect(writeManifestMock).toHaveBeenCalledTimes(2);
+ expect(fileStore.get(recordPath).annotations['image:kept.png'].note).toBe('Boot');
+
+ // Success stops retries; a later boot with identical content writes nothing.
+ await vi.advanceTimersByTimeAsync(20 * 60_000);
+ expect(writeManifestMock).toHaveBeenCalledTimes(2);
+ svc.__resetAnnotationsSyncForTests();
+ svc.initAnnotationsSync();
+ await vi.advanceTimersByTimeAsync(2000);
+ expect(writeManifestMock).toHaveBeenCalledTimes(2);
+ } finally {
+ svc.__resetAnnotationsSyncForTests();
+ vi.useRealTimers();
+ errorLog.mockRestore();
+ logSpy.mockRestore();
+ }
+ });
});