Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

fix: publish verified bucket assets atomically (#12564)

[._.]/ Adam Eivyb344c5437233
diff --git a/server/services/sharing/assetLimits.js b/server/services/sharing/assetLimits.js
index f76b386c26..85d04a38d5 100644
--- a/server/services/sharing/assetLimits.js
+++ b/server/services/sharing/assetLimits.js
@@ -1,2 +1,3 @@
-// Shared ingress ceiling for peer pulls and bucket files, including sidecars.
+// Shared ingress ceilings for peer pulls and bucket files.
export const ASSET_PULL_MAX_BYTES = 100 * 1024 * 1024;
+export const SIDECAR_MAX_BYTES = 256 * 1024;
diff --git a/server/services/sharing/importer.js b/server/services/sharing/importer.js
index 35aa7d3ad5..77fbb2f1c8 100644
--- a/server/services/sharing/importer.js
+++ b/server/services/sharing/importer.js
@@ -22,13 +22,13 @@
*/
import { createKeyCachedQueue } from '../../lib/createKeyCachedQueue.js';
-import { join, basename, relative, resolve, sep } from 'path';
-import { createHash } from 'crypto';
-import { readdir, lstat, realpath, open } from 'fs/promises';
+import { join, basename, relative, resolve, sep, dirname } from 'path';
+import { createHash, randomUUID } from 'crypto';
+import { readdir, lstat, realpath, open, link } from 'fs/promises';
import { existsSync, constants } from 'fs';
import { EventEmitter } from 'events';
import { withBackupAssetPublication } from '../../lib/backupSnapshotBoundary.js';
-import { PATHS, writeFileGuarded, ensureDir, atomicWrite, readJSONFile } from '../../lib/fileUtils.js';
+import { PATHS, unlinkGuarded, ensureDir, atomicWrite, readJSONFile } from '../../lib/fileUtils.js';
import { isSafeRecordId } from '../../lib/validation.js';
import { logFailureWithStack } from '../../lib/failureLogging.js';
import { getBucket, bucketBlobPath, bucketBlobSidecarPath, bucketRecordsDir, bucketRecordPath, imageSidecarName, isHexHash } from './buckets.js';
@@ -49,7 +49,7 @@ import { isPlainObject } from '../../lib/objects.js';
import { maybeJournalBeforeOverwrite, flushBaseHashes, setSyncBaseHash, contentHashForRecord } from '../../lib/conflictJournal.js';
import { isStr } from '../../lib/textUtils.js';
import { detectImageFormat, detectMediaManifest } from '../../lib/mimeTypes.js';
-import { ASSET_PULL_MAX_BYTES } from './assetLimits.js';
+import { ASSET_PULL_MAX_BYTES, SIDECAR_MAX_BYTES } from './assetLimits.js';
import { mapWithConcurrency } from '../../lib/mapWithConcurrency.js';
// Record kinds that participate in the non-blocking conflict journal. Universe,
@@ -311,18 +311,18 @@ function copyAssetsLocally(bucketPath, assetRefs) {
// Pin reads to one regular-file descriptor. A cloud-sync writer can replace
// paths during inspection; neither a symlink nor a later path swap may select
// the bytes we publish. The extra byte detects growth without an unbounded read.
-async function readBucketAsset(bucketPath, sourcePath) {
+async function readBucketAsset(bucketPath, sourcePath, maxBytes) {
const root = await realpath(bucketPath);
const source = await realpath(sourcePath);
const expected = resolve(root, relative(resolve(bucketPath), resolve(sourcePath)));
if (source !== expected || !source.startsWith(`${root}${sep}`)) return null;
const info = await lstat(sourcePath);
- if (!info.isFile() || info.size > ASSET_PULL_MAX_BYTES) return null;
+ if (!info.isFile() || info.size > maxBytes) return null;
const handle = await open(sourcePath, constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK);
try {
const opened = await handle.stat();
if (!opened.isFile() || opened.dev !== info.dev || opened.ino !== info.ino
- || opened.size > ASSET_PULL_MAX_BYTES || await realpath(sourcePath) !== source) return null;
+ || opened.size > maxBytes || await realpath(sourcePath) !== source) return null;
const buffer = Buffer.alloc(opened.size + 1);
let length = 0;
while (length < buffer.length) {
@@ -337,20 +337,28 @@ async function readBucketAsset(bucketPath, sourcePath) {
}
}
-async function verifiedBucketAsset(bucketPath, sourcePath, { hash, image = false } = {}) {
- const buffer = await readBucketAsset(bucketPath, sourcePath).catch(() => null);
+async function verifiedBucketAsset(bucketPath, sourcePath, { hash, image = false, maxBytes = ASSET_PULL_MAX_BYTES } = {}) {
+ const buffer = await readBucketAsset(bucketPath, sourcePath, maxBytes).catch(() => null);
if (!buffer?.length || detectMediaManifest(buffer) || image && !detectImageFormat(buffer)) return null;
if (hash && createHash('sha256').update(buffer).digest('hex') !== hash) return null;
return buffer;
}
-// Exclusive creation preserves skip-if-present semantics even for overlapping
-// imports, without following a destination symlink created between checks.
-function publishBucketAsset(targetPath, buffer) {
- return writeFileGuarded(targetPath, buffer, { flag: 'wx' }).then(() => true).catch(err => {
- if (err?.code === 'EEXIST') return false;
- throw err;
- });
+// Publish durable bytes atomically without replacing an existing destination.
+// The short temporary name also supports asset filenames already at NAME_MAX.
+async function publishBucketAsset(targetPath, buffer) {
+ const temporary = join(dirname(targetPath), `.sharing-${randomUUID()}.tmp`);
+ try {
+ await atomicWrite(temporary, buffer);
+ return await link(temporary, targetPath).then(() => true).catch(err => {
+ if (err?.code === 'EEXIST') return false;
+ throw err;
+ });
+ } finally {
+ await unlinkGuarded(temporary).catch(err => {
+ if (err?.code !== 'ENOENT') throw err;
+ });
+ }
}
async function copyAssetsLocallyAdmitted(bucketPath, assetRefs) {
@@ -374,7 +382,7 @@ async function copyAssetsLocallyAdmitted(bucketPath, assetRefs) {
}
const buffer = await verifiedBucketAsset(bucketPath, blobPath, { hash: ref.hash, image: kind !== 'video' });
const hasSidecar = sidecarPath && existsSync(sidecarPath);
- const sidecar = hasSidecar ? await verifiedBucketAsset(bucketPath, sidecarPath) : null;
+ const sidecar = hasSidecar ? await verifiedBucketAsset(bucketPath, sidecarPath, { maxBytes: SIDECAR_MAX_BYTES }) : null;
if (!buffer || hasSidecar && !sidecar) {
console.warn(`⚠️ sharing.importer: refused asset ${kind}/${JSON.stringify(filename)}`);
missing.push({ kind, ref: filename });
diff --git a/server/services/sharing/integration.test.js b/server/services/sharing/integration.test.js
index c647adbbb3..60fd969262 100644
--- a/server/services/sharing/integration.test.js
+++ b/server/services/sharing/integration.test.js
@@ -13,6 +13,7 @@ import { mkdtempSync, rmSync, writeFileSync, existsSync, mkdirSync, readFileSync
import { tmpdir } from 'os';
import { join } from 'path';
import { createHash } from 'crypto';
+import { ASSET_PULL_MAX_BYTES, SIDECAR_MAX_BYTES } from './assetLimits.js';
import { makePathsProxy, mockNoPeerSync, mockNoPeers, mockTestIdentity, ownTestBodies } from '../../lib/mockPathsDataRoot.js';
const PNG = Buffer.from('iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAwMCAO+jRZkAAAAASUVORK5CYII=', 'base64');
@@ -20,7 +21,6 @@ const pngBytes = label => Buffer.concat([PNG, Buffer.from(label)]);
const fileUtilsMock = vi.hoisted(() => ({
atomicWrite: vi.fn(), realAtomicWrite: null,
- writeFileGuarded: vi.fn(), realWriteFileGuarded: null,
}));
function sha256Hex(buf) {
@@ -56,13 +56,11 @@ async function cleanupBuckets() {
vi.mock('../../lib/fileUtils.js', async () =>
makePathsProxy(await vi.importActual('../../lib/fileUtils.js'), {
dataRoot: tempData,
- overrides: { atomicWrite: fileUtilsMock.atomicWrite, writeFileGuarded: fileUtilsMock.writeFileGuarded },
+ overrides: { atomicWrite: fileUtilsMock.atomicWrite },
}));
const fileUtilsActual = await vi.importActual('../../lib/fileUtils.js');
fileUtilsMock.realAtomicWrite = fileUtilsActual.atomicWrite;
-fileUtilsMock.realWriteFileGuarded = fileUtilsActual.writeFileGuarded;
-fileUtilsMock.writeFileGuarded.mockImplementation((...args) => fileUtilsMock.realWriteFileGuarded(...args));
fileUtilsMock.atomicWrite.mockImplementation((...args) => fileUtilsMock.realAtomicWrite(...args));
// Stub instanceIdentity.getInstanceId so the exporter doesn't try to read the
@@ -116,8 +114,6 @@ function simulateRemoteSender(bucketPath, filename, peerId = 'remote-peer-id') {
describe('sharing round-trip', () => {
beforeEach(() => {
- fileUtilsMock.writeFileGuarded.mockReset();
- fileUtilsMock.writeFileGuarded.mockImplementation((...args) => fileUtilsMock.realWriteFileGuarded(...args));
fileUtilsMock.atomicWrite.mockClear();
fileUtilsMock.atomicWrite.mockImplementation((...args) => fileUtilsMock.realAtomicWrite(...args));
// Wipe and re-seed the temp data dir + a fake asset for each test.
@@ -232,7 +228,7 @@ describe('sharing round-trip', () => {
} else if (scenario.startsWith('oversized')) {
const oversized = scenario === 'oversized sidecar' ? `${source}.metadata.json` : source;
writeFileSync(oversized, '');
- truncateSync(oversized, 100 * 1024 * 1024 + 1);
+ truncateSync(oversized, (scenario === 'oversized sidecar' ? SIDECAR_MAX_BYTES : ASSET_PULL_MAX_BYTES) + 1);
}
const warn = vi.spyOn(console, 'warn').mockImplementation(() => {});
try {
@@ -258,20 +254,58 @@ describe('sharing round-trip', () => {
writeFileSync(source, PNG);
const target = join(tempData, 'images', ref.ref);
let replaced = false;
- fileUtilsMock.writeFileGuarded.mockImplementation((path, ...args) => {
- if (path === target) {
+ fileUtilsMock.atomicWrite.mockImplementation((path, ...args) => {
+ if (path.startsWith(join(tempData, 'images', '.sharing-'))) {
replaced = true;
rmSync(source);
symlinkSync(join(tempData, 'outside.png'), source, 'file');
writeFileSync(join(tempData, 'outside.png'), 'unverified replacement');
}
- return fileUtilsMock.realWriteFileGuarded(path, ...args);
+ return fileUtilsMock.realAtomicWrite(path, ...args);
});
expect(await importer.processManifest(bucket.id, writeAssetManifest([ref]))).toMatchObject({ processed: true });
expect(replaced).toBe(true);
expect(readFileSync(target)).toEqual(PNG);
});
+ it('leaves no final or temporary file after a failed publication, then imports on retry', async () => {
+ const bucket = await buckets.createBucket({ name: 'Publication failure', path: tempBucket, mode: 'auto-merge' });
+ const ref = { kind: 'image', ref: 'retry.png', hash: sha256Hex(PNG) };
+ mkdirSync(join(tempBucket, 'assets', 'blobs'), { recursive: true });
+ writeFileSync(buckets.bucketBlobPath(tempBucket, ref.hash), PNG);
+ const filename = writeAssetManifest([ref]);
+ const imageDir = join(tempData, 'images');
+ fileUtilsMock.atomicWrite.mockImplementation(async (path, ...args) => {
+ if (path.startsWith(join(imageDir, '.sharing-'))) {
+ await fileUtilsMock.realAtomicWrite(path, 'partial bytes');
+ throw Object.assign(new Error('simulated disk full'), { code: 'ENOSPC' });
+ }
+ return fileUtilsMock.realAtomicWrite(path, ...args);
+ });
+ await expect(importer.processManifest(bucket.id, filename)).rejects.toThrow('simulated disk full');
+ expect(existsSync(join(imageDir, ref.ref))).toBe(false);
+ const { readdirSync } = await import('fs');
+ expect(readdirSync(imageDir).filter(name => name.startsWith('.sharing-'))).toEqual([]);
+ fileUtilsMock.atomicWrite.mockImplementation((...args) => fileUtilsMock.realAtomicWrite(...args));
+ await importer.processManifest(bucket.id, filename);
+ expect(readFileSync(join(imageDir, ref.ref))).toEqual(PNG);
+ });
+
+ it('preserves a destination created while verified bytes are being staged', async () => {
+ const bucket = await buckets.createBucket({ name: 'Concurrent publication', path: tempBucket, mode: 'auto-merge' });
+ const ref = { kind: 'image', ref: 'existing.png', hash: sha256Hex(PNG) };
+ mkdirSync(join(tempBucket, 'assets', 'blobs'), { recursive: true });
+ writeFileSync(buckets.bucketBlobPath(tempBucket, ref.hash), PNG);
+ const target = join(tempData, 'images', ref.ref);
+ const existing = pngBytes('existing');
+ fileUtilsMock.atomicWrite.mockImplementation((path, ...args) => {
+ if (path.startsWith(join(tempData, 'images', '.sharing-'))) writeFileSync(target, existing);
+ return fileUtilsMock.realAtomicWrite(path, ...args);
+ });
+ await importer.processManifest(bucket.id, writeAssetManifest([ref]));
+ expect(readFileSync(target)).toEqual(existing);
+ });
+
it('revalidates bucket bytes during promotion even when a local file already exists', async () => {
const bucket = await buckets.createBucket({ name: 'Inbox integrity', path: tempBucket, mode: 'inbox' });
const ref = { kind: 'image', ref: 'promote.png' };
diff --git a/server/services/sharing/sidecarSync.js b/server/services/sharing/sidecarSync.js
index 755dc49de0..c75c07a468 100644
--- a/server/services/sharing/sidecarSync.js
+++ b/server/services/sharing/sidecarSync.js
@@ -26,8 +26,8 @@ import { peerFetch } from '../../lib/peerHttpClient.js';
import { withAbortTimeout } from '../../lib/abortTimeout.js';
import { getPeers } from '../instances.js';
import { peerBaseUrl } from '../../lib/peerUrl.js';
+import { SIDECAR_MAX_BYTES } from './assetLimits.js';
-const SIDECAR_MAX_BYTES = 256 * 1024;
// Mirror the image-pull timeout (peerSync.js ASSET_PULL_TIMEOUT_MS) so a hung
// peer connection aborts instead of holding the pull open indefinitely.
const SIDECAR_PULL_TIMEOUT_MS = 60000;