Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
fix: publish verified bucket assets atomically (#12564)
[._.]/ Adam Eivyb344c5437233
diff --git a/server/services/sharing/assetLimits.js b/server/services/sharing/assetLimits.jsindex f76b386c26..85d04a38d5 100644--- a/server/services/sharing/assetLimits.js+++ b/server/services/sharing/assetLimits.js@@ -1,2 +1,3 @@-// Shared ingress ceiling for peer pulls and bucket files, including sidecars.+// Shared ingress ceilings for peer pulls and bucket files.export const ASSET_PULL_MAX_BYTES = 100 * 1024 * 1024;+export const SIDECAR_MAX_BYTES = 256 * 1024;diff --git a/server/services/sharing/importer.js b/server/services/sharing/importer.jsindex 35aa7d3ad5..77fbb2f1c8 100644--- a/server/services/sharing/importer.js+++ b/server/services/sharing/importer.js@@ -22,13 +22,13 @@*/import { createKeyCachedQueue } from '../../lib/createKeyCachedQueue.js';-import { join, basename, relative, resolve, sep } from 'path';-import { createHash } from 'crypto';-import { readdir, lstat, realpath, open } from 'fs/promises';+import { join, basename, relative, resolve, sep, dirname } from 'path';+import { createHash, randomUUID } from 'crypto';+import { readdir, lstat, realpath, open, link } from 'fs/promises';import { existsSync, constants } from 'fs';import { EventEmitter } from 'events';import { withBackupAssetPublication } from '../../lib/backupSnapshotBoundary.js';-import { PATHS, writeFileGuarded, ensureDir, atomicWrite, readJSONFile } from '../../lib/fileUtils.js';+import { PATHS, unlinkGuarded, ensureDir, atomicWrite, readJSONFile } from '../../lib/fileUtils.js';import { isSafeRecordId } from '../../lib/validation.js';import { logFailureWithStack } from '../../lib/failureLogging.js';import { getBucket, bucketBlobPath, bucketBlobSidecarPath, bucketRecordsDir, bucketRecordPath, imageSidecarName, isHexHash } from './buckets.js';@@ -49,7 +49,7 @@ import { isPlainObject } from '../../lib/objects.js';import { maybeJournalBeforeOverwrite, flushBaseHashes, setSyncBaseHash, contentHashForRecord } from '../../lib/conflictJournal.js';import { isStr } from '../../lib/textUtils.js';import { detectImageFormat, detectMediaManifest } from '../../lib/mimeTypes.js';-import { ASSET_PULL_MAX_BYTES } from './assetLimits.js';+import { ASSET_PULL_MAX_BYTES, SIDECAR_MAX_BYTES } from './assetLimits.js';import { mapWithConcurrency } from '../../lib/mapWithConcurrency.js';// Record kinds that participate in the non-blocking conflict journal. Universe,@@ -311,18 +311,18 @@ function copyAssetsLocally(bucketPath, assetRefs) {// Pin reads to one regular-file descriptor. A cloud-sync writer can replace// paths during inspection; neither a symlink nor a later path swap may select// the bytes we publish. The extra byte detects growth without an unbounded read.-async function readBucketAsset(bucketPath, sourcePath) {+async function readBucketAsset(bucketPath, sourcePath, maxBytes) {const root = await realpath(bucketPath);const source = await realpath(sourcePath);const expected = resolve(root, relative(resolve(bucketPath), resolve(sourcePath)));if (source !== expected || !source.startsWith(`${root}${sep}`)) return null;const info = await lstat(sourcePath);- if (!info.isFile() || info.size > ASSET_PULL_MAX_BYTES) return null;+ if (!info.isFile() || info.size > maxBytes) return null;const handle = await open(sourcePath, constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK);try {const opened = await handle.stat();if (!opened.isFile() || opened.dev !== info.dev || opened.ino !== info.ino- || opened.size > ASSET_PULL_MAX_BYTES || await realpath(sourcePath) !== source) return null;+ || opened.size > maxBytes || await realpath(sourcePath) !== source) return null;const buffer = Buffer.alloc(opened.size + 1);let length = 0;while (length < buffer.length) {@@ -337,20 +337,28 @@ async function readBucketAsset(bucketPath, sourcePath) {}}-async function verifiedBucketAsset(bucketPath, sourcePath, { hash, image = false } = {}) {- const buffer = await readBucketAsset(bucketPath, sourcePath).catch(() => null);+async function verifiedBucketAsset(bucketPath, sourcePath, { hash, image = false, maxBytes = ASSET_PULL_MAX_BYTES } = {}) {+ const buffer = await readBucketAsset(bucketPath, sourcePath, maxBytes).catch(() => null);if (!buffer?.length || detectMediaManifest(buffer) || image && !detectImageFormat(buffer)) return null;if (hash && createHash('sha256').update(buffer).digest('hex') !== hash) return null;return buffer;}-// Exclusive creation preserves skip-if-present semantics even for overlapping-// imports, without following a destination symlink created between checks.-function publishBucketAsset(targetPath, buffer) {- return writeFileGuarded(targetPath, buffer, { flag: 'wx' }).then(() => true).catch(err => {- if (err?.code === 'EEXIST') return false;- throw err;- });+// Publish durable bytes atomically without replacing an existing destination.+// The short temporary name also supports asset filenames already at NAME_MAX.+async function publishBucketAsset(targetPath, buffer) {+ const temporary = join(dirname(targetPath), `.sharing-${randomUUID()}.tmp`);+ try {+ await atomicWrite(temporary, buffer);+ return await link(temporary, targetPath).then(() => true).catch(err => {+ if (err?.code === 'EEXIST') return false;+ throw err;+ });+ } finally {+ await unlinkGuarded(temporary).catch(err => {+ if (err?.code !== 'ENOENT') throw err;+ });+ }}async function copyAssetsLocallyAdmitted(bucketPath, assetRefs) {@@ -374,7 +382,7 @@ async function copyAssetsLocallyAdmitted(bucketPath, assetRefs) {}const buffer = await verifiedBucketAsset(bucketPath, blobPath, { hash: ref.hash, image: kind !== 'video' });const hasSidecar = sidecarPath && existsSync(sidecarPath);- const sidecar = hasSidecar ? await verifiedBucketAsset(bucketPath, sidecarPath) : null;+ const sidecar = hasSidecar ? await verifiedBucketAsset(bucketPath, sidecarPath, { maxBytes: SIDECAR_MAX_BYTES }) : null;if (!buffer || hasSidecar && !sidecar) {console.warn(`⚠️ sharing.importer: refused asset ${kind}/${JSON.stringify(filename)}`);missing.push({ kind, ref: filename });diff --git a/server/services/sharing/integration.test.js b/server/services/sharing/integration.test.jsindex c647adbbb3..60fd969262 100644--- a/server/services/sharing/integration.test.js+++ b/server/services/sharing/integration.test.js@@ -13,6 +13,7 @@ import { mkdtempSync, rmSync, writeFileSync, existsSync, mkdirSync, readFileSyncimport { tmpdir } from 'os';import { join } from 'path';import { createHash } from 'crypto';+import { ASSET_PULL_MAX_BYTES, SIDECAR_MAX_BYTES } from './assetLimits.js';import { makePathsProxy, mockNoPeerSync, mockNoPeers, mockTestIdentity, ownTestBodies } from '../../lib/mockPathsDataRoot.js';const PNG = Buffer.from('iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAwMCAO+jRZkAAAAASUVORK5CYII=', 'base64');@@ -20,7 +21,6 @@ const pngBytes = label => Buffer.concat([PNG, Buffer.from(label)]);const fileUtilsMock = vi.hoisted(() => ({atomicWrite: vi.fn(), realAtomicWrite: null,- writeFileGuarded: vi.fn(), realWriteFileGuarded: null,}));function sha256Hex(buf) {@@ -56,13 +56,11 @@ async function cleanupBuckets() {vi.mock('../../lib/fileUtils.js', async () =>makePathsProxy(await vi.importActual('../../lib/fileUtils.js'), {dataRoot: tempData,- overrides: { atomicWrite: fileUtilsMock.atomicWrite, writeFileGuarded: fileUtilsMock.writeFileGuarded },+ overrides: { atomicWrite: fileUtilsMock.atomicWrite },}));const fileUtilsActual = await vi.importActual('../../lib/fileUtils.js');fileUtilsMock.realAtomicWrite = fileUtilsActual.atomicWrite;-fileUtilsMock.realWriteFileGuarded = fileUtilsActual.writeFileGuarded;-fileUtilsMock.writeFileGuarded.mockImplementation((...args) => fileUtilsMock.realWriteFileGuarded(...args));fileUtilsMock.atomicWrite.mockImplementation((...args) => fileUtilsMock.realAtomicWrite(...args));// Stub instanceIdentity.getInstanceId so the exporter doesn't try to read the@@ -116,8 +114,6 @@ function simulateRemoteSender(bucketPath, filename, peerId = 'remote-peer-id') {describe('sharing round-trip', () => {beforeEach(() => {- fileUtilsMock.writeFileGuarded.mockReset();- fileUtilsMock.writeFileGuarded.mockImplementation((...args) => fileUtilsMock.realWriteFileGuarded(...args));fileUtilsMock.atomicWrite.mockClear();fileUtilsMock.atomicWrite.mockImplementation((...args) => fileUtilsMock.realAtomicWrite(...args));// Wipe and re-seed the temp data dir + a fake asset for each test.@@ -232,7 +228,7 @@ describe('sharing round-trip', () => {} else if (scenario.startsWith('oversized')) {const oversized = scenario === 'oversized sidecar' ? `${source}.metadata.json` : source;writeFileSync(oversized, '');- truncateSync(oversized, 100 * 1024 * 1024 + 1);+ truncateSync(oversized, (scenario === 'oversized sidecar' ? SIDECAR_MAX_BYTES : ASSET_PULL_MAX_BYTES) + 1);}const warn = vi.spyOn(console, 'warn').mockImplementation(() => {});try {@@ -258,20 +254,58 @@ describe('sharing round-trip', () => {writeFileSync(source, PNG);const target = join(tempData, 'images', ref.ref);let replaced = false;- fileUtilsMock.writeFileGuarded.mockImplementation((path, ...args) => {- if (path === target) {+ fileUtilsMock.atomicWrite.mockImplementation((path, ...args) => {+ if (path.startsWith(join(tempData, 'images', '.sharing-'))) {replaced = true;rmSync(source);symlinkSync(join(tempData, 'outside.png'), source, 'file');writeFileSync(join(tempData, 'outside.png'), 'unverified replacement');}- return fileUtilsMock.realWriteFileGuarded(path, ...args);+ return fileUtilsMock.realAtomicWrite(path, ...args);});expect(await importer.processManifest(bucket.id, writeAssetManifest([ref]))).toMatchObject({ processed: true });expect(replaced).toBe(true);expect(readFileSync(target)).toEqual(PNG);});+ it('leaves no final or temporary file after a failed publication, then imports on retry', async () => {+ const bucket = await buckets.createBucket({ name: 'Publication failure', path: tempBucket, mode: 'auto-merge' });+ const ref = { kind: 'image', ref: 'retry.png', hash: sha256Hex(PNG) };+ mkdirSync(join(tempBucket, 'assets', 'blobs'), { recursive: true });+ writeFileSync(buckets.bucketBlobPath(tempBucket, ref.hash), PNG);+ const filename = writeAssetManifest([ref]);+ const imageDir = join(tempData, 'images');+ fileUtilsMock.atomicWrite.mockImplementation(async (path, ...args) => {+ if (path.startsWith(join(imageDir, '.sharing-'))) {+ await fileUtilsMock.realAtomicWrite(path, 'partial bytes');+ throw Object.assign(new Error('simulated disk full'), { code: 'ENOSPC' });+ }+ return fileUtilsMock.realAtomicWrite(path, ...args);+ });+ await expect(importer.processManifest(bucket.id, filename)).rejects.toThrow('simulated disk full');+ expect(existsSync(join(imageDir, ref.ref))).toBe(false);+ const { readdirSync } = await import('fs');+ expect(readdirSync(imageDir).filter(name => name.startsWith('.sharing-'))).toEqual([]);+ fileUtilsMock.atomicWrite.mockImplementation((...args) => fileUtilsMock.realAtomicWrite(...args));+ await importer.processManifest(bucket.id, filename);+ expect(readFileSync(join(imageDir, ref.ref))).toEqual(PNG);+ });++ it('preserves a destination created while verified bytes are being staged', async () => {+ const bucket = await buckets.createBucket({ name: 'Concurrent publication', path: tempBucket, mode: 'auto-merge' });+ const ref = { kind: 'image', ref: 'existing.png', hash: sha256Hex(PNG) };+ mkdirSync(join(tempBucket, 'assets', 'blobs'), { recursive: true });+ writeFileSync(buckets.bucketBlobPath(tempBucket, ref.hash), PNG);+ const target = join(tempData, 'images', ref.ref);+ const existing = pngBytes('existing');+ fileUtilsMock.atomicWrite.mockImplementation((path, ...args) => {+ if (path.startsWith(join(tempData, 'images', '.sharing-'))) writeFileSync(target, existing);+ return fileUtilsMock.realAtomicWrite(path, ...args);+ });+ await importer.processManifest(bucket.id, writeAssetManifest([ref]));+ expect(readFileSync(target)).toEqual(existing);+ });+it('revalidates bucket bytes during promotion even when a local file already exists', async () => {const bucket = await buckets.createBucket({ name: 'Inbox integrity', path: tempBucket, mode: 'inbox' });const ref = { kind: 'image', ref: 'promote.png' };diff --git a/server/services/sharing/sidecarSync.js b/server/services/sharing/sidecarSync.jsindex 755dc49de0..c75c07a468 100644--- a/server/services/sharing/sidecarSync.js+++ b/server/services/sharing/sidecarSync.js@@ -26,8 +26,8 @@ import { peerFetch } from '../../lib/peerHttpClient.js';import { withAbortTimeout } from '../../lib/abortTimeout.js';import { getPeers } from '../instances.js';import { peerBaseUrl } from '../../lib/peerUrl.js';+import { SIDECAR_MAX_BYTES } from './assetLimits.js';-const SIDECAR_MAX_BYTES = 256 * 1024;// Mirror the image-pull timeout (peerSync.js ASSET_PULL_TIMEOUT_MS) so a hung// peer connection aborts instead of holding the pull open indefinitely.const SIDECAR_PULL_TIMEOUT_MS = 60000;