Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

fix: verify shared bucket assets before gallery import (#12564)

[._.]/ Adam Eivy85c079193951
diff --git a/server/services/sharing/assetLimits.js b/server/services/sharing/assetLimits.js
new file mode 100644
index 0000000000..f76b386c26
--- /dev/null
+++ b/server/services/sharing/assetLimits.js
@@ -0,0 +1,2 @@
+// Shared ingress ceiling for peer pulls and bucket files, including sidecars.
+export const ASSET_PULL_MAX_BYTES = 100 * 1024 * 1024;
diff --git a/server/services/sharing/importer.js b/server/services/sharing/importer.js
index 51aee6e082..35aa7d3ad5 100644
--- a/server/services/sharing/importer.js
+++ b/server/services/sharing/importer.js
@@ -22,13 +22,13 @@
*/
import { createKeyCachedQueue } from '../../lib/createKeyCachedQueue.js';
-import { join, basename } from 'path';
+import { join, basename, relative, resolve, sep } from 'path';
import { createHash } from 'crypto';
-import { readdir } from 'fs/promises';
-import { existsSync } from 'fs';
+import { readdir, lstat, realpath, open } from 'fs/promises';
+import { existsSync, constants } from 'fs';
import { EventEmitter } from 'events';
import { withBackupAssetPublication } from '../../lib/backupSnapshotBoundary.js';
-import { PATHS, copyFileGuarded, ensureDir, atomicWrite, readJSONFile } from '../../lib/fileUtils.js';
+import { PATHS, writeFileGuarded, ensureDir, atomicWrite, readJSONFile } from '../../lib/fileUtils.js';
import { isSafeRecordId } from '../../lib/validation.js';
import { logFailureWithStack } from '../../lib/failureLogging.js';
import { getBucket, bucketBlobPath, bucketBlobSidecarPath, bucketRecordsDir, bucketRecordPath, imageSidecarName, isHexHash } from './buckets.js';
@@ -48,6 +48,9 @@ import { mergePeerAnnotations } from '../mediaAnnotations.js';
import { isPlainObject } from '../../lib/objects.js';
import { maybeJournalBeforeOverwrite, flushBaseHashes, setSyncBaseHash, contentHashForRecord } from '../../lib/conflictJournal.js';
import { isStr } from '../../lib/textUtils.js';
+import { detectImageFormat, detectMediaManifest } from '../../lib/mimeTypes.js';
+import { ASSET_PULL_MAX_BYTES } from './assetLimits.js';
+import { mapWithConcurrency } from '../../lib/mapWithConcurrency.js';
// Record kinds that participate in the non-blocking conflict journal. Universe,
// series, AND issue all seed a base hash on first import and archive a losing
@@ -305,6 +308,51 @@ function copyAssetsLocally(bucketPath, assetRefs) {
return withBackupAssetPublication(() => copyAssetsLocallyAdmitted(bucketPath, assetRefs));
}
+// Pin reads to one regular-file descriptor. A cloud-sync writer can replace
+// paths during inspection; neither a symlink nor a later path swap may select
+// the bytes we publish. The extra byte detects growth without an unbounded read.
+async function readBucketAsset(bucketPath, sourcePath) {
+ const root = await realpath(bucketPath);
+ const source = await realpath(sourcePath);
+ const expected = resolve(root, relative(resolve(bucketPath), resolve(sourcePath)));
+ if (source !== expected || !source.startsWith(`${root}${sep}`)) return null;
+ const info = await lstat(sourcePath);
+ if (!info.isFile() || info.size > ASSET_PULL_MAX_BYTES) return null;
+ const handle = await open(sourcePath, constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK);
+ try {
+ const opened = await handle.stat();
+ if (!opened.isFile() || opened.dev !== info.dev || opened.ino !== info.ino
+ || opened.size > ASSET_PULL_MAX_BYTES || await realpath(sourcePath) !== source) return null;
+ const buffer = Buffer.alloc(opened.size + 1);
+ let length = 0;
+ while (length < buffer.length) {
+ const { bytesRead } = await handle.read(buffer, length, buffer.length - length, length);
+ if (!bytesRead) break;
+ length += bytesRead;
+ }
+ if (length > opened.size) return null;
+ return buffer.subarray(0, length);
+ } finally {
+ await handle.close();
+ }
+}
+
+async function verifiedBucketAsset(bucketPath, sourcePath, { hash, image = false } = {}) {
+ const buffer = await readBucketAsset(bucketPath, sourcePath).catch(() => null);
+ if (!buffer?.length || detectMediaManifest(buffer) || image && !detectImageFormat(buffer)) return null;
+ if (hash && createHash('sha256').update(buffer).digest('hex') !== hash) return null;
+ return buffer;
+}
+
+// Exclusive creation preserves skip-if-present semantics even for overlapping
+// imports, without following a destination symlink created between checks.
+function publishBucketAsset(targetPath, buffer) {
+ return writeFileGuarded(targetPath, buffer, { flag: 'wx' }).then(() => true).catch(err => {
+ if (err?.code === 'EEXIST') return false;
+ throw err;
+ });
+}
+
async function copyAssetsLocallyAdmitted(bucketPath, assetRefs) {
await ensureDir(PATHS.images);
await ensureDir(PATHS.videos);
@@ -312,7 +360,8 @@ async function copyAssetsLocallyAdmitted(bucketPath, assetRefs) {
const copied = [];
const available = [];
const missing = [];
- await Promise.all((assetRefs || []).map(async (ref) => {
+ // Buffers remain live through publication; do not read an entire manifest at once.
+ await mapWithConcurrency(assetRefs || [], 2, async (ref) => {
if (!ref || !isStr(ref.ref)) return;
const filename = basename(ref.ref);
const kind = ref.kind;
@@ -323,17 +372,24 @@ async function copyAssetsLocallyAdmitted(bucketPath, assetRefs) {
missing.push({ kind, ref: filename });
return;
}
+ const buffer = await verifiedBucketAsset(bucketPath, blobPath, { hash: ref.hash, image: kind !== 'video' });
+ const hasSidecar = sidecarPath && existsSync(sidecarPath);
+ const sidecar = hasSidecar ? await verifiedBucketAsset(bucketPath, sidecarPath) : null;
+ if (!buffer || hasSidecar && !sidecar) {
+ console.warn(`⚠️ sharing.importer: refused asset ${kind}/${JSON.stringify(filename)}`);
+ missing.push({ kind, ref: filename });
+ return;
+ }
available.push({ kind, ref: filename });
- if (!existsSync(targetPath)) {
- await copyFileGuarded(blobPath, targetPath);
+ if (!existsSync(targetPath) && await publishBucketAsset(targetPath, buffer)) {
copied.push({ kind, ref: filename });
}
- if (sidecarPath && existsSync(sidecarPath)) {
+ if (sidecar) {
const sidecarTarget = join(targetDir, imageSidecarName(filename));
- if (!existsSync(sidecarTarget)) await copyFileGuarded(sidecarPath, sidecarTarget);
+ if (!existsSync(sidecarTarget)) await publishBucketAsset(sidecarTarget, sidecar);
}
if (kind === 'image') await import('../mediaAssetIndex/index.js').then(m => m.indexImage({ filename })).catch(err => logFailureWithStack('❌ Media index imported asset refresh', err));
- }));
+ });
return { copied, available, missing };
}
diff --git a/server/services/sharing/integration.test.js b/server/services/sharing/integration.test.js
index 5d98c55d7e..c647adbbb3 100644
--- a/server/services/sharing/integration.test.js
+++ b/server/services/sharing/integration.test.js
@@ -9,13 +9,19 @@
*/
import { describe, it as vitestIt, expect, beforeEach, afterEach, afterAll, vi } from 'vitest';
-import { mkdtempSync, rmSync, writeFileSync, existsSync, mkdirSync, readFileSync, statSync, utimesSync } from 'fs';
+import { mkdtempSync, rmSync, writeFileSync, existsSync, mkdirSync, readFileSync, statSync, utimesSync, symlinkSync, truncateSync } from 'fs';
import { tmpdir } from 'os';
import { join } from 'path';
import { createHash } from 'crypto';
import { makePathsProxy, mockNoPeerSync, mockNoPeers, mockTestIdentity, ownTestBodies } from '../../lib/mockPathsDataRoot.js';
-const fileUtilsMock = vi.hoisted(() => ({ atomicWrite: vi.fn(), realAtomicWrite: null }));
+const PNG = Buffer.from('iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAwMCAO+jRZkAAAAASUVORK5CYII=', 'base64');
+const pngBytes = label => Buffer.concat([PNG, Buffer.from(label)]);
+
+const fileUtilsMock = vi.hoisted(() => ({
+ atomicWrite: vi.fn(), realAtomicWrite: null,
+ writeFileGuarded: vi.fn(), realWriteFileGuarded: null,
+}));
function sha256Hex(buf) {
return createHash('sha256').update(buf).digest('hex');
@@ -50,11 +56,13 @@ async function cleanupBuckets() {
vi.mock('../../lib/fileUtils.js', async () =>
makePathsProxy(await vi.importActual('../../lib/fileUtils.js'), {
dataRoot: tempData,
- overrides: { atomicWrite: fileUtilsMock.atomicWrite },
+ overrides: { atomicWrite: fileUtilsMock.atomicWrite, writeFileGuarded: fileUtilsMock.writeFileGuarded },
}));
const fileUtilsActual = await vi.importActual('../../lib/fileUtils.js');
fileUtilsMock.realAtomicWrite = fileUtilsActual.atomicWrite;
+fileUtilsMock.realWriteFileGuarded = fileUtilsActual.writeFileGuarded;
+fileUtilsMock.writeFileGuarded.mockImplementation((...args) => fileUtilsMock.realWriteFileGuarded(...args));
fileUtilsMock.atomicWrite.mockImplementation((...args) => fileUtilsMock.realAtomicWrite(...args));
// Stub instanceIdentity.getInstanceId so the exporter doesn't try to read the
@@ -108,6 +116,8 @@ function simulateRemoteSender(bucketPath, filename, peerId = 'remote-peer-id') {
describe('sharing round-trip', () => {
beforeEach(() => {
+ fileUtilsMock.writeFileGuarded.mockReset();
+ fileUtilsMock.writeFileGuarded.mockImplementation((...args) => fileUtilsMock.realWriteFileGuarded(...args));
fileUtilsMock.atomicWrite.mockClear();
fileUtilsMock.atomicWrite.mockImplementation((...args) => fileUtilsMock.realAtomicWrite(...args));
// Wipe and re-seed the temp data dir + a fake asset for each test.
@@ -115,7 +125,7 @@ describe('sharing round-trip', () => {
mkdirSync(tempData, { recursive: true });
mkdirSync(join(tempData, 'images'), { recursive: true });
mkdirSync(join(tempData, 'videos'), { recursive: true });
- writeFileSync(join(tempData, 'images', 'fakeasset.png'), 'PNGSTUB');
+ writeFileSync(join(tempData, 'images', 'fakeasset.png'), pngBytes('PNGSTUB'));
tempBucket = mkdtempSync(join(tmpdir(), 'portos-sharing-roundtrip-bucket-'));
tempBuckets.add(tempBucket);
});
@@ -132,6 +142,154 @@ describe('sharing round-trip', () => {
}
});
+ function writeAssetManifest(assetRefs, id = 'asset-ingress') {
+ const filename = `${id}.json`;
+ writeFileSync(join(tempBucket, 'manifests', filename), JSON.stringify({
+ id, senderInstanceId: 'example-peer', kind: 'media', recordIds: [], assetRefs,
+ }));
+ return filename;
+ }
+
+ for (const mode of ['auto-merge', 'inbox']) {
+ it(`imports verified v1/v2 images, reference sheets and videos through ${mode}`, async () => {
+ const bucket = await buckets.createBucket({ name: 'Verified assets', path: tempBucket, mode });
+ const mp4 = Buffer.from('000000186674797069736f6d0000020069736f6d6d703432', 'hex');
+ const refs = [];
+ for (const version of [1, 2]) {
+ for (const kind of ['image', 'image-ref', 'video']) {
+ const ref = `${kind}-v${version}.${kind === 'video' ? 'mp4' : 'png'}`;
+ const bytes = kind === 'video' ? mp4 : pngBytes(ref);
+ const hash = version === 2 ? sha256Hex(bytes) : undefined;
+ const dir = join(tempBucket, 'assets', hash ? 'blobs' : kind === 'video' ? 'videos' : 'images');
+ mkdirSync(dir, { recursive: true });
+ const source = join(dir, hash || ref);
+ writeFileSync(source, bytes);
+ if (kind !== 'video') writeFileSync(hash ? `${source}.metadata.json` : join(dir, buckets.imageSidecarName(ref)), '{"prompt":"example"}');
+ refs.push({ kind, ref, ...(hash ? { hash } : {}) });
+ }
+ }
+ const filename = writeAssetManifest(refs);
+ expect(await importer.processManifest(bucket.id, filename)).toMatchObject({ processed: true });
+ if (mode === 'inbox') {
+ // Force promotion to publish again, rather than merely use existing files.
+ for (const dir of ['images', 'image-refs', 'videos']) rmSync(join(tempData, dir), { recursive: true, force: true });
+ await importer.promoteInboxItem(bucket.id, 'asset-ingress');
+ expect(await importer.listInbox(bucket.id)).toEqual([]);
+ }
+ for (const { kind, ref } of refs) {
+ const dir = join(tempData, kind === 'video' ? 'videos' : kind === 'image-ref' ? 'image-refs' : 'images');
+ expect(readFileSync(join(dir, ref))).toEqual(kind === 'video' ? mp4 : pngBytes(ref));
+ if (kind !== 'video') expect(JSON.parse(readFileSync(join(dir, buckets.imageSidecarName(ref)), 'utf8'))).toEqual({ prompt: 'example' });
+ }
+ });
+ }
+
+ for (const scenario of [
+ 'v2 symlink', 'legacy symlink', 'blobs directory symlink', 'assets directory symlink',
+ 'internal directory symlink', 'nonregular blob',
+ 'hash mismatch', 'video playlist', 'image playlist', 'reference non-image',
+ 'sidecar symlink', 'sidecar playlist', 'oversized blob', 'oversized sidecar',
+ ]) {
+ it(`defers ${scenario} without publishing unverified bytes`, async () => {
+ const bucket = await buckets.createBucket({ name: 'Untrusted assets', path: tempBucket, mode: 'auto-merge' });
+ const kind = scenario === 'video playlist' ? 'video' : scenario === 'reference non-image' ? 'image-ref' : 'image';
+ const ref = { kind, ref: 'untrusted.png' };
+ const good = pngBytes('verified');
+ const hash = sha256Hex(good);
+ const legacy = scenario === 'legacy symlink';
+ if (!legacy) ref.hash = hash;
+ const dir = join(tempBucket, 'assets', legacy ? 'images' : 'blobs');
+ mkdirSync(dir, { recursive: true });
+ let source = join(dir, legacy ? ref.ref : hash);
+ writeFileSync(source, good);
+ const outside = join(tempData, 'outside.png');
+ writeFileSync(outside, good);
+ if (scenario === 'v2 symlink' || legacy) {
+ rmSync(source);
+ symlinkSync(outside, source, 'file');
+ } else if (scenario.endsWith('directory symlink')) {
+ const original = scenario === 'assets directory symlink' ? join(tempBucket, 'assets') : dir;
+ const outsideDir = join(scenario === 'internal directory symlink' ? tempBucket : tempData, 'outside-dir');
+ mkdirSync(outsideDir);
+ if (scenario === 'assets directory symlink') mkdirSync(join(outsideDir, 'blobs'));
+ writeFileSync(join(outsideDir, scenario === 'assets directory symlink' ? 'blobs' : '', hash), good);
+ rmSync(original, { recursive: true });
+ symlinkSync(outsideDir, original, process.platform === 'win32' ? 'junction' : 'dir');
+ } else if (scenario === 'nonregular blob') {
+ rmSync(source);
+ mkdirSync(source);
+ } else if (scenario === 'hash mismatch') {
+ writeFileSync(source, pngBytes('different'));
+ } else if (scenario === 'video playlist' || scenario === 'image playlist' || scenario === 'reference non-image') {
+ const bytes = Buffer.from(scenario === 'reference non-image' ? 'not an image' : '#EXTM3U\n#EXT-X-TARGETDURATION:10\nexample.ts\n');
+ ref.hash = sha256Hex(bytes);
+ source = join(dir, ref.hash);
+ writeFileSync(source, bytes);
+ } else if (scenario === 'sidecar symlink') {
+ symlinkSync(outside, `${source}.metadata.json`, 'file');
+ } else if (scenario === 'sidecar playlist') {
+ writeFileSync(`${source}.metadata.json`, 'ffconcat version 1.0\nfile example.mp4\n');
+ } else if (scenario.startsWith('oversized')) {
+ const oversized = scenario === 'oversized sidecar' ? `${source}.metadata.json` : source;
+ writeFileSync(oversized, '');
+ truncateSync(oversized, 100 * 1024 * 1024 + 1);
+ }
+ const warn = vi.spyOn(console, 'warn').mockImplementation(() => {});
+ try {
+ const result = await importer.processManifest(bucket.id, writeAssetManifest([ref]));
+ expect(result).toMatchObject({ processed: true, pending: true, outcome: { pendingAssets: [{ kind, ref: ref.ref }] } });
+ const targetDir = join(tempData, kind === 'video' ? 'videos' : kind === 'image-ref' ? 'image-refs' : 'images');
+ expect(existsSync(join(targetDir, ref.ref))).toBe(false);
+ expect(existsSync(join(targetDir, buckets.imageSidecarName(ref.ref)))).toBe(false);
+ expect(warn.mock.calls.filter(([line]) => line.startsWith('⚠️ sharing.importer: refused asset'))).toEqual([
+ [`⚠️ sharing.importer: refused asset ${kind}/${JSON.stringify(ref.ref)}`],
+ ]);
+ } finally {
+ warn.mockRestore();
+ }
+ });
+ }
+
+ it('publishes the verified buffer when the bucket path changes before publication', async () => {
+ const bucket = await buckets.createBucket({ name: 'Publication race', path: tempBucket, mode: 'auto-merge' });
+ const ref = { kind: 'image', ref: 'verified-race.png', hash: sha256Hex(PNG) };
+ const source = buckets.bucketBlobPath(tempBucket, ref.hash);
+ mkdirSync(join(tempBucket, 'assets', 'blobs'), { recursive: true });
+ writeFileSync(source, PNG);
+ const target = join(tempData, 'images', ref.ref);
+ let replaced = false;
+ fileUtilsMock.writeFileGuarded.mockImplementation((path, ...args) => {
+ if (path === target) {
+ replaced = true;
+ rmSync(source);
+ symlinkSync(join(tempData, 'outside.png'), source, 'file');
+ writeFileSync(join(tempData, 'outside.png'), 'unverified replacement');
+ }
+ return fileUtilsMock.realWriteFileGuarded(path, ...args);
+ });
+ expect(await importer.processManifest(bucket.id, writeAssetManifest([ref]))).toMatchObject({ processed: true });
+ expect(replaced).toBe(true);
+ expect(readFileSync(target)).toEqual(PNG);
+ });
+
+ it('revalidates bucket bytes during promotion even when a local file already exists', async () => {
+ const bucket = await buckets.createBucket({ name: 'Inbox integrity', path: tempBucket, mode: 'inbox' });
+ const ref = { kind: 'image', ref: 'promote.png' };
+ const dir = join(tempBucket, 'assets', 'images');
+ mkdirSync(dir, { recursive: true });
+ writeFileSync(join(dir, ref.ref), PNG);
+ await importer.processManifest(bucket.id, writeAssetManifest([ref]));
+ writeFileSync(join(dir, ref.ref), '#EXTM3U\nexample.ts\n');
+ await expect(importer.promoteInboxItem(bucket.id, 'asset-ingress')).rejects.toMatchObject({
+ code: 'SHARING_ASSETS_PENDING', missingAssets: [ref],
+ });
+ expect(readFileSync(join(tempData, 'images', ref.ref))).toEqual(PNG);
+ expect(await importer.listInbox(bucket.id)).toHaveLength(1);
+ writeFileSync(join(dir, ref.ref), PNG);
+ await importer.promoteInboxItem(bucket.id, 'asset-ingress');
+ expect(await importer.listInbox(bucket.id)).toEqual([]);
+ });
+
it('preserves local video holds while importing bundled media-job records', async () => {
const bucket = await buckets.createBucket({ name: 'Example Bucket', path: tempBucket, mode: 'inbox' });
const local = { id: 'retained-local', kind: 'video', status: 'queued', params: { modelId: 'example-video' } };
@@ -175,7 +333,7 @@ describe('sharing round-trip', () => {
expect(existsSync(join(tempBucket, 'manifests', exp.filename))).toBe(true);
expect(existsSync(join(tempBucket, 'records', 'series', `${s.id}.json`))).toBe(true);
expect(existsSync(join(tempBucket, 'records', 'issues', `${iss.id}.json`))).toBe(true);
- const fakeHash = sha256Hex('PNGSTUB');
+ const fakeHash = sha256Hex(pngBytes('PNGSTUB'));
expect(existsSync(join(tempBucket, 'assets', 'blobs', fakeHash))).toBe(true);
// Process as inbox (the bucket mode is 'inbox'). The manifest should
@@ -592,7 +750,7 @@ describe('sharing round-trip', () => {
// collection. Each item.ref points at a real file in tempData/images
// (we seeded 'fakeasset.png' in beforeEach, but for variety, add another).
const fs = await import('fs');
- fs.writeFileSync(join(tempData, 'images', 'second.png'), 'PNG2');
+ fs.writeFileSync(join(tempData, 'images', 'second.png'), pngBytes('PNG2'));
const collection = await mediaCollections.findOrCreateCollectionByName({
name: `Universe: ${u.name}`, description: 'Linked', universeId: u.id,
});
@@ -607,7 +765,7 @@ describe('sharing round-trip', () => {
const manifest = JSON.parse(fs.readFileSync(join(tempBucket, 'manifests', manifestFile), 'utf-8'));
expect(manifest.collection).toMatchObject({ universeId: u.id });
expect(manifest.collection.items).toHaveLength(2);
- expect(fs.existsSync(join(tempBucket, 'assets', 'blobs', sha256Hex('PNG2')))).toBe(true);
+ expect(fs.existsSync(join(tempBucket, 'assets', 'blobs', sha256Hex(pngBytes('PNG2'))))).toBe(true);
// Drop the local collection + universe, then re-process: the importer
// should recreate both, find-or-create the local collection by
@@ -639,8 +797,8 @@ describe('sharing round-trip', () => {
const s = await series.createSeries({ name: 'Indie Saga', logline: 'L' });
const fs = await import('fs');
- fs.writeFileSync(join(tempData, 'images', 'cover-a.png'), 'COVA');
- fs.writeFileSync(join(tempData, 'images', 'cover-b.png'), 'COVB');
+ fs.writeFileSync(join(tempData, 'images', 'cover-a.png'), pngBytes('COVA'));
+ fs.writeFileSync(join(tempData, 'images', 'cover-b.png'), pngBytes('COVB'));
const collection = await mediaCollections.findOrCreateSeriesCollection({
seriesId: s.id, seriesName: s.name,
});
@@ -653,8 +811,8 @@ describe('sharing round-trip', () => {
expect(manifest.collection).toMatchObject({ seriesId: s.id });
expect(manifest.collection.universeId).toBeUndefined();
expect(manifest.collection.items.map((i) => i.ref).sort()).toEqual(['cover-a.png', 'cover-b.png']);
- expect(fs.existsSync(join(tempBucket, 'assets', 'blobs', sha256Hex('COVA')))).toBe(true);
- expect(fs.existsSync(join(tempBucket, 'assets', 'blobs', sha256Hex('COVB')))).toBe(true);
+ expect(fs.existsSync(join(tempBucket, 'assets', 'blobs', sha256Hex(pngBytes('COVA'))))).toBe(true);
+ expect(fs.existsSync(join(tempBucket, 'assets', 'blobs', sha256Hex(pngBytes('COVB'))))).toBe(true);
// Drop the local collection + series, then re-process: importer
// restores the series and find-or-creates the local per-series
@@ -690,8 +848,8 @@ describe('sharing round-trip', () => {
const s = await series.createSeries({ name: 'Series', universeId: u.id });
const fs = await import('fs');
- fs.writeFileSync(join(tempData, 'images', 'uni.png'), 'UNI');
- fs.writeFileSync(join(tempData, 'images', 'ser.png'), 'SER');
+ fs.writeFileSync(join(tempData, 'images', 'uni.png'), pngBytes('UNI'));
+ fs.writeFileSync(join(tempData, 'images', 'ser.png'), pngBytes('SER'));
const uniColl = await mediaCollections.findOrCreateUniverseCollection({
universeId: u.id, universeName: u.name,
@@ -724,7 +882,7 @@ describe('sharing round-trip', () => {
const s = await series.createSeries({ name: 'Bound', universeId: u.id });
const fs = await import('fs');
- fs.writeFileSync(join(tempData, 'images', 'stale.png'), 'STALE');
+ fs.writeFileSync(join(tempData, 'images', 'stale.png'), pngBytes('STALE'));
// Stamp a seriesId collection directly (simulating a leftover from a
// prior universeless phase). No universe collection exists.
const serColl = await mediaCollections.findOrCreateSeriesCollection({
@@ -753,7 +911,7 @@ describe('sharing round-trip', () => {
// Sender side: universeless series + per-series collection with one cover.
const s = await series.createSeries({ name: 'WillLink', logline: 'L' });
- fs.writeFileSync(join(tempData, 'images', 'stale-cover.png'), 'STALE');
+ fs.writeFileSync(join(tempData, 'images', 'stale-cover.png'), pngBytes('STALE'));
const senderColl = await mediaCollections.findOrCreateSeriesCollection({
seriesId: s.id, seriesName: s.name,
});
@@ -789,7 +947,7 @@ describe('sharing round-trip', () => {
const mediaCollections = await import('../mediaCollections.js');
const s = await series.createSeries({ name: 'Will Vanish' });
const fs = await import('fs');
- fs.writeFileSync(join(tempData, 'images', 'late-cover.png'), 'LATE');
+ fs.writeFileSync(join(tempData, 'images', 'late-cover.png'), pngBytes('LATE'));
const collection = await mediaCollections.findOrCreateSeriesCollection({
seriesId: s.id, seriesName: s.name,
});
@@ -822,7 +980,7 @@ describe('sharing round-trip', () => {
const mediaCollections = await import('../mediaCollections.js');
const u = await universeBuilder.createUniverse({ name: 'Post-Epoc' });
const fs = await import('fs');
- fs.writeFileSync(join(tempData, 'images', 'legacy.png'), 'PNG');
+ fs.writeFileSync(join(tempData, 'images', 'legacy.png'), pngBytes('PNG'));
// Unlinked collection with the conventional name — exporter must NOT
// adopt this for the universe.
@@ -846,7 +1004,7 @@ describe('sharing round-trip', () => {
const { readCursor, hasBeenProcessed } = await import('./manifest.js');
const u = await universeBuilder.createUniverse({ name: 'Slow Sync Universe' });
const fs = await import('fs');
- fs.writeFileSync(join(tempData, 'images', 'late.png'), 'LATEPNG');
+ fs.writeFileSync(join(tempData, 'images', 'late.png'), pngBytes('LATEPNG'));
const collection = await mediaCollections.findOrCreateCollectionByName({
name: `Universe: ${u.name}`, description: 'Linked', universeId: u.id,
@@ -855,7 +1013,7 @@ describe('sharing round-trip', () => {
await mediaCollections.addItem(collection.id, { kind: 'image', ref: 'late.png' });
const exp = await exporter.exportUniverse(u.id, bucket.id);
- const lateHash = sha256Hex('LATEPNG');
+ const lateHash = sha256Hex(pngBytes('LATEPNG'));
fs.rmSync(join(tempBucket, 'assets', 'blobs', lateHash), { force: true });
fs.rmSync(join(tempData, 'images', 'late.png'), { force: true });
await mediaCollections.deleteCollection(collection.id);
@@ -873,7 +1031,7 @@ describe('sharing round-trip', () => {
let cursor = await readCursor(bucket.id);
expect(hasBeenProcessed(cursor, exp.filename, exp.manifestId)).toBe(false);
- fs.writeFileSync(join(tempBucket, 'assets', 'blobs', lateHash), 'LATEPNG');
+ fs.writeFileSync(join(tempBucket, 'assets', 'blobs', lateHash), pngBytes('LATEPNG'));
const retry = await importer.processBacklog(bucket.id);
expect(retry.processed).toBe(1);
@@ -1474,7 +1632,7 @@ describe('sharing round-trip', () => {
const mediaCollections = await import('../mediaCollections.js');
// Two distinct filenames, identical bytes.
- const sharedBytes = 'SHARED-PNG-PAYLOAD';
+ const sharedBytes = pngBytes('SHARED-PNG-PAYLOAD');
fs.writeFileSync(join(tempData, 'images', 'alpha.png'), sharedBytes);
fs.writeFileSync(join(tempData, 'images', 'beta.png'), sharedBytes);
const hash = sha256Hex(sharedBytes);
@@ -1528,8 +1686,8 @@ describe('sharing round-trip', () => {
expect(fs.existsSync(join(tempData, 'images', 'alpha.png'))).toBe(true);
expect(fs.existsSync(join(tempData, 'images', 'beta.png'))).toBe(true);
- expect(fs.readFileSync(join(tempData, 'images', 'alpha.png'), 'utf-8')).toBe(sharedBytes);
- expect(fs.readFileSync(join(tempData, 'images', 'beta.png'), 'utf-8')).toBe(sharedBytes);
+ expect(fs.readFileSync(join(tempData, 'images', 'alpha.png'))).toEqual(sharedBytes);
+ expect(fs.readFileSync(join(tempData, 'images', 'beta.png'))).toEqual(sharedBytes);
});
it('importer rejects manifest asset refs whose hash is not a 64-char hex string (path-traversal guard)', async () => {
@@ -1625,7 +1783,7 @@ describe('sharing round-trip', () => {
// Hand-craft a v1-style bucket: blob lives at assets/images/<filename>,
// manifest's assetRefs has no `hash` field, schemaVersion: 1.
fs.mkdirSync(join(tempBucket, 'assets', 'images'), { recursive: true });
- fs.writeFileSync(join(tempBucket, 'assets', 'images', 'legacy-v1.png'), 'V1BYTES');
+ fs.writeFileSync(join(tempBucket, 'assets', 'images', 'legacy-v1.png'), pngBytes('V1BYTES'));
const universeBuilder = await import('../universeBuilder.js');
const u = await universeBuilder.createUniverse({ name: 'Legacy v1 Universe' });
@@ -1665,7 +1823,7 @@ describe('sharing round-trip', () => {
// Asset blob made it to the local data dir via the legacy path.
expect(fs.existsSync(join(tempData, 'images', 'legacy-v1.png'))).toBe(true);
- expect(fs.readFileSync(join(tempData, 'images', 'legacy-v1.png'), 'utf-8')).toBe('V1BYTES');
+ expect(fs.readFileSync(join(tempData, 'images', 'legacy-v1.png'))).toEqual(pngBytes('V1BYTES'));
});
it('migrates legacy series canon arrays into the linked universe on import (pre-B.4 peer)', async () => {
@@ -2222,7 +2380,7 @@ describe('sharing round-trip', () => {
// Stage a bucket asset so peer annotations have a corresponding key.
mkdirSync(join(tempBucket, 'assets', 'images'), { recursive: true });
- writeFileSync(join(tempBucket, 'assets', 'images', 'fakeasset.png'), 'PNGSTUB');
+ writeFileSync(join(tempBucket, 'assets', 'images', 'fakeasset.png'), pngBytes('PNGSTUB'));
// Hand-write a peer's annotation record + manifest. Bypasses the exporter
// (which would stamp the LOCAL instance id and short-circuit as
@@ -2288,7 +2446,7 @@ describe('sharing round-trip', () => {
it('writes <bucket>/assets/blobs/.index.json mapping sourcePath:mtime:size → hash and invalidates on mtime change', async () => {
const bucket = await buckets.createBucket({ name: 'CacheBucket', path: tempBucket, mode: 'auto-merge' });
- const sourcePath = join(tempData, 'images', 'fakeasset.png'); // seeded as 'PNGSTUB' in beforeEach
+ const sourcePath = join(tempData, 'images', 'fakeasset.png'); // seeded as pngBytes('PNGSTUB') in beforeEach
// Single-asset export — exportMedia falls through to copyAssetIfPresent (getJob is mocked → null).
await exporter.exportMedia([{ kind: 'image', ref: 'fakeasset.png' }], bucket.id);
@@ -2298,7 +2456,7 @@ describe('sharing round-trip', () => {
const initialStat = statSync(sourcePath);
const initialKey = `${sourcePath}:${initialStat.mtimeMs}:${initialStat.size}`;
- const initialHash = sha256Hex('PNGSTUB');
+ const initialHash = sha256Hex(pngBytes('PNGSTUB'));
const idx1 = JSON.parse(readFileSync(indexPath, 'utf-8'));
expect(idx1[initialKey]).toBe(initialHash);
@@ -2321,21 +2479,21 @@ describe('sharing round-trip', () => {
expect(idx2[touchedKey]).toBe(initialHash);
// Mutating content invalidates and records a fresh hash.
- writeFileSync(sourcePath, 'PNGSTUB-v2');
+ writeFileSync(sourcePath, pngBytes('PNGSTUB-v2'));
await exporter.exportMedia([{ kind: 'image', ref: 'fakeasset.png' }], bucket.id);
const mutatedStat = statSync(sourcePath);
const mutatedKey = `${sourcePath}:${mutatedStat.mtimeMs}:${mutatedStat.size}`;
const idx3 = JSON.parse(readFileSync(indexPath, 'utf-8'));
- expect(idx3[mutatedKey]).toBe(sha256Hex('PNGSTUB-v2'));
+ expect(idx3[mutatedKey]).toBe(sha256Hex(pngBytes('PNGSTUB-v2')));
// Mutated bytes also land in the bucket under the new hash (content-addressed).
- expect(existsSync(join(tempBucket, 'assets', 'blobs', sha256Hex('PNGSTUB-v2')))).toBe(true);
+ expect(existsSync(join(tempBucket, 'assets', 'blobs', sha256Hex(pngBytes('PNGSTUB-v2'))))).toBe(true);
});
it('parallel exports against the same bucket merge cache entries instead of clobbering (exportByKind fanout)', async () => {
const bucket = await buckets.createBucket({ name: 'MergeBucket', path: tempBucket, mode: 'auto-merge' });
// Two distinct assets, both seeded with distinct bytes so they hash differently.
- writeFileSync(join(tempData, 'images', 'parallelA.png'), 'PARALLEL_A');
- writeFileSync(join(tempData, 'images', 'parallelB.png'), 'PARALLEL_B');
+ writeFileSync(join(tempData, 'images', 'parallelA.png'), pngBytes('PARALLEL_A'));
+ writeFileSync(join(tempData, 'images', 'parallelB.png'), pngBytes('PARALLEL_B'));
// Two singleton media exports run in parallel — each loads its own cache view,
// each saves with merge-on-save. Both new keys must survive in the persisted index.
await Promise.all([
@@ -2348,8 +2506,8 @@ describe('sharing round-trip', () => {
const statB = statSync(join(tempData, 'images', 'parallelB.png'));
const keyA = `${join(tempData, 'images', 'parallelA.png')}:${statA.mtimeMs}:${statA.size}`;
const keyB = `${join(tempData, 'images', 'parallelB.png')}:${statB.mtimeMs}:${statB.size}`;
- expect(idx[keyA]).toBe(sha256Hex('PARALLEL_A'));
- expect(idx[keyB]).toBe(sha256Hex('PARALLEL_B'));
+ expect(idx[keyA]).toBe(sha256Hex(pngBytes('PARALLEL_A')));
+ expect(idx[keyB]).toBe(sha256Hex(pngBytes('PARALLEL_B')));
});
it('retries a blob-index write after a transient failure instead of poisoning later exports', async () => {
@@ -2565,7 +2723,7 @@ describe('sharing round-trip', () => {
const { hasBeenProcessed, readCursor } = await import('./manifest.js');
const u = await universeBuilder.createUniverse({ name: 'Has A Collection Universe' });
- fs.writeFileSync(join(tempData, 'images', 'late-universe-cover.png'), 'LATEUNI');
+ fs.writeFileSync(join(tempData, 'images', 'late-universe-cover.png'), pngBytes('LATEUNI'));
const collection = await mediaCollections.findOrCreateCollectionByName({
name: `Universe: ${u.name}`, description: 'Linked', universeId: u.id,
});
diff --git a/server/services/sharing/peerSyncAssets.js b/server/services/sharing/peerSyncAssets.js
index 01d86b5c5b..c06b84088e 100644
--- a/server/services/sharing/peerSyncAssets.js
+++ b/server/services/sharing/peerSyncAssets.js
@@ -40,6 +40,7 @@ import { isStr } from '../../lib/textUtils.js';
import { detectMediaManifest } from '../../lib/mimeTypes.js';
import { mapWithConcurrency } from '../../lib/mapWithConcurrency.js';
import { withBackupAssetPublication } from '../../lib/backupSnapshotBoundary.js';
+import { ASSET_PULL_MAX_BYTES } from './assetLimits.js';
// Bound file hashing to avoid starving unrelated filesystem work when a
// large record (especially a deck) advertises hundreds of images.
@@ -862,7 +863,6 @@ const ASSET_KIND_TO_URL_PREFIX = Object.freeze({
});
export const ASSET_PULL_TIMEOUT_MS = 60000;
-const ASSET_PULL_MAX_BYTES = 100 * 1024 * 1024; // 100MB hard cap per asset
// In-flight pull dedup. A peer can push multiple records that reference the
// same asset in quick succession (universe edit → child collection re-link