Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

perf([issue-11606]): reuse fresh compact video history across gallery pages (#11623)

[._.]/850125202fc8
diff --git a/server/services/mediaAssetIndex/gallery.js b/server/services/mediaAssetIndex/gallery.js
index 3c02d0b3af..e4c8fff75f 100644
--- a/server/services/mediaAssetIndex/gallery.js
+++ b/server/services/mediaAssetIndex/gallery.js
@@ -40,7 +40,9 @@ export async function listGalleryPage({
kind = 'image', media = false, cover = false, videoSnapshot, collectionSnapshot, collectionId, universeId, entryCategory, entryKind, mediaKeys,
} = {}, diskReader) {
const mixed = media || kind !== 'image';
- const videos = mixed ? videoSnapshot ?? await (await import('../videoGen/history.js')).loadHistory() : undefined;
+ const videoReader = mixed && videoSnapshot == null ? await import('../videoGen/history.js') : null;
+ const videos = mixed ? videoSnapshot ?? await (compact && !q.trim()
+ ? videoReader.loadCompactHistory() : videoReader.loadHistory()) : undefined;
const scope = await resolveScope({ collectionId, starred, mediaKeys, collectionSnapshot });
const filters = { kind: kind === 'all' ? undefined : kind, q, hidden, filename,
universeId, entryCategory, entryKind, mediaKeys: scope.mediaKeys, excludeKeys: scope.excludeKeys, videos };
diff --git a/server/services/mediaAssetIndex/gallery.test.js b/server/services/mediaAssetIndex/gallery.test.js
index 43564151dd..b9ed946677 100644
--- a/server/services/mediaAssetIndex/gallery.test.js
+++ b/server/services/mediaAssetIndex/gallery.test.js
@@ -4,7 +4,7 @@ import { imageToRow } from './logic.js';
import { query } from '../../lib/db.js';
const { listAnnotations, loadHistory, listCollections, getCollection } = vi.hoisted(() => ({ listAnnotations: vi.fn(), loadHistory: vi.fn(), listCollections: vi.fn(), getCollection: vi.fn() }));
-vi.mock('../videoGen/history.js', () => ({ loadHistory }));
+vi.mock('../videoGen/history.js', () => ({ loadHistory, loadCompactHistory: loadHistory }));
vi.mock('../mediaCollections.js', () => ({ listCollections, getCollection }));
vi.mock('../mediaAnnotations.js', () => ({ listAnnotations }));
diff --git a/server/services/mediaAssetIndex/galleryCollections.db.test.js b/server/services/mediaAssetIndex/galleryCollections.db.test.js
index aafe5de7b0..41eea99486 100644
--- a/server/services/mediaAssetIndex/galleryCollections.db.test.js
+++ b/server/services/mediaAssetIndex/galleryCollections.db.test.js
@@ -9,7 +9,10 @@ import { compactGalleryRecord } from './logic.js';
const state = vi.hoisted(() => ({ file: false, images: [], videos: [], collections: [], annotations: {}, query: vi.fn() }));
vi.mock('../../lib/runtimeEnv.js', () => ({ isTestRunner: () => state.file }));
vi.mock('../../lib/db.js', () => ({ query: (...args) => state.query(...args) }));
-vi.mock('../videoGen/history.js', () => ({ loadHistory: async () => state.videos }));
+vi.mock('../videoGen/history.js', async () => {
+ const { compactSource } = await import('./logic.js');
+ return { loadHistory: async () => state.videos, loadCompactHistory: async () => state.videos.map(compactSource) };
+});
vi.mock('../mediaCollections.js', () => ({ listCollections: async () => state.collections,
getCollection: async id => state.collections.find(collection => collection.id === id) }));
vi.mock('../mediaAnnotations.js', () => ({ listAnnotations: async () => state.annotations }));
@@ -28,7 +31,14 @@ beforeAll(async () => {
// A session-private table shadows the index, including for EXPLAIN: no live rows.
const ddl = mediaDdl.filter(sql => /(?:TABLE IF NOT EXISTS media_assets|INDEX IF NOT EXISTS idx_media_assets_)/.test(sql));
for (const sql of ddl) await client.query(sql.replace('CREATE TABLE IF NOT EXISTS', 'CREATE TEMP TABLE'));
- state.query.mockImplementation((...args) => client.query(...args));
+ // Production uses a pool; this fixture shares one session for its temp table.
+ // Serialize that session rather than relying on pg's deprecated query queue.
+ let queryTail = Promise.resolve();
+ state.query.mockImplementation((...args) => {
+ const result = queryTail.then(() => client.query(...args));
+ queryTail = result.catch(() => {});
+ return result;
+ });
available = true;
});
afterAll(async () => { await client?.end(); vi.unstubAllEnvs(); });
diff --git a/server/services/mediaAssetIndex/logic.js b/server/services/mediaAssetIndex/logic.js
index af86d2de20..0dcb9a5636 100644
--- a/server/services/mediaAssetIndex/logic.js
+++ b/server/services/mediaAssetIndex/logic.js
@@ -101,7 +101,10 @@ export function compactSource(data) {
if (!data || typeof data !== 'object') return data;
const source = {};
for (const field of COMPACT_SOURCE_FIELDS) if (data[field] !== undefined) source[field] = data[field];
- const prompt = nonBlank(data.prompt) || nonBlank(data.metadata?.prompt);
+ // Re-projecting a cached source must preserve whitespace-only prefixes: the
+ // original nonblank text may lie beyond the preview boundary.
+ const prompt = nonBlank(data.prompt) || nonBlank(data.metadata?.prompt)
+ || (typeof data[COMPACT_PREVIEW_KEY] === 'string' ? data[COMPACT_PREVIEW_KEY] : null);
if (prompt) source[COMPACT_PREVIEW_KEY] = prompt.slice(0, COMPACT_PROMPT_CHARS + 1);
return source;
}
diff --git a/server/services/videoGen/compactHistory.test.js b/server/services/videoGen/compactHistory.test.js
new file mode 100644
index 0000000000..e2cd1cacad
--- /dev/null
+++ b/server/services/videoGen/compactHistory.test.js
@@ -0,0 +1,151 @@
+import { afterAll, beforeEach, describe, expect, it, vi } from 'vitest';
+import { mkdtemp, readFile, writeFile, rename, rm, utimes, stat } from 'node:fs/promises';
+import { join } from 'node:path';
+import { tmpdir } from 'node:os';
+import { mediaFixture, CARDINALITIES } from '../../../scripts/perf/collectionFixtureData.js';
+import { compactGalleryRecord } from '../mediaAssetIndex/logic.js';
+
+const io = vi.hoisted(() => ({ root: '', reads: 0, afterRead: null, accessError: null }));
+vi.mock('node:fs/promises', async importOriginal => {
+ const real = await importOriginal();
+ return { ...real, access: async (...args) => {
+ if (io.accessError) throw io.accessError;
+ return real.access(...args);
+ } };
+});
+vi.mock('../../lib/fileUtils.js', async () => {
+ const { readJSONFile } = await import('../../lib/jsonIo.js');
+ const { atomicWrite } = await import('../../lib/fileCore.js');
+ return { PATHS: { get data() { return io.root; } }, atomicWrite,
+ readJSONFile: async (...args) => {
+ io.reads++;
+ const result = await readJSONFile(...args);
+ if (io.afterRead) await io.afterRead();
+ return result;
+ } };
+});
+vi.mock('../mediaCollections.js', () => ({ listCollections: async () => [], getCollection: async () => null }));
+vi.mock('../mediaAnnotations.js', () => ({ listAnnotations: async () => ({ 'video:synthetic-video-00001': { own: { starred: true } } }) }));
+
+let history;
+let file;
+let listGalleryPage;
+beforeEach(async () => {
+ vi.resetModules();
+ if (io.root) await rm(io.root, { recursive: true, force: true });
+ io.root = await mkdtemp(join(tmpdir(), 'compact-history-'));
+ io.reads = 0;
+ io.afterRead = null;
+ io.accessError = null;
+ file = join(io.root, 'video-history.json');
+ history = await import('./history.js');
+ ({ listGalleryPage } = await import('../mediaAssetIndex/gallery.js'));
+});
+afterAll(async () => { await rm(io.root, { recursive: true, force: true }); });
+const persist = rows => writeFile(file, JSON.stringify(rows));
+
+describe('compact history through gallery reads', () => {
+ it('reuses one immutable bounded projection for concurrent and repeated 1,200-video pages', async () => {
+ const rows = Array.from({ length: CARDINALITIES.videos }, (_, i) => mediaFixture('video', i));
+ await persist(rows);
+ const fullTimings = [];
+ for (let index = 0; index < 5; index++) {
+ const started = performance.now();
+ await history.loadHistory();
+ fullTimings.push(performance.now() - started);
+ }
+ expect(io.reads).toBe(5);
+ io.reads = 0;
+ const options = { media: true, kind: 'all', summary: true, compact: true, hidden: false, limit: 60, videoSnapshot: null };
+ const start = performance.now();
+ const pages = await Promise.all(Array.from({ length: 5 }, (_, i) => listGalleryPage({ ...options, offset: i * 60 }, async () => [])));
+ const coldMs = performance.now() - start;
+ const warmStart = performance.now();
+ await listGalleryPage(options, async () => []);
+ const warmMs = performance.now() - warmStart;
+ expect(io.reads).toBe(1);
+ expect(pages.every(page => page.items.length === 60 && page.total === 1080)).toBe(true);
+ const snapshot = await history.loadCompactHistory();
+ expect(Object.isFrozen(snapshot)).toBe(true);
+ expect(Object.isFrozen(snapshot[0])).toBe(true);
+ expect(snapshot[0]._promptPreviewSource).toHaveLength(241);
+ expect(snapshot[0]).not.toHaveProperty('status');
+ expect(compactGalleryRecord(snapshot[0])).toEqual(compactGalleryRecord(rows[0]));
+ console.log(`📊 Compact video fixture: fileBytes=${(await stat(file)).size}, compactBytes=${Buffer.byteLength(JSON.stringify(snapshot))}, fullReads=5, fullMedianMs=${fullTimings.sort((a, b) => a - b)[2].toFixed(2)}, compactReads=${io.reads}, concurrentMs=${coldMs.toFixed(2)}, warmPageMs=${warmMs.toFixed(2)}`);
+ });
+
+ it('preserves page/count filters and full search/detail beyond the preview', async () => {
+ const rows = Array.from({ length: 12 }, (_, i) => ({ ...mediaFixture('video', i),
+ prompt: ' '.repeat(300) + 'tailneedle', universeId: i % 2 ? 'one' : 'two', stitchedFrom: ['original'] }));
+ await persist(rows);
+ const collectionSnapshot = [{ id: 'set', items: rows.slice(0, 7).map(row => ({ kind: 'video', ref: row.id })) }];
+ for (const filters of [{}, { kind: 'video' }, { hidden: true }, { collectionId: 'set' },
+ { collectionId: 'unsorted' }, { starred: true }, { universeId: 'one' }, { offset: 3 }]) {
+ const options = { media: true, kind: 'all', summary: true, hidden: false, limit: 3, collectionSnapshot, ...filters };
+ const full = await listGalleryPage({ ...options, videoSnapshot: rows }, async () => []);
+ const compact = await listGalleryPage({ ...options, compact: true }, async () => []);
+ expect(compact).toEqual({ ...full, items: full.items.map(item => ({ ...item, data: compactGalleryRecord(item.data) })) });
+ }
+ expect(io.reads).toBe(1);
+ expect((await listGalleryPage({ media: true, kind: 'video', compact: true, q: 'tailneedle' }, async () => [])).total).toBe(12);
+ expect(await history.getHistoryItem(rows[0].id)).toEqual(rows[0]);
+ const snapshot = await history.loadCompactHistory();
+ expect(() => snapshot[0].stitchedFrom.push('mutated')).toThrow();
+ expect(io.reads).toBe(3);
+ });
+
+ it('refreshes after local create/update/delete and external same-size or atomic replacement', async () => {
+ await history.saveHistory([{ id: 'a', prompt: 'old' }]);
+ expect((await history.loadCompactHistory())[0]._promptPreviewSource).toBe('old');
+ await history.mutateVideoHistory(rows => [...rows, { id: 'b' }]);
+ expect(await history.loadCompactHistory()).toHaveLength(2);
+ await history.mutateVideoHistory(rows => rows.map(row => ({ ...row, hidden: true })));
+ expect((await history.loadCompactHistory())[0].hidden).toBe(true);
+ await history.mutateVideoHistory(rows => rows.filter(row => row.id === 'a'));
+ expect(await history.loadCompactHistory()).toHaveLength(1);
+ const before = await stat(file);
+ const original = await readFile(file, 'utf8');
+ await writeFile(file, original.replace('old', 'new'));
+ await utimes(file, before.atime, before.mtime);
+ expect((await history.loadCompactHistory())[0]._promptPreviewSource).toBe('new');
+ await writeFile(file + '.restore', JSON.stringify([{ id: 'restored' }]));
+ await rename(file + '.restore', file);
+ expect(await history.loadCompactHistory()).toEqual([{ id: 'restored' }]);
+ });
+
+ it('does not publish a read overtaken by a save or external replacement', async () => {
+ await persist([{ id: 'old' }]);
+ io.afterRead = async () => {
+ io.afterRead = null;
+ await history.saveHistory([{ id: 'new' }]);
+ };
+ expect(await history.loadCompactHistory()).toEqual([{ id: 'new' }]);
+ await persist([{ id: 'external-old' }]);
+ io.afterRead = async () => {
+ io.afterRead = null;
+ await writeFile(file + '.restore', JSON.stringify([{ id: 'external-new' }]));
+ await rename(file + '.restore', file);
+ };
+ const results = await Promise.all([history.loadCompactHistory(), history.loadCompactHistory()]);
+ expect(results).toEqual([[{ id: 'external-new' }], [{ id: 'external-new' }]]);
+ expect(io.reads).toBe(4);
+ });
+
+ it('never serves a cached value over corrupt, unreadable, empty or missing history', async () => {
+ await persist([{ id: 'cached' }]);
+ await history.loadCompactHistory();
+ io.accessError = Object.assign(new Error('unreadable'), { code: 'EACCES' });
+ await expect(history.loadCompactHistory()).rejects.toThrow('unreadable');
+ io.accessError = null;
+ await writeFile(file, 'invalid');
+ await expect(history.loadCompactHistory()).rejects.toThrow();
+ await writeFile(file, '');
+ await expect(history.loadCompactHistory()).rejects.toThrow();
+ await persist([]);
+ expect(await history.loadCompactHistory()).toEqual([]);
+ await rm(file);
+ expect(await history.loadCompactHistory()).toEqual([]);
+ await persist([{ id: 'returned' }]);
+ expect(await history.loadCompactHistory()).toEqual([{ id: 'returned' }]);
+ });
+});
diff --git a/server/services/videoGen/history.js b/server/services/videoGen/history.js
index 366dcdde5b..81747b7490 100644
--- a/server/services/videoGen/history.js
+++ b/server/services/videoGen/history.js
@@ -8,6 +8,9 @@
*/
import { join } from 'path';
+import { stat, access } from 'node:fs/promises';
+import { constants } from 'node:fs';
+import { compactSource } from '../mediaAssetIndex/logic.js';
import { PATHS, readJSONFile, atomicWrite } from '../../lib/fileUtils.js';
const HISTORY_FILE = join(PATHS.data, 'video-history.json');
@@ -18,7 +21,57 @@ const HISTORY_FILE = join(PATHS.data, 'video-history.json');
// render history. The Media History page also counts this list ("Show more (N
// remaining)"), where a fake 0 reads as fact. Absent stays a real first-run empty.
export const loadHistory = () => readJSONFile(HISTORY_FILE, [], { strict: true });
-export const saveHistory = (h) => atomicWrite(HISTORY_FILE, h);
+let compactSnapshot;
+let compactRead;
+let revisionEpoch = 0;
+export async function saveHistory(h) {
+ await atomicWrite(HISTORY_FILE, h);
+ revisionEpoch++;
+ compactSnapshot = undefined;
+}
+
+async function historyRevision() {
+ try {
+ const info = await stat(HISTORY_FILE, { bigint: true });
+ await access(HISTORY_FILE, constants.R_OK);
+ return [info.dev, info.ino, info.size, info.mtimeNs, info.ctimeNs].join(':');
+ } catch (error) {
+ if (error.code === 'ENOENT') return null;
+ throw error;
+ }
+}
+
+function freezeRecord(value) {
+ if (value && typeof value === 'object') {
+ for (const child of Object.values(value)) freezeRecord(child);
+ Object.freeze(value);
+ }
+ return value;
+}
+
+/** Fresh, immutable card/filter data; search and detail still use loadHistory. */
+export function loadCompactHistory() {
+ if (compactRead) return compactRead;
+ compactRead = (async () => {
+ // One builder owns admission. A save during its read invalidates the epoch;
+ // external writers/replacements are detected on both sides of the read.
+ for (let attempt = 0; attempt < 3; attempt++) {
+ const epoch = revisionEpoch;
+ const before = await historyRevision();
+ if (before !== null && compactSnapshot?.revision === before && epoch === revisionEpoch) {
+ return compactSnapshot.rows;
+ }
+ const history = await loadHistory();
+ const after = await historyRevision();
+ if (epoch !== revisionEpoch || before !== after) continue;
+ const rows = freezeRecord(history.map(compactSource));
+ compactSnapshot = after === null ? undefined : { revision: after, rows };
+ return rows;
+ }
+ throw new Error('Video history changed repeatedly while reading its compact snapshot');
+ })().finally(() => { compactRead = undefined; });
+ return compactRead;
+}
// Resolve ONE entry by its history id, or `null` when no entry carries it (#4165).
// A history id is NOT the filename stem — `videoGen/local.js` names a clip