Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
perf([issue-11606]): reuse fresh compact video history across gallery pages (#11623)
[._.]/850125202fc8
diff --git a/server/services/mediaAssetIndex/gallery.js b/server/services/mediaAssetIndex/gallery.jsindex 3c02d0b3af..e4c8fff75f 100644--- a/server/services/mediaAssetIndex/gallery.js+++ b/server/services/mediaAssetIndex/gallery.js@@ -40,7 +40,9 @@ export async function listGalleryPage({kind = 'image', media = false, cover = false, videoSnapshot, collectionSnapshot, collectionId, universeId, entryCategory, entryKind, mediaKeys,} = {}, diskReader) {const mixed = media || kind !== 'image';- const videos = mixed ? videoSnapshot ?? await (await import('../videoGen/history.js')).loadHistory() : undefined;+ const videoReader = mixed && videoSnapshot == null ? await import('../videoGen/history.js') : null;+ const videos = mixed ? videoSnapshot ?? await (compact && !q.trim()+ ? videoReader.loadCompactHistory() : videoReader.loadHistory()) : undefined;const scope = await resolveScope({ collectionId, starred, mediaKeys, collectionSnapshot });const filters = { kind: kind === 'all' ? undefined : kind, q, hidden, filename,universeId, entryCategory, entryKind, mediaKeys: scope.mediaKeys, excludeKeys: scope.excludeKeys, videos };diff --git a/server/services/mediaAssetIndex/gallery.test.js b/server/services/mediaAssetIndex/gallery.test.jsindex 43564151dd..b9ed946677 100644--- a/server/services/mediaAssetIndex/gallery.test.js+++ b/server/services/mediaAssetIndex/gallery.test.js@@ -4,7 +4,7 @@ import { imageToRow } from './logic.js';import { query } from '../../lib/db.js';const { listAnnotations, loadHistory, listCollections, getCollection } = vi.hoisted(() => ({ listAnnotations: vi.fn(), loadHistory: vi.fn(), listCollections: vi.fn(), getCollection: vi.fn() }));-vi.mock('../videoGen/history.js', () => ({ loadHistory }));+vi.mock('../videoGen/history.js', () => ({ loadHistory, loadCompactHistory: loadHistory }));vi.mock('../mediaCollections.js', () => ({ listCollections, getCollection }));vi.mock('../mediaAnnotations.js', () => ({ listAnnotations }));diff --git a/server/services/mediaAssetIndex/galleryCollections.db.test.js b/server/services/mediaAssetIndex/galleryCollections.db.test.jsindex aafe5de7b0..41eea99486 100644--- a/server/services/mediaAssetIndex/galleryCollections.db.test.js+++ b/server/services/mediaAssetIndex/galleryCollections.db.test.js@@ -9,7 +9,10 @@ import { compactGalleryRecord } from './logic.js';const state = vi.hoisted(() => ({ file: false, images: [], videos: [], collections: [], annotations: {}, query: vi.fn() }));vi.mock('../../lib/runtimeEnv.js', () => ({ isTestRunner: () => state.file }));vi.mock('../../lib/db.js', () => ({ query: (...args) => state.query(...args) }));-vi.mock('../videoGen/history.js', () => ({ loadHistory: async () => state.videos }));+vi.mock('../videoGen/history.js', async () => {+ const { compactSource } = await import('./logic.js');+ return { loadHistory: async () => state.videos, loadCompactHistory: async () => state.videos.map(compactSource) };+});vi.mock('../mediaCollections.js', () => ({ listCollections: async () => state.collections,getCollection: async id => state.collections.find(collection => collection.id === id) }));vi.mock('../mediaAnnotations.js', () => ({ listAnnotations: async () => state.annotations }));@@ -28,7 +31,14 @@ beforeAll(async () => {// A session-private table shadows the index, including for EXPLAIN: no live rows.const ddl = mediaDdl.filter(sql => /(?:TABLE IF NOT EXISTS media_assets|INDEX IF NOT EXISTS idx_media_assets_)/.test(sql));for (const sql of ddl) await client.query(sql.replace('CREATE TABLE IF NOT EXISTS', 'CREATE TEMP TABLE'));- state.query.mockImplementation((...args) => client.query(...args));+ // Production uses a pool; this fixture shares one session for its temp table.+ // Serialize that session rather than relying on pg's deprecated query queue.+ let queryTail = Promise.resolve();+ state.query.mockImplementation((...args) => {+ const result = queryTail.then(() => client.query(...args));+ queryTail = result.catch(() => {});+ return result;+ });available = true;});afterAll(async () => { await client?.end(); vi.unstubAllEnvs(); });diff --git a/server/services/mediaAssetIndex/logic.js b/server/services/mediaAssetIndex/logic.jsindex af86d2de20..0dcb9a5636 100644--- a/server/services/mediaAssetIndex/logic.js+++ b/server/services/mediaAssetIndex/logic.js@@ -101,7 +101,10 @@ export function compactSource(data) {if (!data || typeof data !== 'object') return data;const source = {};for (const field of COMPACT_SOURCE_FIELDS) if (data[field] !== undefined) source[field] = data[field];- const prompt = nonBlank(data.prompt) || nonBlank(data.metadata?.prompt);+ // Re-projecting a cached source must preserve whitespace-only prefixes: the+ // original nonblank text may lie beyond the preview boundary.+ const prompt = nonBlank(data.prompt) || nonBlank(data.metadata?.prompt)+ || (typeof data[COMPACT_PREVIEW_KEY] === 'string' ? data[COMPACT_PREVIEW_KEY] : null);if (prompt) source[COMPACT_PREVIEW_KEY] = prompt.slice(0, COMPACT_PROMPT_CHARS + 1);return source;}diff --git a/server/services/videoGen/compactHistory.test.js b/server/services/videoGen/compactHistory.test.jsnew file mode 100644index 0000000000..e2cd1cacad--- /dev/null+++ b/server/services/videoGen/compactHistory.test.js@@ -0,0 +1,151 @@+import { afterAll, beforeEach, describe, expect, it, vi } from 'vitest';+import { mkdtemp, readFile, writeFile, rename, rm, utimes, stat } from 'node:fs/promises';+import { join } from 'node:path';+import { tmpdir } from 'node:os';+import { mediaFixture, CARDINALITIES } from '../../../scripts/perf/collectionFixtureData.js';+import { compactGalleryRecord } from '../mediaAssetIndex/logic.js';++const io = vi.hoisted(() => ({ root: '', reads: 0, afterRead: null, accessError: null }));+vi.mock('node:fs/promises', async importOriginal => {+ const real = await importOriginal();+ return { ...real, access: async (...args) => {+ if (io.accessError) throw io.accessError;+ return real.access(...args);+ } };+});+vi.mock('../../lib/fileUtils.js', async () => {+ const { readJSONFile } = await import('../../lib/jsonIo.js');+ const { atomicWrite } = await import('../../lib/fileCore.js');+ return { PATHS: { get data() { return io.root; } }, atomicWrite,+ readJSONFile: async (...args) => {+ io.reads++;+ const result = await readJSONFile(...args);+ if (io.afterRead) await io.afterRead();+ return result;+ } };+});+vi.mock('../mediaCollections.js', () => ({ listCollections: async () => [], getCollection: async () => null }));+vi.mock('../mediaAnnotations.js', () => ({ listAnnotations: async () => ({ 'video:synthetic-video-00001': { own: { starred: true } } }) }));++let history;+let file;+let listGalleryPage;+beforeEach(async () => {+ vi.resetModules();+ if (io.root) await rm(io.root, { recursive: true, force: true });+ io.root = await mkdtemp(join(tmpdir(), 'compact-history-'));+ io.reads = 0;+ io.afterRead = null;+ io.accessError = null;+ file = join(io.root, 'video-history.json');+ history = await import('./history.js');+ ({ listGalleryPage } = await import('../mediaAssetIndex/gallery.js'));+});+afterAll(async () => { await rm(io.root, { recursive: true, force: true }); });+const persist = rows => writeFile(file, JSON.stringify(rows));++describe('compact history through gallery reads', () => {+ it('reuses one immutable bounded projection for concurrent and repeated 1,200-video pages', async () => {+ const rows = Array.from({ length: CARDINALITIES.videos }, (_, i) => mediaFixture('video', i));+ await persist(rows);+ const fullTimings = [];+ for (let index = 0; index < 5; index++) {+ const started = performance.now();+ await history.loadHistory();+ fullTimings.push(performance.now() - started);+ }+ expect(io.reads).toBe(5);+ io.reads = 0;+ const options = { media: true, kind: 'all', summary: true, compact: true, hidden: false, limit: 60, videoSnapshot: null };+ const start = performance.now();+ const pages = await Promise.all(Array.from({ length: 5 }, (_, i) => listGalleryPage({ ...options, offset: i * 60 }, async () => [])));+ const coldMs = performance.now() - start;+ const warmStart = performance.now();+ await listGalleryPage(options, async () => []);+ const warmMs = performance.now() - warmStart;+ expect(io.reads).toBe(1);+ expect(pages.every(page => page.items.length === 60 && page.total === 1080)).toBe(true);+ const snapshot = await history.loadCompactHistory();+ expect(Object.isFrozen(snapshot)).toBe(true);+ expect(Object.isFrozen(snapshot[0])).toBe(true);+ expect(snapshot[0]._promptPreviewSource).toHaveLength(241);+ expect(snapshot[0]).not.toHaveProperty('status');+ expect(compactGalleryRecord(snapshot[0])).toEqual(compactGalleryRecord(rows[0]));+ console.log(`📊 Compact video fixture: fileBytes=${(await stat(file)).size}, compactBytes=${Buffer.byteLength(JSON.stringify(snapshot))}, fullReads=5, fullMedianMs=${fullTimings.sort((a, b) => a - b)[2].toFixed(2)}, compactReads=${io.reads}, concurrentMs=${coldMs.toFixed(2)}, warmPageMs=${warmMs.toFixed(2)}`);+ });++ it('preserves page/count filters and full search/detail beyond the preview', async () => {+ const rows = Array.from({ length: 12 }, (_, i) => ({ ...mediaFixture('video', i),+ prompt: ' '.repeat(300) + 'tailneedle', universeId: i % 2 ? 'one' : 'two', stitchedFrom: ['original'] }));+ await persist(rows);+ const collectionSnapshot = [{ id: 'set', items: rows.slice(0, 7).map(row => ({ kind: 'video', ref: row.id })) }];+ for (const filters of [{}, { kind: 'video' }, { hidden: true }, { collectionId: 'set' },+ { collectionId: 'unsorted' }, { starred: true }, { universeId: 'one' }, { offset: 3 }]) {+ const options = { media: true, kind: 'all', summary: true, hidden: false, limit: 3, collectionSnapshot, ...filters };+ const full = await listGalleryPage({ ...options, videoSnapshot: rows }, async () => []);+ const compact = await listGalleryPage({ ...options, compact: true }, async () => []);+ expect(compact).toEqual({ ...full, items: full.items.map(item => ({ ...item, data: compactGalleryRecord(item.data) })) });+ }+ expect(io.reads).toBe(1);+ expect((await listGalleryPage({ media: true, kind: 'video', compact: true, q: 'tailneedle' }, async () => [])).total).toBe(12);+ expect(await history.getHistoryItem(rows[0].id)).toEqual(rows[0]);+ const snapshot = await history.loadCompactHistory();+ expect(() => snapshot[0].stitchedFrom.push('mutated')).toThrow();+ expect(io.reads).toBe(3);+ });++ it('refreshes after local create/update/delete and external same-size or atomic replacement', async () => {+ await history.saveHistory([{ id: 'a', prompt: 'old' }]);+ expect((await history.loadCompactHistory())[0]._promptPreviewSource).toBe('old');+ await history.mutateVideoHistory(rows => [...rows, { id: 'b' }]);+ expect(await history.loadCompactHistory()).toHaveLength(2);+ await history.mutateVideoHistory(rows => rows.map(row => ({ ...row, hidden: true })));+ expect((await history.loadCompactHistory())[0].hidden).toBe(true);+ await history.mutateVideoHistory(rows => rows.filter(row => row.id === 'a'));+ expect(await history.loadCompactHistory()).toHaveLength(1);+ const before = await stat(file);+ const original = await readFile(file, 'utf8');+ await writeFile(file, original.replace('old', 'new'));+ await utimes(file, before.atime, before.mtime);+ expect((await history.loadCompactHistory())[0]._promptPreviewSource).toBe('new');+ await writeFile(file + '.restore', JSON.stringify([{ id: 'restored' }]));+ await rename(file + '.restore', file);+ expect(await history.loadCompactHistory()).toEqual([{ id: 'restored' }]);+ });++ it('does not publish a read overtaken by a save or external replacement', async () => {+ await persist([{ id: 'old' }]);+ io.afterRead = async () => {+ io.afterRead = null;+ await history.saveHistory([{ id: 'new' }]);+ };+ expect(await history.loadCompactHistory()).toEqual([{ id: 'new' }]);+ await persist([{ id: 'external-old' }]);+ io.afterRead = async () => {+ io.afterRead = null;+ await writeFile(file + '.restore', JSON.stringify([{ id: 'external-new' }]));+ await rename(file + '.restore', file);+ };+ const results = await Promise.all([history.loadCompactHistory(), history.loadCompactHistory()]);+ expect(results).toEqual([[{ id: 'external-new' }], [{ id: 'external-new' }]]);+ expect(io.reads).toBe(4);+ });++ it('never serves a cached value over corrupt, unreadable, empty or missing history', async () => {+ await persist([{ id: 'cached' }]);+ await history.loadCompactHistory();+ io.accessError = Object.assign(new Error('unreadable'), { code: 'EACCES' });+ await expect(history.loadCompactHistory()).rejects.toThrow('unreadable');+ io.accessError = null;+ await writeFile(file, 'invalid');+ await expect(history.loadCompactHistory()).rejects.toThrow();+ await writeFile(file, '');+ await expect(history.loadCompactHistory()).rejects.toThrow();+ await persist([]);+ expect(await history.loadCompactHistory()).toEqual([]);+ await rm(file);+ expect(await history.loadCompactHistory()).toEqual([]);+ await persist([{ id: 'returned' }]);+ expect(await history.loadCompactHistory()).toEqual([{ id: 'returned' }]);+ });+});diff --git a/server/services/videoGen/history.js b/server/services/videoGen/history.jsindex 366dcdde5b..81747b7490 100644--- a/server/services/videoGen/history.js+++ b/server/services/videoGen/history.js@@ -8,6 +8,9 @@*/import { join } from 'path';+import { stat, access } from 'node:fs/promises';+import { constants } from 'node:fs';+import { compactSource } from '../mediaAssetIndex/logic.js';import { PATHS, readJSONFile, atomicWrite } from '../../lib/fileUtils.js';const HISTORY_FILE = join(PATHS.data, 'video-history.json');@@ -18,7 +21,57 @@ const HISTORY_FILE = join(PATHS.data, 'video-history.json');// render history. The Media History page also counts this list ("Show more (N// remaining)"), where a fake 0 reads as fact. Absent stays a real first-run empty.export const loadHistory = () => readJSONFile(HISTORY_FILE, [], { strict: true });-export const saveHistory = (h) => atomicWrite(HISTORY_FILE, h);+let compactSnapshot;+let compactRead;+let revisionEpoch = 0;+export async function saveHistory(h) {+ await atomicWrite(HISTORY_FILE, h);+ revisionEpoch++;+ compactSnapshot = undefined;+}++async function historyRevision() {+ try {+ const info = await stat(HISTORY_FILE, { bigint: true });+ await access(HISTORY_FILE, constants.R_OK);+ return [info.dev, info.ino, info.size, info.mtimeNs, info.ctimeNs].join(':');+ } catch (error) {+ if (error.code === 'ENOENT') return null;+ throw error;+ }+}++function freezeRecord(value) {+ if (value && typeof value === 'object') {+ for (const child of Object.values(value)) freezeRecord(child);+ Object.freeze(value);+ }+ return value;+}++/** Fresh, immutable card/filter data; search and detail still use loadHistory. */+export function loadCompactHistory() {+ if (compactRead) return compactRead;+ compactRead = (async () => {+ // One builder owns admission. A save during its read invalidates the epoch;+ // external writers/replacements are detected on both sides of the read.+ for (let attempt = 0; attempt < 3; attempt++) {+ const epoch = revisionEpoch;+ const before = await historyRevision();+ if (before !== null && compactSnapshot?.revision === before && epoch === revisionEpoch) {+ return compactSnapshot.rows;+ }+ const history = await loadHistory();+ const after = await historyRevision();+ if (epoch !== revisionEpoch || before !== after) continue;+ const rows = freezeRecord(history.map(compactSource));+ compactSnapshot = after === null ? undefined : { revision: after, rows };+ return rows;+ }+ throw new Error('Video history changed repeatedly while reading its compact snapshot');+ })().finally(() => { compactRead = undefined; });+ return compactRead;+}// Resolve ONE entry by its history id, or `null` when no entry carries it (#4165).// A history id is NOT the filename stem — `videoGen/local.js` names a clip