Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

fix: hydrate only the requested music video project page

[._.]/ Adam Eivy68a1541db17a
diff --git a/server/routes/musicVideo.js b/server/routes/musicVideo.js
index 7f3485633d..466560520d 100644
--- a/server/routes/musicVideo.js
+++ b/server/routes/musicVideo.js
@@ -94,7 +94,7 @@ import {
musicVideoDocumentCandidateSchema,
musicVideoMixedMediaRegenerateSchema,
isPaginationRequested,
- paginateArray,
+ parsePagination,
} from '../lib/validation.js';
import { recordRenderPinFields } from '../lib/sharedSchemas.js';
import { PATHS } from '../lib/fileUtils.js';
@@ -104,6 +104,7 @@ import { isSupportedMusicUpload, MUSIC_UPLOAD_MAX_BYTES } from '../services/pipe
import { attachVocalStem, detachVocalStem } from '../services/musicVideo/vocalStem.js';
import {
listProjects,
+ listProjectPage,
getProject,
createProject,
cloneProject,
@@ -267,21 +268,19 @@ async function presentProjectForRead(project) {
router.get('/', asyncHandler(async (req, res) => {
const query = validateRequest(musicVideoProjectListQuerySchema, req.query);
- const projects = await Promise.all((await listProjects()).map((project) => presentProjectForRead(project)));
- if (query.summary === '1' || query.summary === 'true') {
- const summaries = projects.sort(compareMusicVideoProjectsNewestFirst)
- .map((project) => summarizeMusicVideoProject(project, project.productionReadiness));
+ const summary = query.summary === '1' || query.summary === 'true';
+ if (summary || isPaginationRequested(query)) {
// The cursor is the next page's offset; `cursor` wins over `offset`.
const cursor = parseInt(query.cursor, 10);
- const page = paginateArray(summaries, Number.isInteger(cursor) && cursor >= 0 ? { ...query, offset: String(cursor) } : query,
+ const paging = parsePagination(summary && Number.isInteger(cursor) && cursor >= 0 ? { ...query, offset: String(cursor) } : query,
{ defaultLimit: 50, maxLimit: 500 });
+ const page = await listProjectPage({ ...paging, ...(summary ? { compare: compareMusicVideoProjectsNewestFirst } : {}) });
+ const presented = await Promise.all(page.items.map((project) => presentProjectForRead(project)));
+ const items = summary ? presented.map(project => summarizeMusicVideoProject(project, project.productionReadiness)) : presented;
const next = page.offset + page.items.length;
- return res.json({ ...page, nextCursor: next < page.total ? String(next) : null });
+ return res.json({ ...page, items, ...(summary ? { nextCursor: next < page.total ? String(next) : null } : {}) });
}
- if (!isPaginationRequested(query)) {
- return res.json(projects);
- }
- res.json(paginateArray(projects, query, { defaultLimit: 50, maxLimit: 500 }));
+ res.json(await Promise.all((await listProjects()).map((project) => presentProjectForRead(project))));
}));
// Bounded projection for the Tracks page MIDI read-through (#10203): the newest
diff --git a/server/routes/musicVideo.test.js b/server/routes/musicVideo.test.js
index bafa054ab0..8c33d1b7f8 100644
--- a/server/routes/musicVideo.test.js
+++ b/server/routes/musicVideo.test.js
@@ -14,6 +14,7 @@ import { sharingCopyDownload } from '../services/musicVideo/sharingCopy.js';
vi.mock('../services/musicVideo/projects.js', () => ({
listProjects: vi.fn(async () => [{ id: 'mv-1', name: 'A' }]),
+ listProjectPage: vi.fn(),
getProject: vi.fn(),
createProject: vi.fn(async (d) => ({ id: 'mv-new', ...d })),
cloneProject: vi.fn(async (id, options) => ({
@@ -115,6 +116,11 @@ describe('musicVideo routes', () => {
app.use(express.json());
app.use('/api/music-video', musicVideoRoutes);
vi.clearAllMocks();
+ svc.listProjectPage.mockImplementation(async ({ limit, offset, compare }) => {
+ const projects = [...await svc.listProjects()];
+ if (compare) projects.sort(compare);
+ return { items: projects.slice(offset, offset + limit), total: projects.length, limit, offset };
+ });
});
it('GET / lists projects', async () => {
@@ -246,6 +252,20 @@ describe('musicVideo routes', () => {
expect(r2.body.items[0].id).toBe('mv-older');
});
+ it('presents only the requested summary page, including cursor precedence and an empty page', async () => {
+ const readOffPage = vi.fn(() => { throw new Error('Off-page detail was inspected'); });
+ const older = { id: 'older', createdAt: '2026-01-01', get scenes() { return readOffPage(); } };
+ const newer = { id: 'newer', createdAt: '2026-02-01', scenes: [] };
+ svc.listProjects.mockResolvedValue([older, newer]);
+
+ const first = await request(app).get('/api/music-video?summary=1&limit=1&offset=1&cursor=0');
+ expect(first.status).toBe(200);
+ expect(first.body).toMatchObject({ total: 2, offset: 0, nextCursor: '1', items: [{ id: 'newer' }] });
+ const empty = await request(app).get('/api/music-video?summary=1&limit=1&cursor=2');
+ expect(empty.body).toMatchObject({ total: 2, offset: 2, nextCursor: null, items: [] });
+ expect(readOffPage).not.toHaveBeenCalled();
+ });
+
it('GET /:id 404s when missing', async () => {
svc.getProject.mockResolvedValue(null);
const r = await request(app).get('/api/music-video/mv-x');
diff --git a/server/services/musicVideo/projects.js b/server/services/musicVideo/projects.js
index c58a675cda..bd17cee78e 100644
--- a/server/services/musicVideo/projects.js
+++ b/server/services/musicVideo/projects.js
@@ -71,6 +71,10 @@ export async function listProjects(options = {}) {
return (await selectBackend()).listProjects(options);
}
+export async function listProjectPage(options = {}) {
+ return (await selectBackend()).listProjectPage(options);
+}
+
/** Null when missing; a caller that answers 404 for that uses requireProject.js. */
export async function getProject(id, options = {}) {
return presentMusicVideoDependencies(await (await selectBackend()).getProject(id, options));
diff --git a/server/services/musicVideo/projectsDB.js b/server/services/musicVideo/projectsDB.js
index dc022b4e78..4f8461af63 100644
--- a/server/services/musicVideo/projectsDB.js
+++ b/server/services/musicVideo/projectsDB.js
@@ -20,7 +20,7 @@ const store = createProjectDbStore({
const { withLockedProject, rowToProject } = store;
export const {
- persist, listProjects, getProject, getProjectsByIds, listProjectIds,
+ persist, listProjects, listProjectPage, getProject, getProjectsByIds, listProjectIds,
} = store;
async function cloneProjectUnqueued(id, options = {}) {
diff --git a/server/services/musicVideo/projectsDB.test.js b/server/services/musicVideo/projectsDB.test.js
index 88fa5eab9e..1584574cbf 100644
--- a/server/services/musicVideo/projectsDB.test.js
+++ b/server/services/musicVideo/projectsDB.test.js
@@ -12,6 +12,9 @@ import { rmSync } from 'fs';
import { join } from 'path';
import { captureMusicVideoEvidence, musicVideoDependencyImpact } from '../../lib/musicVideoDependencies.js';
import { checkHealth, ensureSchema, query, close } from '../../lib/db.js';
+import * as database from '../../lib/db.js';
+import { toPgJson } from '../../lib/pgJson.js';
+import { compareMusicVideoProjectsNewestFirst } from '../../lib/musicVideoSummary.js';
import { requireDbOrSkip } from '../../lib/dbTestGate.js';
import { getSyncBaseHash, __resetBaseHashCacheForTests } from '../../lib/conflictJournal.js';
@@ -142,6 +145,47 @@ describe.skipIf(!runDb)('music video projects DB adapter', () => {
expect((await db.listProjects()).find((item) => item.name === 'Legacy row')?.id).toBe(id);
});
+ it('hydrates only a requested page while preserving legacy timestamp fallbacks and excluding tombstones', async () => {
+ const records = Array.from({ length: 20 }, (_, i) => ({ id: `mv-page-${i}`,
+ createdAt: i === 19 ? null : `2026-01-${String(i + 1).padStart(2, '0')}`,
+ updatedAt: `2026-02-${String(i + 1).padStart(2, '0')}`,
+ composition: { document: { source: 'x'.repeat(128 * 1024) } } }));
+ for (const record of records) {
+ await query(`INSERT INTO music_video_projects (id, status, data, created_at, updated_at, deleted)
+ VALUES ($1, 'draft', $2::jsonb, '2026-01-01', '2026-01-01', $3)`,
+ [record.id, toPgJson(record), record.id === 'mv-page-18']);
+ }
+ const read = vi.spyOn(database, 'query');
+ try {
+ const all = await db.listProjects();
+ const baselineCalls = [...read.mock.calls];
+ read.mockClear();
+ const expected = [...all].sort(compareMusicVideoProjectsNewestFirst);
+ expect(expected[0].id).toBe('mv-page-19');
+ const page = await db.listProjectPage({ limit: 1, offset: 0, compare: compareMusicVideoProjectsNewestFirst });
+ expect(page).toEqual({ items: expected.slice(0, 1), total: 19, limit: 1, offset: 0 });
+ const rows = (await Promise.all(read.mock.results.map(result => result.value))).flatMap(result => result.rows);
+ expect(rows.filter(row => row.data).map(row => row.id)).toEqual([expected[0].id]);
+ if (process.env.PORTOS_COLLECTION_BENCHMARK === '1') {
+ process.stdout.write(JSON.stringify({ projects: all.length, fullBytes: Buffer.byteLength(toPgJson(all)),
+ pageReadBytes: Buffer.byteLength(toPgJson(rows)), hydratedProjects: rows.filter(row => row.data).length }) + '\n');
+ const calls = [...baselineCalls, ...read.mock.calls];
+ for (const [sql, params] of calls) {
+ const { rows: explained } = await query(`EXPLAIN (ANALYZE, BUFFERS, FORMAT JSON) ${sql}`, params);
+ const plan = explained[0]['QUERY PLAN'][0];
+ process.stdout.write(JSON.stringify({ operation: sql.includes('ANY') ? 'page' : sql.includes('AS "createdAt"') ? 'headers' : 'full',
+ node: plan.Plan['Node Type'], executionMs: plan['Execution Time'], rows: plan.Plan['Actual Rows'],
+ tempReadBlocks: plan.Plan['Temp Read Blocks'], tempWrittenBlocks: plan.Plan['Temp Written Blocks'] }) + '\n');
+ }
+ }
+ expect(await db.listProjectPage({ limit: 1, offset: 19, compare: compareMusicVideoProjectsNewestFirst }))
+ .toEqual({ items: [], total: 19, limit: 1, offset: 19 });
+ expect((await db.listProjectPage({ limit: 2, offset: 1 })).items).toEqual(all.slice(1, 3));
+ } finally {
+ read.mockRestore();
+ }
+ });
+
it('rejects persistence when the project id is missing', async () => {
await expect(db.persist(query, { name: 'Missing id' })).rejects.toMatchObject({ code: 'PROJECT_ID_MISSING' });
});
diff --git a/server/services/musicVideo/projectsFile.js b/server/services/musicVideo/projectsFile.js
index 6887dbadd9..448188065f 100644
--- a/server/services/musicVideo/projectsFile.js
+++ b/server/services/musicVideo/projectsFile.js
@@ -19,7 +19,7 @@ const store = createProjectFileStore({
});
export const {
- loadAll, saveAll, loadAllAndIndex, listProjects, getProject, getProjectsByIds, listProjectIds,
+ loadAll, saveAll, loadAllAndIndex, listProjects, listProjectPage, getProject, getProjectsByIds, listProjectIds,
} = store;
// Every mutator below is a load → modify → save of the ONE projects file, so
diff --git a/server/services/projectDbStore.js b/server/services/projectDbStore.js
index 1ad6f05f14..733443900e 100644
--- a/server/services/projectDbStore.js
+++ b/server/services/projectDbStore.js
@@ -60,6 +60,19 @@ export function createProjectDbStore({ table, kind, idPrefix, logEmoji, logLabel
return result.rows.map(rowToProject);
}
+ // Sort the small headers with the caller's legacy comparator before loading
+ // detail JSONB. Timestamp fallbacks need not match the mirrored SQL columns.
+ async function listProjectPage({ includeDeleted = false, limit = 50, offset = 0, compare } = {}) {
+ const where = includeDeleted ? '' : ' WHERE deleted = FALSE';
+ const { rows: headers } = await query(`SELECT id, data->'createdAt' AS "createdAt", data->'updatedAt' AS "updatedAt"
+ FROM ${table}${where} ORDER BY created_at ASC`);
+ if (compare) headers.sort(compare);
+ const page = headers.slice(offset, offset + limit);
+ const projects = new Map((await getProjectsByIds(page.map(({ id }) => id), { includeDeleted }))
+ .map(project => [project.id, project]));
+ return { items: page.map(({ id }) => projects.get(id)).filter(Boolean), total: headers.length, limit, offset };
+ }
+
async function getProject(id, { includeDeleted = false } = {}) {
const result = await query(`SELECT id, data FROM ${table} WHERE id = $1`, [id]);
const project = rowToProject(result.rows[0]);
@@ -171,6 +184,7 @@ export function createProjectDbStore({ table, kind, idPrefix, logEmoji, logLabel
rowToProject,
persist,
listProjects,
+ listProjectPage,
getProject,
getProjectsByIds,
listProjectIds,
diff --git a/server/services/projectFileStore.js b/server/services/projectFileStore.js
index 40a4a1cb22..7bb62c245a 100644
--- a/server/services/projectFileStore.js
+++ b/server/services/projectFileStore.js
@@ -41,6 +41,12 @@ export function createProjectFileStore({ file, kind, idPrefix, logEmoji, logLabe
return includeDeleted ? all : all.filter((project) => !project.deleted);
}
+ async function listProjectPage({ includeDeleted = false, limit = 50, offset = 0, compare } = {}) {
+ const projects = await listProjects({ includeDeleted });
+ if (compare) projects.sort(compare);
+ return { items: projects.slice(offset, offset + limit), total: projects.length, limit, offset };
+ }
+
async function getProject(id, { includeDeleted = false } = {}) {
const all = await loadAll();
const found = all.find((project) => project.id === id);
@@ -122,6 +128,7 @@ export function createProjectFileStore({ file, kind, idPrefix, logEmoji, logLabe
saveAll,
loadAllAndIndex,
listProjects,
+ listProjectPage,
getProject,
getProjectsByIds,
listProjectIds,