Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
perf: catalog ingredient list reads each identity view once per statement (#12989)
[._.]/ Adam Eivy9f10d2c44b77
diff --git a/scripts/lib/dbTestFiles.js b/scripts/lib/dbTestFiles.jsindex 318cf683c3..2e9e87f3fd 100644--- a/scripts/lib/dbTestFiles.js+++ b/scripts/lib/dbTestFiles.js@@ -31,6 +31,7 @@ export const DB_TEST_INCLUDE = ['services/backup.db.test.js','services/catalogDB.test.js','services/catalogDB.facets.db.test.js',+ 'services/catalogDB/listIngredients.db.test.js','services/catalogDB.media.db.test.js','services/catalogSync.tombstoneRevival.db.test.js','services/catalogSync.changeClock.db.test.js',diff --git a/server/services/catalogDB/ingredients.js b/server/services/catalogDB/ingredients.jsindex f4ff9dc1fb..34a27a526a 100644--- a/server/services/catalogDB/ingredients.js+++ b/server/services/catalogDB/ingredients.js@@ -23,6 +23,8 @@ import {REVISION_SOURCES,rowToIngredient,rowToRevision,+ CANONICAL_REFS_CTE,+ inSet,HAS_LIVE_HOMING_REF,HAS_ANY_HOMING_REF,HOMING_REF_KINDS_SQL,@@ -360,33 +362,47 @@ export async function reviveDeletedIngredient(id, { type, name, payload = {}, ta// directly instead of fetching-then-JS-filtering.const INGREDIENT_LIGHT_COLS = 'id, type, name, payload, tags, embedding_model, origin_instance_id, created_at, updated_at, deleted, deleted_at, sync_sequence';-// Correlated subquery that yields the card-thumbnail media key for each row:-// the live portrait when set, otherwise the most recent live reference image.-// Non-image media kinds (audio/video/document) are excluded so a card never-// points an <img> at an unrenderable key. Working images filed with a role —-// reference sheets (`sheet:*`), looks (`look:*`) and plates — never become the-// cover. Used only by listIngredients' light path (see the note there).-const THUMBNAIL_KEY_SUBQUERY = `(- SELECT m.media_key FROM catalog_identity_read_media m- WHERE catalog_canonical_ingredient_id(m.ingredient_id) = catalog_ingredients.id- AND m.deleted = false- AND m.kind IN ('portrait', 'reference')- AND (m.role IS NULL OR NOT (m.role LIKE 'sheet:%' OR m.role LIKE 'look:%' OR m.role LIKE 'archive:%' OR m.role = 'plate'))- ORDER BY (m.kind = 'portrait') DESC, m.created_at DESC- LIMIT 1- ) AS thumbnail_key`;--// Per-kind count of the distinct records each row is linked to ({ universe: 1,-// series: 2 }), for the "used in" line on library cards. Same light-path scope-// as the thumbnail subquery.-const REF_USAGE_SUBQUERY = `(- SELECT jsonb_object_agg(u.ref_kind, u.n) FROM (- SELECT r.ref_kind, COUNT(DISTINCT r.ref_id) AS n FROM catalog_identity_read_refs r- WHERE catalog_canonical_ingredient_id(r.ingredient_id) = catalog_ingredients.id AND r.deleted = false- AND (r.ref_kind NOT IN (${HOMING_REF_KINDS_SQL}) OR ${liveHomingTargetSql('r')})- GROUP BY r.ref_kind- ) u- ) AS ref_usage`;+// Card decorations for one already-limited page, in one grouped pass over each+// identity read view (#12989 — a per-row correlated subquery re-evaluated the+// whole view for every row). Light list path only (see listIngredients).+// - thumbnail_key: the live portrait when set, otherwise the most recent live+// reference image. Mirrors listMediaForIngredient's `(kind='portrait') DESC,+// created_at DESC` ordering so the card shows the detail page's avatar.+// Non-image kinds are excluded so a card never points an <img> at an+// unrenderable key, and role-filed working images — reference sheets+// (`sheet:*`), looks (`look:*`), archives and plates — never become the cover.+// - ref_usage: per-kind count of the distinct records the row is linked to+// ({ universe: 1, series: 2 }) for the "used in" line; a homing kind counts+// only live targets.+async function decorateIngredientPage(ids) {+ if (ids.length === 0) return new Map();+ const result = await query(+ `WITH page AS (SELECT DISTINCT unnest($1::text[]) AS id),+ usage AS (+ SELECT cid, jsonb_object_agg(ref_kind, n) AS ref_usage FROM (+ SELECT r.cid, r.ref_kind, COUNT(DISTINCT r.ref_id) AS n+ FROM (SELECT catalog_canonical_ingredient_id(ingredient_id) AS cid, ref_kind, ref_id+ FROM catalog_identity_read_refs WHERE deleted = false) r+ WHERE ${inSet('r.cid', 'SELECT id FROM page')}+ AND (r.ref_kind NOT IN (${HOMING_REF_KINDS_SQL}) OR ${liveHomingTargetSql('r')})+ GROUP BY r.cid, r.ref_kind+ ) u GROUP BY cid+ ),+ thumbs AS (+ SELECT DISTINCT ON (m.cid) m.cid, m.media_key+ FROM (SELECT catalog_canonical_ingredient_id(ingredient_id) AS cid, media_key, kind, created_at+ FROM catalog_identity_read_media+ WHERE deleted = false AND kind IN ('portrait', 'reference')+ AND (role IS NULL OR NOT (role LIKE 'sheet:%' OR role LIKE 'look:%' OR role LIKE 'archive:%' OR role = 'plate'))) m+ WHERE ${inSet('m.cid', 'SELECT id FROM page')}+ ORDER BY m.cid, (m.kind = 'portrait') DESC, m.created_at DESC+ )+ SELECT page.id, thumbs.media_key AS thumbnail_key, usage.ref_usage+ FROM page LEFT JOIN thumbs ON thumbs.cid = page.id LEFT JOIN usage ON usage.cid = page.id`,+ [ids],+ );+ return new Map(result.rows.map((row) => [row.id, row]));+}export async function listIngredients({ ids, type, tag, query: q, refKind = null, refId = null, unlinked = false, orphaned = false, scrapId = null, limit = 50, offset = 0, includeEmbedding = false, embeddingMissing = false, staleEmbeddingModel = null } = {}) {const conditions = ['deleted = false', 'id = catalog_canonical_ingredient_id(id)'];@@ -397,6 +413,8 @@ export async function listIngredients({ ids, type, tag, query: q, refKind = null// caller gets exactly the requested (still non-deleted) rows.const byIds = Array.isArray(ids) && ids.length > 0;let qIdx = null;+ // Set by any filter that reads canonical_refs (CANONICAL_REFS_CTE).+ let needsRefs = false;if (byIds) {conditions.push(`id IN (SELECT catalog_canonical_ingredient_id(requested) FROM unnest($${idx++}::text[]) requested)`);params.push(ids);@@ -414,10 +432,12 @@ export async function listIngredients({ ids, type, tag, query: q, refKind = nullconditions.push(`search_tsv @@ websearch_to_tsquery('english', $${qIdx})`);params.push(q);}- // Album/facet filters (#1762). EXISTS subquery (not a JOIN) so a row linked- // under multiple roles isn't duplicated — no DISTINCT needed. Composes with- // type/tag/q above. `unlinked` and `orphaned` are mutually exclusive views- // of the un-homed set; the route schema rejects combining them with ref.+ // Album/facet filters (#1762). A set-membership test (not a JOIN) so a row+ // linked under multiple roles isn't duplicated — no DISTINCT needed — over+ // the statement-level canonical_refs CTE, so the read view is evaluated once+ // per statement, not once per row (#12989). Composes with type/tag/q above.+ // `unlinked` and `orphaned` are mutually exclusive views of the un-homed+ // set; the route schema rejects combining them with ref.if (refKind === 'universe' && refId) {// A universe album/filter is the universe's WHOLE membership: ingredients// linked directly to the universe OR to any live series under it (decision@@ -425,33 +445,35 @@ export async function listIngredients({ ids, type, tag, query: q, refKind = null// ingredient must still land in its parent universe). $refId is referenced// twice via one placeholder.const refIdx = idx++;- conditions.push(`EXISTS (SELECT 1 FROM catalog_identity_read_refs r- WHERE catalog_canonical_ingredient_id(r.ingredient_id) = catalog_ingredients.id AND r.deleted = false- AND ((r.ref_kind = 'universe' AND r.ref_id = $${refIdx})+ needsRefs = true;+ conditions.push(inSet('id', `SELECT r.cid FROM canonical_refs r+ WHERE (r.ref_kind = 'universe' AND r.ref_id = $${refIdx})OR (r.ref_kind = 'series' AND r.ref_id IN (- SELECT id FROM pipeline_series WHERE universe_id = $${refIdx} AND deleted = false))))`);+ SELECT id FROM pipeline_series WHERE universe_id = $${refIdx} AND deleted = false))`));params.push(refId);} else if (refKind && refId) {- conditions.push(`EXISTS (SELECT 1 FROM catalog_identity_read_refs r- WHERE catalog_canonical_ingredient_id(r.ingredient_id) = catalog_ingredients.id AND r.deleted = false- AND r.ref_kind = $${idx++} AND r.ref_id = $${idx++})`);+ needsRefs = true;+ conditions.push(inSet('id', `SELECT r.cid FROM canonical_refs r WHERE r.ref_kind = $${idx++} AND r.ref_id = $${idx++}`));params.push(refKind, refId);} else if (unlinked) {// "Unsorted / Raw": no universe/series ref at all.- conditions.push(`NOT ${HAS_ANY_HOMING_REF}`);+ needsRefs = true;+ conditions.push(`NOT (${HAS_ANY_HOMING_REF})`);} else if (orphaned) {// "Orphaned": has a universe/series ref, but none resolve to a live target.+ needsRefs = true;conditions.push(HAS_ANY_HOMING_REF);- conditions.push(`NOT ${HAS_LIVE_HOMING_REF}`);+ conditions.push(`NOT (${HAS_LIVE_HOMING_REF})`);}}// Source-scrap filter (#7617): "everything extracted from this piece" — a// provenance dimension independent of the ids/type/tag/q/ref branches above,// so it composes with any of them (unlike unlinked/orphaned/refKind, which// are mutually exclusive album views enforced at the schema level).+ // One hashed pass over the sources read view (#12989).if (scrapId) {- conditions.push(`EXISTS (SELECT 1 FROM catalog_identity_read_sources cis- WHERE catalog_canonical_ingredient_id(cis.ingredient_id) = catalog_ingredients.id AND cis.scrap_id = $${idx++})`);+ conditions.push(inSet('id', `SELECT catalog_canonical_ingredient_id(cis.ingredient_id)+ FROM catalog_identity_read_sources cis WHERE cis.scrap_id = $${idx++}`));params.push(scrapId);}if (embeddingMissing) {@@ -471,21 +493,23 @@ export async function listIngredients({ ids, type, tag, query: q, refKind = null? `ORDER BY ts_rank_cd(search_tsv, websearch_to_tsquery('english', $${qIdx})) DESC, created_at DESC`: `ORDER BY created_at DESC`;params.push(limit, offset);- // Card-thumbnail key: the live portrait if one exists, else the most recent- // reference image. Mirrors listMediaForIngredient's `(kind='portrait') DESC,- // created_at DESC` ordering so the card shows the same image the detail page- // treats as the avatar. Scoped to the light-column list path only — the- // includeEmbedding admin paths (re-embed/backfill) don't render cards and- // don't pay for the correlated subquery. The field rides ALONGSIDE- // rowToIngredient's shape (attached below), so the sync/revision payloads- // that share rowToIngredient stay untouched.- const cols = includeEmbedding ? '*' : `${INGREDIENT_LIGHT_COLS}, ${THUMBNAIL_KEY_SUBQUERY}, ${REF_USAGE_SUBQUERY}`;+ // Card decorations (thumbnailKey, usage) are computed for the returned page+ // only, and only on the light-column path — the includeEmbedding admin paths+ // (re-embed/backfill) don't render cards. They ride ALONGSIDE+ // rowToIngredient's shape, so the sync/revision payloads that share+ // rowToIngredient stay untouched.+ const cols = includeEmbedding ? '*' : INGREDIENT_LIGHT_COLS;+ const withClause = needsRefs ? `WITH ${CANONICAL_REFS_CTE} ` : '';const result = await query(- `SELECT ${cols} FROM catalog_ingredients ${where} ${orderBy} LIMIT $${idx++} OFFSET $${idx}`,+ `${withClause}SELECT ${cols} FROM catalog_ingredients ${where} ${orderBy} LIMIT $${idx++} OFFSET $${idx}`,params,);+ const decorations = includeEmbedding ? new Map() : await decorateIngredientPage(result.rows.map((row) => row.id));return {- items: result.rows.map((row) => ({ ...rowToIngredient(row), thumbnailKey: row.thumbnail_key ?? null, usage: row.ref_usage ?? {} })),+ items: result.rows.map((row) => {+ const decoration = decorations.get(row.id);+ return { ...rowToIngredient(row), thumbnailKey: decoration?.thumbnail_key ?? null, usage: decoration?.ref_usage ?? {} };+ }),nextOffset: offset + result.rows.length,};}diff --git a/server/services/catalogDB/listIngredients.db.test.js b/server/services/catalogDB/listIngredients.db.test.jsnew file mode 100644index 0000000000..c56383cc05--- /dev/null+++ b/server/services/catalogDB/listIngredients.db.test.js@@ -0,0 +1,230 @@+/**+ * listIngredients reads the identity read views (catalog_identity_read_refs /+ * _media / _sources) once per statement, not once per ingredient row (#12989).+ *+ * The equivalence suite pins the rewritten album filters and card decorations+ * to the correlated-subquery statement they replaced (LEGACY_* below, frozen+ * from the pre-#12989 implementation) on a fixture that exercises every view+ * branch: live rows, a local merge (catalog_ingredient_merges) and a committed+ * shared-identity operation whose plan and snapshot rows feed the views.+ *+ * `*.db.test.js` → runs ONLY via `npm run test:db` against `portos_test`. The+ * DB is shared, so every comparison is scoped to this run's nonce tag.+ */++import { randomUUID } from 'node:crypto';+import { afterAll, beforeAll, describe, expect, it, vi } from 'vitest';++vi.mock('../../lib/db.js', async (importOriginal) => {+ const actual = await importOriginal();+ return { ...actual, query: vi.fn(actual.query) };+});+vi.mock('../instanceIdentity.js', () => ({ getInstanceId: async () => 'list-test-local', UNKNOWN_INSTANCE_ID: 'unknown' }));++const { checkHealth, close, ensureSchema, query } = await import('../../lib/db.js');+const { requireDbOrSkip } = await import('../../lib/dbTestGate.js');+const { listIngredients } = await import('./ingredients.js');+const { HOMING_REF_KINDS_SQL, liveHomingTargetSql } = await import('./shared.js');++const health = await checkHealth().catch((error) => ({ connected: false, error: error.message }));+const ready = requireDbOrSkip('services/catalogDB/listIngredients.db.test', health.connected, health.error);+if (ready) await ensureSchema();++const P = `list-${randomUUID()}`;+const TAG = `${P}-tag`;+const UNI = `${P}-uni`;+const SER = `${P}-ser`;+const DEAD_UNI = `${P}-dead-uni`;+const CD_DEAD = `${P}-cd`;+const SCRAP = `${P}-scrap`;+const OP = `${P}-op`;+const MERGE = `${P}-merge`;+const id = (name) => `${P}-${name}`;+const now = '2026-01-02T00:00:00.000Z';++// Frozen pre-#12989 statement: correlated subqueries over the read views.+const LEGACY_HAS_LIVE = `EXISTS (SELECT 1 FROM catalog_identity_read_refs r+ WHERE catalog_canonical_ingredient_id(r.ingredient_id) = catalog_ingredients.id AND r.deleted = false AND ${liveHomingTargetSql('r')})`;+const LEGACY_HAS_ANY = `EXISTS (SELECT 1 FROM catalog_identity_read_refs r+ WHERE catalog_canonical_ingredient_id(r.ingredient_id) = catalog_ingredients.id AND r.deleted = false AND r.ref_kind IN (${HOMING_REF_KINDS_SQL}))`;+const LEGACY_THUMBNAIL = `(SELECT m.media_key FROM catalog_identity_read_media m+ WHERE catalog_canonical_ingredient_id(m.ingredient_id) = catalog_ingredients.id AND m.deleted = false+ AND m.kind IN ('portrait', 'reference')+ AND (m.role IS NULL OR NOT (m.role LIKE 'sheet:%' OR m.role LIKE 'look:%' OR m.role LIKE 'archive:%' OR m.role = 'plate'))+ ORDER BY (m.kind = 'portrait') DESC, m.created_at DESC LIMIT 1) AS thumbnail_key`;+const LEGACY_USAGE = `(SELECT jsonb_object_agg(u.ref_kind, u.n) FROM (+ SELECT r.ref_kind, COUNT(DISTINCT r.ref_id) AS n FROM catalog_identity_read_refs r+ WHERE catalog_canonical_ingredient_id(r.ingredient_id) = catalog_ingredients.id AND r.deleted = false+ AND (r.ref_kind NOT IN (${HOMING_REF_KINDS_SQL}) OR ${liveHomingTargetSql('r')})+ GROUP BY r.ref_kind) u) AS ref_usage`;++async function legacyList({ refKind, refId, unlinked, orphaned, scrapId }) {+ const conditions = ['deleted = false', 'id = catalog_canonical_ingredient_id(id)', '$1 = ANY(tags)'];+ const params = [TAG];+ if (refKind === 'universe') {+ params.push(refId);+ conditions.push(`EXISTS (SELECT 1 FROM catalog_identity_read_refs r+ WHERE catalog_canonical_ingredient_id(r.ingredient_id) = catalog_ingredients.id AND r.deleted = false+ AND ((r.ref_kind = 'universe' AND r.ref_id = $2)+ OR (r.ref_kind = 'series' AND r.ref_id IN (SELECT id FROM pipeline_series WHERE universe_id = $2 AND deleted = false))))`);+ } else if (refKind) {+ params.push(refKind, refId);+ conditions.push(`EXISTS (SELECT 1 FROM catalog_identity_read_refs r+ WHERE catalog_canonical_ingredient_id(r.ingredient_id) = catalog_ingredients.id AND r.deleted = false+ AND r.ref_kind = $2 AND r.ref_id = $3)`);+ } else if (unlinked) {+ conditions.push(`NOT ${LEGACY_HAS_ANY}`);+ } else if (orphaned) {+ conditions.push(LEGACY_HAS_ANY, `NOT ${LEGACY_HAS_LIVE}`);+ }+ if (scrapId) {+ params.push(scrapId);+ conditions.push(`EXISTS (SELECT 1 FROM catalog_identity_read_sources cis+ WHERE catalog_canonical_ingredient_id(cis.ingredient_id) = catalog_ingredients.id AND cis.scrap_id = $${params.length})`);+ }+ const { rows } = await query(+ `SELECT id, ${LEGACY_THUMBNAIL}, ${LEGACY_USAGE} FROM catalog_ingredients+ WHERE ${conditions.join(' AND ')} ORDER BY created_at DESC`,+ params,+ );+ return rows.map((row) => ({ id: row.id, thumbnailKey: row.thumbnail_key ?? null, usage: row.ref_usage ?? {} }));+}++async function insertIngredient(name, secondsAgo, owner = 'list-test-local') {+ await query(+ `INSERT INTO catalog_ingredients (id, type, name, payload, tags, origin_instance_id, created_at)+ VALUES ($1, 'character', $2, '{}'::jsonb, ARRAY[$3], $4, NOW() - make_interval(secs => $5))`,+ [id(name), `Example ${name}`, TAG, owner, secondsAgo],+ );+}+const insertRef = (name, refKind, refId, { role = 'cast', deleted = false } = {}) => query(+ `INSERT INTO catalog_ingredient_refs (ingredient_id, ref_kind, ref_id, role, deleted) VALUES ($1, $2, $3, $4, $5)`,+ [id(name), refKind, refId, role, deleted],+);+const insertMedia = (name, key, kind, { role = null, secondsAgo = 0 } = {}) => query(+ `INSERT INTO catalog_ingredient_media (ingredient_id, media_key, kind, role, created_at)+ VALUES ($1, $2, $3, $4, NOW() - make_interval(secs => $5))`,+ [id(name), `${P}-${key}`, kind, role, secondsAgo],+);+const refRow = (name, refKind, refId) => ({ ingredient_id: id(name), ref_kind: refKind, ref_id: refId, role: 'cast', created_at: now, updated_at: now, deleted: false });+const mediaRow = (name, key, kind) => ({ ingredient_id: id(name), media_key: `${P}-${key}`, kind, role: null, caption: null, metadata: {}, library: {}, created_at: now, updated_at: now, deleted: false });++beforeAll(async () => {+ if (!ready) return;+ await query('INSERT INTO universes (id, name) VALUES ($1, $2)', [UNI, 'Example Universe']);+ await query('INSERT INTO pipeline_series (id, name, universe_id) VALUES ($1, $2, $3)', [SER, 'Example Series', UNI]);+ await query("INSERT INTO creative_director_projects (id, data, deleted, deleted_at) VALUES ($1, '{}'::jsonb, TRUE, NOW())", [CD_DEAD]);+ await query("INSERT INTO catalog_scraps (id, title, raw_text, source_kind) VALUES ($1, 'Example scrap', 'Example text', 'paste')", [SCRAP]);++ const names = ['raw', 'linked', 'series-only', 'orphan', 'mixed', 'dead-ref-gone', 'thumbs', 'merge-source', 'merge-survivor', 'op-source', 'op-survivor'];+ for (const [i, name] of names.entries()) {+ const owner = name === 'op-survivor' ? 'list-test-peer' : 'list-test-local';+ await insertIngredient(name, (names.length - i) * 10, owner);+ }+ await insertRef('linked', 'universe', UNI);+ await insertRef('linked', 'universe', UNI, { role: 'mentioned' });+ await insertRef('linked', 'issue', `${P}-issue`);+ await insertRef('series-only', 'series', SER);+ await insertRef('orphan', 'universe', DEAD_UNI);+ await insertRef('orphan', 'creative-director', CD_DEAD);+ await insertRef('mixed', 'universe', DEAD_UNI);+ await insertRef('mixed', 'series', SER);+ await insertRef('dead-ref-gone', 'universe', UNI, { deleted: true });+ await insertMedia('thumbs', 'portrait', 'portrait', { secondsAgo: 30 });+ await insertMedia('thumbs', 'newer-ref', 'reference', { secondsAgo: 5 });+ await insertMedia('linked', 'old-ref', 'reference', { secondsAgo: 20 });+ await insertMedia('linked', 'new-ref', 'reference', { secondsAgo: 10 });+ await insertMedia('linked', 'sheet', 'reference', { role: 'sheet:front', secondsAgo: 1 });+ await query('INSERT INTO catalog_ingredient_sources (ingredient_id, scrap_id) VALUES ($1, $2), ($3, $2)', [id('raw'), SCRAP, id('merge-source')]);++ // Local merge: merge-source's own rows resolve to merge-survivor.+ await insertRef('merge-source', 'universe', DEAD_UNI);+ await insertMedia('merge-source', 'merged-portrait', 'portrait');+ await query(`INSERT INTO catalog_ingredient_merges (id, source_id, survivor_id, before_state, after_fingerprint)+ VALUES ($1, $2, $3, '{}'::jsonb, 'fixture')`, [MERGE, id('merge-source'), id('merge-survivor')]);++ // Committed shared-identity operation owned by a peer survivor: the views read+ // the source's plan rows (priority 0) and the survivor snapshot (priority 2).+ const source = { refs: [refRow('op-source', 'universe', UNI)], media: [mediaRow('op-source', 'op-ref', 'reference')], sources: [{ ingredient_id: id('op-source'), scrap_id: SCRAP, span: null, extracted_at: now }] };+ const snapshot = { refs: [refRow('op-survivor', 'series', SER), refRow('op-survivor', 'creative-director', CD_DEAD)], media: [mediaRow('op-survivor', 'op-portrait', 'portrait')], sources: [] };+ await query(`INSERT INTO catalog_identity_operations (id, source_id, survivor_id, source_owner, survivor_owner, local_owner, state, plan, plan_hash, current_snapshot)+ VALUES ($1, $2, $3, 'list-test-local', 'list-test-peer', 'list-test-local', 'committed', $4::jsonb, 'fixture', $5::jsonb)`,+ [OP, id('op-source'), id('op-survivor'), JSON.stringify({ version: 1, source, survivor: snapshot }), JSON.stringify(snapshot)]);+});++afterAll(async () => {+ if (!ready) return;+ await query('DELETE FROM catalog_identity_operations WHERE id = $1', [OP]);+ await query('DELETE FROM catalog_ingredient_merges WHERE id = $1', [MERGE]);+ await query('DELETE FROM catalog_ingredients WHERE $1 = ANY(tags)', [TAG]);+ await query('DELETE FROM catalog_scraps WHERE id = $1', [SCRAP]);+ await query('DELETE FROM creative_director_projects WHERE id = $1', [CD_DEAD]);+ await query('DELETE FROM pipeline_series WHERE id = $1', [SER]);+ await query('DELETE FROM universes WHERE id = $1', [UNI]);+ await close();+});++const FILTERS = {+ none: {},+ unlinked: { unlinked: true },+ orphaned: { orphaned: true },+ universe: { refKind: 'universe', refId: UNI },+ series: { refKind: 'series', refId: SER },+ scrap: { scrapId: SCRAP },+};++const project = ({ items }) => items.map(({ id: itemId, thumbnailKey, usage }) => ({ id: itemId, thumbnailKey, usage }));++describe.skipIf(!ready)('listIngredients identity read views (#12989)', () => {+ // Uniquely catches a rewritten filter or decoration diverging from the+ // correlated statement on any view branch (live, merge, shared operation).+ it.each(Object.entries(FILTERS))('matches the correlated statement for the %s view', async (_label, filter) => {+ const current = project(await listIngredients({ ...filter, tag: TAG, limit: 200 }));+ expect(current).toEqual(await legacyList(filter));+ expect(current.length).toBeGreaterThan(0);+ });++ it('classifies the fixture the way the albums promise', async () => {+ const ids = async (filter) => project(await listIngredients({ ...filter, tag: TAG, limit: 200 })).map((row) => row.id);+ expect(await ids(FILTERS.orphaned)).toEqual([id('orphan')]);+ expect(await ids(FILTERS.universe)).toEqual([id('op-survivor'), id('mixed'), id('series-only'), id('linked')]);+ expect(await ids(FILTERS.scrap)).toEqual([id('op-survivor'), id('merge-survivor'), id('raw')]);+ const byId = new Map(project(await listIngredients({ tag: TAG, limit: 200 })).map((row) => [row.id, row]));+ expect(byId.get(id('linked'))).toEqual({ id: id('linked'), thumbnailKey: `${P}-new-ref`, usage: { universe: 1, issue: 1 } });+ expect(byId.get(id('thumbs')).thumbnailKey).toBe(`${P}-portrait`);+ expect(byId.get(id('op-survivor'))).toEqual({ id: id('op-survivor'), thumbnailKey: `${P}-op-portrait`, usage: { universe: 1, series: 1 } });+ });++ it('pages without changing order and decorates only the returned page', async () => {+ const all = project(await listIngredients({ tag: TAG, limit: 200 }));+ const pages = [];+ for (let offset = 0; offset < all.length; offset += 3) {+ const page = await listIngredients({ tag: TAG, limit: 3, offset });+ expect(page.nextOffset).toBe(offset + page.items.length);+ pages.push(...project(page));+ }+ expect(pages).toEqual(all);+ });++ // Uniquely catches a regression to per-row view evaluation: a correlated+ // subquery rescans the view's base tables once per outer row.+ it('evaluates each read view once per statement', async () => {+ const plans = [];+ for (const filter of [FILTERS.orphaned, FILTERS.unlinked, FILTERS.universe, FILTERS.scrap]) {+ query.mockClear();+ await listIngredients({ ...filter, limit: 50 });+ for (const [sql, params] of [...query.mock.calls]) {+ const { rows } = await query(`EXPLAIN (ANALYZE, FORMAT JSON) ${sql}`, params);+ plans.push(rows[0]['QUERY PLAN'][0].Plan);+ }+ }+ const viewScans = [];+ const walk = (node) => {+ if (/^catalog_ingredient_(refs|media|sources)$/.test(node['Relation Name'] || '')) viewScans.push(node);+ for (const child of node.Plans || []) walk(child);+ };+ plans.forEach(walk);+ expect(viewScans.length).toBeGreaterThan(0);+ for (const node of viewScans) expect({ relation: node['Relation Name'], loops: node['Actual Loops'] }).toEqual({ relation: node['Relation Name'], loops: 1 });+ });+});diff --git a/server/services/catalogDB/shared.js b/server/services/catalogDB/shared.jsindex 365951bb0f..02707e0271 100644--- a/server/services/catalogDB/shared.js+++ b/server/services/catalogDB/shared.js@@ -256,16 +256,27 @@ export const liveHomingTargetSql = (r) => `(OR (${r}.ref_kind = 'series' AND EXISTS (SELECT 1 FROM pipeline_series s WHERE s.id = ${r}.ref_id AND s.deleted = false))OR (${r}.ref_kind = 'creative-director' AND EXISTS (SELECT 1 FROM creative_director_projects p WHERE p.id = ${r}.ref_id AND p.deleted = false)))`;-// SQL predicate matching ingredients that have at least one LIVE homing ref-// (target row still resolves).-export const HAS_LIVE_HOMING_REF = `EXISTS (- SELECT 1 FROM catalog_identity_read_refs r- WHERE catalog_canonical_ingredient_id(r.ingredient_id) = catalog_ingredients.id AND r.deleted = false- AND ${liveHomingTargetSql('r')})`;+// The identity read views are built from CTEs, UNION ALL and plan JSON, so a+// predicate correlated on `catalog_canonical_ingredient_id(r.ingredient_id) =+// catalog_ingredients.id` cannot use an index and re-evaluates the whole view+// once per outer row (#12989: 35 s for the Orphaned album). A statement that+// filters on refs adds this CTE so the view is evaluated once.+export const CANONICAL_REFS_CTE = `canonical_refs AS MATERIALIZED (+ SELECT catalog_canonical_ingredient_id(ingredient_id) AS cid, ref_kind, ref_id+ FROM catalog_identity_read_refs WHERE deleted = false)`;++// `<expr> IN (<uncorrelated subquery>)` evaluated as ONE hashed subplan. A bare+// top-level IN is pulled up into a semi-join, and the read views' tiny row+// estimates make the planner pick a nested loop that rescans the subquery once+// per outer row — quadratic again (#12989). Wrapping it in IS TRUE keeps it+// a hashed subplan with the old EXISTS semantics: a NULL from IN (a NULL in the+// set) reads as false, so NOT (inSet(…)) also matches the old NOT EXISTS.+export const inSet = (expr, subquery) => `(${expr} IN (${subquery})) IS TRUE`;++// Predicates over CANONICAL_REFS_CTE (the statement must include it).+// HAS_LIVE_HOMING_REF: at least one homing ref whose target still resolves.+export const HAS_LIVE_HOMING_REF = inSet('id', `SELECT r.cid FROM canonical_refs r WHERE ${liveHomingTargetSql('r')}`);// At least one homing ref row exists (live OR dangling) — distinguishes// "orphaned" (has a ref, none resolve) from "unlinked" (no ref at all).-export const HAS_ANY_HOMING_REF = `EXISTS (- SELECT 1 FROM catalog_identity_read_refs r- WHERE catalog_canonical_ingredient_id(r.ingredient_id) = catalog_ingredients.id AND r.deleted = false- AND r.ref_kind IN (${HOMING_REF_KINDS_SQL}))`;+export const HAS_ANY_HOMING_REF = inSet('id', `SELECT r.cid FROM canonical_refs r WHERE r.ref_kind IN (${HOMING_REF_KINDS_SQL})`);diff --git a/server/services/catalogDBMedia.test.js b/server/services/catalogDBMedia.test.jsindex 8765bb3f32..178bfe8d13 100644--- a/server/services/catalogDBMedia.test.js+++ b/server/services/catalogDBMedia.test.js@@ -206,41 +206,33 @@ describe('listIngredients card thumbnail', () => {...overrides,});- it('selects the portrait-or-latest-reference media key as thumbnail_key', async () => {+ // The DB suites pin the decoration SQL; this pins the id join that attaches+ // the one page-level decoration query back onto the page rows (#12989).+ it('decorates the returned page by id, defaulting to a null thumbnail and empty usage', async () => {const db = await import('../lib/db.js');- db.query.mockResolvedValueOnce({ rows: [ingredientRow({ thumbnail_key: 'hero.png' })] });+ db.query+ .mockResolvedValueOnce({ rows: [ingredientRow({ id: 'cat-chr-a' }), ingredientRow({ id: 'cat-chr-b' })] })+ .mockResolvedValueOnce({ rows: [+ { id: 'cat-chr-b', thumbnail_key: null, ref_usage: null },+ { id: 'cat-chr-a', thumbnail_key: 'hero.png', ref_usage: { universe: 1, 'creative-director': 2 } },+ ] });const { items } = await catalogDB.listIngredients({ limit: 10 });- // mockResolvedValueOnce bypasses the factory's `calls` push, so read the- // captured SQL from the mock's own call log (as the getMaxSequences test does).- const sql = db.query.mock.calls.at(-1)[0].replace(/\s+/g, ' ');- // The correlated subquery prefers a live portrait, then the most recent- // live reference, and excludes unrenderable (audio/video/document) kinds.- expect(sql).toMatch(/FROM catalog_identity_read_media m/i);- expect(sql).toMatch(/m\.kind IN \('portrait', 'reference'\)/i);- expect(sql).toMatch(/ORDER BY \(m\.kind = 'portrait'\) DESC, m\.created_at DESC/i);- expect(sql).toMatch(/AS thumbnail_key/i);- expect(items[0].thumbnailKey).toBe('hero.png');+ expect(db.query.mock.calls.at(-1)[1]).toEqual([['cat-chr-a', 'cat-chr-b']]);+ expect(items.map(({ id, thumbnailKey, usage }) => ({ id, thumbnailKey, usage }))).toEqual([+ { id: 'cat-chr-a', thumbnailKey: 'hero.png', usage: { universe: 1, 'creative-director': 2 } },+ { id: 'cat-chr-b', thumbnailKey: null, usage: {} },+ ]);});- it('maps a null thumbnail_key (no image attached) to null', async () => {+ it('skips the decoration query for an empty page and the embedding admin path', async () => {const db = await import('../lib/db.js');- db.query.mockResolvedValueOnce({ rows: [ingredientRow({ thumbnail_key: null })] });- const { items } = await catalogDB.listIngredients({ limit: 10 });- expect(items[0].thumbnailKey).toBeNull();- });-- it('returns per-kind link counts as usage, defaulting to an empty object', async () => {- const db = await import('../lib/db.js');- db.query.mockResolvedValueOnce({ rows: [- ingredientRow({ id: 'cat-chr-a', ref_usage: { universe: 1, 'creative-director': 2 } }),- ingredientRow({ id: 'cat-chr-b', ref_usage: null }),- ] });- const { items } = await catalogDB.listIngredients({ limit: 10 });- const sql = db.query.mock.calls.at(-1)[0].replace(/\s+/g, ' ');- expect(sql).toMatch(/COUNT\(DISTINCT r\.ref_id\)/i);- expect(sql).toMatch(/r\.deleted = false/i);- expect(items[0].usage).toEqual({ universe: 1, 'creative-director': 2 });- expect(items[1].usage).toEqual({});+ db.query.mockResolvedValueOnce({ rows: [] });+ const before = db.query.mock.calls.length;+ expect((await catalogDB.listIngredients({ limit: 10 })).items).toEqual([]);+ db.query.mockResolvedValueOnce({ rows: [ingredientRow()] });+ const { items } = await catalogDB.listIngredients({ limit: 10, includeEmbedding: true });+ expect(db.query.mock.calls.length - before).toBe(2);+ expect(items[0]).toMatchObject({ thumbnailKey: null, usage: {} });});});