Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

perf: catalog ingredient list reads each identity view once per statement (#12989)

[._.]/ Adam Eivy9f10d2c44b77
diff --git a/scripts/lib/dbTestFiles.js b/scripts/lib/dbTestFiles.js
index 318cf683c3..2e9e87f3fd 100644
--- a/scripts/lib/dbTestFiles.js
+++ b/scripts/lib/dbTestFiles.js
@@ -31,6 +31,7 @@ export const DB_TEST_INCLUDE = [
'services/backup.db.test.js',
'services/catalogDB.test.js',
'services/catalogDB.facets.db.test.js',
+ 'services/catalogDB/listIngredients.db.test.js',
'services/catalogDB.media.db.test.js',
'services/catalogSync.tombstoneRevival.db.test.js',
'services/catalogSync.changeClock.db.test.js',
diff --git a/server/services/catalogDB/ingredients.js b/server/services/catalogDB/ingredients.js
index f4ff9dc1fb..34a27a526a 100644
--- a/server/services/catalogDB/ingredients.js
+++ b/server/services/catalogDB/ingredients.js
@@ -23,6 +23,8 @@ import {
REVISION_SOURCES,
rowToIngredient,
rowToRevision,
+ CANONICAL_REFS_CTE,
+ inSet,
HAS_LIVE_HOMING_REF,
HAS_ANY_HOMING_REF,
HOMING_REF_KINDS_SQL,
@@ -360,33 +362,47 @@ export async function reviveDeletedIngredient(id, { type, name, payload = {}, ta
// directly instead of fetching-then-JS-filtering.
const INGREDIENT_LIGHT_COLS = 'id, type, name, payload, tags, embedding_model, origin_instance_id, created_at, updated_at, deleted, deleted_at, sync_sequence';
-// Correlated subquery that yields the card-thumbnail media key for each row:
-// the live portrait when set, otherwise the most recent live reference image.
-// Non-image media kinds (audio/video/document) are excluded so a card never
-// points an <img> at an unrenderable key. Working images filed with a role —
-// reference sheets (`sheet:*`), looks (`look:*`) and plates — never become the
-// cover. Used only by listIngredients' light path (see the note there).
-const THUMBNAIL_KEY_SUBQUERY = `(
- SELECT m.media_key FROM catalog_identity_read_media m
- WHERE catalog_canonical_ingredient_id(m.ingredient_id) = catalog_ingredients.id
- AND m.deleted = false
- AND m.kind IN ('portrait', 'reference')
- AND (m.role IS NULL OR NOT (m.role LIKE 'sheet:%' OR m.role LIKE 'look:%' OR m.role LIKE 'archive:%' OR m.role = 'plate'))
- ORDER BY (m.kind = 'portrait') DESC, m.created_at DESC
- LIMIT 1
- ) AS thumbnail_key`;
-
-// Per-kind count of the distinct records each row is linked to ({ universe: 1,
-// series: 2 }), for the "used in" line on library cards. Same light-path scope
-// as the thumbnail subquery.
-const REF_USAGE_SUBQUERY = `(
- SELECT jsonb_object_agg(u.ref_kind, u.n) FROM (
- SELECT r.ref_kind, COUNT(DISTINCT r.ref_id) AS n FROM catalog_identity_read_refs r
- WHERE catalog_canonical_ingredient_id(r.ingredient_id) = catalog_ingredients.id AND r.deleted = false
- AND (r.ref_kind NOT IN (${HOMING_REF_KINDS_SQL}) OR ${liveHomingTargetSql('r')})
- GROUP BY r.ref_kind
- ) u
- ) AS ref_usage`;
+// Card decorations for one already-limited page, in one grouped pass over each
+// identity read view (#12989 — a per-row correlated subquery re-evaluated the
+// whole view for every row). Light list path only (see listIngredients).
+// - thumbnail_key: the live portrait when set, otherwise the most recent live
+// reference image. Mirrors listMediaForIngredient's `(kind='portrait') DESC,
+// created_at DESC` ordering so the card shows the detail page's avatar.
+// Non-image kinds are excluded so a card never points an <img> at an
+// unrenderable key, and role-filed working images — reference sheets
+// (`sheet:*`), looks (`look:*`), archives and plates — never become the cover.
+// - ref_usage: per-kind count of the distinct records the row is linked to
+// ({ universe: 1, series: 2 }) for the "used in" line; a homing kind counts
+// only live targets.
+async function decorateIngredientPage(ids) {
+ if (ids.length === 0) return new Map();
+ const result = await query(
+ `WITH page AS (SELECT DISTINCT unnest($1::text[]) AS id),
+ usage AS (
+ SELECT cid, jsonb_object_agg(ref_kind, n) AS ref_usage FROM (
+ SELECT r.cid, r.ref_kind, COUNT(DISTINCT r.ref_id) AS n
+ FROM (SELECT catalog_canonical_ingredient_id(ingredient_id) AS cid, ref_kind, ref_id
+ FROM catalog_identity_read_refs WHERE deleted = false) r
+ WHERE ${inSet('r.cid', 'SELECT id FROM page')}
+ AND (r.ref_kind NOT IN (${HOMING_REF_KINDS_SQL}) OR ${liveHomingTargetSql('r')})
+ GROUP BY r.cid, r.ref_kind
+ ) u GROUP BY cid
+ ),
+ thumbs AS (
+ SELECT DISTINCT ON (m.cid) m.cid, m.media_key
+ FROM (SELECT catalog_canonical_ingredient_id(ingredient_id) AS cid, media_key, kind, created_at
+ FROM catalog_identity_read_media
+ WHERE deleted = false AND kind IN ('portrait', 'reference')
+ AND (role IS NULL OR NOT (role LIKE 'sheet:%' OR role LIKE 'look:%' OR role LIKE 'archive:%' OR role = 'plate'))) m
+ WHERE ${inSet('m.cid', 'SELECT id FROM page')}
+ ORDER BY m.cid, (m.kind = 'portrait') DESC, m.created_at DESC
+ )
+ SELECT page.id, thumbs.media_key AS thumbnail_key, usage.ref_usage
+ FROM page LEFT JOIN thumbs ON thumbs.cid = page.id LEFT JOIN usage ON usage.cid = page.id`,
+ [ids],
+ );
+ return new Map(result.rows.map((row) => [row.id, row]));
+}
export async function listIngredients({ ids, type, tag, query: q, refKind = null, refId = null, unlinked = false, orphaned = false, scrapId = null, limit = 50, offset = 0, includeEmbedding = false, embeddingMissing = false, staleEmbeddingModel = null } = {}) {
const conditions = ['deleted = false', 'id = catalog_canonical_ingredient_id(id)'];
@@ -397,6 +413,8 @@ export async function listIngredients({ ids, type, tag, query: q, refKind = null
// caller gets exactly the requested (still non-deleted) rows.
const byIds = Array.isArray(ids) && ids.length > 0;
let qIdx = null;
+ // Set by any filter that reads canonical_refs (CANONICAL_REFS_CTE).
+ let needsRefs = false;
if (byIds) {
conditions.push(`id IN (SELECT catalog_canonical_ingredient_id(requested) FROM unnest($${idx++}::text[]) requested)`);
params.push(ids);
@@ -414,10 +432,12 @@ export async function listIngredients({ ids, type, tag, query: q, refKind = null
conditions.push(`search_tsv @@ websearch_to_tsquery('english', $${qIdx})`);
params.push(q);
}
- // Album/facet filters (#1762). EXISTS subquery (not a JOIN) so a row linked
- // under multiple roles isn't duplicated — no DISTINCT needed. Composes with
- // type/tag/q above. `unlinked` and `orphaned` are mutually exclusive views
- // of the un-homed set; the route schema rejects combining them with ref.
+ // Album/facet filters (#1762). A set-membership test (not a JOIN) so a row
+ // linked under multiple roles isn't duplicated — no DISTINCT needed — over
+ // the statement-level canonical_refs CTE, so the read view is evaluated once
+ // per statement, not once per row (#12989). Composes with type/tag/q above.
+ // `unlinked` and `orphaned` are mutually exclusive views of the un-homed
+ // set; the route schema rejects combining them with ref.
if (refKind === 'universe' && refId) {
// A universe album/filter is the universe's WHOLE membership: ingredients
// linked directly to the universe OR to any live series under it (decision
@@ -425,33 +445,35 @@ export async function listIngredients({ ids, type, tag, query: q, refKind = null
// ingredient must still land in its parent universe). $refId is referenced
// twice via one placeholder.
const refIdx = idx++;
- conditions.push(`EXISTS (SELECT 1 FROM catalog_identity_read_refs r
- WHERE catalog_canonical_ingredient_id(r.ingredient_id) = catalog_ingredients.id AND r.deleted = false
- AND ((r.ref_kind = 'universe' AND r.ref_id = $${refIdx})
+ needsRefs = true;
+ conditions.push(inSet('id', `SELECT r.cid FROM canonical_refs r
+ WHERE (r.ref_kind = 'universe' AND r.ref_id = $${refIdx})
OR (r.ref_kind = 'series' AND r.ref_id IN (
- SELECT id FROM pipeline_series WHERE universe_id = $${refIdx} AND deleted = false))))`);
+ SELECT id FROM pipeline_series WHERE universe_id = $${refIdx} AND deleted = false))`));
params.push(refId);
} else if (refKind && refId) {
- conditions.push(`EXISTS (SELECT 1 FROM catalog_identity_read_refs r
- WHERE catalog_canonical_ingredient_id(r.ingredient_id) = catalog_ingredients.id AND r.deleted = false
- AND r.ref_kind = $${idx++} AND r.ref_id = $${idx++})`);
+ needsRefs = true;
+ conditions.push(inSet('id', `SELECT r.cid FROM canonical_refs r WHERE r.ref_kind = $${idx++} AND r.ref_id = $${idx++}`));
params.push(refKind, refId);
} else if (unlinked) {
// "Unsorted / Raw": no universe/series ref at all.
- conditions.push(`NOT ${HAS_ANY_HOMING_REF}`);
+ needsRefs = true;
+ conditions.push(`NOT (${HAS_ANY_HOMING_REF})`);
} else if (orphaned) {
// "Orphaned": has a universe/series ref, but none resolve to a live target.
+ needsRefs = true;
conditions.push(HAS_ANY_HOMING_REF);
- conditions.push(`NOT ${HAS_LIVE_HOMING_REF}`);
+ conditions.push(`NOT (${HAS_LIVE_HOMING_REF})`);
}
}
// Source-scrap filter (#7617): "everything extracted from this piece" — a
// provenance dimension independent of the ids/type/tag/q/ref branches above,
// so it composes with any of them (unlike unlinked/orphaned/refKind, which
// are mutually exclusive album views enforced at the schema level).
+ // One hashed pass over the sources read view (#12989).
if (scrapId) {
- conditions.push(`EXISTS (SELECT 1 FROM catalog_identity_read_sources cis
- WHERE catalog_canonical_ingredient_id(cis.ingredient_id) = catalog_ingredients.id AND cis.scrap_id = $${idx++})`);
+ conditions.push(inSet('id', `SELECT catalog_canonical_ingredient_id(cis.ingredient_id)
+ FROM catalog_identity_read_sources cis WHERE cis.scrap_id = $${idx++}`));
params.push(scrapId);
}
if (embeddingMissing) {
@@ -471,21 +493,23 @@ export async function listIngredients({ ids, type, tag, query: q, refKind = null
? `ORDER BY ts_rank_cd(search_tsv, websearch_to_tsquery('english', $${qIdx})) DESC, created_at DESC`
: `ORDER BY created_at DESC`;
params.push(limit, offset);
- // Card-thumbnail key: the live portrait if one exists, else the most recent
- // reference image. Mirrors listMediaForIngredient's `(kind='portrait') DESC,
- // created_at DESC` ordering so the card shows the same image the detail page
- // treats as the avatar. Scoped to the light-column list path only — the
- // includeEmbedding admin paths (re-embed/backfill) don't render cards and
- // don't pay for the correlated subquery. The field rides ALONGSIDE
- // rowToIngredient's shape (attached below), so the sync/revision payloads
- // that share rowToIngredient stay untouched.
- const cols = includeEmbedding ? '*' : `${INGREDIENT_LIGHT_COLS}, ${THUMBNAIL_KEY_SUBQUERY}, ${REF_USAGE_SUBQUERY}`;
+ // Card decorations (thumbnailKey, usage) are computed for the returned page
+ // only, and only on the light-column path — the includeEmbedding admin paths
+ // (re-embed/backfill) don't render cards. They ride ALONGSIDE
+ // rowToIngredient's shape, so the sync/revision payloads that share
+ // rowToIngredient stay untouched.
+ const cols = includeEmbedding ? '*' : INGREDIENT_LIGHT_COLS;
+ const withClause = needsRefs ? `WITH ${CANONICAL_REFS_CTE} ` : '';
const result = await query(
- `SELECT ${cols} FROM catalog_ingredients ${where} ${orderBy} LIMIT $${idx++} OFFSET $${idx}`,
+ `${withClause}SELECT ${cols} FROM catalog_ingredients ${where} ${orderBy} LIMIT $${idx++} OFFSET $${idx}`,
params,
);
+ const decorations = includeEmbedding ? new Map() : await decorateIngredientPage(result.rows.map((row) => row.id));
return {
- items: result.rows.map((row) => ({ ...rowToIngredient(row), thumbnailKey: row.thumbnail_key ?? null, usage: row.ref_usage ?? {} })),
+ items: result.rows.map((row) => {
+ const decoration = decorations.get(row.id);
+ return { ...rowToIngredient(row), thumbnailKey: decoration?.thumbnail_key ?? null, usage: decoration?.ref_usage ?? {} };
+ }),
nextOffset: offset + result.rows.length,
};
}
diff --git a/server/services/catalogDB/listIngredients.db.test.js b/server/services/catalogDB/listIngredients.db.test.js
new file mode 100644
index 0000000000..c56383cc05
--- /dev/null
+++ b/server/services/catalogDB/listIngredients.db.test.js
@@ -0,0 +1,230 @@
+/**
+ * listIngredients reads the identity read views (catalog_identity_read_refs /
+ * _media / _sources) once per statement, not once per ingredient row (#12989).
+ *
+ * The equivalence suite pins the rewritten album filters and card decorations
+ * to the correlated-subquery statement they replaced (LEGACY_* below, frozen
+ * from the pre-#12989 implementation) on a fixture that exercises every view
+ * branch: live rows, a local merge (catalog_ingredient_merges) and a committed
+ * shared-identity operation whose plan and snapshot rows feed the views.
+ *
+ * `*.db.test.js` → runs ONLY via `npm run test:db` against `portos_test`. The
+ * DB is shared, so every comparison is scoped to this run's nonce tag.
+ */
+
+import { randomUUID } from 'node:crypto';
+import { afterAll, beforeAll, describe, expect, it, vi } from 'vitest';
+
+vi.mock('../../lib/db.js', async (importOriginal) => {
+ const actual = await importOriginal();
+ return { ...actual, query: vi.fn(actual.query) };
+});
+vi.mock('../instanceIdentity.js', () => ({ getInstanceId: async () => 'list-test-local', UNKNOWN_INSTANCE_ID: 'unknown' }));
+
+const { checkHealth, close, ensureSchema, query } = await import('../../lib/db.js');
+const { requireDbOrSkip } = await import('../../lib/dbTestGate.js');
+const { listIngredients } = await import('./ingredients.js');
+const { HOMING_REF_KINDS_SQL, liveHomingTargetSql } = await import('./shared.js');
+
+const health = await checkHealth().catch((error) => ({ connected: false, error: error.message }));
+const ready = requireDbOrSkip('services/catalogDB/listIngredients.db.test', health.connected, health.error);
+if (ready) await ensureSchema();
+
+const P = `list-${randomUUID()}`;
+const TAG = `${P}-tag`;
+const UNI = `${P}-uni`;
+const SER = `${P}-ser`;
+const DEAD_UNI = `${P}-dead-uni`;
+const CD_DEAD = `${P}-cd`;
+const SCRAP = `${P}-scrap`;
+const OP = `${P}-op`;
+const MERGE = `${P}-merge`;
+const id = (name) => `${P}-${name}`;
+const now = '2026-01-02T00:00:00.000Z';
+
+// Frozen pre-#12989 statement: correlated subqueries over the read views.
+const LEGACY_HAS_LIVE = `EXISTS (SELECT 1 FROM catalog_identity_read_refs r
+ WHERE catalog_canonical_ingredient_id(r.ingredient_id) = catalog_ingredients.id AND r.deleted = false AND ${liveHomingTargetSql('r')})`;
+const LEGACY_HAS_ANY = `EXISTS (SELECT 1 FROM catalog_identity_read_refs r
+ WHERE catalog_canonical_ingredient_id(r.ingredient_id) = catalog_ingredients.id AND r.deleted = false AND r.ref_kind IN (${HOMING_REF_KINDS_SQL}))`;
+const LEGACY_THUMBNAIL = `(SELECT m.media_key FROM catalog_identity_read_media m
+ WHERE catalog_canonical_ingredient_id(m.ingredient_id) = catalog_ingredients.id AND m.deleted = false
+ AND m.kind IN ('portrait', 'reference')
+ AND (m.role IS NULL OR NOT (m.role LIKE 'sheet:%' OR m.role LIKE 'look:%' OR m.role LIKE 'archive:%' OR m.role = 'plate'))
+ ORDER BY (m.kind = 'portrait') DESC, m.created_at DESC LIMIT 1) AS thumbnail_key`;
+const LEGACY_USAGE = `(SELECT jsonb_object_agg(u.ref_kind, u.n) FROM (
+ SELECT r.ref_kind, COUNT(DISTINCT r.ref_id) AS n FROM catalog_identity_read_refs r
+ WHERE catalog_canonical_ingredient_id(r.ingredient_id) = catalog_ingredients.id AND r.deleted = false
+ AND (r.ref_kind NOT IN (${HOMING_REF_KINDS_SQL}) OR ${liveHomingTargetSql('r')})
+ GROUP BY r.ref_kind) u) AS ref_usage`;
+
+async function legacyList({ refKind, refId, unlinked, orphaned, scrapId }) {
+ const conditions = ['deleted = false', 'id = catalog_canonical_ingredient_id(id)', '$1 = ANY(tags)'];
+ const params = [TAG];
+ if (refKind === 'universe') {
+ params.push(refId);
+ conditions.push(`EXISTS (SELECT 1 FROM catalog_identity_read_refs r
+ WHERE catalog_canonical_ingredient_id(r.ingredient_id) = catalog_ingredients.id AND r.deleted = false
+ AND ((r.ref_kind = 'universe' AND r.ref_id = $2)
+ OR (r.ref_kind = 'series' AND r.ref_id IN (SELECT id FROM pipeline_series WHERE universe_id = $2 AND deleted = false))))`);
+ } else if (refKind) {
+ params.push(refKind, refId);
+ conditions.push(`EXISTS (SELECT 1 FROM catalog_identity_read_refs r
+ WHERE catalog_canonical_ingredient_id(r.ingredient_id) = catalog_ingredients.id AND r.deleted = false
+ AND r.ref_kind = $2 AND r.ref_id = $3)`);
+ } else if (unlinked) {
+ conditions.push(`NOT ${LEGACY_HAS_ANY}`);
+ } else if (orphaned) {
+ conditions.push(LEGACY_HAS_ANY, `NOT ${LEGACY_HAS_LIVE}`);
+ }
+ if (scrapId) {
+ params.push(scrapId);
+ conditions.push(`EXISTS (SELECT 1 FROM catalog_identity_read_sources cis
+ WHERE catalog_canonical_ingredient_id(cis.ingredient_id) = catalog_ingredients.id AND cis.scrap_id = $${params.length})`);
+ }
+ const { rows } = await query(
+ `SELECT id, ${LEGACY_THUMBNAIL}, ${LEGACY_USAGE} FROM catalog_ingredients
+ WHERE ${conditions.join(' AND ')} ORDER BY created_at DESC`,
+ params,
+ );
+ return rows.map((row) => ({ id: row.id, thumbnailKey: row.thumbnail_key ?? null, usage: row.ref_usage ?? {} }));
+}
+
+async function insertIngredient(name, secondsAgo, owner = 'list-test-local') {
+ await query(
+ `INSERT INTO catalog_ingredients (id, type, name, payload, tags, origin_instance_id, created_at)
+ VALUES ($1, 'character', $2, '{}'::jsonb, ARRAY[$3], $4, NOW() - make_interval(secs => $5))`,
+ [id(name), `Example ${name}`, TAG, owner, secondsAgo],
+ );
+}
+const insertRef = (name, refKind, refId, { role = 'cast', deleted = false } = {}) => query(
+ `INSERT INTO catalog_ingredient_refs (ingredient_id, ref_kind, ref_id, role, deleted) VALUES ($1, $2, $3, $4, $5)`,
+ [id(name), refKind, refId, role, deleted],
+);
+const insertMedia = (name, key, kind, { role = null, secondsAgo = 0 } = {}) => query(
+ `INSERT INTO catalog_ingredient_media (ingredient_id, media_key, kind, role, created_at)
+ VALUES ($1, $2, $3, $4, NOW() - make_interval(secs => $5))`,
+ [id(name), `${P}-${key}`, kind, role, secondsAgo],
+);
+const refRow = (name, refKind, refId) => ({ ingredient_id: id(name), ref_kind: refKind, ref_id: refId, role: 'cast', created_at: now, updated_at: now, deleted: false });
+const mediaRow = (name, key, kind) => ({ ingredient_id: id(name), media_key: `${P}-${key}`, kind, role: null, caption: null, metadata: {}, library: {}, created_at: now, updated_at: now, deleted: false });
+
+beforeAll(async () => {
+ if (!ready) return;
+ await query('INSERT INTO universes (id, name) VALUES ($1, $2)', [UNI, 'Example Universe']);
+ await query('INSERT INTO pipeline_series (id, name, universe_id) VALUES ($1, $2, $3)', [SER, 'Example Series', UNI]);
+ await query("INSERT INTO creative_director_projects (id, data, deleted, deleted_at) VALUES ($1, '{}'::jsonb, TRUE, NOW())", [CD_DEAD]);
+ await query("INSERT INTO catalog_scraps (id, title, raw_text, source_kind) VALUES ($1, 'Example scrap', 'Example text', 'paste')", [SCRAP]);
+
+ const names = ['raw', 'linked', 'series-only', 'orphan', 'mixed', 'dead-ref-gone', 'thumbs', 'merge-source', 'merge-survivor', 'op-source', 'op-survivor'];
+ for (const [i, name] of names.entries()) {
+ const owner = name === 'op-survivor' ? 'list-test-peer' : 'list-test-local';
+ await insertIngredient(name, (names.length - i) * 10, owner);
+ }
+ await insertRef('linked', 'universe', UNI);
+ await insertRef('linked', 'universe', UNI, { role: 'mentioned' });
+ await insertRef('linked', 'issue', `${P}-issue`);
+ await insertRef('series-only', 'series', SER);
+ await insertRef('orphan', 'universe', DEAD_UNI);
+ await insertRef('orphan', 'creative-director', CD_DEAD);
+ await insertRef('mixed', 'universe', DEAD_UNI);
+ await insertRef('mixed', 'series', SER);
+ await insertRef('dead-ref-gone', 'universe', UNI, { deleted: true });
+ await insertMedia('thumbs', 'portrait', 'portrait', { secondsAgo: 30 });
+ await insertMedia('thumbs', 'newer-ref', 'reference', { secondsAgo: 5 });
+ await insertMedia('linked', 'old-ref', 'reference', { secondsAgo: 20 });
+ await insertMedia('linked', 'new-ref', 'reference', { secondsAgo: 10 });
+ await insertMedia('linked', 'sheet', 'reference', { role: 'sheet:front', secondsAgo: 1 });
+ await query('INSERT INTO catalog_ingredient_sources (ingredient_id, scrap_id) VALUES ($1, $2), ($3, $2)', [id('raw'), SCRAP, id('merge-source')]);
+
+ // Local merge: merge-source's own rows resolve to merge-survivor.
+ await insertRef('merge-source', 'universe', DEAD_UNI);
+ await insertMedia('merge-source', 'merged-portrait', 'portrait');
+ await query(`INSERT INTO catalog_ingredient_merges (id, source_id, survivor_id, before_state, after_fingerprint)
+ VALUES ($1, $2, $3, '{}'::jsonb, 'fixture')`, [MERGE, id('merge-source'), id('merge-survivor')]);
+
+ // Committed shared-identity operation owned by a peer survivor: the views read
+ // the source's plan rows (priority 0) and the survivor snapshot (priority 2).
+ const source = { refs: [refRow('op-source', 'universe', UNI)], media: [mediaRow('op-source', 'op-ref', 'reference')], sources: [{ ingredient_id: id('op-source'), scrap_id: SCRAP, span: null, extracted_at: now }] };
+ const snapshot = { refs: [refRow('op-survivor', 'series', SER), refRow('op-survivor', 'creative-director', CD_DEAD)], media: [mediaRow('op-survivor', 'op-portrait', 'portrait')], sources: [] };
+ await query(`INSERT INTO catalog_identity_operations (id, source_id, survivor_id, source_owner, survivor_owner, local_owner, state, plan, plan_hash, current_snapshot)
+ VALUES ($1, $2, $3, 'list-test-local', 'list-test-peer', 'list-test-local', 'committed', $4::jsonb, 'fixture', $5::jsonb)`,
+ [OP, id('op-source'), id('op-survivor'), JSON.stringify({ version: 1, source, survivor: snapshot }), JSON.stringify(snapshot)]);
+});
+
+afterAll(async () => {
+ if (!ready) return;
+ await query('DELETE FROM catalog_identity_operations WHERE id = $1', [OP]);
+ await query('DELETE FROM catalog_ingredient_merges WHERE id = $1', [MERGE]);
+ await query('DELETE FROM catalog_ingredients WHERE $1 = ANY(tags)', [TAG]);
+ await query('DELETE FROM catalog_scraps WHERE id = $1', [SCRAP]);
+ await query('DELETE FROM creative_director_projects WHERE id = $1', [CD_DEAD]);
+ await query('DELETE FROM pipeline_series WHERE id = $1', [SER]);
+ await query('DELETE FROM universes WHERE id = $1', [UNI]);
+ await close();
+});
+
+const FILTERS = {
+ none: {},
+ unlinked: { unlinked: true },
+ orphaned: { orphaned: true },
+ universe: { refKind: 'universe', refId: UNI },
+ series: { refKind: 'series', refId: SER },
+ scrap: { scrapId: SCRAP },
+};
+
+const project = ({ items }) => items.map(({ id: itemId, thumbnailKey, usage }) => ({ id: itemId, thumbnailKey, usage }));
+
+describe.skipIf(!ready)('listIngredients identity read views (#12989)', () => {
+ // Uniquely catches a rewritten filter or decoration diverging from the
+ // correlated statement on any view branch (live, merge, shared operation).
+ it.each(Object.entries(FILTERS))('matches the correlated statement for the %s view', async (_label, filter) => {
+ const current = project(await listIngredients({ ...filter, tag: TAG, limit: 200 }));
+ expect(current).toEqual(await legacyList(filter));
+ expect(current.length).toBeGreaterThan(0);
+ });
+
+ it('classifies the fixture the way the albums promise', async () => {
+ const ids = async (filter) => project(await listIngredients({ ...filter, tag: TAG, limit: 200 })).map((row) => row.id);
+ expect(await ids(FILTERS.orphaned)).toEqual([id('orphan')]);
+ expect(await ids(FILTERS.universe)).toEqual([id('op-survivor'), id('mixed'), id('series-only'), id('linked')]);
+ expect(await ids(FILTERS.scrap)).toEqual([id('op-survivor'), id('merge-survivor'), id('raw')]);
+ const byId = new Map(project(await listIngredients({ tag: TAG, limit: 200 })).map((row) => [row.id, row]));
+ expect(byId.get(id('linked'))).toEqual({ id: id('linked'), thumbnailKey: `${P}-new-ref`, usage: { universe: 1, issue: 1 } });
+ expect(byId.get(id('thumbs')).thumbnailKey).toBe(`${P}-portrait`);
+ expect(byId.get(id('op-survivor'))).toEqual({ id: id('op-survivor'), thumbnailKey: `${P}-op-portrait`, usage: { universe: 1, series: 1 } });
+ });
+
+ it('pages without changing order and decorates only the returned page', async () => {
+ const all = project(await listIngredients({ tag: TAG, limit: 200 }));
+ const pages = [];
+ for (let offset = 0; offset < all.length; offset += 3) {
+ const page = await listIngredients({ tag: TAG, limit: 3, offset });
+ expect(page.nextOffset).toBe(offset + page.items.length);
+ pages.push(...project(page));
+ }
+ expect(pages).toEqual(all);
+ });
+
+ // Uniquely catches a regression to per-row view evaluation: a correlated
+ // subquery rescans the view's base tables once per outer row.
+ it('evaluates each read view once per statement', async () => {
+ const plans = [];
+ for (const filter of [FILTERS.orphaned, FILTERS.unlinked, FILTERS.universe, FILTERS.scrap]) {
+ query.mockClear();
+ await listIngredients({ ...filter, limit: 50 });
+ for (const [sql, params] of [...query.mock.calls]) {
+ const { rows } = await query(`EXPLAIN (ANALYZE, FORMAT JSON) ${sql}`, params);
+ plans.push(rows[0]['QUERY PLAN'][0].Plan);
+ }
+ }
+ const viewScans = [];
+ const walk = (node) => {
+ if (/^catalog_ingredient_(refs|media|sources)$/.test(node['Relation Name'] || '')) viewScans.push(node);
+ for (const child of node.Plans || []) walk(child);
+ };
+ plans.forEach(walk);
+ expect(viewScans.length).toBeGreaterThan(0);
+ for (const node of viewScans) expect({ relation: node['Relation Name'], loops: node['Actual Loops'] }).toEqual({ relation: node['Relation Name'], loops: 1 });
+ });
+});
diff --git a/server/services/catalogDB/shared.js b/server/services/catalogDB/shared.js
index 365951bb0f..02707e0271 100644
--- a/server/services/catalogDB/shared.js
+++ b/server/services/catalogDB/shared.js
@@ -256,16 +256,27 @@ export const liveHomingTargetSql = (r) => `(
OR (${r}.ref_kind = 'series' AND EXISTS (SELECT 1 FROM pipeline_series s WHERE s.id = ${r}.ref_id AND s.deleted = false))
OR (${r}.ref_kind = 'creative-director' AND EXISTS (SELECT 1 FROM creative_director_projects p WHERE p.id = ${r}.ref_id AND p.deleted = false)))`;
-// SQL predicate matching ingredients that have at least one LIVE homing ref
-// (target row still resolves).
-export const HAS_LIVE_HOMING_REF = `EXISTS (
- SELECT 1 FROM catalog_identity_read_refs r
- WHERE catalog_canonical_ingredient_id(r.ingredient_id) = catalog_ingredients.id AND r.deleted = false
- AND ${liveHomingTargetSql('r')})`;
+// The identity read views are built from CTEs, UNION ALL and plan JSON, so a
+// predicate correlated on `catalog_canonical_ingredient_id(r.ingredient_id) =
+// catalog_ingredients.id` cannot use an index and re-evaluates the whole view
+// once per outer row (#12989: 35 s for the Orphaned album). A statement that
+// filters on refs adds this CTE so the view is evaluated once.
+export const CANONICAL_REFS_CTE = `canonical_refs AS MATERIALIZED (
+ SELECT catalog_canonical_ingredient_id(ingredient_id) AS cid, ref_kind, ref_id
+ FROM catalog_identity_read_refs WHERE deleted = false)`;
+
+// `<expr> IN (<uncorrelated subquery>)` evaluated as ONE hashed subplan. A bare
+// top-level IN is pulled up into a semi-join, and the read views' tiny row
+// estimates make the planner pick a nested loop that rescans the subquery once
+// per outer row — quadratic again (#12989). Wrapping it in IS TRUE keeps it
+// a hashed subplan with the old EXISTS semantics: a NULL from IN (a NULL in the
+// set) reads as false, so NOT (inSet(…)) also matches the old NOT EXISTS.
+export const inSet = (expr, subquery) => `(${expr} IN (${subquery})) IS TRUE`;
+
+// Predicates over CANONICAL_REFS_CTE (the statement must include it).
+// HAS_LIVE_HOMING_REF: at least one homing ref whose target still resolves.
+export const HAS_LIVE_HOMING_REF = inSet('id', `SELECT r.cid FROM canonical_refs r WHERE ${liveHomingTargetSql('r')}`);
// At least one homing ref row exists (live OR dangling) — distinguishes
// "orphaned" (has a ref, none resolve) from "unlinked" (no ref at all).
-export const HAS_ANY_HOMING_REF = `EXISTS (
- SELECT 1 FROM catalog_identity_read_refs r
- WHERE catalog_canonical_ingredient_id(r.ingredient_id) = catalog_ingredients.id AND r.deleted = false
- AND r.ref_kind IN (${HOMING_REF_KINDS_SQL}))`;
+export const HAS_ANY_HOMING_REF = inSet('id', `SELECT r.cid FROM canonical_refs r WHERE r.ref_kind IN (${HOMING_REF_KINDS_SQL})`);
diff --git a/server/services/catalogDBMedia.test.js b/server/services/catalogDBMedia.test.js
index 8765bb3f32..178bfe8d13 100644
--- a/server/services/catalogDBMedia.test.js
+++ b/server/services/catalogDBMedia.test.js
@@ -206,41 +206,33 @@ describe('listIngredients card thumbnail', () => {
...overrides,
});
- it('selects the portrait-or-latest-reference media key as thumbnail_key', async () => {
+ // The DB suites pin the decoration SQL; this pins the id join that attaches
+ // the one page-level decoration query back onto the page rows (#12989).
+ it('decorates the returned page by id, defaulting to a null thumbnail and empty usage', async () => {
const db = await import('../lib/db.js');
- db.query.mockResolvedValueOnce({ rows: [ingredientRow({ thumbnail_key: 'hero.png' })] });
+ db.query
+ .mockResolvedValueOnce({ rows: [ingredientRow({ id: 'cat-chr-a' }), ingredientRow({ id: 'cat-chr-b' })] })
+ .mockResolvedValueOnce({ rows: [
+ { id: 'cat-chr-b', thumbnail_key: null, ref_usage: null },
+ { id: 'cat-chr-a', thumbnail_key: 'hero.png', ref_usage: { universe: 1, 'creative-director': 2 } },
+ ] });
const { items } = await catalogDB.listIngredients({ limit: 10 });
- // mockResolvedValueOnce bypasses the factory's `calls` push, so read the
- // captured SQL from the mock's own call log (as the getMaxSequences test does).
- const sql = db.query.mock.calls.at(-1)[0].replace(/\s+/g, ' ');
- // The correlated subquery prefers a live portrait, then the most recent
- // live reference, and excludes unrenderable (audio/video/document) kinds.
- expect(sql).toMatch(/FROM catalog_identity_read_media m/i);
- expect(sql).toMatch(/m\.kind IN \('portrait', 'reference'\)/i);
- expect(sql).toMatch(/ORDER BY \(m\.kind = 'portrait'\) DESC, m\.created_at DESC/i);
- expect(sql).toMatch(/AS thumbnail_key/i);
- expect(items[0].thumbnailKey).toBe('hero.png');
+ expect(db.query.mock.calls.at(-1)[1]).toEqual([['cat-chr-a', 'cat-chr-b']]);
+ expect(items.map(({ id, thumbnailKey, usage }) => ({ id, thumbnailKey, usage }))).toEqual([
+ { id: 'cat-chr-a', thumbnailKey: 'hero.png', usage: { universe: 1, 'creative-director': 2 } },
+ { id: 'cat-chr-b', thumbnailKey: null, usage: {} },
+ ]);
});
- it('maps a null thumbnail_key (no image attached) to null', async () => {
+ it('skips the decoration query for an empty page and the embedding admin path', async () => {
const db = await import('../lib/db.js');
- db.query.mockResolvedValueOnce({ rows: [ingredientRow({ thumbnail_key: null })] });
- const { items } = await catalogDB.listIngredients({ limit: 10 });
- expect(items[0].thumbnailKey).toBeNull();
- });
-
- it('returns per-kind link counts as usage, defaulting to an empty object', async () => {
- const db = await import('../lib/db.js');
- db.query.mockResolvedValueOnce({ rows: [
- ingredientRow({ id: 'cat-chr-a', ref_usage: { universe: 1, 'creative-director': 2 } }),
- ingredientRow({ id: 'cat-chr-b', ref_usage: null }),
- ] });
- const { items } = await catalogDB.listIngredients({ limit: 10 });
- const sql = db.query.mock.calls.at(-1)[0].replace(/\s+/g, ' ');
- expect(sql).toMatch(/COUNT\(DISTINCT r\.ref_id\)/i);
- expect(sql).toMatch(/r\.deleted = false/i);
- expect(items[0].usage).toEqual({ universe: 1, 'creative-director': 2 });
- expect(items[1].usage).toEqual({});
+ db.query.mockResolvedValueOnce({ rows: [] });
+ const before = db.query.mock.calls.length;
+ expect((await catalogDB.listIngredients({ limit: 10 })).items).toEqual([]);
+ db.query.mockResolvedValueOnce({ rows: [ingredientRow()] });
+ const { items } = await catalogDB.listIngredients({ limit: 10, includeEmbedding: true });
+ expect(db.query.mock.calls.length - before).toBe(2);
+ expect(items[0]).toMatchObject({ thumbnailKey: null, usage: {} });
});
});