Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
fix: preserve catalog identity and media when re-importing exports (#11008)
Adam Eivy7192bab8828f
diff --git a/server/lib/catalogBulkParsers.js b/server/lib/catalogBulkParsers.jsindex ffd0fbb0dc..5d55ec8ea2 100644--- a/server/lib/catalogBulkParsers.js+++ b/server/lib/catalogBulkParsers.js@@ -108,6 +108,10 @@ export function parseJsonBulk(payload) {}const entries = rawEntries.map((entry, i) => {const normalized = normalizeEntry(entry, i);+ // Export identity and attachments are validated by the bulk-entry schema.+ for (const key of ['id', 'media', 'scraps']) {+ if (entry[key] !== undefined) attachNonEnumerable(normalized, key, entry[key]);+ }// Preserve the per-row role the export bundle stamped so the bulk-import// route can re-create each ingredient's ref link with its original role,// instead of collapsing every row onto the batch-level default. Rides as@@ -360,13 +364,8 @@ export function parseMarkdownBulk(payload) {}const entries = out.map((entry, i) => {const normalized = normalizeEntry(entry, i);- // Parse scraps into a NON-enumerable sibling so the markdown re-import is- // lossy-but-not-corrupt (per PLAN [catalog-markdown-roundtrip-fences-scraps]):- // the bulk-import route does NOT persist scraps today — it spreads each- // entry into the `.strict()` catalogIngredientCreateSchema, which has no- // `scraps` field, so an enumerable key would reject the whole import. The- // data is preserved on `entry.scraps` for any consumer; persisting it into- // catalog_scraps/catalog_ingredient_sources is a tracked follow-up.+ // Keep parser output compatible with callers consuming only draft fields.+ // The bulk-import route explicitly validates and persists this metadata.if (Array.isArray(entry.scraps) && entry.scraps.length > 0) {attachNonEnumerable(normalized, 'scraps', entry.scraps);}diff --git a/server/lib/catalogValidation.js b/server/lib/catalogValidation.jsindex 2c8bfebf37..28a73cc9bd 100644--- a/server/lib/catalogValidation.js+++ b/server/lib/catalogValidation.js@@ -518,6 +518,22 @@ export const catalogMigrationRerunSchema = z.object({force: z.boolean().optional(),}).strict();+// Export rows carry identity and provenance in addition to create fields.+// Strip export-only timestamps and source/media row bookkeeping.+export const catalogBulkEntrySchema = catalogIngredientCreateSchema.extend({+ id: z.string().trim().min(1).max(80).optional(),+ scraps: z.array(z.object({+ id: z.string().trim().min(1).max(80).optional(),+ title: z.string().max(300).nullable().optional(),+ rawText: z.string().min(1).max(2_000_000),+ sourceKind: z.string().trim().min(1).max(32).optional(),+ metadata: z.record(z.string(), z.unknown()).optional(),+ })).optional(),+ media: z.array(catalogMediaAttachSchema.strip().extend({+ metadata: z.record(z.string(), z.unknown()).optional(),+ })).optional(),+}).strict();+// /bulk-import — accept a structured payload in one of three formats. The// route parses `payload` into a list of ingredient drafts (per the format),// then validates each entry against catalogIngredientCreateSchema beforediff --git a/server/routes/catalog.js b/server/routes/catalog.jsindex dc8dfdb8cd..a9f2fc079b 100644--- a/server/routes/catalog.js+++ b/server/routes/catalog.js@@ -30,6 +30,7 @@ import {catalogEmbeddingsBackfillSchema,catalogMigrationRerunSchema,catalogBulkImportSchema,+ catalogBulkEntrySchema,catalogExportQuerySchema,catalogRevisionQuerySchema,catalogRevisionRestoreSchema,@@ -584,23 +585,21 @@ router.post('/bulk-import', asyncHandler(async (req, res) => {// Zod-validated shape so we can stamp each ref link with its original role.const entries = [];const perRowRoles = [];- // Round-tripped scraps (markdown `### Scraps`) ride as a non-enumerable- // `entry.scraps` sibling — captured here alongside the Zod-validated shape so- // they can be persisted as catalog_scraps rows in the same transaction below.- const perRowScraps = [];+ // Explicitly validate the non-enumerable JSON/markdown metadata too.for (let i = 0; i < parsed.length; i++) {const entry = parsed[i];// Merge default tags onto each row (dedup), then validate the full// per-row shape against the same schema the single-create endpoint uses.const mergedTags = Array.from(new Set([...(entry.tags || []), ...defaultTags]));- const result = catalogIngredientCreateSchema.safeParse({ ...entry, tags: mergedTags });+ const result = catalogBulkEntrySchema.safeParse({+ ...entry, tags: mergedTags, id: entry.id, scraps: entry.scraps, media: entry.media,+ });if (!result.success) {const msg = result.error.issues?.[0]?.message || result.error.message;throw new ServerError(`Bulk import entry ${i} invalid: ${msg}`, { status: 400 });}entries.push(result.data);perRowRoles.push(typeof entry.roleForExportedRef === 'string' ? entry.roleForExportedRef : null);- perRowScraps.push(Array.isArray(entry.scraps) ? entry.scraps : []);}// Optional ref-links: same shape as the /link route, applied once per@@ -635,18 +634,34 @@ router.post('/bulk-import', asyncHandler(async (req, res) => {let scrapsCreated = 0;const created = await withTransaction(async (client) => {+ // Serialize retries even when their explicit ingredient/scrap IDs do not+ // exist yet. The transaction releases this lock on commit or rollback.+ await client.query("SELECT pg_advisory_xact_lock(hashtext('catalog-bulk-import'))");const out = [];for (let i = 0; i < entries.length; i++) {const entry = entries[i];const e = embeds[i];- const ing = await catalogDB.createIngredient({+ const existing = entry.id+ ? (await client.query('SELECT type, deleted FROM catalog_ingredients WHERE id = $1 FOR UPDATE', [entry.id])).rows[0]+ : null;+ if (existing && (existing.deleted || existing.type !== entry.type)) {+ throw new ServerError('Imported ingredient ID is deleted or has a different type', { status: 409 });+ }+ const values = {+ id: entry.id,type: entry.type,name: entry.name,payload: entry.payload || {},tags: entry.tags || [],embedding: e?.embedding ?? null,embeddingModel: e?.model ?? null,- }, { client });+ };+ const ing = existing+ ? await catalogDB.updateIngredient(entry.id, values, { client })+ : await catalogDB.createIngredient(values, { client });+ for (const media of entry.media || []) {+ await catalogDB.attachMedia(ing.id, media.mediaKey, media.kind, { ...media, client });+ }// Stamp ref links inside the same transaction so a mid-batch failure// rolls back the link rows alongside their ingredients. Role precedence:// explicit `defaults.role` > the bundle row's own `roleForExportedRef` >@@ -665,21 +680,32 @@ router.post('/bulk-import', asyncHandler(async (req, res) => {[ing.id, target.refKind, target.refId, role],);}- // Persist any round-tripped scraps as catalog_scraps rows + source links- // in the same transaction, so they roll back with the ingredient on a- // mid-batch failure. Only markdown imports carry scraps today; JSON/CSV- // rows have an empty list. No embedding is generated for an imported scrap- // (the ingredient already carries the searchable embedding) — the scrap is- // provenance, not a separate search target.- for (const s of perRowScraps[i]) {- if (!s || typeof s.rawText !== 'string' || !s.rawText.trim()) continue;- const scrap = await catalogDB.createScrap({- rawText: s.rawText,- sourceKind: (typeof s.sourceKind === 'string' && s.sourceKind.trim() ? s.sourceKind.trim() : 'import').slice(0, 32),- metadata: { importedVia: 'bulk-import', format },+ // Preserve shared scrap identity. ID-less additions to an identified+ // ingredient deduplicate by text/source, so retrying an edited bundle+ // does not create new provenance on each attempt.+ for (const s of entry.scraps || []) {+ if (!s.rawText.trim()) continue;+ const sourceKind = s.sourceKind || 'import';+ const existingScrap = s.id+ ? (await client.query('SELECT id, deleted FROM catalog_scraps WHERE id = $1 FOR UPDATE', [s.id])).rows[0]+ : entry.id+ ? (await client.query(+ `SELECT s.id, s.deleted FROM catalog_scraps s+ JOIN catalog_ingredient_sources src ON src.scrap_id = s.id+ WHERE src.ingredient_id = $1 AND s.raw_text = $2+ AND s.source_kind = $3 AND s.deleted = false LIMIT 1`,+ [ing.id, s.rawText, sourceKind],+ )).rows[0]+ : null;+ if (existingScrap?.deleted) {+ throw new ServerError('Imported scrap ID is deleted', { status: 409 });+ }+ const scrap = existingScrap || await catalogDB.createScrap({+ ...s, sourceKind,+ metadata: s.metadata || { importedVia: 'bulk-import', format },}, { client });- await catalogDB.linkIngredientToSource(ing.id, scrap.id, null, { client });- scrapsCreated++;+ await catalogDB.linkIngredientToSource(ing.id, scrap.id, null, { client, preserveExisting: true });+ if (!existingScrap) scrapsCreated++;}out.push({ id: ing.id, type: ing.type, name: ing.name });}diff --git a/server/routes/catalog.test.js b/server/routes/catalog.test.jsindex 6a341a9645..947625d306 100644--- a/server/routes/catalog.test.js+++ b/server/routes/catalog.test.js@@ -213,6 +213,110 @@ describe.skipIf(!runDb)('POST /api/catalog/bulk-import — export-bundle ref rec});});++describe.skipIf(!runDb)('catalog export identity round trip', () => {+ const send = (bundle) => request(makeApp()).post('/api/catalog/bulk-import')+ .send({ format: 'json', payload: JSON.stringify(bundle) });++ it('imports an exported slice twice and reuses IDs, shared scraps and media while applying edits', async () => {+ const refId = `roundtrip-${NONCE}`;+ const original = await catalogDB.createIngredient({ type: 'idea', name: 'Export original' });+ const scrap = await catalogDB.createScrap({ rawText: 'Original provenance' });+ createdIngredientIds.add(original.id);+ createdScrapIds.add(scrap.id);+ await catalogDB.linkIngredientToSource(original.id, scrap.id);+ await catalogDB.linkIngredientToRef(original.id, 'series', refId, 'cast');+ await catalogDB.attachMedia(original.id, 'example.png', 'portrait', { caption: 'Example', metadata: { prompt: 'Example prompt' } });+ await catalogDB.attachMedia(original.id, 'reference.png', 'reference');+ const bundle = await catalogDB.exportSliceForRef('series', refId);+ // Simulate a receiving catalog without either exported record.+ await catalogDB.deleteIngredient(original.id, { hard: true });+ await catalogDB.deleteScrap(scrap.id, { hard: true });+ bundle.ingredients[0].name = 'Edited export';+ bundle.ingredients[0].scraps.push({ rawText: 'New notes', sourceKind: 'import' });+ const first = await send(bundle);+ await catalogDB.linkIngredientToSource(original.id, scrap.id, { start: 2, end: 8 });+ const results = [first, await send(bundle)];+ expect(results.map((r) => r.status)).toEqual([201, 201]);+ expect(results.map((r) => r.body.scrapsCreated)).toEqual([2, 0]);+ expect(results.map((r) => r.body.created[0].id)).toEqual([original.id, original.id]);+ const exported = await catalogDB.exportSliceForRef('series', refId);+ expect(exported.ingredients).toHaveLength(1);+ const row = exported.ingredients[0];+ for (const item of row.scraps) createdScrapIds.add(item.id);+ expect(row.name).toBe('Edited export');+ expect(row.scraps).toHaveLength(2);+ expect((await catalogDB.listSourcesForIngredient(original.id)).find((source) => source.scrapId === scrap.id).span)+ .toEqual({ start: 2, end: 8 });+ expect(row.scraps.some((item) => item.id === scrap.id)).toBe(true);+ expect(row.media.map((item) => item.mediaKey).sort()).toEqual(['example.png', 'reference.png']);+ expect(row.media.find((item) => item.kind === 'portrait').metadata).toMatchObject({ prompt: 'Example prompt' });+ const storedMedia = await query('SELECT * FROM catalog_ingredient_media WHERE ingredient_id = $1', [original.id]);+ expect(storedMedia.rows).toHaveLength(2);+ // Overlapping retries use the same transaction guard, including new IDs.+ const concurrentId = `cat-idea-concurrent-${NONCE}`;+ createdIngredientIds.add(concurrentId);+ bundle.ingredients[0].id = concurrentId;+ bundle.ingredients[0].scraps = [scrap];+ const concurrent = await Promise.all([send(bundle), send(bundle)]);+ expect(concurrent.map((r) => r.status)).toEqual([201, 201]);+ expect(await catalogDB.listSourcesForIngredient(concurrentId)).toHaveLength(1);+ });++ it.each(['json', 'csv'])('keeps ID-less %s imports as new creates', async (format) => {+ const payload = format === 'json'+ ? JSON.stringify([{ type: 'idea', name: 'No identity' }])+ : 'type,name\nidea,No identity';+ const ids = [];+ for (let i = 0; i < 2; i++) {+ const response = await request(makeApp()).post('/api/catalog/bulk-import').send({ format, payload });+ expect(response.status).toBe(201);+ ids.push(response.body.created[0].id);+ createdIngredientIds.add(ids.at(-1));+ }+ expect(new Set(ids).size).toBe(2);+ });++ it('rolls back ingredient updates, new ingredients, refs, scraps and media on a later SQL failure', async () => {+ const old = await catalogDB.createIngredient({ type: 'idea', name: 'Before import' });+ createdIngredientIds.add(old.id);+ await catalogDB.attachMedia(old.id, 'before.png', 'portrait');+ const freshId = `cat-idea-bulk-rollback-${NONCE}`;+ const scrapId = `cat-scrap-bulk-rollback-${NONCE}`;+ createdIngredientIds.add(freshId);+ createdScrapIds.add(scrapId);+ const bundle = {+ ref: { kind: 'series', id: `rollback-${NONCE}` },+ ingredients: [+ { id: old.id, type: 'idea', name: 'After import', media: [{ mediaKey: 'after.png', kind: 'portrait' }] },+ { id: freshId, type: 'idea', name: 'New import', scraps: [{ id: scrapId, rawText: 'Rollback notes' }],+ media: [{ mediaKey: 'new.png', kind: 'reference' }] },+ ],+ };+ const response = await withRevisionFailure(/^INSERT INTO catalog_ingredient_sources/i, () => send(bundle));+ expect(response.status).toBe(500);+ expect((await catalogDB.getIngredient(old.id)).name).toBe('Before import');+ expect(await catalogDB.getIngredient(freshId)).toBeNull();+ expect(await catalogDB.getScrap(scrapId)).toBeNull();+ expect((await catalogDB.listMediaForIngredient(old.id)).map((row) => row.mediaKey)).toEqual(['before.png']);+ expect(await catalogDB.listRefsForIngredient(old.id)).toHaveLength(0);+ expect((await query('SELECT * FROM catalog_ingredient_media WHERE ingredient_id = ANY($1)', [[old.id, freshId]])).rows).toHaveLength(1);+ });++ it('rejects malformed identity and media before writes, and refuses deleted or conflicting IDs', async () => {+ const row = await catalogDB.createIngredient({ type: 'idea', name: 'Keep identity' });+ createdIngredientIds.add(row.id);+ for (const entry of [+ { id: 123, type: 'idea', name: 'Invalid ID' },+ { type: 'idea', name: 'Invalid media', media: [{ mediaKey: 'x', kind: 'invalid' }] },+ ]) expect((await send([entry])).status).toBe(400);+ expect((await send([{ id: row.id, type: 'scene', name: 'Wrong type' }])).status).toBe(409);+ await catalogDB.deleteIngredient(row.id);+ expect((await send([{ id: row.id, type: 'idea', name: 'Deleted' }])).status).toBe(409);+ expect(await catalogDB.getIngredient(row.id)).toBeNull();+ });+});+// Inject a real SQL error at the pg transport, retaining its promise/callback// interface. All other statements (including rollback) still reach portos_test.async function withRevisionFailure(statement, operation, matchesParams = () => true) {diff --git a/server/services/catalogDB/media.js b/server/services/catalogDB/media.jsindex d2d6690179..06328c0e5e 100644--- a/server/services/catalogDB/media.js+++ b/server/services/catalogDB/media.js@@ -20,7 +20,7 @@ export async function attachMedia(ingredientId, mediaKey, kind, options = {}) {// Every local portrait entry point shares the replacement transaction,// including the generic media API and file uploads.if (kind === 'portrait') return setPortraitMedia(ingredientId, mediaKey, options);- return insertMedia(query, ingredientId, mediaKey, kind, options);+ return insertMedia(options.client ? options.client.query.bind(options.client) : query, ingredientId, mediaKey, kind, options);}async function insertMedia(runQuery, ingredientId, mediaKey, kind, options) {@@ -64,7 +64,7 @@ export async function detachMedia(ingredientId, mediaKey, kind) {// attachment commit together, so overlapping HTTP/generation writes serialize// and a failed replacement leaves the previous portrait intact.export async function setPortraitMedia(ingredientId, mediaKey, options = {}) {- return withTransaction(async (client) => {+ const replace = async (client) => {await client.query('SELECT id FROM catalog_ingredients WHERE id = $1 FOR UPDATE', [ingredientId]);await client.query(`UPDATE catalog_ingredient_media@@ -74,7 +74,8 @@ export async function setPortraitMedia(ingredientId, mediaKey, options = {}) {[ingredientId, mediaKey],);return insertMedia(client.query.bind(client), ingredientId, mediaKey, 'portrait', options);- });+ };+ return options.client ? replace(options.client) : withTransaction(replace);}// Live (non-tombstoned) media rows for an ingredient's detail "Media" panel,diff --git a/server/services/catalogDB/refs.js b/server/services/catalogDB/refs.jsindex e3effb179c..a9340a311e 100644--- a/server/services/catalogDB/refs.js+++ b/server/services/catalogDB/refs.js@@ -17,12 +17,12 @@ import { rowToRef, rowToSource, rowToIngredient, groupRowsByIngredient } from '.// `{ client }` is optional — see the createIngredient comment. Passing the same// client used to insert the ingredient row keeps the source-link row in the// same transaction so a mid-batch failure rolls back both halves.-export async function linkIngredientToSource(ingredientId, scrapId, span = null, { client } = {}) {+export async function linkIngredientToSource(ingredientId, scrapId, span = null, { client, preserveExisting = false } = {}) {const exec = client ? client.query.bind(client) : query;await exec(`INSERT INTO catalog_ingredient_sources (ingredient_id, scrap_id, span)VALUES ($1, $2, $3::jsonb)- ON CONFLICT (ingredient_id, scrap_id) DO UPDATE SET span = EXCLUDED.span`,+ ON CONFLICT (ingredient_id, scrap_id) ${preserveExisting ? 'DO NOTHING' : 'DO UPDATE SET span = EXCLUDED.span'}`,[ingredientId, scrapId, span ? JSON.stringify(span) : null],);}diff --git a/server/services/catalogDB/scraps.js b/server/services/catalogDB/scraps.jsindex 4e29acdd53..c26354216d 100644--- a/server/services/catalogDB/scraps.js+++ b/server/services/catalogDB/scraps.js@@ -16,9 +16,9 @@ import { newScrapId, rowToScrap } from './shared.js';// alongside the ingredients + source links (see POST /api/catalog/bulk-import).// Absent, falls through to the pool-level `query` as before. Mirrors the same// option on `createIngredient` / `linkIngredientToSource`.-export async function createScrap({ title, rawText, sourceKind = 'paste', metadata = {}, embedding = null, embeddingModel = null, chunkIndex = 0, parentScrapId = null } = {}, { client } = {}) {+export async function createScrap({ id: explicitId, title, rawText, sourceKind = 'paste', metadata = {}, embedding = null, embeddingModel = null, chunkIndex = 0, parentScrapId = null } = {}, { client } = {}) {if (!rawText) throw new Error('rawText is required');- const id = newScrapId();+ const id = explicitId || newScrapId();const originInstanceId = await getInstanceId();const exec = client ? client.query.bind(client) : query;const result = await exec(