Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

fix: preserve catalog identity and media when re-importing exports (#11008)

Adam Eivy7192bab8828f
diff --git a/server/lib/catalogBulkParsers.js b/server/lib/catalogBulkParsers.js
index ffd0fbb0dc..5d55ec8ea2 100644
--- a/server/lib/catalogBulkParsers.js
+++ b/server/lib/catalogBulkParsers.js
@@ -108,6 +108,10 @@ export function parseJsonBulk(payload) {
}
const entries = rawEntries.map((entry, i) => {
const normalized = normalizeEntry(entry, i);
+ // Export identity and attachments are validated by the bulk-entry schema.
+ for (const key of ['id', 'media', 'scraps']) {
+ if (entry[key] !== undefined) attachNonEnumerable(normalized, key, entry[key]);
+ }
// Preserve the per-row role the export bundle stamped so the bulk-import
// route can re-create each ingredient's ref link with its original role,
// instead of collapsing every row onto the batch-level default. Rides as
@@ -360,13 +364,8 @@ export function parseMarkdownBulk(payload) {
}
const entries = out.map((entry, i) => {
const normalized = normalizeEntry(entry, i);
- // Parse scraps into a NON-enumerable sibling so the markdown re-import is
- // lossy-but-not-corrupt (per PLAN [catalog-markdown-roundtrip-fences-scraps]):
- // the bulk-import route does NOT persist scraps today — it spreads each
- // entry into the `.strict()` catalogIngredientCreateSchema, which has no
- // `scraps` field, so an enumerable key would reject the whole import. The
- // data is preserved on `entry.scraps` for any consumer; persisting it into
- // catalog_scraps/catalog_ingredient_sources is a tracked follow-up.
+ // Keep parser output compatible with callers consuming only draft fields.
+ // The bulk-import route explicitly validates and persists this metadata.
if (Array.isArray(entry.scraps) && entry.scraps.length > 0) {
attachNonEnumerable(normalized, 'scraps', entry.scraps);
}
diff --git a/server/lib/catalogValidation.js b/server/lib/catalogValidation.js
index 2c8bfebf37..28a73cc9bd 100644
--- a/server/lib/catalogValidation.js
+++ b/server/lib/catalogValidation.js
@@ -518,6 +518,22 @@ export const catalogMigrationRerunSchema = z.object({
force: z.boolean().optional(),
}).strict();
+// Export rows carry identity and provenance in addition to create fields.
+// Strip export-only timestamps and source/media row bookkeeping.
+export const catalogBulkEntrySchema = catalogIngredientCreateSchema.extend({
+ id: z.string().trim().min(1).max(80).optional(),
+ scraps: z.array(z.object({
+ id: z.string().trim().min(1).max(80).optional(),
+ title: z.string().max(300).nullable().optional(),
+ rawText: z.string().min(1).max(2_000_000),
+ sourceKind: z.string().trim().min(1).max(32).optional(),
+ metadata: z.record(z.string(), z.unknown()).optional(),
+ })).optional(),
+ media: z.array(catalogMediaAttachSchema.strip().extend({
+ metadata: z.record(z.string(), z.unknown()).optional(),
+ })).optional(),
+}).strict();
+
// /bulk-import — accept a structured payload in one of three formats. The
// route parses `payload` into a list of ingredient drafts (per the format),
// then validates each entry against catalogIngredientCreateSchema before
diff --git a/server/routes/catalog.js b/server/routes/catalog.js
index dc8dfdb8cd..a9f2fc079b 100644
--- a/server/routes/catalog.js
+++ b/server/routes/catalog.js
@@ -30,6 +30,7 @@ import {
catalogEmbeddingsBackfillSchema,
catalogMigrationRerunSchema,
catalogBulkImportSchema,
+ catalogBulkEntrySchema,
catalogExportQuerySchema,
catalogRevisionQuerySchema,
catalogRevisionRestoreSchema,
@@ -584,23 +585,21 @@ router.post('/bulk-import', asyncHandler(async (req, res) => {
// Zod-validated shape so we can stamp each ref link with its original role.
const entries = [];
const perRowRoles = [];
- // Round-tripped scraps (markdown `### Scraps`) ride as a non-enumerable
- // `entry.scraps` sibling — captured here alongside the Zod-validated shape so
- // they can be persisted as catalog_scraps rows in the same transaction below.
- const perRowScraps = [];
+ // Explicitly validate the non-enumerable JSON/markdown metadata too.
for (let i = 0; i < parsed.length; i++) {
const entry = parsed[i];
// Merge default tags onto each row (dedup), then validate the full
// per-row shape against the same schema the single-create endpoint uses.
const mergedTags = Array.from(new Set([...(entry.tags || []), ...defaultTags]));
- const result = catalogIngredientCreateSchema.safeParse({ ...entry, tags: mergedTags });
+ const result = catalogBulkEntrySchema.safeParse({
+ ...entry, tags: mergedTags, id: entry.id, scraps: entry.scraps, media: entry.media,
+ });
if (!result.success) {
const msg = result.error.issues?.[0]?.message || result.error.message;
throw new ServerError(`Bulk import entry ${i} invalid: ${msg}`, { status: 400 });
}
entries.push(result.data);
perRowRoles.push(typeof entry.roleForExportedRef === 'string' ? entry.roleForExportedRef : null);
- perRowScraps.push(Array.isArray(entry.scraps) ? entry.scraps : []);
}
// Optional ref-links: same shape as the /link route, applied once per
@@ -635,18 +634,34 @@ router.post('/bulk-import', asyncHandler(async (req, res) => {
let scrapsCreated = 0;
const created = await withTransaction(async (client) => {
+ // Serialize retries even when their explicit ingredient/scrap IDs do not
+ // exist yet. The transaction releases this lock on commit or rollback.
+ await client.query("SELECT pg_advisory_xact_lock(hashtext('catalog-bulk-import'))");
const out = [];
for (let i = 0; i < entries.length; i++) {
const entry = entries[i];
const e = embeds[i];
- const ing = await catalogDB.createIngredient({
+ const existing = entry.id
+ ? (await client.query('SELECT type, deleted FROM catalog_ingredients WHERE id = $1 FOR UPDATE', [entry.id])).rows[0]
+ : null;
+ if (existing && (existing.deleted || existing.type !== entry.type)) {
+ throw new ServerError('Imported ingredient ID is deleted or has a different type', { status: 409 });
+ }
+ const values = {
+ id: entry.id,
type: entry.type,
name: entry.name,
payload: entry.payload || {},
tags: entry.tags || [],
embedding: e?.embedding ?? null,
embeddingModel: e?.model ?? null,
- }, { client });
+ };
+ const ing = existing
+ ? await catalogDB.updateIngredient(entry.id, values, { client })
+ : await catalogDB.createIngredient(values, { client });
+ for (const media of entry.media || []) {
+ await catalogDB.attachMedia(ing.id, media.mediaKey, media.kind, { ...media, client });
+ }
// Stamp ref links inside the same transaction so a mid-batch failure
// rolls back the link rows alongside their ingredients. Role precedence:
// explicit `defaults.role` > the bundle row's own `roleForExportedRef` >
@@ -665,21 +680,32 @@ router.post('/bulk-import', asyncHandler(async (req, res) => {
[ing.id, target.refKind, target.refId, role],
);
}
- // Persist any round-tripped scraps as catalog_scraps rows + source links
- // in the same transaction, so they roll back with the ingredient on a
- // mid-batch failure. Only markdown imports carry scraps today; JSON/CSV
- // rows have an empty list. No embedding is generated for an imported scrap
- // (the ingredient already carries the searchable embedding) — the scrap is
- // provenance, not a separate search target.
- for (const s of perRowScraps[i]) {
- if (!s || typeof s.rawText !== 'string' || !s.rawText.trim()) continue;
- const scrap = await catalogDB.createScrap({
- rawText: s.rawText,
- sourceKind: (typeof s.sourceKind === 'string' && s.sourceKind.trim() ? s.sourceKind.trim() : 'import').slice(0, 32),
- metadata: { importedVia: 'bulk-import', format },
+ // Preserve shared scrap identity. ID-less additions to an identified
+ // ingredient deduplicate by text/source, so retrying an edited bundle
+ // does not create new provenance on each attempt.
+ for (const s of entry.scraps || []) {
+ if (!s.rawText.trim()) continue;
+ const sourceKind = s.sourceKind || 'import';
+ const existingScrap = s.id
+ ? (await client.query('SELECT id, deleted FROM catalog_scraps WHERE id = $1 FOR UPDATE', [s.id])).rows[0]
+ : entry.id
+ ? (await client.query(
+ `SELECT s.id, s.deleted FROM catalog_scraps s
+ JOIN catalog_ingredient_sources src ON src.scrap_id = s.id
+ WHERE src.ingredient_id = $1 AND s.raw_text = $2
+ AND s.source_kind = $3 AND s.deleted = false LIMIT 1`,
+ [ing.id, s.rawText, sourceKind],
+ )).rows[0]
+ : null;
+ if (existingScrap?.deleted) {
+ throw new ServerError('Imported scrap ID is deleted', { status: 409 });
+ }
+ const scrap = existingScrap || await catalogDB.createScrap({
+ ...s, sourceKind,
+ metadata: s.metadata || { importedVia: 'bulk-import', format },
}, { client });
- await catalogDB.linkIngredientToSource(ing.id, scrap.id, null, { client });
- scrapsCreated++;
+ await catalogDB.linkIngredientToSource(ing.id, scrap.id, null, { client, preserveExisting: true });
+ if (!existingScrap) scrapsCreated++;
}
out.push({ id: ing.id, type: ing.type, name: ing.name });
}
diff --git a/server/routes/catalog.test.js b/server/routes/catalog.test.js
index 6a341a9645..947625d306 100644
--- a/server/routes/catalog.test.js
+++ b/server/routes/catalog.test.js
@@ -213,6 +213,110 @@ describe.skipIf(!runDb)('POST /api/catalog/bulk-import — export-bundle ref rec
});
});
+
+describe.skipIf(!runDb)('catalog export identity round trip', () => {
+ const send = (bundle) => request(makeApp()).post('/api/catalog/bulk-import')
+ .send({ format: 'json', payload: JSON.stringify(bundle) });
+
+ it('imports an exported slice twice and reuses IDs, shared scraps and media while applying edits', async () => {
+ const refId = `roundtrip-${NONCE}`;
+ const original = await catalogDB.createIngredient({ type: 'idea', name: 'Export original' });
+ const scrap = await catalogDB.createScrap({ rawText: 'Original provenance' });
+ createdIngredientIds.add(original.id);
+ createdScrapIds.add(scrap.id);
+ await catalogDB.linkIngredientToSource(original.id, scrap.id);
+ await catalogDB.linkIngredientToRef(original.id, 'series', refId, 'cast');
+ await catalogDB.attachMedia(original.id, 'example.png', 'portrait', { caption: 'Example', metadata: { prompt: 'Example prompt' } });
+ await catalogDB.attachMedia(original.id, 'reference.png', 'reference');
+ const bundle = await catalogDB.exportSliceForRef('series', refId);
+ // Simulate a receiving catalog without either exported record.
+ await catalogDB.deleteIngredient(original.id, { hard: true });
+ await catalogDB.deleteScrap(scrap.id, { hard: true });
+ bundle.ingredients[0].name = 'Edited export';
+ bundle.ingredients[0].scraps.push({ rawText: 'New notes', sourceKind: 'import' });
+ const first = await send(bundle);
+ await catalogDB.linkIngredientToSource(original.id, scrap.id, { start: 2, end: 8 });
+ const results = [first, await send(bundle)];
+ expect(results.map((r) => r.status)).toEqual([201, 201]);
+ expect(results.map((r) => r.body.scrapsCreated)).toEqual([2, 0]);
+ expect(results.map((r) => r.body.created[0].id)).toEqual([original.id, original.id]);
+ const exported = await catalogDB.exportSliceForRef('series', refId);
+ expect(exported.ingredients).toHaveLength(1);
+ const row = exported.ingredients[0];
+ for (const item of row.scraps) createdScrapIds.add(item.id);
+ expect(row.name).toBe('Edited export');
+ expect(row.scraps).toHaveLength(2);
+ expect((await catalogDB.listSourcesForIngredient(original.id)).find((source) => source.scrapId === scrap.id).span)
+ .toEqual({ start: 2, end: 8 });
+ expect(row.scraps.some((item) => item.id === scrap.id)).toBe(true);
+ expect(row.media.map((item) => item.mediaKey).sort()).toEqual(['example.png', 'reference.png']);
+ expect(row.media.find((item) => item.kind === 'portrait').metadata).toMatchObject({ prompt: 'Example prompt' });
+ const storedMedia = await query('SELECT * FROM catalog_ingredient_media WHERE ingredient_id = $1', [original.id]);
+ expect(storedMedia.rows).toHaveLength(2);
+ // Overlapping retries use the same transaction guard, including new IDs.
+ const concurrentId = `cat-idea-concurrent-${NONCE}`;
+ createdIngredientIds.add(concurrentId);
+ bundle.ingredients[0].id = concurrentId;
+ bundle.ingredients[0].scraps = [scrap];
+ const concurrent = await Promise.all([send(bundle), send(bundle)]);
+ expect(concurrent.map((r) => r.status)).toEqual([201, 201]);
+ expect(await catalogDB.listSourcesForIngredient(concurrentId)).toHaveLength(1);
+ });
+
+ it.each(['json', 'csv'])('keeps ID-less %s imports as new creates', async (format) => {
+ const payload = format === 'json'
+ ? JSON.stringify([{ type: 'idea', name: 'No identity' }])
+ : 'type,name\nidea,No identity';
+ const ids = [];
+ for (let i = 0; i < 2; i++) {
+ const response = await request(makeApp()).post('/api/catalog/bulk-import').send({ format, payload });
+ expect(response.status).toBe(201);
+ ids.push(response.body.created[0].id);
+ createdIngredientIds.add(ids.at(-1));
+ }
+ expect(new Set(ids).size).toBe(2);
+ });
+
+ it('rolls back ingredient updates, new ingredients, refs, scraps and media on a later SQL failure', async () => {
+ const old = await catalogDB.createIngredient({ type: 'idea', name: 'Before import' });
+ createdIngredientIds.add(old.id);
+ await catalogDB.attachMedia(old.id, 'before.png', 'portrait');
+ const freshId = `cat-idea-bulk-rollback-${NONCE}`;
+ const scrapId = `cat-scrap-bulk-rollback-${NONCE}`;
+ createdIngredientIds.add(freshId);
+ createdScrapIds.add(scrapId);
+ const bundle = {
+ ref: { kind: 'series', id: `rollback-${NONCE}` },
+ ingredients: [
+ { id: old.id, type: 'idea', name: 'After import', media: [{ mediaKey: 'after.png', kind: 'portrait' }] },
+ { id: freshId, type: 'idea', name: 'New import', scraps: [{ id: scrapId, rawText: 'Rollback notes' }],
+ media: [{ mediaKey: 'new.png', kind: 'reference' }] },
+ ],
+ };
+ const response = await withRevisionFailure(/^INSERT INTO catalog_ingredient_sources/i, () => send(bundle));
+ expect(response.status).toBe(500);
+ expect((await catalogDB.getIngredient(old.id)).name).toBe('Before import');
+ expect(await catalogDB.getIngredient(freshId)).toBeNull();
+ expect(await catalogDB.getScrap(scrapId)).toBeNull();
+ expect((await catalogDB.listMediaForIngredient(old.id)).map((row) => row.mediaKey)).toEqual(['before.png']);
+ expect(await catalogDB.listRefsForIngredient(old.id)).toHaveLength(0);
+ expect((await query('SELECT * FROM catalog_ingredient_media WHERE ingredient_id = ANY($1)', [[old.id, freshId]])).rows).toHaveLength(1);
+ });
+
+ it('rejects malformed identity and media before writes, and refuses deleted or conflicting IDs', async () => {
+ const row = await catalogDB.createIngredient({ type: 'idea', name: 'Keep identity' });
+ createdIngredientIds.add(row.id);
+ for (const entry of [
+ { id: 123, type: 'idea', name: 'Invalid ID' },
+ { type: 'idea', name: 'Invalid media', media: [{ mediaKey: 'x', kind: 'invalid' }] },
+ ]) expect((await send([entry])).status).toBe(400);
+ expect((await send([{ id: row.id, type: 'scene', name: 'Wrong type' }])).status).toBe(409);
+ await catalogDB.deleteIngredient(row.id);
+ expect((await send([{ id: row.id, type: 'idea', name: 'Deleted' }])).status).toBe(409);
+ expect(await catalogDB.getIngredient(row.id)).toBeNull();
+ });
+});
+
// Inject a real SQL error at the pg transport, retaining its promise/callback
// interface. All other statements (including rollback) still reach portos_test.
async function withRevisionFailure(statement, operation, matchesParams = () => true) {
diff --git a/server/services/catalogDB/media.js b/server/services/catalogDB/media.js
index d2d6690179..06328c0e5e 100644
--- a/server/services/catalogDB/media.js
+++ b/server/services/catalogDB/media.js
@@ -20,7 +20,7 @@ export async function attachMedia(ingredientId, mediaKey, kind, options = {}) {
// Every local portrait entry point shares the replacement transaction,
// including the generic media API and file uploads.
if (kind === 'portrait') return setPortraitMedia(ingredientId, mediaKey, options);
- return insertMedia(query, ingredientId, mediaKey, kind, options);
+ return insertMedia(options.client ? options.client.query.bind(options.client) : query, ingredientId, mediaKey, kind, options);
}
async function insertMedia(runQuery, ingredientId, mediaKey, kind, options) {
@@ -64,7 +64,7 @@ export async function detachMedia(ingredientId, mediaKey, kind) {
// attachment commit together, so overlapping HTTP/generation writes serialize
// and a failed replacement leaves the previous portrait intact.
export async function setPortraitMedia(ingredientId, mediaKey, options = {}) {
- return withTransaction(async (client) => {
+ const replace = async (client) => {
await client.query('SELECT id FROM catalog_ingredients WHERE id = $1 FOR UPDATE', [ingredientId]);
await client.query(
`UPDATE catalog_ingredient_media
@@ -74,7 +74,8 @@ export async function setPortraitMedia(ingredientId, mediaKey, options = {}) {
[ingredientId, mediaKey],
);
return insertMedia(client.query.bind(client), ingredientId, mediaKey, 'portrait', options);
- });
+ };
+ return options.client ? replace(options.client) : withTransaction(replace);
}
// Live (non-tombstoned) media rows for an ingredient's detail "Media" panel,
diff --git a/server/services/catalogDB/refs.js b/server/services/catalogDB/refs.js
index e3effb179c..a9340a311e 100644
--- a/server/services/catalogDB/refs.js
+++ b/server/services/catalogDB/refs.js
@@ -17,12 +17,12 @@ import { rowToRef, rowToSource, rowToIngredient, groupRowsByIngredient } from '.
// `{ client }` is optional — see the createIngredient comment. Passing the same
// client used to insert the ingredient row keeps the source-link row in the
// same transaction so a mid-batch failure rolls back both halves.
-export async function linkIngredientToSource(ingredientId, scrapId, span = null, { client } = {}) {
+export async function linkIngredientToSource(ingredientId, scrapId, span = null, { client, preserveExisting = false } = {}) {
const exec = client ? client.query.bind(client) : query;
await exec(
`INSERT INTO catalog_ingredient_sources (ingredient_id, scrap_id, span)
VALUES ($1, $2, $3::jsonb)
- ON CONFLICT (ingredient_id, scrap_id) DO UPDATE SET span = EXCLUDED.span`,
+ ON CONFLICT (ingredient_id, scrap_id) ${preserveExisting ? 'DO NOTHING' : 'DO UPDATE SET span = EXCLUDED.span'}`,
[ingredientId, scrapId, span ? JSON.stringify(span) : null],
);
}
diff --git a/server/services/catalogDB/scraps.js b/server/services/catalogDB/scraps.js
index 4e29acdd53..c26354216d 100644
--- a/server/services/catalogDB/scraps.js
+++ b/server/services/catalogDB/scraps.js
@@ -16,9 +16,9 @@ import { newScrapId, rowToScrap } from './shared.js';
// alongside the ingredients + source links (see POST /api/catalog/bulk-import).
// Absent, falls through to the pool-level `query` as before. Mirrors the same
// option on `createIngredient` / `linkIngredientToSource`.
-export async function createScrap({ title, rawText, sourceKind = 'paste', metadata = {}, embedding = null, embeddingModel = null, chunkIndex = 0, parentScrapId = null } = {}, { client } = {}) {
+export async function createScrap({ id: explicitId, title, rawText, sourceKind = 'paste', metadata = {}, embedding = null, embeddingModel = null, chunkIndex = 0, parentScrapId = null } = {}, { client } = {}) {
if (!rawText) throw new Error('rawText is required');
- const id = newScrapId();
+ const id = explicitId || newScrapId();
const originInstanceId = await getInstanceId();
const exec = client ? client.query.bind(client) : query;
const result = await exec(