Skip to content

Open-source agentic AI Search Execution Platform. Track AI visibility, find opportunities, and let agents turn signals into prioritized actions and drafted fixes across ChatGPT, Claude, Gemini, Google AI Overviews, Google AI Mode, Perplexity, Grok, and Microsoft Copilot.

feat(catalog): add the product catalog and its batch sync (00278, #1263 part 1) (#1489)

Gökhan Geyik7a91993de2b4
diff --git a/server/src/lib/catalog/sync.js b/server/src/lib/catalog/sync.js
new file mode 100644
index 000000000..d95b778a5
--- /dev/null
+++ b/server/src/lib/catalog/sync.js
@@ -0,0 +1,331 @@
+import { createHash } from 'node:crypto';
+import supabaseAdmin from '../../config/supabase.js';
+
+/**
+ * Writes a brand's catalog from one source (#1263, 00278): the one batch
+ * sync every catalog source goes through (a feed URL, a CSV upload, a store
+ * connection), from the feed parsers' items (lib/feeds).
+ *
+ * - A feed's item group is one product, and each item one of its variants.
+ * An item with no group is its own product.
+ * - Rows are keyed by (brand, source, external id) and updated in place.
+ * A row whose content didn't change isn't written again (`content_hash`),
+ * so a sync of the same file writes nothing.
+ * - A product or variant gone from the source is marked removed
+ * (`removed_at`), never deleted. One that comes back is live again.
+ * - A changed product's images and categories are written again.
+ *
+ * Values are clipped to the columns' limits, and an address that isn't
+ * http(s) is left out, so one odd product doesn't fail the sync.
+ */
+
+const PAGE = 1000;
+const CHUNK = 500;
+
+const clip = (value, max) => {
+ if (typeof value !== 'string') return null;
+ const text = value.trim();
+ return text ? text.slice(0, max) : null;
+};
+const httpUrl = (value) => {
+ const url = clip(value, 2048);
+ return url && /^https?:\/\//i.test(url) ? url : null;
+};
+
+/** A stable hash of a row's content. */
+export function hashOf(value) {
+ return createHash('sha1').update(JSON.stringify(value)).digest('hex');
+}
+
+/**
+ * An item group's title: the words its items' titles start with, without a
+ * trailing separator ("Trailrunner Boot - Red", "Trailrunner Boot - Blue" →
+ * "Trailrunner Boot"). The first title when they share less than half.
+ */
+export function groupTitle(titles) {
+ const [first] = titles;
+ if (titles.length < 2) return first;
+ const split = titles.map((t) => t.split(/\s+/));
+ const shared = [];
+ for (let i = 0; i < split[0].length; i++) {
+ const word = split[0][i];
+ if (split.every((words) => words[i] === word)) shared.push(word);
+ else break;
+ }
+ const title = shared
+ .join(' ')
+ .replace(/[\s\-–—|/,:(]+$/u, '')
+ .trim();
+ return title && title.length * 2 >= Math.min(...titles.map((t) => t.length)) ? title : first;
+}
+
+/**
+ * The products and variants a source's items make.
+ *
+ * @param {import('../feeds/item.js').FeedItem[]} items
+ */
+export function catalogRecords(items) {
+ const groups = new Map();
+ for (const item of items) {
+ const key = item.itemGroupId ?? item.id;
+ const group = groups.get(key) ?? [];
+ group.push(item);
+ groups.set(key, group);
+ }
+ const products = [];
+ const variants = [];
+ for (const [key, group] of groups) {
+ const [first] = group;
+ const product = {
+ external_id: clip(key, 300),
+ title: clip(groupTitle(group.map((i) => i.title)), 1000),
+ description: clip(first.description, 20000),
+ product_brand: clip(first.brand, 300),
+ url: httpUrl(first.link),
+ product_type: clip(first.productType, 750),
+ };
+ products.push(product);
+ for (const item of group) {
+ const images = [item.imageLink, ...item.additionalImageLinks].map(httpUrl).filter(Boolean);
+ const attributes = Object.fromEntries(
+ [
+ ['color', item.color],
+ ['size', item.size],
+ ['material', item.material],
+ ]
+ .map(([name, value]) => [name, clip(value, 300)])
+ .filter(([, value]) => value),
+ );
+ variants.push({
+ product_external_id: product.external_id,
+ external_id: clip(item.id, 300),
+ title: clip(item.title, 1000),
+ gtin: clip(item.gtin, 50),
+ mpn: clip(item.mpn, 300),
+ url: httpUrl(item.link),
+ price: item.price?.amount ?? null,
+ sale_price: item.salePrice?.amount ?? null,
+ currency: item.price?.currency ?? item.salePrice?.currency ?? null,
+ availability: clip(item.availability, 50),
+ condition: clip(item.condition, 50),
+ attributes,
+ images: [...new Set(images)],
+ });
+ }
+ }
+ return { products, variants };
+}
+
+async function readAll(db, table, columns, brandId, source) {
+ const rows = [];
+ for (let start = 0; ; start += PAGE) {
+ const { data, error } = await db
+ .from(table)
+ .select(columns)
+ .eq('brand_id', brandId)
+ .eq('source', source)
+ .order('id')
+ .range(start, start + PAGE - 1);
+ if (error) throw new Error(`${table} read: ${error.message}`);
+ rows.push(...(data ?? []));
+ if (!data || data.length < PAGE) return rows;
+ }
+}
+
+function chunks(rows) {
+ const out = [];
+ for (let i = 0; i < rows.length; i += CHUNK) out.push(rows.slice(i, i + CHUNK));
+ return out;
+}
+
+async function upsertRows(db, table, rows) {
+ const ids = new Map();
+ for (const chunk of chunks(rows)) {
+ const { data, error } = await db
+ .from(table)
+ .upsert(chunk, { onConflict: 'brand_id,source,external_id' })
+ .select('id, external_id');
+ if (error) throw new Error(`${table} write: ${error.message}`);
+ for (const row of data ?? []) ids.set(row.external_id, row.id);
+ }
+ return ids;
+}
+
+async function markRemoved(db, table, ids, at) {
+ for (const chunk of chunks(ids)) {
+ const { error } = await db
+ .from(table)
+ .update({ removed_at: at, updated_at: at })
+ .in('id', chunk);
+ if (error) throw new Error(`${table} remove: ${error.message}`);
+ }
+}
+
+async function replaceIn(db, table, productIds, rows) {
+ for (const chunk of chunks(productIds)) {
+ const { error } = await db.from(table).delete().in('product_id', chunk);
+ if (error) throw new Error(`${table} clear: ${error.message}`);
+ }
+ for (const chunk of chunks(rows)) {
+ const { error } = await db.from(table).insert(chunk);
+ if (error) throw new Error(`${table} write: ${error.message}`);
+ }
+}
+
+/**
+ * Brings a brand's catalog from `source` in line with `items`, and says
+ * what changed.
+ *
+ * @param {string} brandId
+ * @param {string} source e.g. 'feed', 'csv', 'shopify'
+ * @param {import('../feeds/item.js').FeedItem[]} items
+ * @param {{ db?: object, now?: Date }} [options]
+ */
+export async function syncCatalog(
+ brandId,
+ source,
+ items,
+ { db = supabaseAdmin, now = new Date() } = {},
+) {
+ const at = now.toISOString();
+ const { products, variants } = catalogRecords(items);
+ const counts = {
+ products: { added: 0, changed: 0, removed: 0, unchanged: 0 },
+ variants: { added: 0, changed: 0, removed: 0, unchanged: 0 },
+ };
+
+ // Products: write the new, the changed and the returned.
+ const keptProducts = await readAll(
+ db,
+ 'catalog_products',
+ 'id, external_id, content_hash, removed_at',
+ brandId,
+ source,
+ );
+ const productBy = new Map(keptProducts.map((row) => [row.external_id, row]));
+ const productIds = new Map(keptProducts.map((row) => [row.external_id, row.id]));
+ const productWrites = [];
+ for (const product of products) {
+ const content_hash = hashOf(product);
+ const kept = productBy.get(product.external_id);
+ if (kept && kept.content_hash === content_hash && !kept.removed_at) {
+ counts.products.unchanged++;
+ continue;
+ }
+ counts.products[kept ? 'changed' : 'added']++;
+ productWrites.push({
+ brand_id: brandId,
+ source,
+ ...product,
+ content_hash,
+ removed_at: null,
+ updated_at: at,
+ });
+ }
+ for (const [externalId, id] of await upsertRows(db, 'catalog_products', productWrites)) {
+ productIds.set(externalId, id);
+ }
+ const seenProducts = new Set(products.map((p) => p.external_id));
+ const goneProducts = keptProducts.filter(
+ (r) => !seenProducts.has(r.external_id) && !r.removed_at,
+ );
+ counts.products.removed = goneProducts.length;
+ await markRemoved(
+ db,
+ 'catalog_products',
+ goneProducts.map((r) => r.id),
+ at,
+ );
+
+ // Variants, under their products.
+ const keptVariants = await readAll(
+ db,
+ 'catalog_variants',
+ 'id, external_id, content_hash, removed_at',
+ brandId,
+ source,
+ );
+ const variantBy = new Map(keptVariants.map((row) => [row.external_id, row]));
+ const variantWrites = [];
+ const touched = new Set(productWrites.map((p) => p.external_id));
+ for (const variant of variants) {
+ const content_hash = hashOf(variant);
+ const kept = variantBy.get(variant.external_id);
+ if (kept && kept.content_hash === content_hash && !kept.removed_at) {
+ counts.variants.unchanged++;
+ continue;
+ }
+ counts.variants[kept ? 'changed' : 'added']++;
+ touched.add(variant.product_external_id);
+ const { product_external_id, images: _images, ...columns } = variant;
+ variantWrites.push({
+ brand_id: brandId,
+ source,
+ product_id: productIds.get(product_external_id),
+ ...columns,
+ content_hash,
+ removed_at: null,
+ updated_at: at,
+ });
+ }
+ const variantIds = new Map(keptVariants.map((row) => [row.external_id, row.id]));
+ for (const [externalId, id] of await upsertRows(db, 'catalog_variants', variantWrites)) {
+ variantIds.set(externalId, id);
+ }
+ const seenVariants = new Set(variants.map((v) => v.external_id));
+ const goneVariants = keptVariants.filter(
+ (r) => !seenVariants.has(r.external_id) && !r.removed_at,
+ );
+ counts.variants.removed = goneVariants.length;
+ await markRemoved(
+ db,
+ 'catalog_variants',
+ goneVariants.map((r) => r.id),
+ at,
+ );
+
+ // A touched product's images and categories, written again.
+ const touchedIds = [...touched].map((key) => productIds.get(key)).filter(Boolean);
+ if (touchedIds.length) {
+ const images = variants
+ .filter((v) => touched.has(v.product_external_id))
+ .flatMap((v) =>
+ v.images.map((url, position) => ({
+ brand_id: brandId,
+ product_id: productIds.get(v.product_external_id),
+ variant_id: variantIds.get(v.external_id) ?? null,
+ url,
+ position,
+ })),
+ );
+ await replaceIn(db, 'catalog_images', touchedIds, images);
+
+ const names = [
+ ...new Set(
+ products
+ .filter((p) => touched.has(p.external_id) && p.product_type)
+ .map((p) => p.product_type),
+ ),
+ ];
+ const categoryIds = new Map();
+ for (const chunk of chunks(names.map((name) => ({ brand_id: brandId, name })))) {
+ const { data, error } = await db
+ .from('catalog_categories')
+ .upsert(chunk, { onConflict: 'brand_id,name' })
+ .select('id, name');
+ if (error) throw new Error(`catalog_categories write: ${error.message}`);
+ for (const row of data ?? []) categoryIds.set(row.name, row.id);
+ }
+ const links = products
+ .filter((p) => touched.has(p.external_id) && p.product_type)
+ .map((p) => ({
+ brand_id: brandId,
+ product_id: productIds.get(p.external_id),
+ category_id: categoryIds.get(p.product_type),
+ }))
+ .filter((link) => link.product_id && link.category_id);
+ await replaceIn(db, 'catalog_product_categories', touchedIds, links);
+ }
+
+ return counts;
+}
diff --git a/server/src/lib/catalog/sync.test.js b/server/src/lib/catalog/sync.test.js
new file mode 100644
index 000000000..e248befef
--- /dev/null
+++ b/server/src/lib/catalog/sync.test.js
@@ -0,0 +1,298 @@
+import { randomUUID } from 'node:crypto';
+import { readFileSync } from 'node:fs';
+import { beforeEach, describe, expect, it, vi } from 'vitest';
+
+// The module reaches the supabase admin client at import time, whose config
+// hard-exits when SUPABASE_* env is absent (as in CI).
+vi.mock('../../config/supabase.js', () => ({ default: {} }));
+
+import { parseCsvFeed, parseGoogleFeedXml } from '../feeds/index.js';
+import { catalogRecords, groupTitle, syncCatalog } from './sync.js';
+
+const BRAND = 'brand-1';
+const feed = () =>
+ parseGoogleFeedXml(
+ readFileSync(new URL('../feeds/__fixtures__/google-feed.xml', import.meta.url), 'utf8'),
+ ).items;
+
+/**
+ * An in-memory stand-in for the tables the sync writes, with PostgREST's
+ * calls as the sync makes them, counting every write.
+ */
+function fakeDb() {
+ const tables = new Map();
+ const rows = (name) => {
+ if (!tables.has(name)) tables.set(name, []);
+ return tables.get(name);
+ };
+ const writes = [];
+ const uniqueKey = (table, row) =>
+ table === 'catalog_categories'
+ ? `${row.brand_id}|${row.name}`
+ : `${row.brand_id}|${row.source}|${row.external_id}`;
+ // The unique keys' rows, by table, as the database's unique indexes find them.
+ const byKey = new Map();
+ const keyed = (table) => {
+ if (!byKey.has(table)) byKey.set(table, new Map());
+ return byKey.get(table);
+ };
+
+ function from(table) {
+ const state = { filters: [], range: null, op: 'select', payload: null, returning: null };
+ const run = () => {
+ const all = rows(table);
+ const match = (row) =>
+ state.filters.every(([kind, column, value]) =>
+ kind === 'eq' ? row[column] === value : value.includes(row[column]),
+ );
+ if (state.op === 'select') {
+ const found = all.filter(match).sort((a, b) => (a.id < b.id ? -1 : 1));
+ const [start, end] = state.range ?? [0, found.length];
+ return { data: found.slice(start, end + 1), error: null };
+ }
+ writes.push({
+ table,
+ op: state.op,
+ count: Array.isArray(state.payload) ? state.payload.length : 1,
+ });
+ if (state.op === 'upsert') {
+ const out = [];
+ for (const values of state.payload) {
+ const key = uniqueKey(table, values);
+ let row = keyed(table).get(key);
+ if (row) Object.assign(row, values);
+ else {
+ row = { id: randomUUID(), removed_at: null, ...values };
+ all.push(row);
+ keyed(table).set(key, row);
+ }
+ out.push(row);
+ }
+ return { data: out, error: null };
+ }
+ if (state.op === 'insert') {
+ for (const values of state.payload) all.push({ id: randomUUID(), ...values });
+ return { data: null, error: null };
+ }
+ if (state.op === 'update') {
+ for (const row of all.filter(match)) Object.assign(row, state.payload);
+ return { data: null, error: null };
+ }
+ if (state.op === 'delete') {
+ tables.set(
+ table,
+ all.filter((row) => !match(row)),
+ );
+ return { data: null, error: null };
+ }
+ throw new Error(`unexpected ${state.op}`);
+ };
+ const builder = {
+ select: () => builder,
+ eq: (column, value) => {
+ state.filters.push(['eq', column, value]);
+ return builder;
+ },
+ in: (column, values) => {
+ state.filters.push(['in', column, values]);
+ return builder;
+ },
+ order: () => builder,
+ range: (start, end) => {
+ state.range = [start, end];
+ return builder;
+ },
+ upsert: (payload) => {
+ state.op = 'upsert';
+ state.payload = payload;
+ return builder;
+ },
+ insert: (payload) => {
+ state.op = 'insert';
+ state.payload = payload;
+ return builder;
+ },
+ update: (payload) => {
+ state.op = 'update';
+ state.payload = payload;
+ return builder;
+ },
+ delete: () => {
+ state.op = 'delete';
+ return builder;
+ },
+ then: (onFulfilled, onRejected) => Promise.resolve(run()).then(onFulfilled, onRejected),
+ };
+ return builder;
+ }
+ return { db: { from }, rows, writes };
+}
+
+let fake;
+beforeEach(() => {
+ fake = fakeDb();
+});
+
+describe('groupTitle', () => {
+ it("takes the words an item group's titles start with, without the separator", () => {
+ expect(
+ groupTitle([
+ 'Trailrunner Hiking Boot - Red',
+ 'Trailrunner Hiking Boot - Blue',
+ 'Trailrunner Hiking Boot - Green',
+ ]),
+ ).toBe('Trailrunner Hiking Boot');
+ expect(groupTitle(['Sock'])).toBe('Sock');
+ // Too little in common: the first title.
+ expect(groupTitle(['A red boot for hiking', 'A blue sock'])).toBe('A red boot for hiking');
+ });
+});
+
+describe('catalogRecords', () => {
+ it('makes one product per item group and a variant per item', () => {
+ const { products, variants } = catalogRecords(feed());
+ expect(products.map((p) => [p.external_id, p.title])).toEqual([
+ ['TR-100', 'Trailrunner Hiking Boot'],
+ ['SOCK-1', 'Merino Hiking Sock'],
+ ]);
+ expect(variants.map((v) => [v.product_external_id, v.external_id])).toEqual([
+ ['TR-100', 'TR-100-RED'],
+ ['TR-100', 'TR-100-BLUE'],
+ ['TR-100', 'TR-100-GREEN'],
+ ['SOCK-1', 'SOCK-1'],
+ ]);
+ expect(variants[0]).toMatchObject({
+ price: 129,
+ sale_price: 99,
+ currency: 'USD',
+ availability: 'in_stock',
+ attributes: { color: 'Red', size: '42', material: 'Leather' },
+ images: [
+ 'https://shop.example/img/tr-red.jpg',
+ 'https://shop.example/img/tr-red-2.jpg',
+ 'https://shop.example/img/tr-red-3.jpg',
+ ],
+ });
+ });
+});
+
+describe('syncCatalog', () => {
+ it('writes products, variants, images and categories', async () => {
+ const counts = await syncCatalog(BRAND, 'feed', feed(), { db: fake.db });
+ expect(counts).toEqual({
+ products: { added: 2, changed: 0, removed: 0, unchanged: 0 },
+ variants: { added: 4, changed: 0, removed: 0, unchanged: 0 },
+ });
+ const products = fake.rows('catalog_products');
+ const boot = products.find((p) => p.external_id === 'TR-100');
+ expect(boot).toMatchObject({
+ brand_id: BRAND,
+ source: 'feed',
+ title: 'Trailrunner Hiking Boot',
+ });
+ const red = fake.rows('catalog_variants').find((v) => v.external_id === 'TR-100-RED');
+ expect(red.product_id).toBe(boot.id);
+ const images = fake.rows('catalog_images').filter((i) => i.variant_id === red.id);
+ expect(images.map((i) => [i.position, i.url])).toEqual([
+ [0, 'https://shop.example/img/tr-red.jpg'],
+ [1, 'https://shop.example/img/tr-red-2.jpg'],
+ [2, 'https://shop.example/img/tr-red-3.jpg'],
+ ]);
+ expect(fake.rows('catalog_categories').map((c) => c.name)).toEqual(['Shoes > Hiking Boots']);
+ expect(fake.rows('catalog_product_categories')).toEqual([
+ expect.objectContaining({ product_id: boot.id }),
+ ]);
+ });
+
+ it('writes nothing when the same feed comes again', async () => {
+ await syncCatalog(BRAND, 'feed', feed(), { db: fake.db });
+ fake.writes.length = 0;
+ const counts = await syncCatalog(BRAND, 'feed', feed(), { db: fake.db });
+ expect(fake.writes).toEqual([]);
+ expect(counts.products).toEqual({ added: 0, changed: 0, removed: 0, unchanged: 2 });
+ expect(counts.variants).toEqual({ added: 0, changed: 0, removed: 0, unchanged: 4 });
+ });
+
+ it('marks a product gone from the feed as removed, and live again when it comes back', async () => {
+ await syncCatalog(BRAND, 'feed', feed(), {
+ db: fake.db,
+ now: new Date('2026-10-20T00:00:00Z'),
+ });
+ const withoutSock = feed().filter((i) => i.id !== 'SOCK-1');
+ const counts = await syncCatalog(BRAND, 'feed', withoutSock, {
+ db: fake.db,
+ now: new Date('2026-10-21T00:00:00Z'),
+ });
+ expect(counts.products.removed).toBe(1);
+ expect(counts.variants.removed).toBe(1);
+ const sock = fake.rows('catalog_products').find((p) => p.external_id === 'SOCK-1');
+ expect(sock.removed_at).toBe('2026-10-21T00:00:00.000Z');
+ // Nothing is deleted.
+ expect(fake.rows('catalog_products')).toHaveLength(2);
+
+ const back = await syncCatalog(BRAND, 'feed', feed(), { db: fake.db });
+ expect(back.products).toMatchObject({ changed: 1, unchanged: 1 });
+ expect(sock.removed_at).toBeNull();
+ });
+
+ it("writes a changed price on its variant only, with its product's images again", async () => {
+ await syncCatalog(BRAND, 'feed', feed(), { db: fake.db });
+ const items = feed();
+ items[1] = { ...items[1], price: { amount: 119, currency: 'USD' } };
+ const counts = await syncCatalog(BRAND, 'feed', items, { db: fake.db });
+ expect(counts.variants).toMatchObject({ changed: 1, unchanged: 3 });
+ expect(counts.products).toMatchObject({ unchanged: 2 });
+ expect(fake.rows('catalog_variants').find((v) => v.external_id === 'TR-100-BLUE').price).toBe(
+ 119,
+ );
+ // The boot's images were written again, once each.
+ const bootImages = fake.rows('catalog_images').filter((i) => i.url.includes('/tr-'));
+ expect(bootImages).toHaveLength(5);
+ });
+
+ it('keeps sources and brands apart', async () => {
+ await syncCatalog(BRAND, 'feed', feed(), { db: fake.db });
+ const counts = await syncCatalog(BRAND, 'csv', [], { db: fake.db });
+ expect(counts.products.removed).toBe(0);
+ const other = await syncCatalog('brand-2', 'feed', feed(), { db: fake.db });
+ expect(other.products.added).toBe(2);
+ });
+
+ it("leaves out an address that isn't http(s), and clips long values", async () => {
+ const [item] = feed();
+ await syncCatalog(
+ BRAND,
+ 'feed',
+ [
+ {
+ ...item,
+ link: 'javascript:alert(1)',
+ imageLink: 'ftp://x/y.jpg',
+ title: 'x'.repeat(1200),
+ },
+ ],
+ { db: fake.db },
+ );
+ const [product] = fake.rows('catalog_products');
+ expect(product.url).toBeNull();
+ expect(product.title).toHaveLength(1000);
+ expect(fake.rows('catalog_images').map((i) => i.url)).not.toContain('ftp://x/y.jpg');
+ });
+
+ it('imports a 10,000-row CSV, and a second import writes nothing', async () => {
+ const lines = ['id,item_group_id,title,link,price,availability,color,product_type'];
+ for (let i = 0; i < 10_000; i++) {
+ lines.push(
+ `P-${i},G-${Math.floor(i / 4)},Item ${Math.floor(i / 4)} - ${i % 4},https://s.example/p/${i},${i}.99 USD,in stock,C${i % 4},Cat ${Math.floor(i / 4) % 10}`,
+ );
+ }
+ const { items } = parseCsvFeed(lines.join('\n'));
+ const first = await syncCatalog(BRAND, 'csv', items, { db: fake.db });
+ expect(first.products.added).toBe(2_500);
+ expect(first.variants.added).toBe(10_000);
+ expect(fake.rows('catalog_categories')).toHaveLength(10);
+ fake.writes.length = 0;
+ await syncCatalog(BRAND, 'csv', items, { db: fake.db });
+ expect(fake.writes).toEqual([]);
+ });
+});
diff --git a/supabase/migrations/00278_catalog.sql b/supabase/migrations/00278_catalog.sql
new file mode 100644
index 000000000..f297b329f
--- /dev/null
+++ b/supabase/migrations/00278_catalog.sql
@@ -0,0 +1,147 @@
+-- ============================================================================
+-- 00278: the product catalog (#1263, part 1)
+-- ============================================================================
+-- One shape every catalog source writes into: a store connection, a product
+-- feed URL, a CSV upload. Everything is a brand's:
+--
+-- * catalog_products: a product, keyed by (brand, source, external id). A
+-- feed's item group is one product; an item with no group is its own.
+-- * catalog_variants: what's sold: SKU, GTIN, MPN, a marketplace's item id
+-- (ASIN), price, availability and the options that make it (color,
+-- size, material, as `attributes`).
+-- * catalog_images: a product's or a variant's images, in order.
+-- * catalog_categories: a flat list per brand (a feed's product type, as
+-- written), and catalog_product_categories, a product's (several).
+--
+-- Only the server writes (the service role), through one batch sync
+-- (server/src/lib/catalog/sync.js) that updates rows in place and marks a
+-- product or variant gone from its source as removed (removed_at), never
+-- deleting it. `content_hash` is the sync's: a row whose content didn't
+-- change isn't written again. Members read their brands' rows.
+
+create table if not exists public.catalog_products (
+ id uuid primary key default gen_random_uuid(),
+ brand_id uuid not null references public.brands(id) on delete cascade,
+ source text not null check (source ~ '^[a-z][a-z0-9_]{1,39}$'),
+ external_id text not null check (char_length(external_id) between 1 and 300),
+ title text not null check (char_length(title) between 1 and 1000),
+ description text check (description is null or char_length(description) <= 20000),
+ product_brand text check (product_brand is null or char_length(product_brand) <= 300),
+ url text check (url is null or (char_length(url) <= 2048 and url ~ '^https?://')),
+ product_type text check (product_type is null or char_length(product_type) <= 750),
+ content_hash text not null check (content_hash ~ '^[0-9a-f]{40}$'),
+ removed_at timestamptz,
+ first_seen_at timestamptz not null default now(),
+ updated_at timestamptz not null default now(),
+ unique (brand_id, source, external_id)
+);
+
+create index if not exists catalog_products_brand_live_idx
+ on public.catalog_products (brand_id)
+ where removed_at is null;
+
+create table if not exists public.catalog_variants (
+ id uuid primary key default gen_random_uuid(),
+ brand_id uuid not null references public.brands(id) on delete cascade,
+ product_id uuid not null references public.catalog_products(id) on delete cascade,
+ source text not null check (source ~ '^[a-z][a-z0-9_]{1,39}$'),
+ external_id text not null check (char_length(external_id) between 1 and 300),
+ title text check (title is null or char_length(title) <= 1000),
+ sku text check (sku is null or char_length(sku) <= 300),
+ gtin text check (gtin is null or char_length(gtin) <= 50),
+ mpn text check (mpn is null or char_length(mpn) <= 300),
+ marketplace_item_id text check (marketplace_item_id is null or char_length(marketplace_item_id) <= 300),
+ url text check (url is null or (char_length(url) <= 2048 and url ~ '^https?://')),
+ price numeric(14, 2) check (price is null or price >= 0),
+ sale_price numeric(14, 2) check (sale_price is null or sale_price >= 0),
+ currency text check (currency is null or currency ~ '^[A-Z]{3}$'),
+ availability text check (availability is null or char_length(availability) <= 50),
+ condition text check (condition is null or char_length(condition) <= 50),
+ attributes jsonb not null default '{}'::jsonb
+ check (jsonb_typeof(attributes) = 'object' and octet_length(attributes::text) <= 4096),
+ content_hash text not null check (content_hash ~ '^[0-9a-f]{40}$'),
+ removed_at timestamptz,
+ first_seen_at timestamptz not null default now(),
+ updated_at timestamptz not null default now(),
+ unique (brand_id, source, external_id)
+);
+
+create index if not exists catalog_variants_product_idx on public.catalog_variants (product_id);
+create index if not exists catalog_variants_brand_gtin_idx
+ on public.catalog_variants (brand_id, gtin)
+ where gtin is not null;
+
+create table if not exists public.catalog_images (
+ id uuid primary key default gen_random_uuid(),
+ brand_id uuid not null references public.brands(id) on delete cascade,
+ product_id uuid not null references public.catalog_products(id) on delete cascade,
+ variant_id uuid references public.catalog_variants(id) on delete cascade,
+ url text not null check (char_length(url) <= 2048 and url ~ '^https?://'),
+ position smallint not null default 0 check (position >= 0)
+);
+
+create index if not exists catalog_images_product_idx on public.catalog_images (product_id);
+create index if not exists catalog_images_variant_idx on public.catalog_images (variant_id);
+
+create table if not exists public.catalog_categories (
+ id uuid primary key default gen_random_uuid(),
+ brand_id uuid not null references public.brands(id) on delete cascade,
+ name text not null check (char_length(name) between 1 and 750),
+ created_at timestamptz not null default now(),
+ unique (brand_id, name)
+);
+
+create table if not exists public.catalog_product_categories (
+ brand_id uuid not null references public.brands(id) on delete cascade,
+ product_id uuid not null references public.catalog_products(id) on delete cascade,
+ category_id uuid not null references public.catalog_categories(id) on delete cascade,
+ primary key (product_id, category_id)
+);
+
+create index if not exists catalog_product_categories_category_idx
+ on public.catalog_product_categories (category_id);
+
+-- Members read their brands' catalog; only the service role writes.
+do $$
+declare
+ t text;
+begin
+ foreach t in array array[
+ 'catalog_products',
+ 'catalog_variants',
+ 'catalog_images',
+ 'catalog_categories',
+ 'catalog_product_categories'
+ ]
+ loop
+ execute format('alter table public.%I enable row level security', t);
+ execute format('revoke all on table public.%I from anon, authenticated', t);
+ execute format('grant select on table public.%I to authenticated', t);
+ execute format('grant all on table public.%I to service_role', t);
+ execute format('drop policy if exists %I on public.%I', t || ': member select', t);
+ execute format(
+ 'create policy %I on public.%I for select using (
+ brand_id in (
+ select b.id from public.brands b
+ join public.profiles p on p.organization_id = b.organization_id
+ where p.id = auth.uid()
+ )
+ )',
+ t || ': member select',
+ t
+ );
+ end loop;
+end $$;
+
+comment on table public.catalog_products is
+ 'Catalog (#1263, 00278): a brand''s products from a source (store, feed, CSV), keyed by (brand, source, external id); removed_at once gone from the source. Server writes, members read.';
+comment on table public.catalog_variants is
+ 'Catalog (#1263, 00278): a product''s variants: SKU, GTIN, MPN, marketplace item id, price, availability and options (attributes). Server writes, members read.';
+comment on table public.catalog_images is
+ 'Catalog (#1263, 00278): a product''s or variant''s images, in order. Server writes, members read.';
+comment on table public.catalog_categories is
+ 'Catalog (#1263, 00278): a brand''s categories, a flat list (a feed''s product type as written). Server writes, members read.';
+comment on table public.catalog_product_categories is
+ 'Catalog (#1263, 00278): the categories a product is in. Server writes, members read.';
+
+notify pgrst, 'reload schema';
diff --git a/supabase/schema.sql b/supabase/schema.sql
index e9eccb52e..04a041f5a 100644
--- a/supabase/schema.sql
+++ b/supabase/schema.sql
@@ -39491,3 +39491,154 @@ alter table public.brands
comment on column public.brands.openai_product_feed is
'Shopping (#1281): the brand says it sends OpenAI a direct product feed, so its catalog is in OpenAI''s product index (ChatGPT Shopping, US). Off by default.';
+-- ─────────────────────────────────────────────────────────────────────────
+-- migrations/00278_catalog.sql
+-- ─────────────────────────────────────────────────────────────────────────
+-- ============================================================================
+-- 00278: the product catalog (#1263, part 1)
+-- ============================================================================
+-- One shape every catalog source writes into: a store connection, a product
+-- feed URL, a CSV upload. Everything is a brand's:
+--
+-- * catalog_products: a product, keyed by (brand, source, external id). A
+-- feed's item group is one product; an item with no group is its own.
+-- * catalog_variants: what's sold: SKU, GTIN, MPN, a marketplace's item id
+-- (ASIN), price, availability and the options that make it (color,
+-- size, material, as `attributes`).
+-- * catalog_images: a product's or a variant's images, in order.
+-- * catalog_categories: a flat list per brand (a feed's product type, as
+-- written), and catalog_product_categories, a product's (several).
+--
+-- Only the server writes (the service role), through one batch sync
+-- (server/src/lib/catalog/sync.js) that updates rows in place and marks a
+-- product or variant gone from its source as removed (removed_at), never
+-- deleting it. `content_hash` is the sync's: a row whose content didn't
+-- change isn't written again. Members read their brands' rows.
+
+create table if not exists public.catalog_products (
+ id uuid primary key default gen_random_uuid(),
+ brand_id uuid not null references public.brands(id) on delete cascade,
+ source text not null check (source ~ '^[a-z][a-z0-9_]{1,39}$'),
+ external_id text not null check (char_length(external_id) between 1 and 300),
+ title text not null check (char_length(title) between 1 and 1000),
+ description text check (description is null or char_length(description) <= 20000),
+ product_brand text check (product_brand is null or char_length(product_brand) <= 300),
+ url text check (url is null or (char_length(url) <= 2048 and url ~ '^https?://')),
+ product_type text check (product_type is null or char_length(product_type) <= 750),
+ content_hash text not null check (content_hash ~ '^[0-9a-f]{40}$'),
+ removed_at timestamptz,
+ first_seen_at timestamptz not null default now(),
+ updated_at timestamptz not null default now(),
+ unique (brand_id, source, external_id)
+);
+
+create index if not exists catalog_products_brand_live_idx
+ on public.catalog_products (brand_id)
+ where removed_at is null;
+
+create table if not exists public.catalog_variants (
+ id uuid primary key default gen_random_uuid(),
+ brand_id uuid not null references public.brands(id) on delete cascade,
+ product_id uuid not null references public.catalog_products(id) on delete cascade,
+ source text not null check (source ~ '^[a-z][a-z0-9_]{1,39}$'),
+ external_id text not null check (char_length(external_id) between 1 and 300),
+ title text check (title is null or char_length(title) <= 1000),
+ sku text check (sku is null or char_length(sku) <= 300),
+ gtin text check (gtin is null or char_length(gtin) <= 50),
+ mpn text check (mpn is null or char_length(mpn) <= 300),
+ marketplace_item_id text check (marketplace_item_id is null or char_length(marketplace_item_id) <= 300),
+ url text check (url is null or (char_length(url) <= 2048 and url ~ '^https?://')),
+ price numeric(14, 2) check (price is null or price >= 0),
+ sale_price numeric(14, 2) check (sale_price is null or sale_price >= 0),
+ currency text check (currency is null or currency ~ '^[A-Z]{3}$'),
+ availability text check (availability is null or char_length(availability) <= 50),
+ condition text check (condition is null or char_length(condition) <= 50),
+ attributes jsonb not null default '{}'::jsonb
+ check (jsonb_typeof(attributes) = 'object' and octet_length(attributes::text) <= 4096),
+ content_hash text not null check (content_hash ~ '^[0-9a-f]{40}$'),
+ removed_at timestamptz,
+ first_seen_at timestamptz not null default now(),
+ updated_at timestamptz not null default now(),
+ unique (brand_id, source, external_id)
+);
+
+create index if not exists catalog_variants_product_idx on public.catalog_variants (product_id);
+create index if not exists catalog_variants_brand_gtin_idx
+ on public.catalog_variants (brand_id, gtin)
+ where gtin is not null;
+
+create table if not exists public.catalog_images (
+ id uuid primary key default gen_random_uuid(),
+ brand_id uuid not null references public.brands(id) on delete cascade,
+ product_id uuid not null references public.catalog_products(id) on delete cascade,
+ variant_id uuid references public.catalog_variants(id) on delete cascade,
+ url text not null check (char_length(url) <= 2048 and url ~ '^https?://'),
+ position smallint not null default 0 check (position >= 0)
+);
+
+create index if not exists catalog_images_product_idx on public.catalog_images (product_id);
+create index if not exists catalog_images_variant_idx on public.catalog_images (variant_id);
+
+create table if not exists public.catalog_categories (
+ id uuid primary key default gen_random_uuid(),
+ brand_id uuid not null references public.brands(id) on delete cascade,
+ name text not null check (char_length(name) between 1 and 750),
+ created_at timestamptz not null default now(),
+ unique (brand_id, name)
+);
+
+create table if not exists public.catalog_product_categories (
+ brand_id uuid not null references public.brands(id) on delete cascade,
+ product_id uuid not null references public.catalog_products(id) on delete cascade,
+ category_id uuid not null references public.catalog_categories(id) on delete cascade,
+ primary key (product_id, category_id)
+);
+
+create index if not exists catalog_product_categories_category_idx
+ on public.catalog_product_categories (category_id);
+
+-- Members read their brands' catalog; only the service role writes.
+do $$
+declare
+ t text;
+begin
+ foreach t in array array[
+ 'catalog_products',
+ 'catalog_variants',
+ 'catalog_images',
+ 'catalog_categories',
+ 'catalog_product_categories'
+ ]
+ loop
+ execute format('alter table public.%I enable row level security', t);
+ execute format('revoke all on table public.%I from anon, authenticated', t);
+ execute format('grant select on table public.%I to authenticated', t);
+ execute format('grant all on table public.%I to service_role', t);
+ execute format('drop policy if exists %I on public.%I', t || ': member select', t);
+ execute format(
+ 'create policy %I on public.%I for select using (
+ brand_id in (
+ select b.id from public.brands b
+ join public.profiles p on p.organization_id = b.organization_id
+ where p.id = auth.uid()
+ )
+ )',
+ t || ': member select',
+ t
+ );
+ end loop;
+end $$;
+
+comment on table public.catalog_products is
+ 'Catalog (#1263, 00278): a brand''s products from a source (store, feed, CSV), keyed by (brand, source, external id); removed_at once gone from the source. Server writes, members read.';
+comment on table public.catalog_variants is
+ 'Catalog (#1263, 00278): a product''s variants: SKU, GTIN, MPN, marketplace item id, price, availability and options (attributes). Server writes, members read.';
+comment on table public.catalog_images is
+ 'Catalog (#1263, 00278): a product''s or variant''s images, in order. Server writes, members read.';
+comment on table public.catalog_categories is
+ 'Catalog (#1263, 00278): a brand''s categories, a flat list (a feed''s product type as written). Server writes, members read.';
+comment on table public.catalog_product_categories is
+ 'Catalog (#1263, 00278): the categories a product is in. Server writes, members read.';
+
+notify pgrst, 'reload schema';
+