Skip to content

Open-source agentic AI Search Execution Platform. Track AI visibility, find opportunities, and let agents turn signals into prioritized actions and drafted fixes across ChatGPT, Claude, Gemini, Google AI Overviews, Google AI Mode, Perplexity, Grok, and Microsoft Copilot.

fix(catalog): match cards right after a sync, keep the product cap under parallel syncs, and count suggestion categories in SQL (Product Visibility follow-ups, part 1) (#1665)

Cihanf906d4db180d
diff --git a/server/src/lib/catalog-suggestions.js b/server/src/lib/catalog-suggestions.js
index ba1a6422d..b612326ab 100644
--- a/server/src/lib/catalog-suggestions.js
+++ b/server/src/lib/catalog-suggestions.js
@@ -4,22 +4,20 @@
* Shopping questions start from what the brand sells: its catalog's
* categories (00278), each with how many live products it has and their
* price band, so the model can ask what a shopper asks an assistant before
- * buying there ("best hiking backpacks under $150"). Everything up to the
+ * buying there ("best hiking backpacks under $150"). The categories are
+ * counted in the database (catalog_suggestion_categories, 00371), which
+ * returns only those offered, however big the catalog. Everything up to the
* LLM handoff is deterministic. Any failure returns [] so the base
- * suggestion flow never degrades; brands without a catalog short-circuit on
- * the first (empty) read.
+ * suggestion flow never degrades; a brand without a catalog gets no rows.
*/
import supabaseAdmin from '../config/supabase.js';
-import { readAllRpcRows } from './rpc-pages.js';
import { logger } from './logger.js';
/** Categories offered to the model at most, those with the most products first. */
export const MAX_CANDIDATES = 8;
/** A category with fewer live products than this isn't offered. */
-const MIN_PRODUCTS = 2;
-/** Prices read for a band: the share of the category's products under its "under" price. */
-const UNDER_SHARE = 0.75;
+export const MIN_PRODUCTS = 2;
/** A price rounded up to one a shopper would name: 40, 150, 2,500. */
export function roundPrice(value) {
@@ -32,58 +30,26 @@ export function roundPrice(value) {
}
/**
- * The categories as candidates: each one's products (live, priced or not)
- * and, when they share a currency, their lowest and highest price and the
- * round price most of them sell under.
+ * The database's categories as candidates, in its order: each one's live
+ * products and, when its priced products share a currency, their lowest
+ * and highest price and the round price most of them sell under (the 75th
+ * percentile, rounded up).
*
- * @param {{ categories: { id: string, name: string }[], links: { product_id: string, category_id: string }[], variants: { product_id: string, price: number|null, sale_price: number|null, currency: string|null }[] }} catalog
+ * @param {{ category: string, products: number, price_low: number|string|null, price_high: number|string|null, price_p75: number|string|null, currency: string|null }[]} rows
*/
-export function composeCatalogCandidates({ categories, links, variants }) {
- // Each product's price: its lowest variant's, on sale or not.
- const priceOf = new Map();
- for (const v of variants) {
- const amount = Number(v.sale_price ?? v.price);
- if (!(amount > 0)) continue;
- const kept = priceOf.get(v.product_id);
- if (!kept || amount < kept.amount) {
- priceOf.set(v.product_id, { amount, currency: v.currency ?? null });
- }
- }
- const productsOf = new Map();
- for (const link of links) {
- if (!productsOf.has(link.category_id)) productsOf.set(link.category_id, new Set());
- productsOf.get(link.category_id).add(link.product_id);
- }
- const candidates = [];
- for (const category of categories) {
- const products = [...(productsOf.get(category.id) ?? [])];
- if (products.length < MIN_PRODUCTS) continue;
- const prices = products.map((id) => priceOf.get(id)).filter(Boolean);
- const currencies = new Set(prices.map((p) => p.currency));
- const band =
- prices.length && currencies.size === 1 && [...currencies][0]
- ? (() => {
- const amounts = prices.map((p) => p.amount).sort((a, b) => a - b);
- const at = Math.min(amounts.length - 1, Math.ceil(amounts.length * UNDER_SHARE) - 1);
- return {
- priceLow: amounts[0],
- priceHigh: amounts.at(-1),
- priceUnder: roundPrice(amounts[at]),
- currency: [...currencies][0],
- };
- })()
- : { priceLow: null, priceHigh: null, priceUnder: null, currency: null };
- candidates.push({ category: category.name, products: products.length, ...band });
- }
- return candidates
- .sort((a, b) => b.products - a.products || a.category.localeCompare(b.category))
- .slice(0, MAX_CANDIDATES);
-}
-
-async function readAll(what, page) {
- const { data, error } = await readAllRpcRows(what, page);
- if (error) throw new Error(`${what} read: ${error.message}`);
- return data;
+export function catalogCandidates(rows) {
+ return rows.slice(0, MAX_CANDIDATES).map((row) => {
+ const under = roundPrice(Number(row.price_p75));
+ const band = Boolean(row.currency) && under !== null;
+ return {
+ category: row.category,
+ products: Number(row.products),
+ priceLow: band ? Number(row.price_low) : null,
+ priceHigh: band ? Number(row.price_high) : null,
+ priceUnder: band ? under : null,
+ currency: band ? row.currency : null,
+ };
+ });
}
/**
@@ -92,50 +58,13 @@ async function readAll(what, page) {
*/
export async function getCatalogSuggestionCandidates(brandId, { db = supabaseAdmin } = {}) {
try {
- const { data: categories, error } = await db
- .from('catalog_categories')
- .select('id, name')
- .eq('brand_id', brandId)
- .order('id')
- .limit(1000);
- if (error) throw new Error(error.message);
- if (!categories?.length) return [];
- const [products, links, variants] = await Promise.all([
- readAll('catalog_products', (from, to) =>
- db
- .from('catalog_products')
- .select('id')
- .eq('brand_id', brandId)
- .is('removed_at', null)
- .order('id')
- .range(from, to),
- ),
- readAll('catalog_product_categories', (from, to) =>
- db
- .from('catalog_product_categories')
- .select('product_id, category_id')
- .eq('brand_id', brandId)
- .order('product_id')
- .order('category_id')
- .range(from, to),
- ),
- readAll('catalog_variants', (from, to) =>
- db
- .from('catalog_variants')
- .select('product_id, price, sale_price, currency')
- .eq('brand_id', brandId)
- .is('removed_at', null)
- .order('id')
- .range(from, to),
- ),
- ]);
- // A removed product's links stay; only the live ones count.
- const live = new Set(products.map((p) => p.id));
- return composeCatalogCandidates({
- categories,
- links: links.filter((link) => live.has(link.product_id)),
- variants,
+ const { data, error } = await db.rpc('catalog_suggestion_categories', {
+ p_brand_id: brandId,
+ p_limit: MAX_CANDIDATES,
+ p_min_products: MIN_PRODUCTS,
});
+ if (error) throw new Error(error.message);
+ return catalogCandidates(data ?? []);
} catch (err) {
logger.error({ err, brandId }, '[catalog-suggestions] candidate read failed');
return [];
diff --git a/server/src/lib/catalog-suggestions.test.js b/server/src/lib/catalog-suggestions.test.js
index 8a946ef53..459a4287a 100644
--- a/server/src/lib/catalog-suggestions.test.js
+++ b/server/src/lib/catalog-suggestions.test.js
@@ -1,13 +1,14 @@
+import { readFileSync } from 'node:fs';
import { describe, expect, it, vi } from 'vitest';
// The module reaches the supabase admin client at import time, whose config
// hard-exits when SUPABASE_* env is absent (as in CI).
vi.mock('../config/supabase.js', () => ({ default: {} }));
-import { fakeDb } from './catalog/__fixtures__/fake-db.js';
import {
MAX_CANDIDATES,
- composeCatalogCandidates,
+ MIN_PRODUCTS,
+ catalogCandidates,
getCatalogSuggestionCandidates,
roundPrice,
} from './catalog-suggestions.js';
@@ -24,40 +25,38 @@ describe('roundPrice', () => {
});
});
-describe('composeCatalogCandidates', () => {
- const categories = [
- { id: 'packs', name: 'Bags > Hiking backpacks' },
- { id: 'socks', name: 'Socks' },
- { id: 'mixed', name: 'Gift cards' },
- { id: 'one', name: 'Tents' },
- ];
- const links = [
- ...['p1', 'p2', 'p3', 'p4'].map((product_id) => ({ product_id, category_id: 'packs' })),
- ...['s1', 's2'].map((product_id) => ({ product_id, category_id: 'socks' })),
- ...['g1', 'g2'].map((product_id) => ({ product_id, category_id: 'mixed' })),
- { product_id: 't1', category_id: 'one' },
- ];
- const variant = (product_id, price, extra = {}) => ({
- product_id,
- price,
- sale_price: null,
- currency: 'USD',
- ...extra,
- });
- const variants = [
- variant('p1', 60),
- variant('p1', 80),
- variant('p2', 119),
- variant('p3', 140, { sale_price: 129 }),
- variant('p4', 180),
- // Socks without prices; gift cards in two currencies.
- variant('s1', null),
- variant('g1', 25),
- variant('g2', 25, { currency: 'EUR' }),
- ];
-
- it('offers each category with its products and price band, the biggest first', () => {
- expect(composeCatalogCandidates({ categories, links, variants })).toEqual([
+describe('catalogCandidates', () => {
+ it("offers the database's categories in its order, with a band where they share a currency", () => {
+ expect(
+ catalogCandidates([
+ {
+ category: 'Bags > Hiking backpacks',
+ products: 4,
+ // PostgREST sends numeric as a number, or as a string when told to.
+ price_low: '60.00',
+ price_high: 180,
+ price_p75: 140,
+ currency: 'USD',
+ },
+ // Two currencies, or none priced: no band.
+ {
+ category: 'Gift cards',
+ products: 2,
+ price_low: null,
+ price_high: null,
+ price_p75: null,
+ currency: null,
+ },
+ {
+ category: 'Socks',
+ products: 2,
+ price_low: null,
+ price_high: null,
+ price_p75: null,
+ currency: 'USD',
+ },
+ ]),
+ ).toEqual([
{
category: 'Bags > Hiking backpacks',
products: 4,
@@ -86,54 +85,35 @@ describe('composeCatalogCandidates', () => {
});
it(`offers ${MAX_CANDIDATES} categories at most`, () => {
- const many = Array.from({ length: 12 }, (_, i) => ({ id: `c${i}`, name: `Cat ${i}` }));
- const manyLinks = many.flatMap((c) =>
- ['a', 'b'].map((p) => ({ product_id: `${c.id}${p}`, category_id: c.id })),
- );
- expect(
- composeCatalogCandidates({ categories: many, links: manyLinks, variants: [] }),
- ).toHaveLength(MAX_CANDIDATES);
+ const rows = Array.from({ length: 12 }, (_, i) => ({
+ category: `Cat ${i}`,
+ products: 2,
+ price_low: null,
+ price_high: null,
+ price_p75: null,
+ currency: null,
+ }));
+ expect(catalogCandidates(rows)).toHaveLength(MAX_CANDIDATES);
});
});
describe('getCatalogSuggestionCandidates', () => {
- it("reads the brand's live catalog, and nothing for a brand without one", async () => {
- const fake = fakeDb();
- expect(await getCatalogSuggestionCandidates('brand-1', { db: fake.db })).toEqual([]);
- fake.rows('catalog_categories').push({ id: 'c1', brand_id: 'brand-1', name: 'Boots' });
- fake
- .rows('catalog_products')
- .push(
- { id: 'p1', brand_id: 'brand-1', removed_at: null },
- { id: 'p2', brand_id: 'brand-1', removed_at: null },
- { id: 'p3', brand_id: 'brand-1', removed_at: '2026-10-01T00:00:00Z' },
- );
- fake.rows('catalog_product_categories').push(
- ...['p1', 'p2', 'p3'].map((product_id) => ({
- brand_id: 'brand-1',
- product_id,
- category_id: 'c1',
- })),
- );
- fake.rows('catalog_variants').push(
- {
- id: 'v1',
- brand_id: 'brand-1',
- product_id: 'p1',
- price: 99,
- currency: 'USD',
- removed_at: null,
- },
- {
- id: 'v2',
- brand_id: 'brand-1',
- product_id: 'p2',
- price: 129,
- currency: 'USD',
- removed_at: null,
- },
- );
- expect(await getCatalogSuggestionCandidates('brand-1', { db: fake.db })).toEqual([
+ it('asks the database for the categories offered only, not the whole catalog', async () => {
+ const from = vi.fn();
+ const rpc = vi.fn(async () => ({
+ data: [
+ {
+ category: 'Boots',
+ products: 2,
+ price_low: 99,
+ price_high: 129,
+ price_p75: 129,
+ currency: 'USD',
+ },
+ ],
+ error: null,
+ }));
+ expect(await getCatalogSuggestionCandidates('brand-1', { db: { from, rpc } })).toEqual([
{
category: 'Boots',
products: 2,
@@ -143,16 +123,51 @@ describe('getCatalogSuggestionCandidates', () => {
currency: 'USD',
},
]);
+ expect(rpc).toHaveBeenCalledExactlyOnceWith('catalog_suggestion_categories', {
+ p_brand_id: 'brand-1',
+ p_limit: MAX_CANDIDATES,
+ p_min_products: MIN_PRODUCTS,
+ });
+ expect(from).not.toHaveBeenCalled();
});
- it('returns nothing when the catalog cannot be read', async () => {
- const db = {
- from: () => ({
- select: () => {
- throw new Error('down');
- },
- }),
+ it('returns nothing for a brand without a catalog, or when it cannot be read', async () => {
+ const empty = { rpc: async () => ({ data: [], error: null }) };
+ expect(await getCatalogSuggestionCandidates('brand-1', { db: empty })).toEqual([]);
+ const missing = {
+ rpc: async () => ({ data: null, error: { message: 'Could not find the function' } }),
+ };
+ expect(await getCatalogSuggestionCandidates('brand-1', { db: missing })).toEqual([]);
+ const down = {
+ rpc: () => {
+ throw new Error('down');
+ },
};
- expect(await getCatalogSuggestionCandidates('brand-1', { db })).toEqual([]);
+ expect(await getCatalogSuggestionCandidates('brand-1', { db: down })).toEqual([]);
+ });
+});
+
+describe('catalog_suggestion_categories (00371)', () => {
+ const sql = readFileSync(
+ new URL(
+ '../../../supabase/migrations/00371_catalog_suggestion_categories.sql',
+ import.meta.url,
+ ),
+ 'utf8',
+ );
+
+ it('prices a live product by its lowest live variant, sale price first, for the service role', () => {
+ // The brand's live products, as catalog_products_brand_live_idx (00278) reads them.
+ expect(sql).toMatch(
+ /join public\.catalog_products p\s+on p\.id = pc\.product_id and p\.brand_id = p_brand_id and p\.removed_at is null/,
+ );
+ // Its live variants, as catalog_variants_brand_live_idx (00281) reads them.
+ expect(sql).toMatch(/where v\.brand_id = p_brand_id\s+and v\.removed_at is null/);
+ expect(sql).toMatch(/order by v\.product_id, coalesce\(v\.sale_price, v\.price\), v\.id/);
+ // The "under" price: three quarters of the products at or under it.
+ expect(sql).toMatch(/percentile_disc\(0\.75\) within group \(order by pr\.amount\)/);
+ expect(sql).toMatch(
+ /revoke all on function public\.catalog_suggestion_categories\(uuid, integer, integer\)\s+from public, anon, authenticated;/,
+ );
});
});
diff --git a/server/src/lib/catalog/card-match.js b/server/src/lib/catalog/card-match.js
index 6aaa42254..bd0e24f05 100644
--- a/server/src/lib/catalog/card-match.js
+++ b/server/src/lib/catalog/card-match.js
@@ -26,7 +26,8 @@ import { readAllRpcRows } from '../rpc-pages.js';
* A key two products share names neither. Cards are matched as they're
* written (cloro-result-handler.js), against a catalog read at most every
* INDEX_TTL_MS, and all of a brand's recent cards again after its catalog
- * syncs (sweepShoppingCardMatches), a slice at a time.
+ * syncs, a slice at a time: by the daily sweep (sweepShoppingCardMatches),
+ * and right after a sync the person asked for (matchBrandLater).
*/
/** The share of two titles' words in common (Dice) a title match needs… */
@@ -441,6 +442,47 @@ export async function matchBrandShoppingCards(
return { cards: cards.length, matched, changed };
}
+/** Brands being matched by matchBrandLater: brandId → { again, done }. */
+const later = new Map();
+
+/**
+ * A brand's recent cards matched to its catalog after a sync the person
+ * asked for (a store or an account connected, a sync from Settings, a file
+ * uploaded), without waiting on it: the daily sweep matches them only the
+ * next day, and until then the Shelf counts none of the brand's products on
+ * it. A brand being matched already is matched once more after, against its
+ * catalog as it is then. A failure is logged, never thrown.
+ *
+ * @returns {Promise<void>} settled once the brand is matched (the tests wait on it)
+ */
+export function matchBrandLater(brandId, { match = matchBrandShoppingCards } = {}) {
+ const running = later.get(brandId);
+ if (running) {
+ running.again = true;
+ return running.done;
+ }
+ const entry = { again: true, done: null };
+ later.set(brandId, entry);
+ entry.done = (async () => {
+ try {
+ while (entry.again) {
+ entry.again = false;
+ try {
+ await match(brandId);
+ } catch (err) {
+ logger.error(
+ { err: { name: err?.name, message: err?.message }, brandId },
+ '[card-match] match after sync failed',
+ );
+ }
+ }
+ } finally {
+ later.delete(brandId);
+ }
+ })();
+ return entry.done;
+}
+
/** Catalogs read again within this long are matched again by the daily sweep. */
const SWEEP_SYNCED_WITHIN_MS = 26 * 60 * 60 * 1000;
diff --git a/server/src/lib/catalog/card-match.test.js b/server/src/lib/catalog/card-match.test.js
index 9de9ac558..c5bf80a32 100644
--- a/server/src/lib/catalog/card-match.test.js
+++ b/server/src/lib/catalog/card-match.test.js
@@ -17,6 +17,7 @@ import {
catalogIndex,
catalogIndexCache,
gtinKey,
+ matchBrandLater,
matchBrandShoppingCards,
matchCard,
matchCards,
@@ -429,6 +430,52 @@ describe('matchBrandShoppingCards', () => {
});
});
+describe('matchBrandLater', () => {
+ it('matches a brand once more after the match running, not once per sync', async () => {
+ const runs = [];
+ let finish;
+ const match = vi.fn(
+ (brandId) =>
+ new Promise((resolve) => {
+ runs.push(brandId);
+ finish = resolve;
+ }),
+ );
+ const first = matchBrandLater('b1', { match });
+ // Two syncs while the first match runs: one more match after it.
+ const second = matchBrandLater('b1', { match });
+ matchBrandLater('b1', { match });
+ expect(second).toBe(first);
+ expect(runs).toEqual(['b1']);
+ finish();
+ await vi.waitFor(() => expect(runs).toEqual(['b1', 'b1']));
+ finish();
+ await first;
+ expect(match).toHaveBeenCalledTimes(2);
+ // Done: the next sync matches again.
+ const next = matchBrandLater('b1', { match });
+ expect(next).not.toBe(first);
+ finish();
+ await next;
+ expect(match).toHaveBeenCalledTimes(3);
+ });
+
+ it('logs a failed match and never throws', async () => {
+ logger.error.mockClear();
+ const match = vi.fn(async () => {
+ throw new Error('catalog_products read: down');
+ });
+ await expect(matchBrandLater('b2', { match })).resolves.toBeUndefined();
+ expect(logger.error).toHaveBeenCalledWith(
+ { err: { name: 'Error', message: 'catalog_products read: down' }, brandId: 'b2' },
+ '[card-match] match after sync failed',
+ );
+ // A brand whose match failed is matched again on its next sync.
+ await matchBrandLater('b2', { match });
+ expect(match).toHaveBeenCalledTimes(2);
+ });
+});
+
describe('sweepShoppingCardMatches', () => {
it('matches each brand whose catalog synced in the last day once, and goes on past a failure', async () => {
const fake = fakeDb();
diff --git a/server/src/lib/catalog/sources.js b/server/src/lib/catalog/sources.js
index 1d547ec45..61a6db3f1 100644
--- a/server/src/lib/catalog/sources.js
+++ b/server/src/lib/catalog/sources.js
@@ -23,7 +23,9 @@ import { syncCatalog } from './sync.js';
* (sign-in.js), and the issues it reports per item are kept (issues.js).
* On the managed cloud, the plan caps the workspace's live products
* (ECOM-019): a source keeps what's left of it, and records how many it
- * couldn't.
+ * couldn't. A capped workspace's sources write one at a time
+ * (writeWithinCap), so two read at once can't both count the room the other
+ * takes.
*/
/** Feeds and stores read again after this long, by the daily run. */
@@ -36,9 +38,19 @@ export const SWEEP_DEADLINE_MS = 10 * 60 * 1000;
const ERRORS_KEPT = 5;
/** A store's read stops calling it after this: one slow store can't hold the daily run. */
export const STORE_READ_MS = 10 * 60 * 1000;
+/**
+ * A capped write waits this long at most for the one before it in its
+ * workspace's line (inLine): a write stuck past it can't hold the line.
+ */
+export const LINE_WAIT_MS = 5 * 60 * 1000;
/** The sources being read in this process: a source isn't read twice at once. */
const reading = new Set();
+/**
+ * Each workspace's line of capped writes in this process (writeWithinCap):
+ * the last write's promise, settled either way, by organization id.
+ */
+const capLines = new Map();
/** The catalog's `source` for a catalog source: `src_` and its id without dashes. */
export function sourceKey(id) {
@@ -172,6 +184,88 @@ export async function productRoom(
return { room: Math.max(0, cap - (count ?? 0)), cap };
}
+/**
+ * `work` run after the work in `key`'s line before it, alone, and its result.
+ * The line is dropped once it's empty. Work waits `waitMs` at most: if the
+ * work before it is still running then, it runs anyway (and the work after
+ * it waits for it), so one stuck write doesn't hold its workspace's line.
+ */
+export function inLine(key, work, lines = capLines, waitMs = LINE_WAIT_MS) {
+ const before = lines.get(key);
+ const waited = before ? waitAtMost(before, waitMs, key) : Promise.resolve();
+ const run = waited.then(() => work());
+ const tail = run.then(
+ () => undefined,
+ () => undefined,
+ );
+ lines.set(key, tail);
+ tail.then(() => {
+ if (lines.get(key) === tail) lines.delete(key);
+ });
+ return run;
+}
+
+/** `before` settled, or `ms` gone by first: said in the log, as the line goes on without it. */
+function waitAtMost(before, ms, key) {
+ let timer;
+ const late = new Promise((resolve) => {
+ timer = setTimeout(() => {
+ logger.warn({ line: key, waitMs: ms }, '[catalog] capped write stopped waiting in line');
+ resolve();
+ }, ms);
+ timer.unref?.();
+ });
+ return Promise.race([before, late]).finally(() => clearTimeout(timer));
+}
+
+/**
+ * Writes a source's items into its brand's catalog within the workspace's
+ * cap (ECOM-019), and says what changed and the cap (-1 for none).
+ *
+ * On the managed cloud, a workspace's sources write one at a time: the room
+ * a source has is its workspace's cap less the live products of its other
+ * sources, and two sources counting that at once (a sync from Settings
+ * while the daily run reads another store) would each count the other's
+ * products as they were before either wrote, and keep more than the cap
+ * between them. Only the count and the writes wait in line; the read of a
+ * store or a feed, the long part, doesn't. A workspace whose plan has no cap
+ * doesn't wait at all; one that waits recounts its room when its turn comes,
+ * and waits LINE_WAIT_MS at most. The line is this process's (as `reading`
+ * is): the API and the daily run both sync here.
+ *
+ * Products past the cap are marked removed (removed_at), as if gone from
+ * the source, and come back live when there's room again. A product marked
+ * so stays marked from the day it went over: the daily purge of products
+ * removed for more than 90 days (#1632) deletes it then, with its variants,
+ * images and category links, and its cards' match is cleared (00360, on
+ * delete set null). If there's room again later, the next sync adds it back
+ * as a new row, and the daily card match finds its cards again (90 days).
+ * Telling these apart from products gone from the source would take a
+ * column of its own (a removed reason), not added here.
+ */
+async function writeWithinCap(source, items, { db, now, capOf = getCatalogProductCap }) {
+ const write = async (known) => {
+ const { room, cap } = known ?? (await productRoom(source, { db, capOf }));
+ const counts = await syncCatalog(source.brand_id, sourceKey(source.id), items, {
+ db,
+ now,
+ room,
+ });
+ return { counts, cap };
+ };
+ const uncapped = { room: Infinity, cap: -1 };
+ if (!isCloud()) return write(uncapped);
+ const { data: brand, error } = await db
+ .from('brands')
+ .select('organization_id')
+ .eq('id', source.brand_id)
+ .maybeSingle();
+ if (error) throw new Error(`brands read: ${error.message}`);
+ // Nothing to count for a plan without a cap: its sources write at once.
+ if ((await capOf(brand?.organization_id ?? null)) < 0) return write(uncapped);
+ return inLine(brand?.organization_id ?? `brand:${source.brand_id}`, () => write());
+}
+
async function record(db, id, at, outcome) {
const { error } = await db
.from('catalog_sources')
@@ -277,12 +371,7 @@ async function syncOnce(
}
}
if (!outcome) {
- const { room, cap } = await productRoom(source, { db, capOf });
- const counts = await syncCatalog(source.brand_id, sourceKey(source.id), parsed.items, {
- db,
- now,
- room,
- });
+ const { counts, cap } = await writeWithinCap(source, parsed.items, { db, now, capOf });
outcome = {
status: 'ok',
counts: {
diff --git a/server/src/lib/catalog/sources.test.js b/server/src/lib/catalog/sources.test.js
index b5389e3bb..29e86cb04 100644
--- a/server/src/lib/catalog/sources.test.js
+++ b/server/src/lib/catalog/sources.test.js
@@ -1,5 +1,6 @@
import { randomBytes } from 'node:crypto';
import { readFileSync } from 'node:fs';
+import { setImmediate } from 'node:timers';
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
// The module reaches the supabase admin client at import time, whose config
@@ -21,6 +22,7 @@ import feedUrl, { FEED_MAX_BYTES, FEED_TYPES } from '../../connectors/feed-url/i
import { PlanLimitError } from '../plan-guard.js';
import {
failureOf,
+ inLine,
productRoom,
redact,
sourceKey,
@@ -647,6 +649,65 @@ describe("the plan's product cap (ECOM-019)", () => {
}
});
+ it("keeps a workspace's cap when two of its sources sync at once", async () => {
+ process.env.IS_CLOUD = 'true';
+ // A cap of 4, less brand-2's live product (p1): room for 3 between the two.
+ getCatalogProductCap.mockResolvedValue(4);
+ const second = { ...source, id: '7c8d6a1e-2a3c-4d5e-8f90-123456789abc', brand_id: 'brand-2' };
+ fake.rows('catalog_sources').push({ ...second, last_synced_at: null });
+ const fetch = vi.fn().mockResolvedValue({ html: fixture('google-feed.xml') });
+ const [one, two] = await Promise.all([
+ syncSource(source, { db: fake.db, now: NOW, fetch }),
+ syncSource(second, { db: fake.db, now: NOW, fetch }),
+ ]);
+ // The first keeps the feed's two products (OLD left it); the second, what's left.
+ expect(one.counts).toMatchObject({ products: { added: 2, removed: 1 } });
+ expect(one.counts.overLimit).toBeUndefined();
+ expect(two.counts).toMatchObject({ products: { added: 1 }, overLimit: 1, limit: 4 });
+ const live = fake
+ .rows('catalog_products')
+ .filter((p) => ['brand-1', 'brand-2'].includes(p.brand_id) && !p.removed_at);
+ expect(live).toHaveLength(4);
+ });
+
+ it('makes only a capped workspace wait in its line, and recounts its room on its turn', async () => {
+ process.env.IS_CLOUD = 'true';
+ const fetch = vi.fn().mockResolvedValue({ html: fixture('google-feed.xml') });
+ // Another write of the workspace holds its line.
+ let release;
+ const holding = inLine(
+ 'org-1',
+ () =>
+ new Promise((resolve) => {
+ release = resolve;
+ }),
+ );
+ try {
+ // A plan without a cap writes at once.
+ getCatalogProductCap.mockResolvedValue(-1);
+ const uncapped = await syncSource(source, { db: fake.db, now: NOW, fetch });
+ expect(uncapped).toMatchObject({ status: 'ok', counts: { products: { added: 2 } } });
+ expect(uncapped.counts.limit).toBeUndefined();
+
+ // A capped one waits for its turn.
+ getCatalogProductCap.mockResolvedValue(4);
+ let done = false;
+ const capped = syncSource(source, { db: fake.db, now: NOW, fetch }).then((outcome) => {
+ done = true;
+ return outcome;
+ });
+ await new Promise((resolve) => setTimeout(resolve, 20));
+ expect(done).toBe(false);
+ // The room is counted when its turn comes: a cap of 2 by then.
+ getCatalogProductCap.mockResolvedValue(2);
+ release();
+ expect(await capped).toMatchObject({ status: 'ok', counts: { limit: 2 } });
+ } finally {
+ release();
+ await holding;
+ }
+ });
+
it("records a workspace without an active subscription in the plan's words", () => {
expect(failureOf(new PlanLimitError('An active subscription is required.', 402))).toEqual({
status: 'failed',
@@ -654,3 +715,71 @@ describe("the plan's product cap (ECOM-019)", () => {
});
});
});
+
+describe('inLine', () => {
+ it("runs a key's work one at a time, in order, past a failure, and forgets an empty line", async () => {
+ const lines = new Map();
+ const ran = [];
+ let release;
+ const first = inLine(
+ 'org-1',
+ () =>
+ new Promise((resolve) => {
+ ran.push('first');
+ release = resolve;
+ }),
+ lines,
+ );
+ const failing = inLine(
+ 'org-1',
+ async () => {
+ ran.push('failing');
+ throw new Error('write failed');
+ },
+ lines,
+ );
+ const third = inLine('org-1', async () => ran.push('third') && 'done', lines);
+ // Another workspace doesn't wait.
+ await inLine('org-2', async () => ran.push('other'), lines);
+ expect(ran).toEqual(['first', 'other']);
+ release('kept');
+ await expect(first).resolves.toBe('kept');
+ await expect(failing).rejects.toThrow('write failed');
+ await expect(third).resolves.toBe('done');
+ expect(ran).toEqual(['first', 'other', 'failing', 'third']);
+ await new Promise((resolve) => setImmediate(resolve));
+ expect(lines.size).toBe(0);
+ });
+
+ it('stops waiting for work stuck past the wait, and keeps the line in order after it', async () => {
+ const warn = vi.spyOn(logger, 'warn').mockImplementation(() => {});
+ const lines = new Map();
+ const ran = [];
+ let release;
+ const stuck = inLine(
+ 'org-1',
+ () =>
+ new Promise((resolve) => {
+ ran.push('stuck');
+ release = resolve;
+ }),
+ lines,
+ 20,
+ );
+ const next = inLine('org-1', async () => ran.push('next') && 'written', lines, 20);
+ // Waits its whole time (LINE_WAIT_MS) for the work before it, which ends.
+ const after = inLine('org-1', async () => ran.push('after'), lines);
+ await expect(next).resolves.toBe('written');
+ await after;
+ // The stuck work is still running: the line went on without it, in order.
+ expect(ran).toEqual(['stuck', 'next', 'after']);
+ expect(warn).toHaveBeenCalledExactlyOnceWith(
+ { line: 'org-1', waitMs: 20 },
+ '[catalog] capped write stopped waiting in line',
+ );
+ release('late');
+ await expect(stuck).resolves.toBe('late');
+ await new Promise((resolve) => setImmediate(resolve));
+ expect(lines.size).toBe(0);
+ });
+});
diff --git a/server/src/routes/catalog.js b/server/src/routes/catalog.js
index f47f3b589..156bd1c80 100644
--- a/server/src/routes/catalog.js
+++ b/server/src/routes/catalog.js
@@ -9,6 +9,7 @@ import {
categoryPages,
setCategoryPage,
} from '../lib/catalog/category-pages.js';
+import { matchBrandLater } from '../lib/catalog/card-match.js';
import { credentialsConfigured } from '../lib/catalog/credentials.js';
import { syncSource } from '../lib/catalog/sources.js';
import {
@@ -47,6 +48,9 @@ import { catalogCategoryLimiter, catalogLimiter } from '../middleware/rate-limit
* ?brandId=…&category=… puts it back to the automatic match;
* - DELETE /api/catalog/sources/:id: a store removed, with its sign-in and
* its products.
+ *
+ * A sync here that reads the catalog has the brand's AI shopping cards
+ * matched to it right after, without waiting (matchAfter).
*/
const router = Router();
@@ -117,11 +121,23 @@ async function sourceById(id) {
return data ?? null;
}
+/**
+ * After a sync that read the catalog, the brand's recent AI shopping cards
+ * matched to it in the background (card-match.js): the daily sweep would
+ * match them only the next day, and the Shelf would count none of the
+ * brand's products until then.
+ */
+function matchAfter(source, outcome) {
+ if (outcome?.status === 'ok') matchBrandLater(source.brand_id);
+}
+
/** A store's first read, or its read after a new sign-in: not waited on, recorded on its row. */
function syncLater(req, source) {
- syncSource(source).catch((err) =>
- req.log.error({ err: { name: err?.name }, sourceId: source.id }, 'catalog store sync error'),
- );
+ syncSource(source)
+ .then((outcome) => matchAfter(source, outcome))
+ .catch((err) =>
+ req.log.error({ err: { name: err?.name }, sourceId: source.id }, 'catalog store sync error'),
+ );
}
// GET /api/catalog/connectors — the stores a person can connect, and the sign-in platforms.
@@ -218,7 +234,9 @@ router.post('/sources/:id/sync', catalogLimiter, async (req, res) => {
if (source.last_synced_at && Date.now() - Date.parse(source.last_synced_at) < RESYNC_AFTER_MS) {
return res.status(409).json({ error: 'Synced a moment ago', code: 'recent' });
}
- return res.json(outcomeBody(await syncSource(source)));
+ const outcome = await syncSource(source);
+ matchAfter(source, outcome);
+ return res.json(outcomeBody(outcome));
} catch (err) {
return fail(req, res, err, 'catalog sync error');
}
@@ -317,7 +335,9 @@ router.post(
await supabaseAdmin.from('catalog_sources').update({ format }).eq('id', source.id);
source = { ...source, format };
}
- return res.json(outcomeBody(await syncSource(source, { text: req.body })));
+ const outcome = await syncSource(source, { text: req.body });
+ matchAfter(source, outcome);
+ return res.json(outcomeBody(outcome));
} catch (err) {
return fail(req, res, err, 'catalog upload error');
}
diff --git a/server/src/routes/catalog.test.js b/server/src/routes/catalog.test.js
index 8e6190075..7bf9d1343 100644
--- a/server/src/routes/catalog.test.js
+++ b/server/src/routes/catalog.test.js
@@ -21,6 +21,7 @@ const connectSignInSource = vi.fn();
const categoryPages = vi.fn();
const setCategoryPage = vi.fn();
const listSignInAccounts = vi.fn();
+const matchBrandLater = vi.fn();
let configured;
let googleConfigured;
@@ -66,6 +67,9 @@ vi.mock('../lib/access.js', () => ({
vi.mock('../lib/catalog/sources.js', () => ({
syncSource: (...args) => syncSource(...args),
}));
+vi.mock('../lib/catalog/card-match.js', () => ({
+ matchBrandLater: (...args) => matchBrandLater(...args),
+}));
// The limiters' buckets would end this file's run part way: each request
// notes the bucket it took instead.
const buckets = [];
@@ -157,6 +161,7 @@ beforeEach(async () => {
assertBrandAccess.mockReset().mockResolvedValue({ orgId: 'org-1' });
buckets.length = 0;
syncSource.mockReset().mockResolvedValue({ status: 'ok', counts: { items: 4 } });
+ matchBrandLater.mockReset().mockResolvedValue(undefined);
googleConfigured = true;
categoryPages.mockReset().mockResolvedValue([
{
@@ -266,6 +271,16 @@ describe('POST /api/catalog/sources/:id/sync', () => {
expect(res.status).toBe(409);
expect((await res.json()).code).toBe('recent');
expect(syncSource).not.toHaveBeenCalled();
+ expect(matchBrandLater).not.toHaveBeenCalled();
+ });
+
+ it("matches the brand's shopping cards to the catalog it read, not after a failed read", async () => {
+ expect((await sync()).status).toBe(200);
+ expect(matchBrandLater).toHaveBeenCalledExactlyOnceWith(BRAND);
+ matchBrandLater.mockClear();
+ syncSource.mockResolvedValueOnce({ status: 'failed', error: 'The feed took too long.' });
+ expect((await sync()).status).toBe(200);
+ expect(matchBrandLater).not.toHaveBeenCalled();
});
});
@@ -284,6 +299,15 @@ describe('POST /api/catalog/upload', () => {
});
});
+ it("matches the brand's shopping cards once the file is in the catalog", async () => {
+ await upload('id,title\n1,Boot');
+ expect(matchBrandLater).toHaveBeenCalledExactlyOnceWith(BRAND);
+ matchBrandLater.mockClear();
+ syncSource.mockResolvedValueOnce({ status: 'format_error', error: 'Not a feed.' });
+ await upload('name\nAda');
+ expect(matchBrandLater).not.toHaveBeenCalled();
+ });
+
it('keeps one upload source a brand, with the format asked for', async () => {
sources.push({ id: 'up-1', brand_id: BRAND, kind: 'upload', format: 'auto', url: null });
await upload('<rss/>', `brandId=${BRAND}&format=google_xml`);
@@ -415,6 +439,8 @@ describe('POST /api/catalog/sources', () => {
userId: 'user-1',
});
expect(syncSource).toHaveBeenCalledWith(expect.objectContaining({ id: 'store-1' }));
+ // Its cards are matched once its first read is in.
+ await vi.waitFor(() => expect(matchBrandLater).toHaveBeenCalledExactlyOnceWith(BRAND));
});
it("answers the store's refusal in the person's words, and connects nothing for an analyst", async () => {
diff --git a/supabase/migrations/00371_catalog_suggestion_categories.sql b/supabase/migrations/00371_catalog_suggestion_categories.sql
new file mode 100644
index 000000000..0aefc61b0
--- /dev/null
+++ b/supabase/migrations/00371_catalog_suggestion_categories.sql
@@ -0,0 +1,96 @@
+-- ============================================================================
+-- 00371: catalog categories for shopping suggestions, counted in the database
+-- ============================================================================
+-- Shopping prompt suggestions (ECOM-003, 00362) offer the model a brand's
+-- biggest catalog categories, each with its price band. The server read the
+-- whole live catalog for that, a page at a time (every product, category
+-- link and variant), to keep 8 categories, and cut the categories at an
+-- arbitrary 1,000. `catalog_suggestion_categories` counts here instead and
+-- returns only the categories offered:
+--
+-- * a category's live products (removed_at null), p_min_products at least;
+-- * a product's price: its lowest live variant's, on sale or not, above
+-- zero, in that variant's currency;
+-- * when every priced product of the category shares one currency: the
+-- lowest and the highest price, and the price three quarters of them
+-- sell at or under (price_p75; the server rounds it to one a shopper
+-- would name);
+-- * the most products first, then by name, p_limit of them.
+--
+-- Products and variants are read by brand, live only, so their brand
+-- indexes (00278, 00281) serve the read, not every brand's rows.
+--
+-- The service role calls it, as 00350's catalog_category_counts.
+
+create or replace function public.catalog_suggestion_categories(
+ p_brand_id uuid,
+ p_limit integer default 8,
+ p_min_products integer default 2
+)
+returns table (
+ category text,
+ products integer,
+ price_low numeric,
+ price_high numeric,
+ price_p75 numeric,
+ currency text
+)
+language sql
+stable
+set search_path = public
+as $$
+ with live as (
+ select pc.category_id, pc.product_id
+ from public.catalog_product_categories pc
+ join public.catalog_products p
+ on p.id = pc.product_id and p.brand_id = p_brand_id and p.removed_at is null
+ where pc.brand_id = p_brand_id
+ ),
+ priced as (
+ select distinct on (v.product_id)
+ v.product_id,
+ coalesce(v.sale_price, v.price) as amount,
+ v.currency
+ from public.catalog_variants v
+ where v.brand_id = p_brand_id
+ and v.removed_at is null
+ and v.product_id in (select l.product_id from live l)
+ and coalesce(v.sale_price, v.price) > 0
+ order by v.product_id, coalesce(v.sale_price, v.price), v.id
+ ),
+ counted as (
+ select c.name as name,
+ count(*)::integer as product_count,
+ count(distinct coalesce(pr.currency, '')) filter (where pr.product_id is not null)
+ as currencies,
+ max(pr.currency) as one_currency,
+ min(pr.amount) as low,
+ max(pr.amount) as high,
+ percentile_disc(0.75) within group (order by pr.amount) as p75
+ from public.catalog_categories c
+ join live l on l.category_id = c.id
+ left join priced pr on pr.product_id = l.product_id
+ where c.brand_id = p_brand_id
+ group by c.id, c.name
+ having count(*) >= p_min_products
+ )
+ select k.name,
+ k.product_count,
+ case when k.currencies = 1 and k.one_currency is not null then k.low end,
+ case when k.currencies = 1 and k.one_currency is not null then k.high end,
+ case when k.currencies = 1 and k.one_currency is not null then k.p75 end,
+ case when k.currencies = 1 then k.one_currency end
+ from counted k
+ order by k.product_count desc, k.name
+ limit greatest(p_limit, 0)
+$$;
+
+revoke all on function public.catalog_suggestion_categories(uuid, integer, integer)
+ from public, anon, authenticated;
+grant execute on function public.catalog_suggestion_categories(uuid, integer, integer)
+ to service_role;
+
+comment on function public.catalog_suggestion_categories(uuid, integer, integer) is
+ 'Shopping prompt suggestions (ECOM-003, 00371): a brand''s catalog categories with the most live products (p_min_products at least, p_limit of them), each with the lowest, highest and 75th-percentile price of its products (a product''s lowest live variant price, sale price first) when they share one currency. Service role only.';
+
+notify pgrst, 'reload schema';
diff --git a/supabase/schema.sql b/supabase/schema.sql
index 71f6f2ef9..a2e743071 100644
--- a/supabase/schema.sql
+++ b/supabase/schema.sql
@@ -42495,6 +42495,106 @@ comment on function public.prompt_demand_brand_updates() is
notify pgrst, 'reload schema';
+-- ─────────────────────────────────────────────────────────────────────────
+-- migrations/00371_catalog_suggestion_categories.sql
+-- ─────────────────────────────────────────────────────────────────────────
+-- ============================================================================
+-- 00371: catalog categories for shopping suggestions, counted in the database
+-- ============================================================================
+-- Shopping prompt suggestions (ECOM-003, 00362) offer the model a brand's
+-- biggest catalog categories, each with its price band. The server read the
+-- whole live catalog for that, a page at a time (every product, category
+-- link and variant), to keep 8 categories, and cut the categories at an
+-- arbitrary 1,000. `catalog_suggestion_categories` counts here instead and
+-- returns only the categories offered:
+--
+-- * a category's live products (removed_at null), p_min_products at least;
+-- * a product's price: its lowest live variant's, on sale or not, above
+-- zero, in that variant's currency;
+-- * when every priced product of the category shares one currency: the
+-- lowest and the highest price, and the price three quarters of them
+-- sell at or under (price_p75; the server rounds it to one a shopper
+-- would name);
+-- * the most products first, then by name, p_limit of them.
+--
+-- Products and variants are read by brand, live only, so their brand
+-- indexes (00278, 00281) serve the read, not every brand's rows.
+--
+-- The service role calls it, as 00350's catalog_category_counts.
+
+create or replace function public.catalog_suggestion_categories(
+ p_brand_id uuid,
+ p_limit integer default 8,
+ p_min_products integer default 2
+)
+returns table (
+ category text,
+ products integer,
+ price_low numeric,
+ price_high numeric,
+ price_p75 numeric,
+ currency text
+)
+language sql
+stable
+set search_path = public
+as $$
+ with live as (
+ select pc.category_id, pc.product_id
+ from public.catalog_product_categories pc
+ join public.catalog_products p
+ on p.id = pc.product_id and p.brand_id = p_brand_id and p.removed_at is null
+ where pc.brand_id = p_brand_id
+ ),
+ priced as (
+ select distinct on (v.product_id)
+ v.product_id,
+ coalesce(v.sale_price, v.price) as amount,
+ v.currency
+ from public.catalog_variants v
+ where v.brand_id = p_brand_id
+ and v.removed_at is null
+ and v.product_id in (select l.product_id from live l)
+ and coalesce(v.sale_price, v.price) > 0
+ order by v.product_id, coalesce(v.sale_price, v.price), v.id
+ ),
+ counted as (
+ select c.name as name,
+ count(*)::integer as product_count,
+ count(distinct coalesce(pr.currency, '')) filter (where pr.product_id is not null)
+ as currencies,
+ max(pr.currency) as one_currency,
+ min(pr.amount) as low,
+ max(pr.amount) as high,
+ percentile_disc(0.75) within group (order by pr.amount) as p75
+ from public.catalog_categories c
+ join live l on l.category_id = c.id
+ left join priced pr on pr.product_id = l.product_id
+ where c.brand_id = p_brand_id
+ group by c.id, c.name
+ having count(*) >= p_min_products
+ )
+ select k.name,
+ k.product_count,
+ case when k.currencies = 1 and k.one_currency is not null then k.low end,
+ case when k.currencies = 1 and k.one_currency is not null then k.high end,
+ case when k.currencies = 1 and k.one_currency is not null then k.p75 end,
+ case when k.currencies = 1 then k.one_currency end
+ from counted k
+ order by k.product_count desc, k.name
+ limit greatest(p_limit, 0)
+$$;
+
+revoke all on function public.catalog_suggestion_categories(uuid, integer, integer)
+ from public, anon, authenticated;
+grant execute on function public.catalog_suggestion_categories(uuid, integer, integer)
+ to service_role;
+
+comment on function public.catalog_suggestion_categories(uuid, integer, integer) is
+ 'Shopping prompt suggestions (ECOM-003, 00371): a brand''s catalog categories with the most live products (p_min_products at least, p_limit of them), each with the lowest, highest and 75th-percentile price of its products (a product''s lowest live variant price, sale price first) when they share one currency. Service role only.';
+
+notify pgrst, 'reload schema';
+
-- ─────────────────────────────────────────────────────────────────────────
-- migrations/00374_menu_access.sql
-- ─────────────────────────────────────────────────────────────────────────