Skip to content

Open-source agentic AI Search Execution Platform. Track AI visibility, find opportunities, and let agents turn signals into prioritized actions and drafted fixes across ChatGPT, Claude, Gemini, Google AI Overviews, Google AI Mode, Perplexity, Grok, and Microsoft Copilot.

fix(catalog): send a sync's removals and rewrites in filters of 100 ids, not 500 (#1633)

Cihandafb89cbff07
diff --git a/server/src/lib/catalog/sync.js b/server/src/lib/catalog/sync.js
index 9174e995d..6fa2ab516 100644
--- a/server/src/lib/catalog/sync.js
+++ b/server/src/lib/catalog/sync.js
@@ -1,5 +1,6 @@
import { createHash } from 'node:crypto';
import supabaseAdmin from '../../config/supabase.js';
+import { chunkIds } from '../chunked-in.js';
/**
* Writes a brand's catalog from one source (#1263, 00278): the one batch
@@ -22,6 +23,7 @@ import supabaseAdmin from '../../config/supabase.js';
*/
const PAGE = 1000;
+/** Rows per write: they go in the request's body. */
const CHUNK = 500;
const clip = (value, max) => {
@@ -153,8 +155,10 @@ async function upsertRows(db, table, rows) {
return ids;
}
+// Ids in a filter go in the URL: chunkIds keeps each request near 4 kB,
+// where 500 uuids (~19 kB) could be refused before reaching PostgREST.
async function markRemoved(db, table, ids, at) {
- for (const chunk of chunks(ids)) {
+ for (const chunk of chunkIds(ids)) {
const { error } = await db
.from(table)
.update({ removed_at: at, updated_at: at })
@@ -164,7 +168,7 @@ async function markRemoved(db, table, ids, at) {
}
async function replaceIn(db, table, productIds, rows) {
- for (const chunk of chunks(productIds)) {
+ for (const chunk of chunkIds(productIds)) {
const { error } = await db.from(table).delete().in('product_id', chunk);
if (error) throw new Error(`${table} clear: ${error.message}`);
}
diff --git a/server/src/lib/catalog/sync.test.js b/server/src/lib/catalog/sync.test.js
index d6b4481bc..db825f818 100644
--- a/server/src/lib/catalog/sync.test.js
+++ b/server/src/lib/catalog/sync.test.js
@@ -166,6 +166,52 @@ describe('syncCatalog', () => {
expect(fake.rows('catalog_images').map((i) => i.url)).not.toContain('ftp://x/y.jpg');
});
+ it('removes and rewrites many products with filters of at most 100 ids, so no request runs too long', async () => {
+ const lines = ['id,item_group_id,title,link,price,availability,product_type'];
+ for (let i = 0; i < 250; i++) {
+ lines.push(
+ `M-${i},M-${i},Mug ${i},https://s.example/m/${i},10.00 USD,in stock,Kitchen > Mugs`,
+ );
+ }
+ const { items: many } = parseCsvFeed(lines.join('\n'));
+ await syncCatalog(BRAND, 'feed', many, { db: fake.db, now: new Date('2026-10-20T00:00:00Z') });
+ // Each .in() filter goes in the request's URL: count the ids per filter.
+ const sizes = [];
+ const db = {
+ from: (table) => {
+ const builder = fake.db.from(table);
+ const watch = (b) =>
+ new Proxy(b, {
+ get(target, key) {
+ const value = target[key];
+ if (key === 'in') {
+ return (column, values) => {
+ sizes.push(values.length);
+ return watch(value.call(target, column, values));
+ };
+ }
+ if (typeof value !== 'function' || key === 'then') return value;
+ return (...args) => {
+ const out = value.apply(target, args);
+ return out === target ? watch(target) : out;
+ };
+ },
+ });
+ return watch(builder);
+ },
+ };
+ const counts = await syncCatalog(BRAND, 'feed', [], {
+ db,
+ now: new Date('2026-10-21T00:00:00Z'),
+ });
+ expect(counts.products.removed).toBe(250);
+ expect(
+ fake.rows('catalog_products').every((p) => p.removed_at === '2026-10-21T00:00:00.000Z'),
+ ).toBe(true);
+ expect(sizes.length).toBeGreaterThan(2);
+ expect(Math.max(...sizes)).toBeLessThanOrEqual(100);
+ });
+
it('imports a 10,000-row CSV, and a second import writes nothing', async () => {
const lines = ['id,item_group_id,title,link,price,availability,color,product_type'];
for (let i = 0; i < 10_000; i++) {