Open-source agentic AI Search Execution Platform. Track AI visibility, find opportunities, and let agents turn signals into prioritized actions and drafted fixes across ChatGPT, Claude, Gemini, Google AI Overviews, Google AI Mode, Perplexity, Grok, and Microsoft Copilot.
TypeScript aeoai-agentai-searchanswer-engine-optimizationbrand-monitoringchatgptclaudecopilotdockergeminigeogoogle-ai-overviewsgrokmcpnextjsopen-sourceperplexityseosupabasetypescript
chore(catalog): delete catalog products removed more than 90 days ago (#1632)
Cihan944bca4480e0
diff --git a/server/src/lib/catalog/__fixtures__/fake-db.js b/server/src/lib/catalog/__fixtures__/fake-db.jsindex 008d68dc1..2d8778844 100644--- a/server/src/lib/catalog/__fixtures__/fake-db.js+++ b/server/src/lib/catalog/__fixtures__/fake-db.js@@ -23,6 +23,15 @@ function compareBy(orders) {};}+/** The rows deleted with a row of these tables (on delete cascade, 00278): table, column. */+const CASCADES = {+ catalog_products: [+ ['catalog_variants', 'product_id'],+ ['catalog_images', 'product_id'],+ ['catalog_product_categories', 'product_id'],+ ],+};+/*** An in-memory stand-in for the tables the catalog writes, with PostgREST's* calls as the sync and the sources make them, counting every write.@@ -67,6 +76,8 @@ export function fakeDb({ maxRows = Infinity } = {}) {if (kind === 'eq') return at === value;if (kind === 'neq') return at !== value;if (kind === 'is') return value === null ? at == null : at === value;+ // SQL's: a null is never less.+ if (kind === 'lt') return at != null && at < value;return value.includes(at);});if (state.op === 'select') {@@ -107,11 +118,19 @@ export function fakeDb({ maxRows = Infinity } = {}) {return { data: state.returning ? changed : null, error: null };}if (state.op === 'delete') {+ const deleted = all.filter(match);tables.set(table,all.filter((row) => !match(row)),);- return { data: null, error: null };+ const ids = new Set(deleted.map((row) => row.id));+ for (const [child, column] of CASCADES[table] ?? []) {+ tables.set(+ child,+ rows(child).filter((row) => !ids.has(row[column])),+ );+ }+ return { data: state.returning ? deleted : null, error: null };}throw new Error(`unexpected ${state.op}`);};@@ -133,6 +152,10 @@ export function fakeDb({ maxRows = Infinity } = {}) {state.filters.push(['is', column, value]);return builder;},+ lt: (column, value) => {+ state.filters.push(['lt', column, value]);+ return builder;+ },single: () => {state.one = 'single';return builder;diff --git a/server/src/lib/catalog/purge.js b/server/src/lib/catalog/purge.jsnew file mode 100644index 000000000..26c03edef--- /dev/null+++ b/server/src/lib/catalog/purge.js@@ -0,0 +1,86 @@+import supabaseAdmin from '../../config/supabase.js';+import { chunkIds } from '../chunked-in.js';+import { logger } from '../logger.js';++/**+ * A product gone from its source is marked removed by the sync (sync.js,+ * `removed_at`) and kept REMOVED_KEEP_DAYS, so one that comes back within+ * them is live again with the same id. The daily run deletes older ones+ * here, so the catalog doesn't grow with every product a shop ever sold.+ *+ * - Their variants, images and category links go with them (on delete+ * cascade, 00278).+ * - Merchant Center's item issues (00296) aren't theirs: they're keyed by+ * source and item id, replaced by each sync of the source and deleted with+ * it. An item Merchant Center still lists keeps its issues even when the+ * catalog can't keep the item, so they aren't deleted here.+ * - A draft's shopping path (00351) keeps product ids in jsonb. A deleted+ * product reads as a removed one, gone from the catalog: the path's build+ * (shopping-path.js keptProducts) and the editor's check (the web's+ * getShoppingCheck) both say `gone` for an id with no row.+ */++/** How long a removed product is kept. */+export const REMOVED_KEEP_DAYS = 90;+/** Products read per batch. */+export const PURGE_BATCH = 500;+/** Products deleted per daily run at most; the rest waits a day. */+export const MAX_PURGED = 10_000;++const DAY_MS = 24 * 60 * 60 * 1000;++/**+ * Deletes the products removed over REMOVED_KEEP_DAYS ago, PURGE_BATCH at a+ * time and MAX_PURGED a run at most, and says how many it deleted. Each+ * delete asks again that the product is still removed, so one a sync brought+ * back meanwhile stays. Never throws: a failure is logged and the rest waits+ * a day.+ *+ * @param {{ db?: object, now?: Date }} [options]+ * @returns {Promise<number>}+ */+export async function purgeRemovedCatalogProducts({ db = supabaseAdmin, now = new Date() } = {}) {+ const before = new Date(now.getTime() - REMOVED_KEEP_DAYS * DAY_MS).toISOString();+ let purged = 0;+ try {+ while (purged < MAX_PURGED) {+ const size = Math.min(PURGE_BATCH, MAX_PURGED - purged);+ const { data, error } = await db+ .from('catalog_products')+ .select('id')+ .lt('removed_at', before)+ .limit(size);+ if (error) throw new Error(`catalog_products read: ${error.message}`);+ const ids = (data ?? []).map((row) => row.id);+ let batch = 0;+ // A delete's ids go in its address: IN_FILTER_CHUNK at a time.+ for (const chunk of chunkIds(ids)) {+ const { data: deleted, error: failed } = await db+ .from('catalog_products')+ .delete()+ .in('id', chunk)+ .lt('removed_at', before)+ .select('id');+ if (failed) throw new Error(`catalog_products delete: ${failed.message}`);+ batch += deleted?.length ?? 0;+ }+ purged += batch;+ if (ids.length < size || batch === 0) break;+ }+ } catch (err) {+ logger.warn(+ { err: { name: err?.name, message: err?.message }, purged },+ '[catalog] removed products purge failed; the rest goes tomorrow',+ );+ return purged;+ }+ if (purged >= MAX_PURGED) {+ logger.warn(+ { purged },+ '[catalog] removed products purge stopped early; the rest goes tomorrow',+ );+ } else {+ logger.info({ purged }, '[catalog] removed products purged');+ }+ return purged;+}diff --git a/server/src/lib/catalog/purge.test.js b/server/src/lib/catalog/purge.test.jsnew file mode 100644index 000000000..32cfd56de--- /dev/null+++ b/server/src/lib/catalog/purge.test.js@@ -0,0 +1,158 @@+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';++// The module reaches the supabase admin client at import time, whose config+// hard-exits when SUPABASE_* env is absent (as in CI).+vi.mock('../../config/supabase.js', () => ({ default: {} }));++import { IN_FILTER_CHUNK } from '../chunked-in.js';+import { logger } from '../logger.js';+import { fakeDb } from './__fixtures__/fake-db.js';+import {+ MAX_PURGED,+ PURGE_BATCH,+ REMOVED_KEEP_DAYS,+ purgeRemovedCatalogProducts,+} from './purge.js';++const NOW = new Date('2026-10-28T06:00:00.000Z');+const daysAgo = (days) => new Date(NOW.getTime() - days * 24 * 60 * 60 * 1000).toISOString();++let fake;+let info;+let warn;+beforeEach(() => {+ fake = fakeDb();+ info = vi.spyOn(logger, 'info').mockImplementation(() => {});+ warn = vi.spyOn(logger, 'warn').mockImplementation(() => {});+});+afterEach(() => {+ vi.restoreAllMocks();+});++/** A product with one variant, an image and a category link. */+function product(id, removedAt) {+ fake.rows('catalog_products').push({ id, brand_id: 'brand-1', removed_at: removedAt });+ fake.rows('catalog_variants').push({ id: `${id}-v`, product_id: id, removed_at: removedAt });+ fake.rows('catalog_images').push({ id: `${id}-i`, product_id: id });+ fake.rows('catalog_product_categories').push({ product_id: id, category_id: 'c1' });+}++/** The fake's db, counting each read's limit and each delete's ids. */+function counted() {+ const reads = [];+ const deletes = [];+ const db = {+ from: (table) => {+ const builder = fake.db.from(table);+ const { limit, delete: remove, in: within } = builder;+ let deleting = false;+ builder.limit = (n) => {+ reads.push(n);+ return limit(n);+ };+ builder.delete = () => {+ deleting = true;+ return remove();+ };+ builder.in = (column, values) => {+ if (deleting) deletes.push(values.length);+ return within(column, values);+ };+ return builder;+ },+ };+ return { db, reads, deletes };+}++describe('purgeRemovedCatalogProducts', () => {+ it('deletes products removed over 90 days ago, with their variants, images and categories', async () => {+ product('old', daysAgo(REMOVED_KEEP_DAYS + 1));+ product('recent', daysAgo(REMOVED_KEEP_DAYS - 1));+ product('live', null);++ const purged = await purgeRemovedCatalogProducts({ db: fake.db, now: NOW });++ expect(purged).toBe(1);+ expect(fake.rows('catalog_products').map((p) => p.id)).toEqual(['recent', 'live']);+ for (const table of ['catalog_variants', 'catalog_images', 'catalog_product_categories']) {+ expect(fake.rows(table).map((r) => r.product_id)).toEqual(['recent', 'live']);+ }+ expect(info).toHaveBeenCalledWith({ purged: 1 }, '[catalog] removed products purged');+ });++ it('keeps a product a sync brought back between the read and the delete', async () => {+ product('old', daysAgo(REMOVED_KEEP_DAYS + 5));+ const db = {+ from: (table) => {+ const builder = fake.db.from(table);+ const { delete: remove } = builder;+ builder.delete = () => {+ // The sync makes it live again first.+ fake.rows('catalog_products')[0].removed_at = null;+ return remove();+ };+ return builder;+ },+ };++ expect(await purgeRemovedCatalogProducts({ db, now: NOW })).toBe(0);+ expect(fake.rows('catalog_products')).toHaveLength(1);+ });++ it('reads PURGE_BATCH at a time, deletes IN_FILTER_CHUNK ids a request, and stops at MAX_PURGED', async () => {+ const over = MAX_PURGED + 50;+ for (let i = 0; i < over; i++) {+ fake.rows('catalog_products').push({+ id: `p${String(i).padStart(6, '0')}`,+ removed_at: daysAgo(REMOVED_KEEP_DAYS + 10),+ });+ }+ const { db, reads, deletes } = counted();++ const purged = await purgeRemovedCatalogProducts({ db, now: NOW });++ expect(purged).toBe(MAX_PURGED);+ expect(fake.rows('catalog_products')).toHaveLength(50);+ expect(reads).toEqual(Array(MAX_PURGED / PURGE_BATCH).fill(PURGE_BATCH));+ expect(Math.max(...deletes)).toBe(IN_FILTER_CHUNK);+ expect(deletes.reduce((a, b) => a + b, 0)).toBe(MAX_PURGED);+ expect(warn).toHaveBeenCalledWith(+ { purged: MAX_PURGED },+ '[catalog] removed products purge stopped early; the rest goes tomorrow',+ );++ // The rest the next day.+ expect(await purgeRemovedCatalogProducts({ db: fake.db, now: NOW })).toBe(50);+ expect(fake.rows('catalog_products')).toHaveLength(0);+ });++ it('logs a failure and resolves, so the daily run goes on', async () => {+ product('old', daysAgo(REMOVED_KEEP_DAYS + 1));+ const failing = {+ from: () => {+ const query = {+ select: () => query,+ lt: () => query,+ limit: () => query,+ then: (resolve) => resolve({ data: null, error: { message: 'statement timeout' } }),+ };+ return query;+ },+ };++ await expect(purgeRemovedCatalogProducts({ db: failing, now: NOW })).resolves.toBe(0);+ expect(warn).toHaveBeenCalledWith(+ { err: { name: 'Error', message: 'catalog_products read: statement timeout' }, purged: 0 },+ '[catalog] removed products purge failed; the rest goes tomorrow',+ );++ // A client that throws outright is caught as well.+ const throwing = {+ from: () => {+ throw new TypeError('fetch failed');+ },+ };+ await expect(purgeRemovedCatalogProducts({ db: throwing, now: NOW })).resolves.toBe(0);+ expect(fake.rows('catalog_products')).toHaveLength(1);+ });+});diff --git a/server/src/lib/catalog/sync.js b/server/src/lib/catalog/sync.jsindex d95b778a5..9174e995d 100644--- a/server/src/lib/catalog/sync.js+++ b/server/src/lib/catalog/sync.js@@ -12,7 +12,9 @@ import supabaseAdmin from '../../config/supabase.js';* A row whose content didn't change isn't written again (`content_hash`),* so a sync of the same file writes nothing.* - A product or variant gone from the source is marked removed- * (`removed_at`), never deleted. One that comes back is live again.+ * (`removed_at`). One that comes back is live again. A product removed+ * over 90 days ago is deleted by the daily run, its variants with it+ * (purge.js).* - A changed product's images and categories are written again.** Values are clipped to the columns' limits, and an address that isn'tdiff --git a/server/src/server.js b/server/src/server.jsindex 6c3fc90bd..00b0ea7f4 100644--- a/server/src/server.js+++ b/server/src/server.js@@ -30,6 +30,7 @@ import { rebuildQueuedCitationRollups, REBUILD_QUEUE_SCHEDULE } from './lib/citaimport { sweepActionValidation } from './lib/action-center/validate.js';import { sweepImproveOutcomes } from './lib/action-center/improve-outcome.js';import { sweepBrandFacts } from './lib/brand-facts-sweep.js';+import { purgeRemovedCatalogProducts } from './lib/catalog/purge.js';import { sweepCatalogSources } from './lib/catalog/sources.js';import { sweepWebflowPublished } from './lib/cms/webflow-status.js';import { generateBriefForOpportunity } from './routes/content.js';@@ -298,9 +299,16 @@ async function runDailyTracking() {});// Catalog feed URLs (#1263): read again once a day, never blocking tracking.- sweepCatalogSources().catch((err) => {- logger.error({ err }, '[catalog] daily sweep failed');- });+ // Then the products removed from their source over 90 days ago are+ // deleted: after the sweep, so it isn't deleting while a sync brings one back.+ sweepCatalogSources()+ .catch((err) => {+ logger.error({ err }, '[catalog] daily sweep failed');+ })+ .then(() => purgeRemovedCatalogProducts())+ .catch((err) => {+ logger.error({ err }, '[catalog] daily purge failed');+ });// Pages sent to Webflow, published there since (#1331): read once a day.sweepWebflowPublished().catch((err) => {