Skip to content

Open-source agentic AI Search Execution Platform. Track AI visibility, find opportunities, and let agents turn signals into prioritized actions and drafted fixes across ChatGPT, Claude, Gemini, Google AI Overviews, Google AI Mode, Perplexity, Grok, and Microsoft Copilot.

feat(catalog): fill a brand's catalog from a product feed URL or an uploaded file (00279, #1263 part 2) (#1493)

Gökhan Geyike3252054a233
diff --git a/server/src/connectors/feed-url/index.js b/server/src/connectors/feed-url/index.js
new file mode 100644
index 000000000..89759dc82
--- /dev/null
+++ b/server/src/connectors/feed-url/index.js
@@ -0,0 +1,37 @@
+import { parseFeed } from '../../lib/feeds/index.js';
+import { fetchPage } from '../../lib/safe-fetch.js';
+
+/**
+ * The product feed URL connector (#1263): a Google product feed (XML), a Meta
+ * catalog CSV or a store's CSV export at a public address, read daily.
+ *
+ * The feed is fetched like any page the server reads (lib/safe-fetch.js):
+ * public addresses only, checked again on every redirect, standard ports, no
+ * credentials. It may be up to 50 MB and take up to a minute.
+ */
+
+export const FEED_MAX_BYTES = 50 * 1024 * 1024;
+export const FEED_TIMEOUT_MS = 60_000;
+
+/** What a feed is served as: XML, CSV or text, or a generic file type. */
+export const FEED_TYPES =
+ /^\s*$|^\s*(text\/(xml|csv|plain|tab-separated-values)|application\/(xml|rss\+xml|atom\+xml|csv|octet-stream|vnd\.ms-excel))\b/i;
+const FEED_ACCEPT = 'application/xml,text/xml,text/csv,text/plain;q=0.9,*/*;q=0.1';
+
+export default {
+ kind: 'feed_url',
+
+ /**
+ * @param {{ url: string, format: string }} source
+ * @param {{ fetch?: typeof fetchPage }} [ctx] `fetch` is the tests' stand-in
+ */
+ async readCatalog(source, { fetch = fetchPage } = {}) {
+ const { html: text } = await fetch(source.url, {
+ timeoutMs: FEED_TIMEOUT_MS,
+ maxBytes: FEED_MAX_BYTES,
+ types: FEED_TYPES,
+ accept: FEED_ACCEPT,
+ });
+ return parseFeed(text, source.format);
+ },
+};
diff --git a/server/src/connectors/index.js b/server/src/connectors/index.js
new file mode 100644
index 000000000..db2859b95
--- /dev/null
+++ b/server/src/connectors/index.js
@@ -0,0 +1,28 @@
+import feedUrl from './feed-url/index.js';
+
+/**
+ * Catalog connectors (#1263): one folder each under `server/src/connectors/`,
+ * all writing the same catalog (00278) through `syncCatalog`
+ * (lib/catalog/sync.js). A connector says what a source of its kind holds;
+ * the plumbing (lib/catalog/sources.js) syncs it, records how it went, and
+ * runs it daily and on demand.
+ *
+ * A connector is an object with:
+ * - `kind`: the `catalog_sources.kind` it reads (00279);
+ * - `readCatalog(source, ctx)`: the source's products, as the feed
+ * parsers' items (`{ items, errors, format }`); it throws
+ * FeedFormatError for a file that isn't a feed, or a fetch error;
+ * - later, for stores: `connect`, `syncOrders(since)`,
+ * `handleWebhook(event)` and `disconnect`.
+ *
+ * A connector never logs credentials or a buyer's personal data, and reads
+ * only what its feature needs.
+ */
+export const CONNECTORS = {
+ [feedUrl.kind]: feedUrl,
+};
+
+/** The connector for a source's kind, or null. */
+export function connectorFor(kind) {
+ return CONNECTORS[kind] ?? null;
+}
diff --git a/server/src/lib/catalog/__fixtures__/fake-db.js b/server/src/lib/catalog/__fixtures__/fake-db.js
new file mode 100644
index 000000000..a62170715
--- /dev/null
+++ b/server/src/lib/catalog/__fixtures__/fake-db.js
@@ -0,0 +1,120 @@
+import { randomUUID } from 'node:crypto';
+
+/**
+ * An in-memory stand-in for the tables the catalog writes, with PostgREST's
+ * calls as the sync and the sources make them, counting every write.
+ */
+export function fakeDb() {
+ const tables = new Map();
+ const rows = (name) => {
+ if (!tables.has(name)) tables.set(name, []);
+ return tables.get(name);
+ };
+ const writes = [];
+ const uniqueKey = (table, row) =>
+ table === 'catalog_categories'
+ ? `${row.brand_id}|${row.name}`
+ : `${row.brand_id}|${row.source}|${row.external_id}`;
+ // The unique keys' rows, by table, as the database's unique indexes find them.
+ const byKey = new Map();
+ const keyed = (table) => {
+ if (!byKey.has(table)) byKey.set(table, new Map());
+ return byKey.get(table);
+ };
+
+ function from(table) {
+ const state = { filters: [], range: null, op: 'select', payload: null, returning: null };
+ const run = () => {
+ const all = rows(table);
+ const match = (row) =>
+ state.filters.every(([kind, column, value]) =>
+ kind === 'eq' ? row[column] === value : value.includes(row[column]),
+ );
+ if (state.op === 'select') {
+ const found = all.filter(match).sort((a, b) => (a.id < b.id ? -1 : 1));
+ const [start, end] = state.range ?? [0, found.length];
+ return { data: found.slice(start, end + 1), error: null };
+ }
+ writes.push({
+ table,
+ op: state.op,
+ count: Array.isArray(state.payload) ? state.payload.length : 1,
+ });
+ if (state.op === 'upsert') {
+ const out = [];
+ for (const values of state.payload) {
+ const key = uniqueKey(table, values);
+ let row = keyed(table).get(key);
+ if (row) Object.assign(row, values);
+ else {
+ row = { id: randomUUID(), removed_at: null, ...values };
+ all.push(row);
+ keyed(table).set(key, row);
+ }
+ out.push(row);
+ }
+ return { data: out, error: null };
+ }
+ if (state.op === 'insert') {
+ for (const values of state.payload) all.push({ id: randomUUID(), ...values });
+ return { data: null, error: null };
+ }
+ if (state.op === 'update') {
+ for (const row of all.filter(match)) Object.assign(row, state.payload);
+ return { data: null, error: null };
+ }
+ if (state.op === 'delete') {
+ tables.set(
+ table,
+ all.filter((row) => !match(row)),
+ );
+ return { data: null, error: null };
+ }
+ throw new Error(`unexpected ${state.op}`);
+ };
+ const builder = {
+ select: () => builder,
+ eq: (column, value) => {
+ state.filters.push(['eq', column, value]);
+ return builder;
+ },
+ in: (column, values) => {
+ state.filters.push(['in', column, values]);
+ return builder;
+ },
+ order: () => builder,
+ // The sweep's due filter: the tests' rows are all due.
+ or: () => builder,
+ limit: (n) => {
+ state.range = [0, n - 1];
+ return builder;
+ },
+ range: (start, end) => {
+ state.range = [start, end];
+ return builder;
+ },
+ upsert: (payload) => {
+ state.op = 'upsert';
+ state.payload = payload;
+ return builder;
+ },
+ insert: (payload) => {
+ state.op = 'insert';
+ state.payload = payload;
+ return builder;
+ },
+ update: (payload) => {
+ state.op = 'update';
+ state.payload = payload;
+ return builder;
+ },
+ delete: () => {
+ state.op = 'delete';
+ return builder;
+ },
+ then: (onFulfilled, onRejected) => Promise.resolve(run()).then(onFulfilled, onRejected),
+ };
+ return builder;
+ }
+ return { db: { from }, rows, writes };
+}
diff --git a/server/src/lib/catalog/sources.js b/server/src/lib/catalog/sources.js
new file mode 100644
index 000000000..7f1a3e229
--- /dev/null
+++ b/server/src/lib/catalog/sources.js
@@ -0,0 +1,146 @@
+import supabaseAdmin from '../../config/supabase.js';
+import { isSubscriptionActive } from '../../config/plans.js';
+import { connectorFor } from '../../connectors/index.js';
+import { FeedFormatError, parseFeed } from '../feeds/index.js';
+import { PageFetchError } from '../safe-fetch.js';
+import { logger } from '../logger.js';
+import { syncCatalog } from './sync.js';
+
+/**
+ * A brand's catalog sources (#1263, 00279) synced into its catalog: a feed
+ * URL, read through its connector, daily and on demand; or an uploaded CSV.
+ * Each source keeps its own products under `sourceKey(id)`, and its last
+ * sync's outcome on its row (`last_*`), which the Settings card shows.
+ */
+
+/** Feeds read again after this long, by the daily run. */
+export const SYNC_EVERY_MS = 20 * 60 * 60 * 1000;
+/** Feeds a daily run reads at most, the longest waiting first, one at a time. */
+export const SWEEP_LIMIT = 50;
+/** A run stops starting feeds after this; the rest wait first in line for the next. */
+export const SWEEP_DEADLINE_MS = 10 * 60 * 1000;
+/** Skipped products' errors kept on the source, for the person to see. */
+const ERRORS_KEPT = 5;
+
+/** The catalog's `source` for a catalog source: `src_` and its id without dashes. */
+export function sourceKey(id) {
+ return `src_${String(id).replace(/-/g, '')}`;
+}
+
+/** Why a sync failed, in words a person can act on. No page or feed text. */
+export function failureOf(err) {
+ if (err instanceof FeedFormatError) return { status: 'format_error', error: err.message };
+ if (err instanceof PageFetchError) {
+ const words = {
+ invalid_url: "The feed's address isn't a valid http(s) address.",
+ blocked: "The feed's address isn't a public one.",
+ unreachable: "The feed's address couldn't be reached.",
+ timeout: 'The feed took too long to answer.',
+ too_large: 'The feed is larger than 50 MB.',
+ not_html: "The address doesn't serve a feed (XML or CSV).",
+ http_error: `The feed's address answered ${err.status ?? 'an error'}.`,
+ too_many_redirects: "The feed's address redirects too many times.",
+ };
+ return { status: 'failed', error: words[err.code] ?? "The feed couldn't be read." };
+ }
+ return { status: 'failed', error: "The feed couldn't be synced. Try again later." };
+}
+
+async function record(db, id, at, outcome) {
+ const { error } = await db
+ .from('catalog_sources')
+ .update({
+ last_synced_at: at,
+ last_status: outcome.status,
+ last_error: outcome.error ?? null,
+ last_counts: outcome.counts ?? null,
+ })
+ .eq('id', id);
+ if (error) throw new Error(`catalog_sources write: ${error.message}`);
+}
+
+/**
+ * Syncs one source and records how it went. A feed is read through its
+ * connector; an upload's text is passed in. Never throws for the source's
+ * own failure: it's recorded and returned.
+ *
+ * @param {{ id: string, brand_id: string, kind: string, format: string, url?: string|null }} source
+ * @param {{ text?: string, db?: object, now?: Date, fetch?: Function }} [options]
+ */
+export async function syncSource(
+ source,
+ { text, db = supabaseAdmin, now = new Date(), fetch } = {},
+) {
+ const at = now.toISOString();
+ let outcome;
+ try {
+ let parsed;
+ if (text !== undefined) {
+ parsed = parseFeed(text, source.format);
+ } else {
+ const connector = connectorFor(source.kind);
+ if (!connector) throw new Error(`No connector reads ${source.kind}`);
+ parsed = await connector.readCatalog(source, { fetch });
+ }
+ const counts = await syncCatalog(source.brand_id, sourceKey(source.id), parsed.items, {
+ db,
+ now,
+ });
+ outcome = {
+ status: 'ok',
+ counts: {
+ format: parsed.format,
+ items: parsed.items.length,
+ skipped: parsed.errors.length,
+ errors: parsed.errors.slice(0, ERRORS_KEPT).map((e) => e.message),
+ products: counts.products,
+ variants: counts.variants,
+ },
+ };
+ } catch (err) {
+ outcome = failureOf(err);
+ if (outcome.status === 'failed' && !(err instanceof PageFetchError)) {
+ logger.error(
+ { err: { name: err?.name, message: err?.message }, sourceId: source.id },
+ '[catalog] sync failed',
+ );
+ }
+ }
+ await record(db, source.id, at, outcome);
+ return outcome;
+}
+
+/**
+ * The daily run: feed sources not read for SYNC_EVERY_MS, the longest
+ * waiting first, SWEEP_LIMIT a run, one at a time, until SWEEP_DEADLINE_MS.
+ * On the managed cloud, only workspaces with an active subscription.
+ */
+export async function sweepCatalogSources({ db = supabaseAdmin, now = new Date(), fetch } = {}) {
+ const started = Date.now();
+ const due = new Date(now.getTime() - SYNC_EVERY_MS).toISOString();
+ const { data, error } = await db
+ .from('catalog_sources')
+ .select(
+ 'id, brand_id, kind, format, url, last_synced_at, brands!inner(organizations!inner(subscription_status))',
+ )
+ .eq('kind', 'feed_url')
+ .or(`last_synced_at.is.null,last_synced_at.lt.${due}`)
+ .order('last_synced_at', { ascending: true, nullsFirst: true })
+ .limit(SWEEP_LIMIT);
+ if (error) throw new Error(`catalog_sources read: ${error.message}`);
+ let synced = 0;
+ for (const source of data ?? []) {
+ if (Date.now() - started > SWEEP_DEADLINE_MS) break;
+ if (!isSubscriptionActive(source.brands?.organizations?.subscription_status)) continue;
+ try {
+ await syncSource(source, { db, now, fetch });
+ synced++;
+ } catch (err) {
+ logger.error(
+ { err: { name: err?.name, message: err?.message }, sourceId: source.id },
+ '[catalog] sweep source failed',
+ );
+ }
+ }
+ return { due: (data ?? []).length, synced };
+}
diff --git a/server/src/lib/catalog/sources.test.js b/server/src/lib/catalog/sources.test.js
new file mode 100644
index 000000000..ff3e66d6e
--- /dev/null
+++ b/server/src/lib/catalog/sources.test.js
@@ -0,0 +1,139 @@
+import { readFileSync } from 'node:fs';
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
+
+// The module reaches the supabase admin client at import time, whose config
+// hard-exits when SUPABASE_* env is absent (as in CI).
+vi.mock('../../config/supabase.js', () => ({ default: {} }));
+
+import { PageFetchError } from '../safe-fetch.js';
+import { fakeDb } from './__fixtures__/fake-db.js';
+import { FEED_MAX_BYTES, FEED_TYPES } from '../../connectors/feed-url/index.js';
+import { sourceKey, sweepCatalogSources, syncSource } from './sources.js';
+
+const fixture = (name) =>
+ readFileSync(new URL(`../feeds/__fixtures__/${name}`, import.meta.url), 'utf8');
+
+const SOURCE_ID = '0b8d6a1e-2a3c-4d5e-8f90-123456789abc';
+const NOW = new Date('2026-10-25T09:00:00Z');
+
+let fake;
+let source;
+beforeEach(() => {
+ fake = fakeDb();
+ source = {
+ id: SOURCE_ID,
+ brand_id: 'brand-1',
+ kind: 'feed_url',
+ format: 'auto',
+ url: 'https://shop.example/feed.xml',
+ };
+ fake.rows('catalog_sources').push({ ...source, last_synced_at: null });
+});
+afterEach(() => {
+ delete process.env.IS_CLOUD;
+});
+
+const kept = () => fake.rows('catalog_sources').find((s) => s.id === SOURCE_ID);
+
+describe('sourceKey', () => {
+ it("is the catalog's source: src_ and the id without dashes, as 00279 keys it", () => {
+ expect(sourceKey(SOURCE_ID)).toBe('src_0b8d6a1e2a3c4d5e8f90123456789abc');
+ expect(sourceKey(SOURCE_ID)).toMatch(/^[a-z][a-z0-9_]{1,39}$/);
+ });
+});
+
+describe('syncSource', () => {
+ it('reads a feed through its connector, safely, and records how it went', async () => {
+ const fetch = vi.fn().mockResolvedValue({ html: fixture('google-feed.xml') });
+ const outcome = await syncSource(source, { db: fake.db, now: NOW, fetch });
+ expect(fetch).toHaveBeenCalledWith('https://shop.example/feed.xml', {
+ timeoutMs: 60_000,
+ maxBytes: FEED_MAX_BYTES,
+ types: FEED_TYPES,
+ accept: expect.stringContaining('application/xml'),
+ });
+ expect(outcome).toMatchObject({
+ status: 'ok',
+ counts: {
+ format: 'google_xml',
+ items: 4,
+ skipped: 0,
+ products: { added: 2 },
+ variants: { added: 4 },
+ },
+ });
+ expect(kept()).toMatchObject({
+ last_synced_at: NOW.toISOString(),
+ last_status: 'ok',
+ last_error: null,
+ });
+ // Its products are kept under its own key.
+ expect(new Set(fake.rows('catalog_products').map((p) => p.source))).toEqual(
+ new Set([sourceKey(SOURCE_ID)]),
+ );
+ });
+
+ it("imports an upload's text, with the products it skipped and why", async () => {
+ const outcome = await syncSource(
+ { ...source, kind: 'upload', url: null },
+ { db: fake.db, now: NOW, text: fixture('store-export.csv') },
+ );
+ expect(outcome.counts).toMatchObject({
+ format: 'csv',
+ items: 2,
+ skipped: 1,
+ errors: ['Row 3: price is missing.'],
+ });
+ });
+
+ it("records a feed that couldn't be fetched, in plain words, and writes no product", async () => {
+ const fetch = vi
+ .fn()
+ .mockRejectedValue(
+ new PageFetchError('http_error', 'The page answered 404', { status: 404 }),
+ );
+ const outcome = await syncSource(source, { db: fake.db, now: NOW, fetch });
+ expect(outcome).toEqual({ status: 'failed', error: "The feed's address answered 404." });
+ expect(kept()).toMatchObject({ last_status: 'failed', last_counts: null });
+ expect(fake.rows('catalog_products')).toEqual([]);
+ });
+
+ it("records a file that isn't a feed as a format error, and keeps the products it had", async () => {
+ await syncSource(source, {
+ db: fake.db,
+ now: NOW,
+ fetch: async () => ({ html: fixture('google-feed.xml') }),
+ });
+ const outcome = await syncSource(source, {
+ db: fake.db,
+ now: NOW,
+ fetch: async () => ({ html: 'name,email\nAda,[email protected]' }),
+ });
+ expect(outcome.status).toBe('format_error');
+ expect(outcome.error).toMatch(/no id, link, price, availability column/);
+ // A broken feed doesn't mark the catalog removed.
+ expect(fake.rows('catalog_products').every((p) => p.removed_at === null)).toBe(true);
+ });
+});
+
+describe('sweepCatalogSources', () => {
+ it('syncs the due feeds, skipping workspaces without an active subscription on the cloud', async () => {
+ process.env.IS_CLOUD = 'true';
+ const [first] = fake.rows('catalog_sources');
+ first.brands = { organizations: { subscription_status: 'active' } };
+ fake.rows('catalog_sources').push({
+ id: '1b8d6a1e-2a3c-4d5e-8f90-123456789abc',
+ brand_id: 'brand-2',
+ kind: 'feed_url',
+ format: 'auto',
+ url: 'https://other.example/feed.xml',
+ last_synced_at: null,
+ brands: { organizations: { subscription_status: 'canceled' } },
+ });
+ const fetch = vi.fn().mockResolvedValue({ html: fixture('google-feed.xml') });
+ const result = await sweepCatalogSources({ db: fake.db, now: NOW, fetch });
+ expect(result).toEqual({ due: 2, synced: 1 });
+ expect(fetch).toHaveBeenCalledTimes(1);
+ expect(fetch.mock.calls[0][0]).toBe('https://shop.example/feed.xml');
+ });
+});
diff --git a/server/src/lib/catalog/sync.test.js b/server/src/lib/catalog/sync.test.js
index e248befef..d6b4481bc 100644
--- a/server/src/lib/catalog/sync.test.js
+++ b/server/src/lib/catalog/sync.test.js
@@ -1,4 +1,3 @@
-import { randomUUID } from 'node:crypto';
import { readFileSync } from 'node:fs';
import { beforeEach, describe, expect, it, vi } from 'vitest';
@@ -7,6 +6,7 @@ import { beforeEach, describe, expect, it, vi } from 'vitest';
vi.mock('../../config/supabase.js', () => ({ default: {} }));
import { parseCsvFeed, parseGoogleFeedXml } from '../feeds/index.js';
+import { fakeDb } from './__fixtures__/fake-db.js';
import { catalogRecords, groupTitle, syncCatalog } from './sync.js';
const BRAND = 'brand-1';
@@ -15,119 +15,6 @@ const feed = () =>
readFileSync(new URL('../feeds/__fixtures__/google-feed.xml', import.meta.url), 'utf8'),
).items;
-/**
- * An in-memory stand-in for the tables the sync writes, with PostgREST's
- * calls as the sync makes them, counting every write.
- */
-function fakeDb() {
- const tables = new Map();
- const rows = (name) => {
- if (!tables.has(name)) tables.set(name, []);
- return tables.get(name);
- };
- const writes = [];
- const uniqueKey = (table, row) =>
- table === 'catalog_categories'
- ? `${row.brand_id}|${row.name}`
- : `${row.brand_id}|${row.source}|${row.external_id}`;
- // The unique keys' rows, by table, as the database's unique indexes find them.
- const byKey = new Map();
- const keyed = (table) => {
- if (!byKey.has(table)) byKey.set(table, new Map());
- return byKey.get(table);
- };
-
- function from(table) {
- const state = { filters: [], range: null, op: 'select', payload: null, returning: null };
- const run = () => {
- const all = rows(table);
- const match = (row) =>
- state.filters.every(([kind, column, value]) =>
- kind === 'eq' ? row[column] === value : value.includes(row[column]),
- );
- if (state.op === 'select') {
- const found = all.filter(match).sort((a, b) => (a.id < b.id ? -1 : 1));
- const [start, end] = state.range ?? [0, found.length];
- return { data: found.slice(start, end + 1), error: null };
- }
- writes.push({
- table,
- op: state.op,
- count: Array.isArray(state.payload) ? state.payload.length : 1,
- });
- if (state.op === 'upsert') {
- const out = [];
- for (const values of state.payload) {
- const key = uniqueKey(table, values);
- let row = keyed(table).get(key);
- if (row) Object.assign(row, values);
- else {
- row = { id: randomUUID(), removed_at: null, ...values };
- all.push(row);
- keyed(table).set(key, row);
- }
- out.push(row);
- }
- return { data: out, error: null };
- }
- if (state.op === 'insert') {
- for (const values of state.payload) all.push({ id: randomUUID(), ...values });
- return { data: null, error: null };
- }
- if (state.op === 'update') {
- for (const row of all.filter(match)) Object.assign(row, state.payload);
- return { data: null, error: null };
- }
- if (state.op === 'delete') {
- tables.set(
- table,
- all.filter((row) => !match(row)),
- );
- return { data: null, error: null };
- }
- throw new Error(`unexpected ${state.op}`);
- };
- const builder = {
- select: () => builder,
- eq: (column, value) => {
- state.filters.push(['eq', column, value]);
- return builder;
- },
- in: (column, values) => {
- state.filters.push(['in', column, values]);
- return builder;
- },
- order: () => builder,
- range: (start, end) => {
- state.range = [start, end];
- return builder;
- },
- upsert: (payload) => {
- state.op = 'upsert';
- state.payload = payload;
- return builder;
- },
- insert: (payload) => {
- state.op = 'insert';
- state.payload = payload;
- return builder;
- },
- update: (payload) => {
- state.op = 'update';
- state.payload = payload;
- return builder;
- },
- delete: () => {
- state.op = 'delete';
- return builder;
- },
- then: (onFulfilled, onRejected) => Promise.resolve(run()).then(onFulfilled, onRejected),
- };
- return builder;
- }
- return { db: { from }, rows, writes };
-}
-
let fake;
beforeEach(() => {
fake = fakeDb();
diff --git a/server/src/lib/feeds/feeds.test.js b/server/src/lib/feeds/feeds.test.js
index 6635e6c50..b75e1b2b7 100644
--- a/server/src/lib/feeds/feeds.test.js
+++ b/server/src/lib/feeds/feeds.test.js
@@ -3,8 +3,10 @@ import { describe, expect, it } from 'vitest';
import { csvRows } from './csv.js';
import {
FeedFormatError,
+ detectFormat,
parseCsvFeed,
parseFacebookCatalogCsv,
+ parseFeed,
parseGoogleFeedXml,
} from './index.js';
import { parseMoney } from './item.js';
@@ -214,3 +216,21 @@ describe('parseCsvFeed', () => {
});
});
});
+
+describe('detectFormat and parseFeed', () => {
+ it('reads XML, a Meta catalog CSV and a plain CSV by their text', () => {
+ expect(detectFormat(fixture('google-feed.xml'))).toBe('google_xml');
+ expect(detectFormat(fixture('google-feed-atom.xml'))).toBe('google_xml');
+ expect(detectFormat(fixture('facebook-catalog.csv'))).toBe('meta_csv');
+ expect(detectFormat(fixture('store-export.csv'))).toBe('csv');
+ });
+
+ it('parses in the format named, or the one it reads, and says which', () => {
+ const auto = parseFeed(fixture('facebook-catalog.csv'));
+ expect(auto.format).toBe('meta_csv');
+ expect(auto.items).toHaveLength(4);
+ expect(parseFeed(fixture('store-export.csv'), 'csv').format).toBe('csv');
+ expect(() => parseFeed(' ')).toThrow('The file is empty.');
+ expect(() => parseFeed(fixture('store-export.csv'), 'google_xml')).toThrow(FeedFormatError);
+ });
+});
diff --git a/server/src/lib/feeds/index.js b/server/src/lib/feeds/index.js
index c17d9ceaa..3d1c9c6cc 100644
--- a/server/src/lib/feeds/index.js
+++ b/server/src/lib/feeds/index.js
@@ -135,6 +135,44 @@ export function parseFacebookCatalogCsv(text) {
return parseCsv(text, undefined, { description: 'rich_text_description' });
}
+/**
+ * The format a feed's text is in: XML when it starts with a tag, a Meta
+ * catalog CSV when its header has Meta's own columns, else a plain CSV.
+ *
+ * @param {string} text
+ * @returns {'google_xml' | 'meta_csv' | 'csv'}
+ */
+export function detectFormat(text) {
+ const start = text.replace(/^\uFEFF/, '').trimStart();
+ if (start.startsWith('<')) return 'google_xml';
+ const header = start.slice(0, start.search(/\r?\n|$/)).toLowerCase();
+ return /(^|[,\t"])(fb_product_category|rich_text_description|quantity_to_sell_on_facebook)\b/.test(
+ header,
+ )
+ ? 'meta_csv'
+ : 'csv';
+}
+
+/**
+ * A feed in any of the formats read here: `format` names it, or 'auto'
+ * reads it from the text (`detectFormat`).
+ *
+ * @param {string} text
+ * @param {'auto' | 'google_xml' | 'csv' | 'meta_csv'} [format]
+ * @returns {import('./item.js').FeedResult & { format: string }}
+ */
+export function parseFeed(text, format = 'auto') {
+ if (typeof text !== 'string' || !text.trim()) throw new FeedFormatError('The file is empty.');
+ const read = format === 'auto' ? detectFormat(text) : format;
+ const result =
+ read === 'google_xml'
+ ? parseGoogleFeedXml(text)
+ : read === 'meta_csv'
+ ? parseFacebookCatalogCsv(text)
+ : parseCsvFeed(text);
+ return { ...result, format: read };
+}
+
const ITEM = /<item(?:\s[^>]*)?>([\s\S]*?)<\/item\s*>/g;
const ENTRY = /<entry(?:\s[^>]*)?>([\s\S]*?)<\/entry\s*>/g;
diff --git a/server/src/lib/safe-fetch.js b/server/src/lib/safe-fetch.js
index 83c09ea66..5d5f46968 100644
--- a/server/src/lib/safe-fetch.js
+++ b/server/src/lib/safe-fetch.js
@@ -27,6 +27,10 @@ export const FETCH_TIMEOUT_MS = 10_000;
export const MAX_PAGE_BYTES = 3 * 1024 * 1024;
export const MAX_REDIRECTS = 5;
+/** The content types a web page comes as, and what a request asks for. */
+const HTML_TYPES = /^\s*(text\/html|application\/xhtml\+xml)\b/i;
+const HTML_ACCEPT = 'text/html,application/xhtml+xml;q=0.9,*/*;q=0.1';
+
const USER_AGENT = 'Mozilla/5.0 (compatible; AnsvisorBot/1.0; +https://ansvisor.com)';
export class PageFetchError extends Error {
@@ -160,7 +164,7 @@ function decode(buffer, contentType) {
* One request, no redirects followed: the response, or where it redirects to.
* A body over `maxBytes` is refused, or with `truncate` cut off there.
*/
-function requestOnce(url, { signal, lookup, maxBytes, truncate }) {
+function requestOnce(url, { signal, lookup, maxBytes, truncate, types, accept }) {
return new Promise((resolve, reject) => {
const client = url.protocol === 'https:' ? https : http;
const req = client.request(
@@ -172,7 +176,7 @@ function requestOnce(url, { signal, lookup, maxBytes, truncate }) {
signal,
headers: {
'user-agent': USER_AGENT,
- accept: 'text/html,application/xhtml+xml;q=0.9,*/*;q=0.1',
+ accept,
'accept-encoding': 'gzip, deflate, br',
},
},
@@ -190,7 +194,7 @@ function requestOnce(url, { signal, lookup, maxBytes, truncate }) {
return;
}
const contentType = res.headers['content-type'] ?? '';
- if (!/^\s*(text\/html|application\/xhtml\+xml)\b/i.test(contentType)) {
+ if (!types.test(contentType)) {
res.destroy();
reject(new PageFetchError('not_html', `The address is ${contentType || 'not a page'}`));
return;
@@ -236,12 +240,14 @@ function requestOnce(url, { signal, lookup, maxBytes, truncate }) {
* A web page's HTML, fetched safely (see above).
*
* @param {string} address
- * @param {{ timeoutMs?: number, maxBytes?: number, truncate?: boolean, maxRedirects?: number, lookup?: Function, ports?: string[], signal?: AbortSignal }} [opts]
+ * @param {{ timeoutMs?: number, maxBytes?: number, truncate?: boolean, maxRedirects?: number, lookup?: Function, ports?: string[], signal?: AbortSignal, types?: RegExp, accept?: string }} [opts]
* `signal` ends the read early too (a run's Stop), as its timeout does.
* `truncate` reads a page over `maxBytes` up to it, instead of refusing it
* ('too_large'), for a caller that needs only its start.
* `lookup` and `ports` replace the public-only lookup and the web ports;
* tests use them to reach a local server.
+ * `types` and `accept` read something other than a web page (a product
+ * feed): the content types taken, else 'not_html', and the Accept header.
* @returns {Promise<{ url: string, status: number, html: string }>} url is the address after redirects
*/
export async function fetchPage(address, opts = {}) {
@@ -249,6 +255,8 @@ export async function fetchPage(address, opts = {}) {
timeoutMs = FETCH_TIMEOUT_MS,
maxBytes = MAX_PAGE_BYTES,
truncate = false,
+ types = HTML_TYPES,
+ accept = HTML_ACCEPT,
maxRedirects = MAX_REDIRECTS,
lookup = publicOnlyLookup,
ports = WEB_PORTS,
@@ -261,7 +269,7 @@ export async function fetchPage(address, opts = {}) {
for (let hops = 0; ; hops += 1) {
let res;
try {
- res = await requestOnce(url, { signal, lookup, maxBytes, truncate });
+ res = await requestOnce(url, { signal, lookup, maxBytes, truncate, types, accept });
} catch (err) {
if (err instanceof PageFetchError) throw err;
if (signal.aborted) throw new PageFetchError('timeout', 'The page took too long to answer');
diff --git a/server/src/lib/safe-fetch.test.js b/server/src/lib/safe-fetch.test.js
index 6e20d81ce..3aec610f3 100644
--- a/server/src/lib/safe-fetch.test.js
+++ b/server/src/lib/safe-fetch.test.js
@@ -127,6 +127,10 @@ describe('fetchPage against a local site', () => {
res.writeHead(200, { 'content-type': 'application/pdf' });
res.end('%PDF');
},
+ '/feed.xml': (res) => {
+ res.writeHead(200, { 'content-type': 'application/xml; charset=utf-8' });
+ res.end('<rss version="2.0"><channel></channel></rss>');
+ },
'/missing': (res) => {
res.writeHead(404, { 'content-type': 'text/html' });
res.end('nope');
@@ -194,6 +198,13 @@ describe('fetchPage against a local site', () => {
await expect(get('/slow', { timeoutMs: 50 })).rejects.toMatchObject({ code: 'timeout' });
});
+ it('reads another content type only when asked to', async () => {
+ await expect(get('/feed.xml')).rejects.toMatchObject({ code: 'not_html' });
+ await expect(
+ get('/feed.xml', { types: /^\s*application\/xml\b/i, accept: 'application/xml' }),
+ ).resolves.toMatchObject({ html: '<rss version="2.0"><channel></channel></rss>' });
+ });
+
it('reads the start of a page over the cap when asked to, instead of refusing it', async () => {
await expect(get('/big', { maxBytes: 1_000, truncate: true })).resolves.toMatchObject({
status: 200,
diff --git a/server/src/middleware/rate-limiter.js b/server/src/middleware/rate-limiter.js
index 62f22904a..7553b8e83 100644
--- a/server/src/middleware/rate-limiter.js
+++ b/server/src/middleware/rate-limiter.js
@@ -117,6 +117,20 @@ export const pageReadLimiter = rateLimit({
// A brand's facts read from its site again when asked (Quality CE-047): a
// few model calls, so its own bucket.
+// Catalog syncs and uploads (#1263): each reads a feed or a file of up to
+// 50 MB / 4 MB, so a member's are few.
+export const catalogLimiter = rateLimit({
+ windowMs: 15 * 60 * 1000,
+ max: 20,
+ keyGenerator: apiLimiterKey,
+ standardHeaders: true,
+ legacyHeaders: false,
+ message: {
+ success: false,
+ message: 'Too many requests, please try again later.',
+ },
+});
+
export const brandFactsLimiter = rateLimit({
windowMs: 15 * 60 * 1000,
max: 30,
diff --git a/server/src/routes/catalog.js b/server/src/routes/catalog.js
new file mode 100644
index 000000000..8cb18d8e0
--- /dev/null
+++ b/server/src/routes/catalog.js
@@ -0,0 +1,136 @@
+import express, { Router } from 'express';
+import { z } from 'zod';
+import supabaseAdmin from '../config/supabase.js';
+import { assertBrandAccess } from '../lib/access.js';
+import { syncSource } from '../lib/catalog/sources.js';
+import { catalogLimiter } from '../middleware/rate-limiter.js';
+
+/**
+ * A brand's catalog sources (#1263, 00279). Members see them through RLS;
+ * admins and managers add and remove feed URLs there too. The server syncs:
+ * - POST /api/catalog/sources/:id/sync: a feed read now;
+ * - POST /api/catalog/upload?brandId=…&format=…: a CSV (or XML) file as
+ * the request's body, kept as the brand's upload source.
+ */
+
+const router = Router();
+
+const WRITE_ROLES = ['admin', 'manager'];
+/** What the API's proxy takes in one request; a larger catalog is a feed URL. */
+export const UPLOAD_MAX = '4mb';
+/** A feed synced this recently isn't read again on demand. */
+const RESYNC_AFTER_MS = 2 * 60 * 1000;
+const FORMATS = ['auto', 'google_xml', 'csv', 'meta_csv'];
+
+async function assertWriter(brandId, userId) {
+ await assertBrandAccess(brandId, userId);
+ const { data: profile } = await supabaseAdmin
+ .from('profiles')
+ .select('role')
+ .eq('id', userId)
+ .single();
+ if (!WRITE_ROLES.includes(profile?.role)) {
+ const err = new Error('Only admins and managers can sync the catalog.');
+ err.status = 403;
+ throw err;
+ }
+}
+
+function fail(req, res, err, what) {
+ if (err?.status === 403 || err?.status === 404) {
+ return res.status(err.status).json({
+ error: err.message,
+ code:
+ err.code === 'not_in_workspace'
+ ? 'not_in_workspace'
+ : err.status === 404
+ ? 'not_found'
+ : 'forbidden',
+ });
+ }
+ req.log.error({ err: { name: err?.name, message: err?.message } }, what);
+ return res.status(500).json({ error: 'Failed to sync the catalog', code: 'failed' });
+}
+
+const outcomeBody = (outcome) => ({
+ status: outcome.status,
+ error: outcome.error ?? null,
+ counts: outcome.counts ?? null,
+});
+
+// POST /api/catalog/sources/:id/sync — read a feed now.
+router.post('/sources/:id/sync', catalogLimiter, async (req, res) => {
+ const parsed = z.guid().safeParse(req.params.id);
+ if (!parsed.success) return res.status(400).json({ error: 'invalid', code: 'invalid' });
+ try {
+ const { data: source } = await supabaseAdmin
+ .from('catalog_sources')
+ .select('id, brand_id, kind, format, url, last_synced_at')
+ .eq('id', parsed.data)
+ .maybeSingle();
+ if (!source || source.kind !== 'feed_url') {
+ return res.status(404).json({ error: 'Not found', code: 'not_found' });
+ }
+ await assertWriter(source.brand_id, req.user.id);
+ if (source.last_synced_at && Date.now() - Date.parse(source.last_synced_at) < RESYNC_AFTER_MS) {
+ return res.status(409).json({ error: 'Synced a moment ago', code: 'recent' });
+ }
+ return res.json(outcomeBody(await syncSource(source)));
+ } catch (err) {
+ return fail(req, res, err, 'catalog sync error');
+ }
+});
+
+// POST /api/catalog/upload?brandId=…&format=… — a file's text as the body.
+router.post(
+ '/upload',
+ catalogLimiter,
+ express.text({ type: () => true, limit: UPLOAD_MAX }),
+ async (req, res) => {
+ const parsed = z
+ .object({ brandId: z.guid(), format: z.enum(FORMATS).default('auto') })
+ .safeParse(req.query);
+ if (!parsed.success || typeof req.body !== 'string' || !req.body.trim()) {
+ return res.status(400).json({ error: 'invalid', code: 'invalid' });
+ }
+ const { brandId, format } = parsed.data;
+ try {
+ await assertWriter(brandId, req.user.id);
+ // The brand's one upload source: made on its first upload, its format kept.
+ let { data: source } = await supabaseAdmin
+ .from('catalog_sources')
+ .select('id, brand_id, kind, format, url')
+ .eq('brand_id', brandId)
+ .eq('kind', 'upload')
+ .maybeSingle();
+ if (!source) {
+ const { data, error } = await supabaseAdmin
+ .from('catalog_sources')
+ .insert({ brand_id: brandId, kind: 'upload', format, created_by: req.user.id })
+ .select('id, brand_id, kind, format, url')
+ .single();
+ if (error?.code === '23514') {
+ return res.status(409).json({ error: error.message, code: 'too_many' });
+ }
+ if (error) throw new Error(error.message);
+ source = data;
+ } else if (source.format !== format) {
+ await supabaseAdmin.from('catalog_sources').update({ format }).eq('id', source.id);
+ source = { ...source, format };
+ }
+ return res.json(outcomeBody(await syncSource(source, { text: req.body })));
+ } catch (err) {
+ return fail(req, res, err, 'catalog upload error');
+ }
+ },
+);
+
+// A body over UPLOAD_MAX: said, not a 500.
+router.use((err, req, res, next) => {
+ if (err?.type === 'entity.too.large') {
+ return res.status(413).json({ error: 'The file is larger than 4 MB', code: 'too_large' });
+ }
+ return next(err);
+});
+
+export default router;
diff --git a/server/src/routes/catalog.test.js b/server/src/routes/catalog.test.js
new file mode 100644
index 000000000..588d00b3a
--- /dev/null
+++ b/server/src/routes/catalog.test.js
@@ -0,0 +1,177 @@
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
+import express from 'express';
+
+/**
+ * A brand's catalog sources (#1263): a feed synced now, a file uploaded;
+ * admins and managers only, a brand of their own workspace only.
+ */
+
+const BRAND = '33333333-3333-3333-3333-333333333333';
+const SOURCE = '0b8d6a1e-2a3c-4d5e-8f90-123456789abc';
+let role;
+let sources;
+let inserted;
+const assertBrandAccess = vi.fn();
+const syncSource = vi.fn();
+
+vi.mock('../config/supabase.js', () => ({
+ default: {
+ from: (table) => {
+ const filters = {};
+ const chain = {
+ select: () => chain,
+ eq: (column, value) => {
+ filters[column] = value;
+ return chain;
+ },
+ single: async () =>
+ table === 'profiles'
+ ? { data: { role }, error: null }
+ : { data: inserted, error: inserted ? null : { message: 'no' } },
+ maybeSingle: async () => ({
+ data: sources.find((s) => Object.entries(filters).every(([k, v]) => s[k] === v)) ?? null,
+ error: null,
+ }),
+ insert: (row) => {
+ inserted = { id: 'new-upload', url: null, ...row };
+ sources.push(inserted);
+ return chain;
+ },
+ update: (values) => {
+ chain.then = (onFulfilled) => {
+ const row = sources.find((s) => s.id === filters.id);
+ if (row) Object.assign(row, values);
+ return Promise.resolve({ error: null }).then(onFulfilled);
+ };
+ return chain;
+ },
+ };
+ return chain;
+ },
+ },
+}));
+vi.mock('../lib/access.js', () => ({
+ assertBrandAccess: (...args) => assertBrandAccess(...args),
+}));
+vi.mock('../lib/catalog/sources.js', () => ({
+ syncSource: (...args) => syncSource(...args),
+}));
+
+const { default: router } = await import('./catalog.js');
+
+let server;
+let base;
+beforeEach(async () => {
+ role = 'admin';
+ inserted = null;
+ sources = [
+ {
+ id: SOURCE,
+ brand_id: BRAND,
+ kind: 'feed_url',
+ format: 'auto',
+ url: 'https://shop.example/feed.xml',
+ last_synced_at: null,
+ },
+ ];
+ assertBrandAccess.mockReset().mockResolvedValue({ orgId: 'org-1' });
+ syncSource.mockReset().mockResolvedValue({ status: 'ok', counts: { items: 4 } });
+ const app = express();
+ app.use(express.json());
+ app.use((req, _res, next) => {
+ req.user = { id: 'user-1' };
+ req.log = { error: vi.fn() };
+ next();
+ });
+ app.use('/api/catalog', router);
+ await new Promise((resolve) => {
+ server = app.listen(0, '127.0.0.1', resolve);
+ });
+ base = `http://127.0.0.1:${server.address().port}/api/catalog`;
+});
+afterEach(() => new Promise((resolve) => server.close(resolve)));
+
+const sync = (id = SOURCE) => fetch(`${base}/sources/${id}/sync`, { method: 'POST' });
+const upload = (body, query = `brandId=${BRAND}`) =>
+ fetch(`${base}/upload?${query}`, {
+ method: 'POST',
+ headers: { 'content-type': 'text/csv' },
+ body,
+ });
+
+describe('POST /api/catalog/sources/:id/sync', () => {
+ it("syncs a feed for an admin or a manager of the brand's workspace", async () => {
+ const res = await sync();
+ expect(res.status).toBe(200);
+ expect(await res.json()).toEqual({ status: 'ok', error: null, counts: { items: 4 } });
+ expect(assertBrandAccess).toHaveBeenCalledWith(BRAND, 'user-1');
+ expect(syncSource).toHaveBeenCalledWith(expect.objectContaining({ id: SOURCE }));
+ });
+
+ it('refuses an analyst, another workspace, an upload source and a bad id', async () => {
+ role = 'analyst';
+ expect((await sync()).status).toBe(403);
+ role = 'manager';
+ assertBrandAccess.mockRejectedValueOnce(
+ Object.assign(new Error("This brand isn't in your workspace"), {
+ status: 403,
+ code: 'not_in_workspace',
+ }),
+ );
+ const other = await sync();
+ expect(other.status).toBe(403);
+ expect((await other.json()).code).toBe('not_in_workspace');
+ sources[0].kind = 'upload';
+ expect((await sync()).status).toBe(404);
+ expect((await sync('nope')).status).toBe(400);
+ expect(syncSource).not.toHaveBeenCalled();
+ });
+
+ it('says a feed synced a moment ago is fresh, without reading it again', async () => {
+ sources[0].last_synced_at = new Date(Date.now() - 30_000).toISOString();
+ const res = await sync();
+ expect(res.status).toBe(409);
+ expect((await res.json()).code).toBe('recent');
+ expect(syncSource).not.toHaveBeenCalled();
+ });
+});
+
+describe('POST /api/catalog/upload', () => {
+ it("makes the brand's upload source on its first upload, and imports the file's text", async () => {
+ const res = await upload('id,title\n1,Boot');
+ expect(res.status).toBe(200);
+ expect(inserted).toMatchObject({
+ brand_id: BRAND,
+ kind: 'upload',
+ format: 'auto',
+ created_by: 'user-1',
+ });
+ expect(syncSource).toHaveBeenCalledWith(expect.objectContaining({ kind: 'upload' }), {
+ text: 'id,title\n1,Boot',
+ });
+ });
+
+ it('keeps one upload source a brand, with the format asked for', async () => {
+ sources.push({ id: 'up-1', brand_id: BRAND, kind: 'upload', format: 'auto', url: null });
+ await upload('<rss/>', `brandId=${BRAND}&format=google_xml`);
+ expect(inserted).toBeNull();
+ expect(sources.find((s) => s.id === 'up-1').format).toBe('google_xml');
+ expect(syncSource).toHaveBeenCalledWith(
+ expect.objectContaining({ id: 'up-1', format: 'google_xml' }),
+ { text: '<rss/>' },
+ );
+ });
+
+ it('refuses an analyst, an empty body, a bad brand or format, and a file over 4 MB', async () => {
+ role = 'analyst';
+ expect((await upload('id\n1')).status).toBe(403);
+ role = 'admin';
+ expect((await upload(' ')).status).toBe(400);
+ expect((await upload('id\n1', 'brandId=nope')).status).toBe(400);
+ expect((await upload('id\n1', `brandId=${BRAND}&format=xlsx`)).status).toBe(400);
+ const big = await upload('x'.repeat(4 * 1024 * 1024 + 10));
+ expect(big.status).toBe(413);
+ expect((await big.json()).code).toBe('too_large');
+ expect(syncSource).not.toHaveBeenCalled();
+ });
+});
diff --git a/server/src/routes/index.js b/server/src/routes/index.js
index 3294b10ad..f99b936b2 100644
--- a/server/src/routes/index.js
+++ b/server/src/routes/index.js
@@ -16,6 +16,7 @@ import actionCenterRoutes from './action-center.js';
import botRoutes from './bots.js';
import billingRoutes from './billing.js';
import brandFactRoutes from './brand-facts.js';
+import catalogRoutes from './catalog.js';
const router = Router();
@@ -36,6 +37,7 @@ router.use('/action-center', actionCenterRoutes);
router.use('/bots', botRoutes);
router.use('/billing', billingRoutes);
router.use('/brand-facts', brandFactRoutes);
+router.use('/catalog', catalogRoutes);
router.get('/health', (req, res) => {
res.json({ status: 'ok', user: req.user?.id });
diff --git a/server/src/server.js b/server/src/server.js
index 3b32eb189..cb6bad217 100644
--- a/server/src/server.js
+++ b/server/src/server.js
@@ -28,6 +28,7 @@ import { rebuildQueuedCitationRollups, REBUILD_QUEUE_SCHEDULE } from './lib/cita
import { sweepActionValidation } from './lib/action-center/validate.js';
import { sweepImproveOutcomes } from './lib/action-center/improve-outcome.js';
import { sweepBrandFacts } from './lib/brand-facts-sweep.js';
+import { sweepCatalogSources } from './lib/catalog/sources.js';
import { parseScraperResponse } from './lib/cloro-scraper.js';
import { handleScraperResult } from './lib/cloro-result-handler.js';
import { verifyCloroWebhook } from './lib/cloro-webhook-verify.js';
@@ -284,6 +285,11 @@ async function runDailyTracking() {
logger.error({ err }, '[content-history] daily prune failed');
});
+ // Catalog feed URLs (#1263): read again once a day, never blocking tracking.
+ sweepCatalogSources().catch((err) => {
+ logger.error({ err }, '[catalog] daily sweep failed');
+ });
+
// Google Analytics sync (#704) — same contract as the Search Console sync
// above: independent, never blocks or fails the tracking run, no-ops when
// Composio or the Analytics auth config isn't set.
diff --git a/supabase/migrations/00279_catalog_sources.sql b/supabase/migrations/00279_catalog_sources.sql
new file mode 100644
index 000000000..42d52e59e
--- /dev/null
+++ b/supabase/migrations/00279_catalog_sources.sql
@@ -0,0 +1,154 @@
+-- ============================================================================
+-- 00279: where a brand's catalog comes from (#1263, part 2)
+-- ============================================================================
+-- A brand's catalog sources: a product feed URL the server reads every day
+-- (and on demand), or the CSV a person uploads. Each source's products are
+-- kept in the catalog (00278) under its own `source` key,
+-- `src_<the source's id without dashes>`, so two feeds of one brand never
+-- mark each other's products removed.
+--
+-- * Members read their brands' sources; admins and managers add, change
+-- and remove them (as the rest of a brand). A feed URL must be http(s);
+-- the server fetches it only to public addresses.
+-- * The last sync's outcome (`last_*`) is the server's to write.
+-- * At most 10 sources a brand, one upload source each.
+-- * Removing a source removes its products from the catalog: nothing of
+-- a source outlives it.
+
+create table if not exists public.catalog_sources (
+ id uuid primary key default gen_random_uuid(),
+ brand_id uuid not null references public.brands(id) on delete cascade,
+ kind text not null check (kind in ('feed_url', 'upload')),
+ format text not null default 'auto'
+ check (format in ('auto', 'google_xml', 'csv', 'meta_csv')),
+ url text check (url is null or (char_length(url) <= 2048 and url ~ '^https?://')),
+ label text check (label is null or char_length(label) between 1 and 200),
+ last_synced_at timestamptz,
+ last_status text check (last_status is null or last_status in ('ok', 'failed', 'format_error')),
+ last_error text check (last_error is null or char_length(last_error) <= 1000),
+ last_counts jsonb check (
+ last_counts is null
+ or (jsonb_typeof(last_counts) = 'object' and octet_length(last_counts::text) <= 4096)
+ ),
+ created_by uuid references public.profiles(id) on delete set null,
+ created_at timestamptz not null default now(),
+ constraint catalog_sources_url_kind check ((kind = 'feed_url') = (url is not null)),
+ unique (brand_id, url)
+);
+
+create unique index if not exists catalog_sources_one_upload
+ on public.catalog_sources (brand_id)
+ where kind = 'upload';
+create index if not exists catalog_sources_feed_due_idx
+ on public.catalog_sources (last_synced_at nulls first)
+ where kind = 'feed_url';
+
+alter table public.catalog_sources enable row level security;
+revoke all on table public.catalog_sources from anon, authenticated;
+grant select on table public.catalog_sources to authenticated;
+grant insert (brand_id, kind, format, url, label, created_by) on table public.catalog_sources to authenticated;
+grant update (format, label) on table public.catalog_sources to authenticated;
+grant delete on table public.catalog_sources to authenticated;
+grant all on table public.catalog_sources to service_role;
+
+drop policy if exists "catalog_sources: member select" on public.catalog_sources;
+create policy "catalog_sources: member select"
+ on public.catalog_sources
+ for select
+ using (
+ brand_id in (
+ select b.id from public.brands b
+ join public.profiles p on p.organization_id = b.organization_id
+ where p.id = auth.uid()
+ )
+ );
+
+drop policy if exists "catalog_sources: admin/manager insert" on public.catalog_sources;
+create policy "catalog_sources: admin/manager insert"
+ on public.catalog_sources
+ for insert
+ with check (
+ brand_id in (
+ select b.id from public.brands b
+ join public.profiles p on p.organization_id = b.organization_id
+ where p.id = auth.uid() and p.role = any (array['admin'::public.user_role, 'manager'::public.user_role])
+ )
+ and created_by = auth.uid()
+ );
+
+drop policy if exists "catalog_sources: admin/manager update" on public.catalog_sources;
+create policy "catalog_sources: admin/manager update"
+ on public.catalog_sources
+ for update
+ using (
+ brand_id in (
+ select b.id from public.brands b
+ join public.profiles p on p.organization_id = b.organization_id
+ where p.id = auth.uid() and p.role = any (array['admin'::public.user_role, 'manager'::public.user_role])
+ )
+ );
+
+drop policy if exists "catalog_sources: admin/manager delete" on public.catalog_sources;
+create policy "catalog_sources: admin/manager delete"
+ on public.catalog_sources
+ for delete
+ using (
+ brand_id in (
+ select b.id from public.brands b
+ join public.profiles p on p.organization_id = b.organization_id
+ where p.id = auth.uid() and p.role = any (array['admin'::public.user_role, 'manager'::public.user_role])
+ )
+ );
+
+-- At most 10 sources a brand. Serialized per brand, so two inserts at once
+-- can't both pass.
+create or replace function public.catalog_sources_cap()
+returns trigger
+language plpgsql
+set search_path = ''
+as $$
+begin
+ perform pg_advisory_xact_lock(hashtext('catalog_sources'), hashtext(new.brand_id::text));
+ if (select count(*) from public.catalog_sources where brand_id = new.brand_id) >= 10 then
+ raise exception 'A brand has 10 catalog sources at most' using errcode = '23514';
+ end if;
+ return new;
+end;
+$$;
+
+drop trigger if exists catalog_sources_cap on public.catalog_sources;
+create trigger catalog_sources_cap
+ before insert on public.catalog_sources
+ for each row execute function public.catalog_sources_cap();
+
+-- A removed source takes its products with it. Members can't write the
+-- catalog, so this runs as the table's owner; it touches only the removed
+-- source's own rows (its brand and its source key).
+create or replace function public.catalog_sources_forget()
+returns trigger
+language plpgsql
+security definer
+set search_path = ''
+as $$
+begin
+ delete from public.catalog_products
+ where brand_id = old.brand_id
+ and source = 'src_' || replace(old.id::text, '-', '');
+ delete from public.catalog_variants
+ where brand_id = old.brand_id
+ and source = 'src_' || replace(old.id::text, '-', '');
+ return old;
+end;
+$$;
+
+revoke all on function public.catalog_sources_forget() from public, anon, authenticated;
+
+drop trigger if exists catalog_sources_forget on public.catalog_sources;
+create trigger catalog_sources_forget
+ after delete on public.catalog_sources
+ for each row execute function public.catalog_sources_forget();
+
+comment on table public.catalog_sources is
+ 'Catalog (#1263, 00279): where a brand''s catalog comes from: a feed URL read daily, or an uploaded CSV. Its products are kept under source src_<id>. Members read; admins and managers add, change and remove; the server writes last_*. Removing one removes its products.';
+
+notify pgrst, 'reload schema';
diff --git a/supabase/schema.sql b/supabase/schema.sql
index 04a041f5a..1584564af 100644
--- a/supabase/schema.sql
+++ b/supabase/schema.sql
@@ -39642,3 +39642,161 @@ comment on table public.catalog_product_categories is
notify pgrst, 'reload schema';
+-- ─────────────────────────────────────────────────────────────────────────
+-- migrations/00279_catalog_sources.sql
+-- ─────────────────────────────────────────────────────────────────────────
+-- ============================================================================
+-- 00279: where a brand's catalog comes from (#1263, part 2)
+-- ============================================================================
+-- A brand's catalog sources: a product feed URL the server reads every day
+-- (and on demand), or the CSV a person uploads. Each source's products are
+-- kept in the catalog (00278) under its own `source` key,
+-- `src_<the source's id without dashes>`, so two feeds of one brand never
+-- mark each other's products removed.
+--
+-- * Members read their brands' sources; admins and managers add, change
+-- and remove them (as the rest of a brand). A feed URL must be http(s);
+-- the server fetches it only to public addresses.
+-- * The last sync's outcome (`last_*`) is the server's to write.
+-- * At most 10 sources a brand, one upload source each.
+-- * Removing a source removes its products from the catalog: nothing of
+-- a source outlives it.
+
+create table if not exists public.catalog_sources (
+ id uuid primary key default gen_random_uuid(),
+ brand_id uuid not null references public.brands(id) on delete cascade,
+ kind text not null check (kind in ('feed_url', 'upload')),
+ format text not null default 'auto'
+ check (format in ('auto', 'google_xml', 'csv', 'meta_csv')),
+ url text check (url is null or (char_length(url) <= 2048 and url ~ '^https?://')),
+ label text check (label is null or char_length(label) between 1 and 200),
+ last_synced_at timestamptz,
+ last_status text check (last_status is null or last_status in ('ok', 'failed', 'format_error')),
+ last_error text check (last_error is null or char_length(last_error) <= 1000),
+ last_counts jsonb check (
+ last_counts is null
+ or (jsonb_typeof(last_counts) = 'object' and octet_length(last_counts::text) <= 4096)
+ ),
+ created_by uuid references public.profiles(id) on delete set null,
+ created_at timestamptz not null default now(),
+ constraint catalog_sources_url_kind check ((kind = 'feed_url') = (url is not null)),
+ unique (brand_id, url)
+);
+
+create unique index if not exists catalog_sources_one_upload
+ on public.catalog_sources (brand_id)
+ where kind = 'upload';
+create index if not exists catalog_sources_feed_due_idx
+ on public.catalog_sources (last_synced_at nulls first)
+ where kind = 'feed_url';
+
+alter table public.catalog_sources enable row level security;
+revoke all on table public.catalog_sources from anon, authenticated;
+grant select on table public.catalog_sources to authenticated;
+grant insert (brand_id, kind, format, url, label, created_by) on table public.catalog_sources to authenticated;
+grant update (format, label) on table public.catalog_sources to authenticated;
+grant delete on table public.catalog_sources to authenticated;
+grant all on table public.catalog_sources to service_role;
+
+drop policy if exists "catalog_sources: member select" on public.catalog_sources;
+create policy "catalog_sources: member select"
+ on public.catalog_sources
+ for select
+ using (
+ brand_id in (
+ select b.id from public.brands b
+ join public.profiles p on p.organization_id = b.organization_id
+ where p.id = auth.uid()
+ )
+ );
+
+drop policy if exists "catalog_sources: admin/manager insert" on public.catalog_sources;
+create policy "catalog_sources: admin/manager insert"
+ on public.catalog_sources
+ for insert
+ with check (
+ brand_id in (
+ select b.id from public.brands b
+ join public.profiles p on p.organization_id = b.organization_id
+ where p.id = auth.uid() and p.role = any (array['admin'::public.user_role, 'manager'::public.user_role])
+ )
+ and created_by = auth.uid()
+ );
+
+drop policy if exists "catalog_sources: admin/manager update" on public.catalog_sources;
+create policy "catalog_sources: admin/manager update"
+ on public.catalog_sources
+ for update
+ using (
+ brand_id in (
+ select b.id from public.brands b
+ join public.profiles p on p.organization_id = b.organization_id
+ where p.id = auth.uid() and p.role = any (array['admin'::public.user_role, 'manager'::public.user_role])
+ )
+ );
+
+drop policy if exists "catalog_sources: admin/manager delete" on public.catalog_sources;
+create policy "catalog_sources: admin/manager delete"
+ on public.catalog_sources
+ for delete
+ using (
+ brand_id in (
+ select b.id from public.brands b
+ join public.profiles p on p.organization_id = b.organization_id
+ where p.id = auth.uid() and p.role = any (array['admin'::public.user_role, 'manager'::public.user_role])
+ )
+ );
+
+-- At most 10 sources a brand. Serialized per brand, so two inserts at once
+-- can't both pass.
+create or replace function public.catalog_sources_cap()
+returns trigger
+language plpgsql
+set search_path = ''
+as $$
+begin
+ perform pg_advisory_xact_lock(hashtext('catalog_sources'), hashtext(new.brand_id::text));
+ if (select count(*) from public.catalog_sources where brand_id = new.brand_id) >= 10 then
+ raise exception 'A brand has 10 catalog sources at most' using errcode = '23514';
+ end if;
+ return new;
+end;
+$$;
+
+drop trigger if exists catalog_sources_cap on public.catalog_sources;
+create trigger catalog_sources_cap
+ before insert on public.catalog_sources
+ for each row execute function public.catalog_sources_cap();
+
+-- A removed source takes its products with it. Members can't write the
+-- catalog, so this runs as the table's owner; it touches only the removed
+-- source's own rows (its brand and its source key).
+create or replace function public.catalog_sources_forget()
+returns trigger
+language plpgsql
+security definer
+set search_path = ''
+as $$
+begin
+ delete from public.catalog_products
+ where brand_id = old.brand_id
+ and source = 'src_' || replace(old.id::text, '-', '');
+ delete from public.catalog_variants
+ where brand_id = old.brand_id
+ and source = 'src_' || replace(old.id::text, '-', '');
+ return old;
+end;
+$$;
+
+revoke all on function public.catalog_sources_forget() from public, anon, authenticated;
+
+drop trigger if exists catalog_sources_forget on public.catalog_sources;
+create trigger catalog_sources_forget
+ after delete on public.catalog_sources
+ for each row execute function public.catalog_sources_forget();
+
+comment on table public.catalog_sources is
+ 'Catalog (#1263, 00279): where a brand''s catalog comes from: a feed URL read daily, or an uploaded CSV. Its products are kept under source src_<id>. Members read; admins and managers add, change and remove; the server writes last_*. Removing one removes its products.';
+
+notify pgrst, 'reload schema';
+
diff --git a/web/messages/en.json b/web/messages/en.json
index 76c4e9f81..31d753049 100644
--- a/web/messages/en.json
+++ b/web/messages/en.json
@@ -7472,6 +7472,46 @@
"integrations_notMapped": "Not mapped",
"integrations_selectProperty": "Select a property",
"integrations_description": "Connect external data sources. Connections are shared with your whole organization.",
+ "catalog": {
+ "title": "Product catalog",
+ "description": "Where {brand}'s products come from: a product feed URL, read every day, or a file you upload. Google product feeds (XML), Meta catalog CSVs and store CSV exports work.",
+ "noBrand": "Pick a brand to see its catalog.",
+ "unavailable": "Couldn't read the catalog's sources right now.",
+ "empty": "No catalog yet. Add your product feed's URL, or upload a CSV.",
+ "upload": "Uploaded file",
+ "status": {
+ "ok": "Synced",
+ "failed": "Couldn't sync",
+ "format_error": "Not a feed",
+ "never": "Not synced yet"
+ },
+ "products": "{count, plural, one {# product} other {# products}}",
+ "lastSync": "synced {when}",
+ "skipped": "{count, plural, one {# product was skipped} other {# products were skipped}}",
+ "syncNow": "Sync now",
+ "remove": "Remove",
+ "removeConfirm": "Remove this source? Its products leave the catalog.",
+ "urlLabel": "Product feed URL",
+ "urlPlaceholder": "https://example.com/products.xml",
+ "addFeed": "Add feed",
+ "uploadFile": "Upload a file",
+ "uploadHint": "CSV or XML, up to 4 MB. For a larger catalog, add its feed URL.",
+ "managersOnly": "Admins and managers can add and sync sources.",
+ "synced": "{count, plural, one {# product read} other {# products read}}",
+ "errors": {
+ "invalid": "Enter a feed address starting with http:// or https://.",
+ "duplicate": "This feed is already a source.",
+ "too_many": "A brand can have 10 sources at most.",
+ "forbidden": "Only admins and managers can change the catalog.",
+ "recent": "Synced a moment ago. Try again in a few minutes.",
+ "too_large": "The file is larger than 4 MB. Add its feed URL instead.",
+ "not_found": "This source no longer exists.",
+ "session": "Your session expired. Sign in again.",
+ "workspace": "This brand isn't in your workspace.",
+ "busy": "Too many syncs. Wait a few minutes.",
+ "failed": "Couldn't sync the catalog. Try again later."
+ }
+ },
"integrations_connected": "Connected",
"integrations_notConfigured": "Not configured",
"integrations_disconnect": "Disconnect",
diff --git a/web/src/components/settings/catalog-sources-card.test.tsx b/web/src/components/settings/catalog-sources-card.test.tsx
new file mode 100644
index 000000000..232aee34a
--- /dev/null
+++ b/web/src/components/settings/catalog-sources-card.test.tsx
@@ -0,0 +1,166 @@
+import { act } from 'react';
+import { createRoot, type Root } from 'react-dom/client';
+import { NextIntlClientProvider } from 'next-intl';
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
+import messages from '../../../messages/en.json';
+import type { CatalogSource } from '@/lib/catalog';
+
+const listCatalogSources = vi.fn();
+const addFeedSource = vi.fn();
+const removeCatalogSource = vi.fn();
+vi.mock('@/lib/actions/catalog', () => ({
+ listCatalogSources: (...args: unknown[]) => listCatalogSources(...args),
+ addFeedSource: (...args: unknown[]) => addFeedSource(...args),
+ removeCatalogSource: (...args: unknown[]) => removeCatalogSource(...args),
+}));
+const syncCatalogSource = vi.fn();
+const uploadCatalogFile = vi.fn();
+vi.mock('@/lib/catalog-client', () => ({
+ syncCatalogSource: (...args: unknown[]) => syncCatalogSource(...args),
+ uploadCatalogFile: (...args: unknown[]) => uploadCatalogFile(...args),
+}));
+const role = vi.hoisted(() => ({ canManage: true }));
+vi.mock('@/hooks/use-user-role', () => ({ useUserRole: () => role }));
+vi.mock('@/stores/use-brand-store', () => ({
+ useBrandStore: () => ({ activeBrandId: 'b1', brands: [{ id: 'b1', name: 'Acme' }] }),
+}));
+const toast = vi.hoisted(() => ({ error: vi.fn(), success: vi.fn() }));
+vi.mock('sonner', () => ({ toast }));
+
+const { CatalogSourcesCard } = await import('./catalog-sources-card');
+
+(globalThis as typeof globalThis & { IS_REACT_ACT_ENVIRONMENT: boolean }).IS_REACT_ACT_ENVIRONMENT =
+ true;
+
+/** Settings › Integrations › Product catalog (#1263): a brand's feed URLs and uploads. */
+
+let container: HTMLDivElement;
+let root: Root;
+
+const FEED: CatalogSource = {
+ id: 's1',
+ kind: 'feed_url',
+ format: 'auto',
+ url: 'https://shop.example/feed.xml',
+ lastSyncedAt: '2026-10-25T08:00:00Z',
+ lastStatus: 'ok',
+ lastError: null,
+ lastCounts: {
+ format: 'google_xml',
+ items: 4,
+ skipped: 1,
+ errors: ['Item 2: price is missing.'],
+ products: { added: 2, changed: 0, removed: 0, unchanged: 0 },
+ variants: { added: 3, changed: 0, removed: 0, unchanged: 0 },
+ },
+ products: 2,
+};
+
+async function render() {
+ await act(async () => {
+ root.render(
+ <NextIntlClientProvider locale="en" messages={messages} timeZone="UTC">
+ <CatalogSourcesCard />
+ </NextIntlClientProvider>,
+ );
+ });
+}
+const button = (label: string) =>
+ [...container.querySelectorAll('button')].find(
+ (b) => b.textContent === label || b.getAttribute('aria-label') === label,
+ );
+
+beforeEach(() => {
+ role.canManage = true;
+ listCatalogSources.mockReset().mockResolvedValue([FEED]);
+ addFeedSource.mockReset().mockResolvedValue({ ok: true, id: 's2' });
+ removeCatalogSource.mockReset().mockResolvedValue(undefined);
+ syncCatalogSource.mockReset().mockResolvedValue({
+ ok: true,
+ status: 'ok',
+ error: null,
+ counts: { items: 12 },
+ });
+ uploadCatalogFile.mockReset();
+ toast.error.mockReset();
+ toast.success.mockReset();
+ container = document.createElement('div');
+ document.body.appendChild(container);
+ root = createRoot(container);
+});
+afterEach(() => {
+ act(() => root.unmount());
+ container.remove();
+});
+
+describe('CatalogSourcesCard', () => {
+ it("lists the brand's sources with their products, status and skipped products", async () => {
+ await render();
+ expect(listCatalogSources).toHaveBeenCalledWith('b1');
+ const text = container.textContent ?? '';
+ expect(text).toContain("Where Acme's products come from");
+ expect(text).toContain('https://shop.example/feed.xml');
+ expect(text).toContain('Synced');
+ expect(text).toContain('2 products');
+ expect(text).toContain('1 product was skipped');
+ expect(text).toContain('Item 2: price is missing.');
+ });
+
+ it('adds a feed URL and reads it at once', async () => {
+ await render();
+ const input = container.querySelector<HTMLInputElement>('input[type="url"]')!;
+ await act(async () => {
+ const setter = Object.getOwnPropertyDescriptor(HTMLInputElement.prototype, 'value')!.set!;
+ setter.call(input, 'https://shop.example/meta.csv');
+ input.dispatchEvent(new Event('input', { bubbles: true }));
+ });
+ await act(async () => {
+ container
+ .querySelector('form')!
+ .dispatchEvent(new Event('submit', { bubbles: true, cancelable: true }));
+ });
+ expect(addFeedSource).toHaveBeenCalledWith('b1', 'https://shop.example/meta.csv');
+ expect(syncCatalogSource).toHaveBeenCalledWith('s2');
+ expect(toast.success).toHaveBeenCalledWith('12 products read');
+ expect(listCatalogSources).toHaveBeenCalledTimes(2);
+ });
+
+ it("says why a feed couldn't be added, or read", async () => {
+ addFeedSource.mockResolvedValue({ ok: false, code: 'duplicate' });
+ await render();
+ const input = container.querySelector<HTMLInputElement>('input[type="url"]')!;
+ await act(async () => {
+ const setter = Object.getOwnPropertyDescriptor(HTMLInputElement.prototype, 'value')!.set!;
+ setter.call(input, 'https://shop.example/feed.xml');
+ input.dispatchEvent(new Event('input', { bubbles: true }));
+ });
+ await act(async () => {
+ container
+ .querySelector('form')!
+ .dispatchEvent(new Event('submit', { bubbles: true, cancelable: true }));
+ });
+ expect(toast.error).toHaveBeenCalledWith('This feed is already a source.');
+ expect(syncCatalogSource).not.toHaveBeenCalled();
+
+ syncCatalogSource.mockResolvedValue({
+ ok: true,
+ status: 'format_error',
+ error: "This isn't a Google product feed: it has no <rss> or <feed> element.",
+ counts: null,
+ });
+ await act(async () => button('Sync now')!.click());
+ expect(toast.error).toHaveBeenLastCalledWith(
+ "This isn't a Google product feed: it has no <rss> or <feed> element.",
+ );
+ });
+
+ it("shows the sources, not to change, to members who aren't admins or managers", async () => {
+ role.canManage = false;
+ await render();
+ expect(container.textContent).toContain('https://shop.example/feed.xml');
+ expect(container.textContent).toContain('Admins and managers can add and sync sources.');
+ expect(container.querySelector('input[type="url"]')).toBeNull();
+ expect(button('Sync now')).toBeUndefined();
+ expect(button('Remove')).toBeUndefined();
+ });
+});
diff --git a/web/src/components/settings/catalog-sources-card.tsx b/web/src/components/settings/catalog-sources-card.tsx
new file mode 100644
index 000000000..8049adaf1
--- /dev/null
+++ b/web/src/components/settings/catalog-sources-card.tsx
@@ -0,0 +1,262 @@
+'use client';
+
+import { useCallback, useEffect, useRef, useState } from 'react';
+import { useFormatter, useTranslations } from 'next-intl';
+import { toast } from 'sonner';
+import { Loader2, Package, RefreshCw, Trash2, Upload } from 'lucide-react';
+import { Badge } from '@/components/ui/badge';
+import { Button } from '@/components/ui/button';
+import { Input } from '@/components/ui/input';
+import { Skeleton } from '@/components/ui/skeleton';
+import { useUserRole } from '@/hooks/use-user-role';
+import { addFeedSource, listCatalogSources, removeCatalogSource } from '@/lib/actions/catalog';
+import type { CatalogSource } from '@/lib/catalog';
+import { syncCatalogSource, uploadCatalogFile, type CatalogSyncResult } from '@/lib/catalog-client';
+import { useBrandStore } from '@/stores/use-brand-store';
+
+/**
+ * Product catalog (#1263): where the active brand's catalog comes from, a
+ * product feed URL read daily, or a CSV / feed file uploaded. Admins and
+ * managers add, sync and remove; members see the sources and how they went.
+ */
+export function CatalogSourcesCard() {
+ const t = useTranslations('settings.catalog');
+ const format = useFormatter();
+ const { activeBrandId, brands } = useBrandStore();
+ const { canManage } = useUserRole();
+ const brand = brands.find((b) => b.id === activeBrandId) ?? null;
+
+ // The sources as read for a brand, and when ("synced 3 hours ago"): another
+ // brand's never show.
+ const [loaded, setLoaded] = useState<{
+ brandId: string;
+ sources: CatalogSource[] | null;
+ at: Date;
+ } | null>(null);
+ const [url, setUrl] = useState('');
+ const [busy, setBusy] = useState<string | null>(null);
+ const fileInput = useRef<HTMLInputElement>(null);
+
+ // Bumped to read the sources again, after a sync, an add or a removal.
+ const [reads, setReads] = useState(0);
+ const load = useCallback(() => setReads((n) => n + 1), []);
+
+ useEffect(() => {
+ if (!activeBrandId) return;
+ let cancelled = false;
+ listCatalogSources(activeBrandId).then(
+ (sources) => {
+ if (!cancelled) setLoaded({ brandId: activeBrandId, sources, at: new Date() });
+ },
+ (err) => {
+ console.error('Failed to read the catalog sources:', err);
+ if (!cancelled) setLoaded({ brandId: activeBrandId, sources: null, at: new Date() });
+ },
+ );
+ return () => {
+ cancelled = true;
+ };
+ }, [activeBrandId, reads]);
+
+ const current = loaded?.brandId === activeBrandId ? loaded : null;
+ const sources = current?.sources ?? null;
+ const failed = current !== null && current.sources === null;
+ const now = current?.at ?? new Date();
+
+ /** Says how a sync went, then reads the sources again. */
+ const report = (result: CatalogSyncResult) => {
+ if (!result.ok) toast.error(t(`errors.${result.code}`));
+ else if (result.status === 'ok') {
+ toast.success(t('synced', { count: result.counts?.items ?? 0 }));
+ } else toast.error(result.error ?? t('errors.failed'));
+ load();
+ };
+
+ const add = async () => {
+ if (!activeBrandId) return;
+ setBusy('add');
+ const added = await addFeedSource(activeBrandId, url);
+ if (!added.ok) {
+ toast.error(t(`errors.${added.code}`));
+ setBusy(null);
+ return;
+ }
+ setUrl('');
+ // Read it now, so the person sees whether it's a feed.
+ report(await syncCatalogSource(added.id));
+ setBusy(null);
+ };
+
+ const sync = async (id: string) => {
+ setBusy(id);
+ report(await syncCatalogSource(id));
+ setBusy(null);
+ };
+
+ const remove = async (id: string) => {
+ if (!window.confirm(t('removeConfirm'))) return;
+ setBusy(id);
+ try {
+ await removeCatalogSource(id);
+ load();
+ } catch {
+ toast.error(t('errors.forbidden'));
+ }
+ setBusy(null);
+ };
+
+ const upload = async (file: File | undefined) => {
+ if (!file || !activeBrandId) return;
+ setBusy('upload');
+ report(await uploadCatalogFile(activeBrandId, file));
+ setBusy(null);
+ if (fileInput.current) fileInput.current.value = '';
+ };
+
+ return (
+ <div className="space-y-4 rounded-lg border p-4">
+ <div className="flex items-start gap-3">
+ <div className="flex h-10 w-10 shrink-0 items-center justify-center rounded-md border bg-muted/50">
+ <Package className="h-5 w-5 text-muted-foreground" />
+ </div>
+ <div className="min-w-0">
+ <p className="font-medium">{t('title')}</p>
+ <p className="text-sm text-muted-foreground">
+ {brand ? t('description', { brand: brand.name }) : t('noBrand')}
+ </p>
+ </div>
+ </div>
+
+ {activeBrandId &&
+ (failed ? (
+ <p className="text-sm text-muted-foreground">{t('unavailable')}</p>
+ ) : sources === null ? (
+ <Skeleton className="h-16 w-full" />
+ ) : (
+ <>
+ {sources.length === 0 ? (
+ <p className="text-sm text-muted-foreground">{t('empty')}</p>
+ ) : (
+ <ul className="divide-y rounded-md border">
+ {sources.map((source) => (
+ <li key={source.id} className="space-y-1 p-3 text-sm">
+ <div className="flex flex-wrap items-center justify-between gap-2">
+ <span
+ className="min-w-0 truncate font-medium"
+ title={source.url ?? undefined}
+ >
+ {source.kind === 'feed_url' ? source.url : t('upload')}
+ </span>
+ <div className="flex items-center gap-2">
+ <Badge variant={source.lastStatus === 'ok' ? 'secondary' : 'outline'}>
+ {source.lastStatus ? t(`status.${source.lastStatus}`) : t('status.never')}
+ </Badge>
+ {canManage && source.kind === 'feed_url' && (
+ <Button
+ size="sm"
+ variant="ghost"
+ disabled={busy !== null}
+ onClick={() => sync(source.id)}
+ aria-label={t('syncNow')}
+ >
+ {busy === source.id ? (
+ <Loader2 className="h-4 w-4 animate-spin" />
+ ) : (
+ <RefreshCw className="h-4 w-4" />
+ )}
+ </Button>
+ )}
+ {canManage && (
+ <Button
+ size="sm"
+ variant="ghost"
+ disabled={busy !== null}
+ onClick={() => remove(source.id)}
+ aria-label={t('remove')}
+ >
+ <Trash2 className="h-4 w-4" />
+ </Button>
+ )}
+ </div>
+ </div>
+ <p className="text-xs text-muted-foreground">
+ {t('products', { count: source.products })}
+ {source.lastSyncedAt &&
+ ` · ${t('lastSync', { when: format.relativeTime(new Date(source.lastSyncedAt), now) })}`}
+ </p>
+ {source.lastStatus !== 'ok' && source.lastError && (
+ <p className="text-xs text-destructive">{source.lastError}</p>
+ )}
+ {source.lastStatus === 'ok' && (source.lastCounts?.skipped ?? 0) > 0 && (
+ <details className="text-xs text-muted-foreground">
+ <summary className="cursor-pointer">
+ {t('skipped', { count: source.lastCounts!.skipped })}
+ </summary>
+ <ul className="mt-1 list-disc space-y-0.5 pl-4">
+ {source.lastCounts!.errors.map((error) => (
+ <li key={error}>{error}</li>
+ ))}
+ </ul>
+ </details>
+ )}
+ </li>
+ ))}
+ </ul>
+ )}
+
+ {canManage ? (
+ <div className="space-y-2">
+ <form
+ className="flex flex-col gap-2 sm:flex-row"
+ onSubmit={(e) => {
+ e.preventDefault();
+ void add();
+ }}
+ >
+ <Input
+ type="url"
+ inputMode="url"
+ placeholder={t('urlPlaceholder')}
+ aria-label={t('urlLabel')}
+ value={url}
+ maxLength={2048}
+ onChange={(e) => setUrl(e.target.value)}
+ />
+ <Button type="submit" disabled={!url.trim() || busy !== null}>
+ {busy === 'add' && <Loader2 className="mr-2 h-4 w-4 animate-spin" />}
+ {t('addFeed')}
+ </Button>
+ </form>
+ <div className="flex flex-wrap items-center gap-2">
+ <input
+ ref={fileInput}
+ type="file"
+ accept=".csv,.tsv,.txt,.xml,text/csv,text/plain,application/xml,text/xml"
+ className="hidden"
+ onChange={(e) => void upload(e.target.files?.[0])}
+ />
+ <Button
+ type="button"
+ variant="outline"
+ size="sm"
+ disabled={busy !== null}
+ onClick={() => fileInput.current?.click()}
+ >
+ {busy === 'upload' ? (
+ <Loader2 className="mr-2 h-4 w-4 animate-spin" />
+ ) : (
+ <Upload className="mr-2 h-4 w-4" />
+ )}
+ {t('uploadFile')}
+ </Button>
+ <span className="text-xs text-muted-foreground">{t('uploadHint')}</span>
+ </div>
+ </div>
+ ) : (
+ <p className="text-xs text-muted-foreground">{t('managersOnly')}</p>
+ )}
+ </>
+ ))}
+ </div>
+ );
+}
diff --git a/web/src/components/settings/integrations-section.tsx b/web/src/components/settings/integrations-section.tsx
index 785b66ce6..0b6b5838f 100644
--- a/web/src/components/settings/integrations-section.tsx
+++ b/web/src/components/settings/integrations-section.tsx
@@ -38,6 +38,7 @@ import {
type IntegrationStatus,
} from '@/lib/actions/integrations';
import { matchProperty } from '@/lib/property-match';
+import { CatalogSourcesCard } from './catalog-sources-card';
import { useIntegrationConnect } from '@/hooks/use-integration-connect';
/**
@@ -76,6 +77,8 @@ export function IntegrationsSection() {
>
<GaPropertyMapping />
</IntegrationCard>
+
+ <CatalogSourcesCard />
</CardContent>
</Card>
);
diff --git a/web/src/lib/actions/catalog.ts b/web/src/lib/actions/catalog.ts
new file mode 100644
index 000000000..8c312dfff
--- /dev/null
+++ b/web/src/lib/actions/catalog.ts
@@ -0,0 +1,102 @@
+'use server';
+
+import { z } from 'zod';
+import { createClient } from '@/lib/supabase/server';
+import {
+ catalogSourceKey,
+ type CatalogCounts,
+ type CatalogFormat,
+ type CatalogSource,
+ type CatalogSourceKind,
+ type CatalogSyncStatus,
+} from '@/lib/catalog';
+
+/**
+ * A brand's catalog sources (#1263, 00279), read and changed through RLS:
+ * members read them; admins and managers add and remove feed URLs. The API
+ * reads the feeds and imports uploads (lib/catalog-client.ts).
+ */
+
+const FORMATS = ['auto', 'google_xml', 'csv', 'meta_csv'] as const;
+
+/** The brand's sources, oldest first, each with its live products in the catalog. */
+export async function listCatalogSources(brandId: string): Promise<CatalogSource[]> {
+ if (!z.guid().safeParse(brandId).success) throw new Error('invalid');
+ const supabase = await createClient();
+ const { data, error } = await supabase
+ .from('catalog_sources')
+ .select('id, kind, format, url, last_synced_at, last_status, last_error, last_counts')
+ .eq('brand_id', brandId)
+ .order('created_at', { ascending: true });
+ if (error) throw new Error(error.message);
+ return Promise.all(
+ (data ?? []).map(async (row) => {
+ const { count } = await supabase
+ .from('catalog_products')
+ .select('id', { count: 'exact', head: true })
+ .eq('brand_id', brandId)
+ .eq('source', catalogSourceKey(row.id))
+ .is('removed_at', null);
+ return {
+ id: row.id,
+ kind: row.kind as CatalogSourceKind,
+ format: row.format as CatalogFormat,
+ url: row.url,
+ lastSyncedAt: row.last_synced_at,
+ lastStatus: row.last_status as CatalogSyncStatus | null,
+ lastError: row.last_error,
+ lastCounts: row.last_counts as unknown as CatalogCounts | null,
+ products: count ?? 0,
+ };
+ }),
+ );
+}
+
+export type AddFeedResult =
+ | { ok: true; id: string }
+ | { ok: false; code: 'invalid' | 'duplicate' | 'too_many' | 'forbidden' | 'failed' };
+
+/** Adds a feed URL to a brand's sources: admins and managers. The API reads it daily. */
+export async function addFeedSource(
+ brandId: string,
+ url: string,
+ format: CatalogFormat = 'auto',
+): Promise<AddFeedResult> {
+ const address = typeof url === 'string' ? url.trim() : '';
+ if (
+ !z.guid().safeParse(brandId).success ||
+ !FORMATS.includes(format) ||
+ address.length > 2048 ||
+ !/^https?:\/\/[^\s/]+/i.test(address) ||
+ !URL.canParse(address)
+ ) {
+ return { ok: false, code: 'invalid' };
+ }
+ const supabase = await createClient();
+ const {
+ data: { user },
+ } = await supabase.auth.getUser();
+ if (!user) return { ok: false, code: 'forbidden' };
+ const { data, error } = await supabase
+ .from('catalog_sources')
+ .insert({ brand_id: brandId, kind: 'feed_url', format, url: address, created_by: user.id })
+ .select('id')
+ .single();
+ if (error?.code === '23505') return { ok: false, code: 'duplicate' };
+ if (error?.code === '23514') return { ok: false, code: 'too_many' };
+ if (error?.code === '42501') return { ok: false, code: 'forbidden' };
+ if (error || !data) {
+ console.error('Failed to add the feed:', error);
+ return { ok: false, code: 'failed' };
+ }
+ return { ok: true, id: data.id };
+}
+
+/** Removes a source, and with it its products: admins and managers. */
+export async function removeCatalogSource(id: string): Promise<void> {
+ if (!z.guid().safeParse(id).success) throw new Error('invalid');
+ const supabase = await createClient();
+ const { data, error } = await supabase.from('catalog_sources').delete().eq('id', id).select('id');
+ if (error) throw new Error(error.message);
+ if (!data?.length) throw new Error('forbidden');
+}
diff --git a/web/src/lib/catalog-client.test.ts b/web/src/lib/catalog-client.test.ts
new file mode 100644
index 000000000..cd2c0d569
--- /dev/null
+++ b/web/src/lib/catalog-client.test.ts
@@ -0,0 +1,64 @@
+import { beforeEach, describe, expect, it, vi } from 'vitest';
+
+/** #1263: a catalog sync from the browser, through the API's proxy. */
+
+vi.mock('@/lib/supabase/client', () => ({
+ createClient: () => ({
+ auth: { getSession: async () => ({ data: { session: { access_token: 'tok' } } }) },
+ }),
+}));
+
+const { syncCatalogSource, uploadCatalogFile } = await import('./catalog-client');
+
+const fetchMock = vi.fn();
+beforeEach(() => {
+ fetchMock.mockReset();
+ vi.stubGlobal('fetch', fetchMock);
+});
+
+const answer = (status: number, body: unknown) =>
+ fetchMock.mockResolvedValue(new Response(JSON.stringify(body), { status }));
+
+describe('syncCatalogSource', () => {
+ it('reads a feed now through the proxy, with the session', async () => {
+ answer(200, { status: 'ok', error: null, counts: { items: 4 } });
+ expect(await syncCatalogSource('s1')).toEqual({
+ ok: true,
+ status: 'ok',
+ error: null,
+ counts: { items: 4 },
+ });
+ const [url, init] = fetchMock.mock.calls[0];
+ expect(url).toBe('/api/aeo/api/catalog/sources/s1/sync');
+ expect(init).toMatchObject({ method: 'POST', headers: { Authorization: 'Bearer tok' } });
+ });
+
+ it("names the API's reasons, and the general ones", async () => {
+ answer(409, { code: 'recent' });
+ expect(await syncCatalogSource('s1')).toEqual({ ok: false, code: 'recent' });
+ answer(403, { code: 'not_in_workspace' });
+ expect(await syncCatalogSource('s1')).toEqual({ ok: false, code: 'workspace' });
+ answer(429, { message: 'Too many requests' });
+ expect(await syncCatalogSource('s1')).toEqual({ ok: false, code: 'busy' });
+ answer(500, { code: 'failed' });
+ expect(await syncCatalogSource('s1')).toEqual({ ok: false, code: 'failed' });
+ });
+});
+
+describe('uploadCatalogFile', () => {
+ it("sends the file's text as the body, for the brand", async () => {
+ answer(200, { status: 'ok', error: null, counts: null });
+ // jsdom's File has no text(); a browser's does.
+ const file = { size: 15, text: async () => 'id,title\n1,Boot' } as unknown as File;
+ await uploadCatalogFile('b1', file);
+ const [url, init] = fetchMock.mock.calls[0];
+ expect(url).toBe('/api/aeo/api/catalog/upload?brandId=b1&format=auto');
+ expect(init.body).toBe('id,title\n1,Boot');
+ });
+
+ it('refuses a file over 4 MB before sending it', async () => {
+ const file = new File(['x'.repeat(4 * 1024 * 1024 + 1)], 'big.csv');
+ expect(await uploadCatalogFile('b1', file)).toEqual({ ok: false, code: 'too_large' });
+ expect(fetchMock).not.toHaveBeenCalled();
+ });
+});
diff --git a/web/src/lib/catalog-client.ts b/web/src/lib/catalog-client.ts
new file mode 100644
index 000000000..eebf2fd0c
--- /dev/null
+++ b/web/src/lib/catalog-client.ts
@@ -0,0 +1,108 @@
+import { BROWSER_API_BASE_URL } from '@/config/api';
+import { apiFailure } from '@/lib/api-failure';
+import { createClient } from '@/lib/supabase/client';
+import {
+ CATALOG_UPLOAD_MAX_BYTES,
+ type CatalogCounts,
+ type CatalogFormat,
+ type CatalogSyncStatus,
+} from '@/lib/catalog';
+
+/**
+ * A catalog sync from the browser (#1263), through the API's same-origin
+ * proxy: a feed read now, or a file uploaded. Not a server action, whose
+ * body is limited to 1 MB.
+ */
+
+export type CatalogSyncError =
+ | 'recent'
+ | 'too_large'
+ | 'too_many'
+ | 'forbidden'
+ | 'not_found'
+ | 'invalid'
+ | 'session'
+ | 'workspace'
+ | 'busy'
+ | 'failed';
+
+export type CatalogSyncResult =
+ | {
+ ok: true;
+ /** How the sync went: a feed that can't be read is still an answer. */
+ status: CatalogSyncStatus;
+ error: string | null;
+ counts: CatalogCounts | null;
+ }
+ | { ok: false; code: CatalogSyncError };
+
+const KNOWN: readonly CatalogSyncError[] = [
+ 'recent',
+ 'too_large',
+ 'too_many',
+ 'forbidden',
+ 'not_found',
+ 'invalid',
+];
+
+async function call(path: string, init: RequestInit): Promise<CatalogSyncResult> {
+ try {
+ const {
+ data: { session },
+ } = await createClient().auth.getSession();
+ if (!session) return { ok: false, code: 'session' };
+ const res = await fetch(`${BROWSER_API_BASE_URL}/api/catalog${path}`, {
+ ...init,
+ headers: { ...init.headers, Authorization: `Bearer ${session.access_token}` },
+ });
+ const body = (await res.json().catch(() => ({}))) as {
+ code?: string;
+ status?: CatalogSyncStatus;
+ error?: string | null;
+ counts?: CatalogCounts | null;
+ };
+ if (!res.ok) {
+ if (res.status === 413) return { ok: false, code: 'too_large' };
+ const known = KNOWN.find((c) => c === body.code);
+ if (known) return { ok: false, code: known };
+ const general = apiFailure(res.status, body);
+ return {
+ ok: false,
+ code:
+ general === 'session' || general === 'workspace' || general === 'busy'
+ ? general
+ : 'failed',
+ };
+ }
+ return {
+ ok: true,
+ status: body.status ?? 'failed',
+ error: body.error ?? null,
+ counts: body.counts ?? null,
+ };
+ } catch (err) {
+ console.error('Failed to sync the catalog:', err);
+ return { ok: false, code: 'failed' };
+ }
+}
+
+/** Reads a feed source now. */
+export function syncCatalogSource(id: string): Promise<CatalogSyncResult> {
+ return call(`/sources/${encodeURIComponent(id)}/sync`, { method: 'POST' });
+}
+
+/** Imports a file as the brand's upload: up to 4 MB, CSV or a Google feed's XML. */
+export async function uploadCatalogFile(
+ brandId: string,
+ file: File,
+ format: CatalogFormat = 'auto',
+): Promise<CatalogSyncResult> {
+ if (file.size > CATALOG_UPLOAD_MAX_BYTES) return { ok: false, code: 'too_large' };
+ const text = await file.text();
+ const query = new URLSearchParams({ brandId, format });
+ return call(`/upload?${query}`, {
+ method: 'POST',
+ headers: { 'Content-Type': 'text/plain; charset=utf-8' },
+ body: text,
+ });
+}
diff --git a/web/src/lib/catalog.ts b/web/src/lib/catalog.ts
new file mode 100644
index 000000000..a0994aaeb
--- /dev/null
+++ b/web/src/lib/catalog.ts
@@ -0,0 +1,40 @@
+/**
+ * A brand's catalog sources (#1263, 00279), as Settings › Integrations shows
+ * them: a feed URL the API reads daily, or the file a person uploads.
+ */
+
+export type CatalogSourceKind = 'feed_url' | 'upload';
+export type CatalogFormat = 'auto' | 'google_xml' | 'csv' | 'meta_csv';
+export type CatalogSyncStatus = 'ok' | 'failed' | 'format_error';
+
+/** What a sync did, as the API records it on the source (`last_counts`). */
+export interface CatalogCounts {
+ format: Exclude<CatalogFormat, 'auto'>;
+ items: number;
+ skipped: number;
+ /** The first products skipped, and why ("Row 3: price is missing."). */
+ errors: string[];
+ products: { added: number; changed: number; removed: number; unchanged: number };
+ variants: { added: number; changed: number; removed: number; unchanged: number };
+}
+
+export interface CatalogSource {
+ id: string;
+ kind: CatalogSourceKind;
+ format: CatalogFormat;
+ url: string | null;
+ lastSyncedAt: string | null;
+ lastStatus: CatalogSyncStatus | null;
+ lastError: string | null;
+ lastCounts: CatalogCounts | null;
+ /** Its products in the catalog now (not removed). */
+ products: number;
+}
+
+/** The largest file an upload takes: what the API's proxy takes in one request. */
+export const CATALOG_UPLOAD_MAX_BYTES = 4 * 1024 * 1024;
+
+/** The catalog's `source` for a source: `src_` and its id without dashes, as 00279 keys it. */
+export function catalogSourceKey(id: string): string {
+ return `src_${id.replace(/-/g, '')}`;
+}
diff --git a/web/src/types/supabase.ts b/web/src/types/supabase.ts
index 1a6ae7c08..8a81d333c 100644
--- a/web/src/types/supabase.ts
+++ b/web/src/types/supabase.ts
@@ -821,6 +821,122 @@ export type Database = {
},
]
}
+ catalog_products: {
+ Row: {
+ brand_id: string
+ content_hash: string
+ description: string | null
+ external_id: string
+ first_seen_at: string
+ id: string
+ product_brand: string | null
+ product_type: string | null
+ removed_at: string | null
+ source: string
+ title: string
+ updated_at: string
+ url: string | null
+ }
+ Insert: {
+ brand_id: string
+ content_hash: string
+ description?: string | null
+ external_id: string
+ first_seen_at?: string
+ id?: string
+ product_brand?: string | null
+ product_type?: string | null
+ removed_at?: string | null
+ source: string
+ title: string
+ updated_at?: string
+ url?: string | null
+ }
+ Update: {
+ brand_id?: string
+ content_hash?: string
+ description?: string | null
+ external_id?: string
+ first_seen_at?: string
+ id?: string
+ product_brand?: string | null
+ product_type?: string | null
+ removed_at?: string | null
+ source?: string
+ title?: string
+ updated_at?: string
+ url?: string | null
+ }
+ Relationships: [
+ {
+ foreignKeyName: "catalog_products_brand_id_fkey"
+ columns: ["brand_id"]
+ isOneToOne: false
+ referencedRelation: "brands"
+ referencedColumns: ["id"]
+ },
+ ]
+ }
+ catalog_sources: {
+ Row: {
+ brand_id: string
+ created_at: string
+ created_by: string | null
+ format: string
+ id: string
+ kind: string
+ label: string | null
+ last_counts: Json | null
+ last_error: string | null
+ last_status: string | null
+ last_synced_at: string | null
+ url: string | null
+ }
+ Insert: {
+ brand_id: string
+ created_at?: string
+ created_by?: string | null
+ format?: string
+ id?: string
+ kind: string
+ label?: string | null
+ last_counts?: Json | null
+ last_error?: string | null
+ last_status?: string | null
+ last_synced_at?: string | null
+ url?: string | null
+ }
+ Update: {
+ brand_id?: string
+ created_at?: string
+ created_by?: string | null
+ format?: string
+ id?: string
+ kind?: string
+ label?: string | null
+ last_counts?: Json | null
+ last_error?: string | null
+ last_status?: string | null
+ last_synced_at?: string | null
+ url?: string | null
+ }
+ Relationships: [
+ {
+ foreignKeyName: "catalog_sources_brand_id_fkey"
+ columns: ["brand_id"]
+ isOneToOne: false
+ referencedRelation: "brands"
+ referencedColumns: ["id"]
+ },
+ {
+ foreignKeyName: "catalog_sources_created_by_fkey"
+ columns: ["created_by"]
+ isOneToOne: false
+ referencedRelation: "profiles"
+ referencedColumns: ["id"]
+ },
+ ]
+ }
citation_urls: {
Row: {
domain: string