Skip to content

Open-source agentic AI Search Execution Platform. Track AI visibility, find opportunities, and let agents turn signals into prioritized actions and drafted fixes across ChatGPT, Claude, Gemini, Google AI Overviews, Google AI Mode, Perplexity, Grok, and Microsoft Copilot.

feat(feeds): parse store CSVs, Google product feed XML and Meta catalog CSVs into one item shape (#1264) (#1488)

Gökhan Geyik8496228cfbc3
diff --git a/server/src/lib/feeds/__fixtures__/facebook-catalog.csv b/server/src/lib/feeds/__fixtures__/facebook-catalog.csv
new file mode 100644
index 000000000..39bc1ef52
--- /dev/null
+++ b/server/src/lib/feeds/__fixtures__/facebook-catalog.csv
@@ -0,0 +1,5 @@
+id,title,description,availability,condition,price,sale_price,link,image_link,additional_image_link,brand,gtin,mpn,item_group_id,color,size,material,product_type,google_product_category,fb_product_category,quantity_to_sell_on_facebook
+TR-100-RED,Trailrunner Hiking Boot - Red,Waterproof leather & mesh. <b>Grips wet rock.</b>,in stock,new,129.00 USD,99.00 USD,https://shop.example/trailrunner?color=red,https://shop.example/img/tr-red.jpg,"https://shop.example/img/tr-red-2.jpg,https://shop.example/img/tr-red-3.jpg",Acme,4006381333931,,TR-100,Red,42,Leather,Shoes > Hiking Boots,Apparel & Accessories > Shoes,,12
+TR-100-BLUE,Trailrunner Hiking Boot - Blue,Waterproof leather & mesh. <b>Grips wet rock.</b>,in stock,new,129.00 USD,,https://shop.example/trailrunner?color=blue,https://shop.example/img/tr-blue.jpg,,Acme,,,TR-100,Blue,42,Leather,Shoes > Hiking Boots,Apparel & Accessories > Shoes,,8
+TR-100-GREEN,Trailrunner Hiking Boot - Green,Waterproof leather & mesh. <b>Grips wet rock.</b>,out of stock,new,129.00 USD,,https://shop.example/trailrunner?color=green,https://shop.example/img/tr-green.jpg,,Acme,,,TR-100,Green,42,Leather,Shoes > Hiking Boots,Apparel & Accessories > Shoes,,0
+SOCK-1,Merino Hiking Sock,"Soft merino wool, cushioned heel.",preorder,,18.50,,https://shop.example/merino-sock,https://shop.example/img/sock.jpg,,Acme,,SOCK-MER-1,,,,,,,,
diff --git a/server/src/lib/feeds/__fixtures__/google-feed-atom.xml b/server/src/lib/feeds/__fixtures__/google-feed-atom.xml
new file mode 100644
index 000000000..ef88a4c4d
--- /dev/null
+++ b/server/src/lib/feeds/__fixtures__/google-feed-atom.xml
@@ -0,0 +1,69 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<feed xmlns="http://www.w3.org/2005/Atom" xmlns:g="http://base.google.com/ns/1.0">
+ <title>Acme Outdoor</title>
+ <link rel="self" href="https://shop.example/feed.xml"/>
+ <updated>2026-10-20T10:00:00Z</updated>
+ <entry>
+ <g:id>TR-100-RED</g:id>
+ <g:item_group_id>TR-100</g:item_group_id>
+ <title>Trailrunner Hiking Boot - Red</title>
+ <summary><![CDATA[Waterproof leather & mesh. <b>Grips wet rock.</b>]]></summary>
+ <link rel="alternate" href="https://shop.example/trailrunner?color=red"/>
+ <g:image_link>https://shop.example/img/tr-red.jpg</g:image_link>
+ <g:additional_image_link>https://shop.example/img/tr-red-2.jpg</g:additional_image_link>
+ <g:additional_image_link>https://shop.example/img/tr-red-3.jpg</g:additional_image_link>
+ <g:brand>Acme</g:brand>
+ <g:gtin>4006381333931</g:gtin>
+ <g:price>129.00 USD</g:price>
+ <g:sale_price>99.00 USD</g:sale_price>
+ <g:availability>in stock</g:availability>
+ <g:condition>new</g:condition>
+ <g:color>Red</g:color>
+ <g:size>42</g:size>
+ <g:material>Leather</g:material>
+ <g:product_type>Shoes &gt; Hiking Boots</g:product_type>
+ </entry>
+ <entry>
+ <g:id>TR-100-BLUE</g:id>
+ <g:item_group_id>TR-100</g:item_group_id>
+ <title>Trailrunner Hiking Boot - Blue</title>
+ <summary><![CDATA[Waterproof leather & mesh. <b>Grips wet rock.</b>]]></summary>
+ <link href="https://shop.example/trailrunner?color=blue"/>
+ <g:image_link>https://shop.example/img/tr-blue.jpg</g:image_link>
+ <g:brand>Acme</g:brand>
+ <g:price>129.00 USD</g:price>
+ <g:availability>in stock</g:availability>
+ <g:condition>new</g:condition>
+ <g:color>Blue</g:color>
+ <g:size>42</g:size>
+ <g:material>Leather</g:material>
+ <g:product_type>Shoes &gt; Hiking Boots</g:product_type>
+ </entry>
+ <entry>
+ <g:id>TR-100-GREEN</g:id>
+ <g:item_group_id>TR-100</g:item_group_id>
+ <title>Trailrunner Hiking Boot - Green</title>
+ <summary><![CDATA[Waterproof leather & mesh. <b>Grips wet rock.</b>]]></summary>
+ <link href="https://shop.example/trailrunner?color=green"/>
+ <g:image_link>https://shop.example/img/tr-green.jpg</g:image_link>
+ <g:brand>Acme</g:brand>
+ <g:price>129.00 USD</g:price>
+ <g:availability>out of stock</g:availability>
+ <g:condition>new</g:condition>
+ <g:color>Green</g:color>
+ <g:size>42</g:size>
+ <g:material>Leather</g:material>
+ <g:product_type>Shoes &gt; Hiking Boots</g:product_type>
+ </entry>
+ <entry>
+ <g:id>SOCK-1</g:id>
+ <title>Merino Hiking Sock</title>
+ <summary>Soft merino wool, cushioned heel.</summary>
+ <link href="https://shop.example/merino-sock"/>
+ <g:image_link>https://shop.example/img/sock.jpg</g:image_link>
+ <g:brand>Acme</g:brand>
+ <g:mpn>SOCK-MER-1</g:mpn>
+ <g:price>18.50</g:price>
+ <g:availability>preorder</g:availability>
+ </entry>
+</feed>
diff --git a/server/src/lib/feeds/__fixtures__/google-feed.xml b/server/src/lib/feeds/__fixtures__/google-feed.xml
new file mode 100644
index 000000000..134116a0f
--- /dev/null
+++ b/server/src/lib/feeds/__fixtures__/google-feed.xml
@@ -0,0 +1,71 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<rss xmlns:g="http://base.google.com/ns/1.0" version="2.0">
+ <channel>
+ <title>Acme Outdoor</title>
+ <link>https://shop.example</link>
+ <description>Acme Outdoor product feed</description>
+ <item>
+ <g:id>TR-100-RED</g:id>
+ <g:item_group_id>TR-100</g:item_group_id>
+ <title>Trailrunner Hiking Boot - Red</title>
+ <description><![CDATA[Waterproof leather & mesh. <b>Grips wet rock.</b>]]></description>
+ <link>https://shop.example/trailrunner?color=red</link>
+ <g:image_link>https://shop.example/img/tr-red.jpg</g:image_link>
+ <g:additional_image_link>https://shop.example/img/tr-red-2.jpg</g:additional_image_link>
+ <g:additional_image_link>https://shop.example/img/tr-red-3.jpg</g:additional_image_link>
+ <g:brand>Acme</g:brand>
+ <g:gtin>4006381333931</g:gtin>
+ <g:price>129.00 USD</g:price>
+ <g:sale_price>99.00 USD</g:sale_price>
+ <g:availability>in stock</g:availability>
+ <g:condition>new</g:condition>
+ <g:color>Red</g:color>
+ <g:size>42</g:size>
+ <g:material>Leather</g:material>
+ <g:product_type>Shoes &gt; Hiking Boots</g:product_type>
+ </item>
+ <item>
+ <g:id>TR-100-BLUE</g:id>
+ <g:item_group_id>TR-100</g:item_group_id>
+ <title>Trailrunner Hiking Boot - Blue</title>
+ <description><![CDATA[Waterproof leather & mesh. <b>Grips wet rock.</b>]]></description>
+ <link>https://shop.example/trailrunner?color=blue</link>
+ <g:image_link>https://shop.example/img/tr-blue.jpg</g:image_link>
+ <g:brand>Acme</g:brand>
+ <g:price>129.00 USD</g:price>
+ <g:availability>in stock</g:availability>
+ <g:condition>new</g:condition>
+ <g:color>Blue</g:color>
+ <g:size>42</g:size>
+ <g:material>Leather</g:material>
+ <g:product_type>Shoes &gt; Hiking Boots</g:product_type>
+ </item>
+ <item>
+ <g:id>TR-100-GREEN</g:id>
+ <g:item_group_id>TR-100</g:item_group_id>
+ <title>Trailrunner Hiking Boot - Green</title>
+ <description><![CDATA[Waterproof leather & mesh. <b>Grips wet rock.</b>]]></description>
+ <link>https://shop.example/trailrunner?color=green</link>
+ <g:image_link>https://shop.example/img/tr-green.jpg</g:image_link>
+ <g:brand>Acme</g:brand>
+ <g:price>129.00 USD</g:price>
+ <g:availability>out of stock</g:availability>
+ <g:condition>new</g:condition>
+ <g:color>Green</g:color>
+ <g:size>42</g:size>
+ <g:material>Leather</g:material>
+ <g:product_type>Shoes &gt; Hiking Boots</g:product_type>
+ </item>
+ <item>
+ <g:id>SOCK-1</g:id>
+ <title>Merino Hiking Sock</title>
+ <description>Soft merino wool, cushioned heel.</description>
+ <link>https://shop.example/merino-sock</link>
+ <g:image_link>https://shop.example/img/sock.jpg</g:image_link>
+ <g:brand>Acme</g:brand>
+ <g:mpn>SOCK-MER-1</g:mpn>
+ <g:price>18.50</g:price>
+ <g:availability>preorder</g:availability>
+ </item>
+ </channel>
+</rss>
diff --git a/server/src/lib/feeds/__fixtures__/store-export.csv b/server/src/lib/feeds/__fixtures__/store-export.csv
new file mode 100644
index 000000000..cc456cf93
--- /dev/null
+++ b/server/src/lib/feeds/__fixtures__/store-export.csv
@@ -0,0 +1,5 @@
+ID,Title,Description,Link,Image Link,Price,GTIN,Brand,Availability
+B-1,Summit Backpack 30L,"Roomy, light and
+rain-proof: ""ready for anything"".",https://shop.example/summit-30,https://shop.example/img/summit.jpg,"1,249.90 TRY",8690000000017,Acme,in stock
+B-2,Summit Backpack 45L,No price on this one,https://shop.example/summit-45,https://shop.example/img/summit45.jpg,,,Acme,in stock
+B-3,Day Pack,Small daily pack,https://shop.example/day-pack,,EUR 39,,Acme,In Stock
diff --git a/server/src/lib/feeds/csv.js b/server/src/lib/feeds/csv.js
new file mode 100644
index 000000000..26723c486
--- /dev/null
+++ b/server/src/lib/feeds/csv.js
@@ -0,0 +1,86 @@
+/**
+ * A small RFC 4180 reader: quoted fields, doubled quotes, commas and line
+ * breaks inside quotes, CRLF or LF, and a leading byte-order mark. Tab-
+ * separated text (Google's text feeds) is read the same way when the header
+ * line has tabs and no commas.
+ *
+ * Rows are yielded one at a time, so a large file isn't copied into an array
+ * of rows before it's read.
+ */
+
+/** The header line's delimiter: a tab when it has tabs and no commas outside quotes. */
+export function delimiterOf(text) {
+ let quoted = false;
+ let tabs = 0;
+ let commas = 0;
+ for (let i = 0; i < text.length; i++) {
+ const ch = text[i];
+ if (ch === '"') quoted = !quoted;
+ else if (!quoted && (ch === '\n' || ch === '\r')) break;
+ else if (!quoted && ch === '\t') tabs++;
+ else if (!quoted && ch === ',') commas++;
+ }
+ return tabs > 0 && commas === 0 ? '\t' : ',';
+}
+
+/**
+ * Each row of a CSV text, as an array of field strings, with the line it
+ * starts on (1-based). A blank line yields nothing.
+ *
+ * @param {string} input
+ * @returns {Generator<{ line: number, fields: string[] }>}
+ */
+export function* csvRows(input) {
+ const text = input.charCodeAt(0) === 0xfeff ? input.slice(1) : input;
+ const delimiter = delimiterOf(text);
+ let fields = [];
+ let field = '';
+ let quoted = false;
+ let line = 1;
+ let rowLine = 1;
+ let sawAny = false;
+
+ const endRow = function* () {
+ fields.push(field);
+ // A blank line: one empty field and nothing else.
+ if (fields.length > 1 || fields[0] !== '') yield { line: rowLine, fields };
+ fields = [];
+ field = '';
+ sawAny = false;
+ };
+
+ for (let i = 0; i < text.length; i++) {
+ const ch = text[i];
+ if (quoted) {
+ if (ch === '"') {
+ if (text[i + 1] === '"') {
+ field += '"';
+ i++;
+ } else {
+ quoted = false;
+ }
+ } else {
+ if (ch === '\n') line++;
+ field += ch;
+ }
+ continue;
+ }
+ if (ch === '"' && field === '') {
+ quoted = true;
+ sawAny = true;
+ } else if (ch === delimiter) {
+ fields.push(field);
+ field = '';
+ sawAny = true;
+ } else if (ch === '\r' || ch === '\n') {
+ if (ch === '\r' && text[i + 1] === '\n') i++;
+ yield* endRow();
+ line++;
+ rowLine = line;
+ } else {
+ field += ch;
+ sawAny = true;
+ }
+ }
+ if (sawAny || field !== '' || fields.length) yield* endRow();
+}
diff --git a/server/src/lib/feeds/feeds.test.js b/server/src/lib/feeds/feeds.test.js
new file mode 100644
index 000000000..6635e6c50
--- /dev/null
+++ b/server/src/lib/feeds/feeds.test.js
@@ -0,0 +1,216 @@
+import { readFileSync } from 'node:fs';
+import { describe, expect, it } from 'vitest';
+import { csvRows } from './csv.js';
+import {
+ FeedFormatError,
+ parseCsvFeed,
+ parseFacebookCatalogCsv,
+ parseGoogleFeedXml,
+} from './index.js';
+import { parseMoney } from './item.js';
+
+const fixture = (name) => readFileSync(new URL(`./__fixtures__/${name}`, import.meta.url), 'utf8');
+
+describe('csvRows', () => {
+ it('reads quoted fields with commas, doubled quotes and line breaks, and CRLF', () => {
+ const rows = [...csvRows('a,b,c\r\n1,"x, ""y""\nz",3\r\n\r\n4,5,6')];
+ expect(rows).toEqual([
+ { line: 1, fields: ['a', 'b', 'c'] },
+ { line: 2, fields: ['1', 'x, "y"\nz', '3'] },
+ { line: 5, fields: ['4', '5', '6'] },
+ ]);
+ });
+
+ it('reads a tab-separated text feed', () => {
+ expect([...csvRows('id\ttitle\n1\tBoot, red')].map((r) => r.fields)).toEqual([
+ ['id', 'title'],
+ ['1', 'Boot, red'],
+ ]);
+ });
+});
+
+describe('parseMoney', () => {
+ it('reads a price with or without a currency, either decimal mark', () => {
+ expect(parseMoney('15.00 USD')).toEqual({ amount: 15, currency: 'USD' });
+ expect(parseMoney('usd 15.5')).toEqual({ amount: 15.5, currency: 'USD' });
+ expect(parseMoney('18.50')).toEqual({ amount: 18.5, currency: null });
+ expect(parseMoney('1,249.90 TRY')).toEqual({ amount: 1249.9, currency: 'TRY' });
+ expect(parseMoney('1.249,90 EUR')).toEqual({ amount: 1249.9, currency: 'EUR' });
+ expect(parseMoney('15,00 EUR')).toEqual({ amount: 15, currency: 'EUR' });
+ expect(parseMoney('€39')).toEqual({ amount: 39, currency: 'EUR' });
+ });
+
+ it("isn't a price when it says something else", () => {
+ expect(parseMoney('')).toBeNull();
+ expect(parseMoney('free')).toBeNull();
+ expect(parseMoney('15.00 USD EUR')).toBeNull();
+ expect(parseMoney('USD 15 EUR')).toBeNull();
+ expect(parseMoney('-5 USD')).toBeNull();
+ });
+});
+
+describe('parseGoogleFeedXml', () => {
+ it('reads an RSS 2.0 feed: a variant group, repeated images, CDATA and entities', () => {
+ const { items, errors } = parseGoogleFeedXml(fixture('google-feed.xml'));
+ expect(errors).toEqual([]);
+ expect(items.map((i) => i.id)).toEqual(['TR-100-RED', 'TR-100-BLUE', 'TR-100-GREEN', 'SOCK-1']);
+ // One item group, three colors.
+ const group = items.filter((i) => i.itemGroupId === 'TR-100');
+ expect(group.map((i) => i.color)).toEqual(['Red', 'Blue', 'Green']);
+ expect(items[0]).toEqual({
+ id: 'TR-100-RED',
+ itemGroupId: 'TR-100',
+ title: 'Trailrunner Hiking Boot - Red',
+ description: 'Waterproof leather & mesh. <b>Grips wet rock.</b>',
+ link: 'https://shop.example/trailrunner?color=red',
+ imageLink: 'https://shop.example/img/tr-red.jpg',
+ additionalImageLinks: [
+ 'https://shop.example/img/tr-red-2.jpg',
+ 'https://shop.example/img/tr-red-3.jpg',
+ ],
+ brand: 'Acme',
+ gtin: '4006381333931',
+ mpn: null,
+ price: { amount: 129, currency: 'USD' },
+ salePrice: { amount: 99, currency: 'USD' },
+ availability: 'in_stock',
+ condition: 'new',
+ color: 'Red',
+ size: '42',
+ material: 'Leather',
+ productType: 'Shoes > Hiking Boots',
+ });
+ // A price with no currency, and no group.
+ expect(items[3]).toMatchObject({
+ itemGroupId: null,
+ price: { amount: 18.5, currency: null },
+ availability: 'preorder',
+ condition: null,
+ });
+ });
+
+ it('reads the same products from an Atom feed', () => {
+ expect(parseGoogleFeedXml(fixture('google-feed-atom.xml'))).toEqual(
+ parseGoogleFeedXml(fixture('google-feed.xml')),
+ );
+ });
+
+ it('skips an item missing a required field, naming its item number', () => {
+ const xml = `<rss xmlns:g="http://base.google.com/ns/1.0" version="2.0"><channel>
+ <item><g:id>A</g:id><title>A</title><link>https://s.example/a</link><g:price>1 USD</g:price><g:availability>in stock</g:availability></item>
+ <item><g:id>B</g:id><title>B</title><link>https://s.example/b</link><g:availability>in stock</g:availability></item>
+ <item><g:id>A</g:id><title>A again</title><link>https://s.example/a2</link><g:price>2 USD</g:price><g:availability>in stock</g:availability></item>
+ </channel></rss>`;
+ const { items, errors } = parseGoogleFeedXml(xml);
+ expect(items.map((i) => i.id)).toEqual(['A']);
+ expect(errors).toEqual([
+ { row: 2, field: 'price', found: '', message: 'Item 2: price is missing.' },
+ {
+ row: 3,
+ field: 'id',
+ found: 'A',
+ message: 'Item 3: id "A" is already used by an earlier product.',
+ },
+ ]);
+ });
+
+ it("fails once on a file that isn't a feed", () => {
+ expect(() => parseGoogleFeedXml('<html><body>Not a feed</body></html>')).toThrow(
+ FeedFormatError,
+ );
+ expect(() => parseGoogleFeedXml('')).toThrow('The file is empty.');
+ });
+});
+
+describe('parseFacebookCatalogCsv', () => {
+ it('reads the same products as the Google feed', () => {
+ expect(parseFacebookCatalogCsv(fixture('facebook-catalog.csv'))).toEqual(
+ parseGoogleFeedXml(fixture('google-feed.xml')),
+ );
+ });
+
+ it('uses rich_text_description when there is no description column', () => {
+ const csv =
+ 'id,title,rich_text_description,availability,price,link\n' +
+ '1,Boot,<p>Waterproof</p>,in stock,10 USD,https://s.example/1';
+ expect(parseFacebookCatalogCsv(csv).items[0].description).toBe('<p>Waterproof</p>');
+ });
+});
+
+describe('parseCsvFeed', () => {
+ it("reads a store's export: its header names, a BOM, quoted fields, and a row missing its price", () => {
+ const { items, errors } = parseCsvFeed(fixture('store-export.csv'));
+ expect(items.map((i) => i.id)).toEqual(['B-1', 'B-3']);
+ expect(items[0]).toMatchObject({
+ title: 'Summit Backpack 30L',
+ description: 'Roomy, light and\nrain-proof: "ready for anything".',
+ imageLink: 'https://shop.example/img/summit.jpg',
+ gtin: '8690000000017',
+ price: { amount: 1249.9, currency: 'TRY' },
+ availability: 'in_stock',
+ });
+ expect(items[1]).toMatchObject({
+ imageLink: null,
+ price: { amount: 39, currency: 'EUR' },
+ availability: 'in_stock',
+ });
+ // Row 3: the header is row 1, and B-1's quoted line break doesn't count as a row.
+ expect(errors).toEqual([
+ { row: 3, field: 'price', found: '', message: 'Row 3: price is missing.' },
+ ]);
+ });
+
+ it('reads other headers through a mapping', () => {
+ const csv =
+ 'SKU,Product name,URL,Cost,Stock\n' + 'S-1,Camp Mug,https://s.example/mug,7.5 USD,in stock';
+ const { items } = parseCsvFeed(csv, {
+ id: 'SKU',
+ title: 'Product name',
+ price: 'Cost',
+ availability: 'stock',
+ });
+ expect(items[0]).toMatchObject({
+ id: 'S-1',
+ title: 'Camp Mug',
+ link: 'https://s.example/mug',
+ price: { amount: 7.5, currency: 'USD' },
+ });
+ });
+
+ it('names a value that is there but invalid', () => {
+ const csv = 'id,title,link,price,availability\n1,Mug,https://s.example/m,free,in stock';
+ expect(parseCsvFeed(csv).errors).toEqual([
+ {
+ row: 2,
+ field: 'price',
+ found: 'free',
+ message: 'Row 2: price "free" isn\'t a valid price.',
+ },
+ ]);
+ });
+
+ it("fails once on a file that isn't a product feed", () => {
+ expect(() => parseCsvFeed('name,email\nAda,[email protected]')).toThrow(
+ /no id, link, price, availability column/,
+ );
+ expect(() => parseCsvFeed('id,title\n1,x', { colour: 'id' })).toThrow(FeedFormatError);
+ expect(() => parseCsvFeed('id,title\n1,x', { link: 'Url' })).toThrow(
+ 'The mapping\'s column "Url" isn\'t in the file.',
+ );
+ expect(() => parseCsvFeed(' ')).toThrow('The file is empty.');
+ });
+
+ it('reads a 10,000-row export', () => {
+ const rows = ['id,title,link,price,availability'];
+ for (let i = 0; i < 10_000; i++) {
+ rows.push(`P-${i},"Product ${i}, blue",https://s.example/p/${i},${i}.99 USD,in stock`);
+ }
+ const { items, errors } = parseCsvFeed(rows.join('\n'));
+ expect(errors).toEqual([]);
+ expect(items).toHaveLength(10_000);
+ expect(items[9_999]).toMatchObject({
+ title: 'Product 9999, blue',
+ price: { amount: 9999.99, currency: 'USD' },
+ });
+ });
+});
diff --git a/server/src/lib/feeds/index.js b/server/src/lib/feeds/index.js
new file mode 100644
index 000000000..c17d9ceaa
--- /dev/null
+++ b/server/src/lib/feeds/index.js
@@ -0,0 +1,194 @@
+import * as cheerio from 'cheerio';
+import { csvRows } from './csv.js';
+import { FEED_ATTRIBUTES, FeedFormatError, REQUIRED_FIELDS, toItem } from './item.js';
+
+export { FeedFormatError } from './item.js';
+
+/**
+ * Product feed parsers (#1264): a store's CSV export, a Google product feed
+ * (XML) and a Meta (Facebook) catalog CSV, each into the catalog's item shape
+ * (`item.js`), for the feed connector and the CSV import (#1263) and the
+ * Merchant Center connector. Pure: no network, no database.
+ *
+ * Each returns `{ items, errors }`. A product missing a required field (id,
+ * title, link, price, availability) is left out, with an error naming its row
+ * or item, the field and what was there. A product whose id comes again is
+ * left out the same way: the catalog keys products by it. A file that isn't
+ * the expected format throws one FeedFormatError.
+ */
+
+/** Header names read as a feed attribute, besides the attribute's own name. */
+const HEADER_ALIASES = {
+ url: 'link',
+ product_url: 'link',
+ image_url: 'image_link',
+ additional_image_links: 'additional_image_link',
+ name: 'title',
+ product_name: 'title',
+};
+
+/** "Image Link", "g:image_link", "image-link" → "image_link". */
+function headerKey(header) {
+ return header
+ .trim()
+ .toLowerCase()
+ .replace(/^g:/, '')
+ .replace(/[\s-]+/g, '_');
+}
+
+/**
+ * The column of each feed attribute: an explicit mapping first (attribute →
+ * header, any case), then the attribute's own name, then a common alias.
+ */
+function columnsOf(header, mapping = {}, fallbacks = {}) {
+ const keys = header.map(headerKey);
+ const columns = {};
+ for (const [index, key] of keys.entries()) {
+ const attribute = key in FEED_ATTRIBUTES ? key : HEADER_ALIASES[key];
+ if (attribute && !(attribute in columns)) columns[attribute] = index;
+ }
+ for (const [attribute, fallback] of Object.entries(fallbacks)) {
+ const index = keys.indexOf(fallback);
+ if (!(attribute in columns) && index >= 0) columns[attribute] = index;
+ }
+ for (const [attribute, name] of Object.entries(mapping)) {
+ if (!(attribute in FEED_ATTRIBUTES)) {
+ throw new FeedFormatError(`The mapping names an unknown field: ${attribute}.`);
+ }
+ const index = keys.indexOf(headerKey(name));
+ if (index < 0) throw new FeedFormatError(`The mapping's column "${name}" isn't in the file.`);
+ columns[attribute] = index;
+ }
+ return columns;
+}
+
+/** Keeps the first product with each id; a later one is an error. */
+function collect(results, unit) {
+ const items = [];
+ const errors = [];
+ const seen = new Set();
+ for (const { row, result } of results) {
+ if ('error' in result) {
+ errors.push(result.error);
+ } else if (seen.has(result.item.id)) {
+ errors.push({
+ row,
+ field: 'id',
+ found: result.item.id,
+ message: `${unit} ${row}: id "${result.item.id}" is already used by an earlier product.`,
+ });
+ } else {
+ seen.add(result.item.id);
+ items.push(result.item);
+ }
+ }
+ return { items, errors };
+}
+
+function parseCsv(text, mapping, fallbacks) {
+ if (typeof text !== 'string' || !text.trim()) throw new FeedFormatError('The file is empty.');
+ const rows = csvRows(text);
+ const first = rows.next();
+ if (first.done) throw new FeedFormatError('The file is empty.');
+ const columns = columnsOf(first.value.fields, mapping, fallbacks);
+ const missing = REQUIRED_FIELDS.filter((field) => !(field in columns));
+ if (missing.length) {
+ throw new FeedFormatError(
+ `This isn't a product feed CSV: no ${missing.join(', ')} column. Check the header row, or map the columns.`,
+ );
+ }
+ // Rows as a spreadsheet numbers them: the header is row 1.
+ let row = 1;
+ const results = (function* () {
+ for (const { fields } of rows) {
+ row++;
+ const attrs = {};
+ for (const [attribute, index] of Object.entries(columns)) attrs[attribute] = fields[index];
+ yield { row, result: toItem(attrs, row, 'Row') };
+ }
+ })();
+ return collect(results, 'Row');
+}
+
+/**
+ * A CSV with a header row. Columns named after Google's attributes (`id`,
+ * `title`, `description`, `link`, `image_link`, `price`, `gtin`, `brand`,
+ * `availability`, …) map by themselves; `mapping` names others, e.g.
+ * `{ id: 'SKU', title: 'Product name' }`.
+ *
+ * @param {string} text
+ * @param {Record<string, string>} [mapping] feed attribute → header
+ * @returns {import('./item.js').FeedResult}
+ */
+export function parseCsvFeed(text, mapping) {
+ return parseCsv(text, mapping, {});
+}
+
+/**
+ * A Meta (Facebook) catalog CSV. Its columns share Google's names; a
+ * catalog with only `rich_text_description` uses it as the description.
+ *
+ * @param {string} text
+ * @returns {import('./item.js').FeedResult}
+ */
+export function parseFacebookCatalogCsv(text) {
+ return parseCsv(text, undefined, { description: 'rich_text_description' });
+}
+
+const ITEM = /<item(?:\s[^>]*)?>([\s\S]*?)<\/item\s*>/g;
+const ENTRY = /<entry(?:\s[^>]*)?>([\s\S]*?)<\/entry\s*>/g;
+
+/** An item's or entry's attributes, by feed attribute name. */
+function xmlAttrs(block) {
+ const $ = cheerio.load(block, { xml: true });
+ const attrs = {};
+ $.root()
+ .children()
+ .each((_, el) => {
+ const key = headerKey(el.tagName ?? '');
+ const attribute = key === 'summary' || key === 'content' ? 'description' : key;
+ if (!(attribute in FEED_ATTRIBUTES)) return;
+ const node = $(el);
+ // Atom: <link href="…"/>, the product's own page (rel alternate, or none).
+ const value =
+ attribute === 'link' && node.attr('href') !== undefined
+ ? ['', 'alternate'].includes(node.attr('rel') ?? '')
+ ? node.attr('href')
+ : undefined
+ : node.text();
+ if (value === undefined || !value.trim()) return;
+ if (attribute === 'additional_image_link') {
+ attrs[attribute] = [...(attrs[attribute] ?? []), value];
+ } else if (attrs[attribute] === undefined) {
+ attrs[attribute] = value;
+ }
+ });
+ return attrs;
+}
+
+/**
+ * A Google product feed in XML: RSS 2.0 with the `g:` namespace, or Atom.
+ * Items are read one at a time, so a large feed isn't held twice.
+ *
+ * @param {string} text
+ * @returns {import('./item.js').FeedResult}
+ */
+export function parseGoogleFeedXml(text) {
+ if (typeof text !== 'string' || !text.trim()) throw new FeedFormatError('The file is empty.');
+ const head = text.slice(0, 4096);
+ const atom = /<feed[\s>]/.test(head);
+ if (!atom && !/<rss[\s>]/.test(head)) {
+ throw new FeedFormatError(
+ "This isn't a Google product feed: it has no <rss> or <feed> element.",
+ );
+ }
+ const pattern = new RegExp(atom ? ENTRY : ITEM);
+ let item = 0;
+ const results = (function* () {
+ for (let m = pattern.exec(text); m; m = pattern.exec(text)) {
+ item++;
+ yield { row: item, result: toItem(xmlAttrs(m[1]), item, 'Item') };
+ }
+ })();
+ return collect(results, 'Item');
+}
diff --git a/server/src/lib/feeds/item.js b/server/src/lib/feeds/item.js
new file mode 100644
index 000000000..23839f3fb
--- /dev/null
+++ b/server/src/lib/feeds/item.js
@@ -0,0 +1,209 @@
+/**
+ * One product from a feed, in the catalog's shape (#1263): every parser in
+ * this folder returns these, whatever the format.
+ *
+ * @typedef {{ amount: number, currency: string | null }} Money
+ *
+ * @typedef {{
+ * id: string,
+ * itemGroupId: string | null,
+ * title: string,
+ * description: string | null,
+ * link: string,
+ * imageLink: string | null,
+ * additionalImageLinks: string[],
+ * brand: string | null,
+ * gtin: string | null,
+ * mpn: string | null,
+ * price: Money,
+ * salePrice: Money | null,
+ * availability: string,
+ * condition: string | null,
+ * color: string | null,
+ * size: string | null,
+ * material: string | null,
+ * productType: string | null,
+ * }} FeedItem
+ *
+ * A product the feed lists but that can't be kept: its row (CSV) or item
+ * (XML) number, counted from 1, the field, and what was there.
+ * @typedef {{ row: number, field: string, found: string, message: string }} FeedError
+ *
+ * @typedef {{ items: FeedItem[], errors: FeedError[] }} FeedResult
+ */
+
+/** The fields a product must have to be kept. */
+export const REQUIRED_FIELDS = ['id', 'title', 'link', 'price', 'availability'];
+
+/**
+ * The feed attributes read, by their Google / Meta name, and the item field
+ * each fills. `additional_image_link` may repeat (XML) or hold several
+ * addresses (CSV).
+ */
+export const FEED_ATTRIBUTES = {
+ id: 'id',
+ item_group_id: 'itemGroupId',
+ title: 'title',
+ description: 'description',
+ link: 'link',
+ image_link: 'imageLink',
+ additional_image_link: 'additionalImageLinks',
+ brand: 'brand',
+ gtin: 'gtin',
+ mpn: 'mpn',
+ price: 'price',
+ sale_price: 'salePrice',
+ availability: 'availability',
+ condition: 'condition',
+ color: 'color',
+ size: 'size',
+ material: 'material',
+ product_type: 'productType',
+};
+
+/** A format the parser doesn't read: one error for the whole file. */
+export class FeedFormatError extends Error {
+ constructor(message) {
+ super(message);
+ this.name = 'FeedFormatError';
+ }
+}
+
+const SYMBOL_CURRENCY = { '€': 'EUR', '£': 'GBP', '₺': 'TRY', '¥': 'JPY', '₹': 'INR' };
+
+/** A decimal written with either separator, thousands grouped or not: "1,234.50", "1.234,50", "15,00". */
+function amountOf(raw) {
+ // \s covers the no-break space too.
+ const digits = raw.replace(/[\s']/g, '');
+ if (!/^\d[\d.,]*$/.test(digits)) return null;
+ const lastDot = digits.lastIndexOf('.');
+ const lastComma = digits.lastIndexOf(',');
+ let normalized;
+ if (lastDot >= 0 && lastComma >= 0) {
+ // Both: the later one is the decimal mark.
+ const decimal = lastDot > lastComma ? '.' : ',';
+ const thousands = decimal === '.' ? ',' : '.';
+ normalized = digits.split(thousands).join('').replace(decimal, '.');
+ } else {
+ const sep = lastDot >= 0 ? '.' : lastComma >= 0 ? ',' : null;
+ if (!sep) normalized = digits;
+ else {
+ const parts = digits.split(sep);
+ const tail = parts[parts.length - 1];
+ // One mark with one or two digits after it is a decimal; groups of three are thousands.
+ normalized = parts.length === 2 && tail.length !== 3 ? parts.join('.') : parts.join('');
+ }
+ }
+ const amount = Number(normalized);
+ return Number.isFinite(amount) && amount >= 0 ? amount : null;
+}
+
+/**
+ * A price as feeds write it: "15.00 USD", "USD 15.00", "15.00" (no
+ * currency), "€15,00". Null when it isn't one.
+ *
+ * @param {string | null | undefined} value
+ * @returns {Money | null}
+ */
+export function parseMoney(value) {
+ let text = String(value ?? '').trim();
+ if (!text) return null;
+ let currency = null;
+ const symbol = text.match(/^[€£₺¥₹$]|[€£₺¥₹$]$/u);
+ if (symbol) {
+ currency = SYMBOL_CURRENCY[symbol[0]] ?? null;
+ text = text.replace(symbol[0], '').trim();
+ }
+ const m = /^(?:([A-Za-z]{3})\s*)?(.+?)(?:\s*([A-Za-z]{3}))?$/.exec(text);
+ if (!m) return null;
+ const [, before, number, after] = m;
+ const code = (before ?? after)?.toUpperCase() ?? null;
+ if (before && after && before.toUpperCase() !== after.toUpperCase()) return null;
+ const amount = amountOf(number);
+ if (amount === null) return null;
+ return { amount, currency: code ?? currency };
+}
+
+/** "in stock", "In-Stock", "in_stock" → "in_stock". */
+export function normalizeAvailability(value) {
+ const text = String(value ?? '')
+ .trim()
+ .toLowerCase()
+ .replace(/[\s-]+/g, '_');
+ return text || null;
+}
+
+/** Several image addresses in one field, split where the next address starts. */
+export function splitImageLinks(value) {
+ return String(value ?? '')
+ .split(/[,\s]+(?=https?:\/\/)/i)
+ .map((s) => s.trim())
+ .filter(Boolean);
+}
+
+const clean = (value) => {
+ const text = typeof value === 'string' ? value.trim() : '';
+ return text || null;
+};
+
+/**
+ * A product from its feed attributes, or the first required field it lacks.
+ *
+ * @param {Record<string, string | string[] | undefined>} attrs by feed attribute name
+ * @param {number} row
+ * @param {'Row' | 'Item'} unit how the message names it: a CSV's row, an XML feed's item
+ * @returns {{ item: FeedItem } | { error: FeedError }}
+ */
+export function toItem(attrs, row, unit = 'Row') {
+ const first = (name) => {
+ const value = attrs[name];
+ return clean(Array.isArray(value) ? value[0] : value);
+ };
+ const price = parseMoney(first('price'));
+ const values = {
+ id: first('id'),
+ title: first('title'),
+ link: first('link'),
+ price,
+ availability: normalizeAvailability(first('availability')),
+ };
+ for (const field of REQUIRED_FIELDS) {
+ if (values[field] === null) {
+ const found = first(field);
+ return {
+ error: {
+ row,
+ field,
+ found: found ?? '',
+ message: found
+ ? `${unit} ${row}: ${field} "${found}" isn't a valid ${field}.`
+ : `${unit} ${row}: ${field} is missing.`,
+ },
+ };
+ }
+ }
+ const extra = attrs.additional_image_link;
+ const additional = (Array.isArray(extra) ? extra : [extra ?? '']).flatMap(splitImageLinks);
+ return {
+ item: {
+ id: values.id,
+ itemGroupId: first('item_group_id'),
+ title: values.title,
+ description: first('description'),
+ link: values.link,
+ imageLink: first('image_link'),
+ additionalImageLinks: [...new Set(additional)],
+ brand: first('brand'),
+ gtin: first('gtin'),
+ mpn: first('mpn'),
+ price: values.price,
+ salePrice: parseMoney(first('sale_price')),
+ availability: values.availability,
+ condition: first('condition')?.toLowerCase() ?? null,
+ color: first('color'),
+ size: first('size'),
+ material: first('material'),
+ productType: first('product_type'),
+ },
+ };
+}