Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

fix(utils): count a microdata property once when itemref reaches it again (#4344)

SwastikTripathicf98a4a3ce2d
diff --git a/packages/utils/src/internals/extract-microdata.ts b/packages/utils/src/internals/extract-microdata.ts
index ab2b2f4c3..169b228d9 100644
--- a/packages/utils/src/internals/extract-microdata.ts
+++ b/packages/utils/src/internals/extract-microdata.ts
@@ -75,33 +75,38 @@ function parseItem(context: ExtractionContext, element: Element, ancestors: Set<
function collectPropertyElements(context: ExtractionContext, scope: Element): Element[] {
const elements: Element[] = [];
+ // An `itemref` can point at an element already reached from the item's own subtree, or at one that
+ // contains another referenced element. The spec visits every element once, so each property counts once.
+ const visited = new Set<Element>();
- collectFromNodes(scope.children, elements);
+ collectFromNodes(scope.children, elements, visited);
for (const id of uniqueTokens(scope.attribs.itemref)) {
const referenced = (context.idIndex ??= indexIds(context.$)).get(id);
if (referenced) {
- collectFromNodes([referenced], elements);
+ collectFromNodes([referenced], elements, visited);
}
}
return elements;
}
-function collectFromNodes(nodes: AnyNode[], elements: Element[]): void {
+function collectFromNodes(nodes: AnyNode[], elements: Element[], visited: Set<Element>): void {
for (const node of nodes) {
- if (!isTag(node)) {
+ if (!isTag(node) || visited.has(node)) {
continue;
}
+ visited.add(node);
+
if ('itemprop' in node.attribs) {
elements.push(node);
}
// A nested item owns everything below it, so its subtree is not part of the enclosing item.
if (!('itemscope' in node.attribs)) {
- collectFromNodes(node.children, elements);
+ collectFromNodes(node.children, elements, visited);
}
}
}
diff --git a/test/utils/extract-microdata.test.ts b/test/utils/extract-microdata.test.ts
index 37491f9f8..da50b712d 100644
--- a/test/utils/extract-microdata.test.ts
+++ b/test/utils/extract-microdata.test.ts
@@ -127,6 +127,20 @@ describe('extractMicrodata', () => {
]);
});
+ it('counts a property once when itemref reaches it a second time', async () => {
+ // `name` is a child of the item and also referenced, and `details` contains the referenced `price`.
+ const items = await extractMicrodata(`
+ <div itemscope itemref="product-name product-details product-price">
+ <span id="product-name" itemprop="name">Lamp</span>
+ </div>
+ <div id="product-details">
+ <span itemprop="color">Red</span>
+ <span id="product-price" itemprop="price">25</span>
+ </div>`);
+
+ expect(items).toEqual([{ properties: { name: 'Lamp', color: 'Red', price: '25' } }]);
+ });
+
it('stops at an itemref cycle instead of recursing forever', async () => {
// `loop-a` and `loop-b` reference each other, so the second visit of `loop-a` must be cut short.
const items = await extractMicrodata(`