Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
feat(utils): add `extractMicrodata` helper (#3233)
Danish Aroraa6961b21779e
diff --git a/docs/public-api/crawlee-utils.api.md b/docs/public-api/crawlee-utils.api.mdindex e9ab3d9c4..f0f360100 100644--- a/docs/public-api/crawlee-utils.api.md+++ b/docs/public-api/crawlee-utils.api.md@@ -65,6 +65,12 @@ export enum EnqueueStrategy {SameOrigin = "same-origin"}+// @public+export function extractMicrodata(raw: string): Promise<MicrodataItem[]>;++// @public (undocumented)+export function extractMicrodata($: CheerioAPI): Promise<MicrodataItem[]>;+// @publicexport function extractUrls(options: ExtractUrlsOptions): string[];@@ -95,6 +101,16 @@ const LINKEDIN_REGEX: RegExp;// @publicconst LINKEDIN_REGEX_GLOBAL: RegExp;+// @public+export interface MicrodataItem {+ id?: string;+ properties: Record<string, MicrodataValue | MicrodataValue[]>;+ type?: string[];+}++// @public+export type MicrodataValue = string | MicrodataItem;+// Not exported by the entry point; reachable only as a referenced type.// @public (undocumented)interface NestedSitemap {diff --git a/docs/public-api/crawlee.api.md b/docs/public-api/crawlee.api.mdindex 86806b621..0f03126c3 100644--- a/docs/public-api/crawlee.api.md+++ b/docs/public-api/crawlee.api.md@@ -5,6 +5,7 @@```tsimport { downloadListOfUrls } from '@crawlee/utils';+import { extractMicrodata } from '@crawlee/utils';import { Log } from '@apify/log';import { parseOpenGraph } from '@crawlee/utils';import { playwrightUtils } from '@crawlee/playwright';@@ -21,6 +22,7 @@ export const utils: {sleep: typeof sleep;downloadListOfUrls: typeof downloadListOfUrls;parseOpenGraph: typeof parseOpenGraph;+ extractMicrodata: typeof extractMicrodata;};diff --git a/packages/crawlee/src/index.ts b/packages/crawlee/src/index.tsindex 83940ac1d..54c490ccf 100644--- a/packages/crawlee/src/index.ts+++ b/packages/crawlee/src/index.ts@@ -1,7 +1,7 @@import { log } from '@crawlee/core';import { playwrightUtils } from '@crawlee/playwright';import { puppeteerUtils } from '@crawlee/puppeteer';-import { downloadListOfUrls, parseOpenGraph, sleep, social } from '@crawlee/utils';+import { downloadListOfUrls, extractMicrodata, parseOpenGraph, sleep, social } from '@crawlee/utils';export * from '@crawlee/core';export * from '@crawlee/utils';@@ -24,4 +24,5 @@ export const utils = {sleep,downloadListOfUrls,parseOpenGraph,+ extractMicrodata,};diff --git a/packages/utils/src/index.ts b/packages/utils/src/index.tsindex 0a81aadf9..70f212653 100644--- a/packages/utils/src/index.ts+++ b/packages/utils/src/index.ts@@ -4,6 +4,7 @@ export { EnqueueStrategy } from './internals/url.js';export type { DownloadListOfUrlsOptions, ExtractUrlsOptions } from './internals/extract-urls.js';export { sleep, expandShadowRoots } from './internals/general.js';export * as social from './internals/social.js';+export * from './internals/extract-microdata.js';export * from './internals/open_graph_parser.js';export * from './internals/robots.js';export * from './internals/sitemap.js';diff --git a/packages/utils/src/internals/extract-microdata.ts b/packages/utils/src/internals/extract-microdata.tsnew file mode 100644index 000000000..a9b222196--- /dev/null+++ b/packages/utils/src/internals/extract-microdata.ts@@ -0,0 +1,182 @@+import type { CheerioAPI } from 'cheerio';+import type { AnyNode, Element } from 'domhandler';+import { isTag } from 'domhandler';++/** The value of a microdata property: either text or a nested item. */+export type MicrodataValue = string | MicrodataItem;++/** A single schema.org item extracted from a document. */+export interface MicrodataItem {+ /** Tokens of the item's `itemtype` attribute. */+ type?: string[];+ /** The item's `itemid` attribute. */+ id?: string;+ /** Values keyed by `itemprop` name, an array where the property repeats. */+ properties: Record<string, MicrodataValue | MicrodataValue[]>;+}++interface ExtractionContext {+ $: CheerioAPI;+ /** Built on first `itemref` lookup, which most documents never trigger. */+ idIndex?: Map<string, Element>;+}++/**+ * Easily parse all schema.org microdata from a page with just a `CheerioAPI` object or raw HTML,+ * following the [microdata processing model](https://html.spec.whatwg.org/multipage/microdata.html#microdata).+ *+ * Text values are trimmed and their inner whitespace collapsed. URL-valued attributes are returned+ * verbatim rather than resolved against the document's base URL.+ *+ * @param htmlOrCheerioElement A `CheerioAPI` object, or a string of raw HTML.+ * @returns The document's top-level items. Nested items are the property values of their parent.+ */+export async function extractMicrodata(raw: string): Promise<MicrodataItem[]>;+export async function extractMicrodata($: CheerioAPI): Promise<MicrodataItem[]>;+export async function extractMicrodata(htmlOrCheerioElement: string | CheerioAPI): Promise<MicrodataItem[]> {+ // Dynamic so that importing `@crawlee/utils` does not pull in cheerio - see #3836.+ const { load } = await import('cheerio');+ const $ = typeof htmlOrCheerioElement === 'string' ? load(htmlOrCheerioElement) : htmlOrCheerioElement;+ const context: ExtractionContext = { $ };++ return $('[itemscope]')+ .toArray()+ .filter((element) => !('itemprop' in element.attribs))+ .map((element) => parseItem(context, element, new Set()));+}++function parseItem(context: ExtractionContext, element: Element, ancestors: Set<Element>): MicrodataItem {+ const item: MicrodataItem = { properties: {} };+ const type = uniqueTokens(element.attribs.itemtype);+ const id = element.attribs.itemid;++ if (type.length > 0) {+ item.type = type;+ }++ if (id) {+ item.id = id.trim();+ }++ ancestors.add(element);++ for (const propertyElement of collectPropertyElements(context, element)) {+ const value = getPropertyValue(context, propertyElement, ancestors);++ for (const name of uniqueTokens(propertyElement.attribs.itemprop)) {+ addProperty(item.properties, name, value);+ }+ }++ ancestors.delete(element);++ return item;+}++function collectPropertyElements(context: ExtractionContext, scope: Element): Element[] {+ const elements: Element[] = [];++ collectFromNodes(scope.children, elements);++ for (const id of uniqueTokens(scope.attribs.itemref)) {+ const referenced = (context.idIndex ??= indexIds(context.$)).get(id);++ if (referenced) {+ collectFromNodes([referenced], elements);+ }+ }++ return elements;+}++function collectFromNodes(nodes: AnyNode[], elements: Element[]): void {+ for (const node of nodes) {+ if (!isTag(node)) {+ continue;+ }++ if ('itemprop' in node.attribs) {+ elements.push(node);+ }++ // A nested item owns everything below it, so its subtree is not part of the enclosing item.+ if (!('itemscope' in node.attribs)) {+ collectFromNodes(node.children, elements);+ }+ }+}++function indexIds($: CheerioAPI): Map<string, Element> {+ const index = new Map<string, Element>();++ for (const element of $('[id]').toArray()) {+ // Duplicate ids are invalid HTML; `getElementById` resolves them to the first element.+ if (!index.has(element.attribs.id)) {+ index.set(element.attribs.id, element);+ }+ }++ return index;+}++function getPropertyValue(context: ExtractionContext, element: Element, ancestors: Set<Element>): MicrodataValue {+ if ('itemscope' in element.attribs) {+ // `itemref` can point back at an enclosing item, which the spec treats as an error.+ return ancestors.has(element) ? { properties: {} } : parseItem(context, element, ancestors);+ }++ const { attribs } = element;++ switch (element.tagName.toLowerCase()) {+ case 'meta':+ return attribs.content ?? '';+ case 'audio':+ case 'embed':+ case 'iframe':+ case 'img':+ case 'source':+ case 'track':+ case 'video':+ return attribs.src ?? '';+ case 'a':+ case 'area':+ case 'link':+ return attribs.href ?? '';+ case 'object':+ return attribs.data ?? '';+ case 'data':+ case 'meter':+ return attribs.value ?? '';+ case 'time':+ return attribs.datetime ?? context.$(element).text().replace(/\s+/g, ' ').trim();+ default:+ return context.$(element).text().replace(/\s+/g, ' ').trim();+ }+}++function addProperty(+ properties: Record<string, MicrodataValue | MicrodataValue[]>,+ name: string,+ value: MicrodataValue,+): void {+ const existing = properties[name];++ if (existing === undefined) {+ properties[name] = value;+ } else if (Array.isArray(existing)) {+ existing.push(value);+ } else {+ properties[name] = [existing, value];+ }+}++/** `itemtype`, `itemprop` and `itemref` are all unordered sets of unique space-separated tokens. */+function uniqueTokens(value: string | undefined): string[] {+ if (!value) {+ return [];+ }++ const tokens = value.split(/\s+/).filter(Boolean);++ return tokens.length > 1 ? [...new Set(tokens)] : tokens;+}diff --git a/test/utils/extract-microdata.test.ts b/test/utils/extract-microdata.test.tsnew file mode 100644index 000000000..37491f9f8--- /dev/null+++ b/test/utils/extract-microdata.test.ts@@ -0,0 +1,157 @@+import { extractMicrodata } from '@crawlee/utils';+import { load } from 'cheerio';++describe('extractMicrodata', () => {+ it('returns no items for a document without microdata', async () => {+ expect(await extractMicrodata('<p>Nothing to see here</p>')).toEqual([]);+ });++ it('accepts a CheerioAPI object', async () => {+ const $ = load('<div itemscope><span itemprop="name">Existing DOM</span></div>');++ expect(await extractMicrodata($)).toEqual([{ properties: { name: 'Existing DOM' } }]);+ });++ it('splits itemtype and itemprop token sets', async () => {+ const items = await extractMicrodata(`+ <div itemscope itemtype="https://schema.org/Product https://schema.org/Thing" itemid="urn:isbn:1">+ <span itemprop="favorite-color favorite-fruit">orange</span>+ </div>`);++ expect(items).toEqual([+ {+ type: ['https://schema.org/Product', 'https://schema.org/Thing'],+ id: 'urn:isbn:1',+ properties: { 'favorite-color': 'orange', 'favorite-fruit': 'orange' },+ },+ ]);+ });++ it('reads the value from the attribute belonging to each element type', async () => {+ const items = await extractMicrodata(`+ <div itemscope>+ <meta itemprop="sku" content="LAMP-1">+ <img itemprop="image" src="/lamp.jpg" alt="A lamp">+ <a itemprop="url" href="/products/lamp">View the lamp</a>+ <time itemprop="releaseDate" datetime="2026-08-29">Yesterday</time>+ <data itemprop="price" value="19.99">$19.99</data>+ <meter itemprop="rating" value="4.5">Great</meter>+ <object itemprop="model" data="/lamp.glb">A 3D model</object>+ <p itemprop="description"> Collapses any+ inner whitespace </p>+ <a itemprop="permalink">Not a link</a>+ </div>`);++ expect(items[0].properties).toEqual({+ sku: 'LAMP-1',+ image: '/lamp.jpg',+ url: '/products/lamp',+ releaseDate: '2026-08-29',+ price: '19.99',+ rating: '4.5',+ model: '/lamp.glb',+ description: 'Collapses any inner whitespace',+ permalink: '',+ });+ });++ it('groups a repeated property into an array', async () => {+ const items = await extractMicrodata(`+ <div itemscope>+ <span itemprop="tag">first</span>+ <span itemprop="tag">second</span>+ <span itemprop="tag">third</span>+ </div>`);++ expect(items[0].properties).toEqual({ tag: ['first', 'second', 'third'] });+ });++ it('nests an item used as a property value', async () => {+ const items = await extractMicrodata(`+ <div itemscope itemtype="https://schema.org/Product">+ <span itemprop="name">Coffee</span>+ <div itemprop="offers" itemscope itemtype="https://schema.org/Offer">+ <meta itemprop="priceCurrency" content="USD">+ <span itemprop="price">12</span>+ </div>+ </div>`);++ expect(items).toEqual([+ {+ type: ['https://schema.org/Product'],+ properties: {+ name: 'Coffee',+ offers: {+ type: ['https://schema.org/Offer'],+ properties: { priceCurrency: 'USD', price: '12' },+ },+ },+ },+ ]);+ });++ it('treats a nested itemscope without itemprop as a separate top-level item', async () => {+ const items = await extractMicrodata(`+ <div itemscope itemtype="https://schema.org/Article">+ <span itemprop="headline">Outer</span>+ <div itemscope itemtype="https://schema.org/Comment">+ <span itemprop="text">Inner</span>+ </div>+ </div>`);++ expect(items).toEqual([+ { type: ['https://schema.org/Article'], properties: { headline: 'Outer' } },+ { type: ['https://schema.org/Comment'], properties: { text: 'Inner' } },+ ]);+ });++ it('collects properties referenced with itemref', async () => {+ const items = await extractMicrodata(`+ <div itemscope itemtype="https://schema.org/Person" itemref="external-name external-contact">+ <span itemprop="jobTitle">Engineer</span>+ </div>+ <p id="external-name" itemprop="name">Ada Lovelace</p>+ <div id="external-contact">+ <a itemprop="url" href="https://example.com/ada">Profile</a>+ </div>`);++ expect(items).toEqual([+ {+ type: ['https://schema.org/Person'],+ properties: {+ jobTitle: 'Engineer',+ name: 'Ada Lovelace',+ url: 'https://example.com/ada',+ },+ },+ ]);+ });++ it('stops at an itemref cycle instead of recursing forever', async () => {+ // `loop-a` and `loop-b` reference each other, so the second visit of `loop-a` must be cut short.+ const items = await extractMicrodata(`+ <div itemscope itemref="loop-a">+ <span itemprop="name">root</span>+ </div>+ <div id="loop-a" itemscope itemprop="self" itemref="loop-b">+ <span itemprop="name">a</span>+ </div>+ <div id="loop-b" itemscope itemprop="self" itemref="loop-a">+ <span itemprop="name">b</span>+ </div>`);++ expect(items).toEqual([+ {+ properties: {+ name: 'root',+ self: {+ properties: {+ name: 'a',+ self: { properties: { name: 'b', self: { properties: {} } } },+ },+ },+ },+ },+ ]);+ });+});