Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
import { parseOpenGraph } from '@crawlee/utils';
import { load } from 'cheerio';
describe('parseOpenGraph', () => {
const case1 = load(`<meta property="og:title" content="Under Pressure"/>
<meta property="og:type" content="music.song"/>`);
const case2 = load(`<meta property="video:actor" content="foo"/>
<meta property="video:actor" content="bar"/>
<meta property="video:actor" content="baz"/>`);
const case3 = load(`<meta property="og:locale" content="test"/>
<meta property="og:locale:alternate" content="foo"/>
<meta property="og:locale:alternate" content="bar"/>`);
const case4 = load(`<meta property="music:song:disc" content="hello"/>
<meta property="music:song:track" content="world"/>`);
const case5 = load(`<meta property="og:custom:test" content="hello"/>`);
const case6 = `<meta property="og:title" content="My Website"/>
<meta property="og:type" content="website"/>`;
const case7 = load(`<meta property="og:type" content="article"/>
<meta property="article:published_time" content="2024-01-01T00:00:00Z"/>
<meta property="article:modified_time" content="2024-01-02T00:00:00Z"/>
<meta property="article:expiration_time" content="2024-01-03T00:00:00Z"/>
<meta property="article:author" content="Jane Doe"/>
<meta property="article:section" content="Tech"/>
<meta property="article:tag" content="ai"/>`);
it('Should scrape properties', async () => {
expect(await parseOpenGraph(case1)).toEqual({
title: 'Under Pressure',
type: 'music.song',
});
});
it('Should return a property as an array if there are multiple attributes under the same property name', async () => {
const parsed = (await parseOpenGraph(case2)) as {
videoInfo: { actor: { actorValue: string[] } };
};
expect(parsed).toHaveProperty('videoInfo');
expect(parsed.videoInfo.actor.actorValue).toContain('foo');
expect(parsed.videoInfo.actor.actorValue).toContain('bar');
expect(parsed.videoInfo.actor.actorValue).toContain('baz');
const parsed2 = (await parseOpenGraph(case3)) as {
locale: { localeValue: string; alternate: string[] };
};
expect(parsed2).toHaveProperty('locale');
expect(parsed2.locale.alternate).toContain('foo');
expect(parsed2.locale.alternate).toContain('bar');
});
it('Should parse properties regardless of how deeply they are nested', async () => {
expect(await parseOpenGraph(case4)).toEqual({
musicInfo: { song: { disc: 'hello', track: 'world' } },
});
});
it('Should accept additional OpenGraphProperties', async () => {
const parsed = await parseOpenGraph(case5, [
{
name: 'og:custom',
outputName: 'custom',
children: [
{
name: 'og:custom:test',
outputName: 'test',
children: [],
},
],
},
]);
expect(parsed).toEqual({ custom: { test: 'hello' } });
});
it('Should accept strings as a substitute for CheerioAPI objects', async () => {
expect(await parseOpenGraph(case6)).toEqual({
title: 'My Website',
type: 'website',
});
});
it('Should parse article properties into articleInfo', async () => {
expect(await parseOpenGraph(case7)).toEqual({
type: 'article',
articleInfo: {
publishedTime: '2024-01-01T00:00:00Z',
modifiedTime: '2024-01-02T00:00:00Z',
expirationTime: '2024-01-03T00:00:00Z',
author: 'Jane Doe',
section: 'Tech',
tag: 'ai',
},
});
});
});