Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

fix(utils): use article:* selectors for article Open Graph properties (#3789)

Anas Khanb6605065fc4d
diff --git a/packages/utils/src/internals/open_graph_parser.ts b/packages/utils/src/internals/open_graph_parser.ts
index ad6d8a371..edeb5ecc6 100644
--- a/packages/utils/src/internals/open_graph_parser.ts
+++ b/packages/utils/src/internals/open_graph_parser.ts
@@ -268,32 +268,32 @@ const OPEN_GRAPH_PROPERTIES: OpenGraphProperty[] = [
outputName: 'articleInfo',
children: [
{
- name: 'music:published_time',
+ name: 'article:published_time',
outputName: 'publishedTime',
children: [],
},
{
- name: 'music:modified_time',
+ name: 'article:modified_time',
outputName: 'modifiedTime',
children: [],
},
{
- name: 'music:expiration_time',
+ name: 'article:expiration_time',
outputName: 'expirationTime',
children: [],
},
{
- name: 'music:author',
+ name: 'article:author',
outputName: 'author',
children: [],
},
{
- name: 'music:section',
+ name: 'article:section',
outputName: 'section',
children: [],
},
{
- name: 'music:tag',
+ name: 'article:tag',
outputName: 'tag',
children: [],
},
diff --git a/test/utils/open_graph_parser.test.ts b/test/utils/open_graph_parser.test.ts
index ac68674fd..0d490d578 100644
--- a/test/utils/open_graph_parser.test.ts
+++ b/test/utils/open_graph_parser.test.ts
@@ -21,6 +21,14 @@ describe('parseOpenGraph', () => {
const case6 = `<meta property="og:title" content="My Website"/>
<meta property="og:type" content="website"/>`;
+ const case7 = load(`<meta property="og:type" content="article"/>
+ <meta property="article:published_time" content="2024-01-01T00:00:00Z"/>
+ <meta property="article:modified_time" content="2024-01-02T00:00:00Z"/>
+ <meta property="article:expiration_time" content="2024-01-03T00:00:00Z"/>
+ <meta property="article:author" content="Jane Doe"/>
+ <meta property="article:section" content="Tech"/>
+ <meta property="article:tag" content="ai"/>`);
+
it('Should scrape properties', () => {
expect(parseOpenGraph(case1)).toEqual({
title: 'Under Pressure',
@@ -77,4 +85,18 @@ describe('parseOpenGraph', () => {
type: 'website',
});
});
+
+ it('Should parse article properties into articleInfo', () => {
+ expect(parseOpenGraph(case7)).toEqual({
+ type: 'article',
+ articleInfo: {
+ publishedTime: '2024-01-01T00:00:00Z',
+ modifiedTime: '2024-01-02T00:00:00Z',
+ expirationTime: '2024-01-03T00:00:00Z',
+ author: 'Jane Doe',
+ section: 'Tech',
+ tag: 'ai',
+ },
+ });
+ });
});