Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
fix(utils): use article:* selectors for article Open Graph properties (#3789)
Anas Khanb6605065fc4d
diff --git a/packages/utils/src/internals/open_graph_parser.ts b/packages/utils/src/internals/open_graph_parser.tsindex ad6d8a371..edeb5ecc6 100644--- a/packages/utils/src/internals/open_graph_parser.ts+++ b/packages/utils/src/internals/open_graph_parser.ts@@ -268,32 +268,32 @@ const OPEN_GRAPH_PROPERTIES: OpenGraphProperty[] = [outputName: 'articleInfo',children: [{- name: 'music:published_time',+ name: 'article:published_time',outputName: 'publishedTime',children: [],},{- name: 'music:modified_time',+ name: 'article:modified_time',outputName: 'modifiedTime',children: [],},{- name: 'music:expiration_time',+ name: 'article:expiration_time',outputName: 'expirationTime',children: [],},{- name: 'music:author',+ name: 'article:author',outputName: 'author',children: [],},{- name: 'music:section',+ name: 'article:section',outputName: 'section',children: [],},{- name: 'music:tag',+ name: 'article:tag',outputName: 'tag',children: [],},diff --git a/test/utils/open_graph_parser.test.ts b/test/utils/open_graph_parser.test.tsindex ac68674fd..0d490d578 100644--- a/test/utils/open_graph_parser.test.ts+++ b/test/utils/open_graph_parser.test.ts@@ -21,6 +21,14 @@ describe('parseOpenGraph', () => {const case6 = `<meta property="og:title" content="My Website"/><meta property="og:type" content="website"/>`;+ const case7 = load(`<meta property="og:type" content="article"/>+ <meta property="article:published_time" content="2024-01-01T00:00:00Z"/>+ <meta property="article:modified_time" content="2024-01-02T00:00:00Z"/>+ <meta property="article:expiration_time" content="2024-01-03T00:00:00Z"/>+ <meta property="article:author" content="Jane Doe"/>+ <meta property="article:section" content="Tech"/>+ <meta property="article:tag" content="ai"/>`);+it('Should scrape properties', () => {expect(parseOpenGraph(case1)).toEqual({title: 'Under Pressure',@@ -77,4 +85,18 @@ describe('parseOpenGraph', () => {type: 'website',});});++ it('Should parse article properties into articleInfo', () => {+ expect(parseOpenGraph(case7)).toEqual({+ type: 'article',+ articleInfo: {+ publishedTime: '2024-01-01T00:00:00Z',+ modifiedTime: '2024-01-02T00:00:00Z',+ expirationTime: '2024-01-03T00:00:00Z',+ author: 'Jane Doe',+ section: 'Tech',+ tag: 'ai',+ },+ });+ });});