Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
fix(CheerioCrawler): pass ixXml down to response parser (#1807)
Vlad Franguaf7a5c4efa94
diff --git a/packages/cheerio-crawler/src/internals/cheerio-crawler.ts b/packages/cheerio-crawler/src/internals/cheerio-crawler.tsindex 7bd781772..aa940e85d 100644--- a/packages/cheerio-crawler/src/internals/cheerio-crawler.ts+++ b/packages/cheerio-crawler/src/internals/cheerio-crawler.ts@@ -134,7 +134,7 @@ export class CheerioCrawler extends HttpCrawler<CheerioCrawlingContext> {}protected override async _parseHTML(response: IncomingMessage, isXml: boolean, crawlingContext: CheerioCrawlingContext) {- const dom = await this._parseHtmlToDom(response);+ const dom = await this._parseHtmlToDom(response, isXml);const $ = cheerio.load(dom as string, {xmlMode: isXml,@@ -162,13 +162,13 @@ export class CheerioCrawler extends HttpCrawler<CheerioCrawlingContext> {};}- protected async _parseHtmlToDom(response: IncomingMessage) {+ protected async _parseHtmlToDom(response: IncomingMessage, isXml: boolean) {return new Promise((resolve, reject) => {const domHandler = new DomHandler((err, dom) => {if (err) reject(err);else resolve(dom);- });- const parser = new WritableStream(domHandler, { decodeEntities: true });+ }, { xmlMode: isXml });+ const parser = new WritableStream(domHandler, { decodeEntities: true, xmlMode: isXml });parser.on('error', reject);response.on('error', reject)diff --git a/packages/cheerio-crawler/test/xml.test.ts b/packages/cheerio-crawler/test/xml.test.tsnew file mode 100644index 000000000..3ded9a3ef--- /dev/null+++ b/packages/cheerio-crawler/test/xml.test.ts@@ -0,0 +1,35 @@+import type { Server } from 'node:http';+import type { CheerioCrawlingContext } from '@crawlee/cheerio';+import { CheerioCrawler } from '@crawlee/cheerio';+import { runExampleComServer } from '../../../test/shared/_helper';++let serverAddress = 'http://localhost:';+let port: number;+let server: Server;++beforeAll(async () => {+ [server, port] = await runExampleComServer();+ serverAddress += port;+});++afterAll(() => {+ server.close();+});++describe('CheerioCrawler - XML should be parsed correctly', () => {+ test('should parse XML', async () => {+ let value!: string;++ function handler({ $ }: CheerioCrawlingContext) {+ value = $('item').first().find('link').text();+ }++ const crawler = new CheerioCrawler({+ requestHandler: handler,+ });++ await crawler.run([`${serverAddress}/special/complex-xml`]);++ expect(value).toBeTruthy();+ });+});diff --git a/test/shared/_helper.ts b/test/shared/_helper.tsindex fba8e0690..2b235e7a2 100644--- a/test/shared/_helper.ts+++ b/test/shared/_helper.ts@@ -23,6 +23,7 @@ export const responseSamples = {+ ' <title>Web Scraping, Data Extraction and Automation · Apify</title>\n'+ '</item>\n'+ '</items>',+ complexXml: fs.readFileSync(path.join(__dirname, 'data/complex.xml'), 'utf-8'),image: fs.readFileSync(path.join(__dirname, 'data/apify.png')),html: `<!doctype html><html>@@ -37,7 +38,7 @@ export const responseSamples = {margin: 0;padding: 0;font-family: -apple-system, system-ui, BlinkMacSystemFont, "Segoe UI", "Open Sans", "Helvetica Neue", Helvetica, Arial, sans-serif;-+}div {width: 600px;@@ -57,9 +58,9 @@ export const responseSamples = {width: auto;}}- </style>+ </style></head>-+<body><div><h1>Example Domain</h1>@@ -79,7 +80,7 @@ export const responseSamples = {<meta http-equiv="Content-type" content="text/html; charset=utf-8"><meta name="viewport" content="width=device-width, initial-scale=1"><link rel="stylesheet" href="cacheable/style.css">- <script src="cacheable/script.js"></script>+ <script src="cacheable/script.js"></script></body></html>`,@@ -163,6 +164,10 @@ export async function runExampleComServer(): Promise<[Server, number]> {res.type('application/xml');res.send(responseSamples.xml);});+ special.get('/complex-xml', (_req, res) => {+ res.type('application/xml');+ res.send(responseSamples.complexXml);+ });special.get('/image-type', (_req, res) => {res.type('image/png');res.send(responseSamples.image);diff --git a/test/shared/data/complex.xml b/test/shared/data/complex.xmlnew file mode 100644index 000000000..d02e232bd--- /dev/null+++ b/test/shared/data/complex.xml@@ -0,0 +1,46 @@+<?xml version="1.0" encoding="UTF-8" standalone="yes"?>+<rss version="2.0"+ xmlns:media="http://search.yahoo.com/mrss/">+ <channel>+ <generator>NFE/5.0</generator>+ <title>"test" - Google Actualités</title>+ <link>https://news.google.com/search?q=test&hl=fr&gl=FR&ceid=FR:fr</link>+ <language>fr</language>+ <webMaster>[email protected]</webMaster>+ <copyright>2023 Google Inc.</copyright>+ <lastBuildDate>Mon, 06 Mar 2023 22:12:03 GMT</lastBuildDate>+ <description>Google Actualités</description>+ <item>+ <title>Test des Flic, des mini interrupteurs compatibles HomeKit qui en font beaucoup - iGeneration</title>+ <link>https://news.google.com/rss/articles/CBMidmh0dHBzOi8vd3d3LmlnZW4uZnIvdGVzdHMvMjAyMy8wMy90ZXN0LWRlcy1mbGljLWRlcy1taW5pLWludGVycnVwdGV1cnMtY29tcGF0aWJsZXMtaG9tZWtpdC1xdWktZW4tZm9udC1iZWF1Y291cC0xMzU4NjfSAQA?oc=5</link>+ <guid isPermaLink="false">CBMidmh0dHBzOi8vd3d3LmlnZW4uZnIvdGVzdHMvMjAyMy8wMy90ZXN0LWRlcy1mbGljLWRlcy1taW5pLWludGVycnVwdGV1cnMtY29tcGF0aWJsZXMtaG9tZWtpdC1xdWktZW4tZm9udC1iZWF1Y291cC0xMzU4NjfSAQA</guid>+ <pubDate>Mon, 06 Mar 2023 19:32:38 GMT</pubDate>+ <description><a href="https://news.google.com/rss/articles/CBMidmh0dHBzOi8vd3d3LmlnZW4uZnIvdGVzdHMvMjAyMy8wMy90ZXN0LWRlcy1mbGljLWRlcy1taW5pLWludGVycnVwdGV1cnMtY29tcGF0aWJsZXMtaG9tZWtpdC1xdWktZW4tZm9udC1iZWF1Y291cC0xMzU4NjfSAQA?oc=5" target="_blank">Test des Flic, des mini interrupteurs compatibles HomeKit qui en font beaucoup</a>&nbsp;&nbsp;<font color="#6f6f6f">iGeneration</font></description>+ <source url="https://www.igen.fr">iGeneration</source>+ </item>+ <item>+ <title>TEST de Dead Cells - Return to Castlevania : une évidence et un hommage de qualité - Gameblog.fr</title>+ <link>https://news.google.com/rss/articles/CBMiggFodHRwczovL3d3dy5nYW1lYmxvZy5mci9qZXUtdmlkZW8vamV1eC90ZXN0cy90ZXN0LWRlLWRlYWQtY2VsbHMtcmV0dXJuLXRvLWNhc3RsZXZhbmlhLXVuZS1ldmlkZW5jZS1ldC11bi1ob21tYWdlLWRlLXF1YWxpdGUtNDE4NDc30gEA?oc=5</link>+ <guid isPermaLink="false">CBMiggFodHRwczovL3d3dy5nYW1lYmxvZy5mci9qZXUtdmlkZW8vamV1eC90ZXN0cy90ZXN0LWRlLWRlYWQtY2VsbHMtcmV0dXJuLXRvLWNhc3RsZXZhbmlhLXVuZS1ldmlkZW5jZS1ldC11bi1ob21tYWdlLWRlLXF1YWxpdGUtNDE4NDc30gEA</guid>+ <pubDate>Mon, 06 Mar 2023 19:03:49 GMT</pubDate>+ <description><a href="https://news.google.com/rss/articles/CBMiggFodHRwczovL3d3dy5nYW1lYmxvZy5mci9qZXUtdmlkZW8vamV1eC90ZXN0cy90ZXN0LWRlLWRlYWQtY2VsbHMtcmV0dXJuLXRvLWNhc3RsZXZhbmlhLXVuZS1ldmlkZW5jZS1ldC11bi1ob21tYWdlLWRlLXF1YWxpdGUtNDE4NDc30gEA?oc=5" target="_blank">TEST de Dead Cells - Return to Castlevania : une évidence et un hommage de qualité</a>&nbsp;&nbsp;<font color="#6f6f6f">Gameblog.fr</font></description>+ <source url="https://www.gameblog.fr">Gameblog.fr</source>+ </item>+ <item>+ <title>Choisissez un animal et découvrez vos plus grands traits de caractère. Il faut absolument faire ce test de personnalité ! - Wk Vet</title>+ <link>https://news.google.com/rss/articles/CBMijwFodHRwczovL3drLXZldC5mci90ZXN0cy9jaG9pc2lzc2V6LXVuLWFuaW1hbC1ldC1kZWNvdXZyZXotdm9zLXBsdXMtZ3JhbmRzLXRyYWl0cy1kZS1jYXJhY3RlcmUtaWwtZmF1dC1hYnNvbHVtZW50LWZhaXJlLWNlLXRlc3QtZGUtcGVyc29ubmFsaXRlL9IBAA?oc=5</link>+ <guid isPermaLink="false">CBMijwFodHRwczovL3drLXZldC5mci90ZXN0cy9jaG9pc2lzc2V6LXVuLWFuaW1hbC1ldC1kZWNvdXZyZXotdm9zLXBsdXMtZ3JhbmRzLXRyYWl0cy1kZS1jYXJhY3RlcmUtaWwtZmF1dC1hYnNvbHVtZW50LWZhaXJlLWNlLXRlc3QtZGUtcGVyc29ubmFsaXRlL9IBAA</guid>+ <pubDate>Mon, 06 Mar 2023 05:00:00 GMT</pubDate>+ <description><a href="https://news.google.com/rss/articles/CBMijwFodHRwczovL3drLXZldC5mci90ZXN0cy9jaG9pc2lzc2V6LXVuLWFuaW1hbC1ldC1kZWNvdXZyZXotdm9zLXBsdXMtZ3JhbmRzLXRyYWl0cy1kZS1jYXJhY3RlcmUtaWwtZmF1dC1hYnNvbHVtZW50LWZhaXJlLWNlLXRlc3QtZGUtcGVyc29ubmFsaXRlL9IBAA?oc=5" target="_blank">Choisissez un animal et découvrez vos plus grands traits de caractère. Il faut absolument faire ce test de personnalité !</a>&nbsp;&nbsp;<font color="#6f6f6f">Wk Vet</font></description>+ <source url="https://wk-vet.fr">Wk Vet</source>+ </item>+ <item>+ <title>Test de personnalité : Découvrez ce que votre choix de pierre révèle sur vous ! - Radiotips.fr</title>+ <link>https://news.google.com/rss/articles/CBMiZWh0dHBzOi8vd3d3LnJhZGlvdGlwcy5mci90ZXN0LWRlLXBlcnNvbm5hbGl0ZS1kZWNvdXZyZXotY2UtcXVlLXZvdHJlLWNob2l4LWRlLXBpZXJyZS1yZXZlbGUtc3VyLXZvdXMv0gEA?oc=5</link>+ <guid isPermaLink="false">CBMiZWh0dHBzOi8vd3d3LnJhZGlvdGlwcy5mci90ZXN0LWRlLXBlcnNvbm5hbGl0ZS1kZWNvdXZyZXotY2UtcXVlLXZvdHJlLWNob2l4LWRlLXBpZXJyZS1yZXZlbGUtc3VyLXZvdXMv0gEA</guid>+ <pubDate>Mon, 06 Mar 2023 08:00:17 GMT</pubDate>+ <description><a href="https://news.google.com/rss/articles/CBMiZWh0dHBzOi8vd3d3LnJhZGlvdGlwcy5mci90ZXN0LWRlLXBlcnNvbm5hbGl0ZS1kZWNvdXZyZXotY2UtcXVlLXZvdHJlLWNob2l4LWRlLXBpZXJyZS1yZXZlbGUtc3VyLXZvdXMv0gEA?oc=5" target="_blank">Test de personnalité : Découvrez ce que votre choix de pierre révèle sur vous !</a>&nbsp;&nbsp;<font color="#6f6f6f">Radiotips.fr</font></description>+ <source url="https://www.radiotips.fr">Radiotips.fr</source>+ </item>+ </channel>+</rss>