Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

chore: pin cheerio to `1.0.0-rc.12`

Martin Adámek61e3ed518012
diff --git a/packages/cheerio-crawler/package.json b/packages/cheerio-crawler/package.json
index c1853737f..d628094be 100644
--- a/packages/cheerio-crawler/package.json
+++ b/packages/cheerio-crawler/package.json
@@ -56,7 +56,7 @@
"@crawlee/http": "3.11.1",
"@crawlee/types": "3.11.1",
"@crawlee/utils": "3.11.1",
- "cheerio": "^1.0.0-rc.12",
+ "cheerio": "1.0.0-rc.12",
"htmlparser2": "^9.0.0",
"tslib": "^2.4.0"
}
diff --git a/packages/http-crawler/package.json b/packages/http-crawler/package.json
index 10781fff8..f963c5909 100644
--- a/packages/http-crawler/package.json
+++ b/packages/http-crawler/package.json
@@ -59,7 +59,7 @@
"@crawlee/types": "3.11.1",
"@crawlee/utils": "3.11.1",
"@types/content-type": "^1.1.5",
- "cheerio": "^1.0.0-rc.12",
+ "cheerio": "1.0.0-rc.12",
"content-type": "^1.0.4",
"got-scraping": "^4.0.0",
"iconv-lite": "^0.6.3",
diff --git a/packages/jsdom-crawler/package.json b/packages/jsdom-crawler/package.json
index 7303f5a18..fcf7515e6 100644
--- a/packages/jsdom-crawler/package.json
+++ b/packages/jsdom-crawler/package.json
@@ -59,7 +59,7 @@
"@crawlee/types": "3.11.1",
"@crawlee/utils": "3.11.1",
"@types/jsdom": "^21.0.0",
- "cheerio": "^1.0.0-rc.12",
+ "cheerio": "1.0.0-rc.12",
"jsdom": "^24.0.0",
"ow": "^0.28.2",
"tslib": "^2.4.0"
diff --git a/packages/playwright-crawler/package.json b/packages/playwright-crawler/package.json
index 80f0946d3..670cb8254 100644
--- a/packages/playwright-crawler/package.json
+++ b/packages/playwright-crawler/package.json
@@ -61,7 +61,7 @@
"@crawlee/core": "3.11.1",
"@crawlee/types": "3.11.1",
"@crawlee/utils": "3.11.1",
- "cheerio": "^1.0.0-rc.12",
+ "cheerio": "1.0.0-rc.12",
"idcac-playwright": "^0.1.2",
"jquery": "^3.6.0",
"lodash.isequal": "^4.5.0",
diff --git a/packages/puppeteer-crawler/package.json b/packages/puppeteer-crawler/package.json
index 0c33ebfa1..840506720 100644
--- a/packages/puppeteer-crawler/package.json
+++ b/packages/puppeteer-crawler/package.json
@@ -59,7 +59,7 @@
"@crawlee/browser-pool": "3.11.1",
"@crawlee/types": "3.11.1",
"@crawlee/utils": "3.11.1",
- "cheerio": "^1.0.0-rc.12",
+ "cheerio": "1.0.0-rc.12",
"devtools-protocol": "*",
"idcac-playwright": "^0.1.2",
"jquery": "^3.6.0",
diff --git a/packages/utils/package.json b/packages/utils/package.json
index 34c8d01c0..c102499b4 100644
--- a/packages/utils/package.json
+++ b/packages/utils/package.json
@@ -51,7 +51,7 @@
"@apify/ps-tree": "^1.2.0",
"@crawlee/types": "3.11.1",
"@types/sax": "^1.2.7",
- "cheerio": "^1.0.0-rc.12",
+ "cheerio": "1.0.0-rc.12",
"file-type": "^19.0.0",
"got-scraping": "^4.0.3",
"ow": "^0.28.1",
diff --git a/test/core/crawlers/cheerio_crawler.test.ts b/test/core/crawlers/cheerio_crawler.test.ts
index 11271c4e9..55c47f734 100644
--- a/test/core/crawlers/cheerio_crawler.test.ts
+++ b/test/core/crawlers/cheerio_crawler.test.ts
@@ -1,7 +1,7 @@
import type { IncomingHttpHeaders, Server } from 'http';
import { Readable } from 'stream';
-import log, { Log, LogLevel } from '@apify/log';
+import log, { Log } from '@apify/log';
import type { CheerioRequestHandler, CheerioCrawlingContext, ProxyInfo, Source } from '@crawlee/cheerio';
import {
AutoscaledPool,
@@ -255,20 +255,27 @@ describe('CheerioCrawler', () => {
expect.assertions(3);
const sources = [`${serverAddress}/special/html-type`];
const requestList = await RequestList.open(null, sources);
+ const tmp: any[] = [];
const cheerioCrawler = new CheerioCrawler({
requestList,
maxRequestRetries: 0,
maxConcurrency: 1,
requestHandler: ({ $, body, request }) => {
- expect(body).toBe(responseSamples.html);
- // test that `request.loadedUrl` is no longer optional
- expect(request.loadedUrl.length).toBe(sources[0].length);
- expect($.html()).toBe(body);
+ tmp.push(body, $.html(), request.loadedUrl);
+ console.log(request.loadedUrl.length, sources[0].length);
+ console.log(body);
+ console.log($.html());
},
});
await cheerioCrawler.run();
+
+ expect(tmp).toHaveLength(3);
+ expect(tmp[0]).toBe(responseSamples.html);
+ expect(tmp[1]).toBe(tmp[0]);
+ // test that `request.loadedUrl` is no longer optional
+ expect(tmp[2].length).toBe(sources[0].length);
});
describe('should timeout', () => {
diff --git a/yarn.lock b/yarn.lock
index a4a5b8b38..d9c9d7cac 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -557,7 +557,7 @@ __metadata:
"@crawlee/http": "npm:3.11.1"
"@crawlee/types": "npm:3.11.1"
"@crawlee/utils": "npm:3.11.1"
- cheerio: "npm:^1.0.0-rc.12"
+ cheerio: "npm:1.0.0-rc.12"
htmlparser2: "npm:^9.0.0"
tslib: "npm:^2.4.0"
languageName: unknown
@@ -620,7 +620,7 @@ __metadata:
"@crawlee/types": "npm:3.11.1"
"@crawlee/utils": "npm:3.11.1"
"@types/content-type": "npm:^1.1.5"
- cheerio: "npm:^1.0.0-rc.12"
+ cheerio: "npm:1.0.0-rc.12"
content-type: "npm:^1.0.4"
got-scraping: "npm:^4.0.0"
iconv-lite: "npm:^0.6.3"
@@ -641,7 +641,7 @@ __metadata:
"@crawlee/types": "npm:3.11.1"
"@crawlee/utils": "npm:3.11.1"
"@types/jsdom": "npm:^21.0.0"
- cheerio: "npm:^1.0.0-rc.12"
+ cheerio: "npm:1.0.0-rc.12"
jsdom: "npm:^24.0.0"
ow: "npm:^0.28.2"
tslib: "npm:^2.4.0"
@@ -691,7 +691,7 @@ __metadata:
"@crawlee/core": "npm:3.11.1"
"@crawlee/types": "npm:3.11.1"
"@crawlee/utils": "npm:3.11.1"
- cheerio: "npm:^1.0.0-rc.12"
+ cheerio: "npm:1.0.0-rc.12"
idcac-playwright: "npm:^0.1.2"
jquery: "npm:^3.6.0"
lodash.isequal: "npm:^4.5.0"
@@ -718,7 +718,7 @@ __metadata:
"@crawlee/browser-pool": "npm:3.11.1"
"@crawlee/types": "npm:3.11.1"
"@crawlee/utils": "npm:3.11.1"
- cheerio: "npm:^1.0.0-rc.12"
+ cheerio: "npm:1.0.0-rc.12"
devtools-protocol: "npm:*"
idcac-playwright: "npm:^0.1.2"
jquery: "npm:^3.6.0"
@@ -827,7 +827,7 @@ __metadata:
"@crawlee/types": "npm:3.11.1"
"@types/sax": "npm:^1.2.7"
"@types/whatwg-mimetype": "npm:^3.0.2"
- cheerio: "npm:^1.0.0-rc.12"
+ cheerio: "npm:1.0.0-rc.12"
file-type: "npm:^19.0.0"
got-scraping: "npm:^4.0.3"
ow: "npm:^0.28.1"
@@ -3855,7 +3855,7 @@ __metadata:
languageName: node
linkType: hard
-"cheerio@npm:^1.0.0-rc.12":
+"cheerio@npm:1.0.0-rc.12":
version: 1.0.0-rc.12
resolution: "cheerio@npm:1.0.0-rc.12"
dependencies: