Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

Fix Node 14 (#789)

Ondra Urban3e46becc1ed4
diff --git a/.travis.yml b/.travis.yml
index c300ca743..0ffb997c1 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -3,6 +3,7 @@ dist: bionic
language: node_js
if: (tag IS blank) OR tag =~ /^v\d+\.\d+\.\d+$/
node_js:
+- '14'
- '12'
- '10'
cache:
diff --git a/package.json b/package.json
index e21d68881..2aa69dc29 100644
--- a/package.json
+++ b/package.json
@@ -54,6 +54,7 @@
"@apify/http-request": "^2.0.1",
"@apify/ps-tree": "^1.1.3",
"@types/cheerio": "^0.22.18",
+ "@types/domhandler": "^2.4.1",
"@types/node": "^12",
"@types/puppeteer": "^3.0.0",
"@types/socket.io": "^2.1.4",
diff --git a/src/crawlers/cheerio_crawler.js b/src/crawlers/cheerio_crawler.js
index acddd5169..c17a9dd0e 100644
--- a/src/crawlers/cheerio_crawler.js
+++ b/src/crawlers/cheerio_crawler.js
@@ -743,7 +743,7 @@ export default CheerioCrawler;
/**
* @typedef CheerioHandlePageInputs
- * @property {CheerioSelector} [$]
+ * @property {cheerio.Selector} [$]
* The [Cheerio](https://cheerio.js.org/) object with parsed HTML.
* @property {(string|Buffer)} body
* The request body of the web page.
diff --git a/src/serialization.js b/src/serialization.js
index e556a26c8..6de1e8369 100644
--- a/src/serialization.js
+++ b/src/serialization.js
@@ -19,10 +19,12 @@ const pipeline = util.promisify(stream.pipeline);
*/
class ArrayToJson extends stream.Readable {
constructor(data, options = {}) {
- super(options);
- const {
- batchSize = 10000,
- } = options;
+ super({
+ ...options,
+ autoDestroy: true,
+ emitClose: true,
+ });
+ const { batchSize = 10000 } = options;
this.offset = 0;
this.batchSize = batchSize;
this.data = data;
@@ -42,10 +44,9 @@ class ArrayToJson extends stream.Readable {
} else {
this.push(']');
this.push(null);
- this.destroy();
}
} catch (err) {
- this.destroy(err);
+ this.emit('error', err);
}
}
}
diff --git a/test/serialization.test.js b/test/serialization.test.js
index 66e01462f..ce3987957 100644
--- a/test/serialization.test.js
+++ b/test/serialization.test.js
@@ -6,11 +6,10 @@ import { serializeArray, deserializeArray, createDeserialize } from '../build/se
const TEST_JSON_PATH = path.join(__dirname, 'data', 'sample.json.gz');
-
const gunzip = util.promisify(zlib.gunzip);
describe('Data Compression:', () => {
- test('compressData should compress', async () => {
+ test('serializeArray should compress', async () => {
const compressedTestJson = await fs.readFile(TEST_JSON_PATH);
const jsonBuffer = await gunzip(compressedTestJson);
const expectedArray = JSON.parse(jsonBuffer.toString('utf8'));
@@ -25,7 +24,7 @@ describe('Data Compression:', () => {
expect(receivedArray).toEqual(expectedArray);
});
- test('decompressData should decompress', async () => {
+ test('deserializeArray should decompress', async () => {
const compressedTestJson = await fs.readFile(TEST_JSON_PATH);
const jsonBuffer = await gunzip(compressedTestJson);
const expectedArray = JSON.parse(jsonBuffer.toString('utf8'));
@@ -37,7 +36,7 @@ describe('Data Compression:', () => {
expect(receivedArray).toEqual(expectedArray);
});
- test('compressData + decompressData should produce original data', async () => {
+ test('serializeArray + deserializeArray should produce original data', async () => {
const data = [];
for (let i = 0; i < 10000; i++) {
data.push({ [`${Math.random()}`]: Math.random() });
@@ -47,7 +46,7 @@ describe('Data Compression:', () => {
expect(decompressed).toEqual(data);
});
- test('createDecompress should work', async () => {
+ test('createDeserialize should work', async () => {
const compressedTestJson = await fs.readFile(TEST_JSON_PATH);
const jsonBuffer = await gunzip(compressedTestJson);
const expectedArray = JSON.parse(jsonBuffer.toString('utf8'));
diff --git a/test/typescript/crawlers/cheerio_crawler.test.ts b/test/typescript/crawlers/cheerio_crawler.test.ts
index 43a5c4e11..5871014f0 100644
--- a/test/typescript/crawlers/cheerio_crawler.test.ts
+++ b/test/typescript/crawlers/cheerio_crawler.test.ts
@@ -47,7 +47,7 @@ describe('CheerioCrawler TS', () => {
test('Can pass around and call `handler({ var }: { var: Type})`', async () => {
// This form can also be easily reused as above.
// Auto-completion works on defined input variables in parameter list.
- const y = async ({$}: { $?: CheerioSelector }) => {
+ const y = async ({$}: { $?: cheerio.Selector }) => {
expect($!('a').attr('href')).toEqual('#');
};