Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
Fix Node 14 (#789)
Ondra Urban3e46becc1ed4
diff --git a/.travis.yml b/.travis.ymlindex c300ca743..0ffb997c1 100644--- a/.travis.yml+++ b/.travis.yml@@ -3,6 +3,7 @@ dist: bioniclanguage: node_jsif: (tag IS blank) OR tag =~ /^v\d+\.\d+\.\d+$/node_js:+- '14'- '12'- '10'cache:diff --git a/package.json b/package.jsonindex e21d68881..2aa69dc29 100644--- a/package.json+++ b/package.json@@ -54,6 +54,7 @@"@apify/http-request": "^2.0.1","@apify/ps-tree": "^1.1.3","@types/cheerio": "^0.22.18",+ "@types/domhandler": "^2.4.1","@types/node": "^12","@types/puppeteer": "^3.0.0","@types/socket.io": "^2.1.4",diff --git a/src/crawlers/cheerio_crawler.js b/src/crawlers/cheerio_crawler.jsindex acddd5169..c17a9dd0e 100644--- a/src/crawlers/cheerio_crawler.js+++ b/src/crawlers/cheerio_crawler.js@@ -743,7 +743,7 @@ export default CheerioCrawler;/*** @typedef CheerioHandlePageInputs- * @property {CheerioSelector} [$]+ * @property {cheerio.Selector} [$]* The [Cheerio](https://cheerio.js.org/) object with parsed HTML.* @property {(string|Buffer)} body* The request body of the web page.diff --git a/src/serialization.js b/src/serialization.jsindex e556a26c8..6de1e8369 100644--- a/src/serialization.js+++ b/src/serialization.js@@ -19,10 +19,12 @@ const pipeline = util.promisify(stream.pipeline);*/class ArrayToJson extends stream.Readable {constructor(data, options = {}) {- super(options);- const {- batchSize = 10000,- } = options;+ super({+ ...options,+ autoDestroy: true,+ emitClose: true,+ });+ const { batchSize = 10000 } = options;this.offset = 0;this.batchSize = batchSize;this.data = data;@@ -42,10 +44,9 @@ class ArrayToJson extends stream.Readable {} else {this.push(']');this.push(null);- this.destroy();}} catch (err) {- this.destroy(err);+ this.emit('error', err);}}}diff --git a/test/serialization.test.js b/test/serialization.test.jsindex 66e01462f..ce3987957 100644--- a/test/serialization.test.js+++ b/test/serialization.test.js@@ -6,11 +6,10 @@ import { serializeArray, deserializeArray, createDeserialize } from '../build/seconst TEST_JSON_PATH = path.join(__dirname, 'data', 'sample.json.gz');-const gunzip = util.promisify(zlib.gunzip);describe('Data Compression:', () => {- test('compressData should compress', async () => {+ test('serializeArray should compress', async () => {const compressedTestJson = await fs.readFile(TEST_JSON_PATH);const jsonBuffer = await gunzip(compressedTestJson);const expectedArray = JSON.parse(jsonBuffer.toString('utf8'));@@ -25,7 +24,7 @@ describe('Data Compression:', () => {expect(receivedArray).toEqual(expectedArray);});- test('decompressData should decompress', async () => {+ test('deserializeArray should decompress', async () => {const compressedTestJson = await fs.readFile(TEST_JSON_PATH);const jsonBuffer = await gunzip(compressedTestJson);const expectedArray = JSON.parse(jsonBuffer.toString('utf8'));@@ -37,7 +36,7 @@ describe('Data Compression:', () => {expect(receivedArray).toEqual(expectedArray);});- test('compressData + decompressData should produce original data', async () => {+ test('serializeArray + deserializeArray should produce original data', async () => {const data = [];for (let i = 0; i < 10000; i++) {data.push({ [`${Math.random()}`]: Math.random() });@@ -47,7 +46,7 @@ describe('Data Compression:', () => {expect(decompressed).toEqual(data);});- test('createDecompress should work', async () => {+ test('createDeserialize should work', async () => {const compressedTestJson = await fs.readFile(TEST_JSON_PATH);const jsonBuffer = await gunzip(compressedTestJson);const expectedArray = JSON.parse(jsonBuffer.toString('utf8'));diff --git a/test/typescript/crawlers/cheerio_crawler.test.ts b/test/typescript/crawlers/cheerio_crawler.test.tsindex 43a5c4e11..5871014f0 100644--- a/test/typescript/crawlers/cheerio_crawler.test.ts+++ b/test/typescript/crawlers/cheerio_crawler.test.ts@@ -47,7 +47,7 @@ describe('CheerioCrawler TS', () => {test('Can pass around and call `handler({ var }: { var: Type})`', async () => {// This form can also be easily reused as above.// Auto-completion works on defined input variables in parameter list.- const y = async ({$}: { $?: CheerioSelector }) => {+ const y = async ({$}: { $?: cheerio.Selector }) => {expect($!('a').attr('href')).toEqual('#');};