Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
chore: fix dependency updates stuck because of `stream-chain`
Martin Adámeka453e5850bef
diff --git a/packages/core/package.json b/packages/core/package.jsonindex b6b6cb93e..02f90f1dd 100644--- a/packages/core/package.json+++ b/packages/core/package.json@@ -71,8 +71,7 @@"json5": "^2.2.3","minimatch": "^9.0.0","ow": "^0.28.1",- "stream-chain": "^3.0.0",- "stream-json": "^1.7.4",+ "stream-json": "^1.8.0","tldts": "^6.0.0","tough-cookie": "^4.0.0","tslib": "^2.4.0",diff --git a/packages/core/src/serialization.ts b/packages/core/src/serialization.tsindex b3ed3d353..616bc9e06 100644--- a/packages/core/src/serialization.ts+++ b/packages/core/src/serialization.ts@@ -3,11 +3,12 @@ import zlib from 'node:zlib';import { pipeline as streamPipeline, Readable, Writable } from 'stream';import ow from 'ow';-import type Chain from 'stream-chain';import StreamArray from 'stream-json/streamers/StreamArray';const pipeline = util.promisify(streamPipeline);+type Chain = ReturnType<typeof StreamArray.withParser>;+/*** Transforms an array of items to a JSON in a streaming* fashion to save memory. It operates in batches to speeddiff --git a/yarn.lock b/yarn.lockindex 792cd66d5..b302c83ea 100644--- a/yarn.lock+++ b/yarn.lock@@ -601,8 +601,7 @@ __metadata:json5: "npm:^2.2.3"minimatch: "npm:^9.0.0"ow: "npm:^0.28.1"- stream-chain: "npm:^3.0.0"- stream-json: "npm:^1.7.4"+ stream-json: "npm:^1.8.0"tldts: "npm:^6.0.0"tough-cookie: "npm:^4.0.0"tslib: "npm:^2.4.0"@@ -11408,13 +11407,6 @@ __metadata:languageName: nodelinkType: hard-"stream-chain@npm:^3.0.0":- version: 3.1.0- resolution: "stream-chain@npm:3.1.0"- checksum: 10c0/9deac4c0e42525659f39e0751e0fa4ffef3dccfac9938b3e664a6f7e8e52347aec7963e785c19f6c47033b5f5ee4251ebe86d717de00807321779351367c9844- languageName: node- linkType: hard-"stream-combiner@npm:~0.0.4":version: 0.0.4resolution: "stream-combiner@npm:0.0.4"@@ -11424,7 +11416,7 @@ __metadata:languageName: nodelinkType: hard-"stream-json@npm:^1.7.4":+"stream-json@npm:^1.8.0":version: 1.8.0resolution: "stream-json@npm:1.8.0"dependencies: