Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
fix(impit-client): pause fromWeb stream to prevent early consumption (#3347)
nikitachapovskii-dev72aacb44eaa4
diff --git a/packages/impit-client/src/index.ts b/packages/impit-client/src/index.tsindex 590f1e1b4..ee1e840e2 100644--- a/packages/impit-client/src/index.ts+++ b/packages/impit-client/src/index.ts@@ -1,4 +1,4 @@-import { Readable } from 'node:stream';+import { pipeline, Readable, Transform } from 'node:stream';import { type ReadableStream } from 'node:stream/web';import { isGeneratorObject } from 'node:util/types';@@ -210,19 +210,24 @@ export class ImpitHttpClient implements BaseHttpClient {const responseStream = Readable.fromWeb(response.body as ReadableStream<any>);let transferred = 0;const total = Number(response.headers.get('content-length') ?? 0);- responseStream.on('data', (chunk) => {- transferred += chunk.length;+ const counter = new Transform({+ transform(chunk, _enc, cb) {+ transferred += chunk.length;+ cb(null, chunk);+ },});- const getDownloadProgress = () => {- return {- percent: Math.round((transferred / total) * 100),- transferred,- total,- };- };+ pipeline(responseStream, counter, (err) => {+ if (err) counter.destroy(err);+ });++ const getDownloadProgress = () => ({+ percent: total > 0 ? Math.round((transferred / total) * 100) : 0,+ transferred,+ total,+ });- return [responseStream, getDownloadProgress];+ return [counter, getDownloadProgress];}/**