Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
feat: add `MinimumSpeedStream` and `ByteCounterStream` helpers (#2970)
Jindřich Bär921c4ee3401b
diff --git a/packages/http-crawler/src/internals/file-download.ts b/packages/http-crawler/src/internals/file-download.tsindex 6c095c352..1e5f66ef2 100644--- a/packages/http-crawler/src/internals/file-download.ts+++ b/packages/http-crawler/src/internals/file-download.ts@@ -1,3 +1,4 @@+import { Transform } from 'node:stream';import { finished } from 'node:stream/promises';import { isPromise } from 'node:util/types';@@ -58,6 +59,85 @@ export type FileDownloadRequestHandler<JSONData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler> = RequestHandler<FileDownloadCrawlingContext<UserData, JSONData>>;+/**+ * Creates a transform stream that throws an error if the source data speed is below the specified minimum speed.+ * This `Transform` checks the amount of data every `checkProgressInterval` milliseconds.+ * If the stream has received less than `minSpeedKbps * historyLengthMs / 1000` bytes in the last `historyLengthMs` milliseconds,+ * it will throw an error.+ *+ * Can be used e.g. to abort a download if the network speed is too slow.+ * @returns Transform stream that monitors the speed of the incoming data.+ */+export function MinimumSpeedStream({+ minSpeedKbps,+ historyLengthMs = 10e3,+ checkProgressInterval: checkProgressIntervalMs = 5e3,+}: {+ minSpeedKbps: number;+ historyLengthMs?: number;+ checkProgressInterval?: number;+}): Transform {+ let snapshots: { timestamp: number; bytes: number }[] = [];++ const checkInterval = setInterval(() => {+ const now = Date.now();++ snapshots = snapshots.filter((snapshot) => now - snapshot.timestamp < historyLengthMs);+ const totalBytes = snapshots.reduce((acc, snapshot) => acc + snapshot.bytes, 0);+ const elapsed = (now - (snapshots[0]?.timestamp ?? 0)) / 1000;++ if (totalBytes / 1024 / elapsed < minSpeedKbps) {+ clearInterval(checkInterval);+ stream.emit('error', new Error(`Stream speed too slow, aborting...`));+ }+ }, checkProgressIntervalMs);++ const stream = new Transform({+ transform: (chunk, _, callback) => {+ snapshots.push({ timestamp: Date.now(), bytes: chunk.length });+ callback(null, chunk);+ },+ final: (callback) => {+ clearInterval(checkInterval);+ callback();+ },+ });++ return stream;+}++/**+ * Creates a transform stream that logs the progress of the incoming data.+ * This `Transform` calls the `logProgress` function every `loggingInterval` milliseconds with the number of bytes received so far.+ *+ * Can be used e.g. to log the progress of a download.+ * @returns Transform stream logging the progress of the incoming data.+ */+export function ByteCounterStream({+ logTransferredBytes,+ loggingInterval = 5000,+}: { logTransferredBytes: (transferredBytes: number) => void; loggingInterval?: number }): Transform {+ let transferredBytes = 0;+ let lastLogTime = Date.now();++ return new Transform({+ transform: (chunk, _, callback) => {+ transferredBytes += chunk.length;++ if (Date.now() - lastLogTime > loggingInterval) {+ lastLogTime = Date.now();+ logTransferredBytes(transferredBytes);+ }++ callback(null, chunk);+ },+ flush: (callback) => {+ logTransferredBytes(transferredBytes);+ callback();+ },+ });+}+/*** Provides a framework for downloading files in parallel using plain HTTP requests. The URLs to download are fed either from a static list of URLs or they can be added on the fly from another crawler.*