Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
[fix] Compatibility fixes for Typescript compiler (#542)
Matej Gagyi7ea84bacd6e4
diff --git a/src/autoscaling/snapshotter.js b/src/autoscaling/snapshotter.jsindex 4df492204..5315cfd44 100644--- a/src/autoscaling/snapshotter.js+++ b/src/autoscaling/snapshotter.js@@ -285,7 +285,8 @@ class Snapshotter {*/_memoryOverloadWarning({ memCurrentBytes }) {const now = new Date();- if (this.lastLoggedCriticalMemoryOverloadAt && now < this.lastLoggedCriticalMemoryOverloadAt + CRITICAL_OVERLOAD_RATE_LIMIT_MILLIS) return;+ if (this.lastLoggedCriticalMemoryOverloadAt && now.getTime() < this.lastLoggedCriticalMemoryOverloadAt.getTime()+ + CRITICAL_OVERLOAD_RATE_LIMIT_MILLIS) return;const maxDesiredMemoryBytes = this.maxUsedMemoryRatio * this.maxMemoryBytes;const reserveMemory = this.maxMemoryBytes * (1 - this.maxUsedMemoryRatio) * RESERVE_MEMORY_RATIO;@@ -318,7 +319,7 @@ class Snapshotter {const previousSnapshot = this.eventLoopSnapshots[this.eventLoopSnapshots.length - 1];if (previousSnapshot) {const { createdAt } = previousSnapshot;- const delta = now - createdAt - this.eventLoopSnapshotIntervalMillis;+ const delta = now.getTime() - createdAt - this.eventLoopSnapshotIntervalMillis;if (delta > this.maxBlockedMillis) snapshot.isOverloaded = true;snapshot.exceededMillis = Math.max(delta - this.maxBlockedMillis, 0);@@ -437,7 +438,7 @@ class Snapshotter {let oldCount = 0;for (let i = 0; i < snapshots.length; i++) {const { createdAt } = snapshots[i];- if (now - createdAt > this.snapshotHistoryMillis) oldCount++;+ if (now.getTime() - createdAt > this.snapshotHistoryMillis) oldCount++;else break;}snapshots.splice(0, oldCount);diff --git a/src/crawlers/basic_crawler.js b/src/crawlers/basic_crawler.jsindex 1796a0e55..d8f433759 100644--- a/src/crawlers/basic_crawler.js+++ b/src/crawlers/basic_crawler.js@@ -10,6 +10,7 @@ import events from '../events';import { addTimeoutToPromise } from '../utils';import Statistics from './statistics';import { openSessionPool } from '../session_pool/session_pool';+import Request from '../request'; // eslint-disable-line no-unused-vars/*** Since there's no set number of seconds before the container is terminated after@@ -437,7 +438,7 @@ class BasicCrawler {* @param {Error} error* @param {Request} request* @param {RequestList|RequestQueue} source- * @return {Boolean} willBeRetried+ * @return {Promise<Boolean>} willBeRetried* @ignore*/async _requestFunctionErrorHandler(error, request, source) {diff --git a/tsconfig.json b/tsconfig.jsonnew file mode 100644index 000000000..5570755b9--- /dev/null+++ b/tsconfig.json@@ -0,0 +1,25 @@+{+ "compilerOptions": {+ "target": "es2015",+ "module": "commonjs",+ "moduleResolution": "node",+ "lib": [+ "es2017",+ "dom"+ ],+ "types": [+ "node"+ ],+ "allowJs": true,+ "checkJs": false,+ "noEmit": true,+ "declaration": false,+ "strict": false,+ "noImplicitThis": true,+ "alwaysStrict": true,+ "esModuleInterop": true+ },+ "include": [+ "src/index.js"+ ]+}