Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

[fix] Compatibility fixes for Typescript compiler (#542)

Matej Gagyi7ea84bacd6e4
diff --git a/src/autoscaling/snapshotter.js b/src/autoscaling/snapshotter.js
index 4df492204..5315cfd44 100644
--- a/src/autoscaling/snapshotter.js
+++ b/src/autoscaling/snapshotter.js
@@ -285,7 +285,8 @@ class Snapshotter {
*/
_memoryOverloadWarning({ memCurrentBytes }) {
const now = new Date();
- if (this.lastLoggedCriticalMemoryOverloadAt && now < this.lastLoggedCriticalMemoryOverloadAt + CRITICAL_OVERLOAD_RATE_LIMIT_MILLIS) return;
+ if (this.lastLoggedCriticalMemoryOverloadAt && now.getTime() < this.lastLoggedCriticalMemoryOverloadAt.getTime()
+ + CRITICAL_OVERLOAD_RATE_LIMIT_MILLIS) return;
const maxDesiredMemoryBytes = this.maxUsedMemoryRatio * this.maxMemoryBytes;
const reserveMemory = this.maxMemoryBytes * (1 - this.maxUsedMemoryRatio) * RESERVE_MEMORY_RATIO;
@@ -318,7 +319,7 @@ class Snapshotter {
const previousSnapshot = this.eventLoopSnapshots[this.eventLoopSnapshots.length - 1];
if (previousSnapshot) {
const { createdAt } = previousSnapshot;
- const delta = now - createdAt - this.eventLoopSnapshotIntervalMillis;
+ const delta = now.getTime() - createdAt - this.eventLoopSnapshotIntervalMillis;
if (delta > this.maxBlockedMillis) snapshot.isOverloaded = true;
snapshot.exceededMillis = Math.max(delta - this.maxBlockedMillis, 0);
@@ -437,7 +438,7 @@ class Snapshotter {
let oldCount = 0;
for (let i = 0; i < snapshots.length; i++) {
const { createdAt } = snapshots[i];
- if (now - createdAt > this.snapshotHistoryMillis) oldCount++;
+ if (now.getTime() - createdAt > this.snapshotHistoryMillis) oldCount++;
else break;
}
snapshots.splice(0, oldCount);
diff --git a/src/crawlers/basic_crawler.js b/src/crawlers/basic_crawler.js
index 1796a0e55..d8f433759 100644
--- a/src/crawlers/basic_crawler.js
+++ b/src/crawlers/basic_crawler.js
@@ -10,6 +10,7 @@ import events from '../events';
import { addTimeoutToPromise } from '../utils';
import Statistics from './statistics';
import { openSessionPool } from '../session_pool/session_pool';
+import Request from '../request'; // eslint-disable-line no-unused-vars
/**
* Since there's no set number of seconds before the container is terminated after
@@ -437,7 +438,7 @@ class BasicCrawler {
* @param {Error} error
* @param {Request} request
* @param {RequestList|RequestQueue} source
- * @return {Boolean} willBeRetried
+ * @return {Promise<Boolean>} willBeRetried
* @ignore
*/
async _requestFunctionErrorHandler(error, request, source) {
diff --git a/tsconfig.json b/tsconfig.json
new file mode 100644
index 000000000..5570755b9
--- /dev/null
+++ b/tsconfig.json
@@ -0,0 +1,25 @@
+{
+ "compilerOptions": {
+ "target": "es2015",
+ "module": "commonjs",
+ "moduleResolution": "node",
+ "lib": [
+ "es2017",
+ "dom"
+ ],
+ "types": [
+ "node"
+ ],
+ "allowJs": true,
+ "checkJs": false,
+ "noEmit": true,
+ "declaration": false,
+ "strict": false,
+ "noImplicitThis": true,
+ "alwaysStrict": true,
+ "esModuleInterop": true
+ },
+ "include": [
+ "src/index.js"
+ ]
+}