Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

fix: remove memory leaks from migration event handling (#1679)

Szymon Marczak49bba252ebc3
diff --git a/packages/basic-crawler/src/internals/basic-crawler.ts b/packages/basic-crawler/src/internals/basic-crawler.ts
index cf15f2c83..4a2719a88 100644
--- a/packages/basic-crawler/src/internals/basic-crawler.ts
+++ b/packages/basic-crawler/src/internals/basic-crawler.ts
@@ -575,10 +575,6 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContext
};
this.autoscaledPoolOptions = { ...autoscaledPoolOptions, ...basicCrawlerAutoscaledPoolConfiguration };
-
- // Attach a listener to handle migration and aborting events gracefully.
- this.events.on(EventType.MIGRATING, this._pauseOnMigration.bind(this));
- this.events.on(EventType.ABORTING, this._pauseOnMigration.bind(this));
}
/**
@@ -599,11 +595,17 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContext
await this._init();
await this.stats.startCapturing();
- process.once('SIGINT', async () => {
+ const sigintHandler = async () => {
this.log.warning('Pausing... Press CTRL+C again to force exit. To resume, do: CRAWLEE_PURGE_ON_START=0 npm run start');
await this._pauseOnMigration();
await this.autoscaledPool!.abort();
- });
+ };
+
+ // Attach a listener to handle migration and aborting events gracefully.
+ const boundPauseOnMigration = this._pauseOnMigration.bind(this);
+ process.once('SIGINT', sigintHandler);
+ this.events.on(EventType.MIGRATING, boundPauseOnMigration);
+ this.events.on(EventType.ABORTING, boundPauseOnMigration);
try {
this.log.info('Starting the crawl');
@@ -611,6 +613,10 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContext
} finally {
await this.teardown();
await this.stats.stopCapturing();
+
+ process.off('SIGINT', sigintHandler);
+ this.events.off(EventType.MIGRATING, boundPauseOnMigration);
+ this.events.off(EventType.ABORTING, boundPauseOnMigration);
}
const finalStats = this.stats.calculate();
diff --git a/test/core/crawlers/basic_crawler.test.ts b/test/core/crawlers/basic_crawler.test.ts
index 5535b7527..4e742a422 100644
--- a/test/core/crawlers/basic_crawler.test.ts
+++ b/test/core/crawlers/basic_crawler.test.ts
@@ -77,6 +77,20 @@ describe('BasicCrawler', () => {
server.close();
});
+ test('does not leak sigint events', async () => {
+ let count = 0;
+
+ const crawler = new BasicCrawler({
+ requestHandler: () => {
+ count = process.listenerCount('SIGINT');
+ },
+ });
+
+ await crawler.run(['https://example.com']);
+
+ expect(process.listenerCount('SIGINT')).toBe(count - 1);
+ });
+
test('should run in parallel thru all the requests', async () => {
const sources = [...Array(500).keys()].map((index) => ({ url: `https://example.com/${index}` }));
const sourcesCopy = JSON.parse(JSON.stringify(sources));