Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

fix(basic): settle teardown during crawler startup (#4094)

Michael Bujnovský51a74cabc16f
diff --git a/packages/basic-crawler/src/internals/basic-crawler.ts b/packages/basic-crawler/src/internals/basic-crawler.ts
index fdf396938..03fb0d41e 100644
--- a/packages/basic-crawler/src/internals/basic-crawler.ts
+++ b/packages/basic-crawler/src/internals/basic-crawler.ts
@@ -1683,20 +1683,6 @@ export class BasicCrawler<
'This crawler instance is already running, you can add more requests to it via `crawler.addRequests()`.',
);
}
-
- if (this.#hasFinishedBefore) {
- // A supplied statistics instance keeps whatever state it was handed - only wipe a default we built.
- await this.#statisticsDep.ifOwned(async (stats) => {
- stats.reset();
- await stats.resetStore();
- });
- // A session pool the crawler built follows the same blank-slate rule as its statistics.
- await this.#sessionPoolDep.ifOwned(async (pool) => {
- pool.reset();
- await pool.resetStore();
- });
- }
-
// An owned governor is rebuilt for every run, so it always starts from a clean slate — stale
// resource snapshots or a previous run's scaled desired concurrency would otherwise distort this
// run's scaling. An injected one is long-lived and its lifecycle belongs to the caller.
@@ -1711,6 +1697,19 @@ export class BasicCrawler<
this.#run = run;
try {
+ if (this.#hasFinishedBefore) {
+ // A supplied statistics instance keeps whatever state it was handed - only wipe a default we built.
+ await this.#statisticsDep.ifOwned(async (stats) => {
+ stats.reset();
+ await stats.resetStore();
+ });
+ // A session pool the crawler built follows the same blank-slate rule as its statistics.
+ await this.#sessionPoolDep.ifOwned(async (pool) => {
+ pool.reset();
+ await pool.resetStore();
+ });
+ }
+
await purgeDefaultStorages({
onlyPurgeOnce: true,
storageBackend: serviceLocator.getStorageBackend(),
diff --git a/packages/basic-crawler/src/internals/crawler-run.ts b/packages/basic-crawler/src/internals/crawler-run.ts
index 1c5920133..0ae7690e1 100644
--- a/packages/basic-crawler/src/internals/crawler-run.ts
+++ b/packages/basic-crawler/src/internals/crawler-run.ts
@@ -57,6 +57,8 @@ export class CrawlerRun {
* running by now — the task loop refuses to book anything against one that is not.
*/
async dispatchRequests(): Promise<void> {
+ if (this.#ended) return;
+
this.#dispatching = true;
try {
diff --git a/test/core/crawlers/basic-crawler.test.ts b/test/core/crawlers/basic-crawler.test.ts
index 6f53b395d..5dbf533d8 100644
--- a/test/core/crawlers/basic-crawler.test.ts
+++ b/test/core/crawlers/basic-crawler.test.ts
@@ -442,6 +442,94 @@ describe('BasicCrawler', () => {
await crawler.run(['https://example.com/2']);
});
+ describe('teardown during startup', () => {
+ test.each([
+ { label: 'a crawler-owned', injectSystem: false },
+ { label: 'an injected', injectSystem: true },
+ ])('settles run() with $label ConcurrencySystem', async ({ injectSystem }) => {
+ const initialized = Promise.withResolvers<void>();
+ const resumeInitialization = Promise.withResolvers<void>();
+
+ const concurrencySystem = injectSystem ? new ConcurrencySystem({ maxConcurrency: 1 }) : undefined;
+ await concurrencySystem?.start();
+
+ // Override the `init` method to allow weaving a teardown call into it reliably
+ const crawler = new (class DelayedStartupCrawler extends BasicCrawler {
+ protected override async init() {
+ await super.init();
+ initialized.resolve();
+ await resumeInitialization.promise;
+ }
+ })({
+ keepAlive: true,
+ ...(concurrencySystem && { concurrencySystem }),
+ requestHandler: async () => {},
+ });
+
+ const runPromise = crawler.run().then(
+ () => 'resolved',
+ (error: Error) => `rejected: ${error.message}`,
+ );
+
+ await initialized.promise;
+ await crawler.teardown();
+ resumeInitialization.resolve();
+
+ const outcome = await Promise.race([runPromise, sleep(500).then(() => 'still pending')]);
+
+ // Without this, a regression would leak the pool interval into the following tests.
+ if (outcome === 'still pending') await crawler.teardown();
+ await concurrencySystem?.stop();
+
+ expect(outcome).toBe('resolved');
+ });
+
+ test('settles a repeated run() torn down while it resets the previous run state', async () => {
+ const crawler = new BasicCrawler({ keepAlive: true, requestHandler: async () => {} });
+ const firstRun = crawler.run();
+ await vitest.waitFor(() => expect(crawler.running).toBe(true));
+ await crawler.teardown();
+ await firstRun;
+
+ const resetStarted = Promise.withResolvers<void>();
+ const resumeReset = Promise.withResolvers<void>();
+ vitest.spyOn(crawler.statistics as Statistics, 'resetStore').mockImplementation(async () => {
+ resetStarted.resolve();
+ await resumeReset.promise;
+ });
+
+ const runPromise = crawler.run().then(
+ () => 'resolved',
+ (error: Error) => `rejected: ${error.message}`,
+ );
+
+ await resetStarted.promise;
+ await crawler.teardown();
+ resumeReset.resolve();
+
+ const outcome = await Promise.race([runPromise, sleep(500).then(() => 'still pending')]);
+
+ // Without this, a regression would leak the pool interval into the following tests.
+ if (outcome === 'still pending') await crawler.teardown();
+
+ expect(outcome).toBe('resolved');
+ });
+ });
+
+ test('teardown before run() does not cancel the next run', async () => {
+ const processed: string[] = [];
+ const crawler = new BasicCrawler({
+ requestHandler: async ({ request }) => {
+ processed.push(request.url);
+ },
+ });
+
+ await crawler.teardown();
+ await crawler.run(['https://example.com/after-teardown']);
+
+ expect(processed).toEqual(['https://example.com/after-teardown']);
+ });
+
test('a startup that fails before the crawl leaves the instance runnable', async () => {
const processed: string[] = [];
const crawler = new BasicCrawler({