Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
fix: don't give up on a stalled domain while another one is cooling down (#4325)
SwastikTripathid10a4200aa1e
diff --git a/docs/upgrading/upgrading-v4/request-loaders.md b/docs/upgrading/upgrading-v4/request-loaders.mdindex 47d386bbf..86b969e52 100644--- a/docs/upgrading/upgrading-v4/request-loaders.md+++ b/docs/upgrading/upgrading-v4/request-loaders.md@@ -101,7 +101,7 @@ type RequestSourceStatus =One probe instead of two, which a task loop runs several times a second, plus two answers the booleans could not express: `waiting` can name when it expects work again (`readyAt`) instead of leaving the caller to poll, and `stalled` reports requests a source cannot make progress on, which the crawler turns into a `PersistentRateLimitError`.-If you implemented either interface, return `ready` without evaluating anything further — it is the most common answer and the only one a caller can act on immediately. Reading from two sources, the precedence is `ready` > `stalled` > `waiting` > `finished`, and a combined `waiting` carries the earlier `readyAt`.+If you implemented either interface, return `ready` without evaluating anything further — it is the most common answer and the only one a caller can act on immediately. Reading from two sources, the precedence is `ready` > `waiting` > `stalled` > `finished`, and a combined `waiting` carries the earlier `readyAt`.**Storage backends keep the two booleans** — see [`StorageBackend` interface simplified](./storage-backends.md#storagebackend-interface-simplified).diff --git a/packages/basic-crawler/src/internals/throttling-request-manager.ts b/packages/basic-crawler/src/internals/throttling-request-manager.tsindex 02d2d7962..24f44f5d5 100644--- a/packages/basic-crawler/src/internals/throttling-request-manager.ts+++ b/packages/basic-crawler/src/internals/throttling-request-manager.ts@@ -194,6 +194,18 @@ function throttledUntil(state: DomainState): number {return Math.max(state.backoffUntil, state.crawlDelayUntil);}+/**+ * Still refusing, and has been for longer than the window. Not dispatched to, so this lapses a window after the+ * last 429 and the domain gets one burst to prove itself with.+ */+function isStalling(state: DomainState, now: number, maxStallMs: number): boolean {+ return (+ state.rateLimitedSince !== 0 &&+ now - state.lastRateLimitedAt <= maxStallMs &&+ now - state.rateLimitedSince > maxStallMs+ );+}+/** How long a domain must be left alone after a dispatch: what it asked for, or our floor, whichever is longer. */function crawlDelayMs(state: DomainState, minCrawlDelayMs: number): number {return Math.max(state.declaredCrawlDelayMs ?? 0, minCrawlDelayMs);@@ -596,7 +608,12 @@ export class ThrottlingRequestManager<T extends IRequestManager = IRequestManage#fetchableDomains(): string[] {const now = Date.now();return Array.from(this.#domainStates.values())- .filter((state) => now >= throttledUntil(state) && this.#subManagers.has(state.domain))+ .filter(+ (state) =>+ now >= throttledUntil(state) &&+ !isStalling(state, now, this.#maxDomainStallMs) &&+ this.#subManagers.has(state.domain),+ ).sort((a, b) => throttledUntil(a) - throttledUntil(b)).map((state) => state.domain);}@@ -927,7 +944,8 @@ export class ThrottlingRequestManager<T extends IRequestManager = IRequestManage** `ready` from anywhere else outranks a stalling domain and is returned without looking at the stall clocks,* so one hopeless domain never ends a crawl making progress elsewhere. It cannot outrank itself, though -- * see the traversal.+ * see the traversal. A domain that is only cooling down holds off a stalling one the same way, because the+ * crawl can continue once its cooldown ends.*/async checkReadiness(): Promise<RequestSourceStatus> {await this.#ensureSubManagers();@@ -944,15 +962,7 @@ export class ThrottlingRequestManager<T extends IRequestManager = IRequestManagecontinue;}- if (- // Together: still turning us away, and has been without a break for longer than the window - a- // domain that has merely been idle starts this clock at its first 429, not with idle time on it.- state.rateLimitedSince !== 0 &&- now - state.lastRateLimitedAt <= this.#maxDomainStallMs &&- now - state.rateLimitedSince > this.#maxDomainStallMs- ) {- // Deliberately not dispatchable: a domain that has refused every request for the whole window- // is not progress just because its backoff momentarily lapsed - that is what it does between 429s.+ if (isStalling(state, now, this.#maxDomainStallMs)) {(stallCandidates ??= []).push(state);continue;}@@ -980,6 +990,15 @@ export class ThrottlingRequestManager<T extends IRequestManager = IRequestManagereturn probed;}+ if (readyAt !== undefined) {+ return joinRequestSourceStatuses(probed, { status: 'waiting', readyAt });+ }++ // Batches still landing in the background are work nobody can see in a queue yet.+ if (this.#inProgressBatchCount > 0 && probed.status === 'finished') {+ return { status: 'waiting' };+ }+if (stallCandidates !== undefined) {const stalled = (await Promise.all(@@ -996,26 +1015,17 @@ export class ThrottlingRequestManager<T extends IRequestManager = IRequestManage.map((state) => `"${state.domain}" (${((now - state.rateLimitedSince) / 1000).toFixed(0)}s)`).join(', ');- return {+ return joinRequestSourceStatuses(probed, {status: 'stalled',reason:`${summary} rate-limited every request for longer than maxDomainStallSecs ` +`(${(this.#maxDomainStallMs / 1000).toFixed(0)}s). Waiting longer will not help - lower the ` +`crawler's concurrency, or drop these domains. Their requests are still queued, so ` +`re-running with \`purgeOnStart\` disabled will resume them if the rate limit lifts.`,- };+ });}}- if (readyAt !== undefined) {- return joinRequestSourceStatuses(probed, { status: 'waiting', readyAt });- }-- // Batches still landing in the background are work nobody can see in a queue yet.- if (this.#inProgressBatchCount > 0 && probed.status === 'finished') {- return { status: 'waiting' };- }-return probed;}@@ -1166,7 +1176,8 @@ export class ThrottlingRequestManager<T extends IRequestManager = IRequestManageconst state = this.#getDomainState(request.url);- if (state === null || Date.now() >= throttledUntil(state)) {+ const now = Date.now();+ if (state === null || (now >= throttledUntil(state) && !isStalling(state, now, this.#maxDomainStallMs))) {return request;}diff --git a/packages/core/src/storages/request-loader.ts b/packages/core/src/storages/request-loader.tsindex 107e61b40..e4987196b 100644--- a/packages/core/src/storages/request-loader.ts+++ b/packages/core/src/storages/request-loader.ts@@ -31,7 +31,7 @@ export type RequestSourceStatus =export type RequestLoaderStatus = Exclude<RequestSourceStatus, { status: 'stalled' }>;/**- * Combines two request sources' statuses, with the precedence `ready` > `stalled` > `waiting` > `finished`.+ * Combines two request sources' statuses, with the precedence `ready` > `waiting` > `stalled` > `finished`.** Binary rather than variadic on purpose: it is on the task loop's probe path and folding a pair allocates* nothing.@@ -46,18 +46,9 @@ export function joinRequestSourceStatuses(a: RequestSourceStatus, b: RequestSourreturn b;}- // `ready` outranking `stalled` masks a stalled source while the other still has work. That is parity with- // the crawler before this was a single answer: the stall check was only reached from `isFinishedFunction`,- // which the task loop calls only when nothing is in flight and nothing is ready. "Fixing" the masking- // turns a crawl that is progressing elsewhere into a `PersistentRateLimitError`. `stalled` outranking- // `waiting` is the same parity - that call site fired regardless of other domains' clocks.- if (a.status === 'stalled') {- return a;- }- if (b.status === 'stalled') {- return b;- }-+ // `ready` and `waiting` outranking `stalled` mask a stalled source while the other still has work, now or+ // later. The crawler turns `stalled` into a `PersistentRateLimitError`, so "fixing" the masking turns a crawl+ // that can still progress elsewhere into a failure. The stall surfaces once the other source runs out of work.if (a.status === 'waiting') {// The earlier of the two known wake-up times - unknown only if neither source announced one.if (b.status !== 'waiting' || b.readyAt === undefined) {@@ -65,6 +56,13 @@ export function joinRequestSourceStatuses(a: RequestSourceStatus, b: RequestSour}return a.readyAt !== undefined && a.readyAt <= b.readyAt ? a : b;}+ if (b.status === 'waiting') {+ return b;+ }++ if (a.status === 'stalled') {+ return a;+ }// `a` is finished, so `b` decides.return b;diff --git a/test/basic-crawler/throttling-request-manager.test.ts b/test/basic-crawler/throttling-request-manager.test.tsindex 2010affef..217bae3d5 100644--- a/test/basic-crawler/throttling-request-manager.test.ts+++ b/test/basic-crawler/throttling-request-manager.test.ts@@ -786,14 +786,69 @@ describe('ThrottlingRequestManager', () => {expect((await manager.checkReadiness()).status).toBe('ready');});+ test('a domain cooling down with requests left holds off a stalling domain until its cooldown ends', async () => {+ const manager = new ThrottlingRequestManager({+ inner: await createQueue(),+ domains: ['blocked.com', 'slow.com'],+ baseDelaySecs: 10,+ maxDomainStallSecs: 30,+ });+ await manager.addRequest({ url: 'https://blocked.com/1' });+ manager.recordPacingSignal({ url: 'https://blocked.com/1', reason: 'rateLimited' });+ await manager.addRequest({ url: 'https://slow.com/1' });++ // `slow.com` starts its first backoff, then `blocked.com` refuses again past its stall window.+ advanceClock(31_000);+ manager.recordPacingSignal({ url: 'https://slow.com/1', reason: 'rateLimited' });+ advanceClock(5_000);+ manager.recordPacingSignal({ url: 'https://blocked.com/1', reason: 'rateLimited' });++ const cooling = await manager.checkReadiness();+ expect(cooling).toMatchObject({ status: 'waiting', readyAt: expect.any(Number) });++ // Once `slow.com` can be dispatched again, the crawl has work to do.+ advanceClock((cooling as { readyAt: number }).readyAt - Date.now());+ expect((await manager.checkReadiness()).status).toBe('ready');++ // With that request handled, only the stalling domain is left to report.+ const request = (await manager.fetchNextRequest())!;+ expect(request.url).toBe('https://slow.com/1');+ await manager.markRequestAsHandled(request);+ expect((await manager.checkReadiness()).status).toBe('stalled');+ });++ test('a stalling domain is not dispatched to while work elsewhere keeps the crawl alive', async () => {+ const manager = await stallingManager();+ await manager.addRequest({ url: 'https://example.com/1' });+ manager.recordPacingSignal({ url: 'https://example.com/1', reason: 'rateLimited' });+ stallFor(manager, 'https://example.com/1');+ await manager.addRequest({ url: 'https://slow.com/1' });++ // Its backoff lapsed long ago, but the crawl is not going to keep hammering a domain it has given up on.+ advanceClock(1_000);+ const slow = (await manager.fetchNextRequest())!;+ expect(slow.url).toBe('https://slow.com/1');+ expect(await manager.fetchNextRequest()).toBeNull();+ await manager.markRequestAsHandled(slow);++ // One window without a 429 earns it a single burst - which it refuses, so it is stalled again at once.+ advanceClock(30_000);+ const probe = (await manager.fetchNextRequest())!;+ expect(probe.url).toBe('https://example.com/1');+ manager.recordPacingSignal({ url: probe.url, reason: 'rateLimited' });+ await manager.reclaimRequest(probe, { forefront: true });+ expect((await manager.checkReadiness()).status).toBe('stalled');+ });+test('a handled request resets the clock', async () => {const manager = await stallingManager();await manager.addRequest({ url: 'https://example.com/1' });await manager.addRequest({ url: 'https://example.com/2' });manager.recordPacingSignal({ url: 'https://example.com/1', reason: 'rateLimited' });stallFor(manager, 'https://example.com/1');- advanceClock(1_000);+ // The one burst it is granted per window lets a request through this time.+ advanceClock(31_000);await manager.markRequestAsHandled((await manager.fetchNextRequest())!);expect((await manager.checkReadiness()).status).not.toBe('stalled');diff --git a/test/core/storages/request-source-status.test.ts b/test/core/storages/request-source-status.test.tsindex 26c8f239b..a6e3e9ce1 100644--- a/test/core/storages/request-source-status.test.ts+++ b/test/core/storages/request-source-status.test.ts@@ -20,7 +20,7 @@ describe('joinRequestSourceStatuses', () => {[ready, waiting, ready],[ready, finished, ready],[stalled, stalled, stalled],- [stalled, waiting, stalled],+ [stalled, waiting, waiting],[stalled, finished, stalled],[waiting, waiting, waiting],[waiting, finished, waiting],@@ -33,12 +33,6 @@ describe('joinRequestSourceStatuses', () => {},);- test('a stalled source is masked while the other one still has work', () => {- // The crawler turns `stalled` into a `PersistentRateLimitError`, so this precedence is what keeps a- // crawl that is making progress elsewhere from being abandoned over one hopeless domain.- expect(joinRequestSourceStatuses(stalled, ready)).toEqual(ready);- });-test('the earlier of two known wake-up times wins', () => {expect(joinRequestSourceStatuses({ status: 'waiting', readyAt: 200 }, { status: 'waiting', readyAt: 100 }),@@ -58,8 +52,4 @@ describe('joinRequestSourceStatuses', () => {readyAt: 100,});});-- test('no wake-up time is invented when neither source knows one', () => {- expect(joinRequestSourceStatuses(waiting, waiting)).toEqual({ status: 'waiting' });- });});