Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
fix(basic): ignore completed domains when waiting for cooldowns (#4294)
Junghwan09d3ffe5b268
diff --git a/packages/basic-crawler/src/internals/throttling-request-manager.ts b/packages/basic-crawler/src/internals/throttling-request-manager.tsindex 24f44f5d5..915956681 100644--- a/packages/basic-crawler/src/internals/throttling-request-manager.ts+++ b/packages/basic-crawler/src/internals/throttling-request-manager.ts@@ -938,9 +938,8 @@ export class ThrottlingRequestManager<T extends IRequestManager = IRequestManage/*** Reports whether anything can be dispatched right now, and if not, when — or why never.*- * One traversal of the domain clocks answers all of it: only domains whose delays have run out are probed,- * the rest merely contribute the moment they come due. Throttled requests count as outstanding work, so a- * crawler gated on this idles for the backoff instead of concluding it is done.+ * Dispatchable domains are probed first. When nothing can be dispatched, cooling domains contribute their+ * deadlines only while they still have outstanding work, so a finished crawl does not wait out a cooldown.** `ready` from anywhere else outranks a stalling domain and is returned without looking at the stall clocks,* so one hopeless domain never ends a crawl making progress elsewhere. It cannot outrank itself, though -@@ -952,13 +951,14 @@ export class ThrottlingRequestManager<T extends IRequestManager = IRequestManageconst now = Date.now();const dispatchable: Promise<T>[] = [];- let readyAt: number | undefined;+ const cooling: { subManager: Promise<T>; readyAt: number }[] = [];let stallCandidates: DomainState[] | undefined;for (const state of this.#domainStates.values()) {// A `Crawl-delay` can give a domain a clock before its first request gives it a queue - nothing// to fetch from and nothing to wait for until then.- if (!this.#subManagers.has(state.domain)) {+ const subManager = this.#subManagers.get(state.domain);+ if (!subManager) {continue;}@@ -967,12 +967,12 @@ export class ThrottlingRequestManager<T extends IRequestManager = IRequestManagecontinue;}- const until = throttledUntil(state);+ const readyAt = throttledUntil(state);- if (now >= until) {- dispatchable.push(this.#subManagers.get(state.domain)!);- } else if (readyAt === undefined || until < readyAt) {- readyAt = until;+ if (now >= readyAt) {+ dispatchable.push(subManager);+ } else {+ cooling.push({ subManager, readyAt });}}@@ -990,8 +990,14 @@ export class ThrottlingRequestManager<T extends IRequestManager = IRequestManagereturn probed;}- if (readyAt !== undefined) {- return joinRequestSourceStatuses(probed, { status: 'waiting', readyAt });+ // A cooling domain with nothing left to dispatch is nothing to wait for; in deadline order, the first+ // with work left is the earliest wake-up.+ cooling.sort((a, b) => a.readyAt - b.readyAt);+ for (const { subManager, readyAt } of cooling) {+ const { status } = await (await subManager).checkReadiness();+ if (status !== 'finished') {+ return joinRequestSourceStatuses(probed, { status: 'waiting', readyAt });+ }}// Batches still landing in the background are work nobody can see in a queue yet.diff --git a/test/basic-crawler/throttling-request-manager.test.ts b/test/basic-crawler/throttling-request-manager.test.tsindex 217bae3d5..dd2d0643c 100644--- a/test/basic-crawler/throttling-request-manager.test.ts+++ b/test/basic-crawler/throttling-request-manager.test.ts@@ -726,6 +726,56 @@ describe('ThrottlingRequestManager', () => {expect(await subQueue.getPendingCount()).toBe(0);});+ describe('readiness during cooldown', () => {+ beforeEach(() => {+ freezeClock();+ });++ test('finishes as soon as the last request is handled', async () => {+ const manager = new ThrottlingRequestManager({+ inner: await createQueue(),+ domains: ['example.com'],+ minCrawlDelaySecs: 60,+ });+ await manager.addRequest({ url: 'https://example.com/1' });+ await manager.markRequestAsHandled(await pollForNextRequest(manager));++ expect(await manager.checkReadiness()).toEqual({ status: 'finished' });+ });++ test('waits while a domain has in-flight work', async () => {+ const manager = new ThrottlingRequestManager({+ inner: await createQueue(),+ domains: ['example.com'],+ minCrawlDelaySecs: 60,+ });+ await manager.addRequest({ url: 'https://example.com/1' });+ await pollForNextRequest(manager);++ expect(await manager.checkReadiness()).toEqual({ status: 'waiting', readyAt: Date.now() + 60_000 });+ });++ test('a finished domain does not set the wake-up time ahead of one with work left', async () => {+ const manager = new ThrottlingRequestManager({ inner: await createQueue(), domains: 'all' });+ for (const [domain, intervalMs] of [+ ['finished.com', 10_000],+ ['busy.com', 20_000],+ ] as const) {+ manager.recordPacingSignal({+ url: `https://${domain}/1`,+ reason: 'minInterval',+ intervalMs,+ scope: 'hostname',+ });+ await manager.addRequest({ url: `https://${domain}/1` });+ await manager.markRequestAsHandled(await pollForNextRequest(manager));+ }+ await manager.addRequest({ url: 'https://busy.com/2' });++ expect(await manager.checkReadiness()).toEqual({ status: 'waiting', readyAt: Date.now() + 20_000 });+ });+ });+describe('stall detection', () => {beforeEach(() => {freezeClock();diff --git a/test/core/request-manager-tandem.test.ts b/test/core/request-manager-tandem.test.tsindex 547c91536..0812be445 100644--- a/test/core/request-manager-tandem.test.ts+++ b/test/core/request-manager-tandem.test.ts@@ -298,7 +298,9 @@ describe('RequestManagerTandem', () => {false,);- // ...and the pacer acted on the signals rather than merely accepting them.+ // ...and the pacer acted on the signals rather than merely accepting them. The clocks only show on a+ // domain with work left; the list's request is not the throttler's.+ await throttler.addRequest({ url: 'https://example.com/2' });await expect(throttler.checkReadiness()).resolves.toMatchObject({ status: 'waiting' });// A floor covering every domain reaches it the same way, and this pacer covers only `example.com`.