Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
fix: Reset consecutive http 429 count on crawl progress (#4354)
Jan Bucharddd9491bc1eb
diff --git a/packages/basic-crawler/src/internals/throttling-request-manager.ts b/packages/basic-crawler/src/internals/throttling-request-manager.tsindex f88758ffd..02d2d7962 100644--- a/packages/basic-crawler/src/internals/throttling-request-manager.ts+++ b/packages/basic-crawler/src/internals/throttling-request-manager.ts@@ -771,11 +771,15 @@ export class ThrottlingRequestManager<T extends IRequestManager = IRequestManagereturn true;}- /** Records that a domain let a request through, which ends any rate-limit run stall detection was timing. */+ /**+ * Records that a domain let a request through, which ends any rate-limit run stall detection was timing+ * and the run of 429s the backoff was doubling for.+ */#recordProgress(url: string): void {const state = this.#getDomainState(url);if (state) {state.rateLimitedSince = 0;+ state.consecutive429Count = 0;}}diff --git a/test/basic-crawler/throttling-request-manager.test.ts b/test/basic-crawler/throttling-request-manager.test.tsindex c52902c2b..2010affef 100644--- a/test/basic-crawler/throttling-request-manager.test.ts+++ b/test/basic-crawler/throttling-request-manager.test.ts@@ -631,6 +631,27 @@ describe('ThrottlingRequestManager', () => {expect(await readyAt(manager)).toBe(Date.now() + 10_000);});+ test('a request getting through restarts the backoff exponent', async () => {+ const manager = new ThrottlingRequestManager({+ inner: await createQueue(),+ domains: ['example.com'],+ baseDelaySecs: 10,+ });+ await manager.addRequest({ url: 'https://example.com/1' });+ await manager.addRequest({ url: 'https://example.com/2' });+ const request = await pollForNextRequest(manager);+ freezeClock();++ // Part of a burst comes back 429 while the rest is still being served; once that lands, the next+ // burst's 429 is a fresh one - concurrency over the domain's rate must not double the delay forever.+ manager.recordPacingSignal({ url: 'https://example.com/2', reason: 'rateLimited' });+ advanceClock(11_000);+ await manager.markRequestAsHandled(request);+ manager.recordPacingSignal({ url: 'https://example.com/2', reason: 'rateLimited' });++ expect(await readyAt(manager)).toBe(Date.now() + 10_000);+ });+test('caps the delay at maxDelaySecs', async () => {const manager = new ThrottlingRequestManager({inner: await createQueue(),