Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
fix(RequestQueue): always clear locks when a request is reclaimed (#2263)
Vlad Frangu0fafe2901036
diff --git a/packages/core/src/storages/request_queue_v2.ts b/packages/core/src/storages/request_queue_v2.tsindex d4fbe163e..28e59955b 100644--- a/packages/core/src/storages/request_queue_v2.ts+++ b/packages/core/src/storages/request_queue_v2.ts@@ -136,6 +136,23 @@ class RequestQueue extends RequestProvider {return request;}+ override async reclaimRequest(...args: Parameters<RequestProvider['reclaimRequest']>): ReturnType<RequestProvider['reclaimRequest']> {+ const res = await super.reclaimRequest(...args);++ if (res) {+ const [request, options] = args;++ // Try to delete the request lock if possible+ try {+ await this.client.deleteRequestLock(request.id!, { forefront: options?.forefront ?? false });+ } catch (err) {+ this.log.debug(`Failed to delete request lock for request ${request.id}`, { err });+ }+ }++ return res;+ }+protected async ensureHeadIsNonEmpty() {// Stop fetching if we are paused for migrationif (this.queuePausedForMigration) {