Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
Fixing RequestQueue to better handle DynamoDB data inconsistency
Jan Curn3eb514b1682b
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex 065153b91..a9fa944b1 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -2,6 +2,7 @@ xxx====================- `Snapshotter` will now log critical memory overload warnings at most once per 10 seconds._ Live view snapshots are now made right after navigation finishes, instead of right before page close.+- Fixes in `RequestQueue` to deal with inconsistencies in the underlying data storage0.14.11 / 2019-05-28====================diff --git a/src/request_queue.js b/src/request_queue.jsindex c04a5d12e..6eadfc2fe 100644--- a/src/request_queue.js+++ b/src/request_queue.js@@ -32,7 +32,7 @@ const RECENTLY_HANDLED_CACHE_SIZE = 1000;// Indicates how long it usually takes for the underlying storage to propagate all writes// to be available to subsequent reads.-const STORAGE_CONSISTENCY_DELAY_MILLIS = 3000;+export const STORAGE_CONSISTENCY_DELAY_MILLIS = 3000;const writeFilePromised = promisify(fs.writeFile);@@ -546,7 +546,7 @@ export class RequestQueue {*/async _ensureHeadIsNonEmpty(ensureConsistency = false,- limit = Math.max(this.inProgressCount * QUERY_HEAD_BUFFER, QUERY_HEAD_MIN_LENGTH),+ limit = Math.max(this.inProgressCount() * QUERY_HEAD_BUFFER, QUERY_HEAD_MIN_LENGTH),iteration = 0,) {checkParamOrThrow(ensureConsistency, 'ensureConsistency', 'Boolean');@@ -706,6 +706,9 @@ const filePathToQueueOrderNo = (filepath) => {/*** Local directory-based implementation of the `RequestQueue` class.+ * TODO: We should implement this class using the RequestQueueRemote, just replace+ * the underlying API calls with their emulation on filesystem. That will bring+ * all the goodies such as caching and will enable better and more consistent testing** @ignore*/diff --git a/test/request_queue.js b/test/request_queue.jsindex a72c76623..573daec10 100644--- a/test/request_queue.js+++ b/test/request_queue.js@@ -7,7 +7,7 @@ import * as Apify from '../build/index';import * as utils from '../build/utils';import {RequestQueueLocal, RequestQueue,- LOCAL_STORAGE_SUBDIR, QUERY_HEAD_MIN_LENGTH, API_PROCESSED_REQUESTS_DELAY_MILLIS,+ LOCAL_STORAGE_SUBDIR, QUERY_HEAD_MIN_LENGTH, API_PROCESSED_REQUESTS_DELAY_MILLIS, STORAGE_CONSISTENCY_DELAY_MILLIS,} from '../build/request_queue';import { emptyLocalStorageSubdir, LOCAL_STORAGE_DIR, expectNotUsingLocalStorage, expectDirEmpty, expectDirNonEmpty } from './_helper';@@ -293,6 +293,7 @@ describe('RequestQueue', () => {}).returns(Promise.resolve({ requestId: 'a', wasAlreadyHandled: false, wasAlreadyPresent: false, request: requestA }));await queue.addRequest(requestA);+ expect(queue.queueHeadDict.length()).to.be.eql(1);const requestB = new Request({ url: 'http://example.com/b' });mock.expects('addRequest')@@ -305,7 +306,7 @@ describe('RequestQueue', () => {}).returns(Promise.resolve({ requestId: 'b', wasAlreadyHandled: false, wasAlreadyPresent: false, request: requestB }));await queue.addRequest(requestB, { forefront: true });- expect(queue.queueHeadDict.length()).to.be.eql(1);+ expect(queue.queueHeadDict.length()).to.be.eql(2);expect(queue.inProgressCount()).to.be.eql(0);// Forefronted request was added to the queue.@@ -318,7 +319,7 @@ describe('RequestQueue', () => {.returns(Promise.resolve(_.extend(requestB, { id: 'b' })));const requestBFromQueue = await queue.fetchNextRequest();expect(requestBFromQueue).to.be.eql(requestB);- expect(queue.queueHeadDict.length()).to.be.eql(0);+ expect(queue.queueHeadDict.length()).to.be.eql(1);expect(queue.inProgressCount()).to.be.eql(1);// getRequest() returns null if object was not found.@@ -345,6 +346,9 @@ describe('RequestQueue', () => {await queue.reclaimRequest(requestB, { forefront: true });expect(queue.queueHeadDict.length()).to.be.eql(1);expect(queue.inProgressCount()).to.be.eql(0);+ await Apify.utils.sleep(STORAGE_CONSISTENCY_DELAY_MILLIS + 10);+ expect(queue.queueHeadDict.length()).to.be.eql(2);+ expect(queue.inProgressCount()).to.be.eql(0);// Fetch again.expect(queue.queueHeadDict.length()).to.be.eql(1);@@ -357,7 +361,7 @@ describe('RequestQueue', () => {.returns(Promise.resolve(_.extend(requestB, { id: 'b' })));const requestBFromQueue2 = await queue.fetchNextRequest();expect(requestBFromQueue2).to.be.eql(requestB);- expect(queue.queueHeadDict.length()).to.be.eql(0);+ expect(queue.queueHeadDict.length()).to.be.eql(1);expect(queue.inProgressCount()).to.be.eql(1);// Mark handled.@@ -370,7 +374,7 @@ describe('RequestQueue', () => {}).returns(Promise.resolve({ requestId: requestB.id, wasAlreadyHandled: false, wasAlreadyPresent: true, request: requestB }));await queue.markRequestHandled(requestB);- expect(queue.queueHeadDict.length()).to.be.eql(0);+ expect(queue.queueHeadDict.length()).to.be.eql(1);expect(queue.inProgressCount()).to.be.eql(0);// Query queue head.@@ -766,7 +770,11 @@ describe('RequestQueue', () => {.withArgs({ queueId, request: requestA, forefront: true, clientKey }).returns(Promise.resolve({ requestId: requestA.id, wasAlreadyHandled: false, wasAlreadyPresent: true, request: requestA }));await queue.reclaimRequest(requestA, { forefront: true });-+ expect(queue.queueHeadDict.length()).to.be.eql(0);+ expect(queue.inProgressCount()).to.be.eql(0);+ expect(queue.assumedTotalCount).to.be.eql(2);+ expect(queue.assumedHandledCount).to.be.eql(1);+ await Apify.utils.sleep(STORAGE_CONSISTENCY_DELAY_MILLIS + 10);expect(queue.queueHeadDict.length()).to.be.eql(1);expect(queue.inProgressCount()).to.be.eql(0);expect(queue.assumedTotalCount).to.be.eql(2);