Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
Add test
Ondra Urbanec2b0917bfcd
diff --git a/src/request_list.js b/src/request_list.jsindex c37910429..b6961aa1d 100644--- a/src/request_list.js+++ b/src/request_list.js@@ -1,16 +1,14 @@import { checkParamOrThrow } from 'apify-client/build/utils';import log from 'apify-shared/log';import _ from 'underscore';-import { ENV_VARS } from 'apify-shared/consts';import { ACTOR_EVENT_NAMES_EX } from './constants';import Request from './request';import events from './events';-import { ensureTokenOrLocalStorageEnvExists, getFirstKey, openLocalStorage, openRemoteStorage, publicUtils } from './utils';+import { getFirstKey, publicUtils } from './utils';import { getValue, setValue } from './key_value_store';-import { RequestQueue, RequestQueueLocal } from './request_queue';-const STATE_PERSISTENCE_KEY = 'REQUEST_LIST_STATE';-const SOURCES_PERSISTENCE_KEY = 'REQUEST_LIST_SOURCES';+export const STATE_PERSISTENCE_KEY = 'REQUEST_LIST_STATE';+export const SOURCES_PERSISTENCE_KEY = 'REQUEST_LIST_SOURCES';/*** Represents a static list of URLs to crawl.@@ -159,6 +157,7 @@ class RequestList {this.persistStateKey = persistStateKey;// If the stateKeyPrefix is set then we persist url list and state into default key-value store under keys with this prefix.+ this.initialState = state;this.shouldPersist = !!stateKeyPrefix;this.stateKey = null;this.sourcesKey = null;@@ -170,24 +169,13 @@ class RequestList {// If this option is set then all requests will get a pre-generated unique ID and duplicate URLs will be kept in the list.this.keepDuplicateUrls = keepDuplicateUrls;- this.isCurrentStatePersisted = true;+ // Starts as true because until we handle the first request, the list is effectively persisted by doing nothing.+ this.isStatePersisted = true;+ // Starts as false because we don't know yet and sources might change in the meantime (eg. download from live list).+ this.areSourcesPersisted = false;this.isLoading = false;this.isInitialized = false;this.sources = sources;-- // We will initialize everything from this state in this.initialize();- if (state) {- this.initialStatePromise = Promise.resolve([state]);- } else if (stateKeyPrefix) {- this.initialStatePromise = Promise.all([- getValue(this.stateKey),- getValue(this.sourcesKey),- ]);- } else if (persistStateKey) {- this.initialStatePromise = Promise.all([getValue(persistStateKey)]);- } else {- this.initialStatePromise = Promise.resolve([]);- }}/**@@ -202,9 +190,23 @@ class RequestList {}this.isLoading = true;- const [state, sources] = await this.initialStatePromise;+ let state;+ let sources;+ if (this.initialState) {+ log.debug('RequestList: Loading previous state from options.state argument.');+ state = this.initialState;+ } else if (this.shouldPersist) {+ log.debug('RequestList: Loading previous state and sources from key value store.');+ [state, sources] = await Promise.all([+ getValue(this.stateKey),+ getValue(this.sourcesKey),+ ]);+ } else if (this.persistStateKey) {+ state = await getValue(this.persistStateKey);+ }// If there are no sources, it just means that we've not persisted any (yet).+ if (sources) this.areSourcesPersisted = true;const actualSources = sources || this.sources;// We'll load all sources in sequence to ensure that they get loaded in the right order.@@ -216,7 +218,7 @@ class RequestList {this._restoreState(state);this.isInitialized = true;// TODO refactor after persistStateKey removal.- if (this.shouldPersist) await this._persistSources();+ if (this.shouldPersist && !this.areSourcesPersisted) await this._persistSources();if (this.shouldPersist || this.persistStateKey) {events.on(ACTOR_EVENT_NAMES_EX.PERSIST_STATE, this.persistState.bind(this));}@@ -237,10 +239,10 @@ class RequestList {throw new Error('RequestList: Cannot persist state. options.stateKeyPrefix is not set.');}const key = this.stateKey || this.persistStateKey;- if (this.isCurrentStatePersisted) return;+ if (this.isStatePersisted) return;try {await setValue(key, this.getState());- this.isCurrentStatePersisted = true;+ this.isStatePersisted = true;} catch (err) {log.exception(err, 'RequestList attempted to persist state, but failed.');}@@ -255,7 +257,8 @@ class RequestList {* @ignore*/async _persistSources() {- return setValue(this.sourcesKey, this.sources);+ await setValue(this.sourcesKey, this.sources);+ this.areSourcesPersisted = true;}/**@@ -385,7 +388,7 @@ class RequestList {const request = this.requests[this.nextIndex];this.inProgress[request.uniqueKey] = true;this.nextIndex++;- this.isCurrentStatePersisted = false;+ this.isStatePersisted = false;return request;}@@ -407,7 +410,7 @@ class RequestList {this._ensureIsInitialized();delete this.inProgress[uniqueKey];- this.isCurrentStatePersisted = false;+ this.isStatePersisted = false;}/**diff --git a/test/request_list.js b/test/request_list.jsindex 26b81455f..94fe691cb 100644--- a/test/request_list.js+++ b/test/request_list.js@@ -9,6 +9,7 @@ import { ACTOR_EVENT_NAMES_EX } from '../build/constants';import Apify from '../build/index';import * as keyValueStore from '../build/key_value_store';import * as utils from '../build/utils';+import { STATE_PERSISTENCE_KEY, SOURCES_PERSISTENCE_KEY } from '../build/request_list';chai.use(chaiAsPromised);@@ -419,11 +420,11 @@ describe('Apify.RequestList', () => {const requestList = new Apify.RequestList(opts);await requestList.initialize();- expect(requestList.isCurrentStatePersisted).to.be.eql(true);+ expect(requestList.isStatePersisted).to.be.eql(true);// Fetch one request and check that state is not persisted.const request1 = await requestList.fetchNextRequest();- expect(requestList.isCurrentStatePersisted).to.be.eql(false);+ expect(requestList.isStatePersisted).to.be.eql(false);// Persist state.mock.expects('setValue')@@ -432,24 +433,24 @@ describe('Apify.RequestList', () => {.returns(Promise.resolve());Apify.events.emit(ACTOR_EVENT_NAMES_EX.PERSIST_STATE);await delayPromise(1);- expect(requestList.isCurrentStatePersisted).to.be.eql(true);+ expect(requestList.isStatePersisted).to.be.eql(true);// Do some other changes and persist it again.const request2 = await requestList.fetchNextRequest();- expect(requestList.isCurrentStatePersisted).to.be.eql(false);+ expect(requestList.isStatePersisted).to.be.eql(false);await requestList.markRequestHandled(request2);- expect(requestList.isCurrentStatePersisted).to.be.eql(false);+ expect(requestList.isStatePersisted).to.be.eql(false);mock.expects('setValue').once().withArgs(PERSIST_STATE_KEY, requestList.getState()).returns(Promise.resolve());Apify.events.emit(ACTOR_EVENT_NAMES_EX.PERSIST_STATE);await delayPromise(1);- expect(requestList.isCurrentStatePersisted).to.be.eql(true);+ expect(requestList.isStatePersisted).to.be.eql(true);// Reclaim event doesn't change the state.await requestList.reclaimRequest(request1);- expect(requestList.isCurrentStatePersisted).to.be.eql(true);+ expect(requestList.isStatePersisted).to.be.eql(true);// Now initiate new request list from saved state and check that it's same as state// of original request list.@@ -465,6 +466,100 @@ describe('Apify.RequestList', () => {mock.restore();});+ it('should correctly persist its state when stateKeyPrefix is set', async () => {+ const STATE_PREFIX = 'prefix';+ const STATE_KEY = `${STATE_PREFIX}-${STATE_PERSISTENCE_KEY}`;+ const SOURCES_KEY = `${STATE_PREFIX}-${SOURCES_PERSISTENCE_KEY}`;+ const sources = [+ { url: 'https://example.com/1' },+ { url: 'https://example.com/2' },+ { url: 'https://example.com/3' },+ ];+ const mock = sinon.mock(keyValueStore);++ mock.expects('getValue')+ .once()+ .withArgs(STATE_KEY)+ .returns(null);+ mock.expects('getValue')+ .once()+ .withArgs(SOURCES_KEY)+ .returns(null);++ const opts = {+ sources: [...sources],+ stateKeyPrefix: STATE_PREFIX,+ };++ const requestList = new Apify.RequestList(opts);+ expect(requestList.areSourcesPersisted).to.be.eql(false);++ mock.expects('setValue')+ .once()+ .withArgs(SOURCES_KEY, sources)+ .returns(Promise.resolve());++ await requestList.initialize();+ expect(requestList.isStatePersisted).to.be.eql(true);+ expect(requestList.areSourcesPersisted).to.be.eql(true);++ // Fetch one request and check that state is not persisted.+ const request1 = await requestList.fetchNextRequest();+ expect(requestList.isStatePersisted).to.be.eql(false);+ expect(requestList.areSourcesPersisted).to.be.eql(true);++ // Persist state.+ mock.expects('setValue')+ .once()+ .withArgs(STATE_KEY, requestList.getState())+ .returns(Promise.resolve());+ Apify.events.emit(ACTOR_EVENT_NAMES_EX.PERSIST_STATE);+ await delayPromise(1);+ expect(requestList.isStatePersisted).to.be.eql(true);+ expect(requestList.areSourcesPersisted).to.be.eql(true);++ // Do some other changes and persist it again.+ const request2 = await requestList.fetchNextRequest();+ expect(requestList.isStatePersisted).to.be.eql(false);+ expect(requestList.areSourcesPersisted).to.be.eql(true);+ await requestList.markRequestHandled(request2);+ expect(requestList.isStatePersisted).to.be.eql(false);+ expect(requestList.areSourcesPersisted).to.be.eql(true);+ mock.expects('setValue')+ .once()+ .withArgs(STATE_KEY, requestList.getState())+ .returns(Promise.resolve());+ Apify.events.emit(ACTOR_EVENT_NAMES_EX.PERSIST_STATE);+ await delayPromise(1);+ expect(requestList.isStatePersisted).to.be.eql(true);+ expect(requestList.areSourcesPersisted).to.be.eql(true);++ // Reclaim event doesn't change the state.+ await requestList.reclaimRequest(request1);+ expect(requestList.isStatePersisted).to.be.eql(true);+ expect(requestList.areSourcesPersisted).to.be.eql(true);++ // Now initiate new request list from saved state and check that it's same as state+ // of original request list.+ mock.expects('getValue')+ .once()+ .withArgs(STATE_KEY)+ .returns(Promise.resolve(requestList.getState()));+ mock.expects('getValue')+ .once()+ .withArgs(SOURCES_KEY)+ .returns(sources);+ const requestList2 = new Apify.RequestList({+ sources: [{ url: 'https://iana.com' }], // change the sources to see if overridden+ stateKeyPrefix: STATE_PREFIX,+ });+ await requestList2.initialize();+ expect(requestList2.getState()).to.be.eql(requestList.getState());+ expect(requestList2.requests).to.be.eql(requestList.requests);++ mock.verify();+ });+it('handles correctly inconsistent inProgress fields in state', async () => {// NOTE: This is a test for the deleteFromInProgress hotfix - see RequestList.initialize()