Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

Feature/url list persist (#61)

Marek Trunkátbd2476bc3d17
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 322acaaa4..a99400c37 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -11,6 +11,8 @@ xxxxxx / xxxxxxxxxx
- `AutoscaledPool` now scales down less aggresively.
- `PuppeteerCrawler` and `BasicCrawler` now allow its underlying `AutoscaledPool` function `isFunction` to be overriden.
- New events `persistState` and `migrating` added. Check documentation of `Apify.events` for more information.
+- `RequestList` has a new parameter `persistStateKey`. If this is used then `RequestList` persists its state in the default
+ key-value store at regular intervals.
0.5.17 / 2018-03-27
===================
diff --git a/src/constants.js b/src/constants.js
index 4b3cfaa71..ce52ba6cf 100644
--- a/src/constants.js
+++ b/src/constants.js
@@ -48,4 +48,10 @@ consts.KEY_VALUE_STORE_KEYS = {
OUTPUT: 'OUTPUT',
};
+/**
+ * These events are just internal for Apify package so we don't need them
+ * in apify-shared package.
+ */
+consts.ACTOR_EVENT_NAMES.PERSIST_STATE = 'persistState';
+
module.exports = consts;
diff --git a/src/events.js b/src/events.js
index 2b2052919..14c73d63f 100644
--- a/src/events.js
+++ b/src/events.js
@@ -46,6 +46,7 @@ let persistStateInterval = null;
* <table class="table table-bordered table-condensed">
* <thead>
* <tr>
+ * <th>Event</th>
* <th>Name</th>
* <th>Constant</th>
* <ht>Message</th>
diff --git a/src/request_list.js b/src/request_list.js
index e1c3c0ddd..384f5e3f4 100644
--- a/src/request_list.js
+++ b/src/request_list.js
@@ -5,7 +5,10 @@ import Promise from 'bluebird';
import requestPromise from 'request-promise';
import { sequentializePromises } from 'apify-shared/utilities';
import Request from './request';
+import events from './events';
+import { ACTOR_EVENT_NAMES } from './constants';
import { getFirstKey } from './utils';
+import { getValue, setValue } from './key_value_store';
// TODO: better tests
const URL_REGEX = '(http|https)://[\\w-]+(\\.[\\w-]+)+([\\w-.,@?^=%&:/~+#-]*[\\w@?^=%&;/~+#-])?';
@@ -43,17 +46,10 @@ const ensureUniqueKeyValid = (uniqueKey) => {
* // Note that all URLs must start with http:// or https://
* { requestsFromUrl: 'http://www.example.com/my-url-list.txt', userData: { isFromUrl: true } },
* ],
- * // Initialize from previous state if act was restarted due to some error
- * state: await Apify.getValue('my-request-list-state'),
* });
*
* await requestList.initialize(); // Load requests.
*
- * // Save state of the RequestList instance every 5 seconds.
- * setInterval(() => {
- * Apify.setValue('my-request-list-state', requestList.getState());
- * }, 5000);
- *
* // Get requests from list
* const request1 = requestList.fetchNextRequest();
* const request2 = requestList.fetchNextRequest();
@@ -87,18 +83,24 @@ const ensureUniqueKeyValid = (uniqueKey) => {
* },
* }
* ```
+ * @param {String} [options.persistStateKey] Key-value store key under which RequestList persists its state. If this is set then RequestList
+ * persists its state in regular intervals and loads the state from there in a case thats restarted
+ * due to some error or migration to another worker machine.
*/
export default class RequestList {
constructor(opts = {}) {
checkParamOrThrow(opts, 'options', 'Object');
- const { sources, state } = opts;
+ const { sources, persistStateKey, state } = opts;
checkParamOrThrow(sources, 'options.sources', 'Array');
checkParamOrThrow(state, 'options.state', 'Maybe Object');
+ checkParamOrThrow(persistStateKey, 'options.persistStateKey', 'Maybe String');
// We will initialize everything from this state in this.initialize();
- this.initialState = state;
+ this.initialStatePromise = persistStateKey && !state
+ ? getValue(persistStateKey)
+ : Promise.resolve(state);
// Array of all requests from all sources, in the order as they appeared in sources.
// All requests in the array have distinct uniqueKey!
@@ -119,6 +121,10 @@ export default class RequestList {
// Note that reclaimedRequests is always a subset of inProgressRequests!
this.reclaimed = {};
+ // If this key is set then we persist url list into default key-value store under this key.
+ this.persistStateKey = persistStateKey;
+ this.isStatePersisted = true;
+
this.isLoading = false;
this.isInitialized = false;
@@ -143,9 +149,8 @@ export default class RequestList {
this.isLoading = true;
return sequentializePromises(this.loadSourcesPromiseGenerators)
- .then(() => {
- const state = this.initialState;
-
+ .then(() => this.initialStatePromise)
+ .then((state) => {
if (!state) return;
// Restore state
@@ -174,6 +179,20 @@ export default class RequestList {
})
.then(() => {
this.isInitialized = true;
+
+ if (!this.persistStateKey) return;
+
+ events.on(ACTOR_EVENT_NAMES.PERSIST_STATE, () => {
+ if (this.isStatePersisted) return;
+
+ return setValue(this.persistStateKey, this.getState())
+ .then(() => {
+ this.isStatePersisted = true;
+ })
+ .catch((err) => {
+ log.exception(err, 'Cannot persist state of UrlList', { persistStateKey: this.persistStateKey });
+ });
+ });
});
}
@@ -249,6 +268,7 @@ export default class RequestList {
const request = this.requests[this.nextIndex];
this.inProgress[request.uniqueKey] = true;
this.nextIndex++;
+ this.isStatePersisted = false;
return request;
}
@@ -274,6 +294,7 @@ export default class RequestList {
this._ensureIsInitialized();
delete this.inProgress[uniqueKey];
+ this.isStatePersisted = false;
});
}
diff --git a/test/request_list.js b/test/request_list.js
index 6ab7c74d6..a920f77e1 100644
--- a/test/request_list.js
+++ b/test/request_list.js
@@ -3,7 +3,10 @@ import chai, { expect } from 'chai';
import chaiAsPromised from 'chai-as-promised';
import request from 'request-promise';
import sinon from 'sinon';
+import { delayPromise } from 'apify-shared/utilities';
import Apify from '../build/index';
+import * as keyValueStore from '../build/key_value_store';
+import { ACTOR_EVENT_NAMES } from '../build/constants';
chai.use(chaiAsPromised);
@@ -381,4 +384,70 @@ describe('Apify.RequestList', () => {
expect(await requestList.isFinished()).to.be.eql(true);
expect(requestList.inProgress).to.include(requestList.reclaimed);
});
+
+ it('should correctly persists its state when persistStateKey is set', async () => {
+ const PERSIST_STATE_KEY = 'some-key';
+ const mock = sinon.mock(keyValueStore);
+
+ mock.expects('getValue')
+ .once()
+ .withArgs(PERSIST_STATE_KEY)
+ .returns(null);
+
+ const opts = {
+ sources: [
+ { url: 'https://example.com/1' },
+ { url: 'https://example.com/2' },
+ { url: 'https://example.com/3' },
+ ],
+ persistStateKey: PERSIST_STATE_KEY,
+ };
+
+ const requestList = new Apify.RequestList(opts);
+ await requestList.initialize();
+ expect(requestList.isStatePersisted).to.be.eql(true);
+
+ // Fetch one request and check that state is not persisted.
+ const request1 = await requestList.fetchNextRequest();
+ expect(requestList.isStatePersisted).to.be.eql(false);
+
+ // Persist state.
+ mock.expects('setValue')
+ .once()
+ .withArgs(PERSIST_STATE_KEY, requestList.getState())
+ .returns(Promise.resolve());
+ Apify.events.emit(ACTOR_EVENT_NAMES.PERSIST_STATE);
+ await delayPromise(1);
+ expect(requestList.isStatePersisted).to.be.eql(true);
+
+ // Do some other changes and persist it again.
+ const request2 = await requestList.fetchNextRequest();
+ expect(requestList.isStatePersisted).to.be.eql(false);
+ await requestList.markRequestHandled(request2);
+ expect(requestList.isStatePersisted).to.be.eql(false);
+ mock.expects('setValue')
+ .once()
+ .withArgs(PERSIST_STATE_KEY, requestList.getState())
+ .returns(Promise.resolve());
+ Apify.events.emit(ACTOR_EVENT_NAMES.PERSIST_STATE);
+ await delayPromise(1);
+ expect(requestList.isStatePersisted).to.be.eql(true);
+
+ // Reclaim event doesn't change the state.
+ await requestList.reclaimRequest(request1);
+ expect(requestList.isStatePersisted).to.be.eql(true);
+
+ // Now initiate new request list from saved state and check that it's same as state
+ // of original request list.
+ mock.expects('getValue')
+ .once()
+ .withArgs(PERSIST_STATE_KEY)
+ .returns(Promise.resolve(requestList.getState()));
+ const requestList2 = new Apify.RequestList(opts);
+ await requestList2.initialize();
+ expect(requestList2.getState()).to.be.eql(requestList.getState());
+
+ mock.verify();
+ mock.restore();
+ });
});