Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

Add test

Ondra Urbana360fbcab7f1
diff --git a/test/basic_crawler.js b/test/basic_crawler.js
index 1b9b87e23..c639dcf44 100644
--- a/test/basic_crawler.js
+++ b/test/basic_crawler.js
@@ -3,8 +3,10 @@ import chaiAsPromised from 'chai-as-promised';
import _ from 'underscore';
import sinon from 'sinon';
import log from 'apify-shared/log';
+import { ACTOR_EVENT_NAMES } from 'apify-shared/consts';
import { delayPromise } from 'apify-shared/utilities';
import * as Apify from '../build/index';
+import * as keyValueStore from '../build/key_value_store';
import { RequestQueue, RequestQueueLocal } from '../build/request_queue';
import { LOCAL_STORAGE_DIR } from './_helper';
@@ -93,6 +95,50 @@ describe('BasicCrawler', () => {
expect(await requestList.isEmpty()).to.be.eql(true);
});
+ it('should pause on migration event and persist RequestList state', async () => {
+ const sources = _.range(500).map(index => ({ url: `https://example.com/${index + 1}` }));
+
+ let persistResolve;
+ const persistPromise = new Promise((res) => { persistResolve = res; });
+
+ // Mock the calls to persist sources.
+ const mock = sinon.mock(keyValueStore);
+ mock.expects('getValue').twice().resolves(null);
+ mock.expects('setValue').once().resolves();
+
+ const processed = [];
+ const requestList = await Apify.openRequestList('reqList', sources);
+ const handleRequestFunction = async ({ request }) => {
+ if (request.url.endsWith('200')) Apify.events.emit(ACTOR_EVENT_NAMES.MIGRATING);
+ processed.push(_.pick(request, 'url'));
+ };
+
+ const basicCrawler = new Apify.BasicCrawler({
+ requestList,
+ minConcurrency: 25,
+ maxConcurrency: 25,
+ handleRequestFunction,
+ });
+
+ let finished = false;
+ // Mock the call to persist state.
+ mock.expects('setValue').once().callsFake(async () => { persistResolve(); });
+ // The crawler will pause after 200 requests
+ const runPromise = basicCrawler.run();
+ runPromise.then(() => { finished = true; });
+ await persistPromise;
+
+ expect(finished).to.be.eql(false);
+ expect(await requestList.isFinished()).to.be.eql(false);
+ expect(await requestList.isEmpty()).to.be.eql(false);
+ expect(processed.length).to.be.eql(200);
+
+ mock.verify();
+
+ // clean up
+ await basicCrawler.autoscaledPool._destroy(); // eslint-disable-line no-underscore-dangle
+ });
+
it('should retry failed requests', async () => {
const sources = [
{ url: 'http://example.com/1' },