Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
Add openRequestList tests
Ondra Urbancb2d80fd8b9a
diff --git a/src/basic_crawler.js b/src/basic_crawler.jsindex 71ba92769..8706c4db1 100644--- a/src/basic_crawler.js+++ b/src/basic_crawler.js@@ -3,7 +3,7 @@ import _ from 'underscore';import log from 'apify-shared/log';import { checkParamPrototypeOrThrow } from 'apify-shared/utilities';import AutoscaledPool from './autoscaling/autoscaled_pool';-import RequestList from './request_list';+import { RequestList } from './request_list';import { RequestQueue, RequestQueueLocal } from './request_queue';const DEFAULT_OPTIONS = {diff --git a/src/index.js b/src/index.jsindex f81fb4fab..a6e58464c 100644--- a/src/index.js+++ b/src/index.js@@ -12,7 +12,7 @@ import { launchPuppeteer } from './puppeteer';import PuppeteerCrawler from './puppeteer_crawler';import PuppeteerPool from './puppeteer_pool';import Request from './request';-import RequestList from './request_list';+import { RequestList, openRequestList } from './request_list';import { openRequestQueue } from './request_queue';import SettingsRotator from './settings_rotator';import { apifyClient, getMemoryInfo, isAtHome, publicUtils } from './utils';@@ -88,6 +88,7 @@ module.exports = {// RequestsRequest,RequestList,+ openRequestList,openRequestQueue,// Settings rotatordiff --git a/src/request_list.js b/src/request_list.jsindex b6961aa1d..59a0dbdd9 100644--- a/src/request_list.js+++ b/src/request_list.js@@ -117,7 +117,7 @@ export const SOURCES_PERSISTENCE_KEY = 'REQUEST_LIST_SOURCES';* `uniqueKey`s however. It is the user's responsibility to ensure uniqueness of their unique keys* if they wish to keep more than just a single copy in the `RequestList`.*/-class RequestList {+export class RequestList {constructor(options = {}) {checkParamOrThrow(options, 'options', 'Object');@@ -562,8 +562,6 @@ class RequestList {}}-export default RequestList;-/*** Opens a request list and returns a promise resolving to an instance* of the {@link RequestList} class that is already initialized.@@ -606,5 +604,6 @@ export const openRequestList = async (listName, sources, options = {}) => {stateKeyPrefix: listName,sources,});- return rl.initialize();+ await rl.initialize();+ return rl;};diff --git a/test/request_list.js b/test/request_list.jsindex 94fe691cb..b11b1aceb 100644--- a/test/request_list.js+++ b/test/request_list.js@@ -687,4 +687,100 @@ describe('Apify.RequestList', () => {logStub.restore();log.setLevel(log.LEVELS.ERROR);});++ describe('Apify.openRequestList()', () => {+ it('should work', async () => {+ const mock = sinon.mock(keyValueStore);+ mock.expects('getValue').atLeast(1).resolves();+ mock.expects('setValue').atLeast(1).resolves();++ const name = 'xxx';+ const sources = [{ url: 'https://example.com' }];++ const rl = await Apify.openRequestList(name, sources);+ expect(rl).to.be.instanceof(Apify.RequestList);+ expect(rl.stateKey.startsWith(name)).to.be.eql(true);+ expect(rl.sourcesKey.startsWith(name)).to.be.eql(true);+ expect(rl.sources).to.be.eql(sources);+ expect(rl.isInitialized).to.be.eql(true);++ mock.verify();+ });+ it('should work with string sources', async () => {+ const mock = sinon.mock(keyValueStore);+ mock.expects('getValue').atLeast(1).resolves();+ mock.expects('setValue').atLeast(1).resolves();++ const name = 'xxx';+ const sources = ['https://example.com'];++ const rl = await Apify.openRequestList(name, sources);+ expect(rl).to.be.instanceof(Apify.RequestList);+ expect(rl.stateKey.startsWith(name)).to.be.eql(true);+ expect(rl.sourcesKey.startsWith(name)).to.be.eql(true);+ expect(rl.sources).to.be.eql(sources.map(url => ({ url })));+ expect(rl.isInitialized).to.be.eql(true);++ mock.verify();+ });+ it('should correctly pass options', async () => {+ const mock = sinon.mock(keyValueStore);+ mock.expects('getValue').atLeast(1).resolves();+ mock.expects('setValue').atLeast(1).resolves();++ const name = 'xxx';+ const sources = [{ url: 'https://example.com' }];+ const options = {+ keepDuplicateUrls: true,+ stateKeyPrefix: 'yyy',+ };++ const rl = await Apify.openRequestList(name, sources, options);+ expect(rl).to.be.instanceof(Apify.RequestList);+ expect(rl.stateKey.startsWith(name)).to.be.eql(true);+ expect(rl.sourcesKey.startsWith(name)).to.be.eql(true);+ expect(rl.sources).to.be.eql(sources);+ expect(rl.isInitialized).to.be.eql(true);+ expect(rl.keepDuplicateUrls).to.be.eql(true);++ mock.verify();+ });+ it('should work with null name', async () => {+ const mock = sinon.mock(keyValueStore);+ mock.expects('getValue').never().resolves();+ mock.expects('setValue').never().resolves();++ const name = null;+ const sources = [{ url: 'https://example.com' }];++ const rl = await Apify.openRequestList(name, sources);+ expect(rl).to.be.instanceof(Apify.RequestList);+ expect(rl.shouldPersist).to.be.eql(false);+ expect(rl.stateKey).to.be.eql(null);+ expect(rl.sourcesKey).to.be.eql(null);+ expect(rl.sources).to.be.eql(sources);+ expect(rl.isInitialized).to.be.eql(true);++ mock.verify();+ });+ it('should throw on invalid parameters', async () => {+ const args = [+ [],+ ['x', []],+ ['x', [6]],+ ['x', [[]], {}],+ ['x', [[]], []],+ ];+ for (const arg of args) {+ try {+ await Apify.openRequestList(...arg);+ throw new Error('wrong error');+ } catch (err) {+ expect(err.message).to.not.be.eql('wrong error');+ expect(err.message).to.include('Parameter');+ expect(err.message).to.not.be.eql('must');+ }+ }+ });+ });});