Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
Revert "RequestList: Persist downloaded requests from requestsFromUrl"
drobnikjd88d7ae9532b
diff --git a/src/request_list.js b/src/request_list.jsindex 598a57543..19b2c1cd4 100644--- a/src/request_list.js+++ b/src/request_list.js@@ -199,21 +199,11 @@ export class RequestList {const actualSources = sources || this.sources;// We'll load all sources in sequence to ensure that they get loaded in the right order.- const sourcesWithRequestsFromUrl = [];for (const source of actualSources) {- if (source.requestsFromUrl) {- const fetchedRequests = await this._fetchRequestsFromUrl(source);- sourcesWithRequestsFromUrl.push(...fetchedRequests);- await this._addFetchedRequests(source, fetchedRequests);- } else {- sourcesWithRequestsFromUrl.push(source);- this._addRequest(source);- }+ if (source.requestsFromUrl) await this._addRequestsFromUrl(source);+ else this._addRequest(source);}- // Replace source with source with request from remote URLs- this.sources = sourcesWithRequestsFromUrl;-this._restoreState(state);this.isInitialized = true;if (this.persistSourcesKey && !this.areSourcesPersisted) await this._persistSources();@@ -451,36 +441,11 @@ export class RequestList {}/**- * Adds all fetched requests from a URL from a remote resource.+ * Adds all requests from a URL fetched from a remote resource.** @ignore*/- async _addFetchedRequests(source, fetchedRequests) {- const { requestsFromUrl, regex } = source;- const originalLength = this.requests.length;-- fetchedRequests.forEach(request => this._addRequest(request));-- const fetchedCount = fetchedRequests.length;- const importedCount = this.requests.length - originalLength;-- log.info('RequestList: list fetched.', {- requestsFromUrl,- regex,- fetchedCount,- importedCount,- duplicateCount: fetchedCount - importedCount,- sample: JSON.stringify(fetchedRequests.slice(0, 5)),- });- }-- /**- * Fetches URLs from requestsFromUrl and returns them in format of list of requests- * @param source- * @return {Promise<Object[]|Array>}- * @ignore- */- async _fetchRequestsFromUrl(source) {+ async _addRequestsFromUrl(source) {const sharedOpts = _.omit(source, 'requestsFromUrl', 'regex');const { requestsFromUrl, regex } = source;const { downloadListOfUrls } = publicUtils;@@ -495,11 +460,24 @@ export class RequestList {// Skip if resource contained no URLs.if (!urlsArr.length) {- log.warning('RequestList: list fetched, but it is empty.', { requestsFromUrl, regex });- return [];+ return log.warning('RequestList: list fetched, but it is empty.', { requestsFromUrl, regex });}- return urlsArr.map(url => _.extend({ url }, sharedOpts));+ // Process downloaded URLs.+ const originalLength = this.requests.length;+ urlsArr.forEach(url => this._addRequest(_.extend({ url }, sharedOpts)));++ const fetchedCount = urlsArr.length;+ const importedCount = this.requests.length - originalLength;++ log.info('RequestList: list fetched.', {+ requestsFromUrl,+ regex,+ fetchedCount,+ importedCount,+ duplicateCount: fetchedCount - importedCount,+ sample: JSON.stringify(urlsArr.slice(0, 5)),+ });}/**diff --git a/test/request_list.js b/test/request_list.jsindex e31f080ec..a30d3c06f 100644--- a/test/request_list.js+++ b/test/request_list.js@@ -521,51 +521,6 @@ describe('Apify.RequestList', () => {mock.verify();});- it('should correctly persist sources from requestsFromUrl if persistSourcesKey is set', async () => {- const PERSIST_SOURCES_KEY = 'some-key';- const kvsMock = sinon.mock(keyValueStore);- const publicUtilsMock = sinon.mock(utils.publicUtils);-- const opts = {- sources: [- { url: 'https://example.com/1' },- { url: 'https://example.com/2' },- { requestsFromUrl: 'http://example.com/list-urls.txt', userData: { isFromUrl: true } },- { url: 'https://example.com/5' },- ],- persistSourcesKey: PERSIST_SOURCES_KEY,- };-- const urlsFromTxt = ['http://example.com/3', 'http://example.com/4'];-- const requestList = new Apify.RequestList(opts);- expect(requestList.areSourcesPersisted).to.be.eql(false);-- // Expect an attempt to load sources.- kvsMock.expects('getValue')- .once()- .withArgs(PERSIST_SOURCES_KEY)- .returns(null);-- // Expect persist sources.- const sourcesFromTxt = urlsFromTxt.map(url => ({ url, userData: { isFromUrl: true } }));- kvsMock.expects('setValue')- .once()- .withArgs(PERSIST_SOURCES_KEY, [opts.sources[0], opts.sources[1], ...sourcesFromTxt, opts.sources[3]])- .returns(Promise.resolve());- // Expect downloadListOfUrls returns list of URLs- publicUtilsMock.expects('downloadListOfUrls')- .once()- .withArgs({ url: 'http://example.com/list-urls.txt', urlRegExp: undefined })- .returns(Promise.resolve(urlsFromTxt));-- await requestList.initialize();- expect(requestList.areSourcesPersisted).to.be.eql(true);-- kvsMock.verify();- publicUtilsMock.verify();- });-it('handles correctly inconsistent inProgress fields in state', async () => {// NOTE: This is a test for the deleteFromInProgress hotfix - see RequestList.initialize()