Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
Feature/request list length (#100)
Jan Čurn129da3a4ba7e
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex 6d585abea..c76ff584a 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -1,3 +1,7 @@+xxx+===================+- Added `RequestList.length()` function+0.5.32 / 2018-05-14===================- Bugfix in `RequestList` - skip invalid in-progress entries when restoring statediff --git a/src/request_list.js b/src/request_list.jsindex 7ec0a43c8..743bea014 100644--- a/src/request_list.js+++ b/src/request_list.js@@ -244,7 +244,7 @@ export default class RequestList {}/**- * Returns `true` if the next call to fetchNextRequest() will return null, otherwise it returns `false`.+ * Returns `true` if the next call to `fetchNextRequest()` will return null, otherwise it returns `false`.* Note that even if the list is empty, there might be some pending requests currently being processed.** @returns {Promise<boolean>}@@ -396,7 +396,7 @@ export default class RequestList {/*** Adds given request.- * If opts partameter is plain object not instance of an Requests then creates it.+ * If opts parameter is plain object not instance of an Requests then creates it.** @ignore*/@@ -439,4 +439,11 @@ export default class RequestList {throw new Error('RequestList is not initialized. You must call "await requestList.initialize();" before using it!');}}++ /**+ * Returns the total number of unique requests present in the `RequestList`.+ */+ length() {+ return this.requests.length;+ }}diff --git a/test/request_list.js b/test/request_list.jsindex 8926bfe00..9f2d44bbc 100644--- a/test/request_list.js+++ b/test/request_list.js@@ -493,4 +493,22 @@ describe('Apify.RequestList', () => {await requestList.reclaimRequest(reqs[i]); // eslint-disable-line}});++ it('it gets correct length()', async () => {+ const sources = [+ { url: 'https://www.example.com' },+ { url: 'https://www.ams360.com' },+ { url: 'https://www.anybus.com' },+ { url: 'https://www.anychart.com' },+ { url: 'https://www.example.com' },+ ];++ const requestList = new Apify.RequestList({+ sources,+ });++ await requestList.initialize();++ expect(requestList.length()).to.be.eql(4);+ });});