Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

Fixing documentation and bug in BasicCrawler

Marek Trunkat10f115a49744
diff --git a/package.json b/package.json
index f1bf157e1..acd493193 100644
--- a/package.json
+++ b/package.json
@@ -1,6 +1,6 @@
{
"name": "apify",
- "version": "0.5.0",
+ "version": "0.5.1",
"description": "Apify Actor runtime for JavaScript",
"main": "build/index.js",
"keywords": [
diff --git a/src/basic_crawler.js b/src/basic_crawler.js
index 18e7972e9..4d752c522 100644
--- a/src/basic_crawler.js
+++ b/src/basic_crawler.js
@@ -24,7 +24,7 @@ const DEFAULT_OPTIONS = {
* const request = require('request-promise');
*
* const crawler = new Apify.BasicCrawler({
- * urlList,
+ * requestList,
* handleRequestFunction: async ({ request }) => {
* await Apify.pushData({
* html: await request(request.url),
@@ -101,20 +101,24 @@ export default class BasicCrawler {
const handlePromise = this.handleRequestFunction({ request });
if (!isPromise(handlePromise)) throw new Error('User provided handleRequestFunction must return a Promise.');
- return handlePromise.catch((err) => {
- request.pushErrorMessage(err);
+ return handlePromise
+ .then(() => {
+ this.requestList.markRequestHandled(request);
+ })
+ .catch((err) => {
+ request.pushErrorMessage(err);
- // Retry request.
- if (request.retryCount < this.maxRequestRetries) {
- request.retryCount++;
- this.requestList.reclaimRequest(request);
- return;
- }
+ // Retry request.
+ if (request.retryCount < this.maxRequestRetries) {
+ request.retryCount++;
+ this.requestList.reclaimRequest(request);
+ return;
+ }
- // Mark as failed.
- this.requestList.markRequestHandled(request);
+ // Mark as failed.
+ this.requestList.markRequestHandled(request);
- return this.handleFailedRequestFunction({ request });
- });
+ return this.handleFailedRequestFunction({ request });
+ });
}
}
diff --git a/src/puppeteer_crawler.js b/src/puppeteer_crawler.js
index c4fb70edb..f3e998e27 100644
--- a/src/puppeteer_crawler.js
+++ b/src/puppeteer_crawler.js
@@ -23,8 +23,8 @@ const DEFAULT_OPTIONS = {
* ```javascript
* const request = require('request-promise');
*
- * const crawler = new Apify.BasicCrawler({
- * urlList,
+ * const crawler = new Apify.PuppeteerCrawler({
+ * requestList,
* // Parameter page here is an intance of Puppeteer.Page with page.goto(request.url) already called
* handlePageFunction: async ({ page, request }) => {
* await Apify.pushData({
diff --git a/src/request_list.js b/src/request_list.js
index 20d963379..383180c18 100644
--- a/src/request_list.js
+++ b/src/request_list.js
@@ -47,7 +47,7 @@ const ensureUniqueKeyValid = (uniqueKey) => {
* { url: 'http://www.example.com/page-1', method: 'GET', headers: {} },
* { url: 'http://www.example.com/page-2', userData: { foo: 'bar' }},
* // Bulk load of urls from file `http://www.example.com/my-url-list.txt`
- * { pagesFromUrl: 'http://www.example.com/my-url-list.txt', userData: { isFromUrl: true } },
+ * { requestsFromUrl: 'http://www.example.com/my-url-list.txt', userData: { isFromUrl: true } },
* ],
* // Initialize from previous state if act was restarted due to some error
* state: await Apify.getValue('my-request-list-state'),
diff --git a/test/basic_crawler.js b/test/basic_crawler.js
index 7b04044ab..fd81d5558 100644
--- a/test/basic_crawler.js
+++ b/test/basic_crawler.js
@@ -31,6 +31,8 @@ describe('basic_crawler', () => {
expect(processed).to.be.eql(sources);
expect(Date.now() - startedAt).to.be.within(200, 400);
+ expect(requestList.isFinished()).to.be.eql(true);
+ expect(requestList.isEmpty()).to.be.eql(true);
});
it('retries failed requests', async () => {
@@ -74,6 +76,9 @@ describe('basic_crawler', () => {
expect(processed['http://example.com/2'].userData.foo).to.be.a('undefined');
expect(processed['http://example.com/2'].errorMessages).to.have.lengthOf(11);
expect(processed['http://example.com/2'].retryCount).to.be.eql(10);
+
+ expect(requestList.isFinished()).to.be.eql(true);
+ expect(requestList.isEmpty()).to.be.eql(true);
});
it('should allow to handle failed requests', async () => {
@@ -112,5 +117,7 @@ describe('basic_crawler', () => {
expect(failed['http://example.com/3'].retryCount).to.be.eql(3);
expect(_.values(failed)).to.have.length.of(3);
expect(_.values(processed)).to.have.length.of(0);
+ expect(requestList.isFinished()).to.be.eql(true);
+ expect(requestList.isEmpty()).to.be.eql(true);
});
});