Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

Fix deprecation warning, fix keepDuplicateUrls

Ondra Urban1edd99a4585b
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 6de8ffb55..42f1f57dd 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,3 +1,9 @@
+0.6.3 / 2018-08-24
+==================
+- Fixed persistent deprecation warning of `pageOpsTimeoutMillis`.
+- Moved `cheerio` to dependencies.
+- Fixed `keepDuplicateUrls` errors with persistent RequestList.
+
0.6.2 / 2018-08-23
===================
- Added `getInfo()` method to Dataset to get meta-information about a dataset.
diff --git a/package.json b/package.json
index bc8b6361c..bd603cd90 100644
--- a/package.json
+++ b/package.json
@@ -1,6 +1,6 @@
{
"name": "apify",
- "version": "0.6.2",
+ "version": "0.6.3",
"description": "Web scraping and automation SDK",
"engines": {
"node": ">=8.0.0"
@@ -51,6 +51,7 @@
"apify-client": "^0.2.9",
"apify-shared": "0.0.50",
"bluebird": "^3.5.0",
+ "cheerio": "^1.0.0-rc.2",
"content-type": "^1.0.3",
"fs-extra": "^7.0.0",
"jquery": "^3.3.1",
@@ -93,7 +94,6 @@
},
"optionalDependencies": {
"puppeteer": "^1.7.0",
- "selenium-webdriver": "^3.6.0",
- "cheerio": "^1.0.0-rc.2"
+ "selenium-webdriver": "^3.6.0"
}
}
diff --git a/src/puppeteer_crawler.js b/src/puppeteer_crawler.js
index 555aef574..98a0d1096 100644
--- a/src/puppeteer_crawler.js
+++ b/src/puppeteer_crawler.js
@@ -7,7 +7,6 @@ import { createTimeoutPromise } from './utils';
const DEFAULT_OPTIONS = {
gotoFunction: async ({ request, page }) => page.goto(request.url, { timeout: 60000 }),
- pageOpsTimeoutMillis: 300000,
handlePageTimeoutSecs: 300,
};
diff --git a/src/request_list.js b/src/request_list.js
index fcf459818..e504105d7 100644
--- a/src/request_list.js
+++ b/src/request_list.js
@@ -95,8 +95,8 @@ const ensureUniqueKeyValid = (uniqueKey) => {
* on the `uniqueKey` property of passed sources. If not present, it is generated by normalizing
* the URL. If present, it is kept intact. In any case, only one request per `uniqueKey` is added
* to the `RequestList` resulting in removing of duplicate URLs / unique keys.
- * Setting `keepDuplicateUrls` to true will generate a random unique identifier for each request
- * that does not already include a `uniqueKey` to be used as the unique key. Therefore, duplicate
+ * Setting `keepDuplicateUrls` to true will append an additional identifier to the `uniqueKey`
+ * of each request that does not already include a `uniqueKey`. Therefore, duplicate
* URLs will be kept in the list. It does not protect the user from having duplicates in user set
* `uniqueKey`s however. It is the user's responsibility to ensure uniqueness of their unique keys,
* if they wish to keep more than just a single copy in the `RequestList`.
@@ -419,15 +419,17 @@ export default class RequestList {
* @ignore
*/
_addRequest(opts) {
- // Generate random uniqueKey if duplicates are to be kept
- if (this.keepDuplicateUrls && !opts.uniqueKey) {
- opts.uniqueKey = crypto.randomBytes(20).toString('base64').replace(/[+/=]/g, '');
- }
+ const hasUniqueKey = !!opts.uniqueKey;
const request = opts instanceof Request
? opts
: new Request(opts);
+ // Add index to uniqueKey if duplicates are to be kept
+ if (this.keepDuplicateUrls && !hasUniqueKey) {
+ request.uniqueKey += `-${this.requests.length}`;
+ }
+
const { uniqueKey } = request;
ensureUniqueKeyValid(uniqueKey);