Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
Fix deprecation warning, fix keepDuplicateUrls
Ondra Urban1edd99a4585b
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex 6de8ffb55..42f1f57dd 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -1,3 +1,9 @@+0.6.3 / 2018-08-24+==================+- Fixed persistent deprecation warning of `pageOpsTimeoutMillis`.+- Moved `cheerio` to dependencies.+- Fixed `keepDuplicateUrls` errors with persistent RequestList.+0.6.2 / 2018-08-23===================- Added `getInfo()` method to Dataset to get meta-information about a dataset.diff --git a/package.json b/package.jsonindex bc8b6361c..bd603cd90 100644--- a/package.json+++ b/package.json@@ -1,6 +1,6 @@{"name": "apify",- "version": "0.6.2",+ "version": "0.6.3","description": "Web scraping and automation SDK","engines": {"node": ">=8.0.0"@@ -51,6 +51,7 @@"apify-client": "^0.2.9","apify-shared": "0.0.50","bluebird": "^3.5.0",+ "cheerio": "^1.0.0-rc.2","content-type": "^1.0.3","fs-extra": "^7.0.0","jquery": "^3.3.1",@@ -93,7 +94,6 @@},"optionalDependencies": {"puppeteer": "^1.7.0",- "selenium-webdriver": "^3.6.0",- "cheerio": "^1.0.0-rc.2"+ "selenium-webdriver": "^3.6.0"}}diff --git a/src/puppeteer_crawler.js b/src/puppeteer_crawler.jsindex 555aef574..98a0d1096 100644--- a/src/puppeteer_crawler.js+++ b/src/puppeteer_crawler.js@@ -7,7 +7,6 @@ import { createTimeoutPromise } from './utils';const DEFAULT_OPTIONS = {gotoFunction: async ({ request, page }) => page.goto(request.url, { timeout: 60000 }),- pageOpsTimeoutMillis: 300000,handlePageTimeoutSecs: 300,};diff --git a/src/request_list.js b/src/request_list.jsindex fcf459818..e504105d7 100644--- a/src/request_list.js+++ b/src/request_list.js@@ -95,8 +95,8 @@ const ensureUniqueKeyValid = (uniqueKey) => {* on the `uniqueKey` property of passed sources. If not present, it is generated by normalizing* the URL. If present, it is kept intact. In any case, only one request per `uniqueKey` is added* to the `RequestList` resulting in removing of duplicate URLs / unique keys.- * Setting `keepDuplicateUrls` to true will generate a random unique identifier for each request- * that does not already include a `uniqueKey` to be used as the unique key. Therefore, duplicate+ * Setting `keepDuplicateUrls` to true will append an additional identifier to the `uniqueKey`+ * of each request that does not already include a `uniqueKey`. Therefore, duplicate* URLs will be kept in the list. It does not protect the user from having duplicates in user set* `uniqueKey`s however. It is the user's responsibility to ensure uniqueness of their unique keys,* if they wish to keep more than just a single copy in the `RequestList`.@@ -419,15 +419,17 @@ export default class RequestList {* @ignore*/_addRequest(opts) {- // Generate random uniqueKey if duplicates are to be kept- if (this.keepDuplicateUrls && !opts.uniqueKey) {- opts.uniqueKey = crypto.randomBytes(20).toString('base64').replace(/[+/=]/g, '');- }+ const hasUniqueKey = !!opts.uniqueKey;const request = opts instanceof Request? opts: new Request(opts);+ // Add index to uniqueKey if duplicates are to be kept+ if (this.keepDuplicateUrls && !hasUniqueKey) {+ request.uniqueKey += `-${this.requests.length}`;+ }+const { uniqueKey } = request;ensureUniqueKeyValid(uniqueKey);