Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

Fixed RequestQueue.addRequest() to gracefully handle invalid URLs

Jan Curnec1ca223b45b
diff --git a/src/request.js b/src/request.js
index 4625a6597..9aed83d6e 100644
--- a/src/request.js
+++ b/src/request.js
@@ -130,9 +130,12 @@ class Request {
if (method === 'GET' && payload) throw new Error('Request with GET method cannot have a payload.');
+ if (!url) throw new Error('The "url" option cannot be empty string.');
+
this.id = id;
this.url = url;
- this.uniqueKey = uniqueKey || computeUniqueKey(url, keepUrlFragment);
+ // NOTE: If URL is invalid, computeUniqueKey() returns null which was causing weird errors
+ this.uniqueKey = uniqueKey || computeUniqueKey(url, keepUrlFragment) || url;
this.method = method;
this.payload = payload;
this.retryCount = retryCount;
diff --git a/src/request_queue.js b/src/request_queue.js
index 345dd9142..a7ff3d526 100644
--- a/src/request_queue.js
+++ b/src/request_queue.js
@@ -213,7 +213,7 @@ export class RequestQueue {
* it will not be updated. You can find out whether this happened from the resulting
* {@link RequestOperationInfo} object.
*
- * @param {Request|Object} request Request object, or an Object to construct a Request from.
+ * @param {Request|Object} request {@link Request} object, or an object to construct a `Request` instance from.
* @param {Object} [options]
* @param {Boolean} [options.forefront=false] If `true`, the request will be added to the foremost position in the queue.
* @return {RequestOperationInfo}
diff --git a/test/request_queue.js b/test/request_queue.js
index 5bc875de1..0a8f65692 100644
--- a/test/request_queue.js
+++ b/test/request_queue.js
@@ -1,4 +1,4 @@
-import chai, { expect } from 'chai';
+import chai, { assert, expect } from 'chai';
import chaiAsPromised from 'chai-as-promised';
import _ from 'underscore';
import 'babel-polyfill';
@@ -83,6 +83,32 @@ describe('RequestQueue', () => {
expectDirEmpty(queueDir);
});
+ it('handles invalid URLs gracefully', async () => {
+ const queue = new RequestQueueLocal('my-queue-x', LOCAL_STORAGE_DIR);
+
+ try {
+ await queue.addRequest({ url: '' });
+ assert.fail('This should not happen');
+ } catch (e) {
+ expect(e.message).to.contain('The "url" option cannot be empty string');
+ }
+ await queue.addRequest(new Apify.Request({ url: 'something' }));
+ await Apify.utils.sleep(20);
+ await queue.addRequest({ url: 'dummy' });
+
+ const request2 = await queue.fetchNextRequest();
+ const request1 = await queue.fetchNextRequest();
+
+ expect(request1.url).to.be.eql('dummy');
+ expect(request1.uniqueKey).to.be.eql('dummy');
+
+ expect(request2.url).to.be.eql('something');
+ expect(request2.uniqueKey).to.be.eql('something');
+
+ expect(await queue.getRequest(request1.id)).to.be.eql(request1);
+ expect(await queue.getRequest(request2.id)).to.be.eql(request2);
+ });
+
it('supports forefront param in reclaimRequest()', async () => {
const queue = new RequestQueueLocal('my-queue-1', LOCAL_STORAGE_DIR);