Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

Updated comments, added new TODOs, added new (failing) unit tests

Jan Curn60f5197a28ad
diff --git a/src/crawler.js b/src/crawler.js
index 2673e568e..6b0c54f9d 100644
--- a/src/crawler.js
+++ b/src/crawler.js
@@ -2,6 +2,7 @@ import _ from 'underscore';
import { newPromise, objectToQueryString } from './utils';
+// TODO: please use requestPromise from utils
const request = require('request-promise');
const BASE_URL = 'https://api.apifier.com/v1';
diff --git a/src/key-value-store.js b/src/key-value-store.js
index 9c081ed2d..9aa766316 100644
--- a/src/key-value-store.js
+++ b/src/key-value-store.js
@@ -114,8 +114,8 @@ const methods = {
};
/**
- * Get or creates key value store and returns it's object.
- * There are 3 possible initializations:
+ * Gets or creates a key-value store and returns an object representing it.
+ * There are 3 possible initializations (performed in this order):
*
* 1) If `options.storeId` is provided then store with this id gets opened.
* 2) If `options.name`, `options.token` and one of `options.ownerUserId` and `options.ownerUser`
@@ -137,11 +137,11 @@ const createOrGetStore = (baseUrl, options) => {
if (process.env.APIFY_ACT_RUN_ID) return methods.getStore(baseUrl, process.env.APIFY_ACT_RUN_ID);
- throw new Error('Error: cannot identify store via storeId, credentials or ENV variable.');
+ throw new Error('Cannot identify the key-value store to open.');
};
/**
- * Opens key value store and returns object containing preconfigured methods for that store.
+ * Opens a key-value store and returns object containing preconfigured methods for that store.
* Possible options are:
* - protocol (http/https)
* - host
@@ -154,7 +154,8 @@ const createOrGetStore = (baseUrl, options) => {
* - name
*/
export default (options = {}) => {
- if (!_.isObject(options)) throw new Error('Error: `options` must be object or empty.');
+ // TODO: Why is this default export? Not sure this is a good idea, it might be confusing
+ if (!_.isObject(options)) throw new Error('The `options` parameter must be an object.');
// eslint-disable-next-line prefer-template
const baseUrl = (options.protocol || 'https')
diff --git a/test/utils.js b/test/utils.js
index 979673279..7eb80fd6b 100644
--- a/test/utils.js
+++ b/test/utils.js
@@ -42,13 +42,19 @@ describe('utils.objectToQueryString()', () => {
expect(utils.objectToQueryString(obj)).to.be.eql('?val1=something&val2=else');
});
- it('should encode uri components', () => {
+ it('should encode URI components', () => {
const obj = {
val1: 'something+else',
- val2: 'ěščřžýá',
+ 'val&?/': 'ěščřžýá',
};
- expect(utils.objectToQueryString(obj)).to.be.eql('?val1=something%2Belse&val2=%C4%9B%C5%A1%C4%8D%C5%99%C5%BE%C3%BD%C3%A1');
+ expect(utils.objectToQueryString(obj)).to.be.eql('?val1=something%2Belse&val%26%3F%2F=%C4%9B%C5%A1%C4%8D%C5%99%C5%BE%C3%BD%C3%A1');
+ });
+
+ it('handles empty args', () => {
+ expect(utils.objectToQueryString({})).to.be.eql('');
+ expect(utils.objectToQueryString(null)).to.be.eql('');
+ expect(utils.objectToQueryString(undefined)).to.be.eql('');
});
});