Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
Updated comments, added new TODOs, added new (failing) unit tests
Jan Curn60f5197a28ad
diff --git a/src/crawler.js b/src/crawler.jsindex 2673e568e..6b0c54f9d 100644--- a/src/crawler.js+++ b/src/crawler.js@@ -2,6 +2,7 @@ import _ from 'underscore';import { newPromise, objectToQueryString } from './utils';+// TODO: please use requestPromise from utilsconst request = require('request-promise');const BASE_URL = 'https://api.apifier.com/v1';diff --git a/src/key-value-store.js b/src/key-value-store.jsindex 9c081ed2d..9aa766316 100644--- a/src/key-value-store.js+++ b/src/key-value-store.js@@ -114,8 +114,8 @@ const methods = {};/**- * Get or creates key value store and returns it's object.- * There are 3 possible initializations:+ * Gets or creates a key-value store and returns an object representing it.+ * There are 3 possible initializations (performed in this order):** 1) If `options.storeId` is provided then store with this id gets opened.* 2) If `options.name`, `options.token` and one of `options.ownerUserId` and `options.ownerUser`@@ -137,11 +137,11 @@ const createOrGetStore = (baseUrl, options) => {if (process.env.APIFY_ACT_RUN_ID) return methods.getStore(baseUrl, process.env.APIFY_ACT_RUN_ID);- throw new Error('Error: cannot identify store via storeId, credentials or ENV variable.');+ throw new Error('Cannot identify the key-value store to open.');};/**- * Opens key value store and returns object containing preconfigured methods for that store.+ * Opens a key-value store and returns object containing preconfigured methods for that store.* Possible options are:* - protocol (http/https)* - host@@ -154,7 +154,8 @@ const createOrGetStore = (baseUrl, options) => {* - name*/export default (options = {}) => {- if (!_.isObject(options)) throw new Error('Error: `options` must be object or empty.');+ // TODO: Why is this default export? Not sure this is a good idea, it might be confusing+ if (!_.isObject(options)) throw new Error('The `options` parameter must be an object.');// eslint-disable-next-line prefer-templateconst baseUrl = (options.protocol || 'https')diff --git a/test/utils.js b/test/utils.jsindex 979673279..7eb80fd6b 100644--- a/test/utils.js+++ b/test/utils.js@@ -42,13 +42,19 @@ describe('utils.objectToQueryString()', () => {expect(utils.objectToQueryString(obj)).to.be.eql('?val1=something&val2=else');});- it('should encode uri components', () => {+ it('should encode URI components', () => {const obj = {val1: 'something+else',- val2: 'ěščřžýá',+ 'val&?/': 'ěščřžýá',};- expect(utils.objectToQueryString(obj)).to.be.eql('?val1=something%2Belse&val2=%C4%9B%C5%A1%C4%8D%C5%99%C5%BE%C3%BD%C3%A1');+ expect(utils.objectToQueryString(obj)).to.be.eql('?val1=something%2Belse&val%26%3F%2F=%C4%9B%C5%A1%C4%8D%C5%99%C5%BE%C3%BD%C3%A1');+ });++ it('handles empty args', () => {+ expect(utils.objectToQueryString({})).to.be.eql('');+ expect(utils.objectToQueryString(null)).to.be.eql('');+ expect(utils.objectToQueryString(undefined)).to.be.eql('');});});