Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
fix: workaround for decodeURI (#1206)
Szymon Marczake248fb2490a9
diff --git a/package.json b/package.jsonindex 2536b99f8..96a97d15d 100644--- a/package.json+++ b/package.json@@ -1,6 +1,6 @@{"name": "apify",- "version": "2.1.0",+ "version": "2.1.1","description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.","engines": {"node": ">=15.10.0"@@ -59,7 +59,7 @@"browser-pool": "^2.0.2","cheerio": "1.0.0-rc.10","content-type": "^1.0.4",- "got-scraping": "^3.2.3",+ "got-scraping": "^3.2.5","htmlparser2": "^7.0.0","iconv-lite": "^0.6.3","jquery": "^3.6.0",diff --git a/src/utils_request.js b/src/utils_request.jsindex 3561d16ef..a0c814812 100644--- a/src/utils_request.js+++ b/src/utils_request.js@@ -170,9 +170,6 @@ export const requestAsBrowser = async (options = {}) => {},};- // Encode the URL if necessary- gotScrapingOptions.url = fixUrl(gotScrapingOptions.url);-// Order is importantnormalizePayloadOption(payload, gotScrapingOptions);normalizeJsonOption(json, gotScrapingOptions);@@ -221,53 +218,6 @@ export const requestAsBrowser = async (options = {}) => {});};-/**- * Fixes malformed URIs. Takes ~13s per 1M executions.- * @example fixUrl('https://example.com/%cf') => 'https://example.com/%EF%BF%BD'- * @example fixUrl('https://example.com/%xx') => 'https://example.com/%25xx'- * @example fixUrl('https://example.com/%0fexample%cc%0f') => 'https://example.com/%0Fexample%EF%BF%BD%0F'- * @see https://tc39.es/ecma262/multipage/global-object.html#sec-decodeuri-encodeduri- * @see https://en.wikipedia.org/wiki/Percent-encoding- * @param {string} url- * @private- */-function fixUrl(url) {- const hexChars = '0123456789abcdefABCDEF';- let chunks = '';-- let index = url.indexOf('%');- while (index !== -1 && index < url.length) {- const start = index;-- do {- const buffer = url.slice(index + 1, index + 3);-- if (hexChars.indexOf(buffer[0]) !== -1 && hexChars.indexOf(buffer[1]) !== -1) {- chunks += buffer;-- index += 3;- } else {- chunks += '25'; // %-- index += 1;- break;- }- } while (url[index] === '%');-- if (chunks.length) {- const encoded = encodeURIComponent(Buffer.from(chunks, 'hex').toString());- url = `${url.slice(0, start)}${encoded}${url.slice(index)}`;-- index = start + encoded.length;- chunks = '';- }-- index = url.indexOf('%', index);- }-- return url;-}-/*** `got` has a `body` option and 2 helpers, `json` and `form`, to provide specific bodies.* Those options are mutually exclusive. `requestAsBrowser` also supports `payload` asdiff --git a/test/crawlers/cheerio_crawler.test.js b/test/crawlers/cheerio_crawler.test.jsindex 23f564744..8cbb0367f 100644--- a/test/crawlers/cheerio_crawler.test.js+++ b/test/crawlers/cheerio_crawler.test.js@@ -284,8 +284,8 @@ describe('CheerioCrawler', () => {expect(failed).toHaveLength(0);expect(processed[0].loadedUrl).toBe(`http://${HOST}:${port}/mirror?q=abc`);- expect(processed[1].loadedUrl).toBe(`http://${HOST}:${port}/mirror?q=%25`);- expect(processed[2].loadedUrl).toBe(`http://${HOST}:${port}/mirror?q=%EF%BF%BD`);+ expect(processed[1].loadedUrl).toBe(`http://${HOST}:${port}/mirror?q=%`);+ expect(processed[2].loadedUrl).toBe(`http://${HOST}:${port}/mirror?q=%cf`);});test('postResponseFunction should work', async () => {diff --git a/test/utils_request.test.js b/test/utils_request.test.jsindex 93692e149..ead4dfc52 100644--- a/test/utils_request.test.js+++ b/test/utils_request.test.js@@ -110,7 +110,7 @@ describe('Apify.utils_request', () => {// [query, decoded, raw]['abc', 'abc', 'abc'],['%20', ' ', '%20'],- ['%cf', '�', '%EF%BF%BD'],+ ['%cf', '%cf', '%cf'],['helios-–-the-primordial-sun', 'helios-–-the-primordial-sun', 'helios-%E2%80%93-the-primordial-sun'],['helios-%E2%80%93-the-primordial-sun', 'helios-–-the-primordial-sun', 'helios-%E2%80%93-the-primordial-sun'],['%C3%A8----%C3%A9', 'è----é', '%C3%A8----%C3%A9'],