Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

fix: workaround for decodeURI (#1206)

Szymon Marczake248fb2490a9
diff --git a/package.json b/package.json
index 2536b99f8..96a97d15d 100644
--- a/package.json
+++ b/package.json
@@ -1,6 +1,6 @@
{
"name": "apify",
- "version": "2.1.0",
+ "version": "2.1.1",
"description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
"engines": {
"node": ">=15.10.0"
@@ -59,7 +59,7 @@
"browser-pool": "^2.0.2",
"cheerio": "1.0.0-rc.10",
"content-type": "^1.0.4",
- "got-scraping": "^3.2.3",
+ "got-scraping": "^3.2.5",
"htmlparser2": "^7.0.0",
"iconv-lite": "^0.6.3",
"jquery": "^3.6.0",
diff --git a/src/utils_request.js b/src/utils_request.js
index 3561d16ef..a0c814812 100644
--- a/src/utils_request.js
+++ b/src/utils_request.js
@@ -170,9 +170,6 @@ export const requestAsBrowser = async (options = {}) => {
},
};
- // Encode the URL if necessary
- gotScrapingOptions.url = fixUrl(gotScrapingOptions.url);
-
// Order is important
normalizePayloadOption(payload, gotScrapingOptions);
normalizeJsonOption(json, gotScrapingOptions);
@@ -221,53 +218,6 @@ export const requestAsBrowser = async (options = {}) => {
});
};
-/**
- * Fixes malformed URIs. Takes ~13s per 1M executions.
- * @example fixUrl('https://example.com/%cf') => 'https://example.com/%EF%BF%BD'
- * @example fixUrl('https://example.com/%xx') => 'https://example.com/%25xx'
- * @example fixUrl('https://example.com/%0fexample%cc%0f') => 'https://example.com/%0Fexample%EF%BF%BD%0F'
- * @see https://tc39.es/ecma262/multipage/global-object.html#sec-decodeuri-encodeduri
- * @see https://en.wikipedia.org/wiki/Percent-encoding
- * @param {string} url
- * @private
- */
-function fixUrl(url) {
- const hexChars = '0123456789abcdefABCDEF';
- let chunks = '';
-
- let index = url.indexOf('%');
- while (index !== -1 && index < url.length) {
- const start = index;
-
- do {
- const buffer = url.slice(index + 1, index + 3);
-
- if (hexChars.indexOf(buffer[0]) !== -1 && hexChars.indexOf(buffer[1]) !== -1) {
- chunks += buffer;
-
- index += 3;
- } else {
- chunks += '25'; // %
-
- index += 1;
- break;
- }
- } while (url[index] === '%');
-
- if (chunks.length) {
- const encoded = encodeURIComponent(Buffer.from(chunks, 'hex').toString());
- url = `${url.slice(0, start)}${encoded}${url.slice(index)}`;
-
- index = start + encoded.length;
- chunks = '';
- }
-
- index = url.indexOf('%', index);
- }
-
- return url;
-}
-
/**
* `got` has a `body` option and 2 helpers, `json` and `form`, to provide specific bodies.
* Those options are mutually exclusive. `requestAsBrowser` also supports `payload` as
diff --git a/test/crawlers/cheerio_crawler.test.js b/test/crawlers/cheerio_crawler.test.js
index 23f564744..8cbb0367f 100644
--- a/test/crawlers/cheerio_crawler.test.js
+++ b/test/crawlers/cheerio_crawler.test.js
@@ -284,8 +284,8 @@ describe('CheerioCrawler', () => {
expect(failed).toHaveLength(0);
expect(processed[0].loadedUrl).toBe(`http://${HOST}:${port}/mirror?q=abc`);
- expect(processed[1].loadedUrl).toBe(`http://${HOST}:${port}/mirror?q=%25`);
- expect(processed[2].loadedUrl).toBe(`http://${HOST}:${port}/mirror?q=%EF%BF%BD`);
+ expect(processed[1].loadedUrl).toBe(`http://${HOST}:${port}/mirror?q=%`);
+ expect(processed[2].loadedUrl).toBe(`http://${HOST}:${port}/mirror?q=%cf`);
});
test('postResponseFunction should work', async () => {
diff --git a/test/utils_request.test.js b/test/utils_request.test.js
index 93692e149..ead4dfc52 100644
--- a/test/utils_request.test.js
+++ b/test/utils_request.test.js
@@ -110,7 +110,7 @@ describe('Apify.utils_request', () => {
// [query, decoded, raw]
['abc', 'abc', 'abc'],
['%20', ' ', '%20'],
- ['%cf', '�', '%EF%BF%BD'],
+ ['%cf', '%cf', '%cf'],
['helios-–-the-primordial-sun', 'helios-–-the-primordial-sun', 'helios-%E2%80%93-the-primordial-sun'],
['helios-%E2%80%93-the-primordial-sun', 'helios-–-the-primordial-sun', 'helios-%E2%80%93-the-primordial-sun'],
['%C3%A8----%C3%A9', 'è----é', '%C3%A8----%C3%A9'],