Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
fix: requestAsBrowser legacy options json/payload (#1029)
Ondra Urban3cedd4de6101
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex e926f2afc..0bebcb6e5 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -1,3 +1,7 @@+1.2.1 / 2021/05/14+====================+- Fix `requestAsBrowser` behavior with various combinations of `json`, `payload` legacy options.+1.2.0 / 2021/05/10====================This release brings the long awaited HTTP2 capabilities to `requestAsBrowser`. It could make HTTP2 requests even before, but it was not very helpful in making browser-like ones. This is very important for disguising as a browser and reduction in the number of blocked requests. `requestAsBrowser` now uses [`got-scraping`](https://github.com/apify/got-scraping).diff --git a/package.json b/package.jsonindex e6999c701..b3ad06a2d 100644--- a/package.json+++ b/package.json@@ -1,6 +1,6 @@{"name": "apify",- "version": "1.2.0",+ "version": "1.2.1","description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.","engines": {"node": ">=10.17.0"diff --git a/src/utils_request.js b/src/utils_request.jsindex 82e944ad1..8af6121fe 100644--- a/src/utils_request.js+++ b/src/utils_request.js@@ -232,7 +232,7 @@ function areBodyOptionsCompatible(requestAsBrowserOptions) {const { payload, json, body, form } = requestAsBrowserOptions;// A boolean is old requestAsBrowser interface and not a real "body"// See the normalizeJsonOption function.- const jsonBody = typeof json !== 'boolean' ? undefined : json;+ const jsonBody = typeof json === 'boolean' ? undefined : json;const possibleOpts = [payload, jsonBody, body, form];const usedOpts = possibleOpts.filter((opt) => opt !== undefined);@@ -263,14 +263,14 @@ function normalizePayloadOption(payload, gotScrapingOptions) {*/function normalizeJsonOption(json, gotScrapingOptions) {// If it's a boolean, then it's the old requestAsBrowser API.- if (typeof json === 'boolean') {+ // If it's true, it means the user expects a JSON response.+ if (json === true) {gotScrapingOptions.responseType = 'json';gotScrapingOptions.ciphers = undefined;- // If there is a body, we move it under `json` to get the automatic- // 'content-type' header injection.- if (gotScrapingOptions.body !== undefined) gotScrapingOptions.json = gotScrapingOptions.body;+ } else if (json === false) {+ // Do nothing, it means the user expects something else than JSON.} else {- // If it's something else, we let `got` handle it.+ // If it's something else, we let `got` handle it as a request body.gotScrapingOptions.json = json;}}diff --git a/test/utils_request.test.js b/test/utils_request.test.jsindex 5dc4e45c8..79ad67a9d 100644--- a/test/utils_request.test.js+++ b/test/utils_request.test.js@@ -5,6 +5,7 @@ import {import { startExpressAppPromise } from './_helper';const CONTENT = 'CONTENT';+const JSON_CONTENT = JSON.stringify({ content: CONTENT });const HOSTNAME = '127.0.0.1';describe('Apify.utils_request', () => {@@ -27,9 +28,14 @@ describe('Apify.utils_request', () => {res.send(JSON.stringify(req.rawHeaders));});- app.get('/invalidContentType', (req, res) => {- res.setHeader('content-type', 'application/json');- res.send(CONTENT);+ app.get('/json', (req, res) => {+ res.setHeader('content-type', 'application/json; charset=utf-8');+ res.send(JSON_CONTENT);+ });++ app.post('/jsonEcho', (req, res) => {+ res.setHeader('content-type', 'application/json; charset=utf-8');+ req.pipe(res);});app.get('/invalidContentHeader', (req, res) => {@@ -256,5 +262,109 @@ describe('Apify.utils_request', () => {const body = chunks.join();expect(body).toBe('TEST');});++ describe('old requestAsBrowser API', () => {+ test('correctly handles json: true without payload', async () => {+ const host = `${HOSTNAME}:${port}`;+ const options = {+ url: `http://${host}/json`,+ json: true,+ };++ const response = await requestAsBrowser(options);+ expect(response.statusCode).toBe(200);+ const parsedContent = JSON.parse(JSON_CONTENT);+ expect(response.body).toEqual(parsedContent);+ });++ test('correctly handles json: true with payload', async () => {+ const host = `${HOSTNAME}:${port}`;+ const body = { hello: 'world' };+ const options = {+ url: `http://${host}/jsonEcho`,+ method: 'POST',+ json: true,+ headers: { 'content-type': 'application/json' },+ payload: JSON.stringify(body),+ };++ const response = await requestAsBrowser(options);+ expect(response.statusCode).toBe(200);+ expect(response.body).toEqual(body);+ });++ test('correctly handles json: false without payload', async () => {+ const host = `${HOSTNAME}:${port}`;+ const options = {+ url: `http://${host}/json`,+ json: false,+ };++ const response = await requestAsBrowser(options);+ expect(response.statusCode).toBe(200);+ expect(response.body).toEqual(JSON_CONTENT);+ });++ test('correctly handles json: false with payload', async () => {+ const host = `${HOSTNAME}:${port}`;+ const payload = JSON.stringify({ hello: 'world' });+ const options = {+ url: `http://${host}/jsonEcho`,+ json: false,+ method: 'POST',+ headers: { 'content-type': 'application/json' },+ payload,+ };++ const response = await requestAsBrowser(options);+ expect(response.statusCode).toBe(200);+ expect(response.body).toEqual(payload);+ });++ test('correctly handles json: undefined without payload', async () => {+ const host = `${HOSTNAME}:${port}`;+ const options = {+ url: `http://${host}/json`,+ };++ const response = await requestAsBrowser(options);+ expect(response.statusCode).toBe(200);+ expect(response.body).toEqual(JSON_CONTENT);+ });++ test('correctly handles json: undefined with payload', async () => {+ const host = `${HOSTNAME}:${port}`;+ const payload = JSON.stringify({ hello: 'world' });+ const options = {+ url: `http://${host}/jsonEcho`,+ method: 'POST',+ headers: { 'content-type': 'application/json' },+ payload,+ };++ const response = await requestAsBrowser(options);+ expect(response.statusCode).toBe(200);+ expect(response.body).toEqual(payload);+ });++ test('correctly handles json: object', async () => {+ // This is a check if we did not break the got API.+ // got needs responseType: 'json' to automatically parse the response.+ // Here we test that it correctly sends the request body,+ // but the response stays unparsed.++ const host = `${HOSTNAME}:${port}`;+ const json = { foo: 'bar' };+ const options = {+ url: `http://${host}/jsonEcho`,+ method: 'POST',+ json,+ };++ const response = await requestAsBrowser(options);+ expect(response.statusCode).toBe(200);+ expect(response.body).toEqual(JSON.stringify(json));+ });+ });});});