Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

Added request better and requestLikeBrowser functions with tests.

Petr Patek3ac8c76d7e71
diff --git a/package.json b/package.json
index 4a38685e9..fe93a51cd 100644
--- a/package.json
+++ b/package.json
@@ -56,6 +56,7 @@
"cheerio": "^1.0.0-rc.2",
"content-type": "^1.0.4",
"fs-extra": "^7.0.1",
+ "iltorb": "^2.4.2",
"jquery": "^3.3.1",
"mime": "^2.3.1",
"proxy-chain": "^0.2.5",
@@ -82,6 +83,7 @@
"eslint-plugin-jsx-a11y": "^6.1.2",
"eslint-plugin-promise": "^4.0.1",
"eslint-plugin-react": "^7.11.1",
+ "express": "^4.16.4",
"jsdoc": "^3.5.5",
"jsdoc-export-default-interop": "^0.3.1",
"mocha": "^5.2.0",
diff --git a/src/utils_request.js b/src/utils_request.js
index a5a420ee2..03bc158bf 100644
--- a/src/utils_request.js
+++ b/src/utils_request.js
@@ -1,3 +1,13 @@
+import rqst from 'request';
+import contentType from 'content-type';
+import util from 'util';
+import zlib from 'zlib';
+import * as url from 'url';
+import { decompressStream } from 'iltorb';
+
+const FIREFOX_MOBILE_USER_AGENT = 'Mozilla/5.0 (Android 9.0; Mobile; rv:66.0) Gecko/66.0 Firefox/66.0';
+const FIREFOX_DESKTOP_USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_3) AppleWebKit/605.1.15 (KHTML, like Gecko)' +
+ ' Version/11.1.1 Safari/605.1.15';
/**
* Sends a HTTP request and returns the response.
@@ -25,13 +35,7 @@
* they are defined in the `headers` parameter, in which case the function leaves them untouched.
* For example, even if you define `{ 'Content-Length': null }`, the function doesn't define
* the 'Content-Length' header and the request will not contain it (due to the `null` value).
- * @param [options.headers={}]
- * HTTP headers.
- * Note that the function generates several headers itself, unless
- * they are defined in the `headers` parameter, in which case the function leaves them untouched.
- * For example, even if you define `{ 'Content-Type': null }`, the function doesn't override
- * the 'Content-Type' header and the request contains none.
- * @param [options.body]
+ * @param [options.body]
* HTTP payload for PATCH, POST and PUT requests. Must be a `Buffer` or `String`.
* @param [options.followRedirect=true]
* Follow HTTP 3xx responses as redirects (default: true).
@@ -79,12 +83,107 @@
* `body` is a `String`, `Buffer` or `Object`, depending on the `encoding` and `json` options.
*/
export const requestBetter = async (options) => {
- // TODO: Implement this
- // TODO: You can use a lot of the code from CheerioCrawler that does abort()
- console.dir(options);
- throw new Error('Not implemented yet');
+ return new Promise((resolve, reject) => {
+ // Using the streaming API of Request to be able to
+ // handle the response based on headers receieved.
+ const defaultOptions = {
+ json: false,
+ gzip: false,
+ maxRedirects: 10,
+ followRedirect: true,
+ removeRefererHeader: false,
+ headers: {},
+ method: 'GET',
+ };
+
+ const opts = Object.assign(defaultOptions, options);
+ const method = opts.method.toLowerCase();
+ rqst[method](opts)
+ .on('error', err => reject(err))
+ .on('response', async (res) => {
+ // no need to catch invalid content header - it is already caught by request
+ const { type, encoding } = contentType.parse(res);
+
+ // 500 codes are handled as errors, requests will be retried.
+ const status = res.statusCode;
+ if (status >= 500) {
+ let body;
+ try {
+ body = await readStreamIntoString(res, encoding);
+ } catch (err) {
+ // Error in reading the body.
+ return reject(err);
+ }
+ // Errors are often sent as JSON, so attempt to parse them,
+ // despite Accept header being set to something different.
+ if (type === 'application/json') {
+ const errorResponse = JSON.parse(body);
+ let { message } = errorResponse;
+ if (!message) message = util.inspect(errorResponse, { depth: 1, maxArrayLength: 10 });
+ return reject(new Error(`${status} - ${message}`));
+ }
+ // It's not a JSON so it's probably some text. Get the first 100 chars of it.
+ return reject(new Error(`requestBetter: ${status} - Internal Server Error: ${body.substr(0, 100)}`));
+ }
+
+ // Content-Type is fine. Read the body and respond.
+ try {
+ res.body = await readStreamIntoString(res, encoding);
+ resolve(res);
+ } catch (err) {
+ // Error in reading the body.
+ reject(err);
+ }
+ });
+ });
};
+/**
+ * Flushes the provided stream into a Buffer and transforms
+ * it to a String using the provided encoding or utf-8 as default.
+ *
+ * If the stream data is compressed, decompresses it using
+ * the Content-Encoding header.
+ *
+ * @param {http.IncomingMessage} response
+ * @param {String} [encoding]
+ * @returns {Promise<String>}
+ * @private
+ */
+async function readStreamIntoString(response, encoding) { // eslint-disable-line class-methods-use-this
+ const compression = response.headers['content-encoding'];
+ let stream = response;
+ if (compression) {
+ let decompressor;
+ switch (compression) {
+ case 'gzip':
+ decompressor = zlib.createGunzip();
+ break;
+ case 'deflate':
+ decompressor = zlib.createInflate();
+ break;
+ case 'br':
+ decompressor = decompressStream();
+ break;
+ case undefined:
+ break;
+ default:
+ throw new Error(`requestBetter: Invalid Content-Encoding header. Expected gzip, deflate or br, but received: ${compression}`);
+ }
+ stream = response.pipe(decompressor);
+ }
+
+ return new Promise((resolve, reject) => {
+ const chunks = [];
+ stream
+ .on('data', chunk => chunks.push(chunk))
+ .on('error', err => reject(err))
+ .on('end', () => {
+ const buffer = Buffer.concat(chunks);
+ resolve(buffer.toString(encoding));
+ });
+ });
+}
/**
* Sends a HTTP request that looks like a request sent by a web browser,
* fully emulating browser's HTTP headers.
@@ -120,74 +219,46 @@ export const requestBetter = async (options) => {
* Node's [`http.IncomingMessage`](https://nodejs.org/api/http.html#http_class_http_incomingmessage) class,
* `body` is a `String`, `Buffer` or `Object`, depending on the `encoding` and `json` options.
*/
-export const requestLikeBrowser = (options) => {
- // TODO: Implement this
- // TODO: This function should use requestBetter(),
- // perhaps we could pass unknown options to the parent function
- console.dir(options);
- throw new Error('Not implemented yet');
-
- /* HERE'S SAMPLE CODE YOU CAN USE FOR START
+export const requestLikeBrowser = async (options) => {
+ const defaultOptions = {
+ countryCode: 'US',
+ languageCode: 'en',
+ headers: {},
+ method: 'GET',
+ useMobileVersion: false,
+ };
+ const opts = Object.assign(defaultOptions, options);
- const gzip = Promise.promisify(zlib.gzip, { context: zlib });
- const gunzip = Promise.promisify(zlib.gunzip, { context: zlib });
- const deflate = Promise.promisify(zlib.deflate, { context: zlib });
+ const parsedUrl = url.parse(opts.url);
- const reqOpts = {
- url,
- // Emulate Firefox HTTP headers
- // TODO: We should move this to apify-js or apify-shared-js
- headers: {
- Host: parsedUrlModified.host,
- 'User-Agent': useMobileVersion ? FIREFOX_MOBILE_USER_AGENT : FIREFOX_DESKTOP_USER_AGENT,
- Accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,*' + '/' + '*;q=0.8',
- 'Accept-Language': languageCode ? `${languageCode}-${countryCode},${languageCode};q=0.5` : '*', // TODO: get this from country !
- 'Accept-Encoding': 'gzip, deflate, br',
- DNT: '1',
- Connection: 'keep-alive',
- 'Upgrade-Insecure-Requests': '1',
- },
- // Return response as raw Buffer
- encoding: null,
+ const browserHeaders = {
+ Host: parsedUrl.host,
+ 'User-Agent': opts.useMobileVersion ? FIREFOX_MOBILE_USER_AGENT : FIREFOX_DESKTOP_USER_AGENT,
+ Accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
+ 'Accept-Language': `${opts.languageCode}-${opts.countryCode},${opts.languageCode};q=0.5`,
+ 'Accept-Encoding': 'gzip, deflate, br',
+ DNT: '1',
+ Connection: 'keep-alive',
+ 'Upgrade-Insecure-Requests': '1',
};
+ opts.headers = Object.assign(browserHeaders, opts.headers);
- const result = await utils.requestPromised(reqOpts, false);
- let body;
+ const response = await requestBetter(opts);
+ const receivedContentType = response.headers['content-type'];
- try {
- // eslint-disable-next-line prefer-destructuring
- body = result.body;
+ // Handle situations where the server explicitly states that
+ // it will not serve the resource as text/html by skipping.
+ if (response.statusCode === 406) {
+ throw new Error(`requestLikeBrowser: Resource ${options.url} is not available in HTML format. Skipping resource.`);
+ }
- // Decode response body
- const contentEncoding = result.response.headers['content-encoding'];
- switch (contentEncoding) {
- case 'br':
- body = await brotli.decompress(body);
- break;
- case 'gzip':
- body = await gunzip(body);
- break;
- case 'deflate':
- body = await deflate(body);
- break;
- case 'identity':
- case null:
- case undefined:
- break;
- default:
- throw new Error(`Received unexpected Content-Encoding: ${contentEncoding}`);
- }
- body = body.toString('utf8');
+ if (receivedContentType.toLowerCase() !== 'text/html; charset=utf-8') {
+ throw new Error(`Received unexpected Content-Type: ${receivedContentType}`);
+ }
- const { statusCode } = result;
- if (statusCode !== 200) {
- throw new Error(`Received HTTP error response status ${statusCode}`);
- }
+ const { body } = response;
- const contentType = result.response.headers['content-type'];
- if (contentType !== 'text/html; charset=UTF-8') {
- throw new Error(`Received unexpected Content-Type: ${contentType}`);
- }
+ if (!body) throw new Error('The response body is empty');
- if (!body) throw new Error('The response body is empty'); */
+ return { response, body };
};
diff --git a/test/_helper.js b/test/_helper.js
index b63f96e0f..36fa06636 100644
--- a/test/_helper.js
+++ b/test/_helper.js
@@ -39,3 +39,9 @@ export const expectDirNonEmpty = (dirPath) => {
const content = fs.readdirSync(dirPath);
expect(content).to.have.lengthOf.above(0);
};
+
+export const startExpressAppPromise = (app, port) => {
+ return new Promise((resolve) => {
+ const server = app.listen(port, () => resolve(server));
+ });
+};
diff --git a/test/utils_request.js b/test/utils_request.js
new file mode 100644
index 000000000..3541f18a3
--- /dev/null
+++ b/test/utils_request.js
@@ -0,0 +1,310 @@
+import { expect } from 'chai';
+import zlib from 'zlib';
+import express from 'express';
+import { compress } from 'iltorb';
+import { requestBetter, requestLikeBrowser } from '../build/utils_request';
+import { startExpressAppPromise } from './_helper';
+
+const CONTENT = 'CONTENT';
+const HOST = '127.0.0.1';
+const ERROR_BODY = 'CUSTOM_ERROR';
+const JSON_BODY = {
+ message: ERROR_BODY,
+};
+
+describe('Apify.utils_request', () => {
+ let port;
+ let server;
+ before(async () => {
+ const app = express();
+
+ app.get('/406', (req, res) => {
+ res.setHeader('content-type', 'text/html; charset=utf-8');
+ res.status(406);
+ res.send('CONTENT');
+ });
+
+ app.get('/invalidContentType', (req, res) => {
+ res.setHeader('content-type', 'application/json');
+ res.send('CONTENT');
+ });
+
+ app.get('/gzip', (req, res) => {
+ zlib.gzip(CONTENT, (error, result) => {
+ if (error) throw error;
+ res.setHeader('content-encoding', 'gzip');
+ res.send(result);
+ });
+ });
+
+ app.get('/deflate', (req, res) => {
+ // return zlib.compress(CONTENT);
+ zlib.deflate(CONTENT, (error, result) => {
+ if (error) throw error;
+ res.setHeader('content-encoding', 'deflate');
+ res.send(result);
+ });
+ });
+
+ app.get('/brotli', async (req, res) => {
+ // return zlib.compress(CONTENT);
+ const compressed = await compress(Buffer.from(CONTENT, 'utf8'));
+
+ res.setHeader('content-encoding', 'br');
+ res.send(compressed);
+ });
+
+ app.get('/500', (req, res) => {
+ res.status(500);
+ res.send(ERROR_BODY);
+ });
+
+ app.get('/500/invalidBody', async (req, res) => {
+ const compressed = await compress(Buffer.from(CONTENT, 'utf8'));
+
+ res.setHeader('content-encoding', 'deflate');
+ res.status(500);
+ res.send(compressed);
+ });
+
+ app.get('/500/json', async (req, res) => {
+ res.setHeader('Content-Type', 'application/json');
+ res.status(500);
+ res.send(JSON.stringify(JSON_BODY));
+ });
+
+ app.get('/invalidContentHeader', (req, res) => {
+ res.setHeader('Content-Type', 'non-existent-content-type');
+ res.send(CONTENT);
+ });
+
+ app.get('/invalidBody', async (req, res) => {
+ const compressed = await compress(Buffer.from(CONTENT, 'utf8'));
+
+ res.setHeader('content-encoding', 'deflate');
+ res.status(500);
+ res.send(compressed);
+ });
+
+ app.get('/empty', async (req, res) => {
+ res.setHeader('Content-Type', 'text/html; charset=utf-8');
+ res.send();
+ });
+
+ server = await startExpressAppPromise(app, 0);
+ port = server.address().port; //eslint-disable-line
+ });
+
+ after(() => {
+ server.close();
+ });
+
+ describe('Apify.requestBetter', async () => {
+ it('works', async () => {
+ const data = {
+ url: 'https://api.apify.com/v2/browser-info',
+ };
+ const response = await requestBetter(data);
+ expect(response.statusCode)
+ .to
+ .eql(200);
+ expect(response.request.headers).to.be.empty; // eslint-disable-line no-unused-expressions
+ });
+
+ it('decompress gzip', async () => {
+ const options = {
+ url: `http://${HOST}:${port}/gzip`,
+
+ };
+
+ const response = await requestBetter(options);
+ expect(response.body)
+ .to
+ .eql(CONTENT);
+ });
+
+ it('decompress deflate', async () => {
+ const options = {
+ url: `http://${HOST}:${port}/deflate`,
+
+ };
+
+ const response = await requestBetter(options);
+ expect(response.body)
+ .to
+ .eql(CONTENT);
+ });
+
+ it('decompress brotli', async () => {
+ const options = {
+ url: `http://${HOST}:${port}/brotli`,
+
+ };
+
+ const response = await requestBetter(options);
+ expect(response.body).to.eql(CONTENT);
+ });
+
+ it('it throws error for 500+', async () => {
+ const options = {
+ url: `http://${HOST}:${port}/500`,
+
+ };
+ let error;
+ try {
+ await requestBetter(options);
+ } catch (e) {
+ error = e;
+ }
+ expect(error.message).to.exist; // eslint-disable-line
+ expect(error.message.includes(ERROR_BODY)).to.be.eql(true);
+ });
+
+ it('it throws error when the body cannot be parsed and the code is 500', async () => {
+ const options = {
+ url: `http://${HOST}:${port}/500/invalidBody`,
+
+ };
+ let error;
+ try {
+ await requestBetter(options);
+ } catch (e) {
+ error = e;
+ }
+ expect(error.message).to.exist; // eslint-disable-line
+ });
+
+ it('it throws error when the body cannot be parsed', async () => {
+ const options = {
+ url: `http://${HOST}:${port}/invalidBody`,
+
+ };
+ let error;
+ try {
+ await requestBetter(options);
+ } catch (e) {
+ error = e;
+ }
+ expect(error.message).to.exist; // eslint-disable-line
+ });
+
+ it('it returns json when 500 even if content-type is different', async () => {
+ const options = {
+ url: `http://${HOST}:${port}/500/json`,
+
+ };
+ let error;
+ try {
+ await requestBetter(options);
+ } catch (e) {
+ error = e;
+ }
+ expect(error.message).to.exist; // eslint-disable-line
+ expect(error.message.includes(JSON_BODY.message)).to.be.eql(true);
+ });
+ });
+
+ describe('Apify.requestAsBrowser', async () => {
+ it('passes crunchbase.com non browser request blocking', async () => {
+ const data = {
+ url: 'https://www.crunchbase.com/',
+ html: true,
+ };
+ const { response } = await requestLikeBrowser(data);
+ expect(response.statusCode).to.eql(200);
+ });
+
+ it('it uses mobile user-agent whe mobile property is set to true ', async () => {
+ const data = {
+ url: 'https://www.crunchbase.com/',
+ html: true,
+ useMobileVersion: true,
+ };
+ const { response } = await requestLikeBrowser(data);
+ expect(response.statusCode).to.eql(200);
+ expect(response.request.headers['User-Agent']).to.be.eql('Mozilla/5.0 (Android 9.0; Mobile; rv:66.0) Gecko/66.0 Firefox/66.0');
+ });
+
+ it('it uses desktop user-agent by default ', async () => {
+ const data = {
+ url: 'https://www.crunchbase.com/',
+ html: true,
+ };
+ const { response } = await requestLikeBrowser(data);
+ expect(response.statusCode).to.eql(200);
+ expect(response.request.headers['User-Agent'])
+ .to
+ .be
+ .eql('Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_3) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/11.1.1 Safari/605.1.15');
+ });
+
+ it('it sets correct hosts', async () => {
+ const host = 'www.crunchbase.com';
+ const options = {
+ url: `https://${host}`,
+ };
+
+ const { response } = await requestLikeBrowser(options);
+
+ expect(response.statusCode).to.eql(200);
+ expect(response.request.headers.Host).to.be.eql(host);
+ });
+
+ it('it uses correct default language', async () => {
+ const host = 'www.crunchbase.com';
+ const options = {
+ url: `https://${host}`,
+ };
+
+ const { response } = await requestLikeBrowser(options);
+
+ expect(response.statusCode).to.eql(200);
+ expect(response.request.headers.Host).to.be.eql(host);
+ });
+
+ it('it throws error for 406', async () => {
+ const options = {
+ url: `http://${HOST}:${port}/406`,
+ };
+ let error;
+ try {
+ await requestLikeBrowser(options);
+ } catch (e) {
+ error = e;
+ }
+
+ expect(error).to.exist; //eslint-disable-line
+ expect(error.message).to.eql(`requestLikeBrowser: Resource ${options.url} is not available in HTML format. Skipping resource.`);
+ });
+
+ it('it throws for empty response body', async () => {
+ const options = {
+ url: `http://${HOST}:${port}/empty`,
+ };
+ let error;
+ try {
+ await requestLikeBrowser(options);
+ } catch (e) {
+ error = e;
+ }
+
+ expect(error).to.exist; //eslint-disable-line
+ expect(error.message).to.eql('The response body is empty');
+ });
+
+ it('it throws for other contentType then - text/html; charset=utf-8', async () => {
+ const options = {
+ url: `http://${HOST}:${port}/invalidContentType`,
+ };
+ let error;
+ try {
+ await requestLikeBrowser(options);
+ } catch (e) {
+ error = e;
+ }
+
+ expect(error).to.exist; //eslint-disable-line
+ expect(error.message.includes('Received unexpected Content-Type:')).to.eql(true);
+ });
+ });
+});