Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

Rewrote to use the http request.

Petr Patek4eca61426dbb
diff --git a/package.json b/package.json
index 27dae7931..1b48bc078 100644
--- a/package.json
+++ b/package.json
@@ -57,14 +57,14 @@
"content-type": "^1.0.4",
"express": "^4.16.4",
"fs-extra": "^7.0.1",
- "iltorb": "^2.4.2",
+ "iltorb": "^2.4.3",
"jquery": "^3.3.1",
"mime": "^2.3.1",
"proxy-chain": "^0.2.5",
- "socket.io": "^2.2.0",
"request": "^2.88.0",
"request-promise-native": "^1.0.5",
"rimraf": "^2.6.2",
+ "socket.io": "^2.2.0",
"underscore": "^1.9.1",
"ws": "^6.1.2",
"xregexp": "^4.2.4"
diff --git a/src/utils_request.js b/src/utils_request.js
index 58566b0b7..0690b0b3f 100644
--- a/src/utils_request.js
+++ b/src/utils_request.js
@@ -1,274 +1,21 @@
-import rqst from 'request';
import contentType from 'content-type';
-import util from 'util';
-import zlib from 'zlib';
import * as url from 'url';
-import { decompressStream } from 'iltorb';
import _ from 'underscore';
-import log from 'apify-shared/log';
+import httpRequest from 'http-request';
const FIREFOX_MOBILE_USER_AGENT = 'Mozilla/5.0 (Android 9.0; Mobile; rv:66.0) Gecko/66.0 Firefox/66.0';
const FIREFOX_DESKTOP_USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_3) AppleWebKit/605.1.15 (KHTML, like Gecko)'
+ ' Version/11.1.1 Safari/605.1.15';
-const REQUEST_EXTENDED_DEFAULT_OPTIONS = {
- json: false,
- gzip: true,
- maxRedirects: 10,
- followRedirect: true,
- removeRefererHeader: false,
- headers: {},
- method: 'GET',
- throwOnHttpError: false,
- abortFunction: () => false,
-};
-
-const REQUEST_LIKE_BROWSER_DEFAULT_OPTIONS = {
+export const REQUEST_AS_BROWSER_DEFAULT_OPTIONS = {
countryCode: 'US',
languageCode: 'en',
headers: {},
method: 'GET',
useMobileVersion: false,
+ useBrotli: true,
+ json: false,
};
-const TRUNCATED_ERROR_CHARS = 100;
-
-let tunnelAgentExceptionListener;
-/**
- * Sends a HTTP request and returns the response.
- * The function has similar functionality and options as the [request](https://www.npmjs.com/package/request) NPM package,
- * but it brings several additional improvements and fixes:
- *
- * - It support not only Gzip compression, but also Brotli and Deflate. To activate this feature,
- * simply add `Accept-Encoding: gzip, deflate, br` to `options.headers` (or a combination).
- * - Enables abortion of the request based on the response headers, before the data is downloaded.
- * See `options.abortFunction` parameter.
- * - SSL connections over proxy do not leak sockets in CLOSE_WAIT state (https://github.com/request/request/issues/2440)
- * - Gzip implementation doesn't fail (https://github.com/apifytech/apify-js/issues/266)
- * - There is no tunnel-agent AssertionError (https://github.com/request/tunnel-agent/issues/20)
- *
- * NOTE: Most of the options below are simply copied from NPM request. Perhaps we don't need to copy
- * them here and can just pass them down. Well, we can decide later.
- *
- * @param options.url
- * URL of the target endpoint. Supports both HTTP and HTTPS schemes.
- * @param [options.method=GET]
- * HTTP method.
- * @param [options.headers={}]
- * HTTP headers.
- * Note that the function generates several headers itself, unless
- * they are defined in the `headers` parameter, in which case the function leaves them untouched.
- * For example, even if you define `{ 'Content-Length': null }`, the function doesn't define
- * the 'Content-Length' header and the request will not contain it (due to the `null` value).
- * @param [options.body]
- * HTTP payload for PATCH, POST and PUT requests. Must be a `Buffer` or `String`.
- * @param [options.followRedirect=true]
- * Follow HTTP 3xx responses as redirects (default: true).
- * OPTIONALLY: This property can also be implemented as function which gets response object as
- * a single argument and should return `true` if redirects should continue or `false` otherwise.
- * @param [options.maxRedirects=10]
- * The maximum number of redirects to follow.
- * @param [options.removeRefererHeader=false]
- * Removes the referer header when a redirect happens.
- * If `true`, referer header set in the initial request is preserved during redirect chain.
- * @param [options.encoding]
- * Encoding to be used on `setEncoding` of response data.
- * If `null`, the body is returned as a `Buffer`.
- * Anything else (including the default value of undefined) will be passed as the encoding parameter to `toString()`,
- * (meaning this is effectively utf8 by default).
- * (Note: if you expect binary data, you should set encoding: null.)
- * @param [options.gzip=true]
- * If `true`, the function adds an `Accept-Encoding: gzip` header to request compressed content encodings from the server
- * (if not already present) and decode supported content encodings in the response.
- * Note that you can achieve the same effect by adding the `Accept-Encoding: gzip` header directly to `options.headers`,
- * similarly as `deflate` as `br` encodings.
- * @param [options.json=false]
- * Sets body to JSON representation of value and adds `Content-type: application/json` header.
- * Additionally, parses the response body as JSON, i.e. the `body` property of the returned object
- * is the result of `JSON.parse()`. Throws an error if response cannot be parsed as JSON.
- * @param [options.timeout]
- * Integer containing the number of milliseconds to wait for a server to send
- * response headers (and start the response body) before aborting the request.
- * Note that if the underlying TCP connection cannot be established, the OS-wide
- * TCP connection timeout will overrule the timeout option (the default in Linux can be anywhere from 20-120 seconds).
- * @param [options.proxy]
- * An HTTP proxy to be used. Supports proxy authentication with Basic Auth.
- * @param [options.strictSSL=true]
- * If `true`, requires SSL/TLS certificates to be valid.
- * @param [options.abortFunction]
- * A function that determines whether the request should be aborted. It is called when the server
- * responds with the HTTP headers, but before the actual data is downloaded.
- * The function receives a single argument - an instance of Node's
- * [`http.IncomingMessage`](https://nodejs.org/api/http.html#http_class_http_incomingmessage)
- * class and it should return `true` if request should be aborted, or `false` otherwise.
- * @param [options.throwOnHttpError=false]
- * If set to true function throws and error on 4XX and 5XX response codes.
- * @return {http.IncomingMessage}
- * @memberOf utils
- * @name requestExtended
- */
-export const requestExtended = async (options) => {
- suppressTunnelAgentAssertError();
-
- return new Promise((resolve, reject) => {
- const opts = _.defaults({}, options, REQUEST_EXTENDED_DEFAULT_OPTIONS);
-
- const {
- abortFunction,
- throwOnHttpError,
- } = opts;
- const method = opts.method.toLowerCase();
-
- // Using the streaming API of Request to be able to
- const request = rqst[method](opts);
- request
- .on('error', err => reject(err))
- .on('response', async (res) => {
- let shouldAbort;
-
- try {
- shouldAbort = abortFunction(res);
- } catch (e) {
- reject(e);
- }
-
- if (shouldAbort) {
- request.abort();
- res.destroy();
-
- return reject(new Error(`utils.requestBetter: Request for ${opts.url} aborted due to abortFunction`));
- }
-
- let cType;
- try {
- cType = contentType.parse(res);
- } catch (err) {
- res.destroy();
- // No reason to parse the body if the Content-Type header is invalid.
- return reject(new Error(`utils.requestBetter: Invalid Content-Type header for URL: ${request.url}`));
- }
-
- const { encoding } = cType;
-
- // 5XX and 4XX codes are handled as errors, requests will be retried.
- const status = res.statusCode;
-
- if (status >= 400 && throwOnHttpError) {
- const error = await getMoreErrorInfo(res, cType);
- return reject(error);
- }
-
- // Content-Type is fine. Read the body and respond.
- try {
- res.body = await readStreamIntoString(res, encoding);
- resolve(res);
- } catch (err) {
- // Error in reading the body.
- reject(err);
- }
- });
- });
-};
-
-/**
- * Gets more info about the error.
- * Errors are often sent as JSON, so attempt to parse them, despite Accept header being set to something different.
- * @param {http.IncomingMessage} response
- * @param {Object} cType
- * @param {String} cType.type
- * @param {String} cType.encoding
- * @returns {Promise<Error>}
- * @ignore
- */
-async function getMoreErrorInfo(response, cType) {
- const { type, encoding } = cType;
- const { status } = response;
- let body;
- try {
- body = await readStreamIntoString(response, encoding);
- } catch (err) {
- // Error in reading the body.
- return err;
- }
-
- if (type === 'application/json') {
- let errorResponse;
- let message;
- try {
- errorResponse = JSON.parse(body);
- message = errorResponse.message; // eslint-disable-line
- if (!message) {
- message = util.inspect(errorResponse, {
- depth: 1,
- maxArrayLength: 10,
- });
- }
- } catch (e) {
- message = `${body.substr(0, TRUNCATED_ERROR_CHARS)}...`;
- }
-
- return new Error(`${status} - ${message}`);
- }
- // It's not a JSON so it's probably some text. Get the first 100 chars of it.
- return new Error(`utils.requestBetter: ${status} - Internal Server Error: ${body.substr(0, TRUNCATED_ERROR_CHARS)}...`);
-}
-
-/**
- * Flushes the provided stream into a Buffer and transforms
- * it to a String using the provided encoding or utf-8 as default.
- *
- *
- * @param {http.IncomingMessage} response
- * @param {String} [encoding]
- * @returns {Promise<String>}
- * @ignore
- */
-async function readStreamIntoString(response, encoding) { // eslint-disable-line class-methods-use-this
- const stream = decompress(response);
-
- return new Promise((resolve, reject) => {
- const chunks = [];
- stream
- .on('data', chunk => chunks.push(chunk))
- .on('error', err => reject(err))
- .on('end', () => {
- const buffer = Buffer.concat(chunks);
- resolve(buffer.toString(encoding));
- });
- });
-}
-
-/**
- * Gets decompressed response from
- * If the stream data is compressed, decompresses it using the Content-Encoding header.
- * @param {http.IncomingMessage} response
- * @returns {http.IncomingMessage|Stream} - Decompressed response
- * @ignore
- */
-function decompress(response) {
- const compression = response.headers['content-encoding'];
- let stream = response;
- if (compression) {
- let decompressor;
- switch (compression) {
- case 'gzip':
- decompressor = zlib.createGunzip();
- break;
- case 'deflate':
- decompressor = zlib.createInflate();
- break;
- case 'br':
- decompressor = decompressStream();
- break;
- case undefined:
- break;
- default:
- throw new Error(`requestBetter: Invalid Content-Encoding header. Expected gzip, deflate or br, but received: ${compression}`);
- }
- stream = response.pipe(decompressor);
- }
- return stream;
-}
-
/**
* Sends a HTTP request that looks like a request sent by a web browser,
* fully emulating browser's HTTP headers.
@@ -302,10 +49,10 @@ function decompress(response) {
*
* @return {http.IncomingMessage}
* @memberOf utils
- * @name requestLikeBrowser
+ * @name requestAsBrowser
*/
-export const requestLikeBrowser = async (options) => {
- const opts = _.defaults({}, options, REQUEST_LIKE_BROWSER_DEFAULT_OPTIONS);
+export const requestAsBrowser = async (options) => {
+ const opts = _.defaults({}, options, REQUEST_AS_BROWSER_DEFAULT_OPTIONS);
const parsedUrl = url.parse(opts.url);
const abortFunction = (res) => {
@@ -326,39 +73,5 @@ export const requestLikeBrowser = async (options) => {
opts.headers = _.defaults({}, opts.headers, browserHeaders);
opts.abortFunction = abortFunction;
- return requestExtended(opts);
+ return httpRequest(opts);
};
-
-/**
- * The handler this function attaches overcomes a long standing bug in
- * the tunnel-agent NPM package that is used by the Request package internally.
- * The package throws an assertion error in a callback scope that cannot be
- * caught by conventional means and shuts down the running process.
- * @ignore
- */
-function suppressTunnelAgentAssertError() {
- // Only set the handler if it's not already set.
- if (tunnelAgentExceptionListener) return;
- tunnelAgentExceptionListener = (err) => {
- try {
- const code = err.code === 'ERR_ASSERTION';
- const name = err.name === 'AssertionError [ERR_ASSERTION]';
- const operator = err.operator === '==';
- const value = err.expected === 0;
- const stack = err.stack.includes('/tunnel-agent/index.js');
- // If this passes, we can be reasonably sure that it's
- // the right error from tunnel-agent.
- if (code && name && operator && value && stack) {
- log.error('utils.requestExtended: Tunnel-Agent assertion error intercepted.');
- return;
- }
- } catch (caughtError) {
- // Catch any exception resulting from the duck-typing
- // check. It only means that the error is not the one
- // we're looking for.
- }
- // Rethrow the original error if it's not a match.
- throw err;
- };
- process.on('uncaughtException', tunnelAgentExceptionListener);
-}
diff --git a/test/utils_request.js b/test/utils_request.js
index d9b775d86..a88120d5c 100644
--- a/test/utils_request.js
+++ b/test/utils_request.js
@@ -1,18 +1,12 @@
import { expect } from 'chai';
-import zlib from 'zlib';
import express from 'express';
import { compress } from 'iltorb';
-import sinon from 'sinon';
-import { requestExtended, requestLikeBrowser } from '../build/utils_request';
+import { requestAsBrowser, REQUEST_AS_BROWSER_DEFAULT_OPTIONS } from '../build/utils_request';
import { startExpressAppPromise } from './_helper';
-import Apify from '../build';
const CONTENT = 'CONTENT';
const HOST = '127.0.0.1';
const ERROR_BODY = 'CUSTOM_ERROR';
-const JSON_BODY = {
- message: ERROR_BODY,
-};
describe('Apify.utils_request', () => {
let mochaListener;
@@ -30,7 +24,7 @@ describe('Apify.utils_request', () => {
});
app.get('/echo', (req, res) => {
- res.send(CONTENT);
+ res.send(JSON.stringify(req.headers));
});
app.get('/invalidContentType', (req, res) => {
@@ -38,50 +32,6 @@ describe('Apify.utils_request', () => {
res.send(CONTENT);
});
- app.get('/gzip', (req, res) => {
- zlib.gzip(CONTENT, (error, result) => {
- if (error) throw error;
- res.setHeader('content-encoding', 'gzip');
- res.send(result);
- });
- });
-
- app.get('/deflate', (req, res) => {
- // return zlib.compress(CONTENT);
- zlib.deflate(CONTENT, (error, result) => {
- if (error) throw error;
- res.setHeader('content-encoding', 'deflate');
- res.send(result);
- });
- });
-
- app.get('/brotli', async (req, res) => {
- // return zlib.compress(CONTENT);
- const compressed = await compress(Buffer.from(CONTENT, 'utf8'));
-
- res.setHeader('content-encoding', 'br');
- res.send(compressed);
- });
-
- app.get('/500', (req, res) => {
- res.status(500);
- res.send(ERROR_BODY);
- });
-
- app.get('/500/invalidBody', async (req, res) => {
- const compressed = await compress(Buffer.from(CONTENT, 'utf8'));
-
- res.setHeader('content-encoding', 'deflate');
- res.status(500);
- res.send(compressed);
- });
-
- app.get('/500/json', async (req, res) => {
- res.setHeader('Content-Type', 'application/json');
- res.status(500);
- res.send(JSON.stringify(JSON_BODY));
- });
-
app.get('/invalidContentHeader', (req, res) => {
res.setHeader('Content-Type', 'non-existent-content-type');
res.send(CONTENT);
@@ -104,288 +54,76 @@ describe('Apify.utils_request', () => {
port = server.address().port; //eslint-disable-line
});
- after(() => {
- server.close();
- process.on('uncaughtException', mochaListener);
- });
-
- describe('Apify.requestExtended', async () => {
- it('works', async () => {
- const data = {
- url: 'https://api.apify.com/v2/browser-info',
- };
- const response = await requestExtended(data);
- expect(response.statusCode)
- .to
- .eql(200);
- });
-
- it('passes response to abortFunction and aborts request', async () => {
- let constructorName;
- let aborted = false;
- const data = {
- url: `http://${HOST}:${port}/empty`,
- abortFunction: (response) => {
- constructorName = response.constructor.name;
- response.request.on('abort', () => {
- aborted = true;
- });
- return true;
- },
-
- };
-
- let error;
- try {
- await requestExtended(data);
- } catch (e) {
- error = e;
- }
-
- expect(constructorName).to.be.eql('IncomingMessage');
- expect(error.message).to.eql(`utils.requestBetter: Request for ${data.url} aborted due to abortFunction`);
- expect(aborted).to.be.eql(true);
- });
-
- it('should suppress tunnel-agent errors', async () => {
- const throwNextTick = (err) => {
- process.nextTick(() => {
- throw err;
- });
- };
- const abortFunction = async () => {
- const err = new Error();
- err.code = 'ERR_ASSERTION';
- err.name = 'AssertionError [ERR_ASSERTION]';
- err.operator = '==';
- err.expected = 0;
- err.stack = ('xxx/tunnel-agent/index.js/yyyy');
- throwNextTick(err);
- // will never resolve
- await new Promise(() => {
- });
- };
- const data = {
- url: `http://${HOST}:${port}/gzip`,
- abortFunction,
-
- };
- let message;
- const stubbedErrorLog = sinon
- .stub(Apify.utils.log, 'error')
- .callsFake(async (msg) => {
- message = msg;
- });
- let error;
- try {
- await requestExtended(data);
- } catch (e) {
- error = e;
- }
-
- expect(message).to.be.eql('utils.requestExtended: Tunnel-Agent assertion error intercepted.');
- expect(error).to.exist //eslint-disable-line
-
- stubbedErrorLog.restore();
- });
-
- it('it does not aborts request when aborts function returns false', async () => {
- let aborted = false;
- const data = {
- url: `http://${HOST}:${port}/gzip`,
- abortFunction: (response) => {
- response.on('aborted', () => {
- aborted = true;
- });
- return false;
- },
-
- };
- await requestExtended(data);
- expect(aborted).to.be.eql(false);
- });
-
- it('decompress gzip', async () => {
- const options = {
- url: `http://${HOST}:${port}/gzip`,
-
- };
-
- const response = await requestExtended(options);
- expect(response.body)
- .to
- .eql(CONTENT);
- });
-
- it('decompress deflate', async () => {
- const options = {
- url: `http://${HOST}:${port}/deflate`,
-
- };
-
- const response = await requestExtended(options);
- expect(response.body)
- .to
- .eql(CONTENT);
- });
-
- it('decompress brotli', async () => {
- const options = {
- url: `http://${HOST}:${port}/brotli`,
-
- };
-
- const response = await requestExtended(options);
- expect(response.body).to.eql(CONTENT);
- });
-
- it('it does not throw error for 400+ error codes when throwOnHttpError is false', async () => {
- const options = {
- url: `http://${HOST}:${port}/500`,
-
- };
- let error;
- try {
- await requestExtended(options);
- } catch (e) {
- error = e;
- }
- expect(error).to.be.undefined; // eslint-disable-line
- });
-
- it('it does not throw error for 400+ error codes when throwOnHttpError is true', async () => {
- const options = {
- url: `http://${HOST}:${port}/500`,
- throwOnHttpError: true,
-
- };
- let error;
- try {
- await requestExtended(options);
- } catch (e) {
- error = e;
- }
- expect(error.message).to.exist; // eslint-disable-line
- expect(error.message.includes(ERROR_BODY)).to.be.eql(true);
- });
-
- it('it throws error when the body cannot be parsed and the code is 500 when throwOnHttpError is true', async () => {
- const options = {
- url: `http://${HOST}:${port}/500/invalidBody`,
- throwOnHttpError: true,
-
- };
- let error;
- try {
- await requestExtended(options);
- } catch (e) {
- error = e;
- }
- expect(error.message).to.exist; // eslint-disable-line
- });
-
- it('it throws error when the body cannot be parsed', async () => {
- const options = {
- url: `http://${HOST}:${port}/invalidBody`,
-
- };
- let error;
- try {
- await requestExtended(options);
- } catch (e) {
- error = e;
- }
- expect(error.message).to.exist; // eslint-disable-line
- });
-
- it('it returns json when 500 even if content-type is different, throwOnHttpError is true ', async () => {
- const options = {
- url: `http://${HOST}:${port}/500/json`,
- throwOnHttpError: true,
-
- };
- let error;
- try {
- await requestExtended(options);
- } catch (e) {
- error = e;
- }
- expect(error.message).to.exist; // eslint-disable-line
- expect(error.message.includes(JSON_BODY.message)).to.be.eql(true);
- });
- });
-
describe('Apify.requestAsBrowser', async () => {
it('it uses mobile user-agent whe mobile property is set to true ', async () => {
const data = {
url: `http://${HOST}:${port}/echo`,
- html: true,
useMobileVersion: true,
};
- const response = await requestLikeBrowser(data);
+ const response = await requestAsBrowser(data);
expect(response.statusCode).to.eql(200);
- expect(response.request.headers['User-Agent']).to.be.eql('Mozilla/5.0 (Android 9.0; Mobile; rv:66.0) Gecko/66.0 Firefox/66.0');
+ expect(JSON.parse(response.body)['user-agent']).to.be.eql('Mozilla/5.0 (Android 9.0; Mobile; rv:66.0) Gecko/66.0 Firefox/66.0');
});
- it('it uses desktop user-agent by default ', async () => {
+ it('uses desktop user-agent by default ', async () => {
const data = {
url: `http://${HOST}:${port}/echo`,
- html: true,
};
- const response = await requestLikeBrowser(data);
+ const response = await requestAsBrowser(data);
expect(response.statusCode).to.eql(200);
- expect(response.request.headers['User-Agent'])
+ expect(JSON.parse(response.body)['user-agent'])
.to
.be
.eql('Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_3) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/11.1.1 Safari/605.1.15');
});
- it('it sets correct hosts', async () => {
+ it('sets correct hosts', async () => {
const host = `${HOST}:${port}`;
const options = {
url: `http://${host}/echo`,
};
- const response = await requestLikeBrowser(options);
+ const response = await requestAsBrowser(options);
expect(response.statusCode).to.eql(200);
- expect(response.request.headers.Host).to.be.eql(host);
+ expect(JSON.parse(response.body).host).to.be.eql(host);
});
- it('it uses correct default language', async () => {
+ it('uses correct default language', async () => {
+ const { languageCode, countryCode } = REQUEST_AS_BROWSER_DEFAULT_OPTIONS;
const host = `${HOST}:${port}`;
const options = {
url: `http://${host}/echo`,
};
- const response = await requestLikeBrowser(options);
+ const response = await requestAsBrowser(options);
expect(response.statusCode).to.eql(200);
- expect(response.request.headers.Host).to.be.eql(host);
+ expect(JSON.parse(response.body)['accept-language']).to.be.eql(`${languageCode}-${countryCode},${languageCode};q=0.5`);
});
- it('it throws error for 406', async () => {
+ it('throws error for 406', async () => {
const options = {
url: `http://${HOST}:${port}/406`,
};
let error;
try {
- await requestLikeBrowser(options);
+ await requestAsBrowser(options);
} catch (e) {
error = e;
}
expect(error).to.exist; //eslint-disable-line
- expect(error.message).to.eql(`utils.requestBetter: Request for ${options.url} aborted due to abortFunction`);
+ expect(error.message).to.eql(`Request for ${options.url} aborted due to abortFunction`);
});
- it('it does not throw for empty response body', async () => {
+ it('does not throw for empty response body', async () => {
const options = {
url: `http://${HOST}:${port}/empty`,
};
let error;
try {
- await requestLikeBrowser(options);
+ await requestAsBrowser(options);
} catch (e) {
error = e;
}
@@ -393,19 +131,44 @@ describe('Apify.utils_request', () => {
expect(error).to.not.exist; //eslint-disable-line
});
- it('it throws for other contentType then - text/html', async () => {
+ it('throws for other contentType then - text/html', async () => {
const options = {
url: `http://${HOST}:${port}/invalidContentType`,
};
let error;
try {
- await requestLikeBrowser(options);
+ await requestAsBrowser(options);
} catch (e) {
error = e;
}
expect(error).to.exist; //eslint-disable-line
- expect(error.message).to.eql(`utils.requestBetter: Request for ${options.url} aborted due to abortFunction`);
+ expect(error.message).to.eql(`Request for ${options.url} aborted due to abortFunction`);
+ });
+
+ it('overrides defaults', async () => {
+ const host = `${HOST}:${port}`;
+ const options = {
+ url: `http://${host}/echo`,
+ headers: {
+ 'User-Agent': 'chrome',
+ },
+ };
+
+ const response = await requestAsBrowser(options);
+
+ expect(response.statusCode).to.eql(200);
+ expect(JSON.parse(response.body)['user-agent']).to.be.eql(options.headers['User-Agent']);
+ });
+
+ it('overrides defaults', async () => {
+ const options = {
+ url: 'https://www.ebay.com/sch/sis.html?_nkw=Beechcraft',
+ };
+
+ const response = await requestAsBrowser(options);
+
+ console.log(response);
});
});
});