Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

Removing chaiAsPromised initialization from tests, moving to _helper.js

Marek Trunkatd1127edc8b03
diff --git a/test/actor.js b/test/actor.js
index 8b75256a5..09d313394 100644
--- a/test/actor.js
+++ b/test/actor.js
@@ -1,13 +1,11 @@
import path from 'path';
import _ from 'underscore';
-import chai, { expect } from 'chai';
+import { expect } from 'chai';
import sinon from 'sinon';
import { delayPromise } from 'apify-shared/utilities';
import { ENV_VARS, ACT_JOB_STATUSES, LOCAL_ENV_VARS } from 'apify-shared/consts';
import { ApifyCallError } from '../build/errors';
-chai.use(chaiAsPromised);
-
// NOTE: test use of require() here because this is how its done in acts
const Apify = require('../build/index');
diff --git a/test/basic_crawler.js b/test/basic_crawler.js
index 1e918963e..652166c14 100644
--- a/test/basic_crawler.js
+++ b/test/basic_crawler.js
@@ -1,5 +1,4 @@
-import chai, { expect } from 'chai';
-import chaiAsPromised from 'chai-as-promised';
+import { expect } from 'chai';
import _ from 'underscore';
import sinon from 'sinon';
import log from 'apify-shared/log';
@@ -10,8 +9,6 @@ import * as keyValueStore from '../build/key_value_store';
import { RequestQueue, RequestQueueLocal } from '../build/request_queue';
import { LOCAL_STORAGE_DIR } from './_helper';
-chai.use(chaiAsPromised);
-
describe('BasicCrawler', () => {
let logLevel;
diff --git a/test/cheerio_crawler.js b/test/cheerio_crawler.js
index 7649a9c9d..e9af002df 100644
--- a/test/cheerio_crawler.js
+++ b/test/cheerio_crawler.js
@@ -4,15 +4,12 @@ import path from 'path';
import zlib from 'zlib';
import EventEmitter from 'events';
import rqst from 'request';
-import chai, { expect } from 'chai';
-import chaiAsPromised from 'chai-as-promised';
+import { expect } from 'chai';
import log from 'apify-shared/log';
import { delayPromise } from 'apify-shared/utilities';
import { ENV_VARS } from 'apify-shared/consts';
import Apify from '../build/index';
-chai.use(chaiAsPromised);
-
// Add common props to mocked request responses.
const responseMock = {
request: {
diff --git a/test/dataset.js b/test/dataset.js
index 8d9700f96..3f84ea83e 100644
--- a/test/dataset.js
+++ b/test/dataset.js
@@ -1,5 +1,4 @@
-import chai, { expect } from 'chai';
-import chaiAsPromised from 'chai-as-promised';
+import { expect } from 'chai';
import fs from 'fs-extra';
import path from 'path';
import sinon from 'sinon';
@@ -13,8 +12,6 @@ import { LOCAL_STORAGE_DIR, emptyLocalStorageSubdir, expectDirEmpty, expectDirNo
const { apifyClient } = utils;
-chai.use(chaiAsPromised);
-
const read = (datasetName, index) => {
const fileName = `${leftpad(index, LOCAL_FILENAME_DIGITS, 0)}.json`;
const filePath = path.join(LOCAL_STORAGE_DIR, LOCAL_STORAGE_SUBDIR, datasetName, fileName);
diff --git a/test/key_value_store.js b/test/key_value_store.js
index 5683834de..41b9d7436 100644
--- a/test/key_value_store.js
+++ b/test/key_value_store.js
@@ -1,5 +1,4 @@
-import chai, { expect } from 'chai';
-import chaiAsPromised from 'chai-as-promised';
+import { expect } from 'chai';
import sinon from 'sinon';
import path from 'path';
import { ENV_VARS, KEY_VALUE_STORE_KEYS } from 'apify-shared/consts';
@@ -10,8 +9,6 @@ import { LOCAL_STORAGE_DIR, emptyLocalStorageSubdir, expectDirEmpty, expectDirNo
const { apifyClient } = utils;
-chai.use(chaiAsPromised);
-
describe('KeyValueStore', () => {
before(() => apifyClient.setOptions({ token: 'xxx' }));
after(() => apifyClient.setOptions({ token: undefined }));
diff --git a/test/puppeteer_crawler.js b/test/puppeteer_crawler.js
index 00cead365..6052f24ae 100644
--- a/test/puppeteer_crawler.js
+++ b/test/puppeteer_crawler.js
@@ -1,11 +1,8 @@
-import chai, { expect } from 'chai';
-import chaiAsPromised from 'chai-as-promised';
+import { expect } from 'chai';
import log from 'apify-shared/log';
import { ENV_VARS } from 'apify-shared/consts';
import * as Apify from '../build/index';
-chai.use(chaiAsPromised);
-
describe('PuppeteerCrawler', () => {
let prevEnvHeadless;
let logLevel;
diff --git a/test/puppeteer_pool.js b/test/puppeteer_pool.js
index 8e40102b9..6252e3ab0 100644
--- a/test/puppeteer_pool.js
+++ b/test/puppeteer_pool.js
@@ -1,14 +1,11 @@
-import chai, { expect } from 'chai';
+import { expect } from 'chai';
import fs from 'fs';
-import chaiAsPromised from 'chai-as-promised';
import _ from 'underscore';
import log from 'apify-shared/log';
import { ENV_VARS } from 'apify-shared/consts';
import * as Apify from '../build/index';
import { launchPuppeteer } from '../build/puppeteer';
-chai.use(chaiAsPromised);
-
const shortSleep = (millis = 25) => new Promise(resolve => setTimeout(resolve, millis));
describe('PuppeteerPool', () => {
diff --git a/test/puppeteer_utils.js b/test/puppeteer_utils.js
index 4c2c753d7..5912f7d03 100644
--- a/test/puppeteer_utils.js
+++ b/test/puppeteer_utils.js
@@ -1,11 +1,8 @@
-import chai, { expect } from 'chai';
-import chaiAsPromised from 'chai-as-promised';
+import { expect } from 'chai';
import Apify from '../build/index';
const { utils: { log } } = Apify;
-chai.use(chaiAsPromised);
-
/* global describe, it */
describe('Apify.utils.puppeteer', () => {
diff --git a/test/request_list.js b/test/request_list.js
index fc5dde88f..a30d3c06f 100644
--- a/test/request_list.js
+++ b/test/request_list.js
@@ -1,6 +1,5 @@
import _ from 'underscore';
-import chai, { expect } from 'chai';
-import chaiAsPromised from 'chai-as-promised';
+import { expect } from 'chai';
import request from 'request-promise-native';
import sinon from 'sinon';
import log from 'apify-shared/log';
@@ -10,8 +9,6 @@ import Apify from '../build/index';
import * as keyValueStore from '../build/key_value_store';
import * as utils from '../build/utils';
-chai.use(chaiAsPromised);
-
describe('Apify.RequestList', () => {
let ll;
before(() => {
diff --git a/test/request_queue.js b/test/request_queue.js
index 1ee87fa99..776027e99 100644
--- a/test/request_queue.js
+++ b/test/request_queue.js
@@ -1,5 +1,4 @@
-import chai, { assert, expect } from 'chai';
-import chaiAsPromised from 'chai-as-promised';
+import { assert, expect } from 'chai';
import _ from 'underscore';
import sinon from 'sinon';
import path from 'path';
@@ -14,8 +13,6 @@ import { emptyLocalStorageSubdir, LOCAL_STORAGE_DIR, expectNotUsingLocalStorage,
const { apifyClient } = utils;
-chai.use(chaiAsPromised);
-
const expectRequestsSame = (req1, req2) => {
expect(_.omit(req1, 'id')).to.be.eql(_.omit(req2, 'id'));
};
diff --git a/test/utils.js b/test/utils.js
index 74170d5c4..b5fc81002 100644
--- a/test/utils.js
+++ b/test/utils.js
@@ -1,5 +1,4 @@
-import chai, { expect } from 'chai';
-import chaiAsPromised from 'chai-as-promised';
+import { expect } from 'chai';
import sinon from 'sinon';
import _ from 'underscore';
import fs from 'fs';
@@ -12,8 +11,6 @@ import { ENV_VARS, LOCAL_ENV_VARS } from 'apify-shared/consts';
import * as utils from '../build/utils';
import Apify from '../build/index';
-chai.use(chaiAsPromised);
-
/* global process, describe, it */
describe('utils.newClient()', () => {
diff --git a/test/utils_social.js b/test/utils_social.js
index 259fd66c0..d2d745fe9 100644
--- a/test/utils_social.js
+++ b/test/utils_social.js
@@ -1,11 +1,8 @@
import _ from 'underscore';
-import chai, { expect } from 'chai';
-import chaiAsPromised from 'chai-as-promised';
+import { expect } from 'chai';
// import * as utils from '../build/utils';
import Apify from '../build/index';
-chai.use(chaiAsPromised);
-
const { social } = Apify.utils;
/* global describe, it */
@@ -42,7 +39,7 @@ describe('utils.social', () => {
testEmailsFromText(`
this'is'also'valid'[email protected]
not @ an.email.com
@also.not.an.email
`, [
@@ -130,10 +127,10 @@ describe('utils.social', () => {
it('extracts phones correctly', () => {
testPhonesFromText(`
+420775123456 +420775123456
-
+
+420 775 123 456
-
- 775123456 775123456 \n\n
+
+ 775123456 775123456 \n\n
00420775123456
1234567 1234567890
+44 7911 123456
@@ -222,7 +219,7 @@ describe('utils.social', () => {
000-0000
000 0000
000.0000
-
+
0000000
0000000000
(000)0000000
@@ -337,26 +334,26 @@ describe('utils.social', () => {
expect(social.parseHandlesFromHtml(`
<html>
<head>
- <title>Bla</title>
+ <title>Bla</title>
</head>
<a>
- [email protected] testing skipping duplicates xxx@blabla
+ [email protected] testing skipping duplicates xxx@blabla
<p>carl&#64;example.com</p>
<a href="mailto:[email protected]"></a>
- <a href="mailto:david&#64;example.com"></a>
-
- <a href="[email protected]"></a>
+ <a href="mailto:david&#64;example.com"></a>
+
+ <a href="[email protected]"></a>
<img src="http://somewhere.com/ [email protected] " />
<a href="http://somewhere.com/ [email protected] "></a>
-
- &#43;420775222222
+
+ &#43;420775222222
+4207751111111
+4207751111111 test duplicate
1 2 3 4 5 6 7 8 9 0
<a href="skip.this.one: +42099999999"></a>
<a href="tel://+42077533333"></a>
-
+
https://www.linkedin.com/in/bobnewman
https://www.linkedin.com/in/alicenewman/
https://www.linkedin.com/in/alicenewman/ duplicate
@@ -366,26 +363,26 @@ describe('utils.social', () => {
https://www.linkedin.com/in/first-last-123456a
<a href="https://www.linkedin.com/in/jancurn">Profile</a>
<a href="https://www.linkedin.com/in/carl-newman-5555555a/detail/recent-activity/">Sub-link</a>
-
+
https://www.instagram.com/old_prague/
https://www.instagram.com/old_prague/ duplicate
- instagram.com/old_prague
- https://www.instagram.com/newyorkarea/
+ instagram.com/old_prague
+ https://www.instagram.com/newyorkarea/
<a href="https://www.instagram.com/york">link</a>
<a href="https://www.instagram.com/york2/something">sub-link</a>
-
+
<a href="twitter.com/betasomething">link</a>
https://www.twitter.com/apify
- https://www.twitter.com/apify duplicate
+ https://www.twitter.com/apify duplicate
<a href="twitter.com/cblabla/sub-dir/">link</a>
-
- <a href="facebook.com/carl.username123/sub-dir/">link</a>
+
+ <a href="facebook.com/carl.username123/sub-dir/">link</a>
https://www.facebook.com/bob.username123/
https://www.facebook.com/bob.username123/ duplicate
http://www.facebook.com/alice.username123
<a href="https://www.facebook.com/profile.php?id=1155802082&xxx=5">link x</a>
<a href="fb.com/dada5678?query=1">link</a>
-
+
</body>
</html>
`)).to.eql({
@@ -429,7 +426,7 @@ describe('utils.social', () => {
social.parseHandlesFromHtml(`
<html>
<head>
- <title>Bla</title>
+ <title>Bla</title>
</head>
<body>
Body content
@@ -518,7 +515,7 @@ describe('utils.social', () => {
expect(social.LINKEDIN_REGEX_GLOBAL.test('https://www.linkedin.com/in/bobnewman')).to.eql(true);
expect(`
- https://www.linkedin.com/in/bobnewman
+ https://www.linkedin.com/in/bobnewman
"http://ie.linkedin.com/in/alicenewman"
https://www.linkedin.com/in/someverylongnamesomeverylongnamesomeverylongnamesomeverylongnamesomeverylongnamesomeverylongname
linkedin.com/in/carlnewman
@@ -587,7 +584,7 @@ describe('utils.social', () => {
expect(social.INSTAGRAM_REGEX_GLOBAL.test('https://www.instagram.com/old_prague')).to.eql(true);
expect(`
https://www.instagram.com/old_prague
- https://www.instagram.com/someverylongusernamethatisnotgood
+ https://www.instagram.com/someverylongusernamethatisnotgood
"instagram.com/old_brno"
http://instagr.am/old_plzen
`.match(social.INSTAGRAM_REGEX_GLOBAL))