Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
Removing chaiAsPromised initialization from tests, moving to _helper.js
Marek Trunkatd1127edc8b03
diff --git a/test/actor.js b/test/actor.jsindex 8b75256a5..09d313394 100644--- a/test/actor.js+++ b/test/actor.js@@ -1,13 +1,11 @@import path from 'path';import _ from 'underscore';-import chai, { expect } from 'chai';+import { expect } from 'chai';import sinon from 'sinon';import { delayPromise } from 'apify-shared/utilities';import { ENV_VARS, ACT_JOB_STATUSES, LOCAL_ENV_VARS } from 'apify-shared/consts';import { ApifyCallError } from '../build/errors';-chai.use(chaiAsPromised);-// NOTE: test use of require() here because this is how its done in actsconst Apify = require('../build/index');diff --git a/test/basic_crawler.js b/test/basic_crawler.jsindex 1e918963e..652166c14 100644--- a/test/basic_crawler.js+++ b/test/basic_crawler.js@@ -1,5 +1,4 @@-import chai, { expect } from 'chai';-import chaiAsPromised from 'chai-as-promised';+import { expect } from 'chai';import _ from 'underscore';import sinon from 'sinon';import log from 'apify-shared/log';@@ -10,8 +9,6 @@ import * as keyValueStore from '../build/key_value_store';import { RequestQueue, RequestQueueLocal } from '../build/request_queue';import { LOCAL_STORAGE_DIR } from './_helper';-chai.use(chaiAsPromised);-describe('BasicCrawler', () => {let logLevel;diff --git a/test/cheerio_crawler.js b/test/cheerio_crawler.jsindex 7649a9c9d..e9af002df 100644--- a/test/cheerio_crawler.js+++ b/test/cheerio_crawler.js@@ -4,15 +4,12 @@ import path from 'path';import zlib from 'zlib';import EventEmitter from 'events';import rqst from 'request';-import chai, { expect } from 'chai';-import chaiAsPromised from 'chai-as-promised';+import { expect } from 'chai';import log from 'apify-shared/log';import { delayPromise } from 'apify-shared/utilities';import { ENV_VARS } from 'apify-shared/consts';import Apify from '../build/index';-chai.use(chaiAsPromised);-// Add common props to mocked request responses.const responseMock = {request: {diff --git a/test/dataset.js b/test/dataset.jsindex 8d9700f96..3f84ea83e 100644--- a/test/dataset.js+++ b/test/dataset.js@@ -1,5 +1,4 @@-import chai, { expect } from 'chai';-import chaiAsPromised from 'chai-as-promised';+import { expect } from 'chai';import fs from 'fs-extra';import path from 'path';import sinon from 'sinon';@@ -13,8 +12,6 @@ import { LOCAL_STORAGE_DIR, emptyLocalStorageSubdir, expectDirEmpty, expectDirNoconst { apifyClient } = utils;-chai.use(chaiAsPromised);-const read = (datasetName, index) => {const fileName = `${leftpad(index, LOCAL_FILENAME_DIGITS, 0)}.json`;const filePath = path.join(LOCAL_STORAGE_DIR, LOCAL_STORAGE_SUBDIR, datasetName, fileName);diff --git a/test/key_value_store.js b/test/key_value_store.jsindex 5683834de..41b9d7436 100644--- a/test/key_value_store.js+++ b/test/key_value_store.js@@ -1,5 +1,4 @@-import chai, { expect } from 'chai';-import chaiAsPromised from 'chai-as-promised';+import { expect } from 'chai';import sinon from 'sinon';import path from 'path';import { ENV_VARS, KEY_VALUE_STORE_KEYS } from 'apify-shared/consts';@@ -10,8 +9,6 @@ import { LOCAL_STORAGE_DIR, emptyLocalStorageSubdir, expectDirEmpty, expectDirNoconst { apifyClient } = utils;-chai.use(chaiAsPromised);-describe('KeyValueStore', () => {before(() => apifyClient.setOptions({ token: 'xxx' }));after(() => apifyClient.setOptions({ token: undefined }));diff --git a/test/puppeteer_crawler.js b/test/puppeteer_crawler.jsindex 00cead365..6052f24ae 100644--- a/test/puppeteer_crawler.js+++ b/test/puppeteer_crawler.js@@ -1,11 +1,8 @@-import chai, { expect } from 'chai';-import chaiAsPromised from 'chai-as-promised';+import { expect } from 'chai';import log from 'apify-shared/log';import { ENV_VARS } from 'apify-shared/consts';import * as Apify from '../build/index';-chai.use(chaiAsPromised);-describe('PuppeteerCrawler', () => {let prevEnvHeadless;let logLevel;diff --git a/test/puppeteer_pool.js b/test/puppeteer_pool.jsindex 8e40102b9..6252e3ab0 100644--- a/test/puppeteer_pool.js+++ b/test/puppeteer_pool.js@@ -1,14 +1,11 @@-import chai, { expect } from 'chai';+import { expect } from 'chai';import fs from 'fs';-import chaiAsPromised from 'chai-as-promised';import _ from 'underscore';import log from 'apify-shared/log';import { ENV_VARS } from 'apify-shared/consts';import * as Apify from '../build/index';import { launchPuppeteer } from '../build/puppeteer';-chai.use(chaiAsPromised);-const shortSleep = (millis = 25) => new Promise(resolve => setTimeout(resolve, millis));describe('PuppeteerPool', () => {diff --git a/test/puppeteer_utils.js b/test/puppeteer_utils.jsindex 4c2c753d7..5912f7d03 100644--- a/test/puppeteer_utils.js+++ b/test/puppeteer_utils.js@@ -1,11 +1,8 @@-import chai, { expect } from 'chai';-import chaiAsPromised from 'chai-as-promised';+import { expect } from 'chai';import Apify from '../build/index';const { utils: { log } } = Apify;-chai.use(chaiAsPromised);-/* global describe, it */describe('Apify.utils.puppeteer', () => {diff --git a/test/request_list.js b/test/request_list.jsindex fc5dde88f..a30d3c06f 100644--- a/test/request_list.js+++ b/test/request_list.js@@ -1,6 +1,5 @@import _ from 'underscore';-import chai, { expect } from 'chai';-import chaiAsPromised from 'chai-as-promised';+import { expect } from 'chai';import request from 'request-promise-native';import sinon from 'sinon';import log from 'apify-shared/log';@@ -10,8 +9,6 @@ import Apify from '../build/index';import * as keyValueStore from '../build/key_value_store';import * as utils from '../build/utils';-chai.use(chaiAsPromised);-describe('Apify.RequestList', () => {let ll;before(() => {diff --git a/test/request_queue.js b/test/request_queue.jsindex 1ee87fa99..776027e99 100644--- a/test/request_queue.js+++ b/test/request_queue.js@@ -1,5 +1,4 @@-import chai, { assert, expect } from 'chai';-import chaiAsPromised from 'chai-as-promised';+import { assert, expect } from 'chai';import _ from 'underscore';import sinon from 'sinon';import path from 'path';@@ -14,8 +13,6 @@ import { emptyLocalStorageSubdir, LOCAL_STORAGE_DIR, expectNotUsingLocalStorage,const { apifyClient } = utils;-chai.use(chaiAsPromised);-const expectRequestsSame = (req1, req2) => {expect(_.omit(req1, 'id')).to.be.eql(_.omit(req2, 'id'));};diff --git a/test/utils.js b/test/utils.jsindex 74170d5c4..b5fc81002 100644--- a/test/utils.js+++ b/test/utils.js@@ -1,5 +1,4 @@-import chai, { expect } from 'chai';-import chaiAsPromised from 'chai-as-promised';+import { expect } from 'chai';import sinon from 'sinon';import _ from 'underscore';import fs from 'fs';@@ -12,8 +11,6 @@ import { ENV_VARS, LOCAL_ENV_VARS } from 'apify-shared/consts';import * as utils from '../build/utils';import Apify from '../build/index';-chai.use(chaiAsPromised);-/* global process, describe, it */describe('utils.newClient()', () => {diff --git a/test/utils_social.js b/test/utils_social.jsindex 259fd66c0..d2d745fe9 100644--- a/test/utils_social.js+++ b/test/utils_social.js@@ -1,11 +1,8 @@import _ from 'underscore';-import chai, { expect } from 'chai';-import chaiAsPromised from 'chai-as-promised';+import { expect } from 'chai';// import * as utils from '../build/utils';import Apify from '../build/index';-chai.use(chaiAsPromised);-const { social } = Apify.utils;/* global describe, it */@@ -42,7 +39,7 @@ describe('utils.social', () => {testEmailsFromText(`this'is'also'valid'[email protected][email protected] \n\nnot @ an.email.com@also.not.an.email`, [@@ -130,10 +127,10 @@ describe('utils.social', () => {it('extracts phones correctly', () => {testPhonesFromText(`+420775123456 +420775123456-++420 775 123 456-- 775123456 775123456 \n\n++ 775123456 775123456 \n\n004207751234561234567 1234567890+44 7911 123456@@ -222,7 +219,7 @@ describe('utils.social', () => {000-0000000 0000000.0000-+00000000000000000(000)0000000@@ -337,26 +334,26 @@ describe('utils.social', () => {expect(social.parseHandlesFromHtml(`<html><head>- <title>Bla</title>+ <title>Bla</title></head><a>- <p>[email protected]</p>- [email protected] testing skipping duplicates xxx@blabla+ <p>[email protected]</p>+ [email protected] testing skipping duplicates xxx@blabla<p>carl@example.com</p><a href="mailto:[email protected]"></a>- <a href="mailto:david@example.com"></a>-- <a href="[email protected]"></a>+ <a href="mailto:david@example.com"></a>++ <a href="[email protected]"></a><img src="http://somewhere.com/ [email protected] " /><a href="http://somewhere.com/ [email protected] "></a>-- +420775222222++ +420775222222+4207751111111+4207751111111 test duplicate1 2 3 4 5 6 7 8 9 0<a href="skip.this.one: +42099999999"></a><a href="tel://+42077533333"></a>-+https://www.linkedin.com/in/bobnewmanhttps://www.linkedin.com/in/alicenewman/https://www.linkedin.com/in/alicenewman/ duplicate@@ -366,26 +363,26 @@ describe('utils.social', () => {https://www.linkedin.com/in/first-last-123456a<a href="https://www.linkedin.com/in/jancurn">Profile</a><a href="https://www.linkedin.com/in/carl-newman-5555555a/detail/recent-activity/">Sub-link</a>-+https://www.instagram.com/old_prague/https://www.instagram.com/old_prague/ duplicate- instagram.com/old_prague- https://www.instagram.com/newyorkarea/+ instagram.com/old_prague+ https://www.instagram.com/newyorkarea/<a href="https://www.instagram.com/york">link</a><a href="https://www.instagram.com/york2/something">sub-link</a>-+<a href="twitter.com/betasomething">link</a>https://www.twitter.com/apify- https://www.twitter.com/apify duplicate+ https://www.twitter.com/apify duplicate<a href="twitter.com/cblabla/sub-dir/">link</a>-- <a href="facebook.com/carl.username123/sub-dir/">link</a>++ <a href="facebook.com/carl.username123/sub-dir/">link</a>https://www.facebook.com/bob.username123/https://www.facebook.com/bob.username123/ duplicatehttp://www.facebook.com/alice.username123<a href="https://www.facebook.com/profile.php?id=1155802082&xxx=5">link x</a><a href="fb.com/dada5678?query=1">link</a>-+</body></html>`)).to.eql({@@ -429,7 +426,7 @@ describe('utils.social', () => {social.parseHandlesFromHtml(`<html><head>- <title>Bla</title>+ <title>Bla</title></head><body>Body content@@ -518,7 +515,7 @@ describe('utils.social', () => {expect(social.LINKEDIN_REGEX_GLOBAL.test('https://www.linkedin.com/in/bobnewman')).to.eql(true);expect(`- https://www.linkedin.com/in/bobnewman+ https://www.linkedin.com/in/bobnewman"http://ie.linkedin.com/in/alicenewman"https://www.linkedin.com/in/someverylongnamesomeverylongnamesomeverylongnamesomeverylongnamesomeverylongnamesomeverylongnamelinkedin.com/in/carlnewman@@ -587,7 +584,7 @@ describe('utils.social', () => {expect(social.INSTAGRAM_REGEX_GLOBAL.test('https://www.instagram.com/old_prague')).to.eql(true);expect(`https://www.instagram.com/old_prague- https://www.instagram.com/someverylongusernamethatisnotgood+ https://www.instagram.com/someverylongusernamethatisnotgood"instagram.com/old_brno"http://instagr.am/old_plzen`.match(social.INSTAGRAM_REGEX_GLOBAL))