Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
Restructure test folder
Ondra Urban9a51af45768a
diff --git a/__tests__/.eslintrc.json b/test/.eslintrc.jsonsimilarity index 100%rename from __tests__/.eslintrc.jsonrename to test/.eslintrc.jsondiff --git a/__tests__/_helper.js b/test/_helper.jssimilarity index 97%rename from __tests__/_helper.jsrename to test/_helper.jsindex 36fa06636..d845f112e 100644--- a/__tests__/_helper.js+++ b/test/_helper.js@@ -1,6 +1,5 @@import fs from 'fs-extra';import path from 'path';-import chai, { expect } from 'chai';import chaiSubset from 'chai-subset';import chaiAsPromised from 'chai-as-promised';import { ENV_VARS } from 'apify-shared/consts';diff --git a/__tests__/actor.js b/test/actor.test.jssimilarity index 94%rename from __tests__/actor.jsrename to test/actor.test.jsindex 2b127bfb8..8dcc4d5d7 100644--- a/__tests__/actor.js+++ b/test/actor.test.js@@ -1,6 +1,5 @@import path from 'path';import _ from 'underscore';-import chai, { expect } from 'chai';import chaiSubset from 'chai-subset';import sinon from 'sinon';import { delayPromise } from 'apify-shared/utilities';@@ -131,15 +130,15 @@ const setEnv = (env) => {describe('Apify.getEnv()', () => {let prevEnv;- before(() => {+ beforeAll(() => {prevEnv = Apify.getEnv();});- after(() => {+ afterAll(() => {setEnv(prevEnv);});- it('works with null values', () => {+ test('works with null values', () => {const expectedEnv = getEmptyEnv();setEnv(expectedEnv);@@ -147,7 +146,7 @@ describe('Apify.getEnv()', () => {expect(env).to.containSubset(expectedEnv);});- it('works with with non-null values', () => {+ test('works with with non-null values', () => {const expectedEnv = _.extend(getEmptyEnv(), {// internalPort: 12345,actId: 'test actId',@@ -168,20 +167,20 @@ describe('Apify.getEnv()', () => {});describe('Apify.main()', () => {- it('throws on invalid args', () => {+ test('throws on invalid args', () => {expect(() => {Apify.main();}).to.throw(Error);});- it('works with simple user function', () => {+ test('works with simple user function', () => {return testMain({userFunc: () => {},exitCode: 0,});});- it('sets default APIFY_LOCAL_STORAGE_DIR', async () => {+ test('sets default APIFY_LOCAL_STORAGE_DIR', async () => {delete process.env[ENV_VARS.LOCAL_STORAGE_DIR];delete process.env[ENV_VARS.TOKEN];@@ -195,7 +194,7 @@ describe('Apify.main()', () => {delete process.env[ENV_VARS.LOCAL_STORAGE_DIR];});- it('works with promised user function', () => {+ test('works with promised user function', () => {let called = false;return testMain({userFunc: () => {@@ -214,28 +213,34 @@ describe('Apify.main()', () => {});});- it('on exception in simple user function the process exits with code 91', () => {- return testMain({- userFunc: () => {- throw new Error('Test exception I');- },- exitCode: 91,- });- });-- it('on exception in promised user function the process exits with code 91', () => {- return testMain({- userFunc: async () => {- await delayPromise(20);- throw new Error('Test exception II');- },- exitCode: 91,- });- });+ test(+ 'on exception in simple user function the process exits with code 91',+ () => {+ return testMain({+ userFunc: () => {+ throw new Error('Test exception I');+ },+ exitCode: 91,+ });+ }+ );++ test(+ 'on exception in promised user function the process exits with code 91',+ () => {+ return testMain({+ userFunc: async () => {+ await delayPromise(20);+ throw new Error('Test exception II');+ },+ exitCode: 91,+ });+ }+ );});describe('Apify.call()', () => {- it('works as expected', () => {+ test('works as expected', () => {const actId = 'some-act-id';const token = 'some-token';const defaultKeyValueStoreId = 'some-store-id';@@ -289,7 +294,7 @@ describe('Apify.call()', () => {});});- it('supports legacy "memory" option', () => {+ test('supports legacy "memory" option', () => {const actId = 'some-act-id';const token = 'some-token';const defaultKeyValueStoreId = 'some-store-id';@@ -341,7 +346,7 @@ describe('Apify.call()', () => {});});- it('works without opts and input', () => {+ test('works without opts and input', () => {const actId = 'some-act-id';const token = 'token';const defaultKeyValueStoreId = 'some-store-id';@@ -382,7 +387,7 @@ describe('Apify.call()', () => {});});- it('should not fail when run get stuck in READY state', () => {+ test('should not fail when run get stuck in READY state', () => {const actId = 'some-act-id';const token = 'token';const defaultKeyValueStoreId = 'some-store-id';@@ -413,7 +418,7 @@ describe('Apify.call()', () => {});});- it('works without opts with null input', () => {+ test('works without opts with null input', () => {const actId = 'some-act-id';const token = 'token';const defaultKeyValueStoreId = 'some-store-id';@@ -454,7 +459,7 @@ describe('Apify.call()', () => {});});- it('works without opts with non-null input', () => {+ test('works without opts with non-null input', () => {const actId = 'some-act-id';const token = 'token';const defaultKeyValueStoreId = 'some-store-id';@@ -497,7 +502,7 @@ describe('Apify.call()', () => {});});- it('stringifies to JSON', () => {+ test('stringifies to JSON', () => {const actId = 'some-act-id';const token = 'some-token';const defaultKeyValueStoreId = 'some-store-id';@@ -538,7 +543,7 @@ describe('Apify.call()', () => {});});- it('works as expected with fetchOutput = false', () => {+ test('works as expected with fetchOutput = false', () => {const actId = 'some-act-id';const token = 'some-token';const defaultKeyValueStoreId = 'some-store-id';@@ -572,7 +577,7 @@ describe('Apify.call()', () => {});});- it('timeouts as expected with unfinished run', () => {+ test('timeouts as expected with unfinished run', () => {const actId = 'some-act-id';const token = 'some-token';const defaultKeyValueStoreId = 'some-store-id';@@ -604,7 +609,7 @@ describe('Apify.call()', () => {});});- it('handles getRun() returning null the first time', () => {+ test('handles getRun() returning null the first time', () => {const actId = 'some-act-id';const token = 'some-token';const defaultKeyValueStoreId = 'some-store-id';@@ -646,7 +651,7 @@ describe('Apify.call()', () => {});});- it('returns immediately with zero ', () => {+ test('returns immediately with zero ', () => {const actId = 'some-act-id';const token = 'some-token';const defaultKeyValueStoreId = 'some-store-id';@@ -673,7 +678,7 @@ describe('Apify.call()', () => {});});- it('throws if run doesn\'t succeed', () => {+ test('throws if run doesn\'t succeed', () => {const actId = 'some-act-id';const token = 'some-token';const run = { id: 'some-run-id' };@@ -702,7 +707,7 @@ describe('Apify.call()', () => {});describe('Apify.callTask()', () => {- it('works as expected', () => {+ test('works as expected', () => {const taskId = 'some-act-id';const actId = 'xxx';const token = 'some-token';@@ -758,7 +763,7 @@ describe('Apify.callTask()', () => {});});- it('works as expected with fetchOutput = false', () => {+ test('works as expected with fetchOutput = false', () => {const taskId = 'some-act-id';const actId = 'xxx';const token = 'some-token';@@ -796,7 +801,7 @@ describe('Apify.callTask()', () => {});});- it('s as expected with unfinished run', () => {+ test('s as expected with unfinished run', () => {const waitSecs = 1;const taskId = 'some-act-id';const actId = 'xxx';@@ -832,7 +837,7 @@ describe('Apify.callTask()', () => {});});- it('handles getRun() returning null the first time', () => {+ test('handles getRun() returning null the first time', () => {const actId = 'some-act-id';const taskId = 'some-act-id';const token = 'some-token';@@ -876,7 +881,7 @@ describe('Apify.callTask()', () => {});});- it('returns immediately with zero ', () => {+ test('returns immediately with zero ', () => {const waitSecs = 0;const taskId = 'some-act-id';const actId = 'xxx';@@ -907,7 +912,7 @@ describe('Apify.callTask()', () => {});});- it('throws if run doesn\'t succeed', () => {+ test('throws if run doesn\'t succeed', () => {const taskId = 'some-act-id';const actId = 'xxx';const defaultKeyValueStoreId = 'some-store-id';@@ -940,7 +945,7 @@ describe('Apify.callTask()', () => {});describe('Apify.metamorph()', () => {- it('works as expected', async () => {+ test('works as expected', async () => {const runId = 'some-run-id';const actorId = 'some-actor-id';const targetActorId = 'some-target-actor-id';@@ -974,7 +979,7 @@ describe('Apify.metamorph()', () => {actsMock.restore();});- it('works without opts and input', async () => {+ test('works without opts and input', async () => {const runId = 'some-run-id';const actorId = 'some-actor-id';const targetActorId = 'some-target-actor-id';@@ -1005,7 +1010,7 @@ describe('Apify.metamorph()', () => {actsMock.restore();});- it('stringifies to JSON including functions', async () => {+ test('stringifies to JSON including functions', async () => {const runId = 'some-run-id';const actorId = 'some-actor-id';const targetActorId = 'some-target-actor-id';@@ -1041,7 +1046,7 @@ describe('Apify.metamorph()', () => {});describe('Apify.getApifyProxyUrl()', () => {- it('should work', () => {+ test('should work', () => {process.env[ENV_VARS.PROXY_PASSWORD] = 'abc123';process.env[ENV_VARS.PROXY_HOSTNAME] = 'my.host.com';process.env[ENV_VARS.PROXY_PORT] = 123;@@ -1078,7 +1083,7 @@ describe('Apify.getApifyProxyUrl()', () => {});// Test old params - session, groups- it('should be backwards compatible', () => {+ test('should be backwards compatible', () => {const ll = log.getLevel();log.setLevel(log.LEVELS.ERROR);process.env[ENV_VARS.PROXY_PASSWORD] = 'abc123';@@ -1117,7 +1122,7 @@ describe('Apify.getApifyProxyUrl()', () => {log.setLevel(ll);});- it('should throw on invalid proxy args', () => {+ test('should throw on invalid proxy args', () => {process.env[ENV_VARS.PROXY_PASSWORD] = 'abc123';process.env[ENV_VARS.PROXY_HOSTNAME] = 'my.host.com';process.env[ENV_VARS.PROXY_PORT] = 123;@@ -1150,7 +1155,7 @@ describe('Apify.getApifyProxyUrl()', () => {describe('Apify.addWebhook()', () => {- it('works', async () => {+ test('works', async () => {const runId = 'my-run-id';const expectedEventTypes = ['ACTOR.RUN.SUCCEEDED'];const expectedRequestUrl = 'http://example.com/api';@@ -1192,7 +1197,7 @@ describe('Apify.addWebhook()', () => {webhooksMock.verify();});- it('on local logs warning and does nothing', async () => {+ test('on local logs warning and does nothing', async () => {const expectedEventTypes = ['ACTOR.RUN.SUCCEEDED'];const expectedRequestUrl = 'http://example.com/api';@@ -1208,7 +1213,7 @@ describe('Apify.addWebhook()', () => {logMock.verify();});- it('should fail without actor run ID', async () => {+ test('should fail without actor run ID', async () => {const expectedEventTypes = ['ACTOR.RUN.SUCCEEDED'];const expectedRequestUrl = 'http://example.com/api';diff --git a/__tests__/autoscaling/autoscaled_pool.js b/test/autoscaling/autoscaled_pool.test.jssimilarity index 82%rename from __tests__/autoscaling/autoscaled_pool.jsrename to test/autoscaling/autoscaled_pool.test.jsindex 760e68583..48f8d515b 100644--- a/__tests__/autoscaling/autoscaled_pool.js+++ b/test/autoscaling/autoscaled_pool.test.js@@ -1,4 +1,3 @@-import chai, { expect } from 'chai';import chaiAsPromised from 'chai-as-promised';import _ from 'underscore';import { delayPromise } from 'apify-shared/utilities';@@ -11,16 +10,16 @@ chai.use(chaiAsPromised);describe('AutoscaledPool', () => {let logLevel;- before(() => {+ beforeAll(() => {logLevel = log.getLevel();log.setLevel(log.LEVELS.ERROR);});- after(() => {+ afterAll(() => {log.setLevel(logLevel);});- it('should work with concurrency 1', async () => {+ test('should work with concurrency 1', async () => {const range = _.range(0, 10);const result = [];@@ -52,7 +51,7 @@ describe('AutoscaledPool', () => {expect(Date.now() - startedAt).to.be.within(50, 200);});- it('should work with concurrency 10', async () => {+ test('should work with concurrency 10', async () => {const range = _.range(0, 100);const result = [];@@ -85,7 +84,7 @@ describe('AutoscaledPool', () => {expect(Date.now() - startedAt).to.be.within(50, 200);});- it('enables setting concurrency', async () => {+ test('enables setting concurrency', async () => {const range = _.range(0, 100);const result = [];@@ -160,7 +159,7 @@ describe('AutoscaledPool', () => {pool.systemStatus = systemStatus;});- it('works with low values', () => {+ test('works with low values', () => {pool._autoscale(cb);expect(pool.desiredConcurrency).to.be.eql(2);@@ -181,7 +180,7 @@ describe('AutoscaledPool', () => {expect(pool.desiredConcurrency).to.be.eql(3);});- it('works with high values', () => {+ test('works with high values', () => {// Should not scale because current concurrency is too low.pool.desiredConcurrency = 50;pool._currentConcurrency = Math.floor(pool.desiredConcurrency * pool.desiredConcurrencyRatio) - 1;@@ -202,7 +201,7 @@ describe('AutoscaledPool', () => {expect(pool.desiredConcurrency).to.be.eql(newConcurrency);});- it('works at minConcurrency when currently overloaded', async () => {+ test('works at minConcurrency when currently overloaded', async () => {let limit = 5;let concurrencyLog = [];let count = 0;@@ -244,16 +243,16 @@ describe('AutoscaledPool', () => {describe('should throw', () => {// Turn off unnecessary error logging.let originalLevel;- before(() => {+ beforeAll(() => {originalLevel = log.getLevel();log.setLevel(log.LEVELS.OFF);});- after(() => {+ afterAll(() => {log.setLevel(originalLevel);});- it('when some of the promises throws', async () => {+ test('when some of the promises throws', async () => {let counter = 0;const runTaskFunction = () => {counter++;@@ -280,7 +279,7 @@ describe('AutoscaledPool', () => {await expect(pool.run()).to.be.rejectedWith('some-promise-error');});- it('when runTaskFunction throws', async () => {+ test('when runTaskFunction throws', async () => {const runTaskFunction = () => {throw new Error('some-runtask-error');};@@ -297,7 +296,7 @@ describe('AutoscaledPool', () => {await expect(pool.run()).to.be.rejectedWith('some-runtask-error');});- it('when isFinishedFunction throws', async () => {+ test('when isFinishedFunction throws', async () => {let count = 0;const pool = new AutoscaledPool({minConcurrency: 10,@@ -311,7 +310,7 @@ describe('AutoscaledPool', () => {await expect(pool.run()).to.be.rejectedWith('some-finished-error');});- it('when isTaskReadyFunction throws', async () => {+ test('when isTaskReadyFunction throws', async () => {let count = 0;const pool = new AutoscaledPool({minConcurrency: 10,@@ -330,52 +329,58 @@ describe('AutoscaledPool', () => {});- it('should not handle tasks added after isFinishedFunction returned true', async () => {- const isFinished = async () => count > 10;- let count = 0;+ test(+ 'should not handle tasks added after isFinishedFunction returned true',+ async () => {+ const isFinished = async () => count > 10;+ let count = 0;- // Run the pool and close it after 3s.- const pool = new AutoscaledPool({- minConcurrency: 3,- runTaskFunction: async () => delayPromise(1).then(() => { count++; }),- isFinishedFunction: async () => isFinished(),- isTaskReadyFunction: async () => !await isFinished(),- });- pool.maybeRunIntervalMillis = 5;+ // Run the pool and close it after 3s.+ const pool = new AutoscaledPool({+ minConcurrency: 3,+ runTaskFunction: async () => delayPromise(1).then(() => { count++; }),+ isFinishedFunction: async () => isFinished(),+ isTaskReadyFunction: async () => !await isFinished(),+ });+ pool.maybeRunIntervalMillis = 5;- await pool.run();- await delayPromise(10);- // Check finished tasks.- expect(count).to.be.within(11, 13);- });+ await pool.run();+ await delayPromise(10);+ // Check finished tasks.+ expect(count).to.be.within(11, 13);+ }+ );- it('should break and resume when the task queue is empty for a while', async () => {- const finished = [];- let isFinished = false;- let isTaskReady = true;+ test(+ 'should break and resume when the task queue is empty for a while',+ async () => {+ const finished = [];+ let isFinished = false;+ let isTaskReady = true;- let counter = 0;- const pool = new AutoscaledPool({- maxConcurrency: 1,- runTaskFunction: async () => {- await delayPromise(1);- if (counter === 10) { isTaskReady = false; setTimeout(() => { isTaskReady = true; }, 10); }- if (counter === 19) { isTaskReady = false; isFinished = true; }- counter++;- finished.push(Date.now());- },- isFinishedFunction: async () => isFinished,- isTaskReadyFunction: async () => !isFinished && isTaskReady,- });- pool.maybeRunIntervalMillis = 1;- await pool.run();+ let counter = 0;+ const pool = new AutoscaledPool({+ maxConcurrency: 1,+ runTaskFunction: async () => {+ await delayPromise(1);+ if (counter === 10) { isTaskReady = false; setTimeout(() => { isTaskReady = true; }, 10); }+ if (counter === 19) { isTaskReady = false; isFinished = true; }+ counter++;+ finished.push(Date.now());+ },+ isFinishedFunction: async () => isFinished,+ isTaskReadyFunction: async () => !isFinished && isTaskReady,+ });+ pool.maybeRunIntervalMillis = 1;+ await pool.run();- // Check finished tasks.- expect(finished).to.have.lengthOf(20);- expect(finished[11] - finished[10]).to.be.above(9);- });+ // Check finished tasks.+ expect(finished).to.have.lengthOf(20);+ expect(finished[11] - finished[10]).to.be.above(9);+ }+ );- it('should work with loggingIntervalMillis = null', async () => {+ test('should work with loggingIntervalMillis = null', async () => {const pool = new AutoscaledPool({minConcurrency: 1,maxConcurrency: 100,@@ -388,7 +393,7 @@ describe('AutoscaledPool', () => {expect(pool.desiredConcurrency).to.be.eql(2);});- it('should abort', async () => {+ test('should abort', async () => {let finished = false;let aborted = false;const pool = new AutoscaledPool({@@ -407,7 +412,7 @@ describe('AutoscaledPool', () => {expect(finished).to.be.eql(false);});- it('should only finish after tasks complete', async () => {+ test('should only finish after tasks complete', async () => {let started = false;let completed = false;@@ -432,7 +437,7 @@ describe('AutoscaledPool', () => {expect(completed).to.be.eql(true);});- it('should pause and resume', async () => {+ test('should pause and resume', async () => {let count = 0;const results = [];let pauseResolve;diff --git a/__tests__/autoscaling/snapshotter.js b/test/autoscaling/snapshotter.test.jssimilarity index 72%rename from __tests__/autoscaling/snapshotter.jsrename to test/autoscaling/snapshotter.test.jsindex 1dcc36cb6..27905a413 100644--- a/__tests__/autoscaling/snapshotter.js+++ b/test/autoscaling/snapshotter.test.js@@ -13,16 +13,16 @@ const toBytes = x => x * 1024 * 1024;describe('Snapshotter', () => {let logLevel;- before(() => {+ beforeAll(() => {logLevel = log.getLevel();log.setLevel(log.LEVELS.ERROR);});- after(() => {+ afterAll(() => {log.setLevel(logLevel);});- it('should collect snapshots with some values', async () => {+ test('should collect snapshots with some values', async () => {// mock client dataconst oldStats = utils.apifyClient.stats;utils.apifyClient.stats = {};@@ -76,7 +76,7 @@ describe('Snapshotter', () => {utils.apifyClient.stats = oldStats;});- it('should override default timers', async () => {+ test('should override default timers', async () => {const options = {eventLoopSnapshotIntervalSecs: 0.05,memorySnapshotIntervalSecs: 0.1,@@ -95,7 +95,7 @@ describe('Snapshotter', () => {expect(eventLoopSnapshots.length).toBeGreaterThan(4);});- it('correctly marks CPU overloaded using Platform event', async () => {+ test('correctly marks CPU overloaded using Platform event', async () => {process.env[ENV_VARS.IS_AT_HOME] = '1';let count = 0;const emitAndWait = async (delay) => {@@ -129,7 +129,7 @@ describe('Snapshotter', () => {}});- it('correctly marks CPU overloaded using OS metrics', () => {+ test('correctly marks CPU overloaded using OS metrics', () => {const mock = sinon.mock(os);const fakeCpu = [{times: {@@ -171,7 +171,7 @@ describe('Snapshotter', () => {mock.verify();});- it('correctly marks eventLoopOverloaded', () => {+ test('correctly marks eventLoopOverloaded', () => {const clock = sinon.useFakeTimers();try {const noop = () => {};@@ -198,41 +198,44 @@ describe('Snapshotter', () => {}});- it('correctly marks memoryOverloaded using OS metrics', async () => { /* eslint-disable no-underscore-dangle */- const noop = () => {};- const memoryData = {- mainProcessBytes: toBytes(1000),- childProcessesBytes: toBytes(1000),- };- const getMem = async () => Object.assign({}, memoryData);- const stub = sinon.stub(utils, 'getMemoryInfo');- stub.callsFake(getMem);-- process.env[ENV_VARS.MEMORY_MBYTES] = '10000';-- const snapshotter = new Snapshotter({ maxUsedMemoryRatio: 0.5 });- await snapshotter._snapshotMemoryOnLocal(noop);- memoryData.mainProcessBytes = toBytes(2000);- await snapshotter._snapshotMemoryOnLocal(noop);- memoryData.childProcessesBytes = toBytes(2000);- await snapshotter._snapshotMemoryOnLocal(noop);- memoryData.mainProcessBytes = toBytes(3001);- await snapshotter._snapshotMemoryOnLocal(noop);- memoryData.childProcessesBytes = toBytes(1999);- await snapshotter._snapshotMemoryOnLocal(noop);- const memorySnapshots = snapshotter.getMemorySample();-- expect(memorySnapshots.length).toBe(5);- expect(memorySnapshots[0].isOverloaded).toBe(false);- expect(memorySnapshots[1].isOverloaded).toBe(false);- expect(memorySnapshots[2].isOverloaded).toBe(false);- expect(memorySnapshots[3].isOverloaded).toBe(true);- expect(memorySnapshots[4].isOverloaded).toBe(false);-- delete process.env[ENV_VARS.MEMORY_MBYTES];- });+ test(+ 'correctly marks memoryOverloaded using OS metrics',+ async () => { /* eslint-disable no-underscore-dangle */+ const noop = () => {};+ const memoryData = {+ mainProcessBytes: toBytes(1000),+ childProcessesBytes: toBytes(1000),+ };+ const getMem = async () => Object.assign({}, memoryData);+ const stub = sinon.stub(utils, 'getMemoryInfo');+ stub.callsFake(getMem);++ process.env[ENV_VARS.MEMORY_MBYTES] = '10000';++ const snapshotter = new Snapshotter({ maxUsedMemoryRatio: 0.5 });+ await snapshotter._snapshotMemoryOnLocal(noop);+ memoryData.mainProcessBytes = toBytes(2000);+ await snapshotter._snapshotMemoryOnLocal(noop);+ memoryData.childProcessesBytes = toBytes(2000);+ await snapshotter._snapshotMemoryOnLocal(noop);+ memoryData.mainProcessBytes = toBytes(3001);+ await snapshotter._snapshotMemoryOnLocal(noop);+ memoryData.childProcessesBytes = toBytes(1999);+ await snapshotter._snapshotMemoryOnLocal(noop);+ const memorySnapshots = snapshotter.getMemorySample();++ expect(memorySnapshots.length).toBe(5);+ expect(memorySnapshots[0].isOverloaded).toBe(false);+ expect(memorySnapshots[1].isOverloaded).toBe(false);+ expect(memorySnapshots[2].isOverloaded).toBe(false);+ expect(memorySnapshots[3].isOverloaded).toBe(true);+ expect(memorySnapshots[4].isOverloaded).toBe(false);++ delete process.env[ENV_VARS.MEMORY_MBYTES];+ }+ );- it('correctly logs critical memory overload', () => {+ test('correctly logs critical memory overload', () => {const memoryDataOverloaded = {memCurrentBytes: toBytes(7600),};@@ -257,34 +260,37 @@ describe('Snapshotter', () => {delete process.env[ENV_VARS.MEMORY_MBYTES];});- it('correctly marks clientOverloaded', () => { /* eslint-disable no-underscore-dangle */- const noop = () => {};- // mock client data- const oldStats = utils.apifyClient.stats;- utils.apifyClient.stats = {};- utils.apifyClient.stats.rateLimitErrors = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0];-- const snapshotter = new Snapshotter({ maxClientErrors: 1 });- snapshotter._snapshotClient(noop);- utils.apifyClient.stats.rateLimitErrors = [1, 1, 1, 0, 0, 0, 0, 0, 0, 0];- snapshotter._snapshotClient(noop);- utils.apifyClient.stats.rateLimitErrors = [10, 5, 2, 0, 0, 0, 0, 0, 0, 0];- snapshotter._snapshotClient(noop);- utils.apifyClient.stats.rateLimitErrors = [100, 24, 4, 2, 0, 0, 0, 0, 0, 0];- snapshotter._snapshotClient(noop);-- const clientSnapshots = snapshotter.getClientSample();-- expect(clientSnapshots.length).toBe(4);- expect(clientSnapshots[0].isOverloaded).toBe(false);- expect(clientSnapshots[1].isOverloaded).toBe(false);- expect(clientSnapshots[2].isOverloaded).toBe(false);- expect(clientSnapshots[3].isOverloaded).toBe(true);-- utils.apifyClient.stats = oldStats;- });+ test(+ 'correctly marks clientOverloaded',+ () => { /* eslint-disable no-underscore-dangle */+ const noop = () => {};+ // mock client data+ const oldStats = utils.apifyClient.stats;+ utils.apifyClient.stats = {};+ utils.apifyClient.stats.rateLimitErrors = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0];++ const snapshotter = new Snapshotter({ maxClientErrors: 1 });+ snapshotter._snapshotClient(noop);+ utils.apifyClient.stats.rateLimitErrors = [1, 1, 1, 0, 0, 0, 0, 0, 0, 0];+ snapshotter._snapshotClient(noop);+ utils.apifyClient.stats.rateLimitErrors = [10, 5, 2, 0, 0, 0, 0, 0, 0, 0];+ snapshotter._snapshotClient(noop);+ utils.apifyClient.stats.rateLimitErrors = [100, 24, 4, 2, 0, 0, 0, 0, 0, 0];+ snapshotter._snapshotClient(noop);++ const clientSnapshots = snapshotter.getClientSample();++ expect(clientSnapshots.length).toBe(4);+ expect(clientSnapshots[0].isOverloaded).toBe(false);+ expect(clientSnapshots[1].isOverloaded).toBe(false);+ expect(clientSnapshots[2].isOverloaded).toBe(false);+ expect(clientSnapshots[3].isOverloaded).toBe(true);++ utils.apifyClient.stats = oldStats;+ }+ );- it('.get[.*]Sample limits amount of samples', async () => {+ test('.get[.*]Sample limits amount of samples', async () => {const SAMPLE_SIZE_MILLIS = 120;const options = {eventLoopSnapshotIntervalSecs: 0.01,diff --git a/__tests__/autoscaling/system_status.js b/test/autoscaling/system_status.test.jssimilarity index 94%rename from __tests__/autoscaling/system_status.jsrename to test/autoscaling/system_status.test.jsindex f4a28d2e9..559f258a3 100644--- a/__tests__/autoscaling/system_status.js+++ b/test/autoscaling/system_status.test.js@@ -4,12 +4,12 @@ import Snapshotter from '../../build/autoscaling/snapshotter';describe('SystemStatus', () => {let logLevel;- before(() => {+ beforeAll(() => {logLevel = log.getLevel();log.setLevel(log.LEVELS.ERROR);});- after(() => {+ afterAll(() => {log.setLevel(logLevel);});@@ -50,7 +50,7 @@ describe('SystemStatus', () => {return snaps;};- it('should return OK for OK snapshots', () => {+ test('should return OK for OK snapshots', () => {const snaps = generateSnapsSync(100, false);const systemStatus = new SystemStatus({snapshotter: new MockSnapshotter(snaps, snaps, snaps, snaps),@@ -59,7 +59,7 @@ describe('SystemStatus', () => {expect(systemStatus.getHistoricalStatus().isSystemIdle).toBe(true);});- it('should return overloaded for overloaded snapshots', () => {+ test('should return overloaded for overloaded snapshots', () => {const snaps = generateSnapsSync(100, true);const systemStatus = new SystemStatus({snapshotter: new MockSnapshotter(snaps, snaps, snaps, snaps),@@ -68,7 +68,7 @@ describe('SystemStatus', () => {expect(systemStatus.getHistoricalStatus().isSystemIdle).toBe(false);});- it('should work with some samples empty', () => {+ test('should work with some samples empty', () => {const snaps = generateSnapsSync(100, true);let systemStatus = new SystemStatus({snapshotter: new MockSnapshotter(snaps, [], [], []),@@ -92,7 +92,7 @@ describe('SystemStatus', () => {expect(systemStatus.getHistoricalStatus().isSystemIdle).toBe(true);});- it('should overload if only one sample is overloaded', () => {+ test('should overload if only one sample is overloaded', () => {const overloaded = generateSnapsSync(100, true);const fine = generateSnapsSync(100, false);let systemStatus = new SystemStatus({@@ -117,7 +117,7 @@ describe('SystemStatus', () => {expect(systemStatus.getHistoricalStatus().isSystemIdle).toBe(false);});- it('should overload when threshold is crossed', () => {+ test('should overload when threshold is crossed', () => {const snaps = generateSnapsSync(50, true);const systemStatus = new SystemStatus({snapshotter: new MockSnapshotter(snaps, snaps, snaps, snaps),@@ -151,7 +151,7 @@ describe('SystemStatus', () => {expect(systemStatus.getHistoricalStatus().isSystemIdle).toBe(true);});- it('should show different values for now and lately', () => {+ test('should show different values for now and lately', () => {let snaps = generateSnapsSync(95, false);let systemStatus = new SystemStatus({snapshotter: new MockSnapshotter(snaps, snaps, snaps, snaps),@@ -193,7 +193,7 @@ describe('SystemStatus', () => {expect(systemStatus.getHistoricalStatus().isSystemIdle).toBe(false);});- it('creates a snapshotter when none is passed', () => {+ test('creates a snapshotter when none is passed', () => {const systemStatus = new SystemStatus();expect(systemStatus.snapshotter).toBeInstanceOf(Snapshotter);});diff --git a/__tests__/constants.js b/test/constants.test.jssimilarity index 81%rename from __tests__/constants.jsrename to test/constants.test.jsindex 89715f651..101da3f01 100644--- a/__tests__/constants.js+++ b/test/constants.test.js@@ -2,12 +2,12 @@ import { ACTOR_BASE_DOCKER_IMAGES, BUILD_TAG_LATEST, ENV_VARS } from 'apify-sharimport { ACTOR_EVENT_NAMES_EX } from '../build/constants';describe('consts', () => {- it('should contain constants from apify-shared', () => {+ test('should contain constants from apify-shared', () => {expect(Array.isArray(ACTOR_BASE_DOCKER_IMAGES)).toBe(true);expect(typeof BUILD_TAG_LATEST).toBe('string');});- it('should contain local constants', () => {+ test('should contain local constants', () => {expect(ENV_VARS).toBeInstanceOf(Object);expect(ACTOR_EVENT_NAMES_EX.CPU_INFO).toBe('cpuInfo');expect(ACTOR_EVENT_NAMES_EX.PERSIST_STATE).toBe('persistState');diff --git a/__tests__/crawlers/basic_crawler.js b/test/crawlers/basic_crawler.test.jssimilarity index 78%rename from __tests__/crawlers/basic_crawler.jsrename to test/crawlers/basic_crawler.test.jsindex 6831b4f77..b01d9720d 100644--- a/__tests__/crawlers/basic_crawler.js+++ b/test/crawlers/basic_crawler.test.js@@ -11,16 +11,16 @@ import { LOCAL_STORAGE_DIR } from '../_helper';describe('BasicCrawler', () => {let logLevel;- before(() => {+ beforeAll(() => {logLevel = log.getLevel();log.setLevel(log.LEVELS.OFF);});- after(() => {+ afterAll(() => {log.setLevel(logLevel);});- it('should run in parallel thru all the requests', async () => {+ test('should run in parallel thru all the requests', async () => {const startedAt = Date.now();const sources = _.range(0, 500).map(index => ({ url: `https://example.com/${index}` }));@@ -48,51 +48,54 @@ describe('BasicCrawler', () => {expect(await requestList.isEmpty()).toBe(true);});- it('should pause on migration event and persist RequestList state', async () => {- const sources = _.range(500).map(index => ({ url: `https://example.com/${index + 1}` }));-- let persistResolve;- const persistPromise = new Promise((res) => { persistResolve = res; });-- // Mock the calls to persist sources.- const mock = sinon.mock(keyValueStore);- mock.expects('getValue').twice().resolves(null);- mock.expects('setValue').once().resolves();-- const processed = [];- const requestList = await Apify.openRequestList('reqList', sources);- const handleRequestFunction = async ({ request }) => {- if (request.url.endsWith('200')) Apify.events.emit(ACTOR_EVENT_NAMES.MIGRATING);- processed.push(_.pick(request, 'url'));- };-- const basicCrawler = new Apify.BasicCrawler({- requestList,- minConcurrency: 25,- maxConcurrency: 25,- handleRequestFunction,- });-- let finished = false;- // Mock the call to persist state.- mock.expects('setValue').once().callsFake(async () => { persistResolve(); });- // The crawler will pause after 200 requests- const runPromise = basicCrawler.run();- runPromise.then(() => { finished = true; });- await persistPromise;-- expect(finished).toBe(false);- expect(await requestList.isFinished()).toBe(false);- expect(await requestList.isEmpty()).toBe(false);- expect(processed.length).toBe(200);-- mock.verify();-- // clean up- await basicCrawler.autoscaledPool._destroy(); // eslint-disable-line no-underscore-dangle- });+ test(+ 'should pause on migration event and persist RequestList state',+ async () => {+ const sources = _.range(500).map(index => ({ url: `https://example.com/${index + 1}` }));++ let persistResolve;+ const persistPromise = new Promise((res) => { persistResolve = res; });++ // Mock the calls to persist sources.+ const mock = sinon.mock(keyValueStore);+ mock.expects('getValue').twice().resolves(null);+ mock.expects('setValue').once().resolves();++ const processed = [];+ const requestList = await Apify.openRequestList('reqList', sources);+ const handleRequestFunction = async ({ request }) => {+ if (request.url.endsWith('200')) Apify.events.emit(ACTOR_EVENT_NAMES.MIGRATING);+ processed.push(_.pick(request, 'url'));+ };++ const basicCrawler = new Apify.BasicCrawler({+ requestList,+ minConcurrency: 25,+ maxConcurrency: 25,+ handleRequestFunction,+ });++ let finished = false;+ // Mock the call to persist state.+ mock.expects('setValue').once().callsFake(async () => { persistResolve(); });+ // The crawler will pause after 200 requests+ const runPromise = basicCrawler.run();+ runPromise.then(() => { finished = true; });+ await persistPromise;++ expect(finished).toBe(false);+ expect(await requestList.isFinished()).toBe(false);+ expect(await requestList.isEmpty()).toBe(false);+ expect(processed.length).toBe(200);++ mock.verify();++ // clean up+ await basicCrawler.autoscaledPool._destroy(); // eslint-disable-line no-underscore-dangle+ }+ );- it('should retry failed requests', async () => {+ test('should retry failed requests', async () => {const sources = [{ url: 'http://example.com/1' },{ url: 'http://example.com/2' },@@ -138,7 +141,7 @@ describe('BasicCrawler', () => {expect(await requestList.isEmpty()).toBe(true);});- it('should not retry requests with noRetry set to true ', async () => {+ test('should not retry requests with noRetry set to true ', async () => {const noRetryRequest = new Apify.Request({ url: 'http://example.com/3' });try {noRetryRequest.doNotRetry('no retry');@@ -196,7 +199,7 @@ describe('BasicCrawler', () => {expect(await requestList.isEmpty()).toBe(true);});- it('should allow to handle failed requests', async () => {+ test('should allow to handle failed requests', async () => {const sources = [{ url: 'http://example.com/1' },{ url: 'http://example.com/2' },@@ -239,7 +242,7 @@ describe('BasicCrawler', () => {errors.forEach(error => expect(error).toBeInstanceOf(Error));});- it('should require at least one of RequestQueue and RequestList', () => {+ test('should require at least one of RequestQueue and RequestList', () => {const requestList = new Apify.RequestList({ sources: [] });const requestQueue = new RequestQueue('xxx');const handleRequestFunction = () => {};@@ -250,7 +253,7 @@ describe('BasicCrawler', () => {expect(() => new Apify.BasicCrawler({ handleRequestFunction, requestQueue, requestList })).not.toThrowError();});- it('should also support RequestQueueLocal', () => {+ test('should also support RequestQueueLocal', () => {const requestQueue = new RequestQueue('xxx');const requestQueueLocal = new RequestQueueLocal('xxx', LOCAL_STORAGE_DIR);const handleRequestFunction = () => {};@@ -259,7 +262,7 @@ describe('BasicCrawler', () => {expect(() => new Apify.BasicCrawler({ handleRequestFunction, requestQueue: requestQueueLocal })).not.toThrowError();});- it('should correctly combine RequestList and RequestQueue', async () => {+ test('should correctly combine RequestList and RequestQueue', async () => {const sources = [{ url: 'http://example.com/0' },{ url: 'http://example.com/1' },@@ -385,66 +388,72 @@ describe('BasicCrawler', () => {mock.verify();});- it('should say that task is not ready requestList is not set and requestQueue is empty', async () => {- const requestQueue = new RequestQueue('xxx');- requestQueue.isEmpty = () => Promise.resolve(true);-- const crawler = new Apify.BasicCrawler({- requestQueue,- handleRequestFunction: () => {},- });+ test(+ 'should say that task is not ready requestList is not set and requestQueue is empty',+ async () => {+ const requestQueue = new RequestQueue('xxx');+ requestQueue.isEmpty = () => Promise.resolve(true);- expect(await crawler._isTaskReadyFunction()).toBe(false); // eslint-disable-line no-underscore-dangle- });+ const crawler = new Apify.BasicCrawler({+ requestQueue,+ handleRequestFunction: () => {},+ });- it('should be possible to override isFinishedFunction of underlying AutoscaledPool', async () => {- const requestQueue = new RequestQueue('xxx');- const processed = [];- const queue = [];- let isFinished = false;-- const basicCrawler = new Apify.BasicCrawler({- requestQueue,- autoscaledPoolOptions: {- minConcurrency: 1,- maxConcurrency: 1,- isFinishedFunction: () => {- return Promise.resolve(isFinished);+ expect(await crawler._isTaskReadyFunction()).toBe(false); // eslint-disable-line no-underscore-dangle+ }+ );++ test(+ 'should be possible to override isFinishedFunction of underlying AutoscaledPool',+ async () => {+ const requestQueue = new RequestQueue('xxx');+ const processed = [];+ const queue = [];+ let isFinished = false;++ const basicCrawler = new Apify.BasicCrawler({+ requestQueue,+ autoscaledPoolOptions: {+ minConcurrency: 1,+ maxConcurrency: 1,+ isFinishedFunction: () => {+ return Promise.resolve(isFinished);+ },},- },- handleRequestFunction: async ({ request }) => {- await delayPromise(10);- processed.push(request);- },- });+ handleRequestFunction: async ({ request }) => {+ await delayPromise(10);+ processed.push(request);+ },+ });- // Speed up the test- basicCrawler.autoscaledPoolOptions.maybeRunIntervalMillis = 50;+ // Speed up the test+ basicCrawler.autoscaledPoolOptions.maybeRunIntervalMillis = 50;- const request0 = new Apify.Request({ url: 'http://example.com/0' });- const request1 = new Apify.Request({ url: 'http://example.com/1' });+ const request0 = new Apify.Request({ url: 'http://example.com/0' });+ const request1 = new Apify.Request({ url: 'http://example.com/1' });- const mock = sinon.mock(requestQueue);- mock.expects('handledCount').once().returns(Promise.resolve());- mock.expects('markRequestHandled').once().withArgs(request0).returns(Promise.resolve());- mock.expects('markRequestHandled').once().withArgs(request1).returns(Promise.resolve());- mock.expects('isFinished').never();- requestQueue.fetchNextRequest = () => Promise.resolve(queue.pop());- requestQueue.isEmpty = () => Promise.resolve(!queue.length);+ const mock = sinon.mock(requestQueue);+ mock.expects('handledCount').once().returns(Promise.resolve());+ mock.expects('markRequestHandled').once().withArgs(request0).returns(Promise.resolve());+ mock.expects('markRequestHandled').once().withArgs(request1).returns(Promise.resolve());+ mock.expects('isFinished').never();+ requestQueue.fetchNextRequest = () => Promise.resolve(queue.pop());+ requestQueue.isEmpty = () => Promise.resolve(!queue.length);- setTimeout(() => queue.push(request0), 10);- setTimeout(() => queue.push(request1), 100);- setTimeout(() => { isFinished = true; }, 150);+ setTimeout(() => queue.push(request0), 10);+ setTimeout(() => queue.push(request1), 100);+ setTimeout(() => { isFinished = true; }, 150);- await basicCrawler.run();+ await basicCrawler.run();- expect(processed.includes(request0, request1)).toBe(true);+ expect(processed.includes(request0, request1)).toBe(true);- mock.verify();- sinon.restore();- });+ mock.verify();+ sinon.restore();+ }+ );- it('should support maxRequestsPerCrawl parameter', async () => {+ test('should support maxRequestsPerCrawl parameter', async () => {const sources = [{ url: 'http://example.com/1' },{ url: 'http://example.com/2' },@@ -496,7 +505,7 @@ describe('BasicCrawler', () => {expect(await requestList.isEmpty()).toBe(false);});- it('should load handledRequestCount from storages', async () => {+ test('should load handledRequestCount from storages', async () => {const requestQueue = new RequestQueue('id');requestQueue.isEmpty = async () => false;requestQueue.isFinished = async () => false;@@ -579,7 +588,7 @@ describe('BasicCrawler', () => {sinon.restore();});- it('should timeout after handleRequestTimeoutSecs', async () => {+ test('should timeout after handleRequestTimeoutSecs', async () => {const url = 'https://example.com';const requestList = new Apify.RequestList({ sources: [{ url }] });await requestList.initialize();diff --git a/__tests__/crawlers/cheerio_crawler.js b/test/crawlers/cheerio_crawler.test.jssimilarity index 95%rename from __tests__/crawlers/cheerio_crawler.jsrename to test/crawlers/cheerio_crawler.test.jsindex e3773f50c..42b364b4e 100644--- a/__tests__/crawlers/cheerio_crawler.js+++ b/test/crawlers/cheerio_crawler.test.js@@ -89,7 +89,7 @@ describe('CheerioCrawler', () => {};- before(async () => {+ beforeAll(async () => {logLevel = log.getLevel();log.setLevel(log.LEVELS.ERROR);const app = express();@@ -145,11 +145,11 @@ describe('CheerioCrawler', () => {port = server.address().port; //eslint-disable-line});- after(() => {+ afterAll(() => {log.setLevel(logLevel);});- it('should work', async () => {+ test('should work', async () => {const sources = [{ url: 'http://example.com/?q=1' },{ url: 'http://example.com/?q=2' },@@ -190,7 +190,7 @@ describe('CheerioCrawler', () => {});});- it('should trigger prepareRequestFunction', async () => {+ test('should trigger prepareRequestFunction', async () => {const MODIFIED_URL = 'http://example.com/?q=2';const sources = [{ url: 'http://example.com/' },@@ -222,16 +222,16 @@ describe('CheerioCrawler', () => {describe('should timeout', () => {let ll;- before(() => {+ beforeAll(() => {ll = log.getLevel();log.setLevel(log.LEVELS.OFF);});- after(() => {+ afterAll(() => {log.setLevel(ll);});- it('after requestTimeoutSecs', async () => {+ test('after requestTimeoutSecs', async () => {const sources = [{ url: 'http://example.com/?q=0' },{ url: 'http://example.com/?q=1' },@@ -272,7 +272,7 @@ describe('CheerioCrawler', () => {});});- it('after handlePageTimeoutSecs', async () => {+ test('after handlePageTimeoutSecs', async () => {const sources = [{ url: 'http://example.com/?q=0' },{ url: 'http://example.com/?q=1' },@@ -346,7 +346,7 @@ describe('CheerioCrawler', () => {});describe('should ensure text/html Content-Type', () => {- it('by setting a correct Accept header', async () => {+ test('by setting a correct Accept header', async () => {const headers = [];const requestList = await getRequestListForMirror();const crawler = new Apify.CheerioCrawler({@@ -382,7 +382,7 @@ describe('CheerioCrawler', () => {});- it('when invalid Content-Type header is received', async () => {+ test('when invalid Content-Type header is received', async () => {// Mock Request to inject invalid response headers.crawler = new Apify.CheerioCrawler({@@ -466,7 +466,7 @@ describe('CheerioCrawler', () => {errorMessages.forEach(msg => expect(msg).toEqual(expect.arrayContaining(['Request Error.'])));});- it('when statusCode >= 500 and text/html is received', async () => {+ test('when statusCode >= 500 and text/html is received', async () => {// sometimes if you get blocked you can get 500+ with some html insidecrawler = new Apify.CheerioCrawler({requestList: await getRequestListForMock({@@ -491,7 +491,7 @@ describe('CheerioCrawler', () => {errorMessages.forEach(msg => expect(msg).toEqual(expect.arrayContaining(['Internal Server Error'])));});- it('when statusCode >= 500 and application/json is received', async () => {+ test('when statusCode >= 500 and application/json is received', async () => {crawler = new Apify.CheerioCrawler({requestList: await getRequestListForMock({}, 'jsonError'),maxRequestRetries: 1,@@ -509,7 +509,7 @@ describe('CheerioCrawler', () => {errorMessages.forEach(msg => expect(msg).toEqual(expect.arrayContaining(['CUSTOM_ERROR'])));});- it('when 406 is received', async () => {+ test('when 406 is received', async () => {// Mock Request to respond with a 406.crawler = new Apify.CheerioCrawler({requestList: await getRequestListForMock({@@ -571,7 +571,7 @@ describe('CheerioCrawler', () => {});});- it('should work with all defaults content types', async () => {+ test('should work with all defaults content types', async () => {let handledRequests = 0;const contentTypes = ['text/html', 'application/xhtml+xml'];const sources = contentTypes.map(contentType => ({@@ -597,7 +597,7 @@ describe('CheerioCrawler', () => {describe('should works with all content types from options.additionalMimeTypes', () => {const handlePageInvocationParams = [];let handleFailedInvocationCount = 0;- before(async () => {+ beforeAll(async () => {const sources = [{ url: `http://${HOST}:${port}/json-type` },{ url: `http://${HOST}:${port}/xml-type` },@@ -621,20 +621,20 @@ describe('CheerioCrawler', () => {expect(handleFailedInvocationCount).toBe(0);expect(handlePageInvocationParams.length).toEqual(sources.length);});- it('when response is application/json', async () => {+ test('when response is application/json', async () => {const jsonRequestParams = handlePageInvocationParams[0];expect(jsonRequestParams.json).toBeInstanceOf(Object);expect(jsonRequestParams.body).toEqual(Buffer.from(JSON.stringify(responseSamples.json)));expect(jsonRequestParams.contentType.type).toBe('application/json');});- it('when response is application/xml', async () => {+ test('when response is application/xml', async () => {const xmlRequestParams = handlePageInvocationParams[1];expect(typeof xmlRequestParams.body).toBe('string');expect(xmlRequestParams.body).toEqual(responseSamples.xml);expect(xmlRequestParams.$).toBeInstanceOf(Function);expect(xmlRequestParams.contentType.type).toBe('application/xml');});- it('when response is image/png', async () => {+ test('when response is image/png', async () => {const imageRequestParams = handlePageInvocationParams[2];expect(imageRequestParams.body).toBeInstanceOf(Buffer);expect(imageRequestParams.body).toEqual(responseSamples.image);@@ -660,7 +660,7 @@ describe('CheerioCrawler', () => {requestList = null;});- it('should work with proxyUrls array', async () => {+ test('should work with proxyUrls array', async () => {const proxies = [];const crawler = new Apify.CheerioCrawler({requestList,@@ -686,7 +686,7 @@ describe('CheerioCrawler', () => {expect(proxies[3]).toEqual(shuffled[0]);});- it('should work with useApifyProxy', async () => {+ test('should work with useApifyProxy', async () => {process.env[ENV_VARS.PROXY_PASSWORD] = 'abc123';const proxies = [];const useApifyProxy = true;@@ -717,7 +717,7 @@ describe('CheerioCrawler', () => {expect(proxies[3]).toEqual(proxy);});- it('should work with useApifyProxy and other opts', async () => {+ test('should work with useApifyProxy and other opts', async () => {process.env[ENV_VARS.PROXY_PASSWORD] = 'abc123';const proxies = [];const useApifyProxy = true;@@ -764,7 +764,7 @@ describe('CheerioCrawler', () => {log.setLevel(log.LEVELS.ERROR);});- it('when proxyUrls is used together with useApifyProxy', async () => {+ test('when proxyUrls is used together with useApifyProxy', async () => {try {new Apify.CheerioCrawler({requestList,@@ -779,7 +779,7 @@ describe('CheerioCrawler', () => {}});- it('when proxyUrls array is empty', async () => {+ test('when proxyUrls array is empty', async () => {try {new Apify.CheerioCrawler({requestList,diff --git a/__tests__/crawlers/data/apify.png b/test/crawlers/data/apify.pngsimilarity index 100%rename from __tests__/crawlers/data/apify.pngrename to test/crawlers/data/apify.pngdiff --git a/__tests__/crawlers/data/sample.html b/test/crawlers/data/sample.htmlsimilarity index 100%rename from __tests__/crawlers/data/sample.htmlrename to test/crawlers/data/sample.htmldiff --git a/__tests__/crawlers/puppeteer_crawler.js b/test/crawlers/puppeteer_crawler.test.jssimilarity index 95%rename from __tests__/crawlers/puppeteer_crawler.jsrename to test/crawlers/puppeteer_crawler.test.jsindex 367c8ccaa..6a48cfef2 100644--- a/__tests__/crawlers/puppeteer_crawler.js+++ b/test/crawlers/puppeteer_crawler.test.js@@ -6,19 +6,19 @@ describe('PuppeteerCrawler', () => {let prevEnvHeadless;let logLevel;- before(() => {+ beforeAll(() => {prevEnvHeadless = process.env[ENV_VARS.HEADLESS];process.env[ENV_VARS.HEADLESS] = '1';logLevel = log.getLevel();log.setLevel(log.LEVELS.ERROR);});- after(() => {+ afterAll(() => {log.setLevel(logLevel);process.env[ENV_VARS.HEADLESS] = prevEnvHeadless;});- it('should work', async () => {+ test('should work', async () => {const sources = [{ url: 'http://example.com/?q=1' },{ url: 'http://example.com/?q=2' },@@ -58,7 +58,7 @@ describe('PuppeteerCrawler', () => {});});- it('should ignore errors in Page.close()', async () => {+ test('should ignore errors in Page.close()', async () => {for (let i = 0; i < 2; i++) {const requestList = new Apify.RequestList({sources: [diff --git a/__tests__/crawlers/statistics.js b/test/crawlers/statistics.test.jssimilarity index 93%rename from __tests__/crawlers/statistics.jsrename to test/crawlers/statistics.test.jsindex 59b221242..b56c1cd7a 100644--- a/__tests__/crawlers/statistics.js+++ b/test/crawlers/statistics.test.js@@ -21,7 +21,7 @@ describe('Statistics', () => {clock = null;});- it('should finish a job', () => {+ test('should finish a job', () => {stats.startJob(0);clock.tick(1);stats.finishJob(0);@@ -36,7 +36,7 @@ describe('Statistics', () => {});});- it('should fail a job', () => {+ test('should fail a job', () => {stats.startJob(0);clock.tick(0);stats.failJob(0);@@ -51,7 +51,7 @@ describe('Statistics', () => {});});- it('should collect retries', () => {+ test('should collect retries', () => {stats.startJob(0);stats.startJob(1);stats.startJob(2);@@ -71,7 +71,7 @@ describe('Statistics', () => {});});- it('should return correct stats for multiple parallel jobs', () => {+ test('should return correct stats for multiple parallel jobs', () => {stats.startJob(0);clock.tick(1);stats.startJob(1);@@ -95,7 +95,7 @@ describe('Statistics', () => {});});- it('should regularly log stats', () => {+ test('should regularly log stats', () => {const logged = [];sinon.stub(log, 'info').callsFake((...args) => {logged.push(args);diff --git a/__tests__/data/html_to_text_test.html b/test/data/html_to_text_test.htmlsimilarity index 100%rename from __tests__/data/html_to_text_test.htmlrename to test/data/html_to_text_test.htmldiff --git a/__tests__/data/html_to_text_test.txt b/test/data/html_to_text_test.txtsimilarity index 100%rename from __tests__/data/html_to_text_test.txtrename to test/data/html_to_text_test.txtdiff --git a/__tests__/data/inject_file.txt b/test/data/inject_file.txtsimilarity index 100%rename from __tests__/data/inject_file.txtrename to test/data/inject_file.txtdiff --git a/__tests__/data/invalid_url_list.txt b/test/data/invalid_url_list.txtsimilarity index 100%rename from __tests__/data/invalid_url_list.txtrename to test/data/invalid_url_list.txtdiff --git a/__tests__/data/lipsum.txt b/test/data/lipsum.txtsimilarity index 100%rename from __tests__/data/lipsum.txtrename to test/data/lipsum.txtdiff --git a/__tests__/data/simple_url_list.txt b/test/data/simple_url_list.txtsimilarity index 100%rename from __tests__/data/simple_url_list.txtrename to test/data/simple_url_list.txtdiff --git a/__tests__/data/tricky_url_list.txt b/test/data/tricky_url_list.txtsimilarity index 100%rename from __tests__/data/tricky_url_list.txtrename to test/data/tricky_url_list.txtdiff --git a/__tests__/data/unicode+comma_url_list.txt b/test/data/unicode+comma_url_list.txtsimilarity index 100%rename from __tests__/data/unicode+comma_url_list.txtrename to test/data/unicode+comma_url_list.txtdiff --git a/__tests__/data/unicode_url_list.txt b/test/data/unicode_url_list.txtsimilarity index 100%rename from __tests__/data/unicode_url_list.txtrename to test/data/unicode_url_list.txtdiff --git a/__tests__/dataset.js b/test/dataset.test.jssimilarity index 92%rename from __tests__/dataset.jsrename to test/dataset.test.jsindex 9ec9c60cc..033dc7d75 100644--- a/__tests__/dataset.js+++ b/test/dataset.test.js@@ -20,13 +20,13 @@ const read = (datasetName, index) => {};describe('dataset', () => {- before(() => apifyClient.setOptions({ token: 'xxx' }));- after(() => apifyClient.setOptions({ token: undefined }));+ beforeAll(() => apifyClient.setOptions({ token: 'xxx' }));+ afterAll(() => apifyClient.setOptions({ token: undefined }));beforeEach(() => emptyLocalStorageSubdir(LOCAL_STORAGE_SUBDIR));afterEach(() => emptyLocalStorageSubdir(LOCAL_STORAGE_SUBDIR));describe('local', async () => {- it('should successfully save data', async () => {+ test('should successfully save data', async () => {const dataset = new DatasetLocal('my-dataset', LOCAL_STORAGE_DIR);await dataset.pushData({ foo: 'bar' });@@ -60,7 +60,7 @@ describe('dataset', () => {return dataset;};- it('getData() should work', async () => {+ test('getData() should work', async () => {const dataset = await getLocalDataset([{ foo: 'a' },{ foo: 'b' },@@ -112,7 +112,7 @@ describe('dataset', () => {});});- it('getInfo() should work', async () => {+ test('getInfo() should work', async () => {const datasetName = 'stats-dataset';const dataset = new DatasetLocal(datasetName, LOCAL_STORAGE_DIR);await Apify.utils.sleep(2);@@ -184,7 +184,7 @@ describe('dataset', () => {if (originalUserId) process.env[ENV_VARS.USER_ID] = originalUserId;});- it('forEach() should work', async () => {+ test('forEach() should work', async () => {const dataset = await getLocalDataset([{ foo: 'a' },{ foo: 'b' },@@ -209,7 +209,7 @@ describe('dataset', () => {expect(indexes).toEqual([0, 1, 2, 3]);});- it('map() should work', async () => {+ test('map() should work', async () => {const dataset = await getLocalDataset([{ foo: 'a' },{ foo: 'b' },@@ -229,7 +229,7 @@ describe('dataset', () => {]);});- it('map() should support promises', async () => {+ test('map() should support promises', async () => {const dataset = await getLocalDataset([{ foo: 'a' },{ foo: 'b' },@@ -250,7 +250,7 @@ describe('dataset', () => {]);});- it('reduce() should work', async () => {+ test('reduce() should work', async () => {const dataset = await getLocalDataset([{ foo: 'a' },{ foo: 'b' },@@ -273,7 +273,7 @@ describe('dataset', () => {]);});- it('reduce() should support promises', async () => {+ test('reduce() should support promises', async () => {const dataset = await getLocalDataset([{ foo: 'a' },{ foo: 'b' },@@ -296,7 +296,7 @@ describe('dataset', () => {]);});- it('reduce() uses first value as memo if no memo is provided', async () => {+ test('reduce() uses first value as memo if no memo is provided', async () => {const dataset = await getLocalDataset([{ foo: 4 },{ foo: 5 },@@ -315,7 +315,7 @@ describe('dataset', () => {expect(calledForIndexes).toEqual([1, 2, 3]);});- it('deprecated delete() still works', async () => {+ test('deprecated delete() still works', async () => {const dataset = new DatasetLocal('to-delete', LOCAL_STORAGE_DIR);await dataset.pushData({ foo: 'bar' });@@ -329,7 +329,7 @@ describe('dataset', () => {describe('remote', async () => {const mockData = bytes => 'x'.repeat(bytes);- it('should succesfully save simple data', async () => {+ test('should succesfully save simple data', async () => {const dataset = new Dataset('some-id');const mock = sinon.mock(apifyClient.datasets);@@ -359,7 +359,7 @@ describe('dataset', () => {mock.restore();});- it('should successfully save large data', async () => {+ test('should successfully save large data', async () => {const half = mockData(MAX_PAYLOAD_SIZE_BYTES / 2);const dataset = new Dataset('some-id');@@ -390,7 +390,7 @@ describe('dataset', () => {mock.restore();});- it('should successfully save lots of small data', async () => {+ test('should successfully save lots of small data', async () => {const count = 20;const string = mockData(MAX_PAYLOAD_SIZE_BYTES / count);const chunk = { foo: string, bar: 'baz' };@@ -423,7 +423,7 @@ describe('dataset', () => {mock.restore();});- it('should throw on too large file', async () => {+ test('should throw on too large file', async () => {const mock = sinon.mock(apifyClient.datasets);const full = mockData(MAX_PAYLOAD_SIZE_BYTES);const dataset = new Dataset('some-id');@@ -442,7 +442,7 @@ describe('dataset', () => {mock.verify();mock.restore();});- it('should throw on too large file in an array', async () => {+ test('should throw on too large file in an array', async () => {const mock = sinon.mock(apifyClient.datasets);const full = mockData(MAX_PAYLOAD_SIZE_BYTES);const dataset = new Dataset('some-id', 'some-name');@@ -469,7 +469,7 @@ describe('dataset', () => {});- it('getData() should work', async () => {+ test('getData() should work', async () => {const dataset = new Dataset('some-id', 'some-name');const mock = sinon.mock(apifyClient.datasets);@@ -500,7 +500,7 @@ describe('dataset', () => {mock.restore();});- it('getInfo() should work', async () => {+ test('getInfo() should work', async () => {const dataset = new Dataset('some-id', 'some-name');const mock = sinon.mock(apifyClient.datasets);@@ -574,7 +574,7 @@ describe('dataset', () => {};- it('forEach() should work', async () => {+ test('forEach() should work', async () => {const { dataset, restoreAndVerify } = getRemoteDataset();const items = [];@@ -597,7 +597,7 @@ describe('dataset', () => {restoreAndVerify();});- it('map() should work', async () => {+ test('map() should work', async () => {const { dataset, restoreAndVerify } = getRemoteDataset();const result = await dataset.map((item, index) => {@@ -616,7 +616,7 @@ describe('dataset', () => {restoreAndVerify();});- it('map() should support promises', async () => {+ test('map() should support promises', async () => {const { dataset, restoreAndVerify } = getRemoteDataset();const result = await dataset.map((item, index) => {@@ -636,7 +636,7 @@ describe('dataset', () => {restoreAndVerify();});- it('reduce() should work', async () => {+ test('reduce() should work', async () => {const { dataset, restoreAndVerify } = getRemoteDataset();const result = await dataset.reduce((memo, item, index) => {@@ -658,7 +658,7 @@ describe('dataset', () => {restoreAndVerify();});- it('reduce() should support promises', async () => {+ test('reduce() should support promises', async () => {const { dataset, restoreAndVerify } = getRemoteDataset();const result = await dataset.reduce((memo, item, index) => {@@ -680,7 +680,7 @@ describe('dataset', () => {restoreAndVerify();});- it('reduce() uses first value as memo if no memo is provided', async () => {+ test('reduce() uses first value as memo if no memo is provided', async () => {const dataset = new Dataset('some-id', 'some-name');const mock = sinon.mock(apifyClient.datasets);@@ -732,7 +732,7 @@ describe('dataset', () => {expect(calledForIndexes).toEqual([1, 2, 3]);});- it('deprecated delete() still works', async () => {+ test('deprecated delete() still works', async () => {const mock = sinon.mock(apifyClient.datasets);const dataset = new Dataset('some-id', 'some-name');mock.expects('deleteDataset')@@ -747,7 +747,7 @@ describe('dataset', () => {});describe('Apify.openDataset', async () => {- it('should work', () => {+ test('should work', () => {const mock = sinon.mock(utils);process.env[ENV_VARS.LOCAL_STORAGE_DIR] = LOCAL_STORAGE_DIR;@@ -774,7 +774,7 @@ describe('dataset', () => {});describe('pushData', async () => {- it('throws on invalid args', async () => {+ test('throws on invalid args', async () => {process.env[ENV_VARS.DEFAULT_DATASET_ID] = 'some-id-8';process.env[ENV_VARS.LOCAL_STORAGE_DIR] = LOCAL_STORAGE_DIR;@@ -795,20 +795,23 @@ describe('dataset', () => {delete process.env[ENV_VARS.LOCAL_STORAGE_DIR];});- it('throws if DEFAULT_DATASET_ID env var is not defined and we use cloud storage', async () => {- delete process.env[ENV_VARS.LOCAL_STORAGE_DIR];- process.env[ENV_VARS.TOKEN] = 'xxx';+ test(+ 'throws if DEFAULT_DATASET_ID env var is not defined and we use cloud storage',+ async () => {+ delete process.env[ENV_VARS.LOCAL_STORAGE_DIR];+ process.env[ENV_VARS.TOKEN] = 'xxx';- process.env[ENV_VARS.DEFAULT_DATASET_ID] = '';- await expect(Apify.pushData({ something: 123 })).to.be.rejectedWith(Error);+ process.env[ENV_VARS.DEFAULT_DATASET_ID] = '';+ await expect(Apify.pushData({ something: 123 })).to.be.rejectedWith(Error);- delete process.env[ENV_VARS.DEFAULT_DATASET_ID];- await expect(Apify.pushData({ something: 123 })).to.be.rejectedWith(Error);+ delete process.env[ENV_VARS.DEFAULT_DATASET_ID];+ await expect(Apify.pushData({ something: 123 })).to.be.rejectedWith(Error);- delete process.env[ENV_VARS.TOKEN];- });+ delete process.env[ENV_VARS.TOKEN];+ }+ );- it('correctly stores records', async () => {+ test('correctly stores records', async () => {process.env[ENV_VARS.LOCAL_STORAGE_DIR] = LOCAL_STORAGE_DIR;process.env[ENV_VARS.DEFAULT_DATASET_ID] = 'some-id-9';@@ -824,7 +827,7 @@ describe('dataset', () => {});describe('utils', async () => {- it('checkAndSerialize() works', () => {+ test('checkAndSerialize() works', () => {// Basicconst obj = { foo: 'bar' };const json = JSON.stringify(obj);@@ -845,7 +848,7 @@ describe('dataset', () => {expect(() => checkAndSerialize([], 100)).toThrowError(Error);expect(() => checkAndSerialize([str, str], 100)).toThrowError(Error);});- it('chunkBySize', () => {+ test('chunkBySize', () => {const obj = { foo: 'bar' };const json = JSON.stringify(obj);const size = Buffer.byteLength(json);diff --git a/__tests__/enqueue_links/click_elements.js b/test/enqueue_links/click_elements.test.jssimilarity index 85%rename from __tests__/enqueue_links/click_elements.jsrename to test/enqueue_links/click_elements.test.jsindex 020769352..b4e59fcf8 100644--- a/__tests__/enqueue_links/click_elements.js+++ b/test/enqueue_links/click_elements.test.js@@ -6,11 +6,11 @@ describe('enqueueLinksByClickingElements()', () => {let browser;let page;- before(async () => {+ beforeAll(async () => {browser = await Apify.launchPuppeteer({ headless: true });});- after(async () => {+ afterAll(async () => {await browser.close();});@@ -22,7 +22,7 @@ describe('enqueueLinksByClickingElements()', () => {await page.close();});- it('should work', async () => {+ test('should work', async () => {const addedRequests = [];const requestQueue = Object.create(RequestQueue.prototype);requestQueue.addRequest = async request => addedRequests.push(request);@@ -53,7 +53,7 @@ describe('enqueueLinksByClickingElements()', () => {});describe('clickElements()', () => {- it('should click an element', async () => {+ test('should click an element', async () => {const html = `<html><body>@@ -67,7 +67,7 @@ describe('enqueueLinksByClickingElements()', () => {expect(wasClicked).toBe(true);});- it('should click an empty element', async () => {+ test('should click an empty element', async () => {const html = `<html><body>@@ -80,7 +80,7 @@ describe('enqueueLinksByClickingElements()', () => {const wasClicked = await page.evaluate(() => window.clicked);expect(wasClicked).toBe(true);});- it('should click multiple elements', async () => {+ test('should click multiple elements', async () => {const html = `<html><script>@@ -101,7 +101,7 @@ describe('enqueueLinksByClickingElements()', () => {expect(clickedElements).toEqual(['HEADER', 'DIV', 'P', 'FOOTER']);});- it('should click hidden elements', async () => {+ test('should click hidden elements', async () => {const html = `<html><body>@@ -115,7 +115,7 @@ describe('enqueueLinksByClickingElements()', () => {expect(wasClicked).toBe(true);});- it('should click elements stacked on top of each other', async () => {+ test('should click elements stacked on top of each other', async () => {const html = `<html><script>@@ -148,7 +148,7 @@ describe('enqueueLinksByClickingElements()', () => {};}- it('should intercept navigation by clicking a link', async () => {+ test('should intercept navigation by clicking a link', async () => {const html = `<html><body>@@ -165,7 +165,7 @@ describe('enqueueLinksByClickingElements()', () => {expect(page.url()).toBe('about:blank');});- it('should intercept navigation with window.location', async () => {+ test('should intercept navigation with window.location', async () => {const html = `<html><body>@@ -182,24 +182,27 @@ describe('enqueueLinksByClickingElements()', () => {expect(pageContent).toEqual(expect.arrayContaining(['onclick="return window.location = ']));});- it('should save the hash when changing it with window.location', async () => {- const html = `-<html>- <body>- <div onclick="return window.location = '#foo'">div</div>- </body>-</html>- `;- await page.goto('https://example.com');- await page.setContent(html);- const interceptedRequests = await clickElementsAndInterceptNavigationRequests(getOpts());- expect(interceptedRequests).toHaveLength(1);- expect(interceptedRequests[0].url).toBe('https://example.com/#foo');- const pageContent = await page.content();- expect(pageContent).toEqual(expect.arrayContaining(['onclick="return window.location = ']));- });+ test(+ 'should save the hash when changing it with window.location',+ async () => {+ const html = `+ <html>+ <body>+ <div onclick="return window.location = '#foo'">div</div>+ </body>+ </html>+ `;+ await page.goto('https://example.com');+ await page.setContent(html);+ const interceptedRequests = await clickElementsAndInterceptNavigationRequests(getOpts());+ expect(interceptedRequests).toHaveLength(1);+ expect(interceptedRequests[0].url).toBe('https://example.com/#foo');+ const pageContent = await page.content();+ expect(pageContent).toEqual(expect.arrayContaining(['onclick="return window.location = ']));+ }+ );- it('should prevent reload from cache with window.reload()', async () => {+ test('should prevent reload from cache with window.reload()', async () => {const html = `<html><body>@@ -216,7 +219,7 @@ describe('enqueueLinksByClickingElements()', () => {expect(pageContent).toEqual(expect.arrayContaining(['onclick="return window.location.reload()']));});- it('should prevent forced reload with window.reload(true)', async () => {+ test('should prevent forced reload with window.reload(true)', async () => {const html = `<html><body>@@ -233,7 +236,7 @@ describe('enqueueLinksByClickingElements()', () => {expect(pageContent).toEqual(expect.arrayContaining(['onclick="return window.location.reload(true)']));});- it('should prevent manipulation with window.history', async () => {+ test('should prevent manipulation with window.history', async () => {const html = `<html><script>@@ -259,7 +262,7 @@ describe('enqueueLinksByClickingElements()', () => {expect(pageContent).toEqual(expect.arrayContaining(['onclick="return handleClick();']));});- it('should save urls pushed to window.history', async () => {+ test('should save urls pushed to window.history', async () => {const html = `<html><body>@@ -276,7 +279,7 @@ describe('enqueueLinksByClickingElements()', () => {expect(pageContent).toEqual(expect.arrayContaining(['onclick="return window.history.pushState']));});- it('should close newly opened tabs', async () => {+ test('should close newly opened tabs', async () => {const html = `<html><body>@@ -311,7 +314,7 @@ describe('enqueueLinksByClickingElements()', () => {expect(pageContent).toEqual(expect.arrayContaining(['onclick="return window.open(']));});- it('should save urls from newly opened tabs', async () => {+ test('should save urls from newly opened tabs', async () => {const html = `<html><body>diff --git a/__tests__/enqueue_links/enqueue_links.js b/test/enqueue_links/enqueue_links.test.jssimilarity index 94%rename from __tests__/enqueue_links/enqueue_links.jsrename to test/enqueue_links/enqueue_links.test.jsindex 35e9311f9..068777102 100644--- a/__tests__/enqueue_links/enqueue_links.js+++ b/test/enqueue_links/enqueue_links.test.js@@ -32,12 +32,12 @@ const HTML = `describe('enqueueLinks()', () => {let ll;- before(() => {+ beforeAll(() => {ll = log.getLevel();log.setLevel(log.LEVELS.ERROR);});- after(() => {+ afterAll(() => {log.setLevel(ll);});@@ -57,7 +57,7 @@ describe('enqueueLinks()', () => {browser = null;});- it('works with PseudoUrl instances', async () => {+ test('works with PseudoUrl instances', async () => {const enqueued = [];const requestQueue = new RequestQueue('xxx');requestQueue.addRequest = async (request) => {@@ -85,7 +85,7 @@ describe('enqueueLinks()', () => {expect(enqueued[2].userData.foo).toBe('bar');});- it('works with Actor UI output object', async () => {+ test('works with Actor UI output object', async () => {const enqueued = [];const requestQueue = new RequestQueue('xxx');requestQueue.addRequest = async (request) => {@@ -113,7 +113,7 @@ describe('enqueueLinks()', () => {expect(enqueued[2].userData.foo).toBe('bar');});- it('works with string pseudoUrls', async () => {+ test('works with string pseudoUrls', async () => {const enqueued = [];const requestQueue = new RequestQueue('xxx');requestQueue.addRequest = async (request) => {@@ -141,7 +141,7 @@ describe('enqueueLinks()', () => {expect(enqueued[2].userData).toEqual({});});- it('works with RegExp pseudoUrls', async () => {+ test('works with RegExp pseudoUrls', async () => {const enqueued = [];const requestQueue = new RequestQueue('xxx');requestQueue.addRequest = async (request) => {@@ -169,7 +169,7 @@ describe('enqueueLinks()', () => {expect(enqueued[2].userData).toEqual({});});- it('works with undefined pseudoUrls[]', async () => {+ test('works with undefined pseudoUrls[]', async () => {const enqueued = [];const requestQueue = new RequestQueue('xxx');requestQueue.addRequest = async (request) => {@@ -197,7 +197,7 @@ describe('enqueueLinks()', () => {expect(enqueued[3].userData).toEqual({});});- it('works with null pseudoUrls[]', async () => {+ test('works with null pseudoUrls[]', async () => {const enqueued = [];const requestQueue = new RequestQueue('xxx');requestQueue.addRequest = async (request) => {@@ -225,7 +225,7 @@ describe('enqueueLinks()', () => {expect(enqueued[3].userData).toEqual({});});- it('works with empty pseudoUrls[]', async () => {+ test('works with empty pseudoUrls[]', async () => {const enqueued = [];const requestQueue = new RequestQueue('xxx');requestQueue.addRequest = async (request) => {@@ -253,7 +253,7 @@ describe('enqueueLinks()', () => {expect(enqueued[3].userData).toEqual({});});- it('throws with sparse pseudoUrls[]', async () => {+ test('throws with sparse pseudoUrls[]', async () => {const enqueued = [];const requestQueue = new RequestQueue('xxx');requestQueue.addRequest = async (request) => {@@ -274,7 +274,7 @@ describe('enqueueLinks()', () => {}});- it('DEPRECATED: enqueueRequestsFromClickableElements()', async () => {+ test('DEPRECATED: enqueueRequestsFromClickableElements()', async () => {const enqueuedUrls = [];const queue = new RequestQueue('xxx');queue.addRequest = (request) => {@@ -309,7 +309,7 @@ describe('enqueueLinks()', () => {$ = null;});- it('works from utils namespace', async () => {+ test('works from utils namespace', async () => {const enqueued = [];const requestQueue = new RequestQueue('xxx');requestQueue.addRequest = async (request) => {@@ -337,7 +337,7 @@ describe('enqueueLinks()', () => {expect(enqueued[2].userData.foo).toBe('bar');});- it('works with PseudoUrl instances', async () => {+ test('works with PseudoUrl instances', async () => {const enqueued = [];const requestQueue = new RequestQueue('xxx');requestQueue.addRequest = async (request) => {@@ -365,7 +365,7 @@ describe('enqueueLinks()', () => {expect(enqueued[2].userData.foo).toBe('bar');});- it('works with Actor UI output object', async () => {+ test('works with Actor UI output object', async () => {const enqueued = [];const requestQueue = new RequestQueue('xxx');requestQueue.addRequest = async (request) => {@@ -393,7 +393,7 @@ describe('enqueueLinks()', () => {expect(enqueued[2].userData.foo).toBe('bar');});- it('works with string pseudoUrls', async () => {+ test('works with string pseudoUrls', async () => {const enqueued = [];const requestQueue = new RequestQueue('xxx');requestQueue.addRequest = async (request) => {@@ -421,7 +421,7 @@ describe('enqueueLinks()', () => {expect(enqueued[2].userData).toEqual({});});- it('works with RegExp pseudoUrls', async () => {+ test('works with RegExp pseudoUrls', async () => {const enqueued = [];const requestQueue = new RequestQueue('xxx');requestQueue.addRequest = async (request) => {@@ -449,7 +449,7 @@ describe('enqueueLinks()', () => {expect(enqueued[2].userData).toEqual({});});- it('works with undefined pseudoUrls[]', async () => {+ test('works with undefined pseudoUrls[]', async () => {const enqueued = [];const requestQueue = new RequestQueue('xxx');requestQueue.addRequest = async (request) => {@@ -477,7 +477,7 @@ describe('enqueueLinks()', () => {expect(enqueued[3].userData).toEqual({});});- it('works with null pseudoUrls[]', async () => {+ test('works with null pseudoUrls[]', async () => {const enqueued = [];const requestQueue = new RequestQueue('xxx');requestQueue.addRequest = async (request) => {@@ -505,7 +505,7 @@ describe('enqueueLinks()', () => {expect(enqueued[3].userData).toEqual({});});- it('works with empty pseudoUrls[]', async () => {+ test('works with empty pseudoUrls[]', async () => {const enqueued = [];const requestQueue = new RequestQueue('xxx');requestQueue.addRequest = async (request) => {@@ -533,7 +533,7 @@ describe('enqueueLinks()', () => {expect(enqueued[3].userData).toEqual({});});- it('throws with sparse pseudoUrls[]', async () => {+ test('throws with sparse pseudoUrls[]', async () => {const enqueued = [];const requestQueue = new RequestQueue('xxx');requestQueue.addRequest = async (request) => {@@ -554,7 +554,7 @@ describe('enqueueLinks()', () => {}});- it('correctly resolves relative URLs', async () => {+ test('correctly resolves relative URLs', async () => {const enqueued = [];const requestQueue = new RequestQueue('xxx');requestQueue.addRequest = async (request) => {@@ -594,7 +594,7 @@ describe('enqueueLinks()', () => {expect(enqueued[6].userData).toEqual({});});- it('throws on finding a relative link with no baseUrl set', async () => {+ test('throws on finding a relative link with no baseUrl set', async () => {const enqueued = [];const requestQueue = new RequestQueue('xxx');requestQueue.addRequest = async (request) => {diff --git a/__tests__/enqueue_links/shared.js b/test/enqueue_links/shared.test.jssimilarity index 95%rename from __tests__/enqueue_links/shared.jsrename to test/enqueue_links/shared.test.jsindex a68aa7691..1d2dd7089 100644--- a/__tests__/enqueue_links/shared.js+++ b/test/enqueue_links/shared.test.js@@ -3,7 +3,7 @@ import * as shared from '../../build/enqueue_links/shared';describe('Enqueue links shared functions', () => {describe('constructPseudoUrlInstances()', () => {- it('should work', () => {+ test('should work', () => {const pseudoUrlSources = [new PseudoUrl(/^https?:\/\/example\.com/, { userData: { foo: 'bar' } }),/^https?:\/\/example\.com/,@@ -21,7 +21,7 @@ describe('Enqueue links shared functions', () => {expect(request.userData).toEqual({ foo: 'bar' });});- it('should cache items', () => {+ test('should cache items', () => {const pseudoUrls = shared.constructPseudoUrlInstances(['http[s?]://example.com/[.*]']);const pseudoUrls2 = shared.constructPseudoUrlInstances(['http[s?]://example.com/[.*]']);expect(pseudoUrls[0] === pseudoUrls2[0]).toBe(true);@@ -29,7 +29,7 @@ describe('Enqueue links shared functions', () => {});describe('createRequests()', () => {- it('should work', () => {+ test('should work', () => {const sources = ['http://example.com/foo',{ url: 'https://example.com/bar', method: 'POST' },@@ -53,7 +53,7 @@ describe('Enqueue links shared functions', () => {});describe('addRequestsToQueueInBatches()', () => {- it('should work', async () => {+ test('should work', async () => {const fakeRequestQueue = {requests: [],async addRequest(request) {diff --git a/__tests__/events.js b/test/events.test.jssimilarity index 95%rename from __tests__/events.jsrename to test/events.test.jsindex 0dd4368c3..5765d8e34 100644--- a/__tests__/events.js+++ b/test/events.test.js@@ -7,7 +7,7 @@ import { ACTOR_EVENT_NAMES_EX } from '../build/constants';import Apify from '../build';describe('Apify.events', () => {- it('is there and works as EventEmitter', () => {+ test('is there and works as EventEmitter', () => {return new Promise((resolve, reject) => {try {Apify.events.on('foo', resolve);@@ -21,7 +21,7 @@ describe('Apify.events', () => {});});- it('should work in Apify.main()', (done) => {+ test('should work in Apify.main()', (done) => {const wss = new WebSocket.Server({ port: 9099 });const eventsReceived = [];let isWsConnected = false;@@ -74,7 +74,7 @@ describe('Apify.events', () => {});});- it('should work without Apify.main()', async () => {+ test('should work without Apify.main()', async () => {const wss = new WebSocket.Server({ port: 9099 });const eventsReceived = [];let isWsConnected = false;@@ -118,7 +118,7 @@ describe('Apify.events', () => {delete process.env[ENV_VARS.ACTOR_EVENTS_WS_URL];});- it('should send persist state events in regular interval', async () => {+ test('should send persist state events in regular interval', async () => {process.env.APIFY_TEST_PERSIST_INTERVAL_MILLIS = 1;const eventsReceived = [];diff --git a/__tests__/key_value_store.js b/test/key_value_store.test.jssimilarity index 89%rename from __tests__/key_value_store.jsrename to test/key_value_store.test.jsindex dec607346..fc1811da3 100644--- a/__tests__/key_value_store.js+++ b/test/key_value_store.test.js@@ -9,13 +9,13 @@ import { LOCAL_STORAGE_DIR, emptyLocalStorageSubdir, expectDirEmpty, expectDirNoconst { apifyClient } = utils;describe('KeyValueStore', () => {- before(() => apifyClient.setOptions({ token: 'xxx' }));- after(() => apifyClient.setOptions({ token: undefined }));+ beforeAll(() => apifyClient.setOptions({ token: 'xxx' }));+ afterAll(() => apifyClient.setOptions({ token: undefined }));beforeEach(() => emptyLocalStorageSubdir(LOCAL_STORAGE_SUBDIR));afterEach(() => emptyLocalStorageSubdir(LOCAL_STORAGE_SUBDIR));describe('maybeStringify()', () => {- it('should work', () => {+ test('should work', () => {expect(maybeStringify({ foo: 'bar' }, { contentType: null })).toBe('{\n "foo": "bar"\n}');expect(maybeStringify({ foo: 'bar' }, { contentType: undefined })).toBe('{\n "foo": "bar"\n}');@@ -31,7 +31,7 @@ describe('KeyValueStore', () => {});describe('getFileNameRegexp()', () => {- it('should work', () => {+ test('should work', () => {const key = 'hel.lo';const filenames = ['hel.lo.txt', // valid@@ -51,7 +51,7 @@ describe('KeyValueStore', () => {});describe('local', async () => {- it('should work', async () => {+ test('should work', async () => {const store = new KeyValueStoreLocal('my-store-id', LOCAL_STORAGE_DIR);const store2 = new KeyValueStoreLocal('another-store-id', LOCAL_STORAGE_DIR);const buffer = Buffer.from('some text value');@@ -100,7 +100,7 @@ describe('KeyValueStore', () => {expectDirEmpty(storeDir);});- it('deprecated delete() still works', async () => {+ test('deprecated delete() still works', async () => {const kvs = new KeyValueStoreLocal('to-delete', LOCAL_STORAGE_DIR);await kvs.setValue('dummy', { foo: 'bar' });@@ -112,7 +112,7 @@ describe('KeyValueStore', () => {});describe('remote', async () => {- it('works', async () => {+ test('works', async () => {const store = new KeyValueStore('some-id-1');const mock = sinon.mock(apifyClient.keyValueStores);const record = { foo: 'bar' };@@ -164,7 +164,7 @@ describe('KeyValueStore', () => {mock.restore();});- it('deprecated delete() still works', async () => {+ test('deprecated delete() still works', async () => {const mock = sinon.mock(apifyClient.keyValueStores);const kvs = new KeyValueStore('some-id', 'some-name');mock.expects('deleteStore')@@ -179,7 +179,7 @@ describe('KeyValueStore', () => {});describe('Apify.openKeyValueStore', async () => {- it('should work', async () => {+ test('should work', async () => {const mock = sinon.mock(utils);process.env[ENV_VARS.LOCAL_STORAGE_DIR] = LOCAL_STORAGE_DIR;@@ -206,7 +206,7 @@ describe('KeyValueStore', () => {});describe('getValue', async () => {- it('throws on invalid args', async () => {+ test('throws on invalid args', async () => {process.env[ENV_VARS.DEFAULT_KEY_VALUE_STORE_ID] = '1234';process.env[ENV_VARS.LOCAL_STORAGE_DIR] = LOCAL_STORAGE_DIR;await expect(Apify.getValue()).to.be.rejectedWith('Parameter "key" of type String must be provided');@@ -216,20 +216,23 @@ describe('KeyValueStore', () => {delete process.env[ENV_VARS.LOCAL_STORAGE_DIR];});- it('throws if APIFY_DEFAULT_KEY_VALUE_STORE_ID env var is not defined and we use cloud storage', async () => {- delete process.env[ENV_VARS.LOCAL_STORAGE_DIR];- delete process.env[ENV_VARS.DEFAULT_KEY_VALUE_STORE_ID];- process.env[ENV_VARS.TOKEN] = 'xxx';+ test(+ 'throws if APIFY_DEFAULT_KEY_VALUE_STORE_ID env var is not defined and we use cloud storage',+ async () => {+ delete process.env[ENV_VARS.LOCAL_STORAGE_DIR];+ delete process.env[ENV_VARS.DEFAULT_KEY_VALUE_STORE_ID];+ process.env[ENV_VARS.TOKEN] = 'xxx';- const errMsg = 'The \'APIFY_DEFAULT_KEY_VALUE_STORE_ID\' environment variable is not defined';- await expect(Apify.getValue('KEY')).to.be.rejectedWith(errMsg);+ const errMsg = 'The \'APIFY_DEFAULT_KEY_VALUE_STORE_ID\' environment variable is not defined';+ await expect(Apify.getValue('KEY')).to.be.rejectedWith(errMsg);- delete process.env[ENV_VARS.TOKEN];- });+ delete process.env[ENV_VARS.TOKEN];+ }+ );});describe('setValue', async () => {- it('throws on invalid args', async () => {+ test('throws on invalid args', async () => {process.env[ENV_VARS.DEFAULT_KEY_VALUE_STORE_ID] = '12345';process.env[ENV_VARS.LOCAL_STORAGE_DIR] = LOCAL_STORAGE_DIR;@@ -274,7 +277,7 @@ describe('KeyValueStore', () => {delete process.env[ENV_VARS.LOCAL_STORAGE_DIR];});- it('throws on invalid key', async () => {+ test('throws on invalid key', async () => {const store = new KeyValueStoreLocal('my-store-id', LOCAL_STORAGE_DIR);const INVALID_CHARACTERS = '?|\\/"*<>%:';let counter = 0;@@ -300,18 +303,21 @@ describe('KeyValueStore', () => {}});- it('throws if APIFY_DEFAULT_KEY_VALUE_STORE_ID env var is not defined and we use cloud storage', async () => {- delete process.env[ENV_VARS.DEFAULT_KEY_VALUE_STORE_ID];- delete process.env[ENV_VARS.LOCAL_STORAGE_DIR];- process.env[ENV_VARS.TOKEN] = 'xxx';+ test(+ 'throws if APIFY_DEFAULT_KEY_VALUE_STORE_ID env var is not defined and we use cloud storage',+ async () => {+ delete process.env[ENV_VARS.DEFAULT_KEY_VALUE_STORE_ID];+ delete process.env[ENV_VARS.LOCAL_STORAGE_DIR];+ process.env[ENV_VARS.TOKEN] = 'xxx';- const errMsg = 'The \'APIFY_DEFAULT_KEY_VALUE_STORE_ID\' environment variable is not defined';- await expect(Apify.setValue('KEY', {})).to.be.rejectedWith(errMsg);+ const errMsg = 'The \'APIFY_DEFAULT_KEY_VALUE_STORE_ID\' environment variable is not defined';+ await expect(Apify.setValue('KEY', {})).to.be.rejectedWith(errMsg);- delete process.env[ENV_VARS.TOKEN];- });+ delete process.env[ENV_VARS.TOKEN];+ }+ );- it('correctly adds charset to content type', async () => {+ test('correctly adds charset to content type', async () => {const store = new KeyValueStore('some-id-1');const mock = sinon.mock(apifyClient.keyValueStores);@@ -329,7 +335,7 @@ describe('KeyValueStore', () => {mock.restore();});- it('correctly passes object values as JSON', async () => {+ test('correctly passes object values as JSON', async () => {const store = new KeyValueStore('some-id-1', 'some-name-1');const mock = sinon.mock(apifyClient.keyValueStores);const record = { foo: 'bar' };@@ -349,7 +355,7 @@ describe('KeyValueStore', () => {mock.restore();});- it('correctly passes raw string values', async () => {+ test('correctly passes raw string values', async () => {const store = new KeyValueStore('some-id-1', 'some-name-1');const mock = sinon.mock(apifyClient.keyValueStores);@@ -367,7 +373,7 @@ describe('KeyValueStore', () => {mock.restore();});- it('correctly passes raw Buffer values', async () => {+ test('correctly passes raw Buffer values', async () => {const store = new KeyValueStore('some-id-1', 'some-name-1');const mock = sinon.mock(apifyClient.keyValueStores);const value = Buffer.from('some text value');@@ -389,14 +395,14 @@ describe('KeyValueStore', () => {describe('getPublicUrl', () => {- it('should return the local url of a file', () => {+ test('should return the local url of a file', () => {process.env[ENV_VARS.LOCAL_STORAGE_DIR] = LOCAL_STORAGE_DIR;const store = new KeyValueStoreLocal('my-store-id', LOCAL_STORAGE_DIR);expect(store.getPublicUrl('file.txt')).toBe(`file://${store.localStoragePath}/file.txt`);delete process.env[ENV_VARS.LOCAL_STORAGE_DIR];});- it('should return the url of a file in apify cloud', async () => {+ test('should return the url of a file in apify cloud', async () => {process.env[ENV_VARS.TOKEN] = 'xxx';const publicUrl = 'https://api.apify.com/v2/key-value-stores';const store = new KeyValueStore('my-store-id');@@ -407,7 +413,7 @@ describe('KeyValueStore', () => {});describe('forEachKey', () => {- it('should work remotely', async () => {+ test('should work remotely', async () => {const storeId = 'some-id-1';const store = new KeyValueStore(storeId, 'some-name-1');const mock = sinon.mock(apifyClient.keyValueStores);@@ -467,7 +473,7 @@ describe('KeyValueStore', () => {mock.verify();});- it('should work locally', async () => {+ test('should work locally', async () => {const storeId = 'some-id-1';const store = new KeyValueStoreLocal(storeId, LOCAL_STORAGE_DIR);@@ -496,7 +502,7 @@ describe('KeyValueStore', () => {});describe('getInput', async () => {- it('should work', async () => {+ test('should work', async () => {process.env[ENV_VARS.LOCAL_STORAGE_DIR] = LOCAL_STORAGE_DIR;const defaultStore = await Apify.openKeyValueStore();// Uses default value.diff --git a/__tests__/live_view/live_view_server.js b/test/live_view/live_view_server.test.jssimilarity index 90%rename from __tests__/live_view/live_view_server.jsrename to test/live_view/live_view_server.test.jsindex b26585311..36edd7a04 100644--- a/__tests__/live_view/live_view_server.js+++ b/test/live_view/live_view_server.test.js@@ -17,12 +17,12 @@ const PORT = LOCAL_ENV_VARS[ENV_VARS.CONTAINER_PORT];const BASE_URL = `http://localhost:${PORT}`;let originalLogLevel;-before(() => {+beforeAll(() => {originalLogLevel = log.getLevel();log.setLevel(log.LEVELS.ERROR);});-after(() => {+afterAll(() => {log.setLevel(originalLogLevel);});@@ -49,18 +49,18 @@ describe('LiveViewServer', () => {await emptyDir(LOCAL_STORAGE_SUBDIR);});- it('should construct', async () => {+ test('should construct', async () => {expect(lvs).toBeInstanceOf(LiveViewServer);});- it('should start and stop', async () => {+ test('should start and stop', async () => {await lvs.start();expect(lvs.isRunning()).toBe(true);await lvs.stop();expect(lvs.isRunning()).toBe(false);});- it('should connect over websocket', async () => {+ test('should connect over websocket', async () => {const socket = io(BASE_URL);await lvs.start();await new Promise(resolve => socket.on('connect', resolve));@@ -69,7 +69,7 @@ describe('LiveViewServer', () => {await lvs.stop();});- it('should make one snapshot even without clients', async () => {+ test('should make one snapshot even without clients', async () => {await lvs.start();expect(lvs.hasClients()).toBe(true);await lvs.serve(fakePage);@@ -96,14 +96,14 @@ describe('LiveViewServer', () => {socket = null;});- it('should serve snapshot', async () => {+ test('should serve snapshot', async () => {await lvs.serve(fakePage);const snapshot = await new Promise(resolve => socket.on('snapshot', resolve));expect(snapshot).toMatchObject({ pageUrl: 'url', htmlContent: 'content', screenshotIndex: 0 });expect(`"${snapshot.createdAt}"`).toEqual(JSON.stringify(new Date(snapshot.createdAt)));});- it('should return screenshots', async () => {+ test('should return screenshots', async () => {const snapshot0 = new Promise(resolve => socket.on('snapshot', resolve));await lvs.serve(fakePage);const response0 = await rqst(`${BASE_URL}/screenshot/${((await snapshot0).screenshotIndex)}`);@@ -114,7 +114,7 @@ describe('LiveViewServer', () => {expect(response1).toBe('screenshot1');});- it('should not store more than maxScreenshotFiles screenshots', async () => {+ test('should not store more than maxScreenshotFiles screenshots', async () => {const snapshots = [];socket.on('snapshot', s => snapshots.push(s));for (let i = 0; i < 5; i++) {diff --git a/__tests__/pseudo_url.js b/test/pseudo_url.test.jssimilarity index 90%rename from __tests__/pseudo_url.jsrename to test/pseudo_url.test.jsindex c6573f2e2..819a9b84c 100644--- a/__tests__/pseudo_url.js+++ b/test/pseudo_url.test.js@@ -1,7 +1,7 @@import Apify from '../build/index';describe('Apify.PseudoUrl', () => {- it('matches() should work', () => {+ test('matches() should work', () => {let purl = new Apify.PseudoUrl('http://www.example.com/PAGES/[(\\w|-)*]'); // eslint-disable-lineexpect(purl.matches('http://www.example.com/PAGES/')).toBe(true);@@ -15,7 +15,7 @@ describe('Apify.PseudoUrl', () => {expect(purl.matches('http://www.example.com/pages/not@working')).toBe(true);});- it('createRequest() should work with a string', () => {+ test('createRequest() should work with a string', () => {const purl = new Apify.PseudoUrl('something', { method: 'POST', userData: { foo: 'bar' } });const request = purl.createRequest('http://example.com');@@ -25,7 +25,7 @@ describe('Apify.PseudoUrl', () => {expect(request.userData).toEqual({ foo: 'bar' });});- it('createRequest() should work with an object', () => {+ test('createRequest() should work with an object', () => {const purl = new Apify.PseudoUrl('something', { method: 'POST', userData: { foo: 'bar' } });const request = purl.createRequest({url: 'http://example.com',diff --git a/__tests__/puppeteer.js b/test/puppeteer.test.jssimilarity index 91%rename from __tests__/puppeteer.jsrename to test/puppeteer.test.jsindex 22c6d1eef..8fbfe636f 100644--- a/__tests__/puppeteer.js+++ b/test/puppeteer.test.js@@ -1,4 +1,3 @@-import chai, { expect } from 'chai';import chaiAsPromised from 'chai-as-promised';import proxy from 'proxy';import http from 'http';@@ -21,7 +20,7 @@ const proxyAuth = { scheme: 'Basic', username: 'username', password: 'password'let wasProxyCalled = false; // eslint-disable-line no-unused-vars// Setup local proxy server for the tests-before(() => {+beforeAll(() => {prevEnvHeadless = process.env[ENV_VARS.HEADLESS];process.env[ENV_VARS.HEADLESS] = '1';@@ -58,7 +57,7 @@ before(() => {});});-after(function () {+afterAll(() => {process.env[ENV_VARS.HEADLESS] = prevEnvHeadless;this.timeout(5 * 1000);@@ -67,7 +66,7 @@ after(function () {describe('Apify.launchPuppeteer()', () => {- it('throws on invalid args', () => {+ test('throws on invalid args', () => {expect(Apify.launchPuppeteer('some non-object')).to.be.rejectedWith(Error);expect(Apify.launchPuppeteer(1234)).to.be.rejectedWith(Error);@@ -86,7 +85,7 @@ describe('Apify.launchPuppeteer()', () => {expect(Apify.launchPuppeteer({ args: [12, 34] })).to.be.rejectedWith(Error);});- it('opens supports non-HTTP proxies without authentication', async () => {+ test('opens supports non-HTTP proxies without authentication', async () => {const browser1 = await Apify.launchPuppeteer({ proxyUrl: 'socks4://example.com:1234' });browser1.close();@@ -101,7 +100,7 @@ describe('Apify.launchPuppeteer()', () => {});- it('opens https://www.example.com', () => {+ test('opens https://www.example.com', () => {let browser;let page;@@ -122,7 +121,7 @@ describe('Apify.launchPuppeteer()', () => {.then(() => browser.close());});- it('opens https://www.example.com via proxy with authentication', () => {+ test('opens https://www.example.com via proxy with authentication', () => {let browser;let page;@@ -152,7 +151,7 @@ describe('Apify.launchPuppeteer()', () => {.then(() => browser.close());});- it('supports userAgent option', () => {+ test('supports userAgent option', () => {let browser;let page;const opts = {@@ -180,7 +179,7 @@ describe('Apify.launchPuppeteer()', () => {});});- it('supports useChrome option', async () => {+ test('supports useChrome option', async () => {const spy = sinon.spy(utils, 'getTypicalChromeExecutablePath');let browser;@@ -209,7 +208,7 @@ describe('Apify.launchPuppeteer()', () => {}});- it('should allow to use Apify proxy', async () => {+ test('should allow to use Apify proxy', async () => {process.env[ENV_VARS.PROXY_PASSWORD] = 'abc123';process.env[ENV_VARS.PROXY_HOSTNAME] = 'my.host.com';process.env[ENV_VARS.PROXY_PORT] = 123;@@ -243,14 +242,17 @@ describe('Apify.launchPuppeteer()', () => {}});- it('should throw when useApifyProxy=true and proxy password is not set', () => {- const opts = {- useApifyProxy: true,- apifyProxySession: 'xxx',- apifyProxyGroups: ['yyy'],- headless: true,- };+ test(+ 'should throw when useApifyProxy=true and proxy password is not set',+ () => {+ const opts = {+ useApifyProxy: true,+ apifyProxySession: 'xxx',+ apifyProxyGroups: ['yyy'],+ headless: true,+ };- expect(Apify.launchPuppeteer(opts)).to.be.rejectedWith(Error);- });+ expect(Apify.launchPuppeteer(opts)).to.be.rejectedWith(Error);+ }+ );});diff --git a/__tests__/puppeteer_pool.js b/test/puppeteer_pool.test.jssimilarity index 97%rename from __tests__/puppeteer_pool.jsrename to test/puppeteer_pool.test.jsindex f57923dfe..5b08c368c 100644--- a/__tests__/puppeteer_pool.js+++ b/test/puppeteer_pool.test.js@@ -11,19 +11,19 @@ describe('PuppeteerPool', () => {let prevEnvHeadless;let logLevel;- before(() => {+ beforeAll(() => {logLevel = log.getLevel();log.setLevel(log.LEVELS.ERROR);prevEnvHeadless = process.env[ENV_VARS.HEADLESS];process.env[ENV_VARS.HEADLESS] = '1';});- after(() => {+ afterAll(() => {log.setLevel(logLevel);process.env[ENV_VARS.HEADLESS] = prevEnvHeadless;});- it('should work', async () => {+ test('should work', async () => {process.env[ENV_VARS.PROXY_PASSWORD] = 'abc123';process.env[ENV_VARS.PROXY_HOSTNAME] = 'my.host.com';process.env[ENV_VARS.PROXY_PORT] = 123;@@ -99,7 +99,7 @@ describe('PuppeteerPool', () => {delete process.env[ENV_VARS.PROXY_PORT];});- it('kills hanging retired instances', async () => {+ test('kills hanging retired instances', async () => {process.env[ENV_VARS.PROXY_PASSWORD] = 'abc123';process.env[ENV_VARS.PROXY_HOSTNAME] = 'my.host.com';process.env[ENV_VARS.PROXY_PORT] = 123;@@ -150,7 +150,7 @@ describe('PuppeteerPool', () => {delete process.env[ENV_VARS.PROXY_PORT];});- it('retire manually', async () => {+ test('retire manually', async () => {process.env[ENV_VARS.PROXY_PASSWORD] = 'abc123';process.env[ENV_VARS.PROXY_HOSTNAME] = 'my.host.com';process.env[ENV_VARS.PROXY_PORT] = 123;@@ -193,7 +193,7 @@ describe('PuppeteerPool', () => {delete process.env[ENV_VARS.PROXY_PORT];});- it('works with one page per instance', async () => {+ test('works with one page per instance', async () => {const pool = new Apify.PuppeteerPool({maxOpenPagesPerInstance: 1,retireInstanceAfterRequestCount: 1,@@ -225,7 +225,7 @@ describe('PuppeteerPool', () => {await pool.destroy();});- it('supports recycleDiskCache option', async () => {+ test('supports recycleDiskCache option', async () => {const pool = new Apify.PuppeteerPool({maxOpenPagesPerInstance: 1,retireInstanceAfterRequestCount: 1,@@ -472,7 +472,7 @@ describe('PuppeteerPool', () => {});describe('useIncognitoPages', () => {- it('opens a page in incognito browser context', async () => {+ test('opens a page in incognito browser context', async () => {const pool = new Apify.PuppeteerPool({useIncognitoPages: true,});@@ -489,7 +489,7 @@ describe('PuppeteerPool', () => {});describe('the proxyUrls parameter', () => {- it('supports rotation of custom proxies', async () => {+ test('supports rotation of custom proxies', async () => {const optionsLog = [];const pool = new Apify.PuppeteerPool({maxOpenPagesPerInstance: 1,@@ -524,7 +524,7 @@ describe('PuppeteerPool', () => {if (pool) await pool.destroy();});- it('when used with useApifyProxy', async () => {+ test('when used with useApifyProxy', async () => {pool = new Apify.PuppeteerPool({maxOpenPagesPerInstance: 1,proxyUrls: ['http://proxy.com:1111', 'http://proxy.com:2222', 'http://proxy.com:3333'],@@ -542,7 +542,7 @@ describe('PuppeteerPool', () => {}});- it('when empty', async () => {+ test('when empty', async () => {try {pool = new Apify.PuppeteerPool({maxOpenPagesPerInstance: 1,@@ -571,7 +571,7 @@ describe('PuppeteerPool', () => {await pool.destroy();});- it('should work', async () => {+ test('should work', async () => {// Start browser;await pool._openNewTab(); // eslint-disable-line no-underscore-danglepool.puppeteerOperationTimeoutMillis = 0.005;@@ -585,13 +585,13 @@ describe('PuppeteerPool', () => {});describe('enables LiveView', () => {- before(() => {+ beforeAll(() => {process.env[ENV_VARS.IS_AT_HOME] = '1';});- after(() => {+ afterAll(() => {delete process.env[ENV_VARS.IS_AT_HOME];});- it('should work', async () => {+ test('should work', async () => {const serveCalledWith = [];let started = 0;let stopped = 0;diff --git a/__tests__/puppeteer_request_interception.js b/test/puppeteer_request_interception.test.jssimilarity index 77%rename from __tests__/puppeteer_request_interception.jsrename to test/puppeteer_request_interception.test.jsindex eca97d66f..6a9f661d2 100644--- a/__tests__/puppeteer_request_interception.js+++ b/test/puppeteer_request_interception.test.js@@ -10,7 +10,7 @@ const HTML_PAGE = `<html><body></body></html>`;describe('Apify.utils.puppeteer.addInterceptRequestHandler|removeInterceptRequestHandler()', () => {- it('should allow multiple handlers', async () => {+ test('should allow multiple handlers', async () => {const browser = await Apify.launchPuppeteer({ headless: true });const allUrls = [];@@ -56,45 +56,48 @@ describe('Apify.utils.puppeteer.addInterceptRequestHandler|removeInterceptReques]));});- it('should not propagate aborted/responded requests to following handlers', async () => {- const browser = await Apify.launchPuppeteer({ headless: true });- const propagatedUrls = [];+ test(+ 'should not propagate aborted/responded requests to following handlers',+ async () => {+ const browser = await Apify.launchPuppeteer({ headless: true });+ const propagatedUrls = [];- try {- const page = await browser.newPage();+ try {+ const page = await browser.newPage();- // Abort images.- await addInterceptRequestHandler(page, (request) => {- if (request.resourceType() === 'image') return request.abort();- return request.continue();- });+ // Abort images.+ await addInterceptRequestHandler(page, (request) => {+ if (request.resourceType() === 'image') return request.abort();+ return request.continue();+ });- // Respond scripts.- await addInterceptRequestHandler(page, (request) => {- if (request.resourceType() === 'script') {- return request.respond({- status: 404,- });- }- return request.continue();- });+ // Respond scripts.+ await addInterceptRequestHandler(page, (request) => {+ if (request.resourceType() === 'script') {+ return request.respond({+ status: 404,+ });+ }+ return request.continue();+ });- // Just collect all URLs propagated to the last handler.- await addInterceptRequestHandler(page, (request) => {- propagatedUrls.push(request.url());- return request.continue();- });- await page.setContent(HTML_PAGE, { waitUntil: 'networkidle0' });- } finally {- await browser.close();- }+ // Just collect all URLs propagated to the last handler.+ await addInterceptRequestHandler(page, (request) => {+ propagatedUrls.push(request.url());+ return request.continue();+ });+ await page.setContent(HTML_PAGE, { waitUntil: 'networkidle0' });+ } finally {+ await browser.close();+ }- expect(propagatedUrls).toEqual(expect.arrayContaining([- 'https://example.com/style.css',- ]));- });+ expect(propagatedUrls).toEqual(expect.arrayContaining([+ 'https://example.com/style.css',+ ]));+ }+ );- it('should allow to modify request', async () => {+ test('should allow to modify request', async () => {const browser = await Apify.launchPuppeteer({ headless: true });try {@@ -133,7 +136,7 @@ describe('Apify.utils.puppeteer.addInterceptRequestHandler|removeInterceptReques});describe('Apify.utils.puppeteer.removeInterceptRequestHandler()', () => {- it('works', async () => {+ test('works', async () => {const browser = await Apify.launchPuppeteer({ headless: true });const loadedUrls = [];diff --git a/__tests__/puppeteer_utils.js b/test/puppeteer_utils.test.jssimilarity index 89%rename from __tests__/puppeteer_utils.jsrename to test/puppeteer_utils.test.jsindex 7e3b87ddb..61b149057 100644--- a/__tests__/puppeteer_utils.js+++ b/test/puppeteer_utils.test.js@@ -5,15 +5,15 @@ const { utils: { log } } = Apify;describe('Apify.utils.puppeteer', () => {let ll;- before(() => {+ beforeAll(() => {ll = log.getLevel();log.setLevel(log.LEVELS.ERROR);});- after(() => {+ afterAll(() => {log.setLevel(ll);});- it('injectFile()', async () => {+ test('injectFile()', async () => {/* eslint-disable no-shadow */const browser = await Apify.launchPuppeteer({ headless: true });const survive = async (browser) => {@@ -53,7 +53,7 @@ describe('Apify.utils.puppeteer', () => {}});- it('injectJQuery()', async () => {+ test('injectJQuery()', async () => {const browser = await Apify.launchPuppeteer({ headless: true });try {@@ -88,7 +88,7 @@ describe('Apify.utils.puppeteer', () => {}});- it('injectUnderscore()', async () => {+ test('injectUnderscore()', async () => {const browser = await Apify.launchPuppeteer({ headless: true });try {@@ -111,7 +111,7 @@ describe('Apify.utils.puppeteer', () => {}});- it('hideWebDriver()', async () => {+ test('hideWebDriver()', async () => {const browser = await Apify.launchPuppeteer({ headless: true });try {@@ -137,7 +137,7 @@ describe('Apify.utils.puppeteer', () => {// TODO Remove with 1.0.0. This is here as a backwards comp for moving// the function from utils.puppeteer to utils.- it('enqueueLinks() exists in this namespace', async () => {+ test('enqueueLinks() exists in this namespace', async () => {expect(Apify.utils.puppeteer.enqueueLinks).toBeInstanceOf(Function);try {await Apify.utils.puppeteer.enqueueLinks({ page: {}, $: () => {} });@@ -148,7 +148,7 @@ describe('Apify.utils.puppeteer', () => {}});- it('blockRequests() works with default values', async () => {+ test('blockRequests() works with default values', async () => {const browser = await Apify.launchPuppeteer({ headless: true });const loadedUrls = [];@@ -170,7 +170,7 @@ describe('Apify.utils.puppeteer', () => {]));});- it('blockRequests() works with overridden values', async () => {+ test('blockRequests() works with overridden values', async () => {const browser = await Apify.launchPuppeteer({ headless: true });const loadedUrls = [];@@ -196,7 +196,7 @@ describe('Apify.utils.puppeteer', () => {]));});- it('supports blockResources() with default values', async () => {+ test('supports blockResources() with default values', async () => {const browser = await Apify.launchPuppeteer({ headless: true });const loadedUrls = [];@@ -218,7 +218,7 @@ describe('Apify.utils.puppeteer', () => {]));});- it('supports blockResources() with nondefault values', async () => {+ test('supports blockResources() with nondefault values', async () => {const browser = await Apify.launchPuppeteer({ headless: true });const loadedUrls = [];@@ -241,7 +241,7 @@ describe('Apify.utils.puppeteer', () => {]));});- it('supports cacheResponses()', async () => {+ test('supports cacheResponses()', async () => {const browser = await Apify.launchPuppeteer({ headless: true });const cache = {};@@ -273,31 +273,34 @@ describe('Apify.utils.puppeteer', () => {}});- it('cacheResponses() throws when rule with invalid type is provided', async () => {- const mockedPage = {- setRequestInterception: () => {},- on: () => {},- };+ test(+ 'cacheResponses() throws when rule with invalid type is provided',+ async () => {+ const mockedPage = {+ setRequestInterception: () => {},+ on: () => {},+ };- const testRuleType = async (value) => {- try {- await Apify.utils.puppeteer.cacheResponses(mockedPage, {}, [value]);- } catch (error) {- // this is valid path for this test- return;- }+ const testRuleType = async (value) => {+ try {+ await Apify.utils.puppeteer.cacheResponses(mockedPage, {}, [value]);+ } catch (error) {+ // this is valid path for this test+ return;+ }- expect(`Rule '${value}' should have thrown error`).toBe('');- };- await testRuleType(0);- await testRuleType(1);- await testRuleType(null);- await testRuleType([]);- await testRuleType(['']);- await testRuleType(() => {});- });+ expect(`Rule '${value}' should have thrown error`).toBe('');+ };+ await testRuleType(0);+ await testRuleType(1);+ await testRuleType(null);+ await testRuleType([]);+ await testRuleType(['']);+ await testRuleType(() => {});+ }+ );- it('compileScript() works', async () => {+ test('compileScript() works', async () => {const { compileScript } = Apify.utils.puppeteer;const scriptStringGood = 'await page.goto("about:blank"); return await page.content();';const scriptStringBad = 'for const while';@@ -324,7 +327,7 @@ describe('Apify.utils.puppeteer', () => {}});- it('gotoExtended() works', async () => {+ test('gotoExtended() works', async () => {const browser = await Apify.launchPuppeteer({ headless: true });try {@@ -349,7 +352,7 @@ describe('Apify.utils.puppeteer', () => {}});- it('infiniteScroll() works', async () => {+ test('infiniteScroll() works', async () => {function isAtBottom() {return (window.innerHeight + window.pageYOffset) >= document.body.offsetHeight;}diff --git a/__tests__/request.js b/test/request.test.jssimilarity index 79%rename from __tests__/request.jsrename to test/request.test.jsindex c65049825..0703a611b 100644--- a/__tests__/request.js+++ b/test/request.test.js@@ -4,12 +4,12 @@ import { hashPayload } from '../build/request';import Apify from '../build/index';describe('Apify.Request', () => {- it('should not accept invalid values', () => {+ test('should not accept invalid values', () => {expect(() => new Apify.Request({ url: 1 })).toThrowError();expect(() => new Apify.Request({ url: 'xxx' })).not.toThrowError();});- it('should create unique key based on url for GET requests', () => {+ test('should create unique key based on url for GET requests', () => {const url = 'https://user:[email protected]/a/vb/c /d?q=1&q=kjnjkn$lkn#lkmlkml';const normalizedUrl = normalizeUrl(url);const request = new Apify.Request({ url });@@ -18,19 +18,22 @@ describe('Apify.Request', () => {expect(request.uniqueKey).not.toEqual(request.url);});- it('should create unique key based on url, method and payload for POST requests', () => {- const url = 'https://user:[email protected]/a/vb/c /d?q=1&q=kjnjkn$lkn#lkmlkml';- const payload = JSON.stringify({ foo: 'bar' });- const payloadHash = hashPayload(payload);- const normalizedUrl = normalizeUrl(url);- const request = new Apify.Request({ url, method: 'post', payload, useExtendedUniqueKey: true });+ test(+ 'should create unique key based on url, method and payload for POST requests',+ () => {+ const url = 'https://user:[email protected]/a/vb/c /d?q=1&q=kjnjkn$lkn#lkmlkml';+ const payload = JSON.stringify({ foo: 'bar' });+ const payloadHash = hashPayload(payload);+ const normalizedUrl = normalizeUrl(url);+ const request = new Apify.Request({ url, method: 'post', payload, useExtendedUniqueKey: true });- const uniqueKey = `POST(${payloadHash}):${normalizedUrl}`;+ const uniqueKey = `POST(${payloadHash}):${normalizedUrl}`;- expect(request.uniqueKey).toEqual(uniqueKey);- });+ expect(request.uniqueKey).toEqual(uniqueKey);+ }+ );- it('works', () => {+ test('works', () => {const data = {id: '123',url: 'http://www.example.com',@@ -56,7 +59,7 @@ describe('Apify.Request', () => {expect((new Apify.Request(data)).handledAt).toBeInstanceOf(Date);});- it('should allow to push error messages', () => {+ test('should allow to push error messages', () => {const request = new Apify.Request({ url: 'http://example.com' });expect(request.errorMessages).toBe(null);@@ -112,7 +115,7 @@ describe('Apify.Request', () => {expect(last).toEqual(expect.arrayContaining([__filename.split(/[\\/]/g).pop()]));});- it('should not allow to have a GET request with payload', () => {+ test('should not allow to have a GET request with payload', () => {expect(() => new Apify.Request({ url: 'http://example.com', payload: 'foo' })).toThrowError();expect(() => new Apify.Request({ url: 'http://example.com', payload: 'foo', method: 'POST' })).not.toThrowError();});diff --git a/__tests__/request_list.js b/test/request_list.test.jssimilarity index 62%rename from __tests__/request_list.jsrename to test/request_list.test.jsindex 2679dedbf..be9c9c3d2 100644--- a/__tests__/request_list.js+++ b/test/request_list.test.js@@ -10,16 +10,16 @@ import * as utils from '../build/utils';describe('Apify.RequestList', () => {let ll;- before(() => {+ beforeAll(() => {ll = log.getLevel();log.setLevel(log.LEVELS.ERROR);});- after(() => {+ afterAll(() => {log.setLevel(ll);});- it('should not accept to pages with same uniqueKey', async () => {+ test('should not accept to pages with same uniqueKey', async () => {const requestList = new Apify.RequestList({sources: [{ url: 'https://example.com/1' },@@ -44,7 +44,7 @@ describe('Apify.RequestList', () => {expect(await requestList.isFinished()).toBe(true);});- it('must be initialized before using any of the methods', async () => {+ test('must be initialized before using any of the methods', async () => {const requestList = new Apify.RequestList({ sources: [{ url: 'https://example.com' }] });const requestObj = new Apify.Request({ url: 'https://example.com' });@@ -66,7 +66,7 @@ describe('Apify.RequestList', () => {await expect(requestList.markRequestHandled(requestObj)).to.not.be.rejectedWith();});- it('should correctly initialize itself', async () => {+ test('should correctly initialize itself', async () => {const sources = [{ url: 'https://example.com/1' },{ url: 'https://example.com/2' },@@ -108,48 +108,51 @@ describe('Apify.RequestList', () => {expect(await newList.isEmpty()).toBe(true);});- it('should correctly load list from hosted files in correct order', async () => {- const mock = sinon.mock(utils.publicUtils);- const list1 = [- 'https://example.com',- 'https://google.com',- 'https://wired.com',- ];- const list2 = [- 'https://another.com',- 'https://page.com',- ];+ test(+ 'should correctly load list from hosted files in correct order',+ async () => {+ const mock = sinon.mock(utils.publicUtils);+ const list1 = [+ 'https://example.com',+ 'https://google.com',+ 'https://wired.com',+ ];+ const list2 = [+ 'https://another.com',+ 'https://page.com',+ ];- mock.expects('downloadListOfUrls')- .once()- .withArgs({ url: 'http://example.com/list-1', urlRegExp: undefined })- .returns(new Promise(resolve => setTimeout(resolve(list1), 100)));+ mock.expects('downloadListOfUrls')+ .once()+ .withArgs({ url: 'http://example.com/list-1', urlRegExp: undefined })+ .returns(new Promise(resolve => setTimeout(resolve(list1), 100)));- mock.expects('downloadListOfUrls')- .once()- .withArgs({ url: 'http://example.com/list-2', urlRegExp: undefined })- .returns(Promise.resolve(list2), 0);+ mock.expects('downloadListOfUrls')+ .once()+ .withArgs({ url: 'http://example.com/list-2', urlRegExp: undefined })+ .returns(Promise.resolve(list2), 0);- const requestList = new Apify.RequestList({- sources: [- { method: 'GET', requestsFromUrl: 'http://example.com/list-1' },- { method: 'POST', requestsFromUrl: 'http://example.com/list-2' },- ],- });+ const requestList = new Apify.RequestList({+ sources: [+ { method: 'GET', requestsFromUrl: 'http://example.com/list-1' },+ { method: 'POST', requestsFromUrl: 'http://example.com/list-2' },+ ],+ });- await requestList.initialize();+ await requestList.initialize();- expect(await requestList.fetchNextRequest()).toMatchObject({ method: 'GET', url: list1[0] });- expect(await requestList.fetchNextRequest()).toMatchObject({ method: 'GET', url: list1[1] });- expect(await requestList.fetchNextRequest()).toMatchObject({ method: 'GET', url: list1[2] });- expect(await requestList.fetchNextRequest()).toMatchObject({ method: 'POST', url: list2[0] });- expect(await requestList.fetchNextRequest()).toMatchObject({ method: 'POST', url: list2[1] });+ expect(await requestList.fetchNextRequest()).toMatchObject({ method: 'GET', url: list1[0] });+ expect(await requestList.fetchNextRequest()).toMatchObject({ method: 'GET', url: list1[1] });+ expect(await requestList.fetchNextRequest()).toMatchObject({ method: 'GET', url: list1[2] });+ expect(await requestList.fetchNextRequest()).toMatchObject({ method: 'POST', url: list2[0] });+ expect(await requestList.fetchNextRequest()).toMatchObject({ method: 'POST', url: list2[1] });- mock.verify();- mock.restore();- });+ mock.verify();+ mock.restore();+ }+ );- it('should use regex parameter to parse urls', async () => {+ test('should use regex parameter to parse urls', async () => {const mock = sinon.mock(request);const listStr = 'kjnjkn"https://example.com/a/b/c?q=1#abc";,"HTTP://google.com/a/b/c";dgg:dd';const listArr = ['https://example.com', 'HTTP://google.com'];@@ -179,7 +182,7 @@ describe('Apify.RequestList', () => {mock.restore();});- it('should handle requestsFromUrl with no URLs', async () => {+ test('should handle requestsFromUrl with no URLs', async () => {const mock = sinon.mock(utils.publicUtils);mock.expects('downloadListOfUrls').once()@@ -203,7 +206,7 @@ describe('Apify.RequestList', () => {mock.restore();});- it('should correctly handle reclaimed pages', async () => {+ test('should correctly handle reclaimed pages', async () => {const requestList = new Apify.RequestList({sources: [{ url: 'https://example.com/1' },@@ -395,177 +398,186 @@ describe('Apify.RequestList', () => {expect(requestList.inProgress).toEqual(expect.arrayContaining([requestList.reclaimed]));});- it('should correctly persist its state when persistStateKey is set', async () => {- const PERSIST_STATE_KEY = 'some-key';- const mock = sinon.mock(keyValueStore);-- mock.expects('getValue')- .once()- .withArgs(PERSIST_STATE_KEY)- .returns(null);-- const opts = {- sources: [- { url: 'https://example.com/1' },- { url: 'https://example.com/2' },- { url: 'https://example.com/3' },- ],- persistStateKey: PERSIST_STATE_KEY,- };-- const requestList = new Apify.RequestList(opts);- await requestList.initialize();- expect(requestList.isStatePersisted).toBe(true);-- // Fetch one request and check that state is not persisted.- const request1 = await requestList.fetchNextRequest();- expect(requestList.isStatePersisted).toBe(false);-- // Persist state.- mock.expects('setValue')- .once()- .withArgs(PERSIST_STATE_KEY, requestList.getState())- .returns(Promise.resolve());- Apify.events.emit(ACTOR_EVENT_NAMES_EX.PERSIST_STATE);- await delayPromise(1);- expect(requestList.isStatePersisted).toBe(true);-- // Do some other changes and persist it again.- const request2 = await requestList.fetchNextRequest();- expect(requestList.isStatePersisted).toBe(false);- await requestList.markRequestHandled(request2);- expect(requestList.isStatePersisted).toBe(false);- mock.expects('setValue')- .once()- .withArgs(PERSIST_STATE_KEY, requestList.getState())- .returns(Promise.resolve());- Apify.events.emit(ACTOR_EVENT_NAMES_EX.PERSIST_STATE);- await delayPromise(1);- expect(requestList.isStatePersisted).toBe(true);-- // Reclaim event doesn't change the state.- await requestList.reclaimRequest(request1);- expect(requestList.isStatePersisted).toBe(true);-- // Now initiate new request list from saved state and check that it's same as state- // of original request list.- mock.expects('getValue')- .once()- .withArgs(PERSIST_STATE_KEY)- .returns(Promise.resolve(requestList.getState()));- const requestList2 = new Apify.RequestList(opts);- await requestList2.initialize();- expect(requestList2.getState()).toEqual(requestList.getState());-- mock.verify();- });-- it('should correctly persist its sources when persistSourcesKey is set', async () => {- const PERSIST_SOURCES_KEY = 'some-key';- const mock = sinon.mock(keyValueStore);-- const opts = {- sources: [- { url: 'https://example.com/1' },- { url: 'https://example.com/2' },- { url: 'https://example.com/3' },- ],- persistSourcesKey: PERSIST_SOURCES_KEY,- };-- const requestList = new Apify.RequestList(opts);- expect(requestList.areSourcesPersisted).toBe(false);-- // Expect an attempt to load sources.- mock.expects('getValue')- .once()- .withArgs(PERSIST_SOURCES_KEY)- .returns(null);-- // Expect persist sources.- mock.expects('setValue')- .once()- .withArgs(PERSIST_SOURCES_KEY, requestList.sources)- .returns(Promise.resolve());-- await requestList.initialize();- expect(requestList.areSourcesPersisted).toBe(true);-- const opts2 = {- sources: [- { url: 'https://test.com/1' },- { url: 'https://test.com/2' },- { url: 'https://test.com/3' },- ],- persistSourcesKey: PERSIST_SOURCES_KEY,- };+ test(+ 'should correctly persist its state when persistStateKey is set',+ async () => {+ const PERSIST_STATE_KEY = 'some-key';+ const mock = sinon.mock(keyValueStore);- const requestList2 = new Apify.RequestList(opts2);- expect(requestList2.areSourcesPersisted).toBe(false);+ mock.expects('getValue')+ .once()+ .withArgs(PERSIST_STATE_KEY)+ .returns(null);++ const opts = {+ sources: [+ { url: 'https://example.com/1' },+ { url: 'https://example.com/2' },+ { url: 'https://example.com/3' },+ ],+ persistStateKey: PERSIST_STATE_KEY,+ };- // Now initialize new request list from saved sources and check that- // they are same as state of original request list.- mock.expects('getValue')- .once()- .withArgs(PERSIST_SOURCES_KEY)- .returns(Promise.resolve(requestList.sources));+ const requestList = new Apify.RequestList(opts);+ await requestList.initialize();+ expect(requestList.isStatePersisted).toBe(true);++ // Fetch one request and check that state is not persisted.+ const request1 = await requestList.fetchNextRequest();+ expect(requestList.isStatePersisted).toBe(false);++ // Persist state.+ mock.expects('setValue')+ .once()+ .withArgs(PERSIST_STATE_KEY, requestList.getState())+ .returns(Promise.resolve());+ Apify.events.emit(ACTOR_EVENT_NAMES_EX.PERSIST_STATE);+ await delayPromise(1);+ expect(requestList.isStatePersisted).toBe(true);++ // Do some other changes and persist it again.+ const request2 = await requestList.fetchNextRequest();+ expect(requestList.isStatePersisted).toBe(false);+ await requestList.markRequestHandled(request2);+ expect(requestList.isStatePersisted).toBe(false);+ mock.expects('setValue')+ .once()+ .withArgs(PERSIST_STATE_KEY, requestList.getState())+ .returns(Promise.resolve());+ Apify.events.emit(ACTOR_EVENT_NAMES_EX.PERSIST_STATE);+ await delayPromise(1);+ expect(requestList.isStatePersisted).toBe(true);++ // Reclaim event doesn't change the state.+ await requestList.reclaimRequest(request1);+ expect(requestList.isStatePersisted).toBe(true);++ // Now initiate new request list from saved state and check that it's same as state+ // of original request list.+ mock.expects('getValue')+ .once()+ .withArgs(PERSIST_STATE_KEY)+ .returns(Promise.resolve(requestList.getState()));+ const requestList2 = new Apify.RequestList(opts);+ await requestList2.initialize();+ expect(requestList2.getState()).toEqual(requestList.getState());- mock.expects('setValue')- .never();+ mock.verify();+ }+ );- await requestList2.initialize();- expect(requestList2.areSourcesPersisted).toBe(true);- expect(requestList2.requests).toEqual(requestList.requests);+ test(+ 'should correctly persist its sources when persistSourcesKey is set',+ async () => {+ const PERSIST_SOURCES_KEY = 'some-key';+ const mock = sinon.mock(keyValueStore);- mock.verify();- });+ const opts = {+ sources: [+ { url: 'https://example.com/1' },+ { url: 'https://example.com/2' },+ { url: 'https://example.com/3' },+ ],+ persistSourcesKey: PERSIST_SOURCES_KEY,+ };- it('should correctly persist sources from requestsFromUrl if persistSourcesKey is set', async () => {- const PERSIST_SOURCES_KEY = 'some-key';- const kvsMock = sinon.mock(keyValueStore);- const publicUtilsMock = sinon.mock(utils.publicUtils);+ const requestList = new Apify.RequestList(opts);+ expect(requestList.areSourcesPersisted).toBe(false);++ // Expect an attempt to load sources.+ mock.expects('getValue')+ .once()+ .withArgs(PERSIST_SOURCES_KEY)+ .returns(null);++ // Expect persist sources.+ mock.expects('setValue')+ .once()+ .withArgs(PERSIST_SOURCES_KEY, requestList.sources)+ .returns(Promise.resolve());++ await requestList.initialize();+ expect(requestList.areSourcesPersisted).toBe(true);++ const opts2 = {+ sources: [+ { url: 'https://test.com/1' },+ { url: 'https://test.com/2' },+ { url: 'https://test.com/3' },+ ],+ persistSourcesKey: PERSIST_SOURCES_KEY,+ };- const opts = {- sources: [- { url: 'https://example.com/1' },- { url: 'https://example.com/2' },- { requestsFromUrl: 'http://example.com/list-urls.txt', userData: { isFromUrl: true } },- { url: 'https://example.com/5' },- ],- persistSourcesKey: PERSIST_SOURCES_KEY,- };+ const requestList2 = new Apify.RequestList(opts2);+ expect(requestList2.areSourcesPersisted).toBe(false);- const urlsFromTxt = ['http://example.com/3', 'http://example.com/4'];+ // Now initialize new request list from saved sources and check that+ // they are same as state of original request list.+ mock.expects('getValue')+ .once()+ .withArgs(PERSIST_SOURCES_KEY)+ .returns(Promise.resolve(requestList.sources));- const requestList = new Apify.RequestList(opts);- expect(requestList.areSourcesPersisted).toBe(false);+ mock.expects('setValue')+ .never();- // Expect an attempt to load sources.- kvsMock.expects('getValue')- .once()- .withArgs(PERSIST_SOURCES_KEY)- .returns(null);+ await requestList2.initialize();+ expect(requestList2.areSourcesPersisted).toBe(true);+ expect(requestList2.requests).toEqual(requestList.requests);- // Expect persist sources.- const sourcesFromTxt = urlsFromTxt.map(url => ({ url, userData: { isFromUrl: true } }));- kvsMock.expects('setValue')- .once()- .withArgs(PERSIST_SOURCES_KEY, [opts.sources[0], opts.sources[1], ...sourcesFromTxt, opts.sources[3]])- .returns(Promise.resolve());- // Expect downloadListOfUrls returns list of URLs- publicUtilsMock.expects('downloadListOfUrls')- .once()- .withArgs({ url: 'http://example.com/list-urls.txt', urlRegExp: undefined })- .returns(Promise.resolve(urlsFromTxt));-- await requestList.initialize();- expect(requestList.areSourcesPersisted).toBe(true);+ mock.verify();+ }+ );++ test(+ 'should correctly persist sources from requestsFromUrl if persistSourcesKey is set',+ async () => {+ const PERSIST_SOURCES_KEY = 'some-key';+ const kvsMock = sinon.mock(keyValueStore);+ const publicUtilsMock = sinon.mock(utils.publicUtils);++ const opts = {+ sources: [+ { url: 'https://example.com/1' },+ { url: 'https://example.com/2' },+ { requestsFromUrl: 'http://example.com/list-urls.txt', userData: { isFromUrl: true } },+ { url: 'https://example.com/5' },+ ],+ persistSourcesKey: PERSIST_SOURCES_KEY,+ };- kvsMock.verify();- publicUtilsMock.verify();- });+ const urlsFromTxt = ['http://example.com/3', 'http://example.com/4'];++ const requestList = new Apify.RequestList(opts);+ expect(requestList.areSourcesPersisted).toBe(false);++ // Expect an attempt to load sources.+ kvsMock.expects('getValue')+ .once()+ .withArgs(PERSIST_SOURCES_KEY)+ .returns(null);++ // Expect persist sources.+ const sourcesFromTxt = urlsFromTxt.map(url => ({ url, userData: { isFromUrl: true } }));+ kvsMock.expects('setValue')+ .once()+ .withArgs(PERSIST_SOURCES_KEY, [opts.sources[0], opts.sources[1], ...sourcesFromTxt, opts.sources[3]])+ .returns(Promise.resolve());+ // Expect downloadListOfUrls returns list of URLs+ publicUtilsMock.expects('downloadListOfUrls')+ .once()+ .withArgs({ url: 'http://example.com/list-urls.txt', urlRegExp: undefined })+ .returns(Promise.resolve(urlsFromTxt));++ await requestList.initialize();+ expect(requestList.areSourcesPersisted).toBe(true);++ kvsMock.verify();+ publicUtilsMock.verify();+ }+ );- it('handles correctly inconsistent inProgress fields in state', async () => {+ test('handles correctly inconsistent inProgress fields in state', async () => {// NOTE: This is a test for the deleteFromInProgress hotfix - see RequestList.initialize()const sources = [@@ -607,7 +619,7 @@ describe('Apify.RequestList', () => {}});- it('it gets correct length()', async () => {+ test('it gets correct length()', async () => {const sources = [{ url: 'https://www.example.com' },{ url: 'https://www.ams360.com' },@@ -625,7 +637,7 @@ describe('Apify.RequestList', () => {expect(requestList.length()).toBe(4);});- it('it gets correct handledCount()', async () => {+ test('it gets correct handledCount()', async () => {const sources = [{ url: 'https://www.example.com' },{ url: 'https://www.ams360.com' },@@ -655,46 +667,49 @@ describe('Apify.RequestList', () => {expect(requestList.handledCount()).toBe(2);});- it('should correctly keep duplicate URLs while keepDuplicateUrls is set', async () => {- const sources = [- { url: 'https://www.example.com' },- { url: 'https://www.example.com' },- { url: 'https://www.example.com' },- { url: 'https://www.ex2mple.com' },- ];+ test(+ 'should correctly keep duplicate URLs while keepDuplicateUrls is set',+ async () => {+ const sources = [+ { url: 'https://www.example.com' },+ { url: 'https://www.example.com' },+ { url: 'https://www.example.com' },+ { url: 'https://www.ex2mple.com' },+ ];- let requestList = new Apify.RequestList({- sources,- keepDuplicateUrls: true,- });+ let requestList = new Apify.RequestList({+ sources,+ keepDuplicateUrls: true,+ });- await requestList.initialize();- expect(requestList.length()).toBe(4);+ await requestList.initialize();+ expect(requestList.length()).toBe(4);- log.setLevel(log.LEVELS.INFO);- const logStub = sinon.stub(console, 'log');-- requestList = new Apify.RequestList({- sources: sources.concat([- { url: 'https://www.example.com', uniqueKey: '123' },- { url: 'https://www.example.com', uniqueKey: '123' },- { url: 'https://www.example.com', uniqueKey: '456' },- { url: 'https://www.ex2mple.com', uniqueKey: '456' },- ]),- keepDuplicateUrls: true,- });+ log.setLevel(log.LEVELS.INFO);+ const logStub = sinon.stub(console, 'log');- await requestList.initialize();- expect(requestList.length()).toBe(6);- expect(logStub.called).toBe(true);- expect(logStub.getCall(0).args[0]).toEqual(expect.arrayContaining(['Check your sources\' unique keys.']));+ requestList = new Apify.RequestList({+ sources: sources.concat([+ { url: 'https://www.example.com', uniqueKey: '123' },+ { url: 'https://www.example.com', uniqueKey: '123' },+ { url: 'https://www.example.com', uniqueKey: '456' },+ { url: 'https://www.ex2mple.com', uniqueKey: '456' },+ ]),+ keepDuplicateUrls: true,+ });- logStub.restore();- log.setLevel(log.LEVELS.ERROR);- });+ await requestList.initialize();+ expect(requestList.length()).toBe(6);+ expect(logStub.called).toBe(true);+ expect(logStub.getCall(0).args[0]).toEqual(expect.arrayContaining(['Check your sources\' unique keys.']));++ logStub.restore();+ log.setLevel(log.LEVELS.ERROR);+ }+ );describe('Apify.openRequestList()', () => {- it('should work', async () => {+ test('should work', async () => {const mock = sinon.mock(keyValueStore);mock.expects('getValue').atLeast(1).resolves();mock.expects('setValue').atLeast(1).resolves();@@ -711,7 +726,7 @@ describe('Apify.RequestList', () => {mock.verify();});- it('should work with string sources', async () => {+ test('should work with string sources', async () => {const mock = sinon.mock(keyValueStore);mock.expects('getValue').atLeast(1).resolves();mock.expects('setValue').atLeast(1).resolves();@@ -728,7 +743,7 @@ describe('Apify.RequestList', () => {mock.verify();});- it('should correctly pass options', async () => {+ test('should correctly pass options', async () => {const mock = sinon.mock(keyValueStore);mock.expects('getValue').atLeast(1).resolves();mock.expects('setValue').atLeast(1).resolves();@@ -750,7 +765,7 @@ describe('Apify.RequestList', () => {mock.verify();});- it('should work with null name', async () => {+ test('should work with null name', async () => {const mock = sinon.mock(keyValueStore);mock.expects('getValue').never().resolves();mock.expects('setValue').never().resolves();@@ -767,7 +782,7 @@ describe('Apify.RequestList', () => {mock.verify();});- it('should throw on invalid parameters', async () => {+ test('should throw on invalid parameters', async () => {const args = [[],['x', []],diff --git a/__tests__/request_queue.js b/test/request_queue.test.jssimilarity index 66%rename from __tests__/request_queue.jsrename to test/request_queue.test.jsindex dfba1e85a..692d6bde9 100644--- a/__tests__/request_queue.js+++ b/test/request_queue.test.js@@ -1,4 +1,3 @@-import { assert, expect } from 'chai';import _ from 'underscore';import sinon from 'sinon';import path from 'path';@@ -18,13 +17,13 @@ const expectRequestsSame = (req1, req2) => {};describe('RequestQueue', () => {- before(() => apifyClient.setOptions({ token: 'xxx' }));- after(() => apifyClient.setOptions({ token: undefined }));+ beforeAll(() => apifyClient.setOptions({ token: 'xxx' }));+ afterAll(() => apifyClient.setOptions({ token: undefined }));beforeEach(() => emptyLocalStorageSubdir(LOCAL_STORAGE_SUBDIR));afterEach(() => emptyLocalStorageSubdir(LOCAL_STORAGE_SUBDIR));describe('local', async () => {- it('should work', async () => {+ test('should work', async () => {const queue = new RequestQueueLocal('my-queue-0', LOCAL_STORAGE_DIR);const req1 = new Apify.Request({ url: 'http://example.com/first' });@@ -117,12 +116,12 @@ describe('RequestQueue', () => {expectDirEmpty(queueDir);});- it('handles invalid URLs gracefully', async () => {+ test('handles invalid URLs gracefully', async () => {const queue = new RequestQueueLocal('my-queue-x', LOCAL_STORAGE_DIR);try {await queue.addRequest({ url: '' });- assert.fail('This should not happen');+ expect(false).toBe(true);} catch (e) {expect(e.message).toEqual(expect.arrayContaining(['The "url" option cannot be empty string']));}@@ -143,7 +142,7 @@ describe('RequestQueue', () => {expect(await queue.getRequest(request2.id)).toEqual(request2);});- it('supports forefront param in reclaimRequest()', async () => {+ test('supports forefront param in reclaimRequest()', async () => {const queue = new RequestQueueLocal('my-queue-1', LOCAL_STORAGE_DIR);await queue.addRequest(new Apify.Request({ url: 'http://example.com/first' }));@@ -171,7 +170,7 @@ describe('RequestQueue', () => {expect(request3.url).toBe('http://example.com/third');});- it('should get initialized from existing dir', async () => {+ test('should get initialized from existing dir', async () => {const request1 = new Apify.Request({ url: 'http://example.com/first' });const request2 = new Apify.Request({ url: 'http://example.com/middle' });const request3 = new Apify.Request({ url: 'http://example.com/last-but-first' });@@ -203,7 +202,7 @@ describe('RequestQueue', () => {expect(await anotherQueue.isFinished()).toBe(true);});- it('should accept plain object in addRequest()', async () => {+ test('should accept plain object in addRequest()', async () => {const queue = new RequestQueueLocal('some-id', LOCAL_STORAGE_DIR);await queue.addRequest({ url: 'http://example.com/a' });expect(@@ -211,7 +210,7 @@ describe('RequestQueue', () => {).toBe('http://example.com/a');});- it('getInfo() and handledCount() should work', async () => {+ test('getInfo() and handledCount() should work', async () => {const queueName = 'stats-queue';const queue = new RequestQueueLocal(queueName, LOCAL_STORAGE_DIR);let count = await queue.handledCount();@@ -283,7 +282,7 @@ describe('RequestQueue', () => {expect(info.handledRequestCount).toBe(3);});- it('deprecated delete() still works', async () => {+ test('deprecated delete() still works', async () => {const rq = new RequestQueueLocal('to-delete', LOCAL_STORAGE_DIR);await rq.addRequest({ url: 'https://example.com' });@@ -295,7 +294,7 @@ describe('RequestQueue', () => {});describe('remote', async () => {- it('should work', async () => {+ test('should work', async () => {expectNotUsingLocalStorage();const { Request } = Apify;@@ -477,7 +476,7 @@ describe('RequestQueue', () => {mock.restore();});- it('should cache new requests locally', async () => {+ test('should cache new requests locally', async () => {expectNotUsingLocalStorage();const { Request } = Apify;@@ -519,49 +518,52 @@ describe('RequestQueue', () => {mock.restore();});- it('should cache requests locally with info if request was already handled', async () => {- expectNotUsingLocalStorage();-- const { Request } = Apify;-- const clientKey = 'my-client';- const queue = new RequestQueue('some-id', undefined, clientKey);- const mock = sinon.mock(apifyClient.requestQueues);-- const requestX = new Request({ url: 'http://example.com/x' });- const requestY = new Request({ url: 'http://example.com/x' }); // Has same uniqueKey as X-- // Add request X.- mock.expects('addRequest')- .once()- .withArgs({- queueId: 'some-id',- request: requestX,- forefront: false,- clientKey,- })- .returns(Promise.resolve({+ test(+ 'should cache requests locally with info if request was already handled',+ async () => {+ expectNotUsingLocalStorage();++ const { Request } = Apify;++ const clientKey = 'my-client';+ const queue = new RequestQueue('some-id', undefined, clientKey);+ const mock = sinon.mock(apifyClient.requestQueues);++ const requestX = new Request({ url: 'http://example.com/x' });+ const requestY = new Request({ url: 'http://example.com/x' }); // Has same uniqueKey as X++ // Add request X.+ mock.expects('addRequest')+ .once()+ .withArgs({+ queueId: 'some-id',+ request: requestX,+ forefront: false,+ clientKey,+ })+ .returns(Promise.resolve({+ requestId: 'x',+ wasAlreadyHandled: true,+ wasAlreadyPresent: true,+ request: requestX,+ }));+ await queue.addRequest(requestX);++ // Add request Y that has same unique so that addRequest() is not called because it's already cached.+ mock.expects('addRequest').never();+ expect(await queue.addRequest(requestY)).toEqual({requestId: 'x',- wasAlreadyHandled: true,wasAlreadyPresent: true,- request: requestX,- }));- await queue.addRequest(requestX);-- // Add request Y that has same unique so that addRequest() is not called because it's already cached.- mock.expects('addRequest').never();- expect(await queue.addRequest(requestY)).toEqual({- requestId: 'x',- wasAlreadyPresent: true,- wasAlreadyHandled: true,- request: requestY,- });+ wasAlreadyHandled: true,+ request: requestY,+ });- mock.verify();- mock.restore();- });+ mock.verify();+ mock.restore();+ }+ );- it('should cache requests from queue head', async () => {+ test('should cache requests from queue head', async () => {expectNotUsingLocalStorage();const { Request } = Apify;@@ -599,76 +601,79 @@ describe('RequestQueue', () => {mock.restore();});- it('should handle situation when newly created request is not available yet', async () => {- expectNotUsingLocalStorage();-- const { Request } = Apify;-- const clientKey = 'my-client';- const queue = new RequestQueue('some-id', 'some-name', clientKey);- const mock = sinon.mock(apifyClient.requestQueues);- mock.expects('getHead').never();-- const requestA = new Request({ url: 'http://example.com/a' });-- // Add request A- mock.expects('addRequest')- .once()- .withArgs({- queueId: 'some-id',- request: requestA,- forefront: true,- clientKey,- })- .returns(Promise.resolve({- requestId: 'a',- wasAlreadyHandled: false,- wasAlreadyPresent: false,- request: requestA,- }));- await queue.addRequest(requestA, { forefront: true });- expect(queue.queueHeadDict.length()).toBe(1);-- // Try to get requestA which is not available yet.- mock.expects('getRequest')- .once()- .withArgs({- queueId: 'some-id',- requestId: 'a',- })- .returns(Promise.resolve(null));- expect(await queue.fetchNextRequest()).toBe(null);-- // Give queue time to mark request 'a' as not in progress- await Apify.utils.sleep(STORAGE_CONSISTENCY_DELAY_MILLIS + 10);-- // Should try it once again (the queue head is queried again)- mock.expects('getRequest')- .once()- .withArgs({- queueId: 'some-id',- requestId: 'a',- })- .returns(Promise.resolve(requestA));- mock.expects('getHead')- .once()- .withArgs({- queueId: 'some-id',- limit: QUERY_HEAD_MIN_LENGTH,- clientKey,- })- .returns(Promise.resolve({- items: [- { id: 'a', uniqueKey: 'aaa' },- ],- }));- expect(await queue.fetchNextRequest()).toEqual(requestA);-- mock.verify();- mock.restore();- });+ test(+ 'should handle situation when newly created request is not available yet',+ async () => {+ expectNotUsingLocalStorage();++ const { Request } = Apify;++ const clientKey = 'my-client';+ const queue = new RequestQueue('some-id', 'some-name', clientKey);+ const mock = sinon.mock(apifyClient.requestQueues);+ mock.expects('getHead').never();++ const requestA = new Request({ url: 'http://example.com/a' });++ // Add request A+ mock.expects('addRequest')+ .once()+ .withArgs({+ queueId: 'some-id',+ request: requestA,+ forefront: true,+ clientKey,+ })+ .returns(Promise.resolve({+ requestId: 'a',+ wasAlreadyHandled: false,+ wasAlreadyPresent: false,+ request: requestA,+ }));+ await queue.addRequest(requestA, { forefront: true });+ expect(queue.queueHeadDict.length()).toBe(1);++ // Try to get requestA which is not available yet.+ mock.expects('getRequest')+ .once()+ .withArgs({+ queueId: 'some-id',+ requestId: 'a',+ })+ .returns(Promise.resolve(null));+ expect(await queue.fetchNextRequest()).toBe(null);++ // Give queue time to mark request 'a' as not in progress+ await Apify.utils.sleep(STORAGE_CONSISTENCY_DELAY_MILLIS + 10);++ // Should try it once again (the queue head is queried again)+ mock.expects('getRequest')+ .once()+ .withArgs({+ queueId: 'some-id',+ requestId: 'a',+ })+ .returns(Promise.resolve(requestA));+ mock.expects('getHead')+ .once()+ .withArgs({+ queueId: 'some-id',+ limit: QUERY_HEAD_MIN_LENGTH,+ clientKey,+ })+ .returns(Promise.resolve({+ items: [+ { id: 'a', uniqueKey: 'aaa' },+ ],+ }));+ expect(await queue.fetchNextRequest()).toEqual(requestA);++ mock.verify();+ mock.restore();+ }+ );- it('should not add handled request to queue head dict', async () => {+ test('should not add handled request to queue head dict', async () => {expectNotUsingLocalStorage();const { Request } = Apify;@@ -711,7 +716,7 @@ describe('RequestQueue', () => {mock.restore();});- it('should accept plain object in addRequest()', async () => {+ test('should accept plain object in addRequest()', async () => {expectNotUsingLocalStorage();const clientKey = 'my-client';const queue = new RequestQueue('some-id', undefined, clientKey);@@ -729,7 +734,7 @@ describe('RequestQueue', () => {mock.restore();});- it('should return correct handledCount', async () => {+ test('should return correct handledCount', async () => {const stub = sinon.stub(apifyClient.requestQueues, 'getQueue').returns(Promise.resolve({@@ -742,175 +747,181 @@ describe('RequestQueue', () => {sinon.restore();});- it('should always wait for a queue head to become consistent before marking queue as finished (hadMultipleClients = true)', async () => {- expectNotUsingLocalStorage();-- const clientKey = 'my-client';- const queue = new RequestQueue('some-id', 'some-name', clientKey);- const mock = sinon.mock(apifyClient.requestQueues);-- // Return head with modifiedAt = now so it will retry the call.- mock.expects('getHead')- .once()- .withArgs({- queueId: 'some-id',- limit: QUERY_HEAD_MIN_LENGTH,- clientKey,- })- .returns(Promise.resolve({- limit: 5,- queueModifiedAt: new Date(Date.now() - API_PROCESSED_REQUESTS_DELAY_MILLIS * 0.75),- items: [],- hadMultipleClients: true,- }));-- // And now return return date which makes the queue consistent.- mock.expects('getHead')- .once()- .withArgs({- queueId: 'some-id',- limit: QUERY_HEAD_MIN_LENGTH,- clientKey,- })- .returns(Promise.resolve({- limit: 5,- queueModifiedAt: new Date(Date.now() - API_PROCESSED_REQUESTS_DELAY_MILLIS),- items: [],- hadMultipleClients: true,- }));-- expect(await queue.isFinished()).toBe(true);-- mock.verify();- mock.restore();- });-- it('should always wait for a queue head to become consistent before marking queue as finished (hadMultipleClients = true)', async () => {- expectNotUsingLocalStorage();-- const clientKey = 'my-client';- const queueId = 'some-id';- const queue = new RequestQueue(queueId, 'some-name', clientKey);- const mock = sinon.mock(apifyClient.requestQueues);-- expect(queue.assumedTotalCount).toBe(0);- expect(queue.assumedHandledCount).toBe(0);-- // Add some requests.- const requestA = new Apify.Request({ url: 'http://example.com/a' });- const requestB = new Apify.Request({ url: 'http://example.com/b' });- mock.expects('addRequest').once()- .withArgs({ queueId, request: requestA, forefront: true, clientKey })- .returns(Promise.resolve({ requestId: 'a', wasAlreadyHandled: false, wasAlreadyPresent: false, request: requestA }));- mock.expects('addRequest').once()- .withArgs({ queueId, request: requestB, forefront: true, clientKey })- .returns(Promise.resolve({ requestId: 'b', wasAlreadyHandled: false, wasAlreadyPresent: false, request: requestA }));- await queue.addRequest(requestA, { forefront: true });- await queue.addRequest(requestB, { forefront: true });-- expect(queue.queueHeadDict.length()).toBe(2);- expect(queue.inProgressCount()).toBe(0);- expect(queue.assumedTotalCount).toBe(2);- expect(queue.assumedHandledCount).toBe(0);-- // It won't query the head as there is something in progress or pending.- mock.expects('getHead').never();- expect(await queue.isFinished()).toBe(false);-- // Fetch them from queue.- mock.expects('getRequest').once()- .withArgs({ queueId: 'some-id', requestId: 'b' })- .returns(Promise.resolve(_.extend(requestB, { id: 'b' })));- mock.expects('getRequest').once()- .withArgs({ queueId: 'some-id', requestId: 'a' })- .returns(Promise.resolve(_.extend(requestA, { id: 'a' })));- const requestBFromQueue = await queue.fetchNextRequest();- expect(requestBFromQueue).toEqual(requestB);- const requestAFromQueue = await queue.fetchNextRequest();- expect(requestAFromQueue).toEqual(requestA);-- expect(queue.queueHeadDict.length()).toBe(0);- expect(queue.inProgressCount()).toBe(2);- expect(queue.assumedTotalCount).toBe(2);- expect(queue.assumedHandledCount).toBe(0);-- // It won't query the head as there is something in progress or pending.- mock.expects('getHead').never();- expect(await queue.isFinished()).toBe(false);-- // Reclaim one and mark another one handled.- mock.expects('updateRequest').once()- .withArgs({ queueId, request: requestB, clientKey })- .returns(Promise.resolve({ requestId: requestB.id, wasAlreadyHandled: false, wasAlreadyPresent: true, request: requestB }));- await queue.markRequestHandled(requestB);- mock.expects('updateRequest').once()- .withArgs({ queueId, request: requestA, forefront: true, clientKey })- .returns(Promise.resolve({ requestId: requestA.id, wasAlreadyHandled: false, wasAlreadyPresent: true, request: requestA }));- await queue.reclaimRequest(requestA, { forefront: true });- expect(queue.queueHeadDict.length()).toBe(0);- expect(queue.inProgressCount()).toBe(1);- expect(queue.assumedTotalCount).toBe(2);- expect(queue.assumedHandledCount).toBe(1);- await Apify.utils.sleep(STORAGE_CONSISTENCY_DELAY_MILLIS + 10);- expect(queue.queueHeadDict.length()).toBe(1);- expect(queue.inProgressCount()).toBe(0);- expect(queue.assumedTotalCount).toBe(2);- expect(queue.assumedHandledCount).toBe(1);-- // It won't query the head as there is something in progress or pending.- mock.expects('getHead').never();- expect(await queue.isFinished()).toBe(false);-- // Fetch again.- mock.expects('getRequest').once()- .withArgs({ queueId: 'some-id', requestId: 'a' })- .returns(Promise.resolve(_.extend(requestA, { id: 'a' })));- const requestAFromQueue2 = await queue.fetchNextRequest();- expect(requestAFromQueue2).toEqual(requestA);-- expect(queue.queueHeadDict.length()).toBe(0);- expect(queue.inProgressCount()).toBe(1);- expect(queue.assumedTotalCount).toBe(2);- expect(queue.assumedHandledCount).toBe(1);-- // It won't query the head as there is something in progress or pending.- mock.expects('getHead').never();- expect(await queue.isFinished()).toBe(false);-- // Mark handled.- mock.expects('updateRequest').once()- .withArgs({ queueId, request: requestA, clientKey })- .returns(Promise.resolve({ requestId: requestA.id, wasAlreadyHandled: false, wasAlreadyPresent: true, request: requestA }));- await queue.markRequestHandled(requestA);-- expect(queue.queueHeadDict.length()).toBe(0);- expect(queue.inProgressCount()).toBe(0);- expect(queue.assumedTotalCount).toBe(2);- expect(queue.assumedHandledCount).toBe(2);-- // Return head with modifiedAt = now so it would retry the query for queue to become consistent but because hadMultipleClients=true- // it will finish immediately.- mock.expects('getHead')- .once()- .withArgs({- queueId: 'some-id',- limit: QUERY_HEAD_MIN_LENGTH,- clientKey,- })- .returns(Promise.resolve({- limit: 5,- queueModifiedAt: new Date(),- items: [],- hadMultipleClients: false,- }));-- expect(await queue.isFinished()).toBe(true);-- mock.verify();- mock.restore();- });+ test(+ 'should always wait for a queue head to become consistent before marking queue as finished (hadMultipleClients = true)',+ async () => {+ expectNotUsingLocalStorage();++ const clientKey = 'my-client';+ const queue = new RequestQueue('some-id', 'some-name', clientKey);+ const mock = sinon.mock(apifyClient.requestQueues);++ // Return head with modifiedAt = now so it will retry the call.+ mock.expects('getHead')+ .once()+ .withArgs({+ queueId: 'some-id',+ limit: QUERY_HEAD_MIN_LENGTH,+ clientKey,+ })+ .returns(Promise.resolve({+ limit: 5,+ queueModifiedAt: new Date(Date.now() - API_PROCESSED_REQUESTS_DELAY_MILLIS * 0.75),+ items: [],+ hadMultipleClients: true,+ }));++ // And now return return date which makes the queue consistent.+ mock.expects('getHead')+ .once()+ .withArgs({+ queueId: 'some-id',+ limit: QUERY_HEAD_MIN_LENGTH,+ clientKey,+ })+ .returns(Promise.resolve({+ limit: 5,+ queueModifiedAt: new Date(Date.now() - API_PROCESSED_REQUESTS_DELAY_MILLIS),+ items: [],+ hadMultipleClients: true,+ }));++ expect(await queue.isFinished()).toBe(true);++ mock.verify();+ mock.restore();+ }+ );++ test(+ 'should always wait for a queue head to become consistent before marking queue as finished (hadMultipleClients = true)',+ async () => {+ expectNotUsingLocalStorage();++ const clientKey = 'my-client';+ const queueId = 'some-id';+ const queue = new RequestQueue(queueId, 'some-name', clientKey);+ const mock = sinon.mock(apifyClient.requestQueues);++ expect(queue.assumedTotalCount).toBe(0);+ expect(queue.assumedHandledCount).toBe(0);++ // Add some requests.+ const requestA = new Apify.Request({ url: 'http://example.com/a' });+ const requestB = new Apify.Request({ url: 'http://example.com/b' });+ mock.expects('addRequest').once()+ .withArgs({ queueId, request: requestA, forefront: true, clientKey })+ .returns(Promise.resolve({ requestId: 'a', wasAlreadyHandled: false, wasAlreadyPresent: false, request: requestA }));+ mock.expects('addRequest').once()+ .withArgs({ queueId, request: requestB, forefront: true, clientKey })+ .returns(Promise.resolve({ requestId: 'b', wasAlreadyHandled: false, wasAlreadyPresent: false, request: requestA }));+ await queue.addRequest(requestA, { forefront: true });+ await queue.addRequest(requestB, { forefront: true });++ expect(queue.queueHeadDict.length()).toBe(2);+ expect(queue.inProgressCount()).toBe(0);+ expect(queue.assumedTotalCount).toBe(2);+ expect(queue.assumedHandledCount).toBe(0);++ // It won't query the head as there is something in progress or pending.+ mock.expects('getHead').never();+ expect(await queue.isFinished()).toBe(false);++ // Fetch them from queue.+ mock.expects('getRequest').once()+ .withArgs({ queueId: 'some-id', requestId: 'b' })+ .returns(Promise.resolve(_.extend(requestB, { id: 'b' })));+ mock.expects('getRequest').once()+ .withArgs({ queueId: 'some-id', requestId: 'a' })+ .returns(Promise.resolve(_.extend(requestA, { id: 'a' })));+ const requestBFromQueue = await queue.fetchNextRequest();+ expect(requestBFromQueue).toEqual(requestB);+ const requestAFromQueue = await queue.fetchNextRequest();+ expect(requestAFromQueue).toEqual(requestA);++ expect(queue.queueHeadDict.length()).toBe(0);+ expect(queue.inProgressCount()).toBe(2);+ expect(queue.assumedTotalCount).toBe(2);+ expect(queue.assumedHandledCount).toBe(0);++ // It won't query the head as there is something in progress or pending.+ mock.expects('getHead').never();+ expect(await queue.isFinished()).toBe(false);++ // Reclaim one and mark another one handled.+ mock.expects('updateRequest').once()+ .withArgs({ queueId, request: requestB, clientKey })+ .returns(Promise.resolve({ requestId: requestB.id, wasAlreadyHandled: false, wasAlreadyPresent: true, request: requestB }));+ await queue.markRequestHandled(requestB);+ mock.expects('updateRequest').once()+ .withArgs({ queueId, request: requestA, forefront: true, clientKey })+ .returns(Promise.resolve({ requestId: requestA.id, wasAlreadyHandled: false, wasAlreadyPresent: true, request: requestA }));+ await queue.reclaimRequest(requestA, { forefront: true });+ expect(queue.queueHeadDict.length()).toBe(0);+ expect(queue.inProgressCount()).toBe(1);+ expect(queue.assumedTotalCount).toBe(2);+ expect(queue.assumedHandledCount).toBe(1);+ await Apify.utils.sleep(STORAGE_CONSISTENCY_DELAY_MILLIS + 10);+ expect(queue.queueHeadDict.length()).toBe(1);+ expect(queue.inProgressCount()).toBe(0);+ expect(queue.assumedTotalCount).toBe(2);+ expect(queue.assumedHandledCount).toBe(1);++ // It won't query the head as there is something in progress or pending.+ mock.expects('getHead').never();+ expect(await queue.isFinished()).toBe(false);++ // Fetch again.+ mock.expects('getRequest').once()+ .withArgs({ queueId: 'some-id', requestId: 'a' })+ .returns(Promise.resolve(_.extend(requestA, { id: 'a' })));+ const requestAFromQueue2 = await queue.fetchNextRequest();+ expect(requestAFromQueue2).toEqual(requestA);++ expect(queue.queueHeadDict.length()).toBe(0);+ expect(queue.inProgressCount()).toBe(1);+ expect(queue.assumedTotalCount).toBe(2);+ expect(queue.assumedHandledCount).toBe(1);++ // It won't query the head as there is something in progress or pending.+ mock.expects('getHead').never();+ expect(await queue.isFinished()).toBe(false);++ // Mark handled.+ mock.expects('updateRequest').once()+ .withArgs({ queueId, request: requestA, clientKey })+ .returns(Promise.resolve({ requestId: requestA.id, wasAlreadyHandled: false, wasAlreadyPresent: true, request: requestA }));+ await queue.markRequestHandled(requestA);++ expect(queue.queueHeadDict.length()).toBe(0);+ expect(queue.inProgressCount()).toBe(0);+ expect(queue.assumedTotalCount).toBe(2);+ expect(queue.assumedHandledCount).toBe(2);++ // Return head with modifiedAt = now so it would retry the query for queue to become consistent but because hadMultipleClients=true+ // it will finish immediately.+ mock.expects('getHead')+ .once()+ .withArgs({+ queueId: 'some-id',+ limit: QUERY_HEAD_MIN_LENGTH,+ clientKey,+ })+ .returns(Promise.resolve({+ limit: 5,+ queueModifiedAt: new Date(),+ items: [],+ hadMultipleClients: false,+ }));++ expect(await queue.isFinished()).toBe(true);++ mock.verify();+ mock.restore();+ }+ );- it('getInfo() should work', async () => {+ test('getInfo() should work', async () => {const queue = new RequestQueue('some-id', 'some-name');const mock = sinon.mock(apifyClient.requestQueues);@@ -938,7 +949,7 @@ describe('RequestQueue', () => {mock.restore();});- it('deprecated delete() still works', async () => {+ test('deprecated delete() still works', async () => {const mock = sinon.mock(apifyClient.requestQueues);const rq = new RequestQueue('some-id', 'some-name');mock.expects('deleteQueue')@@ -953,7 +964,7 @@ describe('RequestQueue', () => {});describe('Apify.openRequestQueue', async () => {- it('should work', () => {+ test('should work', () => {const mock = sinon.mock(utils);process.env[ENV_VARS.LOCAL_STORAGE_DIR] = LOCAL_STORAGE_DIR;diff --git a/__tests__/session_pool/session.js b/test/session_pool/session.test.jssimilarity index 76%rename from __tests__/session_pool/session.jsrename to test/session_pool/session.test.jsindex a5d66b75a..9e6e302f1 100644--- a/__tests__/session_pool/session.js+++ b/test/session_pool/session.test.js@@ -14,7 +14,7 @@ describe('Session - testing session behaviour ', async () => {session = new Session({ sessionPool });});- it('should markGood session and lower the errorScore', () => {+ test('should markGood session and lower the errorScore', () => {expect(session.usageCount).toBe(0);expect(session.errorScore).toBe(0);session.markGood();@@ -25,50 +25,53 @@ describe('Session - testing session behaviour ', async () => {expect(session.errorScore).toBe(0.5);});- it('should throw error when param sessionPool is not EventEmitter instance', () => {- let err;- try {- const session = new Session({ sessionPool: {} }); // eslint-disable-line- } catch (e) {- err = e;+ test(+ 'should throw error when param sessionPool is not EventEmitter instance',+ () => {+ let err;+ try {+ const session = new Session({ sessionPool: {} }); // eslint-disable-line+ } catch (e) {+ err = e;+ }+ expect(err).toBeDefined(); // eslint-disable-line+ expect(err.message.includes('Session: sessionPool must be instance of SessionPool')).toBe(true); // eslint-disable-line}- expect(err).toBeDefined(); // eslint-disable-line- expect(err.message.includes('Session: sessionPool must be instance of SessionPool')).toBe(true); // eslint-disable-line- });+ );- it('should mark session markBaded', () => {+ test('should mark session markBaded', () => {session.markBad();expect(session.errorScore).toBe(1);expect(session.usageCount).toBe(1);});- it('should expire session', async () => {+ test('should expire session', async () => {session = new Session({ maxAgeSecs: 1 / 100, sessionPool });await Apify.utils.sleep(101);expect(session.isExpired()).toBe(true);expect(session.isUsable()).toBe(false);});- it('should max out session usage', () => {+ test('should max out session usage', () => {session.maxSessionUsageCount = 1;session.markGood();expect(session.isMaxUsageCountReached()).toBe(true);expect(session.isUsable()).toBe(false);});- it('should block session', () => {+ test('should block session', () => {session.errorScore += session.maxErrorScore;expect(session.isBlocked()).toBe(true);expect(session.isUsable()).toBe(false);});- it('should markGood session', () => {+ test('should markGood session', () => {session.markGood();expect(session.usageCount).toBe(1);expect(session.isUsable()).toBe(true);});- it('should retire session', () => {+ test('should retire session', () => {let discarded = false;sessionPool.on(EVENTS.SESSION_RETIRED, (ses) => {expect(ses instanceof Session).toBe(true);@@ -79,7 +82,7 @@ describe('Session - testing session behaviour ', async () => {expect(session.usageCount).toBe(1);});- it('should get state', () => {+ test('should get state', () => {const state = session.getState();expect(state.id).toBeDefined(); // eslint-disable-linediff --git a/__tests__/session_pool/session_pool.js b/test/session_pool/session_pool.test.jssimilarity index 81%rename from __tests__/session_pool/session_pool.jsrename to test/session_pool/session_pool.test.jsindex ec31b3fa3..faa18add4 100644--- a/__tests__/session_pool/session_pool.js+++ b/test/session_pool/session_pool.test.js@@ -9,7 +9,7 @@ import { Session } from '../../src/session_pool/session';describe('SessionPool - testing session pool', async () => {let sessionPool;- before(() => {+ beforeAll(() => {process.env.APIFY_LOCAL_STORAGE_DIR = LOCAL_STORAGE_DIR;});@@ -18,7 +18,7 @@ describe('SessionPool - testing session pool', async () => {sessionPool = await Apify.openSessionPool();});- it('should initialize with default values for first time', async () => {+ test('should initialize with default values for first time', async () => {expect(sessionPool.sessions.length).toBeDefined(); // eslint-disable-lineexpect(sessionPool.maxPoolSize).toBeDefined(); // eslint-disable-lineexpect(sessionPool.maxSessionAgeSecs).toBeDefined(); // eslint-disable-line@@ -27,7 +27,7 @@ describe('SessionPool - testing session pool', async () => {expect(sessionPool.createSessionFunction).toEqual(sessionPool._defaultCreateSessionFunction); // eslint-disable-line});- it('should override default values', async () => {+ test('should override default values', async () => {const opts = {maxPoolSize: 3000,maxSessionAgeSecs: 100,@@ -47,7 +47,7 @@ describe('SessionPool - testing session pool', async () => {});});- it('should work using openSessionPool', async () => {+ test('should work using openSessionPool', async () => {const opts = {maxPoolSize: 3000,maxSessionAgeSecs: 100,@@ -67,7 +67,7 @@ describe('SessionPool - testing session pool', async () => {});describe('should retrieve session', () => {- it('should retrieve session with correct shape', async () => {+ test('should retrieve session with correct shape', async () => {const session = await sessionPool.getSession();expect(sessionPool.sessions.length).toBe(1);expect(session.id).toBeDefined(); // eslint-disable-line@@ -77,7 +77,7 @@ describe('SessionPool - testing session pool', async () => {expect(session.sessionPool).toEqual(sessionPool);});- it('should pick session when pool is full', async () => {+ test('should pick session when pool is full', async () => {sessionPool.maxPoolSize = 2;await sessionPool.getSession();await sessionPool.getSession();@@ -94,30 +94,33 @@ describe('SessionPool - testing session pool', async () => {expect(isCalled).toBe(true); //eslint-disable-line});- it('should delete picked session when it is usable a create a new one', async () => {- sessionPool.maxPoolSize = 1;- await sessionPool.getSession();- const session = sessionPool.sessions[0];+ test(+ 'should delete picked session when it is usable a create a new one',+ async () => {+ sessionPool.maxPoolSize = 1;+ await sessionPool.getSession();+ const session = sessionPool.sessions[0];- session.errorScore += session.maxErrorScore;- let isCalled = false;- const oldRemove = sessionPool._removeSession; //eslint-disable-line+ session.errorScore += session.maxErrorScore;+ let isCalled = false;+ const oldRemove = sessionPool._removeSession; //eslint-disable-line- sessionPool._removeSession = (session) => { //eslint-disable-line- isCalled = true;- return oldRemove.bind(sessionPool)(session);- };+ sessionPool._removeSession = (session) => { //eslint-disable-line+ isCalled = true;+ return oldRemove.bind(sessionPool)(session);+ };- await sessionPool.getSession();+ await sessionPool.getSession();- expect(isCalled).toBe(true); //eslint-disable-line- expect(sessionPool.sessions[0].id === session.id).toBe(false); //eslint-disable-line- expect(sessionPool.sessions).toHaveLength(1);- });+ expect(isCalled).toBe(true); //eslint-disable-line+ expect(sessionPool.sessions[0].id === session.id).toBe(false); //eslint-disable-line+ expect(sessionPool.sessions).toHaveLength(1);+ }+ );});- it('should persist state and recreate it from storage', async () => {+ test('should persist state and recreate it from storage', async () => {await sessionPool.getSession();await sessionPool.persistState();@@ -147,7 +150,7 @@ describe('SessionPool - testing session pool', async () => {expect(loadedSessionPool).toEqual(expect.arrayContaining(Object.keys(sessionPool)));});- it('should create only maxPoolSize number of sessions', async () => {+ test('should create only maxPoolSize number of sessions', async () => {const max = sessionPool.maxPoolSize;for (let i = 0; i < max + 100; i++) {await sessionPool.getSession();@@ -155,7 +158,7 @@ describe('SessionPool - testing session pool', async () => {expect(sessionPool.sessions.length).toEqual(sessionPool.maxPoolSize);});- it('should create session', async () => {+ test('should create session', async () => {await sessionPool._createSession(); // eslint-disable-lineexpect(sessionPool.sessions.length).toBe(1);expect(sessionPool.sessions[0].id).toBeDefined(); // eslint-disable-line@@ -173,7 +176,7 @@ describe('SessionPool - testing session pool', async () => {await emptyLocalStorageSubdir(`key_value_stores/${KV_STORE}`);});- it('on persist event', async () => {+ test('on persist event', async () => {await sessionPool.getSession();expect(sessionPool.sessions.length).toBe(1);@@ -196,7 +199,7 @@ describe('SessionPool - testing session pool', async () => {});});- it('should remove session', async () => {+ test('should remove session', async () => {for (let i = 0; i < 10; i++) {await sessionPool.getSession();}@@ -205,7 +208,7 @@ describe('SessionPool - testing session pool', async () => {expect(sessionPool.sessions.find(s => s.id === picked.id)).toEqual(undefined);});- it('should recreate only usable sessions', async () => {+ test('should recreate only usable sessions', async () => {let invalidSessionsCount = 0;for (let i = 0; i < 10; i++) {const session = await sessionPool.getSession();@@ -225,7 +228,7 @@ describe('SessionPool - testing session pool', async () => {expect(newSessionPool.sessions).toHaveLength(10 - invalidSessionsCount);});- it('should createSessionFunction work', async () => {+ test('should createSessionFunction work', async () => {let isCalled;const createSessionFunction = (sessionPool2) => {isCalled = true;diff --git a/__tests__/settings_rotator.js b/test/settings_rotator.test.jssimilarity index 93%rename from __tests__/settings_rotator.jsrename to test/settings_rotator.test.jsindex 5dca986d6..9126c6470 100644--- a/__tests__/settings_rotator.js+++ b/test/settings_rotator.test.js@@ -2,7 +2,7 @@ import _ from 'underscore';import Apify from '../build/index';describe('Apify.SettingsRotator', () => {- it('works', async () => {+ test('works', async () => {const maxUsages = 12;const totalCalls = 167;diff --git a/__tests__/stealth/stealth.js b/test/stealth/stealth.test.jssimilarity index 83%rename from __tests__/stealth/stealth.jsrename to test/stealth/stealth.test.jsindex 6b2f2709d..a1d339f13 100644--- a/__tests__/stealth/stealth.js+++ b/test/stealth/stealth.test.js@@ -13,11 +13,11 @@ const getFingerPrint = async (page) => {// we can speed up the test to make the requests to the local static htmldescribe('Stealth - testing headless chrome hiding tricks', () => {- before(async () => {+ beforeAll(async () => {process.env.APIFY_LOCAL_STORAGE_DIR = LOCAL_STORAGE_DIR;await emptyLocalStorageSubdir('key_value_stores/default');});- it('it adds plugins, mimeTypes and passes', async () => {+ test('it adds plugins, mimeTypes and passes', async () => {const browser = await Apify.launchPuppeteer({stealth: true,stealthOptions: {@@ -49,7 +49,7 @@ describe('Stealth - testing headless chrome hiding tricks', () => {return browser.close();});- it('it hides webDriver', async () => {+ test('it hides webDriver', async () => {const browser = await Apify.launchPuppeteer({stealth: true,headless: true,@@ -65,7 +65,7 @@ describe('Stealth - testing headless chrome hiding tricks', () => {return browser.close();});- it('it hacks permissions', async () => {+ test('it hacks permissions', async () => {const browser = await Apify.launchPuppeteer({stealth: true,headless: true,@@ -81,7 +81,7 @@ describe('Stealth - testing headless chrome hiding tricks', () => {return browser.close();});- it('it adds language to navigator', async () => {+ test('it adds language to navigator', async () => {const browser = await Apify.launchPuppeteer({stealth: true,headless: true,@@ -98,7 +98,7 @@ describe('Stealth - testing headless chrome hiding tricks', () => {return browser.close();});- it('it emulates WebGL', async () => {+ test('it emulates WebGL', async () => {const browser = await Apify.launchPuppeteer({stealth: true,headless: true,@@ -115,7 +115,7 @@ describe('Stealth - testing headless chrome hiding tricks', () => {return browser.close();});- it('it emulates windowFrame', async () => {+ test('it emulates windowFrame', async () => {const browser = await Apify.launchPuppeteer({stealth: true,headless: true,@@ -132,7 +132,7 @@ describe('Stealth - testing headless chrome hiding tricks', () => {return browser.close();});- it('it emulates console.debug', async () => {+ test('it emulates console.debug', async () => {const browser = await Apify.launchPuppeteer({stealth: true,headless: true,@@ -147,7 +147,7 @@ describe('Stealth - testing headless chrome hiding tricks', () => {return browser.close();});- it('it should mock window.chrome to plain object', async () => {+ test('it should mock window.chrome to plain object', async () => {const browser = await Apify.launchPuppeteer({stealth: true,headless: true,@@ -165,7 +165,7 @@ describe('Stealth - testing headless chrome hiding tricks', () => {return browser.close();});- it('it should mock chrome when iframe is created', async () => {+ test('it should mock chrome when iframe is created', async () => {const browser = await Apify.launchPuppeteer({stealth: true,headless: true,@@ -181,7 +181,7 @@ describe('Stealth - testing headless chrome hiding tricks', () => {return browser.close();});- it('it should not break iframe ', async () => {+ test('it should not break iframe ', async () => {const browser = await Apify.launchPuppeteer({stealth: true,headless: true,@@ -206,7 +206,7 @@ describe('Stealth - testing headless chrome hiding tricks', () => {return browser.close();});- it('it should mock device memory', async () => {+ test('it should mock device memory', async () => {const browser = await Apify.launchPuppeteer({stealth: true,headless: true,@@ -222,25 +222,28 @@ describe('Stealth - testing headless chrome hiding tricks', () => {return browser.close();});- it('it should bypass all of the known tests for browser fingerprinting', async () => {- const browser = await Apify.launchPuppeteer({- stealth: true,- headless: true,- useChrome: true,- });+ test(+ 'it should bypass all of the known tests for browser fingerprinting',+ async () => {+ const browser = await Apify.launchPuppeteer({+ stealth: true,+ headless: true,+ useChrome: true,+ });- const page = await browser.newPage();- await page.goto('file://test_html.html');- const fingerPrint = await getFingerPrint(page);- const testedFingerprint = scanner.analyseFingerprint(fingerPrint);- const failedChecks = Object.values(testedFingerprint).filter(val => val.consistent < 3);+ const page = await browser.newPage();+ await page.goto('file://test_html.html');+ const fingerPrint = await getFingerPrint(page);+ const testedFingerprint = scanner.analyseFingerprint(fingerPrint);+ const failedChecks = Object.values(testedFingerprint).filter(val => val.consistent < 3);- expect(failedChecks.length).toBe(0);+ expect(failedChecks.length).toBe(0);- return browser.close();- });+ return browser.close();+ }+ );- it('should work in crawler', async () => {+ test('should work in crawler', async () => {const requestList = await Apify.openRequestList('test', ['file://test_html.html']);const values = [];const puppeteerCrawler = new Apify.PuppeteerCrawler({diff --git a/__tests__/stealth/test_html.html b/test/stealth/test_html.htmlsimilarity index 100%rename from __tests__/stealth/test_html.htmlrename to test/stealth/test_html.htmldiff --git a/__tests__/utils.js b/test/utils.test.jssimilarity index 78%rename from __tests__/utils.jsrename to test/utils.test.jsindex 07ed401ad..3dbdac23c 100644--- a/__tests__/utils.js+++ b/test/utils.test.js@@ -13,7 +13,7 @@ import * as utils from '../build/utils';import Apify from '../build/index';describe('utils.newClient()', () => {- it('reads environment variables correctly', () => {+ test('reads environment variables correctly', () => {process.env[ENV_VARS.API_BASE_URL] = 'http://www.example.com:1234/path/';process.env[ENV_VARS.USER_ID] = 'userId';process.env[ENV_VARS.TOKEN] = 'token';@@ -27,7 +27,7 @@ describe('utils.newClient()', () => {expect(opts.baseUrl).toBe('http://www.example.com:1234/path/');});- it('uses correct default if APIFY_API_BASE_URL is not defined', () => {+ test('uses correct default if APIFY_API_BASE_URL is not defined', () => {delete process.env[ENV_VARS.API_BASE_URL];process.env[ENV_VARS.USER_ID] = 'userId';process.env[ENV_VARS.TOKEN] = 'token';@@ -42,7 +42,7 @@ describe('utils.newClient()', () => {});describe('utils.addCharsetToContentType()', () => {- it('works', () => {+ test('works', () => {expect(utils.addCharsetToContentType('application/json; charset=something')).toBe('application/json; charset=something');expect(utils.addCharsetToContentType('application/json; foo=bar; charset=something')).toBe('application/json; foo=bar; charset=something');expect(utils.addCharsetToContentType('application/json; foo=bar')).toBe('application/json; charset=utf-8; foo=bar');@@ -53,7 +53,7 @@ describe('utils.addCharsetToContentType()', () => {});describe('utils.isProduction()', () => {- it('works', () => {+ test('works', () => {const prev = process.env.NODE_ENV;try {process.env.NODE_ENV = 'production';@@ -68,7 +68,7 @@ describe('utils.isProduction()', () => {});describe('utils.isDocker()', () => {- it('works for dockerenv && cgroup', () => {+ test('works for dockerenv && cgroup', () => {sinon.stub(fs, 'stat').callsFake((filePath, callback) => callback(null));sinon.stub(fs, 'readFile').callsFake((filePath, encoding, callback) => callback(null, 'something ... docker ... something'));@@ -81,7 +81,7 @@ describe('utils.isDocker()', () => {});});- it('works for dockerenv', () => {+ test('works for dockerenv', () => {sinon.stub(fs, 'stat').callsFake((filePath, callback) => callback(null));sinon.stub(fs, 'readFile').callsFake((filePath, encoding, callback) => callback(null, 'something ... ... something'));@@ -94,7 +94,7 @@ describe('utils.isDocker()', () => {});});- it('works for cgroup', () => {+ test('works for cgroup', () => {sinon.stub(fs, 'stat').callsFake((filePath, callback) => callback(new Error()));sinon.stub(fs, 'readFile').callsFake((filePath, encoding, callback) => callback(null, 'something ... docker ... something'));@@ -107,7 +107,7 @@ describe('utils.isDocker()', () => {});});- it('works for nothing', () => {+ test('works for nothing', () => {sinon.stub(fs, 'stat').callsFake((filePath, callback) => callback(new Error()));sinon.stub(fs, 'readFile').callsFake((filePath, encoding, callback) => callback(null, 'something ... ... something'));@@ -122,7 +122,7 @@ describe('utils.isDocker()', () => {});describe('utils.getMemoryInfo()', () => {- it('works WITHOUT child process outside the container', () => {+ test('works WITHOUT child process outside the container', () => {const osMock = sinon.mock(os);const utilsMock = sinon.mock(utils);@@ -157,7 +157,7 @@ describe('utils.getMemoryInfo()', () => {});});- it('works WITHOUT child process inside the container', () => {+ test('works WITHOUT child process inside the container', () => {const utilsMock = sinon.mock(utils);utilsMock@@ -189,7 +189,7 @@ describe('utils.getMemoryInfo()', () => {});});- it('works WITH child process outside the container', () => {+ test('works WITH child process outside the container', () => {const osMock = sinon.mock(os);const utilsMock = sinon.mock(utils);process.env[ENV_VARS.HEADLESS] = '1';@@ -229,7 +229,7 @@ describe('utils.getMemoryInfo()', () => {});});- it('works WITH child process inside the container', () => {+ test('works WITH child process inside the container', () => {const utilsMock = sinon.mock(utils);process.env[ENV_VARS.HEADLESS] = '1';@@ -268,7 +268,7 @@ describe('utils.getMemoryInfo()', () => {});describe('utils.isPromise()', () => {- it('works', () => {+ test('works', () => {const rejected = Promise.reject();expect(utils.isPromise(new Promise(resolve => setTimeout(resolve, 1000)))).toBe(true);@@ -284,13 +284,13 @@ describe('utils.isPromise()', () => {});describe('utils.newPromise()', () => {- it('works', () => {+ test('works', () => {if (!utils.isPromise(utils.newPromise())) throw new Error('utils.newPromise() must return a promise!');});});describe('utils.isAtHome()', () => {- it('works', () => {+ test('works', () => {expect(utils.isAtHome()).toBe(false);process.env[ENV_VARS.IS_AT_HOME] = 1;expect(utils.isAtHome()).toBe(true);@@ -300,7 +300,7 @@ describe('utils.isAtHome()', () => {});describe('utils.sum()', () => {- it('works', () => {+ test('works', () => {expect(utils.sum([1, 2, 3, 1.2])).toBe(7.2);expect(utils.sum([])).toBe(0);expect(utils.sum([9])).toBe(9);@@ -308,7 +308,7 @@ describe('utils.sum()', () => {});describe('utils.avg()', () => {- it('works', () => {+ test('works', () => {expect(utils.avg([1, 2, 3, 1.2])).toEqual(7.2 / 4);expect(utils.avg([])).toEqual(NaN);expect(utils.avg([9])).toBe(9);@@ -316,7 +316,7 @@ describe('utils.avg()', () => {});describe('utils.weightedAvg()', () => {- it('works', () => {+ test('works', () => {expect(utils.weightedAvg([10, 10, 10], [1, 1, 1])).toBe(10);expect(utils.weightedAvg([5, 10, 15], [1, 1, 1])).toBe(10);expect(utils.weightedAvg([10, 10, 10], [0.5, 1, 1.5])).toBe(10);@@ -328,7 +328,7 @@ describe('utils.weightedAvg()', () => {});describe('Apify.utils.sleep()', () => {- it('works', () => {+ test('works', () => {let timeBefore;return Promise.resolve().then(() => {@@ -390,116 +390,116 @@ describe('Apify.utils.extractUrls()', () => {return array.reduce((string, url) => string.replace(ID, ` ${url} `), text.join(''));};- it('extracts simple URLs', () => {+ test('extracts simple URLs', () => {const { string, array } = getURLData(SIMPLE_URL_LIST);const extracted = extractUrls({ string });expect(extracted).toEqual(array);});- it('extracts unicode URLs', () => {+ test('extracts unicode URLs', () => {const { string, array } = getURLData(UNICODE_URL_LIST);const extracted = extractUrls({ string });expect(extracted).toEqual(array);});- it('extracts unicode URLs with commas', () => {+ test('extracts unicode URLs with commas', () => {const { string, array } = getURLData(COMMA_URL_LIST);const extracted = extractUrls({ string, urlRegExp: URL_WITH_COMMAS_REGEX });expect(extracted).toEqual(array);});- it('extracts tricky URLs', () => {+ test('extracts tricky URLs', () => {const { string, array } = getURLData(TRICKY_URL_LIST);const extracted = extractUrls({ string });expect(extracted).toEqual(array);});- it('does not extract invalid URLs', () => {+ test('does not extract invalid URLs', () => {const { string } = getURLData(INVALID_URL_LIST);const extracted = extractUrls({ string });expect(extracted).toEqual(['http://www.foo.bar']);});- it('extracts simple URLs from JSON', () => {+ test('extracts simple URLs from JSON', () => {const d = getURLData(SIMPLE_URL_LIST);const string = makeJSON(d);const extracted = extractUrls({ string });expect(extracted).toEqual(d.array.concat(d.array));});- it('extracts unicode URLs from JSON', () => {+ test('extracts unicode URLs from JSON', () => {const d = getURLData(UNICODE_URL_LIST);const string = makeJSON(d);const extracted = extractUrls({ string });expect(extracted).toEqual(d.array.concat(d.array));});- it('extracts unicode URLs with commas from JSON', () => {+ test('extracts unicode URLs with commas from JSON', () => {const d = getURLData(COMMA_URL_LIST);const string = makeJSON(d);const extracted = extractUrls({ string, urlRegExp: URL_WITH_COMMAS_REGEX });expect(extracted).toEqual(d.array.concat(d.array));});- it('extracts tricky URLs from JSON', () => {+ test('extracts tricky URLs from JSON', () => {const d = getURLData(TRICKY_URL_LIST);const string = makeJSON(d);const extracted = extractUrls({ string });expect(extracted).toEqual(d.array.concat(d.array));});- it('does not extract invalid URLs from JSON', () => {+ test('does not extract invalid URLs from JSON', () => {const d = getURLData(INVALID_URL_LIST);const string = makeJSON(d);const extracted = extractUrls({ string });expect(extracted).toEqual(['http://www.foo.bar', 'http://www.foo.bar']);});- it('extracts simple URLs from CSV', () => {+ test('extracts simple URLs from CSV', () => {const { array } = getURLData(SIMPLE_URL_LIST);const string = makeCSV(array);const extracted = extractUrls({ string });expect(extracted).toEqual(array);});- it('extracts unicode URLs from CSV', () => {+ test('extracts unicode URLs from CSV', () => {const { array } = getURLData(UNICODE_URL_LIST);const string = makeCSV(array);const extracted = extractUrls({ string });expect(extracted).toEqual(array);});- it('extracts unicode URLs with commas from semicolon CSV', () => {+ test('extracts unicode URLs with commas from semicolon CSV', () => {const { array } = getURLData(COMMA_URL_LIST);const string = makeCSV(array, ';');const extracted = extractUrls({ string, urlRegExp: URL_WITH_COMMAS_REGEX });expect(extracted).toEqual(array);});- it('extracts tricky URLs from CSV', () => {+ test('extracts tricky URLs from CSV', () => {const { array } = getURLData(TRICKY_URL_LIST);const string = makeCSV(array);const extracted = extractUrls({ string });expect(extracted).toEqual(array);});- it('does not extract invalid URLs from CSV', () => {+ test('does not extract invalid URLs from CSV', () => {const { array } = getURLData(INVALID_URL_LIST);const string = makeCSV(array);const extracted = extractUrls({ string });expect(extracted).toEqual(['http://www.foo.bar']);});- it('extracts simple URLs from Text', () => {+ test('extracts simple URLs from Text', () => {const { array } = getURLData(SIMPLE_URL_LIST);const string = makeText(array);const extracted = extractUrls({ string });expect(extracted).toEqual(array);});- it('extracts unicode URLs from Text', () => {+ test('extracts unicode URLs from Text', () => {const { array } = getURLData(UNICODE_URL_LIST);const string = makeText(array);const extracted = extractUrls({ string });expect(extracted).toEqual(array);});- it('extracts unicode URLs with commas from Text', () => {+ test('extracts unicode URLs with commas from Text', () => {const { array } = getURLData(COMMA_URL_LIST);const string = makeText(array);const extracted = extractUrls({ string, urlRegExp: URL_WITH_COMMAS_REGEX });expect(extracted).toEqual(array);});- it('extracts tricky URLs from Text', () => {+ test('extracts tricky URLs from Text', () => {const { array } = getURLData(TRICKY_URL_LIST);const string = makeText(array);const extracted = extractUrls({ string });expect(extracted).toEqual(array);});- it('does not extract invalid URLs from Text', () => {+ test('does not extract invalid URLs from Text', () => {const { array } = getURLData(INVALID_URL_LIST);const string = makeText(array);const extracted = extractUrls({ string });@@ -517,7 +517,7 @@ describe('Apify.utils.downloadListOfUrls()', () => {requestPromise.get.restore();});- it('downloads a list of URLs', () => {+ test('downloads a list of URLs', () => {const text = fs.readFileSync(path.join(__dirname, 'data', 'simple_url_list.txt'), 'utf8');const arr = text.trim().split(/[\r\n]+/g).map(u => u.trim());stub.resolves(text);@@ -529,7 +529,7 @@ describe('Apify.utils.downloadListOfUrls()', () => {});describe('Apify.utils.getRandomUserAgent()', () => {- it('works', () => {+ test('works', () => {const agent = utils.publicUtils.getRandomUserAgent();expect(typeof agent).toBe('string');expect(agent.length).not.toBe(0);@@ -537,121 +537,139 @@ describe('Apify.utils.getRandomUserAgent()', () => {});describe('utils.openLocalStorage()', async () => {- it('should return item from cache if available and create new one otherwise', async () => {- const cache = new LruCache({ maxLength: 5 });- class MyStore {}+ test(+ 'should return item from cache if available and create new one otherwise',+ async () => {+ const cache = new LruCache({ maxLength: 5 });+ class MyStore {}- expect(cache.length()).toBe(0);+ expect(cache.length()).toBe(0);- const store = await utils.openLocalStorage('some-id', 'some-env', MyStore, cache);- expect(store).toBeInstanceOf(MyStore);- expect(cache.length()).toBe(1);+ const store = await utils.openLocalStorage('some-id', 'some-env', MyStore, cache);+ expect(store).toBeInstanceOf(MyStore);+ expect(cache.length()).toBe(1);- const store2 = await utils.openLocalStorage('some-id', 'some-env', MyStore, cache);- expect(store2).toBe(store);- expect(cache.length()).toBe(1);+ const store2 = await utils.openLocalStorage('some-id', 'some-env', MyStore, cache);+ expect(store2).toBe(store);+ expect(cache.length()).toBe(1);- const store3 = await utils.openLocalStorage('some-other-id', 'some-env', MyStore, cache);- expect(store3).not.toBe(store);- expect(cache.length()).toBe(2);- });-- it('should use ID from ENV variable if no parameter is provided', async () => {- const cache = new LruCache({ maxLength: 5 });- class MyStore {- constructor(id) {- this.id = id;- }+ const store3 = await utils.openLocalStorage('some-other-id', 'some-env', MyStore, cache);+ expect(store3).not.toBe(store);+ expect(cache.length()).toBe(2);}+ );++ test(+ 'should use ID from ENV variable if no parameter is provided',+ async () => {+ const cache = new LruCache({ maxLength: 5 });+ class MyStore {+ constructor(id) {+ this.id = id;+ }+ }- process.env['some-env'] = 'id-from-env';-- const store = await utils.openLocalStorage(null, 'some-env', MyStore, cache);- expect(store.id).toBe('id-from-env');+ process.env['some-env'] = 'id-from-env';- delete process.env['some-env'];- });+ const store = await utils.openLocalStorage(null, 'some-env', MyStore, cache);+ expect(store.id).toBe('id-from-env');- it('should use ID from shared if neither parameter nor ENV var is provided', async () => {- const cache = new LruCache({ maxLength: 5 });- class MyStore {- constructor(id) {- this.id = id;- }+ delete process.env['some-env'];}+ );++ test(+ 'should use ID from shared if neither parameter nor ENV var is provided',+ async () => {+ const cache = new LruCache({ maxLength: 5 });+ class MyStore {+ constructor(id) {+ this.id = id;+ }+ }- // There is some default in shared constants.- const defaultLocalValue = LOCAL_ENV_VARS[ENV_VARS.DEFAULT_KEY_VALUE_STORE_ID];- expect(typeof defaultLocalValue).toBe('string');- expect(defaultLocalValue.length).toBeGreaterThan(1);+ // There is some default in shared constants.+ const defaultLocalValue = LOCAL_ENV_VARS[ENV_VARS.DEFAULT_KEY_VALUE_STORE_ID];+ expect(typeof defaultLocalValue).toBe('string');+ expect(defaultLocalValue.length).toBeGreaterThan(1);- // There is no env var!- expect(process.env[ENV_VARS.DEFAULT_KEY_VALUE_STORE_ID]).toEqual(undefined);+ // There is no env var!+ expect(process.env[ENV_VARS.DEFAULT_KEY_VALUE_STORE_ID]).toEqual(undefined);- const store = await utils.openLocalStorage(null, ENV_VARS.DEFAULT_KEY_VALUE_STORE_ID, MyStore, cache);- expect(store.id).toEqual(defaultLocalValue);- });+ const store = await utils.openLocalStorage(null, ENV_VARS.DEFAULT_KEY_VALUE_STORE_ID, MyStore, cache);+ expect(store.id).toEqual(defaultLocalValue);+ }+ );});describe('utils.openRemoteStorage()', async () => {- it('should return item from cache if available and create new one otherwise', async () => {- const cache = new LruCache({ maxLength: 5 });- class MyStore {- constructor(id) {- this.id = id;+ test(+ 'should return item from cache if available and create new one otherwise',+ async () => {+ const cache = new LruCache({ maxLength: 5 });+ class MyStore {+ constructor(id) {+ this.id = id;+ }}- }- delete process.env['some-env'];+ delete process.env['some-env'];- expect(cache.length()).toBe(0);+ expect(cache.length()).toBe(0);- const store = await utils.openRemoteStorage('some-id', 'some-env', MyStore, cache, async () => ({ id: 'some-id' }));- expect(store.id).toBe('some-id');- expect(cache.length()).toBe(1);+ const store = await utils.openRemoteStorage('some-id', 'some-env', MyStore, cache, async () => ({ id: 'some-id' }));+ expect(store.id).toBe('some-id');+ expect(cache.length()).toBe(1);- const store2 = await utils.openRemoteStorage('some-id', 'some-env', MyStore, cache, async () => { throw new Error('Should not be called!'); }); // eslint-disable-line- expect(store2.id).toBe('some-id');- expect(store2).toBe(store);- expect(cache.length()).toBe(1);+ const store2 = await utils.openRemoteStorage('some-id', 'some-env', MyStore, cache, async () => { throw new Error('Should not be called!'); }); // eslint-disable-line+ expect(store2.id).toBe('some-id');+ expect(store2).toBe(store);+ expect(cache.length()).toBe(1);- const store3 = await utils.openRemoteStorage('some-other-id', 'some-env', MyStore, cache, async () => ({ id: 'some-other-id' }));- expect(store3).not.toBe(store);- expect(store3.id).toBe('some-other-id');- expect(cache.length()).toBe(2);- });-- it('should use ID from ENV variable if no parameter is provided', async () => {- const cache = new LruCache({ maxLength: 5 });- class MyStore {- constructor(id) {- this.id = id;- }+ const store3 = await utils.openRemoteStorage('some-other-id', 'some-env', MyStore, cache, async () => ({ id: 'some-other-id' }));+ expect(store3).not.toBe(store);+ expect(store3.id).toBe('some-other-id');+ expect(cache.length()).toBe(2);}+ );++ test(+ 'should use ID from ENV variable if no parameter is provided',+ async () => {+ const cache = new LruCache({ maxLength: 5 });+ class MyStore {+ constructor(id) {+ this.id = id;+ }+ }- process.env['some-env'] = 'id-from-env';-- const store = await utils.openLocalStorage(null, 'some-env', MyStore, cache);- expect(store.id).toBe('id-from-env');+ process.env['some-env'] = 'id-from-env';- delete process.env['some-env'];- });+ const store = await utils.openLocalStorage(null, 'some-env', MyStore, cache);+ expect(store.id).toBe('id-from-env');- it('should use ID from ENV variable and not call getOrCreateStoreFunction parameter is not provided', async () => {- const cache = new LruCache({ maxLength: 5 });- class MyStore {- constructor(id) {- this.id = id;- }+ delete process.env['some-env'];}+ );++ test(+ 'should use ID from ENV variable and not call getOrCreateStoreFunction parameter is not provided',+ async () => {+ const cache = new LruCache({ maxLength: 5 });+ class MyStore {+ constructor(id) {+ this.id = id;+ }+ }- process.env['some-env'] = 'id-from-env';+ process.env['some-env'] = 'id-from-env';- const store = await utils.openRemoteStorage(null, 'some-env', MyStore, cache, async () => { throw new Error('Should not be called!'); }); // eslint-disable-line- expect(store.id).toBe('id-from-env');+ const store = await utils.openRemoteStorage(null, 'some-env', MyStore, cache, async () => { throw new Error('Should not be called!'); }); // eslint-disable-line+ expect(store.id).toBe('id-from-env');- delete process.env['some-env'];- });+ delete process.env['some-env'];+ }+ );});const checkHtmlToText = (html, expectedText, hasBody = false) => {@@ -681,14 +699,14 @@ const checkHtmlToText = (html, expectedText, hasBody = false) => {};describe('utils.htmlToText()', () => {- it('handles invalid args', () => {+ test('handles invalid args', () => {checkHtmlToText(null, '');checkHtmlToText('', '');checkHtmlToText(0, '');checkHtmlToText(undefined, '');});- it('handles basic HTML elements correctly', () => {+ test('handles basic HTML elements correctly', () => {checkHtmlToText('Plain text node', 'Plain text node');checkHtmlToText(' Plain text node ', 'Plain text node');checkHtmlToText(' \nPlain text node \n ', 'Plain text node');@@ -738,11 +756,11 @@ describe('utils.htmlToText()', () => {);});- it('handles HTML entities correctly', () => {+ test('handles HTML entities correctly', () => {checkHtmlToText('<span>á é</span>', 'á é');});- it('handles larger HTML documents', () => {+ test('handles larger HTML documents', () => {const html1 = fs.readFileSync(path.join(__dirname, 'data', 'html_to_text_test.html'), 'utf8');const text1 = fs.readFileSync(path.join(__dirname, 'data', 'html_to_text_test.txt'), 'utf8');@@ -750,7 +768,7 @@ describe('utils.htmlToText()', () => {checkHtmlToText(html1, text1, true);});- it('works with Cheerio object', () => {+ test('works with Cheerio object', () => {const html1 = '<html><body>Some text</body></html>';checkHtmlToText(cheerio.load(html1, { decodeEntities: true }), 'Some text');@@ -760,7 +778,7 @@ describe('utils.htmlToText()', () => {});describe('utils.createRequestDebugInfo()', () => {- it('handles Puppeteer response', () => {+ test('handles Puppeteer response', () => {const request = {id: 'some-id',url: 'https://example.com',@@ -793,7 +811,7 @@ describe('utils.createRequestDebugInfo()', () => {});});- it('handles NodeJS response', () => {+ test('handles NodeJS response', () => {const request = {id: 'some-id',url: 'https://example.com',@@ -828,7 +846,7 @@ describe('utils.createRequestDebugInfo()', () => {});describe('utils.snakeCaseToCamelCase()', () => {- it('should camel case all sneaky cases of snake case', () => {+ test('should camel case all sneaky cases of snake case', () => {const tests = {aaa_bbb_: 'aaaBbb','': '',@@ -843,7 +861,7 @@ describe('utils.snakeCaseToCamelCase()', () => {});describe('utils.addTimeoutToPromise()', () => {- it('should timeout', async () => {+ test('should timeout', async () => {const clock = sinon.useFakeTimers();try {const p = utils.addTimeoutToPromise(@@ -861,7 +879,7 @@ describe('utils.addTimeoutToPromise()', () => {}});- it('should not timeout too soon', async () => {+ test('should not timeout too soon', async () => {const clock = sinon.useFakeTimers();try {const p = utils.addTimeoutToPromise(@@ -893,13 +911,13 @@ describe('utils.printOutdatedSdkWarning()', () => {logMock.restore();});- it('should do nothing when ENV_VARS.SDK_LATEST_VERSION is not set', () => {+ test('should do nothing when ENV_VARS.SDK_LATEST_VERSION is not set', () => {delete process.env[ENV_VARS.SDK_LATEST_VERSION];logMock.expects('warning').never();utils.printOutdatedSdkWarning();});- it('should correctly work when outdated', () => {+ test('should correctly work when outdated', () => {process.env[ENV_VARS.SDK_LATEST_VERSION] = semver.inc(currentVersion, 'minor');console.log(process.env[ENV_VARS.SDK_LATEST_VERSION]);logMock.expects('warning').once();@@ -907,7 +925,7 @@ describe('utils.printOutdatedSdkWarning()', () => {delete process.env[ENV_VARS.SDK_LATEST_VERSION];});- it('should correctly work when up to date', () => {+ test('should correctly work when up to date', () => {process.env[ENV_VARS.SDK_LATEST_VERSION] = '0.13.0';logMock.expects('warning').never();utils.printOutdatedSdkWarning();diff --git a/__tests__/utils_request.js b/test/utils_request.test.jssimilarity index 84%rename from __tests__/utils_request.jsrename to test/utils_request.test.jsindex a62194762..d5c2d6ef6 100644--- a/__tests__/utils_request.js+++ b/test/utils_request.test.js@@ -14,7 +14,7 @@ describe('Apify.utils_request', () => {let mochaListener;let port;let server;- before(async () => {+ beforeAll(async () => {mochaListener = process.listeners('uncaughtException').shift();process.removeListener('uncaughtException', mochaListener);const app = express();@@ -59,17 +59,20 @@ describe('Apify.utils_request', () => {});describe('Apify.requestAsBrowser', async () => {- it('it uses mobile user-agent whe mobile property is set to true ', async () => {- const data = {- url: `http://${HOST}:${port}/echo`,- useMobileVersion: true,- };- const response = await requestAsBrowser(data);- expect(response.statusCode).toBe(200);- expect(JSON.parse(response.body)['user-agent']).toEqual(FIREFOX_MOBILE_USER_AGENT);- });+ test(+ 'it uses mobile user-agent whe mobile property is set to true ',+ async () => {+ const data = {+ url: `http://${HOST}:${port}/echo`,+ useMobileVersion: true,+ };+ const response = await requestAsBrowser(data);+ expect(response.statusCode).toBe(200);+ expect(JSON.parse(response.body)['user-agent']).toEqual(FIREFOX_MOBILE_USER_AGENT);+ }+ );- it('uses desktop user-agent by default ', async () => {+ test('uses desktop user-agent by default ', async () => {const data = {url: `http://${HOST}:${port}/echo`,};@@ -78,7 +81,7 @@ describe('Apify.utils_request', () => {expect(JSON.parse(response.body)['user-agent']).toEqual(FIREFOX_DESKTOP_USER_AGENT);});- it('sets correct hosts', async () => {+ test('sets correct hosts', async () => {const host = `${HOST}:${port}`;const options = {url: `http://${host}/echo`,@@ -90,7 +93,7 @@ describe('Apify.utils_request', () => {expect(JSON.parse(response.body).host).toEqual(host);});- it('uses correct default language', async () => {+ test('uses correct default language', async () => {const { languageCode, countryCode } = REQUEST_AS_BROWSER_DEFAULT_OPTIONS;const host = `${HOST}:${port}`;const options = {@@ -103,7 +106,7 @@ describe('Apify.utils_request', () => {expect(JSON.parse(response.body)['accept-language']).toEqual(`${languageCode}-${countryCode},${languageCode};q=0.5`);});- it('throws error for 406', async () => {+ test('throws error for 406', async () => {const options = {url: `http://${HOST}:${port}/406`,};@@ -118,7 +121,7 @@ describe('Apify.utils_request', () => {expect(error.message).toEqual(`Request for ${options.url} aborted due to abortFunction`);});- it('does not throw for empty response body', async () => {+ test('does not throw for empty response body', async () => {const options = {url: `http://${HOST}:${port}/empty`,};@@ -132,7 +135,7 @@ describe('Apify.utils_request', () => {expect(error).toBeFalsy(); //eslint-disable-line});- it('throws for other contentType then - text/html', async () => {+ test('throws for other contentType then - text/html', async () => {const options = {url: `http://${HOST}:${port}/invalidContentType`,};@@ -147,7 +150,7 @@ describe('Apify.utils_request', () => {expect(error.message).toEqual(`Request for ${options.url} aborted due to abortFunction`);});- it('overrides defaults', async () => {+ test('overrides defaults', async () => {const host = `${HOST}:${port}`;const options = {url: `http://${host}/echo`,@@ -162,7 +165,7 @@ describe('Apify.utils_request', () => {expect(JSON.parse(response.body)['user-agent']).toEqual(options.headers['User-Agent']);});- it('headers has same format as in firefox', async () => {+ test('headers has same format as in firefox', async () => {const host = `${HOST}:${port}`;const options = {url: `http://${host}/rawHeaders`,diff --git a/__tests__/utils_social.js b/test/utils_social.test.jssimilarity index 97%rename from __tests__/utils_social.jsrename to test/utils_social.test.jsindex 8cc75135c..d1729a723 100644--- a/__tests__/utils_social.js+++ b/test/utils_social.test.js@@ -1,5 +1,4 @@import _ from 'underscore';-import chai, { expect } from 'chai';import chaiAsPromised from 'chai-as-promised';// import * as utils from '../build/utils';import Apify from '../build/index';@@ -14,7 +13,7 @@ describe('utils.social', () => {expect(social.emailsFromText(text)).to.eql(phones);};- it('works with arg with no emails or invalid', () => {+ test('works with arg with no emails or invalid', () => {expect(social.emailsFromText()).to.eql([]);testEmailsFromText('', []);testEmailsFromText(null, []);@@ -24,7 +23,7 @@ describe('utils.social', () => {testEmailsFromText(' \n\n\r\t\n ', []);});- it('extracts emails correctly', () => {+ test('extracts emails correctly', () => {testEmailsFromText(' [email protected] ', ['[email protected]']);testEmailsFromText(`@@ -55,7 +54,7 @@ describe('utils.social', () => {});describe('emailsFromUrls()', () => {- it('throws on invalid arg', () => {+ test('throws on invalid arg', () => {expect(() => {social.emailsFromUrls();}).to.throw(/must be an array/);@@ -73,7 +72,7 @@ describe('utils.social', () => {}).to.throw(/must be an array/);});- it('extracts emails correctly', () => {+ test('extracts emails correctly', () => {expect(social.emailsFromUrls([])).to.eql([]);expect(social.emailsFromUrls([1, 2, {}, 'fwef', null, undefined])).to.eql([]);@@ -114,7 +113,7 @@ describe('utils.social', () => {expect(social.phonesFromText(text)).to.eql(phones);};- it('works with arg with no phones or invalid', () => {+ test('works with arg with no phones or invalid', () => {expect(social.phonesFromText()).to.eql([]);testPhonesFromText('', []);@@ -125,7 +124,7 @@ describe('utils.social', () => {testPhonesFromText(' \n\n\r\t\n ', []);});- it('extracts phones correctly', () => {+ test('extracts phones correctly', () => {testPhonesFromText(`+420775123456 +420775123456@@ -234,7 +233,7 @@ describe('utils.social', () => {]);});- it('skips invalid phones', () => {+ test('skips invalid phones', () => {testPhonesFromText(`2018-10-11 123456789 345@@ -244,7 +243,7 @@ describe('utils.social', () => {});describe('phonesFromUrls()', () => {- it('throws on invalid arg', () => {+ test('throws on invalid arg', () => {expect(() => {social.phonesFromUrls();}).to.throw(/must be an array/);@@ -262,7 +261,7 @@ describe('utils.social', () => {}).to.throw(/must be an array/);});- it('extracts phones correctly', () => {+ test('extracts phones correctly', () => {expect(social.phonesFromUrls([])).to.eql([]);expect(social.phonesFromUrls([1, 2, {}, 'fwef', null, undefined])).to.eql([]);@@ -320,7 +319,7 @@ describe('utils.social', () => {facebooks: [],};- it('handles invalid arg', () => {+ test('handles invalid arg', () => {expect(social.parseHandlesFromHtml()).to.eql(EMPTY_RESULT);expect(social.parseHandlesFromHtml(undefined)).to.eql(EMPTY_RESULT);expect(social.parseHandlesFromHtml(null)).to.eql(EMPTY_RESULT);@@ -328,7 +327,7 @@ describe('utils.social', () => {expect(social.parseHandlesFromHtml(1234)).to.eql(EMPTY_RESULT);});- it('works', () => {+ test('works', () => {expect(social.parseHandlesFromHtml('')).to.eql(EMPTY_RESULT);expect(social.parseHandlesFromHtml(' ')).to.eql(EMPTY_RESULT);@@ -422,7 +421,7 @@ describe('utils.social', () => {});});- it('data is set correctly', () => {+ test('data is set correctly', () => {const data = {};social.parseHandlesFromHtml(`<html>@@ -441,7 +440,7 @@ describe('utils.social', () => {});describe('EMAIL_REGEX', () => {- it('works', () => {+ test('works', () => {expect(_.isRegExp(social.EMAIL_REGEX)).to.eql(true);expect(_.isRegExp(social.EMAIL_REGEX_GLOBAL)).to.eql(true);@@ -471,7 +470,7 @@ describe('utils.social', () => {});describe('LINKEDIN_REGEX', () => {- it('works', () => {+ test('works', () => {expect(_.isRegExp(social.LINKEDIN_REGEX)).to.eql(true);expect(_.isRegExp(social.LINKEDIN_REGEX_GLOBAL)).to.eql(true);@@ -542,7 +541,7 @@ describe('utils.social', () => {});describe('INSTAGRAM_REGEX', () => {- it('works', () => {+ test('works', () => {expect(_.isRegExp(social.INSTAGRAM_REGEX)).to.eql(true);expect(_.isRegExp(social.INSTAGRAM_REGEX_GLOBAL)).to.eql(true);@@ -610,7 +609,7 @@ describe('utils.social', () => {});describe('TWITTER_REGEX', () => {- it('works', () => {+ test('works', () => {expect(_.isRegExp(social.TWITTER_REGEX)).to.eql(true);expect(_.isRegExp(social.TWITTER_REGEX_GLOBAL)).to.eql(true);@@ -679,7 +678,7 @@ describe('utils.social', () => {});describe('FACEBOOK_REGEX', () => {- it('works', () => {+ test('works', () => {expect(_.isRegExp(social.FACEBOOK_REGEX)).to.eql(true);expect(_.isRegExp(social.FACEBOOK_REGEX_GLOBAL)).to.eql(true);diff --git a/__tests__/integration_tests/utils.js b/test_integration/utils.jssimilarity index 93%rename from __tests__/integration_tests/utils.jsrename to test_integration/utils.jsindex 1d253fd61..6e0676b55 100644--- a/__tests__/integration_tests/utils.js+++ b/test_integration/utils.js@@ -1,6 +1,6 @@// import { expect } from 'chai';-import * as utils from '../../build/utils';-import Apify from '../../build/index';+import * as utils from '../build/utils';+import Apify from '../build';describe('utils.infiniteScroll()', () => {xit('exits after it reaches the bottom', async () => {