Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
Remove time reliant expects
Ondra Urban245d2758365d
diff --git a/package.json b/package.jsonindex 129d57816..05f94b86b 100644--- a/package.json+++ b/package.json@@ -40,7 +40,7 @@"build": "rm -rf ./build && babel src --out-dir build --copy-files","build-docs": "npm run clean && npm run build && node ./website/tools/build_docs.js","build-readme": "node ./tools/build_readme.js",- "test": "npm run build && jest --testTimeout=60000 --coverage --maxWorkers=2",+ "test": "npm run build && jest --testTimeout=10000 --coverage --maxWorkers=2","prepare": "npm run build","prepublishOnly": "(test $TRAVIS || (echo \"Publishing is reserved to the Travis CI!\"; exit 1))","clean": "rm -rf build",diff --git a/test/autoscaling/autoscaled_pool.test.js b/test/autoscaling/autoscaled_pool.test.jsindex 9d5777796..61c5b92fe 100644--- a/test/autoscaling/autoscaled_pool.test.js+++ b/test/autoscaling/autoscaled_pool.test.js@@ -41,12 +41,9 @@ describe('AutoscaledPool', () => {isFinishedFunction: () => Promise.resolve(isFinished),isTaskReadyFunction: () => Promise.resolve(!isFinished),});- const startedAt = Date.now();await pool.run();expect(result).toEqual(_.range(0, 10));- expect(Date.now() - startedAt).toBeGreaterThanOrEqual(50);- expect(Date.now() - startedAt).toBeLessThanOrEqual(200);});test('should work with concurrency 10', async () => {@@ -75,12 +72,9 @@ describe('AutoscaledPool', () => {isTaskReadyFunction: () => Promise.resolve(!isFinished),});- const startedAt = Date.now();await pool.run();expect(result).toEqual(_.range(0, 100));- expect(Date.now() - startedAt).toBeGreaterThanOrEqual(50);- expect(Date.now() - startedAt).toBeLessThanOrEqual(200);});test('enables setting concurrency', async () => {@@ -115,7 +109,6 @@ describe('AutoscaledPool', () => {expect(pool.maxConcurrency).toBe(13);expect(pool.desiredConcurrency).toBe(9);- const startedAt = Date.now();const promise = await pool.run();// Test setting concurrency@@ -130,8 +123,6 @@ describe('AutoscaledPool', () => {await promise;expect(result).toEqual(_.range(0, 100));- expect(Date.now() - startedAt).toBeGreaterThanOrEqual(50);- expect(Date.now() - startedAt).toBeLessThanOrEqual(200);});describe('should scale correctly', () => {diff --git a/test/crawlers/cheerio_crawler.test.js b/test/crawlers/cheerio_crawler.test.jsindex 1e0db23a0..7c2dc96ee 100644--- a/test/crawlers/cheerio_crawler.test.js+++ b/test/crawlers/cheerio_crawler.test.js@@ -314,8 +314,7 @@ describe('CheerioCrawler', () => {});describe('should not timeout by the default httpRequest timeoutSecs', () => {- xit('when requestTimeoutSecs is grater than 30', async function () {- this.timeout(5 * 60 * 1000);+ it('when requestTimeoutSecs is greater than 30', async () => {const sources = [{ url: `http://${HOST}:${port}/timeout?a=12` },{ url: `http://${HOST}:${port}/timeout?a=23` },@@ -342,7 +341,7 @@ describe('CheerioCrawler', () => {expect(processed).toHaveLength(2);expect(failed).toHaveLength(0);- });+ }, 40000);});describe('should ensure text/html Content-Type', () => {diff --git a/test/utils.test.js b/test/utils.test.jsindex 4ca815aeb..9c507e400 100644--- a/test/utils.test.js+++ b/test/utils.test.js@@ -930,36 +930,36 @@ describe('utils.printOutdatedSdkWarning()', () => {});describe('utils.parseContentTypeFromResponse', () => {- it('should parse content type from header', () => {+ test('should parse content type from header', () => {const parsed = utils.parseContentTypeFromResponse({ url: 'http://example.com', headers: { 'content-type': 'text/html; charset=utf-8' } });- expect(parsed.type).to.be.eql('text/html');- expect(parsed.parameters.charset).to.be.eql('utf-8');+ expect(parsed.type).toBe('text/html');+ expect(parsed.parameters.charset).toBe('utf-8');});- it('should parse content type from file extension', () => {+ test('should parse content type from file extension', () => {const parsedHtml = utils.parseContentTypeFromResponse({ url: 'http://www.example.com/foo/file.html?someparam=foo', headers: {} });- expect(parsedHtml.type).to.be.eql('text/html');- expect(parsedHtml.parameters.charset).to.be.eql('utf-8');+ expect(parsedHtml.type).toBe('text/html');+ expect(parsedHtml.parameters.charset).toBe('utf-8');const parsedTxt = utils.parseContentTypeFromResponse({ url: 'http://www.example.com/foo/file.txt', headers: {} });- expect(parsedTxt.type).to.be.eql('text/plain');- expect(parsedTxt.parameters.charset).to.be.eql('utf-8');+ expect(parsedTxt.type).toBe('text/plain');+ expect(parsedTxt.parameters.charset).toBe('utf-8');});- it('should return default content type for bad content type headers', () => {+ test('should return default content type for bad content type headers', () => {const parsedWithoutCt = utils.parseContentTypeFromResponse({ url: 'http://www.example.com/foo/file', headers: {} });- expect(parsedWithoutCt.type).to.be.eql('application/octet-stream');- expect(parsedWithoutCt.parameters.charset).to.be.eql('utf-8');+ expect(parsedWithoutCt.type).toBe('application/octet-stream');+ expect(parsedWithoutCt.parameters.charset).toBe('utf-8');const parsedBadHeader = utils.parseContentTypeFromResponse({url: 'http://www.example.com/foo/file.html',headers: { 'content-type': 'text/html,text/html' },});- expect(parsedBadHeader.type).to.be.eql('text/html');- expect(parsedBadHeader.parameters.charset).to.be.eql('utf-8');+ expect(parsedBadHeader.type).toBe('text/html');+ expect(parsedBadHeader.parameters.charset).toBe('utf-8');const parsedReallyBad = utils.parseContentTypeFromResponse({ url: 'http://www.example.com/foo', headers: { 'content-type': 'crazy-stuff' } });- expect(parsedReallyBad.type).to.be.eql('application/octet-stream');- expect(parsedReallyBad.parameters.charset).to.be.eql('utf-8');+ expect(parsedReallyBad.type).toBe('application/octet-stream');+ expect(parsedReallyBad.parameters.charset).toBe('utf-8');});});