Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
Development of .travis.yml, udpated unit tests
Jan Curnb7456f3a54f3
diff --git a/.travis.yml b/.travis.ymlindex 2ae6f7447..4d5408075 100644--- a/.travis.yml+++ b/.travis.yml@@ -6,13 +6,14 @@ before_install:- sudo curl -sS http://chromedriver.storage.googleapis.com/2.29/chromedriver_linux64.zip > /tmp/chromedriver.zip- sudo unzip /tmp/chromedriver.zip -d /tmp- sudo rm /tmp/chromedriver.zip- - sudo mv /tmp/chromedriver /bin+ - sudo mv /tmp/chromedriver /usr/bin/- # Travis' VM has preinstalled google-chrome stable, which has no headless mode!- sudo mv -f /usr/bin/google-chrome-beta /usr/bin/google-chrome | tee- google-chrome --version- which google-chrome | tee- which google-chrome-beta | tee- google-chrome --headless --disable-gpu --screenshot https://www.chromestatus.com/+ - chromedriver --version | teelanguage: node_jsnode_js:- "8"diff --git a/test/browser.js b/test/browser.jsindex 34d8171ec..0ff796862 100644--- a/test/browser.js+++ b/test/browser.js@@ -25,7 +25,10 @@ describe('getDefaultBrowseOptions()', () => {});-describe('Apifier.browse()', () => {+describe('Apifier.browse()', function () {+ // Need a large timeout to run unit tests on Travis CI+ this.timeout(300 * 1000);+it('opens about:blank with no args', () => {process.env.APIFY_HEADLESS = '1';let browser;@@ -39,7 +42,7 @@ describe('Apifier.browse()', () => {expect(url).to.eql('about:blank');return browser.close();});- }).timeout(300 * 1000);+ });it('opens https://www.example.com in headless mode', () => {delete process.env.APIFY_HEADLESS;@@ -54,11 +57,14 @@ describe('Apifier.browse()', () => {expect(url).to.eql('https://www.example.com/');return browser.close();});- }).timeout(300 * 1000);+ });});-describe('launchChrome()', () => {+describe('launchChrome()', function () {+ // Need a large timeout to run unit tests on Travis CI+ this.timeout(300 * 1000);+let launcher;before(() => {@@ -76,7 +82,7 @@ describe('launchChrome()', () => {expect(version.Browser).to.contain('HeadlessChrome');// console.dir(version);});- }).timeout(300 * 1000);+ });it('opens http://www.example.com', () => {function onPageLoad(Runtime) {@@ -115,7 +121,7 @@ describe('launchChrome()', () => {}).on('error', reject);});- }).timeout(300 * 1000);+ });// it('test github requests', () => {// return new Promise((resolve, reject) => {@@ -149,6 +155,6 @@ describe('launchChrome()', () => {// });after(() => {- return launcher.kill();+ if (launcher) return launcher.kill();});});