Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
Development of .travis.yml, fixed unit tests
Jan Curna55c46bad719
diff --git a/.travis.yml b/.travis.ymlindex 69e268b7b..ca9b57c90 100644--- a/.travis.yml+++ b/.travis.yml@@ -6,8 +6,6 @@ before_install:- sudo apt-get install -y google-chrome-beta- sudo curl -sS http://chromedriver.storage.googleapis.com/2.29/chromedriver_linux64.zip > /tmp/chromedriver.zip- sudo unzip /tmp/chromedriver.zip -d /tmp- - sudo ls -l /tmp- - sudo ls -l /- sudo rm /tmp/chromedriver.zip- sudo mv /tmp/chromedriver /binlanguage: node_jsdiff --git a/test/browser.js b/test/browser.jsindex bf3700a24..f5e4d1948 100644--- a/test/browser.js+++ b/test/browser.js@@ -11,6 +11,7 @@ describe('getDefaultBrowseOptions()', () => {expect(opts1).to.eql({browser: 'chrome',headless: true,+ proxyUrl: null,});delete process.env.APIFY_HEADLESS;@@ -18,6 +19,7 @@ describe('getDefaultBrowseOptions()', () => {expect(opts2).to.eql({browser: 'chrome',headless: false,+ proxyUrl: null,});});});@@ -37,7 +39,7 @@ describe('Apifier.browse()', () => {expect(url).to.eql('about:blank');return browser.close();});- }).timeout(10000);+ }).timeout(60 * 1000);it('opens https://www.example.com in headless mode', () => {delete process.env.APIFY_HEADLESS;@@ -52,7 +54,7 @@ describe('Apifier.browse()', () => {expect(url).to.eql('https://www.example.com/');return browser.close();});- }).timeout(10000);+ }).timeout(60 * 1000);});@@ -66,7 +68,7 @@ describe('launchChrome()', () => {});});- it('reports head-less version', () => {+ it('reports headless browser version', () => {return Promise.resolve().then(() => {return CDP.Version();})@@ -74,7 +76,7 @@ describe('launchChrome()', () => {expect(version.Browser).to.contain('HeadlessChrome');// console.dir(version);});- });+ }).timeout(60 * 1000);it('opens http://www.example.com', () => {function onPageLoad(Runtime) {@@ -113,7 +115,7 @@ describe('launchChrome()', () => {}).on('error', reject);});- }).timeout(10000);+ }).timeout(60 * 1000);// it('test github requests', () => {// return new Promise((resolve, reject) => {