Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
removed the local testing html file since it is not working in travis.
Petr Patek669ee2aca359
diff --git a/test/stealth/stealth.js b/test/stealth/stealth.jsindex 15f0d35d9..f1242d32e 100644--- a/test/stealth/stealth.js+++ b/test/stealth/stealth.js@@ -22,7 +22,7 @@ describe('Stealth - testing headless chrome hiding tricks', () => {});const page = await browser.newPage();- await page.goto('file://test_website.html');+ await page.goto('http://example.com');const { plugins, mimeTypes } = await getFingerPrint(page);@@ -42,7 +42,7 @@ describe('Stealth - testing headless chrome hiding tricks', () => {});const page = await browser.newPage();- await page.goto('file://test_website.html');+ await page.goto('http://example.com');const { webDriver } = await getFingerPrint(page);expect(webDriver).to.be.eql(false);@@ -60,7 +60,7 @@ describe('Stealth - testing headless chrome hiding tricks', () => {});const page = await browser.newPage();- await page.goto('file://test_website.html');+ await page.goto('http://example.com');const { permissions } = await getFingerPrint(page);expect(permissions.state).to.be.eql('denied');@@ -78,7 +78,7 @@ describe('Stealth - testing headless chrome hiding tricks', () => {});const page = await browser.newPage();- await page.goto('file://test_website.html');+ await page.goto('http://example.com');const { languages } = await getFingerPrint(page);expect(languages).to.be.an('array');@@ -98,7 +98,7 @@ describe('Stealth - testing headless chrome hiding tricks', () => {});const page = await browser.newPage();- await page.goto('file://test_website.html');+ await page.goto('http://example.com');const { videoCard } = await getFingerPrint(page);expect(videoCard[0]).to.be.eql('Intel Inc.');@@ -117,7 +117,7 @@ describe('Stealth - testing headless chrome hiding tricks', () => {});const page = await browser.newPage();- await page.goto('file://test_website.html');+ await page.goto('http://example.com');const { screen } = await getFingerPrint(page);expect(screen.wOuterHeight).to.be.eql(screen.wInnerHeight + 85);@@ -135,7 +135,7 @@ describe('Stealth - testing headless chrome hiding tricks', () => {});const page = await browser.newPage();- await page.goto('file://test_website.html');+ await page.goto('http://example.com');const returnValue = await page.evaluate(() => console.debug('TEST'));expect(returnValue).to.be.eql(null);@@ -152,7 +152,7 @@ describe('Stealth - testing headless chrome hiding tricks', () => {});const page = await browser.newPage();- await page.goto('file://test_website.html');+ await page.goto('http://example.com');const { hasChrome } = await getFingerPrint(page);const chrome = await page.evaluate(() => window.chrome); //eslint-disable-lineexpect(chrome).to.be.an('object');@@ -172,7 +172,7 @@ describe('Stealth - testing headless chrome hiding tricks', () => {});const page = await browser.newPage();- await page.goto('file://test_website.html');+ await page.goto('http://example.com');const { iframeChrome } = await getFingerPrint(page);expect(iframeChrome).to.be.eql('object');@@ -190,7 +190,7 @@ describe('Stealth - testing headless chrome hiding tricks', () => {});const page = await browser.newPage();- await page.goto('file://test_website.html');+ await page.goto('http://example.com');const { deviceMemory } = await getFingerPrint(page);expect(deviceMemory).not.to.be.eql(0);@@ -217,7 +217,7 @@ describe('Stealth - testing headless chrome hiding tricks', () => {});const page = await browser.newPage();- await page.goto('file://test_website.html');+ await page.goto('http://example.com');const fingerPrint = await getFingerPrint(page);const testedFingerprint = scanner.analyseFingerprint(fingerPrint);const failedChecks = Object.values(testedFingerprint).filter(val => val.consistent < 3);diff --git a/test/stealth/test_website.html b/test/stealth/test_website.htmldeleted file mode 100644index 23f895e68..000000000--- a/test/stealth/test_website.html+++ /dev/null@@ -1,15 +0,0 @@-<!doctype html>-<html lang="">--<head>- <meta charset="utf-8">- <title>Test website</title>- <meta name="description" content="">- <meta name="viewport" content="width=device-width, initial-scale=1">-</head>--<body>-<p>Hello world! This is HTML5 Boilerplate.</p>-</body>--</html>