Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
fixed test
Petr Patekf5d64326399b
diff --git a/test/crawlers/cheerio_crawler.test.js b/test/crawlers/cheerio_crawler.test.jsindex 97c819860..42ab1f9d3 100644--- a/test/crawlers/cheerio_crawler.test.js+++ b/test/crawlers/cheerio_crawler.test.js@@ -152,6 +152,10 @@ describe('CheerioCrawler', () => {log.setLevel(logLevel);});+ beforeEach(async () => {+ process.env.APIFY_LOCAL_STORAGE_DIR = LOCAL_STORAGE_DIR;+ });+test('should work', async () => {const sources = [{ url: 'http://example.com/?q=1' },@@ -276,22 +280,15 @@ describe('CheerioCrawler', () => {});test('after handlePageTimeoutSecs', async () => {- const sources = [- { url: 'http://example.com/?q=0' },- { url: 'http://example.com/?q=1' },- { url: 'http://example.com/?q=2' },- ];- const processed = [];const failed = [];- const requestList = new Apify.RequestList({ sources });- const handlePageFunction = async ({ request }) => {- await sleep(3000);- processed.push(request);+ const requestList = await getRequestListForMirror();+ const handlePageFunction = async () => {+ await sleep(2000);};const cheerioCrawler = new Apify.CheerioCrawler({requestList,- handlePageTimeoutSecs: 0.05,+ handlePageTimeoutSecs: 1,maxRequestRetries: 1,minConcurrency: 2,maxConcurrency: 2,@@ -302,10 +299,8 @@ describe('CheerioCrawler', () => {// Override low value to prevent seeing timeouts from BasicCrawlercheerioCrawler.basicCrawler.handleRequestTimeoutMillis = 10000;- await requestList.initialize();await cheerioCrawler.run();- expect(processed).toHaveLength(0);expect(failed).toHaveLength(3);failed.forEach((request) => {