Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
Move crawlers to their own folder
Ondra Urbande447648384e
diff --git a/src/basic_crawler.js b/src/crawlers/basic_crawler.jssimilarity index 98%rename from src/basic_crawler.jsrename to src/crawlers/basic_crawler.jsindex b014d476f..6ae6335b0 100644--- a/src/basic_crawler.js+++ b/src/crawlers/basic_crawler.js@@ -3,11 +3,11 @@ import _ from 'underscore';import log from 'apify-shared/log';import { ACTOR_EVENT_NAMES } from 'apify-shared/consts';import { checkParamPrototypeOrThrow } from 'apify-shared/utilities';-import AutoscaledPool from './autoscaling/autoscaled_pool';-import { RequestList } from './request_list';-import { RequestQueue, RequestQueueLocal } from './request_queue';-import events from './events';-import { addTimeoutToPromise } from './utils';+import AutoscaledPool from '../autoscaling/autoscaled_pool';+import { RequestList } from '../request_list';+import { RequestQueue, RequestQueueLocal } from '../request_queue';+import events from '../events';+import { addTimeoutToPromise } from '../utils';/*** Since there's no set number of seconds before the container is terminated afterdiff --git a/src/cheerio_crawler.js b/src/crawlers/cheerio_crawler.jssimilarity index 99%rename from src/cheerio_crawler.jsrename to src/crawlers/cheerio_crawler.jsindex e5d7555af..839802575 100644--- a/src/cheerio_crawler.js+++ b/src/crawlers/cheerio_crawler.js@@ -10,9 +10,9 @@ import htmlparser from 'htmlparser2';import log from 'apify-shared/log';import { checkParamOrThrow } from 'apify-client/build/utils';import BasicCrawler from './basic_crawler';-import { addTimeoutToPromise } from './utils';-import { getApifyProxyUrl } from './actor';-import { BASIC_CRAWLER_TIMEOUT_MULTIPLIER } from './constants';+import { addTimeoutToPromise } from '../utils';+import { getApifyProxyUrl } from '../actor';+import { BASIC_CRAWLER_TIMEOUT_MULTIPLIER } from '../constants';const DEFAULT_OPTIONS = {requestTimeoutSecs: 30,diff --git a/src/puppeteer_crawler.js b/src/crawlers/puppeteer_crawler.jssimilarity index 98%rename from src/puppeteer_crawler.jsrename to src/crawlers/puppeteer_crawler.jsindex 47560efa7..5b3b59b4c 100644--- a/src/puppeteer_crawler.js+++ b/src/crawlers/puppeteer_crawler.js@@ -2,10 +2,10 @@ import { checkParamOrThrow } from 'apify-client/build/utils';import log from 'apify-shared/log';import _ from 'underscore';import BasicCrawler from './basic_crawler';-import PuppeteerPool from './puppeteer_pool';-import { addTimeoutToPromise } from './utils';-import { BASIC_CRAWLER_TIMEOUT_MULTIPLIER } from './constants';-import { gotoExtended } from './puppeteer_utils';+import PuppeteerPool from '../puppeteer_pool';+import { addTimeoutToPromise } from '../utils';+import { BASIC_CRAWLER_TIMEOUT_MULTIPLIER } from '../constants';+import { gotoExtended } from '../puppeteer_utils';const DEFAULT_OPTIONS = {gotoFunction: async ({ request, page }) => gotoExtended(page, request, { timeout: 60000 }),diff --git a/src/crawlers/run_statistics.js b/src/crawlers/run_statistics.jsnew file mode 100644index 000000000..a3eda3723--- /dev/null+++ b/src/crawlers/run_statistics.js@@ -0,0 +1,3 @@+class RunStatistics {++}diff --git a/src/index.js b/src/index.jsindex 9bce6dbbb..bfd21540a 100644--- a/src/index.js+++ b/src/index.js@@ -3,13 +3,13 @@ import log from 'apify-shared/log';import { ENV_VARS } from 'apify-shared/consts';import { main, getEnv, call, callTask, getApifyProxyUrl, metamorph, addWebhook } from './actor';import AutoscaledPool from './autoscaling/autoscaled_pool';-import BasicCrawler from './basic_crawler';-import CheerioCrawler from './cheerio_crawler';+import BasicCrawler from './crawlers/basic_crawler';+import CheerioCrawler from './crawlers/cheerio_crawler';import { pushData, openDataset } from './dataset';import events, { initializeEvents, stopEvents } from './events';import { getValue, setValue, getInput, openKeyValueStore } from './key_value_store';import { launchPuppeteer } from './puppeteer';-import PuppeteerCrawler from './puppeteer_crawler';+import PuppeteerCrawler from './crawlers/puppeteer_crawler';import PuppeteerPool from './puppeteer_pool';import Request from './request';import { RequestList, openRequestList } from './request_list';