Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

Move crawlers to their own folder

Ondra Urbande447648384e
diff --git a/src/basic_crawler.js b/src/crawlers/basic_crawler.js
similarity index 98%
rename from src/basic_crawler.js
rename to src/crawlers/basic_crawler.js
index b014d476f..6ae6335b0 100644
--- a/src/basic_crawler.js
+++ b/src/crawlers/basic_crawler.js
@@ -3,11 +3,11 @@ import _ from 'underscore';
import log from 'apify-shared/log';
import { ACTOR_EVENT_NAMES } from 'apify-shared/consts';
import { checkParamPrototypeOrThrow } from 'apify-shared/utilities';
-import AutoscaledPool from './autoscaling/autoscaled_pool';
-import { RequestList } from './request_list';
-import { RequestQueue, RequestQueueLocal } from './request_queue';
-import events from './events';
-import { addTimeoutToPromise } from './utils';
+import AutoscaledPool from '../autoscaling/autoscaled_pool';
+import { RequestList } from '../request_list';
+import { RequestQueue, RequestQueueLocal } from '../request_queue';
+import events from '../events';
+import { addTimeoutToPromise } from '../utils';
/**
* Since there's no set number of seconds before the container is terminated after
diff --git a/src/cheerio_crawler.js b/src/crawlers/cheerio_crawler.js
similarity index 99%
rename from src/cheerio_crawler.js
rename to src/crawlers/cheerio_crawler.js
index e5d7555af..839802575 100644
--- a/src/cheerio_crawler.js
+++ b/src/crawlers/cheerio_crawler.js
@@ -10,9 +10,9 @@ import htmlparser from 'htmlparser2';
import log from 'apify-shared/log';
import { checkParamOrThrow } from 'apify-client/build/utils';
import BasicCrawler from './basic_crawler';
-import { addTimeoutToPromise } from './utils';
-import { getApifyProxyUrl } from './actor';
-import { BASIC_CRAWLER_TIMEOUT_MULTIPLIER } from './constants';
+import { addTimeoutToPromise } from '../utils';
+import { getApifyProxyUrl } from '../actor';
+import { BASIC_CRAWLER_TIMEOUT_MULTIPLIER } from '../constants';
const DEFAULT_OPTIONS = {
requestTimeoutSecs: 30,
diff --git a/src/puppeteer_crawler.js b/src/crawlers/puppeteer_crawler.js
similarity index 98%
rename from src/puppeteer_crawler.js
rename to src/crawlers/puppeteer_crawler.js
index 47560efa7..5b3b59b4c 100644
--- a/src/puppeteer_crawler.js
+++ b/src/crawlers/puppeteer_crawler.js
@@ -2,10 +2,10 @@ import { checkParamOrThrow } from 'apify-client/build/utils';
import log from 'apify-shared/log';
import _ from 'underscore';
import BasicCrawler from './basic_crawler';
-import PuppeteerPool from './puppeteer_pool';
-import { addTimeoutToPromise } from './utils';
-import { BASIC_CRAWLER_TIMEOUT_MULTIPLIER } from './constants';
-import { gotoExtended } from './puppeteer_utils';
+import PuppeteerPool from '../puppeteer_pool';
+import { addTimeoutToPromise } from '../utils';
+import { BASIC_CRAWLER_TIMEOUT_MULTIPLIER } from '../constants';
+import { gotoExtended } from '../puppeteer_utils';
const DEFAULT_OPTIONS = {
gotoFunction: async ({ request, page }) => gotoExtended(page, request, { timeout: 60000 }),
diff --git a/src/crawlers/run_statistics.js b/src/crawlers/run_statistics.js
new file mode 100644
index 000000000..a3eda3723
--- /dev/null
+++ b/src/crawlers/run_statistics.js
@@ -0,0 +1,3 @@
+class RunStatistics {
+
+}
diff --git a/src/index.js b/src/index.js
index 9bce6dbbb..bfd21540a 100644
--- a/src/index.js
+++ b/src/index.js
@@ -3,13 +3,13 @@ import log from 'apify-shared/log';
import { ENV_VARS } from 'apify-shared/consts';
import { main, getEnv, call, callTask, getApifyProxyUrl, metamorph, addWebhook } from './actor';
import AutoscaledPool from './autoscaling/autoscaled_pool';
-import BasicCrawler from './basic_crawler';
-import CheerioCrawler from './cheerio_crawler';
+import BasicCrawler from './crawlers/basic_crawler';
+import CheerioCrawler from './crawlers/cheerio_crawler';
import { pushData, openDataset } from './dataset';
import events, { initializeEvents, stopEvents } from './events';
import { getValue, setValue, getInput, openKeyValueStore } from './key_value_store';
import { launchPuppeteer } from './puppeteer';
-import PuppeteerCrawler from './puppeteer_crawler';
+import PuppeteerCrawler from './crawlers/puppeteer_crawler';
import PuppeteerPool from './puppeteer_pool';
import Request from './request';
import { RequestList, openRequestList } from './request_list';