Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
chore: use apify.com as the default start url in project templates
Martin Adámek5e8428d1c45c
diff --git a/packages/templates/templates/cheerio-js/src/main.js b/packages/templates/templates/cheerio-js/src/main.jsindex 448dc74fa..189a945bd 100644--- a/packages/templates/templates/cheerio-js/src/main.js+++ b/packages/templates/templates/cheerio-js/src/main.js@@ -2,7 +2,7 @@import { CheerioCrawler, KeyValueStore, log, ProxyConfiguration } from 'crawlee';import { router } from './routes.js';-const { startUrls = [], debug } = await KeyValueStore.getInput() ?? {};+const { startUrls = ['https://apify.com'], debug } = await KeyValueStore.getInput() ?? {};if (debug) {log.setLevel(log.LEVELS.DEBUG);diff --git a/packages/templates/templates/cheerio-ts/src/main.ts b/packages/templates/templates/cheerio-ts/src/main.tsindex aa5d8c0d0..302c17cbc 100644--- a/packages/templates/templates/cheerio-ts/src/main.ts+++ b/packages/templates/templates/cheerio-ts/src/main.ts@@ -7,7 +7,7 @@ interface InputSchema {debug?: boolean;}-const { startUrls = [], debug } = await KeyValueStore.getInput<InputSchema>() ?? {};+const { startUrls = ['https://apify.com'], debug } = await KeyValueStore.getInput<InputSchema>() ?? {};log.setLevel(log.LEVELS.DEBUG);if (debug) {diff --git a/packages/templates/templates/playwright-js/src/main.js b/packages/templates/templates/playwright-js/src/main.jsindex d202ffbd5..0bd5e149c 100644--- a/packages/templates/templates/playwright-js/src/main.js+++ b/packages/templates/templates/playwright-js/src/main.js@@ -2,7 +2,7 @@import { KeyValueStore, PlaywrightCrawler, ProxyConfiguration, log } from 'crawlee';import { router } from './routes.js';-const { startUrls = [], debug } = await KeyValueStore.getInput() ?? {};+const { startUrls = ['https://apify.com'], debug } = await KeyValueStore.getInput() ?? {};if (debug) {log.setLevel(log.LEVELS.DEBUG);diff --git a/packages/templates/templates/playwright-ts/src/main.ts b/packages/templates/templates/playwright-ts/src/main.tsindex d472de160..d272162d8 100644--- a/packages/templates/templates/playwright-ts/src/main.ts+++ b/packages/templates/templates/playwright-ts/src/main.ts@@ -7,7 +7,7 @@ interface InputSchema {debug?: boolean;}-const { startUrls = [], debug } = await KeyValueStore.getInput<InputSchema>() ?? {};+const { startUrls = ['https://apify.com'], debug } = await KeyValueStore.getInput<InputSchema>() ?? {};if (debug) {log.setLevel(log.LEVELS.DEBUG);diff --git a/packages/templates/templates/puppeteer-js/src/main.js b/packages/templates/templates/puppeteer-js/src/main.jsindex bd8dd7d44..41180604a 100644--- a/packages/templates/templates/puppeteer-js/src/main.js+++ b/packages/templates/templates/puppeteer-js/src/main.js@@ -2,7 +2,7 @@import { KeyValueStore, PuppeteerCrawler, ProxyConfiguration, log } from 'crawlee';import { router } from './routes.js';-const { startUrls = [], debug } = await KeyValueStore.getInput() ?? {};+const { startUrls = ['https://apify.com'], debug } = await KeyValueStore.getInput() ?? {};if (debug) {log.setLevel(log.LEVELS.DEBUG);diff --git a/packages/templates/templates/puppeteer-ts/src/main.ts b/packages/templates/templates/puppeteer-ts/src/main.tsindex 5625479d5..d230e3bac 100644--- a/packages/templates/templates/puppeteer-ts/src/main.ts+++ b/packages/templates/templates/puppeteer-ts/src/main.ts@@ -7,7 +7,7 @@ interface InputSchema {debug?: boolean;}-const { startUrls = [], debug } = await KeyValueStore.getInput<InputSchema>() ?? {};+const { startUrls = ['https://apify.com'], debug } = await KeyValueStore.getInput<InputSchema>() ?? {};if (debug) {log.setLevel(log.LEVELS.DEBUG);