Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
feat(templates): use ImpitHttpClient and install optional deps in Docker (#3603)
Jindřich Bär113dca699960
diff --git a/packages/templates/templates/camoufox-ts/Dockerfile b/packages/templates/templates/camoufox-ts/Dockerfileindex 637ca8dea..f33195aed 100644--- a/packages/templates/templates/camoufox-ts/Dockerfile+++ b/packages/templates/templates/camoufox-ts/Dockerfile@@ -28,7 +28,7 @@ COPY --from=builder --chown=myuser /home/myuser/dist ./dist# to speed up the build using Docker layer cache.COPY --chown=myuser package*.json ./-# Install NPM packages, skip optional and development dependencies to+# Install NPM packages, skip development dependencies to# keep the image small. Avoid logging too much and print the dependency# tree for debuggingRUN npm --quiet set progress=false \diff --git a/packages/templates/templates/camoufox-ts/package.json b/packages/templates/templates/camoufox-ts/package.jsonindex e9d247a66..9e4bcbe30 100644--- a/packages/templates/templates/camoufox-ts/package.json+++ b/packages/templates/templates/camoufox-ts/package.json@@ -4,6 +4,7 @@"type": "module","description": "This is an example of a Crawlee project.","dependencies": {+ "@crawlee/impit-client": "^3.0.0","camoufox-js": "^0.9.0","crawlee": "^3.0.0","playwright": "1.58.2"diff --git a/packages/templates/templates/camoufox-ts/src/main.ts b/packages/templates/templates/camoufox-ts/src/main.tsindex a997a2841..8d29644ff 100644--- a/packages/templates/templates/camoufox-ts/src/main.ts+++ b/packages/templates/templates/camoufox-ts/src/main.ts@@ -1,4 +1,5 @@// For more information, see https://crawlee.dev/+import { Browser, ImpitHttpClient } from '@crawlee/impit-client';import { launchOptions } from 'camoufox-js';import { PlaywrightCrawler, ProxyConfiguration } from 'crawlee';import { firefox } from 'playwright';@@ -9,6 +10,7 @@ const startUrls = ['https://crawlee.dev'];const crawler = new PlaywrightCrawler({// proxyConfiguration: new ProxyConfiguration({ proxyUrls: ['...'] }),+ httpClient: new ImpitHttpClient({ browser: Browser.Firefox }),requestHandler: router,// Comment this option to scrape the full website.maxRequestsPerCrawl: 20,diff --git a/packages/templates/templates/cheerio-js/Dockerfile b/packages/templates/templates/cheerio-js/Dockerfileindex 4c8d11fc3..86de12656 100644--- a/packages/templates/templates/cheerio-js/Dockerfile+++ b/packages/templates/templates/cheerio-js/Dockerfile@@ -7,11 +7,11 @@ FROM apify/actor-node:20# to speed up the build using Docker layer cache.COPY package*.json ./-# Install NPM packages, skip optional and development dependencies to+# Install NPM packages, skip development dependencies to# keep the image small. Avoid logging too much and print the dependency# tree for debuggingRUN npm --quiet set progress=false \- && npm install --omit=dev --omit=optional \+ && npm install --omit=dev \&& echo "Installed NPM packages:" \&& (npm list --omit=dev --all || true) \&& echo "Node.js version:" \diff --git a/packages/templates/templates/cheerio-js/package.json b/packages/templates/templates/cheerio-js/package.jsonindex 51fb96156..0a751aa42 100644--- a/packages/templates/templates/cheerio-js/package.json+++ b/packages/templates/templates/cheerio-js/package.json@@ -4,6 +4,7 @@"type": "module","description": "This is an example of a Crawlee project.","dependencies": {+ "@crawlee/impit-client": "^3.0.0","crawlee": "^3.0.0"},"scripts": {diff --git a/packages/templates/templates/cheerio-js/src/main.js b/packages/templates/templates/cheerio-js/src/main.jsindex f8b62e352..51130b8b8 100644--- a/packages/templates/templates/cheerio-js/src/main.js+++ b/packages/templates/templates/cheerio-js/src/main.js@@ -1,4 +1,5 @@// For more information, see https://crawlee.dev/+import { Browser, ImpitHttpClient } from '@crawlee/impit-client';import { CheerioCrawler, ProxyConfiguration } from 'crawlee';import { router } from './routes.js';@@ -6,6 +7,7 @@ const startUrls = ['https://crawlee.dev'];const crawler = new CheerioCrawler({// proxyConfiguration: new ProxyConfiguration({ proxyUrls: ['...'] }),+ httpClient: new ImpitHttpClient({ browser: Browser.Chrome }),requestHandler: router,// Comment this option to scrape the full website.maxRequestsPerCrawl: 20,diff --git a/packages/templates/templates/cheerio-ts/Dockerfile b/packages/templates/templates/cheerio-ts/Dockerfileindex 995a3d815..befbf5e84 100644--- a/packages/templates/templates/cheerio-ts/Dockerfile+++ b/packages/templates/templates/cheerio-ts/Dockerfile@@ -28,11 +28,11 @@ COPY --from=builder /usr/src/app/dist ./dist# to speed up the build using Docker layer cache.COPY package*.json ./-# Install NPM packages, skip optional and development dependencies to+# Install NPM packages, skip development dependencies to# keep the image small. Avoid logging too much and print the dependency# tree for debuggingRUN npm --quiet set progress=false \- && npm install --omit=dev --omit=optional \+ && npm install --omit=dev \&& echo "Installed NPM packages:" \&& (npm list --omit=dev --all || true) \&& echo "Node.js version:" \diff --git a/packages/templates/templates/cheerio-ts/package.json b/packages/templates/templates/cheerio-ts/package.jsonindex 234babb1e..64284b238 100644--- a/packages/templates/templates/cheerio-ts/package.json+++ b/packages/templates/templates/cheerio-ts/package.json@@ -4,6 +4,7 @@"type": "module","description": "This is an example of a Crawlee project.","dependencies": {+ "@crawlee/impit-client": "^3.0.0","crawlee": "^3.0.0"},"devDependencies": {diff --git a/packages/templates/templates/cheerio-ts/src/main.ts b/packages/templates/templates/cheerio-ts/src/main.tsindex fb586fb6c..c777aa6c3 100644--- a/packages/templates/templates/cheerio-ts/src/main.ts+++ b/packages/templates/templates/cheerio-ts/src/main.ts@@ -1,4 +1,5 @@// For more information, see https://crawlee.dev/+import { Browser, ImpitHttpClient } from '@crawlee/impit-client';import { CheerioCrawler, ProxyConfiguration } from 'crawlee';import { router } from './routes.js';@@ -7,6 +8,7 @@ const startUrls = ['https://crawlee.dev'];const crawler = new CheerioCrawler({// proxyConfiguration: new ProxyConfiguration({ proxyUrls: ['...'] }),+ httpClient: new ImpitHttpClient({ browser: Browser.Chrome }),requestHandler: router,// Comment this option to scrape the full website.maxRequestsPerCrawl: 20,diff --git a/packages/templates/templates/getting-started-js/Dockerfile b/packages/templates/templates/getting-started-js/Dockerfileindex 09574c8d2..5f0911aaa 100644--- a/packages/templates/templates/getting-started-js/Dockerfile+++ b/packages/templates/templates/getting-started-js/Dockerfile@@ -7,11 +7,11 @@ FROM apify/actor-node-playwright-chrome:24-1.58.2# to speed up the build using Docker layer cache.COPY --chown=myuser package*.json ./-# Install NPM packages, skip optional and development dependencies to+# Install NPM packages, skip development dependencies to# keep the image small. Avoid logging too much and print the dependency# tree for debuggingRUN npm --quiet set progress=false \- && npm install --omit=dev --omit=optional \+ && npm install --omit=dev \&& echo "Installed NPM packages:" \&& (npm list --omit=dev --all || true) \&& echo "Node.js version:" \diff --git a/packages/templates/templates/getting-started-js/package.json b/packages/templates/templates/getting-started-js/package.jsonindex eaa47083b..fcec25879 100644--- a/packages/templates/templates/getting-started-js/package.json+++ b/packages/templates/templates/getting-started-js/package.json@@ -4,6 +4,7 @@"type": "module","description": "This is an example of a Crawlee project.","dependencies": {+ "@crawlee/impit-client": "^3.0.0","crawlee": "^3.0.0","playwright": "1.58.2"},diff --git a/packages/templates/templates/getting-started-js/src/main.js b/packages/templates/templates/getting-started-js/src/main.jsindex d8e39bb84..0d876d0e0 100644--- a/packages/templates/templates/getting-started-js/src/main.js+++ b/packages/templates/templates/getting-started-js/src/main.js@@ -1,9 +1,11 @@// For more information, see https://crawlee.dev/+import { Browser, ImpitHttpClient } from '@crawlee/impit-client';import { PlaywrightCrawler } from 'crawlee';// PlaywrightCrawler crawls the web using a headless// browser controlled by the Playwright library.const crawler = new PlaywrightCrawler({+ httpClient: new ImpitHttpClient({ browser: Browser.Chrome }),// Use the requestHandler to process each of the crawled pages.async requestHandler({ request, page, enqueueLinks, log, pushData }) {const title = await page.title();diff --git a/packages/templates/templates/getting-started-ts/Dockerfile b/packages/templates/templates/getting-started-ts/Dockerfileindex f728f7ce5..92d00d920 100644--- a/packages/templates/templates/getting-started-ts/Dockerfile+++ b/packages/templates/templates/getting-started-ts/Dockerfile@@ -28,11 +28,11 @@ COPY --from=builder --chown=myuser /home/myuser/dist ./dist# to speed up the build using Docker layer cache.COPY --chown=myuser package*.json ./-# Install NPM packages, skip optional and development dependencies to+# Install NPM packages, skip development dependencies to# keep the image small. Avoid logging too much and print the dependency# tree for debuggingRUN npm --quiet set progress=false \- && npm install --omit=dev --omit=optional \+ && npm install --omit=dev \&& echo "Installed NPM packages:" \&& (npm list --omit=dev --all || true) \&& echo "Node.js version:" \diff --git a/packages/templates/templates/getting-started-ts/package.json b/packages/templates/templates/getting-started-ts/package.jsonindex 5b4f49218..c771d9724 100644--- a/packages/templates/templates/getting-started-ts/package.json+++ b/packages/templates/templates/getting-started-ts/package.json@@ -4,6 +4,7 @@"type": "module","description": "This is an example of a Crawlee project.","dependencies": {+ "@crawlee/impit-client": "^3.0.0","crawlee": "^3.0.0","playwright": "1.58.2"},diff --git a/packages/templates/templates/getting-started-ts/src/main.ts b/packages/templates/templates/getting-started-ts/src/main.tsindex d8e39bb84..0d876d0e0 100644--- a/packages/templates/templates/getting-started-ts/src/main.ts+++ b/packages/templates/templates/getting-started-ts/src/main.ts@@ -1,9 +1,11 @@// For more information, see https://crawlee.dev/+import { Browser, ImpitHttpClient } from '@crawlee/impit-client';import { PlaywrightCrawler } from 'crawlee';// PlaywrightCrawler crawls the web using a headless// browser controlled by the Playwright library.const crawler = new PlaywrightCrawler({+ httpClient: new ImpitHttpClient({ browser: Browser.Chrome }),// Use the requestHandler to process each of the crawled pages.async requestHandler({ request, page, enqueueLinks, log, pushData }) {const title = await page.title();diff --git a/packages/templates/templates/playwright-js/Dockerfile b/packages/templates/templates/playwright-js/Dockerfileindex 8be44c49f..0fbac3b9e 100644--- a/packages/templates/templates/playwright-js/Dockerfile+++ b/packages/templates/templates/playwright-js/Dockerfile@@ -7,11 +7,11 @@ FROM apify/actor-node-playwright-chrome:24-1.58.2# to speed up the build using Docker layer cache.COPY --chown=myuser package*.json ./-# Install NPM packages, skip optional and development dependencies to+# Install NPM packages, skip development dependencies to# keep the image small. Avoid logging too much and print the dependency# tree for debuggingRUN npm --quiet set progress=false \- && npm install --omit=dev --omit=optional \+ && npm install --omit=dev \&& echo "Installed NPM packages:" \&& (npm list --omit=dev --all || true) \&& echo "Node.js version:" \diff --git a/packages/templates/templates/playwright-js/package.json b/packages/templates/templates/playwright-js/package.jsonindex 8b64a45c1..ef87404bc 100644--- a/packages/templates/templates/playwright-js/package.json+++ b/packages/templates/templates/playwright-js/package.json@@ -4,6 +4,7 @@"type": "module","description": "This is an example of a Crawlee project.","dependencies": {+ "@crawlee/impit-client": "^3.0.0","crawlee": "^3.0.0","playwright": "1.58.2"},diff --git a/packages/templates/templates/playwright-js/src/main.js b/packages/templates/templates/playwright-js/src/main.jsindex ccda8b9ea..c6f16b425 100644--- a/packages/templates/templates/playwright-js/src/main.js+++ b/packages/templates/templates/playwright-js/src/main.js@@ -1,4 +1,5 @@// For more information, see https://crawlee.dev/+import { Browser, ImpitHttpClient } from '@crawlee/impit-client';import { PlaywrightCrawler, ProxyConfiguration } from 'crawlee';import { router } from './routes.js';@@ -6,6 +7,7 @@ const startUrls = ['https://crawlee.dev'];const crawler = new PlaywrightCrawler({// proxyConfiguration: new ProxyConfiguration({ proxyUrls: ['...'] }),+ httpClient: new ImpitHttpClient({ browser: Browser.Chrome }),requestHandler: router,// Comment this option to scrape the full website.maxRequestsPerCrawl: 20,diff --git a/packages/templates/templates/playwright-ts/Dockerfile b/packages/templates/templates/playwright-ts/Dockerfileindex f728f7ce5..92d00d920 100644--- a/packages/templates/templates/playwright-ts/Dockerfile+++ b/packages/templates/templates/playwright-ts/Dockerfile@@ -28,11 +28,11 @@ COPY --from=builder --chown=myuser /home/myuser/dist ./dist# to speed up the build using Docker layer cache.COPY --chown=myuser package*.json ./-# Install NPM packages, skip optional and development dependencies to+# Install NPM packages, skip development dependencies to# keep the image small. Avoid logging too much and print the dependency# tree for debuggingRUN npm --quiet set progress=false \- && npm install --omit=dev --omit=optional \+ && npm install --omit=dev \&& echo "Installed NPM packages:" \&& (npm list --omit=dev --all || true) \&& echo "Node.js version:" \diff --git a/packages/templates/templates/playwright-ts/package.json b/packages/templates/templates/playwright-ts/package.jsonindex 4edc98077..43c03c95a 100644--- a/packages/templates/templates/playwright-ts/package.json+++ b/packages/templates/templates/playwright-ts/package.json@@ -4,6 +4,7 @@"type": "module","description": "This is an example of a Crawlee project.","dependencies": {+ "@crawlee/impit-client": "^3.0.0","crawlee": "^3.0.0","playwright": "1.58.2"},diff --git a/packages/templates/templates/playwright-ts/src/main.ts b/packages/templates/templates/playwright-ts/src/main.tsindex 53ecb8a76..4502b3fdf 100644--- a/packages/templates/templates/playwright-ts/src/main.ts+++ b/packages/templates/templates/playwright-ts/src/main.ts@@ -1,4 +1,5 @@// For more information, see https://crawlee.dev/+import { Browser, ImpitHttpClient } from '@crawlee/impit-client';import { PlaywrightCrawler, ProxyConfiguration } from 'crawlee';import { router } from './routes.js';@@ -7,6 +8,7 @@ const startUrls = ['https://crawlee.dev'];const crawler = new PlaywrightCrawler({// proxyConfiguration: new ProxyConfiguration({ proxyUrls: ['...'] }),+ httpClient: new ImpitHttpClient({ browser: Browser.Chrome }),requestHandler: router,// Comment this option to scrape the full website.maxRequestsPerCrawl: 20,diff --git a/packages/templates/templates/puppeteer-js/Dockerfile b/packages/templates/templates/puppeteer-js/Dockerfileindex fa86c423f..9e150f167 100644--- a/packages/templates/templates/puppeteer-js/Dockerfile+++ b/packages/templates/templates/puppeteer-js/Dockerfile@@ -7,11 +7,11 @@ FROM apify/actor-node-puppeteer-chrome:20# to speed up the build using Docker layer cache.COPY --chown=myuser package*.json ./-# Install NPM packages, skip optional and development dependencies to+# Install NPM packages, skip development dependencies to# keep the image small. Avoid logging too much and print the dependency# tree for debuggingRUN npm --quiet set progress=false \- && npm install --omit=dev --omit=optional \+ && npm install --omit=dev \&& echo "Installed NPM packages:" \&& (npm list --omit=dev --all || true) \&& echo "Node.js version:" \diff --git a/packages/templates/templates/puppeteer-js/package.json b/packages/templates/templates/puppeteer-js/package.jsonindex e34913979..3f39f6ddd 100644--- a/packages/templates/templates/puppeteer-js/package.json+++ b/packages/templates/templates/puppeteer-js/package.json@@ -4,6 +4,7 @@"type": "module","description": "This is an example of a Crawlee project.","dependencies": {+ "@crawlee/impit-client": "^3.0.0","crawlee": "^3.0.0","puppeteer": "*"},diff --git a/packages/templates/templates/puppeteer-js/src/main.js b/packages/templates/templates/puppeteer-js/src/main.jsindex d7d2882ce..28450cf1c 100644--- a/packages/templates/templates/puppeteer-js/src/main.js+++ b/packages/templates/templates/puppeteer-js/src/main.js@@ -1,4 +1,5 @@// For more information, see https://crawlee.dev/+import { Browser, ImpitHttpClient } from '@crawlee/impit-client';import { PuppeteerCrawler, ProxyConfiguration } from 'crawlee';import { router } from './routes.js';@@ -6,6 +7,7 @@ const startUrls = ['https://crawlee.dev'];const crawler = new PuppeteerCrawler({// proxyConfiguration: new ProxyConfiguration({ proxyUrls: ['...'] }),+ httpClient: new ImpitHttpClient({ browser: Browser.Chrome }),requestHandler: router,// Comment this option to scrape the full website.maxRequestsPerCrawl: 20,diff --git a/packages/templates/templates/puppeteer-ts/Dockerfile b/packages/templates/templates/puppeteer-ts/Dockerfileindex 292b6f4a1..cd384312e 100644--- a/packages/templates/templates/puppeteer-ts/Dockerfile+++ b/packages/templates/templates/puppeteer-ts/Dockerfile@@ -28,11 +28,11 @@ COPY --from=builder --chown=myuser /home/myuser/dist ./dist# to speed up the build using Docker layer cache.COPY --chown=myuser package*.json ./-# Install NPM packages, skip optional and development dependencies to+# Install NPM packages, skip development dependencies to# keep the image small. Avoid logging too much and print the dependency# tree for debuggingRUN npm --quiet set progress=false \- && npm install --omit=dev --omit=optional \+ && npm install --omit=dev \&& echo "Installed NPM packages:" \&& (npm list --omit=dev --all || true) \&& echo "Node.js version:" \diff --git a/packages/templates/templates/puppeteer-ts/package.json b/packages/templates/templates/puppeteer-ts/package.jsonindex eb67e0c44..7e3049e2a 100644--- a/packages/templates/templates/puppeteer-ts/package.json+++ b/packages/templates/templates/puppeteer-ts/package.json@@ -4,6 +4,7 @@"type": "module","description": "This is an example of a Crawlee project.","dependencies": {+ "@crawlee/impit-client": "^3.0.0","crawlee": "^3.0.0","puppeteer": "*"},diff --git a/packages/templates/templates/puppeteer-ts/src/main.ts b/packages/templates/templates/puppeteer-ts/src/main.tsindex 04b43ec21..0dfd6bee7 100644--- a/packages/templates/templates/puppeteer-ts/src/main.ts+++ b/packages/templates/templates/puppeteer-ts/src/main.ts@@ -1,4 +1,5 @@// For more information, see https://crawlee.dev/+import { Browser, ImpitHttpClient } from '@crawlee/impit-client';import { log, PuppeteerCrawler } from 'crawlee';import { router } from './routes.js';@@ -7,6 +8,7 @@ const startUrls = ['https://crawlee.dev'];const crawler = new PuppeteerCrawler({// proxyConfiguration: new ProxyConfiguration({ proxyUrls: ['...'] }),+ httpClient: new ImpitHttpClient({ browser: Browser.Chrome }),requestHandler: router,// Comment this option to scrape the full website.maxRequestsPerCrawl: 20,