Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
docs: add new GIFs for quickstart (#1622)
Vlad Frangu2f701e658ee5
diff --git a/docs/introduction/01-setting-up.mdx b/docs/introduction/01-setting-up.mdxindex 3a6e1b51b..01e17f700 100644--- a/docs/introduction/01-setting-up.mdx+++ b/docs/introduction/01-setting-up.mdx@@ -4,6 +4,8 @@ title: "Setting up"description: Your first steps into the world of scraping with Crawlee---+import ThemedImage from '@theme/ThemedImage';+import ApiLink from '@site/src/components/ApiLink';To run Crawlee on your own computer, you need to meet the following pre-requisites first:@@ -71,7 +73,19 @@ headless: falseWhen you run the example again, after a second a Chromium browser window will open. In the window, you'll see quickly changing pages as the crawler does its job.-+:::note++For the sake of this show off, we've slowed down the crawler, but rest assured, it's blazing fast in real world usage.++:::++<ThemedImage+ alt="An image showing off Crawlee scraping the Crawlee website using Puppeteer/Playwright and Chromium"+ sources={{+ light: '/img/chrome-scrape-light-compressed.gif',+ dark: '/img/chrome-scrape-dark-compressed.gif',+ }}+/>;## Next lessondiff --git a/docs/quick-start/index.mdx b/docs/quick-start/index.mdxindex 1acc52bca..3a9b7ee96 100644--- a/docs/quick-start/index.mdx+++ b/docs/quick-start/index.mdx@@ -10,6 +10,7 @@ import Admonition from '@theme/Admonition';import Tabs from '@theme/Tabs';import TabItem from '@theme/TabItem';import CodeBlock from '@theme/CodeBlock';+import ThemedImage from '@theme/ThemedImage';import CheerioSource from '!!raw-loader!./quick_start_cheerio.ts';import PlaywrightSource from '!!raw-loader!./quick_start_playwright.ts';@@ -128,7 +129,19 @@ Browsers controlled by Puppeteer and Playwright run headless (without a visibleWhen you run the example code, you'll see an automated browser blaze through the Crawlee website.-+:::note++For the sake of this show off, we've slowed down the crawler, but rest assured, it's blazing fast in real world usage.++:::++<ThemedImage+ alt="An image showing off Crawlee scraping the Crawlee website using Puppeteer/Playwright and Chromium"+ sources={{+ light: '/img/chrome-scrape-light-compressed.gif',+ dark: '/img/chrome-scrape-dark-compressed.gif',+ }}+/>;## Resultsdiff --git a/website/static/img/chrome-scrape-dark-compressed.gif b/website/static/img/chrome-scrape-dark-compressed.gifnew file mode 100644index 000000000..aaf6c16fdBinary files /dev/null and b/website/static/img/chrome-scrape-dark-compressed.gif differdiff --git a/website/static/img/chrome-scrape-light-compressed.gif b/website/static/img/chrome-scrape-light-compressed.gifnew file mode 100644index 000000000..e6fd44858Binary files /dev/null and b/website/static/img/chrome-scrape-light-compressed.gif differdiff --git a/website/static/img/chrome_scrape.gif b/website/static/img/chrome_scrape.gifdeleted file mode 100644index 42a8c272f..000000000Binary files a/website/static/img/chrome_scrape.gif and /dev/null differdiff --git a/website/tools/website_gif/chrome-scrape-dark.gif b/website/tools/website_gif/chrome-scrape-dark.gifnew file mode 100644index 000000000..7f99b2e36Binary files /dev/null and b/website/tools/website_gif/chrome-scrape-dark.gif differdiff --git a/website/tools/website_gif/chrome-scrape-dark.mp4 b/website/tools/website_gif/chrome-scrape-dark.mp4new file mode 100644index 000000000..7fb4b0cf5Binary files /dev/null and b/website/tools/website_gif/chrome-scrape-dark.mp4 differdiff --git a/website/tools/website_gif/chrome-scrape-light.gif b/website/tools/website_gif/chrome-scrape-light.gifnew file mode 100644index 000000000..ed7ee4014Binary files /dev/null and b/website/tools/website_gif/chrome-scrape-light.gif differdiff --git a/website/tools/website_gif/chrome-scrape-light.mp4 b/website/tools/website_gif/chrome-scrape-light.mp4new file mode 100644index 000000000..3c018b968Binary files /dev/null and b/website/tools/website_gif/chrome-scrape-light.mp4 differdiff --git a/website/tools/website_gif/website_gif.mjs b/website/tools/website_gif/website_gif.mjsnew file mode 100644index 000000000..919c3584d--- /dev/null+++ b/website/tools/website_gif/website_gif.mjs@@ -0,0 +1,84 @@+/**+ * How to generate the gifs:+ *+ * 1. Set a breakpoint on the marked line+ * 2. Run the crawler with the debugger+ * 3. Setup your chrome and recording+ * 4. Resume, record, ???, profit!+ */++import { PuppeteerCrawler, RequestQueue, sleep } from 'crawlee';++const crawler = new PuppeteerCrawler({+ headless: false,+ maxConcurrency: 1,+ navigationTimeoutSecs: 100000,+ requestHandlerTimeoutSecs: 10000,+ browserPoolOptions: {+ closeInactiveBrowserAfterSecs: 100000,+ operationTimeoutSecs: 100000,+ },+ async requestHandler({ request }) {+ if (request.userData.label === 'start') {+ console.log('Waiting 10s, prepare recording!');+ await sleep(10000); // <--- Set breakpoint here+ } else {+ await sleep(2500); // <--- Set breakpoint here for initial request, remove after+ }+ },+});++await crawler.run([+ {+ url: 'https://crawlee.dev',+ userData: { label: 'start' },+ uniqueKey: 'dark-start'+ },+ {+ url: 'https://crawlee.dev/docs/quick-start',+ uniqueKey: 'dark-1'+ },+ {+ url: 'https://crawlee.dev/docs/introduction/setting-up',+ uniqueKey: 'dark-2'+ },+ {+ url: 'https://crawlee.dev/docs/introduction/first-crawler',+ uniqueKey: 'dark-3'+ },+ {+ url: 'https://crawlee.dev/docs/introduction/adding-urls',+ uniqueKey: 'dark-4'+ },+ {+ url: 'https://crawlee.dev/docs/introduction/real-world-project',+ uniqueKey: 'dark-5'+ },++ // Light theme+ {+ url: 'https://crawlee.dev',+ userData: { label: 'start' },+ uniqueKey: 'light th-start'+ },+ {+ url: 'https://crawlee.dev/docs/quick-start',+ uniqueKey: 'light th-1'+ },+ {+ url: 'https://crawlee.dev/docs/introduction/setting-up',+ uniqueKey: 'light th-2'+ },+ {+ url: 'https://crawlee.dev/docs/introduction/first-crawler',+ uniqueKey: 'light th-3'+ },+ {+ url: 'https://crawlee.dev/docs/introduction/adding-urls',+ uniqueKey: 'light th-4'+ },+ {+ url: 'https://crawlee.dev/docs/introduction/real-world-project',+ uniqueKey: 'light th-5'+ }+]);diff --git a/website/versioned_docs/version-3.0/introduction/01-setting-up.mdx b/website/versioned_docs/version-3.0/introduction/01-setting-up.mdxindex 9aee7e42e..4ac8fb3e3 100644--- a/website/versioned_docs/version-3.0/introduction/01-setting-up.mdx+++ b/website/versioned_docs/version-3.0/introduction/01-setting-up.mdx@@ -4,6 +4,8 @@ title: "Setting up"description: Your first steps into the world of scraping with Crawlee---+import ThemedImage from '@theme/ThemedImage';+import ApiLink from '@site/src/components/ApiLink';To run Crawlee on your own computer, you need to meet the following pre-requisites first:@@ -71,7 +73,19 @@ headless: falseWhen you run the example again, after a second a Chromium browser window will open. In the window, you'll see quickly changing pages as the crawler does its job.-+:::note++For the sake of this show off, we've slowed down the crawler, but rest assured, it's blazing fast in real world usage.++:::++<ThemedImage+ alt="An image showing off Crawlee scraping the Crawlee website using Puppeteer/Playwright and Chromium"+ sources={{+ light: '/img/chrome-scrape-light-compressed.gif',+ dark: '/img/chrome-scrape-dark-compressed.gif',+ }}+/>;## Next lessondiff --git a/website/versioned_docs/version-3.0/quick-start/index.mdx b/website/versioned_docs/version-3.0/quick-start/index.mdxindex 1acc52bca..3a9b7ee96 100644--- a/website/versioned_docs/version-3.0/quick-start/index.mdx+++ b/website/versioned_docs/version-3.0/quick-start/index.mdx@@ -10,6 +10,7 @@ import Admonition from '@theme/Admonition';import Tabs from '@theme/Tabs';import TabItem from '@theme/TabItem';import CodeBlock from '@theme/CodeBlock';+import ThemedImage from '@theme/ThemedImage';import CheerioSource from '!!raw-loader!./quick_start_cheerio.ts';import PlaywrightSource from '!!raw-loader!./quick_start_playwright.ts';@@ -128,7 +129,19 @@ Browsers controlled by Puppeteer and Playwright run headless (without a visibleWhen you run the example code, you'll see an automated browser blaze through the Crawlee website.-+:::note++For the sake of this show off, we've slowed down the crawler, but rest assured, it's blazing fast in real world usage.++:::++<ThemedImage+ alt="An image showing off Crawlee scraping the Crawlee website using Puppeteer/Playwright and Chromium"+ sources={{+ light: '/img/chrome-scrape-light-compressed.gif',+ dark: '/img/chrome-scrape-dark-compressed.gif',+ }}+/>;## Resultsdiff --git a/website/versioned_docs/version-3.1/introduction/01-setting-up.mdx b/website/versioned_docs/version-3.1/introduction/01-setting-up.mdxindex 3a6e1b51b..01e17f700 100644--- a/website/versioned_docs/version-3.1/introduction/01-setting-up.mdx+++ b/website/versioned_docs/version-3.1/introduction/01-setting-up.mdx@@ -4,6 +4,8 @@ title: "Setting up"description: Your first steps into the world of scraping with Crawlee---+import ThemedImage from '@theme/ThemedImage';+import ApiLink from '@site/src/components/ApiLink';To run Crawlee on your own computer, you need to meet the following pre-requisites first:@@ -71,7 +73,19 @@ headless: falseWhen you run the example again, after a second a Chromium browser window will open. In the window, you'll see quickly changing pages as the crawler does its job.-+:::note++For the sake of this show off, we've slowed down the crawler, but rest assured, it's blazing fast in real world usage.++:::++<ThemedImage+ alt="An image showing off Crawlee scraping the Crawlee website using Puppeteer/Playwright and Chromium"+ sources={{+ light: '/img/chrome-scrape-light-compressed.gif',+ dark: '/img/chrome-scrape-dark-compressed.gif',+ }}+/>;## Next lessondiff --git a/website/versioned_docs/version-3.1/quick-start/index.mdx b/website/versioned_docs/version-3.1/quick-start/index.mdxindex 1acc52bca..3a9b7ee96 100644--- a/website/versioned_docs/version-3.1/quick-start/index.mdx+++ b/website/versioned_docs/version-3.1/quick-start/index.mdx@@ -10,6 +10,7 @@ import Admonition from '@theme/Admonition';import Tabs from '@theme/Tabs';import TabItem from '@theme/TabItem';import CodeBlock from '@theme/CodeBlock';+import ThemedImage from '@theme/ThemedImage';import CheerioSource from '!!raw-loader!./quick_start_cheerio.ts';import PlaywrightSource from '!!raw-loader!./quick_start_playwright.ts';@@ -128,7 +129,19 @@ Browsers controlled by Puppeteer and Playwright run headless (without a visibleWhen you run the example code, you'll see an automated browser blaze through the Crawlee website.-+:::note++For the sake of this show off, we've slowed down the crawler, but rest assured, it's blazing fast in real world usage.++:::++<ThemedImage+ alt="An image showing off Crawlee scraping the Crawlee website using Puppeteer/Playwright and Chromium"+ sources={{+ light: '/img/chrome-scrape-light-compressed.gif',+ dark: '/img/chrome-scrape-dark-compressed.gif',+ }}+/>;## Results