Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
docs: adjust HTML title and description
Martin Adámekbc447451744b
diff --git a/website/docusaurus.config.js b/website/docusaurus.config.jsindex 17051d1dd..f07477162 100644--- a/website/docusaurus.config.js+++ b/website/docusaurus.config.js@@ -35,7 +35,7 @@ const packagesOrder = [/** @type {Partial<import('@docusaurus/types').DocusaurusConfig>} */module.exports = {- title: 'Crawlee',+ title: 'Crawlee · Build reliable crawlers. Fast.',tagline: 'Build reliable crawlers. Fast.',url: 'https://crawlee.dev',baseUrl: '/',@@ -73,6 +73,7 @@ module.exports = {},blog: {blogTitle: 'Crawlee Blog - learn how to build better scrapers',+ // eslint-disable-next-line max-lenblogDescription: 'Guides and tutorials on using Crawlee, the most reliable open-source web scraping and browser automation library for JavaScript and Node.js developers.',},theme: {@@ -241,7 +242,12 @@ module.exports = {darkTheme: require('prism-react-renderer').themes.dracula,additionalLanguages: ['docker', 'log', 'bash', 'diff', 'json'],},- metadata: [],+ metadata: [+ // eslint-disable-next-line max-len+ { name: 'description', content: `Crawlee helps you build and maintain your crawlers. It's open source, but built by developers who scrape millions of pages every day for a living.` },+ // eslint-disable-next-line max-len+ { name: 'og:description', content: `Crawlee helps you build and maintain your crawlers. It's open source, but built by developers who scrape millions of pages every day for a living.` },+ ],image: 'img/crawlee-og.png',footer: {links: [diff --git a/website/src/pages/index.js b/website/src/pages/index.jsindex c723d44ca..765872652 100644--- a/website/src/pages/index.js+++ b/website/src/pages/index.js@@ -241,7 +241,6 @@ export default function Home() {const { siteConfig } = useDocusaurusContext();return (<Layout- title={`${siteConfig.title} · ${siteConfig.tagline}`}description={siteConfig.description}><Hero /><Features />