Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

docs: adjust HTML title and description

Martin Adámekbc447451744b
diff --git a/website/docusaurus.config.js b/website/docusaurus.config.js
index 17051d1dd..f07477162 100644
--- a/website/docusaurus.config.js
+++ b/website/docusaurus.config.js
@@ -35,7 +35,7 @@ const packagesOrder = [
/** @type {Partial<import('@docusaurus/types').DocusaurusConfig>} */
module.exports = {
- title: 'Crawlee',
+ title: 'Crawlee · Build reliable crawlers. Fast.',
tagline: 'Build reliable crawlers. Fast.',
url: 'https://crawlee.dev',
baseUrl: '/',
@@ -73,6 +73,7 @@ module.exports = {
},
blog: {
blogTitle: 'Crawlee Blog - learn how to build better scrapers',
+ // eslint-disable-next-line max-len
blogDescription: 'Guides and tutorials on using Crawlee, the most reliable open-source web scraping and browser automation library for JavaScript and Node.js developers.',
},
theme: {
@@ -241,7 +242,12 @@ module.exports = {
darkTheme: require('prism-react-renderer').themes.dracula,
additionalLanguages: ['docker', 'log', 'bash', 'diff', 'json'],
},
- metadata: [],
+ metadata: [
+ // eslint-disable-next-line max-len
+ { name: 'description', content: `Crawlee helps you build and maintain your crawlers. It's open source, but built by developers who scrape millions of pages every day for a living.` },
+ // eslint-disable-next-line max-len
+ { name: 'og:description', content: `Crawlee helps you build and maintain your crawlers. It's open source, but built by developers who scrape millions of pages every day for a living.` },
+ ],
image: 'img/crawlee-og.png',
footer: {
links: [
diff --git a/website/src/pages/index.js b/website/src/pages/index.js
index c723d44ca..765872652 100644
--- a/website/src/pages/index.js
+++ b/website/src/pages/index.js
@@ -241,7 +241,6 @@ export default function Home() {
const { siteConfig } = useDocusaurusContext();
return (
<Layout
- title={`${siteConfig.title} · ${siteConfig.tagline}`}
description={siteConfig.description}>
<Hero />
<Features />