Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
docs: add new LLM dropdown (#3267)
Richard Birošd7cb5a099a9f
diff --git a/website/package.json b/website/package.jsonindex b3d434dd5..04680b6b0 100644--- a/website/package.json+++ b/website/package.json@@ -35,6 +35,7 @@},"dependencies": {"@apify/docusaurus-plugin-typedoc-api": "^4.4.8",+ "@apify/ui-icons": "^1.23.0","@apify/utilities": "^2.8.0","@docusaurus/core": "3.9.2","@docusaurus/faster": "3.9.2",diff --git a/website/src/components/LLMButtons.jsx b/website/src/components/LLMButtons.jsxnew file mode 100644index 000000000..6f4f070fa--- /dev/null+++ b/website/src/components/LLMButtons.jsx@@ -0,0 +1,513 @@+import {+ AnthropicIcon,+ ChatGptIcon,+ CheckIcon,+ ChevronDownIcon,+ CopyIcon,+ ExternalLinkIcon,+ LoaderIcon,+ MarkdownIcon,+ PerplexityIcon,+} from '@apify/ui-icons';+import clsx from 'clsx';+import { useLocation } from '@docusaurus/router';+import React, {+ useCallback,+ useEffect,+ useMemo,+ useRef,+ useState,+} from 'react';++import styles from './LLMButtons.module.css';++const DROPDOWN_OPTIONS = [+ {+ label: 'Copy for LLM',+ description: 'Copy page as Markdown for LLMs',+ showExternalIcon: false,+ icon: CopyIcon,+ value: 'copyForLLM',+ analytics: {+ buttonText: 'Copy for LLM',+ element: 'llm-buttons.copyForLLM',+ },+ },+ {+ label: 'View as Markdown',+ description: 'View this page as plain text',+ icon: MarkdownIcon,+ value: 'viewAsMarkdown',+ showExternalIcon: true,+ analytics: {+ buttonText: 'View as Markdown',+ element: 'llm-buttons.viewAsMarkdown',+ },+ },+ {+ label: 'Open in ChatGPT',+ description: 'Ask questions about this page',+ icon: ChatGptIcon,+ value: 'openInChatGPT',+ showExternalIcon: true,+ analytics: {+ buttonText: 'Open in ChatGPT',+ element: 'llm-buttons.openInChatGPT',+ },+ },+ {+ label: 'Open in Claude',+ description: 'Ask questions about this page',+ icon: AnthropicIcon,+ value: 'openInClaude',+ showExternalIcon: true,+ analytics: {+ buttonText: 'Open in Claude',+ element: 'llm-buttons.openInClaude',+ },+ },+ {+ label: 'Open in Perplexity',+ description: 'Ask questions about this page',+ icon: PerplexityIcon,+ value: 'openInPerplexity',+ showExternalIcon: true,+ analytics: {+ buttonText: 'Open in Perplexity',+ element: 'llm-buttons.openInPerplexity',+ },+ },+];++const CHAT_GPT_BASE = 'https://chatgpt.com/?hints=search&q=';+const CLAUDE_BASE = 'https://claude.ai/new?q=';+const PERPLEXITY_BASE = 'https://www.perplexity.ai/search/new?q=';++const getPrompt = (currentUrl) => `Read from ${currentUrl} so I can ask questions about it.`;+const getMarkdownUrl = (currentUrl) => {+ const url = new URL(currentUrl);+ url.pathname = `${url.pathname.replace(/\/$/, '')}.md`;+ return url.toString();+};++const trackClick = (buttonText, element) => {+ if (typeof window !== 'undefined' && window.analytics) {+ window.analytics.track('Clicked', {+ app: 'crawlee',+ button_text: buttonText,+ element,+ });+ }+};++const getOptionHref = (value, currentUrl) => {+ if (!currentUrl) {+ return undefined;+ }++ switch (value) {+ case 'viewAsMarkdown':+ return getMarkdownUrl(currentUrl);+ case 'openInChatGPT':+ return `${CHAT_GPT_BASE}${encodeURIComponent(getPrompt(currentUrl))}`;+ case 'openInClaude':+ return `${CLAUDE_BASE}${encodeURIComponent(getPrompt(currentUrl))}`;+ case 'openInPerplexity':+ return `${PERPLEXITY_BASE}${encodeURIComponent(getPrompt(currentUrl))}`;+ default:+ return undefined;+ }+};++const Menu = ({+ className,+ components = {},+ onMenuOpen,+ onSelect,+ options = [],+}) => {+ const [isOpen, setIsOpen] = useState(false);+ const [focusedIndex, setFocusedIndex] = useState(0);+ const menuRef = useRef(null);+ const menuItemRefs = useRef([]);++ const MenuBaseComponent = components.MenuBase;++ const closeMenu = useCallback(() => {+ setIsOpen(false);+ setFocusedIndex(0);+ }, []);++ const toggleMenu = useCallback(() => {+ setIsOpen((prev) => {+ if (!prev) {+ setFocusedIndex(0);+ }+ return !prev;+ });+ }, []);++ const handleKeyDown = useCallback(+ (event) => {+ if (event.key === 'Enter' || event.key === ' ') {+ event.preventDefault();+ toggleMenu();+ } else if (event.key === 'ArrowDown') {+ event.preventDefault();+ if (!isOpen) {+ toggleMenu();+ } else {+ setFocusedIndex((prev) => (prev + 1) % options.length);+ }+ } else if (event.key === 'ArrowUp') {+ event.preventDefault();+ if (isOpen) {+ setFocusedIndex((prev) => (prev - 1 + options.length) % options.length);+ }+ }+ },+ [toggleMenu, isOpen, options.length],+ );++ const handleOptionSelect = useCallback(+ (option, event) => {+ onSelect?.(option, event);+ closeMenu();+ },+ [closeMenu, onSelect],+ );++ const handleMenuItemKeyDown = useCallback(+ (event, option, index) => {+ if (event.key === 'Enter' || event.key === ' ') {+ event.preventDefault();+ event.currentTarget.click();+ return;+ }++ if (event.key === 'ArrowDown') {+ event.preventDefault();+ setFocusedIndex((index + 1) % options.length);+ return;+ }++ if (event.key === 'ArrowUp') {+ event.preventDefault();+ setFocusedIndex((index - 1 + options.length) % options.length);+ return;+ }++ if (event.key === 'Escape') {+ event.preventDefault();+ closeMenu();+ }+ },+ [options.length, closeMenu],+ );++ useEffect(() => {+ onMenuOpen?.(isOpen);+ }, [isOpen, onMenuOpen]);++ useEffect(() => {+ if (isOpen && menuItemRefs.current[focusedIndex]) {+ menuItemRefs.current[focusedIndex].focus();+ }+ }, [isOpen, focusedIndex]);++ useEffect(() => {+ if (!isOpen) {+ return undefined;+ }++ const handleClickOutside = (event) => {+ if (!menuRef.current?.contains(event.target)) {+ closeMenu();+ }+ };++ const handleEscape = (event) => {+ if (event.key === 'Escape') {+ closeMenu();+ }+ };++ document.addEventListener('mousedown', handleClickOutside);+ document.addEventListener('keydown', handleEscape);++ return () => {+ document.removeEventListener('mousedown', handleClickOutside);+ document.removeEventListener('keydown', handleEscape);+ };+ }, [closeMenu, isOpen]);++ return (+ <div className={clsx(styles.menu, className)} ref={menuRef}>+ <MenuBaseComponent+ onClick={toggleMenu}+ onKeyDown={handleKeyDown}+ aria-haspopup="menu"+ aria-expanded={isOpen}+ aria-controls="llm-menu"+ />+ {isOpen && (+ <div className={styles.menuDropdown} role="menu" id="llm-menu">+ {options.map((option, index) => {+ const WrapperComponent = option.href ? 'a' : 'button';++ return (+ <WrapperComponent+ key={option.value}+ ref={(el) => {+ menuItemRefs.current[index] = el;+ }}+ className={styles.menuOptionWrapper}+ role="menuitem"+ tabIndex={0}+ href={option.href}+ target={option.target}+ rel={option.rel}+ type={option.href ? undefined : 'button'}+ onClick={(event) => {+ if (!option.href) {+ event.preventDefault();+ }+ handleOptionSelect(option, event);+ }}+ onKeyDown={(e) => handleMenuItemKeyDown(e, option, index)}+ >+ <Option {...option} />+ </WrapperComponent>+ );+ })}+ </div>+ )}+ </div>+ );+};++function getButtonText({ status }) {+ switch (status) {+ case 'loading':+ return 'Copying...';+ case 'copied':+ return 'Copied';+ default:+ return 'Copy for LLM';+ }+}++const onCopyAsMarkdownClick = async ({ setCopyingStatus, currentUrl }) => {+ const sourceUrl = currentUrl || (typeof window !== 'undefined' ? window.location.href : '');++ if (!sourceUrl) {+ return;+ }++ trackClick('Copy for LLM', 'llm-buttons.copyForLLM');++ const markdownUrl = getMarkdownUrl(sourceUrl);++ try {+ setCopyingStatus('loading');++ const response = await fetch(markdownUrl);++ if (!response.ok) {+ throw new Error(`Failed to fetch markdown: ${response.status}`);+ }++ const markdownContent = await response.text();+ await navigator.clipboard.writeText(markdownContent);+ setCopyingStatus('copied');+ } catch (error) {+ console.error('Failed to copy markdown content:', error);+ } finally {+ setTimeout(() => setCopyingStatus('idle'), 2000);+ }+};++const COPYING_STATUS_ICON = {+ loading: <LoaderIcon size={16} />,+ copied: <CheckIcon size={16} />,+ idle: <CopyIcon size={16} />,+}++const MenuBase = React.forwardRef(({+ copyingStatus,+ setCopyingStatus,+ chevronIconRef,+ currentUrl,+ ...buttonProps+}, ref) => {+ const mergedButtonProps = {+ ...buttonProps,+ tabIndex: buttonProps.tabIndex ?? 0,+ };++ return (+ <div className={styles.llmButtonWrapper}>+ <div+ ref={ref}+ className={styles.llmButton}+ {...mergedButtonProps}+ >+ <div+ className={styles.copyUpIconWrapper}+ onClick={(event) => {+ event.stopPropagation();+ onCopyAsMarkdownClick({ setCopyingStatus, currentUrl });+ }}+ >+ {COPYING_STATUS_ICON[copyingStatus]}+ </div>+ <span+ onClick={(event) => {+ event.stopPropagation();+ onCopyAsMarkdownClick({ setCopyingStatus, currentUrl });+ }}+ className={styles.llmButtonText}+ >+ {getButtonText({ status: copyingStatus })}+ </span>+ <div className={styles.chevronIconWrapper}>+ <ChevronDownIcon+ size="16"+ color="currentColor"+ className={styles.chevronIcon}+ ref={chevronIconRef}+ />+ </div>+ </div>+ </div>+ );+});+MenuBase.displayName = 'MenuBase';++const Option = ({ label, description, showExternalIcon, icon }) => {+ const Icon = icon ?? CopyIcon;++ return (+ <div className={styles.menuOption}>+ <Icon size={16} className={styles.menuOptionIcon} />+ <div className={styles.menuOptionText}>+ <span className={styles.menuOptionLabel}>{label}</span>+ <span className={styles.menuOptionDescription}>{description}</span>+ </div>+ {showExternalIcon && (+ <ExternalLinkIcon+ size={16}+ className={styles.menuOptionExternalIcon}+ />+ )}+ </div>+ );+};++export default function LLMButtons() {+ const location = useLocation();+ const [copyingStatus, setCopyingStatus] = useState('idle');+ const [currentUrl, setCurrentUrl] = useState('');+ const [isMarkdownAvailable, setIsMarkdownAvailable] = useState(false);+ const chevronIconRef = useRef(null);++ useEffect(() => {+ if (typeof window !== 'undefined') {+ setCurrentUrl(window.location.href);+ }+ }, [location]);++ useEffect(() => {+ if (!currentUrl) {+ setIsMarkdownAvailable(false);+ return undefined;+ }++ const controller = new AbortController();+ const markdownUrl = getMarkdownUrl(currentUrl);++ const checkMarkdownAvailability = async () => {+ try {+ const response = await fetch(markdownUrl, {+ method: 'HEAD',+ signal: controller.signal,+ });+ setIsMarkdownAvailable(response.ok);+ } catch (error) {+ if (error.name === 'AbortError') {+ return;+ }+ setIsMarkdownAvailable(false);+ }+ };++ checkMarkdownAvailability();++ return () => {+ controller.abort();+ };+ }, [currentUrl]);++ const menuOptions = useMemo(+ () =>+ DROPDOWN_OPTIONS.map((option) => {+ const href = getOptionHref(option.value, currentUrl);+++ if (option.value === 'viewAsMarkdown') {+ if (!isMarkdownAvailable) {+ return null;+ }+ }++ return {+ ...option,+ href,+ target: href ? '_blank' : undefined,+ rel: href ? 'noopener noreferrer' : undefined,+ };+ }).filter(Boolean),+ [currentUrl, isMarkdownAvailable],+ );++ const onMenuOptionClick = useCallback(+ (option, event) => {+ if (!option) {+ return;+ }++ if (option.analytics) {+ trackClick(option.analytics.buttonText, option.analytics.element);+ }++ if (option.value === 'copyForLLM') {+ event?.preventDefault();+ onCopyAsMarkdownClick({ setCopyingStatus, currentUrl });+ }+ },+ [currentUrl, setCopyingStatus],+ );++ return (+ <Menu+ className={styles.llmMenu}+ onMenuOpen={(isOpen) => chevronIconRef.current?.classList.toggle(+ styles.chevronIconOpen,+ isOpen,+ )}+ components={{+ MenuBase: (props) => (+ <MenuBase+ copyingStatus={copyingStatus}+ setCopyingStatus={setCopyingStatus}+ chevronIconRef={chevronIconRef}+ currentUrl={currentUrl}+ {...props}+ />+ ),+ }}+ onSelect={onMenuOptionClick}+ options={menuOptions}+ />+ );+}diff --git a/website/src/components/LLMButtons.module.css b/website/src/components/LLMButtons.module.cssnew file mode 100644index 000000000..7d203d8b5--- /dev/null+++ b/website/src/components/LLMButtons.module.css@@ -0,0 +1,151 @@+.llmMenu {+ display: flex;+ justify-content: flex-end;+ flex: 0 0 auto;+}++@media (max-width: 996px) {+ .llmMenu {+ width: 100%;+ justify-content: flex-start;+ }+}++.llmButtonWrapper {+ display: flex;+ justify-content: flex-end;+ width: auto;+}++.llmButton {+ display: flex;+ align-items: center;+ border-radius: 0.5rem;+ border: 1px solid var(--color-separator);+ background-color: var(--color-background-subtle);+ cursor: pointer;+ transition: background-color 0.2s ease-in-out, border-color 0.2s ease-in-out;+}++.copyUpIconWrapper {+ display: flex;+ align-items: center;+ justify-content: center;+ padding: 0.6rem 0.5rem 0.6rem 0.8rem;+}++.llmButtonText {+ display: flex;+ align-items: center;+ padding-right: 0.8rem;+ border-right: 1px solid var(--color-separator);+ margin: 0;+ font: 400 0.875rem/1.4 Inter, sans-serif;+}++.chevronIconWrapper {+ display: flex;+ align-items: center;+ justify-content: center;+ padding-inline: 0.25rem;+}++.chevronIcon {+ transition: transform 0.2s ease-in-out;+}++.chevronIconOpen {+ transform: rotate(180deg);+}++.menu {+ position: relative;+ width: fit-content;+}++.menuDropdown {+ position: absolute;+ right: 0;+ margin-top: 0.5rem;+ padding: 0.375rem;+ border-radius: 0.75rem;+ border: 1px solid var(--color-separator);+ background-color: var(--color-background);+ box-shadow: 0 12px 32px rgb(10 11 36 / 20%);+ min-width: 17rem;+ max-width: min(20rem, calc(100vw - 1.5rem));+ z-index: 2;+ display: flex;+ flex-direction: column;+ gap: 0.25rem;+}++@media (max-width: 996px) {+ .menuDropdown {+ left: 0;+ right: auto;+ width: min(20rem, calc(100vw - 1.5rem));+ }+}++.menuOption {+ display: flex;+ gap: 0.5rem;+ padding: 0.25rem 0.5rem;+ border-radius: 0.5rem;+ transition: background-color 0.15s ease-in-out;+}++.menuOption:hover {+ background: var(--color-hover);+}++.menuOptionWrapper {+ border: none;+ background: transparent;+ padding: 0;+ text-align: left;+ width: 100%;+ display: block;+ text-decoration: none;+ color: inherit;+ cursor: pointer;+ outline: none;+}++.menuOptionWrapper:focus-visible .menuOption {+ background: var(--color-hover);+ outline-offset: -2px;+}++.menuOptionIcon,+.menuOptionExternalIcon {+ flex-shrink: 0;+}++.menuOptionIcon {+ margin-top: 0.2rem;+}++.menuOptionText {+ flex: 1;+ display: flex;+ flex-direction: column;+ gap: 0.125rem;+ line-height: 1rem;+ padding: 4px 0;+}++.menuOptionLabel {+ margin: 0;+ font-size: 0.875rem;+ line-height: 1rem;+ font-weight: 400;+ color: var(--ifm-font-color-base);+}++.menuOptionDescription {+ margin: 0;+ font-size: 0.8rem;+ color: var(--color-text-subtle);+}\ No newline at end of filediff --git a/website/src/theme/DocItem/Content/index.js b/website/src/theme/DocItem/Content/index.jsnew file mode 100644index 000000000..d870c4635--- /dev/null+++ b/website/src/theme/DocItem/Content/index.js@@ -0,0 +1,37 @@+import { useDoc } from '@docusaurus/plugin-content-docs/client';+import { ThemeClassNames } from '@docusaurus/theme-common';+import LLMButtons from '@site/src/components/LLMButtons';+import Heading from '@theme/Heading';+import MDXContent from '@theme/MDXContent';+import clsx from 'clsx';+import React from 'react';++import styles from './styles.module.css';++function useSyntheticTitle() {+ const { metadata, frontMatter, contentTitle } = useDoc();+ const shouldRender = !frontMatter.hide_title && typeof contentTitle === 'undefined';++ if (!shouldRender) {+ return null;+ }++ return metadata.title;+ }++export default function DocItemContent({ children }) {+ const syntheticTitle = useSyntheticTitle();++ return (+ <div className={clsx(ThemeClassNames.docs.docMarkdown, 'markdown')}>+ {syntheticTitle && (+ <div className={styles.docItemContent}>+ {syntheticTitle && <Heading as="h1">{syntheticTitle}</Heading>}+ <LLMButtons />+ </div>+ )}+ <MDXContent>{children}</MDXContent>+ </div>+ );+}+diff --git a/website/src/theme/DocItem/Content/styles.module.css b/website/src/theme/DocItem/Content/styles.module.cssnew file mode 100644index 000000000..f20bbd05c--- /dev/null+++ b/website/src/theme/DocItem/Content/styles.module.css@@ -0,0 +1,22 @@+.docItemContent {+ display: flex;+ align-items: center;+ justify-content: space-between;+ gap: 1rem;+ flex-wrap: wrap;+ padding-bottom: calc(+ var(--ifm-h1-vertical-rhythm-bottom) * var(--ifm-leading)+ );++ h1 {+ margin: 0 !important;+ flex: 1 1 auto;+ min-width: 12rem;+ }++ @media (max-width: 767px) {+ flex-direction: column;+ align-items: flex-start;+ gap: 0.75rem;+ }+}diff --git a/website/yarn.lock b/website/yarn.lockindex ad05bf237..b980cf218 100644--- a/website/yarn.lock+++ b/website/yarn.lock@@ -335,6 +335,18 @@ __metadata:languageName: nodelinkType: hard+"@apify/ui-icons@npm:^1.23.0":+ version: 1.23.0+ resolution: "@apify/ui-icons@npm:1.23.0"+ dependencies:+ clsx: "npm:^2.0.0"+ peerDependencies:+ react: 17.x || 18.x+ react-dom: 17.x || 18.x+ checksum: 10c0/960399ba6878ae564e23d095a3f65d13c1f7a2bff8c3dd3804c55b9bbfa084e7f464be35bc11cee4d7c68164fb729aa821b5232b703bac2609c85dd891c6ef19+ languageName: node+ linkType: hard+"@apify/utilities@npm:^2.8.0":version: 2.20.3resolution: "@apify/utilities@npm:2.20.3"@@ -14630,6 +14642,7 @@ __metadata:"@apify/docusaurus-plugin-typedoc-api": "npm:^4.4.8""@apify/eslint-config-ts": "npm:^0.4.0""@apify/tsconfig": "npm:^0.1.0"+ "@apify/ui-icons": "npm:^1.23.0""@apify/utilities": "npm:^2.8.0""@docusaurus/core": "npm:3.9.2""@docusaurus/faster": "npm:3.9.2"