Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
map $http_accept $serve_markdown {
default 0;
~*text/plain 1;
~*text/markdown 1;
}
map $request_uri $has_no_extension {
~^[^.]*$ 1;
default 0;
}
# Nginx reverse proxy configuration for crawlee.dev
# Routes to GitHub Pages and handles legacy URL redirects
server {
listen 0.0.0.0:8080;
server_name 'crawlee.dev';
# comment out the resolver and use localhost:3000 for local development
set $backend "https://apify.github.io/crawlee";
resolver 1.1.1.1 8.8.8.8 valid=30s ipv6=off;
# Health check endpoint
location /health {
access_log off;
return 200 '{"status":"UP"}';
add_header Content-Type application/json;
}
location = / {
if ($serve_markdown) {
rewrite ^ /llms.txt last;
}
proxy_pass $backend/;
}
location ~ ^/(llms|llms-full)\.txt$ {
proxy_hide_header Content-Type;
add_header Content-Type 'text/markdown; charset=utf-8' always;
proxy_pass $backend$uri;
}
# remove trailing slashes from all URLs (except root /)
# exact match locations (e.g., location = /python/) take priority over this regex
# Only match URIs composed of safe characters (letters, digits, dots, hyphens,
# underscores, forward slashes). This prevents open redirect via %5C (backslash):
# nginx decodes %5C to \ in $uri, and \ in the Location header gets normalized
# to / by browsers, turning /\evil.com into //evil.com (protocol-relative URL).
location ~ ^(/[a-zA-Z0-9][a-zA-Z0-9_./-]*)/$ {
rewrite ^(.+)/$ $1$is_args$args? redirect;
}
location / {
set $rewrite_condition "$serve_markdown$has_no_extension";
set $proxy_path $request_uri;
if ($rewrite_condition = "11") {
set $proxy_path "${request_uri}.md";
}
proxy_pass $backend$proxy_path;
}
### Repository path: "/python"
location = /python {
if ($serve_markdown) {
rewrite ^ /python/llms.txt last;
}
proxy_pass https://apify.github.io/crawlee-python/;
}
location = /python/ {
if ($serve_markdown) {
rewrite ^ /python/llms.txt last;
}
proxy_pass https://apify.github.io/crawlee-python/;
}
location ~ ^/python/(llms|llms-full)\.txt$ {
proxy_hide_header Content-Type;
add_header Content-Type 'text/markdown; charset=utf-8' always;
proxy_pass https://apify.github.io/crawlee-python/$1.txt;
}
location ~ ^/python/(.*)$ {
set $path_suffix $1;
set $proxy_path "/$path_suffix";
set $rewrite_condition "$serve_markdown$has_no_extension";
if ($rewrite_condition = "11") {
set $proxy_path "${proxy_path}.md";
}
proxy_pass https://apify.github.io/crawlee-python$proxy_path;
}
# So that we can have both GH pages and crawlee.dev/python working and loading assets from the same path
location /crawlee-python {
proxy_pass https://apify.github.io/crawlee-python/;
}
# Redirect rule for old JS docs to go under /js prefix
rewrite ^/docs(.*)$ /js/docs$1 permanent;
rewrite ^/api(.*)$ /js/api$1 permanent;
# Remove version numbers from /js/api/3.[0-9]/* and /js/docs/3.[0-9]/*
rewrite ^/js/api/3\.\d(/.*)?$ /js/api$1 permanent;
rewrite ^/js/docs/3\.\d(/.*)?$ /js/docs$1 permanent;
# Redirect rule for "upgrading-to-v03" to "upgrading-to-v0x"
rewrite ^/python/docs/upgrading/upgrading-to-v03$ /python/docs/upgrading/upgrading-to-v0x permanent;
# Redirect rule so that /python/docs actually leads somewhere
rewrite ^/python/docs/?$ /python/docs/quick-start;
rewrite ^/versions/?$ /js/api/core/changelog permanent;
}