Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

docs: file download examples (#2546)

Jindřich Bära167ec3440b9
diff --git a/docs/examples/file_download.mdx b/docs/examples/file_download.mdx
new file mode 100644
index 000000000..090320eda
--- /dev/null
+++ b/docs/examples/file_download.mdx
@@ -0,0 +1,17 @@
+---
+id: file-download
+title: Download a file
+---
+
+import RunnableCodeBlock from '@site/src/components/RunnableCodeBlock';
+import ApiLink from '@site/src/components/ApiLink';
+import FileDownloadSource from '!!raw-loader!roa-loader!./file_download.ts';
+
+When webcrawling, you sometimes need to download files such as images, PDFs, or other binary files. This example demonstrates how to download files using Crawlee and save them to the default key-value store.
+
+The script simply downloads several files with plain HTTP requests using the custom <ApiLink to="http-crawler/class/FileDownload">`FileDownload`</ApiLink> crawler class and stores their contents in the default key-value store.
+In local configuration, the data will be stored as files in `./storage/key_value_stores/default`.
+
+<RunnableCodeBlock className="language-js" type="cheerio">
+ {FileDownloadSource}
+</RunnableCodeBlock>
diff --git a/docs/examples/file_download.ts b/docs/examples/file_download.ts
new file mode 100644
index 000000000..a6b42555e
--- /dev/null
+++ b/docs/examples/file_download.ts
@@ -0,0 +1,23 @@
+import { FileDownload } from 'crawlee';
+
+// Create a FileDownload - a custom crawler instance that will download files from URLs.
+const crawler = new FileDownload({
+ async requestHandler({ body, request, contentType, getKeyValueStore }) {
+ const url = new URL(request.url);
+ const kvs = await getKeyValueStore();
+
+ await kvs.setValue(url.pathname.replace(/\//g, '_'), body, { contentType: contentType.type });
+ },
+});
+
+// The initial list of URLs to crawl. Here we use just a few hard-coded URLs.
+await crawler.addRequests([
+ 'https://pdfobject.com/pdf/sample.pdf',
+ 'https://download.blender.org/peach/bigbuckbunny_movies/BigBuckBunny_320x180.mp4',
+ 'https://upload.wikimedia.org/wikipedia/commons/c/c8/Example.ogg',
+]);
+
+// Run the downloader and wait for it to finish.
+await crawler.run();
+
+console.log('Crawler finished.');
diff --git a/docs/examples/file_download_stream.mdx b/docs/examples/file_download_stream.mdx
new file mode 100644
index 000000000..bd7c130eb
--- /dev/null
+++ b/docs/examples/file_download_stream.mdx
@@ -0,0 +1,17 @@
+---
+id: file-download-stream
+title: Download a file with Node.js streams
+---
+
+import RunnableCodeBlock from '@site/src/components/RunnableCodeBlock';
+import ApiLink from '@site/src/components/ApiLink';
+import FileDownloadSource from '!!raw-loader!roa-loader!./file_download_stream.ts';
+
+For larger files, it is more efficient to use Node.js streams to download and transfer the files. This example demonstrates how to download files using streams.
+
+The script uses the <ApiLink to="http-crawler/class/FileDownload">`FileDownload`</ApiLink> crawler class to download files with streams, log the progress, and store the data in the key-value store.
+In local configuration, the data will be stored as files in `./storage/key_value_stores/default`.
+
+<RunnableCodeBlock className="language-js" type="cheerio">
+ {FileDownloadSource}
+</RunnableCodeBlock>
diff --git a/docs/examples/file_download_stream.ts b/docs/examples/file_download_stream.ts
new file mode 100644
index 000000000..a7f39a70f
--- /dev/null
+++ b/docs/examples/file_download_stream.ts
@@ -0,0 +1,62 @@
+import { pipeline, Transform } from 'stream';
+
+import { FileDownload, type Log } from 'crawlee';
+
+// A sample Transform stream logging the download progress.
+function createProgressTracker({ url, log, totalBytes }: { url: URL; log: Log; totalBytes: number }) {
+ let downloadedBytes = 0;
+
+ return new Transform({
+ transform(chunk, _, callback) {
+ if (downloadedBytes % 1e6 > (downloadedBytes + chunk.length) % 1e6) {
+ log.info(
+ `Downloaded ${downloadedBytes / 1e6} MB (${Math.floor((downloadedBytes / totalBytes) * 100)}%) for ${url}.`,
+ );
+ }
+ downloadedBytes += chunk.length;
+
+ this.push(chunk);
+ callback();
+ },
+ });
+}
+
+// Create a FileDownload - a custom crawler instance that will download files from URLs.
+const crawler = new FileDownload({
+ async streamHandler({ stream, request, log, getKeyValueStore }) {
+ const url = new URL(request.url);
+
+ log.info(`Downloading ${url} to ${url.pathname.replace(/\//g, '_')}...`);
+
+ await new Promise<void>((resolve, reject) => {
+ // With the 'response' event, we have received the headers of the response.
+ stream.on('response', async (response) => {
+ const kvs = await getKeyValueStore();
+ await kvs.setValue(
+ url.pathname.replace(/\//g, '_'),
+ pipeline(
+ stream,
+ createProgressTracker({ url, log, totalBytes: Number(response.headers['content-length']) }),
+ (error) => {
+ if (error) reject(error);
+ },
+ ),
+ { contentType: response.headers['content-type'] },
+ );
+
+ log.info(`Downloaded ${url} to ${url.pathname.replace(/\//g, '_')}.`);
+
+ resolve();
+ });
+ });
+ },
+});
+
+// The initial list of URLs to crawl. Here we use just a few hard-coded URLs.
+await crawler.addRequests([
+ 'https://download.blender.org/peach/bigbuckbunny_movies/BigBuckBunny_320x180.mp4',
+ 'https://download.blender.org/peach/bigbuckbunny_movies/BigBuckBunny_640x360.m4v',
+]);
+
+// Run the downloader and wait for it to finish.
+await crawler.run();
diff --git a/packages/http-crawler/src/internals/file-download.ts b/packages/http-crawler/src/internals/file-download.ts
index 5d7cf8760..46046e930 100644
--- a/packages/http-crawler/src/internals/file-download.ts
+++ b/packages/http-crawler/src/internals/file-download.ts
@@ -2,6 +2,8 @@ import { finished } from 'stream/promises';
import { isPromise } from 'util/types';
import type { Dictionary } from '@crawlee/types';
+// @ts-expect-error got-scraping is ESM only
+import type { Request } from 'got-scraping';
import type {
ErrorHandler,
@@ -23,7 +25,7 @@ export type StreamHandlerContext = Omit<
FileDownloadCrawlingContext,
'body' | 'response' | 'parseWithCheerio' | 'json' | 'addRequests' | 'contentType'
> & {
- stream: ReadableStream;
+ stream: Request;
};
type StreamHandler = (context: StreamHandlerContext) => void | Promise<void>;