Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

feat: pass `response` to `FileDownload.streamHandler` (#2930)

Jindřich Bär008c4c7d8791
diff --git a/packages/http-crawler/src/internals/file-download.ts b/packages/http-crawler/src/internals/file-download.ts
index 48cd2e9c0..6c095c352 100644
--- a/packages/http-crawler/src/internals/file-download.ts
+++ b/packages/http-crawler/src/internals/file-download.ts
@@ -23,7 +23,7 @@ export type FileDownloadErrorHandler<
export type StreamHandlerContext = Omit<
FileDownloadCrawlingContext,
- 'body' | 'response' | 'parseWithCheerio' | 'json' | 'addRequests' | 'contentType'
+ 'body' | 'parseWithCheerio' | 'json' | 'addRequests' | 'contentType'
> & {
stream: Request; // TODO BC - remove in v4
};
@@ -117,7 +117,7 @@ export class FileDownload extends HttpCrawler<FileDownloadCrawlingContext> {
this.streamHandler = streamHandler;
if (this.streamHandler) {
- this.requestHandler = this.streamRequestHandler;
+ this.requestHandler = this.streamRequestHandler as any;
}
// The base HttpCrawler class only supports a handful of text based mime types.
@@ -170,6 +170,7 @@ export class FileDownload extends HttpCrawler<FileDownloadCrawlingContext> {
try {
context.stream = response.stream;
+ context.response = response as any;
streamHandlerResult = this.streamHandler!(context as any);
} catch (e) {
cleanUp();
diff --git a/test/core/crawlers/file_download.test.ts b/test/core/crawlers/file_download.test.ts
index 50170be59..123432ece 100644
--- a/test/core/crawlers/file_download.test.ts
+++ b/test/core/crawlers/file_download.test.ts
@@ -119,6 +119,23 @@ test('streamHandler works', async () => {
expect(result).toEqual(await ReadableStreamGenerator.getBuffer(1024, 456));
});
+test('streamHandler receives response', async () => {
+ const crawler = new FileDownload({
+ maxRequestRetries: 0,
+ streamHandler: async ({ response }) => {
+ expect(response.headers['content-type']).toBe('application/octet-stream');
+ expect(response.rawHeaders[0]).toBe('content-type');
+ expect(response.rawHeaders[1]).toBe('application/octet-stream');
+ expect(response.statusCode).toBe(200);
+ expect(response.statusMessage).toBe('OK');
+ },
+ });
+
+ const fileUrl = new URL('/file?size=1024&seed=456', url).toString();
+
+ await crawler.run([fileUrl]);
+});
+
test('crawler with streamHandler waits for the stream to finish', async () => {
const bufferingStream = new Duplex({
read() {},