Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(scrapeURL/engines/pdf,docx): support authorization
Móricz Gergő3c614a2e5c9d
diff --git a/apps/api/src/scraper/scrapeURL/engines/docx/index.ts b/apps/api/src/scraper/scrapeURL/engines/docx/index.tsindex 933d4d747..e7d9efe84 100644--- a/apps/api/src/scraper/scrapeURL/engines/docx/index.ts+++ b/apps/api/src/scraper/scrapeURL/engines/docx/index.ts@@ -4,7 +4,9 @@ import { downloadFile } from "../utils/downloadFile";import mammoth from "mammoth";export async function scrapeDOCX(meta: Meta): Promise<EngineScrapeResult> {- const { response, tempFilePath } = await downloadFile(meta.id, meta.url);+ const { response, tempFilePath } = await downloadFile(meta.id, meta.url, {+ headers: meta.options.headers,+ });return {url: response.url,diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/index.tsindex b774cd0ef..101c9a535 100644--- a/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts@@ -74,7 +74,9 @@ export async function scrapePDF(timeToRun: number | undefined,): Promise<EngineScrapeResult> {if (!meta.options.parsePDF) {- const file = await fetchFileToBuffer(meta.url);+ const file = await fetchFileToBuffer(meta.url, {+ headers: meta.options.headers,+ });const content = file.buffer.toString("base64");return {url: file.response.url,@@ -85,7 +87,9 @@ export async function scrapePDF(};}- const { response, tempFilePath } = await downloadFile(meta.id, meta.url);+ const { response, tempFilePath } = await downloadFile(meta.id, meta.url, {+ headers: meta.options.headers,+ });let result: PDFProcessorResult | null = null;diff --git a/apps/api/src/scraper/scrapeURL/engines/utils/downloadFile.ts b/apps/api/src/scraper/scrapeURL/engines/utils/downloadFile.tsindex d4932f34b..55be08c32 100644--- a/apps/api/src/scraper/scrapeURL/engines/utils/downloadFile.ts+++ b/apps/api/src/scraper/scrapeURL/engines/utils/downloadFile.ts@@ -7,11 +7,11 @@ import { v4 as uuid } from "uuid";import * as undici from "undici";import { makeSecureDispatcher } from "./safeFetch";-export async function fetchFileToBuffer(url: string): Promise<{- response: Response;+export async function fetchFileToBuffer(url: string, init?: undici.RequestInit): Promise<{+ response: undici.Response;buffer: Buffer;}> {- const response = await fetch(url); // TODO: maybe we could use tlsclient for this? for proxying+ const response = await undici.fetch(url, { ...init, redirect: "follow", dispatcher: await makeSecureDispatcher(url) });return {response,buffer: Buffer.from(await response.arrayBuffer()),@@ -21,6 +21,7 @@ export async function fetchFileToBuffer(url: string): Promise<{export async function downloadFile(id: string,url: string,+ init?: undici.RequestInit,): Promise<{response: undici.Response;tempFilePath: string;@@ -29,7 +30,7 @@ export async function downloadFile(const tempFileWrite = createWriteStream(tempFilePath);// TODO: maybe we could use tlsclient for this? for proxying- const response = await undici.fetch(url, { dispatcher: await makeSecureDispatcher(url) });+ const response = await undici.fetch(url, { ...init, redirect: "follow", dispatcher: await makeSecureDispatcher(url) });// This should never happen in the current state of JS/Undici (2024), but let's check anyways.if (response.body === null) {