Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(scrapeURL/pdf): fix llamaparse upload
Gergő Móricz9ace2ad071e9
diff --git a/apps/api/package.json b/apps/api/package.jsonindex bb4ea268b..aebd90a52 100644--- a/apps/api/package.json+++ b/apps/api/package.json@@ -81,7 +81,6 @@"escape-html": "^1.0.3","express-rate-limit": "^7.3.1","express-ws": "^5.0.2",- "form-data": "^4.0.0","glob": "^10.4.2","gpt3-tokenizer": "^1.1.5","ioredis": "^5.4.1",diff --git a/apps/api/pnpm-lock.yaml b/apps/api/pnpm-lock.yamlindex 3350c74ea..f98055fba 100644--- a/apps/api/pnpm-lock.yaml+++ b/apps/api/pnpm-lock.yaml@@ -101,9 +101,6 @@ importers:express-ws:specifier: ^5.0.2version: 5.0.2([email protected])- form-data:- specifier: ^4.0.0- version: 4.0.0glob:specifier: ^10.4.2version: 10.4.2@@ -3932,8 +3929,8 @@ packages:engines: {node: '>=14.17'}hasBin: true- resolution: {integrity: sha512-NW8ByodCSNCwZeghjN3o+JX5OFH0Ojg6sadjEKY4huZ52TqbJTJnDo5+Tw98lSy63NZvi4n+ez5m2u5d4PkZyw==}+ resolution: {integrity: sha512-hjcS1mhfuyi4WW8IWtjP7brDrG2cuDZukyrYrSauoXGNgx0S7zceP07adYkJycEr56BOUTNPzbInooiN3fn1qw==}engines: {node: '>=14.17'}hasBin: true@@ -7742,7 +7739,7 @@ snapshots:csv-parse: 5.5.6gpt3-tokenizer: 1.1.5openai: 3.3.0- typescript: 5.6.2+ typescript: 5.6.3uuid: 9.0.1zod: 3.23.8transitivePeerDependencies:@@ -8320,7 +8317,7 @@ snapshots:- [email protected]: {}+ [email protected]: {}[email protected](@babel/[email protected]):dependencies:diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/index.tsindex ea44b0510..8b42ee716 100644--- a/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts@@ -1,5 +1,4 @@import { createReadStream, promises as fs } from "node:fs";-import FormData from "form-data";import { Meta } from "../..";import { EngineScrapeResult } from "..";import * as marked from "marked";@@ -16,10 +15,26 @@ async function scrapePDFWithLlamaParse(meta: Meta, tempFilePath: string): Promismeta.logger.debug("Processing PDF document with LlamaIndex", { tempFilePath });const uploadForm = new FormData();- uploadForm.append("file", createReadStream(tempFilePath), {- filename: tempFilePath,- contentType: "application/pdf", // NOTE: request.headers["Content-Type"]?- });++ // This is utterly stupid but it works! - mogery+ uploadForm.append("file", {+ [Symbol.toStringTag]: "Blob",+ name: tempFilePath,+ stream() {+ return createReadStream(tempFilePath) as unknown as ReadableStream<Uint8Array>+ },+ arrayBuffer() {+ throw Error("Unimplemented in mock Blob: arrayBuffer")+ },+ size: (await fs.stat(tempFilePath)).size,+ text() {+ throw Error("Unimplemented in mock Blob: text")+ },+ slice(start, end, contentType) {+ throw Error("Unimplemented in mock Blob: slice")+ },+ type: "application/pdf",+ } as Blob);const upload = await robustFetch({url: "https://api.cloud.llamaindex.ai/api/parsing/upload",diff --git a/apps/api/src/scraper/scrapeURL/lib/fetch.ts b/apps/api/src/scraper/scrapeURL/lib/fetch.tsindex 03bbd80cf..738e240e2 100644--- a/apps/api/src/scraper/scrapeURL/lib/fetch.ts+++ b/apps/api/src/scraper/scrapeURL/lib/fetch.ts@@ -2,7 +2,6 @@ import { Logger } from "winston";import { z, ZodError } from "zod";import { v4 as uuid } from "uuid";import * as Sentry from "@sentry/node";-import FormData from "form-data";export type RobustFetchParams<Schema extends z.Schema<any>> = {url: string;@@ -38,14 +37,14 @@ export async function robustFetch<Schema extends z.Schema<any>, Output = z.infermethod,headers: {...(body instanceof FormData- ? body.getHeaders()+ ? ({}): body !== undefined ? ({"Content-Type": "application/json",}) : {}),...(headers !== undefined ? headers : {}),},...(body instanceof FormData ? ({- body: body.getBuffer(),+ body,}) : body !== undefined ? ({body: JSON.stringify(body),}) : {}),