Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

Update index.ts

Eric Ciarlaed7d15d2af23
diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts
index 24d5f002a..38dc6b5ee 100644
--- a/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts
@@ -140,9 +140,23 @@ export async function scrapePDF(meta: Meta, timeToRun: number | undefined): Prom
const { response, tempFilePath } = await downloadFile(meta.id, meta.url);
let result: PDFProcessorResult | null = null;
- if (process.env.LLAMAPARSE_API_KEY) {
+
+ // First, try parsing with PdfParse
+ result = await scrapePDFWithParsePDF(
+ {
+ ...meta,
+ logger: meta.logger.child({
+ method: "scrapePDF/scrapePDFWithParsePDF",
+ }),
+ },
+ tempFilePath,
+ );
+
+
+ // If the parsed text is under 500 characters and LLAMAPARSE_API_KEY exists, try LlamaParse
+ if (result.markdown && result.markdown.length < 500 && process.env.LLAMAPARSE_API_KEY) {
try {
- result = await scrapePDFWithLlamaParse(
+ const llamaResult = await scrapePDFWithLlamaParse(
{
...meta,
logger: meta.logger.child({
@@ -152,16 +166,17 @@ export async function scrapePDF(meta: Meta, timeToRun: number | undefined): Prom
tempFilePath,
timeToRun,
);
+ result = llamaResult; // Use LlamaParse result if successful
} catch (error) {
if (error instanceof Error && error.message === "LlamaParse timed out") {
- meta.logger.warn("LlamaParse timed out -- falling back to parse-pdf", {
+ meta.logger.warn("LlamaParse timed out -- using parse-pdf result", {
error,
});
} else if (error instanceof RemoveFeatureError) {
throw error;
} else {
meta.logger.warn(
- "LlamaParse failed to parse PDF -- falling back to parse-pdf",
+ "LlamaParse failed to parse PDF -- using parse-pdf result",
{ error },
);
Sentry.captureException(error);
@@ -169,18 +184,6 @@ export async function scrapePDF(meta: Meta, timeToRun: number | undefined): Prom
}
}
- if (result === null) {
- result = await scrapePDFWithParsePDF(
- {
- ...meta,
- logger: meta.logger.child({
- method: "scrapePDF/scrapePDFWithParsePDF",
- }),
- },
- tempFilePath,
- );
- }
-
await fs.unlink(tempFilePath);
return {