Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(pdf): Track MU Errors (#2894)
Abimael Martell66da13b7f148
diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/index.tsindex 3d5b4e22d..636e7ba54 100644--- a/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts@@ -28,6 +28,7 @@ import {getPDFMaxPages,} from "../../../../controllers/v2/types";import { getPdfMetadata } from "@mendable/firecrawl-rs";+import { captureExceptionWithZdrCheck } from "../../../../services/sentry";type PDFProcessorResult = { html: string; markdown?: string };@@ -147,9 +148,11 @@ async function scrapePDFWithRunPodMU(schema: z.object({id: z.string(),status: z.string(),+ error: z.string().optional(),output: z.object({- markdown: z.string(),+ markdown: z.string().optional(),+ status: z.string().optional(),}).optional(),}),@@ -158,7 +161,11 @@ async function scrapePDFWithRunPodMU(});let status: string = podStart.status;- let result: { markdown: string } | undefined = podStart.output;+ let result: { markdown?: string } | undefined =+ podStart.output?.markdown !== undefined+ ? { markdown: podStart.output.markdown }+ : undefined;+ let lastRunPodError: string | undefined = podStart.error;if (status === "IN_QUEUE" || status === "IN_PROGRESS") {do {@@ -176,9 +183,11 @@ async function scrapePDFWithRunPodMU(}),schema: z.object({status: z.string(),+ error: z.string().optional(),output: z.object({- markdown: z.string(),+ markdown: z.string().optional(),+ status: z.string().optional(),}).optional(),}),@@ -186,22 +195,43 @@ async function scrapePDFWithRunPodMU(abort: meta.abort.asSignal(),});status = podStatus.status;- result = podStatus.output;+ result =+ podStatus.output?.markdown !== undefined+ ? { markdown: podStatus.output.markdown }+ : undefined;+ if (podStatus.error !== undefined) {+ lastRunPodError = podStatus.error;+ }} while (status !== "COMPLETED" && status !== "FAILED");}if (status === "FAILED") {const durationMs = Date.now() - muV1StartedAt;+ const errorMessage =+ lastRunPodError ?? "RunPod MU failed to parse PDF (no error message)";+ const err = new Error(`RunPod MU failed to parse PDF: ${errorMessage}`);meta.logger.child({ method: "scrapePDF/MUv1" }).warn("MU v1 failed", {durationMs,url: meta.rewrittenUrl ?? meta.url,mu_id: podStart.id,+ runpodError: lastRunPodError,...(pagesProcessed !== undefined && { pagesProcessed }),});- throw new Error("RunPod MU failed to parse PDF");+ captureExceptionWithZdrCheck(err, {+ extra: {+ zeroDataRetention: meta.internalOptions.zeroDataRetention ?? false,+ scrapeId: meta.id,+ teamId: meta.internalOptions.teamId,+ url: meta.rewrittenUrl ?? meta.url,+ runpodId: podStart.id,+ runpodError: lastRunPodError,+ durationMs,+ },+ });+ throw err;}- if (!result) {+ if (!result?.markdown) {const durationMs = Date.now() - muV1StartedAt;meta.logger.child({ method: "scrapePDF/MUv1" }).warn("MU v1 failed", {durationMs,@@ -215,7 +245,7 @@ async function scrapePDFWithRunPodMU(const processorResult = {markdown: result.markdown,html: await marked.parse(result.markdown, { async: true }),- };+ } satisfies PDFProcessorResult;if (!meta.internalOptions.zeroDataRetention) {try {