Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

8.3 KB
Raw Blame
import { Meta } from "../..";
import { config } from "../../../../config";
import { robustFetch } from "../../lib/fetch";
import { safeMarkdownToHtml } from "./markdownToHtml";
import { z } from "zod";
import path from "node:path";
import {
  getPdfResultFromCache,
  pdfCacheConfigured,
  resolvePdfCacheKey,
  savePdfResultToCache,
} from "../../../../lib/gcs-pdf-cache";
import type { PDFProcessorResult } from "./types";
import { getPDFRefresh } from "../../../../controllers/v2/types";
import { consumeRefresh } from "./fire-pdf/refresh-budget";

export async function scrapePDFWithRunPodMU(
  meta: Meta,
  tempFilePath: string,
  base64Content: string,
  maxPages?: number,
  pagesProcessed?: number,
): Promise<PDFProcessorResult> {
  meta.logger.debug("Processing PDF document with RunPod MU", {
    tempFilePath,
  });

  // The cache key (sha256 of the payload, a full pass over the document) is
  // computed at most once per request and reused for the read, the write
  // and the log lines. This cache lives under the legacy `pdf-cache-v2/`
  // prefix; the key is logged so a report can name the entry.
  let cacheKeyMemo: string | undefined;
  const cacheKeyOf = (): string =>
    (cacheKeyMemo ??= resolvePdfCacheKey(base64Content));

  // Without a bucket there is no cache: nothing is hashed and no refresh
  // budget is spent. Zero-data-retention requests never read the shared
  // cache, as on the fire-pdf path. `parsers: [{ type: "pdf", refresh: true }]`
  // skips it too, under the same per-team budget: a MinerU-diverted request
  // never reaches the fire-pdf path, so the budget has to be applied here as
  // well, and the decision is taken once per request, so a fire-pdf → MU
  // fallback keeps the decision the first engine got instead of spending a
  // second token. Over budget, or with the limiter unavailable, the entry is
  // served.
  const cacheOn = pdfCacheConfigured();
  if (cacheOn && !maxPages && !meta.internalOptions.zeroDataRetention) {
    const cacheKey = cacheKeyOf();
    let bypass = false;
    if (getPDFRefresh(meta.options?.parsers)) {
      const decision = await consumeRefresh(
        meta.internalOptions.teamId,
        meta.id,
      );
      bypass = decision === "allowed";
      if (bypass) {
        meta.logger.info("RunPod MU cache bypassed by refresh", {
          tempFilePath,
          cacheKey,
          cacheProvider: "runpod",
        });
      } else {
        meta.logger.warn("RunPod MU cache refresh not applied", {
          tempFilePath,
          cacheKey,
          cacheProvider: "runpod",
          decision,
          perMinute: config.FIRE_PDF_CACHE_REFRESH_PER_MINUTE,
        });
      }
    }
    if (!bypass) {
      try {
        const cachedResult = await getPdfResultFromCache({ key: cacheKey });
        if (cachedResult) {
          meta.logger.info("Using cached RunPod MU result for PDF", {
            tempFilePath,
            cacheKey,
            cacheProvider: "runpod",
          });
          return cachedResult;
        }
      } catch (error) {
        meta.logger.warn(
          "Error checking PDF cache, proceeding with RunPod MU",
          { error, tempFilePath, cacheKey, cacheProvider: "runpod" },
        );
      }
    }
  }

  meta.abort.throwIfAborted();

  meta.logger.info("Max Pdf pages", {
    tempFilePath,
    maxPages,
  });

  if (
    config.PDF_MU_V2_EXPERIMENT === "true" &&
    config.PDF_MU_V2_BASE_URL &&
    Math.random() * 100 < config.PDF_MU_V2_EXPERIMENT_PERCENT
  ) {
    (async () => {
      const pdfParseId = crypto.randomUUID();
      const startedAt = Date.now();
      const logger = meta.logger.child({ method: "scrapePDF/MUv2Experiment" });
      logger.info("MU v2 experiment started", {
        scrapeId: meta.id,
        pdfParseId,
        url: meta.rewrittenUrl ?? meta.url,
        maxPages,
      });
      try {
        const resp = await robustFetch({
          url: config.PDF_MU_V2_BASE_URL ?? "",
          method: "POST",
          headers: config.PDF_MU_V2_API_KEY
            ? { Authorization: `Bearer ${config.PDF_MU_V2_API_KEY}` }
            : undefined,
          body: {
            input: {
              file_content: base64Content,
              filename: path.basename(tempFilePath) + ".pdf",
              timeout: meta.abort.scrapeTimeout(),
              created_at: Date.now(),
              id: pdfParseId,
              ...(maxPages !== undefined && { max_pages: maxPages }),
            },
          },
          logger,
          schema: z.any(),
          mock: meta.mock,
          abort: meta.abort.asSignal(),
        });
        const body: any = resp as any;
        const tokensIn = body?.metadata?.["total-input-tokens"];
        const tokensOut = body?.metadata?.["total-output-tokens"];
        const pages = body?.metadata?.["pdf-total-pages"];
        const durationMs = Date.now() - startedAt;
        logger.info("MU v2 experiment completed", {
          durationMs,
          url: meta.rewrittenUrl ?? meta.url,
          tokensIn,
          tokensOut,
          pages,
        });
      } catch (error) {
        const durationMs = Date.now() - startedAt;
        logger.warn("MU v2 experiment failed", { error, durationMs });
      }
    })();
  }

  const muV1StartedAt = Date.now();
  const podStart = await robustFetch({
    url: "https://api.runpod.ai/v2/" + config.RUNPOD_MU_POD_ID + "/runsync",
    method: "POST",
    headers: {
      Authorization: `Bearer ${config.RUNPOD_MU_API_KEY}`,
    },
    body: {
      input: {
        file_content: base64Content,
        filename: path.basename(tempFilePath) + ".pdf",
        timeout: meta.abort.scrapeTimeout(),
        created_at: Date.now(),
        ...(maxPages !== undefined && { max_pages: maxPages }),
      },
    },
    logger: meta.logger.child({
      method: "scrapePDFWithRunPodMU/runsync/robustFetch",
    }),
    schema: z.object({
      id: z.string(),
      status: z.string(),
      output: z
        .object({
          markdown: z.string(),
        })
        .optional(),
    }),
    mock: meta.mock,
    abort: meta.abort.asSignal(),
  });

  let status: string = podStart.status;
  let result: { markdown: string } | undefined = podStart.output;

  if (status === "IN_QUEUE" || status === "IN_PROGRESS") {
    do {
      meta.abort.throwIfAborted();
      await new Promise(resolve => setTimeout(resolve, 2500));
      meta.abort.throwIfAborted();
      const podStatus = await robustFetch({
        url: `https://api.runpod.ai/v2/${config.RUNPOD_MU_POD_ID}/status/${podStart.id}`,
        method: "GET",
        headers: {
          Authorization: `Bearer ${config.RUNPOD_MU_API_KEY}`,
        },
        logger: meta.logger.child({
          method: "scrapePDFWithRunPodMU/status/robustFetch",
        }),
        schema: z.object({
          status: z.string(),
          output: z
            .object({
              markdown: z.string(),
            })
            .optional(),
        }),
        mock: meta.mock,
        abort: meta.abort.asSignal(),
      });
      status = podStatus.status;
      result = podStatus.output;
    } while (status !== "COMPLETED" && status !== "FAILED");
  }

  if (status === "FAILED") {
    const durationMs = Date.now() - muV1StartedAt;
    meta.logger.child({ method: "scrapePDF/MUv1" }).warn("MU v1 failed", {
      durationMs,
      url: meta.rewrittenUrl ?? meta.url,
      pagesProcessed,
    });
    throw new Error("RunPod MU failed to parse PDF");
  }

  if (!result) {
    const durationMs = Date.now() - muV1StartedAt;
    meta.logger.child({ method: "scrapePDF/MUv1" }).warn("MU v1 failed", {
      durationMs,
      url: meta.rewrittenUrl ?? meta.url,
      pagesProcessed,
    });
    throw new Error("RunPod MU returned no result");
  }

  const processorResult = {
    markdown: result.markdown,
    html: await safeMarkdownToHtml(result.markdown, meta.logger, meta.id),
  };

  if (cacheOn && !meta.internalOptions.zeroDataRetention) {
    try {
      await savePdfResultToCache({ key: cacheKeyOf() }, processorResult);
    } catch (error) {
      meta.logger.warn("Error saving PDF to cache", {
        error,
        tempFilePath,
      });
    }
  }

  {
    const durationMs = Date.now() - muV1StartedAt;
    meta.logger.child({ method: "scrapePDF/MUv1" }).info("MU v1 completed", {
      durationMs,
      url: meta.rewrittenUrl ?? meta.url,
      pagesProcessed,
    });
  }

  return processorResult;
}