Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
add pdf prefetch log for debugging (ENG-2542) (#1734)
Gergő Móricz13f012c583cb
diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/index.tsindex 1135264fa..7d14bd0dc 100644--- a/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts@@ -10,6 +10,7 @@ import { downloadFile, fetchFileToBuffer } from "../utils/downloadFile";import {PDFAntibotError,PDFInsufficientTimeError,+ PDFPrefetchFailed,RemoveFeatureError,TimeoutError,} from "../../error";@@ -194,7 +195,11 @@ export async function scrapePDF(const ct = file.response.headers.get("Content-Type");if (ct && !ct.includes("application/pdf")) {// if downloaded file wasn't a PDF- throw new PDFAntibotError();+ if (meta.pdfPrefetch === undefined) {+ throw new PDFAntibotError();+ } else {+ throw new PDFPrefetchFailed();+ }}const content = file.buffer.toString("base64");@@ -223,7 +228,11 @@ export async function scrapePDF(const ct = r.headers.get("Content-Type");if (ct && !ct.includes("application/pdf")) {// if downloaded file wasn't a PDF- throw new PDFAntibotError();+ if (meta.pdfPrefetch === undefined) {+ throw new PDFAntibotError();+ } else {+ throw new PDFPrefetchFailed();+ }}}diff --git a/apps/api/src/scraper/scrapeURL/engines/utils/specialtyHandler.ts b/apps/api/src/scraper/scrapeURL/engines/utils/specialtyHandler.tsindex 02d04aca3..c67581cd4 100644--- a/apps/api/src/scraper/scrapeURL/engines/utils/specialtyHandler.ts+++ b/apps/api/src/scraper/scrapeURL/engines/utils/specialtyHandler.ts@@ -6,8 +6,9 @@ import os from "os";import { writeFile } from "fs/promises";import { Meta } from "../..";-async function feResToPdfPrefetch(feRes: FireEngineCheckStatusSuccess | undefined): Promise<Meta["pdfPrefetch"]> {+async function feResToPdfPrefetch(logger: Logger, feRes: FireEngineCheckStatusSuccess | undefined): Promise<Meta["pdfPrefetch"]> {if (!feRes?.file) {+ logger.warn("No file in pdf prefetch");return null;}@@ -40,7 +41,7 @@ export async function specialtyScrapeCheck(contentType.startsWith("application/pdf;")) {- throw new AddFeatureError(["pdf"], await feResToPdfPrefetch(feRes));+ throw new AddFeatureError(["pdf"], await feResToPdfPrefetch(logger, feRes));} else if (contentType ==="application/vnd.openxmlformats-officedocument.wordprocessingml.document" ||diff --git a/apps/api/src/scraper/scrapeURL/error.ts b/apps/api/src/scraper/scrapeURL/error.tsindex f139cfbd2..df3dd05e0 100644--- a/apps/api/src/scraper/scrapeURL/error.ts+++ b/apps/api/src/scraper/scrapeURL/error.ts@@ -109,3 +109,9 @@ export class IndexMissError extends Error {super("Index doesn't have the page we're looking for");}}++export class PDFPrefetchFailed extends Error {+ constructor() {+ super("Failed to prefetch PDF that is protected by anti-bot. Please contact [email protected]");+ }+}diff --git a/apps/api/src/scraper/scrapeURL/index.ts b/apps/api/src/scraper/scrapeURL/index.tsindex 31f65617b..e0f74ef08 100644--- a/apps/api/src/scraper/scrapeURL/index.ts+++ b/apps/api/src/scraper/scrapeURL/index.ts@@ -25,6 +25,7 @@ import {PDFInsufficientTimeError,IndexMissError,DNSResolutionError,+ PDFPrefetchFailed,} from "./error";import { executeTransformers } from "./transformers";import { LLMRefusalError } from "./transformers/llmExtract";@@ -550,6 +551,8 @@ export async function scrapeURL(});} else if (error instanceof PDFInsufficientTimeError) {meta.logger.warn("scrapeURL: Insufficient time to process PDF", { error });+ } else if (error instanceof PDFPrefetchFailed) {+ meta.logger.warn("scrapeURL: Failed to prefetch PDF that is protected by anti-bot", { error });} else if (error instanceof TimeoutSignal) {throw error;} else {