Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(pdf-parser): get PDF title and show in metadata (#2062)
Gergő Móriczb3607eb18941
diff --git a/apps/api/sharedLibs/pdf-parser/Cargo.lock b/apps/api/sharedLibs/pdf-parser/Cargo.lockindex 6509899bf..5635c68aa 100644--- a/apps/api/sharedLibs/pdf-parser/Cargo.lock+++ b/apps/api/sharedLibs/pdf-parser/Cargo.lock@@ -483,6 +483,8 @@ version = "0.1.0"dependencies = ["libc","lopdf",+ "serde",+ "serde_json",][[package]]@@ -600,6 +602,12 @@ version = "1.0.21"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "8a0d197bd2c9dc6e53b84da9556a69ba4cdfab8619eb41a8bd1cc2027a0f6b1d"+[[package]]+name = "ryu"+version = "1.0.20"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "28d3b2b1366ec20994f1fd18c3c594f05c5dd4bc44d8bb0c1c632c8d6829481f"+[[package]]name = "serde"version = "1.0.219"@@ -620,6 +628,18 @@ dependencies = ["syn",]+[[package]]+name = "serde_json"+version = "1.0.143"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "d401abef1d108fbd9cbaebc3e46611f4b1021f714a0597a71f41ee463f5f4a5a"+dependencies = [+ "itoa",+ "memchr",+ "ryu",+ "serde",+]+[[package]]name = "sha2"version = "0.10.9"diff --git a/apps/api/sharedLibs/pdf-parser/Cargo.toml b/apps/api/sharedLibs/pdf-parser/Cargo.tomlindex ec2e163a4..7f3945f64 100644--- a/apps/api/sharedLibs/pdf-parser/Cargo.toml+++ b/apps/api/sharedLibs/pdf-parser/Cargo.toml@@ -6,6 +6,8 @@ edition = "2021"[dependencies]libc = "0.2.0"lopdf = "0.36.0"+serde = { version = "1.0", features = ["derive"] }+serde_json = "1.0"[lib]crate-type = ["cdylib"]diff --git a/apps/api/sharedLibs/pdf-parser/src/lib.rs b/apps/api/sharedLibs/pdf-parser/src/lib.rsindex dbe64a065..034723b98 100644--- a/apps/api/sharedLibs/pdf-parser/src/lib.rs+++ b/apps/api/sharedLibs/pdf-parser/src/lib.rs@@ -1,24 +1,67 @@-use std::{ffi::CStr};+use std::ffi::{CStr, CString};+use serde::Serialize;-/// Returns the number of pages in a PDF file+#[derive(Debug, Clone, Serialize)]+#[serde(rename_all = "camelCase")]+pub struct PDFMetadata {+ num_pages: i32,++ #[serde(skip_serializing_if = "Option::is_none")]+ title: Option<String>,+}++fn _get_pdf_metadata(path: &str) -> Result<PDFMetadata, String> {+ let doc = match lopdf::Document::load(path) {+ Ok(x) => x,+ Err(_) => {+ return Err("Failed to load PDF".to_string());+ }+ };++ let num_pages = doc.get_pages().len() as i32;+ let title = doc.trailer.get(b"Info")+ .and_then(|info| info.as_dict()+ .and_then(|info| info.get(b"Title"))+ .and_then(|title| title.as_str().map(|s| String::from_utf8_lossy(s).to_string()))+ ).ok()+ .or_else(|| doc.objects.iter()+ .find_map(|(_i, obj)| obj.as_dict()+ .and_then(|obj| obj.get(b"Title"))+ .and_then(|title| title.as_str())+ .map(|s| String::from_utf8_lossy(s).to_string())+ .ok()+ )+ );++ Ok(PDFMetadata { num_pages, title })+}++/// Returns the metadata of a PDF file////// # Safety-/// Input path must be a C string of a path pointing to a PDF file. Output will be an integer, either the number of pages in the PDF or -1 indicating an error.+/// Input path must be a C string of a path pointing to a PDF file. Output will be a JSON string of the PDF metadata.#[no_mangle]-pub unsafe extern "C" fn get_page_count(path: *const libc::c_char) -> i32 {+pub unsafe extern "C" fn get_pdf_metadata(path: *const libc::c_char) -> *const libc::c_char {let path: String = match unsafe { CStr::from_ptr(path) }.to_str().map_err(|_| ()) {Ok(x) => x.to_string(),Err(_) => {- return -1;+ return CString::new("RUSTFC:ERROR:Failed to parse input path as C string").unwrap().into_raw();}};- let doc = match lopdf::Document::load(&path) {+ let metadata = match _get_pdf_metadata(&path) {Ok(x) => x,- Err(_) => {- return -1;+ Err(e) => {+ return CString::new(format!("RUSTFC:ERROR:Failed to get PDF metadata: {}", e)).unwrap().into_raw();+ }+ };++ let json = match serde_json::to_string(&metadata) {+ Ok(x) => x,+ Err(e) => {+ return CString::new(format!("RUSTFC:ERROR:Serde failed to serialize metadata: {}", e)).unwrap().into_raw();}};- doc.get_pages().len() as i32-}\ No newline at end of file+ CString::new(json).unwrap().into_raw()+}diff --git a/apps/api/src/__tests__/snips/v1/scrape.test.ts b/apps/api/src/__tests__/snips/v1/scrape.test.tsindex 0b4d590c9..febea502b 100644--- a/apps/api/src/__tests__/snips/v1/scrape.test.ts+++ b/apps/api/src/__tests__/snips/v1/scrape.test.ts@@ -889,6 +889,17 @@ describe("Scrape tests", () => {});describe("PDF (f-e dependant)", () => {+ it.concurrent("works", async () => {+ const response = await scrape({+ url: "https://www.orimi.com/pdf-test.pdf",+ timeout: scrapeTimeout * 2,+ }, identity);++ expect(response.markdown).toContain("PDF Test File");+ expect(response.metadata.title).toBe("PDF Test Page");+ expect(response.metadata.numPages).toBe(1);+ }, scrapeTimeout * 2);+// Temporarily disabled, too flaky// it.concurrent("works for PDFs behind anti-bot", async () => {// const response = await scrape({diff --git a/apps/api/src/__tests__/snips/v2/scrape.test.ts b/apps/api/src/__tests__/snips/v2/scrape.test.tsindex b811b0cc3..dc0b73596 100644--- a/apps/api/src/__tests__/snips/v2/scrape.test.ts+++ b/apps/api/src/__tests__/snips/v2/scrape.test.ts@@ -791,6 +791,17 @@ describe("Scrape tests", () => {});describe("PDF (f-e dependant)", () => {+ it.concurrent("works", async () => {+ const response = await scrape({+ url: "https://www.orimi.com/pdf-test.pdf",+ maxAge: 0,+ }, identity);++ expect(response.markdown).toContain("PDF Test File");+ expect(response.metadata.title).toBe("PDF Test Page");+ expect(response.metadata.numPages).toBe(1);+ }, scrapeTimeout);+// Temporarily disabled, too flaky// it.concurrent("works for PDFs behind anti-bot", async () => {// const response = await scrape({diff --git a/apps/api/src/lib/pdf-parser.ts b/apps/api/src/lib/pdf-parser.tsindex 524eb9d04..0c24637d6 100644--- a/apps/api/src/lib/pdf-parser.ts+++ b/apps/api/src/lib/pdf-parser.ts@@ -10,13 +10,18 @@ const rustExecutablePath = join(platform() === "darwin" ? "libpdf_parser.dylib" : "libpdf_parser.so");+export type PDFMetadata = {+ numPages: number;+ title?: string;+}+class RustPDFParser {private static instance: RustPDFParser;- private _getPageCount: KoffiFunction;+ private _getPDFMetadata: KoffiFunction;private constructor() {const lib = koffi.load(rustExecutablePath);- this._getPageCount = lib.func("get_page_count", "int32", ["string"]);+ this._getPDFMetadata = lib.func("get_pdf_metadata", "string", ["string"]);}public static async isParserAvailable(): Promise<boolean> {@@ -45,16 +50,21 @@ class RustPDFParser {return RustPDFParser.instance;}- public async getPageCount(path: string): Promise<number> {- return new Promise<number>((resolve, reject) => {- this._getPageCount.async(path, (err: Error, res: number) => {+ public async getPDFMetadata(path: string): Promise<PDFMetadata> {+ return new Promise<PDFMetadata>((resolve, reject) => {+ this._getPDFMetadata.async(path, (err: Error, res: string) => {if (err) {reject(err);} else {- if (res === -1) {- reject(new Error("Failed to parse PDF."));+ if (res.startsWith("RUSTFC:ERROR:")) {+ reject(new Error(res.replace("RUSTFC:ERROR:", "")));} else {- resolve(res);+ try {+ const metadata = JSON.parse(res) as PDFMetadata;+ resolve(metadata);+ } catch (e) {+ reject(new Error("Failed to parse PDF metadata."));+ }}}});@@ -62,9 +72,9 @@ class RustPDFParser {}}-export async function getPageCount(+export async function getPDFMetadata(path: string,-): Promise<number> {+): Promise<PDFMetadata> {const converter = await RustPDFParser.getInstance();- return await converter.getPageCount(path);+ return await converter.getPDFMetadata(path);}diff --git a/apps/api/src/scraper/scrapeURL/engines/index.ts b/apps/api/src/scraper/scrapeURL/engines/index.tsindex 4addc40b6..cc2529479 100644--- a/apps/api/src/scraper/scrapeURL/engines/index.ts+++ b/apps/api/src/scraper/scrapeURL/engines/index.ts@@ -13,6 +13,7 @@ import { playwrightMaxReasonableTime, scrapeURLWithPlaywright } from "./playwrigimport { indexMaxReasonableTime, scrapeURLWithIndex } from "./index/index";import { useIndex } from "../../../services";import { hasFormatOfType } from "../../../lib/format-utils";+import { PDFMetadata } from "../../../lib/pdf-parser";export type Engine =| "fire-engine;chrome-cdp"@@ -114,7 +115,7 @@ export type EngineScrapeResult = {pdfs: string[];};- numPages?: number;+ pdfMetadata?: PDFMetadata;cacheInfo?: {created_at: Date;diff --git a/apps/api/src/scraper/scrapeURL/engines/index/index.ts b/apps/api/src/scraper/scrapeURL/engines/index/index.tsindex 740dda874..b076da894 100644--- a/apps/api/src/scraper/scrapeURL/engines/index/index.ts+++ b/apps/api/src/scraper/scrapeURL/engines/index/index.ts@@ -53,7 +53,10 @@ export async function sendDocumentToIndex(meta: Meta, document: Document) {statusCode: document.metadata.statusCode,error: document.metadata.error,screenshot: document.screenshot,- numPages: document.metadata.numPages,+ pdfMetadata: document.metadata.numPages !== undefined ? { // reconstruct pdfMetadata from numPages and title+ numPages: document.metadata.numPages,+ title: document.metadata.title ?? undefined,+ } : undefined,contentType: document.metadata.contentType,});} catch (error) {@@ -267,7 +270,9 @@ export async function scrapeURLWithIndex(meta: Meta): Promise<EngineScrapeResultstatusCode: doc.statusCode,error: doc.error,screenshot: doc.screenshot,- numPages: doc.numPages,+ pdfMetadata: doc.pdfMetadata ?? (doc.numPages !== undefined ? { // backwards-compatible shim of pdfMetadata without title+ numPages: doc.numPages+ } : undefined),contentType: doc.contentType,cacheInfo: {diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/index.tsindex 310544eb8..4f726e62c 100644--- a/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts@@ -16,7 +16,7 @@ import {import { readFile, unlink } from "node:fs/promises";import path from "node:path";import type { Response } from "undici";-import { getPageCount } from "../../../../lib/pdf-parser";+import { getPDFMetadata } from "../../../../lib/pdf-parser";import { getPdfResultFromCache, savePdfResultToCache } from "../../../../lib/gcs-pdf-cache";import { AbortManagerThrownError } from "../../lib/abortManager";@@ -227,11 +227,12 @@ export async function scrapePDF(}}- const pageCount = await getPageCount(tempFilePath);- if (pageCount * MILLISECONDS_PER_PAGE > (meta.abort.scrapeTimeout() ?? Infinity)) {+ const pdfMetadata = await getPDFMetadata(tempFilePath);++ if (pdfMetadata.numPages * MILLISECONDS_PER_PAGE > (meta.abort.scrapeTimeout() ?? Infinity)) {throw new PDFInsufficientTimeError(- pageCount,- pageCount * MILLISECONDS_PER_PAGE + 5000,+ pdfMetadata.numPages,+ pdfMetadata.numPages * MILLISECONDS_PER_PAGE + 5000,);}@@ -291,7 +292,7 @@ export async function scrapePDF(statusCode: response.status,html: result?.html ?? "",markdown: result?.markdown ?? "",- numPages: pageCount,+ pdfMetadata,proxyUsed: "basic",};diff --git a/apps/api/src/scraper/scrapeURL/index.ts b/apps/api/src/scraper/scrapeURL/index.tsindex 6f811a049..f16aea173 100644--- a/apps/api/src/scraper/scrapeURL/index.ts+++ b/apps/api/src/scraper/scrapeURL/index.ts@@ -510,7 +510,8 @@ async function scrapeURLLoop(meta: Meta): Promise<ScrapeUrlResponse> {url: result.result.url,statusCode: result.result.statusCode,error: result.result.error,- numPages: result.result.numPages,+ numPages: result.result.pdfMetadata?.numPages,+ title: result.result.pdfMetadata?.title,contentType: result.result.contentType,proxyUsed: meta.featureFlags.has("stealthProxy") ? "stealth" : "basic",...(fallbackList.find(x => ["index", "index;documents"].includes(x.engine)) ? (diff --git a/apps/api/src/services/index.ts b/apps/api/src/services/index.tsindex 4ede28992..a846e0736 100644--- a/apps/api/src/services/index.ts+++ b/apps/api/src/services/index.ts@@ -7,6 +7,7 @@ import { redisEvictConnection } from "./redis";import type { Logger } from "winston";import psl from "psl";import { MapDocument } from "../controllers/v2/types";+import { PDFMetadata } from "../lib/pdf-parser";configDotenv();// SupabaseService class initializes the Supabase client conditionally based on environment variables.@@ -97,7 +98,7 @@ export async function saveIndexToGCS(id: string, doc: {statusCode: number;error?: string;screenshot?: string;- numPages?: number;+ pdfMetadata?: PDFMetadata;contentType?: string;}): Promise<void> {try {