Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

feat(pdf-parser): get PDF title and show in metadata (#2062)

Gergő Móriczb3607eb18941
diff --git a/apps/api/sharedLibs/pdf-parser/Cargo.lock b/apps/api/sharedLibs/pdf-parser/Cargo.lock
index 6509899bf..5635c68aa 100644
--- a/apps/api/sharedLibs/pdf-parser/Cargo.lock
+++ b/apps/api/sharedLibs/pdf-parser/Cargo.lock
@@ -483,6 +483,8 @@ version = "0.1.0"
dependencies = [
"libc",
"lopdf",
+ "serde",
+ "serde_json",
]
[[package]]
@@ -600,6 +602,12 @@ version = "1.0.21"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8a0d197bd2c9dc6e53b84da9556a69ba4cdfab8619eb41a8bd1cc2027a0f6b1d"
+[[package]]
+name = "ryu"
+version = "1.0.20"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "28d3b2b1366ec20994f1fd18c3c594f05c5dd4bc44d8bb0c1c632c8d6829481f"
+
[[package]]
name = "serde"
version = "1.0.219"
@@ -620,6 +628,18 @@ dependencies = [
"syn",
]
+[[package]]
+name = "serde_json"
+version = "1.0.143"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d401abef1d108fbd9cbaebc3e46611f4b1021f714a0597a71f41ee463f5f4a5a"
+dependencies = [
+ "itoa",
+ "memchr",
+ "ryu",
+ "serde",
+]
+
[[package]]
name = "sha2"
version = "0.10.9"
diff --git a/apps/api/sharedLibs/pdf-parser/Cargo.toml b/apps/api/sharedLibs/pdf-parser/Cargo.toml
index ec2e163a4..7f3945f64 100644
--- a/apps/api/sharedLibs/pdf-parser/Cargo.toml
+++ b/apps/api/sharedLibs/pdf-parser/Cargo.toml
@@ -6,6 +6,8 @@ edition = "2021"
[dependencies]
libc = "0.2.0"
lopdf = "0.36.0"
+serde = { version = "1.0", features = ["derive"] }
+serde_json = "1.0"
[lib]
crate-type = ["cdylib"]
diff --git a/apps/api/sharedLibs/pdf-parser/src/lib.rs b/apps/api/sharedLibs/pdf-parser/src/lib.rs
index dbe64a065..034723b98 100644
--- a/apps/api/sharedLibs/pdf-parser/src/lib.rs
+++ b/apps/api/sharedLibs/pdf-parser/src/lib.rs
@@ -1,24 +1,67 @@
-use std::{ffi::CStr};
+use std::ffi::{CStr, CString};
+use serde::Serialize;
-/// Returns the number of pages in a PDF file
+#[derive(Debug, Clone, Serialize)]
+#[serde(rename_all = "camelCase")]
+pub struct PDFMetadata {
+ num_pages: i32,
+
+ #[serde(skip_serializing_if = "Option::is_none")]
+ title: Option<String>,
+}
+
+fn _get_pdf_metadata(path: &str) -> Result<PDFMetadata, String> {
+ let doc = match lopdf::Document::load(path) {
+ Ok(x) => x,
+ Err(_) => {
+ return Err("Failed to load PDF".to_string());
+ }
+ };
+
+ let num_pages = doc.get_pages().len() as i32;
+ let title = doc.trailer.get(b"Info")
+ .and_then(|info| info.as_dict()
+ .and_then(|info| info.get(b"Title"))
+ .and_then(|title| title.as_str().map(|s| String::from_utf8_lossy(s).to_string()))
+ ).ok()
+ .or_else(|| doc.objects.iter()
+ .find_map(|(_i, obj)| obj.as_dict()
+ .and_then(|obj| obj.get(b"Title"))
+ .and_then(|title| title.as_str())
+ .map(|s| String::from_utf8_lossy(s).to_string())
+ .ok()
+ )
+ );
+
+ Ok(PDFMetadata { num_pages, title })
+}
+
+/// Returns the metadata of a PDF file
///
/// # Safety
-/// Input path must be a C string of a path pointing to a PDF file. Output will be an integer, either the number of pages in the PDF or -1 indicating an error.
+/// Input path must be a C string of a path pointing to a PDF file. Output will be a JSON string of the PDF metadata.
#[no_mangle]
-pub unsafe extern "C" fn get_page_count(path: *const libc::c_char) -> i32 {
+pub unsafe extern "C" fn get_pdf_metadata(path: *const libc::c_char) -> *const libc::c_char {
let path: String = match unsafe { CStr::from_ptr(path) }.to_str().map_err(|_| ()) {
Ok(x) => x.to_string(),
Err(_) => {
- return -1;
+ return CString::new("RUSTFC:ERROR:Failed to parse input path as C string").unwrap().into_raw();
}
};
- let doc = match lopdf::Document::load(&path) {
+ let metadata = match _get_pdf_metadata(&path) {
Ok(x) => x,
- Err(_) => {
- return -1;
+ Err(e) => {
+ return CString::new(format!("RUSTFC:ERROR:Failed to get PDF metadata: {}", e)).unwrap().into_raw();
+ }
+ };
+
+ let json = match serde_json::to_string(&metadata) {
+ Ok(x) => x,
+ Err(e) => {
+ return CString::new(format!("RUSTFC:ERROR:Serde failed to serialize metadata: {}", e)).unwrap().into_raw();
}
};
- doc.get_pages().len() as i32
-}
\ No newline at end of file
+ CString::new(json).unwrap().into_raw()
+}
diff --git a/apps/api/src/__tests__/snips/v1/scrape.test.ts b/apps/api/src/__tests__/snips/v1/scrape.test.ts
index 0b4d590c9..febea502b 100644
--- a/apps/api/src/__tests__/snips/v1/scrape.test.ts
+++ b/apps/api/src/__tests__/snips/v1/scrape.test.ts
@@ -889,6 +889,17 @@ describe("Scrape tests", () => {
});
describe("PDF (f-e dependant)", () => {
+ it.concurrent("works", async () => {
+ const response = await scrape({
+ url: "https://www.orimi.com/pdf-test.pdf",
+ timeout: scrapeTimeout * 2,
+ }, identity);
+
+ expect(response.markdown).toContain("PDF Test File");
+ expect(response.metadata.title).toBe("PDF Test Page");
+ expect(response.metadata.numPages).toBe(1);
+ }, scrapeTimeout * 2);
+
// Temporarily disabled, too flaky
// it.concurrent("works for PDFs behind anti-bot", async () => {
// const response = await scrape({
diff --git a/apps/api/src/__tests__/snips/v2/scrape.test.ts b/apps/api/src/__tests__/snips/v2/scrape.test.ts
index b811b0cc3..dc0b73596 100644
--- a/apps/api/src/__tests__/snips/v2/scrape.test.ts
+++ b/apps/api/src/__tests__/snips/v2/scrape.test.ts
@@ -791,6 +791,17 @@ describe("Scrape tests", () => {
});
describe("PDF (f-e dependant)", () => {
+ it.concurrent("works", async () => {
+ const response = await scrape({
+ url: "https://www.orimi.com/pdf-test.pdf",
+ maxAge: 0,
+ }, identity);
+
+ expect(response.markdown).toContain("PDF Test File");
+ expect(response.metadata.title).toBe("PDF Test Page");
+ expect(response.metadata.numPages).toBe(1);
+ }, scrapeTimeout);
+
// Temporarily disabled, too flaky
// it.concurrent("works for PDFs behind anti-bot", async () => {
// const response = await scrape({
diff --git a/apps/api/src/lib/pdf-parser.ts b/apps/api/src/lib/pdf-parser.ts
index 524eb9d04..0c24637d6 100644
--- a/apps/api/src/lib/pdf-parser.ts
+++ b/apps/api/src/lib/pdf-parser.ts
@@ -10,13 +10,18 @@ const rustExecutablePath = join(
platform() === "darwin" ? "libpdf_parser.dylib" : "libpdf_parser.so"
);
+export type PDFMetadata = {
+ numPages: number;
+ title?: string;
+}
+
class RustPDFParser {
private static instance: RustPDFParser;
- private _getPageCount: KoffiFunction;
+ private _getPDFMetadata: KoffiFunction;
private constructor() {
const lib = koffi.load(rustExecutablePath);
- this._getPageCount = lib.func("get_page_count", "int32", ["string"]);
+ this._getPDFMetadata = lib.func("get_pdf_metadata", "string", ["string"]);
}
public static async isParserAvailable(): Promise<boolean> {
@@ -45,16 +50,21 @@ class RustPDFParser {
return RustPDFParser.instance;
}
- public async getPageCount(path: string): Promise<number> {
- return new Promise<number>((resolve, reject) => {
- this._getPageCount.async(path, (err: Error, res: number) => {
+ public async getPDFMetadata(path: string): Promise<PDFMetadata> {
+ return new Promise<PDFMetadata>((resolve, reject) => {
+ this._getPDFMetadata.async(path, (err: Error, res: string) => {
if (err) {
reject(err);
} else {
- if (res === -1) {
- reject(new Error("Failed to parse PDF."));
+ if (res.startsWith("RUSTFC:ERROR:")) {
+ reject(new Error(res.replace("RUSTFC:ERROR:", "")));
} else {
- resolve(res);
+ try {
+ const metadata = JSON.parse(res) as PDFMetadata;
+ resolve(metadata);
+ } catch (e) {
+ reject(new Error("Failed to parse PDF metadata."));
+ }
}
}
});
@@ -62,9 +72,9 @@ class RustPDFParser {
}
}
-export async function getPageCount(
+export async function getPDFMetadata(
path: string,
-): Promise<number> {
+): Promise<PDFMetadata> {
const converter = await RustPDFParser.getInstance();
- return await converter.getPageCount(path);
+ return await converter.getPDFMetadata(path);
}
diff --git a/apps/api/src/scraper/scrapeURL/engines/index.ts b/apps/api/src/scraper/scrapeURL/engines/index.ts
index 4addc40b6..cc2529479 100644
--- a/apps/api/src/scraper/scrapeURL/engines/index.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/index.ts
@@ -13,6 +13,7 @@ import { playwrightMaxReasonableTime, scrapeURLWithPlaywright } from "./playwrig
import { indexMaxReasonableTime, scrapeURLWithIndex } from "./index/index";
import { useIndex } from "../../../services";
import { hasFormatOfType } from "../../../lib/format-utils";
+import { PDFMetadata } from "../../../lib/pdf-parser";
export type Engine =
| "fire-engine;chrome-cdp"
@@ -114,7 +115,7 @@ export type EngineScrapeResult = {
pdfs: string[];
};
- numPages?: number;
+ pdfMetadata?: PDFMetadata;
cacheInfo?: {
created_at: Date;
diff --git a/apps/api/src/scraper/scrapeURL/engines/index/index.ts b/apps/api/src/scraper/scrapeURL/engines/index/index.ts
index 740dda874..b076da894 100644
--- a/apps/api/src/scraper/scrapeURL/engines/index/index.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/index/index.ts
@@ -53,7 +53,10 @@ export async function sendDocumentToIndex(meta: Meta, document: Document) {
statusCode: document.metadata.statusCode,
error: document.metadata.error,
screenshot: document.screenshot,
- numPages: document.metadata.numPages,
+ pdfMetadata: document.metadata.numPages !== undefined ? { // reconstruct pdfMetadata from numPages and title
+ numPages: document.metadata.numPages,
+ title: document.metadata.title ?? undefined,
+ } : undefined,
contentType: document.metadata.contentType,
});
} catch (error) {
@@ -267,7 +270,9 @@ export async function scrapeURLWithIndex(meta: Meta): Promise<EngineScrapeResult
statusCode: doc.statusCode,
error: doc.error,
screenshot: doc.screenshot,
- numPages: doc.numPages,
+ pdfMetadata: doc.pdfMetadata ?? (doc.numPages !== undefined ? { // backwards-compatible shim of pdfMetadata without title
+ numPages: doc.numPages
+ } : undefined),
contentType: doc.contentType,
cacheInfo: {
diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts
index 310544eb8..4f726e62c 100644
--- a/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts
@@ -16,7 +16,7 @@ import {
import { readFile, unlink } from "node:fs/promises";
import path from "node:path";
import type { Response } from "undici";
-import { getPageCount } from "../../../../lib/pdf-parser";
+import { getPDFMetadata } from "../../../../lib/pdf-parser";
import { getPdfResultFromCache, savePdfResultToCache } from "../../../../lib/gcs-pdf-cache";
import { AbortManagerThrownError } from "../../lib/abortManager";
@@ -227,11 +227,12 @@ export async function scrapePDF(
}
}
- const pageCount = await getPageCount(tempFilePath);
- if (pageCount * MILLISECONDS_PER_PAGE > (meta.abort.scrapeTimeout() ?? Infinity)) {
+ const pdfMetadata = await getPDFMetadata(tempFilePath);
+
+ if (pdfMetadata.numPages * MILLISECONDS_PER_PAGE > (meta.abort.scrapeTimeout() ?? Infinity)) {
throw new PDFInsufficientTimeError(
- pageCount,
- pageCount * MILLISECONDS_PER_PAGE + 5000,
+ pdfMetadata.numPages,
+ pdfMetadata.numPages * MILLISECONDS_PER_PAGE + 5000,
);
}
@@ -291,7 +292,7 @@ export async function scrapePDF(
statusCode: response.status,
html: result?.html ?? "",
markdown: result?.markdown ?? "",
- numPages: pageCount,
+ pdfMetadata,
proxyUsed: "basic",
};
diff --git a/apps/api/src/scraper/scrapeURL/index.ts b/apps/api/src/scraper/scrapeURL/index.ts
index 6f811a049..f16aea173 100644
--- a/apps/api/src/scraper/scrapeURL/index.ts
+++ b/apps/api/src/scraper/scrapeURL/index.ts
@@ -510,7 +510,8 @@ async function scrapeURLLoop(meta: Meta): Promise<ScrapeUrlResponse> {
url: result.result.url,
statusCode: result.result.statusCode,
error: result.result.error,
- numPages: result.result.numPages,
+ numPages: result.result.pdfMetadata?.numPages,
+ title: result.result.pdfMetadata?.title,
contentType: result.result.contentType,
proxyUsed: meta.featureFlags.has("stealthProxy") ? "stealth" : "basic",
...(fallbackList.find(x => ["index", "index;documents"].includes(x.engine)) ? (
diff --git a/apps/api/src/services/index.ts b/apps/api/src/services/index.ts
index 4ede28992..a846e0736 100644
--- a/apps/api/src/services/index.ts
+++ b/apps/api/src/services/index.ts
@@ -7,6 +7,7 @@ import { redisEvictConnection } from "./redis";
import type { Logger } from "winston";
import psl from "psl";
import { MapDocument } from "../controllers/v2/types";
+import { PDFMetadata } from "../lib/pdf-parser";
configDotenv();
// SupabaseService class initializes the Supabase client conditionally based on environment variables.
@@ -97,7 +98,7 @@ export async function saveIndexToGCS(id: string, doc: {
statusCode: number;
error?: string;
screenshot?: string;
- numPages?: number;
+ pdfMetadata?: PDFMetadata;
contentType?: string;
}): Promise<void> {
try {