Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.
TypeScript agentaiai-agentsall-in-onecrmcrm-systememaillinearmcpmessagingnotesnotionofficerustslackslack-alternativestartupstartup-toolstypescriptworkspace
/// extract text from a pdf
/// insert references
/// store rectangles and text
use pdfium_render::prelude::{PdfDocument, PdfRect, PdfiumError, PdfiumInternalError};
use std::{rc::Rc, sync::Arc};
use uuid::Uuid;
use crate::{
model::key::DocumentKey,
service::{self, db::DocumentTextOutcome},
};
use anyhow::{Context, Result};
use lambda_runtime::{
Error,
tracing::{self},
};
use model::citations::{DocumentReference, TextReference, UserPdfRect};
const DELIMETER: char = '\n';
const NPARTS_PER_ID: i32 = 4;
struct TextPart {
pub ph: f32,
pub pw: f32,
pub id: String,
pub text: String,
pub bounds: PdfRect,
pub page_index: u32,
}
// transform from page to user space
pub fn get_user_pdf_rect(rect: &PdfRect, page_index: u32, ph: f32, pw: f32) -> UserPdfRect {
let width = (rect.right().value - rect.left().value) * 100.0 / pw;
let height = (rect.top().value - rect.bottom().value) * 100.0 / ph;
let left = rect.left().value * 100.0 / pw;
let top = (ph - rect.top().value) * 100.0 / ph;
let page_index = page_index + 1;
UserPdfRect {
page_index,
left,
top,
width,
height,
}
}
fn unzip_text_parts(parts: Vec<TextPart>) -> (Vec<TextReference>, String) {
let text_with_refs = parts.iter().fold(String::new(), |text, part| {
format!("{}{}[[{}]]{}", text, part.text, part.id, DELIMETER)
});
let ids = parts
.into_iter()
.map(|part| TextReference {
reference: DocumentReference::Pdf(get_user_pdf_rect(
&part.bounds,
part.page_index,
part.ph,
part.pw,
)),
id: part.id,
})
.collect();
(ids, text_with_refs)
}
// simple single rect is used for citations
// expand the rectangle to encompass an entire sentence
fn expand_bounds(a: Option<PdfRect>, b: Result<PdfRect, PdfiumError>) -> Option<PdfRect> {
match (a, b) {
(Some(a), Ok(b)) => {
if b.left().value < a.left().value {
a.left().value = b.left().value;
}
if b.right().value > a.right().value {
a.right().value = b.right().value;
}
if b.top().value > a.top().value {
a.top().value = b.top().value;
}
if b.bottom().value < a.bottom().value {
a.bottom().value = b.bottom().value
}
Some(a)
}
(None, Ok(b)) => Some(b),
_ => a,
}
}
pub fn ref_id_extract_text(document: &PdfDocument) -> (Vec<TextReference>, String) {
let mut sentences: Vec<TextPart> = Vec::new();
for (page_index, page) in document.pages().iter().enumerate() {
let (ph, pw) = (page.height().value, page.width().value);
if let Ok(page_text) = page.text() {
let mut part = String::new();
let mut page_parts: Vec<TextPart> = Vec::new();
let mut part_bounds = None::<PdfRect>;
let mut count = 0;
for c in page_text.chars().iter() {
match c.unicode_char() {
None => (),
Some('\0') => (),
Some(DELIMETER) if count >= NPARTS_PER_ID => {
part_bounds = expand_bounds(part_bounds, c.loose_bounds());
part.push(DELIMETER);
page_parts.push(TextPart {
ph,
pw,
page_index: page_index as u32,
bounds: part_bounds.unwrap_or(PdfRect::zero()),
id: Uuid::new_v4().to_string(),
text: part,
});
part = String::new();
part_bounds = None;
count = 0;
}
Some(DELIMETER) => {
part.push(DELIMETER);
count += 1;
}
Some(content) if !content.is_whitespace() => {
part.push(content);
part_bounds = expand_bounds(part_bounds, c.loose_bounds());
}
Some(content) => part.push(content),
}
}
if !part.is_empty() {
page_parts.push(TextPart {
ph,
pw,
page_index: page_index as u32,
bounds: part_bounds.unwrap_or(PdfRect::zero()),
id: Uuid::new_v4().to_string(),
text: part,
});
}
sentences.extend(page_parts);
}
}
unzip_text_parts(sentences)
}
#[tracing::instrument(skip(pdfium, s3_client, db) fields(key = key, bucket = bucket))]
pub async fn extract_text_from_document(
key: &str,
bucket: &str,
pdfium: Rc<pdfium_render::prelude::Pdfium>,
s3_client: Arc<service::s3::S3>,
db: Arc<service::db::DB>,
) -> Result<Option<DocumentKey>, Error> {
let document_key = DocumentKey::from_s3_key(key).map_err(|e| {
tracing::error!(error=?e, key=%key, "invalid key format");
Error::from("invalid key format")
})?;
if document_key.is_bom_part() || document_key.is_sync_service_snapshot() {
tracing::trace!(?document_key, "skipping non-document key");
return Ok(None);
}
let document_id = document_key.document_id().expect("document key has id");
let file_type = if document_key.is_converted_pdf() {
Some(model::document::FileType::Pdf)
} else if document_key.is_temp() {
None
} else {
db.get_document_file_type(document_id).await.map_err(|e| {
tracing::error!(error=?e, "unable to get file type from db");
Error::from("unable to get file type from db")
})?
};
match file_type {
Some(model::document::FileType::Pdf) => {}
_ => {
tracing::info!(file_type=?file_type, "skipping non-pdf file for extraction");
return Ok(None);
}
}
tracing::info!(document_id=%document_id, "starting extracting text process for document");
let raw_file = match s3_client
.get_document_bytes(bucket, &document_key.to_key())
.await
{
Ok(bytes) => bytes,
Err(e) => {
// If we can't get the document from s3, then most likely the document was deleted
// this shouldn't cause an error, but mostly we want to know about it
tracing::warn!(error=?e, "unable to retrieve document from s3");
return Ok(None);
}
};
let file = match pdfium.load_pdf_from_byte_vec(raw_file, None) {
Ok(file) => file,
// there are still some lingering encrypted pdfs what we don't care about
Err(PdfiumError::PdfiumLibraryInternalError(PdfiumInternalError::PasswordError)) => {
tracing::warn!("ran into encrypted pdf, skipping");
return Ok(None);
}
// corrupt or unsupported pdfs fail the same way on every retry
Err(PdfiumError::PdfiumLibraryInternalError(
error @ (PdfiumInternalError::FormatError | PdfiumInternalError::SecurityError),
)) => {
tracing::warn!(error=?error, "pdfium cannot open pdf, skipping");
return Ok(None);
}
Err(e) => {
tracing::error!(error=?e, "pdfium failed to load pdf from bytes");
return Err(Error::from(e));
}
};
let (references, extracted_text) = ref_id_extract_text(&file);
let token_count = tiktoken_rs::o200k_base()
.map(|enc| enc.encode_with_special_tokens(&extracted_text).len() as i64)
.map_err(|e| {
tracing::error!(error=?e, "unable to count tokens");
Error::from(e)
})?;
match db
.create_document_text(document_id, &extracted_text, token_count)
.await
.context("unable to store extracted text in db")?
{
DocumentTextOutcome::Stored => {}
DocumentTextOutcome::DocumentMissing => {
tracing::warn!(document_id=%document_id, "document no longer exists, skipping");
return Ok(None);
}
}
db.insert_references(&references, document_id)
.await
.context("coult not insert sentences")?;
tracing::info!(document_id=%document_id, "extraction complete");
Ok(Some(document_key))
}
// locally run extraction
#[cfg(test)]
#[cfg(feature = "local-extract")]
mod test {
use pdfium_render::prelude::*;
use sqlx::PgPool;
use std::fs::File;
use std::io::Read;
use crate::handler::extract_text_citations::ref_id_extract_text;
use crate::service::db::DBClient;
#[ignore = "this test requires a very specific system environment which is hard to replicate in CI"]
#[tokio::test]
pub async fn test_local_extract() {
let db_url = std::env::var("DATABASE_URL").unwrap();
let pool = PgPool::connect(db_url.as_str()).await.unwrap();
let db = DBClient::new(pool);
let pdfium_path = "./pdfium-lib/macos";
let pdfium = Pdfium::new(
Pdfium::bind_to_library(Pdfium::pdfium_platform_library_name_at_path(pdfium_path))
.unwrap(),
);
let test_file = "deepseek.pdf";
let mut f = File::open(test_file).unwrap();
let mut data: Vec<u8> = vec![];
f.read_to_end(&mut data).unwrap();
let pdf_file = pdfium.load_pdf_from_byte_vec(data, None).unwrap();
let (refs, text) = ref_id_extract_text(&pdf_file);
println!("{}", text);
// println!("__REFERENCES__\n{:?}", refs);
// println!("__TEXT__\n{:?}", text);
db.create_document_text("test", text.as_str(), 0)
.await
.unwrap();
db.insert_references(&refs, "test").await.unwrap();
println!("pdfium initialized");
}
}