Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.
TypeScript agentaiai-agentsall-in-onecrmcrm-systememaillinearmcpmessagingnotesnotionofficerustslackslack-alternativestartupstartup-toolstypescriptworkspace
use crate::outbound::email_api::GmailApi;
use base64::Engine;
use base64::engine::general_purpose::STANDARD;
use document_storage_service_client::DocumentStorageServiceClient;
use email_api_client::domain::models::EmailApiError;
use macro_user_id::lowercased::Lowercase;
use macro_user_id::user_id::{MacroUserId, MacroUserIdStr};
use model::document::MAX_DOCUMENT_NAME_GRAPHEMES;
use model::document::response::{CreateDocumentRequest, CreateDocumentResponse};
use models_email::service::attachment::{
AttachmentSfs, AttachmentUploadArgs, AttachmentUploadMetadata,
};
use models_email::service::link;
use sha2::{Digest, Sha256};
use static_file_service_client::StaticFileServiceClient;
use std::sync::Arc;
use system_properties::{
EmailAttachmentInput, EmailAttachmentProperty, PgSystemPropertiesRepository, SourceEntity,
SystemPropertiesService, SystemPropertiesServiceImpl,
};
use thiserror::Error;
use unicode_segmentation::UnicodeSegmentation;
use uuid::Uuid;
#[cfg(test)]
mod test;
#[derive(Error, Debug)]
pub enum UploadAttachmentError {
#[error("Gmail API rate limit exceeded")]
RateLimited(EmailApiError),
#[error("Failed to fetch attachment data from Gmail: {0}")]
GmailFetchFailed(EmailApiError),
#[error("Failed to upload media to SFS: {0}")]
SfsUploadFailed(String),
#[error("Parse error: {0}")]
ParseError(String),
#[error("Database error: {0}")]
DatabaseError(String),
#[error("Attachment filename is missing")]
FilenameMissing,
#[error(
"Failed to determine file extension from mime type ({mime_type}) or filename ({filename})"
)]
FileExtensionDeterminationFailed { mime_type: String, filename: String },
#[error("Failed to create document record in DSS: {0}")]
DssCreateFailed(String),
#[error("DSS response did not include a presigned URL")]
PresignedUrlMissing,
#[error("Failed to upload attachment to presigned URL. Status: {0}, Body: {1}")]
PresignedUrlUploadFailed(reqwest::StatusCode, String),
#[error("Failed to set email attachment properties: {0}")]
SystemPropertiesSetFailed(String),
}
/// Context required for uploading an email attachment.
pub struct UploadAttachmentContext<'a> {
pub db: &'a sqlx::Pool<sqlx::Postgres>,
pub email_api: &'a GmailApi,
pub dss_client: &'a DocumentStorageServiceClient,
pub sfs_client: &'a StaticFileServiceClient,
pub system_properties_service:
&'a Arc<SystemPropertiesServiceImpl<PgSystemPropertiesRepository>>,
pub link: &'a link::Link,
}
/// Provider-independent storage context for already downloaded attachment bytes.
pub struct AttachmentStorageContext<'a> {
pub db: &'a sqlx::PgPool,
pub dss_client: &'a DocumentStorageServiceClient,
pub sfs_client: &'a StaticFileServiceClient,
pub system_properties_service:
&'a Arc<SystemPropertiesServiceImpl<PgSystemPropertiesRepository>>,
pub link: &'a link::Link,
}
/// Upload an email attachment to DSS as a document or SFS as media.
pub async fn upload_attachment(
ctx: UploadAttachmentContext<'_>,
args: &AttachmentUploadArgs,
) -> Result<String, UploadAttachmentError> {
// Fetching through the email API service applies this context's token and quota policy.
let attachment_data =
fetch_gmail_attachment_data(ctx.email_api, ctx.link, &args.attachment_metadata).await?;
store_attachment(
AttachmentStorageContext {
db: ctx.db,
dss_client: ctx.dss_client,
sfs_client: ctx.sfs_client,
system_properties_service: ctx.system_properties_service,
link: ctx.link,
},
args,
attachment_data,
)
.await
}
/// Share DSS/SFS persistence across providers after scoped download succeeds.
pub async fn store_attachment(
ctx: AttachmentStorageContext<'_>,
args: &AttachmentUploadArgs,
attachment_data: Vec<u8>,
) -> Result<String, UploadAttachmentError> {
let mime_type = args.attachment_metadata.mime_type.clone();
match args.upload_destination {
models_email::service::attachment::AttachmentUploadDestination::Sfs => {
upload_media_attachment(&ctx, args, attachment_data, mime_type).await
}
models_email::service::attachment::AttachmentUploadDestination::Dss => {
upload_document_attachment(&ctx, args, attachment_data).await
}
}
}
/// Uploads an image or video attachment to SFS.
#[tracing::instrument(skip(ctx, attachment_data), err)]
async fn upload_media_attachment(
ctx: &AttachmentStorageContext<'_>,
args: &AttachmentUploadArgs,
attachment_data: Vec<u8>,
mime_type: String,
) -> Result<String, UploadAttachmentError> {
// Upload to SFS
let sfs_response = ctx
.sfs_client
.put_file_with_bytes("a", bytes::Bytes::from(attachment_data), mime_type)
.await
.map_err(|e| UploadAttachmentError::SfsUploadFailed(e.to_string()))?;
// Store metadata in email_attachments_sfs table
let attachment_sfs_id = macro_uuid::generate_uuid_v7();
let sfs_id = Uuid::parse_str(&sfs_response.id).map_err(|e| {
UploadAttachmentError::ParseError(format!("Failed to parse SFS ID as UUID: {}", e))
})?;
email_db_client::attachments::sfs::insert_attachment_sfs(
ctx.db,
&AttachmentSfs {
id: attachment_sfs_id,
attachment_id: Some(args.attachment_metadata.attachment_db_id),
sfs_id,
},
)
.await
.map_err(|e| UploadAttachmentError::DatabaseError(e.to_string()))?;
Ok(sfs_response.id)
}
/// Uploads a document attachment to DSS.
#[tracing::instrument(skip(ctx, attachment_data), err)]
async fn upload_document_attachment(
ctx: &AttachmentStorageContext<'_>,
args: &AttachmentUploadArgs,
attachment_data: Vec<u8>,
) -> Result<String, UploadAttachmentError> {
// 1. Calculate hashes required for the upload process.
let (hex_hash, base64_hash) = calculate_hashes(&attachment_data);
// 2. Determine file metadata from the payload.
let (file_name, file_type) = determine_file_metadata(&args.attachment_metadata)?;
// 3. Create the document record in DSS and get a presigned URL for the upload.
let dss_response = create_dss_document_record(
ctx.dss_client,
ctx.link,
&args.attachment_metadata,
&hex_hash,
&file_name,
&file_type,
args.backfill,
)
.await?;
if dss_response.data.document_response.presigned_url.is_some() {
upload_data_to_presigned_url(&dss_response, attachment_data, &base64_hash).await?;
}
// 5. Get document id
let document_id = dss_response
.data
.document_response
.document_metadata
.document_id
.clone();
// 6. Set properties for attachment
set_email_attachment_properties(ctx.system_properties_service, &document_id, args).await?;
Ok(document_id)
}
/// Fetches the raw attachment data from the Gmail API.
async fn fetch_gmail_attachment_data(
email_api: &GmailApi,
link: &link::Link,
p: &AttachmentUploadMetadata,
) -> Result<Vec<u8>, UploadAttachmentError> {
email_api
.get_attachment(link.id, &p.email_provider_id, &p.provider_attachment_id)
.await
.map_err(|error| match error {
error @ EmailApiError::RateLimited { .. } => UploadAttachmentError::RateLimited(error),
// Preserve the provider error so API handlers can map it to the
// right status instead of collapsing everything to 500.
error => UploadAttachmentError::GmailFetchFailed(error),
})
}
/// Calculates the SHA256 hash of the attachment data in both hex and base64 formats.
fn calculate_hashes(data: &[u8]) -> (String, String) {
let mut hasher = Sha256::new();
hasher.update(data);
let hash_bytes = hasher.finalize();
let hex_hash = format!("{:x}", hash_bytes);
let base64_hash = STANDARD.encode(hash_bytes);
(hex_hash, base64_hash)
}
/// Determines the file name (without extension) and file type (extension) from the payload.
fn determine_file_metadata(
p: &AttachmentUploadMetadata,
) -> Result<(String, String), UploadAttachmentError> {
// documents must have a file name to be inserted into Document table.
let original_file_name = p
.filename
.as_deref()
.ok_or(UploadAttachmentError::FilenameMissing)?;
let file_name = original_file_name
.split('.')
.next()
.unwrap_or(original_file_name);
let file_name = truncate_document_name(file_name).to_string();
let file_type = match original_file_name
.rsplit_once('.')
.map(|(_, ext)| ext.trim())
.filter(|ext| !ext.is_empty())
{
// if it's a heic, the mime_type can sometimes be heif. hardcode file_type to match file name
Some(ext) if ext.eq_ignore_ascii_case("heic") => "heic".to_string(),
_ => mime_guess::get_mime_extensions_str(&p.mime_type)
.and_then(|exts| exts.first().map(|s| s.to_string()))
.or_else(|| {
// if mime_guess fails, use everything after the last '.' in the original filename
original_file_name
.rsplit_once('.')
.map(|(_, ext)| ext.trim())
.filter(|ext| !ext.is_empty())
.map(|ext| ext.to_string())
})
.ok_or_else(|| UploadAttachmentError::FileExtensionDeterminationFailed {
mime_type: p.mime_type.clone(),
filename: original_file_name.to_string(),
})?,
};
Ok((file_name, file_type))
}
/// DSS rejects longer names, so keep only the leading graphemes rather than drop the attachment.
fn truncate_document_name(name: &str) -> &str {
name.grapheme_indices(true)
.nth(MAX_DOCUMENT_NAME_GRAPHEMES)
.map_or(name, |(end, _)| &name[..end])
}
/// Creates a document record in the Document Storage Service (DSS) and returns the response,
/// which includes the presigned URL for the upload.
async fn create_dss_document_record(
dss_client: &DocumentStorageServiceClient,
link: &link::Link,
p: &AttachmentUploadMetadata,
hex_hash: &str,
file_name: &str,
file_type: &str,
backfill: bool,
) -> Result<CreateDocumentResponse, UploadAttachmentError> {
// if we are backfilling, use the email timestamp. if it's an on-demand upload, use the current
// time so the document shows up at the top of soup views.
let created_at = backfill.then_some(p.internal_date_ts);
let request = CreateDocumentRequest {
id: None,
sha: hex_hash.to_string(),
document_name: file_name.to_string(),
file_type: Some(file_type.to_string()),
mime_type: Some(p.mime_type.clone()),
document_family_id: None,
branched_from_id: None,
branched_from_version_id: None,
job_id: None,
project_id: None,
team_id: None,
created_at,
email_attachment_id: Some(p.attachment_db_id),
is_task: false,
skip_history: true,
};
dss_client
.create_document_internal(request, link.macro_id.0.as_ref())
.await
.map_err(|e| UploadAttachmentError::DssCreateFailed(e.to_string()))
}
/// Uploads the provided data to the presigned URL from the DSS response.
async fn upload_data_to_presigned_url(
dss_response: &CreateDocumentResponse,
attachment_data: Vec<u8>,
base64_hash: &str,
) -> Result<(), UploadAttachmentError> {
let presigned_url = dss_response
.data
.document_response
.presigned_url
.as_ref()
.ok_or(UploadAttachmentError::PresignedUrlMissing)?;
let response = reqwest::Client::new()
.put(presigned_url)
.header("content-type", &dss_response.data.content_type)
.header("x-amz-checksum-sha256", base64_hash)
.body(attachment_data)
.send()
.await
.map_err(|e| {
UploadAttachmentError::PresignedUrlUploadFailed(
reqwest::StatusCode::INTERNAL_SERVER_ERROR,
e.to_string(),
)
})?;
if !response.status().is_success() {
let status = response.status();
let body = response.text().await.unwrap_or_default();
return Err(UploadAttachmentError::PresignedUrlUploadFailed(
status, body,
));
}
Ok(())
}
/// Sets email attachment properties for the uploaded document.
#[tracing::instrument(skip(system_properties_service, p))]
async fn set_email_attachment_properties(
system_properties_service: &Arc<SystemPropertiesServiceImpl<PgSystemPropertiesRepository>>,
document_id: &str,
p: &AttachmentUploadArgs,
) -> Result<(), UploadAttachmentError> {
system_properties_service
.set_email_attachment_properties(vec![EmailAttachmentInput {
entity_id: document_id.to_string(),
properties: email_attachment_property(p),
}])
.await
.map_err(|e| UploadAttachmentError::SystemPropertiesSetFailed(e.to_string()))?;
Ok(())
}
/// Builds the attachment's properties from its message.
///
/// Header addresses are stored as Gmail sent them, so some are not valid Macro
/// user ids. Those are left out rather than dropping every property.
fn email_attachment_property(p: &AttachmentUploadArgs) -> EmailAttachmentProperty<'static> {
EmailAttachmentProperty {
source: Some(SourceEntity {
entity_type: models_properties::EntityType::Thread,
entity_id: p.attachment_metadata.thread_db_id.to_string(),
specific_message_id: Some(p.attachment_metadata.message_db_id),
}),
// TODO: companies support
companies: None,
sender: macro_user_id(p, &p.attachment_metadata.sender_email),
recipients: Some(
p.recipient_emails
.iter()
.filter_map(|email| macro_user_id(p, email))
.collect(),
),
subject: p.attachment_metadata.subject.clone(),
}
}
fn macro_user_id(p: &AttachmentUploadArgs, email: &str) -> Option<MacroUserId<Lowercase<'static>>> {
match MacroUserIdStr::try_from_email(email) {
Ok(id) => Some(id.0),
// The parse error echoes the address, so it stays out of the log.
Err(_) => {
tracing::warn!(
message_id = %p.attachment_metadata.message_db_id,
"Leaving an address that is not a valid Macro user id out of attachment properties"
);
None
}
}
}