Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.
TypeScript agentaiai-agentsall-in-onecrmcrm-systememaillinearmcpmessagingnotesnotionofficerustslackslack-alternativestartupstartup-toolstypescriptworkspace
fix(convert): keep LibreOffice scratch files from filling the task disk (#9546)
teof763d8a605b7
diff --git a/infra/stacks/convert-service/index.ts b/infra/stacks/convert-service/index.tsindex 39b6118399..9ceaa6167e 100644--- a/infra/stacks/convert-service/index.ts+++ b/infra/stacks/convert-service/index.ts@@ -13,6 +13,10 @@ const tags = {const convertQueue = new Queue('convert-service', {tags,maxReceiveCount: 2,+ // Above the slowest conversion (`conversion_timeout` in the service's+ // utils.rs, 120s) even when it waits behind its batch, so a running+ // conversion is not redelivered and a failed one retries after a pause.+ visibilityTimeoutSeconds: 300,});export const convertQueueArn = convertQueue.queue.arn;diff --git a/services/convert_service/AGENTS.md b/services/convert_service/AGENTS.mdindex ab08e9e8e2..546aeda03a 100644--- a/services/convert_service/AGENTS.md+++ b/services/convert_service/AGENTS.md@@ -38,6 +38,16 @@ synchronously via HTTP or asynchronously via SQS.LOK shared library in the environment.- The `disable_cleanup`/`disable_worker`/`local_queue` Cargo features gatelocal-dev behavior — don't assume they're enabled in deployed builds.+- Removing the job folder is the only disk cleanup; the conversion child's+ `TMPDIR` points there so LibreOffice's scratch files go with it. Anything+ written elsewhere stays on the task's 20 GiB ephemeral disk until ECS+ replaces the task. A full disk looks like LibreOffice failing+ (`loadComponentFromURL returned an empty reference`, `Failed to save+ document`) next to `No space left on device`; check+ `ecs.fargate.ephemeral_storage.utilized` before debugging LibreOffice.+- The convert queue's visibility timeout (`infra/stacks/convert-service`)+ must stay above `conversion_timeout`; failed messages go to the DLQ after+ their second receive.## Relateddiff --git a/services/convert_service/src/api/convert.rs b/services/convert_service/src/api/convert.rsindex b0d0d1b88b..afad53f8aa 100644--- a/services/convert_service/src/api/convert.rs+++ b/services/convert_service/src/api/convert.rs@@ -1,4 +1,4 @@-use std::{fs::File, io::Write, str::FromStr, thread, time::Duration};+use std::{fs::File, io::Write, str::FromStr};use anyhow::Context;use axum::{@@ -14,15 +14,11 @@ use model::{document::FileType,response::{EmptyResponse, ErrorResponse},};-use nix::{- libc::exit,- unistd::{ForkResult, fork},-};-use rs_libreoffice_bindings::Office;use crate::{config::LOK_PATH,- utils::{cleanup_folder, get_lok_filter_from_file_types},+ process::convert::run_lok_conversion,+ utils::{cleanup_folder, conversion_timeout, get_lok_filter_from_file_types},};use super::context::{ApiContext, AuthorizationService};@@ -124,8 +120,11 @@ pub async fn handler(.into_response()})?;- let mut file =- File::create(format!("./{}/IN.{}", job_id, from_file_type.as_str())).map_err(|e| {+ let in_file_path = format!("./{}/IN.{}", job_id, from_file_type.as_str());+ let out_file_path = format!("./{}/OUT.{}", job_id, to_file_type.as_str());++ let result = async {+ let mut file = File::create(&in_file_path).map_err(|e| {tracing::error!(error=?e, job_id=%job_id, "unable to create file");(StatusCode::INTERNAL_SERVER_ERROR,@@ -136,138 +135,68 @@ pub async fn handler(.into_response()})?;- file.write_all(&file_content).map_err(|e| {- tracing::error!(error=?e, job_id=%job_id, "unable to write file");- (- StatusCode::INTERNAL_SERVER_ERROR,- Json(ErrorResponse {- message: "unable to write file".into(),- }),- )- .into_response()- })?;-- let in_file_path = format!("./{}/IN.{}", job_id, from_file_type.as_str());- let out_file_path = format!("./{}/OUT.{}", job_id, to_file_type.as_str());-- let office_path = &*LOK_PATH;- let mut child_pids = vec![];- match unsafe { fork() } {- Ok(ForkResult::Parent { child }) => {- tracing::info!(job_id=%job_id, "parent process: spawned child with PID: {}", child);- child_pids.push(child);- }- Ok(ForkResult::Child) => unsafe {- tracing::info!(job_id=%job_id, "child process: initializing lok");- std::env::set_var("SAL_LOG", "");- let result: anyhow::Result<()> = (|| {- let office = Office::new(office_path)- .map_err(|e| anyhow::anyhow!(e))- .context("unable to initialize lok")?;- let document = office- .load_document(&in_file_path)- .map_err(|e| anyhow::anyhow!(e))- .context("unable to load document")?;- document- .save_as(&out_file_path, to_file_type.as_str(), &filter)- .map_err(|e| anyhow::anyhow!(e))- .context("unable to save document")?;-- drop(document);- drop(office);- Ok(())- })();-- match result {- Ok(_) => exit(0),- Err(e) => {- tracing::error!(job_id=%job_id, error=?e, "unable to convert");- exit(1)- }- }- },- Err(err) => {- tracing::error!(job_id=%job_id, error=?err, "fork failed");- return Err((+ file.write_all(&file_content).map_err(|e| {+ tracing::error!(error=?e, job_id=%job_id, "unable to write file");+ (StatusCode::INTERNAL_SERVER_ERROR,Json(ErrorResponse {- message: "fork failed".into(),+ message: "unable to write file".into(),}),)- .into_response());- }- }-- let mut status_code = 1;- loop {- if child_pids.is_empty() {- break;- }-- thread::sleep(Duration::from_millis(100));-- // check for completed children- child_pids.retain(|&pid| {- match nix::sys::wait::waitpid(- nix::unistd::Pid::from_raw(pid.into()),- Some(nix::sys::wait::WaitPidFlag::WNOHANG),- ) {- Ok(nix::sys::wait::WaitStatus::StillAlive) => true,- Ok(nix::sys::wait::WaitStatus::Exited(_, status)) => {- tracing::debug!(job_id=%job_id, status=status, "pid {} completed", pid);- status_code = status;- false- }- _ => {- tracing::warn!(job_id=%job_id, "pid {} completed without exiting", pid);- false- }- }- });- }-- if status_code != 0 {- tracing::error!(job_id=%job_id, status=status_code, "conversion failed with non-zero status code");- if let Err(e) = cleanup_folder(&job_id) {- tracing::error!(error=?e, job_id=%job_id, "unable to cleanup folder");- }- return Err((- StatusCode::INTERNAL_SERVER_ERROR,- Json(ErrorResponse {- message: "conversion failed".into(),- }),- )- .into_response());- }+ .into_response()+ })?;- let file_content = std::fs::read(out_file_path).map_err(|e| {- tracing::error!(error=?e, job_id=%job_id, "unable to read file");- (- StatusCode::INTERNAL_SERVER_ERROR,- Json(ErrorResponse {- message: "unable to read file".into(),- }),+ run_lok_conversion(+ &job_id,+ &LOK_PATH,+ &in_file_path,+ &out_file_path,+ to_file_type.as_str(),+ &filter,+ conversion_timeout(&from_file_type, &to_file_type),)- .into_response()- })?;-- ctx.s3_client- .put(&req.to_bucket, &req.to_key, &file_content)- .await.map_err(|e| {- tracing::error!(error=?e, job_id=%job_id, "unable to upload file");+ tracing::error!(error=?e, job_id=%job_id, "conversion failed");+ (+ StatusCode::INTERNAL_SERVER_ERROR,+ Json(ErrorResponse {+ message: "conversion failed".into(),+ }),+ )+ .into_response()+ })?;++ let file_content = std::fs::read(&out_file_path).map_err(|e| {+ tracing::error!(error=?e, job_id=%job_id, "unable to read file");(StatusCode::INTERNAL_SERVER_ERROR,Json(ErrorResponse {- message: "unable to upload file".into(),+ message: "unable to read file".into(),}),).into_response()})?;+ ctx.s3_client+ .put(&req.to_bucket, &req.to_key, &file_content)+ .await+ .map_err(|e| {+ tracing::error!(error=?e, job_id=%job_id, "unable to upload file");+ (+ StatusCode::INTERNAL_SERVER_ERROR,+ Json(ErrorResponse {+ message: "unable to upload file".into(),+ }),+ )+ .into_response()+ })+ }+ .await;+if let Err(e) = cleanup_folder(&job_id) {tracing::error!(error=?e, job_id=%job_id, "unable to cleanup folder");}+ result?;Ok((StatusCode::OK, Json(EmptyResponse::default())).into_response())}diff --git a/services/convert_service/src/process/convert.rs b/services/convert_service/src/process/convert.rsindex 97abb8adfd..e2c20903f4 100644--- a/services/convert_service/src/process/convert.rs+++ b/services/convert_service/src/process/convert.rs@@ -3,7 +3,7 @@ use nix::{unistd::{ForkResult, fork},};use rs_libreoffice_bindings::Office;-use std::{borrow::Cow, fs::File, io::Write, thread, time::Duration};+use std::{borrow::Cow, fs::File, io::Write, path::Path, thread, time::Duration};use anyhow::Context;use model::convert::ConvertQueueMessage;@@ -184,7 +184,8 @@ pub async fn convert(/// child process, killing it after `max_wait`.////// The child process keeps a LibreOffice crash or hang from taking down the-/// worker.+/// worker. LibreOffice writes its scratch files into the input file's folder,+/// so removing that folder cleans up after the child however it ended.pub(crate) fn run_lok_conversion(job_id: &str,office_path: &str,@@ -194,6 +195,12 @@ pub(crate) fn run_lok_conversion(filter: &str,max_wait: Duration,) -> anyhow::Result<()> {+ let job_folder = Path::new(in_file_path)+ .parent()+ .map(std::path::absolute)+ .context("input file should be inside a job folder")?+ .context("unable to resolve job folder")?;+let mut child_pids = vec![];match unsafe { fork() } {Ok(ForkResult::Parent { child }) => {@@ -204,6 +211,10 @@ pub(crate) fn run_lok_conversion(tracing::info!(job_id=%job_id, "child process: initializing lok");let result: anyhow::Result<()> = (|| {std::env::set_var("SAL_LOG", "");+ // LibreOffice deletes its TMPDIR scratch files only in its own+ // teardown, which a child killed at its timeout, crashed, or+ // ended with `_exit` below never runs.+ std::env::set_var("TMPDIR", &job_folder);let office = Office::new(office_path).map_err(|e| anyhow::anyhow!(e)).context("unable to initialize lok")?;diff --git a/services/convert_service/src/process/mod.rs b/services/convert_service/src/process/mod.rsindex 28f1b900b3..83ef16364c 100644--- a/services/convert_service/src/process/mod.rs+++ b/services/convert_service/src/process/mod.rs@@ -1,4 +1,4 @@-mod convert;+pub(crate) mod convert;pub mod runner;#[cfg(test)]diff --git a/services/convert_service/src/process/test.rs b/services/convert_service/src/process/test.rsindex 509117c051..8fb73affa5 100644--- a/services/convert_service/src/process/test.rs+++ b/services/convert_service/src/process/test.rs@@ -259,6 +259,29 @@ mod legacy_upgrade {assert!(result.is_err());}+ #[test]+ #[ignore = "needs LibreOfficeKit; see module docs"]+ fn conversions_leave_no_libreoffice_scratch_files_outside_the_job_folder() {+ // The worker only removes the job folder, and a child killed at its+ // timeout never runs LibreOffice's own scratch cleanup.+ let scratch = || {+ let mut names: Vec<_> = std::fs::read_dir(std::env::temp_dir())+ .unwrap()+ .filter_map(|entry| entry.ok())+ .map(|entry| entry.file_name().to_string_lossy().into_owned())+ .filter(|name| name.starts_with("lu") && name.ends_with(".tmp"))+ .collect();+ names.sort();+ names+ };+ let before = scratch();++ let docx = upgrade("memo.doc", FileType::Doc, FileType::Docx);+ convert_bytes(&docx, FileType::Docx, FileType::Pdf);++ assert_eq!(scratch(), before);+ }+#[test]#[ignore = "needs LibreOfficeKit; see module docs"]fn upgraded_docx_still_exports_to_pdf() {