Skip to content

armgit / macro

Public Imported from GitHub
0

Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.

fix(convert): keep LibreOffice scratch files from filling the task disk (#9546)

teof763d8a605b7
diff --git a/infra/stacks/convert-service/index.ts b/infra/stacks/convert-service/index.ts
index 39b6118399..9ceaa6167e 100644
--- a/infra/stacks/convert-service/index.ts
+++ b/infra/stacks/convert-service/index.ts
@@ -13,6 +13,10 @@ const tags = {
const convertQueue = new Queue('convert-service', {
tags,
maxReceiveCount: 2,
+ // Above the slowest conversion (`conversion_timeout` in the service's
+ // utils.rs, 120s) even when it waits behind its batch, so a running
+ // conversion is not redelivered and a failed one retries after a pause.
+ visibilityTimeoutSeconds: 300,
});
export const convertQueueArn = convertQueue.queue.arn;
diff --git a/services/convert_service/AGENTS.md b/services/convert_service/AGENTS.md
index ab08e9e8e2..546aeda03a 100644
--- a/services/convert_service/AGENTS.md
+++ b/services/convert_service/AGENTS.md
@@ -38,6 +38,16 @@ synchronously via HTTP or asynchronously via SQS.
LOK shared library in the environment.
- The `disable_cleanup`/`disable_worker`/`local_queue` Cargo features gate
local-dev behavior — don't assume they're enabled in deployed builds.
+- Removing the job folder is the only disk cleanup; the conversion child's
+ `TMPDIR` points there so LibreOffice's scratch files go with it. Anything
+ written elsewhere stays on the task's 20 GiB ephemeral disk until ECS
+ replaces the task. A full disk looks like LibreOffice failing
+ (`loadComponentFromURL returned an empty reference`, `Failed to save
+ document`) next to `No space left on device`; check
+ `ecs.fargate.ephemeral_storage.utilized` before debugging LibreOffice.
+- The convert queue's visibility timeout (`infra/stacks/convert-service`)
+ must stay above `conversion_timeout`; failed messages go to the DLQ after
+ their second receive.
## Related
diff --git a/services/convert_service/src/api/convert.rs b/services/convert_service/src/api/convert.rs
index b0d0d1b88b..afad53f8aa 100644
--- a/services/convert_service/src/api/convert.rs
+++ b/services/convert_service/src/api/convert.rs
@@ -1,4 +1,4 @@
-use std::{fs::File, io::Write, str::FromStr, thread, time::Duration};
+use std::{fs::File, io::Write, str::FromStr};
use anyhow::Context;
use axum::{
@@ -14,15 +14,11 @@ use model::{
document::FileType,
response::{EmptyResponse, ErrorResponse},
};
-use nix::{
- libc::exit,
- unistd::{ForkResult, fork},
-};
-use rs_libreoffice_bindings::Office;
use crate::{
config::LOK_PATH,
- utils::{cleanup_folder, get_lok_filter_from_file_types},
+ process::convert::run_lok_conversion,
+ utils::{cleanup_folder, conversion_timeout, get_lok_filter_from_file_types},
};
use super::context::{ApiContext, AuthorizationService};
@@ -124,8 +120,11 @@ pub async fn handler(
.into_response()
})?;
- let mut file =
- File::create(format!("./{}/IN.{}", job_id, from_file_type.as_str())).map_err(|e| {
+ let in_file_path = format!("./{}/IN.{}", job_id, from_file_type.as_str());
+ let out_file_path = format!("./{}/OUT.{}", job_id, to_file_type.as_str());
+
+ let result = async {
+ let mut file = File::create(&in_file_path).map_err(|e| {
tracing::error!(error=?e, job_id=%job_id, "unable to create file");
(
StatusCode::INTERNAL_SERVER_ERROR,
@@ -136,138 +135,68 @@ pub async fn handler(
.into_response()
})?;
- file.write_all(&file_content).map_err(|e| {
- tracing::error!(error=?e, job_id=%job_id, "unable to write file");
- (
- StatusCode::INTERNAL_SERVER_ERROR,
- Json(ErrorResponse {
- message: "unable to write file".into(),
- }),
- )
- .into_response()
- })?;
-
- let in_file_path = format!("./{}/IN.{}", job_id, from_file_type.as_str());
- let out_file_path = format!("./{}/OUT.{}", job_id, to_file_type.as_str());
-
- let office_path = &*LOK_PATH;
- let mut child_pids = vec![];
- match unsafe { fork() } {
- Ok(ForkResult::Parent { child }) => {
- tracing::info!(job_id=%job_id, "parent process: spawned child with PID: {}", child);
- child_pids.push(child);
- }
- Ok(ForkResult::Child) => unsafe {
- tracing::info!(job_id=%job_id, "child process: initializing lok");
- std::env::set_var("SAL_LOG", "");
- let result: anyhow::Result<()> = (|| {
- let office = Office::new(office_path)
- .map_err(|e| anyhow::anyhow!(e))
- .context("unable to initialize lok")?;
- let document = office
- .load_document(&in_file_path)
- .map_err(|e| anyhow::anyhow!(e))
- .context("unable to load document")?;
- document
- .save_as(&out_file_path, to_file_type.as_str(), &filter)
- .map_err(|e| anyhow::anyhow!(e))
- .context("unable to save document")?;
-
- drop(document);
- drop(office);
- Ok(())
- })();
-
- match result {
- Ok(_) => exit(0),
- Err(e) => {
- tracing::error!(job_id=%job_id, error=?e, "unable to convert");
- exit(1)
- }
- }
- },
- Err(err) => {
- tracing::error!(job_id=%job_id, error=?err, "fork failed");
- return Err((
+ file.write_all(&file_content).map_err(|e| {
+ tracing::error!(error=?e, job_id=%job_id, "unable to write file");
+ (
StatusCode::INTERNAL_SERVER_ERROR,
Json(ErrorResponse {
- message: "fork failed".into(),
+ message: "unable to write file".into(),
}),
)
- .into_response());
- }
- }
-
- let mut status_code = 1;
- loop {
- if child_pids.is_empty() {
- break;
- }
-
- thread::sleep(Duration::from_millis(100));
-
- // check for completed children
- child_pids.retain(|&pid| {
- match nix::sys::wait::waitpid(
- nix::unistd::Pid::from_raw(pid.into()),
- Some(nix::sys::wait::WaitPidFlag::WNOHANG),
- ) {
- Ok(nix::sys::wait::WaitStatus::StillAlive) => true,
- Ok(nix::sys::wait::WaitStatus::Exited(_, status)) => {
- tracing::debug!(job_id=%job_id, status=status, "pid {} completed", pid);
- status_code = status;
- false
- }
- _ => {
- tracing::warn!(job_id=%job_id, "pid {} completed without exiting", pid);
- false
- }
- }
- });
- }
-
- if status_code != 0 {
- tracing::error!(job_id=%job_id, status=status_code, "conversion failed with non-zero status code");
- if let Err(e) = cleanup_folder(&job_id) {
- tracing::error!(error=?e, job_id=%job_id, "unable to cleanup folder");
- }
- return Err((
- StatusCode::INTERNAL_SERVER_ERROR,
- Json(ErrorResponse {
- message: "conversion failed".into(),
- }),
- )
- .into_response());
- }
+ .into_response()
+ })?;
- let file_content = std::fs::read(out_file_path).map_err(|e| {
- tracing::error!(error=?e, job_id=%job_id, "unable to read file");
- (
- StatusCode::INTERNAL_SERVER_ERROR,
- Json(ErrorResponse {
- message: "unable to read file".into(),
- }),
+ run_lok_conversion(
+ &job_id,
+ &LOK_PATH,
+ &in_file_path,
+ &out_file_path,
+ to_file_type.as_str(),
+ &filter,
+ conversion_timeout(&from_file_type, &to_file_type),
)
- .into_response()
- })?;
-
- ctx.s3_client
- .put(&req.to_bucket, &req.to_key, &file_content)
- .await
.map_err(|e| {
- tracing::error!(error=?e, job_id=%job_id, "unable to upload file");
+ tracing::error!(error=?e, job_id=%job_id, "conversion failed");
+ (
+ StatusCode::INTERNAL_SERVER_ERROR,
+ Json(ErrorResponse {
+ message: "conversion failed".into(),
+ }),
+ )
+ .into_response()
+ })?;
+
+ let file_content = std::fs::read(&out_file_path).map_err(|e| {
+ tracing::error!(error=?e, job_id=%job_id, "unable to read file");
(
StatusCode::INTERNAL_SERVER_ERROR,
Json(ErrorResponse {
- message: "unable to upload file".into(),
+ message: "unable to read file".into(),
}),
)
.into_response()
})?;
+ ctx.s3_client
+ .put(&req.to_bucket, &req.to_key, &file_content)
+ .await
+ .map_err(|e| {
+ tracing::error!(error=?e, job_id=%job_id, "unable to upload file");
+ (
+ StatusCode::INTERNAL_SERVER_ERROR,
+ Json(ErrorResponse {
+ message: "unable to upload file".into(),
+ }),
+ )
+ .into_response()
+ })
+ }
+ .await;
+
if let Err(e) = cleanup_folder(&job_id) {
tracing::error!(error=?e, job_id=%job_id, "unable to cleanup folder");
}
+ result?;
Ok((StatusCode::OK, Json(EmptyResponse::default())).into_response())
}
diff --git a/services/convert_service/src/process/convert.rs b/services/convert_service/src/process/convert.rs
index 97abb8adfd..e2c20903f4 100644
--- a/services/convert_service/src/process/convert.rs
+++ b/services/convert_service/src/process/convert.rs
@@ -3,7 +3,7 @@ use nix::{
unistd::{ForkResult, fork},
};
use rs_libreoffice_bindings::Office;
-use std::{borrow::Cow, fs::File, io::Write, thread, time::Duration};
+use std::{borrow::Cow, fs::File, io::Write, path::Path, thread, time::Duration};
use anyhow::Context;
use model::convert::ConvertQueueMessage;
@@ -184,7 +184,8 @@ pub async fn convert(
/// child process, killing it after `max_wait`.
///
/// The child process keeps a LibreOffice crash or hang from taking down the
-/// worker.
+/// worker. LibreOffice writes its scratch files into the input file's folder,
+/// so removing that folder cleans up after the child however it ended.
pub(crate) fn run_lok_conversion(
job_id: &str,
office_path: &str,
@@ -194,6 +195,12 @@ pub(crate) fn run_lok_conversion(
filter: &str,
max_wait: Duration,
) -> anyhow::Result<()> {
+ let job_folder = Path::new(in_file_path)
+ .parent()
+ .map(std::path::absolute)
+ .context("input file should be inside a job folder")?
+ .context("unable to resolve job folder")?;
+
let mut child_pids = vec![];
match unsafe { fork() } {
Ok(ForkResult::Parent { child }) => {
@@ -204,6 +211,10 @@ pub(crate) fn run_lok_conversion(
tracing::info!(job_id=%job_id, "child process: initializing lok");
let result: anyhow::Result<()> = (|| {
std::env::set_var("SAL_LOG", "");
+ // LibreOffice deletes its TMPDIR scratch files only in its own
+ // teardown, which a child killed at its timeout, crashed, or
+ // ended with `_exit` below never runs.
+ std::env::set_var("TMPDIR", &job_folder);
let office = Office::new(office_path)
.map_err(|e| anyhow::anyhow!(e))
.context("unable to initialize lok")?;
diff --git a/services/convert_service/src/process/mod.rs b/services/convert_service/src/process/mod.rs
index 28f1b900b3..83ef16364c 100644
--- a/services/convert_service/src/process/mod.rs
+++ b/services/convert_service/src/process/mod.rs
@@ -1,4 +1,4 @@
-mod convert;
+pub(crate) mod convert;
pub mod runner;
#[cfg(test)]
diff --git a/services/convert_service/src/process/test.rs b/services/convert_service/src/process/test.rs
index 509117c051..8fb73affa5 100644
--- a/services/convert_service/src/process/test.rs
+++ b/services/convert_service/src/process/test.rs
@@ -259,6 +259,29 @@ mod legacy_upgrade {
assert!(result.is_err());
}
+ #[test]
+ #[ignore = "needs LibreOfficeKit; see module docs"]
+ fn conversions_leave_no_libreoffice_scratch_files_outside_the_job_folder() {
+ // The worker only removes the job folder, and a child killed at its
+ // timeout never runs LibreOffice's own scratch cleanup.
+ let scratch = || {
+ let mut names: Vec<_> = std::fs::read_dir(std::env::temp_dir())
+ .unwrap()
+ .filter_map(|entry| entry.ok())
+ .map(|entry| entry.file_name().to_string_lossy().into_owned())
+ .filter(|name| name.starts_with("lu") && name.ends_with(".tmp"))
+ .collect();
+ names.sort();
+ names
+ };
+ let before = scratch();
+
+ let docx = upgrade("memo.doc", FileType::Doc, FileType::Docx);
+ convert_bytes(&docx, FileType::Docx, FileType::Pdf);
+
+ assert_eq!(scratch(), before);
+ }
+
#[test]
#[ignore = "needs LibreOfficeKit; see module docs"]
fn upgraded_docx_still_exports_to_pdf() {