Papermark is the open-source DocSend alternative and secure data rooms with built-in analytics and custom domains.
fix: libreoffice fixes in conversion
Marc Seitz0909ddf50e4f
diff --git a/ee/features/conversions/python/docx-sanitizer.py b/ee/features/conversions/python/docx-sanitizer.pyindex bf6bf321..eb9c09a3 100644--- a/ee/features/conversions/python/docx-sanitizer.py+++ b/ee/features/conversions/python/docx-sanitizer.py@@ -1,15 +1,23 @@#!/usr/bin/env python3"""-Minimal DOCX Sanitizer — only unwraps <w:sdt> blocks.+DOCX Sanitizer — fixes common issues that cause LibreOffice conversion failures.-Google Docs exports wrap content in structured document tag (<w:sdt>) blocks-for suggested-edit tracking (goog_rdk_*) and TOC wrappers. LibreOffice crashes-on these — especially empty ones. This script unwraps them, keeping the inner-content intact and touching nothing else.+Supported fixes:+ - RTL compat: Downgrade compatibilityMode 15→14 for Arabic/RTL documents+ that cause LibreOffice's layout engine to hang indefinitely.+ - Glossary removal: Remove corrupt glossary parts (0-byte fontTable.xml etc.)+ - SDT unwrap: Unwrap <w:sdt> blocks from Google Docs exports that crash+ LibreOffice.Usage:python docx-sanitizer.py input.docx [output.docx]- python docx-sanitizer.py -v input.docx+ python docx-sanitizer.py --mode rtl -v input.docx output.docx+ python docx-sanitizer.py --mode all -v input.docx output.docx++Modes:+ rtl — RTL compat fix + glossary removal only+ sdt — SDT unwrap only (original behavior)+ all — All fixes (default)If output is omitted, the input file is overwritten in place."""@@ -21,11 +29,71 @@ import loggingimport zipfileimport tempfileimport argparse+import shutillog = logging.getLogger("docx-sanitizer")-def unwrap_sdt(content: str) -> str:+def has_rtl_content(doc_content: str) -> bool:+ """Check whether document.xml contains RTL / complex-script markers."""+ return '<w:rtl/>' in doc_content or '<w:bidi/>' in doc_content+++def downgrade_compat_mode(content: str) -> tuple:+ """Downgrade compatibilityMode from 15 to 14 in settings.xml content."""+ new_content = re.sub(+ r'(<w:compatSetting\b'+ r'(?=[^>]*\bw:name="compatibilityMode")'+ r'(?=[^>]*\bw:uri="http://schemas\.microsoft\.com/office/word")'+ r'[^>]*\bw:val=")15(")',+ r'\g<1>14\2',+ content,+ )+ changed = new_content != content+ return new_content, changed+++def remove_glossary(tmp_dir: str) -> bool:+ """Remove word/glossary/ directory and clean up its references."""+ glossary_dir = os.path.join(tmp_dir, 'word', 'glossary')+ if not os.path.isdir(glossary_dir):+ return False++ shutil.rmtree(glossary_dir)+ log.info("Removed word/glossary/ directory")++ rels_path = os.path.join(tmp_dir, 'word', '_rels', 'document.xml.rels')+ if os.path.exists(rels_path):+ with open(rels_path, 'r', encoding='utf-8') as f:+ rels_content = f.read()+ new_rels = re.sub(+ r'<Relationship[^>]*Target="glossary/[^"]*"[^>]*/>\s*',+ '',+ rels_content,+ )+ if new_rels != rels_content:+ with open(rels_path, 'w', encoding='utf-8') as f:+ f.write(new_rels)+ log.info("Removed glossary relationship from document.xml.rels")++ ct_path = os.path.join(tmp_dir, '[Content_Types].xml')+ if os.path.exists(ct_path):+ with open(ct_path, 'r', encoding='utf-8') as f:+ ct_content = f.read()+ new_ct = re.sub(+ r'<Override[^>]*PartName="/word/glossary/[^"]*"[^>]*/>\s*',+ '',+ ct_content,+ )+ if new_ct != ct_content:+ with open(ct_path, 'w', encoding='utf-8') as f:+ f.write(new_ct)+ log.info("Removed glossary overrides from [Content_Types].xml")++ return True+++def unwrap_sdt(content: str) -> tuple:"""Replace <w:sdt>...<w:sdtContent>X</w:sdtContent></w:sdt> with X."""count = 0while '<w:sdt>' in content:@@ -43,10 +111,30 @@ def unwrap_sdt(content: str) -> str:return content, count-def sanitize_docx(input_path: str, output_path: str) -> bool:+def repackage_docx(tmp_dir: str, output_path: str):+ """Repackage extracted DOCX with [Content_Types].xml first."""+ entries = []+ ct_entry = None+ for root, _dirs, files in os.walk(tmp_dir):+ for file in files:+ fp = os.path.join(root, file)+ arc = os.path.relpath(fp, tmp_dir)+ if arc == '[Content_Types].xml':+ ct_entry = (fp, arc)+ else:+ entries.append((fp, arc))+ if ct_entry:+ entries.insert(0, ct_entry)++ with zipfile.ZipFile(output_path, 'w', zipfile.ZIP_DEFLATED) as z:+ for fp, arc in entries:+ z.write(fp, arc)+++def sanitize_docx(input_path: str, output_path: str, mode: str = 'all') -> bool:try:input_size = os.path.getsize(input_path)- log.info("Input: %s (%d bytes)", input_path, input_size)+ log.info("Input: %s (%d bytes), mode: %s", input_path, input_size, mode)with tempfile.TemporaryDirectory() as tmp:with zipfile.ZipFile(input_path, 'r') as z:@@ -58,35 +146,48 @@ def sanitize_docx(input_path: str, output_path: str) -> bool:return Falsewith open(doc_path, 'r', encoding='utf-8') as f:- content = f.read()-- new_content, count = unwrap_sdt(content)-- if count:- log.info("Unwrapped %d <w:sdt> block(s) (removed %d bytes)",- count, len(content) - len(new_content))- with open(doc_path, 'w', encoding='utf-8') as f:- f.write(new_content)- else:- log.info("No <w:sdt> blocks found — file unchanged")-- # Repackage with [Content_Types].xml first- entries = []- ct_entry = None- for root, _dirs, files in os.walk(tmp):- for file in files:- fp = os.path.join(root, file)- arc = os.path.relpath(fp, tmp)- if arc == '[Content_Types].xml':- ct_entry = (fp, arc)+ doc_content = f.read()++ doc_modified = False++ # --- RTL compat fixes (glossary removal + compat mode downgrade) ---+ if mode in ('rtl', 'all'):+ remove_glossary(tmp)++ if has_rtl_content(doc_content):+ log.info("RTL content detected — checking compatibilityMode")+ settings_path = os.path.join(tmp, 'word', 'settings.xml')+ if os.path.exists(settings_path):+ with open(settings_path, 'r', encoding='utf-8') as f:+ settings_content = f.read()+ new_settings, changed = downgrade_compat_mode(settings_content)+ if changed:+ with open(settings_path, 'w', encoding='utf-8') as f:+ f.write(new_settings)+ log.info("Downgraded compatibilityMode 15 → 14")+ else:+ log.info("compatibilityMode is not 15 — no change needed")else:- entries.append((fp, arc))- if ct_entry:- entries.insert(0, ct_entry)+ log.info("No word/settings.xml found — skipping compat fix")+ else:+ log.info("No RTL content detected — skipping compat downgrade")++ # --- SDT unwrap ---+ if mode in ('sdt', 'all'):+ new_content, sdt_count = unwrap_sdt(doc_content)+ if sdt_count:+ log.info("Unwrapped %d <w:sdt> block(s) (removed %d bytes)",+ sdt_count, len(doc_content) - len(new_content))+ doc_content = new_content+ doc_modified = True+ else:+ log.info("No <w:sdt> blocks found")++ if doc_modified:+ with open(doc_path, 'w', encoding='utf-8') as f:+ f.write(doc_content)- with zipfile.ZipFile(output_path, 'w', zipfile.ZIP_DEFLATED) as z:- for fp, arc in entries:- z.write(fp, arc)+ repackage_docx(tmp, output_path)output_size = os.path.getsize(output_path)log.info("Output: %s (%d bytes, %+d)", output_path, output_size, output_size - input_size)@@ -97,11 +198,31 @@ def sanitize_docx(input_path: str, output_path: str) -> bool:return False+def check_rtl(input_path: str) -> bool:+ """Open DOCX, read document.xml, return whether RTL content is present."""+ try:+ with zipfile.ZipFile(input_path, 'r') as z:+ with z.open('word/document.xml') as f:+ content = f.read().decode('utf-8')+ return has_rtl_content(content)+ except (KeyError, Exception) as e:+ log.warning("Could not check RTL: %s", e)+ return False++def main():- parser = argparse.ArgumentParser(description="Minimal DOCX sanitizer — unwrap sdt blocks only")+ parser = argparse.ArgumentParser(+ description="DOCX sanitizer for LibreOffice conversion issues")parser.add_argument("input", help="Input .docx file")- parser.add_argument("output", nargs="?", default=None, help="Output .docx (default: overwrite input)")+ parser.add_argument("output", nargs="?", default=None,+ help="Output .docx (default: overwrite input)")parser.add_argument("-v", "--verbose", action="count", default=0)+ parser.add_argument("--mode", choices=["rtl", "sdt", "all"], default="all",+ help="Sanitization mode: rtl (compat fix only), "+ "sdt (unwrap only), all (default)")+ parser.add_argument("--check-rtl", action="store_true",+ help="Check if document has RTL content and exit. "+ "Prints 'true' or 'false' to stdout.")args = parser.parse_args()level = logging.WARNING@@ -109,14 +230,19 @@ def main():level = logging.DEBUGelif args.verbose >= 1:level = logging.INFO- logging.basicConfig(level=level, format="%(levelname)-5s %(message)s", stream=sys.stderr)+ logging.basicConfig(level=level, format="%(levelname)-5s %(message)s",+ stream=sys.stderr)if not os.path.exists(args.input):log.error("File not found: %s", args.input)sys.exit(1)+ if args.check_rtl:+ print("true" if check_rtl(args.input) else "false")+ sys.exit(0)+output = args.output or args.input- if sanitize_docx(args.input, output):+ if sanitize_docx(args.input, output, mode=args.mode):print(f"Sanitized DOCX written to: {output}")else:sys.exit(1)diff --git a/lib/trigger/convert-files.ts b/lib/trigger/convert-files.tsindex d60898a1..cf656216 100644--- a/lib/trigger/convert-files.ts+++ b/lib/trigger/convert-files.ts@@ -1,6 +1,6 @@import { python } from "@trigger.dev/python";import { logger, retry, task } from "@trigger.dev/sdk/v3";-import { writeFile, readFile } from "node:fs/promises";+import { writeFile, readFile, unlink } from "node:fs/promises";import { tmpdir } from "node:os";import { join } from "node:path";@@ -82,45 +82,38 @@ export const convertFilesToPdfTask = task({type: document.versions[0].storageType,});- // Prepare form data- const formData = new FormData();- formData.append(- "downloadFrom",- JSON.stringify([- {- url: fileUrl,- },- ]),- );- formData.append("quality", "75");+ const CONVERSION_TIMEOUT_MS = 3 * 60 * 1000; // 3 min client-side timeout- updateStatus({ progress: 20, text: "Converting document..." });+ updateStatus({ progress: 20, text: "Converting document…" });- // Make the conversion request- const conversionResponse = await retry.fetch(- `${process.env.NEXT_PRIVATE_CONVERSION_BASE_URL}/forms/libreoffice/convert`,- {- method: "POST",- body: formData,- headers: {- Authorization: `Basic ${process.env.NEXT_PRIVATE_INTERNAL_AUTH_TOKEN}`,- },- retry: {- byStatus: {- "500-599": {- strategy: "backoff",- maxAttempts: 3,- factor: 2,- minTimeoutInMs: 1_000,- maxTimeoutInMs: 30_000,- randomize: false,- },+ let conversionResponse: Response;+ try {+ const fd = new FormData();+ fd.append("downloadFrom", JSON.stringify([{ url: fileUrl }]));+ fd.append("quality", "75");++ conversionResponse = await fetch(+ `${process.env.NEXT_PRIVATE_CONVERSION_BASE_URL}/forms/libreoffice/convert`,+ {+ method: "POST",+ body: fd,+ headers: {+ Authorization: `Basic ${process.env.NEXT_PRIVATE_INTERNAL_AUTH_TOKEN}`,},+ signal: AbortSignal.timeout(CONVERSION_TIMEOUT_MS),},- },- );+ );+ } catch (err) {+ logger.warn("Gotenberg conversion failed, will attempt sanitization", {+ error: String(err),+ });+ conversionResponse = new Response(null, {+ status: 504,+ statusText: "Conversion timed out or unreachable",+ });+ }- let conversionBuffer: Buffer;+ let conversionBuffer!: Buffer;if (!conversionResponse.ok) {const contentType = document.versions[0].contentType;@@ -129,58 +122,118 @@ export const convertFilesToPdfTask = task({"application/vnd.openxmlformats-officedocument.wordprocessingml.document";if (isDocx) {- logger.warn("DOCX conversion failed, attempting sanitization…", {- status: conversionResponse.status,- });- updateStatus({ progress: 25, text: "Sanitizing document…" });-- const docxResponse = await fetch(fileUrl);- const docxBuffer = Buffer.from(await docxResponse.arrayBuffer());-const inputPath = join(tmpdir(), `input-${Date.now()}.docx`);const outputPath = join(tmpdir(), `output-${Date.now()}.docx`);- await writeFile(inputPath, docxBuffer);-- const result = await python.runScript(- "./ee/features/conversions/python/docx-sanitizer.py",- ["-v", inputPath, outputPath],- );- logger.info("Sanitizer output", { stderr: result.stderr });-- const sanitizedBuffer = await readFile(outputPath);- const retryFormData = new FormData();- retryFormData.append(- "files",- new Blob([new Uint8Array(sanitizedBuffer)], { type: contentType }),- document.name,- );- retryFormData.append("quality", "75");-- const retryResponse = await fetch(- `${process.env.NEXT_PRIVATE_CONVERSION_BASE_URL}/forms/libreoffice/convert`,- {- method: "POST",- body: retryFormData,- headers: {- Authorization: `Basic ${process.env.NEXT_PRIVATE_INTERNAL_AUTH_TOKEN}`,- },- },- );-- if (!retryResponse.ok) {- updateStatus({ progress: 0, text: "Conversion failed" });- const body = await retryResponse.json();- throw new Error(- `Conversion failed after sanitization: ${body.message} ${retryResponse.status}`,+ try {+ let docxResponse: Response;+ try {+ docxResponse = await fetch(fileUrl);+ } catch (err) {+ updateStatus({ progress: 0, text: "Failed to retrieve DOCX file" });+ throw new Error(+ `Failed to fetch DOCX from signed URL for sanitization: ${String(err)}`,+ );+ }++ if (!docxResponse.ok) {+ updateStatus({ progress: 0, text: "Failed to retrieve DOCX file" });+ throw new Error(+ `Failed to fetch DOCX from signed URL for sanitization: HTTP ${docxResponse.status} ${docxResponse.statusText}`,+ );+ }++ const responseContentType = docxResponse.headers.get("content-type");+ if (+ responseContentType &&+ !responseContentType.includes(+ "application/vnd.openxmlformats-officedocument.wordprocessingml.document",+ ) &&+ !responseContentType.includes("application/octet-stream")+ ) {+ updateStatus({ progress: 0, text: "Invalid DOCX download response" });+ throw new Error(+ `Signed URL returned unexpected content-type for DOCX sanitization: ${responseContentType}`,+ );+ }++ const docxBuffer = Buffer.from(await docxResponse.arrayBuffer());+ await writeFile(inputPath, docxBuffer);++ const attemptConversion = async (+ sanitizedPath: string,+ ): Promise<Response> => {+ const buf = await readFile(sanitizedPath);+ const fd = new FormData();+ fd.append(+ "files",+ new Blob([new Uint8Array(buf)], { type: contentType }),+ document.name,+ );+ fd.append("quality", "75");+ return fetch(+ `${process.env.NEXT_PRIVATE_CONVERSION_BASE_URL}/forms/libreoffice/convert`,+ {+ method: "POST",+ body: fd,+ headers: {+ Authorization: `Basic ${process.env.NEXT_PRIVATE_INTERNAL_AUTH_TOKEN}`,+ },+ signal: AbortSignal.timeout(CONVERSION_TIMEOUT_MS),+ },+ );+ };++ logger.warn("DOCX conversion failed, sanitizing document…", {+ status: conversionResponse.status,+ });++ updateStatus({ progress: 25, text: "Sanitizing document…" });++ const result = await python.runScript(+ "./ee/features/conversions/python/docx-sanitizer.py",+ ["-v", "--mode", "all", inputPath, outputPath],);+ logger.info("Sanitizer output", { stderr: result.stderr });++ let retryResponse: Response;+ try {+ retryResponse = await attemptConversion(outputPath);+ } catch (err) {+ updateStatus({ progress: 0, text: "Conversion timed out" });+ throw new Error(+ `Conversion timed out after sanitization — LibreOffice could not process this file within ${CONVERSION_TIMEOUT_MS / 1000}s`,+ );+ }++ if (!retryResponse.ok) {+ updateStatus({ progress: 0, text: "Conversion failed" });+ let message: string;+ try {+ const body = await retryResponse.json();+ message = body.message ?? retryResponse.statusText;+ } catch {+ message = retryResponse.statusText || "Unknown error";+ }+ throw new Error(+ `Conversion failed after sanitization: ${message} (${retryResponse.status})`,+ );+ }++ conversionBuffer = Buffer.from(await retryResponse.arrayBuffer());+ } finally {+ await Promise.allSettled([unlink(inputPath), unlink(outputPath)]);}-- conversionBuffer = Buffer.from(await retryResponse.arrayBuffer());} else {updateStatus({ progress: 0, text: "Conversion failed" });- const body = await conversionResponse.json();+ let message: string;+ try {+ const body = await conversionResponse.json();+ message = body.message ?? conversionResponse.statusText;+ } catch {+ message = conversionResponse.statusText || "Unknown error";+ }throw new Error(- `Conversion failed: ${body.message} ${conversionResponse.status}`,+ `Conversion failed: ${message} (${conversionResponse.status})`,);}} else {