Papermark is the open-source DocSend alternative and secure data rooms with built-in analytics and custom domains.
feat: add document sanitizer
Marc Seitz502db47be750
diff --git a/ee/features/conversions/python/docx-sanitizer.py b/ee/features/conversions/python/docx-sanitizer.pynew file mode 100644index 00000000..bf6bf321--- /dev/null+++ b/ee/features/conversions/python/docx-sanitizer.py@@ -0,0 +1,126 @@+#!/usr/bin/env python3+"""+Minimal DOCX Sanitizer — only unwraps <w:sdt> blocks.++Google Docs exports wrap content in structured document tag (<w:sdt>) blocks+for suggested-edit tracking (goog_rdk_*) and TOC wrappers. LibreOffice crashes+on these — especially empty ones. This script unwraps them, keeping the inner+content intact and touching nothing else.++Usage:+ python docx-sanitizer.py input.docx [output.docx]+ python docx-sanitizer.py -v input.docx++If output is omitted, the input file is overwritten in place.+"""++import sys+import os+import re+import logging+import zipfile+import tempfile+import argparse++log = logging.getLogger("docx-sanitizer")+++def unwrap_sdt(content: str) -> str:+ """Replace <w:sdt>...<w:sdtContent>X</w:sdtContent></w:sdt> with X."""+ count = 0+ while '<w:sdt>' in content:+ old = content+ content = re.sub(+ r'<w:sdt><w:sdtPr>.*?</w:sdtPr><w:sdtContent>(.*?)</w:sdtContent></w:sdt>',+ r'\1',+ content,+ count=1,+ flags=re.DOTALL,+ )+ if content == old:+ break+ count += 1+ return content, count+++def sanitize_docx(input_path: str, output_path: str) -> bool:+ try:+ input_size = os.path.getsize(input_path)+ log.info("Input: %s (%d bytes)", input_path, input_size)++ with tempfile.TemporaryDirectory() as tmp:+ with zipfile.ZipFile(input_path, 'r') as z:+ z.extractall(tmp)++ doc_path = os.path.join(tmp, 'word', 'document.xml')+ if not os.path.exists(doc_path):+ log.error("No word/document.xml found")+ return False++ with open(doc_path, 'r', encoding='utf-8') as f:+ content = f.read()++ new_content, count = unwrap_sdt(content)++ if count:+ log.info("Unwrapped %d <w:sdt> block(s) (removed %d bytes)",+ count, len(content) - len(new_content))+ with open(doc_path, 'w', encoding='utf-8') as f:+ f.write(new_content)+ else:+ log.info("No <w:sdt> blocks found — file unchanged")++ # Repackage with [Content_Types].xml first+ entries = []+ ct_entry = None+ for root, _dirs, files in os.walk(tmp):+ for file in files:+ fp = os.path.join(root, file)+ arc = os.path.relpath(fp, tmp)+ if arc == '[Content_Types].xml':+ ct_entry = (fp, arc)+ else:+ entries.append((fp, arc))+ if ct_entry:+ entries.insert(0, ct_entry)++ with zipfile.ZipFile(output_path, 'w', zipfile.ZIP_DEFLATED) as z:+ for fp, arc in entries:+ z.write(fp, arc)++ output_size = os.path.getsize(output_path)+ log.info("Output: %s (%d bytes, %+d)", output_path, output_size, output_size - input_size)+ return True++ except Exception as e:+ log.exception("Error: %s", e)+ return False+++def main():+ parser = argparse.ArgumentParser(description="Minimal DOCX sanitizer — unwrap sdt blocks only")+ parser.add_argument("input", help="Input .docx file")+ parser.add_argument("output", nargs="?", default=None, help="Output .docx (default: overwrite input)")+ parser.add_argument("-v", "--verbose", action="count", default=0)+ args = parser.parse_args()++ level = logging.WARNING+ if args.verbose >= 2:+ level = logging.DEBUG+ elif args.verbose >= 1:+ level = logging.INFO+ logging.basicConfig(level=level, format="%(levelname)-5s %(message)s", stream=sys.stderr)++ if not os.path.exists(args.input):+ log.error("File not found: %s", args.input)+ sys.exit(1)++ output = args.output or args.input+ if sanitize_docx(args.input, output):+ print(f"Sanitized DOCX written to: {output}")+ else:+ sys.exit(1)+++if __name__ == '__main__':+ main()diff --git a/lib/trigger/convert-files.ts b/lib/trigger/convert-files.tsindex c73c2c57..d60898a1 100644--- a/lib/trigger/convert-files.ts+++ b/lib/trigger/convert-files.ts@@ -1,4 +1,8 @@+import { python } from "@trigger.dev/python";import { logger, retry, task } from "@trigger.dev/sdk/v3";+import { writeFile, readFile } from "node:fs/promises";+import { tmpdir } from "node:os";+import { join } from "node:path";import { getFile } from "@/lib/files/get-file";import { putFileServer } from "@/lib/files/put-file-server";@@ -116,18 +120,73 @@ export const convertFilesToPdfTask = task({},);+ let conversionBuffer: Buffer;+if (!conversionResponse.ok) {- updateStatus({ progress: 0, text: "Conversion failed" });- const body = await conversionResponse.json();- throw new Error(- `Conversion failed: ${body.message} ${conversionResponse.status}`,- );+ const contentType = document.versions[0].contentType;+ const isDocx =+ contentType ===+ "application/vnd.openxmlformats-officedocument.wordprocessingml.document";++ if (isDocx) {+ logger.warn("DOCX conversion failed, attempting sanitization…", {+ status: conversionResponse.status,+ });+ updateStatus({ progress: 25, text: "Sanitizing document…" });++ const docxResponse = await fetch(fileUrl);+ const docxBuffer = Buffer.from(await docxResponse.arrayBuffer());++ const inputPath = join(tmpdir(), `input-${Date.now()}.docx`);+ const outputPath = join(tmpdir(), `output-${Date.now()}.docx`);+ await writeFile(inputPath, docxBuffer);++ const result = await python.runScript(+ "./ee/features/conversions/python/docx-sanitizer.py",+ ["-v", inputPath, outputPath],+ );+ logger.info("Sanitizer output", { stderr: result.stderr });++ const sanitizedBuffer = await readFile(outputPath);+ const retryFormData = new FormData();+ retryFormData.append(+ "files",+ new Blob([new Uint8Array(sanitizedBuffer)], { type: contentType }),+ document.name,+ );+ retryFormData.append("quality", "75");++ const retryResponse = await fetch(+ `${process.env.NEXT_PRIVATE_CONVERSION_BASE_URL}/forms/libreoffice/convert`,+ {+ method: "POST",+ body: retryFormData,+ headers: {+ Authorization: `Basic ${process.env.NEXT_PRIVATE_INTERNAL_AUTH_TOKEN}`,+ },+ },+ );++ if (!retryResponse.ok) {+ updateStatus({ progress: 0, text: "Conversion failed" });+ const body = await retryResponse.json();+ throw new Error(+ `Conversion failed after sanitization: ${body.message} ${retryResponse.status}`,+ );+ }++ conversionBuffer = Buffer.from(await retryResponse.arrayBuffer());+ } else {+ updateStatus({ progress: 0, text: "Conversion failed" });+ const body = await conversionResponse.json();+ throw new Error(+ `Conversion failed: ${body.message} ${conversionResponse.status}`,+ );+ }+ } else {+ conversionBuffer = Buffer.from(await conversionResponse.arrayBuffer());}- const conversionBuffer = Buffer.from(- await conversionResponse.arrayBuffer(),- );-console.log("conversionBuffer", conversionBuffer);// get docId from url with starts with "doc_" with regexdiff --git a/package-lock.json b/package-lock.jsonindex ee17a1ee..b43c4baa 100644--- a/package-lock.json+++ b/package-lock.json@@ -67,6 +67,7 @@"@tiptap/react": "^3.19.0","@tiptap/starter-kit": "^3.19.0","@tremor/react": "^3.18.7",+ "@trigger.dev/python": "^3.3.17","@trigger.dev/react-hooks": "^3.3.17","@trigger.dev/sdk": "^3.3.17","@tus/s3-store": "^1.9.1",@@ -10604,7 +10605,6 @@"version": "3.3.17","resolved": "https://registry.npmjs.org/@trigger.dev/build/-/build-3.3.17.tgz","integrity": "sha512-dfreMuVeLAcZypS3kkUA9nWNviiuOPIQ3ldy2ywPCmwmbHyd0BE8tI5D3A4kmVq/f53TdRMls4c+cYafxlwubQ==",- "dev": true,"dependencies": {"@trigger.dev/core": "3.3.17","pkg-types": "^1.1.3",@@ -10676,6 +10676,29 @@"url": "https://github.com/sponsors/colinhacks"}},+ "node_modules/@trigger.dev/python": {+ "version": "3.3.17",+ "resolved": "https://registry.npmjs.org/@trigger.dev/python/-/python-3.3.17.tgz",+ "integrity": "sha512-8Wv1xjlLLfemZ1poYAB5MF2QHgGpiBVgu8M5km/NRV5nhngTIlY4xZIKjCWoIf6A/4+IJNwvimzt10ZGKp09Rg==",+ "license": "MIT",+ "dependencies": {+ "@trigger.dev/core": "3.3.17",+ "tinyexec": "^0.3.2"+ },+ "engines": {+ "node": ">=18.20.0"+ },+ "peerDependencies": {+ "@trigger.dev/build": "^3.3.17",+ "@trigger.dev/sdk": "^3.3.17"+ }+ },+ "node_modules/@trigger.dev/python/node_modules/tinyexec": {+ "version": "0.3.2",+ "resolved": "https://registry.npmjs.org/tinyexec/-/tinyexec-0.3.2.tgz",+ "integrity": "sha512-KQQR9yN7R5+OSwaK0XQoj22pwHoTlgYqmUscPYoknOoWCWfj/5/ABTMRi69FrKU5ffPVh5QcFikpWJI/P1ocHA==",+ "license": "MIT"+ },"node_modules/@trigger.dev/react-hooks": {"version": "3.3.17","resolved": "https://registry.npmjs.org/@trigger.dev/react-hooks/-/react-hooks-3.3.17.tgz",@@ -13263,8 +13286,7 @@"node_modules/confbox": {"version": "0.1.8","resolved": "https://registry.npmjs.org/confbox/-/confbox-0.1.8.tgz",- "integrity": "sha512-RMtmw0iFkeR4YV+fUOSucriAQNb9g8zFR52MWCtl+cCZOFRNL6zeB395vPzFhEjjn4fMxXudmELnl/KF/WrK6w==",- "dev": true+ "integrity": "sha512-RMtmw0iFkeR4YV+fUOSucriAQNb9g8zFR52MWCtl+cCZOFRNL6zeB395vPzFhEjjn4fMxXudmELnl/KF/WrK6w=="},"node_modules/consola": {"version": "3.4.2",@@ -19516,7 +19538,6 @@"version": "1.8.0","resolved": "https://registry.npmjs.org/mlly/-/mlly-1.8.0.tgz","integrity": "sha512-l8D9ODSRWLe2KHJSifWGwBqpTZXIXTeo8mlKjY+E2HAakaTeNpqAyBZ8GSqLzHgw4XmHmC8whvpjJNMbFZN7/g==",- "dev": true,"license": "MIT","dependencies": {"acorn": "^8.15.0",@@ -21222,7 +21243,6 @@"version": "1.3.1","resolved": "https://registry.npmjs.org/pkg-types/-/pkg-types-1.3.1.tgz","integrity": "sha512-/Jm5M4RvtBFVkKWRu2BLUTNP8/M2a+UwuAX+ae4770q1qVGtfjG+WTCupoZixokjmHiry8uI+dlY8KXYV5HVVQ==",- "dev": true,"license": "MIT","dependencies": {"confbox": "^0.1.8",@@ -25153,7 +25173,6 @@"version": "0.2.10","resolved": "https://registry.npmjs.org/tinyglobby/-/tinyglobby-0.2.10.tgz","integrity": "sha512-Zc+8eJlFMvgatPZTl6A9L/yht8QqdmUNtURHaKZLmKBE12hNPSrqNkUp2cs3M/UKmNVVAMFQYSjYIVHDjW5zew==",- "dev": true,"dependencies": {"fdir": "^6.4.2","picomatch": "^4.0.2"@@ -25166,7 +25185,6 @@"version": "6.4.2","resolved": "https://registry.npmjs.org/fdir/-/fdir-6.4.2.tgz","integrity": "sha512-KnhMXsKSPZlAhp7+IjUkRZKPb4fUyccpDrdFXbi4QL1qkmFh9kVY09Yox+n4MaOb3lHZ1Tv829C3oaaXoMYPDQ==",- "dev": true,"peerDependencies": {"picomatch": "^3 || ^4"},@@ -25180,7 +25198,6 @@"version": "4.0.2","resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.2.tgz","integrity": "sha512-M7BAV6Rlcy5u+m6oPhAPFgJTzAioX/6B0DxyvDlo9l8+T3nLKbrczg2WLUyzd45L8RqfUMyGPzekbMvX2Ldkwg==",- "dev": true,"engines": {"node": ">=12"},@@ -25298,7 +25315,6 @@"version": "3.1.3","resolved": "https://registry.npmjs.org/tsconfck/-/tsconfck-3.1.3.tgz","integrity": "sha512-ulNZP1SVpRDesxeMLON/LtWM8HIgAJEIVpVVhBM6gsmvQ8+Rh+ZG7FWGvHh7Ah3pRABwVJWklWCr/BTZSv0xnQ==",- "dev": true,"bin": {"tsconfck": "bin/tsconfck.js"},diff --git a/package.json b/package.jsonindex 79ad2c80..551c481a 100644--- a/package.json+++ b/package.json@@ -78,6 +78,7 @@"@tiptap/react": "^3.19.0","@tiptap/starter-kit": "^3.19.0","@tremor/react": "^3.18.7",+ "@trigger.dev/python": "^3.3.17","@trigger.dev/react-hooks": "^3.3.17","@trigger.dev/sdk": "^3.3.17","@tus/s3-store": "^1.9.1",diff --git a/trigger.config.ts b/trigger.config.tsindex 4c0cc7fa..fa757b32 100644--- a/trigger.config.ts+++ b/trigger.config.ts@@ -1,5 +1,6 @@import { ffmpeg } from "@trigger.dev/build/extensions/core";import { prismaExtension } from "@trigger.dev/build/extensions/prisma";+import { pythonExtension } from "@trigger.dev/python/extension";import { defineConfig, timeout } from "@trigger.dev/sdk/v3";export default defineConfig({@@ -22,6 +23,9 @@ export default defineConfig({schema: "prisma/schema/schema.prisma",}),ffmpeg(),+ pythonExtension({+ scripts: ["./**/*.py"],+ }),],},});