A platform for community discussion. Free, open, simple.
DEV: Split document encoding out of UploadEncoder (#43008)
Régis Hanol1e0b2f60043a
diff --git a/plugins/discourse-ai/lib/completions/dialects/dialect.rb b/plugins/discourse-ai/lib/completions/dialects/dialect.rbindex 1f97b2f117a..e478effd2af 100644--- a/plugins/discourse-ai/lib/completions/dialects/dialect.rb+++ b/plugins/discourse-ai/lib/completions/dialects/dialect.rb@@ -517,7 +517,7 @@ module DiscourseAiext = File.extname(encoded[:filename].to_s).delete_prefix(".")allowed_types.include?(- DiscourseAi::Completions::UploadEncoder.attachment_type_for(ext, encoded[:mime_type]),+ DiscourseAi::Completions::DocumentEncoder.attachment_type_for(ext, encoded[:mime_type]),)endenddiff --git a/plugins/discourse-ai/lib/completions/doc_to_text.rb b/plugins/discourse-ai/lib/completions/doc_to_text.rbindex abce4a203a3..84b33091060 100644--- a/plugins/discourse-ai/lib/completions/doc_to_text.rb+++ b/plugins/discourse-ai/lib/completions/doc_to_text.rb@@ -6,7 +6,7 @@ module DiscourseAimodule Completionsclass DocToTextANTIWORD_TIMEOUT_SECONDS = 5- MAX_CONVERSION_OUTPUT_BYTES = 4 * 100_001+ MAX_CONVERSION_OUTPUT_BYTES = 4 * TextNormalization::MAX_EXTRACTED_TEXT_CHARSSAFE_EXEC_ENV = { "PATH" => ENV["PATH"].to_s }.freezeANTIWORD_RLIMITS = {cpu_seconds: ANTIWORD_TIMEOUT_SECONDS,diff --git a/plugins/discourse-ai/lib/completions/document_encoder.rb b/plugins/discourse-ai/lib/completions/document_encoder.rbnew file mode 100644index 00000000000..b0df5310835--- /dev/null+++ b/plugins/discourse-ai/lib/completions/document_encoder.rb@@ -0,0 +1,179 @@+# frozen_string_literal: true++module DiscourseAi+ module Completions+ class DocumentEncoder+ extend UploadEncoding++ TEXT_CONVERTERS = {+ "doc" => DocToText,+ "docx" => DocxToText,+ "xls" => XlsToText,+ "xlsx" => XlsxToText,+ "odt" => OdtToText,+ "ods" => OdsToText,+ "rtf" => RtfToText,+ }.freeze++ PLAIN_TEXT_ATTACHMENT_TYPES = %w[csv md txt].freeze+ RAW_DOCUMENT_ATTACHMENT_TYPES = %w[pdf].freeze++ SUPPORTED_ATTACHMENT_TYPES =+ (TEXT_CONVERTERS.keys + PLAIN_TEXT_ATTACHMENT_TYPES + RAW_DOCUMENT_ATTACHMENT_TYPES).freeze++ UNKNOWN_ATTACHMENT_TYPE = "file"++ # order matters: the first fragment a mime type contains wins+ MIME_ATTACHMENT_TYPES = {+ "pdf" => "pdf",+ "wordprocessingml.document" => "docx",+ "application/msword" => "doc",+ "spreadsheetml.sheet" => "xlsx",+ "application/vnd.ms-excel" => "xls",+ "opendocument.text" => "odt",+ "opendocument.spreadsheet" => "ods",+ "csv" => "csv",+ "text/plain" => "txt",+ "rtf" => "rtf",+ "html" => "html",+ "markdown" => "md",+ }.freeze++ MAX_TEXT_FILE_BYTES = 1 * 1024 * 1024+ MAX_RAW_DOCUMENT_BYTES = 10 * 1024 * 1024++ def self.encode(upload, mime_type, attachment_type)+ path = fetch_path(upload)++ if path.blank?+ log_document_upload_skip(upload, attachment_type, "file is not available in the store")+ return+ end++ extract_text_payload(upload, path, attachment_type) ||+ raw_document_payload(upload, path, mime_type, attachment_type)+ end++ def self.attachment_type_for(extension, mime_type)+ ext = extension.to_s.delete_prefix(".").downcase+ ext = LlmModel::ATTACHMENT_TYPE_ALIASES.fetch(ext, ext)+ return ext if SUPPORTED_ATTACHMENT_TYPES.include?(ext)++ mime = mime_type.to_s.downcase+ MIME_ATTACHMENT_TYPES.find { |fragment, _| mime.include?(fragment) }&.last ||+ UNKNOWN_ATTACHMENT_TYPE+ end++ class << self+ private++ def extract_text_payload(upload, path, attachment_type)+ converter = TEXT_CONVERTERS[attachment_type]+ return if converter.nil? && PLAIN_TEXT_ATTACHMENT_TYPES.exclude?(attachment_type)++ raw = converter ? converter.convert(path) : read_utf8_text_file(path)+ text = normalize_extracted_text(raw)++ if text.blank?+ log_document_conversion_failure(+ upload,+ attachment_type,+ "#{attachment_type.upcase} converter returned blank output",+ )+ return+ end++ text_document_payload(upload, path, text, converted_from: attachment_type)+ rescue StandardError => e+ log_document_conversion_failure(upload, attachment_type, "#{e.class}: #{e.message}")+ nil+ end++ def raw_document_payload(upload, path, mime_type, attachment_type)+ if RAW_DOCUMENT_ATTACHMENT_TYPES.exclude?(attachment_type)+ log_document_upload_skip(+ upload,+ attachment_type,+ "raw upload is not supported for this attachment type; it must be converted to text",+ )+ return+ end++ bytesize = File.size(path)+ if bytesize > MAX_RAW_DOCUMENT_BYTES+ log_document_upload_skip(+ upload,+ attachment_type,+ "raw upload size #{ActiveSupport::NumberHelper.number_to_human_size(bytesize)} " \+ "exceeds the #{ActiveSupport::NumberHelper.number_to_human_size(MAX_RAW_DOCUMENT_BYTES)} limit",+ )+ return+ end++ {+ base64: Base64.strict_encode64(File.binread(path)),+ mime_type: mime_type,+ kind: :document,+ filename: upload.original_filename,+ }+ rescue SystemCallError => e+ log_document_upload_skip(upload, attachment_type, "#{e.class}: #{e.message}")+ nil+ end++ def read_utf8_text_file(path)+ text = File.binread(path, MAX_TEXT_FILE_BYTES + 1) || +""+ truncated = text.bytesize > MAX_TEXT_FILE_BYTES+ text = text.byteslice(0, MAX_TEXT_FILE_BYTES) if truncated++ text.force_encoding("UTF-8")+ return text if !truncated++ text ++ "\n\n[Document text truncated after #{ActiveSupport::NumberHelper.number_to_human_size(MAX_TEXT_FILE_BYTES)}.]"+ end++ def normalize_extracted_text(output)+ Encodings.force_utf8(output.to_s).strip+ end++ def truncate_extracted_text(text)+ return text if text.length <= TextNormalization::DOCUMENT_TEXT_BUDGET++ text.first(TextNormalization::DOCUMENT_TEXT_BUDGET) ++ "\n\n[Document text truncated after #{TextNormalization::DOCUMENT_TEXT_BUDGET} characters.]"+ end++ def text_document_payload(upload, path, text, converted_from:)+ {+ kind: :document,+ filename: upload.original_filename,+ mime_type: "text/plain",+ text: document_text_preamble(upload, path) + truncate_extracted_text(text),+ converted_from: converted_from,+ }+ end++ def document_text_preamble(upload, path)+ filename = upload.original_filename.presence || "document"+ filesize = upload.filesize || File.size(path)+ "Uploaded document: #{filename} (#{ActiveSupport::NumberHelper.number_to_human_size(filesize)})\n\n"+ end++ def log_document_conversion_failure(upload, extension, message)+ Rails.logger.warn(+ "Discourse AI: Failed to convert .#{extension} upload to text " \+ "(upload_id=#{upload.id}, filename=#{upload.original_filename.inspect}): #{message}",+ )+ end++ def log_document_upload_skip(upload, extension, message)+ Rails.logger.warn(+ "Discourse AI: Skipping .#{extension} upload " \+ "(upload_id=#{upload.id}, filename=#{upload.original_filename.inspect}): #{message}",+ )+ end+ end+ end+ end+enddiff --git a/plugins/discourse-ai/lib/completions/docx_to_text.rb b/plugins/discourse-ai/lib/completions/docx_to_text.rbindex d5d699f8d52..2aafc6d1322 100644--- a/plugins/discourse-ai/lib/completions/docx_to_text.rb+++ b/plugins/discourse-ai/lib/completions/docx_to_text.rb@@ -6,10 +6,11 @@ require "compression/safe_zip_reader"module DiscourseAimodule Completionsclass DocxToText+ include TextNormalization+MAX_ENTRY_XML_BYTES = 10 * 1024 * 1024MAX_TOTAL_XML_BYTES = 30 * 1024 * 1024MAX_ZIP_ENTRIES = 1_000- MAX_EXTRACTED_TEXT_CHARS = 100_001MAX_PARAGRAPHS = 20_000SUPPORTED_PARTS = %w[word/document.xml word/footnotes.xml word/endnotes.xml word/comments.xml]@@ -20,6 +21,8 @@ module DiscourseAiendclass Numbering+ include TextNormalization+Level = Struct.new(:num_format, :level_text, :start, keyword_init: true)BULLET_FORMAT = "bullet"@@ -269,10 +272,6 @@ module DiscourseAidef integer_value(value, default = 0)value.present? ? value.to_i : defaultend-- def force_utf8(text)- text.to_s.encode("UTF-8", invalid: :replace, undef: :replace, replace: "")- endenddef self.convert(path)@@ -414,10 +413,6 @@ module DiscourseAi.join(" - ")end- def normalize_inline_text(text)- force_utf8(text).gsub("\u00A0", " ").gsub(/\s+/, " ").strip- end-def normalize_paragraph_text(text)force_utf8(text).gsub("\u00A0", " ")@@ -426,19 +421,6 @@ module DiscourseAi.gsub(/\n[ \t]+/, "\n").rstripend-- def normalize_document_text(text)- force_utf8(text)- .gsub("\u00A0", " ")- .gsub(/\r\n?/, "\n")- .gsub(/[ \t]+\n/, "\n")- .gsub(/\n{3,}/, "\n\n")- .strip- end-- def force_utf8(text)- text.to_s.encode("UTF-8", invalid: :replace, undef: :replace, replace: "")- endendendenddiff --git a/plugins/discourse-ai/lib/completions/ods_to_text.rb b/plugins/discourse-ai/lib/completions/ods_to_text.rbindex 6608256fe2a..debc40838a6 100644--- a/plugins/discourse-ai/lib/completions/ods_to_text.rb+++ b/plugins/discourse-ai/lib/completions/ods_to_text.rb@@ -6,10 +6,11 @@ require "compression/safe_zip_reader"module DiscourseAimodule Completionsclass OdsToText+ include TextNormalization+MAX_ENTRY_XML_BYTES = 10 * 1024 * 1024MAX_TOTAL_XML_BYTES = 30 * 1024 * 1024MAX_ZIP_ENTRIES = 1_000- MAX_EXTRACTED_TEXT_CHARS = 100_001MAX_SHEETS = 50MAX_ROWS_PER_SHEET = 10_000MAX_COLUMNS = 200@@ -171,19 +172,6 @@ module DiscourseAidef normalize_inline(text)force_utf8(text).gsub("\u00A0", " ").gsub(/[ \t\r\n]+/, " ").stripend-- def normalize_document_text(text)- force_utf8(text)- .gsub("\u00A0", " ")- .gsub(/\r\n?/, "\n")- .gsub(/[ \t]+\n/, "\n")- .gsub(/\n{3,}/, "\n\n")- .strip- end-- def force_utf8(text)- text.to_s.encode("UTF-8", invalid: :replace, undef: :replace, replace: "")- endendendenddiff --git a/plugins/discourse-ai/lib/completions/odt_to_text.rb b/plugins/discourse-ai/lib/completions/odt_to_text.rbindex 60a77d29d09..9fb5e06eabb 100644--- a/plugins/discourse-ai/lib/completions/odt_to_text.rb+++ b/plugins/discourse-ai/lib/completions/odt_to_text.rb@@ -6,10 +6,11 @@ require "compression/safe_zip_reader"module DiscourseAimodule Completionsclass OdtToText+ include TextNormalization+MAX_ENTRY_XML_BYTES = 10 * 1024 * 1024MAX_TOTAL_XML_BYTES = 30 * 1024 * 1024MAX_ZIP_ENTRIES = 1_000- MAX_EXTRACTED_TEXT_CHARS = 100_001MAX_PARAGRAPHS = 20_000MAX_LIST_DEPTH = 20@@ -176,19 +177,6 @@ module DiscourseAidef normalize_inline(text)force_utf8(text).gsub("\u00A0", " ").gsub(/[ \t\r\n]+/, " ").stripend-- def normalize_document_text(text)- force_utf8(text)- .gsub("\u00A0", " ")- .gsub(/\r\n?/, "\n")- .gsub(/[ \t]+\n/, "\n")- .gsub(/\n{3,}/, "\n\n")- .strip- end-- def force_utf8(text)- text.to_s.encode("UTF-8", invalid: :replace, undef: :replace, replace: "")- endendendenddiff --git a/plugins/discourse-ai/lib/completions/prompt_messages_builder.rb b/plugins/discourse-ai/lib/completions/prompt_messages_builder.rbindex 2a56f441ca0..881077ddd3e 100644--- a/plugins/discourse-ai/lib/completions/prompt_messages_builder.rb+++ b/plugins/discourse-ai/lib/completions/prompt_messages_builder.rb@@ -360,7 +360,7 @@ module DiscourseAiMiniMime.lookup_by_filename(upload.original_filename)&.content_type ||"application/octet-stream"attachment_type =- DiscourseAi::Completions::UploadEncoder.attachment_type_for(upload.extension, mime_type)+ DiscourseAi::Completions::DocumentEncoder.attachment_type_for(upload.extension, mime_type)allowed_attachment_types.include?(attachment_type)enddiff --git a/plugins/discourse-ai/lib/completions/rtf_to_text.rb b/plugins/discourse-ai/lib/completions/rtf_to_text.rbindex 49d390cf2d3..9595f206ed8 100644--- a/plugins/discourse-ai/lib/completions/rtf_to_text.rb+++ b/plugins/discourse-ai/lib/completions/rtf_to_text.rb@@ -3,8 +3,13 @@module DiscourseAimodule Completionsclass RtfToText+ include TextNormalization++ def normalize_document_text(text)+ super.gsub(/\n[ \t]+/, "\n")+ end+MAX_INPUT_BYTES = 2 * 1024 * 1024- MAX_EXTRACTED_TEXT_CHARS = 100_001MAX_GROUP_DEPTH = 100MAX_CONTROL_WORD_CHARS = 64@@ -387,18 +392,6 @@ module DiscourseAidef digit?(byte)byte >= 48 && byte <= 57end-- def normalize_document_text(text)- text- .to_s- .encode("UTF-8", invalid: :replace, undef: :replace, replace: "")- .gsub("\u00A0", " ")- .gsub(/\r\n?/, "\n")- .gsub(/[ \t]+\n/, "\n")- .gsub(/\n[ \t]+/, "\n")- .gsub(/\n{3,}/, "\n\n")- .strip- endendendenddiff --git a/plugins/discourse-ai/lib/completions/text_normalization.rb b/plugins/discourse-ai/lib/completions/text_normalization.rbnew file mode 100644index 00000000000..79b1c1af97b--- /dev/null+++ b/plugins/discourse-ai/lib/completions/text_normalization.rb@@ -0,0 +1,31 @@+# frozen_string_literal: true++module DiscourseAi+ module Completions+ module TextNormalization+ DOCUMENT_TEXT_BUDGET = 100_000++ # one past the budget, so DocumentEncoder#truncate_extracted_text can tell a document+ # was cut short and say so in the prompt+ MAX_EXTRACTED_TEXT_CHARS = DOCUMENT_TEXT_BUDGET + 1++ def force_utf8(text)+ ::Encodings.force_utf8(text.to_s)+ end++ def normalize_document_text(text)+ force_utf8(text)+ .first(MAX_EXTRACTED_TEXT_CHARS)+ .gsub("\u00A0", " ")+ .gsub(/\r\n?/, "\n")+ .gsub(/[ \t]+\n/, "\n")+ .gsub(/\n{3,}/, "\n\n")+ .strip+ end++ def normalize_inline_text(text)+ force_utf8(text).gsub("\u00A0", " ").gsub(/\s+/, " ").strip+ end+ end+ end+enddiff --git a/plugins/discourse-ai/lib/completions/upload_encoder.rb b/plugins/discourse-ai/lib/completions/upload_encoder.rbindex e9d8d49a2a4..0600110b9f0 100644--- a/plugins/discourse-ai/lib/completions/upload_encoder.rb+++ b/plugins/discourse-ai/lib/completions/upload_encoder.rb@@ -3,6 +3,8 @@module DiscourseAimodule Completionsclass UploadEncoder+ extend UploadEncoding+def self.image_upload?(upload)mime_type = MiniMime.lookup_by_filename(upload.original_filename)&.content_typemime_type&.start_with?("image/") == true@@ -18,11 +20,10 @@ module DiscourseAiallowed_kinds: [:image],allowed_attachment_types: nil)- uploads = []allowed_attachment_types = normalize_attachment_types(allowed_attachment_types)uploads_by_id = Upload.where(id: upload_ids).index_by(&:id)- upload_ids.each do |upload_id|+ upload_ids.filter_map do |upload_id|upload = uploads_by_id[upload_id]next if upload.blank?@@ -36,12 +37,10 @@ module DiscourseAiMiniMime.lookup_by_filename(upload.original_filename)&.content_type ||"application/octet-stream"- attachment_type = attachment_type_for(upload.extension, mime_type)- next if disallowed_attachment?(allowed_attachment_types, attachment_type)+ attachment_type = DocumentEncoder.attachment_type_for(upload.extension, mime_type)+ next if allowed_attachment_types&.exclude?(attachment_type)- payload = encode_document(upload, mime_type, attachment_type)- uploads << payload if payload- next+ next DocumentEncoder.encode(upload, mime_type, attachment_type)endnext if upload.width.to_i == 0 || upload.height.to_i == 0@@ -54,35 +53,8 @@ module DiscourseAi# this keeps it very simple format wise given everyone supports png and jpgnext if !%w[jpeg png].include?(desired_extension)- payload = encode_image(upload, desired_extension, max_pixels)- uploads << payload if payload+ encode_image(upload, desired_extension, max_pixels)end- uploads- end-- MAX_EXTRACTED_DOCUMENT_TEXT_CHARS = 100_000- MAX_TEXT_FILE_BYTES = 1 * 1024 * 1024- MAX_RAW_DOCUMENT_BYTES = 10 * 1024 * 1024- RAW_DOCUMENT_ATTACHMENT_TYPES = %w[pdf]-- def self.attachment_type_for(extension, mime_type)- ext = extension.to_s.delete_prefix(".").downcase- mime = mime_type.to_s.downcase-- return "pdf" if ext == "pdf" || mime.include?("pdf")- return "docx" if ext == "docx" || mime.include?("wordprocessingml.document")- return "doc" if ext == "doc" || mime == "application/msword"- return "xlsx" if ext == "xlsx" || mime.include?("spreadsheetml.sheet")- return "xls" if ext == "xls" || mime == "application/vnd.ms-excel"- return "odt" if ext == "odt" || mime.include?("opendocument.text")- return "ods" if ext == "ods" || mime.include?("opendocument.spreadsheet")- return "csv" if ext == "csv" || mime.include?("text/csv") || mime.include?("csv")- return "txt" if ext == "txt" || mime.include?("text/plain")- return "rtf" if ext == "rtf" || mime.include?("rtf")- return "html" if %w[html htm].include?(ext) || mime.include?("html")- return "md" if %w[md markdown].include?(ext) || mime.include?("markdown")-- "file"endclass << self@@ -94,271 +66,10 @@ module DiscourseAiLlmModel.normalize_attachment_types(types)end- def disallowed_attachment?(allowed_types, attachment_type)- !allowed_types.nil? && !allowed_types.include?(attachment_type)- end-def image_extension?(ext)%w[jpg jpeg png gif webp].include?(ext)end- def encode_document(upload, mime_type, attachment_type)- path = fetch_path(upload)- return if path.blank?-- if attachment_type == "doc"- text_payload = doc_to_text_payload(upload, path)- return text_payload if text_payload- elsif attachment_type == "docx"- text_payload = docx_to_text_payload(upload, path)- return text_payload if text_payload- elsif attachment_type == "xls"- text_payload = xls_to_text_payload(upload, path)- return text_payload if text_payload- elsif attachment_type == "xlsx"- text_payload = xlsx_to_text_payload(upload, path)- return text_payload if text_payload- elsif attachment_type == "odt"- text_payload = odt_to_text_payload(upload, path)- return text_payload if text_payload- elsif attachment_type == "ods"- text_payload = ods_to_text_payload(upload, path)- return text_payload if text_payload- elsif attachment_type == "rtf"- text_payload = rtf_to_text_payload(upload, path)- return text_payload if text_payload- elsif %w[csv md txt].include?(attachment_type)- text_payload = text_file_payload(upload, path, attachment_type)- return text_payload if text_payload- end-- raw_document_payload(upload, path, mime_type, attachment_type)- end-- def raw_document_payload(upload, path, mime_type, attachment_type)- if RAW_DOCUMENT_ATTACHMENT_TYPES.exclude?(attachment_type)- log_document_upload_skip(- upload,- attachment_type,- "raw upload is not supported for this attachment type; it must be converted to text",- )- return- end-- bytesize = File.size(path)- if bytesize > MAX_RAW_DOCUMENT_BYTES- log_document_upload_skip(- upload,- attachment_type,- "raw upload size #{human_filesize(bytesize)} exceeds the #{human_filesize(MAX_RAW_DOCUMENT_BYTES)} limit",- )- return- end-- {- base64: Base64.strict_encode64(File.binread(path)),- mime_type: mime_type,- kind: :document,- filename: upload.original_filename,- }- rescue SystemCallError => e- log_document_upload_skip(upload, attachment_type, "#{e.class}: #{e.message}")- nil- end-- def doc_to_text_payload(upload, path)- text = normalize_extracted_text(DiscourseAi::Completions::DocToText.convert(path))-- if text.blank?- log_document_conversion_failure(upload, "doc", "DOC converter returned blank output")- return- end-- text_document_payload(upload, path, text, converted_from: "doc")- rescue StandardError => e- log_document_conversion_failure(upload, "doc", "#{e.class}: #{e.message}")- nil- end-- def docx_to_text_payload(upload, path)- text = normalize_extracted_text(DiscourseAi::Completions::DocxToText.convert(path))-- if text.blank?- log_document_conversion_failure(upload, "docx", "DOCX converter returned blank output")- return- end-- text_document_payload(upload, path, text, converted_from: "docx")- rescue StandardError => e- log_document_conversion_failure(upload, "docx", "#{e.class}: #{e.message}")- nil- end-- def xls_to_text_payload(upload, path)- text = normalize_extracted_text(DiscourseAi::Completions::XlsToText.convert(path))-- if text.blank?- log_document_conversion_failure(upload, "xls", "XLS converter returned blank output")- return- end-- text_document_payload(upload, path, text, converted_from: "xls")- rescue StandardError => e- log_document_conversion_failure(upload, "xls", "#{e.class}: #{e.message}")- nil- end-- def xlsx_to_text_payload(upload, path)- text = normalize_extracted_text(DiscourseAi::Completions::XlsxToText.convert(path))-- if text.blank?- log_document_conversion_failure(upload, "xlsx", "XLSX converter returned blank output")- return- end-- text_document_payload(upload, path, text, converted_from: "xlsx")- rescue StandardError => e- log_document_conversion_failure(upload, "xlsx", "#{e.class}: #{e.message}")- nil- end-- def odt_to_text_payload(upload, path)- text = normalize_extracted_text(DiscourseAi::Completions::OdtToText.convert(path))-- if text.blank?- log_document_conversion_failure(upload, "odt", "ODT converter returned blank output")- return- end-- text_document_payload(upload, path, text, converted_from: "odt")- rescue StandardError => e- log_document_conversion_failure(upload, "odt", "#{e.class}: #{e.message}")- nil- end-- def ods_to_text_payload(upload, path)- text = normalize_extracted_text(DiscourseAi::Completions::OdsToText.convert(path))-- if text.blank?- log_document_conversion_failure(upload, "ods", "ODS converter returned blank output")- return- end-- text_document_payload(upload, path, text, converted_from: "ods")- rescue StandardError => e- log_document_conversion_failure(upload, "ods", "#{e.class}: #{e.message}")- nil- end-- def text_file_payload(upload, path, attachment_type)- text = normalize_extracted_text(read_utf8_text_file(path))-- if text.blank?- log_document_conversion_failure(upload, attachment_type, "text file was blank")- return- end-- text_document_payload(upload, path, text, converted_from: attachment_type)- rescue SystemCallError => e- log_document_conversion_failure(upload, attachment_type, "#{e.class}: #{e.message}")- nil- end-- def rtf_to_text_payload(upload, path)- text = normalize_extracted_text(DiscourseAi::Completions::RtfToText.convert(path))-- if text.blank?- log_document_conversion_failure(upload, "rtf", "RTF converter returned blank output")- return- end-- text_document_payload(upload, path, text, converted_from: "rtf")- rescue StandardError => e- log_document_conversion_failure(upload, "rtf", "#{e.class}: #{e.message}")- nil- end-- def read_utf8_text_file(path)- text = +""- truncated = false-- File.open(path, "rb") do |file|- text = file.read(MAX_TEXT_FILE_BYTES + 1).to_s- if text.bytesize > MAX_TEXT_FILE_BYTES- text = text.byteslice(0, MAX_TEXT_FILE_BYTES)- truncated = true- end- end-- text = text.delete_prefix("\xEF\xBB\xBF".b)- text.force_encoding("UTF-8")- text = text.encode("UTF-8", invalid: :replace, undef: :replace, replace: "")-- if truncated- text << "\n\n[Document text truncated after #{human_filesize(MAX_TEXT_FILE_BYTES)}.]"- end-- text- end-- def normalize_extracted_text(output)- output.to_s.encode("UTF-8", invalid: :replace, undef: :replace, replace: "").strip- end-- def truncate_extracted_text(text)- return text if text.length <= MAX_EXTRACTED_DOCUMENT_TEXT_CHARS-- text.first(MAX_EXTRACTED_DOCUMENT_TEXT_CHARS) +- "\n\n[Document text truncated after #{MAX_EXTRACTED_DOCUMENT_TEXT_CHARS} characters.]"- end-- def text_document_payload(upload, path, text, converted_from:)- {- kind: :document,- filename: upload.original_filename,- mime_type: "text/plain",- text: document_text_preamble(upload, path) + truncate_extracted_text(text),- converted_from: converted_from,- }- end-- def document_text_preamble(upload, path)- filename = upload.original_filename.presence || "document"- filesize = upload.filesize || File.size(path)- "Uploaded document: #{filename} (#{human_filesize(filesize)})\n\n"- end-- def human_filesize(bytes)- bytes = bytes.to_i- units = %w[Bytes KB MB GB TB]- size = bytes.to_f- unit = units.shift-- while size >= 1024 && units.any?- size /= 1024.0- unit = units.shift- end-- return "#{bytes} #{bytes == 1 ? "Byte" : "Bytes"}" if unit == "Bytes"-- formatted_size = size >= 10 ? size.round.to_s : format("%.1f", size).sub(/\.0\z/, "")- "#{formatted_size} #{unit}"- end-- def log_document_conversion_failure(upload, extension, message)- Rails.logger.warn(- "Discourse AI: Failed to convert .#{extension} upload to text " \- "(upload_id=#{upload.id}, filename=#{upload.original_filename.inspect}): #{message}",- )- end-- def log_document_upload_skip(upload, extension, message)- Rails.logger.warn(- "Discourse AI: Skipping .#{extension} upload " \- "(upload_id=#{upload.id}, filename=#{upload.original_filename.inspect}): #{message}",- )- end-def encode_image(upload, desired_extension, max_pixels)original_pixels = upload.width * upload.height@@ -392,16 +103,6 @@ module DiscourseAifilename: upload.original_filename,}end-- def fetch_path(upload)- path = Discourse.store.path_for(upload)- path = Discourse.store.download(upload) if path.blank?-- return if path.blank?- return unless File.exist?(path)-- path- endendendenddiff --git a/plugins/discourse-ai/lib/completions/upload_encoding.rb b/plugins/discourse-ai/lib/completions/upload_encoding.rbnew file mode 100644index 00000000000..14d0125f7f8--- /dev/null+++ b/plugins/discourse-ai/lib/completions/upload_encoding.rb@@ -0,0 +1,18 @@+# frozen_string_literal: true++module DiscourseAi+ module Completions+ # plumbing shared by the image and document halves of UploadEncoder+ module UploadEncoding+ def fetch_path(upload)+ path = Discourse.store.path_for(upload)+ path = Discourse.store.download(upload) if path.blank?++ return if path.blank?+ return unless File.exist?(path)++ path+ end+ end+ end+enddiff --git a/plugins/discourse-ai/lib/completions/xls_to_text.rb b/plugins/discourse-ai/lib/completions/xls_to_text.rbindex a434cc489dc..fe557864864 100644--- a/plugins/discourse-ai/lib/completions/xls_to_text.rb+++ b/plugins/discourse-ai/lib/completions/xls_to_text.rb@@ -6,7 +6,7 @@ module DiscourseAimodule Completionsclass XlsToTextXLS2CSV_TIMEOUT_SECONDS = 5- MAX_CONVERSION_OUTPUT_BYTES = 4 * 100_001+ MAX_CONVERSION_OUTPUT_BYTES = 4 * TextNormalization::MAX_EXTRACTED_TEXT_CHARSSAFE_EXEC_ENV = { "PATH" => ENV["PATH"].to_s }.freezeXLS2CSV_RLIMITS = {cpu_seconds: XLS2CSV_TIMEOUT_SECONDS,diff --git a/plugins/discourse-ai/lib/completions/xlsx_to_text.rb b/plugins/discourse-ai/lib/completions/xlsx_to_text.rbindex 69aad720533..d69932cf9af 100644--- a/plugins/discourse-ai/lib/completions/xlsx_to_text.rb+++ b/plugins/discourse-ai/lib/completions/xlsx_to_text.rb@@ -7,10 +7,11 @@ require "compression/safe_zip_reader"module DiscourseAimodule Completionsclass XlsxToText+ include TextNormalization+MAX_ENTRY_XML_BYTES = 10 * 1024 * 1024MAX_TOTAL_XML_BYTES = 30 * 1024 * 1024MAX_ZIP_ENTRIES = 1_000- MAX_EXTRACTED_TEXT_CHARS = 100_001MAX_SHEETS = 50MAX_ROWS_PER_SHEET = 10_000MAX_COLUMNS = 200@@ -242,23 +243,6 @@ module DiscourseAicolumn_name.upcase.each_byte.reduce(0) { |index, char| (index * 26) + char - "A".ord + 1 } -1end-- def normalize_inline_text(text)- force_utf8(text).gsub("\u00A0", " ").gsub(/[ \t\r\n]+/, " ").strip- end-- def normalize_document_text(text)- force_utf8(text)- .gsub("\u00A0", " ")- .gsub(/\r\n?/, "\n")- .gsub(/[ \t]+\n/, "\n")- .gsub(/\n{3,}/, "\n\n")- .strip- end-- def force_utf8(text)- text.to_s.encode("UTF-8", invalid: :replace, undef: :replace, replace: "")- endendendenddiff --git a/plugins/discourse-ai/spec/lib/completions/document_encoder_spec.rb b/plugins/discourse-ai/spec/lib/completions/document_encoder_spec.rbnew file mode 100644index 00000000000..a139377b9bc--- /dev/null+++ b/plugins/discourse-ai/spec/lib/completions/document_encoder_spec.rb@@ -0,0 +1,46 @@+# frozen_string_literal: true++RSpec.describe DiscourseAi::Completions::DocumentEncoder do+ before { enable_current_plugin }++ describe ".attachment_type_for" do+ {+ "pdf" => "pdf",+ "docx" => "docx",+ "doc" => "doc",+ "xlsx" => "xlsx",+ "xls" => "xls",+ "odt" => "odt",+ "ods" => "ods",+ "csv" => "csv",+ "txt" => "txt",+ "rtf" => "rtf",+ "html" => "html",+ "htm" => "html",+ "md" => "md",+ "markdown" => "md",+ }.each do |extension, expected|+ it "maps .#{extension} to #{expected}" do+ mime = MiniMime.lookup_by_filename("sample.#{extension}")&.content_type.to_s++ expect(described_class.attachment_type_for(extension, mime)).to eq(expected)+ expect(described_class.attachment_type_for(".#{extension.upcase}", mime)).to eq(expected)+ end+ end++ it "falls back to the mime type when the extension is unknown" do+ expect(described_class.attachment_type_for("", "application/pdf")).to eq("pdf")+ expect(described_class.attachment_type_for("bin", "text/plain")).to eq("txt")+ expect(described_class.attachment_type_for("bin", "application/vnd.ms-excel")).to eq("xls")+ end++ it "prefers a known extension over the mime type" do+ expect(described_class.attachment_type_for("txt", "application/pdf")).to eq("txt")+ end++ it "returns the unknown sentinel for anything else" do+ expect(described_class.attachment_type_for("zip", "application/zip")).to eq("file")+ expect(described_class.attachment_type_for("", "")).to eq("file")+ end+ end+enddiff --git a/plugins/discourse-ai/spec/lib/completions/docx_to_text_spec.rb b/plugins/discourse-ai/spec/lib/completions/docx_to_text_spec.rbindex b58353661bf..6d744f4cd74 100644--- a/plugins/discourse-ai/spec/lib/completions/docx_to_text_spec.rb+++ b/plugins/discourse-ai/spec/lib/completions/docx_to_text_spec.rb@@ -1,24 +1,6 @@# frozen_string_literal: trueRSpec.describe DiscourseAi::Completions::DocxToText do- def with_docx(entries)- tempfile = Tempfile.new(%w[document .docx])- path = tempfile.path- tempfile.close- FileUtils.rm_f(path)-- ::Zip::File.open(path, create: true) do |zip_file|- entries.each do |name, content|- zip_file.get_output_stream(name) { |stream| stream.write(content) }- end- end-- yield path- ensure- tempfile&.close- FileUtils.rm_f(path) if path- end-def word_xml(body)<<~XML<?xml version="1.0" encoding="UTF-8" standalone="yes"?>@@ -75,7 +57,7 @@ RSpec.describe DiscourseAi::Completions::DocxToText doendit "extracts paragraph text from the main document" do- with_docx("word/document.xml" => word_xml(<<~XML)) do |path|+ with_zipped_document("docx", "word/document.xml" => word_xml(<<~XML)) do |path|<w:body><w:p><w:r><w:t>Hello</w:t></w:r>@@ -94,7 +76,8 @@ RSpec.describe DiscourseAi::Completions::DocxToText doit "extracts supported text parts in a stable order" dopart = ->(text) { word_xml("<w:p><w:r><w:t>#{text}</w:t></w:r></w:p>") }- with_docx(+ with_zipped_document(+ "docx","word/footer1.xml" => part.call("Footer"),"word/header2.xml" => part.call("Header two"),"word/document.xml" => part.call("Body"),@@ -110,7 +93,7 @@ RSpec.describe DiscourseAi::Completions::DocxToText doendit "extracts image alt text inline" do- with_docx("word/document.xml" => word_xml(<<~XML)) do |path|+ with_zipped_document("docx", "word/document.xml" => word_xml(<<~XML)) do |path|<w:body><w:p><w:r><w:t>See </w:t></w:r>@@ -132,7 +115,8 @@ RSpec.describe DiscourseAi::Completions::DocxToText doendit "adds prefixes for numbered and bulleted lists" do- with_docx(+ with_zipped_document(+ "docx","word/numbering.xml" => numbering_xml,"word/document.xml" => word_xml(<<~XML),<w:body>@@ -151,7 +135,7 @@ RSpec.describe DiscourseAi::Completions::DocxToText doendit "returns blank text when the docx has no supported text parts" do- with_docx("docProps/core.xml" => "<properties />") do |path|+ with_zipped_document("docx", "docProps/core.xml" => "<properties />") do |path|expect(described_class.convert(path)).to eq("")endenddiff --git a/plugins/discourse-ai/spec/lib/completions/ods_to_text_spec.rb b/plugins/discourse-ai/spec/lib/completions/ods_to_text_spec.rbindex ea4968acee8..b05199e801b 100644--- a/plugins/discourse-ai/spec/lib/completions/ods_to_text_spec.rb+++ b/plugins/discourse-ai/spec/lib/completions/ods_to_text_spec.rb@@ -1,24 +1,6 @@# frozen_string_literal: trueRSpec.describe DiscourseAi::Completions::OdsToText do- def with_ods(entries)- tempfile = Tempfile.new(%w[spreadsheet .ods])- path = tempfile.path- tempfile.close- FileUtils.rm_f(path)-- ::Zip::File.open(path, create: true) do |zip_file|- entries.each do |name, content|- zip_file.get_output_stream(name) { |stream| stream.write(content) }- end- end-- yield path- ensure- tempfile&.close- FileUtils.rm_f(path) if path- end-def ods_content(body)<<~XML<?xml version="1.0" encoding="UTF-8"?>@@ -36,7 +18,7 @@ RSpec.describe DiscourseAi::Completions::OdsToText doendit "extracts cells from each named sheet" do- with_ods("content.xml" => ods_content(<<~XML)) do |path|+ with_zipped_document("ods", "content.xml" => ods_content(<<~XML)) do |path|<table:table table:name="Summary"><table:table-row><table:table-cell office:value-type="string"><text:p>Name</text:p></table:table-cell>@@ -60,7 +42,7 @@ RSpec.describe DiscourseAi::Completions::OdsToText doendit "uses Sheet1, Sheet2 ... when no name is set" do- with_ods("content.xml" => ods_content(<<~XML)) do |path|+ with_zipped_document("ods", "content.xml" => ods_content(<<~XML)) do |path|<table:table><table:table-row><table:table-cell office:value-type="string"><text:p>only</text:p></table:table-cell>@@ -72,7 +54,7 @@ RSpec.describe DiscourseAi::Completions::OdsToText doendit "renders typed values when no inline paragraph is present" do- with_ods("content.xml" => ods_content(<<~XML)) do |path|+ with_zipped_document("ods", "content.xml" => ods_content(<<~XML)) do |path|<table:table table:name="Types"><table:table-row><table:table-cell office:value-type="boolean" office:boolean-value="true"/>@@ -86,7 +68,7 @@ RSpec.describe DiscourseAi::Completions::OdsToText doendit "expands a non-empty number-columns-repeated up to MAX_COLUMNS" do- with_ods("content.xml" => ods_content(<<~XML)) do |path|+ with_zipped_document("ods", "content.xml" => ods_content(<<~XML)) do |path|<table:table table:name="Repeats"><table:table-row><table:table-cell office:value-type="string" table:number-columns-repeated="3">@@ -101,7 +83,7 @@ RSpec.describe DiscourseAi::Completions::OdsToText doendit "ignores covered (merge-continuation) cells and trims trailing empties" do- with_ods("content.xml" => ods_content(<<~XML)) do |path|+ with_zipped_document("ods", "content.xml" => ods_content(<<~XML)) do |path|<table:table table:name="Merge"><table:table-row><table:table-cell office:value-type="string"><text:p>head</text:p></table:table-cell>@@ -115,7 +97,7 @@ RSpec.describe DiscourseAi::Completions::OdsToText doendit "returns blank text when content.xml is missing" do- with_ods("META-INF/manifest.xml" => "<manifest/>") do |path|+ with_zipped_document("ods", "META-INF/manifest.xml" => "<manifest/>") do |path|expect(described_class.convert(path)).to eq("")endenddiff --git a/plugins/discourse-ai/spec/lib/completions/odt_to_text_spec.rb b/plugins/discourse-ai/spec/lib/completions/odt_to_text_spec.rbindex 45c059e093b..88e0bdfa7e2 100644--- a/plugins/discourse-ai/spec/lib/completions/odt_to_text_spec.rb+++ b/plugins/discourse-ai/spec/lib/completions/odt_to_text_spec.rb@@ -1,24 +1,6 @@# frozen_string_literal: trueRSpec.describe DiscourseAi::Completions::OdtToText do- def with_odt(entries)- tempfile = Tempfile.new(%w[document .odt])- path = tempfile.path- tempfile.close- FileUtils.rm_f(path)-- ::Zip::File.open(path, create: true) do |zip_file|- entries.each do |name, content|- zip_file.get_output_stream(name) { |stream| stream.write(content) }- end- end-- yield path- ensure- tempfile&.close- FileUtils.rm_f(path) if path- end-def odt_content(body)<<~XML<?xml version="1.0" encoding="UTF-8"?>@@ -37,7 +19,7 @@ RSpec.describe DiscourseAi::Completions::OdtToText doendit "extracts headings, paragraphs, tabs, line breaks, and spans" do- with_odt("content.xml" => odt_content(<<~XML)) do |path|+ with_zipped_document("odt", "content.xml" => odt_content(<<~XML)) do |path|<text:h text:outline-level="1">Title</text:h><text:p>Hello<text:tab/>world<text:line-break/>second line</text:p><text:p>Run with <text:span>nested <text:span>span</text:span></text:span> text</text:p>@@ -49,7 +31,7 @@ RSpec.describe DiscourseAi::Completions::OdtToText doendit "expands repeated spaces from text:s c=N" do- with_odt("content.xml" => odt_content(<<~XML)) do |path|+ with_zipped_document("odt", "content.xml" => odt_content(<<~XML)) do |path|<text:p>A<text:s text:c="3"/>B</text:p>XMLexpect(described_class.convert(path)).to eq("A B")@@ -57,7 +39,7 @@ RSpec.describe DiscourseAi::Completions::OdtToText doendit "renders nested lists with depth-aware bullet prefixes" do- with_odt("content.xml" => odt_content(<<~XML)) do |path|+ with_zipped_document("odt", "content.xml" => odt_content(<<~XML)) do |path|<text:list><text:list-item><text:p>One</text:p></text:list-item><text:list-item>@@ -75,7 +57,7 @@ RSpec.describe DiscourseAi::Completions::OdtToText doendit "renders tables as tab-separated rows" do- with_odt("content.xml" => odt_content(<<~XML)) do |path|+ with_zipped_document("odt", "content.xml" => odt_content(<<~XML)) do |path|<table:table><table:table-row><table:table-cell><text:p>Name</text:p></table:table-cell>@@ -92,7 +74,7 @@ RSpec.describe DiscourseAi::Completions::OdtToText doendit "walks into frames and text-boxes for callout text" do- with_odt("content.xml" => odt_content(<<~XML)) do |path|+ with_zipped_document("odt", "content.xml" => odt_content(<<~XML)) do |path|<text:p>Body before</text:p><draw:frame><draw:text-box>@@ -106,7 +88,7 @@ RSpec.describe DiscourseAi::Completions::OdtToText doendit "returns blank text when content.xml is missing" do- with_odt("META-INF/manifest.xml" => "<manifest/>") do |path|+ with_zipped_document("odt", "META-INF/manifest.xml" => "<manifest/>") do |path|expect(described_class.convert(path)).to eq("")endenddiff --git a/plugins/discourse-ai/spec/lib/completions/rtf_to_text_spec.rb b/plugins/discourse-ai/spec/lib/completions/rtf_to_text_spec.rbindex 4afa84d919b..70928e0679a 100644--- a/plugins/discourse-ai/spec/lib/completions/rtf_to_text_spec.rb+++ b/plugins/discourse-ai/spec/lib/completions/rtf_to_text_spec.rb@@ -1,19 +1,8 @@# frozen_string_literal: trueRSpec.describe DiscourseAi::Completions::RtfToText do- def with_rtf(contents)- tempfile = Tempfile.new(%w[document .rtf])- tempfile.binmode- tempfile.write(contents)- tempfile.rewind-- yield tempfile.path- ensure- tempfile&.close!- end-it "extracts formatted body text" do- with_rtf(<<~'RTF') do |path|+ with_document_file("rtf", <<~'RTF') do |path|{\rtf1\ansi\ansicpg1252 This is {\b bold}\par Caf\'e9\par Unicode \u8212? dash\tab done}RTFexpect(described_class.convert(path)).to eq("This is bold\nCafé\nUnicode — dash\tdone")@@ -21,13 +10,15 @@ RSpec.describe DiscourseAi::Completions::RtfToText doendit "ignores formatting tables and embedded binary destinations" do- with_rtf(<<~'RTF') { |path| expect(described_class.convert(path)).to eq("Visible\nMore text") }+ with_document_file("rtf", <<~'RTF') do |path|{\rtf1\ansi{\fonttbl{\f0 Arial;}}{\colortbl;\red255\green0\blue0;}{\pict\pngblip abcdef}Visible\par More text}RTF+ expect(described_class.convert(path)).to eq("Visible\nMore text")+ endendit "preserves escaped literal braces and backslashes" do- with_rtf(<<~'RTF') do |path|+ with_document_file("rtf", <<~'RTF') do |path|{\rtf1\ansi literal \{brace\} and \\slash}RTFexpect(described_class.convert(path)).to eq("literal {brace} and \\slash")diff --git a/plugins/discourse-ai/spec/lib/completions/text_normalization_spec.rb b/plugins/discourse-ai/spec/lib/completions/text_normalization_spec.rbnew file mode 100644index 00000000000..9907020a9fd--- /dev/null+++ b/plugins/discourse-ai/spec/lib/completions/text_normalization_spec.rb@@ -0,0 +1,39 @@+# frozen_string_literal: true++RSpec.describe DiscourseAi::Completions::TextNormalization do+ subject(:normalizer) { Class.new { include DiscourseAi::Completions::TextNormalization }.new }++ before { enable_current_plugin }++ describe "#force_utf8" do+ it "scrubs invalid bytes" do+ expect(normalizer.force_utf8("caf\xFFe".dup.force_encoding("UTF-8"))).to eq("cafe")+ end++ it "strips a byte order mark" do+ expect(normalizer.force_utf8("\xEF\xBB\xBFhello".dup.force_encoding("UTF-8"))).to eq("hello")+ end++ it "tolerates nil" do+ expect(normalizer.force_utf8(nil)).to eq("")+ end+ end++ describe "#normalize_document_text" do+ it "collapses whitespace and trims" do+ expect(normalizer.normalize_document_text(" a b\r\nc \n\n\n\nd ")).to eq("a b\nc\n\nd")+ end++ it "caps the output one character past the budget, so the encoder can detect truncation" do+ text = normalizer.normalize_document_text("a" * (described_class::DOCUMENT_TEXT_BUDGET + 50))++ expect(text.length).to eq(described_class::DOCUMENT_TEXT_BUDGET + 1)+ end+ end++ describe "#normalize_inline_text" do+ it "flattens all whitespace to single spaces" do+ expect(normalizer.normalize_inline_text(" a b\n\tc ")).to eq("a b c")+ end+ end+enddiff --git a/plugins/discourse-ai/spec/lib/completions/upload_encoder_spec.rb b/plugins/discourse-ai/spec/lib/completions/upload_encoder_spec.rbindex 9085ba750e4..39b47f74c57 100644--- a/plugins/discourse-ai/spec/lib/completions/upload_encoder_spec.rb+++ b/plugins/discourse-ai/spec/lib/completions/upload_encoder_spec.rb@@ -10,6 +10,17 @@ RSpec.describe DiscourseAi::Completions::UploadEncoder dobefore { enable_current_plugin }+ MAX_PIXELS = 1_048_576++ def encode_document(upload, attachment_type)+ described_class.encode(+ upload_ids: [upload.id],+ max_pixels: MAX_PIXELS,+ allowed_kinds: %i[document],+ allowed_attachment_types: Array(attachment_type),+ )+ end+def create_doc_upload(contents: "raw doc bytes", filename: "sample.doc")extension = File.extname(filename)tempfile = Tempfile.new([File.basename(filename, extension), extension.presence || ".upload"])@@ -22,40 +33,6 @@ RSpec.describe DiscourseAi::Completions::UploadEncoder dotempfile&.close!end- def zip_bytes(entries, extension)- tempfile = Tempfile.new(["sample", extension])- path = tempfile.path- tempfile.close- FileUtils.rm_f(path)-- ::Zip::File.open(path, create: true) do |zip_file|- entries.each do |name, content|- zip_file.get_output_stream(name) { |stream| stream.write(content) }- end- end-- File.binread(path)- ensure- tempfile&.close- FileUtils.rm_f(path) if path- end-- def docx_bytes(entries)- zip_bytes(entries, ".docx")- end-- def xlsx_bytes(entries)- zip_bytes(entries, ".xlsx")- end-- def odt_bytes(entries)- zip_bytes(entries, ".odt")- end-- def ods_bytes(entries)- zip_bytes(entries, ".ods")- end-def odt_content_xml(text)<<~XML<?xml version="1.0" encoding="UTF-8"?>@@ -104,7 +81,7 @@ RSpec.describe DiscourseAi::Completions::UploadEncoder doit "automatically converts gifs to pngs" doupload = UploadCreator.new(gif, "1x1.gif").create_for(Discourse.system_user.id)- encoded = described_class.encode(upload_ids: [upload.id], max_pixels: 1_048_576)+ encoded = described_class.encode(upload_ids: [upload.id], max_pixels: MAX_PIXELS)expect(encoded.length).to eq(1)expect(encoded[0][:base64]).to be_presentexpect(encoded[0][:mime_type]).to eq("image/png")@@ -112,7 +89,7 @@ RSpec.describe DiscourseAi::Completions::UploadEncoder doit "automatically converts webp to pngs" doupload = UploadCreator.new(webp, "1x1.webp").create_for(Discourse.system_user.id)- encoded = described_class.encode(upload_ids: [upload.id], max_pixels: 1_048_576)+ encoded = described_class.encode(upload_ids: [upload.id], max_pixels: MAX_PIXELS)expect(encoded.length).to eq(1)expect(encoded[0][:base64]).to be_presentexpect(encoded[0][:mime_type]).to eq("image/png")@@ -129,7 +106,7 @@ RSpec.describe DiscourseAi::Completions::UploadEncoder doit "supports jpg" doupload = UploadCreator.new(jpg, "1x1.jpg").create_for(Discourse.system_user.id)- encoded = described_class.encode(upload_ids: [upload.id], max_pixels: 1_048_576)+ encoded = described_class.encode(upload_ids: [upload.id], max_pixels: MAX_PIXELS)expect(encoded.length).to eq(1)expect(encoded[0][:base64]).to be_presentexpect(encoded[0][:mime_type]).to eq("image/jpeg")@@ -138,7 +115,7 @@ RSpec.describe DiscourseAi::Completions::UploadEncoder doit "does not raise when an upload no longer exists" domissing_id = Upload.maximum(:id).to_i + 1- expect(described_class.encode(upload_ids: [missing_id], max_pixels: 1_048_576)).to be_empty+ expect(described_class.encode(upload_ids: [missing_id], max_pixels: MAX_PIXELS)).to be_emptyenddescribe ".doc, .docx, .xls, and .xlsx uploads" do@@ -151,13 +128,7 @@ RSpec.describe DiscourseAi::Completions::UploadEncoder do"Converted document text\n",)- encoded =- described_class.encode(- upload_ids: [upload.id],- max_pixels: 1_048_576,- allowed_kinds: %i[document],- allowed_attachment_types: ["doc"],- )+ encoded = encode_document(upload, ["doc"])expect(DiscourseAi::Completions::DocToText).to have_received(:convert).with(a_string_matching(/\.doc\z/),@@ -180,13 +151,7 @@ RSpec.describe DiscourseAi::Completions::UploadEncoder doallow(DiscourseAi::Completions::DocToText).to receive(:convert).and_return(nil)allow(Rails.logger).to receive(:warn)- encoded =- described_class.encode(- upload_ids: [upload.id],- max_pixels: 1_048_576,- allowed_kinds: %i[document],- allowed_attachment_types: ["doc"],- )+ encoded = encode_document(upload, ["doc"])expect(encoded).to be_emptyexpect(Rails.logger).to have_received(:warn).with(@@ -204,17 +169,14 @@ RSpec.describe DiscourseAi::Completions::UploadEncoder doupload =create_doc_upload(contents:- docx_bytes("word/document.xml" => docx_document_xml("Converted DOCX document text")),+ zipped_document_bytes(+ "docx",+ "word/document.xml" => docx_document_xml("Converted DOCX document text"),+ ),filename: "sample.docx",)- encoded =- described_class.encode(- upload_ids: [upload.id],- max_pixels: 1_048_576,- allowed_kinds: %i[document],- allowed_attachment_types: ["docx"],- )+ encoded = encode_document(upload, ["docx"])expect(encoded.length).to eq(1)expect(encoded.first).to include(@@ -228,30 +190,6 @@ RSpec.describe DiscourseAi::Completions::UploadEncoder doexpect(encoded.first).not_to have_key(:base64)end- it "logs docx conversion failures and skips the upload" do- upload = create_doc_upload(contents: "raw docx bytes", filename: "sample.docx")-- allow(Rails.logger).to receive(:warn)-- encoded =- described_class.encode(- upload_ids: [upload.id],- max_pixels: 1_048_576,- allowed_kinds: %i[document],- allowed_attachment_types: ["docx"],- )-- expect(Rails.logger).to have_received(:warn).with(- a_string_including(- "Failed to convert .docx upload to text",- "upload_id=#{upload.id}",- "sample.docx",- "Zip",- ),- )- expect(encoded).to be_empty- end-it "converts .xls files to text" doupload = create_doc_upload(contents: "raw xls bytes", filename: "sample.xls")@@ -259,13 +197,7 @@ RSpec.describe DiscourseAi::Completions::UploadEncoder do"Name,Value\nAlice,1\n",)- encoded =- described_class.encode(- upload_ids: [upload.id],- max_pixels: 1_048_576,- allowed_kinds: %i[document],- allowed_attachment_types: ["xls"],- )+ encoded = encode_document(upload, ["xls"])expect(DiscourseAi::Completions::XlsToText).to have_received(:convert).with(a_string_matching(/\.xls\z/),@@ -288,13 +220,7 @@ RSpec.describe DiscourseAi::Completions::UploadEncoder doallow(DiscourseAi::Completions::XlsToText).to receive(:convert).and_return(nil)allow(Rails.logger).to receive(:warn)- encoded =- described_class.encode(- upload_ids: [upload.id],- max_pixels: 1_048_576,- allowed_kinds: %i[document],- allowed_attachment_types: ["xls"],- )+ encoded = encode_document(upload, ["xls"])expect(encoded).to be_emptyexpect(Rails.logger).to have_received(:warn).with(@@ -311,7 +237,7 @@ RSpec.describe DiscourseAi::Completions::UploadEncoder doit "converts .xlsx files to text" doupload =create_doc_upload(- contents: xlsx_bytes("xl/worksheets/sheet1.xml" => <<~XML),+ contents: zipped_document_bytes("xlsx", "xl/worksheets/sheet1.xml" => <<~XML),<worksheet xmlns="http://schemas.openxmlformats.org/spreadsheetml/2006/main"><sheetData><row><c t="inlineStr"><is><t>Converted XLSX spreadsheet text</t></is></c></row>@@ -321,13 +247,7 @@ RSpec.describe DiscourseAi::Completions::UploadEncoder dofilename: "sample.xlsx",)- encoded =- described_class.encode(- upload_ids: [upload.id],- max_pixels: 1_048_576,- allowed_kinds: %i[document],- allowed_attachment_types: ["xlsx"],- )+ encoded = encode_document(upload, ["xlsx"])expect(encoded.length).to eq(1)expect(encoded.first).to include(@@ -343,44 +263,18 @@ RSpec.describe DiscourseAi::Completions::UploadEncoder doexpect(encoded.first).not_to have_key(:base64)end- it "logs xlsx conversion failures and skips the upload" do- upload = create_doc_upload(contents: "raw xlsx bytes", filename: "sample.xlsx")-- allow(Rails.logger).to receive(:warn)-- encoded =- described_class.encode(- upload_ids: [upload.id],- max_pixels: 1_048_576,- allowed_kinds: %i[document],- allowed_attachment_types: ["xlsx"],- )-- expect(Rails.logger).to have_received(:warn).with(- a_string_including(- "Failed to convert .xlsx upload to text",- "upload_id=#{upload.id}",- "sample.xlsx",- "Zip",- ),- )- expect(encoded).to be_empty- end-it "converts .odt files to text" doupload =create_doc_upload(- contents: odt_bytes("content.xml" => odt_content_xml("Converted ODT document text")),+ contents:+ zipped_document_bytes(+ "odt",+ "content.xml" => odt_content_xml("Converted ODT document text"),+ ),filename: "sample.odt",)- encoded =- described_class.encode(- upload_ids: [upload.id],- max_pixels: 1_048_576,- allowed_kinds: %i[document],- allowed_attachment_types: ["odt"],- )+ encoded = encode_document(upload, ["odt"])expect(encoded.length).to eq(1)expect(encoded.first).to include(@@ -394,44 +288,18 @@ RSpec.describe DiscourseAi::Completions::UploadEncoder doexpect(encoded.first).not_to have_key(:base64)end- it "logs odt conversion failures and skips the upload" do- upload = create_doc_upload(contents: "raw odt bytes", filename: "sample.odt")-- allow(Rails.logger).to receive(:warn)-- encoded =- described_class.encode(- upload_ids: [upload.id],- max_pixels: 1_048_576,- allowed_kinds: %i[document],- allowed_attachment_types: ["odt"],- )-- expect(Rails.logger).to have_received(:warn).with(- a_string_including(- "Failed to convert .odt upload to text",- "upload_id=#{upload.id}",- "sample.odt",- "Zip",- ),- )- expect(encoded).to be_empty- end-it "converts .ods files to text" doupload =create_doc_upload(- contents: ods_bytes("content.xml" => ods_content_xml("Sales", "Converted ODS cell")),+ contents:+ zipped_document_bytes(+ "ods",+ "content.xml" => ods_content_xml("Sales", "Converted ODS cell"),+ ),filename: "sample.ods",)- encoded =- described_class.encode(- upload_ids: [upload.id],- max_pixels: 1_048_576,- allowed_kinds: %i[document],- allowed_attachment_types: ["ods"],- )+ encoded = encode_document(upload, ["ods"])expect(encoded.length).to eq(1)expect(encoded.first).to include(@@ -445,28 +313,25 @@ RSpec.describe DiscourseAi::Completions::UploadEncoder doexpect(encoded.first).not_to have_key(:base64)end- it "logs ods conversion failures and skips the upload" do- upload = create_doc_upload(contents: "raw ods bytes", filename: "sample.ods")+ %w[docx xlsx odt ods].each do |extension|+ it "logs #{extension} conversion failures and skips the upload" do+ upload =+ create_doc_upload(contents: "raw #{extension} bytes", filename: "sample.#{extension}")- allow(Rails.logger).to receive(:warn)+ allow(Rails.logger).to receive(:warn)- encoded =- described_class.encode(- upload_ids: [upload.id],- max_pixels: 1_048_576,- allowed_kinds: %i[document],- allowed_attachment_types: ["ods"],- )+ encoded = encode_document(upload, [extension])- expect(Rails.logger).to have_received(:warn).with(- a_string_including(- "Failed to convert .ods upload to text",- "upload_id=#{upload.id}",- "sample.ods",- "Zip",- ),- )- expect(encoded).to be_empty+ expect(Rails.logger).to have_received(:warn).with(+ a_string_including(+ "Failed to convert .#{extension} upload to text",+ "upload_id=#{upload.id}",+ "sample.#{extension}",+ "Zip",+ ),+ )+ expect(encoded).to be_empty+ endendit "converts .rtf files to text" do@@ -476,13 +341,7 @@ RSpec.describe DiscourseAi::Completions::UploadEncoder dofilename: "sample.rtf",)- encoded =- described_class.encode(- upload_ids: [upload.id],- max_pixels: 1_048_576,- allowed_kinds: %i[document],- allowed_attachment_types: ["rtf"],- )+ encoded = encode_document(upload, ["rtf"])expect(encoded.length).to eq(1)expect(encoded.first).to include(@@ -516,7 +375,7 @@ Alice,1",encoded =described_class.encode(upload_ids: uploads.map(&:id),- max_pixels: 1_048_576,+ max_pixels: MAX_PIXELS,allowed_kinds: %i[document],allowed_attachment_types: %w[txt md csv],)@@ -551,7 +410,7 @@ Alice,1",encoded =described_class.encode(upload_ids: [md_upload.id, txt_upload.id],- max_pixels: 1_048_576,+ max_pixels: MAX_PIXELS,allowed_kinds: %i[document],allowed_attachment_types: %w[markdown text],)@@ -565,13 +424,8 @@ Alice,1",upload = create_doc_upload(contents: "0123456789abcdef", filename: "large.txt")encoded =- stub_const(described_class, :MAX_TEXT_FILE_BYTES, 10) do- described_class.encode(- upload_ids: [upload.id],- max_pixels: 1_048_576,- allowed_kinds: %i[document],- allowed_attachment_types: ["txt"],- )+ stub_const(DiscourseAi::Completions::DocumentEncoder, :MAX_TEXT_FILE_BYTES, 10) do+ encode_document(upload, ["txt"])endexpect(encoded.length).to eq(1)@@ -587,13 +441,7 @@ Alice,1",allow(Rails.logger).to receive(:warn)- encoded =- described_class.encode(- upload_ids: [upload.id],- max_pixels: 1_048_576,- allowed_kinds: %i[document],- allowed_attachment_types: ["txt"],- )+ encoded = encode_document(upload, ["txt"])expect(Rails.logger).to have_received(:warn).with(a_string_including(@@ -613,13 +461,7 @@ Alice,1",allow(DiscourseAi::Completions::DocToText).to receive(:convert).and_raise(error)allow(Rails.logger).to receive(:warn)- encoded =- described_class.encode(- upload_ids: [upload.id],- max_pixels: 1_048_576,- allowed_kinds: %i[document],- allowed_attachment_types: ["doc"],- )+ encoded = encode_document(upload, ["doc"])expect(Rails.logger).to have_received(:warn).with(a_string_including(@@ -639,13 +481,7 @@ Alice,1",allow(DiscourseAi::Completions::DocToText).to receive(:convert).and_return("\n \n")allow(Rails.logger).to receive(:warn)- encoded =- described_class.encode(- upload_ids: [upload.id],- max_pixels: 1_048_576,- allowed_kinds: %i[document],- allowed_attachment_types: ["doc"],- )+ encoded = encode_document(upload, ["doc"])expect(Rails.logger).to have_received(:warn).with(a_string_including(@@ -661,13 +497,7 @@ Alice,1",it "sends allowed raw PDF documents when they are within the byte limit" doupload = create_doc_upload(contents: "%PDF raw bytes", filename: "sample.pdf")- encoded =- described_class.encode(- upload_ids: [upload.id],- max_pixels: 1_048_576,- allowed_kinds: %i[document],- allowed_attachment_types: ["pdf"],- )+ encoded = encode_document(upload, ["pdf"])expect(encoded.length).to eq(1)expect(encoded.first[:kind]).to eq(:document)@@ -681,13 +511,8 @@ Alice,1",allow(Rails.logger).to receive(:warn)encoded =- stub_const(described_class, :MAX_RAW_DOCUMENT_BYTES, 4) do- described_class.encode(- upload_ids: [upload.id],- max_pixels: 1_048_576,- allowed_kinds: %i[document],- allowed_attachment_types: ["pdf"],- )+ stub_const(DiscourseAi::Completions::DocumentEncoder, :MAX_RAW_DOCUMENT_BYTES, 4) do+ encode_document(upload, ["pdf"])endexpect(encoded).to be_emptydiff --git a/plugins/discourse-ai/spec/lib/completions/xlsx_to_text_spec.rb b/plugins/discourse-ai/spec/lib/completions/xlsx_to_text_spec.rbindex 6c3452d494f..222de8c04a4 100644--- a/plugins/discourse-ai/spec/lib/completions/xlsx_to_text_spec.rb+++ b/plugins/discourse-ai/spec/lib/completions/xlsx_to_text_spec.rb@@ -1,24 +1,6 @@# frozen_string_literal: trueRSpec.describe DiscourseAi::Completions::XlsxToText do- def with_xlsx(entries)- tempfile = Tempfile.new(%w[workbook .xlsx])- path = tempfile.path- tempfile.close- FileUtils.rm_f(path)-- ::Zip::File.open(path, create: true) do |zip_file|- entries.each do |name, content|- zip_file.get_output_stream(name) { |stream| stream.write(content) }- end- end-- yield path- ensure- tempfile&.close- FileUtils.rm_f(path) if path- end-def workbook_xml<<~XML<?xml version="1.0" encoding="UTF-8" standalone="yes"?>@@ -54,7 +36,8 @@ RSpec.describe DiscourseAi::Completions::XlsxToText doendit "extracts sheets in workbook order as tab-delimited text" do- with_xlsx(+ with_zipped_document(+ "xlsx","xl/workbook.xml" => workbook_xml,"xl/_rels/workbook.xml.rels" => workbook_rels_xml,"xl/sharedStrings.xml" => shared_strings_xml,@@ -82,15 +65,19 @@ RSpec.describe DiscourseAi::Completions::XlsxToText doendit "falls back to worksheet entries when workbook metadata is missing" do- with_xlsx("xl/worksheets/sheet2.xml" => <<~XML, "xl/worksheets/sheet1.xml" => <<~XML) do |path|+ with_zipped_document(+ "xlsx",+ "xl/worksheets/sheet2.xml" => <<~XML,<worksheet xmlns="http://schemas.openxmlformats.org/spreadsheetml/2006/main"><sheetData><row><c t="inlineStr"><is><t>Second</t></is></c></row></sheetData></worksheet>XML+ "xl/worksheets/sheet1.xml" => <<~XML,<worksheet xmlns="http://schemas.openxmlformats.org/spreadsheetml/2006/main"><sheetData><row><c t="inlineStr"><is><t>First</t></is></c></row></sheetData></worksheet>XML+ ) do |path|expect(described_class.convert(path)).to eq("Sheet: Sheet1\n\nFirst\n\nSheet: Sheet2\n\nSecond",)@@ -98,7 +85,7 @@ RSpec.describe DiscourseAi::Completions::XlsxToText doendit "limits very wide sparse rows" do- with_xlsx("xl/worksheets/sheet1.xml" => <<~XML) do |path|+ with_zipped_document("xlsx", "xl/worksheets/sheet1.xml" => <<~XML) do |path|<worksheet xmlns="http://schemas.openxmlformats.org/spreadsheetml/2006/main"><sheetData><row><c r="A1" t="inlineStr"><is><t>Visible</t></is></c><c r="ZZ1" t="inlineStr"><is><t>Too far away</t></is></c></row>@@ -110,7 +97,7 @@ RSpec.describe DiscourseAi::Completions::XlsxToText doendit "returns blank text when the workbook has no readable sheets" do- with_xlsx("docProps/core.xml" => "<properties />") do |path|+ with_zipped_document("xlsx", "docProps/core.xml" => "<properties />") do |path|expect(described_class.convert(path)).to eq("")endenddiff --git a/plugins/discourse-ai/spec/support/document_converter_fixtures.rb b/plugins/discourse-ai/spec/support/document_converter_fixtures.rbnew file mode 100644index 00000000000..f22fc267b04--- /dev/null+++ b/plugins/discourse-ai/spec/support/document_converter_fixtures.rb@@ -0,0 +1,38 @@+# frozen_string_literal: true++module DocumentConverterFixtures+ def with_document_file(extension, contents)+ tempfile = Tempfile.new(["document", ".#{extension}"])+ tempfile.binmode+ tempfile.write(contents)+ tempfile.rewind++ yield tempfile.path+ ensure+ tempfile&.close!+ end++ def with_zipped_document(extension, entries)+ tempfile = Tempfile.new(["document", ".#{extension}"])+ path = tempfile.path+ tempfile.close+ FileUtils.rm_f(path)++ ::Zip::File.open(path, create: true) do |zip_file|+ entries.each do |name, content|+ zip_file.get_output_stream(name) { |stream| stream.write(content) }+ end+ end++ yield path+ ensure+ tempfile&.close+ FileUtils.rm_f(path) if path+ end++ def zipped_document_bytes(extension, entries)+ with_zipped_document(extension, entries) { |path| File.binread(path) }+ end+end++RSpec.configure { |config| config.include DocumentConverterFixtures }