Skip to content

A platform for community discussion. Free, open, simple.

DEV: Split document encoding out of UploadEncoder (#43008)

Régis Hanol1e0b2f60043a
diff --git a/plugins/discourse-ai/lib/completions/dialects/dialect.rb b/plugins/discourse-ai/lib/completions/dialects/dialect.rb
index 1f97b2f117a..e478effd2af 100644
--- a/plugins/discourse-ai/lib/completions/dialects/dialect.rb
+++ b/plugins/discourse-ai/lib/completions/dialects/dialect.rb
@@ -517,7 +517,7 @@ module DiscourseAi
ext = File.extname(encoded[:filename].to_s).delete_prefix(".")
allowed_types.include?(
- DiscourseAi::Completions::UploadEncoder.attachment_type_for(ext, encoded[:mime_type]),
+ DiscourseAi::Completions::DocumentEncoder.attachment_type_for(ext, encoded[:mime_type]),
)
end
end
diff --git a/plugins/discourse-ai/lib/completions/doc_to_text.rb b/plugins/discourse-ai/lib/completions/doc_to_text.rb
index abce4a203a3..84b33091060 100644
--- a/plugins/discourse-ai/lib/completions/doc_to_text.rb
+++ b/plugins/discourse-ai/lib/completions/doc_to_text.rb
@@ -6,7 +6,7 @@ module DiscourseAi
module Completions
class DocToText
ANTIWORD_TIMEOUT_SECONDS = 5
- MAX_CONVERSION_OUTPUT_BYTES = 4 * 100_001
+ MAX_CONVERSION_OUTPUT_BYTES = 4 * TextNormalization::MAX_EXTRACTED_TEXT_CHARS
SAFE_EXEC_ENV = { "PATH" => ENV["PATH"].to_s }.freeze
ANTIWORD_RLIMITS = {
cpu_seconds: ANTIWORD_TIMEOUT_SECONDS,
diff --git a/plugins/discourse-ai/lib/completions/document_encoder.rb b/plugins/discourse-ai/lib/completions/document_encoder.rb
new file mode 100644
index 00000000000..b0df5310835
--- /dev/null
+++ b/plugins/discourse-ai/lib/completions/document_encoder.rb
@@ -0,0 +1,179 @@
+# frozen_string_literal: true
+
+module DiscourseAi
+ module Completions
+ class DocumentEncoder
+ extend UploadEncoding
+
+ TEXT_CONVERTERS = {
+ "doc" => DocToText,
+ "docx" => DocxToText,
+ "xls" => XlsToText,
+ "xlsx" => XlsxToText,
+ "odt" => OdtToText,
+ "ods" => OdsToText,
+ "rtf" => RtfToText,
+ }.freeze
+
+ PLAIN_TEXT_ATTACHMENT_TYPES = %w[csv md txt].freeze
+ RAW_DOCUMENT_ATTACHMENT_TYPES = %w[pdf].freeze
+
+ SUPPORTED_ATTACHMENT_TYPES =
+ (TEXT_CONVERTERS.keys + PLAIN_TEXT_ATTACHMENT_TYPES + RAW_DOCUMENT_ATTACHMENT_TYPES).freeze
+
+ UNKNOWN_ATTACHMENT_TYPE = "file"
+
+ # order matters: the first fragment a mime type contains wins
+ MIME_ATTACHMENT_TYPES = {
+ "pdf" => "pdf",
+ "wordprocessingml.document" => "docx",
+ "application/msword" => "doc",
+ "spreadsheetml.sheet" => "xlsx",
+ "application/vnd.ms-excel" => "xls",
+ "opendocument.text" => "odt",
+ "opendocument.spreadsheet" => "ods",
+ "csv" => "csv",
+ "text/plain" => "txt",
+ "rtf" => "rtf",
+ "html" => "html",
+ "markdown" => "md",
+ }.freeze
+
+ MAX_TEXT_FILE_BYTES = 1 * 1024 * 1024
+ MAX_RAW_DOCUMENT_BYTES = 10 * 1024 * 1024
+
+ def self.encode(upload, mime_type, attachment_type)
+ path = fetch_path(upload)
+
+ if path.blank?
+ log_document_upload_skip(upload, attachment_type, "file is not available in the store")
+ return
+ end
+
+ extract_text_payload(upload, path, attachment_type) ||
+ raw_document_payload(upload, path, mime_type, attachment_type)
+ end
+
+ def self.attachment_type_for(extension, mime_type)
+ ext = extension.to_s.delete_prefix(".").downcase
+ ext = LlmModel::ATTACHMENT_TYPE_ALIASES.fetch(ext, ext)
+ return ext if SUPPORTED_ATTACHMENT_TYPES.include?(ext)
+
+ mime = mime_type.to_s.downcase
+ MIME_ATTACHMENT_TYPES.find { |fragment, _| mime.include?(fragment) }&.last ||
+ UNKNOWN_ATTACHMENT_TYPE
+ end
+
+ class << self
+ private
+
+ def extract_text_payload(upload, path, attachment_type)
+ converter = TEXT_CONVERTERS[attachment_type]
+ return if converter.nil? && PLAIN_TEXT_ATTACHMENT_TYPES.exclude?(attachment_type)
+
+ raw = converter ? converter.convert(path) : read_utf8_text_file(path)
+ text = normalize_extracted_text(raw)
+
+ if text.blank?
+ log_document_conversion_failure(
+ upload,
+ attachment_type,
+ "#{attachment_type.upcase} converter returned blank output",
+ )
+ return
+ end
+
+ text_document_payload(upload, path, text, converted_from: attachment_type)
+ rescue StandardError => e
+ log_document_conversion_failure(upload, attachment_type, "#{e.class}: #{e.message}")
+ nil
+ end
+
+ def raw_document_payload(upload, path, mime_type, attachment_type)
+ if RAW_DOCUMENT_ATTACHMENT_TYPES.exclude?(attachment_type)
+ log_document_upload_skip(
+ upload,
+ attachment_type,
+ "raw upload is not supported for this attachment type; it must be converted to text",
+ )
+ return
+ end
+
+ bytesize = File.size(path)
+ if bytesize > MAX_RAW_DOCUMENT_BYTES
+ log_document_upload_skip(
+ upload,
+ attachment_type,
+ "raw upload size #{ActiveSupport::NumberHelper.number_to_human_size(bytesize)} " \
+ "exceeds the #{ActiveSupport::NumberHelper.number_to_human_size(MAX_RAW_DOCUMENT_BYTES)} limit",
+ )
+ return
+ end
+
+ {
+ base64: Base64.strict_encode64(File.binread(path)),
+ mime_type: mime_type,
+ kind: :document,
+ filename: upload.original_filename,
+ }
+ rescue SystemCallError => e
+ log_document_upload_skip(upload, attachment_type, "#{e.class}: #{e.message}")
+ nil
+ end
+
+ def read_utf8_text_file(path)
+ text = File.binread(path, MAX_TEXT_FILE_BYTES + 1) || +""
+ truncated = text.bytesize > MAX_TEXT_FILE_BYTES
+ text = text.byteslice(0, MAX_TEXT_FILE_BYTES) if truncated
+
+ text.force_encoding("UTF-8")
+ return text if !truncated
+
+ text +
+ "\n\n[Document text truncated after #{ActiveSupport::NumberHelper.number_to_human_size(MAX_TEXT_FILE_BYTES)}.]"
+ end
+
+ def normalize_extracted_text(output)
+ Encodings.force_utf8(output.to_s).strip
+ end
+
+ def truncate_extracted_text(text)
+ return text if text.length <= TextNormalization::DOCUMENT_TEXT_BUDGET
+
+ text.first(TextNormalization::DOCUMENT_TEXT_BUDGET) +
+ "\n\n[Document text truncated after #{TextNormalization::DOCUMENT_TEXT_BUDGET} characters.]"
+ end
+
+ def text_document_payload(upload, path, text, converted_from:)
+ {
+ kind: :document,
+ filename: upload.original_filename,
+ mime_type: "text/plain",
+ text: document_text_preamble(upload, path) + truncate_extracted_text(text),
+ converted_from: converted_from,
+ }
+ end
+
+ def document_text_preamble(upload, path)
+ filename = upload.original_filename.presence || "document"
+ filesize = upload.filesize || File.size(path)
+ "Uploaded document: #{filename} (#{ActiveSupport::NumberHelper.number_to_human_size(filesize)})\n\n"
+ end
+
+ def log_document_conversion_failure(upload, extension, message)
+ Rails.logger.warn(
+ "Discourse AI: Failed to convert .#{extension} upload to text " \
+ "(upload_id=#{upload.id}, filename=#{upload.original_filename.inspect}): #{message}",
+ )
+ end
+
+ def log_document_upload_skip(upload, extension, message)
+ Rails.logger.warn(
+ "Discourse AI: Skipping .#{extension} upload " \
+ "(upload_id=#{upload.id}, filename=#{upload.original_filename.inspect}): #{message}",
+ )
+ end
+ end
+ end
+ end
+end
diff --git a/plugins/discourse-ai/lib/completions/docx_to_text.rb b/plugins/discourse-ai/lib/completions/docx_to_text.rb
index d5d699f8d52..2aafc6d1322 100644
--- a/plugins/discourse-ai/lib/completions/docx_to_text.rb
+++ b/plugins/discourse-ai/lib/completions/docx_to_text.rb
@@ -6,10 +6,11 @@ require "compression/safe_zip_reader"
module DiscourseAi
module Completions
class DocxToText
+ include TextNormalization
+
MAX_ENTRY_XML_BYTES = 10 * 1024 * 1024
MAX_TOTAL_XML_BYTES = 30 * 1024 * 1024
MAX_ZIP_ENTRIES = 1_000
- MAX_EXTRACTED_TEXT_CHARS = 100_001
MAX_PARAGRAPHS = 20_000
SUPPORTED_PARTS = %w[word/document.xml word/footnotes.xml word/endnotes.xml word/comments.xml]
@@ -20,6 +21,8 @@ module DiscourseAi
end
class Numbering
+ include TextNormalization
+
Level = Struct.new(:num_format, :level_text, :start, keyword_init: true)
BULLET_FORMAT = "bullet"
@@ -269,10 +272,6 @@ module DiscourseAi
def integer_value(value, default = 0)
value.present? ? value.to_i : default
end
-
- def force_utf8(text)
- text.to_s.encode("UTF-8", invalid: :replace, undef: :replace, replace: "")
- end
end
def self.convert(path)
@@ -414,10 +413,6 @@ module DiscourseAi
.join(" - ")
end
- def normalize_inline_text(text)
- force_utf8(text).gsub("\u00A0", " ").gsub(/\s+/, " ").strip
- end
-
def normalize_paragraph_text(text)
force_utf8(text)
.gsub("\u00A0", " ")
@@ -426,19 +421,6 @@ module DiscourseAi
.gsub(/\n[ \t]+/, "\n")
.rstrip
end
-
- def normalize_document_text(text)
- force_utf8(text)
- .gsub("\u00A0", " ")
- .gsub(/\r\n?/, "\n")
- .gsub(/[ \t]+\n/, "\n")
- .gsub(/\n{3,}/, "\n\n")
- .strip
- end
-
- def force_utf8(text)
- text.to_s.encode("UTF-8", invalid: :replace, undef: :replace, replace: "")
- end
end
end
end
diff --git a/plugins/discourse-ai/lib/completions/ods_to_text.rb b/plugins/discourse-ai/lib/completions/ods_to_text.rb
index 6608256fe2a..debc40838a6 100644
--- a/plugins/discourse-ai/lib/completions/ods_to_text.rb
+++ b/plugins/discourse-ai/lib/completions/ods_to_text.rb
@@ -6,10 +6,11 @@ require "compression/safe_zip_reader"
module DiscourseAi
module Completions
class OdsToText
+ include TextNormalization
+
MAX_ENTRY_XML_BYTES = 10 * 1024 * 1024
MAX_TOTAL_XML_BYTES = 30 * 1024 * 1024
MAX_ZIP_ENTRIES = 1_000
- MAX_EXTRACTED_TEXT_CHARS = 100_001
MAX_SHEETS = 50
MAX_ROWS_PER_SHEET = 10_000
MAX_COLUMNS = 200
@@ -171,19 +172,6 @@ module DiscourseAi
def normalize_inline(text)
force_utf8(text).gsub("\u00A0", " ").gsub(/[ \t\r\n]+/, " ").strip
end
-
- def normalize_document_text(text)
- force_utf8(text)
- .gsub("\u00A0", " ")
- .gsub(/\r\n?/, "\n")
- .gsub(/[ \t]+\n/, "\n")
- .gsub(/\n{3,}/, "\n\n")
- .strip
- end
-
- def force_utf8(text)
- text.to_s.encode("UTF-8", invalid: :replace, undef: :replace, replace: "")
- end
end
end
end
diff --git a/plugins/discourse-ai/lib/completions/odt_to_text.rb b/plugins/discourse-ai/lib/completions/odt_to_text.rb
index 60a77d29d09..9fb5e06eabb 100644
--- a/plugins/discourse-ai/lib/completions/odt_to_text.rb
+++ b/plugins/discourse-ai/lib/completions/odt_to_text.rb
@@ -6,10 +6,11 @@ require "compression/safe_zip_reader"
module DiscourseAi
module Completions
class OdtToText
+ include TextNormalization
+
MAX_ENTRY_XML_BYTES = 10 * 1024 * 1024
MAX_TOTAL_XML_BYTES = 30 * 1024 * 1024
MAX_ZIP_ENTRIES = 1_000
- MAX_EXTRACTED_TEXT_CHARS = 100_001
MAX_PARAGRAPHS = 20_000
MAX_LIST_DEPTH = 20
@@ -176,19 +177,6 @@ module DiscourseAi
def normalize_inline(text)
force_utf8(text).gsub("\u00A0", " ").gsub(/[ \t\r\n]+/, " ").strip
end
-
- def normalize_document_text(text)
- force_utf8(text)
- .gsub("\u00A0", " ")
- .gsub(/\r\n?/, "\n")
- .gsub(/[ \t]+\n/, "\n")
- .gsub(/\n{3,}/, "\n\n")
- .strip
- end
-
- def force_utf8(text)
- text.to_s.encode("UTF-8", invalid: :replace, undef: :replace, replace: "")
- end
end
end
end
diff --git a/plugins/discourse-ai/lib/completions/prompt_messages_builder.rb b/plugins/discourse-ai/lib/completions/prompt_messages_builder.rb
index 2a56f441ca0..881077ddd3e 100644
--- a/plugins/discourse-ai/lib/completions/prompt_messages_builder.rb
+++ b/plugins/discourse-ai/lib/completions/prompt_messages_builder.rb
@@ -360,7 +360,7 @@ module DiscourseAi
MiniMime.lookup_by_filename(upload.original_filename)&.content_type ||
"application/octet-stream"
attachment_type =
- DiscourseAi::Completions::UploadEncoder.attachment_type_for(upload.extension, mime_type)
+ DiscourseAi::Completions::DocumentEncoder.attachment_type_for(upload.extension, mime_type)
allowed_attachment_types.include?(attachment_type)
end
diff --git a/plugins/discourse-ai/lib/completions/rtf_to_text.rb b/plugins/discourse-ai/lib/completions/rtf_to_text.rb
index 49d390cf2d3..9595f206ed8 100644
--- a/plugins/discourse-ai/lib/completions/rtf_to_text.rb
+++ b/plugins/discourse-ai/lib/completions/rtf_to_text.rb
@@ -3,8 +3,13 @@
module DiscourseAi
module Completions
class RtfToText
+ include TextNormalization
+
+ def normalize_document_text(text)
+ super.gsub(/\n[ \t]+/, "\n")
+ end
+
MAX_INPUT_BYTES = 2 * 1024 * 1024
- MAX_EXTRACTED_TEXT_CHARS = 100_001
MAX_GROUP_DEPTH = 100
MAX_CONTROL_WORD_CHARS = 64
@@ -387,18 +392,6 @@ module DiscourseAi
def digit?(byte)
byte >= 48 && byte <= 57
end
-
- def normalize_document_text(text)
- text
- .to_s
- .encode("UTF-8", invalid: :replace, undef: :replace, replace: "")
- .gsub("\u00A0", " ")
- .gsub(/\r\n?/, "\n")
- .gsub(/[ \t]+\n/, "\n")
- .gsub(/\n[ \t]+/, "\n")
- .gsub(/\n{3,}/, "\n\n")
- .strip
- end
end
end
end
diff --git a/plugins/discourse-ai/lib/completions/text_normalization.rb b/plugins/discourse-ai/lib/completions/text_normalization.rb
new file mode 100644
index 00000000000..79b1c1af97b
--- /dev/null
+++ b/plugins/discourse-ai/lib/completions/text_normalization.rb
@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+module DiscourseAi
+ module Completions
+ module TextNormalization
+ DOCUMENT_TEXT_BUDGET = 100_000
+
+ # one past the budget, so DocumentEncoder#truncate_extracted_text can tell a document
+ # was cut short and say so in the prompt
+ MAX_EXTRACTED_TEXT_CHARS = DOCUMENT_TEXT_BUDGET + 1
+
+ def force_utf8(text)
+ ::Encodings.force_utf8(text.to_s)
+ end
+
+ def normalize_document_text(text)
+ force_utf8(text)
+ .first(MAX_EXTRACTED_TEXT_CHARS)
+ .gsub("\u00A0", " ")
+ .gsub(/\r\n?/, "\n")
+ .gsub(/[ \t]+\n/, "\n")
+ .gsub(/\n{3,}/, "\n\n")
+ .strip
+ end
+
+ def normalize_inline_text(text)
+ force_utf8(text).gsub("\u00A0", " ").gsub(/\s+/, " ").strip
+ end
+ end
+ end
+end
diff --git a/plugins/discourse-ai/lib/completions/upload_encoder.rb b/plugins/discourse-ai/lib/completions/upload_encoder.rb
index e9d8d49a2a4..0600110b9f0 100644
--- a/plugins/discourse-ai/lib/completions/upload_encoder.rb
+++ b/plugins/discourse-ai/lib/completions/upload_encoder.rb
@@ -3,6 +3,8 @@
module DiscourseAi
module Completions
class UploadEncoder
+ extend UploadEncoding
+
def self.image_upload?(upload)
mime_type = MiniMime.lookup_by_filename(upload.original_filename)&.content_type
mime_type&.start_with?("image/") == true
@@ -18,11 +20,10 @@ module DiscourseAi
allowed_kinds: [:image],
allowed_attachment_types: nil
)
- uploads = []
allowed_attachment_types = normalize_attachment_types(allowed_attachment_types)
uploads_by_id = Upload.where(id: upload_ids).index_by(&:id)
- upload_ids.each do |upload_id|
+ upload_ids.filter_map do |upload_id|
upload = uploads_by_id[upload_id]
next if upload.blank?
@@ -36,12 +37,10 @@ module DiscourseAi
MiniMime.lookup_by_filename(upload.original_filename)&.content_type ||
"application/octet-stream"
- attachment_type = attachment_type_for(upload.extension, mime_type)
- next if disallowed_attachment?(allowed_attachment_types, attachment_type)
+ attachment_type = DocumentEncoder.attachment_type_for(upload.extension, mime_type)
+ next if allowed_attachment_types&.exclude?(attachment_type)
- payload = encode_document(upload, mime_type, attachment_type)
- uploads << payload if payload
- next
+ next DocumentEncoder.encode(upload, mime_type, attachment_type)
end
next if upload.width.to_i == 0 || upload.height.to_i == 0
@@ -54,35 +53,8 @@ module DiscourseAi
# this keeps it very simple format wise given everyone supports png and jpg
next if !%w[jpeg png].include?(desired_extension)
- payload = encode_image(upload, desired_extension, max_pixels)
- uploads << payload if payload
+ encode_image(upload, desired_extension, max_pixels)
end
- uploads
- end
-
- MAX_EXTRACTED_DOCUMENT_TEXT_CHARS = 100_000
- MAX_TEXT_FILE_BYTES = 1 * 1024 * 1024
- MAX_RAW_DOCUMENT_BYTES = 10 * 1024 * 1024
- RAW_DOCUMENT_ATTACHMENT_TYPES = %w[pdf]
-
- def self.attachment_type_for(extension, mime_type)
- ext = extension.to_s.delete_prefix(".").downcase
- mime = mime_type.to_s.downcase
-
- return "pdf" if ext == "pdf" || mime.include?("pdf")
- return "docx" if ext == "docx" || mime.include?("wordprocessingml.document")
- return "doc" if ext == "doc" || mime == "application/msword"
- return "xlsx" if ext == "xlsx" || mime.include?("spreadsheetml.sheet")
- return "xls" if ext == "xls" || mime == "application/vnd.ms-excel"
- return "odt" if ext == "odt" || mime.include?("opendocument.text")
- return "ods" if ext == "ods" || mime.include?("opendocument.spreadsheet")
- return "csv" if ext == "csv" || mime.include?("text/csv") || mime.include?("csv")
- return "txt" if ext == "txt" || mime.include?("text/plain")
- return "rtf" if ext == "rtf" || mime.include?("rtf")
- return "html" if %w[html htm].include?(ext) || mime.include?("html")
- return "md" if %w[md markdown].include?(ext) || mime.include?("markdown")
-
- "file"
end
class << self
@@ -94,271 +66,10 @@ module DiscourseAi
LlmModel.normalize_attachment_types(types)
end
- def disallowed_attachment?(allowed_types, attachment_type)
- !allowed_types.nil? && !allowed_types.include?(attachment_type)
- end
-
def image_extension?(ext)
%w[jpg jpeg png gif webp].include?(ext)
end
- def encode_document(upload, mime_type, attachment_type)
- path = fetch_path(upload)
- return if path.blank?
-
- if attachment_type == "doc"
- text_payload = doc_to_text_payload(upload, path)
- return text_payload if text_payload
- elsif attachment_type == "docx"
- text_payload = docx_to_text_payload(upload, path)
- return text_payload if text_payload
- elsif attachment_type == "xls"
- text_payload = xls_to_text_payload(upload, path)
- return text_payload if text_payload
- elsif attachment_type == "xlsx"
- text_payload = xlsx_to_text_payload(upload, path)
- return text_payload if text_payload
- elsif attachment_type == "odt"
- text_payload = odt_to_text_payload(upload, path)
- return text_payload if text_payload
- elsif attachment_type == "ods"
- text_payload = ods_to_text_payload(upload, path)
- return text_payload if text_payload
- elsif attachment_type == "rtf"
- text_payload = rtf_to_text_payload(upload, path)
- return text_payload if text_payload
- elsif %w[csv md txt].include?(attachment_type)
- text_payload = text_file_payload(upload, path, attachment_type)
- return text_payload if text_payload
- end
-
- raw_document_payload(upload, path, mime_type, attachment_type)
- end
-
- def raw_document_payload(upload, path, mime_type, attachment_type)
- if RAW_DOCUMENT_ATTACHMENT_TYPES.exclude?(attachment_type)
- log_document_upload_skip(
- upload,
- attachment_type,
- "raw upload is not supported for this attachment type; it must be converted to text",
- )
- return
- end
-
- bytesize = File.size(path)
- if bytesize > MAX_RAW_DOCUMENT_BYTES
- log_document_upload_skip(
- upload,
- attachment_type,
- "raw upload size #{human_filesize(bytesize)} exceeds the #{human_filesize(MAX_RAW_DOCUMENT_BYTES)} limit",
- )
- return
- end
-
- {
- base64: Base64.strict_encode64(File.binread(path)),
- mime_type: mime_type,
- kind: :document,
- filename: upload.original_filename,
- }
- rescue SystemCallError => e
- log_document_upload_skip(upload, attachment_type, "#{e.class}: #{e.message}")
- nil
- end
-
- def doc_to_text_payload(upload, path)
- text = normalize_extracted_text(DiscourseAi::Completions::DocToText.convert(path))
-
- if text.blank?
- log_document_conversion_failure(upload, "doc", "DOC converter returned blank output")
- return
- end
-
- text_document_payload(upload, path, text, converted_from: "doc")
- rescue StandardError => e
- log_document_conversion_failure(upload, "doc", "#{e.class}: #{e.message}")
- nil
- end
-
- def docx_to_text_payload(upload, path)
- text = normalize_extracted_text(DiscourseAi::Completions::DocxToText.convert(path))
-
- if text.blank?
- log_document_conversion_failure(upload, "docx", "DOCX converter returned blank output")
- return
- end
-
- text_document_payload(upload, path, text, converted_from: "docx")
- rescue StandardError => e
- log_document_conversion_failure(upload, "docx", "#{e.class}: #{e.message}")
- nil
- end
-
- def xls_to_text_payload(upload, path)
- text = normalize_extracted_text(DiscourseAi::Completions::XlsToText.convert(path))
-
- if text.blank?
- log_document_conversion_failure(upload, "xls", "XLS converter returned blank output")
- return
- end
-
- text_document_payload(upload, path, text, converted_from: "xls")
- rescue StandardError => e
- log_document_conversion_failure(upload, "xls", "#{e.class}: #{e.message}")
- nil
- end
-
- def xlsx_to_text_payload(upload, path)
- text = normalize_extracted_text(DiscourseAi::Completions::XlsxToText.convert(path))
-
- if text.blank?
- log_document_conversion_failure(upload, "xlsx", "XLSX converter returned blank output")
- return
- end
-
- text_document_payload(upload, path, text, converted_from: "xlsx")
- rescue StandardError => e
- log_document_conversion_failure(upload, "xlsx", "#{e.class}: #{e.message}")
- nil
- end
-
- def odt_to_text_payload(upload, path)
- text = normalize_extracted_text(DiscourseAi::Completions::OdtToText.convert(path))
-
- if text.blank?
- log_document_conversion_failure(upload, "odt", "ODT converter returned blank output")
- return
- end
-
- text_document_payload(upload, path, text, converted_from: "odt")
- rescue StandardError => e
- log_document_conversion_failure(upload, "odt", "#{e.class}: #{e.message}")
- nil
- end
-
- def ods_to_text_payload(upload, path)
- text = normalize_extracted_text(DiscourseAi::Completions::OdsToText.convert(path))
-
- if text.blank?
- log_document_conversion_failure(upload, "ods", "ODS converter returned blank output")
- return
- end
-
- text_document_payload(upload, path, text, converted_from: "ods")
- rescue StandardError => e
- log_document_conversion_failure(upload, "ods", "#{e.class}: #{e.message}")
- nil
- end
-
- def text_file_payload(upload, path, attachment_type)
- text = normalize_extracted_text(read_utf8_text_file(path))
-
- if text.blank?
- log_document_conversion_failure(upload, attachment_type, "text file was blank")
- return
- end
-
- text_document_payload(upload, path, text, converted_from: attachment_type)
- rescue SystemCallError => e
- log_document_conversion_failure(upload, attachment_type, "#{e.class}: #{e.message}")
- nil
- end
-
- def rtf_to_text_payload(upload, path)
- text = normalize_extracted_text(DiscourseAi::Completions::RtfToText.convert(path))
-
- if text.blank?
- log_document_conversion_failure(upload, "rtf", "RTF converter returned blank output")
- return
- end
-
- text_document_payload(upload, path, text, converted_from: "rtf")
- rescue StandardError => e
- log_document_conversion_failure(upload, "rtf", "#{e.class}: #{e.message}")
- nil
- end
-
- def read_utf8_text_file(path)
- text = +""
- truncated = false
-
- File.open(path, "rb") do |file|
- text = file.read(MAX_TEXT_FILE_BYTES + 1).to_s
- if text.bytesize > MAX_TEXT_FILE_BYTES
- text = text.byteslice(0, MAX_TEXT_FILE_BYTES)
- truncated = true
- end
- end
-
- text = text.delete_prefix("\xEF\xBB\xBF".b)
- text.force_encoding("UTF-8")
- text = text.encode("UTF-8", invalid: :replace, undef: :replace, replace: "")
-
- if truncated
- text << "\n\n[Document text truncated after #{human_filesize(MAX_TEXT_FILE_BYTES)}.]"
- end
-
- text
- end
-
- def normalize_extracted_text(output)
- output.to_s.encode("UTF-8", invalid: :replace, undef: :replace, replace: "").strip
- end
-
- def truncate_extracted_text(text)
- return text if text.length <= MAX_EXTRACTED_DOCUMENT_TEXT_CHARS
-
- text.first(MAX_EXTRACTED_DOCUMENT_TEXT_CHARS) +
- "\n\n[Document text truncated after #{MAX_EXTRACTED_DOCUMENT_TEXT_CHARS} characters.]"
- end
-
- def text_document_payload(upload, path, text, converted_from:)
- {
- kind: :document,
- filename: upload.original_filename,
- mime_type: "text/plain",
- text: document_text_preamble(upload, path) + truncate_extracted_text(text),
- converted_from: converted_from,
- }
- end
-
- def document_text_preamble(upload, path)
- filename = upload.original_filename.presence || "document"
- filesize = upload.filesize || File.size(path)
- "Uploaded document: #{filename} (#{human_filesize(filesize)})\n\n"
- end
-
- def human_filesize(bytes)
- bytes = bytes.to_i
- units = %w[Bytes KB MB GB TB]
- size = bytes.to_f
- unit = units.shift
-
- while size >= 1024 && units.any?
- size /= 1024.0
- unit = units.shift
- end
-
- return "#{bytes} #{bytes == 1 ? "Byte" : "Bytes"}" if unit == "Bytes"
-
- formatted_size = size >= 10 ? size.round.to_s : format("%.1f", size).sub(/\.0\z/, "")
- "#{formatted_size} #{unit}"
- end
-
- def log_document_conversion_failure(upload, extension, message)
- Rails.logger.warn(
- "Discourse AI: Failed to convert .#{extension} upload to text " \
- "(upload_id=#{upload.id}, filename=#{upload.original_filename.inspect}): #{message}",
- )
- end
-
- def log_document_upload_skip(upload, extension, message)
- Rails.logger.warn(
- "Discourse AI: Skipping .#{extension} upload " \
- "(upload_id=#{upload.id}, filename=#{upload.original_filename.inspect}): #{message}",
- )
- end
-
def encode_image(upload, desired_extension, max_pixels)
original_pixels = upload.width * upload.height
@@ -392,16 +103,6 @@ module DiscourseAi
filename: upload.original_filename,
}
end
-
- def fetch_path(upload)
- path = Discourse.store.path_for(upload)
- path = Discourse.store.download(upload) if path.blank?
-
- return if path.blank?
- return unless File.exist?(path)
-
- path
- end
end
end
end
diff --git a/plugins/discourse-ai/lib/completions/upload_encoding.rb b/plugins/discourse-ai/lib/completions/upload_encoding.rb
new file mode 100644
index 00000000000..14d0125f7f8
--- /dev/null
+++ b/plugins/discourse-ai/lib/completions/upload_encoding.rb
@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+module DiscourseAi
+ module Completions
+ # plumbing shared by the image and document halves of UploadEncoder
+ module UploadEncoding
+ def fetch_path(upload)
+ path = Discourse.store.path_for(upload)
+ path = Discourse.store.download(upload) if path.blank?
+
+ return if path.blank?
+ return unless File.exist?(path)
+
+ path
+ end
+ end
+ end
+end
diff --git a/plugins/discourse-ai/lib/completions/xls_to_text.rb b/plugins/discourse-ai/lib/completions/xls_to_text.rb
index a434cc489dc..fe557864864 100644
--- a/plugins/discourse-ai/lib/completions/xls_to_text.rb
+++ b/plugins/discourse-ai/lib/completions/xls_to_text.rb
@@ -6,7 +6,7 @@ module DiscourseAi
module Completions
class XlsToText
XLS2CSV_TIMEOUT_SECONDS = 5
- MAX_CONVERSION_OUTPUT_BYTES = 4 * 100_001
+ MAX_CONVERSION_OUTPUT_BYTES = 4 * TextNormalization::MAX_EXTRACTED_TEXT_CHARS
SAFE_EXEC_ENV = { "PATH" => ENV["PATH"].to_s }.freeze
XLS2CSV_RLIMITS = {
cpu_seconds: XLS2CSV_TIMEOUT_SECONDS,
diff --git a/plugins/discourse-ai/lib/completions/xlsx_to_text.rb b/plugins/discourse-ai/lib/completions/xlsx_to_text.rb
index 69aad720533..d69932cf9af 100644
--- a/plugins/discourse-ai/lib/completions/xlsx_to_text.rb
+++ b/plugins/discourse-ai/lib/completions/xlsx_to_text.rb
@@ -7,10 +7,11 @@ require "compression/safe_zip_reader"
module DiscourseAi
module Completions
class XlsxToText
+ include TextNormalization
+
MAX_ENTRY_XML_BYTES = 10 * 1024 * 1024
MAX_TOTAL_XML_BYTES = 30 * 1024 * 1024
MAX_ZIP_ENTRIES = 1_000
- MAX_EXTRACTED_TEXT_CHARS = 100_001
MAX_SHEETS = 50
MAX_ROWS_PER_SHEET = 10_000
MAX_COLUMNS = 200
@@ -242,23 +243,6 @@ module DiscourseAi
column_name.upcase.each_byte.reduce(0) { |index, char| (index * 26) + char - "A".ord + 1 } -
1
end
-
- def normalize_inline_text(text)
- force_utf8(text).gsub("\u00A0", " ").gsub(/[ \t\r\n]+/, " ").strip
- end
-
- def normalize_document_text(text)
- force_utf8(text)
- .gsub("\u00A0", " ")
- .gsub(/\r\n?/, "\n")
- .gsub(/[ \t]+\n/, "\n")
- .gsub(/\n{3,}/, "\n\n")
- .strip
- end
-
- def force_utf8(text)
- text.to_s.encode("UTF-8", invalid: :replace, undef: :replace, replace: "")
- end
end
end
end
diff --git a/plugins/discourse-ai/spec/lib/completions/document_encoder_spec.rb b/plugins/discourse-ai/spec/lib/completions/document_encoder_spec.rb
new file mode 100644
index 00000000000..a139377b9bc
--- /dev/null
+++ b/plugins/discourse-ai/spec/lib/completions/document_encoder_spec.rb
@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+RSpec.describe DiscourseAi::Completions::DocumentEncoder do
+ before { enable_current_plugin }
+
+ describe ".attachment_type_for" do
+ {
+ "pdf" => "pdf",
+ "docx" => "docx",
+ "doc" => "doc",
+ "xlsx" => "xlsx",
+ "xls" => "xls",
+ "odt" => "odt",
+ "ods" => "ods",
+ "csv" => "csv",
+ "txt" => "txt",
+ "rtf" => "rtf",
+ "html" => "html",
+ "htm" => "html",
+ "md" => "md",
+ "markdown" => "md",
+ }.each do |extension, expected|
+ it "maps .#{extension} to #{expected}" do
+ mime = MiniMime.lookup_by_filename("sample.#{extension}")&.content_type.to_s
+
+ expect(described_class.attachment_type_for(extension, mime)).to eq(expected)
+ expect(described_class.attachment_type_for(".#{extension.upcase}", mime)).to eq(expected)
+ end
+ end
+
+ it "falls back to the mime type when the extension is unknown" do
+ expect(described_class.attachment_type_for("", "application/pdf")).to eq("pdf")
+ expect(described_class.attachment_type_for("bin", "text/plain")).to eq("txt")
+ expect(described_class.attachment_type_for("bin", "application/vnd.ms-excel")).to eq("xls")
+ end
+
+ it "prefers a known extension over the mime type" do
+ expect(described_class.attachment_type_for("txt", "application/pdf")).to eq("txt")
+ end
+
+ it "returns the unknown sentinel for anything else" do
+ expect(described_class.attachment_type_for("zip", "application/zip")).to eq("file")
+ expect(described_class.attachment_type_for("", "")).to eq("file")
+ end
+ end
+end
diff --git a/plugins/discourse-ai/spec/lib/completions/docx_to_text_spec.rb b/plugins/discourse-ai/spec/lib/completions/docx_to_text_spec.rb
index b58353661bf..6d744f4cd74 100644
--- a/plugins/discourse-ai/spec/lib/completions/docx_to_text_spec.rb
+++ b/plugins/discourse-ai/spec/lib/completions/docx_to_text_spec.rb
@@ -1,24 +1,6 @@
# frozen_string_literal: true
RSpec.describe DiscourseAi::Completions::DocxToText do
- def with_docx(entries)
- tempfile = Tempfile.new(%w[document .docx])
- path = tempfile.path
- tempfile.close
- FileUtils.rm_f(path)
-
- ::Zip::File.open(path, create: true) do |zip_file|
- entries.each do |name, content|
- zip_file.get_output_stream(name) { |stream| stream.write(content) }
- end
- end
-
- yield path
- ensure
- tempfile&.close
- FileUtils.rm_f(path) if path
- end
-
def word_xml(body)
<<~XML
<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
@@ -75,7 +57,7 @@ RSpec.describe DiscourseAi::Completions::DocxToText do
end
it "extracts paragraph text from the main document" do
- with_docx("word/document.xml" => word_xml(<<~XML)) do |path|
+ with_zipped_document("docx", "word/document.xml" => word_xml(<<~XML)) do |path|
<w:body>
<w:p>
<w:r><w:t>Hello</w:t></w:r>
@@ -94,7 +76,8 @@ RSpec.describe DiscourseAi::Completions::DocxToText do
it "extracts supported text parts in a stable order" do
part = ->(text) { word_xml("<w:p><w:r><w:t>#{text}</w:t></w:r></w:p>") }
- with_docx(
+ with_zipped_document(
+ "docx",
"word/footer1.xml" => part.call("Footer"),
"word/header2.xml" => part.call("Header two"),
"word/document.xml" => part.call("Body"),
@@ -110,7 +93,7 @@ RSpec.describe DiscourseAi::Completions::DocxToText do
end
it "extracts image alt text inline" do
- with_docx("word/document.xml" => word_xml(<<~XML)) do |path|
+ with_zipped_document("docx", "word/document.xml" => word_xml(<<~XML)) do |path|
<w:body>
<w:p>
<w:r><w:t>See </w:t></w:r>
@@ -132,7 +115,8 @@ RSpec.describe DiscourseAi::Completions::DocxToText do
end
it "adds prefixes for numbered and bulleted lists" do
- with_docx(
+ with_zipped_document(
+ "docx",
"word/numbering.xml" => numbering_xml,
"word/document.xml" => word_xml(<<~XML),
<w:body>
@@ -151,7 +135,7 @@ RSpec.describe DiscourseAi::Completions::DocxToText do
end
it "returns blank text when the docx has no supported text parts" do
- with_docx("docProps/core.xml" => "<properties />") do |path|
+ with_zipped_document("docx", "docProps/core.xml" => "<properties />") do |path|
expect(described_class.convert(path)).to eq("")
end
end
diff --git a/plugins/discourse-ai/spec/lib/completions/ods_to_text_spec.rb b/plugins/discourse-ai/spec/lib/completions/ods_to_text_spec.rb
index ea4968acee8..b05199e801b 100644
--- a/plugins/discourse-ai/spec/lib/completions/ods_to_text_spec.rb
+++ b/plugins/discourse-ai/spec/lib/completions/ods_to_text_spec.rb
@@ -1,24 +1,6 @@
# frozen_string_literal: true
RSpec.describe DiscourseAi::Completions::OdsToText do
- def with_ods(entries)
- tempfile = Tempfile.new(%w[spreadsheet .ods])
- path = tempfile.path
- tempfile.close
- FileUtils.rm_f(path)
-
- ::Zip::File.open(path, create: true) do |zip_file|
- entries.each do |name, content|
- zip_file.get_output_stream(name) { |stream| stream.write(content) }
- end
- end
-
- yield path
- ensure
- tempfile&.close
- FileUtils.rm_f(path) if path
- end
-
def ods_content(body)
<<~XML
<?xml version="1.0" encoding="UTF-8"?>
@@ -36,7 +18,7 @@ RSpec.describe DiscourseAi::Completions::OdsToText do
end
it "extracts cells from each named sheet" do
- with_ods("content.xml" => ods_content(<<~XML)) do |path|
+ with_zipped_document("ods", "content.xml" => ods_content(<<~XML)) do |path|
<table:table table:name="Summary">
<table:table-row>
<table:table-cell office:value-type="string"><text:p>Name</text:p></table:table-cell>
@@ -60,7 +42,7 @@ RSpec.describe DiscourseAi::Completions::OdsToText do
end
it "uses Sheet1, Sheet2 ... when no name is set" do
- with_ods("content.xml" => ods_content(<<~XML)) do |path|
+ with_zipped_document("ods", "content.xml" => ods_content(<<~XML)) do |path|
<table:table>
<table:table-row>
<table:table-cell office:value-type="string"><text:p>only</text:p></table:table-cell>
@@ -72,7 +54,7 @@ RSpec.describe DiscourseAi::Completions::OdsToText do
end
it "renders typed values when no inline paragraph is present" do
- with_ods("content.xml" => ods_content(<<~XML)) do |path|
+ with_zipped_document("ods", "content.xml" => ods_content(<<~XML)) do |path|
<table:table table:name="Types">
<table:table-row>
<table:table-cell office:value-type="boolean" office:boolean-value="true"/>
@@ -86,7 +68,7 @@ RSpec.describe DiscourseAi::Completions::OdsToText do
end
it "expands a non-empty number-columns-repeated up to MAX_COLUMNS" do
- with_ods("content.xml" => ods_content(<<~XML)) do |path|
+ with_zipped_document("ods", "content.xml" => ods_content(<<~XML)) do |path|
<table:table table:name="Repeats">
<table:table-row>
<table:table-cell office:value-type="string" table:number-columns-repeated="3">
@@ -101,7 +83,7 @@ RSpec.describe DiscourseAi::Completions::OdsToText do
end
it "ignores covered (merge-continuation) cells and trims trailing empties" do
- with_ods("content.xml" => ods_content(<<~XML)) do |path|
+ with_zipped_document("ods", "content.xml" => ods_content(<<~XML)) do |path|
<table:table table:name="Merge">
<table:table-row>
<table:table-cell office:value-type="string"><text:p>head</text:p></table:table-cell>
@@ -115,7 +97,7 @@ RSpec.describe DiscourseAi::Completions::OdsToText do
end
it "returns blank text when content.xml is missing" do
- with_ods("META-INF/manifest.xml" => "<manifest/>") do |path|
+ with_zipped_document("ods", "META-INF/manifest.xml" => "<manifest/>") do |path|
expect(described_class.convert(path)).to eq("")
end
end
diff --git a/plugins/discourse-ai/spec/lib/completions/odt_to_text_spec.rb b/plugins/discourse-ai/spec/lib/completions/odt_to_text_spec.rb
index 45c059e093b..88e0bdfa7e2 100644
--- a/plugins/discourse-ai/spec/lib/completions/odt_to_text_spec.rb
+++ b/plugins/discourse-ai/spec/lib/completions/odt_to_text_spec.rb
@@ -1,24 +1,6 @@
# frozen_string_literal: true
RSpec.describe DiscourseAi::Completions::OdtToText do
- def with_odt(entries)
- tempfile = Tempfile.new(%w[document .odt])
- path = tempfile.path
- tempfile.close
- FileUtils.rm_f(path)
-
- ::Zip::File.open(path, create: true) do |zip_file|
- entries.each do |name, content|
- zip_file.get_output_stream(name) { |stream| stream.write(content) }
- end
- end
-
- yield path
- ensure
- tempfile&.close
- FileUtils.rm_f(path) if path
- end
-
def odt_content(body)
<<~XML
<?xml version="1.0" encoding="UTF-8"?>
@@ -37,7 +19,7 @@ RSpec.describe DiscourseAi::Completions::OdtToText do
end
it "extracts headings, paragraphs, tabs, line breaks, and spans" do
- with_odt("content.xml" => odt_content(<<~XML)) do |path|
+ with_zipped_document("odt", "content.xml" => odt_content(<<~XML)) do |path|
<text:h text:outline-level="1">Title</text:h>
<text:p>Hello<text:tab/>world<text:line-break/>second line</text:p>
<text:p>Run with <text:span>nested <text:span>span</text:span></text:span> text</text:p>
@@ -49,7 +31,7 @@ RSpec.describe DiscourseAi::Completions::OdtToText do
end
it "expands repeated spaces from text:s c=N" do
- with_odt("content.xml" => odt_content(<<~XML)) do |path|
+ with_zipped_document("odt", "content.xml" => odt_content(<<~XML)) do |path|
<text:p>A<text:s text:c="3"/>B</text:p>
XML
expect(described_class.convert(path)).to eq("A B")
@@ -57,7 +39,7 @@ RSpec.describe DiscourseAi::Completions::OdtToText do
end
it "renders nested lists with depth-aware bullet prefixes" do
- with_odt("content.xml" => odt_content(<<~XML)) do |path|
+ with_zipped_document("odt", "content.xml" => odt_content(<<~XML)) do |path|
<text:list>
<text:list-item><text:p>One</text:p></text:list-item>
<text:list-item>
@@ -75,7 +57,7 @@ RSpec.describe DiscourseAi::Completions::OdtToText do
end
it "renders tables as tab-separated rows" do
- with_odt("content.xml" => odt_content(<<~XML)) do |path|
+ with_zipped_document("odt", "content.xml" => odt_content(<<~XML)) do |path|
<table:table>
<table:table-row>
<table:table-cell><text:p>Name</text:p></table:table-cell>
@@ -92,7 +74,7 @@ RSpec.describe DiscourseAi::Completions::OdtToText do
end
it "walks into frames and text-boxes for callout text" do
- with_odt("content.xml" => odt_content(<<~XML)) do |path|
+ with_zipped_document("odt", "content.xml" => odt_content(<<~XML)) do |path|
<text:p>Body before</text:p>
<draw:frame>
<draw:text-box>
@@ -106,7 +88,7 @@ RSpec.describe DiscourseAi::Completions::OdtToText do
end
it "returns blank text when content.xml is missing" do
- with_odt("META-INF/manifest.xml" => "<manifest/>") do |path|
+ with_zipped_document("odt", "META-INF/manifest.xml" => "<manifest/>") do |path|
expect(described_class.convert(path)).to eq("")
end
end
diff --git a/plugins/discourse-ai/spec/lib/completions/rtf_to_text_spec.rb b/plugins/discourse-ai/spec/lib/completions/rtf_to_text_spec.rb
index 4afa84d919b..70928e0679a 100644
--- a/plugins/discourse-ai/spec/lib/completions/rtf_to_text_spec.rb
+++ b/plugins/discourse-ai/spec/lib/completions/rtf_to_text_spec.rb
@@ -1,19 +1,8 @@
# frozen_string_literal: true
RSpec.describe DiscourseAi::Completions::RtfToText do
- def with_rtf(contents)
- tempfile = Tempfile.new(%w[document .rtf])
- tempfile.binmode
- tempfile.write(contents)
- tempfile.rewind
-
- yield tempfile.path
- ensure
- tempfile&.close!
- end
-
it "extracts formatted body text" do
- with_rtf(<<~'RTF') do |path|
+ with_document_file("rtf", <<~'RTF') do |path|
{\rtf1\ansi\ansicpg1252 This is {\b bold}\par Caf\'e9\par Unicode \u8212? dash\tab done}
RTF
expect(described_class.convert(path)).to eq("This is bold\nCafé\nUnicode — dash\tdone")
@@ -21,13 +10,15 @@ RSpec.describe DiscourseAi::Completions::RtfToText do
end
it "ignores formatting tables and embedded binary destinations" do
- with_rtf(<<~'RTF') { |path| expect(described_class.convert(path)).to eq("Visible\nMore text") }
+ with_document_file("rtf", <<~'RTF') do |path|
{\rtf1\ansi{\fonttbl{\f0 Arial;}}{\colortbl;\red255\green0\blue0;}{\pict\pngblip abcdef}Visible\par More text}
RTF
+ expect(described_class.convert(path)).to eq("Visible\nMore text")
+ end
end
it "preserves escaped literal braces and backslashes" do
- with_rtf(<<~'RTF') do |path|
+ with_document_file("rtf", <<~'RTF') do |path|
{\rtf1\ansi literal \{brace\} and \\slash}
RTF
expect(described_class.convert(path)).to eq("literal {brace} and \\slash")
diff --git a/plugins/discourse-ai/spec/lib/completions/text_normalization_spec.rb b/plugins/discourse-ai/spec/lib/completions/text_normalization_spec.rb
new file mode 100644
index 00000000000..9907020a9fd
--- /dev/null
+++ b/plugins/discourse-ai/spec/lib/completions/text_normalization_spec.rb
@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+RSpec.describe DiscourseAi::Completions::TextNormalization do
+ subject(:normalizer) { Class.new { include DiscourseAi::Completions::TextNormalization }.new }
+
+ before { enable_current_plugin }
+
+ describe "#force_utf8" do
+ it "scrubs invalid bytes" do
+ expect(normalizer.force_utf8("caf\xFFe".dup.force_encoding("UTF-8"))).to eq("cafe")
+ end
+
+ it "strips a byte order mark" do
+ expect(normalizer.force_utf8("\xEF\xBB\xBFhello".dup.force_encoding("UTF-8"))).to eq("hello")
+ end
+
+ it "tolerates nil" do
+ expect(normalizer.force_utf8(nil)).to eq("")
+ end
+ end
+
+ describe "#normalize_document_text" do
+ it "collapses whitespace and trims" do
+ expect(normalizer.normalize_document_text(" a b\r\nc \n\n\n\nd ")).to eq("a b\nc\n\nd")
+ end
+
+ it "caps the output one character past the budget, so the encoder can detect truncation" do
+ text = normalizer.normalize_document_text("a" * (described_class::DOCUMENT_TEXT_BUDGET + 50))
+
+ expect(text.length).to eq(described_class::DOCUMENT_TEXT_BUDGET + 1)
+ end
+ end
+
+ describe "#normalize_inline_text" do
+ it "flattens all whitespace to single spaces" do
+ expect(normalizer.normalize_inline_text(" a b\n\tc ")).to eq("a b c")
+ end
+ end
+end
diff --git a/plugins/discourse-ai/spec/lib/completions/upload_encoder_spec.rb b/plugins/discourse-ai/spec/lib/completions/upload_encoder_spec.rb
index 9085ba750e4..39b47f74c57 100644
--- a/plugins/discourse-ai/spec/lib/completions/upload_encoder_spec.rb
+++ b/plugins/discourse-ai/spec/lib/completions/upload_encoder_spec.rb
@@ -10,6 +10,17 @@ RSpec.describe DiscourseAi::Completions::UploadEncoder do
before { enable_current_plugin }
+ MAX_PIXELS = 1_048_576
+
+ def encode_document(upload, attachment_type)
+ described_class.encode(
+ upload_ids: [upload.id],
+ max_pixels: MAX_PIXELS,
+ allowed_kinds: %i[document],
+ allowed_attachment_types: Array(attachment_type),
+ )
+ end
+
def create_doc_upload(contents: "raw doc bytes", filename: "sample.doc")
extension = File.extname(filename)
tempfile = Tempfile.new([File.basename(filename, extension), extension.presence || ".upload"])
@@ -22,40 +33,6 @@ RSpec.describe DiscourseAi::Completions::UploadEncoder do
tempfile&.close!
end
- def zip_bytes(entries, extension)
- tempfile = Tempfile.new(["sample", extension])
- path = tempfile.path
- tempfile.close
- FileUtils.rm_f(path)
-
- ::Zip::File.open(path, create: true) do |zip_file|
- entries.each do |name, content|
- zip_file.get_output_stream(name) { |stream| stream.write(content) }
- end
- end
-
- File.binread(path)
- ensure
- tempfile&.close
- FileUtils.rm_f(path) if path
- end
-
- def docx_bytes(entries)
- zip_bytes(entries, ".docx")
- end
-
- def xlsx_bytes(entries)
- zip_bytes(entries, ".xlsx")
- end
-
- def odt_bytes(entries)
- zip_bytes(entries, ".odt")
- end
-
- def ods_bytes(entries)
- zip_bytes(entries, ".ods")
- end
-
def odt_content_xml(text)
<<~XML
<?xml version="1.0" encoding="UTF-8"?>
@@ -104,7 +81,7 @@ RSpec.describe DiscourseAi::Completions::UploadEncoder do
it "automatically converts gifs to pngs" do
upload = UploadCreator.new(gif, "1x1.gif").create_for(Discourse.system_user.id)
- encoded = described_class.encode(upload_ids: [upload.id], max_pixels: 1_048_576)
+ encoded = described_class.encode(upload_ids: [upload.id], max_pixels: MAX_PIXELS)
expect(encoded.length).to eq(1)
expect(encoded[0][:base64]).to be_present
expect(encoded[0][:mime_type]).to eq("image/png")
@@ -112,7 +89,7 @@ RSpec.describe DiscourseAi::Completions::UploadEncoder do
it "automatically converts webp to pngs" do
upload = UploadCreator.new(webp, "1x1.webp").create_for(Discourse.system_user.id)
- encoded = described_class.encode(upload_ids: [upload.id], max_pixels: 1_048_576)
+ encoded = described_class.encode(upload_ids: [upload.id], max_pixels: MAX_PIXELS)
expect(encoded.length).to eq(1)
expect(encoded[0][:base64]).to be_present
expect(encoded[0][:mime_type]).to eq("image/png")
@@ -129,7 +106,7 @@ RSpec.describe DiscourseAi::Completions::UploadEncoder do
it "supports jpg" do
upload = UploadCreator.new(jpg, "1x1.jpg").create_for(Discourse.system_user.id)
- encoded = described_class.encode(upload_ids: [upload.id], max_pixels: 1_048_576)
+ encoded = described_class.encode(upload_ids: [upload.id], max_pixels: MAX_PIXELS)
expect(encoded.length).to eq(1)
expect(encoded[0][:base64]).to be_present
expect(encoded[0][:mime_type]).to eq("image/jpeg")
@@ -138,7 +115,7 @@ RSpec.describe DiscourseAi::Completions::UploadEncoder do
it "does not raise when an upload no longer exists" do
missing_id = Upload.maximum(:id).to_i + 1
- expect(described_class.encode(upload_ids: [missing_id], max_pixels: 1_048_576)).to be_empty
+ expect(described_class.encode(upload_ids: [missing_id], max_pixels: MAX_PIXELS)).to be_empty
end
describe ".doc, .docx, .xls, and .xlsx uploads" do
@@ -151,13 +128,7 @@ RSpec.describe DiscourseAi::Completions::UploadEncoder do
"Converted document text\n",
)
- encoded =
- described_class.encode(
- upload_ids: [upload.id],
- max_pixels: 1_048_576,
- allowed_kinds: %i[document],
- allowed_attachment_types: ["doc"],
- )
+ encoded = encode_document(upload, ["doc"])
expect(DiscourseAi::Completions::DocToText).to have_received(:convert).with(
a_string_matching(/\.doc\z/),
@@ -180,13 +151,7 @@ RSpec.describe DiscourseAi::Completions::UploadEncoder do
allow(DiscourseAi::Completions::DocToText).to receive(:convert).and_return(nil)
allow(Rails.logger).to receive(:warn)
- encoded =
- described_class.encode(
- upload_ids: [upload.id],
- max_pixels: 1_048_576,
- allowed_kinds: %i[document],
- allowed_attachment_types: ["doc"],
- )
+ encoded = encode_document(upload, ["doc"])
expect(encoded).to be_empty
expect(Rails.logger).to have_received(:warn).with(
@@ -204,17 +169,14 @@ RSpec.describe DiscourseAi::Completions::UploadEncoder do
upload =
create_doc_upload(
contents:
- docx_bytes("word/document.xml" => docx_document_xml("Converted DOCX document text")),
+ zipped_document_bytes(
+ "docx",
+ "word/document.xml" => docx_document_xml("Converted DOCX document text"),
+ ),
filename: "sample.docx",
)
- encoded =
- described_class.encode(
- upload_ids: [upload.id],
- max_pixels: 1_048_576,
- allowed_kinds: %i[document],
- allowed_attachment_types: ["docx"],
- )
+ encoded = encode_document(upload, ["docx"])
expect(encoded.length).to eq(1)
expect(encoded.first).to include(
@@ -228,30 +190,6 @@ RSpec.describe DiscourseAi::Completions::UploadEncoder do
expect(encoded.first).not_to have_key(:base64)
end
- it "logs docx conversion failures and skips the upload" do
- upload = create_doc_upload(contents: "raw docx bytes", filename: "sample.docx")
-
- allow(Rails.logger).to receive(:warn)
-
- encoded =
- described_class.encode(
- upload_ids: [upload.id],
- max_pixels: 1_048_576,
- allowed_kinds: %i[document],
- allowed_attachment_types: ["docx"],
- )
-
- expect(Rails.logger).to have_received(:warn).with(
- a_string_including(
- "Failed to convert .docx upload to text",
- "upload_id=#{upload.id}",
- "sample.docx",
- "Zip",
- ),
- )
- expect(encoded).to be_empty
- end
-
it "converts .xls files to text" do
upload = create_doc_upload(contents: "raw xls bytes", filename: "sample.xls")
@@ -259,13 +197,7 @@ RSpec.describe DiscourseAi::Completions::UploadEncoder do
"Name,Value\nAlice,1\n",
)
- encoded =
- described_class.encode(
- upload_ids: [upload.id],
- max_pixels: 1_048_576,
- allowed_kinds: %i[document],
- allowed_attachment_types: ["xls"],
- )
+ encoded = encode_document(upload, ["xls"])
expect(DiscourseAi::Completions::XlsToText).to have_received(:convert).with(
a_string_matching(/\.xls\z/),
@@ -288,13 +220,7 @@ RSpec.describe DiscourseAi::Completions::UploadEncoder do
allow(DiscourseAi::Completions::XlsToText).to receive(:convert).and_return(nil)
allow(Rails.logger).to receive(:warn)
- encoded =
- described_class.encode(
- upload_ids: [upload.id],
- max_pixels: 1_048_576,
- allowed_kinds: %i[document],
- allowed_attachment_types: ["xls"],
- )
+ encoded = encode_document(upload, ["xls"])
expect(encoded).to be_empty
expect(Rails.logger).to have_received(:warn).with(
@@ -311,7 +237,7 @@ RSpec.describe DiscourseAi::Completions::UploadEncoder do
it "converts .xlsx files to text" do
upload =
create_doc_upload(
- contents: xlsx_bytes("xl/worksheets/sheet1.xml" => <<~XML),
+ contents: zipped_document_bytes("xlsx", "xl/worksheets/sheet1.xml" => <<~XML),
<worksheet xmlns="http://schemas.openxmlformats.org/spreadsheetml/2006/main">
<sheetData>
<row><c t="inlineStr"><is><t>Converted XLSX spreadsheet text</t></is></c></row>
@@ -321,13 +247,7 @@ RSpec.describe DiscourseAi::Completions::UploadEncoder do
filename: "sample.xlsx",
)
- encoded =
- described_class.encode(
- upload_ids: [upload.id],
- max_pixels: 1_048_576,
- allowed_kinds: %i[document],
- allowed_attachment_types: ["xlsx"],
- )
+ encoded = encode_document(upload, ["xlsx"])
expect(encoded.length).to eq(1)
expect(encoded.first).to include(
@@ -343,44 +263,18 @@ RSpec.describe DiscourseAi::Completions::UploadEncoder do
expect(encoded.first).not_to have_key(:base64)
end
- it "logs xlsx conversion failures and skips the upload" do
- upload = create_doc_upload(contents: "raw xlsx bytes", filename: "sample.xlsx")
-
- allow(Rails.logger).to receive(:warn)
-
- encoded =
- described_class.encode(
- upload_ids: [upload.id],
- max_pixels: 1_048_576,
- allowed_kinds: %i[document],
- allowed_attachment_types: ["xlsx"],
- )
-
- expect(Rails.logger).to have_received(:warn).with(
- a_string_including(
- "Failed to convert .xlsx upload to text",
- "upload_id=#{upload.id}",
- "sample.xlsx",
- "Zip",
- ),
- )
- expect(encoded).to be_empty
- end
-
it "converts .odt files to text" do
upload =
create_doc_upload(
- contents: odt_bytes("content.xml" => odt_content_xml("Converted ODT document text")),
+ contents:
+ zipped_document_bytes(
+ "odt",
+ "content.xml" => odt_content_xml("Converted ODT document text"),
+ ),
filename: "sample.odt",
)
- encoded =
- described_class.encode(
- upload_ids: [upload.id],
- max_pixels: 1_048_576,
- allowed_kinds: %i[document],
- allowed_attachment_types: ["odt"],
- )
+ encoded = encode_document(upload, ["odt"])
expect(encoded.length).to eq(1)
expect(encoded.first).to include(
@@ -394,44 +288,18 @@ RSpec.describe DiscourseAi::Completions::UploadEncoder do
expect(encoded.first).not_to have_key(:base64)
end
- it "logs odt conversion failures and skips the upload" do
- upload = create_doc_upload(contents: "raw odt bytes", filename: "sample.odt")
-
- allow(Rails.logger).to receive(:warn)
-
- encoded =
- described_class.encode(
- upload_ids: [upload.id],
- max_pixels: 1_048_576,
- allowed_kinds: %i[document],
- allowed_attachment_types: ["odt"],
- )
-
- expect(Rails.logger).to have_received(:warn).with(
- a_string_including(
- "Failed to convert .odt upload to text",
- "upload_id=#{upload.id}",
- "sample.odt",
- "Zip",
- ),
- )
- expect(encoded).to be_empty
- end
-
it "converts .ods files to text" do
upload =
create_doc_upload(
- contents: ods_bytes("content.xml" => ods_content_xml("Sales", "Converted ODS cell")),
+ contents:
+ zipped_document_bytes(
+ "ods",
+ "content.xml" => ods_content_xml("Sales", "Converted ODS cell"),
+ ),
filename: "sample.ods",
)
- encoded =
- described_class.encode(
- upload_ids: [upload.id],
- max_pixels: 1_048_576,
- allowed_kinds: %i[document],
- allowed_attachment_types: ["ods"],
- )
+ encoded = encode_document(upload, ["ods"])
expect(encoded.length).to eq(1)
expect(encoded.first).to include(
@@ -445,28 +313,25 @@ RSpec.describe DiscourseAi::Completions::UploadEncoder do
expect(encoded.first).not_to have_key(:base64)
end
- it "logs ods conversion failures and skips the upload" do
- upload = create_doc_upload(contents: "raw ods bytes", filename: "sample.ods")
+ %w[docx xlsx odt ods].each do |extension|
+ it "logs #{extension} conversion failures and skips the upload" do
+ upload =
+ create_doc_upload(contents: "raw #{extension} bytes", filename: "sample.#{extension}")
- allow(Rails.logger).to receive(:warn)
+ allow(Rails.logger).to receive(:warn)
- encoded =
- described_class.encode(
- upload_ids: [upload.id],
- max_pixels: 1_048_576,
- allowed_kinds: %i[document],
- allowed_attachment_types: ["ods"],
- )
+ encoded = encode_document(upload, [extension])
- expect(Rails.logger).to have_received(:warn).with(
- a_string_including(
- "Failed to convert .ods upload to text",
- "upload_id=#{upload.id}",
- "sample.ods",
- "Zip",
- ),
- )
- expect(encoded).to be_empty
+ expect(Rails.logger).to have_received(:warn).with(
+ a_string_including(
+ "Failed to convert .#{extension} upload to text",
+ "upload_id=#{upload.id}",
+ "sample.#{extension}",
+ "Zip",
+ ),
+ )
+ expect(encoded).to be_empty
+ end
end
it "converts .rtf files to text" do
@@ -476,13 +341,7 @@ RSpec.describe DiscourseAi::Completions::UploadEncoder do
filename: "sample.rtf",
)
- encoded =
- described_class.encode(
- upload_ids: [upload.id],
- max_pixels: 1_048_576,
- allowed_kinds: %i[document],
- allowed_attachment_types: ["rtf"],
- )
+ encoded = encode_document(upload, ["rtf"])
expect(encoded.length).to eq(1)
expect(encoded.first).to include(
@@ -516,7 +375,7 @@ Alice,1",
encoded =
described_class.encode(
upload_ids: uploads.map(&:id),
- max_pixels: 1_048_576,
+ max_pixels: MAX_PIXELS,
allowed_kinds: %i[document],
allowed_attachment_types: %w[txt md csv],
)
@@ -551,7 +410,7 @@ Alice,1",
encoded =
described_class.encode(
upload_ids: [md_upload.id, txt_upload.id],
- max_pixels: 1_048_576,
+ max_pixels: MAX_PIXELS,
allowed_kinds: %i[document],
allowed_attachment_types: %w[markdown text],
)
@@ -565,13 +424,8 @@ Alice,1",
upload = create_doc_upload(contents: "0123456789abcdef", filename: "large.txt")
encoded =
- stub_const(described_class, :MAX_TEXT_FILE_BYTES, 10) do
- described_class.encode(
- upload_ids: [upload.id],
- max_pixels: 1_048_576,
- allowed_kinds: %i[document],
- allowed_attachment_types: ["txt"],
- )
+ stub_const(DiscourseAi::Completions::DocumentEncoder, :MAX_TEXT_FILE_BYTES, 10) do
+ encode_document(upload, ["txt"])
end
expect(encoded.length).to eq(1)
@@ -587,13 +441,7 @@ Alice,1",
allow(Rails.logger).to receive(:warn)
- encoded =
- described_class.encode(
- upload_ids: [upload.id],
- max_pixels: 1_048_576,
- allowed_kinds: %i[document],
- allowed_attachment_types: ["txt"],
- )
+ encoded = encode_document(upload, ["txt"])
expect(Rails.logger).to have_received(:warn).with(
a_string_including(
@@ -613,13 +461,7 @@ Alice,1",
allow(DiscourseAi::Completions::DocToText).to receive(:convert).and_raise(error)
allow(Rails.logger).to receive(:warn)
- encoded =
- described_class.encode(
- upload_ids: [upload.id],
- max_pixels: 1_048_576,
- allowed_kinds: %i[document],
- allowed_attachment_types: ["doc"],
- )
+ encoded = encode_document(upload, ["doc"])
expect(Rails.logger).to have_received(:warn).with(
a_string_including(
@@ -639,13 +481,7 @@ Alice,1",
allow(DiscourseAi::Completions::DocToText).to receive(:convert).and_return("\n \n")
allow(Rails.logger).to receive(:warn)
- encoded =
- described_class.encode(
- upload_ids: [upload.id],
- max_pixels: 1_048_576,
- allowed_kinds: %i[document],
- allowed_attachment_types: ["doc"],
- )
+ encoded = encode_document(upload, ["doc"])
expect(Rails.logger).to have_received(:warn).with(
a_string_including(
@@ -661,13 +497,7 @@ Alice,1",
it "sends allowed raw PDF documents when they are within the byte limit" do
upload = create_doc_upload(contents: "%PDF raw bytes", filename: "sample.pdf")
- encoded =
- described_class.encode(
- upload_ids: [upload.id],
- max_pixels: 1_048_576,
- allowed_kinds: %i[document],
- allowed_attachment_types: ["pdf"],
- )
+ encoded = encode_document(upload, ["pdf"])
expect(encoded.length).to eq(1)
expect(encoded.first[:kind]).to eq(:document)
@@ -681,13 +511,8 @@ Alice,1",
allow(Rails.logger).to receive(:warn)
encoded =
- stub_const(described_class, :MAX_RAW_DOCUMENT_BYTES, 4) do
- described_class.encode(
- upload_ids: [upload.id],
- max_pixels: 1_048_576,
- allowed_kinds: %i[document],
- allowed_attachment_types: ["pdf"],
- )
+ stub_const(DiscourseAi::Completions::DocumentEncoder, :MAX_RAW_DOCUMENT_BYTES, 4) do
+ encode_document(upload, ["pdf"])
end
expect(encoded).to be_empty
diff --git a/plugins/discourse-ai/spec/lib/completions/xlsx_to_text_spec.rb b/plugins/discourse-ai/spec/lib/completions/xlsx_to_text_spec.rb
index 6c3452d494f..222de8c04a4 100644
--- a/plugins/discourse-ai/spec/lib/completions/xlsx_to_text_spec.rb
+++ b/plugins/discourse-ai/spec/lib/completions/xlsx_to_text_spec.rb
@@ -1,24 +1,6 @@
# frozen_string_literal: true
RSpec.describe DiscourseAi::Completions::XlsxToText do
- def with_xlsx(entries)
- tempfile = Tempfile.new(%w[workbook .xlsx])
- path = tempfile.path
- tempfile.close
- FileUtils.rm_f(path)
-
- ::Zip::File.open(path, create: true) do |zip_file|
- entries.each do |name, content|
- zip_file.get_output_stream(name) { |stream| stream.write(content) }
- end
- end
-
- yield path
- ensure
- tempfile&.close
- FileUtils.rm_f(path) if path
- end
-
def workbook_xml
<<~XML
<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
@@ -54,7 +36,8 @@ RSpec.describe DiscourseAi::Completions::XlsxToText do
end
it "extracts sheets in workbook order as tab-delimited text" do
- with_xlsx(
+ with_zipped_document(
+ "xlsx",
"xl/workbook.xml" => workbook_xml,
"xl/_rels/workbook.xml.rels" => workbook_rels_xml,
"xl/sharedStrings.xml" => shared_strings_xml,
@@ -82,15 +65,19 @@ RSpec.describe DiscourseAi::Completions::XlsxToText do
end
it "falls back to worksheet entries when workbook metadata is missing" do
- with_xlsx("xl/worksheets/sheet2.xml" => <<~XML, "xl/worksheets/sheet1.xml" => <<~XML) do |path|
+ with_zipped_document(
+ "xlsx",
+ "xl/worksheets/sheet2.xml" => <<~XML,
<worksheet xmlns="http://schemas.openxmlformats.org/spreadsheetml/2006/main">
<sheetData><row><c t="inlineStr"><is><t>Second</t></is></c></row></sheetData>
</worksheet>
XML
+ "xl/worksheets/sheet1.xml" => <<~XML,
<worksheet xmlns="http://schemas.openxmlformats.org/spreadsheetml/2006/main">
<sheetData><row><c t="inlineStr"><is><t>First</t></is></c></row></sheetData>
</worksheet>
XML
+ ) do |path|
expect(described_class.convert(path)).to eq(
"Sheet: Sheet1\n\nFirst\n\nSheet: Sheet2\n\nSecond",
)
@@ -98,7 +85,7 @@ RSpec.describe DiscourseAi::Completions::XlsxToText do
end
it "limits very wide sparse rows" do
- with_xlsx("xl/worksheets/sheet1.xml" => <<~XML) do |path|
+ with_zipped_document("xlsx", "xl/worksheets/sheet1.xml" => <<~XML) do |path|
<worksheet xmlns="http://schemas.openxmlformats.org/spreadsheetml/2006/main">
<sheetData>
<row><c r="A1" t="inlineStr"><is><t>Visible</t></is></c><c r="ZZ1" t="inlineStr"><is><t>Too far away</t></is></c></row>
@@ -110,7 +97,7 @@ RSpec.describe DiscourseAi::Completions::XlsxToText do
end
it "returns blank text when the workbook has no readable sheets" do
- with_xlsx("docProps/core.xml" => "<properties />") do |path|
+ with_zipped_document("xlsx", "docProps/core.xml" => "<properties />") do |path|
expect(described_class.convert(path)).to eq("")
end
end
diff --git a/plugins/discourse-ai/spec/support/document_converter_fixtures.rb b/plugins/discourse-ai/spec/support/document_converter_fixtures.rb
new file mode 100644
index 00000000000..f22fc267b04
--- /dev/null
+++ b/plugins/discourse-ai/spec/support/document_converter_fixtures.rb
@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+module DocumentConverterFixtures
+ def with_document_file(extension, contents)
+ tempfile = Tempfile.new(["document", ".#{extension}"])
+ tempfile.binmode
+ tempfile.write(contents)
+ tempfile.rewind
+
+ yield tempfile.path
+ ensure
+ tempfile&.close!
+ end
+
+ def with_zipped_document(extension, entries)
+ tempfile = Tempfile.new(["document", ".#{extension}"])
+ path = tempfile.path
+ tempfile.close
+ FileUtils.rm_f(path)
+
+ ::Zip::File.open(path, create: true) do |zip_file|
+ entries.each do |name, content|
+ zip_file.get_output_stream(name) { |stream| stream.write(content) }
+ end
+ end
+
+ yield path
+ ensure
+ tempfile&.close
+ FileUtils.rm_f(path) if path
+ end
+
+ def zipped_document_bytes(extension, entries)
+ with_zipped_document(extension, entries) { |path| File.binread(path) }
+ end
+end
+
+RSpec.configure { |config| config.include DocumentConverterFixtures }