Skip to content

A platform for community discussion. Free, open, simple.

FEATURE: extract text from ODT and ODS document uploads (#39711)

Rafael dos Santos Silva2e3b64fa742b
diff --git a/plugins/discourse-ai/assets/javascripts/discourse/components/ai-llm-attachment-types.gjs b/plugins/discourse-ai/assets/javascripts/discourse/components/ai-llm-attachment-types.gjs
index 4c5af18de5a..e2a8b4bb2bb 100644
--- a/plugins/discourse-ai/assets/javascripts/discourse/components/ai-llm-attachment-types.gjs
+++ b/plugins/discourse-ai/assets/javascripts/discourse/components/ai-llm-attachment-types.gjs
@@ -11,9 +11,11 @@ const DEFAULT_CHOICES = [
"txt",
"doc",
"docx",
+ "odt",
"csv",
"xls",
"xlsx",
+ "ods",
"rtf",
"html",
"md",
diff --git a/plugins/discourse-ai/lib/completions/ods_to_text.rb b/plugins/discourse-ai/lib/completions/ods_to_text.rb
new file mode 100644
index 00000000000..6608256fe2a
--- /dev/null
+++ b/plugins/discourse-ai/lib/completions/ods_to_text.rb
@@ -0,0 +1,189 @@
+# frozen_string_literal: true
+
+require "nokogiri"
+require "compression/safe_zip_reader"
+
+module DiscourseAi
+ module Completions
+ class OdsToText
+ MAX_ENTRY_XML_BYTES = 10 * 1024 * 1024
+ MAX_TOTAL_XML_BYTES = 30 * 1024 * 1024
+ MAX_ZIP_ENTRIES = 1_000
+ MAX_EXTRACTED_TEXT_CHARS = 100_001
+ MAX_SHEETS = 50
+ MAX_ROWS_PER_SHEET = 10_000
+ MAX_COLUMNS = 200
+
+ CONTENT_PATH = "content.xml"
+
+ class EntryTooLargeError < StandardError
+ end
+
+ def self.convert(path)
+ new(path).convert
+ end
+
+ def initialize(path)
+ @path = path
+ end
+
+ def convert
+ Compression::SafeZipReader.open(
+ path,
+ max_entries: MAX_ZIP_ENTRIES,
+ max_total_bytes: MAX_TOTAL_XML_BYTES,
+ ) do |zip_file|
+ xml = read_xml_entry(zip_file, CONTENT_PATH)
+ return "" if xml.blank?
+
+ normalize_document_text(extract_sheets(xml))
+ end
+ end
+
+ private
+
+ attr_reader :path
+
+ def read_xml_entry(zip_file, name)
+ entry = zip_file.find_entry(name)
+ return if entry.blank? || entry.directory?
+
+ zip_file.read_entry(entry, max_bytes: MAX_ENTRY_XML_BYTES)
+ rescue Compression::SafeZipReader::EntryTooLargeError => e
+ raise EntryTooLargeError, e.message
+ end
+
+ def extract_sheets(xml)
+ doc = Nokogiri.XML(force_utf8(xml)) { |config| config.recover.nonet }
+ doc.remove_namespaces!
+
+ sheets = doc.xpath("//body/spreadsheet/table")
+ return "" if sheets.blank?
+
+ sheet_texts = []
+ extracted_chars = 0
+
+ sheets
+ .first(MAX_SHEETS)
+ .each_with_index do |sheet, index|
+ name = sheet["name"].presence || "Sheet#{index + 1}"
+ rows_text = sheet_rows(sheet)
+ next if rows_text.blank?
+
+ block = "Sheet: #{name}\n\n#{rows_text}"
+ sheet_texts << block
+ extracted_chars += block.length + 2
+ break if extracted_chars > MAX_EXTRACTED_TEXT_CHARS
+ end
+
+ sheet_texts.join("\n\n")
+ end
+
+ def sheet_rows(sheet)
+ rows = sheet.xpath("./table-row")
+ return if rows.blank?
+
+ row_texts = []
+ extracted_chars = 0
+
+ rows
+ .first(MAX_ROWS_PER_SHEET)
+ .each do |row|
+ line = row_line(row)
+ next if line.blank?
+
+ row_texts << line
+ extracted_chars += line.length + 1
+ break if extracted_chars > MAX_EXTRACTED_TEXT_CHARS
+ end
+
+ row_texts.join("\n")
+ end
+
+ def row_line(row)
+ values = []
+
+ row
+ .xpath("./table-cell | ./covered-table-cell")
+ .each do |cell|
+ break if values.length >= MAX_COLUMNS
+
+ repeat = cell["number-columns-repeated"].to_i
+ repeat = 1 if repeat < 1
+
+ text = cell.name == "covered-table-cell" ? "" : cell_text(cell)
+
+ if text.blank?
+ values << ""
+ else
+ slots = [repeat, MAX_COLUMNS - values.length].min
+ slots.times { values << text }
+ end
+ end
+
+ values.pop while values.last&.blank? && values.any?
+ values.join("\t")
+ end
+
+ def cell_text(cell)
+ paragraphs = cell.xpath("./p")
+ return normalize_inline(paragraphs.map { |p| inline_text(p) }.join("\n")) if paragraphs.any?
+
+ case cell["value-type"]
+ when "boolean"
+ cell["boolean-value"].to_s == "true" ? "TRUE" : "FALSE"
+ when "date"
+ normalize_inline(cell["date-value"])
+ when "time"
+ normalize_inline(cell["time-value"])
+ else
+ normalize_inline(cell["value"])
+ end
+ end
+
+ def inline_text(node)
+ out = +""
+ collect_inline(node, out)
+ out
+ end
+
+ def collect_inline(node, out)
+ node.children.each do |child|
+ if child.text?
+ out << child.content
+ elsif child.element?
+ case child.name
+ when "tab"
+ out << "\t"
+ when "s"
+ count = child["c"].to_i
+ count = 1 if count < 1
+ out << (" " * count)
+ when "line-break"
+ out << "\n"
+ else
+ collect_inline(child, out)
+ end
+ end
+ end
+ end
+
+ def normalize_inline(text)
+ force_utf8(text).gsub("\u00A0", " ").gsub(/[ \t\r\n]+/, " ").strip
+ end
+
+ def normalize_document_text(text)
+ force_utf8(text)
+ .gsub("\u00A0", " ")
+ .gsub(/\r\n?/, "\n")
+ .gsub(/[ \t]+\n/, "\n")
+ .gsub(/\n{3,}/, "\n\n")
+ .strip
+ end
+
+ def force_utf8(text)
+ text.to_s.encode("UTF-8", invalid: :replace, undef: :replace, replace: "")
+ end
+ end
+ end
+end
diff --git a/plugins/discourse-ai/lib/completions/odt_to_text.rb b/plugins/discourse-ai/lib/completions/odt_to_text.rb
new file mode 100644
index 00000000000..60a77d29d09
--- /dev/null
+++ b/plugins/discourse-ai/lib/completions/odt_to_text.rb
@@ -0,0 +1,194 @@
+# frozen_string_literal: true
+
+require "nokogiri"
+require "compression/safe_zip_reader"
+
+module DiscourseAi
+ module Completions
+ class OdtToText
+ MAX_ENTRY_XML_BYTES = 10 * 1024 * 1024
+ MAX_TOTAL_XML_BYTES = 30 * 1024 * 1024
+ MAX_ZIP_ENTRIES = 1_000
+ MAX_EXTRACTED_TEXT_CHARS = 100_001
+ MAX_PARAGRAPHS = 20_000
+ MAX_LIST_DEPTH = 20
+
+ CONTENT_PATH = "content.xml"
+
+ class EntryTooLargeError < StandardError
+ end
+
+ def self.convert(path)
+ new(path).convert
+ end
+
+ def initialize(path)
+ @path = path
+ end
+
+ def convert
+ Compression::SafeZipReader.open(
+ path,
+ max_entries: MAX_ZIP_ENTRIES,
+ max_total_bytes: MAX_TOTAL_XML_BYTES,
+ ) do |zip_file|
+ xml = read_xml_entry(zip_file, CONTENT_PATH)
+ return "" if xml.blank?
+
+ normalize_document_text(extract_text(xml))
+ end
+ end
+
+ private
+
+ attr_reader :path
+
+ def read_xml_entry(zip_file, name)
+ entry = zip_file.find_entry(name)
+ return if entry.blank? || entry.directory?
+
+ zip_file.read_entry(entry, max_bytes: MAX_ENTRY_XML_BYTES)
+ rescue Compression::SafeZipReader::EntryTooLargeError => e
+ raise EntryTooLargeError, e.message
+ end
+
+ def extract_text(xml)
+ doc = Nokogiri.XML(force_utf8(xml)) { |config| config.recover.nonet }
+ doc.remove_namespaces!
+
+ body = doc.at_xpath("//body/text") || doc.at_xpath("//body") || doc.root
+ return "" if body.nil?
+
+ @text_parts = []
+ @extracted_chars = 0
+ @paragraph_count = 0
+
+ walk(body, list_depth: 0)
+
+ @text_parts.join("\n")
+ end
+
+ def walk(node, list_depth:)
+ node.element_children.each do |child|
+ break if hit_limits?
+
+ case child.name
+ when "p", "h"
+ push_paragraph(child)
+ when "list"
+ walk_list(child, depth: list_depth) if list_depth < MAX_LIST_DEPTH
+ when "table"
+ walk_table(child)
+ when "frame", "section", "text-box"
+ walk(child, list_depth: list_depth)
+ else
+ walk(child, list_depth: list_depth) if child.element_children.any?
+ end
+ end
+ end
+
+ def walk_list(list_node, depth:)
+ list_node
+ .xpath("./list-item")
+ .each do |item|
+ break if hit_limits?
+
+ item.element_children.each do |child|
+ case child.name
+ when "p", "h"
+ push_paragraph(child, prefix: "#{" " * depth}- ")
+ when "list"
+ walk_list(child, depth: depth + 1) if depth + 1 < MAX_LIST_DEPTH
+ else
+ walk(child, list_depth: depth + 1) if child.element_children.any?
+ end
+ end
+ end
+ end
+
+ def walk_table(table_node)
+ table_node
+ .xpath(".//table-row")
+ .each do |row|
+ break if hit_limits?
+
+ cells = row.xpath("./table-cell").map { |cell| normalize_inline(inline_text(cell)) }
+ cells.pop while cells.last&.blank? && cells.any?
+ next if cells.empty?
+
+ push_line(cells.join("\t"))
+ end
+ end
+
+ def push_paragraph(node, prefix: "")
+ text = normalize_paragraph(inline_text(node))
+ return if text.blank?
+
+ push_line("#{prefix}#{text}")
+ end
+
+ def push_line(line)
+ @text_parts << line
+ @extracted_chars += line.length + 1
+ @paragraph_count += 1
+ end
+
+ def hit_limits?
+ @paragraph_count >= MAX_PARAGRAPHS || @extracted_chars > MAX_EXTRACTED_TEXT_CHARS
+ end
+
+ def inline_text(node)
+ out = +""
+ collect_inline(node, out)
+ out
+ end
+
+ def collect_inline(node, out)
+ node.children.each do |child|
+ if child.text?
+ out << child.content
+ elsif child.element?
+ case child.name
+ when "tab"
+ out << "\t"
+ when "s"
+ count = child["c"].to_i
+ count = 1 if count < 1
+ out << (" " * count)
+ when "line-break"
+ out << "\n"
+ else
+ collect_inline(child, out)
+ end
+ end
+ end
+ end
+
+ def normalize_paragraph(text)
+ force_utf8(text)
+ .gsub("\u00A0", " ")
+ .gsub(/\r\n?/, "\n")
+ .gsub(/[ \t]+\n/, "\n")
+ .gsub(/\n[ \t]+/, "\n")
+ .rstrip
+ end
+
+ def normalize_inline(text)
+ force_utf8(text).gsub("\u00A0", " ").gsub(/[ \t\r\n]+/, " ").strip
+ end
+
+ def normalize_document_text(text)
+ force_utf8(text)
+ .gsub("\u00A0", " ")
+ .gsub(/\r\n?/, "\n")
+ .gsub(/[ \t]+\n/, "\n")
+ .gsub(/\n{3,}/, "\n\n")
+ .strip
+ end
+
+ def force_utf8(text)
+ text.to_s.encode("UTF-8", invalid: :replace, undef: :replace, replace: "")
+ end
+ end
+ end
+end
diff --git a/plugins/discourse-ai/lib/completions/upload_encoder.rb b/plugins/discourse-ai/lib/completions/upload_encoder.rb
index c4b64b525b2..a4ab43778f3 100644
--- a/plugins/discourse-ai/lib/completions/upload_encoder.rb
+++ b/plugins/discourse-ai/lib/completions/upload_encoder.rb
@@ -63,6 +63,8 @@ module DiscourseAi
return "doc" if ext == "doc" || mime == "application/msword"
return "xlsx" if ext == "xlsx" || mime.include?("spreadsheetml.sheet")
return "xls" if ext == "xls" || mime == "application/vnd.ms-excel"
+ return "odt" if ext == "odt" || mime.include?("opendocument.text")
+ return "ods" if ext == "ods" || mime.include?("opendocument.spreadsheet")
return "csv" if ext == "csv" || mime.include?("text/csv") || mime.include?("csv")
return "txt" if ext == "txt" || mime.include?("text/plain")
return "rtf" if ext == "rtf" || mime.include?("rtf")
@@ -105,6 +107,12 @@ module DiscourseAi
elsif attachment_type == "xlsx"
text_payload = xlsx_to_text_payload(upload, path)
return text_payload if text_payload
+ elsif attachment_type == "odt"
+ text_payload = odt_to_text_payload(upload, path)
+ return text_payload if text_payload
+ elsif attachment_type == "ods"
+ text_payload = ods_to_text_payload(upload, path)
+ return text_payload if text_payload
elsif attachment_type == "rtf"
text_payload = rtf_to_text_payload(upload, path)
return text_payload if text_payload
@@ -203,6 +211,34 @@ module DiscourseAi
nil
end
+ def odt_to_text_payload(upload, path)
+ text = normalize_extracted_text(DiscourseAi::Completions::OdtToText.convert(path))
+
+ if text.blank?
+ log_document_conversion_failure(upload, "odt", "ODT converter returned blank output")
+ return
+ end
+
+ text_document_payload(upload, path, text, converted_from: "odt")
+ rescue StandardError => e
+ log_document_conversion_failure(upload, "odt", "#{e.class}: #{e.message}")
+ nil
+ end
+
+ def ods_to_text_payload(upload, path)
+ text = normalize_extracted_text(DiscourseAi::Completions::OdsToText.convert(path))
+
+ if text.blank?
+ log_document_conversion_failure(upload, "ods", "ODS converter returned blank output")
+ return
+ end
+
+ text_document_payload(upload, path, text, converted_from: "ods")
+ rescue StandardError => e
+ log_document_conversion_failure(upload, "ods", "#{e.class}: #{e.message}")
+ nil
+ end
+
def text_file_payload(upload, path, attachment_type)
text = normalize_extracted_text(read_utf8_text_file(path))
diff --git a/plugins/discourse-ai/spec/lib/completions/ods_to_text_spec.rb b/plugins/discourse-ai/spec/lib/completions/ods_to_text_spec.rb
new file mode 100644
index 00000000000..ea4968acee8
--- /dev/null
+++ b/plugins/discourse-ai/spec/lib/completions/ods_to_text_spec.rb
@@ -0,0 +1,122 @@
+# frozen_string_literal: true
+
+RSpec.describe DiscourseAi::Completions::OdsToText do
+ def with_ods(entries)
+ tempfile = Tempfile.new(%w[spreadsheet .ods])
+ path = tempfile.path
+ tempfile.close
+ FileUtils.rm_f(path)
+
+ ::Zip::File.open(path, create: true) do |zip_file|
+ entries.each do |name, content|
+ zip_file.get_output_stream(name) { |stream| stream.write(content) }
+ end
+ end
+
+ yield path
+ ensure
+ tempfile&.close
+ FileUtils.rm_f(path) if path
+ end
+
+ def ods_content(body)
+ <<~XML
+ <?xml version="1.0" encoding="UTF-8"?>
+ <office:document-content
+ xmlns:office="urn:oasis:names:tc:opendocument:xmlns:office:1.0"
+ xmlns:text="urn:oasis:names:tc:opendocument:xmlns:text:1.0"
+ xmlns:table="urn:oasis:names:tc:opendocument:xmlns:table:1.0">
+ <office:body>
+ <office:spreadsheet>
+ #{body}
+ </office:spreadsheet>
+ </office:body>
+ </office:document-content>
+ XML
+ end
+
+ it "extracts cells from each named sheet" do
+ with_ods("content.xml" => ods_content(<<~XML)) do |path|
+ <table:table table:name="Summary">
+ <table:table-row>
+ <table:table-cell office:value-type="string"><text:p>Name</text:p></table:table-cell>
+ <table:table-cell office:value-type="string"><text:p>Value</text:p></table:table-cell>
+ </table:table-row>
+ <table:table-row>
+ <table:table-cell office:value-type="string"><text:p>Alice</text:p></table:table-cell>
+ <table:table-cell office:value-type="float" office:value="1"><text:p>1</text:p></table:table-cell>
+ </table:table-row>
+ </table:table>
+ <table:table table:name="Notes">
+ <table:table-row>
+ <table:table-cell office:value-type="string"><text:p>hello</text:p></table:table-cell>
+ </table:table-row>
+ </table:table>
+ XML
+ expect(described_class.convert(path)).to eq(
+ "Sheet: Summary\n\nName\tValue\nAlice\t1\n\nSheet: Notes\n\nhello",
+ )
+ end
+ end
+
+ it "uses Sheet1, Sheet2 ... when no name is set" do
+ with_ods("content.xml" => ods_content(<<~XML)) do |path|
+ <table:table>
+ <table:table-row>
+ <table:table-cell office:value-type="string"><text:p>only</text:p></table:table-cell>
+ </table:table-row>
+ </table:table>
+ XML
+ expect(described_class.convert(path)).to eq("Sheet: Sheet1\n\nonly")
+ end
+ end
+
+ it "renders typed values when no inline paragraph is present" do
+ with_ods("content.xml" => ods_content(<<~XML)) do |path|
+ <table:table table:name="Types">
+ <table:table-row>
+ <table:table-cell office:value-type="boolean" office:boolean-value="true"/>
+ <table:table-cell office:value-type="date" office:date-value="2026-05-04"/>
+ <table:table-cell office:value-type="float" office:value="42"/>
+ </table:table-row>
+ </table:table>
+ XML
+ expect(described_class.convert(path)).to eq("Sheet: Types\n\nTRUE\t2026-05-04\t42")
+ end
+ end
+
+ it "expands a non-empty number-columns-repeated up to MAX_COLUMNS" do
+ with_ods("content.xml" => ods_content(<<~XML)) do |path|
+ <table:table table:name="Repeats">
+ <table:table-row>
+ <table:table-cell office:value-type="string" table:number-columns-repeated="3">
+ <text:p>x</text:p>
+ </table:table-cell>
+ <table:table-cell office:value-type="string"><text:p>y</text:p></table:table-cell>
+ </table:table-row>
+ </table:table>
+ XML
+ expect(described_class.convert(path)).to eq("Sheet: Repeats\n\nx\tx\tx\ty")
+ end
+ end
+
+ it "ignores covered (merge-continuation) cells and trims trailing empties" do
+ with_ods("content.xml" => ods_content(<<~XML)) do |path|
+ <table:table table:name="Merge">
+ <table:table-row>
+ <table:table-cell office:value-type="string"><text:p>head</text:p></table:table-cell>
+ <table:covered-table-cell/>
+ <table:table-cell/>
+ </table:table-row>
+ </table:table>
+ XML
+ expect(described_class.convert(path)).to eq("Sheet: Merge\n\nhead")
+ end
+ end
+
+ it "returns blank text when content.xml is missing" do
+ with_ods("META-INF/manifest.xml" => "<manifest/>") do |path|
+ expect(described_class.convert(path)).to eq("")
+ end
+ end
+end
diff --git a/plugins/discourse-ai/spec/lib/completions/odt_to_text_spec.rb b/plugins/discourse-ai/spec/lib/completions/odt_to_text_spec.rb
new file mode 100644
index 00000000000..45c059e093b
--- /dev/null
+++ b/plugins/discourse-ai/spec/lib/completions/odt_to_text_spec.rb
@@ -0,0 +1,113 @@
+# frozen_string_literal: true
+
+RSpec.describe DiscourseAi::Completions::OdtToText do
+ def with_odt(entries)
+ tempfile = Tempfile.new(%w[document .odt])
+ path = tempfile.path
+ tempfile.close
+ FileUtils.rm_f(path)
+
+ ::Zip::File.open(path, create: true) do |zip_file|
+ entries.each do |name, content|
+ zip_file.get_output_stream(name) { |stream| stream.write(content) }
+ end
+ end
+
+ yield path
+ ensure
+ tempfile&.close
+ FileUtils.rm_f(path) if path
+ end
+
+ def odt_content(body)
+ <<~XML
+ <?xml version="1.0" encoding="UTF-8"?>
+ <office:document-content
+ xmlns:office="urn:oasis:names:tc:opendocument:xmlns:office:1.0"
+ xmlns:text="urn:oasis:names:tc:opendocument:xmlns:text:1.0"
+ xmlns:table="urn:oasis:names:tc:opendocument:xmlns:table:1.0"
+ xmlns:draw="urn:oasis:names:tc:opendocument:xmlns:drawing:1.0">
+ <office:body>
+ <office:text>
+ #{body}
+ </office:text>
+ </office:body>
+ </office:document-content>
+ XML
+ end
+
+ it "extracts headings, paragraphs, tabs, line breaks, and spans" do
+ with_odt("content.xml" => odt_content(<<~XML)) do |path|
+ <text:h text:outline-level="1">Title</text:h>
+ <text:p>Hello<text:tab/>world<text:line-break/>second line</text:p>
+ <text:p>Run with <text:span>nested <text:span>span</text:span></text:span> text</text:p>
+ XML
+ expect(described_class.convert(path)).to eq(
+ "Title\nHello\tworld\nsecond line\nRun with nested span text",
+ )
+ end
+ end
+
+ it "expands repeated spaces from text:s c=N" do
+ with_odt("content.xml" => odt_content(<<~XML)) do |path|
+ <text:p>A<text:s text:c="3"/>B</text:p>
+ XML
+ expect(described_class.convert(path)).to eq("A B")
+ end
+ end
+
+ it "renders nested lists with depth-aware bullet prefixes" do
+ with_odt("content.xml" => odt_content(<<~XML)) do |path|
+ <text:list>
+ <text:list-item><text:p>One</text:p></text:list-item>
+ <text:list-item>
+ <text:p>Two</text:p>
+ <text:list>
+ <text:list-item><text:p>Two.a</text:p></text:list-item>
+ <text:list-item><text:p>Two.b</text:p></text:list-item>
+ </text:list>
+ </text:list-item>
+ <text:list-item><text:p>Three</text:p></text:list-item>
+ </text:list>
+ XML
+ expect(described_class.convert(path)).to eq("- One\n- Two\n - Two.a\n - Two.b\n- Three")
+ end
+ end
+
+ it "renders tables as tab-separated rows" do
+ with_odt("content.xml" => odt_content(<<~XML)) do |path|
+ <table:table>
+ <table:table-row>
+ <table:table-cell><text:p>Name</text:p></table:table-cell>
+ <table:table-cell><text:p>Value</text:p></table:table-cell>
+ </table:table-row>
+ <table:table-row>
+ <table:table-cell><text:p>Alice</text:p></table:table-cell>
+ <table:table-cell><text:p>1</text:p></table:table-cell>
+ </table:table-row>
+ </table:table>
+ XML
+ expect(described_class.convert(path)).to eq("Name\tValue\nAlice\t1")
+ end
+ end
+
+ it "walks into frames and text-boxes for callout text" do
+ with_odt("content.xml" => odt_content(<<~XML)) do |path|
+ <text:p>Body before</text:p>
+ <draw:frame>
+ <draw:text-box>
+ <text:p>Callout</text:p>
+ </draw:text-box>
+ </draw:frame>
+ <text:p>Body after</text:p>
+ XML
+ expect(described_class.convert(path)).to eq("Body before\nCallout\nBody after")
+ end
+ end
+
+ it "returns blank text when content.xml is missing" do
+ with_odt("META-INF/manifest.xml" => "<manifest/>") do |path|
+ expect(described_class.convert(path)).to eq("")
+ end
+ end
+end
diff --git a/plugins/discourse-ai/spec/lib/completions/upload_encoder_spec.rb b/plugins/discourse-ai/spec/lib/completions/upload_encoder_spec.rb
index 8614397694d..6c3ec1c049f 100644
--- a/plugins/discourse-ai/spec/lib/completions/upload_encoder_spec.rb
+++ b/plugins/discourse-ai/spec/lib/completions/upload_encoder_spec.rb
@@ -47,6 +47,49 @@ RSpec.describe DiscourseAi::Completions::UploadEncoder do
zip_bytes(entries, ".xlsx")
end
+ def odt_bytes(entries)
+ zip_bytes(entries, ".odt")
+ end
+
+ def ods_bytes(entries)
+ zip_bytes(entries, ".ods")
+ end
+
+ def odt_content_xml(text)
+ <<~XML
+ <?xml version="1.0" encoding="UTF-8"?>
+ <office:document-content
+ xmlns:office="urn:oasis:names:tc:opendocument:xmlns:office:1.0"
+ xmlns:text="urn:oasis:names:tc:opendocument:xmlns:text:1.0">
+ <office:body>
+ <office:text>
+ <text:p>#{text}</text:p>
+ </office:text>
+ </office:body>
+ </office:document-content>
+ XML
+ end
+
+ def ods_content_xml(sheet_name, cell_text)
+ <<~XML
+ <?xml version="1.0" encoding="UTF-8"?>
+ <office:document-content
+ xmlns:office="urn:oasis:names:tc:opendocument:xmlns:office:1.0"
+ xmlns:text="urn:oasis:names:tc:opendocument:xmlns:text:1.0"
+ xmlns:table="urn:oasis:names:tc:opendocument:xmlns:table:1.0">
+ <office:body>
+ <office:spreadsheet>
+ <table:table table:name="#{sheet_name}">
+ <table:table-row>
+ <table:table-cell office:value-type="string"><text:p>#{cell_text}</text:p></table:table-cell>
+ </table:table-row>
+ </table:table>
+ </office:spreadsheet>
+ </office:body>
+ </office:document-content>
+ XML
+ end
+
def docx_document_xml(text)
<<~XML
<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
@@ -308,6 +351,108 @@ RSpec.describe DiscourseAi::Completions::UploadEncoder do
expect(encoded).to be_empty
end
+ it "converts .odt files to text" do
+ upload =
+ create_doc_upload(
+ contents: odt_bytes("content.xml" => odt_content_xml("Converted ODT document text")),
+ filename: "sample.odt",
+ )
+
+ encoded =
+ described_class.encode(
+ upload_ids: [upload.id],
+ max_pixels: 1_048_576,
+ allowed_kinds: %i[document],
+ allowed_attachment_types: ["odt"],
+ )
+
+ expect(encoded.length).to eq(1)
+ expect(encoded.first).to include(
+ kind: :document,
+ filename: "sample.odt",
+ mime_type: "text/plain",
+ converted_from: "odt",
+ )
+ expect(encoded.first[:text]).to start_with("Uploaded document: sample.odt (")
+ expect(encoded.first[:text]).to include("\n\nConverted ODT document text")
+ expect(encoded.first).not_to have_key(:base64)
+ end
+
+ it "logs odt conversion failures and skips the upload" do
+ upload = create_doc_upload(contents: "raw odt bytes", filename: "sample.odt")
+
+ allow(Rails.logger).to receive(:warn)
+
+ encoded =
+ described_class.encode(
+ upload_ids: [upload.id],
+ max_pixels: 1_048_576,
+ allowed_kinds: %i[document],
+ allowed_attachment_types: ["odt"],
+ )
+
+ expect(Rails.logger).to have_received(:warn).with(
+ a_string_including(
+ "Failed to convert .odt upload to text",
+ "upload_id=#{upload.id}",
+ "sample.odt",
+ "Zip",
+ ),
+ )
+ expect(encoded).to be_empty
+ end
+
+ it "converts .ods files to text" do
+ upload =
+ create_doc_upload(
+ contents: ods_bytes("content.xml" => ods_content_xml("Sales", "Converted ODS cell")),
+ filename: "sample.ods",
+ )
+
+ encoded =
+ described_class.encode(
+ upload_ids: [upload.id],
+ max_pixels: 1_048_576,
+ allowed_kinds: %i[document],
+ allowed_attachment_types: ["ods"],
+ )
+
+ expect(encoded.length).to eq(1)
+ expect(encoded.first).to include(
+ kind: :document,
+ filename: "sample.ods",
+ mime_type: "text/plain",
+ converted_from: "ods",
+ )
+ expect(encoded.first[:text]).to start_with("Uploaded document: sample.ods (")
+ expect(encoded.first[:text]).to include("\n\nSheet: Sales\n\nConverted ODS cell")
+ expect(encoded.first).not_to have_key(:base64)
+ end
+
+ it "logs ods conversion failures and skips the upload" do
+ upload = create_doc_upload(contents: "raw ods bytes", filename: "sample.ods")
+
+ allow(Rails.logger).to receive(:warn)
+
+ encoded =
+ described_class.encode(
+ upload_ids: [upload.id],
+ max_pixels: 1_048_576,
+ allowed_kinds: %i[document],
+ allowed_attachment_types: ["ods"],
+ )
+
+ expect(Rails.logger).to have_received(:warn).with(
+ a_string_including(
+ "Failed to convert .ods upload to text",
+ "upload_id=#{upload.id}",
+ "sample.ods",
+ "Zip",
+ ),
+ )
+ expect(encoded).to be_empty
+ end
+
it "converts .rtf files to text" do
upload =
create_doc_upload(