Skip to content

A platform for community discussion. Free, open, simple.

FEATURE: Extract text from HTML attachments (#43009)

Régis Hanol7cd2a78f87f8
diff --git a/plugins/discourse-ai/lib/completions/document_encoder.rb b/plugins/discourse-ai/lib/completions/document_encoder.rb
index b0df5310835..719a861a9c8 100644
--- a/plugins/discourse-ai/lib/completions/document_encoder.rb
+++ b/plugins/discourse-ai/lib/completions/document_encoder.rb
@@ -13,6 +13,7 @@ module DiscourseAi
"odt" => OdtToText,
"ods" => OdsToText,
"rtf" => RtfToText,
+ "html" => HtmlToText,
}.freeze
PLAIN_TEXT_ATTACHMENT_TYPES = %w[csv md txt].freeze
diff --git a/plugins/discourse-ai/lib/completions/html_to_text.rb b/plugins/discourse-ai/lib/completions/html_to_text.rb
new file mode 100644
index 00000000000..09c8a47e2bc
--- /dev/null
+++ b/plugins/discourse-ai/lib/completions/html_to_text.rb
@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+module DiscourseAi
+ module Completions
+ class HtmlToText
+ include TextNormalization
+
+ MAX_INPUT_BYTES = 1 * 1024 * 1024
+
+ def self.convert(path)
+ new(path).convert
+ end
+
+ def initialize(path)
+ @path = path
+ end
+
+ def convert
+ normalize_document_text(::HtmlToMarkdown.new(read_input).to_markdown)
+ end
+
+ private
+
+ attr_reader :path
+
+ def read_input
+ input = File.binread(path, MAX_INPUT_BYTES + 1) || +""
+ input = input.byteslice(0, MAX_INPUT_BYTES) if input.bytesize > MAX_INPUT_BYTES
+
+ Encodings.force_utf8(input.force_encoding(Encoding::UTF_8))
+ end
+ end
+ end
+end
diff --git a/plugins/discourse-ai/spec/lib/completions/html_to_text_spec.rb b/plugins/discourse-ai/spec/lib/completions/html_to_text_spec.rb
new file mode 100644
index 00000000000..f737490e43b
--- /dev/null
+++ b/plugins/discourse-ai/spec/lib/completions/html_to_text_spec.rb
@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+RSpec.describe DiscourseAi::Completions::HtmlToText do
+ before { enable_current_plugin }
+
+ def convert(contents)
+ with_document_file("html", contents) { |path| described_class.convert(path) }
+ end
+
+ it "converts structure to markdown" do
+ text =
+ convert("<html><body><h2>Title</h2><p>A <i>word</i>.</p><ul><li>x</li></ul></body></html>")
+
+ expect(text).to include("## Title")
+ expect(text).to include("A *word*.")
+ expect(text).to include("x")
+ end
+
+ it "drops scripts and styles" do
+ text =
+ convert("<html><body><script>alert(1)</script><style>b{}</style><p>kept</p></body></html>")
+
+ expect(text).to eq("kept")
+ end
+
+ it "preserves multi-byte characters" do
+ expect(convert("<html><body><p>café ünïcode 日本語</p></body></html>")).to include(
+ "café ünïcode 日本語",
+ )
+ end
+
+ it "survives invalid byte sequences" do
+ expect(convert("<html><body><p>caf\xFF\xFEe</p></body></html>")).to include("caf")
+ end
+
+ it "returns an empty string for an empty document" do
+ expect(convert("")).to eq("")
+ end
+
+ it "caps the extracted text" do
+ stub_const(DiscourseAi::Completions::TextNormalization, :MAX_EXTRACTED_TEXT_CHARS, 10) do
+ expect(convert("<html><body><p>#{"a" * 200}</p></body></html>").length).to eq(10)
+ end
+ end
+end
diff --git a/plugins/discourse-ai/spec/lib/completions/upload_encoder_spec.rb b/plugins/discourse-ai/spec/lib/completions/upload_encoder_spec.rb
index 39b47f74c57..b0924de15b7 100644
--- a/plugins/discourse-ai/spec/lib/completions/upload_encoder_spec.rb
+++ b/plugins/discourse-ai/spec/lib/completions/upload_encoder_spec.rb
@@ -334,6 +334,19 @@ RSpec.describe DiscourseAi::Completions::UploadEncoder do
end
end
+ it "routes .html uploads through the html converter" do
+ upload =
+ create_doc_upload(
+ contents: "<html><body><h1>Heading</h1></body></html>",
+ filename: "page.html",
+ )
+
+ encoded = encode_document(upload, ["html"])
+
+ expect(encoded.first).to include(converted_from: "html", mime_type: "text/plain")
+ expect(encoded.first[:text]).to include("# Heading")
+ end
+
it "converts .rtf files to text" do
upload =
create_doc_upload(