A platform for community discussion. Free, open, simple.
FEATURE: Extract text from HTML attachments (#43009)
Régis Hanol7cd2a78f87f8
diff --git a/plugins/discourse-ai/lib/completions/document_encoder.rb b/plugins/discourse-ai/lib/completions/document_encoder.rbindex b0df5310835..719a861a9c8 100644--- a/plugins/discourse-ai/lib/completions/document_encoder.rb+++ b/plugins/discourse-ai/lib/completions/document_encoder.rb@@ -13,6 +13,7 @@ module DiscourseAi"odt" => OdtToText,"ods" => OdsToText,"rtf" => RtfToText,+ "html" => HtmlToText,}.freezePLAIN_TEXT_ATTACHMENT_TYPES = %w[csv md txt].freezediff --git a/plugins/discourse-ai/lib/completions/html_to_text.rb b/plugins/discourse-ai/lib/completions/html_to_text.rbnew file mode 100644index 00000000000..09c8a47e2bc--- /dev/null+++ b/plugins/discourse-ai/lib/completions/html_to_text.rb@@ -0,0 +1,34 @@+# frozen_string_literal: true++module DiscourseAi+ module Completions+ class HtmlToText+ include TextNormalization++ MAX_INPUT_BYTES = 1 * 1024 * 1024++ def self.convert(path)+ new(path).convert+ end++ def initialize(path)+ @path = path+ end++ def convert+ normalize_document_text(::HtmlToMarkdown.new(read_input).to_markdown)+ end++ private++ attr_reader :path++ def read_input+ input = File.binread(path, MAX_INPUT_BYTES + 1) || +""+ input = input.byteslice(0, MAX_INPUT_BYTES) if input.bytesize > MAX_INPUT_BYTES++ Encodings.force_utf8(input.force_encoding(Encoding::UTF_8))+ end+ end+ end+enddiff --git a/plugins/discourse-ai/spec/lib/completions/html_to_text_spec.rb b/plugins/discourse-ai/spec/lib/completions/html_to_text_spec.rbnew file mode 100644index 00000000000..f737490e43b--- /dev/null+++ b/plugins/discourse-ai/spec/lib/completions/html_to_text_spec.rb@@ -0,0 +1,45 @@+# frozen_string_literal: true++RSpec.describe DiscourseAi::Completions::HtmlToText do+ before { enable_current_plugin }++ def convert(contents)+ with_document_file("html", contents) { |path| described_class.convert(path) }+ end++ it "converts structure to markdown" do+ text =+ convert("<html><body><h2>Title</h2><p>A <i>word</i>.</p><ul><li>x</li></ul></body></html>")++ expect(text).to include("## Title")+ expect(text).to include("A *word*.")+ expect(text).to include("x")+ end++ it "drops scripts and styles" do+ text =+ convert("<html><body><script>alert(1)</script><style>b{}</style><p>kept</p></body></html>")++ expect(text).to eq("kept")+ end++ it "preserves multi-byte characters" do+ expect(convert("<html><body><p>café ünïcode 日本語</p></body></html>")).to include(+ "café ünïcode 日本語",+ )+ end++ it "survives invalid byte sequences" do+ expect(convert("<html><body><p>caf\xFF\xFEe</p></body></html>")).to include("caf")+ end++ it "returns an empty string for an empty document" do+ expect(convert("")).to eq("")+ end++ it "caps the extracted text" do+ stub_const(DiscourseAi::Completions::TextNormalization, :MAX_EXTRACTED_TEXT_CHARS, 10) do+ expect(convert("<html><body><p>#{"a" * 200}</p></body></html>").length).to eq(10)+ end+ end+enddiff --git a/plugins/discourse-ai/spec/lib/completions/upload_encoder_spec.rb b/plugins/discourse-ai/spec/lib/completions/upload_encoder_spec.rbindex 39b47f74c57..b0924de15b7 100644--- a/plugins/discourse-ai/spec/lib/completions/upload_encoder_spec.rb+++ b/plugins/discourse-ai/spec/lib/completions/upload_encoder_spec.rb@@ -334,6 +334,19 @@ RSpec.describe DiscourseAi::Completions::UploadEncoder doendend+ it "routes .html uploads through the html converter" do+ upload =+ create_doc_upload(+ contents: "<html><body><h1>Heading</h1></body></html>",+ filename: "page.html",+ )++ encoded = encode_document(upload, ["html"])++ expect(encoded.first).to include(converted_from: "html", mime_type: "text/plain")+ expect(encoded.first[:text]).to include("# Heading")+ end+it "converts .rtf files to text" doupload =create_doc_upload(