Skip to content

A platform for community discussion. Free, open, simple.

FIX: Decode binary text responses as UTF-8 (#44880)

Sam971fc4e6f6b3
diff --git a/plugins/discourse-ai/lib/rag/web_page_fetcher.rb b/plugins/discourse-ai/lib/rag/web_page_fetcher.rb
index 451afca8fee..e168d14c5fa 100644
--- a/plugins/discourse-ai/lib/rag/web_page_fetcher.rb
+++ b/plugins/discourse-ai/lib/rag/web_page_fetcher.rb
@@ -116,7 +116,12 @@ module DiscourseAi
end
body = response[:body]
- text = content_type.in?(%w[text/plain text/markdown]) ? body : extract_html(body)
+ text =
+ if content_type.in?(%w[text/plain text/markdown])
+ body.force_encoding(Encoding::UTF_8).scrub
+ else
+ extract_html(body)
+ end
raise FetchError, "The page did not contain readable text" if text.blank?
{
diff --git a/plugins/discourse-ai/spec/lib/rag/document_source_refresher_spec.rb b/plugins/discourse-ai/spec/lib/rag/document_source_refresher_spec.rb
index 81648862f06..80875f1da45 100644
--- a/plugins/discourse-ai/spec/lib/rag/document_source_refresher_spec.rb
+++ b/plugins/discourse-ai/spec/lib/rag/document_source_refresher_spec.rb
@@ -67,6 +67,32 @@ RSpec.describe DiscourseAi::Rag::DocumentSourceRefresher do
expect(UploadReference.exists?(target: agent, upload_id: pending_upload.id)).to eq(true)
end
+ it "uploads and digests non-ASCII Markdown fetched as binary data" do
+ stub_request(:get, source.url).to_return(
+ status: 200,
+ headers: {
+ "Content-Type" => "text/markdown; charset=utf-8",
+ },
+ body: "Knowledge — café\nTruncated: \xE2".b,
+ )
+
+ expect { described_class.refresh(source) }.to change(Jobs::DigestRagUpload.jobs, :size).by(1)
+
+ source.reload
+ expect(source.last_error).to be_nil
+ expect(source.pending_upload).to be_present
+
+ Jobs::DigestRagUpload.new.execute(
+ upload_id: source.pending_upload_id,
+ target_id: agent.id,
+ target_type: "AiAgent",
+ )
+
+ fragment = RagDocumentFragment.find_by!(target: agent, upload_id: source.pending_upload_id)
+ expect(fragment.fragment).to eq("Knowledge — café\nTruncated: �")
+ expect(JSON.parse(fragment.metadata)).to eq("source_url" => source.url)
+ end
+
it "retains existing content and retries later when refresh fails" do
upload = Fabricate(:upload)
source.update_columns(upload_id: upload.id)
diff --git a/plugins/discourse-ai/spec/lib/rag/web_page_fetcher_spec.rb b/plugins/discourse-ai/spec/lib/rag/web_page_fetcher_spec.rb
index cdde62a7506..097539b63d5 100644
--- a/plugins/discourse-ai/spec/lib/rag/web_page_fetcher_spec.rb
+++ b/plugins/discourse-ai/spec/lib/rag/web_page_fetcher_spec.rb
@@ -134,6 +134,40 @@ RSpec.describe DiscourseAi::Rag::WebPageFetcher do
).to have_been_made.once
end
+ %w[text/plain text/markdown].each do |content_type|
+ it "decodes binary #{content_type} responses as UTF-8 and replaces invalid bytes" do
+ url = "https://example.com/docs"
+ stub_request(:get, url).to_return(
+ status: 200,
+ headers: {
+ "Content-Type" => "#{content_type}; charset=utf-8",
+ },
+ body: "Knowledge — café\nTruncated: \xE2".b,
+ )
+
+ result = described_class.fetch(url: url)
+
+ expect(result[:text].encoding).to eq(Encoding::UTF_8)
+ expect(result[:text]).to eq("Knowledge — café\nTruncated: �")
+ end
+ end
+
+ it "preserves UTF-8 characters split across binary response chunks" do
+ url = "https://example.com/docs"
+ response = Net::HTTPOK.new("1.1", "200", "OK")
+ response["Content-Type"] = "text/markdown"
+ response.stubs(:read_body).multiple_yields(["Knowledge \xE2".b], ["\x80\x94 café".b])
+ http = stub
+ http.stubs(:read_timeout=)
+ http.stubs(:request).yields(response)
+ FinalDestination::HTTP.stubs(:start).yields(http)
+
+ result = described_class.fetch(url: url)
+
+ expect(result[:text].encoding).to eq(Encoding::UTF_8)
+ expect(result[:text]).to eq("Knowledge — café")
+ end
+
it "follows redirects without an additional discovery request" do
original_url = "https://example.com/docs"
redirected_url = "https://docs.example.com/guide"