A platform for community discussion. Free, open, simple.
FIX: Decode binary text responses as UTF-8 (#44880)
Sam971fc4e6f6b3
diff --git a/plugins/discourse-ai/lib/rag/web_page_fetcher.rb b/plugins/discourse-ai/lib/rag/web_page_fetcher.rbindex 451afca8fee..e168d14c5fa 100644--- a/plugins/discourse-ai/lib/rag/web_page_fetcher.rb+++ b/plugins/discourse-ai/lib/rag/web_page_fetcher.rb@@ -116,7 +116,12 @@ module DiscourseAiendbody = response[:body]- text = content_type.in?(%w[text/plain text/markdown]) ? body : extract_html(body)+ text =+ if content_type.in?(%w[text/plain text/markdown])+ body.force_encoding(Encoding::UTF_8).scrub+ else+ extract_html(body)+ endraise FetchError, "The page did not contain readable text" if text.blank?{diff --git a/plugins/discourse-ai/spec/lib/rag/document_source_refresher_spec.rb b/plugins/discourse-ai/spec/lib/rag/document_source_refresher_spec.rbindex 81648862f06..80875f1da45 100644--- a/plugins/discourse-ai/spec/lib/rag/document_source_refresher_spec.rb+++ b/plugins/discourse-ai/spec/lib/rag/document_source_refresher_spec.rb@@ -67,6 +67,32 @@ RSpec.describe DiscourseAi::Rag::DocumentSourceRefresher doexpect(UploadReference.exists?(target: agent, upload_id: pending_upload.id)).to eq(true)end+ it "uploads and digests non-ASCII Markdown fetched as binary data" do+ stub_request(:get, source.url).to_return(+ status: 200,+ headers: {+ "Content-Type" => "text/markdown; charset=utf-8",+ },+ body: "Knowledge — café\nTruncated: \xE2".b,+ )++ expect { described_class.refresh(source) }.to change(Jobs::DigestRagUpload.jobs, :size).by(1)++ source.reload+ expect(source.last_error).to be_nil+ expect(source.pending_upload).to be_present++ Jobs::DigestRagUpload.new.execute(+ upload_id: source.pending_upload_id,+ target_id: agent.id,+ target_type: "AiAgent",+ )++ fragment = RagDocumentFragment.find_by!(target: agent, upload_id: source.pending_upload_id)+ expect(fragment.fragment).to eq("Knowledge — café\nTruncated: �")+ expect(JSON.parse(fragment.metadata)).to eq("source_url" => source.url)+ end+it "retains existing content and retries later when refresh fails" doupload = Fabricate(:upload)source.update_columns(upload_id: upload.id)diff --git a/plugins/discourse-ai/spec/lib/rag/web_page_fetcher_spec.rb b/plugins/discourse-ai/spec/lib/rag/web_page_fetcher_spec.rbindex cdde62a7506..097539b63d5 100644--- a/plugins/discourse-ai/spec/lib/rag/web_page_fetcher_spec.rb+++ b/plugins/discourse-ai/spec/lib/rag/web_page_fetcher_spec.rb@@ -134,6 +134,40 @@ RSpec.describe DiscourseAi::Rag::WebPageFetcher do).to have_been_made.onceend+ %w[text/plain text/markdown].each do |content_type|+ it "decodes binary #{content_type} responses as UTF-8 and replaces invalid bytes" do+ url = "https://example.com/docs"+ stub_request(:get, url).to_return(+ status: 200,+ headers: {+ "Content-Type" => "#{content_type}; charset=utf-8",+ },+ body: "Knowledge — café\nTruncated: \xE2".b,+ )++ result = described_class.fetch(url: url)++ expect(result[:text].encoding).to eq(Encoding::UTF_8)+ expect(result[:text]).to eq("Knowledge — café\nTruncated: �")+ end+ end++ it "preserves UTF-8 characters split across binary response chunks" do+ url = "https://example.com/docs"+ response = Net::HTTPOK.new("1.1", "200", "OK")+ response["Content-Type"] = "text/markdown"+ response.stubs(:read_body).multiple_yields(["Knowledge \xE2".b], ["\x80\x94 café".b])+ http = stub+ http.stubs(:read_timeout=)+ http.stubs(:request).yields(response)+ FinalDestination::HTTP.stubs(:start).yields(http)++ result = described_class.fetch(url: url)++ expect(result[:text].encoding).to eq(Encoding::UTF_8)+ expect(result[:text]).to eq("Knowledge — café")+ end+it "follows redirects without an additional discovery request" dooriginal_url = "https://example.com/docs"redirected_url = "https://docs.example.com/guide"