A platform for community discussion. Free, open, simple.
DEV: Improve handling of RAG web fetcher (#42788)
Penar Musaraj8e14b6246624
diff --git a/plugins/discourse-ai/app/jobs/regular/digest_rag_upload.rb b/plugins/discourse-ai/app/jobs/regular/digest_rag_upload.rbindex 09570aefaab..9c8e053cc6d 100644--- a/plugins/discourse-ai/app/jobs/regular/digest_rag_upload.rb+++ b/plugins/discourse-ai/app/jobs/regular/digest_rag_upload.rb@@ -38,6 +38,8 @@ module Jobschunk_tokens:,overlap_tokens:,) do |chunk, metadata|+ next if chunk.blank?+fragment_ids << RagDocumentFragment.create!(target:,fragment: chunk,diff --git a/plugins/discourse-ai/lib/rag/web_page_fetcher.rb b/plugins/discourse-ai/lib/rag/web_page_fetcher.rbindex 2b96b82866c..451afca8fee 100644--- a/plugins/discourse-ai/lib/rag/web_page_fetcher.rb+++ b/plugins/discourse-ai/lib/rag/web_page_fetcher.rb@@ -5,7 +5,8 @@ module DiscourseAiclass WebPageFetcherMAX_RESPONSE_BODY_LENGTH = 5.megabytesMAX_REDIRECTS = 5- SUPPORTED_CONTENT_TYPES = %w[text/html application/xhtml+xml text/plain].freeze+ ACCEPT_HEADER = "text/markdown, */*"+ SUPPORTED_CONTENT_TYPES = %w[text/html application/xhtml+xml text/plain text/markdown].freezeFetchError = Class.new(StandardError)@@ -44,7 +45,7 @@ module DiscourseAiprivatedef request_headers- {}.tap do |headers|+ { "Accept" => ACCEPT_HEADER }.tap do |headers|headers["If-None-Match"] = @etag if @etag.present?headers["If-Modified-Since"] = @last_modified if @last_modified.present?end@@ -115,7 +116,7 @@ module DiscourseAiendbody = response[:body]- text = content_type == "text/plain" ? body : extract_html(body)+ text = content_type.in?(%w[text/plain text/markdown]) ? body : extract_html(body)raise FetchError, "The page did not contain readable text" if text.blank?{@@ -136,7 +137,62 @@ module DiscourseAidocument.at("#main") || document.at(".main") || document.at("#content") ||document.at(".content") || document.at("body")- main_content&.xpath(".//text()")&.map(&:text)&.join(" ").to_s.gsub(/\s+/, " ").strip+ text_content(main_content)+ end++ def text_content(node)+ return "" if node.blank?++ segments =+ node+ .xpath(".//text()[not(ancestor::table)] | .//table[not(ancestor::table)]")+ .filter_map do |child|+ if child.name == "table"+ table_text = table_to_markdown(child) || normalize_whitespace(child.text)+ "\n\n#{table_text}\n\n" if table_text.present?+ else+ normalize_whitespace(child.text) if child.text.present?+ end+ end++ segments.join(" ").gsub(/ *\n */, "\n").gsub(/\n{3,}/, "\n\n").strip+ end++ def table_to_markdown(table)+ rows = table.xpath("./tr | ./thead/tr | ./tbody/tr | ./tfoot/tr")+ cells_by_row = rows.map { |row| row.xpath("./th | ./td") }+ column_count = cells_by_row.map(&:length).max.to_i+ return if column_count < 2++ cells_by_row.select! do |cells|+ cells.length == column_count && cells.all? { |cell| cell_span(cell) == [1, 1] }+ end+ return if cells_by_row.length < 2++ lines =+ cells_by_row.map do |cells|+ "| #{cells.map { |cell| table_cell_text(cell) }.join(" | ")} |"+ end+ lines.insert(1, "| #{Array.new(column_count, "---").join(" | ")} |")+ lines.join("\n")+ end++ def table_cell_text(cell)+ text = cell.xpath(".//text()[not(ancestor::table[ancestor::table])]").map(&:text)+ labels =+ cell+ .xpath(".//*[@aria-label] | self::*[@aria-label]")+ .filter_map { |element| element["aria-label"] if element.text.blank? }++ normalize_whitespace((text + labels.uniq).join(" ")).gsub("|", "\\|")+ end++ def cell_span(cell)+ [cell["rowspan"].presence&.to_i || 1, cell["colspan"].presence&.to_i || 1]+ end++ def normalize_whitespace(text)+ text.to_s.gsub(/\s+/, " ").stripendendenddiff --git a/plugins/discourse-ai/spec/jobs/regular/digest_rag_upload_spec.rb b/plugins/discourse-ai/spec/jobs/regular/digest_rag_upload_spec.rbindex facbef7ce8d..005cef5191a 100644--- a/plugins/discourse-ai/spec/jobs/regular/digest_rag_upload_spec.rb+++ b/plugins/discourse-ai/spec/jobs/regular/digest_rag_upload_spec.rb@@ -176,6 +176,22 @@ describe Jobs::DigestRagUpload doexpect(indexed_content).to eq("before 猫 after 犬")endend++ context "when the document produces a blank trailing chunk" do+ let(:document_file) do+ StringIO.new(+ "[[metadata {\"source_url\":\"https://example.com\"}]]\n#{"some text " * 200}\n",+ )+ end++ it "only creates fragments containing text" do+ job.execute(upload_id: upload.id, target_id: agent.id, target_type: agent.class.to_s)++ fragments = RagDocumentFragment.where(upload:, target: agent).pluck(:fragment)++ expect(fragments).to all(be_present)+ end+ endendit "doesn't generate new fragments if we already processed the upload" dodiff --git a/plugins/discourse-ai/spec/lib/rag/web_page_fetcher_spec.rb b/plugins/discourse-ai/spec/lib/rag/web_page_fetcher_spec.rbindex 666e66f79cb..cdde62a7506 100644--- a/plugins/discourse-ai/spec/lib/rag/web_page_fetcher_spec.rb+++ b/plugins/discourse-ai/spec/lib/rag/web_page_fetcher_spec.rb@@ -5,6 +5,10 @@ RSpec.describe DiscourseAi::Rag::WebPageFetcher doit "extracts readable page content and response validators" dourl = "https://example.com/docs"+ navigation = "Home Documentation Pricing"+ article_heading = "Installation"+ article_paragraph = "Install the package."+ footer = "Privacy Terms Copyright"stub_request(:get, url).to_return(status: 200,headers: {@@ -15,8 +19,12 @@ RSpec.describe DiscourseAi::Rag::WebPageFetcher dobody: <<~HTML,<html><body>- <nav>Navigation</nav>- <main><h1>Installation</h1><p>Install the package.</p></main>+ <nav>#{navigation}</nav>+ <main>+ <h1>#{article_heading}</h1>+ <p>#{article_paragraph}</p>+ </main>+ <footer>#{footer}</footer><script>ignoreMe()</script></body></html>@@ -28,12 +36,56 @@ RSpec.describe DiscourseAi::Rag::WebPageFetcher doexpect(result).to include(not_modified: false,url: url,- text: "Installation Install the package.",+ text: "#{article_heading} #{article_paragraph}",etag: '"revision-2"',last_modified: "Wed, 19 Aug 2026 16:00:00 GMT",)end+ it "converts rectangular HTML table rows to Markdown" do+ url = "https://example.com/pricing"+ stub_request(:get, url).to_return(+ status: 200,+ headers: {+ "Content-Type" => "text/html",+ },+ body: <<~HTML,+ <main>+ <h1>Compare plans & features</h1>+ <table>+ <thead>+ <tr><th colspan="5">Community</th></tr>+ <tr><th>Feature</th><th>Free</th><th>Pro</th><th>Business</th><th>Enterprise</th></tr>+ </thead>+ <tbody>+ <tr><td></td><td colspan="4">Staff seats</td></tr>+ <tr><td>Staff seats</td><td>2</td><td>5</td><td>15</td><td>Unlimited</td></tr>+ <tr><td></td><td colspan="4">Custom groups</td></tr>+ <tr>+ <td>Custom groups</td>+ <td></td>+ <td><i aria-label="Included"></i></td>+ <td><i aria-label="Included"></i></td>+ <td><i aria-label="Included"></i></td>+ </tr>+ </tbody>+ </table>+ </main>+ HTML+ )++ result = described_class.fetch(url: url)++ expect(result[:text]).to eq(<<~MARKDOWN.strip)+ Compare plans & features++ | Feature | Free | Pro | Business | Enterprise |+ | --- | --- | --- | --- | --- |+ | Staff seats | 2 | 5 | 15 | Unlimited |+ | Custom groups | | Included | Included | Included |+ MARKDOWN+ end+it "uses conditional request headers and handles an unchanged page" dourl = "https://example.com/docs"stub_request(:get, url).to_return(status: 304)@@ -56,6 +108,32 @@ RSpec.describe DiscourseAi::Rag::WebPageFetcher do).to have_been_made.onceend+ it "requests and preserves Markdown content" do+ url = "https://example.com/docs.md"+ markdown = <<~MARKDOWN+ # Installation++ Run `bundle install`, then:++ - Configure the service+ - Restart the process+ MARKDOWN+ stub_request(:get, url).to_return(+ status: 200,+ headers: {+ "Content-Type" => "text/markdown; charset=utf-8",+ },+ body: markdown,+ )++ result = described_class.fetch(url: url)++ expect(result).to include(not_modified: false, url: url, text: markdown)+ expect(+ a_request(:get, url).with(headers: { "Accept" => "text/markdown, */*" }),+ ).to have_been_made.once+ end+it "follows redirects without an additional discovery request" dooriginal_url = "https://example.com/docs"redirected_url = "https://docs.example.com/guide"