Skip to content

A platform for community discussion. Free, open, simple.

DEV: Improve handling of RAG web fetcher (#42788)

Penar Musaraj8e14b6246624
diff --git a/plugins/discourse-ai/app/jobs/regular/digest_rag_upload.rb b/plugins/discourse-ai/app/jobs/regular/digest_rag_upload.rb
index 09570aefaab..9c8e053cc6d 100644
--- a/plugins/discourse-ai/app/jobs/regular/digest_rag_upload.rb
+++ b/plugins/discourse-ai/app/jobs/regular/digest_rag_upload.rb
@@ -38,6 +38,8 @@ module Jobs
chunk_tokens:,
overlap_tokens:,
) do |chunk, metadata|
+ next if chunk.blank?
+
fragment_ids << RagDocumentFragment.create!(
target:,
fragment: chunk,
diff --git a/plugins/discourse-ai/lib/rag/web_page_fetcher.rb b/plugins/discourse-ai/lib/rag/web_page_fetcher.rb
index 2b96b82866c..451afca8fee 100644
--- a/plugins/discourse-ai/lib/rag/web_page_fetcher.rb
+++ b/plugins/discourse-ai/lib/rag/web_page_fetcher.rb
@@ -5,7 +5,8 @@ module DiscourseAi
class WebPageFetcher
MAX_RESPONSE_BODY_LENGTH = 5.megabytes
MAX_REDIRECTS = 5
- SUPPORTED_CONTENT_TYPES = %w[text/html application/xhtml+xml text/plain].freeze
+ ACCEPT_HEADER = "text/markdown, */*"
+ SUPPORTED_CONTENT_TYPES = %w[text/html application/xhtml+xml text/plain text/markdown].freeze
FetchError = Class.new(StandardError)
@@ -44,7 +45,7 @@ module DiscourseAi
private
def request_headers
- {}.tap do |headers|
+ { "Accept" => ACCEPT_HEADER }.tap do |headers|
headers["If-None-Match"] = @etag if @etag.present?
headers["If-Modified-Since"] = @last_modified if @last_modified.present?
end
@@ -115,7 +116,7 @@ module DiscourseAi
end
body = response[:body]
- text = content_type == "text/plain" ? body : extract_html(body)
+ text = content_type.in?(%w[text/plain text/markdown]) ? body : extract_html(body)
raise FetchError, "The page did not contain readable text" if text.blank?
{
@@ -136,7 +137,62 @@ module DiscourseAi
document.at("#main") || document.at(".main") || document.at("#content") ||
document.at(".content") || document.at("body")
- main_content&.xpath(".//text()")&.map(&:text)&.join(" ").to_s.gsub(/\s+/, " ").strip
+ text_content(main_content)
+ end
+
+ def text_content(node)
+ return "" if node.blank?
+
+ segments =
+ node
+ .xpath(".//text()[not(ancestor::table)] | .//table[not(ancestor::table)]")
+ .filter_map do |child|
+ if child.name == "table"
+ table_text = table_to_markdown(child) || normalize_whitespace(child.text)
+ "\n\n#{table_text}\n\n" if table_text.present?
+ else
+ normalize_whitespace(child.text) if child.text.present?
+ end
+ end
+
+ segments.join(" ").gsub(/ *\n */, "\n").gsub(/\n{3,}/, "\n\n").strip
+ end
+
+ def table_to_markdown(table)
+ rows = table.xpath("./tr | ./thead/tr | ./tbody/tr | ./tfoot/tr")
+ cells_by_row = rows.map { |row| row.xpath("./th | ./td") }
+ column_count = cells_by_row.map(&:length).max.to_i
+ return if column_count < 2
+
+ cells_by_row.select! do |cells|
+ cells.length == column_count && cells.all? { |cell| cell_span(cell) == [1, 1] }
+ end
+ return if cells_by_row.length < 2
+
+ lines =
+ cells_by_row.map do |cells|
+ "| #{cells.map { |cell| table_cell_text(cell) }.join(" | ")} |"
+ end
+ lines.insert(1, "| #{Array.new(column_count, "---").join(" | ")} |")
+ lines.join("\n")
+ end
+
+ def table_cell_text(cell)
+ text = cell.xpath(".//text()[not(ancestor::table[ancestor::table])]").map(&:text)
+ labels =
+ cell
+ .xpath(".//*[@aria-label] | self::*[@aria-label]")
+ .filter_map { |element| element["aria-label"] if element.text.blank? }
+
+ normalize_whitespace((text + labels.uniq).join(" ")).gsub("|", "\\|")
+ end
+
+ def cell_span(cell)
+ [cell["rowspan"].presence&.to_i || 1, cell["colspan"].presence&.to_i || 1]
+ end
+
+ def normalize_whitespace(text)
+ text.to_s.gsub(/\s+/, " ").strip
end
end
end
diff --git a/plugins/discourse-ai/spec/jobs/regular/digest_rag_upload_spec.rb b/plugins/discourse-ai/spec/jobs/regular/digest_rag_upload_spec.rb
index facbef7ce8d..005cef5191a 100644
--- a/plugins/discourse-ai/spec/jobs/regular/digest_rag_upload_spec.rb
+++ b/plugins/discourse-ai/spec/jobs/regular/digest_rag_upload_spec.rb
@@ -176,6 +176,22 @@ describe Jobs::DigestRagUpload do
expect(indexed_content).to eq("before 猫 after 犬")
end
end
+
+ context "when the document produces a blank trailing chunk" do
+ let(:document_file) do
+ StringIO.new(
+ "[[metadata {\"source_url\":\"https://example.com\"}]]\n#{"some text " * 200}\n",
+ )
+ end
+
+ it "only creates fragments containing text" do
+ job.execute(upload_id: upload.id, target_id: agent.id, target_type: agent.class.to_s)
+
+ fragments = RagDocumentFragment.where(upload:, target: agent).pluck(:fragment)
+
+ expect(fragments).to all(be_present)
+ end
+ end
end
it "doesn't generate new fragments if we already processed the upload" do
diff --git a/plugins/discourse-ai/spec/lib/rag/web_page_fetcher_spec.rb b/plugins/discourse-ai/spec/lib/rag/web_page_fetcher_spec.rb
index 666e66f79cb..cdde62a7506 100644
--- a/plugins/discourse-ai/spec/lib/rag/web_page_fetcher_spec.rb
+++ b/plugins/discourse-ai/spec/lib/rag/web_page_fetcher_spec.rb
@@ -5,6 +5,10 @@ RSpec.describe DiscourseAi::Rag::WebPageFetcher do
it "extracts readable page content and response validators" do
url = "https://example.com/docs"
+ navigation = "Home Documentation Pricing"
+ article_heading = "Installation"
+ article_paragraph = "Install the package."
+ footer = "Privacy Terms Copyright"
stub_request(:get, url).to_return(
status: 200,
headers: {
@@ -15,8 +19,12 @@ RSpec.describe DiscourseAi::Rag::WebPageFetcher do
body: <<~HTML,
<html>
<body>
- <nav>Navigation</nav>
- <main><h1>Installation</h1><p>Install the package.</p></main>
+ <nav>#{navigation}</nav>
+ <main>
+ <h1>#{article_heading}</h1>
+ <p>#{article_paragraph}</p>
+ </main>
+ <footer>#{footer}</footer>
<script>ignoreMe()</script>
</body>
</html>
@@ -28,12 +36,56 @@ RSpec.describe DiscourseAi::Rag::WebPageFetcher do
expect(result).to include(
not_modified: false,
url: url,
- text: "Installation Install the package.",
+ text: "#{article_heading} #{article_paragraph}",
etag: '"revision-2"',
last_modified: "Wed, 19 Aug 2026 16:00:00 GMT",
)
end
+ it "converts rectangular HTML table rows to Markdown" do
+ url = "https://example.com/pricing"
+ stub_request(:get, url).to_return(
+ status: 200,
+ headers: {
+ "Content-Type" => "text/html",
+ },
+ body: <<~HTML,
+ <main>
+ <h1>Compare plans &amp; features</h1>
+ <table>
+ <thead>
+ <tr><th colspan="5">Community</th></tr>
+ <tr><th>Feature</th><th>Free</th><th>Pro</th><th>Business</th><th>Enterprise</th></tr>
+ </thead>
+ <tbody>
+ <tr><td></td><td colspan="4">Staff seats</td></tr>
+ <tr><td>Staff seats</td><td>2</td><td>5</td><td>15</td><td>Unlimited</td></tr>
+ <tr><td></td><td colspan="4">Custom groups</td></tr>
+ <tr>
+ <td>Custom groups</td>
+ <td></td>
+ <td><i aria-label="Included"></i></td>
+ <td><i aria-label="Included"></i></td>
+ <td><i aria-label="Included"></i></td>
+ </tr>
+ </tbody>
+ </table>
+ </main>
+ HTML
+ )
+
+ result = described_class.fetch(url: url)
+
+ expect(result[:text]).to eq(<<~MARKDOWN.strip)
+ Compare plans & features
+
+ | Feature | Free | Pro | Business | Enterprise |
+ | --- | --- | --- | --- | --- |
+ | Staff seats | 2 | 5 | 15 | Unlimited |
+ | Custom groups | | Included | Included | Included |
+ MARKDOWN
+ end
+
it "uses conditional request headers and handles an unchanged page" do
url = "https://example.com/docs"
stub_request(:get, url).to_return(status: 304)
@@ -56,6 +108,32 @@ RSpec.describe DiscourseAi::Rag::WebPageFetcher do
).to have_been_made.once
end
+ it "requests and preserves Markdown content" do
+ url = "https://example.com/docs.md"
+ markdown = <<~MARKDOWN
+ # Installation
+
+ Run `bundle install`, then:
+
+ - Configure the service
+ - Restart the process
+ MARKDOWN
+ stub_request(:get, url).to_return(
+ status: 200,
+ headers: {
+ "Content-Type" => "text/markdown; charset=utf-8",
+ },
+ body: markdown,
+ )
+
+ result = described_class.fetch(url: url)
+
+ expect(result).to include(not_modified: false, url: url, text: markdown)
+ expect(
+ a_request(:get, url).with(headers: { "Accept" => "text/markdown, */*" }),
+ ).to have_been_made.once
+ end
+
it "follows redirects without an additional discovery request" do
original_url = "https://example.com/docs"
redirected_url = "https://docs.example.com/guide"