RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix(data_source): scope table and link state to ancestors in format_document_soup (#17045)
Aaria0e091e75051
diff --git a/common/data_source/html_utils.py b/common/data_source/html_utils.pyindex b39569e664..9fd33a7b0e 100644--- a/common/data_source/html_utils.py+++ b/common/data_source/html_utils.py@@ -76,12 +76,24 @@ def format_document_soup(document: bs4.BeautifulSoup, table_cell_separator: strtext = ""list_element_start = Falseverbatim_output = 0- in_table = Falselast_added_newline = False- link_href: str | None = None++ # ``descendants`` yields opening tags only, so a flag set on <table>/<a> would+ # never clear. Precompute scope by registering each <table>/<a> descendant;+ # this avoids O(depth) ancestor walks while keeping lookups O(1).+ table_scope = {id(d) for table in document.find_all("table") for d in table.descendants}+ href_scope = {}+ for anchor in document.find_all("a"): # document order, so a nested <a> wins over its parent+ href_value = anchor.get("href", None)+ # mostly for typing, having multiple hrefs is not valid HTML+ link_href = href_value[0] if isinstance(href_value, list) else href_value+ href_scope.update((id(d), link_href) for d in anchor.descendants)for e in document.descendants:verbatim_output -= 1+ in_table = id(e) in table_scope+ link_href = href_scope.get(id(e))+if isinstance(e, bs4.element.NavigableString):if isinstance(e, (bs4.element.Comment, bs4.element.Doctype)):continue@@ -109,26 +121,15 @@ def format_document_soup(document: bs4.BeautifulSoup, table_cell_separator: strlist_element_start = Falseelif isinstance(e, bs4.element.Tag):- # table is standard HTML element- if e.name == "table":- in_table = True# TR is for rows- elif e.name == "tr" and in_table:+ if e.name == "tr" and in_table:text += "\n"# td for data cell, th for headerelif e.name in ["td", "th"] and in_table:text += table_cell_separator- elif e.name == "/table":- in_table = Falseelif in_table:# don't handle other cases while in tablepass- elif e.name == "a":- href_value = e.get("href", None)- # mostly for typing, having multiple hrefs is not valid HTML- link_href = href_value[0] if isinstance(href_value, list) else href_value- elif e.name == "/a":- link_href = Noneelif e.name in ["p", "div"]:if not list_element_start:text += "\n"diff --git a/test/unit_test/data_source/test_html_utils.py b/test/unit_test/data_source/test_html_utils.pynew file mode 100644index 0000000000..242ae90d3b--- /dev/null+++ b/test/unit_test/data_source/test_html_utils.py@@ -0,0 +1,91 @@+#+# Copyright 2025 The InfiniFlow Authors. All Rights Reserved.+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+#+import bs4+import pytest++from common.data_source import html_utils+from common.data_source.config import HtmlBasedConnectorTransformLinksStrategy+from common.data_source.html_utils import format_document_soup+++def _fmt(html: str) -> str:+ return format_document_soup(bs4.BeautifulSoup(html, "html.parser"))+++def markdown_links(monkeypatch):+ """``format_element_text`` only renders links under the markdown strategy."""+ monkeypatch.setattr(+ html_utils,+ "HTML_BASED_CONNECTOR_TRANSFORM_LINKS_STRATEGY",+ HtmlBasedConnectorTransformLinksStrategy.MARKDOWN,+ )+++TABLE = "<table><tr><td>A</td><td>B</td></tr></table>"+++def test_paragraph_after_table_keeps_its_newline():+ assert "\nAfter" in _fmt(f"<p>Before</p>{TABLE}<p>After</p>")+++def test_list_after_table_keeps_hyphen_markers():+ assert "\n- item1" in _fmt(f"<p>Before</p>{TABLE}<ul><li>item1</li><li>item2</li></ul>")+++def test_block_after_table_starts_on_a_new_line():+ # The div must not be folded onto the last table row.+ assert "\nTrailing" in _fmt(f"{TABLE}<div>Trailing</div>")+++def test_table_still_separates_rows_and_cells():+ # Control: the table itself must keep working — rows on newlines, cells tab-separated.+ assert _fmt("<table><tr><td>A</td><td>B</td></tr><tr><td>C</td><td>D</td></tr></table>") == "A\tB\n\tC\tD"+++def test_content_after_table_matches_the_same_content_without_a_table(markdown_links):+ tail = "<p>After</p><ul><li>item1</li></ul>"+ with_table = _fmt(f"{TABLE}{tail}")+ without_table = _fmt(tail)+ assert with_table.endswith(without_table.lstrip("\n"))+++def test_text_after_link_in_same_paragraph_is_not_linkified(markdown_links):+ assert _fmt('<p>see <a href="http://x.com">link</a> after</p>') == "see [link](http://x.com) after"+++def test_paragraph_after_link_is_not_linkified(markdown_links):+ out = _fmt('<p><a href="http://x.com">link</a></p><p>next paragraph</p>')+ assert "[next paragraph]" not in out+++def test_link_inside_anchor_is_still_linkified(markdown_links):+ # The anchor text itself must keep its markdown link.+ assert _fmt('<p><a href="http://x.com">link</a></p>') == "[link](http://x.com)"+++def test_table_cells_do_not_inherit_a_preceding_links_href(markdown_links):+ out = _fmt(f'<p><a href="http://x.com">pre</a></p>{TABLE}')+ assert "[A](http://x.com)" not in out+++def test_link_inside_a_table_cell_is_linkified(markdown_links):+ assert _fmt('<table><tr><td><a href="http://x.com">cell</a></td></tr></table>') == "[cell](http://x.com)"+++def test_link_is_stripped_under_the_default_strategy():+ # Default strategy is STRIP: no markdown link syntax at all.+ assert _fmt('<p>see <a href="http://x.com">link</a> after</p>') == "see link after"