RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix(agent): preserve whitespace around message references (#18666)
primorLeea11311678099
diff --git a/agent/canvas.py b/agent/canvas.pyindex 14b5743b9c..0f390e6bd2 100644--- a/agent/canvas.py+++ b/agent/canvas.py@@ -215,30 +215,19 @@ class Graph:# Reference the canonical pre-compiled regex from ComponentBase so# the source-pattern and the runtime-pattern can never drift apart.pat = ComponentBase.variable_ref_patt_re- out_parts = []- last = 0- for m in pat.finditer(value):- out_parts.append(value[last : m.start()])+ def replace(m):key = m.group(1)v = self.get_variable_value(key)if v is None:- rep = ""+ return ""elif isinstance(v, partial):- buf = []- for chunk in v():- buf.append(chunk)- rep = "".join(buf)+ return "".join(v())elif isinstance(v, str):- rep = v- else:- rep = json.dumps(v, ensure_ascii=False)-- out_parts.append(rep)- last = m.end()+ return v+ return json.dumps(v, ensure_ascii=False)- out_parts.append(value[last:])- return "".join(out_parts)+ return ComponentBase._replace_template_matches(pat, value, replace)def get_variable_value(self, exp: str) -> Any:exp = exp.strip("{").strip("}").strip(" ").strip("{").strip("}")diff --git a/agent/component/base.py b/agent/component/base.pyindex 6139ef14e9..5c49b50481 100644--- a/agent/component/base.py+++ b/agent/component/base.py@@ -40,6 +40,19 @@ _USER_FEEDED_PARAMS = "_user_feeded_params"_IS_RAW_CONF = "_is_raw_conf"+def _build_template_ref_pattern(reference_pattern: str) -> str:+ """Match a plain reference or one balanced outer brace pair.++ Group 1 remains the reference for existing callers. The named ``outer``+ group records whether the match used the wrapped form.+ """+ return (+ rf"(?=(?:\{{ *)?\{{({reference_pattern})\}})"+ rf"(?:(?P<outer>\{{) *)?\{{(?:{reference_pattern})\}}"+ r"(?(outer) *\}|(?! *\}))"+ )++class ComponentParamBase(ABC):def __init__(self):self.message_history_window_size = 13@@ -360,9 +373,9 @@ class ComponentBase(ABC):# `cpn_id` allows underscores (frontend ids like `userfillup_abc`,# `retrieval_xyz`) and colons (legacy DSL ids like `UserFillUp:CateInput`,# `Retrieval:KBSearch`).- variable_ref_patt = r"\{* *\{([a-zA-Z0-9_:]+@[A-Za-z0-9_.-]+|sys\.[A-Za-z0-9_.]+|env\.[A-Za-z0-9_.]+)\} *\}*"+ variable_ref_patt = _build_template_ref_pattern(r"[a-zA-Z0-9_:]+@[A-Za-z0-9_.-]+|sys\.[A-Za-z0-9_.]+|env\.[A-Za-z0-9_.]+")variable_ref_patt_re = re.compile(variable_ref_patt, flags=re.IGNORECASE | re.DOTALL)- iteration_alias_patt = r"\{* *\{(item|index|result)\} *\}*"+ iteration_alias_patt = _build_template_ref_pattern(r"item|index|result")iteration_alias_patt_re = re.compile(iteration_alias_patt, flags=re.IGNORECASE | re.DOTALL)def __str__(self):@@ -528,7 +541,7 @@ class ComponentBase(ABC):def get_input_elements_from_text(self, txt: str) -> dict[str, dict[str, str]]:res = {}- for r in self.variable_ref_patt_re.finditer(txt):+ for r in self._iter_template_matches(self.variable_ref_patt_re, txt):exp = r.group(1)# Use maxsplit=1 to be defensive: although `exp` here comes# from `variable_ref_patt` (which constrains `var_nm` to@@ -542,7 +555,7 @@ class ComponentBase(ABC):"_retrieval": self._canvas.get_variable_value(f"{cpn_id}@_references") if cpn_id else None,"_cpn_id": cpn_id,}- for r in self.iteration_alias_patt_re.finditer(txt):+ for r in self._iter_template_matches(self.iteration_alias_patt_re, txt):exp = r.group(1)if exp in res:continue@@ -604,13 +617,48 @@ class ComponentBase(ABC):return cpn_nms@staticmethod- def string_format(content: str, kv: dict[str, str]) -> str:- for n, v in kv.items():+ def _is_complete_template_match(content: str, match: re.Match) -> bool:+ if match.groupdict().get("outer") is not None:+ return True++ # A failed wrapped match can otherwise be rediscovered as its inner+ # plain reference (for example, ``{ {A@x}``). Keep such malformed+ # input literal instead of consuming only half of the brace pair.+ before = content[: match.start()].rstrip(" ")+ after = content[match.end() :].lstrip(" ")+ return not before.endswith("{") and not after.startswith("}")++ @classmethod+ def _iter_template_matches(cls, pattern: re.Pattern, content: str):+ for match in pattern.finditer(content):+ if cls._is_complete_template_match(content, match):+ yield match+ else:+ _logger.debug("Ignored incomplete template reference candidate at offset %d", match.start())++ @classmethod+ def _replace_template_matches(cls, pattern: re.Pattern, content: str, replacement) -> str:+ out = []+ last = 0+ for match in cls._iter_template_matches(pattern, content):+ out.append(content[last : match.start()])+ out.append(replacement(match))+ last = match.end()+ out.append(content[last:])+ return "".join(out)++ @classmethod+ def string_format(cls, content: str, kv: dict[str, str]) -> str:+ for pattern in (cls.variable_ref_patt_re, cls.iteration_alias_patt_re):- def repl(_match, val=v):- return str(val) if val is not None else ""+ def replace(match):+ key = match.group(1)+ if key in kv:+ value = kv[key]+ return str(value) if value is not None else ""+ return match.group(0)- content = re.sub(r"\{%s\}" % re.escape(n), repl, content)+ content = cls._replace_template_matches(pattern, content, replace)return contentdef exception_handler(self):diff --git a/agent/component/docs_generator.py b/agent/component/docs_generator.pyindex 9c89eefbf2..e0ef6acd78 100644--- a/agent/component/docs_generator.py+++ b/agent/component/docs_generator.py@@ -182,11 +182,10 @@ class DocGenerator(Message, ABC):logging.warning("Error resolving variable %s: %s", match, str(e))return f"[ERROR: {str(e)}]"- content = re.sub(- self.variable_ref_patt,- _replace_variable,+ content = self._replace_template_matches(+ re.compile(self.variable_ref_patt, flags=re.DOTALL),content,- flags=re.DOTALL,+ _replace_variable,)return self._strip_thinking(content)diff --git a/agent/component/invoke.py b/agent/component/invoke.pyindex a6cf36e678..4ace2197b4 100644--- a/agent/component/invoke.py+++ b/agent/component/invoke.py@@ -122,7 +122,7 @@ class Invoke(ComponentBase, ABC):self.set_input_value(variable_name, value)return "" if value is None else value- def _render_template(self, content: str, pattern: str, kwargs: dict | None = None, *, flags: int = 0) -> str:+ def _render_template(self, content: str, pattern: str, kwargs: dict | None = None, *, flags: int = 0, complete_matches: bool = False) -> str:content = content or ""if not content:return content@@ -130,10 +130,13 @@ class Invoke(ComponentBase, ABC):def replace_variable(match_obj):return str(self._resolve_variable_value(match_obj.group(1), kwargs))- return re.sub(pattern, replace_variable, content, flags=flags)+ compiled_pattern = re.compile(pattern, flags=flags)+ if complete_matches:+ return self._replace_template_matches(compiled_pattern, content, replace_variable)+ return compiled_pattern.sub(replace_variable, content)def _resolve_template_text(self, content: str, kwargs: dict | None = None) -> str:- return self._render_template(content, self.variable_ref_patt, kwargs, flags=re.DOTALL)+ return self._render_template(content, self.variable_ref_patt, kwargs, flags=re.DOTALL, complete_matches=True)def _resolve_header_text(self, content: str, kwargs: dict | None = None) -> str:# Headers support plain {token} placeholders, so they cannot reuse the canvas variable regex.diff --git a/agent/component/message.py b/agent/component/message.pyindex d9077313de..4108772f6a 100644--- a/agent/component/message.py+++ b/agent/component/message.py@@ -190,7 +190,8 @@ class Message(ComponentBase):all_content = ""cache = {}downloads = []- for r in re.finditer(self.variable_ref_patt, rand_cnt, flags=re.DOTALL):+ pattern = re.compile(self.variable_ref_patt, flags=re.DOTALL)+ for r in self._iter_template_matches(pattern, rand_cnt):if self.check_if_canceled("Message streaming"):returndiff --git a/agent/tools/retrieval.py b/agent/tools/retrieval.pyindex bee162f3fb..5c98ebef78 100644--- a/agent/tools/retrieval.py+++ b/agent/tools/retrieval.py@@ -88,6 +88,27 @@ class Retrieval(ToolBase, ABC):"""Get dataset IDs with backward compatibility for kb_ids."""return self._param.dataset_ids or getattr(self._param, "kb_ids", None) or []+ def _resolve_manual_filter(self, flt: dict) -> dict:+ # Return a new dict instead of mutating `flt` in place. The caller+ # passes filters straight out of self._param.meta_data_filter, so+ # mutating them would make later invocations reuse a stale value.+ pat = re.compile(self.variable_ref_patt)+ content = flt.get("value", "")++ def replace(match):+ value = self._canvas.get_variable_value(match.group(1))+ if value is None:+ return ""+ elif isinstance(value, partial):+ return "".join(value())+ elif isinstance(value, str):+ return value+ return json.dumps(value, ensure_ascii=False)++ resolved = dict(flt)+ resolved["value"] = self._replace_template_matches(pat, content, replace)+ return resolved+async def _retrieve_kb(self, query_text: str):kb_ids: list[str] = []for id in self._dataset_ids:@@ -137,41 +158,6 @@ class Retrieval(ToolBase, ABC):def _load_metas() -> dict:return DocMetadataService.get_flatted_meta_by_kbs(kb_ids)- def _resolve_manual_filter(flt: dict) -> dict:- # Return a new dict instead of mutating `flt` in place. The- # caller passes filters straight out of self._param.meta_data_filter,- # so mutating them would replace the variable reference with its- # resolved value and every subsequent invocation (e.g. inside an- # Iteration component) would reuse that stale value.- pat = re.compile(self.variable_ref_patt)- s = flt.get("value", "")- out_parts = []- last = 0-- for m in pat.finditer(s):- out_parts.append(s[last : m.start()])- key = m.group(1)- v = self._canvas.get_variable_value(key)- if v is None:- rep = ""- elif isinstance(v, partial):- buf = []- for chunk in v():- buf.append(chunk)- rep = "".join(buf)- elif isinstance(v, str):- rep = v- else:- rep = json.dumps(v, ensure_ascii=False)-- out_parts.append(rep)- last = m.end()-- out_parts.append(s[last:])- resolved = dict(flt)- resolved["value"] = "".join(out_parts)- return resolved-chat_mdl = Noneif self._param.meta_data_filter.get("method") in ["auto", "semi_auto"]:tenant_id = self._canvas.get_tenant_id()@@ -184,7 +170,7 @@ class Retrieval(ToolBase, ABC):query,chat_mdl,doc_ids,- _resolve_manual_filter if self._param.meta_data_filter.get("method") == "manual" else None,+ self._resolve_manual_filter if self._param.meta_data_filter.get("method") == "manual" else None,kb_ids=kb_ids,metas_loader=_load_metas,)diff --git a/test/testcases/test_web_api/test_canvas_app/test_variable_ref_pattern_unit.py b/test/testcases/test_web_api/test_canvas_app/test_variable_ref_pattern_unit.pyindex 966614655c..3927410809 100644--- a/test/testcases/test_web_api/test_canvas_app/test_variable_ref_pattern_unit.py+++ b/test/testcases/test_web_api/test_canvas_app/test_variable_ref_pattern_unit.py@@ -58,6 +58,7 @@ These tests pin four contracts:"""import importlib.util+import loggingimport reimport sysfrom pathlib import Path@@ -85,9 +86,84 @@ def base_module(monkeypatch):fake_pandas.DataFrame = type("DataFrame", (), {})monkeypatch.setitem(sys.modules, "pandas", fake_pandas)- spec = importlib.util.spec_from_file_location(- "_base_for_regex_test", repo_root / "agent" / "component" / "base.py"+ fake_connection_utils = ModuleType("common.connection_utils")+ fake_connection_utils.timeout = lambda _seconds: lambda function: function+ monkeypatch.setitem(sys.modules, "common.connection_utils", fake_connection_utils)++ fake_misc_utils = ModuleType("common.misc_utils")+ fake_misc_utils.thread_pool_exec = lambda function, *args, **kwargs: function(*args, **kwargs)+ monkeypatch.setitem(sys.modules, "common.misc_utils", fake_misc_utils)++ spec = importlib.util.spec_from_file_location("_base_for_regex_test", repo_root / "agent" / "component" / "base.py")+ module = importlib.util.module_from_spec(spec)+ spec.loader.exec_module(module)+ return module+++def retrieval_module(monkeypatch, base_module):+ """Load the Retrieval class without its service-layer dependencies."""+ repo_root = Path(__file__).resolve().parents[4]++ def install_module(name, *, package=False, **attributes):+ module = ModuleType(name)+ if package:+ module.__path__ = []+ for key, value in attributes.items():+ setattr(module, key, value)+ monkeypatch.setitem(sys.modules, name, module)+ if "." in name:+ parent_name, child_name = name.rsplit(".", 1)+ setattr(sys.modules[parent_name], child_name, module)+ return module++ install_module("agent", package=True)+ install_module("agent.component", package=True)+ monkeypatch.setitem(sys.modules, "agent.component.base", base_module)+ sys.modules["agent.component"].base = base_module+ install_module("agent.tools", package=True)++ class ToolParamBase(base_module.ComponentParamBase):+ pass++ class ToolBase(base_module.ComponentBase):+ pass++ install_module("agent.tools.base", ToolParamBase=ToolParamBase, ToolBase=ToolBase, ToolMeta=dict)++ install_module("api", package=True)+ install_module("api.db", package=True)+ install_module("api.db.services", package=True)+ install_module("api.db.joint_services", package=True)+ install_module("api.db.services.doc_metadata_service", DocMetadataService=type("DocMetadataService", (), {}))+ install_module("api.db.services.knowledgebase_service", KnowledgebaseService=type("KnowledgebaseService", (), {}))+ install_module("api.db.services.llm_service", LLMBundle=type("LLMBundle", (), {}))+ install_module("api.db.services.memory_service", MemoryService=type("MemoryService", (), {}))+ install_module("api.db.joint_services.memory_message_service")+ install_module(+ "api.db.joint_services.tenant_model_service",+ get_tenant_default_model_by_type=lambda *_args, **_kwargs: None,+ resolve_model_config=lambda *_args, **_kwargs: None,+ )++ install_module("common", package=True)+ install_module("common.constants", LLMType=SimpleNamespace(EMBEDDING="embedding", RERANK="rerank", CHAT="chat"))+ install_module("common.metadata_utils", apply_meta_data_filter=lambda *_args, **_kwargs: [])+ install_module("common.settings")+ install_module("common.connection_utils", timeout=lambda _seconds: lambda function: function)++ install_module("rag", package=True)+ install_module("rag.app", package=True)+ install_module("rag.app.tag", label_question=lambda *_args, **_kwargs: None)+ install_module("rag.prompts", package=True)+ install_module(+ "rag.prompts.generator",+ cross_languages=lambda *_args, **_kwargs: None,+ kb_prompt=lambda *_args, **_kwargs: [],+ memory_prompt=lambda *_args, **_kwargs: [],)++ spec = importlib.util.spec_from_file_location("_retrieval_for_regex_test", repo_root / "agent" / "tools" / "retrieval.py")module = importlib.util.module_from_spec(spec)spec.loader.exec_module(module)return module@@ -111,10 +187,7 @@ def test_variable_ref_patt_matches_underscored_component_ids(base_module):for text, expected in cases:matches = list(patt.finditer(text))assert matches, f"Expected {text!r} to match variable_ref_patt"- assert matches[0].group(1) == expected, (- f"{text!r}: wrong capture — got {matches[0].group(1)!r}, "- f"expected {expected!r}"- )+ assert matches[0].group(1) == expected, f"{text!r}: wrong capture — got {matches[0].group(1)!r}, expected {expected!r}"@pytest.mark.p2@@ -141,14 +214,8 @@ def test_variable_ref_patt_matches_colon_bearing_component_ids(base_module):for text, expected in cases:matches = list(patt.finditer(text))- assert matches, (- f"Expected {text!r} to match variable_ref_patt — colon-bearing "- f"cqn_id lost its support."- )- assert matches[0].group(1) == expected, (- f"{text!r}: wrong capture — got {matches[0].group(1)!r}, "- f"expected {expected!r}"- )+ assert matches, f"Expected {text!r} to match variable_ref_patt — colon-bearing cqn_id lost its support."+ assert matches[0].group(1) == expected, f"{text!r}: wrong capture — got {matches[0].group(1)!r}, expected {expected!r}"@pytest.mark.p2@@ -186,30 +253,15 @@ def test_variable_ref_patt_re_matches_variable_ref_patt(base_module):canonical = base_module.ComponentBase.variable_ref_patt_re# Same source pattern & flags.- assert canonical.pattern == rebuilt.pattern, (- "variable_ref_patt_re must be compiled from variable_ref_patt "- "(patterns differ)."- )- assert canonical.flags == rebuilt.flags, (- "variable_ref_patt_re flags changed unexpectedly."- )+ assert canonical.pattern == rebuilt.pattern, "variable_ref_patt_re must be compiled from variable_ref_patt (patterns differ)."+ assert canonical.flags == rebuilt.flags, "variable_ref_patt_re flags changed unexpectedly."# Same match positions / groups on a representative sample.- sample = (- "Repeat: {userfillup_abc@line} / also {Retrieval:KBSearch@f} / "- "sys={sys.query}"- )+ sample = "Repeat: {userfillup_abc@line} / also {Retrieval:KBSearch@f} / sys={sys.query}"- canonical_matches = [- (m.start(), m.end(), m.group(1)) for m in canonical.finditer(sample)- ]- rebuilt_matches = [- (m.start(), m.end(), m.group(1)) for m in rebuilt.finditer(sample)- ]- assert canonical_matches == rebuilt_matches, (- "variable_ref_patt_re produces different matches than a fresh "- "compile of variable_ref_patt — they have silently diverged."- )+ canonical_matches = [(m.start(), m.end(), m.group(1)) for m in canonical.finditer(sample)]+ rebuilt_matches = [(m.start(), m.end(), m.group(1)) for m in rebuilt.finditer(sample)]+ assert canonical_matches == rebuilt_matches, "variable_ref_patt_re produces different matches than a fresh compile of variable_ref_patt — they have silently diverged."@pytest.mark.p2@@ -226,10 +278,7 @@ def test_get_input_elements_from_text_resolves_underscored_id(base_module):)elements = cpn.get_input_elements_from_text("Repeat: {userfillup_abc@line}")- assert "userfillup_abc@line" in elements, (- "Underscored `cpn_id@var_nm` template ref was not extracted — "- "see #16758: Await-response variable ignored by Agent."- )+ assert "userfillup_abc@line" in elements, "Underscored `cpn_id@var_nm` template ref was not extracted — see #16758: Await-response variable ignored by Agent."assert elements["userfillup_abc@line"]["value"] == "user-text"assert elements["userfillup_abc@line"]["_cpn_id"] == "userfillup_abc"@@ -255,7 +304,108 @@ def test_variable_ref_patt_does_not_match_bare_var_name(base_module):"""patt = base_module.ComponentBase.variable_ref_patt_rematches = list(patt.finditer("{line}"))- assert not matches, (- "Bare `{line}` should not match — only `cpn_id@var` / `sys.*` / "- "`env.*` are valid template refs."+ assert not matches, "Bare `{line}` should not match — only `cpn_id@var` / `sys.*` / `env.*` are valid template refs."++[email protected]("content", ["{ {A@x}", "{A@x} }"])+def test_unbalanced_outer_braces_remain_literal(base_module, content):+ cpn = base_module.ComponentBase.__new__(base_module.ComponentBase)+ cpn._canvas = SimpleNamespace(+ get_component_name=lambda _cid: "A",+ get_variable_value=lambda exp: "VALUE" if exp == "A@x" else None,)++ assert base_module.ComponentBase.variable_ref_patt_re.fullmatch(content) is None+ assert cpn.get_input_elements_from_text(content) == {}+ assert cpn.string_format(content, {"A@x": "VALUE"}) == content++[email protected]("content", ["{ {A@x}", "{A@x} }"])+def test_manual_metadata_filter_preserves_unbalanced_outer_braces(retrieval_module, content):+ retrieval = retrieval_module.Retrieval.__new__(retrieval_module.Retrieval)+ retrieval._canvas = SimpleNamespace(get_variable_value=lambda exp: "VALUE" if exp == "A@x" else None)+ original = {"key": "author", "op": "=", "value": content}++ assert retrieval._resolve_manual_filter(original) == original++[email protected]("content", ["{A@x}", "{{A@x}}", "{ {A@x} }"])+def test_manual_metadata_filter_resolves_complete_references(retrieval_module, content):+ retrieval = retrieval_module.Retrieval.__new__(retrieval_module.Retrieval)+ retrieval._canvas = SimpleNamespace(get_variable_value=lambda exp: "VALUE" if exp == "A@x" else None)+ original = {"key": "author", "op": "=", "value": f"before {content} after"}++ assert retrieval._resolve_manual_filter(original)["value"] == "before VALUE after"+ assert original["value"] == f"before {content} after"+++def test_rejected_incomplete_match_is_logged_without_template_value(base_module, caplog):+ cpn = base_module.ComponentBase.__new__(base_module.ComponentBase)++ with caplog.at_level(logging.DEBUG):+ assert list(cpn._iter_template_matches(cpn.variable_ref_patt_re, "{ {SecretNode@token}")) == []++ messages = [record.getMessage() for record in caplog.records]+ assert any("Ignored incomplete template reference candidate" in message for message in messages)+ assert all("SecretNode@token" not in message for message in messages)+++ ("pattern_name", "content", "values", "matched_text", "expected"),+ [+ (+ "variable_ref_patt_re",+ "Decision: {Agent:x@content} — done",+ {"Agent:x@content": "VALUE"},+ ["{Agent:x@content}"],+ "Decision: VALUE — done",+ ),+ (+ "variable_ref_patt_re",+ "{A@x} {B@y}",+ {"A@x": "ONE", "B@y": "TWO"},+ ["{A@x}", "{B@y}"],+ "ONE TWO",+ ),+ (+ "variable_ref_patt_re",+ "a {{X@y}} b",+ {"X@y": "VALUE"},+ ["{{X@y}}"],+ "a VALUE b",+ ),+ (+ "variable_ref_patt_re",+ "a { {X@y} } b",+ {"X@y": "VALUE"},+ ["{ {X@y} }"],+ "a VALUE b",+ ),+ (+ "iteration_alias_patt_re",+ "before {item} after",+ {"item": "VALUE"},+ ["{item}"],+ "before VALUE after",+ ),+ ],+)+def test_reference_patterns_preserve_adjacent_literal_whitespace(base_module, pattern_name, content, values, matched_text, expected):+ cpn = base_module.ComponentBase.__new__(base_module.ComponentBase)+ pattern = getattr(base_module.ComponentBase, pattern_name)+ matches = list(pattern.finditer(content))++ assert [match.group(0) for match in matches] == matched_text+ if pattern_name == "variable_ref_patt_re":+ cpn._canvas = SimpleNamespace(+ get_component_name=lambda cpn_id: cpn_id,+ get_variable_value=lambda exp: values.get(exp),+ )+ elements = cpn.get_input_elements_from_text(content)+ assert {key: meta["value"] for key, meta in elements.items()} == values+ assert cpn.string_format(content, values) == expected