RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix: fix security issue in ob_conn_base.py (#19524)
Anupam Mediratta3a9b84495c94
diff --git a/common/doc_store/ob_conn_base.py b/common/doc_store/ob_conn_base.pyindex 29b2379c59..8f0d450fa4 100644--- a/common/doc_store/ob_conn_base.py+++ b/common/doc_store/ob_conn_base.py@@ -20,6 +20,7 @@ import reimport threadingimport timefrom abc import abstractmethod+from collections.abc import Iterablefrom typing import Anyfrom pymysql.converters import escape_string@@ -36,7 +37,7 @@ index_name_template = "ix_%s_%s"fulltext_index_name_template = "fts_idx_%s"fulltext_search_template = "MATCH (%s) AGAINST ('%s' IN NATURAL LANGUAGE MODE)"vector_search_template = "cosine_distance(%s, '%s')"-vector_column_pattern = re.compile(r"q_(?P<vector_size>\d+)_vec")+vector_column_pattern = re.compile(r"q_(?P<vector_size>\d+)_vec$")# Document metadata table columnsdoc_meta_columns = [@@ -65,6 +66,15 @@ def get_value_str(value: Any) -> str:return str(value)+def validate_column_name(column_name: Any, valid_columns: Iterable[str] | None = None, pattern: re.Pattern | None = None) -> str:+ """Validate a dynamic SQL identifier before it is interpolated into a query."""+ if not isinstance(column_name, str) or not column_name.isidentifier():+ raise ValueError(f"Invalid column name: {column_name!r}")+ if valid_columns is not None and column_name not in valid_columns and not (pattern and pattern.match(column_name)):+ raise ValueError(f"Invalid column name: {column_name!r}")+ return column_name++def _try_with_lock(lock_name: str, process_func, check_func, timeout: int = None):"""Execute function with distributed lock."""if not timeout:@@ -581,10 +591,18 @@ class OBConnectionBase(DocStoreConnection):return "kb_id"def _get_filters(self, condition: dict) -> list[str]:+ """Build SQL WHERE-clause fragments from a condition dict, validating every dynamic column name first."""filters: list[str] = []for k, v in condition.items():if not v:continue+ if k == "exists":+ column_name = v+ elif k == "must_not" and isinstance(v, dict) and "exists" in v:+ column_name = v.get("exists")+ else:+ column_name = k+ validate_column_name(column_name)if k == "exists":filters.append(f"{v} IS NOT NULL")elif k == "must_not" and isinstance(v, dict) and "exists" in v:diff --git a/memory/utils/ob_conn.py b/memory/utils/ob_conn.pyindex 45f78c7802..c9d7d9514b 100644--- a/memory/utils/ob_conn.py+++ b/memory/utils/ob_conn.py@@ -27,7 +27,7 @@ from common.decorator import singletonfrom memory.utils.aggregation_utils import aggregate_by_fieldfrom memory.utils.highlight_utils import get_highlight_from_messagesfrom common.doc_store.doc_store_base import MatchExpr, OrderByExpr, FusionExpr, MatchTextExpr, MatchDenseExpr-from common.doc_store.ob_conn_base import OBConnectionBase, get_value_str, vector_search_template+from common.doc_store.ob_conn_base import OBConnectionBase, get_value_str, validate_column_name, vector_column_pattern, vector_search_templatefrom common.float_utils import get_floatfrom rag.nlp import is_englishfrom rag.nlp.rag_tokenizer import tokenize, fine_grained_tokenize@@ -53,6 +53,10 @@ COLUMN_DEFINITIONS: list[Column] = [COLUMN_NAMES: list[str] = [col.name for col in COLUMN_DEFINITIONS]+# Field names that are valid as convert_field_name() inputs: real columns, plus pseudo-fields used+# only as dict keys to dispatch query logic (never themselves interpolated into SQL as identifiers).+_VALID_FIELD_NAMES: frozenset[str] = frozenset(COLUMN_NAMES) | {"_score", "exists", "must_not", "remove"}+# Index columns for creating indexesINDEX_COLUMNS: list[str] = ["message_id",@@ -121,7 +125,7 @@ class OBConnection(OBConnectionBase):@staticmethoddef convert_field_name(field_name: str, use_tokenized_content=False) -> str:- """Convert message field name to database column name."""+ """Convert message field name to database column name, rejecting names outside the known schema."""match field_name:case "message_type":return "message_type_kwd"@@ -132,7 +136,7 @@ class OBConnection(OBConnectionBase):return "tokenized_content_ltks"return "content_ltks"case _:- return field_name+ return validate_column_name(field_name, _VALID_FIELD_NAMES, vector_column_pattern)@staticmethoddef map_message_to_ob_fields(message: dict) -> dict:@@ -491,7 +495,7 @@ class OBConnection(OBConnectionBase):for k, v in update_dict.items():if k == "remove":if isinstance(v, str):- set_values.append(f"{v} = NULL")+ set_values.append(f"{self.convert_field_name(v)} = NULL")elif k == "status":set_values.append(f"status_int = {1 if v else 0}")else:diff --git a/rag/utils/ob_conn.py b/rag/utils/ob_conn.pyindex 4205215a23..0a47237c4e 100644--- a/rag/utils/ob_conn.py+++ b/rag/utils/ob_conn.py@@ -34,6 +34,7 @@ from common.doc_store.doc_store_base import MatchExpr, OrderByExpr, FusionExpr,from common.doc_store.ob_conn_base import (OBConnectionBase,get_value_str,+ validate_column_name,vector_search_template,vector_column_pattern,fulltext_index_name_template,@@ -1201,6 +1202,7 @@ class OBConnection(OBConnectionBase):for k, v in new_value.items():if k == "remove":if isinstance(v, str):+ validate_column_name(v, _VALID_FILTER_COLUMNS, vector_column_pattern)set_values.append(f"{v} = NULL")else:if not isinstance(v, dict):@@ -1230,6 +1232,7 @@ class OBConnection(OBConnectionBase):if title:set_values.append(f"docnm_kwd = {get_value_str(title)}")else:+ validate_column_name(k, _VALID_FILTER_COLUMNS, vector_column_pattern)set_values.append(f"{k} = {get_value_str(v)}")if not set_values:diff --git a/test/unit_test/common/test_ob_conn_base.py b/test/unit_test/common/test_ob_conn_base.pynew file mode 100644index 0000000000..2a38fbff49--- /dev/null+++ b/test/unit_test/common/test_ob_conn_base.py@@ -0,0 +1,119 @@+#+# Copyright 2025 The InfiniFlow Authors. All Rights Reserved.+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+#++"""+Unit tests for shared OceanBase dynamic-SQL-identifier validation (V-004 follow-up).+"""++import re++import pytest++from common.doc_store.ob_conn_base import OBConnectionBase, validate_column_name, vector_column_pattern+++class TestValidateColumnName:+ """Test cases for the validate_column_name helper."""++ def test_valid_identifier_no_allowlist(self):+ assert validate_column_name("kb_id") == "kb_id"++ def test_non_string_rejected(self):+ with pytest.raises(ValueError):+ validate_column_name(123)++ def test_sql_injection_attempt_rejected(self):+ with pytest.raises(ValueError):+ validate_column_name("id; DROP TABLE users;--")++ def test_syntactically_valid_but_not_in_allowlist_rejected(self):+ with pytest.raises(ValueError):+ validate_column_name("not_a_real_column", {"kb_id", "doc_id"})++ def test_in_allowlist_accepted(self):+ assert validate_column_name("kb_id", {"kb_id", "doc_id"}) == "kb_id"++ def test_pattern_fallback_accepted(self):+ pattern = re.compile(r"q_(?P<vector_size>\d+)_vec")+ assert validate_column_name("q_768_vec", {"kb_id"}, pattern) == "q_768_vec"++ def test_pattern_fallback_does_not_bypass_identifier_check(self):+ pattern = re.compile(r".*")+ with pytest.raises(ValueError):+ validate_column_name("id; DROP TABLE users;--", {"kb_id"}, pattern)+++class TestVectorColumnPattern:+ """vector_column_pattern must anchor at both ends when used as an allowlist pattern."""++ def test_matches_exact_vector_column(self):+ assert vector_column_pattern.match("q_1024_vec")++ def test_rejects_suffixed_lookalike(self):+ with pytest.raises(ValueError):+ validate_column_name("q_1024_vec_x", {"kb_id"}, vector_column_pattern)++ def test_rejects_appended_lookalike(self):+ with pytest.raises(ValueError):+ validate_column_name("q_1024_vecabc", {"kb_id"}, vector_column_pattern)+++def _unimplemented(self, *args, **kwargs):+ raise NotImplementedError+++class _DummyOBConnection(OBConnectionBase):+ """Minimal concrete subclass so the abstract base class's _get_filters() can be exercised directly."""++ get_aggregation = _unimplemented+ get_column_definitions = _unimplemented+ get_doc_ids = _unimplemented+ get_fields = _unimplemented+ get_fulltext_columns = _unimplemented+ get_highlight = _unimplemented+ get_index_columns = _unimplemented+ get_lock_prefix = _unimplemented+ get_total = _unimplemented+ insert = _unimplemented+ search = _unimplemented+ update = _unimplemented+++class TestOBConnectionBaseGetFilters:+ """Test cases for OBConnectionBase._get_filters, which underlies memory/utils/ob_conn.py."""++ def _base(self):+ return object.__new__(_DummyOBConnection)++ def test_malicious_column_key_rejected(self):+ with pytest.raises(ValueError):+ self._base()._get_filters({"id; DROP TABLE users;--": "foo"})++ def test_malicious_exists_value_rejected(self):+ with pytest.raises(ValueError):+ self._base()._get_filters({"exists": "id; DROP TABLE users;--"})++ def test_malicious_must_not_exists_value_rejected(self):+ with pytest.raises(ValueError):+ self._base()._get_filters({"must_not": {"exists": "id; DROP TABLE users;--"}})++ def test_legitimate_condition_produces_expected_filter(self):+ filters = self._base()._get_filters({"kb_id": "kb1"})+ assert filters == ["kb_id = 'kb1'"]++ def test_legitimate_exists_condition(self):+ filters = self._base()._get_filters({"exists": "forget_at"})+ assert filters == ["forget_at IS NOT NULL"]diff --git a/test/unit_test/memory/utils/test_ob_conn_column_validation.py b/test/unit_test/memory/utils/test_ob_conn_column_validation.pynew file mode 100644index 0000000000..e75b772d1e--- /dev/null+++ b/test/unit_test/memory/utils/test_ob_conn_column_validation.py@@ -0,0 +1,113 @@+#+# Copyright 2025 The InfiniFlow Authors. All Rights Reserved.+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+#++"""+Unit tests for dynamic-SQL-identifier validation in memory/utils/ob_conn.py (V-004 follow-up).+"""++from unittest.mock import MagicMock++import pytest++from memory.utils.ob_conn import OBConnection+++def _ob_connection_class():+ return next(cell.cell_contents for cell in OBConnection.__closure__ if isinstance(cell.cell_contents, type))+++class TestConvertFieldName:+ """convert_field_name() must reject any name outside the known schema."""++ def test_malicious_field_name_rejected(self):+ with pytest.raises(ValueError):+ _ob_connection_class().convert_field_name("id; DROP TABLE users;--")++ def test_known_alias_message_type(self):+ assert _ob_connection_class().convert_field_name("message_type") == "message_type_kwd"++ def test_known_alias_status(self):+ assert _ob_connection_class().convert_field_name("status") == "status_int"++ def test_known_alias_content(self):+ assert _ob_connection_class().convert_field_name("content") == "content_ltks"+ assert _ob_connection_class().convert_field_name("content", use_tokenized_content=True) == "tokenized_content_ltks"++ def test_real_column_passthrough(self):+ assert _ob_connection_class().convert_field_name("memory_id") == "memory_id"++ def test_reserved_query_keywords_passthrough(self):+ assert _ob_connection_class().convert_field_name("exists") == "exists"+ assert _ob_connection_class().convert_field_name("must_not") == "must_not"+ assert _ob_connection_class().convert_field_name("remove") == "remove"++ def test_score_passthrough(self):+ assert _ob_connection_class().convert_field_name("_score") == "_score"++ def test_vector_column_passthrough(self):+ assert _ob_connection_class().convert_field_name("q_768_vec") == "q_768_vec"+++class TestConnectionGetFilters:+ """_get_filters() is inherited from OBConnectionBase; verify it rejects malicious columns for this connector too."""++ def _connection(self):+ return object.__new__(_ob_connection_class())++ def test_malicious_column_key_rejected(self):+ with pytest.raises(ValueError):+ self._connection()._get_filters({"id; DROP TABLE users;--": "foo"})+++class TestUpdateColumnValidation:+ """Regression tests: OBConnection.update() must validate dynamic identifiers before building SQL."""++ def _connection(self):+ connection = object.__new__(_ob_connection_class())+ connection._check_table_exists_cached = lambda index_name: True+ connection.client = MagicMock()+ connection.logger = MagicMock()+ return connection++ def test_malicious_new_value_key_rejected(self):+ connection = self._connection()+ with pytest.raises(ValueError):+ connection.update({}, {"id; DROP TABLE users;--": "foo"}, "test_table", "mem1")+ connection.client.perform_raw_text_sql.assert_not_called()++ def test_malicious_remove_value_rejected(self):+ connection = self._connection()+ with pytest.raises(ValueError):+ connection.update({}, {"remove": "id; DROP TABLE users;--"}, "test_table", "mem1")+ connection.client.perform_raw_text_sql.assert_not_called()++ def test_legitimate_update_succeeds(self):+ connection = self._connection()+ result = connection.update({"id": "m1"}, {"session_id": "s1"}, "test_table", "mem1")+ assert result is True+ connection.client.perform_raw_text_sql.assert_called_once()++ def test_legitimate_remove_succeeds(self):+ connection = self._connection()+ result = connection.update({"id": "m1"}, {"remove": "session_id"}, "test_table", "mem1")+ assert result is True+ connection.client.perform_raw_text_sql.assert_called_once()++ def test_remove_with_alias_resolves_column(self):+ connection = self._connection()+ connection.update({"id": "m1"}, {"remove": "content"}, "test_table", "mem1")+ sql = connection.client.perform_raw_text_sql.call_args[0][0]+ assert "content_ltks = NULL" in sqldiff --git a/test/unit_test/rag/utils/test_ob_conn.py b/test/unit_test/rag/utils/test_ob_conn.pyindex 7f89088e19..d78e822b06 100644--- a/test/unit_test/rag/utils/test_ob_conn.py+++ b/test/unit_test/rag/utils/test_ob_conn.py@@ -19,8 +19,11 @@ Unit tests for OceanBase connection utility functions."""import logging+from unittest.mock import MagicMock-from rag.utils.ob_conn import OBConnection, SearchResult, get_metadata_filter_expression, get_value_str+import pytest++from rag.utils.ob_conn import OBConnection, SearchResult, get_filters, get_metadata_filter_expression, get_value_strdef _ob_connection_class():@@ -265,3 +268,66 @@ class TestGetMetadataFilterExpression:result = get_metadata_filter_expression(filter_dict)assert result.startswith("(")assert result.endswith(")")+++class TestGetFiltersColumnValidation:+ """Regression tests: get_filters() must not let malicious column names reach SQL (V-004 follow-up)."""++ def test_malicious_column_key_silently_skipped(self):+ assert get_filters({"id; DROP TABLE users;--": "foo"}) == []++ def test_malicious_compound_key_silently_skipped(self):+ assert get_filters({"id = 1, malicious_column": "foo"}) == []++ def test_malicious_exists_value_silently_skipped(self):+ assert get_filters({"exists": "id; DROP TABLE users;--"}) == []++ def test_legitimate_column_still_filtered(self):+ assert get_filters({"kb_id": "kb1"}) == ["kb_id = 'kb1'"]+++class TestUpdateColumnValidation:+ """Regression tests: OBConnection.update() must validate dynamic identifiers before building SQL (V-004 follow-up)."""++ def _connection(self):+ connection = object.__new__(_ob_connection_class())+ connection._check_table_exists_cached = lambda index_name: True+ connection.client = MagicMock()+ return connection++ def test_malicious_new_value_key_rejected(self):+ connection = self._connection()+ with pytest.raises(ValueError):+ connection.update({}, {"id; DROP TABLE users;--": "foo"}, "test_table", "kb1")+ connection.client.perform_raw_text_sql.assert_not_called()++ def test_malicious_remove_value_rejected(self):+ connection = self._connection()+ with pytest.raises(ValueError):+ connection.update({}, {"remove": "id; DROP TABLE users;--"}, "test_table", "kb1")+ connection.client.perform_raw_text_sql.assert_not_called()++ def test_legitimate_update_succeeds(self):+ connection = self._connection()+ result = connection.update({"id": "c1"}, {"docnm_kwd": "foo"}, "test_table", "kb1")+ assert result is True+ connection.client.perform_raw_text_sql.assert_called_once()++ def test_legitimate_remove_succeeds(self):+ connection = self._connection()+ result = connection.update({"id": "c1"}, {"remove": "docnm_kwd"}, "test_table", "kb1")+ assert result is True+ connection.client.perform_raw_text_sql.assert_called_once()++ def test_vector_column_update_succeeds(self):+ """Regression test: dynamic q_<n>_vec columns must not be rejected by the static allowlist."""+ connection = self._connection()+ result = connection.update({"id": "c1"}, {"q_1024_vec": [0.1, 0.2, 0.3]}, "test_table", "kb1")+ assert result is True+ connection.client.perform_raw_text_sql.assert_called_once()++ def test_vector_column_remove_succeeds(self):+ connection = self._connection()+ result = connection.update({"id": "c1"}, {"remove": "q_1024_vec"}, "test_table", "kb1")+ assert result is True+ connection.client.perform_raw_text_sql.assert_called_once()