Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

fix: fix security issue in ob_conn_base.py (#19524)

Anupam Mediratta3a9b84495c94
diff --git a/common/doc_store/ob_conn_base.py b/common/doc_store/ob_conn_base.py
index 29b2379c59..8f0d450fa4 100644
--- a/common/doc_store/ob_conn_base.py
+++ b/common/doc_store/ob_conn_base.py
@@ -20,6 +20,7 @@ import re
import threading
import time
from abc import abstractmethod
+from collections.abc import Iterable
from typing import Any
from pymysql.converters import escape_string
@@ -36,7 +37,7 @@ index_name_template = "ix_%s_%s"
fulltext_index_name_template = "fts_idx_%s"
fulltext_search_template = "MATCH (%s) AGAINST ('%s' IN NATURAL LANGUAGE MODE)"
vector_search_template = "cosine_distance(%s, '%s')"
-vector_column_pattern = re.compile(r"q_(?P<vector_size>\d+)_vec")
+vector_column_pattern = re.compile(r"q_(?P<vector_size>\d+)_vec$")
# Document metadata table columns
doc_meta_columns = [
@@ -65,6 +66,15 @@ def get_value_str(value: Any) -> str:
return str(value)
+def validate_column_name(column_name: Any, valid_columns: Iterable[str] | None = None, pattern: re.Pattern | None = None) -> str:
+ """Validate a dynamic SQL identifier before it is interpolated into a query."""
+ if not isinstance(column_name, str) or not column_name.isidentifier():
+ raise ValueError(f"Invalid column name: {column_name!r}")
+ if valid_columns is not None and column_name not in valid_columns and not (pattern and pattern.match(column_name)):
+ raise ValueError(f"Invalid column name: {column_name!r}")
+ return column_name
+
+
def _try_with_lock(lock_name: str, process_func, check_func, timeout: int = None):
"""Execute function with distributed lock."""
if not timeout:
@@ -581,10 +591,18 @@ class OBConnectionBase(DocStoreConnection):
return "kb_id"
def _get_filters(self, condition: dict) -> list[str]:
+ """Build SQL WHERE-clause fragments from a condition dict, validating every dynamic column name first."""
filters: list[str] = []
for k, v in condition.items():
if not v:
continue
+ if k == "exists":
+ column_name = v
+ elif k == "must_not" and isinstance(v, dict) and "exists" in v:
+ column_name = v.get("exists")
+ else:
+ column_name = k
+ validate_column_name(column_name)
if k == "exists":
filters.append(f"{v} IS NOT NULL")
elif k == "must_not" and isinstance(v, dict) and "exists" in v:
diff --git a/memory/utils/ob_conn.py b/memory/utils/ob_conn.py
index 45f78c7802..c9d7d9514b 100644
--- a/memory/utils/ob_conn.py
+++ b/memory/utils/ob_conn.py
@@ -27,7 +27,7 @@ from common.decorator import singleton
from memory.utils.aggregation_utils import aggregate_by_field
from memory.utils.highlight_utils import get_highlight_from_messages
from common.doc_store.doc_store_base import MatchExpr, OrderByExpr, FusionExpr, MatchTextExpr, MatchDenseExpr
-from common.doc_store.ob_conn_base import OBConnectionBase, get_value_str, vector_search_template
+from common.doc_store.ob_conn_base import OBConnectionBase, get_value_str, validate_column_name, vector_column_pattern, vector_search_template
from common.float_utils import get_float
from rag.nlp import is_english
from rag.nlp.rag_tokenizer import tokenize, fine_grained_tokenize
@@ -53,6 +53,10 @@ COLUMN_DEFINITIONS: list[Column] = [
COLUMN_NAMES: list[str] = [col.name for col in COLUMN_DEFINITIONS]
+# Field names that are valid as convert_field_name() inputs: real columns, plus pseudo-fields used
+# only as dict keys to dispatch query logic (never themselves interpolated into SQL as identifiers).
+_VALID_FIELD_NAMES: frozenset[str] = frozenset(COLUMN_NAMES) | {"_score", "exists", "must_not", "remove"}
+
# Index columns for creating indexes
INDEX_COLUMNS: list[str] = [
"message_id",
@@ -121,7 +125,7 @@ class OBConnection(OBConnectionBase):
@staticmethod
def convert_field_name(field_name: str, use_tokenized_content=False) -> str:
- """Convert message field name to database column name."""
+ """Convert message field name to database column name, rejecting names outside the known schema."""
match field_name:
case "message_type":
return "message_type_kwd"
@@ -132,7 +136,7 @@ class OBConnection(OBConnectionBase):
return "tokenized_content_ltks"
return "content_ltks"
case _:
- return field_name
+ return validate_column_name(field_name, _VALID_FIELD_NAMES, vector_column_pattern)
@staticmethod
def map_message_to_ob_fields(message: dict) -> dict:
@@ -491,7 +495,7 @@ class OBConnection(OBConnectionBase):
for k, v in update_dict.items():
if k == "remove":
if isinstance(v, str):
- set_values.append(f"{v} = NULL")
+ set_values.append(f"{self.convert_field_name(v)} = NULL")
elif k == "status":
set_values.append(f"status_int = {1 if v else 0}")
else:
diff --git a/rag/utils/ob_conn.py b/rag/utils/ob_conn.py
index 4205215a23..0a47237c4e 100644
--- a/rag/utils/ob_conn.py
+++ b/rag/utils/ob_conn.py
@@ -34,6 +34,7 @@ from common.doc_store.doc_store_base import MatchExpr, OrderByExpr, FusionExpr,
from common.doc_store.ob_conn_base import (
OBConnectionBase,
get_value_str,
+ validate_column_name,
vector_search_template,
vector_column_pattern,
fulltext_index_name_template,
@@ -1201,6 +1202,7 @@ class OBConnection(OBConnectionBase):
for k, v in new_value.items():
if k == "remove":
if isinstance(v, str):
+ validate_column_name(v, _VALID_FILTER_COLUMNS, vector_column_pattern)
set_values.append(f"{v} = NULL")
else:
if not isinstance(v, dict):
@@ -1230,6 +1232,7 @@ class OBConnection(OBConnectionBase):
if title:
set_values.append(f"docnm_kwd = {get_value_str(title)}")
else:
+ validate_column_name(k, _VALID_FILTER_COLUMNS, vector_column_pattern)
set_values.append(f"{k} = {get_value_str(v)}")
if not set_values:
diff --git a/test/unit_test/common/test_ob_conn_base.py b/test/unit_test/common/test_ob_conn_base.py
new file mode 100644
index 0000000000..2a38fbff49
--- /dev/null
+++ b/test/unit_test/common/test_ob_conn_base.py
@@ -0,0 +1,119 @@
+#
+# Copyright 2025 The InfiniFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+"""
+Unit tests for shared OceanBase dynamic-SQL-identifier validation (V-004 follow-up).
+"""
+
+import re
+
+import pytest
+
+from common.doc_store.ob_conn_base import OBConnectionBase, validate_column_name, vector_column_pattern
+
+
+class TestValidateColumnName:
+ """Test cases for the validate_column_name helper."""
+
+ def test_valid_identifier_no_allowlist(self):
+ assert validate_column_name("kb_id") == "kb_id"
+
+ def test_non_string_rejected(self):
+ with pytest.raises(ValueError):
+ validate_column_name(123)
+
+ def test_sql_injection_attempt_rejected(self):
+ with pytest.raises(ValueError):
+ validate_column_name("id; DROP TABLE users;--")
+
+ def test_syntactically_valid_but_not_in_allowlist_rejected(self):
+ with pytest.raises(ValueError):
+ validate_column_name("not_a_real_column", {"kb_id", "doc_id"})
+
+ def test_in_allowlist_accepted(self):
+ assert validate_column_name("kb_id", {"kb_id", "doc_id"}) == "kb_id"
+
+ def test_pattern_fallback_accepted(self):
+ pattern = re.compile(r"q_(?P<vector_size>\d+)_vec")
+ assert validate_column_name("q_768_vec", {"kb_id"}, pattern) == "q_768_vec"
+
+ def test_pattern_fallback_does_not_bypass_identifier_check(self):
+ pattern = re.compile(r".*")
+ with pytest.raises(ValueError):
+ validate_column_name("id; DROP TABLE users;--", {"kb_id"}, pattern)
+
+
+class TestVectorColumnPattern:
+ """vector_column_pattern must anchor at both ends when used as an allowlist pattern."""
+
+ def test_matches_exact_vector_column(self):
+ assert vector_column_pattern.match("q_1024_vec")
+
+ def test_rejects_suffixed_lookalike(self):
+ with pytest.raises(ValueError):
+ validate_column_name("q_1024_vec_x", {"kb_id"}, vector_column_pattern)
+
+ def test_rejects_appended_lookalike(self):
+ with pytest.raises(ValueError):
+ validate_column_name("q_1024_vecabc", {"kb_id"}, vector_column_pattern)
+
+
+def _unimplemented(self, *args, **kwargs):
+ raise NotImplementedError
+
+
+class _DummyOBConnection(OBConnectionBase):
+ """Minimal concrete subclass so the abstract base class's _get_filters() can be exercised directly."""
+
+ get_aggregation = _unimplemented
+ get_column_definitions = _unimplemented
+ get_doc_ids = _unimplemented
+ get_fields = _unimplemented
+ get_fulltext_columns = _unimplemented
+ get_highlight = _unimplemented
+ get_index_columns = _unimplemented
+ get_lock_prefix = _unimplemented
+ get_total = _unimplemented
+ insert = _unimplemented
+ search = _unimplemented
+ update = _unimplemented
+
+
+class TestOBConnectionBaseGetFilters:
+ """Test cases for OBConnectionBase._get_filters, which underlies memory/utils/ob_conn.py."""
+
+ def _base(self):
+ return object.__new__(_DummyOBConnection)
+
+ def test_malicious_column_key_rejected(self):
+ with pytest.raises(ValueError):
+ self._base()._get_filters({"id; DROP TABLE users;--": "foo"})
+
+ def test_malicious_exists_value_rejected(self):
+ with pytest.raises(ValueError):
+ self._base()._get_filters({"exists": "id; DROP TABLE users;--"})
+
+ def test_malicious_must_not_exists_value_rejected(self):
+ with pytest.raises(ValueError):
+ self._base()._get_filters({"must_not": {"exists": "id; DROP TABLE users;--"}})
+
+ def test_legitimate_condition_produces_expected_filter(self):
+ filters = self._base()._get_filters({"kb_id": "kb1"})
+ assert filters == ["kb_id = 'kb1'"]
+
+ def test_legitimate_exists_condition(self):
+ filters = self._base()._get_filters({"exists": "forget_at"})
+ assert filters == ["forget_at IS NOT NULL"]
diff --git a/test/unit_test/memory/utils/test_ob_conn_column_validation.py b/test/unit_test/memory/utils/test_ob_conn_column_validation.py
new file mode 100644
index 0000000000..e75b772d1e
--- /dev/null
+++ b/test/unit_test/memory/utils/test_ob_conn_column_validation.py
@@ -0,0 +1,113 @@
+#
+# Copyright 2025 The InfiniFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+"""
+Unit tests for dynamic-SQL-identifier validation in memory/utils/ob_conn.py (V-004 follow-up).
+"""
+
+from unittest.mock import MagicMock
+
+import pytest
+
+from memory.utils.ob_conn import OBConnection
+
+
+def _ob_connection_class():
+ return next(cell.cell_contents for cell in OBConnection.__closure__ if isinstance(cell.cell_contents, type))
+
+
+class TestConvertFieldName:
+ """convert_field_name() must reject any name outside the known schema."""
+
+ def test_malicious_field_name_rejected(self):
+ with pytest.raises(ValueError):
+ _ob_connection_class().convert_field_name("id; DROP TABLE users;--")
+
+ def test_known_alias_message_type(self):
+ assert _ob_connection_class().convert_field_name("message_type") == "message_type_kwd"
+
+ def test_known_alias_status(self):
+ assert _ob_connection_class().convert_field_name("status") == "status_int"
+
+ def test_known_alias_content(self):
+ assert _ob_connection_class().convert_field_name("content") == "content_ltks"
+ assert _ob_connection_class().convert_field_name("content", use_tokenized_content=True) == "tokenized_content_ltks"
+
+ def test_real_column_passthrough(self):
+ assert _ob_connection_class().convert_field_name("memory_id") == "memory_id"
+
+ def test_reserved_query_keywords_passthrough(self):
+ assert _ob_connection_class().convert_field_name("exists") == "exists"
+ assert _ob_connection_class().convert_field_name("must_not") == "must_not"
+ assert _ob_connection_class().convert_field_name("remove") == "remove"
+
+ def test_score_passthrough(self):
+ assert _ob_connection_class().convert_field_name("_score") == "_score"
+
+ def test_vector_column_passthrough(self):
+ assert _ob_connection_class().convert_field_name("q_768_vec") == "q_768_vec"
+
+
+class TestConnectionGetFilters:
+ """_get_filters() is inherited from OBConnectionBase; verify it rejects malicious columns for this connector too."""
+
+ def _connection(self):
+ return object.__new__(_ob_connection_class())
+
+ def test_malicious_column_key_rejected(self):
+ with pytest.raises(ValueError):
+ self._connection()._get_filters({"id; DROP TABLE users;--": "foo"})
+
+
+class TestUpdateColumnValidation:
+ """Regression tests: OBConnection.update() must validate dynamic identifiers before building SQL."""
+
+ def _connection(self):
+ connection = object.__new__(_ob_connection_class())
+ connection._check_table_exists_cached = lambda index_name: True
+ connection.client = MagicMock()
+ connection.logger = MagicMock()
+ return connection
+
+ def test_malicious_new_value_key_rejected(self):
+ connection = self._connection()
+ with pytest.raises(ValueError):
+ connection.update({}, {"id; DROP TABLE users;--": "foo"}, "test_table", "mem1")
+ connection.client.perform_raw_text_sql.assert_not_called()
+
+ def test_malicious_remove_value_rejected(self):
+ connection = self._connection()
+ with pytest.raises(ValueError):
+ connection.update({}, {"remove": "id; DROP TABLE users;--"}, "test_table", "mem1")
+ connection.client.perform_raw_text_sql.assert_not_called()
+
+ def test_legitimate_update_succeeds(self):
+ connection = self._connection()
+ result = connection.update({"id": "m1"}, {"session_id": "s1"}, "test_table", "mem1")
+ assert result is True
+ connection.client.perform_raw_text_sql.assert_called_once()
+
+ def test_legitimate_remove_succeeds(self):
+ connection = self._connection()
+ result = connection.update({"id": "m1"}, {"remove": "session_id"}, "test_table", "mem1")
+ assert result is True
+ connection.client.perform_raw_text_sql.assert_called_once()
+
+ def test_remove_with_alias_resolves_column(self):
+ connection = self._connection()
+ connection.update({"id": "m1"}, {"remove": "content"}, "test_table", "mem1")
+ sql = connection.client.perform_raw_text_sql.call_args[0][0]
+ assert "content_ltks = NULL" in sql
diff --git a/test/unit_test/rag/utils/test_ob_conn.py b/test/unit_test/rag/utils/test_ob_conn.py
index 7f89088e19..d78e822b06 100644
--- a/test/unit_test/rag/utils/test_ob_conn.py
+++ b/test/unit_test/rag/utils/test_ob_conn.py
@@ -19,8 +19,11 @@ Unit tests for OceanBase connection utility functions.
"""
import logging
+from unittest.mock import MagicMock
-from rag.utils.ob_conn import OBConnection, SearchResult, get_metadata_filter_expression, get_value_str
+import pytest
+
+from rag.utils.ob_conn import OBConnection, SearchResult, get_filters, get_metadata_filter_expression, get_value_str
def _ob_connection_class():
@@ -265,3 +268,66 @@ class TestGetMetadataFilterExpression:
result = get_metadata_filter_expression(filter_dict)
assert result.startswith("(")
assert result.endswith(")")
+
+
+class TestGetFiltersColumnValidation:
+ """Regression tests: get_filters() must not let malicious column names reach SQL (V-004 follow-up)."""
+
+ def test_malicious_column_key_silently_skipped(self):
+ assert get_filters({"id; DROP TABLE users;--": "foo"}) == []
+
+ def test_malicious_compound_key_silently_skipped(self):
+ assert get_filters({"id = 1, malicious_column": "foo"}) == []
+
+ def test_malicious_exists_value_silently_skipped(self):
+ assert get_filters({"exists": "id; DROP TABLE users;--"}) == []
+
+ def test_legitimate_column_still_filtered(self):
+ assert get_filters({"kb_id": "kb1"}) == ["kb_id = 'kb1'"]
+
+
+class TestUpdateColumnValidation:
+ """Regression tests: OBConnection.update() must validate dynamic identifiers before building SQL (V-004 follow-up)."""
+
+ def _connection(self):
+ connection = object.__new__(_ob_connection_class())
+ connection._check_table_exists_cached = lambda index_name: True
+ connection.client = MagicMock()
+ return connection
+
+ def test_malicious_new_value_key_rejected(self):
+ connection = self._connection()
+ with pytest.raises(ValueError):
+ connection.update({}, {"id; DROP TABLE users;--": "foo"}, "test_table", "kb1")
+ connection.client.perform_raw_text_sql.assert_not_called()
+
+ def test_malicious_remove_value_rejected(self):
+ connection = self._connection()
+ with pytest.raises(ValueError):
+ connection.update({}, {"remove": "id; DROP TABLE users;--"}, "test_table", "kb1")
+ connection.client.perform_raw_text_sql.assert_not_called()
+
+ def test_legitimate_update_succeeds(self):
+ connection = self._connection()
+ result = connection.update({"id": "c1"}, {"docnm_kwd": "foo"}, "test_table", "kb1")
+ assert result is True
+ connection.client.perform_raw_text_sql.assert_called_once()
+
+ def test_legitimate_remove_succeeds(self):
+ connection = self._connection()
+ result = connection.update({"id": "c1"}, {"remove": "docnm_kwd"}, "test_table", "kb1")
+ assert result is True
+ connection.client.perform_raw_text_sql.assert_called_once()
+
+ def test_vector_column_update_succeeds(self):
+ """Regression test: dynamic q_<n>_vec columns must not be rejected by the static allowlist."""
+ connection = self._connection()
+ result = connection.update({"id": "c1"}, {"q_1024_vec": [0.1, 0.2, 0.3]}, "test_table", "kb1")
+ assert result is True
+ connection.client.perform_raw_text_sql.assert_called_once()
+
+ def test_vector_column_remove_succeeds(self):
+ connection = self._connection()
+ result = connection.update({"id": "c1"}, {"remove": "q_1024_vec"}, "test_table", "kb1")
+ assert result is True
+ connection.client.perform_raw_text_sql.assert_called_once()