Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

Fix oversized important keywords in OceanBase and SeekDB (#18872)

wangyunlai1b5cb3caa0a3
diff --git a/internal/engine/oceanbase/codec.go b/internal/engine/oceanbase/codec.go
index f25c2d5284..e72fae0af8 100644
--- a/internal/engine/oceanbase/codec.go
+++ b/internal/engine/oceanbase/codec.go
@@ -25,10 +25,17 @@ import (
"sort"
"strconv"
"strings"
+ "unicode"
+ "unicode/utf8"
+ "ragflow/internal/common"
"ragflow/internal/tokenizer"
+
+ "go.uber.org/zap"
)
+const importantKeywordMaxBytes = 256
+
var vectorColumnPattern = regexp.MustCompile(`^q_(\d+)_vec$`)
var arrayColumns = map[string]bool{
@@ -62,6 +69,7 @@ var memoryColumnToField = map[string]string{
}
func normalizeChunk(document map[string]interface{}) (map[string]interface{}, error) {
+ document = normalizeImportantKeywordFields(document)
result := make(map[string]interface{}, len(chunkColumns)+1)
extra := make(map[string]interface{})
if existing, ok := document["extra"].(map[string]interface{}); ok {
@@ -258,6 +266,12 @@ func encodeColumnValue(columnName string, value interface{}) (interface{}, error
return string(encoded), err
}
}
+ if columnName == "important_kwd" {
+ normalized, _, _ := normalizeImportantKeywords(value)
+ // JSON escaping must not expand the VARCHAR value stored in each array element.
+ encoded, err := json.Marshal(normalized)
+ return string(encoded), err
+ }
if arrayColumns[columnName] {
return encodeArray(value)
}
@@ -271,6 +285,77 @@ func encodeColumnValue(columnName string, value interface{}) (interface{}, error
return value, nil
}
+func sanitizeImportantKeyword(keyword string) (string, bool) {
+ keyword = strings.TrimSpace(keyword)
+ if len(keyword) <= importantKeywordMaxBytes {
+ return keyword, false
+ }
+
+ length := 0
+ end := 0
+ for _, character := range keyword {
+ size := utf8.RuneLen(character)
+ if length+size > importantKeywordMaxBytes {
+ break
+ }
+ length += size
+ end += size
+ }
+ truncated := strings.TrimRightFunc(keyword[:end], unicode.IsSpace)
+ common.Warn(
+ "Sanitizing oversized OceanBase/SeekDB important keyword",
+ zap.Int("original_bytes", len(keyword)),
+ zap.Int("stored_bytes", len(truncated)),
+ zap.Int("limit_bytes", importantKeywordMaxBytes),
+ )
+ return truncated, true
+}
+
+func normalizeImportantKeywords(value interface{}) (interface{}, []string, bool) {
+ values, ok := interfaceSlice(value)
+ if !ok {
+ return value, nil, false
+ }
+
+ normalized := make([]interface{}, len(values))
+ textKeywords := make([]string, 0, len(values))
+ truncated := false
+ for i, item := range values {
+ text, ok := item.(string)
+ if !ok {
+ normalized[i] = item
+ continue
+ }
+ normalizedText, wasTruncated := sanitizeImportantKeyword(text)
+ normalized[i] = normalizedText
+ textKeywords = append(textKeywords, normalizedText)
+ truncated = truncated || wasTruncated
+ }
+ return normalized, textKeywords, truncated
+}
+
+func normalizeImportantKeywordFields(fields map[string]interface{}) map[string]interface{} {
+ keywords, ok := fields["important_kwd"]
+ if !ok {
+ return fields
+ }
+
+ normalizedKeywords, textKeywords, truncated := normalizeImportantKeywords(keywords)
+ if !truncated {
+ return fields
+ }
+
+ normalizedFields := copyMap(fields)
+ normalizedFields["important_kwd"] = normalizedKeywords
+ joinedKeywords := strings.Join(textKeywords, " ")
+ tokenizedKeywords, err := tokenizer.Tokenize(joinedKeywords)
+ if err != nil {
+ tokenizedKeywords = joinedKeywords
+ }
+ normalizedFields["important_tks"] = tokenizedKeywords
+ return normalizedFields
+}
+
func encodeUpdateValue(kind, columnName string, value interface{}) (interface{}, error) {
if vectorColumnPattern.MatchString(columnName) {
return encodeVector(value)
diff --git a/internal/engine/oceanbase/codec_test.go b/internal/engine/oceanbase/codec_test.go
new file mode 100644
index 0000000000..3650dbc872
--- /dev/null
+++ b/internal/engine/oceanbase/codec_test.go
@@ -0,0 +1,84 @@
+//
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package oceanbase
+
+import (
+ "encoding/json"
+ "strings"
+ "testing"
+
+ "ragflow/internal/tokenizer"
+)
+
+func TestNormalizeChunkBoundsImportantKeywordsAndRecomputesTokens(t *testing.T) {
+ tokenizer.SetEngineType("infinity")
+ defer tokenizer.SetEngineType("")
+
+ asciiKeyword := strings.Repeat("x", importantKeywordMaxBytes+1)
+ unicodeKeyword := strings.Repeat("中", 100)
+ escapedKeyword := strings.Repeat(`\`, 200)
+ document := map[string]interface{}{
+ "id": "chunk-1",
+ "kb_id": "kb-1",
+ "important_kwd": []string{asciiKeyword, unicodeKeyword, escapedKeyword},
+ "important_tks": "stale tokens",
+ }
+
+ normalized, err := normalizeChunk(document)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ var keywords []string
+ if err := json.Unmarshal([]byte(normalized["important_kwd"].(string)), &keywords); err != nil {
+ t.Fatal(err)
+ }
+ if len(keywords) != 3 {
+ t.Fatalf("important_kwd = %#v, want three keywords", keywords)
+ }
+ for _, keyword := range keywords {
+ if len(keyword) > importantKeywordMaxBytes {
+ t.Errorf("important keyword byte length = %d, want <= %d", len(keyword), importantKeywordMaxBytes)
+ }
+ if strings.ToValidUTF8(keyword, "") != keyword {
+ t.Errorf("important keyword is not valid UTF-8: %q", keyword)
+ }
+ }
+ if normalized["important_tks"] != strings.Join(keywords, " ") {
+ t.Errorf("important_tks = %q, want tokens from stored keywords", normalized["important_tks"])
+ }
+ if document["important_tks"] != "stale tokens" {
+ t.Errorf("normalizeChunk mutated input important_tks: %q", document["important_tks"])
+ }
+}
+
+func TestEncodeUpdateValueBoundsImportantKeyword(t *testing.T) {
+ keyword := strings.Repeat("中", 100)
+
+ encoded, err := encodeUpdateValue("chunk", "important_kwd", []string{keyword})
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ var keywords []string
+ if err := json.Unmarshal([]byte(encoded.(string)), &keywords); err != nil {
+ t.Fatal(err)
+ }
+ if len(keywords) != 1 || len(keywords[0]) > importantKeywordMaxBytes {
+ t.Fatalf("important_kwd = %#v, want one keyword bounded to %d bytes", keywords, importantKeywordMaxBytes)
+ }
+}
diff --git a/internal/engine/oceanbase/crud.go b/internal/engine/oceanbase/crud.go
index c273b1a849..acd1ac5760 100644
--- a/internal/engine/oceanbase/crud.go
+++ b/internal/engine/oceanbase/crud.go
@@ -201,6 +201,9 @@ func (e *Engine) UpdateChunks(ctx context.Context, condition, newValue map[strin
if !exists {
return fmt.Errorf("%w: '%s'", types.ErrIndexNotFound, baseName)
}
+ if kind == "chunk" {
+ newValue = normalizeImportantKeywordFields(newValue)
+ }
whereSQL, args, err := buildFilter(condition, kind)
if err != nil {
return err
@@ -248,6 +251,11 @@ func (e *Engine) UpdateChunks(ctx context.Context, condition, newValue map[strin
if kind != "chunk" || !arrayColumns[column] {
return fmt.Errorf("column %s is not an array column", column)
}
+ if column == "important_kwd" {
+ if text, ok := item.(string); ok {
+ item, _ = sanitizeImportantKeyword(text)
+ }
+ }
setParts = append(setParts, fmt.Sprintf("%s = ARRAY_APPEND(%s, ?)", quoteIdentifier(column), quoteIdentifier(column)))
setArgs = append(setArgs, item)
}
diff --git a/rag/utils/ob_conn.py b/rag/utils/ob_conn.py
index ee4ea22259..24e9d6f400 100644
--- a/rag/utils/ob_conn.py
+++ b/rag/utils/ob_conn.py
@@ -45,6 +45,49 @@ from rag.nlp import rag_tokenizer
logger = logging.getLogger("ragflow.ob_conn")
+_IMPORTANT_KEYWORD_MAX_BYTES = 256
+
+
+def _sanitize_important_keywords(keywords: list[Any]) -> tuple[list[Any], bool]:
+ """Bound keyword array elements to the OceanBase VARCHAR byte limit."""
+ normalized: list[Any] = []
+ truncated = False
+ for keyword in keywords:
+ if not isinstance(keyword, str):
+ normalized.append(keyword)
+ continue
+
+ keyword = keyword.strip()
+ encoded = keyword.encode("utf-8")
+ if len(encoded) <= _IMPORTANT_KEYWORD_MAX_BYTES:
+ normalized.append(keyword)
+ continue
+
+ sanitized = encoded[:_IMPORTANT_KEYWORD_MAX_BYTES].decode("utf-8", errors="ignore").rstrip()
+ logger.warning(
+ "Sanitizing oversized OceanBase/SeekDB important keyword (%d bytes, stored %d bytes, limit %d)",
+ len(encoded),
+ len(sanitized.encode("utf-8")),
+ _IMPORTANT_KEYWORD_MAX_BYTES,
+ )
+ normalized.append(sanitized)
+ truncated = True
+ return normalized, truncated
+
+
+def _normalize_important_keyword_fields(fields: dict[str, Any]) -> dict[str, Any]:
+ keywords = fields.get("important_kwd")
+ if not isinstance(keywords, list):
+ return fields
+
+ normalized_keywords, truncated = _sanitize_important_keywords(keywords)
+ normalized_fields = dict(fields)
+ normalized_fields["important_kwd"] = normalized_keywords
+ if truncated:
+ normalized_fields["important_tks"] = rag_tokenizer.tokenize(" ".join(keyword for keyword in normalized_keywords if isinstance(keyword, str)))
+ return normalized_fields
+
+
column_order_id = Column("_order_id", Integer, nullable=True, comment="chunk order id for maintaining sequence")
column_chunk_order_int = Column("chunk_order_int", Integer, nullable=True, comment="chunk order index for retrieval sorting")
column_group_id = Column("group_id", String(256), nullable=True, comment="group id for external retrieval")
@@ -1039,6 +1082,7 @@ class OBConnection(OBConnectionBase):
docs: list[dict] = []
ids: list[str] = []
for document in documents:
+ document = _normalize_important_keyword_fields(document)
d: dict = {}
for k, v in document.items():
if vector_column_pattern.match(k):
@@ -1059,6 +1103,9 @@ class OBConnection(OBConnectionBase):
d[k] = json.dumps(v, ensure_ascii=False)
elif k == "position_int":
d[k] = json.dumps([list(vv) for vv in v], ensure_ascii=False)
+ elif k == "important_kwd" and isinstance(v, list):
+ # Let JSON encoding escape control characters without expanding the stored array elements.
+ d[k] = json.dumps(v, ensure_ascii=False)
elif isinstance(v, list):
# remove characters like '\t' for JSON dump and clean special characters
cleaned_v = []
@@ -1142,6 +1189,8 @@ class OBConnection(OBConnectionBase):
if not self._check_table_exists_cached(index_name):
return True
+ new_value = _normalize_important_keyword_fields(new_value)
+
# For doc_meta tables, don't force kb_id in condition
if not index_name.startswith("ragflow_doc_meta_"):
condition["kb_id"] = knowledgebase_id
@@ -1164,6 +1213,8 @@ class OBConnection(OBConnectionBase):
for kk, vv in v.items():
if kk not in array_columns:
raise ValueError(f"Column '{kk}' is not an array column.")
+ if kk == "important_kwd" and isinstance(vv, str):
+ vv = _sanitize_important_keywords([vv])[0][0]
set_values.append(f"{kk} = array_append({kk}, {get_value_str(vv)})")
elif k == "metadata":
if not isinstance(v, dict):
diff --git a/test/unit_test/rag/utils/test_ob_conn_keyword_limit.py b/test/unit_test/rag/utils/test_ob_conn_keyword_limit.py
new file mode 100644
index 0000000000..8297471ded
--- /dev/null
+++ b/test/unit_test/rag/utils/test_ob_conn_keyword_limit.py
@@ -0,0 +1,58 @@
+#
+# Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import json
+import logging
+
+from rag.utils import ob_conn
+
+
+def test_truncates_at_utf8_boundary_without_logging_content(caplog):
+ keyword = "sensitive-" + "中" * 100
+
+ with caplog.at_level(logging.WARNING, logger="ragflow.ob_conn"):
+ normalized, truncated = ob_conn._sanitize_important_keywords([keyword])
+
+ assert truncated is True
+ assert len(normalized[0].encode("utf-8")) <= ob_conn._IMPORTANT_KEYWORD_MAX_BYTES
+ assert normalized[0].encode("utf-8").decode("utf-8") == normalized[0]
+ assert caplog.records
+ assert keyword not in caplog.records[0].getMessage()
+ assert str(len(keyword.encode("utf-8"))) in caplog.records[0].getMessage()
+
+
+def test_recomputes_tokens_from_stored_keywords_without_mutating_input(monkeypatch):
+ keyword = "x" * (ob_conn._IMPORTANT_KEYWORD_MAX_BYTES + 1)
+ fields = {"important_kwd": [keyword], "important_tks": "stale tokens"}
+ monkeypatch.setattr(ob_conn.rag_tokenizer, "tokenize", lambda text: f"tokens:{text}")
+
+ normalized = ob_conn._normalize_important_keyword_fields(fields)
+
+ assert fields["important_kwd"] == [keyword]
+ assert fields["important_tks"] == "stale tokens"
+ assert normalized["important_kwd"] == ["x" * ob_conn._IMPORTANT_KEYWORD_MAX_BYTES]
+ assert normalized["important_tks"] == "tokens:" + "x" * ob_conn._IMPORTANT_KEYWORD_MAX_BYTES
+
+
+def test_json_escaping_does_not_change_stored_keyword_length():
+ keyword = "\\" * 200
+
+ normalized, truncated = ob_conn._sanitize_important_keywords([keyword])
+ stored = json.loads(json.dumps(normalized, ensure_ascii=False))[0]
+
+ assert truncated is False
+ assert stored == keyword
+ assert len(stored.encode("utf-8")) == 200