RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
Fix oversized important keywords in OceanBase and SeekDB (#18872)
wangyunlai1b5cb3caa0a3
diff --git a/internal/engine/oceanbase/codec.go b/internal/engine/oceanbase/codec.goindex f25c2d5284..e72fae0af8 100644--- a/internal/engine/oceanbase/codec.go+++ b/internal/engine/oceanbase/codec.go@@ -25,10 +25,17 @@ import ("sort""strconv""strings"+ "unicode"+ "unicode/utf8"+ "ragflow/internal/common""ragflow/internal/tokenizer"++ "go.uber.org/zap")+const importantKeywordMaxBytes = 256+var vectorColumnPattern = regexp.MustCompile(`^q_(\d+)_vec$`)var arrayColumns = map[string]bool{@@ -62,6 +69,7 @@ var memoryColumnToField = map[string]string{}func normalizeChunk(document map[string]interface{}) (map[string]interface{}, error) {+ document = normalizeImportantKeywordFields(document)result := make(map[string]interface{}, len(chunkColumns)+1)extra := make(map[string]interface{})if existing, ok := document["extra"].(map[string]interface{}); ok {@@ -258,6 +266,12 @@ func encodeColumnValue(columnName string, value interface{}) (interface{}, errorreturn string(encoded), err}}+ if columnName == "important_kwd" {+ normalized, _, _ := normalizeImportantKeywords(value)+ // JSON escaping must not expand the VARCHAR value stored in each array element.+ encoded, err := json.Marshal(normalized)+ return string(encoded), err+ }if arrayColumns[columnName] {return encodeArray(value)}@@ -271,6 +285,77 @@ func encodeColumnValue(columnName string, value interface{}) (interface{}, errorreturn value, nil}+func sanitizeImportantKeyword(keyword string) (string, bool) {+ keyword = strings.TrimSpace(keyword)+ if len(keyword) <= importantKeywordMaxBytes {+ return keyword, false+ }++ length := 0+ end := 0+ for _, character := range keyword {+ size := utf8.RuneLen(character)+ if length+size > importantKeywordMaxBytes {+ break+ }+ length += size+ end += size+ }+ truncated := strings.TrimRightFunc(keyword[:end], unicode.IsSpace)+ common.Warn(+ "Sanitizing oversized OceanBase/SeekDB important keyword",+ zap.Int("original_bytes", len(keyword)),+ zap.Int("stored_bytes", len(truncated)),+ zap.Int("limit_bytes", importantKeywordMaxBytes),+ )+ return truncated, true+}++func normalizeImportantKeywords(value interface{}) (interface{}, []string, bool) {+ values, ok := interfaceSlice(value)+ if !ok {+ return value, nil, false+ }++ normalized := make([]interface{}, len(values))+ textKeywords := make([]string, 0, len(values))+ truncated := false+ for i, item := range values {+ text, ok := item.(string)+ if !ok {+ normalized[i] = item+ continue+ }+ normalizedText, wasTruncated := sanitizeImportantKeyword(text)+ normalized[i] = normalizedText+ textKeywords = append(textKeywords, normalizedText)+ truncated = truncated || wasTruncated+ }+ return normalized, textKeywords, truncated+}++func normalizeImportantKeywordFields(fields map[string]interface{}) map[string]interface{} {+ keywords, ok := fields["important_kwd"]+ if !ok {+ return fields+ }++ normalizedKeywords, textKeywords, truncated := normalizeImportantKeywords(keywords)+ if !truncated {+ return fields+ }++ normalizedFields := copyMap(fields)+ normalizedFields["important_kwd"] = normalizedKeywords+ joinedKeywords := strings.Join(textKeywords, " ")+ tokenizedKeywords, err := tokenizer.Tokenize(joinedKeywords)+ if err != nil {+ tokenizedKeywords = joinedKeywords+ }+ normalizedFields["important_tks"] = tokenizedKeywords+ return normalizedFields+}+func encodeUpdateValue(kind, columnName string, value interface{}) (interface{}, error) {if vectorColumnPattern.MatchString(columnName) {return encodeVector(value)diff --git a/internal/engine/oceanbase/codec_test.go b/internal/engine/oceanbase/codec_test.gonew file mode 100644index 0000000000..3650dbc872--- /dev/null+++ b/internal/engine/oceanbase/codec_test.go@@ -0,0 +1,84 @@+//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+//++package oceanbase++import (+ "encoding/json"+ "strings"+ "testing"++ "ragflow/internal/tokenizer"+)++func TestNormalizeChunkBoundsImportantKeywordsAndRecomputesTokens(t *testing.T) {+ tokenizer.SetEngineType("infinity")+ defer tokenizer.SetEngineType("")++ asciiKeyword := strings.Repeat("x", importantKeywordMaxBytes+1)+ unicodeKeyword := strings.Repeat("中", 100)+ escapedKeyword := strings.Repeat(`\`, 200)+ document := map[string]interface{}{+ "id": "chunk-1",+ "kb_id": "kb-1",+ "important_kwd": []string{asciiKeyword, unicodeKeyword, escapedKeyword},+ "important_tks": "stale tokens",+ }++ normalized, err := normalizeChunk(document)+ if err != nil {+ t.Fatal(err)+ }++ var keywords []string+ if err := json.Unmarshal([]byte(normalized["important_kwd"].(string)), &keywords); err != nil {+ t.Fatal(err)+ }+ if len(keywords) != 3 {+ t.Fatalf("important_kwd = %#v, want three keywords", keywords)+ }+ for _, keyword := range keywords {+ if len(keyword) > importantKeywordMaxBytes {+ t.Errorf("important keyword byte length = %d, want <= %d", len(keyword), importantKeywordMaxBytes)+ }+ if strings.ToValidUTF8(keyword, "") != keyword {+ t.Errorf("important keyword is not valid UTF-8: %q", keyword)+ }+ }+ if normalized["important_tks"] != strings.Join(keywords, " ") {+ t.Errorf("important_tks = %q, want tokens from stored keywords", normalized["important_tks"])+ }+ if document["important_tks"] != "stale tokens" {+ t.Errorf("normalizeChunk mutated input important_tks: %q", document["important_tks"])+ }+}++func TestEncodeUpdateValueBoundsImportantKeyword(t *testing.T) {+ keyword := strings.Repeat("中", 100)++ encoded, err := encodeUpdateValue("chunk", "important_kwd", []string{keyword})+ if err != nil {+ t.Fatal(err)+ }++ var keywords []string+ if err := json.Unmarshal([]byte(encoded.(string)), &keywords); err != nil {+ t.Fatal(err)+ }+ if len(keywords) != 1 || len(keywords[0]) > importantKeywordMaxBytes {+ t.Fatalf("important_kwd = %#v, want one keyword bounded to %d bytes", keywords, importantKeywordMaxBytes)+ }+}diff --git a/internal/engine/oceanbase/crud.go b/internal/engine/oceanbase/crud.goindex c273b1a849..acd1ac5760 100644--- a/internal/engine/oceanbase/crud.go+++ b/internal/engine/oceanbase/crud.go@@ -201,6 +201,9 @@ func (e *Engine) UpdateChunks(ctx context.Context, condition, newValue map[strinif !exists {return fmt.Errorf("%w: '%s'", types.ErrIndexNotFound, baseName)}+ if kind == "chunk" {+ newValue = normalizeImportantKeywordFields(newValue)+ }whereSQL, args, err := buildFilter(condition, kind)if err != nil {return err@@ -248,6 +251,11 @@ func (e *Engine) UpdateChunks(ctx context.Context, condition, newValue map[strinif kind != "chunk" || !arrayColumns[column] {return fmt.Errorf("column %s is not an array column", column)}+ if column == "important_kwd" {+ if text, ok := item.(string); ok {+ item, _ = sanitizeImportantKeyword(text)+ }+ }setParts = append(setParts, fmt.Sprintf("%s = ARRAY_APPEND(%s, ?)", quoteIdentifier(column), quoteIdentifier(column)))setArgs = append(setArgs, item)}diff --git a/rag/utils/ob_conn.py b/rag/utils/ob_conn.pyindex ee4ea22259..24e9d6f400 100644--- a/rag/utils/ob_conn.py+++ b/rag/utils/ob_conn.py@@ -45,6 +45,49 @@ from rag.nlp import rag_tokenizerlogger = logging.getLogger("ragflow.ob_conn")+_IMPORTANT_KEYWORD_MAX_BYTES = 256+++def _sanitize_important_keywords(keywords: list[Any]) -> tuple[list[Any], bool]:+ """Bound keyword array elements to the OceanBase VARCHAR byte limit."""+ normalized: list[Any] = []+ truncated = False+ for keyword in keywords:+ if not isinstance(keyword, str):+ normalized.append(keyword)+ continue++ keyword = keyword.strip()+ encoded = keyword.encode("utf-8")+ if len(encoded) <= _IMPORTANT_KEYWORD_MAX_BYTES:+ normalized.append(keyword)+ continue++ sanitized = encoded[:_IMPORTANT_KEYWORD_MAX_BYTES].decode("utf-8", errors="ignore").rstrip()+ logger.warning(+ "Sanitizing oversized OceanBase/SeekDB important keyword (%d bytes, stored %d bytes, limit %d)",+ len(encoded),+ len(sanitized.encode("utf-8")),+ _IMPORTANT_KEYWORD_MAX_BYTES,+ )+ normalized.append(sanitized)+ truncated = True+ return normalized, truncated+++def _normalize_important_keyword_fields(fields: dict[str, Any]) -> dict[str, Any]:+ keywords = fields.get("important_kwd")+ if not isinstance(keywords, list):+ return fields++ normalized_keywords, truncated = _sanitize_important_keywords(keywords)+ normalized_fields = dict(fields)+ normalized_fields["important_kwd"] = normalized_keywords+ if truncated:+ normalized_fields["important_tks"] = rag_tokenizer.tokenize(" ".join(keyword for keyword in normalized_keywords if isinstance(keyword, str)))+ return normalized_fields++column_order_id = Column("_order_id", Integer, nullable=True, comment="chunk order id for maintaining sequence")column_chunk_order_int = Column("chunk_order_int", Integer, nullable=True, comment="chunk order index for retrieval sorting")column_group_id = Column("group_id", String(256), nullable=True, comment="group id for external retrieval")@@ -1039,6 +1082,7 @@ class OBConnection(OBConnectionBase):docs: list[dict] = []ids: list[str] = []for document in documents:+ document = _normalize_important_keyword_fields(document)d: dict = {}for k, v in document.items():if vector_column_pattern.match(k):@@ -1059,6 +1103,9 @@ class OBConnection(OBConnectionBase):d[k] = json.dumps(v, ensure_ascii=False)elif k == "position_int":d[k] = json.dumps([list(vv) for vv in v], ensure_ascii=False)+ elif k == "important_kwd" and isinstance(v, list):+ # Let JSON encoding escape control characters without expanding the stored array elements.+ d[k] = json.dumps(v, ensure_ascii=False)elif isinstance(v, list):# remove characters like '\t' for JSON dump and clean special characterscleaned_v = []@@ -1142,6 +1189,8 @@ class OBConnection(OBConnectionBase):if not self._check_table_exists_cached(index_name):return True+ new_value = _normalize_important_keyword_fields(new_value)+# For doc_meta tables, don't force kb_id in conditionif not index_name.startswith("ragflow_doc_meta_"):condition["kb_id"] = knowledgebase_id@@ -1164,6 +1213,8 @@ class OBConnection(OBConnectionBase):for kk, vv in v.items():if kk not in array_columns:raise ValueError(f"Column '{kk}' is not an array column.")+ if kk == "important_kwd" and isinstance(vv, str):+ vv = _sanitize_important_keywords([vv])[0][0]set_values.append(f"{kk} = array_append({kk}, {get_value_str(vv)})")elif k == "metadata":if not isinstance(v, dict):diff --git a/test/unit_test/rag/utils/test_ob_conn_keyword_limit.py b/test/unit_test/rag/utils/test_ob_conn_keyword_limit.pynew file mode 100644index 0000000000..8297471ded--- /dev/null+++ b/test/unit_test/rag/utils/test_ob_conn_keyword_limit.py@@ -0,0 +1,58 @@+#+# Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+#++import json+import logging++from rag.utils import ob_conn+++def test_truncates_at_utf8_boundary_without_logging_content(caplog):+ keyword = "sensitive-" + "中" * 100++ with caplog.at_level(logging.WARNING, logger="ragflow.ob_conn"):+ normalized, truncated = ob_conn._sanitize_important_keywords([keyword])++ assert truncated is True+ assert len(normalized[0].encode("utf-8")) <= ob_conn._IMPORTANT_KEYWORD_MAX_BYTES+ assert normalized[0].encode("utf-8").decode("utf-8") == normalized[0]+ assert caplog.records+ assert keyword not in caplog.records[0].getMessage()+ assert str(len(keyword.encode("utf-8"))) in caplog.records[0].getMessage()+++def test_recomputes_tokens_from_stored_keywords_without_mutating_input(monkeypatch):+ keyword = "x" * (ob_conn._IMPORTANT_KEYWORD_MAX_BYTES + 1)+ fields = {"important_kwd": [keyword], "important_tks": "stale tokens"}+ monkeypatch.setattr(ob_conn.rag_tokenizer, "tokenize", lambda text: f"tokens:{text}")++ normalized = ob_conn._normalize_important_keyword_fields(fields)++ assert fields["important_kwd"] == [keyword]+ assert fields["important_tks"] == "stale tokens"+ assert normalized["important_kwd"] == ["x" * ob_conn._IMPORTANT_KEYWORD_MAX_BYTES]+ assert normalized["important_tks"] == "tokens:" + "x" * ob_conn._IMPORTANT_KEYWORD_MAX_BYTES+++def test_json_escaping_does_not_change_stored_keyword_length():+ keyword = "\\" * 200++ normalized, truncated = ob_conn._sanitize_important_keywords([keyword])+ stored = json.loads(json.dumps(normalized, ensure_ascii=False))[0]++ assert truncated is False+ assert stored == keyword+ assert len(stored.encode("utf-8")) == 200