Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

feat: Gaussdb adaptation (#17703)

zws202606d02723ca7603
diff --git a/.github/workflows/sep-tests.yml b/.github/workflows/sep-tests.yml
index ffdf121ba5..01ec38fffb 100644
--- a/.github/workflows/sep-tests.yml
+++ b/.github/workflows/sep-tests.yml
@@ -623,7 +623,7 @@ jobs:
echo "SANDBOX_EXECUTOR_MANAGER_PORT=${SANDBOX_EXECUTOR_MANAGER_PORT}"
echo "SVR_WEB_HTTP_PORT=${SVR_WEB_HTTP_PORT}"
echo "SVR_WEB_HTTPS_PORT=${SVR_WEB_HTTPS_PORT}"
- echo "COMPOSE_PROFILES=${DOC_ENGINE},cpu,ragflow-go,tei-cpu,deepdoc"
+ echo "COMPOSE_PROFILES=${DOC_ENGINE},cpu,ragflow-go,tei-cpu,deepdoc,metadata-mysql"
echo "TEI_MODEL=BAAI/bge-small-en-v1.5"
echo "RAGFLOW_IMAGE=${RAGFLOW_IMAGE}"
echo "DOC_ENGINE=${DOC_ENGINE}"
@@ -1242,7 +1242,7 @@ jobs:
echo "SANDBOX_EXECUTOR_MANAGER_PORT=${SANDBOX_EXECUTOR_MANAGER_PORT}"
echo "SVR_WEB_HTTP_PORT=${SVR_WEB_HTTP_PORT}"
echo "SVR_WEB_HTTPS_PORT=${SVR_WEB_HTTPS_PORT}"
- echo "COMPOSE_PROFILES=${DOC_ENGINE},cpu,ragflow-go,tei-cpu,deepdoc"
+ echo "COMPOSE_PROFILES=${DOC_ENGINE},cpu,ragflow-go,tei-cpu,deepdoc,metadata-mysql"
echo "TEI_MODEL=BAAI/bge-small-en-v1.5"
echo "RAGFLOW_IMAGE=${RAGFLOW_IMAGE}"
echo "DOC_ENGINE=${DOC_ENGINE}"
diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
index 8ab708253e..cdc9c1ac66 100644
--- a/.github/workflows/tests.yml
+++ b/.github/workflows/tests.yml
@@ -393,7 +393,7 @@ jobs:
echo "SANDBOX_EXECUTOR_MANAGER_PORT=${SANDBOX_EXECUTOR_MANAGER_PORT}"
echo "SVR_WEB_HTTP_PORT=${SVR_WEB_HTTP_PORT}"
echo "SVR_WEB_HTTPS_PORT=${SVR_WEB_HTTPS_PORT}"
- echo "COMPOSE_PROFILES=${DOC_ENGINE},cpu,tei-cpu,deepdoc"
+ echo "COMPOSE_PROFILES=${DOC_ENGINE},cpu,tei-cpu,deepdoc,metadata-mysql"
echo "TEI_MODEL=BAAI/bge-small-en-v1.5"
echo "RAGFLOW_IMAGE=${RAGFLOW_IMAGE}"
echo "DOC_ENGINE=${DOC_ENGINE}"
@@ -872,7 +872,7 @@ jobs:
echo "SANDBOX_EXECUTOR_MANAGER_PORT=${SANDBOX_EXECUTOR_MANAGER_PORT}"
echo "SVR_WEB_HTTP_PORT=${SVR_WEB_HTTP_PORT}"
echo "SVR_WEB_HTTPS_PORT=${SVR_WEB_HTTPS_PORT}"
- echo "COMPOSE_PROFILES=${DOC_ENGINE},cpu,tei-cpu,deepdoc"
+ echo "COMPOSE_PROFILES=${DOC_ENGINE},cpu,tei-cpu,deepdoc,metadata-mysql"
echo "TEI_MODEL=BAAI/bge-small-en-v1.5"
echo "RAGFLOW_IMAGE=${RAGFLOW_IMAGE}"
echo "DOC_ENGINE=${DOC_ENGINE}"
diff --git a/admin/server/admin_server.py b/admin/server/admin_server.py
index 677d8a1c96..9e54a1cf21 100644
--- a/admin/server/admin_server.py
+++ b/admin/server/admin_server.py
@@ -36,6 +36,7 @@ from config import load_configurations, SERVICE_CONFIGS
from auth import init_default_admin, setup_auth
from flask_session import Session
from common.versions import get_ragflow_version
+from api.db.db_models import close_connection
stop_event = threading.Event()
@@ -60,6 +61,18 @@ if __name__ == "__main__":
show_configs()
login_manager = LoginManager()
login_manager.init_app(app)
+
+ @app.teardown_request
+ def _db_close(exception):
+ if settings.DATABASE_TYPE.lower() == "gaussdb":
+ if exception:
+ logging.error(
+ "Admin request failed: %s",
+ exception,
+ exc_info=(type(exception), exception, exception.__traceback__),
+ )
+ close_connection()
+
settings.init_settings()
setup_auth(login_manager)
init_default_admin()
diff --git a/admin/server/config.py b/admin/server/config.py
index eac8047974..c64ec98a84 100644
--- a/admin/server/config.py
+++ b/admin/server/config.py
@@ -16,6 +16,8 @@
import logging
+import os
+import re
import threading
from enum import Enum
@@ -46,6 +48,67 @@ class ServiceConfigs:
SERVICE_CONFIGS = ServiceConfigs
+# The Admin service list reads the same service_conf/local.service_conf files,
+# but the GaussDB metadata connection does not come from those files because
+# their top-level gaussdb section belongs to DocEngine and Memory Store. This
+# lightweight parser keeps Admin's metadata display aligned with the main
+# service DATABASE settings instead of showing an inactive mysql/postgres block.
+GAUSSDB_ENV_DEFAULTS = {
+ "name": "rag_flow",
+ "user": "rag_flow",
+ "password": "infini_rag_flow",
+ "host": "gaussdb",
+ "port": 8000,
+ "schema": "public",
+}
+_GAUSSDB_SCHEMA_PATTERN = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$")
+
+
+def normalize_database_type(database_type: str | None = None) -> str:
+ # Only normalize the new GaussDB values. Existing metadata database
+ # selection retains the upstream behavior.
+ raw_value = database_type or "mysql"
+ normalized = raw_value.strip().lower()
+ if normalized in {"gaussdb", "gauss"}:
+ return "gaussdb"
+ return raw_value
+
+
+def _get_int_env(name: str, default: int) -> int:
+ raw_value = os.environ.get(name)
+ if raw_value is None:
+ return default
+ try:
+ return int(raw_value)
+ except (TypeError, ValueError):
+ logging.warning("Ignoring invalid %s=%r; using default %d", name, raw_value, default)
+ return default
+
+
+def _normalize_gaussdb_metadata_schema(value: str | None = None) -> str:
+ # Admin only displays the schema and does not create the connection, but it
+ # must apply the same validation. Otherwise an invalid
+ # GAUSSDB_METADATA_SCHEMA could look valid in Admin while the main service
+ # fails during startup.
+ schema = (value or GAUSSDB_ENV_DEFAULTS["schema"]).strip() or GAUSSDB_ENV_DEFAULTS["schema"]
+ if not _GAUSSDB_SCHEMA_PATTERN.match(schema):
+ raise ValueError(f"invalid GAUSSDB_METADATA_SCHEMA: {schema}")
+ return schema
+
+
+def _gaussdb_env_config() -> dict[str, Any]:
+ # This configuration is only for Admin display and health checks; it does
+ # not create the application ORM connection. Keep the flat metadata fields
+ # aligned with common.settings._gaussdb_env_config() so both surfaces report
+ # the same connection information.
+ return {
+ "name": os.environ.get("GAUSSDB_METADATA_DBNAME", GAUSSDB_ENV_DEFAULTS["name"]),
+ "user": os.environ.get("GAUSSDB_METADATA_USER", GAUSSDB_ENV_DEFAULTS["user"]),
+ "password": os.environ.get("GAUSSDB_METADATA_PASSWORD", GAUSSDB_ENV_DEFAULTS["password"]),
+ "host": os.environ.get("GAUSSDB_METADATA_HOST", GAUSSDB_ENV_DEFAULTS["host"]),
+ "port": _get_int_env("GAUSSDB_METADATA_PORT", GAUSSDB_ENV_DEFAULTS["port"]),
+ "schema": _normalize_gaussdb_metadata_schema(os.environ.get("GAUSSDB_METADATA_SCHEMA")),
+ }
class ServiceType(Enum):
@@ -85,6 +148,16 @@ class MySQLConfig(MetaConfig):
return result
+class GaussDBMetadataConfig(MySQLConfig):
+ metadata_schema: str
+
+ def to_dict(self) -> dict[str, Any]:
+ result = super().to_dict()
+ result["extra"]["schema"] = self.metadata_schema
+ result["extra"]["password"] = "*" * 8
+ return result
+
+
class PostgresConfig(MetaConfig):
def to_dict(self) -> dict[str, Any]:
result = super().to_dict()
@@ -106,6 +179,18 @@ class RetrievalConfig(BaseConfig):
return result
+class GaussDBRetrievalConfig(RetrievalConfig):
+ database: str
+ retrieval_schema: str
+
+ def to_dict(self) -> dict[str, Any]:
+ result = super().to_dict()
+ extra_dict = result["extra"].copy()
+ extra_dict.update({"database": self.database, "schema": self.retrieval_schema})
+ result["extra"] = extra_dict
+ return result
+
+
class InfinityConfig(RetrievalConfig):
db_name: str
@@ -222,6 +307,7 @@ def load_configurations(config_path: str) -> list[BaseConfig]:
configurations = []
ragflow_count = 0
id_count = 0
+ metadata_db_type = normalize_database_type(os.getenv("DB_TYPE", "mysql"))
for k, v in raw_configs.items():
match k:
case "ragflow":
@@ -341,13 +427,55 @@ def load_configurations(config_path: str) -> list[BaseConfig]:
configurations.append(config)
id_count += 1
case "mysql":
+ if metadata_db_type == "gaussdb":
+ continue
name: str = "mysql"
host: str = v.get("host")
port: int = v.get("port")
username = v.get("user")
password = v.get("password")
config = MySQLConfig(
- id=id_count, name=name, host=host, port=port, username=username, password=password, service_type="meta_data", meta_type="mysql", detail_func_name="get_mysql_status"
+ id=id_count,
+ name=name,
+ host=host,
+ port=port,
+ username=username,
+ password=password,
+ service_type="meta_data",
+ meta_type="mysql",
+ detail_func_name="get_mysql_status",
+ )
+ configurations.append(config)
+ id_count += 1
+ case "gaussdb":
+ # The gaussdb block in the configuration file belongs to
+ # DocEngine and Memory Store, not the DB_TYPE=gaussdb metadata
+ # connection. Add metadata GaussDB separately from
+ # GAUSSDB_METADATA_* after the loop so Admin does not merge the
+ # two services.
+ doc_config = v
+ host = doc_config.get("host")
+ port = doc_config.get("port")
+ if not host or port in (None, ""):
+ # service_conf.yaml.template always contains a GaussDB
+ # placeholder block. Default deployments leave it empty;
+ # do not let an inactive DocEngine config break Admin.
+ continue
+ try:
+ port = int(port)
+ except (TypeError, ValueError):
+ logging.warning("Ignoring invalid GaussDB DocEngine port in Admin configuration")
+ continue
+ config = GaussDBRetrievalConfig(
+ id=id_count,
+ name="gaussdb",
+ host=host,
+ port=port,
+ database=doc_config.get("database"),
+ retrieval_schema=doc_config.get("schema") or "public",
+ service_type="retrieval",
+ retrieval_type="gaussdb",
+ detail_func_name="get_gaussdb_status",
)
configurations.append(config)
id_count += 1
@@ -374,4 +502,23 @@ def load_configurations(config_path: str) -> list[BaseConfig]:
logging.warning(f"Unknown configuration key: {k}")
continue
+ if metadata_db_type == "gaussdb":
+ # Display the metadata database from the same GAUSSDB_METADATA_*
+ # variables as the main service, never from the DocEngine/Memory Store
+ # GAUSSDB_* variables.
+ v = _gaussdb_env_config()
+ config = GaussDBMetadataConfig(
+ id=id_count,
+ name="gaussdb",
+ host=v.get("host"),
+ port=v.get("port"),
+ username=v.get("user"),
+ password=v.get("password"),
+ metadata_schema=v.get("schema"),
+ service_type="meta_data",
+ meta_type="gaussdb",
+ detail_func_name="get_database_status",
+ )
+ configurations.append(config)
+
return configurations
diff --git a/api/apps/restful_apis/system_api.py b/api/apps/restful_apis/system_api.py
index 4e587035f8..c3473cf53c 100644
--- a/api/apps/restful_apis/system_api.py
+++ b/api/apps/restful_apis/system_api.py
@@ -23,13 +23,14 @@ from quart import jsonify
from api.apps import login_required, current_user
from api.utils.api_utils import get_json_result, get_data_error_result, server_error_response, generate_confirmation_token
-from api.utils.health_utils import run_health_checks, get_oceanbase_status
+from api.utils.health_utils import run_health_checks, get_oceanbase_status, get_gaussdb_status
from common.versions import get_ragflow_version
from common.time_utils import current_timestamp, datetime_format
from api.db.db_models import APIToken
from api.db.services.api_service import APITokenService
from api.db.services.knowledgebase_service import KnowledgebaseService
from api.db.services.user_service import UserTenantService
+from common.doc_store.gaussdb_conn_base import mask_gaussdb_text
from common.log_utils import get_log_levels, set_log_level
from common import settings
from rag.utils.redis_conn import REDIS_CONN
@@ -208,6 +209,38 @@ def oceanbase_status():
return get_json_result(data={"status": "error", "message": f"Failed to get OceanBase status: {str(e)}"}, code=500)
[email protected]("/system/gaussdb/status", methods=["GET"]) # noqa: F821
+@login_required
+def gaussdb_status():
+ """
+ Get GaussDB health status and performance metrics.
+ ---
+ tags:
+ - System
+ security:
+ - ApiKeyAuth: []
+ responses:
+ 200:
+ description: GaussDB status retrieved successfully.
+ schema:
+ type: object
+ properties:
+ status:
+ type: string
+ description: Status (alive/timeout/not_configured).
+ message:
+ type: object
+ description: Detailed status information including health and performance metrics.
+ """
+ try:
+ status_info = get_gaussdb_status()
+ return get_json_result(data=status_info)
+ except Exception as e:
+ masked_error = mask_gaussdb_text(e)
+ logging.error("GaussDB status route failed (%s): %s", type(e).__name__, masked_error)
+ return get_json_result(data={"status": "error", "message": f"Failed to get GaussDB status: {masked_error}"}, code=500)
+
+
@manager.route("/system/config", methods=["GET"]) # noqa: F821
def get_config():
"""
diff --git a/api/apps/services/dataset_api_service.py b/api/apps/services/dataset_api_service.py
index f71b973bc3..fdceb392c0 100644
--- a/api/apps/services/dataset_api_service.py
+++ b/api/apps/services/dataset_api_service.py
@@ -788,7 +788,9 @@ def delete_tags(dataset_id: str, tenant_id: str, tags: list[str]):
from rag.nlp import search
for t in tags:
- settings.docStoreConn.update({"tag_kwd": t, "kb_id": [dataset_id]}, {"remove": {"tag_kwd": t}}, search.index_name(kb.tenant_id), dataset_id)
+ updated = settings.docStoreConn.update({"tag_kwd": t, "kb_id": [dataset_id]}, {"remove": {"tag_kwd": t}}, search.index_name(kb.tenant_id), dataset_id)
+ if callable(getattr(settings.docStoreConn, "db_type", None)) and settings.docStoreConn.db_type() == "gaussdb" and not updated:
+ return False, "Failed to update dataset tags in document store"
return True, {}
@@ -991,7 +993,11 @@ def rename_tag(dataset_id: str, tenant_id: str, from_tag: str, to_tag: str):
from rag.nlp import search
- settings.docStoreConn.update({"tag_kwd": from_tag, "kb_id": [dataset_id]}, {"remove": {"tag_kwd": from_tag.strip()}, "add": {"tag_kwd": to_tag}}, search.index_name(kb.tenant_id), dataset_id)
+ updated = settings.docStoreConn.update(
+ {"tag_kwd": from_tag, "kb_id": [dataset_id]}, {"remove": {"tag_kwd": from_tag.strip()}, "add": {"tag_kwd": to_tag}}, search.index_name(kb.tenant_id), dataset_id
+ )
+ if callable(getattr(settings.docStoreConn, "db_type", None)) and settings.docStoreConn.db_type() == "gaussdb" and not updated:
+ return False, "Failed to update dataset tags in document store"
return True, {"from": from_tag, "to": to_tag}
@@ -1261,7 +1267,11 @@ def check_embedding(dataset_id: str, tenant_id: str, req: dict):
cid = ids[0]
full_doc = docStoreConn.get(cid, index_nm, [kb_id]) or {}
vec_field = _guess_vec_field(full_doc)
- vec = _as_float_vec(full_doc.get(vec_field))
+ vec_valid = full_doc.get(f"{vec_field}_valid") if vec_field else None
+ if callable(getattr(docStoreConn, "db_type", None)) and docStoreConn.db_type() == "gaussdb" and vec_valid is False:
+ vec = []
+ else:
+ vec = _as_float_vec(full_doc.get(vec_field))
out.append(
{
diff --git a/api/db/db_models.py b/api/db/db_models.py
index 3867612210..04009562fc 100644
--- a/api/db/db_models.py
+++ b/api/db/db_models.py
@@ -27,6 +27,7 @@ from functools import wraps
from quart_auth import AuthUser
from itsdangerous.url_safe import URLSafeTimedSerializer as Serializer
+from psycopg2 import sql as psycopg2_sql
from peewee import (
fn,
InterfaceError,
@@ -42,6 +43,7 @@ from peewee import (
IntegerField,
Metadata,
Model,
+ ModelInsert,
TextField,
)
from playhouse.migrate import MySQLMigrator, PostgresqlMigrator, migrate
@@ -49,6 +51,13 @@ from playhouse.pool import PooledMySQLDatabase, PooledPostgresqlDatabase
from api import utils
from api.db import SerializedType
+from api.db.gaussdb_error_utils import (
+ is_duplicate_column_error,
+ is_duplicate_object_error,
+ is_psycopg_connection_error,
+ is_undefined_object_error,
+ sqlstate_from_exception,
+)
from api.utils.json_encode import json_dumps, json_loads
from api.utils.configs import deserialize_b64, serialize_b64
@@ -60,18 +69,96 @@ from common import settings
CONTINUOUS_FIELD_TYPE = {IntegerField, FloatField, DateTimeField}
AUTO_DATE_TIMESTAMP_FIELD_PREFIX = {"create", "start", "end", "update", "read_access", "write_access"}
+GAUSSDB_COMPATIBLE_DATABASE_TYPES = {"GAUSSDB"}
+
+
+def is_gaussdb_compatible_database() -> bool:
+ return settings.DATABASE_TYPE.upper() in GAUSSDB_COMPATIBLE_DATABASE_TYPES
class TextFieldType(Enum):
MYSQL = "LONGTEXT"
OCEANBASE = "LONGTEXT"
POSTGRES = "TEXT"
+ # GaussDB does not support MySQL LONGTEXT semantics. Map long text to TEXT
+ # for JSONField and LongTextField while retaining a distinct enum entry so
+ # DB_TYPE=gaussdb is not treated as DB_TYPE=postgres.
+ GAUSSDB = "TEXT"
class LongTextField(TextField):
field_type = TextFieldType[settings.DATABASE_TYPE.upper()].value
+class EmptyStringFieldMixin:
+ """Preserve application-level empty-string semantics on GaussDB.
+
+ A/ORA-compatible GaussDB treats empty strings in SQL literals and bound
+ parameters as NULL. Some RAGFlow fields use an empty string to mean "not
+ configured" and can remain NOT NULL with a default of "" on MySQL and
+ PostgreSQL. On GaussDB, those values would violate a NOT NULL constraint.
+
+ Apply this field only to explicitly selected columns. Do not perform a
+ global NULL-to-empty-string conversion in DB.execute_sql or the cursor
+ layer, because NULL remains meaningful for other fields.
+ """
+
+ __hash__ = Field.__hash__
+
+ def __init__(self, *args, **kwargs):
+ if is_gaussdb_compatible_database():
+ # These fields represent application-level empty strings as NULL
+ # in GaussDB. Their columns must therefore be nullable to avoid a
+ # NOT NULL violation. Other databases retain their original
+ # constraints.
+ kwargs["null"] = True
+ super().__init__(*args, **kwargs)
+
+ def db_value(self, value):
+ if is_gaussdb_compatible_database() and value == "":
+ # Normalize the application-level empty string explicitly so the
+ # storage semantics remain stable even if a test or user database
+ # stops converting empty strings to NULL automatically.
+ return None
+ return super().db_value(value)
+
+ def python_value(self, value):
+ if is_gaussdb_compatible_database() and value is None:
+ # Preserve the existing application contract without scattering
+ # `is None` branches throughout the business code.
+ return ""
+ return super().python_value(value)
+
+ def __eq__(self, rhs):
+ if is_gaussdb_compatible_database() and isinstance(rhs, str) and rhs == "":
+ # On A/ORA-compatible GaussDB, `field = ''` becomes `field = NULL`
+ # and never matches. Express the application-level empty value as
+ # `IS NULL OR LENGTH(field) = 0` to match both normalized NULLs and
+ # any historical empty strings.
+ return self.is_null(True) | (fn.LENGTH(self) == 0)
+ return super().__eq__(rhs)
+
+ def __ne__(self, rhs):
+ if is_gaussdb_compatible_database() and isinstance(rhs, str) and rhs == "":
+ # Symmetrically, a value unequal to the application-level empty
+ # string must contain data. LENGTH(field) > 0 avoids comparing with
+ # NULL and excludes both NULL and historical empty strings.
+ return fn.LENGTH(self) > 0
+ return super().__ne__(rhs)
+
+
+class EmptyStringCharField(EmptyStringFieldMixin, CharField):
+ pass
+
+
+class EmptyStringTextField(EmptyStringFieldMixin, TextField):
+ pass
+
+
+class EmptyStringLongTextField(EmptyStringFieldMixin, LongTextField):
+ pass
+
+
class JSONField(LongTextField):
default_value = {}
@@ -317,6 +404,80 @@ class RetryingPooledMySQLDatabase(PooledMySQLDatabase):
return None
+class GaussDBPsycopgRetryMixin:
+ """Connection retry behavior used only by the GaussDB adapter."""
+
+ database_display_name = "GaussDB"
+
+ def __init__(self, *args, **kwargs):
+ self.max_retries = kwargs.pop("max_retries", 5)
+ self.retry_delay = kwargs.pop("retry_delay", 1)
+ super().__init__(*args, **kwargs)
+
+ def _prepare_sql_for_execution(self, sql):
+ return sql
+
+ def execute_sql(self, sql, params=None, commit=True):
+ for attempt in range(self.max_retries + 1):
+ try:
+ prepared_sql = self._prepare_sql_for_execution(sql)
+ return super().execute_sql(prepared_sql, params, commit)
+ except (OperationalError, InterfaceError) as e:
+ # A reconnect cannot restore an active transaction. Let the
+ # atomic block fail as a unit instead of retrying one statement
+ # on a different connection.
+ if self.in_transaction():
+ logging.error(f"{self.database_display_name} execution failure: {e}")
+ raise
+
+ # Peewee may hide the driver SQLSTATE in `__context__`. Use the
+ # shared classifier to recognize 08xxx/57P0x connection states
+ # first and fall back to observed GaussDB disconnect text.
+ should_retry = is_psycopg_connection_error(e)
+
+ if should_retry and attempt < self.max_retries:
+ logging.warning(f"{self.database_display_name} connection issue (attempt {attempt + 1}/{self.max_retries}): {e}")
+ self._handle_connection_loss()
+ time.sleep(self.retry_delay * (2**attempt))
+ else:
+ logging.error(f"{self.database_display_name} execution failure: {e}")
+ raise
+ return None
+
+ def _handle_connection_loss(self):
+ try:
+ self.close()
+ except Exception:
+ pass
+ try:
+ self.connect()
+ except Exception as e:
+ logging.error(f"Failed to reconnect to {self.database_display_name}: {e}")
+ time.sleep(0.1)
+ try:
+ self.connect()
+ except Exception as e2:
+ logging.error(f"Failed to reconnect to {self.database_display_name} on second attempt: {e2}")
+ raise
+
+ def begin(self):
+ for attempt in range(self.max_retries + 1):
+ try:
+ return super().begin()
+ except (OperationalError, InterfaceError) as e:
+ # Apply the same connection-error classification during
+ # transaction startup and statement execution.
+ should_retry = is_psycopg_connection_error(e)
+
+ if should_retry and attempt < self.max_retries:
+ logging.warning(f"{self.database_display_name} connection lost during transaction (attempt {attempt + 1}/{self.max_retries})")
+ self._handle_connection_loss()
+ time.sleep(self.retry_delay * (2**attempt))
+ else:
+ raise
+ return None
+
+
class RetryingPooledPostgresqlDatabase(PooledPostgresqlDatabase):
def __init__(self, *args, **kwargs):
self.max_retries = kwargs.pop("max_retries", 5)
@@ -382,6 +543,184 @@ class RetryingPooledPostgresqlDatabase(PooledPostgresqlDatabase):
return None
+class RetryingPooledGaussDBDatabase(GaussDBPsycopgRetryMixin, PooledPostgresqlDatabase):
+ """Connection pool for the GaussDB business metadata database.
+
+ GaussDB exposes a psycopg/libpq-compatible protocol, so Peewee can use
+ PooledPostgresqlDatabase as its transport. The adapter remains distinct so
+ DB_TYPE=gaussdb dialect differences, compatibility checks, logging, and
+ diagnostics stay in this class or GaussDBMigrator instead of aliasing
+ GaussDB to PostgreSQL.
+ """
+
+ database_display_name = "GaussDB"
+ # These tables have both an id primary key and a business unique key that
+ # does not contain id. Distributed HASH tables cannot enforce both sets of
+ # constraints globally, so preserve the existing schema with replication.
+ distributed_replication_tables = frozenset(
+ {
+ "compilation_template",
+ "compilation_template_group",
+ "file_commit_item",
+ "tenant_llm",
+ "tenant_model_provider",
+ "user",
+ }
+ )
+
+ def __init__(self, *args, **kwargs):
+ self._is_distributed = None
+ # Force UTF-8 so metadata containing non-ASCII text, JSON, or template
+ # data does not depend on the connection's default encoding.
+ kwargs.setdefault("options", "-c client_encoding=UTF8")
+ super().__init__(*args, **kwargs)
+
+ def _initialize_connection(self, conn):
+ """Initialize a checked-out connection and detect its deployment type."""
+ super()._initialize_connection(conn)
+
+ with conn.cursor() as cursor:
+ cursor.execute("SELECT current_schema()")
+ row = cursor.fetchone()
+ schema = row[0] if row else None
+ if not schema:
+ raise OperationalError("GaussDB metadata search_path has no existing schema")
+ # SET is repeated after connection startup because a distributed CN
+ # may not propagate the libpq search_path option to every DN.
+ cursor.execute(psycopg2_sql.SQL("SET search_path TO {}").format(psycopg2_sql.Identifier(schema)))
+
+ try:
+ cursor.execute("SELECT EXISTS (SELECT 1 FROM pg_catalog.pgxc_node WHERE node_type = 'D')")
+ topology_row = cursor.fetchone()
+ is_distributed = bool(topology_row and topology_row[0])
+ except Exception as exc:
+ # Centralized editions may not expose pgxc_node at all, or may
+ # reject querying it as an unsupported feature.
+ if sqlstate_from_exception(exc) not in {"0A000", "42P01"}:
+ raise
+ rollback = getattr(conn, "rollback", None)
+ if callable(rollback):
+ rollback()
+ # A failed catalog probe can abort the startup transaction and
+ # roll back the preceding SET. Restore the configured schema so
+ # the centralized connection remains immediately usable.
+ cursor.execute(psycopg2_sql.SQL("SET search_path TO {}").format(psycopg2_sql.Identifier(schema)))
+ is_distributed = False
+
+ if self._is_distributed is not None and self._is_distributed != is_distributed:
+ raise RuntimeError("GaussDB deployment topology changed across pooled connections")
+ if self._is_distributed is None:
+ logging.info(
+ "Detected %s GaussDB metadata deployment",
+ "distributed" if is_distributed else "centralized",
+ )
+ self._is_distributed = is_distributed
+
+ @property
+ def is_distributed(self):
+ if self._is_distributed is None:
+ raise RuntimeError("GaussDB deployment topology is unavailable before connection initialization")
+ return self._is_distributed
+
+ def _ensure_topology(self):
+ if self._is_distributed is None:
+ # This goes through GaussDBPsycopgRetryMixin so a transient first-connect
+ # failure gets the same retry treatment as any other SQL execution.
+ self.execute_sql("SELECT 1")
+
+ def prepare_update_by_id(self, model, pid, data):
+ """Remove only redundant primary-key assignments on distributed GaussDB."""
+ self._ensure_topology()
+ if not self.is_distributed:
+ return data
+
+ primary_key = model._meta.primary_key
+ if isinstance(primary_key, CompositeKey) or primary_key.name not in data:
+ return data
+
+ field_name = primary_key.name
+ locator_value = primary_key.db_value(pid)
+ update_value = primary_key.db_value(data[field_name])
+ if locator_value != update_value:
+ return data
+
+ prepared = data.copy()
+ del prepared[field_name]
+ return prepared
+
+ def replace_update_by_id(self, model, pid, data):
+ """Replace a row when distributed GaussDB must change its primary key.
+
+ A distributed table commonly uses its primary key as the distribution
+ key, which GaussDB does not allow an UPDATE statement to modify. Keep
+ the generic UPDATE path for centralized GaussDB and for updates that do
+ not really change the key. A real key change is performed atomically as
+ delete plus insert while preserving every existing column.
+
+ Return ``None`` when the caller should execute its normal UPDATE, or
+ the affected-row count when this method handled the replacement.
+ """
+ self._ensure_topology()
+ if not self.is_distributed:
+ return None
+
+ primary_key = model._meta.primary_key
+ if isinstance(primary_key, CompositeKey) or primary_key.name not in data:
+ return None
+
+ field_name = primary_key.name
+ locator_value = primary_key.db_value(pid)
+ update_value = primary_key.db_value(data[field_name])
+ if locator_value == update_value:
+ return None
+
+ with self.atomic():
+ record = model.get_or_none(primary_key == pid)
+ if record is None:
+ return 0
+
+ replacement = record.__data__.copy()
+ replacement.update(data)
+ deleted = model.delete().where(primary_key == pid).execute()
+ if deleted != 1:
+ return deleted
+ # Bypass BaseModel.insert(), which refreshes create_time. Replacing
+ # a distribution key must retain the original row's creation data.
+ ModelInsert(model, replacement).execute()
+ return deleted
+
+ def _prepare_distributed_ddl(self, sql):
+ if not self.is_distributed or not isinstance(sql, str):
+ return sql
+
+ statement = sql.rstrip()
+ trailing_whitespace = sql[len(statement) :]
+ terminator = ";" if statement.endswith(";") else ""
+ if terminator:
+ statement = statement[:-1].rstrip()
+
+ for table_name in self.distributed_replication_tables:
+ quoted_name = f'"{table_name}"'
+ if statement.startswith(f"CREATE TABLE IF NOT EXISTS {quoted_name} ") or statement.startswith(f"CREATE TABLE {quoted_name} "):
+ return f"{statement} DISTRIBUTE BY REPLICATION{terminator}{trailing_whitespace}"
+ return sql
+
+ def _prepare_sql_for_execution(self, sql):
+ # Topology initialization intentionally runs inside the retry loop in
+ # GaussDBPsycopgRetryMixin. This also ensures the first CREATE TABLE receives
+ # its distributed table clause before Peewee executes it.
+ if self._is_distributed is None:
+ self.connection()
+ return self._prepare_distributed_ddl(sql)
+
+ def get_tables(self, schema=None):
+ if schema is not None:
+ return super().get_tables(schema)
+
+ cursor = self.execute_sql("SELECT tablename FROM pg_catalog.pg_tables WHERE schemaname = current_schema() ORDER BY tablename")
+ return [table for (table,) in cursor.fetchall()]
+
+
class RetryingPooledOceanBaseDatabase(PooledMySQLDatabase):
"""Pooled OceanBase database with retry mechanism.
@@ -459,12 +798,26 @@ class PooledDatabase(Enum):
MYSQL = RetryingPooledMySQLDatabase
OCEANBASE = RetryingPooledOceanBaseDatabase
POSTGRES = RetryingPooledPostgresqlDatabase
+ # Keep GaussDB distinct instead of rewriting settings.DATABASE_TYPE to
+ # postgres so logs, health checks, and Admin report the real database type.
+ GAUSSDB = RetryingPooledGaussDBDatabase
+
+
+class GaussDBMigrator(PostgresqlMigrator):
+ # Peewee has no built-in GaussDB migrator. Its standard ADD COLUMN, ALTER
+ # COLUMN, and ADD INDEX statements are compatible with those generated by
+ # PostgresqlMigrator. Keep a separate class so GaussDB-specific DDL can be
+ # added without changing DB_TYPE=postgres.
+ pass
class DatabaseMigrator(Enum):
MYSQL = MySQLMigrator
OCEANBASE = MySQLMigrator
POSTGRES = PostgresqlMigrator
+ # Route DB_TYPE=gaussdb through its own migrator entry even though the
+ # current DDL implementation derives from PostgresqlMigrator.
+ GAUSSDB = GaussDBMigrator
@singleton
@@ -477,6 +830,9 @@ class BaseDataBase:
"max_retries": 5,
"retry_delay": 1,
}
+ # Retry settings are runtime behavior shared by the database adapters,
+ # not database configuration fields. GaussDBPsycopgRetryMixin uses
+ # SQLSTATE to decide whether an operation is retryable.
database_config.update(pool_config)
self.database_connection = PooledDatabase[settings.DATABASE_TYPE.upper()].value(db_name, **database_config)
# self.database_connection = PooledDatabase[settings.DATABASE_TYPE.upper()].value(db_name, **database_config)
@@ -571,6 +927,44 @@ class PostgresDatabaseLock:
return magic
+class GaussDBDatabaseLock(PostgresDatabaseLock):
+ # GaussDB supports pg_try_advisory_lock/pg_advisory_unlock, so its current
+ # lock SQL matches PostgreSQL. Keep a distinct class so future lock APIs,
+ # timeout behavior, or error handling remain isolated.
+ database_label = "gaussdb"
+ poll_interval = 0.1
+
+ def __init__(self, lock_name, timeout=10, db=None):
+ super().__init__(lock_name, timeout=timeout, db=db)
+ self.timeout = max(float(timeout), 0.0)
+
+ def lock(self):
+ deadline = time.monotonic() + self.timeout
+ while True:
+ cursor = self.db.execute_sql("SELECT pg_try_advisory_lock(%s)", (self.lock_id,))
+ row = cursor.fetchone()
+ value = row[0] if row else None
+ if value in (1, True):
+ return True
+ if value not in (0, False):
+ raise Exception(f"failed to acquire lock {self.lock_name}")
+
+ remaining = deadline - time.monotonic()
+ if remaining <= 0:
+ raise Exception(f"acquire {self.database_label} lock {self.lock_name} timeout")
+ time.sleep(min(self.poll_interval, remaining))
+
+ @with_retry(max_retries=3, retry_delay=1.0)
+ def unlock(self):
+ cursor = self.db.execute_sql("SELECT pg_advisory_unlock(%s)", (self.lock_id,))
+ ret = cursor.fetchone()
+ if ret[0] == 0:
+ raise Exception(f"gaussdb lock {self.lock_name} was not established by this thread")
+ if ret[0] == 1:
+ return True
+ raise Exception(f"gaussdb lock {self.lock_name} does not exist")
+
+
class MysqlDatabaseLock:
def __init__(self, lock_name, timeout=10, db=None):
self.lock_name = lock_name
@@ -622,6 +1016,10 @@ class DatabaseLock(Enum):
MYSQL = MysqlDatabaseLock
OCEANBASE = MysqlDatabaseLock
POSTGRES = PostgresDatabaseLock
+ # Initialization and migrations require cross-process exclusion. GaussDB
+ # uses advisory locks instead of MySQL GET_LOCK/RELEASE_LOCK while retaining
+ # a distinct enum entry from PostgreSQL.
+ GAUSSDB = GaussDBDatabaseLock
DB = BaseDataBase().database_connection
@@ -631,7 +1029,12 @@ DB.lock = DatabaseLock[settings.DATABASE_TYPE.upper()].value
def close_connection():
try:
if DB:
- DB.close_stale(age=30)
+ if settings.DATABASE_TYPE.upper() in GAUSSDB_COMPATIBLE_DATABASE_TYPES:
+ if not DB.is_closed():
+ # Return this worker thread's connection to the GaussDB pool.
+ DB.close()
+ else:
+ DB.close_stale(age=30)
except Exception as e:
logging.exception(e)
@@ -681,7 +1084,10 @@ class User(DataBaseModel, AuthUser):
id = CharField(max_length=32, primary_key=True)
access_token = CharField(max_length=255, null=True, index=True)
- nickname = CharField(max_length=100, null=False, help_text="nicky name", index=True)
+ # A/ORA-compatible GaussDB stores the application-level empty string as
+ # NULL. Admin retains nickname="" while the field restores that value when
+ # reading from storage.
+ nickname = EmptyStringCharField(max_length=100, null=False, help_text="nicky name", index=True)
password = CharField(max_length=255, null=True, help_text="password", index=True)
email = CharField(max_length=255, null=False, help_text="email", unique=True)
avatar = TextField(null=True, help_text="avatar base64 string")
@@ -723,15 +1129,18 @@ class Tenant(DataBaseModel):
id = CharField(max_length=32, primary_key=True)
name = CharField(max_length=100, null=True, help_text="Tenant name", index=True)
public_key = CharField(max_length=255, null=True, index=True)
- llm_id = CharField(max_length=128, null=False, help_text="default llm ID", index=True)
+ # These default model IDs historically use "" for an unconfigured value.
+ # EmptyStringCharField maps GaussDB storage NULL back to that application
+ # value.
+ llm_id = EmptyStringCharField(max_length=128, null=False, help_text="default llm ID", index=True)
tenant_llm_id = CharField(max_length=32, null=True, help_text="id in tenant_model", index=True)
- embd_id = CharField(max_length=128, null=False, help_text="default embedding model ID", index=True)
+ embd_id = EmptyStringCharField(max_length=128, null=False, help_text="default embedding model ID", index=True)
tenant_embd_id = CharField(max_length=32, null=True, help_text="id in tenant_model", index=True)
- asr_id = CharField(max_length=128, null=False, help_text="default ASR model ID", index=True)
+ asr_id = EmptyStringCharField(max_length=128, null=False, help_text="default ASR model ID", index=True)
tenant_asr_id = CharField(max_length=32, null=True, help_text="id in tenant_model", index=True)
- img2txt_id = CharField(max_length=128, null=False, help_text="default image to text model ID", index=True)
+ img2txt_id = EmptyStringCharField(max_length=128, null=False, help_text="default image to text model ID", index=True)
tenant_img2txt_id = CharField(max_length=32, null=True, help_text="id in tenant_model", index=True)
- rerank_id = CharField(max_length=128, null=False, help_text="default rerank model ID", index=True)
+ rerank_id = EmptyStringCharField(max_length=128, null=False, help_text="default rerank model ID", index=True)
tenant_rerank_id = CharField(max_length=32, null=True, help_text="id in tenant_model", index=True)
tts_id = CharField(max_length=256, null=True, help_text="default tts model ID", index=True)
tenant_tts_id = CharField(max_length=32, null=True, help_text="id in tenant_model", index=True)
@@ -841,7 +1250,9 @@ class Knowledgebase(DataBaseModel):
name = CharField(max_length=128, null=False, help_text="KB name", index=True)
language = CharField(max_length=32, null=True, default="Chinese" if "zh_CN" in os.getenv("LANG", "") else "English", help_text="English|Chinese", index=True)
description = TextField(null=True, help_text="KB description")
- embd_id = CharField(max_length=128, null=False, help_text="default embedding model ID", index=True)
+ # A dataset may inherit an application-level empty string. GaussDB stores it
+ # as NULL and restores it to "" when read.
+ embd_id = EmptyStringCharField(max_length=128, null=False, help_text="default embedding model ID", index=True)
tenant_embd_id = CharField(max_length=32, null=True, help_text="id in tenant_model", index=True)
permission = CharField(max_length=16, null=False, help_text="me|team", default="me", index=True)
created_by = CharField(max_length=32, null=False, index=True)
@@ -930,7 +1341,9 @@ class File(DataBaseModel):
location = CharField(max_length=255, null=True, help_text="where dose it store", index=True)
size = BigIntegerField(default=0, index=True)
type = CharField(max_length=32, null=False, help_text="file extension", index=True)
- source_type = CharField(max_length=128, null=False, default="", help_text="where dose this document come from", index=True)
+ # File.source_type historically defaults to "". Only GaussDB makes the
+ # storage column nullable; MySQL and PostgreSQL retain NOT NULL.
+ source_type = EmptyStringCharField(max_length=128, null=False, default="", help_text="where dose this document come from", index=True)
class Meta:
db_table = "file"
@@ -1004,17 +1417,25 @@ class Task(DataBaseModel):
doc_id = CharField(max_length=32, null=False, index=True)
from_page = IntegerField(default=0)
to_page = IntegerField(default=MAXIMUM_TASK_PAGE_NUMBER)
- task_type = CharField(max_length=32, null=False, default="")
+ # Standard document parsing tasks historically use task_type="" to mean
+ # that they are not dataflow, graph, memory, or another specialized task.
+ # A/ORA-compatible GaussDB stores the bound empty string as NULL, so a NOT
+ # NULL column would make /documents/parse fail while inserting the task.
+ # Only DB_TYPE=gaussdb stores NULL, and the ORM restores "" when reading.
+ task_type = EmptyStringCharField(max_length=32, null=False, default="")
priority = IntegerField(default=0)
begin_at = DateTimeField(null=True, index=True)
process_duration = FloatField(default=0)
progress = FloatField(default=0, index=True)
- progress_msg = TextField(null=True, help_text="process message", default="")
+ # progress_msg and chunk_ids are nullable, but callers commonly concatenate,
+ # strip, or split them as strings. The field maps GaussDB NULL to "" without
+ # affecting MySQL or PostgreSQL.
+ progress_msg = EmptyStringTextField(null=True, help_text="process message", default="")
retry_count = IntegerField(default=0)
digest = TextField(null=True, help_text="task digest", default="")
- chunk_ids = LongTextField(null=True, help_text="chunk ids", default="")
+ chunk_ids = EmptyStringLongTextField(null=True, help_text="chunk ids", default="")
class Dialog(DataBaseModel):
@@ -1024,7 +1445,9 @@ class Dialog(DataBaseModel):
description = TextField(null=True, help_text="Dialog description")
icon = TextField(null=True, help_text="icon base64 string")
language = CharField(max_length=32, null=True, default="Chinese" if "zh_CN" in os.getenv("LANG", "") else "English", help_text="English|Chinese", index=True)
- llm_id = CharField(max_length=128, null=False, help_text="default llm ID")
+ # Map application-level empty chat/reranker model IDs to storage NULL in the
+ # field instead of handling None throughout the business code.
+ llm_id = EmptyStringCharField(max_length=128, null=False, help_text="default llm ID")
tenant_llm_id = CharField(max_length=32, null=True, help_text="id in tenant_model", index=True)
llm_setting = JSONField(null=False, default={"temperature": 0.1, "top_p": 0.3, "frequency_penalty": 0.7, "presence_penalty": 0.4, "max_tokens": 512})
@@ -1044,7 +1467,7 @@ class Dialog(DataBaseModel):
do_refer = CharField(max_length=1, null=False, default="1", help_text="it needs to insert reference index into answer or not")
- rerank_id = CharField(max_length=128, null=False, help_text="default rerank model ID")
+ rerank_id = EmptyStringCharField(max_length=128, null=False, help_text="default rerank model ID")
tenant_rerank_id = CharField(max_length=32, null=True, help_text="id in tenant_model", index=True)
kb_ids = JSONField(null=False, default=[])
status = CharField(max_length=1, null=True, help_text="is it validate(0: wasted, 1: validate)", default="1", index=True)
@@ -1081,7 +1504,7 @@ class API4Conversation(DataBaseModel):
id = CharField(max_length=32, primary_key=True)
name = CharField(max_length=255, null=True, help_text="conversation name", index=False)
dialog_id = CharField(max_length=32, null=False, index=True)
- user_id = CharField(max_length=255, null=False, help_text="user_id", index=True)
+ user_id = EmptyStringCharField(max_length=255, null=False, help_text="user_id", index=True)
exp_user_id = CharField(max_length=255, null=True, help_text="exp_user_id", index=True)
message = JSONField(null=True)
reference = JSONField(null=True, default=[])
@@ -1109,7 +1532,13 @@ class UserCanvas(DataBaseModel):
description = TextField(null=True, help_text="Canvas description")
canvas_type = CharField(max_length=32, null=True, help_text="Canvas type", index=True)
canvas_category = CharField(max_length=32, null=False, default="agent_canvas", help_text="Canvas category: agent_canvas|dataflow_canvas", index=True)
- tags = CharField(max_length=512, null=False, default="", help_text="Comma-separated tags for organizing agents", index=True)
+ tags = EmptyStringCharField(
+ max_length=512,
+ null=False,
+ default="",
+ help_text="Comma-separated tags for organizing agents",
+ index=True,
+ )
dsl = JSONField(null=True, default={})
class Meta:
@@ -1345,9 +1774,15 @@ class SyncLogs(DataBaseModel):
new_docs_indexed = IntegerField(default=0, index=False)
total_docs_indexed = IntegerField(default=0, index=False)
docs_removed_from_index = IntegerField(default=0, index=False)
- error_msg = TextField(null=False, help_text="process message", default="")
+ # Connector scheduling logs use error_msg="" to mean that no error is
+ # present. A/ORA-compatible GaussDB stores it as NULL, so keeping this column
+ # NOT NULL would break PUT /datasets/<id> when it schedules sync_logs. The
+ # field permits storage NULL and restores the application-level empty string.
+ error_msg = EmptyStringTextField(null=False, help_text="process message", default="")
error_count = IntegerField(default=0, index=False)
- full_exception_trace = TextField(null=True, help_text="process message", default="")
+ # full_exception_trace is nullable, but callers append stack traces as text.
+ # Hide GaussDB storage NULL behind the same application-level empty string.
+ full_exception_trace = EmptyStringTextField(null=True, help_text="process message", default="")
time_started = DateTimeField(null=True, index=True)
poll_range_start = DateTimeTzField(max_length=255, null=True, index=True)
poll_range_end = DateTimeTzField(max_length=255, null=True, index=True)
@@ -1364,9 +1799,11 @@ class Memory(DataBaseModel):
tenant_id = CharField(max_length=32, null=False, index=True)
memory_type = IntegerField(null=False, default=1, index=True, help_text="Bit flags (LSB->MSB): 1=raw, 2=semantic, 4=episodic, 8=procedural. E.g., 5 enables raw + episodic.")
storage_type = CharField(max_length=32, default="table", null=False, index=True, help_text="table|graph")
- embd_id = CharField(max_length=128, null=False, index=False, help_text="embedding model ID")
+ # Keep empty model-ID handling in the field so storage NULL semantics do not
+ # leak into Memory Store callers.
+ embd_id = EmptyStringCharField(max_length=128, null=False, index=False, help_text="embedding model ID")
tenant_embd_id = CharField(max_length=32, null=True, help_text="id in tenant_model", index=True)
- llm_id = CharField(max_length=128, null=False, index=False, help_text="chat model ID")
+ llm_id = EmptyStringCharField(max_length=128, null=False, index=False, help_text="chat model ID")
tenant_llm_id = CharField(max_length=32, null=True, help_text="id in tenant_model", index=True)
permissions = CharField(max_length=16, null=False, index=True, help_text="me|team", default="me")
description = TextField(null=True, help_text="description")
@@ -1384,7 +1821,10 @@ class SystemSettings(DataBaseModel):
name = CharField(max_length=128, primary_key=True)
source = CharField(max_length=32, null=False, index=False)
data_type = CharField(max_length=32, null=False, index=False)
- value = TextField(null=False, help_text="Configuration value (JSON, string, etc.)")
+ # system_settings.json contains empty configuration values. Store them as
+ # NULL on GaussDB and restore "" here instead of globally converting SQL
+ # results and affecting fields where NULL is meaningful.
+ value = EmptyStringTextField(null=False, help_text="Configuration value (JSON, string, etc.)")
class Meta:
db_table = "system_settings"
@@ -1448,7 +1888,63 @@ class TenantModelGroupMapping(DataBaseModel):
primary_key = CompositeKey("group_id", "provider_id", "instance_id", "model_id")
+GAUSSDB_EMPTY_STRING_COMPATIBLE_COLUMNS = (
+ ("user", ("nickname",)),
+ ("tenant", ("llm_id", "embd_id", "asr_id", "img2txt_id", "rerank_id")),
+ ("knowledgebase", ("embd_id",)),
+ ("dialog", ("llm_id", "rerank_id")),
+ ("memory", ("embd_id", "llm_id")),
+ ("file", ("source_type",)),
+ ("system_settings", ("value",)),
+ ("task", ("task_type",)),
+ ("sync_logs", ("error_msg", "full_exception_trace")),
+ ("api_4_conversation", ("user_id",)),
+ ("user_canvas", ("tags",)),
+)
+
+
+def _quote_identifier_for_gaussdb(identifier: str) -> str:
+ return '"' + identifier.replace('"', '""') + '"'
+
+
+def relax_gaussdb_empty_string_compatible_columns():
+ if not is_gaussdb_compatible_database():
+ return
+
+ # These fields retain their historical application-level empty-string
+ # semantics, but A/ORA-compatible GaussDB stores "" as NULL. Existing
+ # databases must therefore make only these listed columns nullable. MySQL,
+ # PostgreSQL, and unlisted fields remain unchanged.
+ for table_name, column_names in GAUSSDB_EMPTY_STRING_COMPATIBLE_COLUMNS:
+ quoted_table = _quote_identifier_for_gaussdb(table_name)
+ for column_name in column_names:
+ quoted_column = _quote_identifier_for_gaussdb(column_name)
+ try:
+ DB.execute_sql(f"ALTER TABLE {quoted_table} ALTER COLUMN {quoted_column} DROP NOT NULL")
+ except Exception as ex:
+ # Match the migration framework's behavior for failed column
+ # additions or type changes: log at critical and continue
+ # startup. This avoids aborting on a missing historical column
+ # while identifying the constraint that was not relaxed.
+ logging.critical(
+ "Failed to relax GaussDB empty-string compatible column %s.%s: %s",
+ table_name,
+ column_name,
+ ex,
+ )
+
+
def alter_db_add_column(migrator, table_name, column_name, column_type):
+ if is_gaussdb_compatible_database():
+ try:
+ migrate(migrator.add_column(table_name, column_name, column_type))
+ except (OperationalError, ProgrammingError) as ex:
+ if not is_duplicate_column_error(ex):
+ logging.critical(f"Failed to add {settings.DATABASE_TYPE.upper()}.{table_name} column {column_name}, operation error: {ex}")
+ except Exception as ex:
+ logging.critical(f"Failed to add {settings.DATABASE_TYPE.upper()}.{table_name} column {column_name}, error: {ex}")
+ return
+
try:
migrate(migrator.add_column(table_name, column_name, column_type))
except OperationalError as ex:
@@ -1528,11 +2024,38 @@ def ensure_model_indexes(migrator):
logging.error(f"Failed to create {'unique ' if unique else ''}index on {table_name} ({', '.join(columns)}): {ex}")
+def _gaussdb_user_email_unique_index_exists() -> bool:
+ cursor = DB.execute_sql(
+ """
+ SELECT COUNT(*)
+ FROM pg_indexes
+ WHERE schemaname = current_schema()
+ AND tablename = 'user'
+ AND lower(indexdef) LIKE %s
+ AND (
+ lower(indexdef) LIKE %s
+ OR lower(indexdef) LIKE %s
+ )
+ """,
+ ("create unique index%", "%(email)%", '%("email")%'),
+ )
+ result = cursor.fetchone()
+ return bool(result and result[0] > 0)
+
+
def migrate_add_unique_email(migrator):
"""Deduplicates user emails and add UNIQUE constraint to email column (idempotent)"""
# step 0: check existing index state on user.email and prepare for unique constraint
try:
- if settings.DATABASE_TYPE.upper() == "POSTGRES":
+ if is_gaussdb_compatible_database():
+ # GaussDB cannot use MySQL information_schema.statistics or
+ # backtick syntax. Query pg_indexes through a distinct GaussDB path
+ # so future catalog changes remain isolated in
+ # _gaussdb_user_email_unique_index_exists.
+ if _gaussdb_user_email_unique_index_exists():
+ logging.info("UNIQUE index on user.email already exists, skipping migration")
+ return
+ elif settings.DATABASE_TYPE.upper() == "POSTGRES":
cursor = DB.execute_sql("""
SELECT COUNT(*)
FROM pg_indexes
@@ -1586,8 +2109,12 @@ def migrate_add_unique_email(migrator):
migrate(migrator.add_index("user", ("email",), unique=True))
except (OperationalError, ProgrammingError) as ex:
msg = str(ex)
- # MySQL 1061 "Duplicate key name" or PostgreSQL "already exists" -> already migrated
- if "1061" in msg or "Duplicate key name" in msg or "already exists" in msg.lower():
+ if is_gaussdb_compatible_database():
+ already_exists = is_duplicate_object_error(ex)
+ else:
+ # Preserve the upstream MySQL/PostgreSQL handling.
+ already_exists = "1061" in msg or "Duplicate key name" in msg or "already exists" in msg.lower()
+ if already_exists:
pass
else:
logging.critical("Failed to add UNIQUE constraint on user.email: %s", ex)
@@ -1597,7 +2124,12 @@ def migrate_add_unique_email(migrator):
def update_tenant_llm_to_id_primary_key():
"""Add ID and set to primary key step by step."""
- if settings.DATABASE_TYPE.upper() == "POSTGRES":
+ if is_gaussdb_compatible_database():
+ # Use the GaussDB implementation for sequences, catalogs, and constraint
+ # DDL instead of MySQL AUTO_INCREMENT/INFORMATION_SCHEMA or a branch
+ # labeled as PostgreSQL.
+ _update_tenant_llm_to_id_primary_key_gaussdb()
+ elif settings.DATABASE_TYPE.upper() == "POSTGRES":
_update_tenant_llm_to_id_primary_key_postgres()
else:
_update_tenant_llm_to_id_primary_key_mysql()
@@ -1734,12 +2266,113 @@ def _update_tenant_llm_to_id_primary_key_postgres():
DB.execute_sql("ALTER TABLE tenant_llm DROP COLUMN temp_id")
+def _update_tenant_llm_to_id_primary_key_gaussdb():
+ """GaussDB-compatible implementation: add sequence-backed primary key."""
+ # GaussDB can perform this migration with pg_attribute, pg_constraint, and
+ # a sequence. Keep the entry point distinct so catalog, ctid, or sequence
+ # differences can be handled without changing the PostgreSQL path.
+ _update_tenant_llm_to_id_primary_key_gaussdb_catalog()
+
+
+def _update_tenant_llm_to_id_primary_key_gaussdb_catalog():
+ """GaussDB pg_catalog implementation for the tenant_llm migration."""
+ try:
+ with DB.atomic():
+ # 0. Check if 'id' column already exists
+ # information_schema rowcount is unreliable on GaussDB. Query
+ # pg_attribute and use fetchone() to prevent a duplicate migration.
+ cursor = DB.execute_sql("""
+ SELECT a.attname
+ FROM pg_attribute a
+ JOIN pg_class c ON c.oid = a.attrelid
+ JOIN pg_namespace n ON n.oid = c.relnamespace
+ WHERE n.nspname = current_schema()
+ AND c.relname = 'tenant_llm'
+ AND a.attname = 'id'
+ AND NOT a.attisdropped
+ """)
+ if cursor.fetchone():
+ return
+
+ # 1. Add nullable integer column
+ DB.execute_sql("ALTER TABLE tenant_llm ADD COLUMN temp_id INTEGER NULL")
+
+ # 2. Assign sequential row numbers ordered consistently
+ DB.execute_sql("""
+ UPDATE tenant_llm
+ SET temp_id = subq.rn
+ FROM (
+ SELECT ctid,
+ ROW_NUMBER() OVER (ORDER BY tenant_id, llm_factory, llm_name) AS rn
+ FROM tenant_llm
+ ) AS subq
+ WHERE tenant_llm.ctid = subq.ctid
+ """)
+
+ # 3. Drop old composite primary key constraint
+ # Older tenant_llm tables use (tenant_id, llm_factory, llm_name) as
+ # a composite primary key. The new schema needs an id primary key
+ # and a composite unique constraint. Read the actual constraint name
+ # from pg_constraint instead of assuming a fixed name.
+ cursor = DB.execute_sql("""
+ SELECT con.conname
+ FROM pg_constraint con
+ JOIN pg_class c ON c.oid = con.conrelid
+ JOIN pg_namespace n ON n.oid = c.relnamespace
+ WHERE n.nspname = current_schema()
+ AND c.relname = 'tenant_llm'
+ AND con.contype = 'p'
+ """)
+ row = cursor.fetchone()
+ if row:
+ DB.execute_sql(f'ALTER TABLE tenant_llm DROP CONSTRAINT "{row[0]}"')
+
+ # 4. Make temp_id NOT NULL and create a sequence for it
+ # GaussDB and PostgreSQL do not use MySQL AUTO_INCREMENT here. A
+ # sequence with nextval() provides the generated key, and OWNED BY
+ # keeps its lifecycle tied to the column after temp_id becomes id.
+ DB.execute_sql("ALTER TABLE tenant_llm ALTER COLUMN temp_id SET NOT NULL")
+ DB.execute_sql("CREATE SEQUENCE IF NOT EXISTS tenant_llm_id_seq")
+ DB.execute_sql("""
+ SELECT setval('tenant_llm_id_seq', COALESCE((SELECT MAX(temp_id) FROM tenant_llm), 0))
+ """)
+ DB.execute_sql("ALTER TABLE tenant_llm ALTER COLUMN temp_id SET DEFAULT nextval('tenant_llm_id_seq')")
+ DB.execute_sql("ALTER SEQUENCE tenant_llm_id_seq OWNED BY tenant_llm.temp_id")
+ DB.execute_sql("ALTER TABLE tenant_llm ADD PRIMARY KEY (temp_id)")
+
+ # 5. Add unique constraint
+ DB.execute_sql("""
+ ALTER TABLE tenant_llm
+ ADD CONSTRAINT uk_tenant_llm UNIQUE (tenant_id, llm_factory, llm_name)
+ """)
+
+ # 6. Rename temp_id to id
+ DB.execute_sql("ALTER TABLE tenant_llm RENAME COLUMN temp_id TO id")
+
+ logging.info("Successfully updated tenant_llm to id primary key (GaussDB).")
+
+ except Exception as e:
+ logging.error(str(e))
+ cursor = DB.execute_sql("""
+ SELECT a.attname
+ FROM pg_attribute a
+ JOIN pg_class c ON c.oid = a.attrelid
+ JOIN pg_namespace n ON n.oid = c.relnamespace
+ WHERE n.nspname = current_schema()
+ AND c.relname = 'tenant_llm'
+ AND a.attname = 'temp_id'
+ AND NOT a.attisdropped
+ """)
+ if cursor.fetchone():
+ DB.execute_sql("ALTER TABLE tenant_llm DROP COLUMN temp_id")
+
+
def migrate_db():
logging.disable(logging.ERROR)
migrator = DatabaseMigrator[settings.DATABASE_TYPE.upper()].value(DB)
- alter_db_add_column(migrator, "file", "source_type", CharField(max_length=128, null=False, default="", help_text="where dose this document come from", index=True))
- alter_db_add_column(migrator, "tenant", "rerank_id", CharField(max_length=128, null=False, default="BAAI/bge-reranker-v2-m3", help_text="default rerank model ID"))
- alter_db_add_column(migrator, "dialog", "rerank_id", CharField(max_length=128, null=False, default="", help_text="default rerank model ID"))
+ alter_db_add_column(migrator, "file", "source_type", EmptyStringCharField(max_length=128, null=False, default="", help_text="where dose this document come from", index=True))
+ alter_db_add_column(migrator, "tenant", "rerank_id", EmptyStringCharField(max_length=128, null=False, default="BAAI/bge-reranker-v2-m3", help_text="default rerank model ID"))
+ alter_db_add_column(migrator, "dialog", "rerank_id", EmptyStringCharField(max_length=128, null=False, default="", help_text="default rerank model ID"))
alter_db_column_type(migrator, "dialog", "top_k", IntegerField(default=1024))
alter_db_add_column(migrator, "tenant_llm", "api_key", CharField(max_length=2048, null=True, help_text="API KEY", index=True))
alter_db_add_column(migrator, "api_token", "source", CharField(max_length=16, null=True, help_text="none|agent|dialog", index=True))
@@ -1752,9 +2385,9 @@ def migrate_db():
alter_db_add_column(migrator, "knowledgebase", "pagerank", IntegerField(default=0, index=False))
alter_db_add_column(migrator, "api_token", "beta", CharField(max_length=255, null=True, index=True))
alter_db_add_column(migrator, "task", "digest", TextField(null=True, help_text="task digest", default=""))
- alter_db_add_column(migrator, "task", "chunk_ids", LongTextField(null=True, help_text="chunk ids", default=""))
+ alter_db_add_column(migrator, "task", "chunk_ids", EmptyStringLongTextField(null=True, help_text="chunk ids", default=""))
alter_db_add_column(migrator, "conversation", "user_id", CharField(max_length=255, null=True, help_text="user_id", index=True))
- alter_db_add_column(migrator, "task", "task_type", CharField(max_length=32, null=False, default=""))
+ alter_db_add_column(migrator, "task", "task_type", EmptyStringCharField(max_length=32, null=False, default=""))
alter_db_add_column(migrator, "task", "priority", IntegerField(default=0))
alter_db_add_column(migrator, "user_canvas", "permission", CharField(max_length=16, null=False, help_text="me|team", default="me", index=True))
alter_db_add_column(migrator, "user_canvas", "release", BooleanField(null=False, help_text="is released", default=False, index=True))
@@ -1795,11 +2428,24 @@ def migrate_db():
alter_db_add_column(migrator, "api_4_conversation", "name", CharField(max_length=255, null=True, help_text="conversation name", index=False))
alter_db_add_column(migrator, "api_4_conversation", "exp_user_id", CharField(max_length=255, null=True, help_text="exp_user_id", index=True))
alter_db_add_column(migrator, "sync_logs", "task_type", CharField(max_length=32, null=False, default="sync", index=True))
- # Migrate system_settings.value from CharField to TextField for longer sandbox configs
- alter_db_column_type(migrator, "system_settings", "value", TextField(null=False, help_text="Configuration value (JSON, string, etc.)"))
+ # Migrate system_settings.value from CharField to TextField for longer sandbox configs.
+ # Use the dedicated field so the GaussDB column remains nullable after the
+ # type change and the ORM restores NULL to the application-level "".
+ alter_db_column_type(migrator, "system_settings", "value", EmptyStringTextField(null=False, help_text="Configuration value (JSON, string, etc.)"))
alter_db_add_column(migrator, "document", "content_hash", CharField(max_length=32, null=True, help_text="xxhash128 of document content for change detection", default="", index=True))
alter_db_add_column(migrator, "user_canvas_version", "release", BooleanField(null=False, help_text="is released", default=False, index=True))
- alter_db_add_column(migrator, "user_canvas", "tags", CharField(max_length=512, null=False, default="", help_text="Comma-separated tags for organizing agents", index=True))
+ alter_db_add_column(
+ migrator,
+ "user_canvas",
+ "tags",
+ EmptyStringCharField(
+ max_length=512,
+ null=False,
+ default="",
+ help_text="Comma-separated tags for organizing agents",
+ index=True,
+ ),
+ )
alter_db_add_column(migrator, "api_4_conversation", "version_title", CharField(max_length=255, null=True, help_text="canvas version title when session created", index=False))
alter_db_column_type(migrator, "document", "size", BigIntegerField(default=0, index=True))
alter_db_column_type(migrator, "file", "size", BigIntegerField(default=0, index=True))
@@ -1817,6 +2463,9 @@ def migrate_db():
alter_db_drop_index(migrator, "tenant_langfuse", "idx_tenant_langfuse_secret_key")
alter_db_drop_index(migrator, "tenant_langfuse", "idx_tenant_langfuse_public_key")
alter_db_drop_index(migrator, "tenant_langfuse", "idx_tenant_langfuse_host")
+ # Run after all alter_db_* calls so newly added compatible columns, such as
+ # user_canvas.tags, exist before their GaussDB NOT NULL constraints relax.
+ relax_gaussdb_empty_string_compatible_columns()
# Drop both the explicit "idx_*" name from later migrations AND the
# Peewee-auto-derived "<table-as-classname>_<col1>_<col2>" name from the
@@ -1842,7 +2491,11 @@ def migrate_db():
migrate(migrator.drop_index(table_name, index_name))
except (OperationalError, ProgrammingError) as ex:
msg = str(ex)
- if "1091" in msg or "can't DROP" in msg.lower() or "does not exist" in msg.lower() or "already exists" in msg.lower():
+ if is_gaussdb_compatible_database():
+ can_skip = is_undefined_object_error(ex) or is_duplicate_object_error(ex)
+ else:
+ can_skip = "1091" in msg or "can't DROP" in msg.lower() or "does not exist" in msg.lower() or "already exists" in msg.lower()
+ if can_skip:
pass
else:
logging.critical(f"Failed to drop index {index_name} on {table_name}: {ex}")
diff --git a/api/db/db_utils.py b/api/db/db_utils.py
index 98e44f8d53..2500156acf 100644
--- a/api/db/db_utils.py
+++ b/api/db/db_utils.py
@@ -20,7 +20,47 @@ from playhouse.pool import PooledMySQLDatabase
from common.time_utils import current_timestamp, timestamp_to_date
-from api.db.db_models import DB, DataBaseModel
+from api.db.db_models import DB, DataBaseModel, is_gaussdb_compatible_database
+
+
+def _gaussdb_replace_insert_by_id(model, batch, preserve):
+ """Replace Peewee ``ON CONFLICT`` upserts on A/ORA-compatible GaussDB.
+
+ Peewee's PostgreSQL dialect renders
+ ``insert_many(...).on_conflict(conflict_target="id")`` as
+ ``INSERT ... ON CONFLICT (...) DO UPDATE ... RETURNING``. A/ORA-compatible
+ GaussDB rejects that syntax, which breaks APIs such as
+ ``/datasets/{id}/documents/parse`` when they batch-write tasks.
+
+ For ``DB_TYPE=gaussdb`` only, split this replace-by-id operation into two
+ steps:
+
+ * Query the IDs already present in the batch and update those rows using
+ the same columns that Peewee's PostgreSQL path would preserve.
+ * Insert rows with new IDs without generating unsupported ``ON CONFLICT``
+ SQL.
+
+ MySQL retains its on-duplicate-key behavior, and PostgreSQL/OceanBase retain
+ Peewee's existing generation path. SQL for other databases is unchanged.
+ """
+ ids = [data.get("id") for data in batch if data.get("id") is not None]
+ existing_ids = set()
+ if ids:
+ existing_ids = {row[0] for row in model.select(model.id).where(model.id.in_(ids)).tuples()}
+
+ insert_rows = []
+ update_columns = [column for column in preserve if column != "id"]
+ for data in batch:
+ row_id = data.get("id")
+ if row_id in existing_ids:
+ update_payload = {column: data[column] for column in update_columns if column in data}
+ if update_payload:
+ model.update(update_payload).where(model.id == row_id).execute()
+ else:
+ insert_rows.append(data)
+
+ if insert_rows:
+ model.insert_many(insert_rows).execute()
@DB.connection_context()
@@ -46,6 +86,13 @@ def bulk_insert_into_db(model, data_source, replace_on_conflict=False):
if replace_on_conflict:
if isinstance(DB, PooledMySQLDatabase):
query = query.on_conflict(preserve=preserve)
+ elif is_gaussdb_compatible_database():
+ # Peewee's PostgreSQL path emits `ON CONFLICT (...) DO
+ # UPDATE ... RETURNING`, which A/ORA-compatible GaussDB
+ # rejects. Query existing IDs first, then UPDATE or INSERT
+ # to preserve the operation without leaking PostgreSQL SQL.
+ _gaussdb_replace_insert_by_id(model, data_source[i : i + batch_size], preserve)
+ continue
else:
query = query.on_conflict(conflict_target="id", preserve=preserve)
query.execute()
diff --git a/api/db/gaussdb_error_utils.py b/api/db/gaussdb_error_utils.py
new file mode 100644
index 0000000000..d42d51c26a
--- /dev/null
+++ b/api/db/gaussdb_error_utils.py
@@ -0,0 +1,174 @@
+#
+# Copyright 2024 The InfiniFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+"""GaussDB metadata adapter exception classification helpers.
+
+This module deliberately avoids importing ``db_models`` or service classes so
+the GaussDB pool, migrations, and retry decorator do not form import cycles.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Iterable
+
+
+# Connection-related SQLSTATE values reported by GaussDB/PostgreSQL/libpq.
+# GaussDB exposes errors through psycopg2/libpq, so connection failures and
+# server rejections should be classified by SQLSTATE instead of English error
+# text. Text matching is only a fallback when the driver omits pgcode. The
+# 08xxx class covers connection exceptions, while 57P0x covers shutdowns and
+# temporarily unavailable servers.
+PSYCOPG_CONNECTION_SQLSTATES = {
+ "08000", # connection_exception
+ "08001", # sqlclient_unable_to_establish_sqlconnection
+ "08003", # connection_does_not_exist
+ "08004", # sqlserver_rejected_establishment_of_sqlconnection
+ "08006", # connection_failure
+ "08007", # transaction_resolution_unknown
+ "08P01", # protocol_violation
+ "57P01", # admin_shutdown
+ "57P02", # crash_shutdown
+ "57P03", # cannot_connect_now
+}
+
+# Transaction-conflict SQLSTATE values observed on GaussDB:
+# 40P01: deadlock detected;
+# 40001: could not serialize access due to concurrent update;
+# 55P03: FOR UPDATE NOWAIT could not acquire the row lock.
+# Each indicates a statement or transaction that can be retried as a complete
+# RAGFlow database operation.
+PSYCOPG_RETRYABLE_TRANSACTION_SQLSTATES = {"40P01", "40001", "55P03"}
+
+# SQLSTATE values used to make DDL migrations idempotent. Migration code must
+# distinguish an already-applied change from a real failure. GaussDB reports
+# duplicate columns as 42701, duplicate table/index names as 42P07, and missing
+# index/constraint objects as 42704.
+PSYCOPG_DUPLICATE_COLUMN_SQLSTATES = {"42701"}
+PSYCOPG_DUPLICATE_OBJECT_SQLSTATES = {"42P07"}
+PSYCOPG_UNDEFINED_OBJECT_SQLSTATES = {"42704"}
+
+CONNECTION_ERROR_MESSAGES = (
+ "server closed",
+ "connection refused",
+ "no connection to the server",
+ "terminating connection",
+ "ssl connection has been closed unexpectedly",
+ "ssl syscall error: eof detected",
+)
+
+
+def _iter_exception_chain(exc: BaseException) -> Iterable[BaseException | object]:
+ """Iterate through Peewee wrappers, chained errors, and driver errors in args."""
+ seen: set[int] = set()
+ stack: list[BaseException | object] = [exc]
+ while stack:
+ current = stack.pop(0)
+ current_id = id(current)
+ if current_id in seen:
+ continue
+ seen.add(current_id)
+ yield current
+
+ for attr in ("__cause__", "__context__", "orig", "original_exception"):
+ nested = getattr(current, attr, None)
+ if nested is not None:
+ stack.append(nested)
+
+ for arg in getattr(current, "args", ()) or ():
+ if hasattr(arg, "pgcode"):
+ stack.append(arg)
+
+
+def sqlstate_from_exception(exc: BaseException) -> str | None:
+ """Extract SQLSTATE from a psycopg2 error or a Peewee-wrapped error."""
+ for current in _iter_exception_chain(exc):
+ pgcode = getattr(current, "pgcode", None)
+ if pgcode:
+ return str(pgcode)
+
+ diag = getattr(current, "diag", None)
+ sqlstate = getattr(diag, "sqlstate", None) if diag else None
+ if sqlstate:
+ return str(sqlstate)
+
+ return None
+
+
+def exception_text(exc: BaseException) -> str:
+ return str(exc).lower()
+
+
+def mysql_errno_from_exception(exc: BaseException) -> int | None:
+ args = getattr(exc, "args", ())
+ if args and isinstance(args[0], int):
+ return args[0]
+ return None
+
+
+def is_psycopg_connection_error(exc: BaseException) -> bool:
+ sqlstate = sqlstate_from_exception(exc)
+ if sqlstate is not None:
+ return sqlstate.startswith("08") or sqlstate in {"57P01", "57P02", "57P03"}
+
+ if exc.__class__.__name__ == "InterfaceError":
+ return True
+
+ text = exception_text(exc)
+ return any(message in text for message in CONNECTION_ERROR_MESSAGES)
+
+
+def is_retryable_transaction_error(exc: BaseException) -> bool:
+ sqlstate = sqlstate_from_exception(exc)
+ if sqlstate in PSYCOPG_RETRYABLE_TRANSACTION_SQLSTATES:
+ return True
+
+ text = exception_text(exc)
+ return "deadlock detected" in text or "could not serialize access" in text or "could not obtain lock" in text
+
+
+def is_duplicate_column_error(exc: BaseException) -> bool:
+ if mysql_errno_from_exception(exc) == 1060:
+ return True
+
+ sqlstate = sqlstate_from_exception(exc)
+ if sqlstate in PSYCOPG_DUPLICATE_COLUMN_SQLSTATES:
+ return True
+
+ text = exception_text(exc)
+ return "duplicate column" in text or ("column" in text and "already exists" in text)
+
+
+def is_duplicate_object_error(exc: BaseException) -> bool:
+ if mysql_errno_from_exception(exc) == 1061:
+ return True
+
+ sqlstate = sqlstate_from_exception(exc)
+ if sqlstate in PSYCOPG_DUPLICATE_OBJECT_SQLSTATES:
+ return True
+
+ text = exception_text(exc)
+ return "duplicate key name" in text or "already exists" in text
+
+
+def is_undefined_object_error(exc: BaseException) -> bool:
+ if mysql_errno_from_exception(exc) == 1091:
+ return True
+
+ sqlstate = sqlstate_from_exception(exc)
+ if sqlstate in PSYCOPG_UNDEFINED_OBJECT_SQLSTATES:
+ return True
+
+ text = exception_text(exc)
+ return "can't drop" in text or "does not exist" in text
diff --git a/api/db/services/common_service.py b/api/db/services/common_service.py
index 93fd5f8a83..2b7046a0cf 100644
--- a/api/db/services/common_service.py
+++ b/api/db/services/common_service.py
@@ -23,12 +23,26 @@ import peewee
from peewee import InterfaceError, OperationalError
from api.db.db_models import DB
+from api.db.gaussdb_error_utils import is_retryable_transaction_error
+from common import settings
from common.misc_utils import get_uuid
from common.time_utils import current_timestamp, datetime_format
def _is_deadlock_error(exc: OperationalError) -> bool:
- return isinstance(exc, OperationalError) and bool(getattr(exc, "args", ())) and exc.args[0] == 1213
+ if not isinstance(exc, OperationalError):
+ return False
+
+ args = getattr(exc, "args", ())
+ if args and args[0] == 1213:
+ return True
+
+ # Keep the existing helper name to avoid changing every caller, although it
+ # now recognizes all retryable transaction conflicts. GaussDB reports
+ # 40P01 for deadlocks, 40001 for serialization failures, and 55P03 when a
+ # NOWAIT row lock is unavailable. Peewee omits pgcode on the outer error, so
+ # the SQLSTATE must be read from the psycopg2 error in __context__.
+ return settings.DATABASE_TYPE.lower() == "gaussdb" and is_retryable_transaction_error(exc)
def retry_deadlock_operation(max_retries=3, retry_delay=0.1):
@@ -274,6 +288,16 @@ class CommonService:
# Number of records updated
data["update_time"] = current_timestamp()
data["update_date"] = datetime_format(datetime.now())
+ # Only the GaussDB adapter exposes this hook. Centralized GaussDB and
+ # every other database retain the original update_by_id behavior.
+ replace_update = getattr(DB, "replace_update_by_id", None)
+ if replace_update is not None:
+ num = replace_update(cls.model, pid, data)
+ if num is not None:
+ return num
+ prepare_update = getattr(DB, "prepare_update_by_id", None)
+ if prepare_update is not None:
+ data = prepare_update(cls.model, pid, data)
num = cls.model.update(data).where(cls.model.id == pid).execute()
return num
diff --git a/api/db/services/connector_service.py b/api/db/services/connector_service.py
index 502bd1f51d..b940e6e663 100644
--- a/api/db/services/connector_service.py
+++ b/api/db/services/connector_service.py
@@ -27,6 +27,7 @@ from api.db.services.common_service import CommonService
from api.db.services.document_service import DocumentService
from api.db.services.document_service import DocMetadataService
from api.utils.common import hash128
+from common import settings
from common.misc_utils import get_uuid
from common.constants import ConnectorTaskType, TaskStatus
from common.settings import TIMEZONE
@@ -35,6 +36,24 @@ from common.time_utils import current_timestamp, timestamp_to_date
LOGGER = logging.getLogger(__name__)
+def _is_gaussdb_compatible_metadata_db() -> bool:
+ return settings.normalize_database_type(settings.DATABASE_TYPE) == "gaussdb"
+
+
+def _gaussdb_poll_interval_expr(freq_field: str) -> SQL:
+ return SQL(f"NOW() AT TIME ZONE '{settings.TIMEZONE}' - (t2.{freq_field} * INTERVAL '1 minute')")
+
+
+def _append_text_expr(field, suffix: str):
+ if not _is_gaussdb_compatible_metadata_db():
+ return field + suffix
+
+ suffix = str(suffix or "")
+ if not suffix:
+ return None
+ return fn.COALESCE(field + suffix, suffix)
+
+
class ConnectorService(CommonService):
model = Connector
@@ -226,6 +245,10 @@ class SyncLogsService(CommonService):
Knowledgebase.name.alias("kb_name"),
cls.model.status,
]
+ if _is_gaussdb_compatible_metadata_db():
+ # GaussDB requires a DISTINCT query's ORDER BY expression in the
+ # select list.
+ fields.append(cls.model.update_time)
if not connector_id:
fields.append(Connector.config)
@@ -239,11 +262,14 @@ class SyncLogsService(CommonService):
if connector_id:
query = query.where(cls.model.connector_id == connector_id)
else:
- database_type = os.getenv("DB_TYPE", "mysql")
- if "postgres" in database_type.lower():
- expr = SQL(f"NOW() AT TIME ZONE '{TIMEZONE}' - make_interval(mins => t2.refresh_freq)")
+ if _is_gaussdb_compatible_metadata_db():
+ expr = _gaussdb_poll_interval_expr("refresh_freq")
else:
- expr = SQL("NOW() - INTERVAL `t2`.`refresh_freq` MINUTE")
+ database_type = os.getenv("DB_TYPE", "mysql")
+ if "postgres" in database_type.lower():
+ expr = SQL(f"NOW() AT TIME ZONE '{TIMEZONE}' - make_interval(mins => t2.refresh_freq)")
+ else:
+ expr = SQL("NOW() - INTERVAL `t2`.`refresh_freq` MINUTE")
query = query.where(Connector.input_type == InputType.POLL, Connector.status == TaskStatus.SCHEDULE, cls.model.status == TaskStatus.SCHEDULE, cls.model.update_date < expr)
query = query.distinct().order_by(cls.model.update_time.desc())
@@ -318,11 +344,14 @@ class SyncLogsService(CommonService):
cls.model.task_type == task_type,
)
- database_type = os.getenv("DB_TYPE", "mysql")
- if "postgres" in database_type.lower():
- expr = SQL(f"NOW() AT TIME ZONE '{TIMEZONE}' - make_interval(mins => t2.{freq_field})")
+ if _is_gaussdb_compatible_metadata_db():
+ expr = _gaussdb_poll_interval_expr(freq_field)
else:
- expr = SQL(f"NOW() - INTERVAL `t2`.`{freq_field}` MINUTE")
+ database_type = os.getenv("DB_TYPE", "mysql")
+ if "postgres" in database_type.lower():
+ expr = SQL(f"NOW() AT TIME ZONE '{TIMEZONE}' - make_interval(mins => t2.{freq_field})")
+ else:
+ expr = SQL(f"NOW() - INTERVAL `t2`.`{freq_field}` MINUTE")
query = query.where(cls.model.update_date < expr)
return list(query.distinct().order_by(cls.model.update_time.desc()).dicts())
@@ -397,34 +426,48 @@ class SyncLogsService(CommonService):
logging.exception(e)
task = cls.get_latest_task(connector_id, kb_id, task_type)
if task:
- cls.model.update(
- status=TaskStatus.SCHEDULE, poll_range_start=poll_range_start, error_msg=cls.model.error_msg + str(e), full_exception_trace=cls.model.full_exception_trace + str(e)
- ).where(cls.model.id == task.id).execute()
+ update_payload = {
+ "status": TaskStatus.SCHEDULE,
+ "poll_range_start": poll_range_start,
+ }
+ error_msg_expr = _append_text_expr(cls.model.error_msg, str(e))
+ trace_expr = _append_text_expr(cls.model.full_exception_trace, str(e))
+ if error_msg_expr is not None:
+ update_payload["error_msg"] = error_msg_expr
+ if trace_expr is not None:
+ update_payload["full_exception_trace"] = trace_expr
+ cls.model.update(update_payload).where(cls.model.id == task.id).execute()
ConnectorService.update_by_id(connector_id, {"status": TaskStatus.SCHEDULE})
@classmethod
def increase_docs(cls, id, max_update, doc_num, err_msg="", error_count=0):
# Keep sync monotonic.
- cls.model.update(
- new_docs_indexed=cls.model.new_docs_indexed + doc_num,
- total_docs_indexed=cls.model.total_docs_indexed + doc_num,
- poll_range_start=fn.COALESCE(fn.GREATEST(cls.model.poll_range_start, max_update), max_update),
- poll_range_end=fn.COALESCE(fn.GREATEST(cls.model.poll_range_end, max_update), max_update),
- error_msg=cls.model.error_msg + err_msg,
- error_count=cls.model.error_count + error_count,
- update_time=current_timestamp(),
- update_date=timestamp_to_date(current_timestamp()),
- ).where(cls.model.id == id).execute()
+ update_payload = {
+ "new_docs_indexed": cls.model.new_docs_indexed + doc_num,
+ "total_docs_indexed": cls.model.total_docs_indexed + doc_num,
+ "poll_range_start": fn.COALESCE(fn.GREATEST(cls.model.poll_range_start, max_update), max_update),
+ "poll_range_end": fn.COALESCE(fn.GREATEST(cls.model.poll_range_end, max_update), max_update),
+ "error_count": cls.model.error_count + error_count,
+ "update_time": current_timestamp(),
+ "update_date": timestamp_to_date(current_timestamp()),
+ }
+ error_msg_expr = _append_text_expr(cls.model.error_msg, err_msg)
+ if error_msg_expr is not None:
+ update_payload["error_msg"] = error_msg_expr
+ cls.model.update(update_payload).where(cls.model.id == id).execute()
@classmethod
def increase_removed_docs(cls, id, removed_count, err_msg="", error_count=0):
- cls.model.update(
- docs_removed_from_index=cls.model.docs_removed_from_index + removed_count,
- error_msg=cls.model.error_msg + err_msg,
- error_count=cls.model.error_count + error_count,
- update_time=current_timestamp(),
- update_date=timestamp_to_date(current_timestamp()),
- ).where(cls.model.id == id).execute()
+ update_payload = {
+ "docs_removed_from_index": cls.model.docs_removed_from_index + removed_count,
+ "error_count": cls.model.error_count + error_count,
+ "update_time": current_timestamp(),
+ "update_date": timestamp_to_date(current_timestamp()),
+ }
+ error_msg_expr = _append_text_expr(cls.model.error_msg, err_msg)
+ if error_msg_expr is not None:
+ update_payload["error_msg"] = error_msg_expr
+ cls.model.update(update_payload).where(cls.model.id == id).execute()
@classmethod
def duplicate_and_parse(cls, kb, docs, tenant_id, src, auto_parse=True):
diff --git a/api/db/services/dialog_service.py b/api/db/services/dialog_service.py
index a7ded8e8f1..d7168a69ba 100644
--- a/api/db/services/dialog_service.py
+++ b/api/db/services/dialog_service.py
@@ -54,6 +54,7 @@ from rag.utils.web_search_conn import create_web_search_provider, has_web_search
from rag.utils.tts_cache import synthesize_with_cache
from common.string_utils import remove_redundant_spaces
from common import settings
+from rag.utils import gaussdb_text_to_sql
def _chunk_kb_id_for_doc(row_dict, kb_ids, doc_id):
@@ -1019,6 +1020,8 @@ async def use_sql(question, field_map, tenant_id, chat_mdl, quota=True, kb_ids=N
doc_engine = "infinity"
elif settings.DOC_ENGINE_OCEANBASE:
doc_engine = "oceanbase"
+ elif settings.DOC_ENGINE_GAUSSDB:
+ doc_engine = "gaussdb"
else:
doc_engine = "es"
@@ -1051,6 +1054,12 @@ async def use_sql(question, field_map, tenant_id, chat_mdl, quota=True, kb_ids=N
logging.debug(f"use_sql: Using ES/OS table name: {table_name}")
expected_doc_name_column = "docnm" if doc_engine == "infinity" else "docnm_kwd"
+ if doc_engine == "gaussdb":
+ if not kb_ids:
+ raise ValueError("GaussDB Text-to-SQL requires kb_ids")
+ for kid in kb_ids:
+ _assert_valid_uuid(kid, "kb_id")
+ gaussdb_validator = gaussdb_text_to_sql.build_validator(table_name, kb_ids, field_map)
def has_source_columns(columns):
"""Return True if the result set contains the columns needed to build source citations."""
@@ -1059,6 +1068,8 @@ async def use_sql(question, field_map, tenant_id, chat_mdl, quota=True, kb_ids=N
def is_aggregate_sql(sql_text):
"""Return True if *sql_text* contains an aggregate function (COUNT, SUM, AVG, MAX, MIN, DISTINCT)."""
+ if doc_engine == "gaussdb":
+ return gaussdb_text_to_sql.is_aggregate_sql(sql_text)
return bool(re.search(r"(count|sum|avg|max|min|distinct)\s*\(", (sql_text or "").lower()))
def normalize_sql(sql):
@@ -1080,12 +1091,13 @@ async def use_sql(question, field_map, tenant_id, chat_mdl, quota=True, kb_ids=N
def add_kb_filter(sql):
"""Inject validated scope filters into *sql*.
- Infinity encodes single-KB scope in the table name, so only document
- scope is injected there. All ids are validated before interpolation.
+ Infinity encodes single-KB scope in the table name and the GaussDB
+ validator injects its KB boundary, so only document scope is injected
+ for those engines. All ids are validated before interpolation.
"""
scope_filters = []
sql_lower = sql.lower()
- if doc_engine != "infinity" and kb_ids and "kb_id =" not in sql_lower and "kb_id=" not in sql_lower:
+ if doc_engine not in ("infinity", "gaussdb") and kb_ids and "kb_id =" not in sql_lower and "kb_id=" not in sql_lower:
for kid in kb_ids:
_assert_valid_uuid(kid, "kb_id")
if len(kb_ids) == 1:
@@ -1180,6 +1192,10 @@ Question: {}
Write SQL using json_extract_string() with exact field names. Include doc_id, docnm_kwd for data queries. Only SQL.""".format(
table_name, ", ".join(json_field_names), "\n".join([f" - {field}" for field in json_field_names]), question
)
+ elif doc_engine == "gaussdb":
+ row_count_override = f"SELECT COUNT(*) AS rows FROM {table_name}" if is_row_count_question(question) else None
+ sys_prompt = gaussdb_text_to_sql.build_sql_prompt(table_name, field_map, question)
+ user_prompt = gaussdb_text_to_sql.build_user_prompt(table_name, field_map, question)
else:
# Build ES/OS prompts with direct field access
row_count_override = None
@@ -1209,6 +1225,8 @@ Write SQL using exact field names above. Include doc_id, docnm_kwd for data quer
sql = await chat_mdl.async_chat(sys_prompt, [{"role": "user", "content": prompt}], {"temperature": 0.06})
sql = normalize_sql(sql)
sql = add_kb_filter(sql)
+ if doc_engine == "gaussdb":
+ sql = gaussdb_validator.validate_and_patch(sql).sql
logging.debug(f"{question} get SQL(refined): {sql}")
tried_times += 1
@@ -1216,6 +1234,8 @@ Write SQL using exact field names above. Include doc_id, docnm_kwd for data quer
tbl = settings.retriever.sql_retrieval(sql, format="json")
if tbl is None:
logging.debug("use_sql: SQL retrieval failed (returned None)")
+ if doc_engine == "gaussdb":
+ raise RuntimeError("SQL execution returned no result")
return None, sql
row_count = len(tbl.get("rows", []))
if row_count == 0:
@@ -1225,6 +1245,15 @@ Write SQL using exact field names above. Include doc_id, docnm_kwd for data quer
return tbl, sql
async def repair_table_for_missing_source_columns(previous_sql):
+ if doc_engine == "gaussdb":
+ return await get_table(
+ custom_user_prompt=gaussdb_text_to_sql.build_repair_prompt(
+ table_name,
+ field_map,
+ question,
+ previous_sql,
+ )
+ )
if doc_engine in ("infinity", "oceanbase"):
json_field_names = list(field_map.keys())
repair_prompt = """Table name: {};
@@ -1260,7 +1289,9 @@ Return ONLY SQL.""".format(table_name, "\n".join([f" - {k} ({v})" for k, v in f
except Exception as e:
logging.warning(f"use_sql: Initial SQL execution FAILED with error: {e}")
# Build retry prompt with error information
- if doc_engine in ("infinity", "oceanbase"):
+ if doc_engine == "gaussdb":
+ user_prompt = gaussdb_text_to_sql.build_retry_prompt(table_name, field_map, question, e)
+ elif doc_engine in ("infinity", "oceanbase"):
# Build Infinity error retry prompt
json_field_names = list(field_map.keys())
user_prompt = """
@@ -1414,6 +1445,23 @@ Please correct the error and write SQL again using json_extract_string(chunk_dat
# Keep original table format as answer
answer = "\n".join([columns, line, rows])
+ if doc_engine == "gaussdb":
+ try:
+ chunks_sql = gaussdb_text_to_sql.build_aggregate_source_sql(
+ sql,
+ expected_doc_name_column,
+ include_kb_id=not (kb_ids and len(kb_ids) == 1),
+ )
+ chunks_sql = gaussdb_validator.validate_and_patch(chunks_sql).sql
+ logging.debug(f"use_sql: Fetching chunks with SQL: {chunks_sql}")
+ chunks_tbl = settings.retriever.sql_retrieval(chunks_sql, format="json")
+ reference = gaussdb_text_to_sql.build_source_reference(chunks_tbl, kb_ids)
+ if reference:
+ return {"answer": answer, "reference": reference, "prompt": sys_prompt}
+ except Exception as e:
+ logging.warning(f"use_sql: Failed to fetch chunks: {e}")
+ return {"answer": answer, "reference": {"chunks": [], "doc_aggs": []}, "prompt": sys_prompt}
+
# Now fetch doc_id, docnm_kwd to provide source chunks
# Extract WHERE clause from the original SQL
where_match = re.search(r"\bwhere\b(.+?)(?:\bgroup by\b|\border by\b|\blimit\b|$)", sql, re.IGNORECASE)
@@ -1495,6 +1543,14 @@ Please correct the error and write SQL again using json_extract_string(chunk_dat
},
"prompt": sys_prompt,
}
+ if doc_engine == "gaussdb":
+ gaussdb_text_to_sql.complete_reference_kb_ids(
+ result,
+ table_name,
+ kb_ids,
+ gaussdb_validator,
+ settings.retriever.sql_retrieval,
+ )
logging.debug(f"use_sql: Returning answer with {len(result['reference']['chunks'])} chunks from {len(doc_aggs)} documents")
return result
diff --git a/api/db/services/doc_metadata_service.py b/api/db/services/doc_metadata_service.py
index f9cfefeaeb..4943fe276c 100644
--- a/api/db/services/doc_metadata_service.py
+++ b/api/db/services/doc_metadata_service.py
@@ -272,6 +272,10 @@ class DocMetadataService:
total_count = total_hits.get("value", len(page_docs))
else:
total_count = total_hits if total_hits else len(page_docs)
+ # Handle GaussDB/OceanBase SearchResult(total, chunks) format
+ elif hasattr(results, "chunks") and hasattr(results, "total"):
+ page_docs = list(results.chunks or [])
+ total_count = results.total
# Handle list/iterable results
elif hasattr(results, "__iter__") and not isinstance(results, dict):
page_docs = list(results)
@@ -469,6 +473,22 @@ class DocMetadataService:
logging.debug(f"[update_document_metadata] Updating doc_id: {doc_id}, kb_id: {kb_id}, meta_fields: {processed_meta}")
+ if settings.DOC_ENGINE_GAUSSDB:
+ if not settings.docStoreConn.index_exist(index_name, kb_id):
+ result = settings.docStoreConn.create_doc_meta_idx(index_name)
+ if result is False:
+ logging.error(f"Failed to create metadata index {index_name}")
+ return False
+ insert_errors = settings.docStoreConn.insert(
+ [{"id": doc_id, "kb_id": kb_id, "meta_fields": processed_meta}],
+ index_name,
+ kb_id,
+ )
+ if insert_errors:
+ logging.error(f"Failed to update metadata for document {doc_id}: {insert_errors}")
+ return False
+ return True
+
# For Elasticsearch, use efficient partial update
if not settings.DOC_ENGINE_INFINITY and not settings.DOC_ENGINE_OCEANBASE and not settings.DOC_ENGINE_SERENEDB:
# Check if index exists first
@@ -864,6 +884,8 @@ class DocMetadataService:
if settings.DOC_ENGINE_INFINITY:
return cls._filter_doc_ids_by_metadata_infinity(index_name, kb_ids, filters, logic)
+ elif settings.DOC_ENGINE_GAUSSDB:
+ return cls._filter_doc_ids_by_metadata_gaussdb(index_name, kb_ids, filters, logic, limit)
else:
return cls._filter_doc_ids_by_metadata_es(index_name, kb_ids, filters, logic, limit)
@@ -941,6 +963,54 @@ class DocMetadataService:
logging.debug(f"ES metadata filter returned {len(unique)} matches for KBs {kb_ids}")
return unique
+ @classmethod
+ def _filter_doc_ids_by_metadata_gaussdb(
+ cls,
+ index_name: str,
+ kb_ids: List[str],
+ filters: List[Dict],
+ logic: str,
+ limit: int,
+ ) -> Optional[List[str]]:
+ """GaussDB push-down path for metadata filtering."""
+ from common.metadata_gaussdb_filter import (
+ UnsupportedGaussDBMetaFilter,
+ build_gaussdb_filter,
+ fetch_gaussdb_metadata_doc_ids,
+ is_pushdown_supported,
+ )
+
+ if not is_pushdown_supported(filters):
+ return None
+
+ try:
+ sql_filter, filter_params = build_gaussdb_filter(filters, logic)
+ except UnsupportedGaussDBMetaFilter as e:
+ logging.error("GaussDB build metadata filter failed: %s", e.reason)
+ return None
+
+ try:
+ probe_limit = int(limit or 0) + 1 if limit else limit
+ doc_ids = fetch_gaussdb_metadata_doc_ids(settings.docStoreConn, index_name, kb_ids, sql_filter, filter_params, probe_limit)
+ except Exception:
+ logging.warning("GaussDB metadata filter push-down failed; falling back to in-memory filter", exc_info=True)
+ return None
+
+ seen: set[str] = set()
+ unique: List[str] = []
+ for did in doc_ids:
+ if did in seen:
+ continue
+ seen.add(did)
+ unique.append(did)
+
+ if limit and len(unique) > limit:
+ logging.warning(f"GaussDB metadata filter hit push-down cap, falling back to in-memory: cap={limit}, kb_ids={kb_ids}")
+ return None
+
+ logging.debug(f"GaussDB metadata filter returned {len(unique)} matches for KBs {kb_ids}")
+ return unique[:limit] if limit else unique
+
@classmethod
def _filter_doc_ids_by_metadata_infinity(
cls,
diff --git a/api/utils/health_utils.py b/api/utils/health_utils.py
index 896f62c132..de11a58bc0 100644
--- a/api/utils/health_utils.py
+++ b/api/utils/health_utils.py
@@ -17,6 +17,7 @@ from datetime import datetime
import json
import logging
import os
+import re
import requests
from timeit import default_timer as timer
@@ -25,7 +26,15 @@ from rag.utils.redis_conn import REDIS_CONN
from rag.utils.es_conn import ESConnection
from rag.utils.infinity_conn import InfinityConnection
from rag.utils.ob_conn import OBConnection
+from rag.utils.gaussdb_conn import GaussDBConnection
from common import settings
+from common.doc_store.gaussdb_conn_base import mask_gaussdb_text
+
+
+_GAUSSDB_SENSITIVE_KEY_PATTERN = re.compile(
+ r"(?:^|[_-])(?:password|passwd|pwd|secret|token|api[_-]?key|dsn)$",
+ re.IGNORECASE,
+)
def _ok_nok(ok: bool) -> str:
@@ -122,6 +131,113 @@ def get_oceanbase_status():
}
+def _mask_gaussdb_string(value: str) -> str:
+ return mask_gaussdb_text(value)
+
+
+def _log_gaussdb_error(context: str, exc: Exception) -> str:
+ masked_error = _mask_gaussdb_string(str(exc))
+ logging.error("%s (%s): %s", context, type(exc).__name__, masked_error)
+ return masked_error
+
+
+def _mask_gaussdb_secret(value):
+ if isinstance(value, dict):
+ return {key: "***" if isinstance(key, str) and _GAUSSDB_SENSITIVE_KEY_PATTERN.search(key) else _mask_gaussdb_secret(item) for key, item in value.items()}
+ if isinstance(value, list):
+ return [_mask_gaussdb_secret(v) for v in value]
+ if isinstance(value, tuple):
+ return tuple(_mask_gaussdb_secret(v) for v in value)
+ if isinstance(value, str):
+ return _mask_gaussdb_string(value)
+ return value
+
+
+def _get_gaussdb_connection():
+ conn = getattr(settings, "docStoreConn", None)
+ if conn is not None and getattr(conn, "db_type", lambda: None)() == "gaussdb":
+ return conn
+ return GaussDBConnection()
+
+
+def get_gaussdb_status():
+ doc_engine = os.getenv("DOC_ENGINE", "elasticsearch").lower()
+ if doc_engine != "gaussdb":
+ return {
+ "status": "not_configured",
+ "message": "GaussDB is not configured as the document engine",
+ }
+ try:
+ conn = _get_gaussdb_connection()
+ health_info = _mask_gaussdb_secret(conn.health())
+ performance_metrics = _mask_gaussdb_secret(conn.get_performance_metrics())
+ status = "alive" if health_info.get("status") == "healthy" else "timeout"
+ return {
+ "status": status,
+ "message": {
+ "health": health_info,
+ "performance": performance_metrics,
+ },
+ }
+ except Exception as e:
+ masked_error = _log_gaussdb_error("GaussDB status check failed", e)
+ return {
+ "status": "timeout",
+ "message": f"error: {masked_error}",
+ }
+
+
+def check_gaussdb_health() -> dict:
+ doc_engine = os.getenv("DOC_ENGINE", "elasticsearch").lower()
+ if doc_engine != "gaussdb":
+ return {
+ "status": "not_configured",
+ "details": {
+ "connection": "not_configured",
+ "message": "GaussDB is not configured as the document engine",
+ },
+ }
+
+ try:
+ conn = _get_gaussdb_connection()
+ health_info = _mask_gaussdb_secret(conn.health())
+ performance_metrics = _mask_gaussdb_secret(conn.get_performance_metrics())
+ connection_status = performance_metrics.get("connection", "unknown")
+ if connection_status == "disconnected" or health_info.get("status") != "healthy":
+ return {
+ "status": "unhealthy",
+ "details": {
+ "connection": connection_status,
+ "latency_ms": performance_metrics.get("latency_ms", 0),
+ "uri": health_info.get("uri", "unknown"),
+ "version": health_info.get("version_comment", "unknown"),
+ "sql_compatibility": health_info.get("sql_compatibility", "unknown"),
+ "error": health_info.get("error", performance_metrics.get("error", "")),
+ },
+ }
+
+ is_healthy = connection_status == "connected" and performance_metrics.get("latency_ms", float("inf")) < 1000
+ return {
+ "status": "healthy" if is_healthy else "degraded",
+ "details": {
+ "connection": connection_status,
+ "latency_ms": performance_metrics.get("latency_ms", 0),
+ "uri": health_info.get("uri", "unknown"),
+ "version": health_info.get("version_comment", "unknown"),
+ "sql_compatibility": health_info.get("sql_compatibility", "unknown"),
+ },
+ }
+ except Exception as e:
+ masked_error = _log_gaussdb_error("GaussDB health check failed", e)
+ return {
+ "status": "unhealthy",
+ "details": {
+ "connection": "disconnected",
+ "error": masked_error,
+ },
+ }
+
+
def check_oceanbase_health() -> dict:
"""
Check OceanBase health status with comprehensive metrics.
@@ -193,6 +309,10 @@ def check_oceanbase_health() -> dict:
def get_mysql_status():
+ if settings.DATABASE_TYPE.lower() == "gaussdb":
+ # GaussDB cannot execute MySQL's SHOW PROCESSLIST.
+ return get_database_status()
+
try:
cursor = DB.execute_sql("SHOW PROCESSLIST;")
res_rows = cursor.fetchall()
@@ -206,6 +326,29 @@ def get_mysql_status():
}
+def get_database_status():
+ try:
+ # SELECT 1 is the smallest probe supported by MySQL, PostgreSQL, and
+ # GaussDB. Admin uses it for a GaussDB metadata database instead of a
+ # MySQL-specific status query.
+ cursor = DB.execute_sql("SELECT 1;")
+ row = cursor.fetchone()
+ cursor.close()
+ return {
+ "status": "alive",
+ "message": {
+ "database": settings.DATABASE_TYPE.lower(),
+ "result": row[0] if row else None,
+ },
+ }
+ except Exception as e:
+ masked_error = _log_gaussdb_error("GaussDB metadata database status check failed", e)
+ return {
+ "status": "timeout",
+ "message": f"error: {masked_error}",
+ }
+
+
def _minio_scheme_and_verify():
"""
Determine URL scheme (http/https) and SSL verify flag for MinIO health check.
diff --git a/common/config_utils.py b/common/config_utils.py
index 524732265d..06db79debf 100644
--- a/common/config_utils.py
+++ b/common/config_utils.py
@@ -79,6 +79,9 @@ def show_configs():
msg = f"Current configs, from {conf_realpath(SERVICE_CONF)}:"
for k, v in CONFIGS.items():
if isinstance(v, dict):
+ if k == "gaussdb" and isinstance(v.get("config"), dict) and "password" in v["config"]:
+ v = copy.deepcopy(v)
+ v["config"]["password"] = "*" * 8
if "password" in v:
v = copy.deepcopy(v)
v["password"] = "*" * 8
diff --git a/common/doc_store/gaussdb_conn_base.py b/common/doc_store/gaussdb_conn_base.py
new file mode 100644
index 0000000000..684248b6d0
--- /dev/null
+++ b/common/doc_store/gaussdb_conn_base.py
@@ -0,0 +1,1687 @@
+#
+# Copyright 2025 The InfiniFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+import hashlib
+import json
+import logging
+import re
+from dataclasses import dataclass
+from numbers import Integral
+from timeit import default_timer as timer
+from typing import Any
+
+from common.doc_store.doc_store_base import DocStoreConnection, MatchExpr, OrderByExpr
+
+
+_GAUSSDB_DSN_PATTERN = re.compile(r"(?i)\b(?:postgres(?:ql)?|gaussdb)://[^\s'\"<>]+")
+_GAUSSDB_SECRET_PATTERN = re.compile(
+ r"""
+ (?P<prefix>
+ (?P<key_quote>["']?)
+ (?P<key>\b(?:password|passwd|access_token|token|api_key|secret|dsn)\b)
+ (?P=key_quote)
+ (?P<sep>\s*[:=]\s*)
+ )
+ (?P<value>
+ "(?:\\.|""|[^"\\])*"
+ | '(?:\\.|''|[^'\\])*'
+ | "(?:\\.|[^"\\])*\Z
+ | '(?:\\.|[^'\\])*\Z
+ | [^\s]+
+ )
+ """,
+ flags=re.IGNORECASE | re.VERBOSE,
+)
+
+
+def mask_gaussdb_text(value: Any) -> str:
+ message = str(value)
+ message = _GAUSSDB_DSN_PATTERN.sub("***", message)
+
+ def replace_secret(match: re.Match[str]) -> str:
+ secret = match.group("value")
+ if len(secret) >= 2 and secret[0] in {"'", '"'} and secret[-1] == secret[0]:
+ masked = f"{secret[0]}***{secret[0]}"
+ else:
+ masked = "***"
+ return f"{match.group('prefix')}{masked}"
+
+ return _GAUSSDB_SECRET_PATTERN.sub(replace_secret, message)
+
+
+class InvalidGaussDBObjectName(ValueError):
+ pass
+
+
+class UnsafeGaussDBSQL(ValueError):
+ pass
+
+
+EXTRA_FIELD_RE = re.compile(r"^[A-Za-z_][A-Za-z0-9_]{0,127}$")
+
+
+def validate_extra_field(name: str) -> str:
+ field = str(name or "")
+ if not EXTRA_FIELD_RE.fullmatch(field):
+ raise InvalidGaussDBObjectName(name)
+ return field
+
+
+def extra_field_expr(name: str, *, prefix: str | None = None, text: bool = False) -> str:
+ field = validate_extra_field(name)
+ source = f"{prefix}.extra" if prefix else "extra"
+ operator = "->>" if text else "->"
+ return f"({source} {operator} '{field}')"
+
+
+def _parse_gaussdb_statements(sql: str, dialect: str = "postgres"):
+ from sqlglot import exp, parse
+
+ try:
+ return exp, parse(sql, read=dialect)
+ except Exception:
+ raise UnsafeGaussDBSQL("SQL parse failed") from None
+
+
+def is_gaussdb_aggregate_sql(sql: str) -> bool:
+ exp, expressions = _parse_gaussdb_statements(sql)
+ if len(expressions) != 1:
+ raise UnsafeGaussDBSQL("exactly one SQL statement is allowed")
+ return any(isinstance(node, exp.AggFunc) for node in expressions[0].walk())
+
+
+_FIELD_DESCRIPTOR_TYPES = {
+ "bool",
+ "boolean",
+ "date",
+ "datetime",
+ "double",
+ "float",
+ "integer",
+ "json",
+ "number",
+ "string",
+ "text",
+}
+
+
+def gaussdb_field_json_path_parts(field: str, descriptor: Any = None) -> tuple[str, ...]:
+ candidate = str(descriptor or "").strip()
+ if candidate and candidate.lower() not in _FIELD_DESCRIPTOR_TYPES:
+ return tuple(part for part in candidate.split(".") if part)
+ return tuple(part for part in str(field).split(".") if part)
+
+
+@dataclass(frozen=True)
+class ExposedGaussDBTable:
+ logical_name: str
+ physical_name: str
+ allowed_columns: frozenset[str]
+ json_fields: dict[str, tuple[str, ...]]
+ required_kb_ids: tuple[str, ...]
+
+ @classmethod
+ def from_field_map(cls, physical_name: str, kb_ids: list[str] | tuple[str, ...], field_map: dict[str, Any]):
+ json_fields = {}
+ for field, descriptor in (field_map or {}).items():
+ parts = gaussdb_field_json_path_parts(str(field), descriptor)
+ if parts:
+ json_fields[str(field)] = parts
+ return cls(
+ logical_name=physical_name,
+ physical_name=physical_name,
+ allowed_columns=frozenset({"doc_id", "docnm_kwd", "kb_id", "chunk_data"}),
+ json_fields=json_fields,
+ required_kb_ids=tuple(str(kid) for kid in kb_ids or () if str(kid)),
+ )
+
+
+@dataclass(frozen=True)
+class ValidatedGaussDBSQL:
+ sql: str
+ columns: list[str] | None = None
+ is_aggregation: bool = False
+
+
+def jsonb_path_literal(parts: list[str] | tuple[str, ...]) -> str:
+ if not isinstance(parts, (list, tuple)) or not parts:
+ raise UnsafeGaussDBSQL("empty JSONB path")
+ encoded = []
+ for part in parts:
+ if not isinstance(part, str):
+ raise UnsafeGaussDBSQL("invalid JSONB path segment")
+ segment = part
+ if not segment:
+ raise UnsafeGaussDBSQL("empty JSONB path segment")
+ if re.fullmatch(r"[A-Za-z0-9_]+", segment):
+ encoded.append(segment)
+ else:
+ escaped = segment.replace("\\", "\\\\").replace('"', '\\"')
+ encoded.append(f'"{escaped}"')
+ path = "{" + ",".join(encoded) + "}"
+ return "'" + path.replace("'", "''") + "'"
+
+
+def _parse_jsonb_path_literal(value: str) -> tuple[str, ...]:
+ text = str(value or "").strip()
+ if len(text) >= 2 and text[0] == "'" and text[-1] == "'":
+ text = text[1:-1].replace("''", "'")
+ if not (text.startswith("{") and text.endswith("}")):
+ raise UnsafeGaussDBSQL("dynamic JSONB path is not allowed")
+ body = text[1:-1]
+ parts: list[str] = []
+ buf: list[str] = []
+ quoted = False
+ escaped = False
+ for char in body:
+ if escaped:
+ buf.append(char)
+ escaped = False
+ continue
+ if quoted and char == "\\":
+ escaped = True
+ continue
+ if char == '"':
+ quoted = not quoted
+ continue
+ if char == "," and not quoted:
+ parts.append("".join(buf))
+ buf = []
+ continue
+ buf.append(char)
+ if quoted or escaped:
+ raise UnsafeGaussDBSQL("invalid JSONB path literal")
+ parts.append("".join(buf))
+ if not parts or any(part == "" for part in parts):
+ raise UnsafeGaussDBSQL("invalid JSONB path literal")
+ return tuple(parts)
+
+
+class GaussDBSQLValidator:
+ PARSE_DIALECT = "postgres"
+ FORBIDDEN_FUNCTIONS = {
+ "pg_sleep",
+ "sleep",
+ "now",
+ "current_user",
+ "current_date",
+ "current_time",
+ "current_timestamp",
+ "current_database",
+ "current_catalog",
+ "localtime",
+ "localtimestamp",
+ "session_user",
+ "user",
+ "version",
+ "current_schema",
+ "json_extract",
+ "json_extract_string",
+ "json_extract_isnull",
+ "jsonb_each",
+ "jsonb_each_text",
+ "jsonb_to_record",
+ "jsonb_to_recordset",
+ "jsonb_array_elements",
+ "jsonb_array_elements_text",
+ }
+
+ def __init__(
+ self,
+ tables: dict[str, ExposedGaussDBTable] | set[str] | list[str] | tuple[str, ...] | None = None,
+ kb_ids: list[str] | tuple[str, ...] | None = None,
+ default_limit: int = 128,
+ readonly_only: bool = False,
+ runtime_readonly_guard: bool = False,
+ execution_schema: str | None = None,
+ ):
+ if tables is None:
+ self.tables = {}
+ elif isinstance(tables, dict):
+ self.tables = dict(tables)
+ else:
+ self.tables = {str(table): ExposedGaussDBTable.from_field_map(str(table), kb_ids or (), {}) for table in tables}
+ self.default_limit = int(default_limit or 128)
+ self.readonly_only = readonly_only
+ self.runtime_readonly_guard = runtime_readonly_guard
+ self.execution_schema = str(execution_schema).strip() if execution_schema else None
+
+ @classmethod
+ def readonly_guard(cls, default_limit: int = 128, execution_schema: str | None = None):
+ return cls(
+ default_limit=default_limit,
+ readonly_only=True,
+ runtime_readonly_guard=True,
+ execution_schema=execution_schema,
+ )
+
+ def validate_and_patch(self, raw_sql: str) -> ValidatedGaussDBSQL:
+ sql = self.normalize_sql(raw_sql)
+ ast = self._parse_one(sql)
+ self._validate_readonly_ast(ast)
+ if self.runtime_readonly_guard:
+ self._validate_runtime_readonly_context(ast)
+ if not self.readonly_only:
+ self._validate_tables(ast)
+ self._validate_jsonb_paths(ast)
+ self._validate_columns(ast)
+ sql = self._enforce_kb_boundary(sql)
+ ast = self._parse_one(sql)
+ self._validate_tables(ast)
+ self._validate_jsonb_paths(ast)
+ self._validate_columns(ast)
+ sql = self._enforce_limit(sql)
+ ast = self._parse_one(sql)
+ self._validate_readonly_ast(ast)
+ if self.runtime_readonly_guard:
+ self._validate_runtime_readonly_context(ast)
+ if not self.readonly_only:
+ self._validate_tables(ast)
+ self._validate_jsonb_paths(ast)
+ self._validate_columns(ast)
+ if self.execution_schema:
+ self._qualify_execution_tables(ast)
+ sql = ast.sql(dialect=self.PARSE_DIALECT)
+ return ValidatedGaussDBSQL(
+ sql=sql,
+ columns=self._select_columns(ast),
+ is_aggregation=is_gaussdb_aggregate_sql(sql),
+ )
+
+ def normalize_sql(self, raw_sql: str) -> str:
+ sql = str(raw_sql or "").strip()
+ sql = re.sub(r"</think>\s*.*?\s*", "", sql, flags=re.DOTALL)
+ sql = re.sub(r"```(?:sql)?\s*", "", sql, flags=re.IGNORECASE)
+ sql = re.sub(r"```\s*$", "", sql, flags=re.IGNORECASE)
+ sql = sql.strip().strip("`").strip().rstrip(";").strip()
+ if not sql:
+ raise UnsafeGaussDBSQL("empty SQL")
+ if ";" in sql:
+ raise UnsafeGaussDBSQL("multiple statements are not allowed")
+ return sql
+
+ def _parse_one(self, sql: str):
+ exp, expressions = _parse_gaussdb_statements(sql, self.PARSE_DIALECT)
+ if len(expressions) != 1:
+ raise UnsafeGaussDBSQL("exactly one SQL statement is allowed")
+ if not isinstance(expressions[0], exp.Select):
+ raise UnsafeGaussDBSQL("only SELECT statements are allowed")
+ return expressions[0]
+
+ def _validate_readonly_ast(self, ast) -> None:
+ from sqlglot import exp
+
+ forbidden_types = (
+ exp.Delete,
+ exp.Update,
+ exp.Insert,
+ exp.Create,
+ exp.Drop,
+ exp.Command,
+ )
+ if any(isinstance(node, forbidden_types) for node in ast.walk()):
+ raise UnsafeGaussDBSQL("SQL contains a non-read-only expression")
+ if any(isinstance(node, exp.Lateral) for node in ast.walk()):
+ raise UnsafeGaussDBSQL("LATERAL is not allowed")
+ self._validate_limit_clauses(ast)
+ self._validate_jsonb_empty_string_comparisons(ast)
+ forbidden_system_types = tuple(
+ cls
+ for cls in (
+ getattr(exp, "CurrentUser", None),
+ getattr(exp, "CurrentDate", None),
+ getattr(exp, "CurrentTime", None),
+ getattr(exp, "CurrentTimestamp", None),
+ getattr(exp, "CurrentDatabase", None),
+ getattr(exp, "CurrentCatalog", None),
+ getattr(exp, "CurrentSchema", None),
+ )
+ if cls
+ )
+ for node in ast.walk():
+ if isinstance(node, exp.Star) and not isinstance(getattr(node, "parent", None), exp.Count):
+ raise UnsafeGaussDBSQL("SELECT * is not allowed")
+ if isinstance(node, exp.Window):
+ raise UnsafeGaussDBSQL("window functions are not allowed")
+ if isinstance(node, forbidden_system_types):
+ raise UnsafeGaussDBSQL("system functions are not allowed")
+ allowed_func_types = tuple(
+ cls
+ for cls in (
+ exp.Count,
+ exp.Sum,
+ exp.Avg,
+ exp.Max,
+ exp.Min,
+ exp.Cast,
+ getattr(exp, "StrToDate", None),
+ getattr(exp, "JSONBExtractScalar", None),
+ getattr(exp, "JSONBExtract", None),
+ )
+ if cls
+ )
+ for node in ast.find_all(exp.Func):
+ if isinstance(node, (exp.Binary, exp.Connector, exp.Predicate)) and not isinstance(node, allowed_func_types):
+ continue
+ name = str(getattr(node, "name", "") or "").lower()
+ if not name:
+ sql_name = str(node.sql_name() if hasattr(node, "sql_name") else "").lower()
+ name = sql_name.removeprefix("exploding_") or node.__class__.__name__.lower()
+ if isinstance(node, forbidden_system_types) or name in self.FORBIDDEN_FUNCTIONS:
+ raise UnsafeGaussDBSQL(f"function {name} is not allowed")
+ if not isinstance(node, allowed_func_types):
+ raise UnsafeGaussDBSQL(f"function {name} is not allowed")
+
+ def _validate_limit_clauses(self, ast) -> None:
+ from sqlglot import exp
+
+ for node in ast.walk():
+ if isinstance(node, exp.Fetch):
+ expression = node.args.get("count")
+ elif isinstance(node, exp.Limit):
+ expression = node.args.get("expression")
+ else:
+ continue
+ self._positive_static_limit_value(expression)
+
+ @staticmethod
+ def _positive_static_limit_value(expression) -> int:
+ from sqlglot import exp
+
+ if not isinstance(expression, exp.Literal) or expression.is_string or not re.fullmatch(r"[0-9]+", str(expression.this)) or int(expression.this) <= 0:
+ raise UnsafeGaussDBSQL("LIMIT must be a positive static integer")
+ return int(expression.this)
+
+ def _validate_jsonb_empty_string_comparisons(self, ast) -> None:
+ from sqlglot import exp
+
+ jsonb_text_types = (exp.JSONBExtractScalar,)
+
+ def is_empty_string(expression) -> bool:
+ while isinstance(expression, (exp.Cast, exp.Paren)):
+ expression = expression.this
+ return isinstance(expression, exp.Literal) and expression.is_string and expression.this == ""
+
+ def contains_jsonb_text(expression) -> bool:
+ return any(isinstance(node, jsonb_text_types) for node in expression.walk())
+
+ comparison_types = tuple(cls for cls in (exp.EQ, exp.NEQ, getattr(exp, "NullSafeEQ", None), getattr(exp, "NullSafeNEQ", None)) if cls)
+ for comparison in ast.walk():
+ if not isinstance(comparison, comparison_types):
+ continue
+ left, right = comparison.this, comparison.expression
+ if (is_empty_string(left) and contains_jsonb_text(right)) or (is_empty_string(right) and contains_jsonb_text(left)):
+ raise UnsafeGaussDBSQL("JSONB text cannot be compared with an empty SQL string")
+
+ def _validate_runtime_readonly_context(self, ast) -> None:
+ base_tables = self._base_tables(ast)
+ if not base_tables:
+ raise UnsafeGaussDBSQL("SQL must read from a DocEngine table")
+ for table in base_tables:
+ if not re.fullmatch(r"ragflow_[A-Za-z0-9_]{1,56}", table):
+ raise UnsafeGaussDBSQL(f"table {table} is not allowed")
+ if self._has_complex_boundary(ast.sql(dialect=self.PARSE_DIALECT)):
+ raise UnsafeGaussDBSQL("complex SQL must use a simpler single-table kb_id boundary")
+ for select in self._selects_with_base_tables(ast):
+ if not self._select_has_static_kb_boundary(select):
+ raise UnsafeGaussDBSQL("each base table scope must include a static kb_id boundary")
+ self._validate_runtime_columns(ast)
+ self._validate_jsonb_paths(ast)
+
+ def _validate_runtime_columns(self, ast) -> None:
+ from sqlglot import exp
+
+ allowed_columns = {"doc_id", "docnm_kwd", "kb_id", "chunk_data"}
+ cte_outputs = self._cte_output_columns(ast)
+ select_alias_refs = self._select_alias_reference_columns(ast)
+ for column in ast.find_all(exp.Column):
+ name = column.name
+ if name in allowed_columns:
+ continue
+ if self._is_cte_output_column(column, cte_outputs):
+ continue
+ if id(column) in select_alias_refs:
+ continue
+ raise UnsafeGaussDBSQL(f"column {name} is not allowed")
+
+ def _cte_names(self, ast) -> set[str]:
+ from sqlglot import exp
+
+ return {cte.alias_or_name for cte in ast.find_all(exp.CTE) if cte.alias_or_name}
+
+ def _base_tables(self, ast) -> list[str]:
+ from sqlglot import exp
+
+ cte_names = self._cte_names(ast)
+ tables = []
+ for table in ast.find_all(exp.Table):
+ if table.db or table.catalog:
+ raise UnsafeGaussDBSQL("cross-schema SQL is not allowed")
+ name = table.name
+ if name and name not in cte_names:
+ tables.append(name)
+ return tables
+
+ def _qualify_execution_tables(self, ast) -> None:
+ from sqlglot import exp
+
+ cte_names = self._cte_names(ast)
+ schema = exp.to_identifier(self.execution_schema, quoted=True)
+ for table in ast.find_all(exp.Table):
+ if table.db or table.catalog or table.name in cte_names:
+ continue
+ table.set("db", schema.copy())
+
+ def _validate_tables(self, ast) -> None:
+ allowed = {table.physical_name for table in self.tables.values()} | set(self.tables)
+ for table in self._base_tables(ast):
+ if table not in allowed:
+ raise UnsafeGaussDBSQL(f"table {table} is not allowed")
+
+ def _validate_columns(self, ast) -> None:
+ from sqlglot import exp
+
+ allowed_columns = set()
+ for table in self.tables.values():
+ allowed_columns.update(table.allowed_columns)
+ cte_outputs = self._cte_output_columns(ast)
+ select_alias_refs = self._select_alias_reference_columns(ast)
+ for column in ast.find_all(exp.Column):
+ name = column.name
+ if name in allowed_columns:
+ continue
+ if self._is_cte_output_column(column, cte_outputs):
+ continue
+ if id(column) in select_alias_refs:
+ continue
+ raise UnsafeGaussDBSQL(f"column {name} is not allowed")
+
+ def _select_alias_reference_columns(self, ast) -> set[int]:
+ from sqlglot import exp
+
+ refs = set()
+ for select in ast.find_all(exp.Select):
+ aliases = {str(getattr(expression, "alias_or_name", "") or "") for expression in getattr(select, "expressions", []) or [] if getattr(expression, "alias", None)}
+ aliases.discard("")
+ if not aliases:
+ continue
+ for clause_name in ("group", "order"):
+ clause = select.args.get(clause_name)
+ if not clause:
+ continue
+ for column in clause.find_all(exp.Column):
+ if not column.table and column.name in aliases:
+ refs.add(id(column))
+ return refs
+
+ def _cte_output_columns(self, ast) -> dict[str, set[str]]:
+ from sqlglot import exp
+
+ outputs: dict[str, set[str]] = {}
+ for cte in ast.find_all(exp.CTE):
+ name = cte.alias_or_name
+ query = cte.this
+ if not name or not isinstance(query, exp.Select):
+ continue
+ columns = set()
+ for expression in query.expressions or []:
+ alias = getattr(expression, "alias_or_name", None)
+ if alias:
+ columns.add(str(alias))
+ outputs[name] = columns
+ return outputs
+
+ def _is_cte_output_column(self, column, cte_outputs: dict[str, set[str]]) -> bool:
+ from sqlglot import exp
+
+ name = column.name
+ table = column.table
+ if table:
+ return table in cte_outputs and name in cte_outputs[table]
+ select = column.parent
+ while select is not None and not isinstance(select, exp.Select):
+ select = getattr(select, "parent", None)
+ if select is None:
+ return False
+ source_tables = self._direct_source_tables(select)
+ if not source_tables or not all(source in cte_outputs for source in source_tables):
+ return False
+ return any(name in cte_outputs[source] for source in source_tables)
+
+ def _validate_jsonb_paths(self, ast) -> None:
+ from sqlglot import exp
+
+ allowed_paths = {path for table in self.tables.values() for path in table.json_fields.values()}
+ json_classes = (exp.JSONExtractScalar, exp.JSONExtract)
+ jsonb_classes = (exp.JSONBExtractScalar, exp.JSONBExtract)
+ allowed_chunk_data_columns = set()
+ for node in ast.walk():
+ if isinstance(node, json_classes):
+ raise UnsafeGaussDBSQL("only GaussDB #> / #>> JSONB operators are allowed")
+ if not isinstance(node, jsonb_classes):
+ continue
+ source = node.this
+ if not isinstance(source, exp.Column) or source.name != "chunk_data":
+ raise UnsafeGaussDBSQL("only chunk_data JSONB paths are allowed")
+ allowed_chunk_data_columns.add(id(source))
+ expression = node.expression
+ if not isinstance(expression, exp.Literal) or not expression.is_string:
+ raise UnsafeGaussDBSQL("dynamic JSONB path is not allowed")
+ path = _parse_jsonb_path_literal(expression.this)
+ if not self.readonly_only and not allowed_paths:
+ raise UnsafeGaussDBSQL(f"JSONB path {path} is not exposed")
+ if allowed_paths and path not in allowed_paths:
+ raise UnsafeGaussDBSQL(f"JSONB path {path} is not exposed")
+ for column in ast.find_all(exp.Column):
+ if column.name == "chunk_data" and id(column) not in allowed_chunk_data_columns:
+ raise UnsafeGaussDBSQL("chunk_data may only be accessed through #> / #>>")
+
+ def _required_kb_ids(self) -> tuple[str, ...]:
+ kb_ids = []
+ for table in self.tables.values():
+ kb_ids.extend(table.required_kb_ids)
+ return tuple(dict.fromkeys(kb_ids))
+
+ def _enforce_kb_boundary(self, sql: str) -> str:
+ kb_ids = self._required_kb_ids()
+ if not kb_ids:
+ raise UnsafeGaussDBSQL("kb_id boundary is required")
+ ast = self._parse_one(sql)
+ if not self._base_tables(ast):
+ raise UnsafeGaussDBSQL("SQL must read from an exposed DocEngine table")
+ if self._has_complex_boundary(sql):
+ raise UnsafeGaussDBSQL("complex SQL must use a simpler single-table kb_id boundary")
+ missing = []
+ for select in self._selects_with_base_tables(ast):
+ if self._select_has_allowed_kb_boundary(select, kb_ids):
+ continue
+ missing.append(select)
+ if not missing:
+ return sql
+ if len(missing) == 1 and missing[0] is ast and len(self._selects_with_base_tables(ast)) == 1 and not ast.args.get("with_") and len(set(self._direct_base_tables(ast))) == 1:
+ if self._where_mentions_kb_id(ast):
+ raise UnsafeGaussDBSQL("kb_id boundary must be a positive top-level predicate")
+ return self._insert_condition(sql, self._kb_condition(kb_ids))
+ if len(set(self._base_tables(ast))) != 1:
+ raise UnsafeGaussDBSQL("SQL must reference exactly one base table to inject kb_id")
+ raise UnsafeGaussDBSQL("each base table scope must include a kb_id boundary")
+
+ def _has_complex_boundary(self, sql: str) -> bool:
+ from sqlglot import exp
+
+ ast = self._parse_one(sql)
+ if any(isinstance(node, exp.Or) for node in ast.walk()):
+ return True
+ if re.search(r"\b(join|union|intersect|except)\b", sql, flags=re.IGNORECASE):
+ return True
+ return False
+
+ def _selects_with_base_tables(self, ast) -> list:
+ from sqlglot import exp
+
+ return [select for select in ast.find_all(exp.Select) if self._direct_base_tables(select)]
+
+ def _direct_base_tables(self, select) -> list[str]:
+ from sqlglot import exp
+
+ cte_names = self._cte_names(select)
+ tables = []
+ from_expr = select.args.get("from_")
+ if from_expr and isinstance(from_expr.this, exp.Table):
+ if from_expr.this.name not in cte_names:
+ tables.append(from_expr.this.name)
+ for join in select.args.get("joins") or []:
+ target = join.this
+ if isinstance(target, exp.Table) and target.name not in cte_names:
+ tables.append(target.name)
+ return tables
+
+ def _direct_source_tables(self, select) -> list[str]:
+ from sqlglot import exp
+
+ tables = []
+ from_expr = select.args.get("from_")
+ if from_expr and isinstance(from_expr.this, exp.Table):
+ tables.append(from_expr.this.name)
+ for join in select.args.get("joins") or []:
+ target = join.this
+ if isinstance(target, exp.Table):
+ tables.append(target.name)
+ return tables
+
+ def _select_has_allowed_kb_boundary(self, select, allowed_kb_ids: tuple[str, ...]) -> bool:
+ from sqlglot import exp
+
+ where = select.args.get("where")
+ if not where:
+ return False
+ allowed = set(allowed_kb_ids)
+ for node in self._top_level_and_predicates(where.this):
+ if isinstance(node, exp.EQ):
+ values = self._kb_values_from_equality(node)
+ if values is None:
+ continue
+ if not values or not set(values).issubset(allowed):
+ raise UnsafeGaussDBSQL("SQL crosses the allowed kb_id boundary")
+ return True
+ elif isinstance(node, exp.In):
+ values = self._kb_values_from_in(node)
+ if values is None:
+ continue
+ if not values or not set(values).issubset(allowed):
+ raise UnsafeGaussDBSQL("SQL crosses the allowed kb_id boundary")
+ return True
+ return False
+
+ def _select_has_static_kb_boundary(self, select) -> bool:
+ from sqlglot import exp
+
+ where = select.args.get("where")
+ if not where:
+ return False
+ for node in self._top_level_and_predicates(where.this):
+ if isinstance(node, exp.EQ):
+ values = self._kb_values_from_equality(node)
+ elif isinstance(node, exp.In):
+ values = self._kb_values_from_in(node)
+ else:
+ values = None
+ if values is None:
+ continue
+ if not values:
+ raise UnsafeGaussDBSQL("kb_id boundary is empty")
+ self._validate_literal_values(values, "kb_id")
+ return True
+ return False
+
+ def _top_level_and_predicates(self, node) -> list:
+ from sqlglot import exp
+
+ if isinstance(node, exp.And):
+ return [*self._top_level_and_predicates(node.this), *self._top_level_and_predicates(node.expression)]
+ return [node]
+
+ def _where_mentions_kb_id(self, select) -> bool:
+ from sqlglot import exp
+
+ where = select.args.get("where")
+ if not where:
+ return False
+ return any(self._is_kb_column(column) for column in where.find_all(exp.Column))
+
+ def _kb_values_from_equality(self, node) -> list[str] | None:
+ from sqlglot import exp
+
+ left, right = node.this, node.expression
+ if self._is_kb_column(left) and isinstance(right, exp.Literal) and right.is_string:
+ return [str(right.this)]
+ if self._is_kb_column(right) and isinstance(left, exp.Literal) and left.is_string:
+ return [str(left.this)]
+ return None
+
+ def _kb_values_from_in(self, node) -> list[str] | None:
+ from sqlglot import exp
+
+ if not self._is_kb_column(node.this):
+ return None
+ values = []
+ for item in node.expressions or []:
+ if not isinstance(item, exp.Literal) or not item.is_string:
+ raise UnsafeGaussDBSQL("kb_id IN must use static string literals")
+ values.append(str(item.this))
+ return values
+
+ def _is_kb_column(self, node) -> bool:
+ from sqlglot import exp
+
+ return isinstance(node, exp.Column) and node.name.lower() == "kb_id"
+
+ def _kb_condition(self, kb_ids: tuple[str, ...]) -> str:
+ self._validate_literal_values(kb_ids, "kb_id")
+ if len(kb_ids) == 1:
+ return f"kb_id = '{kb_ids[0]}'"
+ return "kb_id IN (" + ", ".join(f"'{kid}'" for kid in kb_ids) + ")"
+
+ def _validate_literal_values(self, values: tuple[str, ...] | list[str], label: str) -> None:
+ for value in values:
+ if not re.fullmatch(r"[A-Za-z0-9_.:-]{1,256}", str(value)):
+ raise UnsafeGaussDBSQL(f"unsafe {label} literal")
+
+ def _insert_condition(self, sql: str, condition: str) -> str:
+ from sqlglot import exp
+
+ ast = self._parse_one(sql)
+ condition_ast = self._parse_one(f"SELECT 1 WHERE {condition}").args["where"].this
+ where = ast.args.get("where")
+ if where:
+ ast.set("where", exp.Where(this=exp.and_(where.this, condition_ast)))
+ else:
+ ast.set("where", exp.Where(this=condition_ast))
+ return ast.sql(dialect=self.PARSE_DIALECT)
+
+ def _enforce_limit(self, sql: str) -> str:
+ from sqlglot import exp
+
+ ast = self._parse_one(sql)
+ limit = ast.args.get("limit")
+ if not limit:
+ if self.default_limit <= 0:
+ return sql
+ return ast.limit(self.default_limit).sql(dialect=self.PARSE_DIALECT)
+ expression_key = "count" if isinstance(limit, exp.Fetch) else "expression"
+ current = self._positive_static_limit_value(limit.args.get(expression_key))
+ if self.default_limit <= 0 or current <= self.default_limit:
+ return sql
+ limit.set(expression_key, exp.Literal.number(self.default_limit))
+ return ast.sql(dialect=self.PARSE_DIALECT)
+
+ def _select_columns(self, ast) -> list[str]:
+ columns = []
+ for expression in getattr(ast, "expressions", []) or []:
+ alias = getattr(expression, "alias_or_name", None)
+ if alias:
+ columns.append(str(alias))
+ return columns
+
+
+class GaussDBDDLBuilder:
+ MAX_IDENTIFIER_LENGTH = 63
+ REGULAR_INDEX_COLUMNS = (
+ "doc_id",
+ "available_int",
+ "knowledge_graph_kwd",
+ "entity_type_kwd",
+ "removed_kwd",
+ )
+ FTS_COLUMNS = (
+ "title_tks",
+ "title_sm_tks",
+ "important_tks",
+ "question_tks",
+ "content_ltks",
+ "content_sm_ltks",
+ )
+
+ def __init__(self, schema: str):
+ self.schema = self.validate_identifier(schema)
+
+ def validate_identifier(self, name: str) -> str:
+ if not re.fullmatch(r"(?:[A-Za-z_]|[^\x00-\x7F])(?:[A-Za-z0-9_#$]|[^\x00-\x7F]){0,62}", name or ""):
+ raise InvalidGaussDBObjectName(name)
+ return name
+
+ def quote_identifier(self, name: str) -> str:
+ escaped = self.validate_identifier(name).replace('"', '""')
+ return f'"{escaped}"'
+
+ def qualified_name(self, table: str) -> str:
+ return f"{self.quote_identifier(self.schema)}.{self.quote_identifier(table)}"
+
+ def index_name(self, table: str, suffix: str) -> str:
+ name = f"idx_gdb_{self.validate_identifier(table)}_{self.validate_identifier(suffix)}"
+ if len(name) <= self.MAX_IDENTIFIER_LENGTH:
+ return name
+ digest = hashlib.sha1(name.encode("utf-8")).hexdigest()[:10]
+ prefix_len = self.MAX_IDENTIFIER_LENGTH - len(digest) - 1
+ return f"{name[:prefix_len]}_{digest}"
+
+ def build_chunk_table_ddl(self, table: str) -> str:
+ name = self.qualified_name(table)
+ return f"""CREATE TABLE IF NOT EXISTS {name} (
+ id VARCHAR(256) NOT NULL,
+ kb_id VARCHAR(256) NOT NULL,
+ doc_id VARCHAR(256),
+ docnm_kwd VARCHAR(256),
+ doc_type_kwd VARCHAR(256),
+ title_tks VARCHAR(256),
+ title_sm_tks VARCHAR(256),
+ content_with_weight TEXT,
+ content_ltks TEXT,
+ content_sm_ltks TEXT,
+ important_kwd JSONB,
+ important_tks TEXT,
+ question_kwd JSONB,
+ question_tks TEXT,
+ tag_kwd JSONB,
+ tag_feas JSONB,
+ available_int INTEGER DEFAULT 1 NOT NULL,
+ pagerank_fea INTEGER,
+ create_time VARCHAR(19),
+ create_timestamp_flt DOUBLE PRECISION,
+ img_id VARCHAR(128),
+ position_int JSONB,
+ page_num_int JSONB,
+ top_int JSONB,
+ metadata JSONB,
+ chunk_data JSONB,
+ extra JSONB,
+ _order_id INTEGER,
+ group_id VARCHAR(256),
+ mom_id VARCHAR(256),
+ knowledge_graph_kwd VARCHAR(256),
+ source_id JSONB,
+ entity_kwd VARCHAR(256),
+ entity_type_kwd VARCHAR(256),
+ from_entity_kwd VARCHAR(256),
+ to_entity_kwd VARCHAR(256),
+ weight_int INTEGER,
+ weight_flt DOUBLE PRECISION,
+ entities_kwd JSONB,
+ rank_flt DOUBLE PRECISION,
+ n_hop_with_weight TEXT,
+ removed_kwd VARCHAR(256) DEFAULT 'N',
+ raptor_kwd VARCHAR(256),
+ raptor_layer_int INTEGER,
+ PRIMARY KEY (kb_id, id)
+) WITH (storage_type=USTORE)"""
+
+ def build_doc_meta_table_ddls(self, meta_table: str) -> list[str]:
+ name = self.qualified_name(meta_table)
+ idx = self.quote_identifier(self.index_name(meta_table, "kb_id"))
+ return [
+ f"""CREATE TABLE IF NOT EXISTS {name} (
+ id VARCHAR(256) NOT NULL,
+ kb_id VARCHAR(256) NOT NULL,
+ meta_fields JSONB,
+ PRIMARY KEY (id)
+) WITH (storage_type=USTORE)""",
+ f"CREATE INDEX IF NOT EXISTS {idx} ON {name} (kb_id)",
+ ]
+
+ def build_regular_index_ddls(self, table: str) -> list[str]:
+ name = self.qualified_name(table)
+ return [f"CREATE INDEX IF NOT EXISTS {self.quote_identifier(self.index_name(table, column))} ON {name} ({column})" for column in self.REGULAR_INDEX_COLUMNS]
+
+ def build_fulltext_ugin_ddl(self, table: str) -> str:
+ name = self.qualified_name(table)
+ idx = self.quote_identifier(self.index_name(table, "fts_all"))
+ expression = " || ' ' || ".join(f"coalesce({column}, ' ')" for column in self.FTS_COLUMNS)
+ return f"""CREATE INDEX IF NOT EXISTS {idx}
+ ON {name}
+ USING ugin(to_tsvector('simple', {expression}))"""
+
+ def build_ngram_fulltext_ugin_ddl(self, table: str) -> str:
+ name = self.qualified_name(table)
+ idx = self.quote_identifier(self.index_name(table, "fts_all_ngram"))
+ expression = " || ' ' || ".join(f"coalesce({column}, ' ')" for column in self.FTS_COLUMNS)
+ return f"""CREATE INDEX IF NOT EXISTS {idx}
+ ON {name}
+ USING ugin(to_tsvector('ngram', {expression}))"""
+
+ def build_vector_column_ddls(self, table: str, dim: int) -> list[str]:
+ dim = self.validate_vector_dim(dim)
+ name = self.qualified_name(table)
+ vector_col = self.vector_column_name(dim)
+ valid_col = self.vector_valid_column_name(dim)
+ return [
+ f"ALTER TABLE {name} ADD COLUMN IF NOT EXISTS {vector_col} floatvector({dim}) DEFAULT (array_fill(0, ARRAY[{dim}])::text::floatvector({dim}))",
+ f"ALTER TABLE {name} ADD COLUMN IF NOT EXISTS {valid_col} BOOLEAN DEFAULT FALSE NOT NULL",
+ ]
+
+ def build_diskann_index_ddl(self, table: str, dim: int) -> str:
+ dim = self.validate_vector_dim(dim)
+ name = self.qualified_name(table)
+ vector_col = self.vector_column_name(dim)
+ idx = self.quote_identifier(self.index_name(table, f"{vector_col}_diskann"))
+ options = "subgraph_count=1"
+ if dim > 1024:
+ options += ", enable_vector_copy=false"
+ return f"CREATE INDEX IF NOT EXISTS {idx} ON {name} USING gsdiskann ({vector_col} COSINE) WITH ({options})"
+
+ def build_advisory_lock_sql(self, lock_name: str) -> tuple[str, list[str]]:
+ return ("SELECT pg_advisory_xact_lock(hashtext(%s))", [str(lock_name)])
+
+ def validate_vector_dim(self, dim: int) -> int:
+ if isinstance(dim, bool) or not isinstance(dim, Integral):
+ raise ValueError("vector dimension must be an integer")
+ value = int(dim)
+ if value <= 0:
+ raise ValueError("vector dimension must be positive")
+ if value > 4096:
+ raise ValueError("GaussDB floatvector dimensions cannot exceed 4096")
+ return value
+
+ def vector_column_name(self, dim: int) -> str:
+ return f"q_{self.validate_vector_dim(dim)}_vec"
+
+ def vector_valid_column_name(self, dim: int) -> str:
+ return f"q_{self.validate_vector_dim(dim)}_vec_valid"
+
+
+class GaussDBSearchBuilder:
+ CHUNK_COLUMNS = {
+ "id",
+ "kb_id",
+ "doc_id",
+ "docnm_kwd",
+ "doc_type_kwd",
+ "title_tks",
+ "title_sm_tks",
+ "content_with_weight",
+ "content_ltks",
+ "content_sm_ltks",
+ "important_kwd",
+ "important_tks",
+ "question_kwd",
+ "question_tks",
+ "tag_kwd",
+ "tag_feas",
+ "available_int",
+ "pagerank_fea",
+ "create_time",
+ "create_timestamp_flt",
+ "img_id",
+ "position_int",
+ "page_num_int",
+ "top_int",
+ "metadata",
+ "chunk_data",
+ "extra",
+ "_order_id",
+ "chunk_order_int",
+ "group_id",
+ "mom_id",
+ "knowledge_graph_kwd",
+ "source_id",
+ "entity_kwd",
+ "entity_type_kwd",
+ "from_entity_kwd",
+ "to_entity_kwd",
+ "weight_int",
+ "weight_flt",
+ "entities_kwd",
+ "rank_flt",
+ "n_hop_with_weight",
+ "removed_kwd",
+ "raptor_kwd",
+ "raptor_layer_int",
+ "compile_kwd",
+ "row_id()",
+ }
+ JSONB_MULTI_VALUE_COLUMNS = {"important_kwd", "question_kwd", "tag_kwd", "source_id", "entities_kwd"}
+ JSONB_ARRAY_AGG_COLUMNS = JSONB_MULTI_VALUE_COLUMNS | {"entities_kwd"}
+ JSONB_EXTRA_SCALAR_COLUMNS = {"compile_kwd"}
+ COLUMN_ALIASES = {"chunk_order_int": "_order_id"}
+ FTS_WEIGHTS = {
+ "title_tks": 10.0,
+ "title_sm_tks": 5.0,
+ "important_tks": 20.0,
+ "question_tks": 20.0,
+ "content_ltks": 2.0,
+ "content_sm_ltks": 1.0,
+ }
+ VECTOR_COLUMN_RE = re.compile(r"^q_(?P<dim>\d+)_vec$")
+ VECTOR_VALID_COLUMN_RE = re.compile(r"^q_(?P<dim>\d+)_vec_valid$")
+ CJK_RUN_RE = re.compile(r"[\u3400-\u4dbf\u4e00-\u9fff\uf900-\ufaff]+")
+
+ def __init__(self, schema: str):
+ self.ddl = GaussDBDDLBuilder(schema=schema)
+
+ def build_search_sql(
+ self,
+ table: str,
+ select_fields: list[str],
+ condition: dict,
+ keywords: list[str] | None,
+ vector: list[float] | None,
+ vector_dim: int | None,
+ vector_weight: float,
+ offset: int,
+ limit: int,
+ similarity_threshold: float | None = None,
+ topn: int | None = None,
+ highlight_fields: list[str] | None = None,
+ order_by: OrderByExpr | None = None,
+ pagerank_weight: float = 0.0,
+ ) -> tuple[str, list[Any]]:
+ query_keywords = [str(keyword).strip() for keyword in keywords or [] if str(keyword).strip()]
+ has_searchable_text = any(self.split_text_query_terms(query_keywords))
+ query_vector = self._vector_param(vector, vector_dim) if vector is not None else None
+ effective_limit = max(int(limit or 0), 0)
+ effective_offset = max(int(offset or 0), 0)
+ page_limit = effective_limit if effective_limit > 0 else max(int(topn or 0), 10000)
+ candidate_limit = self._candidate_limit(page_limit, effective_offset, topn)
+
+ if has_searchable_text and query_vector is not None:
+ return self._build_hybrid_search_sql(
+ table=table,
+ select_fields=select_fields,
+ condition=condition,
+ keywords=query_keywords,
+ vector=query_vector,
+ vector_dim=vector_dim,
+ vector_weight=vector_weight,
+ similarity_threshold=similarity_threshold,
+ candidate_limit=candidate_limit,
+ offset=effective_offset,
+ limit=page_limit,
+ highlight_fields=highlight_fields,
+ pagerank_weight=pagerank_weight,
+ )
+ if query_vector is not None:
+ return self._build_vector_search_sql(
+ table=table,
+ select_fields=select_fields,
+ condition=condition,
+ vector=query_vector,
+ vector_dim=vector_dim,
+ similarity_threshold=similarity_threshold,
+ candidate_limit=candidate_limit,
+ offset=effective_offset,
+ limit=page_limit,
+ pagerank_weight=pagerank_weight,
+ )
+ if query_keywords:
+ return self._build_fulltext_search_sql(
+ table=table,
+ select_fields=select_fields,
+ condition=condition,
+ keywords=query_keywords,
+ offset=effective_offset,
+ limit=page_limit,
+ highlight_fields=highlight_fields,
+ pagerank_weight=pagerank_weight,
+ )
+ return self._build_filter_search_sql(
+ table=table,
+ select_fields=select_fields,
+ condition=condition,
+ offset=effective_offset,
+ limit=page_limit,
+ order_by=order_by,
+ pagerank_weight=pagerank_weight,
+ )
+
+ def build_condition_where(self, condition: dict | None) -> tuple[str, list[Any]]:
+ fragments: list[str] = []
+ params: list[Any] = []
+ for key, value in (condition or {}).items():
+ if key == "exists":
+ column = self.validate_column(value)
+ expression = self._column_expr(column)
+ if self._is_jsonb_dynamic_column(column):
+ fragments.append(f"({expression} IS NOT NULL AND {expression} <> 'null'::jsonb)")
+ else:
+ fragments.append(f"{expression} IS NOT NULL")
+ continue
+ if key == "must_not" and isinstance(value, dict) and "exists" in value:
+ column = self.validate_column(value["exists"])
+ expression = self._column_expr(column)
+ if self._is_jsonb_dynamic_column(column):
+ fragments.append(f"({expression} IS NULL OR {expression} = 'null'::jsonb)")
+ else:
+ fragments.append(f"{expression} IS NULL")
+ continue
+
+ column = self.validate_column(key)
+ expression = self._column_expr(column)
+ if self._is_jsonb_dynamic_column(column):
+ values = self._list_values(value)
+ predicates = []
+ for item in values:
+ predicates.append(f"({expression} = %s::jsonb OR {expression} @> %s::jsonb)")
+ params.extend(
+ [
+ json.dumps(item, ensure_ascii=False),
+ json.dumps([item], ensure_ascii=False),
+ ]
+ )
+ fragments.append("(" + " OR ".join(predicates) + ")")
+ continue
+ storage_column = self._storage_column(column)
+ if storage_column in self.JSONB_MULTI_VALUE_COLUMNS:
+ values = self._list_values(value)
+ fragments.append("(" + " OR ".join([f"{storage_column} @> %s::jsonb"] * len(values)) + ")")
+ params.extend(json.dumps([item], ensure_ascii=False) for item in values)
+ continue
+ if isinstance(value, (list, tuple, set)):
+ values = self._list_values(value)
+ fragments.append(f"{expression} IN ({', '.join(['%s'] * len(values))})")
+ params.extend(values)
+ continue
+ if value is None:
+ fragments.append(f"{expression} IS NULL")
+ continue
+ fragments.append(f"{expression} = %s")
+ params.append(value)
+ return " AND ".join(fragments), params
+
+ def build_text_score_expr(self, keywords: list[str]) -> tuple[str, list[Any]]:
+ score_exprs: list[str] = []
+ params: list[Any] = []
+ for config, query_text in self._text_queries(keywords):
+ weighted_score = " + ".join(
+ f"{weight} * COALESCE(ts_rank(to_tsvector('{config}', coalesce({column}, ' ')), plainto_tsquery('{config}', %s)), 0)" for column, weight in self.FTS_WEIGHTS.items()
+ )
+ score_exprs.append(f"({weighted_score})")
+ params.extend([query_text] * len(self.FTS_WEIGHTS))
+ if not score_exprs:
+ return "0.0", []
+ if len(score_exprs) == 1:
+ return score_exprs[0], params
+ return f"(({' + '.join(score_exprs)}) / {len(score_exprs)}.0)", params
+
+ def build_highlight_expr(self, field_name: str, keywords: list[str]) -> tuple[str, list[Any]]:
+ field = self.validate_column(field_name)
+ expression = self._column_expr(field, text=self._is_dynamic_column(field))
+ _simple_terms, ngram_terms = self.split_text_query_terms(keywords)
+ if ngram_terms:
+ return f"COALESCE({expression}, ' ') AS _highlight_source", []
+ return (
+ f"ts_headline('simple', COALESCE({expression}, ' '), plainto_tsquery('simple', %s), 'StartSel=<em>, StopSel=</em>') AS _highlight",
+ [self._text_query_param(keywords)],
+ )
+
+ def build_aggregation_sql(
+ self,
+ table: str,
+ field_name: str,
+ condition: dict | None,
+ limit: int = 1000,
+ ) -> tuple[str, list[Any]]:
+ table_name = self.ddl.qualified_name(table)
+ field = self.validate_column(field_name)
+ field_expression = self._column_expr(field)
+ where_sql, where_params = self.build_condition_where(condition)
+ where_clause = where_sql or "TRUE"
+ if field in self.JSONB_ARRAY_AGG_COLUMNS:
+ sql = (
+ "SELECT value, COUNT(1) AS count "
+ "FROM ("
+ f"SELECT jsonb_array_elements_text(COALESCE({field_expression}, '[]'::jsonb)) AS value "
+ f"FROM {table_name} WHERE {where_clause}"
+ ") AS expanded "
+ "GROUP BY value ORDER BY count DESC, value ASC LIMIT %s"
+ )
+ else:
+ sql = (
+ f"SELECT {field_expression} AS value, COUNT(1) AS count "
+ f"FROM {table_name} "
+ f"WHERE {where_clause} AND {field_expression} IS NOT NULL "
+ "GROUP BY value ORDER BY count DESC, value ASC LIMIT %s"
+ )
+ return sql, [*where_params, int(limit)]
+
+ def build_position_order_sql(self) -> str:
+ return "COALESCE((page_num_int #>> '{0}')::int, 100000000) ASC, COALESCE((position_int #>> '{0,3}')::int, 100000000) ASC, COALESCE((top_int #>> '{0}')::int, 100000000) ASC"
+
+ def build_fts_vector_expr(self, config: str = "simple") -> str:
+ expression = " || ' ' || ".join(f"coalesce({column}, ' ')" for column in self.FTS_WEIGHTS)
+ return f"to_tsvector('{config}', {expression})"
+
+ def validate_column(self, column: str) -> str:
+ if column == "row_id()":
+ return column
+ column = str(column or "")
+ if column in self.CHUNK_COLUMNS or self.VECTOR_COLUMN_RE.fullmatch(column) or self.VECTOR_VALID_COLUMN_RE.fullmatch(column):
+ return column
+ return validate_extra_field(column)
+
+ def normalize_select_fields(self, fields: list[str] | None) -> list[str]:
+ if not fields or "*" in fields:
+ return ["id", "kb_id"]
+ normalized: list[str] = ["id", "kb_id"]
+ for field in fields:
+ if field == "_score":
+ continue
+ column = self.validate_column(field)
+ if column not in normalized:
+ normalized.append(column)
+ match = self.VECTOR_COLUMN_RE.fullmatch(column)
+ if match:
+ valid_column = self.ddl.vector_valid_column_name(int(match.group("dim")))
+ if valid_column not in normalized:
+ normalized.append(valid_column)
+ return normalized
+
+ def _build_filter_search_sql(
+ self,
+ table: str,
+ select_fields: list[str],
+ condition: dict,
+ offset: int,
+ limit: int,
+ order_by: OrderByExpr | None,
+ pagerank_weight: float,
+ ) -> tuple[str, list[Any]]:
+ table_name = self.ddl.qualified_name(table)
+ columns = self.normalize_select_fields(select_fields)
+ where_sql, where_params = self.build_condition_where(condition)
+ order_sql = self._build_order_by(order_by) or "kb_id ASC, id ASC"
+ score_expr, score_params = self._score_with_pagerank("0.0", pagerank_weight)
+ sql = f"SELECT {', '.join(self._select_exprs(columns))}, {score_expr} AS _score, COUNT(*) OVER() AS __total FROM {table_name}"
+ if where_sql:
+ sql += f" WHERE {where_sql}"
+ sql += f" ORDER BY {order_sql} LIMIT %s OFFSET %s"
+ return sql, [*score_params, *where_params, limit, offset]
+
+ def _build_fulltext_search_sql(
+ self,
+ table: str,
+ select_fields: list[str],
+ condition: dict,
+ keywords: list[str],
+ offset: int,
+ limit: int,
+ highlight_fields: list[str] | None,
+ pagerank_weight: float,
+ ) -> tuple[str, list[Any]]:
+ table_name = self.ddl.qualified_name(table)
+ columns = self.normalize_select_fields(select_fields)
+ score_expr, score_params = self.build_text_score_expr(keywords)
+ score_expr, pagerank_params = self._score_with_pagerank(score_expr, pagerank_weight)
+ match_expr, match_params = self._build_text_match_expr(keywords)
+ where_sql, where_params = self.build_condition_where(condition)
+ where_parts = [part for part in (where_sql, match_expr) if part]
+ select_exprs = [*self._select_exprs(columns), f"{score_expr} AS _score", "COUNT(*) OVER() AS __total"]
+ highlight_params: list[Any] = []
+ if highlight_fields:
+ highlight_expr, highlight_params = self.build_highlight_expr(highlight_fields[0], keywords)
+ select_exprs.append(highlight_expr)
+ sql = f"SELECT {', '.join(select_exprs)} FROM {table_name}"
+ if where_parts:
+ sql += f" WHERE {' AND '.join(where_parts)}"
+ sql += " ORDER BY _score DESC, kb_id ASC, id ASC LIMIT %s OFFSET %s"
+ return sql, [*score_params, *pagerank_params, *highlight_params, *where_params, *match_params, limit, offset]
+
+ def _build_vector_search_sql(
+ self,
+ table: str,
+ select_fields: list[str],
+ condition: dict,
+ vector: str,
+ vector_dim: int,
+ similarity_threshold: float | None,
+ candidate_limit: int,
+ offset: int,
+ limit: int,
+ pagerank_weight: float,
+ ) -> tuple[str, list[Any]]:
+ table_name = self.ddl.qualified_name(table)
+ columns = self.normalize_select_fields(select_fields)
+ dim = self.ddl.validate_vector_dim(vector_dim)
+ vector_col = self.ddl.vector_column_name(dim)
+ valid_col = self.ddl.vector_valid_column_name(dim)
+ where_sql, where_params = self.build_condition_where(condition)
+ where_parts = [part for part in (where_sql, f"{valid_col} = TRUE") if part]
+ threshold = 0.0 if similarity_threshold is None else float(similarity_threshold)
+ score_expr, score_params = self._score_with_pagerank(f"1 - ({vector_col} <+> %s::floatvector({dim}))", pagerank_weight)
+ sql = (
+ "WITH vec AS ("
+ f" SELECT {', '.join(self._select_exprs(columns))}, "
+ f"{vector_col} <+> %s::floatvector({dim}) AS distance, "
+ f"{score_expr} AS _score "
+ f"FROM {table_name} "
+ f"WHERE {' AND '.join(where_parts)} "
+ f"ORDER BY {vector_col} <+> %s::floatvector({dim}) ASC "
+ "LIMIT %s"
+ ") "
+ "SELECT vec.*, COUNT(*) OVER() AS __total FROM vec "
+ "WHERE _score >= %s "
+ "ORDER BY distance ASC, kb_id ASC, id ASC LIMIT %s OFFSET %s"
+ )
+ return sql, [vector, vector, *score_params, *where_params, vector, candidate_limit, threshold, limit, offset]
+
+ def _build_hybrid_search_sql(
+ self,
+ table: str,
+ select_fields: list[str],
+ condition: dict,
+ keywords: list[str],
+ vector: str,
+ vector_dim: int,
+ vector_weight: float,
+ similarity_threshold: float | None,
+ candidate_limit: int,
+ offset: int,
+ limit: int,
+ highlight_fields: list[str] | None,
+ pagerank_weight: float,
+ ) -> tuple[str, list[Any]]:
+ table_name = self.ddl.qualified_name(table)
+ columns = self.normalize_select_fields(select_fields)
+ joined_columns = ", ".join(self._select_exprs(columns, prefix="c"))
+ dim = self.ddl.validate_vector_dim(vector_dim)
+ vector_col = self.ddl.vector_column_name(dim)
+ valid_col = self.ddl.vector_valid_column_name(dim)
+ text_score_expr, text_score_params = self.build_text_score_expr(keywords)
+ match_expr, match_params = self._build_text_match_expr(keywords)
+ where_sql, where_params = self.build_condition_where(condition)
+ base_where = where_sql or "TRUE"
+ fts_where = " AND ".join([base_where, match_expr])
+ vector_where = " AND ".join([base_where, f"{valid_col} = TRUE"])
+ threshold = 0.0 if similarity_threshold is None else float(similarity_threshold)
+ select_exprs = [joined_columns, "merged.score AS _score", "COUNT(*) OVER() AS __total"]
+ highlight_params: list[Any] = []
+ if highlight_fields:
+ highlight_expr, highlight_params = self.build_highlight_expr(highlight_fields[0], keywords)
+ select_exprs.append(highlight_expr)
+ final_score_expr, final_score_params = self._score_with_pagerank("merged.score", pagerank_weight, table_alias="c")
+ select_exprs[1] = f"{final_score_expr} AS _score"
+ threshold_expr, threshold_score_params = self._score_with_pagerank("merged.score", pagerank_weight, table_alias="c")
+ sql = (
+ "WITH fts_raw AS ("
+ f" SELECT kb_id, id, {text_score_expr} AS raw_fts_score "
+ f"FROM {table_name} WHERE {fts_where} "
+ "ORDER BY raw_fts_score DESC, kb_id ASC, id ASC LIMIT %s"
+ "), fts AS ("
+ " SELECT kb_id, id, COALESCE(raw_fts_score / NULLIF(MAX(raw_fts_score) OVER (), 0), 0) AS fts_score "
+ "FROM fts_raw"
+ "), vec AS ("
+ f" SELECT kb_id, id, 1 - ({vector_col} <+> %s::floatvector({dim})) AS vector_score "
+ f"FROM {table_name} WHERE {vector_where} "
+ f"ORDER BY {vector_col} <+> %s::floatvector({dim}) ASC LIMIT %s"
+ "), merged AS ("
+ " SELECT COALESCE(fts.kb_id, vec.kb_id) AS kb_id, "
+ "COALESCE(fts.id, vec.id) AS id, "
+ "(1 - %s) * COALESCE(fts.fts_score, 0) + %s * COALESCE(vec.vector_score, 0) AS score "
+ "FROM fts FULL OUTER JOIN vec ON fts.kb_id = vec.kb_id AND fts.id = vec.id"
+ ") "
+ f"SELECT {', '.join(select_exprs)} "
+ f"FROM merged JOIN {table_name} c ON c.kb_id = merged.kb_id AND c.id = merged.id "
+ f"WHERE {threshold_expr} >= %s "
+ "ORDER BY _score DESC, merged.kb_id ASC, merged.id ASC LIMIT %s OFFSET %s"
+ )
+ return sql, [
+ *text_score_params,
+ *where_params,
+ *match_params,
+ candidate_limit,
+ vector,
+ *where_params,
+ vector,
+ candidate_limit,
+ float(vector_weight),
+ float(vector_weight),
+ *final_score_params,
+ *highlight_params,
+ *threshold_score_params,
+ threshold,
+ limit,
+ offset,
+ ]
+
+ def _build_text_match_expr(self, keywords: list[str]) -> tuple[str, list[Any]]:
+ match_exprs: list[str] = []
+ params: list[Any] = []
+ for config, query_text in self._text_queries(keywords):
+ match_exprs.append(f"{self.build_fts_vector_expr(config)} @@ plainto_tsquery('{config}', %s)")
+ params.append(query_text)
+ return (" AND ".join(match_exprs), params) if match_exprs else ("FALSE", [])
+
+ def _build_order_by(self, order_by: OrderByExpr | None) -> str:
+ fields = getattr(order_by, "fields", None) or []
+ parts: list[str] = []
+ for field, direction in fields:
+ column = self.validate_column(field)
+ order = "DESC" if direction else "ASC"
+ if column in {"page_num_int", "position_int", "top_int"}:
+ parts.append(self.build_position_order_sql())
+ else:
+ parts.append(f"{self._column_expr(column, text=self._is_dynamic_column(column))} {order}")
+ return ", ".join(parts)
+
+ def _select_exprs(self, columns: list[str], prefix: str | None = None) -> list[str]:
+ expressions = []
+ for column in columns:
+ if column == "row_id()":
+ expressions.append('NULL AS "row_id()"')
+ elif self._is_dynamic_column(column):
+ expressions.append(f"{self._column_expr(column, prefix=prefix)} AS {column}")
+ elif column in self.COLUMN_ALIASES:
+ storage_column = self._storage_column(column)
+ source = f"{prefix}.{storage_column}" if prefix else storage_column
+ expressions.append(f"{source} AS {column}")
+ elif prefix:
+ expressions.append(f"{prefix}.{column}")
+ else:
+ expressions.append(column)
+ return expressions
+
+ def _is_dynamic_column(self, column: str) -> bool:
+ return column in self.JSONB_EXTRA_SCALAR_COLUMNS or not (
+ column in self.CHUNK_COLUMNS or column == "row_id()" or self.VECTOR_COLUMN_RE.fullmatch(column) or self.VECTOR_VALID_COLUMN_RE.fullmatch(column)
+ )
+
+ def _is_jsonb_dynamic_column(self, column: str) -> bool:
+ return self._is_dynamic_column(column) and column not in self.JSONB_EXTRA_SCALAR_COLUMNS
+
+ def _column_expr(
+ self,
+ column: str,
+ *,
+ prefix: str | None = None,
+ text: bool = False,
+ ) -> str:
+ if self._is_dynamic_column(column):
+ if column in self.JSONB_EXTRA_SCALAR_COLUMNS:
+ source = f"{prefix}.extra" if prefix else "extra"
+ return f"({source} #>> '{{{column}}}')"
+ return extra_field_expr(
+ column,
+ prefix=prefix,
+ text=text,
+ )
+ storage_column = self._storage_column(column)
+ return f"{prefix}.{storage_column}" if prefix else storage_column
+
+ def _storage_column(self, column: str) -> str:
+ if column in self.JSONB_EXTRA_SCALAR_COLUMNS:
+ return f"(extra #>> '{{{column}}}')"
+ return self.COLUMN_ALIASES.get(column, column)
+
+ def _score_with_pagerank(self, score_expr: str, pagerank_weight: float, table_alias: str | None = None) -> tuple[str, list[Any]]:
+ weight = float(pagerank_weight or 0.0)
+ if weight <= 0.0:
+ return score_expr, []
+ column = "pagerank_fea" if table_alias is None else f"{table_alias}.pagerank_fea"
+ pagerank_expr = f"(COALESCE({column}, 0)::DOUBLE PRECISION / 100.0 * %s)"
+ return f"({score_expr} + {pagerank_expr})", [weight]
+
+ def _text_query_param(self, keywords: list[str]) -> str:
+ return " ".join(str(keyword).strip() for keyword in keywords if str(keyword).strip())
+
+ @classmethod
+ def split_text_query_terms(cls, keywords: list[str]) -> tuple[list[str], list[str]]:
+ simple_terms: list[str] = []
+ ngram_terms: list[str] = []
+ for keyword in keywords:
+ text = str(keyword).strip()
+ if not text:
+ continue
+ matches = list(cls.CJK_RUN_RE.finditer(text))
+ if not matches:
+ simple_terms.append(text)
+ continue
+ cursor = 0
+ for match in matches:
+ simple_part = text[cursor : match.start()].strip()
+ if simple_part and any(char.isalnum() for char in simple_part):
+ simple_terms.append(simple_part)
+ cjk_part = match.group()
+ if len(cjk_part) >= 2:
+ ngram_terms.append(cjk_part)
+ cursor = match.end()
+ simple_part = text[cursor:].strip()
+ if simple_part and any(char.isalnum() for char in simple_part):
+ simple_terms.append(simple_part)
+ return simple_terms, ngram_terms
+
+ def _text_queries(self, keywords: list[str]) -> list[tuple[str, str]]:
+ simple_terms, ngram_terms = self.split_text_query_terms(keywords)
+ queries = []
+ if simple_terms:
+ queries.append(("simple", self._text_query_param(simple_terms)))
+ if ngram_terms:
+ queries.append(("ngram", self._text_query_param(ngram_terms)))
+ return queries
+
+ def _vector_param(self, vector: list[float] | tuple[float, ...], vector_dim: int | None) -> str:
+ if vector_dim is None:
+ raise ValueError("vector_dim is required for vector search")
+ dim = self.ddl.validate_vector_dim(vector_dim)
+ values = list(vector)
+ if len(values) != dim:
+ raise ValueError(f"vector dimension mismatch: expected {dim}, got {len(values)}")
+ return "[" + ",".join(str(float(value)) for value in values) + "]"
+
+ def _list_values(self, value) -> list[Any]:
+ values = list(value) if isinstance(value, (list, tuple, set)) else [value]
+ if not values:
+ raise ValueError("empty condition values are not supported")
+ return values
+
+ def _candidate_limit(self, limit: int, offset: int, topn: int | None) -> int:
+ base = max(int(limit or 0) + int(offset or 0), int(limit or 0), 1)
+ if topn and int(topn) > 0:
+ base = max(base, int(topn))
+ return base
+
+
+class GaussDBConnectionBase(DocStoreConnection):
+ def __init__(self, pool: Any | None = None, logger_name: str = "ragflow.gaussdb_conn"):
+ if pool is None:
+ from common.doc_store.gaussdb_conn_pool import GAUSSDB_CONN
+
+ pool = GAUSSDB_CONN
+ self.logger = logging.getLogger(logger_name)
+ self.pool = pool
+ self.masked_uri = self.pool.masked_uri
+ self.resolved_schema = self.pool.resolved_schema
+ self.schema = self.resolved_schema
+ self.ddl = GaussDBDDLBuilder(schema=self.resolved_schema)
+ self.pool.check_schema_access()
+ self.logger.info("GaussDB %s connection initialized.", self.masked_uri)
+
+ def db_type(self) -> str:
+ return "gaussdb"
+
+ def health(self) -> dict:
+ result = {
+ "status": "unhealthy",
+ "uri": self.masked_uri,
+ "version_comment": "unknown",
+ "schema": self.resolved_schema,
+ "server_encoding": "unknown",
+ "client_encoding": "unknown",
+ }
+ try:
+ result["version_comment"] = self._query_version()
+ result["sql_compatibility"] = self._query_sql_compatibility()
+ if result["sql_compatibility"] not in {"A", "ORA"}:
+ result["error"] = f"unsupported GaussDB compatibility, expected A/ORA: sql_compatibility={result['sql_compatibility']}"
+ return result
+ result["server_encoding"] = self._query_server_encoding()
+ result["client_encoding"] = self._query_client_encoding()
+ normalized_client_encoding = result["client_encoding"].replace("-", "").replace("_", "")
+ if normalized_client_encoding != "UTF8":
+ result["error"] = f"unsupported GaussDB client encoding, expected UTF8: client_encoding={result['client_encoding']}"
+ return result
+ if result["server_encoding"] == "SQL_ASCII":
+ result["warning"] = "server_encoding=SQL_ASCII does not validate or convert stored bytes; RAGFlow clients are forced to UTF8"
+ result["status"] = "healthy"
+ return result
+ except Exception as exc:
+ result["error"] = mask_gaussdb_text(exc)
+ return result
+
+ def _query_version(self) -> str:
+ return self._query_required_scalar("SELECT version()", "version")
+
+ def _query_sql_compatibility(self) -> str:
+ return self._query_required_scalar("SHOW sql_compatibility", "sql_compatibility").upper()
+
+ def _query_server_encoding(self) -> str:
+ return self._query_required_scalar("SHOW server_encoding", "server_encoding").upper()
+
+ def _query_client_encoding(self) -> str:
+ return self._query_required_scalar("SHOW client_encoding", "client_encoding").upper()
+
+ def _query_required_scalar(self, sql: str, field_name: str) -> str:
+ row = self.pool.fetch_one(sql)
+ if not row or row[0] is None or str(row[0]).strip() == "":
+ raise RuntimeError(f"GaussDB {field_name} query returned no rows")
+ return str(row[0]).strip()
+
+ def get_performance_metrics(self) -> dict:
+ st = timer()
+ try:
+ self.pool.fetch_one("SELECT 1")
+ return {
+ "connection": "connected",
+ "latency_ms": round((timer() - st) * 1000.0, 3),
+ "schema": self.resolved_schema,
+ }
+ except Exception as exc:
+ return {
+ "connection": "disconnected",
+ "latency_ms": round((timer() - st) * 1000.0, 3),
+ "error": mask_gaussdb_text(exc),
+ }
+
+ def create_idx(self, index_name: str, dataset_id: str, vector_size: int, parser_id: str = None):
+ raise NotImplementedError("GaussDB create_idx is implemented in the DDL task")
+
+ def delete_idx(self, index_name: str, dataset_id: str):
+ raise NotImplementedError("GaussDB delete_idx is implemented in the CRUD task")
+
+ def index_exist(self, index_name: str, dataset_id: str) -> bool:
+ raise NotImplementedError("GaussDB index_exist is implemented in the DDL task")
+
+ def search(
+ self,
+ select_fields: list[str],
+ highlight_fields: list[str],
+ condition: dict,
+ match_expressions: list[MatchExpr],
+ order_by: OrderByExpr,
+ offset: int,
+ limit: int,
+ index_names: str | list[str],
+ dataset_ids: list[str],
+ agg_fields: list[str] | None = None,
+ rank_feature: dict | None = None,
+ ):
+ raise NotImplementedError("GaussDB search is implemented in the search task")
+
+ def get(self, data_id: str, index_name: str, dataset_ids: list[str]) -> dict | None:
+ raise NotImplementedError("GaussDB get is implemented in the CRUD task")
+
+ def insert(self, rows: list[dict], index_name: str, dataset_id: str = None) -> list[str]:
+ raise NotImplementedError("GaussDB insert is implemented in the CRUD task")
+
+ def update(self, condition: dict, new_value: dict, index_name: str, dataset_id: str) -> bool:
+ raise NotImplementedError("GaussDB update is implemented in the CRUD task")
+
+ def delete(self, condition: dict, index_name: str, dataset_id: str) -> int:
+ raise NotImplementedError("GaussDB delete is implemented in the CRUD task")
+
+ def get_total(self, res):
+ raise NotImplementedError("GaussDB get_total is implemented in the adapter task")
+
+ def get_doc_ids(self, res):
+ raise NotImplementedError("GaussDB get_doc_ids is implemented in the adapter task")
+
+ def get_fields(self, res, fields: list[str]) -> dict[str, dict]:
+ raise NotImplementedError("GaussDB get_fields is implemented in the adapter task")
+
+ def get_highlight(self, res, keywords: list[str], field_name: str):
+ raise NotImplementedError("GaussDB get_highlight is implemented in the search task")
+
+ def get_aggregation(self, res, field_name: str):
+ raise NotImplementedError("GaussDB get_aggregation is implemented in the search task")
+
+ def sql(self, sql: str, fetch_size: int, format: str):
+ raise NotImplementedError("GaussDB sql is implemented in the Text-to-SQL task")
diff --git a/common/doc_store/gaussdb_conn_pool.py b/common/doc_store/gaussdb_conn_pool.py
new file mode 100644
index 0000000000..a0dd334ad4
--- /dev/null
+++ b/common/doc_store/gaussdb_conn_pool.py
@@ -0,0 +1,282 @@
+#
+# Copyright 2025 The InfiniFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+import logging
+import re
+import threading
+from dataclasses import dataclass
+from typing import Any
+
+from psycopg2 import pool as psycopg2_pool
+
+logger = logging.getLogger("ragflow.gaussdb_conn_pool")
+_IDENTIFIER_PATTERN = re.compile(r"^(?:[A-Za-z_]|[^\x00-\x7F])(?:[A-Za-z0-9_#$]|[^\x00-\x7F]){0,62}$")
+_DECIMAL_PORT_PATTERN = re.compile(r"^[0-9]+$")
+_DATABASE_KEYS = ("database", "db_name", "name")
+
+
+class GaussDBError(Exception):
+ """Base exception for GaussDB DocEngine setup and connectivity."""
+
+
+class InvalidGaussDBConfig(ValueError, GaussDBError):
+ """Raised when GaussDB DocEngine config is missing or invalid."""
+
+
+class GaussDBConnectionError(GaussDBError):
+ """Raised when the GaussDB connection cannot be established or used."""
+
+
+class GaussDBAuthenticationError(GaussDBConnectionError):
+ """Raised when GaussDB rejects configured credentials."""
+
+
+class GaussDBPermissionError(GaussDBConnectionError):
+ """Raised when the configured user cannot use the target schema."""
+
+
+@dataclass(frozen=True)
+class GaussDBConfig:
+ host: str
+ port: int
+ database: str
+ user: str
+ password: str
+ schema: str = "public"
+
+
+def _normalize_schema(schema: Any) -> str:
+ value = str(schema or "").strip() or "public"
+ if not _IDENTIFIER_PATTERN.match(value):
+ raise InvalidGaussDBConfig(f"invalid gaussdb schema: {value}")
+ return value
+
+
+def _normalize_required_string(value: Any, field: str, *, strip: bool = True) -> str:
+ if not isinstance(value, str) or not value.strip():
+ raise InvalidGaussDBConfig(f"invalid gaussdb config field: {field}")
+ return value.strip() if strip else value
+
+
+def _normalize_port(value: Any) -> int:
+ if isinstance(value, bool):
+ raise InvalidGaussDBConfig("invalid gaussdb config field: port")
+ if isinstance(value, int):
+ port = value
+ elif isinstance(value, str) and _DECIMAL_PORT_PATTERN.fullmatch(value):
+ port = int(value)
+ else:
+ raise InvalidGaussDBConfig("invalid gaussdb config field: port")
+ if port <= 0 or port > 65535:
+ raise InvalidGaussDBConfig("invalid gaussdb config field: port")
+ return port
+
+
+def load_gaussdb_config(raw: dict[str, Any] | None = None) -> GaussDBConfig:
+ if raw is None:
+ from common import settings
+
+ raw = getattr(settings, "GAUSSDB", None) or settings.get_base_config("gaussdb", {})
+ config = raw if isinstance(raw, dict) else {}
+ database_key = next((key for key in _DATABASE_KEYS if key in config), None)
+ configured_keys = {
+ "host": "host" if "host" in config else None,
+ "port": "port" if "port" in config else None,
+ "database": database_key,
+ "user": "user" if "user" in config else None,
+ "password": "password" if "password" in config else None,
+ }
+ missing = [field for field, key in configured_keys.items() if key is None]
+ if missing:
+ raise InvalidGaussDBConfig(f"missing gaussdb config field(s): {', '.join(missing)}")
+
+ return GaussDBConfig(
+ host=_normalize_required_string(config["host"], "host"),
+ port=_normalize_port(config["port"]),
+ database=_normalize_required_string(config[database_key], "database"),
+ user=_normalize_required_string(config["user"], "user"),
+ password=_normalize_required_string(config["password"], "password", strip=False),
+ schema=_normalize_schema(config.get("schema")),
+ )
+
+
+def mask_gaussdb_uri(cfg: GaussDBConfig) -> str:
+ return f"{cfg.user}@{cfg.host}:{cfg.port}/{cfg.database}?schema={cfg.schema}"
+
+
+def classify_gaussdb_exception(exc: Exception) -> GaussDBConnectionError:
+ if isinstance(exc, GaussDBConnectionError):
+ return exc
+ text = str(exc).lower()
+ if "password" in text or "authentication" in text or "invalid username" in text:
+ return GaussDBAuthenticationError(str(exc))
+ if "permission" in text or "privilege" in text:
+ return GaussDBPermissionError(str(exc))
+ return GaussDBConnectionError(str(exc))
+
+
+class GaussDBConnectionPool:
+ def __init__(
+ self,
+ config: GaussDBConfig | None = None,
+ pool: Any | None = None,
+ minconn: int = 1,
+ maxconn: int = 8,
+ ):
+ self.config = config or load_gaussdb_config()
+ self.resolved_schema = self.config.schema
+ self.masked_uri = mask_gaussdb_uri(self.config)
+ self._pool = pool or self._create_pool(minconn=minconn, maxconn=maxconn)
+
+ def _create_pool(self, minconn: int, maxconn: int):
+ try:
+ return psycopg2_pool.ThreadedConnectionPool(
+ minconn,
+ maxconn,
+ host=self.config.host,
+ port=self.config.port,
+ dbname=self.config.database,
+ user=self.config.user,
+ password=self.config.password,
+ options=(f"-c search_path={self.resolved_schema} -c client_encoding=UTF8 -c default_transaction_read_only=off"),
+ )
+ except Exception as exc:
+ raise classify_gaussdb_exception(exc) from exc
+
+ def _discard_conn(self, conn) -> None:
+ if conn is None:
+ return
+ self._pool.putconn(conn, close=True)
+
+ def _validate_conn(self, conn) -> None:
+ if getattr(conn, "closed", False):
+ raise GaussDBConnectionError("GaussDB connection is closed")
+ cur = None
+ try:
+ cur = conn.cursor()
+ cur.execute("SELECT 1")
+ conn.rollback()
+ finally:
+ if cur is not None:
+ cur.close()
+
+ def get_conn(self):
+ last_exc: Exception = RuntimeError("GaussDB connection checkout failed without an exception")
+ for _attempt in range(2):
+ conn = None
+ try:
+ conn = self._pool.getconn()
+ self._validate_conn(conn)
+ return conn
+ except Exception as exc:
+ last_exc = exc
+ self._discard_conn(conn)
+ raise classify_gaussdb_exception(last_exc) from last_exc
+
+ def put_conn(self, conn) -> None:
+ if conn is None:
+ return
+ try:
+ conn.rollback()
+ except Exception as exc:
+ logger.warning("Discarding GaussDB connection after rollback failure: %s", type(exc).__name__)
+ try:
+ self._discard_conn(conn)
+ except Exception as discard_exc:
+ logger.warning("Failed to discard GaussDB connection: %s", type(discard_exc).__name__)
+ return
+ self._pool.putconn(conn)
+
+ def close_all(self) -> None:
+ self._pool.closeall()
+
+ def check_schema_access(self) -> None:
+ conn = self.get_conn()
+ cur = None
+ try:
+ cur = conn.cursor()
+ cur.execute(
+ """
+ SELECT
+ has_schema_privilege(%s, %s, %s) AS has_usage,
+ has_schema_privilege(%s, %s, %s) AS has_create
+ """,
+ (
+ self.config.user,
+ self.resolved_schema,
+ "USAGE",
+ self.config.user,
+ self.resolved_schema,
+ "CREATE",
+ ),
+ )
+ row = cur.fetchone()
+ has_usage, has_create = (bool(row[0]), bool(row[1])) if row else (False, False)
+ missing = []
+ if not has_usage:
+ missing.append("USAGE")
+ if not has_create:
+ missing.append("CREATE")
+ if missing:
+ raise GaussDBPermissionError(f"GaussDB user {self.config.user} lacks {', '.join(missing)} on schema {self.resolved_schema}")
+ except GaussDBPermissionError:
+ raise
+ except Exception as exc:
+ raise classify_gaussdb_exception(exc) from exc
+ finally:
+ if cur is not None:
+ cur.close()
+ self.put_conn(conn)
+
+ def fetch_one(self, sql: str, params: tuple[Any, ...] | None = None):
+ conn = self.get_conn()
+ cur = None
+ try:
+ cur = conn.cursor()
+ cur.execute(sql, params)
+ return cur.fetchone()
+ except Exception as exc:
+ raise classify_gaussdb_exception(exc) from exc
+ finally:
+ if cur is not None:
+ cur.close()
+ self.put_conn(conn)
+
+
+class _LazyGaussDBConnectionPool:
+ """Shared lazy pool used by the GaussDB DocEngine and Memory Store."""
+
+ def __init__(self):
+ self._lock = threading.Lock()
+ self._pool: GaussDBConnectionPool | None = None
+
+ def get_pool(self) -> GaussDBConnectionPool:
+ if self._pool is None:
+ with self._lock:
+ if self._pool is None:
+ self._pool = GaussDBConnectionPool()
+ return self._pool
+
+ def close_all(self) -> None:
+ with self._lock:
+ if self._pool is not None:
+ self._pool.close_all()
+ self._pool = None
+
+ def __getattr__(self, name: str):
+ return getattr(self.get_pool(), name)
+
+
+GAUSSDB_CONN = _LazyGaussDBConnectionPool()
diff --git a/common/metadata_gaussdb_filter.py b/common/metadata_gaussdb_filter.py
new file mode 100644
index 0000000000..cdd5b92511
--- /dev/null
+++ b/common/metadata_gaussdb_filter.py
@@ -0,0 +1,460 @@
+#
+# Copyright 2025 The InfiniFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+"""Translate RAGFlow document-metadata filters into GaussDB JSONB SQL predicates."""
+
+from __future__ import annotations
+
+import ast
+import json
+import re
+from dataclasses import dataclass
+from datetime import datetime
+from typing import Any, Sequence
+
+from common.doc_store.gaussdb_conn_base import jsonb_path_literal as encode_jsonb_path_literal
+
+OP_NE = "\u2260"
+OP_GE = "\u2265"
+OP_LE = "\u2264"
+
+SUPPORTED_OPERATORS: frozenset[str] = frozenset(
+ {
+ "=",
+ OP_NE,
+ ">",
+ OP_GE,
+ "<",
+ OP_LE,
+ "in",
+ "not in",
+ "contains",
+ "not contains",
+ "start with",
+ "end with",
+ "empty",
+ "not empty",
+ }
+)
+
+_CANONICAL_TO_INTERNAL = {
+ "=": "eq",
+ OP_NE: "ne",
+ ">": "gt",
+ OP_GE: "gte",
+ "<": "lt",
+ OP_LE: "lte",
+ "in": "in",
+ "not in": "not_in",
+ "contains": "contains",
+ "not contains": "not_contains",
+ "start with": "start_with",
+ "end with": "end_with",
+ "empty": "empty",
+ "not empty": "not_empty",
+}
+
+_OP_ALIASES = {
+ "is": "=",
+ "is not": OP_NE,
+ "not is": OP_NE,
+ "!=": OP_NE,
+ "<>": OP_NE,
+ ">=": OP_GE,
+ "<=": OP_LE,
+}
+
+_INTERNAL_RANGE_SQL = {
+ "gt": ">",
+ "gte": ">=",
+ "lt": "<",
+ "lte": "<=",
+}
+
+_DATETIME_PREFIX_RE = re.compile(
+ r"^(?P<year>\d{4})-(?P<month>\d{2})"
+ r"(?:-(?P<day>\d{2})"
+ r"(?:(?:T| )(?P<hour>\d{2})"
+ r"(?::(?P<minute>\d{2})"
+ r"(?::(?P<second>\d{2}))?"
+ r")?"
+ r")?"
+ r")?$"
+)
+_NUMBER_RE = re.compile(r"^-?[0-9]+(\.[0-9]+)?$")
+_IDENTIFIER_RE = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$")
+
+
+class UnsupportedGaussDBMetaFilter(ValueError):
+ def __init__(self, reason: str) -> None:
+ super().__init__(reason)
+ self.reason = reason
+
+
+@dataclass(frozen=True)
+class TranslatedGaussDBMetaFilter:
+ sql: str
+ params: list[Any]
+
+
+@dataclass(frozen=True)
+class GaussDBFilterPlan:
+ logic: str
+ translated: list[TranslatedGaussDBMetaFilter]
+
+ def to_predicate(self) -> TranslatedGaussDBMetaFilter:
+ if not self.translated:
+ return TranslatedGaussDBMetaFilter("1=1", [])
+ joiner = " AND " if self.logic == "and" else " OR "
+ sql = joiner.join(f"({item.sql})" for item in self.translated)
+ params: list[Any] = []
+ for item in self.translated:
+ params.extend(item.params)
+ return TranslatedGaussDBMetaFilter(sql, params)
+
+
+class GaussDBMetaFilterTranslator:
+ def __init__(self, jsonb_column: str = "meta_fields") -> None:
+ self.jsonb_column = validate_jsonb_column(jsonb_column)
+
+ def translate(self, flt: dict) -> TranslatedGaussDBMetaFilter:
+ key = validate_meta_key(flt.get("key"))
+ op = normalize_metadata_filter_op(flt)
+ value = flt.get("value")
+
+ if op == "empty":
+ return TranslatedGaussDBMetaFilter(self._empty_predicate(key), [])
+ if op == "not_empty":
+ return TranslatedGaussDBMetaFilter(self._not_empty_predicate(key), [])
+ if op == "eq":
+ return self._translate_equal(key, value, flt)
+ if op == "ne":
+ return self._translate_not_equal(key, value, flt)
+ if op in _INTERNAL_RANGE_SQL:
+ return self._translate_range(key, op, value, flt)
+ if op == "in":
+ return self._translate_in(key, value, flt)
+ if op == "not_in":
+ return self._translate_not_in(key, value, flt)
+ if op == "contains":
+ return self._translate_contains(key, value, flt)
+ if op == "not_contains":
+ return self._translate_not_contains(key, value, flt)
+ if op == "start_with":
+ return self._translate_start_with(key, value, flt)
+ return self._translate_end_with(key, value, flt)
+
+ def _translate_equal(self, key: str, value: Any, flt: dict) -> TranslatedGaussDBMetaFilter:
+ sql, params = self._equal_predicate(key, value, flt)
+ return TranslatedGaussDBMetaFilter(sql, params)
+
+ def _translate_not_equal(self, key: str, value: Any, flt: dict) -> TranslatedGaussDBMetaFilter:
+ sql, params = self._equal_predicate(key, value, flt)
+ return TranslatedGaussDBMetaFilter(f"{self._key_exists_expr(key)} AND ({sql}) IS NOT TRUE", params)
+
+ def _translate_range(self, key: str, op: str, value: Any, flt: dict) -> TranslatedGaussDBMetaFilter:
+ value_type, coerced = coerce_range_value(value, flt)
+ text_expr = self._text_expr(key)
+ sql_op = _INTERNAL_RANGE_SQL[op]
+ if value_type == "datetime":
+ datetime_expr = self._datetime_expr(text_expr)
+ return TranslatedGaussDBMetaFilter(
+ f"{datetime_expr} {sql_op} to_timestamp(%s, 'YYYY-MM-DD HH24:MI:SS')",
+ [coerced],
+ )
+ return TranslatedGaussDBMetaFilter(
+ f"CASE WHEN {text_expr} ~ '^-?[0-9]+(\\.[0-9]+)?$' THEN ({text_expr})::DOUBLE PRECISION {sql_op} %s ELSE FALSE END",
+ [coerced],
+ )
+
+ def _translate_in(self, key: str, value: Any, flt: dict) -> TranslatedGaussDBMetaFilter:
+ fragments, params = self._membership_predicates(key, value, flt)
+ return TranslatedGaussDBMetaFilter(" OR ".join(f"({sql})" for sql in fragments), params)
+
+ def _translate_not_in(self, key: str, value: Any, flt: dict) -> TranslatedGaussDBMetaFilter:
+ fragments, params = self._membership_predicates(key, value, flt)
+ sql = " AND ".join(f"({fragment}) IS NOT TRUE" for fragment in fragments)
+ return TranslatedGaussDBMetaFilter(f"{self._key_exists_expr(key)} AND {sql}", params)
+
+ def _translate_contains(self, key: str, value: Any, flt: dict) -> TranslatedGaussDBMetaFilter:
+ text = coerce_string_value(value, flt)
+ array_expr = f"jsonb_exists({self._value_expr(key)}, %s)"
+ like_expr = f"lower({self._text_expr(key)}) LIKE %s ESCAPE '\\'"
+ return TranslatedGaussDBMetaFilter(
+ f"{array_expr} OR {like_expr}",
+ [text.lower(), f"%{escape_like_pattern(text.lower())}%"],
+ )
+
+ def _translate_not_contains(self, key: str, value: Any, flt: dict) -> TranslatedGaussDBMetaFilter:
+ positive = self._translate_contains(key, value, flt)
+ return TranslatedGaussDBMetaFilter(
+ f"{self._key_exists_expr(key)} AND ({positive.sql}) IS NOT TRUE",
+ positive.params,
+ )
+
+ def _translate_start_with(self, key: str, value: Any, flt: dict) -> TranslatedGaussDBMetaFilter:
+ text = coerce_string_value(value, flt).lower()
+ return TranslatedGaussDBMetaFilter(
+ f"lower({self._text_expr(key)}) LIKE %s ESCAPE '\\'",
+ [f"{escape_like_pattern(text)}%"],
+ )
+
+ def _translate_end_with(self, key: str, value: Any, flt: dict) -> TranslatedGaussDBMetaFilter:
+ text = coerce_string_value(value, flt).lower()
+ return TranslatedGaussDBMetaFilter(
+ f"lower({self._text_expr(key)}) LIKE %s ESCAPE '\\'",
+ [f"%{escape_like_pattern(text)}"],
+ )
+
+ def _equal_predicate(self, key: str, value: Any, flt: dict) -> tuple[str, list[Any]]:
+ value = coerce_scalar_value(value, flt)
+ value_expr = self._value_expr(key)
+ if value is None:
+ return f"{self._key_exists_expr(key)} AND {value_expr} = 'null'::jsonb", []
+ if isinstance(value, str):
+ if value == "":
+ return f"{value_expr} = '\"\"'::jsonb", []
+ value = value.lower()
+ return f"lower({self._text_expr(key)}) = %s OR jsonb_exists({value_expr}, %s)", [value, value]
+ return f"{value_expr} @> %s::jsonb", [jsonb_param(value)]
+
+ def _membership_predicates(self, key: str, value: Any, flt: dict) -> tuple[list[str], list[Any]]:
+ fragments: list[str] = []
+ params: list[Any] = []
+ for member in coerce_membership_values(value, flt):
+ sql, member_params = self._equal_predicate(key, member, flt)
+ fragments.append(sql)
+ params.extend(member_params)
+ return fragments, params
+
+ def _empty_predicate(self, key: str) -> str:
+ value_expr = self._value_expr(key)
+ return f"{self._key_missing_expr(key)} OR {value_expr} = 'null'::jsonb OR {value_expr} = '\"\"'::jsonb OR {value_expr} = '[]'::jsonb OR {value_expr} = '{{}}'::jsonb"
+
+ def _not_empty_predicate(self, key: str) -> str:
+ value_expr = self._value_expr(key)
+ return (
+ f"{self._key_exists_expr(key)} AND "
+ f"({value_expr} = 'null'::jsonb) IS NOT TRUE AND "
+ f"({value_expr} = '\"\"'::jsonb) IS NOT TRUE AND "
+ f"({value_expr} = '[]'::jsonb) IS NOT TRUE AND "
+ f"({value_expr} = '{{}}'::jsonb) IS NOT TRUE"
+ )
+
+ def _value_expr(self, key: str) -> str:
+ return f"{self.jsonb_column} #> {jsonb_path_literal(key)}"
+
+ def _text_expr(self, key: str) -> str:
+ return f"{self.jsonb_column} #>> {jsonb_path_literal(key)}"
+
+ def _datetime_expr(self, text_expr: str) -> str:
+ normalized = f"replace({text_expr}, 'T', ' ')"
+ return (
+ "CASE "
+ f"WHEN {text_expr} ~ '^[0-9]{{4}}-[0-9]{{2}}$' "
+ f"THEN to_timestamp({text_expr} || '-01 00:00:00', 'YYYY-MM-DD HH24:MI:SS') "
+ f"WHEN {text_expr} ~ '^[0-9]{{4}}-[0-9]{{2}}-[0-9]{{2}}$' "
+ f"THEN to_timestamp({text_expr} || ' 00:00:00', 'YYYY-MM-DD HH24:MI:SS') "
+ f"WHEN {text_expr} ~ '^[0-9]{{4}}-[0-9]{{2}}-[0-9]{{2}}[T ][0-9]{{2}}$' "
+ f"THEN to_timestamp({normalized} || ':00:00', 'YYYY-MM-DD HH24:MI:SS') "
+ f"WHEN {text_expr} ~ '^[0-9]{{4}}-[0-9]{{2}}-[0-9]{{2}}[T ][0-9]{{2}}:[0-9]{{2}}$' "
+ f"THEN to_timestamp({normalized} || ':00', 'YYYY-MM-DD HH24:MI:SS') "
+ f"WHEN {text_expr} ~ '^[0-9]{{4}}-[0-9]{{2}}-[0-9]{{2}}[T ][0-9]{{2}}:[0-9]{{2}}:[0-9]{{2}}$' "
+ f"THEN to_timestamp({normalized}, 'YYYY-MM-DD HH24:MI:SS') "
+ "END"
+ )
+
+ def _key_exists_expr(self, key: str) -> str:
+ # JSON null is a non-NULL jsonb value; #> yields SQL NULL only when the path is missing.
+ return f"({self._value_expr(key)}) IS NOT NULL"
+
+ def _key_missing_expr(self, key: str) -> str:
+ return f"({self._value_expr(key)}) IS NULL"
+
+
+def build_gaussdb_filter(
+ filters: Sequence[dict],
+ logic: str,
+) -> tuple[str, list[Any]]:
+ translated = plan_pushdown(filters, logic).to_predicate()
+ return translated.sql, translated.params
+
+
+def plan_pushdown(
+ filters: Sequence[dict],
+ logic: str,
+) -> GaussDBFilterPlan:
+ if logic not in {"and", "or"}:
+ raise UnsupportedGaussDBMetaFilter(f"unknown logic {logic!r}")
+ translator = GaussDBMetaFilterTranslator()
+ return GaussDBFilterPlan(logic=logic, translated=[translator.translate(flt) for flt in filters])
+
+
+def fetch_gaussdb_metadata_doc_ids(
+ doc_store,
+ index_name: str,
+ kb_ids: Sequence[str],
+ sql_filter: str,
+ filter_params: Sequence[Any],
+ limit: int,
+) -> list[str]:
+ return doc_store.fetch_metadata_doc_ids(index_name, kb_ids, sql_filter, filter_params, limit)
+
+
+def is_pushdown_supported(filters: Sequence[dict]) -> bool:
+ try:
+ plan_pushdown(filters, "and")
+ return True
+ except UnsupportedGaussDBMetaFilter:
+ return False
+
+
+def normalize_gaussdb_meta_operator(op: Any) -> str:
+ if op is None:
+ raise UnsupportedGaussDBMetaFilter("metadata filter operator is missing")
+ normalized = " ".join(str(op).strip().lower().split())
+ canonical = _OP_ALIASES.get(normalized, normalized)
+ if canonical not in SUPPORTED_OPERATORS:
+ raise UnsupportedGaussDBMetaFilter(f"unsupported metadata filter operator {op!r}")
+ return canonical
+
+
+def normalize_metadata_filter_op(flt: dict | Any) -> str:
+ op = flt.get("op", flt.get("operator", flt.get("comparison_operator"))) if isinstance(flt, dict) else flt
+ return _CANONICAL_TO_INTERNAL[normalize_gaussdb_meta_operator(op)]
+
+
+def validate_jsonb_column(column: str) -> str:
+ if not _IDENTIFIER_RE.fullmatch(column or ""):
+ raise UnsupportedGaussDBMetaFilter(f"invalid JSONB column {column!r}")
+ return column
+
+
+def validate_meta_key(key: Any) -> str:
+ split_meta_key(key)
+ return key
+
+
+def split_meta_key(key: Any) -> list[str]:
+ if not isinstance(key, str) or not key:
+ raise UnsupportedGaussDBMetaFilter("invalid metadata key")
+ segments = key.split(".")
+ if not segments or any(not segment for segment in segments):
+ raise UnsupportedGaussDBMetaFilter("metadata key contains an empty path segment")
+ if any("\x00" in segment for segment in segments):
+ raise UnsupportedGaussDBMetaFilter("metadata key contains NUL")
+ return segments
+
+
+def jsonb_path_literal(key: str) -> str:
+ return encode_jsonb_path_literal(split_meta_key(key))
+
+
+def coerce_scalar_value(value: Any, flt: dict) -> Any:
+ if isinstance(value, (list, dict, tuple)):
+ raise UnsupportedGaussDBMetaFilter("scalar comparison value is non-scalar")
+ if isinstance(value, str):
+ stripped = value.strip()
+ if stripped == "":
+ return value
+ try:
+ parsed = ast.literal_eval(stripped)
+ except Exception:
+ return value
+ if isinstance(parsed, (int, float, bool)) or parsed is None:
+ return parsed
+ return value
+
+
+def coerce_range_value(value: Any, flt: dict) -> tuple[str, int | float | str]:
+ if value is None:
+ raise UnsupportedGaussDBMetaFilter("range comparison value is None")
+ if isinstance(value, bool):
+ raise UnsupportedGaussDBMetaFilter("range comparison value is boolean")
+ if isinstance(value, (int, float)):
+ return "number", value
+
+ text = str(value).strip()
+ datetime_value = normalize_datetime_prefix(text)
+ if datetime_value is not None:
+ return "datetime", datetime_value
+
+ try:
+ parsed = ast.literal_eval(text)
+ except Exception:
+ parsed = None
+ if isinstance(parsed, (int, float)) and not isinstance(parsed, bool):
+ return "number", parsed
+ if _NUMBER_RE.fullmatch(text):
+ return ("number", float(text) if "." in text else int(text))
+ raise UnsupportedGaussDBMetaFilter("unsupported range comparison value")
+
+
+def normalize_datetime_prefix(text: str) -> str | None:
+ match = _DATETIME_PREFIX_RE.fullmatch(text)
+ if not match:
+ return None
+ parts = match.groupdict()
+ values = {
+ "year": int(parts["year"]),
+ "month": int(parts["month"]),
+ "day": int(parts["day"] or 1),
+ "hour": int(parts["hour"] or 0),
+ "minute": int(parts["minute"] or 0),
+ "second": int(parts["second"] or 0),
+ }
+ try:
+ return datetime(**values).strftime("%Y-%m-%d %H:%M:%S")
+ except ValueError:
+ return None
+
+
+def coerce_string_value(value: Any, flt: dict) -> str:
+ if value is None or isinstance(value, (list, dict, tuple)):
+ raise UnsupportedGaussDBMetaFilter("string operator value must be a scalar")
+ text = str(value)
+ if not text:
+ raise UnsupportedGaussDBMetaFilter("string operator value is empty")
+ return text
+
+
+def coerce_membership_values(value: Any, flt: dict) -> list[Any]:
+ if value is None:
+ raise UnsupportedGaussDBMetaFilter("membership value is None")
+ if isinstance(value, (list, tuple)):
+ members = list(value)
+ elif isinstance(value, str):
+ stripped = value.strip()
+ try:
+ parsed = ast.literal_eval(stripped)
+ except Exception:
+ parsed = value
+ if isinstance(parsed, (list, tuple)):
+ members = list(parsed)
+ else:
+ members = [item.strip() for item in value.split(",") if item.strip()]
+ else:
+ members = [value]
+ if not members:
+ raise UnsupportedGaussDBMetaFilter("membership value resolved to empty list")
+ return [coerce_scalar_value(member, flt) for member in members]
+
+
+def jsonb_param(value: Any) -> str:
+ return json.dumps(value, ensure_ascii=False)
+
+
+def escape_like_pattern(value: str) -> str:
+ return value.replace("\\", "\\\\").replace("%", "\\%").replace("_", "\\_")
diff --git a/common/settings.py b/common/settings.py
index 2eff4a7be7..7db73d6330 100644
--- a/common/settings.py
+++ b/common/settings.py
@@ -17,6 +17,7 @@ import os
import json
import secrets
import logging
+import re
from datetime import date
from common.constants import RAG_FLOW_SERVICE_NAME
@@ -30,6 +31,7 @@ import rag.utils.es_conn
import rag.utils.infinity_conn
import rag.utils.ob_conn
import rag.utils.opensearch_conn
+import rag.utils.gaussdb_conn
from rag.utils.azure_sas_conn import RAGFlowAzureSasBlob
from rag.utils.azure_spn_conn import RAGFlowAzureSpnBlob
from rag.utils.gcs_conn import RAGFlowGCS
@@ -44,6 +46,7 @@ from rag.nlp import search
import memory.utils.es_conn as memory_es_conn
import memory.utils.infinity_conn as memory_infinity_conn
import memory.utils.ob_conn as memory_ob_conn
+import memory.utils.gaussdb_conn as memory_gaussdb_conn
TIMEZONE = os.getenv("TZ", "Asia/Shanghai")
@@ -70,8 +73,89 @@ SECRET_KEY = None
FACTORY_LLM_INFOS = None
ALLOWED_LLM_FACTORIES = None
-DATABASE_TYPE = os.getenv("DB_TYPE", "mysql")
-DATABASE = decrypt_database_config(name=DATABASE_TYPE)
+# The metadata database and DocEngine/Memory Store may all use GaussDB while
+# targeting different databases, schemas, compatibility modes, or credentials.
+# The metadata database therefore reads only GAUSSDB_METADATA_*. The gaussdb
+# section in service_conf.yaml remains exclusive to DOC_ENGINE=gaussdb.
+GAUSSDB_ENV_DEFAULTS = {
+ "name": "rag_flow",
+ "user": "rag_flow",
+ "password": "infini_rag_flow",
+ "host": "gaussdb",
+ "port": 8000,
+ "schema": "public",
+ "max_connections": 100,
+ "stale_timeout": 30,
+ "options": "-c client_encoding=UTF8 -c default_transaction_read_only=off",
+}
+_GAUSSDB_SCHEMA_PATTERN = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$")
+
+
+def normalize_database_type(database_type: str | None = None) -> str:
+ # Only normalize the new GaussDB values. Existing database names retain
+ # their upstream spelling and lookup behavior.
+ raw_value = database_type or "mysql"
+ normalized = raw_value.strip().lower()
+ if normalized in {"gaussdb", "gauss"}:
+ return "gaussdb"
+ return raw_value
+
+
+def _get_int_env(name: str, default: int) -> int:
+ try:
+ return int(os.environ.get(name, default))
+ except (TypeError, ValueError):
+ return default
+
+
+def _normalize_gaussdb_metadata_schema(value: str | None = None) -> str:
+ # The schema is interpolated into search_path in the libpq options. Restrict
+ # it to a plain SQL identifier so quotes, semicolons, or extra options cannot
+ # be injected. The metadata database currently accepts one schema only.
+ schema = (value or GAUSSDB_ENV_DEFAULTS["schema"]).strip() or GAUSSDB_ENV_DEFAULTS["schema"]
+ if not _GAUSSDB_SCHEMA_PATTERN.match(schema):
+ raise ValueError(f"invalid GAUSSDB_METADATA_SCHEMA: {schema}")
+ return schema
+
+
+def _gaussdb_metadata_options(schema: str) -> str:
+ explicit_options = os.environ.get("GAUSSDB_METADATA_OPTIONS")
+ if explicit_options is not None:
+ # An explicit value replaces the complete options string. Advanced
+ # deployments can control search_path, encoding, read-only behavior,
+ # and other libpq options, but must also include any required defaults.
+ return explicit_options
+ return f"-c search_path={schema} {GAUSSDB_ENV_DEFAULTS['options']}"
+
+
+def _gaussdb_env_config() -> dict:
+ # Read only GAUSSDB_METADATA_* so the metadata database does not share
+ # connection parameters with DOC_ENGINE=gaussdb.
+ schema = _normalize_gaussdb_metadata_schema(os.environ.get("GAUSSDB_METADATA_SCHEMA"))
+ return {
+ "name": os.environ.get("GAUSSDB_METADATA_DBNAME", GAUSSDB_ENV_DEFAULTS["name"]),
+ "user": os.environ.get("GAUSSDB_METADATA_USER", GAUSSDB_ENV_DEFAULTS["user"]),
+ "password": os.environ.get("GAUSSDB_METADATA_PASSWORD", GAUSSDB_ENV_DEFAULTS["password"]),
+ "host": os.environ.get("GAUSSDB_METADATA_HOST", GAUSSDB_ENV_DEFAULTS["host"]),
+ "port": _get_int_env("GAUSSDB_METADATA_PORT", GAUSSDB_ENV_DEFAULTS["port"]),
+ "max_connections": _get_int_env("GAUSSDB_METADATA_MAX_CONNECTIONS", GAUSSDB_ENV_DEFAULTS["max_connections"]),
+ "stale_timeout": _get_int_env("GAUSSDB_METADATA_STALE_TIMEOUT", GAUSSDB_ENV_DEFAULTS["stale_timeout"]),
+ "options": _gaussdb_metadata_options(schema),
+ }
+
+
+def load_database_config(database_type: str) -> dict:
+ database_type = normalize_database_type(database_type)
+ if database_type == "gaussdb":
+ # Always build DB_TYPE=gaussdb from GAUSSDB_METADATA_* so the metadata
+ # connection remains isolated from the gaussdb section used by
+ # DOC_ENGINE=gaussdb.
+ return decrypt_database_config(database=_gaussdb_env_config())
+ return decrypt_database_config(name=database_type)
+
+
+DATABASE_TYPE = normalize_database_type(os.getenv("DB_TYPE", "mysql"))
+DATABASE = load_database_config(DATABASE_TYPE)
# authentication
AUTHENTICATION_CONF = None
@@ -85,6 +169,7 @@ OAUTH_CONFIG = None
DOC_ENGINE = os.getenv("DOC_ENGINE", "elasticsearch")
DOC_ENGINE_INFINITY = DOC_ENGINE.lower() == "infinity"
DOC_ENGINE_OCEANBASE = DOC_ENGINE.lower() == "oceanbase"
+DOC_ENGINE_GAUSSDB = DOC_ENGINE.lower() == "gaussdb"
DOC_ENGINE_SERENEDB = DOC_ENGINE.lower() == "serenedb"
@@ -124,6 +209,7 @@ OB = {}
OSS = {}
OS = {}
GCS = {}
+GAUSSDB = {}
SERENEDB = {}
DOC_MAXIMUM_SIZE: int = 128 * 1024 * 1024
@@ -220,8 +306,8 @@ class StorageFactory:
def init_settings():
global DATABASE_TYPE, DATABASE
- DATABASE_TYPE = os.getenv("DB_TYPE", "mysql")
- DATABASE = decrypt_database_config(name=DATABASE_TYPE)
+ DATABASE_TYPE = normalize_database_type(os.getenv("DB_TYPE", "mysql"))
+ DATABASE = load_database_config(DATABASE_TYPE)
global ALLOWED_LLM_FACTORIES, LLM_FACTORY, LLM_BASE_URL
llm_settings = get_base_config("user_default_llm", {}) or {}
@@ -303,10 +389,11 @@ def init_settings():
FEISHU_OAUTH = get_base_config("oauth", {}).get("feishu")
OAUTH_CONFIG = get_base_config("oauth", {})
- global DOC_ENGINE, DOC_ENGINE_INFINITY, DOC_ENGINE_OCEANBASE, DOC_ENGINE_SERENEDB, docStoreConn, ES, OB, OS, INFINITY, SERENEDB
+ global DOC_ENGINE, DOC_ENGINE_INFINITY, DOC_ENGINE_OCEANBASE, DOC_ENGINE_GAUSSDB, DOC_ENGINE_SERENEDB, docStoreConn, ES, OB, OS, INFINITY, GAUSSDB, SERENEDB
DOC_ENGINE = os.environ.get("DOC_ENGINE", "elasticsearch").strip()
DOC_ENGINE_INFINITY = DOC_ENGINE.lower() == "infinity"
DOC_ENGINE_OCEANBASE = DOC_ENGINE.lower() == "oceanbase"
+ DOC_ENGINE_GAUSSDB = DOC_ENGINE.lower() == "gaussdb"
DOC_ENGINE_SERENEDB = DOC_ENGINE.lower() == "serenedb"
lower_case_doc_engine = DOC_ENGINE.lower()
if lower_case_doc_engine == "elasticsearch":
@@ -324,6 +411,9 @@ def init_settings():
elif lower_case_doc_engine == "seekdb":
OB = get_base_config("seekdb", {})
docStoreConn = rag.utils.ob_conn.OBConnection()
+ elif lower_case_doc_engine == "gaussdb":
+ GAUSSDB = get_base_config("gaussdb", {})
+ docStoreConn = rag.utils.gaussdb_conn.GaussDBConnection()
elif lower_case_doc_engine == "serenedb":
SERENEDB = get_base_config("serenedb", {})
# Imported lazily so psycopg2/SereneDB is only touched when selected.
@@ -343,6 +433,11 @@ def init_settings():
msgStoreConn = memory_infinity_conn.InfinityConnection()
elif lower_case_doc_engine in ["oceanbase", "seekdb"]:
msgStoreConn = memory_ob_conn.OBConnection()
+ elif lower_case_doc_engine == "gaussdb":
+ # Memory Store uses a dedicated adapter for message tables. It reads the
+ # same GaussDB configuration and shares the lazy connection pool with
+ # docStoreConn, but keeps its own table layout and query semantics.
+ msgStoreConn = memory_gaussdb_conn.GaussDBMemoryConnection()
global AZURE, S3, MINIO, OSS, GCS
if STORAGE_IMPL_TYPE in ["AZURE_SPN", "AZURE_SAS"]:
diff --git a/docker/.env b/docker/.env
index 0940323198..b8af55624f 100644
--- a/docker/.env
+++ b/docker/.env
@@ -17,15 +17,25 @@
# - `oceanbase` (https://github.com/oceanbase/oceanbase)
# - `opensearch` (https://github.com/opensearch-project/OpenSearch)
# - `seekdb` (https://github.com/oceanbase/seekdb)
+# - `gaussdb` GaussDB Centralized / Distributed DocEngine
DOC_ENGINE=${DOC_ENGINE:-elasticsearch}
+# The business metadata database type used by DB_TYPE.
+# Available options: mysql (default), postgres, gaussdb, oceanbase.
+DB_TYPE=${DB_TYPE:-mysql}
+
# Device on which deepdoc inference run.
# Available levels:
# - `cpu` (default)
# - `gpu`
DEVICE=${DEVICE:-cpu}
-COMPOSE_PROFILES=${DOC_ENGINE},${DEVICE}
+# This controls only the bundled metadata service started by Compose; DB_TYPE
+# independently selects the metadata backend used by RAGFlow. Keep `mysql` to
+# start the in-cluster MySQL service. For external GaussDB, set both
+# DB_TYPE=gaussdb and METADATA_DB_PROFILE=gaussdb so MySQL stays disabled.
+METADATA_DB_PROFILE=${METADATA_DB_PROFILE:-mysql}
+COMPOSE_PROFILES=${DOC_ENGINE},${DEVICE},metadata-${METADATA_DB_PROFILE}
# The version of Elasticsearch.
STACK_VERSION=${STACK_VERSION:-8.11.3}
@@ -91,6 +101,21 @@ OCEANBASE_PASSWORD=infini_rag_flow
# The doc database of the OceanBase service to use
OCEANBASE_DOC_DBNAME=ragflow_doc
+# GaussDB DocEngine connection. RAGFlow does not start or manage GaussDB;
+# configure an existing GaussDB instance here.
+# GaussDB host.
+GAUSSDB_HOST=
+# GaussDB port.
+GAUSSDB_PORT=
+# GaussDB database name.
+GAUSSDB_DATABASE=postgres
+# GaussDB login user.
+GAUSSDB_USER=
+# GaussDB login password.
+GAUSSDB_PASSWORD=
+# GaussDB schema for RAGFlow DocEngine objects.
+GAUSSDB_SCHEMA=
+
# OceanBase container configuration
OB_CLUSTER_NAME=${OB_CLUSTER_NAME:-ragflow}
OB_TENANT_NAME=${OB_TENANT_NAME:-ragflow}
@@ -130,6 +155,17 @@ EXPOSE_MYSQL_PORT=3306
# The maximum size of communication packets sent to the MySQL server
MYSQL_MAX_PACKET=1073741824
+# External GaussDB metadata database settings, used when DB_TYPE=gaussdb.
+# These are intentionally separate from the GAUSSDB_* DocEngine settings.
+GAUSSDB_METADATA_HOST=
+GAUSSDB_METADATA_PORT=
+GAUSSDB_METADATA_USER=
+GAUSSDB_METADATA_PASSWORD=
+GAUSSDB_METADATA_DBNAME=rag_flow
+GAUSSDB_METADATA_SCHEMA=public
+GAUSSDB_METADATA_MAX_CONNECTIONS=100
+GAUSSDB_METADATA_STALE_TIMEOUT=30
+
# The hostname where the MinIO service is exposed
MINIO_HOST=minio
# The port used to expose the MinIO console interface to the host machine,
diff --git a/docker/README.md b/docker/README.md
index b5cdb13838..522aa24e69 100644
--- a/docker/README.md
+++ b/docker/README.md
@@ -24,6 +24,17 @@
The [.env](./.env) file contains important environment variables for Docker.
+### Metadata database
+
+- `DB_TYPE`
+ The business metadata database type. Defaults to `mysql`. Supported values include `mysql`, `postgres`, `gaussdb`, and `oceanbase`.
+- `COMPOSE_PROFILES`
+ The Docker Compose profiles to enable. By default it contains `${DOC_ENGINE},${DEVICE},metadata-${METADATA_DB_PROFILE}`.
+- `METADATA_DB_PROFILE`
+ Defaults to `mysql`, preserving the in-cluster MySQL service. Set it to `gaussdb` together with `DB_TYPE=gaussdb` to use an external GaussDB metadata database without starting MySQL.
+- `GAUSSDB_METADATA_HOST`, `GAUSSDB_METADATA_PORT`, `GAUSSDB_METADATA_USER`, `GAUSSDB_METADATA_PASSWORD`, `GAUSSDB_METADATA_DBNAME`, `GAUSSDB_METADATA_SCHEMA`
+ External GaussDB metadata database connection settings, used when `DB_TYPE=gaussdb`. Set `METADATA_DB_PROFILE=gaussdb` to keep the in-cluster MySQL service disabled.
+
### Elasticsearch
- `STACK_VERSION`
@@ -50,7 +61,7 @@ The [.env](./.env) file contains important environment variables for Docker.
### MySQL
- `MYSQL_PASSWORD`
- The password for MySQL.
+ The password for MySQL. Required only when `DB_TYPE=mysql` starts the in-cluster MySQL metadata database.
- `MYSQL_PORT`
The port to connect to MySQL from RAGFlow container. Defaults to `3306`. Change this if you use an external MySQL.
- `EXPOSE_MYSQL_PORT`
@@ -207,6 +218,15 @@ Before setting `DOC_ENGINE=oceanbase`, make sure the host OS allows the file des
- `host`: The hostname of the OceanBase service.
- `port`: The port of OceanBase.
+- `gaussdb`
+ - `host`: The hostname or IP address of the GaussDB instance.
+ - `port`: The GaussDB port.
+ - `database`: The GaussDB database name. Defaults to `postgres`.
+ - `user`: The username for GaussDB.
+ - `password`: The password for GaussDB.
+ - `schema`: Optional schema used by the DocEngine. Defaults to `public`.
+ - RAGFlow does not start or manage GaussDB; set `DOC_ENGINE=gaussdb` only after preparing a GaussDB instance.
+
- `oss`
- `access_key`: The access key ID used to authenticate requests to the OSS service.
- `secret_key`: The secret access key used to authenticate requests to the OSS service.
diff --git a/docker/docker-compose-CN-oc9.yml b/docker/docker-compose-CN-oc9.yml
index 501b2044f3..c07a53b5d1 100644
--- a/docker/docker-compose-CN-oc9.yml
+++ b/docker/docker-compose-CN-oc9.yml
@@ -8,6 +8,7 @@ services:
depends_on:
mysql:
condition: service_healthy
+ required: false
profiles:
- cpu
image: edwardelric233/ragflow:oc9
@@ -33,6 +34,7 @@ services:
depends_on:
mysql:
condition: service_healthy
+ required: false
profiles:
- gpu
image: edwardelric233/ragflow:oc9
diff --git a/docker/docker-compose-base.yml b/docker/docker-compose-base.yml
index 6722071ab1..aa9f4ac652 100644
--- a/docker/docker-compose-base.yml
+++ b/docker/docker-compose-base.yml
@@ -205,6 +205,11 @@ services:
restart: unless-stopped
mysql:
+ profiles:
+ - mysql
+ - metadata-mysql
+ - metadata-MySQL
+ - metadata-MYSQL
# mysql:5.7 linux/arm64 image is unavailable.
image: mysql:8.0.40
env_file: .env
diff --git a/docker/docker-compose-macos.yml b/docker/docker-compose-macos.yml
index b667108fc4..587266b395 100644
--- a/docker/docker-compose-macos.yml
+++ b/docker/docker-compose-macos.yml
@@ -7,6 +7,7 @@ services:
depends_on:
mysql:
condition: service_healthy
+ required: false
build:
context: ../
dockerfile: Dockerfile
diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml
index 14415f8219..5194984bae 100644
--- a/docker/docker-compose.yml
+++ b/docker/docker-compose.yml
@@ -22,6 +22,7 @@ services:
depends_on:
mysql:
condition: service_healthy
+ required: false
profiles:
- cpu
image: ${RAGFLOW_IMAGE}
@@ -73,6 +74,7 @@ services:
depends_on:
mysql:
condition: service_healthy
+ required: false
profiles:
- gpu
image: ${RAGFLOW_IMAGE}
diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh
index cfd8fb65fb..7979b7e72d 100755
--- a/docker/entrypoint.sh
+++ b/docker/entrypoint.sh
@@ -255,7 +255,15 @@ ensure_docling
ensure_db_init
if [[ "${INIT_MODEL_PROVIDER_TABLES}" -eq 1 ]]; then
- tools/scripts/run_migrations.sh
+ DB_TYPE_NORMALIZED="${DB_TYPE:-mysql}"
+ DB_TYPE_NORMALIZED="${DB_TYPE_NORMALIZED,,}"
+ if [[ "${DB_TYPE_NORMALIZED}" == "gaussdb" || "${DB_TYPE_NORMALIZED}" == "gauss" ]]; then
+ # This migration script contains MySQL-only SQL and cannot run against
+ # a GaussDB metadata database.
+ echo "Skipping MySQL-specific model provider table migrations for DB_TYPE=${DB_TYPE:-mysql}."
+ else
+ tools/scripts/run_migrations.sh
+ fi
fi
if [[ "${ENABLE_ADMIN_SERVER}" -eq 1 ]]; then
diff --git a/docker/launch_backend_service.sh b/docker/launch_backend_service.sh
index 52d9661779..017297ecc2 100755
--- a/docker/launch_backend_service.sh
+++ b/docker/launch_backend_service.sh
@@ -211,6 +211,15 @@ ensure_db_init() {
}
run_mysql_migrations() {
+ local db_type="${DB_TYPE:-mysql}"
+ db_type="${db_type,,}"
+ if [ "$db_type" = "gaussdb" ] || [ "$db_type" = "gauss" ]; then
+ # This migration script contains MySQL-only SQL and cannot run against
+ # a GaussDB metadata database.
+ echo "Skipping MySQL-specific model provider table migrations for DB_TYPE=${DB_TYPE:-mysql}."
+ return 0
+ fi
+
tools/scripts/run_migrations.sh
}
diff --git a/docker/service_conf.yaml.template b/docker/service_conf.yaml.template
index b60e1317c4..2070f96f27 100644
--- a/docker/service_conf.yaml.template
+++ b/docker/service_conf.yaml.template
@@ -55,6 +55,13 @@ oceanbase:
password: '${OCEANBASE_PASSWORD:-infini_rag_flow}'
host: '${OCEANBASE_HOST:-oceanbase}'
port: ${OCEANBASE_PORT:-2881}
+gaussdb:
+ host: '${GAUSSDB_HOST}'
+ port: ${GAUSSDB_PORT}
+ database: '${GAUSSDB_DATABASE:-postgres}'
+ user: '${GAUSSDB_USER}'
+ password: '${GAUSSDB_PASSWORD}'
+ schema: '${GAUSSDB_SCHEMA:-public}'
seekdb:
scheme: 'oceanbase' # SeekDB is the lite version of OceanBase
config:
@@ -103,6 +110,8 @@ user_default_llm:
# port: 5432
# max_connections: 100
# stale_timeout: 30
+# GaussDB metadata DB is configured through GAUSSDB_METADATA_* environment
+# variables and is intentionally not rendered into service_conf.yaml.
# s3:
# access_key: 'access_key'
# secret_key: 'secret_key'
diff --git a/memory/utils/gaussdb_conn.py b/memory/utils/gaussdb_conn.py
new file mode 100644
index 0000000000..8b48fbe2af
--- /dev/null
+++ b/memory/utils/gaussdb_conn.py
@@ -0,0 +1,1607 @@
+#
+# Copyright 2025 The InfiniFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+"""GaussDB A/ORA-compatible Memory Store adapter.
+
+This module stores Memory messages only; it does not manage memory metadata in
+the metadata database. The service constructs a logical index_name per tenant,
+which this adapter maps to a physical GaussDB table. Rows within the shared
+tenant table are isolated by memory_id.
+
+The implementation reuses the connection pool, identifier validation, and
+selected utilities from common.doc_store.gaussdb_conn_base. Its table schema,
+field mapping, full-text/vector/hybrid queries, capacity accounting, and FIFO
+operations remain independent so DocEngine chunk semantics never leak into
+Memory messages.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import logging
+import re
+from datetime import datetime
+from decimal import Decimal, InvalidOperation
+from typing import Any, Iterable
+
+import numpy as np
+from pydantic import BaseModel
+
+from common.doc_store.doc_store_base import FusionExpr, MatchDenseExpr, MatchExpr, MatchTextExpr, OrderByExpr
+from common.doc_store.gaussdb_conn_base import GaussDBConnectionBase, GaussDBDDLBuilder, InvalidGaussDBObjectName
+from common.doc_store.gaussdb_conn_pool import GaussDBConnectionPool, classify_gaussdb_exception
+from common.float_utils import get_float
+from memory.utils.aggregation_utils import aggregate_by_field
+from memory.utils.highlight_utils import get_highlight_from_messages
+from rag.nlp import is_english
+from rag.nlp.rag_tokenizer import fine_grained_tokenize, tokenize
+
+logger = logging.getLogger("ragflow.memory_gaussdb_conn")
+
+VECTOR_COLUMN_RE = re.compile(r"^q_(?P<dim>\d+)_vec$")
+VECTOR_EMPTY_COLUMN_RE = re.compile(r"^q_(?P<dim>\d+)_vec_empty$")
+
+# Base columns for Memory messages. Vector columns are added dynamically for
+# each embedding dimension and are not listed here. These names also form the
+# SQL allowlist: every external field must resolve to a base column or a
+# q_{dim}_vec / q_{dim}_vec_empty dynamic column.
+BASE_COLUMNS = (
+ "id",
+ "message_id",
+ "message_type_kwd",
+ "source_id",
+ "memory_id",
+ "user_id",
+ "agent_id",
+ "session_id",
+ "zone_id",
+ "valid_at",
+ "invalid_at",
+ "forget_at",
+ "status_int",
+ "content_ltks",
+ "tokenized_content_ltks",
+)
+BASE_COLUMN_SET = set(BASE_COLUMNS)
+
+TIME_COLUMNS = {"valid_at", "invalid_at", "forget_at"}
+NUMERIC_COLUMNS = {"message_id", "source_id", "zone_id", "status_int"}
+# MemoryService uses backend-neutral field names. GaussDB physical tables use
+# suffixed names to avoid conflicts with full-text, vector, and status columns.
+# Map every field before constructing SQL.
+MEMORY_FIELD_MAP = {
+ "message_type": "message_type_kwd",
+ "status": "status_int",
+ "content": "content_ltks",
+}
+REVERSE_MEMORY_FIELD_MAP = {
+ "message_type_kwd": "message_type",
+ "status_int": "status",
+ "content_ltks": "content",
+}
+RESULT_FIELD_DEFAULTS = {
+ "source_id": None,
+ "user_id": "",
+ "zone_id": 0,
+ "invalid_at": "-",
+ "forget_at": "-",
+ "content": "",
+ "content_embed": [],
+}
+
+
+def normalize_fulltext_query(text: Any) -> str:
+ # Writes store fine_grained_tokenize(tokenize(content)). Apply the same
+ # normalization before plainto_tsquery so short terms, English stems, and
+ # mixed-language content do not produce false negatives from mismatched
+ # write-time and query-time tokens.
+ query = str(text or "").strip()
+ if not query:
+ return ""
+ tokenized = fine_grained_tokenize(tokenize(query)).strip()
+ return tokenized or query
+
+
+class SearchResult(BaseModel):
+ # Match other Memory Store adapters: search() returns (SearchResult, total),
+ # while get_total/get_doc_ids/get_fields accept either that tuple or a bare
+ # SearchResult.
+ total: int
+ messages: list[dict]
+
+
+class GaussDBMemoryDDLBuilder(GaussDBDDLBuilder):
+ """Build DDL for Memory message tables.
+
+ The shared GaussDBDDLBuilder handles identifiers, safe table names, and the
+ advisory lock. Memory tables additionally need A/ORA-compatible types,
+ UStore, UGIN full-text indexes, gsdiskann vector indexes, and vector-empty
+ markers, all of which are maintained here.
+ """
+
+ # Standard indexes cover listing, deletion, capacity accounting, FIFO,
+ # recent-message access, and source/raw relationships. Vector and full-text
+ # indexes are generated separately because they depend on dynamic dimensions
+ # or expression indexes.
+ REGULAR_INDEXES = (
+ ("message_id", ("message_id",)),
+ ("memory_id", ("memory_id",)),
+ ("message_type", ("message_type_kwd",)),
+ ("source_id", ("source_id",)),
+ ("agent_session", ("agent_id", "session_id")),
+ ("status_valid", ("status_int", "valid_at")),
+ ("forget_at", ("forget_at",)),
+ )
+
+ def physical_table_name(self, index_name: str) -> str:
+ logical = str(index_name or "").strip()
+ if not logical:
+ raise InvalidGaussDBObjectName(index_name)
+ # A logical index_name may contain tenant IDs, hyphens, or other
+ # service-generated characters. Expose only a hash suffix to keep the
+ # physical name stable, short, and valid for GaussDB. memory_id is not
+ # part of the table name; it isolates memories within the tenant table.
+ digest = hashlib.sha1(logical.encode("utf-8")).hexdigest()[:32]
+ return f"ragflow_mem_{digest}"
+
+ def build_memory_table_ddl(self, table: str) -> str:
+ # Use A/ORA-compatible VARCHAR2/NUMBER types for bounded text and
+ # integers while retaining TEXT for full-text content. UStore is required
+ # by the gsdiskann/vector retrieval path and is selected at table creation.
+ name = self.qualified_name(table)
+ pk = self.index_name(table, "pk")
+ return f"""CREATE TABLE IF NOT EXISTS {name} (
+ id VARCHAR2(96) NOT NULL,
+ message_id NUMBER(19) NOT NULL,
+ message_type_kwd VARCHAR2(64),
+ source_id NUMBER(19),
+ memory_id VARCHAR2(32) NOT NULL,
+ user_id VARCHAR2(64),
+ agent_id VARCHAR2(64),
+ session_id VARCHAR2(128),
+ zone_id NUMBER(10) DEFAULT 0,
+ valid_at TIMESTAMP,
+ invalid_at TIMESTAMP,
+ forget_at TIMESTAMP,
+ status_int NUMBER(10) DEFAULT 1 NOT NULL,
+ content_ltks TEXT,
+ tokenized_content_ltks TEXT,
+ CONSTRAINT {pk} PRIMARY KEY (id)
+) WITH (storage_type=USTORE)"""
+
+ def build_regular_index_ddls(self, table: str) -> list[str]:
+ name = self.qualified_name(table)
+ return [f"CREATE INDEX IF NOT EXISTS {self.index_name(table, suffix)} ON {name} ({', '.join(columns)})" for suffix, columns in self.REGULAR_INDEXES]
+
+ def build_fulltext_ugin_ddl(self, table: str) -> str:
+ # Index the simple tsvector expression over tokenized_content_ltks. The
+ # query path uses the same simple configuration with plainto_tsquery so
+ # behavior does not depend on the database's default language.
+ name = self.qualified_name(table)
+ idx = self.index_name(table, "tokenized_ugin")
+ return f"""CREATE INDEX IF NOT EXISTS {idx}
+ ON {name}
+ USING ugin (to_tsvector('simple', tokenized_content_ltks))"""
+
+ def vector_empty_column_name(self, dim: int) -> str:
+ return f"q_{self.validate_vector_dim(dim)}_vec_empty"
+
+ def build_vector_column_ddls(self, table: str, dim: int) -> list[str]:
+ # GaussDB floatvector columns cannot use NULL to represent a message
+ # without a vector of this dimension. Pair every vector column with an
+ # *_empty marker and use a zero-vector placeholder. Retrieval filters on
+ # *_empty = FALSE, and reads restore only marked non-empty dimensions.
+ dim = self.validate_vector_dim(dim)
+ name = self.qualified_name(table)
+ vector_col = self.vector_column_name(dim)
+ empty_col = self.vector_empty_column_name(dim)
+ return [
+ f"ALTER TABLE {name} ADD COLUMN IF NOT EXISTS {vector_col} floatvector({dim}) DEFAULT (array_fill(0, ARRAY[{dim}])::text::floatvector({dim})) NOT NULL",
+ f"ALTER TABLE {name} ADD COLUMN IF NOT EXISTS {empty_col} BOOLEAN DEFAULT TRUE NOT NULL",
+ ]
+
+ def build_vector_empty_index_ddl(self, table: str, dim: int) -> str:
+ dim = self.validate_vector_dim(dim)
+ name = self.qualified_name(table)
+ empty_col = self.vector_empty_column_name(dim)
+ idx = self.index_name(table, f"{empty_col}_idx")
+ return f"CREATE INDEX IF NOT EXISTS {idx} ON {name} (memory_id, {empty_col})"
+
+ def build_diskann_index_ddl(self, table: str, dim: int) -> str:
+ # gsdiskann provides approximate nearest-neighbor lookup. Include the
+ # dimension column in the index name so one tenant table can hold several
+ # embedding dimensions during model migration.
+ dim = self.validate_vector_dim(dim)
+ name = self.qualified_name(table)
+ vector_col = self.vector_column_name(dim)
+ idx = self.index_name(table, f"{vector_col}_diskann")
+ return f"CREATE INDEX IF NOT EXISTS {idx} ON {name} USING gsdiskann ({vector_col} cosine)"
+
+
+class GaussDBMemoryConnection(GaussDBConnectionBase):
+ """Store Memory messages in GaussDB.
+
+ This class implements the Message Store interface expected by MemoryService.
+ It exposes no arbitrary SQL and does not reuse DocEngine chunk queries. Every
+ write, update, delete, and retrieval enforces a memory_id boundary so messages
+ cannot cross memories within a shared tenant table.
+ """
+
+ def __init__(self, pool: GaussDBConnectionPool | None = None):
+ super().__init__(pool=pool, logger_name="ragflow.memory_gaussdb_conn")
+ # The base class initializes the shared pool and validates schema access.
+ # Replace its DDL builder so create_idx and vector-column maintenance use
+ # the message schema instead of the document chunk schema.
+ self.ddl = GaussDBMemoryDDLBuilder(schema=self.resolved_schema)
+
+ def create_idx(self, index_name: str, memory_id: str, vector_size: int, parser_id: str = None):
+ table = self.physical_table(index_name)
+ # Acquire the advisory lock before creating the base table, standard
+ # indexes, full-text index, vector column, and vector-empty index. The
+ # idempotent DDL supports repeated calls, concurrent initialization, and
+ # first writes from multiple memories in one tenant.
+ statements: list[str | tuple[str, list[Any]]] = [
+ self.ddl.build_advisory_lock_sql(f"gaussdb_memory_create_table:{table}"),
+ self.ddl.build_memory_table_ddl(table),
+ self.ddl.build_advisory_lock_sql(f"gaussdb_memory_base_index:{table}"),
+ *self.ddl.build_regular_index_ddls(table),
+ self.ddl.build_advisory_lock_sql(f"gaussdb_memory_fulltext_index:{table}"),
+ self.ddl.build_fulltext_ugin_ddl(table),
+ ]
+ statements.extend(self.ddl.build_vector_column_ddls(table, vector_size))
+ statements.append(self.ddl.build_vector_empty_index_ddl(table, vector_size))
+ self._execute_statements(statements)
+ self._create_diskann_index_with_retry(table, vector_size)
+ return True
+
+ def delete_idx(self, index_name: str, memory_id: str):
+ # The physical table boundary is the tenant-level index_name, not
+ # memory_id. Tenant/user deletion may call delete_idx once per memory, so
+ # DROP TABLE IF EXISTS must be repeatable. memory_id remains for interface
+ # compatibility only.
+ table = self.ddl.qualified_name(self.physical_table(index_name))
+ self._execute_write(f"DROP TABLE IF EXISTS {table} PURGE", [])
+ return True
+
+ def index_exist(self, index_name: str, memory_id: str = None) -> bool:
+ # has_index() checks the required base columns and indexes as well as the
+ # table itself. Legacy or partially initialized tables therefore enter
+ # the create/ensure path instead of failing later during a query.
+ table = self.physical_table(index_name)
+ if not self._table_exists(table):
+ return False
+ required_columns = set(BASE_COLUMNS)
+ existing_columns = set(self._column_names(table))
+ if not required_columns.issubset(existing_columns):
+ return False
+ required_indexes = {self.ddl.index_name(table, suffix) for suffix, _columns in self.ddl.REGULAR_INDEXES}
+ required_indexes.add(self.ddl.index_name(table, "tokenized_ugin"))
+ existing_indexes = set(self._index_names(table))
+ return required_indexes.issubset(existing_indexes)
+
+ def insert(self, documents: list[dict], index_name: str, memory_id: str = None) -> list[str]:
+ if not documents:
+ return []
+
+ document_ids = [str(document.get("id") or "") for document in documents]
+ errors: list[str] = []
+ rows: list[dict] = []
+ dim = None
+ for document in documents:
+ doc_id = str(document.get("id") or "")
+ try:
+ # One batch must use one embedding dimension because a MERGE can
+ # bind only one q_{dim}_vec column. Reject mixed dimensions before
+ # executing SQL and return the failed IDs.
+ row, row_dim = self._message_to_row(document, memory_id)
+ dim = row_dim if dim is None else dim
+ if row_dim != dim:
+ raise ValueError(f"inconsistent content_embed dimension: expected {dim}, got {row_dim}")
+ rows.append(row)
+ except Exception as exc:
+ logger.error("GaussDB memory normalize failed id=%s error=%s", doc_id, exc)
+ errors.append(doc_id or str(exc))
+ if errors:
+ return [document_id for document_id in document_ids if document_id] or errors
+
+ table = self.physical_table(index_name)
+ if not self._table_exists(table):
+ # The first write creates the tenant message table. memory_id stays
+ # in row data and WHERE predicates as the intra-table boundary.
+ self.create_idx(index_name, memory_id, dim)
+ else:
+ # An existing tenant table may lack the new column after an embedding
+ # model dimension changes. Add the column and index on demand without
+ # dropping older dimensions so historical rows remain readable via
+ # their *_empty markers.
+ self._ensure_vector_column_exists(table, dim)
+
+ existing_dims = self._vector_dimensions(table)
+ sql, params = self._build_merge_sql(table, dim, existing_dims, rows)
+ try:
+ self._execute_write(sql, params, many=True)
+ return []
+ except Exception as exc:
+ ids = [row["id"] for row in rows]
+ logger.error("GaussDB memory insert failed table=%s ids=%s error=%s", table, ids, exc)
+ return ids or [str(exc)]
+
+ def update(self, condition: dict, new_value: dict, index_name: str, memory_id: str) -> bool:
+ if not condition or not new_value:
+ return False
+ table = self.physical_table(index_name)
+ if not self._table_exists(table):
+ return True
+
+ try:
+ set_sql, set_params = self._build_update_set(table, index_name, memory_id, new_value)
+ if not set_sql:
+ return True
+ where_sql, where_params = self._build_where_clause(condition, memory_ids=[memory_id], force_memory_filter=True)
+ if not where_sql:
+ return False
+ # Always add the memory_id boundary, even when the caller supplies
+ # only message_id, so an update cannot cross memories in one tenant.
+ sql = f"UPDATE {self.ddl.qualified_name(table)} SET {set_sql} WHERE {where_sql}"
+ self._execute_write(sql, [*set_params, *where_params])
+ return True
+ except Exception as exc:
+ logger.error("GaussDB memory update failed table=%s condition=%s error=%s", table, condition, exc)
+ return False
+
+ def delete(self, condition: dict, index_name: str, memory_id: str) -> int:
+ if not condition:
+ return 0
+ if self._has_empty_delete_list(condition):
+ # An empty-list delete is a no-op. Building SQL from it could collapse
+ # to a DELETE constrained only by memory_id and erase the memory.
+ return 0
+ table = self.physical_table(index_name)
+ if not self._table_exists(table):
+ return 0
+ try:
+ where_sql, where_params = self._build_where_clause(condition, memory_ids=[memory_id], force_memory_filter=True)
+ if not where_sql:
+ return 0
+ # delete_message() often provides only message_id or source_id. Add
+ # memory_id consistently to isolate rows in the shared tenant table.
+ return self._execute_write(f"DELETE FROM {self.ddl.qualified_name(table)} WHERE {where_sql}", where_params)
+ except Exception as exc:
+ logger.error("GaussDB memory delete failed table=%s condition=%s error=%s", table, condition, exc)
+ return 0
+
+ def get(self, doc_id: str, index_name: str, memory_ids: list[str]) -> dict | None:
+ if not doc_id:
+ return None
+ table = self.physical_table(index_name)
+ if not self._table_exists(table):
+ return None
+ # Read all base columns plus every existing vector and empty-marker
+ # column. _message_from_row() restores the valid content_embed dimension.
+ columns = [*BASE_COLUMNS]
+ columns.extend(self._vector_columns_for_select(table))
+ sql = f"SELECT {', '.join(columns)} FROM {self.ddl.qualified_name(table)} WHERE id = %s"
+ row, description = self._fetch_one_with_description(sql, [doc_id])
+ if row is None:
+ return None
+ return self._message_from_row(self._row_to_dict(row, description))
+
+ def search(
+ self,
+ select_fields: list[str],
+ highlight_fields: list[str],
+ condition: dict,
+ match_expressions: list[MatchExpr],
+ order_by: OrderByExpr,
+ offset: int,
+ limit: int,
+ index_names: str | list[str],
+ memory_ids: list[str],
+ agg_fields: list[str] | None = None,
+ rank_feature: dict | None = None,
+ hide_forgotten: bool = True,
+ **kwargs,
+ ):
+ tables = [self.physical_table(name) for name in normalize_index_names(index_names)]
+ memory_ids = clean_list_values(memory_ids)
+ if not tables or not memory_ids:
+ return SearchResult(total=0, messages=[]), 0
+
+ parsed = self._parse_match_expressions(match_expressions)
+ has_match = bool(parsed["text_query"] or parsed["vector"])
+ # For a multi-tenant fan-out, fetch offset+limit candidates per table and
+ # merge, sort, and slice them in memory. A single-table query pushes
+ # offset and limit directly into SQL.
+ collection_limit = max(int(offset or 0), 0) + max(int(limit or 0), 0)
+ if collection_limit <= 0:
+ collection_limit = 10000
+
+ result = SearchResult(total=0, messages=[])
+ for table in tables:
+ if not self._table_exists(table):
+ continue
+ sql, params = self._build_search_sql(
+ table=table,
+ select_fields=select_fields,
+ highlight_fields=highlight_fields,
+ condition=condition,
+ parsed=parsed,
+ order_by=order_by,
+ offset=0 if len(tables) > 1 else max(int(offset or 0), 0),
+ limit=collection_limit if len(tables) > 1 else max(int(limit or 0), 0),
+ memory_ids=memory_ids,
+ hide_forgotten=hide_forgotten,
+ )
+ rows, description = self._fetch_all_with_description(sql, params)
+ table_total, messages = self._rows_to_messages(rows, description)
+ result.total += table_total
+ result.messages.extend(messages)
+
+ result.messages = self._sort_messages(result.messages, order_by, has_match)
+ if len(tables) > 1:
+ # Apply final pagination only after globally sorting the merged
+ # tenant results; concatenating locally paged results is incorrect.
+ effective_offset = max(int(offset or 0), 0)
+ effective_limit = max(int(limit or 0), 0)
+ if effective_limit:
+ result.messages = result.messages[effective_offset : effective_offset + effective_limit]
+ return result, result.total
+
+ def get_forgotten_messages(self, select_fields: list[str], index_name: str, memory_id: str, limit: int = 512):
+ table = self.physical_table(index_name)
+ if not self._table_exists(table):
+ return None
+ columns = self._select_columns(table, select_fields)
+ sql = (
+ f"SELECT {', '.join(columns)} "
+ f"FROM {self.ddl.qualified_name(table)} "
+ "WHERE memory_id = %s AND forget_at IS NOT NULL "
+ # LIMIT is more reliable than FETCH FIRST for these queries on
+ # A/ORA-compatible GaussDB and avoids dialect errors in maintenance.
+ "ORDER BY forget_at ASC LIMIT %s"
+ )
+ rows, description = self._fetch_all_with_description(sql, [memory_id, int(limit)])
+ _total, messages = self._rows_to_messages(rows, description)
+ return SearchResult(total=len(messages), messages=messages)
+
+ def get_missing_field_message(
+ self,
+ select_fields: list[str],
+ index_name: str,
+ memory_id: str,
+ field_name: str,
+ limit: int = 512,
+ ):
+ table = self.physical_table(index_name)
+ if not self._table_exists(table):
+ return None
+ db_field = self.convert_field_name(field_name)
+ self._validate_column(db_field)
+ columns = self._select_columns(table, select_fields)
+ sql = (
+ f"SELECT {', '.join(columns)} "
+ f"FROM {self.ddl.qualified_name(table)} "
+ f"WHERE memory_id = %s AND {db_field} IS NULL "
+ # Keep maintenance scans consistent with get_forgotten_messages.
+ "ORDER BY valid_at ASC LIMIT %s"
+ )
+ rows, description = self._fetch_all_with_description(sql, [memory_id, int(limit)])
+ _total, messages = self._rows_to_messages(rows, description)
+ return SearchResult(total=len(messages), messages=messages)
+
+ def get_total(self, res) -> int:
+ if isinstance(res, tuple):
+ return int(res[1] or 0)
+ return int(getattr(res, "total", 0) or 0)
+
+ def get_doc_ids(self, res) -> list[str]:
+ if isinstance(res, tuple):
+ res = res[0]
+ return [row["id"] for row in getattr(res, "messages", []) if row.get("id")]
+
+ def get_fields(self, res, fields: list[str]) -> dict[str, dict]:
+ if isinstance(res, tuple):
+ res = res[0]
+ requested = set(fields or [])
+ if not requested:
+ return {}
+ result: dict[str, dict] = {}
+ for row in getattr(res, "messages", []) or []:
+ message = self._message_from_row(row)
+ doc_id = row.get("id") or message.get("id")
+ if not doc_id:
+ continue
+ item = {}
+ for field in fields:
+ if field in message:
+ item[field] = message[field]
+ elif field in RESULT_FIELD_DEFAULTS:
+ item[field] = RESULT_FIELD_DEFAULTS[field]
+ if item:
+ result[str(doc_id)] = item
+ return result
+
+ def get_highlight(self, res, keywords: list[str], field_name: str):
+ if isinstance(res, tuple):
+ res = res[0]
+ return get_highlight_from_messages(
+ getattr(res, "messages", None),
+ keywords,
+ field_name,
+ is_english_fn=lambda s: is_english([s]),
+ )
+
+ def get_aggregation(self, res, field_name: str):
+ if isinstance(res, tuple):
+ res = res[0]
+ return aggregate_by_field(getattr(res, "messages", None), field_name)
+
+ def sql(self, sql: str, fetch_size: int = 128, format: str = "json"):
+ # Memory Store exposes no arbitrary SQL. DocEngine SQL Q&A has its own
+ # read-only validator, while Memory tables must never be accessed by
+ # external SQL that can bypass the memory_id boundary.
+ logger.warning("GaussDB Memory Store does not expose raw SQL execution.")
+ return None
+
+ def physical_table(self, index_name: str) -> str:
+ return self.ddl.physical_table_name(index_name)
+
+ @staticmethod
+ def convert_field_name(field_name: str, use_tokenized_content: bool = False) -> str:
+ # Normal reads and writes map content to the original-text column.
+ # Full-text matching explicitly requests tokenized_content_ltks so
+ # get_fields() does not return tokenized content.
+ if field_name == "content" and use_tokenized_content:
+ return "tokenized_content_ltks"
+ return MEMORY_FIELD_MAP.get(field_name, field_name)
+
+ def _message_to_row(self, message: dict, memory_id: str | None) -> tuple[dict, int]:
+ content_embed = message.get("content_embed")
+ if content_embed is None or len(content_embed) == 0:
+ raise ValueError("content_embed is required for GaussDB memory insert")
+ dim = self.ddl.validate_vector_dim(len(content_embed))
+ target_memory_id = str(message.get("memory_id") or memory_id or "")
+ if not target_memory_id:
+ raise ValueError("memory_id is required")
+ # Normalize each row by mapping application fields to physical columns,
+ # converting empty user/agent/session IDs to NULL, and writing
+ # content_embed to its dimension-specific vector column with *_empty set
+ # to FALSE.
+ row = {
+ "id": str(message.get("id") or f"{target_memory_id}_{message['message_id']}"),
+ "message_id": to_int_or_none(message.get("message_id")),
+ "message_type_kwd": message.get("message_type"),
+ "source_id": to_int_or_none(message.get("source_id")),
+ "memory_id": target_memory_id,
+ "user_id": none_if_empty(message.get("user_id")),
+ "agent_id": none_if_empty(message.get("agent_id")),
+ "session_id": none_if_empty(message.get("session_id")),
+ "zone_id": to_int_or_none(message.get("zone_id", 0)) or 0,
+ "valid_at": normalize_timestamp(message.get("valid_at")),
+ "invalid_at": normalize_timestamp(message.get("invalid_at")),
+ "forget_at": normalize_timestamp(message.get("forget_at")),
+ "status_int": 1 if bool(message.get("status")) else 0,
+ "content_ltks": message.get("content") or "",
+ "tokenized_content_ltks": fine_grained_tokenize(tokenize(message.get("content") or "")),
+ self.ddl.vector_column_name(dim): vector_literal(content_embed, dim),
+ self.ddl.vector_empty_column_name(dim): False,
+ }
+ return row, dim
+
+ def _message_from_row(self, row: dict) -> dict:
+ # Restore backend-neutral field names and defaults. Empty invalid_at and
+ # forget_at remain "-", while empty content and user_id remain "", so
+ # callers do not need GaussDB-specific NULL handling.
+ message = {
+ "id": row.get("id"),
+ "message_id": to_int_or_original(row.get("message_id")),
+ "message_type": row.get("message_type_kwd"),
+ "source_id": to_int_or_original(row.get("source_id")) if row.get("source_id") is not None else None,
+ "memory_id": row.get("memory_id"),
+ "user_id": row.get("user_id") or "",
+ "agent_id": row.get("agent_id"),
+ "session_id": row.get("session_id"),
+ "zone_id": to_int_or_original(row.get("zone_id")) if row.get("zone_id") is not None else 0,
+ "valid_at": format_timestamp(row.get("valid_at")),
+ "invalid_at": format_timestamp(row.get("invalid_at")) or "-",
+ "forget_at": format_timestamp(row.get("forget_at")) or "-",
+ "status": bool(int(row.get("status_int") or 0)),
+ "content": row.get("content_ltks") or "",
+ "content_embed": self._content_embed_from_row(row),
+ }
+ if row.get("_score") is not None:
+ message["_score"] = float(row.get("_score") or 0.0)
+ return message
+
+ def _content_embed_from_row(self, row: dict) -> list[float]:
+ candidates = []
+ for key, value in row.items():
+ match = VECTOR_COLUMN_RE.fullmatch(str(key))
+ if not match:
+ continue
+ dim = int(match.group("dim"))
+ if row.get(self.ddl.vector_empty_column_name(dim)) is False:
+ candidates.append((dim, parse_vector_value(value)))
+ if not candidates:
+ return []
+ if len(candidates) > 1:
+ # A message normally has one non-empty vector dimension. If legacy
+ # data or a manual repair exposes several, log it and return the
+ # highest dimension instead of failing the read.
+ logger.warning("GaussDB memory row %s has multiple non-empty vector dimensions.", row.get("id"))
+ return sorted(candidates, key=lambda item: item[0], reverse=True)[0][1]
+
+ def _build_merge_sql(
+ self,
+ table: str,
+ dim: int,
+ existing_dims: list[int],
+ rows: list[dict],
+ ) -> tuple[str, list[list[Any]]]:
+ vector_col = self.ddl.vector_column_name(dim)
+ empty_col = self.ddl.vector_empty_column_name(dim)
+ table_name = self.ddl.qualified_name(table)
+ reset_other_dims = []
+ for other_dim in existing_dims:
+ if other_dim == dim:
+ continue
+ other_vector = self.ddl.vector_column_name(other_dim)
+ other_empty = self.ddl.vector_empty_column_name(other_dim)
+ reset_other_dims.append(f"{other_vector} = '{zero_vector_literal(other_dim)}'::floatvector")
+ reset_other_dims.append(f"{other_empty} = TRUE")
+ reset_clause = "".join(f",\n {assignment}" for assignment in reset_other_dims)
+ # A/ORA-compatible GaussDB uses MERGE ... USING (SELECT ... FROM dual)
+ # for upserts instead of PostgreSQL ON CONFLICT. Updating the active
+ # dimension resets all others to zero vectors with *_empty=TRUE so each
+ # row exposes only one real content_embed.
+ sql = f"""
+MERGE INTO {table_name} t
+USING (
+ SELECT
+ %s AS id,
+ %s AS message_id,
+ %s AS message_type_kwd,
+ %s AS source_id,
+ %s AS memory_id,
+ %s AS user_id,
+ %s AS agent_id,
+ %s AS session_id,
+ %s AS zone_id,
+ %s::timestamp AS valid_at,
+ %s::timestamp AS invalid_at,
+ %s::timestamp AS forget_at,
+ %s AS status_int,
+ %s AS content_ltks,
+ %s AS tokenized_content_ltks,
+ %s::floatvector({dim}) AS {vector_col}
+ FROM dual
+) s
+ON (t.id = s.id)
+WHEN MATCHED THEN UPDATE SET
+ message_id = s.message_id,
+ message_type_kwd = s.message_type_kwd,
+ source_id = s.source_id,
+ memory_id = s.memory_id,
+ user_id = s.user_id,
+ agent_id = s.agent_id,
+ session_id = s.session_id,
+ zone_id = s.zone_id,
+ valid_at = s.valid_at,
+ invalid_at = s.invalid_at,
+ forget_at = s.forget_at,
+ status_int = s.status_int,
+ content_ltks = s.content_ltks,
+ tokenized_content_ltks = s.tokenized_content_ltks,
+ {vector_col} = s.{vector_col},
+ {empty_col} = FALSE{reset_clause}
+WHEN NOT MATCHED THEN INSERT (
+ id, message_id, message_type_kwd, source_id, memory_id, user_id, agent_id,
+ session_id, zone_id, valid_at, invalid_at, forget_at, status_int,
+ content_ltks, tokenized_content_ltks, {vector_col}, {empty_col}
+) VALUES (
+ s.id, s.message_id, s.message_type_kwd, s.source_id, s.memory_id, s.user_id,
+ s.agent_id, s.session_id, s.zone_id, s.valid_at, s.invalid_at, s.forget_at,
+ s.status_int, s.content_ltks, s.tokenized_content_ltks, s.{vector_col}, FALSE
+)"""
+ params = [
+ [
+ row.get("id"),
+ row.get("message_id"),
+ row.get("message_type_kwd"),
+ row.get("source_id"),
+ row.get("memory_id"),
+ row.get("user_id"),
+ row.get("agent_id"),
+ row.get("session_id"),
+ row.get("zone_id"),
+ row.get("valid_at"),
+ row.get("invalid_at"),
+ row.get("forget_at"),
+ row.get("status_int"),
+ row.get("content_ltks"),
+ row.get("tokenized_content_ltks"),
+ row.get(vector_col),
+ ]
+ for row in rows
+ ]
+ return sql.strip(), params
+
+ def _build_update_set(self, table: str, index_name: str, memory_id: str, new_value: dict) -> tuple[str, list[Any]]:
+ fragments: list[str] = []
+ params: list[Any] = []
+ for field, value in (new_value or {}).items():
+ if field == "remove":
+ # remove clears fields. Vector columns cannot hold NULL, so use a
+ # zero vector with the corresponding empty marker; set ordinary
+ # columns directly to NULL.
+ remove_fields = [value] if isinstance(value, str) else list(value or [])
+ for remove_field in remove_fields:
+ db_field = self.convert_field_name(remove_field)
+ if VECTOR_COLUMN_RE.fullmatch(db_field):
+ dim = int(VECTOR_COLUMN_RE.fullmatch(db_field).group("dim"))
+ self._ensure_vector_column_exists(table, dim)
+ fragments.append(f"{db_field} = %s::floatvector({dim})")
+ params.append(zero_vector_literal(dim))
+ fragments.append(f"{self.ddl.vector_empty_column_name(dim)} = TRUE")
+ else:
+ self._validate_column(db_field)
+ fragments.append(f"{db_field} = NULL")
+ if db_field == "content_ltks":
+ # content_ltks owns its derived full-text tokens.
+ fragments.append("tokenized_content_ltks = NULL")
+ continue
+ if field == "content_embed":
+ if value is None or len(value) == 0:
+ continue
+ dim = self.ddl.validate_vector_dim(len(value))
+ self._ensure_vector_column_exists(table, dim)
+ vector_col = self.ddl.vector_column_name(dim)
+ # As with insert, retain only the active dimension as a real
+ # vector and reset every other dimension to prevent one message
+ # from matching more than once.
+ fragments.append(f"{vector_col} = %s::floatvector({dim})")
+ params.append(vector_literal(value, dim))
+ fragments.append(f"{self.ddl.vector_empty_column_name(dim)} = FALSE")
+ for other_dim in self._vector_dimensions(table):
+ if other_dim == dim:
+ continue
+ fragments.append(f"{self.ddl.vector_column_name(other_dim)} = %s::floatvector({other_dim})")
+ params.append(zero_vector_literal(other_dim))
+ fragments.append(f"{self.ddl.vector_empty_column_name(other_dim)} = TRUE")
+ continue
+
+ db_field = self.convert_field_name(field)
+ self._validate_column(db_field)
+ if db_field == "content_ltks":
+ # Refresh tokenized_content_ltks whenever the original content
+ # changes so full-text tokens remain current.
+ fragments.append("content_ltks = %s")
+ params.append(value or "")
+ fragments.append("tokenized_content_ltks = %s")
+ params.append(fine_grained_tokenize(tokenize(value or "")))
+ elif db_field in TIME_COLUMNS:
+ fragments.append(f"{db_field} = %s::timestamp")
+ params.append(normalize_timestamp(value))
+ else:
+ fragments.append(f"{db_field} = %s")
+ if db_field in NUMERIC_COLUMNS:
+ params.append(to_int_or_none(value))
+ else:
+ params.append(none_if_empty(value))
+ return ", ".join(fragments), params
+
+ def _build_search_sql(
+ self,
+ table: str,
+ select_fields: list[str],
+ highlight_fields: list[str],
+ condition: dict,
+ parsed: dict[str, Any],
+ order_by: OrderByExpr,
+ offset: int,
+ limit: int,
+ memory_ids: list[str],
+ hide_forgotten: bool,
+ ) -> tuple[str, list[Any]]:
+ text_query = parsed["text_query"]
+ vector = parsed["vector"]
+ # Dispatch according to expression semantics: text plus vector uses
+ # fusion, vector alone uses ANN ordering, text alone uses full-text
+ # ordering, and no match expression falls back to filtering.
+ if text_query and vector:
+ return self._build_fusion_search_sql(table, select_fields, condition, parsed, offset, limit, memory_ids, hide_forgotten)
+ if vector:
+ return self._build_vector_search_sql(table, select_fields, condition, parsed, offset, limit, memory_ids, hide_forgotten)
+ if text_query:
+ return self._build_fulltext_search_sql(table, select_fields, highlight_fields, condition, parsed, offset, limit, memory_ids, hide_forgotten)
+ return self._build_filter_search_sql(table, select_fields, condition, order_by, offset, limit, memory_ids, hide_forgotten)
+
+ def _build_filter_search_sql(
+ self,
+ table: str,
+ select_fields: list[str],
+ condition: dict,
+ order_by: OrderByExpr,
+ offset: int,
+ limit: int,
+ memory_ids: list[str],
+ hide_forgotten: bool,
+ ) -> tuple[str, list[Any]]:
+ columns = self._select_columns(table, select_fields)
+ where_sql, where_params = self._build_where_clause(condition, memory_ids=memory_ids, hide_forgotten=hide_forgotten)
+ order_sql = self._build_order_by(order_by) or "id ASC"
+ # Filter-only lists, recent messages, and capacity accounting reuse
+ # search(). COUNT(*) OVER() returns the total without another query.
+ sql = f"SELECT {', '.join(columns)}, COUNT(*) OVER() AS __total FROM {self.ddl.qualified_name(table)}"
+ if where_sql:
+ sql += f" WHERE {where_sql}"
+ sql += f" ORDER BY {order_sql}"
+ params = [*where_params]
+ if limit and int(limit) > 0:
+ sql += " OFFSET %s LIMIT %s"
+ params.extend([max(int(offset or 0), 0), int(limit)])
+ return sql, params
+
+ def _build_fulltext_search_sql(
+ self,
+ table: str,
+ select_fields: list[str],
+ highlight_fields: list[str],
+ condition: dict,
+ parsed: dict[str, Any],
+ offset: int,
+ limit: int,
+ memory_ids: list[str],
+ hide_forgotten: bool,
+ ) -> tuple[str, list[Any]]:
+ columns = self._select_columns(table, select_fields)
+ text_query = parsed["text_query"]
+ fts_expr = "to_tsvector('simple', tokenized_content_ltks)"
+ # _parse_match_expressions already normalizes text_query with the
+ # write-time tokenizer. This method only builds the GaussDB full-text
+ # expression and ranking.
+ score_expr = f"ts_rank({fts_expr}, plainto_tsquery('simple', %s))"
+ match_expr = f"{fts_expr} @@ plainto_tsquery('simple', %s)"
+ where_sql, where_params = self._build_where_clause(condition, memory_ids=memory_ids, hide_forgotten=hide_forgotten)
+ where_parts = [part for part in (where_sql, match_expr) if part]
+ sql = (
+ f"SELECT {', '.join(columns)}, {score_expr} AS _score, COUNT(*) OVER() AS __total "
+ f"FROM {self.ddl.qualified_name(table)} "
+ f"WHERE {' AND '.join(where_parts)} "
+ "ORDER BY _score DESC, valid_at DESC OFFSET %s LIMIT %s"
+ )
+ return sql, [text_query, *where_params, text_query, max(int(offset or 0), 0), effective_limit(limit, parsed["topn"])]
+
+ def _build_vector_search_sql(
+ self,
+ table: str,
+ select_fields: list[str],
+ condition: dict,
+ parsed: dict[str, Any],
+ offset: int,
+ limit: int,
+ memory_ids: list[str],
+ hide_forgotten: bool,
+ ) -> tuple[str, list[Any]]:
+ dim = parsed["vector_dim"]
+ vector_col = self.ddl.vector_column_name(dim)
+ empty_col = self.ddl.vector_empty_column_name(dim)
+ columns = self._select_columns(table, select_fields)
+ vector = parsed["vector"]
+ threshold = float(parsed["similarity_threshold"] or 0.0)
+ # `<+>` returns cosine distance, while RAGFlow supplies a similarity
+ # threshold. Score and filter with 1 - distance, excluding zero-vector
+ # placeholders through *_empty=FALSE.
+ score_expr = f"1 - ({vector_col} <+> %s::floatvector({dim}))"
+ distance_expr = f"{vector_col} <+> %s::floatvector({dim})"
+ where_sql, where_params = self._build_where_clause(condition, memory_ids=memory_ids, hide_forgotten=hide_forgotten)
+ where_parts = [part for part in (where_sql, f"{empty_col} = FALSE", f"1 - ({vector_col} <+> %s::floatvector({dim})) >= %s") if part]
+ sql = (
+ f"SELECT {', '.join(columns)}, {score_expr} AS _score, COUNT(*) OVER() AS __total "
+ f"FROM {self.ddl.qualified_name(table)} "
+ f"WHERE {' AND '.join(where_parts)} "
+ f"ORDER BY {distance_expr} ASC OFFSET %s LIMIT %s"
+ )
+ return sql, [
+ vector,
+ *where_params,
+ vector,
+ threshold,
+ vector,
+ max(int(offset or 0), 0),
+ effective_limit(limit, parsed["topn"]),
+ ]
+
+ def _build_fusion_search_sql(
+ self,
+ table: str,
+ select_fields: list[str],
+ condition: dict,
+ parsed: dict[str, Any],
+ offset: int,
+ limit: int,
+ memory_ids: list[str],
+ hide_forgotten: bool,
+ ) -> tuple[str, list[Any]]:
+ dim = parsed["vector_dim"]
+ vector_col = self.ddl.vector_column_name(dim)
+ empty_col = self.ddl.vector_empty_column_name(dim)
+ vector = parsed["vector"]
+ threshold = float(parsed["similarity_threshold"] or 0.0)
+ vector_weight = float(parsed["vector_weight"])
+ text_weight = 1.0 - vector_weight
+ text_query = parsed["text_query"]
+ columns = self._select_columns(table, select_fields)
+ inner_columns = unique_preserve_order([*columns, "tokenized_content_ltks", vector_col, empty_col, "valid_at"])
+ fts_expr = "to_tsvector('simple', tokenized_content_ltks)"
+ score_expr = f"ts_rank({fts_expr}, plainto_tsquery('simple', %s))"
+ match_expr = f"{fts_expr} @@ plainto_tsquery('simple', %s)"
+ where_sql, where_params = self._build_where_clause(condition, memory_ids=memory_ids, hide_forgotten=hide_forgotten)
+ where_parts = [part for part in (where_sql, match_expr) if part]
+ candidate_limit = max(int(parsed["topn"] or 0), effective_limit(limit, parsed["topn"]), 1)
+ # Fusion first selects full-text candidates, then applies vector filtering
+ # and a weighted sum within that set. This avoids computing vector scores
+ # for rows without a text match and matches weighted_sum semantics.
+ sql = (
+ "WITH fulltext_results AS ("
+ f" SELECT {', '.join(inner_columns)}, {score_expr} AS relevance "
+ f"FROM {self.ddl.qualified_name(table)} "
+ f"WHERE {' AND '.join(where_parts)} "
+ "ORDER BY relevance DESC LIMIT %s"
+ ") "
+ f"SELECT {', '.join(columns)}, "
+ f"relevance * %s + (1 - ({vector_col} <+> %s::floatvector({dim}))) * %s AS _score, "
+ "COUNT(*) OVER() AS __total "
+ "FROM fulltext_results "
+ f"WHERE {empty_col} = FALSE "
+ f"AND 1 - ({vector_col} <+> %s::floatvector({dim})) >= %s "
+ "ORDER BY _score DESC, valid_at DESC OFFSET %s LIMIT %s"
+ )
+ return sql, [
+ text_query,
+ *where_params,
+ text_query,
+ candidate_limit,
+ text_weight,
+ vector,
+ vector_weight,
+ vector,
+ threshold,
+ max(int(offset or 0), 0),
+ effective_limit(limit, parsed["topn"]),
+ ]
+
+ def _parse_match_expressions(self, match_expressions: list[MatchExpr] | None) -> dict[str, Any]:
+ text_query = ""
+ vector = None
+ vector_dim = None
+ topn = None
+ similarity_threshold = 0.0
+ vector_weight = 0.5
+
+ for expr in match_expressions or []:
+ if isinstance(expr, MatchTextExpr):
+ # MsgTextQuery stores the original query in
+ # extra_options["original_query"]. Prefer it and apply this
+ # adapter's tokenizer instead of consuming matching_text syntax
+ # generated for another backend.
+ text_query = normalize_fulltext_query((expr.extra_options or {}).get("original_query") or expr.matching_text)
+ topn = expr.topn if topn is None else min(topn, expr.topn)
+ elif isinstance(expr, MatchDenseExpr):
+ # vector_column_name may be q_{dim}_vec or the generic
+ # content_embed field. Infer the latter's dimension from the
+ # embedding data.
+ vector_dim = parse_vector_dim(expr.vector_column_name) or len(expr.embedding_data)
+ vector = vector_literal(expr.embedding_data, vector_dim)
+ topn = expr.topn if topn is None else min(topn, expr.topn)
+ similarity_threshold = float((expr.extra_options or {}).get("similarity", 0.0))
+ elif isinstance(expr, FusionExpr):
+ # FusionExpr orders weights as text_weight,vector_weight. Default
+ # to 0.5 when the value is absent or incomplete.
+ weights = (expr.fusion_params or {}).get("weights", "0.5,0.5")
+ parts = str(weights).split(",")
+ if len(parts) > 1:
+ vector_weight = get_float(parts[1])
+ topn = expr.topn if topn is None else min(topn, expr.topn)
+
+ return {
+ "text_query": text_query,
+ "vector": vector,
+ "vector_dim": vector_dim,
+ "topn": topn,
+ "similarity_threshold": similarity_threshold,
+ "vector_weight": vector_weight,
+ }
+
+ def _select_columns(self, table: str, select_fields: list[str] | None) -> list[str]:
+ requested = select_fields or []
+ columns = ["id"]
+ wants_content_embed = False
+ for field in requested:
+ if field in {"_score", "id"}:
+ continue
+ if field == "content_embed":
+ # content_embed is virtual; a physical table may contain several
+ # q_{dim}_vec and *_empty columns. Expand it from the actual table
+ # columns after processing the requested fields.
+ wants_content_embed = True
+ continue
+ db_field = self.convert_field_name(field)
+ self._validate_column(db_field)
+ if db_field not in columns:
+ columns.append(db_field)
+ if wants_content_embed:
+ columns.extend(column for column in self._vector_columns_for_select(table) if column not in columns)
+ return columns
+
+ def _vector_columns_for_select(self, table: str) -> list[str]:
+ columns: list[str] = []
+ # Read each vector with its empty marker to distinguish a real zero
+ # vector from a placeholder.
+ for dim in self._vector_dimensions(table):
+ columns.append(self.ddl.vector_column_name(dim))
+ columns.append(self.ddl.vector_empty_column_name(dim))
+ return columns
+
+ def _build_where_clause(
+ self,
+ condition: dict | None,
+ memory_ids: list[str] | None = None,
+ hide_forgotten: bool = False,
+ force_memory_filter: bool = False,
+ ) -> tuple[str, list[Any]]:
+ fragments: list[str] = []
+ params: list[Any] = []
+
+ memory_values = clean_list_values(memory_ids)
+ if memory_values:
+ # memory_id is the mandatory boundary within a shared tenant table.
+ # Add it from search/update/delete arguments even when the caller's
+ # condition omits it.
+ fragments.append(f"memory_id IN ({', '.join(['%s'] * len(memory_values))})")
+ params.extend(memory_values)
+ elif force_memory_filter:
+ # Return an always-false predicate when a write lacks a memory_id
+ # boundary so it cannot affect the entire tenant table.
+ return "1=0", []
+
+ if hide_forgotten:
+ fragments.append("forget_at IS NULL")
+
+ for field, value in (condition or {}).items():
+ if field == "memory_id" and memory_values:
+ continue
+ if field == "exists":
+ # Startup maintenance and missing-field scans use exists and
+ # must_not-exists semantics. Map and validate the column instead
+ # of interpolating external input directly.
+ db_field = self.convert_field_name(str(value))
+ self._validate_column(db_field)
+ fragments.append(f"{db_field} IS NOT NULL")
+ continue
+ if field == "must_not" and isinstance(value, dict) and "exists" in value:
+ db_field = self.convert_field_name(str(value["exists"]))
+ self._validate_column(db_field)
+ fragments.append(f"{db_field} IS NULL")
+ continue
+ if value is None or value == "":
+ # Treat empty filter values as absent, matching the ES/OB Memory
+ # adapters. Real empty strings are not searchable business data.
+ continue
+ db_field = self.convert_field_name(field)
+ self._validate_column(db_field)
+ if isinstance(value, (list, tuple, set)):
+ values = clean_list_values(value)
+ if not values:
+ continue
+ fragments.append(f"{db_field} IN ({', '.join(['%s'] * len(values))})")
+ params.extend(to_int_or_none(v) if db_field in NUMERIC_COLUMNS else v for v in values)
+ continue
+ fragments.append(f"{db_field} = %s")
+ params.append(to_int_or_none(value) if db_field in NUMERIC_COLUMNS else value)
+ return " AND ".join(fragments), params
+
+ def _build_order_by(self, order_by: OrderByExpr | None) -> str:
+ fields = getattr(order_by, "fields", None) or []
+ parts = []
+ for field, direction in fields:
+ db_field = self.convert_field_name(field)
+ # Apply the same allowlist to ORDER BY fields to prevent SQL injection.
+ self._validate_column(db_field)
+ parts.append(f"{db_field} {'DESC' if direction else 'ASC'}")
+ return ", ".join(parts)
+
+ def _validate_column(self, column: str) -> None:
+ # SQL accepts only base columns or regex-protected dynamic vector columns.
+ # This is the final guard before interpolating any dynamic field name.
+ if column in BASE_COLUMN_SET or VECTOR_COLUMN_RE.fullmatch(column) or VECTOR_EMPTY_COLUMN_RE.fullmatch(column):
+ return
+ raise InvalidGaussDBObjectName(column)
+
+ def _ensure_vector_column_exists(self, table: str, dim: int) -> None:
+ dim = self.ddl.validate_vector_dim(dim)
+ vector_col = self.ddl.vector_column_name(dim)
+ empty_col = self.ddl.vector_empty_column_name(dim)
+ columns_exist = self._column_exists(table, vector_col) and self._column_exists(table, empty_col)
+ statements: list[str | tuple[str, list[Any]]] = [
+ self.ddl.build_advisory_lock_sql(f"gaussdb_memory_vector_column:{table}:{dim}"),
+ ]
+ if not columns_exist:
+ # Add columns only when absent. The empty-marker index uses IF NOT
+ # EXISTS and can run on every ensure call to repair a partial state
+ # where the columns exist but the index does not.
+ statements.extend(self.ddl.build_vector_column_ddls(table, dim))
+ statements.append(self.ddl.build_vector_empty_index_ddl(table, dim))
+ self._execute_statements(statements)
+ if not self._diskann_index_exists(table, dim):
+ # Some GaussDB versions still perform costly gsdiskann initialization
+ # for CREATE INDEX IF NOT EXISTS. Check pg_indexes first to avoid
+ # changing work_mem on every write.
+ self._create_diskann_index_with_retry(table, dim)
+
+ def _create_diskann_index_with_retry(self, table: str, dim: int) -> None:
+ ddl = self.ddl.build_diskann_index_ddl(table, dim)
+ lock = self.ddl.build_advisory_lock_sql(f"gaussdb_memory_vector_index:{table}:{dim}")
+ for work_mem in ("1GB", "2GB", "4GB"):
+ try:
+ # gsdiskann index creation depends on maintenance_work_mem. Retry
+ # with bounded increases only for insufficient-memory errors;
+ # propagate all other DDL failures.
+ self._execute_statements([lock, f"SET LOCAL maintenance_work_mem = '{work_mem}'", ddl])
+ return
+ except Exception as exc:
+ if not is_maintenance_work_mem_error(exc) or work_mem == "4GB":
+ raise
+ logger.warning("Retrying GaussDB gsdiskann index with larger maintenance_work_mem after: %s", exc)
+
+ def _table_exists(self, table: str) -> bool:
+ row = self._fetch_one(
+ """
+ SELECT 1
+ FROM information_schema.tables
+ WHERE table_schema = %s
+ AND table_name = %s
+ """,
+ [self.schema, table],
+ )
+ return bool(row)
+
+ def _column_exists(self, table: str, column: str) -> bool:
+ row = self._fetch_one(
+ """
+ SELECT 1
+ FROM information_schema.columns
+ WHERE table_schema = %s
+ AND table_name = %s
+ AND column_name = %s
+ """,
+ [self.schema, table, column],
+ )
+ return bool(row)
+
+ def _column_names(self, table: str) -> list[str]:
+ rows = self._fetch_all(
+ """
+ SELECT column_name
+ FROM information_schema.columns
+ WHERE table_schema = %s
+ AND table_name = %s
+ """,
+ [self.schema, table],
+ )
+ return [row_value(row, "column_name", 0) for row in rows or []]
+
+ def _index_names(self, table: str) -> list[str]:
+ rows = self._fetch_all(
+ """
+ SELECT indexname
+ FROM pg_indexes
+ WHERE schemaname = %s
+ AND tablename = %s
+ """,
+ [self.schema, table],
+ )
+ return [row_value(row, "indexname", 0) for row in rows or []]
+
+ def _diskann_index_exists(self, table: str, dim: int) -> bool:
+ dim = self.ddl.validate_vector_dim(dim)
+ vector_col = self.ddl.vector_column_name(dim)
+ expected_index = self.ddl.index_name(table, f"{vector_col}_diskann")
+ # A matching name is insufficient: a historical index with another type
+ # cannot support ANN retrieval. Confirm `using gsdiskann` in indexdef.
+ row = self._fetch_one(
+ """
+ SELECT indexdef
+ FROM pg_indexes
+ WHERE schemaname = %s
+ AND tablename = %s
+ AND indexname = %s
+ """,
+ [self.schema, table, expected_index],
+ )
+ if not row:
+ return False
+ return "using gsdiskann" in str(row_value(row, "indexdef", 0)).lower()
+
+ def _vector_dimensions(self, table: str) -> list[int]:
+ dims = []
+ # Infer every embedding dimension used by the tenant table from its
+ # q_{dim}_vec catalog columns. Writes, reads, and content_embed expansion
+ # all depend on this list.
+ for column in self._column_names(table):
+ match = VECTOR_COLUMN_RE.fullmatch(str(column or ""))
+ if match:
+ dims.append(int(match.group("dim")))
+ return sorted(set(dims))
+
+ def _execute_statements(self, statements: Iterable[str | tuple[str, list[Any]]]) -> None:
+ # Commit DDL and maintenance statements as one transaction. Roll back the
+ # group on any failure to avoid columns without their required indexes.
+ conn = self.pool.get_conn()
+ cur = None
+ try:
+ cur = conn.cursor()
+ for statement in statements:
+ if isinstance(statement, tuple):
+ cur.execute(statement[0], statement[1])
+ else:
+ cur.execute(statement)
+ conn.commit()
+ except Exception as exc:
+ conn.rollback()
+ raise classify_gaussdb_exception(exc) from exc
+ finally:
+ close_cursor(cur)
+ self.pool.put_conn(conn)
+
+ def _execute_write(self, sql: str, params: list[Any], many: bool = False) -> int:
+ # Centralize write commits and rollbacks, and classify psycopg2 failures
+ # as GaussDB connection, permission, or authentication errors for logs
+ # and health checks.
+ conn = self.pool.get_conn()
+ cur = None
+ try:
+ cur = conn.cursor()
+ if many:
+ cur.executemany(sql, params)
+ else:
+ cur.execute(sql, params)
+ conn.commit()
+ return int(getattr(cur, "rowcount", 0) or 0)
+ except Exception as exc:
+ conn.rollback()
+ raise classify_gaussdb_exception(exc) from exc
+ finally:
+ close_cursor(cur)
+ self.pool.put_conn(conn)
+
+ def _fetch_one(self, sql: str, params: list[Any]):
+ row, _description = self._fetch_one_with_description(sql, params)
+ return row
+
+ def _fetch_one_with_description(self, sql: str, params: list[Any]):
+ conn = self.pool.get_conn()
+ cur = None
+ try:
+ cur = conn.cursor()
+ cur.execute(sql, params)
+ return cur.fetchone(), getattr(cur, "description", None) or []
+ finally:
+ close_cursor(cur)
+ self.pool.put_conn(conn)
+
+ def _fetch_all(self, sql: str, params: list[Any]):
+ conn = self.pool.get_conn()
+ cur = None
+ try:
+ cur = conn.cursor()
+ cur.execute(sql, params)
+ return cur.fetchall()
+ finally:
+ close_cursor(cur)
+ self.pool.put_conn(conn)
+
+ def _fetch_all_with_description(self, sql: str, params: list[Any]):
+ conn = self.pool.get_conn()
+ cur = None
+ try:
+ cur = conn.cursor()
+ cur.execute(sql, params)
+ return cur.fetchall(), getattr(cur, "description", None) or []
+ finally:
+ close_cursor(cur)
+ self.pool.put_conn(conn)
+
+ def _row_to_dict(self, row, description) -> dict:
+ columns = [desc[0] for desc in description]
+ return dict(zip(columns, row)) if not isinstance(row, dict) else dict(row)
+
+ def _rows_to_messages(self, rows, description) -> tuple[int, list[dict]]:
+ total = 0
+ messages = []
+ for row in rows or []:
+ raw = self._row_to_dict(row, description)
+ if raw.get("__total") is not None:
+ # Search SQL appends the total to each row through COUNT(*)
+ # OVER(). Do not expose the internal __total column to callers.
+ total = int(raw.pop("__total") or 0)
+ messages.append(raw)
+ if total == 0:
+ total = len(messages)
+ return total, messages
+
+ def _sort_messages(self, messages: list[dict], order_by: OrderByExpr | None, has_match: bool) -> list[dict]:
+ if has_match:
+ # The database guarantees ordering only within each tenant table.
+ # Merge match results stably by descending score, descending valid_at,
+ # and ascending id to mirror the SQL ORDER BY.
+ return sorted(
+ messages,
+ key=lambda row: (
+ -float(numeric_sort_value(row.get("_score"))),
+ descending_timestamp_sort_value(row.get("valid_at")),
+ str(row.get("id") or ""),
+ ),
+ )
+ fields = getattr(order_by, "fields", None) or []
+ if not fields:
+ return sorted(messages, key=lambda row: str(row.get("id") or ""))
+ sorted_messages = list(messages)
+ for field, direction in reversed(fields):
+ db_field = self.convert_field_name(field)
+ # Preserve field types while merging: message_id/status sort
+ # numerically and valid_at/forget_at sort chronologically rather than
+ # as strings.
+ sorted_messages.sort(key=lambda row, db_field=db_field: sortable_value(row.get(db_field), db_field), reverse=bool(direction))
+ return sorted_messages
+
+ @staticmethod
+ def _has_empty_delete_list(condition: dict) -> bool:
+ for key in ("message_id", "id"):
+ if key in condition and isinstance(condition[key], (list, tuple, set)) and not clean_list_values(condition[key]):
+ return True
+ return False
+
+
+def normalize_index_names(index_names: str | list[str]) -> list[str]:
+ # Callers may pass one index_name, a comma-separated string, or a list.
+ # Normalize each form to logical names before resolving physical tables.
+ # List elements are stringified, so None becomes "None" under this contract.
+ if isinstance(index_names, str):
+ return [name.strip() for name in index_names.split(",") if name.strip()]
+ return [str(name).strip() for name in index_names or [] if str(name).strip()]
+
+
+def clean_list_values(values) -> list[Any]:
+ # Remove None and empty strings to avoid IN () or treating an empty string as
+ # business data. Wrap one string as a list for consistent ID handling.
+ if values is None:
+ return []
+ if isinstance(values, (str, bytes)):
+ values = [values]
+ result = []
+ for value in values:
+ if value is None or value == "":
+ continue
+ result.append(value)
+ return result
+
+
+def effective_limit(limit: int, topn: int | None = None) -> int:
+ # topn comes from the match expression and limit from pagination. Honor the
+ # smaller positive constraint, or use a conservative cap when neither exists.
+ candidates = [int(value) for value in (limit, topn) if value and int(value) > 0]
+ return min(candidates) if candidates else 10000
+
+
+def parse_vector_dim(column: str) -> int | None:
+ match = VECTOR_COLUMN_RE.fullmatch(str(column or ""))
+ return int(match.group("dim")) if match else None
+
+
+def vector_literal(value, dim: int) -> str:
+ # Bind GaussDB floatvectors as "[1.0,2.0]" and cast them explicitly to
+ # floatvector(dim) in SQL. Reformat parseable strings; leave an unparseable
+ # string unchanged so the database reports the actual input error.
+ if isinstance(value, str):
+ parsed = parse_vector_value(value)
+ if parsed:
+ value = parsed
+ else:
+ return value
+ if isinstance(value, np.ndarray):
+ value = value.tolist()
+ if not isinstance(value, (list, tuple)) or len(value) != int(dim):
+ raise ValueError(f"vector dimension mismatch: expected {dim}, got {len(value) if hasattr(value, '__len__') else 'unknown'}")
+ return "[" + ",".join(str(float(item)) for item in value) + "]"
+
+
+def parse_vector_value(value) -> list[float]:
+ # Reads may return numpy arrays, lists, tuples, JSON arrays, or GaussDB
+ # "[...]" text. Normalize them to a float list; return an empty list when no
+ # usable content_embed can be decoded.
+ if value is None:
+ return []
+ if isinstance(value, np.ndarray):
+ return [float(item) for item in value.tolist()]
+ if isinstance(value, (list, tuple)):
+ return [float(item) for item in value]
+ text = str(value).strip()
+ if not text:
+ return []
+ try:
+ parsed = json.loads(text)
+ except json.JSONDecodeError:
+ if text.startswith("[") and text.endswith("]"):
+ body = text[1:-1].strip()
+ if not body:
+ return []
+ return [float(item.strip()) for item in body.split(",")]
+ return []
+ if isinstance(parsed, list):
+ return [float(item) for item in parsed]
+ return []
+
+
+def zero_vector_literal(dim: int) -> str:
+ # Use a zero vector as the floatvector placeholder; q_{dim}_vec_empty carries
+ # the actual empty-value semantics.
+ return "[" + ",".join(["0"] * int(dim)) + "]"
+
+
+def normalize_timestamp(value) -> str | None:
+ # The Memory interface uses "-", "", and None for empty invalid_at/forget_at
+ # values. Store each as SQL NULL.
+ if value in (None, "", "-"):
+ return None
+ if isinstance(value, datetime):
+ return value.strftime("%Y-%m-%d %H:%M:%S")
+ return str(value)
+
+
+def format_timestamp(value) -> str | None:
+ # Format values without adding "-". _message_from_row owns field defaults so
+ # other internal paths can still distinguish a real None.
+ if value in (None, "", "-"):
+ return None
+ if isinstance(value, datetime):
+ return value.strftime("%Y-%m-%d %H:%M:%S")
+ return str(value)
+
+
+def none_if_empty(value):
+ # Optional user_id/agent_id/session_id fields do not need a stored empty
+ # string. NULL better represents an omitted filter value.
+ if value == "":
+ return None
+ return value
+
+
+def to_int_or_none(value):
+ if value in (None, ""):
+ return None
+ if isinstance(value, bool):
+ return int(value)
+ if isinstance(value, (int, np.integer)):
+ return int(value)
+ if isinstance(value, Decimal):
+ return int(value)
+ return int(value)
+
+
+def to_int_or_original(value):
+ if value is None:
+ return None
+ if isinstance(value, bool):
+ return int(value)
+ if isinstance(value, (int, np.integer)):
+ return int(value)
+ if isinstance(value, Decimal):
+ return int(value)
+ try:
+ return int(value)
+ except (TypeError, ValueError):
+ return value
+
+
+def row_value(row, key: str, index: int):
+ if isinstance(row, dict):
+ return row.get(key)
+ return row[index]
+
+
+def unique_preserve_order(values: list[str]) -> list[str]:
+ result = []
+ for value in values:
+ if value not in result:
+ result.append(value)
+ return result
+
+
+def sortable_value(value, field_name: str | None = None):
+ # Cross-table merges cannot sort every value as text because numeric IDs such
+ # as 100 and 99 would be ordered incorrectly. Use the physical column type
+ # for numeric and timestamp fields, and text ordering for the rest.
+ if field_name in NUMERIC_COLUMNS:
+ return numeric_sort_value(value)
+ if field_name in TIME_COLUMNS:
+ return timestamp_sort_value(value)
+ if value is None:
+ return ""
+ if isinstance(value, datetime):
+ return value.isoformat()
+ return str(value)
+
+
+def numeric_sort_value(value) -> Decimal:
+ # Decimal compares integers, numeric strings, and booleans consistently.
+ # Map invalid or empty values to negative infinity so descending order places
+ # them last.
+ if value in (None, ""):
+ return Decimal("-Infinity")
+ if isinstance(value, bool):
+ return Decimal(int(value))
+ try:
+ return Decimal(str(value))
+ except (InvalidOperation, TypeError, ValueError):
+ return Decimal("-Infinity")
+
+
+def timestamp_sort_value(value) -> float:
+ # Accept datetime objects, ISO strings, and the project's common
+ # "%Y-%m-%d %H:%M:%S" format. Treat empty timestamps as negative infinity
+ # for merged ascending or descending sorts.
+ if value in (None, "", "-"):
+ return float("-inf")
+ if isinstance(value, datetime):
+ return value.timestamp()
+ text = str(value).strip()
+ if not text:
+ return float("-inf")
+ try:
+ return datetime.fromisoformat(text).timestamp()
+ except ValueError:
+ pass
+ try:
+ return datetime.strptime(text, "%Y-%m-%d %H:%M:%S").timestamp()
+ except ValueError:
+ return float("-inf")
+
+
+def descending_timestamp_sort_value(value) -> float:
+ # sorted() is ascending, so negate valid timestamps to obtain valid_at DESC.
+ # Map empty values to positive infinity so they remain last.
+ timestamp = timestamp_sort_value(value)
+ if timestamp == float("-inf"):
+ return float("inf")
+ return -timestamp
+
+
+def is_maintenance_work_mem_error(exc: Exception) -> bool:
+ # Retry gsdiskann creation only when maintenance_work_mem is insufficient.
+ # Dialect, permission, and extension errors must propagate.
+ text = str(exc).lower()
+ return "maintenance_work_mem" in text and ("required" in text or "below" in text or "insufficient" in text)
+
+
+def close_cursor(cur) -> None:
+ # Ignore cleanup failures so they do not hide the original SQL error.
+ if cur is None:
+ return
+ try:
+ cur.close()
+ except Exception:
+ pass
diff --git a/rag/app/table.py b/rag/app/table.py
index 114f62dd58..dc76316588 100644
--- a/rag/app/table.py
+++ b/rag/app/table.py
@@ -547,10 +547,12 @@ def chunk(filename, binary=None, from_page=0, to_page=MAXIMUM_TASK_PAGE_NUMBER,
clmns_map = [(py_clmns[i].lower() + fields_map[clmn_tys[i]], str(clmns[i]).replace("_", " ")) for i in range(len(clmns))]
# field_map: only columns stored in chunk_data (metadata or both) — used for retrieval/SQL
stored_indices = [i for i in range(len(clmns)) if column_roles.get(clmns[i], "both") in ("metadata", "both")]
- if settings.DOC_ENGINE_INFINITY or settings.DOC_ENGINE_OCEANBASE or settings.DOC_ENGINE_SERENEDB:
+ if settings.DOC_ENGINE_INFINITY or settings.DOC_ENGINE_OCEANBASE or settings.DOC_ENGINE_GAUSSDB or settings.DOC_ENGINE_SERENEDB:
field_map = {py_clmns[i].lower(): str(clmns[i]).replace("_", " ") for i in stored_indices}
else:
field_map = {clmns_map[i][0]: clmns_map[i][1] for i in stored_indices}
+ if settings.DOC_ENGINE_GAUSSDB:
+ field_map = {py_clmns[i].lower(): str(clmns[i]) for i in stored_indices}
logging.debug(f"Field map (sheet): {field_map}")
sheet_specs.append(
{
@@ -608,7 +610,7 @@ def chunk(filename, binary=None, from_page=0, to_page=MAXIMUM_TASK_PAGE_NUMBER,
if role in ("indexing", "vectorize", "both"):
text_fields.append((col_name, row[col_name]))
if role in ("metadata", "both"):
- if settings.DOC_ENGINE_INFINITY or settings.DOC_ENGINE_OCEANBASE or settings.DOC_ENGINE_SERENEDB:
+ if settings.DOC_ENGINE_INFINITY or settings.DOC_ENGINE_OCEANBASE or settings.DOC_ENGINE_GAUSSDB or settings.DOC_ENGINE_SERENEDB:
stored[str(col_name)] = row[col_name]
else:
fld = clmns_map[j][0]
@@ -631,7 +633,7 @@ def chunk(filename, binary=None, from_page=0, to_page=MAXIMUM_TASK_PAGE_NUMBER,
stored[f"{py_clmns[j].lower()}_raw"] = raw_s
if not text_fields and not stored:
continue
- if settings.DOC_ENGINE_INFINITY or settings.DOC_ENGINE_OCEANBASE or settings.DOC_ENGINE_SERENEDB:
+ if settings.DOC_ENGINE_INFINITY or settings.DOC_ENGINE_OCEANBASE or settings.DOC_ENGINE_GAUSSDB or settings.DOC_ENGINE_SERENEDB:
if stored:
d["chunk_data"] = stored
else:
@@ -642,7 +644,7 @@ def chunk(filename, binary=None, from_page=0, to_page=MAXIMUM_TASK_PAGE_NUMBER,
logger.debug(f"[TABLE_PARSER_DEBUG] Chunk content_with_weight length: {len(d.get('content_with_weight', '') or '')}")
_cd = d.get("chunk_data")
logger.debug(f"[TABLE_PARSER_DEBUG] Chunk chunk_data keys: {list(_cd.keys()) if isinstance(_cd, dict) else 'N/A'}")
- if not (settings.DOC_ENGINE_INFINITY or settings.DOC_ENGINE_OCEANBASE or settings.DOC_ENGINE_SERENEDB):
+ if not (settings.DOC_ENGINE_INFINITY or settings.DOC_ENGINE_OCEANBASE or settings.DOC_ENGINE_GAUSSDB or settings.DOC_ENGINE_SERENEDB):
_extra = [k for k in d if k not in ("docnm_kwd", "title_tks", "content_with_weight", "content_ltks", "content_sm_ltks")]
logger.debug(f"[TABLE_PARSER_DEBUG] Chunk ES extra field keys (sample): {_extra[:20]}")
res.append(d)
diff --git a/rag/nlp/search.py b/rag/nlp/search.py
index 78b876afe7..fd73e2ea09 100644
--- a/rag/nlp/search.py
+++ b/rag/nlp/search.py
@@ -207,7 +207,11 @@ class Dealer:
if settings.DOC_ENGINE_OCEANBASE or settings.DOC_ENGINE_SERENEDB:
src.append(f"q_{len(q_vec)}_vec")
- fusionExpr = FusionExpr("weighted_sum", topk, {"weights": "0.001,1"})
+ if settings.DOC_ENGINE_GAUSSDB:
+ vector_weight = req.get("vector_similarity_weight", 0.3)
+ fusionExpr = FusionExpr("weighted_sum", topk, {"weights": f"{1 - float(vector_weight)},{float(vector_weight)}"})
+ else:
+ fusionExpr = FusionExpr("weighted_sum", topk, {"weights": "0.001,1"})
matchExprs = [matchText, matchDense, fusionExpr] if matchText else [matchDense]
res = await thread_pool_exec(self.dataStore.search, src, highlightFields, filters, matchExprs, orderBy, offset, limit, idx_names, kb_ids, rank_feature=rank_feature)
@@ -337,20 +341,14 @@ class Dealer:
return res, seted
- def _rank_feature_scores(self, query_rfea, search_res):
- ## For rank feature(tag_fea) scores.
+ def _tag_feature_scores(self, query_rfea, search_res):
rank_fea = []
- pageranks = []
- for chunk_id in search_res.ids:
- pageranks.append(search_res.field[chunk_id].get(PAGERANK_FLD, 0))
- pageranks = np.array(pageranks, dtype=float)
-
if not query_rfea:
- return np.array([0 for _ in range(len(search_res.ids))]) + pageranks
+ return np.zeros(len(search_res.ids), dtype=float)
q_denor = np.sqrt(np.sum([s * s for t, s in query_rfea.items() if t != PAGERANK_FLD]))
if q_denor == 0:
- return np.array([0 for _ in range(len(search_res.ids))]) + pageranks
+ return np.zeros(len(search_res.ids), dtype=float)
for i in search_res.ids:
nor, denor = 0, 0
if not search_res.field[i].get(TAG_FLD):
@@ -368,7 +366,12 @@ class Dealer:
rank_fea.append(0)
else:
rank_fea.append(nor / np.sqrt(denor) / q_denor)
- return np.array(rank_fea) * 10.0 + pageranks
+ return np.array(rank_fea, dtype=float) * 10.0
+
+ def _rank_feature_scores(self, query_rfea, search_res):
+ ## For rank feature(tag_fea) scores.
+ pageranks = np.array([search_res.field[chunk_id].get(PAGERANK_FLD, 0) for chunk_id in search_res.ids], dtype=float)
+ return self._tag_feature_scores(query_rfea, search_res) + pageranks
async def _knn_scores(self, sres: "Dealer.SearchResult", idx_names: str | list[str], kb_ids: list[str]) -> dict[str, float]:
"""
@@ -597,6 +600,7 @@ class Dealer:
"topk": top,
"similarity": similarity_threshold,
"available_int": 1,
+ "vector_similarity_weight": vector_similarity_weight,
}
if isinstance(must_not, dict) and must_not:
req["must_not"] = must_not
@@ -652,6 +656,14 @@ class Dealer:
vector_similarity_weight,
rank_feature=rank_feature,
)
+ elif settings.DOC_ENGINE_GAUSSDB:
+ # GaussDB computes fusion and PageRank in SQL; tag features are
+ # applied locally to the returned candidate window.
+ sql_scores = [sres.field[id].get("_score", 0.0) for id in sres.ids]
+ sql_scores = np.array([s if s is not None else 0.0 for s in sql_scores], dtype=np.float64)
+ sim = sql_scores + self._tag_feature_scores(rank_feature, sres)
+ tsim = sql_scores
+ vsim = sql_scores
else:
# ES path: ask ES for the clean cosine score via a second
# KNN-only call filtered by the candidate ids, then merge it
diff --git a/rag/utils/gaussdb_conn.py b/rag/utils/gaussdb_conn.py
new file mode 100644
index 0000000000..0ce2958dec
--- /dev/null
+++ b/rag/utils/gaussdb_conn.py
@@ -0,0 +1,1431 @@
+#
+# Copyright 2025 The InfiniFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+from __future__ import annotations
+
+import json
+import logging
+import re
+from math import isfinite
+from typing import Any, Iterable
+
+from psycopg2 import Error as PsycopgError, errorcodes
+from pydantic import BaseModel
+
+from common.constants import PAGERANK_FLD
+from common.doc_store.doc_store_base import FusionExpr, MatchDenseExpr, MatchTextExpr
+from common.doc_store.gaussdb_conn_base import (
+ GaussDBConnectionBase,
+ GaussDBSQLValidator,
+ GaussDBSearchBuilder,
+ extra_field_expr,
+ validate_extra_field,
+)
+
+logger = logging.getLogger("ragflow.gaussdb_conn")
+
+SQL_QUERY_TIMEOUT_MS = 30000
+VECTOR_COLUMN_RE = re.compile(r"^q_(?P<dim>\d+)_vec$")
+VECTOR_VALID_COLUMN_RE = re.compile(r"^q_(?P<dim>\d+)_vec_valid$")
+
+
+def _tokenize_query_terms(query_text: Any) -> list[str]:
+ from rag.nlp import rag_tokenizer
+
+ return [term for term in rag_tokenizer.tokenize(str(query_text or "")).split() if term]
+
+
+def _application_highlight(text: Any, keywords: list[str]) -> str | None:
+ if text is None:
+ return None
+ simple_terms, ngram_terms = GaussDBSearchBuilder.split_text_query_terms(keywords)
+ unique_terms: dict[str, str] = {}
+ for term in [*simple_terms, *ngram_terms]:
+ unique_terms.setdefault(term.casefold(), term)
+ terms = sorted(unique_terms.values(), key=len, reverse=True)
+ if not terms:
+ return None
+ pattern = re.compile("|".join(re.escape(term) for term in terms), re.IGNORECASE)
+ highlighted, count = pattern.subn(lambda match: f"<em>{match.group(0)}</em>", str(text))
+ return highlighted if count else None
+
+
+class GaussDBError(Exception):
+ pass
+
+
+CHUNK_COLUMNS = (
+ "id",
+ "kb_id",
+ "doc_id",
+ "docnm_kwd",
+ "doc_type_kwd",
+ "title_tks",
+ "title_sm_tks",
+ "content_with_weight",
+ "content_ltks",
+ "content_sm_ltks",
+ "important_kwd",
+ "important_tks",
+ "question_kwd",
+ "question_tks",
+ "tag_kwd",
+ "tag_feas",
+ "available_int",
+ "pagerank_fea",
+ "create_time",
+ "create_timestamp_flt",
+ "img_id",
+ "position_int",
+ "page_num_int",
+ "top_int",
+ "metadata",
+ "chunk_data",
+ "extra",
+ "_order_id",
+ "group_id",
+ "mom_id",
+ "knowledge_graph_kwd",
+ "source_id",
+ "entity_kwd",
+ "entity_type_kwd",
+ "from_entity_kwd",
+ "to_entity_kwd",
+ "weight_int",
+ "weight_flt",
+ "entities_kwd",
+ "rank_flt",
+ "n_hop_with_weight",
+ "removed_kwd",
+ "raptor_kwd",
+ "raptor_layer_int",
+)
+CHUNK_COLUMN_SET = set(CHUNK_COLUMNS)
+JSONB_COLUMNS = {
+ "important_kwd",
+ "question_kwd",
+ "tag_kwd",
+ "tag_feas",
+ "position_int",
+ "page_num_int",
+ "top_int",
+ "metadata",
+ "chunk_data",
+ "extra",
+ "source_id",
+ "entities_kwd",
+}
+JSONB_MULTI_VALUE_COLUMNS = {
+ "important_kwd",
+ "question_kwd",
+ "tag_kwd",
+ "source_id",
+ "entities_kwd",
+}
+KEY_COLUMNS = {"id", "kb_id"}
+DOC_META_COLUMNS = ("id", "kb_id", "meta_fields")
+DOC_META_COLUMN_SET = set(DOC_META_COLUMNS)
+DOC_META_JSONB_COLUMNS = {"meta_fields"}
+DEFAULT_VALUES = {
+ "available_int": 1,
+ "removed_kwd": "N",
+}
+_UNDEFINED_TABLE_SQLSTATE = "42P01"
+
+
+def _is_undefined_table_error(exc: Exception) -> bool:
+ return getattr(exc, "pgcode", None) == _UNDEFINED_TABLE_SQLSTATE or getattr(exc, "sqlstate", None) == _UNDEFINED_TABLE_SQLSTATE
+
+
+def _infer_chunk_vector_size(documents: list[dict]) -> int:
+ dimensions = {int(match.group("dim")) for document in documents for column in document if (match := VECTOR_COLUMN_RE.fullmatch(str(column)))}
+ if len(dimensions) != 1:
+ raise ValueError("cannot infer a single GaussDB vector dimension from chunk documents")
+ return dimensions.pop()
+
+
+class SearchResult(BaseModel):
+ total: int
+ chunks: list[dict]
+
+
+class GaussDBConnection(GaussDBConnectionBase):
+ def db_type(self) -> str:
+ return "gaussdb"
+
+ def create_idx(self, index_name: str, dataset_id: str, vector_size: int, parser_id: str = None):
+ # Keep the functional table even when the optional gsdiskann build
+ # fails, while still propagating the index error to the caller.
+ statements = [
+ self.ddl.build_advisory_lock_sql(f"create_idx:{self.schema}:{index_name}"),
+ self.ddl.build_chunk_table_ddl(index_name),
+ ]
+ statements.extend(self.ddl.build_regular_index_ddls(index_name))
+ statements.append(self.ddl.build_fulltext_ugin_ddl(index_name))
+ statements.append(self.ddl.build_ngram_fulltext_ugin_ddl(index_name))
+ statements.extend(self.ddl.build_vector_column_ddls(index_name, vector_size))
+ self._execute_statements(statements)
+ self._create_diskann_index(index_name, vector_size)
+ return True
+
+ def _create_diskann_index(self, index_name: str, vector_size: int) -> None:
+ statements = [
+ self.ddl.build_advisory_lock_sql(f"create_vector_idx:{self.schema}:{index_name}:{vector_size}"),
+ "SET LOCAL maintenance_work_mem = '1GB'",
+ self.ddl.build_diskann_index_ddl(index_name, vector_size),
+ ]
+ self._execute_statements(statements)
+
+ def create_doc_meta_idx(self, index_name: str):
+ if not is_doc_meta_table(index_name):
+ raise ValueError(f"invalid GaussDB document metadata table name: {index_name}")
+ statements = [
+ self.ddl.build_advisory_lock_sql(f"create_doc_meta_idx:{self.schema}:{index_name}"),
+ *self.ddl.build_doc_meta_table_ddls(index_name),
+ ]
+ self._execute_statements(statements)
+ return True
+
+ def delete_idx(self, index_name: str, dataset_id: str | None):
+ table = self.ddl.qualified_name(index_name)
+ if dataset_id:
+ try:
+ self._execute_write(f"DELETE FROM {table} WHERE kb_id = %s", [dataset_id])
+ except PsycopgError as exc:
+ if exc.pgcode != errorcodes.UNDEFINED_TABLE:
+ raise
+ else:
+ self._execute_write(f"DROP TABLE IF EXISTS {table}", [])
+
+ def index_exist(self, index_name: str, dataset_id: str | None = None) -> bool:
+ self.ddl.validate_identifier(index_name)
+ row = self._fetch_one(
+ """
+ SELECT 1
+ FROM information_schema.tables
+ WHERE table_schema = %s
+ AND table_name = %s
+ LIMIT 1
+ """,
+ [self.schema, index_name],
+ )
+ return bool(row)
+
+ def insert(self, documents: list[dict], index_name: str, knowledgebase_id: str = None, refresh: str | bool = "wait_for") -> list[str]:
+ if not documents:
+ return []
+ document_ids = [str(document.get("id") or "") for document in documents]
+ try:
+ is_meta = is_doc_meta_table(index_name)
+ if is_meta:
+ sql, params = self._build_doc_meta_upsert(index_name, documents, knowledgebase_id)
+ else:
+ normalized, errors = self._normalize_chunk_rows(index_name, documents, knowledgebase_id)
+ if errors:
+ return [document_id for document_id in document_ids if document_id] or errors
+ sql, params = self._build_chunk_upsert(index_name, normalized)
+ try:
+ self._execute_write(sql, params, many=True)
+ except Exception as exc:
+ if is_meta or not _is_undefined_table_error(exc) or self.index_exist(index_name, knowledgebase_id):
+ raise
+ vector_size = _infer_chunk_vector_size(documents)
+ logger.info(
+ "GaussDB chunk table missing during insert; creating table=%s vector_size=%s",
+ index_name,
+ vector_size,
+ )
+ self.create_idx(index_name, knowledgebase_id, vector_size)
+ self._execute_write(sql, params, many=True)
+ return []
+ except Exception as exc:
+ logger.error("GaussDB insert failed for table=%s ids=%s error=%s", index_name, document_ids, exc)
+ return [document_id for document_id in document_ids if document_id] or [str(exc)]
+
+ def get(self, chunk_id: str, index_name: str, knowledgebase_ids: list[str]) -> dict | None:
+ if not chunk_id:
+ return None
+ is_meta = is_doc_meta_table(index_name)
+ kb_ids = normalize_kb_ids(knowledgebase_ids)
+ if not is_meta and not kb_ids:
+ return None
+ if not self.index_exist(index_name):
+ return None
+ table = self.ddl.qualified_name(index_name)
+ params = [chunk_id]
+ where = "id = %s"
+ if kb_ids:
+ placeholders = ", ".join(["%s"] * len(kb_ids))
+ where += f" AND kb_id IN ({placeholders})"
+ params.extend(kb_ids)
+ if not is_meta and len(kb_ids) > 1:
+ sql = f"SELECT * FROM {table} WHERE {where} ORDER BY kb_id ASC LIMIT 2"
+ rows, description = self._fetch_all_with_description(sql, params)
+ chunks = [self._row_to_chunk(row, description) for row in rows or []]
+ self._check_cross_kb_duplicate_chunk_ids(chunks)
+ return chunks[0] if chunks else None
+ sql = f"SELECT * FROM {table} WHERE {where} LIMIT 1"
+ row, description = self._fetch_one_with_description(sql, params)
+ if row is None:
+ return None
+ return self._row_to_chunk(row, description)
+
+ def search(
+ self,
+ select_fields: list[str],
+ highlight_fields: list[str],
+ condition: dict,
+ match_expressions: list,
+ order_by,
+ offset: int,
+ limit: int,
+ index_names: str | list[str],
+ knowledgebase_ids: list[str] | None = None,
+ agg_fields: list[str] | None = None,
+ rank_feature: dict | None = None,
+ **kwargs,
+ ) -> SearchResult:
+ if knowledgebase_ids is None:
+ knowledgebase_ids = kwargs.get("dataset_ids") or []
+ tables = normalize_table_names(index_names)
+ if not tables:
+ return SearchResult(total=0, chunks=[])
+ if len(tables) == 1 and is_doc_meta_table(tables[0]) and not match_expressions and not agg_fields:
+ return self._search_doc_meta_table(
+ select_fields=select_fields,
+ condition=condition,
+ order_by=order_by,
+ offset=offset,
+ limit=limit,
+ index_name=tables[0],
+ knowledgebase_ids=knowledgebase_ids,
+ )
+ if any(is_doc_meta_table(table) for table in tables):
+ raise ValueError("document metadata tables cannot be mixed with chunk search")
+
+ if agg_fields:
+ return self._search_chunk_aggregation(
+ tables=tables,
+ condition=condition,
+ knowledgebase_ids=knowledgebase_ids,
+ agg_fields=agg_fields,
+ )
+
+ parsed = self._parse_match_expressions(match_expressions, rank_feature)
+ scoped_condition = self._scoped_search_condition(condition, knowledgebase_ids)
+ if len(tables) == 1:
+ return self._search_chunk_table(
+ table=tables[0],
+ select_fields=select_fields,
+ highlight_fields=highlight_fields,
+ condition=scoped_condition,
+ parsed=parsed,
+ order_by=order_by,
+ offset=offset,
+ limit=limit,
+ )
+
+ effective_offset = max(int(offset or 0), 0)
+ effective_limit = max(int(limit or 0), 0)
+ if effective_limit == 0:
+ effective_limit = 10000
+ collection_limit = effective_offset + effective_limit
+ total = 0
+ chunks: list[dict] = []
+ for table in tables:
+ result = self._search_chunk_table(
+ table=table,
+ select_fields=select_fields,
+ highlight_fields=highlight_fields,
+ condition=scoped_condition,
+ parsed=parsed,
+ order_by=order_by,
+ offset=0,
+ limit=collection_limit,
+ )
+ total += result.total
+ chunks.extend(result.chunks)
+ chunks = self._sort_search_chunks(chunks, order_by, has_match=bool(parsed["keywords"] or parsed["vector"]))
+ chunks = chunks[effective_offset : effective_offset + effective_limit]
+ return SearchResult(total=total, chunks=chunks)
+
+ def update(self, condition: dict, new_value: dict, index_name: str, knowledgebase_id: str) -> bool:
+ if not condition or not new_value:
+ return False
+ if not (normalize_kb_id(knowledgebase_id) or normalize_kb_id(condition.get("kb_id"))):
+ return False
+ try:
+ table = self.ddl.qualified_name(index_name)
+ is_meta = is_doc_meta_table(index_name)
+ dynamic_values = {validate_extra_field(key): value for key, value in new_value.items() if key not in {"add", "remove"} and is_dynamic_chunk_field(key, is_meta)}
+ if dynamic_values:
+ fixed_values = {key: value for key, value in new_value.items() if key not in dynamic_values}
+ return self._update_with_dynamic_values(
+ table=table,
+ condition=condition,
+ knowledgebase_id=knowledgebase_id,
+ fixed_values=fixed_values,
+ dynamic_values=dynamic_values,
+ )
+ set_sql, set_params = self._build_set_clause(new_value, is_meta=is_meta, condition=condition)
+ where_sql, where_params = self._build_where_clause(condition, knowledgebase_id, is_meta=is_meta)
+ sql = f"UPDATE {table} SET {set_sql} WHERE {where_sql}"
+ self._execute_write(sql, [*set_params, *where_params])
+ return True
+ except Exception as exc:
+ logger.error("GaussDB update failed for table=%s condition=%s error=%s", index_name, condition, exc)
+ return False
+
+ def _update_with_dynamic_values(
+ self,
+ *,
+ table: str,
+ condition: dict,
+ knowledgebase_id: str,
+ fixed_values: dict,
+ dynamic_values: dict[str, Any],
+ ) -> bool:
+ supplied_extra = fixed_values.pop("extra", None)
+ set_sql, set_params = self._build_set_clause(
+ fixed_values,
+ is_meta=False,
+ condition=condition,
+ )
+ where_sql, where_params = self._build_where_clause(
+ condition,
+ knowledgebase_id,
+ is_meta=False,
+ )
+ if not where_sql:
+ return False
+
+ conn = self.pool.get_conn()
+ cur = None
+ try:
+ cur = conn.cursor()
+ cur.execute(
+ f"SELECT id, kb_id, extra FROM {table} WHERE {where_sql} FOR UPDATE",
+ where_params,
+ )
+ rows = cur.fetchall()
+ for row in rows or []:
+ row_id, kb_id, stored_extra = row
+ base_extra = supplied_extra if supplied_extra is not None else stored_extra
+ merged_extra = dict(parse_json_dict(base_extra))
+ merged_extra.update(dynamic_values)
+ assignments = [part for part in (set_sql, "extra = %s::jsonb") if part]
+ cur.execute(
+ f"UPDATE {table} SET {', '.join(assignments)} WHERE kb_id = %s AND id = %s",
+ [
+ *set_params,
+ json.dumps(merged_extra, ensure_ascii=False),
+ kb_id,
+ row_id,
+ ],
+ )
+ conn.commit()
+ return True
+ except Exception:
+ conn.rollback()
+ raise
+ finally:
+ close_cursor(cur)
+ self.pool.put_conn(conn)
+
+ def delete(self, condition: dict, index_name: str, knowledgebase_id: str) -> int:
+ if not condition:
+ return 0
+ if not (normalize_kb_id(knowledgebase_id) or normalize_kb_id(condition.get("kb_id"))):
+ return 0
+ try:
+ table = self.ddl.qualified_name(index_name)
+ where_sql, where_params = self._build_where_clause(
+ condition,
+ knowledgebase_id,
+ is_meta=is_doc_meta_table(index_name),
+ )
+ return self._execute_write(f"DELETE FROM {table} WHERE {where_sql}", where_params)
+ except Exception as exc:
+ logger.error("GaussDB delete failed for table=%s condition=%s error=%s", index_name, condition, exc)
+ return 0
+
+ def fetch_metadata_doc_ids(
+ self,
+ index_name: str,
+ kb_ids: list[str],
+ sql_filter: str,
+ filter_params: list[Any],
+ limit: int,
+ ) -> list[str]:
+ if not is_doc_meta_table(index_name):
+ raise ValueError(f"invalid GaussDB document metadata table name: {index_name}")
+ scoped_kb_ids = normalize_kb_ids(kb_ids)
+ if not scoped_kb_ids or not sql_filter:
+ return []
+ table = self.ddl.qualified_name(index_name)
+ placeholders = ", ".join(["%s"] * len(scoped_kb_ids))
+ effective_limit = limit if limit and limit > 0 else 10000
+ sql = f"SELECT id FROM {table} WHERE kb_id IN ({placeholders}) AND ({sql_filter}) ORDER BY id LIMIT %s"
+ rows = self._fetch_all(sql, [*scoped_kb_ids, *(filter_params or []), effective_limit])
+ doc_ids = []
+ for row in rows or []:
+ if isinstance(row, dict):
+ value = row.get("id")
+ elif isinstance(row, (list, tuple)) and row:
+ value = row[0]
+ else:
+ value = None
+ if value is not None:
+ doc_ids.append(str(value))
+ return doc_ids
+
+ def sql(self, sql: str, fetch_size: int = 128, format: str = "json"):
+ self.logger.debug("GaussDBConnection.sql get sql: %s", sql)
+ fetch_size = int(fetch_size or 128)
+ validated = GaussDBSQLValidator.readonly_guard(
+ default_limit=fetch_size,
+ execution_schema=self.schema,
+ ).validate_and_patch(sql)
+ rows, description = self._fetch_all_with_description(
+ validated.sql,
+ [],
+ statement_timeout_ms=SQL_QUERY_TIMEOUT_MS,
+ )
+ columns = [desc[0] for desc in description]
+
+ def coerce_value(value):
+ if isinstance(value, bytes):
+ return value.decode("utf-8", errors="ignore")
+ if isinstance(value, (dict, list)):
+ return json.dumps(value, ensure_ascii=False)
+ return value
+
+ rows_list = [[coerce_value(value) for value in list(row)] for row in rows or []]
+ result = {
+ "columns": [{"name": column, "type": "text"} for column in columns],
+ "rows": rows_list,
+ }
+ if format == "markdown":
+ header = "|" + "|".join(columns) + "|" if columns else ""
+ separator = "|" + "|".join(["---" for _ in columns]) + "|" if columns else ""
+ body = "\n".join(["|" + "|".join([str(value) for value in row]) + "|" for row in rows_list])
+ result["markdown"] = "\n".join([line for line in [header, separator, body] if line])
+ return result
+
+ def get_total(self, res) -> int:
+ return int(res.total)
+
+ def get_doc_ids(self, res) -> list[str]:
+ return [row["id"] for row in res.chunks if "id" in row]
+
+ @staticmethod
+ def _check_cross_kb_duplicate_chunk_ids(rows) -> None:
+ seen_kb_ids = {}
+ for row in rows or []:
+ chunk_id = row.get("id")
+ if chunk_id is None:
+ continue
+ kb_id = row.get("kb_id")
+ if kb_id is None:
+ continue
+ previous_kb_id = seen_kb_ids.get(chunk_id)
+ if previous_kb_id is not None and previous_kb_id != kb_id:
+ raise GaussDBError(f"cross-KB duplicate chunk id: {chunk_id}")
+ seen_kb_ids[chunk_id] = kb_id
+
+ def get_fields(self, res, fields: list[str]) -> dict[str, dict]:
+ self._check_cross_kb_duplicate_chunk_ids(res.chunks)
+ result = {}
+ for row in res.chunks:
+ chunk_id = row.get("id")
+ if chunk_id is None:
+ continue
+ result[chunk_id] = {field: row[field] for field in fields if row.get(field) is not None}
+ return result
+
+ def get_highlight(self, res, keywords: list[str], field_name: str):
+ self._check_cross_kb_duplicate_chunk_ids(res.chunks)
+ highlights = {}
+ for row in res.chunks:
+ chunk_id = row.get("id")
+ if row.get("_highlight_source") is not None:
+ value = _application_highlight(row["_highlight_source"], keywords)
+ else:
+ value = row.get("_highlight") or row.get("highlight")
+ if chunk_id and value:
+ highlights[chunk_id] = value
+ return highlights
+
+ def get_aggregation(self, res, field_name: str):
+ counts = {}
+ result = []
+ for row in res.chunks:
+ if "value" in row and "count" in row:
+ result.append((row["value"], row["count"]))
+ continue
+ value = row.get(field_name)
+ if isinstance(value, list):
+ for item in value:
+ if isinstance(item, str) and item.strip():
+ counts[item] = counts.get(item, 0) + 1
+ elif isinstance(value, str) and value.strip():
+ counts[value] = counts.get(value, 0) + 1
+ return result or list(counts.items())
+
+ def get_scores(self, res) -> dict[str, float]:
+ self._check_cross_kb_duplicate_chunk_ids(res.chunks)
+ return {row["id"]: float(row.get("_score") or 0.0) for row in res.chunks if row.get("id") is not None}
+
+ def get_vector_dimensions(self, index_name: str) -> list[int]:
+ self.ddl.validate_identifier(index_name)
+ sql = """
+ SELECT column_name
+ FROM information_schema.columns
+ WHERE table_schema = %s AND table_name = %s
+ """
+ rows = self._fetch_all(sql, [self.schema, index_name])
+ dims = []
+ for row in rows:
+ col = row[0] if isinstance(row, (list, tuple)) else row.get("column_name")
+ match = VECTOR_COLUMN_RE.fullmatch(str(col or ""))
+ if match:
+ dims.append(int(match.group("dim")))
+ return sorted(set(dims))
+
+ def adjust_chunk_pagerank_fea(
+ self,
+ chunk_id: str,
+ index_name: str,
+ dataset_id: str,
+ delta: int,
+ min_w: int = 0,
+ max_w: int = 100,
+ **kwargs,
+ ) -> bool:
+ if not chunk_id or not dataset_id:
+ return False
+ table = self.ddl.qualified_name(index_name)
+ sql = f"UPDATE {table} SET pagerank_fea = GREATEST(%s, LEAST(%s, COALESCE(pagerank_fea, 0) + %s)) WHERE kb_id = %s AND id = %s"
+ return self._execute_write(sql, [min_w, max_w, delta, dataset_id, chunk_id]) > 0
+
+ def _normalize_chunk_rows(self, index_name: str, documents: list[dict], knowledgebase_id: str | None):
+ errors = []
+ rows = []
+ batch_dims = sorted({int(VECTOR_COLUMN_RE.fullmatch(key).group("dim")) for document in documents for key in document if VECTOR_COLUMN_RE.fullmatch(key)})
+ for document in documents:
+ doc_id = str(document.get("id") or "")
+ try:
+ rows.append(self._normalize_chunk_row(index_name, document, knowledgebase_id, batch_dims))
+ except Exception as exc:
+ logger.error("GaussDB normalize chunk failed id=%s error=%s", doc_id, exc)
+ errors.append(doc_id or str(exc))
+ return rows, errors
+
+ def _normalize_chunk_row(self, index_name: str, document: dict, knowledgebase_id: str | None, batch_dims: list[int]) -> dict:
+ chunk_id = document.get("id")
+ if not chunk_id:
+ raise ValueError("chunk id is required")
+ row = {"id": str(chunk_id)}
+
+ kb_id = normalize_kb_id(document.get("kb_id")) or knowledgebase_id
+ if not kb_id:
+ raise ValueError("kb_id is required")
+ if knowledgebase_id and kb_id != knowledgebase_id:
+ raise ValueError(f"kb_id {kb_id} does not match dataset_id {knowledgebase_id}")
+ row["kb_id"] = kb_id
+
+ extra = {}
+ for key, value in document.items():
+ if key in {"id", "kb_id"} or VECTOR_COLUMN_RE.fullmatch(key) or VECTOR_VALID_COLUMN_RE.fullmatch(key):
+ continue
+ if key not in CHUNK_COLUMN_SET:
+ extra[key] = value
+ continue
+ row[key] = normalize_column_value(key, value)
+
+ if extra:
+ existing_extra = row.get("extra")
+ if isinstance(existing_extra, str):
+ try:
+ existing_extra = json.loads(existing_extra)
+ except json.JSONDecodeError:
+ existing_extra = {}
+ if not isinstance(existing_extra, dict):
+ existing_extra = {}
+ existing_extra.update(extra)
+ row["extra"] = json.dumps(existing_extra, ensure_ascii=False)
+
+ metadata = row.get("metadata")
+ metadata_dict = parse_json_dict(metadata)
+ if metadata_dict:
+ if metadata_dict.get("_group_id"):
+ row["group_id"] = metadata_dict["_group_id"]
+ elif row.get("doc_id"):
+ row.setdefault("group_id", row.get("doc_id"))
+ if metadata_dict.get("_title"):
+ row["docnm_kwd"] = metadata_dict["_title"]
+ elif row.get("doc_id"):
+ row.setdefault("group_id", row.get("doc_id"))
+
+ vector_columns = [key for key in document if VECTOR_COLUMN_RE.fullmatch(key)]
+ if vector_columns:
+ for vector_col in vector_columns:
+ dim = int(VECTOR_COLUMN_RE.fullmatch(vector_col).group("dim"))
+ row[vector_col] = vector_literal(document[vector_col], dim)
+ row[self.ddl.vector_valid_column_name(dim)] = True
+ else:
+ dims = batch_dims or self.get_vector_dimensions(index_name)
+ if len(dims) != 1:
+ raise ValueError("cannot infer GaussDB vector dimension")
+ dim = dims[0]
+ row[self.ddl.vector_column_name(dim)] = zero_vector_literal(dim)
+ row[self.ddl.vector_valid_column_name(dim)] = False
+
+ for column, default in DEFAULT_VALUES.items():
+ row.setdefault(column, default)
+ return row
+
+ def _build_chunk_upsert(self, index_name: str, rows: list[dict]) -> tuple[str, list[list[Any]]]:
+ if not rows:
+ raise ValueError("rows are required")
+ columns = ordered_columns(rows)
+ table = self.ddl.qualified_name(index_name)
+ placeholders = ", ".join(self._placeholder(column) for column in columns)
+ update_columns = [column for column in columns if column not in {"id", "kb_id"}]
+ update_clause = ", ".join(f"{column} = VALUES({column})" for column in update_columns)
+ sql = f"INSERT INTO {table} ({', '.join(columns)}) VALUES ({placeholders}) ON DUPLICATE KEY UPDATE {update_clause}"
+ params = [[row.get(column) for column in columns] for row in rows]
+ return sql, params
+
+ def _build_doc_meta_upsert(
+ self,
+ index_name: str,
+ documents: list[dict],
+ knowledgebase_id: str | None,
+ ) -> tuple[str, list[list[Any]]]:
+ table = self.ddl.qualified_name(index_name)
+ params = []
+ for document in documents:
+ doc_id = document.get("id")
+ kb_id = normalize_kb_id(document.get("kb_id")) or knowledgebase_id
+ if not doc_id or not kb_id:
+ raise ValueError("doc metadata id and kb_id are required")
+ if knowledgebase_id and kb_id != knowledgebase_id:
+ raise ValueError(f"kb_id {kb_id} does not match dataset_id {knowledgebase_id}")
+ params.append([str(doc_id), kb_id, json.dumps(document.get("meta_fields") or {}, ensure_ascii=False)])
+ sql = f"INSERT INTO {table} (id, kb_id, meta_fields) VALUES (%s, %s, %s::jsonb) ON DUPLICATE KEY UPDATE meta_fields = VALUES(meta_fields)"
+ return sql, params
+
+ def _search_doc_meta_table(
+ self,
+ select_fields: list[str],
+ condition: dict,
+ order_by,
+ offset: int,
+ limit: int,
+ index_name: str,
+ knowledgebase_ids: list[str],
+ ) -> SearchResult:
+ effective_condition = dict(condition or {})
+ kb_ids = normalize_kb_ids(knowledgebase_ids)
+ condition_kb_ids = normalize_kb_ids(effective_condition.get("kb_id"))
+ if kb_ids:
+ if condition_kb_ids and not set(condition_kb_ids).issubset(set(kb_ids)):
+ raise ValueError("condition kb_id must stay within knowledgebase_ids")
+ if not condition_kb_ids:
+ effective_condition["kb_id"] = kb_ids
+ if not normalize_kb_ids(effective_condition.get("kb_id")):
+ raise ValueError("GaussDB document metadata search requires a kb_id boundary")
+
+ table = self.ddl.qualified_name(index_name)
+ columns = select_doc_meta_columns(select_fields)
+ where_sql, where_params = self._build_where_clause(effective_condition, None, is_meta=True)
+ sql = f"SELECT {', '.join(columns)}, COUNT(*) OVER() AS __total FROM {table}"
+ if where_sql:
+ sql += f" WHERE {where_sql}"
+ order_sql = build_doc_meta_order_by(order_by)
+ if order_sql:
+ sql += f" ORDER BY {order_sql}"
+ effective_limit = limit if limit and limit > 0 else 10000
+ effective_offset = max(int(offset or 0), 0)
+ sql += " LIMIT %s OFFSET %s"
+ rows, description = self._fetch_all_with_description(sql, [*where_params, effective_limit, effective_offset])
+ total = 0
+ chunks = []
+ for row in rows or []:
+ chunk = self._row_to_chunk(row, description)
+ total = int(chunk.pop("__total", total or 0) or 0)
+ chunks.append(chunk)
+ if not chunks and effective_offset:
+ count_sql = f"SELECT COUNT(*) FROM {table}"
+ if where_sql:
+ count_sql += f" WHERE {where_sql}"
+ row = self._fetch_one(count_sql, where_params)
+ total = int(row[0]) if row else 0
+ return SearchResult(total=total, chunks=chunks)
+
+ def _search_chunk_aggregation(
+ self,
+ tables: list[str],
+ condition: dict,
+ knowledgebase_ids: list[str] | None,
+ agg_fields: list[str],
+ ) -> SearchResult:
+ if len(agg_fields) != 1:
+ raise ValueError("GaussDB search supports one aggregation field per request")
+ counts: dict[Any, int] = {}
+ scoped_condition = self._scoped_search_condition(condition, knowledgebase_ids)
+ for table in tables:
+ sql, params = self._search_builder().build_aggregation_sql(
+ table=table,
+ field_name=agg_fields[0],
+ condition=scoped_condition,
+ )
+ rows, description = self._fetch_all_with_description(sql, params)
+ for row in rows or []:
+ chunk = self._row_to_chunk(row, description)
+ if chunk.get("count") is not None:
+ chunk["count"] = int(chunk["count"])
+ value = chunk.get("value")
+ if value is not None:
+ counts[value] = counts.get(value, 0) + int(chunk.get("count") or 0)
+ chunks = [{"value": value, "count": count} for value, count in sorted(counts.items(), key=lambda item: (-item[1], str(item[0])))][:1000]
+ return SearchResult(total=len(chunks), chunks=chunks)
+
+ def _search_chunk_table(
+ self,
+ table: str,
+ select_fields: list[str],
+ highlight_fields: list[str],
+ condition: dict,
+ parsed: dict[str, Any],
+ order_by,
+ offset: int,
+ limit: int,
+ ) -> SearchResult:
+ sql, params = self._search_builder().build_search_sql(
+ table=table,
+ select_fields=select_fields,
+ condition=condition,
+ keywords=parsed["keywords"],
+ vector=parsed["vector"],
+ vector_dim=parsed["vector_dim"],
+ vector_weight=parsed["vector_weight"],
+ similarity_threshold=parsed["similarity_threshold"],
+ topn=parsed["topn"],
+ offset=offset,
+ limit=limit,
+ highlight_fields=highlight_fields,
+ order_by=order_by,
+ pagerank_weight=parsed["pagerank_weight"],
+ )
+ rows, description = self._fetch_all_with_description(sql, params)
+ result = self._rows_to_search_result(rows, description)
+ if result.chunks or not max(int(offset or 0), 0):
+ return result
+
+ count_sql, count_params = self._search_builder().build_search_sql(
+ table=table,
+ select_fields=["id"],
+ condition=condition,
+ keywords=parsed["keywords"],
+ vector=parsed["vector"],
+ vector_dim=parsed["vector_dim"],
+ vector_weight=parsed["vector_weight"],
+ similarity_threshold=parsed["similarity_threshold"],
+ topn=parsed["topn"],
+ offset=0,
+ limit=1,
+ highlight_fields=[],
+ order_by=order_by,
+ pagerank_weight=parsed["pagerank_weight"],
+ )
+ count_rows, count_description = self._fetch_all_with_description(count_sql, count_params)
+ count_result = self._rows_to_search_result(count_rows, count_description)
+ return SearchResult(total=count_result.total, chunks=[])
+
+ def _rows_to_search_result(self, rows, description) -> SearchResult:
+ total = 0
+ chunks = []
+ for row in rows or []:
+ chunk = self._row_to_chunk(row, description)
+ total = int(chunk.pop("__total", total or 0) or 0)
+ chunks.append(chunk)
+ return SearchResult(total=total, chunks=chunks)
+
+ def _sort_search_chunks(self, chunks: list[dict], order_by, has_match: bool) -> list[dict]:
+ fields = getattr(order_by, "fields", None) or []
+ if not fields:
+ if has_match:
+ return sorted(
+ chunks,
+ key=lambda row: (
+ -(float(row.get("_score") or 0.0)),
+ str(row.get("kb_id") or ""),
+ str(row.get("id") or ""),
+ ),
+ )
+ return sorted(chunks, key=lambda row: (str(row.get("kb_id") or ""), str(row.get("id") or "")))
+ sorted_chunks = list(chunks)
+ for field, direction in reversed(fields):
+ sorted_chunks.sort(
+ key=lambda row, field=field: sortable_search_value(row.get(field), field),
+ reverse=bool(direction),
+ )
+ return sorted_chunks
+
+ def _parse_match_expressions(
+ self,
+ match_expressions: list,
+ rank_feature: dict | None,
+ ) -> dict[str, Any]:
+ keywords: list[str] = []
+ vector = None
+ vector_dim = None
+ topn = None
+ vector_weight = None
+ similarity_threshold = None
+ pagerank_weight = 10.0
+
+ for expr in match_expressions or []:
+ if isinstance(expr, MatchTextExpr):
+ query_text = (expr.extra_options or {}).get("original_query") or expr.matching_text or ""
+ keywords = _tokenize_query_terms(query_text)
+ topn = expr.topn if topn is None else min(topn, expr.topn)
+ elif isinstance(expr, MatchDenseExpr):
+ if expr.embedding_data_type != "float":
+ raise ValueError(f"unsupported GaussDB vector data type: {expr.embedding_data_type}")
+ vector = list(expr.embedding_data)
+ vector_dim = parse_vector_dim(expr.vector_column_name) or len(vector)
+ topn = expr.topn if topn is None else min(topn, expr.topn)
+ similarity_threshold = float((expr.extra_options or {}).get("similarity", 0.0))
+ elif isinstance(expr, FusionExpr):
+ vector_weight = parse_fusion_vector_weight(expr)
+ topn = expr.topn if topn is None else min(topn, expr.topn)
+
+ if rank_feature and rank_feature.get(PAGERANK_FLD) is not None:
+ pagerank_weight = float(rank_feature[PAGERANK_FLD])
+
+ if vector is None:
+ vector_weight = 0.0
+ elif not keywords:
+ vector_weight = 1.0 if vector_weight is None else float(vector_weight)
+ else:
+ vector_weight = 0.5 if vector_weight is None else float(vector_weight)
+
+ return {
+ "keywords": keywords,
+ "vector": vector,
+ "vector_dim": vector_dim,
+ "vector_weight": vector_weight,
+ "similarity_threshold": similarity_threshold,
+ "topn": topn,
+ "pagerank_weight": pagerank_weight,
+ }
+
+ def _scoped_search_condition(self, condition: dict | None, knowledgebase_ids: list[str] | None) -> dict:
+ effective = dict(condition or {})
+ kb_ids = normalize_kb_ids(knowledgebase_ids)
+ if kb_ids:
+ condition_kb_ids = normalize_kb_ids(effective.get("kb_id"))
+ if condition_kb_ids:
+ if not set(condition_kb_ids).issubset(set(kb_ids)):
+ raise ValueError("condition kb_id must stay within knowledgebase_ids")
+ else:
+ effective["kb_id"] = kb_ids
+ if "doc_ids" in effective and "doc_id" not in effective:
+ effective["doc_id"] = effective.pop("doc_ids")
+ if isinstance(effective.get("doc_id"), (list, tuple, set)) and not effective["doc_id"]:
+ effective.pop("doc_id")
+ if not normalize_kb_ids(effective.get("kb_id")):
+ raise ValueError("GaussDB chunk search requires a kb_id boundary")
+ return effective
+
+ def _search_builder(self) -> GaussDBSearchBuilder:
+ builder = getattr(self, "search_builder", None)
+ if builder is None:
+ builder = GaussDBSearchBuilder(schema=self.schema)
+ self.search_builder = builder
+ return builder
+
+ def _build_set_clause(self, new_value: dict, is_meta: bool, condition: dict) -> tuple[str, list[Any]]:
+ fragments = []
+ params = []
+ allowed_columns = DOC_META_COLUMN_SET if is_meta else CHUNK_COLUMN_SET
+ jsonb_columns = DOC_META_JSONB_COLUMNS if is_meta else JSONB_COLUMNS
+ jsonb_operations: dict[str, dict[str, Any]] = {}
+ updated_vector_dims = {int(match.group("dim")) for key in new_value if not is_meta and (match := VECTOR_COLUMN_RE.fullmatch(str(key)))}
+ for operation, value in new_value.items():
+ if operation in {"remove", "add"} and isinstance(value, dict):
+ for column, item in value.items():
+ jsonb_operations.setdefault(column, {})[operation] = item
+ emitted_jsonb_columns = set()
+
+ def append_jsonb_update(column: str) -> None:
+ if column in emitted_jsonb_columns:
+ return
+ emitted_jsonb_columns.add(column)
+ operations = jsonb_operations[column]
+ remove_expr = None
+ remove_param = None
+ if "remove" in operations:
+ escaped_item = str(operations["remove"]).replace("\\", "\\\\").replace("%", "\\%").replace("_", "\\_")
+ remove_expr = f"json_remove({column}, json_unquote(json_search({column}, 'one', jsonb_array_element_text(%s::jsonb, 0))))::jsonb"
+ remove_param = json.dumps([escaped_item], ensure_ascii=False)
+ if "add" in operations:
+ base_expr = f"COALESCE({remove_expr}, {column}, '[]'::jsonb)" if remove_expr is not None else f"COALESCE({column}, '[]'::jsonb)"
+ expression = (
+ f"(SELECT CASE WHEN candidate @> %s::jsonb THEN candidate ELSE json_array_append(candidate, '$', %s::jsonb)::jsonb END FROM (SELECT {base_expr} AS candidate) AS jsonb_update)"
+ )
+ params.extend(
+ [
+ json.dumps([operations["add"]], ensure_ascii=False),
+ json.dumps(operations["add"], ensure_ascii=False),
+ ]
+ )
+ if remove_param is not None:
+ params.append(remove_param)
+ else:
+ expression = f"COALESCE({remove_expr}, {column})"
+ params.append(remove_param)
+ fragments.append(f"{column} = {expression}")
+
+ for key, value in new_value.items():
+ if key == "remove":
+ if isinstance(value, str):
+ if value in KEY_COLUMNS:
+ raise ValueError(f"key column cannot be updated: {value}")
+ if value not in allowed_columns:
+ raise ValueError(f"unsupported remove target: {value}")
+ fragments.append(f"{value} = NULL")
+ continue
+ if not isinstance(value, dict):
+ raise ValueError(f"unsupported remove target: {value}")
+ for column, item in value.items():
+ if column not in JSONB_MULTI_VALUE_COLUMNS:
+ raise ValueError(f"unsupported JSONB remove target: {column}")
+ append_jsonb_update(column)
+ continue
+ if key == "add":
+ if not isinstance(value, dict):
+ raise ValueError(f"unsupported add target: {value}")
+ for column, item in value.items():
+ if column not in JSONB_MULTI_VALUE_COLUMNS:
+ raise ValueError(f"unsupported JSONB add target: {column}")
+ append_jsonb_update(column)
+ continue
+ if key == "metadata" and not is_meta and isinstance(value, dict):
+ fragments.append("metadata = %s::jsonb")
+ params.append(json.dumps(value, ensure_ascii=False))
+ if value.get("_group_id"):
+ fragments.append("group_id = %s")
+ params.append(value["_group_id"])
+ if value.get("_title"):
+ fragments.append("docnm_kwd = %s")
+ params.append(value["_title"])
+ continue
+ if key in KEY_COLUMNS:
+ if key == "id" and isinstance(value, str) and value == condition.get("id"):
+ continue
+ raise ValueError(f"key column cannot be updated: {key}")
+ if vector_match := VECTOR_COLUMN_RE.fullmatch(key):
+ dim = int(vector_match.group("dim"))
+ fragments.append(f"{key} = {self._placeholder(key)}")
+ params.append(vector_literal(value, dim))
+ fragments.append(f"{self.ddl.vector_valid_column_name(dim)} = TRUE")
+ continue
+ if valid_match := VECTOR_VALID_COLUMN_RE.fullmatch(str(key)):
+ dim = self.ddl.validate_vector_dim(int(valid_match.group("dim")))
+ if dim in updated_vector_dims:
+ continue
+ fragments.append(f"{key} = %s")
+ params.append(bool(value))
+ continue
+ if key not in allowed_columns:
+ raise ValueError(f"unknown column for update: {key}")
+ if key in jsonb_columns:
+ fragments.append(f"{key} = %s::jsonb")
+ params.append(json.dumps(value, ensure_ascii=False))
+ else:
+ fragments.append(f"{key} = %s")
+ params.append(value)
+ return ", ".join(fragments), params
+
+ def _build_where_clause(self, condition: dict, knowledgebase_id: str | None, is_meta: bool) -> tuple[str, list[Any]]:
+ effective = dict(condition or {})
+ if knowledgebase_id:
+ existing_kb = normalize_kb_id(effective.get("kb_id"))
+ if existing_kb and existing_kb != knowledgebase_id:
+ raise ValueError(f"condition kb_id {existing_kb} does not match dataset_id {knowledgebase_id}")
+ effective["kb_id"] = knowledgebase_id
+
+ fragments = []
+ params = []
+
+ def column_expression(column: str) -> str:
+ validate_filter_column(column, is_meta)
+ return filter_column_expr(column, is_meta)
+
+ for key, value in effective.items():
+ if key == "exists":
+ column = column_expression(value)
+ if is_jsonb_dynamic_chunk_field(value, is_meta):
+ fragments.append(f"({column} IS NOT NULL AND {column} <> 'null'::jsonb)")
+ else:
+ fragments.append(f"{column} IS NOT NULL")
+ continue
+ if key == "must_not" and isinstance(value, dict) and "exists" in value:
+ column = column_expression(value["exists"])
+ if is_jsonb_dynamic_chunk_field(value["exists"], is_meta):
+ fragments.append(f"({column} IS NULL OR {column} = 'null'::jsonb)")
+ else:
+ fragments.append(f"{column} IS NULL")
+ continue
+ column = column_expression(key)
+ if is_jsonb_dynamic_chunk_field(key, is_meta):
+ values = list(value) if isinstance(value, (list, tuple, set)) else [value]
+ if not values:
+ raise ValueError(f"empty list condition for {key}")
+ predicates = []
+ for item in values:
+ predicates.append(f"({column} = %s::jsonb OR {column} @> %s::jsonb)")
+ params.extend(
+ [
+ json.dumps(item, ensure_ascii=False),
+ json.dumps([item], ensure_ascii=False),
+ ]
+ )
+ fragments.append("(" + " OR ".join(predicates) + ")")
+ continue
+ if key in JSONB_MULTI_VALUE_COLUMNS:
+ values = list(value) if isinstance(value, (list, tuple, set)) else [value]
+ if not values:
+ raise ValueError(f"empty list condition for {key}")
+ fragments.append("(" + " OR ".join([f"{column} @> %s::jsonb"] * len(values)) + ")")
+ params.extend(json.dumps([item], ensure_ascii=False) for item in values)
+ elif isinstance(value, (list, tuple, set)):
+ values = list(value)
+ if not values:
+ raise ValueError(f"empty list condition for {key}")
+ fragments.append(f"{column} IN ({', '.join(['%s'] * len(values))})")
+ params.extend(values)
+ else:
+ fragments.append(f"{column} = %s")
+ params.append(value)
+ return " AND ".join(fragments), params
+
+ def _placeholder(self, column: str) -> str:
+ match = VECTOR_COLUMN_RE.fullmatch(column)
+ if match:
+ return f"%s::floatvector({match.group('dim')})"
+ if column in JSONB_COLUMNS or column in DOC_META_JSONB_COLUMNS:
+ return "%s::jsonb"
+ return "%s"
+
+ def _row_to_chunk(self, row, description) -> dict:
+ columns = [desc[0] for desc in description]
+ raw = dict(zip(columns, row)) if not isinstance(row, dict) else dict(row)
+ result = {}
+ dynamic_fields = {}
+ invalid_vectors = {f"q_{match.group('dim')}_vec" for key, value in raw.items() if (match := VECTOR_VALID_COLUMN_RE.fullmatch(key)) and value is False}
+ for key, value in raw.items():
+ if key in invalid_vectors:
+ continue
+ decoded = decode_column_value(key, value)
+ result[key] = decoded
+ if key == "extra" and isinstance(decoded, dict):
+ dynamic_fields.update(decoded)
+ for key, value in dynamic_fields.items():
+ result.setdefault(key, value)
+ return result
+
+ def _execute_statements(self, statements: Iterable[str | tuple[str, list[Any]]]) -> None:
+ conn = self.pool.get_conn()
+ cur = None
+ try:
+ cur = conn.cursor()
+ for statement in statements:
+ if isinstance(statement, tuple):
+ cur.execute(statement[0], statement[1])
+ else:
+ cur.execute(statement)
+ conn.commit()
+ except Exception:
+ conn.rollback()
+ raise
+ finally:
+ close_cursor(cur)
+ self.pool.put_conn(conn)
+
+ def _execute_write(self, sql: str, params: list[Any], many: bool = False) -> int:
+ conn = self.pool.get_conn()
+ cur = None
+ try:
+ cur = conn.cursor()
+ if many:
+ cur.executemany(sql, params)
+ else:
+ cur.execute(sql, params)
+ conn.commit()
+ return int(getattr(cur, "rowcount", 0) or 0)
+ except Exception:
+ conn.rollback()
+ raise
+ finally:
+ close_cursor(cur)
+ self.pool.put_conn(conn)
+
+ def _fetch_one(self, sql: str, params: list[Any]):
+ row, _description = self._fetch_one_with_description(sql, params)
+ return row
+
+ def _fetch_one_with_description(self, sql: str, params: list[Any]):
+ conn = self.pool.get_conn()
+ cur = None
+ try:
+ cur = conn.cursor()
+ cur.execute(sql, params)
+ return cur.fetchone(), getattr(cur, "description", None) or []
+ finally:
+ close_cursor(cur)
+ self.pool.put_conn(conn)
+
+ def _fetch_all(self, sql: str, params: list[Any]):
+ conn = self.pool.get_conn()
+ cur = None
+ try:
+ cur = conn.cursor()
+ cur.execute(sql, params)
+ return cur.fetchall()
+ finally:
+ close_cursor(cur)
+ self.pool.put_conn(conn)
+
+ def _fetch_all_with_description(self, sql: str, params: list[Any], statement_timeout_ms: int | None = None):
+ conn = self.pool.get_conn()
+ cur = None
+ try:
+ cur = conn.cursor()
+ if statement_timeout_ms and statement_timeout_ms > 0:
+ cur.execute(f"SET LOCAL statement_timeout = {int(statement_timeout_ms)}")
+ cur.execute(sql, params)
+ return cur.fetchall(), getattr(cur, "description", None) or []
+ finally:
+ close_cursor(cur)
+ self.pool.put_conn(conn)
+
+
+def is_doc_meta_table(index_name: str) -> bool:
+ return str(index_name or "").startswith("ragflow_doc_meta_")
+
+
+def normalize_kb_id(value) -> str | None:
+ if isinstance(value, list):
+ return str(value[0]) if value else None
+ if value is None:
+ return None
+ return str(value)
+
+
+def normalize_kb_ids(values) -> list[str]:
+ if values is None:
+ return []
+ if isinstance(values, (str, bytes)):
+ raw_values = [values]
+ else:
+ raw_values = list(values)
+ normalized = []
+ seen = set()
+ for value in raw_values:
+ if value in (None, ""):
+ continue
+ text = str(value)
+ if text in seen:
+ continue
+ seen.add(text)
+ normalized.append(text)
+ return normalized
+
+
+def normalize_table_names(index_names: str | list[str]) -> list[str]:
+ if isinstance(index_names, str):
+ return [name.strip() for name in index_names.split(",") if name.strip()]
+ return [str(name).strip() for name in index_names or [] if str(name).strip()]
+
+
+def parse_vector_dim(vector_column_name: str) -> int | None:
+ match = VECTOR_COLUMN_RE.fullmatch(str(vector_column_name or ""))
+ return int(match.group("dim")) if match else None
+
+
+def parse_fusion_vector_weight(expr: FusionExpr) -> float | None:
+ params = expr.fusion_params or {}
+ weights = params.get("weights")
+ if not weights:
+ return None
+ try:
+ return float(str(weights).split(",")[-1])
+ except (TypeError, ValueError):
+ return None
+
+
+def sortable_search_value(value: Any, field: str):
+ if field == "page_num_int":
+ return nested_numeric_value(value, [0])
+ if field == "position_int":
+ return nested_numeric_value(value, [0, 3])
+ if field == "top_int":
+ return nested_numeric_value(value, [0])
+ if isinstance(value, (int, float)):
+ return value
+ if value is None:
+ return ""
+ return str(value)
+
+
+def nested_numeric_value(value: Any, path: list[int]) -> float:
+ current = value
+ try:
+ for index in path:
+ current = current[index]
+ numeric = float(current)
+ return numeric if isfinite(numeric) else 100000000.0
+ except (TypeError, ValueError, IndexError, KeyError, OverflowError):
+ return 100000000.0
+
+
+def select_doc_meta_columns(select_fields: list[str]) -> list[str]:
+ if not select_fields or "*" in select_fields:
+ return list(DOC_META_COLUMNS)
+ columns = []
+ for field in select_fields:
+ validate_filter_column(field, is_meta=True)
+ if field not in columns:
+ columns.append(field)
+ return columns
+
+
+def build_doc_meta_order_by(order_by) -> str:
+ fields = getattr(order_by, "fields", None) or []
+ parts = []
+ for field, direction in fields:
+ validate_filter_column(field, is_meta=True)
+ parts.append(f"{field} {'DESC' if direction else 'ASC'}")
+ return ", ".join(parts)
+
+
+def normalize_column_value(column: str, value: Any) -> Any:
+ if value is None:
+ return None
+ if column in JSONB_COLUMNS:
+ return json.dumps(value, ensure_ascii=False) if not isinstance(value, str) else value
+ return value
+
+
+def parse_json_dict(value) -> dict:
+ if isinstance(value, dict):
+ return value
+ if isinstance(value, str):
+ try:
+ parsed = json.loads(value)
+ except json.JSONDecodeError:
+ return {}
+ return parsed if isinstance(parsed, dict) else {}
+ return {}
+
+
+def vector_literal(value, dim: int) -> str:
+ if isinstance(value, str):
+ parsed = parse_vector_literal(value)
+ if parsed is None:
+ raise ValueError("invalid vector literal")
+ value = parsed
+ if not isinstance(value, (list, tuple)) or len(value) != dim:
+ raise ValueError(f"vector dimension mismatch: expected {dim}, got {len(value) if hasattr(value, '__len__') else 'unknown'}")
+ return "[" + ",".join(str(item) for item in value) + "]"
+
+
+def parse_vector_literal(value: str):
+ try:
+ parsed = json.loads(value)
+ except json.JSONDecodeError:
+ return None
+ return parsed if isinstance(parsed, list) else None
+
+
+def zero_vector_literal(dim: int) -> str:
+ return "[" + ",".join(["0"] * dim) + "]"
+
+
+def ordered_columns(rows: list[dict]) -> list[str]:
+ present = set()
+ for row in rows:
+ present.update(row)
+ dynamic_vectors = sorted(column for column in present if VECTOR_COLUMN_RE.fullmatch(column))
+ dynamic_valid = sorted(column for column in present if VECTOR_VALID_COLUMN_RE.fullmatch(column))
+ ordered = [column for column in CHUNK_COLUMNS if column in present]
+ ordered.extend(column for column in dynamic_vectors + dynamic_valid if column not in ordered)
+ return ordered
+
+
+def validate_filter_column(column: str, is_meta: bool) -> None:
+ if is_meta:
+ if column not in DOC_META_COLUMN_SET:
+ raise ValueError(f"unsupported metadata filter column: {column}")
+ return
+ if column not in CHUNK_COLUMN_SET and not VECTOR_COLUMN_RE.fullmatch(str(column)) and not VECTOR_VALID_COLUMN_RE.fullmatch(str(column)):
+ validate_extra_field(column)
+
+
+def is_dynamic_chunk_field(column: str, is_meta: bool = False) -> bool:
+ return not is_meta and column not in CHUNK_COLUMN_SET and not VECTOR_COLUMN_RE.fullmatch(str(column)) and not VECTOR_VALID_COLUMN_RE.fullmatch(str(column))
+
+
+def is_jsonb_dynamic_chunk_field(column: str, is_meta: bool = False) -> bool:
+ return is_dynamic_chunk_field(column, is_meta) and column not in GaussDBSearchBuilder.JSONB_EXTRA_SCALAR_COLUMNS
+
+
+def filter_column_expr(column: str, is_meta: bool = False) -> str:
+ validate_filter_column(column, is_meta)
+ if not is_dynamic_chunk_field(column, is_meta):
+ return str(column)
+ if column in GaussDBSearchBuilder.JSONB_EXTRA_SCALAR_COLUMNS:
+ return f"(extra #>> '{{{column}}}')"
+ return extra_field_expr(column)
+
+
+def decode_column_value(column: str, value: Any) -> Any:
+ if value is None:
+ return None
+ if column in JSONB_COLUMNS or column in DOC_META_JSONB_COLUMNS:
+ if isinstance(value, str):
+ try:
+ return json.loads(value)
+ except json.JSONDecodeError as exc:
+ raise ValueError(f"invalid JSONB value for {column}") from exc
+ return value
+ if VECTOR_COLUMN_RE.fullmatch(column):
+ if isinstance(value, str):
+ parsed = parse_vector_literal(value)
+ if parsed is None:
+ raise ValueError(f"invalid vector value for {column}")
+ return parsed
+ return value
+ return value
+
+
+def close_cursor(cur) -> None:
+ if cur is not None and hasattr(cur, "close"):
+ cur.close()
diff --git a/rag/utils/gaussdb_text_to_sql.py b/rag/utils/gaussdb_text_to_sql.py
new file mode 100644
index 0000000000..ff861e0c18
--- /dev/null
+++ b/rag/utils/gaussdb_text_to_sql.py
@@ -0,0 +1,219 @@
+#
+# Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+import logging
+
+from sqlglot import exp, parse_one
+
+from common.doc_store.gaussdb_conn_base import (
+ ExposedGaussDBTable,
+ GaussDBSQLValidator,
+ gaussdb_field_json_path_parts,
+ is_gaussdb_aggregate_sql,
+ jsonb_path_literal,
+)
+
+
+logger = logging.getLogger(__name__)
+
+_SQL_RULES = """RULES:
+1. Use only the table and fields listed below.
+2. Select doc_id and docnm_kwd for non-aggregate data queries.
+3. Do not add kb_id to SELECT or WHERE; the system injects and validates the KB scope.
+4. Use only the static JSONB path literals listed under Fields.
+5. Do not construct dynamic JSONB paths.
+6. Do not use json_extract, json_extract_string, or json_extract_isnull; use #>> / #> instead.
+7. A/ORA treats '' as NULL; never compare with = '' or <> ''; use IS NULL or IS NOT NULL.
+8. Return exactly one read-only SELECT statement and no other text.
+9. Do not use SELECT *, window functions, or system functions.
+10. Do not use LATERAL, JOIN, UNION, INTERSECT, EXCEPT, or OR.
+11. Do not expand JSON with jsonb_each, jsonb_each_text, jsonb_to_record, jsonb_to_recordset, jsonb_array_elements, or jsonb_array_elements_text.
+12. Do not use DML or DDL, including INSERT, UPDATE, DELETE, MERGE, CREATE, ALTER, DROP, TRUNCATE, GRANT, REVOKE, CALL, COPY, or DO."""
+
+
+def build_validator(table_name, kb_ids, field_map):
+ table = ExposedGaussDBTable.from_field_map(table_name, kb_ids, field_map)
+ return GaussDBSQLValidator({table.logical_name: table}, default_limit=128)
+
+
+def is_aggregate_sql(sql):
+ return is_gaussdb_aggregate_sql(sql)
+
+
+def _json_path_literal(field, field_type=None):
+ return jsonb_path_literal(gaussdb_field_json_path_parts(str(field), field_type))
+
+
+def _field_lines(field_map):
+ return "\n".join(f" - {field} ({descriptor}): chunk_data #>> {_json_path_literal(field, descriptor)}" for field, descriptor in (field_map or {}).items())
+
+
+def build_sql_prompt(table_name, field_map, question):
+ return """You are a Database Administrator. Write SQL for GaussDB A/ORA mode for a table with JSONB 'chunk_data' column.
+
+Table: {table_name}
+JSONB text extraction: chunk_data #>> '{{FieldName}}'
+JSONB value extraction: chunk_data #> '{{FieldName}}'
+Numeric cast: CAST(chunk_data #>> '{{FieldName}}' AS DOUBLE PRECISION)
+Date cast: to_date(chunk_data #>> '{{FieldName}}', 'YYYY-MM-DD')
+NULL check: (chunk_data #>> '{{FieldName}}') IS NOT NULL
+
+{rules}
+
+Fields:
+{fields}
+
+Question: {question}""".format(
+ table_name=table_name,
+ rules=_SQL_RULES,
+ fields=_field_lines(field_map),
+ question=question,
+ )
+
+
+def build_user_prompt(table_name, field_map, question):
+ return """Table: {table_name}
+Fields:
+{fields}
+Question: {question}
+Write SQL using GaussDB JSONB #>> / #> syntax. Include doc_id and docnm_kwd for data queries. Only SQL.""".format(
+ table_name=table_name,
+ fields=_field_lines(field_map),
+ question=question,
+ )
+
+
+def build_repair_prompt(table_name, field_map, question, previous_sql):
+ return """Table name: {table_name};
+GaussDB JSONB fields available in chunk_data:
+{fields}
+
+Question: {question}
+Previous SQL:
+{previous_sql}
+
+The previous SQL result is missing required source columns for citations.
+Rewrite SQL to keep the same query intent and include doc_id and docnm_kwd in the SELECT list.
+Use chunk_data #>> '{{field}}' or chunk_data #> '{{field}}' for JSONB fields.
+For date fields, use to_date(chunk_data #>> '{{field}}', 'YYYY-MM-DD').
+
+{rules}""".format(
+ table_name=table_name,
+ fields=_field_lines(field_map),
+ question=question,
+ previous_sql=previous_sql,
+ rules=_SQL_RULES,
+ )
+
+
+def build_retry_prompt(table_name, field_map, question, error):
+ return """
+Table name: {table_name};
+GaussDB JSONB fields available in chunk_data:
+{fields}
+
+Question: {question}
+Please write SQL using chunk_data #>> '{{field}}' / chunk_data #> '{{field}}' syntax. Include doc_id and docnm_kwd for data queries. Only SQL.
+For date fields, use to_date(chunk_data #>> '{{field}}', 'YYYY-MM-DD').
+
+The previous SQL error is:
+{error}
+
+Correct the SQL for GaussDB A/ORA mode.
+
+{rules}
+""".format(
+ table_name=table_name,
+ fields=_field_lines(field_map),
+ question=question,
+ error=error,
+ rules=_SQL_RULES,
+ )
+
+
+def build_aggregate_source_sql(sql, doc_name_column, include_kb_id):
+ statement = parse_one(sql, read=GaussDBSQLValidator.PARSE_DIALECT)
+ if statement.args.get("having"):
+ raise ValueError("GaussDB aggregate source lookup cannot preserve HAVING safely")
+
+ columns = [exp.column("doc_id"), exp.column(doc_name_column)]
+ if include_kb_id:
+ columns.append(exp.column("kb_id"))
+ statement.set("expressions", columns)
+ statement.set("distinct", None)
+ statement.set("group", None)
+ statement.set("order", None)
+ statement.set("limit", None)
+ statement.set("offset", None)
+ return statement.sql(dialect=GaussDBSQLValidator.PARSE_DIALECT)
+
+
+def build_source_reference(table, kb_ids):
+ rows = table.get("rows", [])
+ if not rows:
+ return None
+
+ columns = table.get("columns", [])
+ doc_idx = next((i for i, column in enumerate(columns) if column["name"].lower() == "doc_id"), None)
+ name_idx = next((i for i, column in enumerate(columns) if column["name"].lower() in ["docnm_kwd", "docnm"]), None)
+ kb_idx = next((i for i, column in enumerate(columns) if column["name"].lower() in ["kb_id", "kb_id_kwd"]), None)
+ if doc_idx is None or name_idx is None:
+ return None
+
+ chunks = []
+ doc_aggs = {}
+ for row in rows:
+ chunk = {"doc_id": row[doc_idx], "docnm_kwd": row[name_idx]}
+ if len(kb_ids or []) == 1:
+ chunk["kb_id"] = kb_ids[0]
+ elif kb_idx is not None:
+ chunk["kb_id"] = row[kb_idx]
+ chunks.append(chunk)
+
+ doc_id = row[doc_idx]
+ if doc_id not in doc_aggs:
+ doc_aggs[doc_id] = {"doc_name": row[name_idx], "count": 0}
+ doc_aggs[doc_id]["count"] += 1
+
+ return {
+ "chunks": chunks,
+ "doc_aggs": [{"doc_id": doc_id, "doc_name": values["doc_name"], "count": values["count"]} for doc_id, values in doc_aggs.items()],
+ }
+
+
+def complete_reference_kb_ids(result, table_name, kb_ids, validator, sql_retrieval):
+ if len(kb_ids or []) <= 1:
+ return
+
+ chunks = result["reference"]["chunks"]
+ doc_ids = list(dict.fromkeys(str(chunk["doc_id"]) for chunk in chunks if chunk.get("doc_id") and not chunk.get("kb_id")))
+ if not doc_ids:
+ return
+
+ quoted_doc_ids = ", ".join("'" + doc_id.replace("'", "''") + "'" for doc_id in doc_ids)
+ lookup_sql = f"SELECT doc_id, kb_id FROM {table_name} WHERE doc_id IN ({quoted_doc_ids})"
+ try:
+ lookup_sql = validator.validate_and_patch(lookup_sql).sql
+ lookup_tbl = sql_retrieval(lookup_sql, format="json")
+ doc_idx = next((i for i, column in enumerate(lookup_tbl.get("columns", [])) if column["name"].lower() == "doc_id"), None)
+ kb_idx = next((i for i, column in enumerate(lookup_tbl.get("columns", [])) if column["name"].lower() in ["kb_id", "kb_id_kwd"]), None)
+ if doc_idx is None or kb_idx is None:
+ return
+ mapping = {row[doc_idx]: row[kb_idx] for row in lookup_tbl.get("rows", []) if len(row) > max(doc_idx, kb_idx)}
+ for chunk in chunks:
+ if not chunk.get("kb_id") and chunk.get("doc_id") in mapping:
+ chunk["kb_id"] = mapping[chunk["doc_id"]]
+ except Exception:
+ logger.warning("use_sql: Failed to complete GaussDB reference kb_id values", exc_info=True)
diff --git a/rag/utils/table_es_metadata.py b/rag/utils/table_es_metadata.py
index c06155c225..b56edcadc4 100644
--- a/rag/utils/table_es_metadata.py
+++ b/rag/utils/table_es_metadata.py
@@ -210,16 +210,20 @@ def aggregate_table_doc_metadata(chunks: list, task: dict) -> dict:
e,
exc_info=True,
)
- if not fm and not (settings.DOC_ENGINE_INFINITY or settings.DOC_ENGINE_OCEANBASE or settings.DOC_ENGINE_SERENEDB):
+ sql_doc_engine = settings.DOC_ENGINE_INFINITY or settings.DOC_ENGINE_OCEANBASE or settings.DOC_ENGINE_GAUSSDB or settings.DOC_ENGINE_SERENEDB
+ if not fm and not sql_doc_engine:
logging.debug(f"[TABLE_META_DEBUG] field_map empty on task snapshot — will use ES key probe on chunk dicts; kb_parser_config keys={list((task.get('kb_parser_config') or {}).keys())}")
- logging.debug(f"[TABLE_META_DEBUG] meta_cols={meta_cols}, field_map entries={len(fm)}, infinity={settings.DOC_ENGINE_INFINITY}, oceanbase={settings.DOC_ENGINE_OCEANBASE}")
+ logging.debug(
+ f"[TABLE_META_DEBUG] meta_cols={meta_cols}, field_map entries={len(fm)}, infinity={settings.DOC_ENGINE_INFINITY}, "
+ f"oceanbase={settings.DOC_ENGINE_OCEANBASE}, gaussdb={settings.DOC_ENGINE_GAUSSDB}, serenedb={settings.DOC_ENGINE_SERENEDB}"
+ )
sample_ck = next((c for c in chunks if isinstance(c, dict)), None)
if sample_ck:
sk = [k for k in sample_ck.keys() if not (str(k).startswith("q_") and str(k).endswith("_vec"))][:50]
logging.debug(f"[TABLE_META_DEBUG] first chunk non-vector keys (sample): {sk}")
es_col_keys: dict[str, tuple[str | None, str]] = {}
- if not (settings.DOC_ENGINE_INFINITY or settings.DOC_ENGINE_OCEANBASE or settings.DOC_ENGINE_SERENEDB):
+ if not sql_doc_engine:
for col in meta_cols:
tk, src = _resolve_es_chunk_field_key(col, fm, sample_ck)
es_col_keys[col] = (tk, src)
@@ -230,7 +234,7 @@ def aggregate_table_doc_metadata(chunks: list, task: dict) -> dict:
for i, ck in enumerate(chunks):
if not isinstance(ck, dict):
continue
- if settings.DOC_ENGINE_INFINITY or settings.DOC_ENGINE_OCEANBASE or settings.DOC_ENGINE_SERENEDB:
+ if sql_doc_engine:
cd = ck.get("chunk_data")
if not isinstance(cd, dict):
continue
diff --git a/test/testcases/test_web_api/test_system_app/test_system_routes_unit.py b/test/testcases/test_web_api/test_system_app/test_system_routes_unit.py
index a9735fa300..4793a97234 100644
--- a/test/testcases/test_web_api/test_system_app/test_system_routes_unit.py
+++ b/test/testcases/test_web_api/test_system_app/test_system_routes_unit.py
@@ -148,6 +148,7 @@ def _load_system_module(monkeypatch):
health_utils_mod = ModuleType("api.utils.health_utils")
health_utils_mod.run_health_checks = lambda: ({"status": "ok"}, True)
health_utils_mod.get_oceanbase_status = lambda: {"status": "alive"}
+ health_utils_mod.get_gaussdb_status = lambda: {"status": "alive"}
monkeypatch.setitem(sys.modules, "api.utils.health_utils", health_utils_mod)
quart_mod = ModuleType("quart")
@@ -220,3 +221,14 @@ def test_get_config_returns_register_enabled_unit(monkeypatch):
res = module.get_config()
assert res["code"] == 0
assert res["data"]["registerEnabled"] is False
+
+
+def test_gaussdb_status_route_unit(monkeypatch):
+ module = _load_system_module(monkeypatch)
+ monkeypatch.setattr(module, "get_gaussdb_status", lambda: {"status": "alive"})
+
+ res = module.gaussdb_status()
+
+ assert res["code"] == 0
+ assert res["data"]["status"] == "alive"
diff --git a/test/unit_test/api/apps/services/test_gaussdb_dataset_embedding.py b/test/unit_test/api/apps/services/test_gaussdb_dataset_embedding.py
new file mode 100644
index 0000000000..4aca9efce5
--- /dev/null
+++ b/test/unit_test/api/apps/services/test_gaussdb_dataset_embedding.py
@@ -0,0 +1,209 @@
+#
+# Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+import importlib.util
+import sys
+from pathlib import Path
+from types import ModuleType, SimpleNamespace
+from unittest.mock import MagicMock
+
+
+def _install_module(monkeypatch, name, **attrs):
+ module = ModuleType(name)
+ for key, value in attrs.items():
+ setattr(module, key, value)
+ monkeypatch.setitem(sys.modules, name, module)
+ if "." in name:
+ parent_name, _, child_name = name.rpartition(".")
+ parent = sys.modules.get(parent_name)
+ if parent is not None:
+ monkeypatch.setattr(parent, child_name, module, raising=False)
+ return module
+
+
+class FakeSearchResult:
+ pass
+
+
+class FakeGaussDBStore:
+ def db_type(self):
+ return "gaussdb"
+
+ def search(self, *_args, **_kwargs):
+ return FakeSearchResult()
+
+ def get_total(self, _res):
+ return 1
+
+ def get_doc_ids(self, _res):
+ return ["chunk-invalid-vector"]
+
+ def get(self, _chunk_id, _index_name, _kb_ids):
+ return {
+ "id": "chunk-invalid-vector",
+ "kb_id": "kb1",
+ "doc_id": "doc1",
+ "docnm_kwd": "doc.txt",
+ "content_with_weight": "sample text",
+ "q_4_vec": [0.0, 0.0, 0.0, 0.0],
+ "q_4_vec_valid": False,
+ }
+
+
+class ValidVectorGaussDBStore(FakeGaussDBStore):
+ def get(self, _chunk_id, _index_name, _kb_ids):
+ return {
+ "id": "chunk-valid-vector",
+ "kb_id": "kb1",
+ "doc_id": "doc1",
+ "docnm_kwd": "doc.txt",
+ "content_with_weight": "sample text",
+ "q_4_vec": [1.0, 0.0, 0.0, 0.0],
+ "q_4_vec_valid": True,
+ }
+
+
+class FailIfCalledBundle:
+ def __init__(self, *_args, **_kwargs):
+ pass
+
+ def encode(self, *_args, **_kwargs):
+ raise AssertionError("invalid placeholder vectors must be skipped before re-embedding")
+
+
+class CompatibleEmbeddingBundle:
+ encode_calls = []
+
+ def __init__(self, *_args, **_kwargs):
+ pass
+
+ def encode(self, *args, **kwargs):
+ import numpy as np
+
+ self.encode_calls.append((args, kwargs))
+ return [np.array([1.0, 0.0, 0.0, 0.0]), np.array([1.0, 0.0, 0.0, 0.0])], None
+
+
+def _load_dataset_module(monkeypatch):
+ repo_root = Path(__file__).resolve().parents[5]
+ kb = SimpleNamespace(id="kb1", tenant_id="tenant1", embd_id="old-embd")
+
+ _install_module(
+ monkeypatch,
+ "api.db.joint_services.tenant_model_service",
+ get_model_config_from_provider_instance=lambda *_args, **_kwargs: {"llm_name": "new-embd"},
+ get_composite_model_name_by_ids=lambda _ids: {},
+ resolve_model_config=lambda *_args, **_kwargs: {"llm_name": "new-embd"},
+ resolve_model_id=lambda *_args, **_kwargs: "new-embd",
+ )
+ _install_module(monkeypatch, "api.db.db_models", File=SimpleNamespace())
+ _install_module(
+ monkeypatch,
+ "api.db.services.document_service",
+ DocumentService=SimpleNamespace(),
+ queue_raptor_o_graphrag_tasks=MagicMock(),
+ )
+ _install_module(monkeypatch, "api.db.services.file2document_service", File2DocumentService=SimpleNamespace())
+ _install_module(monkeypatch, "api.db.services.file_service", FileService=SimpleNamespace())
+ _install_module(
+ monkeypatch,
+ "api.db.services.knowledgebase_service",
+ KnowledgebaseService=SimpleNamespace(
+ accessible=staticmethod(lambda *_args: True),
+ get_by_id=staticmethod(lambda _dataset_id: (True, kb)),
+ ),
+ validate_dataset_embedding_models=lambda *_args, **_kwargs: None,
+ )
+ _install_module(monkeypatch, "api.db.services.connector_service", Connector2KbService=SimpleNamespace())
+ _install_module(
+ monkeypatch,
+ "api.db.services.task_service",
+ GRAPH_RAPTOR_FAKE_DOC_ID="fake-doc",
+ TaskService=SimpleNamespace(),
+ )
+ _install_module(
+ monkeypatch,
+ "api.db.services.user_service",
+ TenantService=SimpleNamespace(),
+ UserService=SimpleNamespace(),
+ UserTenantService=SimpleNamespace(),
+ )
+ _install_module(
+ monkeypatch,
+ "api.db.services.tenant_model_service",
+ TenantModelService=SimpleNamespace(get_by_id=staticmethod(lambda *_args, **_kwargs: (True, None))),
+ )
+ _install_module(monkeypatch, "common.settings", docStoreConn=FakeGaussDBStore())
+ _install_module(
+ monkeypatch,
+ "common.constants",
+ FileSource=SimpleNamespace(KNOWLEDGEBASE="knowledgebase"),
+ LLMType=SimpleNamespace(EMBEDDING="embedding"),
+ PAGERANK_FLD="pagerank_fea",
+ RetCode=SimpleNamespace(NOT_EFFECTIVE=590),
+ StatusEnum=SimpleNamespace(VALID=SimpleNamespace(value="1")),
+ )
+ _install_module(
+ monkeypatch,
+ "api.utils.api_utils",
+ deep_merge=lambda base, update: {**(base or {}), **(update or {})},
+ get_parser_config=lambda *_args, **_kwargs: {},
+ remap_dictionary_keys=lambda value: value,
+ verify_embedding_availability=lambda *_args, **_kwargs: (True, ""),
+ )
+ rag = _install_module(monkeypatch, "rag")
+ rag.__path__ = []
+ advanced_rag = _install_module(monkeypatch, "rag.advanced_rag")
+ advanced_rag.__path__ = []
+ knowledge_compile = _install_module(monkeypatch, "rag.advanced_rag.knowlege_compile")
+ knowledge_compile.__path__ = []
+ _install_module(monkeypatch, "rag.advanced_rag.knowlege_compile.wiki", WIKI_PAGE_COMPILE_KWD="artifact_page")
+ _install_module(monkeypatch, "rag.nlp")
+ _install_module(monkeypatch, "rag.nlp.search", index_name=lambda tenant_id: f"ragflow_{tenant_id}")
+
+ spec = importlib.util.spec_from_file_location(
+ "api.apps.services.dataset_api_service",
+ repo_root / "api" / "apps" / "services" / "dataset_api_service.py",
+ )
+ module = importlib.util.module_from_spec(spec)
+ monkeypatch.setitem(sys.modules, "api.apps.services.dataset_api_service", module)
+ spec.loader.exec_module(module)
+ return module
+
+
+def test_tc_wrt_206_check_embedding_ignores_gaussdb_invalid_placeholder_vectors(monkeypatch):
+ module = _load_dataset_module(monkeypatch)
+
+ _install_module(monkeypatch, "api.db.services.llm_service", LLMBundle=FailIfCalledBundle)
+
+ ok, result = module.check_embedding("kb1", "tenant1", {"embd_id": "new-embd", "check_num": 1})
+
+ assert ok is False
+ assert result == "No embedded chunks are available to compare."
+
+
+def test_tc_wrt_207_check_embedding_compares_gaussdb_valid_vectors_once(monkeypatch):
+ module = _load_dataset_module(monkeypatch)
+ module.settings.docStoreConn = ValidVectorGaussDBStore()
+ CompatibleEmbeddingBundle.encode_calls.clear()
+ _install_module(monkeypatch, "api.db.services.llm_service", LLMBundle=CompatibleEmbeddingBundle)
+
+ ok, result = module.check_embedding("kb1", "tenant1", {"embd_id": "new-embd", "check_num": 1})
+
+ assert CompatibleEmbeddingBundle.encode_calls == [((["doc.txt", "sample text"],), {})]
+ assert ok is True
+ assert result["summary"]["valid"] == 1
+ assert result["summary"]["avg_cos_sim"] == 1.0
+ assert result["results"][0]["vector_dim"] == 4
diff --git a/test/unit_test/api/apps/test_gaussdb_system_routes.py b/test/unit_test/api/apps/test_gaussdb_system_routes.py
new file mode 100644
index 0000000000..d34e2fc2cc
--- /dev/null
+++ b/test/unit_test/api/apps/test_gaussdb_system_routes.py
@@ -0,0 +1,285 @@
+#
+# Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import asyncio
+import importlib.util
+import json
+import sys
+from pathlib import Path
+from types import ModuleType, SimpleNamespace
+from unittest.mock import Mock
+
+import pytest
+
+
+ROUTE_PASSWORD_SENTINEL = "cfg703-password-'quoted![]{}'"
+ROUTE_DSN_SENTINEL = "postgresql://cfg703-user:cfg703-dsn%27%[email protected]:19995/private"
+ROUTE_TOKEN_SENTINEL = "cfg703-token-'quoted![]{}'"
+
+
+class _DummyManager:
+ def route(self, *_args, **_kwargs):
+ def decorator(func):
+ return func
+
+ return decorator
+
+
+class _ExprField:
+ def __init__(self, name):
+ self.name = name
+
+ def __eq__(self, other):
+ return (self.name, other)
+
+
+class _DummyAPITokenModel:
+ tenant_id = _ExprField("tenant_id")
+ token = _ExprField("token")
+
+
+def _load_system_module(monkeypatch, *, apps_module=None, manager=None):
+ repo_root = Path(__file__).resolve().parents[4]
+
+ api_pkg = ModuleType("api")
+ api_pkg.__path__ = [str(repo_root / "api")]
+ monkeypatch.setitem(sys.modules, "api", api_pkg)
+
+ if apps_module is None:
+ apps_mod = ModuleType("api.apps")
+ apps_mod.__path__ = [str(repo_root / "api" / "apps")]
+ apps_mod.login_required = lambda fn: fn
+ apps_mod.current_user = SimpleNamespace(id="user-1")
+ else:
+ apps_mod = apps_module
+ monkeypatch.setitem(sys.modules, "api.apps", apps_mod)
+
+ common_pkg = ModuleType("common")
+ common_pkg.__path__ = [str(repo_root / "common")]
+ monkeypatch.setitem(sys.modules, "common", common_pkg)
+
+ settings_mod = ModuleType("common.settings")
+ settings_mod.docStoreConn = SimpleNamespace(health=lambda: {"type": "doc", "status": "green"})
+ settings_mod.STORAGE_IMPL = SimpleNamespace(health=lambda: True)
+ settings_mod.STORAGE_IMPL_TYPE = "MINIO"
+ settings_mod.DATABASE_TYPE = "MYSQL"
+ settings_mod.REGISTER_ENABLED = True
+ settings_mod.DISABLE_PASSWORD_LOGIN = False
+ common_pkg.settings = settings_mod
+ monkeypatch.setitem(sys.modules, "common.settings", settings_mod)
+
+ versions_mod = ModuleType("common.versions")
+ versions_mod.get_ragflow_version = lambda: "0.0.0-unit"
+ monkeypatch.setitem(sys.modules, "common.versions", versions_mod)
+
+ time_utils_mod = ModuleType("common.time_utils")
+ time_utils_mod.current_timestamp = lambda: 111
+ time_utils_mod.datetime_format = lambda _dt: "2026-01-01 00:00:00"
+ monkeypatch.setitem(sys.modules, "common.time_utils", time_utils_mod)
+
+ api_utils_mod = ModuleType("api.utils.api_utils")
+ api_utils_mod.get_json_result = lambda data=None, message="success", code=0: {
+ "code": code,
+ "message": message,
+ "data": data,
+ }
+ api_utils_mod.get_data_error_result = lambda message="", code=102, data=None: {
+ "code": code,
+ "message": message,
+ "data": data,
+ }
+ api_utils_mod.server_error_response = lambda exc: {
+ "code": 100,
+ "message": repr(exc),
+ "data": None,
+ }
+ api_utils_mod.generate_confirmation_token = lambda: "ragflow-abcdefghijklmnopqrstuvwxyz0123456789"
+ monkeypatch.setitem(sys.modules, "api.utils.api_utils", api_utils_mod)
+
+ api_service_mod = ModuleType("api.db.services.api_service")
+ api_service_mod.APITokenService = SimpleNamespace(
+ save=lambda **_kwargs: True,
+ query=lambda **_kwargs: [],
+ filter_update=lambda *_args, **_kwargs: True,
+ filter_delete=lambda *_args, **_kwargs: True,
+ )
+ monkeypatch.setitem(sys.modules, "api.db.services.api_service", api_service_mod)
+
+ kb_service_mod = ModuleType("api.db.services.knowledgebase_service")
+ kb_service_mod.KnowledgebaseService = SimpleNamespace(get_by_id=lambda _kb_id: True)
+ monkeypatch.setitem(sys.modules, "api.db.services.knowledgebase_service", kb_service_mod)
+
+ user_service_mod = ModuleType("api.db.services.user_service")
+ user_service_mod.UserTenantService = SimpleNamespace(query=lambda **_kwargs: [SimpleNamespace(role="owner", tenant_id="tenant-1")])
+ monkeypatch.setitem(sys.modules, "api.db.services.user_service", user_service_mod)
+
+ db_models_mod = ModuleType("api.db.db_models")
+ db_models_mod.APIToken = _DummyAPITokenModel
+ monkeypatch.setitem(sys.modules, "api.db.db_models", db_models_mod)
+
+ rag_pkg = ModuleType("rag")
+ rag_pkg.__path__ = []
+ monkeypatch.setitem(sys.modules, "rag", rag_pkg)
+
+ rag_utils_pkg = ModuleType("rag.utils")
+ rag_utils_pkg.__path__ = []
+ monkeypatch.setitem(sys.modules, "rag.utils", rag_utils_pkg)
+
+ redis_mod = ModuleType("rag.utils.redis_conn")
+ redis_mod.REDIS_CONN = SimpleNamespace(
+ health=lambda: True,
+ smembers=lambda *_args, **_kwargs: set(),
+ zrangebyscore=lambda *_args, **_kwargs: [],
+ )
+ monkeypatch.setitem(sys.modules, "rag.utils.redis_conn", redis_mod)
+
+ health_utils_mod = ModuleType("api.utils.health_utils")
+ health_utils_mod.run_health_checks = lambda: ({"status": "ok"}, True)
+ health_utils_mod.get_oceanbase_status = lambda: {"status": "alive"}
+ health_utils_mod.get_gaussdb_status = lambda: {"status": "alive"}
+ monkeypatch.setitem(sys.modules, "api.utils.health_utils", health_utils_mod)
+
+ quart_mod = ModuleType("quart")
+ quart_mod.jsonify = lambda payload: payload
+ monkeypatch.setitem(sys.modules, "quart", quart_mod)
+
+ module_path = repo_root / "api" / "apps" / "restful_apis" / "system_api.py"
+ spec = importlib.util.spec_from_file_location("test_gaussdb_system_routes_module", module_path)
+ module = importlib.util.module_from_spec(spec)
+ module.manager = manager or _DummyManager()
+ monkeypatch.setitem(sys.modules, "test_gaussdb_system_routes_module", module)
+ spec.loader.exec_module(module)
+ return module
+
+
+def _load_system_http_app(monkeypatch):
+ from test.testcases.test_web_api.test_system_app.test_apps_init_unit import _load_apps_module
+
+ quart_app, apps_module = _load_apps_module(monkeypatch)
+ module = _load_system_module(monkeypatch, apps_module=apps_module, manager=quart_app)
+ return quart_app, module
+
+
+def _get_without_auth(quart_app, path):
+ async def request():
+ response = await quart_app.test_client().get(path)
+ return response.status_code, await response.get_json()
+
+ return asyncio.run(request())
+
+
+def test_tc_cfg_704_status_branch_matrix_unit(monkeypatch):
+ module = _load_system_module(monkeypatch)
+ expected = {
+ "db": "ok",
+ "redis": "ok",
+ "doc_engine": "ok",
+ "storage": "ok",
+ "status": "ok",
+ }
+ monkeypatch.setattr(module, "run_health_checks", lambda: (expected, True))
+
+ payload, status_code = module.healthz()
+
+ assert status_code == 200
+ assert payload == expected
+
+
+def test_tc_cfg_705_healthz_returns_500_when_any_check_is_not_ok(monkeypatch):
+ module = _load_system_module(monkeypatch)
+ expected = {
+ "db": "ok",
+ "redis": "ok",
+ "doc_engine": "nok",
+ "storage": "ok",
+ "status": "nok",
+ "_meta": {"doc_engine": {"error": "doc down"}},
+ }
+ monkeypatch.setattr(module, "run_health_checks", lambda: (expected, False))
+
+ payload, status_code = module.healthz()
+
+ assert status_code == 500
+ assert payload == expected
+
+
+def test_tc_cfg_706_healthz_route_allows_real_unauthenticated_request(monkeypatch):
+ quart_app, module = _load_system_http_app(monkeypatch)
+ expected = {"status": "ok", "doc_engine": "ok"}
+ monkeypatch.setattr(module, "run_health_checks", lambda: (expected, True))
+
+ status_code, payload = _get_without_auth(quart_app, "/system/healthz")
+
+ assert status_code == 200
+ assert payload == expected
+
+
+def test_tc_cfg_702_gaussdb_status_route_rejects_real_unauthenticated_request(monkeypatch):
+ quart_app, module = _load_system_http_app(monkeypatch)
+ probe = Mock(return_value={"status": "alive"})
+ monkeypatch.setattr(module, "get_gaussdb_status", probe)
+
+ status_code, payload = _get_without_auth(quart_app, "/system/gaussdb/status")
+
+ assert status_code == 401
+ assert payload["code"] == 401
+ assert "Unauthorized" in payload["message"]
+ probe.assert_not_called()
+
+
+def test_tc_cfg_712_gaussdb_status_route_returns_probe_payload(monkeypatch):
+ module = _load_system_module(monkeypatch)
+ monkeypatch.setattr(module, "get_gaussdb_status", lambda: {"status": "alive"})
+
+ res = module.gaussdb_status()
+
+ assert res["code"] == 0
+ assert res == {"code": 0, "message": "success", "data": {"status": "alive"}}
+
+
+def test_tc_cfg_703_gaussdb_status_route_returns_500_when_probe_raises(monkeypatch, caplog):
+ module = _load_system_module(monkeypatch)
+
+ def raise_probe_error():
+ raise RuntimeError(f"probe failed password=\"{ROUTE_PASSWORD_SENTINEL}\" dsn={ROUTE_DSN_SENTINEL} access_token='{ROUTE_TOKEN_SENTINEL}'")
+
+ monkeypatch.setattr(module, "get_gaussdb_status", raise_probe_error)
+ caplog.set_level("ERROR")
+
+ res = module.gaussdb_status()
+ serialized = json.dumps(res, sort_keys=True)
+ log_text = caplog.text
+
+ assert res["code"] == 500
+ assert res["data"]["status"] == "error"
+ assert res["data"]["message"].startswith("Failed to get GaussDB status: probe failed")
+ assert "***" in res["data"]["message"]
+ assert ROUTE_PASSWORD_SENTINEL not in serialized
+ assert ROUTE_DSN_SENTINEL not in serialized
+ assert ROUTE_TOKEN_SENTINEL not in serialized
+ assert "postgresql://" not in serialized
+ assert "cfg703-user" not in serialized
+ assert "GaussDB status route failed (RuntimeError)" in log_text
+ assert "***" in log_text
+ assert ROUTE_PASSWORD_SENTINEL not in log_text
+ assert ROUTE_DSN_SENTINEL not in log_text
+ assert ROUTE_TOKEN_SENTINEL not in log_text
diff --git a/test/unit_test/api/db/gaussdb_test_utils.py b/test/unit_test/api/db/gaussdb_test_utils.py
new file mode 100644
index 0000000000..495ffec0c3
--- /dev/null
+++ b/test/unit_test/api/db/gaussdb_test_utils.py
@@ -0,0 +1,64 @@
+import importlib
+import sys
+import types
+
+
+class DummyConnection:
+ def __init__(self, *args, **kwargs):
+ pass
+
+
+def stub_module(name, **attrs):
+ module = types.ModuleType(name)
+ for attr_name, attr_value in attrs.items():
+ setattr(module, attr_name, attr_value)
+ sys.modules[name] = module
+ return module
+
+
+def stub_settings_import_dependencies():
+ memory_utils = importlib.import_module("memory.utils")
+ from rag.nlp import is_english
+ from rag.nlp import rag_tokenizer
+
+ search = stub_module("rag.nlp.search")
+ rag_nlp = stub_module("rag.nlp", search=search, rag_tokenizer=rag_tokenizer, is_english=is_english)
+ rag_nlp.__path__ = []
+ sys.modules["rag.nlp"] = rag_nlp
+ sys.modules["rag.nlp.rag_tokenizer"] = rag_tokenizer
+
+ redis_conn = stub_module(
+ "rag.utils.redis_conn",
+ REDIS_CONN=types.SimpleNamespace(get_or_create_secret_key=lambda key, value: value, health=lambda: True),
+ )
+ sys.modules["rag.utils.redis_conn"] = redis_conn
+
+ for module_name in [
+ "rag.utils.es_conn",
+ "rag.utils.infinity_conn",
+ "rag.utils.ob_conn",
+ "rag.utils.opensearch_conn",
+ "memory.utils.es_conn",
+ "memory.utils.infinity_conn",
+ "memory.utils.ob_conn",
+ ]:
+ module = stub_module(
+ module_name,
+ ESConnection=DummyConnection,
+ InfinityConnection=DummyConnection,
+ OBConnection=DummyConnection,
+ OSConnection=DummyConnection,
+ )
+ if module_name.startswith("memory.utils."):
+ setattr(memory_utils, module_name.rsplit(".", 1)[-1], module)
+
+ for module_name, class_name in [
+ ("rag.utils.azure_sas_conn", "RAGFlowAzureSasBlob"),
+ ("rag.utils.azure_spn_conn", "RAGFlowAzureSpnBlob"),
+ ("rag.utils.gcs_conn", "RAGFlowGCS"),
+ ("rag.utils.minio_conn", "RAGFlowMinio"),
+ ("rag.utils.opendal_conn", "OpenDALStorage"),
+ ("rag.utils.s3_conn", "RAGFlowS3"),
+ ("rag.utils.oss_conn", "RAGFlowOSS"),
+ ]:
+ stub_module(module_name, **{class_name: DummyConnection})
diff --git a/test/unit_test/api/db/services/test_gaussdb_dialog_sql.py b/test/unit_test/api/db/services/test_gaussdb_dialog_sql.py
new file mode 100644
index 0000000000..665d3e04c2
--- /dev/null
+++ b/test/unit_test/api/db/services/test_gaussdb_dialog_sql.py
@@ -0,0 +1,1866 @@
+#
+# Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+import asyncio
+import importlib
+import logging
+import sys
+import types
+import warnings
+from unittest.mock import AsyncMock, Mock
+
+import pytest
+
+
+GAUSSDB_SQL_RULE_MARKERS = (
+ "Use only the static JSONB path literals listed under Fields.",
+ "Do not construct dynamic JSONB paths.",
+ "A/ORA treats '' as NULL; never compare with = '' or <> ''; use IS NULL or IS NOT NULL.",
+ "Return exactly one read-only SELECT statement and no other text.",
+ "Do not use LATERAL, JOIN, UNION, INTERSECT, EXCEPT, or OR.",
+ "Do not expand JSON with jsonb_each, jsonb_each_text, jsonb_to_record, jsonb_to_recordset, jsonb_array_elements, or jsonb_array_elements_text.",
+ "Do not use DML or DDL, including INSERT, UPDATE, DELETE, MERGE, CREATE, ALTER, DROP, TRUNCATE, GRANT, REVOKE, CALL, COPY, or DO.",
+)
+
+
+def _assert_gaussdb_sql_rule_markers(prompt):
+ for marker in GAUSSDB_SQL_RULE_MARKERS:
+ assert marker in prompt
+
+
+warnings.filterwarnings(
+ "ignore",
+ message="pkg_resources is deprecated as an API.*",
+ category=UserWarning,
+)
+
+
+def _install_cv2_stub_if_unavailable():
+ try:
+ import cv2 # noqa: F401
+
+ return
+ except Exception:
+ pass
+
+ stub = types.ModuleType("cv2")
+ stub.INTER_LINEAR = 1
+ stub.INTER_CUBIC = 2
+ stub.BORDER_CONSTANT = 0
+ stub.BORDER_REPLICATE = 1
+ stub.COLOR_BGR2RGB = 0
+ stub.COLOR_BGR2GRAY = 1
+ stub.COLOR_GRAY2BGR = 2
+ stub.IMREAD_IGNORE_ORIENTATION = 128
+ stub.IMREAD_COLOR = 1
+ stub.RETR_LIST = 1
+ stub.CHAIN_APPROX_SIMPLE = 2
+
+ def _module_getattr(name):
+ if name.isupper():
+ return 0
+ raise RuntimeError(f"cv2.{name} is unavailable in this test environment")
+
+ stub.__getattr__ = _module_getattr
+ sys.modules["cv2"] = stub
+
+
+def _install_settings_import_stubs(monkeypatch):
+ class StubDocEngineConnection:
+ def db_type(self):
+ return "stub"
+
+ class StubStorage:
+ def health(self):
+ return True
+
+ def install_module(name, **attrs):
+ module = types.ModuleType(name)
+ for key, value in attrs.items():
+ setattr(module, key, value)
+ monkeypatch.setitem(sys.modules, name, module)
+ return module
+
+ try:
+ import rag.utils
+ import memory.utils
+ except Exception:
+ return
+
+ rag_modules = {
+ "es_conn": {"ESConnection": StubDocEngineConnection},
+ "infinity_conn": {"InfinityConnection": StubDocEngineConnection},
+ "ob_conn": {"OBConnection": StubDocEngineConnection},
+ "opensearch_conn": {"OSConnection": StubDocEngineConnection},
+ "azure_sas_conn": {"RAGFlowAzureSasBlob": StubStorage},
+ "azure_spn_conn": {"RAGFlowAzureSpnBlob": StubStorage},
+ "gcs_conn": {"RAGFlowGCS": StubStorage},
+ "minio_conn": {"RAGFlowMinio": StubStorage},
+ "opendal_conn": {"OpenDALStorage": StubStorage},
+ "redis_conn": {"REDIS_CONN": types.SimpleNamespace(health=lambda: True, is_alive=lambda: False, REDIS=None)},
+ "s3_conn": {"RAGFlowS3": StubStorage},
+ "oss_conn": {"RAGFlowOSS": StubStorage},
+ }
+ for short_name, attrs in rag_modules.items():
+ module = install_module(f"rag.utils.{short_name}", **attrs)
+ monkeypatch.setattr(rag.utils, short_name, module, raising=False)
+
+ for short_name in ("es_conn", "infinity_conn", "ob_conn"):
+ module = install_module(
+ f"memory.utils.{short_name}",
+ ESConnection=StubDocEngineConnection,
+ InfinityConnection=StubDocEngineConnection,
+ OBConnection=StubDocEngineConnection,
+ )
+ monkeypatch.setattr(memory.utils, short_name, module, raising=False)
+
+ if "json_repair" not in sys.modules:
+ import json
+
+ install_module("json_repair", loads=json.loads)
+ install_module("roman_numbers")
+ install_module("word2number", w2n=types.SimpleNamespace(word_to_num=lambda value: 0))
+ install_module("cn2an", cn2an=lambda value, *_args, **_kwargs: 0)
+ if "langfuse" not in sys.modules:
+
+ class StubLangfuse:
+ pass
+
+ def propagate_attributes(**_kwargs):
+ class Context:
+ def __enter__(self):
+ return self
+
+ def __exit__(self, *_args):
+ return False
+
+ return Context()
+
+ install_module("langfuse", Langfuse=StubLangfuse, propagate_attributes=propagate_attributes)
+ if "peewee" not in sys.modules:
+ install_module("peewee", fn=types.SimpleNamespace())
+ if "mcp.client.session" not in sys.modules:
+ install_module("mcp")
+ install_module("mcp.client")
+
+ class ClientSession:
+ pass
+
+ async def _client(*_args, **_kwargs):
+ raise RuntimeError("mcp client is unavailable in this test")
+
+ class _MCPType:
+ pass
+
+ install_module("mcp.client.session", ClientSession=ClientSession)
+ install_module("mcp.client.sse", sse_client=_client)
+ install_module("mcp.client.streamable_http", streamablehttp_client=_client)
+ install_module(
+ "mcp.types",
+ CallToolResult=_MCPType,
+ ListToolsResult=_MCPType,
+ TextContent=_MCPType,
+ Tool=_MCPType,
+ )
+ if "beartype" not in sys.modules:
+
+ def beartype(obj=None, **_kwargs):
+ if obj is None:
+ return lambda wrapped: wrapped
+ return obj
+
+ install_module("beartype", beartype=beartype)
+ install_module("beartype.claw", beartype_this_package=lambda *_args, **_kwargs: None)
+
+ class _Dummy:
+ def __init__(self, *_args, **_kwargs):
+ pass
+
+ class _Context:
+ def __enter__(self):
+ return self
+
+ def __exit__(self, *_args):
+ return False
+
+ def __call__(self, func):
+ return func
+
+ class _DummyDB:
+ @staticmethod
+ def connection_context():
+ return _Context()
+
+ @staticmethod
+ def atomic():
+ return _Context()
+
+ install_module("api.db.services.user_service", UserService=_Dummy)
+ install_module("api.db.services.file_service", FileService=_Dummy)
+ install_module("api.db.services.common_service", CommonService=_Dummy)
+ install_module("api.db.services.doc_metadata_service", DocMetadataService=_Dummy)
+ install_module(
+ "api.db.services.knowledgebase_service",
+ KnowledgebaseService=types.SimpleNamespace(get_field_map=lambda _kb_ids: {}),
+ validate_dataset_embedding_models=lambda _kbs: None,
+ )
+ install_module("api.db.services.langfuse_service", TenantLangfuseService=_Dummy)
+ install_module("api.db.services.llm_service", LLMBundle=_Dummy)
+ install_module(
+ "api.db.joint_services.tenant_model_service",
+ get_tenant_default_model_by_type=lambda *_args, **_kwargs: {"model_type": "chat", "max_tokens": 8192},
+ get_model_config_from_provider_instance=lambda *_args, **_kwargs: {},
+ get_model_type_by_name=lambda *_args, **_kwargs: [],
+ resolve_model_config=lambda *_args, **_kwargs: ({"model_type": "chat"}, None),
+ resolve_model_type=lambda *_args, **_kwargs: None,
+ get_model_config_by_id=lambda *_args, **_kwargs: {},
+ )
+ install_module("api.db.db_models", DB=_DummyDB, Dialog=_Dummy)
+ install_module("common.metadata_utils", apply_meta_data_filter=lambda *_args, **_kwargs: None)
+ install_module(
+ "api.utils.reference_metadata_utils",
+ enrich_chunks_with_document_metadata=lambda chunks, *_args, **_kwargs: chunks,
+ resolve_reference_metadata_preferences=lambda *_args, **_kwargs: {},
+ )
+ install_module("rag.graphrag.general.mind_map_extractor", MindMapExtractor=_Dummy)
+ advanced_rag = install_module("rag.advanced_rag", DeepResearcher=_Dummy)
+ advanced_rag.__path__ = []
+ install_module("rag.advanced_rag.agentic_rag", RAGTools=_Dummy)
+ install_module("rag.advanced_rag.knowlege_compile")
+ install_module("rag.advanced_rag.knowlege_compile.mind_map_extractor", MindMapExtractor=_Dummy)
+ install_module("rag.app.tag", label_question=lambda *_args, **_kwargs: None)
+ install_module("rag.nlp.search", index_name=lambda tenant_id: f"ragflow_{tenant_id}")
+ install_module(
+ "rag.prompts.generator",
+ chunks_format=lambda *_args, **_kwargs: "",
+ citation_prompt=lambda: "",
+ cross_languages=lambda *_args, **_kwargs: "",
+ full_question=lambda *_args, **_kwargs: "",
+ kb_prompt=lambda *_args, **_kwargs: [],
+ keyword_extraction=lambda *_args, **_kwargs: "",
+ message_fit_in=lambda *_args, **_kwargs: (0, []),
+ PROMPT_JINJA_ENV=types.SimpleNamespace(from_string=lambda *_args, **_kwargs: types.SimpleNamespace(render=lambda **_kw: "")),
+ ASK_SUMMARY="",
+ )
+ install_module("common.token_utils", num_tokens_from_string=lambda *_args, **_kwargs: 0)
+ install_module("rag.utils.tavily_conn", Tavily=_Dummy)
+ install_module("rag.utils.tts_cache", synthesize_with_cache=lambda *_args, **_kwargs: None)
+
+
+_install_cv2_stub_if_unavailable()
+
+
+class FakeChatModel:
+ def __init__(self, responses):
+ self.responses = list(responses)
+ self.calls = []
+
+ async def async_chat(self, sys_prompt, messages, params):
+ self.calls.append((sys_prompt, messages, params))
+ if not self.responses:
+ raise AssertionError("async_chat called more times than expected")
+ return self.responses.pop(0)
+
+
+class RecordingRetriever:
+ def __init__(self, fail_first=False, missing_source_once=False):
+ self.sqls = []
+ self.fail_first = fail_first
+ self.missing_source_once = missing_source_once
+
+ def sql_retrieval(self, sql, format="json"):
+ assert format == "json"
+ self.sqls.append(sql)
+ if self.fail_first and len(self.sqls) == 1:
+ raise RuntimeError("json_extract_string does not exist")
+ missing_source_call = 2 if self.fail_first else 1
+ if self.missing_source_once and len(self.sqls) == missing_source_call:
+ return {"columns": [{"name": "amount"}], "rows": [["120"]]}
+ return {
+ "columns": [{"name": "doc_id"}, {"name": "docnm_kwd"}, {"name": "kb_id"}, {"name": "amount"}],
+ "rows": [["doc1", "finance.csv", "abcdefabcdefabcdefabcdefabcdefab", "120"]],
+ }
+
+
+class AggregateRecordingRetriever:
+ def __init__(self):
+ self.sqls = []
+
+ def sql_retrieval(self, sql, format="json"):
+ assert format == "json"
+ self.sqls.append(sql)
+ if len(self.sqls) == 1:
+ return {"columns": [{"name": "total_amount"}], "rows": [["120"]]}
+ return {
+ "columns": [{"name": "doc_id"}, {"name": "docnm_kwd"}, {"name": "kb_id"}],
+ "rows": [["doc1", "finance.csv", "abcdefabcdefabcdefabcdefabcdefab"]],
+ }
+
+
+class MultiKBReferenceRetriever:
+ def __init__(self):
+ self.sqls = []
+
+ def sql_retrieval(self, sql, format="json"):
+ assert format == "json"
+ self.sqls.append(sql)
+ if len(self.sqls) == 1:
+ return {
+ "columns": [{"name": "doc_id"}, {"name": "docnm_kwd"}, {"name": "amount"}],
+ "rows": [
+ ["doc1", "finance-a.csv", "120"],
+ ["doc2", "finance-b.csv", "80"],
+ ["doc3", "finance-c.csv", "40"],
+ ],
+ }
+ return {
+ "columns": [{"name": "doc_id"}, {"name": "kb_id"}],
+ "rows": [
+ ["doc1", "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"],
+ ["doc2", "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"],
+ ["doc3", "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"],
+ ],
+ }
+
+
+class MultiKBReferenceWithInlineKB:
+ def __init__(self):
+ self.sqls = []
+
+ def sql_retrieval(self, sql, format="json"):
+ assert format == "json"
+ self.sqls.append(sql)
+ return {
+ "columns": [{"name": "doc_id"}, {"name": "docnm_kwd"}, {"name": "kb_id"}],
+ "rows": [["doc1", "finance.csv", "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"]],
+ }
+
+
+class ScriptedRetriever:
+ def __init__(self, sql_results=(), retrieval_result=None):
+ self.sql_results = list(sql_results)
+ self.sqls = []
+ self.retrieval = AsyncMock(
+ return_value=retrieval_result
+ or {
+ "total": 1,
+ "chunks": [
+ {
+ "id": "chunk-1",
+ "doc_id": "d1",
+ "docnm_kwd": "doc.xlsx",
+ "kb_id": "abcdefabcdefabcdefabcdefabcdefab",
+ "content_with_weight": "fallback content",
+ "content_ltks": "fallback content",
+ "vector": [],
+ }
+ ],
+ "doc_aggs": [{"doc_id": "d1", "doc_name": "doc.xlsx", "count": 1}],
+ }
+ )
+
+ def sql_retrieval(self, sql, format="json"):
+ assert format == "json"
+ self.sqls.append(sql)
+ if not self.sql_results:
+ raise AssertionError("sql_retrieval called more times than expected")
+ result = self.sql_results.pop(0)
+ if isinstance(result, BaseException):
+ raise result
+ return result
+
+ @staticmethod
+ def retrieval_by_children(chunks, _tenant_ids):
+ return chunks
+
+ @staticmethod
+ def insert_citations(answer, *_args, **_kwargs):
+ return answer, set()
+
+
+def dialog_service(monkeypatch):
+ _install_settings_import_stubs(monkeypatch)
+ module = importlib.import_module("api.db.services.dialog_service")
+ try:
+ yield module
+ finally:
+ sys.modules.pop("api.db.services.dialog_service", None)
+ package = sys.modules.get("api.db.services")
+ if package is not None and hasattr(package, "dialog_service"):
+ delattr(package, "dialog_service")
+
+
+def enable_gaussdb_docengine(monkeypatch, dialog_service):
+ monkeypatch.setattr(dialog_service.settings, "DOC_ENGINE_INFINITY", False, raising=False)
+ monkeypatch.setattr(dialog_service.settings, "DOC_ENGINE_OCEANBASE", False, raising=False)
+ monkeypatch.setattr(dialog_service.settings, "DOC_ENGINE_GAUSSDB", True, raising=False)
+
+
+def test_tc_sql_601_gaussdb_prompt_uses_jsonb_path_not_oceanbase_json_helpers(dialog_service, enable_gaussdb_docengine):
+ prompt = dialog_service.gaussdb_text_to_sql.build_sql_prompt(
+ table_name="ragflow_0123456789abcdef0123456789abcdef",
+ field_map={"amount": "number", "status": "string"},
+ question="amount greater than 100",
+ )
+
+ assert "JSONB text extraction: chunk_data #>> '{FieldName}'" in prompt
+ assert "JSONB value extraction: chunk_data #> '{FieldName}'" in prompt
+ assert "Numeric cast: CAST(chunk_data #>> '{FieldName}' AS DOUBLE PRECISION)" in prompt
+ assert "Date cast: to_date(chunk_data #>> '{FieldName}', 'YYYY-MM-DD')" in prompt
+ assert "NULL check: (chunk_data #>> '{FieldName}') IS NOT NULL" in prompt
+ assert "#>> '{amount}'" in prompt
+ assert " - status (string): chunk_data #>> '{status}'" in prompt
+ assert "json_extract_string(" not in prompt
+ assert "Select doc_id and docnm_kwd for non-aggregate data queries." in prompt
+ assert "Do not add kb_id" in prompt
+ assert "Do not use json_extract, json_extract_string, or json_extract_isnull" in prompt
+ assert "Restrict the query by kb_id" not in prompt
+ _assert_gaussdb_sql_rule_markers(prompt)
+
+
+def test_tc_sql_506_sql_converts_row_values_after_runtime_validation(dialog_service):
+ from rag.utils.gaussdb_conn import GaussDBConnection
+
+ conn = object.__new__(GaussDBConnection)
+ conn.schema = "public"
+ conn.logger = logging.getLogger("test.gaussdb")
+ conn._fetch_all_with_description = Mock(
+ return_value=(
+ [(b"abc", {"a": 1}, [1, "x"], None)],
+ [("bytes_col",), ("object_col",), ("list_col",), ("null_col",)],
+ )
+ )
+ sql = (
+ "SELECT doc_id AS bytes_col, chunk_data #> '{object_col}' AS object_col, "
+ "chunk_data #> '{list_col}' AS list_col, chunk_data #>> '{null_col}' AS null_col "
+ "FROM ragflow_tc_sql_506 WHERE kb_id = 'kb-506'"
+ )
+
+ result = conn.sql(sql, format="json")
+
+ executed_sql, params = conn._fetch_all_with_description.call_args.args
+ assert executed_sql == (
+ "SELECT doc_id AS bytes_col, chunk_data #> '{object_col}' AS object_col, "
+ "chunk_data #> '{list_col}' AS list_col, chunk_data #>> '{null_col}' AS null_col "
+ "FROM \"public\".ragflow_tc_sql_506 WHERE kb_id = 'kb-506' LIMIT 128"
+ )
+ assert params == []
+ assert conn._fetch_all_with_description.call_args.kwargs == {
+ "statement_timeout_ms": 30000,
+ }
+ assert result == {
+ "columns": [
+ {"name": "bytes_col", "type": "text"},
+ {"name": "object_col", "type": "text"},
+ {"name": "list_col", "type": "text"},
+ {"name": "null_col", "type": "text"},
+ ],
+ "rows": [["abc", '{"a": 1}', '[1, "x"]', None]],
+ }
+
+
+def test_tc_sql_609_gaussdb_prompt_uses_field_map_value_as_jsonb_path(dialog_service, enable_gaussdb_docengine):
+ prompt = dialog_service.gaussdb_text_to_sql.build_sql_prompt(
+ table_name="ragflow_0123456789abcdef0123456789abcdef",
+ field_map={"customer_alias": "customer_name", "amount": "number"},
+ question="show customer amount",
+ )
+
+ assert " - customer_alias (customer_name): chunk_data #>> '{customer_name}'" in prompt
+ assert " - amount (number): chunk_data #>> '{amount}'" in prompt
+ _assert_gaussdb_sql_rule_markers(prompt)
+
+
+def test_tc_sql_410_empty_value_prompt_rules_and_validator_rejects_empty_string_comparison(
+ dialog_service,
+ enable_gaussdb_docengine,
+):
+ prompt = dialog_service.gaussdb_text_to_sql.build_sql_prompt(
+ "ragflow_t1",
+ {"status": "string"},
+ "show rows where status is empty",
+ )
+ validator = dialog_service.gaussdb_text_to_sql.build_validator("ragflow_t1", ["kb-1"], {"status": "string"})
+
+ with pytest.raises(ValueError, match="JSONB text cannot be compared with an empty SQL string"):
+ validator.validate_and_patch("SELECT doc_id FROM ragflow_t1 WHERE chunk_data #>> '{status}' = ''")
+
+ assert "NULL check: (chunk_data #>> '{FieldName}') IS NOT NULL" in prompt
+ assert "Do not use json_extract, json_extract_string, or json_extract_isnull; use #>> / #> instead" in prompt
+ _assert_gaussdb_sql_rule_markers(prompt)
+
+
+def test_tc_sql_001_async_chat_routes_table_field_map_to_gaussdb_use_sql(monkeypatch, dialog_service, enable_gaussdb_docengine):
+ tenant_id = "0123456789abcdef0123456789abcdef"
+ kb_id = "abcdefabcdefabcdefabcdefabcdefab"
+ sql_result = {
+ "answer": "|amount|\n|---|\n|100|",
+ "reference": {"chunks": [], "doc_aggs": []},
+ "prompt": "gaussdb prompt",
+ }
+ get_field_map = Mock(return_value={"amount": "number"})
+ use_sql = AsyncMock(return_value=sql_result)
+ retrieval = AsyncMock()
+
+ chat = object()
+ monkeypatch.setattr(
+ dialog_service.settings,
+ "retriever",
+ types.SimpleNamespace(retrieval=retrieval),
+ raising=False,
+ )
+ monkeypatch.setattr(
+ dialog_service,
+ "KnowledgebaseService",
+ types.SimpleNamespace(get_field_map=get_field_map),
+ raising=False,
+ )
+ monkeypatch.setattr(dialog_service, "use_sql", use_sql)
+ monkeypatch.setattr(dialog_service, "get_models", lambda *_args, **_kwargs: ([], None, None, chat, None))
+ monkeypatch.setattr(dialog_service, "_resolve_reference_metadata", lambda *_args, **_kwargs: (False, []))
+ monkeypatch.setattr(
+ dialog_service,
+ "TenantLangfuseService",
+ types.SimpleNamespace(filter_by_tenant=lambda **_kwargs: None),
+ raising=False,
+ )
+ dialog = types.SimpleNamespace(
+ tenant_id=tenant_id,
+ kb_ids=[kb_id],
+ prompt_config={"quote": True, "parameters": [], "system": ""},
+ llm_id=None,
+ tenant_llm_id=None,
+ meta_data_filter=None,
+ )
+
+ async def collect():
+ results = []
+ async for item in dialog_service.async_chat(dialog, [{"role": "user", "content": "What is the total amount?"}], stream=False):
+ results.append(item)
+ return results
+
+ results = asyncio.run(collect())
+
+ get_field_map.assert_called_once_with([kb_id])
+ use_sql.assert_awaited_once_with(
+ "What is the total amount?",
+ {"amount": "number"},
+ tenant_id,
+ chat,
+ True,
+ [kb_id],
+ doc_ids=None,
+ )
+ retrieval.assert_not_awaited()
+ assert results == [sql_result]
+
+
+def _run_field_map_empty_chat(monkeypatch, dialog_service, chunks, doc_aggs):
+ tenant_id = "0123456789abcdef0123456789abcdef"
+ kb_id = "abcdefabcdefabcdefabcdefabcdefab"
+ get_field_map = Mock(return_value={})
+ use_sql = AsyncMock()
+ retrieval = AsyncMock(return_value={"total": len(chunks), "chunks": chunks, "doc_aggs": doc_aggs})
+ retriever = types.SimpleNamespace(
+ retrieval=retrieval,
+ retrieval_by_children=lambda value, _tenant_ids: value,
+ insert_citations=lambda answer, *_args, **_kwargs: (answer, set()),
+ )
+ chat = FakeChatModel(["fallback answer"])
+
+ def sync_get_field_map(kb_ids):
+ return get_field_map(kb_ids)
+
+ monkeypatch.setattr(dialog_service.settings, "retriever", retriever, raising=False)
+ monkeypatch.setattr(
+ dialog_service,
+ "KnowledgebaseService",
+ types.SimpleNamespace(get_field_map=sync_get_field_map),
+ raising=False,
+ )
+ monkeypatch.setattr(dialog_service, "use_sql", use_sql)
+ monkeypatch.setattr(
+ dialog_service,
+ "get_tenant_default_model_by_type",
+ lambda *_args, **_kwargs: {"model_type": "chat", "max_tokens": 8192},
+ )
+ monkeypatch.setattr(
+ dialog_service,
+ "get_models",
+ lambda *_args, **_kwargs: ([types.SimpleNamespace(tenant_id=tenant_id)], object(), None, chat, None),
+ )
+ monkeypatch.setattr(dialog_service, "kb_prompt", lambda *_args, **_kwargs: ["content"])
+ monkeypatch.setattr(dialog_service, "message_fit_in", lambda messages, _limit: (0, messages))
+ monkeypatch.setattr(dialog_service, "tts", lambda *_args, **_kwargs: None)
+ monkeypatch.setattr(dialog_service, "_resolve_reference_metadata", lambda *_args, **_kwargs: (False, []))
+ monkeypatch.setattr(
+ dialog_service,
+ "TenantLangfuseService",
+ types.SimpleNamespace(filter_by_tenant=lambda **_kwargs: None),
+ raising=False,
+ )
+ dialog = types.SimpleNamespace(
+ tenant_id=tenant_id,
+ kb_ids=[kb_id],
+ prompt_config={
+ "quote": True,
+ "parameters": [{"key": "knowledge", "optional": False}],
+ "system": "Use {knowledge}",
+ },
+ llm_id=None,
+ tenant_llm_id=None,
+ top_n=8,
+ top_k=32,
+ similarity_threshold=0.2,
+ vector_similarity_weight=0.3,
+ meta_data_filter=None,
+ llm_setting={},
+ )
+
+ async def collect():
+ return [
+ item
+ async for item in dialog_service.async_chat(
+ dialog,
+ [{"role": "user", "content": "What is the content?"}],
+ stream=False,
+ )
+ ]
+
+ return asyncio.run(collect()), get_field_map, use_sql, retrieval, kb_id
+
+
+_USE_REAL_SQL = object()
+
+
+def _run_configured_chat(
+ monkeypatch,
+ dialog_service,
+ *,
+ retriever,
+ chat,
+ field_map=None,
+ question="What is the total amount?",
+ use_sql_mock=_USE_REAL_SQL,
+ embd_mdl=None,
+):
+ tenant_id = "0123456789abcdef0123456789abcdef"
+ kb_id = "abcdefabcdefabcdefabcdefabcdefab"
+ if embd_mdl is None:
+ embd_mdl = object()
+ get_field_map = Mock(return_value=field_map or {"amount": "number"})
+ monkeypatch.setattr(dialog_service.settings, "retriever", retriever, raising=False)
+ monkeypatch.setattr(
+ dialog_service,
+ "KnowledgebaseService",
+ types.SimpleNamespace(get_field_map=get_field_map),
+ raising=False,
+ )
+ if use_sql_mock is not _USE_REAL_SQL:
+ monkeypatch.setattr(dialog_service, "use_sql", use_sql_mock)
+ monkeypatch.setattr(
+ dialog_service,
+ "get_tenant_default_model_by_type",
+ lambda *_args, **_kwargs: {"model_type": "chat", "max_tokens": 8192},
+ )
+ monkeypatch.setattr(
+ dialog_service,
+ "get_models",
+ lambda *_args, **_kwargs: ([types.SimpleNamespace(tenant_id=tenant_id)], embd_mdl, None, chat, None),
+ )
+ monkeypatch.setattr(dialog_service, "kb_prompt", lambda *_args, **_kwargs: ["fallback content"])
+ monkeypatch.setattr(dialog_service, "message_fit_in", lambda messages, _limit: (0, messages))
+ monkeypatch.setattr(dialog_service, "tts", lambda *_args, **_kwargs: None)
+ monkeypatch.setattr(dialog_service, "_resolve_reference_metadata", lambda *_args, **_kwargs: (False, []))
+ monkeypatch.setattr(
+ dialog_service,
+ "TenantLangfuseService",
+ types.SimpleNamespace(filter_by_tenant=lambda **_kwargs: None),
+ raising=False,
+ )
+ dialog = types.SimpleNamespace(
+ tenant_id=tenant_id,
+ kb_ids=[kb_id],
+ prompt_config={
+ "quote": True,
+ "parameters": [{"key": "knowledge", "optional": False}],
+ "system": "Use {knowledge}",
+ },
+ llm_id=None,
+ tenant_llm_id=None,
+ top_n=8,
+ top_k=32,
+ similarity_threshold=0.2,
+ vector_similarity_weight=0.3,
+ meta_data_filter=None,
+ llm_setting={},
+ )
+
+ async def collect():
+ return [
+ item
+ async for item in dialog_service.async_chat(
+ dialog,
+ [{"role": "user", "content": question}],
+ stream=False,
+ )
+ ]
+
+ return asyncio.run(collect()), get_field_map, kb_id
+
+
+def _expected_fallback_reference(kb_id):
+ return {
+ "total": 1,
+ "chunks": [
+ {
+ "id": "chunk-1",
+ "doc_id": "d1",
+ "docnm_kwd": "doc.xlsx",
+ "kb_id": kb_id,
+ "content_with_weight": "fallback content",
+ "content_ltks": "fallback content",
+ "vector": [],
+ }
+ ],
+ "doc_aggs": [{"doc_id": "d1", "doc_name": "doc.xlsx", "count": 1}],
+ }
+
+
+def test_tc_sql_002_field_map_empty_uses_normal_retrieval(
+ monkeypatch,
+ dialog_service,
+ enable_gaussdb_docengine,
+ caplog,
+):
+ chunks = [
+ {
+ "id": "chunk-1",
+ "doc_id": "doc-1",
+ "docnm_kwd": "doc.xlsx",
+ "kb_id": "abcdefabcdefabcdefabcdefabcdefab",
+ "content_with_weight": "content",
+ "content_ltks": "content",
+ "vector": [],
+ }
+ ]
+ doc_aggs = [{"doc_id": "doc-1", "doc_name": "doc.xlsx", "count": 1}]
+
+ with caplog.at_level(logging.DEBUG):
+ results, get_field_map, use_sql, retrieval, kb_id = _run_field_map_empty_chat(
+ monkeypatch,
+ dialog_service,
+ chunks,
+ doc_aggs,
+ )
+
+ get_field_map.assert_called_once_with([kb_id])
+ use_sql.assert_not_awaited()
+ retrieval.assert_awaited_once()
+ assert retrieval.await_args.args[3] == [kb_id]
+ assert results[-1]["answer"] == "fallback answer"
+ assert results[-1]["reference"]["chunks"] == chunks
+ assert results[-1]["reference"]["doc_aggs"] == doc_aggs
+ assert "Use SQL to retrieval" not in caplog.text
+
+
+def test_tc_sql_004_use_sql_non_gaussdb_engine_does_not_call_gaussdb_prompt(monkeypatch, dialog_service):
+ tenant_id = "0123456789abcdef0123456789abcdef"
+ kb_id = "abcdefabcdefabcdefabcdefabcdefab"
+ retriever = RecordingRetriever()
+
+ monkeypatch.setattr(dialog_service.settings, "DOC_ENGINE_INFINITY", False, raising=False)
+ monkeypatch.setattr(dialog_service.settings, "DOC_ENGINE_OCEANBASE", False, raising=False)
+ monkeypatch.setattr(dialog_service.settings, "DOC_ENGINE_GAUSSDB", False, raising=False)
+ monkeypatch.setattr(dialog_service.settings, "retriever", retriever, raising=False)
+ gaussdb_prompt = Mock(side_effect=AssertionError("GaussDB prompt must not be used"))
+ gaussdb_validator = Mock(side_effect=AssertionError("GaussDB validator must not be constructed"))
+ monkeypatch.setattr(dialog_service.gaussdb_text_to_sql, "build_sql_prompt", gaussdb_prompt)
+ monkeypatch.setattr(dialog_service.gaussdb_text_to_sql, "build_validator", gaussdb_validator)
+ generated_sql = f"SELECT doc_id, docnm_kwd FROM ragflow_{tenant_id}"
+ chat = FakeChatModel([generated_sql])
+
+ result = asyncio.run(
+ dialog_service.use_sql(
+ "show docs",
+ {"amount": "number"},
+ tenant_id,
+ chat,
+ quota=False,
+ kb_ids=[kb_id],
+ )
+ )
+
+ expected_sys_prompt = """You are a Database Administrator. Write SQL queries.
+
+RULES:
+1. Use EXACT field names from the schema below (e.g., product_tks, not product)
+2. Quote field names starting with digit: "123_field"
+3. Add IS NOT NULL in WHERE clause when:
+ - Question asks to "show me" or "display" specific columns
+4. Include doc_id/docnm in non-aggregate statement
+5. Output ONLY the SQL, no explanations"""
+ expected_user_prompt = f"""Table: ragflow_{tenant_id}
+Available fields:
+ - amount (number)
+Question: show docs
+Write SQL using exact field names above. Include doc_id, docnm_kwd for data queries. Only SQL."""
+ expected_sql = f"{generated_sql} WHERE kb_id = '{kb_id}'"
+
+ gaussdb_prompt.assert_not_called()
+ gaussdb_validator.assert_not_called()
+ assert chat.calls == [
+ (
+ expected_sys_prompt,
+ [{"role": "user", "content": expected_user_prompt}],
+ {"temperature": 0.06},
+ )
+ ]
+ assert retriever.sqls == [expected_sql]
+ assert result == {
+ "answer": "|number|Source|\n|------|------|\n|120| ##0$$|",
+ "reference": {
+ "chunks": [
+ {
+ "doc_id": "doc1",
+ "docnm_kwd": "finance.csv",
+ "kb_id": "abcdefabcdefabcdefabcdefabcdefab",
+ }
+ ],
+ "doc_aggs": [{"doc_id": "doc1", "doc_name": "finance.csv", "count": 1}],
+ },
+ "prompt": expected_sys_prompt,
+ }
+
+ monkeypatch.setattr(dialog_service.settings, "DOC_ENGINE_INFINITY", True, raising=False)
+ infinity_retriever = RecordingRetriever(fail_first=True, missing_source_once=True)
+ monkeypatch.setattr(dialog_service.settings, "retriever", infinity_retriever, raising=False)
+ infinity_table = f"ragflow_{tenant_id}_{kb_id}"
+ infinity_sql = f"SELECT doc_id, docnm FROM {infinity_table}"
+ infinity_chat = FakeChatModel([infinity_sql, infinity_sql, infinity_sql])
+
+ infinity_result = asyncio.run(
+ dialog_service.use_sql(
+ "show docs",
+ {"amount": "number"},
+ tenant_id,
+ infinity_chat,
+ quota=False,
+ kb_ids=[kb_id],
+ )
+ )
+
+ assert len(infinity_chat.calls) == 3
+ retry_prompt = infinity_chat.calls[1][1][0]["content"]
+ source_repair_prompt = infinity_chat.calls[2][1][0]["content"]
+ assert "json_extract_string(chunk_data, '$.field_name')" in retry_prompt
+ assert "Correct the SQL for GaussDB" not in retry_prompt
+ assert "missing required source columns" in source_repair_prompt
+ assert "include doc_id and docnm in the SELECT list" in source_repair_prompt
+ assert "json_extract_string(chunk_data, '$.field_name')" in source_repair_prompt
+ assert "chunk_data #>>" not in source_repair_prompt
+ assert infinity_retriever.sqls == [infinity_sql, infinity_sql, infinity_sql]
+ assert infinity_result["reference"]["chunks"] == [
+ {
+ "doc_id": "doc1",
+ "docnm_kwd": "finance.csv",
+ "kb_id": "abcdefabcdefabcdefabcdefabcdefab",
+ }
+ ]
+ gaussdb_prompt.assert_not_called()
+ gaussdb_validator.assert_not_called()
+
+
+def test_tc_sql_601_use_sql_routes_gaussdb_prompt_through_retriever(monkeypatch, dialog_service, enable_gaussdb_docengine):
+ tenant_id = "0123456789abcdef0123456789abcdef"
+ kb_id = "abcdefabcdefabcdefabcdefabcdefab"
+ table = f"ragflow_{tenant_id}"
+ retriever = RecordingRetriever()
+ monkeypatch.setattr(dialog_service.settings, "retriever", retriever, raising=False)
+ chat = FakeChatModel([f"SELECT doc_id, docnm_kwd, chunk_data #>> '{{amount}}' AS amount FROM {table} WHERE kb_id = '{kb_id}' AND (chunk_data #>> '{{amount}}')::DOUBLE PRECISION > 100"])
+
+ result = asyncio.run(
+ dialog_service.use_sql(
+ "finance amount greater than 100",
+ {"amount": "number", "status": "string"},
+ tenant_id,
+ chat,
+ quota=False,
+ kb_ids=[kb_id],
+ )
+ )
+
+ assert result is not None
+ assert "120" in result["answer"]
+ assert len(chat.calls) == 1
+ system_prompt, messages, params = chat.calls[0]
+ assert f"Table: {table}" in system_prompt
+ assert " - amount (number): chunk_data #>> '{amount}'" in system_prompt
+ assert " - status (string): chunk_data #>> '{status}'" in system_prompt
+ assert "json_extract_string(chunk_data" not in system_prompt
+ assert messages == [
+ {
+ "role": "user",
+ "content": (
+ f"Table: {table}\n"
+ "Fields:\n"
+ " - amount (number): chunk_data #>> '{amount}'\n"
+ " - status (string): chunk_data #>> '{status}'\n"
+ "Question: finance amount greater than 100\n"
+ "Write SQL using GaussDB JSONB #>> / #> syntax. "
+ "Include doc_id and docnm_kwd for data queries. Only SQL."
+ ),
+ }
+ ]
+ assert params == {"temperature": 0.06}
+ assert retriever.sqls == [
+ f"SELECT doc_id, docnm_kwd, chunk_data #>> '{{amount}}' AS amount FROM {table} WHERE kb_id = '{kb_id}' AND CAST((chunk_data #>> '{{amount}}') AS DOUBLE PRECISION) > 100 LIMIT 128"
+ ]
+
+
+def test_tc_sql_601_gaussdb_injects_doc_scope_before_validator(monkeypatch, dialog_service, enable_gaussdb_docengine):
+ tenant_id = "0123456789abcdef0123456789abcdef"
+ kb_id = "abcdefabcdefabcdefabcdefabcdefab"
+ doc_id = "12345678123456781234567812345678"
+ table = f"ragflow_{tenant_id}"
+ retriever = RecordingRetriever()
+ monkeypatch.setattr(dialog_service.settings, "retriever", retriever, raising=False)
+ chat = FakeChatModel([f"SELECT doc_id, docnm_kwd, chunk_data #>> '{{amount}}' AS amount FROM {table}"])
+
+ result = asyncio.run(
+ dialog_service.use_sql(
+ "show the scoped document amount",
+ {"amount": "number"},
+ tenant_id,
+ chat,
+ quota=False,
+ kb_ids=[kb_id],
+ doc_ids=[doc_id],
+ )
+ )
+
+ assert result is not None
+ assert retriever.sqls == [f"SELECT doc_id, docnm_kwd, chunk_data #>> '{{amount}}' AS amount FROM {table} WHERE doc_id = '{doc_id}' AND kb_id = '{kb_id}' LIMIT 128"]
+
+
+def test_tc_sql_602_row_count_override_skips_llm_and_executes_scoped_count(
+ monkeypatch,
+ dialog_service,
+ enable_gaussdb_docengine,
+):
+ tenant_id = "0123456789abcdef0123456789abcdef"
+ kb_id = "abcdefabcdefabcdefabcdefabcdefab"
+ table = f"ragflow_{tenant_id}"
+ retriever = ScriptedRetriever(
+ [
+ {"columns": [{"name": "rows"}], "rows": [[5]]},
+ {"columns": [{"name": "doc_id"}, {"name": "docnm_kwd"}], "rows": []},
+ ]
+ )
+ monkeypatch.setattr(dialog_service.settings, "retriever", retriever, raising=False)
+ chat = FakeChatModel([])
+
+ result = asyncio.run(
+ dialog_service.use_sql(
+ "How many rows in the dataset?",
+ {"amount": "number"},
+ tenant_id,
+ chat,
+ quota=False,
+ kb_ids=[kb_id],
+ )
+ )
+
+ assert chat.calls == []
+ assert retriever.sqls[0] == (f"SELECT COUNT(*) AS rows FROM {table} WHERE kb_id = '{kb_id}' LIMIT 128")
+ assert retriever.sqls[1] == (f"SELECT doc_id, docnm_kwd FROM {table} WHERE kb_id = '{kb_id}' LIMIT 128")
+ assert result["answer"] == "|rows|\n|------\n|5|"
+ assert result["reference"] == {"chunks": [], "doc_aggs": []}
+
+
+def test_tc_sql_603_validator_rejection_builds_retry_prompt(
+ monkeypatch,
+ dialog_service,
+ enable_gaussdb_docengine,
+ caplog,
+):
+ tenant_id = "0123456789abcdef0123456789abcdef"
+ kb_id = "abcdefabcdefabcdefabcdefabcdefab"
+ table = f"ragflow_{tenant_id}"
+ retriever = ScriptedRetriever(
+ [
+ {
+ "columns": [{"name": "doc_id"}, {"name": "docnm_kwd"}, {"name": "amount"}],
+ "rows": [["d1", "doc.xlsx", "120"]],
+ }
+ ]
+ )
+ monkeypatch.setattr(dialog_service.settings, "retriever", retriever, raising=False)
+ chat = FakeChatModel(
+ [
+ f"SELECT * FROM {table}",
+ f"SELECT doc_id, docnm_kwd, chunk_data #>> '{{amount}}' AS amount FROM {table}",
+ ]
+ )
+
+ with caplog.at_level(logging.WARNING):
+ result = asyncio.run(
+ dialog_service.use_sql(
+ "finance amount greater than 100",
+ {"amount": "number", "dept": "string"},
+ tenant_id,
+ chat,
+ quota=False,
+ kb_ids=[kb_id],
+ )
+ )
+
+ assert result["reference"]["chunks"] == [{"doc_id": "d1", "docnm_kwd": "doc.xlsx", "kb_id": kb_id}]
+ assert result["answer"] == "|number|Source|\n|------|------|\n|120| ##0$$|"
+ assert result["reference"]["doc_aggs"] == [{"doc_id": "d1", "doc_name": "doc.xlsx", "count": 1}]
+ assert len(chat.calls) == 2
+ assert len(retriever.sqls) == 1
+ assert f"kb_id = '{kb_id}'" in retriever.sqls[0]
+ retry_prompt = chat.calls[1][1][0]["content"]
+ assert "SELECT * is not allowed" in retry_prompt
+ assert "Initial SQL execution FAILED with error: SELECT * is not allowed" in caplog.text
+ assert "Do not use json_extract, json_extract_string, or json_extract_isnull" in retry_prompt
+ _assert_gaussdb_sql_rule_markers(retry_prompt)
+
+
+def test_tc_sql_604_execution_none_retries(
+ monkeypatch,
+ dialog_service,
+ enable_gaussdb_docengine,
+):
+ tenant_id = "0123456789abcdef0123456789abcdef"
+ kb_id = "abcdefabcdefabcdefabcdefabcdefab"
+ table = f"ragflow_{tenant_id}"
+ sql = f"SELECT doc_id, docnm_kwd, chunk_data #>> '{{amount}}' AS amount FROM {table}"
+ retriever = ScriptedRetriever(
+ [
+ None,
+ {
+ "columns": [{"name": "doc_id"}, {"name": "docnm_kwd"}, {"name": "amount"}],
+ "rows": [["d1", "doc.xlsx", "120"]],
+ },
+ ]
+ )
+ monkeypatch.setattr(dialog_service.settings, "retriever", retriever, raising=False)
+ chat = FakeChatModel([sql, sql])
+
+ result = asyncio.run(
+ dialog_service.use_sql(
+ "Show all amounts",
+ {"amount": "number"},
+ tenant_id,
+ chat,
+ quota=False,
+ kb_ids=[kb_id],
+ )
+ )
+
+ expected_sql = f"{sql} WHERE kb_id = '{kb_id}' LIMIT 128"
+ assert len(chat.calls) == 2
+ assert retriever.sqls == [expected_sql, expected_sql]
+ assert result["answer"] == "|number|Source|\n|------|------|\n|120| ##0$$|"
+ assert result["reference"] == {
+ "chunks": [{"doc_id": "d1", "docnm_kwd": "doc.xlsx", "kb_id": kb_id}],
+ "doc_aggs": [{"doc_id": "d1", "doc_name": "doc.xlsx", "count": 1}],
+ }
+
+
[email protected]("mode", ["validator_rejected", "execution_failed"])
+def test_tc_sql_605_retry_exhaustion_returns_none(
+ mode,
+ monkeypatch,
+ dialog_service,
+ enable_gaussdb_docengine,
+ caplog,
+):
+ tenant_id = "0123456789abcdef0123456789abcdef"
+ kb_id = "abcdefabcdefabcdefabcdefabcdefab"
+ table = f"ragflow_{tenant_id}"
+ if mode == "validator_rejected":
+ chat = FakeChatModel([f"SELECT * FROM {table}", f"SELECT * FROM {table}"])
+ retriever = ScriptedRetriever([])
+ expected_sql_calls = 0
+ else:
+ valid_sql = f"SELECT doc_id, docnm_kwd FROM {table}"
+ chat = FakeChatModel([valid_sql, valid_sql])
+ retriever = ScriptedRetriever([None, None])
+ expected_sql_calls = 2
+ monkeypatch.setattr(dialog_service.settings, "retriever", retriever, raising=False)
+
+ with caplog.at_level(logging.WARNING):
+ result = asyncio.run(
+ dialog_service.use_sql(
+ "Show all amounts",
+ {"amount": "number"},
+ tenant_id,
+ chat,
+ quota=False,
+ kb_ids=[kb_id],
+ )
+ )
+
+ assert result is None
+ assert len(chat.calls) == 2
+ assert len(retriever.sqls) == expected_sql_calls
+ if mode == "validator_rejected":
+ assert "SELECT * is not allowed" in caplog.text
+ assert "Retry SQL execution also FAILED" in caplog.text
+
+
+def test_tc_sql_606_retry_sql_is_revalidated_and_scoped(
+ monkeypatch,
+ dialog_service,
+ enable_gaussdb_docengine,
+):
+ tenant_id = "0123456789abcdef0123456789abcdef"
+ kb_id = "abcdefabcdefabcdefabcdefabcdefab"
+ table = f"ragflow_{tenant_id}"
+ retriever = ScriptedRetriever(
+ [
+ {
+ "columns": [{"name": "doc_id"}, {"name": "docnm_kwd"}, {"name": "amount"}],
+ "rows": [["d1", "doc.xlsx", "120"]],
+ }
+ ]
+ )
+ monkeypatch.setattr(dialog_service.settings, "retriever", retriever, raising=False)
+ chat = FakeChatModel(
+ [
+ f"SELECT doc_id, chunk_data FROM {table}",
+ f"SELECT doc_id, docnm_kwd, chunk_data #>> '{{amount}}' AS amount FROM {table}",
+ ]
+ )
+
+ result = asyncio.run(
+ dialog_service.use_sql(
+ "Show all amounts",
+ {"amount": "number"},
+ tenant_id,
+ chat,
+ quota=False,
+ kb_ids=[kb_id],
+ )
+ )
+
+ assert len(chat.calls) == 2
+ assert retriever.sqls == [f"SELECT doc_id, docnm_kwd, chunk_data #>> '{{amount}}' AS amount FROM {table} WHERE kb_id = '{kb_id}' LIMIT 128"]
+ assert result["reference"]["chunks"][0]["kb_id"] == kb_id
+
+
+def test_tc_sql_607_missing_source_columns_trigger_dedicated_repair(
+ monkeypatch,
+ dialog_service,
+ enable_gaussdb_docengine,
+):
+ tenant_id = "0123456789abcdef0123456789abcdef"
+ kb_id = "abcdefabcdefabcdefabcdefabcdefab"
+ table = f"ragflow_{tenant_id}"
+ retriever = ScriptedRetriever(
+ [
+ {"columns": [{"name": "amount"}], "rows": [["120"]]},
+ {
+ "columns": [{"name": "doc_id"}, {"name": "docnm_kwd"}, {"name": "amount"}],
+ "rows": [["d1", "doc.xlsx", "120"]],
+ },
+ ]
+ )
+ monkeypatch.setattr(dialog_service.settings, "retriever", retriever, raising=False)
+ chat = FakeChatModel(
+ [
+ f"SELECT chunk_data #>> '{{amount}}' AS amount FROM {table}",
+ f"SELECT doc_id, docnm_kwd, chunk_data #>> '{{amount}}' AS amount FROM {table}",
+ ]
+ )
+
+ result = asyncio.run(
+ dialog_service.use_sql(
+ "Show all amounts",
+ {"amount": "number"},
+ tenant_id,
+ chat,
+ quota=False,
+ kb_ids=[kb_id],
+ )
+ )
+
+ assert len(chat.calls) == 2
+ repair_prompt = chat.calls[1][1][0]["content"]
+ assert "missing required source columns" in repair_prompt
+ _assert_gaussdb_sql_rule_markers(repair_prompt)
+ assert retriever.sqls == [
+ f"SELECT chunk_data #>> '{{amount}}' AS amount FROM {table} WHERE kb_id = '{kb_id}' LIMIT 128",
+ f"SELECT doc_id, docnm_kwd, chunk_data #>> '{{amount}}' AS amount FROM {table} WHERE kb_id = '{kb_id}' LIMIT 128",
+ ]
+ assert result["answer"] == "|number|Source|\n|------|------|\n|120| ##0$$|"
+ assert result["reference"] == {
+ "chunks": [{"doc_id": "d1", "docnm_kwd": "doc.xlsx", "kb_id": kb_id}],
+ "doc_aggs": [{"doc_id": "d1", "doc_name": "doc.xlsx", "count": 1}],
+ }
+
+
+def test_tc_sql_608_source_repair_failure_returns_best_effort_answer(
+ monkeypatch,
+ dialog_service,
+ enable_gaussdb_docengine,
+ caplog,
+):
+ tenant_id = "0123456789abcdef0123456789abcdef"
+ kb_id = "abcdefabcdefabcdefabcdefabcdefab"
+ table = f"ragflow_{tenant_id}"
+ retriever = ScriptedRetriever(
+ [
+ {"columns": [{"name": "amount"}], "rows": [["120"]]},
+ RuntimeError("source lookup failed"),
+ ]
+ )
+ chat = FakeChatModel(
+ [
+ f"SELECT chunk_data #>> '{{amount}}' AS amount FROM {table}",
+ f"SELECT doc_id, docnm_kwd, chunk_data #>> '{{amount}}' AS amount FROM {table}",
+ ]
+ )
+
+ with caplog.at_level(logging.WARNING):
+ results, get_field_map, actual_kb_id = _run_configured_chat(
+ monkeypatch,
+ dialog_service,
+ retriever=retriever,
+ chat=chat,
+ question="Show all amounts",
+ )
+
+ assert actual_kb_id == kb_id
+ get_field_map.assert_called_once_with([kb_id])
+ assert len(chat.calls) == 2
+ assert retriever.sqls == [
+ f"SELECT chunk_data #>> '{{amount}}' AS amount FROM {table} WHERE kb_id = '{kb_id}' LIMIT 128",
+ f"SELECT doc_id, docnm_kwd, chunk_data #>> '{{amount}}' AS amount FROM {table} WHERE kb_id = '{kb_id}' LIMIT 128",
+ ]
+ retriever.retrieval.assert_not_awaited()
+ assert len(results) == 1
+ assert results[0]["answer"] == "|number|\n|------\n|120|"
+ assert results[0]["reference"] == {"chunks": [], "doc_aggs": []}
+ _assert_gaussdb_sql_rule_markers(results[0]["prompt"])
+ assert "Source-column SQL repair failed" in caplog.text
+ assert "source lookup failed" in caplog.text
+ assert "SQL failed or returned no results, falling back to vector search" not in caplog.text
+
+
+def test_tc_sql_701_use_sql_aggregate_fallback_fetches_sources_with_kb_scope(monkeypatch, dialog_service, enable_gaussdb_docengine):
+ tenant_id = "0123456789abcdef0123456789abcdef"
+ kb_id = "abcdefabcdefabcdefabcdefabcdefab"
+ table = f"ragflow_{tenant_id}"
+ retriever = AggregateRecordingRetriever()
+ monkeypatch.setattr(dialog_service.settings, "retriever", retriever, raising=False)
+ chat = FakeChatModel([f"SELECT SUM((chunk_data #>> '{{amount}}')::DOUBLE PRECISION) AS total_amount FROM {table} WHERE chunk_data #>> '{{dept}}' = 'finance' LIMIT 3"])
+
+ result = asyncio.run(
+ dialog_service.use_sql(
+ "sum finance amount",
+ {"amount": "number", "dept": "string"},
+ tenant_id,
+ chat,
+ quota=False,
+ kb_ids=[kb_id],
+ )
+ )
+
+ assert result is not None
+ assert len(retriever.sqls) == 2
+ assert retriever.sqls[0].endswith("LIMIT 3")
+ source_sql = retriever.sqls[1]
+ assert source_sql.startswith(f"SELECT doc_id, docnm_kwd FROM {table} WHERE ")
+ assert "chunk_data #>> '{dept}' = 'finance'" in source_sql
+ assert f"kb_id = '{kb_id}'" in source_sql
+ assert source_sql.endswith("LIMIT 128")
+ assert "LIMIT 20" not in source_sql
+ assert result["reference"]["chunks"][0]["kb_id"] == kb_id
+
+
+def test_tc_sql_701_aggregate_source_builder_projects_multikb_scope_and_rejects_having(dialog_service):
+ source_sql = dialog_service.gaussdb_text_to_sql.build_aggregate_source_sql(
+ "SELECT kb_id, SUM((chunk_data #>> '{amount}')::DOUBLE PRECISION) AS total FROM ragflow_tenant WHERE kb_id IN ('k1', 'k2') GROUP BY kb_id ORDER BY total DESC LIMIT 3 OFFSET 1",
+ "docnm_kwd",
+ include_kb_id=True,
+ )
+
+ assert source_sql == "SELECT doc_id, docnm_kwd, kb_id FROM ragflow_tenant WHERE kb_id IN ('k1', 'k2')"
+ reference = dialog_service.gaussdb_text_to_sql.build_source_reference(
+ {
+ "columns": [{"name": "doc_id"}, {"name": "docnm_kwd"}, {"name": "kb_id"}],
+ "rows": [["d1", "one.xlsx", "k1"], ["d2", "two.xlsx", "k2"]],
+ },
+ ["k1", "k2"],
+ )
+ assert reference == {
+ "chunks": [
+ {"doc_id": "d1", "docnm_kwd": "one.xlsx", "kb_id": "k1"},
+ {"doc_id": "d2", "docnm_kwd": "two.xlsx", "kb_id": "k2"},
+ ],
+ "doc_aggs": [
+ {"doc_id": "d1", "doc_name": "one.xlsx", "count": 1},
+ {"doc_id": "d2", "doc_name": "two.xlsx", "count": 1},
+ ],
+ }
+ with pytest.raises(ValueError) as exc_info:
+ dialog_service.gaussdb_text_to_sql.build_aggregate_source_sql(
+ "SELECT SUM((chunk_data #>> '{amount}')::DOUBLE PRECISION) AS total FROM ragflow_tenant GROUP BY kb_id HAVING SUM((chunk_data #>> '{amount}')::DOUBLE PRECISION) > 0",
+ "docnm_kwd",
+ include_kb_id=False,
+ )
+ assert str(exc_info.value) == "GaussDB aggregate source lookup cannot preserve HAVING safely"
+
+
+ ("projection", "value"),
+ [
+ ("'sum(amount)' AS note", "sum(amount)"),
+ ("1 AS note /* sum(amount) */", 1),
+ ],
+ ids=["literal", "comment"],
+)
+def test_tc_sql_705_use_sql_does_not_treat_aggregate_names_in_literals_or_comments_as_aggregates(
+ projection,
+ value,
+ monkeypatch,
+ dialog_service,
+ enable_gaussdb_docengine,
+):
+ tenant_id = "0123456789abcdef0123456789abcdef"
+ kb_id = "abcdefabcdefabcdefabcdefabcdefab"
+ table = f"ragflow_{tenant_id}"
+ retriever = ScriptedRetriever(
+ [
+ {"columns": [{"name": "note"}], "rows": [[value]]},
+ {
+ "columns": [{"name": "doc_id"}, {"name": "docnm_kwd"}, {"name": "note"}],
+ "rows": [["d1", "doc.xlsx", value]],
+ },
+ ]
+ )
+ monkeypatch.setattr(dialog_service.settings, "retriever", retriever, raising=False)
+ chat = FakeChatModel(
+ [
+ f"SELECT {projection} FROM {table}",
+ f"SELECT doc_id, docnm_kwd, {projection} FROM {table}",
+ ]
+ )
+
+ result = asyncio.run(
+ dialog_service.use_sql(
+ "show the note",
+ {"note": "string"},
+ tenant_id,
+ chat,
+ quota=False,
+ kb_ids=[kb_id],
+ )
+ )
+
+ assert len(chat.calls) == 2
+ assert len(retriever.sqls) == 2
+ repair_prompt = chat.calls[1][1][0]["content"]
+ assert "missing required source columns" in repair_prompt
+ _assert_gaussdb_sql_rule_markers(repair_prompt)
+ assert result["reference"]["chunks"] == [{"doc_id": "d1", "docnm_kwd": "doc.xlsx", "kb_id": kb_id}]
+
+
+def test_tc_sql_005_use_sql_requires_kb_ids_for_gaussdb(monkeypatch, dialog_service, enable_gaussdb_docengine):
+ validator = Mock()
+ monkeypatch.setattr(dialog_service.gaussdb_text_to_sql, "build_validator", validator)
+
+ with pytest.raises(ValueError) as exc_info:
+ asyncio.run(
+ dialog_service.use_sql(
+ "show amount",
+ {"amount": "number"},
+ "0123456789abcdef0123456789abcdef",
+ FakeChatModel([]),
+ quota=False,
+ kb_ids=[],
+ )
+ )
+
+ assert str(exc_info.value) == "GaussDB Text-to-SQL requires kb_ids"
+ validator.assert_not_called()
+
+
+def test_tc_sql_703_aggregate_source_lookup_failure_returns_empty_chunks(
+ monkeypatch,
+ dialog_service,
+ enable_gaussdb_docengine,
+ caplog,
+):
+ tenant_id = "0123456789abcdef0123456789abcdef"
+ kb_id = "abcdefabcdefabcdefabcdefabcdefab"
+ table = f"ragflow_{tenant_id}"
+ assert (
+ dialog_service.gaussdb_text_to_sql.build_source_reference(
+ {"columns": [{"name": "total_amount"}], "rows": [["120"]]},
+ [kb_id],
+ )
+ is None
+ )
+
+ class AggregateSourceFailureRetriever:
+ def __init__(self):
+ self.sqls = []
+
+ def sql_retrieval(self, sql, format="json"):
+ assert format == "json"
+ self.sqls.append(sql)
+ if len(self.sqls) == 1:
+ return {"columns": [{"name": "total_amount"}], "rows": [["120"]]}
+ raise RuntimeError("source lookup failed")
+
+ retriever = AggregateSourceFailureRetriever()
+ monkeypatch.setattr(dialog_service.settings, "retriever", retriever, raising=False)
+ chat = FakeChatModel([f"SELECT SUM((chunk_data #>> '{{amount}}')::DOUBLE PRECISION) AS total_amount FROM {table} WHERE kb_id = '{kb_id}'"])
+
+ with caplog.at_level(logging.WARNING):
+ result = asyncio.run(
+ dialog_service.use_sql(
+ "What is the total amount?",
+ {"amount": "number"},
+ tenant_id,
+ chat,
+ quota=False,
+ kb_ids=[kb_id],
+ )
+ )
+
+ assert len(retriever.sqls) == 2
+ assert result == {
+ "answer": "|total_number|\n|------\n|120|",
+ "reference": {"chunks": [], "doc_aggs": []},
+ "prompt": chat.calls[0][0],
+ }
+ assert "Failed to fetch chunks" in caplog.text
+
+
+def test_tc_sql_801_use_sql_completes_multikb_reference_kb_id(monkeypatch, dialog_service, enable_gaussdb_docengine):
+ tenant_id = "0123456789abcdef0123456789abcdef"
+ kb1 = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
+ kb2 = "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"
+ table = f"ragflow_{tenant_id}"
+ retriever = MultiKBReferenceRetriever()
+ monkeypatch.setattr(dialog_service.settings, "retriever", retriever, raising=False)
+ chat = FakeChatModel([f"SELECT doc_id, docnm_kwd, chunk_data #>> '{{amount}}' AS amount FROM {table} WHERE kb_id IN ('{kb1}', '{kb2}')"])
+
+ result = asyncio.run(
+ dialog_service.use_sql(
+ "show amount",
+ {"amount": "number"},
+ tenant_id,
+ chat,
+ quota=False,
+ kb_ids=[kb1, kb2],
+ )
+ )
+
+ assert result["reference"]["chunks"] == [
+ {"doc_id": "doc1", "docnm_kwd": "finance-a.csv", "kb_id": kb1},
+ {"doc_id": "doc2", "docnm_kwd": "finance-b.csv", "kb_id": kb1},
+ {"doc_id": "doc3", "docnm_kwd": "finance-c.csv", "kb_id": kb2},
+ ]
+ assert len(retriever.sqls) == 2
+ assert retriever.sqls[1] == (f"SELECT doc_id, kb_id FROM {table} WHERE doc_id IN ('doc1', 'doc2', 'doc3') AND kb_id IN ('{kb1}', '{kb2}') LIMIT 128")
+
+
+ ("lookup_result", "expects_warning"),
+ [
+ (RuntimeError("lookup failed"), True),
+ ({"columns": [{"name": "doc_id"}], "rows": [["doc1"]]}, False),
+ ({"columns": [{"name": "doc_id"}, {"name": "kb_id"}], "rows": []}, False),
+ ({"columns": [{"name": "doc_id"}, {"name": "kb_id"}], "rows": [["doc1"]]}, False),
+ ],
+ ids=["exception", "missing-kb-column", "empty-rows", "short-row"],
+)
+def test_tc_sql_803_multikb_lookup_failures_preserve_reference(
+ lookup_result,
+ expects_warning,
+ monkeypatch,
+ dialog_service,
+ enable_gaussdb_docengine,
+ caplog,
+):
+ tenant_id = "0123456789abcdef0123456789abcdef"
+ kb1 = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
+ kb2 = "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"
+ table = f"ragflow_{tenant_id}"
+ retriever = ScriptedRetriever(
+ [
+ {
+ "columns": [{"name": "doc_id"}, {"name": "docnm_kwd"}, {"name": "amount"}],
+ "rows": [["doc1", "finance.csv", "120"]],
+ },
+ lookup_result,
+ ]
+ )
+ monkeypatch.setattr(dialog_service.settings, "retriever", retriever, raising=False)
+ chat = FakeChatModel([f"SELECT doc_id, docnm_kwd, chunk_data #>> '{{amount}}' AS amount FROM {table} WHERE kb_id IN ('{kb1}', '{kb2}')"])
+
+ with caplog.at_level(logging.WARNING):
+ result = asyncio.run(
+ dialog_service.use_sql(
+ "show amount",
+ {"amount": "number"},
+ tenant_id,
+ chat,
+ quota=False,
+ kb_ids=[kb1, kb2],
+ )
+ )
+
+ assert result["reference"]["chunks"] == [{"doc_id": "doc1", "docnm_kwd": "finance.csv"}]
+ assert len(retriever.sqls) == 2
+ if expects_warning:
+ assert "Failed to complete GaussDB reference kb_id values" in caplog.text
+ else:
+ assert "Failed to complete GaussDB reference kb_id values" not in caplog.text
+
+
+def test_tc_sql_804_multikb_lookup_is_validated_and_scoped(
+ monkeypatch,
+ dialog_service,
+ enable_gaussdb_docengine,
+):
+ tenant_id = "0123456789abcdef0123456789abcdef"
+ kb1 = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
+ kb2 = "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"
+ table = f"ragflow_{tenant_id}"
+ retriever = ScriptedRetriever(
+ [
+ {
+ "columns": [{"name": "doc_id"}, {"name": "docnm_kwd"}, {"name": "amount"}],
+ "rows": [["doc1", "finance.csv", "120"]],
+ },
+ {
+ "columns": [{"name": "doc_id"}, {"name": "kb_id"}],
+ "rows": [["doc1", kb2]],
+ },
+ ]
+ )
+ monkeypatch.setattr(dialog_service.settings, "retriever", retriever, raising=False)
+ chat = FakeChatModel([f"SELECT doc_id, docnm_kwd, chunk_data #>> '{{amount}}' AS amount FROM {table} WHERE kb_id IN ('{kb1}', '{kb2}')"])
+
+ result = asyncio.run(
+ dialog_service.use_sql(
+ "show amount",
+ {"amount": "number"},
+ tenant_id,
+ chat,
+ quota=False,
+ kb_ids=[kb1, kb2],
+ )
+ )
+
+ assert retriever.sqls[1] == (f"SELECT doc_id, kb_id FROM {table} WHERE doc_id IN ('doc1') AND kb_id IN ('{kb1}', '{kb2}') LIMIT 128")
+ assert result["reference"]["chunks"][0]["kb_id"] == kb2
+
+
+def test_tc_sql_805_use_sql_uses_inline_multikb_reference_kb_id(monkeypatch, dialog_service, enable_gaussdb_docengine):
+ tenant_id = "0123456789abcdef0123456789abcdef"
+ kb1 = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
+ kb2 = "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"
+ table = f"ragflow_{tenant_id}"
+ retriever = MultiKBReferenceWithInlineKB()
+ monkeypatch.setattr(dialog_service.settings, "retriever", retriever, raising=False)
+ chat = FakeChatModel([f"SELECT doc_id, docnm_kwd, kb_id FROM {table} WHERE kb_id IN ('{kb1}', '{kb2}')"])
+
+ result = asyncio.run(
+ dialog_service.use_sql(
+ "show amount",
+ {"amount": "number"},
+ tenant_id,
+ chat,
+ quota=False,
+ kb_ids=[kb1, kb2],
+ )
+ )
+
+ assert result["reference"]["chunks"][0]["kb_id"] == kb1
+ assert len(retriever.sqls) == 1
+
+
+def test_tc_ret_1001_use_sql_validator_rejects_before_sql_retrieval(
+ monkeypatch,
+ dialog_service,
+ enable_gaussdb_docengine,
+ caplog,
+):
+ tenant_id = "0123456789abcdef0123456789abcdef"
+ kb_id = "00000000000000000000000000000001"
+ other_kb_id = "00000000000000000000000000000002"
+ table = f"ragflow_{tenant_id}"
+ invalid_sql = f"SELECT doc_id, docnm_kwd, chunk_data #>> '{{amount}}' AS amount FROM {table} WHERE kb_id = '{other_kb_id}'"
+ chat = types.SimpleNamespace(async_chat=AsyncMock(side_effect=[invalid_sql, invalid_sql]))
+ sql_retrieval = Mock(side_effect=AssertionError("validator must reject before SQL execution"))
+
+ monkeypatch.setattr(
+ dialog_service.settings,
+ "retriever",
+ types.SimpleNamespace(sql_retrieval=sql_retrieval),
+ raising=False,
+ )
+
+ with caplog.at_level(logging.WARNING):
+ result = asyncio.run(
+ dialog_service.use_sql(
+ "show amount",
+ {"amount": "number"},
+ tenant_id,
+ chat,
+ quota=True,
+ kb_ids=[kb_id],
+ )
+ )
+
+ assert result is None
+ assert chat.async_chat.await_count == 2
+ sql_retrieval.assert_not_called()
+ assert "SQL crosses the allowed kb_id boundary" in caplog.text
+
+
+def test_tc_sql_1001_use_sql_none_falls_back_to_retrieval(
+ monkeypatch,
+ dialog_service,
+ enable_gaussdb_docengine,
+ caplog,
+):
+ tenant_id = "0123456789abcdef0123456789abcdef"
+ kb_id = "abcdefabcdefabcdefabcdefabcdefab"
+ question = "What is the total amount?"
+ embd_mdl = object()
+ retriever = ScriptedRetriever([])
+ use_sql = AsyncMock(return_value=None)
+ chat = FakeChatModel(["fallback answer"])
+
+ with caplog.at_level(logging.DEBUG):
+ results, get_field_map, actual_kb_id = _run_configured_chat(
+ monkeypatch,
+ dialog_service,
+ retriever=retriever,
+ chat=chat,
+ question=question,
+ use_sql_mock=use_sql,
+ embd_mdl=embd_mdl,
+ )
+
+ assert actual_kb_id == kb_id
+ get_field_map.assert_called_once_with([kb_id])
+ use_sql.assert_awaited_once_with(
+ question,
+ {"amount": "number"},
+ tenant_id,
+ chat,
+ True,
+ [kb_id],
+ doc_ids=None,
+ )
+ retriever.retrieval.assert_awaited_once_with(
+ question,
+ embd_mdl,
+ [tenant_id],
+ [kb_id],
+ 1,
+ 8,
+ 0.2,
+ 0.3,
+ doc_ids=None,
+ top=32,
+ aggs=True,
+ rerank_mdl=None,
+ rank_feature=None,
+ )
+ assert results[-1]["answer"] == "fallback answer"
+ assert results[-1]["reference"] == _expected_fallback_reference(kb_id)
+ assert "SQL failed or returned no results, falling back to vector search" in caplog.text
+
+
+def test_tc_sql_1002_validator_rejection_falls_back_to_retrieval(
+ monkeypatch,
+ dialog_service,
+ enable_gaussdb_docengine,
+ caplog,
+):
+ tenant_id = "0123456789abcdef0123456789abcdef"
+ kb_id = "abcdefabcdefabcdefabcdefabcdefab"
+ question = "What is the total amount?"
+ embd_mdl = object()
+ invalid_sql = "SELECT doc_id FROM other_tenant.ragflow_t1"
+ retriever = ScriptedRetriever([])
+ chat = FakeChatModel([invalid_sql, invalid_sql, "fallback answer"])
+
+ with caplog.at_level(logging.DEBUG):
+ results, get_field_map, actual_kb_id = _run_configured_chat(
+ monkeypatch,
+ dialog_service,
+ retriever=retriever,
+ chat=chat,
+ question=question,
+ embd_mdl=embd_mdl,
+ )
+
+ assert actual_kb_id == kb_id
+ get_field_map.assert_called_once_with([kb_id])
+ assert len(chat.calls) == 3
+ assert retriever.sqls == []
+ retriever.retrieval.assert_awaited_once_with(
+ question,
+ embd_mdl,
+ [tenant_id],
+ [kb_id],
+ 1,
+ 8,
+ 0.2,
+ 0.3,
+ doc_ids=None,
+ top=32,
+ aggs=True,
+ rerank_mdl=None,
+ rank_feature=None,
+ )
+ assert results[-1]["answer"] == "fallback answer"
+ assert results[-1]["reference"] == _expected_fallback_reference(kb_id)
+ assert "cross-schema SQL is not allowed" in caplog.text
+ assert "SQL failed or returned no results, falling back to vector search" in caplog.text
+
+
+def test_tc_sql_1003_sql_timeout_falls_back_to_retrieval(
+ monkeypatch,
+ dialog_service,
+ enable_gaussdb_docengine,
+ caplog,
+):
+ tenant_id = "0123456789abcdef0123456789abcdef"
+ kb_id = "abcdefabcdefabcdefabcdefabcdefab"
+ question = "What is the total amount?"
+ embd_mdl = object()
+ table = f"ragflow_{tenant_id}"
+ valid_sql = f"SELECT doc_id, docnm_kwd, chunk_data #>> '{{amount}}' AS amount FROM {table}"
+ retriever = ScriptedRetriever([TimeoutError("query timeout"), TimeoutError("query timeout")])
+ chat = FakeChatModel([valid_sql, valid_sql, "fallback answer"])
+
+ with caplog.at_level(logging.DEBUG):
+ results, get_field_map, actual_kb_id = _run_configured_chat(
+ monkeypatch,
+ dialog_service,
+ retriever=retriever,
+ chat=chat,
+ question=question,
+ embd_mdl=embd_mdl,
+ )
+
+ assert actual_kb_id == kb_id
+ get_field_map.assert_called_once_with([kb_id])
+ expected_sql = f"{valid_sql} WHERE kb_id = '{kb_id}' LIMIT 128"
+ assert retriever.sqls == [expected_sql, expected_sql]
+ retriever.retrieval.assert_awaited_once_with(
+ question,
+ embd_mdl,
+ [tenant_id],
+ [kb_id],
+ 1,
+ 8,
+ 0.2,
+ 0.3,
+ doc_ids=None,
+ top=32,
+ aggs=True,
+ rerank_mdl=None,
+ rank_feature=None,
+ )
+ assert results[-1]["answer"] == "fallback answer"
+ assert results[-1]["reference"] == _expected_fallback_reference(kb_id)
+ assert "query timeout" in caplog.text
+ assert "SQL failed or returned no results, falling back to vector search" in caplog.text
+
+
+def test_tc_sql_1004_aggregate_answer_survives_source_lookup_failure(
+ monkeypatch,
+ dialog_service,
+ enable_gaussdb_docengine,
+ caplog,
+):
+ tenant_id = "0123456789abcdef0123456789abcdef"
+ table = f"ragflow_{tenant_id}"
+ retriever = ScriptedRetriever(
+ [
+ {"columns": [{"name": "total"}], "rows": [[450]]},
+ RuntimeError("source lookup failed"),
+ ]
+ )
+ chat = FakeChatModel([f"SELECT SUM(CAST(chunk_data #>> '{{amount}}' AS DOUBLE PRECISION)) AS total FROM {table}"])
+
+ with caplog.at_level(logging.WARNING):
+ results, _get_field_map, _kb_id = _run_configured_chat(
+ monkeypatch,
+ dialog_service,
+ retriever=retriever,
+ chat=chat,
+ )
+
+ retriever.retrieval.assert_not_awaited()
+ assert results[-1]["answer"] == "|total|\n|------\n|450|"
+ assert results[-1]["reference"] == {"chunks": [], "doc_aggs": []}
+ assert "Failed to fetch chunks" in caplog.text
+ assert "source lookup failed" in caplog.text
+ assert "SQL failed or returned no results, falling back to vector search" not in caplog.text
+
+
+def test_tc_sql_1005_nonempty_answer_with_empty_chunks_is_returned(
+ monkeypatch,
+ dialog_service,
+ enable_gaussdb_docengine,
+ caplog,
+):
+ sql_result = {
+ "answer": "|total|\n|------|\n|450|",
+ "reference": {"chunks": [], "doc_aggs": []},
+ }
+ retriever = ScriptedRetriever([])
+ use_sql = AsyncMock(return_value=sql_result)
+ chat = FakeChatModel([])
+
+ with caplog.at_level(logging.DEBUG):
+ results, _get_field_map, _kb_id = _run_configured_chat(
+ monkeypatch,
+ dialog_service,
+ retriever=retriever,
+ chat=chat,
+ use_sql_mock=use_sql,
+ )
+
+ use_sql.assert_awaited_once_with(
+ "What is the total amount?",
+ {"amount": "number"},
+ "0123456789abcdef0123456789abcdef",
+ chat,
+ True,
+ ["abcdefabcdefabcdefabcdefabcdefab"],
+ doc_ids=None,
+ )
+ retriever.retrieval.assert_not_awaited()
+ assert results == [sql_result]
+ assert "use_sql: No rows returned" not in caplog.text
diff --git a/test/unit_test/api/db/services/test_gaussdb_doc_metadata_service.py b/test/unit_test/api/db/services/test_gaussdb_doc_metadata_service.py
new file mode 100644
index 0000000000..0c942bfaf6
--- /dev/null
+++ b/test/unit_test/api/db/services/test_gaussdb_doc_metadata_service.py
@@ -0,0 +1,601 @@
+#
+# Copyright 2025 The InfiniFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+import importlib
+import json
+import sys
+import types
+from contextlib import suppress
+from pathlib import Path
+
+import pytest
+import psycopg2
+
+
+_MISSING = object()
+_OPTIONAL_STUB_MODULES = (
+ "google",
+ "google.cloud",
+ "google.cloud.storage",
+ "google.api_core",
+ "google.api_core.exceptions",
+ "json_repair",
+ "rag.utils.es_conn",
+ "rag.utils.infinity_conn",
+ "rag.utils.ob_conn",
+ "rag.utils.opensearch_conn",
+ "rag.utils.azure_sas_conn",
+ "rag.utils.azure_spn_conn",
+ "rag.utils.gcs_conn",
+ "rag.utils.minio_conn",
+ "rag.utils.opendal_conn",
+ "rag.utils.redis_conn",
+ "rag.utils.s3_conn",
+ "rag.utils.oss_conn",
+ "memory.utils.es_conn",
+ "memory.utils.infinity_conn",
+ "memory.utils.ob_conn",
+)
+_LOADED_MODULES = (
+ "common.settings",
+ "api.db.db_models",
+ "api.db.services.doc_metadata_service",
+)
+
+
+def _module(name, **attrs):
+ mod = types.ModuleType(name)
+ for key, value in attrs.items():
+ setattr(mod, key, value)
+ return mod
+
+
+class FakeDB:
+ @staticmethod
+ def connection_context():
+ def decorator(func):
+ func.__wrapped__ = func
+ return func
+
+ return decorator
+
+
+class FakeField:
+ def __eq__(self, _other):
+ return self
+
+
+def _install_optional_import_stubs(monkeypatch):
+ class StubClient:
+ pass
+
+ class StubNotFound(Exception):
+ pass
+
+ class StubDocEngineConnection:
+ def db_type(self):
+ return "stub"
+
+ class StubStorage:
+ def health(self):
+ return True
+
+ google = _module("google")
+ google_cloud = _module("google.cloud")
+ google_storage = _module("google.cloud.storage", Client=StubClient)
+ google_api_core = _module("google.api_core")
+ google_api_exceptions = _module("google.api_core.exceptions", NotFound=StubNotFound)
+ google_cloud.storage = google_storage
+ google_api_core.exceptions = google_api_exceptions
+ google.cloud = google_cloud
+ google.api_core = google_api_core
+ for name, mod in (
+ ("google", google),
+ ("google.cloud", google_cloud),
+ ("google.cloud.storage", google_storage),
+ ("google.api_core", google_api_core),
+ ("google.api_core.exceptions", google_api_exceptions),
+ ):
+ monkeypatch.setitem(sys.modules, name, mod)
+
+ import rag.utils
+ import memory.utils
+
+ rag_modules = {
+ "es_conn": {"ESConnection": StubDocEngineConnection},
+ "infinity_conn": {"InfinityConnection": StubDocEngineConnection},
+ "ob_conn": {"OBConnection": StubDocEngineConnection},
+ "opensearch_conn": {"OSConnection": StubDocEngineConnection},
+ "azure_sas_conn": {"RAGFlowAzureSasBlob": StubStorage},
+ "azure_spn_conn": {"RAGFlowAzureSpnBlob": StubStorage},
+ "gcs_conn": {"RAGFlowGCS": StubStorage},
+ "minio_conn": {"RAGFlowMinio": StubStorage},
+ "opendal_conn": {"OpenDALStorage": StubStorage},
+ "redis_conn": {"REDIS_CONN": types.SimpleNamespace(health=lambda: True)},
+ "s3_conn": {"RAGFlowS3": StubStorage},
+ "oss_conn": {"RAGFlowOSS": StubStorage},
+ }
+ for short_name, attrs in rag_modules.items():
+ mod = _module(f"rag.utils.{short_name}", **attrs)
+ monkeypatch.setitem(sys.modules, f"rag.utils.{short_name}", mod)
+ monkeypatch.setattr(rag.utils, short_name, mod, raising=False)
+
+ for short_name in ("es_conn", "infinity_conn", "ob_conn"):
+ mod = _module(
+ f"memory.utils.{short_name}",
+ ESConnection=StubDocEngineConnection,
+ InfinityConnection=StubDocEngineConnection,
+ OBConnection=StubDocEngineConnection,
+ )
+ monkeypatch.setitem(sys.modules, f"memory.utils.{short_name}", mod)
+ monkeypatch.setattr(memory.utils, short_name, mod, raising=False)
+
+ monkeypatch.setitem(sys.modules, "json_repair", _module("json_repair", loads=json.loads))
+
+
+def doc_metadata_context(monkeypatch):
+ tracked = (*_OPTIONAL_STUB_MODULES, *_LOADED_MODULES)
+ original_modules = {name: sys.modules.get(name, _MISSING) for name in tracked}
+
+ _install_optional_import_stubs(monkeypatch)
+ import common
+
+ fake_settings = _module(
+ "common.settings",
+ docStoreConn=None,
+ DOC_ENGINE_GAUSSDB=False,
+ DOC_ENGINE_INFINITY=False,
+ DOC_ENGINE_OCEANBASE=False,
+ )
+ monkeypatch.setitem(sys.modules, "common.settings", fake_settings)
+ monkeypatch.setattr(common, "settings", fake_settings, raising=False)
+ db_models = _module(
+ "api.db.db_models",
+ DB=FakeDB(),
+ Document=types.SimpleNamespace(
+ id=FakeField(),
+ kb_id=FakeField(),
+ select=staticmethod(lambda *_args, **_kwargs: None),
+ ),
+ Knowledgebase=types.SimpleNamespace(
+ id=FakeField(),
+ tenant_id=FakeField(),
+ get_by_id=staticmethod(lambda *_args, **_kwargs: None),
+ ),
+ )
+ monkeypatch.setitem(sys.modules, "api.db.db_models", db_models)
+ repo_root = Path(__file__).resolve().parents[5]
+ spec = importlib.util.spec_from_file_location(
+ "api.db.services.doc_metadata_service",
+ repo_root / "api" / "db" / "services" / "doc_metadata_service.py",
+ )
+ module = importlib.util.module_from_spec(spec)
+ monkeypatch.setitem(sys.modules, "api.db.services.doc_metadata_service", module)
+ spec.loader.exec_module(module)
+ monkeypatch.setattr(module.settings, "docStoreConn", None, raising=False)
+ search_result_cls = importlib.import_module("rag.utils.gaussdb_conn").SearchResult
+
+ try:
+ yield types.SimpleNamespace(
+ module=module,
+ DocMetadataService=module.DocMetadataService,
+ settings=module.settings,
+ SearchResult=search_result_cls,
+ )
+ finally:
+ services_pkg = sys.modules.get("api.db.services")
+ if original_modules["api.db.services.doc_metadata_service"] is _MISSING and services_pkg is not None:
+ with suppress(AttributeError):
+ delattr(services_pkg, "doc_metadata_service")
+ for name, original in original_modules.items():
+ if original is _MISSING:
+ sys.modules.pop(name, None)
+ else:
+ sys.modules[name] = original
+
+
+class FakeGaussDBStore:
+ def __init__(self, *, exists=True, create_result=True, create_error=None, insert_result=None):
+ self.created_meta = []
+ self.deleted = []
+ self.inserted = []
+ self.executed = []
+ self.existence_checks = []
+ self.exists = exists
+ self.create_result = create_result
+ self.create_error = create_error
+ self.insert_result = [] if insert_result is None else insert_result
+
+ def db_type(self):
+ return "gaussdb"
+
+ def create_doc_meta_idx(self, index_name):
+ self.created_meta.append(index_name)
+ if self.create_error is not None:
+ raise self.create_error
+ return self.create_result
+
+ def index_exist(self, *args):
+ self.existence_checks.append(args)
+ return self.exists
+
+ def get(self, *_args):
+ return {"id": "doc1", "kb_id": "kb1", "meta_fields": {"old": "value"}}
+
+ def delete(self, condition, index_name, kb_id):
+ self.deleted.append((condition, index_name, kb_id))
+ return 1
+
+ def insert(self, rows, index_name, kb_id):
+ self.inserted.append((rows, index_name, kb_id))
+ return self.insert_result
+
+ def fetch_metadata_doc_ids(self, index_name, kb_ids, sql_filter, filter_params, limit):
+ self.executed.append((index_name, kb_ids, sql_filter, filter_params, limit))
+ return ["doc-risk"]
+
+
+class FakeDocQuery:
+ def join(self, *_args, **_kwargs):
+ return self
+
+ def where(self, *_args, **_kwargs):
+ return self
+
+ def first(self):
+ return types.SimpleNamespace(
+ kb_id="kb1",
+ knowledgebase=types.SimpleNamespace(tenant_id="tenant1"),
+ )
+
+
+class FakeSearchResultStore(FakeGaussDBStore):
+ def __init__(self, search_result_cls):
+ super().__init__()
+ self.search_result_cls = search_result_cls
+
+ def search(self, **kwargs):
+ self.executed.append(kwargs)
+ return self.search_result_cls(
+ total=1,
+ chunks=[{"id": "doc1", "kb_id": "kb1", "meta_fields": {"author": "Alice"}}],
+ )
+
+
+class PagingSearchResultStore(FakeGaussDBStore):
+ def __init__(self, search_result_cls):
+ super().__init__()
+ self.search_result_cls = search_result_cls
+
+ def search(self, **kwargs):
+ self.executed.append(kwargs)
+ offset = kwargs["offset"]
+ limit = kwargs["limit"]
+ if offset == 0:
+ chunks = [{"id": f"doc{i}", "kb_id": "kb1", "meta_fields": {"seq": i}} for i in range(limit)]
+ elif offset == limit:
+ chunks = [{"id": "doc1000", "kb_id": "kb1", "meta_fields": {"seq": 1000}}]
+ else:
+ chunks = []
+ return self.search_result_cls(total=1001, chunks=chunks)
+
+
+def test_tc_ret_407_iter_search_results_accepts_gaussdb_search_result_shape(doc_metadata_context):
+ service = doc_metadata_context.DocMetadataService
+ result = doc_metadata_context.SearchResult(total=1, chunks=[{"id": "doc1", "meta_fields": {"author": "Alice"}}])
+
+ docs = list(service._iter_search_results(result))
+
+ assert docs == [("doc1", {"id": "doc1", "meta_fields": {"author": "Alice"}})]
+
+
+def test_tc_ret_407_search_metadata_accepts_gaussdb_search_result_shape(monkeypatch, doc_metadata_context):
+ service = doc_metadata_context.DocMetadataService
+ module = doc_metadata_context.module
+ settings = doc_metadata_context.settings
+ fake = FakeSearchResultStore(doc_metadata_context.SearchResult)
+ monkeypatch.setattr(settings, "docStoreConn", fake)
+ monkeypatch.setattr(settings, "DOC_ENGINE_GAUSSDB", True)
+ monkeypatch.setattr(settings, "DOC_ENGINE_INFINITY", False)
+ monkeypatch.setattr(
+ module.Knowledgebase,
+ "get_by_id",
+ staticmethod(lambda _kb_id: type("KB", (), {"tenant_id": "tenant1"})()),
+ )
+
+ docs = service._search_metadata("kb1")
+
+ assert docs == [{"id": "doc1", "kb_id": "kb1", "meta_fields": {"author": "Alice"}}]
+ assert fake.executed[0]["index_names"] == "ragflow_doc_meta_tenant1"
+ assert fake.executed[0]["knowledgebase_ids"] == ["kb1"]
+
+
+def test_tc_ret_407_search_metadata_paginates_gaussdb_search_result_total(monkeypatch, doc_metadata_context):
+ service = doc_metadata_context.DocMetadataService
+ module = doc_metadata_context.module
+ settings = doc_metadata_context.settings
+ fake = PagingSearchResultStore(doc_metadata_context.SearchResult)
+ monkeypatch.setattr(settings, "docStoreConn", fake)
+ monkeypatch.setattr(settings, "DOC_ENGINE_GAUSSDB", True)
+ monkeypatch.setattr(settings, "DOC_ENGINE_INFINITY", False)
+ monkeypatch.setattr(
+ module.Knowledgebase,
+ "get_by_id",
+ staticmethod(lambda _kb_id: type("KB", (), {"tenant_id": "tenant1"})()),
+ )
+
+ docs = service._search_metadata("kb1")
+
+ doc_ids = [doc["id"] for doc in docs]
+ assert doc_ids == [f"doc{i}" for i in range(1001)]
+ assert len(set(doc_ids)) == 1001
+ assert [call["offset"] for call in fake.executed] == [0, 1000]
+ assert len(fake.executed) == 2
+
+
+ ("table_exists", "created_meta"),
+ [
+ (True, []),
+ (False, ["ragflow_doc_meta_tenant1"]),
+ ],
+ ids=("existing-table", "missing-table"),
+)
+def test_tc_wrt_306_update_document_metadata_checks_table_before_gaussdb_upsert(
+ monkeypatch,
+ doc_metadata_context,
+ table_exists,
+ created_meta,
+):
+ service = doc_metadata_context.DocMetadataService
+ module = doc_metadata_context.module
+ settings = doc_metadata_context.settings
+ fake = FakeGaussDBStore(exists=table_exists)
+ monkeypatch.setattr(settings, "docStoreConn", fake)
+ monkeypatch.setattr(settings, "DOC_ENGINE_GAUSSDB", True)
+ monkeypatch.setattr(module.Document, "select", staticmethod(lambda *_args, **_kwargs: FakeDocQuery()))
+
+ ok = service.update_document_metadata.__wrapped__(
+ service,
+ "doc1",
+ {"author": "Alice"},
+ )
+
+ assert ok is True
+ assert fake.existence_checks == [("ragflow_doc_meta_tenant1", "kb1")]
+ assert fake.created_meta == created_meta
+ assert fake.deleted == []
+ assert fake.inserted == [([{"id": "doc1", "kb_id": "kb1", "meta_fields": {"author": "Alice"}}], "ragflow_doc_meta_tenant1", "kb1")]
+
+
+ ("create_result", "create_error"),
+ [
+ (False, None),
+ (True, psycopg2.OperationalError("metadata table creation failed")),
+ ],
+ ids=("returns-false", "raises"),
+)
+def test_tc_wrt_307_update_document_metadata_stops_when_gaussdb_meta_table_creation_fails(
+ monkeypatch,
+ doc_metadata_context,
+ create_result,
+ create_error,
+):
+ service = doc_metadata_context.DocMetadataService
+ module = doc_metadata_context.module
+ settings = doc_metadata_context.settings
+ fake = FakeGaussDBStore(exists=False, create_result=create_result, create_error=create_error)
+ monkeypatch.setattr(settings, "docStoreConn", fake)
+ monkeypatch.setattr(settings, "DOC_ENGINE_GAUSSDB", True)
+ monkeypatch.setattr(module.Document, "select", staticmethod(lambda *_args, **_kwargs: FakeDocQuery()))
+
+ ok = service.update_document_metadata.__wrapped__(
+ service,
+ "doc1",
+ {"author": "Alice"},
+ )
+
+ assert ok is False
+ assert fake.existence_checks == [("ragflow_doc_meta_tenant1", "kb1")]
+ assert fake.created_meta == ["ragflow_doc_meta_tenant1"]
+ assert fake.inserted == []
+ assert fake.deleted == []
+
+
+def test_tc_wrt_308_update_document_metadata_returns_false_for_gaussdb_upsert_errors(monkeypatch, doc_metadata_context):
+ service = doc_metadata_context.DocMetadataService
+ module = doc_metadata_context.module
+ settings = doc_metadata_context.settings
+ fake = FakeGaussDBStore(insert_result=["doc1"])
+ monkeypatch.setattr(settings, "docStoreConn", fake)
+ monkeypatch.setattr(settings, "DOC_ENGINE_GAUSSDB", True)
+ monkeypatch.setattr(module.Document, "select", staticmethod(lambda *_args, **_kwargs: FakeDocQuery()))
+
+ ok = service.update_document_metadata.__wrapped__(
+ service,
+ "doc1",
+ {"author": "Alice"},
+ )
+
+ assert ok is False
+ assert fake.existence_checks == [("ragflow_doc_meta_tenant1", "kb1")]
+ assert fake.created_meta == []
+ assert fake.inserted == [([{"id": "doc1", "kb_id": "kb1", "meta_fields": {"author": "Alice"}}], "ragflow_doc_meta_tenant1", "kb1")]
+ assert fake.deleted == []
+
+
+def test_tc_mgf_710_gaussdb_pushdown_routes_scoped_filter_with_probe_limit(monkeypatch, doc_metadata_context):
+ service = doc_metadata_context.DocMetadataService
+ settings = doc_metadata_context.settings
+ fake = FakeGaussDBStore()
+ monkeypatch.setattr(settings, "docStoreConn", fake)
+ monkeypatch.setattr(settings, "DOC_ENGINE_GAUSSDB", True)
+ monkeypatch.setattr(
+ doc_metadata_context.module.Knowledgebase,
+ "get_by_id",
+ staticmethod(lambda _kb_id: type("KB", (), {"tenant_id": "tenant1"})()),
+ )
+
+ doc_ids = service.filter_doc_ids_by_meta_pushdown(
+ ["kb1"],
+ [{"key": "author", "op": "=", "value": "Alice"}],
+ "and",
+ limit=25,
+ )
+
+ assert doc_ids == ["doc-risk"]
+ index_name, kb_ids, sql_filter, params, limit = fake.executed[0]
+ assert index_name == "ragflow_doc_meta_tenant1"
+ assert kb_ids == ["kb1"]
+ assert sql_filter == "(lower(meta_fields #>> '{author}') = %s OR jsonb_exists(meta_fields #> '{author}', %s))"
+ assert params == ["alice", "alice"]
+ assert limit == 26
+ assert fake.created_meta == []
+ assert fake.inserted == []
+ assert fake.deleted == []
+
+
+ ("returned_ids", "expected"),
+ [
+ (["doc1", "doc2"], ["doc1", "doc2"]),
+ (["doc1", "doc1", "doc2"], ["doc1", "doc2"]),
+ (["doc1", "doc2", "doc3"], None),
+ ],
+)
+def test_tc_mgf_710_gaussdb_pushdown_enforces_complete_result_cap(
+ monkeypatch,
+ doc_metadata_context,
+ returned_ids,
+ expected,
+):
+ service = doc_metadata_context.DocMetadataService
+ settings = doc_metadata_context.settings
+ fake = FakeGaussDBStore()
+ calls = []
+
+ def fetch_metadata_doc_ids(*args):
+ calls.append(args)
+ return returned_ids
+
+ fake.fetch_metadata_doc_ids = fetch_metadata_doc_ids
+ monkeypatch.setattr(settings, "docStoreConn", fake)
+ monkeypatch.setattr(settings, "DOC_ENGINE_GAUSSDB", True)
+ monkeypatch.setattr(
+ doc_metadata_context.module.Knowledgebase,
+ "get_by_id",
+ staticmethod(lambda _kb_id: type("KB", (), {"tenant_id": "tenant1"})()),
+ )
+
+ doc_ids = service.filter_doc_ids_by_meta_pushdown(
+ ["kb1"],
+ [{"key": "author", "op": "=", "value": "Alice"}],
+ "and",
+ limit=2,
+ )
+
+ assert doc_ids == expected
+ assert len(calls) == 1
+ index_name, kb_ids, sql_filter, params, probe_limit = calls[0]
+ assert index_name == "ragflow_doc_meta_tenant1"
+ assert kb_ids == ["kb1"]
+ assert sql_filter == "(lower(meta_fields #>> '{author}') = %s OR jsonb_exists(meta_fields #> '{author}', %s))"
+ assert params == ["alice", "alice"]
+ assert probe_limit == 3
+ assert fake.created_meta == []
+ assert fake.inserted == []
+ assert fake.deleted == []
+
+
+ ("filters", "logic"),
+ [
+ ([{"key": "author", "op": "regex", "value": "^A"}], "and"),
+ ([{"key": "author", "op": "=", "value": "Alice"}], "xor"),
+ ],
+)
+def test_tc_mgf_711_gaussdb_pushdown_rejects_unsupported_filter_without_fetch(
+ monkeypatch,
+ doc_metadata_context,
+ caplog,
+ filters,
+ logic,
+):
+ service = doc_metadata_context.DocMetadataService
+ settings = doc_metadata_context.settings
+ fake = FakeGaussDBStore()
+ monkeypatch.setattr(settings, "docStoreConn", fake)
+ monkeypatch.setattr(settings, "DOC_ENGINE_GAUSSDB", True)
+ monkeypatch.setattr(
+ doc_metadata_context.module.Knowledgebase,
+ "get_by_id",
+ staticmethod(lambda _kb_id: type("KB", (), {"tenant_id": "tenant1"})()),
+ )
+
+ caplog.clear()
+ doc_ids = service.filter_doc_ids_by_meta_pushdown(
+ ["kb1"],
+ filters,
+ logic,
+ limit=2,
+ )
+
+ assert doc_ids is None
+ assert fake.executed == []
+ assert fake.created_meta == []
+ assert fake.inserted == []
+ assert fake.deleted == []
+ messages = [record.getMessage() for record in caplog.records]
+ assert all("author" not in message and "Alice" not in message and "^A" not in message for message in messages)
+
+
+def test_tc_mgf_712_gaussdb_pushdown_falls_back_without_partial_ids_on_fetch_failure(monkeypatch, doc_metadata_context):
+ service = doc_metadata_context.DocMetadataService
+ settings = doc_metadata_context.settings
+ fake = FakeGaussDBStore()
+ calls = []
+
+ def raise_fetch(*args, **_kwargs):
+ calls.append((args, _kwargs))
+ raise RuntimeError("gaussdb unavailable")
+
+ fake.fetch_metadata_doc_ids = raise_fetch
+ monkeypatch.setattr(settings, "docStoreConn", fake)
+ monkeypatch.setattr(settings, "DOC_ENGINE_GAUSSDB", True)
+ monkeypatch.setattr(
+ doc_metadata_context.module.Knowledgebase,
+ "get_by_id",
+ staticmethod(lambda _kb_id: type("KB", (), {"tenant_id": "tenant1"})()),
+ )
+
+ doc_ids = service.filter_doc_ids_by_meta_pushdown(
+ ["kb1"],
+ [{"key": "author", "op": "=", "value": "Alice"}],
+ "and",
+ limit=2,
+ )
+
+ assert doc_ids is None
+ assert len(calls) == 1
+ args, kwargs = calls[0]
+ assert args[0:2] == ("ragflow_doc_meta_tenant1", ["kb1"])
+ assert args[2] == "(lower(meta_fields #>> '{author}') = %s OR jsonb_exists(meta_fields #> '{author}', %s))"
+ assert args[3:] == (["alice", "alice"], 3)
+ assert kwargs == {}
+ assert fake.created_meta == []
+ assert fake.inserted == []
+ assert fake.deleted == []
diff --git a/test/unit_test/api/db/services/test_gaussdb_feedback_dataset.py b/test/unit_test/api/db/services/test_gaussdb_feedback_dataset.py
new file mode 100644
index 0000000000..27832fadde
--- /dev/null
+++ b/test/unit_test/api/db/services/test_gaussdb_feedback_dataset.py
@@ -0,0 +1,124 @@
+#
+# Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+import importlib.util
+import sys
+from pathlib import Path
+from types import ModuleType
+
+
+def _install_module(monkeypatch, name, **attrs):
+ module = ModuleType(name)
+ for key, value in attrs.items():
+ setattr(module, key, value)
+ monkeypatch.setitem(sys.modules, name, module)
+ return module
+
+
+def _load_feedback_module(monkeypatch):
+ repo_root = Path(__file__).resolve().parents[5]
+ common_pkg = _install_module(monkeypatch, "common")
+ constants_mod = _install_module(monkeypatch, "common.constants", PAGERANK_FLD="pagerank_fea")
+ settings_mod = _install_module(monkeypatch, "common.settings", DOC_ENGINE="gaussdb")
+ common_pkg.constants = constants_mod
+ common_pkg.settings = settings_mod
+
+ _install_module(monkeypatch, "rag")
+ _install_module(monkeypatch, "rag.nlp")
+ _install_module(monkeypatch, "rag.nlp.search", index_name=lambda tenant_id: f"ragflow_{tenant_id}")
+
+ spec = importlib.util.spec_from_file_location(
+ "api.db.services.chunk_feedback_service",
+ repo_root / "api" / "db" / "services" / "chunk_feedback_service.py",
+ )
+ module = importlib.util.module_from_spec(spec)
+ monkeypatch.setitem(sys.modules, "api.db.services.chunk_feedback_service", module)
+ spec.loader.exec_module(module)
+ return module, settings_mod
+
+
+class FakeGaussDBStore:
+ def __init__(self):
+ self.updated = []
+ self.adjusted = []
+ self.rows = {"chunk1": {"pagerank_fea": 10}}
+
+ def db_type(self):
+ return "gaussdb"
+
+ def adjust_chunk_pagerank_fea(self, *args, **kwargs):
+ self.adjusted.append((args, kwargs))
+ return True
+
+ def get(self, chunk_id, index_name, kb_ids):
+ return self.rows.get(chunk_id)
+
+ def update(self, condition, values, index_name, kb_id):
+ self.updated.append((condition, values, index_name, kb_id))
+ return True
+
+
+class FakeGaussDBStoreWithoutAdjust:
+ def __init__(self):
+ self.updated = []
+ self.rows = {"chunk1": {"pagerank_fea": 10}}
+
+ def db_type(self):
+ return "gaussdb"
+
+ def get(self, chunk_id, index_name, kb_ids):
+ return self.rows.get(chunk_id)
+
+ def update(self, condition, values, index_name, kb_id):
+ self.updated.append((condition, values, index_name, kb_id))
+ return True
+
+
+def test_update_chunk_weight_falls_back_to_update_by_id_and_kb_without_adjust(monkeypatch):
+ module, settings_mod = _load_feedback_module(monkeypatch)
+ fake = FakeGaussDBStoreWithoutAdjust()
+ settings_mod.docStoreConn = fake
+
+ ok = module.ChunkFeedbackService.update_chunk_weight(
+ tenant_id="tenant",
+ chunk_id="chunk1",
+ kb_id="kb1",
+ delta=2,
+ )
+
+ assert ok is True
+ assert fake.updated == [({"id": "chunk1"}, {"pagerank_fea": 12}, "ragflow_tenant", "kb1")]
+
+
+def test_update_chunk_weight_routes_gaussdb_through_atomic_adjust(monkeypatch):
+ module, settings_mod = _load_feedback_module(monkeypatch)
+ fake = FakeGaussDBStore()
+ settings_mod.docStoreConn = fake
+
+ ok = module.ChunkFeedbackService.update_chunk_weight(
+ tenant_id="tenant",
+ chunk_id="chunk1",
+ kb_id="kb1",
+ delta=2,
+ )
+
+ assert ok is True
+ assert fake.adjusted == [
+ (
+ ("chunk1", "ragflow_tenant", "kb1", 2.0, module.MIN_PAGERANK_WEIGHT, module.MAX_PAGERANK_WEIGHT),
+ {},
+ )
+ ]
+ assert fake.updated == []
diff --git a/test/unit_test/api/db/test_gaussdb_adaptation_points.py b/test/unit_test/api/db/test_gaussdb_adaptation_points.py
new file mode 100644
index 0000000000..0c7f46e841
--- /dev/null
+++ b/test/unit_test/api/db/test_gaussdb_adaptation_points.py
@@ -0,0 +1,386 @@
+"""
+Direct tests for each GaussDB metadata database adaptation point.
+"""
+
+import logging
+from pathlib import Path
+
+from peewee import ProgrammingError
+from ruamel.yaml import YAML
+
+REPO_ROOT = Path(__file__).resolve().parents[4]
+
+from api.db import db_models
+from api.db.services import connector_service
+from common import settings
+
+
+def read_repo_file(path: str) -> str:
+ return (REPO_ROOT / path).read_text()
+
+
+def load_yaml(path: str):
+ yaml = YAML(typ="safe", pure=True)
+ return yaml.load(read_repo_file(path))
+
+
+def test_gaussdb_duplicate_column_migration_is_idempotent(monkeypatch, caplog):
+ class Migrator:
+ def add_column(self, table_name, column_name, column_type):
+ return table_name, column_name, column_type
+
+ def duplicate_column(*_operations):
+ raise ProgrammingError('column "f_extra" already exists')
+
+ monkeypatch.setattr(db_models, "migrate", duplicate_column)
+ monkeypatch.setattr(settings, "DATABASE_TYPE", "gaussdb")
+
+ caplog.set_level(logging.CRITICAL)
+ db_models.alter_db_add_column(Migrator(), "tenant", "f_extra", object())
+
+ assert not caplog.records
+
+
+def test_gaussdb_migration_logs_unrelated_programming_errors(monkeypatch, caplog):
+ class Migrator:
+ def add_column(self, table_name, column_name, column_type):
+ return table_name, column_name, column_type
+
+ def syntax_error(*_operations):
+ raise ProgrammingError("syntax error near invalid_sql")
+
+ monkeypatch.setattr(db_models, "migrate", syntax_error)
+ monkeypatch.setattr(settings, "DATABASE_TYPE", "gaussdb")
+
+ caplog.set_level(logging.CRITICAL)
+ db_models.alter_db_add_column(Migrator(), "tenant", "f_extra", object())
+
+ assert "syntax error near invalid_sql" in caplog.text
+
+
+def test_gaussdb_empty_string_compatible_migration_drops_not_null_only_for_gaussdb(monkeypatch):
+ class FakeDB:
+ queries = []
+
+ @classmethod
+ def execute_sql(cls, sql, params=None):
+ cls.queries.append((sql, params))
+
+ monkeypatch.setattr(db_models, "DB", FakeDB)
+ monkeypatch.setattr(settings, "DATABASE_TYPE", "gaussdb")
+
+ db_models.relax_gaussdb_empty_string_compatible_columns()
+
+ expected_query_count = sum(len(columns) for _, columns in db_models.GAUSSDB_EMPTY_STRING_COMPATIBLE_COLUMNS)
+ assert len(FakeDB.queries) == expected_query_count
+ assert ('ALTER TABLE "user" ALTER COLUMN "nickname" DROP NOT NULL', None) in FakeDB.queries
+ assert ('ALTER TABLE "tenant" ALTER COLUMN "llm_id" DROP NOT NULL', None) in FakeDB.queries
+ assert ('ALTER TABLE "system_settings" ALTER COLUMN "value" DROP NOT NULL', None) in FakeDB.queries
+ assert ('ALTER TABLE "task" ALTER COLUMN "task_type" DROP NOT NULL', None) in FakeDB.queries
+ assert ('ALTER TABLE "sync_logs" ALTER COLUMN "error_msg" DROP NOT NULL', None) in FakeDB.queries
+ assert ('ALTER TABLE "sync_logs" ALTER COLUMN "full_exception_trace" DROP NOT NULL', None) in FakeDB.queries
+ assert ('ALTER TABLE "api_4_conversation" ALTER COLUMN "user_id" DROP NOT NULL', None) in FakeDB.queries
+ assert ('ALTER TABLE "user_canvas" ALTER COLUMN "tags" DROP NOT NULL', None) in FakeDB.queries
+
+ monkeypatch.setattr(settings, "DATABASE_TYPE", "mysql")
+ db_models.relax_gaussdb_empty_string_compatible_columns()
+
+ assert len(FakeDB.queries) == expected_query_count
+
+
+def test_gaussdb_migration_adds_compatible_tags_before_relaxing_columns(monkeypatch):
+ events = []
+ migrated_tags_field = None
+
+ def record_add_column(_migrator, table_name, column_name, column_type):
+ nonlocal migrated_tags_field
+ events.append(("add", table_name, column_name))
+ if (table_name, column_name) == ("user_canvas", "tags"):
+ migrated_tags_field = column_type
+
+ monkeypatch.setattr(settings, "DATABASE_TYPE", "gaussdb")
+ monkeypatch.setattr(db_models, "alter_db_add_column", record_add_column)
+ monkeypatch.setattr(db_models, "alter_db_column_type", lambda *_args: events.append(("alter_type",)))
+ monkeypatch.setattr(db_models, "alter_db_rename_column", lambda *_args: events.append(("rename",)))
+ monkeypatch.setattr(db_models, "alter_db_drop_index", lambda *_args: events.append(("drop_index",)))
+ monkeypatch.setattr(db_models, "relax_gaussdb_empty_string_compatible_columns", lambda: events.append(("relax",)))
+ monkeypatch.setattr(db_models, "migrate", lambda *_operations: None)
+ monkeypatch.setattr(db_models, "migrate_add_unique_email", lambda _migrator: None)
+ monkeypatch.setattr(db_models, "migrate_model_type_names", lambda: None)
+ monkeypatch.setattr(db_models, "ensure_model_indexes", lambda _migrator: None)
+
+ db_models.migrate_db()
+
+ assert isinstance(migrated_tags_field, db_models.EmptyStringCharField)
+ assert migrated_tags_field.null is True
+ assert events[-1] == ("relax",)
+
+
+def test_gaussdb_unique_email_migration_checks_unique_email_index_not_fixed_name(monkeypatch):
+ class Cursor:
+ def fetchone(self):
+ return (1,)
+
+ class FakeDB:
+ queries = []
+ params = []
+
+ @classmethod
+ def execute_sql(cls, sql, params=None):
+ cls.queries.append(sql)
+ cls.params.append(params)
+ return Cursor()
+
+ monkeypatch.setattr(settings, "DATABASE_TYPE", "gaussdb")
+ monkeypatch.setattr(db_models, "DB", FakeDB)
+
+ db_models.migrate_add_unique_email(object())
+
+ assert len(FakeDB.queries) == 1
+ assert "pg_indexes" in FakeDB.queries[0]
+ assert "lower(indexdef) LIKE %s" in FakeDB.queries[0]
+ assert FakeDB.params[0] == ("create unique index%", "%(email)%", '%("email")%')
+ assert "indexname = 'user_email'" not in FakeDB.queries[0]
+
+
+def test_gaussdb_deadlock_retry_recognizes_psycopg_sqlstates(monkeypatch):
+ from peewee import OperationalError
+ from api.db.services.common_service import _is_deadlock_error
+
+ class PgError:
+ def __init__(self, pgcode):
+ self.pgcode = pgcode
+
+ monkeypatch.setattr(settings, "DATABASE_TYPE", "gaussdb")
+
+ assert _is_deadlock_error(OperationalError(PgError("40P01")))
+ assert _is_deadlock_error(OperationalError(PgError("40001")))
+ assert _is_deadlock_error(OperationalError(PgError("55P03")))
+ assert not _is_deadlock_error(OperationalError("syntax error"))
+
+
+def test_connector_poll_interval_uses_gaussdb_compatible_sql(monkeypatch):
+ monkeypatch.setattr(connector_service.settings, "DATABASE_TYPE", "gaussdb")
+ expr = connector_service._gaussdb_poll_interval_expr("refresh_freq")
+
+ assert "t2.refresh_freq * INTERVAL '1 minute'" in expr.sql
+ assert "make_interval" not in expr.sql
+ assert "`t2`" not in expr.sql
+
+
+def test_health_check_uses_generic_probe_for_gaussdb(monkeypatch):
+ from api.utils import health_utils
+
+ class Cursor:
+ def fetchone(self):
+ return (1,)
+
+ def close(self):
+ pass
+
+ class FakeDB:
+ queries = []
+
+ @classmethod
+ def execute_sql(cls, sql):
+ cls.queries.append(sql)
+ return Cursor()
+
+ monkeypatch.setattr(settings, "DATABASE_TYPE", "gaussdb")
+ monkeypatch.setattr(health_utils, "DB", FakeDB)
+
+ status = health_utils.get_mysql_status()
+
+ assert status == {
+ "status": "alive",
+ "message": {
+ "database": "gaussdb",
+ "result": 1,
+ },
+ }
+ assert FakeDB.queries == ["SELECT 1;"]
+
+
+def test_gaussdb_metadata_health_masks_connection_secrets(monkeypatch, caplog):
+ from api.utils import health_utils
+
+ password = "metadata-password-sentinel"
+ dsn = "postgresql://metadata-user:[email protected]:19995/ragflow"
+ access_token = "metadata-token-sentinel"
+
+ class FakeDB:
+ @staticmethod
+ def execute_sql(_sql):
+ raise RuntimeError(f"connection failed password={password} dsn={dsn} access_token={access_token}")
+
+ monkeypatch.setattr(settings, "DATABASE_TYPE", "gaussdb")
+ monkeypatch.setattr(health_utils, "DB", FakeDB)
+ caplog.set_level(logging.ERROR)
+
+ status = health_utils.get_database_status()
+ serialized = str(status)
+
+ assert status["status"] == "timeout"
+ assert "***" in status["message"]
+ assert "GaussDB metadata database status check failed (RuntimeError)" in caplog.text
+ for secret in (password, dsn, access_token):
+ assert secret not in serialized
+ assert secret not in caplog.text
+
+
+def test_admin_config_selects_gaussdb_metadata_entry(monkeypatch):
+ from admin.server import config as admin_config
+
+ monkeypatch.setenv("DB_TYPE", "GaussDB")
+ monkeypatch.setenv("GAUSSDB_METADATA_HOST", "metadata.example.com")
+ monkeypatch.setenv("GAUSSDB_METADATA_PORT", "8000")
+ monkeypatch.setenv("GAUSSDB_METADATA_USER", "metadata_user")
+ monkeypatch.setenv("GAUSSDB_METADATA_PASSWORD", "metadata-secret")
+ monkeypatch.setenv("GAUSSDB_METADATA_SCHEMA", "metadata_schema")
+ monkeypatch.setattr(
+ admin_config,
+ "read_config",
+ lambda _path: {
+ "mysql": {"host": "mysql", "port": 3306, "user": "root", "password": "mysql-secret"},
+ "gaussdb": {
+ "host": "doc.example.com",
+ "port": 19995,
+ "database": "doc_db",
+ "user": "doc_user",
+ "password": "doc-secret",
+ "schema": "doc_schema",
+ },
+ },
+ )
+
+ configs = admin_config.load_configurations("ignored.yaml")
+ metadata_configs = [cfg for cfg in configs if cfg.service_type == "meta_data"]
+ retrieval_configs = [cfg for cfg in configs if cfg.service_type == "retrieval"]
+
+ assert len(metadata_configs) == 1
+ assert len(retrieval_configs) == 1
+ assert metadata_configs[0].name == "gaussdb"
+ assert metadata_configs[0].meta_type == "gaussdb"
+ assert metadata_configs[0].host == "metadata.example.com"
+ assert metadata_configs[0].username == "metadata_user"
+ assert metadata_configs[0].metadata_schema == "metadata_schema"
+ assert metadata_configs[0].detail_func_name == "get_database_status"
+
+ retrieval = retrieval_configs[0]
+ assert retrieval.host == "doc.example.com"
+ assert retrieval.port == 19995
+ assert retrieval.database == "doc_db"
+ assert retrieval.retrieval_schema == "doc_schema"
+ assert retrieval.detail_func_name == "get_gaussdb_status"
+ assert "password" not in str(retrieval.to_dict()).lower()
+
+
+def test_admin_service_safe_serialization_redacts_credentials():
+ from admin.server import config as admin_config
+
+ metadata = admin_config.GaussDBMetadataConfig(
+ id=1,
+ name="gaussdb",
+ host="metadata.example.com",
+ port=8000,
+ username="metadata_user",
+ password="metadata-secret",
+ metadata_schema="metadata_schema",
+ service_type="meta_data",
+ meta_type="gaussdb",
+ detail_func_name="get_database_status",
+ )
+
+ serialized = metadata.to_dict()
+
+ assert serialized["extra"]["username"] == "metadata_user"
+ assert serialized["extra"]["schema"] == "metadata_schema"
+ assert serialized["extra"]["password"] == "********"
+ assert "metadata-secret" not in str(serialized)
+
+
+def test_admin_config_falls_back_to_gaussdb_env(monkeypatch):
+ from admin.server import config as admin_config
+
+ monkeypatch.setenv("DB_TYPE", "gaussdb")
+ monkeypatch.setenv("GAUSSDB_METADATA_HOST", "gaussdb-env.example.com")
+ monkeypatch.setenv("GAUSSDB_METADATA_PORT", "19995")
+ monkeypatch.setenv("GAUSSDB_METADATA_USER", "zws")
+ monkeypatch.setenv("GAUSSDB_METADATA_PASSWORD", "env-secret")
+ monkeypatch.setenv("GAUSSDB_METADATA_SCHEMA", "env_schema")
+ monkeypatch.setattr(admin_config, "read_config", lambda _path: {})
+
+ configs = admin_config.load_configurations("ignored.yaml")
+ metadata_configs = [cfg for cfg in configs if cfg.service_type == "meta_data"]
+
+ assert len(metadata_configs) == 1
+ assert metadata_configs[0].name == "gaussdb"
+ assert metadata_configs[0].host == "gaussdb-env.example.com"
+ assert metadata_configs[0].port == 19995
+ assert metadata_configs[0].username == "zws"
+ assert metadata_configs[0].metadata_schema == "env_schema"
+
+
+def test_admin_config_warns_when_gaussdb_metadata_port_is_invalid(monkeypatch, caplog):
+ from admin.server import config as admin_config
+
+ monkeypatch.setenv("DB_TYPE", "gaussdb")
+ monkeypatch.setenv("GAUSSDB_METADATA_PORT", "1999O")
+ monkeypatch.setattr(admin_config, "read_config", lambda _path: {})
+ caplog.set_level(logging.WARNING)
+
+ configs = admin_config.load_configurations("ignored.yaml")
+ metadata_config = next(cfg for cfg in configs if cfg.service_type == "meta_data")
+
+ assert metadata_config.port == admin_config.GAUSSDB_ENV_DEFAULTS["port"]
+ assert "Ignoring invalid GAUSSDB_METADATA_PORT='1999O'; using default 8000" in caplog.text
+
+
+def test_docker_config_files_expose_gaussdb_metadata_settings():
+ env_text = read_repo_file("docker/.env")
+ template = read_repo_file("docker/service_conf.yaml.template")
+ source_conf = read_repo_file("conf/service_conf.yaml")
+
+ assert "DB_TYPE=${DB_TYPE:-mysql}" in env_text
+ assert "GAUSSDB_METADATA_HOST=" in env_text
+ assert "GAUSSDB_METADATA_PASSWORD=" in env_text
+ assert "GAUSSDB_METADATA_SCHEMA=public" in env_text
+ assert "gaussdb:" in template
+ assert template.count("\ngaussdb:") == 1
+ assert "GAUSSDB_METADATA_DBNAME" not in template
+ assert "gaussdb:" not in source_conf
+
+
+def test_docker_compose_metadata_profile_does_not_force_mysql_for_gaussdb():
+ env_text = read_repo_file("docker/.env")
+ base = load_yaml("docker/docker-compose-base.yml")
+ compose = load_yaml("docker/docker-compose.yml")
+ macos_compose = load_yaml("docker/docker-compose-macos.yml")
+ cn_compose = load_yaml("docker/docker-compose-CN-oc9.yml")
+
+ assert "METADATA_DB_PROFILE=${METADATA_DB_PROFILE:-mysql}" in env_text
+ assert "COMPOSE_PROFILES=${DOC_ENGINE},${DEVICE},metadata-${METADATA_DB_PROFILE}" in env_text
+ assert "metadata-mysql" in base["services"]["mysql"]["profiles"]
+ assert "metadata-gaussdb" not in base["services"]["mysql"]["profiles"]
+
+ for service_name in ["ragflow-cpu", "ragflow-gpu"]:
+ mysql_dep = compose["services"][service_name]["depends_on"]["mysql"]
+ assert mysql_dep["condition"] == "service_healthy"
+ assert mysql_dep["required"] is False
+
+ assert macos_compose["services"]["ragflow"]["depends_on"]["mysql"]["required"] is False
+ for service_name in ["ragflow-cpu", "ragflow-gpu"]:
+ assert cn_compose["services"][service_name]["depends_on"]["mysql"]["required"] is False
+
+
+def test_docker_launch_scripts_skip_mysql_migration_for_gaussdb():
+ entrypoint = read_repo_file("docker/entrypoint.sh")
+ launcher = read_repo_file("docker/launch_backend_service.sh")
+
+ assert 'DB_TYPE_NORMALIZED="${DB_TYPE:-mysql}"' in entrypoint
+ assert 'if [[ "${DB_TYPE_NORMALIZED}" == "gaussdb" || "${DB_TYPE_NORMALIZED}" == "gauss" ]]; then' in entrypoint
+ assert "Skipping MySQL-specific model provider table migrations" in entrypoint
+
+ assert 'local db_type="${DB_TYPE:-mysql}"' in launcher
+ assert 'if [ "$db_type" = "gaussdb" ] || [ "$db_type" = "gauss" ]; then' in launcher
+ assert "Skipping MySQL-specific model provider table migrations" in launcher
diff --git a/test/unit_test/api/db/test_gaussdb_error_code_helpers.py b/test/unit_test/api/db/test_gaussdb_error_code_helpers.py
new file mode 100644
index 0000000000..5d5dd3380e
--- /dev/null
+++ b/test/unit_test/api/db/test_gaussdb_error_code_helpers.py
@@ -0,0 +1,99 @@
+from peewee import InterfaceError, OperationalError, ProgrammingError
+
+
+class PgError(Exception):
+ def __init__(self, pgcode: str, message: str = "driver error"):
+ self.pgcode = pgcode
+ self.args = (message,)
+
+
+def peewee_error_with_context(exc_cls, pgcode: str, message: str):
+ exc = exc_cls(message)
+ # Peewee wraps psycopg2 exceptions and keeps the raw driver exception on
+ # __context__; live GaussDB probing confirmed pgcode is not on the outer
+ # Peewee exception itself.
+ exc.__context__ = PgError(pgcode, message)
+ return exc
+
+
+def test_sqlstate_extraction_reads_peewee_context():
+ from api.db.gaussdb_error_utils import sqlstate_from_exception
+
+ exc = peewee_error_with_context(ProgrammingError, "42701", "Column already exists")
+
+ assert sqlstate_from_exception(exc) == "42701"
+
+
+def test_gaussdb_retryable_transaction_errors_include_live_sqlstates(monkeypatch):
+ from api.db.gaussdb_error_utils import is_retryable_transaction_error
+ from api.db.services import common_service
+
+ monkeypatch.setattr(common_service.settings, "DATABASE_TYPE", "gaussdb")
+
+ for sqlstate in ("40P01", "40001", "55P03"):
+ exc = peewee_error_with_context(OperationalError, sqlstate, "retryable conflict")
+ assert is_retryable_transaction_error(exc)
+ assert common_service._is_deadlock_error(exc)
+
+ syntax_error = peewee_error_with_context(OperationalError, "42601", "syntax error")
+ assert not is_retryable_transaction_error(syntax_error)
+ assert not common_service._is_deadlock_error(syntax_error)
+
+
+def test_gaussdb_ddl_idempotency_error_helpers_use_live_sqlstates():
+ from api.db.gaussdb_error_utils import (
+ is_duplicate_column_error,
+ is_duplicate_object_error,
+ is_undefined_object_error,
+ )
+
+ assert is_duplicate_column_error(peewee_error_with_context(ProgrammingError, "42701", "Column already exists"))
+ assert is_duplicate_object_error(peewee_error_with_context(ProgrammingError, "42P07", "Relation already exists"))
+ assert is_undefined_object_error(peewee_error_with_context(ProgrammingError, "42704", "index does not exist"))
+
+ assert not is_duplicate_column_error(peewee_error_with_context(ProgrammingError, "42601", "syntax error"))
+ assert not is_duplicate_object_error(peewee_error_with_context(ProgrammingError, "23505", "duplicate key value"))
+ assert not is_undefined_object_error(peewee_error_with_context(ProgrammingError, "42P07", "already exists"))
+
+
+def test_connection_error_helper_covers_sqlstate_interface_and_gaussdb_ssl_close_text():
+ from api.db.gaussdb_error_utils import is_psycopg_connection_error
+
+ assert is_psycopg_connection_error(peewee_error_with_context(OperationalError, "08006", "connection failure"))
+ assert is_psycopg_connection_error(InterfaceError("connection already closed"))
+ assert is_psycopg_connection_error(OperationalError("SSL connection has been closed unexpectedly"))
+ assert not is_psycopg_connection_error(OperationalError("syntax error at or near FROM"))
+
+
+def test_non_connection_sqlstate_overrides_connection_shaped_text():
+ from api.db.gaussdb_error_utils import is_psycopg_connection_error
+
+ exc = peewee_error_with_context(OperationalError, "22P02", "invalid integer from connection-shaped input")
+
+ assert is_psycopg_connection_error(exc) is False
+
+
+def test_all_08_class_sqlstates_are_connection_errors():
+ from api.db.gaussdb_error_utils import is_psycopg_connection_error
+
+ assert is_psycopg_connection_error(peewee_error_with_context(OperationalError, "08002", "driver state 08002"))
+
+
+def test_no_sqlstate_eof_reset_is_a_connection_error():
+ from api.db.gaussdb_error_utils import is_psycopg_connection_error
+
+ assert is_psycopg_connection_error(OperationalError("SSL SYSCALL error: EOF detected"))
+
+
+def test_no_sqlstate_unrelated_connection_word_is_not_a_connection_error():
+ from api.db.gaussdb_error_utils import is_psycopg_connection_error
+
+ assert not is_psycopg_connection_error(OperationalError("invalid value for application connection label"))
+
+
+def test_nested_non_connection_sqlstate_overrides_outer_interface_error():
+ from api.db.gaussdb_error_utils import is_psycopg_connection_error
+
+ exc = peewee_error_with_context(InterfaceError, "22P02", "invalid integer from connection-shaped input")
+
+ assert is_psycopg_connection_error(exc) is False
diff --git a/test/unit_test/api/db/test_gaussdb_peewee.py b/test/unit_test/api/db/test_gaussdb_peewee.py
new file mode 100644
index 0000000000..df6c828ddd
--- /dev/null
+++ b/test/unit_test/api/db/test_gaussdb_peewee.py
@@ -0,0 +1,444 @@
+"""Tests for GaussDB Peewee ORM support."""
+
+import pytest
+from peewee import CharField, Model, OperationalError, SqliteDatabase
+from playhouse.pool import PooledPostgresqlDatabase
+
+from api.db.db_models import (
+ DatabaseLock,
+ DatabaseMigrator,
+ GaussDBDatabaseLock,
+ GaussDBMigrator,
+ PostgresDatabaseLock,
+ PooledDatabase,
+ RetryingPooledGaussDBDatabase,
+ RetryingPooledPostgresqlDatabase,
+ TextFieldType,
+)
+from common.settings import load_database_config, normalize_database_type
+import api.db.db_models as db_models
+
+
+class RecordingCursor:
+ def __init__(self, rows):
+ self.rows = rows
+
+ def fetchall(self):
+ return self.rows
+
+
+class LockCursor:
+ def __init__(self, value):
+ self.value = value
+
+ def fetchone(self):
+ return (self.value,)
+
+
+class SequenceLockDB:
+ def __init__(self, values):
+ self.values = iter(values)
+ self.calls = []
+
+ def execute_sql(self, sql, params=None):
+ self.calls.append((sql, params))
+ return LockCursor(next(self.values))
+
+
+class InitializationCursor:
+ def __init__(self, rows):
+ self.rows = iter(rows)
+ self.calls = []
+
+ def __enter__(self):
+ return self
+
+ def __exit__(self, exc_type, exc_value, traceback):
+ return False
+
+ def execute(self, sql, params=None):
+ self.calls.append((sql, params))
+
+ def fetchone(self):
+ return next(self.rows)
+
+
+class InitializationConnection:
+ def __init__(self, rows, cursor_class=InitializationCursor):
+ self.cursor_instance = cursor_class(rows)
+ self.rollback_calls = 0
+
+ def cursor(self):
+ return self.cursor_instance
+
+ def rollback(self):
+ self.rollback_calls += 1
+
+
+class UndefinedTableError(Exception):
+ pgcode = "42P01"
+
+
+class FeatureNotSupportedError(Exception):
+ pgcode = "0A000"
+
+
+class MissingPgxcNodeCursor(InitializationCursor):
+ def execute(self, sql, params=None):
+ super().execute(sql, params)
+ if isinstance(sql, str) and "pg_catalog.pgxc_node" in sql:
+ raise UndefinedTableError("relation pg_catalog.pgxc_node does not exist")
+
+
+class DefinitionOnlyPgxcNodeCursor(InitializationCursor):
+ def execute(self, sql, params=None):
+ super().execute(sql, params)
+ if isinstance(sql, str) and "pg_catalog.pgxc_node" in sql:
+ raise FeatureNotSupportedError("pgxc_node data cannot be queried in centralized mode")
+
+
+class UpdateByIdModel(Model):
+ id = CharField(primary_key=True)
+ name = CharField(null=True)
+
+ class Meta:
+ database = None
+
+
+class RecordingCloseDatabase:
+ def __init__(self, closed=False):
+ self.closed = closed
+ self.close_calls = 0
+ self.close_stale_calls = []
+
+ def __bool__(self):
+ return True
+
+ def is_closed(self):
+ return self.closed
+
+ def close(self):
+ self.close_calls += 1
+ self.closed = True
+
+ def close_stale(self, age):
+ self.close_stale_calls.append(age)
+
+
+class TestGaussDBDatabase:
+ """Test cases for GaussDB metadata database support."""
+
+ def test_gaussdb_database_class_exists(self):
+ assert RetryingPooledGaussDBDatabase is not None
+
+ def test_gaussdb_in_pooled_database_enum(self):
+ assert hasattr(PooledDatabase, "GAUSSDB")
+ assert PooledDatabase.GAUSSDB.value == RetryingPooledGaussDBDatabase
+
+ def test_gaussdb_in_database_migrator_enum(self):
+ assert hasattr(DatabaseMigrator, "GAUSSDB")
+ assert DatabaseMigrator.GAUSSDB.value == GaussDBMigrator
+
+ def test_gaussdb_in_database_lock_enum(self):
+ assert hasattr(DatabaseLock, "GAUSSDB")
+ assert DatabaseLock.GAUSSDB.value == GaussDBDatabaseLock
+
+ def test_gaussdb_in_text_field_type_enum(self):
+ assert hasattr(TextFieldType, "GAUSSDB")
+ assert TextFieldType.GAUSSDB.value == "TEXT"
+
+ def test_gaussdb_database_uses_dedicated_adapter(self):
+ assert not issubclass(RetryingPooledGaussDBDatabase, RetryingPooledPostgresqlDatabase)
+ assert issubclass(RetryingPooledGaussDBDatabase, PooledPostgresqlDatabase)
+ assert issubclass(GaussDBMigrator, DatabaseMigrator.POSTGRES.value)
+ assert issubclass(GaussDBDatabaseLock, PostgresDatabaseLock)
+
+ def test_gaussdb_database_init(self):
+ db = RetryingPooledGaussDBDatabase(
+ "test_db",
+ host="localhost",
+ port=8000,
+ user="gaussdb",
+ password="password",
+ )
+ assert db is not None
+ assert db.max_retries == 5
+ assert db.retry_delay == 1
+
+ def test_gaussdb_database_custom_retries(self):
+ db = RetryingPooledGaussDBDatabase(
+ "test_db",
+ host="localhost",
+ max_retries=10,
+ retry_delay=2,
+ )
+ assert db.max_retries == 10
+ assert db.retry_delay == 2
+
+ @pytest.mark.parametrize(("topology_row", "expected"), [((False,), False), ((True,), True)])
+ def test_gaussdb_initializes_search_path_and_detects_topology(self, topology_row, expected):
+ db = RetryingPooledGaussDBDatabase(None)
+ connection = InitializationConnection([("ragflow_meta",), topology_row])
+
+ db._initialize_connection(connection)
+
+ calls = connection.cursor_instance.calls
+ assert calls[0] == ("SELECT current_schema()", None)
+ assert "SET search_path TO" in str(calls[1][0])
+ assert "Identifier('ragflow_meta')" in str(calls[1][0])
+ assert "pg_catalog.pgxc_node" in calls[2][0]
+ assert db.is_distributed is expected
+
+ @pytest.mark.parametrize("cursor_class", [MissingPgxcNodeCursor, DefinitionOnlyPgxcNodeCursor])
+ def test_unavailable_pgxc_node_is_detected_as_centralized(self, cursor_class):
+ db = RetryingPooledGaussDBDatabase(None)
+ connection = InitializationConnection([("ragflow_meta",)], cursor_class=cursor_class)
+
+ db._initialize_connection(connection)
+
+ assert db.is_distributed is False
+ assert connection.rollback_calls == 1
+ assert "SET search_path TO" in str(connection.cursor_instance.calls[-1][0])
+
+ def test_first_connection_failure_is_retried_before_ddl_preparation(self, monkeypatch):
+ db = RetryingPooledGaussDBDatabase(None, max_retries=1, retry_delay=0)
+ connection_calls = []
+ executed = []
+ sql = 'CREATE TABLE IF NOT EXISTS "user" ("id" VARCHAR(32))'
+
+ def connection():
+ connection_calls.append(None)
+ if len(connection_calls) == 1:
+ raise OperationalError("connection refused")
+ db._is_distributed = True
+
+ def base_execute(_self, prepared_sql, params=None, commit=True):
+ executed.append((prepared_sql, params, commit))
+ return "cursor"
+
+ monkeypatch.setattr(db, "connection", connection)
+ monkeypatch.setattr(db, "_handle_connection_loss", lambda: None)
+ monkeypatch.setattr(PooledPostgresqlDatabase, "execute_sql", base_execute)
+
+ assert db.execute_sql(sql) == "cursor"
+ assert len(connection_calls) == 2
+ assert executed == [(f"{sql} DISTRIBUTE BY REPLICATION", None, True)]
+
+ def test_connection_failure_in_transaction_is_not_retried(self, monkeypatch):
+ db = RetryingPooledGaussDBDatabase(None, max_retries=1, retry_delay=0)
+ db._is_distributed = False
+ connection_error = OperationalError("connection refused")
+ executed = []
+ reconnect_calls = []
+
+ def base_execute(_self, sql, params=None, commit=True):
+ executed.append((sql, params, commit))
+ raise connection_error
+
+ monkeypatch.setattr(db, "in_transaction", lambda: True)
+ monkeypatch.setattr(db, "_handle_connection_loss", lambda: reconnect_calls.append(None))
+ monkeypatch.setattr(PooledPostgresqlDatabase, "execute_sql", base_execute)
+
+ with pytest.raises(OperationalError) as exc_info:
+ db.execute_sql("SELECT 1")
+
+ assert exc_info.value is connection_error
+ assert executed == [("SELECT 1", None, True)]
+ assert reconnect_calls == []
+
+ def test_postgresql_execution_keeps_sql_unchanged(self, monkeypatch):
+ db = RetryingPooledPostgresqlDatabase(None, max_retries=0)
+ executed = []
+
+ def base_execute(_self, sql, params=None, commit=True):
+ executed.append((sql, params, commit))
+ return "cursor"
+
+ monkeypatch.setattr(PooledPostgresqlDatabase, "execute_sql", base_execute)
+
+ assert db.execute_sql("SELECT 1") == "cursor"
+ assert executed == [("SELECT 1", None, True)]
+
+ def test_distributed_gaussdb_removes_only_redundant_primary_key_assignment(self):
+ db = RetryingPooledGaussDBDatabase(None)
+ db._is_distributed = True
+ source = {"id": "record-1", "name": "updated"}
+
+ prepared = db.prepare_update_by_id(UpdateByIdModel, "record-1", source)
+
+ assert prepared == {"name": "updated"}
+ assert prepared is not source
+ assert source == {"id": "record-1", "name": "updated"}
+
+ def test_distributed_gaussdb_replaces_row_for_primary_key_change(self, monkeypatch):
+ db = RetryingPooledGaussDBDatabase(None)
+ db._is_distributed = True
+ sqlite_db = SqliteDatabase(":memory:")
+
+ with sqlite_db.bind_ctx([UpdateByIdModel]):
+ sqlite_db.create_tables([UpdateByIdModel])
+ UpdateByIdModel.create(id="record-1", name="original")
+ monkeypatch.setattr(db, "atomic", sqlite_db.atomic)
+
+ assert db.replace_update_by_id(UpdateByIdModel, "record-1", {"id": "record-2", "name": "updated"}) == 1
+ assert UpdateByIdModel.get_or_none(UpdateByIdModel.id == "record-1") is None
+ assert UpdateByIdModel.get_by_id("record-2").name == "updated"
+ sqlite_db.close()
+
+ def test_distributed_gaussdb_replace_returns_zero_for_missing_row(self, monkeypatch):
+ db = RetryingPooledGaussDBDatabase(None)
+ db._is_distributed = True
+ sqlite_db = SqliteDatabase(":memory:")
+
+ with sqlite_db.bind_ctx([UpdateByIdModel]):
+ sqlite_db.create_tables([UpdateByIdModel])
+ monkeypatch.setattr(db, "atomic", sqlite_db.atomic)
+
+ assert db.replace_update_by_id(UpdateByIdModel, "missing", {"id": "record-2"}) == 0
+ sqlite_db.close()
+
+ def test_centralized_gaussdb_keeps_primary_key_assignment_unchanged(self):
+ db = RetryingPooledGaussDBDatabase(None)
+ db._is_distributed = False
+ source = {"id": "record-1", "name": "updated"}
+
+ assert db.prepare_update_by_id(UpdateByIdModel, "record-1", source) is source
+ assert db.replace_update_by_id(UpdateByIdModel, "record-1", {"id": "record-2"}) is None
+
+ def test_postgresql_adapter_has_no_gaussdb_update_hook(self):
+ assert not hasattr(RetryingPooledPostgresqlDatabase(None), "prepare_update_by_id")
+ assert not hasattr(RetryingPooledPostgresqlDatabase(None), "replace_update_by_id")
+
+ @pytest.mark.parametrize(
+ "table_name",
+ [
+ "compilation_template",
+ "compilation_template_group",
+ "file_commit_item",
+ "tenant_llm",
+ "tenant_model_provider",
+ "user",
+ ],
+ )
+ def test_distributed_gaussdb_replicates_independent_unique_key_tables(self, table_name):
+ db = RetryingPooledGaussDBDatabase(None)
+ db._is_distributed = True
+ sql = f'CREATE TABLE IF NOT EXISTS "{table_name}" ("id" VARCHAR(32))'
+
+ assert db._prepare_distributed_ddl(sql) == f"{sql} DISTRIBUTE BY REPLICATION"
+
+ def test_centralized_gaussdb_keeps_candidate_ddl_unchanged(self):
+ db = RetryingPooledGaussDBDatabase(None)
+ db._is_distributed = False
+ sql = 'CREATE TABLE IF NOT EXISTS "user" ("id" VARCHAR(32))'
+
+ assert db._prepare_distributed_ddl(sql) == sql
+
+ def test_gaussdb_get_tables_defaults_to_current_schema(self, monkeypatch):
+ db = RetryingPooledGaussDBDatabase(None)
+ monkeypatch.setattr(db, "execute_sql", lambda *_args, **_kwargs: RecordingCursor([("tenant",), ("user",)]))
+
+ assert db.get_tables() == ["tenant", "user"]
+
+ def test_gaussdb_request_cleanup_returns_current_connection(self, monkeypatch):
+ db = RecordingCloseDatabase()
+ monkeypatch.setattr(db_models, "DB", db)
+ monkeypatch.setattr(db_models.settings, "DATABASE_TYPE", "gaussdb")
+
+ db_models.close_connection()
+
+ assert db.close_calls == 1
+ assert db.close_stale_calls == []
+
+ def test_non_gaussdb_cleanup_preserves_existing_behavior(self, monkeypatch):
+ db = RecordingCloseDatabase()
+ monkeypatch.setattr(db_models, "DB", db)
+ monkeypatch.setattr(db_models.settings, "DATABASE_TYPE", "mysql")
+
+ db_models.close_connection()
+
+ assert db.close_calls == 0
+ assert db.close_stale_calls == [30]
+
+ def test_pooled_database_enum_values(self):
+ expected = {"MYSQL", "OCEANBASE", "POSTGRES", "GAUSSDB"}
+ actual = {e.name for e in PooledDatabase}
+ assert expected.issubset(actual), f"Missing: {expected - actual}"
+
+ def test_database_lock_enum_values(self):
+ expected = {"MYSQL", "OCEANBASE", "POSTGRES", "GAUSSDB"}
+ actual = set(DatabaseLock.__members__.keys())
+ assert expected.issubset(actual), f"Missing: {expected - actual}"
+
+ def test_gaussdb_lock_waits_until_holder_releases(self, monkeypatch):
+ now = [10.0]
+ sleeps = []
+ monkeypatch.setattr(db_models.time, "monotonic", lambda: now[0])
+
+ def sleep(seconds):
+ sleeps.append(seconds)
+ now[0] += seconds
+
+ monkeypatch.setattr(db_models.time, "sleep", sleep)
+ db = SequenceLockDB([False, False, True])
+
+ assert GaussDBDatabaseLock("init_database_tables", timeout=1.0, db=db).lock()
+ assert sleeps == [0.1, 0.1]
+
+ def test_gaussdb_lock_stops_at_timeout(self, monkeypatch):
+ now = [20.0]
+ monkeypatch.setattr(db_models.time, "monotonic", lambda: now[0])
+ monkeypatch.setattr(db_models.time, "sleep", lambda seconds: now.__setitem__(0, now[0] + seconds))
+ db = SequenceLockDB([False, False, False, False])
+
+ with pytest.raises(Exception, match="acquire gaussdb lock init_database_tables timeout"):
+ GaussDBDatabaseLock("init_database_tables", timeout=0.25, db=db).lock()
+
+ assert now[0] == pytest.approx(20.25)
+
+
+class TestGaussDBConfiguration:
+ """Test cases for DB_TYPE normalization."""
+
+ def test_settings_can_use_gaussdb_case_insensitive(self):
+ assert normalize_database_type("GaussDB") == "gaussdb"
+ assert normalize_database_type("gaussdb") == "gaussdb"
+
+ def test_settings_accepts_gauss_alias(self):
+ assert normalize_database_type("gauss") == "gaussdb"
+
+ def test_load_database_config_uses_gaussdb_env(self, monkeypatch):
+ monkeypatch.delenv("GAUSSDB_METADATA_OPTIONS", raising=False)
+ monkeypatch.setenv("GAUSSDB_METADATA_HOST", "127.0.0.1")
+ monkeypatch.setenv("GAUSSDB_METADATA_PORT", "19995")
+ monkeypatch.setenv("GAUSSDB_METADATA_USER", "zws")
+ monkeypatch.setenv("GAUSSDB_METADATA_PASSWORD", "secret")
+ monkeypatch.setenv("GAUSSDB_METADATA_DBNAME", "zws_test")
+ monkeypatch.setenv("GAUSSDB_METADATA_SCHEMA", "ragflow_meta")
+
+ config = load_database_config("GaussDB")
+
+ assert config["host"] == "127.0.0.1"
+ assert config["port"] == 19995
+ assert config["user"] == "zws"
+ assert config["password"] == "secret"
+ assert config["name"] == "zws_test"
+ assert config["options"] == "-c search_path=ragflow_meta -c client_encoding=UTF8 -c default_transaction_read_only=off"
+
+ def test_load_database_config_defaults_schema_to_public(self, monkeypatch):
+ monkeypatch.delenv("GAUSSDB_METADATA_SCHEMA", raising=False)
+ monkeypatch.delenv("GAUSSDB_METADATA_OPTIONS", raising=False)
+
+ assert load_database_config("gaussdb")["options"].startswith("-c search_path=public ")
+
+ def test_load_database_config_rejects_unsafe_schema(self, monkeypatch):
+ monkeypatch.setenv("GAUSSDB_METADATA_SCHEMA", "public;drop schema public")
+
+ with pytest.raises(ValueError, match="GAUSSDB_METADATA_SCHEMA"):
+ load_database_config("gaussdb")
+
+ def test_explicit_options_override_generated_schema_option(self, monkeypatch):
+ monkeypatch.setenv("GAUSSDB_METADATA_SCHEMA", "ragflow_meta")
+ monkeypatch.setenv("GAUSSDB_METADATA_OPTIONS", "-c search_path=custom_meta -c statement_timeout=5000")
+
+ assert load_database_config("gaussdb")["options"] == "-c search_path=custom_meta -c statement_timeout=5000"
diff --git a/test/unit_test/api/db/test_gaussdb_user_flow.py b/test/unit_test/api/db/test_gaussdb_user_flow.py
new file mode 100644
index 0000000000..3e22a21bc4
--- /dev/null
+++ b/test/unit_test/api/db/test_gaussdb_user_flow.py
@@ -0,0 +1,301 @@
+"""
+User-flow tests for running RAGFlow with DB_TYPE=GaussDB.
+"""
+
+import os
+import subprocess
+import sys
+import textwrap
+from pathlib import Path
+
+import pytest
+
+
+REPO_ROOT = Path(__file__).resolve().parents[4]
+HELPER_DIR = Path(__file__).resolve().parent
+
+
+def run_isolated_flow(script: str, env_extra: dict[str, str], timeout: int = 60):
+ env = os.environ.copy()
+ env.update(env_extra)
+ env["PYTHONPATH"] = os.pathsep.join(
+ [
+ str(REPO_ROOT),
+ str(HELPER_DIR),
+ env.get("PYTHONPATH", ""),
+ ]
+ )
+
+ bootstrap = """
+from gaussdb_test_utils import stub_settings_import_dependencies
+stub_settings_import_dependencies()
+"""
+ result = subprocess.run(
+ [sys.executable, "-c", textwrap.dedent(bootstrap + script)],
+ cwd=REPO_ROOT,
+ env=env,
+ text=True,
+ capture_output=True,
+ timeout=timeout,
+ )
+ assert result.returncode == 0, f"stdout:\n{result.stdout}\nstderr:\n{result.stderr}"
+ return result
+
+
+def test_user_env_selects_gaussdb_metadata_adapter(monkeypatch):
+ monkeypatch.delenv("GAUSSDB_METADATA_OPTIONS", raising=False)
+ result = run_isolated_flow(
+ """
+from common import settings
+from api.db.db_models import DB, LongTextField
+
+assert settings.DATABASE_TYPE == "gaussdb"
+assert settings.DATABASE["host"] == "gaussdb.local"
+assert settings.DATABASE["port"] == 8000
+assert settings.DATABASE["user"] == "rag_flow"
+assert settings.DATABASE["name"] == "rag_flow"
+assert settings.DATABASE["options"] == "-c search_path=ragflow_meta -c client_encoding=UTF8 -c default_transaction_read_only=off"
+assert DB.__class__.__name__ == "RetryingPooledGaussDBDatabase"
+assert DB.lock.__name__ == "GaussDBDatabaseLock"
+assert LongTextField.field_type == "TEXT"
+print("gaussdb-adapter-ok")
+""",
+ {
+ "DB_TYPE": "GaussDB",
+ "GAUSSDB_METADATA_HOST": "gaussdb.local",
+ "GAUSSDB_METADATA_PORT": "8000",
+ "GAUSSDB_METADATA_USER": "rag_flow",
+ "GAUSSDB_METADATA_PASSWORD": "secret",
+ "GAUSSDB_METADATA_DBNAME": "rag_flow",
+ "GAUSSDB_METADATA_SCHEMA": "ragflow_meta",
+ },
+ )
+
+ assert "gaussdb-adapter-ok" in result.stdout
+
+
+def test_gaussdb_empty_string_compatible_fields_are_nullable_only_for_gaussdb():
+ gaussdb_result = run_isolated_flow(
+ """
+from api.db.db_models import API4Conversation, Dialog, File, Knowledgebase, Memory, SyncLogs, SystemSettings, Task, Tenant, User, UserCanvas
+
+fields = [
+ User.nickname,
+ Tenant.llm_id,
+ Tenant.embd_id,
+ Tenant.asr_id,
+ Tenant.img2txt_id,
+ Tenant.rerank_id,
+ File.source_type,
+ Knowledgebase.embd_id,
+ Dialog.llm_id,
+ Dialog.rerank_id,
+ Memory.embd_id,
+ Memory.llm_id,
+ SystemSettings.value,
+ Task.task_type,
+ Task.progress_msg,
+ Task.chunk_ids,
+ SyncLogs.error_msg,
+ SyncLogs.full_exception_trace,
+ API4Conversation.user_id,
+ UserCanvas.tags,
+]
+for field in fields:
+ assert field.null is True
+ assert field.db_value("") is None
+ assert field.python_value(None) == ""
+
+eq_sql, eq_params = File.select().where(File.source_type == "").sql()
+ne_sql, ne_params = File.select().where(File.source_type != "").sql()
+field_eq_sql, field_eq_params = Tenant.select().where(Tenant.llm_id == Tenant.embd_id).sql()
+field_ne_sql, field_ne_params = Tenant.select().where(Tenant.llm_id != Tenant.embd_id).sql()
+
+assert '"source_type" IS NULL' in eq_sql
+assert 'LENGTH("t1"."source_type") = %s' in eq_sql
+assert eq_params == [0]
+assert 'LENGTH("t1"."source_type") > %s' in ne_sql
+assert ne_params == [0]
+assert '"t1"."llm_id" = "t1"."embd_id"' in field_eq_sql
+assert field_eq_params == []
+assert '"t1"."llm_id" != "t1"."embd_id"' in field_ne_sql
+assert field_ne_params == []
+print("gaussdb-empty-string-compatible-ok")
+""",
+ {
+ "DB_TYPE": "GaussDB",
+ "GAUSSDB_METADATA_HOST": "gaussdb.local",
+ "GAUSSDB_METADATA_PORT": "8000",
+ "GAUSSDB_METADATA_USER": "rag_flow",
+ "GAUSSDB_METADATA_PASSWORD": "secret",
+ "GAUSSDB_METADATA_DBNAME": "rag_flow",
+ },
+ )
+ assert "gaussdb-empty-string-compatible-ok" in gaussdb_result.stdout
+
+ mysql_result = run_isolated_flow(
+ """
+from api.db.db_models import API4Conversation, Dialog, File, Knowledgebase, Memory, SyncLogs, SystemSettings, Task, Tenant, User, UserCanvas
+
+fields = [
+ (User.nickname, False),
+ (Tenant.llm_id, False),
+ (Tenant.embd_id, False),
+ (Tenant.asr_id, False),
+ (Tenant.img2txt_id, False),
+ (Tenant.rerank_id, False),
+ (File.source_type, False),
+ (Knowledgebase.embd_id, False),
+ (Dialog.llm_id, False),
+ (Dialog.rerank_id, False),
+ (Memory.embd_id, False),
+ (Memory.llm_id, False),
+ (SystemSettings.value, False),
+ (Task.task_type, False),
+ (Task.progress_msg, True),
+ (Task.chunk_ids, True),
+ (SyncLogs.error_msg, False),
+ (SyncLogs.full_exception_trace, True),
+ (API4Conversation.user_id, False),
+ (UserCanvas.tags, False),
+]
+for field, expected_null in fields:
+ assert field.null is expected_null
+ assert field.db_value("") == ""
+
+assert Task.chunk_ids.field_type == "LONGTEXT"
+sql, params = File.select().where(File.source_type == "").sql()
+assert "IS NULL" not in sql
+assert "LENGTH" not in sql
+assert params == [""]
+print("mysql-nullability-ok")
+""",
+ {
+ "DB_TYPE": "mysql",
+ },
+ )
+ assert "mysql-nullability-ok" in mysql_result.stdout
+
+
+def test_live_gaussdb_init_lock_and_user_crud_flow():
+ if os.getenv("RAGFLOW_GAUSSDB_LIVE") != "1":
+ pytest.skip("Set RAGFLOW_GAUSSDB_LIVE=1 and GAUSSDB_METADATA_* env vars to run live GaussDB flow")
+
+ required = ["GAUSSDB_METADATA_HOST", "GAUSSDB_METADATA_PORT", "GAUSSDB_METADATA_USER", "GAUSSDB_METADATA_PASSWORD", "GAUSSDB_METADATA_DBNAME"]
+ missing = [name for name in required if not os.getenv(name)]
+ if missing:
+ pytest.skip(f"Missing live GaussDB env vars: {', '.join(missing)}")
+
+ env_extra = {
+ "DB_TYPE": "GaussDB",
+ "GAUSSDB_METADATA_HOST": os.environ["GAUSSDB_METADATA_HOST"],
+ "GAUSSDB_METADATA_PORT": os.environ["GAUSSDB_METADATA_PORT"],
+ "GAUSSDB_METADATA_USER": os.environ["GAUSSDB_METADATA_USER"],
+ "GAUSSDB_METADATA_PASSWORD": os.environ["GAUSSDB_METADATA_PASSWORD"],
+ "GAUSSDB_METADATA_DBNAME": os.environ["GAUSSDB_METADATA_DBNAME"],
+ "GAUSSDB_METADATA_SCHEMA": os.getenv("GAUSSDB_METADATA_SCHEMA", "public"),
+ "GAUSSDB_METADATA_MAX_CONNECTIONS": os.getenv("GAUSSDB_METADATA_MAX_CONNECTIONS", "20"),
+ "GAUSSDB_METADATA_STALE_TIMEOUT": os.getenv("GAUSSDB_METADATA_STALE_TIMEOUT", "30"),
+ }
+
+ result = run_isolated_flow(
+ """
+import os
+import uuid
+
+from common import settings
+from api.db.db_models import DB, Tenant, User, init_database_tables
+from api.db.services.user_service import TenantService
+from api.utils.health_utils import get_database_status
+
+assert settings.DATABASE_TYPE == "gaussdb"
+init_database_tables()
+
+expected_distributed = os.environ.get("EXPECT_DISTRIBUTED") == "1"
+assert DB.is_distributed is expected_distributed
+if expected_distributed:
+ rows = DB.execute_sql(
+ "SELECT c.relname, x.pclocatortype "
+ "FROM pg_catalog.pgxc_class x "
+ "JOIN pg_catalog.pg_class c ON c.oid = x.pcrelid "
+ "JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace "
+ "WHERE n.nspname = current_schema() "
+ "AND c.relname IN ('compilation_template', 'compilation_template_group', "
+ "'file_commit_item', 'tenant_llm', 'tenant_model_provider', 'user')"
+ ).fetchall()
+ assert dict(rows) == {
+ "compilation_template": "R",
+ "compilation_template_group": "R",
+ "file_commit_item": "R",
+ "tenant_llm": "R",
+ "tenant_model_provider": "R",
+ "user": "R",
+ }
+
+health = get_database_status()
+assert health["status"] == "alive"
+assert health["message"]["database"] == "gaussdb"
+assert health["message"]["result"] == 1
+
+uid = uuid.uuid4().hex
+email = f"{uid}@ragflow-gaussdb-flow.test"
+tenant_id = uuid.uuid4().hex
+replacement_tenant_id = uuid.uuid4().hex
+
+with DB.connection_context():
+ with DB.lock("gaussdb_user_flow_test", 10):
+ try:
+ User.create(id=uid, email=email, nickname="gaussdb-flow", password="x")
+ row = User.get(User.id == uid)
+ assert row.id == uid
+ assert row.email == email
+ finally:
+ User.delete().where(User.id == uid).execute()
+
+ assert User.select().where(User.id == uid).count() == 0
+
+ try:
+ Tenant.create(
+ id=tenant_id,
+ name="gaussdb-empty-string-flow",
+ llm_id="",
+ embd_id="",
+ asr_id="",
+ img2txt_id="",
+ rerank_id="",
+ parser_ids="naive:General",
+ )
+ raw_row = DB.execute_sql(
+ "select llm_id, embd_id, asr_id, img2txt_id, rerank_id from tenant where id=%s",
+ (tenant_id,),
+ ).fetchone()
+ assert raw_row == (None, None, None, None, None)
+
+ tenant = Tenant.get(Tenant.id == tenant_id)
+ assert tenant.llm_id == ""
+ assert tenant.embd_id == ""
+ assert tenant.asr_id == ""
+ assert tenant.img2txt_id == ""
+ assert tenant.rerank_id == ""
+
+ assert Tenant.select().where(Tenant.id == tenant_id, Tenant.llm_id == "").count() == 1
+ assert Tenant.select().where(Tenant.id == tenant_id, Tenant.llm_id != "").count() == 0
+
+ assert TenantService.update_by_id(
+ tenant_id,
+ {"id": replacement_tenant_id, "name": "gaussdb-primary-key-replacement"},
+ ) == 1
+ assert Tenant.select().where(Tenant.id == tenant_id).count() == 0
+ assert Tenant.select().where(Tenant.id == replacement_tenant_id).count() == 1
+ finally:
+ Tenant.delete().where(Tenant.id == tenant_id).execute()
+ Tenant.delete().where(Tenant.id == replacement_tenant_id).execute()
+
+DB.close()
+print("gaussdb-live-flow-ok")
+""",
+ env_extra,
+ timeout=120,
+ )
+
+ assert "gaussdb-live-flow-ok" in result.stdout
diff --git a/test/unit_test/api/utils/test_gaussdb_health.py b/test/unit_test/api/utils/test_gaussdb_health.py
new file mode 100644
index 0000000000..3de468d753
--- /dev/null
+++ b/test/unit_test/api/utils/test_gaussdb_health.py
@@ -0,0 +1,381 @@
+#
+# Copyright 2025 The InfiniFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+import importlib.util
+import json
+from pathlib import Path
+import sys
+import types
+from unittest.mock import Mock
+
+import pytest
+
+CFG614_PASSWORD_SENTINEL = "secret123"
+PASSWORD_SENTINEL = "cfg711-'quoted;password'"
+FULL_DSN_SENTINEL = "postgresql://sqlbuilder:cfg711-dsn%27%22%[email protected]:19995/postgres?schema=private"
+ACCESS_TOKEN_SENTINEL = "cfg711-token-'quoted![]{}'"
+
+
+class FakeGaussDBConnection:
+ def db_type(self):
+ return "gaussdb"
+
+ def health(self):
+ return {
+ "status": "healthy",
+ "uri": "[email protected]:19995/postgres?schema=public",
+ "version_comment": "GaussDB",
+ "sql_compatibility": "A",
+ }
+
+ def get_performance_metrics(self):
+ return {"connection": "connected", "latency_ms": 3.2}
+
+
+class FailingGaussDBConnection:
+ def health(self):
+ raise RuntimeError(f"FATAL: password authentication failed password={CFG614_PASSWORD_SENTINEL} dsn={FULL_DSN_SENTINEL} access_token={ACCESS_TOKEN_SENTINEL}")
+
+
+class UnhealthyGaussDBConnection(FakeGaussDBConnection):
+ def health(self):
+ return {
+ "status": "unhealthy",
+ "error": "compatibility unsupported password=fake-unit-password",
+ }
+
+ def get_performance_metrics(self):
+ return {"connection": "connected", "latency_ms": 1.2, "error": "dsn password:fake-unit-password"}
+
+
+class IncompatibleGaussDBConnection(FakeGaussDBConnection):
+ def health(self):
+ return {
+ "status": "unhealthy",
+ "uri": "[email protected]:19995/postgres?schema=public",
+ "version_comment": "GaussDB",
+ "sql_compatibility": "PG",
+ "error": "unsupported GaussDB compatibility, expected A/ORA: sql_compatibility=PG",
+ }
+
+
+class SecretBearingGaussDBConnection(FakeGaussDBConnection):
+ def health(self):
+ return {
+ "status": "unhealthy",
+ "schema": "public",
+ "credentials": {
+ "password": PASSWORD_SENTINEL,
+ "connections": [{"dsn": FULL_DSN_SENTINEL}, ({"ACCESS_TOKEN": ACCESS_TOKEN_SENTINEL},)],
+ },
+ "error": f'authentication failed password="{PASSWORD_SENTINEL}"',
+ }
+
+ def get_performance_metrics(self):
+ return {
+ "connection": "disconnected",
+ "schema": "public",
+ "details": [
+ {"dsn": FULL_DSN_SENTINEL},
+ ({"access-token": ACCESS_TOKEN_SENTINEL},),
+ ],
+ }
+
+
+def _install_module(monkeypatch, name, **attrs):
+ mod = types.ModuleType(name)
+ for key, value in attrs.items():
+ setattr(mod, key, value)
+ monkeypatch.setitem(sys.modules, name, mod)
+ return mod
+
+
+def _import_health_utils(monkeypatch):
+ import common
+
+ settings_stub = types.SimpleNamespace(
+ docStoreConn=types.SimpleNamespace(health=lambda: {"status": "healthy"}),
+ STORAGE_IMPL=types.SimpleNamespace(health=lambda: True),
+ STORAGE_IMPL_TYPE="MINIO",
+ MINIO={"host": "minio:9000"},
+ )
+ monkeypatch.setitem(sys.modules, "common.settings", settings_stub)
+ monkeypatch.setattr(common, "settings", settings_stub, raising=False)
+
+ _install_module(monkeypatch, "api.db.db_models", DB=types.SimpleNamespace(execute_sql=lambda *_args, **_kwargs: True))
+ _install_module(monkeypatch, "rag.utils.redis_conn", REDIS_CONN=types.SimpleNamespace(health=lambda: True))
+ _install_module(monkeypatch, "rag.utils.es_conn", ESConnection=lambda: types.SimpleNamespace(get_cluster_stats=lambda: {}))
+ _install_module(monkeypatch, "rag.utils.infinity_conn", InfinityConnection=lambda: types.SimpleNamespace(health=lambda: {}))
+ _install_module(monkeypatch, "rag.utils.ob_conn", OBConnection=lambda: types.SimpleNamespace(health=lambda: {}, get_performance_metrics=lambda: {}))
+ _install_module(monkeypatch, "rag.utils.gaussdb_conn", GaussDBConnection=FakeGaussDBConnection)
+
+ module_path = Path(__file__).resolve().parents[4] / "api" / "utils" / "health_utils.py"
+ spec = importlib.util.spec_from_file_location("_gaussdb_health_utils_under_test", module_path)
+ assert spec and spec.loader
+ module = importlib.util.module_from_spec(spec)
+ spec.loader.exec_module(module)
+ return module
+
+
+def test_tc_cfg_611_get_gaussdb_status_returns_complete_not_configured_response(monkeypatch):
+ health_utils = _import_health_utils(monkeypatch)
+ monkeypatch.setenv("DOC_ENGINE", "elasticsearch")
+
+ result = health_utils.get_gaussdb_status()
+
+ assert result == {
+ "status": "not_configured",
+ "message": "GaussDB is not configured as the document engine",
+ }
+
+
+def test_tc_cfg_616_get_gaussdb_status_composes_healthy_probe_results(monkeypatch):
+ health_utils = _import_health_utils(monkeypatch)
+ monkeypatch.setenv("DOC_ENGINE", "gaussdb")
+ connection = FakeGaussDBConnection()
+ get_connection = Mock(return_value=connection)
+ monkeypatch.setattr(health_utils, "_get_gaussdb_connection", get_connection)
+
+ result = health_utils.get_gaussdb_status()
+
+ assert result == {
+ "status": "alive",
+ "message": {
+ "health": {
+ "status": "healthy",
+ "uri": "[email protected]:19995/postgres?schema=public",
+ "version_comment": "GaussDB",
+ "sql_compatibility": "A",
+ },
+ "performance": {"connection": "connected", "latency_ms": 3.2},
+ },
+ }
+ get_connection.assert_called_once_with()
+
+
+def test_tc_cfg_617_get_gaussdb_status_reuses_settings_doc_store_connection(monkeypatch):
+ health_utils = _import_health_utils(monkeypatch)
+ doc_store_conn = FakeGaussDBConnection()
+ monkeypatch.setattr(health_utils.settings, "docStoreConn", doc_store_conn, raising=False)
+ monkeypatch.setenv("DOC_ENGINE", "gaussdb")
+ constructor = Mock(side_effect=AssertionError("unexpected GaussDBConnection construction"))
+ monkeypatch.setattr(health_utils, "GaussDBConnection", constructor)
+
+ result = health_utils.get_gaussdb_status()
+
+ assert result["status"] == "alive"
+ assert result["message"]["performance"]["connection"] == "connected"
+ constructor.assert_not_called()
+
+
+def test_tc_cfg_618_get_gaussdb_status_constructs_adapter_without_shared_connection(monkeypatch):
+ health_utils = _import_health_utils(monkeypatch)
+ connection = object()
+ constructor = Mock(return_value=connection)
+ monkeypatch.setattr(health_utils.settings, "docStoreConn", None, raising=False)
+ monkeypatch.setattr(health_utils, "GaussDBConnection", constructor)
+
+ resolved = health_utils._get_gaussdb_connection()
+
+ assert resolved is connection
+ constructor.assert_called_once_with()
+
+
+def test_tc_cfg_618_get_gaussdb_status_constructs_adapter_for_non_gaussdb_connection(monkeypatch):
+ health_utils = _import_health_utils(monkeypatch)
+ connection = object()
+ constructor = Mock(return_value=connection)
+ other_doc_store = types.SimpleNamespace(db_type=lambda: "elasticsearch")
+ monkeypatch.setattr(health_utils.settings, "docStoreConn", other_doc_store, raising=False)
+ monkeypatch.setattr(health_utils, "GaussDBConnection", constructor)
+
+ resolved = health_utils._get_gaussdb_connection()
+
+ assert resolved is connection
+ constructor.assert_called_once_with()
+
+
+def test_tc_cfg_614_get_gaussdb_status_error_masks_password(monkeypatch, caplog):
+ health_utils = _import_health_utils(monkeypatch)
+ monkeypatch.setenv("DOC_ENGINE", "gaussdb")
+ monkeypatch.setattr(health_utils, "_get_gaussdb_connection", Mock(return_value=FailingGaussDBConnection()))
+ caplog.set_level("ERROR")
+
+ result = health_utils.get_gaussdb_status()
+ log_text = caplog.text
+
+ assert result["status"] == "timeout"
+ assert CFG614_PASSWORD_SENTINEL not in result["message"]
+ assert FULL_DSN_SENTINEL not in result["message"]
+ assert ACCESS_TOKEN_SENTINEL not in result["message"]
+ assert "postgresql://" not in result["message"]
+ assert "***" in result["message"]
+ assert "GaussDB status check failed (RuntimeError)" in log_text
+ assert "***" in log_text
+ assert CFG614_PASSWORD_SENTINEL not in log_text
+ assert FULL_DSN_SENTINEL not in log_text
+ assert ACCESS_TOKEN_SENTINEL not in log_text
+
+
+def test_tc_cfg_613_get_gaussdb_status_reports_timeout_when_health_is_unhealthy(monkeypatch):
+ health_utils = _import_health_utils(monkeypatch)
+ monkeypatch.setenv("DOC_ENGINE", "gaussdb")
+ monkeypatch.setattr(health_utils, "_get_gaussdb_connection", Mock(return_value=UnhealthyGaussDBConnection()))
+
+ result = health_utils.get_gaussdb_status()
+
+ assert result["status"] == "timeout"
+ assert result["message"]["health"]["status"] == "unhealthy"
+ assert result["message"]["health"]["error"] == "compatibility unsupported password=***"
+ assert result["message"]["performance"]["connection"] == "connected"
+ assert result["message"]["performance"]["latency_ms"] == 1.2
+ assert result["message"]["performance"]["error"] == "dsn password:***"
+
+
+def test_tc_cfg_619_check_gaussdb_health_reports_healthy_details(monkeypatch):
+ health_utils = _import_health_utils(monkeypatch)
+ monkeypatch.setenv("DOC_ENGINE", "gaussdb")
+ monkeypatch.setattr(
+ health_utils,
+ "_get_gaussdb_connection",
+ Mock(return_value=FakeGaussDBConnection()),
+ )
+
+ result = health_utils.check_gaussdb_health()
+
+ assert result["status"] == "healthy"
+ assert result["details"]["connection"] == "connected"
+ assert result["details"]["latency_ms"] == 3.2
+ assert result["details"]["sql_compatibility"] == "A"
+
+
+def test_tc_cfg_620_check_gaussdb_health_reports_incompatible_mode(monkeypatch):
+ health_utils = _import_health_utils(monkeypatch)
+ monkeypatch.setenv("DOC_ENGINE", "gaussdb")
+ monkeypatch.setattr(
+ health_utils,
+ "_get_gaussdb_connection",
+ Mock(return_value=IncompatibleGaussDBConnection()),
+ )
+
+ result = health_utils.check_gaussdb_health()
+
+ assert result["status"] == "unhealthy"
+ assert result["details"]["version"] == "GaussDB"
+ assert result["details"]["sql_compatibility"] == "PG"
+ assert "A/ORA" in result["details"]["error"]
+
+
+def test_tc_cfg_614_check_gaussdb_health_logs_masked_exception(monkeypatch, caplog):
+ health_utils = _import_health_utils(monkeypatch)
+ monkeypatch.setenv("DOC_ENGINE", "gaussdb")
+ monkeypatch.setattr(health_utils, "_get_gaussdb_connection", Mock(return_value=FailingGaussDBConnection()))
+ caplog.set_level("ERROR")
+
+ result = health_utils.check_gaussdb_health()
+ log_text = caplog.text
+
+ assert result["status"] == "unhealthy"
+ assert result["details"]["connection"] == "disconnected"
+ assert "GaussDB health check failed (RuntimeError)" in log_text
+ assert "***" in log_text
+ assert CFG614_PASSWORD_SENTINEL not in log_text
+ assert FULL_DSN_SENTINEL not in log_text
+ assert ACCESS_TOKEN_SENTINEL not in log_text
+
+
+def test_tc_cfg_711_check_gaussdb_health_masks_nested_performance_errors(monkeypatch):
+ health_utils = _import_health_utils(monkeypatch)
+ monkeypatch.setenv("DOC_ENGINE", "gaussdb")
+ monkeypatch.setattr(
+ health_utils,
+ "_get_gaussdb_connection",
+ Mock(return_value=SecretBearingGaussDBConnection()),
+ )
+
+ result = health_utils.check_gaussdb_health()
+ serialized = json.dumps(result, sort_keys=True)
+
+ assert result["status"] == "unhealthy"
+ assert PASSWORD_SENTINEL not in serialized
+ assert FULL_DSN_SENTINEL not in serialized
+ assert ACCESS_TOKEN_SENTINEL not in serialized
+
+
+def test_tc_cfg_707_mask_gaussdb_string_masks_password_equals(monkeypatch):
+ health_utils = _import_health_utils(monkeypatch)
+
+ assert health_utils._mask_gaussdb_string("host=h password=secret port=5432") == "host=h password=*** port=5432"
+
+
+def test_tc_cfg_708_mask_gaussdb_string_masks_password_colon(monkeypatch):
+ health_utils = _import_health_utils(monkeypatch)
+
+ assert health_utils._mask_gaussdb_string("connect failed password:fake-unit-password") == "connect failed password:***"
+
+
[email protected]("value", ["PASSWORD=fake-unit-password", "Password=fake-unit-password"])
+def test_tc_cfg_709_mask_gaussdb_string_is_case_insensitive(monkeypatch, value):
+ health_utils = _import_health_utils(monkeypatch)
+
+ assert health_utils._mask_gaussdb_string(value).lower() == "password=***"
+
+
+def test_tc_cfg_710_mask_gaussdb_secret_masks_nested_lists_and_tuples(monkeypatch):
+ health_utils = _import_health_utils(monkeypatch)
+
+ masked = health_utils._mask_gaussdb_secret(
+ {
+ "nested": {
+ "password": PASSWORD_SENTINEL,
+ "error": f'password="{PASSWORD_SENTINEL}"',
+ },
+ "containers": [
+ {"ACCESS_TOKEN": ACCESS_TOKEN_SENTINEL},
+ ({"dsn": FULL_DSN_SENTINEL},),
+ ],
+ }
+ )
+ serialized = json.dumps(masked, sort_keys=True)
+
+ assert masked["nested"]["password"] == "***"
+ assert "***" in masked["nested"]["error"]
+ assert masked["containers"][0]["ACCESS_TOKEN"] == "***"
+ assert masked["containers"][1][0]["dsn"] == "***"
+ assert PASSWORD_SENTINEL not in serialized
+ assert FULL_DSN_SENTINEL not in serialized
+ assert ACCESS_TOKEN_SENTINEL not in serialized
+
+
+def test_tc_cfg_711_get_gaussdb_status_masks_password_dsn_and_access_token_sentinels(monkeypatch):
+ health_utils = _import_health_utils(monkeypatch)
+ monkeypatch.setenv("DOC_ENGINE", "gaussdb")
+ monkeypatch.setattr(health_utils, "_get_gaussdb_connection", Mock(return_value=SecretBearingGaussDBConnection()))
+
+ result = health_utils.get_gaussdb_status()
+ serialized = json.dumps(result, sort_keys=True)
+
+ assert result["status"] == "timeout"
+ assert result["message"]["health"]["status"] == "unhealthy"
+ assert result["message"]["health"]["schema"] == "public"
+ assert result["message"]["performance"]["connection"] == "disconnected"
+ assert result["message"]["performance"]["schema"] == "public"
+ assert result["message"]["health"]["credentials"]["password"] == "***"
+ assert result["message"]["health"]["credentials"]["connections"][0]["dsn"] == "***"
+ assert result["message"]["health"]["credentials"]["connections"][1][0]["ACCESS_TOKEN"] == "***"
+ assert PASSWORD_SENTINEL not in serialized
+ assert FULL_DSN_SENTINEL not in serialized
+ assert ACCESS_TOKEN_SENTINEL not in serialized
+ assert "postgresql://" not in serialized
diff --git a/test/unit_test/common/test_gaussdb_settings.py b/test/unit_test/common/test_gaussdb_settings.py
new file mode 100644
index 0000000000..11b2f66d8e
--- /dev/null
+++ b/test/unit_test/common/test_gaussdb_settings.py
@@ -0,0 +1,280 @@
+#
+# Copyright 2025 The InfiniFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+import importlib
+import sys
+import types
+from unittest.mock import Mock
+
+import pytest
+
+FAKE_GAUSSDB_POOL = object()
+
+
+class FakeDocEngineConnection:
+ def db_type(self):
+ return "stub"
+
+
+class FakeGaussDBConnection:
+ def __init__(self, pool=None):
+ self.pool = pool or FAKE_GAUSSDB_POOL
+
+ def db_type(self):
+ return "gaussdb"
+
+
+class FakeGaussDBMemoryConnection:
+ def __init__(self, pool=None):
+ self.pool = pool or FAKE_GAUSSDB_POOL
+
+ def db_type(self):
+ return "gaussdb"
+
+
+class FakeStorage:
+ def health(self):
+ return True
+
+
+def _install_module(monkeypatch, name, **attrs):
+ mod = types.ModuleType(name)
+ for key, value in attrs.items():
+ setattr(mod, key, value)
+ monkeypatch.setitem(sys.modules, name, mod)
+ parent_name, _, child_name = name.rpartition(".")
+ parent = sys.modules.get(parent_name)
+ if parent is not None:
+ monkeypatch.setattr(parent, child_name, mod, raising=False)
+ return mod
+
+
+def _install_settings_import_stubs(monkeypatch):
+ importlib.import_module("rag.utils")
+ importlib.import_module("memory.utils")
+
+ rag_modules = {
+ "es_conn": {"ESConnection": FakeDocEngineConnection},
+ "infinity_conn": {"InfinityConnection": FakeDocEngineConnection},
+ "ob_conn": {"OBConnection": FakeDocEngineConnection},
+ "opensearch_conn": {"OSConnection": FakeDocEngineConnection},
+ "gaussdb_conn": {"GaussDBConnection": FakeGaussDBConnection},
+ "azure_sas_conn": {"RAGFlowAzureSasBlob": FakeStorage},
+ "azure_spn_conn": {"RAGFlowAzureSpnBlob": FakeStorage},
+ "gcs_conn": {"RAGFlowGCS": FakeStorage},
+ "minio_conn": {"RAGFlowMinio": FakeStorage},
+ "opendal_conn": {"OpenDALStorage": FakeStorage},
+ "s3_conn": {"RAGFlowS3": FakeStorage},
+ "oss_conn": {"RAGFlowOSS": FakeStorage},
+ "redis_conn": {"REDIS_CONN": types.SimpleNamespace(health=lambda: True)},
+ }
+ for short_name, attrs in rag_modules.items():
+ _install_module(monkeypatch, f"rag.utils.{short_name}", **attrs)
+
+ memory_modules = {
+ "es_conn": {"ESConnection": FakeDocEngineConnection},
+ "infinity_conn": {"InfinityConnection": FakeDocEngineConnection},
+ "ob_conn": {"OBConnection": FakeDocEngineConnection},
+ "gaussdb_conn": {"GaussDBMemoryConnection": FakeGaussDBMemoryConnection},
+ }
+ for short_name, attrs in memory_modules.items():
+ _install_module(monkeypatch, f"memory.utils.{short_name}", **attrs)
+
+ fake_search = types.SimpleNamespace(Dealer=lambda conn: ("dealer", conn))
+ fake_kg_search = types.SimpleNamespace(KGSearch=lambda conn: ("kg", conn))
+ _install_module(monkeypatch, "rag.nlp", search=fake_search)
+ _install_module(monkeypatch, "rag.graphrag", search=fake_kg_search)
+ _install_module(monkeypatch, "rag.graphrag.search", KGSearch=fake_kg_search.KGSearch)
+
+
+def _import_settings(monkeypatch):
+ _install_settings_import_stubs(monkeypatch)
+ import common
+
+ # Import machinery assigns a freshly imported submodule to
+ # ``common.settings`` on the parent package. Restoring only
+ # ``sys.modules["common.settings"]`` leaves that parent attribute pointing
+ # at this test's stub-backed module and contaminates later tests.
+ monkeypatch.setattr(
+ common,
+ "settings",
+ getattr(common, "settings", None),
+ raising=False,
+ )
+ monkeypatch.delitem(sys.modules, "common.settings", raising=False)
+ return importlib.import_module("common.settings")
+
+
+def test_doc_engine_gaussdb_initializes_gaussdb_connection(monkeypatch):
+ settings = _import_settings(monkeypatch)
+
+ monkeypatch.setenv("DOC_ENGINE", "gaussdb")
+ monkeypatch.setattr(
+ settings,
+ "get_base_config",
+ lambda name, default=None: (
+ {
+ "host": "db.example",
+ "port": 19995,
+ "database": "postgres",
+ "user": "sqlbuilder",
+ "password": "fake-unit-password",
+ "schema": "ragflow_gaussdb_docengine_it",
+ }
+ if name == "gaussdb"
+ else (default or {})
+ ),
+ )
+
+ settings.init_settings()
+
+ assert settings.DOC_ENGINE == "gaussdb"
+ assert settings.DOC_ENGINE_GAUSSDB is True
+ assert settings.DOC_ENGINE_OCEANBASE is False
+ assert settings.docStoreConn.db_type() == "gaussdb"
+ assert settings.msgStoreConn.db_type() == "gaussdb"
+ assert settings.docStoreConn is not settings.msgStoreConn
+ assert settings.docStoreConn.pool is settings.msgStoreConn.pool
+ assert settings.GAUSSDB["schema"] == "ragflow_gaussdb_docengine_it"
+
+
+def test_doc_engine_gaussdb_does_not_change_database_type(monkeypatch):
+ settings = _import_settings(monkeypatch)
+
+ monkeypatch.setenv("DOC_ENGINE", "gaussdb")
+ monkeypatch.setenv("DB_TYPE", "mysql")
+ monkeypatch.setattr(
+ settings,
+ "get_base_config",
+ lambda name, default=None: (
+ {
+ "host": "h",
+ "port": 1,
+ "database": "d",
+ "user": "u",
+ "password": "fake-unit-password",
+ }
+ if name == "gaussdb"
+ else (default or {})
+ ),
+ )
+
+ settings.init_settings()
+
+ assert settings.DATABASE_TYPE != "gaussdb"
+ assert settings.docStoreConn.db_type() == "gaussdb"
+ assert settings.msgStoreConn.db_type() == "gaussdb"
+ assert settings.docStoreConn.pool is settings.msgStoreConn.pool
+
+
+def test_doc_engine_gaussdb_initializes_message_store_case_insensitive(monkeypatch):
+ settings = _import_settings(monkeypatch)
+
+ monkeypatch.setenv("DOC_ENGINE", "GaussDB")
+ monkeypatch.setattr(
+ settings,
+ "get_base_config",
+ lambda name, default=None: (
+ {
+ "host": "h",
+ "port": 19995,
+ "database": "d",
+ "user": "u",
+ "password": "fake-unit-password",
+ }
+ if name == "gaussdb"
+ else (default or {})
+ ),
+ )
+
+ settings.init_settings()
+
+ assert settings.DOC_ENGINE == "GaussDB"
+ assert settings.DOC_ENGINE_GAUSSDB is True
+ assert settings.docStoreConn.db_type() == "gaussdb"
+ assert settings.msgStoreConn.db_type() == "gaussdb"
+ assert settings.docStoreConn.pool is settings.msgStoreConn.pool
+
+
+def test_tc_cfg_503_default_doc_engine_is_not_gaussdb(monkeypatch):
+ settings = _import_settings(monkeypatch)
+ gaussdb_module = sys.modules["rag.utils.gaussdb_conn"]
+ gaussdb_constructor = Mock(side_effect=AssertionError("GaussDB must not be constructed"))
+ monkeypatch.setattr(gaussdb_module, "GaussDBConnection", gaussdb_constructor)
+
+ monkeypatch.delenv("DOC_ENGINE", raising=False)
+ monkeypatch.setattr(settings, "get_base_config", lambda _name, default=None: default or {})
+
+ settings.init_settings()
+
+ assert settings.DOC_ENGINE == "elasticsearch"
+ assert settings.DOC_ENGINE_GAUSSDB is False
+ assert settings.docStoreConn.db_type() == "stub"
+ gaussdb_constructor.assert_not_called()
+
+
+def test_tc_cfg_504_unknown_doc_engine_is_rejected(monkeypatch):
+ settings = _import_settings(monkeypatch)
+
+ monkeypatch.setenv("DOC_ENGINE", "unknown_db")
+ monkeypatch.setattr(settings, "get_base_config", lambda _name, default=None: default or {})
+ settings.docStoreConn = None
+
+ with pytest.raises(Exception) as exc_info:
+ settings.init_settings()
+
+ assert str(exc_info.value) == "Not supported doc engine: unknown_db"
+ assert settings.DOC_ENGINE == "unknown_db"
+ assert settings.DOC_ENGINE_GAUSSDB is False
+ assert settings.docStoreConn is None
+
+
+def test_db_type_and_doc_engine_gaussdb_use_separate_configs(monkeypatch):
+ settings = _import_settings(monkeypatch)
+
+ monkeypatch.setenv("DB_TYPE", "gaussdb")
+ monkeypatch.setenv("DOC_ENGINE", "gaussdb")
+ monkeypatch.setenv("GAUSSDB_METADATA_HOST", "metadata.example")
+ monkeypatch.setenv("GAUSSDB_METADATA_PORT", "8000")
+ monkeypatch.setenv("GAUSSDB_METADATA_DBNAME", "metadata_db")
+ monkeypatch.setenv("GAUSSDB_METADATA_USER", "metadata_user")
+ monkeypatch.setenv("GAUSSDB_METADATA_PASSWORD", "metadata-secret")
+ monkeypatch.setenv("GAUSSDB_METADATA_SCHEMA", "metadata_schema")
+ monkeypatch.setattr(
+ settings,
+ "get_base_config",
+ lambda name, default=None: (
+ {
+ "host": "doc.example",
+ "port": 19995,
+ "database": "doc_db",
+ "user": "doc_user",
+ "password": "doc-secret",
+ "schema": "doc_schema",
+ }
+ if name == "gaussdb"
+ else (default or {})
+ ),
+ )
+
+ settings.init_settings()
+
+ assert settings.DATABASE_TYPE == "gaussdb"
+ assert settings.DATABASE["host"] == "metadata.example"
+ assert settings.DATABASE["name"] == "metadata_db"
+ assert settings.DATABASE["options"].startswith("-c search_path=metadata_schema ")
+ assert settings.GAUSSDB["host"] == "doc.example"
+ assert settings.GAUSSDB["database"] == "doc_db"
+ assert settings.docStoreConn.pool is settings.msgStoreConn.pool
diff --git a/test/unit_test/deploy/test_gaussdb_docengine_config.py b/test/unit_test/deploy/test_gaussdb_docengine_config.py
new file mode 100644
index 0000000000..a00fd08924
--- /dev/null
+++ b/test/unit_test/deploy/test_gaussdb_docengine_config.py
@@ -0,0 +1,211 @@
+#
+# Copyright 2025 The InfiniFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+from pathlib import Path
+import re
+import subprocess
+from unittest.mock import patch
+
+from ruamel.yaml import YAML
+
+ROOT = Path(__file__).resolve().parents[3]
+SERVICE_CONF_TEMPLATE = ROOT / "docker" / "service_conf.yaml.template"
+_PLACEHOLDER_PATTERN = re.compile(r"\$\{([A-Za-z_][A-Za-z0-9_]*)(?::-([^}]*))?\}")
+
+
+def _expand_service_conf_template(template_path, output_path, environ):
+ def replace_placeholder(match):
+ value = environ.get(match.group(1))
+ return value if value else (match.group(2) or "")
+
+ rendered = _PLACEHOLDER_PATTERN.sub(replace_placeholder, Path(template_path).read_text(encoding="utf-8"))
+ destination = Path(output_path)
+ destination.write_text(rendered, encoding="utf-8")
+ return destination
+
+
+def _load_yaml(path):
+ yaml = YAML(typ="safe", pure=True)
+ return yaml.load(path.read_text(encoding="utf-8"))
+
+
+def _import_gaussdb_conn_pool(monkeypatch):
+ from common import settings
+
+ monkeypatch.setattr(
+ settings,
+ "GAUSSDB",
+ {
+ "host": "gaussdb.local",
+ "port": "19995",
+ "database": "postgres",
+ "user": "sqlbuilder",
+ "password": "fake-unit-password",
+ },
+ raising=False,
+ )
+ with patch("psycopg2.pool.ThreadedConnectionPool", lambda *_args, **_kwargs: object()):
+ from common.doc_store import gaussdb_conn_pool
+
+ return gaussdb_conn_pool
+
+
+def test_tc_cfg_801_docker_template_defines_complete_gaussdb_config(tmp_path):
+ entrypoint = (ROOT / "docker" / "entrypoint.sh").read_text(encoding="utf-8")
+ template = (ROOT / "docker" / "service_conf.yaml.template").read_text(encoding="utf-8")
+ gaussdb_block = template.split("gaussdb:", 1)[1].split("seekdb:", 1)[0]
+
+ assert 'DEF_ENV_VALUE_PATTERN="\\$\\{([^:]+):-([^}]+)\\}"' in entrypoint
+ assert 'done < "${TEMPLATE_FILE}"' in entrypoint
+ assert "common.service_conf_renderer" not in entrypoint
+ assert "host: '${GAUSSDB_HOST}'" in gaussdb_block
+ assert "port: ${GAUSSDB_PORT}" in gaussdb_block
+ assert "database: '${GAUSSDB_DATABASE:-postgres}'" in gaussdb_block
+ assert "user: '${GAUSSDB_USER}'" in gaussdb_block
+ assert "password: '${GAUSSDB_PASSWORD}'" in gaussdb_block
+ assert "schema: '${GAUSSDB_SCHEMA:-public}'" in gaussdb_block
+
+ rendered_path = _expand_service_conf_template(
+ SERVICE_CONF_TEMPLATE,
+ tmp_path / "service_conf.yaml",
+ {
+ "GAUSSDB_HOST": "h",
+ "GAUSSDB_PORT": "5432",
+ "GAUSSDB_DATABASE": "d",
+ "GAUSSDB_USER": "u",
+ "GAUSSDB_PASSWORD": "p",
+ "GAUSSDB_SCHEMA": "zlw",
+ },
+ )
+
+ assert rendered_path.parent == tmp_path
+ assert _load_yaml(rendered_path)["gaussdb"] == {
+ "host": "h",
+ "port": 5432,
+ "database": "d",
+ "user": "u",
+ "password": "p",
+ "schema": "zlw",
+ }
+
+
+def test_tc_cfg_802_docker_template_defaults_empty_schema_to_public(tmp_path):
+ template = (ROOT / "docker" / "service_conf.yaml.template").read_text(encoding="utf-8")
+ gaussdb_block = template.split("gaussdb:", 1)[1].split("seekdb:", 1)[0]
+ assert "schema: '${GAUSSDB_SCHEMA:-public}'" in gaussdb_block
+
+ rendered_path = _expand_service_conf_template(
+ SERVICE_CONF_TEMPLATE,
+ tmp_path / "service_conf.yaml",
+ {
+ "GAUSSDB_HOST": "h",
+ "GAUSSDB_PORT": "5432",
+ "GAUSSDB_USER": "u",
+ "GAUSSDB_PASSWORD": "p",
+ "GAUSSDB_SCHEMA": "",
+ },
+ )
+
+ config = _load_yaml(rendered_path)["gaussdb"]
+ assert config["database"] == "postgres"
+ assert config["schema"] == "public"
+
+
+def test_tc_cfg_804_docker_rendered_gaussdb_config_excludes_tuning_fields(tmp_path):
+ template = (ROOT / "docker" / "service_conf.yaml.template").read_text(encoding="utf-8")
+ gaussdb_block = template.split("gaussdb:", 1)[1].split("seekdb:", 1)[0]
+ rendered_path = _expand_service_conf_template(
+ SERVICE_CONF_TEMPLATE,
+ tmp_path / "service_conf.yaml",
+ {
+ "GAUSSDB_HOST": "h",
+ "GAUSSDB_PORT": "5432",
+ "GAUSSDB_DATABASE": "d",
+ "GAUSSDB_USER": "u",
+ "GAUSSDB_PASSWORD": "p",
+ "GAUSSDB_SCHEMA": "zlw",
+ "GAUSSDB_SSL": "true",
+ "GAUSSDB_TIMEOUT": "30",
+ "GAUSSDB_POOL_MIN": "4",
+ "GAUSSDB_POOL_MAX": "32",
+ },
+ )
+ gaussdb = _load_yaml(rendered_path)["gaussdb"]
+
+ assert "password: '${GAUSSDB_PASSWORD}'" in gaussdb_block
+ assert set(gaussdb) == {"host", "port", "database", "user", "password", "schema"}
+ forbidden = {"ssl", "sslmode", "timeout", "connect_timeout", "pool_min", "pool_max", "minconn", "maxconn"}
+ assert forbidden.isdisjoint(gaussdb)
+ assert all(f"GAUSSDB_{name.upper()}" not in gaussdb_block for name in forbidden)
+
+
+def test_tc_cfg_807_rendered_service_conf_gaussdb_block_loads_as_gaussdb_config(tmp_path, monkeypatch):
+ rendered_path = _expand_service_conf_template(
+ SERVICE_CONF_TEMPLATE,
+ tmp_path / "service_conf.yaml",
+ {
+ "GAUSSDB_HOST": "h",
+ "GAUSSDB_PORT": "5432",
+ "GAUSSDB_DATABASE": "d",
+ "GAUSSDB_USER": "u",
+ "GAUSSDB_PASSWORD": "p",
+ "GAUSSDB_SCHEMA": "zlw",
+ },
+ )
+ rendered = _load_yaml(rendered_path)
+ pool_mod = _import_gaussdb_conn_pool(monkeypatch)
+
+ cfg = pool_mod.load_gaussdb_config(rendered["gaussdb"])
+
+ assert cfg.host == "h"
+ assert cfg.port == 5432
+ assert cfg.database == "d"
+ assert cfg.user == "u"
+ assert cfg.password == "p"
+ assert cfg.schema == "zlw"
+
+
+def test_tc_cfg_808_rendered_artifacts_stay_under_tmp_path_and_git_status_is_stable(tmp_path):
+ before = subprocess.run(
+ ["git", "status", "--porcelain=v1", "--untracked-files=all"],
+ cwd=ROOT,
+ encoding="utf-8",
+ capture_output=True,
+ check=True,
+ ).stdout
+
+ service_conf = _expand_service_conf_template(
+ SERVICE_CONF_TEMPLATE,
+ tmp_path / "service_conf.yaml",
+ {
+ "GAUSSDB_HOST": "gaussdb.local",
+ "GAUSSDB_PORT": "19995",
+ "GAUSSDB_DATABASE": "postgres",
+ "GAUSSDB_USER": "ragflow",
+ "GAUSSDB_PASSWORD": "secret",
+ "GAUSSDB_SCHEMA": "public",
+ },
+ )
+ assert service_conf.is_relative_to(tmp_path)
+ assert _load_yaml(service_conf)["gaussdb"]["host"] == "gaussdb.local"
+
+ after = subprocess.run(
+ ["git", "status", "--porcelain=v1", "--untracked-files=all"],
+ cwd=ROOT,
+ encoding="utf-8",
+ capture_output=True,
+ check=True,
+ ).stdout
+ assert after == before
diff --git a/test/unit_test/memory/utils/test_gaussdb_memory_conn.py b/test/unit_test/memory/utils/test_gaussdb_memory_conn.py
new file mode 100644
index 0000000000..812ff59509
--- /dev/null
+++ b/test/unit_test/memory/utils/test_gaussdb_memory_conn.py
@@ -0,0 +1,643 @@
+#
+# Copyright 2025 The InfiniFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+from datetime import datetime
+from decimal import Decimal
+
+import numpy as np
+import pytest
+
+from common.doc_store.doc_store_base import FusionExpr, MatchDenseExpr, MatchTextExpr, OrderByExpr
+from common.doc_store.gaussdb_conn_base import InvalidGaussDBObjectName
+from memory.utils import gaussdb_conn as gaussdb_mem
+from memory.utils.gaussdb_conn import GaussDBMemoryConnection, GaussDBMemoryDDLBuilder, SearchResult
+
+
+class FakePool:
+ masked_uri = "unit@example:19995/db?schema=public"
+ resolved_schema = "public"
+
+ def check_schema_access(self):
+ return None
+
+
+def make_conn() -> GaussDBMemoryConnection:
+ return GaussDBMemoryConnection(pool=FakePool())
+
+
+def test_physical_table_name_depends_only_on_index_name():
+ builder = GaussDBMemoryDDLBuilder(schema="public")
+
+ table = builder.physical_table_name("memory_tenant_a")
+
+ assert table == builder.physical_table_name("memory_tenant_a")
+ assert table != builder.physical_table_name("memory_tenant_b")
+ assert table.startswith("ragflow_mem_")
+ assert "memory_id" not in table
+ assert len(table) <= 63
+
+
+def test_memory_table_ddl_uses_oracle_compatible_columns_and_ustore():
+ builder = GaussDBMemoryDDLBuilder(schema="public")
+ ddl = builder.build_memory_table_ddl("ragflow_mem_abc")
+
+ assert 'CREATE TABLE IF NOT EXISTS "public"."ragflow_mem_abc"' in ddl
+ assert "message_id NUMBER(19) NOT NULL" in ddl
+ assert "message_type_kwd VARCHAR2(64)" in ddl
+ assert "valid_at TIMESTAMP" in ddl
+ assert "content_ltks TEXT" in ddl
+ assert "WITH (storage_type=USTORE)" in ddl
+
+
+def test_memory_indexes_use_ugin_and_gsdiskann_with_empty_marker():
+ builder = GaussDBMemoryDDLBuilder(schema="public")
+
+ fulltext = builder.build_fulltext_ugin_ddl("ragflow_mem_abc")
+ vector_columns = "\n".join(builder.build_vector_column_ddls("ragflow_mem_abc", 3))
+ empty_index = builder.build_vector_empty_index_ddl("ragflow_mem_abc", 3)
+ diskann = builder.build_diskann_index_ddl("ragflow_mem_abc", 3)
+
+ assert "USING ugin (to_tsvector('simple', tokenized_content_ltks))" in fulltext
+ assert "q_3_vec floatvector(3)" in vector_columns
+ assert "DEFAULT (array_fill(0, ARRAY[3])::text::floatvector(3)) NOT NULL" in vector_columns
+ assert "q_3_vec_empty BOOLEAN DEFAULT TRUE NOT NULL" in vector_columns
+ assert "(memory_id, q_3_vec_empty)" in empty_index
+ assert "USING gsdiskann (q_3_vec cosine)" in diskann
+
+
+def test_merge_sql_uses_dual_and_resets_other_vector_dimensions():
+ conn = make_conn()
+ row, dim = conn._message_to_row(
+ {
+ "id": "mem1_1",
+ "message_id": 1,
+ "message_type": "raw",
+ "source_id": 0,
+ "memory_id": "mem1",
+ "agent_id": "agent1",
+ "session_id": "session1",
+ "content": "hello",
+ "valid_at": "2026-06-29 12:00:00",
+ "status": True,
+ "content_embed": [0.1, 0.2, 0.3],
+ },
+ "mem1",
+ )
+
+ sql, params = conn._build_merge_sql("ragflow_mem_abc", dim, [3, 5], [row])
+
+ assert sql.startswith('MERGE INTO "public"."ragflow_mem_abc"')
+ assert "FROM dual" in sql
+ assert "ON CONFLICT" not in sql
+ assert "q_3_vec = s.q_3_vec" in sql
+ assert "q_3_vec_empty = FALSE" in sql
+ assert "q_5_vec = '[0,0,0,0,0]'::floatvector" in sql
+ assert "q_5_vec_empty = TRUE" in sql
+ assert params[0][0] == "mem1_1"
+ assert params[0][-1] == "[0.1,0.2,0.3]"
+
+
+def test_where_clause_skips_empty_string_filters_and_adds_memory_boundary():
+ conn = make_conn()
+
+ where_sql, params = conn._build_where_clause(
+ {"agent_id": "", "session_id": "s1", "status": 1},
+ memory_ids=["mem1", "mem2"],
+ hide_forgotten=True,
+ )
+
+ assert "memory_id IN (%s, %s)" in where_sql
+ assert "forget_at IS NULL" in where_sql
+ assert "session_id = %s" in where_sql
+ assert "status_int = %s" in where_sql
+ assert "agent_id" not in where_sql
+ assert params == ["mem1", "mem2", "s1", 1]
+
+
+def test_delete_empty_message_id_list_is_noop():
+ conn = make_conn()
+
+ assert conn.delete({"message_id": []}, "memory_tenant", "mem1") == 0
+
+
+def test_get_sql_does_not_add_fetch_first_one(monkeypatch):
+ conn = make_conn()
+ captured = {}
+
+ monkeypatch.setattr(conn, "_table_exists", lambda _table: True)
+ monkeypatch.setattr(conn, "_vector_columns_for_select", lambda _table: [])
+
+ def fake_fetch_one_with_description(sql, params):
+ captured["sql"] = sql
+ captured["params"] = params
+ return None, []
+
+ monkeypatch.setattr(conn, "_fetch_one_with_description", fake_fetch_one_with_description)
+
+ assert conn.get("mem1_1", "memory_tenant", ["mem1"]) is None
+ assert "FETCH FIRST 1 ROWS ONLY" not in captured["sql"]
+ assert "WHERE id = %s" in captured["sql"]
+ assert captured["params"] == ["mem1_1"]
+
+
+def test_catalog_existence_checks_do_not_add_fetch_first_one(monkeypatch):
+ conn = make_conn()
+ captured = []
+
+ def fake_fetch_one(sql, params):
+ captured.append((sql, params))
+ return (1,)
+
+ monkeypatch.setattr(conn, "_fetch_one", fake_fetch_one)
+
+ assert conn._table_exists("ragflow_mem_abc")
+ assert conn._column_exists("ragflow_mem_abc", "q_3_vec")
+ assert all("FETCH FIRST 1 ROWS ONLY" not in sql for sql, _params in captured)
+
+
+def test_sql_returns_none_without_executing_external_sql():
+ conn = make_conn()
+
+ assert conn.sql("SELECT 1") is None
+
+
+def test_update_content_set_refreshes_tokenized_content():
+ conn = make_conn()
+
+ set_sql, params = conn._build_update_set("ragflow_mem_abc", "memory_tenant", "mem1", {"content": "hello world"})
+
+ assert "content_ltks = %s" in set_sql
+ assert "tokenized_content_ltks = %s" in set_sql
+ assert params[0] == "hello world"
+ assert params[1]
+
+
+def test_remove_content_clears_original_and_tokenized_content():
+ conn = make_conn()
+
+ set_sql, params = conn._build_update_set("ragflow_mem_abc", "memory_tenant", "mem1", {"remove": "content"})
+
+ assert set_sql == "content_ltks = NULL, tokenized_content_ltks = NULL"
+ assert params == []
+
+
+def test_vector_search_sql_filters_empty_vectors_and_uses_threshold():
+ conn = make_conn()
+ parsed = conn._parse_match_expressions(
+ [
+ MatchDenseExpr(
+ "q_3_vec",
+ [0.1, 0.2, 0.3],
+ "float",
+ "cosine",
+ topn=5,
+ extra_options={"similarity": 0.62},
+ )
+ ]
+ )
+
+ sql, params = conn._build_vector_search_sql(
+ table="ragflow_mem_abc",
+ select_fields=["message_id", "content"],
+ condition={"status": 1},
+ parsed=parsed,
+ offset=0,
+ limit=10,
+ memory_ids=["mem1"],
+ hide_forgotten=True,
+ )
+
+ assert "1 - (q_3_vec <+> %s::floatvector(3)) AS _score" in sql
+ assert "q_3_vec_empty = FALSE" in sql
+ assert ">= %s" in sql
+ assert "ORDER BY q_3_vec <+> %s::floatvector(3) ASC" in sql
+ assert "OFFSET %s LIMIT %s" in sql
+ assert "FETCH NEXT" not in sql
+ assert params == ["[0.1,0.2,0.3]", "mem1", 1, "[0.1,0.2,0.3]", 0.62, "[0.1,0.2,0.3]", 0, 5]
+
+
+def test_fulltext_search_sql_uses_tokenized_original_query_and_plain_tsquery():
+ conn = make_conn()
+ parsed = conn._parse_match_expressions([MatchTextExpr(["content_ltks"], "ignored", 8, extra_options={"original_query": "contract risk"})])
+
+ sql, params = conn._build_fulltext_search_sql(
+ table="ragflow_mem_abc",
+ select_fields=["message_id", "content"],
+ highlight_fields=[],
+ condition={"status": 1},
+ parsed=parsed,
+ offset=0,
+ limit=10,
+ memory_ids=["mem1"],
+ hide_forgotten=True,
+ )
+
+ assert "to_tsvector('simple', tokenized_content_ltks)" in sql
+ assert "plainto_tsquery('simple', %s)" in sql
+ assert "ts_rank" in sql
+ assert "OFFSET %s LIMIT %s" in sql
+ assert "FETCH NEXT" not in sql
+ assert params[0] == "contract risk"
+ assert params[-2:] == [0, 8]
+
+
+def test_parse_match_expression_tokenizes_fulltext_query_like_stored_content():
+ conn = make_conn()
+
+ parsed = conn._parse_match_expressions([MatchTextExpr(["content_ltks"], "ignored", 8, extra_options={"original_query": "coriander"})])
+
+ assert parsed["text_query"] == "coriand"
+
+
+def test_fusion_sql_uses_fulltext_candidates_then_vector_filter():
+ conn = make_conn()
+ parsed = conn._parse_match_expressions(
+ [
+ MatchTextExpr(["content_ltks"], "ignored", 20, extra_options={"original_query": "risk"}),
+ MatchDenseExpr("q_3_vec", [0.1, 0.2, 0.3], "float", "cosine", topn=10, extra_options={"similarity": 0.5}),
+ FusionExpr("weighted_sum", 10, {"weights": "0.3,0.7"}),
+ ]
+ )
+
+ sql, params = conn._build_fusion_search_sql(
+ table="ragflow_mem_abc",
+ select_fields=["message_id", "content"],
+ condition={"status": 1},
+ parsed=parsed,
+ offset=0,
+ limit=10,
+ memory_ids=["mem1"],
+ hide_forgotten=True,
+ )
+
+ assert "WITH fulltext_results AS" in sql
+ assert "ORDER BY relevance DESC LIMIT %s" in sql
+ assert "q_3_vec_empty = FALSE" in sql
+ assert "relevance * %s + (1 - (q_3_vec <+> %s::floatvector(3))) * %s AS _score" in sql
+ assert "OFFSET %s LIMIT %s" in sql
+ assert "FETCH FIRST" not in sql
+ assert "FETCH NEXT" not in sql
+ assert params[0] == "risk"
+ assert 0.7 in params
+ assert 0.30000000000000004 in params
+
+
+def test_get_fields_restores_content_embed_by_empty_marker():
+ conn = make_conn()
+ result = SearchResult(
+ total=1,
+ messages=[
+ {
+ "id": "mem1_1",
+ "message_id": 1,
+ "message_type_kwd": "raw",
+ "source_id": 0,
+ "memory_id": "mem1",
+ "agent_id": "agent1",
+ "session_id": "session1",
+ "valid_at": "2026-06-29 12:00:00",
+ "status_int": 1,
+ "content_ltks": "hello",
+ "q_3_vec": "[0,0,0]",
+ "q_3_vec_empty": True,
+ "q_5_vec": "[1,2,3,4,5]",
+ "q_5_vec_empty": False,
+ }
+ ],
+ )
+
+ fields = conn.get_fields(result, ["message_id", "message_type", "content", "content_embed"])
+
+ assert fields["mem1_1"]["message_id"] == 1
+ assert fields["mem1_1"]["message_type"] == "raw"
+ assert fields["mem1_1"]["content"] == "hello"
+ assert fields["mem1_1"]["content_embed"] == [1.0, 2.0, 3.0, 4.0, 5.0]
+
+
+def test_filter_search_uses_order_by_and_offset_limit_syntax():
+ conn = make_conn()
+ order_by = OrderByExpr().desc("valid_at")
+
+ sql, params = conn._build_filter_search_sql(
+ table="ragflow_mem_abc",
+ select_fields=["message_id", "content"],
+ condition={},
+ order_by=order_by,
+ offset=20,
+ limit=10,
+ memory_ids=["mem1"],
+ hide_forgotten=False,
+ )
+
+ assert "ORDER BY valid_at DESC" in sql
+ assert "OFFSET %s LIMIT %s" in sql
+ assert "FETCH NEXT" not in sql
+ assert params == ["mem1", 20, 10]
+
+
+def test_sort_messages_uses_numeric_keys_for_message_id():
+ conn = make_conn()
+ order_by = OrderByExpr().desc("message_id")
+
+ messages = [
+ {"id": "mem_b_99", "message_id": 99},
+ {"id": "mem_a_100", "message_id": 100},
+ ]
+
+ sorted_messages = conn._sort_messages(messages, order_by, has_match=False)
+
+ assert [message["message_id"] for message in sorted_messages] == [100, 99]
+
+
+def test_health_reuses_gaussdb_base_health(monkeypatch):
+ conn = make_conn()
+ monkeypatch.setattr(conn, "_query_version", lambda: "GaussDB unit")
+ monkeypatch.setattr(conn, "_query_sql_compatibility", lambda: "PG")
+
+ health = conn.health()
+
+ assert "health" not in GaussDBMemoryConnection.__dict__
+ assert health["status"] == "unhealthy"
+ assert health["sql_compatibility"] == "PG"
+ assert "A/ORA" in health["error"]
+
+
+def make_base_health_conn(monkeypatch, *, server_encoding, client_encoding):
+ from common.doc_store.gaussdb_conn_base import GaussDBConnectionBase
+
+ conn = object.__new__(GaussDBConnectionBase)
+ conn.masked_uri = "gaussdb://[email protected]:19995/postgres?schema=ragflow_doc"
+ conn.resolved_schema = "ragflow_doc"
+ monkeypatch.setattr(conn, "_query_version", lambda: "GaussDB unit")
+ monkeypatch.setattr(conn, "_query_sql_compatibility", lambda: "A")
+ monkeypatch.setattr(conn, "_query_server_encoding", lambda: server_encoding)
+ monkeypatch.setattr(conn, "_query_client_encoding", lambda: client_encoding)
+ return conn
+
+
+def test_health_allows_sql_ascii_server_with_utf8_client_warning(monkeypatch):
+ conn = make_base_health_conn(monkeypatch, server_encoding="SQL_ASCII", client_encoding="UTF8")
+
+ result = conn.health()
+
+ assert result["status"] == "healthy"
+ assert result["server_encoding"] == "SQL_ASCII"
+ assert result["client_encoding"] == "UTF8"
+ assert "does not validate" in result["warning"]
+
+
+def test_health_rejects_non_utf8_client(monkeypatch):
+ conn = make_base_health_conn(monkeypatch, server_encoding="UTF8", client_encoding="SQL_ASCII")
+
+ result = conn.health()
+
+ assert result["status"] == "unhealthy"
+ assert "client_encoding" in result["error"]
+
+
+def test_health_accepts_utf8_server_and_client_without_warning(monkeypatch):
+ conn = make_base_health_conn(monkeypatch, server_encoding="UTF8", client_encoding="UTF8")
+
+ result = conn.health()
+
+ assert result["status"] == "healthy"
+ assert "warning" not in result
+
+
+def test_short_circuit_branches_and_tuple_result_helpers(monkeypatch):
+ conn = make_conn()
+ result = SearchResult(total=2, messages=[{"id": "doc-1", "content_ltks": "contract risk"}, {"id": ""}])
+
+ assert conn.insert([], "memory_tenant", "mem1") == []
+ assert conn.update({}, {"content": "x"}, "memory_tenant", "mem1") is False
+ assert conn.update({"id": "doc-1"}, {}, "memory_tenant", "mem1") is False
+ assert conn.delete({}, "memory_tenant", "mem1") == 0
+ assert conn.delete({"id": ["", None]}, "memory_tenant", "mem1") == 0
+ assert conn.get("", "memory_tenant", ["mem1"]) is None
+ assert conn.get_total((result, 2)) == 2
+ assert conn.get_doc_ids((result, 2)) == ["doc-1"]
+ assert conn.get_fields((result, 2), []) == {}
+ assert "<em>contract</em>" in conn.get_highlight((result, 2), ["contract"], "content_ltks")["doc-1"]
+ assert conn.get_aggregation((result, 2), "content_ltks") == [("contract risk", 1)]
+
+ monkeypatch.setattr(conn, "_table_exists", lambda _table: False)
+ assert conn.update({"id": "doc-1"}, {"content": "x"}, "missing", "mem1") is True
+ assert conn.delete({"id": ["doc-1"]}, "missing", "mem1") == 0
+ assert conn.get("doc-1", "missing", ["mem1"]) is None
+ assert conn.get_forgotten_messages(["message_id"], "missing", "mem1") is None
+ assert conn.get_missing_field_message(["message_id"], "missing", "mem1", "content") is None
+
+ empty_res, empty_total = conn.search(["message_id"], [], {}, [], OrderByExpr(), 0, 10, "", ["mem1"])
+ assert empty_total == 0
+ assert empty_res.messages == []
+ empty_res, empty_total = conn.search(["message_id"], [], {}, [], OrderByExpr(), 0, 10, "idx", [])
+ assert empty_total == 0
+ empty_res, empty_total = conn.search(["message_id"], [], {}, [], OrderByExpr(), 0, 0, "idx", ["mem1"])
+ assert empty_total == 0
+ assert empty_res.messages == []
+
+
+def test_index_exist_false_when_catalog_is_incomplete(monkeypatch):
+ conn = make_conn()
+ monkeypatch.setattr(conn, "_table_exists", lambda _table: True)
+ monkeypatch.setattr(conn, "_column_names", lambda _table: [])
+ assert conn.index_exist("memory_tenant", "mem1") is False
+
+ monkeypatch.setattr(conn, "_column_names", lambda _table: list(gaussdb_mem.BASE_COLUMNS))
+ monkeypatch.setattr(conn, "_index_names", lambda _table: [])
+ assert conn.index_exist("memory_tenant", "mem1") is False
+
+
+def test_message_normalization_vector_selection_and_invalid_inputs(caplog):
+ conn = make_conn()
+
+ with pytest.raises(InvalidGaussDBObjectName):
+ conn.physical_table("")
+ assert conn.convert_field_name("content", use_tokenized_content=True) == "tokenized_content_ltks"
+ with pytest.raises(ValueError, match="content_embed"):
+ conn._message_to_row({"message_id": 1, "content_embed": []}, "mem1")
+ with pytest.raises(ValueError, match="memory_id"):
+ conn._message_to_row({"message_id": 1, "content_embed": [0.1, 0.2, 0.3]}, None)
+
+ row = {
+ "id": "doc-1",
+ "q_2_vec": np.array([1, 2]),
+ "q_2_vec_empty": False,
+ "q_3_vec": "[3,4,5]",
+ "q_3_vec_empty": False,
+ }
+ assert conn._content_embed_from_row(row) == [3.0, 4.0, 5.0]
+ assert "multiple non-empty vector dimensions" in caplog.text
+
+
+def test_insert_normalization_failure_returns_all_batch_ids_without_write(monkeypatch):
+ conn = make_conn()
+ monkeypatch.setattr(conn, "_table_exists", lambda _table: pytest.fail("database access must not occur"))
+
+ errors = conn.insert(
+ [
+ {
+ "id": "mem1_1",
+ "message_id": 1,
+ "status": True,
+ "content": "first",
+ "content_embed": [0.1, 0.2, 0.3],
+ },
+ {
+ "id": "mem1_2",
+ "message_id": 2,
+ "status": True,
+ "content": "second",
+ "content_embed": [0.1, 0.2],
+ },
+ ],
+ "memory_tenant",
+ "mem1",
+ )
+
+ assert errors == ["mem1_1", "mem1_2"]
+
+
+def test_update_set_covers_remove_vector_empty_embed_numeric_and_time_branches(monkeypatch):
+ conn = make_conn()
+ monkeypatch.setattr(conn, "_ensure_vector_column_exists", lambda _table, _dim: None)
+ monkeypatch.setattr(conn, "_vector_dimensions", lambda _table: [3, 5])
+
+ set_sql, params = conn._build_update_set(
+ "ragflow_mem_abc",
+ "memory_tenant",
+ "mem1",
+ {
+ "remove": ["q_3_vec", "agent_id"],
+ "content_embed": [],
+ "message_id": Decimal("7"),
+ "agent_id": "",
+ "invalid_at": "-",
+ "status": True,
+ },
+ )
+
+ assert "q_3_vec = %s::floatvector(3)" in set_sql
+ assert "q_3_vec_empty = TRUE" in set_sql
+ assert "agent_id = NULL" in set_sql
+ assert "message_id = %s" in set_sql
+ assert "invalid_at = %s::timestamp" in set_sql
+ assert "status_int = %s" in set_sql
+ assert params == ["[0,0,0]", 7, None, None, 1]
+
+
+def test_column_where_order_and_parse_helpers_cover_edge_cases(monkeypatch):
+ conn = make_conn()
+ monkeypatch.setattr(conn, "_vector_dimensions", lambda _table: [3])
+
+ parsed = conn._parse_match_expressions(
+ [
+ MatchDenseExpr("content_embed", np.array([0.1, 0.2]), "float", "cosine", 8, {}),
+ FusionExpr("weighted_sum", 6, {"weights": "0.2"}),
+ ]
+ )
+ assert parsed["vector_dim"] == 2
+ assert parsed["vector"] == "[0.1,0.2]"
+ assert parsed["vector_weight"] == 0.5
+ assert parsed["topn"] == 6
+
+ assert conn._select_columns("ragflow_mem_abc", ["id", "_score", "content", "content_embed", "content"]) == [
+ "id",
+ "content_ltks",
+ "q_3_vec",
+ "q_3_vec_empty",
+ ]
+ with pytest.raises(InvalidGaussDBObjectName):
+ conn._select_columns("ragflow_mem_abc", ["bad;column"])
+ assert conn._build_where_clause({}, memory_ids=[], force_memory_filter=True) == ("1=0", [])
+ assert conn._build_where_clause({"message_id": ["", None]}, memory_ids=["mem1"]) == ("memory_id IN (%s)", ["mem1"])
+ assert conn._build_order_by(None) == ""
+
+ assert gaussdb_mem.normalize_index_names(" a, ,b ") == ["a", "b"]
+ assert gaussdb_mem.normalize_index_names([" a ", "", None]) == ["a", "None"]
+ assert gaussdb_mem.clean_list_values(None) == []
+ assert gaussdb_mem.clean_list_values("mem1") == ["mem1"]
+ assert gaussdb_mem.effective_limit(0, None) == 10000
+ assert gaussdb_mem.parse_vector_dim("q_12_vec") == 12
+ assert gaussdb_mem.parse_vector_dim("content_embed") is None
+ assert gaussdb_mem.vector_literal("[1,2]", 2) == "[1.0,2.0]"
+ assert gaussdb_mem.vector_literal("raw", 3) == "raw"
+ assert gaussdb_mem.vector_literal(np.array([1, 2]), 2) == "[1.0,2.0]"
+ with pytest.raises(ValueError, match="dimension mismatch"):
+ gaussdb_mem.vector_literal([1], 2)
+ assert gaussdb_mem.parse_vector_value(None) == []
+ assert gaussdb_mem.parse_vector_value(np.array([1, 2])) == [1.0, 2.0]
+ assert gaussdb_mem.parse_vector_value((1, 2)) == [1.0, 2.0]
+ assert gaussdb_mem.parse_vector_value("") == []
+ assert gaussdb_mem.parse_vector_value("[1, 2]") == [1.0, 2.0]
+ assert gaussdb_mem.parse_vector_value("[]") == []
+ assert gaussdb_mem.parse_vector_value("not-json") == []
+ assert gaussdb_mem.parse_vector_value('{"x": 1}') == []
+ assert gaussdb_mem.zero_vector_literal(3) == "[0,0,0]"
+ assert gaussdb_mem.normalize_timestamp("-") is None
+ assert gaussdb_mem.normalize_timestamp(datetime(2026, 6, 29, 12, 0, 0)) == "2026-06-29 12:00:00"
+ assert gaussdb_mem.format_timestamp(datetime(2026, 6, 29, 12, 0, 0)) == "2026-06-29 12:00:00"
+ assert gaussdb_mem.none_if_empty("") is None
+ assert gaussdb_mem.to_int_or_none(False) == 0
+ assert gaussdb_mem.to_int_or_none(np.int64(3)) == 3
+ assert gaussdb_mem.to_int_or_original(None) is None
+ assert gaussdb_mem.to_int_or_original(True) == 1
+ assert gaussdb_mem.to_int_or_original(np.int64(4)) == 4
+ assert gaussdb_mem.to_int_or_original(Decimal("5")) == 5
+ assert gaussdb_mem.to_int_or_original("abc") == "abc"
+ assert gaussdb_mem.row_value({"x": 1}, "x", 0) == 1
+ assert gaussdb_mem.unique_preserve_order(["a", "b", "a"]) == ["a", "b"]
+ assert gaussdb_mem.sortable_value(datetime(2026, 6, 29, 12, 0, 0)) == "2026-06-29T12:00:00"
+ assert gaussdb_mem.is_maintenance_work_mem_error(RuntimeError("maintenance_work_mem below required")) is True
+ assert gaussdb_mem.is_maintenance_work_mem_error(RuntimeError("other error")) is False
+ gaussdb_mem.close_cursor(None)
+
+
+def test_diskann_retry_handles_maintenance_work_mem_errors(monkeypatch):
+ conn = make_conn()
+ calls = []
+
+ def fail_once(_statements):
+ calls.append(_statements)
+ if len(calls) == 1:
+ raise RuntimeError("maintenance_work_mem below required")
+
+ monkeypatch.setattr(conn, "_execute_statements", fail_once)
+ conn._create_diskann_index_with_retry("ragflow_mem_abc", 3)
+ assert len(calls) == 2
+ assert calls[0][1] == "SET LOCAL maintenance_work_mem = '1GB'"
+ assert calls[1][1] == "SET LOCAL maintenance_work_mem = '2GB'"
+
+ def fail_non_maintenance(_statements):
+ raise RuntimeError("syntax error")
+
+ monkeypatch.setattr(conn, "_execute_statements", fail_non_maintenance)
+ with pytest.raises(RuntimeError, match="syntax error"):
+ conn._create_diskann_index_with_retry("ragflow_mem_abc", 3)
+
+
+def test_diskann_unavailable_propagates_missing_prerequisite(monkeypatch):
+ conn = make_conn()
+
+ class UnsupportedVectorIndexError(Exception):
+ pgcode = "0A000"
+
+ def unsupported(_statements):
+ raise UnsupportedVectorIndexError("The vectordb indexes are supported only by high-order features")
+
+ monkeypatch.setattr(conn, "_execute_statements", unsupported)
+
+ with pytest.raises(UnsupportedVectorIndexError, match="supported only by high-order features"):
+ conn._create_diskann_index_with_retry("ragflow_mem_abc", 3)
diff --git a/test/unit_test/rag/app/test_gaussdb_table_chunk_column_roles.py b/test/unit_test/rag/app/test_gaussdb_table_chunk_column_roles.py
new file mode 100644
index 0000000000..c92a4cda76
--- /dev/null
+++ b/test/unit_test/rag/app/test_gaussdb_table_chunk_column_roles.py
@@ -0,0 +1,253 @@
+import sys
+import types
+from importlib import import_module, reload
+from unittest.mock import MagicMock, patch
+
+import pytest
+
+import common.settings as settings
+
+
+TEST_CSV = b"""row_id,title,content,country,category
+1,Earthquake hits Turkey,A 5.8 magnitude earthquake struck Konya,Turkey,Disaster
+2,Oil prices surge,Brent crude jumped 4.2 percent,Global,Economy
+3,AI regulation proposed,EU unveiled a draft regulation,EU,Technology
+"""
+FILENAME = "test.csv"
+KB_ID = "test_kb_id"
+TABLE_NAME = "ragflow_0123456789abcdef0123456789abcdef"
+
+
+def _install_table_import_stubs():
+ names = (
+ "deepdoc.parser",
+ "deepdoc.parser.utils",
+ "deepdoc.parser.figure_parser",
+ "deepdoc.vision.ocr",
+ "rag.nlp",
+ "rag.app.table",
+ "rag.app.picture",
+ "api.db.services.knowledgebase_service",
+ "xpinyin",
+ )
+ original = {name: sys.modules.get(name) for name in names}
+
+ parser_stub = types.ModuleType("deepdoc.parser")
+ parser_stub.__path__ = []
+ parser_stub.ExcelParser = type("ExcelParser", (), {})
+
+ utils_stub = types.ModuleType("deepdoc.parser.utils")
+ utils_stub.get_text = lambda _filename, binary=None: binary.decode("utf-8") if isinstance(binary, bytes) else str(binary or "")
+
+ figure_stub = types.ModuleType("deepdoc.parser.figure_parser")
+ figure_stub.vision_figure_parser_figure_xlsx_wrapper = lambda *_args, **_kwargs: []
+
+ nlp_stub = types.ModuleType("rag.nlp")
+
+ def tokenize_doc(doc, text="", *_args, **_kwargs):
+ if isinstance(doc, dict):
+ doc["content_with_weight"] = str(text or "")
+ doc["content_ltks"] = str(text or "")
+ doc["content_sm_ltks"] = str(text or "")
+ return doc
+
+ nlp_stub.rag_tokenizer = types.SimpleNamespace(
+ tokenize=lambda text, *_args, **_kwargs: str(text),
+ fine_grained_tokenize=lambda text, *_args, **_kwargs: str(text),
+ )
+ nlp_stub.tokenize = tokenize_doc
+ nlp_stub.tokenize_table = lambda *_args, **_kwargs: []
+
+ kb_service_stub = types.ModuleType("api.db.services.knowledgebase_service")
+ kb_service_stub.KnowledgebaseService = type(
+ "KnowledgebaseService",
+ (),
+ {"update_parser_config": staticmethod(lambda *_args, **_kwargs: None)},
+ )
+
+ xpinyin_stub = types.ModuleType("xpinyin")
+
+ class Pinyin:
+ def get_pinyins(self, text, splitter="_"):
+ return [str(text).replace(" ", splitter)]
+
+ xpinyin_stub.Pinyin = Pinyin
+
+ sys.modules["deepdoc.parser"] = parser_stub
+ sys.modules["deepdoc.parser.utils"] = utils_stub
+ sys.modules["deepdoc.parser.figure_parser"] = figure_stub
+ sys.modules["deepdoc.vision.ocr"] = MagicMock()
+ sys.modules["rag.nlp"] = nlp_stub
+ sys.modules["rag.app.picture"] = MagicMock()
+ sys.modules["api.db.services.knowledgebase_service"] = kb_service_stub
+ sys.modules["xpinyin"] = xpinyin_stub
+ return original
+
+
+def _noop_callback(*_args, **_kwargs):
+ pass
+
+
[email protected](scope="module")
+def table_module():
+ original = _install_table_import_stubs()
+ try:
+ module = import_module("rag.app.table")
+ yield reload(module)
+ finally:
+ for name, previous in original.items():
+ if previous is None:
+ sys.modules.pop(name, None)
+ else:
+ sys.modules[name] = previous
+
+
+def mock_update_kb():
+ with patch("rag.app.table.KnowledgebaseService.update_parser_config") as mock:
+ yield mock
+
+
+def dialog_service(monkeypatch):
+ from test.unit_test.api.db.services.test_gaussdb_dialog_sql import _install_settings_import_stubs
+
+ _install_settings_import_stubs(monkeypatch)
+ sys.modules.pop("api.db.services.dialog_service", None)
+ module = import_module("api.db.services.dialog_service")
+ try:
+ yield module
+ finally:
+ sys.modules.pop("api.db.services.dialog_service", None)
+ package = sys.modules.get("api.db.services")
+ if package is not None and hasattr(package, "dialog_service"):
+ delattr(package, "dialog_service")
+
+
+def _run_chunk(table_module, parser_config, mock_update_kb):
+ return table_module.chunk(
+ FILENAME,
+ binary=TEST_CSV,
+ callback=_noop_callback,
+ kb_id=KB_ID,
+ parser_config=parser_config,
+ lang="Chinese",
+ )
+
+
+def _assert_field_map_prompt_closure(dialog_service, chunks, field_map, expected_field_map):
+ assert field_map == expected_field_map
+ prompt = dialog_service.gaussdb_text_to_sql.build_sql_prompt(TABLE_NAME, field_map, "show table rows")
+ for field, path in expected_field_map.items():
+ assert path in chunks[0]["chunk_data"]
+ assert f" - {field} ({path}): chunk_data #>> '{{{path}}}'" in prompt
+
+
+def test_tc_sql_006_chunk_gaussdb_updates_field_map_and_column_names(
+ table_module,
+ mock_update_kb,
+ monkeypatch,
+ dialog_service,
+):
+ monkeypatch.setattr(settings, "DOC_ENGINE_GAUSSDB", True, raising=False)
+
+ chunks = _run_chunk(table_module, {}, mock_update_kb)
+
+ mock_update_kb.assert_called_once_with(
+ KB_ID,
+ {
+ "field_map": {
+ "row_id": "row_id",
+ "title": "title",
+ "content": "content",
+ "country": "country",
+ "category": "category",
+ },
+ "table_column_names": ["row_id", "title", "content", "country", "category"],
+ },
+ )
+ _assert_field_map_prompt_closure(
+ dialog_service,
+ chunks,
+ mock_update_kb.call_args.args[1]["field_map"],
+ {
+ "row_id": "row_id",
+ "title": "title",
+ "content": "content",
+ "country": "country",
+ "category": "category",
+ },
+ )
+
+
+def test_tc_sql_007_chunk_manual_mode_gaussdb_stores_metadata_in_chunk_data(
+ table_module,
+ mock_update_kb,
+ monkeypatch,
+ dialog_service,
+):
+ monkeypatch.setattr(settings, "DOC_ENGINE_GAUSSDB", True, raising=False)
+ parser_config = {
+ "table_column_mode": "manual",
+ "table_column_roles": {
+ "title": "indexing",
+ "content": "indexing",
+ "row_id": "metadata",
+ "country": "metadata",
+ "category": "metadata",
+ },
+ }
+
+ chunks = _run_chunk(table_module, parser_config, mock_update_kb)
+
+ assert chunks[0]["chunk_data"] == {
+ "row_id": 1,
+ "country": "Turkey",
+ "category": "Disaster",
+ }
+ assert "country_raw" not in chunks[0]
+ assert "- country:" not in chunks[0]["content_with_weight"]
+ assert "- title: Earthquake hits Turkey" in chunks[0]["content_with_weight"]
+ field_map = mock_update_kb.call_args.args[1]["field_map"]
+ assert field_map == {
+ "row_id": "row_id",
+ "country": "country",
+ "category": "category",
+ }
+ _assert_field_map_prompt_closure(
+ dialog_service,
+ chunks,
+ field_map,
+ {
+ "row_id": "row_id",
+ "country": "country",
+ "category": "category",
+ },
+ )
+
+
+def test_tc_sql_008_chunk_auto_mode_preserves_row_id_text_and_prompt_path(
+ table_module,
+ mock_update_kb,
+ monkeypatch,
+ dialog_service,
+):
+ monkeypatch.setattr(settings, "DOC_ENGINE_GAUSSDB", True, raising=False)
+
+ chunks = _run_chunk(table_module, {}, mock_update_kb)
+ field_map = mock_update_kb.call_args.args[1]["field_map"]
+
+ assert "- row_id: 1" in chunks[0]["content_with_weight"].splitlines()
+ assert chunks[0]["chunk_data"]["row_id"] == 1
+ _assert_field_map_prompt_closure(
+ dialog_service,
+ chunks,
+ field_map,
+ {
+ "row_id": "row_id",
+ "title": "title",
+ "content": "content",
+ "country": "country",
+ "category": "category",
+ },
+ )
diff --git a/test/unit_test/rag/nlp/test_gaussdb_retrieval.py b/test/unit_test/rag/nlp/test_gaussdb_retrieval.py
new file mode 100644
index 0000000000..4032eccc5e
--- /dev/null
+++ b/test/unit_test/rag/nlp/test_gaussdb_retrieval.py
@@ -0,0 +1,528 @@
+#
+# Copyright 2025 The InfiniFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+import sys
+import types
+from unittest.mock import AsyncMock, Mock
+
+import pytest
+from common.doc_store.doc_store_base import FusionExpr, MatchDenseExpr, MatchTextExpr
+
+
+class StubDocEngineConnection:
+ def db_type(self):
+ return "stub"
+
+
+class StubStorage:
+ def health(self):
+ return True
+
+
+def _install_module(name, **attrs):
+ module = types.ModuleType(name)
+ for key, value in attrs.items():
+ setattr(module, key, value)
+ sys.modules[name] = module
+ return module
+
+
+def dealer_cls():
+ module_names = [
+ "common.settings",
+ "common.token_utils",
+ "roman_numbers",
+ "word2number",
+ "cn2an",
+ "rag.nlp",
+ "rag.nlp.search",
+ "rag.nlp.query",
+ "rag.nlp.rag_tokenizer",
+ "rag.utils.redis_conn",
+ "memory.utils",
+ "json_repair",
+ ]
+ module_names.extend(
+ f"rag.utils.{name}"
+ for name in (
+ "es_conn",
+ "infinity_conn",
+ "ob_conn",
+ "opensearch_conn",
+ "azure_sas_conn",
+ "azure_spn_conn",
+ "gcs_conn",
+ "minio_conn",
+ "opendal_conn",
+ "s3_conn",
+ "oss_conn",
+ )
+ )
+ module_names.extend(f"memory.utils.{name}" for name in ("es_conn", "infinity_conn", "ob_conn"))
+ saved_modules = {name: sys.modules.get(name) for name in module_names}
+ import common
+ import memory
+
+ common_module = common
+ saved_common_settings = getattr(common_module, "settings", None) if common_module else None
+ saved_common_token_utils = getattr(common_module, "token_utils", None) if common_module else None
+ memory_module = memory
+ saved_memory_utils = getattr(memory_module, "utils", None) if memory_module else None
+
+ settings_stub = _install_module(
+ "common.settings",
+ DOC_ENGINE_GAUSSDB=True,
+ DOC_ENGINE_INFINITY=False,
+ DOC_ENGINE_OCEANBASE=False,
+ DOC_ENGINE_SERENEDB=False,
+ )
+ if common_module is not None:
+ setattr(common_module, "settings", settings_stub)
+ token_utils_stub = _install_module("common.token_utils", num_tokens_from_string=lambda value: len(str(value).split()))
+ if common_module is not None:
+ setattr(common_module, "token_utils", token_utils_stub)
+
+ _install_module("roman_numbers")
+ _install_module("word2number", w2n=types.SimpleNamespace(word_to_num=lambda value: 0))
+ _install_module("cn2an", cn2an=lambda value, *_args, **_kwargs: 0)
+ _install_module(
+ "rag.nlp.rag_tokenizer",
+ tokenize=lambda value: str(value),
+ fine_grained_tokenize=lambda value: str(value),
+ )
+ _install_module("rag.nlp.query", FulltextQueryer=lambda: types.SimpleNamespace(rmWWW=lambda value: value, hybrid_similarity=lambda *_args, **_kwargs: (0, 0, 0)))
+ _install_module("rag.utils.redis_conn", REDIS_CONN=types.SimpleNamespace(health=lambda: True, is_alive=lambda: False, REDIS=None))
+ _install_module("json_repair", loads=lambda value: value)
+ for short_name, attrs in {
+ "es_conn": {"ESConnection": StubDocEngineConnection},
+ "infinity_conn": {"InfinityConnection": StubDocEngineConnection},
+ "ob_conn": {"OBConnection": StubDocEngineConnection},
+ "opensearch_conn": {"OSConnection": StubDocEngineConnection},
+ "azure_sas_conn": {"RAGFlowAzureSasBlob": StubStorage},
+ "azure_spn_conn": {"RAGFlowAzureSpnBlob": StubStorage},
+ "gcs_conn": {"RAGFlowGCS": StubStorage},
+ "minio_conn": {"RAGFlowMinio": StubStorage},
+ "opendal_conn": {"OpenDALStorage": StubStorage},
+ "s3_conn": {"RAGFlowS3": StubStorage},
+ "oss_conn": {"RAGFlowOSS": StubStorage},
+ }.items():
+ _install_module(f"rag.utils.{short_name}", **attrs)
+
+ memory_utils = types.ModuleType("memory.utils")
+ memory_utils.__path__ = []
+ sys.modules["memory.utils"] = memory_utils
+ if memory_module is not None:
+ setattr(memory_module, "utils", memory_utils)
+ for short_name in ("es_conn", "infinity_conn", "ob_conn"):
+ module = _install_module(
+ f"memory.utils.{short_name}",
+ ESConnection=StubDocEngineConnection,
+ InfinityConnection=StubDocEngineConnection,
+ OBConnection=StubDocEngineConnection,
+ )
+ setattr(memory_utils, short_name, module)
+
+ try:
+ import importlib
+
+ sys.modules.pop("rag.nlp.search", None)
+ search_module = importlib.import_module("rag.nlp.search")
+ search_module.settings.DOC_ENGINE_GAUSSDB = True
+ search_module.settings.DOC_ENGINE_INFINITY = False
+ search_module.settings.DOC_ENGINE_OCEANBASE = False
+ search_module.settings.DOC_ENGINE_SERENEDB = False
+ yield search_module.Dealer
+ finally:
+ for name, module in saved_modules.items():
+ if module is None:
+ sys.modules.pop(name, None)
+ else:
+ sys.modules[name] = module
+ if common_module is not None:
+ if saved_common_settings is None:
+ if hasattr(common_module, "settings"):
+ delattr(common_module, "settings")
+ else:
+ setattr(common_module, "settings", saved_common_settings)
+ if saved_common_token_utils is None:
+ if hasattr(common_module, "token_utils"):
+ delattr(common_module, "token_utils")
+ else:
+ setattr(common_module, "token_utils", saved_common_token_utils)
+ if memory_module is not None:
+ if saved_memory_utils is None:
+ if hasattr(memory_module, "utils"):
+ delattr(memory_module, "utils")
+ else:
+ setattr(memory_module, "utils", saved_memory_utils)
+
+
+def make_dealer(dealer_cls):
+ dealer = dealer_cls.__new__(dealer_cls)
+ dealer.dataStore = FakeGaussDBStore()
+ return dealer
+
+
+class FakeGaussDBStore:
+ def db_type(self):
+ return "gaussdb"
+
+ def search(self, *args, **kwargs):
+ self.last_search = (args, kwargs)
+ return type("SearchResult", (), {"total": getattr(self, "total", 1), "chunks": []})()
+
+ def get_total(self, res):
+ return res.total
+
+ def get_doc_ids(self, _res):
+ return getattr(self, "ids", [])
+
+ def get_highlight(self, *_args):
+ return {}
+
+ def get_aggregation(self, *_args):
+ return {}
+
+ def get_fields(self, *_args):
+ return getattr(self, "fields", {})
+
+ async def get_scores(self, *_args):
+ raise AssertionError("GaussDB retrieval must not call ES _knn_scores")
+
+
+class FakeQueryer:
+ def question(self, text, min_match=0.3):
+ return MatchTextExpr(["content_with_weight"], text, 1024), [text]
+
+
+def retrieval_chunk(score, doc_id, doc_name, content="risk contract"):
+ return {
+ "_score": score,
+ "content_ltks": content,
+ "content_with_weight": content,
+ "doc_id": doc_id,
+ "docnm_kwd": doc_name,
+ "kb_id": "kb1",
+ "position_int": [1],
+ }
+
+
+ ("vector_weight", "expected_weights"),
+ [(0.75, "0.25,0.75"), (0.5, "0.5,0.5")],
+)
+async def test_tc_ret_801_dealer_search_uses_requested_gaussdb_fusion_weight(
+ dealer_cls,
+ vector_weight,
+ expected_weights,
+):
+ dealer = make_dealer(dealer_cls)
+ dealer.qryr = FakeQueryer()
+
+ async def fake_get_vector(_text, _emb_mdl, topk=10, similarity=0.1):
+ return MatchDenseExpr("q_4_vec", [0.1, 0.2, 0.3, 0.4], "float", "cosine", topk, {"similarity": similarity})
+
+ dealer.get_vector = fake_get_vector
+
+ await dealer.search(
+ {
+ "question": "risk",
+ "page": 1,
+ "size": 10,
+ "topk": 10,
+ "similarity": 0.2,
+ "vector_similarity_weight": vector_weight,
+ },
+ "ragflow_tenant",
+ ["kb1"],
+ emb_mdl=object(),
+ rank_feature={"pagerank_fea": 7},
+ )
+
+ args, kwargs = dealer.dataStore.last_search
+ match_exprs = args[3]
+ dense_expr = next(expr for expr in match_exprs if isinstance(expr, MatchDenseExpr))
+ fusion_expr = next(expr for expr in match_exprs if isinstance(expr, FusionExpr))
+
+ assert kwargs == {"rank_feature": {"pagerank_fea": 7}}
+ assert dense_expr.topn == 10
+ assert dense_expr.extra_options["similarity"] == 0.2
+ assert fusion_expr.method == "weighted_sum"
+ assert fusion_expr.topn == 10
+ assert fusion_expr.fusion_params == {"weights": expected_weights}
+
+
+async def test_tc_ret_802_dealer_search_uses_default_gaussdb_fusion_weight(dealer_cls):
+ dealer = make_dealer(dealer_cls)
+ dealer.qryr = FakeQueryer()
+
+ async def fake_get_vector(_text, _emb_mdl, topk=10, similarity=0.1):
+ return MatchDenseExpr("q_4_vec", [0.1, 0.2, 0.3, 0.4], "float", "cosine", topk, {"similarity": similarity})
+
+ dealer.get_vector = fake_get_vector
+
+ await dealer.search({"question": "risk", "page": 1, "size": 5}, "ragflow_tenant", ["kb1"], emb_mdl=object())
+
+ args, _kwargs = dealer.dataStore.last_search
+ fusion_expr = next(expr for expr in args[3] if isinstance(expr, FusionExpr))
+
+ assert fusion_expr.fusion_params["weights"] == "0.7,0.3"
+
+
+async def test_tc_ret_306_retrieval_passes_threshold_with_original_similarity_key(dealer_cls):
+ dealer = make_dealer(dealer_cls)
+ captured = {}
+
+ async def fake_search(req, *_args, **_kwargs):
+ captured["req"] = req
+ return type("SearchResult", (), {"total": 0, "chunks": []})()
+
+ async def fake_prune(sres):
+ return sres
+
+ dealer.search = fake_search
+ dealer._prune_deleted_chunks = fake_prune
+
+ await dealer.retrieval(
+ "risk",
+ object(),
+ "tenant",
+ ["kb1"],
+ page=1,
+ page_size=10,
+ similarity_threshold=0.2,
+ vector_similarity_weight=0.75,
+ top=77,
+ doc_ids=["doc1"],
+ )
+
+ assert captured["req"]["doc_ids"] == ["doc1"]
+ assert captured["req"]["topk"] == 77
+ assert captured["req"]["similarity"] == 0.2
+ assert captured["req"]["available_int"] == 1
+ assert captured["req"]["vector_similarity_weight"] == 0.75
+ assert "similarity_threshold" not in captured["req"]
+
+
+async def _run_tc_ret_807_808_retrieval_scenario(dealer_cls):
+ dealer = make_dealer(dealer_cls)
+ dealer.qryr = FakeQueryer()
+ dealer.dataStore.total = 3
+ dealer.dataStore.ids = ["low", "high", "mid"]
+ dealer.dataStore.fields = {
+ "low": retrieval_chunk(0.1, "doc-low", "Low", "low"),
+ "high": retrieval_chunk(0.9, "doc-high", "High", "high"),
+ "mid": retrieval_chunk(0.4, "doc-mid", "Mid", "mid"),
+ }
+
+ async def fake_get_vector(_text, _emb_mdl, topk=10, similarity=0.1):
+ return MatchDenseExpr("q_4_vec", [0.1, 0.2, 0.3, 0.4], "float", "cosine", topk, {"similarity": similarity})
+
+ async def fake_prune(sres):
+ return sres
+
+ dealer.get_vector = fake_get_vector
+ dealer._prune_deleted_chunks = fake_prune
+ dealer._knn_scores = AsyncMock(side_effect=AssertionError("_knn_scores must not be called"))
+ dealer.rerank_by_model = Mock(side_effect=AssertionError("rerank_by_model must not be called"))
+ dealer.rerank = Mock(side_effect=AssertionError("rerank must not be called"))
+ dealer.rerank_with_knn = Mock(side_effect=AssertionError("rerank_with_knn must not be called"))
+
+ ranks = await dealer.retrieval(
+ "risk",
+ object(),
+ "tenant",
+ ["kb1"],
+ page=1,
+ page_size=10,
+ similarity_threshold=0.2,
+ vector_similarity_weight=0.75,
+ rerank_mdl=None,
+ )
+
+ args, kwargs = dealer.dataStore.last_search
+ dense_expr = next(expr for expr in args[3] if isinstance(expr, MatchDenseExpr))
+ fusion_expr = next(expr for expr in args[3] if isinstance(expr, FusionExpr))
+
+ return dealer, ranks, dense_expr, fusion_expr, kwargs
+
+
+async def test_tc_ret_807_retrieval_uses_gaussdb_fusion_weight_without_es_default(monkeypatch, dealer_cls):
+ dealer, _, dense_expr, fusion_expr, kwargs = await _run_tc_ret_807_808_retrieval_scenario(dealer_cls)
+
+ assert kwargs == {"rank_feature": {"pagerank_fea": 10}}
+ assert dense_expr.extra_options["similarity"] == 0.2
+ assert fusion_expr.fusion_params == {"weights": "0.25,0.75"}
+ assert fusion_expr.fusion_params["weights"] != "0.05,0.95"
+
+ search_module = sys.modules[dealer_cls.__module__]
+ monkeypatch.setattr(search_module.settings, "DOC_ENGINE_GAUSSDB", False)
+ await dealer.search(
+ {
+ "question": "risk",
+ "page": 1,
+ "size": 10,
+ "topk": 10,
+ "similarity": 0.2,
+ "vector_similarity_weight": 0.75,
+ },
+ "ragflow_tenant",
+ ["kb1"],
+ emb_mdl=object(),
+ )
+ non_gauss_match_exprs = dealer.dataStore.last_search[0][3]
+ non_gauss_fusion = next(expr for expr in non_gauss_match_exprs if isinstance(expr, FusionExpr))
+ assert non_gauss_fusion.fusion_params == {"weights": "0.001,1"}
+
+
+async def test_tc_ret_808_retrieval_uses_gaussdb_scores_without_local_rerank(dealer_cls):
+ dealer, ranks, _, _, _ = await _run_tc_ret_807_808_retrieval_scenario(dealer_cls)
+
+ assert ranks["total"] == 2
+ assert [chunk["chunk_id"] for chunk in ranks["chunks"]] == ["high", "mid"]
+ assert "low" not in {chunk["chunk_id"] for chunk in ranks["chunks"]}
+ assert [chunk["similarity"] for chunk in ranks["chunks"]] == [0.9, 0.4]
+ assert [chunk["vector_similarity"] for chunk in ranks["chunks"]] == [0.9, 0.4]
+ assert [chunk["term_similarity"] for chunk in ranks["chunks"]] == [0.9, 0.4]
+ assert ranks["chunks"][0]["vector"] == [0.0, 0.0, 0.0, 0.0]
+ assert ranks["doc_aggs"] == [
+ {"doc_name": "High", "doc_id": "doc-high", "count": 1},
+ {"doc_name": "Mid", "doc_id": "doc-mid", "count": 1},
+ ]
+ dealer._knn_scores.assert_not_awaited()
+ dealer.rerank_by_model.assert_not_called()
+ dealer.rerank.assert_not_called()
+ dealer.rerank_with_knn.assert_not_called()
+
+
+async def _run_gaussdb_rank_feature_retrieval(dealer_cls, fields, rank_feature):
+ dealer = make_dealer(dealer_cls)
+ dealer.qryr = FakeQueryer()
+ dealer.dataStore.total = len(fields)
+ dealer.dataStore.ids = list(fields)
+ dealer.dataStore.fields = fields
+
+ async def fake_get_vector(_text, _emb_mdl, topk=10, similarity=0.1):
+ return MatchDenseExpr("q_4_vec", [0.1, 0.2, 0.3, 0.4], "float", "cosine", topk, {"similarity": similarity})
+
+ async def fake_prune(sres):
+ return sres
+
+ dealer.get_vector = fake_get_vector
+ dealer._prune_deleted_chunks = fake_prune
+ dealer._knn_scores = AsyncMock(side_effect=AssertionError("_knn_scores must not be called"))
+ dealer.rerank_by_model = Mock(side_effect=AssertionError("rerank_by_model must not be called"))
+ dealer.rerank = Mock(side_effect=AssertionError("rerank must not be called"))
+ dealer.rerank_with_knn = Mock(side_effect=AssertionError("rerank_with_knn must not be called"))
+
+ return await dealer.retrieval(
+ "risk",
+ object(),
+ "tenant",
+ ["kb1"],
+ page=1,
+ page_size=10,
+ similarity_threshold=0.2,
+ vector_similarity_weight=0.75,
+ rerank_mdl=None,
+ rank_feature=rank_feature,
+ )
+
+
+async def test_tc_ret_704_gaussdb_retrieval_adds_tag_feature_score_to_sql_score(dealer_cls):
+ fields = {
+ "without-tag": retrieval_chunk(0.4, "doc-plain", "Plain"),
+ "with-tag": {**retrieval_chunk(0.4, "doc-tagged", "Tagged"), "tag_feas": {"risk": 1.0}},
+ }
+
+ ranks = await _run_gaussdb_rank_feature_retrieval(dealer_cls, fields, {"risk": 1.0})
+
+ assert [chunk["chunk_id"] for chunk in ranks["chunks"]] == ["with-tag", "without-tag"]
+ assert [chunk["similarity"] for chunk in ranks["chunks"]] == [10.4, 0.4]
+ assert [chunk["vector_similarity"] for chunk in ranks["chunks"]] == [0.4, 0.4]
+ assert [chunk["term_similarity"] for chunk in ranks["chunks"]] == [0.4, 0.4]
+
+
+async def test_tc_ret_704_gaussdb_retrieval_does_not_add_pagerank_twice(dealer_cls):
+ fields = {
+ "higher-sql-score": {**retrieval_chunk(0.8, "doc-high", "High"), "pagerank_fea": 0.1},
+ "higher-pagerank": {**retrieval_chunk(0.7, "doc-pagerank", "PageRank"), "pagerank_fea": 0.9},
+ }
+
+ ranks = await _run_gaussdb_rank_feature_retrieval(dealer_cls, fields, {"pagerank_fea": 10})
+
+ assert [chunk["chunk_id"] for chunk in ranks["chunks"]] == ["higher-sql-score", "higher-pagerank"]
+ assert [chunk["similarity"] for chunk in ranks["chunks"]] == [0.8, 0.7]
+ assert [chunk["vector_similarity"] for chunk in ranks["chunks"]] == [0.8, 0.7]
+ assert [chunk["term_similarity"] for chunk in ranks["chunks"]] == [0.8, 0.7]
+
+
+async def test_tc_ret_311_retrieval_keeps_term_only_gaussdb_scores_when_vector_weight_is_zero(dealer_cls):
+ dealer = make_dealer(dealer_cls)
+ captured = {}
+
+ async def fake_search(req, *_args, **_kwargs):
+ captured["req"] = req
+ return dealer_cls.SearchResult(
+ total=2,
+ ids=["term-low", "term-zero"],
+ query_vector=[0.1, 0.2],
+ field={
+ "term-low": retrieval_chunk(0.1, "doc-low", "Low"),
+ "term-zero": retrieval_chunk(0.0, "doc-zero", "Zero"),
+ },
+ )
+
+ async def fake_prune(sres):
+ return sres
+
+ dealer.search = fake_search
+ dealer._prune_deleted_chunks = fake_prune
+ dealer._knn_scores = AsyncMock(side_effect=AssertionError("_knn_scores must not be called"))
+ dealer.rerank_by_model = Mock(side_effect=AssertionError("rerank_by_model must not be called"))
+ dealer.rerank = Mock(side_effect=AssertionError("rerank must not be called"))
+ dealer.rerank_with_knn = Mock(side_effect=AssertionError("rerank_with_knn must not be called"))
+
+ ranks = await dealer.retrieval(
+ "risk",
+ object(),
+ "tenant",
+ ["kb1"],
+ page=1,
+ page_size=10,
+ similarity_threshold=0.8,
+ vector_similarity_weight=0.0,
+ )
+
+ assert captured["req"]["similarity"] == 0.8
+ assert captured["req"]["vector_similarity_weight"] == 0.0
+ assert ranks["total"] == 2
+ assert [chunk["chunk_id"] for chunk in ranks["chunks"]] == ["term-low", "term-zero"]
+ assert [chunk["similarity"] for chunk in ranks["chunks"]] == [0.1, 0.0]
+ assert [chunk["vector_similarity"] for chunk in ranks["chunks"]] == [0.1, 0.0]
+ assert [chunk["term_similarity"] for chunk in ranks["chunks"]] == [0.1, 0.0]
+ dealer._knn_scores.assert_not_awaited()
+ dealer.rerank_by_model.assert_not_called()
+ dealer.rerank.assert_not_called()
+ dealer.rerank_with_knn.assert_not_called()
diff --git a/test/unit_test/rag/svr/test_gaussdb_table_column_roles_helpers.py b/test/unit_test/rag/svr/test_gaussdb_table_column_roles_helpers.py
new file mode 100644
index 0000000000..c5867aa397
--- /dev/null
+++ b/test/unit_test/rag/svr/test_gaussdb_table_column_roles_helpers.py
@@ -0,0 +1,110 @@
+import common.settings as settings
+
+from rag.utils.table_es_metadata import (
+ aggregate_table_doc_metadata,
+ merge_table_parser_config_from_kb,
+ table_parser_strip_doc_metadata_keys,
+)
+
+
+def test_tc_sql_009_parser_config_merge_and_metadata_cleanup_keys():
+ task = {
+ "parser_id": "table",
+ "parser_config": {"llm_id": "x"},
+ "kb_parser_config": {
+ "table_column_mode": "manual",
+ "table_column_roles": {"a": "metadata"},
+ "table_column_names": ["a", "b"],
+ },
+ }
+
+ assert merge_table_parser_config_from_kb(task) == {
+ "llm_id": "x",
+ "table_column_mode": "manual",
+ "table_column_roles": {"a": "metadata"},
+ "table_column_names": ["a", "b"],
+ }
+ assert merge_table_parser_config_from_kb(
+ {
+ "parser_id": "table",
+ "parser_config": {"foo": 1},
+ "kb_parser_config": {"llm_id": "ignored"},
+ }
+ ) == {"foo": 1}
+ assert table_parser_strip_doc_metadata_keys({"table_column_names": ["Region", " SKU "]}) == frozenset({"Region", "SKU"})
+ assert table_parser_strip_doc_metadata_keys({"table_column_roles": {"x": "metadata", "y": "indexing"}}) == frozenset({"x", "y"})
+ assert table_parser_strip_doc_metadata_keys({"table_column_names": [], "table_column_roles": {"only": "both"}}) == frozenset({"only"})
+
+
+def test_tc_sql_010_es_aggregates_public_chunk_metadata(monkeypatch):
+ monkeypatch.setattr(settings, "DOC_ENGINE_INFINITY", False)
+ monkeypatch.setattr(settings, "DOC_ENGINE_OCEANBASE", False)
+ monkeypatch.setattr(settings, "DOC_ENGINE_GAUSSDB", False, raising=False)
+ chunks = [
+ {
+ "guojia_tks": ["tokenized", "country"],
+ "guojia_raw": "Brazil",
+ "category_tks": ["Risk", "Audit"],
+ "score_long": 3,
+ },
+ {
+ "guojia_tks": ["other"],
+ "guojia_raw": "Brazil",
+ "category_tks": ["Operations"],
+ "score_long": 4,
+ },
+ ]
+ task = {
+ "parser_id": "table",
+ "parser_config": {
+ "table_column_mode": "manual",
+ "table_column_names": ["country", "category", "score", "title"],
+ "table_column_roles": {
+ "country": "metadata",
+ "category": "both",
+ "score": "metadata",
+ "title": "indexing",
+ },
+ },
+ "kb_parser_config": {
+ "field_map": {
+ "guojia_tks": "country",
+ "category_tks": "category",
+ "score_long": "score",
+ }
+ },
+ }
+
+ assert aggregate_table_doc_metadata(chunks, task) == {
+ "country": ["Brazil"],
+ "category": ["Risk Audit", "Operations"],
+ "score": ["3", "4"],
+ }
+
+
+def test_tc_sql_011_gaussdb_aggregates_chunk_data_metadata(monkeypatch):
+ monkeypatch.setattr(settings, "DOC_ENGINE_INFINITY", False)
+ monkeypatch.setattr(settings, "DOC_ENGINE_OCEANBASE", False)
+ monkeypatch.setattr(settings, "DOC_ENGINE_GAUSSDB", True, raising=False)
+ chunks = [
+ {"chunk_data": {"country": "Turkey", "category": "Disaster", "title": "Earthquake"}},
+ {"chunk_data": {"country": "Turkey", "category": "Economy"}},
+ {"chunk_data": {"country": "EU", "category": None}},
+ {"country_raw": "must not be read on GaussDB"},
+ ]
+ task = {
+ "parser_id": "table",
+ "parser_config": {
+ "table_column_names": ["country", "category", "title"],
+ "table_column_roles": {
+ "country": "metadata",
+ "category": "both",
+ "title": "indexing",
+ },
+ },
+ }
+
+ assert aggregate_table_doc_metadata(chunks, task) == {
+ "country": ["Turkey", "EU"],
+ "category": ["Disaster", "Economy"],
+ }
diff --git a/test/unit_test/rag/utils/test_gaussdb_conn.py b/test/unit_test/rag/utils/test_gaussdb_conn.py
new file mode 100644
index 0000000000..03f885b2c1
--- /dev/null
+++ b/test/unit_test/rag/utils/test_gaussdb_conn.py
@@ -0,0 +1,2915 @@
+#
+# Copyright 2025 The InfiniFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+import json
+import math
+import sys
+import threading
+import types
+from unittest.mock import Mock
+
+import pytest
+import psycopg2
+from psycopg2 import errorcodes
+from sqlglot import exp, parse_one
+
+from common.doc_store.doc_store_base import FusionExpr, MatchDenseExpr, MatchTextExpr, OrderByExpr
+from common.doc_store.gaussdb_conn_base import (
+ GaussDBConnectionBase,
+ GaussDBDDLBuilder,
+ InvalidGaussDBObjectName,
+ UnsafeGaussDBSQL,
+)
+import rag.utils.gaussdb_conn as gaussdb_conn_module
+from rag.utils.gaussdb_conn import (
+ GaussDBConnection,
+ GaussDBError,
+ SearchResult,
+ decode_column_value,
+ ordered_columns,
+ parse_json_dict,
+ parse_fusion_vector_weight,
+ vector_literal,
+ zero_vector_literal,
+)
+
+
+class RecordingCursor:
+ def __init__(self):
+ self.executed = []
+ self.rowcount = 0
+ self.description = None
+ self.rows = []
+ self.closed = False
+
+ def execute(self, sql, params=None):
+ self.executed.append((sql, params or []))
+
+ def executemany(self, sql, params):
+ materialized = [list(row) for row in params]
+ self.executed.append((sql, materialized))
+ self.rowcount = len(materialized)
+
+ def fetchone(self):
+ return self.rows[0] if self.rows else None
+
+ def fetchall(self):
+ return self.rows
+
+ def close(self):
+ self.closed = True
+
+
+class UndefinedTableError(Exception):
+ pgcode = errorcodes.UNDEFINED_TABLE
+
+
+class MissingTableOnceCursor(RecordingCursor):
+ def __init__(self):
+ super().__init__()
+ self.insert_attempts = 0
+
+ def executemany(self, sql, params):
+ materialized = [list(row) for row in params]
+ self.executed.append((sql, materialized))
+ self.insert_attempts += 1
+ if self.insert_attempts == 1:
+ raise UndefinedTableError("chunk table does not exist")
+ self.rowcount = len(materialized)
+
+
+class SequencedCursor(RecordingCursor):
+ def __init__(self, results):
+ super().__init__()
+ self.results = list(results)
+
+ def execute(self, sql, params=None):
+ super().execute(sql, params)
+ rows, description = self.results.pop(0)
+ self.rows = rows
+ self.description = description
+
+
+class RecordingConnection:
+ def __init__(self, cursor):
+ self.cursor_obj = cursor
+ self.commits = 0
+ self.rollbacks = 0
+
+ def cursor(self):
+ return self.cursor_obj
+
+ def commit(self):
+ self.commits += 1
+
+ def rollback(self):
+ self.rollbacks += 1
+
+
+class RecordingPool:
+ def __init__(self, cursor):
+ self.cursor = cursor
+ self.conn = RecordingConnection(cursor)
+ self.get_calls = 0
+ self.put_back = []
+
+ def get_conn(self):
+ self.get_calls += 1
+ return self.conn
+
+ def put_conn(self, conn):
+ self.put_back.append(conn)
+
+
+class HealthPool:
+ masked_uri = "u@h:5432/d?schema=public"
+ resolved_schema = "public"
+
+ def __init__(self, rows):
+ self.rows = list(rows)
+ self.checked = False
+ self.sql = []
+
+ def check_schema_access(self):
+ self.checked = True
+
+ def fetch_one(self, sql, _params=None):
+ self.sql.append(sql)
+ row = self.rows.pop(0)
+ if isinstance(row, Exception):
+ raise row
+ return row
+
+
+def make_conn(cursor, vector_dimensions=None):
+ conn = GaussDBConnection.__new__(GaussDBConnection)
+ conn.schema = "public"
+ conn.resolved_schema = "public"
+ conn.pool = RecordingPool(cursor)
+ conn.ddl = GaussDBDDLBuilder(schema="public")
+ conn.logger = type("Logger", (), {"debug": lambda *_args, **_kwargs: None, "error": lambda *_args, **_kwargs: None})()
+ if vector_dimensions is not None:
+ conn.get_vector_dimensions = lambda _table: vector_dimensions
+ return conn
+
+
+CANONICAL_CREATE_IDX_1024_EXECUTIONS = [
+ ("SELECT pg_advisory_xact_lock(hashtext(%s))", ["create_idx:public:ragflow_wrt_012"]),
+ (
+ """CREATE TABLE IF NOT EXISTS "public"."ragflow_wrt_012" (
+ id VARCHAR(256) NOT NULL,
+ kb_id VARCHAR(256) NOT NULL,
+ doc_id VARCHAR(256),
+ docnm_kwd VARCHAR(256),
+ doc_type_kwd VARCHAR(256),
+ title_tks VARCHAR(256),
+ title_sm_tks VARCHAR(256),
+ content_with_weight TEXT,
+ content_ltks TEXT,
+ content_sm_ltks TEXT,
+ important_kwd JSONB,
+ important_tks TEXT,
+ question_kwd JSONB,
+ question_tks TEXT,
+ tag_kwd JSONB,
+ tag_feas JSONB,
+ available_int INTEGER DEFAULT 1 NOT NULL,
+ pagerank_fea INTEGER,
+ create_time VARCHAR(19),
+ create_timestamp_flt DOUBLE PRECISION,
+ img_id VARCHAR(128),
+ position_int JSONB,
+ page_num_int JSONB,
+ top_int JSONB,
+ metadata JSONB,
+ chunk_data JSONB,
+ extra JSONB,
+ _order_id INTEGER,
+ group_id VARCHAR(256),
+ mom_id VARCHAR(256),
+ knowledge_graph_kwd VARCHAR(256),
+ source_id JSONB,
+ entity_kwd VARCHAR(256),
+ entity_type_kwd VARCHAR(256),
+ from_entity_kwd VARCHAR(256),
+ to_entity_kwd VARCHAR(256),
+ weight_int INTEGER,
+ weight_flt DOUBLE PRECISION,
+ entities_kwd JSONB,
+ rank_flt DOUBLE PRECISION,
+ n_hop_with_weight TEXT,
+ removed_kwd VARCHAR(256) DEFAULT 'N',
+ raptor_kwd VARCHAR(256),
+ raptor_layer_int INTEGER,
+ PRIMARY KEY (kb_id, id)
+) WITH (storage_type=USTORE)""",
+ [],
+ ),
+ (
+ 'CREATE INDEX IF NOT EXISTS "idx_gdb_ragflow_wrt_012_doc_id" ON "public"."ragflow_wrt_012" (doc_id)',
+ [],
+ ),
+ (
+ 'CREATE INDEX IF NOT EXISTS "idx_gdb_ragflow_wrt_012_available_int" ON "public"."ragflow_wrt_012" (available_int)',
+ [],
+ ),
+ (
+ 'CREATE INDEX IF NOT EXISTS "idx_gdb_ragflow_wrt_012_knowledge_graph_kwd" ON "public"."ragflow_wrt_012" (knowledge_graph_kwd)',
+ [],
+ ),
+ (
+ 'CREATE INDEX IF NOT EXISTS "idx_gdb_ragflow_wrt_012_entity_type_kwd" ON "public"."ragflow_wrt_012" (entity_type_kwd)',
+ [],
+ ),
+ (
+ 'CREATE INDEX IF NOT EXISTS "idx_gdb_ragflow_wrt_012_removed_kwd" ON "public"."ragflow_wrt_012" (removed_kwd)',
+ [],
+ ),
+ (
+ """CREATE INDEX IF NOT EXISTS "idx_gdb_ragflow_wrt_012_fts_all"
+ ON "public"."ragflow_wrt_012"
+ USING ugin(to_tsvector('simple', """
+ "coalesce(title_tks, ' ') || ' ' || coalesce(title_sm_tks, ' ') || ' ' || "
+ "coalesce(important_tks, ' ') || ' ' || coalesce(question_tks, ' ') || ' ' || "
+ "coalesce(content_ltks, ' ') || ' ' || coalesce(content_sm_ltks, ' ')))",
+ [],
+ ),
+ (
+ """CREATE INDEX IF NOT EXISTS "idx_gdb_ragflow_wrt_012_fts_all_ngram"
+ ON "public"."ragflow_wrt_012"
+ USING ugin(to_tsvector('ngram', """
+ "coalesce(title_tks, ' ') || ' ' || coalesce(title_sm_tks, ' ') || ' ' || "
+ "coalesce(important_tks, ' ') || ' ' || coalesce(question_tks, ' ') || ' ' || "
+ "coalesce(content_ltks, ' ') || ' ' || coalesce(content_sm_ltks, ' ')))",
+ [],
+ ),
+ (
+ 'ALTER TABLE "public"."ragflow_wrt_012" ADD COLUMN IF NOT EXISTS q_1024_vec floatvector(1024) DEFAULT (array_fill(0, ARRAY[1024])::text::floatvector(1024))',
+ [],
+ ),
+ (
+ 'ALTER TABLE "public"."ragflow_wrt_012" ADD COLUMN IF NOT EXISTS q_1024_vec_valid BOOLEAN DEFAULT FALSE NOT NULL',
+ [],
+ ),
+ (
+ "SELECT pg_advisory_xact_lock(hashtext(%s))",
+ ["create_vector_idx:public:ragflow_wrt_012:1024"],
+ ),
+ ("SET LOCAL maintenance_work_mem = '1GB'", []),
+ (
+ 'CREATE INDEX IF NOT EXISTS "idx_gdb_ragflow_wrt_012_q_1024_vec_diskann" ON "public"."ragflow_wrt_012" USING gsdiskann (q_1024_vec COSINE) WITH (subgraph_count=1)',
+ [],
+ ),
+]
+
+
+CANONICAL_CREATE_DOC_META_EXECUTIONS = [
+ (
+ "SELECT pg_advisory_xact_lock(hashtext(%s))",
+ ["create_doc_meta_idx:public:ragflow_doc_meta_tenant"],
+ ),
+ (
+ """CREATE TABLE IF NOT EXISTS "public"."ragflow_doc_meta_tenant" (
+ id VARCHAR(256) NOT NULL,
+ kb_id VARCHAR(256) NOT NULL,
+ meta_fields JSONB,
+ PRIMARY KEY (id)
+) WITH (storage_type=USTORE)""",
+ [],
+ ),
+ (
+ 'CREATE INDEX IF NOT EXISTS "idx_gdb_ragflow_doc_meta_tenant_kb_id" ON "public"."ragflow_doc_meta_tenant" (kb_id)',
+ [],
+ ),
+]
+
+
+def test_tc_sql_501_sql_executes_scoped_docengine_select_with_runtime_guard():
+ cursor = RecordingCursor()
+ cursor.description = [("doc_id",), ("amount",)]
+ cursor.rows = [("doc1", "120")]
+ conn = make_conn(cursor)
+
+ result = conn.sql(
+ "SELECT doc_id, chunk_data #>> '{amount}' AS amount FROM ragflow_tenant WHERE kb_id = 'kb1'",
+ fetch_size=20,
+ )
+
+ sql, params = cursor.executed[-1]
+ assert cursor.executed[0] == ("SET LOCAL statement_timeout = 30000", [])
+ assert 'FROM "public".ragflow_tenant' in sql
+ assert "kb_id = 'kb1'" in sql
+ assert "LIMIT 20" in sql
+ assert params == []
+ assert result == {
+ "columns": [{"name": "doc_id", "type": "text"}, {"name": "amount", "type": "text"}],
+ "rows": [["doc1", "120"]],
+ }
+
+
+def test_tc_sql_306_sql_passes_fetch_size_to_readonly_guard(monkeypatch):
+ cursor = RecordingCursor()
+ cursor.description = [("doc_id",)]
+ cursor.rows = [("doc1",)]
+ conn = make_conn(cursor)
+ source_sql = "SELECT doc_id FROM ragflow_t1 WHERE kb_id = 'kb-1'"
+ guarded_sql = f"{source_sql} LIMIT 64"
+ validator = Mock()
+ validator.validate_and_patch.return_value = types.SimpleNamespace(sql=guarded_sql)
+ readonly_guard = Mock(return_value=validator)
+ gaussdb_module = sys.modules[GaussDBConnection.__module__]
+ monkeypatch.setattr(gaussdb_module.GaussDBSQLValidator, "readonly_guard", readonly_guard)
+
+ result = conn.sql(source_sql, fetch_size=64)
+
+ readonly_guard.assert_called_once_with(default_limit=64, execution_schema="public")
+ validator.validate_and_patch.assert_called_once_with(source_sql)
+ assert cursor.executed == [
+ ("SET LOCAL statement_timeout = 30000", []),
+ (guarded_sql, []),
+ ]
+ assert result == {
+ "columns": [{"name": "doc_id", "type": "text"}],
+ "rows": [["doc1"]],
+ }
+
+
+def test_tc_cfg_312_db_type_reports_gaussdb():
+ assert make_conn(RecordingCursor()).db_type() == "gaussdb"
+ assert GaussDBConnectionBase(pool=HealthPool([])).db_type() == "gaussdb"
+
+
+def test_tc_sql_511_sql_returns_connection_after_query_failure():
+ class FailingCursor(RecordingCursor):
+ def execute(self, sql, params=None):
+ super().execute(sql, params)
+ if not str(sql).startswith("SET LOCAL"):
+ raise RuntimeError("query timeout")
+
+ cursor = FailingCursor()
+ conn = make_conn(cursor)
+
+ with pytest.raises(RuntimeError, match="query timeout"):
+ conn.sql("SELECT doc_id FROM ragflow_tenant WHERE kb_id = 'kb1'")
+
+ assert cursor.executed[0] == ("SET LOCAL statement_timeout = 30000", [])
+ assert cursor.executed[1][0] == ("SELECT doc_id FROM \"public\".ragflow_tenant WHERE kb_id = 'kb1' LIMIT 128")
+ assert cursor.closed is True
+ assert conn.pool.put_back == [conn.pool.conn]
+
+
+ "sql",
+ [
+ "SELECT doc_id FROM pg_class WHERE kb_id = 'kb1'",
+ "SELECT doc_id FROM public.ragflow_tenant WHERE kb_id = 'kb1'",
+ "SELECT doc_id FROM ragflow_tenant",
+ "SELECT doc_id FROM ragflow_tenant WHERE kb_id = 'kb1' OR 1 = 1",
+ "SELECT content_with_weight FROM ragflow_tenant WHERE kb_id = 'kb1'",
+ "SELECT chunk_data ->> 'amount' FROM ragflow_tenant WHERE kb_id = 'kb1'",
+ "SELECT current_database() FROM ragflow_tenant WHERE kb_id = 'kb1'",
+ ],
+)
+def test_tc_sql_510_sql_runtime_guard_rejects_unscoped_or_unsafe_sql(sql):
+ cursor = RecordingCursor()
+ conn = make_conn(cursor)
+
+ with pytest.raises(UnsafeGaussDBSQL):
+ conn.sql(sql)
+
+ assert cursor.executed == []
+
+
+def test_tc_wrt_102_insert_chunk_uses_parameterized_upsert_and_valid_vector_flag():
+ cursor = RecordingCursor()
+ conn = make_conn(cursor, vector_dimensions=[1024])
+ kb_id = "kb-wrt-102"
+ vector = [0.1] * 1024
+ vector_param = "[" + ",".join(["0.1"] * 1024) + "]"
+
+ errors = conn.insert(
+ [{"id": "c1", "kb_id": kb_id, "docnm_kwd": None, "q_1024_vec": vector, "content_with_weight": "x"}],
+ "ragflow_wrt_102",
+ kb_id,
+ )
+
+ expected_sql = (
+ 'INSERT INTO "public"."ragflow_wrt_102" '
+ "(id, kb_id, docnm_kwd, content_with_weight, available_int, removed_kwd, q_1024_vec, q_1024_vec_valid) "
+ "VALUES (%s, %s, %s, %s, %s, %s, %s::floatvector(1024), %s) "
+ "ON DUPLICATE KEY UPDATE docnm_kwd = VALUES(docnm_kwd), content_with_weight = VALUES(content_with_weight), "
+ "available_int = VALUES(available_int), removed_kwd = VALUES(removed_kwd), "
+ "q_1024_vec = VALUES(q_1024_vec), q_1024_vec_valid = VALUES(q_1024_vec_valid)"
+ )
+ expected_params = [["c1", kb_id, None, "x", 1, "N", vector_param, True]]
+ assert errors == []
+ assert cursor.executed == [(expected_sql, expected_params)]
+ assert "c1" not in expected_sql
+ assert kb_id not in expected_sql
+ assert vector_param not in expected_sql
+ assert conn.pool.conn.commits == 1
+ assert conn.pool.conn.rollbacks == 0
+ assert cursor.closed is True
+ assert conn.pool.put_back == [conn.pool.conn]
+
+
+def test_tc_wrt_115_insert_chunk_creates_missing_gaussdb_table_and_retries_once():
+ cursor = MissingTableOnceCursor()
+ conn = make_conn(cursor, vector_dimensions=[4])
+ create_calls = []
+ conn.index_exist = lambda *_args: False
+ conn.create_idx = lambda index_name, dataset_id, vector_size: create_calls.append((index_name, dataset_id, vector_size))
+
+ errors = conn.insert(
+ [
+ {
+ "id": "c1",
+ "kb_id": "kb1",
+ "doc_id": "d1",
+ "content_with_weight": "hello",
+ "q_4_vec": [0.1, 0.2, 0.3, 0.4],
+ }
+ ],
+ "ragflow_tenant",
+ "kb1",
+ )
+
+ assert errors == []
+ assert create_calls == [("ragflow_tenant", "kb1", 4)]
+ assert cursor.insert_attempts == 2
+ assert conn.pool.conn.rollbacks == 1
+ assert conn.pool.conn.commits == 1
+
+
+def test_tc_wrt_115_insert_chunk_does_not_create_table_for_unrelated_write_failure():
+ class InvalidWriteCursor(RecordingCursor):
+ def executemany(self, sql, params):
+ super().executemany(sql, params)
+ raise ValueError("invalid chunk value")
+
+ cursor = InvalidWriteCursor()
+ conn = make_conn(cursor, vector_dimensions=[4])
+ create_calls = []
+ conn.create_idx = lambda *args: create_calls.append(args)
+
+ errors = conn.insert(
+ [{"id": "c1", "kb_id": "kb1", "doc_id": "d1", "q_4_vec": [0.1, 0.2, 0.3, 0.4]}],
+ "ragflow_tenant",
+ "kb1",
+ )
+
+ assert errors == ["c1"]
+ assert create_calls == []
+ assert conn.pool.conn.rollbacks == 1
+
+
+def test_tc_wrt_115_insert_chunk_does_not_create_table_when_target_still_exists():
+ cursor = MissingTableOnceCursor()
+ conn = make_conn(cursor, vector_dimensions=[4])
+ conn.index_exist = lambda *_args: True
+ create_calls = []
+ conn.create_idx = lambda *args: create_calls.append(args)
+
+ errors = conn.insert(
+ [{"id": "c1", "kb_id": "kb1", "doc_id": "d1", "q_4_vec": [0.1, 0.2, 0.3, 0.4]}],
+ "ragflow_tenant",
+ "kb1",
+ )
+
+ assert errors == ["c1"]
+ assert create_calls == []
+ assert cursor.insert_attempts == 1
+
+
+def test_tc_wrt_108_insert_normalizes_kb_id_list_to_first_value():
+ cursor = RecordingCursor()
+ conn = make_conn(cursor, vector_dimensions=[4])
+ vector = [0.1, 0.2, 0.3, 0.4]
+
+ errors = conn.insert(
+ [{"id": "c1", "kb_id": ["kb-wrt-108", "kb-other"], "q_4_vec": vector, "content_with_weight": "test"}],
+ "ragflow_wrt_108",
+ "kb-wrt-108",
+ )
+
+ expected_sql = (
+ 'INSERT INTO "public"."ragflow_wrt_108" '
+ "(id, kb_id, content_with_weight, available_int, removed_kwd, q_4_vec, q_4_vec_valid) "
+ "VALUES (%s, %s, %s, %s, %s, %s::floatvector(4), %s) "
+ "ON DUPLICATE KEY UPDATE content_with_weight = VALUES(content_with_weight), "
+ "available_int = VALUES(available_int), removed_kwd = VALUES(removed_kwd), "
+ "q_4_vec = VALUES(q_4_vec), q_4_vec_valid = VALUES(q_4_vec_valid)"
+ )
+ expected_params = [["c1", "kb-wrt-108", "test", 1, "N", "[0.1,0.2,0.3,0.4]", True]]
+ assert errors == []
+ assert cursor.executed == [(expected_sql, expected_params)]
+ assert "kb-wrt-108" not in expected_sql
+ assert "kb-other" not in expected_sql
+
+
+def test_tc_wrt_201_insert_missing_vector_writes_invalid_placeholder_vector():
+ cursor = RecordingCursor()
+ conn = make_conn(cursor, vector_dimensions=[1024])
+ zero_vector = "[" + ",".join(["0"] * 1024) + "]"
+
+ errors = conn.insert(
+ [{"id": "c2", "kb_id": "k1", "content_with_weight": "mother"}],
+ "ragflow_wrt_201",
+ "k1",
+ )
+
+ expected_sql = (
+ 'INSERT INTO "public"."ragflow_wrt_201" '
+ "(id, kb_id, content_with_weight, available_int, removed_kwd, q_1024_vec, q_1024_vec_valid) "
+ "VALUES (%s, %s, %s, %s, %s, %s::floatvector(1024), %s) "
+ "ON DUPLICATE KEY UPDATE content_with_weight = VALUES(content_with_weight), "
+ "available_int = VALUES(available_int), removed_kwd = VALUES(removed_kwd), "
+ "q_1024_vec = VALUES(q_1024_vec), q_1024_vec_valid = VALUES(q_1024_vec_valid)"
+ )
+ expected_params = [["c2", "k1", "mother", 1, "N", zero_vector, False]]
+ assert errors == []
+ assert cursor.executed == [(expected_sql, expected_params)]
+ assert "c2" not in expected_sql
+ assert "k1" not in expected_sql
+ assert zero_vector not in expected_sql
+ assert conn.pool.conn.commits == 1
+ assert conn.pool.conn.rollbacks == 0
+
+
+def test_tc_wrt_202_insert_missing_vector_rejects_when_existing_dimension_is_unknown():
+ cursor = RecordingCursor()
+ conn = make_conn(cursor, vector_dimensions=[])
+ kb_id = "kb-wrt-202"
+ document = {"id": "c2", "kb_id": kb_id, "content_with_weight": "mother"}
+
+ errors = conn.insert([document], "ragflow_wrt_202", kb_id)
+
+ assert errors == ["c2"]
+ with pytest.raises(ValueError) as exc_info:
+ conn._normalize_chunk_row("ragflow_wrt_202", document, kb_id, [])
+ assert exc_info.type is ValueError
+ assert exc_info.value.args == ("cannot infer GaussDB vector dimension",)
+ assert str(exc_info.value) == "cannot infer GaussDB vector dimension"
+ assert cursor.executed == []
+ assert conn.pool.put_back == []
+
+
+def test_tc_wrt_113_insert_normalization_failure_returns_all_batch_ids_without_sql():
+ cursor = RecordingCursor()
+ conn = make_conn(cursor, vector_dimensions=[4])
+ vector = [0.1, 0.2, 0.3, 0.4]
+
+ errors = conn.insert(
+ [
+ {"id": "c1", "kb_id": "kb1", "doc_id": "d1", "q_4_vec": vector},
+ {"id": "c2", "kb_id": "kb2", "doc_id": "d2", "q_4_vec": vector},
+ ],
+ "ragflow_tenant",
+ "kb1",
+ )
+
+ assert errors == ["c1", "c2"]
+ assert cursor.executed == []
+ assert conn.pool.put_back == []
+
+
+def test_tc_wrt_106_insert_missing_vector_infers_dimension_from_same_batch_real_vector():
+ cursor = RecordingCursor()
+ conn = make_conn(cursor, vector_dimensions=[])
+
+ errors = conn.insert(
+ [
+ {"id": "c1", "kb_id": "kb1", "doc_id": "d1", "q_4_vec": [0.1, 0.2, 0.3, 0.4]},
+ {"id": "c2", "kb_id": "kb1", "doc_id": "d2"},
+ ],
+ "ragflow_tenant",
+ "kb1",
+ )
+
+ expected_sql = (
+ 'INSERT INTO "public"."ragflow_tenant" '
+ "(id, kb_id, doc_id, available_int, group_id, removed_kwd, q_4_vec, q_4_vec_valid) "
+ "VALUES (%s, %s, %s, %s, %s, %s, %s::floatvector(4), %s) "
+ "ON DUPLICATE KEY UPDATE doc_id = VALUES(doc_id), available_int = VALUES(available_int), "
+ "group_id = VALUES(group_id), removed_kwd = VALUES(removed_kwd), q_4_vec = VALUES(q_4_vec), "
+ "q_4_vec_valid = VALUES(q_4_vec_valid)"
+ )
+ expected_params = [
+ ["c1", "kb1", "d1", 1, "d1", "N", "[0.1,0.2,0.3,0.4]", True],
+ ["c2", "kb1", "d2", 1, "d2", "N", "[0,0,0,0]", False],
+ ]
+ assert errors == []
+ assert cursor.executed == [(expected_sql, expected_params)]
+ sentinels = ("c1", "c2", "kb1", "d1", "d2", "[0.1,0.2,0.3,0.4]", "[0,0,0,0]")
+ assert all(value not in expected_sql for value in sentinels)
+ assert conn.pool.conn.commits == 1
+ assert conn.pool.conn.rollbacks == 0
+
+
+def test_tc_wrt_106_insert_preserves_metadata_and_derives_group_and_title_fields():
+ cursor = RecordingCursor()
+ conn = make_conn(cursor, vector_dimensions=[1024])
+ kb_id = "kb-wrt-106"
+ vector_param = "[" + ",".join(["0.1"] * 1024) + "]"
+ fallback_vector_param = "[" + ",".join(["0.2"] * 1024) + "]"
+ metadata = {"_group_id": "g1", "_title": "DocName", "other": 1}
+ metadata_without_group = {"status": "open"}
+ metadata_param = json.dumps(metadata, ensure_ascii=False)
+ metadata_without_group_param = json.dumps(metadata_without_group, ensure_ascii=False)
+
+ errors = conn.insert(
+ [
+ {
+ "id": "c1",
+ "kb_id": kb_id,
+ "doc_id": "doc1",
+ "q_1024_vec": [0.1] * 1024,
+ "metadata": metadata,
+ },
+ {
+ "id": "c2",
+ "kb_id": kb_id,
+ "doc_id": "doc2",
+ "q_1024_vec": [0.2] * 1024,
+ "metadata": metadata_without_group,
+ },
+ ],
+ "ragflow_wrt_106",
+ kb_id,
+ )
+
+ expected_sql = (
+ 'INSERT INTO "public"."ragflow_wrt_106" '
+ "(id, kb_id, doc_id, docnm_kwd, available_int, metadata, group_id, removed_kwd, "
+ "q_1024_vec, q_1024_vec_valid) "
+ "VALUES (%s, %s, %s, %s, %s, %s::jsonb, %s, %s, %s::floatvector(1024), %s) "
+ "ON DUPLICATE KEY UPDATE doc_id = VALUES(doc_id), docnm_kwd = VALUES(docnm_kwd), "
+ "available_int = VALUES(available_int), metadata = VALUES(metadata), group_id = VALUES(group_id), "
+ "removed_kwd = VALUES(removed_kwd), q_1024_vec = VALUES(q_1024_vec), "
+ "q_1024_vec_valid = VALUES(q_1024_vec_valid)"
+ )
+ expected_params = [
+ ["c1", kb_id, "doc1", "DocName", 1, metadata_param, "g1", "N", vector_param, True],
+ ["c2", kb_id, "doc2", None, 1, metadata_without_group_param, "doc2", "N", fallback_vector_param, True],
+ ]
+ assert errors == []
+ assert cursor.executed == [(expected_sql, expected_params)]
+ assert "c1" not in expected_sql
+ assert kb_id not in expected_sql
+ assert "DocName" not in expected_sql
+ assert metadata_param not in expected_sql
+ assert metadata_without_group_param not in expected_sql
+ assert vector_param not in expected_sql
+ assert fallback_vector_param not in expected_sql
+ assert conn.pool.conn.commits == 1
+ assert conn.pool.conn.rollbacks == 0
+
+
+def test_tc_wrt_107_insert_merges_unknown_fields_with_existing_extra_values():
+ cursor = RecordingCursor()
+ conn = make_conn(cursor, vector_dimensions=[4])
+ kb_id = "kb-wrt-107"
+ vector = [0.1, 0.2, 0.3, 0.4]
+ vector_param = "[0.1,0.2,0.3,0.4]"
+
+ errors = conn.insert(
+ [
+ {
+ "id": "c1",
+ "kb_id": kb_id,
+ "q_4_vec": vector,
+ "extra": '{"existing": 1}',
+ "custom_field1": "value1",
+ "custom_field2": 2,
+ },
+ {
+ "id": "c2",
+ "kb_id": kb_id,
+ "q_4_vec": vector,
+ "extra": "{malformed",
+ "custom_field3": 3,
+ },
+ {
+ "id": "c3",
+ "kb_id": kb_id,
+ "q_4_vec": vector,
+ "extra": 1,
+ "custom_field4": 4,
+ },
+ ],
+ "ragflow_wrt_107",
+ kb_id,
+ )
+
+ expected_sql = (
+ 'INSERT INTO "public"."ragflow_wrt_107" '
+ "(id, kb_id, available_int, extra, removed_kwd, q_4_vec, q_4_vec_valid) "
+ "VALUES (%s, %s, %s, %s::jsonb, %s, %s::floatvector(4), %s) "
+ "ON DUPLICATE KEY UPDATE available_int = VALUES(available_int), extra = VALUES(extra), "
+ "removed_kwd = VALUES(removed_kwd), q_4_vec = VALUES(q_4_vec), "
+ "q_4_vec_valid = VALUES(q_4_vec_valid)"
+ )
+ expected_params = [
+ ["c1", kb_id, 1, '{"existing": 1, "custom_field1": "value1", "custom_field2": 2}', "N", vector_param, True],
+ ["c2", kb_id, 1, '{"custom_field3": 3}', "N", vector_param, True],
+ ["c3", kb_id, 1, '{"custom_field4": 4}', "N", vector_param, True],
+ ]
+ assert errors == []
+ assert cursor.executed == [(expected_sql, expected_params)]
+ sentinels = ("c1", "c2", "c3", kb_id, "value1", vector_param)
+ assert all(value not in expected_sql for value in sentinels)
+ assert conn.pool.conn.commits == 1
+ assert conn.pool.conn.rollbacks == 0
+
+
+def test_tc_wrt_301_insert_doc_meta_uses_parameterized_upsert():
+ cursor = RecordingCursor()
+ conn = make_conn(cursor)
+
+ errors = conn.insert(
+ [{"id": "d1", "kb_id": "kb-wrt-301", "meta_fields": {"status": "active"}}],
+ "ragflow_doc_meta_unit",
+ "kb-wrt-301",
+ )
+
+ expected_sql = 'INSERT INTO "public"."ragflow_doc_meta_unit" (id, kb_id, meta_fields) VALUES (%s, %s, %s::jsonb) ON DUPLICATE KEY UPDATE meta_fields = VALUES(meta_fields)'
+ expected_params = [["d1", "kb-wrt-301", '{"status": "active"}']]
+ assert errors == []
+ assert cursor.executed == [(expected_sql, expected_params)]
+ assert "d1" not in expected_sql
+ assert "kb-wrt-301" not in expected_sql
+ assert "active" not in expected_sql
+ assert conn.pool.conn.commits == 1
+ assert conn.pool.conn.rollbacks == 0
+
+
+def test_tc_wrt_101_insert_empty_input_returns_empty_without_database_call():
+ cursor = RecordingCursor()
+ conn = make_conn(cursor)
+
+ assert conn.insert([], "ragflow_tenant", "kb1") == []
+ assert cursor.executed == []
+ with pytest.raises(ValueError, match="rows are required"):
+ conn._build_chunk_upsert("ragflow_tenant", [])
+
+
+def test_tc_wrt_113_insert_returns_all_ids_and_original_error_when_write_fails(monkeypatch):
+ class FailingManyCursor(RecordingCursor):
+ def __init__(self, failure):
+ super().__init__()
+ self.failure = failure
+
+ def executemany(self, sql, params):
+ super().executemany(sql, params)
+ raise self.failure
+
+ failure = psycopg2.Error("Batch insert failed")
+ cursor = FailingManyCursor(failure)
+ conn = make_conn(cursor, vector_dimensions=[4])
+ logger = Mock()
+ gaussdb_module = sys.modules[GaussDBConnection.__module__]
+ monkeypatch.setattr(gaussdb_module, "logger", logger)
+ documents = [{"id": f"c{i}", "kb_id": "k1"} for i in range(1, 4)]
+
+ errors = conn.insert(documents, "ragflow_tenant", "k1")
+
+ expected_sql = (
+ 'INSERT INTO "public"."ragflow_tenant" '
+ "(id, kb_id, available_int, removed_kwd, q_4_vec, q_4_vec_valid) "
+ "VALUES (%s, %s, %s, %s, %s::floatvector(4), %s) "
+ "ON DUPLICATE KEY UPDATE available_int = VALUES(available_int), removed_kwd = VALUES(removed_kwd), "
+ "q_4_vec = VALUES(q_4_vec), q_4_vec_valid = VALUES(q_4_vec_valid)"
+ )
+ expected_params = [[f"c{i}", "k1", 1, "N", "[0,0,0,0]", False] for i in range(1, 4)]
+ assert errors == ["c1", "c2", "c3"]
+ assert len(errors) == 3
+ assert cursor.executed == [(expected_sql, expected_params)]
+ assert all(doc_id not in expected_sql for doc_id in errors)
+ assert "k1" not in expected_sql
+ assert type(failure) is psycopg2.Error
+ assert failure.args == ("Batch insert failed",)
+ assert str(failure) == "Batch insert failed"
+ assert conn.pool.conn.rollbacks == 1
+ assert conn.pool.conn.commits == 0
+ logger.error.assert_called_once_with(
+ "GaussDB insert failed for table=%s ids=%s error=%s",
+ "ragflow_tenant",
+ ["c1", "c2", "c3"],
+ failure,
+ )
+ assert logger.error.call_args.args[-1] is failure
+ assert cursor.closed is True
+ assert conn.pool.put_back == [conn.pool.conn]
+
+
+def test_tc_wrt_109_insert_rejects_missing_chunk_id_without_sql():
+ cursor = RecordingCursor()
+ conn = make_conn(cursor)
+
+ result = conn.insert([{"kb_id": "k1", "q_4_vec": [0.1, 0.2, 0.3, 0.4]}], "ragflow_tenant", "k1")
+
+ assert result == ["chunk id is required"]
+ assert "chunk id is required" in str(result[0])
+ assert cursor.executed == []
+
+
+def test_tc_wrt_110_insert_rejects_missing_kb_id_without_sql():
+ cursor = RecordingCursor()
+ conn = make_conn(cursor)
+
+ result = conn.insert([{"id": "c1", "q_4_vec": [0.1, 0.2, 0.3, 0.4]}], "ragflow_tenant", None)
+
+ assert result == ["c1"]
+ assert len(result) == 1
+ assert cursor.executed == []
+
+
+def test_tc_wrt_111_insert_rejects_kb_id_mismatch_without_sql():
+ cursor = RecordingCursor()
+ conn = make_conn(cursor)
+
+ result = conn.insert([{"id": "c1", "kb_id": "k1", "q_4_vec": [0.1, 0.2, 0.3, 0.4]}], "ragflow_tenant", "k2")
+
+ assert result == ["c1"]
+ assert len(result) == 1
+ assert cursor.executed == []
+
+
+def test_tc_wrt_304_insert_doc_meta_rejects_missing_doc_id_or_kb_id_without_sql():
+ cursor = RecordingCursor()
+ conn = make_conn(cursor)
+
+ missing_doc_id = conn.insert([{"kb_id": "kb1", "meta_fields": {}}], "ragflow_doc_meta_tenant", "kb1")
+ missing_kb_id = conn.insert([{"id": "doc1", "meta_fields": {}}], "ragflow_doc_meta_tenant", None)
+
+ assert missing_doc_id == ["doc metadata id and kb_id are required"]
+ assert missing_kb_id == ["doc1"]
+ with pytest.raises(ValueError, match="^doc metadata id and kb_id are required$"):
+ conn._build_doc_meta_upsert("ragflow_doc_meta_tenant", [{"kb_id": "kb1"}], "kb1")
+ with pytest.raises(ValueError, match="^doc metadata id and kb_id are required$"):
+ conn._build_doc_meta_upsert("ragflow_doc_meta_tenant", [{"id": "doc1"}], None)
+ assert cursor.executed == []
+
+
+def test_tc_wrt_305_insert_doc_meta_rejects_kb_id_mismatch_without_sql():
+ cursor = RecordingCursor()
+ conn = make_conn(cursor)
+
+ result = conn.insert([{"id": "doc1", "kb_id": "kb2", "meta_fields": {}}], "ragflow_doc_meta_tenant", "kb1")
+
+ assert result == ["doc1"]
+ with pytest.raises(ValueError, match="^kb_id kb2 does not match dataset_id kb1$"):
+ conn._build_doc_meta_upsert(
+ "ragflow_doc_meta_tenant",
+ [{"id": "doc1", "kb_id": "kb2", "meta_fields": {}}],
+ "kb1",
+ )
+ assert cursor.executed == []
+
+
+ ("condition", "predicate", "expected_params"),
+ [
+ ({"id": "c1"}, "id = %s", ["c1", "kb1"]),
+ ({"id": ["c1", "c2"]}, "id IN (%s, %s)", ["c1", "c2", "kb1"]),
+ ],
+)
+def test_tc_wrt_602_delete_chunk_scopes_by_id_and_kb_id(condition, predicate, expected_params):
+ cursor = RecordingCursor()
+ cursor.rowcount = 1
+ conn = make_conn(cursor)
+
+ deleted = conn.delete(condition, "ragflow_tenant", "kb1")
+
+ sql, params = cursor.executed[-1]
+ assert deleted == 1
+ assert 'DELETE FROM "public"."ragflow_tenant"' in sql
+ assert predicate in sql
+ assert "AND kb_id = %s" in sql
+ assert params == expected_params
+
+
+def test_tc_wrt_309_delete_doc_meta_scopes_by_id_and_kb_id():
+ cursor = RecordingCursor()
+ cursor.rowcount = 1
+ conn = make_conn(cursor)
+
+ deleted = conn.delete({"id": "doc1"}, "ragflow_doc_meta_tenant", "kb1")
+
+ assert deleted == 1
+ assert cursor.executed == [('DELETE FROM "public"."ragflow_doc_meta_tenant" WHERE id = %s AND kb_id = %s', ["doc1", "kb1"])]
+ sql = cursor.executed[0][0]
+ assert "doc1" not in sql
+ assert "kb1" not in sql
+ assert "DROP TABLE" not in sql
+ assert conn.pool.conn.commits == 1
+ assert conn.pool.conn.rollbacks == 0
+
+
+def test_tc_wrt_601_delete_rejects_unscoped_condition():
+ cursor = RecordingCursor()
+ conn = make_conn(cursor)
+
+ deleted = conn.delete({}, "ragflow_tenant", "kb1")
+
+ assert deleted == 0
+ assert cursor.executed == []
+
+
+def test_tc_wrt_613_delete_without_kb_scope_returns_zero_without_database_call():
+ cursor = RecordingCursor()
+ conn = make_conn(cursor)
+
+ assert conn.delete({"id": "c1"}, "ragflow_tenant", None) == 0
+ assert cursor.executed == []
+
+
+def test_tc_wrt_614_delete_allows_kb_scope_from_condition_without_dataset_id():
+ cursor = RecordingCursor()
+ cursor.rowcount = 1
+ conn = make_conn(cursor)
+
+ deleted = conn.delete({"id": "c1", "kb_id": "kb1"}, "ragflow_tenant", None)
+
+ assert deleted == 1
+ assert cursor.executed == [('DELETE FROM "public"."ragflow_tenant" WHERE id = %s AND kb_id = %s', ["c1", "kb1"])]
+ sql = cursor.executed[0][0]
+ assert "c1" not in sql
+ assert "kb1" not in sql
+ assert "DROP TABLE" not in sql
+
+
+def test_tc_wrt_615_repeated_missing_chunk_delete_emits_same_scoped_sql():
+ cursor = RecordingCursor()
+ cursor.rowcount = 0
+ conn = make_conn(cursor)
+
+ first = conn.delete({"id": "c1"}, "ragflow_tenant", "kb1")
+ second = conn.delete({"id": "c1"}, "ragflow_tenant", "kb1")
+
+ assert first == second == 0
+ assert len(cursor.executed) == 2
+ assert cursor.executed[0] == cursor.executed[1]
+
+
+def test_tc_wrt_503_update_builds_parameterized_set_clause_and_scopes_kb_id():
+ cursor = RecordingCursor()
+ cursor.rowcount = 1
+ conn = make_conn(cursor)
+
+ updated = conn.update(
+ {"id": "c1", "kb_id": "k1"},
+ {"available_int": 0},
+ "ragflow_tenant",
+ "k1",
+ )
+
+ assert updated is True
+ assert cursor.executed == [('UPDATE "public"."ragflow_tenant" SET available_int = %s WHERE id = %s AND kb_id = %s', [0, "c1", "k1"])]
+ sql = cursor.executed[0][0]
+ assert "c1" not in sql
+ assert "k1" not in sql
+ assert all(keyword not in sql for keyword in ("DELETE ", "DROP ", "CREATE ", "ALTER "))
+ assert conn.pool.conn.commits == 1
+ assert conn.pool.conn.rollbacks == 0
+
+
+def test_tc_wrt_502_update_without_kb_scope_returns_false_without_database_call():
+ cursor = RecordingCursor()
+ conn = make_conn(cursor)
+
+ updated = conn.update({"id": "c1"}, {"pagerank_fea": 9}, "ragflow_tenant", None)
+
+ assert updated is False
+ assert cursor.executed == []
+
+
+ ("new_value", "message"),
+ [
+ ({"id": "new-value"}, "key column cannot be updated: id"),
+ ({"kb_id": "new-value"}, "key column cannot be updated: kb_id"),
+ ({"remove": "id"}, "key column cannot be updated: id"),
+ ({"remove": "kb_id"}, "key column cannot be updated: kb_id"),
+ ],
+)
+def test_tc_wrt_508_update_rejects_key_column_changes_without_database_call(new_value, message):
+ cursor = RecordingCursor()
+ conn = make_conn(cursor)
+
+ updated = conn.update({"id": "c1"}, new_value, "ragflow_tenant", "kb1")
+
+ assert updated is False
+ with pytest.raises(ValueError, match=f"^{message}$"):
+ conn._build_set_clause(new_value, is_meta=False, condition={"id": "c1"})
+ assert cursor.executed == []
+
+
+def test_tc_wrt_508_update_ignores_redundant_matching_id():
+ cursor = RecordingCursor()
+ conn = make_conn(cursor)
+
+ updated = conn.update(
+ {"id": "c1"},
+ {"id": "c1", "content_with_weight": "updated", "q_3_vec": [0.1, 0.2, 0.3]},
+ "ragflow_tenant",
+ "kb1",
+ )
+
+ assert updated is True
+ assert cursor.executed == [
+ (
+ 'UPDATE "public"."ragflow_tenant" SET content_with_weight = %s, q_3_vec = %s::floatvector(3), q_3_vec_valid = TRUE WHERE id = %s AND kb_id = %s',
+ ["updated", "[0.1,0.2,0.3]", "c1", "kb1"],
+ )
+ ]
+
+
+def test_tc_wrt_509_update_merges_dynamic_column_into_extra_jsonb():
+ cursor = RecordingCursor()
+ cursor.rows = [("c1", "k1", '{"existing": 1}')]
+ conn = make_conn(cursor)
+
+ result = conn.update({"id": "c1", "kb_id": "k1"}, {"unknown_col": 1}, "ragflow_tenant", "k1")
+
+ assert result is True
+ select_sql, select_params = cursor.executed[-2]
+ update_sql, update_params = cursor.executed[-1]
+ assert "SELECT id, kb_id, extra" in select_sql
+ assert "FOR UPDATE" in select_sql
+ assert select_params == ["c1", "k1"]
+ assert "extra = %s::jsonb" in update_sql
+ assert json.loads(update_params[0]) == {"existing": 1, "unknown_col": 1}
+ assert update_params[1:] == ["k1", "c1"]
+
+
+def test_tc_wrt_509_update_rejects_unsafe_dynamic_column_without_sql():
+ cursor = RecordingCursor()
+ conn = make_conn(cursor)
+
+ result = conn.update(
+ {"id": "c1", "kb_id": "k1"},
+ {"bad'; DROP TABLE user; --": 1},
+ "ragflow_tenant",
+ "k1",
+ )
+
+ assert result is False
+ assert cursor.executed == []
+
+
+def test_tc_wrt_507_update_adds_jsonb_multivalue_with_scoped_parameters():
+ cursor = RecordingCursor()
+ cursor.rowcount = 1
+ conn = make_conn(cursor)
+
+ updated = conn.update({"id": "c1", "kb_id": "k1"}, {"add": {"tag_kwd": "t3"}}, "ragflow_tenant", "k1")
+
+ assert updated is True
+ assert cursor.executed == [
+ (
+ 'UPDATE "public"."ragflow_tenant" '
+ "SET tag_kwd = (SELECT CASE WHEN candidate @> %s::jsonb THEN candidate "
+ "ELSE json_array_append(candidate, '$', %s::jsonb)::jsonb END "
+ "FROM (SELECT COALESCE(tag_kwd, '[]'::jsonb) AS candidate) AS jsonb_update) "
+ "WHERE id = %s AND kb_id = %s",
+ ['["t3"]', '"t3"', "c1", "k1"],
+ )
+ ]
+ sql = cursor.executed[0][0]
+ assert "t3" not in sql
+ assert "c1" not in sql
+ assert "k1" not in sql
+
+
+def test_tc_wrt_506_update_removes_jsonb_value_with_scoped_parameters():
+ cursor = RecordingCursor()
+ cursor.rowcount = 1
+ conn = make_conn(cursor)
+
+ updated = conn.update({"id": "c1", "kb_id": "k1"}, {"remove": {"tag_kwd": "t1"}}, "ragflow_tenant", "k1")
+
+ assert updated is True
+ assert cursor.executed == [
+ (
+ 'UPDATE "public"."ragflow_tenant" '
+ "SET tag_kwd = COALESCE(json_remove(tag_kwd, json_unquote(json_search(tag_kwd, 'one', "
+ "jsonb_array_element_text(%s::jsonb, 0))))::jsonb, tag_kwd) WHERE id = %s AND kb_id = %s",
+ ['["t1"]', "c1", "k1"],
+ )
+ ]
+ sql = cursor.executed[0][0]
+ assert "t1" not in sql
+ assert "c1" not in sql
+ assert "k1" not in sql
+
+
+def test_tc_wrt_506_json_search_treats_wildcards_as_literals():
+ cursor = RecordingCursor()
+ cursor.rowcount = 1
+ conn = make_conn(cursor)
+
+ updated = conn.update(
+ {"id": "c1", "kb_id": "k1"},
+ {"remove": {"tag_kwd": "a%_\\b"}},
+ "ragflow_tenant",
+ "k1",
+ )
+
+ assert updated is True
+ assert json.loads(cursor.executed[0][1][0]) == ["a\\%\\_\\\\b"]
+ assert cursor.executed[0][1][1:] == ["c1", "k1"]
+
+
+def test_tc_wrt_507_same_column_remove_then_add_uses_one_assignment():
+ cursor = RecordingCursor()
+ cursor.rowcount = 1
+ conn = make_conn(cursor)
+
+ updated = conn.update(
+ {"id": "c1", "kb_id": "k1"},
+ {"add": {"tag_kwd": "new"}, "remove": {"tag_kwd": "old"}},
+ "ragflow_tenant",
+ "k1",
+ )
+
+ assert updated is True
+ assert cursor.executed == [
+ (
+ 'UPDATE "public"."ragflow_tenant" '
+ "SET tag_kwd = (SELECT CASE WHEN candidate @> %s::jsonb THEN candidate "
+ "ELSE json_array_append(candidate, '$', %s::jsonb)::jsonb END "
+ "FROM (SELECT COALESCE(json_remove(tag_kwd, json_unquote(json_search(tag_kwd, 'one', "
+ "jsonb_array_element_text(%s::jsonb, 0))))::jsonb, tag_kwd, '[]'::jsonb) AS candidate) AS jsonb_update) "
+ "WHERE id = %s AND kb_id = %s",
+ ['["new"]', '"new"', '["old"]', "c1", "k1"],
+ )
+ ]
+ assert cursor.executed[0][0].count("SET tag_kwd =") == 1
+
+
+def test_tc_wrt_505_update_remove_column_sets_null_with_scoped_parameters():
+ cursor = RecordingCursor()
+ cursor.rowcount = 1
+ conn = make_conn(cursor)
+
+ updated = conn.update({"id": "c1", "kb_id": "k1"}, {"remove": "important_kwd"}, "ragflow_tenant", "k1")
+
+ assert updated is True
+ assert cursor.executed == [('UPDATE "public"."ragflow_tenant" SET important_kwd = NULL WHERE id = %s AND kb_id = %s', ["c1", "k1"])]
+ sql = cursor.executed[0][0]
+ assert "c1" not in sql
+ assert "k1" not in sql
+
+
+ ("new_value", "expected_error"),
+ [
+ ({"remove": {"chunk_data": "x"}}, "unsupported JSONB remove target: chunk_data"),
+ ({"add": {"chunk_data": "x"}}, "unsupported JSONB add target: chunk_data"),
+ ({"remove": 1}, "unsupported remove target: 1"),
+ ({"remove": "bad_column"}, "unsupported remove target: bad_column"),
+ ({"add": "bad"}, "unsupported add target: bad"),
+ ],
+)
+def test_tc_wrt_510_update_rejects_invalid_remove_add_operations_without_sql(new_value, expected_error):
+ cursor = RecordingCursor()
+ conn = make_conn(cursor)
+
+ with pytest.raises(ValueError) as exc_info:
+ conn._build_set_clause(new_value, is_meta=False, condition={"id": "c1", "kb_id": "k1"})
+
+ assert str(exc_info.value) == expected_error
+ result = conn.update({"id": "c1", "kb_id": "k1"}, new_value, "ragflow_tenant", "k1")
+
+ assert result is False
+ assert cursor.executed == []
+
+
+def test_tc_wrt_611_delete_jsonb_multivalue_conditions_use_contains_predicates():
+ cursor = RecordingCursor()
+ cursor.rowcount = 2
+ conn = make_conn(cursor)
+
+ deleted = conn.delete({"tag_kwd": ["t1"]}, "ragflow_tenant", "k1")
+
+ assert deleted == 2
+ assert cursor.executed == [('DELETE FROM "public"."ragflow_tenant" WHERE (tag_kwd @> %s::jsonb) AND kb_id = %s', ['["t1"]', "k1"])]
+ sql = cursor.executed[0][0]
+ assert "t1" not in sql
+ assert "k1" not in sql
+ assert "DROP TABLE" not in sql
+
+
+def test_tc_wrt_607_delete_must_not_exists_uses_is_null_with_kb_scope():
+ cursor = RecordingCursor()
+ cursor.rowcount = 1
+ conn = make_conn(cursor)
+
+ deleted = conn.delete({"must_not": {"exists": "tag_kwd"}}, "ragflow_tenant", "k1")
+
+ assert deleted == 1
+ assert cursor.executed == [('DELETE FROM "public"."ragflow_tenant" WHERE tag_kwd IS NULL AND kb_id = %s', ["k1"])]
+
+
+def test_tc_wrt_616_delete_compile_kwd_uses_extra_jsonb_mapping():
+ cursor = RecordingCursor()
+ cursor.rowcount = 2
+ conn = make_conn(cursor)
+
+ deleted = conn.delete(
+ {"doc_id": "d1", "id": ["c1", "c2"], "must_not": {"exists": "compile_kwd"}},
+ "ragflow_tenant",
+ "k1",
+ )
+
+ assert deleted == 2
+ assert cursor.executed == [
+ (
+ 'DELETE FROM "public"."ragflow_tenant" WHERE doc_id = %s AND id IN (%s, %s) AND (extra #>> \'{compile_kwd}\') IS NULL AND kb_id = %s',
+ ["d1", "c1", "c2", "k1"],
+ )
+ ]
+
+
+ ("condition", "index_name", "is_meta", "expected_error"),
+ [
+ ({"id": []}, "ragflow_tenant", False, "empty list condition for id"),
+ ({"tag_kwd": []}, "ragflow_tenant", False, "empty list condition for tag_kwd"),
+ ({"unknown": "x"}, "ragflow_doc_meta_tenant", True, "unsupported metadata filter column: unknown"),
+ ],
+ ids=["empty-scalar-list", "empty-jsonb-list", "metadata-column"],
+)
+def test_tc_wrt_609_delete_rejects_empty_or_unsupported_conditions_without_sql(
+ condition,
+ index_name,
+ is_meta,
+ expected_error,
+):
+ cursor = RecordingCursor()
+ conn = make_conn(cursor)
+
+ result = conn.delete(condition, index_name, "k1")
+
+ assert result == 0
+ with pytest.raises(ValueError) as exc_info:
+ conn._build_where_clause(condition, "k1", is_meta=is_meta)
+ assert str(exc_info.value) == expected_error
+ assert cursor.executed == []
+
+
+def test_tc_wrt_616_delete_filters_dynamic_fields_through_extra_jsonb():
+ cursor = RecordingCursor()
+ cursor.rowcount = 1
+ conn = make_conn(cursor)
+
+ deleted = conn.delete(
+ {"compile_type": ["artifact_page"]},
+ "ragflow_tenant",
+ "kb1",
+ )
+
+ assert deleted == 1
+ sql, params = cursor.executed[-1]
+ assert "(extra -> 'compile_type') = %s::jsonb" in sql
+ assert params == ['"artifact_page"', '["artifact_page"]', "kb1"]
+
+
+def test_tc_wrt_610_delete_rejects_kb_id_mismatch_without_sql():
+ cursor = RecordingCursor()
+ conn = make_conn(cursor)
+
+ result = conn.delete({"kb_id": "k2"}, "ragflow_tenant", "k1")
+
+ assert result == 0
+ with pytest.raises(ValueError, match="condition kb_id k2 does not match dataset_id k1"):
+ conn._build_where_clause({"kb_id": "k2"}, "k1", is_meta=False)
+ assert cursor.executed == []
+
+
+def test_tc_ret_511_point_get_scopes_by_id_and_kb_id_and_decodes_jsonb():
+ cursor = RecordingCursor()
+ cursor.description = [("id",), ("kb_id",), ("position_int",), ("q_4_vec",), ("q_4_vec_valid",)]
+ cursor.rows = [("c1", "kb1", "[[1, 2, 3, 4]]", "[0,0,0,0]", False)]
+ conn = make_conn(cursor)
+
+ row = conn.get("c1", "ragflow_tenant", ["kb1"])
+
+ sql, params = cursor.executed[-1]
+ assert sql == 'SELECT * FROM "public"."ragflow_tenant" WHERE id = %s AND kb_id IN (%s) LIMIT 1'
+ assert params == ["c1", "kb1"]
+ assert row == {"id": "c1", "kb_id": "kb1", "position_int": [[1, 2, 3, 4]], "q_4_vec_valid": False}
+
+ cursor.rows = [("c1", "kb1", "[[1, 2, 3, 4]]", "[0,0,0,0]", False), ("c2", "kb2", "[]", "[0,0,0,0]", False)]
+ rows = conn.get("c1", "ragflow_tenant", ["kb1", "kb2"])
+ assert rows["id"] == "c1"
+ assert "ORDER BY kb_id ASC LIMIT 2" in cursor.executed[-1][0]
+
+
+def test_tc_ret_513_get_promotes_dynamic_fields_from_extra_jsonb():
+ cursor = RecordingCursor()
+ cursor.description = [("id",), ("kb_id",), ("extra",)]
+ cursor.rows = [("c1", "kb1", '{"compile_type": "artifact_page"}')]
+ conn = make_conn(cursor)
+
+ row = conn.get("c1", "ragflow_tenant", ["kb1"])
+
+ assert row["compile_type"] == "artifact_page"
+ assert row["extra"] == {"compile_type": "artifact_page"}
+
+
+def test_tc_wrt_410_get_doc_meta_allows_empty_kb_scope_and_decodes_jsonb():
+ cursor = RecordingCursor()
+ cursor.description = [("id",), ("kb_id",), ("meta_fields",)]
+ cursor.rows = [("doc1", "kb1", '{"author": "Alice"}')]
+ conn = make_conn(cursor)
+
+ row = conn.get("doc1", "ragflow_doc_meta_tenant", [""])
+
+ sql, params = cursor.executed[-1]
+ assert "WHERE id = %s" in sql
+ assert "kb_id IN" not in sql
+ assert params == ["doc1"]
+ assert row == {"id": "doc1", "kb_id": "kb1", "meta_fields": {"author": "Alice"}}
+
+
+def test_tc_wrt_401_get_empty_chunk_id_returns_none_without_sql():
+ cursor = RecordingCursor()
+ conn = make_conn(cursor)
+
+ assert conn.get("", "ragflow_tenant", ["kb1"]) is None
+ assert cursor.executed == []
+
+
+def test_tc_wrt_409_get_missing_chunk_returns_none_after_scoped_lookup():
+ cursor = SequencedCursor(
+ [
+ ([(1,)], [("?column?",)]),
+ ([], [("id",), ("kb_id",)]),
+ ]
+ )
+ conn = make_conn(cursor)
+
+ result = conn.get("c1", "ragflow_tenant", ["kb1"])
+
+ assert result is None
+ assert "information_schema.tables" in cursor.executed[0][0]
+ assert cursor.executed[0][1] == ["public", "ragflow_tenant"]
+ assert "WHERE id = %s AND kb_id IN (%s) LIMIT 1" in cursor.executed[1][0]
+ assert cursor.executed[1][1] == ["c1", "kb1"]
+
+
+def test_tc_wrt_402_get_non_meta_without_kb_ids_returns_none():
+ cursor = RecordingCursor()
+ conn = make_conn(cursor)
+
+ assert conn.get("c1", "ragflow_tenant", []) is None
+ assert cursor.executed == []
+
+
+def test_tc_wrt_403_get_returns_none_when_table_does_not_exist():
+ cursor = SequencedCursor(
+ [
+ ([], None),
+ ]
+ )
+ conn = make_conn(cursor)
+
+ assert conn.get("c1", "ragflow_tenant", ["kb1"]) is None
+ assert len(cursor.executed) == 1
+ assert "information_schema.tables" in cursor.executed[0][0]
+
+
+def test_tc_ret_003_search_doc_meta_table_returns_search_result():
+ cursor = RecordingCursor()
+ cursor.description = [("id",), ("kb_id",), ("meta_fields",), ("__total",)]
+ cursor.rows = [("doc1", "kb1", '{"author": "Alice"}', 1001)]
+ conn = make_conn(cursor)
+
+ result = conn.search(
+ select_fields=["id", "kb_id", "meta_fields", "id"],
+ highlight_fields=[],
+ condition={"id": "doc1"},
+ match_expressions=[],
+ order_by=OrderByExpr().desc("id"),
+ offset=5,
+ limit=10,
+ index_names="ragflow_doc_meta_tenant",
+ knowledgebase_ids=["kb1", "kb2", "kb1"],
+ )
+
+ sql, params = cursor.executed[-1]
+ assert result == SearchResult(total=1001, chunks=[{"id": "doc1", "kb_id": "kb1", "meta_fields": {"author": "Alice"}}])
+ assert 'SELECT id, kb_id, meta_fields, COUNT(*) OVER() AS __total FROM "public"."ragflow_doc_meta_tenant"' in sql
+ assert "id = %s" in sql
+ assert "kb_id IN (%s, %s)" in sql
+ assert "ORDER BY id DESC" in sql
+ assert "LIMIT %s OFFSET %s" in sql
+ assert "to_tsvector" not in sql.lower()
+ assert "plainto_tsquery" not in sql.lower()
+ assert params == ["doc1", "kb1", "kb2", 10, 5]
+
+
+def test_tc_ret_003_search_doc_meta_empty_page_falls_back_to_count():
+ cursor = RecordingCursor()
+ conn = make_conn(cursor)
+ conn._fetch_all_with_description = lambda _sql, _params: ([], [("id",), ("kb_id",), ("meta_fields",), ("__total",)])
+ count_queries = []
+
+ def fake_fetch_one(sql, params):
+ count_queries.append((sql, params))
+ return (1001,)
+
+ conn._fetch_one = fake_fetch_one
+
+ result = conn.search(
+ select_fields=["*"],
+ highlight_fields=[],
+ condition={"id": "doc-missing"},
+ match_expressions=[],
+ order_by=OrderByExpr(),
+ offset=2000,
+ limit=1000,
+ index_names="ragflow_doc_meta_tenant",
+ knowledgebase_ids=["kb1"],
+ )
+
+ sql, params = count_queries[0]
+ assert result == SearchResult(total=1001, chunks=[])
+ assert 'SELECT COUNT(*) FROM "public"."ragflow_doc_meta_tenant"' in sql
+ assert "id = %s" in sql
+ assert "kb_id IN (%s)" in sql
+ assert "to_tsvector" not in sql.lower()
+ assert "plainto_tsquery" not in sql.lower()
+ assert params == ["doc-missing", "kb1"]
+
+
+def test_tc_mgf_713_fetch_metadata_doc_ids_builds_scoped_jsonb_query():
+ cursor = RecordingCursor()
+ cursor.rows = [("doc2",), ("doc1",)]
+ conn = make_conn(cursor)
+
+ doc_ids = conn.fetch_metadata_doc_ids(
+ "ragflow_doc_meta_tenant",
+ ["kb1", "kb2"],
+ "lower(meta_fields #>> '{author}') = %s",
+ ["alice"],
+ 25,
+ )
+
+ sql, params = cursor.executed[-1]
+ assert doc_ids == ["doc2", "doc1"]
+ assert 'SELECT id FROM "public"."ragflow_doc_meta_tenant"' in sql
+ assert "kb_id IN (%s, %s)" in sql
+ assert "(lower(meta_fields #>> '{author}') = %s)" in sql
+ assert "ORDER BY id" in sql
+ assert "LIMIT %s" in sql
+ assert params == ["kb1", "kb2", "alice", 25]
+ assert cursor.closed is True
+ assert conn.pool.put_back == [conn.pool.conn]
+ assert conn.pool.conn.commits == 0
+ assert conn.pool.conn.rollbacks == 0
+
+
+def test_tc_wrt_202_get_vector_dimensions_reads_q_vector_columns():
+ cursor = RecordingCursor()
+ cursor.rows = [("q_3072_vec",), ("q_1024_vec",), ("q_3072_vec_valid",)]
+ conn = make_conn(cursor)
+
+ dimensions = conn.get_vector_dimensions("ragflow_tenant")
+
+ sql, params = cursor.executed[0]
+ query = parse_one(sql, read="postgres")
+ table = query.find(exp.Table)
+ predicates = list(query.find_all(exp.EQ))
+ assert dimensions == [1024, 3072]
+ assert isinstance(query, exp.Select)
+ assert [expression.name for expression in query.expressions] == ["column_name"]
+ assert (table.db, table.name) == ("information_schema", "columns")
+ assert {predicate.this.name for predicate in predicates} == {"table_schema", "table_name"}
+ assert all(isinstance(predicate.expression, exp.Placeholder) for predicate in predicates)
+ assert params == ["public", "ragflow_tenant"]
+
+
+def test_tc_wrt_705_index_exist_checks_information_schema_tables():
+ cursor = RecordingCursor()
+ cursor.rows = [(1,)]
+ conn = make_conn(cursor)
+
+ result = conn.index_exist("ragflow_tenant")
+
+ sql, params = cursor.executed[0]
+ query = parse_one(sql, read="postgres")
+ table = query.find(exp.Table)
+ predicates = list(query.find_all(exp.EQ))
+ assert result is True
+ assert isinstance(query, exp.Select)
+ assert query.expressions[0].this == "1"
+ assert (table.db, table.name) == ("information_schema", "tables")
+ assert {predicate.this.name for predicate in predicates} == {"table_schema", "table_name"}
+ assert all(isinstance(predicate.expression, exp.Placeholder) for predicate in predicates)
+ assert query.args["limit"].expression.this == "1"
+ assert params == ["public", "ragflow_tenant"]
+ assert conn.pool.conn.commits == 0
+ assert conn.pool.conn.rollbacks == 0
+ assert cursor.closed is True
+ assert conn.pool.put_back == [conn.pool.conn]
+
+
+def test_tc_wrt_708_index_exist_rejects_invalid_index_name_without_sql():
+ cursor = RecordingCursor()
+ conn = make_conn(cursor)
+
+ with pytest.raises(InvalidGaussDBObjectName, match="^1abc$"):
+ conn.index_exist("1abc")
+
+ assert cursor.executed == []
+
+
+def test_tc_wrt_701_delete_idx_with_dataset_id_deletes_only_that_kb():
+ cursor = RecordingCursor()
+ conn = make_conn(cursor)
+
+ result = conn.delete_idx("ragflow_tenant", "k1")
+
+ assert result is None
+ assert cursor.executed == [('DELETE FROM "public"."ragflow_tenant" WHERE kb_id = %s', ["k1"])]
+ sql = cursor.executed[0][0]
+ assert "k1" not in sql
+ assert "DROP TABLE" not in sql
+ assert conn.pool.conn.commits == 1
+ assert conn.pool.conn.rollbacks == 0
+ assert cursor.closed is True
+ assert conn.pool.put_back == [conn.pool.conn]
+
+
+def test_tc_wrt_702_delete_idx_without_dataset_id_drops_table():
+ cursor = RecordingCursor()
+ conn = make_conn(cursor)
+
+ result = conn.delete_idx("ragflow_tenant", None)
+
+ assert result is None
+ assert cursor.executed == [('DROP TABLE IF EXISTS "public"."ragflow_tenant"', [])]
+ assert "DELETE FROM" not in cursor.executed[0][0]
+ assert conn.pool.conn.commits == 1
+ assert conn.pool.conn.rollbacks == 0
+ assert cursor.closed is True
+ assert conn.pool.put_back == [conn.pool.conn]
+
+
+def test_tc_wrt_709_delete_idx_missing_table_only_ignores_undefined_table():
+ class CodedDatabaseError(psycopg2.Error):
+ def __init__(self, code):
+ super().__init__("database error")
+ self._code = code
+
+ @property
+ def pgcode(self):
+ return self._code
+
+ class FailingCursor(RecordingCursor):
+ def __init__(self, failure):
+ super().__init__()
+ self.failure = failure
+
+ def execute(self, sql, params=None):
+ super().execute(sql, params)
+ raise self.failure
+
+ missing_cursor = FailingCursor(CodedDatabaseError(errorcodes.UNDEFINED_TABLE))
+ missing_conn = make_conn(missing_cursor)
+
+ assert missing_conn.delete_idx("ragflow_missing", "k1") is None
+ assert missing_cursor.executed == [('DELETE FROM "public"."ragflow_missing" WHERE kb_id = %s', ["k1"])]
+ assert missing_conn.pool.conn.rollbacks == 1
+ assert missing_conn.pool.conn.commits == 0
+ assert missing_cursor.closed is True
+ assert missing_conn.pool.put_back == [missing_conn.pool.conn]
+
+ permission_cursor = FailingCursor(CodedDatabaseError(errorcodes.INSUFFICIENT_PRIVILEGE))
+ permission_conn = make_conn(permission_cursor)
+ with pytest.raises(CodedDatabaseError) as permission_error:
+ permission_conn.delete_idx("ragflow_forbidden", "k1")
+ assert permission_error.value.pgcode == errorcodes.INSUFFICIENT_PRIVILEGE
+ assert permission_conn.pool.conn.rollbacks == 1
+ assert permission_conn.pool.conn.commits == 0
+ assert permission_cursor.closed is True
+ assert permission_conn.pool.put_back == [permission_conn.pool.conn]
+
+ runtime_cursor = FailingCursor(RuntimeError("connection failure"))
+ runtime_conn = make_conn(runtime_cursor)
+ with pytest.raises(RuntimeError, match="connection failure"):
+ runtime_conn.delete_idx("ragflow_unavailable", "k1")
+ assert runtime_conn.pool.conn.rollbacks == 1
+ assert runtime_conn.pool.conn.commits == 0
+ assert runtime_cursor.closed is True
+ assert runtime_conn.pool.put_back == [runtime_conn.pool.conn]
+
+
+def test_tc_wrt_303_create_doc_meta_idx_rejects_non_metadata_table_without_sql():
+ cursor = RecordingCursor()
+ conn = make_conn(cursor)
+
+ with pytest.raises(ValueError, match="^invalid GaussDB document metadata table name: ragflow_tenant$"):
+ conn.create_doc_meta_idx("ragflow_tenant")
+
+ assert cursor.executed == []
+
+
+def test_tc_wrt_912_create_doc_meta_idx_executes_table_and_kb_index_ddls():
+ cursor = RecordingCursor()
+ conn = make_conn(cursor)
+
+ result = conn.create_doc_meta_idx("ragflow_doc_meta_tenant")
+
+ assert result is True
+ assert cursor.executed == CANONICAL_CREATE_DOC_META_EXECUTIONS
+ assert conn.pool.conn.commits == 1
+ assert conn.pool.conn.rollbacks == 0
+ assert cursor.closed is True
+ assert conn.pool.put_back == [conn.pool.conn]
+
+
+def test_tc_wrt_011_create_idx_uses_schema_lock_before_ddl():
+ cursor = RecordingCursor()
+ conn = make_conn(cursor)
+
+ conn.create_idx("ragflow_tenant", "kb1", 4)
+
+ statements = [sql for sql, _params in cursor.executed]
+ assert statements[0] == "SELECT pg_advisory_xact_lock(hashtext(%s))"
+ assert cursor.executed[0][1] == ["create_idx:public:ragflow_tenant"]
+ work_mem_index = statements.index("SET LOCAL maintenance_work_mem = '1GB'")
+ diskann_index = next(i for i, sql in enumerate(statements) if "USING gsdiskann" in sql)
+ assert work_mem_index < diskann_index
+
+
+def test_tc_wrt_010_create_idx_keeps_functional_table_when_diskann_ddl_fails():
+ class FailingDiskANNCursor(RecordingCursor):
+ def execute(self, sql, params=None):
+ super().execute(sql, params)
+ if "USING gsdiskann" in sql:
+ raise psycopg2.OperationalError("DiskANN index creation failed")
+
+ cursor = FailingDiskANNCursor()
+ conn = make_conn(cursor)
+
+ with pytest.raises(psycopg2.OperationalError) as exc_info:
+ conn.create_idx("ragflow_wrt_012", "kb-diskann-fail", 1024)
+
+ assert exc_info.type is psycopg2.OperationalError
+ assert exc_info.value.args == ("DiskANN index creation failed",)
+ assert cursor.executed == CANONICAL_CREATE_IDX_1024_EXECUTIONS
+ assert conn.pool.conn.rollbacks == 1
+ assert conn.pool.conn.commits == 1
+ assert cursor.closed is True
+ assert conn.pool.put_back == [conn.pool.conn, conn.pool.conn]
+
+
+def test_tc_wrt_012_create_idx_executes_complete_ddl_sequence():
+ cursor = RecordingCursor()
+ conn = make_conn(cursor)
+ table = "ragflow_wrt_012"
+
+ result = conn.create_idx(table, "kb-wrt-012", 1024)
+
+ assert result is True
+ assert len(CANONICAL_CREATE_IDX_1024_EXECUTIONS) == 14
+ assert cursor.executed == CANONICAL_CREATE_IDX_1024_EXECUTIONS
+ assert all("kb-wrt-012" not in sql for sql, _params in cursor.executed)
+ assert conn.pool.conn.commits == 2
+ assert conn.pool.conn.rollbacks == 0
+ assert cursor.closed is True
+ assert conn.pool.put_back == [conn.pool.conn, conn.pool.conn]
+
+
[email protected]("adapter", ["get_fields", "get_highlight", "get_scores", "get"])
+def test_tc_ret_508_chunk_adapters_reject_cross_kb_duplicate_ids(adapter):
+ duplicate_chunks = [
+ {"id": "c1", "kb_id": "kb1", "doc_id": "d1", "_highlight": "old", "_score": 0.1},
+ {"id": "c1", "kb_id": "kb2", "doc_id": "d2", "_highlight": "new", "_score": 0.9},
+ ]
+ if adapter == "get":
+ cursor = RecordingCursor()
+ cursor.description = [("id",), ("kb_id",), ("doc_id",)]
+ cursor.rows = [("c1", "kb1", "d1"), ("c1", "kb2", "d2")]
+ conn = make_conn(cursor)
+
+ def call():
+ return conn.get("c1", "ragflow_tenant", ["kb1", "kb2"])
+ else:
+ conn = GaussDBConnection.__new__(GaussDBConnection)
+ result = SearchResult(total=2, chunks=duplicate_chunks)
+ calls = {
+ "get_fields": lambda: conn.get_fields(result, ["doc_id"]),
+ "get_highlight": lambda: conn.get_highlight(result, [], "content_with_weight"),
+ "get_scores": lambda: conn.get_scores(result),
+ }
+ call = calls[adapter]
+
+ with pytest.raises(GaussDBError, match="^cross-KB duplicate chunk id: c1$"):
+ call()
+
+
+def test_tc_ret_509_search_hides_invalid_placeholder_vector_when_vector_field_is_requested():
+ cursor = RecordingCursor()
+ cursor.description = [
+ ("id",),
+ ("kb_id",),
+ ("q_4_vec",),
+ ("q_4_vec_valid",),
+ ("_score",),
+ ("__total",),
+ ]
+ cursor.rows = [("c1", "kb1", "[0,0,0,0]", False, 0.0, 1)]
+ conn = make_conn(cursor)
+
+ result = conn.search(["q_4_vec"], [], {}, [], OrderByExpr(), 0, 10, "ragflow_tenant", ["kb1"])
+
+ sql, _params = cursor.executed[-1]
+ assert "q_4_vec_valid" in sql
+ assert conn.get_fields(result, ["q_4_vec"]) == {"c1": {}}
+
+
+def test_tc_ret_009_search_empty_deep_page_falls_back_to_total_count():
+ description = [("id",), ("kb_id",), ("_score",), ("__total",)]
+ cursor = SequencedCursor(
+ [
+ ([], description),
+ ([("c1", "kb1", 0.0, 15)], description),
+ ]
+ )
+ conn = make_conn(cursor)
+
+ result = conn.search(["id"], [], {}, [], OrderByExpr(), 20, 10, "ragflow_tenant", ["kb1"])
+
+ assert result.total == 15
+ assert result.chunks == []
+ assert len(cursor.executed) == 2
+ first_sql, first_params = cursor.executed[0]
+ fallback_sql, fallback_params = cursor.executed[1]
+ assert all('FROM "public"."ragflow_tenant"' in sql for sql in (first_sql, fallback_sql))
+ assert all("WHERE kb_id IN (%s)" in sql for sql in (first_sql, fallback_sql))
+ assert "COUNT(*) OVER() AS __total" in fallback_sql
+ assert first_params[-2:] == [10, 20]
+ assert fallback_params[-2:] == [1, 0]
+
+
+def test_tc_ret_002_multi_table_search_applies_global_pagination_and_ordering():
+ description = [("id",), ("kb_id",), ("_score",), ("__total",)]
+ cursor = SequencedCursor(
+ [
+ ([("c1", "kb1", 0.9, 2), ("c3", "kb1", 0.3, 2)], description),
+ ([("c2", "kb1", 0.8, 1)], description),
+ ]
+ )
+ conn = make_conn(cursor)
+ match = MatchTextExpr(["content_ltks"], "risk", 10, {"original_query": "risk"})
+
+ result = conn.search(["id"], [], {}, [match], OrderByExpr(), 1, 1, ["ragflow_tenant_a", "ragflow_tenant_b"], ["kb1"])
+
+ assert result.total == 3
+ assert [chunk["id"] for chunk in result.chunks] == ["c2"]
+ assert len(cursor.executed) == 2
+ assert all("LIMIT %s OFFSET %s" in sql for sql, _params in cursor.executed)
+ assert [params[-2:] for _sql, params in cursor.executed] == [[2, 0], [2, 0]]
+
+
+def test_tc_ret_002_multi_table_search_breaks_score_ties_by_kb_then_id():
+ description = [("id",), ("kb_id",), ("_score",), ("__total",)]
+ cursor = SequencedCursor(
+ [
+ ([("c4", "kb2", 0.8, 2), ("c2", "kb1", 0.8, 2)], description),
+ ([("c3", "kb2", 0.8, 2), ("c1", "kb1", 0.8, 2)], description),
+ ]
+ )
+ conn = make_conn(cursor)
+ match = MatchTextExpr(["content_ltks"], "risk", 10, {"original_query": "risk"})
+
+ result = conn.search(
+ ["id"],
+ [],
+ {},
+ [match],
+ OrderByExpr(),
+ 0,
+ 10,
+ ["ragflow_tenant_a", "ragflow_tenant_b"],
+ ["kb1", "kb2"],
+ )
+
+ assert result.total == 4
+ assert [(chunk["kb_id"], chunk["id"]) for chunk in result.chunks] == [
+ ("kb1", "c1"),
+ ("kb1", "c2"),
+ ("kb2", "c3"),
+ ("kb2", "c4"),
+ ]
+
+
+def test_tc_ret_002_multi_table_without_match_or_order_sorts_by_kb_then_id():
+ description = [("id",), ("kb_id",), ("_score",), ("__total",)]
+ cursor = SequencedCursor(
+ [
+ ([("c4", "kb2", 0.0, 2), ("c2", "kb1", 0.0, 2)], description),
+ ([("c3", "kb2", 0.0, 2), ("c1", "kb1", 0.0, 2)], description),
+ ]
+ )
+ conn = make_conn(cursor)
+
+ result = conn.search(
+ ["id"],
+ [],
+ {},
+ [],
+ OrderByExpr(),
+ 0,
+ 10,
+ ["ragflow_tenant_a", "ragflow_tenant_b"],
+ ["kb1", "kb2"],
+ )
+
+ assert result.total == 4
+ assert [(chunk["kb_id"], chunk["id"]) for chunk in result.chunks] == [
+ ("kb1", "c1"),
+ ("kb1", "c2"),
+ ("kb2", "c3"),
+ ("kb2", "c4"),
+ ]
+
+
+def test_tc_ret_606_multi_table_aggregation_merges_duplicate_buckets():
+ description = [("value",), ("count",)]
+ cursor = SequencedCursor(
+ [
+ ([("doc-a", 2), ("doc-b", 1)], description),
+ ([("doc-a", 3)], description),
+ ]
+ )
+ conn = make_conn(cursor)
+
+ result = conn.search([], [], {}, [], OrderByExpr(), 0, 0, ["ragflow_tenant_a", "ragflow_tenant_b"], ["kb1"], ["docnm_kwd"])
+
+ assert result.chunks == [{"value": "doc-a", "count": 5}, {"value": "doc-b", "count": 1}]
+ assert result.total == 2
+
+
+def test_tc_ret_603_get_aggregation_counts_local_jsonb_array_values():
+ conn = GaussDBConnection.__new__(GaussDBConnection)
+ result = SearchResult(
+ total=4,
+ chunks=[{"tag_kwd": ["a", "b", ""]}, {"tag_kwd": ["a"]}, {"tag_kwd": "c"}, {"tag_kwd": None}],
+ )
+
+ assert sorted(conn.get_aggregation(result, "tag_kwd")) == [("a", 2), ("b", 1), ("c", 1)]
+
+
+def test_tc_ret_604_get_aggregation_uses_value_count_rows_without_recounting():
+ conn = GaussDBConnection.__new__(GaussDBConnection)
+ result = SearchResult(total=2, chunks=[{"value": "active", "count": 2}, {"value": "inactive", "count": 1}])
+
+ assert conn.get_aggregation(result, "doc_type_kwd") == [("active", 2), ("inactive", 1)]
+
+
+def test_tc_cfg_313_base_connection_uses_shared_gaussdb_conn_by_default(monkeypatch):
+ class BasePool:
+ masked_uri = "user@host:19995/postgres?schema=public"
+ resolved_schema = "public"
+
+ def __init__(self):
+ self.checked = False
+
+ def check_schema_access(self):
+ self.checked = True
+
+ pool = BasePool()
+ monkeypatch.setitem(sys.modules, "common.doc_store.gaussdb_conn_pool", types.SimpleNamespace(GAUSSDB_CONN=pool))
+
+ base = GaussDBConnectionBase()
+
+ assert base.pool is pool
+ assert pool.checked is True
+
+
+ ("method_name", "args"),
+ [
+ ("create_idx", ("ragflow_tenant", "kb1", 4)),
+ ("delete_idx", ("ragflow_tenant", "kb1")),
+ ("index_exist", ("ragflow_tenant", "kb1")),
+ (
+ "search",
+ ([], [], {}, [], OrderByExpr(), 0, 10, "ragflow_tenant", ["kb1"]),
+ ),
+ ("get", ("c1", "ragflow_tenant", ["kb1"])),
+ ("insert", ([], "ragflow_tenant", "kb1")),
+ ("update", ({"id": "c1"}, {"pagerank_fea": 1}, "ragflow_tenant", "kb1")),
+ ("delete", ({"id": "c1"}, "ragflow_tenant", "kb1")),
+ ("get_total", (SearchResult(total=0, chunks=[]),)),
+ ("get_doc_ids", (SearchResult(total=0, chunks=[]),)),
+ (
+ "get_fields",
+ (SearchResult(total=0, chunks=[]), ["id"]),
+ ),
+ (
+ "get_highlight",
+ (SearchResult(total=0, chunks=[]), [], "content_with_weight"),
+ ),
+ (
+ "get_aggregation",
+ (SearchResult(total=0, chunks=[]), "doc_type_kwd"),
+ ),
+ ("sql", ("SELECT 1", 1, "json")),
+ ],
+)
+def test_tc_cfg_319_base_unimplemented_interfaces_fail_fast(method_name, args):
+ base = GaussDBConnectionBase(pool=HealthPool([]))
+
+ with pytest.raises(NotImplementedError):
+ getattr(base, method_name)(*args)
+
+
+def test_tc_cfg_615_base_connection_health_reports_healthy_for_a_compatible_gaussdb():
+ pool = HealthPool([("GaussDB 8",), ("A",), ("UTF8",), ("UTF8",)])
+ health = GaussDBConnectionBase(pool=pool).health()
+
+ assert health["status"] == "healthy"
+ assert health["version_comment"] == "GaussDB 8"
+ assert health["sql_compatibility"] == "A"
+ assert health["server_encoding"] == "UTF8"
+ assert health["client_encoding"] == "UTF8"
+ assert health["uri"] == pool.masked_uri
+ assert pool.checked is True
+ assert pool.sql == [
+ "SELECT version()",
+ "SHOW sql_compatibility",
+ "SHOW server_encoding",
+ "SHOW client_encoding",
+ ]
+
+
+def test_tc_cfg_602_base_connection_health_reports_healthy_for_ora_compatible_gaussdb():
+ pool = HealthPool([("GaussDB 8",), ("ORA",), ("SQL_ASCII",), ("UTF-8",)])
+ pool.check_schema_access = Mock(wraps=pool.check_schema_access)
+ health = GaussDBConnectionBase(pool=pool).health()
+
+ assert health["status"] == "healthy"
+ assert health["version_comment"] == "GaussDB 8"
+ assert health["sql_compatibility"] == "ORA"
+ assert health["server_encoding"] == "SQL_ASCII"
+ assert health["client_encoding"] == "UTF-8"
+ assert "RAGFlow clients are forced to UTF8" in health["warning"]
+ assert health["uri"] == "u@h:5432/d?schema=public"
+ pool.check_schema_access.assert_called_once_with()
+ assert pool.checked is True
+ assert pool.sql == [
+ "SELECT version()",
+ "SHOW sql_compatibility",
+ "SHOW server_encoding",
+ "SHOW client_encoding",
+ ]
+ assert "password" not in str(health).lower()
+
+
+def test_tc_cfg_621_base_connection_health_rejects_non_utf8_client_encoding():
+ pool = HealthPool([("GaussDB 8",), ("A",), ("UTF8",), ("LATIN1",)])
+
+ health = GaussDBConnectionBase(pool=pool).health()
+
+ assert health["status"] == "unhealthy"
+ assert health["client_encoding"] == "LATIN1"
+ assert health["error"] == ("unsupported GaussDB client encoding, expected UTF8: client_encoding=LATIN1")
+
+
+def test_tc_cfg_603_base_connection_health_reports_unhealthy_for_b_compatibility():
+ pool = HealthPool([("GaussDB 8",), ("B",)])
+ health = GaussDBConnectionBase(pool=pool).health()
+
+ assert health["status"] == "unhealthy"
+ assert health["sql_compatibility"] == "B"
+ assert health["error"] == "unsupported GaussDB compatibility, expected A/ORA: sql_compatibility=B"
+ assert health["uri"] == pool.masked_uri
+ assert "postgresql://user:secret@host:19995/postgres" not in str(health)
+ assert "password" not in str(health).lower()
+ assert "token" not in str(health).lower()
+
+
+def test_tc_cfg_604_base_connection_health_reports_unhealthy_for_m_compatibility():
+ pool = HealthPool([("GaussDB 3.1.0",), ("M",)])
+ health = GaussDBConnectionBase(pool=pool).health()
+
+ assert health["status"] == "unhealthy"
+ assert health["sql_compatibility"] == "M"
+ assert "unsupported GaussDB compatibility" in health["error"]
+ assert "sql_compatibility=M" in health["error"]
+ assert health["uri"] == pool.masked_uri
+ assert "password" not in health["error"].lower()
+ assert "token" not in health["error"].lower()
+ assert "postgresql://user:secret@host:19995/postgres" not in str(health)
+
+
+def test_tc_cfg_605_base_connection_health_reports_unhealthy_when_version_is_missing():
+ pool = HealthPool([None])
+ health = GaussDBConnectionBase(pool=pool).health()
+
+ assert health["status"] == "unhealthy"
+ assert health["version_comment"] == "unknown"
+ assert health["error"] == "GaussDB version query returned no rows"
+ assert health["uri"] == pool.masked_uri
+ assert "password" not in str(health).lower()
+ assert "postgresql://user:secret@host:19995/postgres" not in str(health)
+
+
+def test_tc_cfg_606_base_connection_health_redacts_secrets_from_query_failures():
+ password = "cfg606-password"
+ dsn = f"postgresql://admin:{password}@db.internal:5432/ragflow"
+ token = "cfg606-token"
+ error = ConnectionError(f"connection failed password={password} dsn={dsn} token={token}")
+
+ pool = HealthPool([error])
+
+ health = GaussDBConnectionBase(pool=pool).health()
+
+ assert health == {
+ "status": "unhealthy",
+ "uri": "u@h:5432/d?schema=public",
+ "version_comment": "unknown",
+ "schema": "public",
+ "server_encoding": "unknown",
+ "client_encoding": "unknown",
+ "error": "connection failed password=*** dsn=*** token=***",
+ }
+ assert pool.checked is True
+ assert pool.sql == ["SELECT version()"]
+ serialized = json.dumps(health)
+ assert password not in serialized
+ assert dsn not in serialized
+ assert token not in serialized
+
+
+def test_tc_cfg_607_base_connection_performance_metrics_reports_connected():
+ pool = HealthPool([(1,)])
+
+ metrics = GaussDBConnectionBase(pool=pool).get_performance_metrics()
+
+ assert metrics["connection"] == "connected"
+ assert isinstance(metrics["latency_ms"], float)
+ assert math.isfinite(metrics["latency_ms"])
+ assert metrics["latency_ms"] >= 0
+ assert metrics["schema"] == "public"
+ assert "error" not in metrics
+ assert pool.sql == ["SELECT 1"]
+ serialized = json.dumps(metrics).lower()
+ for forbidden in ("password", "dsn", "api_key", "access_token", "postgresql://"):
+ assert forbidden not in serialized
+
+
+def test_tc_cfg_608_base_connection_performance_metrics_reports_disconnected():
+ password = "cfg608-password"
+ dsn = f"postgresql://admin:{password}@db.internal:5432/ragflow"
+ token = "cfg608-token"
+ pool = HealthPool([RuntimeError(f"connection failed password={password} dsn={dsn} token={token}")])
+ metrics = GaussDBConnectionBase(pool=pool).get_performance_metrics()
+
+ assert metrics["connection"] == "disconnected"
+ assert isinstance(metrics["latency_ms"], float)
+ assert math.isfinite(metrics["latency_ms"])
+ assert metrics["latency_ms"] >= 0
+ assert metrics["error"] == "connection failed password=*** dsn=*** token=***"
+ serialized = json.dumps(metrics)
+ assert password not in serialized
+ assert dsn not in serialized
+ assert token not in serialized
+ assert pool.sql == ["SELECT 1"]
+
+
+def test_tc_cfg_609_base_connection_health_does_not_start_background_polling(monkeypatch):
+ def fail_thread_creation(*_args, **_kwargs):
+ raise AssertionError("background polling must not start")
+
+ monkeypatch.setattr(threading, "Thread", fail_thread_creation)
+ monkeypatch.setattr(threading, "Timer", fail_thread_creation)
+ pool = HealthPool([("GaussDB 8",), ("A",), ("UTF8",), ("UTF8",)])
+ conn = GaussDBConnectionBase(pool=pool)
+ before_count = threading.active_count()
+
+ health = conn.health()
+
+ assert health["status"] == "healthy"
+ assert threading.active_count() == before_count
+ assert pool.sql == [
+ "SELECT version()",
+ "SHOW sql_compatibility",
+ "SHOW server_encoding",
+ "SHOW client_encoding",
+ ]
+
+
+def test_tc_cfg_610_base_connection_health_does_not_create_business_tables_or_indexes():
+ class TestableGaussDBConnection(GaussDBConnection):
+ def create_idx(self, *_args, **_kwargs):
+ raise AssertionError("DDL must not run")
+
+ def create_doc_meta_idx(self, *_args, **_kwargs):
+ raise AssertionError("DDL must not run")
+
+ pool = HealthPool([("GaussDB 8",), ("A",), ("UTF8",), ("UTF8",)])
+ health = TestableGaussDBConnection(pool=pool).health()
+
+ assert health["status"] == "healthy"
+ assert pool.sql == [
+ "SELECT version()",
+ "SHOW sql_compatibility",
+ "SHOW server_encoding",
+ "SHOW client_encoding",
+ ]
+
+
+def test_tc_ret_008_search_empty_index_names_returns_empty_without_database_call():
+ cursor = RecordingCursor()
+ conn = make_conn(cursor)
+
+ res = conn.search(["id"], [], {}, [], OrderByExpr(), 0, 10, [], ["k1"])
+
+ assert type(res).__name__ == "SearchResult"
+ assert res.total == 0
+ assert type(res.total) is int
+ assert res.chunks == []
+ assert conn.search([], [], {}, [], OrderByExpr(), 0, 10, "", None, dataset_ids=["kb1"]) == SearchResult(total=0, chunks=[])
+ assert cursor.executed == []
+
+
+def test_tc_ret_004_search_rejects_mixed_doc_meta_and_chunk_tables_without_database_call():
+ cursor = RecordingCursor()
+ conn = make_conn(cursor)
+
+ with pytest.raises(ValueError, match="^document metadata tables cannot be mixed with chunk search$"):
+ conn.search(
+ ["id"],
+ [],
+ {"kb_id": "kb1", "doc_id": ["d1"]},
+ [],
+ OrderByExpr(),
+ 0,
+ 10,
+ ["ragflow_doc_meta_tenant", "ragflow_tenant"],
+ ["kb1"],
+ )
+
+ assert cursor.executed == []
+
+
+def test_tc_ret_006_search_rejects_missing_kb_boundary_without_database_call():
+ cursor = RecordingCursor()
+ conn = make_conn(cursor)
+ fetch_all = Mock(side_effect=AssertionError("unscoped search must stop before SQL"))
+ conn._fetch_all_with_description = fetch_all
+
+ with pytest.raises(ValueError, match="^GaussDB chunk search requires a kb_id boundary$"):
+ conn.search(["id"], [], {}, [], OrderByExpr(), 0, 10, "ragflow_t1", [])
+
+ assert cursor.executed == []
+ fetch_all.assert_not_called()
+ assert conn._scoped_search_condition({"doc_ids": ["d1"]}, ["kb1"]) == {
+ "doc_id": ["d1"],
+ "kb_id": ["kb1"],
+ }
+
+
+def test_tc_ret_006_empty_doc_ids_keep_kb_scope():
+ conn = make_conn(RecordingCursor())
+
+ assert conn._scoped_search_condition({"doc_ids": []}, ["kb1"]) == {"kb_id": ["kb1"]}
+
+
+def test_tc_ret_007_search_rejects_condition_kb_id_outside_authorized_kbs_without_sql():
+ cursor = RecordingCursor()
+ conn = make_conn(cursor)
+
+ for index_name in ("ragflow_t1", "ragflow_doc_meta_t1"):
+ with pytest.raises(ValueError, match="^condition kb_id must stay within knowledgebase_ids$"):
+ conn.search(["id"], [], {"kb_id": "k_other"}, [], None, 0, 10, index_name, ["k1"])
+
+ assert cursor.executed == []
+
+
+def test_tc_ret_601_search_executes_single_field_aggregation():
+ cursor = SequencedCursor(
+ [
+ (
+ [("active", 2), ("inactive", 1)],
+ [("value",), ("count",)],
+ )
+ ]
+ )
+ conn = make_conn(cursor)
+
+ result = conn.search(
+ [],
+ [],
+ {},
+ [],
+ OrderByExpr(),
+ 0,
+ 10,
+ "ragflow_tenant",
+ ["kb1"],
+ ["doc_type_kwd"],
+ )
+
+ assert result == SearchResult(
+ total=2,
+ chunks=[{"value": "active", "count": 2}, {"value": "inactive", "count": 1}],
+ )
+ assert conn.get_aggregation(result, "doc_type_kwd") == [("active", 2), ("inactive", 1)]
+ assert cursor.executed == [
+ (
+ 'SELECT doc_type_kwd AS value, COUNT(1) AS count FROM "public"."ragflow_tenant" WHERE kb_id IN (%s) AND doc_type_kwd IS NOT NULL GROUP BY value ORDER BY count DESC, value ASC LIMIT %s',
+ ["kb1", 1000],
+ )
+ ]
+
+
+def test_tc_ret_608_search_rejects_multiple_aggregation_fields_without_partial_query():
+ cursor = RecordingCursor()
+ conn = make_conn(cursor)
+
+ with pytest.raises(ValueError, match="^GaussDB search supports one aggregation field per request$"):
+ conn.search(
+ [],
+ [],
+ {},
+ [],
+ OrderByExpr(),
+ 0,
+ 10,
+ "ragflow_tenant",
+ ["kb1"],
+ ["docnm_kwd", "tag_kwd"],
+ )
+
+ assert cursor.executed == []
+
+
+def test_tc_ret_209_search_rejects_non_float_vector_type_without_sql():
+ cursor = RecordingCursor()
+ conn = make_conn(cursor)
+ bad = MatchDenseExpr("q_1024_vec", [0.1] * 1024, "byte", "cosine", 10, {"similarity": 0.0})
+
+ with pytest.raises(ValueError, match="unsupported GaussDB vector data type|byte"):
+ conn.search(["id"], [], {"kb_id": "k1"}, [bad], OrderByExpr(), 0, 10, "ragflow_t1", ["k1"])
+
+ assert cursor.executed == []
+
+
+def test_tc_ret_207_search_propagates_ann_unavailable_error_without_fallback():
+ conn = make_conn(RecordingCursor())
+ calls = []
+
+ def fail_fetch_all_with_description(sql, params):
+ calls.append((sql, params))
+ raise GaussDBError("ANN gsdiskann index unavailable")
+
+ conn._fetch_all_with_description = fail_fetch_all_with_description
+ dense = MatchDenseExpr("q_1024_vec", [0.1] * 1024, "float", "cosine", 10, {"similarity": 0.0})
+
+ with pytest.raises(GaussDBError, match="ANN|gsdiskann|index"):
+ conn.search(["id"], [], {"kb_id": "k1"}, [dense], OrderByExpr(), 0, 10, "ragflow_t1", ["k1"])
+
+ assert len(calls) == 1
+ assert "q_1024_vec" in calls[0][0]
+
+
+def test_tc_ret_208_search_propagates_missing_vector_column_error():
+ conn = make_conn(RecordingCursor())
+ calls = []
+
+ def fail_fetch_all_with_description(sql, params):
+ calls.append((sql, params))
+ raise psycopg2.errors.UndefinedColumn("column q_512_vec does not exist")
+
+ conn._fetch_all_with_description = fail_fetch_all_with_description
+ dense = MatchDenseExpr("q_512_vec", [0.1] * 512, "float", "cosine", 10, {"similarity": 0.0})
+
+ with pytest.raises(psycopg2.errors.UndefinedColumn, match="q_512_vec"):
+ conn.search(["id"], [], {"kb_id": "k1"}, [dense], OrderByExpr(), 0, 10, "ragflow_t1", ["k1"])
+
+ assert len(calls) == 1
+ assert "q_512_vec" in calls[0][0]
+
+
+def test_tc_ret_010_multi_table_search_limit_zero_uses_default_collection_limit():
+ conn = make_conn(RecordingCursor())
+ calls = []
+ table_chunks = {
+ "ragflow_a": [{"id": f"a-{index:05d}"} for index in range(5001)],
+ "ragflow_b": [{"id": f"b-{index:05d}"} for index in range(5000)],
+ }
+
+ def fake_search_chunk_table(**kwargs):
+ calls.append((kwargs["table"], kwargs["limit"]))
+ chunks = table_chunks[kwargs["table"]]
+ return SearchResult(total=len(chunks), chunks=chunks)
+
+ conn._search_chunk_table = fake_search_chunk_table
+
+ result = conn.search(["id"], [], {"kb_id": "kb1"}, [], OrderByExpr().asc("id"), 0, 0, ["ragflow_a", "ragflow_b"], ["kb1"])
+
+ assert result.total == 10001
+ assert len(result.chunks) == 10000
+ assert result.chunks[0]["id"] == "a-00000"
+ assert result.chunks[-1]["id"] == "b-04998"
+ assert all(chunk["id"] != "b-04999" for chunk in result.chunks)
+ assert calls == [("ragflow_a", 10000), ("ragflow_b", 10000)]
+
+ calls.clear()
+ offset_result = conn.search(["id"], [], {"kb_id": "kb1"}, [], OrderByExpr().asc("id"), 100, 0, ["ragflow_a", "ragflow_b"], ["kb1"])
+
+ assert offset_result.total == 10001
+ assert len(offset_result.chunks) == 9901
+ assert offset_result.chunks[0]["id"] == "a-00100"
+ assert offset_result.chunks[-1]["id"] == "b-04999"
+ assert calls == [("ragflow_a", 10100), ("ragflow_b", 10100)]
+
+
+def test_tc_wrt_511_update_failure_rolls_back_once_without_commit():
+ class FailingCursor(RecordingCursor):
+ def execute(self, sql, params=None):
+ super().execute(sql, params)
+ raise psycopg2.Error("Update failed")
+
+ cursor = FailingCursor()
+ conn = make_conn(cursor)
+
+ assert conn.update({"id": "c1"}, {"pagerank_fea": 1}, "ragflow_tenant", "kb1") is False
+ assert len(cursor.executed) == 1
+ assert conn.pool.conn.rollbacks == 1
+ assert conn.pool.conn.commits == 0
+
+
+def test_tc_wrt_501_update_empty_condition_or_value_returns_false_without_sql():
+ cursor = RecordingCursor()
+ conn = make_conn(cursor)
+
+ assert conn.update({}, {"pagerank_fea": 1}, "ragflow_tenant", "kb1") is False
+ assert conn.update({"id": "c1"}, {}, "ragflow_tenant", "kb1") is False
+ assert cursor.executed == []
+
+
+def test_tc_sql_508_sql_markdown_serializes_bytes_and_json_values():
+ cursor = RecordingCursor()
+ cursor.description = [("doc_id",), ("payload",), ("raw",)]
+ cursor.rows = [("doc1", {"amount": 120}, b"ok")]
+ conn = make_conn(cursor)
+
+ result = conn.sql("SELECT doc_id FROM ragflow_tenant WHERE kb_id = 'kb1'", format="markdown")
+
+ assert result == {
+ "columns": [
+ {"name": "doc_id", "type": "text"},
+ {"name": "payload", "type": "text"},
+ {"name": "raw", "type": "text"},
+ ],
+ "rows": [["doc1", '{"amount": 120}', "ok"]],
+ "markdown": '|doc_id|payload|raw|\n|---|---|---|\n|doc1|{"amount": 120}|ok|',
+ }
+
+
+def test_tc_sql_502_fetch_all_with_description_sets_statement_timeout_before_business_sql():
+ cursor = RecordingCursor()
+ cursor.rows = [("doc1",)]
+ cursor.description = [("doc_id",)]
+ conn = make_conn(cursor)
+
+ rows, description = conn._fetch_all_with_description("SELECT doc_id FROM ragflow_tenant WHERE kb_id = 'kb1'", [], 30000)
+
+ assert rows == [("doc1",)]
+ assert description == [("doc_id",)]
+ assert cursor.executed[0] == ("SET LOCAL statement_timeout = 30000", [])
+ assert cursor.executed[1] == ("SELECT doc_id FROM ragflow_tenant WHERE kb_id = 'kb1'", [])
+ assert cursor.closed is True
+ assert conn.pool.put_back == [conn.pool.conn]
+
+
+def test_tc_sql_503_sql_propagates_timeout_from_fetch_all_with_statement_timeout():
+ conn = make_conn(RecordingCursor())
+ calls = []
+
+ def fail_fetch(sql, params, statement_timeout_ms=None):
+ calls.append((sql, params, statement_timeout_ms))
+ raise TimeoutError("statement timeout")
+
+ conn._fetch_all_with_description = fail_fetch
+
+ with pytest.raises(TimeoutError, match="statement timeout"):
+ conn.sql("SELECT doc_id FROM ragflow_tenant WHERE kb_id = 'kb1'")
+
+ assert calls == [
+ (
+ "SELECT doc_id FROM \"public\".ragflow_tenant WHERE kb_id = 'kb1' LIMIT 128",
+ [],
+ 30000,
+ )
+ ]
+
+
+def test_tc_sql_510_validator_rejection_prevents_database_fetch():
+ cursor = RecordingCursor()
+ conn = make_conn(cursor)
+
+ def fail_fetch(*_args, **_kwargs):
+ raise AssertionError("database fetch must not be called")
+
+ conn._fetch_all_with_description = fail_fetch
+
+ with pytest.raises(UnsafeGaussDBSQL, match="SELECT \\* is not allowed"):
+ conn.sql("SELECT * FROM ragflow_tenant WHERE kb_id = 'kb1'")
+
+ assert cursor.executed == []
+ assert conn.pool.put_back == []
+
+
+def test_tc_ret_605_get_highlight_uses_alias_and_ignores_empty_rows():
+ conn = GaussDBConnection.__new__(GaussDBConnection)
+ result = SearchResult(
+ total=4,
+ chunks=[
+ {"id": "c1", "_highlight": "<em>risk</em>"},
+ {"id": "c2", "highlight": "audit"},
+ {"id": "c3"},
+ {"highlight": "missing id"},
+ ],
+ )
+
+ assert conn.get_highlight(result, ["risk"], "content_with_weight") == {"c1": "<em>risk</em>", "c2": "audit"}
+ assert (
+ conn.get_highlight(
+ SearchResult(total=2, chunks=[{"id": "c3"}, {"id": "c4", "_highlight": None}]),
+ ["risk"],
+ "content_with_weight",
+ )
+ == {}
+ )
+
+
+def test_tc_ret_506_get_fields_filters_none_values_per_chunk():
+ conn = GaussDBConnection.__new__(GaussDBConnection)
+ res = SearchResult(
+ total=3,
+ chunks=[
+ {"id": "c1", "field1": "value1", "field2": None},
+ {"id": "c2", "field1": None, "field2": "value2"},
+ {"field1": "missing id", "field2": "ignored"},
+ ],
+ )
+
+ assert conn.get_fields(res, ["field1", "field2"]) == {
+ "c1": {"field1": "value1"},
+ "c2": {"field2": "value2"},
+ }
+
+
+def test_tc_ret_507_get_scores_defaults_missing_score_to_zero():
+ conn = GaussDBConnection.__new__(GaussDBConnection)
+ res = SearchResult(
+ total=4,
+ chunks=[{"id": "c1", "_score": 0.8}, {"id": "c2", "_score": None}, {"id": "c3"}, {"_score": 1.0}],
+ )
+
+ assert conn.get_scores(res) == {"c1": 0.8, "c2": 0.0, "c3": 0.0}
+
+
+def test_tc_ret_509_invalid_vector_is_removed_before_get_fields_returns_values():
+ conn = GaussDBConnection.__new__(GaussDBConnection)
+ chunk = conn._row_to_chunk(
+ ("c1", "k1", "[0.0,0.0]", False, "hello"),
+ [("id",), ("kb_id",), ("q_2_vec",), ("q_2_vec_valid",), ("content_with_weight",)],
+ )
+ fields = conn.get_fields(SearchResult(total=1, chunks=[chunk]), ["q_2_vec", "content_with_weight"])
+
+ assert chunk == {"id": "c1", "kb_id": "k1", "q_2_vec_valid": False, "content_with_weight": "hello"}
+ assert fields == {"c1": {"content_with_weight": "hello"}}
+
+
+def test_tc_ret_505_get_doc_ids_filters_missing_ids_in_stable_order():
+ conn = GaussDBConnection.__new__(GaussDBConnection)
+ result = SearchResult(
+ total=3,
+ chunks=[
+ {"id": "c2", "kb_id": "kb1"},
+ {"kb_id": "kb1"},
+ {"id": "c1", "kb_id": "kb1"},
+ ],
+ )
+
+ assert conn.get_doc_ids(result) == ["c2", "c1"]
+
+
+def test_tc_ret_510_search_returns_search_result_with_total_and_chunks_shape():
+ cursor = RecordingCursor()
+ cursor.description = [("id",), ("kb_id",), ("_score",), ("__total",)]
+ cursor.rows = [("c1", "k1", 0.7, 2), ("c2", "k1", 0.5, 2)]
+ conn = make_conn(cursor)
+
+ res = conn.search(["id", "kb_id"], [], {"kb_id": "k1"}, [], OrderByExpr(), 0, 10, "ragflow_t1", ["k1"])
+
+ assert type(res).__name__ == "SearchResult"
+ assert res.total == 2
+ assert isinstance(res.total, int)
+ assert res.chunks == [{"id": "c1", "kb_id": "k1", "_score": 0.7}, {"id": "c2", "kb_id": "k1", "_score": 0.5}]
+ assert conn.get_total(SearchResult(total="3", chunks=[])) == 3
+
+
+def test_tc_mgf_713_fetch_metadata_doc_ids_returns_empty_for_empty_scope_or_filter():
+ cursor = RecordingCursor()
+ conn = make_conn(cursor)
+
+ assert conn.fetch_metadata_doc_ids("ragflow_doc_meta_tenant", [], "meta_fields ? 'a'", [], 10) == []
+ assert conn.fetch_metadata_doc_ids("ragflow_doc_meta_tenant", ["kb1"], "", [], 10) == []
+ assert cursor.executed == []
+ assert conn.pool.get_calls == 0
+ assert conn.pool.put_back == []
+ assert conn.pool.conn.commits == 0
+
+
+def test_tc_mgf_811_fetch_metadata_doc_ids_preserves_probe_limit_and_row_shapes():
+ cursor = RecordingCursor()
+ cursor.rows = [{"id": "doc1"}, {"id": None}, ("doc2",), []]
+ conn = make_conn(cursor)
+
+ result = conn.fetch_metadata_doc_ids("ragflow_doc_meta_tenant", ["kb1"], "meta_fields ? 'a'", [], 4)
+
+ assert result == ["doc1", "doc2"]
+ assert cursor.executed == [
+ (
+ 'SELECT id FROM "public"."ragflow_doc_meta_tenant" WHERE kb_id IN (%s) AND (meta_fields ? \'a\') ORDER BY id LIMIT %s',
+ ["kb1", 4],
+ )
+ ]
+ assert cursor.closed is True
+ assert conn.pool.put_back == [conn.pool.conn]
+ assert conn.pool.conn.commits == 0
+ assert conn.pool.conn.rollbacks == 0
+
+
+def test_tc_ret_003_search_doc_meta_without_kb_scope_fails_before_sql():
+ cursor = RecordingCursor()
+ conn = make_conn(cursor)
+
+ with pytest.raises(ValueError, match="^GaussDB document metadata search requires a kb_id boundary$"):
+ conn.search(
+ ["id", "kb_id"],
+ [],
+ {},
+ [],
+ OrderByExpr(),
+ 0,
+ 10,
+ "ragflow_doc_meta_tenant",
+ [],
+ )
+
+ assert cursor.executed == []
+ assert conn.pool.get_calls == 0
+ assert conn.pool.put_back == []
+ assert conn.pool.conn.commits == 0
+
+
+def test_tc_ret_604_search_chunk_aggregation_ignores_null_values_and_missing_counts():
+ cursor = SequencedCursor([([(None, 2), ("risk", None)], [("value",), ("count",)])])
+ conn = make_conn(cursor)
+
+ result = conn.search([], [], {}, [], OrderByExpr(), 0, 10, "ragflow_tenant", ["kb1"], ["docnm_kwd"])
+
+ assert result == SearchResult(total=1, chunks=[{"value": "risk", "count": 0}])
+
+
+def test_tc_ret_607_search_chunk_aggregation_returns_value_counts_for_jsonb_array_fields():
+ cursor = SequencedCursor([([("tag1", 2), ("tag2", 1)], [("value",), ("count",)])])
+ conn = make_conn(cursor)
+
+ result = conn.search([], [], {}, [], OrderByExpr(), 0, 10, "ragflow_tenant", ["kb1"], ["tag_kwd"])
+
+ assert result == SearchResult(total=2, chunks=[{"value": "tag1", "count": 2}, {"value": "tag2", "count": 1}])
+ assert len(cursor.executed) == 1
+ sql, params = cursor.executed[0]
+ assert "jsonb_array_elements_text(COALESCE(tag_kwd, '[]'::jsonb))" in sql
+ assert "LATERAL" not in sql.upper()
+ assert params == ["kb1", 1000]
+
+
+def test_tc_ret_607_search_chunk_aggregation_reraises_non_jsonb_array_builder_errors():
+ class FailingAggregationBuilder:
+ def build_aggregation_sql(self, **_kwargs):
+ raise ValueError("unsupported aggregation field")
+
+ cursor = RecordingCursor()
+ conn = make_conn(cursor)
+ conn._search_builder = lambda: FailingAggregationBuilder()
+
+ with pytest.raises(ValueError, match="unsupported aggregation field"):
+ conn.search([], [], {}, [], OrderByExpr(), 0, 10, "ragflow_tenant", ["kb1"], ["docnm_kwd"])
+
+ assert cursor.executed == []
+
+
+def test_tc_mgf_713_fetch_metadata_doc_ids_rejects_chunk_table():
+ cursor = RecordingCursor()
+ conn = make_conn(cursor)
+
+ with pytest.raises(ValueError, match="document metadata"):
+ conn.fetch_metadata_doc_ids("ragflow_tenant", ["kb1"], "TRUE", [], 10)
+
+ assert cursor.executed == []
+ assert conn.pool.get_calls == 0
+ assert conn.pool.put_back == []
+ assert conn.pool.conn.commits == 0
+
+
+ ("delta", "min_w", "max_w"),
+ [(5, 0, 100), (-5, 0, 100), (200, -10, 120), (-200, -10, 120)],
+)
+def test_tc_wrt_801_adjust_chunk_pagerank_repeated_calls_use_atomic_sql(delta, min_w, max_w):
+ cursor = RecordingCursor()
+ cursor.rowcount = 1
+ conn = make_conn(cursor)
+
+ first = conn.adjust_chunk_pagerank_fea("c1", "ragflow_tenant", "kb1", delta, min_w=min_w, max_w=max_w)
+ second = conn.adjust_chunk_pagerank_fea("c1", "ragflow_tenant", "kb1", delta, min_w=min_w, max_w=max_w)
+
+ expected = (
+ 'UPDATE "public"."ragflow_tenant" SET pagerank_fea = GREATEST(%s, LEAST(%s, COALESCE(pagerank_fea, 0) + %s)) WHERE kb_id = %s AND id = %s',
+ [min_w, max_w, delta, "kb1", "c1"],
+ )
+ assert first is second is True
+ assert cursor.executed == [expected, expected]
+ assert conn.pool.conn.commits == 2
+
+
+def test_tc_wrt_804_adjust_chunk_pagerank_ignores_extra_identity_parameters():
+ cursor = RecordingCursor()
+ cursor.rowcount = 1
+ conn = make_conn(cursor)
+
+ result = conn.adjust_chunk_pagerank_fea(
+ "c1",
+ "ragflow_tenant",
+ "kb1",
+ 5,
+ row_id="row-ignored",
+ extra_param="ignored",
+ )
+
+ sql, params = cursor.executed[-1]
+ assert result is True
+ assert "WHERE kb_id = %s AND id = %s" in sql
+ assert "row_id" not in sql
+ assert "extra_param" not in sql
+ assert params == [0, 100, 5, "kb1", "c1"]
+
+
+def test_tc_wrt_805_adjust_chunk_pagerank_requires_chunk_and_dataset_scope_without_sql():
+ cursor = RecordingCursor()
+ conn = make_conn(cursor)
+
+ assert conn.adjust_chunk_pagerank_fea("", "ragflow_tenant", "kb1", 1) is False
+ assert conn.adjust_chunk_pagerank_fea("c1", "ragflow_tenant", "", 1) is False
+ assert cursor.executed == []
+
+
+ ("field", "first_value", "second_value", "expected_ids", "expected_order"),
+ [
+ (
+ "page_num_int",
+ "[2]",
+ "[1]",
+ ["c-second", "c-first"],
+ "COALESCE((page_num_int #>> '{0}')::int, 100000000) ASC",
+ ),
+ (
+ "position_int",
+ "{}",
+ "[[0, 0, 0, 9]]",
+ ["c-second", "c-first"],
+ "COALESCE((position_int #>> '{0,3}')::int, 100000000) ASC",
+ ),
+ (
+ "top_int",
+ None,
+ "[2]",
+ ["c-second", "c-first"],
+ "COALESCE((top_int #>> '{0}')::int, 100000000) ASC",
+ ),
+ ("pagerank_fea", 2, 1, ["c-second", "c-first"], "ORDER BY pagerank_fea ASC"),
+ ("doc_id", None, "a", ["c-first", "c-second"], "ORDER BY doc_id ASC"),
+ ],
+)
+def test_tc_ret_512_multi_table_search_applies_safe_compatibility_sorting(
+ field,
+ first_value,
+ second_value,
+ expected_ids,
+ expected_order,
+):
+ description = [("id",), ("kb_id",), (field,), ("_score",), ("__total",)]
+ cursor = SequencedCursor(
+ [
+ ([("c-first", "kb1", first_value, 0.0, 1)], description),
+ ([("c-second", "kb1", second_value, 0.0, 1)], description),
+ ]
+ )
+ conn = make_conn(cursor)
+ order_by = type("Order", (), {"fields": [(field, False)]})()
+
+ result = conn.search(
+ ["id", field],
+ [],
+ {},
+ [],
+ order_by,
+ 0,
+ 10,
+ ["ragflow_tenant_a", "ragflow_tenant_b"],
+ ["kb1"],
+ )
+
+ assert result.total == 2
+ assert [chunk["id"] for chunk in result.chunks] == expected_ids
+ assert all(expected_order in sql for sql, _params in cursor.executed)
+
+
+def test_tc_wrt_504_update_overwrites_metadata_and_derives_title_with_scoped_parameters():
+ cursor = RecordingCursor()
+ cursor.rowcount = 1
+ conn = make_conn(cursor)
+ metadata = {"_title": "New", "x": 1}
+
+ updated = conn.update(
+ {"id": "c1", "kb_id": "k1"},
+ {"metadata": metadata},
+ "ragflow_tenant",
+ "k1",
+ )
+
+ assert updated is True
+ assert cursor.executed == [
+ (
+ 'UPDATE "public"."ragflow_tenant" SET metadata = %s::jsonb, docnm_kwd = %s WHERE id = %s AND kb_id = %s',
+ ['{"_title": "New", "x": 1}', "New", "c1", "k1"],
+ )
+ ]
+ sql = cursor.executed[0][0]
+ assert all(value not in sql for value in ("New", "c1", "k1"))
+ assert json.dumps(metadata, ensure_ascii=False) not in sql
+ assert all(keyword not in sql for keyword in ("DELETE ", "DROP ", "CREATE ", "ALTER "))
+
+ group_sql, group_params = conn._build_set_clause(
+ {"metadata": {"_group_id": "g-new"}},
+ is_meta=False,
+ condition={"id": "c1", "kb_id": "k1"},
+ )
+ meta_sql, meta_params = conn._build_set_clause(
+ {"meta_fields": {"status": "open"}},
+ is_meta=True,
+ condition={"id": "doc1", "kb_id": "k1"},
+ )
+ assert group_sql == "metadata = %s::jsonb, group_id = %s"
+ assert group_params == ['{"_group_id": "g-new"}', "g-new"]
+ assert meta_sql == "meta_fields = %s::jsonb"
+ assert meta_params == ['{"status": "open"}']
+
+
+def test_tc_wrt_608_delete_exists_stays_within_kb_scope():
+ cursor = RecordingCursor()
+ cursor.rowcount = 2
+ conn = make_conn(cursor)
+
+ deleted = conn.delete({"exists": "doc_id"}, "ragflow_tenant", "kb1")
+
+ assert deleted == 2
+ assert cursor.executed == [('DELETE FROM "public"."ragflow_tenant" WHERE doc_id IS NOT NULL AND kb_id = %s', ["kb1"])]
+ assert all(" WHERE " in sql and "kb_id = %s" in sql for sql, _params in cursor.executed)
+
+
+def test_tc_ret_804_parse_fusion_vector_weight_uses_last_weight_value():
+ assert parse_fusion_vector_weight(FusionExpr("weighted_sum", 10, {"weights": "0.7,0.3"})) == 0.3
+
+
+def test_tc_ret_805_parse_fusion_vector_weight_returns_none_for_missing_or_bad_weights():
+ assert parse_fusion_vector_weight(FusionExpr("weighted_sum", 10, None)) is None
+ assert parse_fusion_vector_weight(FusionExpr("weighted_sum", 10, {})) is None
+ assert parse_fusion_vector_weight(FusionExpr("weighted_sum", 10, {"weights": ""})) is None
+ assert parse_fusion_vector_weight(FusionExpr("weighted_sum", 10, {"weights": "abc"})) is None
+ assert parse_fusion_vector_weight(FusionExpr("weighted_sum", 10, {"weights": "0.7,bad"})) is None
+
+
+def test_tc_ret_806_parse_match_expressions_applies_gaussdb_vector_weight_defaults(monkeypatch):
+ monkeypatch.setattr(gaussdb_conn_module, "_tokenize_query_terms", lambda query: str(query).split())
+ conn = make_conn(RecordingCursor())
+ text = MatchTextExpr(["content_with_weight"], "hello", 10, {"original_query": "hello"})
+ dense = MatchDenseExpr("q_4_vec", [0.1, 0.2, 0.3, 0.4], "float", "cosine", 10, {"similarity": 0.0})
+ fusion = FusionExpr("weighted_sum", 6, {"weights": "0.7,0.3"})
+
+ text_only = conn._parse_match_expressions([text], rank_feature=None)
+ vector_only = conn._parse_match_expressions([dense], rank_feature=None)
+ hybrid_default = conn._parse_match_expressions([text, dense], rank_feature=None)
+ hybrid_explicit = conn._parse_match_expressions([text, dense, fusion], rank_feature={"pagerank_fea": 7})
+
+ assert text_only["vector_weight"] == 0.0
+ assert vector_only["vector_weight"] == 1.0
+ assert hybrid_default["vector_weight"] == 0.5
+ assert {text_only["pagerank_weight"], vector_only["pagerank_weight"], hybrid_default["pagerank_weight"]} == {10.0}
+ assert hybrid_explicit["vector_weight"] == 0.3
+ assert hybrid_explicit["topn"] == 6
+ assert hybrid_explicit["pagerank_weight"] == 7.0
+
+
+def test_tc_ret_110_parse_match_tokenizes_original_query(monkeypatch):
+ queries = []
+
+ def tokenize(query):
+ queries.append(query)
+ return ["高斯", "数据库", "端", "到", "端", "检索"]
+
+ monkeypatch.setattr(gaussdb_conn_module, "_tokenize_query_terms", tokenize)
+ conn = make_conn(RecordingCursor())
+ text = MatchTextExpr(["content_ltks"], "backend-specific syntax", 10, {"original_query": "高斯数据库端到端检索"})
+
+ parsed = conn._parse_match_expressions([text], rank_feature=None)
+
+ assert queries == ["高斯数据库端到端检索"]
+ assert parsed["keywords"] == ["高斯", "数据库", "端", "到", "端", "检索"]
+
+
+def test_tc_wrt_905_vector_literal_rejects_dimension_mismatch():
+ with pytest.raises(ValueError, match="vector dimension mismatch: expected 3, got 2"):
+ vector_literal([0.1, 0.2], 3)
+
+
+def test_tc_wrt_905_vector_literal_accepts_native_and_json_vectors():
+ assert vector_literal([0.1, 0.2, 0.3], 3) == "[0.1,0.2,0.3]"
+ assert vector_literal("[0.1, 0.2, 0.3]", 3) == "[0.1,0.2,0.3]"
+
+
+def test_tc_wrt_906_zero_vector_literal_returns_all_zero_vector_string():
+ result = zero_vector_literal(3)
+
+ assert result == "[0,0,0]"
+ assert isinstance(result, str)
+
+
+def test_tc_wrt_907_ordered_columns_unifies_heterogeneous_rows():
+ rows = [
+ {"q_1024_vec": "[0,0]", "id": "c1"},
+ {"important_kwd": '["risk"]', "id": "c2"},
+ ]
+
+ assert ordered_columns(rows) == ["id", "important_kwd", "q_1024_vec"]
+
+
+def test_tc_wrt_908_decode_column_value_deserializes_jsonb_and_vector_strings():
+ assert decode_column_value("chunk_data", '{"amount":120,"active":true}') == {"amount": 120, "active": True}
+ assert decode_column_value("chunk_data", {"amount": 120}) == {"amount": 120}
+ assert decode_column_value("q_3_vec", "[0.1, 0.2, 0.3]") == [0.1, 0.2, 0.3]
+ assert decode_column_value("q_3_vec", [0.1, 0.2, 0.3]) == [0.1, 0.2, 0.3]
+ assert parse_json_dict('{"amount":120}') == {"amount": 120}
+ assert parse_json_dict("not-json") == {}
+ assert parse_json_dict("[1,2]") == {}
+ assert parse_json_dict(None) == {}
+
+
+def test_tc_wrt_908_decode_column_value_rejects_malformed_jsonb_readback():
+ with pytest.raises(ValueError, match="^invalid JSONB value for chunk_data$") as exc_info:
+ decode_column_value("chunk_data", "not-json")
+
+ assert exc_info.type is ValueError
+
+
+def test_tc_wrt_908_decode_column_value_rejects_malformed_vector_readback():
+ with pytest.raises(ValueError, match="^invalid vector value for q_3_vec$") as exc_info:
+ decode_column_value("q_3_vec", "not-vector")
+
+ assert exc_info.type is ValueError
+
+
+def test_tc_wrt_908_vector_literal_rejects_invalid_string():
+ with pytest.raises(ValueError, match="^invalid vector literal$") as exc_info:
+ vector_literal("not-json", 2)
+
+ assert exc_info.type is ValueError
diff --git a/test/unit_test/rag/utils/test_gaussdb_conn_pool.py b/test/unit_test/rag/utils/test_gaussdb_conn_pool.py
new file mode 100644
index 0000000000..cb3ce22419
--- /dev/null
+++ b/test/unit_test/rag/utils/test_gaussdb_conn_pool.py
@@ -0,0 +1,881 @@
+#
+# Copyright 2025 The InfiniFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+import importlib.util
+import sys
+from pathlib import Path
+from types import ModuleType
+from unittest.mock import Mock, patch
+
+import pytest
+
+import common
+
+
+class ImportFakePool:
+ def getconn(self):
+ raise RuntimeError("import fake pool should not be used")
+
+ def putconn(self, conn, close=False):
+ pass
+
+ def closeall(self):
+ pass
+
+
+def _load_pool_module_without_import_state_leaks():
+ module_name = "_test_gaussdb_conn_pool_module"
+ module_path = Path(__file__).resolve().parents[4] / "common" / "doc_store" / "gaussdb_conn_pool.py"
+ spec = importlib.util.spec_from_file_location(module_name, module_path)
+ module = importlib.util.module_from_spec(spec)
+ fake_settings = ModuleType("common.settings")
+ fake_settings.GAUSSDB = {
+ "host": "gaussdb.local",
+ "port": "19995",
+ "database": "postgres",
+ "user": "sqlbuilder",
+ "password": "fake-unit-password",
+ }
+ fake_settings.get_base_config = lambda *_args, **_kwargs: {}
+
+ missing = object()
+ previous_settings_module = sys.modules.get("common.settings", missing)
+ previous_test_module = sys.modules.get(module_name, missing)
+ previous_common_settings = getattr(common, "settings", missing)
+ sys.modules["common.settings"] = fake_settings
+ sys.modules[module_name] = module
+ common.settings = fake_settings
+ try:
+ with patch("psycopg2.pool.ThreadedConnectionPool", lambda *_args, **_kwargs: ImportFakePool()):
+ spec.loader.exec_module(module)
+ finally:
+ for name, previous in (
+ ("common.settings", previous_settings_module),
+ (module_name, previous_test_module),
+ ):
+ if previous is missing:
+ sys.modules.pop(name, None)
+ else:
+ sys.modules[name] = previous
+ if previous_common_settings is missing:
+ delattr(common, "settings")
+ else:
+ common.settings = previous_common_settings
+ return module
+
+
+pool_mod = _load_pool_module_without_import_state_leaks()
+GaussDBAuthenticationError = pool_mod.GaussDBAuthenticationError
+GaussDBConfig = pool_mod.GaussDBConfig
+GaussDBConnectionError = pool_mod.GaussDBConnectionError
+GaussDBConnectionPool = pool_mod.GaussDBConnectionPool
+GaussDBPermissionError = pool_mod.GaussDBPermissionError
+InvalidGaussDBConfig = pool_mod.InvalidGaussDBConfig
+LazyGaussDBConnectionPool = pool_mod._LazyGaussDBConnectionPool
+_normalize_schema = pool_mod._normalize_schema
+classify_gaussdb_exception = pool_mod.classify_gaussdb_exception
+load_gaussdb_config = pool_mod.load_gaussdb_config
+mask_gaussdb_uri = pool_mod.mask_gaussdb_uri
+
+
+def _raw_config(**overrides):
+ config = {
+ "host": "127.0.0.1",
+ "port": "19995",
+ "database": "postgres",
+ "user": "sqlbuilder",
+ "password": "fake-unit-password",
+ }
+ config.update(overrides)
+ return config
+
+
+def _install_runtime_settings(monkeypatch, gaussdb, get_base_config):
+ runtime_settings = ModuleType("common.settings")
+ runtime_settings.GAUSSDB = gaussdb
+ runtime_settings.get_base_config = get_base_config
+ monkeypatch.setitem(sys.modules, "common.settings", runtime_settings)
+ monkeypatch.setattr(common, "settings", runtime_settings, raising=False)
+ return runtime_settings
+
+
+def test_tc_cfg_001_load_gaussdb_config_loads_complete_config():
+ cfg = load_gaussdb_config(_raw_config())
+
+ assert cfg == GaussDBConfig(
+ host="127.0.0.1",
+ port=19995,
+ database="postgres",
+ user="sqlbuilder",
+ password="fake-unit-password",
+ schema="public",
+ )
+ assert type(cfg.port) is int
+ for database_key in ("database", "db_name", "name"):
+ raw = _raw_config()
+ raw.pop("database")
+ raw[database_key] = f"{database_key}-literal"
+ assert load_gaussdb_config(raw).database == f"{database_key}-literal"
+ with pytest.raises(InvalidGaussDBConfig, match=r"^missing gaussdb config field\(s\): host, port, database, user, password$"):
+ load_gaussdb_config({"config": _raw_config()})
+
+
+def test_tc_cfg_002_load_gaussdb_config_preserves_explicit_schema():
+ cfg = load_gaussdb_config(_raw_config(schema="tenant_schema"))
+
+ assert cfg.schema == "tenant_schema"
+
+
[email protected]("port", [1, 65535])
+def test_tc_cfg_003_load_gaussdb_config_accepts_port_boundaries(port):
+ cfg = load_gaussdb_config(_raw_config(port=port))
+
+ assert cfg.port == port
+ assert cfg.schema == "public"
+
+
+def test_tc_cfg_004_mask_gaussdb_uri_omits_password_from_masked_uri():
+ cfg = GaussDBConfig("db.example", 19995, "postgres", "sqlbuilder", "secret-password", "public")
+
+ masked = mask_gaussdb_uri(cfg)
+
+ assert masked == "[email protected]:19995/postgres?schema=public"
+ assert "secret-password" not in masked
+
+
[email protected]("schema", ["", " "])
+def test_tc_cfg_102_normalize_empty_schema_to_public(schema):
+ assert _normalize_schema(schema) == "public"
+
+
+def test_tc_cfg_105_load_gaussdb_config_accepts_hash_and_dollar_in_schema_name():
+ cfg = load_gaussdb_config(
+ {
+ "host": "127.0.0.1",
+ "port": "19995",
+ "database": "postgres",
+ "user": "sqlbuilder",
+ "password": "fake-unit-password",
+ "schema": "ragflow#tenant$1",
+ }
+ )
+
+ assert cfg.schema == "ragflow#tenant$1"
+
+
+def test_tc_cfg_106_load_gaussdb_config_accepts_high_bit_schema_name():
+ cfg = load_gaussdb_config(
+ {
+ "host": "127.0.0.1",
+ "port": "19995",
+ "database": "postgres",
+ "user": "sqlbuilder",
+ "password": "fake-unit-password",
+ "schema": "租户_schema1",
+ }
+ )
+
+ assert cfg.schema == "租户_schema1"
+
+
+def test_tc_cfg_201_load_gaussdb_config_rejects_missing_host_without_leaking_values():
+ raw = _raw_config(database="private-db", user="private-user", password="private-password")
+ del raw["host"]
+
+ with pytest.raises(InvalidGaussDBConfig) as exc_info:
+ load_gaussdb_config(raw)
+
+ assert str(exc_info.value) == "missing gaussdb config field(s): host"
+ assert all(secret not in str(exc_info.value) for secret in ("private-db", "private-user", "private-password"))
+
+
[email protected]("field", ["port", "database", "user", "password"])
+def test_tc_cfg_202_load_gaussdb_config_rejects_each_missing_required_field(field):
+ raw = _raw_config(host="private-host", database="private-db", user="private-user", password="private-password")
+ del raw[field]
+
+ with pytest.raises(InvalidGaussDBConfig) as exc_info:
+ load_gaussdb_config(raw)
+
+ assert str(exc_info.value) == f"missing gaussdb config field(s): {field}"
+ assert all(secret not in str(exc_info.value) for secret in ("private-host", "private-db", "private-user", "private-password"))
+
+
+ ("field", "value"),
+ [
+ ("host", 127001),
+ ("database", 1234),
+ ("user", ""),
+ ("password", " "),
+ ],
+)
+def test_tc_cfg_202_load_gaussdb_config_rejects_invalid_required_string_fields(field, value):
+ with pytest.raises(InvalidGaussDBConfig, match=rf"^invalid gaussdb config field: {field}$"):
+ load_gaussdb_config(_raw_config(**{field: value}))
+
+
[email protected]("schema", ["1abc", "sch ema", "public;drop table x", "s" * 64])
+def test_tc_cfg_104_normalize_schema_rejects_unsafe_or_overlong_names(schema):
+ with pytest.raises(InvalidGaussDBConfig) as exc_info:
+ _normalize_schema(schema)
+
+ assert str(exc_info.value) == f"invalid gaussdb schema: {schema}"
+
+
+ ("port", "expected_message"),
+ [
+ (-1, "invalid gaussdb config field: port"),
+ (0, "invalid gaussdb config field: port"),
+ (65536, "invalid gaussdb config field: port"),
+ ],
+)
+def test_tc_cfg_203_load_gaussdb_config_rejects_invalid_port_values(port, expected_message):
+ with pytest.raises(InvalidGaussDBConfig) as exc_info:
+ load_gaussdb_config(_raw_config(port=port))
+
+ assert str(exc_info.value) == expected_message
+
+
[email protected]("port", ["not-a-port", "1.5", 1.5, 5432.0, True])
+def test_tc_cfg_204_load_gaussdb_config_rejects_non_integer_port(port):
+ with pytest.raises(InvalidGaussDBConfig) as exc_info:
+ load_gaussdb_config(_raw_config(port=port))
+
+ assert str(exc_info.value) == "invalid gaussdb config field: port"
+
+
+def test_tc_cfg_205_load_gaussdb_config_rejects_explicit_and_default_empty_config(monkeypatch):
+ expected = "missing gaussdb config field(s): host, port, database, user, password"
+
+ with pytest.raises(InvalidGaussDBConfig) as raw_empty_exc:
+ load_gaussdb_config({})
+
+ get_base_config = Mock(return_value={})
+ _install_runtime_settings(monkeypatch, {}, get_base_config)
+ with pytest.raises(InvalidGaussDBConfig) as default_exc:
+ load_gaussdb_config(None)
+
+ assert str(raw_empty_exc.value) == expected
+ assert str(default_exc.value) == expected
+ get_base_config.assert_called_once_with("gaussdb", {})
+
+
+def test_tc_cfg_005_load_gaussdb_config_without_raw_prefers_gaussdb_block(monkeypatch):
+ get_base_config = Mock(side_effect=AssertionError("fallback used"))
+ _install_runtime_settings(
+ monkeypatch,
+ {
+ "host": "gaussdb.local",
+ "port": "19995",
+ "database": "postgres",
+ "user": "sqlbuilder",
+ "password": "fake-unit-password",
+ },
+ get_base_config,
+ )
+
+ cfg = load_gaussdb_config()
+
+ assert cfg == GaussDBConfig(
+ host="gaussdb.local",
+ port=19995,
+ database="postgres",
+ user="sqlbuilder",
+ password="fake-unit-password",
+ schema="public",
+ )
+ get_base_config.assert_not_called()
+
+
+def test_tc_cfg_006_load_gaussdb_config_without_raw_falls_back_to_base_config(monkeypatch):
+ get_base_config = Mock(
+ return_value={
+ "host": "gaussdb.local",
+ "port": "19995",
+ "database": "postgres",
+ "user": "sqlbuilder",
+ "password": "fake-unit-password",
+ "schema": "ragflow_gaussdb_docengine_it",
+ }
+ )
+ _install_runtime_settings(monkeypatch, None, get_base_config)
+
+ cfg = load_gaussdb_config()
+
+ assert cfg == GaussDBConfig(
+ host="gaussdb.local",
+ port=19995,
+ database="postgres",
+ user="sqlbuilder",
+ password="fake-unit-password",
+ schema="ragflow_gaussdb_docengine_it",
+ )
+ get_base_config.assert_called_once_with("gaussdb", {})
+
+
+def test_tc_cfg_301_pool_initializes_driver_with_complete_connection_options(monkeypatch):
+ created = {}
+ sentinel_pool = object()
+
+ def fake_threaded_pool(*args, **kwargs):
+ created["args"] = args
+ created["kwargs"] = kwargs
+ return sentinel_pool
+
+ monkeypatch.setattr(pool_mod.psycopg2_pool, "ThreadedConnectionPool", fake_threaded_pool)
+ cfg = GaussDBConfig("h", 5432, "d", "u", "p", "zlw")
+
+ pool = GaussDBConnectionPool(cfg, minconn=2, maxconn=4)
+
+ assert pool._pool is sentinel_pool
+ assert pool.resolved_schema == "zlw"
+ assert created["args"] == (2, 4)
+ assert created["kwargs"] == {
+ "host": "h",
+ "port": 5432,
+ "dbname": "d",
+ "user": "u",
+ "password": "p",
+ "options": ("-c search_path=zlw -c client_encoding=UTF8 -c default_transaction_read_only=off"),
+ }
+
+
+def test_tc_cfg_310_gaussdb_config_and_pool_do_not_read_ssl_timeout_or_pool_tuning(monkeypatch):
+ created = {}
+ sentinel_pool = object()
+
+ def fake_threaded_pool(*args, **kwargs):
+ created["args"] = args
+ created["kwargs"] = kwargs
+ return sentinel_pool
+
+ raw = _raw_config(
+ ssl=True,
+ sslmode="require",
+ timeout=30,
+ connect_timeout=30,
+ minconn=4,
+ maxconn=32,
+ )
+ cfg = load_gaussdb_config(raw)
+
+ assert list(GaussDBConfig.__dataclass_fields__) == ["host", "port", "database", "user", "password", "schema"]
+ for attr in ("ssl", "sslmode", "timeout", "connect_timeout", "minconn", "maxconn"):
+ with pytest.raises(AttributeError, match=attr):
+ getattr(cfg, attr)
+
+ monkeypatch.setattr(pool_mod.psycopg2_pool, "ThreadedConnectionPool", fake_threaded_pool)
+ pool = GaussDBConnectionPool(cfg, minconn=1, maxconn=8)
+
+ assert pool._pool is sentinel_pool
+ assert created["args"] == (1, 8)
+ assert set(created["kwargs"]) == {"host", "port", "dbname", "user", "password", "options"}
+ assert created["kwargs"]["options"] == ("-c search_path=public -c client_encoding=UTF8 -c default_transaction_read_only=off")
+
+
+def test_tc_cfg_320_lazy_gaussdb_pool_reuses_one_pool_and_resets_after_close(monkeypatch):
+ created = []
+
+ class SharedPool:
+ masked_uri = "u@h:5432/d?schema=public"
+
+ def __init__(self):
+ self.closed = 0
+
+ def close_all(self):
+ self.closed += 1
+
+ def create_pool():
+ pool = SharedPool()
+ created.append(pool)
+ return pool
+
+ monkeypatch.setattr(pool_mod, "GaussDBConnectionPool", create_pool)
+ lazy_pool = LazyGaussDBConnectionPool()
+
+ first = lazy_pool.get_pool()
+ assert lazy_pool.get_pool() is first
+ assert lazy_pool.masked_uri == first.masked_uri
+ assert created == [first]
+
+ lazy_pool.close_all()
+ assert first.closed == 1
+ second = lazy_pool.get_pool()
+ assert second is not first
+ assert created == [first, second]
+
+
+def test_tc_cfg_404_pool_auth_failure_prevents_adapter_write(monkeypatch):
+ def fake_threaded_pool(*_args, **_kwargs):
+ raise RuntimeError("password authentication failed")
+
+ monkeypatch.setattr(pool_mod.psycopg2_pool, "ThreadedConnectionPool", fake_threaded_pool)
+ cfg = GaussDBConfig("h", 5432, "d", "u", "p", "public")
+
+ with pytest.raises(GaussDBAuthenticationError) as exc_info:
+ GaussDBConnectionPool(cfg)
+
+ assert str(exc_info.value) == "password authentication failed"
+
+ from common.doc_store.gaussdb_conn_base import GaussDBDDLBuilder
+ from rag.utils.gaussdb_conn import GaussDBConnection
+
+ connection = GaussDBConnection.__new__(GaussDBConnection)
+ connection.ddl = GaussDBDDLBuilder(schema="public")
+ connection.schema = "public"
+ connection.pool = Mock()
+ connection.pool.get_conn.side_effect = GaussDBAuthenticationError("auth failed")
+
+ assert connection.update({"id": "c1"}, {"pagerank_fea": 1}, "ragflow_tenant", "kb1") is False
+ connection.pool.get_conn.assert_called_once_with()
+ connection.pool.put_conn.assert_not_called()
+ connection.pool.execute.assert_not_called()
+ connection.pool.commit.assert_not_called()
+ connection.pool.rollback.assert_not_called()
+
+
+ "message",
+ [
+ 'FATAL: password authentication failed for user "test"',
+ "ERROR: invalid authentication method",
+ "invalid username or password",
+ ],
+)
+def test_tc_cfg_401_classify_gaussdb_exception_maps_authentication_errors(message):
+ classified = classify_gaussdb_exception(RuntimeError(message))
+
+ assert type(classified) is GaussDBAuthenticationError
+ assert str(classified) == message
+
+
[email protected]("message", ["ERROR: permission denied for schema public", "ERROR: privilege not granted"])
+def test_tc_cfg_402_classify_gaussdb_exception_maps_permission_errors(message):
+ classified = classify_gaussdb_exception(RuntimeError(message))
+
+ assert type(classified) is GaussDBPermissionError
+ assert str(classified) == message
+
+
[email protected]("message", ["connection timeout", "timeout expired", "network error", "generic connection error"])
+def test_tc_cfg_403_classify_gaussdb_exception_maps_timeout_and_generic_errors(message):
+ classified = classify_gaussdb_exception(RuntimeError(message))
+
+ assert type(classified) is GaussDBConnectionError
+ assert str(classified) == message
+
+
+def test_tc_cfg_405_classify_gaussdb_exception_preserves_existing_classification():
+ existing = GaussDBPermissionError("already classified")
+
+ assert classify_gaussdb_exception(existing) is existing
+
+
+class FakeCursor:
+ def __init__(self, row):
+ self.row = row
+ self.executed = []
+ self.closed = False
+
+ def execute(self, sql, params=None):
+ self.executed.append((sql, params))
+
+ def fetchone(self):
+ return self.row
+
+ def close(self):
+ self.closed = True
+
+
+class FakeConnection:
+ def __init__(self, row):
+ self.cursor_obj = FakeCursor(row)
+ self.rollbacks = 0
+ self.closed = False
+
+ def cursor(self):
+ return self.cursor_obj
+
+ def rollback(self):
+ self.rollbacks += 1
+
+
+class FakePool:
+ def __init__(self, row=(True, True)):
+ self.conn = FakeConnection(row)
+ self.returned = []
+ self.closed = False
+
+ def getconn(self):
+ return self.conn
+
+ def putconn(self, conn, close=False):
+ if close:
+ conn.closed = True
+ self.returned.append(conn)
+
+ def closeall(self):
+ self.closed = True
+
+
+def _assert_schema_privilege_query(sql, params, user, schema):
+ assert " ".join(sql.split()) == ("SELECT has_schema_privilege(%s, %s, %s) AS has_usage, has_schema_privilege(%s, %s, %s) AS has_create")
+ assert params == (user, schema, "USAGE", user, schema, "CREATE")
+
+
+def test_tc_cfg_318_pool_check_schema_access_verifies_usage_and_create_privileges():
+ cfg = GaussDBConfig("db.example", 19995, "postgres", "sqlbuilder", "fake-unit-password", "ragflow_gaussdb_docengine_it")
+ fake_pool = FakePool(row=(True, True))
+ pool = GaussDBConnectionPool(cfg, pool=fake_pool)
+
+ pool.check_schema_access()
+
+ sql, params = fake_pool.conn.cursor_obj.executed[1]
+ _assert_schema_privilege_query(sql, params, "sqlbuilder", "ragflow_gaussdb_docengine_it")
+ assert fake_pool.returned == [fake_pool.conn]
+ assert fake_pool.conn.cursor_obj.closed is True
+ assert fake_pool.conn.rollbacks == 2
+
+
+def test_tc_cfg_308_pool_check_schema_access_rejects_missing_create_privilege():
+ cfg = GaussDBConfig("h", 5432, "d", "u", "p", "public")
+ fake_pool = FakePool(row=(True, False))
+ pool = GaussDBConnectionPool(cfg, pool=fake_pool)
+
+ with pytest.raises(GaussDBPermissionError) as exc_info:
+ pool.check_schema_access()
+
+ assert str(exc_info.value) == "GaussDB user u lacks CREATE on schema public"
+ sql, params = fake_pool.conn.cursor_obj.executed[1]
+ _assert_schema_privilege_query(sql, params, "u", "public")
+ assert fake_pool.conn.cursor_obj.closed is True
+ assert fake_pool.returned == [fake_pool.conn]
+ assert fake_pool.conn.rollbacks == 2
+
+
+def test_tc_cfg_307_pool_check_schema_access_rejects_missing_usage_privilege():
+ cfg = GaussDBConfig("h", 5432, "d", "u", "p", "public")
+ fake_pool = FakePool(row=(False, True))
+ pool = GaussDBConnectionPool(cfg, pool=fake_pool)
+
+ with pytest.raises(GaussDBPermissionError) as exc_info:
+ pool.check_schema_access()
+
+ assert str(exc_info.value) == "GaussDB user u lacks USAGE on schema public"
+ sql, params = fake_pool.conn.cursor_obj.executed[1]
+ _assert_schema_privilege_query(sql, params, "u", "public")
+ assert fake_pool.conn.cursor_obj.closed is True
+ assert fake_pool.returned == [fake_pool.conn]
+ assert fake_pool.conn.rollbacks == 2
+
+
+def test_tc_cfg_314_pool_check_schema_access_classifies_query_failure():
+ class FailingSchemaCursor(FakeCursor):
+ def execute(self, sql, params=None):
+ super().execute(sql, params)
+ if "has_schema_privilege" in sql:
+ raise RuntimeError("permission denied while checking schema")
+
+ class FailingSchemaConnection(FakeConnection):
+ def __init__(self):
+ super().__init__(row=None)
+ self.cursor_obj = FailingSchemaCursor(row=None)
+
+ cfg = GaussDBConfig("db.example", 19995, "postgres", "sqlbuilder", "fake-unit-password", "ragflow_gaussdb_docengine_it")
+ fake_pool = FakePool(row=(True, True))
+ fake_pool.conn = FailingSchemaConnection()
+ pool = GaussDBConnectionPool(cfg, pool=fake_pool)
+
+ with pytest.raises(GaussDBPermissionError) as exc_info:
+ pool.check_schema_access()
+
+ assert str(exc_info.value) == "permission denied while checking schema"
+ assert fake_pool.conn.cursor_obj.closed is True
+ assert fake_pool.returned == [fake_pool.conn]
+
+
+def test_tc_cfg_314_pool_check_schema_access_classifies_cursor_creation_failure():
+ class CursorFailsAfterValidationConnection(FakeConnection):
+ def __init__(self):
+ super().__init__(row=(True, True))
+ self.cursor_calls = 0
+
+ def cursor(self):
+ self.cursor_calls += 1
+ if self.cursor_calls == 1:
+ return self.cursor_obj
+ raise RuntimeError("network timeout before privilege check")
+
+ cfg = GaussDBConfig("db.example", 19995, "postgres", "sqlbuilder", "fake-unit-password", "ragflow_gaussdb_docengine_it")
+ fake_pool = FakePool(row=(True, True))
+ fake_pool.conn = CursorFailsAfterValidationConnection()
+ pool = GaussDBConnectionPool(cfg, pool=fake_pool)
+
+ with pytest.raises(GaussDBConnectionError) as exc_info:
+ pool.check_schema_access()
+
+ assert str(exc_info.value) == "network timeout before privilege check"
+ assert fake_pool.conn.cursor_obj.closed is True
+ assert fake_pool.returned == [fake_pool.conn]
+
+
+class FailingPingCursor(FakeCursor):
+ def execute(self, sql, params=None):
+ self.executed.append((sql, params))
+ if sql == "SELECT 1":
+ raise RuntimeError("SSL SYSCALL error: EOF detected")
+
+
+class SequencedPool:
+ def __init__(self):
+ self.dead_conn = FakeConnection(row=None)
+ self.dead_conn.cursor_obj = FailingPingCursor(row=None)
+ self.live_conn = FakeConnection(row=None)
+ self.returned = []
+ self.closed = []
+
+ def getconn(self):
+ if not self.returned:
+ return self.dead_conn
+ return self.live_conn
+
+ def putconn(self, conn, close=False):
+ self.returned.append(conn)
+ if close:
+ self.closed.append(conn)
+
+
+def test_tc_cfg_315_pool_get_conn_discards_stale_connection_and_retries_once():
+ cfg = GaussDBConfig("db.example", 19995, "postgres", "sqlbuilder", "fake-unit-password", "ragflow_gaussdb_docengine_it")
+ fake_pool = SequencedPool()
+ pool = GaussDBConnectionPool(cfg, pool=fake_pool)
+
+ conn = pool.get_conn()
+
+ assert conn is fake_pool.live_conn
+ assert fake_pool.closed == [fake_pool.dead_conn]
+ assert fake_pool.dead_conn.cursor_obj.closed is True
+ assert fake_pool.live_conn.cursor_obj.closed is True
+ assert fake_pool.live_conn.rollbacks == 1
+
+
+def test_tc_cfg_315_pool_get_conn_retries_checkout_failure_without_discarding_none():
+ class CheckoutFailureThenSuccessPool:
+ def __init__(self):
+ self.calls = 0
+ self.live_conn = FakeConnection(row=(1,))
+ self.returned = []
+
+ def getconn(self):
+ self.calls += 1
+ if self.calls == 1:
+ raise RuntimeError("pool temporarily exhausted")
+ return self.live_conn
+
+ def putconn(self, conn, close=False):
+ self.returned.append((conn, close))
+
+ cfg = GaussDBConfig("db.example", 19995, "postgres", "sqlbuilder", "fake-unit-password", "ragflow_gaussdb_docengine_it")
+ fake_pool = CheckoutFailureThenSuccessPool()
+ pool = GaussDBConnectionPool(cfg, pool=fake_pool)
+
+ conn = pool.get_conn()
+
+ assert conn is fake_pool.live_conn
+ assert fake_pool.calls == 2
+ assert fake_pool.returned == []
+
+
+def test_tc_cfg_304_pool_put_conn_rolls_back_before_returning_connection():
+ cfg = GaussDBConfig("db.example", 19995, "postgres", "sqlbuilder", "fake-unit-password", "ragflow_gaussdb_docengine_it")
+ fake_pool = FakePool(row=(True, True))
+ pool = GaussDBConnectionPool(cfg, pool=fake_pool)
+ conn = FakeConnection(row=None)
+
+ pool.put_conn(conn)
+
+ assert fake_pool.returned == [conn]
+ assert conn.rollbacks == 1
+ assert conn.closed is False
+
+
+class RollbackFailingConnection(FakeConnection):
+ def rollback(self):
+ self.rollbacks += 1
+ raise RuntimeError("connection lost")
+
+
+def test_tc_cfg_305_pool_put_conn_discards_connection_when_rollback_fails():
+ cfg = GaussDBConfig("db.example", 19995, "postgres", "sqlbuilder", "fake-unit-password", "ragflow_gaussdb_docengine_it")
+ fake_pool = FakePool(row=(True, True))
+ pool = GaussDBConnectionPool(cfg, pool=fake_pool)
+ conn = RollbackFailingConnection(row=None)
+
+ pool.put_conn(conn)
+
+ assert fake_pool.returned == [conn]
+ assert conn.closed is True
+ assert conn.rollbacks == 1
+
+
+def test_tc_cfg_316_pool_put_conn_ignores_discard_failure_after_rollback_failure():
+ class DiscardFailingPool(FakePool):
+ def putconn(self, conn, close=False):
+ raise RuntimeError("discard failed")
+
+ cfg = GaussDBConfig("db.example", 19995, "postgres", "sqlbuilder", "fake-unit-password", "ragflow_gaussdb_docengine_it")
+ pool = GaussDBConnectionPool(cfg, pool=DiscardFailingPool(row=(True, True)))
+ conn = RollbackFailingConnection(row=None)
+ discard = Mock(wraps=pool._discard_conn)
+ pool._discard_conn = discard
+
+ pool.put_conn(conn)
+
+ assert conn.rollbacks == 1
+ discard.assert_called_once_with(conn)
+ assert pool._pool.returned == []
+
+
+def test_tc_cfg_303_pool_get_conn_tries_twice_then_raises_closed_error():
+ class AlwaysClosedPool:
+ def __init__(self):
+ self.connections = []
+ self.discarded = []
+
+ def getconn(self):
+ conn = FakeConnection(row=None)
+ conn.closed = True
+ self.connections.append(conn)
+ return conn
+
+ def putconn(self, conn, close=False):
+ if close:
+ self.discarded.append(conn)
+
+ cfg = GaussDBConfig("db.example", 19995, "postgres", "sqlbuilder", "fake-unit-password", "ragflow_gaussdb_docengine_it")
+ fake_pool = AlwaysClosedPool()
+ pool = GaussDBConnectionPool(cfg, pool=fake_pool)
+
+ with pytest.raises(GaussDBConnectionError) as exc_info:
+ pool.get_conn()
+
+ assert str(exc_info.value) == "GaussDB connection is closed"
+ assert len(fake_pool.connections) == 2
+ assert fake_pool.discarded == fake_pool.connections
+
+
+def test_tc_cfg_314_pool_get_conn_classifies_cursor_creation_failure():
+ class CursorCreationFailureConnection(FakeConnection):
+ def cursor(self):
+ raise RuntimeError("network timeout before cursor")
+
+ class CursorCreationFailurePool:
+ def __init__(self):
+ self.conn = CursorCreationFailureConnection(row=None)
+ self.closed = []
+
+ def getconn(self):
+ return self.conn
+
+ def putconn(self, conn, close=False):
+ if close:
+ self.closed.append(conn)
+
+ cfg = GaussDBConfig("db.example", 19995, "postgres", "sqlbuilder", "fake-unit-password", "ragflow_gaussdb_docengine_it")
+ fake_pool = CursorCreationFailurePool()
+ pool = GaussDBConnectionPool(cfg, pool=fake_pool)
+
+ with pytest.raises(GaussDBConnectionError) as exc_info:
+ pool.get_conn()
+
+ assert str(exc_info.value) == "network timeout before cursor"
+ assert fake_pool.closed == [fake_pool.conn, fake_pool.conn]
+
+
+def test_tc_cfg_316_pool_put_conn_ignores_none_and_close_all_delegates():
+ cfg = GaussDBConfig("db.example", 19995, "postgres", "sqlbuilder", "fake-unit-password", "ragflow_gaussdb_docengine_it")
+ fake_pool = FakePool(row=(True, True))
+ pool = GaussDBConnectionPool(cfg, pool=fake_pool)
+
+ pool.put_conn(None)
+ pool.close_all()
+
+ assert fake_pool.returned == []
+ assert fake_pool.closed is True
+
+
+def test_tc_cfg_317_pool_fetch_one_executes_query_and_returns_connection():
+ cfg = GaussDBConfig("db.example", 19995, "postgres", "sqlbuilder", "fake-unit-password", "ragflow_gaussdb_docengine_it")
+ fake_pool = FakePool(row=("GaussDB",))
+ pool = GaussDBConnectionPool(cfg, pool=fake_pool)
+
+ row = pool.fetch_one("SELECT version()", ("arg",))
+
+ assert row == ("GaussDB",)
+ assert fake_pool.conn.cursor_obj.executed[-1] == ("SELECT version()", ("arg",))
+ assert fake_pool.conn.cursor_obj.closed is True
+ assert fake_pool.returned == [fake_pool.conn]
+
+
+def test_tc_cfg_317_pool_fetch_one_classifies_cursor_creation_failure_and_returns_connection():
+ class CursorFailsAfterValidationConnection(FakeConnection):
+ def __init__(self):
+ super().__init__(row=(True,))
+ self.cursor_calls = 0
+
+ def cursor(self):
+ self.cursor_calls += 1
+ if self.cursor_calls == 1:
+ return self.cursor_obj
+ raise RuntimeError("network timeout before query")
+
+ cfg = GaussDBConfig("db.example", 19995, "postgres", "sqlbuilder", "fake-unit-password", "ragflow_gaussdb_docengine_it")
+ fake_pool = FakePool(row=(True, True))
+ fake_pool.conn = CursorFailsAfterValidationConnection()
+ pool = GaussDBConnectionPool(cfg, pool=fake_pool)
+
+ with pytest.raises(GaussDBConnectionError) as exc_info:
+ pool.fetch_one("SELECT 1")
+
+ assert str(exc_info.value) == "network timeout before query"
+ assert fake_pool.conn.cursor_obj.closed is True
+ assert fake_pool.returned == [fake_pool.conn]
+
+
+def test_tc_cfg_317_pool_fetch_one_classifies_query_failure_and_cleans_up():
+ class FailingQueryCursor(FakeCursor):
+ def execute(self, sql, params=None):
+ super().execute(sql, params)
+ if sql != "SELECT 1":
+ raise RuntimeError("authentication failed while querying")
+
+ class FailingQueryConnection(FakeConnection):
+ def __init__(self):
+ super().__init__(row=None)
+ self.cursor_obj = FailingQueryCursor(row=None)
+
+ cfg = GaussDBConfig("db.example", 19995, "postgres", "sqlbuilder", "fake-unit-password", "ragflow_gaussdb_docengine_it")
+ fake_pool = FakePool(row=(True, True))
+ fake_pool.conn = FailingQueryConnection()
+ pool = GaussDBConnectionPool(cfg, pool=fake_pool)
+
+ with pytest.raises(GaussDBAuthenticationError) as exc_info:
+ pool.fetch_one("SELECT version()")
+
+ assert str(exc_info.value) == "authentication failed while querying"
+ assert fake_pool.conn.cursor_obj.closed is True
+ assert fake_pool.returned == [fake_pool.conn]
diff --git a/test/unit_test/rag/utils/test_gaussdb_ddl.py b/test/unit_test/rag/utils/test_gaussdb_ddl.py
new file mode 100644
index 0000000000..a8ec872284
--- /dev/null
+++ b/test/unit_test/rag/utils/test_gaussdb_ddl.py
@@ -0,0 +1,303 @@
+#
+# Copyright 2025 The InfiniFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import pytest
+
+from common.doc_store.gaussdb_conn_base import (
+ GaussDBDDLBuilder,
+ InvalidGaussDBObjectName,
+)
+
+
+def _normalize_ddl(ddl):
+ return " ".join(ddl.split())
+
+
[email protected]("identifier", ["ragflow_unit", "schema#tenant$1", "中文标识符"])
+def test_tc_wrt_901_identifier_helpers_return_exact_valid_and_quoted_names(identifier):
+ builder = GaussDBDDLBuilder(schema="test_schema")
+
+ assert builder.validate_identifier(identifier) == identifier
+ assert builder.quote_identifier(identifier) == f'"{identifier}"'
+ assert builder.qualified_name(identifier) == f'"test_schema"."{identifier}"'
+
+
+INVALID_IDENTIFIERS = ["", "a" * 64, "1abc", "a b", "ta ble", "kb_id;drop"]
+
+
+ "identifier",
+ INVALID_IDENTIFIERS,
+ ids=["empty", "overlong", "numeric-prefix", "embedded-space", "table-space", "sql-punctuation"],
+)
+def test_tc_wrt_008_index_name_rejects_invalid_identifiers(identifier):
+ builder = GaussDBDDLBuilder(schema="test_schema")
+
+ with pytest.raises(InvalidGaussDBObjectName) as exc_info:
+ builder.index_name(identifier, "chunk")
+
+ assert exc_info.type is InvalidGaussDBObjectName
+ assert exc_info.value.args == (identifier,)
+ assert str(exc_info.value) == identifier
+
+
+ "identifier",
+ INVALID_IDENTIFIERS,
+ ids=["empty", "overlong", "numeric-prefix", "embedded-space", "table-space", "sql-punctuation"],
+)
+def test_tc_wrt_902_validate_identifier_rejects_invalid_identifiers(identifier):
+ builder = GaussDBDDLBuilder(schema="test_schema")
+
+ with pytest.raises(InvalidGaussDBObjectName) as exc_info:
+ builder.validate_identifier(identifier)
+
+ assert exc_info.type is InvalidGaussDBObjectName
+ assert exc_info.value.args == (identifier,)
+ assert str(exc_info.value) == identifier
+
+
+def test_tc_wrt_903_chunk_table_ddl_matches_normalized_snapshot():
+ builder = GaussDBDDLBuilder(schema="test_schema")
+
+ ddl = builder.build_chunk_table_ddl("ragflow_unit")
+
+ expected = """CREATE TABLE IF NOT EXISTS "test_schema"."ragflow_unit" (
+ id VARCHAR(256) NOT NULL,
+ kb_id VARCHAR(256) NOT NULL,
+ doc_id VARCHAR(256),
+ docnm_kwd VARCHAR(256),
+ doc_type_kwd VARCHAR(256),
+ title_tks VARCHAR(256),
+ title_sm_tks VARCHAR(256),
+ content_with_weight TEXT,
+ content_ltks TEXT,
+ content_sm_ltks TEXT,
+ important_kwd JSONB,
+ important_tks TEXT,
+ question_kwd JSONB,
+ question_tks TEXT,
+ tag_kwd JSONB,
+ tag_feas JSONB,
+ available_int INTEGER DEFAULT 1 NOT NULL,
+ pagerank_fea INTEGER,
+ create_time VARCHAR(19),
+ create_timestamp_flt DOUBLE PRECISION,
+ img_id VARCHAR(128),
+ position_int JSONB,
+ page_num_int JSONB,
+ top_int JSONB,
+ metadata JSONB,
+ chunk_data JSONB,
+ extra JSONB,
+ _order_id INTEGER,
+ group_id VARCHAR(256),
+ mom_id VARCHAR(256),
+ knowledge_graph_kwd VARCHAR(256),
+ source_id JSONB,
+ entity_kwd VARCHAR(256),
+ entity_type_kwd VARCHAR(256),
+ from_entity_kwd VARCHAR(256),
+ to_entity_kwd VARCHAR(256),
+ weight_int INTEGER,
+ weight_flt DOUBLE PRECISION,
+ entities_kwd JSONB,
+ rank_flt DOUBLE PRECISION,
+ n_hop_with_weight TEXT,
+ removed_kwd VARCHAR(256) DEFAULT 'N',
+ raptor_kwd VARCHAR(256),
+ raptor_layer_int INTEGER,
+ PRIMARY KEY (kb_id, id)
+ ) WITH (storage_type=USTORE)"""
+ assert _normalize_ddl(ddl) == _normalize_ddl(expected)
+
+
+def test_tc_wrt_904_vector_column_ddls_match_exact_snapshot():
+ builder = GaussDBDDLBuilder(schema="test_schema")
+
+ statements = builder.build_vector_column_ddls("ragflow_unit", 1024)
+
+ assert statements == [
+ 'ALTER TABLE "test_schema"."ragflow_unit" ADD COLUMN IF NOT EXISTS q_1024_vec floatvector(1024) DEFAULT (array_fill(0, ARRAY[1024])::text::floatvector(1024))',
+ 'ALTER TABLE "test_schema"."ragflow_unit" ADD COLUMN IF NOT EXISTS q_1024_vec_valid BOOLEAN DEFAULT FALSE NOT NULL',
+ ]
+
+
+ ("dim", "diskann_options"),
+ [
+ (1, "subgraph_count=1"),
+ (4096, "subgraph_count=1, enable_vector_copy=false"),
+ ],
+ ids=["minimum", "maximum"],
+)
+def test_tc_wrt_004_validate_vector_dim_accepts_gaussdb_boundaries(dim, diskann_options):
+ builder = GaussDBDDLBuilder(schema="test_schema")
+
+ assert builder.validate_vector_dim(dim) == dim
+ assert builder.vector_column_name(dim) == f"q_{dim}_vec"
+ assert builder.vector_valid_column_name(dim) == f"q_{dim}_vec_valid"
+ assert builder.build_vector_column_ddls("ragflow_t1", dim) == [
+ f'ALTER TABLE "test_schema"."ragflow_t1" ADD COLUMN IF NOT EXISTS q_{dim}_vec floatvector({dim}) DEFAULT (array_fill(0, ARRAY[{dim}])::text::floatvector({dim}))',
+ f'ALTER TABLE "test_schema"."ragflow_t1" ADD COLUMN IF NOT EXISTS q_{dim}_vec_valid BOOLEAN DEFAULT FALSE NOT NULL',
+ ]
+ assert builder.build_diskann_index_ddl("ragflow_t1", dim) == (
+ f'CREATE INDEX IF NOT EXISTS "idx_gdb_ragflow_t1_q_{dim}_vec_diskann" ON "test_schema"."ragflow_t1" USING gsdiskann (q_{dim}_vec COSINE) WITH ({diskann_options})'
+ )
+
+
[email protected]("dim", [0, -1], ids=["zero", "negative"])
+def test_tc_wrt_005_validate_vector_dim_rejects_non_positive_dimensions(dim):
+ builder = GaussDBDDLBuilder(schema="public")
+
+ with pytest.raises(ValueError) as exc_info:
+ builder.validate_vector_dim(dim)
+
+ assert str(exc_info.value) == "vector dimension must be positive"
+
+
+def test_tc_wrt_006_vector_size_above_gaussdb_limit_is_rejected():
+ builder = GaussDBDDLBuilder(schema="public")
+
+ with pytest.raises(ValueError) as exc_info:
+ builder.validate_vector_dim(4097)
+
+ assert str(exc_info.value) == "GaussDB floatvector dimensions cannot exceed 4096"
+
+
+ "dim",
+ [
+ "not-int",
+ 1024.5,
+ ],
+ ids=["non-numeric", "float"],
+)
+def test_tc_wrt_007_non_integer_vector_size_raises_exact_error(dim):
+ builder = GaussDBDDLBuilder(schema="test_schema")
+
+ with pytest.raises(ValueError) as exc_info:
+ builder.validate_vector_dim(dim)
+
+ assert str(exc_info.value) == "vector dimension must be an integer"
+
+
+ ("dim", "expected_options"),
+ [
+ (768, "subgraph_count=1"),
+ (1024, "subgraph_count=1"),
+ (1025, "subgraph_count=1, enable_vector_copy=false"),
+ (3072, "subgraph_count=1, enable_vector_copy=false"),
+ ],
+ ids=["below-threshold", "threshold", "above-threshold", "high-dimension"],
+)
+def test_tc_wrt_009_diskann_dimension_matrix_matches_exact_ddl(dim, expected_options):
+ builder = GaussDBDDLBuilder(schema="public")
+
+ ddl = builder.build_diskann_index_ddl("ragflow_tenant_a", dim)
+
+ assert ddl == (f'CREATE INDEX IF NOT EXISTS "idx_gdb_ragflow_tenant_a_q_{dim}_vec_diskann" ON "public"."ragflow_tenant_a" USING gsdiskann (q_{dim}_vec COSINE) WITH ({expected_options})')
+
+
+def test_tc_wrt_909_advisory_lock_sql_is_exact_and_parameterized():
+ builder = GaussDBDDLBuilder(schema="test_schema")
+
+ sql, params = builder.build_advisory_lock_sql("create_idx:schema:table")
+
+ assert sql == "SELECT pg_advisory_xact_lock(hashtext(%s))"
+ assert params == ["create_idx:schema:table"]
+
+
+def test_tc_wrt_911_long_table_identifier_produces_bounded_distinct_index_names():
+ builder = GaussDBDDLBuilder(schema="test_schema")
+ table = "ragflow_" + "a" * 32
+ index_columns = ("doc_id", "available_int", "knowledge_graph_kwd", "entity_type_kwd", "removed_kwd")
+ suffixes = (*index_columns, "q_3072_vec_diskann")
+
+ expected_names = [
+ "idx_gdb_ragflow_aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa_doc_id",
+ "idx_gdb_ragflow_aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa_available_int",
+ "idx_gdb_ragflow_aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa_kno_79293b137f",
+ "idx_gdb_ragflow_aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa_ent_2da2f5b85e",
+ "idx_gdb_ragflow_aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa_removed_kwd",
+ "idx_gdb_ragflow_aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa_q_3_83afb848d3",
+ ]
+ names = [builder.index_name(table, suffix) for suffix in suffixes]
+ repeated_names = [builder.index_name(table, suffix) for suffix in suffixes]
+ regular_ddls = builder.build_regular_index_ddls(table)
+ diskann_ddl = builder.build_diskann_index_ddl(table, 3072)
+
+ assert names == expected_names
+ assert repeated_names == expected_names
+ assert len(set(names)) == len(suffixes)
+ assert all(len(name) <= 63 for name in names)
+ assert all(name.startswith("idx_gdb_ragflow_") for name in names)
+ assert len(regular_ddls) == 5
+ for column, index_name, statement in zip(index_columns, expected_names[:5], regular_ddls, strict=True):
+ assert statement == (f'CREATE INDEX IF NOT EXISTS "{index_name}" ON "test_schema"."{table}" ({column})')
+ assert diskann_ddl == (f'CREATE INDEX IF NOT EXISTS "{expected_names[-1]}" ON "test_schema"."{table}" USING gsdiskann (q_3072_vec COSINE) WITH (subgraph_count=1, enable_vector_copy=false)')
+
+
+def test_tc_wrt_912_doc_meta_table_ddl_uses_id_pk_ustore_and_kb_index():
+ builder = GaussDBDDLBuilder(schema="public")
+
+ statements = builder.build_doc_meta_table_ddls("ragflow_doc_meta_tenant_a")
+
+ assert len(statements) == 2
+ assert [_normalize_ddl(statement) for statement in statements] == [
+ _normalize_ddl(
+ """CREATE TABLE IF NOT EXISTS "public"."ragflow_doc_meta_tenant_a" (
+ id VARCHAR(256) NOT NULL,
+ kb_id VARCHAR(256) NOT NULL,
+ meta_fields JSONB,
+ PRIMARY KEY (id)
+ ) WITH (storage_type=USTORE)"""
+ ),
+ 'CREATE INDEX IF NOT EXISTS "idx_gdb_ragflow_doc_meta_tenant_a_kb_id" ON "public"."ragflow_doc_meta_tenant_a" (kb_id)',
+ ]
+ assert "PRIMARY KEY (kb_id, id)" not in _normalize_ddl(statements[0])
+
+
+def test_tc_wrt_913_regular_index_ddls_match_exact_column_matrix():
+ builder = GaussDBDDLBuilder(schema="public")
+
+ statements = builder.build_regular_index_ddls("ragflow_tenant_a")
+
+ assert statements == [
+ f'CREATE INDEX IF NOT EXISTS "idx_gdb_ragflow_tenant_a_{column}" ON "public"."ragflow_tenant_a" ({column})'
+ for column in ("doc_id", "available_int", "knowledge_graph_kwd", "entity_type_kwd", "removed_kwd")
+ ]
+
+
+def test_tc_wrt_914_fulltext_ugin_ddl_matches_normalized_snapshot():
+ builder = GaussDBDDLBuilder(schema="public")
+
+ simple_ddl = builder.build_fulltext_ugin_ddl("ragflow_tenant_a")
+ ngram_ddl = builder.build_ngram_fulltext_ugin_ddl("ragflow_tenant_a")
+
+ expected_simple = """CREATE INDEX IF NOT EXISTS "idx_gdb_ragflow_tenant_a_fts_all"
+ ON "public"."ragflow_tenant_a"
+ USING ugin(to_tsvector('simple', coalesce(title_tks, ' ') || ' ' || coalesce(title_sm_tks, ' ') || ' ' ||
+ coalesce(important_tks, ' ') || ' ' || coalesce(question_tks, ' ') || ' ' || coalesce(content_ltks, ' ') || ' ' ||
+ coalesce(content_sm_ltks, ' ')))"""
+ expected_ngram = """CREATE INDEX IF NOT EXISTS "idx_gdb_ragflow_tenant_a_fts_all_ngram"
+ ON "public"."ragflow_tenant_a"
+ USING ugin(to_tsvector('ngram', coalesce(title_tks, ' ') || ' ' || coalesce(title_sm_tks, ' ') || ' ' ||
+ coalesce(important_tks, ' ') || ' ' || coalesce(question_tks, ' ') || ' ' || coalesce(content_ltks, ' ') || ' ' ||
+ coalesce(content_sm_ltks, ' ')))"""
+ assert _normalize_ddl(simple_ddl) == _normalize_ddl(expected_simple)
+ assert _normalize_ddl(ngram_ddl) == _normalize_ddl(expected_ngram)
diff --git a/test/unit_test/rag/utils/test_gaussdb_metadata_filter.py b/test/unit_test/rag/utils/test_gaussdb_metadata_filter.py
new file mode 100644
index 0000000000..499d78b01a
--- /dev/null
+++ b/test/unit_test/rag/utils/test_gaussdb_metadata_filter.py
@@ -0,0 +1,1155 @@
+#
+# Copyright 2025 The InfiniFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+import pytest
+
+from common import metadata_gaussdb_filter as gaussdb_filter
+from common.metadata_gaussdb_filter import (
+ GaussDBMetaFilterTranslator,
+ SUPPORTED_OPERATORS,
+ UnsupportedGaussDBMetaFilter,
+ build_gaussdb_filter,
+ fetch_gaussdb_metadata_doc_ids,
+ is_pushdown_supported,
+ jsonb_path_literal,
+ normalize_gaussdb_meta_operator,
+ plan_pushdown,
+)
+
+OP_NE = "\u2260"
+OP_GE = "\u2265"
+OP_LE = "\u2264"
+
+
+def _assert_filter(filters, logic, expected_sql, expected_params):
+ sql, params = build_gaussdb_filter(filters, logic)
+
+ assert sql == expected_sql
+ assert params == expected_params
+ return sql
+
+
+def _assert_rejected_filter(flt, expected_reason, logic="and", hidden_inputs=()):
+ with pytest.raises(UnsupportedGaussDBMetaFilter) as exc_info:
+ build_gaussdb_filter([flt], logic)
+
+ assert exc_info.value.reason == expected_reason
+ for hidden_input in hidden_inputs:
+ assert hidden_input not in exc_info.value.reason
+
+
+def test_tc_mgf_002_multiple_filters_join_with_and():
+ sql = _assert_filter(
+ [
+ {"key": "status", "op": "=", "value": "active"},
+ {"key": "amount", "op": ">", "value": 100},
+ ],
+ "and",
+ "(lower(meta_fields #>> '{status}') = %s OR jsonb_exists(meta_fields #> '{status}', %s)) AND "
+ "(CASE WHEN meta_fields #>> '{amount}' ~ '^-?[0-9]+(\\.[0-9]+)?$' "
+ "THEN (meta_fields #>> '{amount}')::DOUBLE PRECISION > %s ELSE FALSE END)",
+ ["active", "active", 100],
+ )
+
+ assert "active" not in sql
+ assert "100" not in sql
+
+
+def test_tc_mgf_003_multiple_filters_join_with_or():
+ _assert_filter(
+ [
+ {"key": "status", "op": "=", "value": "a"},
+ {"key": "status", "op": "=", "value": "b"},
+ ],
+ "or",
+ "(lower(meta_fields #>> '{status}') = %s OR jsonb_exists(meta_fields #> '{status}', %s)) OR (lower(meta_fields #>> '{status}') = %s OR jsonb_exists(meta_fields #> '{status}', %s))",
+ ["a", "a", "b", "b"],
+ )
+
+
+def test_tc_mgf_004_empty_filter_list_returns_tautology():
+ _assert_filter([], "and", "1=1", [])
+
+
+def test_tc_mgf_005_nested_key_path_uses_jsonb_path_literal():
+ sql = _assert_filter(
+ [{"key": "profile.name", "op": "=", "value": "Alice"}],
+ "and",
+ "(lower(meta_fields #>> '{profile,name}') = %s OR jsonb_exists(meta_fields #> '{profile,name}', %s))",
+ ["alice", "alice"],
+ )
+
+ assert jsonb_path_literal("profile.name") == "'{profile,name}'"
+ assert "Alice" not in sql
+ assert "alice" not in sql
+
+
+def test_tc_mgf_006_custom_jsonb_column_is_used_for_translation():
+ translated = GaussDBMetaFilterTranslator(jsonb_column="chunk_data").translate({"key": "amount", "op": "=", "value": 5})
+
+ assert translated.sql == "chunk_data #> '{amount}' @> %s::jsonb"
+ assert translated.params == ["5"]
+ assert "meta_fields" not in translated.sql
+
+
+def test_tc_mgf_101_equal_string_translates_to_scalar_or_array_match():
+ sql = _assert_filter(
+ [{"key": "status", "op": "=", "value": "active"}],
+ "and",
+ "(lower(meta_fields #>> '{status}') = %s OR jsonb_exists(meta_fields #> '{status}', %s))",
+ ["active", "active"],
+ )
+
+ assert "active" not in sql
+
+
+def test_tc_mgf_102_not_equal_string_requires_existing_key_and_negated_match():
+ sql = _assert_filter(
+ [{"key": "status", "op": OP_NE, "value": "active"}],
+ "and",
+ "((meta_fields #> '{status}') IS NOT NULL AND (lower(meta_fields #>> '{status}') = %s OR jsonb_exists(meta_fields #> '{status}', %s)) IS NOT TRUE)",
+ ["active", "active"],
+ )
+
+ assert "active" not in sql
+
+
+def test_tc_mgf_103_greater_than_number_uses_numeric_cast():
+ sql = _assert_filter(
+ [{"key": "amount", "op": ">", "value": 100}],
+ "and",
+ "(CASE WHEN meta_fields #>> '{amount}' ~ '^-?[0-9]+(\\.[0-9]+)?$' THEN (meta_fields #>> '{amount}')::DOUBLE PRECISION > %s ELSE FALSE END)",
+ [100],
+ )
+
+ assert "100" not in sql
+
+
+def test_tc_mgf_104_numeric_comparisons_use_canonical_sql_operators():
+ cases = [
+ (OP_GE, ">="),
+ ("<", "<"),
+ (OP_LE, "<="),
+ ]
+
+ for filter_op, sql_op in cases:
+ sql = _assert_filter(
+ [{"key": "amount", "op": filter_op, "value": 50}],
+ "and",
+ f"(CASE WHEN meta_fields #>> '{{amount}}' ~ '^-?[0-9]+(\\.[0-9]+)?$' THEN (meta_fields #>> '{{amount}}')::DOUBLE PRECISION {sql_op} %s ELSE FALSE END)",
+ [50],
+ )
+ assert "50" not in sql
+
+
+def test_tc_mgf_105_date_iso_prefix_uses_exact_timestamp_predicate():
+ sql = _assert_filter(
+ [{"key": "dt", "op": ">", "value": "2026-07"}],
+ "and",
+ "(CASE "
+ "WHEN meta_fields #>> '{dt}' ~ '^[0-9]{4}-[0-9]{2}$' "
+ "THEN to_timestamp(meta_fields #>> '{dt}' || '-01 00:00:00', 'YYYY-MM-DD HH24:MI:SS') "
+ "WHEN meta_fields #>> '{dt}' ~ '^[0-9]{4}-[0-9]{2}-[0-9]{2}$' "
+ "THEN to_timestamp(meta_fields #>> '{dt}' || ' 00:00:00', 'YYYY-MM-DD HH24:MI:SS') "
+ "WHEN meta_fields #>> '{dt}' ~ '^[0-9]{4}-[0-9]{2}-[0-9]{2}[T ][0-9]{2}$' "
+ "THEN to_timestamp(replace(meta_fields #>> '{dt}', 'T', ' ') || ':00:00', 'YYYY-MM-DD HH24:MI:SS') "
+ "WHEN meta_fields #>> '{dt}' ~ '^[0-9]{4}-[0-9]{2}-[0-9]{2}[T ][0-9]{2}:[0-9]{2}$' "
+ "THEN to_timestamp(replace(meta_fields #>> '{dt}', 'T', ' ') || ':00', 'YYYY-MM-DD HH24:MI:SS') "
+ "WHEN meta_fields #>> '{dt}' ~ '^[0-9]{4}-[0-9]{2}-[0-9]{2}[T ][0-9]{2}:[0-9]{2}:[0-9]{2}$' "
+ "THEN to_timestamp(replace(meta_fields #>> '{dt}', 'T', ' '), 'YYYY-MM-DD HH24:MI:SS') "
+ "END > to_timestamp(%s, 'YYYY-MM-DD HH24:MI:SS'))",
+ ["2026-07-01 00:00:00"],
+ )
+
+ assert gaussdb_filter.normalize_datetime_prefix("2026-07") == "2026-07-01 00:00:00"
+ assert "2026-07" not in sql
+
+
+def test_tc_mgf_106_contains_string_matches_array_member_or_substring():
+ sql = _assert_filter(
+ [{"key": "cust", "op": "contains", "value": "Acme"}],
+ "and",
+ "(jsonb_exists(meta_fields #> '{cust}', %s) OR lower(meta_fields #>> '{cust}') LIKE %s ESCAPE '\\')",
+ ["acme", "%acme%"],
+ )
+
+ assert "Acme" not in sql
+ assert "acme" not in sql
+
+
+def test_tc_mgf_107_not_contains_requires_existing_key_and_negated_contains():
+ sql = _assert_filter(
+ [{"key": "cust", "op": "not contains", "value": "Acme"}],
+ "and",
+ "((meta_fields #> '{cust}') IS NOT NULL AND (jsonb_exists(meta_fields #> '{cust}', %s) OR lower(meta_fields #>> '{cust}') LIKE %s ESCAPE '\\') IS NOT TRUE)",
+ ["acme", "%acme%"],
+ )
+
+ assert "Acme" not in sql
+ assert "acme" not in sql
+
+
+def test_tc_mgf_108_start_with_uses_escaped_like_prefix():
+ sql = _assert_filter(
+ [{"key": "cust", "op": "start with", "value": "Acme"}],
+ "and",
+ "(lower(meta_fields #>> '{cust}') LIKE %s ESCAPE '\\')",
+ ["acme%"],
+ )
+
+ assert "Acme" not in sql
+ assert "acme" not in sql
+
+
+def test_tc_mgf_109_end_with_uses_escaped_like_suffix():
+ sql = _assert_filter(
+ [{"key": "cust", "op": "end with", "value": "Corp"}],
+ "and",
+ "(lower(meta_fields #>> '{cust}') LIKE %s ESCAPE '\\')",
+ ["%corp"],
+ )
+
+ assert "Corp" not in sql
+ assert "corp" not in sql
+
+
+def test_tc_mgf_110_in_operator_expands_members_with_or():
+ _assert_filter(
+ [{"key": "status", "op": "in", "value": ["a", "b"]}],
+ "and",
+ "((lower(meta_fields #>> '{status}') = %s OR jsonb_exists(meta_fields #> '{status}', %s)) OR (lower(meta_fields #>> '{status}') = %s OR jsonb_exists(meta_fields #> '{status}', %s)))",
+ ["a", "a", "b", "b"],
+ )
+
+
+def test_tc_mgf_111_not_in_operator_requires_key_and_ands_members():
+ _assert_filter(
+ [{"key": "status", "op": "not in", "value": ["a", "b"]}],
+ "and",
+ "((meta_fields #> '{status}') IS NOT NULL AND "
+ "(lower(meta_fields #>> '{status}') = %s OR jsonb_exists(meta_fields #> '{status}', %s)) IS NOT TRUE AND "
+ "(lower(meta_fields #>> '{status}') = %s OR jsonb_exists(meta_fields #> '{status}', %s)) IS NOT TRUE)",
+ ["a", "a", "b", "b"],
+ )
+
+
+def test_tc_mgf_112_empty_operator_covers_all_empty_states_and_nested_missing_parent():
+ status_sql = _assert_filter(
+ [{"key": "status", "op": "empty", "value": None}],
+ "and",
+ "((meta_fields #> '{status}') IS NULL OR meta_fields #> '{status}' = 'null'::jsonb OR "
+ "meta_fields #> '{status}' = '\"\"'::jsonb OR meta_fields #> '{status}' = '[]'::jsonb OR "
+ "meta_fields #> '{status}' = '{}'::jsonb)",
+ [],
+ )
+ nested_sql = _assert_filter(
+ [{"key": "vendor.name", "op": "empty", "value": None}],
+ "and",
+ "((meta_fields #> '{vendor,name}') IS NULL OR "
+ "meta_fields #> '{vendor,name}' = 'null'::jsonb OR "
+ "meta_fields #> '{vendor,name}' = '\"\"'::jsonb OR "
+ "meta_fields #> '{vendor,name}' = '[]'::jsonb OR "
+ "meta_fields #> '{vendor,name}' = '{}'::jsonb)",
+ [],
+ )
+ for sql in (status_sql, nested_sql):
+ assert "= ''" not in sql
+ assert "<> ''" not in sql
+
+
+def test_tc_mgf_113_not_empty_operator_excludes_all_empty_states():
+ sql = _assert_filter(
+ [{"key": "status", "op": "not empty", "value": None}],
+ "and",
+ "((meta_fields #> '{status}') IS NOT NULL AND (meta_fields #> '{status}' = 'null'::jsonb) IS NOT TRUE AND "
+ "(meta_fields #> '{status}' = '\"\"'::jsonb) IS NOT TRUE AND "
+ "(meta_fields #> '{status}' = '[]'::jsonb) IS NOT TRUE AND "
+ "(meta_fields #> '{status}' = '{}'::jsonb) IS NOT TRUE)",
+ [],
+ )
+ assert "= ''" not in sql
+ assert "<> ''" not in sql
+
+
+def test_tc_mgf_201_equal_none_generates_json_null_equality():
+ _assert_filter(
+ [{"key": "status", "op": "=", "value": None}],
+ "and",
+ "((meta_fields #> '{status}') IS NOT NULL AND meta_fields #> '{status}' = 'null'::jsonb)",
+ [],
+ )
+ assert gaussdb_filter.coerce_scalar_value(None, {"key": "status", "op": "=", "value": None}) is None
+
+
+def test_tc_mgf_202_equal_empty_string_uses_jsonb_empty_string_literal():
+ sql = _assert_filter(
+ [{"key": "status", "op": "=", "value": ""}],
+ "and",
+ "(meta_fields #> '{status}' = '\"\"'::jsonb)",
+ [],
+ )
+
+ assert "= ''" not in sql
+
+
+def test_tc_mgf_301_equal_integer_uses_jsonb_containment():
+ sql = _assert_filter(
+ [{"key": "amount", "op": "=", "value": 5}],
+ "and",
+ "(meta_fields #> '{amount}' @> %s::jsonb)",
+ ["5"],
+ )
+
+ assert gaussdb_filter.coerce_scalar_value(5, {"key": "amount", "op": "=", "value": 5}) == 5
+ assert gaussdb_filter.jsonb_param(5) == "5"
+ assert "5" not in sql
+
+
+def test_tc_mgf_302_equal_boolean_uses_jsonb_containment():
+ sql = _assert_filter(
+ [{"key": "flag", "op": "=", "value": True}],
+ "and",
+ "(meta_fields #> '{flag}' @> %s::jsonb)",
+ ["true"],
+ )
+
+ assert "true" not in sql
+ assert "True" not in sql
+
+
+def test_tc_mgf_303_equal_numeric_string_uses_jsonb_numeric_containment():
+ flt = {"key": "n", "op": "=", "value": "123"}
+ sql = _assert_filter(
+ [flt],
+ "and",
+ "(meta_fields #> '{n}' @> %s::jsonb)",
+ ["123"],
+ )
+
+ assert gaussdb_filter.coerce_scalar_value("123", flt) == 123
+ assert "123" not in sql
+
+
+def test_tc_mgf_304_equal_python_literals_differ_from_lowercase_json_words():
+ cases = [
+ ("True", True, "(meta_fields #> '{n}' @> %s::jsonb)", ["true"]),
+ ("False", False, "(meta_fields #> '{n}' @> %s::jsonb)", ["false"]),
+ (
+ "None",
+ None,
+ "((meta_fields #> '{n}') IS NOT NULL AND meta_fields #> '{n}' = 'null'::jsonb)",
+ [],
+ ),
+ (
+ "true",
+ "true",
+ "(lower(meta_fields #>> '{n}') = %s OR jsonb_exists(meta_fields #> '{n}', %s))",
+ ["true", "true"],
+ ),
+ (
+ "null",
+ "null",
+ "(lower(meta_fields #>> '{n}') = %s OR jsonb_exists(meta_fields #> '{n}', %s))",
+ ["null", "null"],
+ ),
+ ]
+
+ for value, expected_value, expected_sql, expected_params in cases:
+ flt = {"key": "n", "op": "=", "value": value}
+ sql = _assert_filter([flt], "and", expected_sql, expected_params)
+ coerced = gaussdb_filter.coerce_scalar_value(value, flt)
+
+ if isinstance(expected_value, str):
+ assert coerced == expected_value
+ else:
+ assert coerced is expected_value
+ assert value not in sql
+
+
+def test_tc_mgf_305_range_accepts_negative_number():
+ sql = _assert_filter(
+ [{"key": "amount", "op": ">", "value": -10}],
+ "and",
+ "(CASE WHEN meta_fields #>> '{amount}' ~ '^-?[0-9]+(\\.[0-9]+)?$' THEN (meta_fields #>> '{amount}')::DOUBLE PRECISION > %s ELSE FALSE END)",
+ [-10],
+ )
+
+ assert "-10" not in sql
+
+
+def test_tc_mgf_306_range_accepts_decimal_number():
+ sql = _assert_filter(
+ [{"key": "amount", "op": ">", "value": 1.5}],
+ "and",
+ "(CASE WHEN meta_fields #>> '{amount}' ~ '^-?[0-9]+(\\.[0-9]+)?$' THEN (meta_fields #>> '{amount}')::DOUBLE PRECISION > %s ELSE FALSE END)",
+ [1.5],
+ )
+
+ assert "1.5" not in sql
+
+
+def test_tc_mgf_307_range_numeric_strings_stay_on_numeric_branch():
+ for value, expected_param in (("100", 100), ("2026", 2026), ("01", 1), ("01.5", 1.5)):
+ flt = {"key": "amount", "op": ">", "value": value}
+ sql = _assert_filter(
+ [flt],
+ "and",
+ "(CASE WHEN meta_fields #>> '{amount}' ~ '^-?[0-9]+(\\.[0-9]+)?$' THEN (meta_fields #>> '{amount}')::DOUBLE PRECISION > %s ELSE FALSE END)",
+ [expected_param],
+ )
+ assert gaussdb_filter.coerce_range_value(value, flt) == ("number", expected_param)
+ assert "to_timestamp" not in sql
+ assert value not in sql
+
+
+def test_tc_mgf_308_date_iso_prefixes_normalize_to_exact_timestamp_parameters():
+ cases = [
+ ("2026-07", "2026-07-01 00:00:00"),
+ ("2026-07-08", "2026-07-08 00:00:00"),
+ ("2026-07-08T10", "2026-07-08 10:00:00"),
+ ("2026-07-08 10", "2026-07-08 10:00:00"),
+ ("2026-07-08T10:30", "2026-07-08 10:30:00"),
+ ("2026-07-08 10:30:45", "2026-07-08 10:30:45"),
+ ]
+
+ for value, expected_param in cases:
+ _assert_filter(
+ [{"key": "published_at", "op": ">", "value": value}],
+ "and",
+ "(CASE "
+ "WHEN meta_fields #>> '{published_at}' ~ '^[0-9]{4}-[0-9]{2}$' "
+ "THEN to_timestamp(meta_fields #>> '{published_at}' || '-01 00:00:00', 'YYYY-MM-DD HH24:MI:SS') "
+ "WHEN meta_fields #>> '{published_at}' ~ '^[0-9]{4}-[0-9]{2}-[0-9]{2}$' "
+ "THEN to_timestamp(meta_fields #>> '{published_at}' || ' 00:00:00', 'YYYY-MM-DD HH24:MI:SS') "
+ "WHEN meta_fields #>> '{published_at}' ~ '^[0-9]{4}-[0-9]{2}-[0-9]{2}[T ][0-9]{2}$' "
+ "THEN to_timestamp(replace(meta_fields #>> '{published_at}', 'T', ' ') || ':00:00', 'YYYY-MM-DD HH24:MI:SS') "
+ "WHEN meta_fields #>> '{published_at}' ~ '^[0-9]{4}-[0-9]{2}-[0-9]{2}[T ][0-9]{2}:[0-9]{2}$' "
+ "THEN to_timestamp(replace(meta_fields #>> '{published_at}', 'T', ' ') || ':00', 'YYYY-MM-DD HH24:MI:SS') "
+ "WHEN meta_fields #>> '{published_at}' ~ '^[0-9]{4}-[0-9]{2}-[0-9]{2}[T ][0-9]{2}:[0-9]{2}:[0-9]{2}$' "
+ "THEN to_timestamp(replace(meta_fields #>> '{published_at}', 'T', ' '), 'YYYY-MM-DD HH24:MI:SS') "
+ "END > to_timestamp(%s, 'YYYY-MM-DD HH24:MI:SS'))",
+ [expected_param],
+ )
+ assert gaussdb_filter.normalize_datetime_prefix(value) == expected_param
+
+
+def test_tc_mgf_309_invalid_datetime_prefix_is_rejected_as_range_value():
+ flt = {"key": "dt", "op": ">", "value": "2026-13-01"}
+
+ _assert_rejected_filter(
+ flt,
+ "unsupported range comparison value",
+ hidden_inputs=("2026-13-01",),
+ )
+
+
+def test_tc_mgf_310_in_operator_splits_comma_separated_string_members():
+ sql = _assert_filter(
+ [{"key": "status", "op": "in", "value": "a,b,c"}],
+ "and",
+ "((lower(meta_fields #>> '{status}') = %s OR jsonb_exists(meta_fields #> '{status}', %s)) OR "
+ "(lower(meta_fields #>> '{status}') = %s OR jsonb_exists(meta_fields #> '{status}', %s)) OR "
+ "(lower(meta_fields #>> '{status}') = %s OR jsonb_exists(meta_fields #> '{status}', %s)))",
+ ["a", "a", "b", "b", "c", "c"],
+ )
+
+ assert "a,b,c" not in sql
+
+
+def test_tc_mgf_311_in_operator_parses_json_array_string_members():
+ sql = _assert_filter(
+ [{"key": "n", "op": "in", "value": "[1,2]"}],
+ "and",
+ "((meta_fields #> '{n}' @> %s::jsonb) OR (meta_fields #> '{n}' @> %s::jsonb))",
+ ["1", "2"],
+ )
+
+ assert "[1,2]" not in sql
+
+
+def test_tc_mgf_312_equal_decimal_string_uses_jsonb_numeric_containment():
+ sql = _assert_filter(
+ [{"key": "n", "op": "=", "value": "1.5"}],
+ "and",
+ "(meta_fields #> '{n}' @> %s::jsonb)",
+ ["1.5"],
+ )
+
+ assert "1.5" not in sql
+
+
+def test_tc_mgf_313_in_operator_accepts_single_scalar_member():
+ sql = _assert_filter(
+ [{"key": "n", "op": "in", "value": 7}],
+ "and",
+ "((meta_fields #> '{n}' @> %s::jsonb))",
+ ["7"],
+ )
+
+ assert sql.count("@> %s::jsonb") == 1
+ assert "7" not in sql
+
+
+def test_tc_mgf_401_and_logic_joins_three_mixed_operators_in_order():
+ sql = _assert_filter(
+ [
+ {"key": "status", "op": "=", "value": "active"},
+ {"key": "amount", "op": ">", "value": 100},
+ {"key": "cust", "op": "contains", "value": "acme"},
+ ],
+ "and",
+ "(lower(meta_fields #>> '{status}') = %s OR jsonb_exists(meta_fields #> '{status}', %s)) AND "
+ "(CASE WHEN meta_fields #>> '{amount}' ~ '^-?[0-9]+(\\.[0-9]+)?$' "
+ "THEN (meta_fields #>> '{amount}')::DOUBLE PRECISION > %s ELSE FALSE END) AND "
+ "(jsonb_exists(meta_fields #> '{cust}', %s) OR "
+ "lower(meta_fields #>> '{cust}') LIKE %s ESCAPE '\\')",
+ ["active", "active", 100, "acme", "%acme%"],
+ )
+
+ assert "active" not in sql
+ assert "100" not in sql
+ assert "acme" not in sql
+
+
+def test_tc_mgf_402_or_logic_joins_three_filters_in_order():
+ _assert_filter(
+ [
+ {"key": "status", "op": "=", "value": "a"},
+ {"key": "status", "op": "=", "value": "b"},
+ {"key": "status", "op": "=", "value": "c"},
+ ],
+ "or",
+ "(lower(meta_fields #>> '{status}') = %s OR jsonb_exists(meta_fields #> '{status}', %s)) OR "
+ "(lower(meta_fields #>> '{status}') = %s OR jsonb_exists(meta_fields #> '{status}', %s)) OR "
+ "(lower(meta_fields #>> '{status}') = %s OR jsonb_exists(meta_fields #> '{status}', %s))",
+ ["a", "a", "b", "b", "c", "c"],
+ )
+
+
+def test_tc_mgf_403_and_logic_combines_not_in_with_positive_filter():
+ _assert_filter(
+ [
+ {"key": "status", "op": "not in", "value": ["a", "b"]},
+ {"key": "type", "op": "=", "value": "doc"},
+ ],
+ "and",
+ "((meta_fields #> '{status}') IS NOT NULL AND "
+ "(lower(meta_fields #>> '{status}') = %s OR jsonb_exists(meta_fields #> '{status}', %s)) IS NOT TRUE AND "
+ "(lower(meta_fields #>> '{status}') = %s OR jsonb_exists(meta_fields #> '{status}', %s)) IS NOT TRUE) AND "
+ "(lower(meta_fields #>> '{type}') = %s OR jsonb_exists(meta_fields #> '{type}', %s))",
+ ["a", "a", "b", "b", "doc", "doc"],
+ )
+
+
+def test_tc_mgf_404_nested_key_exists_and_jsonb_number_predicate_join_with_and():
+ sql = _assert_filter(
+ [
+ {"key": "a.b", "op": OP_NE, "value": "x"},
+ {"key": "a.c", "op": "=", "value": 1},
+ ],
+ "and",
+ "((meta_fields #> '{a,b}') IS NOT NULL AND (lower(meta_fields #>> '{a,b}') = %s OR jsonb_exists(meta_fields #> '{a,b}', %s)) IS NOT TRUE) AND (meta_fields #> '{a,c}' @> %s::jsonb)",
+ ["x", "x", "1"],
+ )
+
+ assert jsonb_path_literal("a.c") == "'{a,c}'"
+ assert "'x'" not in sql
+
+
+def test_tc_mgf_405_special_key_segments_are_encoded_without_sql_injection():
+ malicious_segment = "owner') OR TRUE --"
+ key = f"root.a,b.{malicious_segment}"
+ path = "'{root,\"a,b\",\"owner'') OR TRUE --\"}'"
+
+ sql = _assert_filter(
+ [{"key": key, "op": OP_NE, "value": "x"}],
+ "and",
+ f"((meta_fields #> {path}) IS NOT NULL AND (lower(meta_fields #>> {path}) = %s OR jsonb_exists(meta_fields #> {path}, %s)) IS NOT TRUE)",
+ ["x", "x"],
+ )
+
+ assert jsonb_path_literal(key) == path
+ assert malicious_segment not in sql
+ assert sql.count(path) == 3
+
+
+def test_tc_mgf_501_is_alias_translates_to_equal_sql():
+ sql = _assert_filter(
+ [{"key": "status", "op": "is", "value": "active"}],
+ "and",
+ "(lower(meta_fields #>> '{status}') = %s OR jsonb_exists(meta_fields #> '{status}', %s))",
+ ["active", "active"],
+ )
+
+ assert normalize_gaussdb_meta_operator("is") == "="
+ assert "active" not in sql
+
+
+def test_tc_mgf_502_not_equal_aliases_translate_to_identical_sql():
+ expected_sql = "((meta_fields #> '{status}') IS NOT NULL AND (lower(meta_fields #>> '{status}') = %s OR jsonb_exists(meta_fields #> '{status}', %s)) IS NOT TRUE)"
+
+ for op in ("is not", "not is", "!=", "<>"):
+ sql = _assert_filter(
+ [{"key": "status", "op": op, "value": "active"}],
+ "and",
+ expected_sql,
+ ["active", "active"],
+ )
+ assert normalize_gaussdb_meta_operator(op) == OP_NE
+ assert "active" not in sql
+
+
+def test_tc_mgf_503_range_aliases_normalize_and_translate_to_canonical_operators():
+ cases = [(">=", OP_GE, ">="), ("<=", OP_LE, "<=")]
+
+ for alias, canonical, sql_op in cases:
+ sql = _assert_filter(
+ [{"key": "amount", "op": alias, "value": 50}],
+ "and",
+ f"(CASE WHEN meta_fields #>> '{{amount}}' ~ '^-?[0-9]+(\\.[0-9]+)?$' THEN (meta_fields #>> '{{amount}}')::DOUBLE PRECISION {sql_op} %s ELSE FALSE END)",
+ [50],
+ )
+ assert normalize_gaussdb_meta_operator(alias) == canonical
+ assert "50" not in sql
+
+
+def test_tc_mgf_504_operator_normalization_tolerates_case_and_whitespace():
+ cases = [
+ (
+ " IS NOT ",
+ "((meta_fields #> '{status}') IS NOT NULL AND (lower(meta_fields #>> '{status}') = %s OR jsonb_exists(meta_fields #> '{status}', %s)) IS NOT TRUE)",
+ ["acme", "acme"],
+ ),
+ (
+ "Contains",
+ "(jsonb_exists(meta_fields #> '{status}', %s) OR lower(meta_fields #>> '{status}') LIKE %s ESCAPE '\\')",
+ ["acme", "%acme%"],
+ ),
+ (
+ "START WITH",
+ "(lower(meta_fields #>> '{status}') LIKE %s ESCAPE '\\')",
+ ["acme%"],
+ ),
+ ]
+
+ for op, expected_sql, expected_params in cases:
+ sql = _assert_filter(
+ [{"key": "status", "op": op, "value": "acme"}],
+ "and",
+ expected_sql,
+ expected_params,
+ )
+ assert "acme" not in sql
+
+
+def test_tc_mgf_505_before_after_operators_are_not_supported():
+ for op in ("before", "after"):
+ _assert_rejected_filter(
+ {"key": "date", "op": op, "value": "2026-01-01"},
+ f"unsupported metadata filter operator '{op}'",
+ )
+
+
+def test_tc_mgf_601_missing_key_is_rejected_with_stable_reason():
+ _assert_rejected_filter(
+ {"op": "=", "value": "private-metadata-value"},
+ "invalid metadata key",
+ hidden_inputs=("private-metadata-value",),
+ )
+
+
+def test_tc_mgf_602_missing_operator_is_rejected_with_exact_message():
+ _assert_rejected_filter(
+ {"key": "status", "value": "x"},
+ "metadata filter operator is missing",
+ )
+
+
+def test_tc_mgf_603_unknown_operator_is_rejected_with_exact_message():
+ _assert_rejected_filter(
+ {"key": "status", "op": "regex", "value": "x"},
+ "unsupported metadata filter operator 'regex'",
+ )
+
+
+def test_tc_mgf_604_numeric_leading_key_segment_is_encoded():
+ _assert_filter(
+ [{"key": "1abc", "op": "=", "value": 1}],
+ "and",
+ "(meta_fields #> '{1abc}' @> %s::jsonb)",
+ ["1"],
+ )
+ assert jsonb_path_literal("1abc") == "'{1abc}'"
+
+
+def test_tc_mgf_605_empty_or_nul_key_segments_are_rejected():
+ cases = [
+ ("a..b", "metadata key contains an empty path segment"),
+ ("a.", "metadata key contains an empty path segment"),
+ (".a", "metadata key contains an empty path segment"),
+ ("a.\x00b", "metadata key contains NUL"),
+ ]
+
+ for key, expected_reason in cases:
+ _assert_rejected_filter(
+ {"key": key, "op": "=", "value": "x"},
+ expected_reason,
+ hidden_inputs=(key,),
+ )
+
+
+def test_tc_mgf_606_key_segment_with_space_is_encoded():
+ path = "'{a,\"b c\"}'"
+ _assert_filter(
+ [{"key": "a.b c", "op": "=", "value": 1}],
+ "and",
+ f"(meta_fields #> {path} @> %s::jsonb)",
+ ["1"],
+ )
+ assert jsonb_path_literal("a.b c") == path
+
+
+def test_tc_mgf_607_equal_operator_rejects_non_scalar_value():
+ _assert_rejected_filter(
+ {"key": "private-key", "op": "=", "value": ["private-value"]},
+ "scalar comparison value is non-scalar",
+ hidden_inputs=("private-key", "private-value"),
+ )
+
+
+def test_tc_mgf_608_range_operator_rejects_none_value():
+ _assert_rejected_filter(
+ {"key": "private-key", "op": ">", "value": None},
+ "range comparison value is None",
+ hidden_inputs=("private-key",),
+ )
+
+
+def test_tc_mgf_609_range_operator_rejects_boolean_value():
+ _assert_rejected_filter(
+ {"key": "private-key", "op": ">", "value": True},
+ "range comparison value is boolean",
+ hidden_inputs=("private-key",),
+ )
+
+
+def test_tc_mgf_610_range_operator_rejects_non_numeric_non_date_string():
+ _assert_rejected_filter(
+ {"key": "private-key", "op": ">", "value": "private-range-value"},
+ "unsupported range comparison value",
+ hidden_inputs=("private-key", "private-range-value"),
+ )
+
+
+def test_tc_mgf_611_contains_operator_rejects_none_and_container_values():
+ for value in (None, ["private-value"], {"private-value": True}):
+ _assert_rejected_filter(
+ {"key": "private-key", "op": "contains", "value": value},
+ "string operator value must be a scalar",
+ hidden_inputs=("private-key", "private-value"),
+ )
+
+
+def test_tc_mgf_612_contains_operator_rejects_empty_string():
+ _assert_rejected_filter(
+ {"key": "private-key", "op": "contains", "value": ""},
+ "string operator value is empty",
+ hidden_inputs=("private-key",),
+ )
+
+
+def test_tc_mgf_613_in_operator_rejects_none_value():
+ _assert_rejected_filter(
+ {"key": "private-key", "op": "in", "value": None},
+ "membership value is None",
+ hidden_inputs=("private-key",),
+ )
+
+
+def test_tc_mgf_614_in_operator_rejects_every_empty_resolved_member_list():
+ for value in ([], "", ",,"):
+ _assert_rejected_filter(
+ {"key": "private-key", "op": "in", "value": value},
+ "membership value resolved to empty list",
+ hidden_inputs=("private-key",),
+ )
+
+
+def test_tc_mgf_615_unknown_logic_is_rejected():
+ with pytest.raises(UnsupportedGaussDBMetaFilter) as exc_info:
+ build_gaussdb_filter([], "xor")
+
+ assert str(exc_info.value) == "unknown logic 'xor'"
+
+
+def test_tc_mgf_616_invalid_jsonb_column_is_rejected():
+ with pytest.raises(UnsupportedGaussDBMetaFilter) as exc_info:
+ GaussDBMetaFilterTranslator(jsonb_column="chunk_data;drop")
+
+ assert str(exc_info.value) == "invalid JSONB column 'chunk_data;drop'"
+
+
+def test_tc_mgf_617_value_with_sql_metacharacters_is_bound_and_like_escaped():
+ value = r"x%_\'; DROP TABLE--"
+ sql = _assert_filter(
+ [{"key": "status", "op": "contains", "value": value}],
+ "and",
+ "(jsonb_exists(meta_fields #> '{status}', %s) OR lower(meta_fields #>> '{status}') LIKE %s ESCAPE '\\')",
+ [r"x%_\'; drop table--", r"%x\%\_\\'; drop table--%"],
+ )
+
+ assert "DROP TABLE" not in sql
+ assert "drop table" not in sql
+ assert "x%_" not in sql
+
+
+def test_tc_mgf_618_non_ascii_and_at_sign_key_segments_are_encoded():
+ cases = [
+ ("字段", "'{\"字段\"}'"),
+ ("field@name", "'{\"field@name\"}'"),
+ ]
+
+ for key, path in cases:
+ _assert_filter(
+ [{"key": key, "op": "=", "value": 1}],
+ "and",
+ f"(meta_fields #> {path} @> %s::jsonb)",
+ ["1"],
+ )
+ assert jsonb_path_literal(key) == path
+
+
+def test_tc_mgf_701_is_pushdown_supported_returns_true_for_fully_translatable_filter():
+ filters = [{"key": "a", "op": "=", "value": "x"}]
+
+ assert is_pushdown_supported(filters) is True
+ plan = plan_pushdown(filters, "and")
+ assert plan.logic == "and"
+ assert len(plan.translated) == 1
+ assert plan.translated[0].sql == "lower(meta_fields #>> '{a}') = %s OR jsonb_exists(meta_fields #> '{a}', %s)"
+ assert plan.translated[0].params == ["x", "x"]
+ _assert_filter(
+ filters,
+ "and",
+ "(lower(meta_fields #>> '{a}') = %s OR jsonb_exists(meta_fields #> '{a}', %s))",
+ ["x", "x"],
+ )
+
+
+def test_tc_mgf_702_is_pushdown_supported_returns_false_without_raising():
+ filters = [{"key": "a", "op": "before", "value": "x"}]
+
+ assert is_pushdown_supported(filters) is False
+ _assert_rejected_filter(
+ filters[0],
+ "unsupported metadata filter operator 'before'",
+ )
+
+
+def test_tc_mgf_704_all_supported_operators_normalize_and_translate():
+ expected_operators = frozenset(
+ {
+ "=",
+ "\u2260",
+ ">",
+ "\u2265",
+ "<",
+ "\u2264",
+ "in",
+ "not in",
+ "contains",
+ "not contains",
+ "start with",
+ "end with",
+ "empty",
+ "not empty",
+ }
+ )
+ cases = [
+ (
+ "=",
+ "status",
+ "x",
+ "lower(meta_fields #>> '{status}') = %s OR jsonb_exists(meta_fields #> '{status}', %s)",
+ ["x", "x"],
+ ),
+ (
+ OP_NE,
+ "status",
+ "x",
+ "(meta_fields #> '{status}') IS NOT NULL AND (lower(meta_fields #>> '{status}') = %s OR jsonb_exists(meta_fields #> '{status}', %s)) IS NOT TRUE",
+ ["x", "x"],
+ ),
+ (
+ ">",
+ "amount",
+ 1,
+ "CASE WHEN meta_fields #>> '{amount}' ~ '^-?[0-9]+(\\.[0-9]+)?$' THEN (meta_fields #>> '{amount}')::DOUBLE PRECISION > %s ELSE FALSE END",
+ [1],
+ ),
+ (
+ OP_GE,
+ "amount",
+ 1,
+ "CASE WHEN meta_fields #>> '{amount}' ~ '^-?[0-9]+(\\.[0-9]+)?$' THEN (meta_fields #>> '{amount}')::DOUBLE PRECISION >= %s ELSE FALSE END",
+ [1],
+ ),
+ (
+ "<",
+ "amount",
+ 1,
+ "CASE WHEN meta_fields #>> '{amount}' ~ '^-?[0-9]+(\\.[0-9]+)?$' THEN (meta_fields #>> '{amount}')::DOUBLE PRECISION < %s ELSE FALSE END",
+ [1],
+ ),
+ (
+ OP_LE,
+ "amount",
+ 1,
+ "CASE WHEN meta_fields #>> '{amount}' ~ '^-?[0-9]+(\\.[0-9]+)?$' THEN (meta_fields #>> '{amount}')::DOUBLE PRECISION <= %s ELSE FALSE END",
+ [1],
+ ),
+ (
+ "in",
+ "status",
+ ["x"],
+ "(lower(meta_fields #>> '{status}') = %s OR jsonb_exists(meta_fields #> '{status}', %s))",
+ ["x", "x"],
+ ),
+ (
+ "not in",
+ "status",
+ ["x"],
+ "(meta_fields #> '{status}') IS NOT NULL AND (lower(meta_fields #>> '{status}') = %s OR jsonb_exists(meta_fields #> '{status}', %s)) IS NOT TRUE",
+ ["x", "x"],
+ ),
+ (
+ "contains",
+ "status",
+ "x",
+ "jsonb_exists(meta_fields #> '{status}', %s) OR lower(meta_fields #>> '{status}') LIKE %s ESCAPE '\\'",
+ ["x", "%x%"],
+ ),
+ (
+ "not contains",
+ "status",
+ "x",
+ "(meta_fields #> '{status}') IS NOT NULL AND (jsonb_exists(meta_fields #> '{status}', %s) OR lower(meta_fields #>> '{status}') LIKE %s ESCAPE '\\') IS NOT TRUE",
+ ["x", "%x%"],
+ ),
+ (
+ "start with",
+ "status",
+ "x",
+ "lower(meta_fields #>> '{status}') LIKE %s ESCAPE '\\'",
+ ["x%"],
+ ),
+ (
+ "end with",
+ "status",
+ "x",
+ "lower(meta_fields #>> '{status}') LIKE %s ESCAPE '\\'",
+ ["%x"],
+ ),
+ (
+ "empty",
+ "status",
+ None,
+ "(meta_fields #> '{status}') IS NULL OR meta_fields #> '{status}' = 'null'::jsonb OR "
+ "meta_fields #> '{status}' = '\"\"'::jsonb OR meta_fields #> '{status}' = '[]'::jsonb OR "
+ "meta_fields #> '{status}' = '{}'::jsonb",
+ [],
+ ),
+ (
+ "not empty",
+ "status",
+ None,
+ "(meta_fields #> '{status}') IS NOT NULL AND (meta_fields #> '{status}' = 'null'::jsonb) IS NOT TRUE AND "
+ "(meta_fields #> '{status}' = '\"\"'::jsonb) IS NOT TRUE AND "
+ "(meta_fields #> '{status}' = '[]'::jsonb) IS NOT TRUE AND "
+ "(meta_fields #> '{status}' = '{}'::jsonb) IS NOT TRUE",
+ [],
+ ),
+ ]
+
+ case_operators = {op for op, _, _, _, _ in cases}
+ assert len(expected_operators) == 14
+ assert len(cases) == 14
+ assert SUPPORTED_OPERATORS == expected_operators
+ assert case_operators == expected_operators
+ translator = GaussDBMetaFilterTranslator()
+ for op, key, value, expected_sql, expected_params in cases:
+ assert normalize_gaussdb_meta_operator(op) == op
+ translated = translator.translate({"key": key, "op": op, "value": value})
+ assert translated.sql == expected_sql
+ assert translated.params == expected_params
+
+
+def test_tc_mgf_707_coerce_scalar_value_covers_python_literal_matrix():
+ cases = [
+ ("123", 123),
+ ("1.5", 1.5),
+ ("true", "true"),
+ ("false", "false"),
+ ("null", "null"),
+ ("True", True),
+ ("False", False),
+ ("None", None),
+ ("", ""),
+ (" ", " "),
+ ("[1,2]", "[1,2]"),
+ ("abc", "abc"),
+ (5, 5),
+ (None, None),
+ ]
+
+ for value, expected in cases:
+ assert gaussdb_filter.coerce_scalar_value(value, {"key": "a", "op": "=", "value": value}) == expected
+
+
+def test_tc_mgf_708_normalize_datetime_prefix_covers_valid_and_invalid_matrix():
+ cases = [
+ ("2026", None),
+ ("2026-07", "2026-07-01 00:00:00"),
+ ("2026-07-08 10:30:45", "2026-07-08 10:30:45"),
+ ("2026-13-01", None),
+ ("2026-07-32", None),
+ ("abc", None),
+ ]
+
+ for value, expected in cases:
+ assert gaussdb_filter.normalize_datetime_prefix(value) == expected
+
+
+def test_tc_mgf_709_fetch_gaussdb_metadata_doc_ids_delegates_public_boundary():
+ expected_doc_ids = ["doc-1", "doc-2"]
+
+ class RecordingDocStore:
+ def __init__(self):
+ self.calls = []
+
+ def fetch_metadata_doc_ids(self, *args):
+ self.calls.append(args)
+ return expected_doc_ids
+
+ doc_store = RecordingDocStore()
+ kb_ids = ["kb-1", "kb-2"]
+ filter_params = ["active", "active"]
+ result = fetch_gaussdb_metadata_doc_ids(
+ doc_store,
+ "ragflow_doc_meta_tenant",
+ kb_ids,
+ "lower(meta_fields #>> '{status}') = %s",
+ filter_params,
+ 101,
+ )
+
+ assert result is expected_doc_ids
+ assert doc_store.calls == [
+ (
+ "ragflow_doc_meta_tenant",
+ kb_ids,
+ "lower(meta_fields #>> '{status}') = %s",
+ filter_params,
+ 101,
+ )
+ ]
+
+
+def test_tc_mgf_810_all_operators_avoid_bare_empty_string_sql():
+ cases = [
+ (
+ {"key": "status", "op": "=", "value": "x"},
+ "(lower(meta_fields #>> '{status}') = %s OR jsonb_exists(meta_fields #> '{status}', %s))",
+ ["x", "x"],
+ ),
+ (
+ {"key": "status", "op": OP_NE, "value": "x"},
+ "((meta_fields #> '{status}') IS NOT NULL AND (lower(meta_fields #>> '{status}') = %s OR jsonb_exists(meta_fields #> '{status}', %s)) IS NOT TRUE)",
+ ["x", "x"],
+ ),
+ (
+ {"key": "amount", "op": ">", "value": 100},
+ "(CASE WHEN meta_fields #>> '{amount}' ~ '^-?[0-9]+(\\.[0-9]+)?$' THEN (meta_fields #>> '{amount}')::DOUBLE PRECISION > %s ELSE FALSE END)",
+ [100],
+ ),
+ (
+ {"key": "amount", "op": OP_GE, "value": 100},
+ "(CASE WHEN meta_fields #>> '{amount}' ~ '^-?[0-9]+(\\.[0-9]+)?$' THEN (meta_fields #>> '{amount}')::DOUBLE PRECISION >= %s ELSE FALSE END)",
+ [100],
+ ),
+ (
+ {"key": "amount", "op": "<", "value": 100},
+ "(CASE WHEN meta_fields #>> '{amount}' ~ '^-?[0-9]+(\\.[0-9]+)?$' THEN (meta_fields #>> '{amount}')::DOUBLE PRECISION < %s ELSE FALSE END)",
+ [100],
+ ),
+ (
+ {"key": "amount", "op": OP_LE, "value": 100},
+ "(CASE WHEN meta_fields #>> '{amount}' ~ '^-?[0-9]+(\\.[0-9]+)?$' THEN (meta_fields #>> '{amount}')::DOUBLE PRECISION <= %s ELSE FALSE END)",
+ [100],
+ ),
+ (
+ {"key": "status", "op": "contains", "value": "x"},
+ "(jsonb_exists(meta_fields #> '{status}', %s) OR lower(meta_fields #>> '{status}') LIKE %s ESCAPE '\\')",
+ ["x", "%x%"],
+ ),
+ (
+ {"key": "status", "op": "not contains", "value": "x"},
+ "((meta_fields #> '{status}') IS NOT NULL AND (jsonb_exists(meta_fields #> '{status}', %s) OR lower(meta_fields #>> '{status}') LIKE %s ESCAPE '\\') IS NOT TRUE)",
+ ["x", "%x%"],
+ ),
+ (
+ {"key": "status", "op": "start with", "value": "x"},
+ "(lower(meta_fields #>> '{status}') LIKE %s ESCAPE '\\')",
+ ["x%"],
+ ),
+ (
+ {"key": "status", "op": "end with", "value": "x"},
+ "(lower(meta_fields #>> '{status}') LIKE %s ESCAPE '\\')",
+ ["%x"],
+ ),
+ (
+ {"key": "status", "op": "empty", "value": None},
+ "((meta_fields #> '{status}') IS NULL OR meta_fields #> '{status}' = 'null'::jsonb OR "
+ "meta_fields #> '{status}' = '\"\"'::jsonb OR meta_fields #> '{status}' = '[]'::jsonb OR "
+ "meta_fields #> '{status}' = '{}'::jsonb)",
+ [],
+ ),
+ (
+ {"key": "status", "op": "not empty", "value": None},
+ "((meta_fields #> '{status}') IS NOT NULL AND (meta_fields #> '{status}' = 'null'::jsonb) IS NOT TRUE AND "
+ "(meta_fields #> '{status}' = '\"\"'::jsonb) IS NOT TRUE AND "
+ "(meta_fields #> '{status}' = '[]'::jsonb) IS NOT TRUE AND "
+ "(meta_fields #> '{status}' = '{}'::jsonb) IS NOT TRUE)",
+ [],
+ ),
+ (
+ {"key": "status", "op": "in", "value": ["x", "y"]},
+ "((lower(meta_fields #>> '{status}') = %s OR jsonb_exists(meta_fields #> '{status}', %s)) OR (lower(meta_fields #>> '{status}') = %s OR jsonb_exists(meta_fields #> '{status}', %s)))",
+ ["x", "x", "y", "y"],
+ ),
+ (
+ {"key": "status", "op": "not in", "value": ["x", "y"]},
+ "((meta_fields #> '{status}') IS NOT NULL AND "
+ "(lower(meta_fields #>> '{status}') = %s OR jsonb_exists(meta_fields #> '{status}', %s)) IS NOT TRUE AND "
+ "(lower(meta_fields #>> '{status}') = %s OR jsonb_exists(meta_fields #> '{status}', %s)) IS NOT TRUE)",
+ ["x", "x", "y", "y"],
+ ),
+ ]
+
+ assert len(cases) == 14
+ for flt, expected_sql, expected_params in cases:
+ sql = _assert_filter([flt], "and", expected_sql, expected_params)
+ assert "= ''" not in sql
+ assert "<> ''" not in sql
diff --git a/test/unit_test/rag/utils/test_gaussdb_search_sql.py b/test/unit_test/rag/utils/test_gaussdb_search_sql.py
new file mode 100644
index 0000000000..8c27422d1a
--- /dev/null
+++ b/test/unit_test/rag/utils/test_gaussdb_search_sql.py
@@ -0,0 +1,737 @@
+#
+# Copyright 2025 The InfiniFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+import pytest
+
+from common.doc_store.gaussdb_conn_base import GaussDBSearchBuilder, InvalidGaussDBObjectName
+from rag.utils.gaussdb_conn import GaussDBConnection, SearchResult
+
+
+def test_tc_ret_513_dynamic_chunk_field_exists_filter_reads_extra_jsonb():
+ builder = GaussDBSearchBuilder(schema="public")
+
+ sql, params = builder.build_condition_where({"kb_id": "kb1", "must_not": {"exists": "custom_kwd"}})
+
+ assert "(extra -> 'custom_kwd') IS NULL" in sql
+ assert "(extra -> 'custom_kwd') = 'null'::jsonb" in sql
+ assert params == ["kb1"]
+
+
+def test_tc_ret_513_dynamic_chunk_field_select_and_filter_are_backed_by_extra_jsonb():
+ builder = GaussDBSearchBuilder(schema="public")
+
+ sql, params = builder.build_search_sql(
+ table="ragflow_tenant",
+ select_fields=["id", "custom_kwd"],
+ condition={"kb_id": "kb1", "custom_kwd": ["artifact_page"]},
+ keywords=[],
+ vector=None,
+ vector_dim=None,
+ vector_weight=0.0,
+ offset=0,
+ limit=10,
+ )
+
+ assert "(extra -> 'custom_kwd') AS custom_kwd" in sql
+ assert "(extra -> 'custom_kwd') = %s::jsonb" in sql
+ assert params[:3] == ["kb1", '"artifact_page"', '["artifact_page"]']
+
+
+def test_tc_ret_513_dynamic_chunk_field_rejects_unsafe_json_key():
+ builder = GaussDBSearchBuilder(schema="public")
+
+ with pytest.raises(InvalidGaussDBObjectName):
+ builder.build_condition_where({"must_not": {"exists": "custom_kwd'; DROP TABLE user; --"}})
+
+
+def test_tc_ret_101_fulltext_builder_parameterizes_simple_query_and_filter():
+ builder = GaussDBSearchBuilder(schema="public")
+ malicious_keyword = "contract'); DROP TABLE ragflow_tenant; --"
+ query_text = f"{malicious_keyword} risk"
+ sql, params = builder.build_search_sql(
+ table="ragflow_tenant",
+ select_fields=["id", "content_with_weight"],
+ condition={"available_int": 1},
+ keywords=[malicious_keyword, "risk"],
+ vector=None,
+ vector_dim=None,
+ vector_weight=0.0,
+ offset=0,
+ limit=10,
+ )
+
+ assert "to_tsvector('simple'" in sql
+ assert "ts_rank" in sql
+ assert "WHERE available_int = %s AND to_tsvector" in sql
+ assert "plainto_tsquery('simple', %s)" in sql
+ assert "ORDER BY _score DESC, kb_id ASC, id ASC LIMIT %s OFFSET %s" in sql
+ assert malicious_keyword not in sql
+ assert query_text not in sql
+ assert params == [query_text] * 6 + [1, query_text, 10, 0]
+
+
+def test_tc_ret_109_chinese_ngram_query_preserves_english_semantics_and_highlight_text():
+ builder = GaussDBSearchBuilder(schema="public")
+
+ ngram_ddl = builder.ddl.build_ngram_fulltext_ugin_ddl("ragflow_tenant")
+ assert '"idx_gdb_ragflow_tenant_fts_all_ngram"' in ngram_ddl
+ assert "USING ugin(to_tsvector('ngram'" in ngram_ddl
+
+ chinese_sql, chinese_params = builder.build_search_sql(
+ table="ragflow_tenant",
+ select_fields=["id", "content_with_weight"],
+ condition={"kb_id": "kb1"},
+ keywords=["深圳"],
+ vector=None,
+ vector_dim=None,
+ vector_weight=0.0,
+ offset=0,
+ limit=10,
+ highlight_fields=["content_with_weight"],
+ )
+ assert "to_tsvector('ngram'" in chinese_sql
+ assert "plainto_tsquery('ngram', %s)" in chinese_sql
+ assert "to_tsvector('simple'" not in chinese_sql
+ assert "COALESCE(content_with_weight, ' ') AS _highlight_source" in chinese_sql
+ assert "ts_headline('ngram'" not in chinese_sql
+ assert chinese_params == ["深圳"] * 6 + ["kb1", "深圳", 10, 0]
+
+ mixed_sql, mixed_params = builder.build_search_sql(
+ table="ragflow_tenant",
+ select_fields=["id", "content_with_weight"],
+ condition={"kb_id": "kb1"},
+ keywords=["上海", "RAGFlow", "database"],
+ vector=None,
+ vector_dim=None,
+ vector_weight=0.0,
+ offset=0,
+ limit=10,
+ highlight_fields=["content_with_weight"],
+ )
+ assert "to_tsvector('simple'" in mixed_sql
+ assert "to_tsvector('ngram'" in mixed_sql
+ assert "plainto_tsquery('simple', %s)" in mixed_sql
+ assert "plainto_tsquery('ngram', %s)" in mixed_sql
+ assert " / 2.0)" in mixed_sql
+ assert ") @@ plainto_tsquery('simple', %s) AND to_tsvector('ngram'" in mixed_sql
+ assert "COALESCE(content_with_weight, ' ') AS _highlight_source" in mixed_sql
+ assert mixed_params == ["RAGFlow database"] * 6 + ["上海"] * 6 + [
+ "kb1",
+ "RAGFlow database",
+ "上海",
+ 10,
+ 0,
+ ]
+
+ single_character_sql, single_character_params = builder.build_search_sql(
+ table="ragflow_tenant",
+ select_fields=["id"],
+ condition={"kb_id": "kb1"},
+ keywords=["中"],
+ vector=None,
+ vector_dim=None,
+ vector_weight=0.0,
+ offset=0,
+ limit=10,
+ )
+ assert "0.0 AS _score" in single_character_sql
+ assert "WHERE kb_id = %s AND FALSE" in single_character_sql
+ assert single_character_params == ["kb1", 10, 0]
+
+ vector_only_sql, _vector_only_params = builder.build_search_sql(
+ table="ragflow_tenant",
+ select_fields=["id"],
+ condition={"kb_id": "kb1"},
+ keywords=["中"],
+ vector=[0.1, 0.2, 0.3, 0.4],
+ vector_dim=4,
+ vector_weight=0.3,
+ offset=0,
+ limit=10,
+ )
+ assert "WITH vec AS" in vector_only_sql
+ assert "WITH fts_raw AS" not in vector_only_sql
+
+ conn = GaussDBConnection.__new__(GaussDBConnection)
+ result = SearchResult(
+ total=2,
+ chunks=[
+ {"id": "mixed", "_highlight_source": "上海 RAGFlow database 审计"},
+ {"id": "overlap", "_highlight_source": "深圳数据库审计"},
+ ],
+ )
+ assert conn.get_highlight(result, ["上海", "RAGFlow", "database"], "content_with_weight") == {"mixed": "<em>上海</em> <em>RAGFlow</em> <em>database</em> 审计"}
+ assert conn.get_highlight(result, ["深圳数据", "深圳"], "content_with_weight") == {"overlap": "<em>深圳数据</em>库审计"}
+
+
+def test_tc_ret_203_vector_search_filters_invalid_placeholder_vectors():
+ builder = GaussDBSearchBuilder(schema="public")
+ sql, params = builder.build_search_sql(
+ table="ragflow_tenant",
+ select_fields=["id"],
+ condition={"available_int": 1},
+ keywords=[],
+ vector=[0.1, 0.2, 0.3, 0.4],
+ vector_dim=4,
+ vector_weight=1.0,
+ offset=0,
+ limit=5,
+ )
+
+ vector_param = "[0.1,0.2,0.3,0.4]"
+ assert "WHERE available_int = %s AND q_4_vec_valid = TRUE" in sql
+ assert sql.count("q_4_vec <+> %s::floatvector(4)") == 3
+ assert "WHERE _score >= %s" in sql
+ assert params == [vector_param, vector_param, 1, vector_param, 5, 0.0, 5, 0]
+
+
+def test_tc_ret_301_hybrid_search_uses_configured_vector_weight():
+ builder = GaussDBSearchBuilder(schema="public")
+ sql, params = builder.build_search_sql(
+ table="ragflow_tenant",
+ select_fields=["id"],
+ condition={"available_int": 1},
+ keywords=["budget"],
+ vector=[0.1, 0.2, 0.3, 0.4],
+ vector_dim=4,
+ vector_weight=0.7,
+ offset=0,
+ limit=5,
+ )
+
+ assert "%s * COALESCE(vec.vector_score, 0)" in sql
+ assert "(1 - %s) * COALESCE(fts.fts_score, 0)" in sql
+ assert params == [
+ "budget",
+ "budget",
+ "budget",
+ "budget",
+ "budget",
+ "budget",
+ 1,
+ "budget",
+ 5,
+ "[0.1,0.2,0.3,0.4]",
+ 1,
+ "[0.1,0.2,0.3,0.4]",
+ 5,
+ 0.7,
+ 0.7,
+ 0.0,
+ 5,
+ 0,
+ ]
+
+
+def test_tc_ret_303_hybrid_search_full_outer_joins_text_and_vector_candidates():
+ builder = GaussDBSearchBuilder(schema="public")
+ sql, params = builder.build_search_sql(
+ table="ragflow_tenant",
+ select_fields=["id"],
+ condition={"kb_id": "kb1"},
+ keywords=["budget"],
+ vector=[0.1, 0.2, 0.3, 0.4],
+ vector_dim=4,
+ vector_weight=0.3,
+ offset=0,
+ limit=5,
+ )
+
+ assert "COALESCE(fts.kb_id, vec.kb_id) AS kb_id" in sql
+ assert "COALESCE(fts.id, vec.id) AS id" in sql
+ assert "FROM fts FULL OUTER JOIN vec ON fts.kb_id = vec.kb_id AND fts.id = vec.id" in sql
+ assert 'FROM merged JOIN "public"."ragflow_tenant" c ON c.kb_id = merged.kb_id AND c.id = merged.id' in sql
+ assert params[-5:] == [0.3, 0.3, 0.0, 5, 0]
+
+
+def test_tc_ret_310_hybrid_pagerank_is_applied_to_projection_and_threshold():
+ builder = GaussDBSearchBuilder(schema="public")
+ sql, params = builder.build_search_sql(
+ table="ragflow_tenant",
+ select_fields=["id"],
+ condition={"kb_id": "kb1"},
+ keywords=["budget"],
+ vector=[0.1, 0.2, 0.3, 0.4],
+ vector_dim=4,
+ vector_weight=0.7,
+ offset=0,
+ limit=5,
+ pagerank_weight=10.0,
+ )
+
+ pagerank_expr = "(COALESCE(c.pagerank_fea, 0)::DOUBLE PRECISION / 100.0 * %s)"
+ assert f"(merged.score + {pagerank_expr}) AS _score" in sql
+ assert f"WHERE (merged.score + {pagerank_expr}) >= %s" in sql
+ assert params == ["budget"] * 6 + [
+ "kb1",
+ "budget",
+ 5,
+ "[0.1,0.2,0.3,0.4]",
+ "kb1",
+ "[0.1,0.2,0.3,0.4]",
+ 5,
+ 0.7,
+ 0.7,
+ 10.0,
+ 10.0,
+ 0.0,
+ 5,
+ 0,
+ ]
+
+
+def test_tc_ret_701_pagerank_feature_adds_bound_score_component():
+ builder = GaussDBSearchBuilder(schema="public")
+ sql, params = builder.build_search_sql(
+ table="ragflow_tenant",
+ select_fields=["id"],
+ condition={"kb_id": "kb1"},
+ keywords=["risk"],
+ vector=None,
+ vector_dim=None,
+ vector_weight=0.0,
+ offset=0,
+ limit=5,
+ pagerank_weight=10.0,
+ )
+
+ assert "COALESCE(pagerank_fea, 0)::DOUBLE PRECISION / 100.0 * %s" in sql
+ assert "ORDER BY _score DESC, kb_id ASC, id ASC" in sql
+ assert params == ["risk"] * 6 + [10.0, "kb1", "risk", 5, 0]
+
+
+def test_tc_ret_201_vector_only_query_omits_fulltext_predicates():
+ builder = GaussDBSearchBuilder(schema="public")
+ sql, params = builder.build_search_sql(
+ table="ragflow_tenant",
+ select_fields=["id"],
+ condition={"available_int": 1},
+ keywords=[],
+ vector=[0.1, 0.2, 0.3, 0.4],
+ vector_dim=4,
+ vector_weight=1.0,
+ offset=0,
+ limit=5,
+ )
+
+ assert "to_tsvector" not in sql
+ assert "WITH vec AS" in sql
+ assert "ORDER BY q_4_vec <+> %s::floatvector(4) ASC LIMIT %s" in sql
+ assert "ORDER BY distance ASC, kb_id ASC, id ASC LIMIT %s OFFSET %s" in sql
+ assert "[0.1,0.2,0.3,0.4]" not in sql
+ assert "[0.1,0.2,0.3,0.4]" in params
+ assert params[-4:] == [5, 0.0, 5, 0]
+
+
+def test_tc_ret_605_fulltext_search_projects_parameterized_highlight_column():
+ builder = GaussDBSearchBuilder(schema="public")
+ malicious_keyword = "risk'); DROP TABLE ragflow_tenant; --"
+ query_text = f"{malicious_keyword} audit"
+ vector = [0.123456789, 0.234567891, 0.345678912, 0.456789123]
+ sql, params = builder.build_search_sql(
+ table="ragflow_tenant",
+ select_fields=["id"],
+ condition={"kb_id": "kb1"},
+ keywords=[malicious_keyword, "audit"],
+ vector=None,
+ vector_dim=None,
+ vector_weight=0.0,
+ offset=0,
+ limit=5,
+ highlight_fields=["content_with_weight"],
+ )
+
+ assert ("ts_headline('simple', COALESCE(content_with_weight, ' '), plainto_tsquery('simple', %s), 'StartSel=<em>, StopSel=</em>') AS _highlight") in sql
+ assert malicious_keyword not in sql
+ assert query_text not in sql
+ assert params == [query_text] * 7 + ["kb1", query_text, 5, 0]
+
+ hybrid_sql, hybrid_params = builder.build_search_sql(
+ table="ragflow_tenant",
+ select_fields=["id"],
+ condition={"kb_id": "kb1"},
+ keywords=[malicious_keyword, "audit"],
+ vector=vector,
+ vector_dim=4,
+ vector_weight=0.3,
+ offset=0,
+ limit=5,
+ highlight_fields=["content_with_weight"],
+ )
+ vector_param = "[0.123456789,0.234567891,0.345678912,0.456789123]"
+ assert "FULL OUTER JOIN" in hybrid_sql
+ assert ("ts_headline('simple', COALESCE(content_with_weight, ' '), plainto_tsquery('simple', %s), 'StartSel=<em>, StopSel=</em>') AS _highlight") in hybrid_sql
+ assert malicious_keyword not in hybrid_sql
+ assert query_text not in hybrid_sql
+ assert vector_param not in hybrid_sql
+ assert hybrid_params == [query_text] * 6 + [
+ "kb1",
+ query_text,
+ 5,
+ vector_param,
+ "kb1",
+ vector_param,
+ 5,
+ 0.3,
+ 0.3,
+ query_text,
+ 0.0,
+ 5,
+ 0,
+ ]
+
+
+def test_tc_ret_601_aggregation_sql_groups_field_values():
+ builder = GaussDBSearchBuilder(schema="public")
+ sql, params = builder.build_aggregation_sql(
+ table="ragflow_tenant",
+ field_name="docnm_kwd",
+ condition={"kb_id": "kb1"},
+ )
+
+ assert sql == ('SELECT docnm_kwd AS value, COUNT(1) AS count FROM "public"."ragflow_tenant" WHERE kb_id = %s AND docnm_kwd IS NOT NULL GROUP BY value ORDER BY count DESC, value ASC LIMIT %s')
+ assert params == ["kb1", 1000]
+
+
+def test_tc_ret_602_aggregation_sql_expands_jsonb_array_fields_without_lateral():
+ builder = GaussDBSearchBuilder(schema="public")
+ sql, params = builder.build_aggregation_sql(
+ table="ragflow_tenant",
+ field_name="tag_kwd",
+ condition={"kb_id": "k1"},
+ )
+
+ assert sql == (
+ "SELECT value, COUNT(1) AS count "
+ "FROM (SELECT jsonb_array_elements_text(COALESCE(tag_kwd, '[]'::jsonb)) AS value "
+ 'FROM "public"."ragflow_tenant" WHERE kb_id = %s) AS expanded '
+ "GROUP BY value ORDER BY count DESC, value ASC LIMIT %s"
+ )
+ assert "LATERAL" not in sql.upper()
+ assert params == ["k1", 1000]
+
+
+def test_tc_ret_306_hybrid_threshold_is_applied_after_merged_score():
+ builder = GaussDBSearchBuilder(schema="public")
+
+ sql, params = builder.build_search_sql(
+ table="ragflow_tenant",
+ select_fields=["id"],
+ condition={"kb_id": "kb1"},
+ keywords=["risk"],
+ vector=[0.1, 0.2, 0.3, 0.4],
+ vector_dim=4,
+ vector_weight=0.75,
+ similarity_threshold=0.2,
+ offset=0,
+ limit=5,
+ )
+
+ assert "WHERE merged.score >= %s" in sql
+ assert sql.count(">= %s") == 1
+ assert params[-5:] == [0.75, 0.75, 0.2, 5, 0]
+
+
+def test_tc_ret_311_hybrid_zero_vector_weight_is_bound_in_merged_score():
+ builder = GaussDBSearchBuilder(schema="public")
+
+ sql, params = builder.build_search_sql(
+ table="ragflow_tenant",
+ select_fields=["id"],
+ condition={"kb_id": "kb1"},
+ keywords=["risk"],
+ vector=[0.1, 0.2, 0.3, 0.4],
+ vector_dim=4,
+ vector_weight=0.0,
+ similarity_threshold=0.8,
+ offset=0,
+ limit=5,
+ )
+
+ assert "(1 - %s) * COALESCE(fts.fts_score, 0) + %s * COALESCE(vec.vector_score, 0) AS score" in sql
+ assert "WHERE merged.score >= %s" in sql
+ assert params[-5:] == [0.0, 0.0, 0.8, 5, 0]
+
+
+def test_tc_ret_010_filter_only_search_uses_collection_limit_when_limit_is_zero():
+ builder = GaussDBSearchBuilder(schema="public")
+ sql, params = builder.build_search_sql(
+ table="ragflow_tenant",
+ select_fields=["id"],
+ condition={"kb_id": "kb1"},
+ keywords=[],
+ vector=None,
+ vector_dim=None,
+ vector_weight=0.0,
+ offset=0,
+ limit=0,
+ )
+
+ assert "ORDER BY kb_id ASC, id ASC LIMIT %s OFFSET %s" in sql
+ assert params == ["kb1", 10000, 0]
+
+
+def test_tc_ret_108_filter_only_search_parameterizes_kb_limit_and_offset():
+ builder = GaussDBSearchBuilder(schema="public")
+ kb_id = "k1' OR TRUE --"
+ sql, params = builder.build_search_sql(
+ table="ragflow_tenant",
+ select_fields=["id", "docnm_kwd"],
+ condition={"kb_id": kb_id},
+ keywords=[],
+ vector=None,
+ vector_dim=None,
+ vector_weight=0.0,
+ offset=5,
+ limit=10,
+ order_by=type("Order", (), {"fields": [("docnm_kwd", True)]})(),
+ )
+
+ assert sql == ('SELECT id, kb_id, docnm_kwd, 0.0 AS _score, COUNT(*) OVER() AS __total FROM "public"."ragflow_tenant" WHERE kb_id = %s ORDER BY docnm_kwd DESC LIMIT %s OFFSET %s')
+ assert params == [kb_id, 10, 5]
+ assert kb_id not in sql
+ assert all(expression not in sql for expression in ("to_tsvector", "plainto_tsquery", "ts_rank", "@@"))
+
+
+def test_tc_ret_405_condition_builder_supports_exists_must_not_lists_jsonb_and_null():
+ builder = GaussDBSearchBuilder(schema="public")
+
+ where_sql, params = builder.build_condition_where(
+ {
+ "exists": "doc_id",
+ "must_not": {"exists": "img_id"},
+ "doc_id": ["d1", "d2"],
+ "tag_kwd": ["risk", "audit"],
+ "mom_id": None,
+ }
+ )
+
+ assert "doc_id IS NOT NULL" in where_sql
+ assert "img_id IS NULL" in where_sql
+ assert "doc_id IN (%s, %s)" in where_sql
+ assert "(tag_kwd @> %s::jsonb OR tag_kwd @> %s::jsonb)" in where_sql
+ assert "mom_id IS NULL" in where_sql
+ assert params == ["d1", "d2", '["risk"]', '["audit"]']
+
+
+def test_tc_ret_513_compile_kwd_uses_extra_jsonb_mapping():
+ builder = GaussDBSearchBuilder(schema="public")
+
+ sql, params = builder.build_search_sql(
+ table="ragflow_tenant",
+ select_fields=["compile_kwd"],
+ condition={"kb_id": "kb1", "must_not": {"exists": "compile_kwd"}},
+ keywords=[],
+ vector=None,
+ vector_dim=None,
+ vector_weight=0.0,
+ offset=0,
+ limit=10,
+ )
+
+ assert "(extra #>> '{compile_kwd}') AS compile_kwd" in sql
+ assert "(extra #>> '{compile_kwd}') IS NULL" in sql
+ assert params == ["kb1", 10, 0]
+
+
+def test_tc_ret_406_condition_builder_rejects_empty_list_conditions():
+ builder = GaussDBSearchBuilder(schema="public")
+
+ with pytest.raises(ValueError, match="empty condition values"):
+ builder.build_condition_where({"id": []})
+
+
+def test_tc_ret_208_vector_search_rejects_missing_or_mismatched_dimensions():
+ builder = GaussDBSearchBuilder(schema="public")
+
+ with pytest.raises(ValueError, match="^vector_dim is required for vector search$"):
+ builder.build_search_sql(
+ table="ragflow_tenant",
+ select_fields=["id"],
+ condition={"kb_id": "kb1"},
+ keywords=[],
+ vector=[0.1],
+ vector_dim=None,
+ vector_weight=1.0,
+ offset=0,
+ limit=5,
+ )
+
+ with pytest.raises(ValueError, match="^vector dimension mismatch: expected 2, got 1$"):
+ builder.build_search_sql(
+ table="ragflow_tenant",
+ select_fields=["id"],
+ condition={"kb_id": "kb1"},
+ keywords=[],
+ vector=[0.1],
+ vector_dim=2,
+ vector_weight=1.0,
+ offset=0,
+ limit=5,
+ )
+
+
+def test_tc_ret_501_select_empty_defaults_to_id_kb_id():
+ builder = GaussDBSearchBuilder(schema="public")
+
+ assert builder.normalize_select_fields([]) == ["id", "kb_id"]
+
+ sql, params = builder.build_search_sql(
+ table="ragflow_tenant",
+ select_fields=[],
+ condition={"kb_id": "k1"},
+ keywords=[],
+ vector=None,
+ vector_dim=None,
+ vector_weight=0.0,
+ offset=0,
+ limit=10,
+ )
+
+ assert sql == ('SELECT id, kb_id, 0.0 AS _score, COUNT(*) OVER() AS __total FROM "public"."ragflow_tenant" WHERE kb_id = %s ORDER BY kb_id ASC, id ASC LIMIT %s OFFSET %s')
+ assert "SELECT *" not in sql
+ assert params == ["k1", 10, 0]
+
+
+def test_tc_ret_502_select_fields_add_ids_skip_score_and_deduplicate():
+ builder = GaussDBSearchBuilder(schema="public")
+
+ assert builder.normalize_select_fields(["_score", "content_with_weight", "content_with_weight"]) == [
+ "id",
+ "kb_id",
+ "content_with_weight",
+ ]
+
+ sql, params = builder.build_search_sql(
+ table="ragflow_tenant",
+ select_fields=["_score", "content_with_weight", "content_with_weight"],
+ condition={"kb_id": "k1"},
+ keywords=[],
+ vector=None,
+ vector_dim=None,
+ vector_weight=0.0,
+ offset=0,
+ limit=10,
+ )
+
+ assert sql.startswith("SELECT id, kb_id, content_with_weight, 0.0 AS _score")
+ assert sql.count("content_with_weight") == 1
+ assert params == ["k1", 10, 0]
+
+
+def test_tc_ret_503_select_star_defaults_to_id_kb_id():
+ builder = GaussDBSearchBuilder(schema="public")
+
+ assert builder.normalize_select_fields(["*"]) == ["id", "kb_id"]
+
+ sql, params = builder.build_search_sql(
+ table="ragflow_tenant",
+ select_fields=["*"],
+ condition={"kb_id": "k1"},
+ keywords=[],
+ vector=None,
+ vector_dim=None,
+ vector_weight=0.0,
+ offset=0,
+ limit=10,
+ )
+
+ assert sql == ('SELECT id, kb_id, 0.0 AS _score, COUNT(*) OVER() AS __total FROM "public"."ragflow_tenant" WHERE kb_id = %s ORDER BY kb_id ASC, id ASC LIMIT %s OFFSET %s')
+ assert "SELECT *" not in sql
+ assert params == ["k1", 10, 0]
+
+
+def test_tc_ret_504_selecting_vector_column_auto_includes_valid_flag():
+ builder = GaussDBSearchBuilder(schema="public")
+
+ assert builder.normalize_select_fields(["q_1024_vec"]) == ["id", "kb_id", "q_1024_vec", "q_1024_vec_valid"]
+
+
+def test_tc_ret_702_zero_pagerank_weight_does_not_add_pagerank_score_component():
+ builder = GaussDBSearchBuilder(schema="public")
+ sql, params = builder.build_search_sql(
+ table="ragflow_tenant",
+ select_fields=["id"],
+ condition={"kb_id": "k1"},
+ keywords=["hello"],
+ vector=None,
+ vector_dim=None,
+ vector_weight=0.0,
+ offset=0,
+ limit=10,
+ pagerank_weight=0.0,
+ )
+
+ assert "pagerank_fea" not in sql
+ assert "COALESCE(pagerank_fea" not in sql
+ assert params == ["hello"] * 6 + ["k1", "hello", 10, 0]
+
+
+def test_tc_ret_512_compatibility_fields_use_safe_mapping_and_deterministic_orders():
+ builder = GaussDBSearchBuilder(schema="public")
+ sql, params = builder.build_search_sql(
+ table="ragflow_tenant",
+ select_fields=["row_id()", "chunk_order_int"],
+ condition={"kb_id": "kb1"},
+ keywords=[],
+ vector=None,
+ vector_dim=None,
+ vector_weight=0.0,
+ offset=0,
+ limit=5,
+ order_by=type("Order", (), {"fields": [("position_int", False)]})(),
+ )
+
+ assert builder.build_position_order_sql() == (
+ "COALESCE((page_num_int #>> '{0}')::int, 100000000) ASC, COALESCE((position_int #>> '{0,3}')::int, 100000000) ASC, COALESCE((top_int #>> '{0}')::int, 100000000) ASC"
+ )
+ assert sql == (
+ 'SELECT id, kb_id, NULL AS "row_id()", _order_id AS chunk_order_int, '
+ '0.0 AS _score, COUNT(*) OVER() AS __total FROM "public"."ragflow_tenant" '
+ "WHERE kb_id = %s ORDER BY "
+ "COALESCE((page_num_int #>> '{0}')::int, 100000000) ASC, "
+ "COALESCE((position_int #>> '{0,3}')::int, 100000000) ASC, "
+ "COALESCE((top_int #>> '{0}')::int, 100000000) ASC LIMIT %s OFFSET %s"
+ )
+ assert params == ["kb1", 5, 0]
+ assert "row_id() AS" not in sql
+ assert "chunk_order_int AS" not in sql
+
+ default_sql, default_params = builder.build_search_sql(
+ table="ragflow_tenant",
+ select_fields=["row_id()", "chunk_order_int"],
+ condition={"kb_id": "kb1"},
+ keywords=[],
+ vector=None,
+ vector_dim=None,
+ vector_weight=0.0,
+ offset=0,
+ limit=5,
+ )
+ assert "ORDER BY kb_id ASC, id ASC LIMIT %s OFFSET %s" in default_sql
+ assert default_params == ["kb1", 5, 0]
+
+ with pytest.raises(ValueError, match="unsafe_field"):
+ builder.build_search_sql(
+ table="ragflow_tenant",
+ select_fields=["unsafe_field; DROP TABLE chunks"],
+ condition={"kb_id": "kb1"},
+ keywords=[],
+ vector=None,
+ vector_dim=None,
+ vector_weight=0.0,
+ offset=0,
+ limit=5,
+ )
diff --git a/test/unit_test/rag/utils/test_gaussdb_sql_validator.py b/test/unit_test/rag/utils/test_gaussdb_sql_validator.py
new file mode 100644
index 0000000000..5c71cd8b12
--- /dev/null
+++ b/test/unit_test/rag/utils/test_gaussdb_sql_validator.py
@@ -0,0 +1,887 @@
+#
+# Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+import pytest
+
+from common.doc_store import gaussdb_conn_base as gaussdb_base
+from common.doc_store.gaussdb_conn_base import (
+ ExposedGaussDBTable,
+ GaussDBSQLValidator,
+ UnsafeGaussDBSQL,
+ is_gaussdb_aggregate_sql,
+ jsonb_path_literal,
+ mask_gaussdb_text,
+)
+
+
+def _validator(kb_ids=None, field_map=None, fetch_size=128):
+ table = ExposedGaussDBTable.from_field_map(
+ physical_name="ragflow_t1",
+ kb_ids=["kb-1"] if kb_ids is None else kb_ids,
+ field_map={"amount": "number", "dept": "string", "customer,name": "string"} if field_map is None else field_map,
+ )
+ return GaussDBSQLValidator({table.logical_name: table}, default_limit=fetch_size)
+
+
+def _assert_unsafe_sql(sql, message, validator=None):
+ with pytest.raises(UnsafeGaussDBSQL) as exc_info:
+ (validator or _validator()).validate_and_patch(sql)
+
+ assert str(exc_info.value) == message
+
+
+def test_tc_cfg_711_mask_gaussdb_text_redacts_quoted_unquoted_and_delimited_secrets():
+ raw = (
+ "password=plain,semi;value "
+ "passwd='quoted, semi; value' "
+ 'token="token, semi; value" '
+ '"access_token": "access, semi; value", '
+ "api_key=api,semi;value "
+ "secret='secret, semi; value' "
+ "dsn=postgresql://user:[email protected]:5432/ragflow?options=a,b;c"
+ )
+
+ assert mask_gaussdb_text(raw) == ('password=*** passwd=\'***\' token="***" "access_token": "***", api_key=*** secret=\'***\' dsn=***')
+ assert mask_gaussdb_text("uri=gaussdb://user:secret@host/db?x=a,b;c") == "uri=***"
+ assert mask_gaussdb_text("password='quoted ''value,semi;tail' next=ok") == "password='***' next=ok"
+ assert mask_gaussdb_text('token="unterminated, semi; value') == "token=***"
+
+
+def test_tc_sql_412_jsonb_path_literal_encodes_special_keys():
+ assert jsonb_path_literal(["amount"]) == "'{amount}'"
+ assert jsonb_path_literal(["customer", "name"]) == "'{customer,name}'"
+ assert jsonb_path_literal(["customer,name"]) == "'{\"customer,name\"}'"
+ assert jsonb_path_literal(['customer"name']) == '\'{"customer\\"name"}\''
+ assert jsonb_path_literal(["customer\\name"]) == "'{\"customer\\\\name\"}'"
+ assert jsonb_path_literal(["customer name"]) == "'{\"customer name\"}'"
+ assert jsonb_path_literal(["\u5ba2\u6237"]) == "'{\"\u5ba2\u6237\"}'"
+ assert jsonb_path_literal(["O'Brien"]) == "'{\"O''Brien\"}'"
+
+
+def test_tc_sql_412_jsonb_path_literal_rejects_empty_path_or_segment():
+ with pytest.raises(UnsafeGaussDBSQL) as empty_path:
+ jsonb_path_literal([])
+ with pytest.raises(UnsafeGaussDBSQL) as empty_segment:
+ jsonb_path_literal(["amount", ""])
+ with pytest.raises(UnsafeGaussDBSQL) as invalid_segment:
+ jsonb_path_literal([None])
+
+ assert str(empty_path.value) == "empty JSONB path"
+ assert str(empty_segment.value) == "empty JSONB path segment"
+ assert str(invalid_segment.value) == "invalid JSONB path segment"
+
+
+def test_tc_sql_412_jsonb_path_literal_parser_handles_escapes_and_invalid_shapes():
+ assert gaussdb_base._parse_jsonb_path_literal("'{\"customer,name\",dept}'") == ("customer,name", "dept")
+ assert gaussdb_base._parse_jsonb_path_literal("'{\"customer\\\\name\"}'") == ("customer\\name",)
+
+ invalid_literals = {
+ "amount": "dynamic JSONB path is not allowed",
+ "'{amount'": "dynamic JSONB path is not allowed",
+ "'{amount,}'": "invalid JSONB path literal",
+ "'{\"amount}'": "invalid JSONB path literal",
+ }
+ for literal, message in invalid_literals.items():
+ with pytest.raises(UnsafeGaussDBSQL) as exc_info:
+ gaussdb_base._parse_jsonb_path_literal(literal)
+ assert str(exc_info.value) == message
+
+
+def test_tc_sql_101_validator_allows_single_table_select_and_returns_columns():
+ validated = _validator(field_map={}).validate_and_patch("SELECT doc_id, docnm_kwd FROM ragflow_t1 WHERE kb_id = 'kb-1' ORDER BY doc_id LIMIT 5")
+
+ assert validated.sql == ("SELECT doc_id, docnm_kwd FROM ragflow_t1 WHERE kb_id = 'kb-1' ORDER BY doc_id LIMIT 5")
+ assert validated.columns == ["doc_id", "docnm_kwd"]
+ assert validated.is_aggregation is False
+
+
+def test_tc_sql_102_validator_allows_readonly_cte_and_exposed_output_aliases():
+ baseline = _validator(field_map={}).validate_and_patch("WITH x AS (SELECT doc_id FROM ragflow_t1 WHERE kb_id = 'kb-1') SELECT doc_id FROM x")
+ alias = _validator(field_map={"amount": "number"}).validate_and_patch(
+ "WITH rows AS (SELECT chunk_data #>> '{amount}' AS amount, kb_id FROM ragflow_t1 WHERE kb_id = 'kb-1') SELECT amount FROM rows"
+ )
+ qualified_alias = _validator(field_map={"amount": "number"}).validate_and_patch(
+ "WITH rows AS (SELECT chunk_data #>> '{amount}' AS amount, kb_id FROM ragflow_t1 WHERE kb_id = 'kb-1') SELECT rows.amount FROM rows"
+ )
+
+ assert baseline.sql == ("WITH x AS (SELECT doc_id FROM ragflow_t1 WHERE kb_id = 'kb-1') SELECT doc_id FROM x LIMIT 128")
+ assert baseline.columns == ["doc_id"]
+ assert baseline.is_aggregation is False
+ assert alias.sql == ("WITH rows AS (SELECT chunk_data #>> '{amount}' AS amount, kb_id FROM ragflow_t1 WHERE kb_id = 'kb-1') SELECT amount FROM rows LIMIT 128")
+ assert alias.columns == ["amount"]
+ assert alias.is_aggregation is False
+ assert qualified_alias.sql == ("WITH rows AS (SELECT chunk_data #>> '{amount}' AS amount, kb_id FROM ragflow_t1 WHERE kb_id = 'kb-1') SELECT rows.amount FROM rows LIMIT 128")
+ assert qualified_alias.columns == ["amount"]
+ assert qualified_alias.is_aggregation is False
+
+ validator = _validator(field_map={})
+ list_validator = GaussDBSQLValidator(["ragflow_t1"], kb_ids=["kb-1"])
+ assert list_validator.tables["ragflow_t1"].required_kb_ids == ("kb-1",)
+ values_cte = validator._parse_one("WITH x AS (SELECT 1 UNION SELECT 2) SELECT 1")
+ assert validator._cte_output_columns(values_cte) == {}
+ from sqlglot import exp
+
+ assert validator._is_cte_output_column(exp.column("doc_id"), {}) is False
+
+
+def test_tc_sql_103_validator_rejects_multiple_statements_and_wraps_parser_failures():
+ _assert_unsafe_sql(
+ "SELECT doc_id FROM ragflow_t1 WHERE kb_id = 'kb-1'; SELECT 1",
+ "multiple statements are not allowed",
+ )
+
+ validator = _validator()
+ with pytest.raises(UnsafeGaussDBSQL, match="exactly one SQL statement is allowed"):
+ validator._parse_one("SELECT 1; SELECT 2")
+
+ with pytest.raises(UnsafeGaussDBSQL) as parse_error:
+ validator._parse_one("SELECT (")
+ with pytest.raises(UnsafeGaussDBSQL) as aggregate_parse_error:
+ is_gaussdb_aggregate_sql("SELECT (")
+
+ assert str(parse_error.value) == "SQL parse failed"
+ assert str(aggregate_parse_error.value) == "SQL parse failed"
+
+
+def test_tc_sql_104_validator_rejects_dml_including_nested_cte():
+ scenarios = {
+ "INSERT INTO ragflow_t1 VALUES (1)": "only SELECT statements are allowed",
+ "UPDATE ragflow_t1 SET doc_id = 'new'": "only SELECT statements are allowed",
+ "DELETE FROM ragflow_t1": "only SELECT statements are allowed",
+ "WITH x AS (DELETE FROM ragflow_t1 RETURNING doc_id) SELECT doc_id FROM x": ("SQL contains a non-read-only expression"),
+ }
+ for sql, message in scenarios.items():
+ _assert_unsafe_sql(sql, message)
+
+
+def test_tc_sql_105_validator_rejects_ddl():
+ for sql in (
+ "CREATE TABLE test (id int)",
+ "DROP TABLE ragflow_t1",
+ "ALTER TABLE ragflow_t1 ADD COLUMN x int",
+ ):
+ _assert_unsafe_sql(sql, "only SELECT statements are allowed")
+
+
+def test_tc_sql_106_validator_rejects_call():
+ _assert_unsafe_sql("CALL my_proc()", "only SELECT statements are allowed")
+
+
+def test_tc_sql_107_validator_rejects_copy():
+ _assert_unsafe_sql("COPY ragflow_t1 TO '/tmp/out.csv'", "only SELECT statements are allowed")
+
+
+ "sql",
+ [
+ "MERGE INTO ragflow_t1 t USING source_table s ON t.doc_id = s.doc_id WHEN MATCHED THEN DELETE",
+ "TRUNCATE TABLE ragflow_t1",
+ "ANALYZE ragflow_t1",
+ "VACUUM ragflow_t1",
+ "GRANT SELECT ON ragflow_t1 TO app_user",
+ "REVOKE SELECT ON ragflow_t1 FROM app_user",
+ "LOCK TABLE ragflow_t1 IN ACCESS EXCLUSIVE MODE",
+ "DO $$ BEGIN NULL; END $$",
+ ],
+ ids=["merge", "truncate", "analyze", "vacuum", "grant", "revoke", "lock", "do"],
+)
+def test_tc_sql_130_validator_rejects_additional_write_and_admin_statements(sql):
+ with pytest.raises(UnsafeGaussDBSQL) as exc_info:
+ _validator(field_map={}).validate_and_patch(sql)
+
+ assert str(exc_info.value) in {
+ "only SELECT statements are allowed",
+ "multiple statements are not allowed",
+ "SQL parse failed",
+ }
+
+
+def test_tc_sql_108_validator_rejects_select_star():
+ _assert_unsafe_sql("SELECT * FROM ragflow_t1 WHERE kb_id = 'kb-1'", "SELECT * is not allowed")
+
+
+def test_tc_sql_109_validator_allows_count_star_and_marks_aggregation():
+ validated = _validator(field_map={}).validate_and_patch("SELECT COUNT(*) AS cnt FROM ragflow_t1")
+
+ assert validated.sql == "SELECT COUNT(*) AS cnt FROM ragflow_t1 WHERE kb_id = 'kb-1' LIMIT 128"
+ assert validated.columns == ["cnt"]
+ assert validated.is_aggregation is True
+
+
+def test_tc_sql_110_validator_rejects_rank_and_row_number_windows():
+ scenarios = (
+ "SELECT doc_id, RANK() OVER (ORDER BY chunk_data #>> '{amount}') AS rnk FROM ragflow_t1",
+ "SELECT doc_id, ROW_NUMBER() OVER (PARTITION BY kb_id ORDER BY doc_id) AS rn FROM ragflow_t1",
+ )
+ for sql in scenarios:
+ _assert_unsafe_sql(sql, "window functions are not allowed")
+
+
+def test_tc_sql_111_validator_rejects_system_table_and_cross_schema_table():
+ for sql in (
+ "SELECT doc_id FROM pg_catalog.tables",
+ "SELECT doc_id FROM other_schema.ragflow_t1",
+ ):
+ _assert_unsafe_sql(sql, "cross-schema SQL is not allowed")
+
+
+def test_tc_sql_112_validator_rejects_non_whitelisted_table():
+ _assert_unsafe_sql(
+ "SELECT doc_id FROM unknown_table WHERE kb_id = 'kb-1'",
+ "table unknown_table is not allowed",
+ )
+
+
+def test_tc_sql_113_validator_rejects_non_whitelisted_column():
+ _assert_unsafe_sql(
+ "SELECT content_with_weight FROM ragflow_t1 WHERE kb_id = 'kb-1'",
+ "column content_with_weight is not allowed",
+ )
+
+
+def test_tc_sql_114_validator_rejects_forbidden_system_functions():
+ scenarios = {
+ "SELECT pg_sleep(1) FROM ragflow_t1": "function pg_sleep is not allowed",
+ "SELECT now() FROM ragflow_t1": "system functions are not allowed",
+ "SELECT current_user FROM ragflow_t1": "system functions are not allowed",
+ "SELECT version() FROM ragflow_t1": "function version is not allowed",
+ }
+ for sql, message in scenarios.items():
+ _assert_unsafe_sql(sql, message, _validator(field_map={}))
+
+
+def test_tc_sql_115_validator_rejects_oceanbase_json_functions():
+ scenarios = {
+ "SELECT json_extract_string(chunk_data, '$.amount') FROM ragflow_t1": ("function json_extract_string is not allowed"),
+ "SELECT json_extract(chunk_data, '$.amount') FROM ragflow_t1": ("only GaussDB #> / #>> JSONB operators are allowed"),
+ "SELECT json_extract_isnull(chunk_data, '$.amount') FROM ragflow_t1": ("function json_extract_isnull is not allowed"),
+ }
+ for sql, message in scenarios.items():
+ _assert_unsafe_sql(sql, message)
+
+
+def test_tc_sql_116_validator_rejects_forbidden_jsonb_set_returning_functions():
+ scenarios = {
+ "SELECT jsonb_each(chunk_data) FROM ragflow_t1": "function jsonb_each is not allowed",
+ "SELECT jsonb_each_text(chunk_data) FROM ragflow_t1": "function jsonb_each_text is not allowed",
+ "SELECT jsonb_array_elements(chunk_data) FROM ragflow_t1": ("function jsonb_array_elements is not allowed"),
+ "SELECT jsonb_to_recordset(chunk_data) FROM ragflow_t1": ("function jsonb_to_recordset is not allowed"),
+ }
+ for sql, message in scenarios.items():
+ _assert_unsafe_sql(sql, message, _validator(field_map={}))
+
+
+def test_tc_sql_117_validator_rejects_join_even_when_both_aliases_are_scoped():
+ _assert_unsafe_sql(
+ "SELECT a.doc_id FROM ragflow_t1 a JOIN ragflow_t1 b ON a.doc_id = b.doc_id WHERE a.kb_id = 'kb-1' AND b.kb_id = 'kb-1'",
+ "complex SQL must use a simpler single-table kb_id boundary",
+ _validator(field_map={}),
+ )
+
+
+def test_tc_sql_118_validator_rejects_union_as_non_select_root():
+ _assert_unsafe_sql(
+ "SELECT doc_id FROM ragflow_t1 WHERE kb_id = 'kb-1' UNION SELECT doc_id FROM ragflow_t1 WHERE kb_id = 'kb-2'",
+ "only SELECT statements are allowed",
+ _validator(field_map={}),
+ )
+
+
+def test_tc_sql_119_validator_rejects_or_predicate():
+ _assert_unsafe_sql(
+ "SELECT doc_id FROM ragflow_t1 WHERE kb_id = 'kb-1' OR chunk_data #>> '{amount}' = '100'",
+ "complex SQL must use a simpler single-table kb_id boundary",
+ )
+
+
+def test_tc_sql_120_validator_rejects_dynamic_or_wrong_source_jsonb_paths():
+ scenarios = {
+ "SELECT doc_id FROM ragflow_t1 WHERE chunk_data #>> ('{' || param || '}') = '100'": ("dynamic JSONB path is not allowed"),
+ "SELECT doc_id, chunk_data #>> path_col FROM ragflow_t1 WHERE kb_id = 'kb-1'": ("dynamic JSONB path is not allowed"),
+ "SELECT doc_id, other_data #>> '{amount}' FROM ragflow_t1 WHERE kb_id = 'kb-1'": ("only chunk_data JSONB paths are allowed"),
+ }
+ for sql, message in scenarios.items():
+ _assert_unsafe_sql(sql, message)
+
+
+def test_tc_sql_121_validator_rejects_empty_sql_variants():
+ for sql in ("", " ", "```sql```"):
+ _assert_unsafe_sql(sql, "empty SQL")
+
+
+def test_tc_sql_122_validator_rejects_explain():
+ _assert_unsafe_sql(
+ "EXPLAIN SELECT doc_id FROM ragflow_t1 WHERE kb_id = 'kb-1'",
+ "only SELECT statements are allowed",
+ )
+
+
+def test_tc_sql_123_validator_rejects_set_command():
+ _assert_unsafe_sql("SET statement_timeout = 30000", "only SELECT statements are allowed")
+
+
+def test_tc_sql_124_validator_rejects_show_command():
+ _assert_unsafe_sql("SHOW statement_timeout", "only SELECT statements are allowed")
+
+
+ "sql",
+ [
+ "SELECT doc_id FROM ragflow_t1, LATERAL jsonb_each(chunk_data) AS j",
+ "SELECT doc_id FROM ragflow_t1 CROSS JOIN LATERAL (SELECT 1) AS j",
+ "SELECT doc_id FROM ragflow_t1 LEFT JOIN LATERAL (SELECT 1) AS j ON TRUE",
+ ],
+ ids=["set-returning-function", "cross-join-subquery", "left-join-subquery"],
+)
+def test_tc_sql_125_validator_rejects_every_lateral_shape(sql):
+ _assert_unsafe_sql(
+ sql,
+ "LATERAL is not allowed",
+ _validator(field_map={}),
+ )
+
+
+def test_tc_sql_126_validator_rejects_jsonb_path_query():
+ _assert_unsafe_sql(
+ "SELECT doc_id FROM ragflow_t1 WHERE jsonb_path_query(chunk_data, '$.amount') IS NOT NULL",
+ "function jsonb_path_query is not allowed",
+ _validator(field_map={}),
+ )
+
+
+def test_tc_sql_127_validator_rejects_jsonb_to_record():
+ _assert_unsafe_sql(
+ "SELECT doc_id FROM ragflow_t1, jsonb_to_record(chunk_data) AS t(amount text)",
+ "function jsonb_to_record is not allowed",
+ _validator(field_map={}),
+ )
+
+
+def test_tc_sql_128_validator_rejects_generate_series():
+ _assert_unsafe_sql(
+ "SELECT doc_id FROM ragflow_t1, generate_series(1, 10) AS g",
+ "function generate_series is not allowed",
+ _validator(field_map={}),
+ )
+
+
+def test_tc_sql_129_validator_rejects_unnest():
+ _assert_unsafe_sql(
+ "SELECT doc_id FROM ragflow_t1, unnest(ARRAY[1, 2, 3]) AS u",
+ "function unnest is not allowed",
+ _validator(field_map={}),
+ )
+
+
+def test_tc_sql_201_validator_injects_kb_id_for_simple_single_table_query():
+ validator = _validator(field_map={})
+ validated = validator.validate_and_patch("SELECT doc_id FROM ragflow_t1")
+
+ assert validated.sql == "SELECT doc_id FROM ragflow_t1 WHERE kb_id = 'kb-1' LIMIT 128"
+ assert validated.columns == ["doc_id"]
+ assert validated.is_aggregation is False
+
+
+def test_tc_sql_202_validator_injects_multi_kb_boundary():
+ validated = _validator(kb_ids=["kb-1", "kb-2"], field_map={}).validate_and_patch("SELECT doc_id FROM ragflow_t1")
+
+ assert validated.sql == ("SELECT doc_id FROM ragflow_t1 WHERE kb_id IN ('kb-1', 'kb-2') LIMIT 128")
+ assert validated.columns == ["doc_id"]
+ assert validated.is_aggregation is False
+
+
+def test_tc_sql_203_validator_preserves_allowed_kb_boundary_shapes():
+ equality = _validator(field_map={}).validate_and_patch("SELECT doc_id FROM ragflow_t1 WHERE kb_id = 'kb-1'")
+ reversed_equality = _validator(field_map={}).validate_and_patch("SELECT doc_id FROM ragflow_t1 WHERE 'kb-1' = kb_id")
+ in_boundary = _validator(kb_ids=["kb-1", "kb-2"], field_map={}).validate_and_patch("SELECT doc_id FROM ragflow_t1 WHERE kb_id IN ('kb-1', 'kb-2')")
+ predicate_before_boundary = _validator(kb_ids=["kb-1", "kb-2"], field_map={}).validate_and_patch("SELECT doc_id FROM ragflow_t1 WHERE doc_id IN ('doc1', 'doc2') AND kb_id IN ('kb-1')")
+
+ assert equality.sql == "SELECT doc_id FROM ragflow_t1 WHERE kb_id = 'kb-1' LIMIT 128"
+ assert reversed_equality.sql == "SELECT doc_id FROM ragflow_t1 WHERE 'kb-1' = kb_id LIMIT 128"
+ assert in_boundary.sql == ("SELECT doc_id FROM ragflow_t1 WHERE kb_id IN ('kb-1', 'kb-2') LIMIT 128")
+ assert predicate_before_boundary.sql == ("SELECT doc_id FROM ragflow_t1 WHERE doc_id IN ('doc1', 'doc2') AND kb_id IN ('kb-1') LIMIT 128")
+ for validated in (equality, reversed_equality, in_boundary, predicate_before_boundary):
+ assert validated.columns == ["doc_id"]
+ assert validated.is_aggregation is False
+
+
+def test_tc_sql_204_validator_rejects_cross_kb_boundary():
+ _assert_unsafe_sql(
+ "SELECT doc_id FROM ragflow_t1 WHERE kb_id = 'kb-other'",
+ "SQL crosses the allowed kb_id boundary",
+ _validator(field_map={}),
+ )
+
+
+def test_tc_sql_205_validator_rejects_non_positive_or_dynamic_kb_predicates():
+ scenarios = {
+ "SELECT doc_id FROM ragflow_t1 WHERE NOT kb_id = 'kb-1'": ("kb_id boundary must be a positive top-level predicate"),
+ "SELECT doc_id FROM ragflow_t1 WHERE (kb_id = 'kb-1') IS FALSE": ("kb_id boundary must be a positive top-level predicate"),
+ "SELECT doc_id FROM ragflow_t1 WHERE kb_id IN (doc_id)": ("kb_id IN must use static string literals"),
+ "SELECT doc_id FROM ragflow_t1 WHERE kb_id IN ()": "SQL crosses the allowed kb_id boundary",
+ }
+ for sql, message in scenarios.items():
+ _assert_unsafe_sql(sql, message, _validator(field_map={}))
+
+ validator = _validator()
+ from sqlglot import exp
+
+ in_ast = validator._parse_one("SELECT doc_id FROM ragflow_t1 WHERE kb_id IN ('kb-1')")
+ assert validator._select_has_static_kb_boundary(in_ast) is True
+ no_boundary_ast = validator._parse_one("SELECT doc_id FROM ragflow_t1 WHERE doc_id = 'd1'")
+ assert validator._select_has_static_kb_boundary(no_boundary_ast) is False
+ non_boundary_ast = validator._parse_one("SELECT doc_id FROM ragflow_t1 WHERE NOT doc_id = 'd1'")
+ assert validator._select_has_static_kb_boundary(non_boundary_ast) is False
+ empty_in = exp.In(this=exp.column("kb_id"), expressions=[])
+ empty_ast = exp.select("doc_id").from_("ragflow_t1").where(empty_in)
+ with pytest.raises(UnsafeGaussDBSQL, match="kb_id boundary is empty"):
+ validator._select_has_static_kb_boundary(empty_ast)
+
+
+def test_tc_sql_206_validator_rejects_complex_sql_when_scope_cannot_be_patched():
+ _assert_unsafe_sql(
+ "SELECT a.doc_id FROM ragflow_t1 a JOIN ragflow_t1 b ON a.doc_id = b.doc_id",
+ "complex SQL must use a simpler single-table kb_id boundary",
+ _validator(field_map={}),
+ )
+
+
+def test_tc_sql_207_validator_rejects_second_non_whitelisted_table():
+ _assert_unsafe_sql(
+ "SELECT t1.doc_id FROM ragflow_t1 t1 JOIN ragflow_t2 t2 ON t1.doc_id = t2.doc_id",
+ "table ragflow_t2 is not allowed",
+ _validator(field_map={}),
+ )
+
+ validator = _validator()
+ with pytest.raises(UnsafeGaussDBSQL, match="must read from an exposed"):
+ validator._enforce_kb_boundary("SELECT 1")
+ with pytest.raises(UnsafeGaussDBSQL, match="exactly one base table"):
+ validator._enforce_kb_boundary("SELECT doc_id FROM ragflow_t1, ragflow_t2")
+
+ ast = validator._parse_one("SELECT a.doc_id FROM ragflow_t1 a JOIN ragflow_t2 b ON a.id = b.id")
+ assert validator._direct_base_tables(ast) == ["ragflow_t1", "ragflow_t2"]
+ assert validator._direct_source_tables(ast) == ["ragflow_t1", "ragflow_t2"]
+
+
+def test_tc_sql_208_validator_rejects_missing_or_unsafe_required_kb_boundary():
+ missing_table = ExposedGaussDBTable.from_field_map("ragflow_t1", [], {})
+ missing_validator = GaussDBSQLValidator({"ragflow_t1": missing_table})
+
+ _assert_unsafe_sql(
+ "SELECT doc_id FROM ragflow_t1",
+ "kb_id boundary is required",
+ missing_validator,
+ )
+ _assert_unsafe_sql(
+ "SELECT doc_id FROM ragflow_t1",
+ "unsafe kb_id literal",
+ _validator(kb_ids=["kb-1;drop"], field_map={}),
+ )
+
+
+def test_tc_sql_209_validator_requires_each_cte_base_scope_to_prove_kb_boundary():
+ validator = _validator(field_map={})
+ _assert_unsafe_sql(
+ "WITH x AS (SELECT doc_id FROM ragflow_t1) SELECT doc_id FROM x",
+ "each base table scope must include a kb_id boundary",
+ validator,
+ )
+ _assert_unsafe_sql(
+ "WITH rows AS (SELECT doc_id, 'kb-1' AS kb_id FROM ragflow_t1) SELECT doc_id FROM rows WHERE kb_id = 'kb-1'",
+ "each base table scope must include a kb_id boundary",
+ validator,
+ )
+
+ validated = validator.validate_and_patch("WITH x AS (SELECT doc_id FROM ragflow_t1 WHERE kb_id = 'kb-1') SELECT doc_id FROM x")
+ assert validated.sql == ("WITH x AS (SELECT doc_id FROM ragflow_t1 WHERE kb_id = 'kb-1') SELECT doc_id FROM x LIMIT 128")
+ assert validated.columns == ["doc_id"]
+ assert validated.is_aggregation is False
+
+
+ "correlation_predicate",
+ ["", "inner_t.doc_id = outer_t.doc_id AND "],
+ ids=["non-correlated", "correlated"],
+)
+def test_tc_sql_209_validator_requires_each_subquery_scope_to_prove_kb_boundary(correlation_predicate):
+ validator = _validator(field_map={})
+ outer_missing = f"SELECT outer_t.doc_id FROM ragflow_t1 AS outer_t WHERE EXISTS (SELECT 1 FROM ragflow_t1 AS inner_t WHERE {correlation_predicate}inner_t.kb_id = 'kb-1')"
+ inner_missing = (
+ f"SELECT outer_t.doc_id FROM ragflow_t1 AS outer_t WHERE outer_t.kb_id = 'kb-1' AND EXISTS (SELECT 1 FROM ragflow_t1 AS inner_t WHERE {correlation_predicate}inner_t.doc_id = 'doc-1')"
+ )
+ legal = f"SELECT outer_t.doc_id FROM ragflow_t1 AS outer_t WHERE outer_t.kb_id = 'kb-1' AND EXISTS (SELECT 1 FROM ragflow_t1 AS inner_t WHERE {correlation_predicate}inner_t.kb_id = 'kb-1')"
+ cross_kb = f"SELECT outer_t.doc_id FROM ragflow_t1 AS outer_t WHERE outer_t.kb_id = 'kb-1' AND EXISTS (SELECT 1 FROM ragflow_t1 AS inner_t WHERE {correlation_predicate}inner_t.kb_id = 'kb-other')"
+
+ for sql in (outer_missing, inner_missing):
+ _assert_unsafe_sql(sql, "each base table scope must include a kb_id boundary", validator)
+ _assert_unsafe_sql(cross_kb, "SQL crosses the allowed kb_id boundary", validator)
+
+ validated = validator.validate_and_patch(legal)
+ assert validated.sql.endswith("LIMIT 128")
+ assert validated.sql.count("kb_id = 'kb-1'") == 2
+ assert validated.columns == ["doc_id"]
+ assert validated.is_aggregation is False
+
+
+def test_tc_sql_301_validator_adds_default_limit_when_missing():
+ validated = _validator(field_map={}).validate_and_patch("SELECT doc_id FROM ragflow_t1 WHERE kb_id = 'kb-1'")
+
+ assert validated.sql == "SELECT doc_id FROM ragflow_t1 WHERE kb_id = 'kb-1' LIMIT 128"
+ assert validated.columns == ["doc_id"]
+ assert validated.is_aggregation is False
+ no_limit_validator = GaussDBSQLValidator(_validator().tables, default_limit=-1)
+ assert "LIMIT" not in no_limit_validator.validate_and_patch("SELECT doc_id FROM ragflow_t1 WHERE kb_id = 'kb-1'").sql
+
+
+def test_tc_sql_302_validator_caps_large_limit_to_default_limit():
+ validated = _validator(field_map={}).validate_and_patch("SELECT doc_id FROM ragflow_t1 WHERE kb_id = 'kb-1' LIMIT 200")
+
+ assert validated.sql == "SELECT doc_id FROM ragflow_t1 WHERE kb_id = 'kb-1' LIMIT 128"
+ assert validated.columns == ["doc_id"]
+ assert validated.is_aggregation is False
+
+
+def test_tc_sql_303_validator_preserves_limit_under_default_limit():
+ validated = _validator(field_map={}).validate_and_patch("SELECT doc_id FROM ragflow_t1 WHERE kb_id = 'kb-1' LIMIT 100")
+
+ assert validated.sql == "SELECT doc_id FROM ragflow_t1 WHERE kb_id = 'kb-1' LIMIT 100"
+ assert validated.columns == ["doc_id"]
+ assert validated.is_aggregation is False
+
+
+ "sql",
+ [
+ "SELECT doc_id FROM ragflow_t1 WHERE kb_id = 'kb-1' LIMIT 0",
+ "SELECT doc_id FROM ragflow_t1 WHERE kb_id = 'kb-1' LIMIT -1",
+ "SELECT doc_id FROM ragflow_t1 WHERE kb_id = 'kb-1' LIMIT 1.5",
+ "SELECT doc_id FROM ragflow_t1 WHERE kb_id = 'kb-1' LIMIT $1",
+ "SELECT doc_id FROM ragflow_t1 WHERE kb_id = 'kb-1' LIMIT doc_id",
+ "SELECT doc_id FROM ragflow_t1 WHERE kb_id = 'kb-1' LIMIT '1'",
+ "SELECT doc_id FROM ragflow_t1 WHERE kb_id = 'kb-1' FETCH FIRST 0 ROWS ONLY",
+ "SELECT doc_id FROM ragflow_t1 WHERE kb_id = 'kb-1' AND EXISTS (SELECT 1 LIMIT 0)",
+ ],
+ ids=[
+ "zero",
+ "negative",
+ "decimal",
+ "parameter",
+ "column",
+ "string",
+ "zero-fetch",
+ "nested-zero",
+ ],
+)
+def test_tc_sql_304_validator_rejects_non_positive_dynamic_or_non_integer_limits(sql):
+ _assert_unsafe_sql(sql, "LIMIT must be a positive static integer", _validator(field_map={}))
+
+
+def test_tc_sql_305_validator_rejects_complex_scope_before_limit_patch():
+ validator = _validator(field_map={"status": "string"})
+ scenarios = (
+ "SELECT a.doc_id FROM ragflow_t1 a JOIN ragflow_t1 b ON a.doc_id = b.doc_id LIMIT 500",
+ "SELECT doc_id FROM ragflow_t1 WHERE kb_id = 'kb-1' OR chunk_data #>> '{status}' = 'active' LIMIT 500",
+ )
+ for sql in scenarios:
+ _assert_unsafe_sql(
+ sql,
+ "complex SQL must use a simpler single-table kb_id boundary",
+ validator,
+ )
+
+
+def test_tc_sql_306_readonly_guard_uses_fetch_size_as_default_limit():
+ validator = GaussDBSQLValidator.readonly_guard(default_limit=64)
+ validated = validator.validate_and_patch("SELECT doc_id FROM ragflow_t1 WHERE kb_id = 'kb-1'")
+
+ assert validator.default_limit == 64
+ assert validated.sql == "SELECT doc_id FROM ragflow_t1 WHERE kb_id = 'kb-1' LIMIT 64"
+ assert validated.columns == ["doc_id"]
+ assert validated.is_aggregation is False
+
+
+def test_tc_sql_307_validator_caps_fetch_first_to_default_limit():
+ validated = _validator(field_map={}).validate_and_patch("SELECT doc_id FROM ragflow_t1 WHERE kb_id = 'kb-1' FETCH FIRST 500 ROWS ONLY")
+
+ assert validated.sql == ("SELECT doc_id FROM ragflow_t1 WHERE kb_id = 'kb-1' FETCH FIRST 128 ROWS ONLY")
+ assert validated.columns == ["doc_id"]
+ assert validated.is_aggregation is False
+
+
+def test_tc_sql_308_clause_tokens_in_literals_or_subqueries_do_not_block_ast_patches():
+ limit_field = _validator(field_map={"limit": "string"}).validate_and_patch("SELECT doc_id FROM ragflow_t1 WHERE kb_id = 'kb-1' AND chunk_data #>> '{limit}' = '100'")
+ subquery_limit = _validator(field_map={}).validate_and_patch("SELECT doc_id FROM ragflow_t1 WHERE kb_id = 'kb-1' AND EXISTS (SELECT 1 LIMIT 1)")
+ forbidden_word_literal = _validator(field_map={"dept": "string"}).validate_and_patch("SELECT doc_id FROM ragflow_t1 WHERE kb_id = 'kb-1' AND chunk_data #>> '{dept}' = 'update'")
+ order_word_literal = _validator(field_map={"dept": "string"}).validate_and_patch("SELECT doc_id FROM ragflow_t1 WHERE chunk_data #>> '{dept}' = 'order by dept' ORDER BY doc_id")
+
+ assert limit_field.sql == ("SELECT doc_id FROM ragflow_t1 WHERE kb_id = 'kb-1' AND chunk_data #>> '{limit}' = '100' LIMIT 128")
+ assert subquery_limit.sql == ("SELECT doc_id FROM ragflow_t1 WHERE kb_id = 'kb-1' AND EXISTS(SELECT 1 LIMIT 1) LIMIT 128")
+ assert forbidden_word_literal.sql == ("SELECT doc_id FROM ragflow_t1 WHERE kb_id = 'kb-1' AND chunk_data #>> '{dept}' = 'update' LIMIT 128")
+ assert order_word_literal.sql == ("SELECT doc_id FROM ragflow_t1 WHERE chunk_data #>> '{dept}' = 'order by dept' AND kb_id = 'kb-1' ORDER BY doc_id LIMIT 128")
+ for validated in (limit_field, subquery_limit, forbidden_word_literal, order_word_literal):
+ assert validated.columns == ["doc_id"]
+ assert validated.is_aggregation is False
+
+
+def test_tc_sql_401_validator_allows_single_key_jsonb_text_path():
+ validated = _validator(field_map={"amount": "number"}).validate_and_patch("SELECT chunk_data #>> '{amount}' AS amount FROM ragflow_t1 WHERE kb_id = 'kb-1' LIMIT 10")
+
+ assert validated.sql == ("SELECT chunk_data #>> '{amount}' AS amount FROM ragflow_t1 WHERE kb_id = 'kb-1' LIMIT 10")
+ assert validated.columns == ["amount"]
+ assert validated.is_aggregation is False
+
+
+def test_tc_sql_402_validator_allows_multi_key_jsonb_text_path():
+ validated = _validator(field_map={"profile.name": "string"}).validate_and_patch("SELECT doc_id FROM ragflow_t1 WHERE chunk_data #>> '{profile,name}' = 'Alice'")
+
+ assert validated.sql == ("SELECT doc_id FROM ragflow_t1 WHERE chunk_data #>> '{profile,name}' = 'Alice' AND kb_id = 'kb-1' LIMIT 128")
+ assert validated.columns == ["doc_id"]
+ assert validated.is_aggregation is False
+
+
+def test_tc_sql_403_validator_requires_quotes_for_jsonb_key_containing_comma():
+ validator = _validator(field_map={"a,b": "string"})
+ validated = validator.validate_and_patch("SELECT doc_id FROM ragflow_t1 WHERE chunk_data #>> '{\"a,b\"}' = 'value'")
+
+ assert validated.sql == ("SELECT doc_id FROM ragflow_t1 WHERE chunk_data #>> '{\"a,b\"}' = 'value' AND kb_id = 'kb-1' LIMIT 128")
+ assert validated.columns == ["doc_id"]
+ assert validated.is_aggregation is False
+ _assert_unsafe_sql(
+ "SELECT doc_id FROM ragflow_t1 WHERE chunk_data #>> '{a,b}' = 'value'",
+ "JSONB path ('a', 'b') is not exposed",
+ validator,
+ )
+
+
+def test_tc_sql_404_validator_allows_jsonb_object_extraction_operator():
+ validated = _validator(field_map={"profile": "json"}).validate_and_patch("SELECT doc_id FROM ragflow_t1 WHERE chunk_data #> '{profile}' @> '{\"name\": \"Alice\"}'::jsonb")
+
+ assert validated.sql == ("SELECT doc_id FROM ragflow_t1 WHERE chunk_data #> '{profile}' @> CAST('{\"name\": \"Alice\"}' AS JSONB) AND kb_id = 'kb-1' LIMIT 128")
+ assert validated.columns == ["doc_id"]
+ assert validated.is_aggregation is False
+
+
+def test_tc_sql_405_validator_allows_numeric_cast_on_exposed_jsonb_field():
+ validated = _validator(field_map={"amount": "number"}).validate_and_patch("SELECT doc_id FROM ragflow_t1 WHERE CAST(chunk_data #>> '{amount}' AS DOUBLE PRECISION) > 100")
+
+ assert validated.sql == ("SELECT doc_id FROM ragflow_t1 WHERE CAST(chunk_data #>> '{amount}' AS DOUBLE PRECISION) > 100 AND kb_id = 'kb-1' LIMIT 128")
+ assert validated.columns == ["doc_id"]
+ assert validated.is_aggregation is False
+
+
+def test_tc_sql_406_validator_allows_to_date_on_exposed_jsonb_field():
+ validated = _validator(field_map={"dt": "date"}).validate_and_patch("SELECT doc_id FROM ragflow_t1 WHERE to_date(chunk_data #>> '{dt}', 'YYYY-MM-DD') > '2026-01-01'")
+
+ assert validated.sql == ("SELECT doc_id FROM ragflow_t1 WHERE TO_DATE(chunk_data #>> '{dt}', 'YYYY-MM-DD') > '2026-01-01' AND kb_id = 'kb-1' LIMIT 128")
+ assert validated.columns == ["doc_id"]
+ assert validated.is_aggregation is False
+
+
+def test_tc_sql_407_validator_allows_jsonb_text_null_checks():
+ validator = _validator(field_map={"status": "string"})
+ is_null = validator.validate_and_patch("SELECT doc_id FROM ragflow_t1 WHERE (chunk_data #>> '{status}') IS NULL")
+ is_not_null = validator.validate_and_patch("SELECT doc_id FROM ragflow_t1 WHERE (chunk_data #>> '{status}') IS NOT NULL")
+
+ assert is_null.sql == ("SELECT doc_id FROM ragflow_t1 WHERE (chunk_data #>> '{status}') IS NULL AND kb_id = 'kb-1' LIMIT 128")
+ assert is_not_null.sql == ("SELECT doc_id FROM ragflow_t1 WHERE NOT (chunk_data #>> '{status}') IS NULL AND kb_id = 'kb-1' LIMIT 128")
+ for validated in (is_null, is_not_null):
+ assert validated.columns == ["doc_id"]
+ assert validated.is_aggregation is False
+
+
+def test_tc_sql_408_validator_rejects_unexposed_or_empty_map_jsonb_path():
+ _assert_unsafe_sql(
+ "SELECT doc_id, chunk_data #>> '{non_exposed_field}' FROM ragflow_t1 WHERE kb_id = 'kb-1'",
+ "JSONB path ('non_exposed_field',) is not exposed",
+ )
+ _assert_unsafe_sql(
+ "SELECT doc_id, chunk_data #>> '{amount}' AS amount FROM ragflow_t1 WHERE kb_id = 'kb-1'",
+ "JSONB path ('amount',) is not exposed",
+ _validator(field_map={}),
+ )
+
+
+def test_tc_sql_409_validator_rejects_bare_or_non_gaussdb_chunk_data_access():
+ scenarios = {
+ "SELECT doc_id, chunk_data FROM ragflow_t1 WHERE kb_id = 'kb-1'": ("chunk_data may only be accessed through #> / #>>"),
+ "SELECT chunk_data ->> 'amount' FROM ragflow_t1": ("only GaussDB #> / #>> JSONB operators are allowed"),
+ "SELECT chunk_data['amount'] FROM ragflow_t1": ("chunk_data may only be accessed through #> / #>>"),
+ "SELECT chunk_data @> '{\"amount\":1}'::jsonb FROM ragflow_t1": ("chunk_data may only be accessed through #> / #>>"),
+ "SELECT chunk_data ? 'amount' FROM ragflow_t1": ("chunk_data may only be accessed through #> / #>>"),
+ "SELECT chunk_data #- '{amount}' FROM ragflow_t1": ("chunk_data may only be accessed through #> / #>>"),
+ }
+ for sql, message in scenarios.items():
+ _assert_unsafe_sql(sql, message)
+
+
+ "predicate",
+ [
+ "chunk_data #>> '{status}' = ''",
+ "'' = chunk_data #>> '{status}'",
+ "chunk_data #>> '{status}' <> ''",
+ "'' != chunk_data #>> '{status}'",
+ "CAST(chunk_data #>> '{status}' AS TEXT) = ''",
+ "chunk_data #>> '{status}' = CAST('' AS TEXT)",
+ "chunk_data #>> '{status}' IS DISTINCT FROM ''",
+ "chunk_data #>> '{status}' IS NOT DISTINCT FROM ''",
+ ],
+ ids=[
+ "equal",
+ "reversed-equal",
+ "not-equal",
+ "reversed-not-equal",
+ "cast-jsonb",
+ "cast-empty-string",
+ "distinct",
+ "not-distinct",
+ ],
+)
+def test_tc_sql_410_validator_rejects_jsonb_text_comparisons_with_sql_empty_string(predicate):
+ _assert_unsafe_sql(
+ f"SELECT doc_id FROM ragflow_t1 WHERE kb_id = 'kb-1' AND {predicate}",
+ "JSONB text cannot be compared with an empty SQL string",
+ _validator(field_map={"status": "string"}),
+ )
+
+
+def test_tc_sql_410_validator_allows_jsonb_null_literal_comparison():
+ sql = "SELECT doc_id FROM ragflow_t1 WHERE kb_id = 'kb-1' AND chunk_data #> '{status}' = 'null'::jsonb"
+
+ validated = _validator(field_map={"status": "string"}).validate_and_patch(sql)
+
+ assert validated.sql == ("SELECT doc_id FROM ragflow_t1 WHERE kb_id = 'kb-1' AND chunk_data #> '{status}' = CAST('null' AS JSONB) LIMIT 128")
+ assert validated.columns == ["doc_id"]
+ assert validated.is_aggregation is False
+
+
+def test_tc_sql_512_runtime_readonly_guard_enforces_table_and_preserves_safe_scopes():
+ validator = GaussDBSQLValidator.readonly_guard(execution_schema="tenant_schema")
+ _assert_unsafe_sql(
+ "SELECT 1",
+ "SQL must read from a DocEngine table",
+ validator,
+ )
+
+ cte_alias = validator.validate_and_patch("WITH rows AS (SELECT chunk_data #>> '{amount}' AS amount, kb_id FROM ragflow_t1 WHERE kb_id = 'kb-1') SELECT amount FROM rows")
+ predicate_before_boundary = validator.validate_and_patch("SELECT doc_id FROM ragflow_t1 WHERE doc_id = 'doc1' AND kb_id = 'kb-1'")
+ correlated = validator.validate_and_patch(
+ "SELECT outer_t.doc_id FROM ragflow_t1 AS outer_t WHERE outer_t.kb_id = 'kb-1' AND EXISTS ("
+ "SELECT 1 FROM ragflow_t1 AS inner_t WHERE inner_t.doc_id = outer_t.doc_id "
+ "AND inner_t.kb_id = 'kb-1')"
+ )
+ assert cte_alias.sql == ("WITH rows AS (SELECT chunk_data #>> '{amount}' AS amount, kb_id FROM \"tenant_schema\".ragflow_t1 WHERE kb_id = 'kb-1') SELECT amount FROM rows LIMIT 128")
+ assert cte_alias.columns == ["amount"]
+ assert cte_alias.is_aggregation is False
+ assert predicate_before_boundary.sql == ("SELECT doc_id FROM \"tenant_schema\".ragflow_t1 WHERE doc_id = 'doc1' AND kb_id = 'kb-1' LIMIT 128")
+ assert predicate_before_boundary.columns == ["doc_id"]
+ assert predicate_before_boundary.is_aggregation is False
+ assert correlated.sql.count('FROM "tenant_schema".ragflow_t1') == 2
+ assert correlated.sql.count("kb_id = 'kb-1'") == 2
+ assert correlated.columns == ["doc_id"]
+ assert correlated.is_aggregation is False
+
+
+ "sql",
+ [
+ "SELECT 'sum(' AS marker, doc_id FROM ragflow_t1 WHERE kb_id = 'kb-1' LIMIT 5",
+ "SELECT doc_id /* sum( */ FROM ragflow_t1 WHERE kb_id = 'kb-1' LIMIT 5",
+ "SELECT doc_id AS \"sum(\" FROM ragflow_t1 WHERE kb_id = 'kb-1' LIMIT 5",
+ ],
+ ids=["literal", "comment", "alias"],
+)
+def test_tc_sql_705_aggregate_detection_ignores_sum_text_outside_ast_functions(sql):
+ assert is_gaussdb_aggregate_sql(sql) is False
+ assert _validator(field_map={}).validate_and_patch(sql).is_aggregation is False
+
+
+def test_tc_sql_705_aggregate_detection_rejects_multiple_statements():
+ with pytest.raises(UnsafeGaussDBSQL) as exc_info:
+ is_gaussdb_aggregate_sql("SELECT COUNT(*) FROM ragflow_t1; SELECT 1")
+
+ assert str(exc_info.value) == "exactly one SQL statement is allowed"
+
+
+def test_tc_sql_705_validator_detects_all_supported_aggregations_and_non_aggregate():
+ validator = _validator(field_map={"amount": "number"})
+ scenarios = {
+ "SELECT COUNT(*) AS cnt FROM ragflow_t1": (
+ "SELECT COUNT(*) AS cnt FROM ragflow_t1 WHERE kb_id = 'kb-1' LIMIT 128",
+ ["cnt"],
+ True,
+ ),
+ "SELECT SUM(CAST(chunk_data #>> '{amount}' AS DOUBLE PRECISION)) AS total FROM ragflow_t1": (
+ "SELECT SUM(CAST(chunk_data #>> '{amount}' AS DOUBLE PRECISION)) AS total FROM ragflow_t1 WHERE kb_id = 'kb-1' LIMIT 128",
+ ["total"],
+ True,
+ ),
+ "SELECT AVG(CAST(chunk_data #>> '{amount}' AS DOUBLE PRECISION)) AS avg_amt FROM ragflow_t1": (
+ "SELECT AVG(CAST(chunk_data #>> '{amount}' AS DOUBLE PRECISION)) AS avg_amt FROM ragflow_t1 WHERE kb_id = 'kb-1' LIMIT 128",
+ ["avg_amt"],
+ True,
+ ),
+ "SELECT MAX(CAST(chunk_data #>> '{amount}' AS DOUBLE PRECISION)) AS max_amt FROM ragflow_t1": (
+ "SELECT MAX(CAST(chunk_data #>> '{amount}' AS DOUBLE PRECISION)) AS max_amt FROM ragflow_t1 WHERE kb_id = 'kb-1' LIMIT 128",
+ ["max_amt"],
+ True,
+ ),
+ "SELECT MIN(CAST(chunk_data #>> '{amount}' AS DOUBLE PRECISION)) AS min_amt FROM ragflow_t1": (
+ "SELECT MIN(CAST(chunk_data #>> '{amount}' AS DOUBLE PRECISION)) AS min_amt FROM ragflow_t1 WHERE kb_id = 'kb-1' LIMIT 128",
+ ["min_amt"],
+ True,
+ ),
+ "SELECT doc_id FROM ragflow_t1": (
+ "SELECT doc_id FROM ragflow_t1 WHERE kb_id = 'kb-1' LIMIT 128",
+ ["doc_id"],
+ False,
+ ),
+ }
+ for sql, expected in scenarios.items():
+ validated = validator.validate_and_patch(sql)
+ assert (validated.sql, validated.columns, validated.is_aggregation) == expected
+ assert is_gaussdb_aggregate_sql(validated.sql) is expected[2]
+
+
+def test_tc_sql_706_validator_returns_selected_column_names_and_aliases():
+ validator = _validator(field_map={"amount": "number"})
+ aliased = validator.validate_and_patch("SELECT doc_id, chunk_data #>> '{amount}' AS amt FROM ragflow_t1")
+ simple = validator.validate_and_patch("SELECT doc_id FROM ragflow_t1")
+ literal = validator.validate_and_patch("SELECT 1, doc_id FROM ragflow_t1")
+
+ assert aliased.sql == ("SELECT doc_id, chunk_data #>> '{amount}' AS amt FROM ragflow_t1 WHERE kb_id = 'kb-1' LIMIT 128")
+ assert aliased.columns == ["doc_id", "amt"]
+ assert aliased.is_aggregation is False
+ assert simple.sql == "SELECT doc_id FROM ragflow_t1 WHERE kb_id = 'kb-1' LIMIT 128"
+ assert simple.columns == ["doc_id"]
+ assert simple.is_aggregation is False
+ assert literal.sql == "SELECT 1, doc_id FROM ragflow_t1 WHERE kb_id = 'kb-1' LIMIT 128"
+ assert literal.columns == ["1", "doc_id"]
+ assert literal.is_aggregation is False
+
+
+def test_tc_sql_707_select_alias_is_allowed_in_group_and_order_by_for_both_guards():
+ sql = "SELECT chunk_data #>> '{dept}' AS dept, COUNT(*) AS cnt FROM ragflow_t1 WHERE kb_id = 'kb-1' GROUP BY dept ORDER BY dept"
+ validated = _validator(field_map={"dept": "string"}).validate_and_patch(sql)
+ readonly = GaussDBSQLValidator.readonly_guard().validate_and_patch(sql)
+ expected_sql = "SELECT chunk_data #>> '{dept}' AS dept, COUNT(*) AS cnt FROM ragflow_t1 WHERE kb_id = 'kb-1' GROUP BY dept ORDER BY dept LIMIT 128"
+
+ assert validated.sql == expected_sql
+ assert validated.columns == ["dept", "cnt"]
+ assert validated.is_aggregation is True
+ assert readonly.sql == expected_sql
+ assert readonly.columns == ["dept", "cnt"]
+ assert readonly.is_aggregation is True