RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
feat: Gaussdb adaptation (#17703)
zws202606d02723ca7603
diff --git a/.github/workflows/sep-tests.yml b/.github/workflows/sep-tests.ymlindex ffdf121ba5..01ec38fffb 100644--- a/.github/workflows/sep-tests.yml+++ b/.github/workflows/sep-tests.yml@@ -623,7 +623,7 @@ jobs:echo "SANDBOX_EXECUTOR_MANAGER_PORT=${SANDBOX_EXECUTOR_MANAGER_PORT}"echo "SVR_WEB_HTTP_PORT=${SVR_WEB_HTTP_PORT}"echo "SVR_WEB_HTTPS_PORT=${SVR_WEB_HTTPS_PORT}"- echo "COMPOSE_PROFILES=${DOC_ENGINE},cpu,ragflow-go,tei-cpu,deepdoc"+ echo "COMPOSE_PROFILES=${DOC_ENGINE},cpu,ragflow-go,tei-cpu,deepdoc,metadata-mysql"echo "TEI_MODEL=BAAI/bge-small-en-v1.5"echo "RAGFLOW_IMAGE=${RAGFLOW_IMAGE}"echo "DOC_ENGINE=${DOC_ENGINE}"@@ -1242,7 +1242,7 @@ jobs:echo "SANDBOX_EXECUTOR_MANAGER_PORT=${SANDBOX_EXECUTOR_MANAGER_PORT}"echo "SVR_WEB_HTTP_PORT=${SVR_WEB_HTTP_PORT}"echo "SVR_WEB_HTTPS_PORT=${SVR_WEB_HTTPS_PORT}"- echo "COMPOSE_PROFILES=${DOC_ENGINE},cpu,ragflow-go,tei-cpu,deepdoc"+ echo "COMPOSE_PROFILES=${DOC_ENGINE},cpu,ragflow-go,tei-cpu,deepdoc,metadata-mysql"echo "TEI_MODEL=BAAI/bge-small-en-v1.5"echo "RAGFLOW_IMAGE=${RAGFLOW_IMAGE}"echo "DOC_ENGINE=${DOC_ENGINE}"diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.ymlindex 8ab708253e..cdc9c1ac66 100644--- a/.github/workflows/tests.yml+++ b/.github/workflows/tests.yml@@ -393,7 +393,7 @@ jobs:echo "SANDBOX_EXECUTOR_MANAGER_PORT=${SANDBOX_EXECUTOR_MANAGER_PORT}"echo "SVR_WEB_HTTP_PORT=${SVR_WEB_HTTP_PORT}"echo "SVR_WEB_HTTPS_PORT=${SVR_WEB_HTTPS_PORT}"- echo "COMPOSE_PROFILES=${DOC_ENGINE},cpu,tei-cpu,deepdoc"+ echo "COMPOSE_PROFILES=${DOC_ENGINE},cpu,tei-cpu,deepdoc,metadata-mysql"echo "TEI_MODEL=BAAI/bge-small-en-v1.5"echo "RAGFLOW_IMAGE=${RAGFLOW_IMAGE}"echo "DOC_ENGINE=${DOC_ENGINE}"@@ -872,7 +872,7 @@ jobs:echo "SANDBOX_EXECUTOR_MANAGER_PORT=${SANDBOX_EXECUTOR_MANAGER_PORT}"echo "SVR_WEB_HTTP_PORT=${SVR_WEB_HTTP_PORT}"echo "SVR_WEB_HTTPS_PORT=${SVR_WEB_HTTPS_PORT}"- echo "COMPOSE_PROFILES=${DOC_ENGINE},cpu,tei-cpu,deepdoc"+ echo "COMPOSE_PROFILES=${DOC_ENGINE},cpu,tei-cpu,deepdoc,metadata-mysql"echo "TEI_MODEL=BAAI/bge-small-en-v1.5"echo "RAGFLOW_IMAGE=${RAGFLOW_IMAGE}"echo "DOC_ENGINE=${DOC_ENGINE}"diff --git a/admin/server/admin_server.py b/admin/server/admin_server.pyindex 677d8a1c96..9e54a1cf21 100644--- a/admin/server/admin_server.py+++ b/admin/server/admin_server.py@@ -36,6 +36,7 @@ from config import load_configurations, SERVICE_CONFIGSfrom auth import init_default_admin, setup_authfrom flask_session import Sessionfrom common.versions import get_ragflow_version+from api.db.db_models import close_connectionstop_event = threading.Event()@@ -60,6 +61,18 @@ if __name__ == "__main__":show_configs()login_manager = LoginManager()login_manager.init_app(app)++ @app.teardown_request+ def _db_close(exception):+ if settings.DATABASE_TYPE.lower() == "gaussdb":+ if exception:+ logging.error(+ "Admin request failed: %s",+ exception,+ exc_info=(type(exception), exception, exception.__traceback__),+ )+ close_connection()+settings.init_settings()setup_auth(login_manager)init_default_admin()diff --git a/admin/server/config.py b/admin/server/config.pyindex eac8047974..c64ec98a84 100644--- a/admin/server/config.py+++ b/admin/server/config.py@@ -16,6 +16,8 @@import logging+import os+import reimport threadingfrom enum import Enum@@ -46,6 +48,67 @@ class ServiceConfigs:SERVICE_CONFIGS = ServiceConfigs+# The Admin service list reads the same service_conf/local.service_conf files,+# but the GaussDB metadata connection does not come from those files because+# their top-level gaussdb section belongs to DocEngine and Memory Store. This+# lightweight parser keeps Admin's metadata display aligned with the main+# service DATABASE settings instead of showing an inactive mysql/postgres block.+GAUSSDB_ENV_DEFAULTS = {+ "name": "rag_flow",+ "user": "rag_flow",+ "password": "infini_rag_flow",+ "host": "gaussdb",+ "port": 8000,+ "schema": "public",+}+_GAUSSDB_SCHEMA_PATTERN = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$")+++def normalize_database_type(database_type: str | None = None) -> str:+ # Only normalize the new GaussDB values. Existing metadata database+ # selection retains the upstream behavior.+ raw_value = database_type or "mysql"+ normalized = raw_value.strip().lower()+ if normalized in {"gaussdb", "gauss"}:+ return "gaussdb"+ return raw_value+++def _get_int_env(name: str, default: int) -> int:+ raw_value = os.environ.get(name)+ if raw_value is None:+ return default+ try:+ return int(raw_value)+ except (TypeError, ValueError):+ logging.warning("Ignoring invalid %s=%r; using default %d", name, raw_value, default)+ return default+++def _normalize_gaussdb_metadata_schema(value: str | None = None) -> str:+ # Admin only displays the schema and does not create the connection, but it+ # must apply the same validation. Otherwise an invalid+ # GAUSSDB_METADATA_SCHEMA could look valid in Admin while the main service+ # fails during startup.+ schema = (value or GAUSSDB_ENV_DEFAULTS["schema"]).strip() or GAUSSDB_ENV_DEFAULTS["schema"]+ if not _GAUSSDB_SCHEMA_PATTERN.match(schema):+ raise ValueError(f"invalid GAUSSDB_METADATA_SCHEMA: {schema}")+ return schema+++def _gaussdb_env_config() -> dict[str, Any]:+ # This configuration is only for Admin display and health checks; it does+ # not create the application ORM connection. Keep the flat metadata fields+ # aligned with common.settings._gaussdb_env_config() so both surfaces report+ # the same connection information.+ return {+ "name": os.environ.get("GAUSSDB_METADATA_DBNAME", GAUSSDB_ENV_DEFAULTS["name"]),+ "user": os.environ.get("GAUSSDB_METADATA_USER", GAUSSDB_ENV_DEFAULTS["user"]),+ "password": os.environ.get("GAUSSDB_METADATA_PASSWORD", GAUSSDB_ENV_DEFAULTS["password"]),+ "host": os.environ.get("GAUSSDB_METADATA_HOST", GAUSSDB_ENV_DEFAULTS["host"]),+ "port": _get_int_env("GAUSSDB_METADATA_PORT", GAUSSDB_ENV_DEFAULTS["port"]),+ "schema": _normalize_gaussdb_metadata_schema(os.environ.get("GAUSSDB_METADATA_SCHEMA")),+ }class ServiceType(Enum):@@ -85,6 +148,16 @@ class MySQLConfig(MetaConfig):return result+class GaussDBMetadataConfig(MySQLConfig):+ metadata_schema: str++ def to_dict(self) -> dict[str, Any]:+ result = super().to_dict()+ result["extra"]["schema"] = self.metadata_schema+ result["extra"]["password"] = "*" * 8+ return result++class PostgresConfig(MetaConfig):def to_dict(self) -> dict[str, Any]:result = super().to_dict()@@ -106,6 +179,18 @@ class RetrievalConfig(BaseConfig):return result+class GaussDBRetrievalConfig(RetrievalConfig):+ database: str+ retrieval_schema: str++ def to_dict(self) -> dict[str, Any]:+ result = super().to_dict()+ extra_dict = result["extra"].copy()+ extra_dict.update({"database": self.database, "schema": self.retrieval_schema})+ result["extra"] = extra_dict+ return result++class InfinityConfig(RetrievalConfig):db_name: str@@ -222,6 +307,7 @@ def load_configurations(config_path: str) -> list[BaseConfig]:configurations = []ragflow_count = 0id_count = 0+ metadata_db_type = normalize_database_type(os.getenv("DB_TYPE", "mysql"))for k, v in raw_configs.items():match k:case "ragflow":@@ -341,13 +427,55 @@ def load_configurations(config_path: str) -> list[BaseConfig]:configurations.append(config)id_count += 1case "mysql":+ if metadata_db_type == "gaussdb":+ continuename: str = "mysql"host: str = v.get("host")port: int = v.get("port")username = v.get("user")password = v.get("password")config = MySQLConfig(- id=id_count, name=name, host=host, port=port, username=username, password=password, service_type="meta_data", meta_type="mysql", detail_func_name="get_mysql_status"+ id=id_count,+ name=name,+ host=host,+ port=port,+ username=username,+ password=password,+ service_type="meta_data",+ meta_type="mysql",+ detail_func_name="get_mysql_status",+ )+ configurations.append(config)+ id_count += 1+ case "gaussdb":+ # The gaussdb block in the configuration file belongs to+ # DocEngine and Memory Store, not the DB_TYPE=gaussdb metadata+ # connection. Add metadata GaussDB separately from+ # GAUSSDB_METADATA_* after the loop so Admin does not merge the+ # two services.+ doc_config = v+ host = doc_config.get("host")+ port = doc_config.get("port")+ if not host or port in (None, ""):+ # service_conf.yaml.template always contains a GaussDB+ # placeholder block. Default deployments leave it empty;+ # do not let an inactive DocEngine config break Admin.+ continue+ try:+ port = int(port)+ except (TypeError, ValueError):+ logging.warning("Ignoring invalid GaussDB DocEngine port in Admin configuration")+ continue+ config = GaussDBRetrievalConfig(+ id=id_count,+ name="gaussdb",+ host=host,+ port=port,+ database=doc_config.get("database"),+ retrieval_schema=doc_config.get("schema") or "public",+ service_type="retrieval",+ retrieval_type="gaussdb",+ detail_func_name="get_gaussdb_status",)configurations.append(config)id_count += 1@@ -374,4 +502,23 @@ def load_configurations(config_path: str) -> list[BaseConfig]:logging.warning(f"Unknown configuration key: {k}")continue+ if metadata_db_type == "gaussdb":+ # Display the metadata database from the same GAUSSDB_METADATA_*+ # variables as the main service, never from the DocEngine/Memory Store+ # GAUSSDB_* variables.+ v = _gaussdb_env_config()+ config = GaussDBMetadataConfig(+ id=id_count,+ name="gaussdb",+ host=v.get("host"),+ port=v.get("port"),+ username=v.get("user"),+ password=v.get("password"),+ metadata_schema=v.get("schema"),+ service_type="meta_data",+ meta_type="gaussdb",+ detail_func_name="get_database_status",+ )+ configurations.append(config)+return configurationsdiff --git a/api/apps/restful_apis/system_api.py b/api/apps/restful_apis/system_api.pyindex 4e587035f8..c3473cf53c 100644--- a/api/apps/restful_apis/system_api.py+++ b/api/apps/restful_apis/system_api.py@@ -23,13 +23,14 @@ from quart import jsonifyfrom api.apps import login_required, current_userfrom api.utils.api_utils import get_json_result, get_data_error_result, server_error_response, generate_confirmation_token-from api.utils.health_utils import run_health_checks, get_oceanbase_status+from api.utils.health_utils import run_health_checks, get_oceanbase_status, get_gaussdb_statusfrom common.versions import get_ragflow_versionfrom common.time_utils import current_timestamp, datetime_formatfrom api.db.db_models import APITokenfrom api.db.services.api_service import APITokenServicefrom api.db.services.knowledgebase_service import KnowledgebaseServicefrom api.db.services.user_service import UserTenantService+from common.doc_store.gaussdb_conn_base import mask_gaussdb_textfrom common.log_utils import get_log_levels, set_log_levelfrom common import settingsfrom rag.utils.redis_conn import REDIS_CONN@@ -208,6 +209,38 @@ def oceanbase_status():return get_json_result(data={"status": "error", "message": f"Failed to get OceanBase status: {str(e)}"}, code=500)[email protected]("/system/gaussdb/status", methods=["GET"]) # noqa: F821+@login_required+def gaussdb_status():+ """+ Get GaussDB health status and performance metrics.+ ---+ tags:+ - System+ security:+ - ApiKeyAuth: []+ responses:+ 200:+ description: GaussDB status retrieved successfully.+ schema:+ type: object+ properties:+ status:+ type: string+ description: Status (alive/timeout/not_configured).+ message:+ type: object+ description: Detailed status information including health and performance metrics.+ """+ try:+ status_info = get_gaussdb_status()+ return get_json_result(data=status_info)+ except Exception as e:+ masked_error = mask_gaussdb_text(e)+ logging.error("GaussDB status route failed (%s): %s", type(e).__name__, masked_error)+ return get_json_result(data={"status": "error", "message": f"Failed to get GaussDB status: {masked_error}"}, code=500)++@manager.route("/system/config", methods=["GET"]) # noqa: F821def get_config():"""diff --git a/api/apps/services/dataset_api_service.py b/api/apps/services/dataset_api_service.pyindex f71b973bc3..fdceb392c0 100644--- a/api/apps/services/dataset_api_service.py+++ b/api/apps/services/dataset_api_service.py@@ -788,7 +788,9 @@ def delete_tags(dataset_id: str, tenant_id: str, tags: list[str]):from rag.nlp import searchfor t in tags:- settings.docStoreConn.update({"tag_kwd": t, "kb_id": [dataset_id]}, {"remove": {"tag_kwd": t}}, search.index_name(kb.tenant_id), dataset_id)+ updated = settings.docStoreConn.update({"tag_kwd": t, "kb_id": [dataset_id]}, {"remove": {"tag_kwd": t}}, search.index_name(kb.tenant_id), dataset_id)+ if callable(getattr(settings.docStoreConn, "db_type", None)) and settings.docStoreConn.db_type() == "gaussdb" and not updated:+ return False, "Failed to update dataset tags in document store"return True, {}@@ -991,7 +993,11 @@ def rename_tag(dataset_id: str, tenant_id: str, from_tag: str, to_tag: str):from rag.nlp import search- settings.docStoreConn.update({"tag_kwd": from_tag, "kb_id": [dataset_id]}, {"remove": {"tag_kwd": from_tag.strip()}, "add": {"tag_kwd": to_tag}}, search.index_name(kb.tenant_id), dataset_id)+ updated = settings.docStoreConn.update(+ {"tag_kwd": from_tag, "kb_id": [dataset_id]}, {"remove": {"tag_kwd": from_tag.strip()}, "add": {"tag_kwd": to_tag}}, search.index_name(kb.tenant_id), dataset_id+ )+ if callable(getattr(settings.docStoreConn, "db_type", None)) and settings.docStoreConn.db_type() == "gaussdb" and not updated:+ return False, "Failed to update dataset tags in document store"return True, {"from": from_tag, "to": to_tag}@@ -1261,7 +1267,11 @@ def check_embedding(dataset_id: str, tenant_id: str, req: dict):cid = ids[0]full_doc = docStoreConn.get(cid, index_nm, [kb_id]) or {}vec_field = _guess_vec_field(full_doc)- vec = _as_float_vec(full_doc.get(vec_field))+ vec_valid = full_doc.get(f"{vec_field}_valid") if vec_field else None+ if callable(getattr(docStoreConn, "db_type", None)) and docStoreConn.db_type() == "gaussdb" and vec_valid is False:+ vec = []+ else:+ vec = _as_float_vec(full_doc.get(vec_field))out.append({diff --git a/api/db/db_models.py b/api/db/db_models.pyindex 3867612210..04009562fc 100644--- a/api/db/db_models.py+++ b/api/db/db_models.py@@ -27,6 +27,7 @@ from functools import wrapsfrom quart_auth import AuthUserfrom itsdangerous.url_safe import URLSafeTimedSerializer as Serializer+from psycopg2 import sql as psycopg2_sqlfrom peewee import (fn,InterfaceError,@@ -42,6 +43,7 @@ from peewee import (IntegerField,Metadata,Model,+ ModelInsert,TextField,)from playhouse.migrate import MySQLMigrator, PostgresqlMigrator, migrate@@ -49,6 +51,13 @@ from playhouse.pool import PooledMySQLDatabase, PooledPostgresqlDatabasefrom api import utilsfrom api.db import SerializedType+from api.db.gaussdb_error_utils import (+ is_duplicate_column_error,+ is_duplicate_object_error,+ is_psycopg_connection_error,+ is_undefined_object_error,+ sqlstate_from_exception,+)from api.utils.json_encode import json_dumps, json_loadsfrom api.utils.configs import deserialize_b64, serialize_b64@@ -60,18 +69,96 @@ from common import settingsCONTINUOUS_FIELD_TYPE = {IntegerField, FloatField, DateTimeField}AUTO_DATE_TIMESTAMP_FIELD_PREFIX = {"create", "start", "end", "update", "read_access", "write_access"}+GAUSSDB_COMPATIBLE_DATABASE_TYPES = {"GAUSSDB"}+++def is_gaussdb_compatible_database() -> bool:+ return settings.DATABASE_TYPE.upper() in GAUSSDB_COMPATIBLE_DATABASE_TYPESclass TextFieldType(Enum):MYSQL = "LONGTEXT"OCEANBASE = "LONGTEXT"POSTGRES = "TEXT"+ # GaussDB does not support MySQL LONGTEXT semantics. Map long text to TEXT+ # for JSONField and LongTextField while retaining a distinct enum entry so+ # DB_TYPE=gaussdb is not treated as DB_TYPE=postgres.+ GAUSSDB = "TEXT"class LongTextField(TextField):field_type = TextFieldType[settings.DATABASE_TYPE.upper()].value+class EmptyStringFieldMixin:+ """Preserve application-level empty-string semantics on GaussDB.++ A/ORA-compatible GaussDB treats empty strings in SQL literals and bound+ parameters as NULL. Some RAGFlow fields use an empty string to mean "not+ configured" and can remain NOT NULL with a default of "" on MySQL and+ PostgreSQL. On GaussDB, those values would violate a NOT NULL constraint.++ Apply this field only to explicitly selected columns. Do not perform a+ global NULL-to-empty-string conversion in DB.execute_sql or the cursor+ layer, because NULL remains meaningful for other fields.+ """++ __hash__ = Field.__hash__++ def __init__(self, *args, **kwargs):+ if is_gaussdb_compatible_database():+ # These fields represent application-level empty strings as NULL+ # in GaussDB. Their columns must therefore be nullable to avoid a+ # NOT NULL violation. Other databases retain their original+ # constraints.+ kwargs["null"] = True+ super().__init__(*args, **kwargs)++ def db_value(self, value):+ if is_gaussdb_compatible_database() and value == "":+ # Normalize the application-level empty string explicitly so the+ # storage semantics remain stable even if a test or user database+ # stops converting empty strings to NULL automatically.+ return None+ return super().db_value(value)++ def python_value(self, value):+ if is_gaussdb_compatible_database() and value is None:+ # Preserve the existing application contract without scattering+ # `is None` branches throughout the business code.+ return ""+ return super().python_value(value)++ def __eq__(self, rhs):+ if is_gaussdb_compatible_database() and isinstance(rhs, str) and rhs == "":+ # On A/ORA-compatible GaussDB, `field = ''` becomes `field = NULL`+ # and never matches. Express the application-level empty value as+ # `IS NULL OR LENGTH(field) = 0` to match both normalized NULLs and+ # any historical empty strings.+ return self.is_null(True) | (fn.LENGTH(self) == 0)+ return super().__eq__(rhs)++ def __ne__(self, rhs):+ if is_gaussdb_compatible_database() and isinstance(rhs, str) and rhs == "":+ # Symmetrically, a value unequal to the application-level empty+ # string must contain data. LENGTH(field) > 0 avoids comparing with+ # NULL and excludes both NULL and historical empty strings.+ return fn.LENGTH(self) > 0+ return super().__ne__(rhs)+++class EmptyStringCharField(EmptyStringFieldMixin, CharField):+ pass+++class EmptyStringTextField(EmptyStringFieldMixin, TextField):+ pass+++class EmptyStringLongTextField(EmptyStringFieldMixin, LongTextField):+ pass++class JSONField(LongTextField):default_value = {}@@ -317,6 +404,80 @@ class RetryingPooledMySQLDatabase(PooledMySQLDatabase):return None+class GaussDBPsycopgRetryMixin:+ """Connection retry behavior used only by the GaussDB adapter."""++ database_display_name = "GaussDB"++ def __init__(self, *args, **kwargs):+ self.max_retries = kwargs.pop("max_retries", 5)+ self.retry_delay = kwargs.pop("retry_delay", 1)+ super().__init__(*args, **kwargs)++ def _prepare_sql_for_execution(self, sql):+ return sql++ def execute_sql(self, sql, params=None, commit=True):+ for attempt in range(self.max_retries + 1):+ try:+ prepared_sql = self._prepare_sql_for_execution(sql)+ return super().execute_sql(prepared_sql, params, commit)+ except (OperationalError, InterfaceError) as e:+ # A reconnect cannot restore an active transaction. Let the+ # atomic block fail as a unit instead of retrying one statement+ # on a different connection.+ if self.in_transaction():+ logging.error(f"{self.database_display_name} execution failure: {e}")+ raise++ # Peewee may hide the driver SQLSTATE in `__context__`. Use the+ # shared classifier to recognize 08xxx/57P0x connection states+ # first and fall back to observed GaussDB disconnect text.+ should_retry = is_psycopg_connection_error(e)++ if should_retry and attempt < self.max_retries:+ logging.warning(f"{self.database_display_name} connection issue (attempt {attempt + 1}/{self.max_retries}): {e}")+ self._handle_connection_loss()+ time.sleep(self.retry_delay * (2**attempt))+ else:+ logging.error(f"{self.database_display_name} execution failure: {e}")+ raise+ return None++ def _handle_connection_loss(self):+ try:+ self.close()+ except Exception:+ pass+ try:+ self.connect()+ except Exception as e:+ logging.error(f"Failed to reconnect to {self.database_display_name}: {e}")+ time.sleep(0.1)+ try:+ self.connect()+ except Exception as e2:+ logging.error(f"Failed to reconnect to {self.database_display_name} on second attempt: {e2}")+ raise++ def begin(self):+ for attempt in range(self.max_retries + 1):+ try:+ return super().begin()+ except (OperationalError, InterfaceError) as e:+ # Apply the same connection-error classification during+ # transaction startup and statement execution.+ should_retry = is_psycopg_connection_error(e)++ if should_retry and attempt < self.max_retries:+ logging.warning(f"{self.database_display_name} connection lost during transaction (attempt {attempt + 1}/{self.max_retries})")+ self._handle_connection_loss()+ time.sleep(self.retry_delay * (2**attempt))+ else:+ raise+ return None++class RetryingPooledPostgresqlDatabase(PooledPostgresqlDatabase):def __init__(self, *args, **kwargs):self.max_retries = kwargs.pop("max_retries", 5)@@ -382,6 +543,184 @@ class RetryingPooledPostgresqlDatabase(PooledPostgresqlDatabase):return None+class RetryingPooledGaussDBDatabase(GaussDBPsycopgRetryMixin, PooledPostgresqlDatabase):+ """Connection pool for the GaussDB business metadata database.++ GaussDB exposes a psycopg/libpq-compatible protocol, so Peewee can use+ PooledPostgresqlDatabase as its transport. The adapter remains distinct so+ DB_TYPE=gaussdb dialect differences, compatibility checks, logging, and+ diagnostics stay in this class or GaussDBMigrator instead of aliasing+ GaussDB to PostgreSQL.+ """++ database_display_name = "GaussDB"+ # These tables have both an id primary key and a business unique key that+ # does not contain id. Distributed HASH tables cannot enforce both sets of+ # constraints globally, so preserve the existing schema with replication.+ distributed_replication_tables = frozenset(+ {+ "compilation_template",+ "compilation_template_group",+ "file_commit_item",+ "tenant_llm",+ "tenant_model_provider",+ "user",+ }+ )++ def __init__(self, *args, **kwargs):+ self._is_distributed = None+ # Force UTF-8 so metadata containing non-ASCII text, JSON, or template+ # data does not depend on the connection's default encoding.+ kwargs.setdefault("options", "-c client_encoding=UTF8")+ super().__init__(*args, **kwargs)++ def _initialize_connection(self, conn):+ """Initialize a checked-out connection and detect its deployment type."""+ super()._initialize_connection(conn)++ with conn.cursor() as cursor:+ cursor.execute("SELECT current_schema()")+ row = cursor.fetchone()+ schema = row[0] if row else None+ if not schema:+ raise OperationalError("GaussDB metadata search_path has no existing schema")+ # SET is repeated after connection startup because a distributed CN+ # may not propagate the libpq search_path option to every DN.+ cursor.execute(psycopg2_sql.SQL("SET search_path TO {}").format(psycopg2_sql.Identifier(schema)))++ try:+ cursor.execute("SELECT EXISTS (SELECT 1 FROM pg_catalog.pgxc_node WHERE node_type = 'D')")+ topology_row = cursor.fetchone()+ is_distributed = bool(topology_row and topology_row[0])+ except Exception as exc:+ # Centralized editions may not expose pgxc_node at all, or may+ # reject querying it as an unsupported feature.+ if sqlstate_from_exception(exc) not in {"0A000", "42P01"}:+ raise+ rollback = getattr(conn, "rollback", None)+ if callable(rollback):+ rollback()+ # A failed catalog probe can abort the startup transaction and+ # roll back the preceding SET. Restore the configured schema so+ # the centralized connection remains immediately usable.+ cursor.execute(psycopg2_sql.SQL("SET search_path TO {}").format(psycopg2_sql.Identifier(schema)))+ is_distributed = False++ if self._is_distributed is not None and self._is_distributed != is_distributed:+ raise RuntimeError("GaussDB deployment topology changed across pooled connections")+ if self._is_distributed is None:+ logging.info(+ "Detected %s GaussDB metadata deployment",+ "distributed" if is_distributed else "centralized",+ )+ self._is_distributed = is_distributed++ @property+ def is_distributed(self):+ if self._is_distributed is None:+ raise RuntimeError("GaussDB deployment topology is unavailable before connection initialization")+ return self._is_distributed++ def _ensure_topology(self):+ if self._is_distributed is None:+ # This goes through GaussDBPsycopgRetryMixin so a transient first-connect+ # failure gets the same retry treatment as any other SQL execution.+ self.execute_sql("SELECT 1")++ def prepare_update_by_id(self, model, pid, data):+ """Remove only redundant primary-key assignments on distributed GaussDB."""+ self._ensure_topology()+ if not self.is_distributed:+ return data++ primary_key = model._meta.primary_key+ if isinstance(primary_key, CompositeKey) or primary_key.name not in data:+ return data++ field_name = primary_key.name+ locator_value = primary_key.db_value(pid)+ update_value = primary_key.db_value(data[field_name])+ if locator_value != update_value:+ return data++ prepared = data.copy()+ del prepared[field_name]+ return prepared++ def replace_update_by_id(self, model, pid, data):+ """Replace a row when distributed GaussDB must change its primary key.++ A distributed table commonly uses its primary key as the distribution+ key, which GaussDB does not allow an UPDATE statement to modify. Keep+ the generic UPDATE path for centralized GaussDB and for updates that do+ not really change the key. A real key change is performed atomically as+ delete plus insert while preserving every existing column.++ Return ``None`` when the caller should execute its normal UPDATE, or+ the affected-row count when this method handled the replacement.+ """+ self._ensure_topology()+ if not self.is_distributed:+ return None++ primary_key = model._meta.primary_key+ if isinstance(primary_key, CompositeKey) or primary_key.name not in data:+ return None++ field_name = primary_key.name+ locator_value = primary_key.db_value(pid)+ update_value = primary_key.db_value(data[field_name])+ if locator_value == update_value:+ return None++ with self.atomic():+ record = model.get_or_none(primary_key == pid)+ if record is None:+ return 0++ replacement = record.__data__.copy()+ replacement.update(data)+ deleted = model.delete().where(primary_key == pid).execute()+ if deleted != 1:+ return deleted+ # Bypass BaseModel.insert(), which refreshes create_time. Replacing+ # a distribution key must retain the original row's creation data.+ ModelInsert(model, replacement).execute()+ return deleted++ def _prepare_distributed_ddl(self, sql):+ if not self.is_distributed or not isinstance(sql, str):+ return sql++ statement = sql.rstrip()+ trailing_whitespace = sql[len(statement) :]+ terminator = ";" if statement.endswith(";") else ""+ if terminator:+ statement = statement[:-1].rstrip()++ for table_name in self.distributed_replication_tables:+ quoted_name = f'"{table_name}"'+ if statement.startswith(f"CREATE TABLE IF NOT EXISTS {quoted_name} ") or statement.startswith(f"CREATE TABLE {quoted_name} "):+ return f"{statement} DISTRIBUTE BY REPLICATION{terminator}{trailing_whitespace}"+ return sql++ def _prepare_sql_for_execution(self, sql):+ # Topology initialization intentionally runs inside the retry loop in+ # GaussDBPsycopgRetryMixin. This also ensures the first CREATE TABLE receives+ # its distributed table clause before Peewee executes it.+ if self._is_distributed is None:+ self.connection()+ return self._prepare_distributed_ddl(sql)++ def get_tables(self, schema=None):+ if schema is not None:+ return super().get_tables(schema)++ cursor = self.execute_sql("SELECT tablename FROM pg_catalog.pg_tables WHERE schemaname = current_schema() ORDER BY tablename")+ return [table for (table,) in cursor.fetchall()]++class RetryingPooledOceanBaseDatabase(PooledMySQLDatabase):"""Pooled OceanBase database with retry mechanism.@@ -459,12 +798,26 @@ class PooledDatabase(Enum):MYSQL = RetryingPooledMySQLDatabaseOCEANBASE = RetryingPooledOceanBaseDatabasePOSTGRES = RetryingPooledPostgresqlDatabase+ # Keep GaussDB distinct instead of rewriting settings.DATABASE_TYPE to+ # postgres so logs, health checks, and Admin report the real database type.+ GAUSSDB = RetryingPooledGaussDBDatabase+++class GaussDBMigrator(PostgresqlMigrator):+ # Peewee has no built-in GaussDB migrator. Its standard ADD COLUMN, ALTER+ # COLUMN, and ADD INDEX statements are compatible with those generated by+ # PostgresqlMigrator. Keep a separate class so GaussDB-specific DDL can be+ # added without changing DB_TYPE=postgres.+ passclass DatabaseMigrator(Enum):MYSQL = MySQLMigratorOCEANBASE = MySQLMigratorPOSTGRES = PostgresqlMigrator+ # Route DB_TYPE=gaussdb through its own migrator entry even though the+ # current DDL implementation derives from PostgresqlMigrator.+ GAUSSDB = GaussDBMigrator@singleton@@ -477,6 +830,9 @@ class BaseDataBase:"max_retries": 5,"retry_delay": 1,}+ # Retry settings are runtime behavior shared by the database adapters,+ # not database configuration fields. GaussDBPsycopgRetryMixin uses+ # SQLSTATE to decide whether an operation is retryable.database_config.update(pool_config)self.database_connection = PooledDatabase[settings.DATABASE_TYPE.upper()].value(db_name, **database_config)# self.database_connection = PooledDatabase[settings.DATABASE_TYPE.upper()].value(db_name, **database_config)@@ -571,6 +927,44 @@ class PostgresDatabaseLock:return magic+class GaussDBDatabaseLock(PostgresDatabaseLock):+ # GaussDB supports pg_try_advisory_lock/pg_advisory_unlock, so its current+ # lock SQL matches PostgreSQL. Keep a distinct class so future lock APIs,+ # timeout behavior, or error handling remain isolated.+ database_label = "gaussdb"+ poll_interval = 0.1++ def __init__(self, lock_name, timeout=10, db=None):+ super().__init__(lock_name, timeout=timeout, db=db)+ self.timeout = max(float(timeout), 0.0)++ def lock(self):+ deadline = time.monotonic() + self.timeout+ while True:+ cursor = self.db.execute_sql("SELECT pg_try_advisory_lock(%s)", (self.lock_id,))+ row = cursor.fetchone()+ value = row[0] if row else None+ if value in (1, True):+ return True+ if value not in (0, False):+ raise Exception(f"failed to acquire lock {self.lock_name}")++ remaining = deadline - time.monotonic()+ if remaining <= 0:+ raise Exception(f"acquire {self.database_label} lock {self.lock_name} timeout")+ time.sleep(min(self.poll_interval, remaining))++ @with_retry(max_retries=3, retry_delay=1.0)+ def unlock(self):+ cursor = self.db.execute_sql("SELECT pg_advisory_unlock(%s)", (self.lock_id,))+ ret = cursor.fetchone()+ if ret[0] == 0:+ raise Exception(f"gaussdb lock {self.lock_name} was not established by this thread")+ if ret[0] == 1:+ return True+ raise Exception(f"gaussdb lock {self.lock_name} does not exist")++class MysqlDatabaseLock:def __init__(self, lock_name, timeout=10, db=None):self.lock_name = lock_name@@ -622,6 +1016,10 @@ class DatabaseLock(Enum):MYSQL = MysqlDatabaseLockOCEANBASE = MysqlDatabaseLockPOSTGRES = PostgresDatabaseLock+ # Initialization and migrations require cross-process exclusion. GaussDB+ # uses advisory locks instead of MySQL GET_LOCK/RELEASE_LOCK while retaining+ # a distinct enum entry from PostgreSQL.+ GAUSSDB = GaussDBDatabaseLockDB = BaseDataBase().database_connection@@ -631,7 +1029,12 @@ DB.lock = DatabaseLock[settings.DATABASE_TYPE.upper()].valuedef close_connection():try:if DB:- DB.close_stale(age=30)+ if settings.DATABASE_TYPE.upper() in GAUSSDB_COMPATIBLE_DATABASE_TYPES:+ if not DB.is_closed():+ # Return this worker thread's connection to the GaussDB pool.+ DB.close()+ else:+ DB.close_stale(age=30)except Exception as e:logging.exception(e)@@ -681,7 +1084,10 @@ class User(DataBaseModel, AuthUser):id = CharField(max_length=32, primary_key=True)access_token = CharField(max_length=255, null=True, index=True)- nickname = CharField(max_length=100, null=False, help_text="nicky name", index=True)+ # A/ORA-compatible GaussDB stores the application-level empty string as+ # NULL. Admin retains nickname="" while the field restores that value when+ # reading from storage.+ nickname = EmptyStringCharField(max_length=100, null=False, help_text="nicky name", index=True)password = CharField(max_length=255, null=True, help_text="password", index=True)email = CharField(max_length=255, null=False, help_text="email", unique=True)avatar = TextField(null=True, help_text="avatar base64 string")@@ -723,15 +1129,18 @@ class Tenant(DataBaseModel):id = CharField(max_length=32, primary_key=True)name = CharField(max_length=100, null=True, help_text="Tenant name", index=True)public_key = CharField(max_length=255, null=True, index=True)- llm_id = CharField(max_length=128, null=False, help_text="default llm ID", index=True)+ # These default model IDs historically use "" for an unconfigured value.+ # EmptyStringCharField maps GaussDB storage NULL back to that application+ # value.+ llm_id = EmptyStringCharField(max_length=128, null=False, help_text="default llm ID", index=True)tenant_llm_id = CharField(max_length=32, null=True, help_text="id in tenant_model", index=True)- embd_id = CharField(max_length=128, null=False, help_text="default embedding model ID", index=True)+ embd_id = EmptyStringCharField(max_length=128, null=False, help_text="default embedding model ID", index=True)tenant_embd_id = CharField(max_length=32, null=True, help_text="id in tenant_model", index=True)- asr_id = CharField(max_length=128, null=False, help_text="default ASR model ID", index=True)+ asr_id = EmptyStringCharField(max_length=128, null=False, help_text="default ASR model ID", index=True)tenant_asr_id = CharField(max_length=32, null=True, help_text="id in tenant_model", index=True)- img2txt_id = CharField(max_length=128, null=False, help_text="default image to text model ID", index=True)+ img2txt_id = EmptyStringCharField(max_length=128, null=False, help_text="default image to text model ID", index=True)tenant_img2txt_id = CharField(max_length=32, null=True, help_text="id in tenant_model", index=True)- rerank_id = CharField(max_length=128, null=False, help_text="default rerank model ID", index=True)+ rerank_id = EmptyStringCharField(max_length=128, null=False, help_text="default rerank model ID", index=True)tenant_rerank_id = CharField(max_length=32, null=True, help_text="id in tenant_model", index=True)tts_id = CharField(max_length=256, null=True, help_text="default tts model ID", index=True)tenant_tts_id = CharField(max_length=32, null=True, help_text="id in tenant_model", index=True)@@ -841,7 +1250,9 @@ class Knowledgebase(DataBaseModel):name = CharField(max_length=128, null=False, help_text="KB name", index=True)language = CharField(max_length=32, null=True, default="Chinese" if "zh_CN" in os.getenv("LANG", "") else "English", help_text="English|Chinese", index=True)description = TextField(null=True, help_text="KB description")- embd_id = CharField(max_length=128, null=False, help_text="default embedding model ID", index=True)+ # A dataset may inherit an application-level empty string. GaussDB stores it+ # as NULL and restores it to "" when read.+ embd_id = EmptyStringCharField(max_length=128, null=False, help_text="default embedding model ID", index=True)tenant_embd_id = CharField(max_length=32, null=True, help_text="id in tenant_model", index=True)permission = CharField(max_length=16, null=False, help_text="me|team", default="me", index=True)created_by = CharField(max_length=32, null=False, index=True)@@ -930,7 +1341,9 @@ class File(DataBaseModel):location = CharField(max_length=255, null=True, help_text="where dose it store", index=True)size = BigIntegerField(default=0, index=True)type = CharField(max_length=32, null=False, help_text="file extension", index=True)- source_type = CharField(max_length=128, null=False, default="", help_text="where dose this document come from", index=True)+ # File.source_type historically defaults to "". Only GaussDB makes the+ # storage column nullable; MySQL and PostgreSQL retain NOT NULL.+ source_type = EmptyStringCharField(max_length=128, null=False, default="", help_text="where dose this document come from", index=True)class Meta:db_table = "file"@@ -1004,17 +1417,25 @@ class Task(DataBaseModel):doc_id = CharField(max_length=32, null=False, index=True)from_page = IntegerField(default=0)to_page = IntegerField(default=MAXIMUM_TASK_PAGE_NUMBER)- task_type = CharField(max_length=32, null=False, default="")+ # Standard document parsing tasks historically use task_type="" to mean+ # that they are not dataflow, graph, memory, or another specialized task.+ # A/ORA-compatible GaussDB stores the bound empty string as NULL, so a NOT+ # NULL column would make /documents/parse fail while inserting the task.+ # Only DB_TYPE=gaussdb stores NULL, and the ORM restores "" when reading.+ task_type = EmptyStringCharField(max_length=32, null=False, default="")priority = IntegerField(default=0)begin_at = DateTimeField(null=True, index=True)process_duration = FloatField(default=0)progress = FloatField(default=0, index=True)- progress_msg = TextField(null=True, help_text="process message", default="")+ # progress_msg and chunk_ids are nullable, but callers commonly concatenate,+ # strip, or split them as strings. The field maps GaussDB NULL to "" without+ # affecting MySQL or PostgreSQL.+ progress_msg = EmptyStringTextField(null=True, help_text="process message", default="")retry_count = IntegerField(default=0)digest = TextField(null=True, help_text="task digest", default="")- chunk_ids = LongTextField(null=True, help_text="chunk ids", default="")+ chunk_ids = EmptyStringLongTextField(null=True, help_text="chunk ids", default="")class Dialog(DataBaseModel):@@ -1024,7 +1445,9 @@ class Dialog(DataBaseModel):description = TextField(null=True, help_text="Dialog description")icon = TextField(null=True, help_text="icon base64 string")language = CharField(max_length=32, null=True, default="Chinese" if "zh_CN" in os.getenv("LANG", "") else "English", help_text="English|Chinese", index=True)- llm_id = CharField(max_length=128, null=False, help_text="default llm ID")+ # Map application-level empty chat/reranker model IDs to storage NULL in the+ # field instead of handling None throughout the business code.+ llm_id = EmptyStringCharField(max_length=128, null=False, help_text="default llm ID")tenant_llm_id = CharField(max_length=32, null=True, help_text="id in tenant_model", index=True)llm_setting = JSONField(null=False, default={"temperature": 0.1, "top_p": 0.3, "frequency_penalty": 0.7, "presence_penalty": 0.4, "max_tokens": 512})@@ -1044,7 +1467,7 @@ class Dialog(DataBaseModel):do_refer = CharField(max_length=1, null=False, default="1", help_text="it needs to insert reference index into answer or not")- rerank_id = CharField(max_length=128, null=False, help_text="default rerank model ID")+ rerank_id = EmptyStringCharField(max_length=128, null=False, help_text="default rerank model ID")tenant_rerank_id = CharField(max_length=32, null=True, help_text="id in tenant_model", index=True)kb_ids = JSONField(null=False, default=[])status = CharField(max_length=1, null=True, help_text="is it validate(0: wasted, 1: validate)", default="1", index=True)@@ -1081,7 +1504,7 @@ class API4Conversation(DataBaseModel):id = CharField(max_length=32, primary_key=True)name = CharField(max_length=255, null=True, help_text="conversation name", index=False)dialog_id = CharField(max_length=32, null=False, index=True)- user_id = CharField(max_length=255, null=False, help_text="user_id", index=True)+ user_id = EmptyStringCharField(max_length=255, null=False, help_text="user_id", index=True)exp_user_id = CharField(max_length=255, null=True, help_text="exp_user_id", index=True)message = JSONField(null=True)reference = JSONField(null=True, default=[])@@ -1109,7 +1532,13 @@ class UserCanvas(DataBaseModel):description = TextField(null=True, help_text="Canvas description")canvas_type = CharField(max_length=32, null=True, help_text="Canvas type", index=True)canvas_category = CharField(max_length=32, null=False, default="agent_canvas", help_text="Canvas category: agent_canvas|dataflow_canvas", index=True)- tags = CharField(max_length=512, null=False, default="", help_text="Comma-separated tags for organizing agents", index=True)+ tags = EmptyStringCharField(+ max_length=512,+ null=False,+ default="",+ help_text="Comma-separated tags for organizing agents",+ index=True,+ )dsl = JSONField(null=True, default={})class Meta:@@ -1345,9 +1774,15 @@ class SyncLogs(DataBaseModel):new_docs_indexed = IntegerField(default=0, index=False)total_docs_indexed = IntegerField(default=0, index=False)docs_removed_from_index = IntegerField(default=0, index=False)- error_msg = TextField(null=False, help_text="process message", default="")+ # Connector scheduling logs use error_msg="" to mean that no error is+ # present. A/ORA-compatible GaussDB stores it as NULL, so keeping this column+ # NOT NULL would break PUT /datasets/<id> when it schedules sync_logs. The+ # field permits storage NULL and restores the application-level empty string.+ error_msg = EmptyStringTextField(null=False, help_text="process message", default="")error_count = IntegerField(default=0, index=False)- full_exception_trace = TextField(null=True, help_text="process message", default="")+ # full_exception_trace is nullable, but callers append stack traces as text.+ # Hide GaussDB storage NULL behind the same application-level empty string.+ full_exception_trace = EmptyStringTextField(null=True, help_text="process message", default="")time_started = DateTimeField(null=True, index=True)poll_range_start = DateTimeTzField(max_length=255, null=True, index=True)poll_range_end = DateTimeTzField(max_length=255, null=True, index=True)@@ -1364,9 +1799,11 @@ class Memory(DataBaseModel):tenant_id = CharField(max_length=32, null=False, index=True)memory_type = IntegerField(null=False, default=1, index=True, help_text="Bit flags (LSB->MSB): 1=raw, 2=semantic, 4=episodic, 8=procedural. E.g., 5 enables raw + episodic.")storage_type = CharField(max_length=32, default="table", null=False, index=True, help_text="table|graph")- embd_id = CharField(max_length=128, null=False, index=False, help_text="embedding model ID")+ # Keep empty model-ID handling in the field so storage NULL semantics do not+ # leak into Memory Store callers.+ embd_id = EmptyStringCharField(max_length=128, null=False, index=False, help_text="embedding model ID")tenant_embd_id = CharField(max_length=32, null=True, help_text="id in tenant_model", index=True)- llm_id = CharField(max_length=128, null=False, index=False, help_text="chat model ID")+ llm_id = EmptyStringCharField(max_length=128, null=False, index=False, help_text="chat model ID")tenant_llm_id = CharField(max_length=32, null=True, help_text="id in tenant_model", index=True)permissions = CharField(max_length=16, null=False, index=True, help_text="me|team", default="me")description = TextField(null=True, help_text="description")@@ -1384,7 +1821,10 @@ class SystemSettings(DataBaseModel):name = CharField(max_length=128, primary_key=True)source = CharField(max_length=32, null=False, index=False)data_type = CharField(max_length=32, null=False, index=False)- value = TextField(null=False, help_text="Configuration value (JSON, string, etc.)")+ # system_settings.json contains empty configuration values. Store them as+ # NULL on GaussDB and restore "" here instead of globally converting SQL+ # results and affecting fields where NULL is meaningful.+ value = EmptyStringTextField(null=False, help_text="Configuration value (JSON, string, etc.)")class Meta:db_table = "system_settings"@@ -1448,7 +1888,63 @@ class TenantModelGroupMapping(DataBaseModel):primary_key = CompositeKey("group_id", "provider_id", "instance_id", "model_id")+GAUSSDB_EMPTY_STRING_COMPATIBLE_COLUMNS = (+ ("user", ("nickname",)),+ ("tenant", ("llm_id", "embd_id", "asr_id", "img2txt_id", "rerank_id")),+ ("knowledgebase", ("embd_id",)),+ ("dialog", ("llm_id", "rerank_id")),+ ("memory", ("embd_id", "llm_id")),+ ("file", ("source_type",)),+ ("system_settings", ("value",)),+ ("task", ("task_type",)),+ ("sync_logs", ("error_msg", "full_exception_trace")),+ ("api_4_conversation", ("user_id",)),+ ("user_canvas", ("tags",)),+)+++def _quote_identifier_for_gaussdb(identifier: str) -> str:+ return '"' + identifier.replace('"', '""') + '"'+++def relax_gaussdb_empty_string_compatible_columns():+ if not is_gaussdb_compatible_database():+ return++ # These fields retain their historical application-level empty-string+ # semantics, but A/ORA-compatible GaussDB stores "" as NULL. Existing+ # databases must therefore make only these listed columns nullable. MySQL,+ # PostgreSQL, and unlisted fields remain unchanged.+ for table_name, column_names in GAUSSDB_EMPTY_STRING_COMPATIBLE_COLUMNS:+ quoted_table = _quote_identifier_for_gaussdb(table_name)+ for column_name in column_names:+ quoted_column = _quote_identifier_for_gaussdb(column_name)+ try:+ DB.execute_sql(f"ALTER TABLE {quoted_table} ALTER COLUMN {quoted_column} DROP NOT NULL")+ except Exception as ex:+ # Match the migration framework's behavior for failed column+ # additions or type changes: log at critical and continue+ # startup. This avoids aborting on a missing historical column+ # while identifying the constraint that was not relaxed.+ logging.critical(+ "Failed to relax GaussDB empty-string compatible column %s.%s: %s",+ table_name,+ column_name,+ ex,+ )++def alter_db_add_column(migrator, table_name, column_name, column_type):+ if is_gaussdb_compatible_database():+ try:+ migrate(migrator.add_column(table_name, column_name, column_type))+ except (OperationalError, ProgrammingError) as ex:+ if not is_duplicate_column_error(ex):+ logging.critical(f"Failed to add {settings.DATABASE_TYPE.upper()}.{table_name} column {column_name}, operation error: {ex}")+ except Exception as ex:+ logging.critical(f"Failed to add {settings.DATABASE_TYPE.upper()}.{table_name} column {column_name}, error: {ex}")+ return+try:migrate(migrator.add_column(table_name, column_name, column_type))except OperationalError as ex:@@ -1528,11 +2024,38 @@ def ensure_model_indexes(migrator):logging.error(f"Failed to create {'unique ' if unique else ''}index on {table_name} ({', '.join(columns)}): {ex}")+def _gaussdb_user_email_unique_index_exists() -> bool:+ cursor = DB.execute_sql(+ """+ SELECT COUNT(*)+ FROM pg_indexes+ WHERE schemaname = current_schema()+ AND tablename = 'user'+ AND lower(indexdef) LIKE %s+ AND (+ lower(indexdef) LIKE %s+ OR lower(indexdef) LIKE %s+ )+ """,+ ("create unique index%", "%(email)%", '%("email")%'),+ )+ result = cursor.fetchone()+ return bool(result and result[0] > 0)++def migrate_add_unique_email(migrator):"""Deduplicates user emails and add UNIQUE constraint to email column (idempotent)"""# step 0: check existing index state on user.email and prepare for unique constrainttry:- if settings.DATABASE_TYPE.upper() == "POSTGRES":+ if is_gaussdb_compatible_database():+ # GaussDB cannot use MySQL information_schema.statistics or+ # backtick syntax. Query pg_indexes through a distinct GaussDB path+ # so future catalog changes remain isolated in+ # _gaussdb_user_email_unique_index_exists.+ if _gaussdb_user_email_unique_index_exists():+ logging.info("UNIQUE index on user.email already exists, skipping migration")+ return+ elif settings.DATABASE_TYPE.upper() == "POSTGRES":cursor = DB.execute_sql("""SELECT COUNT(*)FROM pg_indexes@@ -1586,8 +2109,12 @@ def migrate_add_unique_email(migrator):migrate(migrator.add_index("user", ("email",), unique=True))except (OperationalError, ProgrammingError) as ex:msg = str(ex)- # MySQL 1061 "Duplicate key name" or PostgreSQL "already exists" -> already migrated- if "1061" in msg or "Duplicate key name" in msg or "already exists" in msg.lower():+ if is_gaussdb_compatible_database():+ already_exists = is_duplicate_object_error(ex)+ else:+ # Preserve the upstream MySQL/PostgreSQL handling.+ already_exists = "1061" in msg or "Duplicate key name" in msg or "already exists" in msg.lower()+ if already_exists:passelse:logging.critical("Failed to add UNIQUE constraint on user.email: %s", ex)@@ -1597,7 +2124,12 @@ def migrate_add_unique_email(migrator):def update_tenant_llm_to_id_primary_key():"""Add ID and set to primary key step by step."""- if settings.DATABASE_TYPE.upper() == "POSTGRES":+ if is_gaussdb_compatible_database():+ # Use the GaussDB implementation for sequences, catalogs, and constraint+ # DDL instead of MySQL AUTO_INCREMENT/INFORMATION_SCHEMA or a branch+ # labeled as PostgreSQL.+ _update_tenant_llm_to_id_primary_key_gaussdb()+ elif settings.DATABASE_TYPE.upper() == "POSTGRES":_update_tenant_llm_to_id_primary_key_postgres()else:_update_tenant_llm_to_id_primary_key_mysql()@@ -1734,12 +2266,113 @@ def _update_tenant_llm_to_id_primary_key_postgres():DB.execute_sql("ALTER TABLE tenant_llm DROP COLUMN temp_id")+def _update_tenant_llm_to_id_primary_key_gaussdb():+ """GaussDB-compatible implementation: add sequence-backed primary key."""+ # GaussDB can perform this migration with pg_attribute, pg_constraint, and+ # a sequence. Keep the entry point distinct so catalog, ctid, or sequence+ # differences can be handled without changing the PostgreSQL path.+ _update_tenant_llm_to_id_primary_key_gaussdb_catalog()+++def _update_tenant_llm_to_id_primary_key_gaussdb_catalog():+ """GaussDB pg_catalog implementation for the tenant_llm migration."""+ try:+ with DB.atomic():+ # 0. Check if 'id' column already exists+ # information_schema rowcount is unreliable on GaussDB. Query+ # pg_attribute and use fetchone() to prevent a duplicate migration.+ cursor = DB.execute_sql("""+ SELECT a.attname+ FROM pg_attribute a+ JOIN pg_class c ON c.oid = a.attrelid+ JOIN pg_namespace n ON n.oid = c.relnamespace+ WHERE n.nspname = current_schema()+ AND c.relname = 'tenant_llm'+ AND a.attname = 'id'+ AND NOT a.attisdropped+ """)+ if cursor.fetchone():+ return++ # 1. Add nullable integer column+ DB.execute_sql("ALTER TABLE tenant_llm ADD COLUMN temp_id INTEGER NULL")++ # 2. Assign sequential row numbers ordered consistently+ DB.execute_sql("""+ UPDATE tenant_llm+ SET temp_id = subq.rn+ FROM (+ SELECT ctid,+ ROW_NUMBER() OVER (ORDER BY tenant_id, llm_factory, llm_name) AS rn+ FROM tenant_llm+ ) AS subq+ WHERE tenant_llm.ctid = subq.ctid+ """)++ # 3. Drop old composite primary key constraint+ # Older tenant_llm tables use (tenant_id, llm_factory, llm_name) as+ # a composite primary key. The new schema needs an id primary key+ # and a composite unique constraint. Read the actual constraint name+ # from pg_constraint instead of assuming a fixed name.+ cursor = DB.execute_sql("""+ SELECT con.conname+ FROM pg_constraint con+ JOIN pg_class c ON c.oid = con.conrelid+ JOIN pg_namespace n ON n.oid = c.relnamespace+ WHERE n.nspname = current_schema()+ AND c.relname = 'tenant_llm'+ AND con.contype = 'p'+ """)+ row = cursor.fetchone()+ if row:+ DB.execute_sql(f'ALTER TABLE tenant_llm DROP CONSTRAINT "{row[0]}"')++ # 4. Make temp_id NOT NULL and create a sequence for it+ # GaussDB and PostgreSQL do not use MySQL AUTO_INCREMENT here. A+ # sequence with nextval() provides the generated key, and OWNED BY+ # keeps its lifecycle tied to the column after temp_id becomes id.+ DB.execute_sql("ALTER TABLE tenant_llm ALTER COLUMN temp_id SET NOT NULL")+ DB.execute_sql("CREATE SEQUENCE IF NOT EXISTS tenant_llm_id_seq")+ DB.execute_sql("""+ SELECT setval('tenant_llm_id_seq', COALESCE((SELECT MAX(temp_id) FROM tenant_llm), 0))+ """)+ DB.execute_sql("ALTER TABLE tenant_llm ALTER COLUMN temp_id SET DEFAULT nextval('tenant_llm_id_seq')")+ DB.execute_sql("ALTER SEQUENCE tenant_llm_id_seq OWNED BY tenant_llm.temp_id")+ DB.execute_sql("ALTER TABLE tenant_llm ADD PRIMARY KEY (temp_id)")++ # 5. Add unique constraint+ DB.execute_sql("""+ ALTER TABLE tenant_llm+ ADD CONSTRAINT uk_tenant_llm UNIQUE (tenant_id, llm_factory, llm_name)+ """)++ # 6. Rename temp_id to id+ DB.execute_sql("ALTER TABLE tenant_llm RENAME COLUMN temp_id TO id")++ logging.info("Successfully updated tenant_llm to id primary key (GaussDB).")++ except Exception as e:+ logging.error(str(e))+ cursor = DB.execute_sql("""+ SELECT a.attname+ FROM pg_attribute a+ JOIN pg_class c ON c.oid = a.attrelid+ JOIN pg_namespace n ON n.oid = c.relnamespace+ WHERE n.nspname = current_schema()+ AND c.relname = 'tenant_llm'+ AND a.attname = 'temp_id'+ AND NOT a.attisdropped+ """)+ if cursor.fetchone():+ DB.execute_sql("ALTER TABLE tenant_llm DROP COLUMN temp_id")++def migrate_db():logging.disable(logging.ERROR)migrator = DatabaseMigrator[settings.DATABASE_TYPE.upper()].value(DB)- alter_db_add_column(migrator, "file", "source_type", CharField(max_length=128, null=False, default="", help_text="where dose this document come from", index=True))- alter_db_add_column(migrator, "tenant", "rerank_id", CharField(max_length=128, null=False, default="BAAI/bge-reranker-v2-m3", help_text="default rerank model ID"))- alter_db_add_column(migrator, "dialog", "rerank_id", CharField(max_length=128, null=False, default="", help_text="default rerank model ID"))+ alter_db_add_column(migrator, "file", "source_type", EmptyStringCharField(max_length=128, null=False, default="", help_text="where dose this document come from", index=True))+ alter_db_add_column(migrator, "tenant", "rerank_id", EmptyStringCharField(max_length=128, null=False, default="BAAI/bge-reranker-v2-m3", help_text="default rerank model ID"))+ alter_db_add_column(migrator, "dialog", "rerank_id", EmptyStringCharField(max_length=128, null=False, default="", help_text="default rerank model ID"))alter_db_column_type(migrator, "dialog", "top_k", IntegerField(default=1024))alter_db_add_column(migrator, "tenant_llm", "api_key", CharField(max_length=2048, null=True, help_text="API KEY", index=True))alter_db_add_column(migrator, "api_token", "source", CharField(max_length=16, null=True, help_text="none|agent|dialog", index=True))@@ -1752,9 +2385,9 @@ def migrate_db():alter_db_add_column(migrator, "knowledgebase", "pagerank", IntegerField(default=0, index=False))alter_db_add_column(migrator, "api_token", "beta", CharField(max_length=255, null=True, index=True))alter_db_add_column(migrator, "task", "digest", TextField(null=True, help_text="task digest", default=""))- alter_db_add_column(migrator, "task", "chunk_ids", LongTextField(null=True, help_text="chunk ids", default=""))+ alter_db_add_column(migrator, "task", "chunk_ids", EmptyStringLongTextField(null=True, help_text="chunk ids", default=""))alter_db_add_column(migrator, "conversation", "user_id", CharField(max_length=255, null=True, help_text="user_id", index=True))- alter_db_add_column(migrator, "task", "task_type", CharField(max_length=32, null=False, default=""))+ alter_db_add_column(migrator, "task", "task_type", EmptyStringCharField(max_length=32, null=False, default=""))alter_db_add_column(migrator, "task", "priority", IntegerField(default=0))alter_db_add_column(migrator, "user_canvas", "permission", CharField(max_length=16, null=False, help_text="me|team", default="me", index=True))alter_db_add_column(migrator, "user_canvas", "release", BooleanField(null=False, help_text="is released", default=False, index=True))@@ -1795,11 +2428,24 @@ def migrate_db():alter_db_add_column(migrator, "api_4_conversation", "name", CharField(max_length=255, null=True, help_text="conversation name", index=False))alter_db_add_column(migrator, "api_4_conversation", "exp_user_id", CharField(max_length=255, null=True, help_text="exp_user_id", index=True))alter_db_add_column(migrator, "sync_logs", "task_type", CharField(max_length=32, null=False, default="sync", index=True))- # Migrate system_settings.value from CharField to TextField for longer sandbox configs- alter_db_column_type(migrator, "system_settings", "value", TextField(null=False, help_text="Configuration value (JSON, string, etc.)"))+ # Migrate system_settings.value from CharField to TextField for longer sandbox configs.+ # Use the dedicated field so the GaussDB column remains nullable after the+ # type change and the ORM restores NULL to the application-level "".+ alter_db_column_type(migrator, "system_settings", "value", EmptyStringTextField(null=False, help_text="Configuration value (JSON, string, etc.)"))alter_db_add_column(migrator, "document", "content_hash", CharField(max_length=32, null=True, help_text="xxhash128 of document content for change detection", default="", index=True))alter_db_add_column(migrator, "user_canvas_version", "release", BooleanField(null=False, help_text="is released", default=False, index=True))- alter_db_add_column(migrator, "user_canvas", "tags", CharField(max_length=512, null=False, default="", help_text="Comma-separated tags for organizing agents", index=True))+ alter_db_add_column(+ migrator,+ "user_canvas",+ "tags",+ EmptyStringCharField(+ max_length=512,+ null=False,+ default="",+ help_text="Comma-separated tags for organizing agents",+ index=True,+ ),+ )alter_db_add_column(migrator, "api_4_conversation", "version_title", CharField(max_length=255, null=True, help_text="canvas version title when session created", index=False))alter_db_column_type(migrator, "document", "size", BigIntegerField(default=0, index=True))alter_db_column_type(migrator, "file", "size", BigIntegerField(default=0, index=True))@@ -1817,6 +2463,9 @@ def migrate_db():alter_db_drop_index(migrator, "tenant_langfuse", "idx_tenant_langfuse_secret_key")alter_db_drop_index(migrator, "tenant_langfuse", "idx_tenant_langfuse_public_key")alter_db_drop_index(migrator, "tenant_langfuse", "idx_tenant_langfuse_host")+ # Run after all alter_db_* calls so newly added compatible columns, such as+ # user_canvas.tags, exist before their GaussDB NOT NULL constraints relax.+ relax_gaussdb_empty_string_compatible_columns()# Drop both the explicit "idx_*" name from later migrations AND the# Peewee-auto-derived "<table-as-classname>_<col1>_<col2>" name from the@@ -1842,7 +2491,11 @@ def migrate_db():migrate(migrator.drop_index(table_name, index_name))except (OperationalError, ProgrammingError) as ex:msg = str(ex)- if "1091" in msg or "can't DROP" in msg.lower() or "does not exist" in msg.lower() or "already exists" in msg.lower():+ if is_gaussdb_compatible_database():+ can_skip = is_undefined_object_error(ex) or is_duplicate_object_error(ex)+ else:+ can_skip = "1091" in msg or "can't DROP" in msg.lower() or "does not exist" in msg.lower() or "already exists" in msg.lower()+ if can_skip:passelse:logging.critical(f"Failed to drop index {index_name} on {table_name}: {ex}")diff --git a/api/db/db_utils.py b/api/db/db_utils.pyindex 98e44f8d53..2500156acf 100644--- a/api/db/db_utils.py+++ b/api/db/db_utils.py@@ -20,7 +20,47 @@ from playhouse.pool import PooledMySQLDatabasefrom common.time_utils import current_timestamp, timestamp_to_date-from api.db.db_models import DB, DataBaseModel+from api.db.db_models import DB, DataBaseModel, is_gaussdb_compatible_database+++def _gaussdb_replace_insert_by_id(model, batch, preserve):+ """Replace Peewee ``ON CONFLICT`` upserts on A/ORA-compatible GaussDB.++ Peewee's PostgreSQL dialect renders+ ``insert_many(...).on_conflict(conflict_target="id")`` as+ ``INSERT ... ON CONFLICT (...) DO UPDATE ... RETURNING``. A/ORA-compatible+ GaussDB rejects that syntax, which breaks APIs such as+ ``/datasets/{id}/documents/parse`` when they batch-write tasks.++ For ``DB_TYPE=gaussdb`` only, split this replace-by-id operation into two+ steps:++ * Query the IDs already present in the batch and update those rows using+ the same columns that Peewee's PostgreSQL path would preserve.+ * Insert rows with new IDs without generating unsupported ``ON CONFLICT``+ SQL.++ MySQL retains its on-duplicate-key behavior, and PostgreSQL/OceanBase retain+ Peewee's existing generation path. SQL for other databases is unchanged.+ """+ ids = [data.get("id") for data in batch if data.get("id") is not None]+ existing_ids = set()+ if ids:+ existing_ids = {row[0] for row in model.select(model.id).where(model.id.in_(ids)).tuples()}++ insert_rows = []+ update_columns = [column for column in preserve if column != "id"]+ for data in batch:+ row_id = data.get("id")+ if row_id in existing_ids:+ update_payload = {column: data[column] for column in update_columns if column in data}+ if update_payload:+ model.update(update_payload).where(model.id == row_id).execute()+ else:+ insert_rows.append(data)++ if insert_rows:+ model.insert_many(insert_rows).execute()@DB.connection_context()@@ -46,6 +86,13 @@ def bulk_insert_into_db(model, data_source, replace_on_conflict=False):if replace_on_conflict:if isinstance(DB, PooledMySQLDatabase):query = query.on_conflict(preserve=preserve)+ elif is_gaussdb_compatible_database():+ # Peewee's PostgreSQL path emits `ON CONFLICT (...) DO+ # UPDATE ... RETURNING`, which A/ORA-compatible GaussDB+ # rejects. Query existing IDs first, then UPDATE or INSERT+ # to preserve the operation without leaking PostgreSQL SQL.+ _gaussdb_replace_insert_by_id(model, data_source[i : i + batch_size], preserve)+ continueelse:query = query.on_conflict(conflict_target="id", preserve=preserve)query.execute()diff --git a/api/db/gaussdb_error_utils.py b/api/db/gaussdb_error_utils.pynew file mode 100644index 0000000000..d42d51c26a--- /dev/null+++ b/api/db/gaussdb_error_utils.py@@ -0,0 +1,174 @@+#+# Copyright 2024 The InfiniFlow Authors. All Rights Reserved.+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+#+"""GaussDB metadata adapter exception classification helpers.++This module deliberately avoids importing ``db_models`` or service classes so+the GaussDB pool, migrations, and retry decorator do not form import cycles.+"""++from __future__ import annotations++from collections.abc import Iterable+++# Connection-related SQLSTATE values reported by GaussDB/PostgreSQL/libpq.+# GaussDB exposes errors through psycopg2/libpq, so connection failures and+# server rejections should be classified by SQLSTATE instead of English error+# text. Text matching is only a fallback when the driver omits pgcode. The+# 08xxx class covers connection exceptions, while 57P0x covers shutdowns and+# temporarily unavailable servers.+PSYCOPG_CONNECTION_SQLSTATES = {+ "08000", # connection_exception+ "08001", # sqlclient_unable_to_establish_sqlconnection+ "08003", # connection_does_not_exist+ "08004", # sqlserver_rejected_establishment_of_sqlconnection+ "08006", # connection_failure+ "08007", # transaction_resolution_unknown+ "08P01", # protocol_violation+ "57P01", # admin_shutdown+ "57P02", # crash_shutdown+ "57P03", # cannot_connect_now+}++# Transaction-conflict SQLSTATE values observed on GaussDB:+# 40P01: deadlock detected;+# 40001: could not serialize access due to concurrent update;+# 55P03: FOR UPDATE NOWAIT could not acquire the row lock.+# Each indicates a statement or transaction that can be retried as a complete+# RAGFlow database operation.+PSYCOPG_RETRYABLE_TRANSACTION_SQLSTATES = {"40P01", "40001", "55P03"}++# SQLSTATE values used to make DDL migrations idempotent. Migration code must+# distinguish an already-applied change from a real failure. GaussDB reports+# duplicate columns as 42701, duplicate table/index names as 42P07, and missing+# index/constraint objects as 42704.+PSYCOPG_DUPLICATE_COLUMN_SQLSTATES = {"42701"}+PSYCOPG_DUPLICATE_OBJECT_SQLSTATES = {"42P07"}+PSYCOPG_UNDEFINED_OBJECT_SQLSTATES = {"42704"}++CONNECTION_ERROR_MESSAGES = (+ "server closed",+ "connection refused",+ "no connection to the server",+ "terminating connection",+ "ssl connection has been closed unexpectedly",+ "ssl syscall error: eof detected",+)+++def _iter_exception_chain(exc: BaseException) -> Iterable[BaseException | object]:+ """Iterate through Peewee wrappers, chained errors, and driver errors in args."""+ seen: set[int] = set()+ stack: list[BaseException | object] = [exc]+ while stack:+ current = stack.pop(0)+ current_id = id(current)+ if current_id in seen:+ continue+ seen.add(current_id)+ yield current++ for attr in ("__cause__", "__context__", "orig", "original_exception"):+ nested = getattr(current, attr, None)+ if nested is not None:+ stack.append(nested)++ for arg in getattr(current, "args", ()) or ():+ if hasattr(arg, "pgcode"):+ stack.append(arg)+++def sqlstate_from_exception(exc: BaseException) -> str | None:+ """Extract SQLSTATE from a psycopg2 error or a Peewee-wrapped error."""+ for current in _iter_exception_chain(exc):+ pgcode = getattr(current, "pgcode", None)+ if pgcode:+ return str(pgcode)++ diag = getattr(current, "diag", None)+ sqlstate = getattr(diag, "sqlstate", None) if diag else None+ if sqlstate:+ return str(sqlstate)++ return None+++def exception_text(exc: BaseException) -> str:+ return str(exc).lower()+++def mysql_errno_from_exception(exc: BaseException) -> int | None:+ args = getattr(exc, "args", ())+ if args and isinstance(args[0], int):+ return args[0]+ return None+++def is_psycopg_connection_error(exc: BaseException) -> bool:+ sqlstate = sqlstate_from_exception(exc)+ if sqlstate is not None:+ return sqlstate.startswith("08") or sqlstate in {"57P01", "57P02", "57P03"}++ if exc.__class__.__name__ == "InterfaceError":+ return True++ text = exception_text(exc)+ return any(message in text for message in CONNECTION_ERROR_MESSAGES)+++def is_retryable_transaction_error(exc: BaseException) -> bool:+ sqlstate = sqlstate_from_exception(exc)+ if sqlstate in PSYCOPG_RETRYABLE_TRANSACTION_SQLSTATES:+ return True++ text = exception_text(exc)+ return "deadlock detected" in text or "could not serialize access" in text or "could not obtain lock" in text+++def is_duplicate_column_error(exc: BaseException) -> bool:+ if mysql_errno_from_exception(exc) == 1060:+ return True++ sqlstate = sqlstate_from_exception(exc)+ if sqlstate in PSYCOPG_DUPLICATE_COLUMN_SQLSTATES:+ return True++ text = exception_text(exc)+ return "duplicate column" in text or ("column" in text and "already exists" in text)+++def is_duplicate_object_error(exc: BaseException) -> bool:+ if mysql_errno_from_exception(exc) == 1061:+ return True++ sqlstate = sqlstate_from_exception(exc)+ if sqlstate in PSYCOPG_DUPLICATE_OBJECT_SQLSTATES:+ return True++ text = exception_text(exc)+ return "duplicate key name" in text or "already exists" in text+++def is_undefined_object_error(exc: BaseException) -> bool:+ if mysql_errno_from_exception(exc) == 1091:+ return True++ sqlstate = sqlstate_from_exception(exc)+ if sqlstate in PSYCOPG_UNDEFINED_OBJECT_SQLSTATES:+ return True++ text = exception_text(exc)+ return "can't drop" in text or "does not exist" in textdiff --git a/api/db/services/common_service.py b/api/db/services/common_service.pyindex 93fd5f8a83..2b7046a0cf 100644--- a/api/db/services/common_service.py+++ b/api/db/services/common_service.py@@ -23,12 +23,26 @@ import peeweefrom peewee import InterfaceError, OperationalErrorfrom api.db.db_models import DB+from api.db.gaussdb_error_utils import is_retryable_transaction_error+from common import settingsfrom common.misc_utils import get_uuidfrom common.time_utils import current_timestamp, datetime_formatdef _is_deadlock_error(exc: OperationalError) -> bool:- return isinstance(exc, OperationalError) and bool(getattr(exc, "args", ())) and exc.args[0] == 1213+ if not isinstance(exc, OperationalError):+ return False++ args = getattr(exc, "args", ())+ if args and args[0] == 1213:+ return True++ # Keep the existing helper name to avoid changing every caller, although it+ # now recognizes all retryable transaction conflicts. GaussDB reports+ # 40P01 for deadlocks, 40001 for serialization failures, and 55P03 when a+ # NOWAIT row lock is unavailable. Peewee omits pgcode on the outer error, so+ # the SQLSTATE must be read from the psycopg2 error in __context__.+ return settings.DATABASE_TYPE.lower() == "gaussdb" and is_retryable_transaction_error(exc)def retry_deadlock_operation(max_retries=3, retry_delay=0.1):@@ -274,6 +288,16 @@ class CommonService:# Number of records updateddata["update_time"] = current_timestamp()data["update_date"] = datetime_format(datetime.now())+ # Only the GaussDB adapter exposes this hook. Centralized GaussDB and+ # every other database retain the original update_by_id behavior.+ replace_update = getattr(DB, "replace_update_by_id", None)+ if replace_update is not None:+ num = replace_update(cls.model, pid, data)+ if num is not None:+ return num+ prepare_update = getattr(DB, "prepare_update_by_id", None)+ if prepare_update is not None:+ data = prepare_update(cls.model, pid, data)num = cls.model.update(data).where(cls.model.id == pid).execute()return numdiff --git a/api/db/services/connector_service.py b/api/db/services/connector_service.pyindex 502bd1f51d..b940e6e663 100644--- a/api/db/services/connector_service.py+++ b/api/db/services/connector_service.py@@ -27,6 +27,7 @@ from api.db.services.common_service import CommonServicefrom api.db.services.document_service import DocumentServicefrom api.db.services.document_service import DocMetadataServicefrom api.utils.common import hash128+from common import settingsfrom common.misc_utils import get_uuidfrom common.constants import ConnectorTaskType, TaskStatusfrom common.settings import TIMEZONE@@ -35,6 +36,24 @@ from common.time_utils import current_timestamp, timestamp_to_dateLOGGER = logging.getLogger(__name__)+def _is_gaussdb_compatible_metadata_db() -> bool:+ return settings.normalize_database_type(settings.DATABASE_TYPE) == "gaussdb"+++def _gaussdb_poll_interval_expr(freq_field: str) -> SQL:+ return SQL(f"NOW() AT TIME ZONE '{settings.TIMEZONE}' - (t2.{freq_field} * INTERVAL '1 minute')")+++def _append_text_expr(field, suffix: str):+ if not _is_gaussdb_compatible_metadata_db():+ return field + suffix++ suffix = str(suffix or "")+ if not suffix:+ return None+ return fn.COALESCE(field + suffix, suffix)++class ConnectorService(CommonService):model = Connector@@ -226,6 +245,10 @@ class SyncLogsService(CommonService):Knowledgebase.name.alias("kb_name"),cls.model.status,]+ if _is_gaussdb_compatible_metadata_db():+ # GaussDB requires a DISTINCT query's ORDER BY expression in the+ # select list.+ fields.append(cls.model.update_time)if not connector_id:fields.append(Connector.config)@@ -239,11 +262,14 @@ class SyncLogsService(CommonService):if connector_id:query = query.where(cls.model.connector_id == connector_id)else:- database_type = os.getenv("DB_TYPE", "mysql")- if "postgres" in database_type.lower():- expr = SQL(f"NOW() AT TIME ZONE '{TIMEZONE}' - make_interval(mins => t2.refresh_freq)")+ if _is_gaussdb_compatible_metadata_db():+ expr = _gaussdb_poll_interval_expr("refresh_freq")else:- expr = SQL("NOW() - INTERVAL `t2`.`refresh_freq` MINUTE")+ database_type = os.getenv("DB_TYPE", "mysql")+ if "postgres" in database_type.lower():+ expr = SQL(f"NOW() AT TIME ZONE '{TIMEZONE}' - make_interval(mins => t2.refresh_freq)")+ else:+ expr = SQL("NOW() - INTERVAL `t2`.`refresh_freq` MINUTE")query = query.where(Connector.input_type == InputType.POLL, Connector.status == TaskStatus.SCHEDULE, cls.model.status == TaskStatus.SCHEDULE, cls.model.update_date < expr)query = query.distinct().order_by(cls.model.update_time.desc())@@ -318,11 +344,14 @@ class SyncLogsService(CommonService):cls.model.task_type == task_type,)- database_type = os.getenv("DB_TYPE", "mysql")- if "postgres" in database_type.lower():- expr = SQL(f"NOW() AT TIME ZONE '{TIMEZONE}' - make_interval(mins => t2.{freq_field})")+ if _is_gaussdb_compatible_metadata_db():+ expr = _gaussdb_poll_interval_expr(freq_field)else:- expr = SQL(f"NOW() - INTERVAL `t2`.`{freq_field}` MINUTE")+ database_type = os.getenv("DB_TYPE", "mysql")+ if "postgres" in database_type.lower():+ expr = SQL(f"NOW() AT TIME ZONE '{TIMEZONE}' - make_interval(mins => t2.{freq_field})")+ else:+ expr = SQL(f"NOW() - INTERVAL `t2`.`{freq_field}` MINUTE")query = query.where(cls.model.update_date < expr)return list(query.distinct().order_by(cls.model.update_time.desc()).dicts())@@ -397,34 +426,48 @@ class SyncLogsService(CommonService):logging.exception(e)task = cls.get_latest_task(connector_id, kb_id, task_type)if task:- cls.model.update(- status=TaskStatus.SCHEDULE, poll_range_start=poll_range_start, error_msg=cls.model.error_msg + str(e), full_exception_trace=cls.model.full_exception_trace + str(e)- ).where(cls.model.id == task.id).execute()+ update_payload = {+ "status": TaskStatus.SCHEDULE,+ "poll_range_start": poll_range_start,+ }+ error_msg_expr = _append_text_expr(cls.model.error_msg, str(e))+ trace_expr = _append_text_expr(cls.model.full_exception_trace, str(e))+ if error_msg_expr is not None:+ update_payload["error_msg"] = error_msg_expr+ if trace_expr is not None:+ update_payload["full_exception_trace"] = trace_expr+ cls.model.update(update_payload).where(cls.model.id == task.id).execute()ConnectorService.update_by_id(connector_id, {"status": TaskStatus.SCHEDULE})@classmethoddef increase_docs(cls, id, max_update, doc_num, err_msg="", error_count=0):# Keep sync monotonic.- cls.model.update(- new_docs_indexed=cls.model.new_docs_indexed + doc_num,- total_docs_indexed=cls.model.total_docs_indexed + doc_num,- poll_range_start=fn.COALESCE(fn.GREATEST(cls.model.poll_range_start, max_update), max_update),- poll_range_end=fn.COALESCE(fn.GREATEST(cls.model.poll_range_end, max_update), max_update),- error_msg=cls.model.error_msg + err_msg,- error_count=cls.model.error_count + error_count,- update_time=current_timestamp(),- update_date=timestamp_to_date(current_timestamp()),- ).where(cls.model.id == id).execute()+ update_payload = {+ "new_docs_indexed": cls.model.new_docs_indexed + doc_num,+ "total_docs_indexed": cls.model.total_docs_indexed + doc_num,+ "poll_range_start": fn.COALESCE(fn.GREATEST(cls.model.poll_range_start, max_update), max_update),+ "poll_range_end": fn.COALESCE(fn.GREATEST(cls.model.poll_range_end, max_update), max_update),+ "error_count": cls.model.error_count + error_count,+ "update_time": current_timestamp(),+ "update_date": timestamp_to_date(current_timestamp()),+ }+ error_msg_expr = _append_text_expr(cls.model.error_msg, err_msg)+ if error_msg_expr is not None:+ update_payload["error_msg"] = error_msg_expr+ cls.model.update(update_payload).where(cls.model.id == id).execute()@classmethoddef increase_removed_docs(cls, id, removed_count, err_msg="", error_count=0):- cls.model.update(- docs_removed_from_index=cls.model.docs_removed_from_index + removed_count,- error_msg=cls.model.error_msg + err_msg,- error_count=cls.model.error_count + error_count,- update_time=current_timestamp(),- update_date=timestamp_to_date(current_timestamp()),- ).where(cls.model.id == id).execute()+ update_payload = {+ "docs_removed_from_index": cls.model.docs_removed_from_index + removed_count,+ "error_count": cls.model.error_count + error_count,+ "update_time": current_timestamp(),+ "update_date": timestamp_to_date(current_timestamp()),+ }+ error_msg_expr = _append_text_expr(cls.model.error_msg, err_msg)+ if error_msg_expr is not None:+ update_payload["error_msg"] = error_msg_expr+ cls.model.update(update_payload).where(cls.model.id == id).execute()@classmethoddef duplicate_and_parse(cls, kb, docs, tenant_id, src, auto_parse=True):diff --git a/api/db/services/dialog_service.py b/api/db/services/dialog_service.pyindex a7ded8e8f1..d7168a69ba 100644--- a/api/db/services/dialog_service.py+++ b/api/db/services/dialog_service.py@@ -54,6 +54,7 @@ from rag.utils.web_search_conn import create_web_search_provider, has_web_searchfrom rag.utils.tts_cache import synthesize_with_cachefrom common.string_utils import remove_redundant_spacesfrom common import settings+from rag.utils import gaussdb_text_to_sqldef _chunk_kb_id_for_doc(row_dict, kb_ids, doc_id):@@ -1019,6 +1020,8 @@ async def use_sql(question, field_map, tenant_id, chat_mdl, quota=True, kb_ids=Ndoc_engine = "infinity"elif settings.DOC_ENGINE_OCEANBASE:doc_engine = "oceanbase"+ elif settings.DOC_ENGINE_GAUSSDB:+ doc_engine = "gaussdb"else:doc_engine = "es"@@ -1051,6 +1054,12 @@ async def use_sql(question, field_map, tenant_id, chat_mdl, quota=True, kb_ids=Nlogging.debug(f"use_sql: Using ES/OS table name: {table_name}")expected_doc_name_column = "docnm" if doc_engine == "infinity" else "docnm_kwd"+ if doc_engine == "gaussdb":+ if not kb_ids:+ raise ValueError("GaussDB Text-to-SQL requires kb_ids")+ for kid in kb_ids:+ _assert_valid_uuid(kid, "kb_id")+ gaussdb_validator = gaussdb_text_to_sql.build_validator(table_name, kb_ids, field_map)def has_source_columns(columns):"""Return True if the result set contains the columns needed to build source citations."""@@ -1059,6 +1068,8 @@ async def use_sql(question, field_map, tenant_id, chat_mdl, quota=True, kb_ids=Ndef is_aggregate_sql(sql_text):"""Return True if *sql_text* contains an aggregate function (COUNT, SUM, AVG, MAX, MIN, DISTINCT)."""+ if doc_engine == "gaussdb":+ return gaussdb_text_to_sql.is_aggregate_sql(sql_text)return bool(re.search(r"(count|sum|avg|max|min|distinct)\s*\(", (sql_text or "").lower()))def normalize_sql(sql):@@ -1080,12 +1091,13 @@ async def use_sql(question, field_map, tenant_id, chat_mdl, quota=True, kb_ids=Ndef add_kb_filter(sql):"""Inject validated scope filters into *sql*.- Infinity encodes single-KB scope in the table name, so only document- scope is injected there. All ids are validated before interpolation.+ Infinity encodes single-KB scope in the table name and the GaussDB+ validator injects its KB boundary, so only document scope is injected+ for those engines. All ids are validated before interpolation."""scope_filters = []sql_lower = sql.lower()- if doc_engine != "infinity" and kb_ids and "kb_id =" not in sql_lower and "kb_id=" not in sql_lower:+ if doc_engine not in ("infinity", "gaussdb") and kb_ids and "kb_id =" not in sql_lower and "kb_id=" not in sql_lower:for kid in kb_ids:_assert_valid_uuid(kid, "kb_id")if len(kb_ids) == 1:@@ -1180,6 +1192,10 @@ Question: {}Write SQL using json_extract_string() with exact field names. Include doc_id, docnm_kwd for data queries. Only SQL.""".format(table_name, ", ".join(json_field_names), "\n".join([f" - {field}" for field in json_field_names]), question)+ elif doc_engine == "gaussdb":+ row_count_override = f"SELECT COUNT(*) AS rows FROM {table_name}" if is_row_count_question(question) else None+ sys_prompt = gaussdb_text_to_sql.build_sql_prompt(table_name, field_map, question)+ user_prompt = gaussdb_text_to_sql.build_user_prompt(table_name, field_map, question)else:# Build ES/OS prompts with direct field accessrow_count_override = None@@ -1209,6 +1225,8 @@ Write SQL using exact field names above. Include doc_id, docnm_kwd for data quersql = await chat_mdl.async_chat(sys_prompt, [{"role": "user", "content": prompt}], {"temperature": 0.06})sql = normalize_sql(sql)sql = add_kb_filter(sql)+ if doc_engine == "gaussdb":+ sql = gaussdb_validator.validate_and_patch(sql).sqllogging.debug(f"{question} get SQL(refined): {sql}")tried_times += 1@@ -1216,6 +1234,8 @@ Write SQL using exact field names above. Include doc_id, docnm_kwd for data quertbl = settings.retriever.sql_retrieval(sql, format="json")if tbl is None:logging.debug("use_sql: SQL retrieval failed (returned None)")+ if doc_engine == "gaussdb":+ raise RuntimeError("SQL execution returned no result")return None, sqlrow_count = len(tbl.get("rows", []))if row_count == 0:@@ -1225,6 +1245,15 @@ Write SQL using exact field names above. Include doc_id, docnm_kwd for data querreturn tbl, sqlasync def repair_table_for_missing_source_columns(previous_sql):+ if doc_engine == "gaussdb":+ return await get_table(+ custom_user_prompt=gaussdb_text_to_sql.build_repair_prompt(+ table_name,+ field_map,+ question,+ previous_sql,+ )+ )if doc_engine in ("infinity", "oceanbase"):json_field_names = list(field_map.keys())repair_prompt = """Table name: {};@@ -1260,7 +1289,9 @@ Return ONLY SQL.""".format(table_name, "\n".join([f" - {k} ({v})" for k, v in fexcept Exception as e:logging.warning(f"use_sql: Initial SQL execution FAILED with error: {e}")# Build retry prompt with error information- if doc_engine in ("infinity", "oceanbase"):+ if doc_engine == "gaussdb":+ user_prompt = gaussdb_text_to_sql.build_retry_prompt(table_name, field_map, question, e)+ elif doc_engine in ("infinity", "oceanbase"):# Build Infinity error retry promptjson_field_names = list(field_map.keys())user_prompt = """@@ -1414,6 +1445,23 @@ Please correct the error and write SQL again using json_extract_string(chunk_dat# Keep original table format as answeranswer = "\n".join([columns, line, rows])+ if doc_engine == "gaussdb":+ try:+ chunks_sql = gaussdb_text_to_sql.build_aggregate_source_sql(+ sql,+ expected_doc_name_column,+ include_kb_id=not (kb_ids and len(kb_ids) == 1),+ )+ chunks_sql = gaussdb_validator.validate_and_patch(chunks_sql).sql+ logging.debug(f"use_sql: Fetching chunks with SQL: {chunks_sql}")+ chunks_tbl = settings.retriever.sql_retrieval(chunks_sql, format="json")+ reference = gaussdb_text_to_sql.build_source_reference(chunks_tbl, kb_ids)+ if reference:+ return {"answer": answer, "reference": reference, "prompt": sys_prompt}+ except Exception as e:+ logging.warning(f"use_sql: Failed to fetch chunks: {e}")+ return {"answer": answer, "reference": {"chunks": [], "doc_aggs": []}, "prompt": sys_prompt}+# Now fetch doc_id, docnm_kwd to provide source chunks# Extract WHERE clause from the original SQLwhere_match = re.search(r"\bwhere\b(.+?)(?:\bgroup by\b|\border by\b|\blimit\b|$)", sql, re.IGNORECASE)@@ -1495,6 +1543,14 @@ Please correct the error and write SQL again using json_extract_string(chunk_dat},"prompt": sys_prompt,}+ if doc_engine == "gaussdb":+ gaussdb_text_to_sql.complete_reference_kb_ids(+ result,+ table_name,+ kb_ids,+ gaussdb_validator,+ settings.retriever.sql_retrieval,+ )logging.debug(f"use_sql: Returning answer with {len(result['reference']['chunks'])} chunks from {len(doc_aggs)} documents")return resultdiff --git a/api/db/services/doc_metadata_service.py b/api/db/services/doc_metadata_service.pyindex f9cfefeaeb..4943fe276c 100644--- a/api/db/services/doc_metadata_service.py+++ b/api/db/services/doc_metadata_service.py@@ -272,6 +272,10 @@ class DocMetadataService:total_count = total_hits.get("value", len(page_docs))else:total_count = total_hits if total_hits else len(page_docs)+ # Handle GaussDB/OceanBase SearchResult(total, chunks) format+ elif hasattr(results, "chunks") and hasattr(results, "total"):+ page_docs = list(results.chunks or [])+ total_count = results.total# Handle list/iterable resultselif hasattr(results, "__iter__") and not isinstance(results, dict):page_docs = list(results)@@ -469,6 +473,22 @@ class DocMetadataService:logging.debug(f"[update_document_metadata] Updating doc_id: {doc_id}, kb_id: {kb_id}, meta_fields: {processed_meta}")+ if settings.DOC_ENGINE_GAUSSDB:+ if not settings.docStoreConn.index_exist(index_name, kb_id):+ result = settings.docStoreConn.create_doc_meta_idx(index_name)+ if result is False:+ logging.error(f"Failed to create metadata index {index_name}")+ return False+ insert_errors = settings.docStoreConn.insert(+ [{"id": doc_id, "kb_id": kb_id, "meta_fields": processed_meta}],+ index_name,+ kb_id,+ )+ if insert_errors:+ logging.error(f"Failed to update metadata for document {doc_id}: {insert_errors}")+ return False+ return True+# For Elasticsearch, use efficient partial updateif not settings.DOC_ENGINE_INFINITY and not settings.DOC_ENGINE_OCEANBASE and not settings.DOC_ENGINE_SERENEDB:# Check if index exists first@@ -864,6 +884,8 @@ class DocMetadataService:if settings.DOC_ENGINE_INFINITY:return cls._filter_doc_ids_by_metadata_infinity(index_name, kb_ids, filters, logic)+ elif settings.DOC_ENGINE_GAUSSDB:+ return cls._filter_doc_ids_by_metadata_gaussdb(index_name, kb_ids, filters, logic, limit)else:return cls._filter_doc_ids_by_metadata_es(index_name, kb_ids, filters, logic, limit)@@ -941,6 +963,54 @@ class DocMetadataService:logging.debug(f"ES metadata filter returned {len(unique)} matches for KBs {kb_ids}")return unique+ @classmethod+ def _filter_doc_ids_by_metadata_gaussdb(+ cls,+ index_name: str,+ kb_ids: List[str],+ filters: List[Dict],+ logic: str,+ limit: int,+ ) -> Optional[List[str]]:+ """GaussDB push-down path for metadata filtering."""+ from common.metadata_gaussdb_filter import (+ UnsupportedGaussDBMetaFilter,+ build_gaussdb_filter,+ fetch_gaussdb_metadata_doc_ids,+ is_pushdown_supported,+ )++ if not is_pushdown_supported(filters):+ return None++ try:+ sql_filter, filter_params = build_gaussdb_filter(filters, logic)+ except UnsupportedGaussDBMetaFilter as e:+ logging.error("GaussDB build metadata filter failed: %s", e.reason)+ return None++ try:+ probe_limit = int(limit or 0) + 1 if limit else limit+ doc_ids = fetch_gaussdb_metadata_doc_ids(settings.docStoreConn, index_name, kb_ids, sql_filter, filter_params, probe_limit)+ except Exception:+ logging.warning("GaussDB metadata filter push-down failed; falling back to in-memory filter", exc_info=True)+ return None++ seen: set[str] = set()+ unique: List[str] = []+ for did in doc_ids:+ if did in seen:+ continue+ seen.add(did)+ unique.append(did)++ if limit and len(unique) > limit:+ logging.warning(f"GaussDB metadata filter hit push-down cap, falling back to in-memory: cap={limit}, kb_ids={kb_ids}")+ return None++ logging.debug(f"GaussDB metadata filter returned {len(unique)} matches for KBs {kb_ids}")+ return unique[:limit] if limit else unique+@classmethoddef _filter_doc_ids_by_metadata_infinity(cls,diff --git a/api/utils/health_utils.py b/api/utils/health_utils.pyindex 896f62c132..de11a58bc0 100644--- a/api/utils/health_utils.py+++ b/api/utils/health_utils.py@@ -17,6 +17,7 @@ from datetime import datetimeimport jsonimport loggingimport os+import reimport requestsfrom timeit import default_timer as timer@@ -25,7 +26,15 @@ from rag.utils.redis_conn import REDIS_CONNfrom rag.utils.es_conn import ESConnectionfrom rag.utils.infinity_conn import InfinityConnectionfrom rag.utils.ob_conn import OBConnection+from rag.utils.gaussdb_conn import GaussDBConnectionfrom common import settings+from common.doc_store.gaussdb_conn_base import mask_gaussdb_text+++_GAUSSDB_SENSITIVE_KEY_PATTERN = re.compile(+ r"(?:^|[_-])(?:password|passwd|pwd|secret|token|api[_-]?key|dsn)$",+ re.IGNORECASE,+)def _ok_nok(ok: bool) -> str:@@ -122,6 +131,113 @@ def get_oceanbase_status():}+def _mask_gaussdb_string(value: str) -> str:+ return mask_gaussdb_text(value)+++def _log_gaussdb_error(context: str, exc: Exception) -> str:+ masked_error = _mask_gaussdb_string(str(exc))+ logging.error("%s (%s): %s", context, type(exc).__name__, masked_error)+ return masked_error+++def _mask_gaussdb_secret(value):+ if isinstance(value, dict):+ return {key: "***" if isinstance(key, str) and _GAUSSDB_SENSITIVE_KEY_PATTERN.search(key) else _mask_gaussdb_secret(item) for key, item in value.items()}+ if isinstance(value, list):+ return [_mask_gaussdb_secret(v) for v in value]+ if isinstance(value, tuple):+ return tuple(_mask_gaussdb_secret(v) for v in value)+ if isinstance(value, str):+ return _mask_gaussdb_string(value)+ return value+++def _get_gaussdb_connection():+ conn = getattr(settings, "docStoreConn", None)+ if conn is not None and getattr(conn, "db_type", lambda: None)() == "gaussdb":+ return conn+ return GaussDBConnection()+++def get_gaussdb_status():+ doc_engine = os.getenv("DOC_ENGINE", "elasticsearch").lower()+ if doc_engine != "gaussdb":+ return {+ "status": "not_configured",+ "message": "GaussDB is not configured as the document engine",+ }+ try:+ conn = _get_gaussdb_connection()+ health_info = _mask_gaussdb_secret(conn.health())+ performance_metrics = _mask_gaussdb_secret(conn.get_performance_metrics())+ status = "alive" if health_info.get("status") == "healthy" else "timeout"+ return {+ "status": status,+ "message": {+ "health": health_info,+ "performance": performance_metrics,+ },+ }+ except Exception as e:+ masked_error = _log_gaussdb_error("GaussDB status check failed", e)+ return {+ "status": "timeout",+ "message": f"error: {masked_error}",+ }+++def check_gaussdb_health() -> dict:+ doc_engine = os.getenv("DOC_ENGINE", "elasticsearch").lower()+ if doc_engine != "gaussdb":+ return {+ "status": "not_configured",+ "details": {+ "connection": "not_configured",+ "message": "GaussDB is not configured as the document engine",+ },+ }++ try:+ conn = _get_gaussdb_connection()+ health_info = _mask_gaussdb_secret(conn.health())+ performance_metrics = _mask_gaussdb_secret(conn.get_performance_metrics())+ connection_status = performance_metrics.get("connection", "unknown")+ if connection_status == "disconnected" or health_info.get("status") != "healthy":+ return {+ "status": "unhealthy",+ "details": {+ "connection": connection_status,+ "latency_ms": performance_metrics.get("latency_ms", 0),+ "uri": health_info.get("uri", "unknown"),+ "version": health_info.get("version_comment", "unknown"),+ "sql_compatibility": health_info.get("sql_compatibility", "unknown"),+ "error": health_info.get("error", performance_metrics.get("error", "")),+ },+ }++ is_healthy = connection_status == "connected" and performance_metrics.get("latency_ms", float("inf")) < 1000+ return {+ "status": "healthy" if is_healthy else "degraded",+ "details": {+ "connection": connection_status,+ "latency_ms": performance_metrics.get("latency_ms", 0),+ "uri": health_info.get("uri", "unknown"),+ "version": health_info.get("version_comment", "unknown"),+ "sql_compatibility": health_info.get("sql_compatibility", "unknown"),+ },+ }+ except Exception as e:+ masked_error = _log_gaussdb_error("GaussDB health check failed", e)+ return {+ "status": "unhealthy",+ "details": {+ "connection": "disconnected",+ "error": masked_error,+ },+ }++def check_oceanbase_health() -> dict:"""Check OceanBase health status with comprehensive metrics.@@ -193,6 +309,10 @@ def check_oceanbase_health() -> dict:def get_mysql_status():+ if settings.DATABASE_TYPE.lower() == "gaussdb":+ # GaussDB cannot execute MySQL's SHOW PROCESSLIST.+ return get_database_status()+try:cursor = DB.execute_sql("SHOW PROCESSLIST;")res_rows = cursor.fetchall()@@ -206,6 +326,29 @@ def get_mysql_status():}+def get_database_status():+ try:+ # SELECT 1 is the smallest probe supported by MySQL, PostgreSQL, and+ # GaussDB. Admin uses it for a GaussDB metadata database instead of a+ # MySQL-specific status query.+ cursor = DB.execute_sql("SELECT 1;")+ row = cursor.fetchone()+ cursor.close()+ return {+ "status": "alive",+ "message": {+ "database": settings.DATABASE_TYPE.lower(),+ "result": row[0] if row else None,+ },+ }+ except Exception as e:+ masked_error = _log_gaussdb_error("GaussDB metadata database status check failed", e)+ return {+ "status": "timeout",+ "message": f"error: {masked_error}",+ }++def _minio_scheme_and_verify():"""Determine URL scheme (http/https) and SSL verify flag for MinIO health check.diff --git a/common/config_utils.py b/common/config_utils.pyindex 524732265d..06db79debf 100644--- a/common/config_utils.py+++ b/common/config_utils.py@@ -79,6 +79,9 @@ def show_configs():msg = f"Current configs, from {conf_realpath(SERVICE_CONF)}:"for k, v in CONFIGS.items():if isinstance(v, dict):+ if k == "gaussdb" and isinstance(v.get("config"), dict) and "password" in v["config"]:+ v = copy.deepcopy(v)+ v["config"]["password"] = "*" * 8if "password" in v:v = copy.deepcopy(v)v["password"] = "*" * 8diff --git a/common/doc_store/gaussdb_conn_base.py b/common/doc_store/gaussdb_conn_base.pynew file mode 100644index 0000000000..684248b6d0--- /dev/null+++ b/common/doc_store/gaussdb_conn_base.py@@ -0,0 +1,1687 @@+#+# Copyright 2025 The InfiniFlow Authors. All Rights Reserved.+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+#+import hashlib+import json+import logging+import re+from dataclasses import dataclass+from numbers import Integral+from timeit import default_timer as timer+from typing import Any++from common.doc_store.doc_store_base import DocStoreConnection, MatchExpr, OrderByExpr+++_GAUSSDB_DSN_PATTERN = re.compile(r"(?i)\b(?:postgres(?:ql)?|gaussdb)://[^\s'\"<>]+")+_GAUSSDB_SECRET_PATTERN = re.compile(+ r"""+ (?P<prefix>+ (?P<key_quote>["']?)+ (?P<key>\b(?:password|passwd|access_token|token|api_key|secret|dsn)\b)+ (?P=key_quote)+ (?P<sep>\s*[:=]\s*)+ )+ (?P<value>+ "(?:\\.|""|[^"\\])*"+ | '(?:\\.|''|[^'\\])*'+ | "(?:\\.|[^"\\])*\Z+ | '(?:\\.|[^'\\])*\Z+ | [^\s]++ )+ """,+ flags=re.IGNORECASE | re.VERBOSE,+)+++def mask_gaussdb_text(value: Any) -> str:+ message = str(value)+ message = _GAUSSDB_DSN_PATTERN.sub("***", message)++ def replace_secret(match: re.Match[str]) -> str:+ secret = match.group("value")+ if len(secret) >= 2 and secret[0] in {"'", '"'} and secret[-1] == secret[0]:+ masked = f"{secret[0]}***{secret[0]}"+ else:+ masked = "***"+ return f"{match.group('prefix')}{masked}"++ return _GAUSSDB_SECRET_PATTERN.sub(replace_secret, message)+++class InvalidGaussDBObjectName(ValueError):+ pass+++class UnsafeGaussDBSQL(ValueError):+ pass+++EXTRA_FIELD_RE = re.compile(r"^[A-Za-z_][A-Za-z0-9_]{0,127}$")+++def validate_extra_field(name: str) -> str:+ field = str(name or "")+ if not EXTRA_FIELD_RE.fullmatch(field):+ raise InvalidGaussDBObjectName(name)+ return field+++def extra_field_expr(name: str, *, prefix: str | None = None, text: bool = False) -> str:+ field = validate_extra_field(name)+ source = f"{prefix}.extra" if prefix else "extra"+ operator = "->>" if text else "->"+ return f"({source} {operator} '{field}')"+++def _parse_gaussdb_statements(sql: str, dialect: str = "postgres"):+ from sqlglot import exp, parse++ try:+ return exp, parse(sql, read=dialect)+ except Exception:+ raise UnsafeGaussDBSQL("SQL parse failed") from None+++def is_gaussdb_aggregate_sql(sql: str) -> bool:+ exp, expressions = _parse_gaussdb_statements(sql)+ if len(expressions) != 1:+ raise UnsafeGaussDBSQL("exactly one SQL statement is allowed")+ return any(isinstance(node, exp.AggFunc) for node in expressions[0].walk())+++_FIELD_DESCRIPTOR_TYPES = {+ "bool",+ "boolean",+ "date",+ "datetime",+ "double",+ "float",+ "integer",+ "json",+ "number",+ "string",+ "text",+}+++def gaussdb_field_json_path_parts(field: str, descriptor: Any = None) -> tuple[str, ...]:+ candidate = str(descriptor or "").strip()+ if candidate and candidate.lower() not in _FIELD_DESCRIPTOR_TYPES:+ return tuple(part for part in candidate.split(".") if part)+ return tuple(part for part in str(field).split(".") if part)+++@dataclass(frozen=True)+class ExposedGaussDBTable:+ logical_name: str+ physical_name: str+ allowed_columns: frozenset[str]+ json_fields: dict[str, tuple[str, ...]]+ required_kb_ids: tuple[str, ...]++ @classmethod+ def from_field_map(cls, physical_name: str, kb_ids: list[str] | tuple[str, ...], field_map: dict[str, Any]):+ json_fields = {}+ for field, descriptor in (field_map or {}).items():+ parts = gaussdb_field_json_path_parts(str(field), descriptor)+ if parts:+ json_fields[str(field)] = parts+ return cls(+ logical_name=physical_name,+ physical_name=physical_name,+ allowed_columns=frozenset({"doc_id", "docnm_kwd", "kb_id", "chunk_data"}),+ json_fields=json_fields,+ required_kb_ids=tuple(str(kid) for kid in kb_ids or () if str(kid)),+ )+++@dataclass(frozen=True)+class ValidatedGaussDBSQL:+ sql: str+ columns: list[str] | None = None+ is_aggregation: bool = False+++def jsonb_path_literal(parts: list[str] | tuple[str, ...]) -> str:+ if not isinstance(parts, (list, tuple)) or not parts:+ raise UnsafeGaussDBSQL("empty JSONB path")+ encoded = []+ for part in parts:+ if not isinstance(part, str):+ raise UnsafeGaussDBSQL("invalid JSONB path segment")+ segment = part+ if not segment:+ raise UnsafeGaussDBSQL("empty JSONB path segment")+ if re.fullmatch(r"[A-Za-z0-9_]+", segment):+ encoded.append(segment)+ else:+ escaped = segment.replace("\\", "\\\\").replace('"', '\\"')+ encoded.append(f'"{escaped}"')+ path = "{" + ",".join(encoded) + "}"+ return "'" + path.replace("'", "''") + "'"+++def _parse_jsonb_path_literal(value: str) -> tuple[str, ...]:+ text = str(value or "").strip()+ if len(text) >= 2 and text[0] == "'" and text[-1] == "'":+ text = text[1:-1].replace("''", "'")+ if not (text.startswith("{") and text.endswith("}")):+ raise UnsafeGaussDBSQL("dynamic JSONB path is not allowed")+ body = text[1:-1]+ parts: list[str] = []+ buf: list[str] = []+ quoted = False+ escaped = False+ for char in body:+ if escaped:+ buf.append(char)+ escaped = False+ continue+ if quoted and char == "\\":+ escaped = True+ continue+ if char == '"':+ quoted = not quoted+ continue+ if char == "," and not quoted:+ parts.append("".join(buf))+ buf = []+ continue+ buf.append(char)+ if quoted or escaped:+ raise UnsafeGaussDBSQL("invalid JSONB path literal")+ parts.append("".join(buf))+ if not parts or any(part == "" for part in parts):+ raise UnsafeGaussDBSQL("invalid JSONB path literal")+ return tuple(parts)+++class GaussDBSQLValidator:+ PARSE_DIALECT = "postgres"+ FORBIDDEN_FUNCTIONS = {+ "pg_sleep",+ "sleep",+ "now",+ "current_user",+ "current_date",+ "current_time",+ "current_timestamp",+ "current_database",+ "current_catalog",+ "localtime",+ "localtimestamp",+ "session_user",+ "user",+ "version",+ "current_schema",+ "json_extract",+ "json_extract_string",+ "json_extract_isnull",+ "jsonb_each",+ "jsonb_each_text",+ "jsonb_to_record",+ "jsonb_to_recordset",+ "jsonb_array_elements",+ "jsonb_array_elements_text",+ }++ def __init__(+ self,+ tables: dict[str, ExposedGaussDBTable] | set[str] | list[str] | tuple[str, ...] | None = None,+ kb_ids: list[str] | tuple[str, ...] | None = None,+ default_limit: int = 128,+ readonly_only: bool = False,+ runtime_readonly_guard: bool = False,+ execution_schema: str | None = None,+ ):+ if tables is None:+ self.tables = {}+ elif isinstance(tables, dict):+ self.tables = dict(tables)+ else:+ self.tables = {str(table): ExposedGaussDBTable.from_field_map(str(table), kb_ids or (), {}) for table in tables}+ self.default_limit = int(default_limit or 128)+ self.readonly_only = readonly_only+ self.runtime_readonly_guard = runtime_readonly_guard+ self.execution_schema = str(execution_schema).strip() if execution_schema else None++ @classmethod+ def readonly_guard(cls, default_limit: int = 128, execution_schema: str | None = None):+ return cls(+ default_limit=default_limit,+ readonly_only=True,+ runtime_readonly_guard=True,+ execution_schema=execution_schema,+ )++ def validate_and_patch(self, raw_sql: str) -> ValidatedGaussDBSQL:+ sql = self.normalize_sql(raw_sql)+ ast = self._parse_one(sql)+ self._validate_readonly_ast(ast)+ if self.runtime_readonly_guard:+ self._validate_runtime_readonly_context(ast)+ if not self.readonly_only:+ self._validate_tables(ast)+ self._validate_jsonb_paths(ast)+ self._validate_columns(ast)+ sql = self._enforce_kb_boundary(sql)+ ast = self._parse_one(sql)+ self._validate_tables(ast)+ self._validate_jsonb_paths(ast)+ self._validate_columns(ast)+ sql = self._enforce_limit(sql)+ ast = self._parse_one(sql)+ self._validate_readonly_ast(ast)+ if self.runtime_readonly_guard:+ self._validate_runtime_readonly_context(ast)+ if not self.readonly_only:+ self._validate_tables(ast)+ self._validate_jsonb_paths(ast)+ self._validate_columns(ast)+ if self.execution_schema:+ self._qualify_execution_tables(ast)+ sql = ast.sql(dialect=self.PARSE_DIALECT)+ return ValidatedGaussDBSQL(+ sql=sql,+ columns=self._select_columns(ast),+ is_aggregation=is_gaussdb_aggregate_sql(sql),+ )++ def normalize_sql(self, raw_sql: str) -> str:+ sql = str(raw_sql or "").strip()+ sql = re.sub(r"</think>\s*.*?\s*", "", sql, flags=re.DOTALL)+ sql = re.sub(r"```(?:sql)?\s*", "", sql, flags=re.IGNORECASE)+ sql = re.sub(r"```\s*$", "", sql, flags=re.IGNORECASE)+ sql = sql.strip().strip("`").strip().rstrip(";").strip()+ if not sql:+ raise UnsafeGaussDBSQL("empty SQL")+ if ";" in sql:+ raise UnsafeGaussDBSQL("multiple statements are not allowed")+ return sql++ def _parse_one(self, sql: str):+ exp, expressions = _parse_gaussdb_statements(sql, self.PARSE_DIALECT)+ if len(expressions) != 1:+ raise UnsafeGaussDBSQL("exactly one SQL statement is allowed")+ if not isinstance(expressions[0], exp.Select):+ raise UnsafeGaussDBSQL("only SELECT statements are allowed")+ return expressions[0]++ def _validate_readonly_ast(self, ast) -> None:+ from sqlglot import exp++ forbidden_types = (+ exp.Delete,+ exp.Update,+ exp.Insert,+ exp.Create,+ exp.Drop,+ exp.Command,+ )+ if any(isinstance(node, forbidden_types) for node in ast.walk()):+ raise UnsafeGaussDBSQL("SQL contains a non-read-only expression")+ if any(isinstance(node, exp.Lateral) for node in ast.walk()):+ raise UnsafeGaussDBSQL("LATERAL is not allowed")+ self._validate_limit_clauses(ast)+ self._validate_jsonb_empty_string_comparisons(ast)+ forbidden_system_types = tuple(+ cls+ for cls in (+ getattr(exp, "CurrentUser", None),+ getattr(exp, "CurrentDate", None),+ getattr(exp, "CurrentTime", None),+ getattr(exp, "CurrentTimestamp", None),+ getattr(exp, "CurrentDatabase", None),+ getattr(exp, "CurrentCatalog", None),+ getattr(exp, "CurrentSchema", None),+ )+ if cls+ )+ for node in ast.walk():+ if isinstance(node, exp.Star) and not isinstance(getattr(node, "parent", None), exp.Count):+ raise UnsafeGaussDBSQL("SELECT * is not allowed")+ if isinstance(node, exp.Window):+ raise UnsafeGaussDBSQL("window functions are not allowed")+ if isinstance(node, forbidden_system_types):+ raise UnsafeGaussDBSQL("system functions are not allowed")+ allowed_func_types = tuple(+ cls+ for cls in (+ exp.Count,+ exp.Sum,+ exp.Avg,+ exp.Max,+ exp.Min,+ exp.Cast,+ getattr(exp, "StrToDate", None),+ getattr(exp, "JSONBExtractScalar", None),+ getattr(exp, "JSONBExtract", None),+ )+ if cls+ )+ for node in ast.find_all(exp.Func):+ if isinstance(node, (exp.Binary, exp.Connector, exp.Predicate)) and not isinstance(node, allowed_func_types):+ continue+ name = str(getattr(node, "name", "") or "").lower()+ if not name:+ sql_name = str(node.sql_name() if hasattr(node, "sql_name") else "").lower()+ name = sql_name.removeprefix("exploding_") or node.__class__.__name__.lower()+ if isinstance(node, forbidden_system_types) or name in self.FORBIDDEN_FUNCTIONS:+ raise UnsafeGaussDBSQL(f"function {name} is not allowed")+ if not isinstance(node, allowed_func_types):+ raise UnsafeGaussDBSQL(f"function {name} is not allowed")++ def _validate_limit_clauses(self, ast) -> None:+ from sqlglot import exp++ for node in ast.walk():+ if isinstance(node, exp.Fetch):+ expression = node.args.get("count")+ elif isinstance(node, exp.Limit):+ expression = node.args.get("expression")+ else:+ continue+ self._positive_static_limit_value(expression)++ @staticmethod+ def _positive_static_limit_value(expression) -> int:+ from sqlglot import exp++ if not isinstance(expression, exp.Literal) or expression.is_string or not re.fullmatch(r"[0-9]+", str(expression.this)) or int(expression.this) <= 0:+ raise UnsafeGaussDBSQL("LIMIT must be a positive static integer")+ return int(expression.this)++ def _validate_jsonb_empty_string_comparisons(self, ast) -> None:+ from sqlglot import exp++ jsonb_text_types = (exp.JSONBExtractScalar,)++ def is_empty_string(expression) -> bool:+ while isinstance(expression, (exp.Cast, exp.Paren)):+ expression = expression.this+ return isinstance(expression, exp.Literal) and expression.is_string and expression.this == ""++ def contains_jsonb_text(expression) -> bool:+ return any(isinstance(node, jsonb_text_types) for node in expression.walk())++ comparison_types = tuple(cls for cls in (exp.EQ, exp.NEQ, getattr(exp, "NullSafeEQ", None), getattr(exp, "NullSafeNEQ", None)) if cls)+ for comparison in ast.walk():+ if not isinstance(comparison, comparison_types):+ continue+ left, right = comparison.this, comparison.expression+ if (is_empty_string(left) and contains_jsonb_text(right)) or (is_empty_string(right) and contains_jsonb_text(left)):+ raise UnsafeGaussDBSQL("JSONB text cannot be compared with an empty SQL string")++ def _validate_runtime_readonly_context(self, ast) -> None:+ base_tables = self._base_tables(ast)+ if not base_tables:+ raise UnsafeGaussDBSQL("SQL must read from a DocEngine table")+ for table in base_tables:+ if not re.fullmatch(r"ragflow_[A-Za-z0-9_]{1,56}", table):+ raise UnsafeGaussDBSQL(f"table {table} is not allowed")+ if self._has_complex_boundary(ast.sql(dialect=self.PARSE_DIALECT)):+ raise UnsafeGaussDBSQL("complex SQL must use a simpler single-table kb_id boundary")+ for select in self._selects_with_base_tables(ast):+ if not self._select_has_static_kb_boundary(select):+ raise UnsafeGaussDBSQL("each base table scope must include a static kb_id boundary")+ self._validate_runtime_columns(ast)+ self._validate_jsonb_paths(ast)++ def _validate_runtime_columns(self, ast) -> None:+ from sqlglot import exp++ allowed_columns = {"doc_id", "docnm_kwd", "kb_id", "chunk_data"}+ cte_outputs = self._cte_output_columns(ast)+ select_alias_refs = self._select_alias_reference_columns(ast)+ for column in ast.find_all(exp.Column):+ name = column.name+ if name in allowed_columns:+ continue+ if self._is_cte_output_column(column, cte_outputs):+ continue+ if id(column) in select_alias_refs:+ continue+ raise UnsafeGaussDBSQL(f"column {name} is not allowed")++ def _cte_names(self, ast) -> set[str]:+ from sqlglot import exp++ return {cte.alias_or_name for cte in ast.find_all(exp.CTE) if cte.alias_or_name}++ def _base_tables(self, ast) -> list[str]:+ from sqlglot import exp++ cte_names = self._cte_names(ast)+ tables = []+ for table in ast.find_all(exp.Table):+ if table.db or table.catalog:+ raise UnsafeGaussDBSQL("cross-schema SQL is not allowed")+ name = table.name+ if name and name not in cte_names:+ tables.append(name)+ return tables++ def _qualify_execution_tables(self, ast) -> None:+ from sqlglot import exp++ cte_names = self._cte_names(ast)+ schema = exp.to_identifier(self.execution_schema, quoted=True)+ for table in ast.find_all(exp.Table):+ if table.db or table.catalog or table.name in cte_names:+ continue+ table.set("db", schema.copy())++ def _validate_tables(self, ast) -> None:+ allowed = {table.physical_name for table in self.tables.values()} | set(self.tables)+ for table in self._base_tables(ast):+ if table not in allowed:+ raise UnsafeGaussDBSQL(f"table {table} is not allowed")++ def _validate_columns(self, ast) -> None:+ from sqlglot import exp++ allowed_columns = set()+ for table in self.tables.values():+ allowed_columns.update(table.allowed_columns)+ cte_outputs = self._cte_output_columns(ast)+ select_alias_refs = self._select_alias_reference_columns(ast)+ for column in ast.find_all(exp.Column):+ name = column.name+ if name in allowed_columns:+ continue+ if self._is_cte_output_column(column, cte_outputs):+ continue+ if id(column) in select_alias_refs:+ continue+ raise UnsafeGaussDBSQL(f"column {name} is not allowed")++ def _select_alias_reference_columns(self, ast) -> set[int]:+ from sqlglot import exp++ refs = set()+ for select in ast.find_all(exp.Select):+ aliases = {str(getattr(expression, "alias_or_name", "") or "") for expression in getattr(select, "expressions", []) or [] if getattr(expression, "alias", None)}+ aliases.discard("")+ if not aliases:+ continue+ for clause_name in ("group", "order"):+ clause = select.args.get(clause_name)+ if not clause:+ continue+ for column in clause.find_all(exp.Column):+ if not column.table and column.name in aliases:+ refs.add(id(column))+ return refs++ def _cte_output_columns(self, ast) -> dict[str, set[str]]:+ from sqlglot import exp++ outputs: dict[str, set[str]] = {}+ for cte in ast.find_all(exp.CTE):+ name = cte.alias_or_name+ query = cte.this+ if not name or not isinstance(query, exp.Select):+ continue+ columns = set()+ for expression in query.expressions or []:+ alias = getattr(expression, "alias_or_name", None)+ if alias:+ columns.add(str(alias))+ outputs[name] = columns+ return outputs++ def _is_cte_output_column(self, column, cte_outputs: dict[str, set[str]]) -> bool:+ from sqlglot import exp++ name = column.name+ table = column.table+ if table:+ return table in cte_outputs and name in cte_outputs[table]+ select = column.parent+ while select is not None and not isinstance(select, exp.Select):+ select = getattr(select, "parent", None)+ if select is None:+ return False+ source_tables = self._direct_source_tables(select)+ if not source_tables or not all(source in cte_outputs for source in source_tables):+ return False+ return any(name in cte_outputs[source] for source in source_tables)++ def _validate_jsonb_paths(self, ast) -> None:+ from sqlglot import exp++ allowed_paths = {path for table in self.tables.values() for path in table.json_fields.values()}+ json_classes = (exp.JSONExtractScalar, exp.JSONExtract)+ jsonb_classes = (exp.JSONBExtractScalar, exp.JSONBExtract)+ allowed_chunk_data_columns = set()+ for node in ast.walk():+ if isinstance(node, json_classes):+ raise UnsafeGaussDBSQL("only GaussDB #> / #>> JSONB operators are allowed")+ if not isinstance(node, jsonb_classes):+ continue+ source = node.this+ if not isinstance(source, exp.Column) or source.name != "chunk_data":+ raise UnsafeGaussDBSQL("only chunk_data JSONB paths are allowed")+ allowed_chunk_data_columns.add(id(source))+ expression = node.expression+ if not isinstance(expression, exp.Literal) or not expression.is_string:+ raise UnsafeGaussDBSQL("dynamic JSONB path is not allowed")+ path = _parse_jsonb_path_literal(expression.this)+ if not self.readonly_only and not allowed_paths:+ raise UnsafeGaussDBSQL(f"JSONB path {path} is not exposed")+ if allowed_paths and path not in allowed_paths:+ raise UnsafeGaussDBSQL(f"JSONB path {path} is not exposed")+ for column in ast.find_all(exp.Column):+ if column.name == "chunk_data" and id(column) not in allowed_chunk_data_columns:+ raise UnsafeGaussDBSQL("chunk_data may only be accessed through #> / #>>")++ def _required_kb_ids(self) -> tuple[str, ...]:+ kb_ids = []+ for table in self.tables.values():+ kb_ids.extend(table.required_kb_ids)+ return tuple(dict.fromkeys(kb_ids))++ def _enforce_kb_boundary(self, sql: str) -> str:+ kb_ids = self._required_kb_ids()+ if not kb_ids:+ raise UnsafeGaussDBSQL("kb_id boundary is required")+ ast = self._parse_one(sql)+ if not self._base_tables(ast):+ raise UnsafeGaussDBSQL("SQL must read from an exposed DocEngine table")+ if self._has_complex_boundary(sql):+ raise UnsafeGaussDBSQL("complex SQL must use a simpler single-table kb_id boundary")+ missing = []+ for select in self._selects_with_base_tables(ast):+ if self._select_has_allowed_kb_boundary(select, kb_ids):+ continue+ missing.append(select)+ if not missing:+ return sql+ if len(missing) == 1 and missing[0] is ast and len(self._selects_with_base_tables(ast)) == 1 and not ast.args.get("with_") and len(set(self._direct_base_tables(ast))) == 1:+ if self._where_mentions_kb_id(ast):+ raise UnsafeGaussDBSQL("kb_id boundary must be a positive top-level predicate")+ return self._insert_condition(sql, self._kb_condition(kb_ids))+ if len(set(self._base_tables(ast))) != 1:+ raise UnsafeGaussDBSQL("SQL must reference exactly one base table to inject kb_id")+ raise UnsafeGaussDBSQL("each base table scope must include a kb_id boundary")++ def _has_complex_boundary(self, sql: str) -> bool:+ from sqlglot import exp++ ast = self._parse_one(sql)+ if any(isinstance(node, exp.Or) for node in ast.walk()):+ return True+ if re.search(r"\b(join|union|intersect|except)\b", sql, flags=re.IGNORECASE):+ return True+ return False++ def _selects_with_base_tables(self, ast) -> list:+ from sqlglot import exp++ return [select for select in ast.find_all(exp.Select) if self._direct_base_tables(select)]++ def _direct_base_tables(self, select) -> list[str]:+ from sqlglot import exp++ cte_names = self._cte_names(select)+ tables = []+ from_expr = select.args.get("from_")+ if from_expr and isinstance(from_expr.this, exp.Table):+ if from_expr.this.name not in cte_names:+ tables.append(from_expr.this.name)+ for join in select.args.get("joins") or []:+ target = join.this+ if isinstance(target, exp.Table) and target.name not in cte_names:+ tables.append(target.name)+ return tables++ def _direct_source_tables(self, select) -> list[str]:+ from sqlglot import exp++ tables = []+ from_expr = select.args.get("from_")+ if from_expr and isinstance(from_expr.this, exp.Table):+ tables.append(from_expr.this.name)+ for join in select.args.get("joins") or []:+ target = join.this+ if isinstance(target, exp.Table):+ tables.append(target.name)+ return tables++ def _select_has_allowed_kb_boundary(self, select, allowed_kb_ids: tuple[str, ...]) -> bool:+ from sqlglot import exp++ where = select.args.get("where")+ if not where:+ return False+ allowed = set(allowed_kb_ids)+ for node in self._top_level_and_predicates(where.this):+ if isinstance(node, exp.EQ):+ values = self._kb_values_from_equality(node)+ if values is None:+ continue+ if not values or not set(values).issubset(allowed):+ raise UnsafeGaussDBSQL("SQL crosses the allowed kb_id boundary")+ return True+ elif isinstance(node, exp.In):+ values = self._kb_values_from_in(node)+ if values is None:+ continue+ if not values or not set(values).issubset(allowed):+ raise UnsafeGaussDBSQL("SQL crosses the allowed kb_id boundary")+ return True+ return False++ def _select_has_static_kb_boundary(self, select) -> bool:+ from sqlglot import exp++ where = select.args.get("where")+ if not where:+ return False+ for node in self._top_level_and_predicates(where.this):+ if isinstance(node, exp.EQ):+ values = self._kb_values_from_equality(node)+ elif isinstance(node, exp.In):+ values = self._kb_values_from_in(node)+ else:+ values = None+ if values is None:+ continue+ if not values:+ raise UnsafeGaussDBSQL("kb_id boundary is empty")+ self._validate_literal_values(values, "kb_id")+ return True+ return False++ def _top_level_and_predicates(self, node) -> list:+ from sqlglot import exp++ if isinstance(node, exp.And):+ return [*self._top_level_and_predicates(node.this), *self._top_level_and_predicates(node.expression)]+ return [node]++ def _where_mentions_kb_id(self, select) -> bool:+ from sqlglot import exp++ where = select.args.get("where")+ if not where:+ return False+ return any(self._is_kb_column(column) for column in where.find_all(exp.Column))++ def _kb_values_from_equality(self, node) -> list[str] | None:+ from sqlglot import exp++ left, right = node.this, node.expression+ if self._is_kb_column(left) and isinstance(right, exp.Literal) and right.is_string:+ return [str(right.this)]+ if self._is_kb_column(right) and isinstance(left, exp.Literal) and left.is_string:+ return [str(left.this)]+ return None++ def _kb_values_from_in(self, node) -> list[str] | None:+ from sqlglot import exp++ if not self._is_kb_column(node.this):+ return None+ values = []+ for item in node.expressions or []:+ if not isinstance(item, exp.Literal) or not item.is_string:+ raise UnsafeGaussDBSQL("kb_id IN must use static string literals")+ values.append(str(item.this))+ return values++ def _is_kb_column(self, node) -> bool:+ from sqlglot import exp++ return isinstance(node, exp.Column) and node.name.lower() == "kb_id"++ def _kb_condition(self, kb_ids: tuple[str, ...]) -> str:+ self._validate_literal_values(kb_ids, "kb_id")+ if len(kb_ids) == 1:+ return f"kb_id = '{kb_ids[0]}'"+ return "kb_id IN (" + ", ".join(f"'{kid}'" for kid in kb_ids) + ")"++ def _validate_literal_values(self, values: tuple[str, ...] | list[str], label: str) -> None:+ for value in values:+ if not re.fullmatch(r"[A-Za-z0-9_.:-]{1,256}", str(value)):+ raise UnsafeGaussDBSQL(f"unsafe {label} literal")++ def _insert_condition(self, sql: str, condition: str) -> str:+ from sqlglot import exp++ ast = self._parse_one(sql)+ condition_ast = self._parse_one(f"SELECT 1 WHERE {condition}").args["where"].this+ where = ast.args.get("where")+ if where:+ ast.set("where", exp.Where(this=exp.and_(where.this, condition_ast)))+ else:+ ast.set("where", exp.Where(this=condition_ast))+ return ast.sql(dialect=self.PARSE_DIALECT)++ def _enforce_limit(self, sql: str) -> str:+ from sqlglot import exp++ ast = self._parse_one(sql)+ limit = ast.args.get("limit")+ if not limit:+ if self.default_limit <= 0:+ return sql+ return ast.limit(self.default_limit).sql(dialect=self.PARSE_DIALECT)+ expression_key = "count" if isinstance(limit, exp.Fetch) else "expression"+ current = self._positive_static_limit_value(limit.args.get(expression_key))+ if self.default_limit <= 0 or current <= self.default_limit:+ return sql+ limit.set(expression_key, exp.Literal.number(self.default_limit))+ return ast.sql(dialect=self.PARSE_DIALECT)++ def _select_columns(self, ast) -> list[str]:+ columns = []+ for expression in getattr(ast, "expressions", []) or []:+ alias = getattr(expression, "alias_or_name", None)+ if alias:+ columns.append(str(alias))+ return columns+++class GaussDBDDLBuilder:+ MAX_IDENTIFIER_LENGTH = 63+ REGULAR_INDEX_COLUMNS = (+ "doc_id",+ "available_int",+ "knowledge_graph_kwd",+ "entity_type_kwd",+ "removed_kwd",+ )+ FTS_COLUMNS = (+ "title_tks",+ "title_sm_tks",+ "important_tks",+ "question_tks",+ "content_ltks",+ "content_sm_ltks",+ )++ def __init__(self, schema: str):+ self.schema = self.validate_identifier(schema)++ def validate_identifier(self, name: str) -> str:+ if not re.fullmatch(r"(?:[A-Za-z_]|[^\x00-\x7F])(?:[A-Za-z0-9_#$]|[^\x00-\x7F]){0,62}", name or ""):+ raise InvalidGaussDBObjectName(name)+ return name++ def quote_identifier(self, name: str) -> str:+ escaped = self.validate_identifier(name).replace('"', '""')+ return f'"{escaped}"'++ def qualified_name(self, table: str) -> str:+ return f"{self.quote_identifier(self.schema)}.{self.quote_identifier(table)}"++ def index_name(self, table: str, suffix: str) -> str:+ name = f"idx_gdb_{self.validate_identifier(table)}_{self.validate_identifier(suffix)}"+ if len(name) <= self.MAX_IDENTIFIER_LENGTH:+ return name+ digest = hashlib.sha1(name.encode("utf-8")).hexdigest()[:10]+ prefix_len = self.MAX_IDENTIFIER_LENGTH - len(digest) - 1+ return f"{name[:prefix_len]}_{digest}"++ def build_chunk_table_ddl(self, table: str) -> str:+ name = self.qualified_name(table)+ return f"""CREATE TABLE IF NOT EXISTS {name} (+ id VARCHAR(256) NOT NULL,+ kb_id VARCHAR(256) NOT NULL,+ doc_id VARCHAR(256),+ docnm_kwd VARCHAR(256),+ doc_type_kwd VARCHAR(256),+ title_tks VARCHAR(256),+ title_sm_tks VARCHAR(256),+ content_with_weight TEXT,+ content_ltks TEXT,+ content_sm_ltks TEXT,+ important_kwd JSONB,+ important_tks TEXT,+ question_kwd JSONB,+ question_tks TEXT,+ tag_kwd JSONB,+ tag_feas JSONB,+ available_int INTEGER DEFAULT 1 NOT NULL,+ pagerank_fea INTEGER,+ create_time VARCHAR(19),+ create_timestamp_flt DOUBLE PRECISION,+ img_id VARCHAR(128),+ position_int JSONB,+ page_num_int JSONB,+ top_int JSONB,+ metadata JSONB,+ chunk_data JSONB,+ extra JSONB,+ _order_id INTEGER,+ group_id VARCHAR(256),+ mom_id VARCHAR(256),+ knowledge_graph_kwd VARCHAR(256),+ source_id JSONB,+ entity_kwd VARCHAR(256),+ entity_type_kwd VARCHAR(256),+ from_entity_kwd VARCHAR(256),+ to_entity_kwd VARCHAR(256),+ weight_int INTEGER,+ weight_flt DOUBLE PRECISION,+ entities_kwd JSONB,+ rank_flt DOUBLE PRECISION,+ n_hop_with_weight TEXT,+ removed_kwd VARCHAR(256) DEFAULT 'N',+ raptor_kwd VARCHAR(256),+ raptor_layer_int INTEGER,+ PRIMARY KEY (kb_id, id)+) WITH (storage_type=USTORE)"""++ def build_doc_meta_table_ddls(self, meta_table: str) -> list[str]:+ name = self.qualified_name(meta_table)+ idx = self.quote_identifier(self.index_name(meta_table, "kb_id"))+ return [+ f"""CREATE TABLE IF NOT EXISTS {name} (+ id VARCHAR(256) NOT NULL,+ kb_id VARCHAR(256) NOT NULL,+ meta_fields JSONB,+ PRIMARY KEY (id)+) WITH (storage_type=USTORE)""",+ f"CREATE INDEX IF NOT EXISTS {idx} ON {name} (kb_id)",+ ]++ def build_regular_index_ddls(self, table: str) -> list[str]:+ name = self.qualified_name(table)+ return [f"CREATE INDEX IF NOT EXISTS {self.quote_identifier(self.index_name(table, column))} ON {name} ({column})" for column in self.REGULAR_INDEX_COLUMNS]++ def build_fulltext_ugin_ddl(self, table: str) -> str:+ name = self.qualified_name(table)+ idx = self.quote_identifier(self.index_name(table, "fts_all"))+ expression = " || ' ' || ".join(f"coalesce({column}, ' ')" for column in self.FTS_COLUMNS)+ return f"""CREATE INDEX IF NOT EXISTS {idx}+ ON {name}+ USING ugin(to_tsvector('simple', {expression}))"""++ def build_ngram_fulltext_ugin_ddl(self, table: str) -> str:+ name = self.qualified_name(table)+ idx = self.quote_identifier(self.index_name(table, "fts_all_ngram"))+ expression = " || ' ' || ".join(f"coalesce({column}, ' ')" for column in self.FTS_COLUMNS)+ return f"""CREATE INDEX IF NOT EXISTS {idx}+ ON {name}+ USING ugin(to_tsvector('ngram', {expression}))"""++ def build_vector_column_ddls(self, table: str, dim: int) -> list[str]:+ dim = self.validate_vector_dim(dim)+ name = self.qualified_name(table)+ vector_col = self.vector_column_name(dim)+ valid_col = self.vector_valid_column_name(dim)+ return [+ f"ALTER TABLE {name} ADD COLUMN IF NOT EXISTS {vector_col} floatvector({dim}) DEFAULT (array_fill(0, ARRAY[{dim}])::text::floatvector({dim}))",+ f"ALTER TABLE {name} ADD COLUMN IF NOT EXISTS {valid_col} BOOLEAN DEFAULT FALSE NOT NULL",+ ]++ def build_diskann_index_ddl(self, table: str, dim: int) -> str:+ dim = self.validate_vector_dim(dim)+ name = self.qualified_name(table)+ vector_col = self.vector_column_name(dim)+ idx = self.quote_identifier(self.index_name(table, f"{vector_col}_diskann"))+ options = "subgraph_count=1"+ if dim > 1024:+ options += ", enable_vector_copy=false"+ return f"CREATE INDEX IF NOT EXISTS {idx} ON {name} USING gsdiskann ({vector_col} COSINE) WITH ({options})"++ def build_advisory_lock_sql(self, lock_name: str) -> tuple[str, list[str]]:+ return ("SELECT pg_advisory_xact_lock(hashtext(%s))", [str(lock_name)])++ def validate_vector_dim(self, dim: int) -> int:+ if isinstance(dim, bool) or not isinstance(dim, Integral):+ raise ValueError("vector dimension must be an integer")+ value = int(dim)+ if value <= 0:+ raise ValueError("vector dimension must be positive")+ if value > 4096:+ raise ValueError("GaussDB floatvector dimensions cannot exceed 4096")+ return value++ def vector_column_name(self, dim: int) -> str:+ return f"q_{self.validate_vector_dim(dim)}_vec"++ def vector_valid_column_name(self, dim: int) -> str:+ return f"q_{self.validate_vector_dim(dim)}_vec_valid"+++class GaussDBSearchBuilder:+ CHUNK_COLUMNS = {+ "id",+ "kb_id",+ "doc_id",+ "docnm_kwd",+ "doc_type_kwd",+ "title_tks",+ "title_sm_tks",+ "content_with_weight",+ "content_ltks",+ "content_sm_ltks",+ "important_kwd",+ "important_tks",+ "question_kwd",+ "question_tks",+ "tag_kwd",+ "tag_feas",+ "available_int",+ "pagerank_fea",+ "create_time",+ "create_timestamp_flt",+ "img_id",+ "position_int",+ "page_num_int",+ "top_int",+ "metadata",+ "chunk_data",+ "extra",+ "_order_id",+ "chunk_order_int",+ "group_id",+ "mom_id",+ "knowledge_graph_kwd",+ "source_id",+ "entity_kwd",+ "entity_type_kwd",+ "from_entity_kwd",+ "to_entity_kwd",+ "weight_int",+ "weight_flt",+ "entities_kwd",+ "rank_flt",+ "n_hop_with_weight",+ "removed_kwd",+ "raptor_kwd",+ "raptor_layer_int",+ "compile_kwd",+ "row_id()",+ }+ JSONB_MULTI_VALUE_COLUMNS = {"important_kwd", "question_kwd", "tag_kwd", "source_id", "entities_kwd"}+ JSONB_ARRAY_AGG_COLUMNS = JSONB_MULTI_VALUE_COLUMNS | {"entities_kwd"}+ JSONB_EXTRA_SCALAR_COLUMNS = {"compile_kwd"}+ COLUMN_ALIASES = {"chunk_order_int": "_order_id"}+ FTS_WEIGHTS = {+ "title_tks": 10.0,+ "title_sm_tks": 5.0,+ "important_tks": 20.0,+ "question_tks": 20.0,+ "content_ltks": 2.0,+ "content_sm_ltks": 1.0,+ }+ VECTOR_COLUMN_RE = re.compile(r"^q_(?P<dim>\d+)_vec$")+ VECTOR_VALID_COLUMN_RE = re.compile(r"^q_(?P<dim>\d+)_vec_valid$")+ CJK_RUN_RE = re.compile(r"[\u3400-\u4dbf\u4e00-\u9fff\uf900-\ufaff]+")++ def __init__(self, schema: str):+ self.ddl = GaussDBDDLBuilder(schema=schema)++ def build_search_sql(+ self,+ table: str,+ select_fields: list[str],+ condition: dict,+ keywords: list[str] | None,+ vector: list[float] | None,+ vector_dim: int | None,+ vector_weight: float,+ offset: int,+ limit: int,+ similarity_threshold: float | None = None,+ topn: int | None = None,+ highlight_fields: list[str] | None = None,+ order_by: OrderByExpr | None = None,+ pagerank_weight: float = 0.0,+ ) -> tuple[str, list[Any]]:+ query_keywords = [str(keyword).strip() for keyword in keywords or [] if str(keyword).strip()]+ has_searchable_text = any(self.split_text_query_terms(query_keywords))+ query_vector = self._vector_param(vector, vector_dim) if vector is not None else None+ effective_limit = max(int(limit or 0), 0)+ effective_offset = max(int(offset or 0), 0)+ page_limit = effective_limit if effective_limit > 0 else max(int(topn or 0), 10000)+ candidate_limit = self._candidate_limit(page_limit, effective_offset, topn)++ if has_searchable_text and query_vector is not None:+ return self._build_hybrid_search_sql(+ table=table,+ select_fields=select_fields,+ condition=condition,+ keywords=query_keywords,+ vector=query_vector,+ vector_dim=vector_dim,+ vector_weight=vector_weight,+ similarity_threshold=similarity_threshold,+ candidate_limit=candidate_limit,+ offset=effective_offset,+ limit=page_limit,+ highlight_fields=highlight_fields,+ pagerank_weight=pagerank_weight,+ )+ if query_vector is not None:+ return self._build_vector_search_sql(+ table=table,+ select_fields=select_fields,+ condition=condition,+ vector=query_vector,+ vector_dim=vector_dim,+ similarity_threshold=similarity_threshold,+ candidate_limit=candidate_limit,+ offset=effective_offset,+ limit=page_limit,+ pagerank_weight=pagerank_weight,+ )+ if query_keywords:+ return self._build_fulltext_search_sql(+ table=table,+ select_fields=select_fields,+ condition=condition,+ keywords=query_keywords,+ offset=effective_offset,+ limit=page_limit,+ highlight_fields=highlight_fields,+ pagerank_weight=pagerank_weight,+ )+ return self._build_filter_search_sql(+ table=table,+ select_fields=select_fields,+ condition=condition,+ offset=effective_offset,+ limit=page_limit,+ order_by=order_by,+ pagerank_weight=pagerank_weight,+ )++ def build_condition_where(self, condition: dict | None) -> tuple[str, list[Any]]:+ fragments: list[str] = []+ params: list[Any] = []+ for key, value in (condition or {}).items():+ if key == "exists":+ column = self.validate_column(value)+ expression = self._column_expr(column)+ if self._is_jsonb_dynamic_column(column):+ fragments.append(f"({expression} IS NOT NULL AND {expression} <> 'null'::jsonb)")+ else:+ fragments.append(f"{expression} IS NOT NULL")+ continue+ if key == "must_not" and isinstance(value, dict) and "exists" in value:+ column = self.validate_column(value["exists"])+ expression = self._column_expr(column)+ if self._is_jsonb_dynamic_column(column):+ fragments.append(f"({expression} IS NULL OR {expression} = 'null'::jsonb)")+ else:+ fragments.append(f"{expression} IS NULL")+ continue++ column = self.validate_column(key)+ expression = self._column_expr(column)+ if self._is_jsonb_dynamic_column(column):+ values = self._list_values(value)+ predicates = []+ for item in values:+ predicates.append(f"({expression} = %s::jsonb OR {expression} @> %s::jsonb)")+ params.extend(+ [+ json.dumps(item, ensure_ascii=False),+ json.dumps([item], ensure_ascii=False),+ ]+ )+ fragments.append("(" + " OR ".join(predicates) + ")")+ continue+ storage_column = self._storage_column(column)+ if storage_column in self.JSONB_MULTI_VALUE_COLUMNS:+ values = self._list_values(value)+ fragments.append("(" + " OR ".join([f"{storage_column} @> %s::jsonb"] * len(values)) + ")")+ params.extend(json.dumps([item], ensure_ascii=False) for item in values)+ continue+ if isinstance(value, (list, tuple, set)):+ values = self._list_values(value)+ fragments.append(f"{expression} IN ({', '.join(['%s'] * len(values))})")+ params.extend(values)+ continue+ if value is None:+ fragments.append(f"{expression} IS NULL")+ continue+ fragments.append(f"{expression} = %s")+ params.append(value)+ return " AND ".join(fragments), params++ def build_text_score_expr(self, keywords: list[str]) -> tuple[str, list[Any]]:+ score_exprs: list[str] = []+ params: list[Any] = []+ for config, query_text in self._text_queries(keywords):+ weighted_score = " + ".join(+ f"{weight} * COALESCE(ts_rank(to_tsvector('{config}', coalesce({column}, ' ')), plainto_tsquery('{config}', %s)), 0)" for column, weight in self.FTS_WEIGHTS.items()+ )+ score_exprs.append(f"({weighted_score})")+ params.extend([query_text] * len(self.FTS_WEIGHTS))+ if not score_exprs:+ return "0.0", []+ if len(score_exprs) == 1:+ return score_exprs[0], params+ return f"(({' + '.join(score_exprs)}) / {len(score_exprs)}.0)", params++ def build_highlight_expr(self, field_name: str, keywords: list[str]) -> tuple[str, list[Any]]:+ field = self.validate_column(field_name)+ expression = self._column_expr(field, text=self._is_dynamic_column(field))+ _simple_terms, ngram_terms = self.split_text_query_terms(keywords)+ if ngram_terms:+ return f"COALESCE({expression}, ' ') AS _highlight_source", []+ return (+ f"ts_headline('simple', COALESCE({expression}, ' '), plainto_tsquery('simple', %s), 'StartSel=<em>, StopSel=</em>') AS _highlight",+ [self._text_query_param(keywords)],+ )++ def build_aggregation_sql(+ self,+ table: str,+ field_name: str,+ condition: dict | None,+ limit: int = 1000,+ ) -> tuple[str, list[Any]]:+ table_name = self.ddl.qualified_name(table)+ field = self.validate_column(field_name)+ field_expression = self._column_expr(field)+ where_sql, where_params = self.build_condition_where(condition)+ where_clause = where_sql or "TRUE"+ if field in self.JSONB_ARRAY_AGG_COLUMNS:+ sql = (+ "SELECT value, COUNT(1) AS count "+ "FROM ("+ f"SELECT jsonb_array_elements_text(COALESCE({field_expression}, '[]'::jsonb)) AS value "+ f"FROM {table_name} WHERE {where_clause}"+ ") AS expanded "+ "GROUP BY value ORDER BY count DESC, value ASC LIMIT %s"+ )+ else:+ sql = (+ f"SELECT {field_expression} AS value, COUNT(1) AS count "+ f"FROM {table_name} "+ f"WHERE {where_clause} AND {field_expression} IS NOT NULL "+ "GROUP BY value ORDER BY count DESC, value ASC LIMIT %s"+ )+ return sql, [*where_params, int(limit)]++ def build_position_order_sql(self) -> str:+ return "COALESCE((page_num_int #>> '{0}')::int, 100000000) ASC, COALESCE((position_int #>> '{0,3}')::int, 100000000) ASC, COALESCE((top_int #>> '{0}')::int, 100000000) ASC"++ def build_fts_vector_expr(self, config: str = "simple") -> str:+ expression = " || ' ' || ".join(f"coalesce({column}, ' ')" for column in self.FTS_WEIGHTS)+ return f"to_tsvector('{config}', {expression})"++ def validate_column(self, column: str) -> str:+ if column == "row_id()":+ return column+ column = str(column or "")+ if column in self.CHUNK_COLUMNS or self.VECTOR_COLUMN_RE.fullmatch(column) or self.VECTOR_VALID_COLUMN_RE.fullmatch(column):+ return column+ return validate_extra_field(column)++ def normalize_select_fields(self, fields: list[str] | None) -> list[str]:+ if not fields or "*" in fields:+ return ["id", "kb_id"]+ normalized: list[str] = ["id", "kb_id"]+ for field in fields:+ if field == "_score":+ continue+ column = self.validate_column(field)+ if column not in normalized:+ normalized.append(column)+ match = self.VECTOR_COLUMN_RE.fullmatch(column)+ if match:+ valid_column = self.ddl.vector_valid_column_name(int(match.group("dim")))+ if valid_column not in normalized:+ normalized.append(valid_column)+ return normalized++ def _build_filter_search_sql(+ self,+ table: str,+ select_fields: list[str],+ condition: dict,+ offset: int,+ limit: int,+ order_by: OrderByExpr | None,+ pagerank_weight: float,+ ) -> tuple[str, list[Any]]:+ table_name = self.ddl.qualified_name(table)+ columns = self.normalize_select_fields(select_fields)+ where_sql, where_params = self.build_condition_where(condition)+ order_sql = self._build_order_by(order_by) or "kb_id ASC, id ASC"+ score_expr, score_params = self._score_with_pagerank("0.0", pagerank_weight)+ sql = f"SELECT {', '.join(self._select_exprs(columns))}, {score_expr} AS _score, COUNT(*) OVER() AS __total FROM {table_name}"+ if where_sql:+ sql += f" WHERE {where_sql}"+ sql += f" ORDER BY {order_sql} LIMIT %s OFFSET %s"+ return sql, [*score_params, *where_params, limit, offset]++ def _build_fulltext_search_sql(+ self,+ table: str,+ select_fields: list[str],+ condition: dict,+ keywords: list[str],+ offset: int,+ limit: int,+ highlight_fields: list[str] | None,+ pagerank_weight: float,+ ) -> tuple[str, list[Any]]:+ table_name = self.ddl.qualified_name(table)+ columns = self.normalize_select_fields(select_fields)+ score_expr, score_params = self.build_text_score_expr(keywords)+ score_expr, pagerank_params = self._score_with_pagerank(score_expr, pagerank_weight)+ match_expr, match_params = self._build_text_match_expr(keywords)+ where_sql, where_params = self.build_condition_where(condition)+ where_parts = [part for part in (where_sql, match_expr) if part]+ select_exprs = [*self._select_exprs(columns), f"{score_expr} AS _score", "COUNT(*) OVER() AS __total"]+ highlight_params: list[Any] = []+ if highlight_fields:+ highlight_expr, highlight_params = self.build_highlight_expr(highlight_fields[0], keywords)+ select_exprs.append(highlight_expr)+ sql = f"SELECT {', '.join(select_exprs)} FROM {table_name}"+ if where_parts:+ sql += f" WHERE {' AND '.join(where_parts)}"+ sql += " ORDER BY _score DESC, kb_id ASC, id ASC LIMIT %s OFFSET %s"+ return sql, [*score_params, *pagerank_params, *highlight_params, *where_params, *match_params, limit, offset]++ def _build_vector_search_sql(+ self,+ table: str,+ select_fields: list[str],+ condition: dict,+ vector: str,+ vector_dim: int,+ similarity_threshold: float | None,+ candidate_limit: int,+ offset: int,+ limit: int,+ pagerank_weight: float,+ ) -> tuple[str, list[Any]]:+ table_name = self.ddl.qualified_name(table)+ columns = self.normalize_select_fields(select_fields)+ dim = self.ddl.validate_vector_dim(vector_dim)+ vector_col = self.ddl.vector_column_name(dim)+ valid_col = self.ddl.vector_valid_column_name(dim)+ where_sql, where_params = self.build_condition_where(condition)+ where_parts = [part for part in (where_sql, f"{valid_col} = TRUE") if part]+ threshold = 0.0 if similarity_threshold is None else float(similarity_threshold)+ score_expr, score_params = self._score_with_pagerank(f"1 - ({vector_col} <+> %s::floatvector({dim}))", pagerank_weight)+ sql = (+ "WITH vec AS ("+ f" SELECT {', '.join(self._select_exprs(columns))}, "+ f"{vector_col} <+> %s::floatvector({dim}) AS distance, "+ f"{score_expr} AS _score "+ f"FROM {table_name} "+ f"WHERE {' AND '.join(where_parts)} "+ f"ORDER BY {vector_col} <+> %s::floatvector({dim}) ASC "+ "LIMIT %s"+ ") "+ "SELECT vec.*, COUNT(*) OVER() AS __total FROM vec "+ "WHERE _score >= %s "+ "ORDER BY distance ASC, kb_id ASC, id ASC LIMIT %s OFFSET %s"+ )+ return sql, [vector, vector, *score_params, *where_params, vector, candidate_limit, threshold, limit, offset]++ def _build_hybrid_search_sql(+ self,+ table: str,+ select_fields: list[str],+ condition: dict,+ keywords: list[str],+ vector: str,+ vector_dim: int,+ vector_weight: float,+ similarity_threshold: float | None,+ candidate_limit: int,+ offset: int,+ limit: int,+ highlight_fields: list[str] | None,+ pagerank_weight: float,+ ) -> tuple[str, list[Any]]:+ table_name = self.ddl.qualified_name(table)+ columns = self.normalize_select_fields(select_fields)+ joined_columns = ", ".join(self._select_exprs(columns, prefix="c"))+ dim = self.ddl.validate_vector_dim(vector_dim)+ vector_col = self.ddl.vector_column_name(dim)+ valid_col = self.ddl.vector_valid_column_name(dim)+ text_score_expr, text_score_params = self.build_text_score_expr(keywords)+ match_expr, match_params = self._build_text_match_expr(keywords)+ where_sql, where_params = self.build_condition_where(condition)+ base_where = where_sql or "TRUE"+ fts_where = " AND ".join([base_where, match_expr])+ vector_where = " AND ".join([base_where, f"{valid_col} = TRUE"])+ threshold = 0.0 if similarity_threshold is None else float(similarity_threshold)+ select_exprs = [joined_columns, "merged.score AS _score", "COUNT(*) OVER() AS __total"]+ highlight_params: list[Any] = []+ if highlight_fields:+ highlight_expr, highlight_params = self.build_highlight_expr(highlight_fields[0], keywords)+ select_exprs.append(highlight_expr)+ final_score_expr, final_score_params = self._score_with_pagerank("merged.score", pagerank_weight, table_alias="c")+ select_exprs[1] = f"{final_score_expr} AS _score"+ threshold_expr, threshold_score_params = self._score_with_pagerank("merged.score", pagerank_weight, table_alias="c")+ sql = (+ "WITH fts_raw AS ("+ f" SELECT kb_id, id, {text_score_expr} AS raw_fts_score "+ f"FROM {table_name} WHERE {fts_where} "+ "ORDER BY raw_fts_score DESC, kb_id ASC, id ASC LIMIT %s"+ "), fts AS ("+ " SELECT kb_id, id, COALESCE(raw_fts_score / NULLIF(MAX(raw_fts_score) OVER (), 0), 0) AS fts_score "+ "FROM fts_raw"+ "), vec AS ("+ f" SELECT kb_id, id, 1 - ({vector_col} <+> %s::floatvector({dim})) AS vector_score "+ f"FROM {table_name} WHERE {vector_where} "+ f"ORDER BY {vector_col} <+> %s::floatvector({dim}) ASC LIMIT %s"+ "), merged AS ("+ " SELECT COALESCE(fts.kb_id, vec.kb_id) AS kb_id, "+ "COALESCE(fts.id, vec.id) AS id, "+ "(1 - %s) * COALESCE(fts.fts_score, 0) + %s * COALESCE(vec.vector_score, 0) AS score "+ "FROM fts FULL OUTER JOIN vec ON fts.kb_id = vec.kb_id AND fts.id = vec.id"+ ") "+ f"SELECT {', '.join(select_exprs)} "+ f"FROM merged JOIN {table_name} c ON c.kb_id = merged.kb_id AND c.id = merged.id "+ f"WHERE {threshold_expr} >= %s "+ "ORDER BY _score DESC, merged.kb_id ASC, merged.id ASC LIMIT %s OFFSET %s"+ )+ return sql, [+ *text_score_params,+ *where_params,+ *match_params,+ candidate_limit,+ vector,+ *where_params,+ vector,+ candidate_limit,+ float(vector_weight),+ float(vector_weight),+ *final_score_params,+ *highlight_params,+ *threshold_score_params,+ threshold,+ limit,+ offset,+ ]++ def _build_text_match_expr(self, keywords: list[str]) -> tuple[str, list[Any]]:+ match_exprs: list[str] = []+ params: list[Any] = []+ for config, query_text in self._text_queries(keywords):+ match_exprs.append(f"{self.build_fts_vector_expr(config)} @@ plainto_tsquery('{config}', %s)")+ params.append(query_text)+ return (" AND ".join(match_exprs), params) if match_exprs else ("FALSE", [])++ def _build_order_by(self, order_by: OrderByExpr | None) -> str:+ fields = getattr(order_by, "fields", None) or []+ parts: list[str] = []+ for field, direction in fields:+ column = self.validate_column(field)+ order = "DESC" if direction else "ASC"+ if column in {"page_num_int", "position_int", "top_int"}:+ parts.append(self.build_position_order_sql())+ else:+ parts.append(f"{self._column_expr(column, text=self._is_dynamic_column(column))} {order}")+ return ", ".join(parts)++ def _select_exprs(self, columns: list[str], prefix: str | None = None) -> list[str]:+ expressions = []+ for column in columns:+ if column == "row_id()":+ expressions.append('NULL AS "row_id()"')+ elif self._is_dynamic_column(column):+ expressions.append(f"{self._column_expr(column, prefix=prefix)} AS {column}")+ elif column in self.COLUMN_ALIASES:+ storage_column = self._storage_column(column)+ source = f"{prefix}.{storage_column}" if prefix else storage_column+ expressions.append(f"{source} AS {column}")+ elif prefix:+ expressions.append(f"{prefix}.{column}")+ else:+ expressions.append(column)+ return expressions++ def _is_dynamic_column(self, column: str) -> bool:+ return column in self.JSONB_EXTRA_SCALAR_COLUMNS or not (+ column in self.CHUNK_COLUMNS or column == "row_id()" or self.VECTOR_COLUMN_RE.fullmatch(column) or self.VECTOR_VALID_COLUMN_RE.fullmatch(column)+ )++ def _is_jsonb_dynamic_column(self, column: str) -> bool:+ return self._is_dynamic_column(column) and column not in self.JSONB_EXTRA_SCALAR_COLUMNS++ def _column_expr(+ self,+ column: str,+ *,+ prefix: str | None = None,+ text: bool = False,+ ) -> str:+ if self._is_dynamic_column(column):+ if column in self.JSONB_EXTRA_SCALAR_COLUMNS:+ source = f"{prefix}.extra" if prefix else "extra"+ return f"({source} #>> '{{{column}}}')"+ return extra_field_expr(+ column,+ prefix=prefix,+ text=text,+ )+ storage_column = self._storage_column(column)+ return f"{prefix}.{storage_column}" if prefix else storage_column++ def _storage_column(self, column: str) -> str:+ if column in self.JSONB_EXTRA_SCALAR_COLUMNS:+ return f"(extra #>> '{{{column}}}')"+ return self.COLUMN_ALIASES.get(column, column)++ def _score_with_pagerank(self, score_expr: str, pagerank_weight: float, table_alias: str | None = None) -> tuple[str, list[Any]]:+ weight = float(pagerank_weight or 0.0)+ if weight <= 0.0:+ return score_expr, []+ column = "pagerank_fea" if table_alias is None else f"{table_alias}.pagerank_fea"+ pagerank_expr = f"(COALESCE({column}, 0)::DOUBLE PRECISION / 100.0 * %s)"+ return f"({score_expr} + {pagerank_expr})", [weight]++ def _text_query_param(self, keywords: list[str]) -> str:+ return " ".join(str(keyword).strip() for keyword in keywords if str(keyword).strip())++ @classmethod+ def split_text_query_terms(cls, keywords: list[str]) -> tuple[list[str], list[str]]:+ simple_terms: list[str] = []+ ngram_terms: list[str] = []+ for keyword in keywords:+ text = str(keyword).strip()+ if not text:+ continue+ matches = list(cls.CJK_RUN_RE.finditer(text))+ if not matches:+ simple_terms.append(text)+ continue+ cursor = 0+ for match in matches:+ simple_part = text[cursor : match.start()].strip()+ if simple_part and any(char.isalnum() for char in simple_part):+ simple_terms.append(simple_part)+ cjk_part = match.group()+ if len(cjk_part) >= 2:+ ngram_terms.append(cjk_part)+ cursor = match.end()+ simple_part = text[cursor:].strip()+ if simple_part and any(char.isalnum() for char in simple_part):+ simple_terms.append(simple_part)+ return simple_terms, ngram_terms++ def _text_queries(self, keywords: list[str]) -> list[tuple[str, str]]:+ simple_terms, ngram_terms = self.split_text_query_terms(keywords)+ queries = []+ if simple_terms:+ queries.append(("simple", self._text_query_param(simple_terms)))+ if ngram_terms:+ queries.append(("ngram", self._text_query_param(ngram_terms)))+ return queries++ def _vector_param(self, vector: list[float] | tuple[float, ...], vector_dim: int | None) -> str:+ if vector_dim is None:+ raise ValueError("vector_dim is required for vector search")+ dim = self.ddl.validate_vector_dim(vector_dim)+ values = list(vector)+ if len(values) != dim:+ raise ValueError(f"vector dimension mismatch: expected {dim}, got {len(values)}")+ return "[" + ",".join(str(float(value)) for value in values) + "]"++ def _list_values(self, value) -> list[Any]:+ values = list(value) if isinstance(value, (list, tuple, set)) else [value]+ if not values:+ raise ValueError("empty condition values are not supported")+ return values++ def _candidate_limit(self, limit: int, offset: int, topn: int | None) -> int:+ base = max(int(limit or 0) + int(offset or 0), int(limit or 0), 1)+ if topn and int(topn) > 0:+ base = max(base, int(topn))+ return base+++class GaussDBConnectionBase(DocStoreConnection):+ def __init__(self, pool: Any | None = None, logger_name: str = "ragflow.gaussdb_conn"):+ if pool is None:+ from common.doc_store.gaussdb_conn_pool import GAUSSDB_CONN++ pool = GAUSSDB_CONN+ self.logger = logging.getLogger(logger_name)+ self.pool = pool+ self.masked_uri = self.pool.masked_uri+ self.resolved_schema = self.pool.resolved_schema+ self.schema = self.resolved_schema+ self.ddl = GaussDBDDLBuilder(schema=self.resolved_schema)+ self.pool.check_schema_access()+ self.logger.info("GaussDB %s connection initialized.", self.masked_uri)++ def db_type(self) -> str:+ return "gaussdb"++ def health(self) -> dict:+ result = {+ "status": "unhealthy",+ "uri": self.masked_uri,+ "version_comment": "unknown",+ "schema": self.resolved_schema,+ "server_encoding": "unknown",+ "client_encoding": "unknown",+ }+ try:+ result["version_comment"] = self._query_version()+ result["sql_compatibility"] = self._query_sql_compatibility()+ if result["sql_compatibility"] not in {"A", "ORA"}:+ result["error"] = f"unsupported GaussDB compatibility, expected A/ORA: sql_compatibility={result['sql_compatibility']}"+ return result+ result["server_encoding"] = self._query_server_encoding()+ result["client_encoding"] = self._query_client_encoding()+ normalized_client_encoding = result["client_encoding"].replace("-", "").replace("_", "")+ if normalized_client_encoding != "UTF8":+ result["error"] = f"unsupported GaussDB client encoding, expected UTF8: client_encoding={result['client_encoding']}"+ return result+ if result["server_encoding"] == "SQL_ASCII":+ result["warning"] = "server_encoding=SQL_ASCII does not validate or convert stored bytes; RAGFlow clients are forced to UTF8"+ result["status"] = "healthy"+ return result+ except Exception as exc:+ result["error"] = mask_gaussdb_text(exc)+ return result++ def _query_version(self) -> str:+ return self._query_required_scalar("SELECT version()", "version")++ def _query_sql_compatibility(self) -> str:+ return self._query_required_scalar("SHOW sql_compatibility", "sql_compatibility").upper()++ def _query_server_encoding(self) -> str:+ return self._query_required_scalar("SHOW server_encoding", "server_encoding").upper()++ def _query_client_encoding(self) -> str:+ return self._query_required_scalar("SHOW client_encoding", "client_encoding").upper()++ def _query_required_scalar(self, sql: str, field_name: str) -> str:+ row = self.pool.fetch_one(sql)+ if not row or row[0] is None or str(row[0]).strip() == "":+ raise RuntimeError(f"GaussDB {field_name} query returned no rows")+ return str(row[0]).strip()++ def get_performance_metrics(self) -> dict:+ st = timer()+ try:+ self.pool.fetch_one("SELECT 1")+ return {+ "connection": "connected",+ "latency_ms": round((timer() - st) * 1000.0, 3),+ "schema": self.resolved_schema,+ }+ except Exception as exc:+ return {+ "connection": "disconnected",+ "latency_ms": round((timer() - st) * 1000.0, 3),+ "error": mask_gaussdb_text(exc),+ }++ def create_idx(self, index_name: str, dataset_id: str, vector_size: int, parser_id: str = None):+ raise NotImplementedError("GaussDB create_idx is implemented in the DDL task")++ def delete_idx(self, index_name: str, dataset_id: str):+ raise NotImplementedError("GaussDB delete_idx is implemented in the CRUD task")++ def index_exist(self, index_name: str, dataset_id: str) -> bool:+ raise NotImplementedError("GaussDB index_exist is implemented in the DDL task")++ def search(+ self,+ select_fields: list[str],+ highlight_fields: list[str],+ condition: dict,+ match_expressions: list[MatchExpr],+ order_by: OrderByExpr,+ offset: int,+ limit: int,+ index_names: str | list[str],+ dataset_ids: list[str],+ agg_fields: list[str] | None = None,+ rank_feature: dict | None = None,+ ):+ raise NotImplementedError("GaussDB search is implemented in the search task")++ def get(self, data_id: str, index_name: str, dataset_ids: list[str]) -> dict | None:+ raise NotImplementedError("GaussDB get is implemented in the CRUD task")++ def insert(self, rows: list[dict], index_name: str, dataset_id: str = None) -> list[str]:+ raise NotImplementedError("GaussDB insert is implemented in the CRUD task")++ def update(self, condition: dict, new_value: dict, index_name: str, dataset_id: str) -> bool:+ raise NotImplementedError("GaussDB update is implemented in the CRUD task")++ def delete(self, condition: dict, index_name: str, dataset_id: str) -> int:+ raise NotImplementedError("GaussDB delete is implemented in the CRUD task")++ def get_total(self, res):+ raise NotImplementedError("GaussDB get_total is implemented in the adapter task")++ def get_doc_ids(self, res):+ raise NotImplementedError("GaussDB get_doc_ids is implemented in the adapter task")++ def get_fields(self, res, fields: list[str]) -> dict[str, dict]:+ raise NotImplementedError("GaussDB get_fields is implemented in the adapter task")++ def get_highlight(self, res, keywords: list[str], field_name: str):+ raise NotImplementedError("GaussDB get_highlight is implemented in the search task")++ def get_aggregation(self, res, field_name: str):+ raise NotImplementedError("GaussDB get_aggregation is implemented in the search task")++ def sql(self, sql: str, fetch_size: int, format: str):+ raise NotImplementedError("GaussDB sql is implemented in the Text-to-SQL task")diff --git a/common/doc_store/gaussdb_conn_pool.py b/common/doc_store/gaussdb_conn_pool.pynew file mode 100644index 0000000000..a0dd334ad4--- /dev/null+++ b/common/doc_store/gaussdb_conn_pool.py@@ -0,0 +1,282 @@+#+# Copyright 2025 The InfiniFlow Authors. All Rights Reserved.+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+#+import logging+import re+import threading+from dataclasses import dataclass+from typing import Any++from psycopg2 import pool as psycopg2_pool++logger = logging.getLogger("ragflow.gaussdb_conn_pool")+_IDENTIFIER_PATTERN = re.compile(r"^(?:[A-Za-z_]|[^\x00-\x7F])(?:[A-Za-z0-9_#$]|[^\x00-\x7F]){0,62}$")+_DECIMAL_PORT_PATTERN = re.compile(r"^[0-9]+$")+_DATABASE_KEYS = ("database", "db_name", "name")+++class GaussDBError(Exception):+ """Base exception for GaussDB DocEngine setup and connectivity."""+++class InvalidGaussDBConfig(ValueError, GaussDBError):+ """Raised when GaussDB DocEngine config is missing or invalid."""+++class GaussDBConnectionError(GaussDBError):+ """Raised when the GaussDB connection cannot be established or used."""+++class GaussDBAuthenticationError(GaussDBConnectionError):+ """Raised when GaussDB rejects configured credentials."""+++class GaussDBPermissionError(GaussDBConnectionError):+ """Raised when the configured user cannot use the target schema."""+++@dataclass(frozen=True)+class GaussDBConfig:+ host: str+ port: int+ database: str+ user: str+ password: str+ schema: str = "public"+++def _normalize_schema(schema: Any) -> str:+ value = str(schema or "").strip() or "public"+ if not _IDENTIFIER_PATTERN.match(value):+ raise InvalidGaussDBConfig(f"invalid gaussdb schema: {value}")+ return value+++def _normalize_required_string(value: Any, field: str, *, strip: bool = True) -> str:+ if not isinstance(value, str) or not value.strip():+ raise InvalidGaussDBConfig(f"invalid gaussdb config field: {field}")+ return value.strip() if strip else value+++def _normalize_port(value: Any) -> int:+ if isinstance(value, bool):+ raise InvalidGaussDBConfig("invalid gaussdb config field: port")+ if isinstance(value, int):+ port = value+ elif isinstance(value, str) and _DECIMAL_PORT_PATTERN.fullmatch(value):+ port = int(value)+ else:+ raise InvalidGaussDBConfig("invalid gaussdb config field: port")+ if port <= 0 or port > 65535:+ raise InvalidGaussDBConfig("invalid gaussdb config field: port")+ return port+++def load_gaussdb_config(raw: dict[str, Any] | None = None) -> GaussDBConfig:+ if raw is None:+ from common import settings++ raw = getattr(settings, "GAUSSDB", None) or settings.get_base_config("gaussdb", {})+ config = raw if isinstance(raw, dict) else {}+ database_key = next((key for key in _DATABASE_KEYS if key in config), None)+ configured_keys = {+ "host": "host" if "host" in config else None,+ "port": "port" if "port" in config else None,+ "database": database_key,+ "user": "user" if "user" in config else None,+ "password": "password" if "password" in config else None,+ }+ missing = [field for field, key in configured_keys.items() if key is None]+ if missing:+ raise InvalidGaussDBConfig(f"missing gaussdb config field(s): {', '.join(missing)}")++ return GaussDBConfig(+ host=_normalize_required_string(config["host"], "host"),+ port=_normalize_port(config["port"]),+ database=_normalize_required_string(config[database_key], "database"),+ user=_normalize_required_string(config["user"], "user"),+ password=_normalize_required_string(config["password"], "password", strip=False),+ schema=_normalize_schema(config.get("schema")),+ )+++def mask_gaussdb_uri(cfg: GaussDBConfig) -> str:+ return f"{cfg.user}@{cfg.host}:{cfg.port}/{cfg.database}?schema={cfg.schema}"+++def classify_gaussdb_exception(exc: Exception) -> GaussDBConnectionError:+ if isinstance(exc, GaussDBConnectionError):+ return exc+ text = str(exc).lower()+ if "password" in text or "authentication" in text or "invalid username" in text:+ return GaussDBAuthenticationError(str(exc))+ if "permission" in text or "privilege" in text:+ return GaussDBPermissionError(str(exc))+ return GaussDBConnectionError(str(exc))+++class GaussDBConnectionPool:+ def __init__(+ self,+ config: GaussDBConfig | None = None,+ pool: Any | None = None,+ minconn: int = 1,+ maxconn: int = 8,+ ):+ self.config = config or load_gaussdb_config()+ self.resolved_schema = self.config.schema+ self.masked_uri = mask_gaussdb_uri(self.config)+ self._pool = pool or self._create_pool(minconn=minconn, maxconn=maxconn)++ def _create_pool(self, minconn: int, maxconn: int):+ try:+ return psycopg2_pool.ThreadedConnectionPool(+ minconn,+ maxconn,+ host=self.config.host,+ port=self.config.port,+ dbname=self.config.database,+ user=self.config.user,+ password=self.config.password,+ options=(f"-c search_path={self.resolved_schema} -c client_encoding=UTF8 -c default_transaction_read_only=off"),+ )+ except Exception as exc:+ raise classify_gaussdb_exception(exc) from exc++ def _discard_conn(self, conn) -> None:+ if conn is None:+ return+ self._pool.putconn(conn, close=True)++ def _validate_conn(self, conn) -> None:+ if getattr(conn, "closed", False):+ raise GaussDBConnectionError("GaussDB connection is closed")+ cur = None+ try:+ cur = conn.cursor()+ cur.execute("SELECT 1")+ conn.rollback()+ finally:+ if cur is not None:+ cur.close()++ def get_conn(self):+ last_exc: Exception = RuntimeError("GaussDB connection checkout failed without an exception")+ for _attempt in range(2):+ conn = None+ try:+ conn = self._pool.getconn()+ self._validate_conn(conn)+ return conn+ except Exception as exc:+ last_exc = exc+ self._discard_conn(conn)+ raise classify_gaussdb_exception(last_exc) from last_exc++ def put_conn(self, conn) -> None:+ if conn is None:+ return+ try:+ conn.rollback()+ except Exception as exc:+ logger.warning("Discarding GaussDB connection after rollback failure: %s", type(exc).__name__)+ try:+ self._discard_conn(conn)+ except Exception as discard_exc:+ logger.warning("Failed to discard GaussDB connection: %s", type(discard_exc).__name__)+ return+ self._pool.putconn(conn)++ def close_all(self) -> None:+ self._pool.closeall()++ def check_schema_access(self) -> None:+ conn = self.get_conn()+ cur = None+ try:+ cur = conn.cursor()+ cur.execute(+ """+ SELECT+ has_schema_privilege(%s, %s, %s) AS has_usage,+ has_schema_privilege(%s, %s, %s) AS has_create+ """,+ (+ self.config.user,+ self.resolved_schema,+ "USAGE",+ self.config.user,+ self.resolved_schema,+ "CREATE",+ ),+ )+ row = cur.fetchone()+ has_usage, has_create = (bool(row[0]), bool(row[1])) if row else (False, False)+ missing = []+ if not has_usage:+ missing.append("USAGE")+ if not has_create:+ missing.append("CREATE")+ if missing:+ raise GaussDBPermissionError(f"GaussDB user {self.config.user} lacks {', '.join(missing)} on schema {self.resolved_schema}")+ except GaussDBPermissionError:+ raise+ except Exception as exc:+ raise classify_gaussdb_exception(exc) from exc+ finally:+ if cur is not None:+ cur.close()+ self.put_conn(conn)++ def fetch_one(self, sql: str, params: tuple[Any, ...] | None = None):+ conn = self.get_conn()+ cur = None+ try:+ cur = conn.cursor()+ cur.execute(sql, params)+ return cur.fetchone()+ except Exception as exc:+ raise classify_gaussdb_exception(exc) from exc+ finally:+ if cur is not None:+ cur.close()+ self.put_conn(conn)+++class _LazyGaussDBConnectionPool:+ """Shared lazy pool used by the GaussDB DocEngine and Memory Store."""++ def __init__(self):+ self._lock = threading.Lock()+ self._pool: GaussDBConnectionPool | None = None++ def get_pool(self) -> GaussDBConnectionPool:+ if self._pool is None:+ with self._lock:+ if self._pool is None:+ self._pool = GaussDBConnectionPool()+ return self._pool++ def close_all(self) -> None:+ with self._lock:+ if self._pool is not None:+ self._pool.close_all()+ self._pool = None++ def __getattr__(self, name: str):+ return getattr(self.get_pool(), name)+++GAUSSDB_CONN = _LazyGaussDBConnectionPool()diff --git a/common/metadata_gaussdb_filter.py b/common/metadata_gaussdb_filter.pynew file mode 100644index 0000000000..cdd5b92511--- /dev/null+++ b/common/metadata_gaussdb_filter.py@@ -0,0 +1,460 @@+#+# Copyright 2025 The InfiniFlow Authors. All Rights Reserved.+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+#+"""Translate RAGFlow document-metadata filters into GaussDB JSONB SQL predicates."""++from __future__ import annotations++import ast+import json+import re+from dataclasses import dataclass+from datetime import datetime+from typing import Any, Sequence++from common.doc_store.gaussdb_conn_base import jsonb_path_literal as encode_jsonb_path_literal++OP_NE = "\u2260"+OP_GE = "\u2265"+OP_LE = "\u2264"++SUPPORTED_OPERATORS: frozenset[str] = frozenset(+ {+ "=",+ OP_NE,+ ">",+ OP_GE,+ "<",+ OP_LE,+ "in",+ "not in",+ "contains",+ "not contains",+ "start with",+ "end with",+ "empty",+ "not empty",+ }+)++_CANONICAL_TO_INTERNAL = {+ "=": "eq",+ OP_NE: "ne",+ ">": "gt",+ OP_GE: "gte",+ "<": "lt",+ OP_LE: "lte",+ "in": "in",+ "not in": "not_in",+ "contains": "contains",+ "not contains": "not_contains",+ "start with": "start_with",+ "end with": "end_with",+ "empty": "empty",+ "not empty": "not_empty",+}++_OP_ALIASES = {+ "is": "=",+ "is not": OP_NE,+ "not is": OP_NE,+ "!=": OP_NE,+ "<>": OP_NE,+ ">=": OP_GE,+ "<=": OP_LE,+}++_INTERNAL_RANGE_SQL = {+ "gt": ">",+ "gte": ">=",+ "lt": "<",+ "lte": "<=",+}++_DATETIME_PREFIX_RE = re.compile(+ r"^(?P<year>\d{4})-(?P<month>\d{2})"+ r"(?:-(?P<day>\d{2})"+ r"(?:(?:T| )(?P<hour>\d{2})"+ r"(?::(?P<minute>\d{2})"+ r"(?::(?P<second>\d{2}))?"+ r")?"+ r")?"+ r")?$"+)+_NUMBER_RE = re.compile(r"^-?[0-9]+(\.[0-9]+)?$")+_IDENTIFIER_RE = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$")+++class UnsupportedGaussDBMetaFilter(ValueError):+ def __init__(self, reason: str) -> None:+ super().__init__(reason)+ self.reason = reason+++@dataclass(frozen=True)+class TranslatedGaussDBMetaFilter:+ sql: str+ params: list[Any]+++@dataclass(frozen=True)+class GaussDBFilterPlan:+ logic: str+ translated: list[TranslatedGaussDBMetaFilter]++ def to_predicate(self) -> TranslatedGaussDBMetaFilter:+ if not self.translated:+ return TranslatedGaussDBMetaFilter("1=1", [])+ joiner = " AND " if self.logic == "and" else " OR "+ sql = joiner.join(f"({item.sql})" for item in self.translated)+ params: list[Any] = []+ for item in self.translated:+ params.extend(item.params)+ return TranslatedGaussDBMetaFilter(sql, params)+++class GaussDBMetaFilterTranslator:+ def __init__(self, jsonb_column: str = "meta_fields") -> None:+ self.jsonb_column = validate_jsonb_column(jsonb_column)++ def translate(self, flt: dict) -> TranslatedGaussDBMetaFilter:+ key = validate_meta_key(flt.get("key"))+ op = normalize_metadata_filter_op(flt)+ value = flt.get("value")++ if op == "empty":+ return TranslatedGaussDBMetaFilter(self._empty_predicate(key), [])+ if op == "not_empty":+ return TranslatedGaussDBMetaFilter(self._not_empty_predicate(key), [])+ if op == "eq":+ return self._translate_equal(key, value, flt)+ if op == "ne":+ return self._translate_not_equal(key, value, flt)+ if op in _INTERNAL_RANGE_SQL:+ return self._translate_range(key, op, value, flt)+ if op == "in":+ return self._translate_in(key, value, flt)+ if op == "not_in":+ return self._translate_not_in(key, value, flt)+ if op == "contains":+ return self._translate_contains(key, value, flt)+ if op == "not_contains":+ return self._translate_not_contains(key, value, flt)+ if op == "start_with":+ return self._translate_start_with(key, value, flt)+ return self._translate_end_with(key, value, flt)++ def _translate_equal(self, key: str, value: Any, flt: dict) -> TranslatedGaussDBMetaFilter:+ sql, params = self._equal_predicate(key, value, flt)+ return TranslatedGaussDBMetaFilter(sql, params)++ def _translate_not_equal(self, key: str, value: Any, flt: dict) -> TranslatedGaussDBMetaFilter:+ sql, params = self._equal_predicate(key, value, flt)+ return TranslatedGaussDBMetaFilter(f"{self._key_exists_expr(key)} AND ({sql}) IS NOT TRUE", params)++ def _translate_range(self, key: str, op: str, value: Any, flt: dict) -> TranslatedGaussDBMetaFilter:+ value_type, coerced = coerce_range_value(value, flt)+ text_expr = self._text_expr(key)+ sql_op = _INTERNAL_RANGE_SQL[op]+ if value_type == "datetime":+ datetime_expr = self._datetime_expr(text_expr)+ return TranslatedGaussDBMetaFilter(+ f"{datetime_expr} {sql_op} to_timestamp(%s, 'YYYY-MM-DD HH24:MI:SS')",+ [coerced],+ )+ return TranslatedGaussDBMetaFilter(+ f"CASE WHEN {text_expr} ~ '^-?[0-9]+(\\.[0-9]+)?$' THEN ({text_expr})::DOUBLE PRECISION {sql_op} %s ELSE FALSE END",+ [coerced],+ )++ def _translate_in(self, key: str, value: Any, flt: dict) -> TranslatedGaussDBMetaFilter:+ fragments, params = self._membership_predicates(key, value, flt)+ return TranslatedGaussDBMetaFilter(" OR ".join(f"({sql})" for sql in fragments), params)++ def _translate_not_in(self, key: str, value: Any, flt: dict) -> TranslatedGaussDBMetaFilter:+ fragments, params = self._membership_predicates(key, value, flt)+ sql = " AND ".join(f"({fragment}) IS NOT TRUE" for fragment in fragments)+ return TranslatedGaussDBMetaFilter(f"{self._key_exists_expr(key)} AND {sql}", params)++ def _translate_contains(self, key: str, value: Any, flt: dict) -> TranslatedGaussDBMetaFilter:+ text = coerce_string_value(value, flt)+ array_expr = f"jsonb_exists({self._value_expr(key)}, %s)"+ like_expr = f"lower({self._text_expr(key)}) LIKE %s ESCAPE '\\'"+ return TranslatedGaussDBMetaFilter(+ f"{array_expr} OR {like_expr}",+ [text.lower(), f"%{escape_like_pattern(text.lower())}%"],+ )++ def _translate_not_contains(self, key: str, value: Any, flt: dict) -> TranslatedGaussDBMetaFilter:+ positive = self._translate_contains(key, value, flt)+ return TranslatedGaussDBMetaFilter(+ f"{self._key_exists_expr(key)} AND ({positive.sql}) IS NOT TRUE",+ positive.params,+ )++ def _translate_start_with(self, key: str, value: Any, flt: dict) -> TranslatedGaussDBMetaFilter:+ text = coerce_string_value(value, flt).lower()+ return TranslatedGaussDBMetaFilter(+ f"lower({self._text_expr(key)}) LIKE %s ESCAPE '\\'",+ [f"{escape_like_pattern(text)}%"],+ )++ def _translate_end_with(self, key: str, value: Any, flt: dict) -> TranslatedGaussDBMetaFilter:+ text = coerce_string_value(value, flt).lower()+ return TranslatedGaussDBMetaFilter(+ f"lower({self._text_expr(key)}) LIKE %s ESCAPE '\\'",+ [f"%{escape_like_pattern(text)}"],+ )++ def _equal_predicate(self, key: str, value: Any, flt: dict) -> tuple[str, list[Any]]:+ value = coerce_scalar_value(value, flt)+ value_expr = self._value_expr(key)+ if value is None:+ return f"{self._key_exists_expr(key)} AND {value_expr} = 'null'::jsonb", []+ if isinstance(value, str):+ if value == "":+ return f"{value_expr} = '\"\"'::jsonb", []+ value = value.lower()+ return f"lower({self._text_expr(key)}) = %s OR jsonb_exists({value_expr}, %s)", [value, value]+ return f"{value_expr} @> %s::jsonb", [jsonb_param(value)]++ def _membership_predicates(self, key: str, value: Any, flt: dict) -> tuple[list[str], list[Any]]:+ fragments: list[str] = []+ params: list[Any] = []+ for member in coerce_membership_values(value, flt):+ sql, member_params = self._equal_predicate(key, member, flt)+ fragments.append(sql)+ params.extend(member_params)+ return fragments, params++ def _empty_predicate(self, key: str) -> str:+ value_expr = self._value_expr(key)+ return f"{self._key_missing_expr(key)} OR {value_expr} = 'null'::jsonb OR {value_expr} = '\"\"'::jsonb OR {value_expr} = '[]'::jsonb OR {value_expr} = '{{}}'::jsonb"++ def _not_empty_predicate(self, key: str) -> str:+ value_expr = self._value_expr(key)+ return (+ f"{self._key_exists_expr(key)} AND "+ f"({value_expr} = 'null'::jsonb) IS NOT TRUE AND "+ f"({value_expr} = '\"\"'::jsonb) IS NOT TRUE AND "+ f"({value_expr} = '[]'::jsonb) IS NOT TRUE AND "+ f"({value_expr} = '{{}}'::jsonb) IS NOT TRUE"+ )++ def _value_expr(self, key: str) -> str:+ return f"{self.jsonb_column} #> {jsonb_path_literal(key)}"++ def _text_expr(self, key: str) -> str:+ return f"{self.jsonb_column} #>> {jsonb_path_literal(key)}"++ def _datetime_expr(self, text_expr: str) -> str:+ normalized = f"replace({text_expr}, 'T', ' ')"+ return (+ "CASE "+ f"WHEN {text_expr} ~ '^[0-9]{{4}}-[0-9]{{2}}$' "+ f"THEN to_timestamp({text_expr} || '-01 00:00:00', 'YYYY-MM-DD HH24:MI:SS') "+ f"WHEN {text_expr} ~ '^[0-9]{{4}}-[0-9]{{2}}-[0-9]{{2}}$' "+ f"THEN to_timestamp({text_expr} || ' 00:00:00', 'YYYY-MM-DD HH24:MI:SS') "+ f"WHEN {text_expr} ~ '^[0-9]{{4}}-[0-9]{{2}}-[0-9]{{2}}[T ][0-9]{{2}}$' "+ f"THEN to_timestamp({normalized} || ':00:00', 'YYYY-MM-DD HH24:MI:SS') "+ f"WHEN {text_expr} ~ '^[0-9]{{4}}-[0-9]{{2}}-[0-9]{{2}}[T ][0-9]{{2}}:[0-9]{{2}}$' "+ f"THEN to_timestamp({normalized} || ':00', 'YYYY-MM-DD HH24:MI:SS') "+ f"WHEN {text_expr} ~ '^[0-9]{{4}}-[0-9]{{2}}-[0-9]{{2}}[T ][0-9]{{2}}:[0-9]{{2}}:[0-9]{{2}}$' "+ f"THEN to_timestamp({normalized}, 'YYYY-MM-DD HH24:MI:SS') "+ "END"+ )++ def _key_exists_expr(self, key: str) -> str:+ # JSON null is a non-NULL jsonb value; #> yields SQL NULL only when the path is missing.+ return f"({self._value_expr(key)}) IS NOT NULL"++ def _key_missing_expr(self, key: str) -> str:+ return f"({self._value_expr(key)}) IS NULL"+++def build_gaussdb_filter(+ filters: Sequence[dict],+ logic: str,+) -> tuple[str, list[Any]]:+ translated = plan_pushdown(filters, logic).to_predicate()+ return translated.sql, translated.params+++def plan_pushdown(+ filters: Sequence[dict],+ logic: str,+) -> GaussDBFilterPlan:+ if logic not in {"and", "or"}:+ raise UnsupportedGaussDBMetaFilter(f"unknown logic {logic!r}")+ translator = GaussDBMetaFilterTranslator()+ return GaussDBFilterPlan(logic=logic, translated=[translator.translate(flt) for flt in filters])+++def fetch_gaussdb_metadata_doc_ids(+ doc_store,+ index_name: str,+ kb_ids: Sequence[str],+ sql_filter: str,+ filter_params: Sequence[Any],+ limit: int,+) -> list[str]:+ return doc_store.fetch_metadata_doc_ids(index_name, kb_ids, sql_filter, filter_params, limit)+++def is_pushdown_supported(filters: Sequence[dict]) -> bool:+ try:+ plan_pushdown(filters, "and")+ return True+ except UnsupportedGaussDBMetaFilter:+ return False+++def normalize_gaussdb_meta_operator(op: Any) -> str:+ if op is None:+ raise UnsupportedGaussDBMetaFilter("metadata filter operator is missing")+ normalized = " ".join(str(op).strip().lower().split())+ canonical = _OP_ALIASES.get(normalized, normalized)+ if canonical not in SUPPORTED_OPERATORS:+ raise UnsupportedGaussDBMetaFilter(f"unsupported metadata filter operator {op!r}")+ return canonical+++def normalize_metadata_filter_op(flt: dict | Any) -> str:+ op = flt.get("op", flt.get("operator", flt.get("comparison_operator"))) if isinstance(flt, dict) else flt+ return _CANONICAL_TO_INTERNAL[normalize_gaussdb_meta_operator(op)]+++def validate_jsonb_column(column: str) -> str:+ if not _IDENTIFIER_RE.fullmatch(column or ""):+ raise UnsupportedGaussDBMetaFilter(f"invalid JSONB column {column!r}")+ return column+++def validate_meta_key(key: Any) -> str:+ split_meta_key(key)+ return key+++def split_meta_key(key: Any) -> list[str]:+ if not isinstance(key, str) or not key:+ raise UnsupportedGaussDBMetaFilter("invalid metadata key")+ segments = key.split(".")+ if not segments or any(not segment for segment in segments):+ raise UnsupportedGaussDBMetaFilter("metadata key contains an empty path segment")+ if any("\x00" in segment for segment in segments):+ raise UnsupportedGaussDBMetaFilter("metadata key contains NUL")+ return segments+++def jsonb_path_literal(key: str) -> str:+ return encode_jsonb_path_literal(split_meta_key(key))+++def coerce_scalar_value(value: Any, flt: dict) -> Any:+ if isinstance(value, (list, dict, tuple)):+ raise UnsupportedGaussDBMetaFilter("scalar comparison value is non-scalar")+ if isinstance(value, str):+ stripped = value.strip()+ if stripped == "":+ return value+ try:+ parsed = ast.literal_eval(stripped)+ except Exception:+ return value+ if isinstance(parsed, (int, float, bool)) or parsed is None:+ return parsed+ return value+++def coerce_range_value(value: Any, flt: dict) -> tuple[str, int | float | str]:+ if value is None:+ raise UnsupportedGaussDBMetaFilter("range comparison value is None")+ if isinstance(value, bool):+ raise UnsupportedGaussDBMetaFilter("range comparison value is boolean")+ if isinstance(value, (int, float)):+ return "number", value++ text = str(value).strip()+ datetime_value = normalize_datetime_prefix(text)+ if datetime_value is not None:+ return "datetime", datetime_value++ try:+ parsed = ast.literal_eval(text)+ except Exception:+ parsed = None+ if isinstance(parsed, (int, float)) and not isinstance(parsed, bool):+ return "number", parsed+ if _NUMBER_RE.fullmatch(text):+ return ("number", float(text) if "." in text else int(text))+ raise UnsupportedGaussDBMetaFilter("unsupported range comparison value")+++def normalize_datetime_prefix(text: str) -> str | None:+ match = _DATETIME_PREFIX_RE.fullmatch(text)+ if not match:+ return None+ parts = match.groupdict()+ values = {+ "year": int(parts["year"]),+ "month": int(parts["month"]),+ "day": int(parts["day"] or 1),+ "hour": int(parts["hour"] or 0),+ "minute": int(parts["minute"] or 0),+ "second": int(parts["second"] or 0),+ }+ try:+ return datetime(**values).strftime("%Y-%m-%d %H:%M:%S")+ except ValueError:+ return None+++def coerce_string_value(value: Any, flt: dict) -> str:+ if value is None or isinstance(value, (list, dict, tuple)):+ raise UnsupportedGaussDBMetaFilter("string operator value must be a scalar")+ text = str(value)+ if not text:+ raise UnsupportedGaussDBMetaFilter("string operator value is empty")+ return text+++def coerce_membership_values(value: Any, flt: dict) -> list[Any]:+ if value is None:+ raise UnsupportedGaussDBMetaFilter("membership value is None")+ if isinstance(value, (list, tuple)):+ members = list(value)+ elif isinstance(value, str):+ stripped = value.strip()+ try:+ parsed = ast.literal_eval(stripped)+ except Exception:+ parsed = value+ if isinstance(parsed, (list, tuple)):+ members = list(parsed)+ else:+ members = [item.strip() for item in value.split(",") if item.strip()]+ else:+ members = [value]+ if not members:+ raise UnsupportedGaussDBMetaFilter("membership value resolved to empty list")+ return [coerce_scalar_value(member, flt) for member in members]+++def jsonb_param(value: Any) -> str:+ return json.dumps(value, ensure_ascii=False)+++def escape_like_pattern(value: str) -> str:+ return value.replace("\\", "\\\\").replace("%", "\\%").replace("_", "\\_")diff --git a/common/settings.py b/common/settings.pyindex 2eff4a7be7..7db73d6330 100644--- a/common/settings.py+++ b/common/settings.py@@ -17,6 +17,7 @@ import osimport jsonimport secretsimport logging+import refrom datetime import datefrom common.constants import RAG_FLOW_SERVICE_NAME@@ -30,6 +31,7 @@ import rag.utils.es_connimport rag.utils.infinity_connimport rag.utils.ob_connimport rag.utils.opensearch_conn+import rag.utils.gaussdb_connfrom rag.utils.azure_sas_conn import RAGFlowAzureSasBlobfrom rag.utils.azure_spn_conn import RAGFlowAzureSpnBlobfrom rag.utils.gcs_conn import RAGFlowGCS@@ -44,6 +46,7 @@ from rag.nlp import searchimport memory.utils.es_conn as memory_es_connimport memory.utils.infinity_conn as memory_infinity_connimport memory.utils.ob_conn as memory_ob_conn+import memory.utils.gaussdb_conn as memory_gaussdb_connTIMEZONE = os.getenv("TZ", "Asia/Shanghai")@@ -70,8 +73,89 @@ SECRET_KEY = NoneFACTORY_LLM_INFOS = NoneALLOWED_LLM_FACTORIES = None-DATABASE_TYPE = os.getenv("DB_TYPE", "mysql")-DATABASE = decrypt_database_config(name=DATABASE_TYPE)+# The metadata database and DocEngine/Memory Store may all use GaussDB while+# targeting different databases, schemas, compatibility modes, or credentials.+# The metadata database therefore reads only GAUSSDB_METADATA_*. The gaussdb+# section in service_conf.yaml remains exclusive to DOC_ENGINE=gaussdb.+GAUSSDB_ENV_DEFAULTS = {+ "name": "rag_flow",+ "user": "rag_flow",+ "password": "infini_rag_flow",+ "host": "gaussdb",+ "port": 8000,+ "schema": "public",+ "max_connections": 100,+ "stale_timeout": 30,+ "options": "-c client_encoding=UTF8 -c default_transaction_read_only=off",+}+_GAUSSDB_SCHEMA_PATTERN = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$")+++def normalize_database_type(database_type: str | None = None) -> str:+ # Only normalize the new GaussDB values. Existing database names retain+ # their upstream spelling and lookup behavior.+ raw_value = database_type or "mysql"+ normalized = raw_value.strip().lower()+ if normalized in {"gaussdb", "gauss"}:+ return "gaussdb"+ return raw_value+++def _get_int_env(name: str, default: int) -> int:+ try:+ return int(os.environ.get(name, default))+ except (TypeError, ValueError):+ return default+++def _normalize_gaussdb_metadata_schema(value: str | None = None) -> str:+ # The schema is interpolated into search_path in the libpq options. Restrict+ # it to a plain SQL identifier so quotes, semicolons, or extra options cannot+ # be injected. The metadata database currently accepts one schema only.+ schema = (value or GAUSSDB_ENV_DEFAULTS["schema"]).strip() or GAUSSDB_ENV_DEFAULTS["schema"]+ if not _GAUSSDB_SCHEMA_PATTERN.match(schema):+ raise ValueError(f"invalid GAUSSDB_METADATA_SCHEMA: {schema}")+ return schema+++def _gaussdb_metadata_options(schema: str) -> str:+ explicit_options = os.environ.get("GAUSSDB_METADATA_OPTIONS")+ if explicit_options is not None:+ # An explicit value replaces the complete options string. Advanced+ # deployments can control search_path, encoding, read-only behavior,+ # and other libpq options, but must also include any required defaults.+ return explicit_options+ return f"-c search_path={schema} {GAUSSDB_ENV_DEFAULTS['options']}"+++def _gaussdb_env_config() -> dict:+ # Read only GAUSSDB_METADATA_* so the metadata database does not share+ # connection parameters with DOC_ENGINE=gaussdb.+ schema = _normalize_gaussdb_metadata_schema(os.environ.get("GAUSSDB_METADATA_SCHEMA"))+ return {+ "name": os.environ.get("GAUSSDB_METADATA_DBNAME", GAUSSDB_ENV_DEFAULTS["name"]),+ "user": os.environ.get("GAUSSDB_METADATA_USER", GAUSSDB_ENV_DEFAULTS["user"]),+ "password": os.environ.get("GAUSSDB_METADATA_PASSWORD", GAUSSDB_ENV_DEFAULTS["password"]),+ "host": os.environ.get("GAUSSDB_METADATA_HOST", GAUSSDB_ENV_DEFAULTS["host"]),+ "port": _get_int_env("GAUSSDB_METADATA_PORT", GAUSSDB_ENV_DEFAULTS["port"]),+ "max_connections": _get_int_env("GAUSSDB_METADATA_MAX_CONNECTIONS", GAUSSDB_ENV_DEFAULTS["max_connections"]),+ "stale_timeout": _get_int_env("GAUSSDB_METADATA_STALE_TIMEOUT", GAUSSDB_ENV_DEFAULTS["stale_timeout"]),+ "options": _gaussdb_metadata_options(schema),+ }+++def load_database_config(database_type: str) -> dict:+ database_type = normalize_database_type(database_type)+ if database_type == "gaussdb":+ # Always build DB_TYPE=gaussdb from GAUSSDB_METADATA_* so the metadata+ # connection remains isolated from the gaussdb section used by+ # DOC_ENGINE=gaussdb.+ return decrypt_database_config(database=_gaussdb_env_config())+ return decrypt_database_config(name=database_type)+++DATABASE_TYPE = normalize_database_type(os.getenv("DB_TYPE", "mysql"))+DATABASE = load_database_config(DATABASE_TYPE)# authenticationAUTHENTICATION_CONF = None@@ -85,6 +169,7 @@ OAUTH_CONFIG = NoneDOC_ENGINE = os.getenv("DOC_ENGINE", "elasticsearch")DOC_ENGINE_INFINITY = DOC_ENGINE.lower() == "infinity"DOC_ENGINE_OCEANBASE = DOC_ENGINE.lower() == "oceanbase"+DOC_ENGINE_GAUSSDB = DOC_ENGINE.lower() == "gaussdb"DOC_ENGINE_SERENEDB = DOC_ENGINE.lower() == "serenedb"@@ -124,6 +209,7 @@ OB = {}OSS = {}OS = {}GCS = {}+GAUSSDB = {}SERENEDB = {}DOC_MAXIMUM_SIZE: int = 128 * 1024 * 1024@@ -220,8 +306,8 @@ class StorageFactory:def init_settings():global DATABASE_TYPE, DATABASE- DATABASE_TYPE = os.getenv("DB_TYPE", "mysql")- DATABASE = decrypt_database_config(name=DATABASE_TYPE)+ DATABASE_TYPE = normalize_database_type(os.getenv("DB_TYPE", "mysql"))+ DATABASE = load_database_config(DATABASE_TYPE)global ALLOWED_LLM_FACTORIES, LLM_FACTORY, LLM_BASE_URLllm_settings = get_base_config("user_default_llm", {}) or {}@@ -303,10 +389,11 @@ def init_settings():FEISHU_OAUTH = get_base_config("oauth", {}).get("feishu")OAUTH_CONFIG = get_base_config("oauth", {})- global DOC_ENGINE, DOC_ENGINE_INFINITY, DOC_ENGINE_OCEANBASE, DOC_ENGINE_SERENEDB, docStoreConn, ES, OB, OS, INFINITY, SERENEDB+ global DOC_ENGINE, DOC_ENGINE_INFINITY, DOC_ENGINE_OCEANBASE, DOC_ENGINE_GAUSSDB, DOC_ENGINE_SERENEDB, docStoreConn, ES, OB, OS, INFINITY, GAUSSDB, SERENEDBDOC_ENGINE = os.environ.get("DOC_ENGINE", "elasticsearch").strip()DOC_ENGINE_INFINITY = DOC_ENGINE.lower() == "infinity"DOC_ENGINE_OCEANBASE = DOC_ENGINE.lower() == "oceanbase"+ DOC_ENGINE_GAUSSDB = DOC_ENGINE.lower() == "gaussdb"DOC_ENGINE_SERENEDB = DOC_ENGINE.lower() == "serenedb"lower_case_doc_engine = DOC_ENGINE.lower()if lower_case_doc_engine == "elasticsearch":@@ -324,6 +411,9 @@ def init_settings():elif lower_case_doc_engine == "seekdb":OB = get_base_config("seekdb", {})docStoreConn = rag.utils.ob_conn.OBConnection()+ elif lower_case_doc_engine == "gaussdb":+ GAUSSDB = get_base_config("gaussdb", {})+ docStoreConn = rag.utils.gaussdb_conn.GaussDBConnection()elif lower_case_doc_engine == "serenedb":SERENEDB = get_base_config("serenedb", {})# Imported lazily so psycopg2/SereneDB is only touched when selected.@@ -343,6 +433,11 @@ def init_settings():msgStoreConn = memory_infinity_conn.InfinityConnection()elif lower_case_doc_engine in ["oceanbase", "seekdb"]:msgStoreConn = memory_ob_conn.OBConnection()+ elif lower_case_doc_engine == "gaussdb":+ # Memory Store uses a dedicated adapter for message tables. It reads the+ # same GaussDB configuration and shares the lazy connection pool with+ # docStoreConn, but keeps its own table layout and query semantics.+ msgStoreConn = memory_gaussdb_conn.GaussDBMemoryConnection()global AZURE, S3, MINIO, OSS, GCSif STORAGE_IMPL_TYPE in ["AZURE_SPN", "AZURE_SAS"]:diff --git a/docker/.env b/docker/.envindex 0940323198..b8af55624f 100644--- a/docker/.env+++ b/docker/.env@@ -17,15 +17,25 @@# - `oceanbase` (https://github.com/oceanbase/oceanbase)# - `opensearch` (https://github.com/opensearch-project/OpenSearch)# - `seekdb` (https://github.com/oceanbase/seekdb)+# - `gaussdb` GaussDB Centralized / Distributed DocEngineDOC_ENGINE=${DOC_ENGINE:-elasticsearch}+# The business metadata database type used by DB_TYPE.+# Available options: mysql (default), postgres, gaussdb, oceanbase.+DB_TYPE=${DB_TYPE:-mysql}+# Device on which deepdoc inference run.# Available levels:# - `cpu` (default)# - `gpu`DEVICE=${DEVICE:-cpu}-COMPOSE_PROFILES=${DOC_ENGINE},${DEVICE}+# This controls only the bundled metadata service started by Compose; DB_TYPE+# independently selects the metadata backend used by RAGFlow. Keep `mysql` to+# start the in-cluster MySQL service. For external GaussDB, set both+# DB_TYPE=gaussdb and METADATA_DB_PROFILE=gaussdb so MySQL stays disabled.+METADATA_DB_PROFILE=${METADATA_DB_PROFILE:-mysql}+COMPOSE_PROFILES=${DOC_ENGINE},${DEVICE},metadata-${METADATA_DB_PROFILE}# The version of Elasticsearch.STACK_VERSION=${STACK_VERSION:-8.11.3}@@ -91,6 +101,21 @@ OCEANBASE_PASSWORD=infini_rag_flow# The doc database of the OceanBase service to useOCEANBASE_DOC_DBNAME=ragflow_doc+# GaussDB DocEngine connection. RAGFlow does not start or manage GaussDB;+# configure an existing GaussDB instance here.+# GaussDB host.+GAUSSDB_HOST=+# GaussDB port.+GAUSSDB_PORT=+# GaussDB database name.+GAUSSDB_DATABASE=postgres+# GaussDB login user.+GAUSSDB_USER=+# GaussDB login password.+GAUSSDB_PASSWORD=+# GaussDB schema for RAGFlow DocEngine objects.+GAUSSDB_SCHEMA=+# OceanBase container configurationOB_CLUSTER_NAME=${OB_CLUSTER_NAME:-ragflow}OB_TENANT_NAME=${OB_TENANT_NAME:-ragflow}@@ -130,6 +155,17 @@ EXPOSE_MYSQL_PORT=3306# The maximum size of communication packets sent to the MySQL serverMYSQL_MAX_PACKET=1073741824+# External GaussDB metadata database settings, used when DB_TYPE=gaussdb.+# These are intentionally separate from the GAUSSDB_* DocEngine settings.+GAUSSDB_METADATA_HOST=+GAUSSDB_METADATA_PORT=+GAUSSDB_METADATA_USER=+GAUSSDB_METADATA_PASSWORD=+GAUSSDB_METADATA_DBNAME=rag_flow+GAUSSDB_METADATA_SCHEMA=public+GAUSSDB_METADATA_MAX_CONNECTIONS=100+GAUSSDB_METADATA_STALE_TIMEOUT=30+# The hostname where the MinIO service is exposedMINIO_HOST=minio# The port used to expose the MinIO console interface to the host machine,diff --git a/docker/README.md b/docker/README.mdindex b5cdb13838..522aa24e69 100644--- a/docker/README.md+++ b/docker/README.md@@ -24,6 +24,17 @@The [.env](./.env) file contains important environment variables for Docker.+### Metadata database++- `DB_TYPE`+ The business metadata database type. Defaults to `mysql`. Supported values include `mysql`, `postgres`, `gaussdb`, and `oceanbase`.+- `COMPOSE_PROFILES`+ The Docker Compose profiles to enable. By default it contains `${DOC_ENGINE},${DEVICE},metadata-${METADATA_DB_PROFILE}`.+- `METADATA_DB_PROFILE`+ Defaults to `mysql`, preserving the in-cluster MySQL service. Set it to `gaussdb` together with `DB_TYPE=gaussdb` to use an external GaussDB metadata database without starting MySQL.+- `GAUSSDB_METADATA_HOST`, `GAUSSDB_METADATA_PORT`, `GAUSSDB_METADATA_USER`, `GAUSSDB_METADATA_PASSWORD`, `GAUSSDB_METADATA_DBNAME`, `GAUSSDB_METADATA_SCHEMA`+ External GaussDB metadata database connection settings, used when `DB_TYPE=gaussdb`. Set `METADATA_DB_PROFILE=gaussdb` to keep the in-cluster MySQL service disabled.+### Elasticsearch- `STACK_VERSION`@@ -50,7 +61,7 @@ The [.env](./.env) file contains important environment variables for Docker.### MySQL- `MYSQL_PASSWORD`- The password for MySQL.+ The password for MySQL. Required only when `DB_TYPE=mysql` starts the in-cluster MySQL metadata database.- `MYSQL_PORT`The port to connect to MySQL from RAGFlow container. Defaults to `3306`. Change this if you use an external MySQL.- `EXPOSE_MYSQL_PORT`@@ -207,6 +218,15 @@ Before setting `DOC_ENGINE=oceanbase`, make sure the host OS allows the file des- `host`: The hostname of the OceanBase service.- `port`: The port of OceanBase.+- `gaussdb`+ - `host`: The hostname or IP address of the GaussDB instance.+ - `port`: The GaussDB port.+ - `database`: The GaussDB database name. Defaults to `postgres`.+ - `user`: The username for GaussDB.+ - `password`: The password for GaussDB.+ - `schema`: Optional schema used by the DocEngine. Defaults to `public`.+ - RAGFlow does not start or manage GaussDB; set `DOC_ENGINE=gaussdb` only after preparing a GaussDB instance.+- `oss`- `access_key`: The access key ID used to authenticate requests to the OSS service.- `secret_key`: The secret access key used to authenticate requests to the OSS service.diff --git a/docker/docker-compose-CN-oc9.yml b/docker/docker-compose-CN-oc9.ymlindex 501b2044f3..c07a53b5d1 100644--- a/docker/docker-compose-CN-oc9.yml+++ b/docker/docker-compose-CN-oc9.yml@@ -8,6 +8,7 @@ services:depends_on:mysql:condition: service_healthy+ required: falseprofiles:- cpuimage: edwardelric233/ragflow:oc9@@ -33,6 +34,7 @@ services:depends_on:mysql:condition: service_healthy+ required: falseprofiles:- gpuimage: edwardelric233/ragflow:oc9diff --git a/docker/docker-compose-base.yml b/docker/docker-compose-base.ymlindex 6722071ab1..aa9f4ac652 100644--- a/docker/docker-compose-base.yml+++ b/docker/docker-compose-base.yml@@ -205,6 +205,11 @@ services:restart: unless-stoppedmysql:+ profiles:+ - mysql+ - metadata-mysql+ - metadata-MySQL+ - metadata-MYSQL# mysql:5.7 linux/arm64 image is unavailable.image: mysql:8.0.40env_file: .envdiff --git a/docker/docker-compose-macos.yml b/docker/docker-compose-macos.ymlindex b667108fc4..587266b395 100644--- a/docker/docker-compose-macos.yml+++ b/docker/docker-compose-macos.yml@@ -7,6 +7,7 @@ services:depends_on:mysql:condition: service_healthy+ required: falsebuild:context: ../dockerfile: Dockerfilediff --git a/docker/docker-compose.yml b/docker/docker-compose.ymlindex 14415f8219..5194984bae 100644--- a/docker/docker-compose.yml+++ b/docker/docker-compose.yml@@ -22,6 +22,7 @@ services:depends_on:mysql:condition: service_healthy+ required: falseprofiles:- cpuimage: ${RAGFLOW_IMAGE}@@ -73,6 +74,7 @@ services:depends_on:mysql:condition: service_healthy+ required: falseprofiles:- gpuimage: ${RAGFLOW_IMAGE}diff --git a/docker/entrypoint.sh b/docker/entrypoint.shindex cfd8fb65fb..7979b7e72d 100755--- a/docker/entrypoint.sh+++ b/docker/entrypoint.sh@@ -255,7 +255,15 @@ ensure_doclingensure_db_initif [[ "${INIT_MODEL_PROVIDER_TABLES}" -eq 1 ]]; then- tools/scripts/run_migrations.sh+ DB_TYPE_NORMALIZED="${DB_TYPE:-mysql}"+ DB_TYPE_NORMALIZED="${DB_TYPE_NORMALIZED,,}"+ if [[ "${DB_TYPE_NORMALIZED}" == "gaussdb" || "${DB_TYPE_NORMALIZED}" == "gauss" ]]; then+ # This migration script contains MySQL-only SQL and cannot run against+ # a GaussDB metadata database.+ echo "Skipping MySQL-specific model provider table migrations for DB_TYPE=${DB_TYPE:-mysql}."+ else+ tools/scripts/run_migrations.sh+ fifiif [[ "${ENABLE_ADMIN_SERVER}" -eq 1 ]]; thendiff --git a/docker/launch_backend_service.sh b/docker/launch_backend_service.shindex 52d9661779..017297ecc2 100755--- a/docker/launch_backend_service.sh+++ b/docker/launch_backend_service.sh@@ -211,6 +211,15 @@ ensure_db_init() {}run_mysql_migrations() {+ local db_type="${DB_TYPE:-mysql}"+ db_type="${db_type,,}"+ if [ "$db_type" = "gaussdb" ] || [ "$db_type" = "gauss" ]; then+ # This migration script contains MySQL-only SQL and cannot run against+ # a GaussDB metadata database.+ echo "Skipping MySQL-specific model provider table migrations for DB_TYPE=${DB_TYPE:-mysql}."+ return 0+ fi+tools/scripts/run_migrations.sh}diff --git a/docker/service_conf.yaml.template b/docker/service_conf.yaml.templateindex b60e1317c4..2070f96f27 100644--- a/docker/service_conf.yaml.template+++ b/docker/service_conf.yaml.template@@ -55,6 +55,13 @@ oceanbase:password: '${OCEANBASE_PASSWORD:-infini_rag_flow}'host: '${OCEANBASE_HOST:-oceanbase}'port: ${OCEANBASE_PORT:-2881}+gaussdb:+ host: '${GAUSSDB_HOST}'+ port: ${GAUSSDB_PORT}+ database: '${GAUSSDB_DATABASE:-postgres}'+ user: '${GAUSSDB_USER}'+ password: '${GAUSSDB_PASSWORD}'+ schema: '${GAUSSDB_SCHEMA:-public}'seekdb:scheme: 'oceanbase' # SeekDB is the lite version of OceanBaseconfig:@@ -103,6 +110,8 @@ user_default_llm:# port: 5432# max_connections: 100# stale_timeout: 30+# GaussDB metadata DB is configured through GAUSSDB_METADATA_* environment+# variables and is intentionally not rendered into service_conf.yaml.# s3:# access_key: 'access_key'# secret_key: 'secret_key'diff --git a/memory/utils/gaussdb_conn.py b/memory/utils/gaussdb_conn.pynew file mode 100644index 0000000000..8b48fbe2af--- /dev/null+++ b/memory/utils/gaussdb_conn.py@@ -0,0 +1,1607 @@+#+# Copyright 2025 The InfiniFlow Authors. All Rights Reserved.+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+#+"""GaussDB A/ORA-compatible Memory Store adapter.++This module stores Memory messages only; it does not manage memory metadata in+the metadata database. The service constructs a logical index_name per tenant,+which this adapter maps to a physical GaussDB table. Rows within the shared+tenant table are isolated by memory_id.++The implementation reuses the connection pool, identifier validation, and+selected utilities from common.doc_store.gaussdb_conn_base. Its table schema,+field mapping, full-text/vector/hybrid queries, capacity accounting, and FIFO+operations remain independent so DocEngine chunk semantics never leak into+Memory messages.+"""++from __future__ import annotations++import hashlib+import json+import logging+import re+from datetime import datetime+from decimal import Decimal, InvalidOperation+from typing import Any, Iterable++import numpy as np+from pydantic import BaseModel++from common.doc_store.doc_store_base import FusionExpr, MatchDenseExpr, MatchExpr, MatchTextExpr, OrderByExpr+from common.doc_store.gaussdb_conn_base import GaussDBConnectionBase, GaussDBDDLBuilder, InvalidGaussDBObjectName+from common.doc_store.gaussdb_conn_pool import GaussDBConnectionPool, classify_gaussdb_exception+from common.float_utils import get_float+from memory.utils.aggregation_utils import aggregate_by_field+from memory.utils.highlight_utils import get_highlight_from_messages+from rag.nlp import is_english+from rag.nlp.rag_tokenizer import fine_grained_tokenize, tokenize++logger = logging.getLogger("ragflow.memory_gaussdb_conn")++VECTOR_COLUMN_RE = re.compile(r"^q_(?P<dim>\d+)_vec$")+VECTOR_EMPTY_COLUMN_RE = re.compile(r"^q_(?P<dim>\d+)_vec_empty$")++# Base columns for Memory messages. Vector columns are added dynamically for+# each embedding dimension and are not listed here. These names also form the+# SQL allowlist: every external field must resolve to a base column or a+# q_{dim}_vec / q_{dim}_vec_empty dynamic column.+BASE_COLUMNS = (+ "id",+ "message_id",+ "message_type_kwd",+ "source_id",+ "memory_id",+ "user_id",+ "agent_id",+ "session_id",+ "zone_id",+ "valid_at",+ "invalid_at",+ "forget_at",+ "status_int",+ "content_ltks",+ "tokenized_content_ltks",+)+BASE_COLUMN_SET = set(BASE_COLUMNS)++TIME_COLUMNS = {"valid_at", "invalid_at", "forget_at"}+NUMERIC_COLUMNS = {"message_id", "source_id", "zone_id", "status_int"}+# MemoryService uses backend-neutral field names. GaussDB physical tables use+# suffixed names to avoid conflicts with full-text, vector, and status columns.+# Map every field before constructing SQL.+MEMORY_FIELD_MAP = {+ "message_type": "message_type_kwd",+ "status": "status_int",+ "content": "content_ltks",+}+REVERSE_MEMORY_FIELD_MAP = {+ "message_type_kwd": "message_type",+ "status_int": "status",+ "content_ltks": "content",+}+RESULT_FIELD_DEFAULTS = {+ "source_id": None,+ "user_id": "",+ "zone_id": 0,+ "invalid_at": "-",+ "forget_at": "-",+ "content": "",+ "content_embed": [],+}+++def normalize_fulltext_query(text: Any) -> str:+ # Writes store fine_grained_tokenize(tokenize(content)). Apply the same+ # normalization before plainto_tsquery so short terms, English stems, and+ # mixed-language content do not produce false negatives from mismatched+ # write-time and query-time tokens.+ query = str(text or "").strip()+ if not query:+ return ""+ tokenized = fine_grained_tokenize(tokenize(query)).strip()+ return tokenized or query+++class SearchResult(BaseModel):+ # Match other Memory Store adapters: search() returns (SearchResult, total),+ # while get_total/get_doc_ids/get_fields accept either that tuple or a bare+ # SearchResult.+ total: int+ messages: list[dict]+++class GaussDBMemoryDDLBuilder(GaussDBDDLBuilder):+ """Build DDL for Memory message tables.++ The shared GaussDBDDLBuilder handles identifiers, safe table names, and the+ advisory lock. Memory tables additionally need A/ORA-compatible types,+ UStore, UGIN full-text indexes, gsdiskann vector indexes, and vector-empty+ markers, all of which are maintained here.+ """++ # Standard indexes cover listing, deletion, capacity accounting, FIFO,+ # recent-message access, and source/raw relationships. Vector and full-text+ # indexes are generated separately because they depend on dynamic dimensions+ # or expression indexes.+ REGULAR_INDEXES = (+ ("message_id", ("message_id",)),+ ("memory_id", ("memory_id",)),+ ("message_type", ("message_type_kwd",)),+ ("source_id", ("source_id",)),+ ("agent_session", ("agent_id", "session_id")),+ ("status_valid", ("status_int", "valid_at")),+ ("forget_at", ("forget_at",)),+ )++ def physical_table_name(self, index_name: str) -> str:+ logical = str(index_name or "").strip()+ if not logical:+ raise InvalidGaussDBObjectName(index_name)+ # A logical index_name may contain tenant IDs, hyphens, or other+ # service-generated characters. Expose only a hash suffix to keep the+ # physical name stable, short, and valid for GaussDB. memory_id is not+ # part of the table name; it isolates memories within the tenant table.+ digest = hashlib.sha1(logical.encode("utf-8")).hexdigest()[:32]+ return f"ragflow_mem_{digest}"++ def build_memory_table_ddl(self, table: str) -> str:+ # Use A/ORA-compatible VARCHAR2/NUMBER types for bounded text and+ # integers while retaining TEXT for full-text content. UStore is required+ # by the gsdiskann/vector retrieval path and is selected at table creation.+ name = self.qualified_name(table)+ pk = self.index_name(table, "pk")+ return f"""CREATE TABLE IF NOT EXISTS {name} (+ id VARCHAR2(96) NOT NULL,+ message_id NUMBER(19) NOT NULL,+ message_type_kwd VARCHAR2(64),+ source_id NUMBER(19),+ memory_id VARCHAR2(32) NOT NULL,+ user_id VARCHAR2(64),+ agent_id VARCHAR2(64),+ session_id VARCHAR2(128),+ zone_id NUMBER(10) DEFAULT 0,+ valid_at TIMESTAMP,+ invalid_at TIMESTAMP,+ forget_at TIMESTAMP,+ status_int NUMBER(10) DEFAULT 1 NOT NULL,+ content_ltks TEXT,+ tokenized_content_ltks TEXT,+ CONSTRAINT {pk} PRIMARY KEY (id)+) WITH (storage_type=USTORE)"""++ def build_regular_index_ddls(self, table: str) -> list[str]:+ name = self.qualified_name(table)+ return [f"CREATE INDEX IF NOT EXISTS {self.index_name(table, suffix)} ON {name} ({', '.join(columns)})" for suffix, columns in self.REGULAR_INDEXES]++ def build_fulltext_ugin_ddl(self, table: str) -> str:+ # Index the simple tsvector expression over tokenized_content_ltks. The+ # query path uses the same simple configuration with plainto_tsquery so+ # behavior does not depend on the database's default language.+ name = self.qualified_name(table)+ idx = self.index_name(table, "tokenized_ugin")+ return f"""CREATE INDEX IF NOT EXISTS {idx}+ ON {name}+ USING ugin (to_tsvector('simple', tokenized_content_ltks))"""++ def vector_empty_column_name(self, dim: int) -> str:+ return f"q_{self.validate_vector_dim(dim)}_vec_empty"++ def build_vector_column_ddls(self, table: str, dim: int) -> list[str]:+ # GaussDB floatvector columns cannot use NULL to represent a message+ # without a vector of this dimension. Pair every vector column with an+ # *_empty marker and use a zero-vector placeholder. Retrieval filters on+ # *_empty = FALSE, and reads restore only marked non-empty dimensions.+ dim = self.validate_vector_dim(dim)+ name = self.qualified_name(table)+ vector_col = self.vector_column_name(dim)+ empty_col = self.vector_empty_column_name(dim)+ return [+ f"ALTER TABLE {name} ADD COLUMN IF NOT EXISTS {vector_col} floatvector({dim}) DEFAULT (array_fill(0, ARRAY[{dim}])::text::floatvector({dim})) NOT NULL",+ f"ALTER TABLE {name} ADD COLUMN IF NOT EXISTS {empty_col} BOOLEAN DEFAULT TRUE NOT NULL",+ ]++ def build_vector_empty_index_ddl(self, table: str, dim: int) -> str:+ dim = self.validate_vector_dim(dim)+ name = self.qualified_name(table)+ empty_col = self.vector_empty_column_name(dim)+ idx = self.index_name(table, f"{empty_col}_idx")+ return f"CREATE INDEX IF NOT EXISTS {idx} ON {name} (memory_id, {empty_col})"++ def build_diskann_index_ddl(self, table: str, dim: int) -> str:+ # gsdiskann provides approximate nearest-neighbor lookup. Include the+ # dimension column in the index name so one tenant table can hold several+ # embedding dimensions during model migration.+ dim = self.validate_vector_dim(dim)+ name = self.qualified_name(table)+ vector_col = self.vector_column_name(dim)+ idx = self.index_name(table, f"{vector_col}_diskann")+ return f"CREATE INDEX IF NOT EXISTS {idx} ON {name} USING gsdiskann ({vector_col} cosine)"+++class GaussDBMemoryConnection(GaussDBConnectionBase):+ """Store Memory messages in GaussDB.++ This class implements the Message Store interface expected by MemoryService.+ It exposes no arbitrary SQL and does not reuse DocEngine chunk queries. Every+ write, update, delete, and retrieval enforces a memory_id boundary so messages+ cannot cross memories within a shared tenant table.+ """++ def __init__(self, pool: GaussDBConnectionPool | None = None):+ super().__init__(pool=pool, logger_name="ragflow.memory_gaussdb_conn")+ # The base class initializes the shared pool and validates schema access.+ # Replace its DDL builder so create_idx and vector-column maintenance use+ # the message schema instead of the document chunk schema.+ self.ddl = GaussDBMemoryDDLBuilder(schema=self.resolved_schema)++ def create_idx(self, index_name: str, memory_id: str, vector_size: int, parser_id: str = None):+ table = self.physical_table(index_name)+ # Acquire the advisory lock before creating the base table, standard+ # indexes, full-text index, vector column, and vector-empty index. The+ # idempotent DDL supports repeated calls, concurrent initialization, and+ # first writes from multiple memories in one tenant.+ statements: list[str | tuple[str, list[Any]]] = [+ self.ddl.build_advisory_lock_sql(f"gaussdb_memory_create_table:{table}"),+ self.ddl.build_memory_table_ddl(table),+ self.ddl.build_advisory_lock_sql(f"gaussdb_memory_base_index:{table}"),+ *self.ddl.build_regular_index_ddls(table),+ self.ddl.build_advisory_lock_sql(f"gaussdb_memory_fulltext_index:{table}"),+ self.ddl.build_fulltext_ugin_ddl(table),+ ]+ statements.extend(self.ddl.build_vector_column_ddls(table, vector_size))+ statements.append(self.ddl.build_vector_empty_index_ddl(table, vector_size))+ self._execute_statements(statements)+ self._create_diskann_index_with_retry(table, vector_size)+ return True++ def delete_idx(self, index_name: str, memory_id: str):+ # The physical table boundary is the tenant-level index_name, not+ # memory_id. Tenant/user deletion may call delete_idx once per memory, so+ # DROP TABLE IF EXISTS must be repeatable. memory_id remains for interface+ # compatibility only.+ table = self.ddl.qualified_name(self.physical_table(index_name))+ self._execute_write(f"DROP TABLE IF EXISTS {table} PURGE", [])+ return True++ def index_exist(self, index_name: str, memory_id: str = None) -> bool:+ # has_index() checks the required base columns and indexes as well as the+ # table itself. Legacy or partially initialized tables therefore enter+ # the create/ensure path instead of failing later during a query.+ table = self.physical_table(index_name)+ if not self._table_exists(table):+ return False+ required_columns = set(BASE_COLUMNS)+ existing_columns = set(self._column_names(table))+ if not required_columns.issubset(existing_columns):+ return False+ required_indexes = {self.ddl.index_name(table, suffix) for suffix, _columns in self.ddl.REGULAR_INDEXES}+ required_indexes.add(self.ddl.index_name(table, "tokenized_ugin"))+ existing_indexes = set(self._index_names(table))+ return required_indexes.issubset(existing_indexes)++ def insert(self, documents: list[dict], index_name: str, memory_id: str = None) -> list[str]:+ if not documents:+ return []++ document_ids = [str(document.get("id") or "") for document in documents]+ errors: list[str] = []+ rows: list[dict] = []+ dim = None+ for document in documents:+ doc_id = str(document.get("id") or "")+ try:+ # One batch must use one embedding dimension because a MERGE can+ # bind only one q_{dim}_vec column. Reject mixed dimensions before+ # executing SQL and return the failed IDs.+ row, row_dim = self._message_to_row(document, memory_id)+ dim = row_dim if dim is None else dim+ if row_dim != dim:+ raise ValueError(f"inconsistent content_embed dimension: expected {dim}, got {row_dim}")+ rows.append(row)+ except Exception as exc:+ logger.error("GaussDB memory normalize failed id=%s error=%s", doc_id, exc)+ errors.append(doc_id or str(exc))+ if errors:+ return [document_id for document_id in document_ids if document_id] or errors++ table = self.physical_table(index_name)+ if not self._table_exists(table):+ # The first write creates the tenant message table. memory_id stays+ # in row data and WHERE predicates as the intra-table boundary.+ self.create_idx(index_name, memory_id, dim)+ else:+ # An existing tenant table may lack the new column after an embedding+ # model dimension changes. Add the column and index on demand without+ # dropping older dimensions so historical rows remain readable via+ # their *_empty markers.+ self._ensure_vector_column_exists(table, dim)++ existing_dims = self._vector_dimensions(table)+ sql, params = self._build_merge_sql(table, dim, existing_dims, rows)+ try:+ self._execute_write(sql, params, many=True)+ return []+ except Exception as exc:+ ids = [row["id"] for row in rows]+ logger.error("GaussDB memory insert failed table=%s ids=%s error=%s", table, ids, exc)+ return ids or [str(exc)]++ def update(self, condition: dict, new_value: dict, index_name: str, memory_id: str) -> bool:+ if not condition or not new_value:+ return False+ table = self.physical_table(index_name)+ if not self._table_exists(table):+ return True++ try:+ set_sql, set_params = self._build_update_set(table, index_name, memory_id, new_value)+ if not set_sql:+ return True+ where_sql, where_params = self._build_where_clause(condition, memory_ids=[memory_id], force_memory_filter=True)+ if not where_sql:+ return False+ # Always add the memory_id boundary, even when the caller supplies+ # only message_id, so an update cannot cross memories in one tenant.+ sql = f"UPDATE {self.ddl.qualified_name(table)} SET {set_sql} WHERE {where_sql}"+ self._execute_write(sql, [*set_params, *where_params])+ return True+ except Exception as exc:+ logger.error("GaussDB memory update failed table=%s condition=%s error=%s", table, condition, exc)+ return False++ def delete(self, condition: dict, index_name: str, memory_id: str) -> int:+ if not condition:+ return 0+ if self._has_empty_delete_list(condition):+ # An empty-list delete is a no-op. Building SQL from it could collapse+ # to a DELETE constrained only by memory_id and erase the memory.+ return 0+ table = self.physical_table(index_name)+ if not self._table_exists(table):+ return 0+ try:+ where_sql, where_params = self._build_where_clause(condition, memory_ids=[memory_id], force_memory_filter=True)+ if not where_sql:+ return 0+ # delete_message() often provides only message_id or source_id. Add+ # memory_id consistently to isolate rows in the shared tenant table.+ return self._execute_write(f"DELETE FROM {self.ddl.qualified_name(table)} WHERE {where_sql}", where_params)+ except Exception as exc:+ logger.error("GaussDB memory delete failed table=%s condition=%s error=%s", table, condition, exc)+ return 0++ def get(self, doc_id: str, index_name: str, memory_ids: list[str]) -> dict | None:+ if not doc_id:+ return None+ table = self.physical_table(index_name)+ if not self._table_exists(table):+ return None+ # Read all base columns plus every existing vector and empty-marker+ # column. _message_from_row() restores the valid content_embed dimension.+ columns = [*BASE_COLUMNS]+ columns.extend(self._vector_columns_for_select(table))+ sql = f"SELECT {', '.join(columns)} FROM {self.ddl.qualified_name(table)} WHERE id = %s"+ row, description = self._fetch_one_with_description(sql, [doc_id])+ if row is None:+ return None+ return self._message_from_row(self._row_to_dict(row, description))++ def search(+ self,+ select_fields: list[str],+ highlight_fields: list[str],+ condition: dict,+ match_expressions: list[MatchExpr],+ order_by: OrderByExpr,+ offset: int,+ limit: int,+ index_names: str | list[str],+ memory_ids: list[str],+ agg_fields: list[str] | None = None,+ rank_feature: dict | None = None,+ hide_forgotten: bool = True,+ **kwargs,+ ):+ tables = [self.physical_table(name) for name in normalize_index_names(index_names)]+ memory_ids = clean_list_values(memory_ids)+ if not tables or not memory_ids:+ return SearchResult(total=0, messages=[]), 0++ parsed = self._parse_match_expressions(match_expressions)+ has_match = bool(parsed["text_query"] or parsed["vector"])+ # For a multi-tenant fan-out, fetch offset+limit candidates per table and+ # merge, sort, and slice them in memory. A single-table query pushes+ # offset and limit directly into SQL.+ collection_limit = max(int(offset or 0), 0) + max(int(limit or 0), 0)+ if collection_limit <= 0:+ collection_limit = 10000++ result = SearchResult(total=0, messages=[])+ for table in tables:+ if not self._table_exists(table):+ continue+ sql, params = self._build_search_sql(+ table=table,+ select_fields=select_fields,+ highlight_fields=highlight_fields,+ condition=condition,+ parsed=parsed,+ order_by=order_by,+ offset=0 if len(tables) > 1 else max(int(offset or 0), 0),+ limit=collection_limit if len(tables) > 1 else max(int(limit or 0), 0),+ memory_ids=memory_ids,+ hide_forgotten=hide_forgotten,+ )+ rows, description = self._fetch_all_with_description(sql, params)+ table_total, messages = self._rows_to_messages(rows, description)+ result.total += table_total+ result.messages.extend(messages)++ result.messages = self._sort_messages(result.messages, order_by, has_match)+ if len(tables) > 1:+ # Apply final pagination only after globally sorting the merged+ # tenant results; concatenating locally paged results is incorrect.+ effective_offset = max(int(offset or 0), 0)+ effective_limit = max(int(limit or 0), 0)+ if effective_limit:+ result.messages = result.messages[effective_offset : effective_offset + effective_limit]+ return result, result.total++ def get_forgotten_messages(self, select_fields: list[str], index_name: str, memory_id: str, limit: int = 512):+ table = self.physical_table(index_name)+ if not self._table_exists(table):+ return None+ columns = self._select_columns(table, select_fields)+ sql = (+ f"SELECT {', '.join(columns)} "+ f"FROM {self.ddl.qualified_name(table)} "+ "WHERE memory_id = %s AND forget_at IS NOT NULL "+ # LIMIT is more reliable than FETCH FIRST for these queries on+ # A/ORA-compatible GaussDB and avoids dialect errors in maintenance.+ "ORDER BY forget_at ASC LIMIT %s"+ )+ rows, description = self._fetch_all_with_description(sql, [memory_id, int(limit)])+ _total, messages = self._rows_to_messages(rows, description)+ return SearchResult(total=len(messages), messages=messages)++ def get_missing_field_message(+ self,+ select_fields: list[str],+ index_name: str,+ memory_id: str,+ field_name: str,+ limit: int = 512,+ ):+ table = self.physical_table(index_name)+ if not self._table_exists(table):+ return None+ db_field = self.convert_field_name(field_name)+ self._validate_column(db_field)+ columns = self._select_columns(table, select_fields)+ sql = (+ f"SELECT {', '.join(columns)} "+ f"FROM {self.ddl.qualified_name(table)} "+ f"WHERE memory_id = %s AND {db_field} IS NULL "+ # Keep maintenance scans consistent with get_forgotten_messages.+ "ORDER BY valid_at ASC LIMIT %s"+ )+ rows, description = self._fetch_all_with_description(sql, [memory_id, int(limit)])+ _total, messages = self._rows_to_messages(rows, description)+ return SearchResult(total=len(messages), messages=messages)++ def get_total(self, res) -> int:+ if isinstance(res, tuple):+ return int(res[1] or 0)+ return int(getattr(res, "total", 0) or 0)++ def get_doc_ids(self, res) -> list[str]:+ if isinstance(res, tuple):+ res = res[0]+ return [row["id"] for row in getattr(res, "messages", []) if row.get("id")]++ def get_fields(self, res, fields: list[str]) -> dict[str, dict]:+ if isinstance(res, tuple):+ res = res[0]+ requested = set(fields or [])+ if not requested:+ return {}+ result: dict[str, dict] = {}+ for row in getattr(res, "messages", []) or []:+ message = self._message_from_row(row)+ doc_id = row.get("id") or message.get("id")+ if not doc_id:+ continue+ item = {}+ for field in fields:+ if field in message:+ item[field] = message[field]+ elif field in RESULT_FIELD_DEFAULTS:+ item[field] = RESULT_FIELD_DEFAULTS[field]+ if item:+ result[str(doc_id)] = item+ return result++ def get_highlight(self, res, keywords: list[str], field_name: str):+ if isinstance(res, tuple):+ res = res[0]+ return get_highlight_from_messages(+ getattr(res, "messages", None),+ keywords,+ field_name,+ is_english_fn=lambda s: is_english([s]),+ )++ def get_aggregation(self, res, field_name: str):+ if isinstance(res, tuple):+ res = res[0]+ return aggregate_by_field(getattr(res, "messages", None), field_name)++ def sql(self, sql: str, fetch_size: int = 128, format: str = "json"):+ # Memory Store exposes no arbitrary SQL. DocEngine SQL Q&A has its own+ # read-only validator, while Memory tables must never be accessed by+ # external SQL that can bypass the memory_id boundary.+ logger.warning("GaussDB Memory Store does not expose raw SQL execution.")+ return None++ def physical_table(self, index_name: str) -> str:+ return self.ddl.physical_table_name(index_name)++ @staticmethod+ def convert_field_name(field_name: str, use_tokenized_content: bool = False) -> str:+ # Normal reads and writes map content to the original-text column.+ # Full-text matching explicitly requests tokenized_content_ltks so+ # get_fields() does not return tokenized content.+ if field_name == "content" and use_tokenized_content:+ return "tokenized_content_ltks"+ return MEMORY_FIELD_MAP.get(field_name, field_name)++ def _message_to_row(self, message: dict, memory_id: str | None) -> tuple[dict, int]:+ content_embed = message.get("content_embed")+ if content_embed is None or len(content_embed) == 0:+ raise ValueError("content_embed is required for GaussDB memory insert")+ dim = self.ddl.validate_vector_dim(len(content_embed))+ target_memory_id = str(message.get("memory_id") or memory_id or "")+ if not target_memory_id:+ raise ValueError("memory_id is required")+ # Normalize each row by mapping application fields to physical columns,+ # converting empty user/agent/session IDs to NULL, and writing+ # content_embed to its dimension-specific vector column with *_empty set+ # to FALSE.+ row = {+ "id": str(message.get("id") or f"{target_memory_id}_{message['message_id']}"),+ "message_id": to_int_or_none(message.get("message_id")),+ "message_type_kwd": message.get("message_type"),+ "source_id": to_int_or_none(message.get("source_id")),+ "memory_id": target_memory_id,+ "user_id": none_if_empty(message.get("user_id")),+ "agent_id": none_if_empty(message.get("agent_id")),+ "session_id": none_if_empty(message.get("session_id")),+ "zone_id": to_int_or_none(message.get("zone_id", 0)) or 0,+ "valid_at": normalize_timestamp(message.get("valid_at")),+ "invalid_at": normalize_timestamp(message.get("invalid_at")),+ "forget_at": normalize_timestamp(message.get("forget_at")),+ "status_int": 1 if bool(message.get("status")) else 0,+ "content_ltks": message.get("content") or "",+ "tokenized_content_ltks": fine_grained_tokenize(tokenize(message.get("content") or "")),+ self.ddl.vector_column_name(dim): vector_literal(content_embed, dim),+ self.ddl.vector_empty_column_name(dim): False,+ }+ return row, dim++ def _message_from_row(self, row: dict) -> dict:+ # Restore backend-neutral field names and defaults. Empty invalid_at and+ # forget_at remain "-", while empty content and user_id remain "", so+ # callers do not need GaussDB-specific NULL handling.+ message = {+ "id": row.get("id"),+ "message_id": to_int_or_original(row.get("message_id")),+ "message_type": row.get("message_type_kwd"),+ "source_id": to_int_or_original(row.get("source_id")) if row.get("source_id") is not None else None,+ "memory_id": row.get("memory_id"),+ "user_id": row.get("user_id") or "",+ "agent_id": row.get("agent_id"),+ "session_id": row.get("session_id"),+ "zone_id": to_int_or_original(row.get("zone_id")) if row.get("zone_id") is not None else 0,+ "valid_at": format_timestamp(row.get("valid_at")),+ "invalid_at": format_timestamp(row.get("invalid_at")) or "-",+ "forget_at": format_timestamp(row.get("forget_at")) or "-",+ "status": bool(int(row.get("status_int") or 0)),+ "content": row.get("content_ltks") or "",+ "content_embed": self._content_embed_from_row(row),+ }+ if row.get("_score") is not None:+ message["_score"] = float(row.get("_score") or 0.0)+ return message++ def _content_embed_from_row(self, row: dict) -> list[float]:+ candidates = []+ for key, value in row.items():+ match = VECTOR_COLUMN_RE.fullmatch(str(key))+ if not match:+ continue+ dim = int(match.group("dim"))+ if row.get(self.ddl.vector_empty_column_name(dim)) is False:+ candidates.append((dim, parse_vector_value(value)))+ if not candidates:+ return []+ if len(candidates) > 1:+ # A message normally has one non-empty vector dimension. If legacy+ # data or a manual repair exposes several, log it and return the+ # highest dimension instead of failing the read.+ logger.warning("GaussDB memory row %s has multiple non-empty vector dimensions.", row.get("id"))+ return sorted(candidates, key=lambda item: item[0], reverse=True)[0][1]++ def _build_merge_sql(+ self,+ table: str,+ dim: int,+ existing_dims: list[int],+ rows: list[dict],+ ) -> tuple[str, list[list[Any]]]:+ vector_col = self.ddl.vector_column_name(dim)+ empty_col = self.ddl.vector_empty_column_name(dim)+ table_name = self.ddl.qualified_name(table)+ reset_other_dims = []+ for other_dim in existing_dims:+ if other_dim == dim:+ continue+ other_vector = self.ddl.vector_column_name(other_dim)+ other_empty = self.ddl.vector_empty_column_name(other_dim)+ reset_other_dims.append(f"{other_vector} = '{zero_vector_literal(other_dim)}'::floatvector")+ reset_other_dims.append(f"{other_empty} = TRUE")+ reset_clause = "".join(f",\n {assignment}" for assignment in reset_other_dims)+ # A/ORA-compatible GaussDB uses MERGE ... USING (SELECT ... FROM dual)+ # for upserts instead of PostgreSQL ON CONFLICT. Updating the active+ # dimension resets all others to zero vectors with *_empty=TRUE so each+ # row exposes only one real content_embed.+ sql = f"""+MERGE INTO {table_name} t+USING (+ SELECT+ %s AS id,+ %s AS message_id,+ %s AS message_type_kwd,+ %s AS source_id,+ %s AS memory_id,+ %s AS user_id,+ %s AS agent_id,+ %s AS session_id,+ %s AS zone_id,+ %s::timestamp AS valid_at,+ %s::timestamp AS invalid_at,+ %s::timestamp AS forget_at,+ %s AS status_int,+ %s AS content_ltks,+ %s AS tokenized_content_ltks,+ %s::floatvector({dim}) AS {vector_col}+ FROM dual+) s+ON (t.id = s.id)+WHEN MATCHED THEN UPDATE SET+ message_id = s.message_id,+ message_type_kwd = s.message_type_kwd,+ source_id = s.source_id,+ memory_id = s.memory_id,+ user_id = s.user_id,+ agent_id = s.agent_id,+ session_id = s.session_id,+ zone_id = s.zone_id,+ valid_at = s.valid_at,+ invalid_at = s.invalid_at,+ forget_at = s.forget_at,+ status_int = s.status_int,+ content_ltks = s.content_ltks,+ tokenized_content_ltks = s.tokenized_content_ltks,+ {vector_col} = s.{vector_col},+ {empty_col} = FALSE{reset_clause}+WHEN NOT MATCHED THEN INSERT (+ id, message_id, message_type_kwd, source_id, memory_id, user_id, agent_id,+ session_id, zone_id, valid_at, invalid_at, forget_at, status_int,+ content_ltks, tokenized_content_ltks, {vector_col}, {empty_col}+) VALUES (+ s.id, s.message_id, s.message_type_kwd, s.source_id, s.memory_id, s.user_id,+ s.agent_id, s.session_id, s.zone_id, s.valid_at, s.invalid_at, s.forget_at,+ s.status_int, s.content_ltks, s.tokenized_content_ltks, s.{vector_col}, FALSE+)"""+ params = [+ [+ row.get("id"),+ row.get("message_id"),+ row.get("message_type_kwd"),+ row.get("source_id"),+ row.get("memory_id"),+ row.get("user_id"),+ row.get("agent_id"),+ row.get("session_id"),+ row.get("zone_id"),+ row.get("valid_at"),+ row.get("invalid_at"),+ row.get("forget_at"),+ row.get("status_int"),+ row.get("content_ltks"),+ row.get("tokenized_content_ltks"),+ row.get(vector_col),+ ]+ for row in rows+ ]+ return sql.strip(), params++ def _build_update_set(self, table: str, index_name: str, memory_id: str, new_value: dict) -> tuple[str, list[Any]]:+ fragments: list[str] = []+ params: list[Any] = []+ for field, value in (new_value or {}).items():+ if field == "remove":+ # remove clears fields. Vector columns cannot hold NULL, so use a+ # zero vector with the corresponding empty marker; set ordinary+ # columns directly to NULL.+ remove_fields = [value] if isinstance(value, str) else list(value or [])+ for remove_field in remove_fields:+ db_field = self.convert_field_name(remove_field)+ if VECTOR_COLUMN_RE.fullmatch(db_field):+ dim = int(VECTOR_COLUMN_RE.fullmatch(db_field).group("dim"))+ self._ensure_vector_column_exists(table, dim)+ fragments.append(f"{db_field} = %s::floatvector({dim})")+ params.append(zero_vector_literal(dim))+ fragments.append(f"{self.ddl.vector_empty_column_name(dim)} = TRUE")+ else:+ self._validate_column(db_field)+ fragments.append(f"{db_field} = NULL")+ if db_field == "content_ltks":+ # content_ltks owns its derived full-text tokens.+ fragments.append("tokenized_content_ltks = NULL")+ continue+ if field == "content_embed":+ if value is None or len(value) == 0:+ continue+ dim = self.ddl.validate_vector_dim(len(value))+ self._ensure_vector_column_exists(table, dim)+ vector_col = self.ddl.vector_column_name(dim)+ # As with insert, retain only the active dimension as a real+ # vector and reset every other dimension to prevent one message+ # from matching more than once.+ fragments.append(f"{vector_col} = %s::floatvector({dim})")+ params.append(vector_literal(value, dim))+ fragments.append(f"{self.ddl.vector_empty_column_name(dim)} = FALSE")+ for other_dim in self._vector_dimensions(table):+ if other_dim == dim:+ continue+ fragments.append(f"{self.ddl.vector_column_name(other_dim)} = %s::floatvector({other_dim})")+ params.append(zero_vector_literal(other_dim))+ fragments.append(f"{self.ddl.vector_empty_column_name(other_dim)} = TRUE")+ continue++ db_field = self.convert_field_name(field)+ self._validate_column(db_field)+ if db_field == "content_ltks":+ # Refresh tokenized_content_ltks whenever the original content+ # changes so full-text tokens remain current.+ fragments.append("content_ltks = %s")+ params.append(value or "")+ fragments.append("tokenized_content_ltks = %s")+ params.append(fine_grained_tokenize(tokenize(value or "")))+ elif db_field in TIME_COLUMNS:+ fragments.append(f"{db_field} = %s::timestamp")+ params.append(normalize_timestamp(value))+ else:+ fragments.append(f"{db_field} = %s")+ if db_field in NUMERIC_COLUMNS:+ params.append(to_int_or_none(value))+ else:+ params.append(none_if_empty(value))+ return ", ".join(fragments), params++ def _build_search_sql(+ self,+ table: str,+ select_fields: list[str],+ highlight_fields: list[str],+ condition: dict,+ parsed: dict[str, Any],+ order_by: OrderByExpr,+ offset: int,+ limit: int,+ memory_ids: list[str],+ hide_forgotten: bool,+ ) -> tuple[str, list[Any]]:+ text_query = parsed["text_query"]+ vector = parsed["vector"]+ # Dispatch according to expression semantics: text plus vector uses+ # fusion, vector alone uses ANN ordering, text alone uses full-text+ # ordering, and no match expression falls back to filtering.+ if text_query and vector:+ return self._build_fusion_search_sql(table, select_fields, condition, parsed, offset, limit, memory_ids, hide_forgotten)+ if vector:+ return self._build_vector_search_sql(table, select_fields, condition, parsed, offset, limit, memory_ids, hide_forgotten)+ if text_query:+ return self._build_fulltext_search_sql(table, select_fields, highlight_fields, condition, parsed, offset, limit, memory_ids, hide_forgotten)+ return self._build_filter_search_sql(table, select_fields, condition, order_by, offset, limit, memory_ids, hide_forgotten)++ def _build_filter_search_sql(+ self,+ table: str,+ select_fields: list[str],+ condition: dict,+ order_by: OrderByExpr,+ offset: int,+ limit: int,+ memory_ids: list[str],+ hide_forgotten: bool,+ ) -> tuple[str, list[Any]]:+ columns = self._select_columns(table, select_fields)+ where_sql, where_params = self._build_where_clause(condition, memory_ids=memory_ids, hide_forgotten=hide_forgotten)+ order_sql = self._build_order_by(order_by) or "id ASC"+ # Filter-only lists, recent messages, and capacity accounting reuse+ # search(). COUNT(*) OVER() returns the total without another query.+ sql = f"SELECT {', '.join(columns)}, COUNT(*) OVER() AS __total FROM {self.ddl.qualified_name(table)}"+ if where_sql:+ sql += f" WHERE {where_sql}"+ sql += f" ORDER BY {order_sql}"+ params = [*where_params]+ if limit and int(limit) > 0:+ sql += " OFFSET %s LIMIT %s"+ params.extend([max(int(offset or 0), 0), int(limit)])+ return sql, params++ def _build_fulltext_search_sql(+ self,+ table: str,+ select_fields: list[str],+ highlight_fields: list[str],+ condition: dict,+ parsed: dict[str, Any],+ offset: int,+ limit: int,+ memory_ids: list[str],+ hide_forgotten: bool,+ ) -> tuple[str, list[Any]]:+ columns = self._select_columns(table, select_fields)+ text_query = parsed["text_query"]+ fts_expr = "to_tsvector('simple', tokenized_content_ltks)"+ # _parse_match_expressions already normalizes text_query with the+ # write-time tokenizer. This method only builds the GaussDB full-text+ # expression and ranking.+ score_expr = f"ts_rank({fts_expr}, plainto_tsquery('simple', %s))"+ match_expr = f"{fts_expr} @@ plainto_tsquery('simple', %s)"+ where_sql, where_params = self._build_where_clause(condition, memory_ids=memory_ids, hide_forgotten=hide_forgotten)+ where_parts = [part for part in (where_sql, match_expr) if part]+ sql = (+ f"SELECT {', '.join(columns)}, {score_expr} AS _score, COUNT(*) OVER() AS __total "+ f"FROM {self.ddl.qualified_name(table)} "+ f"WHERE {' AND '.join(where_parts)} "+ "ORDER BY _score DESC, valid_at DESC OFFSET %s LIMIT %s"+ )+ return sql, [text_query, *where_params, text_query, max(int(offset or 0), 0), effective_limit(limit, parsed["topn"])]++ def _build_vector_search_sql(+ self,+ table: str,+ select_fields: list[str],+ condition: dict,+ parsed: dict[str, Any],+ offset: int,+ limit: int,+ memory_ids: list[str],+ hide_forgotten: bool,+ ) -> tuple[str, list[Any]]:+ dim = parsed["vector_dim"]+ vector_col = self.ddl.vector_column_name(dim)+ empty_col = self.ddl.vector_empty_column_name(dim)+ columns = self._select_columns(table, select_fields)+ vector = parsed["vector"]+ threshold = float(parsed["similarity_threshold"] or 0.0)+ # `<+>` returns cosine distance, while RAGFlow supplies a similarity+ # threshold. Score and filter with 1 - distance, excluding zero-vector+ # placeholders through *_empty=FALSE.+ score_expr = f"1 - ({vector_col} <+> %s::floatvector({dim}))"+ distance_expr = f"{vector_col} <+> %s::floatvector({dim})"+ where_sql, where_params = self._build_where_clause(condition, memory_ids=memory_ids, hide_forgotten=hide_forgotten)+ where_parts = [part for part in (where_sql, f"{empty_col} = FALSE", f"1 - ({vector_col} <+> %s::floatvector({dim})) >= %s") if part]+ sql = (+ f"SELECT {', '.join(columns)}, {score_expr} AS _score, COUNT(*) OVER() AS __total "+ f"FROM {self.ddl.qualified_name(table)} "+ f"WHERE {' AND '.join(where_parts)} "+ f"ORDER BY {distance_expr} ASC OFFSET %s LIMIT %s"+ )+ return sql, [+ vector,+ *where_params,+ vector,+ threshold,+ vector,+ max(int(offset or 0), 0),+ effective_limit(limit, parsed["topn"]),+ ]++ def _build_fusion_search_sql(+ self,+ table: str,+ select_fields: list[str],+ condition: dict,+ parsed: dict[str, Any],+ offset: int,+ limit: int,+ memory_ids: list[str],+ hide_forgotten: bool,+ ) -> tuple[str, list[Any]]:+ dim = parsed["vector_dim"]+ vector_col = self.ddl.vector_column_name(dim)+ empty_col = self.ddl.vector_empty_column_name(dim)+ vector = parsed["vector"]+ threshold = float(parsed["similarity_threshold"] or 0.0)+ vector_weight = float(parsed["vector_weight"])+ text_weight = 1.0 - vector_weight+ text_query = parsed["text_query"]+ columns = self._select_columns(table, select_fields)+ inner_columns = unique_preserve_order([*columns, "tokenized_content_ltks", vector_col, empty_col, "valid_at"])+ fts_expr = "to_tsvector('simple', tokenized_content_ltks)"+ score_expr = f"ts_rank({fts_expr}, plainto_tsquery('simple', %s))"+ match_expr = f"{fts_expr} @@ plainto_tsquery('simple', %s)"+ where_sql, where_params = self._build_where_clause(condition, memory_ids=memory_ids, hide_forgotten=hide_forgotten)+ where_parts = [part for part in (where_sql, match_expr) if part]+ candidate_limit = max(int(parsed["topn"] or 0), effective_limit(limit, parsed["topn"]), 1)+ # Fusion first selects full-text candidates, then applies vector filtering+ # and a weighted sum within that set. This avoids computing vector scores+ # for rows without a text match and matches weighted_sum semantics.+ sql = (+ "WITH fulltext_results AS ("+ f" SELECT {', '.join(inner_columns)}, {score_expr} AS relevance "+ f"FROM {self.ddl.qualified_name(table)} "+ f"WHERE {' AND '.join(where_parts)} "+ "ORDER BY relevance DESC LIMIT %s"+ ") "+ f"SELECT {', '.join(columns)}, "+ f"relevance * %s + (1 - ({vector_col} <+> %s::floatvector({dim}))) * %s AS _score, "+ "COUNT(*) OVER() AS __total "+ "FROM fulltext_results "+ f"WHERE {empty_col} = FALSE "+ f"AND 1 - ({vector_col} <+> %s::floatvector({dim})) >= %s "+ "ORDER BY _score DESC, valid_at DESC OFFSET %s LIMIT %s"+ )+ return sql, [+ text_query,+ *where_params,+ text_query,+ candidate_limit,+ text_weight,+ vector,+ vector_weight,+ vector,+ threshold,+ max(int(offset or 0), 0),+ effective_limit(limit, parsed["topn"]),+ ]++ def _parse_match_expressions(self, match_expressions: list[MatchExpr] | None) -> dict[str, Any]:+ text_query = ""+ vector = None+ vector_dim = None+ topn = None+ similarity_threshold = 0.0+ vector_weight = 0.5++ for expr in match_expressions or []:+ if isinstance(expr, MatchTextExpr):+ # MsgTextQuery stores the original query in+ # extra_options["original_query"]. Prefer it and apply this+ # adapter's tokenizer instead of consuming matching_text syntax+ # generated for another backend.+ text_query = normalize_fulltext_query((expr.extra_options or {}).get("original_query") or expr.matching_text)+ topn = expr.topn if topn is None else min(topn, expr.topn)+ elif isinstance(expr, MatchDenseExpr):+ # vector_column_name may be q_{dim}_vec or the generic+ # content_embed field. Infer the latter's dimension from the+ # embedding data.+ vector_dim = parse_vector_dim(expr.vector_column_name) or len(expr.embedding_data)+ vector = vector_literal(expr.embedding_data, vector_dim)+ topn = expr.topn if topn is None else min(topn, expr.topn)+ similarity_threshold = float((expr.extra_options or {}).get("similarity", 0.0))+ elif isinstance(expr, FusionExpr):+ # FusionExpr orders weights as text_weight,vector_weight. Default+ # to 0.5 when the value is absent or incomplete.+ weights = (expr.fusion_params or {}).get("weights", "0.5,0.5")+ parts = str(weights).split(",")+ if len(parts) > 1:+ vector_weight = get_float(parts[1])+ topn = expr.topn if topn is None else min(topn, expr.topn)++ return {+ "text_query": text_query,+ "vector": vector,+ "vector_dim": vector_dim,+ "topn": topn,+ "similarity_threshold": similarity_threshold,+ "vector_weight": vector_weight,+ }++ def _select_columns(self, table: str, select_fields: list[str] | None) -> list[str]:+ requested = select_fields or []+ columns = ["id"]+ wants_content_embed = False+ for field in requested:+ if field in {"_score", "id"}:+ continue+ if field == "content_embed":+ # content_embed is virtual; a physical table may contain several+ # q_{dim}_vec and *_empty columns. Expand it from the actual table+ # columns after processing the requested fields.+ wants_content_embed = True+ continue+ db_field = self.convert_field_name(field)+ self._validate_column(db_field)+ if db_field not in columns:+ columns.append(db_field)+ if wants_content_embed:+ columns.extend(column for column in self._vector_columns_for_select(table) if column not in columns)+ return columns++ def _vector_columns_for_select(self, table: str) -> list[str]:+ columns: list[str] = []+ # Read each vector with its empty marker to distinguish a real zero+ # vector from a placeholder.+ for dim in self._vector_dimensions(table):+ columns.append(self.ddl.vector_column_name(dim))+ columns.append(self.ddl.vector_empty_column_name(dim))+ return columns++ def _build_where_clause(+ self,+ condition: dict | None,+ memory_ids: list[str] | None = None,+ hide_forgotten: bool = False,+ force_memory_filter: bool = False,+ ) -> tuple[str, list[Any]]:+ fragments: list[str] = []+ params: list[Any] = []++ memory_values = clean_list_values(memory_ids)+ if memory_values:+ # memory_id is the mandatory boundary within a shared tenant table.+ # Add it from search/update/delete arguments even when the caller's+ # condition omits it.+ fragments.append(f"memory_id IN ({', '.join(['%s'] * len(memory_values))})")+ params.extend(memory_values)+ elif force_memory_filter:+ # Return an always-false predicate when a write lacks a memory_id+ # boundary so it cannot affect the entire tenant table.+ return "1=0", []++ if hide_forgotten:+ fragments.append("forget_at IS NULL")++ for field, value in (condition or {}).items():+ if field == "memory_id" and memory_values:+ continue+ if field == "exists":+ # Startup maintenance and missing-field scans use exists and+ # must_not-exists semantics. Map and validate the column instead+ # of interpolating external input directly.+ db_field = self.convert_field_name(str(value))+ self._validate_column(db_field)+ fragments.append(f"{db_field} IS NOT NULL")+ continue+ if field == "must_not" and isinstance(value, dict) and "exists" in value:+ db_field = self.convert_field_name(str(value["exists"]))+ self._validate_column(db_field)+ fragments.append(f"{db_field} IS NULL")+ continue+ if value is None or value == "":+ # Treat empty filter values as absent, matching the ES/OB Memory+ # adapters. Real empty strings are not searchable business data.+ continue+ db_field = self.convert_field_name(field)+ self._validate_column(db_field)+ if isinstance(value, (list, tuple, set)):+ values = clean_list_values(value)+ if not values:+ continue+ fragments.append(f"{db_field} IN ({', '.join(['%s'] * len(values))})")+ params.extend(to_int_or_none(v) if db_field in NUMERIC_COLUMNS else v for v in values)+ continue+ fragments.append(f"{db_field} = %s")+ params.append(to_int_or_none(value) if db_field in NUMERIC_COLUMNS else value)+ return " AND ".join(fragments), params++ def _build_order_by(self, order_by: OrderByExpr | None) -> str:+ fields = getattr(order_by, "fields", None) or []+ parts = []+ for field, direction in fields:+ db_field = self.convert_field_name(field)+ # Apply the same allowlist to ORDER BY fields to prevent SQL injection.+ self._validate_column(db_field)+ parts.append(f"{db_field} {'DESC' if direction else 'ASC'}")+ return ", ".join(parts)++ def _validate_column(self, column: str) -> None:+ # SQL accepts only base columns or regex-protected dynamic vector columns.+ # This is the final guard before interpolating any dynamic field name.+ if column in BASE_COLUMN_SET or VECTOR_COLUMN_RE.fullmatch(column) or VECTOR_EMPTY_COLUMN_RE.fullmatch(column):+ return+ raise InvalidGaussDBObjectName(column)++ def _ensure_vector_column_exists(self, table: str, dim: int) -> None:+ dim = self.ddl.validate_vector_dim(dim)+ vector_col = self.ddl.vector_column_name(dim)+ empty_col = self.ddl.vector_empty_column_name(dim)+ columns_exist = self._column_exists(table, vector_col) and self._column_exists(table, empty_col)+ statements: list[str | tuple[str, list[Any]]] = [+ self.ddl.build_advisory_lock_sql(f"gaussdb_memory_vector_column:{table}:{dim}"),+ ]+ if not columns_exist:+ # Add columns only when absent. The empty-marker index uses IF NOT+ # EXISTS and can run on every ensure call to repair a partial state+ # where the columns exist but the index does not.+ statements.extend(self.ddl.build_vector_column_ddls(table, dim))+ statements.append(self.ddl.build_vector_empty_index_ddl(table, dim))+ self._execute_statements(statements)+ if not self._diskann_index_exists(table, dim):+ # Some GaussDB versions still perform costly gsdiskann initialization+ # for CREATE INDEX IF NOT EXISTS. Check pg_indexes first to avoid+ # changing work_mem on every write.+ self._create_diskann_index_with_retry(table, dim)++ def _create_diskann_index_with_retry(self, table: str, dim: int) -> None:+ ddl = self.ddl.build_diskann_index_ddl(table, dim)+ lock = self.ddl.build_advisory_lock_sql(f"gaussdb_memory_vector_index:{table}:{dim}")+ for work_mem in ("1GB", "2GB", "4GB"):+ try:+ # gsdiskann index creation depends on maintenance_work_mem. Retry+ # with bounded increases only for insufficient-memory errors;+ # propagate all other DDL failures.+ self._execute_statements([lock, f"SET LOCAL maintenance_work_mem = '{work_mem}'", ddl])+ return+ except Exception as exc:+ if not is_maintenance_work_mem_error(exc) or work_mem == "4GB":+ raise+ logger.warning("Retrying GaussDB gsdiskann index with larger maintenance_work_mem after: %s", exc)++ def _table_exists(self, table: str) -> bool:+ row = self._fetch_one(+ """+ SELECT 1+ FROM information_schema.tables+ WHERE table_schema = %s+ AND table_name = %s+ """,+ [self.schema, table],+ )+ return bool(row)++ def _column_exists(self, table: str, column: str) -> bool:+ row = self._fetch_one(+ """+ SELECT 1+ FROM information_schema.columns+ WHERE table_schema = %s+ AND table_name = %s+ AND column_name = %s+ """,+ [self.schema, table, column],+ )+ return bool(row)++ def _column_names(self, table: str) -> list[str]:+ rows = self._fetch_all(+ """+ SELECT column_name+ FROM information_schema.columns+ WHERE table_schema = %s+ AND table_name = %s+ """,+ [self.schema, table],+ )+ return [row_value(row, "column_name", 0) for row in rows or []]++ def _index_names(self, table: str) -> list[str]:+ rows = self._fetch_all(+ """+ SELECT indexname+ FROM pg_indexes+ WHERE schemaname = %s+ AND tablename = %s+ """,+ [self.schema, table],+ )+ return [row_value(row, "indexname", 0) for row in rows or []]++ def _diskann_index_exists(self, table: str, dim: int) -> bool:+ dim = self.ddl.validate_vector_dim(dim)+ vector_col = self.ddl.vector_column_name(dim)+ expected_index = self.ddl.index_name(table, f"{vector_col}_diskann")+ # A matching name is insufficient: a historical index with another type+ # cannot support ANN retrieval. Confirm `using gsdiskann` in indexdef.+ row = self._fetch_one(+ """+ SELECT indexdef+ FROM pg_indexes+ WHERE schemaname = %s+ AND tablename = %s+ AND indexname = %s+ """,+ [self.schema, table, expected_index],+ )+ if not row:+ return False+ return "using gsdiskann" in str(row_value(row, "indexdef", 0)).lower()++ def _vector_dimensions(self, table: str) -> list[int]:+ dims = []+ # Infer every embedding dimension used by the tenant table from its+ # q_{dim}_vec catalog columns. Writes, reads, and content_embed expansion+ # all depend on this list.+ for column in self._column_names(table):+ match = VECTOR_COLUMN_RE.fullmatch(str(column or ""))+ if match:+ dims.append(int(match.group("dim")))+ return sorted(set(dims))++ def _execute_statements(self, statements: Iterable[str | tuple[str, list[Any]]]) -> None:+ # Commit DDL and maintenance statements as one transaction. Roll back the+ # group on any failure to avoid columns without their required indexes.+ conn = self.pool.get_conn()+ cur = None+ try:+ cur = conn.cursor()+ for statement in statements:+ if isinstance(statement, tuple):+ cur.execute(statement[0], statement[1])+ else:+ cur.execute(statement)+ conn.commit()+ except Exception as exc:+ conn.rollback()+ raise classify_gaussdb_exception(exc) from exc+ finally:+ close_cursor(cur)+ self.pool.put_conn(conn)++ def _execute_write(self, sql: str, params: list[Any], many: bool = False) -> int:+ # Centralize write commits and rollbacks, and classify psycopg2 failures+ # as GaussDB connection, permission, or authentication errors for logs+ # and health checks.+ conn = self.pool.get_conn()+ cur = None+ try:+ cur = conn.cursor()+ if many:+ cur.executemany(sql, params)+ else:+ cur.execute(sql, params)+ conn.commit()+ return int(getattr(cur, "rowcount", 0) or 0)+ except Exception as exc:+ conn.rollback()+ raise classify_gaussdb_exception(exc) from exc+ finally:+ close_cursor(cur)+ self.pool.put_conn(conn)++ def _fetch_one(self, sql: str, params: list[Any]):+ row, _description = self._fetch_one_with_description(sql, params)+ return row++ def _fetch_one_with_description(self, sql: str, params: list[Any]):+ conn = self.pool.get_conn()+ cur = None+ try:+ cur = conn.cursor()+ cur.execute(sql, params)+ return cur.fetchone(), getattr(cur, "description", None) or []+ finally:+ close_cursor(cur)+ self.pool.put_conn(conn)++ def _fetch_all(self, sql: str, params: list[Any]):+ conn = self.pool.get_conn()+ cur = None+ try:+ cur = conn.cursor()+ cur.execute(sql, params)+ return cur.fetchall()+ finally:+ close_cursor(cur)+ self.pool.put_conn(conn)++ def _fetch_all_with_description(self, sql: str, params: list[Any]):+ conn = self.pool.get_conn()+ cur = None+ try:+ cur = conn.cursor()+ cur.execute(sql, params)+ return cur.fetchall(), getattr(cur, "description", None) or []+ finally:+ close_cursor(cur)+ self.pool.put_conn(conn)++ def _row_to_dict(self, row, description) -> dict:+ columns = [desc[0] for desc in description]+ return dict(zip(columns, row)) if not isinstance(row, dict) else dict(row)++ def _rows_to_messages(self, rows, description) -> tuple[int, list[dict]]:+ total = 0+ messages = []+ for row in rows or []:+ raw = self._row_to_dict(row, description)+ if raw.get("__total") is not None:+ # Search SQL appends the total to each row through COUNT(*)+ # OVER(). Do not expose the internal __total column to callers.+ total = int(raw.pop("__total") or 0)+ messages.append(raw)+ if total == 0:+ total = len(messages)+ return total, messages++ def _sort_messages(self, messages: list[dict], order_by: OrderByExpr | None, has_match: bool) -> list[dict]:+ if has_match:+ # The database guarantees ordering only within each tenant table.+ # Merge match results stably by descending score, descending valid_at,+ # and ascending id to mirror the SQL ORDER BY.+ return sorted(+ messages,+ key=lambda row: (+ -float(numeric_sort_value(row.get("_score"))),+ descending_timestamp_sort_value(row.get("valid_at")),+ str(row.get("id") or ""),+ ),+ )+ fields = getattr(order_by, "fields", None) or []+ if not fields:+ return sorted(messages, key=lambda row: str(row.get("id") or ""))+ sorted_messages = list(messages)+ for field, direction in reversed(fields):+ db_field = self.convert_field_name(field)+ # Preserve field types while merging: message_id/status sort+ # numerically and valid_at/forget_at sort chronologically rather than+ # as strings.+ sorted_messages.sort(key=lambda row, db_field=db_field: sortable_value(row.get(db_field), db_field), reverse=bool(direction))+ return sorted_messages++ @staticmethod+ def _has_empty_delete_list(condition: dict) -> bool:+ for key in ("message_id", "id"):+ if key in condition and isinstance(condition[key], (list, tuple, set)) and not clean_list_values(condition[key]):+ return True+ return False+++def normalize_index_names(index_names: str | list[str]) -> list[str]:+ # Callers may pass one index_name, a comma-separated string, or a list.+ # Normalize each form to logical names before resolving physical tables.+ # List elements are stringified, so None becomes "None" under this contract.+ if isinstance(index_names, str):+ return [name.strip() for name in index_names.split(",") if name.strip()]+ return [str(name).strip() for name in index_names or [] if str(name).strip()]+++def clean_list_values(values) -> list[Any]:+ # Remove None and empty strings to avoid IN () or treating an empty string as+ # business data. Wrap one string as a list for consistent ID handling.+ if values is None:+ return []+ if isinstance(values, (str, bytes)):+ values = [values]+ result = []+ for value in values:+ if value is None or value == "":+ continue+ result.append(value)+ return result+++def effective_limit(limit: int, topn: int | None = None) -> int:+ # topn comes from the match expression and limit from pagination. Honor the+ # smaller positive constraint, or use a conservative cap when neither exists.+ candidates = [int(value) for value in (limit, topn) if value and int(value) > 0]+ return min(candidates) if candidates else 10000+++def parse_vector_dim(column: str) -> int | None:+ match = VECTOR_COLUMN_RE.fullmatch(str(column or ""))+ return int(match.group("dim")) if match else None+++def vector_literal(value, dim: int) -> str:+ # Bind GaussDB floatvectors as "[1.0,2.0]" and cast them explicitly to+ # floatvector(dim) in SQL. Reformat parseable strings; leave an unparseable+ # string unchanged so the database reports the actual input error.+ if isinstance(value, str):+ parsed = parse_vector_value(value)+ if parsed:+ value = parsed+ else:+ return value+ if isinstance(value, np.ndarray):+ value = value.tolist()+ if not isinstance(value, (list, tuple)) or len(value) != int(dim):+ raise ValueError(f"vector dimension mismatch: expected {dim}, got {len(value) if hasattr(value, '__len__') else 'unknown'}")+ return "[" + ",".join(str(float(item)) for item in value) + "]"+++def parse_vector_value(value) -> list[float]:+ # Reads may return numpy arrays, lists, tuples, JSON arrays, or GaussDB+ # "[...]" text. Normalize them to a float list; return an empty list when no+ # usable content_embed can be decoded.+ if value is None:+ return []+ if isinstance(value, np.ndarray):+ return [float(item) for item in value.tolist()]+ if isinstance(value, (list, tuple)):+ return [float(item) for item in value]+ text = str(value).strip()+ if not text:+ return []+ try:+ parsed = json.loads(text)+ except json.JSONDecodeError:+ if text.startswith("[") and text.endswith("]"):+ body = text[1:-1].strip()+ if not body:+ return []+ return [float(item.strip()) for item in body.split(",")]+ return []+ if isinstance(parsed, list):+ return [float(item) for item in parsed]+ return []+++def zero_vector_literal(dim: int) -> str:+ # Use a zero vector as the floatvector placeholder; q_{dim}_vec_empty carries+ # the actual empty-value semantics.+ return "[" + ",".join(["0"] * int(dim)) + "]"+++def normalize_timestamp(value) -> str | None:+ # The Memory interface uses "-", "", and None for empty invalid_at/forget_at+ # values. Store each as SQL NULL.+ if value in (None, "", "-"):+ return None+ if isinstance(value, datetime):+ return value.strftime("%Y-%m-%d %H:%M:%S")+ return str(value)+++def format_timestamp(value) -> str | None:+ # Format values without adding "-". _message_from_row owns field defaults so+ # other internal paths can still distinguish a real None.+ if value in (None, "", "-"):+ return None+ if isinstance(value, datetime):+ return value.strftime("%Y-%m-%d %H:%M:%S")+ return str(value)+++def none_if_empty(value):+ # Optional user_id/agent_id/session_id fields do not need a stored empty+ # string. NULL better represents an omitted filter value.+ if value == "":+ return None+ return value+++def to_int_or_none(value):+ if value in (None, ""):+ return None+ if isinstance(value, bool):+ return int(value)+ if isinstance(value, (int, np.integer)):+ return int(value)+ if isinstance(value, Decimal):+ return int(value)+ return int(value)+++def to_int_or_original(value):+ if value is None:+ return None+ if isinstance(value, bool):+ return int(value)+ if isinstance(value, (int, np.integer)):+ return int(value)+ if isinstance(value, Decimal):+ return int(value)+ try:+ return int(value)+ except (TypeError, ValueError):+ return value+++def row_value(row, key: str, index: int):+ if isinstance(row, dict):+ return row.get(key)+ return row[index]+++def unique_preserve_order(values: list[str]) -> list[str]:+ result = []+ for value in values:+ if value not in result:+ result.append(value)+ return result+++def sortable_value(value, field_name: str | None = None):+ # Cross-table merges cannot sort every value as text because numeric IDs such+ # as 100 and 99 would be ordered incorrectly. Use the physical column type+ # for numeric and timestamp fields, and text ordering for the rest.+ if field_name in NUMERIC_COLUMNS:+ return numeric_sort_value(value)+ if field_name in TIME_COLUMNS:+ return timestamp_sort_value(value)+ if value is None:+ return ""+ if isinstance(value, datetime):+ return value.isoformat()+ return str(value)+++def numeric_sort_value(value) -> Decimal:+ # Decimal compares integers, numeric strings, and booleans consistently.+ # Map invalid or empty values to negative infinity so descending order places+ # them last.+ if value in (None, ""):+ return Decimal("-Infinity")+ if isinstance(value, bool):+ return Decimal(int(value))+ try:+ return Decimal(str(value))+ except (InvalidOperation, TypeError, ValueError):+ return Decimal("-Infinity")+++def timestamp_sort_value(value) -> float:+ # Accept datetime objects, ISO strings, and the project's common+ # "%Y-%m-%d %H:%M:%S" format. Treat empty timestamps as negative infinity+ # for merged ascending or descending sorts.+ if value in (None, "", "-"):+ return float("-inf")+ if isinstance(value, datetime):+ return value.timestamp()+ text = str(value).strip()+ if not text:+ return float("-inf")+ try:+ return datetime.fromisoformat(text).timestamp()+ except ValueError:+ pass+ try:+ return datetime.strptime(text, "%Y-%m-%d %H:%M:%S").timestamp()+ except ValueError:+ return float("-inf")+++def descending_timestamp_sort_value(value) -> float:+ # sorted() is ascending, so negate valid timestamps to obtain valid_at DESC.+ # Map empty values to positive infinity so they remain last.+ timestamp = timestamp_sort_value(value)+ if timestamp == float("-inf"):+ return float("inf")+ return -timestamp+++def is_maintenance_work_mem_error(exc: Exception) -> bool:+ # Retry gsdiskann creation only when maintenance_work_mem is insufficient.+ # Dialect, permission, and extension errors must propagate.+ text = str(exc).lower()+ return "maintenance_work_mem" in text and ("required" in text or "below" in text or "insufficient" in text)+++def close_cursor(cur) -> None:+ # Ignore cleanup failures so they do not hide the original SQL error.+ if cur is None:+ return+ try:+ cur.close()+ except Exception:+ passdiff --git a/rag/app/table.py b/rag/app/table.pyindex 114f62dd58..dc76316588 100644--- a/rag/app/table.py+++ b/rag/app/table.py@@ -547,10 +547,12 @@ def chunk(filename, binary=None, from_page=0, to_page=MAXIMUM_TASK_PAGE_NUMBER,clmns_map = [(py_clmns[i].lower() + fields_map[clmn_tys[i]], str(clmns[i]).replace("_", " ")) for i in range(len(clmns))]# field_map: only columns stored in chunk_data (metadata or both) — used for retrieval/SQLstored_indices = [i for i in range(len(clmns)) if column_roles.get(clmns[i], "both") in ("metadata", "both")]- if settings.DOC_ENGINE_INFINITY or settings.DOC_ENGINE_OCEANBASE or settings.DOC_ENGINE_SERENEDB:+ if settings.DOC_ENGINE_INFINITY or settings.DOC_ENGINE_OCEANBASE or settings.DOC_ENGINE_GAUSSDB or settings.DOC_ENGINE_SERENEDB:field_map = {py_clmns[i].lower(): str(clmns[i]).replace("_", " ") for i in stored_indices}else:field_map = {clmns_map[i][0]: clmns_map[i][1] for i in stored_indices}+ if settings.DOC_ENGINE_GAUSSDB:+ field_map = {py_clmns[i].lower(): str(clmns[i]) for i in stored_indices}logging.debug(f"Field map (sheet): {field_map}")sheet_specs.append({@@ -608,7 +610,7 @@ def chunk(filename, binary=None, from_page=0, to_page=MAXIMUM_TASK_PAGE_NUMBER,if role in ("indexing", "vectorize", "both"):text_fields.append((col_name, row[col_name]))if role in ("metadata", "both"):- if settings.DOC_ENGINE_INFINITY or settings.DOC_ENGINE_OCEANBASE or settings.DOC_ENGINE_SERENEDB:+ if settings.DOC_ENGINE_INFINITY or settings.DOC_ENGINE_OCEANBASE or settings.DOC_ENGINE_GAUSSDB or settings.DOC_ENGINE_SERENEDB:stored[str(col_name)] = row[col_name]else:fld = clmns_map[j][0]@@ -631,7 +633,7 @@ def chunk(filename, binary=None, from_page=0, to_page=MAXIMUM_TASK_PAGE_NUMBER,stored[f"{py_clmns[j].lower()}_raw"] = raw_sif not text_fields and not stored:continue- if settings.DOC_ENGINE_INFINITY or settings.DOC_ENGINE_OCEANBASE or settings.DOC_ENGINE_SERENEDB:+ if settings.DOC_ENGINE_INFINITY or settings.DOC_ENGINE_OCEANBASE or settings.DOC_ENGINE_GAUSSDB or settings.DOC_ENGINE_SERENEDB:if stored:d["chunk_data"] = storedelse:@@ -642,7 +644,7 @@ def chunk(filename, binary=None, from_page=0, to_page=MAXIMUM_TASK_PAGE_NUMBER,logger.debug(f"[TABLE_PARSER_DEBUG] Chunk content_with_weight length: {len(d.get('content_with_weight', '') or '')}")_cd = d.get("chunk_data")logger.debug(f"[TABLE_PARSER_DEBUG] Chunk chunk_data keys: {list(_cd.keys()) if isinstance(_cd, dict) else 'N/A'}")- if not (settings.DOC_ENGINE_INFINITY or settings.DOC_ENGINE_OCEANBASE or settings.DOC_ENGINE_SERENEDB):+ if not (settings.DOC_ENGINE_INFINITY or settings.DOC_ENGINE_OCEANBASE or settings.DOC_ENGINE_GAUSSDB or settings.DOC_ENGINE_SERENEDB):_extra = [k for k in d if k not in ("docnm_kwd", "title_tks", "content_with_weight", "content_ltks", "content_sm_ltks")]logger.debug(f"[TABLE_PARSER_DEBUG] Chunk ES extra field keys (sample): {_extra[:20]}")res.append(d)diff --git a/rag/nlp/search.py b/rag/nlp/search.pyindex 78b876afe7..fd73e2ea09 100644--- a/rag/nlp/search.py+++ b/rag/nlp/search.py@@ -207,7 +207,11 @@ class Dealer:if settings.DOC_ENGINE_OCEANBASE or settings.DOC_ENGINE_SERENEDB:src.append(f"q_{len(q_vec)}_vec")- fusionExpr = FusionExpr("weighted_sum", topk, {"weights": "0.001,1"})+ if settings.DOC_ENGINE_GAUSSDB:+ vector_weight = req.get("vector_similarity_weight", 0.3)+ fusionExpr = FusionExpr("weighted_sum", topk, {"weights": f"{1 - float(vector_weight)},{float(vector_weight)}"})+ else:+ fusionExpr = FusionExpr("weighted_sum", topk, {"weights": "0.001,1"})matchExprs = [matchText, matchDense, fusionExpr] if matchText else [matchDense]res = await thread_pool_exec(self.dataStore.search, src, highlightFields, filters, matchExprs, orderBy, offset, limit, idx_names, kb_ids, rank_feature=rank_feature)@@ -337,20 +341,14 @@ class Dealer:return res, seted- def _rank_feature_scores(self, query_rfea, search_res):- ## For rank feature(tag_fea) scores.+ def _tag_feature_scores(self, query_rfea, search_res):rank_fea = []- pageranks = []- for chunk_id in search_res.ids:- pageranks.append(search_res.field[chunk_id].get(PAGERANK_FLD, 0))- pageranks = np.array(pageranks, dtype=float)-if not query_rfea:- return np.array([0 for _ in range(len(search_res.ids))]) + pageranks+ return np.zeros(len(search_res.ids), dtype=float)q_denor = np.sqrt(np.sum([s * s for t, s in query_rfea.items() if t != PAGERANK_FLD]))if q_denor == 0:- return np.array([0 for _ in range(len(search_res.ids))]) + pageranks+ return np.zeros(len(search_res.ids), dtype=float)for i in search_res.ids:nor, denor = 0, 0if not search_res.field[i].get(TAG_FLD):@@ -368,7 +366,12 @@ class Dealer:rank_fea.append(0)else:rank_fea.append(nor / np.sqrt(denor) / q_denor)- return np.array(rank_fea) * 10.0 + pageranks+ return np.array(rank_fea, dtype=float) * 10.0++ def _rank_feature_scores(self, query_rfea, search_res):+ ## For rank feature(tag_fea) scores.+ pageranks = np.array([search_res.field[chunk_id].get(PAGERANK_FLD, 0) for chunk_id in search_res.ids], dtype=float)+ return self._tag_feature_scores(query_rfea, search_res) + pageranksasync def _knn_scores(self, sres: "Dealer.SearchResult", idx_names: str | list[str], kb_ids: list[str]) -> dict[str, float]:"""@@ -597,6 +600,7 @@ class Dealer:"topk": top,"similarity": similarity_threshold,"available_int": 1,+ "vector_similarity_weight": vector_similarity_weight,}if isinstance(must_not, dict) and must_not:req["must_not"] = must_not@@ -652,6 +656,14 @@ class Dealer:vector_similarity_weight,rank_feature=rank_feature,)+ elif settings.DOC_ENGINE_GAUSSDB:+ # GaussDB computes fusion and PageRank in SQL; tag features are+ # applied locally to the returned candidate window.+ sql_scores = [sres.field[id].get("_score", 0.0) for id in sres.ids]+ sql_scores = np.array([s if s is not None else 0.0 for s in sql_scores], dtype=np.float64)+ sim = sql_scores + self._tag_feature_scores(rank_feature, sres)+ tsim = sql_scores+ vsim = sql_scoreselse:# ES path: ask ES for the clean cosine score via a second# KNN-only call filtered by the candidate ids, then merge itdiff --git a/rag/utils/gaussdb_conn.py b/rag/utils/gaussdb_conn.pynew file mode 100644index 0000000000..0ce2958dec--- /dev/null+++ b/rag/utils/gaussdb_conn.py@@ -0,0 +1,1431 @@+#+# Copyright 2025 The InfiniFlow Authors. All Rights Reserved.+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+#+from __future__ import annotations++import json+import logging+import re+from math import isfinite+from typing import Any, Iterable++from psycopg2 import Error as PsycopgError, errorcodes+from pydantic import BaseModel++from common.constants import PAGERANK_FLD+from common.doc_store.doc_store_base import FusionExpr, MatchDenseExpr, MatchTextExpr+from common.doc_store.gaussdb_conn_base import (+ GaussDBConnectionBase,+ GaussDBSQLValidator,+ GaussDBSearchBuilder,+ extra_field_expr,+ validate_extra_field,+)++logger = logging.getLogger("ragflow.gaussdb_conn")++SQL_QUERY_TIMEOUT_MS = 30000+VECTOR_COLUMN_RE = re.compile(r"^q_(?P<dim>\d+)_vec$")+VECTOR_VALID_COLUMN_RE = re.compile(r"^q_(?P<dim>\d+)_vec_valid$")+++def _tokenize_query_terms(query_text: Any) -> list[str]:+ from rag.nlp import rag_tokenizer++ return [term for term in rag_tokenizer.tokenize(str(query_text or "")).split() if term]+++def _application_highlight(text: Any, keywords: list[str]) -> str | None:+ if text is None:+ return None+ simple_terms, ngram_terms = GaussDBSearchBuilder.split_text_query_terms(keywords)+ unique_terms: dict[str, str] = {}+ for term in [*simple_terms, *ngram_terms]:+ unique_terms.setdefault(term.casefold(), term)+ terms = sorted(unique_terms.values(), key=len, reverse=True)+ if not terms:+ return None+ pattern = re.compile("|".join(re.escape(term) for term in terms), re.IGNORECASE)+ highlighted, count = pattern.subn(lambda match: f"<em>{match.group(0)}</em>", str(text))+ return highlighted if count else None+++class GaussDBError(Exception):+ pass+++CHUNK_COLUMNS = (+ "id",+ "kb_id",+ "doc_id",+ "docnm_kwd",+ "doc_type_kwd",+ "title_tks",+ "title_sm_tks",+ "content_with_weight",+ "content_ltks",+ "content_sm_ltks",+ "important_kwd",+ "important_tks",+ "question_kwd",+ "question_tks",+ "tag_kwd",+ "tag_feas",+ "available_int",+ "pagerank_fea",+ "create_time",+ "create_timestamp_flt",+ "img_id",+ "position_int",+ "page_num_int",+ "top_int",+ "metadata",+ "chunk_data",+ "extra",+ "_order_id",+ "group_id",+ "mom_id",+ "knowledge_graph_kwd",+ "source_id",+ "entity_kwd",+ "entity_type_kwd",+ "from_entity_kwd",+ "to_entity_kwd",+ "weight_int",+ "weight_flt",+ "entities_kwd",+ "rank_flt",+ "n_hop_with_weight",+ "removed_kwd",+ "raptor_kwd",+ "raptor_layer_int",+)+CHUNK_COLUMN_SET = set(CHUNK_COLUMNS)+JSONB_COLUMNS = {+ "important_kwd",+ "question_kwd",+ "tag_kwd",+ "tag_feas",+ "position_int",+ "page_num_int",+ "top_int",+ "metadata",+ "chunk_data",+ "extra",+ "source_id",+ "entities_kwd",+}+JSONB_MULTI_VALUE_COLUMNS = {+ "important_kwd",+ "question_kwd",+ "tag_kwd",+ "source_id",+ "entities_kwd",+}+KEY_COLUMNS = {"id", "kb_id"}+DOC_META_COLUMNS = ("id", "kb_id", "meta_fields")+DOC_META_COLUMN_SET = set(DOC_META_COLUMNS)+DOC_META_JSONB_COLUMNS = {"meta_fields"}+DEFAULT_VALUES = {+ "available_int": 1,+ "removed_kwd": "N",+}+_UNDEFINED_TABLE_SQLSTATE = "42P01"+++def _is_undefined_table_error(exc: Exception) -> bool:+ return getattr(exc, "pgcode", None) == _UNDEFINED_TABLE_SQLSTATE or getattr(exc, "sqlstate", None) == _UNDEFINED_TABLE_SQLSTATE+++def _infer_chunk_vector_size(documents: list[dict]) -> int:+ dimensions = {int(match.group("dim")) for document in documents for column in document if (match := VECTOR_COLUMN_RE.fullmatch(str(column)))}+ if len(dimensions) != 1:+ raise ValueError("cannot infer a single GaussDB vector dimension from chunk documents")+ return dimensions.pop()+++class SearchResult(BaseModel):+ total: int+ chunks: list[dict]+++class GaussDBConnection(GaussDBConnectionBase):+ def db_type(self) -> str:+ return "gaussdb"++ def create_idx(self, index_name: str, dataset_id: str, vector_size: int, parser_id: str = None):+ # Keep the functional table even when the optional gsdiskann build+ # fails, while still propagating the index error to the caller.+ statements = [+ self.ddl.build_advisory_lock_sql(f"create_idx:{self.schema}:{index_name}"),+ self.ddl.build_chunk_table_ddl(index_name),+ ]+ statements.extend(self.ddl.build_regular_index_ddls(index_name))+ statements.append(self.ddl.build_fulltext_ugin_ddl(index_name))+ statements.append(self.ddl.build_ngram_fulltext_ugin_ddl(index_name))+ statements.extend(self.ddl.build_vector_column_ddls(index_name, vector_size))+ self._execute_statements(statements)+ self._create_diskann_index(index_name, vector_size)+ return True++ def _create_diskann_index(self, index_name: str, vector_size: int) -> None:+ statements = [+ self.ddl.build_advisory_lock_sql(f"create_vector_idx:{self.schema}:{index_name}:{vector_size}"),+ "SET LOCAL maintenance_work_mem = '1GB'",+ self.ddl.build_diskann_index_ddl(index_name, vector_size),+ ]+ self._execute_statements(statements)++ def create_doc_meta_idx(self, index_name: str):+ if not is_doc_meta_table(index_name):+ raise ValueError(f"invalid GaussDB document metadata table name: {index_name}")+ statements = [+ self.ddl.build_advisory_lock_sql(f"create_doc_meta_idx:{self.schema}:{index_name}"),+ *self.ddl.build_doc_meta_table_ddls(index_name),+ ]+ self._execute_statements(statements)+ return True++ def delete_idx(self, index_name: str, dataset_id: str | None):+ table = self.ddl.qualified_name(index_name)+ if dataset_id:+ try:+ self._execute_write(f"DELETE FROM {table} WHERE kb_id = %s", [dataset_id])+ except PsycopgError as exc:+ if exc.pgcode != errorcodes.UNDEFINED_TABLE:+ raise+ else:+ self._execute_write(f"DROP TABLE IF EXISTS {table}", [])++ def index_exist(self, index_name: str, dataset_id: str | None = None) -> bool:+ self.ddl.validate_identifier(index_name)+ row = self._fetch_one(+ """+ SELECT 1+ FROM information_schema.tables+ WHERE table_schema = %s+ AND table_name = %s+ LIMIT 1+ """,+ [self.schema, index_name],+ )+ return bool(row)++ def insert(self, documents: list[dict], index_name: str, knowledgebase_id: str = None, refresh: str | bool = "wait_for") -> list[str]:+ if not documents:+ return []+ document_ids = [str(document.get("id") or "") for document in documents]+ try:+ is_meta = is_doc_meta_table(index_name)+ if is_meta:+ sql, params = self._build_doc_meta_upsert(index_name, documents, knowledgebase_id)+ else:+ normalized, errors = self._normalize_chunk_rows(index_name, documents, knowledgebase_id)+ if errors:+ return [document_id for document_id in document_ids if document_id] or errors+ sql, params = self._build_chunk_upsert(index_name, normalized)+ try:+ self._execute_write(sql, params, many=True)+ except Exception as exc:+ if is_meta or not _is_undefined_table_error(exc) or self.index_exist(index_name, knowledgebase_id):+ raise+ vector_size = _infer_chunk_vector_size(documents)+ logger.info(+ "GaussDB chunk table missing during insert; creating table=%s vector_size=%s",+ index_name,+ vector_size,+ )+ self.create_idx(index_name, knowledgebase_id, vector_size)+ self._execute_write(sql, params, many=True)+ return []+ except Exception as exc:+ logger.error("GaussDB insert failed for table=%s ids=%s error=%s", index_name, document_ids, exc)+ return [document_id for document_id in document_ids if document_id] or [str(exc)]++ def get(self, chunk_id: str, index_name: str, knowledgebase_ids: list[str]) -> dict | None:+ if not chunk_id:+ return None+ is_meta = is_doc_meta_table(index_name)+ kb_ids = normalize_kb_ids(knowledgebase_ids)+ if not is_meta and not kb_ids:+ return None+ if not self.index_exist(index_name):+ return None+ table = self.ddl.qualified_name(index_name)+ params = [chunk_id]+ where = "id = %s"+ if kb_ids:+ placeholders = ", ".join(["%s"] * len(kb_ids))+ where += f" AND kb_id IN ({placeholders})"+ params.extend(kb_ids)+ if not is_meta and len(kb_ids) > 1:+ sql = f"SELECT * FROM {table} WHERE {where} ORDER BY kb_id ASC LIMIT 2"+ rows, description = self._fetch_all_with_description(sql, params)+ chunks = [self._row_to_chunk(row, description) for row in rows or []]+ self._check_cross_kb_duplicate_chunk_ids(chunks)+ return chunks[0] if chunks else None+ sql = f"SELECT * FROM {table} WHERE {where} LIMIT 1"+ row, description = self._fetch_one_with_description(sql, params)+ if row is None:+ return None+ return self._row_to_chunk(row, description)++ def search(+ self,+ select_fields: list[str],+ highlight_fields: list[str],+ condition: dict,+ match_expressions: list,+ order_by,+ offset: int,+ limit: int,+ index_names: str | list[str],+ knowledgebase_ids: list[str] | None = None,+ agg_fields: list[str] | None = None,+ rank_feature: dict | None = None,+ **kwargs,+ ) -> SearchResult:+ if knowledgebase_ids is None:+ knowledgebase_ids = kwargs.get("dataset_ids") or []+ tables = normalize_table_names(index_names)+ if not tables:+ return SearchResult(total=0, chunks=[])+ if len(tables) == 1 and is_doc_meta_table(tables[0]) and not match_expressions and not agg_fields:+ return self._search_doc_meta_table(+ select_fields=select_fields,+ condition=condition,+ order_by=order_by,+ offset=offset,+ limit=limit,+ index_name=tables[0],+ knowledgebase_ids=knowledgebase_ids,+ )+ if any(is_doc_meta_table(table) for table in tables):+ raise ValueError("document metadata tables cannot be mixed with chunk search")++ if agg_fields:+ return self._search_chunk_aggregation(+ tables=tables,+ condition=condition,+ knowledgebase_ids=knowledgebase_ids,+ agg_fields=agg_fields,+ )++ parsed = self._parse_match_expressions(match_expressions, rank_feature)+ scoped_condition = self._scoped_search_condition(condition, knowledgebase_ids)+ if len(tables) == 1:+ return self._search_chunk_table(+ table=tables[0],+ select_fields=select_fields,+ highlight_fields=highlight_fields,+ condition=scoped_condition,+ parsed=parsed,+ order_by=order_by,+ offset=offset,+ limit=limit,+ )++ effective_offset = max(int(offset or 0), 0)+ effective_limit = max(int(limit or 0), 0)+ if effective_limit == 0:+ effective_limit = 10000+ collection_limit = effective_offset + effective_limit+ total = 0+ chunks: list[dict] = []+ for table in tables:+ result = self._search_chunk_table(+ table=table,+ select_fields=select_fields,+ highlight_fields=highlight_fields,+ condition=scoped_condition,+ parsed=parsed,+ order_by=order_by,+ offset=0,+ limit=collection_limit,+ )+ total += result.total+ chunks.extend(result.chunks)+ chunks = self._sort_search_chunks(chunks, order_by, has_match=bool(parsed["keywords"] or parsed["vector"]))+ chunks = chunks[effective_offset : effective_offset + effective_limit]+ return SearchResult(total=total, chunks=chunks)++ def update(self, condition: dict, new_value: dict, index_name: str, knowledgebase_id: str) -> bool:+ if not condition or not new_value:+ return False+ if not (normalize_kb_id(knowledgebase_id) or normalize_kb_id(condition.get("kb_id"))):+ return False+ try:+ table = self.ddl.qualified_name(index_name)+ is_meta = is_doc_meta_table(index_name)+ dynamic_values = {validate_extra_field(key): value for key, value in new_value.items() if key not in {"add", "remove"} and is_dynamic_chunk_field(key, is_meta)}+ if dynamic_values:+ fixed_values = {key: value for key, value in new_value.items() if key not in dynamic_values}+ return self._update_with_dynamic_values(+ table=table,+ condition=condition,+ knowledgebase_id=knowledgebase_id,+ fixed_values=fixed_values,+ dynamic_values=dynamic_values,+ )+ set_sql, set_params = self._build_set_clause(new_value, is_meta=is_meta, condition=condition)+ where_sql, where_params = self._build_where_clause(condition, knowledgebase_id, is_meta=is_meta)+ sql = f"UPDATE {table} SET {set_sql} WHERE {where_sql}"+ self._execute_write(sql, [*set_params, *where_params])+ return True+ except Exception as exc:+ logger.error("GaussDB update failed for table=%s condition=%s error=%s", index_name, condition, exc)+ return False++ def _update_with_dynamic_values(+ self,+ *,+ table: str,+ condition: dict,+ knowledgebase_id: str,+ fixed_values: dict,+ dynamic_values: dict[str, Any],+ ) -> bool:+ supplied_extra = fixed_values.pop("extra", None)+ set_sql, set_params = self._build_set_clause(+ fixed_values,+ is_meta=False,+ condition=condition,+ )+ where_sql, where_params = self._build_where_clause(+ condition,+ knowledgebase_id,+ is_meta=False,+ )+ if not where_sql:+ return False++ conn = self.pool.get_conn()+ cur = None+ try:+ cur = conn.cursor()+ cur.execute(+ f"SELECT id, kb_id, extra FROM {table} WHERE {where_sql} FOR UPDATE",+ where_params,+ )+ rows = cur.fetchall()+ for row in rows or []:+ row_id, kb_id, stored_extra = row+ base_extra = supplied_extra if supplied_extra is not None else stored_extra+ merged_extra = dict(parse_json_dict(base_extra))+ merged_extra.update(dynamic_values)+ assignments = [part for part in (set_sql, "extra = %s::jsonb") if part]+ cur.execute(+ f"UPDATE {table} SET {', '.join(assignments)} WHERE kb_id = %s AND id = %s",+ [+ *set_params,+ json.dumps(merged_extra, ensure_ascii=False),+ kb_id,+ row_id,+ ],+ )+ conn.commit()+ return True+ except Exception:+ conn.rollback()+ raise+ finally:+ close_cursor(cur)+ self.pool.put_conn(conn)++ def delete(self, condition: dict, index_name: str, knowledgebase_id: str) -> int:+ if not condition:+ return 0+ if not (normalize_kb_id(knowledgebase_id) or normalize_kb_id(condition.get("kb_id"))):+ return 0+ try:+ table = self.ddl.qualified_name(index_name)+ where_sql, where_params = self._build_where_clause(+ condition,+ knowledgebase_id,+ is_meta=is_doc_meta_table(index_name),+ )+ return self._execute_write(f"DELETE FROM {table} WHERE {where_sql}", where_params)+ except Exception as exc:+ logger.error("GaussDB delete failed for table=%s condition=%s error=%s", index_name, condition, exc)+ return 0++ def fetch_metadata_doc_ids(+ self,+ index_name: str,+ kb_ids: list[str],+ sql_filter: str,+ filter_params: list[Any],+ limit: int,+ ) -> list[str]:+ if not is_doc_meta_table(index_name):+ raise ValueError(f"invalid GaussDB document metadata table name: {index_name}")+ scoped_kb_ids = normalize_kb_ids(kb_ids)+ if not scoped_kb_ids or not sql_filter:+ return []+ table = self.ddl.qualified_name(index_name)+ placeholders = ", ".join(["%s"] * len(scoped_kb_ids))+ effective_limit = limit if limit and limit > 0 else 10000+ sql = f"SELECT id FROM {table} WHERE kb_id IN ({placeholders}) AND ({sql_filter}) ORDER BY id LIMIT %s"+ rows = self._fetch_all(sql, [*scoped_kb_ids, *(filter_params or []), effective_limit])+ doc_ids = []+ for row in rows or []:+ if isinstance(row, dict):+ value = row.get("id")+ elif isinstance(row, (list, tuple)) and row:+ value = row[0]+ else:+ value = None+ if value is not None:+ doc_ids.append(str(value))+ return doc_ids++ def sql(self, sql: str, fetch_size: int = 128, format: str = "json"):+ self.logger.debug("GaussDBConnection.sql get sql: %s", sql)+ fetch_size = int(fetch_size or 128)+ validated = GaussDBSQLValidator.readonly_guard(+ default_limit=fetch_size,+ execution_schema=self.schema,+ ).validate_and_patch(sql)+ rows, description = self._fetch_all_with_description(+ validated.sql,+ [],+ statement_timeout_ms=SQL_QUERY_TIMEOUT_MS,+ )+ columns = [desc[0] for desc in description]++ def coerce_value(value):+ if isinstance(value, bytes):+ return value.decode("utf-8", errors="ignore")+ if isinstance(value, (dict, list)):+ return json.dumps(value, ensure_ascii=False)+ return value++ rows_list = [[coerce_value(value) for value in list(row)] for row in rows or []]+ result = {+ "columns": [{"name": column, "type": "text"} for column in columns],+ "rows": rows_list,+ }+ if format == "markdown":+ header = "|" + "|".join(columns) + "|" if columns else ""+ separator = "|" + "|".join(["---" for _ in columns]) + "|" if columns else ""+ body = "\n".join(["|" + "|".join([str(value) for value in row]) + "|" for row in rows_list])+ result["markdown"] = "\n".join([line for line in [header, separator, body] if line])+ return result++ def get_total(self, res) -> int:+ return int(res.total)++ def get_doc_ids(self, res) -> list[str]:+ return [row["id"] for row in res.chunks if "id" in row]++ @staticmethod+ def _check_cross_kb_duplicate_chunk_ids(rows) -> None:+ seen_kb_ids = {}+ for row in rows or []:+ chunk_id = row.get("id")+ if chunk_id is None:+ continue+ kb_id = row.get("kb_id")+ if kb_id is None:+ continue+ previous_kb_id = seen_kb_ids.get(chunk_id)+ if previous_kb_id is not None and previous_kb_id != kb_id:+ raise GaussDBError(f"cross-KB duplicate chunk id: {chunk_id}")+ seen_kb_ids[chunk_id] = kb_id++ def get_fields(self, res, fields: list[str]) -> dict[str, dict]:+ self._check_cross_kb_duplicate_chunk_ids(res.chunks)+ result = {}+ for row in res.chunks:+ chunk_id = row.get("id")+ if chunk_id is None:+ continue+ result[chunk_id] = {field: row[field] for field in fields if row.get(field) is not None}+ return result++ def get_highlight(self, res, keywords: list[str], field_name: str):+ self._check_cross_kb_duplicate_chunk_ids(res.chunks)+ highlights = {}+ for row in res.chunks:+ chunk_id = row.get("id")+ if row.get("_highlight_source") is not None:+ value = _application_highlight(row["_highlight_source"], keywords)+ else:+ value = row.get("_highlight") or row.get("highlight")+ if chunk_id and value:+ highlights[chunk_id] = value+ return highlights++ def get_aggregation(self, res, field_name: str):+ counts = {}+ result = []+ for row in res.chunks:+ if "value" in row and "count" in row:+ result.append((row["value"], row["count"]))+ continue+ value = row.get(field_name)+ if isinstance(value, list):+ for item in value:+ if isinstance(item, str) and item.strip():+ counts[item] = counts.get(item, 0) + 1+ elif isinstance(value, str) and value.strip():+ counts[value] = counts.get(value, 0) + 1+ return result or list(counts.items())++ def get_scores(self, res) -> dict[str, float]:+ self._check_cross_kb_duplicate_chunk_ids(res.chunks)+ return {row["id"]: float(row.get("_score") or 0.0) for row in res.chunks if row.get("id") is not None}++ def get_vector_dimensions(self, index_name: str) -> list[int]:+ self.ddl.validate_identifier(index_name)+ sql = """+ SELECT column_name+ FROM information_schema.columns+ WHERE table_schema = %s AND table_name = %s+ """+ rows = self._fetch_all(sql, [self.schema, index_name])+ dims = []+ for row in rows:+ col = row[0] if isinstance(row, (list, tuple)) else row.get("column_name")+ match = VECTOR_COLUMN_RE.fullmatch(str(col or ""))+ if match:+ dims.append(int(match.group("dim")))+ return sorted(set(dims))++ def adjust_chunk_pagerank_fea(+ self,+ chunk_id: str,+ index_name: str,+ dataset_id: str,+ delta: int,+ min_w: int = 0,+ max_w: int = 100,+ **kwargs,+ ) -> bool:+ if not chunk_id or not dataset_id:+ return False+ table = self.ddl.qualified_name(index_name)+ sql = f"UPDATE {table} SET pagerank_fea = GREATEST(%s, LEAST(%s, COALESCE(pagerank_fea, 0) + %s)) WHERE kb_id = %s AND id = %s"+ return self._execute_write(sql, [min_w, max_w, delta, dataset_id, chunk_id]) > 0++ def _normalize_chunk_rows(self, index_name: str, documents: list[dict], knowledgebase_id: str | None):+ errors = []+ rows = []+ batch_dims = sorted({int(VECTOR_COLUMN_RE.fullmatch(key).group("dim")) for document in documents for key in document if VECTOR_COLUMN_RE.fullmatch(key)})+ for document in documents:+ doc_id = str(document.get("id") or "")+ try:+ rows.append(self._normalize_chunk_row(index_name, document, knowledgebase_id, batch_dims))+ except Exception as exc:+ logger.error("GaussDB normalize chunk failed id=%s error=%s", doc_id, exc)+ errors.append(doc_id or str(exc))+ return rows, errors++ def _normalize_chunk_row(self, index_name: str, document: dict, knowledgebase_id: str | None, batch_dims: list[int]) -> dict:+ chunk_id = document.get("id")+ if not chunk_id:+ raise ValueError("chunk id is required")+ row = {"id": str(chunk_id)}++ kb_id = normalize_kb_id(document.get("kb_id")) or knowledgebase_id+ if not kb_id:+ raise ValueError("kb_id is required")+ if knowledgebase_id and kb_id != knowledgebase_id:+ raise ValueError(f"kb_id {kb_id} does not match dataset_id {knowledgebase_id}")+ row["kb_id"] = kb_id++ extra = {}+ for key, value in document.items():+ if key in {"id", "kb_id"} or VECTOR_COLUMN_RE.fullmatch(key) or VECTOR_VALID_COLUMN_RE.fullmatch(key):+ continue+ if key not in CHUNK_COLUMN_SET:+ extra[key] = value+ continue+ row[key] = normalize_column_value(key, value)++ if extra:+ existing_extra = row.get("extra")+ if isinstance(existing_extra, str):+ try:+ existing_extra = json.loads(existing_extra)+ except json.JSONDecodeError:+ existing_extra = {}+ if not isinstance(existing_extra, dict):+ existing_extra = {}+ existing_extra.update(extra)+ row["extra"] = json.dumps(existing_extra, ensure_ascii=False)++ metadata = row.get("metadata")+ metadata_dict = parse_json_dict(metadata)+ if metadata_dict:+ if metadata_dict.get("_group_id"):+ row["group_id"] = metadata_dict["_group_id"]+ elif row.get("doc_id"):+ row.setdefault("group_id", row.get("doc_id"))+ if metadata_dict.get("_title"):+ row["docnm_kwd"] = metadata_dict["_title"]+ elif row.get("doc_id"):+ row.setdefault("group_id", row.get("doc_id"))++ vector_columns = [key for key in document if VECTOR_COLUMN_RE.fullmatch(key)]+ if vector_columns:+ for vector_col in vector_columns:+ dim = int(VECTOR_COLUMN_RE.fullmatch(vector_col).group("dim"))+ row[vector_col] = vector_literal(document[vector_col], dim)+ row[self.ddl.vector_valid_column_name(dim)] = True+ else:+ dims = batch_dims or self.get_vector_dimensions(index_name)+ if len(dims) != 1:+ raise ValueError("cannot infer GaussDB vector dimension")+ dim = dims[0]+ row[self.ddl.vector_column_name(dim)] = zero_vector_literal(dim)+ row[self.ddl.vector_valid_column_name(dim)] = False++ for column, default in DEFAULT_VALUES.items():+ row.setdefault(column, default)+ return row++ def _build_chunk_upsert(self, index_name: str, rows: list[dict]) -> tuple[str, list[list[Any]]]:+ if not rows:+ raise ValueError("rows are required")+ columns = ordered_columns(rows)+ table = self.ddl.qualified_name(index_name)+ placeholders = ", ".join(self._placeholder(column) for column in columns)+ update_columns = [column for column in columns if column not in {"id", "kb_id"}]+ update_clause = ", ".join(f"{column} = VALUES({column})" for column in update_columns)+ sql = f"INSERT INTO {table} ({', '.join(columns)}) VALUES ({placeholders}) ON DUPLICATE KEY UPDATE {update_clause}"+ params = [[row.get(column) for column in columns] for row in rows]+ return sql, params++ def _build_doc_meta_upsert(+ self,+ index_name: str,+ documents: list[dict],+ knowledgebase_id: str | None,+ ) -> tuple[str, list[list[Any]]]:+ table = self.ddl.qualified_name(index_name)+ params = []+ for document in documents:+ doc_id = document.get("id")+ kb_id = normalize_kb_id(document.get("kb_id")) or knowledgebase_id+ if not doc_id or not kb_id:+ raise ValueError("doc metadata id and kb_id are required")+ if knowledgebase_id and kb_id != knowledgebase_id:+ raise ValueError(f"kb_id {kb_id} does not match dataset_id {knowledgebase_id}")+ params.append([str(doc_id), kb_id, json.dumps(document.get("meta_fields") or {}, ensure_ascii=False)])+ sql = f"INSERT INTO {table} (id, kb_id, meta_fields) VALUES (%s, %s, %s::jsonb) ON DUPLICATE KEY UPDATE meta_fields = VALUES(meta_fields)"+ return sql, params++ def _search_doc_meta_table(+ self,+ select_fields: list[str],+ condition: dict,+ order_by,+ offset: int,+ limit: int,+ index_name: str,+ knowledgebase_ids: list[str],+ ) -> SearchResult:+ effective_condition = dict(condition or {})+ kb_ids = normalize_kb_ids(knowledgebase_ids)+ condition_kb_ids = normalize_kb_ids(effective_condition.get("kb_id"))+ if kb_ids:+ if condition_kb_ids and not set(condition_kb_ids).issubset(set(kb_ids)):+ raise ValueError("condition kb_id must stay within knowledgebase_ids")+ if not condition_kb_ids:+ effective_condition["kb_id"] = kb_ids+ if not normalize_kb_ids(effective_condition.get("kb_id")):+ raise ValueError("GaussDB document metadata search requires a kb_id boundary")++ table = self.ddl.qualified_name(index_name)+ columns = select_doc_meta_columns(select_fields)+ where_sql, where_params = self._build_where_clause(effective_condition, None, is_meta=True)+ sql = f"SELECT {', '.join(columns)}, COUNT(*) OVER() AS __total FROM {table}"+ if where_sql:+ sql += f" WHERE {where_sql}"+ order_sql = build_doc_meta_order_by(order_by)+ if order_sql:+ sql += f" ORDER BY {order_sql}"+ effective_limit = limit if limit and limit > 0 else 10000+ effective_offset = max(int(offset or 0), 0)+ sql += " LIMIT %s OFFSET %s"+ rows, description = self._fetch_all_with_description(sql, [*where_params, effective_limit, effective_offset])+ total = 0+ chunks = []+ for row in rows or []:+ chunk = self._row_to_chunk(row, description)+ total = int(chunk.pop("__total", total or 0) or 0)+ chunks.append(chunk)+ if not chunks and effective_offset:+ count_sql = f"SELECT COUNT(*) FROM {table}"+ if where_sql:+ count_sql += f" WHERE {where_sql}"+ row = self._fetch_one(count_sql, where_params)+ total = int(row[0]) if row else 0+ return SearchResult(total=total, chunks=chunks)++ def _search_chunk_aggregation(+ self,+ tables: list[str],+ condition: dict,+ knowledgebase_ids: list[str] | None,+ agg_fields: list[str],+ ) -> SearchResult:+ if len(agg_fields) != 1:+ raise ValueError("GaussDB search supports one aggregation field per request")+ counts: dict[Any, int] = {}+ scoped_condition = self._scoped_search_condition(condition, knowledgebase_ids)+ for table in tables:+ sql, params = self._search_builder().build_aggregation_sql(+ table=table,+ field_name=agg_fields[0],+ condition=scoped_condition,+ )+ rows, description = self._fetch_all_with_description(sql, params)+ for row in rows or []:+ chunk = self._row_to_chunk(row, description)+ if chunk.get("count") is not None:+ chunk["count"] = int(chunk["count"])+ value = chunk.get("value")+ if value is not None:+ counts[value] = counts.get(value, 0) + int(chunk.get("count") or 0)+ chunks = [{"value": value, "count": count} for value, count in sorted(counts.items(), key=lambda item: (-item[1], str(item[0])))][:1000]+ return SearchResult(total=len(chunks), chunks=chunks)++ def _search_chunk_table(+ self,+ table: str,+ select_fields: list[str],+ highlight_fields: list[str],+ condition: dict,+ parsed: dict[str, Any],+ order_by,+ offset: int,+ limit: int,+ ) -> SearchResult:+ sql, params = self._search_builder().build_search_sql(+ table=table,+ select_fields=select_fields,+ condition=condition,+ keywords=parsed["keywords"],+ vector=parsed["vector"],+ vector_dim=parsed["vector_dim"],+ vector_weight=parsed["vector_weight"],+ similarity_threshold=parsed["similarity_threshold"],+ topn=parsed["topn"],+ offset=offset,+ limit=limit,+ highlight_fields=highlight_fields,+ order_by=order_by,+ pagerank_weight=parsed["pagerank_weight"],+ )+ rows, description = self._fetch_all_with_description(sql, params)+ result = self._rows_to_search_result(rows, description)+ if result.chunks or not max(int(offset or 0), 0):+ return result++ count_sql, count_params = self._search_builder().build_search_sql(+ table=table,+ select_fields=["id"],+ condition=condition,+ keywords=parsed["keywords"],+ vector=parsed["vector"],+ vector_dim=parsed["vector_dim"],+ vector_weight=parsed["vector_weight"],+ similarity_threshold=parsed["similarity_threshold"],+ topn=parsed["topn"],+ offset=0,+ limit=1,+ highlight_fields=[],+ order_by=order_by,+ pagerank_weight=parsed["pagerank_weight"],+ )+ count_rows, count_description = self._fetch_all_with_description(count_sql, count_params)+ count_result = self._rows_to_search_result(count_rows, count_description)+ return SearchResult(total=count_result.total, chunks=[])++ def _rows_to_search_result(self, rows, description) -> SearchResult:+ total = 0+ chunks = []+ for row in rows or []:+ chunk = self._row_to_chunk(row, description)+ total = int(chunk.pop("__total", total or 0) or 0)+ chunks.append(chunk)+ return SearchResult(total=total, chunks=chunks)++ def _sort_search_chunks(self, chunks: list[dict], order_by, has_match: bool) -> list[dict]:+ fields = getattr(order_by, "fields", None) or []+ if not fields:+ if has_match:+ return sorted(+ chunks,+ key=lambda row: (+ -(float(row.get("_score") or 0.0)),+ str(row.get("kb_id") or ""),+ str(row.get("id") or ""),+ ),+ )+ return sorted(chunks, key=lambda row: (str(row.get("kb_id") or ""), str(row.get("id") or "")))+ sorted_chunks = list(chunks)+ for field, direction in reversed(fields):+ sorted_chunks.sort(+ key=lambda row, field=field: sortable_search_value(row.get(field), field),+ reverse=bool(direction),+ )+ return sorted_chunks++ def _parse_match_expressions(+ self,+ match_expressions: list,+ rank_feature: dict | None,+ ) -> dict[str, Any]:+ keywords: list[str] = []+ vector = None+ vector_dim = None+ topn = None+ vector_weight = None+ similarity_threshold = None+ pagerank_weight = 10.0++ for expr in match_expressions or []:+ if isinstance(expr, MatchTextExpr):+ query_text = (expr.extra_options or {}).get("original_query") or expr.matching_text or ""+ keywords = _tokenize_query_terms(query_text)+ topn = expr.topn if topn is None else min(topn, expr.topn)+ elif isinstance(expr, MatchDenseExpr):+ if expr.embedding_data_type != "float":+ raise ValueError(f"unsupported GaussDB vector data type: {expr.embedding_data_type}")+ vector = list(expr.embedding_data)+ vector_dim = parse_vector_dim(expr.vector_column_name) or len(vector)+ topn = expr.topn if topn is None else min(topn, expr.topn)+ similarity_threshold = float((expr.extra_options or {}).get("similarity", 0.0))+ elif isinstance(expr, FusionExpr):+ vector_weight = parse_fusion_vector_weight(expr)+ topn = expr.topn if topn is None else min(topn, expr.topn)++ if rank_feature and rank_feature.get(PAGERANK_FLD) is not None:+ pagerank_weight = float(rank_feature[PAGERANK_FLD])++ if vector is None:+ vector_weight = 0.0+ elif not keywords:+ vector_weight = 1.0 if vector_weight is None else float(vector_weight)+ else:+ vector_weight = 0.5 if vector_weight is None else float(vector_weight)++ return {+ "keywords": keywords,+ "vector": vector,+ "vector_dim": vector_dim,+ "vector_weight": vector_weight,+ "similarity_threshold": similarity_threshold,+ "topn": topn,+ "pagerank_weight": pagerank_weight,+ }++ def _scoped_search_condition(self, condition: dict | None, knowledgebase_ids: list[str] | None) -> dict:+ effective = dict(condition or {})+ kb_ids = normalize_kb_ids(knowledgebase_ids)+ if kb_ids:+ condition_kb_ids = normalize_kb_ids(effective.get("kb_id"))+ if condition_kb_ids:+ if not set(condition_kb_ids).issubset(set(kb_ids)):+ raise ValueError("condition kb_id must stay within knowledgebase_ids")+ else:+ effective["kb_id"] = kb_ids+ if "doc_ids" in effective and "doc_id" not in effective:+ effective["doc_id"] = effective.pop("doc_ids")+ if isinstance(effective.get("doc_id"), (list, tuple, set)) and not effective["doc_id"]:+ effective.pop("doc_id")+ if not normalize_kb_ids(effective.get("kb_id")):+ raise ValueError("GaussDB chunk search requires a kb_id boundary")+ return effective++ def _search_builder(self) -> GaussDBSearchBuilder:+ builder = getattr(self, "search_builder", None)+ if builder is None:+ builder = GaussDBSearchBuilder(schema=self.schema)+ self.search_builder = builder+ return builder++ def _build_set_clause(self, new_value: dict, is_meta: bool, condition: dict) -> tuple[str, list[Any]]:+ fragments = []+ params = []+ allowed_columns = DOC_META_COLUMN_SET if is_meta else CHUNK_COLUMN_SET+ jsonb_columns = DOC_META_JSONB_COLUMNS if is_meta else JSONB_COLUMNS+ jsonb_operations: dict[str, dict[str, Any]] = {}+ updated_vector_dims = {int(match.group("dim")) for key in new_value if not is_meta and (match := VECTOR_COLUMN_RE.fullmatch(str(key)))}+ for operation, value in new_value.items():+ if operation in {"remove", "add"} and isinstance(value, dict):+ for column, item in value.items():+ jsonb_operations.setdefault(column, {})[operation] = item+ emitted_jsonb_columns = set()++ def append_jsonb_update(column: str) -> None:+ if column in emitted_jsonb_columns:+ return+ emitted_jsonb_columns.add(column)+ operations = jsonb_operations[column]+ remove_expr = None+ remove_param = None+ if "remove" in operations:+ escaped_item = str(operations["remove"]).replace("\\", "\\\\").replace("%", "\\%").replace("_", "\\_")+ remove_expr = f"json_remove({column}, json_unquote(json_search({column}, 'one', jsonb_array_element_text(%s::jsonb, 0))))::jsonb"+ remove_param = json.dumps([escaped_item], ensure_ascii=False)+ if "add" in operations:+ base_expr = f"COALESCE({remove_expr}, {column}, '[]'::jsonb)" if remove_expr is not None else f"COALESCE({column}, '[]'::jsonb)"+ expression = (+ f"(SELECT CASE WHEN candidate @> %s::jsonb THEN candidate ELSE json_array_append(candidate, '$', %s::jsonb)::jsonb END FROM (SELECT {base_expr} AS candidate) AS jsonb_update)"+ )+ params.extend(+ [+ json.dumps([operations["add"]], ensure_ascii=False),+ json.dumps(operations["add"], ensure_ascii=False),+ ]+ )+ if remove_param is not None:+ params.append(remove_param)+ else:+ expression = f"COALESCE({remove_expr}, {column})"+ params.append(remove_param)+ fragments.append(f"{column} = {expression}")++ for key, value in new_value.items():+ if key == "remove":+ if isinstance(value, str):+ if value in KEY_COLUMNS:+ raise ValueError(f"key column cannot be updated: {value}")+ if value not in allowed_columns:+ raise ValueError(f"unsupported remove target: {value}")+ fragments.append(f"{value} = NULL")+ continue+ if not isinstance(value, dict):+ raise ValueError(f"unsupported remove target: {value}")+ for column, item in value.items():+ if column not in JSONB_MULTI_VALUE_COLUMNS:+ raise ValueError(f"unsupported JSONB remove target: {column}")+ append_jsonb_update(column)+ continue+ if key == "add":+ if not isinstance(value, dict):+ raise ValueError(f"unsupported add target: {value}")+ for column, item in value.items():+ if column not in JSONB_MULTI_VALUE_COLUMNS:+ raise ValueError(f"unsupported JSONB add target: {column}")+ append_jsonb_update(column)+ continue+ if key == "metadata" and not is_meta and isinstance(value, dict):+ fragments.append("metadata = %s::jsonb")+ params.append(json.dumps(value, ensure_ascii=False))+ if value.get("_group_id"):+ fragments.append("group_id = %s")+ params.append(value["_group_id"])+ if value.get("_title"):+ fragments.append("docnm_kwd = %s")+ params.append(value["_title"])+ continue+ if key in KEY_COLUMNS:+ if key == "id" and isinstance(value, str) and value == condition.get("id"):+ continue+ raise ValueError(f"key column cannot be updated: {key}")+ if vector_match := VECTOR_COLUMN_RE.fullmatch(key):+ dim = int(vector_match.group("dim"))+ fragments.append(f"{key} = {self._placeholder(key)}")+ params.append(vector_literal(value, dim))+ fragments.append(f"{self.ddl.vector_valid_column_name(dim)} = TRUE")+ continue+ if valid_match := VECTOR_VALID_COLUMN_RE.fullmatch(str(key)):+ dim = self.ddl.validate_vector_dim(int(valid_match.group("dim")))+ if dim in updated_vector_dims:+ continue+ fragments.append(f"{key} = %s")+ params.append(bool(value))+ continue+ if key not in allowed_columns:+ raise ValueError(f"unknown column for update: {key}")+ if key in jsonb_columns:+ fragments.append(f"{key} = %s::jsonb")+ params.append(json.dumps(value, ensure_ascii=False))+ else:+ fragments.append(f"{key} = %s")+ params.append(value)+ return ", ".join(fragments), params++ def _build_where_clause(self, condition: dict, knowledgebase_id: str | None, is_meta: bool) -> tuple[str, list[Any]]:+ effective = dict(condition or {})+ if knowledgebase_id:+ existing_kb = normalize_kb_id(effective.get("kb_id"))+ if existing_kb and existing_kb != knowledgebase_id:+ raise ValueError(f"condition kb_id {existing_kb} does not match dataset_id {knowledgebase_id}")+ effective["kb_id"] = knowledgebase_id++ fragments = []+ params = []++ def column_expression(column: str) -> str:+ validate_filter_column(column, is_meta)+ return filter_column_expr(column, is_meta)++ for key, value in effective.items():+ if key == "exists":+ column = column_expression(value)+ if is_jsonb_dynamic_chunk_field(value, is_meta):+ fragments.append(f"({column} IS NOT NULL AND {column} <> 'null'::jsonb)")+ else:+ fragments.append(f"{column} IS NOT NULL")+ continue+ if key == "must_not" and isinstance(value, dict) and "exists" in value:+ column = column_expression(value["exists"])+ if is_jsonb_dynamic_chunk_field(value["exists"], is_meta):+ fragments.append(f"({column} IS NULL OR {column} = 'null'::jsonb)")+ else:+ fragments.append(f"{column} IS NULL")+ continue+ column = column_expression(key)+ if is_jsonb_dynamic_chunk_field(key, is_meta):+ values = list(value) if isinstance(value, (list, tuple, set)) else [value]+ if not values:+ raise ValueError(f"empty list condition for {key}")+ predicates = []+ for item in values:+ predicates.append(f"({column} = %s::jsonb OR {column} @> %s::jsonb)")+ params.extend(+ [+ json.dumps(item, ensure_ascii=False),+ json.dumps([item], ensure_ascii=False),+ ]+ )+ fragments.append("(" + " OR ".join(predicates) + ")")+ continue+ if key in JSONB_MULTI_VALUE_COLUMNS:+ values = list(value) if isinstance(value, (list, tuple, set)) else [value]+ if not values:+ raise ValueError(f"empty list condition for {key}")+ fragments.append("(" + " OR ".join([f"{column} @> %s::jsonb"] * len(values)) + ")")+ params.extend(json.dumps([item], ensure_ascii=False) for item in values)+ elif isinstance(value, (list, tuple, set)):+ values = list(value)+ if not values:+ raise ValueError(f"empty list condition for {key}")+ fragments.append(f"{column} IN ({', '.join(['%s'] * len(values))})")+ params.extend(values)+ else:+ fragments.append(f"{column} = %s")+ params.append(value)+ return " AND ".join(fragments), params++ def _placeholder(self, column: str) -> str:+ match = VECTOR_COLUMN_RE.fullmatch(column)+ if match:+ return f"%s::floatvector({match.group('dim')})"+ if column in JSONB_COLUMNS or column in DOC_META_JSONB_COLUMNS:+ return "%s::jsonb"+ return "%s"++ def _row_to_chunk(self, row, description) -> dict:+ columns = [desc[0] for desc in description]+ raw = dict(zip(columns, row)) if not isinstance(row, dict) else dict(row)+ result = {}+ dynamic_fields = {}+ invalid_vectors = {f"q_{match.group('dim')}_vec" for key, value in raw.items() if (match := VECTOR_VALID_COLUMN_RE.fullmatch(key)) and value is False}+ for key, value in raw.items():+ if key in invalid_vectors:+ continue+ decoded = decode_column_value(key, value)+ result[key] = decoded+ if key == "extra" and isinstance(decoded, dict):+ dynamic_fields.update(decoded)+ for key, value in dynamic_fields.items():+ result.setdefault(key, value)+ return result++ def _execute_statements(self, statements: Iterable[str | tuple[str, list[Any]]]) -> None:+ conn = self.pool.get_conn()+ cur = None+ try:+ cur = conn.cursor()+ for statement in statements:+ if isinstance(statement, tuple):+ cur.execute(statement[0], statement[1])+ else:+ cur.execute(statement)+ conn.commit()+ except Exception:+ conn.rollback()+ raise+ finally:+ close_cursor(cur)+ self.pool.put_conn(conn)++ def _execute_write(self, sql: str, params: list[Any], many: bool = False) -> int:+ conn = self.pool.get_conn()+ cur = None+ try:+ cur = conn.cursor()+ if many:+ cur.executemany(sql, params)+ else:+ cur.execute(sql, params)+ conn.commit()+ return int(getattr(cur, "rowcount", 0) or 0)+ except Exception:+ conn.rollback()+ raise+ finally:+ close_cursor(cur)+ self.pool.put_conn(conn)++ def _fetch_one(self, sql: str, params: list[Any]):+ row, _description = self._fetch_one_with_description(sql, params)+ return row++ def _fetch_one_with_description(self, sql: str, params: list[Any]):+ conn = self.pool.get_conn()+ cur = None+ try:+ cur = conn.cursor()+ cur.execute(sql, params)+ return cur.fetchone(), getattr(cur, "description", None) or []+ finally:+ close_cursor(cur)+ self.pool.put_conn(conn)++ def _fetch_all(self, sql: str, params: list[Any]):+ conn = self.pool.get_conn()+ cur = None+ try:+ cur = conn.cursor()+ cur.execute(sql, params)+ return cur.fetchall()+ finally:+ close_cursor(cur)+ self.pool.put_conn(conn)++ def _fetch_all_with_description(self, sql: str, params: list[Any], statement_timeout_ms: int | None = None):+ conn = self.pool.get_conn()+ cur = None+ try:+ cur = conn.cursor()+ if statement_timeout_ms and statement_timeout_ms > 0:+ cur.execute(f"SET LOCAL statement_timeout = {int(statement_timeout_ms)}")+ cur.execute(sql, params)+ return cur.fetchall(), getattr(cur, "description", None) or []+ finally:+ close_cursor(cur)+ self.pool.put_conn(conn)+++def is_doc_meta_table(index_name: str) -> bool:+ return str(index_name or "").startswith("ragflow_doc_meta_")+++def normalize_kb_id(value) -> str | None:+ if isinstance(value, list):+ return str(value[0]) if value else None+ if value is None:+ return None+ return str(value)+++def normalize_kb_ids(values) -> list[str]:+ if values is None:+ return []+ if isinstance(values, (str, bytes)):+ raw_values = [values]+ else:+ raw_values = list(values)+ normalized = []+ seen = set()+ for value in raw_values:+ if value in (None, ""):+ continue+ text = str(value)+ if text in seen:+ continue+ seen.add(text)+ normalized.append(text)+ return normalized+++def normalize_table_names(index_names: str | list[str]) -> list[str]:+ if isinstance(index_names, str):+ return [name.strip() for name in index_names.split(",") if name.strip()]+ return [str(name).strip() for name in index_names or [] if str(name).strip()]+++def parse_vector_dim(vector_column_name: str) -> int | None:+ match = VECTOR_COLUMN_RE.fullmatch(str(vector_column_name or ""))+ return int(match.group("dim")) if match else None+++def parse_fusion_vector_weight(expr: FusionExpr) -> float | None:+ params = expr.fusion_params or {}+ weights = params.get("weights")+ if not weights:+ return None+ try:+ return float(str(weights).split(",")[-1])+ except (TypeError, ValueError):+ return None+++def sortable_search_value(value: Any, field: str):+ if field == "page_num_int":+ return nested_numeric_value(value, [0])+ if field == "position_int":+ return nested_numeric_value(value, [0, 3])+ if field == "top_int":+ return nested_numeric_value(value, [0])+ if isinstance(value, (int, float)):+ return value+ if value is None:+ return ""+ return str(value)+++def nested_numeric_value(value: Any, path: list[int]) -> float:+ current = value+ try:+ for index in path:+ current = current[index]+ numeric = float(current)+ return numeric if isfinite(numeric) else 100000000.0+ except (TypeError, ValueError, IndexError, KeyError, OverflowError):+ return 100000000.0+++def select_doc_meta_columns(select_fields: list[str]) -> list[str]:+ if not select_fields or "*" in select_fields:+ return list(DOC_META_COLUMNS)+ columns = []+ for field in select_fields:+ validate_filter_column(field, is_meta=True)+ if field not in columns:+ columns.append(field)+ return columns+++def build_doc_meta_order_by(order_by) -> str:+ fields = getattr(order_by, "fields", None) or []+ parts = []+ for field, direction in fields:+ validate_filter_column(field, is_meta=True)+ parts.append(f"{field} {'DESC' if direction else 'ASC'}")+ return ", ".join(parts)+++def normalize_column_value(column: str, value: Any) -> Any:+ if value is None:+ return None+ if column in JSONB_COLUMNS:+ return json.dumps(value, ensure_ascii=False) if not isinstance(value, str) else value+ return value+++def parse_json_dict(value) -> dict:+ if isinstance(value, dict):+ return value+ if isinstance(value, str):+ try:+ parsed = json.loads(value)+ except json.JSONDecodeError:+ return {}+ return parsed if isinstance(parsed, dict) else {}+ return {}+++def vector_literal(value, dim: int) -> str:+ if isinstance(value, str):+ parsed = parse_vector_literal(value)+ if parsed is None:+ raise ValueError("invalid vector literal")+ value = parsed+ if not isinstance(value, (list, tuple)) or len(value) != dim:+ raise ValueError(f"vector dimension mismatch: expected {dim}, got {len(value) if hasattr(value, '__len__') else 'unknown'}")+ return "[" + ",".join(str(item) for item in value) + "]"+++def parse_vector_literal(value: str):+ try:+ parsed = json.loads(value)+ except json.JSONDecodeError:+ return None+ return parsed if isinstance(parsed, list) else None+++def zero_vector_literal(dim: int) -> str:+ return "[" + ",".join(["0"] * dim) + "]"+++def ordered_columns(rows: list[dict]) -> list[str]:+ present = set()+ for row in rows:+ present.update(row)+ dynamic_vectors = sorted(column for column in present if VECTOR_COLUMN_RE.fullmatch(column))+ dynamic_valid = sorted(column for column in present if VECTOR_VALID_COLUMN_RE.fullmatch(column))+ ordered = [column for column in CHUNK_COLUMNS if column in present]+ ordered.extend(column for column in dynamic_vectors + dynamic_valid if column not in ordered)+ return ordered+++def validate_filter_column(column: str, is_meta: bool) -> None:+ if is_meta:+ if column not in DOC_META_COLUMN_SET:+ raise ValueError(f"unsupported metadata filter column: {column}")+ return+ if column not in CHUNK_COLUMN_SET and not VECTOR_COLUMN_RE.fullmatch(str(column)) and not VECTOR_VALID_COLUMN_RE.fullmatch(str(column)):+ validate_extra_field(column)+++def is_dynamic_chunk_field(column: str, is_meta: bool = False) -> bool:+ return not is_meta and column not in CHUNK_COLUMN_SET and not VECTOR_COLUMN_RE.fullmatch(str(column)) and not VECTOR_VALID_COLUMN_RE.fullmatch(str(column))+++def is_jsonb_dynamic_chunk_field(column: str, is_meta: bool = False) -> bool:+ return is_dynamic_chunk_field(column, is_meta) and column not in GaussDBSearchBuilder.JSONB_EXTRA_SCALAR_COLUMNS+++def filter_column_expr(column: str, is_meta: bool = False) -> str:+ validate_filter_column(column, is_meta)+ if not is_dynamic_chunk_field(column, is_meta):+ return str(column)+ if column in GaussDBSearchBuilder.JSONB_EXTRA_SCALAR_COLUMNS:+ return f"(extra #>> '{{{column}}}')"+ return extra_field_expr(column)+++def decode_column_value(column: str, value: Any) -> Any:+ if value is None:+ return None+ if column in JSONB_COLUMNS or column in DOC_META_JSONB_COLUMNS:+ if isinstance(value, str):+ try:+ return json.loads(value)+ except json.JSONDecodeError as exc:+ raise ValueError(f"invalid JSONB value for {column}") from exc+ return value+ if VECTOR_COLUMN_RE.fullmatch(column):+ if isinstance(value, str):+ parsed = parse_vector_literal(value)+ if parsed is None:+ raise ValueError(f"invalid vector value for {column}")+ return parsed+ return value+ return value+++def close_cursor(cur) -> None:+ if cur is not None and hasattr(cur, "close"):+ cur.close()diff --git a/rag/utils/gaussdb_text_to_sql.py b/rag/utils/gaussdb_text_to_sql.pynew file mode 100644index 0000000000..ff861e0c18--- /dev/null+++ b/rag/utils/gaussdb_text_to_sql.py@@ -0,0 +1,219 @@+#+# Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+#+import logging++from sqlglot import exp, parse_one++from common.doc_store.gaussdb_conn_base import (+ ExposedGaussDBTable,+ GaussDBSQLValidator,+ gaussdb_field_json_path_parts,+ is_gaussdb_aggregate_sql,+ jsonb_path_literal,+)+++logger = logging.getLogger(__name__)++_SQL_RULES = """RULES:+1. Use only the table and fields listed below.+2. Select doc_id and docnm_kwd for non-aggregate data queries.+3. Do not add kb_id to SELECT or WHERE; the system injects and validates the KB scope.+4. Use only the static JSONB path literals listed under Fields.+5. Do not construct dynamic JSONB paths.+6. Do not use json_extract, json_extract_string, or json_extract_isnull; use #>> / #> instead.+7. A/ORA treats '' as NULL; never compare with = '' or <> ''; use IS NULL or IS NOT NULL.+8. Return exactly one read-only SELECT statement and no other text.+9. Do not use SELECT *, window functions, or system functions.+10. Do not use LATERAL, JOIN, UNION, INTERSECT, EXCEPT, or OR.+11. Do not expand JSON with jsonb_each, jsonb_each_text, jsonb_to_record, jsonb_to_recordset, jsonb_array_elements, or jsonb_array_elements_text.+12. Do not use DML or DDL, including INSERT, UPDATE, DELETE, MERGE, CREATE, ALTER, DROP, TRUNCATE, GRANT, REVOKE, CALL, COPY, or DO."""+++def build_validator(table_name, kb_ids, field_map):+ table = ExposedGaussDBTable.from_field_map(table_name, kb_ids, field_map)+ return GaussDBSQLValidator({table.logical_name: table}, default_limit=128)+++def is_aggregate_sql(sql):+ return is_gaussdb_aggregate_sql(sql)+++def _json_path_literal(field, field_type=None):+ return jsonb_path_literal(gaussdb_field_json_path_parts(str(field), field_type))+++def _field_lines(field_map):+ return "\n".join(f" - {field} ({descriptor}): chunk_data #>> {_json_path_literal(field, descriptor)}" for field, descriptor in (field_map or {}).items())+++def build_sql_prompt(table_name, field_map, question):+ return """You are a Database Administrator. Write SQL for GaussDB A/ORA mode for a table with JSONB 'chunk_data' column.++Table: {table_name}+JSONB text extraction: chunk_data #>> '{{FieldName}}'+JSONB value extraction: chunk_data #> '{{FieldName}}'+Numeric cast: CAST(chunk_data #>> '{{FieldName}}' AS DOUBLE PRECISION)+Date cast: to_date(chunk_data #>> '{{FieldName}}', 'YYYY-MM-DD')+NULL check: (chunk_data #>> '{{FieldName}}') IS NOT NULL++{rules}++Fields:+{fields}++Question: {question}""".format(+ table_name=table_name,+ rules=_SQL_RULES,+ fields=_field_lines(field_map),+ question=question,+ )+++def build_user_prompt(table_name, field_map, question):+ return """Table: {table_name}+Fields:+{fields}+Question: {question}+Write SQL using GaussDB JSONB #>> / #> syntax. Include doc_id and docnm_kwd for data queries. Only SQL.""".format(+ table_name=table_name,+ fields=_field_lines(field_map),+ question=question,+ )+++def build_repair_prompt(table_name, field_map, question, previous_sql):+ return """Table name: {table_name};+GaussDB JSONB fields available in chunk_data:+{fields}++Question: {question}+Previous SQL:+{previous_sql}++The previous SQL result is missing required source columns for citations.+Rewrite SQL to keep the same query intent and include doc_id and docnm_kwd in the SELECT list.+Use chunk_data #>> '{{field}}' or chunk_data #> '{{field}}' for JSONB fields.+For date fields, use to_date(chunk_data #>> '{{field}}', 'YYYY-MM-DD').++{rules}""".format(+ table_name=table_name,+ fields=_field_lines(field_map),+ question=question,+ previous_sql=previous_sql,+ rules=_SQL_RULES,+ )+++def build_retry_prompt(table_name, field_map, question, error):+ return """+Table name: {table_name};+GaussDB JSONB fields available in chunk_data:+{fields}++Question: {question}+Please write SQL using chunk_data #>> '{{field}}' / chunk_data #> '{{field}}' syntax. Include doc_id and docnm_kwd for data queries. Only SQL.+For date fields, use to_date(chunk_data #>> '{{field}}', 'YYYY-MM-DD').++The previous SQL error is:+{error}++Correct the SQL for GaussDB A/ORA mode.++{rules}+""".format(+ table_name=table_name,+ fields=_field_lines(field_map),+ question=question,+ error=error,+ rules=_SQL_RULES,+ )+++def build_aggregate_source_sql(sql, doc_name_column, include_kb_id):+ statement = parse_one(sql, read=GaussDBSQLValidator.PARSE_DIALECT)+ if statement.args.get("having"):+ raise ValueError("GaussDB aggregate source lookup cannot preserve HAVING safely")++ columns = [exp.column("doc_id"), exp.column(doc_name_column)]+ if include_kb_id:+ columns.append(exp.column("kb_id"))+ statement.set("expressions", columns)+ statement.set("distinct", None)+ statement.set("group", None)+ statement.set("order", None)+ statement.set("limit", None)+ statement.set("offset", None)+ return statement.sql(dialect=GaussDBSQLValidator.PARSE_DIALECT)+++def build_source_reference(table, kb_ids):+ rows = table.get("rows", [])+ if not rows:+ return None++ columns = table.get("columns", [])+ doc_idx = next((i for i, column in enumerate(columns) if column["name"].lower() == "doc_id"), None)+ name_idx = next((i for i, column in enumerate(columns) if column["name"].lower() in ["docnm_kwd", "docnm"]), None)+ kb_idx = next((i for i, column in enumerate(columns) if column["name"].lower() in ["kb_id", "kb_id_kwd"]), None)+ if doc_idx is None or name_idx is None:+ return None++ chunks = []+ doc_aggs = {}+ for row in rows:+ chunk = {"doc_id": row[doc_idx], "docnm_kwd": row[name_idx]}+ if len(kb_ids or []) == 1:+ chunk["kb_id"] = kb_ids[0]+ elif kb_idx is not None:+ chunk["kb_id"] = row[kb_idx]+ chunks.append(chunk)++ doc_id = row[doc_idx]+ if doc_id not in doc_aggs:+ doc_aggs[doc_id] = {"doc_name": row[name_idx], "count": 0}+ doc_aggs[doc_id]["count"] += 1++ return {+ "chunks": chunks,+ "doc_aggs": [{"doc_id": doc_id, "doc_name": values["doc_name"], "count": values["count"]} for doc_id, values in doc_aggs.items()],+ }+++def complete_reference_kb_ids(result, table_name, kb_ids, validator, sql_retrieval):+ if len(kb_ids or []) <= 1:+ return++ chunks = result["reference"]["chunks"]+ doc_ids = list(dict.fromkeys(str(chunk["doc_id"]) for chunk in chunks if chunk.get("doc_id") and not chunk.get("kb_id")))+ if not doc_ids:+ return++ quoted_doc_ids = ", ".join("'" + doc_id.replace("'", "''") + "'" for doc_id in doc_ids)+ lookup_sql = f"SELECT doc_id, kb_id FROM {table_name} WHERE doc_id IN ({quoted_doc_ids})"+ try:+ lookup_sql = validator.validate_and_patch(lookup_sql).sql+ lookup_tbl = sql_retrieval(lookup_sql, format="json")+ doc_idx = next((i for i, column in enumerate(lookup_tbl.get("columns", [])) if column["name"].lower() == "doc_id"), None)+ kb_idx = next((i for i, column in enumerate(lookup_tbl.get("columns", [])) if column["name"].lower() in ["kb_id", "kb_id_kwd"]), None)+ if doc_idx is None or kb_idx is None:+ return+ mapping = {row[doc_idx]: row[kb_idx] for row in lookup_tbl.get("rows", []) if len(row) > max(doc_idx, kb_idx)}+ for chunk in chunks:+ if not chunk.get("kb_id") and chunk.get("doc_id") in mapping:+ chunk["kb_id"] = mapping[chunk["doc_id"]]+ except Exception:+ logger.warning("use_sql: Failed to complete GaussDB reference kb_id values", exc_info=True)diff --git a/rag/utils/table_es_metadata.py b/rag/utils/table_es_metadata.pyindex c06155c225..b56edcadc4 100644--- a/rag/utils/table_es_metadata.py+++ b/rag/utils/table_es_metadata.py@@ -210,16 +210,20 @@ def aggregate_table_doc_metadata(chunks: list, task: dict) -> dict:e,exc_info=True,)- if not fm and not (settings.DOC_ENGINE_INFINITY or settings.DOC_ENGINE_OCEANBASE or settings.DOC_ENGINE_SERENEDB):+ sql_doc_engine = settings.DOC_ENGINE_INFINITY or settings.DOC_ENGINE_OCEANBASE or settings.DOC_ENGINE_GAUSSDB or settings.DOC_ENGINE_SERENEDB+ if not fm and not sql_doc_engine:logging.debug(f"[TABLE_META_DEBUG] field_map empty on task snapshot — will use ES key probe on chunk dicts; kb_parser_config keys={list((task.get('kb_parser_config') or {}).keys())}")- logging.debug(f"[TABLE_META_DEBUG] meta_cols={meta_cols}, field_map entries={len(fm)}, infinity={settings.DOC_ENGINE_INFINITY}, oceanbase={settings.DOC_ENGINE_OCEANBASE}")+ logging.debug(+ f"[TABLE_META_DEBUG] meta_cols={meta_cols}, field_map entries={len(fm)}, infinity={settings.DOC_ENGINE_INFINITY}, "+ f"oceanbase={settings.DOC_ENGINE_OCEANBASE}, gaussdb={settings.DOC_ENGINE_GAUSSDB}, serenedb={settings.DOC_ENGINE_SERENEDB}"+ )sample_ck = next((c for c in chunks if isinstance(c, dict)), None)if sample_ck:sk = [k for k in sample_ck.keys() if not (str(k).startswith("q_") and str(k).endswith("_vec"))][:50]logging.debug(f"[TABLE_META_DEBUG] first chunk non-vector keys (sample): {sk}")es_col_keys: dict[str, tuple[str | None, str]] = {}- if not (settings.DOC_ENGINE_INFINITY or settings.DOC_ENGINE_OCEANBASE or settings.DOC_ENGINE_SERENEDB):+ if not sql_doc_engine:for col in meta_cols:tk, src = _resolve_es_chunk_field_key(col, fm, sample_ck)es_col_keys[col] = (tk, src)@@ -230,7 +234,7 @@ def aggregate_table_doc_metadata(chunks: list, task: dict) -> dict:for i, ck in enumerate(chunks):if not isinstance(ck, dict):continue- if settings.DOC_ENGINE_INFINITY or settings.DOC_ENGINE_OCEANBASE or settings.DOC_ENGINE_SERENEDB:+ if sql_doc_engine:cd = ck.get("chunk_data")if not isinstance(cd, dict):continuediff --git a/test/testcases/test_web_api/test_system_app/test_system_routes_unit.py b/test/testcases/test_web_api/test_system_app/test_system_routes_unit.pyindex a9735fa300..4793a97234 100644--- a/test/testcases/test_web_api/test_system_app/test_system_routes_unit.py+++ b/test/testcases/test_web_api/test_system_app/test_system_routes_unit.py@@ -148,6 +148,7 @@ def _load_system_module(monkeypatch):health_utils_mod = ModuleType("api.utils.health_utils")health_utils_mod.run_health_checks = lambda: ({"status": "ok"}, True)health_utils_mod.get_oceanbase_status = lambda: {"status": "alive"}+ health_utils_mod.get_gaussdb_status = lambda: {"status": "alive"}monkeypatch.setitem(sys.modules, "api.utils.health_utils", health_utils_mod)quart_mod = ModuleType("quart")@@ -220,3 +221,14 @@ def test_get_config_returns_register_enabled_unit(monkeypatch):res = module.get_config()assert res["code"] == 0assert res["data"]["registerEnabled"] is False+++def test_gaussdb_status_route_unit(monkeypatch):+ module = _load_system_module(monkeypatch)+ monkeypatch.setattr(module, "get_gaussdb_status", lambda: {"status": "alive"})++ res = module.gaussdb_status()++ assert res["code"] == 0+ assert res["data"]["status"] == "alive"diff --git a/test/unit_test/api/apps/services/test_gaussdb_dataset_embedding.py b/test/unit_test/api/apps/services/test_gaussdb_dataset_embedding.pynew file mode 100644index 0000000000..4aca9efce5--- /dev/null+++ b/test/unit_test/api/apps/services/test_gaussdb_dataset_embedding.py@@ -0,0 +1,209 @@+#+# Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+#+import importlib.util+import sys+from pathlib import Path+from types import ModuleType, SimpleNamespace+from unittest.mock import MagicMock+++def _install_module(monkeypatch, name, **attrs):+ module = ModuleType(name)+ for key, value in attrs.items():+ setattr(module, key, value)+ monkeypatch.setitem(sys.modules, name, module)+ if "." in name:+ parent_name, _, child_name = name.rpartition(".")+ parent = sys.modules.get(parent_name)+ if parent is not None:+ monkeypatch.setattr(parent, child_name, module, raising=False)+ return module+++class FakeSearchResult:+ pass+++class FakeGaussDBStore:+ def db_type(self):+ return "gaussdb"++ def search(self, *_args, **_kwargs):+ return FakeSearchResult()++ def get_total(self, _res):+ return 1++ def get_doc_ids(self, _res):+ return ["chunk-invalid-vector"]++ def get(self, _chunk_id, _index_name, _kb_ids):+ return {+ "id": "chunk-invalid-vector",+ "kb_id": "kb1",+ "doc_id": "doc1",+ "docnm_kwd": "doc.txt",+ "content_with_weight": "sample text",+ "q_4_vec": [0.0, 0.0, 0.0, 0.0],+ "q_4_vec_valid": False,+ }+++class ValidVectorGaussDBStore(FakeGaussDBStore):+ def get(self, _chunk_id, _index_name, _kb_ids):+ return {+ "id": "chunk-valid-vector",+ "kb_id": "kb1",+ "doc_id": "doc1",+ "docnm_kwd": "doc.txt",+ "content_with_weight": "sample text",+ "q_4_vec": [1.0, 0.0, 0.0, 0.0],+ "q_4_vec_valid": True,+ }+++class FailIfCalledBundle:+ def __init__(self, *_args, **_kwargs):+ pass++ def encode(self, *_args, **_kwargs):+ raise AssertionError("invalid placeholder vectors must be skipped before re-embedding")+++class CompatibleEmbeddingBundle:+ encode_calls = []++ def __init__(self, *_args, **_kwargs):+ pass++ def encode(self, *args, **kwargs):+ import numpy as np++ self.encode_calls.append((args, kwargs))+ return [np.array([1.0, 0.0, 0.0, 0.0]), np.array([1.0, 0.0, 0.0, 0.0])], None+++def _load_dataset_module(monkeypatch):+ repo_root = Path(__file__).resolve().parents[5]+ kb = SimpleNamespace(id="kb1", tenant_id="tenant1", embd_id="old-embd")++ _install_module(+ monkeypatch,+ "api.db.joint_services.tenant_model_service",+ get_model_config_from_provider_instance=lambda *_args, **_kwargs: {"llm_name": "new-embd"},+ get_composite_model_name_by_ids=lambda _ids: {},+ resolve_model_config=lambda *_args, **_kwargs: {"llm_name": "new-embd"},+ resolve_model_id=lambda *_args, **_kwargs: "new-embd",+ )+ _install_module(monkeypatch, "api.db.db_models", File=SimpleNamespace())+ _install_module(+ monkeypatch,+ "api.db.services.document_service",+ DocumentService=SimpleNamespace(),+ queue_raptor_o_graphrag_tasks=MagicMock(),+ )+ _install_module(monkeypatch, "api.db.services.file2document_service", File2DocumentService=SimpleNamespace())+ _install_module(monkeypatch, "api.db.services.file_service", FileService=SimpleNamespace())+ _install_module(+ monkeypatch,+ "api.db.services.knowledgebase_service",+ KnowledgebaseService=SimpleNamespace(+ accessible=staticmethod(lambda *_args: True),+ get_by_id=staticmethod(lambda _dataset_id: (True, kb)),+ ),+ validate_dataset_embedding_models=lambda *_args, **_kwargs: None,+ )+ _install_module(monkeypatch, "api.db.services.connector_service", Connector2KbService=SimpleNamespace())+ _install_module(+ monkeypatch,+ "api.db.services.task_service",+ GRAPH_RAPTOR_FAKE_DOC_ID="fake-doc",+ TaskService=SimpleNamespace(),+ )+ _install_module(+ monkeypatch,+ "api.db.services.user_service",+ TenantService=SimpleNamespace(),+ UserService=SimpleNamespace(),+ UserTenantService=SimpleNamespace(),+ )+ _install_module(+ monkeypatch,+ "api.db.services.tenant_model_service",+ TenantModelService=SimpleNamespace(get_by_id=staticmethod(lambda *_args, **_kwargs: (True, None))),+ )+ _install_module(monkeypatch, "common.settings", docStoreConn=FakeGaussDBStore())+ _install_module(+ monkeypatch,+ "common.constants",+ FileSource=SimpleNamespace(KNOWLEDGEBASE="knowledgebase"),+ LLMType=SimpleNamespace(EMBEDDING="embedding"),+ PAGERANK_FLD="pagerank_fea",+ RetCode=SimpleNamespace(NOT_EFFECTIVE=590),+ StatusEnum=SimpleNamespace(VALID=SimpleNamespace(value="1")),+ )+ _install_module(+ monkeypatch,+ "api.utils.api_utils",+ deep_merge=lambda base, update: {**(base or {}), **(update or {})},+ get_parser_config=lambda *_args, **_kwargs: {},+ remap_dictionary_keys=lambda value: value,+ verify_embedding_availability=lambda *_args, **_kwargs: (True, ""),+ )+ rag = _install_module(monkeypatch, "rag")+ rag.__path__ = []+ advanced_rag = _install_module(monkeypatch, "rag.advanced_rag")+ advanced_rag.__path__ = []+ knowledge_compile = _install_module(monkeypatch, "rag.advanced_rag.knowlege_compile")+ knowledge_compile.__path__ = []+ _install_module(monkeypatch, "rag.advanced_rag.knowlege_compile.wiki", WIKI_PAGE_COMPILE_KWD="artifact_page")+ _install_module(monkeypatch, "rag.nlp")+ _install_module(monkeypatch, "rag.nlp.search", index_name=lambda tenant_id: f"ragflow_{tenant_id}")++ spec = importlib.util.spec_from_file_location(+ "api.apps.services.dataset_api_service",+ repo_root / "api" / "apps" / "services" / "dataset_api_service.py",+ )+ module = importlib.util.module_from_spec(spec)+ monkeypatch.setitem(sys.modules, "api.apps.services.dataset_api_service", module)+ spec.loader.exec_module(module)+ return module+++def test_tc_wrt_206_check_embedding_ignores_gaussdb_invalid_placeholder_vectors(monkeypatch):+ module = _load_dataset_module(monkeypatch)++ _install_module(monkeypatch, "api.db.services.llm_service", LLMBundle=FailIfCalledBundle)++ ok, result = module.check_embedding("kb1", "tenant1", {"embd_id": "new-embd", "check_num": 1})++ assert ok is False+ assert result == "No embedded chunks are available to compare."+++def test_tc_wrt_207_check_embedding_compares_gaussdb_valid_vectors_once(monkeypatch):+ module = _load_dataset_module(monkeypatch)+ module.settings.docStoreConn = ValidVectorGaussDBStore()+ CompatibleEmbeddingBundle.encode_calls.clear()+ _install_module(monkeypatch, "api.db.services.llm_service", LLMBundle=CompatibleEmbeddingBundle)++ ok, result = module.check_embedding("kb1", "tenant1", {"embd_id": "new-embd", "check_num": 1})++ assert CompatibleEmbeddingBundle.encode_calls == [((["doc.txt", "sample text"],), {})]+ assert ok is True+ assert result["summary"]["valid"] == 1+ assert result["summary"]["avg_cos_sim"] == 1.0+ assert result["results"][0]["vector_dim"] == 4diff --git a/test/unit_test/api/apps/test_gaussdb_system_routes.py b/test/unit_test/api/apps/test_gaussdb_system_routes.pynew file mode 100644index 0000000000..d34e2fc2cc--- /dev/null+++ b/test/unit_test/api/apps/test_gaussdb_system_routes.py@@ -0,0 +1,285 @@+#+# Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+#++import asyncio+import importlib.util+import json+import sys+from pathlib import Path+from types import ModuleType, SimpleNamespace+from unittest.mock import Mock++import pytest+++ROUTE_PASSWORD_SENTINEL = "cfg703-password-'quoted![]{}'"+ROUTE_DSN_SENTINEL = "postgresql://cfg703-user:cfg703-dsn%27%[email protected]:19995/private"+ROUTE_TOKEN_SENTINEL = "cfg703-token-'quoted![]{}'"+++class _DummyManager:+ def route(self, *_args, **_kwargs):+ def decorator(func):+ return func++ return decorator+++class _ExprField:+ def __init__(self, name):+ self.name = name++ def __eq__(self, other):+ return (self.name, other)+++class _DummyAPITokenModel:+ tenant_id = _ExprField("tenant_id")+ token = _ExprField("token")+++def _load_system_module(monkeypatch, *, apps_module=None, manager=None):+ repo_root = Path(__file__).resolve().parents[4]++ api_pkg = ModuleType("api")+ api_pkg.__path__ = [str(repo_root / "api")]+ monkeypatch.setitem(sys.modules, "api", api_pkg)++ if apps_module is None:+ apps_mod = ModuleType("api.apps")+ apps_mod.__path__ = [str(repo_root / "api" / "apps")]+ apps_mod.login_required = lambda fn: fn+ apps_mod.current_user = SimpleNamespace(id="user-1")+ else:+ apps_mod = apps_module+ monkeypatch.setitem(sys.modules, "api.apps", apps_mod)++ common_pkg = ModuleType("common")+ common_pkg.__path__ = [str(repo_root / "common")]+ monkeypatch.setitem(sys.modules, "common", common_pkg)++ settings_mod = ModuleType("common.settings")+ settings_mod.docStoreConn = SimpleNamespace(health=lambda: {"type": "doc", "status": "green"})+ settings_mod.STORAGE_IMPL = SimpleNamespace(health=lambda: True)+ settings_mod.STORAGE_IMPL_TYPE = "MINIO"+ settings_mod.DATABASE_TYPE = "MYSQL"+ settings_mod.REGISTER_ENABLED = True+ settings_mod.DISABLE_PASSWORD_LOGIN = False+ common_pkg.settings = settings_mod+ monkeypatch.setitem(sys.modules, "common.settings", settings_mod)++ versions_mod = ModuleType("common.versions")+ versions_mod.get_ragflow_version = lambda: "0.0.0-unit"+ monkeypatch.setitem(sys.modules, "common.versions", versions_mod)++ time_utils_mod = ModuleType("common.time_utils")+ time_utils_mod.current_timestamp = lambda: 111+ time_utils_mod.datetime_format = lambda _dt: "2026-01-01 00:00:00"+ monkeypatch.setitem(sys.modules, "common.time_utils", time_utils_mod)++ api_utils_mod = ModuleType("api.utils.api_utils")+ api_utils_mod.get_json_result = lambda data=None, message="success", code=0: {+ "code": code,+ "message": message,+ "data": data,+ }+ api_utils_mod.get_data_error_result = lambda message="", code=102, data=None: {+ "code": code,+ "message": message,+ "data": data,+ }+ api_utils_mod.server_error_response = lambda exc: {+ "code": 100,+ "message": repr(exc),+ "data": None,+ }+ api_utils_mod.generate_confirmation_token = lambda: "ragflow-abcdefghijklmnopqrstuvwxyz0123456789"+ monkeypatch.setitem(sys.modules, "api.utils.api_utils", api_utils_mod)++ api_service_mod = ModuleType("api.db.services.api_service")+ api_service_mod.APITokenService = SimpleNamespace(+ save=lambda **_kwargs: True,+ query=lambda **_kwargs: [],+ filter_update=lambda *_args, **_kwargs: True,+ filter_delete=lambda *_args, **_kwargs: True,+ )+ monkeypatch.setitem(sys.modules, "api.db.services.api_service", api_service_mod)++ kb_service_mod = ModuleType("api.db.services.knowledgebase_service")+ kb_service_mod.KnowledgebaseService = SimpleNamespace(get_by_id=lambda _kb_id: True)+ monkeypatch.setitem(sys.modules, "api.db.services.knowledgebase_service", kb_service_mod)++ user_service_mod = ModuleType("api.db.services.user_service")+ user_service_mod.UserTenantService = SimpleNamespace(query=lambda **_kwargs: [SimpleNamespace(role="owner", tenant_id="tenant-1")])+ monkeypatch.setitem(sys.modules, "api.db.services.user_service", user_service_mod)++ db_models_mod = ModuleType("api.db.db_models")+ db_models_mod.APIToken = _DummyAPITokenModel+ monkeypatch.setitem(sys.modules, "api.db.db_models", db_models_mod)++ rag_pkg = ModuleType("rag")+ rag_pkg.__path__ = []+ monkeypatch.setitem(sys.modules, "rag", rag_pkg)++ rag_utils_pkg = ModuleType("rag.utils")+ rag_utils_pkg.__path__ = []+ monkeypatch.setitem(sys.modules, "rag.utils", rag_utils_pkg)++ redis_mod = ModuleType("rag.utils.redis_conn")+ redis_mod.REDIS_CONN = SimpleNamespace(+ health=lambda: True,+ smembers=lambda *_args, **_kwargs: set(),+ zrangebyscore=lambda *_args, **_kwargs: [],+ )+ monkeypatch.setitem(sys.modules, "rag.utils.redis_conn", redis_mod)++ health_utils_mod = ModuleType("api.utils.health_utils")+ health_utils_mod.run_health_checks = lambda: ({"status": "ok"}, True)+ health_utils_mod.get_oceanbase_status = lambda: {"status": "alive"}+ health_utils_mod.get_gaussdb_status = lambda: {"status": "alive"}+ monkeypatch.setitem(sys.modules, "api.utils.health_utils", health_utils_mod)++ quart_mod = ModuleType("quart")+ quart_mod.jsonify = lambda payload: payload+ monkeypatch.setitem(sys.modules, "quart", quart_mod)++ module_path = repo_root / "api" / "apps" / "restful_apis" / "system_api.py"+ spec = importlib.util.spec_from_file_location("test_gaussdb_system_routes_module", module_path)+ module = importlib.util.module_from_spec(spec)+ module.manager = manager or _DummyManager()+ monkeypatch.setitem(sys.modules, "test_gaussdb_system_routes_module", module)+ spec.loader.exec_module(module)+ return module+++def _load_system_http_app(monkeypatch):+ from test.testcases.test_web_api.test_system_app.test_apps_init_unit import _load_apps_module++ quart_app, apps_module = _load_apps_module(monkeypatch)+ module = _load_system_module(monkeypatch, apps_module=apps_module, manager=quart_app)+ return quart_app, module+++def _get_without_auth(quart_app, path):+ async def request():+ response = await quart_app.test_client().get(path)+ return response.status_code, await response.get_json()++ return asyncio.run(request())+++def test_tc_cfg_704_status_branch_matrix_unit(monkeypatch):+ module = _load_system_module(monkeypatch)+ expected = {+ "db": "ok",+ "redis": "ok",+ "doc_engine": "ok",+ "storage": "ok",+ "status": "ok",+ }+ monkeypatch.setattr(module, "run_health_checks", lambda: (expected, True))++ payload, status_code = module.healthz()++ assert status_code == 200+ assert payload == expected+++def test_tc_cfg_705_healthz_returns_500_when_any_check_is_not_ok(monkeypatch):+ module = _load_system_module(monkeypatch)+ expected = {+ "db": "ok",+ "redis": "ok",+ "doc_engine": "nok",+ "storage": "ok",+ "status": "nok",+ "_meta": {"doc_engine": {"error": "doc down"}},+ }+ monkeypatch.setattr(module, "run_health_checks", lambda: (expected, False))++ payload, status_code = module.healthz()++ assert status_code == 500+ assert payload == expected+++def test_tc_cfg_706_healthz_route_allows_real_unauthenticated_request(monkeypatch):+ quart_app, module = _load_system_http_app(monkeypatch)+ expected = {"status": "ok", "doc_engine": "ok"}+ monkeypatch.setattr(module, "run_health_checks", lambda: (expected, True))++ status_code, payload = _get_without_auth(quart_app, "/system/healthz")++ assert status_code == 200+ assert payload == expected+++def test_tc_cfg_702_gaussdb_status_route_rejects_real_unauthenticated_request(monkeypatch):+ quart_app, module = _load_system_http_app(monkeypatch)+ probe = Mock(return_value={"status": "alive"})+ monkeypatch.setattr(module, "get_gaussdb_status", probe)++ status_code, payload = _get_without_auth(quart_app, "/system/gaussdb/status")++ assert status_code == 401+ assert payload["code"] == 401+ assert "Unauthorized" in payload["message"]+ probe.assert_not_called()+++def test_tc_cfg_712_gaussdb_status_route_returns_probe_payload(monkeypatch):+ module = _load_system_module(monkeypatch)+ monkeypatch.setattr(module, "get_gaussdb_status", lambda: {"status": "alive"})++ res = module.gaussdb_status()++ assert res["code"] == 0+ assert res == {"code": 0, "message": "success", "data": {"status": "alive"}}+++def test_tc_cfg_703_gaussdb_status_route_returns_500_when_probe_raises(monkeypatch, caplog):+ module = _load_system_module(monkeypatch)++ def raise_probe_error():+ raise RuntimeError(f"probe failed password=\"{ROUTE_PASSWORD_SENTINEL}\" dsn={ROUTE_DSN_SENTINEL} access_token='{ROUTE_TOKEN_SENTINEL}'")++ monkeypatch.setattr(module, "get_gaussdb_status", raise_probe_error)+ caplog.set_level("ERROR")++ res = module.gaussdb_status()+ serialized = json.dumps(res, sort_keys=True)+ log_text = caplog.text++ assert res["code"] == 500+ assert res["data"]["status"] == "error"+ assert res["data"]["message"].startswith("Failed to get GaussDB status: probe failed")+ assert "***" in res["data"]["message"]+ assert ROUTE_PASSWORD_SENTINEL not in serialized+ assert ROUTE_DSN_SENTINEL not in serialized+ assert ROUTE_TOKEN_SENTINEL not in serialized+ assert "postgresql://" not in serialized+ assert "cfg703-user" not in serialized+ assert "GaussDB status route failed (RuntimeError)" in log_text+ assert "***" in log_text+ assert ROUTE_PASSWORD_SENTINEL not in log_text+ assert ROUTE_DSN_SENTINEL not in log_text+ assert ROUTE_TOKEN_SENTINEL not in log_textdiff --git a/test/unit_test/api/db/gaussdb_test_utils.py b/test/unit_test/api/db/gaussdb_test_utils.pynew file mode 100644index 0000000000..495ffec0c3--- /dev/null+++ b/test/unit_test/api/db/gaussdb_test_utils.py@@ -0,0 +1,64 @@+import importlib+import sys+import types+++class DummyConnection:+ def __init__(self, *args, **kwargs):+ pass+++def stub_module(name, **attrs):+ module = types.ModuleType(name)+ for attr_name, attr_value in attrs.items():+ setattr(module, attr_name, attr_value)+ sys.modules[name] = module+ return module+++def stub_settings_import_dependencies():+ memory_utils = importlib.import_module("memory.utils")+ from rag.nlp import is_english+ from rag.nlp import rag_tokenizer++ search = stub_module("rag.nlp.search")+ rag_nlp = stub_module("rag.nlp", search=search, rag_tokenizer=rag_tokenizer, is_english=is_english)+ rag_nlp.__path__ = []+ sys.modules["rag.nlp"] = rag_nlp+ sys.modules["rag.nlp.rag_tokenizer"] = rag_tokenizer++ redis_conn = stub_module(+ "rag.utils.redis_conn",+ REDIS_CONN=types.SimpleNamespace(get_or_create_secret_key=lambda key, value: value, health=lambda: True),+ )+ sys.modules["rag.utils.redis_conn"] = redis_conn++ for module_name in [+ "rag.utils.es_conn",+ "rag.utils.infinity_conn",+ "rag.utils.ob_conn",+ "rag.utils.opensearch_conn",+ "memory.utils.es_conn",+ "memory.utils.infinity_conn",+ "memory.utils.ob_conn",+ ]:+ module = stub_module(+ module_name,+ ESConnection=DummyConnection,+ InfinityConnection=DummyConnection,+ OBConnection=DummyConnection,+ OSConnection=DummyConnection,+ )+ if module_name.startswith("memory.utils."):+ setattr(memory_utils, module_name.rsplit(".", 1)[-1], module)++ for module_name, class_name in [+ ("rag.utils.azure_sas_conn", "RAGFlowAzureSasBlob"),+ ("rag.utils.azure_spn_conn", "RAGFlowAzureSpnBlob"),+ ("rag.utils.gcs_conn", "RAGFlowGCS"),+ ("rag.utils.minio_conn", "RAGFlowMinio"),+ ("rag.utils.opendal_conn", "OpenDALStorage"),+ ("rag.utils.s3_conn", "RAGFlowS3"),+ ("rag.utils.oss_conn", "RAGFlowOSS"),+ ]:+ stub_module(module_name, **{class_name: DummyConnection})diff --git a/test/unit_test/api/db/services/test_gaussdb_dialog_sql.py b/test/unit_test/api/db/services/test_gaussdb_dialog_sql.pynew file mode 100644index 0000000000..665d3e04c2--- /dev/null+++ b/test/unit_test/api/db/services/test_gaussdb_dialog_sql.py@@ -0,0 +1,1866 @@+#+# Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+#+import asyncio+import importlib+import logging+import sys+import types+import warnings+from unittest.mock import AsyncMock, Mock++import pytest+++GAUSSDB_SQL_RULE_MARKERS = (+ "Use only the static JSONB path literals listed under Fields.",+ "Do not construct dynamic JSONB paths.",+ "A/ORA treats '' as NULL; never compare with = '' or <> ''; use IS NULL or IS NOT NULL.",+ "Return exactly one read-only SELECT statement and no other text.",+ "Do not use LATERAL, JOIN, UNION, INTERSECT, EXCEPT, or OR.",+ "Do not expand JSON with jsonb_each, jsonb_each_text, jsonb_to_record, jsonb_to_recordset, jsonb_array_elements, or jsonb_array_elements_text.",+ "Do not use DML or DDL, including INSERT, UPDATE, DELETE, MERGE, CREATE, ALTER, DROP, TRUNCATE, GRANT, REVOKE, CALL, COPY, or DO.",+)+++def _assert_gaussdb_sql_rule_markers(prompt):+ for marker in GAUSSDB_SQL_RULE_MARKERS:+ assert marker in prompt+++warnings.filterwarnings(+ "ignore",+ message="pkg_resources is deprecated as an API.*",+ category=UserWarning,+)+++def _install_cv2_stub_if_unavailable():+ try:+ import cv2 # noqa: F401++ return+ except Exception:+ pass++ stub = types.ModuleType("cv2")+ stub.INTER_LINEAR = 1+ stub.INTER_CUBIC = 2+ stub.BORDER_CONSTANT = 0+ stub.BORDER_REPLICATE = 1+ stub.COLOR_BGR2RGB = 0+ stub.COLOR_BGR2GRAY = 1+ stub.COLOR_GRAY2BGR = 2+ stub.IMREAD_IGNORE_ORIENTATION = 128+ stub.IMREAD_COLOR = 1+ stub.RETR_LIST = 1+ stub.CHAIN_APPROX_SIMPLE = 2++ def _module_getattr(name):+ if name.isupper():+ return 0+ raise RuntimeError(f"cv2.{name} is unavailable in this test environment")++ stub.__getattr__ = _module_getattr+ sys.modules["cv2"] = stub+++def _install_settings_import_stubs(monkeypatch):+ class StubDocEngineConnection:+ def db_type(self):+ return "stub"++ class StubStorage:+ def health(self):+ return True++ def install_module(name, **attrs):+ module = types.ModuleType(name)+ for key, value in attrs.items():+ setattr(module, key, value)+ monkeypatch.setitem(sys.modules, name, module)+ return module++ try:+ import rag.utils+ import memory.utils+ except Exception:+ return++ rag_modules = {+ "es_conn": {"ESConnection": StubDocEngineConnection},+ "infinity_conn": {"InfinityConnection": StubDocEngineConnection},+ "ob_conn": {"OBConnection": StubDocEngineConnection},+ "opensearch_conn": {"OSConnection": StubDocEngineConnection},+ "azure_sas_conn": {"RAGFlowAzureSasBlob": StubStorage},+ "azure_spn_conn": {"RAGFlowAzureSpnBlob": StubStorage},+ "gcs_conn": {"RAGFlowGCS": StubStorage},+ "minio_conn": {"RAGFlowMinio": StubStorage},+ "opendal_conn": {"OpenDALStorage": StubStorage},+ "redis_conn": {"REDIS_CONN": types.SimpleNamespace(health=lambda: True, is_alive=lambda: False, REDIS=None)},+ "s3_conn": {"RAGFlowS3": StubStorage},+ "oss_conn": {"RAGFlowOSS": StubStorage},+ }+ for short_name, attrs in rag_modules.items():+ module = install_module(f"rag.utils.{short_name}", **attrs)+ monkeypatch.setattr(rag.utils, short_name, module, raising=False)++ for short_name in ("es_conn", "infinity_conn", "ob_conn"):+ module = install_module(+ f"memory.utils.{short_name}",+ ESConnection=StubDocEngineConnection,+ InfinityConnection=StubDocEngineConnection,+ OBConnection=StubDocEngineConnection,+ )+ monkeypatch.setattr(memory.utils, short_name, module, raising=False)++ if "json_repair" not in sys.modules:+ import json++ install_module("json_repair", loads=json.loads)+ install_module("roman_numbers")+ install_module("word2number", w2n=types.SimpleNamespace(word_to_num=lambda value: 0))+ install_module("cn2an", cn2an=lambda value, *_args, **_kwargs: 0)+ if "langfuse" not in sys.modules:++ class StubLangfuse:+ pass++ def propagate_attributes(**_kwargs):+ class Context:+ def __enter__(self):+ return self++ def __exit__(self, *_args):+ return False++ return Context()++ install_module("langfuse", Langfuse=StubLangfuse, propagate_attributes=propagate_attributes)+ if "peewee" not in sys.modules:+ install_module("peewee", fn=types.SimpleNamespace())+ if "mcp.client.session" not in sys.modules:+ install_module("mcp")+ install_module("mcp.client")++ class ClientSession:+ pass++ async def _client(*_args, **_kwargs):+ raise RuntimeError("mcp client is unavailable in this test")++ class _MCPType:+ pass++ install_module("mcp.client.session", ClientSession=ClientSession)+ install_module("mcp.client.sse", sse_client=_client)+ install_module("mcp.client.streamable_http", streamablehttp_client=_client)+ install_module(+ "mcp.types",+ CallToolResult=_MCPType,+ ListToolsResult=_MCPType,+ TextContent=_MCPType,+ Tool=_MCPType,+ )+ if "beartype" not in sys.modules:++ def beartype(obj=None, **_kwargs):+ if obj is None:+ return lambda wrapped: wrapped+ return obj++ install_module("beartype", beartype=beartype)+ install_module("beartype.claw", beartype_this_package=lambda *_args, **_kwargs: None)++ class _Dummy:+ def __init__(self, *_args, **_kwargs):+ pass++ class _Context:+ def __enter__(self):+ return self++ def __exit__(self, *_args):+ return False++ def __call__(self, func):+ return func++ class _DummyDB:+ @staticmethod+ def connection_context():+ return _Context()++ @staticmethod+ def atomic():+ return _Context()++ install_module("api.db.services.user_service", UserService=_Dummy)+ install_module("api.db.services.file_service", FileService=_Dummy)+ install_module("api.db.services.common_service", CommonService=_Dummy)+ install_module("api.db.services.doc_metadata_service", DocMetadataService=_Dummy)+ install_module(+ "api.db.services.knowledgebase_service",+ KnowledgebaseService=types.SimpleNamespace(get_field_map=lambda _kb_ids: {}),+ validate_dataset_embedding_models=lambda _kbs: None,+ )+ install_module("api.db.services.langfuse_service", TenantLangfuseService=_Dummy)+ install_module("api.db.services.llm_service", LLMBundle=_Dummy)+ install_module(+ "api.db.joint_services.tenant_model_service",+ get_tenant_default_model_by_type=lambda *_args, **_kwargs: {"model_type": "chat", "max_tokens": 8192},+ get_model_config_from_provider_instance=lambda *_args, **_kwargs: {},+ get_model_type_by_name=lambda *_args, **_kwargs: [],+ resolve_model_config=lambda *_args, **_kwargs: ({"model_type": "chat"}, None),+ resolve_model_type=lambda *_args, **_kwargs: None,+ get_model_config_by_id=lambda *_args, **_kwargs: {},+ )+ install_module("api.db.db_models", DB=_DummyDB, Dialog=_Dummy)+ install_module("common.metadata_utils", apply_meta_data_filter=lambda *_args, **_kwargs: None)+ install_module(+ "api.utils.reference_metadata_utils",+ enrich_chunks_with_document_metadata=lambda chunks, *_args, **_kwargs: chunks,+ resolve_reference_metadata_preferences=lambda *_args, **_kwargs: {},+ )+ install_module("rag.graphrag.general.mind_map_extractor", MindMapExtractor=_Dummy)+ advanced_rag = install_module("rag.advanced_rag", DeepResearcher=_Dummy)+ advanced_rag.__path__ = []+ install_module("rag.advanced_rag.agentic_rag", RAGTools=_Dummy)+ install_module("rag.advanced_rag.knowlege_compile")+ install_module("rag.advanced_rag.knowlege_compile.mind_map_extractor", MindMapExtractor=_Dummy)+ install_module("rag.app.tag", label_question=lambda *_args, **_kwargs: None)+ install_module("rag.nlp.search", index_name=lambda tenant_id: f"ragflow_{tenant_id}")+ install_module(+ "rag.prompts.generator",+ chunks_format=lambda *_args, **_kwargs: "",+ citation_prompt=lambda: "",+ cross_languages=lambda *_args, **_kwargs: "",+ full_question=lambda *_args, **_kwargs: "",+ kb_prompt=lambda *_args, **_kwargs: [],+ keyword_extraction=lambda *_args, **_kwargs: "",+ message_fit_in=lambda *_args, **_kwargs: (0, []),+ PROMPT_JINJA_ENV=types.SimpleNamespace(from_string=lambda *_args, **_kwargs: types.SimpleNamespace(render=lambda **_kw: "")),+ ASK_SUMMARY="",+ )+ install_module("common.token_utils", num_tokens_from_string=lambda *_args, **_kwargs: 0)+ install_module("rag.utils.tavily_conn", Tavily=_Dummy)+ install_module("rag.utils.tts_cache", synthesize_with_cache=lambda *_args, **_kwargs: None)+++_install_cv2_stub_if_unavailable()+++class FakeChatModel:+ def __init__(self, responses):+ self.responses = list(responses)+ self.calls = []++ async def async_chat(self, sys_prompt, messages, params):+ self.calls.append((sys_prompt, messages, params))+ if not self.responses:+ raise AssertionError("async_chat called more times than expected")+ return self.responses.pop(0)+++class RecordingRetriever:+ def __init__(self, fail_first=False, missing_source_once=False):+ self.sqls = []+ self.fail_first = fail_first+ self.missing_source_once = missing_source_once++ def sql_retrieval(self, sql, format="json"):+ assert format == "json"+ self.sqls.append(sql)+ if self.fail_first and len(self.sqls) == 1:+ raise RuntimeError("json_extract_string does not exist")+ missing_source_call = 2 if self.fail_first else 1+ if self.missing_source_once and len(self.sqls) == missing_source_call:+ return {"columns": [{"name": "amount"}], "rows": [["120"]]}+ return {+ "columns": [{"name": "doc_id"}, {"name": "docnm_kwd"}, {"name": "kb_id"}, {"name": "amount"}],+ "rows": [["doc1", "finance.csv", "abcdefabcdefabcdefabcdefabcdefab", "120"]],+ }+++class AggregateRecordingRetriever:+ def __init__(self):+ self.sqls = []++ def sql_retrieval(self, sql, format="json"):+ assert format == "json"+ self.sqls.append(sql)+ if len(self.sqls) == 1:+ return {"columns": [{"name": "total_amount"}], "rows": [["120"]]}+ return {+ "columns": [{"name": "doc_id"}, {"name": "docnm_kwd"}, {"name": "kb_id"}],+ "rows": [["doc1", "finance.csv", "abcdefabcdefabcdefabcdefabcdefab"]],+ }+++class MultiKBReferenceRetriever:+ def __init__(self):+ self.sqls = []++ def sql_retrieval(self, sql, format="json"):+ assert format == "json"+ self.sqls.append(sql)+ if len(self.sqls) == 1:+ return {+ "columns": [{"name": "doc_id"}, {"name": "docnm_kwd"}, {"name": "amount"}],+ "rows": [+ ["doc1", "finance-a.csv", "120"],+ ["doc2", "finance-b.csv", "80"],+ ["doc3", "finance-c.csv", "40"],+ ],+ }+ return {+ "columns": [{"name": "doc_id"}, {"name": "kb_id"}],+ "rows": [+ ["doc1", "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"],+ ["doc2", "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"],+ ["doc3", "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"],+ ],+ }+++class MultiKBReferenceWithInlineKB:+ def __init__(self):+ self.sqls = []++ def sql_retrieval(self, sql, format="json"):+ assert format == "json"+ self.sqls.append(sql)+ return {+ "columns": [{"name": "doc_id"}, {"name": "docnm_kwd"}, {"name": "kb_id"}],+ "rows": [["doc1", "finance.csv", "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"]],+ }+++class ScriptedRetriever:+ def __init__(self, sql_results=(), retrieval_result=None):+ self.sql_results = list(sql_results)+ self.sqls = []+ self.retrieval = AsyncMock(+ return_value=retrieval_result+ or {+ "total": 1,+ "chunks": [+ {+ "id": "chunk-1",+ "doc_id": "d1",+ "docnm_kwd": "doc.xlsx",+ "kb_id": "abcdefabcdefabcdefabcdefabcdefab",+ "content_with_weight": "fallback content",+ "content_ltks": "fallback content",+ "vector": [],+ }+ ],+ "doc_aggs": [{"doc_id": "d1", "doc_name": "doc.xlsx", "count": 1}],+ }+ )++ def sql_retrieval(self, sql, format="json"):+ assert format == "json"+ self.sqls.append(sql)+ if not self.sql_results:+ raise AssertionError("sql_retrieval called more times than expected")+ result = self.sql_results.pop(0)+ if isinstance(result, BaseException):+ raise result+ return result++ @staticmethod+ def retrieval_by_children(chunks, _tenant_ids):+ return chunks++ @staticmethod+ def insert_citations(answer, *_args, **_kwargs):+ return answer, set()+++def dialog_service(monkeypatch):+ _install_settings_import_stubs(monkeypatch)+ module = importlib.import_module("api.db.services.dialog_service")+ try:+ yield module+ finally:+ sys.modules.pop("api.db.services.dialog_service", None)+ package = sys.modules.get("api.db.services")+ if package is not None and hasattr(package, "dialog_service"):+ delattr(package, "dialog_service")+++def enable_gaussdb_docengine(monkeypatch, dialog_service):+ monkeypatch.setattr(dialog_service.settings, "DOC_ENGINE_INFINITY", False, raising=False)+ monkeypatch.setattr(dialog_service.settings, "DOC_ENGINE_OCEANBASE", False, raising=False)+ monkeypatch.setattr(dialog_service.settings, "DOC_ENGINE_GAUSSDB", True, raising=False)+++def test_tc_sql_601_gaussdb_prompt_uses_jsonb_path_not_oceanbase_json_helpers(dialog_service, enable_gaussdb_docengine):+ prompt = dialog_service.gaussdb_text_to_sql.build_sql_prompt(+ table_name="ragflow_0123456789abcdef0123456789abcdef",+ field_map={"amount": "number", "status": "string"},+ question="amount greater than 100",+ )++ assert "JSONB text extraction: chunk_data #>> '{FieldName}'" in prompt+ assert "JSONB value extraction: chunk_data #> '{FieldName}'" in prompt+ assert "Numeric cast: CAST(chunk_data #>> '{FieldName}' AS DOUBLE PRECISION)" in prompt+ assert "Date cast: to_date(chunk_data #>> '{FieldName}', 'YYYY-MM-DD')" in prompt+ assert "NULL check: (chunk_data #>> '{FieldName}') IS NOT NULL" in prompt+ assert "#>> '{amount}'" in prompt+ assert " - status (string): chunk_data #>> '{status}'" in prompt+ assert "json_extract_string(" not in prompt+ assert "Select doc_id and docnm_kwd for non-aggregate data queries." in prompt+ assert "Do not add kb_id" in prompt+ assert "Do not use json_extract, json_extract_string, or json_extract_isnull" in prompt+ assert "Restrict the query by kb_id" not in prompt+ _assert_gaussdb_sql_rule_markers(prompt)+++def test_tc_sql_506_sql_converts_row_values_after_runtime_validation(dialog_service):+ from rag.utils.gaussdb_conn import GaussDBConnection++ conn = object.__new__(GaussDBConnection)+ conn.schema = "public"+ conn.logger = logging.getLogger("test.gaussdb")+ conn._fetch_all_with_description = Mock(+ return_value=(+ [(b"abc", {"a": 1}, [1, "x"], None)],+ [("bytes_col",), ("object_col",), ("list_col",), ("null_col",)],+ )+ )+ sql = (+ "SELECT doc_id AS bytes_col, chunk_data #> '{object_col}' AS object_col, "+ "chunk_data #> '{list_col}' AS list_col, chunk_data #>> '{null_col}' AS null_col "+ "FROM ragflow_tc_sql_506 WHERE kb_id = 'kb-506'"+ )++ result = conn.sql(sql, format="json")++ executed_sql, params = conn._fetch_all_with_description.call_args.args+ assert executed_sql == (+ "SELECT doc_id AS bytes_col, chunk_data #> '{object_col}' AS object_col, "+ "chunk_data #> '{list_col}' AS list_col, chunk_data #>> '{null_col}' AS null_col "+ "FROM \"public\".ragflow_tc_sql_506 WHERE kb_id = 'kb-506' LIMIT 128"+ )+ assert params == []+ assert conn._fetch_all_with_description.call_args.kwargs == {+ "statement_timeout_ms": 30000,+ }+ assert result == {+ "columns": [+ {"name": "bytes_col", "type": "text"},+ {"name": "object_col", "type": "text"},+ {"name": "list_col", "type": "text"},+ {"name": "null_col", "type": "text"},+ ],+ "rows": [["abc", '{"a": 1}', '[1, "x"]', None]],+ }+++def test_tc_sql_609_gaussdb_prompt_uses_field_map_value_as_jsonb_path(dialog_service, enable_gaussdb_docengine):+ prompt = dialog_service.gaussdb_text_to_sql.build_sql_prompt(+ table_name="ragflow_0123456789abcdef0123456789abcdef",+ field_map={"customer_alias": "customer_name", "amount": "number"},+ question="show customer amount",+ )++ assert " - customer_alias (customer_name): chunk_data #>> '{customer_name}'" in prompt+ assert " - amount (number): chunk_data #>> '{amount}'" in prompt+ _assert_gaussdb_sql_rule_markers(prompt)+++def test_tc_sql_410_empty_value_prompt_rules_and_validator_rejects_empty_string_comparison(+ dialog_service,+ enable_gaussdb_docengine,+):+ prompt = dialog_service.gaussdb_text_to_sql.build_sql_prompt(+ "ragflow_t1",+ {"status": "string"},+ "show rows where status is empty",+ )+ validator = dialog_service.gaussdb_text_to_sql.build_validator("ragflow_t1", ["kb-1"], {"status": "string"})++ with pytest.raises(ValueError, match="JSONB text cannot be compared with an empty SQL string"):+ validator.validate_and_patch("SELECT doc_id FROM ragflow_t1 WHERE chunk_data #>> '{status}' = ''")++ assert "NULL check: (chunk_data #>> '{FieldName}') IS NOT NULL" in prompt+ assert "Do not use json_extract, json_extract_string, or json_extract_isnull; use #>> / #> instead" in prompt+ _assert_gaussdb_sql_rule_markers(prompt)+++def test_tc_sql_001_async_chat_routes_table_field_map_to_gaussdb_use_sql(monkeypatch, dialog_service, enable_gaussdb_docengine):+ tenant_id = "0123456789abcdef0123456789abcdef"+ kb_id = "abcdefabcdefabcdefabcdefabcdefab"+ sql_result = {+ "answer": "|amount|\n|---|\n|100|",+ "reference": {"chunks": [], "doc_aggs": []},+ "prompt": "gaussdb prompt",+ }+ get_field_map = Mock(return_value={"amount": "number"})+ use_sql = AsyncMock(return_value=sql_result)+ retrieval = AsyncMock()++ chat = object()+ monkeypatch.setattr(+ dialog_service.settings,+ "retriever",+ types.SimpleNamespace(retrieval=retrieval),+ raising=False,+ )+ monkeypatch.setattr(+ dialog_service,+ "KnowledgebaseService",+ types.SimpleNamespace(get_field_map=get_field_map),+ raising=False,+ )+ monkeypatch.setattr(dialog_service, "use_sql", use_sql)+ monkeypatch.setattr(dialog_service, "get_models", lambda *_args, **_kwargs: ([], None, None, chat, None))+ monkeypatch.setattr(dialog_service, "_resolve_reference_metadata", lambda *_args, **_kwargs: (False, []))+ monkeypatch.setattr(+ dialog_service,+ "TenantLangfuseService",+ types.SimpleNamespace(filter_by_tenant=lambda **_kwargs: None),+ raising=False,+ )+ dialog = types.SimpleNamespace(+ tenant_id=tenant_id,+ kb_ids=[kb_id],+ prompt_config={"quote": True, "parameters": [], "system": ""},+ llm_id=None,+ tenant_llm_id=None,+ meta_data_filter=None,+ )++ async def collect():+ results = []+ async for item in dialog_service.async_chat(dialog, [{"role": "user", "content": "What is the total amount?"}], stream=False):+ results.append(item)+ return results++ results = asyncio.run(collect())++ get_field_map.assert_called_once_with([kb_id])+ use_sql.assert_awaited_once_with(+ "What is the total amount?",+ {"amount": "number"},+ tenant_id,+ chat,+ True,+ [kb_id],+ doc_ids=None,+ )+ retrieval.assert_not_awaited()+ assert results == [sql_result]+++def _run_field_map_empty_chat(monkeypatch, dialog_service, chunks, doc_aggs):+ tenant_id = "0123456789abcdef0123456789abcdef"+ kb_id = "abcdefabcdefabcdefabcdefabcdefab"+ get_field_map = Mock(return_value={})+ use_sql = AsyncMock()+ retrieval = AsyncMock(return_value={"total": len(chunks), "chunks": chunks, "doc_aggs": doc_aggs})+ retriever = types.SimpleNamespace(+ retrieval=retrieval,+ retrieval_by_children=lambda value, _tenant_ids: value,+ insert_citations=lambda answer, *_args, **_kwargs: (answer, set()),+ )+ chat = FakeChatModel(["fallback answer"])++ def sync_get_field_map(kb_ids):+ return get_field_map(kb_ids)++ monkeypatch.setattr(dialog_service.settings, "retriever", retriever, raising=False)+ monkeypatch.setattr(+ dialog_service,+ "KnowledgebaseService",+ types.SimpleNamespace(get_field_map=sync_get_field_map),+ raising=False,+ )+ monkeypatch.setattr(dialog_service, "use_sql", use_sql)+ monkeypatch.setattr(+ dialog_service,+ "get_tenant_default_model_by_type",+ lambda *_args, **_kwargs: {"model_type": "chat", "max_tokens": 8192},+ )+ monkeypatch.setattr(+ dialog_service,+ "get_models",+ lambda *_args, **_kwargs: ([types.SimpleNamespace(tenant_id=tenant_id)], object(), None, chat, None),+ )+ monkeypatch.setattr(dialog_service, "kb_prompt", lambda *_args, **_kwargs: ["content"])+ monkeypatch.setattr(dialog_service, "message_fit_in", lambda messages, _limit: (0, messages))+ monkeypatch.setattr(dialog_service, "tts", lambda *_args, **_kwargs: None)+ monkeypatch.setattr(dialog_service, "_resolve_reference_metadata", lambda *_args, **_kwargs: (False, []))+ monkeypatch.setattr(+ dialog_service,+ "TenantLangfuseService",+ types.SimpleNamespace(filter_by_tenant=lambda **_kwargs: None),+ raising=False,+ )+ dialog = types.SimpleNamespace(+ tenant_id=tenant_id,+ kb_ids=[kb_id],+ prompt_config={+ "quote": True,+ "parameters": [{"key": "knowledge", "optional": False}],+ "system": "Use {knowledge}",+ },+ llm_id=None,+ tenant_llm_id=None,+ top_n=8,+ top_k=32,+ similarity_threshold=0.2,+ vector_similarity_weight=0.3,+ meta_data_filter=None,+ llm_setting={},+ )++ async def collect():+ return [+ item+ async for item in dialog_service.async_chat(+ dialog,+ [{"role": "user", "content": "What is the content?"}],+ stream=False,+ )+ ]++ return asyncio.run(collect()), get_field_map, use_sql, retrieval, kb_id+++_USE_REAL_SQL = object()+++def _run_configured_chat(+ monkeypatch,+ dialog_service,+ *,+ retriever,+ chat,+ field_map=None,+ question="What is the total amount?",+ use_sql_mock=_USE_REAL_SQL,+ embd_mdl=None,+):+ tenant_id = "0123456789abcdef0123456789abcdef"+ kb_id = "abcdefabcdefabcdefabcdefabcdefab"+ if embd_mdl is None:+ embd_mdl = object()+ get_field_map = Mock(return_value=field_map or {"amount": "number"})+ monkeypatch.setattr(dialog_service.settings, "retriever", retriever, raising=False)+ monkeypatch.setattr(+ dialog_service,+ "KnowledgebaseService",+ types.SimpleNamespace(get_field_map=get_field_map),+ raising=False,+ )+ if use_sql_mock is not _USE_REAL_SQL:+ monkeypatch.setattr(dialog_service, "use_sql", use_sql_mock)+ monkeypatch.setattr(+ dialog_service,+ "get_tenant_default_model_by_type",+ lambda *_args, **_kwargs: {"model_type": "chat", "max_tokens": 8192},+ )+ monkeypatch.setattr(+ dialog_service,+ "get_models",+ lambda *_args, **_kwargs: ([types.SimpleNamespace(tenant_id=tenant_id)], embd_mdl, None, chat, None),+ )+ monkeypatch.setattr(dialog_service, "kb_prompt", lambda *_args, **_kwargs: ["fallback content"])+ monkeypatch.setattr(dialog_service, "message_fit_in", lambda messages, _limit: (0, messages))+ monkeypatch.setattr(dialog_service, "tts", lambda *_args, **_kwargs: None)+ monkeypatch.setattr(dialog_service, "_resolve_reference_metadata", lambda *_args, **_kwargs: (False, []))+ monkeypatch.setattr(+ dialog_service,+ "TenantLangfuseService",+ types.SimpleNamespace(filter_by_tenant=lambda **_kwargs: None),+ raising=False,+ )+ dialog = types.SimpleNamespace(+ tenant_id=tenant_id,+ kb_ids=[kb_id],+ prompt_config={+ "quote": True,+ "parameters": [{"key": "knowledge", "optional": False}],+ "system": "Use {knowledge}",+ },+ llm_id=None,+ tenant_llm_id=None,+ top_n=8,+ top_k=32,+ similarity_threshold=0.2,+ vector_similarity_weight=0.3,+ meta_data_filter=None,+ llm_setting={},+ )++ async def collect():+ return [+ item+ async for item in dialog_service.async_chat(+ dialog,+ [{"role": "user", "content": question}],+ stream=False,+ )+ ]++ return asyncio.run(collect()), get_field_map, kb_id+++def _expected_fallback_reference(kb_id):+ return {+ "total": 1,+ "chunks": [+ {+ "id": "chunk-1",+ "doc_id": "d1",+ "docnm_kwd": "doc.xlsx",+ "kb_id": kb_id,+ "content_with_weight": "fallback content",+ "content_ltks": "fallback content",+ "vector": [],+ }+ ],+ "doc_aggs": [{"doc_id": "d1", "doc_name": "doc.xlsx", "count": 1}],+ }+++def test_tc_sql_002_field_map_empty_uses_normal_retrieval(+ monkeypatch,+ dialog_service,+ enable_gaussdb_docengine,+ caplog,+):+ chunks = [+ {+ "id": "chunk-1",+ "doc_id": "doc-1",+ "docnm_kwd": "doc.xlsx",+ "kb_id": "abcdefabcdefabcdefabcdefabcdefab",+ "content_with_weight": "content",+ "content_ltks": "content",+ "vector": [],+ }+ ]+ doc_aggs = [{"doc_id": "doc-1", "doc_name": "doc.xlsx", "count": 1}]++ with caplog.at_level(logging.DEBUG):+ results, get_field_map, use_sql, retrieval, kb_id = _run_field_map_empty_chat(+ monkeypatch,+ dialog_service,+ chunks,+ doc_aggs,+ )++ get_field_map.assert_called_once_with([kb_id])+ use_sql.assert_not_awaited()+ retrieval.assert_awaited_once()+ assert retrieval.await_args.args[3] == [kb_id]+ assert results[-1]["answer"] == "fallback answer"+ assert results[-1]["reference"]["chunks"] == chunks+ assert results[-1]["reference"]["doc_aggs"] == doc_aggs+ assert "Use SQL to retrieval" not in caplog.text+++def test_tc_sql_004_use_sql_non_gaussdb_engine_does_not_call_gaussdb_prompt(monkeypatch, dialog_service):+ tenant_id = "0123456789abcdef0123456789abcdef"+ kb_id = "abcdefabcdefabcdefabcdefabcdefab"+ retriever = RecordingRetriever()++ monkeypatch.setattr(dialog_service.settings, "DOC_ENGINE_INFINITY", False, raising=False)+ monkeypatch.setattr(dialog_service.settings, "DOC_ENGINE_OCEANBASE", False, raising=False)+ monkeypatch.setattr(dialog_service.settings, "DOC_ENGINE_GAUSSDB", False, raising=False)+ monkeypatch.setattr(dialog_service.settings, "retriever", retriever, raising=False)+ gaussdb_prompt = Mock(side_effect=AssertionError("GaussDB prompt must not be used"))+ gaussdb_validator = Mock(side_effect=AssertionError("GaussDB validator must not be constructed"))+ monkeypatch.setattr(dialog_service.gaussdb_text_to_sql, "build_sql_prompt", gaussdb_prompt)+ monkeypatch.setattr(dialog_service.gaussdb_text_to_sql, "build_validator", gaussdb_validator)+ generated_sql = f"SELECT doc_id, docnm_kwd FROM ragflow_{tenant_id}"+ chat = FakeChatModel([generated_sql])++ result = asyncio.run(+ dialog_service.use_sql(+ "show docs",+ {"amount": "number"},+ tenant_id,+ chat,+ quota=False,+ kb_ids=[kb_id],+ )+ )++ expected_sys_prompt = """You are a Database Administrator. Write SQL queries.++RULES:+1. Use EXACT field names from the schema below (e.g., product_tks, not product)+2. Quote field names starting with digit: "123_field"+3. Add IS NOT NULL in WHERE clause when:+ - Question asks to "show me" or "display" specific columns+4. Include doc_id/docnm in non-aggregate statement+5. Output ONLY the SQL, no explanations"""+ expected_user_prompt = f"""Table: ragflow_{tenant_id}+Available fields:+ - amount (number)+Question: show docs+Write SQL using exact field names above. Include doc_id, docnm_kwd for data queries. Only SQL."""+ expected_sql = f"{generated_sql} WHERE kb_id = '{kb_id}'"++ gaussdb_prompt.assert_not_called()+ gaussdb_validator.assert_not_called()+ assert chat.calls == [+ (+ expected_sys_prompt,+ [{"role": "user", "content": expected_user_prompt}],+ {"temperature": 0.06},+ )+ ]+ assert retriever.sqls == [expected_sql]+ assert result == {+ "answer": "|number|Source|\n|------|------|\n|120| ##0$$|",+ "reference": {+ "chunks": [+ {+ "doc_id": "doc1",+ "docnm_kwd": "finance.csv",+ "kb_id": "abcdefabcdefabcdefabcdefabcdefab",+ }+ ],+ "doc_aggs": [{"doc_id": "doc1", "doc_name": "finance.csv", "count": 1}],+ },+ "prompt": expected_sys_prompt,+ }++ monkeypatch.setattr(dialog_service.settings, "DOC_ENGINE_INFINITY", True, raising=False)+ infinity_retriever = RecordingRetriever(fail_first=True, missing_source_once=True)+ monkeypatch.setattr(dialog_service.settings, "retriever", infinity_retriever, raising=False)+ infinity_table = f"ragflow_{tenant_id}_{kb_id}"+ infinity_sql = f"SELECT doc_id, docnm FROM {infinity_table}"+ infinity_chat = FakeChatModel([infinity_sql, infinity_sql, infinity_sql])++ infinity_result = asyncio.run(+ dialog_service.use_sql(+ "show docs",+ {"amount": "number"},+ tenant_id,+ infinity_chat,+ quota=False,+ kb_ids=[kb_id],+ )+ )++ assert len(infinity_chat.calls) == 3+ retry_prompt = infinity_chat.calls[1][1][0]["content"]+ source_repair_prompt = infinity_chat.calls[2][1][0]["content"]+ assert "json_extract_string(chunk_data, '$.field_name')" in retry_prompt+ assert "Correct the SQL for GaussDB" not in retry_prompt+ assert "missing required source columns" in source_repair_prompt+ assert "include doc_id and docnm in the SELECT list" in source_repair_prompt+ assert "json_extract_string(chunk_data, '$.field_name')" in source_repair_prompt+ assert "chunk_data #>>" not in source_repair_prompt+ assert infinity_retriever.sqls == [infinity_sql, infinity_sql, infinity_sql]+ assert infinity_result["reference"]["chunks"] == [+ {+ "doc_id": "doc1",+ "docnm_kwd": "finance.csv",+ "kb_id": "abcdefabcdefabcdefabcdefabcdefab",+ }+ ]+ gaussdb_prompt.assert_not_called()+ gaussdb_validator.assert_not_called()+++def test_tc_sql_601_use_sql_routes_gaussdb_prompt_through_retriever(monkeypatch, dialog_service, enable_gaussdb_docengine):+ tenant_id = "0123456789abcdef0123456789abcdef"+ kb_id = "abcdefabcdefabcdefabcdefabcdefab"+ table = f"ragflow_{tenant_id}"+ retriever = RecordingRetriever()+ monkeypatch.setattr(dialog_service.settings, "retriever", retriever, raising=False)+ chat = FakeChatModel([f"SELECT doc_id, docnm_kwd, chunk_data #>> '{{amount}}' AS amount FROM {table} WHERE kb_id = '{kb_id}' AND (chunk_data #>> '{{amount}}')::DOUBLE PRECISION > 100"])++ result = asyncio.run(+ dialog_service.use_sql(+ "finance amount greater than 100",+ {"amount": "number", "status": "string"},+ tenant_id,+ chat,+ quota=False,+ kb_ids=[kb_id],+ )+ )++ assert result is not None+ assert "120" in result["answer"]+ assert len(chat.calls) == 1+ system_prompt, messages, params = chat.calls[0]+ assert f"Table: {table}" in system_prompt+ assert " - amount (number): chunk_data #>> '{amount}'" in system_prompt+ assert " - status (string): chunk_data #>> '{status}'" in system_prompt+ assert "json_extract_string(chunk_data" not in system_prompt+ assert messages == [+ {+ "role": "user",+ "content": (+ f"Table: {table}\n"+ "Fields:\n"+ " - amount (number): chunk_data #>> '{amount}'\n"+ " - status (string): chunk_data #>> '{status}'\n"+ "Question: finance amount greater than 100\n"+ "Write SQL using GaussDB JSONB #>> / #> syntax. "+ "Include doc_id and docnm_kwd for data queries. Only SQL."+ ),+ }+ ]+ assert params == {"temperature": 0.06}+ assert retriever.sqls == [+ f"SELECT doc_id, docnm_kwd, chunk_data #>> '{{amount}}' AS amount FROM {table} WHERE kb_id = '{kb_id}' AND CAST((chunk_data #>> '{{amount}}') AS DOUBLE PRECISION) > 100 LIMIT 128"+ ]+++def test_tc_sql_601_gaussdb_injects_doc_scope_before_validator(monkeypatch, dialog_service, enable_gaussdb_docengine):+ tenant_id = "0123456789abcdef0123456789abcdef"+ kb_id = "abcdefabcdefabcdefabcdefabcdefab"+ doc_id = "12345678123456781234567812345678"+ table = f"ragflow_{tenant_id}"+ retriever = RecordingRetriever()+ monkeypatch.setattr(dialog_service.settings, "retriever", retriever, raising=False)+ chat = FakeChatModel([f"SELECT doc_id, docnm_kwd, chunk_data #>> '{{amount}}' AS amount FROM {table}"])++ result = asyncio.run(+ dialog_service.use_sql(+ "show the scoped document amount",+ {"amount": "number"},+ tenant_id,+ chat,+ quota=False,+ kb_ids=[kb_id],+ doc_ids=[doc_id],+ )+ )++ assert result is not None+ assert retriever.sqls == [f"SELECT doc_id, docnm_kwd, chunk_data #>> '{{amount}}' AS amount FROM {table} WHERE doc_id = '{doc_id}' AND kb_id = '{kb_id}' LIMIT 128"]+++def test_tc_sql_602_row_count_override_skips_llm_and_executes_scoped_count(+ monkeypatch,+ dialog_service,+ enable_gaussdb_docengine,+):+ tenant_id = "0123456789abcdef0123456789abcdef"+ kb_id = "abcdefabcdefabcdefabcdefabcdefab"+ table = f"ragflow_{tenant_id}"+ retriever = ScriptedRetriever(+ [+ {"columns": [{"name": "rows"}], "rows": [[5]]},+ {"columns": [{"name": "doc_id"}, {"name": "docnm_kwd"}], "rows": []},+ ]+ )+ monkeypatch.setattr(dialog_service.settings, "retriever", retriever, raising=False)+ chat = FakeChatModel([])++ result = asyncio.run(+ dialog_service.use_sql(+ "How many rows in the dataset?",+ {"amount": "number"},+ tenant_id,+ chat,+ quota=False,+ kb_ids=[kb_id],+ )+ )++ assert chat.calls == []+ assert retriever.sqls[0] == (f"SELECT COUNT(*) AS rows FROM {table} WHERE kb_id = '{kb_id}' LIMIT 128")+ assert retriever.sqls[1] == (f"SELECT doc_id, docnm_kwd FROM {table} WHERE kb_id = '{kb_id}' LIMIT 128")+ assert result["answer"] == "|rows|\n|------\n|5|"+ assert result["reference"] == {"chunks": [], "doc_aggs": []}+++def test_tc_sql_603_validator_rejection_builds_retry_prompt(+ monkeypatch,+ dialog_service,+ enable_gaussdb_docengine,+ caplog,+):+ tenant_id = "0123456789abcdef0123456789abcdef"+ kb_id = "abcdefabcdefabcdefabcdefabcdefab"+ table = f"ragflow_{tenant_id}"+ retriever = ScriptedRetriever(+ [+ {+ "columns": [{"name": "doc_id"}, {"name": "docnm_kwd"}, {"name": "amount"}],+ "rows": [["d1", "doc.xlsx", "120"]],+ }+ ]+ )+ monkeypatch.setattr(dialog_service.settings, "retriever", retriever, raising=False)+ chat = FakeChatModel(+ [+ f"SELECT * FROM {table}",+ f"SELECT doc_id, docnm_kwd, chunk_data #>> '{{amount}}' AS amount FROM {table}",+ ]+ )++ with caplog.at_level(logging.WARNING):+ result = asyncio.run(+ dialog_service.use_sql(+ "finance amount greater than 100",+ {"amount": "number", "dept": "string"},+ tenant_id,+ chat,+ quota=False,+ kb_ids=[kb_id],+ )+ )++ assert result["reference"]["chunks"] == [{"doc_id": "d1", "docnm_kwd": "doc.xlsx", "kb_id": kb_id}]+ assert result["answer"] == "|number|Source|\n|------|------|\n|120| ##0$$|"+ assert result["reference"]["doc_aggs"] == [{"doc_id": "d1", "doc_name": "doc.xlsx", "count": 1}]+ assert len(chat.calls) == 2+ assert len(retriever.sqls) == 1+ assert f"kb_id = '{kb_id}'" in retriever.sqls[0]+ retry_prompt = chat.calls[1][1][0]["content"]+ assert "SELECT * is not allowed" in retry_prompt+ assert "Initial SQL execution FAILED with error: SELECT * is not allowed" in caplog.text+ assert "Do not use json_extract, json_extract_string, or json_extract_isnull" in retry_prompt+ _assert_gaussdb_sql_rule_markers(retry_prompt)+++def test_tc_sql_604_execution_none_retries(+ monkeypatch,+ dialog_service,+ enable_gaussdb_docengine,+):+ tenant_id = "0123456789abcdef0123456789abcdef"+ kb_id = "abcdefabcdefabcdefabcdefabcdefab"+ table = f"ragflow_{tenant_id}"+ sql = f"SELECT doc_id, docnm_kwd, chunk_data #>> '{{amount}}' AS amount FROM {table}"+ retriever = ScriptedRetriever(+ [+ None,+ {+ "columns": [{"name": "doc_id"}, {"name": "docnm_kwd"}, {"name": "amount"}],+ "rows": [["d1", "doc.xlsx", "120"]],+ },+ ]+ )+ monkeypatch.setattr(dialog_service.settings, "retriever", retriever, raising=False)+ chat = FakeChatModel([sql, sql])++ result = asyncio.run(+ dialog_service.use_sql(+ "Show all amounts",+ {"amount": "number"},+ tenant_id,+ chat,+ quota=False,+ kb_ids=[kb_id],+ )+ )++ expected_sql = f"{sql} WHERE kb_id = '{kb_id}' LIMIT 128"+ assert len(chat.calls) == 2+ assert retriever.sqls == [expected_sql, expected_sql]+ assert result["answer"] == "|number|Source|\n|------|------|\n|120| ##0$$|"+ assert result["reference"] == {+ "chunks": [{"doc_id": "d1", "docnm_kwd": "doc.xlsx", "kb_id": kb_id}],+ "doc_aggs": [{"doc_id": "d1", "doc_name": "doc.xlsx", "count": 1}],+ }++[email protected]("mode", ["validator_rejected", "execution_failed"])+def test_tc_sql_605_retry_exhaustion_returns_none(+ mode,+ monkeypatch,+ dialog_service,+ enable_gaussdb_docengine,+ caplog,+):+ tenant_id = "0123456789abcdef0123456789abcdef"+ kb_id = "abcdefabcdefabcdefabcdefabcdefab"+ table = f"ragflow_{tenant_id}"+ if mode == "validator_rejected":+ chat = FakeChatModel([f"SELECT * FROM {table}", f"SELECT * FROM {table}"])+ retriever = ScriptedRetriever([])+ expected_sql_calls = 0+ else:+ valid_sql = f"SELECT doc_id, docnm_kwd FROM {table}"+ chat = FakeChatModel([valid_sql, valid_sql])+ retriever = ScriptedRetriever([None, None])+ expected_sql_calls = 2+ monkeypatch.setattr(dialog_service.settings, "retriever", retriever, raising=False)++ with caplog.at_level(logging.WARNING):+ result = asyncio.run(+ dialog_service.use_sql(+ "Show all amounts",+ {"amount": "number"},+ tenant_id,+ chat,+ quota=False,+ kb_ids=[kb_id],+ )+ )++ assert result is None+ assert len(chat.calls) == 2+ assert len(retriever.sqls) == expected_sql_calls+ if mode == "validator_rejected":+ assert "SELECT * is not allowed" in caplog.text+ assert "Retry SQL execution also FAILED" in caplog.text+++def test_tc_sql_606_retry_sql_is_revalidated_and_scoped(+ monkeypatch,+ dialog_service,+ enable_gaussdb_docengine,+):+ tenant_id = "0123456789abcdef0123456789abcdef"+ kb_id = "abcdefabcdefabcdefabcdefabcdefab"+ table = f"ragflow_{tenant_id}"+ retriever = ScriptedRetriever(+ [+ {+ "columns": [{"name": "doc_id"}, {"name": "docnm_kwd"}, {"name": "amount"}],+ "rows": [["d1", "doc.xlsx", "120"]],+ }+ ]+ )+ monkeypatch.setattr(dialog_service.settings, "retriever", retriever, raising=False)+ chat = FakeChatModel(+ [+ f"SELECT doc_id, chunk_data FROM {table}",+ f"SELECT doc_id, docnm_kwd, chunk_data #>> '{{amount}}' AS amount FROM {table}",+ ]+ )++ result = asyncio.run(+ dialog_service.use_sql(+ "Show all amounts",+ {"amount": "number"},+ tenant_id,+ chat,+ quota=False,+ kb_ids=[kb_id],+ )+ )++ assert len(chat.calls) == 2+ assert retriever.sqls == [f"SELECT doc_id, docnm_kwd, chunk_data #>> '{{amount}}' AS amount FROM {table} WHERE kb_id = '{kb_id}' LIMIT 128"]+ assert result["reference"]["chunks"][0]["kb_id"] == kb_id+++def test_tc_sql_607_missing_source_columns_trigger_dedicated_repair(+ monkeypatch,+ dialog_service,+ enable_gaussdb_docengine,+):+ tenant_id = "0123456789abcdef0123456789abcdef"+ kb_id = "abcdefabcdefabcdefabcdefabcdefab"+ table = f"ragflow_{tenant_id}"+ retriever = ScriptedRetriever(+ [+ {"columns": [{"name": "amount"}], "rows": [["120"]]},+ {+ "columns": [{"name": "doc_id"}, {"name": "docnm_kwd"}, {"name": "amount"}],+ "rows": [["d1", "doc.xlsx", "120"]],+ },+ ]+ )+ monkeypatch.setattr(dialog_service.settings, "retriever", retriever, raising=False)+ chat = FakeChatModel(+ [+ f"SELECT chunk_data #>> '{{amount}}' AS amount FROM {table}",+ f"SELECT doc_id, docnm_kwd, chunk_data #>> '{{amount}}' AS amount FROM {table}",+ ]+ )++ result = asyncio.run(+ dialog_service.use_sql(+ "Show all amounts",+ {"amount": "number"},+ tenant_id,+ chat,+ quota=False,+ kb_ids=[kb_id],+ )+ )++ assert len(chat.calls) == 2+ repair_prompt = chat.calls[1][1][0]["content"]+ assert "missing required source columns" in repair_prompt+ _assert_gaussdb_sql_rule_markers(repair_prompt)+ assert retriever.sqls == [+ f"SELECT chunk_data #>> '{{amount}}' AS amount FROM {table} WHERE kb_id = '{kb_id}' LIMIT 128",+ f"SELECT doc_id, docnm_kwd, chunk_data #>> '{{amount}}' AS amount FROM {table} WHERE kb_id = '{kb_id}' LIMIT 128",+ ]+ assert result["answer"] == "|number|Source|\n|------|------|\n|120| ##0$$|"+ assert result["reference"] == {+ "chunks": [{"doc_id": "d1", "docnm_kwd": "doc.xlsx", "kb_id": kb_id}],+ "doc_aggs": [{"doc_id": "d1", "doc_name": "doc.xlsx", "count": 1}],+ }+++def test_tc_sql_608_source_repair_failure_returns_best_effort_answer(+ monkeypatch,+ dialog_service,+ enable_gaussdb_docengine,+ caplog,+):+ tenant_id = "0123456789abcdef0123456789abcdef"+ kb_id = "abcdefabcdefabcdefabcdefabcdefab"+ table = f"ragflow_{tenant_id}"+ retriever = ScriptedRetriever(+ [+ {"columns": [{"name": "amount"}], "rows": [["120"]]},+ RuntimeError("source lookup failed"),+ ]+ )+ chat = FakeChatModel(+ [+ f"SELECT chunk_data #>> '{{amount}}' AS amount FROM {table}",+ f"SELECT doc_id, docnm_kwd, chunk_data #>> '{{amount}}' AS amount FROM {table}",+ ]+ )++ with caplog.at_level(logging.WARNING):+ results, get_field_map, actual_kb_id = _run_configured_chat(+ monkeypatch,+ dialog_service,+ retriever=retriever,+ chat=chat,+ question="Show all amounts",+ )++ assert actual_kb_id == kb_id+ get_field_map.assert_called_once_with([kb_id])+ assert len(chat.calls) == 2+ assert retriever.sqls == [+ f"SELECT chunk_data #>> '{{amount}}' AS amount FROM {table} WHERE kb_id = '{kb_id}' LIMIT 128",+ f"SELECT doc_id, docnm_kwd, chunk_data #>> '{{amount}}' AS amount FROM {table} WHERE kb_id = '{kb_id}' LIMIT 128",+ ]+ retriever.retrieval.assert_not_awaited()+ assert len(results) == 1+ assert results[0]["answer"] == "|number|\n|------\n|120|"+ assert results[0]["reference"] == {"chunks": [], "doc_aggs": []}+ _assert_gaussdb_sql_rule_markers(results[0]["prompt"])+ assert "Source-column SQL repair failed" in caplog.text+ assert "source lookup failed" in caplog.text+ assert "SQL failed or returned no results, falling back to vector search" not in caplog.text+++def test_tc_sql_701_use_sql_aggregate_fallback_fetches_sources_with_kb_scope(monkeypatch, dialog_service, enable_gaussdb_docengine):+ tenant_id = "0123456789abcdef0123456789abcdef"+ kb_id = "abcdefabcdefabcdefabcdefabcdefab"+ table = f"ragflow_{tenant_id}"+ retriever = AggregateRecordingRetriever()+ monkeypatch.setattr(dialog_service.settings, "retriever", retriever, raising=False)+ chat = FakeChatModel([f"SELECT SUM((chunk_data #>> '{{amount}}')::DOUBLE PRECISION) AS total_amount FROM {table} WHERE chunk_data #>> '{{dept}}' = 'finance' LIMIT 3"])++ result = asyncio.run(+ dialog_service.use_sql(+ "sum finance amount",+ {"amount": "number", "dept": "string"},+ tenant_id,+ chat,+ quota=False,+ kb_ids=[kb_id],+ )+ )++ assert result is not None+ assert len(retriever.sqls) == 2+ assert retriever.sqls[0].endswith("LIMIT 3")+ source_sql = retriever.sqls[1]+ assert source_sql.startswith(f"SELECT doc_id, docnm_kwd FROM {table} WHERE ")+ assert "chunk_data #>> '{dept}' = 'finance'" in source_sql+ assert f"kb_id = '{kb_id}'" in source_sql+ assert source_sql.endswith("LIMIT 128")+ assert "LIMIT 20" not in source_sql+ assert result["reference"]["chunks"][0]["kb_id"] == kb_id+++def test_tc_sql_701_aggregate_source_builder_projects_multikb_scope_and_rejects_having(dialog_service):+ source_sql = dialog_service.gaussdb_text_to_sql.build_aggregate_source_sql(+ "SELECT kb_id, SUM((chunk_data #>> '{amount}')::DOUBLE PRECISION) AS total FROM ragflow_tenant WHERE kb_id IN ('k1', 'k2') GROUP BY kb_id ORDER BY total DESC LIMIT 3 OFFSET 1",+ "docnm_kwd",+ include_kb_id=True,+ )++ assert source_sql == "SELECT doc_id, docnm_kwd, kb_id FROM ragflow_tenant WHERE kb_id IN ('k1', 'k2')"+ reference = dialog_service.gaussdb_text_to_sql.build_source_reference(+ {+ "columns": [{"name": "doc_id"}, {"name": "docnm_kwd"}, {"name": "kb_id"}],+ "rows": [["d1", "one.xlsx", "k1"], ["d2", "two.xlsx", "k2"]],+ },+ ["k1", "k2"],+ )+ assert reference == {+ "chunks": [+ {"doc_id": "d1", "docnm_kwd": "one.xlsx", "kb_id": "k1"},+ {"doc_id": "d2", "docnm_kwd": "two.xlsx", "kb_id": "k2"},+ ],+ "doc_aggs": [+ {"doc_id": "d1", "doc_name": "one.xlsx", "count": 1},+ {"doc_id": "d2", "doc_name": "two.xlsx", "count": 1},+ ],+ }+ with pytest.raises(ValueError) as exc_info:+ dialog_service.gaussdb_text_to_sql.build_aggregate_source_sql(+ "SELECT SUM((chunk_data #>> '{amount}')::DOUBLE PRECISION) AS total FROM ragflow_tenant GROUP BY kb_id HAVING SUM((chunk_data #>> '{amount}')::DOUBLE PRECISION) > 0",+ "docnm_kwd",+ include_kb_id=False,+ )+ assert str(exc_info.value) == "GaussDB aggregate source lookup cannot preserve HAVING safely"+++ ("projection", "value"),+ [+ ("'sum(amount)' AS note", "sum(amount)"),+ ("1 AS note /* sum(amount) */", 1),+ ],+ ids=["literal", "comment"],+)+def test_tc_sql_705_use_sql_does_not_treat_aggregate_names_in_literals_or_comments_as_aggregates(+ projection,+ value,+ monkeypatch,+ dialog_service,+ enable_gaussdb_docengine,+):+ tenant_id = "0123456789abcdef0123456789abcdef"+ kb_id = "abcdefabcdefabcdefabcdefabcdefab"+ table = f"ragflow_{tenant_id}"+ retriever = ScriptedRetriever(+ [+ {"columns": [{"name": "note"}], "rows": [[value]]},+ {+ "columns": [{"name": "doc_id"}, {"name": "docnm_kwd"}, {"name": "note"}],+ "rows": [["d1", "doc.xlsx", value]],+ },+ ]+ )+ monkeypatch.setattr(dialog_service.settings, "retriever", retriever, raising=False)+ chat = FakeChatModel(+ [+ f"SELECT {projection} FROM {table}",+ f"SELECT doc_id, docnm_kwd, {projection} FROM {table}",+ ]+ )++ result = asyncio.run(+ dialog_service.use_sql(+ "show the note",+ {"note": "string"},+ tenant_id,+ chat,+ quota=False,+ kb_ids=[kb_id],+ )+ )++ assert len(chat.calls) == 2+ assert len(retriever.sqls) == 2+ repair_prompt = chat.calls[1][1][0]["content"]+ assert "missing required source columns" in repair_prompt+ _assert_gaussdb_sql_rule_markers(repair_prompt)+ assert result["reference"]["chunks"] == [{"doc_id": "d1", "docnm_kwd": "doc.xlsx", "kb_id": kb_id}]+++def test_tc_sql_005_use_sql_requires_kb_ids_for_gaussdb(monkeypatch, dialog_service, enable_gaussdb_docengine):+ validator = Mock()+ monkeypatch.setattr(dialog_service.gaussdb_text_to_sql, "build_validator", validator)++ with pytest.raises(ValueError) as exc_info:+ asyncio.run(+ dialog_service.use_sql(+ "show amount",+ {"amount": "number"},+ "0123456789abcdef0123456789abcdef",+ FakeChatModel([]),+ quota=False,+ kb_ids=[],+ )+ )++ assert str(exc_info.value) == "GaussDB Text-to-SQL requires kb_ids"+ validator.assert_not_called()+++def test_tc_sql_703_aggregate_source_lookup_failure_returns_empty_chunks(+ monkeypatch,+ dialog_service,+ enable_gaussdb_docengine,+ caplog,+):+ tenant_id = "0123456789abcdef0123456789abcdef"+ kb_id = "abcdefabcdefabcdefabcdefabcdefab"+ table = f"ragflow_{tenant_id}"+ assert (+ dialog_service.gaussdb_text_to_sql.build_source_reference(+ {"columns": [{"name": "total_amount"}], "rows": [["120"]]},+ [kb_id],+ )+ is None+ )++ class AggregateSourceFailureRetriever:+ def __init__(self):+ self.sqls = []++ def sql_retrieval(self, sql, format="json"):+ assert format == "json"+ self.sqls.append(sql)+ if len(self.sqls) == 1:+ return {"columns": [{"name": "total_amount"}], "rows": [["120"]]}+ raise RuntimeError("source lookup failed")++ retriever = AggregateSourceFailureRetriever()+ monkeypatch.setattr(dialog_service.settings, "retriever", retriever, raising=False)+ chat = FakeChatModel([f"SELECT SUM((chunk_data #>> '{{amount}}')::DOUBLE PRECISION) AS total_amount FROM {table} WHERE kb_id = '{kb_id}'"])++ with caplog.at_level(logging.WARNING):+ result = asyncio.run(+ dialog_service.use_sql(+ "What is the total amount?",+ {"amount": "number"},+ tenant_id,+ chat,+ quota=False,+ kb_ids=[kb_id],+ )+ )++ assert len(retriever.sqls) == 2+ assert result == {+ "answer": "|total_number|\n|------\n|120|",+ "reference": {"chunks": [], "doc_aggs": []},+ "prompt": chat.calls[0][0],+ }+ assert "Failed to fetch chunks" in caplog.text+++def test_tc_sql_801_use_sql_completes_multikb_reference_kb_id(monkeypatch, dialog_service, enable_gaussdb_docengine):+ tenant_id = "0123456789abcdef0123456789abcdef"+ kb1 = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"+ kb2 = "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"+ table = f"ragflow_{tenant_id}"+ retriever = MultiKBReferenceRetriever()+ monkeypatch.setattr(dialog_service.settings, "retriever", retriever, raising=False)+ chat = FakeChatModel([f"SELECT doc_id, docnm_kwd, chunk_data #>> '{{amount}}' AS amount FROM {table} WHERE kb_id IN ('{kb1}', '{kb2}')"])++ result = asyncio.run(+ dialog_service.use_sql(+ "show amount",+ {"amount": "number"},+ tenant_id,+ chat,+ quota=False,+ kb_ids=[kb1, kb2],+ )+ )++ assert result["reference"]["chunks"] == [+ {"doc_id": "doc1", "docnm_kwd": "finance-a.csv", "kb_id": kb1},+ {"doc_id": "doc2", "docnm_kwd": "finance-b.csv", "kb_id": kb1},+ {"doc_id": "doc3", "docnm_kwd": "finance-c.csv", "kb_id": kb2},+ ]+ assert len(retriever.sqls) == 2+ assert retriever.sqls[1] == (f"SELECT doc_id, kb_id FROM {table} WHERE doc_id IN ('doc1', 'doc2', 'doc3') AND kb_id IN ('{kb1}', '{kb2}') LIMIT 128")+++ ("lookup_result", "expects_warning"),+ [+ (RuntimeError("lookup failed"), True),+ ({"columns": [{"name": "doc_id"}], "rows": [["doc1"]]}, False),+ ({"columns": [{"name": "doc_id"}, {"name": "kb_id"}], "rows": []}, False),+ ({"columns": [{"name": "doc_id"}, {"name": "kb_id"}], "rows": [["doc1"]]}, False),+ ],+ ids=["exception", "missing-kb-column", "empty-rows", "short-row"],+)+def test_tc_sql_803_multikb_lookup_failures_preserve_reference(+ lookup_result,+ expects_warning,+ monkeypatch,+ dialog_service,+ enable_gaussdb_docengine,+ caplog,+):+ tenant_id = "0123456789abcdef0123456789abcdef"+ kb1 = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"+ kb2 = "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"+ table = f"ragflow_{tenant_id}"+ retriever = ScriptedRetriever(+ [+ {+ "columns": [{"name": "doc_id"}, {"name": "docnm_kwd"}, {"name": "amount"}],+ "rows": [["doc1", "finance.csv", "120"]],+ },+ lookup_result,+ ]+ )+ monkeypatch.setattr(dialog_service.settings, "retriever", retriever, raising=False)+ chat = FakeChatModel([f"SELECT doc_id, docnm_kwd, chunk_data #>> '{{amount}}' AS amount FROM {table} WHERE kb_id IN ('{kb1}', '{kb2}')"])++ with caplog.at_level(logging.WARNING):+ result = asyncio.run(+ dialog_service.use_sql(+ "show amount",+ {"amount": "number"},+ tenant_id,+ chat,+ quota=False,+ kb_ids=[kb1, kb2],+ )+ )++ assert result["reference"]["chunks"] == [{"doc_id": "doc1", "docnm_kwd": "finance.csv"}]+ assert len(retriever.sqls) == 2+ if expects_warning:+ assert "Failed to complete GaussDB reference kb_id values" in caplog.text+ else:+ assert "Failed to complete GaussDB reference kb_id values" not in caplog.text+++def test_tc_sql_804_multikb_lookup_is_validated_and_scoped(+ monkeypatch,+ dialog_service,+ enable_gaussdb_docengine,+):+ tenant_id = "0123456789abcdef0123456789abcdef"+ kb1 = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"+ kb2 = "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"+ table = f"ragflow_{tenant_id}"+ retriever = ScriptedRetriever(+ [+ {+ "columns": [{"name": "doc_id"}, {"name": "docnm_kwd"}, {"name": "amount"}],+ "rows": [["doc1", "finance.csv", "120"]],+ },+ {+ "columns": [{"name": "doc_id"}, {"name": "kb_id"}],+ "rows": [["doc1", kb2]],+ },+ ]+ )+ monkeypatch.setattr(dialog_service.settings, "retriever", retriever, raising=False)+ chat = FakeChatModel([f"SELECT doc_id, docnm_kwd, chunk_data #>> '{{amount}}' AS amount FROM {table} WHERE kb_id IN ('{kb1}', '{kb2}')"])++ result = asyncio.run(+ dialog_service.use_sql(+ "show amount",+ {"amount": "number"},+ tenant_id,+ chat,+ quota=False,+ kb_ids=[kb1, kb2],+ )+ )++ assert retriever.sqls[1] == (f"SELECT doc_id, kb_id FROM {table} WHERE doc_id IN ('doc1') AND kb_id IN ('{kb1}', '{kb2}') LIMIT 128")+ assert result["reference"]["chunks"][0]["kb_id"] == kb2+++def test_tc_sql_805_use_sql_uses_inline_multikb_reference_kb_id(monkeypatch, dialog_service, enable_gaussdb_docengine):+ tenant_id = "0123456789abcdef0123456789abcdef"+ kb1 = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"+ kb2 = "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"+ table = f"ragflow_{tenant_id}"+ retriever = MultiKBReferenceWithInlineKB()+ monkeypatch.setattr(dialog_service.settings, "retriever", retriever, raising=False)+ chat = FakeChatModel([f"SELECT doc_id, docnm_kwd, kb_id FROM {table} WHERE kb_id IN ('{kb1}', '{kb2}')"])++ result = asyncio.run(+ dialog_service.use_sql(+ "show amount",+ {"amount": "number"},+ tenant_id,+ chat,+ quota=False,+ kb_ids=[kb1, kb2],+ )+ )++ assert result["reference"]["chunks"][0]["kb_id"] == kb1+ assert len(retriever.sqls) == 1+++def test_tc_ret_1001_use_sql_validator_rejects_before_sql_retrieval(+ monkeypatch,+ dialog_service,+ enable_gaussdb_docengine,+ caplog,+):+ tenant_id = "0123456789abcdef0123456789abcdef"+ kb_id = "00000000000000000000000000000001"+ other_kb_id = "00000000000000000000000000000002"+ table = f"ragflow_{tenant_id}"+ invalid_sql = f"SELECT doc_id, docnm_kwd, chunk_data #>> '{{amount}}' AS amount FROM {table} WHERE kb_id = '{other_kb_id}'"+ chat = types.SimpleNamespace(async_chat=AsyncMock(side_effect=[invalid_sql, invalid_sql]))+ sql_retrieval = Mock(side_effect=AssertionError("validator must reject before SQL execution"))++ monkeypatch.setattr(+ dialog_service.settings,+ "retriever",+ types.SimpleNamespace(sql_retrieval=sql_retrieval),+ raising=False,+ )++ with caplog.at_level(logging.WARNING):+ result = asyncio.run(+ dialog_service.use_sql(+ "show amount",+ {"amount": "number"},+ tenant_id,+ chat,+ quota=True,+ kb_ids=[kb_id],+ )+ )++ assert result is None+ assert chat.async_chat.await_count == 2+ sql_retrieval.assert_not_called()+ assert "SQL crosses the allowed kb_id boundary" in caplog.text+++def test_tc_sql_1001_use_sql_none_falls_back_to_retrieval(+ monkeypatch,+ dialog_service,+ enable_gaussdb_docengine,+ caplog,+):+ tenant_id = "0123456789abcdef0123456789abcdef"+ kb_id = "abcdefabcdefabcdefabcdefabcdefab"+ question = "What is the total amount?"+ embd_mdl = object()+ retriever = ScriptedRetriever([])+ use_sql = AsyncMock(return_value=None)+ chat = FakeChatModel(["fallback answer"])++ with caplog.at_level(logging.DEBUG):+ results, get_field_map, actual_kb_id = _run_configured_chat(+ monkeypatch,+ dialog_service,+ retriever=retriever,+ chat=chat,+ question=question,+ use_sql_mock=use_sql,+ embd_mdl=embd_mdl,+ )++ assert actual_kb_id == kb_id+ get_field_map.assert_called_once_with([kb_id])+ use_sql.assert_awaited_once_with(+ question,+ {"amount": "number"},+ tenant_id,+ chat,+ True,+ [kb_id],+ doc_ids=None,+ )+ retriever.retrieval.assert_awaited_once_with(+ question,+ embd_mdl,+ [tenant_id],+ [kb_id],+ 1,+ 8,+ 0.2,+ 0.3,+ doc_ids=None,+ top=32,+ aggs=True,+ rerank_mdl=None,+ rank_feature=None,+ )+ assert results[-1]["answer"] == "fallback answer"+ assert results[-1]["reference"] == _expected_fallback_reference(kb_id)+ assert "SQL failed or returned no results, falling back to vector search" in caplog.text+++def test_tc_sql_1002_validator_rejection_falls_back_to_retrieval(+ monkeypatch,+ dialog_service,+ enable_gaussdb_docengine,+ caplog,+):+ tenant_id = "0123456789abcdef0123456789abcdef"+ kb_id = "abcdefabcdefabcdefabcdefabcdefab"+ question = "What is the total amount?"+ embd_mdl = object()+ invalid_sql = "SELECT doc_id FROM other_tenant.ragflow_t1"+ retriever = ScriptedRetriever([])+ chat = FakeChatModel([invalid_sql, invalid_sql, "fallback answer"])++ with caplog.at_level(logging.DEBUG):+ results, get_field_map, actual_kb_id = _run_configured_chat(+ monkeypatch,+ dialog_service,+ retriever=retriever,+ chat=chat,+ question=question,+ embd_mdl=embd_mdl,+ )++ assert actual_kb_id == kb_id+ get_field_map.assert_called_once_with([kb_id])+ assert len(chat.calls) == 3+ assert retriever.sqls == []+ retriever.retrieval.assert_awaited_once_with(+ question,+ embd_mdl,+ [tenant_id],+ [kb_id],+ 1,+ 8,+ 0.2,+ 0.3,+ doc_ids=None,+ top=32,+ aggs=True,+ rerank_mdl=None,+ rank_feature=None,+ )+ assert results[-1]["answer"] == "fallback answer"+ assert results[-1]["reference"] == _expected_fallback_reference(kb_id)+ assert "cross-schema SQL is not allowed" in caplog.text+ assert "SQL failed or returned no results, falling back to vector search" in caplog.text+++def test_tc_sql_1003_sql_timeout_falls_back_to_retrieval(+ monkeypatch,+ dialog_service,+ enable_gaussdb_docengine,+ caplog,+):+ tenant_id = "0123456789abcdef0123456789abcdef"+ kb_id = "abcdefabcdefabcdefabcdefabcdefab"+ question = "What is the total amount?"+ embd_mdl = object()+ table = f"ragflow_{tenant_id}"+ valid_sql = f"SELECT doc_id, docnm_kwd, chunk_data #>> '{{amount}}' AS amount FROM {table}"+ retriever = ScriptedRetriever([TimeoutError("query timeout"), TimeoutError("query timeout")])+ chat = FakeChatModel([valid_sql, valid_sql, "fallback answer"])++ with caplog.at_level(logging.DEBUG):+ results, get_field_map, actual_kb_id = _run_configured_chat(+ monkeypatch,+ dialog_service,+ retriever=retriever,+ chat=chat,+ question=question,+ embd_mdl=embd_mdl,+ )++ assert actual_kb_id == kb_id+ get_field_map.assert_called_once_with([kb_id])+ expected_sql = f"{valid_sql} WHERE kb_id = '{kb_id}' LIMIT 128"+ assert retriever.sqls == [expected_sql, expected_sql]+ retriever.retrieval.assert_awaited_once_with(+ question,+ embd_mdl,+ [tenant_id],+ [kb_id],+ 1,+ 8,+ 0.2,+ 0.3,+ doc_ids=None,+ top=32,+ aggs=True,+ rerank_mdl=None,+ rank_feature=None,+ )+ assert results[-1]["answer"] == "fallback answer"+ assert results[-1]["reference"] == _expected_fallback_reference(kb_id)+ assert "query timeout" in caplog.text+ assert "SQL failed or returned no results, falling back to vector search" in caplog.text+++def test_tc_sql_1004_aggregate_answer_survives_source_lookup_failure(+ monkeypatch,+ dialog_service,+ enable_gaussdb_docengine,+ caplog,+):+ tenant_id = "0123456789abcdef0123456789abcdef"+ table = f"ragflow_{tenant_id}"+ retriever = ScriptedRetriever(+ [+ {"columns": [{"name": "total"}], "rows": [[450]]},+ RuntimeError("source lookup failed"),+ ]+ )+ chat = FakeChatModel([f"SELECT SUM(CAST(chunk_data #>> '{{amount}}' AS DOUBLE PRECISION)) AS total FROM {table}"])++ with caplog.at_level(logging.WARNING):+ results, _get_field_map, _kb_id = _run_configured_chat(+ monkeypatch,+ dialog_service,+ retriever=retriever,+ chat=chat,+ )++ retriever.retrieval.assert_not_awaited()+ assert results[-1]["answer"] == "|total|\n|------\n|450|"+ assert results[-1]["reference"] == {"chunks": [], "doc_aggs": []}+ assert "Failed to fetch chunks" in caplog.text+ assert "source lookup failed" in caplog.text+ assert "SQL failed or returned no results, falling back to vector search" not in caplog.text+++def test_tc_sql_1005_nonempty_answer_with_empty_chunks_is_returned(+ monkeypatch,+ dialog_service,+ enable_gaussdb_docengine,+ caplog,+):+ sql_result = {+ "answer": "|total|\n|------|\n|450|",+ "reference": {"chunks": [], "doc_aggs": []},+ }+ retriever = ScriptedRetriever([])+ use_sql = AsyncMock(return_value=sql_result)+ chat = FakeChatModel([])++ with caplog.at_level(logging.DEBUG):+ results, _get_field_map, _kb_id = _run_configured_chat(+ monkeypatch,+ dialog_service,+ retriever=retriever,+ chat=chat,+ use_sql_mock=use_sql,+ )++ use_sql.assert_awaited_once_with(+ "What is the total amount?",+ {"amount": "number"},+ "0123456789abcdef0123456789abcdef",+ chat,+ True,+ ["abcdefabcdefabcdefabcdefabcdefab"],+ doc_ids=None,+ )+ retriever.retrieval.assert_not_awaited()+ assert results == [sql_result]+ assert "use_sql: No rows returned" not in caplog.textdiff --git a/test/unit_test/api/db/services/test_gaussdb_doc_metadata_service.py b/test/unit_test/api/db/services/test_gaussdb_doc_metadata_service.pynew file mode 100644index 0000000000..0c942bfaf6--- /dev/null+++ b/test/unit_test/api/db/services/test_gaussdb_doc_metadata_service.py@@ -0,0 +1,601 @@+#+# Copyright 2025 The InfiniFlow Authors. All Rights Reserved.+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+#+import importlib+import json+import sys+import types+from contextlib import suppress+from pathlib import Path++import pytest+import psycopg2+++_MISSING = object()+_OPTIONAL_STUB_MODULES = (+ "google",+ "google.cloud",+ "google.cloud.storage",+ "google.api_core",+ "google.api_core.exceptions",+ "json_repair",+ "rag.utils.es_conn",+ "rag.utils.infinity_conn",+ "rag.utils.ob_conn",+ "rag.utils.opensearch_conn",+ "rag.utils.azure_sas_conn",+ "rag.utils.azure_spn_conn",+ "rag.utils.gcs_conn",+ "rag.utils.minio_conn",+ "rag.utils.opendal_conn",+ "rag.utils.redis_conn",+ "rag.utils.s3_conn",+ "rag.utils.oss_conn",+ "memory.utils.es_conn",+ "memory.utils.infinity_conn",+ "memory.utils.ob_conn",+)+_LOADED_MODULES = (+ "common.settings",+ "api.db.db_models",+ "api.db.services.doc_metadata_service",+)+++def _module(name, **attrs):+ mod = types.ModuleType(name)+ for key, value in attrs.items():+ setattr(mod, key, value)+ return mod+++class FakeDB:+ @staticmethod+ def connection_context():+ def decorator(func):+ func.__wrapped__ = func+ return func++ return decorator+++class FakeField:+ def __eq__(self, _other):+ return self+++def _install_optional_import_stubs(monkeypatch):+ class StubClient:+ pass++ class StubNotFound(Exception):+ pass++ class StubDocEngineConnection:+ def db_type(self):+ return "stub"++ class StubStorage:+ def health(self):+ return True++ google = _module("google")+ google_cloud = _module("google.cloud")+ google_storage = _module("google.cloud.storage", Client=StubClient)+ google_api_core = _module("google.api_core")+ google_api_exceptions = _module("google.api_core.exceptions", NotFound=StubNotFound)+ google_cloud.storage = google_storage+ google_api_core.exceptions = google_api_exceptions+ google.cloud = google_cloud+ google.api_core = google_api_core+ for name, mod in (+ ("google", google),+ ("google.cloud", google_cloud),+ ("google.cloud.storage", google_storage),+ ("google.api_core", google_api_core),+ ("google.api_core.exceptions", google_api_exceptions),+ ):+ monkeypatch.setitem(sys.modules, name, mod)++ import rag.utils+ import memory.utils++ rag_modules = {+ "es_conn": {"ESConnection": StubDocEngineConnection},+ "infinity_conn": {"InfinityConnection": StubDocEngineConnection},+ "ob_conn": {"OBConnection": StubDocEngineConnection},+ "opensearch_conn": {"OSConnection": StubDocEngineConnection},+ "azure_sas_conn": {"RAGFlowAzureSasBlob": StubStorage},+ "azure_spn_conn": {"RAGFlowAzureSpnBlob": StubStorage},+ "gcs_conn": {"RAGFlowGCS": StubStorage},+ "minio_conn": {"RAGFlowMinio": StubStorage},+ "opendal_conn": {"OpenDALStorage": StubStorage},+ "redis_conn": {"REDIS_CONN": types.SimpleNamespace(health=lambda: True)},+ "s3_conn": {"RAGFlowS3": StubStorage},+ "oss_conn": {"RAGFlowOSS": StubStorage},+ }+ for short_name, attrs in rag_modules.items():+ mod = _module(f"rag.utils.{short_name}", **attrs)+ monkeypatch.setitem(sys.modules, f"rag.utils.{short_name}", mod)+ monkeypatch.setattr(rag.utils, short_name, mod, raising=False)++ for short_name in ("es_conn", "infinity_conn", "ob_conn"):+ mod = _module(+ f"memory.utils.{short_name}",+ ESConnection=StubDocEngineConnection,+ InfinityConnection=StubDocEngineConnection,+ OBConnection=StubDocEngineConnection,+ )+ monkeypatch.setitem(sys.modules, f"memory.utils.{short_name}", mod)+ monkeypatch.setattr(memory.utils, short_name, mod, raising=False)++ monkeypatch.setitem(sys.modules, "json_repair", _module("json_repair", loads=json.loads))+++def doc_metadata_context(monkeypatch):+ tracked = (*_OPTIONAL_STUB_MODULES, *_LOADED_MODULES)+ original_modules = {name: sys.modules.get(name, _MISSING) for name in tracked}++ _install_optional_import_stubs(monkeypatch)+ import common++ fake_settings = _module(+ "common.settings",+ docStoreConn=None,+ DOC_ENGINE_GAUSSDB=False,+ DOC_ENGINE_INFINITY=False,+ DOC_ENGINE_OCEANBASE=False,+ )+ monkeypatch.setitem(sys.modules, "common.settings", fake_settings)+ monkeypatch.setattr(common, "settings", fake_settings, raising=False)+ db_models = _module(+ "api.db.db_models",+ DB=FakeDB(),+ Document=types.SimpleNamespace(+ id=FakeField(),+ kb_id=FakeField(),+ select=staticmethod(lambda *_args, **_kwargs: None),+ ),+ Knowledgebase=types.SimpleNamespace(+ id=FakeField(),+ tenant_id=FakeField(),+ get_by_id=staticmethod(lambda *_args, **_kwargs: None),+ ),+ )+ monkeypatch.setitem(sys.modules, "api.db.db_models", db_models)+ repo_root = Path(__file__).resolve().parents[5]+ spec = importlib.util.spec_from_file_location(+ "api.db.services.doc_metadata_service",+ repo_root / "api" / "db" / "services" / "doc_metadata_service.py",+ )+ module = importlib.util.module_from_spec(spec)+ monkeypatch.setitem(sys.modules, "api.db.services.doc_metadata_service", module)+ spec.loader.exec_module(module)+ monkeypatch.setattr(module.settings, "docStoreConn", None, raising=False)+ search_result_cls = importlib.import_module("rag.utils.gaussdb_conn").SearchResult++ try:+ yield types.SimpleNamespace(+ module=module,+ DocMetadataService=module.DocMetadataService,+ settings=module.settings,+ SearchResult=search_result_cls,+ )+ finally:+ services_pkg = sys.modules.get("api.db.services")+ if original_modules["api.db.services.doc_metadata_service"] is _MISSING and services_pkg is not None:+ with suppress(AttributeError):+ delattr(services_pkg, "doc_metadata_service")+ for name, original in original_modules.items():+ if original is _MISSING:+ sys.modules.pop(name, None)+ else:+ sys.modules[name] = original+++class FakeGaussDBStore:+ def __init__(self, *, exists=True, create_result=True, create_error=None, insert_result=None):+ self.created_meta = []+ self.deleted = []+ self.inserted = []+ self.executed = []+ self.existence_checks = []+ self.exists = exists+ self.create_result = create_result+ self.create_error = create_error+ self.insert_result = [] if insert_result is None else insert_result++ def db_type(self):+ return "gaussdb"++ def create_doc_meta_idx(self, index_name):+ self.created_meta.append(index_name)+ if self.create_error is not None:+ raise self.create_error+ return self.create_result++ def index_exist(self, *args):+ self.existence_checks.append(args)+ return self.exists++ def get(self, *_args):+ return {"id": "doc1", "kb_id": "kb1", "meta_fields": {"old": "value"}}++ def delete(self, condition, index_name, kb_id):+ self.deleted.append((condition, index_name, kb_id))+ return 1++ def insert(self, rows, index_name, kb_id):+ self.inserted.append((rows, index_name, kb_id))+ return self.insert_result++ def fetch_metadata_doc_ids(self, index_name, kb_ids, sql_filter, filter_params, limit):+ self.executed.append((index_name, kb_ids, sql_filter, filter_params, limit))+ return ["doc-risk"]+++class FakeDocQuery:+ def join(self, *_args, **_kwargs):+ return self++ def where(self, *_args, **_kwargs):+ return self++ def first(self):+ return types.SimpleNamespace(+ kb_id="kb1",+ knowledgebase=types.SimpleNamespace(tenant_id="tenant1"),+ )+++class FakeSearchResultStore(FakeGaussDBStore):+ def __init__(self, search_result_cls):+ super().__init__()+ self.search_result_cls = search_result_cls++ def search(self, **kwargs):+ self.executed.append(kwargs)+ return self.search_result_cls(+ total=1,+ chunks=[{"id": "doc1", "kb_id": "kb1", "meta_fields": {"author": "Alice"}}],+ )+++class PagingSearchResultStore(FakeGaussDBStore):+ def __init__(self, search_result_cls):+ super().__init__()+ self.search_result_cls = search_result_cls++ def search(self, **kwargs):+ self.executed.append(kwargs)+ offset = kwargs["offset"]+ limit = kwargs["limit"]+ if offset == 0:+ chunks = [{"id": f"doc{i}", "kb_id": "kb1", "meta_fields": {"seq": i}} for i in range(limit)]+ elif offset == limit:+ chunks = [{"id": "doc1000", "kb_id": "kb1", "meta_fields": {"seq": 1000}}]+ else:+ chunks = []+ return self.search_result_cls(total=1001, chunks=chunks)+++def test_tc_ret_407_iter_search_results_accepts_gaussdb_search_result_shape(doc_metadata_context):+ service = doc_metadata_context.DocMetadataService+ result = doc_metadata_context.SearchResult(total=1, chunks=[{"id": "doc1", "meta_fields": {"author": "Alice"}}])++ docs = list(service._iter_search_results(result))++ assert docs == [("doc1", {"id": "doc1", "meta_fields": {"author": "Alice"}})]+++def test_tc_ret_407_search_metadata_accepts_gaussdb_search_result_shape(monkeypatch, doc_metadata_context):+ service = doc_metadata_context.DocMetadataService+ module = doc_metadata_context.module+ settings = doc_metadata_context.settings+ fake = FakeSearchResultStore(doc_metadata_context.SearchResult)+ monkeypatch.setattr(settings, "docStoreConn", fake)+ monkeypatch.setattr(settings, "DOC_ENGINE_GAUSSDB", True)+ monkeypatch.setattr(settings, "DOC_ENGINE_INFINITY", False)+ monkeypatch.setattr(+ module.Knowledgebase,+ "get_by_id",+ staticmethod(lambda _kb_id: type("KB", (), {"tenant_id": "tenant1"})()),+ )++ docs = service._search_metadata("kb1")++ assert docs == [{"id": "doc1", "kb_id": "kb1", "meta_fields": {"author": "Alice"}}]+ assert fake.executed[0]["index_names"] == "ragflow_doc_meta_tenant1"+ assert fake.executed[0]["knowledgebase_ids"] == ["kb1"]+++def test_tc_ret_407_search_metadata_paginates_gaussdb_search_result_total(monkeypatch, doc_metadata_context):+ service = doc_metadata_context.DocMetadataService+ module = doc_metadata_context.module+ settings = doc_metadata_context.settings+ fake = PagingSearchResultStore(doc_metadata_context.SearchResult)+ monkeypatch.setattr(settings, "docStoreConn", fake)+ monkeypatch.setattr(settings, "DOC_ENGINE_GAUSSDB", True)+ monkeypatch.setattr(settings, "DOC_ENGINE_INFINITY", False)+ monkeypatch.setattr(+ module.Knowledgebase,+ "get_by_id",+ staticmethod(lambda _kb_id: type("KB", (), {"tenant_id": "tenant1"})()),+ )++ docs = service._search_metadata("kb1")++ doc_ids = [doc["id"] for doc in docs]+ assert doc_ids == [f"doc{i}" for i in range(1001)]+ assert len(set(doc_ids)) == 1001+ assert [call["offset"] for call in fake.executed] == [0, 1000]+ assert len(fake.executed) == 2+++ ("table_exists", "created_meta"),+ [+ (True, []),+ (False, ["ragflow_doc_meta_tenant1"]),+ ],+ ids=("existing-table", "missing-table"),+)+def test_tc_wrt_306_update_document_metadata_checks_table_before_gaussdb_upsert(+ monkeypatch,+ doc_metadata_context,+ table_exists,+ created_meta,+):+ service = doc_metadata_context.DocMetadataService+ module = doc_metadata_context.module+ settings = doc_metadata_context.settings+ fake = FakeGaussDBStore(exists=table_exists)+ monkeypatch.setattr(settings, "docStoreConn", fake)+ monkeypatch.setattr(settings, "DOC_ENGINE_GAUSSDB", True)+ monkeypatch.setattr(module.Document, "select", staticmethod(lambda *_args, **_kwargs: FakeDocQuery()))++ ok = service.update_document_metadata.__wrapped__(+ service,+ "doc1",+ {"author": "Alice"},+ )++ assert ok is True+ assert fake.existence_checks == [("ragflow_doc_meta_tenant1", "kb1")]+ assert fake.created_meta == created_meta+ assert fake.deleted == []+ assert fake.inserted == [([{"id": "doc1", "kb_id": "kb1", "meta_fields": {"author": "Alice"}}], "ragflow_doc_meta_tenant1", "kb1")]+++ ("create_result", "create_error"),+ [+ (False, None),+ (True, psycopg2.OperationalError("metadata table creation failed")),+ ],+ ids=("returns-false", "raises"),+)+def test_tc_wrt_307_update_document_metadata_stops_when_gaussdb_meta_table_creation_fails(+ monkeypatch,+ doc_metadata_context,+ create_result,+ create_error,+):+ service = doc_metadata_context.DocMetadataService+ module = doc_metadata_context.module+ settings = doc_metadata_context.settings+ fake = FakeGaussDBStore(exists=False, create_result=create_result, create_error=create_error)+ monkeypatch.setattr(settings, "docStoreConn", fake)+ monkeypatch.setattr(settings, "DOC_ENGINE_GAUSSDB", True)+ monkeypatch.setattr(module.Document, "select", staticmethod(lambda *_args, **_kwargs: FakeDocQuery()))++ ok = service.update_document_metadata.__wrapped__(+ service,+ "doc1",+ {"author": "Alice"},+ )++ assert ok is False+ assert fake.existence_checks == [("ragflow_doc_meta_tenant1", "kb1")]+ assert fake.created_meta == ["ragflow_doc_meta_tenant1"]+ assert fake.inserted == []+ assert fake.deleted == []+++def test_tc_wrt_308_update_document_metadata_returns_false_for_gaussdb_upsert_errors(monkeypatch, doc_metadata_context):+ service = doc_metadata_context.DocMetadataService+ module = doc_metadata_context.module+ settings = doc_metadata_context.settings+ fake = FakeGaussDBStore(insert_result=["doc1"])+ monkeypatch.setattr(settings, "docStoreConn", fake)+ monkeypatch.setattr(settings, "DOC_ENGINE_GAUSSDB", True)+ monkeypatch.setattr(module.Document, "select", staticmethod(lambda *_args, **_kwargs: FakeDocQuery()))++ ok = service.update_document_metadata.__wrapped__(+ service,+ "doc1",+ {"author": "Alice"},+ )++ assert ok is False+ assert fake.existence_checks == [("ragflow_doc_meta_tenant1", "kb1")]+ assert fake.created_meta == []+ assert fake.inserted == [([{"id": "doc1", "kb_id": "kb1", "meta_fields": {"author": "Alice"}}], "ragflow_doc_meta_tenant1", "kb1")]+ assert fake.deleted == []+++def test_tc_mgf_710_gaussdb_pushdown_routes_scoped_filter_with_probe_limit(monkeypatch, doc_metadata_context):+ service = doc_metadata_context.DocMetadataService+ settings = doc_metadata_context.settings+ fake = FakeGaussDBStore()+ monkeypatch.setattr(settings, "docStoreConn", fake)+ monkeypatch.setattr(settings, "DOC_ENGINE_GAUSSDB", True)+ monkeypatch.setattr(+ doc_metadata_context.module.Knowledgebase,+ "get_by_id",+ staticmethod(lambda _kb_id: type("KB", (), {"tenant_id": "tenant1"})()),+ )++ doc_ids = service.filter_doc_ids_by_meta_pushdown(+ ["kb1"],+ [{"key": "author", "op": "=", "value": "Alice"}],+ "and",+ limit=25,+ )++ assert doc_ids == ["doc-risk"]+ index_name, kb_ids, sql_filter, params, limit = fake.executed[0]+ assert index_name == "ragflow_doc_meta_tenant1"+ assert kb_ids == ["kb1"]+ assert sql_filter == "(lower(meta_fields #>> '{author}') = %s OR jsonb_exists(meta_fields #> '{author}', %s))"+ assert params == ["alice", "alice"]+ assert limit == 26+ assert fake.created_meta == []+ assert fake.inserted == []+ assert fake.deleted == []+++ ("returned_ids", "expected"),+ [+ (["doc1", "doc2"], ["doc1", "doc2"]),+ (["doc1", "doc1", "doc2"], ["doc1", "doc2"]),+ (["doc1", "doc2", "doc3"], None),+ ],+)+def test_tc_mgf_710_gaussdb_pushdown_enforces_complete_result_cap(+ monkeypatch,+ doc_metadata_context,+ returned_ids,+ expected,+):+ service = doc_metadata_context.DocMetadataService+ settings = doc_metadata_context.settings+ fake = FakeGaussDBStore()+ calls = []++ def fetch_metadata_doc_ids(*args):+ calls.append(args)+ return returned_ids++ fake.fetch_metadata_doc_ids = fetch_metadata_doc_ids+ monkeypatch.setattr(settings, "docStoreConn", fake)+ monkeypatch.setattr(settings, "DOC_ENGINE_GAUSSDB", True)+ monkeypatch.setattr(+ doc_metadata_context.module.Knowledgebase,+ "get_by_id",+ staticmethod(lambda _kb_id: type("KB", (), {"tenant_id": "tenant1"})()),+ )++ doc_ids = service.filter_doc_ids_by_meta_pushdown(+ ["kb1"],+ [{"key": "author", "op": "=", "value": "Alice"}],+ "and",+ limit=2,+ )++ assert doc_ids == expected+ assert len(calls) == 1+ index_name, kb_ids, sql_filter, params, probe_limit = calls[0]+ assert index_name == "ragflow_doc_meta_tenant1"+ assert kb_ids == ["kb1"]+ assert sql_filter == "(lower(meta_fields #>> '{author}') = %s OR jsonb_exists(meta_fields #> '{author}', %s))"+ assert params == ["alice", "alice"]+ assert probe_limit == 3+ assert fake.created_meta == []+ assert fake.inserted == []+ assert fake.deleted == []+++ ("filters", "logic"),+ [+ ([{"key": "author", "op": "regex", "value": "^A"}], "and"),+ ([{"key": "author", "op": "=", "value": "Alice"}], "xor"),+ ],+)+def test_tc_mgf_711_gaussdb_pushdown_rejects_unsupported_filter_without_fetch(+ monkeypatch,+ doc_metadata_context,+ caplog,+ filters,+ logic,+):+ service = doc_metadata_context.DocMetadataService+ settings = doc_metadata_context.settings+ fake = FakeGaussDBStore()+ monkeypatch.setattr(settings, "docStoreConn", fake)+ monkeypatch.setattr(settings, "DOC_ENGINE_GAUSSDB", True)+ monkeypatch.setattr(+ doc_metadata_context.module.Knowledgebase,+ "get_by_id",+ staticmethod(lambda _kb_id: type("KB", (), {"tenant_id": "tenant1"})()),+ )++ caplog.clear()+ doc_ids = service.filter_doc_ids_by_meta_pushdown(+ ["kb1"],+ filters,+ logic,+ limit=2,+ )++ assert doc_ids is None+ assert fake.executed == []+ assert fake.created_meta == []+ assert fake.inserted == []+ assert fake.deleted == []+ messages = [record.getMessage() for record in caplog.records]+ assert all("author" not in message and "Alice" not in message and "^A" not in message for message in messages)+++def test_tc_mgf_712_gaussdb_pushdown_falls_back_without_partial_ids_on_fetch_failure(monkeypatch, doc_metadata_context):+ service = doc_metadata_context.DocMetadataService+ settings = doc_metadata_context.settings+ fake = FakeGaussDBStore()+ calls = []++ def raise_fetch(*args, **_kwargs):+ calls.append((args, _kwargs))+ raise RuntimeError("gaussdb unavailable")++ fake.fetch_metadata_doc_ids = raise_fetch+ monkeypatch.setattr(settings, "docStoreConn", fake)+ monkeypatch.setattr(settings, "DOC_ENGINE_GAUSSDB", True)+ monkeypatch.setattr(+ doc_metadata_context.module.Knowledgebase,+ "get_by_id",+ staticmethod(lambda _kb_id: type("KB", (), {"tenant_id": "tenant1"})()),+ )++ doc_ids = service.filter_doc_ids_by_meta_pushdown(+ ["kb1"],+ [{"key": "author", "op": "=", "value": "Alice"}],+ "and",+ limit=2,+ )++ assert doc_ids is None+ assert len(calls) == 1+ args, kwargs = calls[0]+ assert args[0:2] == ("ragflow_doc_meta_tenant1", ["kb1"])+ assert args[2] == "(lower(meta_fields #>> '{author}') = %s OR jsonb_exists(meta_fields #> '{author}', %s))"+ assert args[3:] == (["alice", "alice"], 3)+ assert kwargs == {}+ assert fake.created_meta == []+ assert fake.inserted == []+ assert fake.deleted == []diff --git a/test/unit_test/api/db/services/test_gaussdb_feedback_dataset.py b/test/unit_test/api/db/services/test_gaussdb_feedback_dataset.pynew file mode 100644index 0000000000..27832fadde--- /dev/null+++ b/test/unit_test/api/db/services/test_gaussdb_feedback_dataset.py@@ -0,0 +1,124 @@+#+# Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+#+import importlib.util+import sys+from pathlib import Path+from types import ModuleType+++def _install_module(monkeypatch, name, **attrs):+ module = ModuleType(name)+ for key, value in attrs.items():+ setattr(module, key, value)+ monkeypatch.setitem(sys.modules, name, module)+ return module+++def _load_feedback_module(monkeypatch):+ repo_root = Path(__file__).resolve().parents[5]+ common_pkg = _install_module(monkeypatch, "common")+ constants_mod = _install_module(monkeypatch, "common.constants", PAGERANK_FLD="pagerank_fea")+ settings_mod = _install_module(monkeypatch, "common.settings", DOC_ENGINE="gaussdb")+ common_pkg.constants = constants_mod+ common_pkg.settings = settings_mod++ _install_module(monkeypatch, "rag")+ _install_module(monkeypatch, "rag.nlp")+ _install_module(monkeypatch, "rag.nlp.search", index_name=lambda tenant_id: f"ragflow_{tenant_id}")++ spec = importlib.util.spec_from_file_location(+ "api.db.services.chunk_feedback_service",+ repo_root / "api" / "db" / "services" / "chunk_feedback_service.py",+ )+ module = importlib.util.module_from_spec(spec)+ monkeypatch.setitem(sys.modules, "api.db.services.chunk_feedback_service", module)+ spec.loader.exec_module(module)+ return module, settings_mod+++class FakeGaussDBStore:+ def __init__(self):+ self.updated = []+ self.adjusted = []+ self.rows = {"chunk1": {"pagerank_fea": 10}}++ def db_type(self):+ return "gaussdb"++ def adjust_chunk_pagerank_fea(self, *args, **kwargs):+ self.adjusted.append((args, kwargs))+ return True++ def get(self, chunk_id, index_name, kb_ids):+ return self.rows.get(chunk_id)++ def update(self, condition, values, index_name, kb_id):+ self.updated.append((condition, values, index_name, kb_id))+ return True+++class FakeGaussDBStoreWithoutAdjust:+ def __init__(self):+ self.updated = []+ self.rows = {"chunk1": {"pagerank_fea": 10}}++ def db_type(self):+ return "gaussdb"++ def get(self, chunk_id, index_name, kb_ids):+ return self.rows.get(chunk_id)++ def update(self, condition, values, index_name, kb_id):+ self.updated.append((condition, values, index_name, kb_id))+ return True+++def test_update_chunk_weight_falls_back_to_update_by_id_and_kb_without_adjust(monkeypatch):+ module, settings_mod = _load_feedback_module(monkeypatch)+ fake = FakeGaussDBStoreWithoutAdjust()+ settings_mod.docStoreConn = fake++ ok = module.ChunkFeedbackService.update_chunk_weight(+ tenant_id="tenant",+ chunk_id="chunk1",+ kb_id="kb1",+ delta=2,+ )++ assert ok is True+ assert fake.updated == [({"id": "chunk1"}, {"pagerank_fea": 12}, "ragflow_tenant", "kb1")]+++def test_update_chunk_weight_routes_gaussdb_through_atomic_adjust(monkeypatch):+ module, settings_mod = _load_feedback_module(monkeypatch)+ fake = FakeGaussDBStore()+ settings_mod.docStoreConn = fake++ ok = module.ChunkFeedbackService.update_chunk_weight(+ tenant_id="tenant",+ chunk_id="chunk1",+ kb_id="kb1",+ delta=2,+ )++ assert ok is True+ assert fake.adjusted == [+ (+ ("chunk1", "ragflow_tenant", "kb1", 2.0, module.MIN_PAGERANK_WEIGHT, module.MAX_PAGERANK_WEIGHT),+ {},+ )+ ]+ assert fake.updated == []diff --git a/test/unit_test/api/db/test_gaussdb_adaptation_points.py b/test/unit_test/api/db/test_gaussdb_adaptation_points.pynew file mode 100644index 0000000000..0c7f46e841--- /dev/null+++ b/test/unit_test/api/db/test_gaussdb_adaptation_points.py@@ -0,0 +1,386 @@+"""+Direct tests for each GaussDB metadata database adaptation point.+"""++import logging+from pathlib import Path++from peewee import ProgrammingError+from ruamel.yaml import YAML++REPO_ROOT = Path(__file__).resolve().parents[4]++from api.db import db_models+from api.db.services import connector_service+from common import settings+++def read_repo_file(path: str) -> str:+ return (REPO_ROOT / path).read_text()+++def load_yaml(path: str):+ yaml = YAML(typ="safe", pure=True)+ return yaml.load(read_repo_file(path))+++def test_gaussdb_duplicate_column_migration_is_idempotent(monkeypatch, caplog):+ class Migrator:+ def add_column(self, table_name, column_name, column_type):+ return table_name, column_name, column_type++ def duplicate_column(*_operations):+ raise ProgrammingError('column "f_extra" already exists')++ monkeypatch.setattr(db_models, "migrate", duplicate_column)+ monkeypatch.setattr(settings, "DATABASE_TYPE", "gaussdb")++ caplog.set_level(logging.CRITICAL)+ db_models.alter_db_add_column(Migrator(), "tenant", "f_extra", object())++ assert not caplog.records+++def test_gaussdb_migration_logs_unrelated_programming_errors(monkeypatch, caplog):+ class Migrator:+ def add_column(self, table_name, column_name, column_type):+ return table_name, column_name, column_type++ def syntax_error(*_operations):+ raise ProgrammingError("syntax error near invalid_sql")++ monkeypatch.setattr(db_models, "migrate", syntax_error)+ monkeypatch.setattr(settings, "DATABASE_TYPE", "gaussdb")++ caplog.set_level(logging.CRITICAL)+ db_models.alter_db_add_column(Migrator(), "tenant", "f_extra", object())++ assert "syntax error near invalid_sql" in caplog.text+++def test_gaussdb_empty_string_compatible_migration_drops_not_null_only_for_gaussdb(monkeypatch):+ class FakeDB:+ queries = []++ @classmethod+ def execute_sql(cls, sql, params=None):+ cls.queries.append((sql, params))++ monkeypatch.setattr(db_models, "DB", FakeDB)+ monkeypatch.setattr(settings, "DATABASE_TYPE", "gaussdb")++ db_models.relax_gaussdb_empty_string_compatible_columns()++ expected_query_count = sum(len(columns) for _, columns in db_models.GAUSSDB_EMPTY_STRING_COMPATIBLE_COLUMNS)+ assert len(FakeDB.queries) == expected_query_count+ assert ('ALTER TABLE "user" ALTER COLUMN "nickname" DROP NOT NULL', None) in FakeDB.queries+ assert ('ALTER TABLE "tenant" ALTER COLUMN "llm_id" DROP NOT NULL', None) in FakeDB.queries+ assert ('ALTER TABLE "system_settings" ALTER COLUMN "value" DROP NOT NULL', None) in FakeDB.queries+ assert ('ALTER TABLE "task" ALTER COLUMN "task_type" DROP NOT NULL', None) in FakeDB.queries+ assert ('ALTER TABLE "sync_logs" ALTER COLUMN "error_msg" DROP NOT NULL', None) in FakeDB.queries+ assert ('ALTER TABLE "sync_logs" ALTER COLUMN "full_exception_trace" DROP NOT NULL', None) in FakeDB.queries+ assert ('ALTER TABLE "api_4_conversation" ALTER COLUMN "user_id" DROP NOT NULL', None) in FakeDB.queries+ assert ('ALTER TABLE "user_canvas" ALTER COLUMN "tags" DROP NOT NULL', None) in FakeDB.queries++ monkeypatch.setattr(settings, "DATABASE_TYPE", "mysql")+ db_models.relax_gaussdb_empty_string_compatible_columns()++ assert len(FakeDB.queries) == expected_query_count+++def test_gaussdb_migration_adds_compatible_tags_before_relaxing_columns(monkeypatch):+ events = []+ migrated_tags_field = None++ def record_add_column(_migrator, table_name, column_name, column_type):+ nonlocal migrated_tags_field+ events.append(("add", table_name, column_name))+ if (table_name, column_name) == ("user_canvas", "tags"):+ migrated_tags_field = column_type++ monkeypatch.setattr(settings, "DATABASE_TYPE", "gaussdb")+ monkeypatch.setattr(db_models, "alter_db_add_column", record_add_column)+ monkeypatch.setattr(db_models, "alter_db_column_type", lambda *_args: events.append(("alter_type",)))+ monkeypatch.setattr(db_models, "alter_db_rename_column", lambda *_args: events.append(("rename",)))+ monkeypatch.setattr(db_models, "alter_db_drop_index", lambda *_args: events.append(("drop_index",)))+ monkeypatch.setattr(db_models, "relax_gaussdb_empty_string_compatible_columns", lambda: events.append(("relax",)))+ monkeypatch.setattr(db_models, "migrate", lambda *_operations: None)+ monkeypatch.setattr(db_models, "migrate_add_unique_email", lambda _migrator: None)+ monkeypatch.setattr(db_models, "migrate_model_type_names", lambda: None)+ monkeypatch.setattr(db_models, "ensure_model_indexes", lambda _migrator: None)++ db_models.migrate_db()++ assert isinstance(migrated_tags_field, db_models.EmptyStringCharField)+ assert migrated_tags_field.null is True+ assert events[-1] == ("relax",)+++def test_gaussdb_unique_email_migration_checks_unique_email_index_not_fixed_name(monkeypatch):+ class Cursor:+ def fetchone(self):+ return (1,)++ class FakeDB:+ queries = []+ params = []++ @classmethod+ def execute_sql(cls, sql, params=None):+ cls.queries.append(sql)+ cls.params.append(params)+ return Cursor()++ monkeypatch.setattr(settings, "DATABASE_TYPE", "gaussdb")+ monkeypatch.setattr(db_models, "DB", FakeDB)++ db_models.migrate_add_unique_email(object())++ assert len(FakeDB.queries) == 1+ assert "pg_indexes" in FakeDB.queries[0]+ assert "lower(indexdef) LIKE %s" in FakeDB.queries[0]+ assert FakeDB.params[0] == ("create unique index%", "%(email)%", '%("email")%')+ assert "indexname = 'user_email'" not in FakeDB.queries[0]+++def test_gaussdb_deadlock_retry_recognizes_psycopg_sqlstates(monkeypatch):+ from peewee import OperationalError+ from api.db.services.common_service import _is_deadlock_error++ class PgError:+ def __init__(self, pgcode):+ self.pgcode = pgcode++ monkeypatch.setattr(settings, "DATABASE_TYPE", "gaussdb")++ assert _is_deadlock_error(OperationalError(PgError("40P01")))+ assert _is_deadlock_error(OperationalError(PgError("40001")))+ assert _is_deadlock_error(OperationalError(PgError("55P03")))+ assert not _is_deadlock_error(OperationalError("syntax error"))+++def test_connector_poll_interval_uses_gaussdb_compatible_sql(monkeypatch):+ monkeypatch.setattr(connector_service.settings, "DATABASE_TYPE", "gaussdb")+ expr = connector_service._gaussdb_poll_interval_expr("refresh_freq")++ assert "t2.refresh_freq * INTERVAL '1 minute'" in expr.sql+ assert "make_interval" not in expr.sql+ assert "`t2`" not in expr.sql+++def test_health_check_uses_generic_probe_for_gaussdb(monkeypatch):+ from api.utils import health_utils++ class Cursor:+ def fetchone(self):+ return (1,)++ def close(self):+ pass++ class FakeDB:+ queries = []++ @classmethod+ def execute_sql(cls, sql):+ cls.queries.append(sql)+ return Cursor()++ monkeypatch.setattr(settings, "DATABASE_TYPE", "gaussdb")+ monkeypatch.setattr(health_utils, "DB", FakeDB)++ status = health_utils.get_mysql_status()++ assert status == {+ "status": "alive",+ "message": {+ "database": "gaussdb",+ "result": 1,+ },+ }+ assert FakeDB.queries == ["SELECT 1;"]+++def test_gaussdb_metadata_health_masks_connection_secrets(monkeypatch, caplog):+ from api.utils import health_utils++ password = "metadata-password-sentinel"+ dsn = "postgresql://metadata-user:[email protected]:19995/ragflow"+ access_token = "metadata-token-sentinel"++ class FakeDB:+ @staticmethod+ def execute_sql(_sql):+ raise RuntimeError(f"connection failed password={password} dsn={dsn} access_token={access_token}")++ monkeypatch.setattr(settings, "DATABASE_TYPE", "gaussdb")+ monkeypatch.setattr(health_utils, "DB", FakeDB)+ caplog.set_level(logging.ERROR)++ status = health_utils.get_database_status()+ serialized = str(status)++ assert status["status"] == "timeout"+ assert "***" in status["message"]+ assert "GaussDB metadata database status check failed (RuntimeError)" in caplog.text+ for secret in (password, dsn, access_token):+ assert secret not in serialized+ assert secret not in caplog.text+++def test_admin_config_selects_gaussdb_metadata_entry(monkeypatch):+ from admin.server import config as admin_config++ monkeypatch.setenv("DB_TYPE", "GaussDB")+ monkeypatch.setenv("GAUSSDB_METADATA_HOST", "metadata.example.com")+ monkeypatch.setenv("GAUSSDB_METADATA_PORT", "8000")+ monkeypatch.setenv("GAUSSDB_METADATA_USER", "metadata_user")+ monkeypatch.setenv("GAUSSDB_METADATA_PASSWORD", "metadata-secret")+ monkeypatch.setenv("GAUSSDB_METADATA_SCHEMA", "metadata_schema")+ monkeypatch.setattr(+ admin_config,+ "read_config",+ lambda _path: {+ "mysql": {"host": "mysql", "port": 3306, "user": "root", "password": "mysql-secret"},+ "gaussdb": {+ "host": "doc.example.com",+ "port": 19995,+ "database": "doc_db",+ "user": "doc_user",+ "password": "doc-secret",+ "schema": "doc_schema",+ },+ },+ )++ configs = admin_config.load_configurations("ignored.yaml")+ metadata_configs = [cfg for cfg in configs if cfg.service_type == "meta_data"]+ retrieval_configs = [cfg for cfg in configs if cfg.service_type == "retrieval"]++ assert len(metadata_configs) == 1+ assert len(retrieval_configs) == 1+ assert metadata_configs[0].name == "gaussdb"+ assert metadata_configs[0].meta_type == "gaussdb"+ assert metadata_configs[0].host == "metadata.example.com"+ assert metadata_configs[0].username == "metadata_user"+ assert metadata_configs[0].metadata_schema == "metadata_schema"+ assert metadata_configs[0].detail_func_name == "get_database_status"++ retrieval = retrieval_configs[0]+ assert retrieval.host == "doc.example.com"+ assert retrieval.port == 19995+ assert retrieval.database == "doc_db"+ assert retrieval.retrieval_schema == "doc_schema"+ assert retrieval.detail_func_name == "get_gaussdb_status"+ assert "password" not in str(retrieval.to_dict()).lower()+++def test_admin_service_safe_serialization_redacts_credentials():+ from admin.server import config as admin_config++ metadata = admin_config.GaussDBMetadataConfig(+ id=1,+ name="gaussdb",+ host="metadata.example.com",+ port=8000,+ username="metadata_user",+ password="metadata-secret",+ metadata_schema="metadata_schema",+ service_type="meta_data",+ meta_type="gaussdb",+ detail_func_name="get_database_status",+ )++ serialized = metadata.to_dict()++ assert serialized["extra"]["username"] == "metadata_user"+ assert serialized["extra"]["schema"] == "metadata_schema"+ assert serialized["extra"]["password"] == "********"+ assert "metadata-secret" not in str(serialized)+++def test_admin_config_falls_back_to_gaussdb_env(monkeypatch):+ from admin.server import config as admin_config++ monkeypatch.setenv("DB_TYPE", "gaussdb")+ monkeypatch.setenv("GAUSSDB_METADATA_HOST", "gaussdb-env.example.com")+ monkeypatch.setenv("GAUSSDB_METADATA_PORT", "19995")+ monkeypatch.setenv("GAUSSDB_METADATA_USER", "zws")+ monkeypatch.setenv("GAUSSDB_METADATA_PASSWORD", "env-secret")+ monkeypatch.setenv("GAUSSDB_METADATA_SCHEMA", "env_schema")+ monkeypatch.setattr(admin_config, "read_config", lambda _path: {})++ configs = admin_config.load_configurations("ignored.yaml")+ metadata_configs = [cfg for cfg in configs if cfg.service_type == "meta_data"]++ assert len(metadata_configs) == 1+ assert metadata_configs[0].name == "gaussdb"+ assert metadata_configs[0].host == "gaussdb-env.example.com"+ assert metadata_configs[0].port == 19995+ assert metadata_configs[0].username == "zws"+ assert metadata_configs[0].metadata_schema == "env_schema"+++def test_admin_config_warns_when_gaussdb_metadata_port_is_invalid(monkeypatch, caplog):+ from admin.server import config as admin_config++ monkeypatch.setenv("DB_TYPE", "gaussdb")+ monkeypatch.setenv("GAUSSDB_METADATA_PORT", "1999O")+ monkeypatch.setattr(admin_config, "read_config", lambda _path: {})+ caplog.set_level(logging.WARNING)++ configs = admin_config.load_configurations("ignored.yaml")+ metadata_config = next(cfg for cfg in configs if cfg.service_type == "meta_data")++ assert metadata_config.port == admin_config.GAUSSDB_ENV_DEFAULTS["port"]+ assert "Ignoring invalid GAUSSDB_METADATA_PORT='1999O'; using default 8000" in caplog.text+++def test_docker_config_files_expose_gaussdb_metadata_settings():+ env_text = read_repo_file("docker/.env")+ template = read_repo_file("docker/service_conf.yaml.template")+ source_conf = read_repo_file("conf/service_conf.yaml")++ assert "DB_TYPE=${DB_TYPE:-mysql}" in env_text+ assert "GAUSSDB_METADATA_HOST=" in env_text+ assert "GAUSSDB_METADATA_PASSWORD=" in env_text+ assert "GAUSSDB_METADATA_SCHEMA=public" in env_text+ assert "gaussdb:" in template+ assert template.count("\ngaussdb:") == 1+ assert "GAUSSDB_METADATA_DBNAME" not in template+ assert "gaussdb:" not in source_conf+++def test_docker_compose_metadata_profile_does_not_force_mysql_for_gaussdb():+ env_text = read_repo_file("docker/.env")+ base = load_yaml("docker/docker-compose-base.yml")+ compose = load_yaml("docker/docker-compose.yml")+ macos_compose = load_yaml("docker/docker-compose-macos.yml")+ cn_compose = load_yaml("docker/docker-compose-CN-oc9.yml")++ assert "METADATA_DB_PROFILE=${METADATA_DB_PROFILE:-mysql}" in env_text+ assert "COMPOSE_PROFILES=${DOC_ENGINE},${DEVICE},metadata-${METADATA_DB_PROFILE}" in env_text+ assert "metadata-mysql" in base["services"]["mysql"]["profiles"]+ assert "metadata-gaussdb" not in base["services"]["mysql"]["profiles"]++ for service_name in ["ragflow-cpu", "ragflow-gpu"]:+ mysql_dep = compose["services"][service_name]["depends_on"]["mysql"]+ assert mysql_dep["condition"] == "service_healthy"+ assert mysql_dep["required"] is False++ assert macos_compose["services"]["ragflow"]["depends_on"]["mysql"]["required"] is False+ for service_name in ["ragflow-cpu", "ragflow-gpu"]:+ assert cn_compose["services"][service_name]["depends_on"]["mysql"]["required"] is False+++def test_docker_launch_scripts_skip_mysql_migration_for_gaussdb():+ entrypoint = read_repo_file("docker/entrypoint.sh")+ launcher = read_repo_file("docker/launch_backend_service.sh")++ assert 'DB_TYPE_NORMALIZED="${DB_TYPE:-mysql}"' in entrypoint+ assert 'if [[ "${DB_TYPE_NORMALIZED}" == "gaussdb" || "${DB_TYPE_NORMALIZED}" == "gauss" ]]; then' in entrypoint+ assert "Skipping MySQL-specific model provider table migrations" in entrypoint++ assert 'local db_type="${DB_TYPE:-mysql}"' in launcher+ assert 'if [ "$db_type" = "gaussdb" ] || [ "$db_type" = "gauss" ]; then' in launcher+ assert "Skipping MySQL-specific model provider table migrations" in launcherdiff --git a/test/unit_test/api/db/test_gaussdb_error_code_helpers.py b/test/unit_test/api/db/test_gaussdb_error_code_helpers.pynew file mode 100644index 0000000000..5d5dd3380e--- /dev/null+++ b/test/unit_test/api/db/test_gaussdb_error_code_helpers.py@@ -0,0 +1,99 @@+from peewee import InterfaceError, OperationalError, ProgrammingError+++class PgError(Exception):+ def __init__(self, pgcode: str, message: str = "driver error"):+ self.pgcode = pgcode+ self.args = (message,)+++def peewee_error_with_context(exc_cls, pgcode: str, message: str):+ exc = exc_cls(message)+ # Peewee wraps psycopg2 exceptions and keeps the raw driver exception on+ # __context__; live GaussDB probing confirmed pgcode is not on the outer+ # Peewee exception itself.+ exc.__context__ = PgError(pgcode, message)+ return exc+++def test_sqlstate_extraction_reads_peewee_context():+ from api.db.gaussdb_error_utils import sqlstate_from_exception++ exc = peewee_error_with_context(ProgrammingError, "42701", "Column already exists")++ assert sqlstate_from_exception(exc) == "42701"+++def test_gaussdb_retryable_transaction_errors_include_live_sqlstates(monkeypatch):+ from api.db.gaussdb_error_utils import is_retryable_transaction_error+ from api.db.services import common_service++ monkeypatch.setattr(common_service.settings, "DATABASE_TYPE", "gaussdb")++ for sqlstate in ("40P01", "40001", "55P03"):+ exc = peewee_error_with_context(OperationalError, sqlstate, "retryable conflict")+ assert is_retryable_transaction_error(exc)+ assert common_service._is_deadlock_error(exc)++ syntax_error = peewee_error_with_context(OperationalError, "42601", "syntax error")+ assert not is_retryable_transaction_error(syntax_error)+ assert not common_service._is_deadlock_error(syntax_error)+++def test_gaussdb_ddl_idempotency_error_helpers_use_live_sqlstates():+ from api.db.gaussdb_error_utils import (+ is_duplicate_column_error,+ is_duplicate_object_error,+ is_undefined_object_error,+ )++ assert is_duplicate_column_error(peewee_error_with_context(ProgrammingError, "42701", "Column already exists"))+ assert is_duplicate_object_error(peewee_error_with_context(ProgrammingError, "42P07", "Relation already exists"))+ assert is_undefined_object_error(peewee_error_with_context(ProgrammingError, "42704", "index does not exist"))++ assert not is_duplicate_column_error(peewee_error_with_context(ProgrammingError, "42601", "syntax error"))+ assert not is_duplicate_object_error(peewee_error_with_context(ProgrammingError, "23505", "duplicate key value"))+ assert not is_undefined_object_error(peewee_error_with_context(ProgrammingError, "42P07", "already exists"))+++def test_connection_error_helper_covers_sqlstate_interface_and_gaussdb_ssl_close_text():+ from api.db.gaussdb_error_utils import is_psycopg_connection_error++ assert is_psycopg_connection_error(peewee_error_with_context(OperationalError, "08006", "connection failure"))+ assert is_psycopg_connection_error(InterfaceError("connection already closed"))+ assert is_psycopg_connection_error(OperationalError("SSL connection has been closed unexpectedly"))+ assert not is_psycopg_connection_error(OperationalError("syntax error at or near FROM"))+++def test_non_connection_sqlstate_overrides_connection_shaped_text():+ from api.db.gaussdb_error_utils import is_psycopg_connection_error++ exc = peewee_error_with_context(OperationalError, "22P02", "invalid integer from connection-shaped input")++ assert is_psycopg_connection_error(exc) is False+++def test_all_08_class_sqlstates_are_connection_errors():+ from api.db.gaussdb_error_utils import is_psycopg_connection_error++ assert is_psycopg_connection_error(peewee_error_with_context(OperationalError, "08002", "driver state 08002"))+++def test_no_sqlstate_eof_reset_is_a_connection_error():+ from api.db.gaussdb_error_utils import is_psycopg_connection_error++ assert is_psycopg_connection_error(OperationalError("SSL SYSCALL error: EOF detected"))+++def test_no_sqlstate_unrelated_connection_word_is_not_a_connection_error():+ from api.db.gaussdb_error_utils import is_psycopg_connection_error++ assert not is_psycopg_connection_error(OperationalError("invalid value for application connection label"))+++def test_nested_non_connection_sqlstate_overrides_outer_interface_error():+ from api.db.gaussdb_error_utils import is_psycopg_connection_error++ exc = peewee_error_with_context(InterfaceError, "22P02", "invalid integer from connection-shaped input")++ assert is_psycopg_connection_error(exc) is Falsediff --git a/test/unit_test/api/db/test_gaussdb_peewee.py b/test/unit_test/api/db/test_gaussdb_peewee.pynew file mode 100644index 0000000000..df6c828ddd--- /dev/null+++ b/test/unit_test/api/db/test_gaussdb_peewee.py@@ -0,0 +1,444 @@+"""Tests for GaussDB Peewee ORM support."""++import pytest+from peewee import CharField, Model, OperationalError, SqliteDatabase+from playhouse.pool import PooledPostgresqlDatabase++from api.db.db_models import (+ DatabaseLock,+ DatabaseMigrator,+ GaussDBDatabaseLock,+ GaussDBMigrator,+ PostgresDatabaseLock,+ PooledDatabase,+ RetryingPooledGaussDBDatabase,+ RetryingPooledPostgresqlDatabase,+ TextFieldType,+)+from common.settings import load_database_config, normalize_database_type+import api.db.db_models as db_models+++class RecordingCursor:+ def __init__(self, rows):+ self.rows = rows++ def fetchall(self):+ return self.rows+++class LockCursor:+ def __init__(self, value):+ self.value = value++ def fetchone(self):+ return (self.value,)+++class SequenceLockDB:+ def __init__(self, values):+ self.values = iter(values)+ self.calls = []++ def execute_sql(self, sql, params=None):+ self.calls.append((sql, params))+ return LockCursor(next(self.values))+++class InitializationCursor:+ def __init__(self, rows):+ self.rows = iter(rows)+ self.calls = []++ def __enter__(self):+ return self++ def __exit__(self, exc_type, exc_value, traceback):+ return False++ def execute(self, sql, params=None):+ self.calls.append((sql, params))++ def fetchone(self):+ return next(self.rows)+++class InitializationConnection:+ def __init__(self, rows, cursor_class=InitializationCursor):+ self.cursor_instance = cursor_class(rows)+ self.rollback_calls = 0++ def cursor(self):+ return self.cursor_instance++ def rollback(self):+ self.rollback_calls += 1+++class UndefinedTableError(Exception):+ pgcode = "42P01"+++class FeatureNotSupportedError(Exception):+ pgcode = "0A000"+++class MissingPgxcNodeCursor(InitializationCursor):+ def execute(self, sql, params=None):+ super().execute(sql, params)+ if isinstance(sql, str) and "pg_catalog.pgxc_node" in sql:+ raise UndefinedTableError("relation pg_catalog.pgxc_node does not exist")+++class DefinitionOnlyPgxcNodeCursor(InitializationCursor):+ def execute(self, sql, params=None):+ super().execute(sql, params)+ if isinstance(sql, str) and "pg_catalog.pgxc_node" in sql:+ raise FeatureNotSupportedError("pgxc_node data cannot be queried in centralized mode")+++class UpdateByIdModel(Model):+ id = CharField(primary_key=True)+ name = CharField(null=True)++ class Meta:+ database = None+++class RecordingCloseDatabase:+ def __init__(self, closed=False):+ self.closed = closed+ self.close_calls = 0+ self.close_stale_calls = []++ def __bool__(self):+ return True++ def is_closed(self):+ return self.closed++ def close(self):+ self.close_calls += 1+ self.closed = True++ def close_stale(self, age):+ self.close_stale_calls.append(age)+++class TestGaussDBDatabase:+ """Test cases for GaussDB metadata database support."""++ def test_gaussdb_database_class_exists(self):+ assert RetryingPooledGaussDBDatabase is not None++ def test_gaussdb_in_pooled_database_enum(self):+ assert hasattr(PooledDatabase, "GAUSSDB")+ assert PooledDatabase.GAUSSDB.value == RetryingPooledGaussDBDatabase++ def test_gaussdb_in_database_migrator_enum(self):+ assert hasattr(DatabaseMigrator, "GAUSSDB")+ assert DatabaseMigrator.GAUSSDB.value == GaussDBMigrator++ def test_gaussdb_in_database_lock_enum(self):+ assert hasattr(DatabaseLock, "GAUSSDB")+ assert DatabaseLock.GAUSSDB.value == GaussDBDatabaseLock++ def test_gaussdb_in_text_field_type_enum(self):+ assert hasattr(TextFieldType, "GAUSSDB")+ assert TextFieldType.GAUSSDB.value == "TEXT"++ def test_gaussdb_database_uses_dedicated_adapter(self):+ assert not issubclass(RetryingPooledGaussDBDatabase, RetryingPooledPostgresqlDatabase)+ assert issubclass(RetryingPooledGaussDBDatabase, PooledPostgresqlDatabase)+ assert issubclass(GaussDBMigrator, DatabaseMigrator.POSTGRES.value)+ assert issubclass(GaussDBDatabaseLock, PostgresDatabaseLock)++ def test_gaussdb_database_init(self):+ db = RetryingPooledGaussDBDatabase(+ "test_db",+ host="localhost",+ port=8000,+ user="gaussdb",+ password="password",+ )+ assert db is not None+ assert db.max_retries == 5+ assert db.retry_delay == 1++ def test_gaussdb_database_custom_retries(self):+ db = RetryingPooledGaussDBDatabase(+ "test_db",+ host="localhost",+ max_retries=10,+ retry_delay=2,+ )+ assert db.max_retries == 10+ assert db.retry_delay == 2++ @pytest.mark.parametrize(("topology_row", "expected"), [((False,), False), ((True,), True)])+ def test_gaussdb_initializes_search_path_and_detects_topology(self, topology_row, expected):+ db = RetryingPooledGaussDBDatabase(None)+ connection = InitializationConnection([("ragflow_meta",), topology_row])++ db._initialize_connection(connection)++ calls = connection.cursor_instance.calls+ assert calls[0] == ("SELECT current_schema()", None)+ assert "SET search_path TO" in str(calls[1][0])+ assert "Identifier('ragflow_meta')" in str(calls[1][0])+ assert "pg_catalog.pgxc_node" in calls[2][0]+ assert db.is_distributed is expected++ @pytest.mark.parametrize("cursor_class", [MissingPgxcNodeCursor, DefinitionOnlyPgxcNodeCursor])+ def test_unavailable_pgxc_node_is_detected_as_centralized(self, cursor_class):+ db = RetryingPooledGaussDBDatabase(None)+ connection = InitializationConnection([("ragflow_meta",)], cursor_class=cursor_class)++ db._initialize_connection(connection)++ assert db.is_distributed is False+ assert connection.rollback_calls == 1+ assert "SET search_path TO" in str(connection.cursor_instance.calls[-1][0])++ def test_first_connection_failure_is_retried_before_ddl_preparation(self, monkeypatch):+ db = RetryingPooledGaussDBDatabase(None, max_retries=1, retry_delay=0)+ connection_calls = []+ executed = []+ sql = 'CREATE TABLE IF NOT EXISTS "user" ("id" VARCHAR(32))'++ def connection():+ connection_calls.append(None)+ if len(connection_calls) == 1:+ raise OperationalError("connection refused")+ db._is_distributed = True++ def base_execute(_self, prepared_sql, params=None, commit=True):+ executed.append((prepared_sql, params, commit))+ return "cursor"++ monkeypatch.setattr(db, "connection", connection)+ monkeypatch.setattr(db, "_handle_connection_loss", lambda: None)+ monkeypatch.setattr(PooledPostgresqlDatabase, "execute_sql", base_execute)++ assert db.execute_sql(sql) == "cursor"+ assert len(connection_calls) == 2+ assert executed == [(f"{sql} DISTRIBUTE BY REPLICATION", None, True)]++ def test_connection_failure_in_transaction_is_not_retried(self, monkeypatch):+ db = RetryingPooledGaussDBDatabase(None, max_retries=1, retry_delay=0)+ db._is_distributed = False+ connection_error = OperationalError("connection refused")+ executed = []+ reconnect_calls = []++ def base_execute(_self, sql, params=None, commit=True):+ executed.append((sql, params, commit))+ raise connection_error++ monkeypatch.setattr(db, "in_transaction", lambda: True)+ monkeypatch.setattr(db, "_handle_connection_loss", lambda: reconnect_calls.append(None))+ monkeypatch.setattr(PooledPostgresqlDatabase, "execute_sql", base_execute)++ with pytest.raises(OperationalError) as exc_info:+ db.execute_sql("SELECT 1")++ assert exc_info.value is connection_error+ assert executed == [("SELECT 1", None, True)]+ assert reconnect_calls == []++ def test_postgresql_execution_keeps_sql_unchanged(self, monkeypatch):+ db = RetryingPooledPostgresqlDatabase(None, max_retries=0)+ executed = []++ def base_execute(_self, sql, params=None, commit=True):+ executed.append((sql, params, commit))+ return "cursor"++ monkeypatch.setattr(PooledPostgresqlDatabase, "execute_sql", base_execute)++ assert db.execute_sql("SELECT 1") == "cursor"+ assert executed == [("SELECT 1", None, True)]++ def test_distributed_gaussdb_removes_only_redundant_primary_key_assignment(self):+ db = RetryingPooledGaussDBDatabase(None)+ db._is_distributed = True+ source = {"id": "record-1", "name": "updated"}++ prepared = db.prepare_update_by_id(UpdateByIdModel, "record-1", source)++ assert prepared == {"name": "updated"}+ assert prepared is not source+ assert source == {"id": "record-1", "name": "updated"}++ def test_distributed_gaussdb_replaces_row_for_primary_key_change(self, monkeypatch):+ db = RetryingPooledGaussDBDatabase(None)+ db._is_distributed = True+ sqlite_db = SqliteDatabase(":memory:")++ with sqlite_db.bind_ctx([UpdateByIdModel]):+ sqlite_db.create_tables([UpdateByIdModel])+ UpdateByIdModel.create(id="record-1", name="original")+ monkeypatch.setattr(db, "atomic", sqlite_db.atomic)++ assert db.replace_update_by_id(UpdateByIdModel, "record-1", {"id": "record-2", "name": "updated"}) == 1+ assert UpdateByIdModel.get_or_none(UpdateByIdModel.id == "record-1") is None+ assert UpdateByIdModel.get_by_id("record-2").name == "updated"+ sqlite_db.close()++ def test_distributed_gaussdb_replace_returns_zero_for_missing_row(self, monkeypatch):+ db = RetryingPooledGaussDBDatabase(None)+ db._is_distributed = True+ sqlite_db = SqliteDatabase(":memory:")++ with sqlite_db.bind_ctx([UpdateByIdModel]):+ sqlite_db.create_tables([UpdateByIdModel])+ monkeypatch.setattr(db, "atomic", sqlite_db.atomic)++ assert db.replace_update_by_id(UpdateByIdModel, "missing", {"id": "record-2"}) == 0+ sqlite_db.close()++ def test_centralized_gaussdb_keeps_primary_key_assignment_unchanged(self):+ db = RetryingPooledGaussDBDatabase(None)+ db._is_distributed = False+ source = {"id": "record-1", "name": "updated"}++ assert db.prepare_update_by_id(UpdateByIdModel, "record-1", source) is source+ assert db.replace_update_by_id(UpdateByIdModel, "record-1", {"id": "record-2"}) is None++ def test_postgresql_adapter_has_no_gaussdb_update_hook(self):+ assert not hasattr(RetryingPooledPostgresqlDatabase(None), "prepare_update_by_id")+ assert not hasattr(RetryingPooledPostgresqlDatabase(None), "replace_update_by_id")++ @pytest.mark.parametrize(+ "table_name",+ [+ "compilation_template",+ "compilation_template_group",+ "file_commit_item",+ "tenant_llm",+ "tenant_model_provider",+ "user",+ ],+ )+ def test_distributed_gaussdb_replicates_independent_unique_key_tables(self, table_name):+ db = RetryingPooledGaussDBDatabase(None)+ db._is_distributed = True+ sql = f'CREATE TABLE IF NOT EXISTS "{table_name}" ("id" VARCHAR(32))'++ assert db._prepare_distributed_ddl(sql) == f"{sql} DISTRIBUTE BY REPLICATION"++ def test_centralized_gaussdb_keeps_candidate_ddl_unchanged(self):+ db = RetryingPooledGaussDBDatabase(None)+ db._is_distributed = False+ sql = 'CREATE TABLE IF NOT EXISTS "user" ("id" VARCHAR(32))'++ assert db._prepare_distributed_ddl(sql) == sql++ def test_gaussdb_get_tables_defaults_to_current_schema(self, monkeypatch):+ db = RetryingPooledGaussDBDatabase(None)+ monkeypatch.setattr(db, "execute_sql", lambda *_args, **_kwargs: RecordingCursor([("tenant",), ("user",)]))++ assert db.get_tables() == ["tenant", "user"]++ def test_gaussdb_request_cleanup_returns_current_connection(self, monkeypatch):+ db = RecordingCloseDatabase()+ monkeypatch.setattr(db_models, "DB", db)+ monkeypatch.setattr(db_models.settings, "DATABASE_TYPE", "gaussdb")++ db_models.close_connection()++ assert db.close_calls == 1+ assert db.close_stale_calls == []++ def test_non_gaussdb_cleanup_preserves_existing_behavior(self, monkeypatch):+ db = RecordingCloseDatabase()+ monkeypatch.setattr(db_models, "DB", db)+ monkeypatch.setattr(db_models.settings, "DATABASE_TYPE", "mysql")++ db_models.close_connection()++ assert db.close_calls == 0+ assert db.close_stale_calls == [30]++ def test_pooled_database_enum_values(self):+ expected = {"MYSQL", "OCEANBASE", "POSTGRES", "GAUSSDB"}+ actual = {e.name for e in PooledDatabase}+ assert expected.issubset(actual), f"Missing: {expected - actual}"++ def test_database_lock_enum_values(self):+ expected = {"MYSQL", "OCEANBASE", "POSTGRES", "GAUSSDB"}+ actual = set(DatabaseLock.__members__.keys())+ assert expected.issubset(actual), f"Missing: {expected - actual}"++ def test_gaussdb_lock_waits_until_holder_releases(self, monkeypatch):+ now = [10.0]+ sleeps = []+ monkeypatch.setattr(db_models.time, "monotonic", lambda: now[0])++ def sleep(seconds):+ sleeps.append(seconds)+ now[0] += seconds++ monkeypatch.setattr(db_models.time, "sleep", sleep)+ db = SequenceLockDB([False, False, True])++ assert GaussDBDatabaseLock("init_database_tables", timeout=1.0, db=db).lock()+ assert sleeps == [0.1, 0.1]++ def test_gaussdb_lock_stops_at_timeout(self, monkeypatch):+ now = [20.0]+ monkeypatch.setattr(db_models.time, "monotonic", lambda: now[0])+ monkeypatch.setattr(db_models.time, "sleep", lambda seconds: now.__setitem__(0, now[0] + seconds))+ db = SequenceLockDB([False, False, False, False])++ with pytest.raises(Exception, match="acquire gaussdb lock init_database_tables timeout"):+ GaussDBDatabaseLock("init_database_tables", timeout=0.25, db=db).lock()++ assert now[0] == pytest.approx(20.25)+++class TestGaussDBConfiguration:+ """Test cases for DB_TYPE normalization."""++ def test_settings_can_use_gaussdb_case_insensitive(self):+ assert normalize_database_type("GaussDB") == "gaussdb"+ assert normalize_database_type("gaussdb") == "gaussdb"++ def test_settings_accepts_gauss_alias(self):+ assert normalize_database_type("gauss") == "gaussdb"++ def test_load_database_config_uses_gaussdb_env(self, monkeypatch):+ monkeypatch.delenv("GAUSSDB_METADATA_OPTIONS", raising=False)+ monkeypatch.setenv("GAUSSDB_METADATA_HOST", "127.0.0.1")+ monkeypatch.setenv("GAUSSDB_METADATA_PORT", "19995")+ monkeypatch.setenv("GAUSSDB_METADATA_USER", "zws")+ monkeypatch.setenv("GAUSSDB_METADATA_PASSWORD", "secret")+ monkeypatch.setenv("GAUSSDB_METADATA_DBNAME", "zws_test")+ monkeypatch.setenv("GAUSSDB_METADATA_SCHEMA", "ragflow_meta")++ config = load_database_config("GaussDB")++ assert config["host"] == "127.0.0.1"+ assert config["port"] == 19995+ assert config["user"] == "zws"+ assert config["password"] == "secret"+ assert config["name"] == "zws_test"+ assert config["options"] == "-c search_path=ragflow_meta -c client_encoding=UTF8 -c default_transaction_read_only=off"++ def test_load_database_config_defaults_schema_to_public(self, monkeypatch):+ monkeypatch.delenv("GAUSSDB_METADATA_SCHEMA", raising=False)+ monkeypatch.delenv("GAUSSDB_METADATA_OPTIONS", raising=False)++ assert load_database_config("gaussdb")["options"].startswith("-c search_path=public ")++ def test_load_database_config_rejects_unsafe_schema(self, monkeypatch):+ monkeypatch.setenv("GAUSSDB_METADATA_SCHEMA", "public;drop schema public")++ with pytest.raises(ValueError, match="GAUSSDB_METADATA_SCHEMA"):+ load_database_config("gaussdb")++ def test_explicit_options_override_generated_schema_option(self, monkeypatch):+ monkeypatch.setenv("GAUSSDB_METADATA_SCHEMA", "ragflow_meta")+ monkeypatch.setenv("GAUSSDB_METADATA_OPTIONS", "-c search_path=custom_meta -c statement_timeout=5000")++ assert load_database_config("gaussdb")["options"] == "-c search_path=custom_meta -c statement_timeout=5000"diff --git a/test/unit_test/api/db/test_gaussdb_user_flow.py b/test/unit_test/api/db/test_gaussdb_user_flow.pynew file mode 100644index 0000000000..3e22a21bc4--- /dev/null+++ b/test/unit_test/api/db/test_gaussdb_user_flow.py@@ -0,0 +1,301 @@+"""+User-flow tests for running RAGFlow with DB_TYPE=GaussDB.+"""++import os+import subprocess+import sys+import textwrap+from pathlib import Path++import pytest+++REPO_ROOT = Path(__file__).resolve().parents[4]+HELPER_DIR = Path(__file__).resolve().parent+++def run_isolated_flow(script: str, env_extra: dict[str, str], timeout: int = 60):+ env = os.environ.copy()+ env.update(env_extra)+ env["PYTHONPATH"] = os.pathsep.join(+ [+ str(REPO_ROOT),+ str(HELPER_DIR),+ env.get("PYTHONPATH", ""),+ ]+ )++ bootstrap = """+from gaussdb_test_utils import stub_settings_import_dependencies+stub_settings_import_dependencies()+"""+ result = subprocess.run(+ [sys.executable, "-c", textwrap.dedent(bootstrap + script)],+ cwd=REPO_ROOT,+ env=env,+ text=True,+ capture_output=True,+ timeout=timeout,+ )+ assert result.returncode == 0, f"stdout:\n{result.stdout}\nstderr:\n{result.stderr}"+ return result+++def test_user_env_selects_gaussdb_metadata_adapter(monkeypatch):+ monkeypatch.delenv("GAUSSDB_METADATA_OPTIONS", raising=False)+ result = run_isolated_flow(+ """+from common import settings+from api.db.db_models import DB, LongTextField++assert settings.DATABASE_TYPE == "gaussdb"+assert settings.DATABASE["host"] == "gaussdb.local"+assert settings.DATABASE["port"] == 8000+assert settings.DATABASE["user"] == "rag_flow"+assert settings.DATABASE["name"] == "rag_flow"+assert settings.DATABASE["options"] == "-c search_path=ragflow_meta -c client_encoding=UTF8 -c default_transaction_read_only=off"+assert DB.__class__.__name__ == "RetryingPooledGaussDBDatabase"+assert DB.lock.__name__ == "GaussDBDatabaseLock"+assert LongTextField.field_type == "TEXT"+print("gaussdb-adapter-ok")+""",+ {+ "DB_TYPE": "GaussDB",+ "GAUSSDB_METADATA_HOST": "gaussdb.local",+ "GAUSSDB_METADATA_PORT": "8000",+ "GAUSSDB_METADATA_USER": "rag_flow",+ "GAUSSDB_METADATA_PASSWORD": "secret",+ "GAUSSDB_METADATA_DBNAME": "rag_flow",+ "GAUSSDB_METADATA_SCHEMA": "ragflow_meta",+ },+ )++ assert "gaussdb-adapter-ok" in result.stdout+++def test_gaussdb_empty_string_compatible_fields_are_nullable_only_for_gaussdb():+ gaussdb_result = run_isolated_flow(+ """+from api.db.db_models import API4Conversation, Dialog, File, Knowledgebase, Memory, SyncLogs, SystemSettings, Task, Tenant, User, UserCanvas++fields = [+ User.nickname,+ Tenant.llm_id,+ Tenant.embd_id,+ Tenant.asr_id,+ Tenant.img2txt_id,+ Tenant.rerank_id,+ File.source_type,+ Knowledgebase.embd_id,+ Dialog.llm_id,+ Dialog.rerank_id,+ Memory.embd_id,+ Memory.llm_id,+ SystemSettings.value,+ Task.task_type,+ Task.progress_msg,+ Task.chunk_ids,+ SyncLogs.error_msg,+ SyncLogs.full_exception_trace,+ API4Conversation.user_id,+ UserCanvas.tags,+]+for field in fields:+ assert field.null is True+ assert field.db_value("") is None+ assert field.python_value(None) == ""++eq_sql, eq_params = File.select().where(File.source_type == "").sql()+ne_sql, ne_params = File.select().where(File.source_type != "").sql()+field_eq_sql, field_eq_params = Tenant.select().where(Tenant.llm_id == Tenant.embd_id).sql()+field_ne_sql, field_ne_params = Tenant.select().where(Tenant.llm_id != Tenant.embd_id).sql()++assert '"source_type" IS NULL' in eq_sql+assert 'LENGTH("t1"."source_type") = %s' in eq_sql+assert eq_params == [0]+assert 'LENGTH("t1"."source_type") > %s' in ne_sql+assert ne_params == [0]+assert '"t1"."llm_id" = "t1"."embd_id"' in field_eq_sql+assert field_eq_params == []+assert '"t1"."llm_id" != "t1"."embd_id"' in field_ne_sql+assert field_ne_params == []+print("gaussdb-empty-string-compatible-ok")+""",+ {+ "DB_TYPE": "GaussDB",+ "GAUSSDB_METADATA_HOST": "gaussdb.local",+ "GAUSSDB_METADATA_PORT": "8000",+ "GAUSSDB_METADATA_USER": "rag_flow",+ "GAUSSDB_METADATA_PASSWORD": "secret",+ "GAUSSDB_METADATA_DBNAME": "rag_flow",+ },+ )+ assert "gaussdb-empty-string-compatible-ok" in gaussdb_result.stdout++ mysql_result = run_isolated_flow(+ """+from api.db.db_models import API4Conversation, Dialog, File, Knowledgebase, Memory, SyncLogs, SystemSettings, Task, Tenant, User, UserCanvas++fields = [+ (User.nickname, False),+ (Tenant.llm_id, False),+ (Tenant.embd_id, False),+ (Tenant.asr_id, False),+ (Tenant.img2txt_id, False),+ (Tenant.rerank_id, False),+ (File.source_type, False),+ (Knowledgebase.embd_id, False),+ (Dialog.llm_id, False),+ (Dialog.rerank_id, False),+ (Memory.embd_id, False),+ (Memory.llm_id, False),+ (SystemSettings.value, False),+ (Task.task_type, False),+ (Task.progress_msg, True),+ (Task.chunk_ids, True),+ (SyncLogs.error_msg, False),+ (SyncLogs.full_exception_trace, True),+ (API4Conversation.user_id, False),+ (UserCanvas.tags, False),+]+for field, expected_null in fields:+ assert field.null is expected_null+ assert field.db_value("") == ""++assert Task.chunk_ids.field_type == "LONGTEXT"+sql, params = File.select().where(File.source_type == "").sql()+assert "IS NULL" not in sql+assert "LENGTH" not in sql+assert params == [""]+print("mysql-nullability-ok")+""",+ {+ "DB_TYPE": "mysql",+ },+ )+ assert "mysql-nullability-ok" in mysql_result.stdout+++def test_live_gaussdb_init_lock_and_user_crud_flow():+ if os.getenv("RAGFLOW_GAUSSDB_LIVE") != "1":+ pytest.skip("Set RAGFLOW_GAUSSDB_LIVE=1 and GAUSSDB_METADATA_* env vars to run live GaussDB flow")++ required = ["GAUSSDB_METADATA_HOST", "GAUSSDB_METADATA_PORT", "GAUSSDB_METADATA_USER", "GAUSSDB_METADATA_PASSWORD", "GAUSSDB_METADATA_DBNAME"]+ missing = [name for name in required if not os.getenv(name)]+ if missing:+ pytest.skip(f"Missing live GaussDB env vars: {', '.join(missing)}")++ env_extra = {+ "DB_TYPE": "GaussDB",+ "GAUSSDB_METADATA_HOST": os.environ["GAUSSDB_METADATA_HOST"],+ "GAUSSDB_METADATA_PORT": os.environ["GAUSSDB_METADATA_PORT"],+ "GAUSSDB_METADATA_USER": os.environ["GAUSSDB_METADATA_USER"],+ "GAUSSDB_METADATA_PASSWORD": os.environ["GAUSSDB_METADATA_PASSWORD"],+ "GAUSSDB_METADATA_DBNAME": os.environ["GAUSSDB_METADATA_DBNAME"],+ "GAUSSDB_METADATA_SCHEMA": os.getenv("GAUSSDB_METADATA_SCHEMA", "public"),+ "GAUSSDB_METADATA_MAX_CONNECTIONS": os.getenv("GAUSSDB_METADATA_MAX_CONNECTIONS", "20"),+ "GAUSSDB_METADATA_STALE_TIMEOUT": os.getenv("GAUSSDB_METADATA_STALE_TIMEOUT", "30"),+ }++ result = run_isolated_flow(+ """+import os+import uuid++from common import settings+from api.db.db_models import DB, Tenant, User, init_database_tables+from api.db.services.user_service import TenantService+from api.utils.health_utils import get_database_status++assert settings.DATABASE_TYPE == "gaussdb"+init_database_tables()++expected_distributed = os.environ.get("EXPECT_DISTRIBUTED") == "1"+assert DB.is_distributed is expected_distributed+if expected_distributed:+ rows = DB.execute_sql(+ "SELECT c.relname, x.pclocatortype "+ "FROM pg_catalog.pgxc_class x "+ "JOIN pg_catalog.pg_class c ON c.oid = x.pcrelid "+ "JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace "+ "WHERE n.nspname = current_schema() "+ "AND c.relname IN ('compilation_template', 'compilation_template_group', "+ "'file_commit_item', 'tenant_llm', 'tenant_model_provider', 'user')"+ ).fetchall()+ assert dict(rows) == {+ "compilation_template": "R",+ "compilation_template_group": "R",+ "file_commit_item": "R",+ "tenant_llm": "R",+ "tenant_model_provider": "R",+ "user": "R",+ }++health = get_database_status()+assert health["status"] == "alive"+assert health["message"]["database"] == "gaussdb"+assert health["message"]["result"] == 1++uid = uuid.uuid4().hex+email = f"{uid}@ragflow-gaussdb-flow.test"+tenant_id = uuid.uuid4().hex+replacement_tenant_id = uuid.uuid4().hex++with DB.connection_context():+ with DB.lock("gaussdb_user_flow_test", 10):+ try:+ User.create(id=uid, email=email, nickname="gaussdb-flow", password="x")+ row = User.get(User.id == uid)+ assert row.id == uid+ assert row.email == email+ finally:+ User.delete().where(User.id == uid).execute()++ assert User.select().where(User.id == uid).count() == 0++ try:+ Tenant.create(+ id=tenant_id,+ name="gaussdb-empty-string-flow",+ llm_id="",+ embd_id="",+ asr_id="",+ img2txt_id="",+ rerank_id="",+ parser_ids="naive:General",+ )+ raw_row = DB.execute_sql(+ "select llm_id, embd_id, asr_id, img2txt_id, rerank_id from tenant where id=%s",+ (tenant_id,),+ ).fetchone()+ assert raw_row == (None, None, None, None, None)++ tenant = Tenant.get(Tenant.id == tenant_id)+ assert tenant.llm_id == ""+ assert tenant.embd_id == ""+ assert tenant.asr_id == ""+ assert tenant.img2txt_id == ""+ assert tenant.rerank_id == ""++ assert Tenant.select().where(Tenant.id == tenant_id, Tenant.llm_id == "").count() == 1+ assert Tenant.select().where(Tenant.id == tenant_id, Tenant.llm_id != "").count() == 0++ assert TenantService.update_by_id(+ tenant_id,+ {"id": replacement_tenant_id, "name": "gaussdb-primary-key-replacement"},+ ) == 1+ assert Tenant.select().where(Tenant.id == tenant_id).count() == 0+ assert Tenant.select().where(Tenant.id == replacement_tenant_id).count() == 1+ finally:+ Tenant.delete().where(Tenant.id == tenant_id).execute()+ Tenant.delete().where(Tenant.id == replacement_tenant_id).execute()++DB.close()+print("gaussdb-live-flow-ok")+""",+ env_extra,+ timeout=120,+ )++ assert "gaussdb-live-flow-ok" in result.stdoutdiff --git a/test/unit_test/api/utils/test_gaussdb_health.py b/test/unit_test/api/utils/test_gaussdb_health.pynew file mode 100644index 0000000000..3de468d753--- /dev/null+++ b/test/unit_test/api/utils/test_gaussdb_health.py@@ -0,0 +1,381 @@+#+# Copyright 2025 The InfiniFlow Authors. All Rights Reserved.+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+#+import importlib.util+import json+from pathlib import Path+import sys+import types+from unittest.mock import Mock++import pytest++CFG614_PASSWORD_SENTINEL = "secret123"+PASSWORD_SENTINEL = "cfg711-'quoted;password'"+FULL_DSN_SENTINEL = "postgresql://sqlbuilder:cfg711-dsn%27%22%[email protected]:19995/postgres?schema=private"+ACCESS_TOKEN_SENTINEL = "cfg711-token-'quoted![]{}'"+++class FakeGaussDBConnection:+ def db_type(self):+ return "gaussdb"++ def health(self):+ return {+ "status": "healthy",+ "uri": "[email protected]:19995/postgres?schema=public",+ "version_comment": "GaussDB",+ "sql_compatibility": "A",+ }++ def get_performance_metrics(self):+ return {"connection": "connected", "latency_ms": 3.2}+++class FailingGaussDBConnection:+ def health(self):+ raise RuntimeError(f"FATAL: password authentication failed password={CFG614_PASSWORD_SENTINEL} dsn={FULL_DSN_SENTINEL} access_token={ACCESS_TOKEN_SENTINEL}")+++class UnhealthyGaussDBConnection(FakeGaussDBConnection):+ def health(self):+ return {+ "status": "unhealthy",+ "error": "compatibility unsupported password=fake-unit-password",+ }++ def get_performance_metrics(self):+ return {"connection": "connected", "latency_ms": 1.2, "error": "dsn password:fake-unit-password"}+++class IncompatibleGaussDBConnection(FakeGaussDBConnection):+ def health(self):+ return {+ "status": "unhealthy",+ "uri": "[email protected]:19995/postgres?schema=public",+ "version_comment": "GaussDB",+ "sql_compatibility": "PG",+ "error": "unsupported GaussDB compatibility, expected A/ORA: sql_compatibility=PG",+ }+++class SecretBearingGaussDBConnection(FakeGaussDBConnection):+ def health(self):+ return {+ "status": "unhealthy",+ "schema": "public",+ "credentials": {+ "password": PASSWORD_SENTINEL,+ "connections": [{"dsn": FULL_DSN_SENTINEL}, ({"ACCESS_TOKEN": ACCESS_TOKEN_SENTINEL},)],+ },+ "error": f'authentication failed password="{PASSWORD_SENTINEL}"',+ }++ def get_performance_metrics(self):+ return {+ "connection": "disconnected",+ "schema": "public",+ "details": [+ {"dsn": FULL_DSN_SENTINEL},+ ({"access-token": ACCESS_TOKEN_SENTINEL},),+ ],+ }+++def _install_module(monkeypatch, name, **attrs):+ mod = types.ModuleType(name)+ for key, value in attrs.items():+ setattr(mod, key, value)+ monkeypatch.setitem(sys.modules, name, mod)+ return mod+++def _import_health_utils(monkeypatch):+ import common++ settings_stub = types.SimpleNamespace(+ docStoreConn=types.SimpleNamespace(health=lambda: {"status": "healthy"}),+ STORAGE_IMPL=types.SimpleNamespace(health=lambda: True),+ STORAGE_IMPL_TYPE="MINIO",+ MINIO={"host": "minio:9000"},+ )+ monkeypatch.setitem(sys.modules, "common.settings", settings_stub)+ monkeypatch.setattr(common, "settings", settings_stub, raising=False)++ _install_module(monkeypatch, "api.db.db_models", DB=types.SimpleNamespace(execute_sql=lambda *_args, **_kwargs: True))+ _install_module(monkeypatch, "rag.utils.redis_conn", REDIS_CONN=types.SimpleNamespace(health=lambda: True))+ _install_module(monkeypatch, "rag.utils.es_conn", ESConnection=lambda: types.SimpleNamespace(get_cluster_stats=lambda: {}))+ _install_module(monkeypatch, "rag.utils.infinity_conn", InfinityConnection=lambda: types.SimpleNamespace(health=lambda: {}))+ _install_module(monkeypatch, "rag.utils.ob_conn", OBConnection=lambda: types.SimpleNamespace(health=lambda: {}, get_performance_metrics=lambda: {}))+ _install_module(monkeypatch, "rag.utils.gaussdb_conn", GaussDBConnection=FakeGaussDBConnection)++ module_path = Path(__file__).resolve().parents[4] / "api" / "utils" / "health_utils.py"+ spec = importlib.util.spec_from_file_location("_gaussdb_health_utils_under_test", module_path)+ assert spec and spec.loader+ module = importlib.util.module_from_spec(spec)+ spec.loader.exec_module(module)+ return module+++def test_tc_cfg_611_get_gaussdb_status_returns_complete_not_configured_response(monkeypatch):+ health_utils = _import_health_utils(monkeypatch)+ monkeypatch.setenv("DOC_ENGINE", "elasticsearch")++ result = health_utils.get_gaussdb_status()++ assert result == {+ "status": "not_configured",+ "message": "GaussDB is not configured as the document engine",+ }+++def test_tc_cfg_616_get_gaussdb_status_composes_healthy_probe_results(monkeypatch):+ health_utils = _import_health_utils(monkeypatch)+ monkeypatch.setenv("DOC_ENGINE", "gaussdb")+ connection = FakeGaussDBConnection()+ get_connection = Mock(return_value=connection)+ monkeypatch.setattr(health_utils, "_get_gaussdb_connection", get_connection)++ result = health_utils.get_gaussdb_status()++ assert result == {+ "status": "alive",+ "message": {+ "health": {+ "status": "healthy",+ "uri": "[email protected]:19995/postgres?schema=public",+ "version_comment": "GaussDB",+ "sql_compatibility": "A",+ },+ "performance": {"connection": "connected", "latency_ms": 3.2},+ },+ }+ get_connection.assert_called_once_with()+++def test_tc_cfg_617_get_gaussdb_status_reuses_settings_doc_store_connection(monkeypatch):+ health_utils = _import_health_utils(monkeypatch)+ doc_store_conn = FakeGaussDBConnection()+ monkeypatch.setattr(health_utils.settings, "docStoreConn", doc_store_conn, raising=False)+ monkeypatch.setenv("DOC_ENGINE", "gaussdb")+ constructor = Mock(side_effect=AssertionError("unexpected GaussDBConnection construction"))+ monkeypatch.setattr(health_utils, "GaussDBConnection", constructor)++ result = health_utils.get_gaussdb_status()++ assert result["status"] == "alive"+ assert result["message"]["performance"]["connection"] == "connected"+ constructor.assert_not_called()+++def test_tc_cfg_618_get_gaussdb_status_constructs_adapter_without_shared_connection(monkeypatch):+ health_utils = _import_health_utils(monkeypatch)+ connection = object()+ constructor = Mock(return_value=connection)+ monkeypatch.setattr(health_utils.settings, "docStoreConn", None, raising=False)+ monkeypatch.setattr(health_utils, "GaussDBConnection", constructor)++ resolved = health_utils._get_gaussdb_connection()++ assert resolved is connection+ constructor.assert_called_once_with()+++def test_tc_cfg_618_get_gaussdb_status_constructs_adapter_for_non_gaussdb_connection(monkeypatch):+ health_utils = _import_health_utils(monkeypatch)+ connection = object()+ constructor = Mock(return_value=connection)+ other_doc_store = types.SimpleNamespace(db_type=lambda: "elasticsearch")+ monkeypatch.setattr(health_utils.settings, "docStoreConn", other_doc_store, raising=False)+ monkeypatch.setattr(health_utils, "GaussDBConnection", constructor)++ resolved = health_utils._get_gaussdb_connection()++ assert resolved is connection+ constructor.assert_called_once_with()+++def test_tc_cfg_614_get_gaussdb_status_error_masks_password(monkeypatch, caplog):+ health_utils = _import_health_utils(monkeypatch)+ monkeypatch.setenv("DOC_ENGINE", "gaussdb")+ monkeypatch.setattr(health_utils, "_get_gaussdb_connection", Mock(return_value=FailingGaussDBConnection()))+ caplog.set_level("ERROR")++ result = health_utils.get_gaussdb_status()+ log_text = caplog.text++ assert result["status"] == "timeout"+ assert CFG614_PASSWORD_SENTINEL not in result["message"]+ assert FULL_DSN_SENTINEL not in result["message"]+ assert ACCESS_TOKEN_SENTINEL not in result["message"]+ assert "postgresql://" not in result["message"]+ assert "***" in result["message"]+ assert "GaussDB status check failed (RuntimeError)" in log_text+ assert "***" in log_text+ assert CFG614_PASSWORD_SENTINEL not in log_text+ assert FULL_DSN_SENTINEL not in log_text+ assert ACCESS_TOKEN_SENTINEL not in log_text+++def test_tc_cfg_613_get_gaussdb_status_reports_timeout_when_health_is_unhealthy(monkeypatch):+ health_utils = _import_health_utils(monkeypatch)+ monkeypatch.setenv("DOC_ENGINE", "gaussdb")+ monkeypatch.setattr(health_utils, "_get_gaussdb_connection", Mock(return_value=UnhealthyGaussDBConnection()))++ result = health_utils.get_gaussdb_status()++ assert result["status"] == "timeout"+ assert result["message"]["health"]["status"] == "unhealthy"+ assert result["message"]["health"]["error"] == "compatibility unsupported password=***"+ assert result["message"]["performance"]["connection"] == "connected"+ assert result["message"]["performance"]["latency_ms"] == 1.2+ assert result["message"]["performance"]["error"] == "dsn password:***"+++def test_tc_cfg_619_check_gaussdb_health_reports_healthy_details(monkeypatch):+ health_utils = _import_health_utils(monkeypatch)+ monkeypatch.setenv("DOC_ENGINE", "gaussdb")+ monkeypatch.setattr(+ health_utils,+ "_get_gaussdb_connection",+ Mock(return_value=FakeGaussDBConnection()),+ )++ result = health_utils.check_gaussdb_health()++ assert result["status"] == "healthy"+ assert result["details"]["connection"] == "connected"+ assert result["details"]["latency_ms"] == 3.2+ assert result["details"]["sql_compatibility"] == "A"+++def test_tc_cfg_620_check_gaussdb_health_reports_incompatible_mode(monkeypatch):+ health_utils = _import_health_utils(monkeypatch)+ monkeypatch.setenv("DOC_ENGINE", "gaussdb")+ monkeypatch.setattr(+ health_utils,+ "_get_gaussdb_connection",+ Mock(return_value=IncompatibleGaussDBConnection()),+ )++ result = health_utils.check_gaussdb_health()++ assert result["status"] == "unhealthy"+ assert result["details"]["version"] == "GaussDB"+ assert result["details"]["sql_compatibility"] == "PG"+ assert "A/ORA" in result["details"]["error"]+++def test_tc_cfg_614_check_gaussdb_health_logs_masked_exception(monkeypatch, caplog):+ health_utils = _import_health_utils(monkeypatch)+ monkeypatch.setenv("DOC_ENGINE", "gaussdb")+ monkeypatch.setattr(health_utils, "_get_gaussdb_connection", Mock(return_value=FailingGaussDBConnection()))+ caplog.set_level("ERROR")++ result = health_utils.check_gaussdb_health()+ log_text = caplog.text++ assert result["status"] == "unhealthy"+ assert result["details"]["connection"] == "disconnected"+ assert "GaussDB health check failed (RuntimeError)" in log_text+ assert "***" in log_text+ assert CFG614_PASSWORD_SENTINEL not in log_text+ assert FULL_DSN_SENTINEL not in log_text+ assert ACCESS_TOKEN_SENTINEL not in log_text+++def test_tc_cfg_711_check_gaussdb_health_masks_nested_performance_errors(monkeypatch):+ health_utils = _import_health_utils(monkeypatch)+ monkeypatch.setenv("DOC_ENGINE", "gaussdb")+ monkeypatch.setattr(+ health_utils,+ "_get_gaussdb_connection",+ Mock(return_value=SecretBearingGaussDBConnection()),+ )++ result = health_utils.check_gaussdb_health()+ serialized = json.dumps(result, sort_keys=True)++ assert result["status"] == "unhealthy"+ assert PASSWORD_SENTINEL not in serialized+ assert FULL_DSN_SENTINEL not in serialized+ assert ACCESS_TOKEN_SENTINEL not in serialized+++def test_tc_cfg_707_mask_gaussdb_string_masks_password_equals(monkeypatch):+ health_utils = _import_health_utils(monkeypatch)++ assert health_utils._mask_gaussdb_string("host=h password=secret port=5432") == "host=h password=*** port=5432"+++def test_tc_cfg_708_mask_gaussdb_string_masks_password_colon(monkeypatch):+ health_utils = _import_health_utils(monkeypatch)++ assert health_utils._mask_gaussdb_string("connect failed password:fake-unit-password") == "connect failed password:***"++[email protected]("value", ["PASSWORD=fake-unit-password", "Password=fake-unit-password"])+def test_tc_cfg_709_mask_gaussdb_string_is_case_insensitive(monkeypatch, value):+ health_utils = _import_health_utils(monkeypatch)++ assert health_utils._mask_gaussdb_string(value).lower() == "password=***"+++def test_tc_cfg_710_mask_gaussdb_secret_masks_nested_lists_and_tuples(monkeypatch):+ health_utils = _import_health_utils(monkeypatch)++ masked = health_utils._mask_gaussdb_secret(+ {+ "nested": {+ "password": PASSWORD_SENTINEL,+ "error": f'password="{PASSWORD_SENTINEL}"',+ },+ "containers": [+ {"ACCESS_TOKEN": ACCESS_TOKEN_SENTINEL},+ ({"dsn": FULL_DSN_SENTINEL},),+ ],+ }+ )+ serialized = json.dumps(masked, sort_keys=True)++ assert masked["nested"]["password"] == "***"+ assert "***" in masked["nested"]["error"]+ assert masked["containers"][0]["ACCESS_TOKEN"] == "***"+ assert masked["containers"][1][0]["dsn"] == "***"+ assert PASSWORD_SENTINEL not in serialized+ assert FULL_DSN_SENTINEL not in serialized+ assert ACCESS_TOKEN_SENTINEL not in serialized+++def test_tc_cfg_711_get_gaussdb_status_masks_password_dsn_and_access_token_sentinels(monkeypatch):+ health_utils = _import_health_utils(monkeypatch)+ monkeypatch.setenv("DOC_ENGINE", "gaussdb")+ monkeypatch.setattr(health_utils, "_get_gaussdb_connection", Mock(return_value=SecretBearingGaussDBConnection()))++ result = health_utils.get_gaussdb_status()+ serialized = json.dumps(result, sort_keys=True)++ assert result["status"] == "timeout"+ assert result["message"]["health"]["status"] == "unhealthy"+ assert result["message"]["health"]["schema"] == "public"+ assert result["message"]["performance"]["connection"] == "disconnected"+ assert result["message"]["performance"]["schema"] == "public"+ assert result["message"]["health"]["credentials"]["password"] == "***"+ assert result["message"]["health"]["credentials"]["connections"][0]["dsn"] == "***"+ assert result["message"]["health"]["credentials"]["connections"][1][0]["ACCESS_TOKEN"] == "***"+ assert PASSWORD_SENTINEL not in serialized+ assert FULL_DSN_SENTINEL not in serialized+ assert ACCESS_TOKEN_SENTINEL not in serialized+ assert "postgresql://" not in serializeddiff --git a/test/unit_test/common/test_gaussdb_settings.py b/test/unit_test/common/test_gaussdb_settings.pynew file mode 100644index 0000000000..11b2f66d8e--- /dev/null+++ b/test/unit_test/common/test_gaussdb_settings.py@@ -0,0 +1,280 @@+#+# Copyright 2025 The InfiniFlow Authors. All Rights Reserved.+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+#+import importlib+import sys+import types+from unittest.mock import Mock++import pytest++FAKE_GAUSSDB_POOL = object()+++class FakeDocEngineConnection:+ def db_type(self):+ return "stub"+++class FakeGaussDBConnection:+ def __init__(self, pool=None):+ self.pool = pool or FAKE_GAUSSDB_POOL++ def db_type(self):+ return "gaussdb"+++class FakeGaussDBMemoryConnection:+ def __init__(self, pool=None):+ self.pool = pool or FAKE_GAUSSDB_POOL++ def db_type(self):+ return "gaussdb"+++class FakeStorage:+ def health(self):+ return True+++def _install_module(monkeypatch, name, **attrs):+ mod = types.ModuleType(name)+ for key, value in attrs.items():+ setattr(mod, key, value)+ monkeypatch.setitem(sys.modules, name, mod)+ parent_name, _, child_name = name.rpartition(".")+ parent = sys.modules.get(parent_name)+ if parent is not None:+ monkeypatch.setattr(parent, child_name, mod, raising=False)+ return mod+++def _install_settings_import_stubs(monkeypatch):+ importlib.import_module("rag.utils")+ importlib.import_module("memory.utils")++ rag_modules = {+ "es_conn": {"ESConnection": FakeDocEngineConnection},+ "infinity_conn": {"InfinityConnection": FakeDocEngineConnection},+ "ob_conn": {"OBConnection": FakeDocEngineConnection},+ "opensearch_conn": {"OSConnection": FakeDocEngineConnection},+ "gaussdb_conn": {"GaussDBConnection": FakeGaussDBConnection},+ "azure_sas_conn": {"RAGFlowAzureSasBlob": FakeStorage},+ "azure_spn_conn": {"RAGFlowAzureSpnBlob": FakeStorage},+ "gcs_conn": {"RAGFlowGCS": FakeStorage},+ "minio_conn": {"RAGFlowMinio": FakeStorage},+ "opendal_conn": {"OpenDALStorage": FakeStorage},+ "s3_conn": {"RAGFlowS3": FakeStorage},+ "oss_conn": {"RAGFlowOSS": FakeStorage},+ "redis_conn": {"REDIS_CONN": types.SimpleNamespace(health=lambda: True)},+ }+ for short_name, attrs in rag_modules.items():+ _install_module(monkeypatch, f"rag.utils.{short_name}", **attrs)++ memory_modules = {+ "es_conn": {"ESConnection": FakeDocEngineConnection},+ "infinity_conn": {"InfinityConnection": FakeDocEngineConnection},+ "ob_conn": {"OBConnection": FakeDocEngineConnection},+ "gaussdb_conn": {"GaussDBMemoryConnection": FakeGaussDBMemoryConnection},+ }+ for short_name, attrs in memory_modules.items():+ _install_module(monkeypatch, f"memory.utils.{short_name}", **attrs)++ fake_search = types.SimpleNamespace(Dealer=lambda conn: ("dealer", conn))+ fake_kg_search = types.SimpleNamespace(KGSearch=lambda conn: ("kg", conn))+ _install_module(monkeypatch, "rag.nlp", search=fake_search)+ _install_module(monkeypatch, "rag.graphrag", search=fake_kg_search)+ _install_module(monkeypatch, "rag.graphrag.search", KGSearch=fake_kg_search.KGSearch)+++def _import_settings(monkeypatch):+ _install_settings_import_stubs(monkeypatch)+ import common++ # Import machinery assigns a freshly imported submodule to+ # ``common.settings`` on the parent package. Restoring only+ # ``sys.modules["common.settings"]`` leaves that parent attribute pointing+ # at this test's stub-backed module and contaminates later tests.+ monkeypatch.setattr(+ common,+ "settings",+ getattr(common, "settings", None),+ raising=False,+ )+ monkeypatch.delitem(sys.modules, "common.settings", raising=False)+ return importlib.import_module("common.settings")+++def test_doc_engine_gaussdb_initializes_gaussdb_connection(monkeypatch):+ settings = _import_settings(monkeypatch)++ monkeypatch.setenv("DOC_ENGINE", "gaussdb")+ monkeypatch.setattr(+ settings,+ "get_base_config",+ lambda name, default=None: (+ {+ "host": "db.example",+ "port": 19995,+ "database": "postgres",+ "user": "sqlbuilder",+ "password": "fake-unit-password",+ "schema": "ragflow_gaussdb_docengine_it",+ }+ if name == "gaussdb"+ else (default or {})+ ),+ )++ settings.init_settings()++ assert settings.DOC_ENGINE == "gaussdb"+ assert settings.DOC_ENGINE_GAUSSDB is True+ assert settings.DOC_ENGINE_OCEANBASE is False+ assert settings.docStoreConn.db_type() == "gaussdb"+ assert settings.msgStoreConn.db_type() == "gaussdb"+ assert settings.docStoreConn is not settings.msgStoreConn+ assert settings.docStoreConn.pool is settings.msgStoreConn.pool+ assert settings.GAUSSDB["schema"] == "ragflow_gaussdb_docengine_it"+++def test_doc_engine_gaussdb_does_not_change_database_type(monkeypatch):+ settings = _import_settings(monkeypatch)++ monkeypatch.setenv("DOC_ENGINE", "gaussdb")+ monkeypatch.setenv("DB_TYPE", "mysql")+ monkeypatch.setattr(+ settings,+ "get_base_config",+ lambda name, default=None: (+ {+ "host": "h",+ "port": 1,+ "database": "d",+ "user": "u",+ "password": "fake-unit-password",+ }+ if name == "gaussdb"+ else (default or {})+ ),+ )++ settings.init_settings()++ assert settings.DATABASE_TYPE != "gaussdb"+ assert settings.docStoreConn.db_type() == "gaussdb"+ assert settings.msgStoreConn.db_type() == "gaussdb"+ assert settings.docStoreConn.pool is settings.msgStoreConn.pool+++def test_doc_engine_gaussdb_initializes_message_store_case_insensitive(monkeypatch):+ settings = _import_settings(monkeypatch)++ monkeypatch.setenv("DOC_ENGINE", "GaussDB")+ monkeypatch.setattr(+ settings,+ "get_base_config",+ lambda name, default=None: (+ {+ "host": "h",+ "port": 19995,+ "database": "d",+ "user": "u",+ "password": "fake-unit-password",+ }+ if name == "gaussdb"+ else (default or {})+ ),+ )++ settings.init_settings()++ assert settings.DOC_ENGINE == "GaussDB"+ assert settings.DOC_ENGINE_GAUSSDB is True+ assert settings.docStoreConn.db_type() == "gaussdb"+ assert settings.msgStoreConn.db_type() == "gaussdb"+ assert settings.docStoreConn.pool is settings.msgStoreConn.pool+++def test_tc_cfg_503_default_doc_engine_is_not_gaussdb(monkeypatch):+ settings = _import_settings(monkeypatch)+ gaussdb_module = sys.modules["rag.utils.gaussdb_conn"]+ gaussdb_constructor = Mock(side_effect=AssertionError("GaussDB must not be constructed"))+ monkeypatch.setattr(gaussdb_module, "GaussDBConnection", gaussdb_constructor)++ monkeypatch.delenv("DOC_ENGINE", raising=False)+ monkeypatch.setattr(settings, "get_base_config", lambda _name, default=None: default or {})++ settings.init_settings()++ assert settings.DOC_ENGINE == "elasticsearch"+ assert settings.DOC_ENGINE_GAUSSDB is False+ assert settings.docStoreConn.db_type() == "stub"+ gaussdb_constructor.assert_not_called()+++def test_tc_cfg_504_unknown_doc_engine_is_rejected(monkeypatch):+ settings = _import_settings(monkeypatch)++ monkeypatch.setenv("DOC_ENGINE", "unknown_db")+ monkeypatch.setattr(settings, "get_base_config", lambda _name, default=None: default or {})+ settings.docStoreConn = None++ with pytest.raises(Exception) as exc_info:+ settings.init_settings()++ assert str(exc_info.value) == "Not supported doc engine: unknown_db"+ assert settings.DOC_ENGINE == "unknown_db"+ assert settings.DOC_ENGINE_GAUSSDB is False+ assert settings.docStoreConn is None+++def test_db_type_and_doc_engine_gaussdb_use_separate_configs(monkeypatch):+ settings = _import_settings(monkeypatch)++ monkeypatch.setenv("DB_TYPE", "gaussdb")+ monkeypatch.setenv("DOC_ENGINE", "gaussdb")+ monkeypatch.setenv("GAUSSDB_METADATA_HOST", "metadata.example")+ monkeypatch.setenv("GAUSSDB_METADATA_PORT", "8000")+ monkeypatch.setenv("GAUSSDB_METADATA_DBNAME", "metadata_db")+ monkeypatch.setenv("GAUSSDB_METADATA_USER", "metadata_user")+ monkeypatch.setenv("GAUSSDB_METADATA_PASSWORD", "metadata-secret")+ monkeypatch.setenv("GAUSSDB_METADATA_SCHEMA", "metadata_schema")+ monkeypatch.setattr(+ settings,+ "get_base_config",+ lambda name, default=None: (+ {+ "host": "doc.example",+ "port": 19995,+ "database": "doc_db",+ "user": "doc_user",+ "password": "doc-secret",+ "schema": "doc_schema",+ }+ if name == "gaussdb"+ else (default or {})+ ),+ )++ settings.init_settings()++ assert settings.DATABASE_TYPE == "gaussdb"+ assert settings.DATABASE["host"] == "metadata.example"+ assert settings.DATABASE["name"] == "metadata_db"+ assert settings.DATABASE["options"].startswith("-c search_path=metadata_schema ")+ assert settings.GAUSSDB["host"] == "doc.example"+ assert settings.GAUSSDB["database"] == "doc_db"+ assert settings.docStoreConn.pool is settings.msgStoreConn.pooldiff --git a/test/unit_test/deploy/test_gaussdb_docengine_config.py b/test/unit_test/deploy/test_gaussdb_docengine_config.pynew file mode 100644index 0000000000..a00fd08924--- /dev/null+++ b/test/unit_test/deploy/test_gaussdb_docengine_config.py@@ -0,0 +1,211 @@+#+# Copyright 2025 The InfiniFlow Authors. All Rights Reserved.+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+#+from pathlib import Path+import re+import subprocess+from unittest.mock import patch++from ruamel.yaml import YAML++ROOT = Path(__file__).resolve().parents[3]+SERVICE_CONF_TEMPLATE = ROOT / "docker" / "service_conf.yaml.template"+_PLACEHOLDER_PATTERN = re.compile(r"\$\{([A-Za-z_][A-Za-z0-9_]*)(?::-([^}]*))?\}")+++def _expand_service_conf_template(template_path, output_path, environ):+ def replace_placeholder(match):+ value = environ.get(match.group(1))+ return value if value else (match.group(2) or "")++ rendered = _PLACEHOLDER_PATTERN.sub(replace_placeholder, Path(template_path).read_text(encoding="utf-8"))+ destination = Path(output_path)+ destination.write_text(rendered, encoding="utf-8")+ return destination+++def _load_yaml(path):+ yaml = YAML(typ="safe", pure=True)+ return yaml.load(path.read_text(encoding="utf-8"))+++def _import_gaussdb_conn_pool(monkeypatch):+ from common import settings++ monkeypatch.setattr(+ settings,+ "GAUSSDB",+ {+ "host": "gaussdb.local",+ "port": "19995",+ "database": "postgres",+ "user": "sqlbuilder",+ "password": "fake-unit-password",+ },+ raising=False,+ )+ with patch("psycopg2.pool.ThreadedConnectionPool", lambda *_args, **_kwargs: object()):+ from common.doc_store import gaussdb_conn_pool++ return gaussdb_conn_pool+++def test_tc_cfg_801_docker_template_defines_complete_gaussdb_config(tmp_path):+ entrypoint = (ROOT / "docker" / "entrypoint.sh").read_text(encoding="utf-8")+ template = (ROOT / "docker" / "service_conf.yaml.template").read_text(encoding="utf-8")+ gaussdb_block = template.split("gaussdb:", 1)[1].split("seekdb:", 1)[0]++ assert 'DEF_ENV_VALUE_PATTERN="\\$\\{([^:]+):-([^}]+)\\}"' in entrypoint+ assert 'done < "${TEMPLATE_FILE}"' in entrypoint+ assert "common.service_conf_renderer" not in entrypoint+ assert "host: '${GAUSSDB_HOST}'" in gaussdb_block+ assert "port: ${GAUSSDB_PORT}" in gaussdb_block+ assert "database: '${GAUSSDB_DATABASE:-postgres}'" in gaussdb_block+ assert "user: '${GAUSSDB_USER}'" in gaussdb_block+ assert "password: '${GAUSSDB_PASSWORD}'" in gaussdb_block+ assert "schema: '${GAUSSDB_SCHEMA:-public}'" in gaussdb_block++ rendered_path = _expand_service_conf_template(+ SERVICE_CONF_TEMPLATE,+ tmp_path / "service_conf.yaml",+ {+ "GAUSSDB_HOST": "h",+ "GAUSSDB_PORT": "5432",+ "GAUSSDB_DATABASE": "d",+ "GAUSSDB_USER": "u",+ "GAUSSDB_PASSWORD": "p",+ "GAUSSDB_SCHEMA": "zlw",+ },+ )++ assert rendered_path.parent == tmp_path+ assert _load_yaml(rendered_path)["gaussdb"] == {+ "host": "h",+ "port": 5432,+ "database": "d",+ "user": "u",+ "password": "p",+ "schema": "zlw",+ }+++def test_tc_cfg_802_docker_template_defaults_empty_schema_to_public(tmp_path):+ template = (ROOT / "docker" / "service_conf.yaml.template").read_text(encoding="utf-8")+ gaussdb_block = template.split("gaussdb:", 1)[1].split("seekdb:", 1)[0]+ assert "schema: '${GAUSSDB_SCHEMA:-public}'" in gaussdb_block++ rendered_path = _expand_service_conf_template(+ SERVICE_CONF_TEMPLATE,+ tmp_path / "service_conf.yaml",+ {+ "GAUSSDB_HOST": "h",+ "GAUSSDB_PORT": "5432",+ "GAUSSDB_USER": "u",+ "GAUSSDB_PASSWORD": "p",+ "GAUSSDB_SCHEMA": "",+ },+ )++ config = _load_yaml(rendered_path)["gaussdb"]+ assert config["database"] == "postgres"+ assert config["schema"] == "public"+++def test_tc_cfg_804_docker_rendered_gaussdb_config_excludes_tuning_fields(tmp_path):+ template = (ROOT / "docker" / "service_conf.yaml.template").read_text(encoding="utf-8")+ gaussdb_block = template.split("gaussdb:", 1)[1].split("seekdb:", 1)[0]+ rendered_path = _expand_service_conf_template(+ SERVICE_CONF_TEMPLATE,+ tmp_path / "service_conf.yaml",+ {+ "GAUSSDB_HOST": "h",+ "GAUSSDB_PORT": "5432",+ "GAUSSDB_DATABASE": "d",+ "GAUSSDB_USER": "u",+ "GAUSSDB_PASSWORD": "p",+ "GAUSSDB_SCHEMA": "zlw",+ "GAUSSDB_SSL": "true",+ "GAUSSDB_TIMEOUT": "30",+ "GAUSSDB_POOL_MIN": "4",+ "GAUSSDB_POOL_MAX": "32",+ },+ )+ gaussdb = _load_yaml(rendered_path)["gaussdb"]++ assert "password: '${GAUSSDB_PASSWORD}'" in gaussdb_block+ assert set(gaussdb) == {"host", "port", "database", "user", "password", "schema"}+ forbidden = {"ssl", "sslmode", "timeout", "connect_timeout", "pool_min", "pool_max", "minconn", "maxconn"}+ assert forbidden.isdisjoint(gaussdb)+ assert all(f"GAUSSDB_{name.upper()}" not in gaussdb_block for name in forbidden)+++def test_tc_cfg_807_rendered_service_conf_gaussdb_block_loads_as_gaussdb_config(tmp_path, monkeypatch):+ rendered_path = _expand_service_conf_template(+ SERVICE_CONF_TEMPLATE,+ tmp_path / "service_conf.yaml",+ {+ "GAUSSDB_HOST": "h",+ "GAUSSDB_PORT": "5432",+ "GAUSSDB_DATABASE": "d",+ "GAUSSDB_USER": "u",+ "GAUSSDB_PASSWORD": "p",+ "GAUSSDB_SCHEMA": "zlw",+ },+ )+ rendered = _load_yaml(rendered_path)+ pool_mod = _import_gaussdb_conn_pool(monkeypatch)++ cfg = pool_mod.load_gaussdb_config(rendered["gaussdb"])++ assert cfg.host == "h"+ assert cfg.port == 5432+ assert cfg.database == "d"+ assert cfg.user == "u"+ assert cfg.password == "p"+ assert cfg.schema == "zlw"+++def test_tc_cfg_808_rendered_artifacts_stay_under_tmp_path_and_git_status_is_stable(tmp_path):+ before = subprocess.run(+ ["git", "status", "--porcelain=v1", "--untracked-files=all"],+ cwd=ROOT,+ encoding="utf-8",+ capture_output=True,+ check=True,+ ).stdout++ service_conf = _expand_service_conf_template(+ SERVICE_CONF_TEMPLATE,+ tmp_path / "service_conf.yaml",+ {+ "GAUSSDB_HOST": "gaussdb.local",+ "GAUSSDB_PORT": "19995",+ "GAUSSDB_DATABASE": "postgres",+ "GAUSSDB_USER": "ragflow",+ "GAUSSDB_PASSWORD": "secret",+ "GAUSSDB_SCHEMA": "public",+ },+ )+ assert service_conf.is_relative_to(tmp_path)+ assert _load_yaml(service_conf)["gaussdb"]["host"] == "gaussdb.local"++ after = subprocess.run(+ ["git", "status", "--porcelain=v1", "--untracked-files=all"],+ cwd=ROOT,+ encoding="utf-8",+ capture_output=True,+ check=True,+ ).stdout+ assert after == beforediff --git a/test/unit_test/memory/utils/test_gaussdb_memory_conn.py b/test/unit_test/memory/utils/test_gaussdb_memory_conn.pynew file mode 100644index 0000000000..812ff59509--- /dev/null+++ b/test/unit_test/memory/utils/test_gaussdb_memory_conn.py@@ -0,0 +1,643 @@+#+# Copyright 2025 The InfiniFlow Authors. All Rights Reserved.+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+#+from datetime import datetime+from decimal import Decimal++import numpy as np+import pytest++from common.doc_store.doc_store_base import FusionExpr, MatchDenseExpr, MatchTextExpr, OrderByExpr+from common.doc_store.gaussdb_conn_base import InvalidGaussDBObjectName+from memory.utils import gaussdb_conn as gaussdb_mem+from memory.utils.gaussdb_conn import GaussDBMemoryConnection, GaussDBMemoryDDLBuilder, SearchResult+++class FakePool:+ masked_uri = "unit@example:19995/db?schema=public"+ resolved_schema = "public"++ def check_schema_access(self):+ return None+++def make_conn() -> GaussDBMemoryConnection:+ return GaussDBMemoryConnection(pool=FakePool())+++def test_physical_table_name_depends_only_on_index_name():+ builder = GaussDBMemoryDDLBuilder(schema="public")++ table = builder.physical_table_name("memory_tenant_a")++ assert table == builder.physical_table_name("memory_tenant_a")+ assert table != builder.physical_table_name("memory_tenant_b")+ assert table.startswith("ragflow_mem_")+ assert "memory_id" not in table+ assert len(table) <= 63+++def test_memory_table_ddl_uses_oracle_compatible_columns_and_ustore():+ builder = GaussDBMemoryDDLBuilder(schema="public")+ ddl = builder.build_memory_table_ddl("ragflow_mem_abc")++ assert 'CREATE TABLE IF NOT EXISTS "public"."ragflow_mem_abc"' in ddl+ assert "message_id NUMBER(19) NOT NULL" in ddl+ assert "message_type_kwd VARCHAR2(64)" in ddl+ assert "valid_at TIMESTAMP" in ddl+ assert "content_ltks TEXT" in ddl+ assert "WITH (storage_type=USTORE)" in ddl+++def test_memory_indexes_use_ugin_and_gsdiskann_with_empty_marker():+ builder = GaussDBMemoryDDLBuilder(schema="public")++ fulltext = builder.build_fulltext_ugin_ddl("ragflow_mem_abc")+ vector_columns = "\n".join(builder.build_vector_column_ddls("ragflow_mem_abc", 3))+ empty_index = builder.build_vector_empty_index_ddl("ragflow_mem_abc", 3)+ diskann = builder.build_diskann_index_ddl("ragflow_mem_abc", 3)++ assert "USING ugin (to_tsvector('simple', tokenized_content_ltks))" in fulltext+ assert "q_3_vec floatvector(3)" in vector_columns+ assert "DEFAULT (array_fill(0, ARRAY[3])::text::floatvector(3)) NOT NULL" in vector_columns+ assert "q_3_vec_empty BOOLEAN DEFAULT TRUE NOT NULL" in vector_columns+ assert "(memory_id, q_3_vec_empty)" in empty_index+ assert "USING gsdiskann (q_3_vec cosine)" in diskann+++def test_merge_sql_uses_dual_and_resets_other_vector_dimensions():+ conn = make_conn()+ row, dim = conn._message_to_row(+ {+ "id": "mem1_1",+ "message_id": 1,+ "message_type": "raw",+ "source_id": 0,+ "memory_id": "mem1",+ "agent_id": "agent1",+ "session_id": "session1",+ "content": "hello",+ "valid_at": "2026-06-29 12:00:00",+ "status": True,+ "content_embed": [0.1, 0.2, 0.3],+ },+ "mem1",+ )++ sql, params = conn._build_merge_sql("ragflow_mem_abc", dim, [3, 5], [row])++ assert sql.startswith('MERGE INTO "public"."ragflow_mem_abc"')+ assert "FROM dual" in sql+ assert "ON CONFLICT" not in sql+ assert "q_3_vec = s.q_3_vec" in sql+ assert "q_3_vec_empty = FALSE" in sql+ assert "q_5_vec = '[0,0,0,0,0]'::floatvector" in sql+ assert "q_5_vec_empty = TRUE" in sql+ assert params[0][0] == "mem1_1"+ assert params[0][-1] == "[0.1,0.2,0.3]"+++def test_where_clause_skips_empty_string_filters_and_adds_memory_boundary():+ conn = make_conn()++ where_sql, params = conn._build_where_clause(+ {"agent_id": "", "session_id": "s1", "status": 1},+ memory_ids=["mem1", "mem2"],+ hide_forgotten=True,+ )++ assert "memory_id IN (%s, %s)" in where_sql+ assert "forget_at IS NULL" in where_sql+ assert "session_id = %s" in where_sql+ assert "status_int = %s" in where_sql+ assert "agent_id" not in where_sql+ assert params == ["mem1", "mem2", "s1", 1]+++def test_delete_empty_message_id_list_is_noop():+ conn = make_conn()++ assert conn.delete({"message_id": []}, "memory_tenant", "mem1") == 0+++def test_get_sql_does_not_add_fetch_first_one(monkeypatch):+ conn = make_conn()+ captured = {}++ monkeypatch.setattr(conn, "_table_exists", lambda _table: True)+ monkeypatch.setattr(conn, "_vector_columns_for_select", lambda _table: [])++ def fake_fetch_one_with_description(sql, params):+ captured["sql"] = sql+ captured["params"] = params+ return None, []++ monkeypatch.setattr(conn, "_fetch_one_with_description", fake_fetch_one_with_description)++ assert conn.get("mem1_1", "memory_tenant", ["mem1"]) is None+ assert "FETCH FIRST 1 ROWS ONLY" not in captured["sql"]+ assert "WHERE id = %s" in captured["sql"]+ assert captured["params"] == ["mem1_1"]+++def test_catalog_existence_checks_do_not_add_fetch_first_one(monkeypatch):+ conn = make_conn()+ captured = []++ def fake_fetch_one(sql, params):+ captured.append((sql, params))+ return (1,)++ monkeypatch.setattr(conn, "_fetch_one", fake_fetch_one)++ assert conn._table_exists("ragflow_mem_abc")+ assert conn._column_exists("ragflow_mem_abc", "q_3_vec")+ assert all("FETCH FIRST 1 ROWS ONLY" not in sql for sql, _params in captured)+++def test_sql_returns_none_without_executing_external_sql():+ conn = make_conn()++ assert conn.sql("SELECT 1") is None+++def test_update_content_set_refreshes_tokenized_content():+ conn = make_conn()++ set_sql, params = conn._build_update_set("ragflow_mem_abc", "memory_tenant", "mem1", {"content": "hello world"})++ assert "content_ltks = %s" in set_sql+ assert "tokenized_content_ltks = %s" in set_sql+ assert params[0] == "hello world"+ assert params[1]+++def test_remove_content_clears_original_and_tokenized_content():+ conn = make_conn()++ set_sql, params = conn._build_update_set("ragflow_mem_abc", "memory_tenant", "mem1", {"remove": "content"})++ assert set_sql == "content_ltks = NULL, tokenized_content_ltks = NULL"+ assert params == []+++def test_vector_search_sql_filters_empty_vectors_and_uses_threshold():+ conn = make_conn()+ parsed = conn._parse_match_expressions(+ [+ MatchDenseExpr(+ "q_3_vec",+ [0.1, 0.2, 0.3],+ "float",+ "cosine",+ topn=5,+ extra_options={"similarity": 0.62},+ )+ ]+ )++ sql, params = conn._build_vector_search_sql(+ table="ragflow_mem_abc",+ select_fields=["message_id", "content"],+ condition={"status": 1},+ parsed=parsed,+ offset=0,+ limit=10,+ memory_ids=["mem1"],+ hide_forgotten=True,+ )++ assert "1 - (q_3_vec <+> %s::floatvector(3)) AS _score" in sql+ assert "q_3_vec_empty = FALSE" in sql+ assert ">= %s" in sql+ assert "ORDER BY q_3_vec <+> %s::floatvector(3) ASC" in sql+ assert "OFFSET %s LIMIT %s" in sql+ assert "FETCH NEXT" not in sql+ assert params == ["[0.1,0.2,0.3]", "mem1", 1, "[0.1,0.2,0.3]", 0.62, "[0.1,0.2,0.3]", 0, 5]+++def test_fulltext_search_sql_uses_tokenized_original_query_and_plain_tsquery():+ conn = make_conn()+ parsed = conn._parse_match_expressions([MatchTextExpr(["content_ltks"], "ignored", 8, extra_options={"original_query": "contract risk"})])++ sql, params = conn._build_fulltext_search_sql(+ table="ragflow_mem_abc",+ select_fields=["message_id", "content"],+ highlight_fields=[],+ condition={"status": 1},+ parsed=parsed,+ offset=0,+ limit=10,+ memory_ids=["mem1"],+ hide_forgotten=True,+ )++ assert "to_tsvector('simple', tokenized_content_ltks)" in sql+ assert "plainto_tsquery('simple', %s)" in sql+ assert "ts_rank" in sql+ assert "OFFSET %s LIMIT %s" in sql+ assert "FETCH NEXT" not in sql+ assert params[0] == "contract risk"+ assert params[-2:] == [0, 8]+++def test_parse_match_expression_tokenizes_fulltext_query_like_stored_content():+ conn = make_conn()++ parsed = conn._parse_match_expressions([MatchTextExpr(["content_ltks"], "ignored", 8, extra_options={"original_query": "coriander"})])++ assert parsed["text_query"] == "coriand"+++def test_fusion_sql_uses_fulltext_candidates_then_vector_filter():+ conn = make_conn()+ parsed = conn._parse_match_expressions(+ [+ MatchTextExpr(["content_ltks"], "ignored", 20, extra_options={"original_query": "risk"}),+ MatchDenseExpr("q_3_vec", [0.1, 0.2, 0.3], "float", "cosine", topn=10, extra_options={"similarity": 0.5}),+ FusionExpr("weighted_sum", 10, {"weights": "0.3,0.7"}),+ ]+ )++ sql, params = conn._build_fusion_search_sql(+ table="ragflow_mem_abc",+ select_fields=["message_id", "content"],+ condition={"status": 1},+ parsed=parsed,+ offset=0,+ limit=10,+ memory_ids=["mem1"],+ hide_forgotten=True,+ )++ assert "WITH fulltext_results AS" in sql+ assert "ORDER BY relevance DESC LIMIT %s" in sql+ assert "q_3_vec_empty = FALSE" in sql+ assert "relevance * %s + (1 - (q_3_vec <+> %s::floatvector(3))) * %s AS _score" in sql+ assert "OFFSET %s LIMIT %s" in sql+ assert "FETCH FIRST" not in sql+ assert "FETCH NEXT" not in sql+ assert params[0] == "risk"+ assert 0.7 in params+ assert 0.30000000000000004 in params+++def test_get_fields_restores_content_embed_by_empty_marker():+ conn = make_conn()+ result = SearchResult(+ total=1,+ messages=[+ {+ "id": "mem1_1",+ "message_id": 1,+ "message_type_kwd": "raw",+ "source_id": 0,+ "memory_id": "mem1",+ "agent_id": "agent1",+ "session_id": "session1",+ "valid_at": "2026-06-29 12:00:00",+ "status_int": 1,+ "content_ltks": "hello",+ "q_3_vec": "[0,0,0]",+ "q_3_vec_empty": True,+ "q_5_vec": "[1,2,3,4,5]",+ "q_5_vec_empty": False,+ }+ ],+ )++ fields = conn.get_fields(result, ["message_id", "message_type", "content", "content_embed"])++ assert fields["mem1_1"]["message_id"] == 1+ assert fields["mem1_1"]["message_type"] == "raw"+ assert fields["mem1_1"]["content"] == "hello"+ assert fields["mem1_1"]["content_embed"] == [1.0, 2.0, 3.0, 4.0, 5.0]+++def test_filter_search_uses_order_by_and_offset_limit_syntax():+ conn = make_conn()+ order_by = OrderByExpr().desc("valid_at")++ sql, params = conn._build_filter_search_sql(+ table="ragflow_mem_abc",+ select_fields=["message_id", "content"],+ condition={},+ order_by=order_by,+ offset=20,+ limit=10,+ memory_ids=["mem1"],+ hide_forgotten=False,+ )++ assert "ORDER BY valid_at DESC" in sql+ assert "OFFSET %s LIMIT %s" in sql+ assert "FETCH NEXT" not in sql+ assert params == ["mem1", 20, 10]+++def test_sort_messages_uses_numeric_keys_for_message_id():+ conn = make_conn()+ order_by = OrderByExpr().desc("message_id")++ messages = [+ {"id": "mem_b_99", "message_id": 99},+ {"id": "mem_a_100", "message_id": 100},+ ]++ sorted_messages = conn._sort_messages(messages, order_by, has_match=False)++ assert [message["message_id"] for message in sorted_messages] == [100, 99]+++def test_health_reuses_gaussdb_base_health(monkeypatch):+ conn = make_conn()+ monkeypatch.setattr(conn, "_query_version", lambda: "GaussDB unit")+ monkeypatch.setattr(conn, "_query_sql_compatibility", lambda: "PG")++ health = conn.health()++ assert "health" not in GaussDBMemoryConnection.__dict__+ assert health["status"] == "unhealthy"+ assert health["sql_compatibility"] == "PG"+ assert "A/ORA" in health["error"]+++def make_base_health_conn(monkeypatch, *, server_encoding, client_encoding):+ from common.doc_store.gaussdb_conn_base import GaussDBConnectionBase++ conn = object.__new__(GaussDBConnectionBase)+ conn.masked_uri = "gaussdb://[email protected]:19995/postgres?schema=ragflow_doc"+ conn.resolved_schema = "ragflow_doc"+ monkeypatch.setattr(conn, "_query_version", lambda: "GaussDB unit")+ monkeypatch.setattr(conn, "_query_sql_compatibility", lambda: "A")+ monkeypatch.setattr(conn, "_query_server_encoding", lambda: server_encoding)+ monkeypatch.setattr(conn, "_query_client_encoding", lambda: client_encoding)+ return conn+++def test_health_allows_sql_ascii_server_with_utf8_client_warning(monkeypatch):+ conn = make_base_health_conn(monkeypatch, server_encoding="SQL_ASCII", client_encoding="UTF8")++ result = conn.health()++ assert result["status"] == "healthy"+ assert result["server_encoding"] == "SQL_ASCII"+ assert result["client_encoding"] == "UTF8"+ assert "does not validate" in result["warning"]+++def test_health_rejects_non_utf8_client(monkeypatch):+ conn = make_base_health_conn(monkeypatch, server_encoding="UTF8", client_encoding="SQL_ASCII")++ result = conn.health()++ assert result["status"] == "unhealthy"+ assert "client_encoding" in result["error"]+++def test_health_accepts_utf8_server_and_client_without_warning(monkeypatch):+ conn = make_base_health_conn(monkeypatch, server_encoding="UTF8", client_encoding="UTF8")++ result = conn.health()++ assert result["status"] == "healthy"+ assert "warning" not in result+++def test_short_circuit_branches_and_tuple_result_helpers(monkeypatch):+ conn = make_conn()+ result = SearchResult(total=2, messages=[{"id": "doc-1", "content_ltks": "contract risk"}, {"id": ""}])++ assert conn.insert([], "memory_tenant", "mem1") == []+ assert conn.update({}, {"content": "x"}, "memory_tenant", "mem1") is False+ assert conn.update({"id": "doc-1"}, {}, "memory_tenant", "mem1") is False+ assert conn.delete({}, "memory_tenant", "mem1") == 0+ assert conn.delete({"id": ["", None]}, "memory_tenant", "mem1") == 0+ assert conn.get("", "memory_tenant", ["mem1"]) is None+ assert conn.get_total((result, 2)) == 2+ assert conn.get_doc_ids((result, 2)) == ["doc-1"]+ assert conn.get_fields((result, 2), []) == {}+ assert "<em>contract</em>" in conn.get_highlight((result, 2), ["contract"], "content_ltks")["doc-1"]+ assert conn.get_aggregation((result, 2), "content_ltks") == [("contract risk", 1)]++ monkeypatch.setattr(conn, "_table_exists", lambda _table: False)+ assert conn.update({"id": "doc-1"}, {"content": "x"}, "missing", "mem1") is True+ assert conn.delete({"id": ["doc-1"]}, "missing", "mem1") == 0+ assert conn.get("doc-1", "missing", ["mem1"]) is None+ assert conn.get_forgotten_messages(["message_id"], "missing", "mem1") is None+ assert conn.get_missing_field_message(["message_id"], "missing", "mem1", "content") is None++ empty_res, empty_total = conn.search(["message_id"], [], {}, [], OrderByExpr(), 0, 10, "", ["mem1"])+ assert empty_total == 0+ assert empty_res.messages == []+ empty_res, empty_total = conn.search(["message_id"], [], {}, [], OrderByExpr(), 0, 10, "idx", [])+ assert empty_total == 0+ empty_res, empty_total = conn.search(["message_id"], [], {}, [], OrderByExpr(), 0, 0, "idx", ["mem1"])+ assert empty_total == 0+ assert empty_res.messages == []+++def test_index_exist_false_when_catalog_is_incomplete(monkeypatch):+ conn = make_conn()+ monkeypatch.setattr(conn, "_table_exists", lambda _table: True)+ monkeypatch.setattr(conn, "_column_names", lambda _table: [])+ assert conn.index_exist("memory_tenant", "mem1") is False++ monkeypatch.setattr(conn, "_column_names", lambda _table: list(gaussdb_mem.BASE_COLUMNS))+ monkeypatch.setattr(conn, "_index_names", lambda _table: [])+ assert conn.index_exist("memory_tenant", "mem1") is False+++def test_message_normalization_vector_selection_and_invalid_inputs(caplog):+ conn = make_conn()++ with pytest.raises(InvalidGaussDBObjectName):+ conn.physical_table("")+ assert conn.convert_field_name("content", use_tokenized_content=True) == "tokenized_content_ltks"+ with pytest.raises(ValueError, match="content_embed"):+ conn._message_to_row({"message_id": 1, "content_embed": []}, "mem1")+ with pytest.raises(ValueError, match="memory_id"):+ conn._message_to_row({"message_id": 1, "content_embed": [0.1, 0.2, 0.3]}, None)++ row = {+ "id": "doc-1",+ "q_2_vec": np.array([1, 2]),+ "q_2_vec_empty": False,+ "q_3_vec": "[3,4,5]",+ "q_3_vec_empty": False,+ }+ assert conn._content_embed_from_row(row) == [3.0, 4.0, 5.0]+ assert "multiple non-empty vector dimensions" in caplog.text+++def test_insert_normalization_failure_returns_all_batch_ids_without_write(monkeypatch):+ conn = make_conn()+ monkeypatch.setattr(conn, "_table_exists", lambda _table: pytest.fail("database access must not occur"))++ errors = conn.insert(+ [+ {+ "id": "mem1_1",+ "message_id": 1,+ "status": True,+ "content": "first",+ "content_embed": [0.1, 0.2, 0.3],+ },+ {+ "id": "mem1_2",+ "message_id": 2,+ "status": True,+ "content": "second",+ "content_embed": [0.1, 0.2],+ },+ ],+ "memory_tenant",+ "mem1",+ )++ assert errors == ["mem1_1", "mem1_2"]+++def test_update_set_covers_remove_vector_empty_embed_numeric_and_time_branches(monkeypatch):+ conn = make_conn()+ monkeypatch.setattr(conn, "_ensure_vector_column_exists", lambda _table, _dim: None)+ monkeypatch.setattr(conn, "_vector_dimensions", lambda _table: [3, 5])++ set_sql, params = conn._build_update_set(+ "ragflow_mem_abc",+ "memory_tenant",+ "mem1",+ {+ "remove": ["q_3_vec", "agent_id"],+ "content_embed": [],+ "message_id": Decimal("7"),+ "agent_id": "",+ "invalid_at": "-",+ "status": True,+ },+ )++ assert "q_3_vec = %s::floatvector(3)" in set_sql+ assert "q_3_vec_empty = TRUE" in set_sql+ assert "agent_id = NULL" in set_sql+ assert "message_id = %s" in set_sql+ assert "invalid_at = %s::timestamp" in set_sql+ assert "status_int = %s" in set_sql+ assert params == ["[0,0,0]", 7, None, None, 1]+++def test_column_where_order_and_parse_helpers_cover_edge_cases(monkeypatch):+ conn = make_conn()+ monkeypatch.setattr(conn, "_vector_dimensions", lambda _table: [3])++ parsed = conn._parse_match_expressions(+ [+ MatchDenseExpr("content_embed", np.array([0.1, 0.2]), "float", "cosine", 8, {}),+ FusionExpr("weighted_sum", 6, {"weights": "0.2"}),+ ]+ )+ assert parsed["vector_dim"] == 2+ assert parsed["vector"] == "[0.1,0.2]"+ assert parsed["vector_weight"] == 0.5+ assert parsed["topn"] == 6++ assert conn._select_columns("ragflow_mem_abc", ["id", "_score", "content", "content_embed", "content"]) == [+ "id",+ "content_ltks",+ "q_3_vec",+ "q_3_vec_empty",+ ]+ with pytest.raises(InvalidGaussDBObjectName):+ conn._select_columns("ragflow_mem_abc", ["bad;column"])+ assert conn._build_where_clause({}, memory_ids=[], force_memory_filter=True) == ("1=0", [])+ assert conn._build_where_clause({"message_id": ["", None]}, memory_ids=["mem1"]) == ("memory_id IN (%s)", ["mem1"])+ assert conn._build_order_by(None) == ""++ assert gaussdb_mem.normalize_index_names(" a, ,b ") == ["a", "b"]+ assert gaussdb_mem.normalize_index_names([" a ", "", None]) == ["a", "None"]+ assert gaussdb_mem.clean_list_values(None) == []+ assert gaussdb_mem.clean_list_values("mem1") == ["mem1"]+ assert gaussdb_mem.effective_limit(0, None) == 10000+ assert gaussdb_mem.parse_vector_dim("q_12_vec") == 12+ assert gaussdb_mem.parse_vector_dim("content_embed") is None+ assert gaussdb_mem.vector_literal("[1,2]", 2) == "[1.0,2.0]"+ assert gaussdb_mem.vector_literal("raw", 3) == "raw"+ assert gaussdb_mem.vector_literal(np.array([1, 2]), 2) == "[1.0,2.0]"+ with pytest.raises(ValueError, match="dimension mismatch"):+ gaussdb_mem.vector_literal([1], 2)+ assert gaussdb_mem.parse_vector_value(None) == []+ assert gaussdb_mem.parse_vector_value(np.array([1, 2])) == [1.0, 2.0]+ assert gaussdb_mem.parse_vector_value((1, 2)) == [1.0, 2.0]+ assert gaussdb_mem.parse_vector_value("") == []+ assert gaussdb_mem.parse_vector_value("[1, 2]") == [1.0, 2.0]+ assert gaussdb_mem.parse_vector_value("[]") == []+ assert gaussdb_mem.parse_vector_value("not-json") == []+ assert gaussdb_mem.parse_vector_value('{"x": 1}') == []+ assert gaussdb_mem.zero_vector_literal(3) == "[0,0,0]"+ assert gaussdb_mem.normalize_timestamp("-") is None+ assert gaussdb_mem.normalize_timestamp(datetime(2026, 6, 29, 12, 0, 0)) == "2026-06-29 12:00:00"+ assert gaussdb_mem.format_timestamp(datetime(2026, 6, 29, 12, 0, 0)) == "2026-06-29 12:00:00"+ assert gaussdb_mem.none_if_empty("") is None+ assert gaussdb_mem.to_int_or_none(False) == 0+ assert gaussdb_mem.to_int_or_none(np.int64(3)) == 3+ assert gaussdb_mem.to_int_or_original(None) is None+ assert gaussdb_mem.to_int_or_original(True) == 1+ assert gaussdb_mem.to_int_or_original(np.int64(4)) == 4+ assert gaussdb_mem.to_int_or_original(Decimal("5")) == 5+ assert gaussdb_mem.to_int_or_original("abc") == "abc"+ assert gaussdb_mem.row_value({"x": 1}, "x", 0) == 1+ assert gaussdb_mem.unique_preserve_order(["a", "b", "a"]) == ["a", "b"]+ assert gaussdb_mem.sortable_value(datetime(2026, 6, 29, 12, 0, 0)) == "2026-06-29T12:00:00"+ assert gaussdb_mem.is_maintenance_work_mem_error(RuntimeError("maintenance_work_mem below required")) is True+ assert gaussdb_mem.is_maintenance_work_mem_error(RuntimeError("other error")) is False+ gaussdb_mem.close_cursor(None)+++def test_diskann_retry_handles_maintenance_work_mem_errors(monkeypatch):+ conn = make_conn()+ calls = []++ def fail_once(_statements):+ calls.append(_statements)+ if len(calls) == 1:+ raise RuntimeError("maintenance_work_mem below required")++ monkeypatch.setattr(conn, "_execute_statements", fail_once)+ conn._create_diskann_index_with_retry("ragflow_mem_abc", 3)+ assert len(calls) == 2+ assert calls[0][1] == "SET LOCAL maintenance_work_mem = '1GB'"+ assert calls[1][1] == "SET LOCAL maintenance_work_mem = '2GB'"++ def fail_non_maintenance(_statements):+ raise RuntimeError("syntax error")++ monkeypatch.setattr(conn, "_execute_statements", fail_non_maintenance)+ with pytest.raises(RuntimeError, match="syntax error"):+ conn._create_diskann_index_with_retry("ragflow_mem_abc", 3)+++def test_diskann_unavailable_propagates_missing_prerequisite(monkeypatch):+ conn = make_conn()++ class UnsupportedVectorIndexError(Exception):+ pgcode = "0A000"++ def unsupported(_statements):+ raise UnsupportedVectorIndexError("The vectordb indexes are supported only by high-order features")++ monkeypatch.setattr(conn, "_execute_statements", unsupported)++ with pytest.raises(UnsupportedVectorIndexError, match="supported only by high-order features"):+ conn._create_diskann_index_with_retry("ragflow_mem_abc", 3)diff --git a/test/unit_test/rag/app/test_gaussdb_table_chunk_column_roles.py b/test/unit_test/rag/app/test_gaussdb_table_chunk_column_roles.pynew file mode 100644index 0000000000..c92a4cda76--- /dev/null+++ b/test/unit_test/rag/app/test_gaussdb_table_chunk_column_roles.py@@ -0,0 +1,253 @@+import sys+import types+from importlib import import_module, reload+from unittest.mock import MagicMock, patch++import pytest++import common.settings as settings+++TEST_CSV = b"""row_id,title,content,country,category+1,Earthquake hits Turkey,A 5.8 magnitude earthquake struck Konya,Turkey,Disaster+2,Oil prices surge,Brent crude jumped 4.2 percent,Global,Economy+3,AI regulation proposed,EU unveiled a draft regulation,EU,Technology+"""+FILENAME = "test.csv"+KB_ID = "test_kb_id"+TABLE_NAME = "ragflow_0123456789abcdef0123456789abcdef"+++def _install_table_import_stubs():+ names = (+ "deepdoc.parser",+ "deepdoc.parser.utils",+ "deepdoc.parser.figure_parser",+ "deepdoc.vision.ocr",+ "rag.nlp",+ "rag.app.table",+ "rag.app.picture",+ "api.db.services.knowledgebase_service",+ "xpinyin",+ )+ original = {name: sys.modules.get(name) for name in names}++ parser_stub = types.ModuleType("deepdoc.parser")+ parser_stub.__path__ = []+ parser_stub.ExcelParser = type("ExcelParser", (), {})++ utils_stub = types.ModuleType("deepdoc.parser.utils")+ utils_stub.get_text = lambda _filename, binary=None: binary.decode("utf-8") if isinstance(binary, bytes) else str(binary or "")++ figure_stub = types.ModuleType("deepdoc.parser.figure_parser")+ figure_stub.vision_figure_parser_figure_xlsx_wrapper = lambda *_args, **_kwargs: []++ nlp_stub = types.ModuleType("rag.nlp")++ def tokenize_doc(doc, text="", *_args, **_kwargs):+ if isinstance(doc, dict):+ doc["content_with_weight"] = str(text or "")+ doc["content_ltks"] = str(text or "")+ doc["content_sm_ltks"] = str(text or "")+ return doc++ nlp_stub.rag_tokenizer = types.SimpleNamespace(+ tokenize=lambda text, *_args, **_kwargs: str(text),+ fine_grained_tokenize=lambda text, *_args, **_kwargs: str(text),+ )+ nlp_stub.tokenize = tokenize_doc+ nlp_stub.tokenize_table = lambda *_args, **_kwargs: []++ kb_service_stub = types.ModuleType("api.db.services.knowledgebase_service")+ kb_service_stub.KnowledgebaseService = type(+ "KnowledgebaseService",+ (),+ {"update_parser_config": staticmethod(lambda *_args, **_kwargs: None)},+ )++ xpinyin_stub = types.ModuleType("xpinyin")++ class Pinyin:+ def get_pinyins(self, text, splitter="_"):+ return [str(text).replace(" ", splitter)]++ xpinyin_stub.Pinyin = Pinyin++ sys.modules["deepdoc.parser"] = parser_stub+ sys.modules["deepdoc.parser.utils"] = utils_stub+ sys.modules["deepdoc.parser.figure_parser"] = figure_stub+ sys.modules["deepdoc.vision.ocr"] = MagicMock()+ sys.modules["rag.nlp"] = nlp_stub+ sys.modules["rag.app.picture"] = MagicMock()+ sys.modules["api.db.services.knowledgebase_service"] = kb_service_stub+ sys.modules["xpinyin"] = xpinyin_stub+ return original+++def _noop_callback(*_args, **_kwargs):+ pass++[email protected](scope="module")+def table_module():+ original = _install_table_import_stubs()+ try:+ module = import_module("rag.app.table")+ yield reload(module)+ finally:+ for name, previous in original.items():+ if previous is None:+ sys.modules.pop(name, None)+ else:+ sys.modules[name] = previous+++def mock_update_kb():+ with patch("rag.app.table.KnowledgebaseService.update_parser_config") as mock:+ yield mock+++def dialog_service(monkeypatch):+ from test.unit_test.api.db.services.test_gaussdb_dialog_sql import _install_settings_import_stubs++ _install_settings_import_stubs(monkeypatch)+ sys.modules.pop("api.db.services.dialog_service", None)+ module = import_module("api.db.services.dialog_service")+ try:+ yield module+ finally:+ sys.modules.pop("api.db.services.dialog_service", None)+ package = sys.modules.get("api.db.services")+ if package is not None and hasattr(package, "dialog_service"):+ delattr(package, "dialog_service")+++def _run_chunk(table_module, parser_config, mock_update_kb):+ return table_module.chunk(+ FILENAME,+ binary=TEST_CSV,+ callback=_noop_callback,+ kb_id=KB_ID,+ parser_config=parser_config,+ lang="Chinese",+ )+++def _assert_field_map_prompt_closure(dialog_service, chunks, field_map, expected_field_map):+ assert field_map == expected_field_map+ prompt = dialog_service.gaussdb_text_to_sql.build_sql_prompt(TABLE_NAME, field_map, "show table rows")+ for field, path in expected_field_map.items():+ assert path in chunks[0]["chunk_data"]+ assert f" - {field} ({path}): chunk_data #>> '{{{path}}}'" in prompt+++def test_tc_sql_006_chunk_gaussdb_updates_field_map_and_column_names(+ table_module,+ mock_update_kb,+ monkeypatch,+ dialog_service,+):+ monkeypatch.setattr(settings, "DOC_ENGINE_GAUSSDB", True, raising=False)++ chunks = _run_chunk(table_module, {}, mock_update_kb)++ mock_update_kb.assert_called_once_with(+ KB_ID,+ {+ "field_map": {+ "row_id": "row_id",+ "title": "title",+ "content": "content",+ "country": "country",+ "category": "category",+ },+ "table_column_names": ["row_id", "title", "content", "country", "category"],+ },+ )+ _assert_field_map_prompt_closure(+ dialog_service,+ chunks,+ mock_update_kb.call_args.args[1]["field_map"],+ {+ "row_id": "row_id",+ "title": "title",+ "content": "content",+ "country": "country",+ "category": "category",+ },+ )+++def test_tc_sql_007_chunk_manual_mode_gaussdb_stores_metadata_in_chunk_data(+ table_module,+ mock_update_kb,+ monkeypatch,+ dialog_service,+):+ monkeypatch.setattr(settings, "DOC_ENGINE_GAUSSDB", True, raising=False)+ parser_config = {+ "table_column_mode": "manual",+ "table_column_roles": {+ "title": "indexing",+ "content": "indexing",+ "row_id": "metadata",+ "country": "metadata",+ "category": "metadata",+ },+ }++ chunks = _run_chunk(table_module, parser_config, mock_update_kb)++ assert chunks[0]["chunk_data"] == {+ "row_id": 1,+ "country": "Turkey",+ "category": "Disaster",+ }+ assert "country_raw" not in chunks[0]+ assert "- country:" not in chunks[0]["content_with_weight"]+ assert "- title: Earthquake hits Turkey" in chunks[0]["content_with_weight"]+ field_map = mock_update_kb.call_args.args[1]["field_map"]+ assert field_map == {+ "row_id": "row_id",+ "country": "country",+ "category": "category",+ }+ _assert_field_map_prompt_closure(+ dialog_service,+ chunks,+ field_map,+ {+ "row_id": "row_id",+ "country": "country",+ "category": "category",+ },+ )+++def test_tc_sql_008_chunk_auto_mode_preserves_row_id_text_and_prompt_path(+ table_module,+ mock_update_kb,+ monkeypatch,+ dialog_service,+):+ monkeypatch.setattr(settings, "DOC_ENGINE_GAUSSDB", True, raising=False)++ chunks = _run_chunk(table_module, {}, mock_update_kb)+ field_map = mock_update_kb.call_args.args[1]["field_map"]++ assert "- row_id: 1" in chunks[0]["content_with_weight"].splitlines()+ assert chunks[0]["chunk_data"]["row_id"] == 1+ _assert_field_map_prompt_closure(+ dialog_service,+ chunks,+ field_map,+ {+ "row_id": "row_id",+ "title": "title",+ "content": "content",+ "country": "country",+ "category": "category",+ },+ )diff --git a/test/unit_test/rag/nlp/test_gaussdb_retrieval.py b/test/unit_test/rag/nlp/test_gaussdb_retrieval.pynew file mode 100644index 0000000000..4032eccc5e--- /dev/null+++ b/test/unit_test/rag/nlp/test_gaussdb_retrieval.py@@ -0,0 +1,528 @@+#+# Copyright 2025 The InfiniFlow Authors. All Rights Reserved.+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+#+import sys+import types+from unittest.mock import AsyncMock, Mock++import pytest+from common.doc_store.doc_store_base import FusionExpr, MatchDenseExpr, MatchTextExpr+++class StubDocEngineConnection:+ def db_type(self):+ return "stub"+++class StubStorage:+ def health(self):+ return True+++def _install_module(name, **attrs):+ module = types.ModuleType(name)+ for key, value in attrs.items():+ setattr(module, key, value)+ sys.modules[name] = module+ return module+++def dealer_cls():+ module_names = [+ "common.settings",+ "common.token_utils",+ "roman_numbers",+ "word2number",+ "cn2an",+ "rag.nlp",+ "rag.nlp.search",+ "rag.nlp.query",+ "rag.nlp.rag_tokenizer",+ "rag.utils.redis_conn",+ "memory.utils",+ "json_repair",+ ]+ module_names.extend(+ f"rag.utils.{name}"+ for name in (+ "es_conn",+ "infinity_conn",+ "ob_conn",+ "opensearch_conn",+ "azure_sas_conn",+ "azure_spn_conn",+ "gcs_conn",+ "minio_conn",+ "opendal_conn",+ "s3_conn",+ "oss_conn",+ )+ )+ module_names.extend(f"memory.utils.{name}" for name in ("es_conn", "infinity_conn", "ob_conn"))+ saved_modules = {name: sys.modules.get(name) for name in module_names}+ import common+ import memory++ common_module = common+ saved_common_settings = getattr(common_module, "settings", None) if common_module else None+ saved_common_token_utils = getattr(common_module, "token_utils", None) if common_module else None+ memory_module = memory+ saved_memory_utils = getattr(memory_module, "utils", None) if memory_module else None++ settings_stub = _install_module(+ "common.settings",+ DOC_ENGINE_GAUSSDB=True,+ DOC_ENGINE_INFINITY=False,+ DOC_ENGINE_OCEANBASE=False,+ DOC_ENGINE_SERENEDB=False,+ )+ if common_module is not None:+ setattr(common_module, "settings", settings_stub)+ token_utils_stub = _install_module("common.token_utils", num_tokens_from_string=lambda value: len(str(value).split()))+ if common_module is not None:+ setattr(common_module, "token_utils", token_utils_stub)++ _install_module("roman_numbers")+ _install_module("word2number", w2n=types.SimpleNamespace(word_to_num=lambda value: 0))+ _install_module("cn2an", cn2an=lambda value, *_args, **_kwargs: 0)+ _install_module(+ "rag.nlp.rag_tokenizer",+ tokenize=lambda value: str(value),+ fine_grained_tokenize=lambda value: str(value),+ )+ _install_module("rag.nlp.query", FulltextQueryer=lambda: types.SimpleNamespace(rmWWW=lambda value: value, hybrid_similarity=lambda *_args, **_kwargs: (0, 0, 0)))+ _install_module("rag.utils.redis_conn", REDIS_CONN=types.SimpleNamespace(health=lambda: True, is_alive=lambda: False, REDIS=None))+ _install_module("json_repair", loads=lambda value: value)+ for short_name, attrs in {+ "es_conn": {"ESConnection": StubDocEngineConnection},+ "infinity_conn": {"InfinityConnection": StubDocEngineConnection},+ "ob_conn": {"OBConnection": StubDocEngineConnection},+ "opensearch_conn": {"OSConnection": StubDocEngineConnection},+ "azure_sas_conn": {"RAGFlowAzureSasBlob": StubStorage},+ "azure_spn_conn": {"RAGFlowAzureSpnBlob": StubStorage},+ "gcs_conn": {"RAGFlowGCS": StubStorage},+ "minio_conn": {"RAGFlowMinio": StubStorage},+ "opendal_conn": {"OpenDALStorage": StubStorage},+ "s3_conn": {"RAGFlowS3": StubStorage},+ "oss_conn": {"RAGFlowOSS": StubStorage},+ }.items():+ _install_module(f"rag.utils.{short_name}", **attrs)++ memory_utils = types.ModuleType("memory.utils")+ memory_utils.__path__ = []+ sys.modules["memory.utils"] = memory_utils+ if memory_module is not None:+ setattr(memory_module, "utils", memory_utils)+ for short_name in ("es_conn", "infinity_conn", "ob_conn"):+ module = _install_module(+ f"memory.utils.{short_name}",+ ESConnection=StubDocEngineConnection,+ InfinityConnection=StubDocEngineConnection,+ OBConnection=StubDocEngineConnection,+ )+ setattr(memory_utils, short_name, module)++ try:+ import importlib++ sys.modules.pop("rag.nlp.search", None)+ search_module = importlib.import_module("rag.nlp.search")+ search_module.settings.DOC_ENGINE_GAUSSDB = True+ search_module.settings.DOC_ENGINE_INFINITY = False+ search_module.settings.DOC_ENGINE_OCEANBASE = False+ search_module.settings.DOC_ENGINE_SERENEDB = False+ yield search_module.Dealer+ finally:+ for name, module in saved_modules.items():+ if module is None:+ sys.modules.pop(name, None)+ else:+ sys.modules[name] = module+ if common_module is not None:+ if saved_common_settings is None:+ if hasattr(common_module, "settings"):+ delattr(common_module, "settings")+ else:+ setattr(common_module, "settings", saved_common_settings)+ if saved_common_token_utils is None:+ if hasattr(common_module, "token_utils"):+ delattr(common_module, "token_utils")+ else:+ setattr(common_module, "token_utils", saved_common_token_utils)+ if memory_module is not None:+ if saved_memory_utils is None:+ if hasattr(memory_module, "utils"):+ delattr(memory_module, "utils")+ else:+ setattr(memory_module, "utils", saved_memory_utils)+++def make_dealer(dealer_cls):+ dealer = dealer_cls.__new__(dealer_cls)+ dealer.dataStore = FakeGaussDBStore()+ return dealer+++class FakeGaussDBStore:+ def db_type(self):+ return "gaussdb"++ def search(self, *args, **kwargs):+ self.last_search = (args, kwargs)+ return type("SearchResult", (), {"total": getattr(self, "total", 1), "chunks": []})()++ def get_total(self, res):+ return res.total++ def get_doc_ids(self, _res):+ return getattr(self, "ids", [])++ def get_highlight(self, *_args):+ return {}++ def get_aggregation(self, *_args):+ return {}++ def get_fields(self, *_args):+ return getattr(self, "fields", {})++ async def get_scores(self, *_args):+ raise AssertionError("GaussDB retrieval must not call ES _knn_scores")+++class FakeQueryer:+ def question(self, text, min_match=0.3):+ return MatchTextExpr(["content_with_weight"], text, 1024), [text]+++def retrieval_chunk(score, doc_id, doc_name, content="risk contract"):+ return {+ "_score": score,+ "content_ltks": content,+ "content_with_weight": content,+ "doc_id": doc_id,+ "docnm_kwd": doc_name,+ "kb_id": "kb1",+ "position_int": [1],+ }+++ ("vector_weight", "expected_weights"),+ [(0.75, "0.25,0.75"), (0.5, "0.5,0.5")],+)+async def test_tc_ret_801_dealer_search_uses_requested_gaussdb_fusion_weight(+ dealer_cls,+ vector_weight,+ expected_weights,+):+ dealer = make_dealer(dealer_cls)+ dealer.qryr = FakeQueryer()++ async def fake_get_vector(_text, _emb_mdl, topk=10, similarity=0.1):+ return MatchDenseExpr("q_4_vec", [0.1, 0.2, 0.3, 0.4], "float", "cosine", topk, {"similarity": similarity})++ dealer.get_vector = fake_get_vector++ await dealer.search(+ {+ "question": "risk",+ "page": 1,+ "size": 10,+ "topk": 10,+ "similarity": 0.2,+ "vector_similarity_weight": vector_weight,+ },+ "ragflow_tenant",+ ["kb1"],+ emb_mdl=object(),+ rank_feature={"pagerank_fea": 7},+ )++ args, kwargs = dealer.dataStore.last_search+ match_exprs = args[3]+ dense_expr = next(expr for expr in match_exprs if isinstance(expr, MatchDenseExpr))+ fusion_expr = next(expr for expr in match_exprs if isinstance(expr, FusionExpr))++ assert kwargs == {"rank_feature": {"pagerank_fea": 7}}+ assert dense_expr.topn == 10+ assert dense_expr.extra_options["similarity"] == 0.2+ assert fusion_expr.method == "weighted_sum"+ assert fusion_expr.topn == 10+ assert fusion_expr.fusion_params == {"weights": expected_weights}+++async def test_tc_ret_802_dealer_search_uses_default_gaussdb_fusion_weight(dealer_cls):+ dealer = make_dealer(dealer_cls)+ dealer.qryr = FakeQueryer()++ async def fake_get_vector(_text, _emb_mdl, topk=10, similarity=0.1):+ return MatchDenseExpr("q_4_vec", [0.1, 0.2, 0.3, 0.4], "float", "cosine", topk, {"similarity": similarity})++ dealer.get_vector = fake_get_vector++ await dealer.search({"question": "risk", "page": 1, "size": 5}, "ragflow_tenant", ["kb1"], emb_mdl=object())++ args, _kwargs = dealer.dataStore.last_search+ fusion_expr = next(expr for expr in args[3] if isinstance(expr, FusionExpr))++ assert fusion_expr.fusion_params["weights"] == "0.7,0.3"+++async def test_tc_ret_306_retrieval_passes_threshold_with_original_similarity_key(dealer_cls):+ dealer = make_dealer(dealer_cls)+ captured = {}++ async def fake_search(req, *_args, **_kwargs):+ captured["req"] = req+ return type("SearchResult", (), {"total": 0, "chunks": []})()++ async def fake_prune(sres):+ return sres++ dealer.search = fake_search+ dealer._prune_deleted_chunks = fake_prune++ await dealer.retrieval(+ "risk",+ object(),+ "tenant",+ ["kb1"],+ page=1,+ page_size=10,+ similarity_threshold=0.2,+ vector_similarity_weight=0.75,+ top=77,+ doc_ids=["doc1"],+ )++ assert captured["req"]["doc_ids"] == ["doc1"]+ assert captured["req"]["topk"] == 77+ assert captured["req"]["similarity"] == 0.2+ assert captured["req"]["available_int"] == 1+ assert captured["req"]["vector_similarity_weight"] == 0.75+ assert "similarity_threshold" not in captured["req"]+++async def _run_tc_ret_807_808_retrieval_scenario(dealer_cls):+ dealer = make_dealer(dealer_cls)+ dealer.qryr = FakeQueryer()+ dealer.dataStore.total = 3+ dealer.dataStore.ids = ["low", "high", "mid"]+ dealer.dataStore.fields = {+ "low": retrieval_chunk(0.1, "doc-low", "Low", "low"),+ "high": retrieval_chunk(0.9, "doc-high", "High", "high"),+ "mid": retrieval_chunk(0.4, "doc-mid", "Mid", "mid"),+ }++ async def fake_get_vector(_text, _emb_mdl, topk=10, similarity=0.1):+ return MatchDenseExpr("q_4_vec", [0.1, 0.2, 0.3, 0.4], "float", "cosine", topk, {"similarity": similarity})++ async def fake_prune(sres):+ return sres++ dealer.get_vector = fake_get_vector+ dealer._prune_deleted_chunks = fake_prune+ dealer._knn_scores = AsyncMock(side_effect=AssertionError("_knn_scores must not be called"))+ dealer.rerank_by_model = Mock(side_effect=AssertionError("rerank_by_model must not be called"))+ dealer.rerank = Mock(side_effect=AssertionError("rerank must not be called"))+ dealer.rerank_with_knn = Mock(side_effect=AssertionError("rerank_with_knn must not be called"))++ ranks = await dealer.retrieval(+ "risk",+ object(),+ "tenant",+ ["kb1"],+ page=1,+ page_size=10,+ similarity_threshold=0.2,+ vector_similarity_weight=0.75,+ rerank_mdl=None,+ )++ args, kwargs = dealer.dataStore.last_search+ dense_expr = next(expr for expr in args[3] if isinstance(expr, MatchDenseExpr))+ fusion_expr = next(expr for expr in args[3] if isinstance(expr, FusionExpr))++ return dealer, ranks, dense_expr, fusion_expr, kwargs+++async def test_tc_ret_807_retrieval_uses_gaussdb_fusion_weight_without_es_default(monkeypatch, dealer_cls):+ dealer, _, dense_expr, fusion_expr, kwargs = await _run_tc_ret_807_808_retrieval_scenario(dealer_cls)++ assert kwargs == {"rank_feature": {"pagerank_fea": 10}}+ assert dense_expr.extra_options["similarity"] == 0.2+ assert fusion_expr.fusion_params == {"weights": "0.25,0.75"}+ assert fusion_expr.fusion_params["weights"] != "0.05,0.95"++ search_module = sys.modules[dealer_cls.__module__]+ monkeypatch.setattr(search_module.settings, "DOC_ENGINE_GAUSSDB", False)+ await dealer.search(+ {+ "question": "risk",+ "page": 1,+ "size": 10,+ "topk": 10,+ "similarity": 0.2,+ "vector_similarity_weight": 0.75,+ },+ "ragflow_tenant",+ ["kb1"],+ emb_mdl=object(),+ )+ non_gauss_match_exprs = dealer.dataStore.last_search[0][3]+ non_gauss_fusion = next(expr for expr in non_gauss_match_exprs if isinstance(expr, FusionExpr))+ assert non_gauss_fusion.fusion_params == {"weights": "0.001,1"}+++async def test_tc_ret_808_retrieval_uses_gaussdb_scores_without_local_rerank(dealer_cls):+ dealer, ranks, _, _, _ = await _run_tc_ret_807_808_retrieval_scenario(dealer_cls)++ assert ranks["total"] == 2+ assert [chunk["chunk_id"] for chunk in ranks["chunks"]] == ["high", "mid"]+ assert "low" not in {chunk["chunk_id"] for chunk in ranks["chunks"]}+ assert [chunk["similarity"] for chunk in ranks["chunks"]] == [0.9, 0.4]+ assert [chunk["vector_similarity"] for chunk in ranks["chunks"]] == [0.9, 0.4]+ assert [chunk["term_similarity"] for chunk in ranks["chunks"]] == [0.9, 0.4]+ assert ranks["chunks"][0]["vector"] == [0.0, 0.0, 0.0, 0.0]+ assert ranks["doc_aggs"] == [+ {"doc_name": "High", "doc_id": "doc-high", "count": 1},+ {"doc_name": "Mid", "doc_id": "doc-mid", "count": 1},+ ]+ dealer._knn_scores.assert_not_awaited()+ dealer.rerank_by_model.assert_not_called()+ dealer.rerank.assert_not_called()+ dealer.rerank_with_knn.assert_not_called()+++async def _run_gaussdb_rank_feature_retrieval(dealer_cls, fields, rank_feature):+ dealer = make_dealer(dealer_cls)+ dealer.qryr = FakeQueryer()+ dealer.dataStore.total = len(fields)+ dealer.dataStore.ids = list(fields)+ dealer.dataStore.fields = fields++ async def fake_get_vector(_text, _emb_mdl, topk=10, similarity=0.1):+ return MatchDenseExpr("q_4_vec", [0.1, 0.2, 0.3, 0.4], "float", "cosine", topk, {"similarity": similarity})++ async def fake_prune(sres):+ return sres++ dealer.get_vector = fake_get_vector+ dealer._prune_deleted_chunks = fake_prune+ dealer._knn_scores = AsyncMock(side_effect=AssertionError("_knn_scores must not be called"))+ dealer.rerank_by_model = Mock(side_effect=AssertionError("rerank_by_model must not be called"))+ dealer.rerank = Mock(side_effect=AssertionError("rerank must not be called"))+ dealer.rerank_with_knn = Mock(side_effect=AssertionError("rerank_with_knn must not be called"))++ return await dealer.retrieval(+ "risk",+ object(),+ "tenant",+ ["kb1"],+ page=1,+ page_size=10,+ similarity_threshold=0.2,+ vector_similarity_weight=0.75,+ rerank_mdl=None,+ rank_feature=rank_feature,+ )+++async def test_tc_ret_704_gaussdb_retrieval_adds_tag_feature_score_to_sql_score(dealer_cls):+ fields = {+ "without-tag": retrieval_chunk(0.4, "doc-plain", "Plain"),+ "with-tag": {**retrieval_chunk(0.4, "doc-tagged", "Tagged"), "tag_feas": {"risk": 1.0}},+ }++ ranks = await _run_gaussdb_rank_feature_retrieval(dealer_cls, fields, {"risk": 1.0})++ assert [chunk["chunk_id"] for chunk in ranks["chunks"]] == ["with-tag", "without-tag"]+ assert [chunk["similarity"] for chunk in ranks["chunks"]] == [10.4, 0.4]+ assert [chunk["vector_similarity"] for chunk in ranks["chunks"]] == [0.4, 0.4]+ assert [chunk["term_similarity"] for chunk in ranks["chunks"]] == [0.4, 0.4]+++async def test_tc_ret_704_gaussdb_retrieval_does_not_add_pagerank_twice(dealer_cls):+ fields = {+ "higher-sql-score": {**retrieval_chunk(0.8, "doc-high", "High"), "pagerank_fea": 0.1},+ "higher-pagerank": {**retrieval_chunk(0.7, "doc-pagerank", "PageRank"), "pagerank_fea": 0.9},+ }++ ranks = await _run_gaussdb_rank_feature_retrieval(dealer_cls, fields, {"pagerank_fea": 10})++ assert [chunk["chunk_id"] for chunk in ranks["chunks"]] == ["higher-sql-score", "higher-pagerank"]+ assert [chunk["similarity"] for chunk in ranks["chunks"]] == [0.8, 0.7]+ assert [chunk["vector_similarity"] for chunk in ranks["chunks"]] == [0.8, 0.7]+ assert [chunk["term_similarity"] for chunk in ranks["chunks"]] == [0.8, 0.7]+++async def test_tc_ret_311_retrieval_keeps_term_only_gaussdb_scores_when_vector_weight_is_zero(dealer_cls):+ dealer = make_dealer(dealer_cls)+ captured = {}++ async def fake_search(req, *_args, **_kwargs):+ captured["req"] = req+ return dealer_cls.SearchResult(+ total=2,+ ids=["term-low", "term-zero"],+ query_vector=[0.1, 0.2],+ field={+ "term-low": retrieval_chunk(0.1, "doc-low", "Low"),+ "term-zero": retrieval_chunk(0.0, "doc-zero", "Zero"),+ },+ )++ async def fake_prune(sres):+ return sres++ dealer.search = fake_search+ dealer._prune_deleted_chunks = fake_prune+ dealer._knn_scores = AsyncMock(side_effect=AssertionError("_knn_scores must not be called"))+ dealer.rerank_by_model = Mock(side_effect=AssertionError("rerank_by_model must not be called"))+ dealer.rerank = Mock(side_effect=AssertionError("rerank must not be called"))+ dealer.rerank_with_knn = Mock(side_effect=AssertionError("rerank_with_knn must not be called"))++ ranks = await dealer.retrieval(+ "risk",+ object(),+ "tenant",+ ["kb1"],+ page=1,+ page_size=10,+ similarity_threshold=0.8,+ vector_similarity_weight=0.0,+ )++ assert captured["req"]["similarity"] == 0.8+ assert captured["req"]["vector_similarity_weight"] == 0.0+ assert ranks["total"] == 2+ assert [chunk["chunk_id"] for chunk in ranks["chunks"]] == ["term-low", "term-zero"]+ assert [chunk["similarity"] for chunk in ranks["chunks"]] == [0.1, 0.0]+ assert [chunk["vector_similarity"] for chunk in ranks["chunks"]] == [0.1, 0.0]+ assert [chunk["term_similarity"] for chunk in ranks["chunks"]] == [0.1, 0.0]+ dealer._knn_scores.assert_not_awaited()+ dealer.rerank_by_model.assert_not_called()+ dealer.rerank.assert_not_called()+ dealer.rerank_with_knn.assert_not_called()diff --git a/test/unit_test/rag/svr/test_gaussdb_table_column_roles_helpers.py b/test/unit_test/rag/svr/test_gaussdb_table_column_roles_helpers.pynew file mode 100644index 0000000000..c5867aa397--- /dev/null+++ b/test/unit_test/rag/svr/test_gaussdb_table_column_roles_helpers.py@@ -0,0 +1,110 @@+import common.settings as settings++from rag.utils.table_es_metadata import (+ aggregate_table_doc_metadata,+ merge_table_parser_config_from_kb,+ table_parser_strip_doc_metadata_keys,+)+++def test_tc_sql_009_parser_config_merge_and_metadata_cleanup_keys():+ task = {+ "parser_id": "table",+ "parser_config": {"llm_id": "x"},+ "kb_parser_config": {+ "table_column_mode": "manual",+ "table_column_roles": {"a": "metadata"},+ "table_column_names": ["a", "b"],+ },+ }++ assert merge_table_parser_config_from_kb(task) == {+ "llm_id": "x",+ "table_column_mode": "manual",+ "table_column_roles": {"a": "metadata"},+ "table_column_names": ["a", "b"],+ }+ assert merge_table_parser_config_from_kb(+ {+ "parser_id": "table",+ "parser_config": {"foo": 1},+ "kb_parser_config": {"llm_id": "ignored"},+ }+ ) == {"foo": 1}+ assert table_parser_strip_doc_metadata_keys({"table_column_names": ["Region", " SKU "]}) == frozenset({"Region", "SKU"})+ assert table_parser_strip_doc_metadata_keys({"table_column_roles": {"x": "metadata", "y": "indexing"}}) == frozenset({"x", "y"})+ assert table_parser_strip_doc_metadata_keys({"table_column_names": [], "table_column_roles": {"only": "both"}}) == frozenset({"only"})+++def test_tc_sql_010_es_aggregates_public_chunk_metadata(monkeypatch):+ monkeypatch.setattr(settings, "DOC_ENGINE_INFINITY", False)+ monkeypatch.setattr(settings, "DOC_ENGINE_OCEANBASE", False)+ monkeypatch.setattr(settings, "DOC_ENGINE_GAUSSDB", False, raising=False)+ chunks = [+ {+ "guojia_tks": ["tokenized", "country"],+ "guojia_raw": "Brazil",+ "category_tks": ["Risk", "Audit"],+ "score_long": 3,+ },+ {+ "guojia_tks": ["other"],+ "guojia_raw": "Brazil",+ "category_tks": ["Operations"],+ "score_long": 4,+ },+ ]+ task = {+ "parser_id": "table",+ "parser_config": {+ "table_column_mode": "manual",+ "table_column_names": ["country", "category", "score", "title"],+ "table_column_roles": {+ "country": "metadata",+ "category": "both",+ "score": "metadata",+ "title": "indexing",+ },+ },+ "kb_parser_config": {+ "field_map": {+ "guojia_tks": "country",+ "category_tks": "category",+ "score_long": "score",+ }+ },+ }++ assert aggregate_table_doc_metadata(chunks, task) == {+ "country": ["Brazil"],+ "category": ["Risk Audit", "Operations"],+ "score": ["3", "4"],+ }+++def test_tc_sql_011_gaussdb_aggregates_chunk_data_metadata(monkeypatch):+ monkeypatch.setattr(settings, "DOC_ENGINE_INFINITY", False)+ monkeypatch.setattr(settings, "DOC_ENGINE_OCEANBASE", False)+ monkeypatch.setattr(settings, "DOC_ENGINE_GAUSSDB", True, raising=False)+ chunks = [+ {"chunk_data": {"country": "Turkey", "category": "Disaster", "title": "Earthquake"}},+ {"chunk_data": {"country": "Turkey", "category": "Economy"}},+ {"chunk_data": {"country": "EU", "category": None}},+ {"country_raw": "must not be read on GaussDB"},+ ]+ task = {+ "parser_id": "table",+ "parser_config": {+ "table_column_names": ["country", "category", "title"],+ "table_column_roles": {+ "country": "metadata",+ "category": "both",+ "title": "indexing",+ },+ },+ }++ assert aggregate_table_doc_metadata(chunks, task) == {+ "country": ["Turkey", "EU"],+ "category": ["Disaster", "Economy"],+ }diff --git a/test/unit_test/rag/utils/test_gaussdb_conn.py b/test/unit_test/rag/utils/test_gaussdb_conn.pynew file mode 100644index 0000000000..03f885b2c1--- /dev/null+++ b/test/unit_test/rag/utils/test_gaussdb_conn.py@@ -0,0 +1,2915 @@+#+# Copyright 2025 The InfiniFlow Authors. All Rights Reserved.+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+#+import json+import math+import sys+import threading+import types+from unittest.mock import Mock++import pytest+import psycopg2+from psycopg2 import errorcodes+from sqlglot import exp, parse_one++from common.doc_store.doc_store_base import FusionExpr, MatchDenseExpr, MatchTextExpr, OrderByExpr+from common.doc_store.gaussdb_conn_base import (+ GaussDBConnectionBase,+ GaussDBDDLBuilder,+ InvalidGaussDBObjectName,+ UnsafeGaussDBSQL,+)+import rag.utils.gaussdb_conn as gaussdb_conn_module+from rag.utils.gaussdb_conn import (+ GaussDBConnection,+ GaussDBError,+ SearchResult,+ decode_column_value,+ ordered_columns,+ parse_json_dict,+ parse_fusion_vector_weight,+ vector_literal,+ zero_vector_literal,+)+++class RecordingCursor:+ def __init__(self):+ self.executed = []+ self.rowcount = 0+ self.description = None+ self.rows = []+ self.closed = False++ def execute(self, sql, params=None):+ self.executed.append((sql, params or []))++ def executemany(self, sql, params):+ materialized = [list(row) for row in params]+ self.executed.append((sql, materialized))+ self.rowcount = len(materialized)++ def fetchone(self):+ return self.rows[0] if self.rows else None++ def fetchall(self):+ return self.rows++ def close(self):+ self.closed = True+++class UndefinedTableError(Exception):+ pgcode = errorcodes.UNDEFINED_TABLE+++class MissingTableOnceCursor(RecordingCursor):+ def __init__(self):+ super().__init__()+ self.insert_attempts = 0++ def executemany(self, sql, params):+ materialized = [list(row) for row in params]+ self.executed.append((sql, materialized))+ self.insert_attempts += 1+ if self.insert_attempts == 1:+ raise UndefinedTableError("chunk table does not exist")+ self.rowcount = len(materialized)+++class SequencedCursor(RecordingCursor):+ def __init__(self, results):+ super().__init__()+ self.results = list(results)++ def execute(self, sql, params=None):+ super().execute(sql, params)+ rows, description = self.results.pop(0)+ self.rows = rows+ self.description = description+++class RecordingConnection:+ def __init__(self, cursor):+ self.cursor_obj = cursor+ self.commits = 0+ self.rollbacks = 0++ def cursor(self):+ return self.cursor_obj++ def commit(self):+ self.commits += 1++ def rollback(self):+ self.rollbacks += 1+++class RecordingPool:+ def __init__(self, cursor):+ self.cursor = cursor+ self.conn = RecordingConnection(cursor)+ self.get_calls = 0+ self.put_back = []++ def get_conn(self):+ self.get_calls += 1+ return self.conn++ def put_conn(self, conn):+ self.put_back.append(conn)+++class HealthPool:+ masked_uri = "u@h:5432/d?schema=public"+ resolved_schema = "public"++ def __init__(self, rows):+ self.rows = list(rows)+ self.checked = False+ self.sql = []++ def check_schema_access(self):+ self.checked = True++ def fetch_one(self, sql, _params=None):+ self.sql.append(sql)+ row = self.rows.pop(0)+ if isinstance(row, Exception):+ raise row+ return row+++def make_conn(cursor, vector_dimensions=None):+ conn = GaussDBConnection.__new__(GaussDBConnection)+ conn.schema = "public"+ conn.resolved_schema = "public"+ conn.pool = RecordingPool(cursor)+ conn.ddl = GaussDBDDLBuilder(schema="public")+ conn.logger = type("Logger", (), {"debug": lambda *_args, **_kwargs: None, "error": lambda *_args, **_kwargs: None})()+ if vector_dimensions is not None:+ conn.get_vector_dimensions = lambda _table: vector_dimensions+ return conn+++CANONICAL_CREATE_IDX_1024_EXECUTIONS = [+ ("SELECT pg_advisory_xact_lock(hashtext(%s))", ["create_idx:public:ragflow_wrt_012"]),+ (+ """CREATE TABLE IF NOT EXISTS "public"."ragflow_wrt_012" (+ id VARCHAR(256) NOT NULL,+ kb_id VARCHAR(256) NOT NULL,+ doc_id VARCHAR(256),+ docnm_kwd VARCHAR(256),+ doc_type_kwd VARCHAR(256),+ title_tks VARCHAR(256),+ title_sm_tks VARCHAR(256),+ content_with_weight TEXT,+ content_ltks TEXT,+ content_sm_ltks TEXT,+ important_kwd JSONB,+ important_tks TEXT,+ question_kwd JSONB,+ question_tks TEXT,+ tag_kwd JSONB,+ tag_feas JSONB,+ available_int INTEGER DEFAULT 1 NOT NULL,+ pagerank_fea INTEGER,+ create_time VARCHAR(19),+ create_timestamp_flt DOUBLE PRECISION,+ img_id VARCHAR(128),+ position_int JSONB,+ page_num_int JSONB,+ top_int JSONB,+ metadata JSONB,+ chunk_data JSONB,+ extra JSONB,+ _order_id INTEGER,+ group_id VARCHAR(256),+ mom_id VARCHAR(256),+ knowledge_graph_kwd VARCHAR(256),+ source_id JSONB,+ entity_kwd VARCHAR(256),+ entity_type_kwd VARCHAR(256),+ from_entity_kwd VARCHAR(256),+ to_entity_kwd VARCHAR(256),+ weight_int INTEGER,+ weight_flt DOUBLE PRECISION,+ entities_kwd JSONB,+ rank_flt DOUBLE PRECISION,+ n_hop_with_weight TEXT,+ removed_kwd VARCHAR(256) DEFAULT 'N',+ raptor_kwd VARCHAR(256),+ raptor_layer_int INTEGER,+ PRIMARY KEY (kb_id, id)+) WITH (storage_type=USTORE)""",+ [],+ ),+ (+ 'CREATE INDEX IF NOT EXISTS "idx_gdb_ragflow_wrt_012_doc_id" ON "public"."ragflow_wrt_012" (doc_id)',+ [],+ ),+ (+ 'CREATE INDEX IF NOT EXISTS "idx_gdb_ragflow_wrt_012_available_int" ON "public"."ragflow_wrt_012" (available_int)',+ [],+ ),+ (+ 'CREATE INDEX IF NOT EXISTS "idx_gdb_ragflow_wrt_012_knowledge_graph_kwd" ON "public"."ragflow_wrt_012" (knowledge_graph_kwd)',+ [],+ ),+ (+ 'CREATE INDEX IF NOT EXISTS "idx_gdb_ragflow_wrt_012_entity_type_kwd" ON "public"."ragflow_wrt_012" (entity_type_kwd)',+ [],+ ),+ (+ 'CREATE INDEX IF NOT EXISTS "idx_gdb_ragflow_wrt_012_removed_kwd" ON "public"."ragflow_wrt_012" (removed_kwd)',+ [],+ ),+ (+ """CREATE INDEX IF NOT EXISTS "idx_gdb_ragflow_wrt_012_fts_all"+ ON "public"."ragflow_wrt_012"+ USING ugin(to_tsvector('simple', """+ "coalesce(title_tks, ' ') || ' ' || coalesce(title_sm_tks, ' ') || ' ' || "+ "coalesce(important_tks, ' ') || ' ' || coalesce(question_tks, ' ') || ' ' || "+ "coalesce(content_ltks, ' ') || ' ' || coalesce(content_sm_ltks, ' ')))",+ [],+ ),+ (+ """CREATE INDEX IF NOT EXISTS "idx_gdb_ragflow_wrt_012_fts_all_ngram"+ ON "public"."ragflow_wrt_012"+ USING ugin(to_tsvector('ngram', """+ "coalesce(title_tks, ' ') || ' ' || coalesce(title_sm_tks, ' ') || ' ' || "+ "coalesce(important_tks, ' ') || ' ' || coalesce(question_tks, ' ') || ' ' || "+ "coalesce(content_ltks, ' ') || ' ' || coalesce(content_sm_ltks, ' ')))",+ [],+ ),+ (+ 'ALTER TABLE "public"."ragflow_wrt_012" ADD COLUMN IF NOT EXISTS q_1024_vec floatvector(1024) DEFAULT (array_fill(0, ARRAY[1024])::text::floatvector(1024))',+ [],+ ),+ (+ 'ALTER TABLE "public"."ragflow_wrt_012" ADD COLUMN IF NOT EXISTS q_1024_vec_valid BOOLEAN DEFAULT FALSE NOT NULL',+ [],+ ),+ (+ "SELECT pg_advisory_xact_lock(hashtext(%s))",+ ["create_vector_idx:public:ragflow_wrt_012:1024"],+ ),+ ("SET LOCAL maintenance_work_mem = '1GB'", []),+ (+ 'CREATE INDEX IF NOT EXISTS "idx_gdb_ragflow_wrt_012_q_1024_vec_diskann" ON "public"."ragflow_wrt_012" USING gsdiskann (q_1024_vec COSINE) WITH (subgraph_count=1)',+ [],+ ),+]+++CANONICAL_CREATE_DOC_META_EXECUTIONS = [+ (+ "SELECT pg_advisory_xact_lock(hashtext(%s))",+ ["create_doc_meta_idx:public:ragflow_doc_meta_tenant"],+ ),+ (+ """CREATE TABLE IF NOT EXISTS "public"."ragflow_doc_meta_tenant" (+ id VARCHAR(256) NOT NULL,+ kb_id VARCHAR(256) NOT NULL,+ meta_fields JSONB,+ PRIMARY KEY (id)+) WITH (storage_type=USTORE)""",+ [],+ ),+ (+ 'CREATE INDEX IF NOT EXISTS "idx_gdb_ragflow_doc_meta_tenant_kb_id" ON "public"."ragflow_doc_meta_tenant" (kb_id)',+ [],+ ),+]+++def test_tc_sql_501_sql_executes_scoped_docengine_select_with_runtime_guard():+ cursor = RecordingCursor()+ cursor.description = [("doc_id",), ("amount",)]+ cursor.rows = [("doc1", "120")]+ conn = make_conn(cursor)++ result = conn.sql(+ "SELECT doc_id, chunk_data #>> '{amount}' AS amount FROM ragflow_tenant WHERE kb_id = 'kb1'",+ fetch_size=20,+ )++ sql, params = cursor.executed[-1]+ assert cursor.executed[0] == ("SET LOCAL statement_timeout = 30000", [])+ assert 'FROM "public".ragflow_tenant' in sql+ assert "kb_id = 'kb1'" in sql+ assert "LIMIT 20" in sql+ assert params == []+ assert result == {+ "columns": [{"name": "doc_id", "type": "text"}, {"name": "amount", "type": "text"}],+ "rows": [["doc1", "120"]],+ }+++def test_tc_sql_306_sql_passes_fetch_size_to_readonly_guard(monkeypatch):+ cursor = RecordingCursor()+ cursor.description = [("doc_id",)]+ cursor.rows = [("doc1",)]+ conn = make_conn(cursor)+ source_sql = "SELECT doc_id FROM ragflow_t1 WHERE kb_id = 'kb-1'"+ guarded_sql = f"{source_sql} LIMIT 64"+ validator = Mock()+ validator.validate_and_patch.return_value = types.SimpleNamespace(sql=guarded_sql)+ readonly_guard = Mock(return_value=validator)+ gaussdb_module = sys.modules[GaussDBConnection.__module__]+ monkeypatch.setattr(gaussdb_module.GaussDBSQLValidator, "readonly_guard", readonly_guard)++ result = conn.sql(source_sql, fetch_size=64)++ readonly_guard.assert_called_once_with(default_limit=64, execution_schema="public")+ validator.validate_and_patch.assert_called_once_with(source_sql)+ assert cursor.executed == [+ ("SET LOCAL statement_timeout = 30000", []),+ (guarded_sql, []),+ ]+ assert result == {+ "columns": [{"name": "doc_id", "type": "text"}],+ "rows": [["doc1"]],+ }+++def test_tc_cfg_312_db_type_reports_gaussdb():+ assert make_conn(RecordingCursor()).db_type() == "gaussdb"+ assert GaussDBConnectionBase(pool=HealthPool([])).db_type() == "gaussdb"+++def test_tc_sql_511_sql_returns_connection_after_query_failure():+ class FailingCursor(RecordingCursor):+ def execute(self, sql, params=None):+ super().execute(sql, params)+ if not str(sql).startswith("SET LOCAL"):+ raise RuntimeError("query timeout")++ cursor = FailingCursor()+ conn = make_conn(cursor)++ with pytest.raises(RuntimeError, match="query timeout"):+ conn.sql("SELECT doc_id FROM ragflow_tenant WHERE kb_id = 'kb1'")++ assert cursor.executed[0] == ("SET LOCAL statement_timeout = 30000", [])+ assert cursor.executed[1][0] == ("SELECT doc_id FROM \"public\".ragflow_tenant WHERE kb_id = 'kb1' LIMIT 128")+ assert cursor.closed is True+ assert conn.pool.put_back == [conn.pool.conn]+++ "sql",+ [+ "SELECT doc_id FROM pg_class WHERE kb_id = 'kb1'",+ "SELECT doc_id FROM public.ragflow_tenant WHERE kb_id = 'kb1'",+ "SELECT doc_id FROM ragflow_tenant",+ "SELECT doc_id FROM ragflow_tenant WHERE kb_id = 'kb1' OR 1 = 1",+ "SELECT content_with_weight FROM ragflow_tenant WHERE kb_id = 'kb1'",+ "SELECT chunk_data ->> 'amount' FROM ragflow_tenant WHERE kb_id = 'kb1'",+ "SELECT current_database() FROM ragflow_tenant WHERE kb_id = 'kb1'",+ ],+)+def test_tc_sql_510_sql_runtime_guard_rejects_unscoped_or_unsafe_sql(sql):+ cursor = RecordingCursor()+ conn = make_conn(cursor)++ with pytest.raises(UnsafeGaussDBSQL):+ conn.sql(sql)++ assert cursor.executed == []+++def test_tc_wrt_102_insert_chunk_uses_parameterized_upsert_and_valid_vector_flag():+ cursor = RecordingCursor()+ conn = make_conn(cursor, vector_dimensions=[1024])+ kb_id = "kb-wrt-102"+ vector = [0.1] * 1024+ vector_param = "[" + ",".join(["0.1"] * 1024) + "]"++ errors = conn.insert(+ [{"id": "c1", "kb_id": kb_id, "docnm_kwd": None, "q_1024_vec": vector, "content_with_weight": "x"}],+ "ragflow_wrt_102",+ kb_id,+ )++ expected_sql = (+ 'INSERT INTO "public"."ragflow_wrt_102" '+ "(id, kb_id, docnm_kwd, content_with_weight, available_int, removed_kwd, q_1024_vec, q_1024_vec_valid) "+ "VALUES (%s, %s, %s, %s, %s, %s, %s::floatvector(1024), %s) "+ "ON DUPLICATE KEY UPDATE docnm_kwd = VALUES(docnm_kwd), content_with_weight = VALUES(content_with_weight), "+ "available_int = VALUES(available_int), removed_kwd = VALUES(removed_kwd), "+ "q_1024_vec = VALUES(q_1024_vec), q_1024_vec_valid = VALUES(q_1024_vec_valid)"+ )+ expected_params = [["c1", kb_id, None, "x", 1, "N", vector_param, True]]+ assert errors == []+ assert cursor.executed == [(expected_sql, expected_params)]+ assert "c1" not in expected_sql+ assert kb_id not in expected_sql+ assert vector_param not in expected_sql+ assert conn.pool.conn.commits == 1+ assert conn.pool.conn.rollbacks == 0+ assert cursor.closed is True+ assert conn.pool.put_back == [conn.pool.conn]+++def test_tc_wrt_115_insert_chunk_creates_missing_gaussdb_table_and_retries_once():+ cursor = MissingTableOnceCursor()+ conn = make_conn(cursor, vector_dimensions=[4])+ create_calls = []+ conn.index_exist = lambda *_args: False+ conn.create_idx = lambda index_name, dataset_id, vector_size: create_calls.append((index_name, dataset_id, vector_size))++ errors = conn.insert(+ [+ {+ "id": "c1",+ "kb_id": "kb1",+ "doc_id": "d1",+ "content_with_weight": "hello",+ "q_4_vec": [0.1, 0.2, 0.3, 0.4],+ }+ ],+ "ragflow_tenant",+ "kb1",+ )++ assert errors == []+ assert create_calls == [("ragflow_tenant", "kb1", 4)]+ assert cursor.insert_attempts == 2+ assert conn.pool.conn.rollbacks == 1+ assert conn.pool.conn.commits == 1+++def test_tc_wrt_115_insert_chunk_does_not_create_table_for_unrelated_write_failure():+ class InvalidWriteCursor(RecordingCursor):+ def executemany(self, sql, params):+ super().executemany(sql, params)+ raise ValueError("invalid chunk value")++ cursor = InvalidWriteCursor()+ conn = make_conn(cursor, vector_dimensions=[4])+ create_calls = []+ conn.create_idx = lambda *args: create_calls.append(args)++ errors = conn.insert(+ [{"id": "c1", "kb_id": "kb1", "doc_id": "d1", "q_4_vec": [0.1, 0.2, 0.3, 0.4]}],+ "ragflow_tenant",+ "kb1",+ )++ assert errors == ["c1"]+ assert create_calls == []+ assert conn.pool.conn.rollbacks == 1+++def test_tc_wrt_115_insert_chunk_does_not_create_table_when_target_still_exists():+ cursor = MissingTableOnceCursor()+ conn = make_conn(cursor, vector_dimensions=[4])+ conn.index_exist = lambda *_args: True+ create_calls = []+ conn.create_idx = lambda *args: create_calls.append(args)++ errors = conn.insert(+ [{"id": "c1", "kb_id": "kb1", "doc_id": "d1", "q_4_vec": [0.1, 0.2, 0.3, 0.4]}],+ "ragflow_tenant",+ "kb1",+ )++ assert errors == ["c1"]+ assert create_calls == []+ assert cursor.insert_attempts == 1+++def test_tc_wrt_108_insert_normalizes_kb_id_list_to_first_value():+ cursor = RecordingCursor()+ conn = make_conn(cursor, vector_dimensions=[4])+ vector = [0.1, 0.2, 0.3, 0.4]++ errors = conn.insert(+ [{"id": "c1", "kb_id": ["kb-wrt-108", "kb-other"], "q_4_vec": vector, "content_with_weight": "test"}],+ "ragflow_wrt_108",+ "kb-wrt-108",+ )++ expected_sql = (+ 'INSERT INTO "public"."ragflow_wrt_108" '+ "(id, kb_id, content_with_weight, available_int, removed_kwd, q_4_vec, q_4_vec_valid) "+ "VALUES (%s, %s, %s, %s, %s, %s::floatvector(4), %s) "+ "ON DUPLICATE KEY UPDATE content_with_weight = VALUES(content_with_weight), "+ "available_int = VALUES(available_int), removed_kwd = VALUES(removed_kwd), "+ "q_4_vec = VALUES(q_4_vec), q_4_vec_valid = VALUES(q_4_vec_valid)"+ )+ expected_params = [["c1", "kb-wrt-108", "test", 1, "N", "[0.1,0.2,0.3,0.4]", True]]+ assert errors == []+ assert cursor.executed == [(expected_sql, expected_params)]+ assert "kb-wrt-108" not in expected_sql+ assert "kb-other" not in expected_sql+++def test_tc_wrt_201_insert_missing_vector_writes_invalid_placeholder_vector():+ cursor = RecordingCursor()+ conn = make_conn(cursor, vector_dimensions=[1024])+ zero_vector = "[" + ",".join(["0"] * 1024) + "]"++ errors = conn.insert(+ [{"id": "c2", "kb_id": "k1", "content_with_weight": "mother"}],+ "ragflow_wrt_201",+ "k1",+ )++ expected_sql = (+ 'INSERT INTO "public"."ragflow_wrt_201" '+ "(id, kb_id, content_with_weight, available_int, removed_kwd, q_1024_vec, q_1024_vec_valid) "+ "VALUES (%s, %s, %s, %s, %s, %s::floatvector(1024), %s) "+ "ON DUPLICATE KEY UPDATE content_with_weight = VALUES(content_with_weight), "+ "available_int = VALUES(available_int), removed_kwd = VALUES(removed_kwd), "+ "q_1024_vec = VALUES(q_1024_vec), q_1024_vec_valid = VALUES(q_1024_vec_valid)"+ )+ expected_params = [["c2", "k1", "mother", 1, "N", zero_vector, False]]+ assert errors == []+ assert cursor.executed == [(expected_sql, expected_params)]+ assert "c2" not in expected_sql+ assert "k1" not in expected_sql+ assert zero_vector not in expected_sql+ assert conn.pool.conn.commits == 1+ assert conn.pool.conn.rollbacks == 0+++def test_tc_wrt_202_insert_missing_vector_rejects_when_existing_dimension_is_unknown():+ cursor = RecordingCursor()+ conn = make_conn(cursor, vector_dimensions=[])+ kb_id = "kb-wrt-202"+ document = {"id": "c2", "kb_id": kb_id, "content_with_weight": "mother"}++ errors = conn.insert([document], "ragflow_wrt_202", kb_id)++ assert errors == ["c2"]+ with pytest.raises(ValueError) as exc_info:+ conn._normalize_chunk_row("ragflow_wrt_202", document, kb_id, [])+ assert exc_info.type is ValueError+ assert exc_info.value.args == ("cannot infer GaussDB vector dimension",)+ assert str(exc_info.value) == "cannot infer GaussDB vector dimension"+ assert cursor.executed == []+ assert conn.pool.put_back == []+++def test_tc_wrt_113_insert_normalization_failure_returns_all_batch_ids_without_sql():+ cursor = RecordingCursor()+ conn = make_conn(cursor, vector_dimensions=[4])+ vector = [0.1, 0.2, 0.3, 0.4]++ errors = conn.insert(+ [+ {"id": "c1", "kb_id": "kb1", "doc_id": "d1", "q_4_vec": vector},+ {"id": "c2", "kb_id": "kb2", "doc_id": "d2", "q_4_vec": vector},+ ],+ "ragflow_tenant",+ "kb1",+ )++ assert errors == ["c1", "c2"]+ assert cursor.executed == []+ assert conn.pool.put_back == []+++def test_tc_wrt_106_insert_missing_vector_infers_dimension_from_same_batch_real_vector():+ cursor = RecordingCursor()+ conn = make_conn(cursor, vector_dimensions=[])++ errors = conn.insert(+ [+ {"id": "c1", "kb_id": "kb1", "doc_id": "d1", "q_4_vec": [0.1, 0.2, 0.3, 0.4]},+ {"id": "c2", "kb_id": "kb1", "doc_id": "d2"},+ ],+ "ragflow_tenant",+ "kb1",+ )++ expected_sql = (+ 'INSERT INTO "public"."ragflow_tenant" '+ "(id, kb_id, doc_id, available_int, group_id, removed_kwd, q_4_vec, q_4_vec_valid) "+ "VALUES (%s, %s, %s, %s, %s, %s, %s::floatvector(4), %s) "+ "ON DUPLICATE KEY UPDATE doc_id = VALUES(doc_id), available_int = VALUES(available_int), "+ "group_id = VALUES(group_id), removed_kwd = VALUES(removed_kwd), q_4_vec = VALUES(q_4_vec), "+ "q_4_vec_valid = VALUES(q_4_vec_valid)"+ )+ expected_params = [+ ["c1", "kb1", "d1", 1, "d1", "N", "[0.1,0.2,0.3,0.4]", True],+ ["c2", "kb1", "d2", 1, "d2", "N", "[0,0,0,0]", False],+ ]+ assert errors == []+ assert cursor.executed == [(expected_sql, expected_params)]+ sentinels = ("c1", "c2", "kb1", "d1", "d2", "[0.1,0.2,0.3,0.4]", "[0,0,0,0]")+ assert all(value not in expected_sql for value in sentinels)+ assert conn.pool.conn.commits == 1+ assert conn.pool.conn.rollbacks == 0+++def test_tc_wrt_106_insert_preserves_metadata_and_derives_group_and_title_fields():+ cursor = RecordingCursor()+ conn = make_conn(cursor, vector_dimensions=[1024])+ kb_id = "kb-wrt-106"+ vector_param = "[" + ",".join(["0.1"] * 1024) + "]"+ fallback_vector_param = "[" + ",".join(["0.2"] * 1024) + "]"+ metadata = {"_group_id": "g1", "_title": "DocName", "other": 1}+ metadata_without_group = {"status": "open"}+ metadata_param = json.dumps(metadata, ensure_ascii=False)+ metadata_without_group_param = json.dumps(metadata_without_group, ensure_ascii=False)++ errors = conn.insert(+ [+ {+ "id": "c1",+ "kb_id": kb_id,+ "doc_id": "doc1",+ "q_1024_vec": [0.1] * 1024,+ "metadata": metadata,+ },+ {+ "id": "c2",+ "kb_id": kb_id,+ "doc_id": "doc2",+ "q_1024_vec": [0.2] * 1024,+ "metadata": metadata_without_group,+ },+ ],+ "ragflow_wrt_106",+ kb_id,+ )++ expected_sql = (+ 'INSERT INTO "public"."ragflow_wrt_106" '+ "(id, kb_id, doc_id, docnm_kwd, available_int, metadata, group_id, removed_kwd, "+ "q_1024_vec, q_1024_vec_valid) "+ "VALUES (%s, %s, %s, %s, %s, %s::jsonb, %s, %s, %s::floatvector(1024), %s) "+ "ON DUPLICATE KEY UPDATE doc_id = VALUES(doc_id), docnm_kwd = VALUES(docnm_kwd), "+ "available_int = VALUES(available_int), metadata = VALUES(metadata), group_id = VALUES(group_id), "+ "removed_kwd = VALUES(removed_kwd), q_1024_vec = VALUES(q_1024_vec), "+ "q_1024_vec_valid = VALUES(q_1024_vec_valid)"+ )+ expected_params = [+ ["c1", kb_id, "doc1", "DocName", 1, metadata_param, "g1", "N", vector_param, True],+ ["c2", kb_id, "doc2", None, 1, metadata_without_group_param, "doc2", "N", fallback_vector_param, True],+ ]+ assert errors == []+ assert cursor.executed == [(expected_sql, expected_params)]+ assert "c1" not in expected_sql+ assert kb_id not in expected_sql+ assert "DocName" not in expected_sql+ assert metadata_param not in expected_sql+ assert metadata_without_group_param not in expected_sql+ assert vector_param not in expected_sql+ assert fallback_vector_param not in expected_sql+ assert conn.pool.conn.commits == 1+ assert conn.pool.conn.rollbacks == 0+++def test_tc_wrt_107_insert_merges_unknown_fields_with_existing_extra_values():+ cursor = RecordingCursor()+ conn = make_conn(cursor, vector_dimensions=[4])+ kb_id = "kb-wrt-107"+ vector = [0.1, 0.2, 0.3, 0.4]+ vector_param = "[0.1,0.2,0.3,0.4]"++ errors = conn.insert(+ [+ {+ "id": "c1",+ "kb_id": kb_id,+ "q_4_vec": vector,+ "extra": '{"existing": 1}',+ "custom_field1": "value1",+ "custom_field2": 2,+ },+ {+ "id": "c2",+ "kb_id": kb_id,+ "q_4_vec": vector,+ "extra": "{malformed",+ "custom_field3": 3,+ },+ {+ "id": "c3",+ "kb_id": kb_id,+ "q_4_vec": vector,+ "extra": 1,+ "custom_field4": 4,+ },+ ],+ "ragflow_wrt_107",+ kb_id,+ )++ expected_sql = (+ 'INSERT INTO "public"."ragflow_wrt_107" '+ "(id, kb_id, available_int, extra, removed_kwd, q_4_vec, q_4_vec_valid) "+ "VALUES (%s, %s, %s, %s::jsonb, %s, %s::floatvector(4), %s) "+ "ON DUPLICATE KEY UPDATE available_int = VALUES(available_int), extra = VALUES(extra), "+ "removed_kwd = VALUES(removed_kwd), q_4_vec = VALUES(q_4_vec), "+ "q_4_vec_valid = VALUES(q_4_vec_valid)"+ )+ expected_params = [+ ["c1", kb_id, 1, '{"existing": 1, "custom_field1": "value1", "custom_field2": 2}', "N", vector_param, True],+ ["c2", kb_id, 1, '{"custom_field3": 3}', "N", vector_param, True],+ ["c3", kb_id, 1, '{"custom_field4": 4}', "N", vector_param, True],+ ]+ assert errors == []+ assert cursor.executed == [(expected_sql, expected_params)]+ sentinels = ("c1", "c2", "c3", kb_id, "value1", vector_param)+ assert all(value not in expected_sql for value in sentinels)+ assert conn.pool.conn.commits == 1+ assert conn.pool.conn.rollbacks == 0+++def test_tc_wrt_301_insert_doc_meta_uses_parameterized_upsert():+ cursor = RecordingCursor()+ conn = make_conn(cursor)++ errors = conn.insert(+ [{"id": "d1", "kb_id": "kb-wrt-301", "meta_fields": {"status": "active"}}],+ "ragflow_doc_meta_unit",+ "kb-wrt-301",+ )++ expected_sql = 'INSERT INTO "public"."ragflow_doc_meta_unit" (id, kb_id, meta_fields) VALUES (%s, %s, %s::jsonb) ON DUPLICATE KEY UPDATE meta_fields = VALUES(meta_fields)'+ expected_params = [["d1", "kb-wrt-301", '{"status": "active"}']]+ assert errors == []+ assert cursor.executed == [(expected_sql, expected_params)]+ assert "d1" not in expected_sql+ assert "kb-wrt-301" not in expected_sql+ assert "active" not in expected_sql+ assert conn.pool.conn.commits == 1+ assert conn.pool.conn.rollbacks == 0+++def test_tc_wrt_101_insert_empty_input_returns_empty_without_database_call():+ cursor = RecordingCursor()+ conn = make_conn(cursor)++ assert conn.insert([], "ragflow_tenant", "kb1") == []+ assert cursor.executed == []+ with pytest.raises(ValueError, match="rows are required"):+ conn._build_chunk_upsert("ragflow_tenant", [])+++def test_tc_wrt_113_insert_returns_all_ids_and_original_error_when_write_fails(monkeypatch):+ class FailingManyCursor(RecordingCursor):+ def __init__(self, failure):+ super().__init__()+ self.failure = failure++ def executemany(self, sql, params):+ super().executemany(sql, params)+ raise self.failure++ failure = psycopg2.Error("Batch insert failed")+ cursor = FailingManyCursor(failure)+ conn = make_conn(cursor, vector_dimensions=[4])+ logger = Mock()+ gaussdb_module = sys.modules[GaussDBConnection.__module__]+ monkeypatch.setattr(gaussdb_module, "logger", logger)+ documents = [{"id": f"c{i}", "kb_id": "k1"} for i in range(1, 4)]++ errors = conn.insert(documents, "ragflow_tenant", "k1")++ expected_sql = (+ 'INSERT INTO "public"."ragflow_tenant" '+ "(id, kb_id, available_int, removed_kwd, q_4_vec, q_4_vec_valid) "+ "VALUES (%s, %s, %s, %s, %s::floatvector(4), %s) "+ "ON DUPLICATE KEY UPDATE available_int = VALUES(available_int), removed_kwd = VALUES(removed_kwd), "+ "q_4_vec = VALUES(q_4_vec), q_4_vec_valid = VALUES(q_4_vec_valid)"+ )+ expected_params = [[f"c{i}", "k1", 1, "N", "[0,0,0,0]", False] for i in range(1, 4)]+ assert errors == ["c1", "c2", "c3"]+ assert len(errors) == 3+ assert cursor.executed == [(expected_sql, expected_params)]+ assert all(doc_id not in expected_sql for doc_id in errors)+ assert "k1" not in expected_sql+ assert type(failure) is psycopg2.Error+ assert failure.args == ("Batch insert failed",)+ assert str(failure) == "Batch insert failed"+ assert conn.pool.conn.rollbacks == 1+ assert conn.pool.conn.commits == 0+ logger.error.assert_called_once_with(+ "GaussDB insert failed for table=%s ids=%s error=%s",+ "ragflow_tenant",+ ["c1", "c2", "c3"],+ failure,+ )+ assert logger.error.call_args.args[-1] is failure+ assert cursor.closed is True+ assert conn.pool.put_back == [conn.pool.conn]+++def test_tc_wrt_109_insert_rejects_missing_chunk_id_without_sql():+ cursor = RecordingCursor()+ conn = make_conn(cursor)++ result = conn.insert([{"kb_id": "k1", "q_4_vec": [0.1, 0.2, 0.3, 0.4]}], "ragflow_tenant", "k1")++ assert result == ["chunk id is required"]+ assert "chunk id is required" in str(result[0])+ assert cursor.executed == []+++def test_tc_wrt_110_insert_rejects_missing_kb_id_without_sql():+ cursor = RecordingCursor()+ conn = make_conn(cursor)++ result = conn.insert([{"id": "c1", "q_4_vec": [0.1, 0.2, 0.3, 0.4]}], "ragflow_tenant", None)++ assert result == ["c1"]+ assert len(result) == 1+ assert cursor.executed == []+++def test_tc_wrt_111_insert_rejects_kb_id_mismatch_without_sql():+ cursor = RecordingCursor()+ conn = make_conn(cursor)++ result = conn.insert([{"id": "c1", "kb_id": "k1", "q_4_vec": [0.1, 0.2, 0.3, 0.4]}], "ragflow_tenant", "k2")++ assert result == ["c1"]+ assert len(result) == 1+ assert cursor.executed == []+++def test_tc_wrt_304_insert_doc_meta_rejects_missing_doc_id_or_kb_id_without_sql():+ cursor = RecordingCursor()+ conn = make_conn(cursor)++ missing_doc_id = conn.insert([{"kb_id": "kb1", "meta_fields": {}}], "ragflow_doc_meta_tenant", "kb1")+ missing_kb_id = conn.insert([{"id": "doc1", "meta_fields": {}}], "ragflow_doc_meta_tenant", None)++ assert missing_doc_id == ["doc metadata id and kb_id are required"]+ assert missing_kb_id == ["doc1"]+ with pytest.raises(ValueError, match="^doc metadata id and kb_id are required$"):+ conn._build_doc_meta_upsert("ragflow_doc_meta_tenant", [{"kb_id": "kb1"}], "kb1")+ with pytest.raises(ValueError, match="^doc metadata id and kb_id are required$"):+ conn._build_doc_meta_upsert("ragflow_doc_meta_tenant", [{"id": "doc1"}], None)+ assert cursor.executed == []+++def test_tc_wrt_305_insert_doc_meta_rejects_kb_id_mismatch_without_sql():+ cursor = RecordingCursor()+ conn = make_conn(cursor)++ result = conn.insert([{"id": "doc1", "kb_id": "kb2", "meta_fields": {}}], "ragflow_doc_meta_tenant", "kb1")++ assert result == ["doc1"]+ with pytest.raises(ValueError, match="^kb_id kb2 does not match dataset_id kb1$"):+ conn._build_doc_meta_upsert(+ "ragflow_doc_meta_tenant",+ [{"id": "doc1", "kb_id": "kb2", "meta_fields": {}}],+ "kb1",+ )+ assert cursor.executed == []+++ ("condition", "predicate", "expected_params"),+ [+ ({"id": "c1"}, "id = %s", ["c1", "kb1"]),+ ({"id": ["c1", "c2"]}, "id IN (%s, %s)", ["c1", "c2", "kb1"]),+ ],+)+def test_tc_wrt_602_delete_chunk_scopes_by_id_and_kb_id(condition, predicate, expected_params):+ cursor = RecordingCursor()+ cursor.rowcount = 1+ conn = make_conn(cursor)++ deleted = conn.delete(condition, "ragflow_tenant", "kb1")++ sql, params = cursor.executed[-1]+ assert deleted == 1+ assert 'DELETE FROM "public"."ragflow_tenant"' in sql+ assert predicate in sql+ assert "AND kb_id = %s" in sql+ assert params == expected_params+++def test_tc_wrt_309_delete_doc_meta_scopes_by_id_and_kb_id():+ cursor = RecordingCursor()+ cursor.rowcount = 1+ conn = make_conn(cursor)++ deleted = conn.delete({"id": "doc1"}, "ragflow_doc_meta_tenant", "kb1")++ assert deleted == 1+ assert cursor.executed == [('DELETE FROM "public"."ragflow_doc_meta_tenant" WHERE id = %s AND kb_id = %s', ["doc1", "kb1"])]+ sql = cursor.executed[0][0]+ assert "doc1" not in sql+ assert "kb1" not in sql+ assert "DROP TABLE" not in sql+ assert conn.pool.conn.commits == 1+ assert conn.pool.conn.rollbacks == 0+++def test_tc_wrt_601_delete_rejects_unscoped_condition():+ cursor = RecordingCursor()+ conn = make_conn(cursor)++ deleted = conn.delete({}, "ragflow_tenant", "kb1")++ assert deleted == 0+ assert cursor.executed == []+++def test_tc_wrt_613_delete_without_kb_scope_returns_zero_without_database_call():+ cursor = RecordingCursor()+ conn = make_conn(cursor)++ assert conn.delete({"id": "c1"}, "ragflow_tenant", None) == 0+ assert cursor.executed == []+++def test_tc_wrt_614_delete_allows_kb_scope_from_condition_without_dataset_id():+ cursor = RecordingCursor()+ cursor.rowcount = 1+ conn = make_conn(cursor)++ deleted = conn.delete({"id": "c1", "kb_id": "kb1"}, "ragflow_tenant", None)++ assert deleted == 1+ assert cursor.executed == [('DELETE FROM "public"."ragflow_tenant" WHERE id = %s AND kb_id = %s', ["c1", "kb1"])]+ sql = cursor.executed[0][0]+ assert "c1" not in sql+ assert "kb1" not in sql+ assert "DROP TABLE" not in sql+++def test_tc_wrt_615_repeated_missing_chunk_delete_emits_same_scoped_sql():+ cursor = RecordingCursor()+ cursor.rowcount = 0+ conn = make_conn(cursor)++ first = conn.delete({"id": "c1"}, "ragflow_tenant", "kb1")+ second = conn.delete({"id": "c1"}, "ragflow_tenant", "kb1")++ assert first == second == 0+ assert len(cursor.executed) == 2+ assert cursor.executed[0] == cursor.executed[1]+++def test_tc_wrt_503_update_builds_parameterized_set_clause_and_scopes_kb_id():+ cursor = RecordingCursor()+ cursor.rowcount = 1+ conn = make_conn(cursor)++ updated = conn.update(+ {"id": "c1", "kb_id": "k1"},+ {"available_int": 0},+ "ragflow_tenant",+ "k1",+ )++ assert updated is True+ assert cursor.executed == [('UPDATE "public"."ragflow_tenant" SET available_int = %s WHERE id = %s AND kb_id = %s', [0, "c1", "k1"])]+ sql = cursor.executed[0][0]+ assert "c1" not in sql+ assert "k1" not in sql+ assert all(keyword not in sql for keyword in ("DELETE ", "DROP ", "CREATE ", "ALTER "))+ assert conn.pool.conn.commits == 1+ assert conn.pool.conn.rollbacks == 0+++def test_tc_wrt_502_update_without_kb_scope_returns_false_without_database_call():+ cursor = RecordingCursor()+ conn = make_conn(cursor)++ updated = conn.update({"id": "c1"}, {"pagerank_fea": 9}, "ragflow_tenant", None)++ assert updated is False+ assert cursor.executed == []+++ ("new_value", "message"),+ [+ ({"id": "new-value"}, "key column cannot be updated: id"),+ ({"kb_id": "new-value"}, "key column cannot be updated: kb_id"),+ ({"remove": "id"}, "key column cannot be updated: id"),+ ({"remove": "kb_id"}, "key column cannot be updated: kb_id"),+ ],+)+def test_tc_wrt_508_update_rejects_key_column_changes_without_database_call(new_value, message):+ cursor = RecordingCursor()+ conn = make_conn(cursor)++ updated = conn.update({"id": "c1"}, new_value, "ragflow_tenant", "kb1")++ assert updated is False+ with pytest.raises(ValueError, match=f"^{message}$"):+ conn._build_set_clause(new_value, is_meta=False, condition={"id": "c1"})+ assert cursor.executed == []+++def test_tc_wrt_508_update_ignores_redundant_matching_id():+ cursor = RecordingCursor()+ conn = make_conn(cursor)++ updated = conn.update(+ {"id": "c1"},+ {"id": "c1", "content_with_weight": "updated", "q_3_vec": [0.1, 0.2, 0.3]},+ "ragflow_tenant",+ "kb1",+ )++ assert updated is True+ assert cursor.executed == [+ (+ 'UPDATE "public"."ragflow_tenant" SET content_with_weight = %s, q_3_vec = %s::floatvector(3), q_3_vec_valid = TRUE WHERE id = %s AND kb_id = %s',+ ["updated", "[0.1,0.2,0.3]", "c1", "kb1"],+ )+ ]+++def test_tc_wrt_509_update_merges_dynamic_column_into_extra_jsonb():+ cursor = RecordingCursor()+ cursor.rows = [("c1", "k1", '{"existing": 1}')]+ conn = make_conn(cursor)++ result = conn.update({"id": "c1", "kb_id": "k1"}, {"unknown_col": 1}, "ragflow_tenant", "k1")++ assert result is True+ select_sql, select_params = cursor.executed[-2]+ update_sql, update_params = cursor.executed[-1]+ assert "SELECT id, kb_id, extra" in select_sql+ assert "FOR UPDATE" in select_sql+ assert select_params == ["c1", "k1"]+ assert "extra = %s::jsonb" in update_sql+ assert json.loads(update_params[0]) == {"existing": 1, "unknown_col": 1}+ assert update_params[1:] == ["k1", "c1"]+++def test_tc_wrt_509_update_rejects_unsafe_dynamic_column_without_sql():+ cursor = RecordingCursor()+ conn = make_conn(cursor)++ result = conn.update(+ {"id": "c1", "kb_id": "k1"},+ {"bad'; DROP TABLE user; --": 1},+ "ragflow_tenant",+ "k1",+ )++ assert result is False+ assert cursor.executed == []+++def test_tc_wrt_507_update_adds_jsonb_multivalue_with_scoped_parameters():+ cursor = RecordingCursor()+ cursor.rowcount = 1+ conn = make_conn(cursor)++ updated = conn.update({"id": "c1", "kb_id": "k1"}, {"add": {"tag_kwd": "t3"}}, "ragflow_tenant", "k1")++ assert updated is True+ assert cursor.executed == [+ (+ 'UPDATE "public"."ragflow_tenant" '+ "SET tag_kwd = (SELECT CASE WHEN candidate @> %s::jsonb THEN candidate "+ "ELSE json_array_append(candidate, '$', %s::jsonb)::jsonb END "+ "FROM (SELECT COALESCE(tag_kwd, '[]'::jsonb) AS candidate) AS jsonb_update) "+ "WHERE id = %s AND kb_id = %s",+ ['["t3"]', '"t3"', "c1", "k1"],+ )+ ]+ sql = cursor.executed[0][0]+ assert "t3" not in sql+ assert "c1" not in sql+ assert "k1" not in sql+++def test_tc_wrt_506_update_removes_jsonb_value_with_scoped_parameters():+ cursor = RecordingCursor()+ cursor.rowcount = 1+ conn = make_conn(cursor)++ updated = conn.update({"id": "c1", "kb_id": "k1"}, {"remove": {"tag_kwd": "t1"}}, "ragflow_tenant", "k1")++ assert updated is True+ assert cursor.executed == [+ (+ 'UPDATE "public"."ragflow_tenant" '+ "SET tag_kwd = COALESCE(json_remove(tag_kwd, json_unquote(json_search(tag_kwd, 'one', "+ "jsonb_array_element_text(%s::jsonb, 0))))::jsonb, tag_kwd) WHERE id = %s AND kb_id = %s",+ ['["t1"]', "c1", "k1"],+ )+ ]+ sql = cursor.executed[0][0]+ assert "t1" not in sql+ assert "c1" not in sql+ assert "k1" not in sql+++def test_tc_wrt_506_json_search_treats_wildcards_as_literals():+ cursor = RecordingCursor()+ cursor.rowcount = 1+ conn = make_conn(cursor)++ updated = conn.update(+ {"id": "c1", "kb_id": "k1"},+ {"remove": {"tag_kwd": "a%_\\b"}},+ "ragflow_tenant",+ "k1",+ )++ assert updated is True+ assert json.loads(cursor.executed[0][1][0]) == ["a\\%\\_\\\\b"]+ assert cursor.executed[0][1][1:] == ["c1", "k1"]+++def test_tc_wrt_507_same_column_remove_then_add_uses_one_assignment():+ cursor = RecordingCursor()+ cursor.rowcount = 1+ conn = make_conn(cursor)++ updated = conn.update(+ {"id": "c1", "kb_id": "k1"},+ {"add": {"tag_kwd": "new"}, "remove": {"tag_kwd": "old"}},+ "ragflow_tenant",+ "k1",+ )++ assert updated is True+ assert cursor.executed == [+ (+ 'UPDATE "public"."ragflow_tenant" '+ "SET tag_kwd = (SELECT CASE WHEN candidate @> %s::jsonb THEN candidate "+ "ELSE json_array_append(candidate, '$', %s::jsonb)::jsonb END "+ "FROM (SELECT COALESCE(json_remove(tag_kwd, json_unquote(json_search(tag_kwd, 'one', "+ "jsonb_array_element_text(%s::jsonb, 0))))::jsonb, tag_kwd, '[]'::jsonb) AS candidate) AS jsonb_update) "+ "WHERE id = %s AND kb_id = %s",+ ['["new"]', '"new"', '["old"]', "c1", "k1"],+ )+ ]+ assert cursor.executed[0][0].count("SET tag_kwd =") == 1+++def test_tc_wrt_505_update_remove_column_sets_null_with_scoped_parameters():+ cursor = RecordingCursor()+ cursor.rowcount = 1+ conn = make_conn(cursor)++ updated = conn.update({"id": "c1", "kb_id": "k1"}, {"remove": "important_kwd"}, "ragflow_tenant", "k1")++ assert updated is True+ assert cursor.executed == [('UPDATE "public"."ragflow_tenant" SET important_kwd = NULL WHERE id = %s AND kb_id = %s', ["c1", "k1"])]+ sql = cursor.executed[0][0]+ assert "c1" not in sql+ assert "k1" not in sql+++ ("new_value", "expected_error"),+ [+ ({"remove": {"chunk_data": "x"}}, "unsupported JSONB remove target: chunk_data"),+ ({"add": {"chunk_data": "x"}}, "unsupported JSONB add target: chunk_data"),+ ({"remove": 1}, "unsupported remove target: 1"),+ ({"remove": "bad_column"}, "unsupported remove target: bad_column"),+ ({"add": "bad"}, "unsupported add target: bad"),+ ],+)+def test_tc_wrt_510_update_rejects_invalid_remove_add_operations_without_sql(new_value, expected_error):+ cursor = RecordingCursor()+ conn = make_conn(cursor)++ with pytest.raises(ValueError) as exc_info:+ conn._build_set_clause(new_value, is_meta=False, condition={"id": "c1", "kb_id": "k1"})++ assert str(exc_info.value) == expected_error+ result = conn.update({"id": "c1", "kb_id": "k1"}, new_value, "ragflow_tenant", "k1")++ assert result is False+ assert cursor.executed == []+++def test_tc_wrt_611_delete_jsonb_multivalue_conditions_use_contains_predicates():+ cursor = RecordingCursor()+ cursor.rowcount = 2+ conn = make_conn(cursor)++ deleted = conn.delete({"tag_kwd": ["t1"]}, "ragflow_tenant", "k1")++ assert deleted == 2+ assert cursor.executed == [('DELETE FROM "public"."ragflow_tenant" WHERE (tag_kwd @> %s::jsonb) AND kb_id = %s', ['["t1"]', "k1"])]+ sql = cursor.executed[0][0]+ assert "t1" not in sql+ assert "k1" not in sql+ assert "DROP TABLE" not in sql+++def test_tc_wrt_607_delete_must_not_exists_uses_is_null_with_kb_scope():+ cursor = RecordingCursor()+ cursor.rowcount = 1+ conn = make_conn(cursor)++ deleted = conn.delete({"must_not": {"exists": "tag_kwd"}}, "ragflow_tenant", "k1")++ assert deleted == 1+ assert cursor.executed == [('DELETE FROM "public"."ragflow_tenant" WHERE tag_kwd IS NULL AND kb_id = %s', ["k1"])]+++def test_tc_wrt_616_delete_compile_kwd_uses_extra_jsonb_mapping():+ cursor = RecordingCursor()+ cursor.rowcount = 2+ conn = make_conn(cursor)++ deleted = conn.delete(+ {"doc_id": "d1", "id": ["c1", "c2"], "must_not": {"exists": "compile_kwd"}},+ "ragflow_tenant",+ "k1",+ )++ assert deleted == 2+ assert cursor.executed == [+ (+ 'DELETE FROM "public"."ragflow_tenant" WHERE doc_id = %s AND id IN (%s, %s) AND (extra #>> \'{compile_kwd}\') IS NULL AND kb_id = %s',+ ["d1", "c1", "c2", "k1"],+ )+ ]+++ ("condition", "index_name", "is_meta", "expected_error"),+ [+ ({"id": []}, "ragflow_tenant", False, "empty list condition for id"),+ ({"tag_kwd": []}, "ragflow_tenant", False, "empty list condition for tag_kwd"),+ ({"unknown": "x"}, "ragflow_doc_meta_tenant", True, "unsupported metadata filter column: unknown"),+ ],+ ids=["empty-scalar-list", "empty-jsonb-list", "metadata-column"],+)+def test_tc_wrt_609_delete_rejects_empty_or_unsupported_conditions_without_sql(+ condition,+ index_name,+ is_meta,+ expected_error,+):+ cursor = RecordingCursor()+ conn = make_conn(cursor)++ result = conn.delete(condition, index_name, "k1")++ assert result == 0+ with pytest.raises(ValueError) as exc_info:+ conn._build_where_clause(condition, "k1", is_meta=is_meta)+ assert str(exc_info.value) == expected_error+ assert cursor.executed == []+++def test_tc_wrt_616_delete_filters_dynamic_fields_through_extra_jsonb():+ cursor = RecordingCursor()+ cursor.rowcount = 1+ conn = make_conn(cursor)++ deleted = conn.delete(+ {"compile_type": ["artifact_page"]},+ "ragflow_tenant",+ "kb1",+ )++ assert deleted == 1+ sql, params = cursor.executed[-1]+ assert "(extra -> 'compile_type') = %s::jsonb" in sql+ assert params == ['"artifact_page"', '["artifact_page"]', "kb1"]+++def test_tc_wrt_610_delete_rejects_kb_id_mismatch_without_sql():+ cursor = RecordingCursor()+ conn = make_conn(cursor)++ result = conn.delete({"kb_id": "k2"}, "ragflow_tenant", "k1")++ assert result == 0+ with pytest.raises(ValueError, match="condition kb_id k2 does not match dataset_id k1"):+ conn._build_where_clause({"kb_id": "k2"}, "k1", is_meta=False)+ assert cursor.executed == []+++def test_tc_ret_511_point_get_scopes_by_id_and_kb_id_and_decodes_jsonb():+ cursor = RecordingCursor()+ cursor.description = [("id",), ("kb_id",), ("position_int",), ("q_4_vec",), ("q_4_vec_valid",)]+ cursor.rows = [("c1", "kb1", "[[1, 2, 3, 4]]", "[0,0,0,0]", False)]+ conn = make_conn(cursor)++ row = conn.get("c1", "ragflow_tenant", ["kb1"])++ sql, params = cursor.executed[-1]+ assert sql == 'SELECT * FROM "public"."ragflow_tenant" WHERE id = %s AND kb_id IN (%s) LIMIT 1'+ assert params == ["c1", "kb1"]+ assert row == {"id": "c1", "kb_id": "kb1", "position_int": [[1, 2, 3, 4]], "q_4_vec_valid": False}++ cursor.rows = [("c1", "kb1", "[[1, 2, 3, 4]]", "[0,0,0,0]", False), ("c2", "kb2", "[]", "[0,0,0,0]", False)]+ rows = conn.get("c1", "ragflow_tenant", ["kb1", "kb2"])+ assert rows["id"] == "c1"+ assert "ORDER BY kb_id ASC LIMIT 2" in cursor.executed[-1][0]+++def test_tc_ret_513_get_promotes_dynamic_fields_from_extra_jsonb():+ cursor = RecordingCursor()+ cursor.description = [("id",), ("kb_id",), ("extra",)]+ cursor.rows = [("c1", "kb1", '{"compile_type": "artifact_page"}')]+ conn = make_conn(cursor)++ row = conn.get("c1", "ragflow_tenant", ["kb1"])++ assert row["compile_type"] == "artifact_page"+ assert row["extra"] == {"compile_type": "artifact_page"}+++def test_tc_wrt_410_get_doc_meta_allows_empty_kb_scope_and_decodes_jsonb():+ cursor = RecordingCursor()+ cursor.description = [("id",), ("kb_id",), ("meta_fields",)]+ cursor.rows = [("doc1", "kb1", '{"author": "Alice"}')]+ conn = make_conn(cursor)++ row = conn.get("doc1", "ragflow_doc_meta_tenant", [""])++ sql, params = cursor.executed[-1]+ assert "WHERE id = %s" in sql+ assert "kb_id IN" not in sql+ assert params == ["doc1"]+ assert row == {"id": "doc1", "kb_id": "kb1", "meta_fields": {"author": "Alice"}}+++def test_tc_wrt_401_get_empty_chunk_id_returns_none_without_sql():+ cursor = RecordingCursor()+ conn = make_conn(cursor)++ assert conn.get("", "ragflow_tenant", ["kb1"]) is None+ assert cursor.executed == []+++def test_tc_wrt_409_get_missing_chunk_returns_none_after_scoped_lookup():+ cursor = SequencedCursor(+ [+ ([(1,)], [("?column?",)]),+ ([], [("id",), ("kb_id",)]),+ ]+ )+ conn = make_conn(cursor)++ result = conn.get("c1", "ragflow_tenant", ["kb1"])++ assert result is None+ assert "information_schema.tables" in cursor.executed[0][0]+ assert cursor.executed[0][1] == ["public", "ragflow_tenant"]+ assert "WHERE id = %s AND kb_id IN (%s) LIMIT 1" in cursor.executed[1][0]+ assert cursor.executed[1][1] == ["c1", "kb1"]+++def test_tc_wrt_402_get_non_meta_without_kb_ids_returns_none():+ cursor = RecordingCursor()+ conn = make_conn(cursor)++ assert conn.get("c1", "ragflow_tenant", []) is None+ assert cursor.executed == []+++def test_tc_wrt_403_get_returns_none_when_table_does_not_exist():+ cursor = SequencedCursor(+ [+ ([], None),+ ]+ )+ conn = make_conn(cursor)++ assert conn.get("c1", "ragflow_tenant", ["kb1"]) is None+ assert len(cursor.executed) == 1+ assert "information_schema.tables" in cursor.executed[0][0]+++def test_tc_ret_003_search_doc_meta_table_returns_search_result():+ cursor = RecordingCursor()+ cursor.description = [("id",), ("kb_id",), ("meta_fields",), ("__total",)]+ cursor.rows = [("doc1", "kb1", '{"author": "Alice"}', 1001)]+ conn = make_conn(cursor)++ result = conn.search(+ select_fields=["id", "kb_id", "meta_fields", "id"],+ highlight_fields=[],+ condition={"id": "doc1"},+ match_expressions=[],+ order_by=OrderByExpr().desc("id"),+ offset=5,+ limit=10,+ index_names="ragflow_doc_meta_tenant",+ knowledgebase_ids=["kb1", "kb2", "kb1"],+ )++ sql, params = cursor.executed[-1]+ assert result == SearchResult(total=1001, chunks=[{"id": "doc1", "kb_id": "kb1", "meta_fields": {"author": "Alice"}}])+ assert 'SELECT id, kb_id, meta_fields, COUNT(*) OVER() AS __total FROM "public"."ragflow_doc_meta_tenant"' in sql+ assert "id = %s" in sql+ assert "kb_id IN (%s, %s)" in sql+ assert "ORDER BY id DESC" in sql+ assert "LIMIT %s OFFSET %s" in sql+ assert "to_tsvector" not in sql.lower()+ assert "plainto_tsquery" not in sql.lower()+ assert params == ["doc1", "kb1", "kb2", 10, 5]+++def test_tc_ret_003_search_doc_meta_empty_page_falls_back_to_count():+ cursor = RecordingCursor()+ conn = make_conn(cursor)+ conn._fetch_all_with_description = lambda _sql, _params: ([], [("id",), ("kb_id",), ("meta_fields",), ("__total",)])+ count_queries = []++ def fake_fetch_one(sql, params):+ count_queries.append((sql, params))+ return (1001,)++ conn._fetch_one = fake_fetch_one++ result = conn.search(+ select_fields=["*"],+ highlight_fields=[],+ condition={"id": "doc-missing"},+ match_expressions=[],+ order_by=OrderByExpr(),+ offset=2000,+ limit=1000,+ index_names="ragflow_doc_meta_tenant",+ knowledgebase_ids=["kb1"],+ )++ sql, params = count_queries[0]+ assert result == SearchResult(total=1001, chunks=[])+ assert 'SELECT COUNT(*) FROM "public"."ragflow_doc_meta_tenant"' in sql+ assert "id = %s" in sql+ assert "kb_id IN (%s)" in sql+ assert "to_tsvector" not in sql.lower()+ assert "plainto_tsquery" not in sql.lower()+ assert params == ["doc-missing", "kb1"]+++def test_tc_mgf_713_fetch_metadata_doc_ids_builds_scoped_jsonb_query():+ cursor = RecordingCursor()+ cursor.rows = [("doc2",), ("doc1",)]+ conn = make_conn(cursor)++ doc_ids = conn.fetch_metadata_doc_ids(+ "ragflow_doc_meta_tenant",+ ["kb1", "kb2"],+ "lower(meta_fields #>> '{author}') = %s",+ ["alice"],+ 25,+ )++ sql, params = cursor.executed[-1]+ assert doc_ids == ["doc2", "doc1"]+ assert 'SELECT id FROM "public"."ragflow_doc_meta_tenant"' in sql+ assert "kb_id IN (%s, %s)" in sql+ assert "(lower(meta_fields #>> '{author}') = %s)" in sql+ assert "ORDER BY id" in sql+ assert "LIMIT %s" in sql+ assert params == ["kb1", "kb2", "alice", 25]+ assert cursor.closed is True+ assert conn.pool.put_back == [conn.pool.conn]+ assert conn.pool.conn.commits == 0+ assert conn.pool.conn.rollbacks == 0+++def test_tc_wrt_202_get_vector_dimensions_reads_q_vector_columns():+ cursor = RecordingCursor()+ cursor.rows = [("q_3072_vec",), ("q_1024_vec",), ("q_3072_vec_valid",)]+ conn = make_conn(cursor)++ dimensions = conn.get_vector_dimensions("ragflow_tenant")++ sql, params = cursor.executed[0]+ query = parse_one(sql, read="postgres")+ table = query.find(exp.Table)+ predicates = list(query.find_all(exp.EQ))+ assert dimensions == [1024, 3072]+ assert isinstance(query, exp.Select)+ assert [expression.name for expression in query.expressions] == ["column_name"]+ assert (table.db, table.name) == ("information_schema", "columns")+ assert {predicate.this.name for predicate in predicates} == {"table_schema", "table_name"}+ assert all(isinstance(predicate.expression, exp.Placeholder) for predicate in predicates)+ assert params == ["public", "ragflow_tenant"]+++def test_tc_wrt_705_index_exist_checks_information_schema_tables():+ cursor = RecordingCursor()+ cursor.rows = [(1,)]+ conn = make_conn(cursor)++ result = conn.index_exist("ragflow_tenant")++ sql, params = cursor.executed[0]+ query = parse_one(sql, read="postgres")+ table = query.find(exp.Table)+ predicates = list(query.find_all(exp.EQ))+ assert result is True+ assert isinstance(query, exp.Select)+ assert query.expressions[0].this == "1"+ assert (table.db, table.name) == ("information_schema", "tables")+ assert {predicate.this.name for predicate in predicates} == {"table_schema", "table_name"}+ assert all(isinstance(predicate.expression, exp.Placeholder) for predicate in predicates)+ assert query.args["limit"].expression.this == "1"+ assert params == ["public", "ragflow_tenant"]+ assert conn.pool.conn.commits == 0+ assert conn.pool.conn.rollbacks == 0+ assert cursor.closed is True+ assert conn.pool.put_back == [conn.pool.conn]+++def test_tc_wrt_708_index_exist_rejects_invalid_index_name_without_sql():+ cursor = RecordingCursor()+ conn = make_conn(cursor)++ with pytest.raises(InvalidGaussDBObjectName, match="^1abc$"):+ conn.index_exist("1abc")++ assert cursor.executed == []+++def test_tc_wrt_701_delete_idx_with_dataset_id_deletes_only_that_kb():+ cursor = RecordingCursor()+ conn = make_conn(cursor)++ result = conn.delete_idx("ragflow_tenant", "k1")++ assert result is None+ assert cursor.executed == [('DELETE FROM "public"."ragflow_tenant" WHERE kb_id = %s', ["k1"])]+ sql = cursor.executed[0][0]+ assert "k1" not in sql+ assert "DROP TABLE" not in sql+ assert conn.pool.conn.commits == 1+ assert conn.pool.conn.rollbacks == 0+ assert cursor.closed is True+ assert conn.pool.put_back == [conn.pool.conn]+++def test_tc_wrt_702_delete_idx_without_dataset_id_drops_table():+ cursor = RecordingCursor()+ conn = make_conn(cursor)++ result = conn.delete_idx("ragflow_tenant", None)++ assert result is None+ assert cursor.executed == [('DROP TABLE IF EXISTS "public"."ragflow_tenant"', [])]+ assert "DELETE FROM" not in cursor.executed[0][0]+ assert conn.pool.conn.commits == 1+ assert conn.pool.conn.rollbacks == 0+ assert cursor.closed is True+ assert conn.pool.put_back == [conn.pool.conn]+++def test_tc_wrt_709_delete_idx_missing_table_only_ignores_undefined_table():+ class CodedDatabaseError(psycopg2.Error):+ def __init__(self, code):+ super().__init__("database error")+ self._code = code++ @property+ def pgcode(self):+ return self._code++ class FailingCursor(RecordingCursor):+ def __init__(self, failure):+ super().__init__()+ self.failure = failure++ def execute(self, sql, params=None):+ super().execute(sql, params)+ raise self.failure++ missing_cursor = FailingCursor(CodedDatabaseError(errorcodes.UNDEFINED_TABLE))+ missing_conn = make_conn(missing_cursor)++ assert missing_conn.delete_idx("ragflow_missing", "k1") is None+ assert missing_cursor.executed == [('DELETE FROM "public"."ragflow_missing" WHERE kb_id = %s', ["k1"])]+ assert missing_conn.pool.conn.rollbacks == 1+ assert missing_conn.pool.conn.commits == 0+ assert missing_cursor.closed is True+ assert missing_conn.pool.put_back == [missing_conn.pool.conn]++ permission_cursor = FailingCursor(CodedDatabaseError(errorcodes.INSUFFICIENT_PRIVILEGE))+ permission_conn = make_conn(permission_cursor)+ with pytest.raises(CodedDatabaseError) as permission_error:+ permission_conn.delete_idx("ragflow_forbidden", "k1")+ assert permission_error.value.pgcode == errorcodes.INSUFFICIENT_PRIVILEGE+ assert permission_conn.pool.conn.rollbacks == 1+ assert permission_conn.pool.conn.commits == 0+ assert permission_cursor.closed is True+ assert permission_conn.pool.put_back == [permission_conn.pool.conn]++ runtime_cursor = FailingCursor(RuntimeError("connection failure"))+ runtime_conn = make_conn(runtime_cursor)+ with pytest.raises(RuntimeError, match="connection failure"):+ runtime_conn.delete_idx("ragflow_unavailable", "k1")+ assert runtime_conn.pool.conn.rollbacks == 1+ assert runtime_conn.pool.conn.commits == 0+ assert runtime_cursor.closed is True+ assert runtime_conn.pool.put_back == [runtime_conn.pool.conn]+++def test_tc_wrt_303_create_doc_meta_idx_rejects_non_metadata_table_without_sql():+ cursor = RecordingCursor()+ conn = make_conn(cursor)++ with pytest.raises(ValueError, match="^invalid GaussDB document metadata table name: ragflow_tenant$"):+ conn.create_doc_meta_idx("ragflow_tenant")++ assert cursor.executed == []+++def test_tc_wrt_912_create_doc_meta_idx_executes_table_and_kb_index_ddls():+ cursor = RecordingCursor()+ conn = make_conn(cursor)++ result = conn.create_doc_meta_idx("ragflow_doc_meta_tenant")++ assert result is True+ assert cursor.executed == CANONICAL_CREATE_DOC_META_EXECUTIONS+ assert conn.pool.conn.commits == 1+ assert conn.pool.conn.rollbacks == 0+ assert cursor.closed is True+ assert conn.pool.put_back == [conn.pool.conn]+++def test_tc_wrt_011_create_idx_uses_schema_lock_before_ddl():+ cursor = RecordingCursor()+ conn = make_conn(cursor)++ conn.create_idx("ragflow_tenant", "kb1", 4)++ statements = [sql for sql, _params in cursor.executed]+ assert statements[0] == "SELECT pg_advisory_xact_lock(hashtext(%s))"+ assert cursor.executed[0][1] == ["create_idx:public:ragflow_tenant"]+ work_mem_index = statements.index("SET LOCAL maintenance_work_mem = '1GB'")+ diskann_index = next(i for i, sql in enumerate(statements) if "USING gsdiskann" in sql)+ assert work_mem_index < diskann_index+++def test_tc_wrt_010_create_idx_keeps_functional_table_when_diskann_ddl_fails():+ class FailingDiskANNCursor(RecordingCursor):+ def execute(self, sql, params=None):+ super().execute(sql, params)+ if "USING gsdiskann" in sql:+ raise psycopg2.OperationalError("DiskANN index creation failed")++ cursor = FailingDiskANNCursor()+ conn = make_conn(cursor)++ with pytest.raises(psycopg2.OperationalError) as exc_info:+ conn.create_idx("ragflow_wrt_012", "kb-diskann-fail", 1024)++ assert exc_info.type is psycopg2.OperationalError+ assert exc_info.value.args == ("DiskANN index creation failed",)+ assert cursor.executed == CANONICAL_CREATE_IDX_1024_EXECUTIONS+ assert conn.pool.conn.rollbacks == 1+ assert conn.pool.conn.commits == 1+ assert cursor.closed is True+ assert conn.pool.put_back == [conn.pool.conn, conn.pool.conn]+++def test_tc_wrt_012_create_idx_executes_complete_ddl_sequence():+ cursor = RecordingCursor()+ conn = make_conn(cursor)+ table = "ragflow_wrt_012"++ result = conn.create_idx(table, "kb-wrt-012", 1024)++ assert result is True+ assert len(CANONICAL_CREATE_IDX_1024_EXECUTIONS) == 14+ assert cursor.executed == CANONICAL_CREATE_IDX_1024_EXECUTIONS+ assert all("kb-wrt-012" not in sql for sql, _params in cursor.executed)+ assert conn.pool.conn.commits == 2+ assert conn.pool.conn.rollbacks == 0+ assert cursor.closed is True+ assert conn.pool.put_back == [conn.pool.conn, conn.pool.conn]++[email protected]("adapter", ["get_fields", "get_highlight", "get_scores", "get"])+def test_tc_ret_508_chunk_adapters_reject_cross_kb_duplicate_ids(adapter):+ duplicate_chunks = [+ {"id": "c1", "kb_id": "kb1", "doc_id": "d1", "_highlight": "old", "_score": 0.1},+ {"id": "c1", "kb_id": "kb2", "doc_id": "d2", "_highlight": "new", "_score": 0.9},+ ]+ if adapter == "get":+ cursor = RecordingCursor()+ cursor.description = [("id",), ("kb_id",), ("doc_id",)]+ cursor.rows = [("c1", "kb1", "d1"), ("c1", "kb2", "d2")]+ conn = make_conn(cursor)++ def call():+ return conn.get("c1", "ragflow_tenant", ["kb1", "kb2"])+ else:+ conn = GaussDBConnection.__new__(GaussDBConnection)+ result = SearchResult(total=2, chunks=duplicate_chunks)+ calls = {+ "get_fields": lambda: conn.get_fields(result, ["doc_id"]),+ "get_highlight": lambda: conn.get_highlight(result, [], "content_with_weight"),+ "get_scores": lambda: conn.get_scores(result),+ }+ call = calls[adapter]++ with pytest.raises(GaussDBError, match="^cross-KB duplicate chunk id: c1$"):+ call()+++def test_tc_ret_509_search_hides_invalid_placeholder_vector_when_vector_field_is_requested():+ cursor = RecordingCursor()+ cursor.description = [+ ("id",),+ ("kb_id",),+ ("q_4_vec",),+ ("q_4_vec_valid",),+ ("_score",),+ ("__total",),+ ]+ cursor.rows = [("c1", "kb1", "[0,0,0,0]", False, 0.0, 1)]+ conn = make_conn(cursor)++ result = conn.search(["q_4_vec"], [], {}, [], OrderByExpr(), 0, 10, "ragflow_tenant", ["kb1"])++ sql, _params = cursor.executed[-1]+ assert "q_4_vec_valid" in sql+ assert conn.get_fields(result, ["q_4_vec"]) == {"c1": {}}+++def test_tc_ret_009_search_empty_deep_page_falls_back_to_total_count():+ description = [("id",), ("kb_id",), ("_score",), ("__total",)]+ cursor = SequencedCursor(+ [+ ([], description),+ ([("c1", "kb1", 0.0, 15)], description),+ ]+ )+ conn = make_conn(cursor)++ result = conn.search(["id"], [], {}, [], OrderByExpr(), 20, 10, "ragflow_tenant", ["kb1"])++ assert result.total == 15+ assert result.chunks == []+ assert len(cursor.executed) == 2+ first_sql, first_params = cursor.executed[0]+ fallback_sql, fallback_params = cursor.executed[1]+ assert all('FROM "public"."ragflow_tenant"' in sql for sql in (first_sql, fallback_sql))+ assert all("WHERE kb_id IN (%s)" in sql for sql in (first_sql, fallback_sql))+ assert "COUNT(*) OVER() AS __total" in fallback_sql+ assert first_params[-2:] == [10, 20]+ assert fallback_params[-2:] == [1, 0]+++def test_tc_ret_002_multi_table_search_applies_global_pagination_and_ordering():+ description = [("id",), ("kb_id",), ("_score",), ("__total",)]+ cursor = SequencedCursor(+ [+ ([("c1", "kb1", 0.9, 2), ("c3", "kb1", 0.3, 2)], description),+ ([("c2", "kb1", 0.8, 1)], description),+ ]+ )+ conn = make_conn(cursor)+ match = MatchTextExpr(["content_ltks"], "risk", 10, {"original_query": "risk"})++ result = conn.search(["id"], [], {}, [match], OrderByExpr(), 1, 1, ["ragflow_tenant_a", "ragflow_tenant_b"], ["kb1"])++ assert result.total == 3+ assert [chunk["id"] for chunk in result.chunks] == ["c2"]+ assert len(cursor.executed) == 2+ assert all("LIMIT %s OFFSET %s" in sql for sql, _params in cursor.executed)+ assert [params[-2:] for _sql, params in cursor.executed] == [[2, 0], [2, 0]]+++def test_tc_ret_002_multi_table_search_breaks_score_ties_by_kb_then_id():+ description = [("id",), ("kb_id",), ("_score",), ("__total",)]+ cursor = SequencedCursor(+ [+ ([("c4", "kb2", 0.8, 2), ("c2", "kb1", 0.8, 2)], description),+ ([("c3", "kb2", 0.8, 2), ("c1", "kb1", 0.8, 2)], description),+ ]+ )+ conn = make_conn(cursor)+ match = MatchTextExpr(["content_ltks"], "risk", 10, {"original_query": "risk"})++ result = conn.search(+ ["id"],+ [],+ {},+ [match],+ OrderByExpr(),+ 0,+ 10,+ ["ragflow_tenant_a", "ragflow_tenant_b"],+ ["kb1", "kb2"],+ )++ assert result.total == 4+ assert [(chunk["kb_id"], chunk["id"]) for chunk in result.chunks] == [+ ("kb1", "c1"),+ ("kb1", "c2"),+ ("kb2", "c3"),+ ("kb2", "c4"),+ ]+++def test_tc_ret_002_multi_table_without_match_or_order_sorts_by_kb_then_id():+ description = [("id",), ("kb_id",), ("_score",), ("__total",)]+ cursor = SequencedCursor(+ [+ ([("c4", "kb2", 0.0, 2), ("c2", "kb1", 0.0, 2)], description),+ ([("c3", "kb2", 0.0, 2), ("c1", "kb1", 0.0, 2)], description),+ ]+ )+ conn = make_conn(cursor)++ result = conn.search(+ ["id"],+ [],+ {},+ [],+ OrderByExpr(),+ 0,+ 10,+ ["ragflow_tenant_a", "ragflow_tenant_b"],+ ["kb1", "kb2"],+ )++ assert result.total == 4+ assert [(chunk["kb_id"], chunk["id"]) for chunk in result.chunks] == [+ ("kb1", "c1"),+ ("kb1", "c2"),+ ("kb2", "c3"),+ ("kb2", "c4"),+ ]+++def test_tc_ret_606_multi_table_aggregation_merges_duplicate_buckets():+ description = [("value",), ("count",)]+ cursor = SequencedCursor(+ [+ ([("doc-a", 2), ("doc-b", 1)], description),+ ([("doc-a", 3)], description),+ ]+ )+ conn = make_conn(cursor)++ result = conn.search([], [], {}, [], OrderByExpr(), 0, 0, ["ragflow_tenant_a", "ragflow_tenant_b"], ["kb1"], ["docnm_kwd"])++ assert result.chunks == [{"value": "doc-a", "count": 5}, {"value": "doc-b", "count": 1}]+ assert result.total == 2+++def test_tc_ret_603_get_aggregation_counts_local_jsonb_array_values():+ conn = GaussDBConnection.__new__(GaussDBConnection)+ result = SearchResult(+ total=4,+ chunks=[{"tag_kwd": ["a", "b", ""]}, {"tag_kwd": ["a"]}, {"tag_kwd": "c"}, {"tag_kwd": None}],+ )++ assert sorted(conn.get_aggregation(result, "tag_kwd")) == [("a", 2), ("b", 1), ("c", 1)]+++def test_tc_ret_604_get_aggregation_uses_value_count_rows_without_recounting():+ conn = GaussDBConnection.__new__(GaussDBConnection)+ result = SearchResult(total=2, chunks=[{"value": "active", "count": 2}, {"value": "inactive", "count": 1}])++ assert conn.get_aggregation(result, "doc_type_kwd") == [("active", 2), ("inactive", 1)]+++def test_tc_cfg_313_base_connection_uses_shared_gaussdb_conn_by_default(monkeypatch):+ class BasePool:+ masked_uri = "user@host:19995/postgres?schema=public"+ resolved_schema = "public"++ def __init__(self):+ self.checked = False++ def check_schema_access(self):+ self.checked = True++ pool = BasePool()+ monkeypatch.setitem(sys.modules, "common.doc_store.gaussdb_conn_pool", types.SimpleNamespace(GAUSSDB_CONN=pool))++ base = GaussDBConnectionBase()++ assert base.pool is pool+ assert pool.checked is True+++ ("method_name", "args"),+ [+ ("create_idx", ("ragflow_tenant", "kb1", 4)),+ ("delete_idx", ("ragflow_tenant", "kb1")),+ ("index_exist", ("ragflow_tenant", "kb1")),+ (+ "search",+ ([], [], {}, [], OrderByExpr(), 0, 10, "ragflow_tenant", ["kb1"]),+ ),+ ("get", ("c1", "ragflow_tenant", ["kb1"])),+ ("insert", ([], "ragflow_tenant", "kb1")),+ ("update", ({"id": "c1"}, {"pagerank_fea": 1}, "ragflow_tenant", "kb1")),+ ("delete", ({"id": "c1"}, "ragflow_tenant", "kb1")),+ ("get_total", (SearchResult(total=0, chunks=[]),)),+ ("get_doc_ids", (SearchResult(total=0, chunks=[]),)),+ (+ "get_fields",+ (SearchResult(total=0, chunks=[]), ["id"]),+ ),+ (+ "get_highlight",+ (SearchResult(total=0, chunks=[]), [], "content_with_weight"),+ ),+ (+ "get_aggregation",+ (SearchResult(total=0, chunks=[]), "doc_type_kwd"),+ ),+ ("sql", ("SELECT 1", 1, "json")),+ ],+)+def test_tc_cfg_319_base_unimplemented_interfaces_fail_fast(method_name, args):+ base = GaussDBConnectionBase(pool=HealthPool([]))++ with pytest.raises(NotImplementedError):+ getattr(base, method_name)(*args)+++def test_tc_cfg_615_base_connection_health_reports_healthy_for_a_compatible_gaussdb():+ pool = HealthPool([("GaussDB 8",), ("A",), ("UTF8",), ("UTF8",)])+ health = GaussDBConnectionBase(pool=pool).health()++ assert health["status"] == "healthy"+ assert health["version_comment"] == "GaussDB 8"+ assert health["sql_compatibility"] == "A"+ assert health["server_encoding"] == "UTF8"+ assert health["client_encoding"] == "UTF8"+ assert health["uri"] == pool.masked_uri+ assert pool.checked is True+ assert pool.sql == [+ "SELECT version()",+ "SHOW sql_compatibility",+ "SHOW server_encoding",+ "SHOW client_encoding",+ ]+++def test_tc_cfg_602_base_connection_health_reports_healthy_for_ora_compatible_gaussdb():+ pool = HealthPool([("GaussDB 8",), ("ORA",), ("SQL_ASCII",), ("UTF-8",)])+ pool.check_schema_access = Mock(wraps=pool.check_schema_access)+ health = GaussDBConnectionBase(pool=pool).health()++ assert health["status"] == "healthy"+ assert health["version_comment"] == "GaussDB 8"+ assert health["sql_compatibility"] == "ORA"+ assert health["server_encoding"] == "SQL_ASCII"+ assert health["client_encoding"] == "UTF-8"+ assert "RAGFlow clients are forced to UTF8" in health["warning"]+ assert health["uri"] == "u@h:5432/d?schema=public"+ pool.check_schema_access.assert_called_once_with()+ assert pool.checked is True+ assert pool.sql == [+ "SELECT version()",+ "SHOW sql_compatibility",+ "SHOW server_encoding",+ "SHOW client_encoding",+ ]+ assert "password" not in str(health).lower()+++def test_tc_cfg_621_base_connection_health_rejects_non_utf8_client_encoding():+ pool = HealthPool([("GaussDB 8",), ("A",), ("UTF8",), ("LATIN1",)])++ health = GaussDBConnectionBase(pool=pool).health()++ assert health["status"] == "unhealthy"+ assert health["client_encoding"] == "LATIN1"+ assert health["error"] == ("unsupported GaussDB client encoding, expected UTF8: client_encoding=LATIN1")+++def test_tc_cfg_603_base_connection_health_reports_unhealthy_for_b_compatibility():+ pool = HealthPool([("GaussDB 8",), ("B",)])+ health = GaussDBConnectionBase(pool=pool).health()++ assert health["status"] == "unhealthy"+ assert health["sql_compatibility"] == "B"+ assert health["error"] == "unsupported GaussDB compatibility, expected A/ORA: sql_compatibility=B"+ assert health["uri"] == pool.masked_uri+ assert "postgresql://user:secret@host:19995/postgres" not in str(health)+ assert "password" not in str(health).lower()+ assert "token" not in str(health).lower()+++def test_tc_cfg_604_base_connection_health_reports_unhealthy_for_m_compatibility():+ pool = HealthPool([("GaussDB 3.1.0",), ("M",)])+ health = GaussDBConnectionBase(pool=pool).health()++ assert health["status"] == "unhealthy"+ assert health["sql_compatibility"] == "M"+ assert "unsupported GaussDB compatibility" in health["error"]+ assert "sql_compatibility=M" in health["error"]+ assert health["uri"] == pool.masked_uri+ assert "password" not in health["error"].lower()+ assert "token" not in health["error"].lower()+ assert "postgresql://user:secret@host:19995/postgres" not in str(health)+++def test_tc_cfg_605_base_connection_health_reports_unhealthy_when_version_is_missing():+ pool = HealthPool([None])+ health = GaussDBConnectionBase(pool=pool).health()++ assert health["status"] == "unhealthy"+ assert health["version_comment"] == "unknown"+ assert health["error"] == "GaussDB version query returned no rows"+ assert health["uri"] == pool.masked_uri+ assert "password" not in str(health).lower()+ assert "postgresql://user:secret@host:19995/postgres" not in str(health)+++def test_tc_cfg_606_base_connection_health_redacts_secrets_from_query_failures():+ password = "cfg606-password"+ dsn = f"postgresql://admin:{password}@db.internal:5432/ragflow"+ token = "cfg606-token"+ error = ConnectionError(f"connection failed password={password} dsn={dsn} token={token}")++ pool = HealthPool([error])++ health = GaussDBConnectionBase(pool=pool).health()++ assert health == {+ "status": "unhealthy",+ "uri": "u@h:5432/d?schema=public",+ "version_comment": "unknown",+ "schema": "public",+ "server_encoding": "unknown",+ "client_encoding": "unknown",+ "error": "connection failed password=*** dsn=*** token=***",+ }+ assert pool.checked is True+ assert pool.sql == ["SELECT version()"]+ serialized = json.dumps(health)+ assert password not in serialized+ assert dsn not in serialized+ assert token not in serialized+++def test_tc_cfg_607_base_connection_performance_metrics_reports_connected():+ pool = HealthPool([(1,)])++ metrics = GaussDBConnectionBase(pool=pool).get_performance_metrics()++ assert metrics["connection"] == "connected"+ assert isinstance(metrics["latency_ms"], float)+ assert math.isfinite(metrics["latency_ms"])+ assert metrics["latency_ms"] >= 0+ assert metrics["schema"] == "public"+ assert "error" not in metrics+ assert pool.sql == ["SELECT 1"]+ serialized = json.dumps(metrics).lower()+ for forbidden in ("password", "dsn", "api_key", "access_token", "postgresql://"):+ assert forbidden not in serialized+++def test_tc_cfg_608_base_connection_performance_metrics_reports_disconnected():+ password = "cfg608-password"+ dsn = f"postgresql://admin:{password}@db.internal:5432/ragflow"+ token = "cfg608-token"+ pool = HealthPool([RuntimeError(f"connection failed password={password} dsn={dsn} token={token}")])+ metrics = GaussDBConnectionBase(pool=pool).get_performance_metrics()++ assert metrics["connection"] == "disconnected"+ assert isinstance(metrics["latency_ms"], float)+ assert math.isfinite(metrics["latency_ms"])+ assert metrics["latency_ms"] >= 0+ assert metrics["error"] == "connection failed password=*** dsn=*** token=***"+ serialized = json.dumps(metrics)+ assert password not in serialized+ assert dsn not in serialized+ assert token not in serialized+ assert pool.sql == ["SELECT 1"]+++def test_tc_cfg_609_base_connection_health_does_not_start_background_polling(monkeypatch):+ def fail_thread_creation(*_args, **_kwargs):+ raise AssertionError("background polling must not start")++ monkeypatch.setattr(threading, "Thread", fail_thread_creation)+ monkeypatch.setattr(threading, "Timer", fail_thread_creation)+ pool = HealthPool([("GaussDB 8",), ("A",), ("UTF8",), ("UTF8",)])+ conn = GaussDBConnectionBase(pool=pool)+ before_count = threading.active_count()++ health = conn.health()++ assert health["status"] == "healthy"+ assert threading.active_count() == before_count+ assert pool.sql == [+ "SELECT version()",+ "SHOW sql_compatibility",+ "SHOW server_encoding",+ "SHOW client_encoding",+ ]+++def test_tc_cfg_610_base_connection_health_does_not_create_business_tables_or_indexes():+ class TestableGaussDBConnection(GaussDBConnection):+ def create_idx(self, *_args, **_kwargs):+ raise AssertionError("DDL must not run")++ def create_doc_meta_idx(self, *_args, **_kwargs):+ raise AssertionError("DDL must not run")++ pool = HealthPool([("GaussDB 8",), ("A",), ("UTF8",), ("UTF8",)])+ health = TestableGaussDBConnection(pool=pool).health()++ assert health["status"] == "healthy"+ assert pool.sql == [+ "SELECT version()",+ "SHOW sql_compatibility",+ "SHOW server_encoding",+ "SHOW client_encoding",+ ]+++def test_tc_ret_008_search_empty_index_names_returns_empty_without_database_call():+ cursor = RecordingCursor()+ conn = make_conn(cursor)++ res = conn.search(["id"], [], {}, [], OrderByExpr(), 0, 10, [], ["k1"])++ assert type(res).__name__ == "SearchResult"+ assert res.total == 0+ assert type(res.total) is int+ assert res.chunks == []+ assert conn.search([], [], {}, [], OrderByExpr(), 0, 10, "", None, dataset_ids=["kb1"]) == SearchResult(total=0, chunks=[])+ assert cursor.executed == []+++def test_tc_ret_004_search_rejects_mixed_doc_meta_and_chunk_tables_without_database_call():+ cursor = RecordingCursor()+ conn = make_conn(cursor)++ with pytest.raises(ValueError, match="^document metadata tables cannot be mixed with chunk search$"):+ conn.search(+ ["id"],+ [],+ {"kb_id": "kb1", "doc_id": ["d1"]},+ [],+ OrderByExpr(),+ 0,+ 10,+ ["ragflow_doc_meta_tenant", "ragflow_tenant"],+ ["kb1"],+ )++ assert cursor.executed == []+++def test_tc_ret_006_search_rejects_missing_kb_boundary_without_database_call():+ cursor = RecordingCursor()+ conn = make_conn(cursor)+ fetch_all = Mock(side_effect=AssertionError("unscoped search must stop before SQL"))+ conn._fetch_all_with_description = fetch_all++ with pytest.raises(ValueError, match="^GaussDB chunk search requires a kb_id boundary$"):+ conn.search(["id"], [], {}, [], OrderByExpr(), 0, 10, "ragflow_t1", [])++ assert cursor.executed == []+ fetch_all.assert_not_called()+ assert conn._scoped_search_condition({"doc_ids": ["d1"]}, ["kb1"]) == {+ "doc_id": ["d1"],+ "kb_id": ["kb1"],+ }+++def test_tc_ret_006_empty_doc_ids_keep_kb_scope():+ conn = make_conn(RecordingCursor())++ assert conn._scoped_search_condition({"doc_ids": []}, ["kb1"]) == {"kb_id": ["kb1"]}+++def test_tc_ret_007_search_rejects_condition_kb_id_outside_authorized_kbs_without_sql():+ cursor = RecordingCursor()+ conn = make_conn(cursor)++ for index_name in ("ragflow_t1", "ragflow_doc_meta_t1"):+ with pytest.raises(ValueError, match="^condition kb_id must stay within knowledgebase_ids$"):+ conn.search(["id"], [], {"kb_id": "k_other"}, [], None, 0, 10, index_name, ["k1"])++ assert cursor.executed == []+++def test_tc_ret_601_search_executes_single_field_aggregation():+ cursor = SequencedCursor(+ [+ (+ [("active", 2), ("inactive", 1)],+ [("value",), ("count",)],+ )+ ]+ )+ conn = make_conn(cursor)++ result = conn.search(+ [],+ [],+ {},+ [],+ OrderByExpr(),+ 0,+ 10,+ "ragflow_tenant",+ ["kb1"],+ ["doc_type_kwd"],+ )++ assert result == SearchResult(+ total=2,+ chunks=[{"value": "active", "count": 2}, {"value": "inactive", "count": 1}],+ )+ assert conn.get_aggregation(result, "doc_type_kwd") == [("active", 2), ("inactive", 1)]+ assert cursor.executed == [+ (+ 'SELECT doc_type_kwd AS value, COUNT(1) AS count FROM "public"."ragflow_tenant" WHERE kb_id IN (%s) AND doc_type_kwd IS NOT NULL GROUP BY value ORDER BY count DESC, value ASC LIMIT %s',+ ["kb1", 1000],+ )+ ]+++def test_tc_ret_608_search_rejects_multiple_aggregation_fields_without_partial_query():+ cursor = RecordingCursor()+ conn = make_conn(cursor)++ with pytest.raises(ValueError, match="^GaussDB search supports one aggregation field per request$"):+ conn.search(+ [],+ [],+ {},+ [],+ OrderByExpr(),+ 0,+ 10,+ "ragflow_tenant",+ ["kb1"],+ ["docnm_kwd", "tag_kwd"],+ )++ assert cursor.executed == []+++def test_tc_ret_209_search_rejects_non_float_vector_type_without_sql():+ cursor = RecordingCursor()+ conn = make_conn(cursor)+ bad = MatchDenseExpr("q_1024_vec", [0.1] * 1024, "byte", "cosine", 10, {"similarity": 0.0})++ with pytest.raises(ValueError, match="unsupported GaussDB vector data type|byte"):+ conn.search(["id"], [], {"kb_id": "k1"}, [bad], OrderByExpr(), 0, 10, "ragflow_t1", ["k1"])++ assert cursor.executed == []+++def test_tc_ret_207_search_propagates_ann_unavailable_error_without_fallback():+ conn = make_conn(RecordingCursor())+ calls = []++ def fail_fetch_all_with_description(sql, params):+ calls.append((sql, params))+ raise GaussDBError("ANN gsdiskann index unavailable")++ conn._fetch_all_with_description = fail_fetch_all_with_description+ dense = MatchDenseExpr("q_1024_vec", [0.1] * 1024, "float", "cosine", 10, {"similarity": 0.0})++ with pytest.raises(GaussDBError, match="ANN|gsdiskann|index"):+ conn.search(["id"], [], {"kb_id": "k1"}, [dense], OrderByExpr(), 0, 10, "ragflow_t1", ["k1"])++ assert len(calls) == 1+ assert "q_1024_vec" in calls[0][0]+++def test_tc_ret_208_search_propagates_missing_vector_column_error():+ conn = make_conn(RecordingCursor())+ calls = []++ def fail_fetch_all_with_description(sql, params):+ calls.append((sql, params))+ raise psycopg2.errors.UndefinedColumn("column q_512_vec does not exist")++ conn._fetch_all_with_description = fail_fetch_all_with_description+ dense = MatchDenseExpr("q_512_vec", [0.1] * 512, "float", "cosine", 10, {"similarity": 0.0})++ with pytest.raises(psycopg2.errors.UndefinedColumn, match="q_512_vec"):+ conn.search(["id"], [], {"kb_id": "k1"}, [dense], OrderByExpr(), 0, 10, "ragflow_t1", ["k1"])++ assert len(calls) == 1+ assert "q_512_vec" in calls[0][0]+++def test_tc_ret_010_multi_table_search_limit_zero_uses_default_collection_limit():+ conn = make_conn(RecordingCursor())+ calls = []+ table_chunks = {+ "ragflow_a": [{"id": f"a-{index:05d}"} for index in range(5001)],+ "ragflow_b": [{"id": f"b-{index:05d}"} for index in range(5000)],+ }++ def fake_search_chunk_table(**kwargs):+ calls.append((kwargs["table"], kwargs["limit"]))+ chunks = table_chunks[kwargs["table"]]+ return SearchResult(total=len(chunks), chunks=chunks)++ conn._search_chunk_table = fake_search_chunk_table++ result = conn.search(["id"], [], {"kb_id": "kb1"}, [], OrderByExpr().asc("id"), 0, 0, ["ragflow_a", "ragflow_b"], ["kb1"])++ assert result.total == 10001+ assert len(result.chunks) == 10000+ assert result.chunks[0]["id"] == "a-00000"+ assert result.chunks[-1]["id"] == "b-04998"+ assert all(chunk["id"] != "b-04999" for chunk in result.chunks)+ assert calls == [("ragflow_a", 10000), ("ragflow_b", 10000)]++ calls.clear()+ offset_result = conn.search(["id"], [], {"kb_id": "kb1"}, [], OrderByExpr().asc("id"), 100, 0, ["ragflow_a", "ragflow_b"], ["kb1"])++ assert offset_result.total == 10001+ assert len(offset_result.chunks) == 9901+ assert offset_result.chunks[0]["id"] == "a-00100"+ assert offset_result.chunks[-1]["id"] == "b-04999"+ assert calls == [("ragflow_a", 10100), ("ragflow_b", 10100)]+++def test_tc_wrt_511_update_failure_rolls_back_once_without_commit():+ class FailingCursor(RecordingCursor):+ def execute(self, sql, params=None):+ super().execute(sql, params)+ raise psycopg2.Error("Update failed")++ cursor = FailingCursor()+ conn = make_conn(cursor)++ assert conn.update({"id": "c1"}, {"pagerank_fea": 1}, "ragflow_tenant", "kb1") is False+ assert len(cursor.executed) == 1+ assert conn.pool.conn.rollbacks == 1+ assert conn.pool.conn.commits == 0+++def test_tc_wrt_501_update_empty_condition_or_value_returns_false_without_sql():+ cursor = RecordingCursor()+ conn = make_conn(cursor)++ assert conn.update({}, {"pagerank_fea": 1}, "ragflow_tenant", "kb1") is False+ assert conn.update({"id": "c1"}, {}, "ragflow_tenant", "kb1") is False+ assert cursor.executed == []+++def test_tc_sql_508_sql_markdown_serializes_bytes_and_json_values():+ cursor = RecordingCursor()+ cursor.description = [("doc_id",), ("payload",), ("raw",)]+ cursor.rows = [("doc1", {"amount": 120}, b"ok")]+ conn = make_conn(cursor)++ result = conn.sql("SELECT doc_id FROM ragflow_tenant WHERE kb_id = 'kb1'", format="markdown")++ assert result == {+ "columns": [+ {"name": "doc_id", "type": "text"},+ {"name": "payload", "type": "text"},+ {"name": "raw", "type": "text"},+ ],+ "rows": [["doc1", '{"amount": 120}', "ok"]],+ "markdown": '|doc_id|payload|raw|\n|---|---|---|\n|doc1|{"amount": 120}|ok|',+ }+++def test_tc_sql_502_fetch_all_with_description_sets_statement_timeout_before_business_sql():+ cursor = RecordingCursor()+ cursor.rows = [("doc1",)]+ cursor.description = [("doc_id",)]+ conn = make_conn(cursor)++ rows, description = conn._fetch_all_with_description("SELECT doc_id FROM ragflow_tenant WHERE kb_id = 'kb1'", [], 30000)++ assert rows == [("doc1",)]+ assert description == [("doc_id",)]+ assert cursor.executed[0] == ("SET LOCAL statement_timeout = 30000", [])+ assert cursor.executed[1] == ("SELECT doc_id FROM ragflow_tenant WHERE kb_id = 'kb1'", [])+ assert cursor.closed is True+ assert conn.pool.put_back == [conn.pool.conn]+++def test_tc_sql_503_sql_propagates_timeout_from_fetch_all_with_statement_timeout():+ conn = make_conn(RecordingCursor())+ calls = []++ def fail_fetch(sql, params, statement_timeout_ms=None):+ calls.append((sql, params, statement_timeout_ms))+ raise TimeoutError("statement timeout")++ conn._fetch_all_with_description = fail_fetch++ with pytest.raises(TimeoutError, match="statement timeout"):+ conn.sql("SELECT doc_id FROM ragflow_tenant WHERE kb_id = 'kb1'")++ assert calls == [+ (+ "SELECT doc_id FROM \"public\".ragflow_tenant WHERE kb_id = 'kb1' LIMIT 128",+ [],+ 30000,+ )+ ]+++def test_tc_sql_510_validator_rejection_prevents_database_fetch():+ cursor = RecordingCursor()+ conn = make_conn(cursor)++ def fail_fetch(*_args, **_kwargs):+ raise AssertionError("database fetch must not be called")++ conn._fetch_all_with_description = fail_fetch++ with pytest.raises(UnsafeGaussDBSQL, match="SELECT \\* is not allowed"):+ conn.sql("SELECT * FROM ragflow_tenant WHERE kb_id = 'kb1'")++ assert cursor.executed == []+ assert conn.pool.put_back == []+++def test_tc_ret_605_get_highlight_uses_alias_and_ignores_empty_rows():+ conn = GaussDBConnection.__new__(GaussDBConnection)+ result = SearchResult(+ total=4,+ chunks=[+ {"id": "c1", "_highlight": "<em>risk</em>"},+ {"id": "c2", "highlight": "audit"},+ {"id": "c3"},+ {"highlight": "missing id"},+ ],+ )++ assert conn.get_highlight(result, ["risk"], "content_with_weight") == {"c1": "<em>risk</em>", "c2": "audit"}+ assert (+ conn.get_highlight(+ SearchResult(total=2, chunks=[{"id": "c3"}, {"id": "c4", "_highlight": None}]),+ ["risk"],+ "content_with_weight",+ )+ == {}+ )+++def test_tc_ret_506_get_fields_filters_none_values_per_chunk():+ conn = GaussDBConnection.__new__(GaussDBConnection)+ res = SearchResult(+ total=3,+ chunks=[+ {"id": "c1", "field1": "value1", "field2": None},+ {"id": "c2", "field1": None, "field2": "value2"},+ {"field1": "missing id", "field2": "ignored"},+ ],+ )++ assert conn.get_fields(res, ["field1", "field2"]) == {+ "c1": {"field1": "value1"},+ "c2": {"field2": "value2"},+ }+++def test_tc_ret_507_get_scores_defaults_missing_score_to_zero():+ conn = GaussDBConnection.__new__(GaussDBConnection)+ res = SearchResult(+ total=4,+ chunks=[{"id": "c1", "_score": 0.8}, {"id": "c2", "_score": None}, {"id": "c3"}, {"_score": 1.0}],+ )++ assert conn.get_scores(res) == {"c1": 0.8, "c2": 0.0, "c3": 0.0}+++def test_tc_ret_509_invalid_vector_is_removed_before_get_fields_returns_values():+ conn = GaussDBConnection.__new__(GaussDBConnection)+ chunk = conn._row_to_chunk(+ ("c1", "k1", "[0.0,0.0]", False, "hello"),+ [("id",), ("kb_id",), ("q_2_vec",), ("q_2_vec_valid",), ("content_with_weight",)],+ )+ fields = conn.get_fields(SearchResult(total=1, chunks=[chunk]), ["q_2_vec", "content_with_weight"])++ assert chunk == {"id": "c1", "kb_id": "k1", "q_2_vec_valid": False, "content_with_weight": "hello"}+ assert fields == {"c1": {"content_with_weight": "hello"}}+++def test_tc_ret_505_get_doc_ids_filters_missing_ids_in_stable_order():+ conn = GaussDBConnection.__new__(GaussDBConnection)+ result = SearchResult(+ total=3,+ chunks=[+ {"id": "c2", "kb_id": "kb1"},+ {"kb_id": "kb1"},+ {"id": "c1", "kb_id": "kb1"},+ ],+ )++ assert conn.get_doc_ids(result) == ["c2", "c1"]+++def test_tc_ret_510_search_returns_search_result_with_total_and_chunks_shape():+ cursor = RecordingCursor()+ cursor.description = [("id",), ("kb_id",), ("_score",), ("__total",)]+ cursor.rows = [("c1", "k1", 0.7, 2), ("c2", "k1", 0.5, 2)]+ conn = make_conn(cursor)++ res = conn.search(["id", "kb_id"], [], {"kb_id": "k1"}, [], OrderByExpr(), 0, 10, "ragflow_t1", ["k1"])++ assert type(res).__name__ == "SearchResult"+ assert res.total == 2+ assert isinstance(res.total, int)+ assert res.chunks == [{"id": "c1", "kb_id": "k1", "_score": 0.7}, {"id": "c2", "kb_id": "k1", "_score": 0.5}]+ assert conn.get_total(SearchResult(total="3", chunks=[])) == 3+++def test_tc_mgf_713_fetch_metadata_doc_ids_returns_empty_for_empty_scope_or_filter():+ cursor = RecordingCursor()+ conn = make_conn(cursor)++ assert conn.fetch_metadata_doc_ids("ragflow_doc_meta_tenant", [], "meta_fields ? 'a'", [], 10) == []+ assert conn.fetch_metadata_doc_ids("ragflow_doc_meta_tenant", ["kb1"], "", [], 10) == []+ assert cursor.executed == []+ assert conn.pool.get_calls == 0+ assert conn.pool.put_back == []+ assert conn.pool.conn.commits == 0+++def test_tc_mgf_811_fetch_metadata_doc_ids_preserves_probe_limit_and_row_shapes():+ cursor = RecordingCursor()+ cursor.rows = [{"id": "doc1"}, {"id": None}, ("doc2",), []]+ conn = make_conn(cursor)++ result = conn.fetch_metadata_doc_ids("ragflow_doc_meta_tenant", ["kb1"], "meta_fields ? 'a'", [], 4)++ assert result == ["doc1", "doc2"]+ assert cursor.executed == [+ (+ 'SELECT id FROM "public"."ragflow_doc_meta_tenant" WHERE kb_id IN (%s) AND (meta_fields ? \'a\') ORDER BY id LIMIT %s',+ ["kb1", 4],+ )+ ]+ assert cursor.closed is True+ assert conn.pool.put_back == [conn.pool.conn]+ assert conn.pool.conn.commits == 0+ assert conn.pool.conn.rollbacks == 0+++def test_tc_ret_003_search_doc_meta_without_kb_scope_fails_before_sql():+ cursor = RecordingCursor()+ conn = make_conn(cursor)++ with pytest.raises(ValueError, match="^GaussDB document metadata search requires a kb_id boundary$"):+ conn.search(+ ["id", "kb_id"],+ [],+ {},+ [],+ OrderByExpr(),+ 0,+ 10,+ "ragflow_doc_meta_tenant",+ [],+ )++ assert cursor.executed == []+ assert conn.pool.get_calls == 0+ assert conn.pool.put_back == []+ assert conn.pool.conn.commits == 0+++def test_tc_ret_604_search_chunk_aggregation_ignores_null_values_and_missing_counts():+ cursor = SequencedCursor([([(None, 2), ("risk", None)], [("value",), ("count",)])])+ conn = make_conn(cursor)++ result = conn.search([], [], {}, [], OrderByExpr(), 0, 10, "ragflow_tenant", ["kb1"], ["docnm_kwd"])++ assert result == SearchResult(total=1, chunks=[{"value": "risk", "count": 0}])+++def test_tc_ret_607_search_chunk_aggregation_returns_value_counts_for_jsonb_array_fields():+ cursor = SequencedCursor([([("tag1", 2), ("tag2", 1)], [("value",), ("count",)])])+ conn = make_conn(cursor)++ result = conn.search([], [], {}, [], OrderByExpr(), 0, 10, "ragflow_tenant", ["kb1"], ["tag_kwd"])++ assert result == SearchResult(total=2, chunks=[{"value": "tag1", "count": 2}, {"value": "tag2", "count": 1}])+ assert len(cursor.executed) == 1+ sql, params = cursor.executed[0]+ assert "jsonb_array_elements_text(COALESCE(tag_kwd, '[]'::jsonb))" in sql+ assert "LATERAL" not in sql.upper()+ assert params == ["kb1", 1000]+++def test_tc_ret_607_search_chunk_aggregation_reraises_non_jsonb_array_builder_errors():+ class FailingAggregationBuilder:+ def build_aggregation_sql(self, **_kwargs):+ raise ValueError("unsupported aggregation field")++ cursor = RecordingCursor()+ conn = make_conn(cursor)+ conn._search_builder = lambda: FailingAggregationBuilder()++ with pytest.raises(ValueError, match="unsupported aggregation field"):+ conn.search([], [], {}, [], OrderByExpr(), 0, 10, "ragflow_tenant", ["kb1"], ["docnm_kwd"])++ assert cursor.executed == []+++def test_tc_mgf_713_fetch_metadata_doc_ids_rejects_chunk_table():+ cursor = RecordingCursor()+ conn = make_conn(cursor)++ with pytest.raises(ValueError, match="document metadata"):+ conn.fetch_metadata_doc_ids("ragflow_tenant", ["kb1"], "TRUE", [], 10)++ assert cursor.executed == []+ assert conn.pool.get_calls == 0+ assert conn.pool.put_back == []+ assert conn.pool.conn.commits == 0+++ ("delta", "min_w", "max_w"),+ [(5, 0, 100), (-5, 0, 100), (200, -10, 120), (-200, -10, 120)],+)+def test_tc_wrt_801_adjust_chunk_pagerank_repeated_calls_use_atomic_sql(delta, min_w, max_w):+ cursor = RecordingCursor()+ cursor.rowcount = 1+ conn = make_conn(cursor)++ first = conn.adjust_chunk_pagerank_fea("c1", "ragflow_tenant", "kb1", delta, min_w=min_w, max_w=max_w)+ second = conn.adjust_chunk_pagerank_fea("c1", "ragflow_tenant", "kb1", delta, min_w=min_w, max_w=max_w)++ expected = (+ 'UPDATE "public"."ragflow_tenant" SET pagerank_fea = GREATEST(%s, LEAST(%s, COALESCE(pagerank_fea, 0) + %s)) WHERE kb_id = %s AND id = %s',+ [min_w, max_w, delta, "kb1", "c1"],+ )+ assert first is second is True+ assert cursor.executed == [expected, expected]+ assert conn.pool.conn.commits == 2+++def test_tc_wrt_804_adjust_chunk_pagerank_ignores_extra_identity_parameters():+ cursor = RecordingCursor()+ cursor.rowcount = 1+ conn = make_conn(cursor)++ result = conn.adjust_chunk_pagerank_fea(+ "c1",+ "ragflow_tenant",+ "kb1",+ 5,+ row_id="row-ignored",+ extra_param="ignored",+ )++ sql, params = cursor.executed[-1]+ assert result is True+ assert "WHERE kb_id = %s AND id = %s" in sql+ assert "row_id" not in sql+ assert "extra_param" not in sql+ assert params == [0, 100, 5, "kb1", "c1"]+++def test_tc_wrt_805_adjust_chunk_pagerank_requires_chunk_and_dataset_scope_without_sql():+ cursor = RecordingCursor()+ conn = make_conn(cursor)++ assert conn.adjust_chunk_pagerank_fea("", "ragflow_tenant", "kb1", 1) is False+ assert conn.adjust_chunk_pagerank_fea("c1", "ragflow_tenant", "", 1) is False+ assert cursor.executed == []+++ ("field", "first_value", "second_value", "expected_ids", "expected_order"),+ [+ (+ "page_num_int",+ "[2]",+ "[1]",+ ["c-second", "c-first"],+ "COALESCE((page_num_int #>> '{0}')::int, 100000000) ASC",+ ),+ (+ "position_int",+ "{}",+ "[[0, 0, 0, 9]]",+ ["c-second", "c-first"],+ "COALESCE((position_int #>> '{0,3}')::int, 100000000) ASC",+ ),+ (+ "top_int",+ None,+ "[2]",+ ["c-second", "c-first"],+ "COALESCE((top_int #>> '{0}')::int, 100000000) ASC",+ ),+ ("pagerank_fea", 2, 1, ["c-second", "c-first"], "ORDER BY pagerank_fea ASC"),+ ("doc_id", None, "a", ["c-first", "c-second"], "ORDER BY doc_id ASC"),+ ],+)+def test_tc_ret_512_multi_table_search_applies_safe_compatibility_sorting(+ field,+ first_value,+ second_value,+ expected_ids,+ expected_order,+):+ description = [("id",), ("kb_id",), (field,), ("_score",), ("__total",)]+ cursor = SequencedCursor(+ [+ ([("c-first", "kb1", first_value, 0.0, 1)], description),+ ([("c-second", "kb1", second_value, 0.0, 1)], description),+ ]+ )+ conn = make_conn(cursor)+ order_by = type("Order", (), {"fields": [(field, False)]})()++ result = conn.search(+ ["id", field],+ [],+ {},+ [],+ order_by,+ 0,+ 10,+ ["ragflow_tenant_a", "ragflow_tenant_b"],+ ["kb1"],+ )++ assert result.total == 2+ assert [chunk["id"] for chunk in result.chunks] == expected_ids+ assert all(expected_order in sql for sql, _params in cursor.executed)+++def test_tc_wrt_504_update_overwrites_metadata_and_derives_title_with_scoped_parameters():+ cursor = RecordingCursor()+ cursor.rowcount = 1+ conn = make_conn(cursor)+ metadata = {"_title": "New", "x": 1}++ updated = conn.update(+ {"id": "c1", "kb_id": "k1"},+ {"metadata": metadata},+ "ragflow_tenant",+ "k1",+ )++ assert updated is True+ assert cursor.executed == [+ (+ 'UPDATE "public"."ragflow_tenant" SET metadata = %s::jsonb, docnm_kwd = %s WHERE id = %s AND kb_id = %s',+ ['{"_title": "New", "x": 1}', "New", "c1", "k1"],+ )+ ]+ sql = cursor.executed[0][0]+ assert all(value not in sql for value in ("New", "c1", "k1"))+ assert json.dumps(metadata, ensure_ascii=False) not in sql+ assert all(keyword not in sql for keyword in ("DELETE ", "DROP ", "CREATE ", "ALTER "))++ group_sql, group_params = conn._build_set_clause(+ {"metadata": {"_group_id": "g-new"}},+ is_meta=False,+ condition={"id": "c1", "kb_id": "k1"},+ )+ meta_sql, meta_params = conn._build_set_clause(+ {"meta_fields": {"status": "open"}},+ is_meta=True,+ condition={"id": "doc1", "kb_id": "k1"},+ )+ assert group_sql == "metadata = %s::jsonb, group_id = %s"+ assert group_params == ['{"_group_id": "g-new"}', "g-new"]+ assert meta_sql == "meta_fields = %s::jsonb"+ assert meta_params == ['{"status": "open"}']+++def test_tc_wrt_608_delete_exists_stays_within_kb_scope():+ cursor = RecordingCursor()+ cursor.rowcount = 2+ conn = make_conn(cursor)++ deleted = conn.delete({"exists": "doc_id"}, "ragflow_tenant", "kb1")++ assert deleted == 2+ assert cursor.executed == [('DELETE FROM "public"."ragflow_tenant" WHERE doc_id IS NOT NULL AND kb_id = %s', ["kb1"])]+ assert all(" WHERE " in sql and "kb_id = %s" in sql for sql, _params in cursor.executed)+++def test_tc_ret_804_parse_fusion_vector_weight_uses_last_weight_value():+ assert parse_fusion_vector_weight(FusionExpr("weighted_sum", 10, {"weights": "0.7,0.3"})) == 0.3+++def test_tc_ret_805_parse_fusion_vector_weight_returns_none_for_missing_or_bad_weights():+ assert parse_fusion_vector_weight(FusionExpr("weighted_sum", 10, None)) is None+ assert parse_fusion_vector_weight(FusionExpr("weighted_sum", 10, {})) is None+ assert parse_fusion_vector_weight(FusionExpr("weighted_sum", 10, {"weights": ""})) is None+ assert parse_fusion_vector_weight(FusionExpr("weighted_sum", 10, {"weights": "abc"})) is None+ assert parse_fusion_vector_weight(FusionExpr("weighted_sum", 10, {"weights": "0.7,bad"})) is None+++def test_tc_ret_806_parse_match_expressions_applies_gaussdb_vector_weight_defaults(monkeypatch):+ monkeypatch.setattr(gaussdb_conn_module, "_tokenize_query_terms", lambda query: str(query).split())+ conn = make_conn(RecordingCursor())+ text = MatchTextExpr(["content_with_weight"], "hello", 10, {"original_query": "hello"})+ dense = MatchDenseExpr("q_4_vec", [0.1, 0.2, 0.3, 0.4], "float", "cosine", 10, {"similarity": 0.0})+ fusion = FusionExpr("weighted_sum", 6, {"weights": "0.7,0.3"})++ text_only = conn._parse_match_expressions([text], rank_feature=None)+ vector_only = conn._parse_match_expressions([dense], rank_feature=None)+ hybrid_default = conn._parse_match_expressions([text, dense], rank_feature=None)+ hybrid_explicit = conn._parse_match_expressions([text, dense, fusion], rank_feature={"pagerank_fea": 7})++ assert text_only["vector_weight"] == 0.0+ assert vector_only["vector_weight"] == 1.0+ assert hybrid_default["vector_weight"] == 0.5+ assert {text_only["pagerank_weight"], vector_only["pagerank_weight"], hybrid_default["pagerank_weight"]} == {10.0}+ assert hybrid_explicit["vector_weight"] == 0.3+ assert hybrid_explicit["topn"] == 6+ assert hybrid_explicit["pagerank_weight"] == 7.0+++def test_tc_ret_110_parse_match_tokenizes_original_query(monkeypatch):+ queries = []++ def tokenize(query):+ queries.append(query)+ return ["高斯", "数据库", "端", "到", "端", "检索"]++ monkeypatch.setattr(gaussdb_conn_module, "_tokenize_query_terms", tokenize)+ conn = make_conn(RecordingCursor())+ text = MatchTextExpr(["content_ltks"], "backend-specific syntax", 10, {"original_query": "高斯数据库端到端检索"})++ parsed = conn._parse_match_expressions([text], rank_feature=None)++ assert queries == ["高斯数据库端到端检索"]+ assert parsed["keywords"] == ["高斯", "数据库", "端", "到", "端", "检索"]+++def test_tc_wrt_905_vector_literal_rejects_dimension_mismatch():+ with pytest.raises(ValueError, match="vector dimension mismatch: expected 3, got 2"):+ vector_literal([0.1, 0.2], 3)+++def test_tc_wrt_905_vector_literal_accepts_native_and_json_vectors():+ assert vector_literal([0.1, 0.2, 0.3], 3) == "[0.1,0.2,0.3]"+ assert vector_literal("[0.1, 0.2, 0.3]", 3) == "[0.1,0.2,0.3]"+++def test_tc_wrt_906_zero_vector_literal_returns_all_zero_vector_string():+ result = zero_vector_literal(3)++ assert result == "[0,0,0]"+ assert isinstance(result, str)+++def test_tc_wrt_907_ordered_columns_unifies_heterogeneous_rows():+ rows = [+ {"q_1024_vec": "[0,0]", "id": "c1"},+ {"important_kwd": '["risk"]', "id": "c2"},+ ]++ assert ordered_columns(rows) == ["id", "important_kwd", "q_1024_vec"]+++def test_tc_wrt_908_decode_column_value_deserializes_jsonb_and_vector_strings():+ assert decode_column_value("chunk_data", '{"amount":120,"active":true}') == {"amount": 120, "active": True}+ assert decode_column_value("chunk_data", {"amount": 120}) == {"amount": 120}+ assert decode_column_value("q_3_vec", "[0.1, 0.2, 0.3]") == [0.1, 0.2, 0.3]+ assert decode_column_value("q_3_vec", [0.1, 0.2, 0.3]) == [0.1, 0.2, 0.3]+ assert parse_json_dict('{"amount":120}') == {"amount": 120}+ assert parse_json_dict("not-json") == {}+ assert parse_json_dict("[1,2]") == {}+ assert parse_json_dict(None) == {}+++def test_tc_wrt_908_decode_column_value_rejects_malformed_jsonb_readback():+ with pytest.raises(ValueError, match="^invalid JSONB value for chunk_data$") as exc_info:+ decode_column_value("chunk_data", "not-json")++ assert exc_info.type is ValueError+++def test_tc_wrt_908_decode_column_value_rejects_malformed_vector_readback():+ with pytest.raises(ValueError, match="^invalid vector value for q_3_vec$") as exc_info:+ decode_column_value("q_3_vec", "not-vector")++ assert exc_info.type is ValueError+++def test_tc_wrt_908_vector_literal_rejects_invalid_string():+ with pytest.raises(ValueError, match="^invalid vector literal$") as exc_info:+ vector_literal("not-json", 2)++ assert exc_info.type is ValueErrordiff --git a/test/unit_test/rag/utils/test_gaussdb_conn_pool.py b/test/unit_test/rag/utils/test_gaussdb_conn_pool.pynew file mode 100644index 0000000000..cb3ce22419--- /dev/null+++ b/test/unit_test/rag/utils/test_gaussdb_conn_pool.py@@ -0,0 +1,881 @@+#+# Copyright 2025 The InfiniFlow Authors. All Rights Reserved.+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+#+import importlib.util+import sys+from pathlib import Path+from types import ModuleType+from unittest.mock import Mock, patch++import pytest++import common+++class ImportFakePool:+ def getconn(self):+ raise RuntimeError("import fake pool should not be used")++ def putconn(self, conn, close=False):+ pass++ def closeall(self):+ pass+++def _load_pool_module_without_import_state_leaks():+ module_name = "_test_gaussdb_conn_pool_module"+ module_path = Path(__file__).resolve().parents[4] / "common" / "doc_store" / "gaussdb_conn_pool.py"+ spec = importlib.util.spec_from_file_location(module_name, module_path)+ module = importlib.util.module_from_spec(spec)+ fake_settings = ModuleType("common.settings")+ fake_settings.GAUSSDB = {+ "host": "gaussdb.local",+ "port": "19995",+ "database": "postgres",+ "user": "sqlbuilder",+ "password": "fake-unit-password",+ }+ fake_settings.get_base_config = lambda *_args, **_kwargs: {}++ missing = object()+ previous_settings_module = sys.modules.get("common.settings", missing)+ previous_test_module = sys.modules.get(module_name, missing)+ previous_common_settings = getattr(common, "settings", missing)+ sys.modules["common.settings"] = fake_settings+ sys.modules[module_name] = module+ common.settings = fake_settings+ try:+ with patch("psycopg2.pool.ThreadedConnectionPool", lambda *_args, **_kwargs: ImportFakePool()):+ spec.loader.exec_module(module)+ finally:+ for name, previous in (+ ("common.settings", previous_settings_module),+ (module_name, previous_test_module),+ ):+ if previous is missing:+ sys.modules.pop(name, None)+ else:+ sys.modules[name] = previous+ if previous_common_settings is missing:+ delattr(common, "settings")+ else:+ common.settings = previous_common_settings+ return module+++pool_mod = _load_pool_module_without_import_state_leaks()+GaussDBAuthenticationError = pool_mod.GaussDBAuthenticationError+GaussDBConfig = pool_mod.GaussDBConfig+GaussDBConnectionError = pool_mod.GaussDBConnectionError+GaussDBConnectionPool = pool_mod.GaussDBConnectionPool+GaussDBPermissionError = pool_mod.GaussDBPermissionError+InvalidGaussDBConfig = pool_mod.InvalidGaussDBConfig+LazyGaussDBConnectionPool = pool_mod._LazyGaussDBConnectionPool+_normalize_schema = pool_mod._normalize_schema+classify_gaussdb_exception = pool_mod.classify_gaussdb_exception+load_gaussdb_config = pool_mod.load_gaussdb_config+mask_gaussdb_uri = pool_mod.mask_gaussdb_uri+++def _raw_config(**overrides):+ config = {+ "host": "127.0.0.1",+ "port": "19995",+ "database": "postgres",+ "user": "sqlbuilder",+ "password": "fake-unit-password",+ }+ config.update(overrides)+ return config+++def _install_runtime_settings(monkeypatch, gaussdb, get_base_config):+ runtime_settings = ModuleType("common.settings")+ runtime_settings.GAUSSDB = gaussdb+ runtime_settings.get_base_config = get_base_config+ monkeypatch.setitem(sys.modules, "common.settings", runtime_settings)+ monkeypatch.setattr(common, "settings", runtime_settings, raising=False)+ return runtime_settings+++def test_tc_cfg_001_load_gaussdb_config_loads_complete_config():+ cfg = load_gaussdb_config(_raw_config())++ assert cfg == GaussDBConfig(+ host="127.0.0.1",+ port=19995,+ database="postgres",+ user="sqlbuilder",+ password="fake-unit-password",+ schema="public",+ )+ assert type(cfg.port) is int+ for database_key in ("database", "db_name", "name"):+ raw = _raw_config()+ raw.pop("database")+ raw[database_key] = f"{database_key}-literal"+ assert load_gaussdb_config(raw).database == f"{database_key}-literal"+ with pytest.raises(InvalidGaussDBConfig, match=r"^missing gaussdb config field\(s\): host, port, database, user, password$"):+ load_gaussdb_config({"config": _raw_config()})+++def test_tc_cfg_002_load_gaussdb_config_preserves_explicit_schema():+ cfg = load_gaussdb_config(_raw_config(schema="tenant_schema"))++ assert cfg.schema == "tenant_schema"++[email protected]("port", [1, 65535])+def test_tc_cfg_003_load_gaussdb_config_accepts_port_boundaries(port):+ cfg = load_gaussdb_config(_raw_config(port=port))++ assert cfg.port == port+ assert cfg.schema == "public"+++def test_tc_cfg_004_mask_gaussdb_uri_omits_password_from_masked_uri():+ cfg = GaussDBConfig("db.example", 19995, "postgres", "sqlbuilder", "secret-password", "public")++ masked = mask_gaussdb_uri(cfg)++ assert masked == "[email protected]:19995/postgres?schema=public"+ assert "secret-password" not in masked++[email protected]("schema", ["", " "])+def test_tc_cfg_102_normalize_empty_schema_to_public(schema):+ assert _normalize_schema(schema) == "public"+++def test_tc_cfg_105_load_gaussdb_config_accepts_hash_and_dollar_in_schema_name():+ cfg = load_gaussdb_config(+ {+ "host": "127.0.0.1",+ "port": "19995",+ "database": "postgres",+ "user": "sqlbuilder",+ "password": "fake-unit-password",+ "schema": "ragflow#tenant$1",+ }+ )++ assert cfg.schema == "ragflow#tenant$1"+++def test_tc_cfg_106_load_gaussdb_config_accepts_high_bit_schema_name():+ cfg = load_gaussdb_config(+ {+ "host": "127.0.0.1",+ "port": "19995",+ "database": "postgres",+ "user": "sqlbuilder",+ "password": "fake-unit-password",+ "schema": "租户_schema1",+ }+ )++ assert cfg.schema == "租户_schema1"+++def test_tc_cfg_201_load_gaussdb_config_rejects_missing_host_without_leaking_values():+ raw = _raw_config(database="private-db", user="private-user", password="private-password")+ del raw["host"]++ with pytest.raises(InvalidGaussDBConfig) as exc_info:+ load_gaussdb_config(raw)++ assert str(exc_info.value) == "missing gaussdb config field(s): host"+ assert all(secret not in str(exc_info.value) for secret in ("private-db", "private-user", "private-password"))++[email protected]("field", ["port", "database", "user", "password"])+def test_tc_cfg_202_load_gaussdb_config_rejects_each_missing_required_field(field):+ raw = _raw_config(host="private-host", database="private-db", user="private-user", password="private-password")+ del raw[field]++ with pytest.raises(InvalidGaussDBConfig) as exc_info:+ load_gaussdb_config(raw)++ assert str(exc_info.value) == f"missing gaussdb config field(s): {field}"+ assert all(secret not in str(exc_info.value) for secret in ("private-host", "private-db", "private-user", "private-password"))+++ ("field", "value"),+ [+ ("host", 127001),+ ("database", 1234),+ ("user", ""),+ ("password", " "),+ ],+)+def test_tc_cfg_202_load_gaussdb_config_rejects_invalid_required_string_fields(field, value):+ with pytest.raises(InvalidGaussDBConfig, match=rf"^invalid gaussdb config field: {field}$"):+ load_gaussdb_config(_raw_config(**{field: value}))++[email protected]("schema", ["1abc", "sch ema", "public;drop table x", "s" * 64])+def test_tc_cfg_104_normalize_schema_rejects_unsafe_or_overlong_names(schema):+ with pytest.raises(InvalidGaussDBConfig) as exc_info:+ _normalize_schema(schema)++ assert str(exc_info.value) == f"invalid gaussdb schema: {schema}"+++ ("port", "expected_message"),+ [+ (-1, "invalid gaussdb config field: port"),+ (0, "invalid gaussdb config field: port"),+ (65536, "invalid gaussdb config field: port"),+ ],+)+def test_tc_cfg_203_load_gaussdb_config_rejects_invalid_port_values(port, expected_message):+ with pytest.raises(InvalidGaussDBConfig) as exc_info:+ load_gaussdb_config(_raw_config(port=port))++ assert str(exc_info.value) == expected_message++[email protected]("port", ["not-a-port", "1.5", 1.5, 5432.0, True])+def test_tc_cfg_204_load_gaussdb_config_rejects_non_integer_port(port):+ with pytest.raises(InvalidGaussDBConfig) as exc_info:+ load_gaussdb_config(_raw_config(port=port))++ assert str(exc_info.value) == "invalid gaussdb config field: port"+++def test_tc_cfg_205_load_gaussdb_config_rejects_explicit_and_default_empty_config(monkeypatch):+ expected = "missing gaussdb config field(s): host, port, database, user, password"++ with pytest.raises(InvalidGaussDBConfig) as raw_empty_exc:+ load_gaussdb_config({})++ get_base_config = Mock(return_value={})+ _install_runtime_settings(monkeypatch, {}, get_base_config)+ with pytest.raises(InvalidGaussDBConfig) as default_exc:+ load_gaussdb_config(None)++ assert str(raw_empty_exc.value) == expected+ assert str(default_exc.value) == expected+ get_base_config.assert_called_once_with("gaussdb", {})+++def test_tc_cfg_005_load_gaussdb_config_without_raw_prefers_gaussdb_block(monkeypatch):+ get_base_config = Mock(side_effect=AssertionError("fallback used"))+ _install_runtime_settings(+ monkeypatch,+ {+ "host": "gaussdb.local",+ "port": "19995",+ "database": "postgres",+ "user": "sqlbuilder",+ "password": "fake-unit-password",+ },+ get_base_config,+ )++ cfg = load_gaussdb_config()++ assert cfg == GaussDBConfig(+ host="gaussdb.local",+ port=19995,+ database="postgres",+ user="sqlbuilder",+ password="fake-unit-password",+ schema="public",+ )+ get_base_config.assert_not_called()+++def test_tc_cfg_006_load_gaussdb_config_without_raw_falls_back_to_base_config(monkeypatch):+ get_base_config = Mock(+ return_value={+ "host": "gaussdb.local",+ "port": "19995",+ "database": "postgres",+ "user": "sqlbuilder",+ "password": "fake-unit-password",+ "schema": "ragflow_gaussdb_docengine_it",+ }+ )+ _install_runtime_settings(monkeypatch, None, get_base_config)++ cfg = load_gaussdb_config()++ assert cfg == GaussDBConfig(+ host="gaussdb.local",+ port=19995,+ database="postgres",+ user="sqlbuilder",+ password="fake-unit-password",+ schema="ragflow_gaussdb_docengine_it",+ )+ get_base_config.assert_called_once_with("gaussdb", {})+++def test_tc_cfg_301_pool_initializes_driver_with_complete_connection_options(monkeypatch):+ created = {}+ sentinel_pool = object()++ def fake_threaded_pool(*args, **kwargs):+ created["args"] = args+ created["kwargs"] = kwargs+ return sentinel_pool++ monkeypatch.setattr(pool_mod.psycopg2_pool, "ThreadedConnectionPool", fake_threaded_pool)+ cfg = GaussDBConfig("h", 5432, "d", "u", "p", "zlw")++ pool = GaussDBConnectionPool(cfg, minconn=2, maxconn=4)++ assert pool._pool is sentinel_pool+ assert pool.resolved_schema == "zlw"+ assert created["args"] == (2, 4)+ assert created["kwargs"] == {+ "host": "h",+ "port": 5432,+ "dbname": "d",+ "user": "u",+ "password": "p",+ "options": ("-c search_path=zlw -c client_encoding=UTF8 -c default_transaction_read_only=off"),+ }+++def test_tc_cfg_310_gaussdb_config_and_pool_do_not_read_ssl_timeout_or_pool_tuning(monkeypatch):+ created = {}+ sentinel_pool = object()++ def fake_threaded_pool(*args, **kwargs):+ created["args"] = args+ created["kwargs"] = kwargs+ return sentinel_pool++ raw = _raw_config(+ ssl=True,+ sslmode="require",+ timeout=30,+ connect_timeout=30,+ minconn=4,+ maxconn=32,+ )+ cfg = load_gaussdb_config(raw)++ assert list(GaussDBConfig.__dataclass_fields__) == ["host", "port", "database", "user", "password", "schema"]+ for attr in ("ssl", "sslmode", "timeout", "connect_timeout", "minconn", "maxconn"):+ with pytest.raises(AttributeError, match=attr):+ getattr(cfg, attr)++ monkeypatch.setattr(pool_mod.psycopg2_pool, "ThreadedConnectionPool", fake_threaded_pool)+ pool = GaussDBConnectionPool(cfg, minconn=1, maxconn=8)++ assert pool._pool is sentinel_pool+ assert created["args"] == (1, 8)+ assert set(created["kwargs"]) == {"host", "port", "dbname", "user", "password", "options"}+ assert created["kwargs"]["options"] == ("-c search_path=public -c client_encoding=UTF8 -c default_transaction_read_only=off")+++def test_tc_cfg_320_lazy_gaussdb_pool_reuses_one_pool_and_resets_after_close(monkeypatch):+ created = []++ class SharedPool:+ masked_uri = "u@h:5432/d?schema=public"++ def __init__(self):+ self.closed = 0++ def close_all(self):+ self.closed += 1++ def create_pool():+ pool = SharedPool()+ created.append(pool)+ return pool++ monkeypatch.setattr(pool_mod, "GaussDBConnectionPool", create_pool)+ lazy_pool = LazyGaussDBConnectionPool()++ first = lazy_pool.get_pool()+ assert lazy_pool.get_pool() is first+ assert lazy_pool.masked_uri == first.masked_uri+ assert created == [first]++ lazy_pool.close_all()+ assert first.closed == 1+ second = lazy_pool.get_pool()+ assert second is not first+ assert created == [first, second]+++def test_tc_cfg_404_pool_auth_failure_prevents_adapter_write(monkeypatch):+ def fake_threaded_pool(*_args, **_kwargs):+ raise RuntimeError("password authentication failed")++ monkeypatch.setattr(pool_mod.psycopg2_pool, "ThreadedConnectionPool", fake_threaded_pool)+ cfg = GaussDBConfig("h", 5432, "d", "u", "p", "public")++ with pytest.raises(GaussDBAuthenticationError) as exc_info:+ GaussDBConnectionPool(cfg)++ assert str(exc_info.value) == "password authentication failed"++ from common.doc_store.gaussdb_conn_base import GaussDBDDLBuilder+ from rag.utils.gaussdb_conn import GaussDBConnection++ connection = GaussDBConnection.__new__(GaussDBConnection)+ connection.ddl = GaussDBDDLBuilder(schema="public")+ connection.schema = "public"+ connection.pool = Mock()+ connection.pool.get_conn.side_effect = GaussDBAuthenticationError("auth failed")++ assert connection.update({"id": "c1"}, {"pagerank_fea": 1}, "ragflow_tenant", "kb1") is False+ connection.pool.get_conn.assert_called_once_with()+ connection.pool.put_conn.assert_not_called()+ connection.pool.execute.assert_not_called()+ connection.pool.commit.assert_not_called()+ connection.pool.rollback.assert_not_called()+++ "message",+ [+ 'FATAL: password authentication failed for user "test"',+ "ERROR: invalid authentication method",+ "invalid username or password",+ ],+)+def test_tc_cfg_401_classify_gaussdb_exception_maps_authentication_errors(message):+ classified = classify_gaussdb_exception(RuntimeError(message))++ assert type(classified) is GaussDBAuthenticationError+ assert str(classified) == message++[email protected]("message", ["ERROR: permission denied for schema public", "ERROR: privilege not granted"])+def test_tc_cfg_402_classify_gaussdb_exception_maps_permission_errors(message):+ classified = classify_gaussdb_exception(RuntimeError(message))++ assert type(classified) is GaussDBPermissionError+ assert str(classified) == message++[email protected]("message", ["connection timeout", "timeout expired", "network error", "generic connection error"])+def test_tc_cfg_403_classify_gaussdb_exception_maps_timeout_and_generic_errors(message):+ classified = classify_gaussdb_exception(RuntimeError(message))++ assert type(classified) is GaussDBConnectionError+ assert str(classified) == message+++def test_tc_cfg_405_classify_gaussdb_exception_preserves_existing_classification():+ existing = GaussDBPermissionError("already classified")++ assert classify_gaussdb_exception(existing) is existing+++class FakeCursor:+ def __init__(self, row):+ self.row = row+ self.executed = []+ self.closed = False++ def execute(self, sql, params=None):+ self.executed.append((sql, params))++ def fetchone(self):+ return self.row++ def close(self):+ self.closed = True+++class FakeConnection:+ def __init__(self, row):+ self.cursor_obj = FakeCursor(row)+ self.rollbacks = 0+ self.closed = False++ def cursor(self):+ return self.cursor_obj++ def rollback(self):+ self.rollbacks += 1+++class FakePool:+ def __init__(self, row=(True, True)):+ self.conn = FakeConnection(row)+ self.returned = []+ self.closed = False++ def getconn(self):+ return self.conn++ def putconn(self, conn, close=False):+ if close:+ conn.closed = True+ self.returned.append(conn)++ def closeall(self):+ self.closed = True+++def _assert_schema_privilege_query(sql, params, user, schema):+ assert " ".join(sql.split()) == ("SELECT has_schema_privilege(%s, %s, %s) AS has_usage, has_schema_privilege(%s, %s, %s) AS has_create")+ assert params == (user, schema, "USAGE", user, schema, "CREATE")+++def test_tc_cfg_318_pool_check_schema_access_verifies_usage_and_create_privileges():+ cfg = GaussDBConfig("db.example", 19995, "postgres", "sqlbuilder", "fake-unit-password", "ragflow_gaussdb_docengine_it")+ fake_pool = FakePool(row=(True, True))+ pool = GaussDBConnectionPool(cfg, pool=fake_pool)++ pool.check_schema_access()++ sql, params = fake_pool.conn.cursor_obj.executed[1]+ _assert_schema_privilege_query(sql, params, "sqlbuilder", "ragflow_gaussdb_docengine_it")+ assert fake_pool.returned == [fake_pool.conn]+ assert fake_pool.conn.cursor_obj.closed is True+ assert fake_pool.conn.rollbacks == 2+++def test_tc_cfg_308_pool_check_schema_access_rejects_missing_create_privilege():+ cfg = GaussDBConfig("h", 5432, "d", "u", "p", "public")+ fake_pool = FakePool(row=(True, False))+ pool = GaussDBConnectionPool(cfg, pool=fake_pool)++ with pytest.raises(GaussDBPermissionError) as exc_info:+ pool.check_schema_access()++ assert str(exc_info.value) == "GaussDB user u lacks CREATE on schema public"+ sql, params = fake_pool.conn.cursor_obj.executed[1]+ _assert_schema_privilege_query(sql, params, "u", "public")+ assert fake_pool.conn.cursor_obj.closed is True+ assert fake_pool.returned == [fake_pool.conn]+ assert fake_pool.conn.rollbacks == 2+++def test_tc_cfg_307_pool_check_schema_access_rejects_missing_usage_privilege():+ cfg = GaussDBConfig("h", 5432, "d", "u", "p", "public")+ fake_pool = FakePool(row=(False, True))+ pool = GaussDBConnectionPool(cfg, pool=fake_pool)++ with pytest.raises(GaussDBPermissionError) as exc_info:+ pool.check_schema_access()++ assert str(exc_info.value) == "GaussDB user u lacks USAGE on schema public"+ sql, params = fake_pool.conn.cursor_obj.executed[1]+ _assert_schema_privilege_query(sql, params, "u", "public")+ assert fake_pool.conn.cursor_obj.closed is True+ assert fake_pool.returned == [fake_pool.conn]+ assert fake_pool.conn.rollbacks == 2+++def test_tc_cfg_314_pool_check_schema_access_classifies_query_failure():+ class FailingSchemaCursor(FakeCursor):+ def execute(self, sql, params=None):+ super().execute(sql, params)+ if "has_schema_privilege" in sql:+ raise RuntimeError("permission denied while checking schema")++ class FailingSchemaConnection(FakeConnection):+ def __init__(self):+ super().__init__(row=None)+ self.cursor_obj = FailingSchemaCursor(row=None)++ cfg = GaussDBConfig("db.example", 19995, "postgres", "sqlbuilder", "fake-unit-password", "ragflow_gaussdb_docengine_it")+ fake_pool = FakePool(row=(True, True))+ fake_pool.conn = FailingSchemaConnection()+ pool = GaussDBConnectionPool(cfg, pool=fake_pool)++ with pytest.raises(GaussDBPermissionError) as exc_info:+ pool.check_schema_access()++ assert str(exc_info.value) == "permission denied while checking schema"+ assert fake_pool.conn.cursor_obj.closed is True+ assert fake_pool.returned == [fake_pool.conn]+++def test_tc_cfg_314_pool_check_schema_access_classifies_cursor_creation_failure():+ class CursorFailsAfterValidationConnection(FakeConnection):+ def __init__(self):+ super().__init__(row=(True, True))+ self.cursor_calls = 0++ def cursor(self):+ self.cursor_calls += 1+ if self.cursor_calls == 1:+ return self.cursor_obj+ raise RuntimeError("network timeout before privilege check")++ cfg = GaussDBConfig("db.example", 19995, "postgres", "sqlbuilder", "fake-unit-password", "ragflow_gaussdb_docengine_it")+ fake_pool = FakePool(row=(True, True))+ fake_pool.conn = CursorFailsAfterValidationConnection()+ pool = GaussDBConnectionPool(cfg, pool=fake_pool)++ with pytest.raises(GaussDBConnectionError) as exc_info:+ pool.check_schema_access()++ assert str(exc_info.value) == "network timeout before privilege check"+ assert fake_pool.conn.cursor_obj.closed is True+ assert fake_pool.returned == [fake_pool.conn]+++class FailingPingCursor(FakeCursor):+ def execute(self, sql, params=None):+ self.executed.append((sql, params))+ if sql == "SELECT 1":+ raise RuntimeError("SSL SYSCALL error: EOF detected")+++class SequencedPool:+ def __init__(self):+ self.dead_conn = FakeConnection(row=None)+ self.dead_conn.cursor_obj = FailingPingCursor(row=None)+ self.live_conn = FakeConnection(row=None)+ self.returned = []+ self.closed = []++ def getconn(self):+ if not self.returned:+ return self.dead_conn+ return self.live_conn++ def putconn(self, conn, close=False):+ self.returned.append(conn)+ if close:+ self.closed.append(conn)+++def test_tc_cfg_315_pool_get_conn_discards_stale_connection_and_retries_once():+ cfg = GaussDBConfig("db.example", 19995, "postgres", "sqlbuilder", "fake-unit-password", "ragflow_gaussdb_docengine_it")+ fake_pool = SequencedPool()+ pool = GaussDBConnectionPool(cfg, pool=fake_pool)++ conn = pool.get_conn()++ assert conn is fake_pool.live_conn+ assert fake_pool.closed == [fake_pool.dead_conn]+ assert fake_pool.dead_conn.cursor_obj.closed is True+ assert fake_pool.live_conn.cursor_obj.closed is True+ assert fake_pool.live_conn.rollbacks == 1+++def test_tc_cfg_315_pool_get_conn_retries_checkout_failure_without_discarding_none():+ class CheckoutFailureThenSuccessPool:+ def __init__(self):+ self.calls = 0+ self.live_conn = FakeConnection(row=(1,))+ self.returned = []++ def getconn(self):+ self.calls += 1+ if self.calls == 1:+ raise RuntimeError("pool temporarily exhausted")+ return self.live_conn++ def putconn(self, conn, close=False):+ self.returned.append((conn, close))++ cfg = GaussDBConfig("db.example", 19995, "postgres", "sqlbuilder", "fake-unit-password", "ragflow_gaussdb_docengine_it")+ fake_pool = CheckoutFailureThenSuccessPool()+ pool = GaussDBConnectionPool(cfg, pool=fake_pool)++ conn = pool.get_conn()++ assert conn is fake_pool.live_conn+ assert fake_pool.calls == 2+ assert fake_pool.returned == []+++def test_tc_cfg_304_pool_put_conn_rolls_back_before_returning_connection():+ cfg = GaussDBConfig("db.example", 19995, "postgres", "sqlbuilder", "fake-unit-password", "ragflow_gaussdb_docengine_it")+ fake_pool = FakePool(row=(True, True))+ pool = GaussDBConnectionPool(cfg, pool=fake_pool)+ conn = FakeConnection(row=None)++ pool.put_conn(conn)++ assert fake_pool.returned == [conn]+ assert conn.rollbacks == 1+ assert conn.closed is False+++class RollbackFailingConnection(FakeConnection):+ def rollback(self):+ self.rollbacks += 1+ raise RuntimeError("connection lost")+++def test_tc_cfg_305_pool_put_conn_discards_connection_when_rollback_fails():+ cfg = GaussDBConfig("db.example", 19995, "postgres", "sqlbuilder", "fake-unit-password", "ragflow_gaussdb_docengine_it")+ fake_pool = FakePool(row=(True, True))+ pool = GaussDBConnectionPool(cfg, pool=fake_pool)+ conn = RollbackFailingConnection(row=None)++ pool.put_conn(conn)++ assert fake_pool.returned == [conn]+ assert conn.closed is True+ assert conn.rollbacks == 1+++def test_tc_cfg_316_pool_put_conn_ignores_discard_failure_after_rollback_failure():+ class DiscardFailingPool(FakePool):+ def putconn(self, conn, close=False):+ raise RuntimeError("discard failed")++ cfg = GaussDBConfig("db.example", 19995, "postgres", "sqlbuilder", "fake-unit-password", "ragflow_gaussdb_docengine_it")+ pool = GaussDBConnectionPool(cfg, pool=DiscardFailingPool(row=(True, True)))+ conn = RollbackFailingConnection(row=None)+ discard = Mock(wraps=pool._discard_conn)+ pool._discard_conn = discard++ pool.put_conn(conn)++ assert conn.rollbacks == 1+ discard.assert_called_once_with(conn)+ assert pool._pool.returned == []+++def test_tc_cfg_303_pool_get_conn_tries_twice_then_raises_closed_error():+ class AlwaysClosedPool:+ def __init__(self):+ self.connections = []+ self.discarded = []++ def getconn(self):+ conn = FakeConnection(row=None)+ conn.closed = True+ self.connections.append(conn)+ return conn++ def putconn(self, conn, close=False):+ if close:+ self.discarded.append(conn)++ cfg = GaussDBConfig("db.example", 19995, "postgres", "sqlbuilder", "fake-unit-password", "ragflow_gaussdb_docengine_it")+ fake_pool = AlwaysClosedPool()+ pool = GaussDBConnectionPool(cfg, pool=fake_pool)++ with pytest.raises(GaussDBConnectionError) as exc_info:+ pool.get_conn()++ assert str(exc_info.value) == "GaussDB connection is closed"+ assert len(fake_pool.connections) == 2+ assert fake_pool.discarded == fake_pool.connections+++def test_tc_cfg_314_pool_get_conn_classifies_cursor_creation_failure():+ class CursorCreationFailureConnection(FakeConnection):+ def cursor(self):+ raise RuntimeError("network timeout before cursor")++ class CursorCreationFailurePool:+ def __init__(self):+ self.conn = CursorCreationFailureConnection(row=None)+ self.closed = []++ def getconn(self):+ return self.conn++ def putconn(self, conn, close=False):+ if close:+ self.closed.append(conn)++ cfg = GaussDBConfig("db.example", 19995, "postgres", "sqlbuilder", "fake-unit-password", "ragflow_gaussdb_docengine_it")+ fake_pool = CursorCreationFailurePool()+ pool = GaussDBConnectionPool(cfg, pool=fake_pool)++ with pytest.raises(GaussDBConnectionError) as exc_info:+ pool.get_conn()++ assert str(exc_info.value) == "network timeout before cursor"+ assert fake_pool.closed == [fake_pool.conn, fake_pool.conn]+++def test_tc_cfg_316_pool_put_conn_ignores_none_and_close_all_delegates():+ cfg = GaussDBConfig("db.example", 19995, "postgres", "sqlbuilder", "fake-unit-password", "ragflow_gaussdb_docengine_it")+ fake_pool = FakePool(row=(True, True))+ pool = GaussDBConnectionPool(cfg, pool=fake_pool)++ pool.put_conn(None)+ pool.close_all()++ assert fake_pool.returned == []+ assert fake_pool.closed is True+++def test_tc_cfg_317_pool_fetch_one_executes_query_and_returns_connection():+ cfg = GaussDBConfig("db.example", 19995, "postgres", "sqlbuilder", "fake-unit-password", "ragflow_gaussdb_docengine_it")+ fake_pool = FakePool(row=("GaussDB",))+ pool = GaussDBConnectionPool(cfg, pool=fake_pool)++ row = pool.fetch_one("SELECT version()", ("arg",))++ assert row == ("GaussDB",)+ assert fake_pool.conn.cursor_obj.executed[-1] == ("SELECT version()", ("arg",))+ assert fake_pool.conn.cursor_obj.closed is True+ assert fake_pool.returned == [fake_pool.conn]+++def test_tc_cfg_317_pool_fetch_one_classifies_cursor_creation_failure_and_returns_connection():+ class CursorFailsAfterValidationConnection(FakeConnection):+ def __init__(self):+ super().__init__(row=(True,))+ self.cursor_calls = 0++ def cursor(self):+ self.cursor_calls += 1+ if self.cursor_calls == 1:+ return self.cursor_obj+ raise RuntimeError("network timeout before query")++ cfg = GaussDBConfig("db.example", 19995, "postgres", "sqlbuilder", "fake-unit-password", "ragflow_gaussdb_docengine_it")+ fake_pool = FakePool(row=(True, True))+ fake_pool.conn = CursorFailsAfterValidationConnection()+ pool = GaussDBConnectionPool(cfg, pool=fake_pool)++ with pytest.raises(GaussDBConnectionError) as exc_info:+ pool.fetch_one("SELECT 1")++ assert str(exc_info.value) == "network timeout before query"+ assert fake_pool.conn.cursor_obj.closed is True+ assert fake_pool.returned == [fake_pool.conn]+++def test_tc_cfg_317_pool_fetch_one_classifies_query_failure_and_cleans_up():+ class FailingQueryCursor(FakeCursor):+ def execute(self, sql, params=None):+ super().execute(sql, params)+ if sql != "SELECT 1":+ raise RuntimeError("authentication failed while querying")++ class FailingQueryConnection(FakeConnection):+ def __init__(self):+ super().__init__(row=None)+ self.cursor_obj = FailingQueryCursor(row=None)++ cfg = GaussDBConfig("db.example", 19995, "postgres", "sqlbuilder", "fake-unit-password", "ragflow_gaussdb_docengine_it")+ fake_pool = FakePool(row=(True, True))+ fake_pool.conn = FailingQueryConnection()+ pool = GaussDBConnectionPool(cfg, pool=fake_pool)++ with pytest.raises(GaussDBAuthenticationError) as exc_info:+ pool.fetch_one("SELECT version()")++ assert str(exc_info.value) == "authentication failed while querying"+ assert fake_pool.conn.cursor_obj.closed is True+ assert fake_pool.returned == [fake_pool.conn]diff --git a/test/unit_test/rag/utils/test_gaussdb_ddl.py b/test/unit_test/rag/utils/test_gaussdb_ddl.pynew file mode 100644index 0000000000..a8ec872284--- /dev/null+++ b/test/unit_test/rag/utils/test_gaussdb_ddl.py@@ -0,0 +1,303 @@+#+# Copyright 2025 The InfiniFlow Authors. All Rights Reserved.+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+#++import pytest++from common.doc_store.gaussdb_conn_base import (+ GaussDBDDLBuilder,+ InvalidGaussDBObjectName,+)+++def _normalize_ddl(ddl):+ return " ".join(ddl.split())++[email protected]("identifier", ["ragflow_unit", "schema#tenant$1", "中文标识符"])+def test_tc_wrt_901_identifier_helpers_return_exact_valid_and_quoted_names(identifier):+ builder = GaussDBDDLBuilder(schema="test_schema")++ assert builder.validate_identifier(identifier) == identifier+ assert builder.quote_identifier(identifier) == f'"{identifier}"'+ assert builder.qualified_name(identifier) == f'"test_schema"."{identifier}"'+++INVALID_IDENTIFIERS = ["", "a" * 64, "1abc", "a b", "ta ble", "kb_id;drop"]+++ "identifier",+ INVALID_IDENTIFIERS,+ ids=["empty", "overlong", "numeric-prefix", "embedded-space", "table-space", "sql-punctuation"],+)+def test_tc_wrt_008_index_name_rejects_invalid_identifiers(identifier):+ builder = GaussDBDDLBuilder(schema="test_schema")++ with pytest.raises(InvalidGaussDBObjectName) as exc_info:+ builder.index_name(identifier, "chunk")++ assert exc_info.type is InvalidGaussDBObjectName+ assert exc_info.value.args == (identifier,)+ assert str(exc_info.value) == identifier+++ "identifier",+ INVALID_IDENTIFIERS,+ ids=["empty", "overlong", "numeric-prefix", "embedded-space", "table-space", "sql-punctuation"],+)+def test_tc_wrt_902_validate_identifier_rejects_invalid_identifiers(identifier):+ builder = GaussDBDDLBuilder(schema="test_schema")++ with pytest.raises(InvalidGaussDBObjectName) as exc_info:+ builder.validate_identifier(identifier)++ assert exc_info.type is InvalidGaussDBObjectName+ assert exc_info.value.args == (identifier,)+ assert str(exc_info.value) == identifier+++def test_tc_wrt_903_chunk_table_ddl_matches_normalized_snapshot():+ builder = GaussDBDDLBuilder(schema="test_schema")++ ddl = builder.build_chunk_table_ddl("ragflow_unit")++ expected = """CREATE TABLE IF NOT EXISTS "test_schema"."ragflow_unit" (+ id VARCHAR(256) NOT NULL,+ kb_id VARCHAR(256) NOT NULL,+ doc_id VARCHAR(256),+ docnm_kwd VARCHAR(256),+ doc_type_kwd VARCHAR(256),+ title_tks VARCHAR(256),+ title_sm_tks VARCHAR(256),+ content_with_weight TEXT,+ content_ltks TEXT,+ content_sm_ltks TEXT,+ important_kwd JSONB,+ important_tks TEXT,+ question_kwd JSONB,+ question_tks TEXT,+ tag_kwd JSONB,+ tag_feas JSONB,+ available_int INTEGER DEFAULT 1 NOT NULL,+ pagerank_fea INTEGER,+ create_time VARCHAR(19),+ create_timestamp_flt DOUBLE PRECISION,+ img_id VARCHAR(128),+ position_int JSONB,+ page_num_int JSONB,+ top_int JSONB,+ metadata JSONB,+ chunk_data JSONB,+ extra JSONB,+ _order_id INTEGER,+ group_id VARCHAR(256),+ mom_id VARCHAR(256),+ knowledge_graph_kwd VARCHAR(256),+ source_id JSONB,+ entity_kwd VARCHAR(256),+ entity_type_kwd VARCHAR(256),+ from_entity_kwd VARCHAR(256),+ to_entity_kwd VARCHAR(256),+ weight_int INTEGER,+ weight_flt DOUBLE PRECISION,+ entities_kwd JSONB,+ rank_flt DOUBLE PRECISION,+ n_hop_with_weight TEXT,+ removed_kwd VARCHAR(256) DEFAULT 'N',+ raptor_kwd VARCHAR(256),+ raptor_layer_int INTEGER,+ PRIMARY KEY (kb_id, id)+ ) WITH (storage_type=USTORE)"""+ assert _normalize_ddl(ddl) == _normalize_ddl(expected)+++def test_tc_wrt_904_vector_column_ddls_match_exact_snapshot():+ builder = GaussDBDDLBuilder(schema="test_schema")++ statements = builder.build_vector_column_ddls("ragflow_unit", 1024)++ assert statements == [+ 'ALTER TABLE "test_schema"."ragflow_unit" ADD COLUMN IF NOT EXISTS q_1024_vec floatvector(1024) DEFAULT (array_fill(0, ARRAY[1024])::text::floatvector(1024))',+ 'ALTER TABLE "test_schema"."ragflow_unit" ADD COLUMN IF NOT EXISTS q_1024_vec_valid BOOLEAN DEFAULT FALSE NOT NULL',+ ]+++ ("dim", "diskann_options"),+ [+ (1, "subgraph_count=1"),+ (4096, "subgraph_count=1, enable_vector_copy=false"),+ ],+ ids=["minimum", "maximum"],+)+def test_tc_wrt_004_validate_vector_dim_accepts_gaussdb_boundaries(dim, diskann_options):+ builder = GaussDBDDLBuilder(schema="test_schema")++ assert builder.validate_vector_dim(dim) == dim+ assert builder.vector_column_name(dim) == f"q_{dim}_vec"+ assert builder.vector_valid_column_name(dim) == f"q_{dim}_vec_valid"+ assert builder.build_vector_column_ddls("ragflow_t1", dim) == [+ f'ALTER TABLE "test_schema"."ragflow_t1" ADD COLUMN IF NOT EXISTS q_{dim}_vec floatvector({dim}) DEFAULT (array_fill(0, ARRAY[{dim}])::text::floatvector({dim}))',+ f'ALTER TABLE "test_schema"."ragflow_t1" ADD COLUMN IF NOT EXISTS q_{dim}_vec_valid BOOLEAN DEFAULT FALSE NOT NULL',+ ]+ assert builder.build_diskann_index_ddl("ragflow_t1", dim) == (+ f'CREATE INDEX IF NOT EXISTS "idx_gdb_ragflow_t1_q_{dim}_vec_diskann" ON "test_schema"."ragflow_t1" USING gsdiskann (q_{dim}_vec COSINE) WITH ({diskann_options})'+ )++[email protected]("dim", [0, -1], ids=["zero", "negative"])+def test_tc_wrt_005_validate_vector_dim_rejects_non_positive_dimensions(dim):+ builder = GaussDBDDLBuilder(schema="public")++ with pytest.raises(ValueError) as exc_info:+ builder.validate_vector_dim(dim)++ assert str(exc_info.value) == "vector dimension must be positive"+++def test_tc_wrt_006_vector_size_above_gaussdb_limit_is_rejected():+ builder = GaussDBDDLBuilder(schema="public")++ with pytest.raises(ValueError) as exc_info:+ builder.validate_vector_dim(4097)++ assert str(exc_info.value) == "GaussDB floatvector dimensions cannot exceed 4096"+++ "dim",+ [+ "not-int",+ 1024.5,+ ],+ ids=["non-numeric", "float"],+)+def test_tc_wrt_007_non_integer_vector_size_raises_exact_error(dim):+ builder = GaussDBDDLBuilder(schema="test_schema")++ with pytest.raises(ValueError) as exc_info:+ builder.validate_vector_dim(dim)++ assert str(exc_info.value) == "vector dimension must be an integer"+++ ("dim", "expected_options"),+ [+ (768, "subgraph_count=1"),+ (1024, "subgraph_count=1"),+ (1025, "subgraph_count=1, enable_vector_copy=false"),+ (3072, "subgraph_count=1, enable_vector_copy=false"),+ ],+ ids=["below-threshold", "threshold", "above-threshold", "high-dimension"],+)+def test_tc_wrt_009_diskann_dimension_matrix_matches_exact_ddl(dim, expected_options):+ builder = GaussDBDDLBuilder(schema="public")++ ddl = builder.build_diskann_index_ddl("ragflow_tenant_a", dim)++ assert ddl == (f'CREATE INDEX IF NOT EXISTS "idx_gdb_ragflow_tenant_a_q_{dim}_vec_diskann" ON "public"."ragflow_tenant_a" USING gsdiskann (q_{dim}_vec COSINE) WITH ({expected_options})')+++def test_tc_wrt_909_advisory_lock_sql_is_exact_and_parameterized():+ builder = GaussDBDDLBuilder(schema="test_schema")++ sql, params = builder.build_advisory_lock_sql("create_idx:schema:table")++ assert sql == "SELECT pg_advisory_xact_lock(hashtext(%s))"+ assert params == ["create_idx:schema:table"]+++def test_tc_wrt_911_long_table_identifier_produces_bounded_distinct_index_names():+ builder = GaussDBDDLBuilder(schema="test_schema")+ table = "ragflow_" + "a" * 32+ index_columns = ("doc_id", "available_int", "knowledge_graph_kwd", "entity_type_kwd", "removed_kwd")+ suffixes = (*index_columns, "q_3072_vec_diskann")++ expected_names = [+ "idx_gdb_ragflow_aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa_doc_id",+ "idx_gdb_ragflow_aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa_available_int",+ "idx_gdb_ragflow_aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa_kno_79293b137f",+ "idx_gdb_ragflow_aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa_ent_2da2f5b85e",+ "idx_gdb_ragflow_aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa_removed_kwd",+ "idx_gdb_ragflow_aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa_q_3_83afb848d3",+ ]+ names = [builder.index_name(table, suffix) for suffix in suffixes]+ repeated_names = [builder.index_name(table, suffix) for suffix in suffixes]+ regular_ddls = builder.build_regular_index_ddls(table)+ diskann_ddl = builder.build_diskann_index_ddl(table, 3072)++ assert names == expected_names+ assert repeated_names == expected_names+ assert len(set(names)) == len(suffixes)+ assert all(len(name) <= 63 for name in names)+ assert all(name.startswith("idx_gdb_ragflow_") for name in names)+ assert len(regular_ddls) == 5+ for column, index_name, statement in zip(index_columns, expected_names[:5], regular_ddls, strict=True):+ assert statement == (f'CREATE INDEX IF NOT EXISTS "{index_name}" ON "test_schema"."{table}" ({column})')+ assert diskann_ddl == (f'CREATE INDEX IF NOT EXISTS "{expected_names[-1]}" ON "test_schema"."{table}" USING gsdiskann (q_3072_vec COSINE) WITH (subgraph_count=1, enable_vector_copy=false)')+++def test_tc_wrt_912_doc_meta_table_ddl_uses_id_pk_ustore_and_kb_index():+ builder = GaussDBDDLBuilder(schema="public")++ statements = builder.build_doc_meta_table_ddls("ragflow_doc_meta_tenant_a")++ assert len(statements) == 2+ assert [_normalize_ddl(statement) for statement in statements] == [+ _normalize_ddl(+ """CREATE TABLE IF NOT EXISTS "public"."ragflow_doc_meta_tenant_a" (+ id VARCHAR(256) NOT NULL,+ kb_id VARCHAR(256) NOT NULL,+ meta_fields JSONB,+ PRIMARY KEY (id)+ ) WITH (storage_type=USTORE)"""+ ),+ 'CREATE INDEX IF NOT EXISTS "idx_gdb_ragflow_doc_meta_tenant_a_kb_id" ON "public"."ragflow_doc_meta_tenant_a" (kb_id)',+ ]+ assert "PRIMARY KEY (kb_id, id)" not in _normalize_ddl(statements[0])+++def test_tc_wrt_913_regular_index_ddls_match_exact_column_matrix():+ builder = GaussDBDDLBuilder(schema="public")++ statements = builder.build_regular_index_ddls("ragflow_tenant_a")++ assert statements == [+ f'CREATE INDEX IF NOT EXISTS "idx_gdb_ragflow_tenant_a_{column}" ON "public"."ragflow_tenant_a" ({column})'+ for column in ("doc_id", "available_int", "knowledge_graph_kwd", "entity_type_kwd", "removed_kwd")+ ]+++def test_tc_wrt_914_fulltext_ugin_ddl_matches_normalized_snapshot():+ builder = GaussDBDDLBuilder(schema="public")++ simple_ddl = builder.build_fulltext_ugin_ddl("ragflow_tenant_a")+ ngram_ddl = builder.build_ngram_fulltext_ugin_ddl("ragflow_tenant_a")++ expected_simple = """CREATE INDEX IF NOT EXISTS "idx_gdb_ragflow_tenant_a_fts_all"+ ON "public"."ragflow_tenant_a"+ USING ugin(to_tsvector('simple', coalesce(title_tks, ' ') || ' ' || coalesce(title_sm_tks, ' ') || ' ' ||+ coalesce(important_tks, ' ') || ' ' || coalesce(question_tks, ' ') || ' ' || coalesce(content_ltks, ' ') || ' ' ||+ coalesce(content_sm_ltks, ' ')))"""+ expected_ngram = """CREATE INDEX IF NOT EXISTS "idx_gdb_ragflow_tenant_a_fts_all_ngram"+ ON "public"."ragflow_tenant_a"+ USING ugin(to_tsvector('ngram', coalesce(title_tks, ' ') || ' ' || coalesce(title_sm_tks, ' ') || ' ' ||+ coalesce(important_tks, ' ') || ' ' || coalesce(question_tks, ' ') || ' ' || coalesce(content_ltks, ' ') || ' ' ||+ coalesce(content_sm_ltks, ' ')))"""+ assert _normalize_ddl(simple_ddl) == _normalize_ddl(expected_simple)+ assert _normalize_ddl(ngram_ddl) == _normalize_ddl(expected_ngram)diff --git a/test/unit_test/rag/utils/test_gaussdb_metadata_filter.py b/test/unit_test/rag/utils/test_gaussdb_metadata_filter.pynew file mode 100644index 0000000000..499d78b01a--- /dev/null+++ b/test/unit_test/rag/utils/test_gaussdb_metadata_filter.py@@ -0,0 +1,1155 @@+#+# Copyright 2025 The InfiniFlow Authors. All Rights Reserved.+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+#+import pytest++from common import metadata_gaussdb_filter as gaussdb_filter+from common.metadata_gaussdb_filter import (+ GaussDBMetaFilterTranslator,+ SUPPORTED_OPERATORS,+ UnsupportedGaussDBMetaFilter,+ build_gaussdb_filter,+ fetch_gaussdb_metadata_doc_ids,+ is_pushdown_supported,+ jsonb_path_literal,+ normalize_gaussdb_meta_operator,+ plan_pushdown,+)++OP_NE = "\u2260"+OP_GE = "\u2265"+OP_LE = "\u2264"+++def _assert_filter(filters, logic, expected_sql, expected_params):+ sql, params = build_gaussdb_filter(filters, logic)++ assert sql == expected_sql+ assert params == expected_params+ return sql+++def _assert_rejected_filter(flt, expected_reason, logic="and", hidden_inputs=()):+ with pytest.raises(UnsupportedGaussDBMetaFilter) as exc_info:+ build_gaussdb_filter([flt], logic)++ assert exc_info.value.reason == expected_reason+ for hidden_input in hidden_inputs:+ assert hidden_input not in exc_info.value.reason+++def test_tc_mgf_002_multiple_filters_join_with_and():+ sql = _assert_filter(+ [+ {"key": "status", "op": "=", "value": "active"},+ {"key": "amount", "op": ">", "value": 100},+ ],+ "and",+ "(lower(meta_fields #>> '{status}') = %s OR jsonb_exists(meta_fields #> '{status}', %s)) AND "+ "(CASE WHEN meta_fields #>> '{amount}' ~ '^-?[0-9]+(\\.[0-9]+)?$' "+ "THEN (meta_fields #>> '{amount}')::DOUBLE PRECISION > %s ELSE FALSE END)",+ ["active", "active", 100],+ )++ assert "active" not in sql+ assert "100" not in sql+++def test_tc_mgf_003_multiple_filters_join_with_or():+ _assert_filter(+ [+ {"key": "status", "op": "=", "value": "a"},+ {"key": "status", "op": "=", "value": "b"},+ ],+ "or",+ "(lower(meta_fields #>> '{status}') = %s OR jsonb_exists(meta_fields #> '{status}', %s)) OR (lower(meta_fields #>> '{status}') = %s OR jsonb_exists(meta_fields #> '{status}', %s))",+ ["a", "a", "b", "b"],+ )+++def test_tc_mgf_004_empty_filter_list_returns_tautology():+ _assert_filter([], "and", "1=1", [])+++def test_tc_mgf_005_nested_key_path_uses_jsonb_path_literal():+ sql = _assert_filter(+ [{"key": "profile.name", "op": "=", "value": "Alice"}],+ "and",+ "(lower(meta_fields #>> '{profile,name}') = %s OR jsonb_exists(meta_fields #> '{profile,name}', %s))",+ ["alice", "alice"],+ )++ assert jsonb_path_literal("profile.name") == "'{profile,name}'"+ assert "Alice" not in sql+ assert "alice" not in sql+++def test_tc_mgf_006_custom_jsonb_column_is_used_for_translation():+ translated = GaussDBMetaFilterTranslator(jsonb_column="chunk_data").translate({"key": "amount", "op": "=", "value": 5})++ assert translated.sql == "chunk_data #> '{amount}' @> %s::jsonb"+ assert translated.params == ["5"]+ assert "meta_fields" not in translated.sql+++def test_tc_mgf_101_equal_string_translates_to_scalar_or_array_match():+ sql = _assert_filter(+ [{"key": "status", "op": "=", "value": "active"}],+ "and",+ "(lower(meta_fields #>> '{status}') = %s OR jsonb_exists(meta_fields #> '{status}', %s))",+ ["active", "active"],+ )++ assert "active" not in sql+++def test_tc_mgf_102_not_equal_string_requires_existing_key_and_negated_match():+ sql = _assert_filter(+ [{"key": "status", "op": OP_NE, "value": "active"}],+ "and",+ "((meta_fields #> '{status}') IS NOT NULL AND (lower(meta_fields #>> '{status}') = %s OR jsonb_exists(meta_fields #> '{status}', %s)) IS NOT TRUE)",+ ["active", "active"],+ )++ assert "active" not in sql+++def test_tc_mgf_103_greater_than_number_uses_numeric_cast():+ sql = _assert_filter(+ [{"key": "amount", "op": ">", "value": 100}],+ "and",+ "(CASE WHEN meta_fields #>> '{amount}' ~ '^-?[0-9]+(\\.[0-9]+)?$' THEN (meta_fields #>> '{amount}')::DOUBLE PRECISION > %s ELSE FALSE END)",+ [100],+ )++ assert "100" not in sql+++def test_tc_mgf_104_numeric_comparisons_use_canonical_sql_operators():+ cases = [+ (OP_GE, ">="),+ ("<", "<"),+ (OP_LE, "<="),+ ]++ for filter_op, sql_op in cases:+ sql = _assert_filter(+ [{"key": "amount", "op": filter_op, "value": 50}],+ "and",+ f"(CASE WHEN meta_fields #>> '{{amount}}' ~ '^-?[0-9]+(\\.[0-9]+)?$' THEN (meta_fields #>> '{{amount}}')::DOUBLE PRECISION {sql_op} %s ELSE FALSE END)",+ [50],+ )+ assert "50" not in sql+++def test_tc_mgf_105_date_iso_prefix_uses_exact_timestamp_predicate():+ sql = _assert_filter(+ [{"key": "dt", "op": ">", "value": "2026-07"}],+ "and",+ "(CASE "+ "WHEN meta_fields #>> '{dt}' ~ '^[0-9]{4}-[0-9]{2}$' "+ "THEN to_timestamp(meta_fields #>> '{dt}' || '-01 00:00:00', 'YYYY-MM-DD HH24:MI:SS') "+ "WHEN meta_fields #>> '{dt}' ~ '^[0-9]{4}-[0-9]{2}-[0-9]{2}$' "+ "THEN to_timestamp(meta_fields #>> '{dt}' || ' 00:00:00', 'YYYY-MM-DD HH24:MI:SS') "+ "WHEN meta_fields #>> '{dt}' ~ '^[0-9]{4}-[0-9]{2}-[0-9]{2}[T ][0-9]{2}$' "+ "THEN to_timestamp(replace(meta_fields #>> '{dt}', 'T', ' ') || ':00:00', 'YYYY-MM-DD HH24:MI:SS') "+ "WHEN meta_fields #>> '{dt}' ~ '^[0-9]{4}-[0-9]{2}-[0-9]{2}[T ][0-9]{2}:[0-9]{2}$' "+ "THEN to_timestamp(replace(meta_fields #>> '{dt}', 'T', ' ') || ':00', 'YYYY-MM-DD HH24:MI:SS') "+ "WHEN meta_fields #>> '{dt}' ~ '^[0-9]{4}-[0-9]{2}-[0-9]{2}[T ][0-9]{2}:[0-9]{2}:[0-9]{2}$' "+ "THEN to_timestamp(replace(meta_fields #>> '{dt}', 'T', ' '), 'YYYY-MM-DD HH24:MI:SS') "+ "END > to_timestamp(%s, 'YYYY-MM-DD HH24:MI:SS'))",+ ["2026-07-01 00:00:00"],+ )++ assert gaussdb_filter.normalize_datetime_prefix("2026-07") == "2026-07-01 00:00:00"+ assert "2026-07" not in sql+++def test_tc_mgf_106_contains_string_matches_array_member_or_substring():+ sql = _assert_filter(+ [{"key": "cust", "op": "contains", "value": "Acme"}],+ "and",+ "(jsonb_exists(meta_fields #> '{cust}', %s) OR lower(meta_fields #>> '{cust}') LIKE %s ESCAPE '\\')",+ ["acme", "%acme%"],+ )++ assert "Acme" not in sql+ assert "acme" not in sql+++def test_tc_mgf_107_not_contains_requires_existing_key_and_negated_contains():+ sql = _assert_filter(+ [{"key": "cust", "op": "not contains", "value": "Acme"}],+ "and",+ "((meta_fields #> '{cust}') IS NOT NULL AND (jsonb_exists(meta_fields #> '{cust}', %s) OR lower(meta_fields #>> '{cust}') LIKE %s ESCAPE '\\') IS NOT TRUE)",+ ["acme", "%acme%"],+ )++ assert "Acme" not in sql+ assert "acme" not in sql+++def test_tc_mgf_108_start_with_uses_escaped_like_prefix():+ sql = _assert_filter(+ [{"key": "cust", "op": "start with", "value": "Acme"}],+ "and",+ "(lower(meta_fields #>> '{cust}') LIKE %s ESCAPE '\\')",+ ["acme%"],+ )++ assert "Acme" not in sql+ assert "acme" not in sql+++def test_tc_mgf_109_end_with_uses_escaped_like_suffix():+ sql = _assert_filter(+ [{"key": "cust", "op": "end with", "value": "Corp"}],+ "and",+ "(lower(meta_fields #>> '{cust}') LIKE %s ESCAPE '\\')",+ ["%corp"],+ )++ assert "Corp" not in sql+ assert "corp" not in sql+++def test_tc_mgf_110_in_operator_expands_members_with_or():+ _assert_filter(+ [{"key": "status", "op": "in", "value": ["a", "b"]}],+ "and",+ "((lower(meta_fields #>> '{status}') = %s OR jsonb_exists(meta_fields #> '{status}', %s)) OR (lower(meta_fields #>> '{status}') = %s OR jsonb_exists(meta_fields #> '{status}', %s)))",+ ["a", "a", "b", "b"],+ )+++def test_tc_mgf_111_not_in_operator_requires_key_and_ands_members():+ _assert_filter(+ [{"key": "status", "op": "not in", "value": ["a", "b"]}],+ "and",+ "((meta_fields #> '{status}') IS NOT NULL AND "+ "(lower(meta_fields #>> '{status}') = %s OR jsonb_exists(meta_fields #> '{status}', %s)) IS NOT TRUE AND "+ "(lower(meta_fields #>> '{status}') = %s OR jsonb_exists(meta_fields #> '{status}', %s)) IS NOT TRUE)",+ ["a", "a", "b", "b"],+ )+++def test_tc_mgf_112_empty_operator_covers_all_empty_states_and_nested_missing_parent():+ status_sql = _assert_filter(+ [{"key": "status", "op": "empty", "value": None}],+ "and",+ "((meta_fields #> '{status}') IS NULL OR meta_fields #> '{status}' = 'null'::jsonb OR "+ "meta_fields #> '{status}' = '\"\"'::jsonb OR meta_fields #> '{status}' = '[]'::jsonb OR "+ "meta_fields #> '{status}' = '{}'::jsonb)",+ [],+ )+ nested_sql = _assert_filter(+ [{"key": "vendor.name", "op": "empty", "value": None}],+ "and",+ "((meta_fields #> '{vendor,name}') IS NULL OR "+ "meta_fields #> '{vendor,name}' = 'null'::jsonb OR "+ "meta_fields #> '{vendor,name}' = '\"\"'::jsonb OR "+ "meta_fields #> '{vendor,name}' = '[]'::jsonb OR "+ "meta_fields #> '{vendor,name}' = '{}'::jsonb)",+ [],+ )+ for sql in (status_sql, nested_sql):+ assert "= ''" not in sql+ assert "<> ''" not in sql+++def test_tc_mgf_113_not_empty_operator_excludes_all_empty_states():+ sql = _assert_filter(+ [{"key": "status", "op": "not empty", "value": None}],+ "and",+ "((meta_fields #> '{status}') IS NOT NULL AND (meta_fields #> '{status}' = 'null'::jsonb) IS NOT TRUE AND "+ "(meta_fields #> '{status}' = '\"\"'::jsonb) IS NOT TRUE AND "+ "(meta_fields #> '{status}' = '[]'::jsonb) IS NOT TRUE AND "+ "(meta_fields #> '{status}' = '{}'::jsonb) IS NOT TRUE)",+ [],+ )+ assert "= ''" not in sql+ assert "<> ''" not in sql+++def test_tc_mgf_201_equal_none_generates_json_null_equality():+ _assert_filter(+ [{"key": "status", "op": "=", "value": None}],+ "and",+ "((meta_fields #> '{status}') IS NOT NULL AND meta_fields #> '{status}' = 'null'::jsonb)",+ [],+ )+ assert gaussdb_filter.coerce_scalar_value(None, {"key": "status", "op": "=", "value": None}) is None+++def test_tc_mgf_202_equal_empty_string_uses_jsonb_empty_string_literal():+ sql = _assert_filter(+ [{"key": "status", "op": "=", "value": ""}],+ "and",+ "(meta_fields #> '{status}' = '\"\"'::jsonb)",+ [],+ )++ assert "= ''" not in sql+++def test_tc_mgf_301_equal_integer_uses_jsonb_containment():+ sql = _assert_filter(+ [{"key": "amount", "op": "=", "value": 5}],+ "and",+ "(meta_fields #> '{amount}' @> %s::jsonb)",+ ["5"],+ )++ assert gaussdb_filter.coerce_scalar_value(5, {"key": "amount", "op": "=", "value": 5}) == 5+ assert gaussdb_filter.jsonb_param(5) == "5"+ assert "5" not in sql+++def test_tc_mgf_302_equal_boolean_uses_jsonb_containment():+ sql = _assert_filter(+ [{"key": "flag", "op": "=", "value": True}],+ "and",+ "(meta_fields #> '{flag}' @> %s::jsonb)",+ ["true"],+ )++ assert "true" not in sql+ assert "True" not in sql+++def test_tc_mgf_303_equal_numeric_string_uses_jsonb_numeric_containment():+ flt = {"key": "n", "op": "=", "value": "123"}+ sql = _assert_filter(+ [flt],+ "and",+ "(meta_fields #> '{n}' @> %s::jsonb)",+ ["123"],+ )++ assert gaussdb_filter.coerce_scalar_value("123", flt) == 123+ assert "123" not in sql+++def test_tc_mgf_304_equal_python_literals_differ_from_lowercase_json_words():+ cases = [+ ("True", True, "(meta_fields #> '{n}' @> %s::jsonb)", ["true"]),+ ("False", False, "(meta_fields #> '{n}' @> %s::jsonb)", ["false"]),+ (+ "None",+ None,+ "((meta_fields #> '{n}') IS NOT NULL AND meta_fields #> '{n}' = 'null'::jsonb)",+ [],+ ),+ (+ "true",+ "true",+ "(lower(meta_fields #>> '{n}') = %s OR jsonb_exists(meta_fields #> '{n}', %s))",+ ["true", "true"],+ ),+ (+ "null",+ "null",+ "(lower(meta_fields #>> '{n}') = %s OR jsonb_exists(meta_fields #> '{n}', %s))",+ ["null", "null"],+ ),+ ]++ for value, expected_value, expected_sql, expected_params in cases:+ flt = {"key": "n", "op": "=", "value": value}+ sql = _assert_filter([flt], "and", expected_sql, expected_params)+ coerced = gaussdb_filter.coerce_scalar_value(value, flt)++ if isinstance(expected_value, str):+ assert coerced == expected_value+ else:+ assert coerced is expected_value+ assert value not in sql+++def test_tc_mgf_305_range_accepts_negative_number():+ sql = _assert_filter(+ [{"key": "amount", "op": ">", "value": -10}],+ "and",+ "(CASE WHEN meta_fields #>> '{amount}' ~ '^-?[0-9]+(\\.[0-9]+)?$' THEN (meta_fields #>> '{amount}')::DOUBLE PRECISION > %s ELSE FALSE END)",+ [-10],+ )++ assert "-10" not in sql+++def test_tc_mgf_306_range_accepts_decimal_number():+ sql = _assert_filter(+ [{"key": "amount", "op": ">", "value": 1.5}],+ "and",+ "(CASE WHEN meta_fields #>> '{amount}' ~ '^-?[0-9]+(\\.[0-9]+)?$' THEN (meta_fields #>> '{amount}')::DOUBLE PRECISION > %s ELSE FALSE END)",+ [1.5],+ )++ assert "1.5" not in sql+++def test_tc_mgf_307_range_numeric_strings_stay_on_numeric_branch():+ for value, expected_param in (("100", 100), ("2026", 2026), ("01", 1), ("01.5", 1.5)):+ flt = {"key": "amount", "op": ">", "value": value}+ sql = _assert_filter(+ [flt],+ "and",+ "(CASE WHEN meta_fields #>> '{amount}' ~ '^-?[0-9]+(\\.[0-9]+)?$' THEN (meta_fields #>> '{amount}')::DOUBLE PRECISION > %s ELSE FALSE END)",+ [expected_param],+ )+ assert gaussdb_filter.coerce_range_value(value, flt) == ("number", expected_param)+ assert "to_timestamp" not in sql+ assert value not in sql+++def test_tc_mgf_308_date_iso_prefixes_normalize_to_exact_timestamp_parameters():+ cases = [+ ("2026-07", "2026-07-01 00:00:00"),+ ("2026-07-08", "2026-07-08 00:00:00"),+ ("2026-07-08T10", "2026-07-08 10:00:00"),+ ("2026-07-08 10", "2026-07-08 10:00:00"),+ ("2026-07-08T10:30", "2026-07-08 10:30:00"),+ ("2026-07-08 10:30:45", "2026-07-08 10:30:45"),+ ]++ for value, expected_param in cases:+ _assert_filter(+ [{"key": "published_at", "op": ">", "value": value}],+ "and",+ "(CASE "+ "WHEN meta_fields #>> '{published_at}' ~ '^[0-9]{4}-[0-9]{2}$' "+ "THEN to_timestamp(meta_fields #>> '{published_at}' || '-01 00:00:00', 'YYYY-MM-DD HH24:MI:SS') "+ "WHEN meta_fields #>> '{published_at}' ~ '^[0-9]{4}-[0-9]{2}-[0-9]{2}$' "+ "THEN to_timestamp(meta_fields #>> '{published_at}' || ' 00:00:00', 'YYYY-MM-DD HH24:MI:SS') "+ "WHEN meta_fields #>> '{published_at}' ~ '^[0-9]{4}-[0-9]{2}-[0-9]{2}[T ][0-9]{2}$' "+ "THEN to_timestamp(replace(meta_fields #>> '{published_at}', 'T', ' ') || ':00:00', 'YYYY-MM-DD HH24:MI:SS') "+ "WHEN meta_fields #>> '{published_at}' ~ '^[0-9]{4}-[0-9]{2}-[0-9]{2}[T ][0-9]{2}:[0-9]{2}$' "+ "THEN to_timestamp(replace(meta_fields #>> '{published_at}', 'T', ' ') || ':00', 'YYYY-MM-DD HH24:MI:SS') "+ "WHEN meta_fields #>> '{published_at}' ~ '^[0-9]{4}-[0-9]{2}-[0-9]{2}[T ][0-9]{2}:[0-9]{2}:[0-9]{2}$' "+ "THEN to_timestamp(replace(meta_fields #>> '{published_at}', 'T', ' '), 'YYYY-MM-DD HH24:MI:SS') "+ "END > to_timestamp(%s, 'YYYY-MM-DD HH24:MI:SS'))",+ [expected_param],+ )+ assert gaussdb_filter.normalize_datetime_prefix(value) == expected_param+++def test_tc_mgf_309_invalid_datetime_prefix_is_rejected_as_range_value():+ flt = {"key": "dt", "op": ">", "value": "2026-13-01"}++ _assert_rejected_filter(+ flt,+ "unsupported range comparison value",+ hidden_inputs=("2026-13-01",),+ )+++def test_tc_mgf_310_in_operator_splits_comma_separated_string_members():+ sql = _assert_filter(+ [{"key": "status", "op": "in", "value": "a,b,c"}],+ "and",+ "((lower(meta_fields #>> '{status}') = %s OR jsonb_exists(meta_fields #> '{status}', %s)) OR "+ "(lower(meta_fields #>> '{status}') = %s OR jsonb_exists(meta_fields #> '{status}', %s)) OR "+ "(lower(meta_fields #>> '{status}') = %s OR jsonb_exists(meta_fields #> '{status}', %s)))",+ ["a", "a", "b", "b", "c", "c"],+ )++ assert "a,b,c" not in sql+++def test_tc_mgf_311_in_operator_parses_json_array_string_members():+ sql = _assert_filter(+ [{"key": "n", "op": "in", "value": "[1,2]"}],+ "and",+ "((meta_fields #> '{n}' @> %s::jsonb) OR (meta_fields #> '{n}' @> %s::jsonb))",+ ["1", "2"],+ )++ assert "[1,2]" not in sql+++def test_tc_mgf_312_equal_decimal_string_uses_jsonb_numeric_containment():+ sql = _assert_filter(+ [{"key": "n", "op": "=", "value": "1.5"}],+ "and",+ "(meta_fields #> '{n}' @> %s::jsonb)",+ ["1.5"],+ )++ assert "1.5" not in sql+++def test_tc_mgf_313_in_operator_accepts_single_scalar_member():+ sql = _assert_filter(+ [{"key": "n", "op": "in", "value": 7}],+ "and",+ "((meta_fields #> '{n}' @> %s::jsonb))",+ ["7"],+ )++ assert sql.count("@> %s::jsonb") == 1+ assert "7" not in sql+++def test_tc_mgf_401_and_logic_joins_three_mixed_operators_in_order():+ sql = _assert_filter(+ [+ {"key": "status", "op": "=", "value": "active"},+ {"key": "amount", "op": ">", "value": 100},+ {"key": "cust", "op": "contains", "value": "acme"},+ ],+ "and",+ "(lower(meta_fields #>> '{status}') = %s OR jsonb_exists(meta_fields #> '{status}', %s)) AND "+ "(CASE WHEN meta_fields #>> '{amount}' ~ '^-?[0-9]+(\\.[0-9]+)?$' "+ "THEN (meta_fields #>> '{amount}')::DOUBLE PRECISION > %s ELSE FALSE END) AND "+ "(jsonb_exists(meta_fields #> '{cust}', %s) OR "+ "lower(meta_fields #>> '{cust}') LIKE %s ESCAPE '\\')",+ ["active", "active", 100, "acme", "%acme%"],+ )++ assert "active" not in sql+ assert "100" not in sql+ assert "acme" not in sql+++def test_tc_mgf_402_or_logic_joins_three_filters_in_order():+ _assert_filter(+ [+ {"key": "status", "op": "=", "value": "a"},+ {"key": "status", "op": "=", "value": "b"},+ {"key": "status", "op": "=", "value": "c"},+ ],+ "or",+ "(lower(meta_fields #>> '{status}') = %s OR jsonb_exists(meta_fields #> '{status}', %s)) OR "+ "(lower(meta_fields #>> '{status}') = %s OR jsonb_exists(meta_fields #> '{status}', %s)) OR "+ "(lower(meta_fields #>> '{status}') = %s OR jsonb_exists(meta_fields #> '{status}', %s))",+ ["a", "a", "b", "b", "c", "c"],+ )+++def test_tc_mgf_403_and_logic_combines_not_in_with_positive_filter():+ _assert_filter(+ [+ {"key": "status", "op": "not in", "value": ["a", "b"]},+ {"key": "type", "op": "=", "value": "doc"},+ ],+ "and",+ "((meta_fields #> '{status}') IS NOT NULL AND "+ "(lower(meta_fields #>> '{status}') = %s OR jsonb_exists(meta_fields #> '{status}', %s)) IS NOT TRUE AND "+ "(lower(meta_fields #>> '{status}') = %s OR jsonb_exists(meta_fields #> '{status}', %s)) IS NOT TRUE) AND "+ "(lower(meta_fields #>> '{type}') = %s OR jsonb_exists(meta_fields #> '{type}', %s))",+ ["a", "a", "b", "b", "doc", "doc"],+ )+++def test_tc_mgf_404_nested_key_exists_and_jsonb_number_predicate_join_with_and():+ sql = _assert_filter(+ [+ {"key": "a.b", "op": OP_NE, "value": "x"},+ {"key": "a.c", "op": "=", "value": 1},+ ],+ "and",+ "((meta_fields #> '{a,b}') IS NOT NULL AND (lower(meta_fields #>> '{a,b}') = %s OR jsonb_exists(meta_fields #> '{a,b}', %s)) IS NOT TRUE) AND (meta_fields #> '{a,c}' @> %s::jsonb)",+ ["x", "x", "1"],+ )++ assert jsonb_path_literal("a.c") == "'{a,c}'"+ assert "'x'" not in sql+++def test_tc_mgf_405_special_key_segments_are_encoded_without_sql_injection():+ malicious_segment = "owner') OR TRUE --"+ key = f"root.a,b.{malicious_segment}"+ path = "'{root,\"a,b\",\"owner'') OR TRUE --\"}'"++ sql = _assert_filter(+ [{"key": key, "op": OP_NE, "value": "x"}],+ "and",+ f"((meta_fields #> {path}) IS NOT NULL AND (lower(meta_fields #>> {path}) = %s OR jsonb_exists(meta_fields #> {path}, %s)) IS NOT TRUE)",+ ["x", "x"],+ )++ assert jsonb_path_literal(key) == path+ assert malicious_segment not in sql+ assert sql.count(path) == 3+++def test_tc_mgf_501_is_alias_translates_to_equal_sql():+ sql = _assert_filter(+ [{"key": "status", "op": "is", "value": "active"}],+ "and",+ "(lower(meta_fields #>> '{status}') = %s OR jsonb_exists(meta_fields #> '{status}', %s))",+ ["active", "active"],+ )++ assert normalize_gaussdb_meta_operator("is") == "="+ assert "active" not in sql+++def test_tc_mgf_502_not_equal_aliases_translate_to_identical_sql():+ expected_sql = "((meta_fields #> '{status}') IS NOT NULL AND (lower(meta_fields #>> '{status}') = %s OR jsonb_exists(meta_fields #> '{status}', %s)) IS NOT TRUE)"++ for op in ("is not", "not is", "!=", "<>"):+ sql = _assert_filter(+ [{"key": "status", "op": op, "value": "active"}],+ "and",+ expected_sql,+ ["active", "active"],+ )+ assert normalize_gaussdb_meta_operator(op) == OP_NE+ assert "active" not in sql+++def test_tc_mgf_503_range_aliases_normalize_and_translate_to_canonical_operators():+ cases = [(">=", OP_GE, ">="), ("<=", OP_LE, "<=")]++ for alias, canonical, sql_op in cases:+ sql = _assert_filter(+ [{"key": "amount", "op": alias, "value": 50}],+ "and",+ f"(CASE WHEN meta_fields #>> '{{amount}}' ~ '^-?[0-9]+(\\.[0-9]+)?$' THEN (meta_fields #>> '{{amount}}')::DOUBLE PRECISION {sql_op} %s ELSE FALSE END)",+ [50],+ )+ assert normalize_gaussdb_meta_operator(alias) == canonical+ assert "50" not in sql+++def test_tc_mgf_504_operator_normalization_tolerates_case_and_whitespace():+ cases = [+ (+ " IS NOT ",+ "((meta_fields #> '{status}') IS NOT NULL AND (lower(meta_fields #>> '{status}') = %s OR jsonb_exists(meta_fields #> '{status}', %s)) IS NOT TRUE)",+ ["acme", "acme"],+ ),+ (+ "Contains",+ "(jsonb_exists(meta_fields #> '{status}', %s) OR lower(meta_fields #>> '{status}') LIKE %s ESCAPE '\\')",+ ["acme", "%acme%"],+ ),+ (+ "START WITH",+ "(lower(meta_fields #>> '{status}') LIKE %s ESCAPE '\\')",+ ["acme%"],+ ),+ ]++ for op, expected_sql, expected_params in cases:+ sql = _assert_filter(+ [{"key": "status", "op": op, "value": "acme"}],+ "and",+ expected_sql,+ expected_params,+ )+ assert "acme" not in sql+++def test_tc_mgf_505_before_after_operators_are_not_supported():+ for op in ("before", "after"):+ _assert_rejected_filter(+ {"key": "date", "op": op, "value": "2026-01-01"},+ f"unsupported metadata filter operator '{op}'",+ )+++def test_tc_mgf_601_missing_key_is_rejected_with_stable_reason():+ _assert_rejected_filter(+ {"op": "=", "value": "private-metadata-value"},+ "invalid metadata key",+ hidden_inputs=("private-metadata-value",),+ )+++def test_tc_mgf_602_missing_operator_is_rejected_with_exact_message():+ _assert_rejected_filter(+ {"key": "status", "value": "x"},+ "metadata filter operator is missing",+ )+++def test_tc_mgf_603_unknown_operator_is_rejected_with_exact_message():+ _assert_rejected_filter(+ {"key": "status", "op": "regex", "value": "x"},+ "unsupported metadata filter operator 'regex'",+ )+++def test_tc_mgf_604_numeric_leading_key_segment_is_encoded():+ _assert_filter(+ [{"key": "1abc", "op": "=", "value": 1}],+ "and",+ "(meta_fields #> '{1abc}' @> %s::jsonb)",+ ["1"],+ )+ assert jsonb_path_literal("1abc") == "'{1abc}'"+++def test_tc_mgf_605_empty_or_nul_key_segments_are_rejected():+ cases = [+ ("a..b", "metadata key contains an empty path segment"),+ ("a.", "metadata key contains an empty path segment"),+ (".a", "metadata key contains an empty path segment"),+ ("a.\x00b", "metadata key contains NUL"),+ ]++ for key, expected_reason in cases:+ _assert_rejected_filter(+ {"key": key, "op": "=", "value": "x"},+ expected_reason,+ hidden_inputs=(key,),+ )+++def test_tc_mgf_606_key_segment_with_space_is_encoded():+ path = "'{a,\"b c\"}'"+ _assert_filter(+ [{"key": "a.b c", "op": "=", "value": 1}],+ "and",+ f"(meta_fields #> {path} @> %s::jsonb)",+ ["1"],+ )+ assert jsonb_path_literal("a.b c") == path+++def test_tc_mgf_607_equal_operator_rejects_non_scalar_value():+ _assert_rejected_filter(+ {"key": "private-key", "op": "=", "value": ["private-value"]},+ "scalar comparison value is non-scalar",+ hidden_inputs=("private-key", "private-value"),+ )+++def test_tc_mgf_608_range_operator_rejects_none_value():+ _assert_rejected_filter(+ {"key": "private-key", "op": ">", "value": None},+ "range comparison value is None",+ hidden_inputs=("private-key",),+ )+++def test_tc_mgf_609_range_operator_rejects_boolean_value():+ _assert_rejected_filter(+ {"key": "private-key", "op": ">", "value": True},+ "range comparison value is boolean",+ hidden_inputs=("private-key",),+ )+++def test_tc_mgf_610_range_operator_rejects_non_numeric_non_date_string():+ _assert_rejected_filter(+ {"key": "private-key", "op": ">", "value": "private-range-value"},+ "unsupported range comparison value",+ hidden_inputs=("private-key", "private-range-value"),+ )+++def test_tc_mgf_611_contains_operator_rejects_none_and_container_values():+ for value in (None, ["private-value"], {"private-value": True}):+ _assert_rejected_filter(+ {"key": "private-key", "op": "contains", "value": value},+ "string operator value must be a scalar",+ hidden_inputs=("private-key", "private-value"),+ )+++def test_tc_mgf_612_contains_operator_rejects_empty_string():+ _assert_rejected_filter(+ {"key": "private-key", "op": "contains", "value": ""},+ "string operator value is empty",+ hidden_inputs=("private-key",),+ )+++def test_tc_mgf_613_in_operator_rejects_none_value():+ _assert_rejected_filter(+ {"key": "private-key", "op": "in", "value": None},+ "membership value is None",+ hidden_inputs=("private-key",),+ )+++def test_tc_mgf_614_in_operator_rejects_every_empty_resolved_member_list():+ for value in ([], "", ",,"):+ _assert_rejected_filter(+ {"key": "private-key", "op": "in", "value": value},+ "membership value resolved to empty list",+ hidden_inputs=("private-key",),+ )+++def test_tc_mgf_615_unknown_logic_is_rejected():+ with pytest.raises(UnsupportedGaussDBMetaFilter) as exc_info:+ build_gaussdb_filter([], "xor")++ assert str(exc_info.value) == "unknown logic 'xor'"+++def test_tc_mgf_616_invalid_jsonb_column_is_rejected():+ with pytest.raises(UnsupportedGaussDBMetaFilter) as exc_info:+ GaussDBMetaFilterTranslator(jsonb_column="chunk_data;drop")++ assert str(exc_info.value) == "invalid JSONB column 'chunk_data;drop'"+++def test_tc_mgf_617_value_with_sql_metacharacters_is_bound_and_like_escaped():+ value = r"x%_\'; DROP TABLE--"+ sql = _assert_filter(+ [{"key": "status", "op": "contains", "value": value}],+ "and",+ "(jsonb_exists(meta_fields #> '{status}', %s) OR lower(meta_fields #>> '{status}') LIKE %s ESCAPE '\\')",+ [r"x%_\'; drop table--", r"%x\%\_\\'; drop table--%"],+ )++ assert "DROP TABLE" not in sql+ assert "drop table" not in sql+ assert "x%_" not in sql+++def test_tc_mgf_618_non_ascii_and_at_sign_key_segments_are_encoded():+ cases = [+ ("字段", "'{\"字段\"}'"),+ ("field@name", "'{\"field@name\"}'"),+ ]++ for key, path in cases:+ _assert_filter(+ [{"key": key, "op": "=", "value": 1}],+ "and",+ f"(meta_fields #> {path} @> %s::jsonb)",+ ["1"],+ )+ assert jsonb_path_literal(key) == path+++def test_tc_mgf_701_is_pushdown_supported_returns_true_for_fully_translatable_filter():+ filters = [{"key": "a", "op": "=", "value": "x"}]++ assert is_pushdown_supported(filters) is True+ plan = plan_pushdown(filters, "and")+ assert plan.logic == "and"+ assert len(plan.translated) == 1+ assert plan.translated[0].sql == "lower(meta_fields #>> '{a}') = %s OR jsonb_exists(meta_fields #> '{a}', %s)"+ assert plan.translated[0].params == ["x", "x"]+ _assert_filter(+ filters,+ "and",+ "(lower(meta_fields #>> '{a}') = %s OR jsonb_exists(meta_fields #> '{a}', %s))",+ ["x", "x"],+ )+++def test_tc_mgf_702_is_pushdown_supported_returns_false_without_raising():+ filters = [{"key": "a", "op": "before", "value": "x"}]++ assert is_pushdown_supported(filters) is False+ _assert_rejected_filter(+ filters[0],+ "unsupported metadata filter operator 'before'",+ )+++def test_tc_mgf_704_all_supported_operators_normalize_and_translate():+ expected_operators = frozenset(+ {+ "=",+ "\u2260",+ ">",+ "\u2265",+ "<",+ "\u2264",+ "in",+ "not in",+ "contains",+ "not contains",+ "start with",+ "end with",+ "empty",+ "not empty",+ }+ )+ cases = [+ (+ "=",+ "status",+ "x",+ "lower(meta_fields #>> '{status}') = %s OR jsonb_exists(meta_fields #> '{status}', %s)",+ ["x", "x"],+ ),+ (+ OP_NE,+ "status",+ "x",+ "(meta_fields #> '{status}') IS NOT NULL AND (lower(meta_fields #>> '{status}') = %s OR jsonb_exists(meta_fields #> '{status}', %s)) IS NOT TRUE",+ ["x", "x"],+ ),+ (+ ">",+ "amount",+ 1,+ "CASE WHEN meta_fields #>> '{amount}' ~ '^-?[0-9]+(\\.[0-9]+)?$' THEN (meta_fields #>> '{amount}')::DOUBLE PRECISION > %s ELSE FALSE END",+ [1],+ ),+ (+ OP_GE,+ "amount",+ 1,+ "CASE WHEN meta_fields #>> '{amount}' ~ '^-?[0-9]+(\\.[0-9]+)?$' THEN (meta_fields #>> '{amount}')::DOUBLE PRECISION >= %s ELSE FALSE END",+ [1],+ ),+ (+ "<",+ "amount",+ 1,+ "CASE WHEN meta_fields #>> '{amount}' ~ '^-?[0-9]+(\\.[0-9]+)?$' THEN (meta_fields #>> '{amount}')::DOUBLE PRECISION < %s ELSE FALSE END",+ [1],+ ),+ (+ OP_LE,+ "amount",+ 1,+ "CASE WHEN meta_fields #>> '{amount}' ~ '^-?[0-9]+(\\.[0-9]+)?$' THEN (meta_fields #>> '{amount}')::DOUBLE PRECISION <= %s ELSE FALSE END",+ [1],+ ),+ (+ "in",+ "status",+ ["x"],+ "(lower(meta_fields #>> '{status}') = %s OR jsonb_exists(meta_fields #> '{status}', %s))",+ ["x", "x"],+ ),+ (+ "not in",+ "status",+ ["x"],+ "(meta_fields #> '{status}') IS NOT NULL AND (lower(meta_fields #>> '{status}') = %s OR jsonb_exists(meta_fields #> '{status}', %s)) IS NOT TRUE",+ ["x", "x"],+ ),+ (+ "contains",+ "status",+ "x",+ "jsonb_exists(meta_fields #> '{status}', %s) OR lower(meta_fields #>> '{status}') LIKE %s ESCAPE '\\'",+ ["x", "%x%"],+ ),+ (+ "not contains",+ "status",+ "x",+ "(meta_fields #> '{status}') IS NOT NULL AND (jsonb_exists(meta_fields #> '{status}', %s) OR lower(meta_fields #>> '{status}') LIKE %s ESCAPE '\\') IS NOT TRUE",+ ["x", "%x%"],+ ),+ (+ "start with",+ "status",+ "x",+ "lower(meta_fields #>> '{status}') LIKE %s ESCAPE '\\'",+ ["x%"],+ ),+ (+ "end with",+ "status",+ "x",+ "lower(meta_fields #>> '{status}') LIKE %s ESCAPE '\\'",+ ["%x"],+ ),+ (+ "empty",+ "status",+ None,+ "(meta_fields #> '{status}') IS NULL OR meta_fields #> '{status}' = 'null'::jsonb OR "+ "meta_fields #> '{status}' = '\"\"'::jsonb OR meta_fields #> '{status}' = '[]'::jsonb OR "+ "meta_fields #> '{status}' = '{}'::jsonb",+ [],+ ),+ (+ "not empty",+ "status",+ None,+ "(meta_fields #> '{status}') IS NOT NULL AND (meta_fields #> '{status}' = 'null'::jsonb) IS NOT TRUE AND "+ "(meta_fields #> '{status}' = '\"\"'::jsonb) IS NOT TRUE AND "+ "(meta_fields #> '{status}' = '[]'::jsonb) IS NOT TRUE AND "+ "(meta_fields #> '{status}' = '{}'::jsonb) IS NOT TRUE",+ [],+ ),+ ]++ case_operators = {op for op, _, _, _, _ in cases}+ assert len(expected_operators) == 14+ assert len(cases) == 14+ assert SUPPORTED_OPERATORS == expected_operators+ assert case_operators == expected_operators+ translator = GaussDBMetaFilterTranslator()+ for op, key, value, expected_sql, expected_params in cases:+ assert normalize_gaussdb_meta_operator(op) == op+ translated = translator.translate({"key": key, "op": op, "value": value})+ assert translated.sql == expected_sql+ assert translated.params == expected_params+++def test_tc_mgf_707_coerce_scalar_value_covers_python_literal_matrix():+ cases = [+ ("123", 123),+ ("1.5", 1.5),+ ("true", "true"),+ ("false", "false"),+ ("null", "null"),+ ("True", True),+ ("False", False),+ ("None", None),+ ("", ""),+ (" ", " "),+ ("[1,2]", "[1,2]"),+ ("abc", "abc"),+ (5, 5),+ (None, None),+ ]++ for value, expected in cases:+ assert gaussdb_filter.coerce_scalar_value(value, {"key": "a", "op": "=", "value": value}) == expected+++def test_tc_mgf_708_normalize_datetime_prefix_covers_valid_and_invalid_matrix():+ cases = [+ ("2026", None),+ ("2026-07", "2026-07-01 00:00:00"),+ ("2026-07-08 10:30:45", "2026-07-08 10:30:45"),+ ("2026-13-01", None),+ ("2026-07-32", None),+ ("abc", None),+ ]++ for value, expected in cases:+ assert gaussdb_filter.normalize_datetime_prefix(value) == expected+++def test_tc_mgf_709_fetch_gaussdb_metadata_doc_ids_delegates_public_boundary():+ expected_doc_ids = ["doc-1", "doc-2"]++ class RecordingDocStore:+ def __init__(self):+ self.calls = []++ def fetch_metadata_doc_ids(self, *args):+ self.calls.append(args)+ return expected_doc_ids++ doc_store = RecordingDocStore()+ kb_ids = ["kb-1", "kb-2"]+ filter_params = ["active", "active"]+ result = fetch_gaussdb_metadata_doc_ids(+ doc_store,+ "ragflow_doc_meta_tenant",+ kb_ids,+ "lower(meta_fields #>> '{status}') = %s",+ filter_params,+ 101,+ )++ assert result is expected_doc_ids+ assert doc_store.calls == [+ (+ "ragflow_doc_meta_tenant",+ kb_ids,+ "lower(meta_fields #>> '{status}') = %s",+ filter_params,+ 101,+ )+ ]+++def test_tc_mgf_810_all_operators_avoid_bare_empty_string_sql():+ cases = [+ (+ {"key": "status", "op": "=", "value": "x"},+ "(lower(meta_fields #>> '{status}') = %s OR jsonb_exists(meta_fields #> '{status}', %s))",+ ["x", "x"],+ ),+ (+ {"key": "status", "op": OP_NE, "value": "x"},+ "((meta_fields #> '{status}') IS NOT NULL AND (lower(meta_fields #>> '{status}') = %s OR jsonb_exists(meta_fields #> '{status}', %s)) IS NOT TRUE)",+ ["x", "x"],+ ),+ (+ {"key": "amount", "op": ">", "value": 100},+ "(CASE WHEN meta_fields #>> '{amount}' ~ '^-?[0-9]+(\\.[0-9]+)?$' THEN (meta_fields #>> '{amount}')::DOUBLE PRECISION > %s ELSE FALSE END)",+ [100],+ ),+ (+ {"key": "amount", "op": OP_GE, "value": 100},+ "(CASE WHEN meta_fields #>> '{amount}' ~ '^-?[0-9]+(\\.[0-9]+)?$' THEN (meta_fields #>> '{amount}')::DOUBLE PRECISION >= %s ELSE FALSE END)",+ [100],+ ),+ (+ {"key": "amount", "op": "<", "value": 100},+ "(CASE WHEN meta_fields #>> '{amount}' ~ '^-?[0-9]+(\\.[0-9]+)?$' THEN (meta_fields #>> '{amount}')::DOUBLE PRECISION < %s ELSE FALSE END)",+ [100],+ ),+ (+ {"key": "amount", "op": OP_LE, "value": 100},+ "(CASE WHEN meta_fields #>> '{amount}' ~ '^-?[0-9]+(\\.[0-9]+)?$' THEN (meta_fields #>> '{amount}')::DOUBLE PRECISION <= %s ELSE FALSE END)",+ [100],+ ),+ (+ {"key": "status", "op": "contains", "value": "x"},+ "(jsonb_exists(meta_fields #> '{status}', %s) OR lower(meta_fields #>> '{status}') LIKE %s ESCAPE '\\')",+ ["x", "%x%"],+ ),+ (+ {"key": "status", "op": "not contains", "value": "x"},+ "((meta_fields #> '{status}') IS NOT NULL AND (jsonb_exists(meta_fields #> '{status}', %s) OR lower(meta_fields #>> '{status}') LIKE %s ESCAPE '\\') IS NOT TRUE)",+ ["x", "%x%"],+ ),+ (+ {"key": "status", "op": "start with", "value": "x"},+ "(lower(meta_fields #>> '{status}') LIKE %s ESCAPE '\\')",+ ["x%"],+ ),+ (+ {"key": "status", "op": "end with", "value": "x"},+ "(lower(meta_fields #>> '{status}') LIKE %s ESCAPE '\\')",+ ["%x"],+ ),+ (+ {"key": "status", "op": "empty", "value": None},+ "((meta_fields #> '{status}') IS NULL OR meta_fields #> '{status}' = 'null'::jsonb OR "+ "meta_fields #> '{status}' = '\"\"'::jsonb OR meta_fields #> '{status}' = '[]'::jsonb OR "+ "meta_fields #> '{status}' = '{}'::jsonb)",+ [],+ ),+ (+ {"key": "status", "op": "not empty", "value": None},+ "((meta_fields #> '{status}') IS NOT NULL AND (meta_fields #> '{status}' = 'null'::jsonb) IS NOT TRUE AND "+ "(meta_fields #> '{status}' = '\"\"'::jsonb) IS NOT TRUE AND "+ "(meta_fields #> '{status}' = '[]'::jsonb) IS NOT TRUE AND "+ "(meta_fields #> '{status}' = '{}'::jsonb) IS NOT TRUE)",+ [],+ ),+ (+ {"key": "status", "op": "in", "value": ["x", "y"]},+ "((lower(meta_fields #>> '{status}') = %s OR jsonb_exists(meta_fields #> '{status}', %s)) OR (lower(meta_fields #>> '{status}') = %s OR jsonb_exists(meta_fields #> '{status}', %s)))",+ ["x", "x", "y", "y"],+ ),+ (+ {"key": "status", "op": "not in", "value": ["x", "y"]},+ "((meta_fields #> '{status}') IS NOT NULL AND "+ "(lower(meta_fields #>> '{status}') = %s OR jsonb_exists(meta_fields #> '{status}', %s)) IS NOT TRUE AND "+ "(lower(meta_fields #>> '{status}') = %s OR jsonb_exists(meta_fields #> '{status}', %s)) IS NOT TRUE)",+ ["x", "x", "y", "y"],+ ),+ ]++ assert len(cases) == 14+ for flt, expected_sql, expected_params in cases:+ sql = _assert_filter([flt], "and", expected_sql, expected_params)+ assert "= ''" not in sql+ assert "<> ''" not in sqldiff --git a/test/unit_test/rag/utils/test_gaussdb_search_sql.py b/test/unit_test/rag/utils/test_gaussdb_search_sql.pynew file mode 100644index 0000000000..8c27422d1a--- /dev/null+++ b/test/unit_test/rag/utils/test_gaussdb_search_sql.py@@ -0,0 +1,737 @@+#+# Copyright 2025 The InfiniFlow Authors. All Rights Reserved.+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+#+import pytest++from common.doc_store.gaussdb_conn_base import GaussDBSearchBuilder, InvalidGaussDBObjectName+from rag.utils.gaussdb_conn import GaussDBConnection, SearchResult+++def test_tc_ret_513_dynamic_chunk_field_exists_filter_reads_extra_jsonb():+ builder = GaussDBSearchBuilder(schema="public")++ sql, params = builder.build_condition_where({"kb_id": "kb1", "must_not": {"exists": "custom_kwd"}})++ assert "(extra -> 'custom_kwd') IS NULL" in sql+ assert "(extra -> 'custom_kwd') = 'null'::jsonb" in sql+ assert params == ["kb1"]+++def test_tc_ret_513_dynamic_chunk_field_select_and_filter_are_backed_by_extra_jsonb():+ builder = GaussDBSearchBuilder(schema="public")++ sql, params = builder.build_search_sql(+ table="ragflow_tenant",+ select_fields=["id", "custom_kwd"],+ condition={"kb_id": "kb1", "custom_kwd": ["artifact_page"]},+ keywords=[],+ vector=None,+ vector_dim=None,+ vector_weight=0.0,+ offset=0,+ limit=10,+ )++ assert "(extra -> 'custom_kwd') AS custom_kwd" in sql+ assert "(extra -> 'custom_kwd') = %s::jsonb" in sql+ assert params[:3] == ["kb1", '"artifact_page"', '["artifact_page"]']+++def test_tc_ret_513_dynamic_chunk_field_rejects_unsafe_json_key():+ builder = GaussDBSearchBuilder(schema="public")++ with pytest.raises(InvalidGaussDBObjectName):+ builder.build_condition_where({"must_not": {"exists": "custom_kwd'; DROP TABLE user; --"}})+++def test_tc_ret_101_fulltext_builder_parameterizes_simple_query_and_filter():+ builder = GaussDBSearchBuilder(schema="public")+ malicious_keyword = "contract'); DROP TABLE ragflow_tenant; --"+ query_text = f"{malicious_keyword} risk"+ sql, params = builder.build_search_sql(+ table="ragflow_tenant",+ select_fields=["id", "content_with_weight"],+ condition={"available_int": 1},+ keywords=[malicious_keyword, "risk"],+ vector=None,+ vector_dim=None,+ vector_weight=0.0,+ offset=0,+ limit=10,+ )++ assert "to_tsvector('simple'" in sql+ assert "ts_rank" in sql+ assert "WHERE available_int = %s AND to_tsvector" in sql+ assert "plainto_tsquery('simple', %s)" in sql+ assert "ORDER BY _score DESC, kb_id ASC, id ASC LIMIT %s OFFSET %s" in sql+ assert malicious_keyword not in sql+ assert query_text not in sql+ assert params == [query_text] * 6 + [1, query_text, 10, 0]+++def test_tc_ret_109_chinese_ngram_query_preserves_english_semantics_and_highlight_text():+ builder = GaussDBSearchBuilder(schema="public")++ ngram_ddl = builder.ddl.build_ngram_fulltext_ugin_ddl("ragflow_tenant")+ assert '"idx_gdb_ragflow_tenant_fts_all_ngram"' in ngram_ddl+ assert "USING ugin(to_tsvector('ngram'" in ngram_ddl++ chinese_sql, chinese_params = builder.build_search_sql(+ table="ragflow_tenant",+ select_fields=["id", "content_with_weight"],+ condition={"kb_id": "kb1"},+ keywords=["深圳"],+ vector=None,+ vector_dim=None,+ vector_weight=0.0,+ offset=0,+ limit=10,+ highlight_fields=["content_with_weight"],+ )+ assert "to_tsvector('ngram'" in chinese_sql+ assert "plainto_tsquery('ngram', %s)" in chinese_sql+ assert "to_tsvector('simple'" not in chinese_sql+ assert "COALESCE(content_with_weight, ' ') AS _highlight_source" in chinese_sql+ assert "ts_headline('ngram'" not in chinese_sql+ assert chinese_params == ["深圳"] * 6 + ["kb1", "深圳", 10, 0]++ mixed_sql, mixed_params = builder.build_search_sql(+ table="ragflow_tenant",+ select_fields=["id", "content_with_weight"],+ condition={"kb_id": "kb1"},+ keywords=["上海", "RAGFlow", "database"],+ vector=None,+ vector_dim=None,+ vector_weight=0.0,+ offset=0,+ limit=10,+ highlight_fields=["content_with_weight"],+ )+ assert "to_tsvector('simple'" in mixed_sql+ assert "to_tsvector('ngram'" in mixed_sql+ assert "plainto_tsquery('simple', %s)" in mixed_sql+ assert "plainto_tsquery('ngram', %s)" in mixed_sql+ assert " / 2.0)" in mixed_sql+ assert ") @@ plainto_tsquery('simple', %s) AND to_tsvector('ngram'" in mixed_sql+ assert "COALESCE(content_with_weight, ' ') AS _highlight_source" in mixed_sql+ assert mixed_params == ["RAGFlow database"] * 6 + ["上海"] * 6 + [+ "kb1",+ "RAGFlow database",+ "上海",+ 10,+ 0,+ ]++ single_character_sql, single_character_params = builder.build_search_sql(+ table="ragflow_tenant",+ select_fields=["id"],+ condition={"kb_id": "kb1"},+ keywords=["中"],+ vector=None,+ vector_dim=None,+ vector_weight=0.0,+ offset=0,+ limit=10,+ )+ assert "0.0 AS _score" in single_character_sql+ assert "WHERE kb_id = %s AND FALSE" in single_character_sql+ assert single_character_params == ["kb1", 10, 0]++ vector_only_sql, _vector_only_params = builder.build_search_sql(+ table="ragflow_tenant",+ select_fields=["id"],+ condition={"kb_id": "kb1"},+ keywords=["中"],+ vector=[0.1, 0.2, 0.3, 0.4],+ vector_dim=4,+ vector_weight=0.3,+ offset=0,+ limit=10,+ )+ assert "WITH vec AS" in vector_only_sql+ assert "WITH fts_raw AS" not in vector_only_sql++ conn = GaussDBConnection.__new__(GaussDBConnection)+ result = SearchResult(+ total=2,+ chunks=[+ {"id": "mixed", "_highlight_source": "上海 RAGFlow database 审计"},+ {"id": "overlap", "_highlight_source": "深圳数据库审计"},+ ],+ )+ assert conn.get_highlight(result, ["上海", "RAGFlow", "database"], "content_with_weight") == {"mixed": "<em>上海</em> <em>RAGFlow</em> <em>database</em> 审计"}+ assert conn.get_highlight(result, ["深圳数据", "深圳"], "content_with_weight") == {"overlap": "<em>深圳数据</em>库审计"}+++def test_tc_ret_203_vector_search_filters_invalid_placeholder_vectors():+ builder = GaussDBSearchBuilder(schema="public")+ sql, params = builder.build_search_sql(+ table="ragflow_tenant",+ select_fields=["id"],+ condition={"available_int": 1},+ keywords=[],+ vector=[0.1, 0.2, 0.3, 0.4],+ vector_dim=4,+ vector_weight=1.0,+ offset=0,+ limit=5,+ )++ vector_param = "[0.1,0.2,0.3,0.4]"+ assert "WHERE available_int = %s AND q_4_vec_valid = TRUE" in sql+ assert sql.count("q_4_vec <+> %s::floatvector(4)") == 3+ assert "WHERE _score >= %s" in sql+ assert params == [vector_param, vector_param, 1, vector_param, 5, 0.0, 5, 0]+++def test_tc_ret_301_hybrid_search_uses_configured_vector_weight():+ builder = GaussDBSearchBuilder(schema="public")+ sql, params = builder.build_search_sql(+ table="ragflow_tenant",+ select_fields=["id"],+ condition={"available_int": 1},+ keywords=["budget"],+ vector=[0.1, 0.2, 0.3, 0.4],+ vector_dim=4,+ vector_weight=0.7,+ offset=0,+ limit=5,+ )++ assert "%s * COALESCE(vec.vector_score, 0)" in sql+ assert "(1 - %s) * COALESCE(fts.fts_score, 0)" in sql+ assert params == [+ "budget",+ "budget",+ "budget",+ "budget",+ "budget",+ "budget",+ 1,+ "budget",+ 5,+ "[0.1,0.2,0.3,0.4]",+ 1,+ "[0.1,0.2,0.3,0.4]",+ 5,+ 0.7,+ 0.7,+ 0.0,+ 5,+ 0,+ ]+++def test_tc_ret_303_hybrid_search_full_outer_joins_text_and_vector_candidates():+ builder = GaussDBSearchBuilder(schema="public")+ sql, params = builder.build_search_sql(+ table="ragflow_tenant",+ select_fields=["id"],+ condition={"kb_id": "kb1"},+ keywords=["budget"],+ vector=[0.1, 0.2, 0.3, 0.4],+ vector_dim=4,+ vector_weight=0.3,+ offset=0,+ limit=5,+ )++ assert "COALESCE(fts.kb_id, vec.kb_id) AS kb_id" in sql+ assert "COALESCE(fts.id, vec.id) AS id" in sql+ assert "FROM fts FULL OUTER JOIN vec ON fts.kb_id = vec.kb_id AND fts.id = vec.id" in sql+ assert 'FROM merged JOIN "public"."ragflow_tenant" c ON c.kb_id = merged.kb_id AND c.id = merged.id' in sql+ assert params[-5:] == [0.3, 0.3, 0.0, 5, 0]+++def test_tc_ret_310_hybrid_pagerank_is_applied_to_projection_and_threshold():+ builder = GaussDBSearchBuilder(schema="public")+ sql, params = builder.build_search_sql(+ table="ragflow_tenant",+ select_fields=["id"],+ condition={"kb_id": "kb1"},+ keywords=["budget"],+ vector=[0.1, 0.2, 0.3, 0.4],+ vector_dim=4,+ vector_weight=0.7,+ offset=0,+ limit=5,+ pagerank_weight=10.0,+ )++ pagerank_expr = "(COALESCE(c.pagerank_fea, 0)::DOUBLE PRECISION / 100.0 * %s)"+ assert f"(merged.score + {pagerank_expr}) AS _score" in sql+ assert f"WHERE (merged.score + {pagerank_expr}) >= %s" in sql+ assert params == ["budget"] * 6 + [+ "kb1",+ "budget",+ 5,+ "[0.1,0.2,0.3,0.4]",+ "kb1",+ "[0.1,0.2,0.3,0.4]",+ 5,+ 0.7,+ 0.7,+ 10.0,+ 10.0,+ 0.0,+ 5,+ 0,+ ]+++def test_tc_ret_701_pagerank_feature_adds_bound_score_component():+ builder = GaussDBSearchBuilder(schema="public")+ sql, params = builder.build_search_sql(+ table="ragflow_tenant",+ select_fields=["id"],+ condition={"kb_id": "kb1"},+ keywords=["risk"],+ vector=None,+ vector_dim=None,+ vector_weight=0.0,+ offset=0,+ limit=5,+ pagerank_weight=10.0,+ )++ assert "COALESCE(pagerank_fea, 0)::DOUBLE PRECISION / 100.0 * %s" in sql+ assert "ORDER BY _score DESC, kb_id ASC, id ASC" in sql+ assert params == ["risk"] * 6 + [10.0, "kb1", "risk", 5, 0]+++def test_tc_ret_201_vector_only_query_omits_fulltext_predicates():+ builder = GaussDBSearchBuilder(schema="public")+ sql, params = builder.build_search_sql(+ table="ragflow_tenant",+ select_fields=["id"],+ condition={"available_int": 1},+ keywords=[],+ vector=[0.1, 0.2, 0.3, 0.4],+ vector_dim=4,+ vector_weight=1.0,+ offset=0,+ limit=5,+ )++ assert "to_tsvector" not in sql+ assert "WITH vec AS" in sql+ assert "ORDER BY q_4_vec <+> %s::floatvector(4) ASC LIMIT %s" in sql+ assert "ORDER BY distance ASC, kb_id ASC, id ASC LIMIT %s OFFSET %s" in sql+ assert "[0.1,0.2,0.3,0.4]" not in sql+ assert "[0.1,0.2,0.3,0.4]" in params+ assert params[-4:] == [5, 0.0, 5, 0]+++def test_tc_ret_605_fulltext_search_projects_parameterized_highlight_column():+ builder = GaussDBSearchBuilder(schema="public")+ malicious_keyword = "risk'); DROP TABLE ragflow_tenant; --"+ query_text = f"{malicious_keyword} audit"+ vector = [0.123456789, 0.234567891, 0.345678912, 0.456789123]+ sql, params = builder.build_search_sql(+ table="ragflow_tenant",+ select_fields=["id"],+ condition={"kb_id": "kb1"},+ keywords=[malicious_keyword, "audit"],+ vector=None,+ vector_dim=None,+ vector_weight=0.0,+ offset=0,+ limit=5,+ highlight_fields=["content_with_weight"],+ )++ assert ("ts_headline('simple', COALESCE(content_with_weight, ' '), plainto_tsquery('simple', %s), 'StartSel=<em>, StopSel=</em>') AS _highlight") in sql+ assert malicious_keyword not in sql+ assert query_text not in sql+ assert params == [query_text] * 7 + ["kb1", query_text, 5, 0]++ hybrid_sql, hybrid_params = builder.build_search_sql(+ table="ragflow_tenant",+ select_fields=["id"],+ condition={"kb_id": "kb1"},+ keywords=[malicious_keyword, "audit"],+ vector=vector,+ vector_dim=4,+ vector_weight=0.3,+ offset=0,+ limit=5,+ highlight_fields=["content_with_weight"],+ )+ vector_param = "[0.123456789,0.234567891,0.345678912,0.456789123]"+ assert "FULL OUTER JOIN" in hybrid_sql+ assert ("ts_headline('simple', COALESCE(content_with_weight, ' '), plainto_tsquery('simple', %s), 'StartSel=<em>, StopSel=</em>') AS _highlight") in hybrid_sql+ assert malicious_keyword not in hybrid_sql+ assert query_text not in hybrid_sql+ assert vector_param not in hybrid_sql+ assert hybrid_params == [query_text] * 6 + [+ "kb1",+ query_text,+ 5,+ vector_param,+ "kb1",+ vector_param,+ 5,+ 0.3,+ 0.3,+ query_text,+ 0.0,+ 5,+ 0,+ ]+++def test_tc_ret_601_aggregation_sql_groups_field_values():+ builder = GaussDBSearchBuilder(schema="public")+ sql, params = builder.build_aggregation_sql(+ table="ragflow_tenant",+ field_name="docnm_kwd",+ condition={"kb_id": "kb1"},+ )++ assert sql == ('SELECT docnm_kwd AS value, COUNT(1) AS count FROM "public"."ragflow_tenant" WHERE kb_id = %s AND docnm_kwd IS NOT NULL GROUP BY value ORDER BY count DESC, value ASC LIMIT %s')+ assert params == ["kb1", 1000]+++def test_tc_ret_602_aggregation_sql_expands_jsonb_array_fields_without_lateral():+ builder = GaussDBSearchBuilder(schema="public")+ sql, params = builder.build_aggregation_sql(+ table="ragflow_tenant",+ field_name="tag_kwd",+ condition={"kb_id": "k1"},+ )++ assert sql == (+ "SELECT value, COUNT(1) AS count "+ "FROM (SELECT jsonb_array_elements_text(COALESCE(tag_kwd, '[]'::jsonb)) AS value "+ 'FROM "public"."ragflow_tenant" WHERE kb_id = %s) AS expanded '+ "GROUP BY value ORDER BY count DESC, value ASC LIMIT %s"+ )+ assert "LATERAL" not in sql.upper()+ assert params == ["k1", 1000]+++def test_tc_ret_306_hybrid_threshold_is_applied_after_merged_score():+ builder = GaussDBSearchBuilder(schema="public")++ sql, params = builder.build_search_sql(+ table="ragflow_tenant",+ select_fields=["id"],+ condition={"kb_id": "kb1"},+ keywords=["risk"],+ vector=[0.1, 0.2, 0.3, 0.4],+ vector_dim=4,+ vector_weight=0.75,+ similarity_threshold=0.2,+ offset=0,+ limit=5,+ )++ assert "WHERE merged.score >= %s" in sql+ assert sql.count(">= %s") == 1+ assert params[-5:] == [0.75, 0.75, 0.2, 5, 0]+++def test_tc_ret_311_hybrid_zero_vector_weight_is_bound_in_merged_score():+ builder = GaussDBSearchBuilder(schema="public")++ sql, params = builder.build_search_sql(+ table="ragflow_tenant",+ select_fields=["id"],+ condition={"kb_id": "kb1"},+ keywords=["risk"],+ vector=[0.1, 0.2, 0.3, 0.4],+ vector_dim=4,+ vector_weight=0.0,+ similarity_threshold=0.8,+ offset=0,+ limit=5,+ )++ assert "(1 - %s) * COALESCE(fts.fts_score, 0) + %s * COALESCE(vec.vector_score, 0) AS score" in sql+ assert "WHERE merged.score >= %s" in sql+ assert params[-5:] == [0.0, 0.0, 0.8, 5, 0]+++def test_tc_ret_010_filter_only_search_uses_collection_limit_when_limit_is_zero():+ builder = GaussDBSearchBuilder(schema="public")+ sql, params = builder.build_search_sql(+ table="ragflow_tenant",+ select_fields=["id"],+ condition={"kb_id": "kb1"},+ keywords=[],+ vector=None,+ vector_dim=None,+ vector_weight=0.0,+ offset=0,+ limit=0,+ )++ assert "ORDER BY kb_id ASC, id ASC LIMIT %s OFFSET %s" in sql+ assert params == ["kb1", 10000, 0]+++def test_tc_ret_108_filter_only_search_parameterizes_kb_limit_and_offset():+ builder = GaussDBSearchBuilder(schema="public")+ kb_id = "k1' OR TRUE --"+ sql, params = builder.build_search_sql(+ table="ragflow_tenant",+ select_fields=["id", "docnm_kwd"],+ condition={"kb_id": kb_id},+ keywords=[],+ vector=None,+ vector_dim=None,+ vector_weight=0.0,+ offset=5,+ limit=10,+ order_by=type("Order", (), {"fields": [("docnm_kwd", True)]})(),+ )++ assert sql == ('SELECT id, kb_id, docnm_kwd, 0.0 AS _score, COUNT(*) OVER() AS __total FROM "public"."ragflow_tenant" WHERE kb_id = %s ORDER BY docnm_kwd DESC LIMIT %s OFFSET %s')+ assert params == [kb_id, 10, 5]+ assert kb_id not in sql+ assert all(expression not in sql for expression in ("to_tsvector", "plainto_tsquery", "ts_rank", "@@"))+++def test_tc_ret_405_condition_builder_supports_exists_must_not_lists_jsonb_and_null():+ builder = GaussDBSearchBuilder(schema="public")++ where_sql, params = builder.build_condition_where(+ {+ "exists": "doc_id",+ "must_not": {"exists": "img_id"},+ "doc_id": ["d1", "d2"],+ "tag_kwd": ["risk", "audit"],+ "mom_id": None,+ }+ )++ assert "doc_id IS NOT NULL" in where_sql+ assert "img_id IS NULL" in where_sql+ assert "doc_id IN (%s, %s)" in where_sql+ assert "(tag_kwd @> %s::jsonb OR tag_kwd @> %s::jsonb)" in where_sql+ assert "mom_id IS NULL" in where_sql+ assert params == ["d1", "d2", '["risk"]', '["audit"]']+++def test_tc_ret_513_compile_kwd_uses_extra_jsonb_mapping():+ builder = GaussDBSearchBuilder(schema="public")++ sql, params = builder.build_search_sql(+ table="ragflow_tenant",+ select_fields=["compile_kwd"],+ condition={"kb_id": "kb1", "must_not": {"exists": "compile_kwd"}},+ keywords=[],+ vector=None,+ vector_dim=None,+ vector_weight=0.0,+ offset=0,+ limit=10,+ )++ assert "(extra #>> '{compile_kwd}') AS compile_kwd" in sql+ assert "(extra #>> '{compile_kwd}') IS NULL" in sql+ assert params == ["kb1", 10, 0]+++def test_tc_ret_406_condition_builder_rejects_empty_list_conditions():+ builder = GaussDBSearchBuilder(schema="public")++ with pytest.raises(ValueError, match="empty condition values"):+ builder.build_condition_where({"id": []})+++def test_tc_ret_208_vector_search_rejects_missing_or_mismatched_dimensions():+ builder = GaussDBSearchBuilder(schema="public")++ with pytest.raises(ValueError, match="^vector_dim is required for vector search$"):+ builder.build_search_sql(+ table="ragflow_tenant",+ select_fields=["id"],+ condition={"kb_id": "kb1"},+ keywords=[],+ vector=[0.1],+ vector_dim=None,+ vector_weight=1.0,+ offset=0,+ limit=5,+ )++ with pytest.raises(ValueError, match="^vector dimension mismatch: expected 2, got 1$"):+ builder.build_search_sql(+ table="ragflow_tenant",+ select_fields=["id"],+ condition={"kb_id": "kb1"},+ keywords=[],+ vector=[0.1],+ vector_dim=2,+ vector_weight=1.0,+ offset=0,+ limit=5,+ )+++def test_tc_ret_501_select_empty_defaults_to_id_kb_id():+ builder = GaussDBSearchBuilder(schema="public")++ assert builder.normalize_select_fields([]) == ["id", "kb_id"]++ sql, params = builder.build_search_sql(+ table="ragflow_tenant",+ select_fields=[],+ condition={"kb_id": "k1"},+ keywords=[],+ vector=None,+ vector_dim=None,+ vector_weight=0.0,+ offset=0,+ limit=10,+ )++ assert sql == ('SELECT id, kb_id, 0.0 AS _score, COUNT(*) OVER() AS __total FROM "public"."ragflow_tenant" WHERE kb_id = %s ORDER BY kb_id ASC, id ASC LIMIT %s OFFSET %s')+ assert "SELECT *" not in sql+ assert params == ["k1", 10, 0]+++def test_tc_ret_502_select_fields_add_ids_skip_score_and_deduplicate():+ builder = GaussDBSearchBuilder(schema="public")++ assert builder.normalize_select_fields(["_score", "content_with_weight", "content_with_weight"]) == [+ "id",+ "kb_id",+ "content_with_weight",+ ]++ sql, params = builder.build_search_sql(+ table="ragflow_tenant",+ select_fields=["_score", "content_with_weight", "content_with_weight"],+ condition={"kb_id": "k1"},+ keywords=[],+ vector=None,+ vector_dim=None,+ vector_weight=0.0,+ offset=0,+ limit=10,+ )++ assert sql.startswith("SELECT id, kb_id, content_with_weight, 0.0 AS _score")+ assert sql.count("content_with_weight") == 1+ assert params == ["k1", 10, 0]+++def test_tc_ret_503_select_star_defaults_to_id_kb_id():+ builder = GaussDBSearchBuilder(schema="public")++ assert builder.normalize_select_fields(["*"]) == ["id", "kb_id"]++ sql, params = builder.build_search_sql(+ table="ragflow_tenant",+ select_fields=["*"],+ condition={"kb_id": "k1"},+ keywords=[],+ vector=None,+ vector_dim=None,+ vector_weight=0.0,+ offset=0,+ limit=10,+ )++ assert sql == ('SELECT id, kb_id, 0.0 AS _score, COUNT(*) OVER() AS __total FROM "public"."ragflow_tenant" WHERE kb_id = %s ORDER BY kb_id ASC, id ASC LIMIT %s OFFSET %s')+ assert "SELECT *" not in sql+ assert params == ["k1", 10, 0]+++def test_tc_ret_504_selecting_vector_column_auto_includes_valid_flag():+ builder = GaussDBSearchBuilder(schema="public")++ assert builder.normalize_select_fields(["q_1024_vec"]) == ["id", "kb_id", "q_1024_vec", "q_1024_vec_valid"]+++def test_tc_ret_702_zero_pagerank_weight_does_not_add_pagerank_score_component():+ builder = GaussDBSearchBuilder(schema="public")+ sql, params = builder.build_search_sql(+ table="ragflow_tenant",+ select_fields=["id"],+ condition={"kb_id": "k1"},+ keywords=["hello"],+ vector=None,+ vector_dim=None,+ vector_weight=0.0,+ offset=0,+ limit=10,+ pagerank_weight=0.0,+ )++ assert "pagerank_fea" not in sql+ assert "COALESCE(pagerank_fea" not in sql+ assert params == ["hello"] * 6 + ["k1", "hello", 10, 0]+++def test_tc_ret_512_compatibility_fields_use_safe_mapping_and_deterministic_orders():+ builder = GaussDBSearchBuilder(schema="public")+ sql, params = builder.build_search_sql(+ table="ragflow_tenant",+ select_fields=["row_id()", "chunk_order_int"],+ condition={"kb_id": "kb1"},+ keywords=[],+ vector=None,+ vector_dim=None,+ vector_weight=0.0,+ offset=0,+ limit=5,+ order_by=type("Order", (), {"fields": [("position_int", False)]})(),+ )++ assert builder.build_position_order_sql() == (+ "COALESCE((page_num_int #>> '{0}')::int, 100000000) ASC, COALESCE((position_int #>> '{0,3}')::int, 100000000) ASC, COALESCE((top_int #>> '{0}')::int, 100000000) ASC"+ )+ assert sql == (+ 'SELECT id, kb_id, NULL AS "row_id()", _order_id AS chunk_order_int, '+ '0.0 AS _score, COUNT(*) OVER() AS __total FROM "public"."ragflow_tenant" '+ "WHERE kb_id = %s ORDER BY "+ "COALESCE((page_num_int #>> '{0}')::int, 100000000) ASC, "+ "COALESCE((position_int #>> '{0,3}')::int, 100000000) ASC, "+ "COALESCE((top_int #>> '{0}')::int, 100000000) ASC LIMIT %s OFFSET %s"+ )+ assert params == ["kb1", 5, 0]+ assert "row_id() AS" not in sql+ assert "chunk_order_int AS" not in sql++ default_sql, default_params = builder.build_search_sql(+ table="ragflow_tenant",+ select_fields=["row_id()", "chunk_order_int"],+ condition={"kb_id": "kb1"},+ keywords=[],+ vector=None,+ vector_dim=None,+ vector_weight=0.0,+ offset=0,+ limit=5,+ )+ assert "ORDER BY kb_id ASC, id ASC LIMIT %s OFFSET %s" in default_sql+ assert default_params == ["kb1", 5, 0]++ with pytest.raises(ValueError, match="unsafe_field"):+ builder.build_search_sql(+ table="ragflow_tenant",+ select_fields=["unsafe_field; DROP TABLE chunks"],+ condition={"kb_id": "kb1"},+ keywords=[],+ vector=None,+ vector_dim=None,+ vector_weight=0.0,+ offset=0,+ limit=5,+ )diff --git a/test/unit_test/rag/utils/test_gaussdb_sql_validator.py b/test/unit_test/rag/utils/test_gaussdb_sql_validator.pynew file mode 100644index 0000000000..5c71cd8b12--- /dev/null+++ b/test/unit_test/rag/utils/test_gaussdb_sql_validator.py@@ -0,0 +1,887 @@+#+# Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+#+import pytest++from common.doc_store import gaussdb_conn_base as gaussdb_base+from common.doc_store.gaussdb_conn_base import (+ ExposedGaussDBTable,+ GaussDBSQLValidator,+ UnsafeGaussDBSQL,+ is_gaussdb_aggregate_sql,+ jsonb_path_literal,+ mask_gaussdb_text,+)+++def _validator(kb_ids=None, field_map=None, fetch_size=128):+ table = ExposedGaussDBTable.from_field_map(+ physical_name="ragflow_t1",+ kb_ids=["kb-1"] if kb_ids is None else kb_ids,+ field_map={"amount": "number", "dept": "string", "customer,name": "string"} if field_map is None else field_map,+ )+ return GaussDBSQLValidator({table.logical_name: table}, default_limit=fetch_size)+++def _assert_unsafe_sql(sql, message, validator=None):+ with pytest.raises(UnsafeGaussDBSQL) as exc_info:+ (validator or _validator()).validate_and_patch(sql)++ assert str(exc_info.value) == message+++def test_tc_cfg_711_mask_gaussdb_text_redacts_quoted_unquoted_and_delimited_secrets():+ raw = (+ "password=plain,semi;value "+ "passwd='quoted, semi; value' "+ 'token="token, semi; value" '+ '"access_token": "access, semi; value", '+ "api_key=api,semi;value "+ "secret='secret, semi; value' "+ "dsn=postgresql://user:[email protected]:5432/ragflow?options=a,b;c"+ )++ assert mask_gaussdb_text(raw) == ('password=*** passwd=\'***\' token="***" "access_token": "***", api_key=*** secret=\'***\' dsn=***')+ assert mask_gaussdb_text("uri=gaussdb://user:secret@host/db?x=a,b;c") == "uri=***"+ assert mask_gaussdb_text("password='quoted ''value,semi;tail' next=ok") == "password='***' next=ok"+ assert mask_gaussdb_text('token="unterminated, semi; value') == "token=***"+++def test_tc_sql_412_jsonb_path_literal_encodes_special_keys():+ assert jsonb_path_literal(["amount"]) == "'{amount}'"+ assert jsonb_path_literal(["customer", "name"]) == "'{customer,name}'"+ assert jsonb_path_literal(["customer,name"]) == "'{\"customer,name\"}'"+ assert jsonb_path_literal(['customer"name']) == '\'{"customer\\"name"}\''+ assert jsonb_path_literal(["customer\\name"]) == "'{\"customer\\\\name\"}'"+ assert jsonb_path_literal(["customer name"]) == "'{\"customer name\"}'"+ assert jsonb_path_literal(["\u5ba2\u6237"]) == "'{\"\u5ba2\u6237\"}'"+ assert jsonb_path_literal(["O'Brien"]) == "'{\"O''Brien\"}'"+++def test_tc_sql_412_jsonb_path_literal_rejects_empty_path_or_segment():+ with pytest.raises(UnsafeGaussDBSQL) as empty_path:+ jsonb_path_literal([])+ with pytest.raises(UnsafeGaussDBSQL) as empty_segment:+ jsonb_path_literal(["amount", ""])+ with pytest.raises(UnsafeGaussDBSQL) as invalid_segment:+ jsonb_path_literal([None])++ assert str(empty_path.value) == "empty JSONB path"+ assert str(empty_segment.value) == "empty JSONB path segment"+ assert str(invalid_segment.value) == "invalid JSONB path segment"+++def test_tc_sql_412_jsonb_path_literal_parser_handles_escapes_and_invalid_shapes():+ assert gaussdb_base._parse_jsonb_path_literal("'{\"customer,name\",dept}'") == ("customer,name", "dept")+ assert gaussdb_base._parse_jsonb_path_literal("'{\"customer\\\\name\"}'") == ("customer\\name",)++ invalid_literals = {+ "amount": "dynamic JSONB path is not allowed",+ "'{amount'": "dynamic JSONB path is not allowed",+ "'{amount,}'": "invalid JSONB path literal",+ "'{\"amount}'": "invalid JSONB path literal",+ }+ for literal, message in invalid_literals.items():+ with pytest.raises(UnsafeGaussDBSQL) as exc_info:+ gaussdb_base._parse_jsonb_path_literal(literal)+ assert str(exc_info.value) == message+++def test_tc_sql_101_validator_allows_single_table_select_and_returns_columns():+ validated = _validator(field_map={}).validate_and_patch("SELECT doc_id, docnm_kwd FROM ragflow_t1 WHERE kb_id = 'kb-1' ORDER BY doc_id LIMIT 5")++ assert validated.sql == ("SELECT doc_id, docnm_kwd FROM ragflow_t1 WHERE kb_id = 'kb-1' ORDER BY doc_id LIMIT 5")+ assert validated.columns == ["doc_id", "docnm_kwd"]+ assert validated.is_aggregation is False+++def test_tc_sql_102_validator_allows_readonly_cte_and_exposed_output_aliases():+ baseline = _validator(field_map={}).validate_and_patch("WITH x AS (SELECT doc_id FROM ragflow_t1 WHERE kb_id = 'kb-1') SELECT doc_id FROM x")+ alias = _validator(field_map={"amount": "number"}).validate_and_patch(+ "WITH rows AS (SELECT chunk_data #>> '{amount}' AS amount, kb_id FROM ragflow_t1 WHERE kb_id = 'kb-1') SELECT amount FROM rows"+ )+ qualified_alias = _validator(field_map={"amount": "number"}).validate_and_patch(+ "WITH rows AS (SELECT chunk_data #>> '{amount}' AS amount, kb_id FROM ragflow_t1 WHERE kb_id = 'kb-1') SELECT rows.amount FROM rows"+ )++ assert baseline.sql == ("WITH x AS (SELECT doc_id FROM ragflow_t1 WHERE kb_id = 'kb-1') SELECT doc_id FROM x LIMIT 128")+ assert baseline.columns == ["doc_id"]+ assert baseline.is_aggregation is False+ assert alias.sql == ("WITH rows AS (SELECT chunk_data #>> '{amount}' AS amount, kb_id FROM ragflow_t1 WHERE kb_id = 'kb-1') SELECT amount FROM rows LIMIT 128")+ assert alias.columns == ["amount"]+ assert alias.is_aggregation is False+ assert qualified_alias.sql == ("WITH rows AS (SELECT chunk_data #>> '{amount}' AS amount, kb_id FROM ragflow_t1 WHERE kb_id = 'kb-1') SELECT rows.amount FROM rows LIMIT 128")+ assert qualified_alias.columns == ["amount"]+ assert qualified_alias.is_aggregation is False++ validator = _validator(field_map={})+ list_validator = GaussDBSQLValidator(["ragflow_t1"], kb_ids=["kb-1"])+ assert list_validator.tables["ragflow_t1"].required_kb_ids == ("kb-1",)+ values_cte = validator._parse_one("WITH x AS (SELECT 1 UNION SELECT 2) SELECT 1")+ assert validator._cte_output_columns(values_cte) == {}+ from sqlglot import exp++ assert validator._is_cte_output_column(exp.column("doc_id"), {}) is False+++def test_tc_sql_103_validator_rejects_multiple_statements_and_wraps_parser_failures():+ _assert_unsafe_sql(+ "SELECT doc_id FROM ragflow_t1 WHERE kb_id = 'kb-1'; SELECT 1",+ "multiple statements are not allowed",+ )++ validator = _validator()+ with pytest.raises(UnsafeGaussDBSQL, match="exactly one SQL statement is allowed"):+ validator._parse_one("SELECT 1; SELECT 2")++ with pytest.raises(UnsafeGaussDBSQL) as parse_error:+ validator._parse_one("SELECT (")+ with pytest.raises(UnsafeGaussDBSQL) as aggregate_parse_error:+ is_gaussdb_aggregate_sql("SELECT (")++ assert str(parse_error.value) == "SQL parse failed"+ assert str(aggregate_parse_error.value) == "SQL parse failed"+++def test_tc_sql_104_validator_rejects_dml_including_nested_cte():+ scenarios = {+ "INSERT INTO ragflow_t1 VALUES (1)": "only SELECT statements are allowed",+ "UPDATE ragflow_t1 SET doc_id = 'new'": "only SELECT statements are allowed",+ "DELETE FROM ragflow_t1": "only SELECT statements are allowed",+ "WITH x AS (DELETE FROM ragflow_t1 RETURNING doc_id) SELECT doc_id FROM x": ("SQL contains a non-read-only expression"),+ }+ for sql, message in scenarios.items():+ _assert_unsafe_sql(sql, message)+++def test_tc_sql_105_validator_rejects_ddl():+ for sql in (+ "CREATE TABLE test (id int)",+ "DROP TABLE ragflow_t1",+ "ALTER TABLE ragflow_t1 ADD COLUMN x int",+ ):+ _assert_unsafe_sql(sql, "only SELECT statements are allowed")+++def test_tc_sql_106_validator_rejects_call():+ _assert_unsafe_sql("CALL my_proc()", "only SELECT statements are allowed")+++def test_tc_sql_107_validator_rejects_copy():+ _assert_unsafe_sql("COPY ragflow_t1 TO '/tmp/out.csv'", "only SELECT statements are allowed")+++ "sql",+ [+ "MERGE INTO ragflow_t1 t USING source_table s ON t.doc_id = s.doc_id WHEN MATCHED THEN DELETE",+ "TRUNCATE TABLE ragflow_t1",+ "ANALYZE ragflow_t1",+ "VACUUM ragflow_t1",+ "GRANT SELECT ON ragflow_t1 TO app_user",+ "REVOKE SELECT ON ragflow_t1 FROM app_user",+ "LOCK TABLE ragflow_t1 IN ACCESS EXCLUSIVE MODE",+ "DO $$ BEGIN NULL; END $$",+ ],+ ids=["merge", "truncate", "analyze", "vacuum", "grant", "revoke", "lock", "do"],+)+def test_tc_sql_130_validator_rejects_additional_write_and_admin_statements(sql):+ with pytest.raises(UnsafeGaussDBSQL) as exc_info:+ _validator(field_map={}).validate_and_patch(sql)++ assert str(exc_info.value) in {+ "only SELECT statements are allowed",+ "multiple statements are not allowed",+ "SQL parse failed",+ }+++def test_tc_sql_108_validator_rejects_select_star():+ _assert_unsafe_sql("SELECT * FROM ragflow_t1 WHERE kb_id = 'kb-1'", "SELECT * is not allowed")+++def test_tc_sql_109_validator_allows_count_star_and_marks_aggregation():+ validated = _validator(field_map={}).validate_and_patch("SELECT COUNT(*) AS cnt FROM ragflow_t1")++ assert validated.sql == "SELECT COUNT(*) AS cnt FROM ragflow_t1 WHERE kb_id = 'kb-1' LIMIT 128"+ assert validated.columns == ["cnt"]+ assert validated.is_aggregation is True+++def test_tc_sql_110_validator_rejects_rank_and_row_number_windows():+ scenarios = (+ "SELECT doc_id, RANK() OVER (ORDER BY chunk_data #>> '{amount}') AS rnk FROM ragflow_t1",+ "SELECT doc_id, ROW_NUMBER() OVER (PARTITION BY kb_id ORDER BY doc_id) AS rn FROM ragflow_t1",+ )+ for sql in scenarios:+ _assert_unsafe_sql(sql, "window functions are not allowed")+++def test_tc_sql_111_validator_rejects_system_table_and_cross_schema_table():+ for sql in (+ "SELECT doc_id FROM pg_catalog.tables",+ "SELECT doc_id FROM other_schema.ragflow_t1",+ ):+ _assert_unsafe_sql(sql, "cross-schema SQL is not allowed")+++def test_tc_sql_112_validator_rejects_non_whitelisted_table():+ _assert_unsafe_sql(+ "SELECT doc_id FROM unknown_table WHERE kb_id = 'kb-1'",+ "table unknown_table is not allowed",+ )+++def test_tc_sql_113_validator_rejects_non_whitelisted_column():+ _assert_unsafe_sql(+ "SELECT content_with_weight FROM ragflow_t1 WHERE kb_id = 'kb-1'",+ "column content_with_weight is not allowed",+ )+++def test_tc_sql_114_validator_rejects_forbidden_system_functions():+ scenarios = {+ "SELECT pg_sleep(1) FROM ragflow_t1": "function pg_sleep is not allowed",+ "SELECT now() FROM ragflow_t1": "system functions are not allowed",+ "SELECT current_user FROM ragflow_t1": "system functions are not allowed",+ "SELECT version() FROM ragflow_t1": "function version is not allowed",+ }+ for sql, message in scenarios.items():+ _assert_unsafe_sql(sql, message, _validator(field_map={}))+++def test_tc_sql_115_validator_rejects_oceanbase_json_functions():+ scenarios = {+ "SELECT json_extract_string(chunk_data, '$.amount') FROM ragflow_t1": ("function json_extract_string is not allowed"),+ "SELECT json_extract(chunk_data, '$.amount') FROM ragflow_t1": ("only GaussDB #> / #>> JSONB operators are allowed"),+ "SELECT json_extract_isnull(chunk_data, '$.amount') FROM ragflow_t1": ("function json_extract_isnull is not allowed"),+ }+ for sql, message in scenarios.items():+ _assert_unsafe_sql(sql, message)+++def test_tc_sql_116_validator_rejects_forbidden_jsonb_set_returning_functions():+ scenarios = {+ "SELECT jsonb_each(chunk_data) FROM ragflow_t1": "function jsonb_each is not allowed",+ "SELECT jsonb_each_text(chunk_data) FROM ragflow_t1": "function jsonb_each_text is not allowed",+ "SELECT jsonb_array_elements(chunk_data) FROM ragflow_t1": ("function jsonb_array_elements is not allowed"),+ "SELECT jsonb_to_recordset(chunk_data) FROM ragflow_t1": ("function jsonb_to_recordset is not allowed"),+ }+ for sql, message in scenarios.items():+ _assert_unsafe_sql(sql, message, _validator(field_map={}))+++def test_tc_sql_117_validator_rejects_join_even_when_both_aliases_are_scoped():+ _assert_unsafe_sql(+ "SELECT a.doc_id FROM ragflow_t1 a JOIN ragflow_t1 b ON a.doc_id = b.doc_id WHERE a.kb_id = 'kb-1' AND b.kb_id = 'kb-1'",+ "complex SQL must use a simpler single-table kb_id boundary",+ _validator(field_map={}),+ )+++def test_tc_sql_118_validator_rejects_union_as_non_select_root():+ _assert_unsafe_sql(+ "SELECT doc_id FROM ragflow_t1 WHERE kb_id = 'kb-1' UNION SELECT doc_id FROM ragflow_t1 WHERE kb_id = 'kb-2'",+ "only SELECT statements are allowed",+ _validator(field_map={}),+ )+++def test_tc_sql_119_validator_rejects_or_predicate():+ _assert_unsafe_sql(+ "SELECT doc_id FROM ragflow_t1 WHERE kb_id = 'kb-1' OR chunk_data #>> '{amount}' = '100'",+ "complex SQL must use a simpler single-table kb_id boundary",+ )+++def test_tc_sql_120_validator_rejects_dynamic_or_wrong_source_jsonb_paths():+ scenarios = {+ "SELECT doc_id FROM ragflow_t1 WHERE chunk_data #>> ('{' || param || '}') = '100'": ("dynamic JSONB path is not allowed"),+ "SELECT doc_id, chunk_data #>> path_col FROM ragflow_t1 WHERE kb_id = 'kb-1'": ("dynamic JSONB path is not allowed"),+ "SELECT doc_id, other_data #>> '{amount}' FROM ragflow_t1 WHERE kb_id = 'kb-1'": ("only chunk_data JSONB paths are allowed"),+ }+ for sql, message in scenarios.items():+ _assert_unsafe_sql(sql, message)+++def test_tc_sql_121_validator_rejects_empty_sql_variants():+ for sql in ("", " ", "```sql```"):+ _assert_unsafe_sql(sql, "empty SQL")+++def test_tc_sql_122_validator_rejects_explain():+ _assert_unsafe_sql(+ "EXPLAIN SELECT doc_id FROM ragflow_t1 WHERE kb_id = 'kb-1'",+ "only SELECT statements are allowed",+ )+++def test_tc_sql_123_validator_rejects_set_command():+ _assert_unsafe_sql("SET statement_timeout = 30000", "only SELECT statements are allowed")+++def test_tc_sql_124_validator_rejects_show_command():+ _assert_unsafe_sql("SHOW statement_timeout", "only SELECT statements are allowed")+++ "sql",+ [+ "SELECT doc_id FROM ragflow_t1, LATERAL jsonb_each(chunk_data) AS j",+ "SELECT doc_id FROM ragflow_t1 CROSS JOIN LATERAL (SELECT 1) AS j",+ "SELECT doc_id FROM ragflow_t1 LEFT JOIN LATERAL (SELECT 1) AS j ON TRUE",+ ],+ ids=["set-returning-function", "cross-join-subquery", "left-join-subquery"],+)+def test_tc_sql_125_validator_rejects_every_lateral_shape(sql):+ _assert_unsafe_sql(+ sql,+ "LATERAL is not allowed",+ _validator(field_map={}),+ )+++def test_tc_sql_126_validator_rejects_jsonb_path_query():+ _assert_unsafe_sql(+ "SELECT doc_id FROM ragflow_t1 WHERE jsonb_path_query(chunk_data, '$.amount') IS NOT NULL",+ "function jsonb_path_query is not allowed",+ _validator(field_map={}),+ )+++def test_tc_sql_127_validator_rejects_jsonb_to_record():+ _assert_unsafe_sql(+ "SELECT doc_id FROM ragflow_t1, jsonb_to_record(chunk_data) AS t(amount text)",+ "function jsonb_to_record is not allowed",+ _validator(field_map={}),+ )+++def test_tc_sql_128_validator_rejects_generate_series():+ _assert_unsafe_sql(+ "SELECT doc_id FROM ragflow_t1, generate_series(1, 10) AS g",+ "function generate_series is not allowed",+ _validator(field_map={}),+ )+++def test_tc_sql_129_validator_rejects_unnest():+ _assert_unsafe_sql(+ "SELECT doc_id FROM ragflow_t1, unnest(ARRAY[1, 2, 3]) AS u",+ "function unnest is not allowed",+ _validator(field_map={}),+ )+++def test_tc_sql_201_validator_injects_kb_id_for_simple_single_table_query():+ validator = _validator(field_map={})+ validated = validator.validate_and_patch("SELECT doc_id FROM ragflow_t1")++ assert validated.sql == "SELECT doc_id FROM ragflow_t1 WHERE kb_id = 'kb-1' LIMIT 128"+ assert validated.columns == ["doc_id"]+ assert validated.is_aggregation is False+++def test_tc_sql_202_validator_injects_multi_kb_boundary():+ validated = _validator(kb_ids=["kb-1", "kb-2"], field_map={}).validate_and_patch("SELECT doc_id FROM ragflow_t1")++ assert validated.sql == ("SELECT doc_id FROM ragflow_t1 WHERE kb_id IN ('kb-1', 'kb-2') LIMIT 128")+ assert validated.columns == ["doc_id"]+ assert validated.is_aggregation is False+++def test_tc_sql_203_validator_preserves_allowed_kb_boundary_shapes():+ equality = _validator(field_map={}).validate_and_patch("SELECT doc_id FROM ragflow_t1 WHERE kb_id = 'kb-1'")+ reversed_equality = _validator(field_map={}).validate_and_patch("SELECT doc_id FROM ragflow_t1 WHERE 'kb-1' = kb_id")+ in_boundary = _validator(kb_ids=["kb-1", "kb-2"], field_map={}).validate_and_patch("SELECT doc_id FROM ragflow_t1 WHERE kb_id IN ('kb-1', 'kb-2')")+ predicate_before_boundary = _validator(kb_ids=["kb-1", "kb-2"], field_map={}).validate_and_patch("SELECT doc_id FROM ragflow_t1 WHERE doc_id IN ('doc1', 'doc2') AND kb_id IN ('kb-1')")++ assert equality.sql == "SELECT doc_id FROM ragflow_t1 WHERE kb_id = 'kb-1' LIMIT 128"+ assert reversed_equality.sql == "SELECT doc_id FROM ragflow_t1 WHERE 'kb-1' = kb_id LIMIT 128"+ assert in_boundary.sql == ("SELECT doc_id FROM ragflow_t1 WHERE kb_id IN ('kb-1', 'kb-2') LIMIT 128")+ assert predicate_before_boundary.sql == ("SELECT doc_id FROM ragflow_t1 WHERE doc_id IN ('doc1', 'doc2') AND kb_id IN ('kb-1') LIMIT 128")+ for validated in (equality, reversed_equality, in_boundary, predicate_before_boundary):+ assert validated.columns == ["doc_id"]+ assert validated.is_aggregation is False+++def test_tc_sql_204_validator_rejects_cross_kb_boundary():+ _assert_unsafe_sql(+ "SELECT doc_id FROM ragflow_t1 WHERE kb_id = 'kb-other'",+ "SQL crosses the allowed kb_id boundary",+ _validator(field_map={}),+ )+++def test_tc_sql_205_validator_rejects_non_positive_or_dynamic_kb_predicates():+ scenarios = {+ "SELECT doc_id FROM ragflow_t1 WHERE NOT kb_id = 'kb-1'": ("kb_id boundary must be a positive top-level predicate"),+ "SELECT doc_id FROM ragflow_t1 WHERE (kb_id = 'kb-1') IS FALSE": ("kb_id boundary must be a positive top-level predicate"),+ "SELECT doc_id FROM ragflow_t1 WHERE kb_id IN (doc_id)": ("kb_id IN must use static string literals"),+ "SELECT doc_id FROM ragflow_t1 WHERE kb_id IN ()": "SQL crosses the allowed kb_id boundary",+ }+ for sql, message in scenarios.items():+ _assert_unsafe_sql(sql, message, _validator(field_map={}))++ validator = _validator()+ from sqlglot import exp++ in_ast = validator._parse_one("SELECT doc_id FROM ragflow_t1 WHERE kb_id IN ('kb-1')")+ assert validator._select_has_static_kb_boundary(in_ast) is True+ no_boundary_ast = validator._parse_one("SELECT doc_id FROM ragflow_t1 WHERE doc_id = 'd1'")+ assert validator._select_has_static_kb_boundary(no_boundary_ast) is False+ non_boundary_ast = validator._parse_one("SELECT doc_id FROM ragflow_t1 WHERE NOT doc_id = 'd1'")+ assert validator._select_has_static_kb_boundary(non_boundary_ast) is False+ empty_in = exp.In(this=exp.column("kb_id"), expressions=[])+ empty_ast = exp.select("doc_id").from_("ragflow_t1").where(empty_in)+ with pytest.raises(UnsafeGaussDBSQL, match="kb_id boundary is empty"):+ validator._select_has_static_kb_boundary(empty_ast)+++def test_tc_sql_206_validator_rejects_complex_sql_when_scope_cannot_be_patched():+ _assert_unsafe_sql(+ "SELECT a.doc_id FROM ragflow_t1 a JOIN ragflow_t1 b ON a.doc_id = b.doc_id",+ "complex SQL must use a simpler single-table kb_id boundary",+ _validator(field_map={}),+ )+++def test_tc_sql_207_validator_rejects_second_non_whitelisted_table():+ _assert_unsafe_sql(+ "SELECT t1.doc_id FROM ragflow_t1 t1 JOIN ragflow_t2 t2 ON t1.doc_id = t2.doc_id",+ "table ragflow_t2 is not allowed",+ _validator(field_map={}),+ )++ validator = _validator()+ with pytest.raises(UnsafeGaussDBSQL, match="must read from an exposed"):+ validator._enforce_kb_boundary("SELECT 1")+ with pytest.raises(UnsafeGaussDBSQL, match="exactly one base table"):+ validator._enforce_kb_boundary("SELECT doc_id FROM ragflow_t1, ragflow_t2")++ ast = validator._parse_one("SELECT a.doc_id FROM ragflow_t1 a JOIN ragflow_t2 b ON a.id = b.id")+ assert validator._direct_base_tables(ast) == ["ragflow_t1", "ragflow_t2"]+ assert validator._direct_source_tables(ast) == ["ragflow_t1", "ragflow_t2"]+++def test_tc_sql_208_validator_rejects_missing_or_unsafe_required_kb_boundary():+ missing_table = ExposedGaussDBTable.from_field_map("ragflow_t1", [], {})+ missing_validator = GaussDBSQLValidator({"ragflow_t1": missing_table})++ _assert_unsafe_sql(+ "SELECT doc_id FROM ragflow_t1",+ "kb_id boundary is required",+ missing_validator,+ )+ _assert_unsafe_sql(+ "SELECT doc_id FROM ragflow_t1",+ "unsafe kb_id literal",+ _validator(kb_ids=["kb-1;drop"], field_map={}),+ )+++def test_tc_sql_209_validator_requires_each_cte_base_scope_to_prove_kb_boundary():+ validator = _validator(field_map={})+ _assert_unsafe_sql(+ "WITH x AS (SELECT doc_id FROM ragflow_t1) SELECT doc_id FROM x",+ "each base table scope must include a kb_id boundary",+ validator,+ )+ _assert_unsafe_sql(+ "WITH rows AS (SELECT doc_id, 'kb-1' AS kb_id FROM ragflow_t1) SELECT doc_id FROM rows WHERE kb_id = 'kb-1'",+ "each base table scope must include a kb_id boundary",+ validator,+ )++ validated = validator.validate_and_patch("WITH x AS (SELECT doc_id FROM ragflow_t1 WHERE kb_id = 'kb-1') SELECT doc_id FROM x")+ assert validated.sql == ("WITH x AS (SELECT doc_id FROM ragflow_t1 WHERE kb_id = 'kb-1') SELECT doc_id FROM x LIMIT 128")+ assert validated.columns == ["doc_id"]+ assert validated.is_aggregation is False+++ "correlation_predicate",+ ["", "inner_t.doc_id = outer_t.doc_id AND "],+ ids=["non-correlated", "correlated"],+)+def test_tc_sql_209_validator_requires_each_subquery_scope_to_prove_kb_boundary(correlation_predicate):+ validator = _validator(field_map={})+ outer_missing = f"SELECT outer_t.doc_id FROM ragflow_t1 AS outer_t WHERE EXISTS (SELECT 1 FROM ragflow_t1 AS inner_t WHERE {correlation_predicate}inner_t.kb_id = 'kb-1')"+ inner_missing = (+ f"SELECT outer_t.doc_id FROM ragflow_t1 AS outer_t WHERE outer_t.kb_id = 'kb-1' AND EXISTS (SELECT 1 FROM ragflow_t1 AS inner_t WHERE {correlation_predicate}inner_t.doc_id = 'doc-1')"+ )+ legal = f"SELECT outer_t.doc_id FROM ragflow_t1 AS outer_t WHERE outer_t.kb_id = 'kb-1' AND EXISTS (SELECT 1 FROM ragflow_t1 AS inner_t WHERE {correlation_predicate}inner_t.kb_id = 'kb-1')"+ cross_kb = f"SELECT outer_t.doc_id FROM ragflow_t1 AS outer_t WHERE outer_t.kb_id = 'kb-1' AND EXISTS (SELECT 1 FROM ragflow_t1 AS inner_t WHERE {correlation_predicate}inner_t.kb_id = 'kb-other')"++ for sql in (outer_missing, inner_missing):+ _assert_unsafe_sql(sql, "each base table scope must include a kb_id boundary", validator)+ _assert_unsafe_sql(cross_kb, "SQL crosses the allowed kb_id boundary", validator)++ validated = validator.validate_and_patch(legal)+ assert validated.sql.endswith("LIMIT 128")+ assert validated.sql.count("kb_id = 'kb-1'") == 2+ assert validated.columns == ["doc_id"]+ assert validated.is_aggregation is False+++def test_tc_sql_301_validator_adds_default_limit_when_missing():+ validated = _validator(field_map={}).validate_and_patch("SELECT doc_id FROM ragflow_t1 WHERE kb_id = 'kb-1'")++ assert validated.sql == "SELECT doc_id FROM ragflow_t1 WHERE kb_id = 'kb-1' LIMIT 128"+ assert validated.columns == ["doc_id"]+ assert validated.is_aggregation is False+ no_limit_validator = GaussDBSQLValidator(_validator().tables, default_limit=-1)+ assert "LIMIT" not in no_limit_validator.validate_and_patch("SELECT doc_id FROM ragflow_t1 WHERE kb_id = 'kb-1'").sql+++def test_tc_sql_302_validator_caps_large_limit_to_default_limit():+ validated = _validator(field_map={}).validate_and_patch("SELECT doc_id FROM ragflow_t1 WHERE kb_id = 'kb-1' LIMIT 200")++ assert validated.sql == "SELECT doc_id FROM ragflow_t1 WHERE kb_id = 'kb-1' LIMIT 128"+ assert validated.columns == ["doc_id"]+ assert validated.is_aggregation is False+++def test_tc_sql_303_validator_preserves_limit_under_default_limit():+ validated = _validator(field_map={}).validate_and_patch("SELECT doc_id FROM ragflow_t1 WHERE kb_id = 'kb-1' LIMIT 100")++ assert validated.sql == "SELECT doc_id FROM ragflow_t1 WHERE kb_id = 'kb-1' LIMIT 100"+ assert validated.columns == ["doc_id"]+ assert validated.is_aggregation is False+++ "sql",+ [+ "SELECT doc_id FROM ragflow_t1 WHERE kb_id = 'kb-1' LIMIT 0",+ "SELECT doc_id FROM ragflow_t1 WHERE kb_id = 'kb-1' LIMIT -1",+ "SELECT doc_id FROM ragflow_t1 WHERE kb_id = 'kb-1' LIMIT 1.5",+ "SELECT doc_id FROM ragflow_t1 WHERE kb_id = 'kb-1' LIMIT $1",+ "SELECT doc_id FROM ragflow_t1 WHERE kb_id = 'kb-1' LIMIT doc_id",+ "SELECT doc_id FROM ragflow_t1 WHERE kb_id = 'kb-1' LIMIT '1'",+ "SELECT doc_id FROM ragflow_t1 WHERE kb_id = 'kb-1' FETCH FIRST 0 ROWS ONLY",+ "SELECT doc_id FROM ragflow_t1 WHERE kb_id = 'kb-1' AND EXISTS (SELECT 1 LIMIT 0)",+ ],+ ids=[+ "zero",+ "negative",+ "decimal",+ "parameter",+ "column",+ "string",+ "zero-fetch",+ "nested-zero",+ ],+)+def test_tc_sql_304_validator_rejects_non_positive_dynamic_or_non_integer_limits(sql):+ _assert_unsafe_sql(sql, "LIMIT must be a positive static integer", _validator(field_map={}))+++def test_tc_sql_305_validator_rejects_complex_scope_before_limit_patch():+ validator = _validator(field_map={"status": "string"})+ scenarios = (+ "SELECT a.doc_id FROM ragflow_t1 a JOIN ragflow_t1 b ON a.doc_id = b.doc_id LIMIT 500",+ "SELECT doc_id FROM ragflow_t1 WHERE kb_id = 'kb-1' OR chunk_data #>> '{status}' = 'active' LIMIT 500",+ )+ for sql in scenarios:+ _assert_unsafe_sql(+ sql,+ "complex SQL must use a simpler single-table kb_id boundary",+ validator,+ )+++def test_tc_sql_306_readonly_guard_uses_fetch_size_as_default_limit():+ validator = GaussDBSQLValidator.readonly_guard(default_limit=64)+ validated = validator.validate_and_patch("SELECT doc_id FROM ragflow_t1 WHERE kb_id = 'kb-1'")++ assert validator.default_limit == 64+ assert validated.sql == "SELECT doc_id FROM ragflow_t1 WHERE kb_id = 'kb-1' LIMIT 64"+ assert validated.columns == ["doc_id"]+ assert validated.is_aggregation is False+++def test_tc_sql_307_validator_caps_fetch_first_to_default_limit():+ validated = _validator(field_map={}).validate_and_patch("SELECT doc_id FROM ragflow_t1 WHERE kb_id = 'kb-1' FETCH FIRST 500 ROWS ONLY")++ assert validated.sql == ("SELECT doc_id FROM ragflow_t1 WHERE kb_id = 'kb-1' FETCH FIRST 128 ROWS ONLY")+ assert validated.columns == ["doc_id"]+ assert validated.is_aggregation is False+++def test_tc_sql_308_clause_tokens_in_literals_or_subqueries_do_not_block_ast_patches():+ limit_field = _validator(field_map={"limit": "string"}).validate_and_patch("SELECT doc_id FROM ragflow_t1 WHERE kb_id = 'kb-1' AND chunk_data #>> '{limit}' = '100'")+ subquery_limit = _validator(field_map={}).validate_and_patch("SELECT doc_id FROM ragflow_t1 WHERE kb_id = 'kb-1' AND EXISTS (SELECT 1 LIMIT 1)")+ forbidden_word_literal = _validator(field_map={"dept": "string"}).validate_and_patch("SELECT doc_id FROM ragflow_t1 WHERE kb_id = 'kb-1' AND chunk_data #>> '{dept}' = 'update'")+ order_word_literal = _validator(field_map={"dept": "string"}).validate_and_patch("SELECT doc_id FROM ragflow_t1 WHERE chunk_data #>> '{dept}' = 'order by dept' ORDER BY doc_id")++ assert limit_field.sql == ("SELECT doc_id FROM ragflow_t1 WHERE kb_id = 'kb-1' AND chunk_data #>> '{limit}' = '100' LIMIT 128")+ assert subquery_limit.sql == ("SELECT doc_id FROM ragflow_t1 WHERE kb_id = 'kb-1' AND EXISTS(SELECT 1 LIMIT 1) LIMIT 128")+ assert forbidden_word_literal.sql == ("SELECT doc_id FROM ragflow_t1 WHERE kb_id = 'kb-1' AND chunk_data #>> '{dept}' = 'update' LIMIT 128")+ assert order_word_literal.sql == ("SELECT doc_id FROM ragflow_t1 WHERE chunk_data #>> '{dept}' = 'order by dept' AND kb_id = 'kb-1' ORDER BY doc_id LIMIT 128")+ for validated in (limit_field, subquery_limit, forbidden_word_literal, order_word_literal):+ assert validated.columns == ["doc_id"]+ assert validated.is_aggregation is False+++def test_tc_sql_401_validator_allows_single_key_jsonb_text_path():+ validated = _validator(field_map={"amount": "number"}).validate_and_patch("SELECT chunk_data #>> '{amount}' AS amount FROM ragflow_t1 WHERE kb_id = 'kb-1' LIMIT 10")++ assert validated.sql == ("SELECT chunk_data #>> '{amount}' AS amount FROM ragflow_t1 WHERE kb_id = 'kb-1' LIMIT 10")+ assert validated.columns == ["amount"]+ assert validated.is_aggregation is False+++def test_tc_sql_402_validator_allows_multi_key_jsonb_text_path():+ validated = _validator(field_map={"profile.name": "string"}).validate_and_patch("SELECT doc_id FROM ragflow_t1 WHERE chunk_data #>> '{profile,name}' = 'Alice'")++ assert validated.sql == ("SELECT doc_id FROM ragflow_t1 WHERE chunk_data #>> '{profile,name}' = 'Alice' AND kb_id = 'kb-1' LIMIT 128")+ assert validated.columns == ["doc_id"]+ assert validated.is_aggregation is False+++def test_tc_sql_403_validator_requires_quotes_for_jsonb_key_containing_comma():+ validator = _validator(field_map={"a,b": "string"})+ validated = validator.validate_and_patch("SELECT doc_id FROM ragflow_t1 WHERE chunk_data #>> '{\"a,b\"}' = 'value'")++ assert validated.sql == ("SELECT doc_id FROM ragflow_t1 WHERE chunk_data #>> '{\"a,b\"}' = 'value' AND kb_id = 'kb-1' LIMIT 128")+ assert validated.columns == ["doc_id"]+ assert validated.is_aggregation is False+ _assert_unsafe_sql(+ "SELECT doc_id FROM ragflow_t1 WHERE chunk_data #>> '{a,b}' = 'value'",+ "JSONB path ('a', 'b') is not exposed",+ validator,+ )+++def test_tc_sql_404_validator_allows_jsonb_object_extraction_operator():+ validated = _validator(field_map={"profile": "json"}).validate_and_patch("SELECT doc_id FROM ragflow_t1 WHERE chunk_data #> '{profile}' @> '{\"name\": \"Alice\"}'::jsonb")++ assert validated.sql == ("SELECT doc_id FROM ragflow_t1 WHERE chunk_data #> '{profile}' @> CAST('{\"name\": \"Alice\"}' AS JSONB) AND kb_id = 'kb-1' LIMIT 128")+ assert validated.columns == ["doc_id"]+ assert validated.is_aggregation is False+++def test_tc_sql_405_validator_allows_numeric_cast_on_exposed_jsonb_field():+ validated = _validator(field_map={"amount": "number"}).validate_and_patch("SELECT doc_id FROM ragflow_t1 WHERE CAST(chunk_data #>> '{amount}' AS DOUBLE PRECISION) > 100")++ assert validated.sql == ("SELECT doc_id FROM ragflow_t1 WHERE CAST(chunk_data #>> '{amount}' AS DOUBLE PRECISION) > 100 AND kb_id = 'kb-1' LIMIT 128")+ assert validated.columns == ["doc_id"]+ assert validated.is_aggregation is False+++def test_tc_sql_406_validator_allows_to_date_on_exposed_jsonb_field():+ validated = _validator(field_map={"dt": "date"}).validate_and_patch("SELECT doc_id FROM ragflow_t1 WHERE to_date(chunk_data #>> '{dt}', 'YYYY-MM-DD') > '2026-01-01'")++ assert validated.sql == ("SELECT doc_id FROM ragflow_t1 WHERE TO_DATE(chunk_data #>> '{dt}', 'YYYY-MM-DD') > '2026-01-01' AND kb_id = 'kb-1' LIMIT 128")+ assert validated.columns == ["doc_id"]+ assert validated.is_aggregation is False+++def test_tc_sql_407_validator_allows_jsonb_text_null_checks():+ validator = _validator(field_map={"status": "string"})+ is_null = validator.validate_and_patch("SELECT doc_id FROM ragflow_t1 WHERE (chunk_data #>> '{status}') IS NULL")+ is_not_null = validator.validate_and_patch("SELECT doc_id FROM ragflow_t1 WHERE (chunk_data #>> '{status}') IS NOT NULL")++ assert is_null.sql == ("SELECT doc_id FROM ragflow_t1 WHERE (chunk_data #>> '{status}') IS NULL AND kb_id = 'kb-1' LIMIT 128")+ assert is_not_null.sql == ("SELECT doc_id FROM ragflow_t1 WHERE NOT (chunk_data #>> '{status}') IS NULL AND kb_id = 'kb-1' LIMIT 128")+ for validated in (is_null, is_not_null):+ assert validated.columns == ["doc_id"]+ assert validated.is_aggregation is False+++def test_tc_sql_408_validator_rejects_unexposed_or_empty_map_jsonb_path():+ _assert_unsafe_sql(+ "SELECT doc_id, chunk_data #>> '{non_exposed_field}' FROM ragflow_t1 WHERE kb_id = 'kb-1'",+ "JSONB path ('non_exposed_field',) is not exposed",+ )+ _assert_unsafe_sql(+ "SELECT doc_id, chunk_data #>> '{amount}' AS amount FROM ragflow_t1 WHERE kb_id = 'kb-1'",+ "JSONB path ('amount',) is not exposed",+ _validator(field_map={}),+ )+++def test_tc_sql_409_validator_rejects_bare_or_non_gaussdb_chunk_data_access():+ scenarios = {+ "SELECT doc_id, chunk_data FROM ragflow_t1 WHERE kb_id = 'kb-1'": ("chunk_data may only be accessed through #> / #>>"),+ "SELECT chunk_data ->> 'amount' FROM ragflow_t1": ("only GaussDB #> / #>> JSONB operators are allowed"),+ "SELECT chunk_data['amount'] FROM ragflow_t1": ("chunk_data may only be accessed through #> / #>>"),+ "SELECT chunk_data @> '{\"amount\":1}'::jsonb FROM ragflow_t1": ("chunk_data may only be accessed through #> / #>>"),+ "SELECT chunk_data ? 'amount' FROM ragflow_t1": ("chunk_data may only be accessed through #> / #>>"),+ "SELECT chunk_data #- '{amount}' FROM ragflow_t1": ("chunk_data may only be accessed through #> / #>>"),+ }+ for sql, message in scenarios.items():+ _assert_unsafe_sql(sql, message)+++ "predicate",+ [+ "chunk_data #>> '{status}' = ''",+ "'' = chunk_data #>> '{status}'",+ "chunk_data #>> '{status}' <> ''",+ "'' != chunk_data #>> '{status}'",+ "CAST(chunk_data #>> '{status}' AS TEXT) = ''",+ "chunk_data #>> '{status}' = CAST('' AS TEXT)",+ "chunk_data #>> '{status}' IS DISTINCT FROM ''",+ "chunk_data #>> '{status}' IS NOT DISTINCT FROM ''",+ ],+ ids=[+ "equal",+ "reversed-equal",+ "not-equal",+ "reversed-not-equal",+ "cast-jsonb",+ "cast-empty-string",+ "distinct",+ "not-distinct",+ ],+)+def test_tc_sql_410_validator_rejects_jsonb_text_comparisons_with_sql_empty_string(predicate):+ _assert_unsafe_sql(+ f"SELECT doc_id FROM ragflow_t1 WHERE kb_id = 'kb-1' AND {predicate}",+ "JSONB text cannot be compared with an empty SQL string",+ _validator(field_map={"status": "string"}),+ )+++def test_tc_sql_410_validator_allows_jsonb_null_literal_comparison():+ sql = "SELECT doc_id FROM ragflow_t1 WHERE kb_id = 'kb-1' AND chunk_data #> '{status}' = 'null'::jsonb"++ validated = _validator(field_map={"status": "string"}).validate_and_patch(sql)++ assert validated.sql == ("SELECT doc_id FROM ragflow_t1 WHERE kb_id = 'kb-1' AND chunk_data #> '{status}' = CAST('null' AS JSONB) LIMIT 128")+ assert validated.columns == ["doc_id"]+ assert validated.is_aggregation is False+++def test_tc_sql_512_runtime_readonly_guard_enforces_table_and_preserves_safe_scopes():+ validator = GaussDBSQLValidator.readonly_guard(execution_schema="tenant_schema")+ _assert_unsafe_sql(+ "SELECT 1",+ "SQL must read from a DocEngine table",+ validator,+ )++ cte_alias = validator.validate_and_patch("WITH rows AS (SELECT chunk_data #>> '{amount}' AS amount, kb_id FROM ragflow_t1 WHERE kb_id = 'kb-1') SELECT amount FROM rows")+ predicate_before_boundary = validator.validate_and_patch("SELECT doc_id FROM ragflow_t1 WHERE doc_id = 'doc1' AND kb_id = 'kb-1'")+ correlated = validator.validate_and_patch(+ "SELECT outer_t.doc_id FROM ragflow_t1 AS outer_t WHERE outer_t.kb_id = 'kb-1' AND EXISTS ("+ "SELECT 1 FROM ragflow_t1 AS inner_t WHERE inner_t.doc_id = outer_t.doc_id "+ "AND inner_t.kb_id = 'kb-1')"+ )+ assert cte_alias.sql == ("WITH rows AS (SELECT chunk_data #>> '{amount}' AS amount, kb_id FROM \"tenant_schema\".ragflow_t1 WHERE kb_id = 'kb-1') SELECT amount FROM rows LIMIT 128")+ assert cte_alias.columns == ["amount"]+ assert cte_alias.is_aggregation is False+ assert predicate_before_boundary.sql == ("SELECT doc_id FROM \"tenant_schema\".ragflow_t1 WHERE doc_id = 'doc1' AND kb_id = 'kb-1' LIMIT 128")+ assert predicate_before_boundary.columns == ["doc_id"]+ assert predicate_before_boundary.is_aggregation is False+ assert correlated.sql.count('FROM "tenant_schema".ragflow_t1') == 2+ assert correlated.sql.count("kb_id = 'kb-1'") == 2+ assert correlated.columns == ["doc_id"]+ assert correlated.is_aggregation is False+++ "sql",+ [+ "SELECT 'sum(' AS marker, doc_id FROM ragflow_t1 WHERE kb_id = 'kb-1' LIMIT 5",+ "SELECT doc_id /* sum( */ FROM ragflow_t1 WHERE kb_id = 'kb-1' LIMIT 5",+ "SELECT doc_id AS \"sum(\" FROM ragflow_t1 WHERE kb_id = 'kb-1' LIMIT 5",+ ],+ ids=["literal", "comment", "alias"],+)+def test_tc_sql_705_aggregate_detection_ignores_sum_text_outside_ast_functions(sql):+ assert is_gaussdb_aggregate_sql(sql) is False+ assert _validator(field_map={}).validate_and_patch(sql).is_aggregation is False+++def test_tc_sql_705_aggregate_detection_rejects_multiple_statements():+ with pytest.raises(UnsafeGaussDBSQL) as exc_info:+ is_gaussdb_aggregate_sql("SELECT COUNT(*) FROM ragflow_t1; SELECT 1")++ assert str(exc_info.value) == "exactly one SQL statement is allowed"+++def test_tc_sql_705_validator_detects_all_supported_aggregations_and_non_aggregate():+ validator = _validator(field_map={"amount": "number"})+ scenarios = {+ "SELECT COUNT(*) AS cnt FROM ragflow_t1": (+ "SELECT COUNT(*) AS cnt FROM ragflow_t1 WHERE kb_id = 'kb-1' LIMIT 128",+ ["cnt"],+ True,+ ),+ "SELECT SUM(CAST(chunk_data #>> '{amount}' AS DOUBLE PRECISION)) AS total FROM ragflow_t1": (+ "SELECT SUM(CAST(chunk_data #>> '{amount}' AS DOUBLE PRECISION)) AS total FROM ragflow_t1 WHERE kb_id = 'kb-1' LIMIT 128",+ ["total"],+ True,+ ),+ "SELECT AVG(CAST(chunk_data #>> '{amount}' AS DOUBLE PRECISION)) AS avg_amt FROM ragflow_t1": (+ "SELECT AVG(CAST(chunk_data #>> '{amount}' AS DOUBLE PRECISION)) AS avg_amt FROM ragflow_t1 WHERE kb_id = 'kb-1' LIMIT 128",+ ["avg_amt"],+ True,+ ),+ "SELECT MAX(CAST(chunk_data #>> '{amount}' AS DOUBLE PRECISION)) AS max_amt FROM ragflow_t1": (+ "SELECT MAX(CAST(chunk_data #>> '{amount}' AS DOUBLE PRECISION)) AS max_amt FROM ragflow_t1 WHERE kb_id = 'kb-1' LIMIT 128",+ ["max_amt"],+ True,+ ),+ "SELECT MIN(CAST(chunk_data #>> '{amount}' AS DOUBLE PRECISION)) AS min_amt FROM ragflow_t1": (+ "SELECT MIN(CAST(chunk_data #>> '{amount}' AS DOUBLE PRECISION)) AS min_amt FROM ragflow_t1 WHERE kb_id = 'kb-1' LIMIT 128",+ ["min_amt"],+ True,+ ),+ "SELECT doc_id FROM ragflow_t1": (+ "SELECT doc_id FROM ragflow_t1 WHERE kb_id = 'kb-1' LIMIT 128",+ ["doc_id"],+ False,+ ),+ }+ for sql, expected in scenarios.items():+ validated = validator.validate_and_patch(sql)+ assert (validated.sql, validated.columns, validated.is_aggregation) == expected+ assert is_gaussdb_aggregate_sql(validated.sql) is expected[2]+++def test_tc_sql_706_validator_returns_selected_column_names_and_aliases():+ validator = _validator(field_map={"amount": "number"})+ aliased = validator.validate_and_patch("SELECT doc_id, chunk_data #>> '{amount}' AS amt FROM ragflow_t1")+ simple = validator.validate_and_patch("SELECT doc_id FROM ragflow_t1")+ literal = validator.validate_and_patch("SELECT 1, doc_id FROM ragflow_t1")++ assert aliased.sql == ("SELECT doc_id, chunk_data #>> '{amount}' AS amt FROM ragflow_t1 WHERE kb_id = 'kb-1' LIMIT 128")+ assert aliased.columns == ["doc_id", "amt"]+ assert aliased.is_aggregation is False+ assert simple.sql == "SELECT doc_id FROM ragflow_t1 WHERE kb_id = 'kb-1' LIMIT 128"+ assert simple.columns == ["doc_id"]+ assert simple.is_aggregation is False+ assert literal.sql == "SELECT 1, doc_id FROM ragflow_t1 WHERE kb_id = 'kb-1' LIMIT 128"+ assert literal.columns == ["1", "doc_id"]+ assert literal.is_aggregation is False+++def test_tc_sql_707_select_alias_is_allowed_in_group_and_order_by_for_both_guards():+ sql = "SELECT chunk_data #>> '{dept}' AS dept, COUNT(*) AS cnt FROM ragflow_t1 WHERE kb_id = 'kb-1' GROUP BY dept ORDER BY dept"+ validated = _validator(field_map={"dept": "string"}).validate_and_patch(sql)+ readonly = GaussDBSQLValidator.readonly_guard().validate_and_patch(sql)+ expected_sql = "SELECT chunk_data #>> '{dept}' AS dept, COUNT(*) AS cnt FROM ragflow_t1 WHERE kb_id = 'kb-1' GROUP BY dept ORDER BY dept LIMIT 128"++ assert validated.sql == expected_sql+ assert validated.columns == ["dept", "cnt"]+ assert validated.is_aggregation is True+ assert readonly.sql == expected_sql+ assert readonly.columns == ["dept", "cnt"]+ assert readonly.is_aggregation is True