RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix: the bigquery connector constructs sql queries u... in... (#17500)
Anupam Medirattad0986424ee33
diff --git a/common/data_source/bigquery_connector.py b/common/data_source/bigquery_connector.pyindex 09ff99db48..89558371e6 100644--- a/common/data_source/bigquery_connector.py+++ b/common/data_source/bigquery_connector.py@@ -16,6 +16,7 @@ import copyimport hashlibimport jsonimport logging+import refrom datetime import date, datetime, time, timezonefrom decimal import Decimalfrom typing import Any, Dict, Generator, List, Optional, Tuple@@ -48,6 +49,28 @@ _CURSOR_TYPE_KEY = "__ragflow_bq_cursor_type__"# Default cost guard: 1 GiB. Users can raise this explicitly.DEFAULT_MAXIMUM_BYTES_BILLED = 1024 * 1024 * 1024+# Allowlist for standard BigQuery unquoted identifiers (columns, datasets, tables).+_IDENTIFIER_RE = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$")+# GCP project IDs may contain hyphens.+_PROJECT_ID_RE = re.compile(r"^[A-Za-z][A-Za-z0-9_-]*$")+++def _validate_identifier(value: Optional[str], name: str) -> Optional[str]:+ if not value:+ return value+ if not _IDENTIFIER_RE.fullmatch(value):+ raise ConnectorValidationError(f"Invalid BigQuery identifier for {name!r}")+ return value+++def _validate_project_id(value: str, name: str) -> str:+ if not value:+ return value+ if not _PROJECT_ID_RE.fullmatch(value):+ raise ConnectorValidationError(f"Invalid BigQuery identifier for {name!r}")+ return value++# Maps a BigQuery field type to the ScalarQueryParameter type used for cursors._CURSOR_PARAM_TYPE_MAP = {"TIMESTAMP": "TIMESTAMP",@@ -115,15 +138,27 @@ class BigQueryConnector(LoadConnector, PollConnector, SlimConnectorWithPermSync)job_timeout_ms: Optional per-job timeout in milliseconds.use_query_cache: Whether to allow BigQuery's query result cache."""- self.project_id = (project_id or "").strip()- self.dataset_id = (dataset_id or "").strip()- self.table_id = (table_id or "").strip()+ self.project_id = _validate_project_id((project_id or "").strip(), "project_id")+ self.dataset_id = _validate_identifier((dataset_id or "").strip(), "dataset_id")+ self.table_id = _validate_identifier((table_id or "").strip(), "table_id")self.location = (location or "").strip()self.query = (query or "").strip()- self.content_columns = [c.strip() for c in (content_columns or "").split(",") if c.strip()]- self.metadata_columns = [c.strip() for c in (metadata_columns or "").split(",") if c.strip()]- self.id_column = id_column.strip() if id_column else None- self.timestamp_column = timestamp_column.strip() if timestamp_column else None+ _content_cols: List[str] = []+ for _c in (content_columns or "").split(","):+ _col = _c.strip()+ if _col:+ _validate_identifier(_col, f"content_columns[{_col!r}]")+ _content_cols.append(_col)+ self.content_columns = _content_cols+ _meta_cols: List[str] = []+ for _c in (metadata_columns or "").split(","):+ _col = _c.strip()+ if _col:+ _validate_identifier(_col, f"metadata_columns[{_col!r}]")+ _meta_cols.append(_col)+ self.metadata_columns = _meta_cols+ self.id_column = _validate_identifier(id_column.strip() if id_column else None, "id_column")+ self.timestamp_column = _validate_identifier(timestamp_column.strip() if timestamp_column else None, "timestamp_column")self.batch_size = batch_sizeself.page_size = page_sizeself.maximum_bytes_billed = maximum_bytes_billed@@ -172,7 +207,7 @@ class BigQueryConnector(LoadConnector, PollConnector, SlimConnectorWithPermSync)Accepts ``service_account_json`` as either a dict or a JSON string."""- logging.debug("Loading credentials for BigQuery project: %s", self.project_id)+ logging.debug("Loading credentials for BigQuery connector.")raw = (credentials or {}).get("service_account_json")if not raw:@@ -225,6 +260,7 @@ class BigQueryConnector(LoadConnector, PollConnector, SlimConnectorWithPermSync)def _build_base_query(self) -> str:"""Return the single base query (custom query takes precedence over table mode)."""if self.query:+ # custom query is trusted, administrator-supplied SQL — not validated as an identifier.return self.query.rstrip(";")if self.dataset_id and self.table_id:return f"SELECT * FROM `{self.project_id}.{self.dataset_id}.{self.table_id}`"diff --git a/test/unit_test/data_source/test_bigquery_connector.py b/test/unit_test/data_source/test_bigquery_connector.pyindex 56eeee9fcb..8514ac473e 100644--- a/test/unit_test/data_source/test_bigquery_connector.py+++ b/test/unit_test/data_source/test_bigquery_connector.py@@ -148,6 +148,51 @@ def test_invalid_credentials_json_raises():connector.load_credentials({"service_account_json": "{not-json"})+# ---------------------------------------------------------------------------+# Identifier validation+# ---------------------------------------------------------------------------+++ "field,value",+ [+ ("project_id", "foo` WHERE 1=1 --"),+ ("dataset_id", "foo` WHERE 1=1 --"),+ ("table_id", "foo` WHERE 1=1 --"),+ ("id_column", "foo; DROP TABLE users --"),+ ("timestamp_column", "ts OR 1=1"),+ ("content_columns", "name` UNION SELECT password FROM users --"),+ ("metadata_columns", "status` UNION SELECT password FROM users --"),+ ],+)+def test_rejects_malicious_identifier(field, value):+ kwargs = dict(+ project_id="my-proj",+ dataset_id="ds",+ table_id="tbl",+ content_columns="name",+ )+ kwargs[field] = value+ with pytest.raises(ConnectorValidationError):+ BigQueryConnector(**kwargs)+++def test_accepts_valid_identifiers():+ connector = BigQueryConnector(+ project_id="my-project",+ dataset_id="my_dataset",+ table_id="my_table",+ content_columns="name,description",+ id_column="id",+ timestamp_column="created_at",+ )+ assert connector.project_id == "my-project"+ assert connector.id_column == "id"+ assert connector.timestamp_column == "created_at"++# ---------------------------------------------------------------------------# Query construction# ---------------------------------------------------------------------------