Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

fix: the bigquery connector constructs sql queries u... in... (#17500)

Anupam Medirattad0986424ee33
diff --git a/common/data_source/bigquery_connector.py b/common/data_source/bigquery_connector.py
index 09ff99db48..89558371e6 100644
--- a/common/data_source/bigquery_connector.py
+++ b/common/data_source/bigquery_connector.py
@@ -16,6 +16,7 @@ import copy
import hashlib
import json
import logging
+import re
from datetime import date, datetime, time, timezone
from decimal import Decimal
from typing import Any, Dict, Generator, List, Optional, Tuple
@@ -48,6 +49,28 @@ _CURSOR_TYPE_KEY = "__ragflow_bq_cursor_type__"
# Default cost guard: 1 GiB. Users can raise this explicitly.
DEFAULT_MAXIMUM_BYTES_BILLED = 1024 * 1024 * 1024
+# Allowlist for standard BigQuery unquoted identifiers (columns, datasets, tables).
+_IDENTIFIER_RE = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$")
+# GCP project IDs may contain hyphens.
+_PROJECT_ID_RE = re.compile(r"^[A-Za-z][A-Za-z0-9_-]*$")
+
+
+def _validate_identifier(value: Optional[str], name: str) -> Optional[str]:
+ if not value:
+ return value
+ if not _IDENTIFIER_RE.fullmatch(value):
+ raise ConnectorValidationError(f"Invalid BigQuery identifier for {name!r}")
+ return value
+
+
+def _validate_project_id(value: str, name: str) -> str:
+ if not value:
+ return value
+ if not _PROJECT_ID_RE.fullmatch(value):
+ raise ConnectorValidationError(f"Invalid BigQuery identifier for {name!r}")
+ return value
+
+
# Maps a BigQuery field type to the ScalarQueryParameter type used for cursors.
_CURSOR_PARAM_TYPE_MAP = {
"TIMESTAMP": "TIMESTAMP",
@@ -115,15 +138,27 @@ class BigQueryConnector(LoadConnector, PollConnector, SlimConnectorWithPermSync)
job_timeout_ms: Optional per-job timeout in milliseconds.
use_query_cache: Whether to allow BigQuery's query result cache.
"""
- self.project_id = (project_id or "").strip()
- self.dataset_id = (dataset_id or "").strip()
- self.table_id = (table_id or "").strip()
+ self.project_id = _validate_project_id((project_id or "").strip(), "project_id")
+ self.dataset_id = _validate_identifier((dataset_id or "").strip(), "dataset_id")
+ self.table_id = _validate_identifier((table_id or "").strip(), "table_id")
self.location = (location or "").strip()
self.query = (query or "").strip()
- self.content_columns = [c.strip() for c in (content_columns or "").split(",") if c.strip()]
- self.metadata_columns = [c.strip() for c in (metadata_columns or "").split(",") if c.strip()]
- self.id_column = id_column.strip() if id_column else None
- self.timestamp_column = timestamp_column.strip() if timestamp_column else None
+ _content_cols: List[str] = []
+ for _c in (content_columns or "").split(","):
+ _col = _c.strip()
+ if _col:
+ _validate_identifier(_col, f"content_columns[{_col!r}]")
+ _content_cols.append(_col)
+ self.content_columns = _content_cols
+ _meta_cols: List[str] = []
+ for _c in (metadata_columns or "").split(","):
+ _col = _c.strip()
+ if _col:
+ _validate_identifier(_col, f"metadata_columns[{_col!r}]")
+ _meta_cols.append(_col)
+ self.metadata_columns = _meta_cols
+ self.id_column = _validate_identifier(id_column.strip() if id_column else None, "id_column")
+ self.timestamp_column = _validate_identifier(timestamp_column.strip() if timestamp_column else None, "timestamp_column")
self.batch_size = batch_size
self.page_size = page_size
self.maximum_bytes_billed = maximum_bytes_billed
@@ -172,7 +207,7 @@ class BigQueryConnector(LoadConnector, PollConnector, SlimConnectorWithPermSync)
Accepts ``service_account_json`` as either a dict or a JSON string.
"""
- logging.debug("Loading credentials for BigQuery project: %s", self.project_id)
+ logging.debug("Loading credentials for BigQuery connector.")
raw = (credentials or {}).get("service_account_json")
if not raw:
@@ -225,6 +260,7 @@ class BigQueryConnector(LoadConnector, PollConnector, SlimConnectorWithPermSync)
def _build_base_query(self) -> str:
"""Return the single base query (custom query takes precedence over table mode)."""
if self.query:
+ # custom query is trusted, administrator-supplied SQL — not validated as an identifier.
return self.query.rstrip(";")
if self.dataset_id and self.table_id:
return f"SELECT * FROM `{self.project_id}.{self.dataset_id}.{self.table_id}`"
diff --git a/test/unit_test/data_source/test_bigquery_connector.py b/test/unit_test/data_source/test_bigquery_connector.py
index 56eeee9fcb..8514ac473e 100644
--- a/test/unit_test/data_source/test_bigquery_connector.py
+++ b/test/unit_test/data_source/test_bigquery_connector.py
@@ -148,6 +148,51 @@ def test_invalid_credentials_json_raises():
connector.load_credentials({"service_account_json": "{not-json"})
+# ---------------------------------------------------------------------------
+# Identifier validation
+# ---------------------------------------------------------------------------
+
+
+ "field,value",
+ [
+ ("project_id", "foo` WHERE 1=1 --"),
+ ("dataset_id", "foo` WHERE 1=1 --"),
+ ("table_id", "foo` WHERE 1=1 --"),
+ ("id_column", "foo; DROP TABLE users --"),
+ ("timestamp_column", "ts OR 1=1"),
+ ("content_columns", "name` UNION SELECT password FROM users --"),
+ ("metadata_columns", "status` UNION SELECT password FROM users --"),
+ ],
+)
+def test_rejects_malicious_identifier(field, value):
+ kwargs = dict(
+ project_id="my-proj",
+ dataset_id="ds",
+ table_id="tbl",
+ content_columns="name",
+ )
+ kwargs[field] = value
+ with pytest.raises(ConnectorValidationError):
+ BigQueryConnector(**kwargs)
+
+
+def test_accepts_valid_identifiers():
+ connector = BigQueryConnector(
+ project_id="my-project",
+ dataset_id="my_dataset",
+ table_id="my_table",
+ content_columns="name,description",
+ id_column="id",
+ timestamp_column="created_at",
+ )
+ assert connector.project_id == "my-project"
+ assert connector.id_column == "id"
+ assert connector.timestamp_column == "created_at"
+
+
# ---------------------------------------------------------------------------
# Query construction
# ---------------------------------------------------------------------------