Skip to content

Graph-Native Infrastructure for Context and Accountable AI Systems

fix: improve SQL fragment validation (#1696)

Sameer Kadam32102f55ced5
diff --git a/semantica/ingest/databricks_ingestor.py b/semantica/ingest/databricks_ingestor.py
index 3a82ed33..09d0a8ff 100644
--- a/semantica/ingest/databricks_ingestor.py
+++ b/semantica/ingest/databricks_ingestor.py
@@ -46,6 +46,7 @@ from typing import Any, Dict, List, Optional
from ..utils.exceptions import ProcessingError, ValidationError
from ..utils.logging import get_logger
from ..utils.progress_tracker import get_progress_tracker
+from .db_ingestor import _validate_sql_fragment
try:
from databricks import sql as databricks_sql
@@ -458,6 +459,11 @@ class DatabricksIngestor:
catalog = catalog or self.connector.catalog
schema = schema or self.connector.schema
+ # Validate SQL fragments BEFORE opening any connection so invalid
+ # inputs are rejected cheaply without a network round-trip.
+ if where:
+ _validate_sql_fragment(where, "where")
+
tracking_id = self.progress_tracker.start_tracking(
file=f"{catalog}.{schema}.{table_name}",
module="ingest",
@@ -474,10 +480,6 @@ class DatabricksIngestor:
query = f"SELECT * FROM {table_ref}"
if where:
- if ";" in where:
- raise ValueError(
- "Invalid WHERE clause: semicolons not permitted."
- )
query += f" WHERE {where}"
if order_by:
@@ -542,6 +544,16 @@ class DatabricksIngestor:
if not already_connected:
self.connector.disconnect()
+ except ValidationError:
+ self.progress_tracker.stop_tracking(
+ tracking_id, status="failed", message="Validation failed"
+ )
+ raise
+ except ProcessingError:
+ self.progress_tracker.stop_tracking(
+ tracking_id, status="failed", message="Processing failed"
+ )
+ raise
except Exception as e:
self.progress_tracker.stop_tracking(
tracking_id, status="failed", message=str(e)
diff --git a/semantica/ingest/db_ingestor.py b/semantica/ingest/db_ingestor.py
index 0d656dee..6d986b6d 100644
--- a/semantica/ingest/db_ingestor.py
+++ b/semantica/ingest/db_ingestor.py
@@ -53,13 +53,15 @@ _SQL_FRAGMENT_BLOCKLIST_RE = re.compile(
re.IGNORECASE,
)
-# SQL single-quoted string literals ('' is the standard escaped-quote) and
-# double-quoted identifiers ("" likewise) — matched only when properly
-# closed, so a malformed/unterminated quote sequence is left alone and
-# still hits the blocklist above rather than being treated as "inside a
-# literal" and skipped.
+# SQL single-quoted string literals ('' is the standard escaped-quote),
+# double-quoted identifiers ("" likewise), and backtick-quoted identifiers
+# (`` is the backtick escape used by MySQL and Databricks SQL) — matched
+# only when properly closed, so a malformed/unterminated quote sequence is
+# left alone and still hits the blocklist above rather than being treated as
+# "inside a literal" and skipped.
_SQL_STRING_LITERAL_RE = re.compile(r"'(?:[^']|'')*'")
_SQL_QUOTED_IDENTIFIER_RE = re.compile(r'"(?:[^"]|"")*"')
+_SQL_BACKTICK_IDENTIFIER_RE = re.compile(r"`(?:[^`]|``)*`")
def _mask_sql_literals(fragment: str) -> str:
@@ -67,11 +69,12 @@ def _mask_sql_literals(fragment: str) -> str:
A legitimate value or quoted identifier that happens to contain a
blocked word or character as *data* — e.g. ``status = 'union'`` or
- ``"my--column" = 1`` — is not SQL syntax and shouldn't be rejected as
- if it were. Only the quoted span's interior is replaced (with `?`,
- keeping the surrounding quotes and the fragment's length/positions
- intact for the error message); text outside any properly closed quote
- is passed through unchanged and still fully scrutinized.
+ ``"my--column" = 1`` or the Databricks/MySQL backtick form
+ `` `union` `` — is not SQL syntax and shouldn't be rejected as if it
+ were. Only the quoted span's interior is replaced (with `?`, keeping
+ the surrounding quotes and the fragment's length/positions intact for
+ the error message); text outside any properly closed quote is passed
+ through unchanged and still fully scrutinised.
"""
fragment = _SQL_STRING_LITERAL_RE.sub(
lambda m: "'" + "?" * (len(m.group(0)) - 2) + "'", fragment
@@ -79,6 +82,9 @@ def _mask_sql_literals(fragment: str) -> str:
fragment = _SQL_QUOTED_IDENTIFIER_RE.sub(
lambda m: '"' + "?" * (len(m.group(0)) - 2) + '"', fragment
)
+ fragment = _SQL_BACKTICK_IDENTIFIER_RE.sub(
+ lambda m: "`" + "?" * (len(m.group(0)) - 2) + "`", fragment
+ )
return fragment
diff --git a/semantica/ingest/snowflake_ingestor.py b/semantica/ingest/snowflake_ingestor.py
index f81c95da..5bedf0ba 100644
--- a/semantica/ingest/snowflake_ingestor.py
+++ b/semantica/ingest/snowflake_ingestor.py
@@ -45,6 +45,7 @@ from typing import Any, Dict, List, Optional
from ..utils.exceptions import ProcessingError, ValidationError
from ..utils.logging import get_logger
from ..utils.progress_tracker import get_progress_tracker
+from .db_ingestor import _validate_sql_fragment
try:
import snowflake.connector
@@ -480,6 +481,11 @@ class SnowflakeIngestor:
database = database or self.connector.database
schema = schema or self.connector.schema
+ # Validate SQL fragments BEFORE opening any connection so invalid
+ # inputs are rejected cheaply without a network round-trip.
+ if where:
+ _validate_sql_fragment(where, "where")
+
tracking_id = self.progress_tracker.start_tracking(
file=f"{database}.{schema}.{table_name}",
module="ingest",
@@ -507,9 +513,6 @@ class SnowflakeIngestor:
if where:
# where is appended verbatim — callers MUST only pass
# trusted, application-controlled predicates here.
- # Reject obvious injection attempts: multiple statements.
- if ";" in where:
- raise ValueError("Invalid WHERE clause: semicolons not permitted.")
query += f" WHERE {where}"
if order_by:
@@ -572,6 +575,16 @@ class SnowflakeIngestor:
metadata={"query": query},
)
+ except ValidationError:
+ self.progress_tracker.stop_tracking(
+ tracking_id, status="failed", message="Validation failed"
+ )
+ raise
+ except ProcessingError:
+ self.progress_tracker.stop_tracking(
+ tracking_id, status="failed", message="Processing failed"
+ )
+ raise
except Exception as e:
self.progress_tracker.stop_tracking(
tracking_id, status="failed", message=str(e)
diff --git a/tests/test_databricks_ingestor.py b/tests/test_databricks_ingestor.py
index 44473afb..ae756b03 100644
--- a/tests/test_databricks_ingestor.py
+++ b/tests/test_databricks_ingestor.py
@@ -877,6 +877,151 @@ class TestDatabricksIngestor:
token="test_token",
)
+ # ------------------------------------------------------------------
+ # WHERE-clause validation regression tests
+ # ------------------------------------------------------------------
+
+ @patch("semantica.ingest.databricks_ingestor.DATABRICKS_AVAILABLE", True)
+ @patch("semantica.ingest.databricks_ingestor.databricks_sql")
+ def test_ingest_table_union_in_where_raises_validation_error(self, mock_sql):
+ """UNION in WHERE must raise ValidationError, not just block semicolons."""
+ from semantica.ingest.databricks_ingestor import DatabricksIngestor
+ from semantica.utils.exceptions import ValidationError
+
+ ingestor = DatabricksIngestor(
+ host="https://adb-xxx.azuredatabricks.net",
+ token="test_token",
+ http_path="/sql/1.0/warehouses/xxxx",
+ )
+ with pytest.raises(ValidationError):
+ ingestor.ingest_table("customers", where="1=1 UNION SELECT * FROM secrets")
+
+ @patch("semantica.ingest.databricks_ingestor.DATABRICKS_AVAILABLE", True)
+ @patch("semantica.ingest.databricks_ingestor.databricks_sql")
+ def test_ingest_table_comment_in_where_raises_validation_error(self, mock_sql):
+ """SQL comment sequences in WHERE must raise ValidationError."""
+ from semantica.ingest.databricks_ingestor import DatabricksIngestor
+ from semantica.utils.exceptions import ValidationError
+
+ ingestor = DatabricksIngestor(
+ host="https://adb-xxx.azuredatabricks.net",
+ token="test_token",
+ http_path="/sql/1.0/warehouses/xxxx",
+ )
+ with pytest.raises(ValidationError):
+ ingestor.ingest_table("customers", where="1=1 -- bypass")
+
+ @patch("semantica.ingest.databricks_ingestor.DATABRICKS_AVAILABLE", True)
+ @patch("semantica.ingest.databricks_ingestor.databricks_sql")
+ def test_ingest_table_quoted_literal_union_passes(
+ self, mock_sql, mock_databricks_connection
+ ):
+ """'union' inside a quoted string literal must NOT be rejected."""
+ from semantica.ingest.databricks_ingestor import DatabricksIngestor
+
+ mock_conn, mock_cursor = mock_databricks_connection
+ mock_sql.connect = Mock(return_value=mock_conn)
+
+ ingestor = DatabricksIngestor(
+ host="https://adb-xxx.azuredatabricks.net",
+ token="test_token",
+ http_path="/sql/1.0/warehouses/xxxx",
+ )
+ # 'Credit Union' is a data value, not SQL syntax — must pass
+ ingestor.ingest_table("customers", where="name = 'Credit Union'")
+
+ executed_query = mock_cursor.execute.call_args[0][0]
+ assert "WHERE name = 'Credit Union'" in executed_query
+
+ @patch("semantica.ingest.databricks_ingestor.DATABRICKS_AVAILABLE", True)
+ @patch("semantica.ingest.databricks_ingestor.databricks_sql")
+ def test_invalid_where_rejected_before_connect(
+ self, mock_sql, mock_databricks_connection
+ ):
+ """ValidationError must be raised before any connection is attempted."""
+ from semantica.ingest.databricks_ingestor import DatabricksIngestor
+ from semantica.utils.exceptions import ValidationError
+
+ mock_conn, _ = mock_databricks_connection
+ mock_sql.connect = Mock(return_value=mock_conn)
+
+ ingestor = DatabricksIngestor(
+ host="https://adb-xxx.azuredatabricks.net",
+ token="test_token",
+ http_path="/sql/1.0/warehouses/xxxx",
+ )
+ with pytest.raises(ValidationError):
+ ingestor.ingest_table("customers", where="1=1 UNION SELECT * FROM secrets")
+
+ mock_sql.connect.assert_not_called()
+
+ @patch("semantica.ingest.databricks_ingestor.DATABRICKS_AVAILABLE", True)
+ @patch("semantica.ingest.databricks_ingestor.databricks_sql")
+ def test_backtick_quoted_identifier_with_blocked_word_passes(
+ self, mock_sql, mock_databricks_connection
+ ):
+ """Backtick-quoted identifiers containing blocked words must not be rejected.
+
+ Databricks SQL uses backtick quoting for column names that collide
+ with reserved words or contain special characters, e.g.:
+ WHERE `union` = 1
+ WHERE `my--column` > 0
+ These are valid column references, not injection syntax.
+ """
+ from semantica.ingest.databricks_ingestor import DatabricksIngestor
+
+ mock_conn, mock_cursor = mock_databricks_connection
+ mock_sql.connect = Mock(return_value=mock_conn)
+
+ ingestor = DatabricksIngestor(
+ host="https://adb-xxx.azuredatabricks.net",
+ token="test_token",
+ http_path="/sql/1.0/warehouses/xxxx",
+ )
+ # backtick-quoted column name that happens to spell a blocked word
+ ingestor.ingest_table("events", where="`union` = 1")
+
+ executed_query = mock_cursor.execute.call_args[0][0]
+ assert "WHERE `union` = 1" in executed_query
+
+ @patch("semantica.ingest.databricks_ingestor.DATABRICKS_AVAILABLE", True)
+ @patch("semantica.ingest.databricks_ingestor.databricks_sql")
+ def test_backtick_quoted_identifier_with_comment_chars_passes(
+ self, mock_sql, mock_databricks_connection
+ ):
+ """Backtick-quoted identifier containing -- must not be rejected."""
+ from semantica.ingest.databricks_ingestor import DatabricksIngestor
+
+ mock_conn, mock_cursor = mock_databricks_connection
+ mock_sql.connect = Mock(return_value=mock_conn)
+
+ ingestor = DatabricksIngestor(
+ host="https://adb-xxx.azuredatabricks.net",
+ token="test_token",
+ http_path="/sql/1.0/warehouses/xxxx",
+ )
+ ingestor.ingest_table("events", where="`my--column` > 0")
+
+ executed_query = mock_cursor.execute.call_args[0][0]
+ assert "WHERE `my--column` > 0" in executed_query
+
+ @patch("semantica.ingest.databricks_ingestor.DATABRICKS_AVAILABLE", True)
+ @patch("semantica.ingest.databricks_ingestor.databricks_sql")
+ def test_bare_union_outside_backticks_still_rejected(
+ self, mock_sql, mock_databricks_connection
+ ):
+ """UNION outside any quoting must still raise ValidationError."""
+ from semantica.ingest.databricks_ingestor import DatabricksIngestor
+ from semantica.utils.exceptions import ValidationError
+
+ ingestor = DatabricksIngestor(
+ host="https://adb-xxx.azuredatabricks.net",
+ token="test_token",
+ http_path="/sql/1.0/warehouses/xxxx",
+ )
+ with pytest.raises(ValidationError):
+ ingestor.ingest_table("events", where="1=1 UNION SELECT * FROM secrets")
+
class TestDatabricksData:
"""Test DatabricksData dataclass."""
diff --git a/tests/test_snowflake_ingestor.py b/tests/test_snowflake_ingestor.py
index f931f58f..0ad267b9 100644
--- a/tests/test_snowflake_ingestor.py
+++ b/tests/test_snowflake_ingestor.py
@@ -568,6 +568,80 @@ class TestSnowflakeIngestor:
password="test_password",
)
+ # ------------------------------------------------------------------
+ # WHERE-clause validation regression tests
+ # ------------------------------------------------------------------
+
+ def test_ingest_table_union_in_where_raises_validation_error(self):
+ """UNION in WHERE must raise ValidationError, not just block semicolons."""
+ from semantica.ingest.snowflake_ingestor import SnowflakeIngestor
+ from semantica.utils.exceptions import ValidationError
+
+ ingestor = SnowflakeIngestor(
+ account="test_account",
+ user="test_user",
+ password="test_password",
+ )
+ with pytest.raises(ValidationError):
+ ingestor.ingest_table("CUSTOMERS", where="1=1 UNION SELECT * FROM secrets")
+
+ def test_ingest_table_comment_in_where_raises_validation_error(self):
+ """SQL comment sequences in WHERE must raise ValidationError."""
+ from semantica.ingest.snowflake_ingestor import SnowflakeIngestor
+ from semantica.utils.exceptions import ValidationError
+
+ ingestor = SnowflakeIngestor(
+ account="test_account",
+ user="test_user",
+ password="test_password",
+ )
+ with pytest.raises(ValidationError):
+ ingestor.ingest_table("CUSTOMERS", where="1=1 -- bypass")
+
+ @patch("semantica.ingest.snowflake_ingestor.SNOWFLAKE_AVAILABLE", True)
+ @patch("semantica.ingest.snowflake_ingestor.snowflake")
+ def test_ingest_table_quoted_literal_union_passes(
+ self, mock_snowflake, mock_snowflake_connection
+ ):
+ """'union' inside a quoted string literal must NOT be rejected."""
+ from semantica.ingest.snowflake_ingestor import SnowflakeIngestor
+
+ mock_conn, mock_cursor = mock_snowflake_connection
+ mock_snowflake.connector.connect = Mock(return_value=mock_conn)
+
+ ingestor = SnowflakeIngestor(
+ account="test_account",
+ user="test_user",
+ password="test_password",
+ )
+ # 'Credit Union' is a data value, not SQL syntax — must pass
+ ingestor.ingest_table("CUSTOMERS", where="NAME = 'Credit Union'")
+
+ executed_query = mock_cursor.execute.call_args[0][0]
+ assert "WHERE NAME = 'Credit Union'" in executed_query
+
+ @patch("semantica.ingest.snowflake_ingestor.SNOWFLAKE_AVAILABLE", True)
+ @patch("semantica.ingest.snowflake_ingestor.snowflake")
+ def test_invalid_where_rejected_before_connect(
+ self, mock_snowflake, mock_snowflake_connection
+ ):
+ """ValidationError must be raised before any connection is attempted."""
+ from semantica.ingest.snowflake_ingestor import SnowflakeIngestor
+ from semantica.utils.exceptions import ValidationError
+
+ mock_conn, _ = mock_snowflake_connection
+ mock_snowflake.connector.connect = Mock(return_value=mock_conn)
+
+ ingestor = SnowflakeIngestor(
+ account="test_account",
+ user="test_user",
+ password="test_password",
+ )
+ with pytest.raises(ValidationError):
+ ingestor.ingest_table("CUSTOMERS", where="1=1 UNION SELECT * FROM secrets")
+
+ mock_snowflake.connector.connect.assert_not_called()
+
class TestSnowflakeData:
"""Test SnowflakeData dataclass."""