Graph-Native Infrastructure for Context and Accountable AI Systems
Python agent-memoryaiai-governanceai-infrastructureartificial-intelligencecontext-engineeringcontext-graphsdata-engineeringdecision-intelligencedeveloper-toolsexplainable-aigenerative-aigraph-ragknowledge-graphllmontologyprovenancepythonreasoningsemantic-search
fix: improve SQL fragment validation (#1696)
Sameer Kadam32102f55ced5
diff --git a/semantica/ingest/databricks_ingestor.py b/semantica/ingest/databricks_ingestor.pyindex 3a82ed33..09d0a8ff 100644--- a/semantica/ingest/databricks_ingestor.py+++ b/semantica/ingest/databricks_ingestor.py@@ -46,6 +46,7 @@ from typing import Any, Dict, List, Optionalfrom ..utils.exceptions import ProcessingError, ValidationErrorfrom ..utils.logging import get_loggerfrom ..utils.progress_tracker import get_progress_tracker+from .db_ingestor import _validate_sql_fragmenttry:from databricks import sql as databricks_sql@@ -458,6 +459,11 @@ class DatabricksIngestor:catalog = catalog or self.connector.catalogschema = schema or self.connector.schema+ # Validate SQL fragments BEFORE opening any connection so invalid+ # inputs are rejected cheaply without a network round-trip.+ if where:+ _validate_sql_fragment(where, "where")+tracking_id = self.progress_tracker.start_tracking(file=f"{catalog}.{schema}.{table_name}",module="ingest",@@ -474,10 +480,6 @@ class DatabricksIngestor:query = f"SELECT * FROM {table_ref}"if where:- if ";" in where:- raise ValueError(- "Invalid WHERE clause: semicolons not permitted."- )query += f" WHERE {where}"if order_by:@@ -542,6 +544,16 @@ class DatabricksIngestor:if not already_connected:self.connector.disconnect()+ except ValidationError:+ self.progress_tracker.stop_tracking(+ tracking_id, status="failed", message="Validation failed"+ )+ raise+ except ProcessingError:+ self.progress_tracker.stop_tracking(+ tracking_id, status="failed", message="Processing failed"+ )+ raiseexcept Exception as e:self.progress_tracker.stop_tracking(tracking_id, status="failed", message=str(e)diff --git a/semantica/ingest/db_ingestor.py b/semantica/ingest/db_ingestor.pyindex 0d656dee..6d986b6d 100644--- a/semantica/ingest/db_ingestor.py+++ b/semantica/ingest/db_ingestor.py@@ -53,13 +53,15 @@ _SQL_FRAGMENT_BLOCKLIST_RE = re.compile(re.IGNORECASE,)-# SQL single-quoted string literals ('' is the standard escaped-quote) and-# double-quoted identifiers ("" likewise) — matched only when properly-# closed, so a malformed/unterminated quote sequence is left alone and-# still hits the blocklist above rather than being treated as "inside a-# literal" and skipped.+# SQL single-quoted string literals ('' is the standard escaped-quote),+# double-quoted identifiers ("" likewise), and backtick-quoted identifiers+# (`` is the backtick escape used by MySQL and Databricks SQL) — matched+# only when properly closed, so a malformed/unterminated quote sequence is+# left alone and still hits the blocklist above rather than being treated as+# "inside a literal" and skipped._SQL_STRING_LITERAL_RE = re.compile(r"'(?:[^']|'')*'")_SQL_QUOTED_IDENTIFIER_RE = re.compile(r'"(?:[^"]|"")*"')+_SQL_BACKTICK_IDENTIFIER_RE = re.compile(r"`(?:[^`]|``)*`")def _mask_sql_literals(fragment: str) -> str:@@ -67,11 +69,12 @@ def _mask_sql_literals(fragment: str) -> str:A legitimate value or quoted identifier that happens to contain ablocked word or character as *data* — e.g. ``status = 'union'`` or- ``"my--column" = 1`` — is not SQL syntax and shouldn't be rejected as- if it were. Only the quoted span's interior is replaced (with `?`,- keeping the surrounding quotes and the fragment's length/positions- intact for the error message); text outside any properly closed quote- is passed through unchanged and still fully scrutinized.+ ``"my--column" = 1`` or the Databricks/MySQL backtick form+ `` `union` `` — is not SQL syntax and shouldn't be rejected as if it+ were. Only the quoted span's interior is replaced (with `?`, keeping+ the surrounding quotes and the fragment's length/positions intact for+ the error message); text outside any properly closed quote is passed+ through unchanged and still fully scrutinised."""fragment = _SQL_STRING_LITERAL_RE.sub(lambda m: "'" + "?" * (len(m.group(0)) - 2) + "'", fragment@@ -79,6 +82,9 @@ def _mask_sql_literals(fragment: str) -> str:fragment = _SQL_QUOTED_IDENTIFIER_RE.sub(lambda m: '"' + "?" * (len(m.group(0)) - 2) + '"', fragment)+ fragment = _SQL_BACKTICK_IDENTIFIER_RE.sub(+ lambda m: "`" + "?" * (len(m.group(0)) - 2) + "`", fragment+ )return fragmentdiff --git a/semantica/ingest/snowflake_ingestor.py b/semantica/ingest/snowflake_ingestor.pyindex f81c95da..5bedf0ba 100644--- a/semantica/ingest/snowflake_ingestor.py+++ b/semantica/ingest/snowflake_ingestor.py@@ -45,6 +45,7 @@ from typing import Any, Dict, List, Optionalfrom ..utils.exceptions import ProcessingError, ValidationErrorfrom ..utils.logging import get_loggerfrom ..utils.progress_tracker import get_progress_tracker+from .db_ingestor import _validate_sql_fragmenttry:import snowflake.connector@@ -480,6 +481,11 @@ class SnowflakeIngestor:database = database or self.connector.databaseschema = schema or self.connector.schema+ # Validate SQL fragments BEFORE opening any connection so invalid+ # inputs are rejected cheaply without a network round-trip.+ if where:+ _validate_sql_fragment(where, "where")+tracking_id = self.progress_tracker.start_tracking(file=f"{database}.{schema}.{table_name}",module="ingest",@@ -507,9 +513,6 @@ class SnowflakeIngestor:if where:# where is appended verbatim — callers MUST only pass# trusted, application-controlled predicates here.- # Reject obvious injection attempts: multiple statements.- if ";" in where:- raise ValueError("Invalid WHERE clause: semicolons not permitted.")query += f" WHERE {where}"if order_by:@@ -572,6 +575,16 @@ class SnowflakeIngestor:metadata={"query": query},)+ except ValidationError:+ self.progress_tracker.stop_tracking(+ tracking_id, status="failed", message="Validation failed"+ )+ raise+ except ProcessingError:+ self.progress_tracker.stop_tracking(+ tracking_id, status="failed", message="Processing failed"+ )+ raiseexcept Exception as e:self.progress_tracker.stop_tracking(tracking_id, status="failed", message=str(e)diff --git a/tests/test_databricks_ingestor.py b/tests/test_databricks_ingestor.pyindex 44473afb..ae756b03 100644--- a/tests/test_databricks_ingestor.py+++ b/tests/test_databricks_ingestor.py@@ -877,6 +877,151 @@ class TestDatabricksIngestor:token="test_token",)+ # ------------------------------------------------------------------+ # WHERE-clause validation regression tests+ # ------------------------------------------------------------------++ @patch("semantica.ingest.databricks_ingestor.DATABRICKS_AVAILABLE", True)+ @patch("semantica.ingest.databricks_ingestor.databricks_sql")+ def test_ingest_table_union_in_where_raises_validation_error(self, mock_sql):+ """UNION in WHERE must raise ValidationError, not just block semicolons."""+ from semantica.ingest.databricks_ingestor import DatabricksIngestor+ from semantica.utils.exceptions import ValidationError++ ingestor = DatabricksIngestor(+ host="https://adb-xxx.azuredatabricks.net",+ token="test_token",+ http_path="/sql/1.0/warehouses/xxxx",+ )+ with pytest.raises(ValidationError):+ ingestor.ingest_table("customers", where="1=1 UNION SELECT * FROM secrets")++ @patch("semantica.ingest.databricks_ingestor.DATABRICKS_AVAILABLE", True)+ @patch("semantica.ingest.databricks_ingestor.databricks_sql")+ def test_ingest_table_comment_in_where_raises_validation_error(self, mock_sql):+ """SQL comment sequences in WHERE must raise ValidationError."""+ from semantica.ingest.databricks_ingestor import DatabricksIngestor+ from semantica.utils.exceptions import ValidationError++ ingestor = DatabricksIngestor(+ host="https://adb-xxx.azuredatabricks.net",+ token="test_token",+ http_path="/sql/1.0/warehouses/xxxx",+ )+ with pytest.raises(ValidationError):+ ingestor.ingest_table("customers", where="1=1 -- bypass")++ @patch("semantica.ingest.databricks_ingestor.DATABRICKS_AVAILABLE", True)+ @patch("semantica.ingest.databricks_ingestor.databricks_sql")+ def test_ingest_table_quoted_literal_union_passes(+ self, mock_sql, mock_databricks_connection+ ):+ """'union' inside a quoted string literal must NOT be rejected."""+ from semantica.ingest.databricks_ingestor import DatabricksIngestor++ mock_conn, mock_cursor = mock_databricks_connection+ mock_sql.connect = Mock(return_value=mock_conn)++ ingestor = DatabricksIngestor(+ host="https://adb-xxx.azuredatabricks.net",+ token="test_token",+ http_path="/sql/1.0/warehouses/xxxx",+ )+ # 'Credit Union' is a data value, not SQL syntax — must pass+ ingestor.ingest_table("customers", where="name = 'Credit Union'")++ executed_query = mock_cursor.execute.call_args[0][0]+ assert "WHERE name = 'Credit Union'" in executed_query++ @patch("semantica.ingest.databricks_ingestor.DATABRICKS_AVAILABLE", True)+ @patch("semantica.ingest.databricks_ingestor.databricks_sql")+ def test_invalid_where_rejected_before_connect(+ self, mock_sql, mock_databricks_connection+ ):+ """ValidationError must be raised before any connection is attempted."""+ from semantica.ingest.databricks_ingestor import DatabricksIngestor+ from semantica.utils.exceptions import ValidationError++ mock_conn, _ = mock_databricks_connection+ mock_sql.connect = Mock(return_value=mock_conn)++ ingestor = DatabricksIngestor(+ host="https://adb-xxx.azuredatabricks.net",+ token="test_token",+ http_path="/sql/1.0/warehouses/xxxx",+ )+ with pytest.raises(ValidationError):+ ingestor.ingest_table("customers", where="1=1 UNION SELECT * FROM secrets")++ mock_sql.connect.assert_not_called()++ @patch("semantica.ingest.databricks_ingestor.DATABRICKS_AVAILABLE", True)+ @patch("semantica.ingest.databricks_ingestor.databricks_sql")+ def test_backtick_quoted_identifier_with_blocked_word_passes(+ self, mock_sql, mock_databricks_connection+ ):+ """Backtick-quoted identifiers containing blocked words must not be rejected.++ Databricks SQL uses backtick quoting for column names that collide+ with reserved words or contain special characters, e.g.:+ WHERE `union` = 1+ WHERE `my--column` > 0+ These are valid column references, not injection syntax.+ """+ from semantica.ingest.databricks_ingestor import DatabricksIngestor++ mock_conn, mock_cursor = mock_databricks_connection+ mock_sql.connect = Mock(return_value=mock_conn)++ ingestor = DatabricksIngestor(+ host="https://adb-xxx.azuredatabricks.net",+ token="test_token",+ http_path="/sql/1.0/warehouses/xxxx",+ )+ # backtick-quoted column name that happens to spell a blocked word+ ingestor.ingest_table("events", where="`union` = 1")++ executed_query = mock_cursor.execute.call_args[0][0]+ assert "WHERE `union` = 1" in executed_query++ @patch("semantica.ingest.databricks_ingestor.DATABRICKS_AVAILABLE", True)+ @patch("semantica.ingest.databricks_ingestor.databricks_sql")+ def test_backtick_quoted_identifier_with_comment_chars_passes(+ self, mock_sql, mock_databricks_connection+ ):+ """Backtick-quoted identifier containing -- must not be rejected."""+ from semantica.ingest.databricks_ingestor import DatabricksIngestor++ mock_conn, mock_cursor = mock_databricks_connection+ mock_sql.connect = Mock(return_value=mock_conn)++ ingestor = DatabricksIngestor(+ host="https://adb-xxx.azuredatabricks.net",+ token="test_token",+ http_path="/sql/1.0/warehouses/xxxx",+ )+ ingestor.ingest_table("events", where="`my--column` > 0")++ executed_query = mock_cursor.execute.call_args[0][0]+ assert "WHERE `my--column` > 0" in executed_query++ @patch("semantica.ingest.databricks_ingestor.DATABRICKS_AVAILABLE", True)+ @patch("semantica.ingest.databricks_ingestor.databricks_sql")+ def test_bare_union_outside_backticks_still_rejected(+ self, mock_sql, mock_databricks_connection+ ):+ """UNION outside any quoting must still raise ValidationError."""+ from semantica.ingest.databricks_ingestor import DatabricksIngestor+ from semantica.utils.exceptions import ValidationError++ ingestor = DatabricksIngestor(+ host="https://adb-xxx.azuredatabricks.net",+ token="test_token",+ http_path="/sql/1.0/warehouses/xxxx",+ )+ with pytest.raises(ValidationError):+ ingestor.ingest_table("events", where="1=1 UNION SELECT * FROM secrets")+class TestDatabricksData:"""Test DatabricksData dataclass."""diff --git a/tests/test_snowflake_ingestor.py b/tests/test_snowflake_ingestor.pyindex f931f58f..0ad267b9 100644--- a/tests/test_snowflake_ingestor.py+++ b/tests/test_snowflake_ingestor.py@@ -568,6 +568,80 @@ class TestSnowflakeIngestor:password="test_password",)+ # ------------------------------------------------------------------+ # WHERE-clause validation regression tests+ # ------------------------------------------------------------------++ def test_ingest_table_union_in_where_raises_validation_error(self):+ """UNION in WHERE must raise ValidationError, not just block semicolons."""+ from semantica.ingest.snowflake_ingestor import SnowflakeIngestor+ from semantica.utils.exceptions import ValidationError++ ingestor = SnowflakeIngestor(+ account="test_account",+ user="test_user",+ password="test_password",+ )+ with pytest.raises(ValidationError):+ ingestor.ingest_table("CUSTOMERS", where="1=1 UNION SELECT * FROM secrets")++ def test_ingest_table_comment_in_where_raises_validation_error(self):+ """SQL comment sequences in WHERE must raise ValidationError."""+ from semantica.ingest.snowflake_ingestor import SnowflakeIngestor+ from semantica.utils.exceptions import ValidationError++ ingestor = SnowflakeIngestor(+ account="test_account",+ user="test_user",+ password="test_password",+ )+ with pytest.raises(ValidationError):+ ingestor.ingest_table("CUSTOMERS", where="1=1 -- bypass")++ @patch("semantica.ingest.snowflake_ingestor.SNOWFLAKE_AVAILABLE", True)+ @patch("semantica.ingest.snowflake_ingestor.snowflake")+ def test_ingest_table_quoted_literal_union_passes(+ self, mock_snowflake, mock_snowflake_connection+ ):+ """'union' inside a quoted string literal must NOT be rejected."""+ from semantica.ingest.snowflake_ingestor import SnowflakeIngestor++ mock_conn, mock_cursor = mock_snowflake_connection+ mock_snowflake.connector.connect = Mock(return_value=mock_conn)++ ingestor = SnowflakeIngestor(+ account="test_account",+ user="test_user",+ password="test_password",+ )+ # 'Credit Union' is a data value, not SQL syntax — must pass+ ingestor.ingest_table("CUSTOMERS", where="NAME = 'Credit Union'")++ executed_query = mock_cursor.execute.call_args[0][0]+ assert "WHERE NAME = 'Credit Union'" in executed_query++ @patch("semantica.ingest.snowflake_ingestor.SNOWFLAKE_AVAILABLE", True)+ @patch("semantica.ingest.snowflake_ingestor.snowflake")+ def test_invalid_where_rejected_before_connect(+ self, mock_snowflake, mock_snowflake_connection+ ):+ """ValidationError must be raised before any connection is attempted."""+ from semantica.ingest.snowflake_ingestor import SnowflakeIngestor+ from semantica.utils.exceptions import ValidationError++ mock_conn, _ = mock_snowflake_connection+ mock_snowflake.connector.connect = Mock(return_value=mock_conn)++ ingestor = SnowflakeIngestor(+ account="test_account",+ user="test_user",+ password="test_password",+ )+ with pytest.raises(ValidationError):+ ingestor.ingest_table("CUSTOMERS", where="1=1 UNION SELECT * FROM secrets")++ mock_snowflake.connector.connect.assert_not_called()+class TestSnowflakeData:"""Test SnowflakeData dataclass."""