Graph-Native Infrastructure for Context and Accountable AI Systems
Python agent-memoryaiai-governanceai-infrastructureartificial-intelligencecontext-engineeringcontext-graphsdata-engineeringdecision-intelligencedeveloper-toolsexplainable-aigenerative-aigraph-ragknowledge-graphllmontologyprovenancepythonreasoningsemantic-search
fix: improve metadata handling (#1734)
Sameer Kadam4b9fda9bf611
diff --git a/semantica/ingest/file_ingestor.py b/semantica/ingest/file_ingestor.pyindex f2a4ca41..61db6cb4 100644--- a/semantica/ingest/file_ingestor.py+++ b/semantica/ingest/file_ingestor.py@@ -39,6 +39,66 @@ from ..utils.exceptions import ProcessingError, ValidationErrorfrom ..utils.logging import get_loggerfrom ..utils.progress_tracker import get_progress_tracker+# ---------------------------------------------------------------------------+# Sensitive-key filter for FileObject.metadata (Hotspot 7)+# ---------------------------------------------------------------------------+# Credential-like option keys must never be copied into FileObject.metadata,+# because metadata is a plain public dict that callers can inspect, serialize,+# export to graph/CSV/JSON, or pass to downstream processors.+#+# Matching is case-insensitive (see _safe_options usage in ingest_file).+# Exact-name matching is used deliberately — following the explicit convention+# in semantica/semantic_extract/cache.py: "Exact-match (not substring) so+# legitimate params such as 'max_tokens' are never dropped." Substring+# matching would silently drop innocuous metadata keys.+#+# Sources used to determine this set:+# - semantica/semantic_extract/cache.py (LLM API credential keys)+# - semantica/ingest/looker_ingestor.py (_SECRET_FIELD_NAMES)+# - semantica/ingest/file_ingestor.py (CloudStorageIngestor: S3/Azure)+# - semantica/ingest/salesforce_ingestor.py (security_token, privatekey, …)+# - semantica/ingest/redshift_ingestor.py (secret_access_key, …)+# - semantica/ingest/servicenow_ingestor.py (client_secret, …)+# - semantica/server.py / explorer (X-API-Key HTTP header name)+_SENSITIVE_METADATA_KEYS: frozenset = frozenset(+ {+ # --- LLM / generic API keys (from cache.py baseline) ---+ "api_key",+ "apikey",+ "api_secret",+ "token",+ "access_token",+ "refresh_token",+ "session_token",+ "bearer_token",+ "password",+ "secret",+ "client_secret",+ "private_key",+ "auth",+ "authorization",+ "credential",+ "credentials",+ # --- Cloud storage credentials (CloudStorageIngestor: S3 / Azure) ---+ "secret_access_key", # boto3 / S3 / Redshift+ "access_key_id", # boto3 / S3 / Redshift+ "connection_string", # Azure Blob BlobServiceClient+ # --- Additional credential patterns found in this codebase's ingestors ---+ "auth_token", # generic auth-token kwarg+ "x-api-key", # HTTP header name (Explorer API)+ "security_token", # Salesforce SOAP auth+ "session_id", # Salesforce pre-existing session+ "consumer_key", # Salesforce JWT Bearer (public app id, but+ # often treated as sensitive in transit)+ "privatekey", # Salesforce JWT Bearer PEM string+ "privatekey_file", # Salesforce JWT Bearer PEM file path+ "deploy_secret", # Looker _SECRET_FIELD_NAMES+ "device_token", # Looker _SECRET_FIELD_NAMES+ "git_password", # Looker _SECRET_FIELD_NAMES+ "pdt_password", # Looker _SECRET_FIELD_NAMES+ }+)+@dataclassclass FileObject:@@ -631,6 +691,19 @@ class FileIngestor:# Detect MIME type for additional metadatamime_type, _ = mimetypes.guess_type(str(file_path))+ # Strip credential-like keys before they reach FileObject.metadata.+ # Sensitive kwargs (api_key, token, password, etc.) are accepted by+ # the public API for forwarding to backends, but must never be copied+ # into the returned object where they could be serialized, exported,+ # or observed by downstream consumers. Matching is case-insensitive+ # so API_KEY and Authorization are caught as well as their lowercase+ # forms. Non-sensitive custom kwargs still reach metadata unchanged.+ _safe_options = {+ k: v+ for k, v in options.items()+ if k.lower() not in _SENSITIVE_METADATA_KEYS+ }+# Create and return FileObject with all metadatafile_obj = FileObject(path=str(file_path.absolute()),@@ -644,7 +717,7 @@ class FileIngestor:"parent": str(file_path.parent),"is_supported": self.type_detector.is_supported(file_type),"read_content": read_content,- **options, # Include any additional options as metadata+ **_safe_options, # credential keys already stripped above},)diff --git a/tests/ingest/test_file_ingestor_metadata_security.py b/tests/ingest/test_file_ingestor_metadata_security.pynew file mode 100644index 00000000..6c69a905--- /dev/null+++ b/tests/ingest/test_file_ingestor_metadata_security.py@@ -0,0 +1,329 @@+"""+Security regression tests for Hotspot 7:+ FileObject.metadata must not retain credential-like kwargs.++These tests verify that sensitive option keys passed to ingest_file() are+stripped before being copied into FileObject.metadata, while non-sensitive+custom kwargs continue to reach metadata unchanged.++No real secret values are used. Every credential-like value is an+obvious placeholder string that is never written to logs or serialized+output by the test runner itself.+"""++import sys+import types+from pathlib import Path+from unittest.mock import MagicMock++import pytest++# ---------------------------------------------------------------------------+# Cloud-SDK stubs — must be in place before the ingest module is imported.+# ---------------------------------------------------------------------------+_CLOUD_STUBS = [+ "boto3",+ "google",+ "google.cloud",+ "google.cloud.storage",+ "azure",+ "azure.storage",+ "azure.storage.blob",+]+for _name in _CLOUD_STUBS:+ if _name not in sys.modules:+ sys.modules[_name] = types.ModuleType(_name)+sys.modules["google.cloud.storage"].Client = MagicMock()+sys.modules["azure.storage.blob"].BlobServiceClient = MagicMock()+sys.modules["boto3"].client = MagicMock()++from semantica.ingest.file_ingestor import ( # noqa: E402+ FileIngestor,+ _SENSITIVE_METADATA_KEYS,+)++# ---------------------------------------------------------------------------+# Helpers+# ---------------------------------------------------------------------------++# Placeholder values — obviously fake, never real credentials.+_PLACEHOLDER = "test-placeholder-value"++# Every key that must be blocked, paired with a case-variant to exercise+# case-insensitive matching.+_SENSITIVE_CASES = [+ # (kwarg_name_to_pass, description)+ # --- LLM / generic API keys (cache.py baseline) ---+ ("api_key", "lowercase canonical"),+ ("API_KEY", "all-caps variant"),+ ("Api_Key", "mixed-case variant"),+ ("apikey", "no-underscore variant"),+ ("api_secret", "api_secret"),+ ("token", "token"),+ ("TOKEN", "TOKEN uppercase"),+ ("access_token", "access_token"),+ ("refresh_token", "refresh_token"),+ ("session_token", "session_token"),+ ("bearer_token", "bearer_token"),+ ("password", "password"),+ ("PASSWORD", "PASSWORD uppercase"),+ ("secret", "secret"),+ ("client_secret", "client_secret"),+ ("private_key", "private_key"),+ ("auth", "auth"),+ ("Auth", "Auth mixed-case"),+ ("authorization", "authorization"),+ ("Authorization", "Authorization mixed-case"),+ ("credential", "credential"),+ ("credentials", "credentials (plural)"),+ # --- Cloud storage credentials (Qodo finding + CloudStorageIngestor) ---+ ("secret_access_key", "S3/Redshift AWS secret key"),+ ("SECRET_ACCESS_KEY", "S3/Redshift uppercase variant"),+ ("access_key_id", "S3/Redshift access key id"),+ ("ACCESS_KEY_ID", "S3/Redshift uppercase variant"),+ ("connection_string", "Azure Blob connection string"),+ ("Connection_String", "Azure Blob mixed-case variant"),+ # --- Additional codebase credential patterns ---+ ("auth_token", "generic auth token"),+ ("AUTH_TOKEN", "auth_token uppercase variant"),+ ("x-api-key", "hyphenated HTTP header name"),+ ("X-API-Key", "hyphenated mixed-case variant"),+ ("security_token", "Salesforce SOAP security token"),+ ("SECURITY_TOKEN", "Salesforce uppercase variant"),+ ("session_id", "Salesforce pre-existing session"),+ ("consumer_key", "Salesforce JWT Bearer consumer key"),+ ("privatekey", "Salesforce JWT Bearer PEM string"),+ ("privatekey_file", "Salesforce JWT Bearer PEM file path"),+ ("deploy_secret", "Looker deploy secret"),+ ("device_token", "Looker device token"),+ ("git_password", "Looker git password"),+ ("pdt_password", "Looker PDT password"),+]+++# ---------------------------------------------------------------------------+# Fixtures+# ---------------------------------------------------------------------------++def txt_file(tmp_path: Path) -> Path:+ """A minimal plaintext file used as an ingestion source."""+ f = tmp_path / "sample.txt"+ f.write_text("security regression fixture", encoding="utf-8")+ return f+++# ---------------------------------------------------------------------------+# 1. Sensitive kwargs are absent from FileObject.metadata+# ---------------------------------------------------------------------------+[email protected]("key,description", _SENSITIVE_CASES)+def test_sensitive_key_absent_from_metadata(+ txt_file: Path, key: str, description: str+) -> None:+ """Each credential-like kwarg must be stripped before reaching metadata."""+ ingestor = FileIngestor()+ result = ingestor.ingest_file(txt_file, **{key: _PLACEHOLDER})++ assert key not in result.metadata, (+ f"Credential key {key!r} ({description}) must not appear in "+ "FileObject.metadata, but it was found there."+ )+++def test_multiple_sensitive_keys_all_absent(txt_file: Path) -> None:+ """Passing several sensitive keys at once — all must be stripped."""+ ingestor = FileIngestor()+ result = ingestor.ingest_file(+ txt_file,+ api_key=_PLACEHOLDER,+ token=_PLACEHOLDER,+ password=_PLACEHOLDER,+ client_secret=_PLACEHOLDER,+ authorization=_PLACEHOLDER,+ )+ for key in ("api_key", "token", "password", "client_secret", "authorization"):+ assert key not in result.metadata, (+ f"{key!r} must not appear in FileObject.metadata"+ )+++# ---------------------------------------------------------------------------+# 2. Nested credential value (credentials={...}) is not retained+# ---------------------------------------------------------------------------++def test_nested_credentials_dict_absent(txt_file: Path) -> None:+ """credentials= with a dict value must also be stripped."""+ ingestor = FileIngestor()+ result = ingestor.ingest_file(+ txt_file,+ credentials={"user": "svc-account", "pass": _PLACEHOLDER},+ )+ assert "credentials" not in result.metadata, (+ "credentials= (nested dict) must not appear in FileObject.metadata"+ )+++# ---------------------------------------------------------------------------+# 3. Non-sensitive custom kwargs still appear in metadata+# ---------------------------------------------------------------------------++def test_non_sensitive_kwargs_reach_metadata(txt_file: Path) -> None:+ """Custom, non-credential kwargs must continue to pass through."""+ ingestor = FileIngestor()+ result = ingestor.ingest_file(+ txt_file,+ source_system="test-suite",+ batch_id="batch-001",+ priority=42,+ )+ assert result.metadata.get("source_system") == "test-suite"+ assert result.metadata.get("batch_id") == "batch-001"+ assert result.metadata.get("priority") == 42+++def test_read_content_option_still_works(txt_file: Path) -> None:+ """read_content is a built-in option that must not be accidentally blocked."""+ ingestor = FileIngestor()+ result_with = ingestor.ingest_file(txt_file, read_content=True)+ result_without = ingestor.ingest_file(txt_file, read_content=False)++ assert result_with.content is not None+ assert result_without.content is None+ # read_content is captured as a named local in ingest_file, so it appears+ # in metadata via the explicit "read_content": read_content assignment.+ assert result_with.metadata["read_content"] is True+ assert result_without.metadata["read_content"] is False+++def test_mixed_kwargs_only_sensitive_stripped(txt_file: Path) -> None:+ """Sensitive and non-sensitive kwargs mixed — only sensitive ones removed."""+ ingestor = FileIngestor()+ result = ingestor.ingest_file(+ txt_file,+ api_key=_PLACEHOLDER, # sensitive — must be absent+ custom_tag="audit-run", # safe — must be present+ token=_PLACEHOLDER, # sensitive — must be absent+ pipeline_stage="ingest", # safe — must be present+ )+ assert "api_key" not in result.metadata+ assert "token" not in result.metadata+ assert result.metadata.get("custom_tag") == "audit-run"+ assert result.metadata.get("pipeline_stage") == "ingest"+++# ---------------------------------------------------------------------------+# 4. Top-level semantica.ingest_file() also does not leak credentials+# (verifies the double-pass path in methods.py)+# ---------------------------------------------------------------------------++def test_top_level_ingest_file_no_credential_leak(txt_file: Path) -> None:+ """+ The public-API ingest_file() in semantica.ingest.methods passes **kwargs+ to both FileIngestor(**config) and ingestor.ingest_file(**kwargs).+ The filter must prevent leakage regardless of which layer the call enters.+ """+ from semantica.ingest.methods import ingest_file as top_level_ingest_file++ result = top_level_ingest_file(+ str(txt_file),+ method="file",+ api_key=_PLACEHOLDER,+ token=_PLACEHOLDER,+ password=_PLACEHOLDER,+ custom_label="top-level-test",+ )+ assert "api_key" not in result.metadata+ assert "token" not in result.metadata+ assert "password" not in result.metadata+ # Non-sensitive kwarg must survive the top-level path as well.+ assert result.metadata.get("custom_label") == "top-level-test"+++# ---------------------------------------------------------------------------+# 5. Constant integrity — _SENSITIVE_METADATA_KEYS contains the required keys+# ---------------------------------------------------------------------------+[email protected]("key", [+ # LLM / generic API keys (cache.py baseline)+ "api_key", "apikey", "api_secret",+ "token", "access_token", "refresh_token", "session_token", "bearer_token",+ "password", "secret", "client_secret", "private_key",+ "auth", "authorization", "credential", "credentials",+ # Cloud storage credentials+ "secret_access_key", "access_key_id", "connection_string",+ # Additional codebase credential patterns+ "auth_token", "x-api-key", "security_token", "session_id",+ "consumer_key", "privatekey", "privatekey_file",+ "deploy_secret", "device_token", "git_password", "pdt_password",+])+def test_sensitive_metadata_keys_constant_completeness(key: str) -> None:+ """_SENSITIVE_METADATA_KEYS must contain every audited key."""+ assert key in _SENSITIVE_METADATA_KEYS, (+ f"Audited sensitive key {key!r} is missing from _SENSITIVE_METADATA_KEYS"+ )+++def test_sensitive_metadata_keys_is_frozenset() -> None:+ """The constant must be immutable so callers cannot accidentally mutate it."""+ assert isinstance(_SENSITIVE_METADATA_KEYS, frozenset)+++# ---------------------------------------------------------------------------+# 6. Qodo HIGH-severity finding — specific regression for the four keys+# that were absent from the initial implementation.+# ---------------------------------------------------------------------------++def test_qodo_finding_secret_access_key_absent(txt_file: Path) -> None:+ """secret_access_key (S3/Redshift AWS credential) must not reach metadata."""+ ingestor = FileIngestor()+ result = ingestor.ingest_file(txt_file, secret_access_key=_PLACEHOLDER)+ assert "secret_access_key" not in result.metadata+++def test_qodo_finding_connection_string_absent(txt_file: Path) -> None:+ """connection_string (Azure Blob) must not reach metadata."""+ ingestor = FileIngestor()+ result = ingestor.ingest_file(txt_file, connection_string=_PLACEHOLDER)+ assert "connection_string" not in result.metadata+++def test_qodo_finding_auth_token_absent(txt_file: Path) -> None:+ """auth_token must not reach metadata."""+ ingestor = FileIngestor()+ result = ingestor.ingest_file(txt_file, auth_token=_PLACEHOLDER)+ assert "auth_token" not in result.metadata+++def test_qodo_finding_x_api_key_absent(txt_file: Path) -> None:+ """x-api-key (hyphenated HTTP header name) must not reach metadata."""+ ingestor = FileIngestor()+ result = ingestor.ingest_file(txt_file, **{"x-api-key": _PLACEHOLDER})+ assert "x-api-key" not in result.metadata+++def test_qodo_finding_all_four_plus_existing_absent(txt_file: Path) -> None:+ """All four Qodo-flagged keys together with pre-existing keys — all absent."""+ ingestor = FileIngestor()+ result = ingestor.ingest_file(+ txt_file,+ # Qodo-flagged keys+ secret_access_key=_PLACEHOLDER,+ connection_string=_PLACEHOLDER,+ auth_token=_PLACEHOLDER,+ **{"x-api-key": _PLACEHOLDER},+ # Pre-existing keys — must also still be absent+ api_key=_PLACEHOLDER,+ token=_PLACEHOLDER,+ password=_PLACEHOLDER,+ # Safe keys — must still be present+ custom_tag="qodo-regression",+ )+ for key in (+ "secret_access_key", "connection_string", "auth_token", "x-api-key",+ "api_key", "token", "password",+ ):+ assert key not in result.metadata, (+ f"{key!r} must not appear in FileObject.metadata"+ )+ assert result.metadata.get("custom_tag") == "qodo-regression"