The Memory Layer for AI Agents - Drop-in memory infrastructure for AI agents and apps. Context that persists. Built for production.
Python agentic-memoryagentic-memory-systemagentsaiai-agentschatgptgenaillmlong-term-memorymemorymemory-managementpythonragstate-management
fix: preserve http_auth in _safe_deepcopy_config for OpenSearch (#3580) (#4418)
Utkarsh46b4b2e9c804
diff --git a/mem0/memory/main.py b/mem0/memory/main.pyindex ac745e800..cd59ba643 100644--- a/mem0/memory/main.py+++ b/mem0/memory/main.py@@ -51,18 +51,77 @@ warnings.filterwarnings("ignore", category=DeprecationWarning, message=".*swigvalogger = logging.getLogger(__name__)+# Fields that hold runtime auth/connection objects and must be preserved.+# These are non-serializable objects (e.g. AWSV4SignerAuth, RequestsHttpConnection)+# needed by clients like OpenSearch — not sensitive strings to redact.+_RUNTIME_FIELDS = frozenset({+ "http_auth",+ "auth",+ "connection_class",+ "ssl_context",+ "use_azure_credential",+})++# Fields that are known to contain sensitive secrets and must be redacted.+_SENSITIVE_FIELDS_EXACT = frozenset({+ "api_key",+ "secret_key",+ "private_key",+ "access_key",+ "password",+ "credentials",+ "credential",+ "secret",+ "token",+ "access_token",+ "refresh_token",+ "auth_token",+ "session_token",+ "client_secret",+ "auth_client_secret",+ "azure_client_secret",+ "service_account_json",+ "aws_session_token",+})++# Suffixes that indicate a field likely holds a secret value.+_SENSITIVE_SUFFIXES = (+ "_password",+ "_secret",+ "_token",+ "_credential",+ "_credentials",+)+++def _is_sensitive_field(field_name: str) -> bool:+ """Check if a field should be redacted for telemetry safety.++ Uses a layered approach:+ 1. Runtime fields (allowlist) — always preserved, highest priority.+ 2. Exact deny list — known secret field names.+ 3. Suffix deny list — catches patterns like db_password, auth_secret, etc.+ """+ name = field_name.lower().strip()+ if name in _RUNTIME_FIELDS:+ return False+ if name in _SENSITIVE_FIELDS_EXACT:+ return True+ return any(name.endswith(suffix) for suffix in _SENSITIVE_SUFFIXES)++def _safe_deepcopy_config(config):- """Safely deepcopy config, falling back to JSON serialization for non-serializable objects."""+ """Safely deepcopy config, falling back to dict-based cloning for non-serializable objects."""try:return deepcopy(config)except Exception as e:- logger.debug(f"Deepcopy failed, using JSON serialization: {e}")-+ logger.debug(f"Deepcopy failed, using dict-based cloning: {e}")+config_class = type(config)-+if hasattr(config, "model_dump"):try:- clone_dict = config.model_dump(mode="json")+ clone_dict = config.model_dump()except Exception:clone_dict = {k: v for k, v in config.__dict__.items()}elif hasattr(config, "__dataclass_fields__"):@@ -70,12 +129,11 @@ def _safe_deepcopy_config(config):clone_dict = asdict(config)else:clone_dict = {k: v for k, v in config.__dict__.items()}-- sensitive_tokens = ("auth", "credential", "password", "token", "secret", "key", "connection_class")+for field_name in list(clone_dict.keys()):- if any(token in field_name.lower() for token in sensitive_tokens):+ if _is_sensitive_field(field_name):clone_dict[field_name] = None-+try:return config_class(**clone_dict)except Exception as reconstruction_error:@@ -222,7 +280,6 @@ class Memory(MemoryBase):telemetry_config_dict['collection_name'] = "mem0migrations"# Set path for file-based vector stores- telemetry_config = _safe_deepcopy_config(self.config.vector_store.config)if self.config.vector_store.provider in ["faiss", "qdrant"]:provider_path = f"migrations_{self.config.vector_store.provider}"telemetry_config_dict['path'] = os.path.join(mem0_dir, provider_path)diff --git a/tests/memory/test_safe_deepcopy_config.py b/tests/memory/test_safe_deepcopy_config.pynew file mode 100644index 000000000..99b69e865--- /dev/null+++ b/tests/memory/test_safe_deepcopy_config.py@@ -0,0 +1,371 @@+"""Tests for _safe_deepcopy_config and _is_sensitive_field (Issue #3580).++Validates that runtime auth objects (http_auth, connection_class, etc.) are+preserved while genuinely sensitive fields (password, api_key, etc.) are+redacted during config cloning for telemetry.+"""++import threading+from dataclasses import dataclass++import pytest++from mem0.memory.main import _is_sensitive_field, _safe_deepcopy_config+++# ---------------------------------------------------------------------------+# _is_sensitive_field tests+# ---------------------------------------------------------------------------+++class TestRuntimeFieldsPreserved:+ """Runtime/allowlist fields must NOT be considered sensitive."""++ @pytest.mark.parametrize("field", [+ "http_auth",+ "auth",+ "connection_class",+ "ssl_context",+ ])+ def test_runtime_fields_are_not_sensitive(self, field):+ assert _is_sensitive_field(field) is False++ def test_runtime_fields_case_insensitive(self):+ assert _is_sensitive_field("HTTP_AUTH") is False+ assert _is_sensitive_field("Connection_Class") is False+++class TestExactDenyList:+ """Known secret field names must be redacted."""++ @pytest.mark.parametrize("field", [+ "api_key",+ "secret_key",+ "private_key",+ "access_key",+ "password",+ "credentials",+ "credential",+ "secret",+ "token",+ "access_token",+ "refresh_token",+ "auth_token",+ "session_token",+ "client_secret",+ "auth_client_secret",+ "azure_client_secret",+ "service_account_json",+ "aws_session_token",+ ])+ def test_exact_sensitive_fields(self, field):+ assert _is_sensitive_field(field) is True++ def test_exact_fields_case_insensitive(self):+ assert _is_sensitive_field("API_KEY") is True+ assert _is_sensitive_field("Password") is True+++class TestSuffixDenyList:+ """Fields ending with sensitive suffixes must be redacted."""++ @pytest.mark.parametrize("field", [+ "db_password",+ "user_password",+ "redis_password",+ "app_secret",+ "client_secret",+ "oauth_token",+ "bearer_token",+ "aws_credential",+ "gcp_credentials",+ ])+ def test_suffix_matches(self, field):+ assert _is_sensitive_field(field) is True+++class TestNonSensitiveFields:+ """Common config fields that must NOT be redacted."""++ @pytest.mark.parametrize("field", [+ "host",+ "port",+ "collection_name",+ "embedding_model_dims",+ "use_ssl",+ "verify_certs",+ "index_name",+ "dimension",+ "metric",+ "path",+ "url",+ "timeout",+ "pool_maxsize",+ ])+ def test_common_config_fields(self, field):+ assert _is_sensitive_field(field) is False+++class TestOverMatchingPrevention:+ """Fields that previously matched due to broad substring matching+ but should NOT be redacted."""++ @pytest.mark.parametrize("field", [+ "primary_key", # contains "key" but is a DB concept+ "partition_key", # contains "key" but is a DB concept+ "shard_key", # contains "key" but is a DB concept+ "token_type", # contains "token" but is metadata+ "token_count", # contains "token" but is a count+ "tokenizer", # contains "token" but is a tool name+ "key_space", # contains "key" but is a namespace+ "keyboard", # contains "key" but is unrelated+ "monkey", # contains "key" but is unrelated+ "authenticate", # contains "auth" but is a verb+ "authorization_url", # contains "auth" but is a URL+ "credentials_path", # contains "credential" but is a file path+ "secret_agent_name", # contains "secret" but is not a suffix match+ ])+ def test_no_over_matching(self, field):+ assert _is_sensitive_field(field) is False+++class TestEdgeCases:+ def test_empty_string(self):+ assert _is_sensitive_field("") is False++ def test_whitespace_stripped(self):+ assert _is_sensitive_field(" api_key ") is True+ assert _is_sensitive_field(" http_auth ") is False+++class TestRealWorldFieldCoverage:+ """Verify behavior for actual field names from mem0 vector store configs."""++ @pytest.mark.parametrize("field,expected", [+ # OpenSearch+ ("password", True),+ ("api_key", True),+ ("http_auth", False),+ ("connection_class", False),+ ("host", False),+ ("port", False),+ ("verify_certs", False),+ ("use_ssl", False),+ ("pool_maxsize", False),+ # Weaviate+ ("auth_client_secret", True),+ # Databricks+ ("access_token", True),+ ("client_secret", True),+ ("azure_client_secret", True),+ # Upstash / Milvus+ ("token", True),+ # Vertex AI+ ("service_account_json", True),+ ("credentials_path", False),+ # AWS+ ("aws_session_token", True),+ # Azure MySQL+ ("use_azure_credential", False),+ # General non-sensitive+ ("collection_name", False),+ ("embedding_model_dims", False),+ ("user", False),+ ("path", False),+ ("url", False),+ ("dimension", False),+ ("metric_type", False),+ ("batch_size", False),+ ("index_type", False),+ ])+ def test_field_sensitivity(self, field, expected):+ assert _is_sensitive_field(field) is expected+++# ---------------------------------------------------------------------------+# _safe_deepcopy_config integration tests+# ---------------------------------------------------------------------------+++class MockNonCopyableAuth:+ """Simulates AWSV4SignerAuth which cannot be deep-copied due to thread locks."""++ def __init__(self):+ self._lock = threading.Lock()+ self.region = "us-east-1"++ def __deepcopy__(self, memo):+ raise TypeError("cannot pickle '_thread.lock' object")+++class MockConnectionClass:++ def __init__(self):+ self._state = {"connected": False}++ def __deepcopy__(self, memo):+ raise TypeError("cannot pickle connection state")+++class PlainConfig:+ """Config object using plain attributes (not Pydantic)."""++ def __init__(self, **kwargs):+ for k, v in kwargs.items():+ setattr(self, k, v)+++class TestSafeDeepcopyClonesNormally:+ """When deepcopy succeeds, config is returned as-is (no sanitization)."""++ def test_deepcopy_success_returns_clone(self):+ config = PlainConfig(host="localhost", port=9200, password="super_secret")+ result = _safe_deepcopy_config(config)++ assert result is not config+ assert result.host == "localhost"+ assert result.port == 9200+ # deepcopy success path does not sanitize+ assert result.password == "super_secret"+++class TestSafeDeepcopyCopiesWithAuth:+ """When deepcopy fails (auth objects), fallback preserves auth and redacts secrets."""++ def test_preserves_http_auth_and_connection_class(self):+ auth = MockNonCopyableAuth()+ conn = MockConnectionClass()+ config = PlainConfig(+ host="localhost",+ port=9200,+ http_auth=auth,+ connection_class=conn,+ api_key="secret123",+ password="hunter2",+ collection_name="test",+ )++ result = _safe_deepcopy_config(config)++ # Runtime objects preserved (not None)+ assert result.http_auth is not None+ assert result.connection_class is not None+ # Sensitive fields redacted+ assert result.api_key is None+ assert result.password is None+ # Normal fields preserved+ assert result.host == "localhost"+ assert result.port == 9200+ assert result.collection_name == "test"++ def test_preserves_auth_field(self):+ auth = MockNonCopyableAuth()+ config = PlainConfig(+ host="localhost",+ auth=auth,+ credentials={"key": "val"},+ )++ result = _safe_deepcopy_config(config)++ assert result.auth is not None+ assert result.credentials is None+++class TestSafeDeepcopyWithPydantic:+ """Test fallback path with Pydantic-like model_dump objects."""++ def test_pydantic_like_config(self):+ class PydanticLikeConfig:+ def __init__(self, **kwargs):+ for k, v in kwargs.items():+ setattr(self, k, v)++ def model_dump(self, mode=None):+ return {k: v for k, v in self.__dict__.items()+ if not k.startswith("_")}++ def __deepcopy__(self, memo):+ raise TypeError("cannot deepcopy")++ config = PydanticLikeConfig(+ host="localhost",+ api_key="secret",+ http_auth="signer_obj",+ )++ result = _safe_deepcopy_config(config)+ assert result.host == "localhost"+ assert result.api_key is None+ assert result.http_auth is not None+++class TestSafeDeepcopyWithRealPydanticModel:+ """Test with real Pydantic BaseModel matching the OpenSearch config pattern.++ This validates the model_dump() path (without mode='json') preserves+ actual auth objects rather than losing them to JSON serialization.+ """++ def test_real_pydantic_model_preserves_auth_objects(self):+ from pydantic import BaseModel, Field+ from typing import Optional++ class OpenSearchLikeConfig(BaseModel):+ host: str = "localhost"+ port: int = 9200+ collection_name: str = "test"+ password: Optional[str] = None+ api_key: Optional[str] = None+ http_auth: Optional[object] = Field(None)+ connection_class: Optional[object] = Field(None)++ auth = MockNonCopyableAuth()+ conn = MockConnectionClass()+ config = OpenSearchLikeConfig(+ host="myhost",+ password="hunter2",+ api_key="sk-secret",+ http_auth=auth,+ connection_class=conn,+ )++ result = _safe_deepcopy_config(config)++ # Auth objects must be the actual objects, not string representations+ assert result.http_auth is auth+ assert result.connection_class is conn+ # Sensitive fields must be redacted+ assert result.password is None+ assert result.api_key is None+ # Normal fields preserved+ assert result.host == "myhost"+ assert result.port == 9200+++class TestSafeDeepcopyWithDataclass:+ """Test fallback path with dataclasses."""++ def test_dataclass_config(self):+ @dataclass+ class DCConfig:+ host: str = "localhost"+ api_key: str = None+ db_password: str = None+ http_auth: object = None++ def __deepcopy__(self, memo):+ raise TypeError("cannot deepcopy")++ config = DCConfig(+ host="myhost",+ api_key="secret",+ db_password="pass123",+ http_auth="auth_obj",+ )++ result = _safe_deepcopy_config(config)+ assert result.host == "myhost"+ assert result.api_key is None+ assert result.db_password is None+ assert result.http_auth is not Nonediff --git a/tests/vector_stores/test_opensearch.py b/tests/vector_stores/test_opensearch.pyindex 9e211ec2e..82e2b41fe 100644--- a/tests/vector_stores/test_opensearch.py+++ b/tests/vector_stores/test_opensearch.py@@ -299,11 +299,13 @@ def test_safe_deepcopy_config_handles_opensearch_auth(mock_sqlite, mock_llm_factconfig_with_auth = MockOpenSearchConfig(collection_name="opensearch_test", include_auth=True)safe_config = _safe_deepcopy_config(config_with_auth)-- assert safe_config.http_auth is None- assert safe_config.auth is None++ # Runtime auth objects must be preserved (Issue #3580)+ assert safe_config.http_auth is not None+ assert safe_config.auth is not None+ assert safe_config.connection_class is not None+ # Credentials dict is a sensitive secret and should be redactedassert safe_config.credentials is None- assert safe_config.connection_class is Noneassert safe_config.collection_name == "opensearch_test"assert safe_config.host == "localhost"