Skip to content

armgit / letta

Public Imported from GitHub
0

Platform for stateful agents: AI with advanced memory that can learn and self-improve over time.

feat: create sdk_v1 test folder [LET-5937] (#5790)

Ari Webb9cab61fe3fd8
diff --git a/tests/sdk_v1/agents_test.py b/tests/sdk_v1/agents_test.py
new file mode 100644
index 000000000..74830702f
--- /dev/null
+++ b/tests/sdk_v1/agents_test.py
@@ -0,0 +1,25 @@
+from conftest import create_test_module
+
+AGENTS_CREATE_PARAMS = [
+ ("caren_agent", {"name": "caren", "model": "openai/gpt-4o-mini", "embedding": "openai/text-embedding-3-small"}, {}, None),
+]
+
+AGENTS_MODIFY_PARAMS = [
+ ("caren_agent", {"name": "caren_updated"}, {}, None),
+]
+
+AGENTS_LIST_PARAMS = [
+ ({}, 1),
+ ({"name": "caren_updated"}, 1),
+]
+
+# Create all test module components at once
+globals().update(
+ create_test_module(
+ resource_name="agents",
+ id_param_name="agent_id",
+ create_params=AGENTS_CREATE_PARAMS,
+ modify_params=AGENTS_MODIFY_PARAMS,
+ list_params=AGENTS_LIST_PARAMS,
+ )
+)
diff --git a/tests/sdk_v1/blocks_test.py b/tests/sdk_v1/blocks_test.py
new file mode 100644
index 000000000..301f2fd60
--- /dev/null
+++ b/tests/sdk_v1/blocks_test.py
@@ -0,0 +1,31 @@
+from conftest import create_test_module
+from letta_client.errors import UnprocessableEntityError
+
+from letta.constants import CORE_MEMORY_HUMAN_CHAR_LIMIT, CORE_MEMORY_PERSONA_CHAR_LIMIT
+
+BLOCKS_CREATE_PARAMS = [
+ ("human_block", {"label": "human", "value": "test"}, {"limit": CORE_MEMORY_HUMAN_CHAR_LIMIT}, None),
+ ("persona_block", {"label": "persona", "value": "test1"}, {"limit": CORE_MEMORY_PERSONA_CHAR_LIMIT}, None),
+]
+
+BLOCKS_MODIFY_PARAMS = [
+ ("human_block", {"value": "test2"}, {}, None),
+ ("persona_block", {"value": "testing testing testing", "limit": 10}, {}, UnprocessableEntityError),
+]
+
+BLOCKS_LIST_PARAMS = [
+ ({}, 2),
+ ({"label": "human"}, 1),
+ ({"label": "persona"}, 1),
+]
+
+# Create all test module components at once
+globals().update(
+ create_test_module(
+ resource_name="blocks",
+ id_param_name="block_id",
+ create_params=BLOCKS_CREATE_PARAMS,
+ modify_params=BLOCKS_MODIFY_PARAMS,
+ list_params=BLOCKS_LIST_PARAMS,
+ )
+)
diff --git a/tests/sdk_v1/conftest.py b/tests/sdk_v1/conftest.py
new file mode 100644
index 000000000..89b5f882f
--- /dev/null
+++ b/tests/sdk_v1/conftest.py
@@ -0,0 +1,319 @@
+import os
+import threading
+import time
+from typing import Any, Dict, List, Optional, Tuple
+
+import pytest
+import requests
+from dotenv import load_dotenv
+from letta_client import Letta
+
+
[email protected](scope="session")
+def server_url() -> str:
+ """
+ Provides the URL for the Letta server.
+ If LETTA_SERVER_URL is not set, starts the server in a background thread
+ and polls until it's accepting connections.
+ """
+
+ def _run_server() -> None:
+ load_dotenv()
+ from letta.server.rest_api.app import start_server
+
+ start_server(debug=True)
+
+ url: str = os.getenv("LETTA_SERVER_URL", "http://localhost:8283")
+
+ if not os.getenv("LETTA_SERVER_URL"):
+ thread = threading.Thread(target=_run_server, daemon=True)
+ thread.start()
+
+ # Poll until the server is up (or timeout)
+ timeout_seconds = 30
+ deadline = time.time() + timeout_seconds
+ while time.time() < deadline:
+ try:
+ resp = requests.get(url + "/v1/health")
+ if resp.status_code < 500:
+ break
+ except requests.exceptions.RequestException:
+ pass
+ time.sleep(0.1)
+ else:
+ raise RuntimeError(f"Could not reach {url} within {timeout_seconds}s")
+
+ return url
+
+
+# This fixture creates a client for each test module
[email protected](scope="session")
+def client(server_url):
+ print("Running client tests with server:", server_url)
+
+ # Overide the base_url if the LETTA_API_URL is set
+ api_url = os.getenv("LETTA_API_URL")
+ base_url = api_url if api_url else server_url
+ # create the Letta client
+ yield Letta(base_url=base_url, token=None, timeout=300.0)
+
+
+def skip_test_if_not_implemented(handler, resource_name, test_name):
+ if not hasattr(handler, test_name):
+ pytest.skip(f"client.{resource_name}.{test_name} not implemented")
+
+
+def create_test_module(
+ resource_name: str,
+ id_param_name: str,
+ create_params: List[Tuple[str, Dict[str, Any], Dict[str, Any], Optional[Exception]]] = [],
+ upsert_params: List[Tuple[str, Dict[str, Any], Dict[str, Any], Optional[Exception]]] = [],
+ modify_params: List[Tuple[str, Dict[str, Any], Dict[str, Any], Optional[Exception]]] = [],
+ list_params: List[Tuple[Dict[str, Any], int]] = [],
+) -> Dict[str, Any]:
+ """Create a test module for a resource.
+
+ This function creates all the necessary test methods and returns them in a dictionary
+ that can be added to the globals() of the module.
+
+ Args:
+ resource_name: Name of the resource (e.g., "blocks", "tools")
+ id_param_name: Name of the ID parameter (e.g., "block_id", "tool_id")
+ create_params: List of (name, params, expected_error) tuples for create tests
+ modify_params: List of (name, params, expected_error) tuples for modify tests
+ list_params: List of (query_params, expected_count) tuples for list tests
+
+ Returns:
+ Dict: A dictionary of all test functions that should be added to the module globals
+ """
+ # Create shared test state
+ test_item_ids = {}
+
+ # Create fixture functions
+ @pytest.fixture(scope="session")
+ def handler(client):
+ return getattr(client, resource_name)
+
+ @pytest.fixture(scope="session")
+ def caren_agent(client, request):
+ """Create an agent to be used as manager in supervisor groups."""
+ agent = client.agents.create(
+ name="caren_agent",
+ model="openai/gpt-4o-mini",
+ embedding="openai/text-embedding-3-small",
+ )
+
+ # Add finalizer to ensure cleanup happens in the right order
+ request.addfinalizer(lambda: client.agents.delete(agent_id=agent.id))
+
+ return agent
+
+ # Create standalone test functions
+ @pytest.mark.order(0)
+ def test_create(handler, caren_agent, name, params, extra_expected_values, expected_error):
+ """Test creating a resource."""
+ skip_test_if_not_implemented(handler, resource_name, "create")
+
+ # Use preprocess_params which adds fixtures
+ processed_params = preprocess_params(params, caren_agent)
+ processed_extra_expected = preprocess_params(extra_expected_values, caren_agent)
+
+ try:
+ item = handler.create(**processed_params)
+ except Exception as e:
+ if expected_error is not None:
+ if hasattr(e, "status_code"):
+ assert e.status_code == expected_error
+ elif hasattr(e, "status"):
+ assert e.status == expected_error
+ else:
+ pytest.fail(f"Expected error with status {expected_error}, but got {type(e)}: {e}")
+ else:
+ raise e
+
+ # Store item ID for later tests
+ test_item_ids[name] = item.id
+
+ # Verify item properties
+ expected_values = processed_params | processed_extra_expected
+ for key, value in expected_values.items():
+ if hasattr(item, key):
+ assert custom_model_dump(getattr(item, key)) == value
+
+ @pytest.mark.order(1)
+ def test_retrieve(handler):
+ """Test retrieving resources."""
+ skip_test_if_not_implemented(handler, resource_name, "retrieve")
+ for name, item_id in test_item_ids.items():
+ kwargs = {id_param_name: item_id}
+ item = handler.retrieve(**kwargs)
+ assert hasattr(item, "id") and item.id == item_id, f"{resource_name.capitalize()} {name} with id {item_id} not found"
+
+ @pytest.mark.order(2)
+ def test_upsert(handler, name, params, extra_expected_values, expected_error):
+ """Test upserting resources."""
+ skip_test_if_not_implemented(handler, resource_name, "upsert")
+ existing_item_id = test_item_ids[name]
+ try:
+ item = handler.upsert(**params)
+ except Exception as e:
+ if expected_error is not None:
+ if hasattr(e, "status_code"):
+ assert e.status_code == expected_error
+ elif hasattr(e, "status"):
+ assert e.status == expected_error
+ else:
+ pytest.fail(f"Expected error with status {expected_error}, but got {type(e)}: {e}")
+ else:
+ raise e
+
+ assert existing_item_id == item.id
+
+ # Verify item properties
+ expected_values = params | extra_expected_values
+ for key, value in expected_values.items():
+ if hasattr(item, key):
+ assert custom_model_dump(getattr(item, key)) == value
+
+ @pytest.mark.order(3)
+ def test_modify(handler, caren_agent, name, params, extra_expected_values, expected_error):
+ """Test modifying a resource."""
+ skip_test_if_not_implemented(handler, resource_name, "modify")
+ if name not in test_item_ids:
+ pytest.skip(f"Item '{name}' not found in test_items")
+
+ kwargs = {id_param_name: test_item_ids[name]}
+ kwargs.update(params)
+ processed_params = preprocess_params(kwargs, caren_agent)
+ processed_extra_expected = preprocess_params(extra_expected_values, caren_agent)
+
+ try:
+ item = handler.modify(**processed_params)
+ except Exception as e:
+ if expected_error is not None:
+ assert isinstance(e, expected_error), f"Expected error with type {expected_error}, but got {type(e)}: {e}"
+ return
+ else:
+ raise e
+
+ # Verify item properties
+ expected_values = processed_params | processed_extra_expected
+ for key, value in expected_values.items():
+ if hasattr(item, key):
+ assert custom_model_dump(getattr(item, key)) == value
+
+ # Verify via retrieve as well
+ retrieve_kwargs = {id_param_name: item.id}
+ retrieved_item = handler.retrieve(**retrieve_kwargs)
+
+ expected_values = processed_params | processed_extra_expected
+ for key, value in expected_values.items():
+ if hasattr(retrieved_item, key):
+ assert custom_model_dump(getattr(retrieved_item, key)) == value
+
+ @pytest.mark.order(4)
+ def test_list(handler, query_params, count):
+ """Test listing resources."""
+ skip_test_if_not_implemented(handler, resource_name, "list")
+ all_items = handler.list(**query_params)
+
+ test_items_list = [item.id for item in all_items if item.id in test_item_ids.values()]
+ assert len(test_items_list) == count
+
+ @pytest.mark.order(-1)
+ def test_delete(handler):
+ """Test deleting resources."""
+ skip_test_if_not_implemented(handler, resource_name, "delete")
+ for item_id in test_item_ids.values():
+ kwargs = {id_param_name: item_id}
+ handler.delete(**kwargs)
+
+ for name, item_id in test_item_ids.items():
+ try:
+ kwargs = {id_param_name: item_id}
+ item = handler.retrieve(**kwargs)
+ raise AssertionError(f"{resource_name.capitalize()} {name} with id {item.id} was not deleted")
+ except Exception as e:
+ if isinstance(e, AssertionError):
+ raise e
+ if hasattr(e, "status_code"):
+ assert e.status_code == 404, f"Expected 404 error, got {e.status_code}"
+ else:
+ raise AssertionError(f"Unexpected error type: {type(e)}")
+
+ test_item_ids.clear()
+
+ # Create test methods dictionary
+ result = {
+ "handler": handler,
+ "caren_agent": caren_agent,
+ "test_create": pytest.mark.parametrize("name, params, extra_expected_values, expected_error", create_params)(test_create),
+ "test_retrieve": test_retrieve,
+ "test_upsert": pytest.mark.parametrize("name, params, extra_expected_values, expected_error", upsert_params)(test_upsert),
+ "test_modify": pytest.mark.parametrize("name, params, extra_expected_values, expected_error", modify_params)(test_modify),
+ "test_delete": test_delete,
+ "test_list": pytest.mark.parametrize("query_params, count", list_params)(test_list),
+ }
+
+ return result
+
+
+def custom_model_dump(model):
+ """
+ Dumps the given model to a form that can be easily compared.
+
+ Args:
+ model: The model to dump
+
+ Returns:
+ The dumped model
+ """
+ if isinstance(model, (str, int, float, bool, type(None))):
+ return model
+ if isinstance(model, list):
+ return [custom_model_dump(item) for item in model]
+ else:
+ return model.model_dump()
+
+
+def add_fixture_params(value, caren_agent):
+ """
+ Replaces string values containing '.id' with their mapped values.
+
+ Args:
+ value: The value to process (should be a string)
+ caren_agent: The agent object to use for ID replacement
+
+ Returns:
+ The processed value with ID strings replaced by actual values
+ """
+ param_to_fixture_mapping = {
+ "caren_agent.id": caren_agent.id,
+ }
+ return param_to_fixture_mapping.get(value, value)
+
+
+def preprocess_params(params, caren_agent):
+ """
+ Recursively processes a nested structure of dictionaries and lists,
+ replacing string values containing '.id' with their mapped values.
+
+ Args:
+ params: The parameters to process (dict, list, or scalar value)
+ caren_agent: The agent object to use for ID replacement
+
+ Returns:
+ The processed parameters with ID strings replaced by actual values
+ """
+ if isinstance(params, dict):
+ # Process each key-value pair in the dictionary
+ return {key: preprocess_params(value, caren_agent) for key, value in params.items()}
+ elif isinstance(params, list):
+ # Process each item in the list
+ return [preprocess_params(item, caren_agent) for item in params]
+ elif isinstance(params, str) and ".id" in params:
+ # Replace string values containing '.id' with their mapped values
+ return add_fixture_params(params, caren_agent)
+ else:
+ # Return other values unchanged
+ return params
diff --git a/tests/sdk_v1/groups_test.py b/tests/sdk_v1/groups_test.py
new file mode 100644
index 000000000..e52a906b4
--- /dev/null
+++ b/tests/sdk_v1/groups_test.py
@@ -0,0 +1,36 @@
+from conftest import create_test_module
+
+GROUPS_CREATE_PARAMS = [
+ ("round_robin_group", {"agent_ids": [], "description": ""}, {"manager_type": "round_robin"}, None),
+ (
+ "supervisor_group",
+ {"agent_ids": [], "description": "", "manager_config": {"manager_type": "supervisor", "manager_agent_id": "caren_agent.id"}},
+ {"manager_type": "supervisor"},
+ None,
+ ),
+]
+
+GROUPS_MODIFY_PARAMS = [
+ (
+ "round_robin_group",
+ {"manager_config": {"manager_type": "round_robin", "max_turns": 10}},
+ {"manager_type": "round_robin", "max_turns": 10},
+ None,
+ ),
+]
+
+GROUPS_LIST_PARAMS = [
+ ({}, 2),
+ ({"manager_type": "round_robin"}, 1),
+]
+
+# Create all test module components at once
+globals().update(
+ create_test_module(
+ resource_name="groups",
+ id_param_name="group_id",
+ create_params=GROUPS_CREATE_PARAMS,
+ modify_params=GROUPS_MODIFY_PARAMS,
+ list_params=GROUPS_LIST_PARAMS,
+ )
+)
diff --git a/tests/sdk_v1/identities_test.py b/tests/sdk_v1/identities_test.py
new file mode 100644
index 000000000..9b06a4103
--- /dev/null
+++ b/tests/sdk_v1/identities_test.py
@@ -0,0 +1,43 @@
+from conftest import create_test_module
+
+IDENTITIES_CREATE_PARAMS = [
+ ("caren1", {"identifier_key": "123", "name": "caren", "identity_type": "user"}, {}, None),
+ ("caren2", {"identifier_key": "456", "name": "caren", "identity_type": "user"}, {}, None),
+]
+
+IDENTITIES_MODIFY_PARAMS = [
+ ("caren1", {"properties": [{"key": "email", "value": "[email protected]", "type": "string"}]}, {}, None),
+ ("caren2", {"properties": [{"key": "email", "value": "[email protected]", "type": "string"}]}, {}, None),
+]
+
+IDENTITIES_UPSERT_PARAMS = [
+ (
+ "caren2",
+ {
+ "identifier_key": "456",
+ "name": "caren",
+ "identity_type": "user",
+ "properties": [{"key": "email", "value": "[email protected]", "type": "string"}],
+ },
+ {},
+ None,
+ ),
+]
+
+IDENTITIES_LIST_PARAMS = [
+ ({}, 2),
+ ({"name": "caren"}, 2),
+ ({"identifier_key": "123"}, 1),
+]
+
+# Create all test module components at once
+globals().update(
+ create_test_module(
+ resource_name="identities",
+ id_param_name="identity_id",
+ create_params=IDENTITIES_CREATE_PARAMS,
+ upsert_params=IDENTITIES_UPSERT_PARAMS,
+ modify_params=IDENTITIES_MODIFY_PARAMS,
+ list_params=IDENTITIES_LIST_PARAMS,
+ )
+)
diff --git a/tests/sdk_v1/tools_test.py b/tests/sdk_v1/tools_test.py
new file mode 100644
index 000000000..62c70be65
--- /dev/null
+++ b/tests/sdk_v1/tools_test.py
@@ -0,0 +1,67 @@
+from conftest import create_test_module
+
+# Sample code for tools
+FRIENDLY_FUNC_SOURCE_CODE = '''
+def friendly_func():
+ """
+ Returns a friendly message.
+
+ Returns:
+ str: A friendly message.
+ """
+ return "HI HI HI HI HI!"
+'''
+
+UNFRIENDLY_FUNC_SOURCE_CODE = '''
+def unfriendly_func():
+ """
+ Returns an unfriendly message.
+
+ Returns:
+ str: An unfriendly message.
+ """
+ return "NO NO NO NO NO!"
+'''
+
+UNFRIENDLY_FUNC_SOURCE_CODE_V2 = '''
+def unfriendly_func():
+ """
+ Returns an unfriendly message.
+
+ Returns:
+ str: An unfriendly message.
+ """
+ return "BYE BYE BYE BYE BYE!"
+'''
+
+# Define test parameters for tools
+TOOLS_CREATE_PARAMS = [
+ ("friendly_func", {"source_code": FRIENDLY_FUNC_SOURCE_CODE}, {"name": "friendly_func"}, None),
+ ("unfriendly_func", {"source_code": UNFRIENDLY_FUNC_SOURCE_CODE}, {"name": "unfriendly_func"}, None),
+]
+
+TOOLS_UPSERT_PARAMS = [
+ ("unfriendly_func", {"source_code": UNFRIENDLY_FUNC_SOURCE_CODE_V2}, {}, None),
+]
+
+TOOLS_MODIFY_PARAMS = [
+ ("friendly_func", {"tags": ["sdk_test"]}, {}, None),
+ ("unfriendly_func", {"return_char_limit": 300}, {}, None),
+]
+
+TOOLS_LIST_PARAMS = [
+ ({}, 2),
+ ({"name": ["friendly_func"]}, 1),
+]
+
+# Create all test module components at once
+globals().update(
+ create_test_module(
+ resource_name="tools",
+ id_param_name="tool_id",
+ create_params=TOOLS_CREATE_PARAMS,
+ upsert_params=TOOLS_UPSERT_PARAMS,
+ modify_params=TOOLS_MODIFY_PARAMS,
+ list_params=TOOLS_LIST_PARAMS,
+ )
+)