Skip to content

armgit / letta

Public Imported from GitHub
0

Platform for stateful agents: AI with advanced memory that can learn and self-improve over time.

test: add sleeptime agent tests (#1661)

cthomas55b9fce827cd
diff --git a/tests/integration_test_offline_memory_agent.py b/tests/integration_test_offline_memory_agent.py
deleted file mode 100644
index 7bdae29bd..000000000
--- a/tests/integration_test_offline_memory_agent.py
+++ /dev/null
@@ -1,222 +0,0 @@
-import pytest
-
-from letta import BasicBlockMemory
-from letta.client.client import create_client
-from letta.constants import DEFAULT_HUMAN, DEFAULT_PERSONA
-from letta.prompts import gpt_system
-from letta.schemas.agent import AgentType
-from letta.schemas.embedding_config import EmbeddingConfig
-from letta.schemas.llm_config import LLMConfig
-from letta.schemas.memory import BasicBlockMemory, Block
-from letta.schemas.tool_rule import TerminalToolRule
-from letta.sleeptime_agent import finish_rethinking_memory, rethink_memory, trigger_rethink_memory
-from letta.utils import get_human_text, get_persona_text
-
-
[email protected](scope="module")
-def client():
- client = create_client()
- client.set_default_llm_config(LLMConfig.default_config("gpt-4o-mini"))
- client.set_default_embedding_config(EmbeddingConfig.default_config(provider="openai"))
-
- yield client
-
-
[email protected](autouse=True)
-def clear_agents(client):
- for agent in client.list_agents():
- client.delete_agent(agent.id)
-
-
-def test_ripple_edit(client, disable_e2b_api_key):
- trigger_rethink_memory_tool = client.create_or_update_tool(trigger_rethink_memory)
- send_message = client.server.tool_manager.get_tool_by_name(tool_name="send_message", actor=client.user)
-
- conversation_human_block = client.create_block(label="human", value=get_human_text(DEFAULT_HUMAN), limit=2000)
- conversation_persona_block = client.create_block(label="persona", value=get_persona_text(DEFAULT_PERSONA), limit=2000)
- offline_human_block = client.create_block(label="human", value=get_human_text(DEFAULT_HUMAN), limit=2000)
- offline_persona_block = client.create_block(label="persona", value=get_persona_text("offline_memory_persona"), limit=2000)
-
- # Figure 1. from Evaluating the Ripple Effects of Knowledge Editing in Language Models (Cohen et al., 2023)
- # https://arxiv.org/pdf/2307.12976
- fact_block = client.create_block(
- label="fact_block",
- value="""Messi resides in the Paris.
- Messi plays in the league Ligue 1.
- Messi plays for the team Paris Saint-Germain.
- The national team Messi plays for is the Argentina team.
- Messi is also known as Leo Messi
- Victor Ulloa plays for Inter Miami""",
- limit=2000,
- )
- new_memory = client.create_block(label="rethink_memory_block", value="[empty]", limit=2000)
-
- # conversation_human_block = Block(name="human", label="human", value=get_human_text(DEFAULT_HUMAN), limit=2000)
- # conversation_persona_block = Block(name="persona", label="persona", value=get_persona_text(DEFAULT_PERSONA), limit=2000)
- # offline_human_block = Block(name="human", label="human", value=get_human_text(DEFAULT_HUMAN), limit=2000)
- # offline_persona_block = Block(name="persona", label="persona", value=get_persona_text("offline_memory_persona"), limit=2000)
-
- ## Figure 1. from Evaluating the Ripple Effects of Knowledge Editing in Language Models (Cohen et al., 2023)
- ## https://arxiv.org/pdf/2307.12976
- # fact_block = Block(
- # name="fact_block",
- # label="fact_block",
- # value="""Messi resides in the Paris.
- # Messi plays in the league Ligue 1.
- # Messi plays for the team Paris Saint-Germain.
- # The national team Messi plays for is the Argentina team.
- # Messi is also known as Leo Messi
- # Victor Ulloa plays for Inter Miami""",
- # limit=2000,
- # )
- # new_memory = Block(name="rethink_memory_block", label="rethink_memory_block", value="[empty]", limit=2000)
- # conversation_memory = BasicBlockMemory(blocks=[conversation_persona_block, conversation_human_block, fact_block, new_memory])
- conversation_memory = BasicBlockMemory(blocks=[conversation_persona_block, conversation_human_block, fact_block])
- # offline_memory = BasicBlockMemory(blocks=[offline_persona_block, offline_human_block, fact_block, new_memory])
- offline_memory = BasicBlockMemory(blocks=[offline_persona_block, offline_human_block, fact_block])
-
- conversation_agent = client.create_agent(
- name="conversation_agent",
- agent_type=AgentType.memgpt_agent,
- system=gpt_system.get_system_text("memgpt_convo_only"),
- llm_config=LLMConfig.default_config("gpt-4"),
- embedding_config=EmbeddingConfig.default_config("text-embedding-ada-002"),
- tool_ids=[send_message.id, trigger_rethink_memory_tool.id],
- memory=conversation_memory,
- block_ids=[new_memory.id],
- include_base_tools=False,
- )
- assert conversation_agent is not None
-
- assert set(conversation_agent.memory.list_block_labels()) == {"persona", "human", "fact_block", "rethink_memory_block"}
-
- rethink_memory_tool = client.create_or_update_tool(rethink_memory)
- finish_rethinking_memory_tool = client.create_or_update_tool(finish_rethinking_memory)
- offline_memory_agent = client.create_agent(
- name="offline_memory_agent",
- agent_type=AgentType.sleeptime_agent,
- system=gpt_system.get_system_text("memgpt_offline_memory"),
- memory=offline_memory,
- llm_config=LLMConfig.default_config("gpt-4"),
- embedding_config=EmbeddingConfig.default_config("text-embedding-ada-002"),
- tool_ids=[rethink_memory_tool.id, finish_rethinking_memory_tool.id],
- tool_rules=[TerminalToolRule(tool_name=finish_rethinking_memory_tool.name)],
- block_ids=[new_memory.id],
- include_base_tools=False,
- )
- assert offline_memory_agent is not None
- assert set(offline_memory_agent.memory.list_block_labels()) == {"persona", "human", "fact_block", "rethink_memory_block"}
- response = client.user_message(
- agent_id=conversation_agent.id, message="[trigger_rethink_memory]: Messi has now moved to playing for Inter Miami"
- )
- offline_memory_agent = client.get_agent(agent_id=offline_memory_agent.id)
-
- assert offline_memory_agent.memory.get_block("rethink_memory_block").value != "[empty]"
- conversation_agent = client.get_agent(agent_id=conversation_agent.id)
- assert conversation_agent.memory.get_block("rethink_memory_block").value != "[empty]"
-
- # Clean up agent
- client.delete_agent(conversation_agent.id)
- client.delete_agent(offline_memory_agent.id)
-
-
-def test_chat_only_agent(client, disable_e2b_api_key):
- from letta.offline_memory_agent import finish_rethinking_memory, rethink_memory
-
- send_message = client.server.tool_manager.get_tool_by_name(tool_name="send_message", actor=client.user)
- rethink_memory = client.create_or_update_tool(rethink_memory)
- finish_rethinking_memory = client.create_or_update_tool(finish_rethinking_memory)
- conversation_human_block = client.create_block(label="chat_only_agent_human", value=get_human_text(DEFAULT_HUMAN), limit=2000)
- conversation_persona_block = client.create_block(label="chat_only_agent_persona", value=get_persona_text(DEFAULT_PERSONA), limit=2000)
-
- offline_persona_block = Block(
- name="offline_memory_persona",
- label="offline_memory_persona",
- value=get_persona_text("offline_memory_persona"),
- limit=2000,
- )
-
- offline_memory = BasicBlockMemory(
- blocks=[
- offline_persona_block,
- conversation_human_block,
- conversation_persona_block,
- ]
- )
-
- offline_memory_agent = client.create_agent(
- name="offline_memory_agent",
- agent_type=AgentType.sleeptime_agent,
- system=gpt_system.get_system_text("memgpt_memory_only"),
- memory=offline_memory,
- llm_config=LLMConfig.default_config("gpt-4"),
- embedding_config=EmbeddingConfig.default_config("text-embedding-ada-002"),
- tool_ids=[rethink_memory.id, finish_rethinking_memory.id],
- include_base_tools=False,
- )
-
- chat_only_agent = client.create_agent(
- name="conversation_agent",
- agent_type=AgentType.memgpt_agent,
- llm_config=LLMConfig.default_config("gpt-4"),
- system=gpt_system.get_system_text("memgpt_convo_only"),
- embedding_config=EmbeddingConfig.default_config("text-embedding-ada-002"),
- tool_ids=[send_message.id],
- memory=BasicBlockMemory(blocks=[conversation_persona_block, conversation_human_block]),
- include_base_tools=False,
- )
-
- assert chat_only_agent is not None
- assert offline_memory_agent is not None
-
- # NOTE: the system messages sent to the offline memory agent here are not the actual messages,
- # from the chat only agent. We need to actually stream the responses from the agent to the offline memory
- # these are just for testing
- for message in [
- ("hello", "user"),
- ("hi chad, how's it going?", "system"),
- ("my name is not chad, my name is swoodily", "user"),
- ("I'm sorry, I'm make a note of that, swoodily", "system"),
- ("what's the weather like today?", "user"),
- ("could you specify where you are at?", "system"),
- ("I'm in SF", "user"),
- ("it's currently 60 degrees in SF", "system"),
- ("actually, I'm in Palo Alto", "user"),
- ("it's currently 60 degrees in Palo Alto", "system"),
- ("that sounds nice, I might go for a hike today then", "user"),
- ("Here are some hikes near Palo Alto: 1. The Dish 2. Redtail Loop 3. Adobe Creek", "system"),
- ("I'm going to try the first one, the Dish", "user"),
- ("That sounds great, hope you enjoy it", "system"),
- ]:
- if message[1] == "user":
- client.send_message(agent_id=chat_only_agent.id, message=message[0], role=message[1])
- client.send_message(agent_id=offline_memory_agent.id, message=message[0], role=message[1])
- else:
- client.send_message(agent_id=offline_memory_agent.id, message=message[0], role=message[1])
-
- offline_memory_agent = client.get_agent(agent_id=offline_memory_agent.id)
- chat_only_agent = client.get_agent(agent_id=chat_only_agent.id)
-
- assert chat_only_agent.memory.get_block("chat_only_agent_human").value != get_human_text(DEFAULT_HUMAN)
- client.delete_agent(chat_only_agent.id)
- client.delete_agent(offline_memory_agent.id)
-
-
-def test_initial_message_sequence(client, disable_e2b_api_key):
- """
- Test that when we set the initial sequence to an empty list,
- we do not get the default initial message sequence.
- """
- offline_memory_agent = client.create_agent(
- name="offline_memory_agent",
- agent_type=AgentType.sleeptime_agent,
- system=gpt_system.get_system_text("memgpt_offline_memory"),
- llm_config=LLMConfig.default_config("gpt-4"),
- embedding_config=EmbeddingConfig.default_config("text-embedding-ada-002"),
- include_base_tools=False,
- initial_message_sequence=[],
- )
- assert offline_memory_agent is not None
- assert len(offline_memory_agent.message_ids) == 1 # There should just the system message
-
- client.delete_agent(offline_memory_agent.id)
diff --git a/tests/integration_test_sleeptime_agent.py b/tests/integration_test_sleeptime_agent.py
new file mode 100644
index 000000000..8aad6a7bc
--- /dev/null
+++ b/tests/integration_test_sleeptime_agent.py
@@ -0,0 +1,284 @@
+import time
+
+import pytest
+from sqlalchemy import delete
+
+from letta.config import LettaConfig
+from letta.constants import DEFAULT_HUMAN
+from letta.orm import Provider, Step
+from letta.orm.enums import JobType
+from letta.orm.errors import NoResultFound
+from letta.schemas.agent import CreateAgent
+from letta.schemas.block import CreateBlock
+from letta.schemas.enums import JobStatus, ToolRuleType
+from letta.schemas.group import GroupUpdate, ManagerType, SleeptimeManagerUpdate
+from letta.schemas.message import MessageCreate
+from letta.schemas.run import Run
+from letta.server.server import SyncServer
+from letta.utils import get_human_text, get_persona_text
+
+
[email protected](scope="module")
+def server():
+ config = LettaConfig.load()
+ print("CONFIG PATH", config.config_path)
+
+ config.save()
+
+ server = SyncServer()
+ return server
+
+
[email protected](scope="module")
+def org_id(server):
+ org = server.organization_manager.create_default_organization()
+
+ yield org.id
+
+ # cleanup
+ with server.organization_manager.session_maker() as session:
+ session.execute(delete(Step))
+ session.execute(delete(Provider))
+ session.commit()
+ server.organization_manager.delete_organization_by_id(org.id)
+
+
[email protected](scope="module")
+def actor(server, org_id):
+ user = server.user_manager.create_default_user()
+ yield user
+
+ # cleanup
+ server.user_manager.delete_user_by_id(user.id)
+
+
+async def test_sleeptime_group_chat(server, actor):
+ # 0. Refresh base tools
+ server.tool_manager.upsert_base_tools(actor=actor)
+
+ # 1. Create sleeptime agent
+ main_agent = server.create_agent(
+ request=CreateAgent(
+ name="main_agent",
+ memory_blocks=[
+ CreateBlock(
+ label="persona",
+ value="You are a personal assistant that helps users with requests.",
+ ),
+ CreateBlock(
+ label="human",
+ value="My favorite plant is the fiddle leaf\nMy favorite color is lavender",
+ ),
+ ],
+ # model="openai/gpt-4o-mini",
+ model="anthropic/claude-3-5-sonnet-20240620",
+ embedding="openai/text-embedding-ada-002",
+ enable_sleeptime=True,
+ ),
+ actor=actor,
+ )
+
+ assert main_agent.enable_sleeptime == True
+ main_agent_tools = [tool.name for tool in main_agent.tools]
+ assert "core_memory_append" not in main_agent_tools
+ assert "core_memory_replace" not in main_agent_tools
+ assert "archival_memory_insert" not in main_agent_tools
+
+ # 2. Override frequency for test
+ group = server.group_manager.modify_group(
+ group_id=main_agent.multi_agent_group.id,
+ group_update=GroupUpdate(
+ manager_config=SleeptimeManagerUpdate(
+ sleeptime_agent_frequency=2,
+ ),
+ ),
+ actor=actor,
+ )
+
+ assert group.manager_type == ManagerType.sleeptime
+ assert group.sleeptime_agent_frequency == 2
+ assert len(group.agent_ids) == 1
+
+ # 3. Verify shared blocks
+ sleeptime_agent_id = group.agent_ids[0]
+ shared_block = server.agent_manager.get_block_with_label(agent_id=main_agent.id, block_label="human", actor=actor)
+ agents = server.block_manager.get_agents_for_block(block_id=shared_block.id, actor=actor)
+ assert len(agents) == 2
+ assert sleeptime_agent_id in [agent.id for agent in agents]
+ assert main_agent.id in [agent.id for agent in agents]
+
+ # 4 Verify sleeptime agent tools
+ sleeptime_agent = server.agent_manager.get_agent_by_id(agent_id=sleeptime_agent_id, actor=actor)
+ sleeptime_agent_tools = [tool.name for tool in sleeptime_agent.tools]
+ assert "rethink_memory" in sleeptime_agent_tools
+ assert "finish_rethinking_memory" in sleeptime_agent_tools
+ assert "view_core_memory_with_line_numbers" in sleeptime_agent_tools
+ assert "core_memory_insert" in sleeptime_agent_tools
+
+ assert len([rule for rule in sleeptime_agent.tool_rules if rule.type == ToolRuleType.parent_last_tool]) > 0
+
+ # 5. Send messages and verify run ids
+ message_text = [
+ "my favorite color is orange",
+ "not particularly. today is a good day",
+ "actually my favorite color is coral",
+ "let's change the subject",
+ "actually my fav plant is the the african spear",
+ "indeed",
+ ]
+ run_ids = []
+ for i, text in enumerate(message_text):
+ response = await server.send_message_to_agent(
+ agent_id=main_agent.id,
+ actor=actor,
+ messages=[
+ MessageCreate(
+ role="user",
+ content=text,
+ ),
+ ],
+ stream_steps=False,
+ stream_tokens=False,
+ )
+
+ assert len(response.messages) > 0
+ assert len(response.usage.run_ids or []) == i % 2
+ run_ids.extend(response.usage.run_ids or [])
+
+ jobs = server.job_manager.list_jobs(actor=actor, job_type=JobType.RUN)
+ runs = [Run.from_job(job) for job in jobs]
+ agent_runs = [run for run in runs if "agent_id" in run.metadata and run.metadata["agent_id"] == sleeptime_agent_id]
+ assert len(agent_runs) == len(run_ids)
+ for run_id in run_ids:
+ job = server.job_manager.get_job_by_id(job_id=run_id, actor=actor)
+ assert job.status == JobStatus.completed
+
+ # 6. Delete agent
+ server.agent_manager.delete_agent(agent_id=main_agent.id, actor=actor)
+
+ with pytest.raises(NoResultFound):
+ server.group_manager.retrieve_group(group_id=group.id, actor=actor)
+ with pytest.raises(NoResultFound):
+ server.agent_manager.get_agent_by_id(agent_id=sleeptime_agent_id, actor=actor)
+
+
+async def test_sleeptime_removes_redundant_information(server, actor):
+ # 1. set up sleep-time agent as in test_sleeptime_group_chat
+ server.tool_manager.upsert_base_tools(actor=actor)
+ main_agent = server.create_agent(
+ request=CreateAgent(
+ name="main_agent",
+ memory_blocks=[
+ CreateBlock(
+ label="persona",
+ value="You are a personal assistant that helps users with requests.",
+ ),
+ CreateBlock(
+ label="human",
+ value="My favorite plant is the fiddle leaf\nMy favorite dog is the husky\nMy favorite plant is the fiddle leaf\nMy favorite plant is the fiddle leaf",
+ ),
+ ],
+ model="anthropic/claude-3-5-sonnet-20240620",
+ embedding="openai/text-embedding-ada-002",
+ enable_sleeptime=True,
+ ),
+ actor=actor,
+ )
+
+ group = server.group_manager.modify_group(
+ group_id=main_agent.multi_agent_group.id,
+ group_update=GroupUpdate(
+ manager_config=SleeptimeManagerUpdate(
+ sleeptime_agent_frequency=1,
+ ),
+ ),
+ actor=actor,
+ )
+ sleeptime_agent_id = group.agent_ids[0]
+ shared_block = server.agent_manager.get_block_with_label(agent_id=main_agent.id, block_label="human", actor=actor)
+ count_before_memory_edits = shared_block.value.count("fiddle leaf")
+ test_messages = ["hello there", "my favorite bird is the sparrow"]
+
+ for test_message in test_messages:
+ _ = await server.send_message_to_agent(
+ agent_id=main_agent.id,
+ actor=actor,
+ messages=[
+ MessageCreate(
+ role="user",
+ content=test_message,
+ ),
+ ],
+ stream_steps=False,
+ stream_tokens=False,
+ )
+ # 2. Allow memory blocks time to update
+ time.sleep(5)
+
+ # 3. Check that the memory blocks have been collapsed
+ shared_block = server.agent_manager.get_block_with_label(agent_id=main_agent.id, block_label="human", actor=actor)
+ count_after_memory_edits = shared_block.value.count("fiddle leaf")
+ assert count_after_memory_edits < count_before_memory_edits
+
+ # 4. Delete agent
+ server.agent_manager.delete_agent(agent_id=main_agent.id, actor=actor)
+
+ with pytest.raises(NoResultFound):
+ server.group_manager.retrieve_group(group_id=group.id, actor=actor)
+ with pytest.raises(NoResultFound):
+ server.agent_manager.get_agent_by_id(agent_id=sleeptime_agent_id, actor=actor)
+
+
+async def test_sleeptime_edit(server, actor):
+ sleeptime_agent = server.create_agent(
+ request=CreateAgent(
+ name="sleeptime_agent",
+ agent_type="sleeptime_agent",
+ memory_blocks=[
+ CreateBlock(
+ label="persona",
+ value=get_human_text(DEFAULT_HUMAN),
+ limit=2000,
+ ),
+ CreateBlock(
+ label="human",
+ value=get_persona_text("offline_memory_persona"),
+ limit=2000,
+ ),
+ CreateBlock(
+ label="fact_block",
+ value="""Messi resides in the Paris.
+ Messi plays in the league Ligue 1.
+ Messi plays for the team Paris Saint-Germain.
+ The national team Messi plays for is the Argentina team.
+ Messi is also known as Leo Messi
+ Victor Ulloa plays for Inter Miami""",
+ limit=2000,
+ ),
+ ],
+ model="anthropic/claude-3-5-sonnet-20240620",
+ embedding="openai/text-embedding-ada-002",
+ enable_sleeptime=True,
+ ),
+ actor=actor,
+ )
+
+ _ = await server.send_message_to_agent(
+ agent_id=sleeptime_agent.id,
+ actor=actor,
+ messages=[
+ MessageCreate(
+ role="user",
+ content="Messi has now moved to playing for Inter Miami",
+ ),
+ ],
+ stream_steps=False,
+ stream_tokens=False,
+ )
+ fact_block = server.agent_manager.get_block_with_label(agent_id=sleeptime_agent.id, block_label="fact_block", actor=actor)
+ print(fact_block.value)
+ assert fact_block.value.count("Inter Miami") > 1
diff --git a/tests/test_multi_agent.py b/tests/test_multi_agent.py
index 4a5b825c5..81dcffd8b 100644
--- a/tests/test_multi_agent.py
+++ b/tests/test_multi_agent.py
@@ -1,26 +1,20 @@
-import time
-
import pytest
from sqlalchemy import delete
from letta.config import LettaConfig
from letta.orm import Provider, Step
-from letta.orm.enums import JobType
-from letta.orm.errors import NoResultFound
from letta.schemas.agent import CreateAgent
from letta.schemas.block import CreateBlock
-from letta.schemas.enums import JobStatus, ToolRuleType
from letta.schemas.group import (
DynamicManager,
+ DynamicManagerUpdate,
GroupCreate,
GroupUpdate,
ManagerType,
- RoundRobinManager,
- SleeptimeManager,
+ RoundRobinManagerUpdate,
SupervisorManager,
)
from letta.schemas.message import MessageCreate
-from letta.schemas.run import Run
from letta.server.server import SyncServer
@@ -189,7 +183,7 @@ async def test_modify_group_pattern(server, actor, participant_agents, manager_a
server.group_manager.modify_group(
group_id=group.id,
group_update=GroupUpdate(
- manager_config=DynamicManager(
+ manager_config=DynamicManagerUpdate(
manager_type=ManagerType.dynamic,
manager_agent_id=manager_agent.id,
),
@@ -263,7 +257,7 @@ async def test_round_robin(server, actor, participant_agents):
group_id=group.id,
group_update=GroupUpdate(
agent_ids=[agent.id for agent in participant_agents][::-1],
- manager_config=RoundRobinManager(
+ manager_config=RoundRobinManagerUpdate(
max_turns=max_turns,
),
),
@@ -440,184 +434,3 @@ async def test_dynamic_group_chat(server, actor, manager_agent, participant_agen
finally:
server.group_manager.delete_group(group_id=group.id, actor=actor)
-
-
-async def test_sleeptime_group_chat(server, actor):
- # 0. Refresh base tools
- server.tool_manager.upsert_base_tools(actor=actor)
-
- # 1. Create sleeptime agent
- main_agent = server.create_agent(
- request=CreateAgent(
- name="main_agent",
- memory_blocks=[
- CreateBlock(
- label="persona",
- value="You are a personal assistant that helps users with requests.",
- ),
- CreateBlock(
- label="human",
- value="My favorite plant is the fiddle leaf\nMy favorite color is lavender",
- ),
- ],
- # model="openai/gpt-4o-mini",
- model="anthropic/claude-3-5-sonnet-20240620",
- embedding="openai/text-embedding-ada-002",
- enable_sleeptime=True,
- ),
- actor=actor,
- )
-
- assert main_agent.enable_sleeptime == True
- main_agent_tools = [tool.name for tool in main_agent.tools]
- assert "core_memory_append" not in main_agent_tools
- assert "core_memory_replace" not in main_agent_tools
- assert "archival_memory_insert" not in main_agent_tools
-
- # 2. Override frequency for test
- group = server.group_manager.modify_group(
- group_id=main_agent.multi_agent_group.id,
- group_update=GroupUpdate(
- manager_config=SleeptimeManager(
- manager_agent_id=main_agent.id,
- sleeptime_agent_frequency=2,
- ),
- ),
- actor=actor,
- )
-
- assert group.manager_type == ManagerType.sleeptime
- assert group.sleeptime_agent_frequency == 2
- assert len(group.agent_ids) == 1
-
- # 3. Verify shared blocks
- sleeptime_agent_id = group.agent_ids[0]
- shared_block = server.agent_manager.get_block_with_label(agent_id=main_agent.id, block_label="human", actor=actor)
- agents = server.block_manager.get_agents_for_block(block_id=shared_block.id, actor=actor)
- assert len(agents) == 2
- assert sleeptime_agent_id in [agent.id for agent in agents]
- assert main_agent.id in [agent.id for agent in agents]
-
- # 4 Verify sleeptime agent tools
- sleeptime_agent = server.agent_manager.get_agent_by_id(agent_id=sleeptime_agent_id, actor=actor)
- sleeptime_agent_tools = [tool.name for tool in sleeptime_agent.tools]
- assert "rethink_memory" in sleeptime_agent_tools
- assert "finish_rethinking_memory" in sleeptime_agent_tools
- assert "view_core_memory_with_line_numbers" in sleeptime_agent_tools
- assert "core_memory_insert" in sleeptime_agent_tools
-
- assert len([rule for rule in sleeptime_agent.tool_rules if rule.type == ToolRuleType.parent_last_tool]) > 0
-
- # 5. Send messages and verify run ids
- message_text = [
- "my favorite color is orange",
- "not particularly. today is a good day",
- "actually my favorite color is coral",
- "let's change the subject",
- "actually my fav plant is the the african spear",
- "indeed",
- ]
- run_ids = []
- for i, text in enumerate(message_text):
- response = await server.send_message_to_agent(
- agent_id=main_agent.id,
- actor=actor,
- messages=[
- MessageCreate(
- role="user",
- content=text,
- ),
- ],
- stream_steps=False,
- stream_tokens=False,
- )
-
- assert len(response.messages) > 0
- assert len(response.usage.run_ids or []) == i % 2
- run_ids.extend(response.usage.run_ids or [])
-
- jobs = server.job_manager.list_jobs(actor=actor, job_type=JobType.RUN)
- runs = [Run.from_job(job) for job in jobs]
- agent_runs = [run for run in runs if "agent_id" in run.metadata and run.metadata["agent_id"] == sleeptime_agent_id]
- assert len(agent_runs) == len(run_ids)
- for run_id in run_ids:
- job = server.job_manager.get_job_by_id(job_id=run_id, actor=actor)
- assert job.status == JobStatus.completed
-
- # 6. Delete agent
- server.agent_manager.delete_agent(agent_id=main_agent.id, actor=actor)
-
- with pytest.raises(NoResultFound):
- server.group_manager.retrieve_group(group_id=group.id, actor=actor)
- with pytest.raises(NoResultFound):
- server.agent_manager.get_agent_by_id(agent_id=sleeptime_agent_id, actor=actor)
-
-
-async def test_sleeptime_removes_redundant_information(server, actor):
- # 1. set up sleep-time agent as in test_sleeptime_group_chat
- server.tool_manager.upsert_base_tools(actor=actor)
- main_agent = server.create_agent(
- request=CreateAgent(
- name="main_agent",
- memory_blocks=[
- CreateBlock(
- label="persona",
- value="You are a personal assistant that helps users with requests.",
- ),
- CreateBlock(
- label="human",
- value="My favorite plant is the fiddle leaf\nMy favorite dog is the husky\nMy favorite plant is the fiddle leaf\nMy favorite plant is the fiddle leaf",
- ),
- ],
- model="anthropic/claude-3-5-sonnet-20240620",
- embedding="openai/text-embedding-ada-002",
- enable_sleeptime=True,
- ),
- actor=actor,
- )
-
- group = server.group_manager.modify_group(
- group_id=main_agent.multi_agent_group.id,
- group_update=GroupUpdate(
- manager_config=SleeptimeManager(
- manager_agent_id=main_agent.id,
- sleeptime_agent_frequency=1,
- ),
- ),
- actor=actor,
- )
- sleeptime_agent_id = group.agent_ids[0]
- shared_block = server.agent_manager.get_block_with_label(agent_id=main_agent.id, block_label="human", actor=actor)
- count_before_memory_edits = shared_block.value.count("fiddle leaf")
- test_messages = ["hello there", "my favorite bird is the sparrow"]
-
- for test_message in test_messages:
- _ = await server.send_message_to_agent(
- agent_id=main_agent.id,
- actor=actor,
- messages=[
- MessageCreate(
- role="user",
- content=test_message,
- ),
- ],
- stream_steps=False,
- stream_tokens=False,
- )
- # 2. Allow memory blocks time to update
- time.sleep(5)
-
- # 3. Check that the memory blocks have been collapsed
- shared_block = server.agent_manager.get_block_with_label(agent_id=main_agent.id, block_label="human", actor=actor)
- count_after_memory_edits = shared_block.value.count("fiddle leaf")
- assert count_after_memory_edits < count_before_memory_edits
-
- # 4. Delete agent
- server.agent_manager.delete_agent(agent_id=main_agent.id, actor=actor)
-
- with pytest.raises(NoResultFound):
- server.group_manager.retrieve_group(group_id=group.id, actor=actor)
- with pytest.raises(NoResultFound):
- server.agent_manager.get_agent_by_id(agent_id=sleeptime_agent_id, actor=actor)