Skip to content

armgit / camel

Public Imported from GitHub
0

🐫 CAMEL: The first and the best multi-agent framework. Finding the Scaling Law of Agents. https://www.camel-ai.org

feat(memories): add timestamp in VectorDBMemory to prevent data retrieval order confusion (#1644)

Lei Zhange89de6e71b77
diff --git a/camel/memories/agent_memories.py b/camel/memories/agent_memories.py
index 788b3fb61..f6e401a66 100644
--- a/camel/memories/agent_memories.py
+++ b/camel/memories/agent_memories.py
@@ -114,6 +114,10 @@ class VectorDBMemory(AgentMemory):
def get_context_creator(self) -> BaseContextCreator:
return self._context_creator
+ def clear(self) -> None:
+ r"""Removes all records from the vector database memory."""
+ self._vectordb_block.clear()
+
class LongtermAgentMemory(AgentMemory):
r"""An implementation of the :obj:`AgentMemory` abstract base class for
diff --git a/camel/memories/blocks/chat_history_block.py b/camel/memories/blocks/chat_history_block.py
index 74b6dfb39..68218dbb4 100644
--- a/camel/memories/blocks/chat_history_block.py
+++ b/camel/memories/blocks/chat_history_block.py
@@ -85,13 +85,21 @@ class ChatHistoryBlock(MemoryBlock):
if record.role_at_backend == OpenAIBackendRole.SYSTEM:
# System messages are always kept.
output_records.append(
- ContextRecord(memory_record=record, score=1.0)
+ ContextRecord(
+ memory_record=record,
+ score=1.0,
+ timestamp=record.timestamp,
+ )
)
else:
# Other messages' score drops down gradually
score *= self.keep_rate
output_records.append(
- ContextRecord(memory_record=record, score=score)
+ ContextRecord(
+ memory_record=record,
+ score=score,
+ timestamp=record.timestamp,
+ )
)
output_records.reverse()
diff --git a/camel/memories/blocks/vectordb_block.py b/camel/memories/blocks/vectordb_block.py
index 6a9f3d0bc..0a11a84e0 100644
--- a/camel/memories/blocks/vectordb_block.py
+++ b/camel/memories/blocks/vectordb_block.py
@@ -74,6 +74,7 @@ class VectorDBBlock(MemoryBlock):
ContextRecord(
memory_record=MemoryRecord.from_dict(result.record.payload),
score=result.similarity,
+ timestamp=result.record.payload['timestamp'],
)
for result in results
if result.record.payload is not None
diff --git a/camel/memories/context_creators/score_based.py b/camel/memories/context_creators/score_based.py
index 669f128ae..1027d9e36 100644
--- a/camel/memories/context_creators/score_based.py
+++ b/camel/memories/context_creators/score_based.py
@@ -104,6 +104,10 @@ class ScoreBasedContextCreator(BaseContextCreator):
# If not exceed token limit, simply return
total_tokens = sum([unit.num_tokens for unit in context_units])
if total_tokens <= self.token_limit:
+ context_units = sorted(
+ context_units,
+ key=lambda unit: (unit.record.timestamp, unit.record.score),
+ )
return self._create_output(context_units)
# Log warning about token limit being exceeded
@@ -114,14 +118,16 @@ class ScoreBasedContextCreator(BaseContextCreator):
# Sort by score
context_units = sorted(
- context_units, key=lambda unit: unit.record.score
+ context_units,
+ key=lambda unit: (unit.record.timestamp, unit.record.score),
)
# Remove the least score messages until total token number is smaller
# than token limit
truncate_idx = None
for i, unit in enumerate(context_units):
- if unit.record.score == 1:
+ if i == len(context_units) - 1:
+ # If we reach the end of the list and still exceed the token
raise RuntimeError(
"Cannot create context: exceed token limit.", total_tokens
)
@@ -144,7 +150,9 @@ class ScoreBasedContextCreator(BaseContextCreator):
for output, specifically a list of OpenAIMessages and an integer
representing the total token count.
"""
- context_units = sorted(context_units, key=lambda unit: unit.idx)
+ context_units = sorted(
+ context_units, key=lambda unit: unit.record.timestamp
+ )
return [
unit.record.memory_record.to_openai_message()
for unit in context_units
diff --git a/camel/memories/records.py b/camel/memories/records.py
index f30b82687..2f3be051d 100644
--- a/camel/memories/records.py
+++ b/camel/memories/records.py
@@ -13,6 +13,7 @@
# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
from dataclasses import asdict
+from datetime import datetime, timezone
from typing import Any, ClassVar, Dict
from uuid import UUID, uuid4
@@ -37,6 +38,7 @@ class MemoryRecord(BaseModel):
extra_info (Dict[str, str], optional): A dictionary of additional
key-value pairs that provide more information. If not given, it
will be an empty `Dict`.
+ timestamp (float, optional): The timestamp when the record was created.
"""
model_config = ConfigDict(arbitrary_types_allowed=True)
@@ -45,6 +47,9 @@ class MemoryRecord(BaseModel):
role_at_backend: OpenAIBackendRole
uuid: UUID = Field(default_factory=uuid4)
extra_info: Dict[str, str] = Field(default_factory=dict)
+ timestamp: float = Field(
+ default_factory=lambda: datetime.now(timezone.utc).timestamp()
+ )
_MESSAGE_TYPES: ClassVar[dict] = {
"BaseMessage": BaseMessage,
@@ -67,6 +72,7 @@ class MemoryRecord(BaseModel):
message=reconstructed_message,
role_at_backend=record_dict["role_at_backend"],
extra_info=record_dict["extra_info"],
+ timestamp=record_dict["timestamp"],
)
def to_dict(self) -> Dict[str, Any]:
@@ -81,6 +87,7 @@ class MemoryRecord(BaseModel):
},
"role_at_backend": self.role_at_backend,
"extra_info": self.extra_info,
+ "timestamp": self.timestamp,
}
def to_openai_message(self) -> OpenAIMessage:
@@ -93,3 +100,6 @@ class ContextRecord(BaseModel):
memory_record: MemoryRecord
score: float
+ timestamp: float = Field(
+ default_factory=lambda: datetime.now(timezone.utc).timestamp()
+ )
diff --git a/test/memories/context_creators/test_score_based.py b/test/memories/context_creators/test_score_based.py
index f55f55133..768cfe3d8 100644
--- a/test/memories/context_creators/test_score_based.py
+++ b/test/memories/context_creators/test_score_based.py
@@ -11,6 +11,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+from datetime import datetime
from camel.memories import (
ContextRecord,
@@ -33,11 +34,12 @@ def test_score_based_context_creator():
"test",
RoleType.ASSISTANT,
meta_dict=None,
- content="Hello world!", # 10
+ content="Nice to meet you.", # 12
),
role_at_backend=OpenAIBackendRole.ASSISTANT,
),
- score=0.9,
+ timestamp=datetime.now().timestamp(),
+ score=0.3,
),
ContextRecord(
memory_record=MemoryRecord(
@@ -45,11 +47,12 @@ def test_score_based_context_creator():
"test",
RoleType.ASSISTANT,
meta_dict=None,
- content="Nice to meet you.", # 12
+ content="Hello world!", # 10
),
role_at_backend=OpenAIBackendRole.ASSISTANT,
),
- score=0.3,
+ timestamp=datetime.now().timestamp() + 1,
+ score=0.9,
),
ContextRecord(
memory_record=MemoryRecord(
@@ -61,13 +64,14 @@ def test_score_based_context_creator():
),
role_at_backend=OpenAIBackendRole.ASSISTANT,
),
+ timestamp=datetime.now().timestamp() + 2,
score=0.7,
),
]
expected_output = [
r.memory_record.to_openai_message()
- for r in [context_records[0], context_records[2]]
+ for r in [context_records[1], context_records[2]]
]
output, _ = context_creator.create_context(records=context_records)
assert expected_output == output
diff --git a/test/memories/test_blocks.py b/test/memories/test_blocks.py
index 9a748dbfa..45bbcf12d 100644
--- a/test/memories/test_blocks.py
+++ b/test/memories/test_blocks.py
@@ -11,6 +11,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+from datetime import datetime
from unittest.mock import MagicMock, create_autospec
from uuid import uuid4
@@ -45,6 +46,7 @@ def generate_mock_records(num_records: int):
},
"role_at_backend": "user",
"extra_info": {},
+ "timestamp": datetime.now().timestamp() + i,
}
for i in range(num_records)
]
diff --git a/test/memories/test_vector_db_memory.py b/test/memories/test_vector_db_memory.py
new file mode 100644
index 000000000..24c59484c
--- /dev/null
+++ b/test/memories/test_vector_db_memory.py
@@ -0,0 +1,171 @@
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+import random
+from typing import Dict, List, Tuple
+
+import pytest
+
+from camel.memories import MemoryRecord, VectorDBMemory
+from camel.memories.context_creators import ScoreBasedContextCreator
+from camel.messages import BaseMessage
+from camel.types import ModelType, OpenAIBackendRole, RoleType
+from camel.utils.token_counting import OpenAITokenCounter
+
+
+def memory_and_message(request):
+ # Initialize memory with context creator
+ memory = VectorDBMemory(
+ context_creator=ScoreBasedContextCreator(
+ OpenAITokenCounter(ModelType.GPT_4),
+ ModelType.GPT_4.token_limit,
+ )
+ )
+
+ # Add system message
+ memory.write_records(
+ [
+ MemoryRecord(
+ message=BaseMessage(
+ "system",
+ role_type=RoleType.DEFAULT,
+ meta_dict=None,
+ content="You are a helpful assistant",
+ ),
+ role_at_backend=OpenAIBackendRole.SYSTEM,
+ )
+ ]
+ )
+
+ # Add conversations in random order
+ messages = request.param["messages"]
+ random.shuffle(messages)
+
+ # Process each conversation
+ for conv in messages:
+ records = [
+ MemoryRecord(
+ message=BaseMessage(
+ "AI user",
+ role_type=RoleType.USER,
+ meta_dict=None,
+ content=conv["user"],
+ ),
+ role_at_backend=OpenAIBackendRole.USER,
+ ),
+ MemoryRecord(
+ message=BaseMessage(
+ "AI assistant",
+ role_type=RoleType.ASSISTANT,
+ meta_dict=None,
+ content=conv["assistant"],
+ ),
+ role_at_backend=OpenAIBackendRole.ASSISTANT,
+ ),
+ ]
+ memory.write_records(records)
+
+ yield memory, messages, request.param["questions"]
+
+
+ "memory_and_message",
+ [
+ {
+ "messages": [
+ {
+ "user": "What is renewable energy?",
+ "assistant": "Renewable energy is energy that is "
+ "collected from renewable resources, "
+ "which are naturally replenished on a human "
+ "timescale, such as sunlight, wind, rain, "
+ "tides, waves, and geothermal heat.",
+ "category": "energy",
+ },
+ {
+ "user": "How does renewable energy impact the "
+ "environment?",
+ "assistant": "Renewable energy has a positive impact on "
+ "the environment as it produces little to no "
+ "greenhouse gas emissions, reduces air and "
+ "water pollution, and conserves natural "
+ "resources.",
+ "category": "energy",
+ },
+ {
+ "user": "How can I improve my study efficiency",
+ "assistant": "You can improve your study efficiency by "
+ "setting goals, creating a study schedule, "
+ "taking breaks, and using active learning "
+ "techniques.",
+ "category": "education",
+ },
+ {
+ "user": "What strategies can help with language learning?",
+ "assistant": "Strategies that can help with language "
+ "learning include practicing regularly, "
+ "immersing yourself in the language, using "
+ "flashcards, and speaking with native "
+ "speakers.",
+ "category": "education",
+ },
+ ],
+ "questions": [
+ {
+ "user": "What are the advantages of using solar energy "
+ "over fossil fuels?",
+ "category": "energy",
+ },
+ {
+ "user": "How do cultural immersion programs contribute to "
+ "language learning success?",
+ "category": "education",
+ },
+ ],
+ }
+ ],
+ indirect=True,
+)
+def test_vector_db_memory(
+ memory_and_message: Tuple[
+ VectorDBMemory, List[Dict[str, str]], List[Dict[str, str]]
+ ],
+):
+ memory, messages, questions = memory_and_message
+
+ for question in questions:
+ # Add user question to memory
+ user_msg = BaseMessage(
+ "AI user", RoleType.USER, None, question["user"]
+ )
+ memory.write_records(
+ [
+ MemoryRecord(
+ message=user_msg, role_at_backend=OpenAIBackendRole.USER
+ )
+ ]
+ )
+ output_messages, _ = memory.get_context()
+
+ # Check that context messages are from correct category
+ for msg in output_messages[:-1]:
+ for conv in messages:
+ if (
+ conv["assistant"] == msg["content"]
+ or conv["user"] == msg["content"]
+ ):
+ assert conv["category"] == question["category"]
+
+ # Check final message matches input question
+ assert output_messages[-1]["content"] == question["user"]