Skip to content

armgit / camel

Public Imported from GitHub
0

🐫 CAMEL: The first and the best multi-agent framework. Finding the Scaling Law of Agents. https://www.camel-ai.org

enhance: history snapshot clean (#3409)

Puzhen Zhangbbcdb3be31ec
diff --git a/camel/agents/chat_agent.py b/camel/agents/chat_agent.py
index 93774568d..3e8e89af0 100644
--- a/camel/agents/chat_agent.py
+++ b/camel/agents/chat_agent.py
@@ -29,6 +29,7 @@ import threading
import time
import uuid
import warnings
+from dataclasses import dataclass
from datetime import datetime
from pathlib import Path
from typing import (
@@ -169,6 +170,16 @@ SIMPLE_FORMAT_PROMPT = TextPrompt(
)
+@dataclass
+class _ToolOutputHistoryEntry:
+ tool_name: str
+ tool_call_id: str
+ result_text: str
+ record_uuids: List[str]
+ record_timestamps: List[float]
+ cached: bool = False
+
+
class StreamContentAccumulator:
r"""Manages content accumulation across streaming responses to ensure
all responses contain complete cumulative content."""
@@ -414,6 +425,11 @@ class ChatAgent(BaseAgent):
usage. When enabled, removes FUNCTION/TOOL role messages and
ASSISTANT messages with tool_calls after each step.
(default: :obj:`False`)
+ enable_snapshot_clean (bool, optional): Whether to clean snapshot
+ markers and references from historical tool outputs in memory.
+ This removes verbose DOM markers (like [ref=...]) from older tool
+ results while keeping the latest output intact for immediate use.
+ (default: :obj:`False`)
retry_attempts (int, optional): Maximum number of retry attempts for
rate limit errors. (default: :obj:`3`)
retry_delay (float, optional): Initial delay in seconds between
@@ -470,6 +486,7 @@ class ChatAgent(BaseAgent):
mask_tool_output: bool = False,
pause_event: Optional[Union[threading.Event, asyncio.Event]] = None,
prune_tool_calls_from_memory: bool = False,
+ enable_snapshot_clean: bool = False,
retry_attempts: int = 3,
retry_delay: float = 1.0,
step_timeout: Optional[float] = None,
@@ -490,6 +507,9 @@ class ChatAgent(BaseAgent):
# Assign unique ID
self.agent_id = agent_id if agent_id else str(uuid.uuid4())
+ self._enable_snapshot_clean = enable_snapshot_clean
+ self._tool_output_history: List[_ToolOutputHistoryEntry] = []
+
# Set up memory
context_creator = ScoreBasedContextCreator(
self.model_backend.token_counter,
@@ -1133,6 +1153,282 @@ class ChatAgent(BaseAgent):
for tool in tools:
self.add_tool(tool)
+ def _serialize_tool_result(self, result: Any) -> str:
+ if isinstance(result, str):
+ return result
+ try:
+ return json.dumps(result, ensure_ascii=False)
+ except (TypeError, ValueError):
+ return str(result)
+
+ def _clean_snapshot_line(self, line: str) -> str:
+ r"""Clean a single snapshot line by removing prefixes and references.
+
+ This method handles snapshot lines in the format:
+ - [prefix] "quoted text" [attributes] [ref=...]: description
+
+ It preserves:
+ - Quoted text content (including brackets inside quotes)
+ - Description text after the colon
+
+ It removes:
+ - Line prefixes (e.g., "- button", "- tooltip", "generic:")
+ - Attribute markers (e.g., [disabled], [ref=e47])
+ - Lines with only element types
+ - All indentation
+
+ Args:
+ line: The original line content.
+
+ Returns:
+ The cleaned line content, or empty string if line should be
+ removed.
+ """
+ original = line.strip()
+ if not original:
+ return ''
+
+ # Check if line is just an element type marker
+ # (e.g., "- generic:", "button:")
+ if re.match(r'^(?:-\s+)?\w+\s*:?\s*$', original):
+ return ''
+
+ # Remove element type prefix
+ line = re.sub(r'^(?:-\s+)?\w+[\s:]+', '', original)
+
+ # Remove bracket markers while preserving quoted text
+ quoted_parts = []
+
+ def save_quoted(match):
+ quoted_parts.append(match.group(0))
+ return f'__QUOTED_{len(quoted_parts)-1}__'
+
+ line = re.sub(r'"[^"]*"', save_quoted, line)
+ line = re.sub(r'\s*\[[^\]]+\]\s*', ' ', line)
+
+ for i, quoted in enumerate(quoted_parts):
+ line = line.replace(f'__QUOTED_{i}__', quoted)
+
+ # Clean up formatting
+ line = re.sub(r'\s+', ' ', line).strip()
+ line = re.sub(r'\s*:\s*', ': ', line)
+ line = line.lstrip(': ').strip()
+
+ return '' if not line else line
+
+ def _clean_snapshot_content(self, content: str) -> str:
+ r"""Clean snapshot content by removing prefixes, references, and
+ deduplicating lines.
+
+ This method identifies snapshot lines (containing element keywords or
+ references) and cleans them while preserving non-snapshot content.
+ It also handles JSON-formatted tool outputs with snapshot fields.
+
+ Args:
+ content: The original snapshot content.
+
+ Returns:
+ The cleaned content with deduplicated lines.
+ """
+ try:
+ import json
+
+ data = json.loads(content)
+ modified = False
+
+ def clean_json_value(obj):
+ nonlocal modified
+ if isinstance(obj, dict):
+ result = {}
+ for key, value in obj.items():
+ if key == 'snapshot' and isinstance(value, str):
+ try:
+ decoded_value = value.encode().decode(
+ 'unicode_escape'
+ )
+ except (UnicodeDecodeError, AttributeError):
+ decoded_value = value
+
+ needs_cleaning = (
+ '- ' in decoded_value
+ or '[ref=' in decoded_value
+ or any(
+ elem + ':' in decoded_value
+ for elem in [
+ 'generic',
+ 'img',
+ 'banner',
+ 'list',
+ 'listitem',
+ 'search',
+ 'navigation',
+ ]
+ )
+ )
+
+ if needs_cleaning:
+ cleaned_snapshot = self._clean_text_snapshot(
+ decoded_value
+ )
+ result[key] = cleaned_snapshot
+ modified = True
+ else:
+ result[key] = value
+ else:
+ result[key] = clean_json_value(value)
+ return result
+ elif isinstance(obj, list):
+ return [clean_json_value(item) for item in obj]
+ else:
+ return obj
+
+ cleaned_data = clean_json_value(data)
+
+ if modified:
+ return json.dumps(cleaned_data, ensure_ascii=False, indent=4)
+ else:
+ return content
+
+ except (json.JSONDecodeError, TypeError):
+ return self._clean_text_snapshot(content)
+
+ def _clean_text_snapshot(self, content: str) -> str:
+ r"""Clean plain text snapshot content.
+
+ This method:
+ - Removes all indentation
+ - Deletes empty lines
+ - Deduplicates all lines
+ - Cleans snapshot-specific markers
+
+ Args:
+ content: The original snapshot text.
+
+ Returns:
+ The cleaned content with deduplicated lines, no indentation,
+ and no empty lines.
+ """
+ lines = content.split('\n')
+ cleaned_lines = []
+ seen = set()
+
+ for line in lines:
+ stripped_line = line.strip()
+
+ if not stripped_line:
+ continue
+
+ # Skip metadata lines (like "- /url:", "- /ref:")
+ if re.match(r'^-?\s*/\w+\s*:', stripped_line):
+ continue
+
+ is_snapshot_line = '[ref=' in stripped_line or re.match(
+ r'^(?:-\s+)?\w+(?:[\s:]|$)', stripped_line
+ )
+
+ if is_snapshot_line:
+ cleaned = self._clean_snapshot_line(stripped_line)
+ if cleaned and cleaned not in seen:
+ cleaned_lines.append(cleaned)
+ seen.add(cleaned)
+ else:
+ if stripped_line not in seen:
+ cleaned_lines.append(stripped_line)
+ seen.add(stripped_line)
+
+ return '\n'.join(cleaned_lines)
+
+ def _register_tool_output_for_cache(
+ self,
+ func_name: str,
+ tool_call_id: str,
+ result_text: str,
+ records: List[MemoryRecord],
+ ) -> None:
+ if not records:
+ return
+
+ entry = _ToolOutputHistoryEntry(
+ tool_name=func_name,
+ tool_call_id=tool_call_id,
+ result_text=result_text,
+ record_uuids=[str(record.uuid) for record in records],
+ record_timestamps=[record.timestamp for record in records],
+ )
+ self._tool_output_history.append(entry)
+ self._process_tool_output_cache()
+
+ def _process_tool_output_cache(self) -> None:
+ if not self._enable_snapshot_clean or not self._tool_output_history:
+ return
+
+ # Only clean older results; keep the latest expanded for immediate use.
+ for entry in self._tool_output_history[:-1]:
+ if entry.cached:
+ continue
+ self._clean_snapshot_in_memory(entry)
+
+ def _clean_snapshot_in_memory(
+ self, entry: _ToolOutputHistoryEntry
+ ) -> None:
+ if not entry.record_uuids:
+ return
+
+ # Clean snapshot markers and references from historical tool output
+ result_text = entry.result_text
+ if '- ' in result_text and '[ref=' in result_text:
+ cleaned_result = self._clean_snapshot_content(result_text)
+
+ # Update the message in memory storage
+ timestamp = (
+ entry.record_timestamps[0]
+ if entry.record_timestamps
+ else time.time_ns() / 1_000_000_000
+ )
+ cleaned_message = FunctionCallingMessage(
+ role_name=self.role_name,
+ role_type=self.role_type,
+ meta_dict={},
+ content="",
+ func_name=entry.tool_name,
+ result=cleaned_result,
+ tool_call_id=entry.tool_call_id,
+ )
+
+ chat_history_block = getattr(
+ self.memory, "_chat_history_block", None
+ )
+ storage = getattr(chat_history_block, "storage", None)
+ if storage is None:
+ return
+
+ existing_records = storage.load()
+ updated_records = [
+ record
+ for record in existing_records
+ if record["uuid"] not in entry.record_uuids
+ ]
+ new_record = MemoryRecord(
+ message=cleaned_message,
+ role_at_backend=OpenAIBackendRole.FUNCTION,
+ timestamp=timestamp,
+ agent_id=self.agent_id,
+ )
+ updated_records.append(new_record.to_dict())
+ updated_records.sort(key=lambda record: record["timestamp"])
+ storage.clear()
+ storage.save(updated_records)
+
+ logger.info(
+ "Cleaned snapshot in memory for tool output '%s' (%s)",
+ entry.tool_name,
+ entry.tool_call_id,
+ )
+
+ entry.cached = True
+ entry.record_uuids = [str(new_record.uuid)]
+ entry.record_timestamps = [timestamp]
+
def add_external_tool(
self, tool: Union[FunctionTool, Callable, Dict[str, Any]]
) -> None:
@@ -1177,7 +1473,8 @@ class ChatAgent(BaseAgent):
message: BaseMessage,
role: OpenAIBackendRole,
timestamp: Optional[float] = None,
- ) -> None:
+ return_records: bool = False,
+ ) -> Optional[List[MemoryRecord]]:
r"""Updates the agent memory with a new message.
Args:
@@ -1187,7 +1484,13 @@ class ChatAgent(BaseAgent):
timestamp (Optional[float], optional): Custom timestamp for the
memory record. If `None`, the current time will be used.
(default: :obj:`None`)
- (default: obj:`None`)
+ return_records (bool, optional): When ``True`` the method returns
+ the list of MemoryRecord objects written to memory.
+ (default: :obj:`False`)
+
+ Returns:
+ Optional[List[MemoryRecord]]: The records that were written when
+ ``return_records`` is ``True``; otherwise ``None``.
"""
record = MemoryRecord(
message=message,
@@ -1199,6 +1502,10 @@ class ChatAgent(BaseAgent):
)
self.memory.write_record(record)
+ if return_records:
+ return [record]
+ return None
+
def load_memory(self, memory: AgentMemory) -> None:
r"""Load the provided memory into the agent.
@@ -3552,16 +3859,30 @@ class ChatAgent(BaseAgent):
base_timestamp = current_time_ns / 1_000_000_000 # Convert to seconds
self.update_memory(
- assist_msg, OpenAIBackendRole.ASSISTANT, timestamp=base_timestamp
+ assist_msg,
+ OpenAIBackendRole.ASSISTANT,
+ timestamp=base_timestamp,
+ return_records=self._enable_snapshot_clean,
)
# Add minimal increment to ensure function message comes after
- self.update_memory(
+ func_records = self.update_memory(
func_msg,
OpenAIBackendRole.FUNCTION,
timestamp=base_timestamp + 1e-6,
+ return_records=self._enable_snapshot_clean,
)
+ # Register tool output for snapshot cleaning if enabled
+ if self._enable_snapshot_clean and not mask_output and func_records:
+ serialized_result = self._serialize_tool_result(result)
+ self._register_tool_output_for_cache(
+ func_name,
+ tool_call_id,
+ serialized_result,
+ cast(List[MemoryRecord], func_records),
+ )
+
# Record information about this tool call
tool_record = ToolCallingRecord(
tool_name=func_name,