Skip to content

AutoGPT is the vision of accessible AI for everyone, to use and to build on. Our mission is to provide the tools, so that you can focus on what matters.

fix(backend): reduce compress_context tokenization work (#13478)

Dexcfbef8e71e7d
diff --git a/autogpt_platform/backend/backend/util/prompt.py b/autogpt_platform/backend/backend/util/prompt.py
index 46ba13ba6c..d8d872237f 100644
--- a/autogpt_platform/backend/backend/util/prompt.py
+++ b/autogpt_platform/backend/backend/util/prompt.py
@@ -849,26 +849,41 @@ async def compress_context(
content_str = _truncate_middle_tokens(content_str, enc, 20_000)
m["content"] = content_str
+ # Steps 3-5 repeatedly check the total size while mutating or deleting one
+ # message at a time. Re-summing all messages there re-tokenizes the whole
+ # history on each iteration, which is quadratic on long transcripts.
+ counts = [_msg_tokens(m, enc) for m in msgs]
+ total = sum(counts)
+
+ def _recount(i: int) -> None:
+ nonlocal total
+ new = _msg_tokens(msgs[i], enc)
+ total += new - counts[i]
+ counts[i] = new
+
# ---- STEP 3: Token-aware content truncation ---------------------------
# Progressively halve per-message cap and truncate bloated content.
# This preserves all messages but shortens their content.
cap = start_cap
- while total_tokens() + reserve > target_tokens and cap >= floor_cap:
- for m in msgs[1:-1]:
+ while total + reserve > target_tokens and cap >= floor_cap:
+ for i in range(1, len(msgs) - 1):
+ m = msgs[i]
if _is_tool_message(m):
_truncate_tool_message_content(m, enc, cap)
+ _recount(i)
continue
if _is_objective_message(m):
continue
content = m.get("content") or ""
if _tok_len(content, enc) > cap:
m["content"] = _truncate_middle_tokens(content, enc, cap)
+ _recount(i)
cap //= 2
# ---- STEP 4: Middle-out deletion --------------------------------------
# Delete messages one at a time from the center outward.
# This is more granular than dropping all old messages at once.
- while total_tokens() + reserve > target_tokens and len(msgs) > 2:
+ while total + reserve > target_tokens and len(msgs) > 2:
deletable: list[int] = []
for i in range(1, len(msgs) - 1):
msg = msgs[i]
@@ -883,21 +898,24 @@ async def compress_context(
centre = len(msgs) // 2
to_delete = min(deletable, key=lambda i: abs(i - centre))
del msgs[to_delete]
+ total -= counts.pop(to_delete)
messages_dropped += 1
# ---- STEP 5: Final trim on first/last ---------------------------------
cap = start_cap
- while total_tokens() + reserve > target_tokens and cap >= floor_cap:
+ while total + reserve > target_tokens and cap >= floor_cap:
for idx in (0, -1):
msg = msgs[idx]
if msg is None:
continue
if _is_tool_message(msg):
_truncate_tool_message_content(msg, enc, cap)
+ _recount(idx)
continue
text = msg.get("content") or ""
if _tok_len(text, enc) > cap:
msg["content"] = _truncate_middle_tokens(text, enc, cap)
+ _recount(idx)
cap //= 2
# Filter out any None values that may have been introduced
diff --git a/autogpt_platform/backend/backend/util/prompt_test.py b/autogpt_platform/backend/backend/util/prompt_test.py
index 9461dbb944..1d7aeee5ff 100644
--- a/autogpt_platform/backend/backend/util/prompt_test.py
+++ b/autogpt_platform/backend/backend/util/prompt_test.py
@@ -8,6 +8,7 @@ from tiktoken import encoding_for_model
from backend.util import json
from backend.util.prompt import (
DEFAULT_TOKEN_THRESHOLD,
+ MAIN_OBJECTIVE_PREFIX,
CompressResult,
_ensure_tool_pairs_intact,
_msg_tokens,
@@ -746,6 +747,107 @@ class TestCompressContext:
# Should have truncated without summarization
assert result.messages_summarized == 0
+ @pytest.mark.asyncio
+ async def test_final_tool_trim_updates_token_count(self):
+ """Keep incremental accounting accurate when Step 5 trims a tool result."""
+ enc = encoding_for_model("gpt-4o")
+ messages = [
+ {
+ "type": "function_call",
+ "call_id": "call_x",
+ "name": "lookup",
+ "arguments": "{}",
+ },
+ {
+ "type": "function_call_output",
+ "call_id": "call_x",
+ "output": "tool result " * 100,
+ },
+ ]
+
+ result = await compress_context(
+ messages,
+ target_tokens=40,
+ client=None,
+ reserve=0,
+ model="gpt-4o",
+ start_cap=20,
+ floor_cap=20,
+ )
+
+ assert result.was_compacted is True
+ assert result.messages_dropped == 0
+ assert result.error is None
+ assert len(enc.encode(result.messages[-1]["output"])) < len(
+ enc.encode(messages[-1]["output"])
+ )
+ assert result.token_count == sum(
+ _msg_tokens(message, enc) for message in result.messages
+ )
+
+ @pytest.mark.asyncio
+ async def test_large_history_middle_out_deletion_stays_correct(self):
+ """Guard incremental token accounting in compress_context."""
+ enc = encoding_for_model("gpt-4o")
+ target = 4000
+ reserve = 256
+
+ messages = [{"role": "system", "content": "You are a helpful assistant."}]
+ messages.append(
+ {
+ "role": "user",
+ "content": MAIN_OBJECTIVE_PREFIX + " keep this objective intact.",
+ }
+ )
+ for i in range(150):
+ role = "user" if i % 2 == 0 else "assistant"
+ body = (
+ f"item{i} " + "lorem ipsum dolor sit amet consectetur " * 15
+ ).strip()
+ messages.append({"role": role, "content": body})
+ messages.append(
+ {
+ "role": "assistant",
+ "tool_calls": [
+ {
+ "id": "call_x",
+ "type": "function",
+ "function": {"name": "f", "arguments": "{}"},
+ }
+ ],
+ }
+ )
+ messages.append(
+ {"role": "tool", "tool_call_id": "call_x", "content": "tool result " * 50}
+ )
+
+ result = await compress_context(
+ messages, target_tokens=target, client=None, reserve=reserve, model="gpt-4o"
+ )
+
+ assert result.was_compacted is True
+ assert result.messages_dropped > 0
+ assert result.error is None
+ assert result.token_count + reserve <= target
+ assert result.token_count == sum(_msg_tokens(m, enc) for m in result.messages)
+
+ assert result.messages[0]["role"] == "system"
+ assert any(
+ isinstance(m.get("content"), str)
+ and m["content"].startswith(MAIN_OBJECTIVE_PREFIX)
+ for m in result.messages
+ )
+
+ tool_call_ids = {
+ tc["id"] for m in result.messages for tc in (m.get("tool_calls") or [])
+ }
+ tool_response_ids = {
+ m["tool_call_id"] for m in result.messages if m.get("tool_call_id")
+ }
+ assert tool_response_ids <= tool_call_ids
+
+ assert result.messages_dropped == 113
+
@pytest.mark.asyncio
async def test_with_mocked_llm_client(self):
"""Test summarization with mocked LLM client."""