Skip to content

πŸ™Œ OpenHands: AI-Driven Development

fix: Disable prompt caching in default condenser (#7781)

Calvin Smith36e092e0ac0c
diff --git a/openhands/memory/condenser/impl/llm_attention_condenser.py b/openhands/memory/condenser/impl/llm_attention_condenser.py
index 8e869a3a08..106e5a46b1 100644
--- a/openhands/memory/condenser/impl/llm_attention_condenser.py
+++ b/openhands/memory/condenser/impl/llm_attention_condenser.py
@@ -114,8 +114,14 @@ class LLMAttentionCondenser(RollingCondenser):
@classmethod
def from_config(cls, config: LLMAttentionCondenserConfig) -> LLMAttentionCondenser:
+ # This condenser cannot take advantage of prompt caching. If it happens
+ # to be set, we'll pay for the cache writes but never get a chance to
+ # save on a read.
+ llm_config = config.llm_config.model_copy()
+ llm_config.caching_prompt = False
+
return LLMAttentionCondenser(
- llm=LLM(config=config.llm_config),
+ llm=LLM(config=llm_config),
max_size=config.max_size,
keep_first=config.keep_first,
)
diff --git a/openhands/memory/condenser/impl/llm_summarizing_condenser.py b/openhands/memory/condenser/impl/llm_summarizing_condenser.py
index 8d5a46fd47..5911e321b2 100644
--- a/openhands/memory/condenser/impl/llm_summarizing_condenser.py
+++ b/openhands/memory/condenser/impl/llm_summarizing_condenser.py
@@ -155,8 +155,14 @@ CURRENT_STATE: Last flip: Heads, Haiku count: 15/20"""
def from_config(
cls, config: LLMSummarizingCondenserConfig
) -> LLMSummarizingCondenser:
+ # This condenser cannot take advantage of prompt caching. If it happens
+ # to be set, we'll pay for the cache writes but never get a chance to
+ # save on a read.
+ llm_config = config.llm_config.model_copy()
+ llm_config.caching_prompt = False
+
return LLMSummarizingCondenser(
- llm=LLM(config=config.llm_config),
+ llm=LLM(config=llm_config),
max_size=config.max_size,
keep_first=config.keep_first,
max_event_length=config.max_event_length,
diff --git a/openhands/memory/condenser/impl/structured_summary_condenser.py b/openhands/memory/condenser/impl/structured_summary_condenser.py
index fa9fb7f630..6bfe875d3e 100644
--- a/openhands/memory/condenser/impl/structured_summary_condenser.py
+++ b/openhands/memory/condenser/impl/structured_summary_condenser.py
@@ -311,8 +311,14 @@ Capture all relevant information, especially:
def from_config(
cls, config: StructuredSummaryCondenserConfig
) -> StructuredSummaryCondenser:
+ # This condenser cannot take advantage of prompt caching. If it happens
+ # to be set, we'll pay for the cache writes but never get a chance to
+ # save on a read.
+ llm_config = config.llm_config.model_copy()
+ llm_config.caching_prompt = False
+
return StructuredSummaryCondenser(
- llm=LLM(config=config.llm_config),
+ llm=LLM(config=llm_config),
max_size=config.max_size,
keep_first=config.keep_first,
max_event_length=config.max_event_length,
diff --git a/tests/unit/test_condenser.py b/tests/unit/test_condenser.py
index c0b82fd53b..af77cb500f 100644
--- a/tests/unit/test_condenser.py
+++ b/tests/unit/test_condenser.py
@@ -331,10 +331,7 @@ def test_llm_summarizing_condenser_from_config():
config = LLMSummarizingCondenserConfig(
max_size=50,
keep_first=10,
- llm_config=LLMConfig(
- model='gpt-4o',
- api_key='test_key',
- ),
+ llm_config=LLMConfig(model='gpt-4o', api_key='test_key', caching_prompt=True),
)
condenser = Condenser.from_config(config)
@@ -344,6 +341,10 @@ def test_llm_summarizing_condenser_from_config():
assert condenser.max_size == 50
assert condenser.keep_first == 10
+ # Since this condenser can't take advantage of caching, we intercept the
+ # passed config and manually flip the caching prompt to False.
+ assert not condenser.llm.config.caching_prompt
+
def test_llm_summarizing_condenser_invalid_config():
"""Test that LLMSummarizingCondenser raises error when keep_first > max_size."""
@@ -474,6 +475,7 @@ def test_llm_attention_condenser_from_config():
llm_config=LLMConfig(
model='gpt-4o',
api_key='test_key',
+ caching_prompt=True,
),
)
condenser = Condenser.from_config(config)
@@ -484,6 +486,10 @@ def test_llm_attention_condenser_from_config():
assert condenser.max_size == 50
assert condenser.keep_first == 10
+ # Since this condenser can't take advantage of caching, we intercept the
+ # passed config and manually flip the caching prompt to False.
+ assert not condenser.llm.config.caching_prompt
+
def test_llm_attention_condenser_invalid_config():
"""Test that LLMAttentionCondenser raises an error if the configured LLM doesn't support response schema."""
@@ -614,6 +620,7 @@ def test_structured_summary_condenser_from_config():
llm_config=LLMConfig(
model='gpt-4o',
api_key='test_key',
+ caching_prompt=True,
),
)
condenser = Condenser.from_config(config)
@@ -624,6 +631,10 @@ def test_structured_summary_condenser_from_config():
assert condenser.max_size == 50
assert condenser.keep_first == 10
+ # Since this condenser can't take advantage of caching, we intercept the
+ # passed config and manually flip the caching prompt to False.
+ assert not condenser.llm.config.caching_prompt
+
def test_structured_summary_condenser_invalid_config():
"""Test that StructuredSummaryCondenser raises error when keep_first > max_size."""