Skip to content

SWE-agent takes a GitHub issue and tries to automatically fix it, using your LM of choice. It can also be employed for offensive cybersecurity or competitive coding challenges. [NeurIPS 2024]

Cache control and resampling for anthropic

Kilian Lieretacf0a7b2a98b
diff --git a/sweagent/agent/best_response_picker.py b/sweagent/agent/best_response_picker.py
index 472beb9d..4d47a65e 100644
--- a/sweagent/agent/best_response_picker.py
+++ b/sweagent/agent/best_response_picker.py
@@ -87,7 +87,7 @@ class AskColleagues(AbstractBestActionPicker):
class BinaryTrajectoryComparison(AbstractBestActionPicker):
type: Literal["binary_trajectory_comparison"] = "binary_trajectory_comparison"
- system_template: str = """<context>You are an expert software engineer overseeing junior developers. They suggest actions to take to solve a problem. You must choose the best action to take. </context>"""
+ system_template: str = """<setting>You are an expert software engineer overseeing junior developers. They suggest actions to take to solve a problem. You must choose the best action to take. </setting>"""
instance_template: str = dedent("""
We're solving the following problem
@@ -100,7 +100,9 @@ class BinaryTrajectoryComparison(AbstractBestActionPicker):
<trajectory>
{{traj}}
</trajectory>
+ """)
+ comparison_template: str = dedent("""
Two junior developers suggested the following actions:
<thought1>
@@ -144,6 +146,7 @@ class BinaryTrajectoryComparison(AbstractBestActionPicker):
action1: str,
thought2: str,
action2: str,
+ use_cache_control: bool = False,
) -> list[dict]:
system_message = self.system_template
logger.debug(f"MODEL INPUT (system)\n{system_message}")
@@ -154,15 +157,30 @@ class BinaryTrajectoryComparison(AbstractBestActionPicker):
user_message = Template(self.instance_template).render(
**ps_format_dict,
traj=self._format_trajectory(trajectory),
- action1=action1,
- action2=action2,
+ )
+ comparison_message = Template(self.comparison_template).render(
thought1=thought1,
+ action1=action1,
thought2=thought2,
+ action2=action2,
)
logger.debug(f"MODEL INPUT (user)\n{user_message}")
+ cache_control_kwargs = {"cache_control": {"type": "ephemeral"}} if use_cache_control else {}
return [
{"role": "system", "content": system_message},
- {"role": "user", "content": user_message},
+ {
+ "role": "user",
+ "content": [{"type": "text", "text": user_message, **cache_control_kwargs}],
+ },
+ {
+ "role": "user",
+ "content": [
+ {
+ "type": "text",
+ "text": comparison_message,
+ }
+ ],
+ },
]
def get_action(
@@ -190,7 +208,6 @@ class BinaryTrajectoryComparison(AbstractBestActionPicker):
best_idx = 0
comparison_log = []
for i in range(1, len(actions)):
- # todo: Can't I just use the messages itself rather than turning them into a string?
messages = self.format_messages(
problem_statement=problem_statement,
trajectory=trajectory,
@@ -198,6 +215,7 @@ class BinaryTrajectoryComparison(AbstractBestActionPicker):
action1=actions[best_idx],
thought2=thoughts[i],
action2=actions[i],
+ use_cache_control=len(actions) >= 3,
)
response = self._model.query(messages)["message"] # type: ignore
logger.info(f"RESPONSE: {response}")
diff --git a/sweagent/agent/history_processors.py b/sweagent/agent/history_processors.py
index 456d49d2..921d088c 100644
--- a/sweagent/agent/history_processors.py
+++ b/sweagent/agent/history_processors.py
@@ -157,7 +157,62 @@ class ClosedWindowHistoryProcessor(BaseModel):
return list(reversed(new_history))
+class CacheControlHistoryProcessor(BaseModel):
+ type: Literal["cache_control"] = "cache_control"
+ """Do not change. Used for (de)serialization."""
+
+ last_n_messages: int = 2
+ """Add cache control to the last n user messages (and clear it for anything else).
+ In most cases this should be set to 2 (caching for multi-turn conversations).
+ When resampling and running concurrent instances, you want to set it to 1.
+ If set to <= 0, any set cache control will be removed from all messages.
+ """
+
+ # pydantic config
+ model_config = ConfigDict(extra="forbid")
+
+ def _get_content_text(self, entry: HistoryItem) -> str:
+ if isinstance(entry["content"], list):
+ assert len(entry["content"]) == 1, "Expected single message in content"
+ return entry["content"][0]["text"]
+ return entry["content"]
+
+ def _clear_cache_control(self, entry: HistoryItem) -> None:
+ if isinstance(entry["content"], list):
+ assert len(entry["content"]) == 1, "Expected single message in content"
+ if "cache_control" in entry["content"][0]:
+ del entry["content"][0]["cache_control"]
+
+ def _set_cache_control(self, entry: HistoryItem) -> None:
+ if not isinstance(entry["content"], list):
+ entry["content"] = [ # type: ignore
+ {
+ "type": "text",
+ "text": self._get_content_text(entry),
+ "cache_control": {"type": "ephemeral"},
+ }
+ ]
+ else:
+ entry["content"][0]["cache_control"] = {"type": "ephemeral"}
+
+ def __call__(self, history: History) -> History:
+ new_history = []
+ n_tagged = 0
+ for entry in reversed(history):
+ # Clear cache control from previous messages
+ self._clear_cache_control(entry)
+ if n_tagged < self.last_n_messages and entry["role"] == "user":
+ self._set_cache_control(entry)
+ n_tagged += 1
+ new_history.append(entry)
+ return list(reversed(new_history))
+
+
HistoryProcessor = Annotated[
- DefaultHistoryProcessor | LastNObservations | ClosedWindowHistoryProcessor | TagToolCallObservations,
+ DefaultHistoryProcessor
+ | LastNObservations
+ | ClosedWindowHistoryProcessor
+ | TagToolCallObservations
+ | CacheControlHistoryProcessor,
Field(discriminator="type"),
]
diff --git a/sweagent/agent/models.py b/sweagent/agent/models.py
index 2d13b981..d4a8ed09 100644
--- a/sweagent/agent/models.py
+++ b/sweagent/agent/models.py
@@ -1,6 +1,5 @@
from __future__ import annotations
-import copy
import json
import random
import shlex
@@ -501,7 +500,15 @@ class LiteLLMModel(AbstractModel):
return None
return random.choice(api_keys)
- def _query(self, messages: list[dict[str, str]], n: int | None = None) -> dict | list[dict]:
+ def _sleep(self) -> None:
+ with GLOBAL_STATS_LOCK:
+ GLOBAL_STATS.last_query_timestamp = time.time()
+ elapsed_time = time.time() - GLOBAL_STATS.last_query_timestamp
+ if elapsed_time < self.config.delay:
+ time.sleep(self.config.delay - elapsed_time)
+
+ def _single_query(self, messages: list[dict[str, str]], n: int | None = None) -> list[dict]:
+ self._sleep()
input_tokens: int = litellm.utils.token_counter(messages=messages, model=self.config.name)
if self.model_max_input_tokens is None:
self.logger.warning(f"No max input tokens found for model {self.config.name!r}")
@@ -548,16 +555,19 @@ class LiteLLMModel(AbstractModel):
output_dict["tool_calls"] = tool_calls
outputs.append(output_dict)
self._update_stats(input_tokens=input_tokens, output_tokens=output_tokens, cost=cost)
+ return outputs
+
+ def _query(self, messages: list[dict[str, str]], n: int | None = None) -> list[dict]:
if n is None:
- return outputs[0]
+ return self._single_query(messages)
+ outputs = []
+ # not needed for openai, but oh well.
+ for _ in range(n):
+ outputs.extend(self._single_query(messages))
return outputs
- def query(self, history: History, n: int | None = None) -> dict:
- elapsed_time = time.time() - GLOBAL_STATS.last_query_timestamp
- if elapsed_time < self.config.delay:
- time.sleep(self.config.delay - elapsed_time)
- with GLOBAL_STATS_LOCK:
- GLOBAL_STATS.last_query_timestamp = time.time()
+ def query(self, history: History, n: int = 1) -> list[dict] | dict:
+ messages = self._history_to_messages(history)
for attempt in Retrying(
stop=stop_after_attempt(self.config.retry.retries),
wait=wait_random_exponential(min=self.config.retry.min_wait, max=self.config.retry.max_wait),
@@ -586,8 +596,9 @@ class LiteLLMModel(AbstractModel):
f"(slept for {attempt.retry_state.idle_for:.2f}s)"
f"{exception_info}"
)
- messages = self._history_to_messages(history)
result = self._query(messages, n=n)
+ if n is None or n == 1:
+ return result[0]
return result
def _history_to_messages(
@@ -617,29 +628,7 @@ class LiteLLMModel(AbstractModel):
)
else:
messages.append({"role": role, "content": history_item["content"]})
- return self._add_cache_control(messages)
-
- def _add_cache_control(self, messages: list[dict[str, str]]) -> list[dict[str, Any]]:
- """Adds cache control headers to the messages"""
- n_cached = 0
- new_messages = []
- messages = copy.deepcopy(messages)
- for message in reversed(messages):
- if (message["role"] == "assistant" and n_cached < 3) or (message["role"] == "system"):
- print("Caching message", message["content"][:50])
- message["content"] = [ # type: ignore
- {
- "type": "text",
- "text": message["content"],
- "cache_control": {"type": "ephemeral"},
- }
- ]
- new_messages.append(message)
- n_cached += 1
- else:
- new_messages.append(message)
- self.logger.info(f"Total cache breakpoints: {n_cached}")
- return list(reversed(new_messages))
+ return messages
def get_model(args: ModelConfig, tools: ToolConfig) -> AbstractModel:
diff --git a/sweagent/types.py b/sweagent/types.py
index 4bae478a..759a68aa 100644
--- a/sweagent/types.py
+++ b/sweagent/types.py
@@ -53,7 +53,7 @@ class TrajectoryStep(TypedDict):
# required fields go here
class _HistoryItem(TypedDict):
role: str
- content: str
+ content: str | list[dict[str, Any]]
message_type: Literal["thought", "action", "observation"]