SWE-agent takes a GitHub issue and tries to automatically fix it, using your LM of choice. It can also be employed for offensive cybersecurity or competitive coding challenges. [NeurIPS 2024]
Cache control and resampling for anthropic
Kilian Lieretacf0a7b2a98b
diff --git a/sweagent/agent/best_response_picker.py b/sweagent/agent/best_response_picker.pyindex 472beb9d..4d47a65e 100644--- a/sweagent/agent/best_response_picker.py+++ b/sweagent/agent/best_response_picker.py@@ -87,7 +87,7 @@ class AskColleagues(AbstractBestActionPicker):class BinaryTrajectoryComparison(AbstractBestActionPicker):type: Literal["binary_trajectory_comparison"] = "binary_trajectory_comparison"- system_template: str = """<context>You are an expert software engineer overseeing junior developers. They suggest actions to take to solve a problem. You must choose the best action to take. </context>"""+ system_template: str = """<setting>You are an expert software engineer overseeing junior developers. They suggest actions to take to solve a problem. You must choose the best action to take. </setting>"""instance_template: str = dedent("""We're solving the following problem@@ -100,7 +100,9 @@ class BinaryTrajectoryComparison(AbstractBestActionPicker):<trajectory>{{traj}}</trajectory>+ """)+ comparison_template: str = dedent("""Two junior developers suggested the following actions:<thought1>@@ -144,6 +146,7 @@ class BinaryTrajectoryComparison(AbstractBestActionPicker):action1: str,thought2: str,action2: str,+ use_cache_control: bool = False,) -> list[dict]:system_message = self.system_templatelogger.debug(f"MODEL INPUT (system)\n{system_message}")@@ -154,15 +157,30 @@ class BinaryTrajectoryComparison(AbstractBestActionPicker):user_message = Template(self.instance_template).render(**ps_format_dict,traj=self._format_trajectory(trajectory),- action1=action1,- action2=action2,+ )+ comparison_message = Template(self.comparison_template).render(thought1=thought1,+ action1=action1,thought2=thought2,+ action2=action2,)logger.debug(f"MODEL INPUT (user)\n{user_message}")+ cache_control_kwargs = {"cache_control": {"type": "ephemeral"}} if use_cache_control else {}return [{"role": "system", "content": system_message},- {"role": "user", "content": user_message},+ {+ "role": "user",+ "content": [{"type": "text", "text": user_message, **cache_control_kwargs}],+ },+ {+ "role": "user",+ "content": [+ {+ "type": "text",+ "text": comparison_message,+ }+ ],+ },]def get_action(@@ -190,7 +208,6 @@ class BinaryTrajectoryComparison(AbstractBestActionPicker):best_idx = 0comparison_log = []for i in range(1, len(actions)):- # todo: Can't I just use the messages itself rather than turning them into a string?messages = self.format_messages(problem_statement=problem_statement,trajectory=trajectory,@@ -198,6 +215,7 @@ class BinaryTrajectoryComparison(AbstractBestActionPicker):action1=actions[best_idx],thought2=thoughts[i],action2=actions[i],+ use_cache_control=len(actions) >= 3,)response = self._model.query(messages)["message"] # type: ignorelogger.info(f"RESPONSE: {response}")diff --git a/sweagent/agent/history_processors.py b/sweagent/agent/history_processors.pyindex 456d49d2..921d088c 100644--- a/sweagent/agent/history_processors.py+++ b/sweagent/agent/history_processors.py@@ -157,7 +157,62 @@ class ClosedWindowHistoryProcessor(BaseModel):return list(reversed(new_history))+class CacheControlHistoryProcessor(BaseModel):+ type: Literal["cache_control"] = "cache_control"+ """Do not change. Used for (de)serialization."""++ last_n_messages: int = 2+ """Add cache control to the last n user messages (and clear it for anything else).+ In most cases this should be set to 2 (caching for multi-turn conversations).+ When resampling and running concurrent instances, you want to set it to 1.+ If set to <= 0, any set cache control will be removed from all messages.+ """++ # pydantic config+ model_config = ConfigDict(extra="forbid")++ def _get_content_text(self, entry: HistoryItem) -> str:+ if isinstance(entry["content"], list):+ assert len(entry["content"]) == 1, "Expected single message in content"+ return entry["content"][0]["text"]+ return entry["content"]++ def _clear_cache_control(self, entry: HistoryItem) -> None:+ if isinstance(entry["content"], list):+ assert len(entry["content"]) == 1, "Expected single message in content"+ if "cache_control" in entry["content"][0]:+ del entry["content"][0]["cache_control"]++ def _set_cache_control(self, entry: HistoryItem) -> None:+ if not isinstance(entry["content"], list):+ entry["content"] = [ # type: ignore+ {+ "type": "text",+ "text": self._get_content_text(entry),+ "cache_control": {"type": "ephemeral"},+ }+ ]+ else:+ entry["content"][0]["cache_control"] = {"type": "ephemeral"}++ def __call__(self, history: History) -> History:+ new_history = []+ n_tagged = 0+ for entry in reversed(history):+ # Clear cache control from previous messages+ self._clear_cache_control(entry)+ if n_tagged < self.last_n_messages and entry["role"] == "user":+ self._set_cache_control(entry)+ n_tagged += 1+ new_history.append(entry)+ return list(reversed(new_history))++HistoryProcessor = Annotated[- DefaultHistoryProcessor | LastNObservations | ClosedWindowHistoryProcessor | TagToolCallObservations,+ DefaultHistoryProcessor+ | LastNObservations+ | ClosedWindowHistoryProcessor+ | TagToolCallObservations+ | CacheControlHistoryProcessor,Field(discriminator="type"),]diff --git a/sweagent/agent/models.py b/sweagent/agent/models.pyindex 2d13b981..d4a8ed09 100644--- a/sweagent/agent/models.py+++ b/sweagent/agent/models.py@@ -1,6 +1,5 @@from __future__ import annotations-import copyimport jsonimport randomimport shlex@@ -501,7 +500,15 @@ class LiteLLMModel(AbstractModel):return Nonereturn random.choice(api_keys)- def _query(self, messages: list[dict[str, str]], n: int | None = None) -> dict | list[dict]:+ def _sleep(self) -> None:+ with GLOBAL_STATS_LOCK:+ GLOBAL_STATS.last_query_timestamp = time.time()+ elapsed_time = time.time() - GLOBAL_STATS.last_query_timestamp+ if elapsed_time < self.config.delay:+ time.sleep(self.config.delay - elapsed_time)++ def _single_query(self, messages: list[dict[str, str]], n: int | None = None) -> list[dict]:+ self._sleep()input_tokens: int = litellm.utils.token_counter(messages=messages, model=self.config.name)if self.model_max_input_tokens is None:self.logger.warning(f"No max input tokens found for model {self.config.name!r}")@@ -548,16 +555,19 @@ class LiteLLMModel(AbstractModel):output_dict["tool_calls"] = tool_callsoutputs.append(output_dict)self._update_stats(input_tokens=input_tokens, output_tokens=output_tokens, cost=cost)+ return outputs++ def _query(self, messages: list[dict[str, str]], n: int | None = None) -> list[dict]:if n is None:- return outputs[0]+ return self._single_query(messages)+ outputs = []+ # not needed for openai, but oh well.+ for _ in range(n):+ outputs.extend(self._single_query(messages))return outputs- def query(self, history: History, n: int | None = None) -> dict:- elapsed_time = time.time() - GLOBAL_STATS.last_query_timestamp- if elapsed_time < self.config.delay:- time.sleep(self.config.delay - elapsed_time)- with GLOBAL_STATS_LOCK:- GLOBAL_STATS.last_query_timestamp = time.time()+ def query(self, history: History, n: int = 1) -> list[dict] | dict:+ messages = self._history_to_messages(history)for attempt in Retrying(stop=stop_after_attempt(self.config.retry.retries),wait=wait_random_exponential(min=self.config.retry.min_wait, max=self.config.retry.max_wait),@@ -586,8 +596,9 @@ class LiteLLMModel(AbstractModel):f"(slept for {attempt.retry_state.idle_for:.2f}s)"f"{exception_info}")- messages = self._history_to_messages(history)result = self._query(messages, n=n)+ if n is None or n == 1:+ return result[0]return resultdef _history_to_messages(@@ -617,29 +628,7 @@ class LiteLLMModel(AbstractModel):)else:messages.append({"role": role, "content": history_item["content"]})- return self._add_cache_control(messages)-- def _add_cache_control(self, messages: list[dict[str, str]]) -> list[dict[str, Any]]:- """Adds cache control headers to the messages"""- n_cached = 0- new_messages = []- messages = copy.deepcopy(messages)- for message in reversed(messages):- if (message["role"] == "assistant" and n_cached < 3) or (message["role"] == "system"):- print("Caching message", message["content"][:50])- message["content"] = [ # type: ignore- {- "type": "text",- "text": message["content"],- "cache_control": {"type": "ephemeral"},- }- ]- new_messages.append(message)- n_cached += 1- else:- new_messages.append(message)- self.logger.info(f"Total cache breakpoints: {n_cached}")- return list(reversed(new_messages))+ return messagesdef get_model(args: ModelConfig, tools: ToolConfig) -> AbstractModel:diff --git a/sweagent/types.py b/sweagent/types.pyindex 4bae478a..759a68aa 100644--- a/sweagent/types.py+++ b/sweagent/types.py@@ -53,7 +53,7 @@ class TrajectoryStep(TypedDict):# required fields go hereclass _HistoryItem(TypedDict):role: str- content: str+ content: str | list[dict[str, Any]]message_type: Literal["thought", "action", "observation"]