SWE-agent takes a GitHub issue and tries to automatically fix it, using your LM of choice. It can also be employed for offensive cybersecurity or competitive coding challenges. [NeurIPS 2024]
Change: Make token counting simpler & more explicit
Kilian Lieretc68ba15eeb46
diff --git a/docs/installation/keys.md b/docs/installation/keys.mdindex 8f24199d..69cf10e5 100644--- a/docs/installation/keys.md+++ b/docs/installation/keys.md@@ -115,7 +115,7 @@ and set `agent.model.api_key` to the key you've configured for your proxy (or aPlease also make sure to set `max_input_tokens` to a non-`None` value to avoid other warnings.-### Custom model registry for cost tracking+### Custom cost trackingIf you want to track costs for models not in the default litellm registry, you can provide a custom model registry file. This is particularly useful for:@@ -157,6 +157,8 @@ agent:...```+If you need to modify the tokenizer that is used when calculating costs, you can set the `custom_tokenizer` setting in the [model config](../reference/model_config.md).+!!! warning "Parsing functions"The default config uses function calling to retrieve actions from the model response, i.e.,diff --git a/sweagent/agent/models.py b/sweagent/agent/models.pyindex a0098889..05f7d5b3 100644--- a/sweagent/agent/models.py+++ b/sweagent/agent/models.py@@ -143,6 +143,12 @@ class GenericAPIModelConfig(PydanticBaseModel):Use this for local models or models not (yet) in the default litellm model registry for tracking costs."""+ custom_tokenizer: dict[str, Any] | None = None+ """Override the default tokenizer for the model.+ Use the arguments of `litellm.create_pretrained_tokenizer`.+ Basic example: `{"identifier": "hf-internal-testing/llama-tokenizer"}`+ """+# pydanticmodel_config = ConfigDict(extra="forbid")@@ -577,7 +583,6 @@ class LiteLLMModel(AbstractModel):self.stats = InstanceStats()self.tools = toolsself.logger = get_logger("swea-lm", emoji="🤖")- self.custom_tokenizer = Noneif tools.use_function_calling:if not litellm.utils.supports_function_calling(model=self.config.name):@@ -615,29 +620,9 @@ class LiteLLMModel(AbstractModel):)self.lm_provider = litellm.model_cost.get(self.config.name, {}).get("litellm_provider", self.config.name)- if self.config.per_instance_cost_limit == 0 and self.config.total_cost_limit == 0: # Local model- if "/" in self.lm_provider:- from transformers import AutoTokenizer-- self.custom_tokenizer = {}- self.custom_tokenizer["provider"] = self.lm_provider.split("/")[0]-- if self.custom_tokenizer["provider"] not in litellm.provider_list:- self.logger.warning(- f"Local model {self.lm_provider} not found in LiteLLM provider list. Using default tokenizer."- )- self.custom_tokenizer = None- else:- self.custom_tokenizer["identifier"] = "/".join(self.lm_provider.split("/")[1:])- self.custom_tokenizer["type"] = "huggingface_tokenizer"- # Use backend tokenizer as workaround for litellm HF tokenizer bug- self.custom_tokenizer["tokenizer"] = AutoTokenizer.from_pretrained(- self.custom_tokenizer["identifier"]- ).backend_tokenizer- else:- self.logger.warning(- f"Local model identifier {self.lm_provider} has an unknown format. Using default tokenizer."- )+ self.custom_tokenizer = None+ if self.config.custom_tokenizer is not None:+ self.custom_tokenizer = litellm.utils.create_pretrained_tokenizer(**self.config.custom_tokenizer)@propertydef instance_cost_limit(self) -> float:@@ -702,7 +687,7 @@ class LiteLLMModel(AbstractModel):del message["cache_control"]input_tokens: int = litellm.utils.token_counter(messages=messages_no_cache_control,- model=self.custom_tokenizer["identifier"] if self.custom_tokenizer else self.config.name,+ model=self.custom_tokenizer["identifier"] if self.custom_tokenizer is not None else self.config.name,custom_tokenizer=self.custom_tokenizer,)if self.model_max_input_tokens is None:@@ -767,7 +752,7 @@ class LiteLLMModel(AbstractModel):output = choices[i].message.content or ""output_tokens += litellm.utils.token_counter(text=output,- model=self.custom_tokenizer["identifier"] if self.custom_tokenizer else self.config.name,+ model=self.custom_tokenizer["identifier"] if self.custom_tokenizer is not None else self.config.name,custom_tokenizer=self.custom_tokenizer,)output_dict = {"message": output}