Skip to content

SWE-agent takes a GitHub issue and tries to automatically fix it, using your LM of choice. It can also be employed for offensive cybersecurity or competitive coding challenges. [NeurIPS 2024]

Change: Make token counting simpler & more explicit

Kilian Lieretc68ba15eeb46
diff --git a/docs/installation/keys.md b/docs/installation/keys.md
index 8f24199d..69cf10e5 100644
--- a/docs/installation/keys.md
+++ b/docs/installation/keys.md
@@ -115,7 +115,7 @@ and set `agent.model.api_key` to the key you've configured for your proxy (or a
Please also make sure to set `max_input_tokens` to a non-`None` value to avoid other warnings.
-### Custom model registry for cost tracking
+### Custom cost tracking
If you want to track costs for models not in the default litellm registry, you can provide a custom model registry file. This is particularly useful for:
@@ -157,6 +157,8 @@ agent:
...
```
+If you need to modify the tokenizer that is used when calculating costs, you can set the `custom_tokenizer` setting in the [model config](../reference/model_config.md).
+
!!! warning "Parsing functions"
The default config uses function calling to retrieve actions from the model response, i.e.,
diff --git a/sweagent/agent/models.py b/sweagent/agent/models.py
index a0098889..05f7d5b3 100644
--- a/sweagent/agent/models.py
+++ b/sweagent/agent/models.py
@@ -143,6 +143,12 @@ class GenericAPIModelConfig(PydanticBaseModel):
Use this for local models or models not (yet) in the default litellm model registry for tracking costs.
"""
+ custom_tokenizer: dict[str, Any] | None = None
+ """Override the default tokenizer for the model.
+ Use the arguments of `litellm.create_pretrained_tokenizer`.
+ Basic example: `{"identifier": "hf-internal-testing/llama-tokenizer"}`
+ """
+
# pydantic
model_config = ConfigDict(extra="forbid")
@@ -577,7 +583,6 @@ class LiteLLMModel(AbstractModel):
self.stats = InstanceStats()
self.tools = tools
self.logger = get_logger("swea-lm", emoji="🤖")
- self.custom_tokenizer = None
if tools.use_function_calling:
if not litellm.utils.supports_function_calling(model=self.config.name):
@@ -615,29 +620,9 @@ class LiteLLMModel(AbstractModel):
)
self.lm_provider = litellm.model_cost.get(self.config.name, {}).get("litellm_provider", self.config.name)
- if self.config.per_instance_cost_limit == 0 and self.config.total_cost_limit == 0: # Local model
- if "/" in self.lm_provider:
- from transformers import AutoTokenizer
-
- self.custom_tokenizer = {}
- self.custom_tokenizer["provider"] = self.lm_provider.split("/")[0]
-
- if self.custom_tokenizer["provider"] not in litellm.provider_list:
- self.logger.warning(
- f"Local model {self.lm_provider} not found in LiteLLM provider list. Using default tokenizer."
- )
- self.custom_tokenizer = None
- else:
- self.custom_tokenizer["identifier"] = "/".join(self.lm_provider.split("/")[1:])
- self.custom_tokenizer["type"] = "huggingface_tokenizer"
- # Use backend tokenizer as workaround for litellm HF tokenizer bug
- self.custom_tokenizer["tokenizer"] = AutoTokenizer.from_pretrained(
- self.custom_tokenizer["identifier"]
- ).backend_tokenizer
- else:
- self.logger.warning(
- f"Local model identifier {self.lm_provider} has an unknown format. Using default tokenizer."
- )
+ self.custom_tokenizer = None
+ if self.config.custom_tokenizer is not None:
+ self.custom_tokenizer = litellm.utils.create_pretrained_tokenizer(**self.config.custom_tokenizer)
@property
def instance_cost_limit(self) -> float:
@@ -702,7 +687,7 @@ class LiteLLMModel(AbstractModel):
del message["cache_control"]
input_tokens: int = litellm.utils.token_counter(
messages=messages_no_cache_control,
- model=self.custom_tokenizer["identifier"] if self.custom_tokenizer else self.config.name,
+ model=self.custom_tokenizer["identifier"] if self.custom_tokenizer is not None else self.config.name,
custom_tokenizer=self.custom_tokenizer,
)
if self.model_max_input_tokens is None:
@@ -767,7 +752,7 @@ class LiteLLMModel(AbstractModel):
output = choices[i].message.content or ""
output_tokens += litellm.utils.token_counter(
text=output,
- model=self.custom_tokenizer["identifier"] if self.custom_tokenizer else self.config.name,
+ model=self.custom_tokenizer["identifier"] if self.custom_tokenizer is not None else self.config.name,
custom_tokenizer=self.custom_tokenizer,
)
output_dict = {"message": output}