Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
fix: add DeepSeek V4/V3.2/R1 tokenizer mappings and context limits (#912)
yiihao0c7087539d92
diff --git a/headroom/providers/openai.py b/headroom/providers/openai.pyindex 320fb59e6..3ebd0c549 100644--- a/headroom/providers/openai.py+++ b/headroom/providers/openai.py@@ -99,9 +99,16 @@ _CONTEXT_LIMITS: dict[str, int] = {# these are the manual fallback when LiteLLM doesn't know the model."deepseek-v4-flash": 1_000_000,"deepseek-v4-pro": 1_000_000,+ "deepseek-v3.2": 128_000,+ "deepseek-v3-0324": 128_000,+ "deepseek-v3": 128_000,+ "deepseek-v2": 128_000,"deepseek-chat": 131_072,"deepseek-reasoner": 131_072,+ "deepseek-r1": 131_072,+ "deepseek-r1-0528": 131_072,"deepseek-coder": 16384,+ "deepseek-coder-v2": 128_000,}# Fallback pricing - LiteLLM is preferred sourcediff --git a/headroom/providers/openai_compatible.py b/headroom/providers/openai_compatible.pyindex 8a590d51c..32c88db70 100644--- a/headroom/providers/openai_compatible.py+++ b/headroom/providers/openai_compatible.py@@ -86,9 +86,22 @@ _DEFAULT_CONTEXT_LIMITS: dict[str, int] = {# DeepSeek"deepseek": 1048576,"deepseek-coder": 128000,+ "deepseek-coder-v2": 128000,+ "deepseek-coder-v2-lite": 128000,"deepseek-v2": 128000,+ "deepseek-v2-lite": 128000,"deepseek-v3": 1048576,+ "deepseek-v3-0324": 128000,+ "deepseek-v3.2": 128000,+ "deepseek-r1": 131072,+ "deepseek-r1-0528": 131072,+ "deepseek-reasoner": 131072,"deepseek-v4": 1048576,+ "deepseek-v4-pro": 1_000_000,+ "deepseek-v4-flash": 1_000_000,+ "deepseek-chat": 131072,+ "deepseek-r1-distill-qwen": 131072,+ "deepseek-r1-distill-llama": 131072,# Yi"yi": 32768,"yi-34b": 32768,diff --git a/headroom/tokenizers/huggingface.py b/headroom/tokenizers/huggingface.pyindex c48c16474..750971f33 100644--- a/headroom/tokenizers/huggingface.py+++ b/headroom/tokenizers/huggingface.py@@ -63,13 +63,30 @@ MODEL_TO_TOKENIZER: dict[str, str] = {"qwen2.5": "Qwen/Qwen2.5-7B","qwen2.5-7b": "Qwen/Qwen2.5-7B","qwen2.5-72b": "Qwen/Qwen2.5-72B",- # DeepSeek+ # DeepSeek V1 / Coder (legacy, 2023-2024)"deepseek": "deepseek-ai/deepseek-llm-7b-base","deepseek-7b": "deepseek-ai/deepseek-llm-7b-base","deepseek-67b": "deepseek-ai/deepseek-llm-67b-base","deepseek-coder": "deepseek-ai/deepseek-coder-6.7b-base",+ "deepseek-coder-v2": "deepseek-ai/DeepSeek-Coder-V2-Lite-Instruct",+ "deepseek-coder-v2-lite": "deepseek-ai/DeepSeek-Coder-V2-Lite-Instruct",+ # DeepSeek V2/V3 family (2024)"deepseek-v2": "deepseek-ai/DeepSeek-V2",+ "deepseek-v2-lite": "deepseek-ai/DeepSeek-V2-Lite","deepseek-v3": "deepseek-ai/DeepSeek-V3",+ "deepseek-v3-0324": "deepseek-ai/DeepSeek-V3-0324",+ "deepseek-v3.2": "deepseek-ai/DeepSeek-V3.2",+ # DeepSeek R1 reasoning family (2025)+ "deepseek-r1": "deepseek-ai/DeepSeek-R1",+ "deepseek-r1-0528": "deepseek-ai/DeepSeek-R1-0528",+ "deepseek-reasoner": "deepseek-ai/DeepSeek-R1",+ # DeepSeek V4 family (2025-2026)+ "deepseek-v4-pro": "deepseek-ai/DeepSeek-V4-Pro",+ "deepseek-v4-flash": "deepseek-ai/DeepSeek-V4-Flash",+ # DeepSeek API aliases (routed through the proxy)+ "deepseek-chat": "deepseek-ai/DeepSeek-V3",+ "deepseek-r1-distill-qwen": "deepseek-ai/DeepSeek-R1-Distill-Qwen-7B",+ "deepseek-r1-distill-llama": "deepseek-ai/DeepSeek-R1-Distill-Llama-8B",# Yi family"yi": "01-ai/Yi-6B","yi-6b": "01-ai/Yi-6B",diff --git a/tests/test_huggingface_tokenizer_timeout.py b/tests/test_huggingface_tokenizer_timeout.pyindex 8d7e8b888..22b9e98ac 100644--- a/tests/test_huggingface_tokenizer_timeout.py+++ b/tests/test_huggingface_tokenizer_timeout.py@@ -113,6 +113,14 @@ def test_count_messages_fails_open_to_estimation(monkeypatch: pytest.MonkeyPatchassert tokens > 0 # estimation fallback, no exception, no hang+def test_deepseek_model_aliases_resolve_to_expected_tokenizers() -> None:+ assert get_tokenizer_name("deepseek-v3.2") == "deepseek-ai/DeepSeek-V3.2"+ assert get_tokenizer_name("deepseek-v4-pro") == "deepseek-ai/DeepSeek-V4-Pro"+ assert get_tokenizer_name("deepseek-v4-flash") == "deepseek-ai/DeepSeek-V4-Flash"+ assert get_tokenizer_name("deepseek-r1") == "deepseek-ai/DeepSeek-R1"+ assert get_tokenizer_name("deepseek-r1-0528") == "deepseek-ai/DeepSeek-R1-0528"++def test_invalid_timeout_env_falls_back_to_default(monkeypatch: pytest.MonkeyPatch) -> None:monkeypatch.setenv("HEADROOM_HF_TOKENIZER_LOAD_TIMEOUT_SECS", "not-a-number")assert hf_mod._load_timeout_secs() == hf_mod._LOAD_TIMEOUT_DEFAULTdiff --git a/tests/test_providers/test_universal.py b/tests/test_providers/test_universal.pyindex 9f34e622e..26190ed46 100644--- a/tests/test_providers/test_universal.py+++ b/tests/test_providers/test_universal.py@@ -88,6 +88,11 @@ class TestOpenAICompatibleProvider:assert provider.get_context_limit("deepseek-v4") == 1048576assert provider.get_context_limit("deepseek") == 1048576assert provider.get_context_limit("deepseek-v2") == 128000+ assert provider.get_context_limit("deepseek-v3.2") == 128000+ assert provider.get_context_limit("deepseek-v4-pro") == 1_000_000+ assert provider.get_context_limit("deepseek-v4-flash") == 1_000_000+ assert provider.get_context_limit("deepseek-r1") == 131072+ assert provider.get_context_limit("deepseek-coder-v2") == 128000def test_get_context_limit_unknown_model(self):"""Test context limit for unknown models (defaults to 128K)."""