The agent engineering platform.
Python agentsaiai-agentsanthropicchatgptdeepagentsenterpriseframeworkgeminigenerative-ailangchainlanggraphllmmultiagentopen-sourceopenaipydanticpythonragtypescript
feat(core): add multimodal support to count_tokens_approximately (#34883)
cc585b691c1d43
diff --git a/libs/core/langchain_core/messages/utils.py b/libs/core/langchain_core/messages/utils.pyindex d4659bf0bb..c15296422f 100644--- a/libs/core/langchain_core/messages/utils.py+++ b/libs/core/langchain_core/messages/utils.py@@ -2184,6 +2184,7 @@ def count_tokens_approximately(chars_per_token: float = 4.0,extra_tokens_per_message: float = 3.0,count_name: bool = True,+ tokens_per_image: int = 85,) -> int:"""Approximate the total number of tokens in messages.@@ -2191,21 +2192,22 @@ def count_tokens_approximately(- For AI messages, the token count also includes stringified tool calls.- For tool messages, the token count also includes the tool call ID.+ - For multimodal messages with images, applies a fixed token penalty per image+ instead of counting base64-encoded characters.Args:messages: List of messages to count tokens for.chars_per_token: Number of characters per token to use for the approximation.-One token corresponds to ~4 chars for common English text.-You can also specify `float` values for more fine-grained control.[See more here](https://platform.openai.com/tokenizer).extra_tokens_per_message: Number of extra tokens to add per message, e.g.special tokens, including beginning/end of message.-You can also specify `float` values for more fine-grained control.[See more here](https://github.com/openai/openai-cookbook/blob/main/examples/How_to_count_tokens_with_tiktoken.ipynb).count_name: Whether to include message names in the count.+ tokens_per_image: Fixed token cost per image (default: 85, aligned with+ OpenAI's low-resolution image token cost).Returns:Approximate number of tokens in the messages.@@ -2214,19 +2216,42 @@ def count_tokens_approximately(This is a simple approximation that may not match the exact token count used byspecific models. For accurate counts, use model-specific tokenizers.- Warning:- This function does not currently support counting image tokens.+ For multimodal messages containing images, a fixed token penalty is applied+ per image instead of counting base64-encoded characters, which provides a+ more realistic approximation.!!! version-added "Added in `langchain-core` 0.3.46""""token_count = 0.0for message in convert_to_messages(messages):message_chars = 0+if isinstance(message.content, str):message_chars += len(message.content)-- # TODO: add support for approximate counting for image blocks+ # Handle multimodal content (list of content blocks)+ elif isinstance(message.content, list):+ for block in message.content:+ if isinstance(block, str):+ # String block+ message_chars += len(block)+ elif isinstance(block, dict):+ block_type = block.get("type", "")++ # Apply fixed penalty for image blocks+ if block_type in ("image", "image_url"):+ token_count += tokens_per_image+ # Count text blocks normally+ elif block_type == "text":+ text = block.get("text", "")+ message_chars += len(text)+ # Conservative estimate for unknown block types+ else:+ message_chars += len(repr(block))+ else:+ # Fallback for unexpected block types+ message_chars += len(repr(block))else:+ # Fallback for other content typescontent = repr(message.content)message_chars += len(content)diff --git a/libs/core/tests/unit_tests/messages/test_utils.py b/libs/core/tests/unit_tests/messages/test_utils.pyindex f206db83cd..518a8a0905 100644--- a/libs/core/tests/unit_tests/messages/test_utils.py+++ b/libs/core/tests/unit_tests/messages/test_utils.py@@ -2661,3 +2661,68 @@ def test_get_buffer_string_xml_all_custom_prefixes() -> None:assert '<message type="assistant">AI message</message>' in resultassert '<message type="fn">Function message</message>' in resultassert '<message type="t">Tool message</message>' in result+++def test_count_tokens_approximately_with_image_content() -> None:+ """Test approximate token counting with image content blocks."""+ message_with_image = HumanMessage(+ content=[+ {"type": "text", "text": "What's in this image?"},+ {+ "type": "image_url",+ "image_url": {"url": "data:image/jpeg;base64," + "A" * 100000},+ },+ ]+ )++ token_count = count_tokens_approximately([message_with_image])++ # Should be ~85 (image) + ~5 (text) + 3 (extra) = ~93 tokens, NOT 25,000++ assert token_count < 200, f"Expected <200 tokens, got {token_count}"+ assert token_count > 80, f"Expected >80 tokens, got {token_count}"+++def test_count_tokens_approximately_with_multiple_images() -> None:+ """Test token counting with multiple images."""+ message = HumanMessage(+ content=[+ {"type": "text", "text": "Compare these images"},+ {"type": "image_url", "image_url": {"url": "data:image/jpeg;base64,AAA"}},+ {"type": "image_url", "image_url": {"url": "data:image/jpeg;base64,BBB"}},+ ]+ )++ token_count = count_tokens_approximately([message])++ # Should be ~85 * 2 (images) + ~6 (text) + 3 (extra) = ~179 tokens+ assert 170 < token_count < 190+++def test_count_tokens_approximately_text_only_backward_compatible() -> None:+ """Test that text-only messages still work correctly."""+ messages = [+ HumanMessage(content="Hello world"),+ AIMessage(content="Hi there!"),+ ]++ token_count = count_tokens_approximately(messages)++ # Should be ~15 tokens+ # (11 chars + 9 chars + roles + 2*3 extra)+ assert 13 <= token_count <= 17+++def test_count_tokens_approximately_with_custom_image_penalty() -> None:+ """Test custom tokens_per_image parameter."""+ message = HumanMessage(+ content=[+ {"type": "text", "text": "test"},+ {"type": "image_url", "image_url": {"url": "data:image/jpeg;base64,XYZ"}},+ ]+ )++ # Using custom image penalty (e.g., for Anthropic models)+ token_count = count_tokens_approximately([message], tokens_per_image=1600)++ # Should be ~1600 (image) + ~1 (text) + 3 (extra) = ~1604 tokens+ assert 1600 < token_count < 1610