Skip to content

The agent engineering platform.

feat(core): add multimodal support to count_tokens_approximately (#34883)

cc585b691c1d43
diff --git a/libs/core/langchain_core/messages/utils.py b/libs/core/langchain_core/messages/utils.py
index d4659bf0bb..c15296422f 100644
--- a/libs/core/langchain_core/messages/utils.py
+++ b/libs/core/langchain_core/messages/utils.py
@@ -2184,6 +2184,7 @@ def count_tokens_approximately(
chars_per_token: float = 4.0,
extra_tokens_per_message: float = 3.0,
count_name: bool = True,
+ tokens_per_image: int = 85,
) -> int:
"""Approximate the total number of tokens in messages.
@@ -2191,21 +2192,22 @@ def count_tokens_approximately(
- For AI messages, the token count also includes stringified tool calls.
- For tool messages, the token count also includes the tool call ID.
+ - For multimodal messages with images, applies a fixed token penalty per image
+ instead of counting base64-encoded characters.
Args:
messages: List of messages to count tokens for.
chars_per_token: Number of characters per token to use for the approximation.
-
One token corresponds to ~4 chars for common English text.
-
You can also specify `float` values for more fine-grained control.
[See more here](https://platform.openai.com/tokenizer).
extra_tokens_per_message: Number of extra tokens to add per message, e.g.
special tokens, including beginning/end of message.
-
You can also specify `float` values for more fine-grained control.
[See more here](https://github.com/openai/openai-cookbook/blob/main/examples/How_to_count_tokens_with_tiktoken.ipynb).
count_name: Whether to include message names in the count.
+ tokens_per_image: Fixed token cost per image (default: 85, aligned with
+ OpenAI's low-resolution image token cost).
Returns:
Approximate number of tokens in the messages.
@@ -2214,19 +2216,42 @@ def count_tokens_approximately(
This is a simple approximation that may not match the exact token count used by
specific models. For accurate counts, use model-specific tokenizers.
- Warning:
- This function does not currently support counting image tokens.
+ For multimodal messages containing images, a fixed token penalty is applied
+ per image instead of counting base64-encoded characters, which provides a
+ more realistic approximation.
!!! version-added "Added in `langchain-core` 0.3.46"
"""
token_count = 0.0
for message in convert_to_messages(messages):
message_chars = 0
+
if isinstance(message.content, str):
message_chars += len(message.content)
-
- # TODO: add support for approximate counting for image blocks
+ # Handle multimodal content (list of content blocks)
+ elif isinstance(message.content, list):
+ for block in message.content:
+ if isinstance(block, str):
+ # String block
+ message_chars += len(block)
+ elif isinstance(block, dict):
+ block_type = block.get("type", "")
+
+ # Apply fixed penalty for image blocks
+ if block_type in ("image", "image_url"):
+ token_count += tokens_per_image
+ # Count text blocks normally
+ elif block_type == "text":
+ text = block.get("text", "")
+ message_chars += len(text)
+ # Conservative estimate for unknown block types
+ else:
+ message_chars += len(repr(block))
+ else:
+ # Fallback for unexpected block types
+ message_chars += len(repr(block))
else:
+ # Fallback for other content types
content = repr(message.content)
message_chars += len(content)
diff --git a/libs/core/tests/unit_tests/messages/test_utils.py b/libs/core/tests/unit_tests/messages/test_utils.py
index f206db83cd..518a8a0905 100644
--- a/libs/core/tests/unit_tests/messages/test_utils.py
+++ b/libs/core/tests/unit_tests/messages/test_utils.py
@@ -2661,3 +2661,68 @@ def test_get_buffer_string_xml_all_custom_prefixes() -> None:
assert '<message type="assistant">AI message</message>' in result
assert '<message type="fn">Function message</message>' in result
assert '<message type="t">Tool message</message>' in result
+
+
+def test_count_tokens_approximately_with_image_content() -> None:
+ """Test approximate token counting with image content blocks."""
+ message_with_image = HumanMessage(
+ content=[
+ {"type": "text", "text": "What's in this image?"},
+ {
+ "type": "image_url",
+ "image_url": {"url": "data:image/jpeg;base64," + "A" * 100000},
+ },
+ ]
+ )
+
+ token_count = count_tokens_approximately([message_with_image])
+
+ # Should be ~85 (image) + ~5 (text) + 3 (extra) = ~93 tokens, NOT 25,000+
+ assert token_count < 200, f"Expected <200 tokens, got {token_count}"
+ assert token_count > 80, f"Expected >80 tokens, got {token_count}"
+
+
+def test_count_tokens_approximately_with_multiple_images() -> None:
+ """Test token counting with multiple images."""
+ message = HumanMessage(
+ content=[
+ {"type": "text", "text": "Compare these images"},
+ {"type": "image_url", "image_url": {"url": "data:image/jpeg;base64,AAA"}},
+ {"type": "image_url", "image_url": {"url": "data:image/jpeg;base64,BBB"}},
+ ]
+ )
+
+ token_count = count_tokens_approximately([message])
+
+ # Should be ~85 * 2 (images) + ~6 (text) + 3 (extra) = ~179 tokens
+ assert 170 < token_count < 190
+
+
+def test_count_tokens_approximately_text_only_backward_compatible() -> None:
+ """Test that text-only messages still work correctly."""
+ messages = [
+ HumanMessage(content="Hello world"),
+ AIMessage(content="Hi there!"),
+ ]
+
+ token_count = count_tokens_approximately(messages)
+
+ # Should be ~15 tokens
+ # (11 chars + 9 chars + roles + 2*3 extra)
+ assert 13 <= token_count <= 17
+
+
+def test_count_tokens_approximately_with_custom_image_penalty() -> None:
+ """Test custom tokens_per_image parameter."""
+ message = HumanMessage(
+ content=[
+ {"type": "text", "text": "test"},
+ {"type": "image_url", "image_url": {"url": "data:image/jpeg;base64,XYZ"}},
+ ]
+ )
+
+ # Using custom image penalty (e.g., for Anthropic models)
+ token_count = count_tokens_approximately([message], tokens_per_image=1600)
+
+ # Should be ~1600 (image) + ~1 (text) + 3 (extra) = ~1604 tokens
+ assert 1600 < token_count < 1610