Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: count ASCII in context estimates without a per-character loop (#15659)
Daniel Han86f509a6b175
diff --git a/studio/backend/core/inference/context_window.py b/studio/backend/core/inference/context_window.pyindex e946df63b01..e4b7d362e26 100644--- a/studio/backend/core/inference/context_window.py+++ b/studio/backend/core/inference/context_window.py@@ -158,6 +158,11 @@ def estimate_messages_tokens(messages: list[dict]) -> int:return sum(estimate_message_tokens(message) for message in messages)+def _ascii_count(text: str) -> int:+ """Number of code points <= 127; ``errors = "ignore"`` drops every other one, surrogates included."""+ return len(text) if text.isascii() else len(text.encode("ascii", "ignore"))++def estimate_messages_tokens_dense(messages: list[dict]) -> int:"""Estimate charging non-ASCII a token per character (2.1x undercount measured on CJK). Not thedefault: eviction must not be pessimistic."""@@ -168,7 +173,7 @@ def estimate_messages_tokens_dense(messages: list[dict]) -> int:except Exception:total += 1continue- dense = sum(1 for char in text if ord(char) > 127)+ dense = len(text) - _ascii_count(text)total += max(1, dense + (len(text) - dense) // 4)return total@@ -192,15 +197,12 @@ def estimate_messages_tokens_conservative(except Exception:total += 1continue- wide = sum(1 for char in text if ord(char) > 127)+ wide = len(text) - _ascii_count(text)if dense_ascii:total += max(1, wide + (len(text) - wide) // _DENSE_RUN_CHARS_PER_TOKEN)continue# ASCII only: a run of CJK is already charged a token a character above.- runs = sum(- sum(1 for char in match.group(0) if ord(char) <= 127)- for match in _DENSE_RUN_RE.finditer(text)- )+ runs = sum(_ascii_count(match.group(0)) for match in _DENSE_RUN_RE.finditer(text))plain = len(text) - wide - runstotal += max(1, wide + runs // _DENSE_RUN_CHARS_PER_TOKEN + plain // 4)return totaldiff --git a/studio/backend/tests/test_context_estimate_ascii_count.py b/studio/backend/tests/test_context_estimate_ascii_count.pynew file mode 100644index 00000000000..040cc8ebfe3--- /dev/null+++ b/studio/backend/tests/test_context_estimate_ascii_count.py@@ -0,0 +1,129 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""The ``str.encode`` ASCII count returns the same integers as the old ``ord()`` loops."""++import json+import random++import pytest++from core.inference import context_window as cw+++def _dense_reference(messages):+ total = 0+ for message in messages:+ try:+ text = json.dumps(message, ensure_ascii = False)+ except Exception:+ total += 1+ continue+ dense = sum(1 for char in text if ord(char) > 127)+ total += max(1, dense + (len(text) - dense) // 4)+ return total+++def _conservative_reference(messages, *, dense_ascii = False):+ total = 0+ for message in messages:+ try:+ text = json.dumps(message, ensure_ascii = False)+ except Exception:+ total += 1+ continue+ wide = sum(1 for char in text if ord(char) > 127)+ if dense_ascii:+ total += max(1, wide + (len(text) - wide) // cw._DENSE_RUN_CHARS_PER_TOKEN)+ continue+ runs = sum(+ sum(1 for char in match.group(0) if ord(char) <= 127)+ for match in cw._DENSE_RUN_RE.finditer(text)+ )+ plain = len(text) - wide - runs+ total += max(1, wide + runs // cw._DENSE_RUN_CHARS_PER_TOKEN + plain // 4)+ return total+++_ALPHABETS = (+ 'abc XYZ 019 {}[]":,\n\t\\/',+ "\x00\x01\x7f\x80\xff",+ "中文日本語한국어",+ "\U0001f600\U0001f680\U0001f9d1\U0001f4bb",+ "\ud800\udfff", # lone surrogates survive json.dumps(ensure_ascii = False)+ "éñЖאا",+ "A" * 8 + "/+=",+)+++def _random_text(rng):+ pieces = []+ for _ in range(rng.randint(0, 12)):+ alphabet = rng.choice(_ALPHABETS)+ run = rng.choice((1, 3, 63, 64, 65, 200))+ pieces.append("".join(rng.choice(alphabet) for _ in range(run)))+ if rng.random() < 0.5:+ pieces.append(" ")+ return "".join(pieces)+++def _random_messages(rng):+ messages = []+ for _ in range(rng.randint(0, 4)):+ roll = rng.random()+ if roll < 0.05:+ messages.append({"role": "user", "content": object()}) # json.dumps raises+ elif roll < 0.15:+ messages.append(+ {+ "role": "assistant",+ "tool_calls": [{"function": {"arguments": _random_text(rng)}}],+ }+ )+ else:+ messages.append(+ {"role": rng.choice(("user", "system", "tool")), "content": _random_text(rng)}+ )+ return messages+++def test_ascii_count_matches_ord_loop_on_every_code_point():+ every = "".join(chr(c) for c in range(0x110000))+ assert cw._ascii_count(every) == sum(1 for char in every if ord(char) <= 127) == 128+ for chunk_start in range(0, 0x110000, 0x8000):+ chunk = every[chunk_start : chunk_start + 0x8000 : 7]+ assert cw._ascii_count(chunk) == sum(1 for char in chunk if ord(char) <= 127)++[email protected]("seed", range(20))+def test_estimators_match_reference_loops(seed):+ rng = random.Random(seed)+ for _ in range(150):+ messages = _random_messages(rng)+ assert cw.estimate_messages_tokens_dense(messages) == _dense_reference(messages)+ assert cw.estimate_messages_tokens_conservative(messages) == _conservative_reference(+ messages+ )+ assert cw.estimate_messages_tokens_conservative(+ messages, dense_ascii = True+ ) == _conservative_reference(messages, dense_ascii = True)+++def test_estimators_on_fixed_edges():+ cases = [+ [],+ [{"content": ""}],+ [{"content": "x" * 64}],+ [{"content": "\U0001f600" * 64 + "a" * 64}],+ [{"content": "\ud83d" + "q" * 100 + "\ude00"}],+ [{"content": ("中" + "a") * 40}],+ [{"content": "base64:" + "QUJD" * 500}, {"content": "日本 " * 300}],+ ]+ for messages in cases:+ assert cw.estimate_messages_tokens_dense(messages) == _dense_reference(messages)+ assert cw.estimate_messages_tokens_conservative(messages) == _conservative_reference(+ messages+ )+ assert cw.estimate_messages_tokens_conservative(+ messages, dense_ascii = True+ ) == _conservative_reference(messages, dense_ascii = True)