Experience-Driven Black-Box Cost Control for Coding Agents
Python agentic-aiagentic-codingai-codingai-coding-assistantanthropicartificial-intelligenceclaude-codecodexcoding-agentcoding-agentscost-optimizationcost-reductiondeveloper-toolsgithub-copilotllmopen-sourceopenaiproductivitytoken-optimization
Read agent input and session logs as UTF-8 on any locale
Aaronfd2c2e51712f
diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.ymlnew file mode 100644index 0000000..d69d03b--- /dev/null+++ b/.github/workflows/tests.yml@@ -0,0 +1,29 @@+name: tests++on:+ push:+ branches: [main]+ pull_request:++permissions:+ contents: read++jobs:+ tests:+ strategy:+ fail-fast: false+ matrix:+ os: [ubuntu-latest, windows-latest]+ python: ["3.10", "3.13"]+ runs-on: ${{ matrix.os }}+ defaults:+ run:+ # Git Bash on Windows: the launchers are `sh` scripts, run there as Claude Code runs them.+ shell: bash+ steps:+ - uses: actions/checkout@v5+ - uses: actions/setup-python@v6+ with:+ python-version: ${{ matrix.python }}+ - run: python -m pip install pytest+ - run: python -m pytest tests -rsdiff --git a/plugin/pi/tokeneconomizer.ts b/plugin/pi/tokeneconomizer.tsindex 26b18b7..8914137 100644--- a/plugin/pi/tokeneconomizer.ts+++ b/plugin/pi/tokeneconomizer.ts@@ -40,6 +40,19 @@ const SECTION = "tokeneconomizer";const truthy = (v: string | undefined) => ["1", "true", "yes", "on"].includes((v ?? "").trim().toLowerCase());+/** The program and arguments that run a launcher. The launchers are POSIX `sh`, which Windows+ * cannot start; there the bootstrap they run is started with Python itself (`TE_PYTHON`, else+ * `python`). */+export function launch(+ script: string,+ args: string[],+ platform: string = process.platform,+): [string, string[]] {+ if (platform !== "win32") return [join(SCRIPTS, script), args];+ const mode = script === "te-hook" ? "hook" : "cli";+ return [process.env.TE_PYTHON || "python", [join(SCRIPTS, "te_bootstrap.py"), mode, ...args]];+}+/** Run one of the plugin's launchers with `input` on stdin; resolve to its stdout. */function run(script: string,@@ -52,8 +65,10 @@ function run(let out = "";let child;try {- child = spawn(join(SCRIPTS, script), args, {- env: { ...process.env, TE_PI_MODE: mode, ...extra },+ const [program, argv] = launch(script, args);+ // UTF-8 mode as the launchers set it, for where they are bypassed (Windows).+ child = spawn(program, argv, {+ env: { ...process.env, PYTHONUTF8: "1", TE_PI_MODE: mode, ...extra },stdio: ["pipe", "pipe", "ignore"],});} catch {@@ -64,7 +79,9 @@ function run(child.kill("SIGKILL");resolve("");}, HOOK_TIMEOUT_MS);- child.stdout.on("data", (d: Buffer) => (out += d.toString()));+ // Decoded as a stream, so a character split across two chunks arrives whole.+ child.stdout.setEncoding("utf8");+ child.stdout.on("data", (d: string) => (out += d));child.on("error", () => {clearTimeout(timer);resolve("");diff --git a/plugin/scripts/te_bootstrap.py b/plugin/scripts/te_bootstrap.pyindex 25b6986..7986a9e 100644--- a/plugin/scripts/te_bootstrap.py+++ b/plugin/scripts/te_bootstrap.py@@ -42,7 +42,7 @@ def _marketplace_sources(plugin_root):home, market = os.sep.join(parts[:-5]), parts[-3]out = [os.path.join(home, ".tmp", "marketplaces", market)]try:- text = open(os.path.join(home, "config.toml")).read()+ text = open(os.path.join(home, "config.toml"), encoding="utf-8", errors="replace").read()except (IOError, OSError):return outsection = None@@ -74,11 +74,11 @@ def _say_missing(plugin_root):"install from the bundle (scripts/build_plugin.py) or set TE_CORE_PATH\n" % plugin_root)path = os.path.join(home, "errors.log")try:- if os.path.exists(path) and line in open(path).read():+ if os.path.exists(path) and line in open(path, encoding="utf-8", errors="replace").read():returnif not os.path.isdir(home):os.makedirs(home)- open(path, "a").write(line)+ open(path, "a", encoding="utf-8").write(line)except (IOError, OSError):pass@@ -88,7 +88,7 @@ def main():rest = sys.argv[2:]if sys.version_info < MIN_VERSION:if mode == "hook":- sys.stdin.read()+ getattr(sys.stdin, "buffer", sys.stdin).read() # bytes: no locale to decode withsys.stdout.write("{}\n")return 0sys.stderr.write("TokenEconomizer needs Python >= %d.%d (this is %s); set TE_PYTHON\n"diff --git a/tests/test_live_hooks.py b/tests/test_live_hooks.pyindex 4f27e18..d24c97c 100644--- a/tests/test_live_hooks.py+++ b/tests/test_live_hooks.py@@ -866,6 +866,21 @@ def test_garbage_input_and_internal_errors_answer_no_change(env, repo, monkeypatassert "RuntimeError: bug" in log+def test_a_non_ascii_checkout_is_found_whatever_the_locale(env, tmp_path):+ """The agents send UTF-8. A text stdin decodes with the locale's code page -- on a Russian+ Windows, cp1251 turns `Программирование` into `РџСЂРѕРіСЂ...`, a directory that does not+ exist -- so the hook reads the bytes, and the session is the checkout's."""+ repo = tmp_path / "Программирование" / "agentsync"+ (repo / ".git").mkdir(parents=True)+ (repo / ".git" / "HEAD").write_text("ref: refs/heads/main\n", encoding="utf-8")+ raw = json.dumps({"hook_event_name": "SessionStart", "session_id": "s1", "source": "startup",+ "transcript_path": "", "cwd": str(repo)}, ensure_ascii=False)+ stdin = io.TextIOWrapper(io.BytesIO(raw.encode("utf-8")), encoding="cp1251")+ hook.main([], stdin, io.StringIO(), env)+ assert store_for(env, str(repo)).read("sessions/s1.json")+ assert not os.path.exists(os.path.join(env["TOKENECONOMIZER_HOME"], "errors.log"))++def test_side_calls_a_disabled_plugin_and_a_payload_without_a_session_do_nothing(env, repo):for flag in ("TE_SIDECALL", "TE_DISABLE"):assert call(dict(env, **{flag: "1"}), {"hook_event_name": "SessionStart",diff --git a/tests/test_plugin_package.py b/tests/test_plugin_package.pyindex 437c791..d970f39 100644--- a/tests/test_plugin_package.py+++ b/tests/test_plugin_package.py@@ -427,3 +427,39 @@ def test_codex_sees_no_claude_format_commands_folder():for manifest in (".claude-plugin/plugin.json", ".github/plugin/plugin.json"):with open(os.path.join(PLUGIN, manifest), encoding="utf-8") as fh:assert json.load(fh)["commands"] == "./slash-commands/"+++def _shipped_python():+ for top in ("tokeneconomizer", os.path.join("plugin", "scripts"), "scripts"):+ for root, _dirs, files in os.walk(os.path.join(REPO, top)):+ for name in sorted(files):+ if name.endswith(".py"):+ yield os.path.join(root, name)+++def test_text_io_names_its_encoding():+ """The agents' payloads, session logs and output are UTF-8 whatever the locale, and Python+ decodes a file or a pipe opened without an encoding with the locale's code page -- cp1251 on+ a Russian Windows, cp1252 on a US one -- which garbles a non-ASCII path and stops a fold at+ the first undecodable byte. So every text-mode `open()`, `read_text`/`write_text` and text+ subprocess call in the shipped code names its encoding."""+ import ast+ missing = []+ for path in _shipped_python():+ for call in ast.walk(ast.parse(open(path, encoding="utf-8").read())):+ if not isinstance(call, ast.Call):+ continue+ f, kw = call.func, {k.arg: k.value for k in call.keywords}+ if isinstance(f, ast.Name) and f.id == "open":+ mode = call.args[1] if len(call.args) > 1 else kw.get("mode")+ text = not (isinstance(mode, ast.Constant) and "b" in mode.value)+ elif isinstance(f, ast.Attribute) and f.attr in ("read_text", "write_text"):+ text = True+ elif (isinstance(f, ast.Attribute) and isinstance(f.value, ast.Name)+ and f.value.id == "subprocess"):+ text = "text" in kw or "universal_newlines" in kw+ else:+ continue+ if text and "encoding" not in kw:+ missing.append(f"{os.path.relpath(path, REPO)}:{call.lineno}")+ assert not missing, missingdiff --git a/tokeneconomizer/microskills/distiller.py b/tokeneconomizer/microskills/distiller.pyindex f46e751..6e3865d 100644--- a/tokeneconomizer/microskills/distiller.py+++ b/tokeneconomizer/microskills/distiller.py@@ -764,7 +764,7 @@ class Distiller:"facts": {k: f.to_dict() for k, f in self.facts.items()},"window": window,"raw_cmds": [[c for c, _ in cmds] for cmds in self.raw_window[-WINDOW:]]},- open(self._path(root, self.repo), "w"), indent=2)+ open(self._path(root, self.repo), "w", encoding="utf-8"), indent=2)@classmethoddef load(cls, repo, root, runner=None, model="agent", external_facts=None, vocab=None):@@ -772,7 +772,7 @@ class Distiller:self = cls(repo, runner, model, external_facts=external_facts, vocab=vocab)if not os.path.exists(p):return self- d = json.load(open(p))+ d = json.load(open(p, encoding="utf-8"))self.now = d.get("now", 0)self.opt_tokens = d.get("opt_tokens", 0.0)self.last_propose = d.get("last_propose", -1)diff --git a/tokeneconomizer/plugin/cli.py b/tokeneconomizer/plugin/cli.pyindex 6e1785c..20df253 100644--- a/tokeneconomizer/plugin/cli.py+++ b/tokeneconomizer/plugin/cli.py@@ -61,7 +61,7 @@ def _unwritable(path):probe = os.path.join(path, f".write-test-{os.getpid()}")try:os.makedirs(path, exist_ok=True)- open(probe, "w").close()+ open(probe, "w", encoding="utf-8").close()os.remove(probe)return Noneexcept OSError as e:@@ -271,7 +271,7 @@ def _wait_and_apply(args):except OSError:passctx.store.ensure()- with open(beat, "w") as fh:+ with open(beat, "w", encoding="utf-8") as fh:fh.write(str(os.getpid()))try:while True:@@ -617,7 +617,7 @@ def cmd_doctor(args):try:os.makedirs(root, exist_ok=True)probe = os.path.join(root, ".write-test")- open(probe, "w").close()+ open(probe, "w", encoding="utf-8").close()os.remove(probe)print(f"state root {root} (writable)")except OSError as e:diff --git a/tokeneconomizer/plugin/context.py b/tokeneconomizer/plugin/context.pyindex ec64057..a391b70 100644--- a/tokeneconomizer/plugin/context.py+++ b/tokeneconomizer/plugin/context.py@@ -79,10 +79,12 @@ class LiveContext:env = dict(self.env)env["PYTHONPATH"] = core + (os.pathsep + env["PYTHONPATH"] if env.get("PYTHONPATH")else "")+ # It reads session logs and the agents' output, which are UTF-8 whatever the locale.+ env["PYTHONUTF8"] = "1"cmd = [sys.executable, "-m", "tokeneconomizer.plugin.cli", *argv,"--host", self.host.name, "--cwd", self.repo.root]self.store.ensure()- log = open(self.store.path(log_name), "a")+ log = open(self.store.path(log_name), "a", encoding="utf-8")try:subprocess.Popen(cmd, stdin=subprocess.DEVNULL, stdout=log, stderr=log,cwd=self.repo.root, env=env, start_new_session=True)diff --git a/tokeneconomizer/plugin/denylist.py b/tokeneconomizer/plugin/denylist.pyindex f778ae5..ad2b62c 100644--- a/tokeneconomizer/plugin/denylist.py+++ b/tokeneconomizer/plugin/denylist.py@@ -386,7 +386,7 @@ def _git_tracked(repo_root, rel):"""Is `rel` committed in this repository? False when git cannot tell."""try:out = subprocess.run(["git", "ls-files", "--error-unmatch", rel], cwd=repo_root,- capture_output=True, text=True, timeout=10)+ capture_output=True, encoding="utf-8", errors="replace", timeout=10)return out.returncode == 0except (OSError, subprocess.SubprocessError):return False@@ -401,8 +401,9 @@ def codex_config_error(repo_root, cli="codex", timeout=30):if not shutil.which(cli):return Nonetry:- out = subprocess.run([cli, "features", "list"], capture_output=True, text=True,- cwd=repo_root, timeout=timeout, env=dict(os.environ, TE_SIDECALL="1"))+ out = subprocess.run([cli, "features", "list"], capture_output=True,+ encoding="utf-8", errors="replace", cwd=repo_root, timeout=timeout,+ env=dict(os.environ, TE_SIDECALL="1"))except (OSError, subprocess.SubprocessError):return Nonereturn None if out.returncode == 0 else ((out.stderr or out.stdout).strip()[:500]diff --git a/tokeneconomizer/plugin/hook.py b/tokeneconomizer/plugin/hook.pyindex ebadd09..dbc2b8b 100644--- a/tokeneconomizer/plugin/hook.py+++ b/tokeneconomizer/plugin/hook.py@@ -54,7 +54,10 @@ def main(argv=None, stdin=None, stdout=None, env=None):hint = argv[i + 1] if i + 1 < len(argv) else Noneout = {}try:- raw = stdin.read()+ # The agents send UTF-8 whatever the locale; a text stdin would decode it with the+ # locale's code page on Windows and garble a non-ASCII path.+ buf = getattr(stdin, "buffer", None)+ raw = buf.read().decode("utf-8") if buf is not None else stdin.read()if not (_truthy(env.get("TE_SIDECALL")) or _truthy(env.get("TE_DISABLE"))):payload = json.loads(raw) if raw.strip() else {}if isinstance(payload, dict):diff --git a/tokeneconomizer/plugin/hosts.py b/tokeneconomizer/plugin/hosts.pyindex 51e2528..34e9af7 100644--- a/tokeneconomizer/plugin/hosts.py+++ b/tokeneconomizer/plugin/hosts.py@@ -426,7 +426,7 @@ def _ancestor_args(limit=4):breaktry:line = subprocess.run(["ps", "-o", "ppid=,args=", "-p", str(pid)], capture_output=True,- text=True, timeout=2).stdout.strip()+ encoding="utf-8", errors="replace", timeout=2).stdout.strip()except (OSError, subprocess.SubprocessError):breakppid, _, args = line.partition(" ")diff --git a/tokeneconomizer/plugin/probe.py b/tokeneconomizer/plugin/probe.pyindex bd4c060..1b939b4 100644--- a/tokeneconomizer/plugin/probe.py+++ b/tokeneconomizer/plugin/probe.py@@ -120,8 +120,8 @@ def run_session(repo, env, plugin, model, cli="copilot", timeout=600):"--output-format", "json"]if model:cmd += ["--model", model]- out = subprocess.run(cmd, cwd=repo, env=env, capture_output=True, text=True,- timeout=timeout, stdin=subprocess.DEVNULL)+ out = subprocess.run(cmd, cwd=repo, env=env, capture_output=True, encoding="utf-8",+ errors="replace", timeout=timeout, stdin=subprocess.DEVNULL)sid, answer, error = None, "", (out.stderr or "").strip().splitlines()[-1:] or [""]for line in (out.stdout or "").splitlines():try:diff --git a/tokeneconomizer/plugin/repomap.py b/tokeneconomizer/plugin/repomap.pyindex 7a4ecef..9d4cbc3 100644--- a/tokeneconomizer/plugin/repomap.py+++ b/tokeneconomizer/plugin/repomap.py@@ -76,7 +76,7 @@ def _write_state(ctx, d):def head_commit(root):try:out = subprocess.run(["git", "rev-parse", "HEAD"], cwd=root, capture_output=True,- text=True, timeout=10)+ encoding="utf-8", errors="replace", timeout=10)return out.stdout.strip() if out.returncode == 0 else ""except (OSError, subprocess.SubprocessError):return ""@@ -140,8 +140,8 @@ def generate(ctx, python, idents=(), timeout=REFRESH_TIMEOUT_S):cmd = [python, GEN, ctx.repo.root, str(ctx.config.repomap_tokens), ",".join(idents),"--git", "--cache", _cache_dir(ctx)]try:- out = subprocess.run(cmd, capture_output=True, text=True, timeout=timeout,- env=_gen_env(ctx), cwd=ctx.repo.root)+ out = subprocess.run(cmd, capture_output=True, encoding="utf-8", errors="replace",+ timeout=timeout, env=_gen_env(ctx), cwd=ctx.repo.root)except subprocess.TimeoutExpired:return "", f"timed out after {timeout}s"except OSError as e:diff --git a/tokeneconomizer/plugin/repomap_gen.py b/tokeneconomizer/plugin/repomap_gen.pyindex 8b8a5e7..53ea0ed 100644--- a/tokeneconomizer/plugin/repomap_gen.py+++ b/tokeneconomizer/plugin/repomap_gen.py@@ -35,7 +35,8 @@ MAX_BYTES = 256 * 1024def _git_files(repo):import subprocess- out = subprocess.run(["git", "ls-files", "-z"], cwd=repo, capture_output=True, text=True)+ out = subprocess.run(["git", "ls-files", "-z"], cwd=repo, capture_output=True,+ encoding="utf-8", errors="replace")files = []for f in out.stdout.split("\0"):path = "/" + fdiff --git a/tokeneconomizer/plugin/repos.py b/tokeneconomizer/plugin/repos.pyindex 12bed6e..ebda1e4 100644--- a/tokeneconomizer/plugin/repos.py+++ b/tokeneconomizer/plugin/repos.py@@ -124,7 +124,7 @@ def prune_pins(root, host, now=None, max_age_s=PIN_MAX_AGE_S):if name.endswith(".json") and now - os.path.getmtime(p) > max_age_s:os.remove(p)n += 1- open(marker, "w").close()+ open(marker, "w", encoding="utf-8").close()except OSError:passreturn n@@ -700,7 +700,7 @@ def start_migration(ctx):return Falseexcept OSError:pass- open(marker, "w").close()+ open(marker, "w", encoding="utf-8").close()return ctx.spawn(["repos", "migrate", "--quiet"], "migrate.log")diff --git a/tokeneconomizer/plugin/sidecall.py b/tokeneconomizer/plugin/sidecall.pyindex 39ef417..e2f11f3 100644--- a/tokeneconomizer/plugin/sidecall.py+++ b/tokeneconomizer/plugin/sidecall.py@@ -111,8 +111,9 @@ def claude_flags(cli="claude"):not a session of the person's, and should not appear in their history (`claude --resume`)."""if cli not in _FLAGS:try:- help_text = subprocess.run([cli, "--help"], capture_output=True, text=True,- timeout=30, stdin=subprocess.DEVNULL).stdout+ help_text = subprocess.run([cli, "--help"], capture_output=True,+ encoding="utf-8", errors="replace", timeout=30,+ stdin=subprocess.DEVNULL).stdoutexcept (OSError, subprocess.SubprocessError):help_text = ""_FLAGS[cli] = ["--no-session-persistence"] if "--no-session-persistence" in help_text else []@@ -123,8 +124,9 @@ def claude_call(prompt, model="haiku", cli="claude", timeout=TIMEOUT_S):cmd = [cli, "-p", "--output-format", "json", "--max-turns", "1", "--tools", "","--model", model or "haiku"] + claude_flags(cli)with tempfile.TemporaryDirectory(prefix="te-side-") as cwd:- out = subprocess.run(cmd, input=prompt, capture_output=True, text=True,- timeout=timeout, env=_env(), cwd=cwd)+ out = subprocess.run(cmd, input=prompt, capture_output=True,+ encoding="utf-8", errors="replace", timeout=timeout, env=_env(),+ cwd=cwd)try:data = json.loads(out.stdout or "{}")except ValueError:@@ -144,8 +146,8 @@ def codex_call(prompt, model="", cli="codex", timeout=TIMEOUT_S):cmd += ["-m", model]cmd.append(prompt)with tempfile.TemporaryDirectory(prefix="te-side-") as cwd:- out = subprocess.run(cmd, capture_output=True, text=True, timeout=timeout,- env=_env(), cwd=cwd)+ out = subprocess.run(cmd, capture_output=True, encoding="utf-8", errors="replace",+ timeout=timeout, env=_env(), cwd=cwd)text, usage = "", Nonefor line in (out.stdout or "").splitlines():try:@@ -183,8 +185,9 @@ def pi_call(prompt, model="", cli="pi", timeout=TIMEOUT_S):with tempfile.TemporaryDirectory(prefix="te-side-") as cwd:for extra in (["--no-mcp"], []):cmd = [cli, *PI_FLAGS, *extra, *model_args, "--", prompt]- out = subprocess.run(cmd, capture_output=True, text=True, timeout=timeout,- env=_env(), cwd=cwd, stdin=subprocess.DEVNULL)+ out = subprocess.run(cmd, capture_output=True, encoding="utf-8", errors="replace",+ timeout=timeout, env=_env(), cwd=cwd,+ stdin=subprocess.DEVNULL)if "Unknown option: --no-mcp" not in (out.stderr or ""):breaktext, usage = trace.pi_json_reply(out.stdout)@@ -243,8 +246,8 @@ def copilot_call(prompt, model="", cli="copilot", timeout=TIMEOUT_S):if model:cmd += ["--model", model]with tempfile.TemporaryDirectory(prefix="te-side-") as cwd:- out = subprocess.run(cmd, capture_output=True, text=True, timeout=timeout,- env=_env(), cwd=cwd, stdin=subprocess.DEVNULL)+ out = subprocess.run(cmd, capture_output=True, encoding="utf-8", errors="replace",+ timeout=timeout, env=_env(), cwd=cwd, stdin=subprocess.DEVNULL)text, sid = "", Nonefor line in (out.stdout or "").splitlines():try:diff --git a/tokeneconomizer/plugin/store.py b/tokeneconomizer/plugin/store.pyindex 9acf686..4d02002 100644--- a/tokeneconomizer/plugin/store.py+++ b/tokeneconomizer/plugin/store.py@@ -81,7 +81,7 @@ class Store:if fcntl is None:yield Truereturn- fh = open(self.path(f".{name}.lock"), "a+")+ fh = open(self.path(f".{name}.lock"), "a+", encoding="utf-8")got = Falsedeadline = time.time() + timeouttry:diff --git a/tokeneconomizer/plugin/toolcheck.py b/tokeneconomizer/plugin/toolcheck.pyindex 319d9e3..9aff29e 100644--- a/tokeneconomizer/plugin/toolcheck.py+++ b/tokeneconomizer/plugin/toolcheck.py@@ -41,8 +41,8 @@ def verify_codex(repo_root, capabilities, cli="codex", timeout=30):if not feats or not shutil.which(cli):return Nonetry:- out = subprocess.run([cli, "features", "list"], capture_output=True, text=True,- cwd=repo_root, timeout=timeout,+ out = subprocess.run([cli, "features", "list"], capture_output=True,+ encoding="utf-8", errors="replace", cwd=repo_root, timeout=timeout,env=dict(os.environ, TE_SIDECALL="1"))except (OSError, subprocess.SubprocessError):return None@@ -62,7 +62,8 @@ def _init_tools(extra_args, cli="claude", timeout=60):"--model", "haiku", "--max-turns", "1"] + claude_flags(cli) + list(extra_args)with tempfile.TemporaryDirectory(prefix="te-probe-") as cwd:p = subprocess.Popen(cmd, stdin=subprocess.DEVNULL, stdout=subprocess.PIPE,- stderr=subprocess.DEVNULL, text=True, env=env, cwd=cwd)+ stderr=subprocess.DEVNULL, encoding="utf-8", errors="replace", env=env,+ cwd=cwd)tools, version = None, Nonedeadline = time.time() + timeouttry:@@ -148,8 +149,9 @@ def copilot_capture(extra_args=(), cwd=None, cli="copilot", timeout=90):cwd = tmptry:subprocess.run([cli, "-p", "Reply with OK", "--no-auto-update", "--no-color"]- + list(extra_args), cwd=cwd, env=env, capture_output=True, text=True,- timeout=timeout, stdin=subprocess.DEVNULL)+ + list(extra_args), cwd=cwd, env=env, capture_output=True,+ encoding="utf-8", errors="replace", timeout=timeout,+ stdin=subprocess.DEVNULL)except (OSError, subprocess.SubprocessError):passfinally:@@ -167,7 +169,8 @@ def _request_tools(body):def _copilot_version(cli="copilot"):try:- out = subprocess.run([cli, "--version"], capture_output=True, text=True, timeout=30)+ out = subprocess.run([cli, "--version"], capture_output=True,+ encoding="utf-8", errors="replace", timeout=30)return (out.stdout or "").strip().splitlines()[0]except (OSError, subprocess.SubprocessError, IndexError):return None@@ -200,8 +203,8 @@ def _prompt_tokens(extra_args, cli="claude", timeout=180):cmd = [cli, "-p", "Reply with exactly: OK", "--output-format", "json", "--model", "haiku","--max-turns", "1"] + claude_flags(cli) + list(extra_args)with tempfile.TemporaryDirectory(prefix="te-measure-") as cwd:- out = subprocess.run(cmd, capture_output=True, text=True, env=env, cwd=cwd,- timeout=timeout)+ out = subprocess.run(cmd, capture_output=True, encoding="utf-8", errors="replace", env=env,+ cwd=cwd, timeout=timeout)data = json.loads(out.stdout or "{}")u = data.get("usage") or {}return (sum(int(u.get(k) or 0) for k in ("input_tokens", "cache_read_input_tokens",@@ -214,8 +217,8 @@ def _codex_prompt_chars(overrides, cli="codex", timeout=60):for table, key, value in overrides:cmd += ["-c", f"{table}.{key}={value}" if table else f"{key}={value}"]with tempfile.TemporaryDirectory(prefix="te-measure-") as cwd:- out = subprocess.run(cmd + ["OK"], capture_output=True, text=True, cwd=cwd,- timeout=timeout, env=dict(os.environ, TE_SIDECALL="1"))+ out = subprocess.run(cmd + ["OK"], capture_output=True, encoding="utf-8", errors="replace",+ cwd=cwd, timeout=timeout, env=dict(os.environ, TE_SIDECALL="1"))return len(out.stdout or "") if out.returncode == 0 else Nonediff --git a/tokeneconomizer/trace/calls.py b/tokeneconomizer/trace/calls.pyindex 8130c8d..b8ce150 100644--- a/tokeneconomizer/trace/calls.py+++ b/tokeneconomizer/trace/calls.py@@ -38,7 +38,7 @@ def load_trajectory(path):envelopes carrying `tool_use` blocks, and anything else is read as Copilot events."""if not os.path.exists(path):return []- lines = [l.strip() for l in open(path) if l.strip()]+ lines = [l.strip() for l in open(path, encoding="utf-8", errors="replace") if l.strip()]if _is_codex_exec(lines):return codex_exec_calls(lines)if any('"tool_use"' in line or '"type": "assistant"' in line or '"type":"assistant"' in line@@ -57,7 +57,7 @@ def call_cwds(path):calls = load_trajectory(path)if not calls or not os.path.exists(path):return [None] * len(calls)- lines = [l.strip() for l in open(path) if l.strip()]+ lines = [l.strip() for l in open(path, encoding="utf-8", errors="replace") if l.strip()]if _is_codex_exec(lines):return [None] * len(calls)cwds = [c[4] for c in claude_calls(lines)]