Skip to content

Experience-Driven Black-Box Cost Control for Coding Agents

Read agent input and session logs as UTF-8 on any locale

Aaronfd2c2e51712f
diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
new file mode 100644
index 0000000..d69d03b
--- /dev/null
+++ b/.github/workflows/tests.yml
@@ -0,0 +1,29 @@
+name: tests
+
+on:
+ push:
+ branches: [main]
+ pull_request:
+
+permissions:
+ contents: read
+
+jobs:
+ tests:
+ strategy:
+ fail-fast: false
+ matrix:
+ os: [ubuntu-latest, windows-latest]
+ python: ["3.10", "3.13"]
+ runs-on: ${{ matrix.os }}
+ defaults:
+ run:
+ # Git Bash on Windows: the launchers are `sh` scripts, run there as Claude Code runs them.
+ shell: bash
+ steps:
+ - uses: actions/checkout@v5
+ - uses: actions/setup-python@v6
+ with:
+ python-version: ${{ matrix.python }}
+ - run: python -m pip install pytest
+ - run: python -m pytest tests -rs
diff --git a/plugin/pi/tokeneconomizer.ts b/plugin/pi/tokeneconomizer.ts
index 26b18b7..8914137 100644
--- a/plugin/pi/tokeneconomizer.ts
+++ b/plugin/pi/tokeneconomizer.ts
@@ -40,6 +40,19 @@ const SECTION = "tokeneconomizer";
const truthy = (v: string | undefined) => ["1", "true", "yes", "on"].includes((v ?? "").trim().toLowerCase());
+/** The program and arguments that run a launcher. The launchers are POSIX `sh`, which Windows
+ * cannot start; there the bootstrap they run is started with Python itself (`TE_PYTHON`, else
+ * `python`). */
+export function launch(
+ script: string,
+ args: string[],
+ platform: string = process.platform,
+): [string, string[]] {
+ if (platform !== "win32") return [join(SCRIPTS, script), args];
+ const mode = script === "te-hook" ? "hook" : "cli";
+ return [process.env.TE_PYTHON || "python", [join(SCRIPTS, "te_bootstrap.py"), mode, ...args]];
+}
+
/** Run one of the plugin's launchers with `input` on stdin; resolve to its stdout. */
function run(
script: string,
@@ -52,8 +65,10 @@ function run(
let out = "";
let child;
try {
- child = spawn(join(SCRIPTS, script), args, {
- env: { ...process.env, TE_PI_MODE: mode, ...extra },
+ const [program, argv] = launch(script, args);
+ // UTF-8 mode as the launchers set it, for where they are bypassed (Windows).
+ child = spawn(program, argv, {
+ env: { ...process.env, PYTHONUTF8: "1", TE_PI_MODE: mode, ...extra },
stdio: ["pipe", "pipe", "ignore"],
});
} catch {
@@ -64,7 +79,9 @@ function run(
child.kill("SIGKILL");
resolve("");
}, HOOK_TIMEOUT_MS);
- child.stdout.on("data", (d: Buffer) => (out += d.toString()));
+ // Decoded as a stream, so a character split across two chunks arrives whole.
+ child.stdout.setEncoding("utf8");
+ child.stdout.on("data", (d: string) => (out += d));
child.on("error", () => {
clearTimeout(timer);
resolve("");
diff --git a/plugin/scripts/te_bootstrap.py b/plugin/scripts/te_bootstrap.py
index 25b6986..7986a9e 100644
--- a/plugin/scripts/te_bootstrap.py
+++ b/plugin/scripts/te_bootstrap.py
@@ -42,7 +42,7 @@ def _marketplace_sources(plugin_root):
home, market = os.sep.join(parts[:-5]), parts[-3]
out = [os.path.join(home, ".tmp", "marketplaces", market)]
try:
- text = open(os.path.join(home, "config.toml")).read()
+ text = open(os.path.join(home, "config.toml"), encoding="utf-8", errors="replace").read()
except (IOError, OSError):
return out
section = None
@@ -74,11 +74,11 @@ def _say_missing(plugin_root):
"install from the bundle (scripts/build_plugin.py) or set TE_CORE_PATH\n" % plugin_root)
path = os.path.join(home, "errors.log")
try:
- if os.path.exists(path) and line in open(path).read():
+ if os.path.exists(path) and line in open(path, encoding="utf-8", errors="replace").read():
return
if not os.path.isdir(home):
os.makedirs(home)
- open(path, "a").write(line)
+ open(path, "a", encoding="utf-8").write(line)
except (IOError, OSError):
pass
@@ -88,7 +88,7 @@ def main():
rest = sys.argv[2:]
if sys.version_info < MIN_VERSION:
if mode == "hook":
- sys.stdin.read()
+ getattr(sys.stdin, "buffer", sys.stdin).read() # bytes: no locale to decode with
sys.stdout.write("{}\n")
return 0
sys.stderr.write("TokenEconomizer needs Python >= %d.%d (this is %s); set TE_PYTHON\n"
diff --git a/tests/test_live_hooks.py b/tests/test_live_hooks.py
index 4f27e18..d24c97c 100644
--- a/tests/test_live_hooks.py
+++ b/tests/test_live_hooks.py
@@ -866,6 +866,21 @@ def test_garbage_input_and_internal_errors_answer_no_change(env, repo, monkeypat
assert "RuntimeError: bug" in log
+def test_a_non_ascii_checkout_is_found_whatever_the_locale(env, tmp_path):
+ """The agents send UTF-8. A text stdin decodes with the locale's code page -- on a Russian
+ Windows, cp1251 turns `Программирование` into `РџСЂРѕРіСЂ...`, a directory that does not
+ exist -- so the hook reads the bytes, and the session is the checkout's."""
+ repo = tmp_path / "Программирование" / "agentsync"
+ (repo / ".git").mkdir(parents=True)
+ (repo / ".git" / "HEAD").write_text("ref: refs/heads/main\n", encoding="utf-8")
+ raw = json.dumps({"hook_event_name": "SessionStart", "session_id": "s1", "source": "startup",
+ "transcript_path": "", "cwd": str(repo)}, ensure_ascii=False)
+ stdin = io.TextIOWrapper(io.BytesIO(raw.encode("utf-8")), encoding="cp1251")
+ hook.main([], stdin, io.StringIO(), env)
+ assert store_for(env, str(repo)).read("sessions/s1.json")
+ assert not os.path.exists(os.path.join(env["TOKENECONOMIZER_HOME"], "errors.log"))
+
+
def test_side_calls_a_disabled_plugin_and_a_payload_without_a_session_do_nothing(env, repo):
for flag in ("TE_SIDECALL", "TE_DISABLE"):
assert call(dict(env, **{flag: "1"}), {"hook_event_name": "SessionStart",
diff --git a/tests/test_plugin_package.py b/tests/test_plugin_package.py
index 437c791..d970f39 100644
--- a/tests/test_plugin_package.py
+++ b/tests/test_plugin_package.py
@@ -427,3 +427,39 @@ def test_codex_sees_no_claude_format_commands_folder():
for manifest in (".claude-plugin/plugin.json", ".github/plugin/plugin.json"):
with open(os.path.join(PLUGIN, manifest), encoding="utf-8") as fh:
assert json.load(fh)["commands"] == "./slash-commands/"
+
+
+def _shipped_python():
+ for top in ("tokeneconomizer", os.path.join("plugin", "scripts"), "scripts"):
+ for root, _dirs, files in os.walk(os.path.join(REPO, top)):
+ for name in sorted(files):
+ if name.endswith(".py"):
+ yield os.path.join(root, name)
+
+
+def test_text_io_names_its_encoding():
+ """The agents' payloads, session logs and output are UTF-8 whatever the locale, and Python
+ decodes a file or a pipe opened without an encoding with the locale's code page -- cp1251 on
+ a Russian Windows, cp1252 on a US one -- which garbles a non-ASCII path and stops a fold at
+ the first undecodable byte. So every text-mode `open()`, `read_text`/`write_text` and text
+ subprocess call in the shipped code names its encoding."""
+ import ast
+ missing = []
+ for path in _shipped_python():
+ for call in ast.walk(ast.parse(open(path, encoding="utf-8").read())):
+ if not isinstance(call, ast.Call):
+ continue
+ f, kw = call.func, {k.arg: k.value for k in call.keywords}
+ if isinstance(f, ast.Name) and f.id == "open":
+ mode = call.args[1] if len(call.args) > 1 else kw.get("mode")
+ text = not (isinstance(mode, ast.Constant) and "b" in mode.value)
+ elif isinstance(f, ast.Attribute) and f.attr in ("read_text", "write_text"):
+ text = True
+ elif (isinstance(f, ast.Attribute) and isinstance(f.value, ast.Name)
+ and f.value.id == "subprocess"):
+ text = "text" in kw or "universal_newlines" in kw
+ else:
+ continue
+ if text and "encoding" not in kw:
+ missing.append(f"{os.path.relpath(path, REPO)}:{call.lineno}")
+ assert not missing, missing
diff --git a/tokeneconomizer/microskills/distiller.py b/tokeneconomizer/microskills/distiller.py
index f46e751..6e3865d 100644
--- a/tokeneconomizer/microskills/distiller.py
+++ b/tokeneconomizer/microskills/distiller.py
@@ -764,7 +764,7 @@ class Distiller:
"facts": {k: f.to_dict() for k, f in self.facts.items()},
"window": window,
"raw_cmds": [[c for c, _ in cmds] for cmds in self.raw_window[-WINDOW:]]},
- open(self._path(root, self.repo), "w"), indent=2)
+ open(self._path(root, self.repo), "w", encoding="utf-8"), indent=2)
@classmethod
def load(cls, repo, root, runner=None, model="agent", external_facts=None, vocab=None):
@@ -772,7 +772,7 @@ class Distiller:
self = cls(repo, runner, model, external_facts=external_facts, vocab=vocab)
if not os.path.exists(p):
return self
- d = json.load(open(p))
+ d = json.load(open(p, encoding="utf-8"))
self.now = d.get("now", 0)
self.opt_tokens = d.get("opt_tokens", 0.0)
self.last_propose = d.get("last_propose", -1)
diff --git a/tokeneconomizer/plugin/cli.py b/tokeneconomizer/plugin/cli.py
index 6e1785c..20df253 100644
--- a/tokeneconomizer/plugin/cli.py
+++ b/tokeneconomizer/plugin/cli.py
@@ -61,7 +61,7 @@ def _unwritable(path):
probe = os.path.join(path, f".write-test-{os.getpid()}")
try:
os.makedirs(path, exist_ok=True)
- open(probe, "w").close()
+ open(probe, "w", encoding="utf-8").close()
os.remove(probe)
return None
except OSError as e:
@@ -271,7 +271,7 @@ def _wait_and_apply(args):
except OSError:
pass
ctx.store.ensure()
- with open(beat, "w") as fh:
+ with open(beat, "w", encoding="utf-8") as fh:
fh.write(str(os.getpid()))
try:
while True:
@@ -617,7 +617,7 @@ def cmd_doctor(args):
try:
os.makedirs(root, exist_ok=True)
probe = os.path.join(root, ".write-test")
- open(probe, "w").close()
+ open(probe, "w", encoding="utf-8").close()
os.remove(probe)
print(f"state root {root} (writable)")
except OSError as e:
diff --git a/tokeneconomizer/plugin/context.py b/tokeneconomizer/plugin/context.py
index ec64057..a391b70 100644
--- a/tokeneconomizer/plugin/context.py
+++ b/tokeneconomizer/plugin/context.py
@@ -79,10 +79,12 @@ class LiveContext:
env = dict(self.env)
env["PYTHONPATH"] = core + (os.pathsep + env["PYTHONPATH"] if env.get("PYTHONPATH")
else "")
+ # It reads session logs and the agents' output, which are UTF-8 whatever the locale.
+ env["PYTHONUTF8"] = "1"
cmd = [sys.executable, "-m", "tokeneconomizer.plugin.cli", *argv,
"--host", self.host.name, "--cwd", self.repo.root]
self.store.ensure()
- log = open(self.store.path(log_name), "a")
+ log = open(self.store.path(log_name), "a", encoding="utf-8")
try:
subprocess.Popen(cmd, stdin=subprocess.DEVNULL, stdout=log, stderr=log,
cwd=self.repo.root, env=env, start_new_session=True)
diff --git a/tokeneconomizer/plugin/denylist.py b/tokeneconomizer/plugin/denylist.py
index f778ae5..ad2b62c 100644
--- a/tokeneconomizer/plugin/denylist.py
+++ b/tokeneconomizer/plugin/denylist.py
@@ -386,7 +386,7 @@ def _git_tracked(repo_root, rel):
"""Is `rel` committed in this repository? False when git cannot tell."""
try:
out = subprocess.run(["git", "ls-files", "--error-unmatch", rel], cwd=repo_root,
- capture_output=True, text=True, timeout=10)
+ capture_output=True, encoding="utf-8", errors="replace", timeout=10)
return out.returncode == 0
except (OSError, subprocess.SubprocessError):
return False
@@ -401,8 +401,9 @@ def codex_config_error(repo_root, cli="codex", timeout=30):
if not shutil.which(cli):
return None
try:
- out = subprocess.run([cli, "features", "list"], capture_output=True, text=True,
- cwd=repo_root, timeout=timeout, env=dict(os.environ, TE_SIDECALL="1"))
+ out = subprocess.run([cli, "features", "list"], capture_output=True,
+ encoding="utf-8", errors="replace", cwd=repo_root, timeout=timeout,
+ env=dict(os.environ, TE_SIDECALL="1"))
except (OSError, subprocess.SubprocessError):
return None
return None if out.returncode == 0 else ((out.stderr or out.stdout).strip()[:500]
diff --git a/tokeneconomizer/plugin/hook.py b/tokeneconomizer/plugin/hook.py
index ebadd09..dbc2b8b 100644
--- a/tokeneconomizer/plugin/hook.py
+++ b/tokeneconomizer/plugin/hook.py
@@ -54,7 +54,10 @@ def main(argv=None, stdin=None, stdout=None, env=None):
hint = argv[i + 1] if i + 1 < len(argv) else None
out = {}
try:
- raw = stdin.read()
+ # The agents send UTF-8 whatever the locale; a text stdin would decode it with the
+ # locale's code page on Windows and garble a non-ASCII path.
+ buf = getattr(stdin, "buffer", None)
+ raw = buf.read().decode("utf-8") if buf is not None else stdin.read()
if not (_truthy(env.get("TE_SIDECALL")) or _truthy(env.get("TE_DISABLE"))):
payload = json.loads(raw) if raw.strip() else {}
if isinstance(payload, dict):
diff --git a/tokeneconomizer/plugin/hosts.py b/tokeneconomizer/plugin/hosts.py
index 51e2528..34e9af7 100644
--- a/tokeneconomizer/plugin/hosts.py
+++ b/tokeneconomizer/plugin/hosts.py
@@ -426,7 +426,7 @@ def _ancestor_args(limit=4):
break
try:
line = subprocess.run(["ps", "-o", "ppid=,args=", "-p", str(pid)], capture_output=True,
- text=True, timeout=2).stdout.strip()
+ encoding="utf-8", errors="replace", timeout=2).stdout.strip()
except (OSError, subprocess.SubprocessError):
break
ppid, _, args = line.partition(" ")
diff --git a/tokeneconomizer/plugin/probe.py b/tokeneconomizer/plugin/probe.py
index bd4c060..1b939b4 100644
--- a/tokeneconomizer/plugin/probe.py
+++ b/tokeneconomizer/plugin/probe.py
@@ -120,8 +120,8 @@ def run_session(repo, env, plugin, model, cli="copilot", timeout=600):
"--output-format", "json"]
if model:
cmd += ["--model", model]
- out = subprocess.run(cmd, cwd=repo, env=env, capture_output=True, text=True,
- timeout=timeout, stdin=subprocess.DEVNULL)
+ out = subprocess.run(cmd, cwd=repo, env=env, capture_output=True, encoding="utf-8",
+ errors="replace", timeout=timeout, stdin=subprocess.DEVNULL)
sid, answer, error = None, "", (out.stderr or "").strip().splitlines()[-1:] or [""]
for line in (out.stdout or "").splitlines():
try:
diff --git a/tokeneconomizer/plugin/repomap.py b/tokeneconomizer/plugin/repomap.py
index 7a4ecef..9d4cbc3 100644
--- a/tokeneconomizer/plugin/repomap.py
+++ b/tokeneconomizer/plugin/repomap.py
@@ -76,7 +76,7 @@ def _write_state(ctx, d):
def head_commit(root):
try:
out = subprocess.run(["git", "rev-parse", "HEAD"], cwd=root, capture_output=True,
- text=True, timeout=10)
+ encoding="utf-8", errors="replace", timeout=10)
return out.stdout.strip() if out.returncode == 0 else ""
except (OSError, subprocess.SubprocessError):
return ""
@@ -140,8 +140,8 @@ def generate(ctx, python, idents=(), timeout=REFRESH_TIMEOUT_S):
cmd = [python, GEN, ctx.repo.root, str(ctx.config.repomap_tokens), ",".join(idents),
"--git", "--cache", _cache_dir(ctx)]
try:
- out = subprocess.run(cmd, capture_output=True, text=True, timeout=timeout,
- env=_gen_env(ctx), cwd=ctx.repo.root)
+ out = subprocess.run(cmd, capture_output=True, encoding="utf-8", errors="replace",
+ timeout=timeout, env=_gen_env(ctx), cwd=ctx.repo.root)
except subprocess.TimeoutExpired:
return "", f"timed out after {timeout}s"
except OSError as e:
diff --git a/tokeneconomizer/plugin/repomap_gen.py b/tokeneconomizer/plugin/repomap_gen.py
index 8b8a5e7..53ea0ed 100644
--- a/tokeneconomizer/plugin/repomap_gen.py
+++ b/tokeneconomizer/plugin/repomap_gen.py
@@ -35,7 +35,8 @@ MAX_BYTES = 256 * 1024
def _git_files(repo):
import subprocess
- out = subprocess.run(["git", "ls-files", "-z"], cwd=repo, capture_output=True, text=True)
+ out = subprocess.run(["git", "ls-files", "-z"], cwd=repo, capture_output=True,
+ encoding="utf-8", errors="replace")
files = []
for f in out.stdout.split("\0"):
path = "/" + f
diff --git a/tokeneconomizer/plugin/repos.py b/tokeneconomizer/plugin/repos.py
index 12bed6e..ebda1e4 100644
--- a/tokeneconomizer/plugin/repos.py
+++ b/tokeneconomizer/plugin/repos.py
@@ -124,7 +124,7 @@ def prune_pins(root, host, now=None, max_age_s=PIN_MAX_AGE_S):
if name.endswith(".json") and now - os.path.getmtime(p) > max_age_s:
os.remove(p)
n += 1
- open(marker, "w").close()
+ open(marker, "w", encoding="utf-8").close()
except OSError:
pass
return n
@@ -700,7 +700,7 @@ def start_migration(ctx):
return False
except OSError:
pass
- open(marker, "w").close()
+ open(marker, "w", encoding="utf-8").close()
return ctx.spawn(["repos", "migrate", "--quiet"], "migrate.log")
diff --git a/tokeneconomizer/plugin/sidecall.py b/tokeneconomizer/plugin/sidecall.py
index 39ef417..e2f11f3 100644
--- a/tokeneconomizer/plugin/sidecall.py
+++ b/tokeneconomizer/plugin/sidecall.py
@@ -111,8 +111,9 @@ def claude_flags(cli="claude"):
not a session of the person's, and should not appear in their history (`claude --resume`)."""
if cli not in _FLAGS:
try:
- help_text = subprocess.run([cli, "--help"], capture_output=True, text=True,
- timeout=30, stdin=subprocess.DEVNULL).stdout
+ help_text = subprocess.run([cli, "--help"], capture_output=True,
+ encoding="utf-8", errors="replace", timeout=30,
+ stdin=subprocess.DEVNULL).stdout
except (OSError, subprocess.SubprocessError):
help_text = ""
_FLAGS[cli] = ["--no-session-persistence"] if "--no-session-persistence" in help_text else []
@@ -123,8 +124,9 @@ def claude_call(prompt, model="haiku", cli="claude", timeout=TIMEOUT_S):
cmd = [cli, "-p", "--output-format", "json", "--max-turns", "1", "--tools", "",
"--model", model or "haiku"] + claude_flags(cli)
with tempfile.TemporaryDirectory(prefix="te-side-") as cwd:
- out = subprocess.run(cmd, input=prompt, capture_output=True, text=True,
- timeout=timeout, env=_env(), cwd=cwd)
+ out = subprocess.run(cmd, input=prompt, capture_output=True,
+ encoding="utf-8", errors="replace", timeout=timeout, env=_env(),
+ cwd=cwd)
try:
data = json.loads(out.stdout or "{}")
except ValueError:
@@ -144,8 +146,8 @@ def codex_call(prompt, model="", cli="codex", timeout=TIMEOUT_S):
cmd += ["-m", model]
cmd.append(prompt)
with tempfile.TemporaryDirectory(prefix="te-side-") as cwd:
- out = subprocess.run(cmd, capture_output=True, text=True, timeout=timeout,
- env=_env(), cwd=cwd)
+ out = subprocess.run(cmd, capture_output=True, encoding="utf-8", errors="replace",
+ timeout=timeout, env=_env(), cwd=cwd)
text, usage = "", None
for line in (out.stdout or "").splitlines():
try:
@@ -183,8 +185,9 @@ def pi_call(prompt, model="", cli="pi", timeout=TIMEOUT_S):
with tempfile.TemporaryDirectory(prefix="te-side-") as cwd:
for extra in (["--no-mcp"], []):
cmd = [cli, *PI_FLAGS, *extra, *model_args, "--", prompt]
- out = subprocess.run(cmd, capture_output=True, text=True, timeout=timeout,
- env=_env(), cwd=cwd, stdin=subprocess.DEVNULL)
+ out = subprocess.run(cmd, capture_output=True, encoding="utf-8", errors="replace",
+ timeout=timeout, env=_env(), cwd=cwd,
+ stdin=subprocess.DEVNULL)
if "Unknown option: --no-mcp" not in (out.stderr or ""):
break
text, usage = trace.pi_json_reply(out.stdout)
@@ -243,8 +246,8 @@ def copilot_call(prompt, model="", cli="copilot", timeout=TIMEOUT_S):
if model:
cmd += ["--model", model]
with tempfile.TemporaryDirectory(prefix="te-side-") as cwd:
- out = subprocess.run(cmd, capture_output=True, text=True, timeout=timeout,
- env=_env(), cwd=cwd, stdin=subprocess.DEVNULL)
+ out = subprocess.run(cmd, capture_output=True, encoding="utf-8", errors="replace",
+ timeout=timeout, env=_env(), cwd=cwd, stdin=subprocess.DEVNULL)
text, sid = "", None
for line in (out.stdout or "").splitlines():
try:
diff --git a/tokeneconomizer/plugin/store.py b/tokeneconomizer/plugin/store.py
index 9acf686..4d02002 100644
--- a/tokeneconomizer/plugin/store.py
+++ b/tokeneconomizer/plugin/store.py
@@ -81,7 +81,7 @@ class Store:
if fcntl is None:
yield True
return
- fh = open(self.path(f".{name}.lock"), "a+")
+ fh = open(self.path(f".{name}.lock"), "a+", encoding="utf-8")
got = False
deadline = time.time() + timeout
try:
diff --git a/tokeneconomizer/plugin/toolcheck.py b/tokeneconomizer/plugin/toolcheck.py
index 319d9e3..9aff29e 100644
--- a/tokeneconomizer/plugin/toolcheck.py
+++ b/tokeneconomizer/plugin/toolcheck.py
@@ -41,8 +41,8 @@ def verify_codex(repo_root, capabilities, cli="codex", timeout=30):
if not feats or not shutil.which(cli):
return None
try:
- out = subprocess.run([cli, "features", "list"], capture_output=True, text=True,
- cwd=repo_root, timeout=timeout,
+ out = subprocess.run([cli, "features", "list"], capture_output=True,
+ encoding="utf-8", errors="replace", cwd=repo_root, timeout=timeout,
env=dict(os.environ, TE_SIDECALL="1"))
except (OSError, subprocess.SubprocessError):
return None
@@ -62,7 +62,8 @@ def _init_tools(extra_args, cli="claude", timeout=60):
"--model", "haiku", "--max-turns", "1"] + claude_flags(cli) + list(extra_args)
with tempfile.TemporaryDirectory(prefix="te-probe-") as cwd:
p = subprocess.Popen(cmd, stdin=subprocess.DEVNULL, stdout=subprocess.PIPE,
- stderr=subprocess.DEVNULL, text=True, env=env, cwd=cwd)
+ stderr=subprocess.DEVNULL, encoding="utf-8", errors="replace", env=env,
+ cwd=cwd)
tools, version = None, None
deadline = time.time() + timeout
try:
@@ -148,8 +149,9 @@ def copilot_capture(extra_args=(), cwd=None, cli="copilot", timeout=90):
cwd = tmp
try:
subprocess.run([cli, "-p", "Reply with OK", "--no-auto-update", "--no-color"]
- + list(extra_args), cwd=cwd, env=env, capture_output=True, text=True,
- timeout=timeout, stdin=subprocess.DEVNULL)
+ + list(extra_args), cwd=cwd, env=env, capture_output=True,
+ encoding="utf-8", errors="replace", timeout=timeout,
+ stdin=subprocess.DEVNULL)
except (OSError, subprocess.SubprocessError):
pass
finally:
@@ -167,7 +169,8 @@ def _request_tools(body):
def _copilot_version(cli="copilot"):
try:
- out = subprocess.run([cli, "--version"], capture_output=True, text=True, timeout=30)
+ out = subprocess.run([cli, "--version"], capture_output=True,
+ encoding="utf-8", errors="replace", timeout=30)
return (out.stdout or "").strip().splitlines()[0]
except (OSError, subprocess.SubprocessError, IndexError):
return None
@@ -200,8 +203,8 @@ def _prompt_tokens(extra_args, cli="claude", timeout=180):
cmd = [cli, "-p", "Reply with exactly: OK", "--output-format", "json", "--model", "haiku",
"--max-turns", "1"] + claude_flags(cli) + list(extra_args)
with tempfile.TemporaryDirectory(prefix="te-measure-") as cwd:
- out = subprocess.run(cmd, capture_output=True, text=True, env=env, cwd=cwd,
- timeout=timeout)
+ out = subprocess.run(cmd, capture_output=True, encoding="utf-8", errors="replace", env=env,
+ cwd=cwd, timeout=timeout)
data = json.loads(out.stdout or "{}")
u = data.get("usage") or {}
return (sum(int(u.get(k) or 0) for k in ("input_tokens", "cache_read_input_tokens",
@@ -214,8 +217,8 @@ def _codex_prompt_chars(overrides, cli="codex", timeout=60):
for table, key, value in overrides:
cmd += ["-c", f"{table}.{key}={value}" if table else f"{key}={value}"]
with tempfile.TemporaryDirectory(prefix="te-measure-") as cwd:
- out = subprocess.run(cmd + ["OK"], capture_output=True, text=True, cwd=cwd,
- timeout=timeout, env=dict(os.environ, TE_SIDECALL="1"))
+ out = subprocess.run(cmd + ["OK"], capture_output=True, encoding="utf-8", errors="replace",
+ cwd=cwd, timeout=timeout, env=dict(os.environ, TE_SIDECALL="1"))
return len(out.stdout or "") if out.returncode == 0 else None
diff --git a/tokeneconomizer/trace/calls.py b/tokeneconomizer/trace/calls.py
index 8130c8d..b8ce150 100644
--- a/tokeneconomizer/trace/calls.py
+++ b/tokeneconomizer/trace/calls.py
@@ -38,7 +38,7 @@ def load_trajectory(path):
envelopes carrying `tool_use` blocks, and anything else is read as Copilot events."""
if not os.path.exists(path):
return []
- lines = [l.strip() for l in open(path) if l.strip()]
+ lines = [l.strip() for l in open(path, encoding="utf-8", errors="replace") if l.strip()]
if _is_codex_exec(lines):
return codex_exec_calls(lines)
if any('"tool_use"' in line or '"type": "assistant"' in line or '"type":"assistant"' in line
@@ -57,7 +57,7 @@ def call_cwds(path):
calls = load_trajectory(path)
if not calls or not os.path.exists(path):
return [None] * len(calls)
- lines = [l.strip() for l in open(path) if l.strip()]
+ lines = [l.strip() for l in open(path, encoding="utf-8", errors="replace") if l.strip()]
if _is_codex_exec(lines):
return [None] * len(calls)
cwds = [c[4] for c in claude_calls(lines)]