Make your computer a home for your AI partner. Avibe is the local-first Agent OS: Vibey, its built-in AI partner, works out of the box, and Claude Code, Codex & OpenCode run unchanged beside it — reach them from your browser, phone, or any chat app.
Python agentagent-osaiai-agentsai-partnerchatopsclaudeclaude-codecodexdevtoolsdiscord-botlark-botllmlocal-firstlong-term-memoryopencodeslack-botvibe-codingvibeywechat
fix(agent-core): one shared decode, exclusive stop reasons, strict file URLs
cyhbccb08319956
diff --git a/core/agent_core/tools/edit.py b/core/agent_core/tools/edit.pyindex adbed37de..e75466f63 100644--- a/core/agent_core/tools/edit.py+++ b/core/agent_core/tools/edit.py@@ -19,6 +19,7 @@ from core.agent_core.tools.args import ToolInputError, error_result, os_error_tefrom core.agent_core.tools.base import ToolContext, ToolResult, ToolSpecfrom core.agent_core.tools.edit_diff import Edit, EditError, apply_edits, generate_diff_string, generate_unified_patchfrom core.agent_core.tools.paths import file_mutation_lock, resolve_to_cwd+from core.agent_core.tools.text import decode_file, encode_filefrom core.agent_core.tools.truncate import format_sizefrom core.agent_core.tools.write import write_bytes@@ -124,9 +125,9 @@ def _plan_edits(absolute: str, edits: list[Edit], path: str) -> tuple[bytes, str``surrogateescape`` carries bytes that are not UTF-8 through unchanged; only replaced spans change."""with open(absolute, "rb") as handle:- text = handle.read().decode("utf-8", "surrogateescape")+ text = decode_file(handle.read())new_text, before, after = apply_edits(text, edits, path)- return new_text.encode("utf-8", "surrogateescape"), before, after+ return encode_file(new_text), before, afterclass EditTool:@@ -161,6 +162,12 @@ class EditTool:"Use bash (for example sed or a short script) to change files this large.")data, base, new_content = await asyncio.to_thread(_plan_edits, absolute, edits, path)+ if len(data) > MAX_EDIT_BYTES:+ return error_result(+ f"File {path} would be {format_size(len(data))} after this edit, over the "+ f"{format_size(MAX_EDIT_BYTES)} edit limit. "+ "Use bash (for example sed or a short script) to change files this large."+ )if ctx.cancel.cancelled:return error_result("Operation aborted")await asyncio.to_thread(write_bytes, absolute, data)@@ -170,7 +177,7 @@ class EditTool:return error_result(os_error_text(exc))result = f"Successfully replaced {len(edits)} block(s) in {path}."- if size > MAX_DIFF_BYTES:+ if max(size, len(data)) > MAX_DIFF_BYTES:# The diff is for display only, and difflib is superlinear on large inputs.return text_result(result, details={"diff_skipped": True})diff, first_changed_line = await asyncio.to_thread(generate_diff_string, base, new_content)diff --git a/core/agent_core/tools/edit_diff.py b/core/agent_core/tools/edit_diff.pyindex f3c4bd02f..7210a6d5a 100644--- a/core/agent_core/tools/edit_diff.py+++ b/core/agent_core/tools/edit_diff.py@@ -21,6 +21,8 @@ import unicodedatafrom dataclasses import dataclassfrom typing import Callable, Optional+from core.agent_core.tools.text import shown+BOM = "\ufeff"# JavaScript's String.prototype.trimEnd set, which Pi uses (Python's isspace differs).@@ -177,7 +179,8 @@ class _Lines:pos = match.end()self.contents.append(text[pos:])self.breaks.append("")- self.view = "\n".join(self.contents)+ # What read showed (one U+FFFD per undecodable byte), one character for one, so offsets map back.+ self.view = shown("\n".join(self.contents))self.view_starts: list[int] = []self.starts: list[int] = []view_pos, pos = 0, prefixdiff --git a/core/agent_core/tools/job_wrapper.py b/core/agent_core/tools/job_wrapper.pyindex 9e8a0262c..b4b9c3587 100644--- a/core/agent_core/tools/job_wrapper.py+++ b/core/agent_core/tools/job_wrapper.py@@ -46,21 +46,26 @@ def _write_atomic(path: str, data: bytes) -> None:os.replace(tmp, path)+def _create_once(path: str, value: str) -> None:+ """Create ``path`` holding ``value`` unless it exists: ``link(2)`` of a written temp file, as the host does."""+ tmp = f"{path}.{value}.{os.getpid()}.tmp"+ with open(tmp, "w") as handle:+ handle.write(f"{value}\n")+ try:+ os.link(tmp, path)+ except FileExistsError:+ pass+ finally:+ os.unlink(tmp)++def _wait_for_decision(job_dir: str, timeout_s: float) -> str:decision = os.path.join(job_dir, "decision")deadline = time.monotonic() + timeout_sdelay = 0.001while not os.path.exists(decision):if time.monotonic() >= deadline:- tmp = f"{decision}.abandon.{os.getpid()}.tmp"- with open(tmp, "w") as handle:- handle.write("abandon\n")- try:- os.link(tmp, decision)- except FileExistsError:- pass- finally:- os.unlink(tmp)+ _create_once(decision, "abandon")breaktime.sleep(delay)delay = min(delay * 2, 0.05)@@ -110,10 +115,8 @@ def _stop_group(job_dir: str, reason: str, proc: Optional[subprocess.Popen]) ->SIGTERM first, which the wrapper ignores for itself; SIGKILL for everything left after 3 s, sobackground children of the command end too."""- stopped = os.path.join(job_dir, "stopped")try:- if not os.path.exists(stopped):- _write_atomic(stopped, reason.encode() + b"\n")+ _create_once(os.path.join(job_dir, "stopped"), reason) # the first stopper's reason winsexcept OSError:pass # the kill matters more than the recordgroup = os.getpgrp()@@ -176,6 +179,9 @@ def main(argv: list[str]) -> int:return 0proc: Optional[subprocess.Popen] = Nonetry:+ if deadline is not None and time.time() >= deadline:+ # The deadline passed during the launch: the command must not start at all.+ _stop_group(job_dir, "timeout", None)log = _BoundedLog(job_dir, head_cap, tail_cap)proc = subprocess.Popen([shell, "-c", command], stdin=subprocess.DEVNULL, stdout=subprocess.PIPE, stderr=subprocess.STDOUTdiff --git a/core/agent_core/tools/jobs.py b/core/agent_core/tools/jobs.pyindex 0b37aa486..6d3c0b19e 100644--- a/core/agent_core/tools/jobs.py+++ b/core/agent_core/tools/jobs.py@@ -197,19 +197,27 @@ class LocalJobHost:# --- decision (J1) ----------------------------------------------------- def _create_decision(self, job_id: str, value: str) -> str:- """Try to decide; return the decision that holds, ours or the wrapper's."""- decision = self._path(job_id, "decision")- tmp = f"{decision}.{value}.{secrets.token_hex(4)}.tmp"+ def _create_once(self, job_id: str, name: str, value: str) -> str:+ """Create ``name`` holding ``value`` unless it exists; return what it holds, ours or the earlier writer's.++ ``link(2)`` of a written temporary file: exclusive, and never visible half-written.+ """+ target = self._path(job_id, name)+ tmp = f"{target}.{value}.{secrets.token_hex(4)}.tmp"with open(tmp, "w", encoding="utf-8") as handle:handle.write(f"{value}\n")try:- os.link(tmp, decision)+ os.link(tmp, target)except FileExistsError:passfinally:os.unlink(tmp)- return _read_text(decision) or _ABANDON+ return _read_text(target) or value++ def _create_decision(self, job_id: str, value: str) -> str:+ """Try to decide; return the decision that holds, ours or the wrapper's."""+ decision = self._create_once(job_id, "decision", value)+ return decision if decision in (_GO, _ABANDON) else _ABANDONdef _decision(self, job_id: str) -> str:"""The job's decision; a job without one is abandoned now, as recovery requires."""@@ -406,8 +414,7 @@ class LocalJobHost:if _group_exists(identity.pid):logger.warning("Job %s process group %s cannot be verified; not signaling", job_id, identity.pid)return- if _read_text(self._path(job_id, "stopped")) is None:- _write_atomic(self._path(job_id, "stopped"), f"{reason}\n")+ self._create_once(job_id, "stopped", reason) # the first stopper's reason winsif not await asyncio.to_thread(_terminate_group, identity.pid):logger.warning("Job %s process group %s survived termination", job_id, identity.pid)diff --git a/core/agent_core/tools/normalize.py b/core/agent_core/tools/normalize.pyindex 79e8c0c96..48f37e23a 100644--- a/core/agent_core/tools/normalize.py+++ b/core/agent_core/tools/normalize.py@@ -15,6 +15,8 @@ from __future__ import annotationsimport codecsimport re+from core.agent_core.tools.truncate import utf8_len+_ANSI_RE = re.compile(r"\x1b\][^\x07\x1b]*(?:\x07|\x1b\\)" # OSC ... BEL or STr"|\x1b[P^_][^\x1b]*\x1b\\" # DCS, PM, APC ... ST@@ -22,9 +24,7 @@ _ANSI_RE = re.compile(r"|\x1b[ -/]*[0-~]" # other escape sequencesr"|\x9b[0-?]*[ -/]*[@-~]" # 8-bit CSI)-# An escape sequence that may still be incomplete at the end of a chunk.-_HOLD_BACK_CHARS = 64-# An open line (no newline yet) is committed once its current segment grows past this.+# An open line (no newline yet) keeps at most this many characters of its last segment._MAX_OPEN_CHARS = 64 * 1024@@ -32,27 +32,23 @@ def strip_ansi(text: str) -> str:return _ANSI_RE.sub("", text).replace("\x1b", "")-def _collapse(segments: list[str]) -> str:- """The last segment that still shows something once escape sequences are gone."""- for segment in reversed(segments):- visible = strip_ansi(segment)- if visible:- return visible- return ""--class OutputNormalizer:"""Incremental normalizer: feed raw bytes, receive normalized text.- ``feed`` returns only text that is final (complete lines, or an over-long- open line); ``peek`` shows the open line as it would read now.+ ``feed`` returns only complete lines, which a later carriage return can no+ longer redraw; ``peek`` shows the open line as it would read now. An open+ line keeps at most its last ``_MAX_OPEN_CHARS`` characters, so memory stays+ bounded; if such a line is final, it starts with a marker saying how many+ bytes were dropped."""def __init__(self) -> None:self._decoder = codecs.getincrementaldecoder("utf-8")(errors="replace")self._carry_cr = False- self._prev = "" # last non-empty finished segment of the open line- self._cur = "" # segment after the last carriage return of the open line+ # The open line: its last segment that still shows something, and the segment after the+ # last carriage return, each with the UTF-8 bytes dropped from its start.+ self._prev, self._prev_dropped = "", 0+ self._cur, self._cur_dropped = "", 0def feed(self, data: bytes) -> str:return self._consume(self._decoder.decode(data))@@ -62,12 +58,12 @@ class OutputNormalizer:if self._carry_cr:self._carry_cr = Falseself._open_segments("\r")- tail = _collapse([self._prev, self._cur])- self._prev = self._cur = ""+ tail = self._line()+ self._reset()return out + taildef peek(self) -> str:- return _collapse([self._prev, self._cur])+ return self._line()def _consume(self, text: str) -> str:if self._carry_cr:@@ -85,31 +81,37 @@ class OutputNormalizer:if line.endswith("\r"):line = line[:-1]self._open_segments(line)- out.append(_collapse([self._prev, self._cur]) + "\n")- self._prev = self._cur = ""+ out.append(self._line() + "\n")+ self._reset()self._open_segments(lines[-1])- if len(self._cur) > _MAX_OPEN_CHARS:- out.append(self._commit_open())return "".join(out)def _open_segments(self, text: str) -> None:segments = (self._cur + text).split("\r")if len(segments) > 1:- for segment in reversed(segments[:-1]):- if strip_ansi(segment):- self._prev = segment+ # The first piece continues the current segment, so it keeps that segment's drop count.+ for index in range(len(segments) - 2, -1, -1):+ if strip_ansi(segments[index]):+ self._prev = segments[index]+ self._prev_dropped = self._cur_dropped if index == 0 else 0break+ self._cur_dropped = 0self._cur = segments[-1]-- def _commit_open(self) -> str:- # Bound memory on a line that never ends; keep a possibly incomplete escape.- cut = len(self._cur)- escape = self._cur.rfind("\x1b", max(0, cut - _HOLD_BACK_CHARS))- if escape != -1 and not _ANSI_RE.match(self._cur, escape):- cut = escape- committed, self._cur = self._cur[:cut], self._cur[cut:]- self._prev = ""- return strip_ansi(committed)+ if len(self._cur) > _MAX_OPEN_CHARS:+ cut = len(self._cur) - _MAX_OPEN_CHARS+ self._cur_dropped += utf8_len(self._cur[:cut])+ self._cur = self._cur[cut:]++ def _line(self) -> str:+ for segment, dropped in ((self._cur, self._cur_dropped), (self._prev, self._prev_dropped)):+ visible = strip_ansi(segment)+ if visible:+ return f"[... {dropped} bytes omitted ...]{visible}" if dropped else visible+ return ""++ def _reset(self) -> None:+ self._prev, self._prev_dropped = "", 0+ self._cur, self._cur_dropped = "", 0def normalize_output(data: bytes) -> str:diff --git a/core/agent_core/tools/paths.py b/core/agent_core/tools/paths.pyindex 44d5db038..e09895a8d 100644--- a/core/agent_core/tools/paths.py+++ b/core/agent_core/tools/paths.py@@ -23,6 +23,7 @@ _AM_PM = re.compile(r" (AM|PM)\.", re.IGNORECASE)_WINDOWS = os.name == "nt"_ENCODED_SEPARATOR = re.compile("%2f", re.IGNORECASE)_ENCODED_SEPARATOR_WINDOWS = re.compile("%2f|%5c", re.IGNORECASE)+_MALFORMED_ESCAPE = re.compile("%(?![0-9a-fA-F]{2})")def expand_path(path: str) -> str:@@ -49,7 +50,7 @@ def _file_url_to_path(url: str) -> str:if _WINDOWS:if _ENCODED_SEPARATOR_WINDOWS.search(parsed.path):raise ToolInputError("Invalid path: a file URL must not include encoded \\ or / characters")- path = unquote(parsed.path).replace("/", "\\")+ path = _decode_url_path(parsed.path).replace("/", "\\")if host:return f"\\\\{host}{path}"if len(path) >= 3 and path[0] == "\\" and path[1].isascii() and path[1].isalpha() and path[2] == ":":@@ -59,7 +60,17 @@ def _file_url_to_path(url: str) -> str:raise ToolInputError("Invalid path: file URL host must be empty or localhost")if _ENCODED_SEPARATOR.search(parsed.path):raise ToolInputError("Invalid path: a file URL must not include encoded / characters")- return unquote(parsed.path)+ return _decode_url_path(parsed.path)+++def _decode_url_path(path: str) -> str:+ """``decodeURIComponent``: a malformed escape or bytes that are not UTF-8 are an error, never a guess."""+ if _MALFORMED_ESCAPE.search(path):+ raise ToolInputError("Invalid path: a file URL must use valid percent-encoded UTF-8")+ try:+ return unquote(path, errors="strict")+ except UnicodeDecodeError:+ raise ToolInputError("Invalid path: a file URL must use valid percent-encoded UTF-8") from Nonedef resolve_to_cwd(path: str, cwd: str) -> str:diff --git a/core/agent_core/tools/read.py b/core/agent_core/tools/read.pyindex ca59a1402..c53232a5a 100644--- a/core/agent_core/tools/read.py+++ b/core/agent_core/tools/read.py@@ -30,6 +30,7 @@ from core.agent_core.tools.args import ()from core.agent_core.tools.base import MAX_BYTES, MAX_LINES, ToolContext, ToolResult, ToolSpecfrom core.agent_core.tools.paths import resolve_read_path+from core.agent_core.tools.text import decode_file, shownfrom core.agent_core.tools.truncate import format_size, truncate_head#: Stores an image for the transcript and returns its media token: ``(data, mime_type, name)``.@@ -252,7 +253,7 @@ def _scan_lines(if collecting and opened != index and index >= start:# An empty file: its one line is empty.lines.append(bytearray())- return [bytes(line).decode("utf-8", "replace") for line in lines], index + 1, first_line_bytes+ return [shown(decode_file(bytes(line))) for line in lines], index + 1, first_line_bytesdef detect_supported_image_mime_type(data: bytes) -> Optional[str]:diff --git a/core/agent_core/tools/text.py b/core/agent_core/tools/text.pynew file mode 100644index 000000000..ee0fb7a1c--- /dev/null+++ b/core/agent_core/tools/text.py@@ -0,0 +1,26 @@+"""How file bytes become the text the model sees, shared by ``read`` and ``edit``.++Bytes are decoded with ``surrogateescape``, so every byte that is not UTF-8 survives as one escape+character and can be written back unchanged. The model sees each such byte as one U+FFFD. The+mapping is one character for one character, so ``edit`` can match what ``read`` showed and still+splice into the original bytes. (Pi, through the WHATWG decoder, shows one U+FFFD per invalid+sequence; Avibe shows one per invalid byte.)+"""++from __future__ import annotations++_SHOWN = {code: "\ufffd" for code in range(0xDC80, 0xDD00)}+++def decode_file(data: bytes) -> str:+ """The file's text with undecodable bytes kept as surrogate escapes."""+ return data.decode("utf-8", "surrogateescape")+++def shown(text: str) -> str:+ """``text`` as the model sees it: each surrogate escape becomes U+FFFD."""+ return text.translate(_SHOWN)+++def encode_file(text: str) -> bytes:+ return text.encode("utf-8", "surrogateescape")diff --git a/tests/agent_core/tools/test_jobs.py b/tests/agent_core/tools/test_jobs.pyindex 48d3f9ba8..4a2c6c51e 100644--- a/tests/agent_core/tools/test_jobs.py+++ b/tests/agent_core/tools/test_jobs.py@@ -245,6 +245,39 @@ async def test_a_finished_wrapper_leaves_the_registry(tmp_path):assert _wait_until(lambda: job_id not in host._children, timeout_s=5)+async def test_the_first_recorded_stop_reason_wins(tmp_path, monkeypatch):+ """Two stoppers that both saw no reason yet: the second must not overwrite the first."""+ host = LocalJobHost(str(tmp_path / "jobs"))+ job_id = await _start(host, tmp_path, "sleep 30")+ stopped = os.path.join(host.job_dir(job_id), "stopped")+ with open(stopped, "w") as handle:+ handle.write("aborted\n") # the other stopper got there first+ real_read = jobs_module._read_text+ monkeypatch.setattr(jobs_module, "_read_text", lambda path: None if path == stopped else real_read(path))++ await host.kill(job_id, reason="timeout") # it checked before the other one wrote+ monkeypatch.undo()++ assert host.stop_reason(job_id) == "aborted"+++async def test_a_deadline_that_passed_during_the_launch_never_starts_the_command(tmp_path, monkeypatch):+ """The deadline is checked before the shell is spawned: a missing shell is never even tried."""+ original = LocalJobHost._await_pid++ async def slow_host(self, *args):+ identity = await original(self, *args)+ await asyncio.sleep(0.3) # the job's deadline passes during the handshake+ return identity++ monkeypatch.setattr(LocalJobHost, "_await_pid", slow_host)+ host = LocalJobHost(str(tmp_path / "jobs"), shell=str(tmp_path / "no-such-shell"))+ job_id = await _start(host, tmp_path, "true", timeout_s=0.1)++ assert _wait_until(lambda: _wrapper_gone(host.job_dir(job_id)))+ assert host.stop_reason(job_id) == "timeout"++async def test_job_files_stay_until_the_call_is_settled(tmp_path):"""J5."""host = LocalJobHost(str(tmp_path / "jobs"))diff --git a/tests/agent_core/tools/test_normalize.py b/tests/agent_core/tools/test_normalize.pyindex fb129e762..8bfef334b 100644--- a/tests/agent_core/tools/test_normalize.py+++ b/tests/agent_core/tools/test_normalize.py@@ -27,3 +27,19 @@ def test_chunks_normalize_to_the_final_screen_text(chunks, expected):out = "".join(normalizer.feed(chunk) for chunk in chunks) + normalizer.flush()assert out == expected+++def test_a_long_open_line_can_still_be_redrawn():+ normalizer = OutputNormalizer()++ out = normalizer.feed(b"x" * 70_000) + normalizer.feed(b"\rdone\n") + normalizer.flush()++ assert out == "done\n"+++def test_a_long_final_line_keeps_its_end_and_says_what_was_dropped():+ normalizer = OutputNormalizer()++ out = "".join(normalizer.feed(b"y" * 10_000) for _ in range(10)) + normalizer.feed(b"\n") + normalizer.flush()++ assert out == f"[... {100_000 - 65_536} bytes omitted ...]" + "y" * 65_536 + "\n"diff --git a/tests/agent_core/tools/test_write_edit.py b/tests/agent_core/tools/test_write_edit.pyindex 5e39e0365..778d0705f 100644--- a/tests/agent_core/tools/test_write_edit.py+++ b/tests/agent_core/tools/test_write_edit.py@@ -100,6 +100,35 @@ async def test_replace_all_replaces_every_occurrence(tmp_path, make_ctx):assert (tmp_path / "f.txt").read_text() == "baz(1)\nqux\nbaz(2)\nbaz(3)\n"+async def test_text_copied_from_read_edits_a_file_with_invalid_bytes(tmp_path, make_ctx):+ """read and edit show undecodable bytes the same way, so a block copied from read matches."""+ (tmp_path / "f.txt").write_bytes(b"a\xffb\xe2\x82c\nkeep\x80\n")+ shown = result_text(await ReadTool().execute({"path": "f.txt"}, make_ctx()))+ first_line = shown.split("\n")[0]++ result = await _edit(make_ctx, "f.txt", {"oldText": first_line, "newText": "fixed"})++ assert not result.is_error, result_text(result)+ assert (tmp_path / "f.txt").read_bytes() == b"fixed\nkeep\x80\n"+++async def test_a_planned_result_over_the_limit_is_refused_and_its_diff_skipped(tmp_path, make_ctx, monkeypatch):+ (tmp_path / "small.txt").write_text("seed\n")+ monkeypatch.setattr(edit_module, "MAX_EDIT_BYTES", 1000)+ monkeypatch.setattr(edit_module, "MAX_DIFF_BYTES", 400)++ grown = await _edit(make_ctx, "small.txt", {"oldText": "seed", "newText": "x" * 600})+ too_big = await _edit(make_ctx, "small.txt", {"oldText": "x" * 600, "newText": "y" * 2000})++ assert grown.details == {"diff_skipped": True}+ assert too_big.is_error+ assert result_text(too_big) == (+ "File small.txt would be 2.0KB after this edit, over the 1000B edit limit. "+ "Use bash (for example sed or a short script) to change files this large."+ )+ assert (tmp_path / "small.txt").read_text() == "x" * 600 + "\n"++async def test_files_over_the_edit_limit_are_refused_and_large_diffs_skipped(tmp_path, make_ctx, monkeypatch):(tmp_path / "big.log").write_text("line\n" * 100)monkeypatch.setattr(edit_module, "MAX_EDIT_BYTES", 400)@@ -314,6 +343,9 @@ async def test_a_path_no_file_can_have_is_an_error_result(make_ctx, tool, argume(False, "file://localhost/tmp/x", "/tmp/x"),(False, "file://server/tmp/x", ToolInputError("Invalid path: file URL host must be empty or localhost")),(False, "file:///tmp/a%2Fb", ToolInputError("Invalid path: a file URL must not include encoded / characters")),+ (False, "file:///tmp/%FF", ToolInputError("Invalid path: a file URL must use valid percent-encoded UTF-8")),+ (False, "file:///tmp/%ZZ", ToolInputError("Invalid path: a file URL must use valid percent-encoded UTF-8")),+ (True, "file:///C:/%E2%82", ToolInputError("Invalid path: a file URL must use valid percent-encoded UTF-8")),(True, "file:///C:/tmp/a%20b.txt", "C:\\tmp\\a b.txt"),(True, "file://localhost/C:/x", "C:\\x"),(True, "file://server/share/x.txt", "\\\\server\\share\\x.txt"),