Skip to content

armgit / avibe

Public Imported from GitHub
0

Make your computer a home for your AI partner. Avibe is the local-first Agent OS: Vibey, its built-in AI partner, works out of the box, and Claude Code, Codex & OpenCode run unchanged beside it — reach them from your browser, phone, or any chat app.

fix(agent-core): byte-exact edits, one atomic write path, wrapper-owned deadlines

cyh1cb25fc4780d
diff --git a/core/agent_core/tools/bash.py b/core/agent_core/tools/bash.py
index 2198ae637..8cc2923d1 100644
--- a/core/agent_core/tools/bash.py
+++ b/core/agent_core/tools/bash.py
@@ -118,7 +118,7 @@ def handover_result(output: JobOutput, watch_id: str) -> ToolResult:
"",
]
if truncation is None:
- lines.append(f"Full output: {output.path}")
+ lines.append(output.where())
lines += [f"Check: vibe watch show {watch_id}", f"Stop: vibe watch remove {watch_id}"]
return text_result("\n".join(lines), details={**output.details(truncation), "watch_id": watch_id})
@@ -161,6 +161,8 @@ class BashTool:
async def execute(self, arguments: Mapping[str, Any], ctx: ToolContext) -> ToolResult:
try:
command = str_arg(arguments, "command")
+ if "\x00" in command:
+ raise ToolInputError("Invalid command: contains a NUL byte")
timeout = _timeout_arg(arguments)
watch = optional_bool_arg(arguments, "watch")
except ToolInputError as exc:
diff --git a/core/agent_core/tools/edit.py b/core/agent_core/tools/edit.py
index 0d661e4ec..9244a4ac8 100644
--- a/core/agent_core/tools/edit.py
+++ b/core/agent_core/tools/edit.py
@@ -12,23 +12,16 @@ import asyncio
import errno
import json
import os
+import re
from typing import Any, Mapping
from core.agent_core.tools.args import ToolInputError, error_result, os_error_text, str_arg, text_result
from core.agent_core.tools.base import ToolContext, ToolResult, ToolSpec
-from core.agent_core.tools.edit_diff import (
- Edit,
- EditError,
- apply_edits_to_normalized_content,
- detect_line_ending,
- generate_diff_string,
- generate_unified_patch,
- normalize_to_lf,
- restore_line_endings,
- split_bom,
-)
+from core.agent_core.tools.edit_diff import Edit, EditError, apply_edits, generate_diff_string, generate_unified_patch
from core.agent_core.tools.paths import file_mutation_lock, resolve_to_cwd
-from core.agent_core.tools.write import write_text
+from core.agent_core.tools.write import write_bytes
+
+_SURROGATE = re.compile("[\ud800-\udfff]")
EDIT_DESCRIPTION = (
"Edit a single file using exact text replacement. Every edits[].oldText must match a unique, non-overlapping "
@@ -115,19 +108,20 @@ def _edits_arg(arguments: Mapping[str, Any]) -> list[Edit]:
replace_all = False
if not isinstance(replace_all, bool):
raise ToolInputError(f"edits[{index}].replaceAll must be a boolean")
- parsed.append(Edit(old_text, new_text, replace_all))
+ # A lone surrogate (possible from JSON) cannot be written; it becomes U+FFFD.
+ parsed.append(Edit(_SURROGATE.sub("\ufffd", old_text), _SURROGATE.sub("\ufffd", new_text), replace_all))
return parsed
-def _plan_edits(absolute: str, edits: list[Edit], path: str) -> tuple[str, str, str, str]:
- """Read the file and compute the edited content: ``(bom, line_ending, before, after)``, LF-normalized."""
+def _plan_edits(absolute: str, edits: list[Edit], path: str) -> tuple[bytes, str, str]:
+ """Read the file and compute its new bytes, with the LF views before and after for the diff.
+
+ ``surrogateescape`` carries bytes that are not UTF-8 through unchanged; only replaced spans change.
+ """
with open(absolute, "rb") as handle:
- raw = handle.read().decode("utf-8", "replace")
- # The model never includes an invisible BOM in oldText.
- bom, content = split_bom(raw)
- ending = detect_line_ending(content)
- base = normalize_to_lf(content)
- return bom, ending, base, apply_edits_to_normalized_content(base, edits, path)
+ text = handle.read().decode("utf-8", "surrogateescape")
+ new_text, before, after = apply_edits(text, edits, path)
+ return new_text.encode("utf-8", "surrogateescape"), before, after
class EditTool:
@@ -154,10 +148,10 @@ class EditTool:
code = errno.errorcode.get(errno.ENOENT if not os.path.exists(absolute) else errno.EACCES, "EACCES")
return error_result(f"Could not edit file: {path}. Error code: {code}.")
try:
- bom, ending, base, new_content = await asyncio.to_thread(_plan_edits, absolute, edits, path)
+ data, base, new_content = await asyncio.to_thread(_plan_edits, absolute, edits, path)
if ctx.cancel.cancelled:
return error_result("Operation aborted")
- await asyncio.to_thread(write_text, absolute, bom + restore_line_endings(new_content, ending))
+ await asyncio.to_thread(write_bytes, absolute, data)
except EditError as exc:
return error_result(str(exc))
except OSError as exc:
diff --git a/core/agent_core/tools/edit_diff.py b/core/agent_core/tools/edit_diff.py
index c836c0932..f3c4bd02f 100644
--- a/core/agent_core/tools/edit_diff.py
+++ b/core/agent_core/tools/edit_diff.py
@@ -7,8 +7,9 @@ original, overlapping edits rejected. Avibe adds ``replaceAll`` (from Claude
Code and OpenCode): every occurrence in the tier that matched, each one a span
for the overlap check. Avibe departures from Pi: each edit matches in its own
tier, so one normalized edit no longer switches the whole batch to normalized
-text; uniqueness is counted in the tier that matched; classic Mac CR line
-endings are restored like CRLF.
+text; uniqueness is counted in the tier that matched; and the file is never
+re-encoded: replacements are spliced into the original text, so its BOM, its
+line breaks (mixed or not), and every byte outside a replaced span survive.
"""
from __future__ import annotations
@@ -23,6 +24,7 @@ from typing import Callable, Optional
BOM = "\ufeff"
# JavaScript's String.prototype.trimEnd set, which Pi uses (Python's isspace differs).
+_BREAK = re.compile(r"\r\n|\r|\n")
_TRAILING_SPACE = re.compile("[ \t\v\f\r\u00a0\u1680\u2000-\u200a\u2028\u2029\u202f\u205f\u3000\ufeff]+$")
_SMART_SINGLE = re.compile("[\u2018\u2019\u201a\u201b]")
_SMART_DOUBLE = re.compile("[\u201c\u201d\u201e\u201f]")
@@ -49,26 +51,10 @@ class _Replacement:
new_text: str
-def split_bom(content: str) -> tuple[str, str]:
- return (BOM, content[1:]) if content.startswith(BOM) else ("", content)
-
-
-def detect_line_ending(content: str) -> str:
- """The style of the first line break. Pi knows CRLF and LF; Avibe also keeps classic Mac CR."""
- cr, lf = content.find("\r"), content.find("\n")
- if cr == -1 or (lf != -1 and lf < cr):
- return "\n"
- return "\r\n" if cr + 1 == lf else "\r"
-
-
def normalize_to_lf(text: str) -> str:
return text.replace("\r\n", "\n").replace("\r", "\n")
-def restore_line_endings(text: str, ending: str) -> str:
- return text if ending == "\n" else text.replace("\n", ending)
-
-
def normalize_for_fuzzy_match(text: str) -> str:
"""NFKC, no trailing whitespace per line, ASCII quotes and dashes, plain spaces."""
text = unicodedata.normalize("NFKC", text)
@@ -172,49 +158,116 @@ def _check_disjoint(path: str, replacements: list[_Replacement]) -> None:
raise _overlap(path, previous.edit_index, current.edit_index)
-def _line_groups(content: str, normalized: str, replacements: list[_Replacement]) -> list[_Replacement]:
- """Normalized-tier replacements as whole-line replacements in original coordinates.
+class _Lines:
+ """The file as lines with their own breaks, and an LF view whose offsets map back to the file.
- Normalization keeps every line break, so line ``i`` of the normalized text is line ``i`` of the
- original. Replacements sharing lines form one group; its lines are rewritten from the
- normalized text with the group's replacements applied (Pi's way of applying a normalized match).
+ Matching happens in the view, as Pi matches LF-normalized text; replacements are spliced into
+ the original, so a BOM, every line break, and every byte outside a replaced span stay as they were.
"""
- original_starts, normalized_starts = _line_starts(content), _line_starts(normalized)
- if len(original_starts) != len(normalized_starts):
+
+ def __init__(self, text: str) -> None:
+ self.text = text
+ prefix = len(BOM) if text.startswith(BOM) else 0
+ self.contents: list[str] = []
+ self.breaks: list[str] = []
+ pos = prefix
+ for match in _BREAK.finditer(text, prefix):
+ self.contents.append(text[pos : match.start()])
+ self.breaks.append(match.group())
+ pos = match.end()
+ self.contents.append(text[pos:])
+ self.breaks.append("")
+ self.view = "\n".join(self.contents)
+ self.view_starts: list[int] = []
+ self.starts: list[int] = []
+ view_pos, pos = 0, prefix
+ for content, brk in zip(self.contents, self.breaks):
+ self.view_starts.append(view_pos)
+ self.starts.append(pos)
+ view_pos += len(content) + 1
+ pos += len(content) + len(brk)
+
+ def to_original(self, view_offset: int) -> int:
+ line = bisect.bisect_right(self.view_starts, view_offset) - 1
+ return self.starts[line] + (view_offset - self.view_starts[line])
+
+ def line_end(self, line: int) -> int:
+ """Original offset just past ``line`` and its break."""
+ return self.starts[line + 1] if line + 1 < len(self.starts) else len(self.text)
+
+ def break_for(self, lo: int, hi: int) -> str:
+ """The break new text uses in ``[lo, hi)``: one inside it, else the next one, else the file's first."""
+ inside = _BREAK.search(self.text, lo, hi)
+ if inside:
+ return inside.group()
+ following = _BREAK.search(self.text, hi)
+ if following:
+ return following.group()
+ return next((brk for brk in self.breaks if brk), "\n")
+
+ def with_breaks(self, lf_text: str, lo: int, hi: int, own: Optional[list[str]] = None) -> str:
+ """``lf_text`` with each ``\\n`` turned into ``own``'s next break, then the span's break."""
+ pieces = lf_text.split("\n")
+ fallback = self.break_for(lo, hi)
+ out = [pieces[0]]
+ for index, piece in enumerate(pieces[1:]):
+ brk = own[index] if own is not None and index < len(own) and own[index] else fallback
+ out.append(brk + piece)
+ return "".join(out)
+
+
+def _exact_replacements(lines: _Lines, matches: list[_Replacement]) -> list[_Replacement]:
+ out = []
+ for match in matches:
+ lo, hi = lines.to_original(match.index), lines.to_original(match.index + match.length)
+ out.append(_Replacement(match.edit_index, lo, hi - lo, lines.with_breaks(match.new_text, lo, hi)))
+ return out
+
+
+def _line_groups(lines: _Lines, normalized: str, matches: list[_Replacement]) -> list[_Replacement]:
+ """Normalized-tier matches as whole-line replacements of the original.
+
+ Normalization keeps every line break, so line ``i`` of the normalized view is line ``i`` of the
+ file. Matches sharing lines form one group; its lines are rewritten from the normalized view with
+ the group's matches applied (Pi's way of applying a normalized match), and each rewritten line
+ keeps its own break.
+ """
+ normalized_starts = _line_starts(normalized)
+ if len(normalized_starts) != len(lines.starts):
raise EditError("Cannot preserve unchanged lines because the base content has a different line count.")
def line_of(index: int) -> int:
return bisect.bisect_right(normalized_starts, index) - 1
groups: list[list] = [] # [first_line, last_line, members]
- for replacement in sorted(replacements, key=lambda r: r.index):
- first, last = line_of(replacement.index), line_of(replacement.index + replacement.length - 1)
+ for match in sorted(matches, key=lambda r: r.index):
+ first, last = line_of(match.index), line_of(match.index + match.length - 1)
if groups and first <= groups[-1][1]:
groups[-1][1] = max(groups[-1][1], last)
- groups[-1][2].append(replacement)
+ groups[-1][2].append(match)
else:
- groups.append([first, last, [replacement]])
-
- def bounds(starts: list[int], text: str, first: int, last: int) -> tuple[int, int]:
- return starts[first], starts[last + 1] if last + 1 < len(starts) else len(text)
+ groups.append([first, last, [match]])
out: list[_Replacement] = []
for first, last, members in groups:
- lo, hi = bounds(normalized_starts, normalized, first, last)
- original_lo, original_hi = bounds(original_starts, content, first, last)
+ lo = normalized_starts[first]
+ hi = normalized_starts[last + 1] if last + 1 < len(normalized_starts) else len(normalized)
rewritten = _apply(normalized[lo:hi], members, lo)
- out.append(_Replacement(members[0].edit_index, original_lo, original_hi - original_lo, rewritten))
+ original_lo, original_hi = lines.starts[first], lines.line_end(last)
+ text = lines.with_breaks(rewritten, original_lo, original_hi, lines.breaks[first : last + 1])
+ out.append(_Replacement(members[0].edit_index, original_lo, original_hi - original_lo, text))
return out
-def apply_edits_to_normalized_content(content: str, edits: list[Edit], path: str) -> str:
- """Apply every edit to LF-normalized ``content`` and return the new content.
+def apply_edits(text: str, edits: list[Edit], path: str) -> tuple[str, str, str]:
+ """Apply every edit to the file's ``text``; return the new text and the LF views before and after.
- Each edit matches in its own tier against the original. Exact edits replace exactly the text
- they matched; a normalized edit rewrites the whole lines it touches from the normalized text.
- Uniqueness, ``replaceAll``, overlap, and application are all per edit in original coordinates,
- so nothing an exact edit did not match is touched. (Pi matches a whole batch in normalized text
- as soon as one edit needs it.)
+ Each edit matches in its own tier against the original, in the file's LF view. Exact edits replace
+ exactly the text they matched; a normalized edit rewrites the whole lines it touches from the
+ normalized view. Uniqueness, ``replaceAll``, overlap, and application are all per edit in original
+ coordinates, and replacements are spliced into the original, so nothing an edit did not match
+ changes: not a byte, not a line break. (Pi matches a whole batch in normalized text as soon as one
+ edit needs it, and re-encodes every line break in the file.)
"""
edits = [Edit(normalize_to_lf(e.old_text), normalize_to_lf(e.new_text), e.replace_all) for e in edits]
total = len(edits)
@@ -222,17 +275,18 @@ def apply_edits_to_normalized_content(content: str, edits: list[Edit], path: str
if not edit.old_text:
raise _empty(path, index, total)
+ lines = _Lines(text)
cache: list[str] = []
def normalized() -> str:
if not cache:
- cache.append(normalize_for_fuzzy_match(content))
+ cache.append(normalize_for_fuzzy_match(lines.view))
return cache[0]
exact: list[_Replacement] = []
fuzzy: list[_Replacement] = []
for index, edit in enumerate(edits):
- used_normalized, spans = _occurrences(content, normalized, edit.old_text)
+ used_normalized, spans = _occurrences(lines.view, normalized, edit.old_text)
if not spans:
raise _not_found(path, index, total)
if not edit.replace_all and len(spans) > 1:
@@ -243,12 +297,12 @@ def apply_edits_to_normalized_content(content: str, edits: list[Edit], path: str
)
_check_disjoint(path, fuzzy) # in normalized coordinates, before they are grouped into lines
- replacements = exact + (_line_groups(content, normalized(), fuzzy) if fuzzy else [])
+ replacements = _exact_replacements(lines, exact) + (_line_groups(lines, normalized(), fuzzy) if fuzzy else [])
_check_disjoint(path, replacements)
- new_content = _apply(content, replacements)
- if new_content == content:
+ new_text = _apply(text, replacements)
+ if new_text == text:
raise _no_change(path, total)
- return new_content
+ return new_text, lines.view, _Lines(new_text).view
def generate_unified_patch(path: str, old: str, new: str, context: int = 4) -> str:
diff --git a/core/agent_core/tools/job_wrapper.py b/core/agent_core/tools/job_wrapper.py
index 30053994d..9e8a0262c 100644
--- a/core/agent_core/tools/job_wrapper.py
+++ b/core/agent_core/tools/job_wrapper.py
@@ -1,8 +1,9 @@
"""The process that runs one job's command (``LocalJobHost``'s ``pipe`` backend).
Run as ``python -I -S job_wrapper.py avibe-job <job_dir> <shell> <command>
-<decision_timeout_s> <head_cap> <tail_cap>`` in a new session, so its pid is
-the job's process group. Standard library only: ``-S`` skips site-packages.
+<decision_timeout_s> <head_cap> <tail_cap> <deadline_epoch_s or ->`` in a new
+session, so its pid is the job's process group. Standard library only: ``-S``
+skips site-packages.
1. Write ``pid`` (temporary file, rename).
2. Wait for ``decision``, at most ``decision_timeout_s``; on timeout, decide
@@ -16,18 +17,26 @@ the job's process group. Standard library only: ``-S`` skips site-packages.
5. When the shell exits, read what it wrote, then write ``exit`` (temporary
file, rename). Keep reading until every holder of the pipe closes it, so
background children are not killed by ``SIGPIPE``.
+6. If the deadline passes before the shell exits, or the wrapper itself fails,
+ write ``stopped`` (``timeout`` or ``wrapper_error``) and end the whole
+ process group: SIGTERM, then SIGKILL after 3 s.
"""
from __future__ import annotations
+import contextlib
import os
import select
+import signal
import subprocess
import sys
import time
+import traceback
+from typing import Optional
_SNAPSHOT_INTERVAL_S = 0.25
_DRAIN_LIMIT_S = 0.5
+_TERMINATE_GRACE_S = 3.0
def _write_atomic(path: str, data: bytes) -> None:
@@ -95,10 +104,29 @@ class _BoundedLog:
self._last_snapshot = now
-def _run(job_dir: str, shell: str, command: str, log: _BoundedLog) -> None:
- proc = subprocess.Popen(
- [shell, "-c", command], stdin=subprocess.DEVNULL, stdout=subprocess.PIPE, stderr=subprocess.STDOUT
- )
+def _stop_group(job_dir: str, reason: str, proc: Optional[subprocess.Popen]) -> None:
+ """Record why, then end the whole process group, this wrapper included; never returns.
+
+ SIGTERM first, which the wrapper ignores for itself; SIGKILL for everything left after 3 s, so
+ background children of the command end too.
+ """
+ stopped = os.path.join(job_dir, "stopped")
+ try:
+ if not os.path.exists(stopped):
+ _write_atomic(stopped, reason.encode() + b"\n")
+ except OSError:
+ pass # the kill matters more than the record
+ group = os.getpgrp()
+ signal.signal(signal.SIGTERM, signal.SIG_IGN)
+ with contextlib.suppress(OSError):
+ os.killpg(group, signal.SIGTERM)
+ deadline = time.monotonic() + _TERMINATE_GRACE_S
+ while proc is not None and proc.poll() is None and time.monotonic() < deadline:
+ time.sleep(0.02)
+ os.killpg(group, signal.SIGKILL)
+
+
+def _run(job_dir: str, proc: subprocess.Popen, log: _BoundedLog, deadline: Optional[float]) -> None:
fd = proc.stdout.fileno()
exit_written = False
eof = False
@@ -108,6 +136,9 @@ def _run(job_dir: str, shell: str, command: str, log: _BoundedLog) -> None:
_write_atomic(os.path.join(job_dir, "exit"), b"%d\n" % (code if code >= 0 else 128 - code))
while not eof:
+ if not exit_written and deadline is not None and time.time() >= deadline:
+ # J3: the deadline holds even when no host is there to enforce it.
+ _stop_group(job_dir, "timeout", proc)
ready, _, _ = select.select([fd], [], [], 0.05)
if ready:
chunk = os.read(fd, 65536)
@@ -117,8 +148,8 @@ def _run(job_dir: str, shell: str, command: str, log: _BoundedLog) -> None:
eof = True
if not exit_written and proc.poll() is not None:
# Everything the shell wrote is in the pipe by now; take it before reporting the exit.
- deadline = time.monotonic() + _DRAIN_LIMIT_S
- while not eof and time.monotonic() < deadline and select.select([fd], [], [], 0)[0]:
+ drain_until = time.monotonic() + _DRAIN_LIMIT_S
+ while not eof and time.monotonic() < drain_until and select.select([fd], [], [], 0)[0]:
chunk = os.read(fd, 65536)
if chunk:
log.write(chunk)
@@ -135,14 +166,26 @@ def _run(job_dir: str, shell: str, command: str, log: _BoundedLog) -> None:
def main(argv: list[str]) -> int:
- if len(argv) != 8 or argv[1] != "avibe-job":
+ if len(argv) != 9 or argv[1] != "avibe-job":
return 64
job_dir, shell, command = argv[2], argv[3], argv[4]
decision_timeout_s, head_cap, tail_cap = float(argv[5]), int(argv[6]), int(argv[7])
+ deadline = None if argv[8] == "-" else float(argv[8])
_write_atomic(os.path.join(job_dir, "pid"), b"%d\n" % os.getpid())
if _wait_for_decision(job_dir, decision_timeout_s) != "go":
return 0
- _run(job_dir, shell, command, _BoundedLog(job_dir, head_cap, tail_cap))
+ proc: Optional[subprocess.Popen] = None
+ try:
+ log = _BoundedLog(job_dir, head_cap, tail_cap)
+ proc = subprocess.Popen(
+ [shell, "-c", command], stdin=subprocess.DEVNULL, stdout=subprocess.PIPE, stderr=subprocess.STDOUT
+ )
+ _run(job_dir, proc, log, deadline)
+ except BaseException:
+ # A wrapper that cannot follow its command (a full disk, a bug) must not leave it running unmanaged.
+ traceback.print_exc()
+ sys.stderr.flush()
+ _stop_group(job_dir, "wrapper_error", proc)
return 0
diff --git a/core/agent_core/tools/jobs.py b/core/agent_core/tools/jobs.py
index 2313affd7..75bad0bae 100644
--- a/core/agent_core/tools/jobs.py
+++ b/core/agent_core/tools/jobs.py
@@ -24,8 +24,9 @@ inherited ``AVIBE_PROCESS_IDENTITY`` marker is set and fingerprinted as well,
but macOS hides the environment of its own binaries (``/bin/bash``,
``/bin/sleep``), so the marker only verifies a group whose wrapper has exited.
-J3: ``meta.json.deadline_at`` is absolute; ``wait`` and ``enforce_deadline``
-kill the tree once it passes, whoever holds the job.
+J3: ``meta.json.deadline_at`` is absolute. The wrapper enforces it itself, so
+it holds while no host runs; ``wait`` and ``enforce_deadline`` are a second,
+idempotent owner. Whoever kills records ``stopped`` first.
POSIX only in v1.
"""
@@ -246,6 +247,7 @@ class LocalJobHost:
os.makedirs(self._jobs_dir, exist_ok=True)
os.mkdir(job_dir, 0o700)
created = datetime.now(timezone.utc)
+ deadline = _ceil_ms(created + timedelta(seconds=timeout_s)) if timeout_s is not None else None
meta: dict[str, Any] = {
"version": 1,
"job_id": job_id,
@@ -260,7 +262,7 @@ class LocalJobHost:
"process": None,
"terminal": None,
"watch_id": None,
- "deadline_at": _iso(_ceil_ms(created + timedelta(seconds=timeout_s))) if timeout_s is not None else None,
+ "deadline_at": _iso(deadline) if deadline is not None else None,
}
self._write_meta(job_id, meta)
@@ -279,6 +281,7 @@ class LocalJobHost:
repr(DECISION_TIMEOUT_S),
str(OUTPUT_HEAD_BYTES),
str(OUTPUT_TAIL_BYTES),
+ repr(deadline.timestamp()) if deadline is not None else "-",
]
open(self.output_path(job_id), "ab").close()
diagnostics = os.open(self._path(job_id, "wrapper.log"), os.O_WRONLY | os.O_CREAT | os.O_APPEND, 0o600)
diff --git a/core/agent_core/tools/output.py b/core/agent_core/tools/output.py
index 5d63e89b0..c5e512063 100644
--- a/core/agent_core/tools/output.py
+++ b/core/agent_core/tools/output.py
@@ -141,11 +141,7 @@ class JobOutput:
text = truncation.content or empty_text
if not truncation.truncated and not self._omitted:
return text, None
- # C-7 section 5: a log bounded on disk (J4) is never called the full output.
- label = "Full output"
- if self._omitted:
- label = f"Output log (middle omitted beyond {format_size(job_host.OUTPUT_HEAD_BYTES + job_host.OUTPUT_TAIL_BYTES)})"
- where = f"{label}: {self.path}"
+ where = self.where()
start = truncation.total_lines - truncation.output_lines + 1
end = truncation.total_lines
if not truncation.truncated:
@@ -167,6 +163,14 @@ class JobOutput:
notice += f"\n[{format_size(self._omitted)} were omitted; the end of the log is in tail.log beside it.]"
return f"{text}\n\n{notice}", truncation
+ def where(self) -> str:
+ """Where the log is. ``Full output`` only for a finished job whose log was never bounded on disk (J4):
+ a running job's log may still pass the cap."""
+ if self._finished and not self._omitted:
+ return f"Full output: {self.path}"
+ cap = format_size(job_host.OUTPUT_HEAD_BYTES + job_host.OUTPUT_TAIL_BYTES)
+ return f"Output log (middle omitted beyond {cap}): {self.path}"
+
def details(self, truncation: Optional[TruncationResult]) -> dict[str, Any]:
out: dict[str, Any] = {"job_id": self._job_id, "output_path": self.path}
if self._omitted:
diff --git a/core/agent_core/tools/read.py b/core/agent_core/tools/read.py
index c701d821e..ca59a1402 100644
--- a/core/agent_core/tools/read.py
+++ b/core/agent_core/tools/read.py
@@ -43,7 +43,7 @@ _PNG_SIGNATURE = b"\x89PNG\r\n\x1a\n"
_READ_CHUNK_BYTES = 1024 * 1024
READ_DESCRIPTION = (
- "Read the contents of a file. Supports text files and images (jpg, png, gif, webp, bmp). Images are sent as "
+ "Read the contents of a file. Supports text files and images (jpg, png, gif, webp). Images are sent as "
f"attachments. For text files, output is truncated to {MAX_LINES} lines or {MAX_BYTES // 1024}KB (whichever is "
"hit first). Use offset/limit for large files. When you need the full file, continue with offset until complete."
)
@@ -132,6 +132,9 @@ def _sniff_image(path: str) -> Optional[str]:
with open(path, "rb") as handle:
head = handle.read(_IMAGE_SNIFF_BYTES)
supported = detect_supported_image_mime_type(head)
+ if supported == "image/png" and _png_file_is_animated(path):
+ # acTL may sit past the sniffed prefix, behind large ancillary chunks.
+ return "image/apng"
if supported is not None:
return supported
if head.startswith(_PNG_SIGNATURE):
@@ -267,6 +270,25 @@ def detect_supported_image_mime_type(data: bytes) -> Optional[str]:
return None
+def _png_file_is_animated(path: str) -> bool:
+ """Walk the chunk headers until IDAT or acTL, seeking over chunk bodies; bounded by the file size."""
+ size = os.path.getsize(path)
+ with open(path, "rb") as handle:
+ offset = len(_PNG_SIGNATURE)
+ while offset + 8 <= size:
+ handle.seek(offset)
+ header = handle.read(8)
+ if len(header) < 8:
+ return False
+ length, kind = struct.unpack(">I", header[:4])[0], header[4:]
+ if kind == b"acTL":
+ return True
+ if kind == b"IDAT":
+ return False
+ offset += 12 + length
+ return False
+
+
def _is_png(data: bytes) -> bool:
return len(data) >= 16 and struct.unpack(">I", data[8:12])[0] == 13 and data[12:16] == b"IHDR"
diff --git a/core/agent_core/tools/write.py b/core/agent_core/tools/write.py
index 519cbe107..3de84a560 100644
--- a/core/agent_core/tools/write.py
+++ b/core/agent_core/tools/write.py
@@ -9,9 +9,9 @@ from __future__ import annotations
import asyncio
import contextlib
import os
+import secrets
import stat
-import tempfile
-from typing import Any, Mapping
+from typing import Any, Mapping, Optional
from core.agent_core.tools.args import ToolInputError, error_result, os_error_text, str_arg, text_result
from core.agent_core.tools.base import ToolContext, ToolResult, ToolSpec
@@ -63,37 +63,41 @@ class WriteTool:
def write_text(path: str, content: str) -> None:
- """UTF-8 without newline translation; a lone surrogate (possible from JSON) is replaced.
+ """UTF-8 without newline translation; a lone surrogate (possible from JSON) is replaced."""
+ write_bytes(path, content.encode("utf-8", "replace"))
- An existing file is replaced only once the new content is completely on disk, so a failed write
- (a full disk, a quota) leaves the original intact. The replacement goes through symlinks and keeps
- the file's mode.
+
+def write_bytes(path: str, data: bytes) -> None:
+ """Create or replace ``path`` only once ``data`` is completely on disk.
+
+ A failed write (a full disk, a quota) leaves the original, or its absence, as it was. The temp
+ file sits beside the real target (through symlinks); it takes the existing file's mode, or the
+ umask default for a new file, is fsynced, and replaces the target with ``os.replace``.
"""
target = os.path.realpath(path)
try:
- mode = stat.S_IMODE(os.stat(target).st_mode)
+ mode: Optional[int] = stat.S_IMODE(os.stat(target).st_mode)
except FileNotFoundError:
- _write_file(target, content)
- return
+ mode = None
+ tmp = os.path.join(os.path.dirname(target), f".{os.path.basename(target)}.{secrets.token_hex(4)}.tmp")
try:
- fd, tmp = tempfile.mkstemp(dir=os.path.dirname(target), prefix=f".{os.path.basename(target)}.", suffix=".tmp")
+ fd = os.open(tmp, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o666)
except PermissionError:
- # A writable file in a directory we cannot add to: only an in-place write is possible.
- _write_file(target, content)
+ if mode is None:
+ raise
+ # An existing, writable file in a directory we cannot add to: only an in-place write is possible.
+ with open(target, "wb") as handle:
+ handle.write(data)
return
try:
- with os.fdopen(fd, "w", encoding="utf-8", errors="replace", newline="") as handle:
- handle.write(content)
+ with os.fdopen(fd, "wb") as handle:
+ handle.write(data)
handle.flush()
os.fsync(handle.fileno())
- os.chmod(tmp, mode)
+ if mode is not None:
+ os.chmod(tmp, mode)
os.replace(tmp, target)
except BaseException:
with contextlib.suppress(FileNotFoundError):
os.unlink(tmp)
raise
-
-
-def _write_file(path: str, content: str) -> None:
- with open(path, "w", encoding="utf-8", errors="replace", newline="") as handle:
- handle.write(content)
diff --git a/tests/agent_core/tools/test_bash.py b/tests/agent_core/tools/test_bash.py
index e6a4f0df0..b5db2f3c3 100644
--- a/tests/agent_core/tools/test_bash.py
+++ b/tests/agent_core/tools/test_bash.py
@@ -78,6 +78,13 @@ async def test_an_out_of_range_timeout_is_an_error_result(tmp_path, make_ctx):
assert result_text(result) == "Invalid timeout: must be a finite number of seconds"
+async def test_a_nul_byte_in_the_command_is_an_error_result(tmp_path, make_ctx):
+ result = await BashTool(_host(tmp_path)).execute({"command": "echo a\x00b"}, make_ctx())
+
+ assert result.is_error
+ assert result_text(result) == "Invalid command: contains a NUL byte"
+
+
async def test_stdin_is_closed(tmp_path, make_ctx):
started = time.monotonic()
result = await BashTool(_host(tmp_path)).execute({"command": 'read x; echo "rc=$?"'}, make_ctx())
@@ -129,7 +136,10 @@ async def test_the_foreground_window_hands_over_and_the_command_runs_once(tmp_pa
assert text.startswith(
"Command is still running and is now Watch wch_1. You will get a follow-up message when it finishes.\n\n"
)
- assert text.endswith(f"\n\nFull output: {path}\nCheck: vibe watch show wch_1\nStop: vibe watch remove wch_1")
+ # A running job's log may still pass the on-disk cap, so it is never called the full output (J4).
+ assert text.endswith(
+ f"\n\nOutput log (middle omitted beyond 3.0MB): {path}\nCheck: vibe watch show wch_1\nStop: vibe watch remove wch_1"
+ )
assert (await host.wait(job_id, deadline_s=5)).exit_code == 0
assert open(path).read() == "working\nfinished\n"
assert (tmp_path / "counter").read_text() == "run\n"
diff --git a/tests/agent_core/tools/test_jobs.py b/tests/agent_core/tools/test_jobs.py
index f68651bb6..27ed52edf 100644
--- a/tests/agent_core/tools/test_jobs.py
+++ b/tests/agent_core/tools/test_jobs.py
@@ -202,6 +202,39 @@ async def test_output_on_disk_keeps_the_head_and_tail_of_a_long_command(tmp_path
assert b"".join(chunks).endswith(b"99999\n100000\n")
+async def test_the_wrapper_enforces_the_deadline_while_no_host_runs(tmp_path):
+ """J3: the host crashed right after the start; the deadline still ends the tree and is recorded."""
+ host = LocalJobHost(str(tmp_path / "jobs"))
+ job_id = await _start(host, tmp_path, f"echo $$ > {tmp_path / 'sh.pid'}; sleep 30 & sleep 30", timeout_s=0.5)
+ shell_pid = None
+ assert _wait_until(lambda: (tmp_path / "sh.pid").exists())
+ shell_pid = int((tmp_path / "sh.pid").read_text())
+ del host # nobody waits on the job from here on
+
+ assert _wait_until(lambda: _wrapper_gone(str(tmp_path / "jobs" / job_id)), timeout_s=6)
+ restarted = LocalJobHost(str(tmp_path / "jobs"))
+ assert restarted.status(job_id).state == "gone"
+ assert restarted.stop_reason(job_id) == "timeout"
+ assert not psutil.pid_exists(shell_pid) or psutil.Process(shell_pid).status() == psutil.STATUS_ZOMBIE
+
+
+async def test_a_wrapper_that_cannot_keep_the_log_stops_its_command(tmp_path, monkeypatch):
+ monkeypatch.setattr(jobs_module, "OUTPUT_HEAD_BYTES", 10)
+ host = LocalJobHost(str(tmp_path / "jobs"))
+ job_id = await _start(host, tmp_path, f"echo $$ > {tmp_path / 'sh.pid'}; sleep 0.5; seq 1 1000; sleep 30")
+ # The tail cannot be written once the head is full, as on a full disk.
+ os.mkdir(os.path.join(host.job_dir(job_id), "tail.log"))
+
+ status = await asyncio.wait_for(host.wait(job_id, deadline_s=None), timeout=10)
+
+ assert status.state == "gone"
+ assert host.stop_reason(job_id) == "wrapper_error"
+ shell_pid = int((tmp_path / "sh.pid").read_text())
+ assert _wait_until(
+ lambda: not psutil.pid_exists(shell_pid) or psutil.Process(shell_pid).status() == psutil.STATUS_ZOMBIE
+ )
+
+
async def test_job_files_stay_until_the_call_is_settled(tmp_path):
"""J5."""
host = LocalJobHost(str(tmp_path / "jobs"))
diff --git a/tests/agent_core/tools/test_read.py b/tests/agent_core/tools/test_read.py
index 5fd918381..f72dd45bc 100644
--- a/tests/agent_core/tools/test_read.py
+++ b/tests/agent_core/tools/test_read.py
@@ -140,8 +140,10 @@ def _chunk(kind: bytes, body: bytes) -> bytes:
return struct.pack(">I", len(body)) + kind + body + struct.pack(">I", zlib.crc32(kind + body))
-def _png(*, animated: bool = False) -> bytes:
+def _png(*, animated: bool = False, padding: int = 0) -> bytes:
ihdr = _chunk(b"IHDR", struct.pack(">IIBBBBB", 1, 1, 8, 2, 0, 0, 0))
+ if padding:
+ ihdr += _chunk(b"tEXt", b"Comment\0" + b"x" * padding)
actl = _chunk(b"acTL", struct.pack(">II", 1, 0)) if animated else b""
idat = _chunk(b"IDAT", zlib.compress(b"\0\xff\0\0"))
return b"\x89PNG\r\n\x1a\n" + ihdr + actl + idat + _chunk(b"IEND", b"")
@@ -180,6 +182,13 @@ async def test_an_image_is_stored_through_the_sink_and_attached(tmp_path, make_c
None,
"Read image file [image/apng]\n[Image omitted: could not be converted to a supported inline image format.]",
),
+ (
+ # acTL behind a large ancillary chunk, past the sniffed prefix.
+ "late.png",
+ _png(animated=True, padding=10_000),
+ None,
+ "Read image file [image/apng]\n[Image omitted: could not be converted to a supported inline image format.]",
+ ),
(
"pic.png",
_png(),
@@ -188,6 +197,7 @@ async def test_an_image_is_stored_through_the_sink_and_attached(tmp_path, make_c
"Images are not resized.]",
),
],
+ ids=["bmp", "animated", "late-actl", "oversized"],
)
async def test_images_that_cannot_be_sent_as_they_are_are_omitted(
tmp_path, make_ctx, monkeypatch, name, data, cap, expected
diff --git a/tests/agent_core/tools/test_write_edit.py b/tests/agent_core/tools/test_write_edit.py
index 930734a78..feb31aa1d 100644
--- a/tests/agent_core/tools/test_write_edit.py
+++ b/tests/agent_core/tools/test_write_edit.py
@@ -3,6 +3,8 @@
from __future__ import annotations
import errno
+import random
+import re
import stat
import pytest
@@ -24,8 +26,10 @@ async def test_write_creates_parent_directories_and_keeps_bytes(tmp_path, make_c
assert (tmp_path / "a/b/notes.txt").read_bytes() == "第一行\r\nsecond\n".encode()
-async def test_a_failed_write_leaves_the_original_intact(tmp_path, make_ctx, monkeypatch):
- (tmp_path / "f.txt").write_text("original\n")
[email protected]("existing", [True, False])
+async def test_a_failed_write_leaves_the_original_intact(tmp_path, make_ctx, monkeypatch, existing):
+ if existing:
+ (tmp_path / "f.txt").write_text("original\n")
def disk_full(fd):
raise OSError(errno.ENOSPC, "No space left on device")
@@ -34,8 +38,9 @@ async def test_a_failed_write_leaves_the_original_intact(tmp_path, make_ctx, mon
result = await WriteTool().execute({"path": "f.txt", "content": "replacement\n"}, make_ctx())
assert result.is_error
- assert (tmp_path / "f.txt").read_text() == "original\n"
- assert sorted(p.name for p in tmp_path.iterdir()) == ["f.txt"]
+ if existing:
+ assert (tmp_path / "f.txt").read_text() == "original\n"
+ assert sorted(p.name for p in tmp_path.iterdir()) == (["f.txt"] if existing else [])
async def test_write_goes_through_a_symlink_and_keeps_the_mode(tmp_path, make_ctx):
@@ -162,6 +167,45 @@ async def test_each_edit_matches_in_its_own_tier(tmp_path, make_ctx, original, e
assert (tmp_path / "f.txt").read_text() == expected
+_BREAKS = ["\r\n", "\r", "\n"]
+_WORDS = ["alpha", "beta", "gamma", "x = 1", " ", "\u201cq\u201d", "\u00e9"]
+
+
+def _random_file(rng: random.Random) -> list[tuple[str, str]]:
+ """Lines with their own breaks, as the file reads back: ``"\\r"`` then ``"\\n"`` is one CRLF."""
+ body = (
+ "".join(" ".join(rng.choice(_WORDS) for _ in range(rng.randint(0, 3))) + rng.choice(_BREAKS) for _ in range(7))
+ + " ".join(rng.choice(_WORDS) for _ in range(rng.randint(0, 3)))
+ + rng.choice(["", *_BREAKS])
+ )
+ parts = re.split(r"(\r\n|\r|\n)", body)
+ return list(zip(parts[0::2], [*parts[1::2], ""]))
+
+
+async def test_an_edit_never_changes_bytes_outside_the_lines_it_replaces(tmp_path, make_ctx):
+ """Mixed line breaks, a BOM, and an invalid byte survive every edit that does not cover them."""
+ rng = random.Random(11)
+ path = tmp_path / "f.txt"
+ for _ in range(300):
+ lines = _random_file(rng)
+ view = "\n".join(content for content, _ in lines) + ("\n" if lines[-1][1] else "")
+ first, last = sorted(rng.sample(range(len(lines)), 2))
+ old = "\n".join(content for content, _ in lines[first : last + 1])
+ if not old.strip() or view.count(old) != 1:
+ continue
+ prefix = "".join(c + b for c, b in lines[:first])
+ suffix = lines[last][1] + "".join(c + b for c, b in lines[last + 1 :])
+ # The replaced lines in the file keep their own breaks; oldText uses LF, as the model sends it.
+ middle = "".join(c + b for c, b in lines[first:last]) + lines[last][0]
+ raw = b"\xef\xbb\xbf\xff" + (prefix + middle + suffix).encode()
+ path.write_bytes(raw)
+
+ result = await _edit(make_ctx, "f.txt", {"oldText": old, "newText": "NEW"})
+
+ assert not result.is_error, (result_text(result), lines, old)
+ assert path.read_bytes() == b"\xef\xbb\xbf\xff" + (prefix + "NEW" + suffix).encode(), (lines, old)
+
+
@pytest.mark.parametrize(
("edits", "message"),
[