Skip to content

armgit / avibe

Public Imported from GitHub
0

Make your computer a home for your AI partner. Avibe is the local-first Agent OS: Vibey, its built-in AI partner, works out of the box, and Claude Code, Codex & OpenCode run unchanged beside it — reach them from your browser, phone, or any chat app.

fix(agent-core): close the pre-push review classes: linear edits, true output labels and sizes, signals to the wrapper

cyh50a463e5d27e
diff --git a/core/agent_core/tools/args.py b/core/agent_core/tools/args.py
index 656db36bd..201c247a3 100644
--- a/core/agent_core/tools/args.py
+++ b/core/agent_core/tools/args.py
@@ -3,10 +3,12 @@
from __future__ import annotations
import math
+from decimal import Decimal
from typing import Any, Mapping, Optional
from core.agent_core.messages import TextBlock
from core.agent_core.tools.base import ToolResult
+from core.agent_core.tools.text import model_text
class ToolInputError(ValueError):
@@ -22,10 +24,11 @@ def error_result(text: str, *, details: Optional[Mapping[str, Any]] = None) -> T
def str_arg(arguments: Mapping[str, Any], name: str) -> str:
+ """A string argument, through the one sanitizer for model text (a lone surrogate becomes U+FFFD)."""
value = arguments.get(name)
if not isinstance(value, str):
raise ToolInputError(f"{name} must be a string")
- return value
+ return model_text(value)
def optional_number_arg(arguments: Mapping[str, Any], name: str) -> Optional[float]:
@@ -65,5 +68,20 @@ def optional_bool_arg(arguments: Mapping[str, Any], name: str) -> bool:
def format_number(value: float) -> str:
- """``5`` for 5.0 and ``0.5`` for 0.5, as JavaScript prints numbers in Pi's messages."""
- return str(int(value)) if float(value).is_integer() else repr(float(value))
+ """JavaScript's ``String(number)``, as Pi prints numbers in its messages: ``5``, ``0.5``, ``0.00001``, ``1e-7``."""
+ value = float(value)
+ if value == 0 or not math.isfinite(value):
+ return {0.0: "0", math.inf: "Infinity", -math.inf: "-Infinity"}.get(value, "NaN")
+ sign = "-" if value < 0 else ""
+ # repr gives the shortest digits that round-trip, as JavaScript does; only the layout differs.
+ _, digit_tuple, exponent = Decimal(repr(abs(value))).normalize().as_tuple()
+ digits = "".join(map(str, digit_tuple))
+ point = exponent + len(digits) # value = 0.<digits> * 10**point
+ if len(digits) <= point <= 21:
+ return sign + digits + "0" * (point - len(digits))
+ if 0 < point <= 21:
+ return f"{sign}{digits[:point]}.{digits[point:]}"
+ if -6 < point <= 0:
+ return f"{sign}0.{'0' * -point}{digits}"
+ mantissa = digits[0] + (f".{digits[1:]}" if len(digits) > 1 else "")
+ return f"{sign}{mantissa}e{'+' if point - 1 >= 0 else '-'}{abs(point - 1)}"
diff --git a/core/agent_core/tools/edit_diff.py b/core/agent_core/tools/edit_diff.py
index 41756c03f..d21d8d4e6 100644
--- a/core/agent_core/tools/edit_diff.py
+++ b/core/agent_core/tools/edit_diff.py
@@ -27,7 +27,7 @@ BOM = "\ufeff"
# JavaScript's String.prototype.trimEnd set, which Pi uses (Python's isspace differs).
_BREAK = re.compile(r"\r\n|\r|\n")
-_TRAILING_SPACE = re.compile("[ \t\v\f\r\u00a0\u1680\u2000-\u200a\u2028\u2029\u202f\u205f\u3000\ufeff]+$")
+_TRAILING_SPACE = " \t\v\f\r\u00a0\u1680" + "".join(map(chr, range(0x2000, 0x200B))) + "\u2028\u2029\u202f\u205f\u3000\ufeff"
_SMART_SINGLE = re.compile("[\u2018\u2019\u201a\u201b]")
_SMART_DOUBLE = re.compile("[\u201c\u201d\u201e\u201f]")
_DASHES = re.compile("[\u2010\u2011\u2012\u2013\u2014\u2015\u2212]")
@@ -60,7 +60,8 @@ def normalize_to_lf(text: str) -> str:
def normalize_for_fuzzy_match(text: str) -> str:
"""NFKC, no trailing whitespace per line, ASCII quotes and dashes, plain spaces."""
text = unicodedata.normalize("NFKC", text)
- text = "\n".join(_TRAILING_SPACE.sub("", line) for line in text.split("\n"))
+ # rstrip, not a "[...]+$" regex: the regex retries at every space of a run, which is quadratic.
+ text = "\n".join(line.rstrip(_TRAILING_SPACE) for line in text.split("\n"))
text = _SMART_SINGLE.sub("'", text)
text = _SMART_DOUBLE.sub('"', text)
text = _DASHES.sub("-", text)
@@ -131,11 +132,15 @@ def _no_change(path: str, total: int) -> EditError:
def _apply(content: str, replacements: list[_Replacement], offset: int = 0) -> str:
- result = content
- for replacement in sorted(replacements, key=lambda r: r.index, reverse=True):
+ """``content`` with disjoint ``replacements`` applied, in one pass (``replaceAll`` can make thousands)."""
+ pieces: list[str] = []
+ pos = 0
+ for replacement in sorted(replacements, key=lambda r: r.index):
start = replacement.index - offset
- result = result[:start] + replacement.new_text + result[start + replacement.length :]
- return result
+ pieces += (content[pos:start], replacement.new_text)
+ pos = start + replacement.length
+ pieces.append(content[pos:])
+ return "".join(pieces)
def _line_starts(text: str) -> list[int]:
@@ -179,6 +184,7 @@ class _Lines:
pos = match.end()
self.contents.append(text[pos:])
self.breaks.append("")
+ self.first_break = next((brk for brk in self.breaks if brk), "\n")
# What read showed (one U+FFFD per undecodable byte), one character for one, so offsets map back.
self.view = shown("\n".join(self.contents))
self.view_starts: list[int] = []
@@ -198,20 +204,17 @@ class _Lines:
"""Original offset just past ``line`` and its break."""
return self.starts[line + 1] if line + 1 < len(self.starts) else len(self.text)
- def break_for(self, lo: int, hi: int) -> str:
- """The break new text uses in ``[lo, hi)``: one inside it, else the next one, else the file's first."""
- inside = _BREAK.search(self.text, lo, hi)
- if inside:
- return inside.group()
- following = _BREAK.search(self.text, hi)
- if following:
- return following.group()
- return next((brk for brk in self.breaks if brk), "\n")
-
- def with_breaks(self, lf_text: str, lo: int, hi: int, own: Optional[list[str]] = None) -> str:
- """``lf_text`` with each ``\\n`` turned into ``own``'s next break, then the span's break."""
+ def break_for(self, lo: int) -> str:
+ """The break new text uses for a span starting at ``lo``: the first break at or after it (inside the
+ span, else the one after it), which is the break of ``lo``'s line, else the file's first."""
+ return self.breaks[bisect.bisect_right(self.starts, lo) - 1] or self.first_break
+
+ def with_breaks(self, lf_text: str, lo: int, own: Optional[list[str]] = None) -> str:
+ """``lf_text`` with each ``\\n`` turned into ``own``'s next break, then the break of the span at ``lo``."""
+ if "\n" not in lf_text:
+ return lf_text
pieces = lf_text.split("\n")
- fallback = self.break_for(lo, hi)
+ fallback = self.break_for(lo)
out = [pieces[0]]
for index, piece in enumerate(pieces[1:]):
brk = own[index] if own is not None and index < len(own) and own[index] else fallback
@@ -223,7 +226,7 @@ def _exact_replacements(lines: _Lines, matches: list[_Replacement]) -> list[_Rep
out = []
for match in matches:
lo, hi = lines.to_original(match.index), lines.to_original(match.index + match.length)
- out.append(_Replacement(match.edit_index, lo, hi - lo, lines.with_breaks(match.new_text, lo, hi)))
+ out.append(_Replacement(match.edit_index, lo, hi - lo, lines.with_breaks(match.new_text, lo)))
return out
@@ -257,7 +260,7 @@ def _line_groups(lines: _Lines, normalized: str, matches: list[_Replacement]) ->
hi = normalized_starts[last + 1] if last + 1 < len(normalized_starts) else len(normalized)
rewritten = _apply(normalized[lo:hi], members, lo)
original_lo, original_hi = lines.starts[first], lines.line_end(last)
- text = lines.with_breaks(rewritten, original_lo, original_hi, lines.breaks[first : last + 1])
+ text = lines.with_breaks(rewritten, original_lo, lines.breaks[first : last + 1])
out.append(_Replacement(members[0].edit_index, original_lo, original_hi - original_lo, text))
return out
diff --git a/core/agent_core/tools/job_wrapper.py b/core/agent_core/tools/job_wrapper.py
index 3f1d18b48..83f1804eb 100644
--- a/core/agent_core/tools/job_wrapper.py
+++ b/core/agent_core/tools/job_wrapper.py
@@ -12,14 +12,17 @@ skips site-packages.
stderr into one pipe.
4. Keep the output bounded on disk: the first ``head_cap`` bytes are appended
to ``output.log`` as they arrive; after that, the last ``tail_cap`` bytes are
- kept in ``tail.log`` (a ``<offset>\\n`` header, then the bytes from that
- offset of the whole stream), replaced atomically a few times a second.
+ kept in ``tail.log`` (a ``[output from byte <offset>]\\n`` header, readable by
+ anyone who opens the file, then the bytes from that offset of the whole
+ stream), replaced atomically a few times a second.
5. When the shell exits, read what it wrote, then write ``exit`` (temporary
file, rename). Keep reading until every holder of the pipe closes it, so
background children are not killed by ``SIGPIPE``.
-6. If the deadline passes before the shell exits, or the wrapper itself fails,
- write ``stopped`` (``timeout`` or ``wrapper_error``) and end the whole
- process group: SIGTERM, then SIGKILL after 3 s.
+6. If the deadline passes before the shell exits, the wrapper itself fails, or
+ a catchable signal reaches the wrapper alone (``pkill -f python``), write
+ ``stopped`` (``timeout``, ``wrapper_error``, or ``killed``) and end the whole
+ process group: SIGTERM, then SIGKILL after 3 s. The wrapper decides timeout
+ versus exit, and it looks at the shell first.
Every wait is bounded or watches the deadline: the decision wait (30 s), the
check just before the shell is spawned, the read loop, the wait after the
@@ -41,6 +44,14 @@ from typing import Optional
_SNAPSHOT_INTERVAL_S = 0.25
_DRAIN_LIMIT_S = 0.5
_TERMINATE_GRACE_S = 3.0
+# Catchable signals that would otherwise end the wrapper alone and leave its command running
+# (``pkill -f python``, a hangup; SIGINT would surface as a failure). Each stops the whole group as
+# "killed" instead. Python already ignores SIGPIPE and SIGXFSZ; SIGKILL cannot be caught.
+_STOP_SIGNALS = tuple(
+ getattr(signal, name)
+ for name in "SIGTERM SIGHUP SIGINT SIGQUIT SIGUSR1 SIGUSR2 SIGALRM SIGVTALRM SIGPROF SIGXCPU SIGPWR SIGIO".split()
+ if hasattr(signal, name)
+)
def _write_atomic(path: str, data: bytes) -> None:
@@ -108,7 +119,7 @@ class _BoundedLog:
if not self._dirty or (not force and now - self._last_snapshot < _SNAPSHOT_INTERVAL_S):
return
kept = bytes(self._tail[-self._tail_cap :])
- _write_atomic(self._tail_path, b"%d\n" % (self._total - len(kept)) + kept)
+ _write_atomic(self._tail_path, b"[output from byte %d]\n" % (self._total - len(kept)) + kept)
self._dirty = False
self._last_snapshot = now
@@ -119,12 +130,13 @@ def _stop_group(job_dir: str, reason: str, proc: Optional[subprocess.Popen]) ->
SIGTERM first, which the wrapper ignores for itself; SIGKILL for everything left after 3 s, so
background children of the command end too.
"""
+ for signum in _STOP_SIGNALS:
+ signal.signal(signum, signal.SIG_IGN) # nothing interrupts the stop, and SIGTERM to the group spares us
try:
_create_once(os.path.join(job_dir, "stopped"), reason) # the first stopper's reason wins
except OSError:
pass # the kill matters more than the record
group = os.getpgrp()
- signal.signal(signal.SIGTERM, signal.SIG_IGN)
with contextlib.suppress(OSError):
os.killpg(group, signal.SIGTERM)
deadline = time.monotonic() + _TERMINATE_GRACE_S
@@ -186,10 +198,16 @@ def main(argv: list[str]) -> int:
job_dir, shell, command = argv[2], argv[3], argv[4]
decision_timeout_s, head_cap, tail_cap = float(argv[5]), int(argv[6]), int(argv[7])
deadline = None if argv[8] == "-" else float(argv[8])
+ proc: Optional[subprocess.Popen] = None
+
+ def stop_on_signal(signum: int, frame: object) -> None:
+ _stop_group(job_dir, "killed", proc)
+
+ for signum in _STOP_SIGNALS:
+ signal.signal(signum, stop_on_signal)
_write_atomic(os.path.join(job_dir, "pid"), b"%d\n" % os.getpid())
if _wait_for_decision(job_dir, decision_timeout_s) != "go":
return 0
- proc: Optional[subprocess.Popen] = None
try:
if deadline is not None and time.time() >= deadline:
# The deadline passed during the launch: the command must not start at all.
diff --git a/core/agent_core/tools/jobs.py b/core/agent_core/tools/jobs.py
index 41573daba..9f874ec23 100644
--- a/core/agent_core/tools/jobs.py
+++ b/core/agent_core/tools/jobs.py
@@ -34,6 +34,7 @@ POSIX only in v1.
from __future__ import annotations
import asyncio
+import contextlib
import json
import logging
import os
@@ -91,6 +92,8 @@ _WRAPPER = os.path.join(os.path.dirname(os.path.abspath(__file__)), "job_wrapper
#: 0.5 s drain, and margin) before stopping the job itself.
_WRAPPER_DECIDES_S = 1.0
_JOB_ID = re.compile(r"^job_[a-z0-9]+$")
+# tail.log starts with this line (written by the wrapper), so a reader of the file sees where it begins.
+_TAIL_HEADER = re.compile(rb"\[output from byte (\d+)\]\n")
_GO = "go"
_ABANDON = "abandon"
@@ -153,7 +156,9 @@ class LocalJobHost:
on_hand_over: Optional[HandOver] = None,
shell: Optional[str] = None,
) -> None:
- self._jobs_dir = os.path.abspath(jobs_dir)
+ # Canonical, so every host names a job's directory (and its wrapper's argv) the same way, whatever
+ # alias it was given (``~/.vibe_remote`` -> ``~/.avibe``, ``/tmp`` -> ``/private/tmp``).
+ self._jobs_dir = os.path.realpath(jobs_dir)
self._on_hand_over = on_hand_over
self._shell = shell or default_shell()
# Wrappers this host spawned. Each has a reaper thread, so a finished wrapper never lingers
@@ -214,10 +219,12 @@ class LocalJobHost:
try:
os.link(tmp, target)
except FileExistsError:
- pass
+ return _read_text(target) or value
finally:
- os.unlink(tmp)
- return _read_text(target) or value
+ with contextlib.suppress(OSError):
+ os.unlink(tmp) # a leftover temp is harmless; it goes with the job directory
+ # Linked: ours is durable, so nothing that fails after this may say otherwise.
+ return value
def _create_decision(self, job_id: str, value: str) -> str:
"""Try to decide; return the decision that holds, ours or the wrapper's."""
@@ -351,7 +358,7 @@ class LocalJobHost:
return JobStatus("exited", exit_code)
if self._decision(job_id) != _GO:
return JobStatus("gone")
- if self._alive(job_id):
+ if self._alive(job_id) or self._orphaned_group_runs(job_id):
return JobStatus("running")
# The wrapper writes `exit` before it exits, so look once more.
exit_code = self._exit_code(job_id)
@@ -395,7 +402,10 @@ class LocalJobHost:
return data, since + len(data)
try:
with open(self._path(job_id, "tail.log"), "rb") as handle:
- tail_start = int(handle.readline())
+ header = _TAIL_HEADER.fullmatch(handle.readline())
+ if header is None:
+ raise ValueError("tail.log header")
+ tail_start = int(header.group(1))
start = max(since, tail_start)
handle.seek(start - tail_start, os.SEEK_CUR)
data = handle.read(OUTPUT_CHUNK_BYTES)
@@ -483,7 +493,8 @@ class LocalJobHost:
job_dir = self.job_dir(name)
if max(entry.stat().st_mtime for entry in os.scandir(job_dir)) > cutoff:
continue
- if self.status(name).state == "running" or not settled(self.meta(name)):
+ # An exited job's wrapper may still drain children that hold the pipe; it needs the directory.
+ if self.status(name).state == "running" or self._alive(name) or not settled(self.meta(name)):
continue
except (KeyError, ValueError, OSError):
continue
@@ -526,6 +537,11 @@ class LocalJobHost:
job_dir = self.job_dir(job_id)
return any(argv[i : i + 2] == ["avibe-job", job_dir] for i in range(len(argv) - 1))
+ def _orphaned_group_runs(self, job_id: str) -> bool:
+ """The wrapper was killed on its own (SIGKILL), but its group still provably runs the job."""
+ identity = self._identity(job_id)
+ return identity is not None and _group_exists(identity.pid) and self._group_carries_marker(identity)
+
def _group_carries_marker(self, identity: PersistedProcessIdentity) -> bool:
return process_group_identity_status(identity.pid, identity, logger, "agent job") == "match"
diff --git a/core/agent_core/tools/normalize.py b/core/agent_core/tools/normalize.py
index 48f37e23a..5b7312633 100644
--- a/core/agent_core/tools/normalize.py
+++ b/core/agent_core/tools/normalize.py
@@ -49,6 +49,8 @@ class OutputNormalizer:
# last carriage return, each with the UTF-8 bytes dropped from its start.
self._prev, self._prev_dropped = "", 0
self._cur, self._cur_dropped = "", 0
+ # UTF-8 bytes of the last finished line, counting what was dropped from its start.
+ self._finished_line_bytes = 0
def feed(self, data: bytes) -> str:
return self._consume(self._decoder.decode(data))
@@ -58,13 +60,24 @@ class OutputNormalizer:
if self._carry_cr:
self._carry_cr = False
self._open_segments("\r")
- tail = self._line()
+ tail, size = self._line_and_size()
+ if tail:
+ self._finished_line_bytes = size
self._reset()
return out + tail
def peek(self) -> str:
return self._line()
+ def last_line_bytes(self) -> int:
+ """UTF-8 bytes of the last line: the open one if it shows anything, else the last finished one.
+
+ Bytes dropped from the start of a long line count (before ANSI stripping), so the size is the
+ line's, not the kept part's.
+ """
+ _, size = self._line_and_size()
+ return size or self._finished_line_bytes
+
def _consume(self, text: str) -> str:
if self._carry_cr:
text = "\r" + text
@@ -81,7 +94,8 @@ class OutputNormalizer:
if line.endswith("\r"):
line = line[:-1]
self._open_segments(line)
- out.append(self._line() + "\n")
+ text, self._finished_line_bytes = self._line_and_size()
+ out.append(text + "\n")
self._reset()
self._open_segments(lines[-1])
return "".join(out)
@@ -103,11 +117,15 @@ class OutputNormalizer:
self._cur = self._cur[cut:]
def _line(self) -> str:
+ return self._line_and_size()[0]
+
+ def _line_and_size(self) -> tuple[str, int]:
for segment, dropped in ((self._cur, self._cur_dropped), (self._prev, self._prev_dropped)):
visible = strip_ansi(segment)
if visible:
- return f"[... {dropped} bytes omitted ...]{visible}" if dropped else visible
- return ""
+ text = f"[... {dropped} bytes omitted ...]{visible}" if dropped else visible
+ return text, dropped + utf8_len(visible)
+ return "", 0
def _reset(self) -> None:
self._prev, self._prev_dropped = "", 0
diff --git a/core/agent_core/tools/output.py b/core/agent_core/tools/output.py
index b7f3678c9..82c3b4e19 100644
--- a/core/agent_core/tools/output.py
+++ b/core/agent_core/tools/output.py
@@ -30,8 +30,6 @@ class OutputAccumulator:
self._tail_starts_at_line_boundary = True
self._total_bytes = 0
self._completed_lines = 0
- # The size of the last line, open or completed: what the partial-line notice names.
- self._last_line_bytes = 0
self._has_open_line = False
def append(self, text: str) -> None:
@@ -44,21 +42,11 @@ class OutputAccumulator:
if self._tail_bytes > self._max_rolling * 2:
self._trim()
newlines = text.count("\n")
- carried = self._last_line_bytes if self._has_open_line else 0
if newlines == 0:
- self._last_line_bytes = carried + size
self._has_open_line = True
return
self._completed_lines += newlines
- last = text.rindex("\n")
- rest = text[last + 1 :]
- if rest:
- self._last_line_bytes = utf8_len(rest)
- else:
- # Avibe fix to Pi, whose count drops to 0 here: a completed last line keeps its size.
- previous = text.rfind("\n", 0, last)
- self._last_line_bytes = (carried if previous == -1 else 0) + utf8_len(text[previous + 1 : last])
- self._has_open_line = bool(rest)
+ self._has_open_line = not text.endswith("\n")
def snapshot(self, open_line: str = "") -> TruncationResult:
"""The tail as the model would see it, with ``open_line`` (not yet final) appended."""
@@ -74,12 +62,6 @@ class OutputAccumulator:
tail, truncated=truncated, truncated_by=truncated_by, total_lines=total_lines, total_bytes=total_bytes
)
- def last_line_bytes(self, open_line: str = "") -> int:
- """Bytes of the last line: the open one plus ``open_line`` (not yet final), else the last completed one."""
- if open_line:
- return (self._last_line_bytes if self._has_open_line else 0) + utf8_len(open_line)
- return self._last_line_bytes
-
def _trim(self) -> None:
data = self._tail.encode("utf-8", "surrogatepass")
if len(data) <= self._max_rolling:
@@ -158,8 +140,9 @@ class JobOutput:
if not truncation.truncated:
notice = f"[{where}]"
elif truncation.last_line_partial:
- open_line = "" if self._finished else self._normalizer.peek()
- line_size = format_size(self._accumulator.last_line_bytes(open_line))
+ # Avibe fix to Pi, whose count drops to 0 once the line ends: the normalizer knows the whole
+ # line's size, including what it dropped from the start of a line too long to keep.
+ line_size = format_size(self._normalizer.last_line_bytes())
notice = (
f"[Showing last {format_size(truncation.output_bytes)} of line {end} (line is {line_size}). {where}]"
)
@@ -170,19 +153,20 @@ class JobOutput:
f"[Showing lines {start}-{end} of {truncation.total_lines} ({format_size(MAX_BYTES)} limit). {where}]"
)
if self._omitted:
- # Only LocalJobHost drops output, and it keeps the end in tail.log next to output.log.
- notice += f"\n[{format_size(self._omitted)} were omitted; the end of the log is in tail.log beside it.]"
+ # Only LocalJobHost drops output: the middle, between output.log and tail.log.
+ notice += f"\n[{format_size(self._omitted)} between output.log and tail.log were omitted.]"
return f"{text}\n\n{notice}", truncation
def where(self) -> str:
"""Where the log is, in one wording for every state (C-7 section 5).
- The log keeps everything up to the cap and drops the middle beyond it, and a job can keep
- appending after its result was built (a background child holding the pipe), so the label states
- that policy rather than the log's current state: it can never become untrue.
+ ``output.log`` keeps the first ``OUTPUT_HEAD_BYTES``; past that, ``tail.log`` beside it keeps the
+ latest ``OUTPUT_TAIL_BYTES``. A job can keep appending after its result was built (a handed-over
+ job, a background child holding the pipe), so the label states that policy rather than the
+ log's current state: it can never become untrue.
"""
- cap = format_size(job_host.OUTPUT_HEAD_BYTES + job_host.OUTPUT_TAIL_BYTES)
- return f"Output log (middle omitted beyond {cap}): {self.path}"
+ head, tail = format_size(job_host.OUTPUT_HEAD_BYTES), format_size(job_host.OUTPUT_TAIL_BYTES)
+ return f"Output log (first {head}, then the last {tail} in tail.log beside it): {self.path}"
def details(self, truncation: Optional[TruncationResult]) -> dict[str, Any]:
out: dict[str, Any] = {"job_id": self._job_id, "output_path": self.path}
diff --git a/core/agent_core/tools/paths.py b/core/agent_core/tools/paths.py
index afb24b64a..36ea44d49 100644
--- a/core/agent_core/tools/paths.py
+++ b/core/agent_core/tools/paths.py
@@ -8,7 +8,9 @@ Mario Zechner).
from __future__ import annotations
import asyncio
+import contextlib
import errno
+import ipaddress
import os
import re
import stat
@@ -34,6 +36,8 @@ _QUERY_OR_FRAGMENT = re.compile("[?#]")
_DRIVE = re.compile(r"[A-Za-z][:|]\Z")
_NORMALIZED_DRIVE = re.compile(r"[A-Za-z]:\Z")
_FORBIDDEN_HOST = re.compile(r"[\x00-\x20#%/:<>?@\[\\\]^|\x7f]")
+_IDNA_JOINERS = re.compile("[\u1806\u200c\u200d]")
+_NUMERIC_LABEL = re.compile(r"(?:[0-9]+|0[xX][0-9a-fA-F]*)\Z")
_SINGLE_DOT = {".", "%2e"}
_DOUBLE_DOT = {"..", ".%2e", "%2e.", "%2e%2e"}
@@ -84,24 +88,53 @@ def _file_url_to_path(url: str) -> str:
def _url_host(authority: str) -> str:
- """The WHATWG host of a file URL: percent-decoded, lowercased, and empty for ``localhost``."""
+ """The WHATWG host of a file URL, lowercased, and empty for ``localhost``.
+
+ Only hosts whose WHATWG form is the text itself are kept: ASCII names, dotted-quad IPv4, and
+ IPv6 in its compressed form. A host that needs IDNA processing or address parsing to reach
+ Node's form is refused rather than guessed (Python has IDNA 2003, Node UTS #46), unless it maps
+ to ``localhost``, so a URL never names a share or a file Node would not.
+ """
if not authority:
return ""
try:
host = unquote(authority, errors="strict")
- if host.startswith("[") and host.endswith("]"):
- host = host.lower() # an IPv6 literal
- else:
- if not host.isascii() or "xn--" in host.lower():
- host = host.encode("idna").decode("idna")
- host = host.lower()
- if _FORBIDDEN_HOST.search(host):
+ except UnicodeError:
+ raise _invalid_host() from None
+ if host.startswith("[") and host.endswith("]"):
+ try:
+ canonical = ipaddress.IPv6Address(host[1:-1]).compressed
+ except ValueError:
+ raise _invalid_host() from None
+ if "." in host or canonical != host[1:-1].lower():
+ raise _invalid_host()
+ return f"[{canonical}]"
+ if not host.isascii() or "xn--" in host.lower():
+ # Node fails a zero-width joiner or non-joiner where Python's IDNA quietly drops it.
+ if not _IDNA_JOINERS.search(host):
+ with contextlib.suppress(UnicodeError):
+ if host.encode("idna").decode("ascii").lower() == "localhost":
+ return ""
+ raise _invalid_host()
+ host = host.lower()
+ if _FORBIDDEN_HOST.search(host):
+ raise _invalid_host()
+ labels = host.split(".")
+ last = labels[-1] or (labels[-2] if len(labels) > 1 else "")
+ if _NUMERIC_LABEL.match(last):
+ # WHATWG parses it as IPv4 and prints it as a dotted quad; only that form is the text itself.
+ try:
+ if str(ipaddress.IPv4Address(host)) != host:
raise ValueError(host)
- except (UnicodeError, ValueError):
- raise ToolInputError("Invalid path: a file URL must have a valid host") from None
+ except ValueError:
+ raise _invalid_host() from None
return "" if host == "localhost" else host
+def _invalid_host() -> ToolInputError:
+ return ToolInputError("Invalid path: a file URL must have a valid host")
+
+
def _url_pathname(path: str) -> str:
"""The WHATWG path of a file URL (still percent-encoded), from the text after the authority's ``/``."""
segments: list[str] = []
diff --git a/core/agent_core/tools/read.py b/core/agent_core/tools/read.py
index 53254bce8..601debf6a 100644
--- a/core/agent_core/tools/read.py
+++ b/core/agent_core/tools/read.py
@@ -12,6 +12,7 @@ loop shares with the rest of Avibe. The result is the same as Pi's.
from __future__ import annotations
import asyncio
+import codecs
import inspect
import os
import shlex
@@ -37,7 +38,7 @@ from core.agent_core.tools.paths import (
target_kind,
)
from core.agent_core.tools.text import decode_file, shown
-from core.agent_core.tools.truncate import format_size, truncate_head
+from core.agent_core.tools.truncate import format_size, truncate_head, utf8_len
#: Stores an image for the transcript and returns its media token: ``(data, mime_type, name)``.
ImageSink = Callable[[bytes, str, str], Union[str, Awaitable[str]]]
@@ -251,10 +252,11 @@ def _scan_lines(
handle: BinaryIO, start: int, stop: Optional[int], cancelled: Callable[[], bool] = lambda: False
) -> tuple[list[str], int, int]:
"""Lines ``[start, stop)`` as Pi's ``text.split("\\n")`` gives them, without the ``\\r`` of a CRLF break,
- the file's line count, and the raw size of line ``start``.
+ the file's line count, and the size of line ``start`` as it is shown.
Avibe drops the ``\\r`` (as tau does; Pi shows it) so read's text is edit's matching view: a copied line
- can no longer end in a bare ``\\r`` that edit would read as a line break.
+ can no longer end in a bare ``\\r`` that edit would read as a line break. A line ending in more than
+ one ``\\r`` keeps them all: edit reads ``\\r\\r\\n`` as two breaks, and so does the copied text.
Collection stops once the kept lines pass either cap, so the kept prefix exceeds a cap exactly when
the whole selection does: decoding with replacement never makes text shorter than its bytes, and
@@ -264,9 +266,19 @@ def _scan_lines(
kept = 0 # raw bytes of "\\n".join(lines)
opened = -1 # file line index of lines[-1]
index = 0 # file line index of the bytes being read
- first_line_bytes = 0
+ first_line_bytes = 0 # UTF-8 bytes of line ``start`` as shown
+ first_decoder = codecs.getincrementaldecoder("utf-8")("surrogateescape")
+ first_line_tail = b"" # its last two bytes, for the dropped "\r"
collecting = stop is None or start < stop
+ def measure(segment: bytes, ends_line: bool) -> None:
+ """Add a piece of line ``start`` to its shown size; ``ends_line`` once its ``\\n`` is reached."""
+ nonlocal first_line_bytes, first_line_tail
+ first_line_bytes += utf8_len(shown(first_decoder.decode(segment, final=ends_line)))
+ first_line_tail = (first_line_tail + segment)[-2:]
+ if ends_line and first_line_tail.endswith(b"\r") and not first_line_tail.endswith(b"\r\r"):
+ first_line_bytes -= 1
+
def collect(segment: bytes, ends_line: bool) -> None:
nonlocal kept, opened, collecting
if opened != index:
@@ -279,7 +291,7 @@ def _scan_lines(
if room > 0:
lines[-1] += segment[:room]
kept += min(len(segment), room)
- if ends_line and len(segment) <= room and lines[-1].endswith(b"\r"):
+ if ends_line and len(segment) <= room and lines[-1].endswith(b"\r") and not lines[-1].endswith(b"\r\r"):
del lines[-1][-1]
kept -= 1
# One line beyond the cap: a final empty piece does not count as a line.
@@ -287,10 +299,10 @@ def _scan_lines(
collecting = False
def count_rest(chunk: bytes, pos: int) -> None:
- nonlocal index, first_line_bytes
+ nonlocal index
if index == start:
newline = chunk.find(b"\n", pos)
- first_line_bytes += (len(chunk) if newline == -1 else newline) - pos
+ measure(chunk[pos : len(chunk) if newline == -1 else newline], newline != -1)
index += chunk.count(b"\n", pos)
handle.seek(0)
@@ -308,7 +320,7 @@ def _scan_lines(
collecting = False
continue
if index == start:
- first_line_bytes += end - pos
+ measure(chunk[pos:end], newline != -1)
if index >= start:
collect(chunk[pos:end], newline != -1)
if newline == -1:
@@ -318,6 +330,9 @@ def _scan_lines(
pos = newline + 1
else:
count_rest(chunk, pos)
+ if index == start:
+ # Line ``start`` is the last line and has no "\n": what the decoder still holds is shown too.
+ first_line_bytes += utf8_len(shown(first_decoder.decode(b"", final=True)))
if collecting and opened != index and index >= start:
# An empty file: its one line is empty.
lines.append(bytearray())
diff --git a/core/agent_core/tools/truncate.py b/core/agent_core/tools/truncate.py
index 6a405eb75..df8ff1ce7 100644
--- a/core/agent_core/tools/truncate.py
+++ b/core/agent_core/tools/truncate.py
@@ -10,6 +10,7 @@ Copyright (c) 2025 Mario Zechner).
from __future__ import annotations
from dataclasses import asdict, dataclass
+from decimal import ROUND_HALF_UP, Decimal
from typing import Any, Literal, Optional
from core.agent_core.tools.base import MAX_BYTES, MAX_LINES
@@ -45,11 +46,16 @@ def utf8_len(value: str) -> int:
def format_size(size: int) -> str:
+ """Pi's ``formatSize``; ``toFixed(1)`` rounds an exact tie up (1280 bytes is ``1.3KB``), unlike ``%.1f``."""
if size < 1024:
return f"{size}B"
if size < 1024 * 1024:
- return f"{size / 1024:.1f}KB"
- return f"{size / (1024 * 1024):.1f}MB"
+ return f"{_to_fixed_1(size / 1024)}KB"
+ return f"{_to_fixed_1(size / (1024 * 1024))}MB"
+
+
+def _to_fixed_1(value: float) -> str:
+ return str(Decimal(value).quantize(Decimal("0.1"), rounding=ROUND_HALF_UP))
def split_lines_for_counting(content: str) -> list[str]:
diff --git a/core/agent_core/tools/write.py b/core/agent_core/tools/write.py
index 057f8228d..88f77fa5a 100644
--- a/core/agent_core/tools/write.py
+++ b/core/agent_core/tools/write.py
@@ -25,11 +25,6 @@ from core.agent_core.tools.paths import (
target_kind,
to_thread_joined,
)
-from core.agent_core.tools.text import model_text
-
-# The process umask, read once: new files get 0o666 under it, as open() would give them.
-_UMASK = os.umask(0)
-os.umask(_UMASK)
WRITE_DESCRIPTION = (
"Write content to a file. Creates the file if it doesn't exist, overwrites if it does. Automatically creates "
@@ -100,8 +95,8 @@ def _prepare_target(absolute: str) -> Optional[str]:
def write_text(path: str, content: str) -> None:
- """UTF-8 without newline translation, after the one sanitizer for model text."""
- write_bytes(path, model_text(content).encode("utf-8"))
+ """UTF-8 without newline translation; ``content`` came through the one sanitizer for model text."""
+ write_bytes(path, content.encode("utf-8"))
class FileChanged(Exception):
@@ -132,9 +127,10 @@ def write_bytes(path: str, data: bytes, expected: Optional[FileIdentity] = None)
"""Create or replace ``path`` only once ``data`` is completely on disk.
A failed write (a full disk, a quota) leaves the original, or its absence, as it was. The temp
- file sits beside the real target (through symlinks) and is private (0600) while it holds the new
- contents; once they are fsynced it takes the existing file's mode, or the umask default for a new
- file, and replaces the target with ``os.replace``.
+ file sits beside the real target (through symlinks). Replacing a file, it is private (0600) while
+ it receives the new contents and takes the file's mode once they are fsynced; a new file's temp is
+ created with the umask default it keeps, so the process umask is never read or changed. Then
+ ``os.replace`` puts it in place.
With ``expected``, the file must still be the one that was read: checked once, right before the
rename, or with ``fstat`` on the descriptor an in-place write goes through. POSIX has no
@@ -147,7 +143,7 @@ def write_bytes(path: str, data: bytes, expected: Optional[FileIdentity] = None)
mode = None
tmp = os.path.join(os.path.dirname(target), f".{os.path.basename(target)}.{secrets.token_hex(4)}.tmp")
try:
- fd = os.open(tmp, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600)
+ fd = os.open(tmp, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600 if mode is not None else 0o666)
except PermissionError:
if mode is None:
raise
@@ -162,7 +158,8 @@ def write_bytes(path: str, data: bytes, expected: Optional[FileIdentity] = None)
handle.write(data)
handle.flush()
os.fsync(handle.fileno())
- os.fchmod(handle.fileno(), mode if mode is not None else 0o666 & ~_UMASK)
+ if mode is not None:
+ os.fchmod(handle.fileno(), mode)
if expected is not None:
current = os.path.realpath(path)
try:
diff --git a/docs/plans/agent-core-contracts/tools.md b/docs/plans/agent-core-contracts/tools.md
index 1f5225e3b..7cb053ce5 100644
--- a/docs/plans/agent-core-contracts/tools.md
+++ b/docs/plans/agent-core-contracts/tools.md
@@ -135,18 +135,21 @@ continue in the background as an Avibe Watch; you get a follow-up message when t
`watch` is an Avibe addition. Behavior:
- stdin is closed; stdout and stderr are merged into the job's `output.log`; the tool follows the file for progress.
-- The tail is kept: `[Showing lines {start}-{end} of {total}. Output log (middle omitted beyond {cap}): {path}]`, or
- with ` (50KB limit)` when the byte cap applied, or `[Showing last {size} of line {n} (line is {size}). Output log
- (middle omitted beyond {cap}): {path}]`. `{cap}` is the on-disk bound (J4). Avibe change to Pi's `Full output:`:
- one label in every state, because it states the log's policy rather than its current size, and a job that is
- handed over keeps appending after its result was written, so no result promises a complete file that may not exist.
+- The tail is kept: `[Showing lines {start}-{end} of {total}. {label}: {path}]`, or with ` (50KB limit)` when the
+ byte cap applied, or `[Showing last {size} of line {n} (line is {size}). {label}: {path}]`, where `{label}` is
+ `Output log (first {head}, then the last {tail} in tail.log beside it)` and `{head}`/`{tail}` are the on-disk bounds
+ (J4: 1.0MB and 2.0MB). `tail.log` starts with the line `[output from byte {offset}]`. When the middle was dropped,
+ `[{size} between output.log and tail.log were omitted.]` follows. Avibe change to Pi's `Full output:`: one label in
+ every state, because it states the log's policy rather than its current size, and a job that is handed over keeps
+ appending after its result was written, so no result promises a complete file that may not exist. The line size
+ counts the whole line, including bytes the normalizer dropped from its start.
- Exit 0 is a normal result; otherwise an error result ending in `Command exited with code {code}`. `(no output)` when
empty.
- `timeout` kills the process tree: `Command timed out after {n} seconds`. Abort kills it: `Command aborted`.
- Handover (Avibe), when `watch: true` or the foreground window passes: the result is not an error and reads
`Command is still running and is now Watch {watch_id}. You will get a follow-up message when it finishes.` followed
- by the output so far (tail rules above), `Output log (middle omitted beyond {cap}): {path}`, and
- `Check: vibe watch show {watch_id}`, `Stop: vibe watch remove {watch_id}`.
+ by the output so far (tail rules above), `{label}: {path}`, and `Check: vibe watch show {watch_id}`,
+ `Stop: vibe watch remove {watch_id}`.
- The Watch follow-up states the command, exit code, elapsed time, the tail under the same rules, and the output log
under the same label.
diff --git a/tests/agent_core/tools/test_args.py b/tests/agent_core/tools/test_args.py
new file mode 100644
index 000000000..0fbc74d1c
--- /dev/null
+++ b/tests/agent_core/tools/test_args.py
@@ -0,0 +1,34 @@
+"""Pi's number and size formatting (JavaScript's ``toFixed`` and ``String(number)``)."""
+
+from __future__ import annotations
+
+import pytest
+
+from core.agent_core.tools.args import format_number
+from core.agent_core.tools.truncate import format_size
+
+
+# Expected values are what Pi's formatSize prints under Node.
+ ("size", "expected"),
+ [(1023, "1023B"), (1280, "1.3KB"), (52_480, "51.3KB"), (1_205_863, "1.2MB")],
+)
+def test_sizes_round_half_up_as_to_fixed_does(size, expected):
+ assert format_size(size) == expected
+
+
+# Expected values are JavaScript's String(number).
+ ("value", "expected"),
+ [
+ (5.0, "5"),
+ (0.5, "0.5"),
+ (2147483.647, "2147483.647"),
+ (0.00001, "0.00001"),
+ (1e-7, "1e-7"),
+ (1e21, "1e+21"),
+ (123456789012345680000.0, "123456789012345680000"),
+ ],
+)
+def test_numbers_print_as_javascript_prints_them(value, expected):
+ assert format_number(value) == expected
diff --git a/tests/agent_core/tools/test_bash.py b/tests/agent_core/tools/test_bash.py
index 657e1e2fd..1ed7b3346 100644
--- a/tests/agent_core/tools/test_bash.py
+++ b/tests/agent_core/tools/test_bash.py
@@ -19,6 +19,8 @@ import core.agent_core.tools.jobs as jobs_module
from core.agent_core.tools.jobs import LocalJobHost
from tests.agent_core.tools.conftest import result_text
+LABEL = "Output log (first 1.0MB, then the last 2.0MB in tail.log beside it)"
+
class Watches:
"""Adopt-or-create keyed by job id, as the adapter's Watch target is."""
@@ -53,7 +55,7 @@ async def test_long_output_keeps_the_tail_and_names_the_full_output(tmp_path, ma
assert not result.is_error
# One label in every state: the log keeps everything up to the cap, so it is never promised as complete.
assert result_text(result).endswith(
- f"2499\n2500\n\n[Showing lines 501-2500 of 2500. Output log (middle omitted beyond 3.0MB): {path}]"
+ f"2499\n2500\n\n[Showing lines 501-2500 of 2500. Output log (first 1.0MB, then the last 2.0MB in tail.log beside it): {path}]"
)
assert open(path).read() == "".join(f"{i}\n" for i in range(1, 2501))
@@ -67,7 +69,8 @@ async def test_a_log_bounded_on_disk_is_not_called_the_full_output(tmp_path, mak
text = result_text(result)
assert text.startswith("1\n2\n3\n")
- assert f"100000\n\n\n[Output log (middle omitted beyond 2.9KB): {result.details['output_path']}]" in text
+ path = result.details["output_path"]
+ assert f"100000\n\n\n[Output log (first 1000B, then the last 2.0KB in tail.log beside it): {path}]" in text
assert "Full output" not in text
assert result.details["omitted_bytes"] == len("".join(f"{i}\n" for i in range(1, 100001))) - 3000
@@ -80,10 +83,37 @@ async def test_a_completed_overlong_line_reports_its_size(tmp_path, make_ctx):
path = result.details["output_path"]
assert result_text(result).endswith(
- f"\n\n[Showing last 50.0KB of line 1 (line is 58.6KB). Output log (middle omitted beyond 3.0MB): {path}]"
+ f"\n\n[Showing last 50.0KB of line 1 (line is 58.6KB). Output log (first 1.0MB, then the last 2.0MB in tail.log beside it): {path}]"
)
+async def test_a_line_longer_than_the_normalizer_keeps_reports_its_whole_size(tmp_path, make_ctx):
+ command = "head -c 900000 /dev/zero | tr '\\0' x; echo"
+
+ result = await BashTool(_host(tmp_path)).execute({"command": command}, make_ctx())
+
+ assert "[Showing last 50.0KB of line 1 (line is 878.9KB). " in result_text(result)
+
+
+async def test_output_past_the_head_names_where_its_end_is(tmp_path, make_ctx):
+ """Between the 1 MiB head and the 3 MiB bound nothing is dropped, but the end is only in tail.log."""
+ result = await BashTool(_host(tmp_path)).execute({"command": "seq 1 200000"}, make_ctx())
+
+ path = result.details["output_path"]
+ assert result_text(result).endswith(f"[Showing lines 198001-200000 of 200000. {LABEL}: {path}]")
+ with open(os.path.join(os.path.dirname(path), "tail.log"), encoding="utf-8") as handle:
+ header, rest = handle.read().split("\n", 1)
+ assert header == f"[output from byte {os.path.getsize(path)}]"
+ assert rest.endswith("199999\n200000\n")
+
+
[email protected](("arguments", "expected"), [({"command": "printf '%s' \ud800"}, "\ufffd")])
+async def test_a_lone_surrogate_in_the_command_is_sanitized(tmp_path, make_ctx, arguments, expected):
+ result = await BashTool(_host(tmp_path)).execute(arguments, make_ctx())
+
+ assert (result.is_error, result_text(result)) == (False, expected)
+
+
async def test_a_non_zero_exit_is_an_error_result(tmp_path, make_ctx):
result = await BashTool(_host(tmp_path)).execute({"command": "echo oops >&2; exit 3"}, make_ctx())
@@ -275,7 +305,7 @@ async def test_the_foreground_window_hands_over_and_the_command_runs_once(tmp_pa
)
# A running job's log may still pass the on-disk cap, so it is never called the full output (J4).
assert text.endswith(
- f"\n\nOutput log (middle omitted beyond 3.0MB): {path}\nCheck: vibe watch show wch_1\nStop: vibe watch remove wch_1"
+ f"\n\nOutput log (first 1.0MB, then the last 2.0MB in tail.log beside it): {path}\nCheck: vibe watch show wch_1\nStop: vibe watch remove wch_1"
)
assert (await host.wait(job_id, deadline_s=5)).exit_code == 0
assert open(path).read() == "working\nfinished\n"
diff --git a/tests/agent_core/tools/test_jobs.py b/tests/agent_core/tools/test_jobs.py
index 579bdfa98..4bbe9e615 100644
--- a/tests/agent_core/tools/test_jobs.py
+++ b/tests/agent_core/tools/test_jobs.py
@@ -11,6 +11,7 @@ from __future__ import annotations
import asyncio
import errno
import os
+import signal
import subprocess
import time
@@ -56,6 +57,13 @@ def _wrapper_gone(job_dir):
return True
+def _gone(pid):
+ try:
+ return psutil.Process(pid).status() == psutil.STATUS_ZOMBIE
+ except psutil.NoSuchProcess:
+ return True
+
+
def _counter(tmp_path):
path = tmp_path / "counter"
return len(path.read_text().splitlines()) if path.exists() else 0
@@ -187,7 +195,7 @@ async def test_output_on_disk_keeps_the_head_and_tail_of_a_long_command(tmp_path
job_dir = host.job_dir(job_id)
assert os.path.getsize(os.path.join(job_dir, "output.log")) == 1000
- assert os.path.getsize(os.path.join(job_dir, "tail.log")) <= 2000 + 16
+ assert os.path.getsize(os.path.join(job_dir, "tail.log")) <= 2000 + 32
chunks, offset, skipped = [], 0, 0
while True:
@@ -319,7 +327,8 @@ async def test_a_kill_whose_reason_cannot_be_recorded_still_stops_the_command(tm
await host.kill(job_id)
assert stopped
- assert host.stop_reason(job_id) is None
+ # The host could not record "aborted"; its SIGTERM reached the wrapper, which recorded what it saw.
+ assert host.stop_reason(job_id) == "killed"
def test_a_shell_that_exited_by_the_deadline_check_is_not_a_timeout(tmp_path, monkeypatch):
@@ -340,6 +349,89 @@ def test_a_shell_that_exited_by_the_deadline_check_is_not_a_timeout(tmp_path, mo
assert (tmp_path / "exit").read_text() == "3\n"
[email protected]("sig", [signal.SIGTERM, signal.SIGHUP, signal.SIGUSR1])
+async def test_a_signal_to_the_wrapper_alone_stops_its_command(tmp_path, sig):
+ """``pkill -f python`` reaches the wrapper but not its shell: the wrapper must take the group with it."""
+ host = LocalJobHost(str(tmp_path / "jobs"))
+ job_id = await _start(host, tmp_path, f"echo $$ > {tmp_path / 'sh.pid'}; sleep 30", timeout_s=60)
+ assert _wait_until(lambda: (tmp_path / "sh.pid").exists())
+ shell_pid = int((tmp_path / "sh.pid").read_text())
+
+ os.kill(int(open(os.path.join(host.job_dir(job_id), "pid")).read()), sig)
+
+ try:
+ assert _wait_until(lambda: _gone(shell_pid), timeout_s=6)
+ finally:
+ await host.kill(job_id)
+ assert host.stop_reason(job_id) == "killed"
+
+
+async def test_a_command_outliving_a_killed_wrapper_is_still_running_and_held_to_its_deadline(tmp_path, monkeypatch):
+ """SIGKILL to the wrapper alone: the group carries the job's marker, so it is not gone (J2, J3)."""
+ import sys
+
+ monkeypatch.setattr(jobs_module, "_WRAPPER_DECIDES_S", 0.2)
+ # Python as the shell: its environment, and so the marker, is readable on macOS too.
+ host = LocalJobHost(str(tmp_path / "jobs"), shell=sys.executable)
+ script = f"import os, time; open({str(tmp_path / 'sh.pid')!r}, 'w').write(str(os.getpid())); time.sleep(30)"
+ job_id = await _start(host, tmp_path, script, timeout_s=1.0)
+ assert _wait_until(lambda: (tmp_path / "sh.pid").exists() and (tmp_path / "sh.pid").read_text())
+ shell_pid = int((tmp_path / "sh.pid").read_text())
+ os.kill(int(open(os.path.join(host.job_dir(job_id), "pid")).read()), signal.SIGKILL)
+ assert _wait_until(lambda: _wrapper_gone(host.job_dir(job_id)))
+
+ try:
+ assert host.status(job_id).state == "running"
+ status = await asyncio.wait_for(host.wait(job_id, deadline_s=None), timeout=10)
+ assert (status.state, host.stop_reason(job_id)) == ("gone", "timeout")
+ assert _wait_until(lambda: _gone(shell_pid))
+ finally:
+ await host.kill(job_id)
+
+
+async def test_start_reports_a_go_it_made_durable_even_if_reading_it_back_fails(tmp_path, monkeypatch):
+ """Once ``go`` is linked, the command runs: start must not report that it could not start."""
+ host = LocalJobHost(str(tmp_path / "jobs"))
+ real_read = jobs_module._read_text
+
+ def read_fails_for_decision(path):
+ if path.endswith(os.sep + "decision"):
+ raise OSError(errno.EMFILE, "Too many open files")
+ return real_read(path)
+
+ monkeypatch.setattr(jobs_module, "_read_text", read_fails_for_decision)
+ job_id = await _start(host, tmp_path, f"echo ran > {tmp_path / 'ran'}")
+ monkeypatch.undo()
+
+ assert (await host.wait(job_id, deadline_s=5)).state == "exited"
+ assert (tmp_path / "ran").read_text() == "ran\n"
+
+
+async def test_prune_keeps_an_exited_job_whose_wrapper_still_drains_children(tmp_path):
+ host = LocalJobHost(str(tmp_path / "jobs"))
+ # The shell exits at once; a background child keeps the pipe, so the wrapper keeps the log.
+ job_id = await _start(host, tmp_path, "sleep 2 &")
+ assert (await host.wait(job_id, deadline_s=5)).state == "exited"
+
+ removed = host.prune(lambda meta: True, older_than_s=-60)
+
+ assert removed == []
+ assert os.path.isdir(host.job_dir(job_id))
+
+
+async def test_hosts_over_different_spellings_of_the_jobs_directory_see_the_same_job(tmp_path):
+ """A legacy alias (``~/.vibe_remote`` -> ``~/.avibe``, ``/tmp`` -> ``/private/tmp``) names the same jobs."""
+ (tmp_path / "real").mkdir()
+ (tmp_path / "alias").symlink_to(tmp_path / "real")
+ host = LocalJobHost(str(tmp_path / "alias" / "jobs"))
+ job_id = await _start(host, tmp_path, "sleep 30")
+ try:
+ other = LocalJobHost(str(tmp_path / "real" / "jobs"))
+ assert other.status(job_id).state == "running"
+ finally:
+ await host.kill(job_id)
+
+
def test_the_wrapper_stops_its_group_even_when_its_diagnostics_fail(tmp_path, monkeypatch):
"""The wrapper's own failure handler must stop the command even if writing the traceback fails (ENOSPC)."""
import core.agent_core.tools.job_wrapper as wrapper
diff --git a/tests/agent_core/tools/test_read.py b/tests/agent_core/tools/test_read.py
index 87875a392..511e1c301 100644
--- a/tests/agent_core/tools/test_read.py
+++ b/tests/agent_core/tools/test_read.py
@@ -72,6 +72,15 @@ async def test_offset_and_limit_are_integers_of_at_least_one(tmp_path, make_ctx,
assert result_text(result) == f"{message} must be an integer of at least 1"
+async def test_an_over_long_first_line_is_measured_as_it_would_be_shown(tmp_path, make_ctx):
+ """Each undecodable byte is shown as U+FFFD (3 bytes), and the limit applies to what is shown."""
+ (tmp_path / "bin.txt").write_bytes(b"\xff" * 20_000 + b"\n")
+
+ text = result_text(await _read(make_ctx, path="bin.txt"))
+
+ assert text.startswith("[Line 1 is 58.6KB, exceeds 50.0KB limit.")
+
+
@pytest.mark.parametrize(
"spelling", ["big file.txt", "~/big file.txt", "file://{home}/big%20file.txt", "@big file.txt"]
)
@@ -117,9 +126,12 @@ async def test_cancel_stops_a_long_scan(tmp_path, make_ctx, monkeypatch):
def _whole_file_read(data: bytes, start: int, stop):
"""Pi's algorithm on the whole file, as the oracle for the streamed reader."""
lines = data.decode("utf-8", "replace").split("\n")
- # The "\r" of a CRLF break is not shown.
- lines = [line[:-1] if line.endswith("\r") and i < len(lines) - 1 else line for i, line in enumerate(lines)]
- return truncate_head("\n".join(lines[start:stop]), max_lines=5, max_bytes=20), len(lines)
+ # The "\r" of a CRLF break is not shown; a line ending in more than one "\r" shows them all.
+ lines = [
+ line[:-1] if line.endswith("\r") and not line.endswith("\r\r") and i < len(lines) - 1 else line
+ for i, line in enumerate(lines)
+ ]
+ return truncate_head("\n".join(lines[start:stop]), max_lines=5, max_bytes=20), lines
async def test_streaming_matches_reading_the_whole_file(tmp_path, monkeypatch):
@@ -138,7 +150,8 @@ async def test_streaming_matches_reading_the_whole_file(tmp_path, monkeypatch):
with open(path, "rb") as handle:
lines, total, first_line_bytes = read_module._scan_lines(handle, start, stop)
- expected, expected_total = _whole_file_read(data, start, stop)
+ expected, whole_lines = _whole_file_read(data, start, stop)
+ expected_total = len(whole_lines)
assert total == expected_total, data
if start >= total:
@@ -151,7 +164,8 @@ async def test_streaming_matches_reading_the_whole_file(tmp_path, monkeypatch):
expected.first_line_exceeds_limit,
), (data, start, stop)
if got.first_line_exceeds_limit:
- assert first_line_bytes == len(data.split(b"\n")[start])
+ # The size of the line as shown, which is what the limit applies to.
+ assert first_line_bytes == len(whole_lines[start].encode()), (data, start)
def _chunk(kind: bytes, body: bytes) -> bytes:
diff --git a/tests/agent_core/tools/test_write_edit.py b/tests/agent_core/tools/test_write_edit.py
index 197e11f5a..d920eb38c 100644
--- a/tests/agent_core/tools/test_write_edit.py
+++ b/tests/agent_core/tools/test_write_edit.py
@@ -130,6 +130,19 @@ async def test_text_copied_from_read_edits_a_file_with_invalid_bytes(tmp_path, m
assert (tmp_path / "f.txt").read_bytes() == b"fixed\nkeep\x80\n"
[email protected]("data", [b"alpha\r\r\nbeta\nkeep\n", b"alpha\r\nbeta\nkeep\n", b"a\rlpha\r\r\r\nbeta\nkeep\n"])
+async def test_lines_copied_from_read_edit_the_file_whatever_their_carriage_returns(tmp_path, make_ctx, data):
+ """read's view is edit's matching view: two lines copied from read are found, ``\\r\\r\\n`` included."""
+ (tmp_path / "f.txt").write_bytes(data)
+ shown = result_text(await ReadTool().execute({"path": "f.txt"}, make_ctx()))
+ first_two = shown.split("\nkeep")[0]
+
+ result = await _edit(make_ctx, "f.txt", {"oldText": first_two, "newText": "fixed"})
+
+ assert not result.is_error, result_text(result)
+ assert (tmp_path / "f.txt").read_bytes() == b"fixed\nkeep\n"
+
+
async def test_text_copied_from_read_edits_the_first_line_of_a_bom_file(tmp_path, make_ctx):
"""read does not show the BOM, edit does not match it, and the BOM survives."""
(tmp_path / "f.txt").write_bytes(b"\xef\xbb\xbfalpha\r\nbeta\r\n")
@@ -359,6 +372,25 @@ async def test_the_display_diff_and_patch_are_pis(tmp_path, make_ctx, original,
}
+ ("content", "edit"),
+ [
+ # Normalized matching strips trailing whitespace per line; a long run must not be quadratic.
+ ("x" + " " * 60_000 + "y\n", {"oldText": "\u201cmissing\u201d", "newText": "z"}),
+ # replaceAll over one long line: tens of thousands of occurrences.
+ ("a," * 40_000 + "\n", {"oldText": "a", "newText": "b", "replaceAll": True}),
+ ],
+ ids=["whitespace-run", "replace-all"],
+)
+async def test_edit_work_stays_linear_on_adversarial_files(tmp_path, make_ctx, content, edit):
+ (tmp_path / "f.txt").write_text(content)
+ started = time.monotonic()
+
+ await _edit(make_ctx, "f.txt", edit)
+
+ assert time.monotonic() - started < 1.0
+
+
async def test_a_large_changed_middle_skips_the_display_diff(tmp_path, make_ctx):
(tmp_path / "f.txt").write_text("".join(f"line {i}\n" for i in range(3000)))
@@ -582,6 +614,19 @@ async def test_a_failing_edit_writes_nothing(tmp_path, make_ctx, edits, message)
assert (tmp_path / "f.txt").read_text() == original
+ ("tool", "arguments", "expected"),
+ [
+ (ReadTool(), {"path": "a\ud800"}, "Cannot read a\ufffd: no such file or directory."),
+ (WriteTool(), {"path": "a\udfff", "content": "x"}, "Successfully wrote to a\ufffd"),
+ ],
+)
+async def test_lone_surrogates_in_a_path_are_sanitized(tmp_path, make_ctx, tool, arguments, expected):
+ result = await tool.execute(arguments, make_ctx())
+
+ assert result_text(result) == expected
+
+
@pytest.mark.parametrize(
("tool", "arguments"),
[
@@ -597,6 +642,25 @@ async def test_a_path_no_file_can_have_is_an_error_result(make_ctx, tool, argume
assert result_text(result) == f"Invalid path: {arguments['path']!r} contains a NUL byte"
+ "url",
+ [
+ "file://h%C3%A9/x", # Node: \\\\hé\\x, through IDNA processing Python does not have
+ "file://xn--9ca/x", # the same host, spelled in punycode
+ "file://127.1/x", # Node: \\\\127.0.0.1\\x, after WHATWG's IPv4 parsing
+ "file://[::ffff:1.2.3.4]/x", # Node: \\\\[::ffff:102:304]\\x
+ ],
+)
+def test_unc_hosts_that_need_idna_or_address_parsing_are_refused(monkeypatch, url):
+ """Refused rather than guessed, so a URL never names a share Node would not."""
+ monkeypatch.setattr(paths_module, "_WINDOWS", True)
+
+ with pytest.raises(ToolInputError) as raised:
+ paths_module.expand_path(url)
+
+ assert str(raised.value) == "Invalid path: a file URL must have a valid host"
+
+
# Expected values are Node's fileURLToPath (WHATWG URL parsing), which Pi uses.
@pytest.mark.parametrize(
("windows", "url", "expected"),
@@ -611,6 +675,12 @@ async def test_a_path_no_file_can_have_is_an_error_result(make_ctx, tool, argume
(True, "file://localhost\\C:\\x", "C:\\x"),
# Pi's gate is case-sensitive: this is a relative path in Pi too.
(False, "FILE:///tmp/a", "FILE:///tmp/a"),
+ # IDNA maps a soft hyphen to nothing; Node fails a host with a zero-width joiner.
+ (False, "file://local\u00adhost/x", "/x"),
+ (False, "file://local\u200dhost/etc/passwd", ToolInputError("Invalid path: a file URL must have a valid host")),
+ (True, "file://Server/Share/x", "\\\\server\\Share\\x"),
+ (True, "file://192.168.1.5/x", "\\\\192.168.1.5\\x"),
+ (True, "file://[::1]/x", "\\\\[::1]\\x"),
(False, "file://localhost/tmp/x", "/tmp/x"),
(False, "file://server/tmp/x", ToolInputError("Invalid path: file URL host must be empty or localhost")),
(False, "file:///tmp/a%2Fb", ToolInputError("Invalid path: a file URL must not include encoded / characters")),