Make your computer a home for your AI partner. Avibe is the local-first Agent OS: Vibey, its built-in AI partner, works out of the box, and Claude Code, Codex & OpenCode run unchanged beside it — reach them from your browser, phone, or any chat app.
Python agentagent-osaiai-agentsai-partnerchatopsclaudeclaude-codecodexdevtoolsdiscord-botlark-botllmlocal-firstlong-term-memoryopencodeslack-botvibe-codingvibeywechat
"""Bounded-memory JSON projection and path-aware string rewriting."""
from __future__ import annotations
import codecs
import json
import re
import tempfile
from collections.abc import Callable, Iterable, Mapping
from dataclasses import dataclass, field
from typing import BinaryIO, Final, Literal
JSONPath = tuple[str, ...]
JSONScope = tuple[int, ...]
JSONEvent = Literal[
"replace",
"start_map",
"start_array",
"scalar",
"nonempty",
"elided_string",
"scope_end",
]
JSONVisitor = Callable[[JSONPath, JSONEvent, object | None, JSONScope], None]
JSON_STRING_TOKEN_BYTES: Final = 16 * 1024
JSON_NUMBER_TOKEN_BYTES: Final = 128
JSON_IO_CHUNK_BYTES: Final = 64 * 1024
_JSON_STRING_SPECIAL: Final = re.compile(br'["\\\x00-\x1f\x80-\xff]')
_UTF8_BOM: Final = b"\xef\xbb\xbf"
_SKIP_MAP_KEY_OR_END: Final = 0
_SKIP_MAP_KEY: Final = 1
_SKIP_MAP_COLON: Final = 2
_SKIP_MAP_VALUE: Final = 3
_SKIP_MAP_COMMA_OR_END: Final = 4
_SKIP_ARRAY_VALUE_OR_END: Final = 5
_SKIP_ARRAY_VALUE: Final = 6
_SKIP_ARRAY_COMMA_OR_END: Final = 7
_SKIP_STATE_BITS: Final = 3
_SKIP_STATE_MASK: Final = (1 << _SKIP_STATE_BITS) - 1
_SKIP_STATES_PER_CHUNK: Final = 64
_SKIP_CHUNK_BYTES: Final = (_SKIP_STATES_PER_CHUNK * _SKIP_STATE_BITS + 7) // 8
@dataclass
class _PathNode:
selected: bool = False
children: dict[str, "_PathNode"] = field(default_factory=dict)
def _path_tree(paths: Iterable[JSONPath]) -> _PathNode:
root = _PathNode()
for path in paths:
node = root
for component in path:
node = node.children.setdefault(component, _PathNode())
node.selected = True
return root
@dataclass
class _Container:
kind: Literal["map", "array"]
path: JSONPath
node: _PathNode
state: str
scope: JSONScope
key: str | None = None
nonempty: bool = False
next_index: int = 0
seen_children: set[str] = field(default_factory=set)
class SelectiveJSONParser:
"""Parse only selected paths while skipping every unrelated value in place.
Strings and numbers are bounded lexical tokens by default. Callers may
retain complete string values at explicit selected paths for facts such as
identifiers. Object keys and unrelated values remain bounded. Selected
containers retain path state; an unrelated subtree uses a packed grammar
stack without weakening JSON validation.
"""
def __init__(
self,
paths: Iterable[JSONPath],
visitor: JSONVisitor,
*,
lossless_string_paths: Iterable[JSONPath] = (),
) -> None:
selected_paths = frozenset(paths)
self._root = _path_tree(selected_paths)
self._visitor = visitor
self._lossless_string_paths = selected_paths.intersection(lossless_string_paths)
self._stack: list[_Container] = []
self._root_state = "value"
self._skip_depth = 0
self._skip_state_chunks: list[int] = []
self._root_prefix = bytearray()
self._root_started = False
self._invalid = False
self._mode: Literal["normal", "string", "number", "literal"] = "normal"
self._string = bytearray()
self._string_too_long = False
self._string_lossless = False
self._escaped = False
self._unicode_escape_digits = 0
self._string_decoder = codecs.getincrementaldecoder("utf-8")()
self._number = bytearray()
self._number_too_long = False
self._number_state = ""
self._literal = bytearray()
@property
def retained_bytes(self) -> int:
return (
len(self._string)
+ len(self._number)
+ len(self._literal)
+ len(self._root_prefix)
+ len(self._skip_state_chunks) * _SKIP_CHUNK_BYTES
+ sum(len(frame.key or "") for frame in self._stack)
+ sum(
sum(len(key) for key in frame.seen_children)
for frame in self._stack
)
)
@property
def valid(self) -> bool:
return (
not self._invalid
and self._root_state == "done"
and not self._stack
and self._skip_depth == 0
and self._mode == "normal"
)
@property
def next_value_path(self) -> JSONPath | None:
"""Return the selected path of the next value, if one is expected."""
if self._invalid or self._skip_depth or self._mode != "normal":
return None
if not self._stack:
return () if self._root_state == "value" else None
frame = self._stack[-1]
if frame.kind == "map" and frame.state == "value" and frame.key is not None:
child = frame.node.children.get(frame.key)
return (*frame.path, frame.key) if child is not None else None
if frame.kind == "array" and frame.state in {"value_or_end", "value"}:
child = frame.node.children.get("*")
return (*frame.path, "*") if child is not None else None
return None
def feed(self, chunk: bytes) -> None:
offset = 0
while offset < len(chunk) and not self._invalid:
if not self._root_started:
self.feed_byte(chunk[offset])
offset += 1
continue
if self._mode == "string":
offset = self._feed_string_chunk(chunk, offset)
continue
self._feed_started_byte(chunk[offset])
offset += 1
def feed_byte(self, byte: int) -> None:
if self._invalid:
return
if not self._root_started:
self._root_prefix.append(byte)
prefix = bytes(self._root_prefix)
if len(prefix) < len(_UTF8_BOM) and _UTF8_BOM.startswith(prefix):
return
self._root_started = True
self._root_prefix.clear()
if prefix.startswith(_UTF8_BOM):
prefix = prefix[len(_UTF8_BOM) :]
for pending_byte in prefix:
self._feed_started_byte(pending_byte)
return
self._feed_started_byte(byte)
def _feed_started_byte(self, byte: int) -> None:
if self._invalid:
return
if self._mode == "string":
self._feed_string(byte)
return
if self._mode == "number":
if byte in b"0123456789+-.eE":
if not self._advance_number(byte):
self._invalid = True
return
if len(self._number) < JSON_NUMBER_TOKEN_BYTES:
self._number.append(byte)
else:
self._number_too_long = True
return
self._finish_number()
self.feed_byte(byte)
return
if self._mode == "literal":
if 0x61 <= byte <= 0x7A:
if len(self._literal) < len(b"false"):
self._literal.append(byte)
else:
self._invalid = True
return
self._finish_literal()
self.feed_byte(byte)
return
if byte in b" \t\r\n":
return
if byte == 0x22:
# Resolve while the parser still expects a value. Keys and skipped
# subtrees have no selected value path and never opt into retention.
self._string_lossless = self.next_value_path in self._lossless_string_paths
self._mode = "string"
self._string.clear()
self._string_too_long = False
self._escaped = False
self._unicode_escape_digits = 0
self._string_decoder.reset()
return
if byte in b"-0123456789":
self._mode = "number"
self._number.clear()
self._number.append(byte)
self._number_too_long = False
self._number_state = "sign" if byte == 0x2D else "zero" if byte == 0x30 else "int"
return
if byte in b"tfn":
self._mode = "literal"
self._literal.clear()
self._literal.append(byte)
return
punctuation = {
0x7B: "start_map",
0x7D: "end_map",
0x5B: "start_array",
0x5D: "end_array",
0x3A: "colon",
0x2C: "comma",
}.get(byte)
if punctuation is None:
self._invalid = True
return
self._accept(punctuation, None)
def finish(self) -> bool:
if self._mode == "number":
self._finish_number()
elif self._mode == "literal":
self._finish_literal()
elif self._mode == "string":
self._invalid = True
return self.valid
def _feed_string(self, byte: int) -> None:
if self._unicode_escape_digits:
if byte not in b"0123456789abcdefABCDEF":
self._invalid = True
return
self._unicode_escape_digits -= 1
elif self._escaped:
if byte not in b'"\\/bfnrtu':
self._invalid = True
return
self._escaped = False
if byte == 0x75:
self._unicode_escape_digits = 4
elif byte == 0x5C:
self._escaped = True
elif byte == 0x22:
try:
self._string_decoder.decode(b"", final=True)
except UnicodeDecodeError:
self._invalid = True
return
raw = bytes(self._string)
too_long = self._string_too_long
self._mode = "normal"
self._string.clear()
if too_long:
self._accept("string_elided", None)
return
try:
value = json.loads(b'"' + raw + b'"')
except (UnicodeDecodeError, ValueError):
self._invalid = True
return
self._accept("string", value)
return
elif byte < 0x20:
self._invalid = True
return
try:
self._string_decoder.decode(bytes((byte,)), final=False)
except UnicodeDecodeError:
self._invalid = True
return
self._retain_string_bytes(bytes((byte,)))
def _feed_string_chunk(self, chunk: bytes, offset: int) -> int:
"""Consume ordinary ASCII string runs without per-byte codec calls."""
while offset < len(chunk) and self._mode == "string" and not self._invalid:
decoder_pending = bool(self._string_decoder.getstate()[0])
if self._escaped or self._unicode_escape_digits or decoder_pending:
self._feed_string(chunk[offset])
offset += 1
continue
special = _JSON_STRING_SPECIAL.search(chunk, offset)
end = special.start() if special is not None else len(chunk)
if end > offset:
self._retain_string_bytes(chunk[offset:end])
offset = end
if special is None:
break
self._feed_string(chunk[offset])
offset += 1
return offset
def _retain_string_bytes(self, payload: bytes) -> None:
if self._string_lossless:
self._string.extend(payload)
return
remaining = JSON_STRING_TOKEN_BYTES - len(self._string)
if remaining > 0:
self._string.extend(payload[:remaining])
if len(payload) > remaining:
self._string_too_long = True
def _finish_number(self) -> None:
raw = bytes(self._number)
too_long = self._number_too_long
self._mode = "normal"
self._number.clear()
if self._number_state not in {"zero", "int", "frac", "exp_int"}:
self._invalid = True
return
if too_long:
self._accept("scalar", None)
return
try:
value = json.loads(raw)
except ValueError:
self._invalid = True
return
self._accept("scalar", value)
def _advance_number(self, byte: int) -> bool:
if self._number_state == "sign":
if byte == 0x30:
self._number_state = "zero"
return True
if byte in b"123456789":
self._number_state = "int"
return True
return False
if self._number_state == "zero":
if byte == 0x2E:
self._number_state = "dot"
return True
if byte in b"eE":
self._number_state = "exp"
return True
return False
if self._number_state == "int":
if byte in b"0123456789":
return True
if byte == 0x2E:
self._number_state = "dot"
return True
if byte in b"eE":
self._number_state = "exp"
return True
return False
if self._number_state == "dot":
if byte in b"0123456789":
self._number_state = "frac"
return True
return False
if self._number_state == "frac":
if byte in b"0123456789":
return True
if byte in b"eE":
self._number_state = "exp"
return True
return False
if self._number_state == "exp":
if byte in b"+-":
self._number_state = "exp_sign"
return True
if byte in b"0123456789":
self._number_state = "exp_int"
return True
return False
if self._number_state == "exp_sign":
if byte in b"0123456789":
self._number_state = "exp_int"
return True
return False
if self._number_state == "exp_int":
return byte in b"0123456789"
return False
def _finish_literal(self) -> None:
raw = bytes(self._literal)
self._mode = "normal"
self._literal.clear()
values = {b"true": True, b"false": False, b"null": None}
if raw not in values:
self._invalid = True
return
self._accept("scalar", values[raw])
def _accept(self, token: str, value: object | None) -> None:
if self._skip_depth:
self._accept_skipped(token)
return
if not self._stack:
if self._root_state != "value":
self._invalid = True
return
self._start_value((), self._root, token, value, scope=())
return
frame = self._stack[-1]
if frame.kind == "map":
self._accept_map(frame, token, value)
else:
self._accept_array(frame, token, value)
def _accept_map(self, frame: _Container, token: str, value: object | None) -> None:
if frame.state in {"key_or_end", "key"}:
if token == "end_map" and frame.state == "key_or_end":
self._end_container("map")
elif token in {"string", "string_elided"}:
frame.key = value if isinstance(value, str) else None
frame.state = "colon"
else:
self._invalid = True
return
if frame.state == "colon":
if token != "colon":
self._invalid = True
else:
frame.state = "value"
return
if frame.state == "value":
key = frame.key
child = frame.node.children.get(key) if key is not None else None
self._mark_nonempty(frame)
if child is None:
self._skip_value(token)
else:
duplicate = key in frame.seen_children
frame.seen_children.add(key)
self._start_value(
(*frame.path, key),
child,
token,
value,
scope=frame.scope,
replace=duplicate,
)
return
if frame.state == "comma_or_end":
if token == "comma":
frame.state = "key"
frame.key = None
elif token == "end_map":
self._end_container("map")
else:
self._invalid = True
def _accept_array(self, frame: _Container, token: str, value: object | None) -> None:
if frame.state in {"value_or_end", "value"}:
if token == "end_array" and frame.state == "value_or_end":
self._end_container("array")
return
self._mark_nonempty(frame)
child = frame.node.children.get("*")
if child is None:
self._skip_value(token)
else:
self._start_value(
(*frame.path, "*"),
child,
token,
value,
scope=(*frame.scope, frame.next_index),
)
return
if frame.state == "comma_or_end":
if token == "comma":
frame.state = "value"
elif token == "end_array":
self._end_container("array")
else:
self._invalid = True
def _start_value(
self,
path: JSONPath,
node: _PathNode,
token: str,
value: object | None,
*,
scope: JSONScope,
replace: bool = False,
) -> None:
if replace and (node.selected or node.children):
self._visitor(path, "replace", None, scope)
if token == "start_map":
if node.selected:
self._visitor(path, "start_map", None, scope)
self._stack.append(_Container("map", path, node, "key_or_end", scope))
elif token == "start_array":
if node.selected:
self._visitor(path, "start_array", None, scope)
self._stack.append(_Container("array", path, node, "value_or_end", scope))
elif token in {"string", "string_elided", "scalar"}:
if node.selected:
if token == "string_elided":
self._visitor(path, "elided_string", None, scope)
self._visitor(path, "scalar", value, scope)
self._complete_value()
else:
self._invalid = True
def _skip_value(self, token: str) -> None:
if token == "start_map":
self._push_skip_state(_SKIP_MAP_KEY_OR_END)
elif token == "start_array":
self._push_skip_state(_SKIP_ARRAY_VALUE_OR_END)
elif token in {"string", "string_elided", "scalar"}:
self._complete_value()
else:
self._invalid = True
def _accept_skipped(self, token: str) -> None:
state = self._skip_state()
if state in {_SKIP_MAP_KEY_OR_END, _SKIP_MAP_KEY}:
if token == "end_map" and state == _SKIP_MAP_KEY_OR_END:
self._pop_skip_state()
elif token in {"string", "string_elided"}:
self._set_skip_state(_SKIP_MAP_COLON)
else:
self._invalid = True
return
if state == _SKIP_MAP_COLON:
if token == "colon":
self._set_skip_state(_SKIP_MAP_VALUE)
else:
self._invalid = True
return
if state == _SKIP_MAP_VALUE:
self._start_skipped_value(token, _SKIP_MAP_COMMA_OR_END)
return
if state == _SKIP_MAP_COMMA_OR_END:
if token == "comma":
self._set_skip_state(_SKIP_MAP_KEY)
elif token == "end_map":
self._pop_skip_state()
else:
self._invalid = True
return
if state in {_SKIP_ARRAY_VALUE_OR_END, _SKIP_ARRAY_VALUE}:
if token == "end_array" and state == _SKIP_ARRAY_VALUE_OR_END:
self._pop_skip_state()
else:
self._start_skipped_value(token, _SKIP_ARRAY_COMMA_OR_END)
return
if state == _SKIP_ARRAY_COMMA_OR_END:
if token == "comma":
self._set_skip_state(_SKIP_ARRAY_VALUE)
elif token == "end_array":
self._pop_skip_state()
else:
self._invalid = True
def _start_skipped_value(self, token: str, completed_state: int) -> None:
if token in {"string", "string_elided", "scalar"}:
self._set_skip_state(completed_state)
elif token == "start_map":
self._set_skip_state(completed_state)
self._push_skip_state(_SKIP_MAP_KEY_OR_END)
elif token == "start_array":
self._set_skip_state(completed_state)
self._push_skip_state(_SKIP_ARRAY_VALUE_OR_END)
else:
self._invalid = True
def _push_skip_state(self, state: int) -> None:
slot = self._skip_depth % _SKIP_STATES_PER_CHUNK
if slot == 0:
self._skip_state_chunks.append(0)
self._skip_state_chunks[-1] |= state << (slot * _SKIP_STATE_BITS)
self._skip_depth += 1
def _set_skip_state(self, state: int) -> None:
slot = (self._skip_depth - 1) % _SKIP_STATES_PER_CHUNK
shift = slot * _SKIP_STATE_BITS
mask = _SKIP_STATE_MASK << shift
self._skip_state_chunks[-1] = (
self._skip_state_chunks[-1] & ~mask
) | (state << shift)
def _skip_state(self) -> int:
slot = (self._skip_depth - 1) % _SKIP_STATES_PER_CHUNK
return (
self._skip_state_chunks[-1] >> (slot * _SKIP_STATE_BITS)
) & _SKIP_STATE_MASK
def _pop_skip_state(self) -> None:
slot = (self._skip_depth - 1) % _SKIP_STATES_PER_CHUNK
if slot == 0:
self._skip_state_chunks.pop()
else:
shift = slot * _SKIP_STATE_BITS
self._skip_state_chunks[-1] &= ~(_SKIP_STATE_MASK << shift)
self._skip_depth -= 1
if self._skip_depth == 0:
self._complete_value()
def _mark_nonempty(self, frame: _Container) -> None:
if not frame.nonempty:
frame.nonempty = True
if frame.node.selected:
self._visitor(frame.path, "nonempty", True, frame.scope)
def _end_container(self, expected: Literal["map", "array"]) -> None:
if not self._stack or self._stack[-1].kind != expected:
self._invalid = True
return
self._stack.pop()
self._complete_value()
def _complete_value(self) -> None:
if not self._stack:
self._root_state = "done"
return
parent = self._stack[-1]
if parent.kind == "array":
child = parent.node.children.get("*")
if child is not None and (child.selected or child.children):
self._visitor(
(*parent.path, "*"),
"scope_end",
None,
(*parent.scope, parent.next_index),
)
parent.next_index += 1
parent.state = "comma_or_end"
def project_json_reader(
reader: BinaryIO,
paths: Iterable[JSONPath],
visitor: JSONVisitor,
*,
lossless_string_paths: Iterable[JSONPath] = (),
) -> bool:
parser = SelectiveJSONParser(paths, visitor, lossless_string_paths=lossless_string_paths)
while chunk := reader.read(JSON_IO_CHUNK_BYTES):
parser.feed(chunk)
return parser.finish()
class _JSONPathStringRewriter:
def __init__(
self,
writer: BinaryIO,
target_paths: Iterable[JSONPath],
replacements: Mapping[str, str],
) -> None:
self._writer = writer
self._targets = frozenset(target_paths)
self._replacements = replacements
self._parser = SelectiveJSONParser(self._targets, lambda *_args: None)
self._output = bytearray()
self._candidate = bytearray()
self._candidate_path: JSONPath | None = None
self._candidate_escaped = False
self._candidate_too_long = False
def feed(self, chunk: bytes) -> None:
for byte in chunk:
if self._candidate_path is not None:
self._candidate.append(byte)
if self._candidate_escaped:
self._candidate_escaped = False
elif byte == 0x5C:
self._candidate_escaped = True
elif byte == 0x22:
self._finish_candidate()
continue
if len(self._candidate) > JSON_STRING_TOKEN_BYTES + 2:
self._candidate_too_long = True
self._emit(bytes(self._candidate))
self._candidate.clear()
self._candidate_path = None
self._parser.feed_byte(byte)
continue
path = self._parser.next_value_path
if byte == 0x22 and path in self._targets:
self._candidate_path = path
self._candidate.clear()
self._candidate.append(byte)
self._candidate_escaped = False
self._candidate_too_long = False
self._parser.feed_byte(byte)
continue
self._emit(bytes((byte,)))
self._parser.feed_byte(byte)
def finish(self) -> bool:
if self._candidate:
self._emit(bytes(self._candidate))
self._candidate.clear()
self._candidate_path = None
valid = self._parser.finish()
self._flush()
return valid
def _finish_candidate(self) -> None:
raw = bytes(self._candidate)
self._parser.feed_byte(raw[-1])
replacement: str | None = None
if not self._candidate_too_long:
try:
value = json.loads(raw)
except (UnicodeDecodeError, ValueError):
value = None
if isinstance(value, str):
replacement = self._replacements.get(value)
if replacement is None:
self._emit(raw)
else:
self._emit(json.dumps(replacement, ensure_ascii=False).encode("utf-8"))
self._candidate.clear()
self._candidate_path = None
def _emit(self, payload: bytes) -> None:
self._output.extend(payload)
if len(self._output) >= JSON_IO_CHUNK_BYTES:
self._flush()
def _flush(self) -> None:
if self._output:
self._writer.write(self._output)
self._output.clear()
def rewrite_json_strings(
reader: BinaryIO,
*,
target_paths: Iterable[JSONPath],
replacements: Mapping[str, str],
) -> BinaryIO | None:
"""Return a rewritten spool, or None when the source is not valid JSON."""
if not replacements:
return None
output = tempfile.SpooledTemporaryFile(max_size=256 * 1024)
rewriter = _JSONPathStringRewriter(output, target_paths, replacements)
while chunk := reader.read(JSON_IO_CHUNK_BYTES):
rewriter.feed(chunk)
if not rewriter.finish():
output.close()
return None
output.seek(0)
return output