Make your computer a home for your AI partner. Avibe is the local-first Agent OS: Vibey, its built-in AI partner, works out of the box, and Claude Code, Codex & OpenCode run unchanged beside it — reach them from your browser, phone, or any chat app.
Python agentagent-osaiai-agentsai-partnerchatopsclaudeclaude-codecodexdevtoolsdiscord-botlark-botllmlocal-firstlong-term-memoryopencodeslack-botvibe-codingvibeywechat
from __future__ import annotations
import io
import json
import pytest
from core.handlers.model_hub.json_wire import (
JSON_STRING_TOKEN_BYTES,
SelectiveJSONParser,
project_json_reader,
rewrite_json_strings,
)
@pytest.mark.parametrize("ensure_ascii", (True, False))
@pytest.mark.parametrize("chunk_size", (1, 4093))
def test_lossless_string_values_preserve_unicode_across_lexical_chunks(
ensure_ascii: bool, chunk_size: int,
) -> None:
identity = '模型🧪/e\u0301"\\' * 2000
payload = json.dumps({"id": identity}, ensure_ascii=ensure_ascii).encode()
observed = []
parser = SelectiveJSONParser(
{(), ("id",)},
lambda path, event, value, scope: observed.append((path, event, value, scope)),
lossless_string_paths={("id",)},
)
for offset in range(0, len(payload), chunk_size):
parser.feed(payload[offset : offset + chunk_size])
assert parser.finish() is True
assert (("id",), "scalar", identity, ()) in observed
def test_lossless_opt_in_never_unbounds_keys_or_unselected_values() -> None:
oversized = "x" * (JSON_STRING_TOKEN_BYTES * 2)
observed = []
parser = SelectiveJSONParser(
{(), ("id",), ("diagnostic",), (oversized,)},
lambda path, event, value, _scope: observed.append((path, event, value)),
lossless_string_paths={("id",), (oversized,), ("unselected",)},
)
parser.feed(b'{"')
parser.feed(oversized.encode())
assert parser.retained_bytes < JSON_STRING_TOKEN_BYTES + 1024
parser.feed(b'":"not a selected key","unselected":"')
parser.feed(oversized.encode())
assert parser.retained_bytes < JSON_STRING_TOKEN_BYTES + 1024
parser.feed(b'","diagnostic":"')
parser.feed(oversized.encode())
assert parser.retained_bytes < JSON_STRING_TOKEN_BYTES + 1024
parser.feed(b'","id":' + json.dumps(oversized).encode() + b"}")
assert parser.finish() is True
assert (("id",), "scalar", oversized) in observed
assert not any(path == (oversized,) for path, _, _ in observed)
assert not any(
event == "scalar" and value == oversized
for path, event, value in observed if path != ("id",)
)
def test_selected_strings_remain_bounded_without_lossless_opt_in() -> None:
observed = []
assert project_json_reader(
io.BytesIO(json.dumps({"id": "x" * (JSON_STRING_TOKEN_BYTES + 1)}).encode()),
{(), ("id",)},
lambda path, event, value, _scope: observed.append((path, event, value)),
)
assert (("id",), "elided_string", None) in observed
@pytest.mark.parametrize("suffix", (b'\\q"}', b'\\u000"}', b'\xff"}', b'"', b'"},', b'","other":[,]}'))
def test_lossless_retention_does_not_relax_json_completion_or_grammar(suffix: bytes) -> None:
assert not project_json_reader(
io.BytesIO(b'{"id":"' + b"x" * (JSON_STRING_TOKEN_BYTES + 1) + suffix),
{(), ("id",)},
lambda *_args: None,
lossless_string_paths={("id",)},
)
@pytest.mark.parametrize(
"payload",
(
b'{"wanted":1,}',
b'{"ignored":[{"nested":1}],"wanted":2,}',
b'{"ignored":[}',
b'{"ignored":"bad\\q","wanted":3}',
b'{"ignored":01,"wanted":4}',
b'{"ignored":1e,"wanted":5}',
b'{"ignored":"\xff","wanted":6}',
b'{"ignored":[,,true],"wanted":7}',
b'{"ignored":{"key" 1},"wanted":8}',
b'{"ignored":[true false],"wanted":9}',
),
)
def test_selective_projection_does_not_accept_invalid_ignored_json(payload: bytes) -> None:
assert not project_json_reader(io.BytesIO(payload), {(), ("wanted",)}, lambda *_args: None)
def test_selective_projection_skips_large_values_but_keeps_later_selected_facts() -> None:
payload = json.dumps(
{
"ignored_string": "x" * (2 * 1024 * 1024),
"ignored_tree": [[{"value": index}] for index in range(20_000)],
"wanted": "kept",
},
separators=(",", ":"),
).encode()
observed: list[object] = []
valid = project_json_reader(
io.BytesIO(payload),
{(), ("wanted",)},
lambda path, event, value, _scope: (
observed.append(value)
if path == ("wanted",) and event == "scalar"
else None
),
)
assert valid is True
assert observed == ["kept"]
def test_selective_projection_scans_large_ascii_strings_in_chunks() -> None:
parser = SelectiveJSONParser({(), ("wanted",)}, lambda *_args: None)
parser.feed(b'{"ignored":"')
delegate = parser._string_decoder
class CountingDecoder:
calls = 0
def decode(self, payload: bytes, final: bool = False) -> str:
self.calls += 1
return delegate.decode(payload, final=final)
def getstate(self):
return delegate.getstate()
def reset(self) -> None:
delegate.reset()
decoder = CountingDecoder()
parser._string_decoder = decoder
parser.feed(b"x" * (2 * 1024 * 1024))
parser.feed(b'","wanted":true}')
assert parser.finish() is True
assert decoder.calls < 10
def test_selective_projection_packs_deep_ignored_grammar_state() -> None:
parser = SelectiveJSONParser({(), ("wanted",)}, lambda *_args: None)
parser.feed(b'{"ignored":' + (b"[" * 100_000))
assert parser.retained_bytes < 64 * 1024
assert parser.finish() is False
def test_selective_projection_closes_deep_ignored_grammar_state() -> None:
parser = SelectiveJSONParser({(), ("wanted",)}, lambda *_args: None)
parser.feed(b'{"ignored":' + (b"[" * 100_000))
parser.feed(b"0" + (b"]" * 100_000) + b',"wanted":true}')
assert parser.finish() is True
assert parser.retained_bytes == 0
def test_selective_projection_accepts_one_leading_utf8_bom() -> None:
observed: list[object] = []
valid = project_json_reader(
io.BytesIO(b'\xef\xbb\xbf{"wanted":true}'),
{(), ("wanted",)},
lambda path, event, value, _scope: (
observed.append(value)
if path == ("wanted",) and event == "scalar"
else None
),
)
assert valid is True
assert observed == [True]
def test_duplicate_wildcard_members_replace_only_their_array_occurrence() -> None:
observed: list[tuple[tuple[str, ...], str, object, tuple[int, ...]]] = []
payload = (
b'{"choices":[{"delta":{"content":"first"}},'
b'{"delta":{"content":"stale","content":null}}]}'
)
valid = project_json_reader(
io.BytesIO(payload),
{(), ("choices", "*", "delta", "content")},
lambda path, event, value, scope: observed.append((path, event, value, scope)),
)
content = [
(event, value, scope)
for path, event, value, scope in observed
if path == ("choices", "*", "delta", "content")
]
assert valid is True
assert content == [
("scalar", "first", (0,)),
("scalar", "stale", (1,)),
("replace", None, (1,)),
("scalar", None, (1,)),
]
def test_wildcard_scope_ends_after_each_array_member() -> None:
scopes: list[tuple[int, ...]] = []
valid = project_json_reader(
io.BytesIO(b'{"choices":[{"value":1},{"value":2}]}'),
{(), ("choices", "*", "value")},
lambda _path, event, _value, scope: (
scopes.append(scope) if event == "scope_end" else None
),
)
assert valid is True
assert scopes == [(0,), (1,)]
def test_path_rewriter_changes_only_selected_string_values() -> None:
payload = b'{"content":"alias","tool":{"name":"alias"},"other":{"name":"alias"}}'
rewritten = rewrite_json_strings(
io.BytesIO(payload),
target_paths={("tool", "name")},
replacements={"alias": "original"},
)
assert rewritten is not None
try:
assert json.load(rewritten) == {
"content": "alias",
"tool": {"name": "original"},
"other": {"name": "alias"},
}
finally:
rewritten.close()