Skip to content

armgit / avibe

Public Imported from GitHub
0

Make your computer a home for your AI partner. Avibe is the local-first Agent OS: Vibey, its built-in AI partner, works out of the box, and Claude Code, Codex & OpenCode run unchanged beside it — reach them from your browser, phone, or any chat app.

7.8 KB
Raw Blame
from __future__ import annotations

import io
import json

import pytest

from core.handlers.model_hub.json_wire import (
    JSON_STRING_TOKEN_BYTES,
    SelectiveJSONParser,
    project_json_reader,
    rewrite_json_strings,
)


@pytest.mark.parametrize("ensure_ascii", (True, False))
@pytest.mark.parametrize("chunk_size", (1, 4093))
def test_lossless_string_values_preserve_unicode_across_lexical_chunks(
    ensure_ascii: bool, chunk_size: int,
) -> None:
    identity = '模型🧪/e\u0301"\\' * 2000
    payload = json.dumps({"id": identity}, ensure_ascii=ensure_ascii).encode()
    observed = []
    parser = SelectiveJSONParser(
        {(), ("id",)},
        lambda path, event, value, scope: observed.append((path, event, value, scope)),
        lossless_string_paths={("id",)},
    )
    for offset in range(0, len(payload), chunk_size):
        parser.feed(payload[offset : offset + chunk_size])
    assert parser.finish() is True
    assert (("id",), "scalar", identity, ()) in observed


def test_lossless_opt_in_never_unbounds_keys_or_unselected_values() -> None:
    oversized = "x" * (JSON_STRING_TOKEN_BYTES * 2)
    observed = []
    parser = SelectiveJSONParser(
        {(), ("id",), ("diagnostic",), (oversized,)},
        lambda path, event, value, _scope: observed.append((path, event, value)),
        lossless_string_paths={("id",), (oversized,), ("unselected",)},
    )
    parser.feed(b'{"')
    parser.feed(oversized.encode())
    assert parser.retained_bytes < JSON_STRING_TOKEN_BYTES + 1024
    parser.feed(b'":"not a selected key","unselected":"')
    parser.feed(oversized.encode())
    assert parser.retained_bytes < JSON_STRING_TOKEN_BYTES + 1024
    parser.feed(b'","diagnostic":"')
    parser.feed(oversized.encode())
    assert parser.retained_bytes < JSON_STRING_TOKEN_BYTES + 1024
    parser.feed(b'","id":' + json.dumps(oversized).encode() + b"}")
    assert parser.finish() is True
    assert (("id",), "scalar", oversized) in observed
    assert not any(path == (oversized,) for path, _, _ in observed)
    assert not any(
        event == "scalar" and value == oversized
        for path, event, value in observed if path != ("id",)
    )


def test_selected_strings_remain_bounded_without_lossless_opt_in() -> None:
    observed = []
    assert project_json_reader(
        io.BytesIO(json.dumps({"id": "x" * (JSON_STRING_TOKEN_BYTES + 1)}).encode()),
        {(), ("id",)},
        lambda path, event, value, _scope: observed.append((path, event, value)),
    )
    assert (("id",), "elided_string", None) in observed


@pytest.mark.parametrize("suffix", (b'\\q"}', b'\\u000"}', b'\xff"}', b'"', b'"},', b'","other":[,]}'))
def test_lossless_retention_does_not_relax_json_completion_or_grammar(suffix: bytes) -> None:
    assert not project_json_reader(
        io.BytesIO(b'{"id":"' + b"x" * (JSON_STRING_TOKEN_BYTES + 1) + suffix),
        {(), ("id",)},
        lambda *_args: None,
        lossless_string_paths={("id",)},
    )


@pytest.mark.parametrize(
    "payload",
    (
        b'{"wanted":1,}',
        b'{"ignored":[{"nested":1}],"wanted":2,}',
        b'{"ignored":[}',
        b'{"ignored":"bad\\q","wanted":3}',
        b'{"ignored":01,"wanted":4}',
        b'{"ignored":1e,"wanted":5}',
        b'{"ignored":"\xff","wanted":6}',
        b'{"ignored":[,,true],"wanted":7}',
        b'{"ignored":{"key" 1},"wanted":8}',
        b'{"ignored":[true false],"wanted":9}',
    ),
)
def test_selective_projection_does_not_accept_invalid_ignored_json(payload: bytes) -> None:
    assert not project_json_reader(io.BytesIO(payload), {(), ("wanted",)}, lambda *_args: None)


def test_selective_projection_skips_large_values_but_keeps_later_selected_facts() -> None:
    payload = json.dumps(
        {
            "ignored_string": "x" * (2 * 1024 * 1024),
            "ignored_tree": [[{"value": index}] for index in range(20_000)],
            "wanted": "kept",
        },
        separators=(",", ":"),
    ).encode()
    observed: list[object] = []

    valid = project_json_reader(
        io.BytesIO(payload),
        {(), ("wanted",)},
        lambda path, event, value, _scope: (
            observed.append(value)
            if path == ("wanted",) and event == "scalar"
            else None
        ),
    )

    assert valid is True
    assert observed == ["kept"]


def test_selective_projection_scans_large_ascii_strings_in_chunks() -> None:
    parser = SelectiveJSONParser({(), ("wanted",)}, lambda *_args: None)
    parser.feed(b'{"ignored":"')

    delegate = parser._string_decoder

    class CountingDecoder:
        calls = 0

        def decode(self, payload: bytes, final: bool = False) -> str:
            self.calls += 1
            return delegate.decode(payload, final=final)

        def getstate(self):
            return delegate.getstate()

        def reset(self) -> None:
            delegate.reset()

    decoder = CountingDecoder()
    parser._string_decoder = decoder
    parser.feed(b"x" * (2 * 1024 * 1024))
    parser.feed(b'","wanted":true}')

    assert parser.finish() is True
    assert decoder.calls < 10


def test_selective_projection_packs_deep_ignored_grammar_state() -> None:
    parser = SelectiveJSONParser({(), ("wanted",)}, lambda *_args: None)
    parser.feed(b'{"ignored":' + (b"[" * 100_000))

    assert parser.retained_bytes < 64 * 1024
    assert parser.finish() is False


def test_selective_projection_closes_deep_ignored_grammar_state() -> None:
    parser = SelectiveJSONParser({(), ("wanted",)}, lambda *_args: None)
    parser.feed(b'{"ignored":' + (b"[" * 100_000))
    parser.feed(b"0" + (b"]" * 100_000) + b',"wanted":true}')

    assert parser.finish() is True
    assert parser.retained_bytes == 0


def test_selective_projection_accepts_one_leading_utf8_bom() -> None:
    observed: list[object] = []

    valid = project_json_reader(
        io.BytesIO(b'\xef\xbb\xbf{"wanted":true}'),
        {(), ("wanted",)},
        lambda path, event, value, _scope: (
            observed.append(value)
            if path == ("wanted",) and event == "scalar"
            else None
        ),
    )

    assert valid is True
    assert observed == [True]


def test_duplicate_wildcard_members_replace_only_their_array_occurrence() -> None:
    observed: list[tuple[tuple[str, ...], str, object, tuple[int, ...]]] = []
    payload = (
        b'{"choices":[{"delta":{"content":"first"}},'
        b'{"delta":{"content":"stale","content":null}}]}'
    )

    valid = project_json_reader(
        io.BytesIO(payload),
        {(), ("choices", "*", "delta", "content")},
        lambda path, event, value, scope: observed.append((path, event, value, scope)),
    )

    content = [
        (event, value, scope)
        for path, event, value, scope in observed
        if path == ("choices", "*", "delta", "content")
    ]
    assert valid is True
    assert content == [
        ("scalar", "first", (0,)),
        ("scalar", "stale", (1,)),
        ("replace", None, (1,)),
        ("scalar", None, (1,)),
    ]


def test_wildcard_scope_ends_after_each_array_member() -> None:
    scopes: list[tuple[int, ...]] = []

    valid = project_json_reader(
        io.BytesIO(b'{"choices":[{"value":1},{"value":2}]}'),
        {(), ("choices", "*", "value")},
        lambda _path, event, _value, scope: (
            scopes.append(scope) if event == "scope_end" else None
        ),
    )

    assert valid is True
    assert scopes == [(0,), (1,)]


def test_path_rewriter_changes_only_selected_string_values() -> None:
    payload = b'{"content":"alias","tool":{"name":"alias"},"other":{"name":"alias"}}'
    rewritten = rewrite_json_strings(
        io.BytesIO(payload),
        target_paths={("tool", "name")},
        replacements={"alias": "original"},
    )

    assert rewritten is not None
    try:
        assert json.load(rewritten) == {
            "content": "alias",
            "tool": {"name": "original"},
            "other": {"name": "alias"},
        }
    finally:
        rewritten.close()