Skip to content

Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.

17.5 KB
Raw Blame
"""Streaming, rollout, cache failure and startup acceptance scenarios."""

import asyncio
import json
import time
from concurrent.futures import ThreadPoolExecutor

import httpx
import pytest

from .harness import (
    CHAT,
    chat,
    control,
    credential_keys,
    document,
    records,
    session_keys,
    wait_for,
    written_messages,
)
from .sdk_probe import probe


def stream_body(model="mock-long", messages=None):
    return {
        "model": model,
        "stream": True,
        "messages": messages or [{"role": "user", "content": "stream continuity"}],
    }


def read_stream(
    harness, pod, model="mock-silent", session="stream", after_first=None, messages=None
):
    lines, timestamps = [], []
    try:
        with harness.stream(
            pod, stream_body(model, messages), headers={"X-OpenViking-Session": session}
        ) as response:
            encoding = response.headers.get("content-encoding")
            for line in response.iter_lines():
                if line:
                    lines.append(line)
                    timestamps.append(time.monotonic())
                    if len(lines) == 1 and after_first:
                        after_first()
    except httpx.RemoteProtocolError:
        return {"lines": lines, "times": timestamps, "truncated": True}
    return {"lines": lines, "times": timestamps, "encoding": encoding, "truncated": False}


def test_02_default_grace_finishes_stream_and_other_pod_delivers(harness):
    harness.redeploy({"default_profile": {"idle_seconds": 45}})
    victim, _ = harness.pods()
    victim_ip = next(
        p["status"]["podIP"] for p in harness.pod_objects() if p["metadata"]["name"] == victim
    )
    history = [{"role": "user", "content": "first pending turn"}]
    history.append(chat(harness, victim, "graceful", history))
    history.append({"role": "user", "content": "finish during termination"})
    result = read_stream(
        harness,
        victim,
        session="graceful",
        messages=history,
        after_first=lambda: harness.kubectl("delete", "pod", victim, "--wait=false"),
    )
    assert "data: [DONE]" in result["lines"] and not result["truncated"]
    snapshot = wait_for(
        lambda: value if len(written_messages(value := records(harness))) == 4 else None,
        timeout=110,
    )
    assert len(snapshot["libraries"]["library-a"]["sessions"]) == 1
    writes = [r for r in snapshot["requests"] if r["path"].endswith("/messages/batch")]
    ids = [
        source for r in writes for m in r["body"]["messages"] for source in m["source_message_ids"]
    ]
    assert len(ids) == len(set(ids)) == 4
    # The next request confirms the first turn, so the victim may deliver it
    # immediately. Only the final, idle-delayed tail must survive on another Pod.
    tail_ids = {
        source
        for message in written_messages(snapshot)[-2:]
        for source in message["source_message_ids"]
    }
    tail_writes = [
        r
        for r in writes
        if any(tail_ids.intersection(m["source_message_ids"]) for m in r["body"]["messages"])
    ]
    assert tail_writes and all(r["source_pod"] != victim_ip for r in tail_writes), tail_writes


@pytest.mark.standalone
def test_03_short_grace_truncates_stream_without_splitting_next_turn(harness):
    victim, _ = harness.pods()
    result = read_stream(
        harness,
        victim,
        model="mock-long",
        session="short-grace",
        after_first=lambda: harness.kubectl(
            "delete", "pod", victim, "--grace-period=25", "--wait=false"
        ),
    )
    assert "data: [DONE]" not in result["lines"], "the negative control must outlive the 25s grace"
    assert result["times"][-1] - result["times"][0] < 40
    pod = next(p for p in harness.pods() if p != victim)
    chat(harness, pod, "short-grace", [{"role": "user", "content": "continue after truncation"}])
    _, capture = session_keys(harness, "short-grace")
    wait_for(lambda: not document(harness, capture).get("pending"))
    sessions = records(harness)["libraries"]["library-a"]["sessions"]
    assert len(sessions) == 1
    assert len(next(iter(sessions.values()))["messages"]) == 2


ROLLOUT_CLIENT = """
import asyncio, http.client, json, time
import aiohttp
base = "http://gateway:1935"
headers = {"Authorization": "Bearer key-a"}
results = {"short": 0, "chat": 0, "keepalive": 0, "keepalive_reuses": 0, "keepalive_connections": 0, "raw_keepalive_reused": False, "errors": [], "done": False}
async def lane(name):
    async with aiohttp.ClientSession(connector=aiohttp.TCPConnector(force_close=True)) as client:
        for i in range(120):
            try:
                if name == "short":
                    response = await client.get(base + "/health")
                else:
                    response = await client.post(base + "/gateway/v1/chat/completions", headers=headers,
                        json={"model":"mock", "messages":[{"role":"user","content":str(i)}]})
                await response.read()
                assert response.status == 200, response.status
                results[name] += 1
            except Exception as error:
                results["errors"].append([name, type(error).__name__, str(error)])
            await asyncio.sleep(.5)
async def keepalive():
    trace = aiohttp.TraceConfig()
    async def reused(session, context, params):
        results["keepalive_reuses"] += 1
    async def created(session, context, params):
        results["keepalive_connections"] += 1
    trace.on_connection_reuseconn.append(reused)
    trace.on_connection_create_end.append(created)
    async with aiohttp.ClientSession(connector=aiohttp.TCPConnector(keepalive_timeout=75),
            timeout=aiohttp.ClientTimeout(total=15), trace_configs=[trace]) as client:
        for _ in range(10):
            try:
                async with client.get(base + "/health") as response:
                    await response.read()
                    assert response.status == 200
                    results["keepalive"] += 1
            except Exception as error:
                results["errors"].append(["keepalive", type(error).__name__, str(error)])
            await asyncio.sleep(6)
async def long_stream():
    async with aiohttp.ClientSession() as client:
        async with client.post(base + "/gateway/v1/chat/completions", headers=headers,
            json={"model":"mock-long","stream":True,"messages":[{"role":"user","content":"long rollout"}]}) as r:
            assert r.status == 200
            body = await r.text()
            results["done"] = "data: [DONE]" in body
def raw_idle_reuse():
    connection = http.client.HTTPConnection("RAW_GATEWAY_HOST", 1935, timeout=15)
    try:
        connection.request("GET", "/health")
        response = connection.getresponse()
        response.read()
        assert response.status == 200
        original = connection.sock
        assert original is not None
        time.sleep(6)
        connection.request("GET", "/health")
        response = connection.getresponse()
        response.read()
        assert response.status == 200 and connection.sock is original
        results["raw_keepalive_reused"] = True
    finally:
        connection.close()
async def main():
    await asyncio.to_thread(raw_idle_reuse)
    await asyncio.gather(lane("short"), lane("chat"), keepalive(), long_stream())
    print(json.dumps(results))
asyncio.run(main())
"""


def test_04_rollout_preserves_three_traffic_lanes_and_long_stream(harness):
    runner = json.loads(harness.kubectl("get", "pods", "-l", "app=mock-ov", "-o", "json"))["items"][
        0
    ]["metadata"]["name"]
    with ThreadPoolExecutor() as executor:
        running = executor.submit(
            harness.kubectl,
            "exec",
            "-i",
            runner,
            "--",
            "python",
            "-",
            input=ROLLOUT_CLIENT.replace(
                "RAW_GATEWAY_HOST", harness.pod_objects()[0]["status"]["podIP"]
            ),
            timeout=110,
        )
        wait_for(
            lambda: any(
                r["body"].get("model") == "mock-long" for r in records(harness, "ark")["requests"]
            )
        )
        harness.kubectl("rollout", "restart", "deployment/gateway")
        result = json.loads(running.result(timeout=115))
        (harness.directory / "rollout-traffic.json").write_text(json.dumps(result, indent=2))
    assert result["errors"] == [], result
    assert (result["short"], result["chat"], result["keepalive"]) == (120, 120, 10), result
    # A real pool observes the FIN when a draining Pod closes an idle connection.
    assert result["raw_keepalive_reused"], result
    assert result["keepalive_reuses"] >= 7, result
    assert 1 <= result["keepalive_connections"] <= 3, result
    assert result["done"], result
    harness.kubectl("rollout", "status", "deployment/gateway", "--timeout=120s")


@pytest.mark.standalone
def test_05_eviction_keeps_permanent_state_and_recovers_credentials(harness):
    pods = harness.pods()
    permanent = "ovgw:v1:m:{a-e2e}:o:policies:permanent"
    tombstone = "ovgw:v1:kd:{deleted-e2e}"
    ready = "ovgw:v1:k:{cap}:ready"
    harness.redis("HSET", permanent, "v", '{"name":"permanent"}', "rev", 1)
    harness.redis("SET", tombstone, 1)
    harness.redis("CONFIG", "SET", "maxmemory", "30mb")
    big = "retained context " * 8192

    def send(number):
        return chat(
            harness,
            pods[number % 2],
            f"eviction-{number}",
            [{"role": "user", "content": f"{number}: {big}"}],
            idle_seconds=120,
        )

    with ThreadPoolExecutor(max_workers=12) as executor:
        list(executor.map(send, range(200)))
    stats = dict(
        line.split(":", 1) for line in harness.redis("INFO", "stats").splitlines() if ":" in line
    )
    assert int(stats["evicted_keys"]) > 0, stats
    assert harness.redis("EXISTS", permanent) == "1"
    assert harness.redis("EXISTS", tombstone) == "1"
    assert harness.redis("TTL", permanent) == harness.redis("TTL", tombstone) == "-1"
    assert harness.redis("TYPE", ready) == "zset" and harness.redis("TTL", ready) == "-1"
    chat(harness, pods[1], "eviction-0", [{"role": "user", "content": "continue after eviction"}])
    harness.redis("CONFIG", "SET", "maxmemory", "96mb")
    chat(
        harness,
        pods[0],
        "credential-rebuild",
        [{"role": "user", "content": "pending before key deletion"}],
    )
    _, capture = session_keys(harness, "credential-rebuild")
    for key in credential_keys(harness, "key-a"):
        harness.redis("DEL", key)
    wait_for(
        lambda: (
            (document(harness, capture).get("error") or {}).get("reason") == "gateway_key_revoked"
        ),
        timeout=40,
    )
    chat(
        harness,
        pods[1],
        "credential-rebuild",
        [{"role": "user", "content": "new request restores key"}],
    )
    assert credential_keys(harness, "key-a")
    wait_for(
        lambda: (
            not document(harness, capture).get("error")
            and not document(harness, capture).get("pending")
        ),
        timeout=90,
    )
    assert "KeyError" not in harness.logs()


def stop_redis(harness, aof):
    harness.kubectl("scale", "statefulset/redis", "--replicas=0")
    harness.kubectl("wait", "--for=delete", "pod/redis-0", "--timeout=40s")
    if not aof:
        value = json.loads(harness.kubectl("get", "statefulset/redis", "-o", "json"))
        args = value["spec"]["template"]["spec"]["containers"][0]["args"]
        args[args.index("--appendonly") + 1] = "no"
        harness.kubectl(
            "patch",
            "statefulset/redis",
            "--type=json",
            "-p",
            json.dumps(
                [{"op": "replace", "path": "/spec/template/spec/containers/0/args", "value": args}]
            ),
        )


def outage_requests(harness, pod):
    counts = {"degraded": 0, "authentication_500": 0}
    for number in range(5):
        response = harness.request(
            pod,
            "POST",
            CHAT,
            json={
                "model": "mock",
                "messages": [{"role": "user", "content": f"redis outage {number}"}],
            },
        )
        if response.status_code == 200:
            counts["degraded"] += 1
        else:
            assert response.status_code == 500 and response.headers["content-type"].startswith(
                "text/plain"
            ), response.text
            counts["authentication_500"] += 1
    (harness.directory / "redis-outage.json").write_text(json.dumps(counts))
    if counts["degraded"]:
        assert "memory_store_failure" in harness.logs()
    return counts


@pytest.mark.standalone
def test_06_aof_restart_preserves_management_tombstones_and_pending_capture(harness):
    harness.redeploy({"default_profile": {"idle_seconds": 45}})
    pod = harness.pods()[0]
    chat(harness, pod, "aof-tail", [{"role": "user", "content": "persist this pending tail"}])
    _, capture = session_keys(harness, "aof-tail")
    keys = credential_keys(harness, "key-a")
    tombstone = "ovgw:v1:kd:{aof-deleted-scope}"
    harness.redis("SET", tombstone, 1)
    stop_redis(harness, aof=True)
    outage_requests(harness, pod)
    harness.kubectl("scale", "statefulset/redis", "--replicas=1")
    harness.kubectl("rollout", "status", "statefulset/redis", "--timeout=90s")
    assert harness.redis("EXISTS", tombstone) == "1"
    assert all(harness.redis("EXISTS", key) == "1" for key in keys)
    assert document(harness, capture)
    wait_for(lambda: len(written_messages(records(harness))) >= 2, timeout=90)
    assert not document(harness, capture).get("pending")


@pytest.mark.standalone
def test_07_restart_without_aof_loses_tail_but_recreates_key(harness):
    harness.redeploy({"default_profile": {"idle_seconds": 120}})
    pod = harness.pods()[0]
    chat(harness, pod, "lost-tail", [{"role": "user", "content": "nonpersistent pending tail"}])
    _, capture = session_keys(harness, "lost-tail")
    old_keys = harness.redis_keys("ovgw:*")
    assert old_keys and document(harness, capture)["pending"]
    stop_redis(harness, aof=False)
    harness.kubectl("scale", "statefulset/redis", "--replicas=1")
    harness.kubectl("rollout", "status", "statefulset/redis", "--timeout=90s")
    assert harness.redis("DBSIZE") == "0"
    chat(harness, pod, "fresh-after-restart", [{"role": "user", "content": "recreated credential"}])
    assert credential_keys(harness, "key-a") and not document(harness, capture)
    assert all(
        all(c["restartCount"] == 0 for c in p["status"]["containerStatuses"])
        for p in harness.pod_objects()
    )


def rollout_candidates(harness, marker):
    pods = json.loads(
        harness.kubectl("get", "pods", "-l", "app.kubernetes.io/instance=gateway", "-o", "json")
    )["items"]
    return [p for p in pods if p["metadata"].get("annotations", {}).get("e2e-startup") == marker]


@pytest.mark.standalone
def test_10_startup_waits_for_503_but_rejects_dev_and_404(harness):
    control(harness, "/_fail", status=503)
    harness.redeploy({}, {"podAnnotations": {"e2e-startup": "unavailable"}}, wait=False)
    pods = wait_for(lambda: rollout_candidates(harness, "unavailable"))
    pod = pods[0]["metadata"]["name"]
    wait_for(lambda: "waiting" in harness.kubectl("logs", pod, check=False).lower(), timeout=45)
    time.sleep(5)
    status = rollout_candidates(harness, "unavailable")[0]["status"]
    assert all(c["restartCount"] == 0 for c in status.get("containerStatuses", []))
    assert not any(c.get("ready") for c in status.get("containerStatuses", []))
    restored = time.monotonic()
    control(harness, "/_fail", on=False)
    # Time only the waiting pod's recovery; the rest of the rollout starts pods from scratch.
    wait_for(lambda: pod in harness.pods(), timeout=40)
    assert time.monotonic() - restored < 30
    harness.kubectl("rollout", "status", "deployment/gateway", "--timeout=120s", timeout=130)
    for marker, config in (
        ("dev", {}),
        ("notfound", {"openviking_url": "http://mock-ov:1933/notfound"}),
    ):
        control(harness, "/_auth_mode", mode="dev" if marker == "dev" else "api_key")
        harness.redeploy(config, {"podAnnotations": {"e2e-startup": marker}}, wait=False)

        def refused(marker=marker):
            for candidate in rollout_candidates(harness, marker):
                for container in candidate["status"].get("containerStatuses", []):
                    terminated = container.get("state", {}).get("terminated") or container.get(
                        "lastState", {}
                    ).get("terminated")
                    if terminated and terminated["exitCode"] != 0:
                        return candidate

        wait_for(refused, timeout=35)


@pytest.mark.standalone
def test_11_sse_heartbeats_preserve_actual_client_parsers(harness):
    pod = harness.pods()[0]
    with ThreadPoolExecutor(max_workers=2) as executor:
        plain = executor.submit(read_stream, harness, pod, "mock-silent", "heartbeat-raw")
        encoded = executor.submit(read_stream, harness, pod, "mock-silent-gzip", "heartbeat-gzip")
        result = asyncio.run(probe(harness.url(pod) + "/gateway", "key-a"))
        (harness.directory / "sdk-probe.json").write_text(json.dumps(result, indent=2))
        assert result["passed"], result
        plain, encoded = plain.result(), encoded.result()
    comments = [
        stamp
        for line, stamp in zip(plain["lines"], plain["times"], strict=True)
        if line.startswith(":")
    ]
    assert len(comments) >= 2, plain
    assert 10 <= comments[0] - plain["times"][0] <= 20
    assert 10 <= comments[1] - comments[0] <= 20
    assert "data: [DONE]" in plain["lines"]
    assert encoded["encoding"] == "gzip"
    assert not any(line.startswith(":") for line in encoded["lines"])
    assert "data: [DONE]" in encoded["lines"]