Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.
Python agent-contextagent-memoryagent-pluginsagentic-ragcontext-databasedsh-pluginknowledge-basememorymemory-managementself-evolvingvector-database
"""Cross-pod session, tenant, delivery and encryption acceptance scenarios."""
import json
import time
from collections import Counter
from concurrent.futures import ThreadPoolExecutor
import pytest
from .harness import (
ADMIN,
CHAT,
MODELS,
account,
chat,
control,
credential_keys,
document,
message_text,
records,
scope,
session_keys,
wait_for,
written_messages,
)
def assert_delivery(snapshot, expected, library="library-a", key="key-a"):
messages = written_messages(snapshot, library)
assert [(m["role"], message_text(m)) for m in messages] == [
(m["role"], m["content"]) for m in expected
]
# Inspect HTTP batches too: a mock that deduplicates IDs must not hide duplicate delivery.
writes = [
message
for request in snapshot["requests"]
if request["key"] == key
and request["path"].endswith("/messages/batch")
and request["status"] == 200
for message in request["body"]["messages"]
]
ids = [source for message in writes for source in message["source_message_ids"]]
assert len(ids) == len(expected)
assert set(Counter(ids).values()) == {1}
assert writes == messages
def test_01_session_continues_across_pods(harness):
pods = harness.pods()
assert len(pods) == 2
models = [harness.request(pod, "GET", MODELS, key="key-a") for pod in pods]
assert [r.status_code for r in models] == [200, 200]
assert models[0].json() == models[1].json()
messages = []
for turn in range(8):
messages.append({"role": "user", "content": f"continuity turn {turn}"})
messages.append(chat(harness, pods[turn % 2], "eight-turns", messages))
replay, capture = session_keys(harness, "eight-turns")
wait_for(lambda: not document(harness, capture).get("pending"))
snapshot = records(harness)
assert len(snapshot["libraries"]["library-a"]["sessions"]) == 1
assert_delivery(snapshot, messages)
assert harness.redis_keys(f"*k:{{{scope()}}}:rp:*") == [replay]
assert harness.redis_keys(f"*k:{{cap}}:{scope()}:*") == [capture]
assert document(harness, capture)["previous"] == []
def test_08_same_identity_libraries_are_isolated(harness):
pods = harness.pods()
histories = {"key-a": [], "key-b": []}
for turn in range(2):
for index, key in enumerate(histories):
history = histories[key]
history.append({"role": "user", "content": f"private {key} turn {turn}"})
history.append(chat(harness, pods[(turn + index) % 2], "same-client-id", history, key))
captures = {}
for key in histories:
_, captures[key] = session_keys(harness, "same-client-id", key)
wait_for(lambda key=key: not document(harness, captures[key]).get("pending"))
assert captures["key-a"] != captures["key-b"]
snapshot = records(harness)
for key, library in (("key-a", "library-a"), ("key-b", "library-b")):
assert len(snapshot["libraries"][library]["sessions"]) == 1
assert_delivery(snapshot, histories[key], library, key)
recalls = snapshot["libraries"][library]["recalls"]
assert len(recalls) == 2, recalls
assert all(key in json.dumps(recall) for recall in recalls)
assert document(harness, captures[key])["account"] == account(key)
ark = records(harness, "ark")["requests"]
for request in ark:
text = json.dumps(request["body"])
own = "library-a" if "private key-a" in text else "library-b"
other = "library-b" if own == "library-a" else "library-a"
assert own in text and other not in text, text
log = harness.logs()
assert account("key-a") in log and account("key-b") in log
@pytest.mark.standalone
def test_09_revocation_pauses_capture_and_replacement_uses_new_partition(harness):
pods = harness.pods()
chat(harness, pods[0], "revoked-tail", [{"role": "user", "content": "pending secret"}])
_, capture = session_keys(harness, "revoked-tail")
assert document(harness, capture)["pending"]
control(harness, "/_revoke", key="key-a")
wait_for(
lambda: (
(document(harness, capture).get("error") or {}).get("reason") == "gateway_key_revoked"
),
)
snapshot = records(harness)
refused = [r for r in snapshot["requests"] if r["key"] == "key-a" and r["status"] == 401]
assert any(r["path"].startswith("/api/v1/sessions") for r in refused), refused
assert not credential_keys(harness, "key-a")
response = harness.request(
pods[1],
"POST",
CHAT,
key="key-a",
json={
"model": "mock",
"messages": [{"role": "user", "content": "refused"}],
},
)
assert response.status_code == 401, response.text
count = sum(r["key"] == "key-a" for r in records(harness)["requests"])
# Exceed the first failure retry interval, proving the paused tail is not retried.
time.sleep(12)
assert sum(r["key"] == "key-a" for r in records(harness)["requests"]) == count
chat(
harness, pods[1], "revoked-tail", [{"role": "user", "content": "replacement"}], "key-a-new"
)
_, replacement = session_keys(harness, "revoked-tail", "key-a-new")
assert replacement != capture
assert document(harness, replacement)["account"] == account("key-a-new")
assert document(harness, capture)["error"]["retry_at"] is None
def test_12_capture_workers_deliver_one_hundred_sessions_once(harness):
pods = harness.pods()
for pod in pods:
assert harness.request(pod, "GET", "/health").status_code == 200
started = time.monotonic()
def send(number):
message = {"role": "user", "content": f"capture race {number:03d}"}
answer = chat(harness, pods[number % 2], f"capture-race-{number}", [message])
return message, answer
with ThreadPoolExecutor(max_workers=32) as executor:
expected = list(executor.map(send, range(100)))
snapshot = wait_for(
lambda: s if len(written_messages(s := records(harness))) == 200 else None,
timeout=120,
)
assert time.monotonic() - started < 180
sessions = snapshot["libraries"]["library-a"]["sessions"]
assert len(sessions) == 100
delivered = {
message_text(value["messages"][0]): value["messages"] for value in sessions.values()
}
assert len(delivered) == 100
for question, answer in expected:
assert [(m["role"], message_text(m)) for m in delivered[question["content"]]] == [
("user", question["content"]),
("assistant", answer["content"]),
]
writes = [r for r in snapshot["requests"] if r["path"].endswith("/messages/batch")]
ids = [
source for r in writes for m in r["body"]["messages"] for source in m["source_message_ids"]
]
assert len(ids) == 200 and set(Counter(ids).values()) == {1}
assert len({r["source_pod"] for r in writes}) == 2
captures = harness.redis_keys(f"*k:{{cap}}:{scope()}:*")
assert len(captures) == 100
for capture in captures:
value = document(harness, capture)
assert value["pending"] == [] and not value["error"]
assert len(harness.redis_keys(f"*k:{{{scope()}}}:rp:*")) == 100
@pytest.mark.standalone
def test_13_unordered_existing_key_never_reaches_upstream(harness):
pods = harness.pods()
chat(harness, pods[0], "ordered", [{"role": "user", "content": "initial request"}])
assert credential_keys(harness, "key-a")
control(harness, "/_unorder", key="key-a")
before = len(records(harness, "ark")["requests"])
response = harness.request(
pods[1],
"POST",
CHAT,
key="key-a",
json={
"model": "mock",
"messages": [{"role": "user", "content": "unpaid request"}],
},
)
assert response.status_code == 402, response.text
assert len(records(harness, "ark")["requests"]) == before
chat(harness, pods[1], "other-key", [{"role": "user", "content": "still ordered"}], "key-b")
assert len(records(harness, "ark")["requests"]) == before + 1
def test_14_tools_and_responses_cross_pods_with_evicted_receipt(harness):
harness.redeploy(config={"default_profile": {"gateway_tools": True}})
pods = harness.pods()
question = {"role": "user", "content": "look up blue"}
answer = chat(harness, pods[0], "hidden-tools", [question], model="mock-tools")
chat(harness, pods[1], "hidden-tools", [question], model="mock-tools")
calls = records(harness)["libraries"]["library-a"]["tool_calls"]
assert len(calls) == 1, calls
chat(
harness,
pods[1],
"hidden-tools",
[
question,
answer,
{
"role": "user",
"content": "look up blue again",
},
],
model="mock-tools",
)
assert len(records(harness)["libraries"]["library-a"]["tool_calls"]) == 2
_, capture = session_keys(harness, "hidden-tools")
assert document(harness, capture)["previous"] == []
previous = None
for turn in range(4):
body = {"model": "mock", "input": [{"role": "user", "content": f"response chain {turn}"}]}
if previous:
body["previous_response_id"] = previous
response = harness.request(
pods[turn % 2],
"POST",
"/gateway/v1/responses",
key="key-a",
json=body,
headers={"X-OpenViking-Session": "responses-chain"},
)
assert response.status_code == 200, response.text
previous = response.json()["id"]
assert previous
chain = [r for r in records(harness, "ark")["requests"] if r["path"] == "/v1/responses"]
assert len(chain) == 4
for turn, request in enumerate(chain):
effective = json.dumps(request["effective_input"])
assert all(f"response chain {earlier}" in effective for earlier in range(turn + 1))
control(harness, "/_tool_delay", seconds=8)
retry_history = [{"role": "user", "content": "evict this in-flight tool receipt"}]
baseline = len(records(harness)["libraries"]["library-a"]["tool_calls"])
with ThreadPoolExecutor(max_workers=2) as executor:
first = executor.submit(
chat, harness, pods[0], "evicted-tool", retry_history, model="mock-tools"
)
wait_for(
lambda: len(records(harness)["libraries"]["library-a"]["tool_calls"]) == baseline + 1,
interval=0.2,
)
receipts = [
key
for key in harness.redis_keys(f"*k:{{{scope()}}}:st:*")
if "owner" in document(harness, key) and "content" not in document(harness, key)
]
assert len(receipts) == 1, receipts
before = len(records(harness, "ark")["requests"])
second = executor.submit(
chat, harness, pods[1], "evicted-tool", retry_history, model="mock-tools"
)
wait_for(lambda: len(records(harness, "ark")["requests"]) > before, interval=0.2)
# The second provider response is immediate; allow its receipt waiter to start.
time.sleep(0.5)
assert not first.done() and not second.done()
assert harness.redis("DEL", receipts[0]).strip() == "1"
second.result(timeout=10)
first.result(timeout=15)
assert len(records(harness)["libraries"]["library-a"]["tool_calls"]) == baseline + 1
retry_requests = records(harness, "ark")["requests"][before:]
assert any("unknown outcome" in json.dumps(r["body"]) for r in retry_requests)
assert "KeyError" not in harness.logs()
@pytest.mark.standalone
def test_15_management_updates_and_revocations_propagate(harness):
harness.redeploy(config={"management_api": True, "tenancy": "account", "management_key": ""})
pods = harness.pods()
studio = harness.request(pods[0], "GET", "/studio/", key="key-a")
assert studio.status_code == 200 and "text/html" in studio.headers["content-type"]
upstream = {
"name": "managed",
"protocol": "chat",
"base_url": "http://mock-ark:8080/v1",
"api_key": "managed-test-key",
"models": ["mock"],
}
response = harness.request(
pods[0], "PUT", ADMIN + "/upstreams/managed", key="key-a", json=upstream
)
assert response.status_code == 200, response.text
response = harness.request(
pods[0],
"PUT",
ADMIN + "/policies/default",
key="key-a",
json={"name": "Default", "idle_seconds": 5},
)
assert response.status_code == 200, response.text
issued = harness.request(
pods[0],
"POST",
ADMIN + "/keys",
key="key-a",
json={
"name": "issued",
"openviking_key": "key-a",
"policy_id": "default",
"upstream_ids": ["managed"],
},
)
assert issued.status_code == 200, issued.text
secret, identifier = issued.json()["key"], issued.json()["id"]
chat(harness, pods[1], "managed-before", [{"role": "user", "content": "before update"}], secret)
upstream["models"] = ["mock", "mock-tools"]
response = harness.request(
pods[0], "PUT", ADMIN + "/upstreams/managed", key="key-a", json=upstream
)
assert response.status_code == 200, response.text
time.sleep(2.2)
response = harness.request(pods[1], "GET", MODELS, key=secret)
assert response.status_code == 200, response.text
assert {row["id"] for row in response.json()["data"]} == {"mock", "mock-tools"}
response = harness.request(pods[0], "DELETE", ADMIN + f"/keys/{identifier}", key="key-a")
assert response.status_code == 200, response.text
before = len(records(harness, "ark")["requests"])
time.sleep(2.2)
response = harness.request(
pods[1],
"POST",
CHAT,
key=secret,
json={
"model": "mock",
"messages": [{"role": "user", "content": "after revoke"}],
},
)
assert response.status_code == 401, response.text
assert len(records(harness, "ark")["requests"]) == before
@pytest.mark.standalone
def test_16_shared_root_key_decrypts_across_pods_and_missing_key_blocks_start(harness):
harness.redeploy(config={"encryption": {"enabled": True, "provider": "local"}})
# Plaintext Pods keep claiming captures during preStop and cannot decrypt them.
def gateway_pods():
listing = harness.kubectl(
"get", "pods", "-l", "app.kubernetes.io/instance=gateway", "-o", "json"
)
return json.loads(listing)["items"]
wait_for(lambda: len(gateway_pods()) == 2)
pods = harness.pods()
messages = [{"role": "user", "content": "encrypted session private marker"}]
messages.append(chat(harness, pods[0], "encrypted-session", messages))
replay, capture = session_keys(harness, "encrypted-session")
assert (
harness.redis(
"EVAL",
"return string.sub(redis.call('HGET',KEYS[1],ARGV[1]),1,4)",
"1",
replay,
"root:",
).strip()
== "OVE1"
)
root_hash = harness.redis(
"EVAL", "return redis.sha1hex(redis.call('HGET',KEYS[1],ARGV[1]))", "1", replay, "root:"
).strip()
messages.append({"role": "user", "content": "continue encrypted session"})
messages.append(chat(harness, pods[1], "encrypted-session", messages))
wait_for(lambda: len(written_messages(records(harness))) == 4)
snapshot = records(harness)
assert len(snapshot["libraries"]["library-a"]["sessions"]) == 1
assert_delivery(snapshot, messages)
assert session_keys(harness, "encrypted-session") == (replay, capture)
assert (
harness.redis(
"EVAL", "return redis.sha1hex(redis.call('HGET',KEYS[1],ARGV[1]))", "1", replay, "root:"
).strip()
== root_hash
)
harness.kubectl(
"patch",
"secret",
"gateway-secrets",
"--type=json",
"-p",
'[{"op":"remove","path":"/data/root-key"}]',
)
harness.redeploy(
config={}, values={"podAnnotations": {"e2e/missing-root-key": "true"}}, wait=False
)
old = set(pods)
# A new ReplicaSet cannot mount the root-key Secret item once the key is removed.
def rejected_pod():
result = json.loads(
harness.kubectl("get", "pods", "-l", "app.kubernetes.io/instance=gateway", "-o", "json")
)
for pod in result["items"]:
if pod["metadata"]["name"] in old:
continue
events = harness.kubectl(
"get",
"events",
"--field-selector",
"involvedObject.name=" + pod["metadata"]["name"],
"-o",
"json",
)
statuses = pod["status"].get("containerStatuses", [])
if (
"FailedMount" in events
and "root-key" in events
and not any(s.get("ready") for s in statuses)
):
return pod
return None
rejected = wait_for(rejected_pod, timeout=90)
assert not any(
c.get("type") == "Ready" and c.get("status") == "True"
for c in rejected["status"].get("conditions", [])
)