Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.
Python agent-contextagent-memoryagent-pluginsagentic-ragcontext-databasedsh-pluginknowledge-basememorymemory-managementself-evolvingvector-database
# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.
# SPDX-License-Identifier: AGPL-3.0
"""Tests for the Jev (TypeSafe System One) rerank client."""
import json
from unittest.mock import MagicMock, patch
import httpx
import pytest
from openviking.models.rerank import JevRerankClient, RerankClient
from openviking.models.rerank.jev_rerank import MODE_CHOICE, MODE_NOUL
from openviking_cli.utils.config.rerank_config import RerankConfig
def _mock_response(payload: dict, status_error=None):
response = MagicMock()
response.json.return_value = payload
response.raise_for_status = MagicMock(side_effect=status_error)
response.status_code = 401 if status_error else 200
response.text = "error" if status_error else ""
return response
def _systemone_payload(scores: list[float]) -> dict:
answers = {
f"relevance_{index}": {"type": "noul", "noul": score} for index, score in enumerate(scores)
}
return {
"model": "jev-1.13.0",
"answers": answers,
"usage": {"input_tokens": 100, "output_tokens": 20},
}
class TestJevRerankClient:
@patch("openviking.models.rerank.jev_rerank.httpx.Client")
def test_rerank_batch_basic(self, mock_client_class):
mock_client = MagicMock()
mock_client_class.return_value = mock_client
mock_client.post.return_value = _mock_response(_systemone_payload([0.97, 0.04, 0.07]))
client = JevRerankClient(api_key="test-key")
scores = client.rerank_batch("What is UCW?", ["doc A", "doc B", "doc C"])
assert client.mode == MODE_NOUL
assert client.candidates_visible is True
assert scores == [0.97, 0.04, 0.07]
body = mock_client.post.call_args[1]["json"]
assert body["model"] == "jev-latest"
assert body["state"] == {
"query": "What is UCW?",
"candidate_documents": ["doc A", "doc B", "doc C"],
}
assert len(body["questions"]) == 3
assert body["questions"]["relevance_0"]["type"] == "noul"
assert body["questions"]["relevance_2"]["instructions"]["candidate_index"] == 2
mock_client.post.assert_called_once_with("https://api.typesafe.ai/v1/systemone", json=body)
@patch("openviking.models.rerank.jev_rerank.httpx.Client")
def test_vercel_gateway_uses_typesafe_compatible_endpoint(self, mock_client_class):
"""Vercel is just another api_base; the wire protocol stays TypeSafe System One."""
mock_client = MagicMock()
mock_client_class.return_value = mock_client
mock_client.post.return_value = _mock_response(_systemone_payload([0.91, 0.08]))
client = JevRerankClient(
api_key="vercel-key",
api_base="https://ai-gateway.vercel.sh/typesafe/v1/systemone",
model_name="typesafe-ai/jev",
mode=MODE_NOUL,
)
assert client.rerank_batch("query", ["first", "second"]) == [0.91, 0.08]
call = mock_client.post.call_args
assert call.args == ("https://ai-gateway.vercel.sh/typesafe/v1/systemone",)
assert call.kwargs["json"]["model"] == "typesafe-ai/jev"
assert call.kwargs["json"]["questions"]["relevance_0"]["type"] == "noul"
assert "headers" not in call.kwargs
@patch("openviking.models.rerank.jev_rerank.httpx.Client")
def test_custom_api_base_appends_systemone_path(self, mock_client_class):
mock_client = MagicMock()
mock_client_class.return_value = mock_client
mock_client.post.return_value = _mock_response(_systemone_payload([0.8]))
client = JevRerankClient(
api_key="custom-key",
api_base="https://gateway.example.com/custom-jev",
)
assert client.rerank_batch("query", ["document"]) == [0.8]
mock_client.post.assert_called_once_with(
"https://gateway.example.com/custom-jev/v1/systemone",
json=mock_client.post.call_args.kwargs["json"],
)
@patch("openviking.models.rerank.jev_rerank.logger.warning")
@patch("openviking.models.rerank.jev_rerank.httpx.Client")
def test_logs_request_and_response_without_api_key(self, mock_client_class, mock_log):
mock_client = MagicMock()
mock_client_class.return_value = mock_client
mock_client.post.return_value = _mock_response(_systemone_payload([0.9]))
client = JevRerankClient(api_key="secret-key", log_payloads=True, mode=MODE_NOUL)
assert client.rerank_batch("query", ["document"]) == [0.9]
messages = [str(call) for call in mock_log.call_args_list]
assert any("[JevRerank] Request" in message for message in messages)
assert any("[JevRerank] Response" in message for message in messages)
assert all("secret-key" not in message for message in messages)
@patch("openviking.models.rerank.jev_rerank.logger.warning")
@patch("openviking.models.rerank.jev_rerank.httpx.Client")
def test_payload_logging_disabled_by_default(self, mock_client_class, mock_log):
mock_client = MagicMock()
mock_client_class.return_value = mock_client
mock_client.post.return_value = _mock_response(_systemone_payload([0.9]))
client = JevRerankClient(api_key="secret-key", mode=MODE_NOUL)
assert client.rerank_batch("query", ["document"]) == [0.9]
mock_log.assert_not_called()
@pytest.mark.parametrize("candidates_visible", [True, False])
@patch("openviking.models.rerank.jev_rerank.httpx.Client")
def test_rerank_batch_empty(self, mock_client_class, candidates_visible):
client = JevRerankClient(api_key="test-key", candidates_visible=candidates_visible)
assert client.rerank_batch("query", []) == []
mock_client_class.return_value.post.assert_not_called()
@pytest.mark.parametrize("candidates_visible", [True, False])
@patch("openviking.models.rerank.jev_rerank.httpx.Client")
def test_rerank_batch_api_error_returns_none(self, mock_client_class, candidates_visible):
mock_client = MagicMock()
mock_client_class.return_value = mock_client
error = httpx.HTTPStatusError("401", request=MagicMock(), response=MagicMock())
mock_client.post.return_value = _mock_response({}, status_error=error)
client = JevRerankClient(
api_key="bad-key", mode=MODE_NOUL, candidates_visible=candidates_visible
)
assert client.rerank_batch("query", ["doc"]) is None
@pytest.mark.parametrize("candidates_visible", [True, False])
@patch("openviking.models.rerank.jev_rerank.httpx.Client")
def test_missing_answer_returns_none(self, mock_client_class, candidates_visible):
mock_client = MagicMock()
mock_client_class.return_value = mock_client
mock_client.post.return_value = _mock_response(
{"answers": {}, "usage": {"input_tokens": 1, "output_tokens": 1}}
)
client = JevRerankClient(
api_key="key", mode=MODE_NOUL, candidates_visible=candidates_visible
)
assert client.rerank_batch("q", ["doc"]) is None
@patch("openviking.models.rerank.jev_rerank.httpx.Client")
def test_each_document_gets_an_independent_question(self, mock_client_class):
mock_client = MagicMock()
mock_client_class.return_value = mock_client
mock_client.post.return_value = _mock_response(_systemone_payload([0.2, 0.8, 0.4]))
client = JevRerankClient(api_key="key", mode=MODE_NOUL)
scores = client.rerank_batch("q", ["a", "b", "c"])
assert scores == [0.2, 0.8, 0.4]
questions = mock_client.post.call_args[1]["json"]["questions"]
assert set(questions) == {"relevance_0", "relevance_1", "relevance_2"}
@pytest.mark.parametrize("candidates_visible", [True, False])
@patch("openviking.models.rerank.jev_rerank.httpx.Client")
def test_invalid_score_returns_none(self, mock_client_class, candidates_visible):
mock_client = MagicMock()
mock_client_class.return_value = mock_client
mock_client.post.return_value = _mock_response(_systemone_payload([1.1]))
client = JevRerankClient(
api_key="key", mode=MODE_NOUL, candidates_visible=candidates_visible
)
assert client.rerank_batch("q", ["doc"]) is None
@patch("openviking.models.rerank.jev_rerank.httpx.Client")
def test_close(self, mock_client_class):
mock_client = MagicMock()
mock_client_class.return_value = mock_client
client = JevRerankClient(api_key="key")
client.close()
mock_client.close.assert_called_once()
class TestIsolatedNoulRerank:
@pytest.mark.parametrize("count", [1, 3, 50])
@patch("openviking.models.rerank.jev_rerank.httpx.Client")
def test_candidates_only_appear_in_their_own_questions(self, mock_client_class, count):
documents = [f'candidate-{index:03d}: "内容"\nnext line' for index in range(count)]
expected_scores = [(index + 1) / count for index in range(count)]
payload = _systemone_payload(expected_scores)
payload["answers"] = dict(reversed(payload["answers"].items()))
mock_client_class.return_value.post.return_value = _mock_response(payload)
client = JevRerankClient(api_key="key", candidates_visible=False)
assert client.rerank_batch("retrieval query", documents) == expected_scores
body = mock_client_class.return_value.post.call_args.kwargs["json"]
assert body["state"] == {"query": "retrieval query"}
assert list(body["questions"]) == [f"relevance_{index}" for index in range(count)]
for index, document in enumerate(documents):
question = body["questions"][f"relevance_{index}"]
assert question["type"] == MODE_NOUL
assert question["instructions"]["candidate_document"] == document
assert "candidate_index" not in question["instructions"]
assert "candidate_documents" not in json.dumps(question)
assert question["criteria"] == {
"true": "The candidate directly answers or is relevant to the query",
"false": "The candidate does not answer and is not relevant to the query",
}
serialized = json.dumps(question, ensure_ascii=False)
assert all(
f"candidate-{other:03d}" not in serialized
for other in range(count)
if other != index
)
mock_client_class.return_value.post.assert_called_once_with(
"https://api.typesafe.ai/v1/systemone", json=body
)
# Caller-owned documents are not modified while building isolated questions.
assert documents == [f'candidate-{index:03d}: "内容"\nnext line' for index in range(count)]
@patch("openviking.models.rerank.jev_rerank.httpx.Client")
def test_duplicate_and_empty_candidates_keep_distinct_question_ids(self, mock_client_class):
mock_client_class.return_value.post.return_value = _mock_response(
_systemone_payload([0.7, 0.7, 0.0])
)
client = JevRerankClient(api_key="key", candidates_visible=False)
assert client.rerank_batch("q", ["same", "same", ""]) == [0.7, 0.7, 0.0]
body = mock_client_class.return_value.post.call_args.kwargs["json"]
assert body["state"] == {"query": "q"}
assert [
question["instructions"]["candidate_document"]
for question in body["questions"].values()
] == ["same", "same", ""]
@patch("openviking.models.rerank.jev_rerank.httpx.Client")
def test_choice_rejects_isolation_before_client_creation(self, mock_client_class):
with pytest.raises(ValueError, match="choice rerank requires candidates_visible=True"):
JevRerankClient(api_key="key", mode=MODE_CHOICE, candidates_visible=False)
mock_client_class.assert_not_called()
class TestChoiceRerank:
@patch("openviking.models.rerank.jev_rerank.httpx.Client")
def test_choice_preserves_document_order(self, mock_client_class):
mock_client = mock_client_class.return_value
mock_client.post.return_value = _mock_response(
{
"answers": {
"relevance": {
"type": "choice",
"choice": "candidate_1",
"confidence": 0.55,
"probabilities": {
"candidate_2": 0.2,
"candidate_1": 0.7,
"candidate_0": 0.1,
},
}
},
"usage": {"input_tokens": None, "output_tokens": 1},
}
)
client = JevRerankClient(api_key="key", mode=MODE_CHOICE)
assert client.mode == MODE_CHOICE
assert client.rerank_batch("query", ["first", "second", "third"]) == [0.1, 0.7, 0.2]
body = mock_client.post.call_args.kwargs["json"]
assert body["state"] == {
"query": "query",
"candidate_documents": ["first", "second", "third"],
}
assert body["questions"] == {
"relevance": {
"type": "choice",
"instructions": (
"Which candidate document best answers or matches the retrieval intent of query?"
),
"criteria": {
"candidate_0": (
"candidate_documents[0] directly answers or is relevant to the query"
),
"candidate_1": (
"candidate_documents[1] directly answers or is relevant to the query"
),
"candidate_2": (
"candidate_documents[2] directly answers or is relevant to the query"
),
},
}
}
mock_client.post.assert_called_once_with("https://api.typesafe.ai/v1/systemone", json=body)
@pytest.mark.parametrize("count", [1, 30])
@patch("openviking.models.rerank.jev_rerank.httpx.Client")
def test_choice_does_not_impose_candidate_limit(self, mock_client_class, count):
probabilities = {f"candidate_{index}": 1 / count for index in range(count)}
mock_client_class.return_value.post.return_value = _mock_response(
{"answers": {"relevance": {"type": "choice", "probabilities": probabilities}}}
)
client = JevRerankClient(api_key="key", mode=MODE_CHOICE)
assert client.rerank_batch("query", ["doc"] * count) == [1 / count] * count
criteria = mock_client_class.return_value.post.call_args.kwargs["json"]["questions"][
"relevance"
]["criteria"]
assert list(criteria) == list(probabilities)
@patch("openviking.models.rerank.jev_rerank.httpx.Client")
def test_choice_empty_input_does_not_send_request(self, mock_client_class):
client = JevRerankClient(api_key="key", mode=MODE_CHOICE)
assert client.rerank_batch("query", []) == []
mock_client_class.return_value.post.assert_not_called()
@pytest.mark.parametrize(
"answer",
[
None,
{"type": "noul", "noul": 0.8},
{"type": "choice", "choice": "candidate_0", "confidence": 0.8},
{"type": "choice", "probabilities": []},
{"type": "choice", "probabilities": {"candidate_0": 0.8}},
{
"type": "choice",
"probabilities": {"candidate_0": 0.8, "candidate_1": "0.2"},
},
{"type": "choice", "probabilities": {"candidate_0": 0.8, "candidate_1": True}},
{"type": "choice", "probabilities": {"candidate_0": 0.8, "candidate_1": -0.1}},
{"type": "choice", "probabilities": {"candidate_0": 0.8, "candidate_1": 1.1}},
{
"type": "choice",
"probabilities": {"candidate_0": 0.8, "candidate_1": float("nan")},
},
],
)
@patch("openviking.models.rerank.jev_rerank.httpx.Client")
def test_invalid_choice_answer_returns_none(self, mock_client_class, answer):
mock_client_class.return_value.post.return_value = _mock_response(
{"answers": {"relevance": answer}}
)
client = JevRerankClient(api_key="key", mode=MODE_CHOICE)
assert client.rerank_batch("query", ["first", "second"]) is None
mock_client_class.return_value.post.assert_called_once()
@patch("openviking.models.rerank.jev_rerank.httpx.Client")
def test_unknown_mode_rejected_before_client_creation(self, mock_client_class):
with pytest.raises(ValueError, match="Unknown Jev rerank mode"):
JevRerankClient(api_key="key", mode="unknown")
mock_client_class.assert_not_called()
class TestJevRerankConfig:
def test_explicit_jev_provider(self):
config = RerankConfig(provider="jev", api_key="key")
assert config._effective_provider() == "jev"
assert config.is_available() is True
assert config.mode == MODE_NOUL
assert config.candidates_visible is True
@pytest.mark.parametrize("mode", [MODE_CHOICE, "CHOICE"])
def test_choice_rejects_isolated_candidates(self, mode):
with pytest.raises(ValueError, match="choice rerank requires candidates_visible=True"):
RerankConfig(provider="jev", api_key="key", mode=mode, candidates_visible=False)
def test_inferred_jev_provider_rejects_choice_isolation(self):
with pytest.raises(ValueError, match="choice rerank requires candidates_visible=True"):
RerankConfig(
api_key="key",
api_base="https://api.typesafe.ai",
mode=MODE_CHOICE,
candidates_visible=False,
)
@pytest.mark.parametrize("mode", [MODE_CHOICE, "CHOICE"])
def test_explicit_choice_mode(self, mode):
config = RerankConfig(provider="jev", api_key="key", mode=mode)
assert config.mode == MODE_CHOICE
def test_null_mode_uses_client_default(self):
config = RerankConfig(provider="jev", api_key="key", mode=None)
assert config.mode is None
def test_unknown_mode_rejected(self):
with pytest.raises(ValueError, match="Jev rerank mode"):
RerankConfig(provider="jev", api_key="key", mode="unknown")
def test_non_jev_provider_does_not_validate_jev_mode(self):
config = RerankConfig(
provider="openai",
api_key="key",
api_base="https://example.com/rerank",
mode="listwise",
)
assert config.mode == "listwise"
def test_jev_auto_detected_from_api_base(self):
config = RerankConfig(
api_key="key",
api_base="https://api.typesafe.ai/v1/systemone",
)
assert config._effective_provider() == "jev"
def test_jev_auto_detected_from_vercel_typesafe_base(self):
config = RerankConfig(
api_key="key",
api_base="https://ai-gateway.vercel.sh/typesafe/v1/systemone",
)
assert config._effective_provider() == "jev"
def test_plain_vercel_base_is_not_jev(self):
config = RerankConfig(api_key="key", api_base="https://ai-gateway.vercel.sh/v1")
assert config._effective_provider() == "openai"
def test_jev_requires_api_key(self):
with pytest.raises(ValueError, match="Jev"):
RerankConfig(provider="jev")
def test_unknown_provider_rejected(self):
with pytest.raises(ValueError, match="Rerank provider"):
RerankConfig(provider="nope", api_key="k")
class TestJevDispatch:
@pytest.mark.parametrize("candidates_visible", [True, False])
@pytest.mark.parametrize("mode", [MODE_NOUL, None])
@patch("openviking.models.rerank.jev_rerank.httpx.Client")
def test_from_config_passes_candidate_visibility(
self, mock_client_class, candidates_visible, mode
):
config = RerankConfig(
provider="jev", api_key="key", mode=mode, candidates_visible=candidates_visible
)
client = RerankClient.from_config(config)
assert client.candidates_visible is candidates_visible
mock_client_class.return_value.post.return_value = _mock_response(_systemone_payload([0.8]))
assert client.rerank_batch("q", ["doc"]) == [0.8]
body = mock_client_class.return_value.post.call_args.kwargs["json"]
if candidates_visible:
assert body["state"] == {"query": "q", "candidate_documents": ["doc"]}
else:
assert body["state"] == {"query": "q"}
assert body["questions"]["relevance_0"]["instructions"]["candidate_document"] == "doc"
@patch("openviking.models.rerank.jev_rerank.httpx.Client")
def test_from_config_creates_jev_client(self, mock_client_class):
config = RerankConfig(provider="jev", api_key="jev-key")
client = RerankClient.from_config(config)
assert isinstance(client, JevRerankClient)
assert client.api_key == "jev-key"
assert client.model_name == "jev-latest"
assert client.mode == MODE_NOUL
@pytest.mark.parametrize("mode", [MODE_CHOICE, None])
@patch("openviking.models.rerank.jev_rerank.httpx.Client")
def test_from_config_resolves_mode(self, mock_client_class, mode):
config = RerankConfig(provider="jev", api_key="key", mode=mode)
client = RerankClient.from_config(config)
assert client.mode == (mode or MODE_NOUL)
@patch("openviking.models.rerank.jev_rerank.httpx.Client")
def test_from_config_uses_custom_model(self, mock_client_class):
config = RerankConfig(provider="jev", api_key="key", model="jev-1.13.0")
client = RerankClient.from_config(config)
assert client.model_name == "jev-1.13.0"
@patch("openviking.models.rerank.jev_rerank.httpx.Client")
def test_from_config_enables_payload_logging(self, mock_client_class):
config = RerankConfig(provider="jev", api_key="key", log_payloads=True)
client = RerankClient.from_config(config)
assert client.log_payloads is True
@patch("openviking.models.rerank.jev_rerank.httpx.Client")
def test_from_config_uses_vercel_model_default(self, mock_client_class):
config = RerankConfig(
provider="jev",
api_key="key",
api_base="https://ai-gateway.vercel.sh/typesafe",
)
client = RerankClient.from_config(config)
assert client.model_name == "typesafe-ai/jev"
assert client.api_url == "https://ai-gateway.vercel.sh/typesafe/v1/systemone"