Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.
Python agent-contextagent-memoryagent-pluginsagentic-ragcontext-databasedsh-pluginknowledge-basememorymemory-managementself-evolvingvector-database
# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.
# SPDX-License-Identifier: Apache-2.0
"""Tests for shared model retry helpers."""
import asyncio
import httpx
import openai
import pytest
from openviking.models.model_error import ModelError, ModelErrorKind, model_error_boundary
from openviking.pyagfs.exceptions import (
AGFSClientError,
AGFSConnectionError,
AGFSNotADirectoryError,
AGFSTimeoutError,
)
from openviking.storage.errors import InvalidPathLockRequestError
from openviking.utils.exceptions import AllCredentialsFailedError
from openviking.utils.model_retry import (
ERROR_CLASS_AUTH,
ERROR_CLASS_CONTENT_SAFETY,
ERROR_CLASS_INPUT_TOO_LARGE,
ERROR_CLASS_PERMANENT,
ERROR_CLASS_QUOTA_EXCEEDED,
ERROR_CLASS_TRANSIENT,
ERROR_CLASS_UNKNOWN,
classify_api_error,
extract_metric_error_code,
retry_async,
retry_sync,
)
@pytest.mark.parametrize("wrapped", [False, True])
@pytest.mark.parametrize(
"error_type",
[
FileNotFoundError,
PermissionError,
IsADirectoryError,
NotADirectoryError,
AGFSNotADirectoryError,
InvalidPathLockRequestError,
],
)
@pytest.mark.parametrize("message", ["/resources/notes.txt", "timeout at /arbitrary/input.md"])
def test_classify_permanent_filesystem_errors(error_type, message, wrapped):
error = error_type(message)
if wrapped:
wrapper = RuntimeError("storage operation failed")
wrapper.__cause__ = error
error = wrapper
assert classify_api_error(error) == ERROR_CLASS_PERMANENT
@pytest.mark.parametrize("wrapped", [False, True])
@pytest.mark.parametrize(
("error", "expected"),
[
(AGFSTimeoutError("request timeout"), ERROR_CLASS_TRANSIENT),
(AGFSConnectionError("connection refused"), ERROR_CLASS_TRANSIENT),
(AGFSClientError("unexpected storage failure"), ERROR_CLASS_UNKNOWN),
],
)
def test_classify_other_agfs_errors(error, expected, wrapped):
if wrapped:
wrapper = RuntimeError("storage operation failed")
wrapper.__cause__ = error
error = wrapper
assert classify_api_error(error) == expected
def test_classify_api_error_recognizes_request_burst_too_fast():
assert classify_api_error(RuntimeError("RequestBurstTooFast")) == ERROR_CLASS_TRANSIENT
def test_classify_api_error_recognizes_timed_out_message():
assert classify_api_error(RuntimeError("Request timed out.")) == ERROR_CLASS_TRANSIENT
@pytest.mark.parametrize("error_type", [TimeoutError, asyncio.TimeoutError])
@pytest.mark.parametrize("wrapped", [False, True])
def test_classify_api_error_treats_bare_timeout_as_transient(error_type, wrapped):
error = error_type()
if wrapped:
wrapper = RuntimeError("vlm call failed")
wrapper.__cause__ = error
error = wrapper
assert classify_api_error(error) == ERROR_CLASS_TRANSIENT
class _ProviderError(RuntimeError):
def __init__(self, *, status_code=None, error_code=None, code=None, body=None):
super().__init__("provider request failed")
self.status_code = status_code
self.error_code = error_code
self.code = code
self.body = body
def test_extract_metric_error_code_prefers_structured_provider_code():
assert (
extract_metric_error_code(_ProviderError(status_code=429, code="RateLimitExceeded"))
== "429"
)
assert extract_metric_error_code(
_ProviderError(body={"error": {"code": "InvalidParameter"}})
) == ("InvalidParameter")
def test_extract_metric_error_code_uses_safe_fallbacks_only():
assert extract_metric_error_code(TimeoutError("request timed out")) == "timeout"
assert extract_metric_error_code(ConnectionError("connection reset")) == "connection_error"
wrapped_timeout = RuntimeError("request failed")
wrapped_timeout.__cause__ = TimeoutError("request timed out")
assert extract_metric_error_code(wrapped_timeout) == "timeout"
assert extract_metric_error_code(RuntimeError("request_id=not-a-metric-label")) == "unknown"
def test_all_credentials_failed_prefers_transient_over_auth():
err = AllCredentialsFailedError(
[
(
"cred-a",
ModelErrorKind.AUTHENTICATION_ERROR,
ModelError(ModelErrorKind.AUTHENTICATION_ERROR, "unauthorized"),
0,
),
(
"cred-b",
ModelErrorKind.TRANSIENT_SERVICE_ERROR,
ModelError(ModelErrorKind.TRANSIENT_SERVICE_ERROR, "server error"),
1,
),
]
)
assert err.kind is ModelErrorKind.TRANSIENT_SERVICE_ERROR
def test_retry_sync_retries_transient_error_until_success(monkeypatch):
attempts = {"count": 0}
monkeypatch.setattr("openviking.utils.model_retry.time.sleep", lambda _delay: None)
sdk_error = openai.InternalServerError(
"Internal server error",
response=httpx.Response(500, request=httpx.Request("POST", "https://example.test")),
body={"error": {"code": 500, "type": "server_error", "message": "Internal server error"}},
)
@model_error_boundary
def _call():
attempts["count"] += 1
if attempts["count"] < 3:
raise sdk_error
return "ok"
assert retry_sync(_call, max_retries=3, model_errors_only=True) == "ok"
assert attempts["count"] == 3
@pytest.mark.asyncio
async def test_retry_async_preserves_local_error_without_parsing_message():
attempts = {"count": 0}
error = RuntimeError("HTTP 503 is input data, not an API failure")
@model_error_boundary
async def _call():
attempts["count"] += 1
raise error
with pytest.raises(RuntimeError) as raised:
await retry_async(_call, max_retries=3, model_errors_only=True)
assert raised.value is error
assert attempts["count"] == 1
# --- quota_exceeded classification ---
def test_classify_account_quota_exceeded():
"""AccountQuotaExceeded is classified as quota_exceeded, not transient."""
error = RuntimeError(
'API Error: 429 {"error":{"code":"AccountQuotaExceeded",'
'"message":"You have exceeded the 5-hour usage quota"}}'
)
assert classify_api_error(error) == ERROR_CLASS_QUOTA_EXCEEDED
def test_classify_quota_limit():
"""'quota limit' is classified as quota_exceeded."""
assert classify_api_error(RuntimeError("quota limit reached")) == ERROR_CLASS_QUOTA_EXCEEDED
def test_classify_quota_exceed():
"""'quota exceed' is classified as quota_exceeded."""
assert classify_api_error(RuntimeError("quota exceed")) == ERROR_CLASS_QUOTA_EXCEEDED
def test_classify_usage_quota():
"""'usage quota' is classified as quota_exceeded."""
assert classify_api_error(RuntimeError("usage quota exceeded")) == ERROR_CLASS_QUOTA_EXCEEDED
def test_quota_exceeded_takes_precedence_over_transient():
"""A 429 with AccountQuotaExceeded is quota_exceeded, not transient."""
error = RuntimeError(
'429 {"error":{"code":"AccountQuotaExceeded","message":"TooManyRequests"}}'
)
assert classify_api_error(error) == ERROR_CLASS_QUOTA_EXCEEDED
def test_auth_takes_precedence_over_quota():
"""Auth errors (e.g. 403) take precedence over the quota substring."""
assert classify_api_error(RuntimeError("403 AccountQuotaExceeded")) == ERROR_CLASS_AUTH
# --- permanent vs auth split (400 vs 401/403) ---
def test_classify_400_is_permanent():
"""A 400 parameter error is request-level permanent (fail-fast)."""
error = RuntimeError("Error code: 400 - invalid parameter `model`")
assert classify_api_error(error) == ERROR_CLASS_PERMANENT
def test_classify_401_is_auth():
"""A 401 is a credential-level auth error (advances in multi-credential mode)."""
assert classify_api_error(RuntimeError("Error code: 401 - Incorrect API key")) == (
ERROR_CLASS_AUTH
)
def test_classify_403_is_auth():
"""A 403 forbidden is a credential-level auth error."""
assert classify_api_error(RuntimeError("403 forbidden")) == ERROR_CLASS_AUTH
def test_classify_unauthorized_is_auth():
assert classify_api_error(RuntimeError("Unauthorized")) == ERROR_CLASS_AUTH
def test_classify_account_overdue_is_auth():
assert classify_api_error(RuntimeError("AccountOverdue")) == ERROR_CLASS_AUTH
# --- content safety classification ---
def test_classify_content_filter_is_content_safety():
assert classify_api_error(RuntimeError("content_filter triggered")) == (
ERROR_CLASS_CONTENT_SAFETY
)
def test_classify_content_policy_is_content_safety():
error = RuntimeError("The response was rejected by the content policy")
assert classify_api_error(error) == ERROR_CLASS_CONTENT_SAFETY
def test_content_safety_takes_precedence_over_400():
"""A moderation rejection containing '400' is content_safety, not permanent."""
error = RuntimeError("Error code: 400 - content_filter: sensitive content detected")
assert classify_api_error(error) == ERROR_CLASS_CONTENT_SAFETY
def test_retry_sync_does_not_retry_quota_exceeded():
"""Structured quota exhaustion takes precedence over the retryable HTTP 429."""
attempts = {"count": 0}
sdk_error = openai.RateLimitError(
"rate limited",
response=httpx.Response(429, request=httpx.Request("POST", "https://example.test")),
body={"error": {"code": "insufficient_quota"}},
)
@model_error_boundary
def _call():
attempts["count"] += 1
raise sdk_error
with pytest.raises(ModelError) as raised:
retry_sync(_call, max_retries=5, model_errors_only=True)
assert raised.value.kind is ModelErrorKind.QUOTA_EXHAUSTED
assert raised.value.__cause__ is sdk_error
assert attempts["count"] == 1
@pytest.mark.asyncio
async def test_retry_async_does_not_retry_quota_exceeded():
"""Quota-exceeded errors should NOT be retried (async)."""
attempts = {"count": 0}
async def _call():
attempts["count"] += 1
raise RuntimeError("AccountQuotaExceeded")
with pytest.raises(RuntimeError, match="AccountQuotaExceeded"):
await retry_async(_call, max_retries=5)
assert attempts["count"] == 1
def test_quota_exceeded_case_insensitive():
"""Quota detection is case-insensitive."""
assert classify_api_error(RuntimeError("QUOTA LIMIT")) == ERROR_CLASS_QUOTA_EXCEEDED
assert classify_api_error(RuntimeError("Quota Exceed")) == ERROR_CLASS_QUOTA_EXCEEDED
@pytest.mark.parametrize(
"message",
[
"BadRequestError: 400 maximum context length is 8192 tokens",
(
"Error code: 400 - InvalidParameter: Total tokens of multi-modal content "
"and text exceed max message tokens."
),
"Error code: 413 - Payload Too Large",
(
"Error code: 500 - {'error': {'code': 500, 'message': "
"'input (8525 tokens) is too large to process. increase the physical batch size "
"(current batch size: 2048)', 'type': 'server_error'}}"
),
# SiliconFlow generic 400 with structured code 20015, dict-repr form (#4676)
(
"Error code: 400 - {'code': 20015, 'message': "
"'The parameter is invalid. Please check again.', 'data': None}"
),
# JSON string form
'Error code: 400 - {"code": 20015, "message": "The parameter is invalid."}',
],
)
def test_classify_input_too_large_errors(message):
assert classify_api_error(RuntimeError(message)) == ERROR_CLASS_INPUT_TOO_LARGE
def test_other_siliconflow_400_codes_stay_permanent():
"""Only code 20015 is reclassified; other SiliconFlow parameter errors
(e.g. a bad model name) must remain permanent request-level failures."""
error = RuntimeError(
"Error code: 400 - {'code': 20012, 'message': 'Model not exists', 'data': None}"
)
assert classify_api_error(error) == ERROR_CLASS_PERMANENT
def test_20015_inside_request_id_is_not_input_too_large():
"""A rate-limit error whose request ID happens to contain 20015 must not
be misclassified as INPUT_TOO_LARGE."""
error = RuntimeError(
"Error code: 429 - {'error': {'code': 'TooManyRequests', "
"'message': 'RPM limit exceeded', "
"'request_id': '0217801248873024200158fe53d7c9130f34413480585e683685bc95'}}"
)
assert classify_api_error(error) == ERROR_CLASS_TRANSIENT
@pytest.mark.parametrize(
"body",
[
{"code": "context_length_exceeded", "message": "Maximum context length exceeded"},
{
"code": 500,
"type": "server_error",
"message": (
"input (8525 tokens) is too large to process. increase the physical batch size "
"(current batch size: 2048)"
),
},
],
)
def test_input_too_large_does_not_retry_or_trip_breaker(body, monkeypatch):
from openviking.utils.circuit_breaker import CircuitBreaker
breaker = CircuitBreaker()
attempts = {"count": 0}
monkeypatch.setattr("openviking.utils.model_retry.time.sleep", lambda _delay: None)
sdk_error = openai.InternalServerError(
"Input rejected",
response=httpx.Response(500, request=httpx.Request("POST", "https://example.test")),
body={"error": body},
)
@model_error_boundary
def _call():
attempts["count"] += 1
raise sdk_error
# Repeated bad inputs must not block a subsequent valid request.
for _ in range(5):
with pytest.raises(ModelError) as raised:
retry_sync(_call, max_retries=3, model_errors_only=True)
assert raised.value.kind is ModelErrorKind.INPUT_TOO_LARGE
breaker.record_failure(raised.value)
assert attempts["count"] == 5
breaker.check()
# --- numeric pattern word-boundary tests ---
def test_429_with_request_id_containing_413_is_transient():
"""A 429 error whose request ID happens to contain '413' must NOT be
misclassified as INPUT_TOO_LARGE (the original bug)."""
error = RuntimeError(
"Volcengine hybrid embedding failed: Error code: 429 - "
"{'error': {'code': 'ModelAccountRpmRateLimitExceeded', "
"'message': 'RPM limit exceeded', 'param': '', "
"'type': 'TooManyRequests'}, "
"'request_id': '0217801248873024288fe53d7c9130f34413480585e683685bc95'}"
)
assert classify_api_error(error) == ERROR_CLASS_TRANSIENT
def test_429_with_hyphenated_request_id_containing_413_is_transient():
"""Numeric status codes must not match hyphen-delimited request ID fragments."""
error = RuntimeError(
"Volcengine hybrid embedding failed: Error code: 429 - "
"{'error': {'code': 'ModelAccountRpmRateLimitExceeded', "
"'message': 'RPM limit exceeded', 'type': 'TooManyRequests'}, "
"'request_id': 'req-413-abcd'}"
)
assert classify_api_error(error) == ERROR_CLASS_TRANSIENT
def test_numeric_status_code_inside_longer_number_is_not_matched():
"""Status code patterns must not match inside longer numbers
(e.g. '400' must not match '1400')."""
assert classify_api_error(RuntimeError("status: 1400 OK")) == "unknown"
assert classify_api_error(RuntimeError("status: 5020 OK")) == "unknown"
def test_numeric_status_code_with_compact_error_code_context_still_matches():
assert classify_api_error(RuntimeError("Error code:413-Payload Too Large")) == (
ERROR_CLASS_INPUT_TOO_LARGE
)