Skip to content

Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.

14.4 KB
Raw Blame
# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.
# SPDX-License-Identifier: AGPL-3.0
"""
JSON stable parsing - Five-Layer Fault Tolerance Architecture.

Layer 1: JSON Cleanup    - extract_json_content()
Layer 2: JSON Repair      - json_repair.loads() (handles markdown too)
Layer 3: Structure Tolerance - list→object conversion + field filtering
Layer 4: Value Tolerance  - value_fault_tolerance()
Layer 5: Validation Tolerance - TypeAdapter(strict=False) + list item filtering
"""

import json
from dataclasses import asdict, is_dataclass
from types import UnionType
from typing import (
    Any,
    List,
    Optional,
    Tuple,
    Type,
    Union,
    get_args,
    get_origin,
    get_type_hints,
)

import json_repair
from pydantic import BaseModel, TypeAdapter

from openviking.telemetry import tracer
from openviking_cli.utils import get_logger

logger = get_logger(__name__)


# Exported for testing
__all__ = [
    "extract_json_content",
    "parse_json_with_stability",
    "value_fault_tolerance",
    "parse_value_with_tolerance",
    "_get_origin_type",
    "_get_arg_type",
    "_any_to_str",
    "JsonUtils",
]


class PydanticEncoder(json.JSONEncoder):
    def default(self, obj):
        if isinstance(obj, BaseModel):
            # 保存类名和属性值
            return {**obj.model_dump(mode="python")}
        elif is_dataclass(obj):
            return asdict(obj)
        return super().default(obj)


class JsonUtils:
    @staticmethod
    def dumps(obj, indent=4, ensure_ascii=False):
        if obj is None:
            return None
        return json.dumps(obj, ensure_ascii=ensure_ascii, indent=indent, cls=PydanticEncoder)

    @staticmethod
    def loads(json_str, clazz=None):
        if not json_str:
            return None
        if clazz:
            return TypeAdapter(clazz).validate_python(json_repair.loads(json_str), strict=False)
        return json_repair.loads(json_str)


def extract_json_content(s: str) -> str:
    """
    Layer 1: Extract JSON content from LLM response, removing both leading and trailing non-JSON content.

    Models often add thinking process, safety warnings, or explanations before or after the JSON.
    This function extracts only the valid JSON part from first {/[ to last }/].

    Args:
        s: Raw LLM response string

    Returns:
        String with only the JSON part (from first {/[ to last }/])
    """
    if not s:
        return s

    original_stripped = s.strip()
    if not original_stripped:
        return s

    temp_s = s

    # Find first { or [
    first_brace = temp_s.find("{")
    first_bracket = temp_s.find("[")

    start_idx = 0
    if first_brace != -1 and first_bracket != -1:
        start_idx = min(first_brace, first_bracket)
    elif first_brace != -1:
        start_idx = first_brace
    elif first_bracket != -1:
        start_idx = first_bracket
    else:
        # No JSON markers found, return original
        return s

    if start_idx > 0:
        temp_s = temp_s[start_idx:]

    # Find last } or ]
    last_brace = temp_s.rfind("}")
    last_bracket = temp_s.rfind("]")

    end_idx = len(temp_s)
    if last_brace != -1 and last_bracket != -1:
        end_idx = max(last_brace, last_bracket) + 1
    elif last_brace != -1:
        end_idx = last_brace + 1
    elif last_bracket != -1:
        end_idx = last_bracket + 1

    if end_idx < len(temp_s):
        temp_s = temp_s[:end_idx]

    result = temp_s.strip()

    # If we stripped everything, return original
    if not result:
        return s

    return result


def _get_origin_type(annotation) -> Type:
    """
    Extract base type from Optional or Union types.

    Similar to BaseModelCompat.get_origin_type().

    Args:
        annotation: Type annotation (could be Union, Optional, List, etc.)

    Returns:
        The underlying origin type
    """
    origin = get_origin(annotation)
    if origin is Union or origin is UnionType:
        args = get_args(annotation)
        # Handle Optional[T] which is Union[T, None]
        if len(args) == 2 and args[1] is type(None):
            return _get_origin_type(args[0])
    elif origin is list:
        return list
    return annotation


def _get_arg_type(annotation) -> Optional[Type]:
    """
    Extract item type from List annotations.

    Similar to BaseModelCompat.get_arg_type().

    Args:
        annotation: Type annotation

    Returns:
        The list item type if annotation is List[T], else None
    """
    origin = get_origin(annotation)
    if origin is Union or origin is UnionType:
        args = get_args(annotation)
        if len(args) == 2 and args[1] is type(None):
            return _get_arg_type(args[0])
    elif origin is list:
        args = get_args(annotation)
        if args:
            return args[0]
    return None


def _any_to_str(value) -> str:
    """
    Convert any value to string, with special handling for containers.

    Similar to BaseModelCompat.any_to_str().

    Args:
        value: Any value

    Returns:
        String representation
    """
    if value is None:
        return ""
    if isinstance(value, list):
        return ",".join(map(str, value))
    elif isinstance(value, dict):
        return json.dumps(value, ensure_ascii=False)
    elif isinstance(value, (int, bool, float)):
        return f"{value}"
    return str(value)


def value_fault_tolerance(field_type, value):
    """
    Layer 4: Value-level fault tolerance - automatic type conversion.

    Similar to BaseModelCompat.value_fault_tolerance().

    Handles common type mismatches:
    - 'None' → None (for non-str types)
    - list/dict/number → str (when target type is str)
    - str → int/float (when target type is number)
    - str/dict → list (when target type is list)

    Args:
        field_type: Target type annotation
        value: Raw value from JSON

    Returns:
        Converted value
    """
    origin_type = _get_origin_type(field_type)

    # Handle json_repair converting None to 'None'
    if value == "None":
        if origin_type is not str:
            return None

    if origin_type is str:
        # Convert any type to string
        return _any_to_str(value)
    elif origin_type is int:
        if isinstance(value, str):
            if value is None or value == "None":
                return 0
            try:
                return int(value)
            except (ValueError, TypeError):
                pass
    elif origin_type is float:
        if isinstance(value, str):
            if value is None or value == "None":
                return 0.0
            try:
                return float(value)
            except (ValueError, TypeError):
                pass
    elif origin_type is list:
        if isinstance(value, str):
            # Wrap single string in list
            return [value]
        elif isinstance(value, dict):
            # Wrap single dict in list
            return [value]

    return value


def parse_value_with_tolerance(value, annotation):
    """
    Layer 4 & 5: Parse value with tolerance and validation.

    Similar to json_adapter.parse_value().

    Args:
        value: Raw value
        annotation: Target type annotation

    Returns:
        Parsed and validated value

    Raises:
        Exception: If parsing fails even after tolerance attempts
    """
    # Handle None string from json_repair
    if annotation is str or annotation is Optional[str]:
        if value is None:
            return None
        if isinstance(value, str):
            return value
        else:
            return (
                json.dumps(value, ensure_ascii=False)
                if isinstance(value, (dict, list))
                else str(value)
            )

    if value == "None":
        return None

    # Apply value fault tolerance (inline for efficiency)
    origin_type = _get_origin_type(annotation)
    if origin_type is str:
        parsed_value = _any_to_str(value)
    elif origin_type is int:
        if isinstance(value, str):
            if value == "None":
                parsed_value = 0
            else:
                try:
                    parsed_value = int(value)
                except (ValueError, TypeError):
                    parsed_value = value
        else:
            parsed_value = value
    elif origin_type is float:
        if isinstance(value, str):
            if value == "None":
                parsed_value = 0.0
            else:
                try:
                    parsed_value = float(value)
                except (ValueError, TypeError):
                    parsed_value = value
        else:
            parsed_value = value
    elif origin_type is list:
        if value is None:
            parsed_value = []
        elif isinstance(value, str):
            parsed_value = [value]
        elif isinstance(value, dict):
            parsed_value = [value]
        else:
            parsed_value = value
    else:
        parsed_value = value

    # Try validation with TypeAdapter
    try:
        return TypeAdapter(annotation).validate_python(parsed_value, strict=False)
    except Exception as e:
        tracer.info(f"TypeAdapter validation failed (recoverable): {e}")

        # For list types, try filtering invalid items
        if get_origin(annotation) is list and isinstance(parsed_value, list):
            filtered_items = []
            item_type = _get_arg_type(annotation)
            if item_type is not None:
                for item in parsed_value:
                    try:
                        validated_item = TypeAdapter(item_type).validate_python(item, strict=False)
                        filtered_items.append(validated_item)
                    except Exception:
                        tracer.info(f"Skipping invalid list item: {item}")
                        continue

            if filtered_items:
                return filtered_items
            else:
                tracer.info("All list items were filtered out, returning empty list")
                return []

        # Re-raise for non-list types
        raise e


def parse_json_with_stability(
    content: str,
    model_class: Optional[Type] = None,
    expected_fields: Optional[List[str]] = None,
) -> Tuple[Optional[Any], Optional[str]]:
    """
    Five-layer JSON parsing with maximum stability.

    Layer 1: Extract JSON content (remove both leading and trailing non-JSON)
    Layer 2: Repair JSON with json_repair
    Layer 3: Structure tolerance (list→object, extra fields filtering)
    Layer 4: Value fault tolerance (type conversion)
    Layer 5: Validation tolerance (strict=False + list item filtering)

    Args:
        content: Raw LLM response string
        model_class: Optional Pydantic model class to validate against
        expected_fields: Optional field names to keep. Non-empty objects containing
            none of these fields are rejected instead of becoming an empty result.

    Returns:
        Tuple of (parsed_data, error_message). error_message is None on success.
    """
    if not content:
        return None, "Empty content"

    # Layer 1: Extract JSON content (both leading and trailing)
    try:
        cleaned_content = extract_json_content(content)
        if not cleaned_content:
            return None, "No JSON content found after cleanup"
    except Exception as e:
        tracer.error(f"Layer 1 cleanup failed: {e}")
        cleaned_content = content

    # Layer 2: Parse with json_repair
    parsed_data = None
    try:
        parsed_data = json_repair.loads(cleaned_content)
    except Exception as e:
        tracer.error(f"Layer 2 json_repair failed: {e}")
        # Fallback: try regular json.loads
        try:
            parsed_data = json.loads(cleaned_content)
        except Exception as e2:
            return None, f"JSON parsing failed: {e} (fallback also failed: {e2})"

    # Layer 3: Structure tolerance
    # Handle case where model returns [{"xxx": ...}] instead of {"xxx": ...}
    if isinstance(parsed_data, list) and len(parsed_data) > 0:
        parsed_data = parsed_data[0]
        tracer.info("Extracted first item from list response")
    elif (
        isinstance(parsed_data, list)
        and len(parsed_data) == 0
        and getattr(model_class, "_allow_empty_list_response", False)
    ):
        # The operations model opts in (via _allow_empty_list_response) to treating a
        # bare `[]` as a valid "no operations" outcome: every field is default_factory=list,
        # so map [] to {} and let it validate to an empty-ops object instead of raising
        # downstream. Every other model keeps the fail-loud "Expected dict" path below.
        parsed_data = {}
        tracer.info("Empty list response treated as empty object (no operations)")

    if not isinstance(parsed_data, dict):
        return None, f"Expected dict after parsing, got {parsed_data}"

    # Filter to only expected fields if provided
    if expected_fields:
        filtered_data = {k: v for k, v in parsed_data.items() if k in expected_fields}
        if parsed_data and not filtered_data:
            fields = ", ".join(sorted(parsed_data))
            return None, f"No expected fields found; got fields: {fields}"
        parsed_data = filtered_data

    # If no model class, return the raw dict
    if model_class is None:
        return parsed_data, None

    # Layer 4 & 5: Validate with model
    try:
        # First try direct model validation
        return model_class.model_validate(parsed_data, strict=False), None
    except Exception as e:
        tracer.info(f"Direct model validation failed, trying parse_value_with_tolerance: {e}")
        tracer.info(f"content={content}")
        # Fallback: Apply value fault tolerance to each field individually
        try:
            field_types = get_type_hints(model_class)
            tolerant_data = {}
            for field_name, field_value in parsed_data.items():
                if field_name in field_types:
                    try:
                        tolerant_data[field_name] = parse_value_with_tolerance(
                            field_value, field_types[field_name]
                        )
                    except Exception as field_e:
                        tracer.error(f"Field {field_name} parsing failed: {field_e}")
                        # Skip this field rather than failing the whole parse
                        continue

            # Now try validating with the tolerant data
            return model_class.model_validate(tolerant_data, strict=False), None
        except Exception as e2:
            return None, f"Model validation failed even after tolerance: {e} (fallback: {e2})"