Speakr is a personal, self-hosted web application designed for transcribing audio recordings
Python
"""
JSON parsing utilities for handling LLM responses and malformed JSON.
This module provides robust JSON parsing functions that can handle common
issues with LLM-generated JSON, including:
- Incomplete/unterminated JSON structures
- Escape sequence problems
- JSON embedded in markdown code blocks
- Nested quotes and special characters
"""
import json
import re
import ast
import logging
# Module-level logger
logger = logging.getLogger(__name__)
def auto_close_json(json_string):
"""
Attempts to close an incomplete JSON string by appending necessary brackets and braces.
This is a simplified parser and may not handle all edge cases, but is
designed to fix unterminated strings from API responses.
"""
if not isinstance(json_string, str):
return json_string
stack = []
in_string = False
escape_next = False
for char in json_string:
if escape_next:
escape_next = False
continue
if char == '\\':
escape_next = True
continue
if char == '"':
# We don't handle escaped quotes inside strings perfectly,
# but this is a simple heuristic.
if not escape_next:
in_string = not in_string
if not in_string:
if char == '{':
stack.append('}')
elif char == '[':
stack.append(']')
elif char == '}':
if stack and stack[-1] == '}':
stack.pop()
elif char == ']':
if stack and stack[-1] == ']':
stack.pop()
# If we are inside a string at the end, close it.
if in_string:
json_string += '"'
# Close any remaining open structures
while stack:
json_string += stack.pop()
return json_string
def preprocess_json_escapes(json_string):
"""
Preprocess JSON string to fix common escape issues from LLM responses.
Uses a more sophisticated approach to handle nested quotes properly.
"""
if not json_string:
return json_string
result = []
i = 0
in_string = False
escape_next = False
expecting_value = False # Track if we're expecting a value (after :)
while i < len(json_string):
char = json_string[i]
if escape_next:
# This character is escaped, add it as-is
result.append(char)
escape_next = False
elif char == '\\':
# This is an escape character
result.append(char)
escape_next = True
elif char == ':' and not in_string:
# We found a colon, next string will be a value
result.append(char)
expecting_value = True
elif char == ',' and not in_string:
# We found a comma, reset expecting_value
result.append(char)
expecting_value = False
elif char == '"':
if not in_string:
# Starting a string
in_string = True
result.append(char)
else:
# We're in a string, check if this quote should be escaped
# Look ahead to see if this is the end of the string value
j = i + 1
while j < len(json_string) and json_string[j].isspace():
j += 1
# For keys (not expecting_value), only end on colon
# For values (expecting_value), end on comma, closing brace, or closing bracket
if expecting_value:
end_chars = ',}]'
else:
end_chars = ':'
if j < len(json_string) and json_string[j] in end_chars:
# This is the end of the string
in_string = False
result.append(char)
if not expecting_value:
# We just finished a key, next will be expecting value
expecting_value = True
else:
# This is an inner quote that should be escaped
result.append('\\"')
elif char == '\n' and in_string:
# Literal newlines inside JSON strings must be escaped
result.append('\\n')
elif char == '\r' and in_string:
# Literal carriage returns inside JSON strings must be escaped
result.append('\\r')
elif char == '\t' and in_string:
# Literal tabs inside JSON strings must be escaped
result.append('\\t')
else:
result.append(char)
i += 1
return ''.join(result)
def extract_json_object(text):
"""
Extract the first complete JSON object or array from text using brace counting.
This is more robust than regex as it properly handles nested structures and
doesn't over-match when there's text after the JSON.
"""
# Find the first { or [ that starts a JSON structure
start = None
close_char = None
for i, ch in enumerate(text):
if ch == '{':
start = i
close_char = '}'
break
elif ch == '[':
start = i
close_char = ']'
break
if start is None:
return text
# Walk through counting braces/brackets, respecting strings
depth = 0
in_string = False
escape_next = False
for i in range(start, len(text)):
ch = text[i]
if escape_next:
escape_next = False
continue
if ch == '\\' and in_string:
escape_next = True
continue
if ch == '"':
in_string = not in_string
continue
if not in_string:
if ch in ('{', '['):
depth += 1
elif ch in ('}', ']'):
depth -= 1
if depth == 0:
return text[start:i + 1]
# If we never balanced, return from start to end (best effort)
return text[start:]
def safe_json_loads(json_string, fallback_value=None):
"""
Safely parse JSON with preprocessing to handle common LLM JSON formatting issues.
Args:
json_string (str): The JSON string to parse
fallback_value: Value to return if parsing fails (default: None)
Returns:
Parsed JSON object or fallback_value if parsing fails
"""
if not json_string or not isinstance(json_string, str):
logger.warning(f"Invalid JSON input: {type(json_string)} - {json_string}")
return fallback_value
# Step 1: Clean the input string
cleaned_json = json_string.strip()
# Step 2: Extract JSON from markdown code blocks if present
json_match = re.search(r'```(?:json)?\s*(.*?)\s*```', cleaned_json, re.DOTALL)
if json_match:
cleaned_json = json_match.group(1).strip()
# Step 3: Try multiple parsing strategies
parsing_strategies = [
# Strategy 1: Direct parsing (for well-formed JSON)
lambda x: json.loads(x),
# Strategy 2: Fix common escape issues
lambda x: json.loads(preprocess_json_escapes(x)),
# Strategy 3: Use ast.literal_eval as fallback for simple cases
lambda x: ast.literal_eval(x) if x.startswith(('{', '[')) else None,
# Strategy 4: Extract JSON object/array using regex
lambda x: json.loads(extract_json_object(x)),
# Strategy 5: Auto-close incomplete JSON and parse
lambda x: json.loads(auto_close_json(x)),
]
for i, strategy in enumerate(parsing_strategies):
try:
result = strategy(cleaned_json)
if result is not None:
if i > 0: # Log if we had to use a fallback strategy
logger.info(f"JSON parsed successfully using strategy {i+1}")
return result
except (json.JSONDecodeError, ValueError, SyntaxError) as e:
if i == 0: # Only log the first failure to avoid spam
logger.debug(f"JSON parsing strategy {i+1} failed: {e}")
continue
# All strategies failed
logger.error(f"All JSON parsing strategies failed for: {cleaned_json[:200]}...")
return fallback_value