Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

fix(native): avoid repeated UTF-8 validation during media extraction (#18654)

Valery Meleshkind3f7bb763c82
diff --git a/packages/native/src/otel_input.rs b/packages/native/src/otel_input.rs
index e23e1672d5..9606f56380 100644
--- a/packages/native/src/otel_input.rs
+++ b/packages/native/src/otel_input.rs
@@ -72,9 +72,7 @@ impl ValidatedOtelJson {
}
};
let batch = ExtractedBatch {
- json: Mutex::new(Some(String::from_utf8(result.compact_json).map_err(
- |_| Error::new("ERR_OTEL_INTERNAL", "compacted JSON is not UTF-8"),
- )?)),
+ json: Mutex::new(Some(result.compact_json)),
media: result.media,
_memory: memory,
};
diff --git a/packages/native/src/otel_media/payload.rs b/packages/native/src/otel_media/payload.rs
index bd507dde71..e2a6466f4e 100644
--- a/packages/native/src/otel_media/payload.rs
+++ b/packages/native/src/otel_media/payload.rs
@@ -28,8 +28,7 @@
use std::borrow::Cow;
use std::collections::HashSet;
-use std::fmt;
-use std::io::Write;
+use std::fmt::{self, Write};
use std::ops::Range;
use std::sync::Arc;
@@ -142,7 +141,7 @@ pub struct ExtractedMedia {
enum MediaStorage {
Owned(Vec<u8>),
Source {
- bytes: Arc<Vec<u8>>,
+ bytes: Arc<String>,
range: Range<usize>,
},
}
@@ -151,9 +150,9 @@ impl ExtractedMedia {
/// Temporary identity travels with this occurrence through normalization.
/// Upload deduplication uses the separate content hash.
pub fn reference(&self) -> String {
- let mut bytes = Vec::with_capacity(reference_length(&self.metadata));
- write_reference(&mut bytes, &self.metadata, self.occurrence_id);
- String::from_utf8(bytes).expect("media reference is UTF-8")
+ let mut reference = String::with_capacity(reference_length(&self.metadata));
+ write_reference(&mut reference, &self.metadata, self.occurrence_id);
+ reference
}
/// Decode a body without consuming its encoded text, allowing retry or inline restoration.
@@ -247,7 +246,7 @@ impl ExtractedMedia {
fn storage_bytes(storage: &MediaStorage) -> &[u8] {
match storage {
MediaStorage::Owned(bytes) => bytes,
- MediaStorage::Source { bytes, range } => &bytes[range.clone()],
+ MediaStorage::Source { bytes, range } => &bytes.as_bytes()[range.clone()],
}
}
@@ -262,16 +261,16 @@ fn storage_retained_bytes(storage: &MediaStorage) -> usize {
#[derive(Clone, Debug, Eq, PartialEq)]
pub struct EarlyMediaResult {
/// Valid JSON with extracted values replaced by temporary occurrence references.
- pub compact_json: Vec<u8>,
+ pub compact_json: String,
/// Media removed from `compact_json`, in source traversal order.
pub media: Vec<ExtractedMedia>,
}
-/// An owned, syntax-validated source snapshot. Discovery is deferred until
+/// An owned UTF-8 string with validated JSON syntax. Discovery is deferred until
/// extraction so masking cannot cause media hashing of a discarded input.
#[derive(Debug, Eq, PartialEq)]
pub struct ValidatedPayload {
- source: Vec<u8>,
+ source: String,
normalized: bool,
}
@@ -285,14 +284,14 @@ pub fn validate(input: Vec<u8>) -> Result<ValidatedPayload, EarlyMediaError> {
};
super::json::validate_json(&input)?;
Ok(ValidatedPayload {
- source: input.into_bytes(),
+ source: input,
normalized,
})
}
impl ValidatedPayload {
pub(crate) fn normalized_bytes(&self) -> Option<&[u8]> {
- self.normalized.then_some(self.source.as_slice())
+ self.normalized.then_some(self.source.as_bytes())
}
/// Retained source allocation used for the validation handle's baseline.
@@ -304,7 +303,7 @@ impl ValidatedPayload {
/// Source-backed candidates keep ranges into the owned source allocation.
pub fn compact(self) -> Result<EarlyMediaResult, EarlyMediaError> {
let ValidatedPayload { source, .. } = self;
- let manifest = super::scanner::discover(source.as_slice())?;
+ let manifest = super::scanner::discover(&source)?;
if manifest.entries.is_empty() {
// Reuse the source allocation when there are no replacements.
return Ok(EarlyMediaResult {
@@ -316,7 +315,7 @@ impl ValidatedPayload {
}
/// Return the validated source allocation without running media discovery.
- pub fn into_source(self) -> Vec<u8> {
+ pub fn into_source(self) -> String {
self.source
}
}
@@ -434,11 +433,11 @@ impl fmt::Display for MediaDecodeError {
impl std::error::Error for MediaDecodeError {}
fn apply_edit_plan(
- source: Arc<Vec<u8>>,
+ source: Arc<String>,
manifest: MediaManifest,
) -> Result<EarlyMediaResult, EarlyMediaError> {
let entries = manifest.entries;
- let input = source.as_slice();
+ let input = source.as_str();
validate_edit_plan(input, &entries)?;
let mut nonce = [0; 16];
getrandom::fill(&mut nonce).map_err(|_| EarlyMediaError::RandomnessUnavailable)?;
@@ -454,7 +453,7 @@ fn apply_edit_plan(
.iter()
.map(|entry| entry.edit_range.len())
.sum::<usize>();
- let mut compact_json = Vec::with_capacity(
+ let mut compact_json = String::with_capacity(
input
.len()
.saturating_sub(removed_bytes)
@@ -464,10 +463,10 @@ fn apply_edit_plan(
let mut media = Vec::with_capacity(entries.len());
for (index, entry) in entries.into_iter().enumerate() {
let occurrence_id = first_id.wrapping_add(index as u128);
- compact_json.extend_from_slice(&input[cursor..entry.edit_range.start]);
+ compact_json.push_str(&input[cursor..entry.edit_range.start]);
write_reference(&mut compact_json, &entry.metadata, occurrence_id);
cursor = entry.edit_range.end;
- let raw = &input[entry.edit_range.clone()];
+ let raw = &input.as_bytes()[entry.edit_range.clone()];
let storage = if raw.contains(&b'\\') {
MediaStorage::Owned(
unescape_json_layer(raw)
@@ -489,7 +488,7 @@ fn apply_edit_plan(
json_decode_layers: entry.json_decode_layers,
});
}
- compact_json.extend_from_slice(&input[cursor..]);
+ compact_json.push_str(&input[cursor..]);
detach_small_source_ranges(&source, &mut media);
Ok(EarlyMediaResult {
@@ -524,7 +523,7 @@ fn escape_json_layer(value: &[u8]) -> Result<Vec<u8>, MediaDecodeError> {
}
pub(super) fn validate_edit_plan(
- input: &[u8],
+ input: &str,
entries: &[MediaManifestEntry],
) -> Result<(), EarlyMediaError> {
// Compaction consumes entries in source order. Keeping this invariant here
@@ -533,7 +532,8 @@ pub(super) fn validate_edit_plan(
for (index, entry) in entries.iter().enumerate() {
if entry.edit_range.start < previous_end
|| entry.edit_range.start > entry.edit_range.end
- || entry.edit_range.end > input.len()
+ || !input.is_char_boundary(entry.edit_range.start)
+ || !input.is_char_boundary(entry.edit_range.end)
{
return Err(EarlyMediaError::InvalidEditPlan { entry: index });
}
@@ -549,7 +549,7 @@ fn reference_length(metadata: &MediaMetadata) -> usize {
+ metadata.source.as_str().len()
}
-fn write_reference(output: &mut Vec<u8>, metadata: &MediaMetadata, id: u128) {
+fn write_reference(output: &mut String, metadata: &MediaMetadata, id: u128) {
// The registry distinguishes pending occurrences from pre-existing public
// references. Both use 22-character IDs, so temporary IDs do not inflate JSON.
let mut encoded_id = [0; 22];
@@ -563,13 +563,13 @@ fn write_reference(output: &mut Vec<u8>, metadata: &MediaMetadata, id: u128) {
metadata.content_type,
metadata.source.as_str()
)
- .expect("writing a media reference to a Vec cannot fail");
+ .expect("writing a media reference to a String cannot fail");
}
/// Detach small source-backed ranges when they account for less than half of
/// the source snapshot. Keeping a source-backed range for a media-heavy
/// document avoids replacing one large allocation with another large copy.
-fn detach_small_source_ranges(source: &Arc<Vec<u8>>, media: &mut [ExtractedMedia]) {
+fn detach_small_source_ranges(source: &Arc<String>, media: &mut [ExtractedMedia]) {
if source.len() < SOURCE_DETACH_MIN_BYTES {
return;
}
@@ -593,7 +593,7 @@ fn detach_small_source_ranges(source: &Arc<Vec<u8>>, media: &mut [ExtractedMedia
}) else {
continue;
};
- let Some(bytes) = source.get(range) else {
+ let Some(bytes) = source.as_bytes().get(range) else {
continue;
};
entry.storage = MediaStorage::Owned(bytes.to_vec());
diff --git a/packages/native/src/otel_media/payload_tests.rs b/packages/native/src/otel_media/payload_tests.rs
index c22ca858ba..fec1e2fba5 100644
--- a/packages/native/src/otel_media/payload_tests.rs
+++ b/packages/native/src/otel_media/payload_tests.rs
@@ -47,7 +47,7 @@ fn compaction_preserves_source_bytes_around_generated_data_uri(
source = format!("{{ \n \"layer{layer}\" : {}, \"number\" : 7.000e0 \n}}", serde_json::to_string(&source).unwrap());
}
let validated = validate(source.as_bytes().to_vec()).unwrap();
- prop_assert_eq!(validated.source.as_slice(), source.as_bytes());
+ prop_assert_eq!(validated.source.as_bytes(), source.as_bytes());
let source_ptr = validated.source.as_ptr();
let compacted = validated.compact().unwrap();
prop_assert_eq!(compacted.media.len(), originals.len(), "provider={}, layers={}", provider, layers);
@@ -60,12 +60,12 @@ fn compaction_preserves_source_bytes_around_generated_data_uri(
match &media.storage {
MediaStorage::Source { bytes, range } => {
prop_assert_eq!(bytes.as_ptr(), source_ptr);
- prop_assert_eq!(&bytes[range.clone()], original.as_bytes());
+ prop_assert_eq!(&bytes.as_bytes()[range.clone()], original.as_bytes());
}
MediaStorage::Owned(_) => prop_assert!(false, "ASCII media should retain the original source allocation"),
}
}
- prop_assert_eq!(String::from_utf8(compacted.compact_json).unwrap(), expected);
+ prop_assert_eq!(compacted.compact_json, expected);
}
// An escaped candidate instead owns decoded text, but must replace its exact source span.
@@ -78,7 +78,7 @@ fn compaction_preserves_source_bytes_around_generated_data_uri(
prop_assert_eq!(media.original_value().unwrap(), uris[0].as_str());
prop_assert_eq!(media.decode().unwrap(), payloads[0].as_slice());
prop_assert!(matches!(&media.storage, MediaStorage::Owned(_)));
- prop_assert_eq!(String::from_utf8(compacted.compact_json).unwrap(), source.replacen(&escaped, &media.reference(), 1));
+ prop_assert_eq!(compacted.compact_json, source.replacen(&escaped, &media.reference(), 1));
}
#[test]
@@ -87,10 +87,10 @@ fn valid_json_values_survive_scan_and_compaction(
) {
let source = serde_json::to_vec(&value).expect("serialize generated JSON");
let validated = validate(source.clone()).expect("valid JSON remains native");
- prop_assert_eq!(validated.source.as_slice(), source.as_slice());
+ prop_assert_eq!(validated.source.as_bytes(), source.as_slice());
let compacted = validated.compact().expect("compact media");
- let compact: Value = serde_json::from_slice(&compacted.compact_json)
+ let compact: Value = serde_json::from_str(&compacted.compact_json)
.expect("compaction preserves valid JSON");
if compacted.media.is_empty() {
// With no candidates, compaction must preserve arbitrary valid JSON.
diff --git a/packages/native/src/otel_media/scanner.rs b/packages/native/src/otel_media/scanner.rs
index 2107a926fa..926c2d59f4 100644
--- a/packages/native/src/otel_media/scanner.rs
+++ b/packages/native/src/otel_media/scanner.rs
@@ -58,7 +58,7 @@ struct ScanState {
}
/// Discover media in an already-validated source using an explicit stack.
-pub(super) fn discover(input: &[u8]) -> Result<MediaManifest, EarlyMediaError> {
+pub(super) fn discover(input: &str) -> Result<MediaManifest, EarlyMediaError> {
discover_inner(
input,
0,
@@ -69,7 +69,7 @@ pub(super) fn discover(input: &[u8]) -> Result<MediaManifest, EarlyMediaError> {
#[cfg(test)]
pub(super) fn discover_measured(
- input: &[u8],
+ input: &str,
) -> Result<(MediaManifest, usize, usize, usize), EarlyMediaError> {
let mut state = ScanState::default();
let mut stats = WalkStats::default();
@@ -83,24 +83,24 @@ pub(super) fn discover_measured(
}
fn discover_inner(
- input: &[u8],
+ input: &str,
embedded_depth: usize,
state: &mut ScanState,
stats: &mut WalkStats,
) -> Result<MediaManifest, EarlyMediaError> {
// Most payloads have no media markers. Avoid allocating the structural walk
// for those documents.
- if std::str::from_utf8(input).is_ok_and(|text| !may_contain_media_candidate(text)) {
+ if !may_contain_media_candidate(input) {
stats.add_bytes(input.len());
return Ok(MediaManifest {
entries: Vec::new(),
});
}
- let candidates = super::structural_walk::scan(input, embedded_depth == 0, stats)?;
+ let candidates = super::structural_walk::scan(input.as_bytes(), embedded_depth == 0, stats)?;
let mut entries = Vec::new();
let mut discovery = MediaDiscovery {
- input,
+ input: input.as_bytes(),
state,
stats,
embedded_depth,
@@ -188,12 +188,7 @@ impl MediaDiscovery<'_, '_, '_> {
return Ok(false);
}
- let mut nested = discover_inner(
- value.as_bytes(),
- self.embedded_depth + 1,
- self.state,
- self.stats,
- )?;
+ let mut nested = discover_inner(value, self.embedded_depth + 1, self.state, self.stats)?;
self.translate_nested_entries(value, token_range, &mut nested.entries)?;
self.entries.extend(nested.entries);
Ok(true)
diff --git a/packages/native/src/otel_media/tests.rs b/packages/native/src/otel_media/tests.rs
index dc0d85c79b..04727ad510 100644
--- a/packages/native/src/otel_media/tests.rs
+++ b/packages/native/src/otel_media/tests.rs
@@ -11,7 +11,8 @@ fn validate_measured(
input: Vec<u8>,
) -> Result<(super::ValidatedPayload, usize, usize, usize, usize), EarlyMediaError> {
let validated = validate(input.clone())?;
- let (manifest, visited, index_bytes, unique) = discover_measured(&input)?;
+ let (manifest, visited, index_bytes, unique) =
+ discover_measured(std::str::from_utf8(&input).unwrap())?;
let retained_bytes = validated.retained_bytes() + manifest.retained_bytes();
Ok((validated, visited, index_bytes, unique, retained_bytes))
}
@@ -84,7 +85,7 @@ fn leaves_small_data_uris_for_the_later_media_pass() {
assert_eq!(result.media.len(), 1);
assert_eq!(result.media[0].decode().unwrap(), large_body);
- let compact = String::from_utf8(result.compact_json).unwrap();
+ let compact = result.compact_json;
assert!(compact.contains(&small));
assert!(compact.contains(&result.media[0].reference()));
}
@@ -100,7 +101,7 @@ fn early_extraction_uses_encoded_size_for_uri_and_provider_candidates() {
let result = extract_media(input.as_bytes()).unwrap();
assert_eq!(result.media.len(), if size < 1024 { 0 } else { 2 });
if size < 1024 {
- assert_eq!(result.compact_json, input.as_bytes());
+ assert_eq!(result.compact_json, input);
}
}
}
@@ -112,7 +113,8 @@ fn validates_and_extracts_data_uri_without_parsing_a_value_tree() {
let result = extract_media(input.as_bytes()).expect("valid JSON");
assert_eq!(result.media.len(), 1);
- assert!(String::from_utf8_lossy(&result.compact_json)
+ assert!(result
+ .compact_json
.contains("@@@langfuseMedia:type=image/png|id="));
assert_eq!(result.media[0].decode().unwrap(), body);
assert_eq!(result.media[0].original_value().unwrap(), uri);
@@ -128,7 +130,7 @@ fn nested_otlp_map_keys_stay_structural() {
let input = json!({"resourceSpans": [{"resource": {"attributes": attributes}}]});
let result = extract_media(&serde_json::to_vec(&input).unwrap()).unwrap();
assert_eq!(result.media.len(), 1);
- let compact: Value = serde_json::from_slice(&result.compact_json).unwrap();
+ let compact: Value = serde_json::from_str(&result.compact_json).unwrap();
let entry = &compact["resourceSpans"][0]["resource"]["attributes"][0]["value"]["arrayValue"]
["values"][0]["kvlistValue"]["values"][0];
assert_eq!(entry["key"], uri);
@@ -191,7 +193,7 @@ fn deep_structural_scan_keeps_provider_objects_inline_but_finds_uri_text() {
let result = extract_media(input.as_bytes()).expect("deep JSON remains valid");
assert_eq!(result.media.len(), 1);
assert_eq!(result.media[0].decode().unwrap(), body);
- let compact: Value = serde_json::from_slice(&result.compact_json).unwrap();
+ let compact: Value = serde_json::from_str(&result.compact_json).unwrap();
let compact_embedded = compact["text"].as_str().unwrap();
assert!(compact_embedded.contains(r#""type":"base64""#));
assert!(compact_embedded.contains(&provider_data));
@@ -227,7 +229,7 @@ fn bounded_text_scan_leaves_escaped_uri_spelling_inline() {
assert_eq!(result.media.len(), 1);
assert_eq!(result.media[0].decode().unwrap(), plain_body);
let expected = input.replacen(&plain_uri, &result.media[0].reference(), 1);
- assert_eq!(result.compact_json, expected.as_bytes());
+ assert_eq!(result.compact_json, expected);
}
}
}
@@ -244,7 +246,7 @@ fn preserves_json_escaping_and_scans_nested_stringified_json() {
let result = extract_media(input.as_bytes()).expect("valid JSON");
assert_eq!(result.media.len(), 1);
- let compact = String::from_utf8(result.compact_json).unwrap();
+ let compact = result.compact_json;
assert!(compact.contains("@@@langfuseMedia:type=image/png|id="));
assert_eq!(result.media[0].kind, MediaPayloadKind::Anthropic);
assert_eq!(result.media[0].original_value().unwrap(), encoded);
@@ -297,7 +299,7 @@ fn scans_sdk_json_string_wrappers_within_the_embedded_json_budget() {
expected = expected.replacen(&uri, &result.media[0].reference(), 1);
expected = expected.replacen(&uri, &result.media[1].reference(), 1);
expected = expected.replace(&provider_data, &result.media[2].reference());
- assert_eq!(String::from_utf8(result.compact_json).unwrap(), expected);
+ assert_eq!(result.compact_json, expected);
}
#[test]
@@ -340,7 +342,7 @@ fn limits_embedded_documents_without_losing_uri_text_or_siblings() {
for media in &result.media {
expected = expected.replacen(&media.original_value().unwrap(), &media.reference(), 1);
}
- assert_eq!(String::from_utf8(result.compact_json).unwrap(), expected);
+ assert_eq!(result.compact_json, expected);
}
}
@@ -487,7 +489,7 @@ fn nested_provider_source_spelling_survives_two_stringified_layers() {
Err(super::payload::MediaDecodeError::JsonLayerLimit)
));
- let compact: Value = serde_json::from_slice(&result.compact_json).unwrap();
+ let compact: Value = serde_json::from_str(&result.compact_json).unwrap();
let compact_first_layer = compact["text"].as_str().unwrap();
let reference = media.reference();
assert_eq!(
@@ -538,7 +540,7 @@ fn classifies_each_root_array_object_before_discovery() {
assert_eq!(result.media.len(), 2);
assert_eq!(result.media[0].decode().unwrap(), provider_body);
assert_eq!(result.media[1].decode().unwrap(), envelope_body);
- let compact: Value = serde_json::from_slice(&result.compact_json).unwrap();
+ let compact: Value = serde_json::from_str(&result.compact_json).unwrap();
assert_eq!(compact[1]["scopeSpans"][0]["scope"]["name"], structural_uri);
assert_eq!(
compact[1]["scopeSpans"][0]["spans"][0]["attributes"][0]["key"],
@@ -659,7 +661,7 @@ fn existing_references_and_tiny_candidates_stay_inline_with_bounded_retention()
);
let result = validated.compact().unwrap();
assert!(result.media.is_empty());
- assert_eq!(result.compact_json, input.as_bytes());
+ assert_eq!(result.compact_json, input);
}
}
@@ -711,7 +713,7 @@ fn validates_large_numbers_without_converting_them_to_floats_or_integers() {
let result = extract_media(input.as_bytes()).expect("valid JSON number lexemes");
assert!(result.media.is_empty());
- assert_eq!(result.compact_json, input.as_bytes());
+ assert_eq!(result.compact_json, input);
}
#[test]
@@ -720,7 +722,7 @@ fn structured_data_uri_must_occupy_the_entire_provider_field() {
let input = format!(r#"{{"type":"file","mediaType":"image/png","data":"{uri} trailing"}}"#);
let result = extract_media(input.as_bytes()).expect("valid JSON");
assert!(result.media.is_empty());
- assert_eq!(result.compact_json, input.as_bytes());
+ assert_eq!(result.compact_json, input);
}
#[test]
@@ -737,7 +739,7 @@ fn matches_data_uri_parameter_validation() {
);
let result = extract_media(invalid.as_bytes()).expect("valid JSON");
assert!(result.media.is_empty());
- assert_eq!(result.compact_json, invalid.as_bytes());
+ assert_eq!(result.compact_json, invalid);
}
#[test]
@@ -755,7 +757,7 @@ fn leaves_invalid_python_bytes_literals_inline_like_typescript() {
.replace("abc", &"abc".repeat(MIN_EARLY_MEDIA_BYTES));
let result = extract_media(input.as_bytes()).expect("valid JSON");
assert!(result.media.is_empty(), "invalid literal extracted");
- assert_eq!(result.compact_json, input.as_bytes());
+ assert_eq!(result.compact_json, input);
}
}
@@ -768,7 +770,7 @@ fn leaves_invalid_and_unsupported_candidates_inline() {
);
let result = extract_media(input.as_bytes()).expect("valid JSON");
assert_eq!(result.media.len(), 1);
- let compact = String::from_utf8(result.compact_json).unwrap();
+ let compact = result.compact_json;
assert!(compact.contains("not#base64"));
assert!(compact.contains("AAAA==="));
assert!(compact.contains("application/x-unknown"));
@@ -780,7 +782,7 @@ fn never_rewrites_media_like_object_keys() {
let input = format!(r#"{{"{uri}":"ordinary value"}}"#);
let result = extract_media(input.as_bytes()).expect("valid JSON");
assert!(result.media.is_empty());
- assert_eq!(result.compact_json, input.as_bytes());
+ assert_eq!(result.compact_json, input);
}
#[test]
@@ -796,7 +798,7 @@ fn extracts_user_payload_values_named_name_or_key() {
});
let source = serde_json::to_vec(&input).expect("serialize payload");
let result = extract_media(&source).expect("valid JSON");
- let compact: Value = serde_json::from_slice(&result.compact_json).expect("compact JSON");
+ let compact: Value = serde_json::from_str(&result.compact_json).expect("compact JSON");
assert_eq!(result.media.len(), 3);
assert!(compact["name"]
@@ -833,7 +835,7 @@ fn does_not_extract_structural_otlp_strings_or_attribute_keys() {
let result = extract_media(&source).expect("valid JSON");
assert_eq!(result.media.len(), 1);
- let compact: Value = serde_json::from_slice(&result.compact_json).expect("compact JSON");
+ let compact: Value = serde_json::from_str(&result.compact_json).expect("compact JSON");
assert_eq!(
compact["resourceSpans"][0]["scopeSpans"][0]["scope"]["name"],
uri
@@ -879,7 +881,7 @@ fn recognizes_a_top_level_resource_span_array_as_an_otlp_envelope() {
}]);
let source = serde_json::to_vec(&input).expect("serialize OTLP envelope");
let result = extract_media(&source).expect("valid JSON");
- let compact: Value = serde_json::from_slice(&result.compact_json).expect("compact JSON");
+ let compact: Value = serde_json::from_str(&result.compact_json).expect("compact JSON");
assert_eq!(result.media.len(), 1);
assert_eq!(compact[0]["scopeSpans"][0]["scope"]["name"], uri);
@@ -903,12 +905,10 @@ fn leaves_lone_surrogate_strings_inline_and_extracts_siblings() {
let result = extract_media(input.as_bytes()).expect("raw surrogate escape is valid JSON");
assert_eq!(result.media.len(), 1);
assert_eq!(result.media[0].decode().unwrap(), body);
- assert!(String::from_utf8(result.compact_json)
- .unwrap()
- .contains(text));
+ assert!(result.compact_json.contains(text));
}
let input = br#"{"value":"\ud83d\ude80"}"#;
- assert_eq!(extract_media(input).unwrap().compact_json, input);
+ assert_eq!(extract_media(input).unwrap().compact_json.as_bytes(), input);
for input in [br#"{"x":"\q"}"#.as_slice(), br#"{"x":"\ud800\uZZZZ"}"#] {
assert!(matches!(
extract_media(input),
@@ -924,7 +924,7 @@ fn malformed_utf8_is_replaced_before_discovery() {
input.extend([0xff, 0xe2, 0x82]);
input.extend_from_slice(format!(r#"","image":"{uri}"}}"#).as_bytes());
let result = extract_media(&input).unwrap();
- let compact: Value = serde_json::from_slice(&result.compact_json).unwrap();
+ let compact: Value = serde_json::from_str(&result.compact_json).unwrap();
assert_eq!(compact["note"], "��");
assert_eq!(result.media[0].decode().unwrap(), body);
assert_eq!(result.media[0].original_value().unwrap(), uri);
@@ -939,7 +939,7 @@ fn embedded_json_keeps_surrogate_strings_and_extracts_sibling_fields() {
assert_eq!(result.media.len(), 1);
assert_eq!(result.media[0].decode().unwrap(), body);
let expected = input.replace(&uri, &result.media[0].reference());
- assert_eq!(String::from_utf8(result.compact_json).unwrap(), expected);
+ assert_eq!(result.compact_json, expected);
}
#[test]
@@ -952,7 +952,7 @@ fn undecodable_keys_leave_their_values_inline_including_shadowed_values() {
let result = extract_media(input.as_bytes()).unwrap();
assert_eq!(result.media.len(), 1);
assert_eq!(result.media[0].decode().unwrap(), body);
- let restored = String::from_utf8(result.compact_json).unwrap().replace(
+ let restored = result.compact_json.replace(
&result.media[0].reference(),
&result.media[0].original_value().unwrap(),
);
@@ -1052,8 +1052,8 @@ fn existing_public_references_are_preserved_beside_new_occurrences() {
assert_eq!(result.media.len(), 1);
let pending = result.media[0].reference();
assert_ne!(pending, expected_reference);
- let restored = String::from_utf8(result.compact_json)
- .unwrap()
+ let restored = result
+ .compact_json
.replace(&pending, &result.media[0].original_value().unwrap());
assert_eq!(restored, input);
}
@@ -1096,7 +1096,7 @@ proptest! {
let expected = input
.replacen(first, &first_reference, 1)
.replacen(second, &second_reference, 1);
- prop_assert_eq!(String::from_utf8(result.compact_json).unwrap(), expected);
+ prop_assert_eq!(result.compact_json, expected);
}
}
@@ -1131,7 +1131,7 @@ proptest! {
let result = validated.compact().unwrap();
prop_assert_eq!(result.media.len(), if should_extract { width } else { 0 });
if !should_extract {
- prop_assert_eq!(result.compact_json, input.as_bytes());
+ prop_assert_eq!(result.compact_json, input);
return Ok(());
}
for media in result.media { let decoded = media.decode().unwrap();
@@ -1161,7 +1161,7 @@ proptest! {
) {
let mut source = serde_json::to_vec(&value).expect("serialize generated JSON");
let validated = validate(source.clone()).expect("generated JSON validates");
- prop_assert_eq!(validated.into_source(), source.as_slice());
+ prop_assert_eq!(validated.into_source().into_bytes(), source.as_slice());
source.push(b'x');
let mut directed = br#"{"traceId":{"type":"Buffer","data":[1,2,3]},"text":"plain"}"#.to_vec();
directed.push(b'x');