Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

16.4 KB
Raw Blame
//! Byte encoding, Data URI syntax checks, and bounded hashing.

use std::sync::LazyLock;

use base64::engine::general_purpose::GeneralPurposeConfig;
use base64::engine::simd::Simd;
use base64::engine::{DecodePaddingMode, Engine};
use sha2::{Digest, Sha256};

use super::payload::{MediaDecodeError, MediaEncoding};
use super::rules::BASE64_MARKER;

// Match the existing Buffer/base64 decoder after the caller has validated the
// character grammar: padding may be omitted or repeated, and trailing bits are
// accepted.
pub(super) static BASE64: LazyLock<Simd> = LazyLock::new(|| {
    Simd::standard(
        GeneralPurposeConfig::new()
            .with_decode_padding_mode(DecodePaddingMode::Indifferent)
            .with_decode_allow_trailing_bits(true),
    )
});

pub(super) const BASE64_HASH_CHUNK_SIZE: usize = 16 * 1024;
pub(super) const BASE64_HASH_DECODED_CHUNK_SIZE: usize = BASE64_HASH_CHUNK_SIZE / 4 * 3;

pub(super) fn is_data_uri_terminator(byte: u8) -> bool {
    matches!(
        byte,
        b'\t'
            | b'\n'
            | b'\r'
            | b' '
            | b'!'
            | b'"'
            | b'\''
            | b')'
            | b','
            | b'.'
            | b';'
            | b'>'
            | b'?'
            | b']'
            | b'}'
    )
}

pub(super) fn is_base64_character(byte: u8) -> bool {
    byte.is_ascii_alphanumeric() || matches!(byte, b'+' | b'/' | b'=')
}

/// Length of the initial base64 alphabet run, stopping before padding.
/// Full blocks must decode without padding. Search the first failing block and
/// short tail bytewise to locate the padding byte or URI terminator.
pub(super) fn base64_unpadded_prefix_len(value: &[u8]) -> usize {
    let mut end = 0;
    let mut decoded = [0; 1024 / 4 * 3];
    for chunk in value.as_chunks::<1024>().0 {
        if BASE64.decode_slice(chunk, &mut decoded) != Ok(decoded.len()) {
            break;
        }
        end += chunk.len();
    }
    end + value[end..]
        .iter()
        .position(|byte| !byte.is_ascii_alphanumeric() && !matches!(byte, b'+' | b'/'))
        .unwrap_or(value.len() - end)
}

pub(super) fn is_valid_content_type(value: &str) -> bool {
    let mut pieces = value.split('/');
    let Some(kind) = pieces.next() else {
        return false;
    };
    let Some(subtype) = pieces.next() else {
        return false;
    };
    pieces.next().is_none()
        && !kind.is_empty()
        && !subtype.is_empty()
        && kind
            .bytes()
            .all(|byte| byte.is_ascii_alphanumeric() || b"!#$&^_.+-".contains(&byte))
        && subtype
            .bytes()
            .all(|byte| byte.is_ascii_alphanumeric() || b"!#$&^_.+-".contains(&byte))
}

/// Validate the optional parameters between a Data URI media type and `;base64`.
///
/// Parameter names use token syntax and values must be non-empty and free of
/// delimiters or whitespace. Malformed headers remain inline text.
pub(super) fn is_valid_data_uri_parameters(value: &str) -> bool {
    if value.is_empty() {
        return true;
    }
    let bytes = value.as_bytes();
    let mut cursor = 0;
    while cursor < bytes.len() {
        if bytes[cursor] != b';' {
            return false;
        }
        cursor += 1;
        let name_start = cursor;
        while cursor < bytes.len()
            && bytes[cursor] != b'='
            && bytes[cursor] != b';'
            && bytes[cursor] != b','
        {
            cursor += 1;
        }
        if cursor == name_start
            || cursor >= bytes.len()
            || bytes[cursor] != b'='
            || !bytes[name_start..cursor]
                .iter()
                .all(|byte| byte.is_ascii_alphanumeric() || b"!#$&^_.+-".contains(byte))
        {
            return false;
        }
        cursor += 1;
        let value_start = cursor;
        while cursor < bytes.len() && bytes[cursor] != b';' {
            cursor += 1;
        }
        if cursor == value_start
            || bytes[value_start..cursor]
                .iter()
                .any(|byte| b",\t\n\r \"'<>[]{}".contains(byte))
            // Use ECMAScript whitespace semantics for compatibility with the
            // existing detector; Rust's `char::is_whitespace` differs here.
            || value[value_start..cursor].chars().any(|character| {
                matches!(character,
                    '\u{0009}'..='\u{000d}' | '\u{0020}' | '\u{00a0}' | '\u{1680}' |
                    '\u{2000}'..='\u{200a}' | '\u{2028}' | '\u{2029}' | '\u{202f}' |
                    '\u{205f}' | '\u{3000}' | '\u{feff}')
            })
        {
            return false;
        }
    }
    true
}

pub(super) fn has_data_uri_boundary(value: &str, start: usize) -> bool {
    start == 0
        || !value.as_bytes()[start - 1].is_ascii_alphanumeric()
            && value.as_bytes()[start - 1] != b'-'
            && value.as_bytes()[start - 1] != b'_'
}

pub(super) fn decode_python_bytes_literal(value: &[u8]) -> Result<Vec<u8>, MediaDecodeError> {
    let mut output = Vec::with_capacity(value.len().saturating_sub(3));
    visit_python_bytes_literal(value, |byte| output.push(byte))?;
    Ok(output)
}

fn visit_python_bytes_literal(
    value: &[u8],
    mut visit_byte: impl FnMut(u8),
) -> Result<(), MediaDecodeError> {
    if value.len() < 3 || value[0] != b'b' || !matches!(value[1], b'\'' | b'"') {
        return Err(MediaDecodeError::InvalidPythonBytes);
    }
    let quote = value[1];
    if *value.last().unwrap_or(&0) != quote {
        return Err(MediaDecodeError::InvalidPythonBytes);
    }
    let mut cursor = 2;
    while cursor + 1 < value.len() {
        let byte = value[cursor];
        if byte != b'\\' {
            // Python's bytes representation escapes non-ASCII/control bytes, and
            // an unescaped delimiter cannot occur inside the literal.
            if !(0x20..=0x7e).contains(&byte) || byte == quote {
                return Err(MediaDecodeError::InvalidPythonBytes);
            }
            visit_byte(byte);
            cursor += 1;
            continue;
        }
        cursor += 1;
        if cursor == value.len() - 1 {
            return Err(MediaDecodeError::InvalidPythonBytes);
        }
        let Some(escaped) = value.get(cursor).copied() else {
            return Err(MediaDecodeError::InvalidPythonBytes);
        };
        match escaped {
            b'\\' | b'\'' | b'"' => visit_byte(escaped),
            b'n' => visit_byte(b'\n'),
            b'r' => visit_byte(b'\r'),
            b't' => visit_byte(b'\t'),
            b'x' => {
                let Some(bytes) = value
                    .get(cursor + 1..cursor + 3)
                    .and_then(|bytes| bytes.first_chunk::<2>())
                else {
                    return Err(MediaDecodeError::InvalidPythonBytes);
                };
                let Some(hi) = hex_digit(bytes[0]) else {
                    return Err(MediaDecodeError::InvalidPythonBytes);
                };
                let Some(lo) = hex_digit(bytes[1]) else {
                    return Err(MediaDecodeError::InvalidPythonBytes);
                };
                visit_byte((hi << 4) | lo);
                cursor += 2;
            }
            _ => return Err(MediaDecodeError::InvalidPythonBytes),
        }
        cursor += 1;
    }
    Ok(())
}

/// Hash decoded media in bounded chunks. Discovery needs only the content ID;
/// full decoding remains deferred to upload.
pub(super) fn hash_encoded_data(
    encoded_data: &[u8],
    encoding: MediaEncoding,
) -> Result<[u8; 32], MediaDecodeError> {
    let mut hasher = Sha256::new();
    match encoding {
        MediaEncoding::Base64DataUri => {
            hash_base64(&mut hasher, data_uri_payload(encoded_data)?)?;
        }
        MediaEncoding::Base64 => hash_base64(&mut hasher, encoded_data)?,
        MediaEncoding::PythonBytesLiteral => hash_python_bytes_literal(&mut hasher, encoded_data)?,
    }
    Ok(hasher.finalize().into())
}

fn data_uri_payload(value: &[u8]) -> Result<&[u8], MediaDecodeError> {
    let marker = value
        .windows(BASE64_MARKER.len())
        .position(|window| window == BASE64_MARKER.as_bytes())
        .ok_or(MediaDecodeError::InvalidDataUri)?;
    Ok(&value[marker + BASE64_MARKER.len()..])
}

/// Match `Buffer.from(..., "base64")` after syntax validation: remove redundant
/// padding and ignore a lone final sextet that cannot produce a byte. The
/// returned prefix is shared by hashing and upload decoding.
fn node_base64_payload(mut value: &[u8]) -> Result<&[u8], MediaDecodeError> {
    while value.last() == Some(&b'=') {
        value = &value[..value.len() - 1];
    }
    if value.len() % 4 == 1 {
        value = &value[..value.len() - 1];
    }
    // Empty decoded payloads are not media candidates.
    if value.is_empty() {
        return Err(MediaDecodeError::InvalidDataUri);
    }
    Ok(value)
}

fn hash_base64(hasher: &mut Sha256, value: &[u8]) -> Result<(), MediaDecodeError> {
    let value = node_base64_payload(value)?;
    // Keep chunks aligned to Base64 groups. The final chunk handles omitted
    // padding and non-zero trailing bits.
    let mut decoded = [0_u8; BASE64_HASH_DECODED_CHUNK_SIZE];
    for chunk in value.chunks(BASE64_HASH_CHUNK_SIZE) {
        let decoded_length = BASE64
            .decode_slice(chunk, &mut decoded)
            .map_err(MediaDecodeError::Base64Slice)?;
        hasher.update(&decoded[..decoded_length]);
    }
    Ok(())
}

fn hash_python_bytes_literal(hasher: &mut Sha256, value: &[u8]) -> Result<(), MediaDecodeError> {
    // Empty Python bytes are not media candidates.
    if value.len() == 3 {
        return Err(MediaDecodeError::InvalidPythonBytes);
    }
    let mut decoded = [0; BASE64_HASH_DECODED_CHUNK_SIZE];
    let mut decoded_len = 0;
    visit_python_bytes_literal(value, |byte| {
        decoded[decoded_len] = byte;
        decoded_len += 1;
        if decoded_len == decoded.len() {
            hasher.update(&decoded[..]);
            decoded_len = 0;
        }
    })?;
    hasher.update(&decoded[..decoded_len]);
    Ok(())
}

pub(super) fn is_python_bytes_literal(value: &str) -> bool {
    let bytes = value.as_bytes();
    bytes.len() >= 3
        && bytes[0] == b'b'
        && matches!(bytes[1], b'\'' | b'"')
        && bytes.last() == bytes.get(1)
}

pub(super) fn is_valid_base64_syntax(value: &[u8]) -> bool {
    if value.is_empty() || value.len() % 4 == 1 {
        return false;
    }
    let mut padding = 0;
    for byte in &value[base64_unpadded_prefix_len(value)..] {
        if *byte == b'=' {
            padding += 1;
            if padding > 2 {
                return false;
            }
        } else if padding > 0 || !is_base64_character(*byte) {
            return false;
        }
    }
    true
}

pub(super) fn hex_digit(byte: u8) -> Option<u8> {
    // Avoid per-digit branches for Python literals with many `\xNN` escapes.
    const DIGITS: [u8; 256] = {
        let mut digits = [u8::MAX; 256];
        let mut index = 0;
        while index < digits.len() {
            digits[index] = match index as u8 {
                b'0'..=b'9' => index as u8 - b'0',
                b'a'..=b'f' => index as u8 - b'a' + 10,
                b'A'..=b'F' => index as u8 - b'A' + 10,
                _ => u8::MAX,
            };
            index += 1;
        }
        digits
    };
    let value = DIGITS[usize::from(byte)];
    (value != u8::MAX).then_some(value)
}

pub(super) fn decode_encoded_data(
    encoded_data: &[u8],
    encoding: MediaEncoding,
) -> Result<Vec<u8>, MediaDecodeError> {
    let value = match encoding {
        MediaEncoding::Base64DataUri => data_uri_payload(encoded_data)?,
        MediaEncoding::Base64 => encoded_data,
        MediaEncoding::PythonBytesLiteral => return decode_python_bytes_literal(encoded_data),
    };
    BASE64
        .decode(node_base64_payload(value)?)
        .map_err(MediaDecodeError::Base64)
}

#[cfg(test)]
mod tests {
    use base64::Engine;
    use proptest::prelude::*;
    use sha2::{Digest, Sha256};

    use super::{
        base64_unpadded_prefix_len, decode_python_bytes_literal, hash_encoded_data, BASE64,
        BASE64_HASH_DECODED_CHUNK_SIZE,
    };
    use super::{MediaDecodeError, MediaEncoding};

    #[test]
    fn python_bytes_decoder_and_hasher_share_escape_validation() {
        for byte in 0..=u8::MAX {
            assert_eq!(
                super::hex_digit(byte),
                char::from(byte).to_digit(16).map(|digit| digit as u8)
            );
        }
        let encoded = br#"b"abc\n\r\t\\\'\"\x41""#;
        let expected = b"abc\n\r\t\\'\"A";
        let expected_hash: [u8; 32] = Sha256::digest(expected).into();
        assert_eq!(decode_python_bytes_literal(encoded).unwrap(), expected);
        assert_eq!(
            hash_encoded_data(encoded, MediaEncoding::PythonBytesLiteral).unwrap(),
            expected_hash
        );

        let malformed_single_quote = b"b'abc\\'";
        let malformed_double_quote = b"b\"abc\\\"";
        for invalid in [
            b"b'bad\\q'".as_slice(),
            b"b'bad\\x0'",
            b"b'caf\xc3\xa9'",
            malformed_single_quote.as_slice(),
            malformed_double_quote.as_slice(),
        ] {
            assert!(matches!(
                decode_python_bytes_literal(invalid),
                Err(MediaDecodeError::InvalidPythonBytes)
            ));
            assert!(matches!(
                hash_encoded_data(invalid, MediaEncoding::PythonBytesLiteral),
                Err(MediaDecodeError::InvalidPythonBytes)
            ));
        }

        let valid_single_quote = b"b'abc\\''";
        let valid_double_quote = b"b\"abc\\\"\"";
        for (literal, expected) in [
            (valid_single_quote.as_slice(), b"abc'".as_slice()),
            (valid_double_quote.as_slice(), b"abc\"".as_slice()),
        ] {
            let expected_hash: [u8; 32] = Sha256::digest(expected).into();
            assert_eq!(decode_python_bytes_literal(literal).unwrap(), expected);
            assert_eq!(
                hash_encoded_data(literal, MediaEncoding::PythonBytesLiteral).unwrap(),
                expected_hash
            );
        }
    }

    #[test]
    fn hashes_large_python_bytes_literals_in_bounded_chunks() {
        let decoded = vec![b'a'; BASE64_HASH_DECODED_CHUNK_SIZE + 17];
        let mut encoded = Vec::with_capacity(decoded.len() + 3);
        encoded.extend_from_slice(b"b'");
        encoded.extend_from_slice(&decoded);
        encoded.push(b'\'');
        let expected_hash: [u8; 32] = Sha256::digest(&decoded).into();

        assert_eq!(
            hash_encoded_data(&encoded, MediaEncoding::PythonBytesLiteral).unwrap(),
            expected_hash
        );
    }

    proptest! {
        #[test]
        fn base64_prefix_matches_scalar_search(
            prefix_len in prop_oneof![Just(1023usize), Just(1024), Just(1025), 0usize..4096],
            suffix in prop::collection::vec(any::<u8>(), 0..128),
            complete_block in any::<bool>(),
        ) {
            let alphabet = b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
            let mut value = alphabet.iter().copied().cycle().take(prefix_len).collect::<Vec<_>>();
            prop_assert_eq!(base64_unpadded_prefix_len(&value), value.len());
            value.extend(suffix);
            if complete_block {
                value.extend([b'A'; 1024]);
            }
            let expected = value.iter().position(|byte| {
                !byte.is_ascii_alphanumeric() && !matches!(byte, b'+' | b'/')
            }).unwrap_or(value.len());
            prop_assert_eq!(base64_unpadded_prefix_len(&value), expected);
        }

        #[test]
        fn bounded_base64_hash_matches_decoded_content(bytes in prop_oneof![
                prop::collection::vec(any::<u8>(), 1..4096),
                prop::collection::vec(
                    any::<u8>(),
                    BASE64_HASH_DECODED_CHUNK_SIZE..(2 * BASE64_HASH_DECODED_CHUNK_SIZE + 17),
                ),
            ]) {
            // Pin the first partial chunk as well as sampling arbitrary bodies and padding.
            let boundary: [u8; BASE64_HASH_DECODED_CHUNK_SIZE + 1] =
                std::array::from_fn(|index| index.wrapping_mul(31) as u8);
            for bytes in [bytes.as_slice(), boundary.as_slice()] {
                let encoded = BASE64.encode(bytes);
                let expected: [u8; 32] = Sha256::digest(bytes).into();
                prop_assert_eq!(hash_encoded_data(encoded.as_bytes(), MediaEncoding::Base64).unwrap(), expected);
                let unpadded = encoded.trim_end_matches('=');
                prop_assert_eq!(hash_encoded_data(unpadded.as_bytes(), MediaEncoding::Base64).unwrap(), expected);
            }
        }
    }
}