π End-to-end encrypted cloud for everything.
Dart 2faandroidauthye2eeencryptionend-to-end-encryptionfluttergolanggoogle-photosgoogle-photos-alternativeiosopensourcephotosprivacyprivacy-toolstypescriptzero-knowledge
// Reproduces libyuv's limited-range BT.601 fixed-point arithmetic instead of
// the textbook full-range matrix because that is what Android camera stacks
// apply when converting frames to RGBA, so YUV and RGBA frames yield the
// same quad.
use crate::cv;
use crate::cv::image::ImageU8;
const YG: i32 = 18997; // round(1.164 * 64 * 256 * 256 / 257)
const YGB: i32 = -1160; // 1.164 * 64 * -16 + 64 * 0.5 (rounding bias folded in)
const UB: i32 = -128; // max(-128, round(-2.018 * 64)): clamped to fit a signed byte
const UG: i32 = 25; // round(0.391 * 64)
const VG: i32 = 52; // round(0.813 * 64)
const VR: i32 = -102; // round(-1.596 * 64)
const BB: i32 = UB * 128 + YGB;
const BG: i32 = UG * 128 + VG * 128 + YGB;
const BR: i32 = VR * 128 + YGB;
fn ceil_half(value: i32) -> i32 {
value / 2 + value % 2
}
fn clamp_u8(value: i32) -> u8 {
value.clamp(0, 255) as u8
}
#[inline]
pub(crate) fn yuv_to_bgr(y: u8, u: u8, v: u8) -> (u8, u8, u8) {
let (y, u, v) = (y as i32, u as i32, v as i32);
let y1 = ((y as u32 * 0x0101 * YG as u32) >> 16) as i32;
let b = clamp_u8((-(u * UB) + y1 + BB) >> 6);
let g = clamp_u8((-(u * UG + v * VG) + y1 + BG) >> 6);
let r = clamp_u8((-(v * VR) + y1 + BR) >> 6);
(b, g, r)
}
#[derive(Clone, Copy, Debug)]
pub struct PlaneLayout {
pub width: i32,
pub height: i32,
pub y_row_stride: i32,
pub uv_row_stride: i32,
pub uv_pixel_stride: i32,
}
fn required_len(rows: i32, row_stride: i32, last_row_bytes: i32) -> Result<usize, String> {
if rows <= 0 || row_stride < 0 || last_row_bytes < 0 {
return Err("invalid plane dimensions".to_owned());
}
((rows - 1) as usize)
.checked_mul(row_stride as usize)
.and_then(|length| length.checked_add(last_row_bytes as usize))
.ok_or_else(|| "plane size overflow".to_owned())
}
fn check_layout(
y_plane: &[u8],
u_plane: &[u8],
v_plane: &[u8],
layout: PlaneLayout,
) -> Result<(i32, i32), String> {
let PlaneLayout {
width,
height,
y_row_stride,
uv_row_stride,
uv_pixel_stride,
} = layout;
if width <= 0 || height <= 0 {
return Err(format!("invalid frame size {width}x{height}"));
}
if uv_pixel_stride < 1 {
return Err(format!("invalid uv pixel stride {uv_pixel_stride}"));
}
if y_row_stride < width {
return Err(format!(
"y row stride {y_row_stride} is smaller than the frame width {width}"
));
}
let cw = ceil_half(width);
let ch = ceil_half(height);
let last_chroma_byte = (cw - 1)
.checked_mul(uv_pixel_stride)
.and_then(|value| value.checked_add(1))
.ok_or_else(|| "chroma stride overflow".to_owned())?;
if uv_row_stride < last_chroma_byte {
return Err(format!(
"uv row stride {uv_row_stride} is too small for {cw} chroma samples \
at pixel stride {uv_pixel_stride}"
));
}
let need_y = required_len(height, y_row_stride, width)?;
let need_uv = required_len(ch, uv_row_stride, last_chroma_byte)?;
if y_plane.len() < need_y {
return Err(format!(
"y plane has {} bytes, needs {need_y}",
y_plane.len()
));
}
if u_plane.len() < need_uv {
return Err(format!(
"u plane has {} bytes, needs {need_uv}",
u_plane.len()
));
}
if v_plane.len() < need_uv {
return Err(format!(
"v plane has {} bytes, needs {need_uv}",
v_plane.len()
));
}
Ok((cw, ch))
}
fn plane_to_image(
plane: &[u8],
width: i32,
height: i32,
row_stride: i32,
pixel_stride: i32,
) -> Result<ImageU8, String> {
let (w, h) = (width as usize, height as usize);
let (row_stride, pixel_stride) = (row_stride as usize, pixel_stride as usize);
if pixel_stride == 1 && row_stride == w {
return ImageU8::new(width, height, 1, plane[..w * h].to_vec());
}
let mut out = vec![0u8; w * h];
for (row, dst) in out.chunks_exact_mut(w).enumerate() {
let base = row * row_stride;
for (col, sample) in dst.iter_mut().enumerate() {
*sample = plane[base + col * pixel_stride];
}
}
ImageU8::new(width, height, 1, out)
}
pub(crate) fn yuv420_to_bgr(
y_plane: &[u8],
u_plane: &[u8],
v_plane: &[u8],
layout: PlaneLayout,
width: i32,
height: i32,
) -> Result<ImageU8, String> {
let (cw, ch) = check_layout(y_plane, u_plane, v_plane, layout)?;
let y = plane_to_image(y_plane, layout.width, layout.height, layout.y_row_stride, 1)?;
let u = plane_to_image(
u_plane,
cw,
ch,
layout.uv_row_stride,
layout.uv_pixel_stride,
)?;
let v = plane_to_image(
v_plane,
cw,
ch,
layout.uv_row_stride,
layout.uv_pixel_stride,
)?;
let y = cv::resize_u8(&y, width, height, cv::Interp::Bilinear)?;
let u = cv::resize_u8(&u, width, height, cv::Interp::Bilinear)?;
let v = cv::resize_u8(&v, width, height, cv::Interp::Bilinear)?;
let mut out = vec![0u8; width as usize * height as usize * 3];
for (pixel, ((&y, &u), &v)) in out
.as_chunks_mut::<3>()
.0
.iter_mut()
.zip(y.data.iter().zip(u.data.iter()).zip(v.data.iter()))
{
let (b, g, r) = yuv_to_bgr(y, u, v);
pixel[0] = b;
pixel[1] = g;
pixel[2] = r;
}
ImageU8::new(width, height, 3, out)
}
pub(crate) fn bgra_to_bgr(
bgra: &[u8],
row_stride: i32,
width: i32,
height: i32,
) -> Result<ImageU8, String> {
if width <= 0 || height <= 0 {
return Err(format!("invalid frame size {width}x{height}"));
}
let row_bytes = width as usize * 4;
if row_stride < 0 || (row_stride as usize) < row_bytes {
return Err(format!(
"row stride {row_stride} is smaller than {row_bytes} bytes per row"
));
}
let row_stride = row_stride as usize;
let needed = required_len(height, row_stride as i32, row_bytes as i32)?;
if bgra.len() < needed {
return Err(format!(
"BGRA buffer has {} bytes, needs {needed}",
bgra.len()
));
}
let mut out = vec![0u8; width as usize * height as usize * 3];
for (row, dst_row) in out.chunks_exact_mut(width as usize * 3).enumerate() {
let src_row = &bgra[row * row_stride..row * row_stride + row_bytes];
for (dst, src) in dst_row
.as_chunks_mut::<3>()
.0
.iter_mut()
.zip(src_row.as_chunks::<4>().0.iter())
{
dst[0] = src[0];
dst[1] = src[1];
dst[2] = src[2];
}
}
ImageU8::new(width, height, 3, out)
}
#[cfg(test)]
mod tests {
use super::*;
fn reference_yuv_to_bgr(y: u8, u: u8, v: u8, ub: f64) -> (u8, u8, u8) {
let yf = (y as f64 - 16.0) * 1.164;
let uf = u as f64 - 128.0;
let vf = v as f64 - 128.0;
let r = yf + 1.596 * vf;
let g = yf - 0.391 * uf - 0.813 * vf;
let b = yf - ub * uf;
let to_u8 = |v: f64| v.round().clamp(0.0, 255.0) as u8;
(to_u8(b), to_u8(g), to_u8(r))
}
fn worst_deviation(triples: impl Iterator<Item = (u8, u8, u8)>, ub: f64) -> [i32; 3] {
let mut worst = [0i32; 3];
for (y, u, v) in triples {
let got = yuv_to_bgr(y, u, v);
let want = reference_yuv_to_bgr(y, u, v, ub);
let pairs = [(got.0, want.0), (got.1, want.1), (got.2, want.2)];
for (i, (a, b)) in pairs.iter().enumerate() {
worst[i] = worst[i].max((*a as i32 - *b as i32).abs());
}
}
worst
}
fn all_triples() -> impl Iterator<Item = (u8, u8, u8)> {
(0..=255u8)
.flat_map(|y| (0..=255u8).map(move |u| (y, u)))
.flat_map(|(y, u)| (0..=255u8).map(move |v| (y, u, v)))
}
fn strided_triples() -> impl Iterator<Item = (u8, u8, u8)> {
let axis = |step: usize| (0..=255u8).step_by(step).chain(std::iter::once(255));
axis(6).flat_map(move |y| axis(7).flat_map(move |u| axis(11).map(move |v| (y, u, v))))
}
struct Rng(u64);
impl Rng {
fn next_u8(&mut self) -> u8 {
let mut x = self.0;
x ^= x >> 12;
x ^= x << 25;
x ^= x >> 27;
self.0 = x;
(x.wrapping_mul(0x2545_F491_4F6C_DD1D) >> 33) as u8
}
}
#[test]
fn anchor_values_match_the_reference_matrix() {
assert_eq!(yuv_to_bgr(16, 128, 128), (0, 0, 0));
assert_eq!(yuv_to_bgr(235, 128, 128), (255, 255, 255));
assert_eq!(yuv_to_bgr(0, 128, 128), (0, 0, 0));
assert_eq!(yuv_to_bgr(255, 128, 128), (255, 255, 255));
let (b, g, r) = yuv_to_bgr(126, 128, 128);
assert_eq!((b, g, r), (128, 128, 128));
}
#[test]
fn fixed_point_matches_the_effective_matrix_on_a_subsample() {
let worst = worst_deviation(strided_triples(), -2.0);
assert!(
worst[0] == 0 && worst[1] <= 1 && worst[2] <= 1,
"B/G/R deviation from the effective matrix changed: {worst:?}"
);
}
#[test]
#[ignore = "exhaustive 2^24 sweep; run explicitly"]
fn fixed_point_matches_the_effective_matrix_exhaustively() {
let worst = worst_deviation(all_triples(), -2.0);
assert_eq!(
worst,
[0, 1, 1],
"B/G/R deviation from the effective matrix changed"
);
}
#[test]
#[ignore = "exhaustive 2^24 sweep; run explicitly"]
fn documented_matrix_differs_only_in_blue_by_the_ub_clamp() {
let worst = worst_deviation(all_triples(), -2.018);
assert_eq!(
worst,
[3, 1, 1],
"B/G/R deviation from the documented matrix changed"
);
}
#[test]
fn grey_ramp_is_monotonic_and_neutral() {
let mut previous = 0u8;
for y in 16..=235u8 {
let (b, g, r) = yuv_to_bgr(y, 128, 128);
assert_eq!(b, g, "neutral chroma must stay grey at Y={y}");
assert_eq!(g, r, "neutral chroma must stay grey at Y={y}");
assert!(b >= previous, "luma ramp must be monotonic at Y={y}");
previous = b;
}
}
fn planes_from_pixels(
width: i32,
height: i32,
pixel: impl Fn(i32, i32) -> (u8, u8, u8),
) -> (Vec<u8>, Vec<u8>, Vec<u8>) {
let cw = ceil_half(width);
let ch = ceil_half(height);
let mut y = vec![0u8; (width * height) as usize];
let mut u = vec![0u8; (cw * ch) as usize];
let mut v = vec![0u8; (cw * ch) as usize];
for row in 0..height {
for col in 0..width {
y[(row * width + col) as usize] = pixel(col, row).0;
}
}
for row in 0..ch {
for col in 0..cw {
let (_, cu, cv) = pixel(col * 2, row * 2);
u[(row * cw + col) as usize] = cu;
v[(row * cw + col) as usize] = cv;
}
}
(y, u, v)
}
#[test]
fn planar_and_interleaved_layouts_agree() {
let (width, height) = (37, 21);
let mut rng = Rng(0xC0FF_EE00_1234_5677);
let samples: Vec<(u8, u8, u8)> = (0..(width * height))
.map(|_| (rng.next_u8(), rng.next_u8(), rng.next_u8()))
.collect();
let pixel = |x: i32, y: i32| samples[(y * width + x) as usize];
let (y_plane, u_plane, v_plane) = planes_from_pixels(width, height, pixel);
let planar = yuv420_to_bgr(
&y_plane,
&u_plane,
&v_plane,
PlaneLayout {
width,
height,
y_row_stride: width,
uv_row_stride: ceil_half(width),
uv_pixel_stride: 1,
},
width,
height,
)
.expect("planar conversion");
let cw = ceil_half(width);
let ch = ceil_half(height);
let mut vu = vec![0u8; (cw * ch * 2) as usize];
for i in 0..(cw * ch) as usize {
vu[i * 2] = v_plane[i];
vu[i * 2 + 1] = u_plane[i];
}
let interleaved = yuv420_to_bgr(
&y_plane,
&vu[1..],
&vu,
PlaneLayout {
width,
height,
y_row_stride: width,
uv_row_stride: cw * 2,
uv_pixel_stride: 2,
},
width,
height,
)
.expect("interleaved conversion");
assert_eq!(planar, interleaved);
let mut uv = vec![0u8; (cw * ch * 2) as usize];
for i in 0..(cw * ch) as usize {
uv[i * 2] = u_plane[i];
uv[i * 2 + 1] = v_plane[i];
}
let nv12 = yuv420_to_bgr(
&y_plane,
&uv,
&uv[1..],
PlaneLayout {
width,
height,
y_row_stride: width,
uv_row_stride: cw * 2,
uv_pixel_stride: 2,
},
width,
height,
)
.expect("nv12 conversion");
assert_eq!(planar, nv12);
}
#[test]
fn row_padding_is_honoured() {
let (width, height) = (10, 6);
let mut rng = Rng(0x1234_5678_9ABC_DEF1);
let samples: Vec<(u8, u8, u8)> = (0..(width * height))
.map(|_| (rng.next_u8(), rng.next_u8(), rng.next_u8()))
.collect();
let pixel = |x: i32, y: i32| samples[(y * width + x) as usize];
let (y_tight, u_tight, v_tight) = planes_from_pixels(width, height, pixel);
let tight = yuv420_to_bgr(
&y_tight,
&u_tight,
&v_tight,
PlaneLayout {
width,
height,
y_row_stride: width,
uv_row_stride: ceil_half(width),
uv_pixel_stride: 1,
},
width,
height,
)
.expect("tight conversion");
let (y_stride, uv_stride) = (width + 7, ceil_half(width) + 3);
let cw = ceil_half(width);
let ch = ceil_half(height);
let mut y_pad = vec![0xA5u8; (y_stride * height) as usize];
let mut u_pad = vec![0x5Au8; (uv_stride * ch) as usize];
let mut v_pad = vec![0x3Cu8; (uv_stride * ch) as usize];
for row in 0..height {
for col in 0..width {
y_pad[(row * y_stride + col) as usize] = y_tight[(row * width + col) as usize];
}
}
for row in 0..ch {
for col in 0..cw {
u_pad[(row * uv_stride + col) as usize] = u_tight[(row * cw + col) as usize];
v_pad[(row * uv_stride + col) as usize] = v_tight[(row * cw + col) as usize];
}
}
let padded = yuv420_to_bgr(
&y_pad,
&u_pad,
&v_pad,
PlaneLayout {
width,
height,
y_row_stride: y_stride,
uv_row_stride: uv_stride,
uv_pixel_stride: 1,
},
width,
height,
)
.expect("padded conversion");
assert_eq!(tight, padded);
}
#[test]
fn odd_sizes_and_short_planes_are_rejected_not_panicked() {
let layout = PlaneLayout {
width: 8,
height: 8,
y_row_stride: 8,
uv_row_stride: 4,
uv_pixel_stride: 1,
};
assert!(yuv420_to_bgr(&[0u8; 63], &[0u8; 16], &[0u8; 16], layout, 8, 8).is_err());
assert!(yuv420_to_bgr(&[0u8; 64], &[0u8; 15], &[0u8; 16], layout, 8, 8).is_err());
assert!(yuv420_to_bgr(&[0u8; 64], &[0u8; 16], &[0u8; 15], layout, 8, 8).is_err());
assert!(
yuv420_to_bgr(
&[0u8; 64],
&[0u8; 16],
&[0u8; 16],
PlaneLayout {
y_row_stride: 4,
..layout
},
8,
8
)
.is_err()
);
assert!(
yuv420_to_bgr(
&[128u8],
&[128u8],
&[128u8],
PlaneLayout {
width: 1,
height: 1,
y_row_stride: 1,
uv_row_stride: 1,
uv_pixel_stride: 1,
},
1,
1
)
.is_ok()
);
}
#[test]
fn bgra_to_bgr_honours_row_padding_and_rejects_bad_geometry() {
let stride = 2 * 4 + 4;
let mut padded = vec![0xAAu8; stride * 2];
let rows = [
[[1u8, 2, 3, 255], [10, 20, 30, 0]],
[[40u8, 50, 60, 255], [70, 80, 90, 128]],
];
for (row, pixels) in rows.iter().enumerate() {
for (col, px) in pixels.iter().enumerate() {
padded[row * stride + col * 4..row * stride + col * 4 + 4].copy_from_slice(px);
}
}
let bgr = bgra_to_bgr(&padded, stride as i32, 2, 2).expect("conversion");
assert_eq!(bgr.data, vec![1, 2, 3, 10, 20, 30, 40, 50, 60, 70, 80, 90]);
let needed = stride + 2 * 4;
assert!(bgra_to_bgr(&padded[..needed], stride as i32, 2, 2).is_ok());
assert!(bgra_to_bgr(&padded[..needed - 1], stride as i32, 2, 2).is_err());
assert!(bgra_to_bgr(&padded, 7, 2, 2).is_err());
assert!(bgra_to_bgr(&padded, -8, 2, 2).is_err());
assert!(bgra_to_bgr(&padded, stride as i32, 0, 2).is_err());
}
}
#[cfg(test)]
mod invalid_metadata_tests {
use super::*;
#[test]
fn huge_camera_metadata_is_rejected_without_overflow() {
for width in [1, 3, i32::MAX] {
for stride in [1, i32::MAX] {
let layout = PlaneLayout {
width,
height: i32::MAX,
y_row_stride: i32::MAX,
uv_row_stride: stride,
uv_pixel_stride: i32::MAX,
};
assert!(yuv420_to_bgr(&[], &[], &[], layout, 256, 256).is_err());
assert!(bgra_to_bgr(&[], stride, width, i32::MAX).is_err());
}
}
}
}