π End-to-end encrypted cloud for everything.
Dart 2faandroidauthye2eeencryptionend-to-end-encryptionfluttergolanggoogle-photosgoogle-photos-alternativeiosopensourcephotosprivacyprivacy-toolstypescriptzero-knowledge
use ort::session::{Session, SessionInputValue};
use ort::value::{Tensor, TensorRef};
use crate::onnx::{SessionRunError, SessionRunResult};
#[derive(Clone)]
pub(super) struct SharedValues(std::sync::Arc<Vec<f32>>);
impl std::ops::Deref for SharedValues {
type Target = Vec<f32>;
fn deref(&self) -> &Self::Target {
&self.0
}
}
impl std::ops::DerefMut for SharedValues {
fn deref_mut(&mut self) -> &mut Self::Target {
std::sync::Arc::make_mut(&mut self.0)
}
}
#[derive(Clone)]
pub(super) struct Input {
pub name: String,
pub shape: Vec<i64>,
pub values: SharedValues,
}
impl Input {
pub fn zeros(name: impl Into<String>, shape: &[usize]) -> Self {
Self {
name: name.into(),
shape: shape.iter().map(|&v| v as i64).collect(),
values: SharedValues(std::sync::Arc::new(vec![0.0; shape.iter().product()])),
}
}
}
pub(super) fn infer(
session: &mut Session,
inputs: &[Input],
) -> SessionRunResult<(Vec<i64>, Vec<f32>)> {
let tensors: Result<Vec<_>, _> = inputs
.iter()
.map(|input| {
if input.name.starts_with("path_") {
return Tensor::from_array((input.shape.clone(), vec![input.values[0] != 0.0]))
.map(|tensor| (input.name.as_str(), SessionInputValue::from(tensor)));
}
TensorRef::from_array_view((input.shape.as_slice(), input.values.as_slice()))
.map(|tensor| (input.name.as_str(), SessionInputValue::from(tensor)))
})
.collect();
let tensors = tensors?;
let result = session.run(tensors);
let outputs = result.map_err(SessionRunError::from_inference_error)?;
let (shape, values) = outputs[0].try_extract_tensor::<f32>()?;
Ok((shape.to_vec(), values.to_vec()))
}
pub(super) fn detector(raw: &[f32], height: usize, width: usize) -> Vec<Input> {
let side = 960;
let mut image = Input::zeros("x", &[1, 3, side, side]);
for channel in 0..3 {
for y in 0..height {
let source = (channel * height + y) * width;
let target = (channel * side + y) * side;
image.values[target..target + width].copy_from_slice(&raw[source..source + width]);
}
}
detector_paths(image, height, width)
}
pub(super) struct Line {
pub values: Vec<f32>,
pub width: usize,
pub content_width: i32,
}
pub(super) fn packed(lines: &[&Line], width: usize, slots: usize) -> (Vec<Input>, Vec<usize>) {
let mut image = Input::zeros("x", &[1, 3, 48, width]);
let mut positions = Vec::with_capacity(lines.len());
let mut cursor = 0;
for line in lines {
for row in 0..3 * 48 {
let source = row * line.width;
let target = row * width + cursor;
image.values[target..target + line.width]
.copy_from_slice(&line.values[source..source + line.width]);
}
positions.push(cursor);
cursor += line.width.div_ceil(8) * 8 + 16;
}
let mut inputs = vec![image];
for (height, scale) in [(48, 1), (24, 2), (12, 2), (12, 4), (6, 4), (3, 4), (1, 8)] {
let size = width / scale;
let mut input = Input::zeros(format!("mask_{height}_{size}"), &[1, 1, 1, size]);
for (line, &offset) in lines.iter().zip(&positions) {
let count = if scale == 8 {
line.width.div_ceil(4) / 2
} else {
line.width.div_ceil(scale)
};
input.values[offset / scale..offset / scale + count].fill(1.0);
}
inputs.push(input);
}
let size = width / 4;
let mut pool = Input::zeros(format!("pool_weights_{size}"), &[1, size, slots]);
let mut gate = Input::zeros(format!("gate_weights_{size}"), &[1, slots, size]);
for (slot, (line, &offset)) in lines.iter().zip(&positions).enumerate() {
let count = line.width.div_ceil(4);
for x in offset / 4..offset / 4 + count {
pool.values[x * slots + slot] = 1.0 / count as f32;
gate.values[slot * size + x] = 1.0;
}
}
inputs.extend([pool, gate]);
let steps = width / 8;
let mut attention = Input::zeros("attention_bias", &[1, 1, steps, steps]);
attention.values.fill(-10000.0);
for query in 0..steps {
attention.values[query * steps] = 0.0;
}
for (line, &offset) in lines.iter().zip(&positions) {
let count = line.width.div_ceil(4) / 2;
let start = offset / 8;
for query in start..start + count {
let row = &mut attention.values[query * steps..(query + 1) * steps];
row.fill(-10000.0);
row[start..start + count].fill(0.0);
}
}
inputs.push(attention);
(inputs, positions)
}
fn detector_auxiliary(index: usize, height: usize, width: usize, h: usize, w: usize) -> Vec<Input> {
let mut inputs = Vec::new();
for scale in [1, 2, 4, 8, 16, 32] {
let mh = h / scale;
let mw = w / scale;
let mut mask = Input::zeros(format!("p{index}_mask_{mh}_{mw}"), &[1, 1, mh, mw]);
for y in 0..height / scale {
mask.values[y * mw..y * mw + width / scale].fill(1.0);
}
inputs.push(mask);
if scale >= 4 {
let mut value = Input::zeros(format!("p{index}_pool_scale_{mh}_{mw}"), &[1, 1, 1, 1]);
value.values[0] = if height > 0 && width > 0 {
(h * w) as f32 / (height * width) as f32
} else {
1.0
};
inputs.push(value);
}
}
inputs
}
fn detector_paths(image: Input, height: usize, width: usize) -> Vec<Input> {
static UNUSED: std::sync::OnceLock<Vec<Vec<Input>>> = std::sync::OnceLock::new();
let shapes = &[(960, 480), (480, 960), (960, 704), (704, 960), (960, 960)];
let unused = UNUSED.get_or_init(|| {
shapes
.iter()
.enumerate()
.map(|(i, &(h, w))| detector_auxiliary(i, 0, 0, h, w))
.collect()
});
let selected = shapes
.iter()
.position(|&(h, w)| height <= h && width <= w)
.unwrap_or(shapes.len() - 1);
let mut inputs = vec![image];
for (index, &(h, w)) in shapes.iter().enumerate() {
inputs.extend(if index == selected {
detector_auxiliary(index, height, width, h, w)
} else {
unused[index].clone()
});
if index < shapes.len() - 1 {
let mut flag = Input::zeros(format!("path_{index}"), &[]);
flag.values[0] = if selected == index { 1.0 } else { 0.0 };
inputs.push(flag);
}
}
inputs
}
pub(super) fn branch_widths() -> &'static [usize] {
&[2048, 7168]
}
pub(super) fn recognizer_paths(mut selected: Vec<Input>, width: usize) -> Vec<Input> {
let original = selected.remove(0);
let image = if width == 7168 {
original
} else {
let mut image = Input::zeros("x", &[1, 3, 48, 7168]);
for row in 0..3 * 48 {
image.values[row * 7168..row * 7168 + width]
.copy_from_slice(&original.values[row * width..(row + 1) * width]);
}
image
};
static UNUSED: std::sync::OnceLock<Vec<Vec<Input>>> = std::sync::OnceLock::new();
let widths = branch_widths();
let unused = UNUSED.get_or_init(|| {
widths
.iter()
.map(|&w| {
let (mut values, _) = packed(&[], w, w / 336);
values.remove(0);
values
})
.collect()
});
let mut inputs = vec![image];
for (index, &branch_width) in widths.iter().enumerate() {
let mut values = if branch_width == width {
std::mem::take(&mut selected)
} else {
unused[index].clone()
};
for input in &mut values {
input.name = format!("p{index}_{}", input.name);
}
inputs.extend(values);
}
for (index, &w) in widths.iter().enumerate().take(widths.len() - 1) {
let mut flag = Input::zeros(format!("path_{index}"), &[]);
flag.values[0] = if width == w { 1.0 } else { 0.0 };
inputs.push(flag);
}
inputs
}
#[cfg(test)]
mod tests {
use super::*;
fn input<'a>(inputs: &'a [Input], name: &str) -> &'a Input {
inputs.iter().find(|input| input.name == name).unwrap()
}
#[test]
fn detector_masks_preserve_area_at_every_scale_and_choose_the_smallest_path() {
for (height, width, path) in [
(32, 32, 0),
(960, 480, 0),
(448, 960, 1),
(960, 672, 2),
(704, 960, 3),
(960, 736, 4),
] {
let raw = vec![0.5; 3 * height * width];
let inputs = detector(&raw, height, width);
let image = input(&inputs, "x");
assert_eq!(image.shape, [1, 3, 960, 960]);
for channel in 0..3 {
for row in 0..960 {
let pixels =
&image.values[(channel * 960 + row) * 960..(channel * 960 + row + 1) * 960];
if row < height {
assert!(pixels[..width].iter().all(|&v| v == 0.5));
assert!(pixels[width..].iter().all(|&v| v == 0.0));
} else {
assert!(pixels.iter().all(|&v| v == 0.0));
}
}
}
for index in 0..4 {
assert_eq!(
input(&inputs, &format!("path_{index}")).values[0],
f32::from(index == path)
);
}
let prefix = format!("p{path}_mask_");
for mask in inputs
.iter()
.filter(|input| input.name.starts_with(&prefix))
{
let scale = match path {
1 => 480,
3 => 704,
_ => 960,
} / mask.shape[2] as usize;
let active = height / scale * (width / scale);
assert_eq!(mask.values.iter().filter(|&&v| v == 1.0).count(), active);
let scale_name = mask.name.replace("mask", "pool_scale");
if let Some(pool_scale) = inputs.iter().find(|input| input.name == scale_name) {
assert!(
(pool_scale.values[0] * active as f32 / mask.values.len() as f32 - 1.0)
.abs()
< 1e-6
);
}
}
}
}
#[test]
fn packing_isolates_odd_width_lines_in_pooling_and_attention() {
let lines: Vec<_> = [321, 327]
.into_iter()
.map(|width| Line {
values: vec![0.25; 3 * 48 * width],
width,
content_width: width as i32,
})
.collect();
let (inputs, offsets) = packed(&[&lines[0], &lines[1]], 2048, 6);
assert_eq!(offsets, [0, 344]);
let mask = input(&inputs, "mask_1_256");
assert_eq!(mask.values.iter().filter(|&&v| v == 1.0).count(), 81);
let attention = input(&inputs, "attention_bias");
for (line, &offset) in lines.iter().zip(&offsets) {
let start = offset / 8;
let count = line.width.div_ceil(4) / 2;
for query in start..start + count {
let row = &attention.values[query * 256..(query + 1) * 256];
assert!(row[start..start + count].iter().all(|&v| v == 0.0));
assert!(
row[..start]
.iter()
.chain(&row[start + count..])
.all(|&v| v == -10000.0)
);
}
}
let pool = input(&inputs, "pool_weights_512");
for slot in 0..2 {
let sum: f32 = pool.values.iter().skip(slot).step_by(6).sum();
assert!((sum - 1.0).abs() < 1e-6);
}
let lifted = recognizer_paths(inputs, 2048);
assert_eq!(input(&lifted, "x").shape, [1, 3, 48, 7168]);
assert_eq!(input(&lifted, "path_0").values[0], 1.0);
assert_eq!(input(&lifted, "p1_attention_bias").shape, [1, 1, 896, 896]);
}
#[test]
fn packing_preserves_the_last_step_at_the_maximum_width() {
for width in [7167, 7168] {
let line = Line {
values: vec![0.5; 3 * 48 * width],
width,
content_width: width as i32,
};
let (inputs, offsets) = packed(&[&line], 7168, 21);
assert_eq!(offsets, [0]);
assert_eq!(input(&inputs, "mask_1_896").values[895], 1.0);
let pixels = input(&inputs, "x").values.as_ptr();
let inputs = recognizer_paths(inputs, 7168);
assert_eq!(input(&inputs, "path_0").values[0], 0.0);
assert_eq!(input(&inputs, "x").values.as_ptr(), pixels);
}
}
}