π End-to-end encrypted cloud for everything.
Dart 2faandroidauthye2eeencryptionend-to-end-encryptionfluttergolanggoogle-photosgoogle-photos-alternativeiosopensourcephotosprivacyprivacy-toolstypescriptzero-knowledge
Optimize mobile ONNX models
laurensprieme606d7990867
diff --git a/infra/ml/playground/optimizations/README.md b/infra/ml/playground/optimizations/README.mdindex 30d8d1b93b..d5ad854a08 100644--- a/infra/ml/playground/optimizations/README.md+++ b/infra/ml/playground/optimizations/README.md@@ -1,8 +1,8 @@# Mobile ML model optimizationsThis directory records the production transformations applied to Ente's mobile-face models. The generated CDN artifacts are written under `models/`; ONNX-files are intentionally gitignored, while `model_manifest.json` records the+ML models. The generated CDN artifacts are written under `models/`; ONNX files+are intentionally gitignored, while `model_manifest.json` records thereproducible output metadata.## Rebuilding the models@@ -11,7 +11,7 @@ Run from the repository root:```shuv run --project infra/ml/playground --no-sync python \- infra/ml/playground/optimizations/optimize_face_models.py \+ infra/ml/playground/optimizations/optimize_models.py \--source-dir infra/ml/test/.cache/local_model_mirror \--output-dir infra/ml/playground/optimizations/models```@@ -20,11 +20,16 @@ The script performs only the transformations selected for production:- YOLO: fix the batch dimension at 1 and use ONNX Runtime's basic optimizer toconstant-fold the resulting shape graph.-- MobileFaceNet: fix the batch dimension at 1, replace 33 decomposed PReLU- expressions with ONNX `PRelu`, make two implicit zero-padding attributes- explicit, and remove the final L2 normalization that the Rust caller already- performs.+- MobileFaceNet: fix the batch dimension at 1 and express each of its 33 trained+ PReLU activations exactly as `Relu(x) - alpha * Relu(x * -1)`. This avoids a+ WebGPU-only runtime kernel while using operators supported by both CoreML+ MLProgram and WebGPU, so the generated artifact can be shared by Android and+ iOS. The script also makes two implicit zero-padding attributes explicit and+ removes the final L2 normalization that the Rust caller already performs.+- MobileCLIP: convert the graph to ONNX opset 20 and replace 54 expanded exact+ GELU expressions with `Gelu(approximate="none")`. This keeps FP32/exact GELU+ semantics while exposing the fused operator to CoreML and WebGPU.-The script verifies the source-model hashes and emits the two ONNX files plus+The script verifies the source-model hashes and emits the three ONNX files plus`model_manifest.json`, which records their output hashes, shapes, sizes, nodecounts, and operator inventories.diff --git a/infra/ml/playground/optimizations/models/model_manifest.json b/infra/ml/playground/optimizations/models/model_manifest.jsonindex 6ceffb7573..0ec8f4ec60 100644--- a/infra/ml/playground/optimizations/models/model_manifest.json+++ b/infra/ml/playground/optimizations/models/model_manifest.json@@ -34,16 +34,18 @@]}},- "mobilefacenet_prelu_static_b1": {- "path": "infra/ml/playground/optimizations/models/mobilefacenet_prelu_static_b1.onnx",- "sha256": "f525daf3089ec41c938a66c47cd8278c2a443f31ee2bd2dac987db531c58ccfd",- "bytes": 5238749,- "nodes": 101,+ "mobilefacenet_portable_static_b1": {+ "path": "infra/ml/playground/optimizations/models/mobilefacenet_portable_static_b1.onnx",+ "sha256": "0763fc33f54e138476194da95987e133b3e976075a6b1d3e1b2caedb251b1a36",+ "bytes": 5278803,+ "nodes": 233,"operators": {"Add": 12,"Conv": 52,- "PRelu": 33,+ "Mul": 66,+ "Relu": 66,"Reshape": 3,+ "Sub": 33,"Transpose": 1},"inputs": {@@ -60,5 +62,47 @@192]}+ },+ "mobileclip_exact_gelu_opset20": {+ "path": "infra/ml/playground/optimizations/models/mobileclip_s2_image_gelu_opset20.onnx",+ "sha256": "205a430af825e501c5138e5bb9abea942482a7a4fd4a680e98e47cf0830dce7e",+ "bytes": 143057352,+ "nodes": 504,+ "operators": {+ "Add": 52,+ "AveragePool": 1,+ "BatchNormalization": 4,+ "Concat": 4,+ "Constant": 53,+ "Conv": 189,+ "Gelu": 54,+ "MatMul": 17,+ "Mul": 55,+ "Pad": 1,+ "ReduceMean": 3,+ "Relu": 3,+ "Reshape": 17,+ "Shape": 4,+ "Sigmoid": 3,+ "Slice": 4,+ "Softmax": 4,+ "Split": 4,+ "Squeeze": 12,+ "Transpose": 20+ },+ "inputs": {+ "input": [+ 1,+ 3,+ 256,+ 256+ ]+ },+ "outputs": {+ "output": [+ 1,+ 512+ ]+ }}}diff --git a/infra/ml/playground/optimizations/optimize_face_models.py b/infra/ml/playground/optimizations/optimize_models.pysimilarity index 54%rename from infra/ml/playground/optimizations/optimize_face_models.pyrename to infra/ml/playground/optimizations/optimize_models.pyindex de2084dc50..13331dc64d 100644--- a/infra/ml/playground/optimizations/optimize_face_models.py+++ b/infra/ml/playground/optimizations/optimize_models.py@@ -1,9 +1,9 @@#!/usr/bin/env python3-"""Build Ente's optimized static face models for the mobile ML pipeline.+"""Build Ente's optimized ONNX models for the mobile ML pipeline.The generated models remain standard ONNX. The transformations make their-graphs easier for CoreML and mobile CPU runtimes to optimize while preserving-the outputs used by the application.+graphs easier for CoreML, WebGPU, and mobile CPU runtimes to optimize while+preserving the outputs used by the application."""from __future__ import annotations@@ -11,19 +11,25 @@ from __future__ import annotationsimport argparseimport hashlibimport json+import mathfrom collections import Counterfrom pathlib import Pathimport onnximport onnxruntime as ort-from onnx import helper, numpy_helper+from onnx import helper, numpy_helper, version_converterYOLO_SOURCE = "yolov5s_face_640_640_dynamic.onnx"YOLO_SOURCE_SHA256 = "71a008707283b03db4881449a24f4da197f9dbd9ddaca5c91fcdb363fbf7e06f"YOLO_OUTPUT = "yolov5s_face_640_640_static_b1.onnx"FACE_SOURCE = "mobilefacenet_opset15.onnx"FACE_SOURCE_SHA256 = "472a0f7e24d0b070cbbdc031b085bc2a06c70655b3bdefb87dbd69bc98662f45"-FACE_OUTPUT = "mobilefacenet_prelu_static_b1.onnx"+FACE_OUTPUT = "mobilefacenet_portable_static_b1.onnx"+PRELU_NEGATIVE_ONE = "ente_prelu_negative_one"+CLIP_SOURCE = "mobileclip_s2_image.onnx"+CLIP_SOURCE_SHA256 = "ef54ec66c687603eb4dd303e20d9b67e81069d3133b1c69a70028c76718b7752"+CLIP_OUTPUT = "mobileclip_s2_image_gelu_opset20.onnx"+EXPECTED_GELU_REWRITES = 54def sha256(path: Path) -> str:@@ -85,13 +91,144 @@ def is_scalar_initializer(return values.size == 1 and float(values[0]) == expected+def constant_scalar(+ producer_by_output: dict[str, onnx.NodeProto],+ tensor_name: str,+) -> float | None:+ """Return a scalar emitted by an ONNX Constant node, if present."""+ node = producer_by_output.get(tensor_name)+ if node is None or node.op_type != "Constant":+ return None+ value = next(+ (attribute.t for attribute in node.attribute if attribute.name == "value"),+ None,+ )+ if value is None:+ return None+ values = numpy_helper.to_array(value).reshape(-1)+ return float(values[0]) if values.size == 1 else None+++def ensure_unique_node_names(model: onnx.ModelProto) -> None:+ """Name adapter nodes inserted by ONNX's opset-version converter."""+ existing = {node.name for node in model.graph.node if node.name}+ for index, node in enumerate(model.graph.node):+ if node.name:+ continue+ base = f"ente_opset20_{index}_{node.op_type}"+ candidate = base+ suffix = 1+ while candidate in existing:+ candidate = f"{base}_{suffix}"+ suffix += 1+ node.name = candidate+ existing.add(candidate)+++def rewrite_exact_gelu(model: onnx.ModelProto) -> int:+ """Collapse x/sqrt(2) -> Erf -> +1 -> *x -> *0.5 into exact GELU."""+ producer_by_output = {+ output: node for node in model.graph.node for output in node.output+ }+ consumers = node_consumers(model)+ replacements: dict[str, onnx.NodeProto] = {}+ removed_names: set[str] = set()+ removable_constants: set[str] = set()++ node_names = [node.name for node in model.graph.node]+ if not all(node_names) or len(node_names) != len(set(node_names)):+ raise ValueError("GELU rewrite requires unique, non-empty ONNX node names")++ for div in (node for node in model.graph.node if node.op_type == "Div"):+ if len(div.input) != 2:+ continue+ div_constant = next(+ (+ input_name+ for input_name in div.input+ if constant_scalar(producer_by_output, input_name) is not None+ ),+ None,+ )+ if div_constant is None:+ continue+ input_name = other_input(div, div_constant)+ divisor = constant_scalar(producer_by_output, div_constant)+ if (+ input_name is None+ or divisor is None+ or not math.isclose(divisor, math.sqrt(2.0), rel_tol=0.0, abs_tol=1e-6)+ ):+ continue++ erf = only_consumer(consumers, div.output[0], "Erf")+ if erf is None:+ continue+ add = only_consumer(consumers, erf.output[0], "Add")+ if add is None:+ continue+ add_constant = other_input(add, erf.output[0])+ if (+ add_constant is None+ or constant_scalar(producer_by_output, add_constant) != 1.0+ ):+ continue++ first_mul = only_consumer(consumers, add.output[0], "Mul")+ if first_mul is None or set(first_mul.input) != {input_name, add.output[0]}:+ continue+ final_mul = only_consumer(consumers, first_mul.output[0], "Mul")+ if final_mul is None:+ continue+ half_constant = other_input(final_mul, first_mul.output[0])+ if (+ half_constant is None+ or constant_scalar(producer_by_output, half_constant) != 0.5+ ):+ continue++ replacements[div.name] = helper.make_node(+ "Gelu",+ [input_name],+ list(final_mul.output),+ name=f"{final_mul.name}/ExactGelu",+ approximate="none",+ )+ removed_names.update(+ node.name for node in (div, erf, add, first_mul, final_mul)+ )+ removable_constants.update(+ producer_by_output[name].name+ for name in (div_constant, add_constant, half_constant)+ if len(consumers.get(name, [])) == 1+ )++ rewritten_nodes = []+ for node in model.graph.node:+ if node.name in replacements:+ rewritten_nodes.append(replacements[node.name])+ elif node.name not in removed_names and node.name not in removable_constants:+ rewritten_nodes.append(node)+ del model.graph.node[:]+ model.graph.node.extend(rewritten_nodes)+ return len(replacements)++def rewrite_prelu_decompositions(model: onnx.ModelProto) -> int:- """Replace max(x, 0) + alpha * min(x, 0) decompositions with PRelu."""+ """Replace exported PReLU expressions with CoreML/WebGPU-compatible ops.++ The source graph expresses PReLU using Relu, Abs, Sub, Mul, Mul, and Add.+ Abs is not supported by CoreML's documented MLProgram operator set, while+ native PRelu is not supported by the WebGPU EP used on Android. The exact+ equivalent below uses only operators documented by both providers:++ PReLU(x, alpha) = Relu(x) - alpha * Relu(x * -1)+ """consumers = node_consumers(model)initializers = {initializer.name: initializer for initializer in model.graph.initializer}- replacements: dict[str, onnx.NodeProto] = {}+ replacements: dict[str, list[onnx.NodeProto]] = {}removed_names: set[str] = set()node_names = [node.name for node in model.graph.node]@@ -123,9 +260,7 @@ def rewrite_prelu_decompositions(model: onnx.ModelProto) -> int:if half_mul is None:continuehalf_name = other_input(half_mul, alpha_mul.output[0])- if half_name is None or not is_scalar_initializer(- initializers, half_name, 0.5- ):+ if half_name is None or not is_scalar_initializer(initializers, half_name, 0.5):continueadd = only_consumer(consumers, relu.output[0], "Add")@@ -135,20 +270,52 @@ def rewrite_prelu_decompositions(model: onnx.ModelProto) -> int:if half_add is None or half_add.name != add.name:continue- # The export stores slopes as [1, C, 1, 1]. ONNX PRelu and the mobile- # accelerators expect a per-channel [C, 1, 1] slope for NCHW tensors.- if list(alpha.dims[-2:]) != [1, 1]:+ # The export stores slopes as [1, C, 1, 1], which broadcasts directly+ # over the NCHW activation tensor used at this point in the graph.+ if (+ len(alpha.dims) != 4+ or alpha.dims[0] != 1+ or list(alpha.dims[-2:]) != [1, 1]+ ):continue- slope_dims = list(alpha.dims[1:])- del alpha.dims[:]- alpha.dims.extend(slope_dims)-- replacements[relu.name] = helper.make_node(- "PRelu",- [input_name, alpha.name],- list(add.output),- name=f"{add.name}/PRelu",- )++ output_name = add.output[0]+ positive_name = f"{output_name}/positive"+ negated_name = f"{output_name}/negated"+ negative_magnitude_name = f"{output_name}/negative_magnitude"+ scaled_negative_name = f"{output_name}/scaled_negative"+ replacements[relu.name] = [+ helper.make_node(+ "Relu",+ [input_name],+ [positive_name],+ name=f"{add.name}/PositiveRelu",+ ),+ helper.make_node(+ "Mul",+ [input_name, PRELU_NEGATIVE_ONE],+ [negated_name],+ name=f"{add.name}/Negate",+ ),+ helper.make_node(+ "Relu",+ [negated_name],+ [negative_magnitude_name],+ name=f"{add.name}/NegativeRelu",+ ),+ helper.make_node(+ "Mul",+ [negative_magnitude_name, alpha.name],+ [scaled_negative_name],+ name=f"{add.name}/ScaleNegative",+ ),+ helper.make_node(+ "Sub",+ [positive_name, scaled_negative_name],+ [output_name],+ name=f"{add.name}/PortablePRelu",+ ),+ ]removed_names.update(node.name for node in (relu, abs_node, sub, alpha_mul, half_mul, add))@@ -156,11 +323,20 @@ def rewrite_prelu_decompositions(model: onnx.ModelProto) -> int:rewritten_nodes = []for node in model.graph.node:if node.name in replacements:- rewritten_nodes.append(replacements[node.name])+ rewritten_nodes.extend(replacements[node.name])elif node.name not in removed_names:rewritten_nodes.append(node)del model.graph.node[:]model.graph.node.extend(rewritten_nodes)+ if replacements:+ model.graph.initializer.append(+ helper.make_tensor(+ PRELU_NEGATIVE_ONE,+ onnx.TensorProto.FLOAT,+ [],+ [-1.0],+ )+ )return len(replacements)@@ -272,9 +448,21 @@ def build_models(source_dir: Path, output_dir: Path) -> dict[str, object]:face_path = output_dir / FACE_OUTPUTface = optimize_with_ort(face, face_path)+ clip = load_source_model(source_dir, CLIP_SOURCE, CLIP_SOURCE_SHA256)+ clip = version_converter.convert_version(clip, 20)+ ensure_unique_node_names(clip)+ if rewrite_exact_gelu(clip) != EXPECTED_GELU_REWRITES:+ raise RuntimeError(+ f"expected exactly {EXPECTED_GELU_REWRITES} MobileCLIP GELU rewrites"+ )+ onnx.checker.check_model(clip, full_check=True)+ clip_path = output_dir / CLIP_OUTPUT+ onnx.save(clip, clip_path)+return {"yolo_static_b1": describe(yolo_path, yolo),- "mobilefacenet_prelu_static_b1": describe(face_path, face),+ "mobilefacenet_portable_static_b1": describe(face_path, face),+ "mobileclip_exact_gelu_opset20": describe(clip_path, clip),}