Skip to content

armgit / ente

Public Imported from GitHub
0

πŸ’š End-to-end encrypted cloud for everything.

Optimize mobile ONNX models

laurensprieme606d7990867
diff --git a/infra/ml/playground/optimizations/README.md b/infra/ml/playground/optimizations/README.md
index 30d8d1b93b..d5ad854a08 100644
--- a/infra/ml/playground/optimizations/README.md
+++ b/infra/ml/playground/optimizations/README.md
@@ -1,8 +1,8 @@
# Mobile ML model optimizations
This directory records the production transformations applied to Ente's mobile
-face models. The generated CDN artifacts are written under `models/`; ONNX
-files are intentionally gitignored, while `model_manifest.json` records the
+ML models. The generated CDN artifacts are written under `models/`; ONNX files
+are intentionally gitignored, while `model_manifest.json` records the
reproducible output metadata.
## Rebuilding the models
@@ -11,7 +11,7 @@ Run from the repository root:
```sh
uv run --project infra/ml/playground --no-sync python \
- infra/ml/playground/optimizations/optimize_face_models.py \
+ infra/ml/playground/optimizations/optimize_models.py \
--source-dir infra/ml/test/.cache/local_model_mirror \
--output-dir infra/ml/playground/optimizations/models
```
@@ -20,11 +20,16 @@ The script performs only the transformations selected for production:
- YOLO: fix the batch dimension at 1 and use ONNX Runtime's basic optimizer to
constant-fold the resulting shape graph.
-- MobileFaceNet: fix the batch dimension at 1, replace 33 decomposed PReLU
- expressions with ONNX `PRelu`, make two implicit zero-padding attributes
- explicit, and remove the final L2 normalization that the Rust caller already
- performs.
+- MobileFaceNet: fix the batch dimension at 1 and express each of its 33 trained
+ PReLU activations exactly as `Relu(x) - alpha * Relu(x * -1)`. This avoids a
+ WebGPU-only runtime kernel while using operators supported by both CoreML
+ MLProgram and WebGPU, so the generated artifact can be shared by Android and
+ iOS. The script also makes two implicit zero-padding attributes explicit and
+ removes the final L2 normalization that the Rust caller already performs.
+- MobileCLIP: convert the graph to ONNX opset 20 and replace 54 expanded exact
+ GELU expressions with `Gelu(approximate="none")`. This keeps FP32/exact GELU
+ semantics while exposing the fused operator to CoreML and WebGPU.
-The script verifies the source-model hashes and emits the two ONNX files plus
+The script verifies the source-model hashes and emits the three ONNX files plus
`model_manifest.json`, which records their output hashes, shapes, sizes, node
counts, and operator inventories.
diff --git a/infra/ml/playground/optimizations/models/model_manifest.json b/infra/ml/playground/optimizations/models/model_manifest.json
index 6ceffb7573..0ec8f4ec60 100644
--- a/infra/ml/playground/optimizations/models/model_manifest.json
+++ b/infra/ml/playground/optimizations/models/model_manifest.json
@@ -34,16 +34,18 @@
]
}
},
- "mobilefacenet_prelu_static_b1": {
- "path": "infra/ml/playground/optimizations/models/mobilefacenet_prelu_static_b1.onnx",
- "sha256": "f525daf3089ec41c938a66c47cd8278c2a443f31ee2bd2dac987db531c58ccfd",
- "bytes": 5238749,
- "nodes": 101,
+ "mobilefacenet_portable_static_b1": {
+ "path": "infra/ml/playground/optimizations/models/mobilefacenet_portable_static_b1.onnx",
+ "sha256": "0763fc33f54e138476194da95987e133b3e976075a6b1d3e1b2caedb251b1a36",
+ "bytes": 5278803,
+ "nodes": 233,
"operators": {
"Add": 12,
"Conv": 52,
- "PRelu": 33,
+ "Mul": 66,
+ "Relu": 66,
"Reshape": 3,
+ "Sub": 33,
"Transpose": 1
},
"inputs": {
@@ -60,5 +62,47 @@
192
]
}
+ },
+ "mobileclip_exact_gelu_opset20": {
+ "path": "infra/ml/playground/optimizations/models/mobileclip_s2_image_gelu_opset20.onnx",
+ "sha256": "205a430af825e501c5138e5bb9abea942482a7a4fd4a680e98e47cf0830dce7e",
+ "bytes": 143057352,
+ "nodes": 504,
+ "operators": {
+ "Add": 52,
+ "AveragePool": 1,
+ "BatchNormalization": 4,
+ "Concat": 4,
+ "Constant": 53,
+ "Conv": 189,
+ "Gelu": 54,
+ "MatMul": 17,
+ "Mul": 55,
+ "Pad": 1,
+ "ReduceMean": 3,
+ "Relu": 3,
+ "Reshape": 17,
+ "Shape": 4,
+ "Sigmoid": 3,
+ "Slice": 4,
+ "Softmax": 4,
+ "Split": 4,
+ "Squeeze": 12,
+ "Transpose": 20
+ },
+ "inputs": {
+ "input": [
+ 1,
+ 3,
+ 256,
+ 256
+ ]
+ },
+ "outputs": {
+ "output": [
+ 1,
+ 512
+ ]
+ }
}
}
diff --git a/infra/ml/playground/optimizations/optimize_face_models.py b/infra/ml/playground/optimizations/optimize_models.py
similarity index 54%
rename from infra/ml/playground/optimizations/optimize_face_models.py
rename to infra/ml/playground/optimizations/optimize_models.py
index de2084dc50..13331dc64d 100644
--- a/infra/ml/playground/optimizations/optimize_face_models.py
+++ b/infra/ml/playground/optimizations/optimize_models.py
@@ -1,9 +1,9 @@
#!/usr/bin/env python3
-"""Build Ente's optimized static face models for the mobile ML pipeline.
+"""Build Ente's optimized ONNX models for the mobile ML pipeline.
The generated models remain standard ONNX. The transformations make their
-graphs easier for CoreML and mobile CPU runtimes to optimize while preserving
-the outputs used by the application.
+graphs easier for CoreML, WebGPU, and mobile CPU runtimes to optimize while
+preserving the outputs used by the application.
"""
from __future__ import annotations
@@ -11,19 +11,25 @@ from __future__ import annotations
import argparse
import hashlib
import json
+import math
from collections import Counter
from pathlib import Path
import onnx
import onnxruntime as ort
-from onnx import helper, numpy_helper
+from onnx import helper, numpy_helper, version_converter
YOLO_SOURCE = "yolov5s_face_640_640_dynamic.onnx"
YOLO_SOURCE_SHA256 = "71a008707283b03db4881449a24f4da197f9dbd9ddaca5c91fcdb363fbf7e06f"
YOLO_OUTPUT = "yolov5s_face_640_640_static_b1.onnx"
FACE_SOURCE = "mobilefacenet_opset15.onnx"
FACE_SOURCE_SHA256 = "472a0f7e24d0b070cbbdc031b085bc2a06c70655b3bdefb87dbd69bc98662f45"
-FACE_OUTPUT = "mobilefacenet_prelu_static_b1.onnx"
+FACE_OUTPUT = "mobilefacenet_portable_static_b1.onnx"
+PRELU_NEGATIVE_ONE = "ente_prelu_negative_one"
+CLIP_SOURCE = "mobileclip_s2_image.onnx"
+CLIP_SOURCE_SHA256 = "ef54ec66c687603eb4dd303e20d9b67e81069d3133b1c69a70028c76718b7752"
+CLIP_OUTPUT = "mobileclip_s2_image_gelu_opset20.onnx"
+EXPECTED_GELU_REWRITES = 54
def sha256(path: Path) -> str:
@@ -85,13 +91,144 @@ def is_scalar_initializer(
return values.size == 1 and float(values[0]) == expected
+def constant_scalar(
+ producer_by_output: dict[str, onnx.NodeProto],
+ tensor_name: str,
+) -> float | None:
+ """Return a scalar emitted by an ONNX Constant node, if present."""
+ node = producer_by_output.get(tensor_name)
+ if node is None or node.op_type != "Constant":
+ return None
+ value = next(
+ (attribute.t for attribute in node.attribute if attribute.name == "value"),
+ None,
+ )
+ if value is None:
+ return None
+ values = numpy_helper.to_array(value).reshape(-1)
+ return float(values[0]) if values.size == 1 else None
+
+
+def ensure_unique_node_names(model: onnx.ModelProto) -> None:
+ """Name adapter nodes inserted by ONNX's opset-version converter."""
+ existing = {node.name for node in model.graph.node if node.name}
+ for index, node in enumerate(model.graph.node):
+ if node.name:
+ continue
+ base = f"ente_opset20_{index}_{node.op_type}"
+ candidate = base
+ suffix = 1
+ while candidate in existing:
+ candidate = f"{base}_{suffix}"
+ suffix += 1
+ node.name = candidate
+ existing.add(candidate)
+
+
+def rewrite_exact_gelu(model: onnx.ModelProto) -> int:
+ """Collapse x/sqrt(2) -> Erf -> +1 -> *x -> *0.5 into exact GELU."""
+ producer_by_output = {
+ output: node for node in model.graph.node for output in node.output
+ }
+ consumers = node_consumers(model)
+ replacements: dict[str, onnx.NodeProto] = {}
+ removed_names: set[str] = set()
+ removable_constants: set[str] = set()
+
+ node_names = [node.name for node in model.graph.node]
+ if not all(node_names) or len(node_names) != len(set(node_names)):
+ raise ValueError("GELU rewrite requires unique, non-empty ONNX node names")
+
+ for div in (node for node in model.graph.node if node.op_type == "Div"):
+ if len(div.input) != 2:
+ continue
+ div_constant = next(
+ (
+ input_name
+ for input_name in div.input
+ if constant_scalar(producer_by_output, input_name) is not None
+ ),
+ None,
+ )
+ if div_constant is None:
+ continue
+ input_name = other_input(div, div_constant)
+ divisor = constant_scalar(producer_by_output, div_constant)
+ if (
+ input_name is None
+ or divisor is None
+ or not math.isclose(divisor, math.sqrt(2.0), rel_tol=0.0, abs_tol=1e-6)
+ ):
+ continue
+
+ erf = only_consumer(consumers, div.output[0], "Erf")
+ if erf is None:
+ continue
+ add = only_consumer(consumers, erf.output[0], "Add")
+ if add is None:
+ continue
+ add_constant = other_input(add, erf.output[0])
+ if (
+ add_constant is None
+ or constant_scalar(producer_by_output, add_constant) != 1.0
+ ):
+ continue
+
+ first_mul = only_consumer(consumers, add.output[0], "Mul")
+ if first_mul is None or set(first_mul.input) != {input_name, add.output[0]}:
+ continue
+ final_mul = only_consumer(consumers, first_mul.output[0], "Mul")
+ if final_mul is None:
+ continue
+ half_constant = other_input(final_mul, first_mul.output[0])
+ if (
+ half_constant is None
+ or constant_scalar(producer_by_output, half_constant) != 0.5
+ ):
+ continue
+
+ replacements[div.name] = helper.make_node(
+ "Gelu",
+ [input_name],
+ list(final_mul.output),
+ name=f"{final_mul.name}/ExactGelu",
+ approximate="none",
+ )
+ removed_names.update(
+ node.name for node in (div, erf, add, first_mul, final_mul)
+ )
+ removable_constants.update(
+ producer_by_output[name].name
+ for name in (div_constant, add_constant, half_constant)
+ if len(consumers.get(name, [])) == 1
+ )
+
+ rewritten_nodes = []
+ for node in model.graph.node:
+ if node.name in replacements:
+ rewritten_nodes.append(replacements[node.name])
+ elif node.name not in removed_names and node.name not in removable_constants:
+ rewritten_nodes.append(node)
+ del model.graph.node[:]
+ model.graph.node.extend(rewritten_nodes)
+ return len(replacements)
+
+
def rewrite_prelu_decompositions(model: onnx.ModelProto) -> int:
- """Replace max(x, 0) + alpha * min(x, 0) decompositions with PRelu."""
+ """Replace exported PReLU expressions with CoreML/WebGPU-compatible ops.
+
+ The source graph expresses PReLU using Relu, Abs, Sub, Mul, Mul, and Add.
+ Abs is not supported by CoreML's documented MLProgram operator set, while
+ native PRelu is not supported by the WebGPU EP used on Android. The exact
+ equivalent below uses only operators documented by both providers:
+
+ PReLU(x, alpha) = Relu(x) - alpha * Relu(x * -1)
+ """
consumers = node_consumers(model)
initializers = {
initializer.name: initializer for initializer in model.graph.initializer
}
- replacements: dict[str, onnx.NodeProto] = {}
+ replacements: dict[str, list[onnx.NodeProto]] = {}
removed_names: set[str] = set()
node_names = [node.name for node in model.graph.node]
@@ -123,9 +260,7 @@ def rewrite_prelu_decompositions(model: onnx.ModelProto) -> int:
if half_mul is None:
continue
half_name = other_input(half_mul, alpha_mul.output[0])
- if half_name is None or not is_scalar_initializer(
- initializers, half_name, 0.5
- ):
+ if half_name is None or not is_scalar_initializer(initializers, half_name, 0.5):
continue
add = only_consumer(consumers, relu.output[0], "Add")
@@ -135,20 +270,52 @@ def rewrite_prelu_decompositions(model: onnx.ModelProto) -> int:
if half_add is None or half_add.name != add.name:
continue
- # The export stores slopes as [1, C, 1, 1]. ONNX PRelu and the mobile
- # accelerators expect a per-channel [C, 1, 1] slope for NCHW tensors.
- if list(alpha.dims[-2:]) != [1, 1]:
+ # The export stores slopes as [1, C, 1, 1], which broadcasts directly
+ # over the NCHW activation tensor used at this point in the graph.
+ if (
+ len(alpha.dims) != 4
+ or alpha.dims[0] != 1
+ or list(alpha.dims[-2:]) != [1, 1]
+ ):
continue
- slope_dims = list(alpha.dims[1:])
- del alpha.dims[:]
- alpha.dims.extend(slope_dims)
-
- replacements[relu.name] = helper.make_node(
- "PRelu",
- [input_name, alpha.name],
- list(add.output),
- name=f"{add.name}/PRelu",
- )
+
+ output_name = add.output[0]
+ positive_name = f"{output_name}/positive"
+ negated_name = f"{output_name}/negated"
+ negative_magnitude_name = f"{output_name}/negative_magnitude"
+ scaled_negative_name = f"{output_name}/scaled_negative"
+ replacements[relu.name] = [
+ helper.make_node(
+ "Relu",
+ [input_name],
+ [positive_name],
+ name=f"{add.name}/PositiveRelu",
+ ),
+ helper.make_node(
+ "Mul",
+ [input_name, PRELU_NEGATIVE_ONE],
+ [negated_name],
+ name=f"{add.name}/Negate",
+ ),
+ helper.make_node(
+ "Relu",
+ [negated_name],
+ [negative_magnitude_name],
+ name=f"{add.name}/NegativeRelu",
+ ),
+ helper.make_node(
+ "Mul",
+ [negative_magnitude_name, alpha.name],
+ [scaled_negative_name],
+ name=f"{add.name}/ScaleNegative",
+ ),
+ helper.make_node(
+ "Sub",
+ [positive_name, scaled_negative_name],
+ [output_name],
+ name=f"{add.name}/PortablePRelu",
+ ),
+ ]
removed_names.update(
node.name for node in (relu, abs_node, sub, alpha_mul, half_mul, add)
)
@@ -156,11 +323,20 @@ def rewrite_prelu_decompositions(model: onnx.ModelProto) -> int:
rewritten_nodes = []
for node in model.graph.node:
if node.name in replacements:
- rewritten_nodes.append(replacements[node.name])
+ rewritten_nodes.extend(replacements[node.name])
elif node.name not in removed_names:
rewritten_nodes.append(node)
del model.graph.node[:]
model.graph.node.extend(rewritten_nodes)
+ if replacements:
+ model.graph.initializer.append(
+ helper.make_tensor(
+ PRELU_NEGATIVE_ONE,
+ onnx.TensorProto.FLOAT,
+ [],
+ [-1.0],
+ )
+ )
return len(replacements)
@@ -272,9 +448,21 @@ def build_models(source_dir: Path, output_dir: Path) -> dict[str, object]:
face_path = output_dir / FACE_OUTPUT
face = optimize_with_ort(face, face_path)
+ clip = load_source_model(source_dir, CLIP_SOURCE, CLIP_SOURCE_SHA256)
+ clip = version_converter.convert_version(clip, 20)
+ ensure_unique_node_names(clip)
+ if rewrite_exact_gelu(clip) != EXPECTED_GELU_REWRITES:
+ raise RuntimeError(
+ f"expected exactly {EXPECTED_GELU_REWRITES} MobileCLIP GELU rewrites"
+ )
+ onnx.checker.check_model(clip, full_check=True)
+ clip_path = output_dir / CLIP_OUTPUT
+ onnx.save(clip, clip_path)
+
return {
"yolo_static_b1": describe(yolo_path, yolo),
- "mobilefacenet_prelu_static_b1": describe(face_path, face),
+ "mobilefacenet_portable_static_b1": describe(face_path, face),
+ "mobileclip_exact_gelu_opset20": describe(clip_path, clip),
}