π End-to-end encrypted cloud for everything.
Dart 2faandroidauthye2eeencryptionend-to-end-encryptionfluttergolanggoogle-photosgoogle-photos-alternativeiosopensourcephotosprivacyprivacy-toolstypescriptzero-knowledge
Use shared fixed OCR models and packed GPU inference
laurenspriem5ec95f8ccdcd
diff --git a/infra/ml/playground/optimizations/README.md b/infra/ml/playground/optimizations/README.mdindex f58323e0ba..63d58792f1 100644--- a/infra/ml/playground/optimizations/README.md+++ b/infra/ml/playground/optimizations/README.md@@ -17,21 +17,23 @@ uv run --no-project --with numpy==2.5.3 --with onnx==1.22.0 python \--output-dir infra/ml/playground/optimizations/models/ocr```-The script keeps FP32 weights and applies these transformations:--- Detection: fold the transposed-convolution head's bias and batch normalization,- and expand `HardSwish` into `HardSigmoid * x` for CoreML compatibility.-- Classification: canonicalize scalar constants and simplify expanded activations- to `HardSigmoid * x`.-- Recognition: fold affine operations into unpadded convolutions, fuse layer- normalization, expand `HardSwish`, and align the final matrix multiplication- for WebGPU without changing the vocabulary. Return FP32 `[N,T,2]` containing- the first winning token index and its probability; Rust retains CTC decoding.--Missing sources are downloaded from `https://models.ente.com/PP-OCRv5`. The script-verifies source and output hashes and writes three ONNX files, the unchanged-dictionary, and `ocr_model_manifest.json`. Publish the ONNX files at new URLs:-the recognizer's output contract differs from the original model.+The script produces three shared FP32 models (23.17 MB total) with fixed inputs:++- Detection: a 960Γ960 canvas with five fixed computation paths. Masks preserve+ the original image boundaries and pooling when selecting a smaller path.+- Classification: six 48Γ192 crops; Rust pads incomplete batches.+- Recognition: a 48Γ7168 canvas with 2048- and 7168-wide paths. Rust packs lines;+ masks isolate their convolutions, pooling, and attention. Aligned matrix+ operations return winning token indices and probabilities for Rust CTC decoding.++Affine folding and equivalent activation rewrites improve GPU support. Shared+weights keep the fixed paths compact. These models require the matching Rust+input adapter; they cannot replace the original models independently.++Missing sources are downloaded from `https://models.ente.com/PP-OCRv5`. Source and+output hashes are verified. Upload only `det_fixed_v1.onnx`, `cls_fixed_v1.onnx`,+and `rec_fixed_v1.onnx`; the dictionary is unchanged. The generated+`ocr_model_manifest.json` records the input shapes, hashes, and sizes.## Rebuilding the modelsdiff --git a/infra/ml/playground/optimizations/models/ocr/ocr_model_manifest.json b/infra/ml/playground/optimizations/models/ocr/ocr_model_manifest.jsonindex 56f26722f7..0610001845 100644--- a/infra/ml/playground/optimizations/models/ocr/ocr_model_manifest.json+++ b/infra/ml/playground/optimizations/models/ocr/ocr_model_manifest.json@@ -1,4 +1,32 @@{+ "format": "ente-ocr-fixed-v1",+ "requires_context_adapter": true,+ "detector_paths": [+ [+ 960,+ 480+ ],+ [+ 480,+ 960+ ],+ [+ 960,+ 704+ ],+ [+ 704,+ 960+ ],+ [+ 960,+ 960+ ]+ ],+ "recognizer_widths": [+ 2048,+ 7168+ ],"source_base_url": "https://models.ente.com/PP-OCRv5","source_sha256": {"cls.onnx": "f4bb53707100c5f3d59ba834eb05bb400369f20aed35d4b26807b1bfadd2a70e",@@ -9,85 +37,467 @@"onnx": "1.22.0","models": [{- "file": "det.onnx",- "sha256": "3e658f85236f1984e186070048d7cdc99ddbb84e12c91a6fd80922b85bd9487a",- "bytes": 4750896,- "operators": [- "Add",- "BatchNormalization",- "Concat",- "Constant",- "Conv",- "ConvTranspose",- "GlobalAveragePool",- "HardSigmoid",- "Mul",- "Relu",- "Reshape",- "Resize",- "Sigmoid"- ],+ "file": "det_fixed_v1.onnx",+ "sha256": "f655f119225b579fa8c3cbf64f6bb7cf56a26c9dc211706a25234d70a543ec8c",+ "bytes": 5548458,+ "inputs": {+ "x": [+ 1,+ 3,+ 960,+ 960+ ],+ "p0_mask_960_480": [+ 1,+ 1,+ 960,+ 480+ ],+ "p0_mask_480_240": [+ 1,+ 1,+ 480,+ 240+ ],+ "p0_mask_240_120": [+ 1,+ 1,+ 240,+ 120+ ],+ "p0_mask_120_60": [+ 1,+ 1,+ 120,+ 60+ ],+ "p0_mask_60_30": [+ 1,+ 1,+ 60,+ 30+ ],+ "p0_mask_30_15": [+ 1,+ 1,+ 30,+ 15+ ],+ "p0_pool_scale_30_15": [+ 1,+ 1,+ 1,+ 1+ ],+ "p0_pool_scale_60_30": [+ 1,+ 1,+ 1,+ 1+ ],+ "p0_pool_scale_120_60": [+ 1,+ 1,+ 1,+ 1+ ],+ "p0_pool_scale_240_120": [+ 1,+ 1,+ 1,+ 1+ ],+ "p1_mask_480_960": [+ 1,+ 1,+ 480,+ 960+ ],+ "p1_mask_240_480": [+ 1,+ 1,+ 240,+ 480+ ],+ "p1_mask_120_240": [+ 1,+ 1,+ 120,+ 240+ ],+ "p1_mask_60_120": [+ 1,+ 1,+ 60,+ 120+ ],+ "p1_mask_30_60": [+ 1,+ 1,+ 30,+ 60+ ],+ "p1_mask_15_30": [+ 1,+ 1,+ 15,+ 30+ ],+ "p1_pool_scale_15_30": [+ 1,+ 1,+ 1,+ 1+ ],+ "p1_pool_scale_30_60": [+ 1,+ 1,+ 1,+ 1+ ],+ "p1_pool_scale_60_120": [+ 1,+ 1,+ 1,+ 1+ ],+ "p1_pool_scale_120_240": [+ 1,+ 1,+ 1,+ 1+ ],+ "p2_mask_960_704": [+ 1,+ 1,+ 960,+ 704+ ],+ "p2_mask_480_352": [+ 1,+ 1,+ 480,+ 352+ ],+ "p2_mask_240_176": [+ 1,+ 1,+ 240,+ 176+ ],+ "p2_mask_120_88": [+ 1,+ 1,+ 120,+ 88+ ],+ "p2_mask_60_44": [+ 1,+ 1,+ 60,+ 44+ ],+ "p2_mask_30_22": [+ 1,+ 1,+ 30,+ 22+ ],+ "p2_pool_scale_30_22": [+ 1,+ 1,+ 1,+ 1+ ],+ "p2_pool_scale_60_44": [+ 1,+ 1,+ 1,+ 1+ ],+ "p2_pool_scale_120_88": [+ 1,+ 1,+ 1,+ 1+ ],+ "p2_pool_scale_240_176": [+ 1,+ 1,+ 1,+ 1+ ],+ "p3_mask_704_960": [+ 1,+ 1,+ 704,+ 960+ ],+ "p3_mask_352_480": [+ 1,+ 1,+ 352,+ 480+ ],+ "p3_mask_176_240": [+ 1,+ 1,+ 176,+ 240+ ],+ "p3_mask_88_120": [+ 1,+ 1,+ 88,+ 120+ ],+ "p3_mask_44_60": [+ 1,+ 1,+ 44,+ 60+ ],+ "p3_mask_22_30": [+ 1,+ 1,+ 22,+ 30+ ],+ "p3_pool_scale_22_30": [+ 1,+ 1,+ 1,+ 1+ ],+ "p3_pool_scale_44_60": [+ 1,+ 1,+ 1,+ 1+ ],+ "p3_pool_scale_88_120": [+ 1,+ 1,+ 1,+ 1+ ],+ "p3_pool_scale_176_240": [+ 1,+ 1,+ 1,+ 1+ ],+ "p4_mask_960_960": [+ 1,+ 1,+ 960,+ 960+ ],+ "p4_mask_480_480": [+ 1,+ 1,+ 480,+ 480+ ],+ "p4_mask_240_240": [+ 1,+ 1,+ 240,+ 240+ ],+ "p4_mask_120_120": [+ 1,+ 1,+ 120,+ 120+ ],+ "p4_mask_60_60": [+ 1,+ 1,+ 60,+ 60+ ],+ "p4_mask_30_30": [+ 1,+ 1,+ 30,+ 30+ ],+ "p4_pool_scale_30_30": [+ 1,+ 1,+ 1,+ 1+ ],+ "p4_pool_scale_60_60": [+ 1,+ 1,+ 1,+ 1+ ],+ "p4_pool_scale_120_120": [+ 1,+ 1,+ 1,+ 1+ ],+ "p4_pool_scale_240_240": [+ 1,+ 1,+ 1,+ 1+ ],+ "path_3": [],+ "path_2": [],+ "path_1": [],+ "path_0": []+ },"output": "unchanged"},{- "file": "cls.onnx",- "sha256": "8675903197a1bab7060e830e9bcc63d29c5cf7d372735338bd9501f829e94ed1",- "bytes": 574875,- "operators": [- "Add",- "BatchNormalization",- "Cast",- "Concat",- "Constant",- "Conv",- "GlobalAveragePool",- "HardSigmoid",- "MatMul",- "MaxPool",- "Mul",- "Relu",- "Reshape",- "Shape",- "Slice",- "Softmax"- ],+ "file": "cls_fixed_v1.onnx",+ "sha256": "378d52a73263828d08d4dda37f1d0aac2e36cbd486fdc6351bf309d515610654",+ "bytes": 590475,+ "inputs": {+ "x": [+ 6,+ 3,+ 48,+ 192+ ]+ },"output": "unchanged"},{- "file": "rec.onnx",- "sha256": "667691bb173a94106b595aa87c7d176b53b68cf831b6ffa09569018754265e57",- "bytes": 16686404,- "operators": [- "Add",- "AveragePool",- "BatchNormalization",- "Ceil",- "Concat",- "Constant",- "Conv",- "Div",- "GlobalAveragePool",- "HardSigmoid",- "LayerNormalization",- "MatMul",- "Mul",- "Pow",- "ReduceMax",- "ReduceMean",- "ReduceMin",- "Relu",- "Reshape",- "Shape",- "Sigmoid",- "Slice",- "Softmax",- "Sqrt",- "Squeeze",- "Sub",- "Transpose",- "Unsqueeze"- ],- "output": "FP32 [N,T,2]: first winning index, original winning probability"+ "file": "rec_fixed_v1.onnx",+ "sha256": "6dda4c0891af5a70c5b0f618b62588df7f3140616d1c560be5ec6496747b54fd",+ "bytes": 17031601,+ "inputs": {+ "x": [+ 1,+ 3,+ 48,+ 7168+ ],+ "p0_mask_48_2048": [+ 1,+ 1,+ 1,+ 2048+ ],+ "p0_mask_24_1024": [+ 1,+ 1,+ 1,+ 1024+ ],+ "p0_mask_12_1024": [+ 1,+ 1,+ 1,+ 1024+ ],+ "p0_mask_12_512": [+ 1,+ 1,+ 1,+ 512+ ],+ "p0_pool_weights_512": [+ 1,+ 512,+ 6+ ],+ "p0_gate_weights_512": [+ 1,+ 6,+ 512+ ],+ "p0_mask_6_512": [+ 1,+ 1,+ 1,+ 512+ ],+ "p0_mask_3_512": [+ 1,+ 1,+ 1,+ 512+ ],+ "p0_mask_1_256": [+ 1,+ 1,+ 1,+ 256+ ],+ "p0_attention_bias": [+ 1,+ 1,+ 256,+ 256+ ],+ "p1_mask_48_7168": [+ 1,+ 1,+ 1,+ 7168+ ],+ "p1_mask_24_3584": [+ 1,+ 1,+ 1,+ 3584+ ],+ "p1_mask_12_3584": [+ 1,+ 1,+ 1,+ 3584+ ],+ "p1_mask_12_1792": [+ 1,+ 1,+ 1,+ 1792+ ],+ "p1_pool_weights_1792": [+ 1,+ 1792,+ 21+ ],+ "p1_gate_weights_1792": [+ 1,+ 21,+ 1792+ ],+ "p1_mask_6_1792": [+ 1,+ 1,+ 1,+ 1792+ ],+ "p1_mask_3_1792": [+ 1,+ 1,+ 1,+ 1792+ ],+ "p1_mask_1_896": [+ 1,+ 1,+ 1,+ 896+ ],+ "p1_attention_bias": [+ 1,+ 1,+ 896,+ 896+ ],+ "path_0": []+ },+ "output": "FP32 [1,896,2]: packed first winning index and probability"}]}diff --git a/infra/ml/playground/optimizations/ocr_fixed_inputs.py b/infra/ml/playground/optimizations/ocr_fixed_inputs.pynew file mode 100644index 0000000000..fa42af5279--- /dev/null+++ b/infra/ml/playground/optimizations/ocr_fixed_inputs.py@@ -0,0 +1,532 @@+import copy+import hashlib++import numpy as np+import onnx+from onnx import helper, numpy_helper++DOMAIN = "ente.ocr.weights"+++def shaped(source_model, shape):+ model = copy.deepcopy(source_model)+ for dim, value in zip(model.graph.input[0].type.tensor_type.shape.dim, shape):+ dim.dim_value = value+ del model.graph.value_info[:]+ for output in model.graph.output:+ output.type.tensor_type.ClearField("shape")+ return onnx.shape_inference.infer_shapes(model, data_prop=True)+++def dimensions(model):+ return {+ value.name: [dim.dim_value for dim in value.type.tensor_type.shape.dim]+ for value in [*model.graph.input, *model.graph.value_info, *model.graph.output]+ }+++def detector_model(source_model, shape):+ model = shaped(source_model, shape)+ sizes = dimensions(model)+ nodes = []+ masks = {}+ mask_cache = {}++ def auxiliary(name, dims, tensor, kind):+ if name not in masks:+ model.graph.input.append(+ helper.make_tensor_value_info(name, onnx.TensorProto.FLOAT, dims)+ )+ masks[name] = {"shape": dims, "tensor": tensor, "kind": kind}+ return name++ def mask(tensor):+ if tensor in mask_cache:+ return mask_cache[tensor]+ n, _, h, w = sizes[tensor]+ dims = [n, 1, h, w]+ name = auxiliary(f"mask_{h}_{w}", dims, tensor, "spatial")+ out = tensor + "_masked"+ nodes.append(helper.make_node("Mul", [tensor, name], [out], name=out))+ mask_cache[tensor] = out+ return out++ for original in model.graph.node:+ node = copy.deepcopy(original)+ if node.op_type in [+ "Conv",+ "AveragePool",+ "GlobalAveragePool",+ ]:+ dims = sizes.get(node.input[0], [])+ attrs = {a.name: helper.get_attribute_value(a) for a in node.attribute}+ spatial = len(dims) == 4 and dims[-1] > 1+ mixing = node.op_type != "Conv" or any(+ k > 1 for k in attrs.get("kernel_shape", [1, 1])+ )+ if spatial and mixing:+ node.input[0] = mask(original.input[0])+ if node.op_type == "GlobalAveragePool":+ n, _, h, w = sizes[original.input[0]]+ scale = auxiliary(+ f"pool_scale_{h}_{w}", [n, 1, 1, 1], original.input[0], "scale"+ )+ output = node.output[0]+ node.output[0] = output + "_uncorrected"+ nodes.append(node)+ nodes.append(+ helper.make_node(+ "Mul", [node.output[0], scale], [output], name=output + "_correct"+ )+ )+ continue+ if node.op_type == "Softmax" and len(sizes[node.input[0]]) == 4:+ n, _, _, w = sizes[node.input[0]]+ name = auxiliary(+ "attention_bias", [n, 1, 1, w], original.input[0], "attention"+ )+ out = node.input[0] + "_masked"+ nodes.append(+ helper.make_node("Add", [node.input[0], name], [out], name=out)+ )+ node.input[0] = out+ nodes.append(node)+ del model.graph.node[:]+ model.graph.node.extend(nodes)+ del model.graph.value_info[:]+ model = onnx.shape_inference.infer_shapes(model, data_prop=True)+ onnx.checker.check_model(model)+ return model, masks+++def recognizer_model(source_model, width, slots):+ model = shaped(source_model, [1, 3, 48, width])+ sizes = dimensions(model)+ nodes = []+ metadata = {}+ seen = {}+ gates = {}++ def aux(name, shape, tensor, kind):+ if name not in metadata:+ model.graph.input.append(+ helper.make_tensor_value_info(name, onnx.TensorProto.FLOAT, shape)+ )+ metadata[name] = {"shape": shape, "tensor": tensor, "kind": kind}+ return name++ def const(name, value):+ model.graph.initializer.append(+ numpy_helper.from_array(np.asarray(value, np.int64), name)+ )+ return name++ def mask(tensor):+ if tensor in seen:+ return seen[tensor]+ _, _, h, w = sizes[tensor]+ name = aux(f"mask_{h}_{w}", [1, 1, 1, w], tensor, "spatial")+ out = tensor + "_masked"+ nodes.append(helper.make_node("Mul", [tensor, name], [out], name=out))+ seen[tensor] = out+ return out++ for original in model.graph.node:+ node = copy.deepcopy(original)+ if node.op_type == "GlobalAveragePool":+ tensor = node.input[0]+ _, c, _, w = sizes[tensor]+ prefix = node.output[0] + "_packed"+ weights = aux(f"pool_weights_{w}", [1, w, slots], tensor, "pool")+ nodes.extend(+ [+ helper.make_node(+ "ReduceMean",+ [tensor],+ [prefix + "_height"],+ axes=[2],+ keepdims=0,+ ),+ helper.make_node(+ "MatMul", [prefix + "_height", weights], [prefix + "_means"]+ ),+ helper.make_node(+ "Transpose",+ [prefix + "_means"],+ [prefix + "_transposed"],+ perm=[2, 1, 0],+ ),+ helper.make_node(+ "Reshape",+ [+ prefix + "_transposed",+ const(prefix + "_shape", [slots, c, 1, 1]),+ ],+ [node.output[0]],+ ),+ ]+ )+ gate = "p2o.pd_op.hardsigmoid." + (+ "0.0" if "pool2d.0" in node.output[0] else "1.0"+ )+ gates[gate] = (c, w, tensor)+ continue+ if node.op_type in ["Conv", "AveragePool"]:+ dims = sizes.get(node.input[0], [])+ attrs = {a.name: helper.get_attribute_value(a) for a in node.attribute}+ if (+ len(dims) == 4+ and dims[-1] > 1+ and (+ node.op_type != "Conv"+ or any(k > 1 for k in attrs.get("kernel_shape", [1, 1]))+ )+ ):+ node.input[0] = mask(node.input[0])+ if node.op_type == "Softmax" and len(sizes[node.input[0]]) == 4:+ t = sizes[node.input[0]][-1]+ name = aux("attention_bias", [1, 1, t, t], node.input[0], "attention")+ out = node.input[0] + "_masked"+ nodes.append(+ helper.make_node("Add", [node.input[0], name], [out], name=out)+ )+ node.input[0] = out+ output = node.output[0]+ if output in gates:+ c, w, tensor = gates[output]+ node.output[0] = output + "_slots"+ weights = aux(f"gate_weights_{w}", [1, slots, w], tensor, "gate")+ nodes.extend(+ [+ node,+ helper.make_node(+ "Reshape",+ [node.output[0], const(output + "_flatshape", [slots, c])],+ [output + "_flat"],+ ),+ helper.make_node(+ "Transpose",+ [output + "_flat"],+ [output + "_transposed"],+ perm=[1, 0],+ ),+ helper.make_node(+ "MatMul",+ [output + "_transposed", weights],+ [output + "_mapped"],+ ),+ helper.make_node(+ "Reshape",+ [output + "_mapped", const(output + "_shape", [1, c, 1, w])],+ [output],+ ),+ ]+ )+ else:+ nodes.append(node)+ del model.graph.node[:]+ model.graph.node.extend(nodes)+ del model.graph.value_info[:]+ for output in model.graph.output:+ output.type.tensor_type.ClearField("shape")+ model = onnx.shape_inference.infer_shapes(model, data_prop=True)+ onnx.checker.check_model(model)+ return model, metadata+++def pad_vocabulary(original):+ model = copy.deepcopy(original)+ values = {v.name: numpy_helper.to_array(v) for v in model.graph.initializer}+ for node in model.graph.node:+ if node.op_type == "Constant":+ for attribute in node.attribute:+ if attribute.name == "value":+ values[node.output[0]] = numpy_helper.to_array(attribute.t)+ removed = next(+ node+ for node in model.graph.node+ if node.op_type == "Slice" and node.input[0] == "modelopt_padded_projection"+ )+ add = next(+ node+ for node in model.graph.node+ if node.op_type == "Add" and removed.output[0] in node.input+ )+ bias_name = next(name for name in add.input if name in values)+ bias = values[bias_name].reshape(-1)+ assert bias.size == 18385+ extended = np.pad(bias, (0, 18432 - bias.size), constant_values=-np.inf)+ model.graph.initializer.append(+ numpy_helper.from_array(extended, "vocabulary_bias_padded")+ )+ add.input[:] = ["modelopt_padded_projection", "vocabulary_bias_padded"]+ model.graph.node.remove(removed)+ for value in model.graph.initializer:+ if value.name == "shared_positions":+ value.CopyFrom(+ numpy_helper.from_array(+ np.arange(18432, dtype=np.float32).reshape(1, 1, -1),+ "shared_positions",+ )+ )+ del model.graph.value_info[:]+ model = onnx.shape_inference.infer_shapes(model, data_prop=True)+ onnx.checker.check_model(model)+ return model+++def align_attention(model):+ model = copy.deepcopy(model)+ sizes = dimensions(model)+ nodes = []++ def constant(name, values):+ model.graph.initializer.append(+ numpy_helper.from_array(np.asarray(values, np.int64), name)+ )+ return name++ def pad(name, tensor, axis):+ pads = [0] * 8+ pads[4 + axis] = 1+ nodes.append(+ helper.make_node(+ "Pad", [tensor, constant(name + "_pads", pads)], [name], name=name+ )+ )+ return name++ for original in model.graph.node:+ node = copy.deepcopy(original)+ if node.op_type == "MatMul":+ a = sizes.get(node.input[0], [])+ b = sizes.get(node.input[1], [])+ if len(a) == 4 and a[-1] == 15 and len(b) == 4 and b[-2] == 15:+ node.input[0] = pad(node.output[0] + "_q16", node.input[0], 3)+ node.input[1] = pad(node.output[0] + "_k16", node.input[1], 2)+ elif len(a) == 4 and len(b) == 4 and a[-1] == b[-2] and b[-1] == 15:+ node.input[1] = pad(node.output[0] + "_v16", node.input[1], 3)+ output = node.output[0]+ node.output[0] = output + "_16"+ nodes.append(node)+ nodes.append(+ helper.make_node(+ "Slice",+ [+ node.output[0],+ constant(output + "_start", [0]),+ constant(output + "_end", [15]),+ constant(output + "_axis", [3]),+ ],+ [output],+ name=output + "_trim",+ )+ )+ continue+ nodes.append(node)+ del model.graph.node[:]+ model.graph.node.extend(nodes)+ del model.graph.value_info[:]+ model = onnx.shape_inference.infer_shapes(model, data_prop=True)+ onnx.checker.check_model(model, full_check=True)+ return model+++def with_fixed_paths(source_model, stage, shapes):+ maximum = [1, 3, 960, 960] if stage == "det" else [1, 3, 48, 7168]+ output_shape = [1, 1, 960, 960] if stage == "det" else [1, 896, 2]+ outer_inputs = [helper.make_tensor_value_info("x", onnx.TensorProto.FLOAT, maximum)]+ weights = {}+ branches = []+ metadata = []+ for index, shape in enumerate(shapes):+ label = f"p{index}"+ if stage == "det":+ model, meta = detector_model(source_model, shape)+ else:+ model, meta = recognizer_model(source_model, shape[-1], shape[-1] // 336)+ model = align_attention(pad_vocabulary(model))+ mapping = {}+ local_weights = set()+ tensors = [+ (n.output[0], a.t)+ for n in model.graph.node+ if n.op_type == "Constant"+ for a in n.attribute+ if a.name == "value" and a.t.data_type == onnx.TensorProto.FLOAT+ ]+ tensors.extend(+ (t.name, t)+ for t in model.graph.initializer+ if t.data_type == onnx.TensorProto.FLOAT+ )+ for original_name, original in tensors:+ tensor = copy.deepcopy(original)+ tensor.name = ""+ name = "w" + hashlib.sha256(tensor.SerializeToString()).hexdigest()+ tensor.name = name+ weights[name] = tensor+ mapping[original_name] = name+ local_weights.add(name)++ def rename(name, mapping=mapping, label=label):+ return mapping.get(name, label + "_" + name) if name else name++ nodes = [+ helper.make_node(name, [], [name], domain=DOMAIN, name=label + "_" + name)+ for name in sorted(local_weights)+ ]+ initializers = []+ for name, values in [+ ("starts", [0, 0]),+ ("ends", shape[-2:]),+ ("axes", [2, 3]),+ ]:+ initializers.append(+ numpy_helper.from_array(+ np.asarray(values, np.int64), label + "_" + name+ )+ )+ nodes.append(+ helper.make_node(+ "Slice",+ ["x", label + "_starts", label + "_ends", label + "_axes"],+ [rename("x")],+ name=label + "_slice",+ )+ )+ for value in list(model.graph.input)[1:]:+ item = copy.deepcopy(value)+ item.name = rename(value.name)+ outer_inputs.append(item)+ for tensor in model.graph.initializer:+ if tensor.name not in mapping:+ item = copy.deepcopy(tensor)+ item.name = rename(item.name)+ initializers.append(item)+ for ni, node in enumerate(model.graph.node):+ if node.output[0] in mapping:+ continue+ item = copy.deepcopy(node)+ item.name = label + "_" + str(ni) + "_" + node.name+ for i, name in enumerate(item.input):+ item.input[i] = rename(name)+ for i, name in enumerate(item.output):+ item.output[i] = rename(name)+ nodes.append(item)+ output = rename(model.graph.output[0].name)+ original_output = (+ [1, 1, *shape[-2:]] if stage == "det" else [1, shape[-1] // 8, 2]+ )+ if original_output != output_shape:+ padding = [0] * len(output_shape) + [+ a - b for a, b in zip(output_shape, original_output)+ ]+ initializers.append(+ numpy_helper.from_array(np.asarray(padding, np.int64), label + "_pads")+ )+ nodes.append(+ helper.make_node(+ "Pad",+ [output, label + "_pads"],+ [label + "_output"],+ name=label + "_pad",+ )+ )+ output = label + "_output"+ graph = helper.make_graph(+ nodes,+ label,+ [],+ [+ helper.make_tensor_value_info(+ output, onnx.TensorProto.FLOAT, output_shape+ )+ ],+ initializer=initializers,+ )+ for value in [*model.graph.input, *model.graph.value_info]:+ item = copy.deepcopy(value)+ item.name = rename(item.name)+ graph.value_info.append(item)+ branches.append(graph)+ metadata.append({"shape": shape, "auxiliary": meta})+ branch = branches[-1]+ for i in range(len(branches) - 2, -1, -1):+ name = f"path_{i}"+ outer_inputs.append(+ helper.make_tensor_value_info(name, onnx.TensorProto.BOOL, [])+ )+ node = helper.make_node(+ "If",+ [name],+ [f"selected_{i}"],+ name=f"select_{i}",+ then_branch=branches[i],+ else_branch=branch,+ )+ branch = helper.make_graph(+ [node],+ f"selection_{i}",+ [],+ [+ helper.make_tensor_value_info(+ f"selected_{i}", onnx.TensorProto.FLOAT, output_shape+ )+ ],+ )+ graph = helper.make_graph(+ branch.node, f"{stage}_static_paths", outer_inputs, branch.output+ )+ model = helper.make_model(+ graph,+ opset_imports=[helper.make_opsetid("", 17), helper.make_opsetid(DOMAIN, 1)],+ ir_version=10,+ )+ for name, tensor in weights.items():+ model.functions.append(+ helper.make_function(+ DOMAIN,+ name,+ [],+ ["value"],+ [helper.make_node("Constant", [], ["value"], value=tensor)],+ [helper.make_opsetid("", 17)],+ )+ )+ onnx.checker.check_model(model, full_check=True)+ return model, metadata+++def fixed_models(models):+ detector, _ = with_fixed_paths(+ models["det.onnx"],+ "det",+ [+ [1, 3, 960, 480],+ [1, 3, 480, 960],+ [1, 3, 960, 704],+ [1, 3, 704, 960],+ [1, 3, 960, 960],+ ],+ )+ recognizer, _ = with_fixed_paths(+ models["rec.onnx"], "rec", [[1, 3, 48, 2048], [1, 3, 48, 7168]]+ )+ classifier = copy.deepcopy(models["cls.onnx"])+ classifier.graph.input[0].type.tensor_type.shape.CopyFrom(+ helper.make_tensor_value_info(+ "x", onnx.TensorProto.FLOAT, [6, 3, 48, 192]+ ).type.tensor_type.shape+ )+ del classifier.graph.value_info[:]+ for value in classifier.graph.output:+ for dim in value.type.tensor_type.shape.dim:+ dim.ClearField("dim_param")+ dim.ClearField("dim_value")+ classifier = onnx.shape_inference.infer_shapes(classifier)+ return {+ "det_fixed_v1.onnx": detector,+ "cls_fixed_v1.onnx": classifier,+ "rec_fixed_v1.onnx": recognizer,+ }diff --git a/infra/ml/playground/optimizations/optimize_ocr_models.py b/infra/ml/playground/optimizations/optimize_ocr_models.pyindex 72284d76cd..e8c7128e39 100644--- a/infra/ml/playground/optimizations/optimize_ocr_models.py+++ b/infra/ml/playground/optimizations/optimize_ocr_models.py@@ -3,10 +3,11 @@ import copyimport hashlibimport jsonfrom pathlib import Path-from urllib.request import urlopen+from urllib.request import Request, urlopenimport numpy as npimport onnx+from ocr_fixed_inputs import fixed_modelsfrom onnx import helper, numpy_helperSOURCE_BASE_URL = "https://models.ente.com/PP-OCRv5"@@ -17,9 +18,9 @@ SOURCE_HASHES = {"rec.onnx": "bf66820f48fa99f779974c4df78e5274a9d8e0458c4137e8c5357e40e2c3faf2",}OUTPUT_HASHES = {- "det.onnx": "3e658f85236f1984e186070048d7cdc99ddbb84e12c91a6fd80922b85bd9487a",- "cls.onnx": "8675903197a1bab7060e830e9bcc63d29c5cf7d372735338bd9501f829e94ed1",- "rec.onnx": "667691bb173a94106b595aa87c7d176b53b68cf831b6ffa09569018754265e57",+ "det_fixed_v1.onnx": "f655f119225b579fa8c3cbf64f6bb7cf56a26c9dc211706a25234d70a543ec8c",+ "cls_fixed_v1.onnx": "378d52a73263828d08d4dda37f1d0aac2e36cbd486fdc6351bf309d515610654",+ "rec_fixed_v1.onnx": "6dda4c0891af5a70c5b0f618b62588df7f3140616d1c560be5ec6496747b54fd",}@@ -462,7 +463,11 @@ def load_sources(directory):for name, expected in SOURCE_HASHES.items():path = directory / nameif not path.exists():- with urlopen(f"{SOURCE_BASE_URL}/{name}", timeout=120) as response:+ request = Request(+ f"{SOURCE_BASE_URL}/{name}",+ headers={"User-Agent": "ente-ml-model-optimizer"},+ )+ with urlopen(request, timeout=120) as response:data = response.read()if hashlib.sha256(data).hexdigest() != expected:raise ValueError(f"Unexpected SHA-256 for downloaded {name}")@@ -505,10 +510,18 @@ def main():args = parser.parse_args()if onnx.__version__ != "1.22.0":raise ValueError("Reproducible model output requires onnx==1.22.0")+ if np.__version__ != "2.5.3":+ raise ValueError("Reproducible model output requires numpy==2.5.3")sources = load_sources(args.source_dir)records = []- for name, model in optimize(sources).items():- onnx.checker.check_model(model)+ for name, model in fixed_models(optimize(sources)).items():+ onnx.checker.check_model(model, full_check=True)+ for value in model.graph.input:+ if any(+ not d.HasField("dim_value") or d.dim_value <= 0+ for d in value.type.tensor_type.shape.dim+ ):+ raise ValueError(f"Non-static input {value.name} in {name}")data = model.SerializeToString()digest = hashlib.sha256(data).hexdigest()if digest != OUTPUT_HASHES[name]:@@ -523,15 +536,24 @@ def main():"file": name,"sha256": digest,"bytes": len(data),- "operators": sorted({node.op_type for node in model.graph.node}),- "output": "FP32 [N,T,2]: first winning index, original winning probability"- if name == "rec.onnx"+ "inputs": {+ value.name: [d.dim_value for d in value.type.tensor_type.shape.dim]+ for value in model.graph.input+ },+ "output": "FP32 [1,896,2]: packed first winning index and probability"+ if name == "rec_fixed_v1.onnx"else "unchanged",})+ if sum(record["bytes"] for record in records) > 25_000_000:+ raise ValueError("The combined OCR models exceed 25 MB")dictionary = args.source_dir / "ppocrv5_dict.txt"(args.output_dir / dictionary.name).write_bytes(dictionary.read_bytes())metadata = {+ "format": "ente-ocr-fixed-v1",+ "requires_context_adapter": True,+ "detector_paths": [[960, 480], [480, 960], [960, 704], [704, 960], [960, 960]],+ "recognizer_widths": [2048, 7168],"source_base_url": SOURCE_BASE_URL,"source_sha256": SOURCE_HASHES,"onnx": onnx.__version__,diff --git a/mobile/apps/photos/lib/services/machine_learning/ocr/rust_ocr_backend.dart b/mobile/apps/photos/lib/services/machine_learning/ocr/rust_ocr_backend.dartindex dfcde2461b..32cf77a8a6 100644--- a/mobile/apps/photos/lib/services/machine_learning/ocr/rust_ocr_backend.dart+++ b/mobile/apps/photos/lib/services/machine_learning/ocr/rust_ocr_backend.dart@@ -12,7 +12,7 @@ import "package:synchronized/synchronized.dart";class RustOcrBackend implements OcrBackend {static final _logger = Logger("RustOcrBackend");- static const _modelVersion = "pp-ocrv5-gpu-v1";+ static const _modelVersion = "pp-ocrv5-fixed-v1";final _engineLock = Lock();OcrEngine? _engine;diff --git a/rust/Cargo.lock b/rust/Cargo.lockindex 275789937b..3161eda8aa 100644--- a/rust/Cargo.lock+++ b/rust/Cargo.lock@@ -2702,6 +2702,7 @@ dependencies = ["tokio","url","usearch",+ "wide",][[package]]diff --git a/rust/crates/ml/Cargo.toml b/rust/crates/ml/Cargo.tomlindex f4310e2093..d220eabf33 100644--- a/rust/crates/ml/Cargo.toml+++ b/rust/crates/ml/Cargo.toml@@ -37,6 +37,7 @@ ort = { workspace = true, features = [] }[target.'cfg(target_os = "android")'.dependencies]+wide = "0.7.33"ash = { version = "0.38.0", default-features = false, features = ["loaded"] }ort = { workspace = true, features = ["ndarray",@@ -78,6 +79,7 @@ ort = { workspace = true, features = [] }[dev-dependencies]+wide = "0.7.33"anyhow = "1.0.0"flate2.workspace = trueserde.workspace = truediff --git a/rust/crates/ml/src/cv/mod.rs b/rust/crates/ml/src/cv/mod.rsindex de0adc365e..d6a93e80cb 100644--- a/rust/crates/ml/src/cv/mod.rs+++ b/rust/crates/ml/src/cv/mod.rs@@ -13,6 +13,8 @@ mod stats;mod structuring;mod transform;mod warp;+#[cfg(any(target_os = "android", test))]+mod warp_rgb;use rayon::prelude::*;diff --git a/rust/crates/ml/src/cv/warp.rs b/rust/crates/ml/src/cv/warp.rsindex 73cfd778c2..d6f670b839 100644--- a/rust/crates/ml/src/cv/warp.rs+++ b/rust/crates/ml/src/cv/warp.rs@@ -1,5 +1,9 @@-use image::{Rgb, RgbImage};-use imageproc::geometric_transformations::{Interpolation, Projection, warp_into};+#[cfg(not(target_os = "android"))]+use image::Rgb;+use image::RgbImage;+use imageproc::geometric_transformations::Projection;+#[cfg(not(target_os = "android"))]+use imageproc::geometric_transformations::{Interpolation, warp_into};use crate::cv::OpResult;use crate::cv::image::ImageU8;@@ -42,13 +46,19 @@ pub(crate) fn warp_rgb_perspective(let narrow = |c: [(f64, f64); 4]| c.map(|(x, y)| (x as f32, y as f32));let projection = Projection::from_control_points(narrow(src_corners), narrow(dst_corners)).ok_or_else(|| "warp_perspective: the corner pairs are degenerate".to_string())?;- let mut out = RgbImage::new(width as u32, height as u32);- warp_into(- source,- &projection,- Interpolation::Bilinear,- Rgb([0, 0, 0]),- &mut out,- );+ #[cfg(target_os = "android")]+ let out = super::warp_rgb::warp(source, projection, width as u32, height as u32);+ #[cfg(not(target_os = "android"))]+ let out = {+ let mut out = RgbImage::new(width as u32, height as u32);+ warp_into(+ source,+ &projection,+ Interpolation::Bilinear,+ Rgb([0, 0, 0]),+ &mut out,+ );+ out+ };ImageU8::new(width, height, 3, out.into_raw())}diff --git a/rust/crates/ml/src/cv/warp_rgb.rs b/rust/crates/ml/src/cv/warp_rgb.rsnew file mode 100644index 0000000000..bcadf5d635--- /dev/null+++ b/rust/crates/ml/src/cv/warp_rgb.rs@@ -0,0 +1,98 @@+use image::RgbImage;+use imageproc::geometric_transformations::Projection;+use wide::f32x4;++pub(super) fn warp(source: &RgbImage, projection: Projection, width: u32, height: u32) -> RgbImage {+ let inverse = projection.invert();+ let mut output = RgbImage::new(width, height);+ let pixels = source.as_raw();+ let stride = source.width() as usize * 3;+ let load = |offset: usize| {+ f32x4::from([+ f32::from(pixels[offset]),+ f32::from(pixels[offset + 1]),+ f32::from(pixels[offset + 2]),+ 0.0,+ ])+ };+ for (y, row) in output+ .as_mut()+ .chunks_exact_mut(width as usize * 3)+ .enumerate()+ {+ for (x, pixel) in row.as_chunks_mut::<3>().0.iter_mut().enumerate() {+ let (px, py) = inverse * (x as f32, y as f32);+ let left = px.floor();+ let top = py.floor();+ if !(left >= 0.0+ && left + 1.0 < source.width() as f32+ && top >= 0.0+ && top + 1.0 < source.height() as f32)+ {+ continue;+ }+ let right_weight = px - left;+ let bottom_weight = py - top;+ let offset = top as usize * stride + left as usize * 3;+ let right = f32x4::splat(right_weight);+ let left = f32x4::splat(1.0 - right_weight);+ let upper =+ f32x4::from_i32x4((left * load(offset) + right * load(offset + 3)).trunc_int());+ let lower = f32x4::from_i32x4(+ (left * load(offset + stride) + right * load(offset + stride + 3)).trunc_int(),+ );+ let blended = (f32x4::splat(1.0 - bottom_weight) * upper+ + f32x4::splat(bottom_weight) * lower)+ .trunc_int()+ .to_array();+ for channel in 0..3 {+ pixel[channel] = blended[channel] as u8;+ }+ }+ }+ output+}++#[cfg(test)]+mod tests {+ use image::Rgb;+ use imageproc::geometric_transformations::{Interpolation, warp_into};++ use super::*;++ #[test]+ fn matches_reference_pixels_for_projective_and_border_cases() {+ let source = RgbImage::from_fn(83, 71, |x, y| {+ Rgb([+ (x * 17 + y * 3) as u8,+ (x * 11 + y * 31) as u8,+ (x * 7 + y * 19) as u8,+ ])+ });+ for index in 0..200 {+ let shift = index as f32 * 0.173 - 13.0;+ let projection = Projection::from_matrix([+ 0.78,+ 0.13,+ shift,+ -0.07,+ 1.14,+ shift / 2.0,+ index as f32 * 0.00001,+ -0.0004,+ 1.0,+ ])+ .expect("valid projection");+ let actual = warp(&source, projection, 113, 97);+ let mut expected = RgbImage::new(113, 97);+ warp_into(+ &source,+ &projection,+ Interpolation::Bilinear,+ Rgb([0, 0, 0]),+ &mut expected,+ );+ assert_eq!(actual, expected, "case {index}");+ }+ }+}diff --git a/rust/crates/ml/src/ocr/assets.rs b/rust/crates/ml/src/ocr/assets.rsindex 5698d696a7..d03b42acc3 100644--- a/rust/crates/ml/src/ocr/assets.rs+++ b/rust/crates/ml/src/ocr/assets.rs@@ -17,27 +17,27 @@ struct OcrModelFile {}const DETECTION: OcrModelFile = OcrModelFile {- key: "ppocrv5_det_gpu_v1",+ key: "ppocrv5_det_fixed_v1",name: "det.onnx",- url: "https://entedevassets.priem.dev/det_opt.onnx",- size: 4_750_896,- sha256: "3e658f85236f1984e186070048d7cdc99ddbb84e12c91a6fd80922b85bd9487a",+ url: "https://entedevassets.priem.dev/det_fixed_v1.onnx",+ size: 5_548_458,+ sha256: "f655f119225b579fa8c3cbf64f6bb7cf56a26c9dc211706a25234d70a543ec8c",};const CLASSIFICATION: OcrModelFile = OcrModelFile {- key: "ppocrv5_cls_gpu_v1",+ key: "ppocrv5_cls_fixed_v1",name: "cls.onnx",- url: "https://entedevassets.priem.dev/cls_opt.onnx",- size: 574_875,- sha256: "8675903197a1bab7060e830e9bcc63d29c5cf7d372735338bd9501f829e94ed1",+ url: "https://entedevassets.priem.dev/cls_fixed_v1.onnx",+ size: 590_475,+ sha256: "378d52a73263828d08d4dda37f1d0aac2e36cbd486fdc6351bf309d515610654",};const RECOGNITION: OcrModelFile = OcrModelFile {- key: "ppocrv5_rec_gpu_v1",+ key: "ppocrv5_rec_fixed_v1",name: "rec.onnx",- url: "https://entedevassets.priem.dev/rec_opt.onnx",- size: 16_686_404,- sha256: "667691bb173a94106b595aa87c7d176b53b68cf831b6ffa09569018754265e57",+ url: "https://entedevassets.priem.dev/rec_fixed_v1.onnx",+ size: 17_031_601,+ sha256: "6dda4c0891af5a70c5b0f618b62588df7f3140616d1c560be5ec6496747b54fd",};const DICTIONARY: OcrModelFile = OcrModelFile {diff --git a/rust/crates/ml/src/ocr/classify.rs b/rust/crates/ml/src/ocr/classify.rsindex b68a3db4b4..c67372b6a0 100644--- a/rust/crates/ml/src/ocr/classify.rs+++ b/rust/crates/ml/src/ocr/classify.rs@@ -47,14 +47,19 @@ impl AngleClassifier {}fn score_batch(&self, batch: &[ImageU8]) -> MlResult<Vec<AngleScores>> {- let input = PreparedF32Input::new(prepare_crops(|| batch_tensor(batch))?);- let count = batch.len() as i64;+ let mut values = prepare_crops(|| batch_tensor(batch))?;+ values.resize(+ BATCH_SIZE * 3 * INPUT_HEIGHT as usize * INPUT_WIDTH as usize,+ 0.0,+ );+ let input = PreparedF32Input::new(values);+ let count = BATCH_SIZE as i64;let expected_shape = [count, CLASS_COUNT as i64];let mut session = self.session.lock().unwrap_or_else(PoisonError::into_inner);let input_shape = [count, 3, i64::from(INPUT_HEIGHT), i64::from(INPUT_WIDTH)];- let (values, _usage) = session.run(input_shape, |session| {+ let (values, _usage) = session.run(|session| {let (shape, values) = run_f32(session, &input, input_shape)?;- if shape != expected_shape {+ if shape != expected_shape || values.len() != BATCH_SIZE * CLASS_COUNT {return Err(SessionRunError::from(MlError::CorruptModel(format!("angle classifier produced output shape {shape:?}, expected {expected_shape:?}"))));@@ -65,6 +70,7 @@ impl AngleClassifier {.as_chunks::<CLASS_COUNT>().0.iter()+ .take(batch.len()).map(|&[p0, p180]| AngleScores { p0, p180 }).collect())}diff --git a/rust/crates/ml/src/ocr/context.rs b/rust/crates/ml/src/ocr/context.rsnew file mode 100644index 0000000000..957fbc95c5--- /dev/null+++ b/rust/crates/ml/src/ocr/context.rs@@ -0,0 +1,362 @@+use ort::session::{Session, SessionInputValue};+use ort::value::{Tensor, TensorRef};++use crate::onnx::{SessionRunError, SessionRunResult};++#[derive(Clone)]+pub(super) struct SharedValues(std::sync::Arc<Vec<f32>>);++impl std::ops::Deref for SharedValues {+ type Target = Vec<f32>;+ fn deref(&self) -> &Self::Target {+ &self.0+ }+}++impl std::ops::DerefMut for SharedValues {+ fn deref_mut(&mut self) -> &mut Self::Target {+ std::sync::Arc::make_mut(&mut self.0)+ }+}++#[derive(Clone)]+pub(super) struct Input {+ pub name: String,+ pub shape: Vec<i64>,+ pub values: SharedValues,+}++impl Input {+ pub fn zeros(name: impl Into<String>, shape: &[usize]) -> Self {+ Self {+ name: name.into(),+ shape: shape.iter().map(|&v| v as i64).collect(),+ values: SharedValues(std::sync::Arc::new(vec![0.0; shape.iter().product()])),+ }+ }+}++pub(super) fn infer(+ session: &mut Session,+ inputs: &[Input],+) -> SessionRunResult<(Vec<i64>, Vec<f32>)> {+ let tensors: Result<Vec<_>, _> = inputs+ .iter()+ .map(|input| {+ if input.name.starts_with("path_") {+ return Tensor::from_array((input.shape.clone(), vec![input.values[0] != 0.0]))+ .map(|tensor| (input.name.as_str(), SessionInputValue::from(tensor)));+ }+ TensorRef::from_array_view((input.shape.as_slice(), input.values.as_slice()))+ .map(|tensor| (input.name.as_str(), SessionInputValue::from(tensor)))+ })+ .collect();+ let tensors = tensors?;+ let result = session.run(tensors);+ let outputs = result.map_err(SessionRunError::from_inference_error)?;+ let (shape, values) = outputs[0].try_extract_tensor::<f32>()?;+ Ok((shape.to_vec(), values.to_vec()))+}++pub(super) fn detector(raw: &[f32], height: usize, width: usize) -> Vec<Input> {+ let side = 960;+ let mut image = Input::zeros("x", &[1, 3, side, side]);+ for channel in 0..3 {+ for y in 0..height {+ let source = (channel * height + y) * width;+ let target = (channel * side + y) * side;+ image.values[target..target + width].copy_from_slice(&raw[source..source + width]);+ }+ }+ detector_paths(image, height, width)+}++pub(super) struct Line {+ pub values: Vec<f32>,+ pub width: usize,+ pub content_width: i32,+}++pub(super) fn packed(lines: &[&Line], width: usize, slots: usize) -> (Vec<Input>, Vec<usize>) {+ let mut image = Input::zeros("x", &[1, 3, 48, width]);+ let mut positions = Vec::with_capacity(lines.len());+ let mut cursor = 0;+ for line in lines {+ for row in 0..3 * 48 {+ let source = row * line.width;+ let target = row * width + cursor;+ image.values[target..target + line.width]+ .copy_from_slice(&line.values[source..source + line.width]);+ }+ positions.push(cursor);+ cursor += line.width.div_ceil(8) * 8 + 16;+ }+ let mut inputs = vec![image];+ for (height, scale) in [(48, 1), (24, 2), (12, 2), (12, 4), (6, 4), (3, 4), (1, 8)] {+ let size = width / scale;+ let mut input = Input::zeros(format!("mask_{height}_{size}"), &[1, 1, 1, size]);+ for (line, &offset) in lines.iter().zip(&positions) {+ let count = if scale == 8 {+ line.width.div_ceil(4) / 2+ } else {+ line.width.div_ceil(scale)+ };+ input.values[offset / scale..offset / scale + count].fill(1.0);+ }+ inputs.push(input);+ }+ let size = width / 4;+ let mut pool = Input::zeros(format!("pool_weights_{size}"), &[1, size, slots]);+ let mut gate = Input::zeros(format!("gate_weights_{size}"), &[1, slots, size]);+ for (slot, (line, &offset)) in lines.iter().zip(&positions).enumerate() {+ let count = line.width.div_ceil(4);+ for x in offset / 4..offset / 4 + count {+ pool.values[x * slots + slot] = 1.0 / count as f32;+ gate.values[slot * size + x] = 1.0;+ }+ }+ inputs.extend([pool, gate]);+ let steps = width / 8;+ let mut attention = Input::zeros("attention_bias", &[1, 1, steps, steps]);+ attention.values.fill(-10000.0);+ for query in 0..steps {+ attention.values[query * steps] = 0.0;+ }+ for (line, &offset) in lines.iter().zip(&positions) {+ let count = line.width.div_ceil(4) / 2;+ let start = offset / 8;+ for query in start..start + count {+ let row = &mut attention.values[query * steps..(query + 1) * steps];+ row.fill(-10000.0);+ row[start..start + count].fill(0.0);+ }+ }+ inputs.push(attention);+ (inputs, positions)+}++fn detector_auxiliary(index: usize, height: usize, width: usize, h: usize, w: usize) -> Vec<Input> {+ let mut inputs = Vec::new();+ for scale in [1, 2, 4, 8, 16, 32] {+ let mh = h / scale;+ let mw = w / scale;+ let mut mask = Input::zeros(format!("p{index}_mask_{mh}_{mw}"), &[1, 1, mh, mw]);+ for y in 0..height / scale {+ mask.values[y * mw..y * mw + width / scale].fill(1.0);+ }+ inputs.push(mask);+ if scale >= 4 {+ let mut value = Input::zeros(format!("p{index}_pool_scale_{mh}_{mw}"), &[1, 1, 1, 1]);+ value.values[0] = if height > 0 && width > 0 {+ (h * w) as f32 / (height * width) as f32+ } else {+ 1.0+ };+ inputs.push(value);+ }+ }+ inputs+}++fn detector_paths(image: Input, height: usize, width: usize) -> Vec<Input> {+ static UNUSED: std::sync::OnceLock<Vec<Vec<Input>>> = std::sync::OnceLock::new();+ let shapes = &[(960, 480), (480, 960), (960, 704), (704, 960), (960, 960)];+ let unused = UNUSED.get_or_init(|| {+ shapes+ .iter()+ .enumerate()+ .map(|(i, &(h, w))| detector_auxiliary(i, 0, 0, h, w))+ .collect()+ });+ let selected = shapes+ .iter()+ .position(|&(h, w)| height <= h && width <= w)+ .unwrap_or(shapes.len() - 1);+ let mut inputs = vec![image];+ for (index, &(h, w)) in shapes.iter().enumerate() {+ inputs.extend(if index == selected {+ detector_auxiliary(index, height, width, h, w)+ } else {+ unused[index].clone()+ });+ if index < shapes.len() - 1 {+ let mut flag = Input::zeros(format!("path_{index}"), &[]);+ flag.values[0] = if selected == index { 1.0 } else { 0.0 };+ inputs.push(flag);+ }+ }+ inputs+}++pub(super) fn branch_widths() -> &'static [usize] {+ &[2048, 7168]+}++pub(super) fn recognizer_paths(mut selected: Vec<Input>, width: usize) -> Vec<Input> {+ let mut original = selected.remove(0);+ let mut image = Input::zeros("x", &[1, 3, 48, 7168]);+ if width == 7168 {+ std::mem::swap(&mut image.values, &mut original.values);+ } else {+ for row in 0..3 * 48 {+ image.values[row * 7168..row * 7168 + width]+ .copy_from_slice(&original.values[row * width..(row + 1) * width]);+ }+ }+ static UNUSED: std::sync::OnceLock<Vec<Vec<Input>>> = std::sync::OnceLock::new();+ let widths = branch_widths();+ let unused = UNUSED.get_or_init(|| {+ widths+ .iter()+ .map(|&w| {+ let (mut values, _) = packed(&[], w, w / 336);+ values.remove(0);+ values+ })+ .collect()+ });+ let mut inputs = vec![image];+ for (index, &branch_width) in widths.iter().enumerate() {+ let mut values = if branch_width == width {+ std::mem::take(&mut selected)+ } else {+ unused[index].clone()+ };+ for input in &mut values {+ input.name = format!("p{index}_{}", input.name);+ }+ inputs.extend(values);+ }+ for (index, &w) in widths.iter().enumerate().take(widths.len() - 1) {+ let mut flag = Input::zeros(format!("path_{index}"), &[]);+ flag.values[0] = if width == w { 1.0 } else { 0.0 };+ inputs.push(flag);+ }+ inputs+}++#[cfg(test)]+mod tests {+ use super::*;++ fn input<'a>(inputs: &'a [Input], name: &str) -> &'a Input {+ inputs.iter().find(|input| input.name == name).unwrap()+ }++ #[test]+ fn detector_masks_preserve_area_at_every_scale_and_choose_the_smallest_path() {+ for (height, width, path) in [+ (32, 32, 0),+ (960, 480, 0),+ (448, 960, 1),+ (960, 672, 2),+ (704, 960, 3),+ (960, 736, 4),+ ] {+ let raw = vec![0.5; 3 * height * width];+ let inputs = detector(&raw, height, width);+ let image = input(&inputs, "x");+ assert_eq!(image.shape, [1, 3, 960, 960]);+ for channel in 0..3 {+ for row in 0..960 {+ let pixels =+ &image.values[(channel * 960 + row) * 960..(channel * 960 + row + 1) * 960];+ if row < height {+ assert!(pixels[..width].iter().all(|&v| v == 0.5));+ assert!(pixels[width..].iter().all(|&v| v == 0.0));+ } else {+ assert!(pixels.iter().all(|&v| v == 0.0));+ }+ }+ }+ for index in 0..4 {+ assert_eq!(+ input(&inputs, &format!("path_{index}")).values[0],+ f32::from(index == path)+ );+ }+ let prefix = format!("p{path}_mask_");+ for mask in inputs+ .iter()+ .filter(|input| input.name.starts_with(&prefix))+ {+ let scale = match path {+ 1 => 480,+ 3 => 704,+ _ => 960,+ } / mask.shape[2] as usize;+ assert_eq!(+ mask.values.iter().filter(|&&v| v == 1.0).count(),+ height * width / (scale * scale)+ );+ let scale_name = mask.name.replace("mask", "pool_scale");+ if let Some(pool_scale) = inputs.iter().find(|input| input.name == scale_name) {+ let active = height * width / (scale * scale);+ assert!(+ (pool_scale.values[0] * active as f32 / mask.values.len() as f32 - 1.0)+ .abs()+ < 1e-6+ );+ }+ }+ }+ }++ #[test]+ fn packing_isolates_odd_width_lines_in_pooling_and_attention() {+ let lines: Vec<_> = [321, 327]+ .into_iter()+ .map(|width| Line {+ values: vec![0.25; 3 * 48 * width],+ width,+ content_width: width as i32,+ })+ .collect();+ let (inputs, offsets) = packed(&[&lines[0], &lines[1]], 2048, 6);+ assert_eq!(offsets, [0, 344]);+ let mask = input(&inputs, "mask_1_256");+ assert_eq!(mask.values.iter().filter(|&&v| v == 1.0).count(), 81);+ let attention = input(&inputs, "attention_bias");+ for (line, &offset) in lines.iter().zip(&offsets) {+ let start = offset / 8;+ let count = line.width.div_ceil(4) / 2;+ for query in start..start + count {+ let row = &attention.values[query * 256..(query + 1) * 256];+ assert!(row[start..start + count].iter().all(|&v| v == 0.0));+ assert!(+ row[..start]+ .iter()+ .chain(&row[start + count..])+ .all(|&v| v == -10000.0)+ );+ }+ }+ let pool = input(&inputs, "pool_weights_512");+ for slot in 0..2 {+ let sum: f32 = pool.values.iter().skip(slot).step_by(6).sum();+ assert!((sum - 1.0).abs() < 1e-6);+ }+ let lifted = recognizer_paths(inputs, 2048);+ assert_eq!(input(&lifted, "x").shape, [1, 3, 48, 7168]);+ assert_eq!(input(&lifted, "path_0").values[0], 1.0);+ assert_eq!(input(&lifted, "p1_attention_bias").shape, [1, 1, 896, 896]);+ }++ #[test]+ fn packing_preserves_the_last_step_at_the_maximum_width() {+ for width in [7167, 7168] {+ let line = Line {+ values: vec![0.5; 3 * 48 * width],+ width,+ content_width: width as i32,+ };+ let (inputs, offsets) = packed(&[&line], 7168, 21);+ assert_eq!(offsets, [0]);+ assert_eq!(input(&inputs, "mask_1_896").values[895], 1.0);+ assert_eq!(+ input(&recognizer_paths(inputs, 7168), "path_0").values[0],+ 0.0+ );+ }+ }+}diff --git a/rust/crates/ml/src/ocr/detect.rs b/rust/crates/ml/src/ocr/detect.rsindex cf440933ea..0731629ca2 100644--- a/rust/crates/ml/src/ocr/detect.rs+++ b/rust/crates/ml/src/ocr/detect.rs@@ -9,7 +9,7 @@ use super::tensor::{BgrNormalization, write_bgr_planes};use crate::cv;use crate::cv::image::{Contour, ImageU8};use crate::error::{MlError, MlResult};-use crate::onnx::{PreparedF32Input, SessionRunError, run_f32};+use crate::onnx::SessionRunError;const MAX_INPUT_SIDE: i32 = 960;const INPUT_STRIDE: i32 = 32;@@ -54,7 +54,7 @@ impl TextDetector {let (input_width, input_height) = detector_input_size(working.width, working.height);let resized = cv::resize_u8(working, input_width, input_height, cv::Interp::Bilinear).map_err(MlError::Preprocess)?;- let input = PreparedF32Input::new(normalized_bgr_planes(&resized)?);+ let input = normalized_bgr_planes(&resized)?;let values = self.infer(&input, input_width, input_height)?;let probability_map = ProbabilityMap {width: input_width as usize,@@ -75,18 +75,24 @@ impl TextDetector {})}- fn infer(&self, input: &PreparedF32Input, width: i32, height: i32) -> MlResult<Vec<f32>> {- let expected_shape = [1i64, 1, i64::from(height), i64::from(width)];+ fn infer(&self, input: &[f32], width: i32, height: i32) -> MlResult<Vec<f32>> {+ let expected_shape = [1i64, 1, 960, 960];let mut session = self.session.lock().unwrap_or_else(PoisonError::into_inner);- let input_shape = [1, 3, i64::from(height), i64::from(width)];- let (values, _usage) = session.run(input_shape, |session| {- let (shape, values) = run_f32(session, input, input_shape)?;- if shape != expected_shape {+ let inputs = super::context::detector(input, height as usize, width as usize);+ let (values, _usage) = session.run(|session| {+ let (shape, values) = super::context::infer(session, &inputs)?;+ if shape != expected_shape || values.len() != 960 * 960 {return Err(SessionRunError::from(MlError::CorruptModel(format!("text detector produced output shape {shape:?}, expected {expected_shape:?}"))));}- Ok(values)+ Ok(values+ .as_chunks::<960>()+ .0+ .iter()+ .take(height as usize)+ .flat_map(|row| row[..width as usize].iter().copied())+ .collect())})?;Ok(values)}diff --git a/rust/crates/ml/src/ocr/mod.rs b/rust/crates/ml/src/ocr/mod.rsindex ddaf78c382..10ab3a4514 100644--- a/rust/crates/ml/src/ocr/mod.rs+++ b/rust/crates/ml/src/ocr/mod.rs@@ -2,6 +2,7 @@ pub mod assets;mod cancel;mod characters;mod classify;+mod context;mod crop;mod detect;mod dictionary;diff --git a/rust/crates/ml/src/ocr/recognize.rs b/rust/crates/ml/src/ocr/recognize.rsindex b0dec5eff8..40600dff6f 100644--- a/rust/crates/ml/src/ocr/recognize.rs+++ b/rust/crates/ml/src/ocr/recognize.rs@@ -8,9 +8,7 @@ use super::tensor::{BgrNormalization, prepare_crops, write_bgr_planes};use crate::cv;use crate::cv::image::ImageU8;use crate::error::{MlError, MlResult};-use crate::onnx::{- BorrowedFloatTensor, FloatTensorData, PreparedF32Input, with_prepared_float_output,-};+use crate::onnx::{FloatTensorData, SessionRunError};use super::session::{OcrModel, OcrSession};const REC_VOCABULARY_SIZE: usize = 18385;@@ -81,37 +79,95 @@ impl TextRecognizer {return Ok(Vec::new());}let dictionary = self.dictionary.entries()?;- recognize_in_batches(crops, request, |batch, layout| {- self.infer_batch(batch, layout, dictionary)- })- }-- fn infer_batch(- &self,- batch: &[&ImageU8],- layout: &BatchLayout,- dictionary: &[String],- ) -> MlResult<Vec<Recognition>> {- let input = PreparedF32Input::new(prepare_crops(|| layout.tensor(batch))?);- let count = batch.len();- let input_shape = [- count as i64,- 3,- i64::from(REC_HEIGHT),- i64::from(layout.target_width),- ];- let mut session = self.session.lock().unwrap_or_else(PoisonError::into_inner);- let (recognized, _usage) = session.run(input_shape, |session| {- with_prepared_float_output(session, &input, input_shape, |shape, values| match values {- BorrowedFloatTensor::F32(values) => {- decode_output(shape, values, layout, dictionary)+ let mut lines = Vec::new();+ let mut line_crops = Vec::new();+ let ordered = recognize_in_batches(crops, request, |batch, layout| {+ let mut indices = Vec::with_capacity(batch.len());+ for (&crop, &content_width) in batch.iter().zip(&layout.content_widths) {+ indices.push(lines.len());+ line_crops.push(crop);+ lines.push(super::context::Line {+ values: Vec::new(),+ width: layout.target_width as usize,+ content_width,+ });+ }+ Ok(indices)+ })?;+ let mut results = vec![Recognition::default(); lines.len()];+ let mut pending: Vec<usize> = (0..lines.len()).collect();+ pending.sort_by_key(|&i| std::cmp::Reverse(lines[i].width));+ while !pending.is_empty() {+ request.check()?;+ let width = choose_packed_width(&pending, &lines);+ let slots = width / 336;+ let mut selected = Vec::new();+ let mut used = 0;+ pending.retain(|&index| {+ let size = lines[index].width.div_ceil(8) * 8;+ if selected.len() < slots && used + size <= width {+ selected.push(index);+ used += size + 16;+ false+ } else {+ true+ }+ });+ if selected.is_empty() {+ return Err(+ MlError::Preprocess("line exceeds fixed packed canvas".to_owned()).into(),+ );+ }+ prepare_crops(|| {+ for &index in &selected {+ let line = &mut lines[index];+ line.values = BatchLayout {+ target_width: line.width as i32,+ content_widths: vec![line.content_width],+ }+ .tensor(&[line_crops[index]])?;}- BorrowedFloatTensor::F16(values) => {- decode_output(shape, values, layout, dictionary)+ Ok(())+ })?;+ let batch: Vec<_> = selected.iter().map(|&index| &lines[index]).collect();+ let (inputs, offsets) = super::context::packed(&batch, width, slots);+ let inputs = super::context::recognizer_paths(inputs, width);+ let mut session = self.session.lock().unwrap_or_else(PoisonError::into_inner);+ let (values, _) = session.run(|session| {+ let (shape, values) = super::context::infer(session, &inputs)?;+ if shape != [1, 896, 2] || values.len() != 896 * 2 {+ return Err(SessionRunError::from(MlError::CorruptModel(format!(+ "text recognizer produced output shape {shape:?}, expected [1, 896, 2]"+ ))));}+ Ok(values)+ })?;+ for (&index, &offset) in selected.iter().zip(&offsets) {+ let line = &lines[index];+ let start = offset / 8 * 2;+ let count = line.width.div_ceil(4) / 2;+ let layout = BatchLayout {+ target_width: line.width as i32,+ content_widths: vec![line.content_width],+ };+ let mut decoded = decode_output(+ &[1, count as i64, 2],+ &values[start..start + count * 2],+ &layout,+ dictionary,+ )?;+ results[index] = decoded.remove(0);+ lines[index].values = Vec::new();+ }+ }+ ordered+ .into_iter()+ .map(|index| {+ results.get(index).cloned().ok_or_else(|| {+ MlError::Preprocess("packed line index out of range".to_owned()).into()+ })})- })?;- Ok(recognized)+ .collect()}}@@ -179,12 +235,12 @@ where.collect()}-fn recognize_in_batches(- crops: &[&ImageU8],+fn recognize_in_batches<'a, T>(+ crops: &[&'a ImageU8],request: &RequestGuard<'_>,- mut infer_batch: impl FnMut(&[&ImageU8], &BatchLayout) -> MlResult<Vec<Recognition>>,-) -> Result<Vec<Recognition>, OcrError> {- let mut results: Vec<Option<Recognition>> = vec![None; crops.len()];+ mut infer_batch: impl FnMut(&[&'a ImageU8], &BatchLayout) -> MlResult<Vec<T>>,+) -> Result<Vec<T>, OcrError> {+ let mut results: Vec<Option<T>> = std::iter::repeat_with(|| None).take(crops.len()).collect();for indices in ascending_aspect_order(crops).chunks(REC_BATCH_SIZE) {request.check()?;let batch: Vec<&ImageU8> = indices.iter().map(|&index| crops[index]).collect();@@ -855,7 +911,8 @@ mod tests {let request = registry.begin(Some("cancelled"));registry.cancel("cancelled");- let error = recognize_in_batches(&[&crop], &request, |_, _| unreachable!()).unwrap_err();+ let error = recognize_in_batches::<Recognition>(&[&crop], &request, |_, _| unreachable!())+ .unwrap_err();assert!(matches!(error, OcrError::Cancelled), "{error}");}@@ -1152,3 +1209,32 @@ mod tests {assert!(matches!(error, OcrError::Cancelled));}}++fn choose_packed_width(pending: &[usize], lines: &[super::context::Line]) -> usize {+ let mut best_width = 7168;+ let mut best_cost = f64::INFINITY;+ for &width in super::context::branch_widths() {+ if width < lines[pending[0]].width {+ continue;+ }+ let mut used = 0;+ let mut payload = 0;+ let mut count = 0;+ for &index in pending {+ let size = lines[index].width.div_ceil(8) * 8;+ if count < width / 336 && used + size <= width {+ used += size + 16;+ payload += lines[index].width;+ count += 1;+ }+ }+ if payload > 0 {+ let cost = (width + 128) as f64 / payload as f64;+ if cost < best_cost {+ best_cost = cost;+ best_width = width;+ }+ }+ }+ best_width+}diff --git a/rust/crates/ml/src/ocr/session.rs b/rust/crates/ml/src/ocr/session.rsindex 71c7c39cdc..4f82a1f44f 100644--- a/rust/crates/ml/src/ocr/session.rs+++ b/rust/crates/ml/src/ocr/session.rs@@ -1,6 +1,4 @@-use std::collections::VecDeque;--use crate::error::{MlError, MlResult};+use crate::error::MlResult;use crate::onnx::{ExecutionMode, GpuOptions, OnnxSession, ProviderUsage, SessionHandle, SessionRunResult,};@@ -12,157 +10,33 @@ pub(super) enum OcrModel {Recognition,}-impl OcrModel {- fn cache_capacity(self) -> usize {- match self {- Self::Detection => 2,- Self::Classification => 6,- Self::Recognition => 8,- }- }-- fn namespace(self) -> &'static str {- match self {- Self::Detection => "ocr-detection",- Self::Classification => "ocr-classification",- Self::Recognition => "ocr-recognition",- }- }-- fn mode(self) -> ExecutionMode {- if matches!(self, Self::Classification)- && !cfg!(any(target_os = "ios", target_os = "macos"))- {- ExecutionMode::CpuOnly- } else {- ExecutionMode::GpuPreferred- }- }-- fn gpu_options(self, [count, _, height, width]: [i64; 4]) -> GpuOptions {- let dimensions = match self {- Self::Detection => vec![- ("DynamicDimension.0", count),- ("DynamicDimension.1", height),- ("DynamicDimension.2", width),- ],- Self::Classification => vec![- ("p2o.DynamicDimension.0", count),- ("p2o.DynamicDimension.1", height),- ("p2o.DynamicDimension.2", width),- ],- Self::Recognition => vec![("DynamicDimension.0", count), ("DynamicDimension.1", width)],- };- GpuOptions {- dimensions,- #[cfg(any(target_os = "android", target_os = "linux", target_os = "windows"))]- prefer_nhwc: matches!(self, Self::Detection),- }- }-}-pub(super) struct OcrSession {- model_path: String,- model: OcrModel,- sessions: VecDeque<([i64; 4], OnnxSession)>,+ session: OnnxSession,}impl OcrSession {pub(super) fn new(model_path: &str, model: OcrModel) -> Self {+ let namespace = match model {+ OcrModel::Detection => "ocr-detection-fixed-v1",+ OcrModel::Classification => "ocr-classification-fixed-v1",+ OcrModel::Recognition => "ocr-recognition-fixed-v1",+ };+ let options = GpuOptions {+ subgraphs: !matches!(model, OcrModel::Classification),+ #[cfg(any(target_os = "android", target_os = "linux", target_os = "windows"))]+ prefer_nhwc: matches!(model, OcrModel::Detection),+ };Self {- model_path: model_path.to_owned(),- model,- sessions: VecDeque::new(),+ session: OnnxSession::new(model_path, namespace, ExecutionMode::GpuPreferred)+ .with_gpu_options(options)+ .with_unvalidated_acceleration(),}}pub(super) fn run<T>(&mut self,- shape: [i64; 4],operation: impl FnMut(&mut SessionHandle) -> SessionRunResult<T>,) -> MlResult<(T, ProviderUsage)> {- self.session_for(shape)?.run(operation)- }-- fn session_for(&mut self, shape: [i64; 4]) -> MlResult<&mut OnnxSession> {- let mode = self.model.mode();- let key = if mode == ExecutionMode::CpuOnly {- [0; 4]- } else {- shape- };- if let Some(index) = self.sessions.iter().position(|(cached, _)| *cached == key) {- self.sessions.make_contiguous()[..=index].rotate_right(1);- } else {- if self.sessions.len() == self.model.cache_capacity() {- self.sessions.pop_back();- }- let [n, c, h, w] = key;- let namespace = format!("{}-{n}x{c}x{h}x{w}-gpu-f32-v1", self.model.namespace());- let mut session = OnnxSession::new(&self.model_path, &namespace, mode);- if mode == ExecutionMode::GpuPreferred {- session = session- .with_gpu_options(self.model.gpu_options(shape))- .with_unvalidated_acceleration();- }- self.sessions.push_front((key, session));- }- self.sessions- .front_mut()- .map(|(_, session)| session)- .ok_or_else(|| MlError::Runtime("OCR session cache is empty".to_owned()))- }-}--#[cfg(test)]-mod tests {- use super::*;-- #[test]- fn reuses_shapes_and_evicts_the_least_recently_used_session() {- let mut cache = OcrSession::new("rec.onnx", OcrModel::Recognition);- let first = [6, 3, 48, 320];- cache.session_for(first).unwrap().initialize_load_state();- for width in 321..328 {- cache.session_for([6, 3, 48, width]).unwrap();- }- assert!(cache.session_for(first).unwrap().has_load_state());- cache.session_for([1, 3, 48, 320]).unwrap();- assert_eq!(cache.sessions.len(), OcrModel::Recognition.cache_capacity());- assert!(cache.session_for(first).unwrap().has_load_state());- assert!(- !cache- .sessions- .iter()- .any(|(shape, _)| *shape == [6, 3, 48, 321])- );- }-- #[test]- fn models_override_their_actual_spatial_dimensions() {- assert_eq!(- OcrModel::Detection.gpu_options([1, 3, 960, 704]).dimensions,- vec![- ("DynamicDimension.0", 1),- ("DynamicDimension.1", 960),- ("DynamicDimension.2", 704)- ]- );- assert_eq!(- OcrModel::Recognition- .gpu_options([3, 3, 48, 641])- .dimensions,- vec![("DynamicDimension.0", 3), ("DynamicDimension.1", 641)]- );- assert_eq!(- OcrModel::Classification- .gpu_options([3, 3, 48, 192])- .dimensions,- vec![- ("p2o.DynamicDimension.0", 3),- ("p2o.DynamicDimension.1", 48),- ("p2o.DynamicDimension.2", 192)- ]- );+ self.session.run(operation)}}diff --git a/rust/crates/ml/src/ocr/tensor.rs b/rust/crates/ml/src/ocr/tensor.rsindex 0f8c976d7b..7c8f914480 100644--- a/rust/crates/ml/src/ocr/tensor.rs+++ b/rust/crates/ml/src/ocr/tensor.rs@@ -71,6 +71,9 @@ pub(crate) fn write_bgr_planes(planes.len())));}+ let lookup: [[f32; 256]; 3] = std::array::from_fn(|channel| {+ std::array::from_fn(|value| normalization.apply(channel, value as u8))+ });let (blue, rest) = planes.split_at_mut(plane);let (green, red) = rest.split_at_mut(plane);let top = (plane_height - height) / 2 * plane_width;@@ -82,9 +85,9 @@ pub(crate) fn write_bgr_planes(for (y, row) in rgb.data.chunks_exact(width * 3).enumerate() {let offset = top + y * plane_width;for (x, px) in row.as_chunks::<3>().0.iter().enumerate() {- blue[offset + x] = normalization.apply(0, px[2]);- green[offset + x] = normalization.apply(1, px[1]);- red[offset + x] = normalization.apply(2, px[0]);+ blue[offset + x] = lookup[0][px[2] as usize];+ green[offset + x] = lookup[1][px[1] as usize];+ red[offset + x] = lookup[2][px[0] as usize];}let padding = offset + width..offset + plane_width;blue[padding.clone()].fill(0.0);diff --git a/rust/crates/ml/src/onnx.rs b/rust/crates/ml/src/onnx.rsindex 6e2255895e..fc57a26d1e 100644--- a/rust/crates/ml/src/onnx.rs+++ b/rust/crates/ml/src/onnx.rs@@ -26,7 +26,7 @@ use providers::{ExecutionProvider, ProviderPlan};#[derive(Clone, Debug)]pub(crate) struct GpuOptions {- pub(crate) dimensions: Vec<(&'static str, i64)>,+ pub(crate) subgraphs: bool,#[cfg(any(target_os = "android", target_os = "linux", target_os = "windows"))]pub(crate) prefer_nhwc: bool,}@@ -237,7 +237,7 @@ impl SessionRunError {Self::Retryable(error)}- fn from_inference_error(error: ort::Error) -> Self {+ pub(crate) fn from_inference_error(error: ort::Error) -> Self {match error.code() {ort::ErrorCode::GenericFailure| ort::ErrorCode::RuntimeExceptiondiff --git a/rust/crates/ml/src/onnx/providers.rs b/rust/crates/ml/src/onnx/providers.rsindex 596cdf04d8..9187c029e3 100644--- a/rust/crates/ml/src/onnx/providers.rs+++ b/rust/crates/ml/src/onnx/providers.rs@@ -157,7 +157,7 @@ pub(super) fn provider_attempt(#[cfg(any(target_os = "ios", target_os = "macos"))]ExecutionProvider::CoreMl => {let (coreml_provider, coreml_cache_dir) =- coreml_provider(_model_path, _model_namespace, _gpu_options.is_some());+ coreml_provider(_model_path, _model_namespace, _gpu_options);ProviderAttempt {providers: if _gpu_options.is_some() {vec![coreml_provider]@@ -175,7 +175,7 @@ pub(super) fn provider_attempt(#[cfg(any(target_os = "android", target_os = "linux", target_os = "windows"))]ExecutionProvider::WebGpu => ProviderAttempt {providers: if let Some(options) = _gpu_options {- vec![webgpu_provider(options.prefer_nhwc)]+ vec![ocr_webgpu_provider(options.prefer_nhwc)]} else {webgpu_attempt_providers()},@@ -200,15 +200,10 @@ pub(super) fn build_session(.with_inter_threads(1)?;if let Some(options) = gpu_options- && matches!(- attempt.execution_provider,- ExecutionProvider::CoreMl | ExecutionProvider::WebGpu- )+ && (attempt.execution_provider == ExecutionProvider::WebGpu+ || (attempt.execution_provider == ExecutionProvider::CoreMl && !options.subgraphs)){builder = builder.with_disable_cpu_fallback()?;- for &(name, value) in &options.dimensions {- builder = builder.with_dimension_override(name, value)?;- }}if attempt.disable_intra_op_spinning {@@ -310,18 +305,19 @@ fn accelerated_provider_allowed(fn coreml_provider(model_path: &str,model_namespace: &str,- full_gpu: bool,+ options: Option<&GpuOptions>,) -> (ExecutionProviderDispatch, Option<PathBuf>) {let mut provider = CoreML::default().with_model_format(ModelFormat::MLProgram).with_compute_units(ComputeUnits::All).with_specialization_strategy(SpecializationStrategy::Default);- if full_gpu {+ if let Some(options) = options {provider = provider.with_compute_units(ComputeUnits::CPUAndGPU).with_low_precision_accumulation_on_gpu(false)- .with_static_input_shapes(true);+ .with_static_input_shapes(true)+ .with_subgraphs(options.subgraphs);}let mut prepared_cache_dir = None;@@ -360,6 +356,23 @@ fn webgpu_provider(prefer_nhwc: bool) -> ExecutionProviderDispatch {provider.build().error_on_failure()}+#[cfg(any(target_os = "android", target_os = "linux", target_os = "windows"))]+fn ocr_webgpu_provider(prefer_nhwc: bool) -> ExecutionProviderDispatch {+ use ort::ep::ArbitrarilyConfigurableExecutionProvider;++ let backend = if cfg!(target_os = "windows") {+ "D3D12"+ } else {+ "Vulkan"+ };+ WebGPU::default()+ .with_arbitrary_config("preferredLayout", if prefer_nhwc { "NHWC" } else { "NCHW" })+ .with_arbitrary_config("dawnBackendType", backend)+ .with_arbitrary_config("enableGraphCapture", "0")+ .build()+ .error_on_failure()+}+#[cfg(target_os = "android")]fn webgpu_attempt_providers() -> Vec<ExecutionProviderDispatch> {vec![