Skip to content

armgit / ente

Public Imported from GitHub
0

πŸ’š End-to-end encrypted cloud for everything.

Use shared fixed OCR models and packed GPU inference

laurenspriem5ec95f8ccdcd
diff --git a/infra/ml/playground/optimizations/README.md b/infra/ml/playground/optimizations/README.md
index f58323e0ba..63d58792f1 100644
--- a/infra/ml/playground/optimizations/README.md
+++ b/infra/ml/playground/optimizations/README.md
@@ -17,21 +17,23 @@ uv run --no-project --with numpy==2.5.3 --with onnx==1.22.0 python \
--output-dir infra/ml/playground/optimizations/models/ocr
```
-The script keeps FP32 weights and applies these transformations:
-
-- Detection: fold the transposed-convolution head's bias and batch normalization,
- and expand `HardSwish` into `HardSigmoid * x` for CoreML compatibility.
-- Classification: canonicalize scalar constants and simplify expanded activations
- to `HardSigmoid * x`.
-- Recognition: fold affine operations into unpadded convolutions, fuse layer
- normalization, expand `HardSwish`, and align the final matrix multiplication
- for WebGPU without changing the vocabulary. Return FP32 `[N,T,2]` containing
- the first winning token index and its probability; Rust retains CTC decoding.
-
-Missing sources are downloaded from `https://models.ente.com/PP-OCRv5`. The script
-verifies source and output hashes and writes three ONNX files, the unchanged
-dictionary, and `ocr_model_manifest.json`. Publish the ONNX files at new URLs:
-the recognizer's output contract differs from the original model.
+The script produces three shared FP32 models (23.17 MB total) with fixed inputs:
+
+- Detection: a 960Γ—960 canvas with five fixed computation paths. Masks preserve
+ the original image boundaries and pooling when selecting a smaller path.
+- Classification: six 48Γ—192 crops; Rust pads incomplete batches.
+- Recognition: a 48Γ—7168 canvas with 2048- and 7168-wide paths. Rust packs lines;
+ masks isolate their convolutions, pooling, and attention. Aligned matrix
+ operations return winning token indices and probabilities for Rust CTC decoding.
+
+Affine folding and equivalent activation rewrites improve GPU support. Shared
+weights keep the fixed paths compact. These models require the matching Rust
+input adapter; they cannot replace the original models independently.
+
+Missing sources are downloaded from `https://models.ente.com/PP-OCRv5`. Source and
+output hashes are verified. Upload only `det_fixed_v1.onnx`, `cls_fixed_v1.onnx`,
+and `rec_fixed_v1.onnx`; the dictionary is unchanged. The generated
+`ocr_model_manifest.json` records the input shapes, hashes, and sizes.
## Rebuilding the models
diff --git a/infra/ml/playground/optimizations/models/ocr/ocr_model_manifest.json b/infra/ml/playground/optimizations/models/ocr/ocr_model_manifest.json
index 56f26722f7..0610001845 100644
--- a/infra/ml/playground/optimizations/models/ocr/ocr_model_manifest.json
+++ b/infra/ml/playground/optimizations/models/ocr/ocr_model_manifest.json
@@ -1,4 +1,32 @@
{
+ "format": "ente-ocr-fixed-v1",
+ "requires_context_adapter": true,
+ "detector_paths": [
+ [
+ 960,
+ 480
+ ],
+ [
+ 480,
+ 960
+ ],
+ [
+ 960,
+ 704
+ ],
+ [
+ 704,
+ 960
+ ],
+ [
+ 960,
+ 960
+ ]
+ ],
+ "recognizer_widths": [
+ 2048,
+ 7168
+ ],
"source_base_url": "https://models.ente.com/PP-OCRv5",
"source_sha256": {
"cls.onnx": "f4bb53707100c5f3d59ba834eb05bb400369f20aed35d4b26807b1bfadd2a70e",
@@ -9,85 +37,467 @@
"onnx": "1.22.0",
"models": [
{
- "file": "det.onnx",
- "sha256": "3e658f85236f1984e186070048d7cdc99ddbb84e12c91a6fd80922b85bd9487a",
- "bytes": 4750896,
- "operators": [
- "Add",
- "BatchNormalization",
- "Concat",
- "Constant",
- "Conv",
- "ConvTranspose",
- "GlobalAveragePool",
- "HardSigmoid",
- "Mul",
- "Relu",
- "Reshape",
- "Resize",
- "Sigmoid"
- ],
+ "file": "det_fixed_v1.onnx",
+ "sha256": "f655f119225b579fa8c3cbf64f6bb7cf56a26c9dc211706a25234d70a543ec8c",
+ "bytes": 5548458,
+ "inputs": {
+ "x": [
+ 1,
+ 3,
+ 960,
+ 960
+ ],
+ "p0_mask_960_480": [
+ 1,
+ 1,
+ 960,
+ 480
+ ],
+ "p0_mask_480_240": [
+ 1,
+ 1,
+ 480,
+ 240
+ ],
+ "p0_mask_240_120": [
+ 1,
+ 1,
+ 240,
+ 120
+ ],
+ "p0_mask_120_60": [
+ 1,
+ 1,
+ 120,
+ 60
+ ],
+ "p0_mask_60_30": [
+ 1,
+ 1,
+ 60,
+ 30
+ ],
+ "p0_mask_30_15": [
+ 1,
+ 1,
+ 30,
+ 15
+ ],
+ "p0_pool_scale_30_15": [
+ 1,
+ 1,
+ 1,
+ 1
+ ],
+ "p0_pool_scale_60_30": [
+ 1,
+ 1,
+ 1,
+ 1
+ ],
+ "p0_pool_scale_120_60": [
+ 1,
+ 1,
+ 1,
+ 1
+ ],
+ "p0_pool_scale_240_120": [
+ 1,
+ 1,
+ 1,
+ 1
+ ],
+ "p1_mask_480_960": [
+ 1,
+ 1,
+ 480,
+ 960
+ ],
+ "p1_mask_240_480": [
+ 1,
+ 1,
+ 240,
+ 480
+ ],
+ "p1_mask_120_240": [
+ 1,
+ 1,
+ 120,
+ 240
+ ],
+ "p1_mask_60_120": [
+ 1,
+ 1,
+ 60,
+ 120
+ ],
+ "p1_mask_30_60": [
+ 1,
+ 1,
+ 30,
+ 60
+ ],
+ "p1_mask_15_30": [
+ 1,
+ 1,
+ 15,
+ 30
+ ],
+ "p1_pool_scale_15_30": [
+ 1,
+ 1,
+ 1,
+ 1
+ ],
+ "p1_pool_scale_30_60": [
+ 1,
+ 1,
+ 1,
+ 1
+ ],
+ "p1_pool_scale_60_120": [
+ 1,
+ 1,
+ 1,
+ 1
+ ],
+ "p1_pool_scale_120_240": [
+ 1,
+ 1,
+ 1,
+ 1
+ ],
+ "p2_mask_960_704": [
+ 1,
+ 1,
+ 960,
+ 704
+ ],
+ "p2_mask_480_352": [
+ 1,
+ 1,
+ 480,
+ 352
+ ],
+ "p2_mask_240_176": [
+ 1,
+ 1,
+ 240,
+ 176
+ ],
+ "p2_mask_120_88": [
+ 1,
+ 1,
+ 120,
+ 88
+ ],
+ "p2_mask_60_44": [
+ 1,
+ 1,
+ 60,
+ 44
+ ],
+ "p2_mask_30_22": [
+ 1,
+ 1,
+ 30,
+ 22
+ ],
+ "p2_pool_scale_30_22": [
+ 1,
+ 1,
+ 1,
+ 1
+ ],
+ "p2_pool_scale_60_44": [
+ 1,
+ 1,
+ 1,
+ 1
+ ],
+ "p2_pool_scale_120_88": [
+ 1,
+ 1,
+ 1,
+ 1
+ ],
+ "p2_pool_scale_240_176": [
+ 1,
+ 1,
+ 1,
+ 1
+ ],
+ "p3_mask_704_960": [
+ 1,
+ 1,
+ 704,
+ 960
+ ],
+ "p3_mask_352_480": [
+ 1,
+ 1,
+ 352,
+ 480
+ ],
+ "p3_mask_176_240": [
+ 1,
+ 1,
+ 176,
+ 240
+ ],
+ "p3_mask_88_120": [
+ 1,
+ 1,
+ 88,
+ 120
+ ],
+ "p3_mask_44_60": [
+ 1,
+ 1,
+ 44,
+ 60
+ ],
+ "p3_mask_22_30": [
+ 1,
+ 1,
+ 22,
+ 30
+ ],
+ "p3_pool_scale_22_30": [
+ 1,
+ 1,
+ 1,
+ 1
+ ],
+ "p3_pool_scale_44_60": [
+ 1,
+ 1,
+ 1,
+ 1
+ ],
+ "p3_pool_scale_88_120": [
+ 1,
+ 1,
+ 1,
+ 1
+ ],
+ "p3_pool_scale_176_240": [
+ 1,
+ 1,
+ 1,
+ 1
+ ],
+ "p4_mask_960_960": [
+ 1,
+ 1,
+ 960,
+ 960
+ ],
+ "p4_mask_480_480": [
+ 1,
+ 1,
+ 480,
+ 480
+ ],
+ "p4_mask_240_240": [
+ 1,
+ 1,
+ 240,
+ 240
+ ],
+ "p4_mask_120_120": [
+ 1,
+ 1,
+ 120,
+ 120
+ ],
+ "p4_mask_60_60": [
+ 1,
+ 1,
+ 60,
+ 60
+ ],
+ "p4_mask_30_30": [
+ 1,
+ 1,
+ 30,
+ 30
+ ],
+ "p4_pool_scale_30_30": [
+ 1,
+ 1,
+ 1,
+ 1
+ ],
+ "p4_pool_scale_60_60": [
+ 1,
+ 1,
+ 1,
+ 1
+ ],
+ "p4_pool_scale_120_120": [
+ 1,
+ 1,
+ 1,
+ 1
+ ],
+ "p4_pool_scale_240_240": [
+ 1,
+ 1,
+ 1,
+ 1
+ ],
+ "path_3": [],
+ "path_2": [],
+ "path_1": [],
+ "path_0": []
+ },
"output": "unchanged"
},
{
- "file": "cls.onnx",
- "sha256": "8675903197a1bab7060e830e9bcc63d29c5cf7d372735338bd9501f829e94ed1",
- "bytes": 574875,
- "operators": [
- "Add",
- "BatchNormalization",
- "Cast",
- "Concat",
- "Constant",
- "Conv",
- "GlobalAveragePool",
- "HardSigmoid",
- "MatMul",
- "MaxPool",
- "Mul",
- "Relu",
- "Reshape",
- "Shape",
- "Slice",
- "Softmax"
- ],
+ "file": "cls_fixed_v1.onnx",
+ "sha256": "378d52a73263828d08d4dda37f1d0aac2e36cbd486fdc6351bf309d515610654",
+ "bytes": 590475,
+ "inputs": {
+ "x": [
+ 6,
+ 3,
+ 48,
+ 192
+ ]
+ },
"output": "unchanged"
},
{
- "file": "rec.onnx",
- "sha256": "667691bb173a94106b595aa87c7d176b53b68cf831b6ffa09569018754265e57",
- "bytes": 16686404,
- "operators": [
- "Add",
- "AveragePool",
- "BatchNormalization",
- "Ceil",
- "Concat",
- "Constant",
- "Conv",
- "Div",
- "GlobalAveragePool",
- "HardSigmoid",
- "LayerNormalization",
- "MatMul",
- "Mul",
- "Pow",
- "ReduceMax",
- "ReduceMean",
- "ReduceMin",
- "Relu",
- "Reshape",
- "Shape",
- "Sigmoid",
- "Slice",
- "Softmax",
- "Sqrt",
- "Squeeze",
- "Sub",
- "Transpose",
- "Unsqueeze"
- ],
- "output": "FP32 [N,T,2]: first winning index, original winning probability"
+ "file": "rec_fixed_v1.onnx",
+ "sha256": "6dda4c0891af5a70c5b0f618b62588df7f3140616d1c560be5ec6496747b54fd",
+ "bytes": 17031601,
+ "inputs": {
+ "x": [
+ 1,
+ 3,
+ 48,
+ 7168
+ ],
+ "p0_mask_48_2048": [
+ 1,
+ 1,
+ 1,
+ 2048
+ ],
+ "p0_mask_24_1024": [
+ 1,
+ 1,
+ 1,
+ 1024
+ ],
+ "p0_mask_12_1024": [
+ 1,
+ 1,
+ 1,
+ 1024
+ ],
+ "p0_mask_12_512": [
+ 1,
+ 1,
+ 1,
+ 512
+ ],
+ "p0_pool_weights_512": [
+ 1,
+ 512,
+ 6
+ ],
+ "p0_gate_weights_512": [
+ 1,
+ 6,
+ 512
+ ],
+ "p0_mask_6_512": [
+ 1,
+ 1,
+ 1,
+ 512
+ ],
+ "p0_mask_3_512": [
+ 1,
+ 1,
+ 1,
+ 512
+ ],
+ "p0_mask_1_256": [
+ 1,
+ 1,
+ 1,
+ 256
+ ],
+ "p0_attention_bias": [
+ 1,
+ 1,
+ 256,
+ 256
+ ],
+ "p1_mask_48_7168": [
+ 1,
+ 1,
+ 1,
+ 7168
+ ],
+ "p1_mask_24_3584": [
+ 1,
+ 1,
+ 1,
+ 3584
+ ],
+ "p1_mask_12_3584": [
+ 1,
+ 1,
+ 1,
+ 3584
+ ],
+ "p1_mask_12_1792": [
+ 1,
+ 1,
+ 1,
+ 1792
+ ],
+ "p1_pool_weights_1792": [
+ 1,
+ 1792,
+ 21
+ ],
+ "p1_gate_weights_1792": [
+ 1,
+ 21,
+ 1792
+ ],
+ "p1_mask_6_1792": [
+ 1,
+ 1,
+ 1,
+ 1792
+ ],
+ "p1_mask_3_1792": [
+ 1,
+ 1,
+ 1,
+ 1792
+ ],
+ "p1_mask_1_896": [
+ 1,
+ 1,
+ 1,
+ 896
+ ],
+ "p1_attention_bias": [
+ 1,
+ 1,
+ 896,
+ 896
+ ],
+ "path_0": []
+ },
+ "output": "FP32 [1,896,2]: packed first winning index and probability"
}
]
}
diff --git a/infra/ml/playground/optimizations/ocr_fixed_inputs.py b/infra/ml/playground/optimizations/ocr_fixed_inputs.py
new file mode 100644
index 0000000000..fa42af5279
--- /dev/null
+++ b/infra/ml/playground/optimizations/ocr_fixed_inputs.py
@@ -0,0 +1,532 @@
+import copy
+import hashlib
+
+import numpy as np
+import onnx
+from onnx import helper, numpy_helper
+
+DOMAIN = "ente.ocr.weights"
+
+
+def shaped(source_model, shape):
+ model = copy.deepcopy(source_model)
+ for dim, value in zip(model.graph.input[0].type.tensor_type.shape.dim, shape):
+ dim.dim_value = value
+ del model.graph.value_info[:]
+ for output in model.graph.output:
+ output.type.tensor_type.ClearField("shape")
+ return onnx.shape_inference.infer_shapes(model, data_prop=True)
+
+
+def dimensions(model):
+ return {
+ value.name: [dim.dim_value for dim in value.type.tensor_type.shape.dim]
+ for value in [*model.graph.input, *model.graph.value_info, *model.graph.output]
+ }
+
+
+def detector_model(source_model, shape):
+ model = shaped(source_model, shape)
+ sizes = dimensions(model)
+ nodes = []
+ masks = {}
+ mask_cache = {}
+
+ def auxiliary(name, dims, tensor, kind):
+ if name not in masks:
+ model.graph.input.append(
+ helper.make_tensor_value_info(name, onnx.TensorProto.FLOAT, dims)
+ )
+ masks[name] = {"shape": dims, "tensor": tensor, "kind": kind}
+ return name
+
+ def mask(tensor):
+ if tensor in mask_cache:
+ return mask_cache[tensor]
+ n, _, h, w = sizes[tensor]
+ dims = [n, 1, h, w]
+ name = auxiliary(f"mask_{h}_{w}", dims, tensor, "spatial")
+ out = tensor + "_masked"
+ nodes.append(helper.make_node("Mul", [tensor, name], [out], name=out))
+ mask_cache[tensor] = out
+ return out
+
+ for original in model.graph.node:
+ node = copy.deepcopy(original)
+ if node.op_type in [
+ "Conv",
+ "AveragePool",
+ "GlobalAveragePool",
+ ]:
+ dims = sizes.get(node.input[0], [])
+ attrs = {a.name: helper.get_attribute_value(a) for a in node.attribute}
+ spatial = len(dims) == 4 and dims[-1] > 1
+ mixing = node.op_type != "Conv" or any(
+ k > 1 for k in attrs.get("kernel_shape", [1, 1])
+ )
+ if spatial and mixing:
+ node.input[0] = mask(original.input[0])
+ if node.op_type == "GlobalAveragePool":
+ n, _, h, w = sizes[original.input[0]]
+ scale = auxiliary(
+ f"pool_scale_{h}_{w}", [n, 1, 1, 1], original.input[0], "scale"
+ )
+ output = node.output[0]
+ node.output[0] = output + "_uncorrected"
+ nodes.append(node)
+ nodes.append(
+ helper.make_node(
+ "Mul", [node.output[0], scale], [output], name=output + "_correct"
+ )
+ )
+ continue
+ if node.op_type == "Softmax" and len(sizes[node.input[0]]) == 4:
+ n, _, _, w = sizes[node.input[0]]
+ name = auxiliary(
+ "attention_bias", [n, 1, 1, w], original.input[0], "attention"
+ )
+ out = node.input[0] + "_masked"
+ nodes.append(
+ helper.make_node("Add", [node.input[0], name], [out], name=out)
+ )
+ node.input[0] = out
+ nodes.append(node)
+ del model.graph.node[:]
+ model.graph.node.extend(nodes)
+ del model.graph.value_info[:]
+ model = onnx.shape_inference.infer_shapes(model, data_prop=True)
+ onnx.checker.check_model(model)
+ return model, masks
+
+
+def recognizer_model(source_model, width, slots):
+ model = shaped(source_model, [1, 3, 48, width])
+ sizes = dimensions(model)
+ nodes = []
+ metadata = {}
+ seen = {}
+ gates = {}
+
+ def aux(name, shape, tensor, kind):
+ if name not in metadata:
+ model.graph.input.append(
+ helper.make_tensor_value_info(name, onnx.TensorProto.FLOAT, shape)
+ )
+ metadata[name] = {"shape": shape, "tensor": tensor, "kind": kind}
+ return name
+
+ def const(name, value):
+ model.graph.initializer.append(
+ numpy_helper.from_array(np.asarray(value, np.int64), name)
+ )
+ return name
+
+ def mask(tensor):
+ if tensor in seen:
+ return seen[tensor]
+ _, _, h, w = sizes[tensor]
+ name = aux(f"mask_{h}_{w}", [1, 1, 1, w], tensor, "spatial")
+ out = tensor + "_masked"
+ nodes.append(helper.make_node("Mul", [tensor, name], [out], name=out))
+ seen[tensor] = out
+ return out
+
+ for original in model.graph.node:
+ node = copy.deepcopy(original)
+ if node.op_type == "GlobalAveragePool":
+ tensor = node.input[0]
+ _, c, _, w = sizes[tensor]
+ prefix = node.output[0] + "_packed"
+ weights = aux(f"pool_weights_{w}", [1, w, slots], tensor, "pool")
+ nodes.extend(
+ [
+ helper.make_node(
+ "ReduceMean",
+ [tensor],
+ [prefix + "_height"],
+ axes=[2],
+ keepdims=0,
+ ),
+ helper.make_node(
+ "MatMul", [prefix + "_height", weights], [prefix + "_means"]
+ ),
+ helper.make_node(
+ "Transpose",
+ [prefix + "_means"],
+ [prefix + "_transposed"],
+ perm=[2, 1, 0],
+ ),
+ helper.make_node(
+ "Reshape",
+ [
+ prefix + "_transposed",
+ const(prefix + "_shape", [slots, c, 1, 1]),
+ ],
+ [node.output[0]],
+ ),
+ ]
+ )
+ gate = "p2o.pd_op.hardsigmoid." + (
+ "0.0" if "pool2d.0" in node.output[0] else "1.0"
+ )
+ gates[gate] = (c, w, tensor)
+ continue
+ if node.op_type in ["Conv", "AveragePool"]:
+ dims = sizes.get(node.input[0], [])
+ attrs = {a.name: helper.get_attribute_value(a) for a in node.attribute}
+ if (
+ len(dims) == 4
+ and dims[-1] > 1
+ and (
+ node.op_type != "Conv"
+ or any(k > 1 for k in attrs.get("kernel_shape", [1, 1]))
+ )
+ ):
+ node.input[0] = mask(node.input[0])
+ if node.op_type == "Softmax" and len(sizes[node.input[0]]) == 4:
+ t = sizes[node.input[0]][-1]
+ name = aux("attention_bias", [1, 1, t, t], node.input[0], "attention")
+ out = node.input[0] + "_masked"
+ nodes.append(
+ helper.make_node("Add", [node.input[0], name], [out], name=out)
+ )
+ node.input[0] = out
+ output = node.output[0]
+ if output in gates:
+ c, w, tensor = gates[output]
+ node.output[0] = output + "_slots"
+ weights = aux(f"gate_weights_{w}", [1, slots, w], tensor, "gate")
+ nodes.extend(
+ [
+ node,
+ helper.make_node(
+ "Reshape",
+ [node.output[0], const(output + "_flatshape", [slots, c])],
+ [output + "_flat"],
+ ),
+ helper.make_node(
+ "Transpose",
+ [output + "_flat"],
+ [output + "_transposed"],
+ perm=[1, 0],
+ ),
+ helper.make_node(
+ "MatMul",
+ [output + "_transposed", weights],
+ [output + "_mapped"],
+ ),
+ helper.make_node(
+ "Reshape",
+ [output + "_mapped", const(output + "_shape", [1, c, 1, w])],
+ [output],
+ ),
+ ]
+ )
+ else:
+ nodes.append(node)
+ del model.graph.node[:]
+ model.graph.node.extend(nodes)
+ del model.graph.value_info[:]
+ for output in model.graph.output:
+ output.type.tensor_type.ClearField("shape")
+ model = onnx.shape_inference.infer_shapes(model, data_prop=True)
+ onnx.checker.check_model(model)
+ return model, metadata
+
+
+def pad_vocabulary(original):
+ model = copy.deepcopy(original)
+ values = {v.name: numpy_helper.to_array(v) for v in model.graph.initializer}
+ for node in model.graph.node:
+ if node.op_type == "Constant":
+ for attribute in node.attribute:
+ if attribute.name == "value":
+ values[node.output[0]] = numpy_helper.to_array(attribute.t)
+ removed = next(
+ node
+ for node in model.graph.node
+ if node.op_type == "Slice" and node.input[0] == "modelopt_padded_projection"
+ )
+ add = next(
+ node
+ for node in model.graph.node
+ if node.op_type == "Add" and removed.output[0] in node.input
+ )
+ bias_name = next(name for name in add.input if name in values)
+ bias = values[bias_name].reshape(-1)
+ assert bias.size == 18385
+ extended = np.pad(bias, (0, 18432 - bias.size), constant_values=-np.inf)
+ model.graph.initializer.append(
+ numpy_helper.from_array(extended, "vocabulary_bias_padded")
+ )
+ add.input[:] = ["modelopt_padded_projection", "vocabulary_bias_padded"]
+ model.graph.node.remove(removed)
+ for value in model.graph.initializer:
+ if value.name == "shared_positions":
+ value.CopyFrom(
+ numpy_helper.from_array(
+ np.arange(18432, dtype=np.float32).reshape(1, 1, -1),
+ "shared_positions",
+ )
+ )
+ del model.graph.value_info[:]
+ model = onnx.shape_inference.infer_shapes(model, data_prop=True)
+ onnx.checker.check_model(model)
+ return model
+
+
+def align_attention(model):
+ model = copy.deepcopy(model)
+ sizes = dimensions(model)
+ nodes = []
+
+ def constant(name, values):
+ model.graph.initializer.append(
+ numpy_helper.from_array(np.asarray(values, np.int64), name)
+ )
+ return name
+
+ def pad(name, tensor, axis):
+ pads = [0] * 8
+ pads[4 + axis] = 1
+ nodes.append(
+ helper.make_node(
+ "Pad", [tensor, constant(name + "_pads", pads)], [name], name=name
+ )
+ )
+ return name
+
+ for original in model.graph.node:
+ node = copy.deepcopy(original)
+ if node.op_type == "MatMul":
+ a = sizes.get(node.input[0], [])
+ b = sizes.get(node.input[1], [])
+ if len(a) == 4 and a[-1] == 15 and len(b) == 4 and b[-2] == 15:
+ node.input[0] = pad(node.output[0] + "_q16", node.input[0], 3)
+ node.input[1] = pad(node.output[0] + "_k16", node.input[1], 2)
+ elif len(a) == 4 and len(b) == 4 and a[-1] == b[-2] and b[-1] == 15:
+ node.input[1] = pad(node.output[0] + "_v16", node.input[1], 3)
+ output = node.output[0]
+ node.output[0] = output + "_16"
+ nodes.append(node)
+ nodes.append(
+ helper.make_node(
+ "Slice",
+ [
+ node.output[0],
+ constant(output + "_start", [0]),
+ constant(output + "_end", [15]),
+ constant(output + "_axis", [3]),
+ ],
+ [output],
+ name=output + "_trim",
+ )
+ )
+ continue
+ nodes.append(node)
+ del model.graph.node[:]
+ model.graph.node.extend(nodes)
+ del model.graph.value_info[:]
+ model = onnx.shape_inference.infer_shapes(model, data_prop=True)
+ onnx.checker.check_model(model, full_check=True)
+ return model
+
+
+def with_fixed_paths(source_model, stage, shapes):
+ maximum = [1, 3, 960, 960] if stage == "det" else [1, 3, 48, 7168]
+ output_shape = [1, 1, 960, 960] if stage == "det" else [1, 896, 2]
+ outer_inputs = [helper.make_tensor_value_info("x", onnx.TensorProto.FLOAT, maximum)]
+ weights = {}
+ branches = []
+ metadata = []
+ for index, shape in enumerate(shapes):
+ label = f"p{index}"
+ if stage == "det":
+ model, meta = detector_model(source_model, shape)
+ else:
+ model, meta = recognizer_model(source_model, shape[-1], shape[-1] // 336)
+ model = align_attention(pad_vocabulary(model))
+ mapping = {}
+ local_weights = set()
+ tensors = [
+ (n.output[0], a.t)
+ for n in model.graph.node
+ if n.op_type == "Constant"
+ for a in n.attribute
+ if a.name == "value" and a.t.data_type == onnx.TensorProto.FLOAT
+ ]
+ tensors.extend(
+ (t.name, t)
+ for t in model.graph.initializer
+ if t.data_type == onnx.TensorProto.FLOAT
+ )
+ for original_name, original in tensors:
+ tensor = copy.deepcopy(original)
+ tensor.name = ""
+ name = "w" + hashlib.sha256(tensor.SerializeToString()).hexdigest()
+ tensor.name = name
+ weights[name] = tensor
+ mapping[original_name] = name
+ local_weights.add(name)
+
+ def rename(name, mapping=mapping, label=label):
+ return mapping.get(name, label + "_" + name) if name else name
+
+ nodes = [
+ helper.make_node(name, [], [name], domain=DOMAIN, name=label + "_" + name)
+ for name in sorted(local_weights)
+ ]
+ initializers = []
+ for name, values in [
+ ("starts", [0, 0]),
+ ("ends", shape[-2:]),
+ ("axes", [2, 3]),
+ ]:
+ initializers.append(
+ numpy_helper.from_array(
+ np.asarray(values, np.int64), label + "_" + name
+ )
+ )
+ nodes.append(
+ helper.make_node(
+ "Slice",
+ ["x", label + "_starts", label + "_ends", label + "_axes"],
+ [rename("x")],
+ name=label + "_slice",
+ )
+ )
+ for value in list(model.graph.input)[1:]:
+ item = copy.deepcopy(value)
+ item.name = rename(value.name)
+ outer_inputs.append(item)
+ for tensor in model.graph.initializer:
+ if tensor.name not in mapping:
+ item = copy.deepcopy(tensor)
+ item.name = rename(item.name)
+ initializers.append(item)
+ for ni, node in enumerate(model.graph.node):
+ if node.output[0] in mapping:
+ continue
+ item = copy.deepcopy(node)
+ item.name = label + "_" + str(ni) + "_" + node.name
+ for i, name in enumerate(item.input):
+ item.input[i] = rename(name)
+ for i, name in enumerate(item.output):
+ item.output[i] = rename(name)
+ nodes.append(item)
+ output = rename(model.graph.output[0].name)
+ original_output = (
+ [1, 1, *shape[-2:]] if stage == "det" else [1, shape[-1] // 8, 2]
+ )
+ if original_output != output_shape:
+ padding = [0] * len(output_shape) + [
+ a - b for a, b in zip(output_shape, original_output)
+ ]
+ initializers.append(
+ numpy_helper.from_array(np.asarray(padding, np.int64), label + "_pads")
+ )
+ nodes.append(
+ helper.make_node(
+ "Pad",
+ [output, label + "_pads"],
+ [label + "_output"],
+ name=label + "_pad",
+ )
+ )
+ output = label + "_output"
+ graph = helper.make_graph(
+ nodes,
+ label,
+ [],
+ [
+ helper.make_tensor_value_info(
+ output, onnx.TensorProto.FLOAT, output_shape
+ )
+ ],
+ initializer=initializers,
+ )
+ for value in [*model.graph.input, *model.graph.value_info]:
+ item = copy.deepcopy(value)
+ item.name = rename(item.name)
+ graph.value_info.append(item)
+ branches.append(graph)
+ metadata.append({"shape": shape, "auxiliary": meta})
+ branch = branches[-1]
+ for i in range(len(branches) - 2, -1, -1):
+ name = f"path_{i}"
+ outer_inputs.append(
+ helper.make_tensor_value_info(name, onnx.TensorProto.BOOL, [])
+ )
+ node = helper.make_node(
+ "If",
+ [name],
+ [f"selected_{i}"],
+ name=f"select_{i}",
+ then_branch=branches[i],
+ else_branch=branch,
+ )
+ branch = helper.make_graph(
+ [node],
+ f"selection_{i}",
+ [],
+ [
+ helper.make_tensor_value_info(
+ f"selected_{i}", onnx.TensorProto.FLOAT, output_shape
+ )
+ ],
+ )
+ graph = helper.make_graph(
+ branch.node, f"{stage}_static_paths", outer_inputs, branch.output
+ )
+ model = helper.make_model(
+ graph,
+ opset_imports=[helper.make_opsetid("", 17), helper.make_opsetid(DOMAIN, 1)],
+ ir_version=10,
+ )
+ for name, tensor in weights.items():
+ model.functions.append(
+ helper.make_function(
+ DOMAIN,
+ name,
+ [],
+ ["value"],
+ [helper.make_node("Constant", [], ["value"], value=tensor)],
+ [helper.make_opsetid("", 17)],
+ )
+ )
+ onnx.checker.check_model(model, full_check=True)
+ return model, metadata
+
+
+def fixed_models(models):
+ detector, _ = with_fixed_paths(
+ models["det.onnx"],
+ "det",
+ [
+ [1, 3, 960, 480],
+ [1, 3, 480, 960],
+ [1, 3, 960, 704],
+ [1, 3, 704, 960],
+ [1, 3, 960, 960],
+ ],
+ )
+ recognizer, _ = with_fixed_paths(
+ models["rec.onnx"], "rec", [[1, 3, 48, 2048], [1, 3, 48, 7168]]
+ )
+ classifier = copy.deepcopy(models["cls.onnx"])
+ classifier.graph.input[0].type.tensor_type.shape.CopyFrom(
+ helper.make_tensor_value_info(
+ "x", onnx.TensorProto.FLOAT, [6, 3, 48, 192]
+ ).type.tensor_type.shape
+ )
+ del classifier.graph.value_info[:]
+ for value in classifier.graph.output:
+ for dim in value.type.tensor_type.shape.dim:
+ dim.ClearField("dim_param")
+ dim.ClearField("dim_value")
+ classifier = onnx.shape_inference.infer_shapes(classifier)
+ return {
+ "det_fixed_v1.onnx": detector,
+ "cls_fixed_v1.onnx": classifier,
+ "rec_fixed_v1.onnx": recognizer,
+ }
diff --git a/infra/ml/playground/optimizations/optimize_ocr_models.py b/infra/ml/playground/optimizations/optimize_ocr_models.py
index 72284d76cd..e8c7128e39 100644
--- a/infra/ml/playground/optimizations/optimize_ocr_models.py
+++ b/infra/ml/playground/optimizations/optimize_ocr_models.py
@@ -3,10 +3,11 @@ import copy
import hashlib
import json
from pathlib import Path
-from urllib.request import urlopen
+from urllib.request import Request, urlopen
import numpy as np
import onnx
+from ocr_fixed_inputs import fixed_models
from onnx import helper, numpy_helper
SOURCE_BASE_URL = "https://models.ente.com/PP-OCRv5"
@@ -17,9 +18,9 @@ SOURCE_HASHES = {
"rec.onnx": "bf66820f48fa99f779974c4df78e5274a9d8e0458c4137e8c5357e40e2c3faf2",
}
OUTPUT_HASHES = {
- "det.onnx": "3e658f85236f1984e186070048d7cdc99ddbb84e12c91a6fd80922b85bd9487a",
- "cls.onnx": "8675903197a1bab7060e830e9bcc63d29c5cf7d372735338bd9501f829e94ed1",
- "rec.onnx": "667691bb173a94106b595aa87c7d176b53b68cf831b6ffa09569018754265e57",
+ "det_fixed_v1.onnx": "f655f119225b579fa8c3cbf64f6bb7cf56a26c9dc211706a25234d70a543ec8c",
+ "cls_fixed_v1.onnx": "378d52a73263828d08d4dda37f1d0aac2e36cbd486fdc6351bf309d515610654",
+ "rec_fixed_v1.onnx": "6dda4c0891af5a70c5b0f618b62588df7f3140616d1c560be5ec6496747b54fd",
}
@@ -462,7 +463,11 @@ def load_sources(directory):
for name, expected in SOURCE_HASHES.items():
path = directory / name
if not path.exists():
- with urlopen(f"{SOURCE_BASE_URL}/{name}", timeout=120) as response:
+ request = Request(
+ f"{SOURCE_BASE_URL}/{name}",
+ headers={"User-Agent": "ente-ml-model-optimizer"},
+ )
+ with urlopen(request, timeout=120) as response:
data = response.read()
if hashlib.sha256(data).hexdigest() != expected:
raise ValueError(f"Unexpected SHA-256 for downloaded {name}")
@@ -505,10 +510,18 @@ def main():
args = parser.parse_args()
if onnx.__version__ != "1.22.0":
raise ValueError("Reproducible model output requires onnx==1.22.0")
+ if np.__version__ != "2.5.3":
+ raise ValueError("Reproducible model output requires numpy==2.5.3")
sources = load_sources(args.source_dir)
records = []
- for name, model in optimize(sources).items():
- onnx.checker.check_model(model)
+ for name, model in fixed_models(optimize(sources)).items():
+ onnx.checker.check_model(model, full_check=True)
+ for value in model.graph.input:
+ if any(
+ not d.HasField("dim_value") or d.dim_value <= 0
+ for d in value.type.tensor_type.shape.dim
+ ):
+ raise ValueError(f"Non-static input {value.name} in {name}")
data = model.SerializeToString()
digest = hashlib.sha256(data).hexdigest()
if digest != OUTPUT_HASHES[name]:
@@ -523,15 +536,24 @@ def main():
"file": name,
"sha256": digest,
"bytes": len(data),
- "operators": sorted({node.op_type for node in model.graph.node}),
- "output": "FP32 [N,T,2]: first winning index, original winning probability"
- if name == "rec.onnx"
+ "inputs": {
+ value.name: [d.dim_value for d in value.type.tensor_type.shape.dim]
+ for value in model.graph.input
+ },
+ "output": "FP32 [1,896,2]: packed first winning index and probability"
+ if name == "rec_fixed_v1.onnx"
else "unchanged",
}
)
+ if sum(record["bytes"] for record in records) > 25_000_000:
+ raise ValueError("The combined OCR models exceed 25 MB")
dictionary = args.source_dir / "ppocrv5_dict.txt"
(args.output_dir / dictionary.name).write_bytes(dictionary.read_bytes())
metadata = {
+ "format": "ente-ocr-fixed-v1",
+ "requires_context_adapter": True,
+ "detector_paths": [[960, 480], [480, 960], [960, 704], [704, 960], [960, 960]],
+ "recognizer_widths": [2048, 7168],
"source_base_url": SOURCE_BASE_URL,
"source_sha256": SOURCE_HASHES,
"onnx": onnx.__version__,
diff --git a/mobile/apps/photos/lib/services/machine_learning/ocr/rust_ocr_backend.dart b/mobile/apps/photos/lib/services/machine_learning/ocr/rust_ocr_backend.dart
index dfcde2461b..32cf77a8a6 100644
--- a/mobile/apps/photos/lib/services/machine_learning/ocr/rust_ocr_backend.dart
+++ b/mobile/apps/photos/lib/services/machine_learning/ocr/rust_ocr_backend.dart
@@ -12,7 +12,7 @@ import "package:synchronized/synchronized.dart";
class RustOcrBackend implements OcrBackend {
static final _logger = Logger("RustOcrBackend");
- static const _modelVersion = "pp-ocrv5-gpu-v1";
+ static const _modelVersion = "pp-ocrv5-fixed-v1";
final _engineLock = Lock();
OcrEngine? _engine;
diff --git a/rust/Cargo.lock b/rust/Cargo.lock
index 275789937b..3161eda8aa 100644
--- a/rust/Cargo.lock
+++ b/rust/Cargo.lock
@@ -2702,6 +2702,7 @@ dependencies = [
"tokio",
"url",
"usearch",
+ "wide",
]
[[package]]
diff --git a/rust/crates/ml/Cargo.toml b/rust/crates/ml/Cargo.toml
index f4310e2093..d220eabf33 100644
--- a/rust/crates/ml/Cargo.toml
+++ b/rust/crates/ml/Cargo.toml
@@ -37,6 +37,7 @@ ort = { workspace = true, features = [
] }
[target.'cfg(target_os = "android")'.dependencies]
+wide = "0.7.33"
ash = { version = "0.38.0", default-features = false, features = ["loaded"] }
ort = { workspace = true, features = [
"ndarray",
@@ -78,6 +79,7 @@ ort = { workspace = true, features = [
] }
[dev-dependencies]
+wide = "0.7.33"
anyhow = "1.0.0"
flate2.workspace = true
serde.workspace = true
diff --git a/rust/crates/ml/src/cv/mod.rs b/rust/crates/ml/src/cv/mod.rs
index de0adc365e..d6a93e80cb 100644
--- a/rust/crates/ml/src/cv/mod.rs
+++ b/rust/crates/ml/src/cv/mod.rs
@@ -13,6 +13,8 @@ mod stats;
mod structuring;
mod transform;
mod warp;
+#[cfg(any(target_os = "android", test))]
+mod warp_rgb;
use rayon::prelude::*;
diff --git a/rust/crates/ml/src/cv/warp.rs b/rust/crates/ml/src/cv/warp.rs
index 73cfd778c2..d6f670b839 100644
--- a/rust/crates/ml/src/cv/warp.rs
+++ b/rust/crates/ml/src/cv/warp.rs
@@ -1,5 +1,9 @@
-use image::{Rgb, RgbImage};
-use imageproc::geometric_transformations::{Interpolation, Projection, warp_into};
+#[cfg(not(target_os = "android"))]
+use image::Rgb;
+use image::RgbImage;
+use imageproc::geometric_transformations::Projection;
+#[cfg(not(target_os = "android"))]
+use imageproc::geometric_transformations::{Interpolation, warp_into};
use crate::cv::OpResult;
use crate::cv::image::ImageU8;
@@ -42,13 +46,19 @@ pub(crate) fn warp_rgb_perspective(
let narrow = |c: [(f64, f64); 4]| c.map(|(x, y)| (x as f32, y as f32));
let projection = Projection::from_control_points(narrow(src_corners), narrow(dst_corners))
.ok_or_else(|| "warp_perspective: the corner pairs are degenerate".to_string())?;
- let mut out = RgbImage::new(width as u32, height as u32);
- warp_into(
- source,
- &projection,
- Interpolation::Bilinear,
- Rgb([0, 0, 0]),
- &mut out,
- );
+ #[cfg(target_os = "android")]
+ let out = super::warp_rgb::warp(source, projection, width as u32, height as u32);
+ #[cfg(not(target_os = "android"))]
+ let out = {
+ let mut out = RgbImage::new(width as u32, height as u32);
+ warp_into(
+ source,
+ &projection,
+ Interpolation::Bilinear,
+ Rgb([0, 0, 0]),
+ &mut out,
+ );
+ out
+ };
ImageU8::new(width, height, 3, out.into_raw())
}
diff --git a/rust/crates/ml/src/cv/warp_rgb.rs b/rust/crates/ml/src/cv/warp_rgb.rs
new file mode 100644
index 0000000000..bcadf5d635
--- /dev/null
+++ b/rust/crates/ml/src/cv/warp_rgb.rs
@@ -0,0 +1,98 @@
+use image::RgbImage;
+use imageproc::geometric_transformations::Projection;
+use wide::f32x4;
+
+pub(super) fn warp(source: &RgbImage, projection: Projection, width: u32, height: u32) -> RgbImage {
+ let inverse = projection.invert();
+ let mut output = RgbImage::new(width, height);
+ let pixels = source.as_raw();
+ let stride = source.width() as usize * 3;
+ let load = |offset: usize| {
+ f32x4::from([
+ f32::from(pixels[offset]),
+ f32::from(pixels[offset + 1]),
+ f32::from(pixels[offset + 2]),
+ 0.0,
+ ])
+ };
+ for (y, row) in output
+ .as_mut()
+ .chunks_exact_mut(width as usize * 3)
+ .enumerate()
+ {
+ for (x, pixel) in row.as_chunks_mut::<3>().0.iter_mut().enumerate() {
+ let (px, py) = inverse * (x as f32, y as f32);
+ let left = px.floor();
+ let top = py.floor();
+ if !(left >= 0.0
+ && left + 1.0 < source.width() as f32
+ && top >= 0.0
+ && top + 1.0 < source.height() as f32)
+ {
+ continue;
+ }
+ let right_weight = px - left;
+ let bottom_weight = py - top;
+ let offset = top as usize * stride + left as usize * 3;
+ let right = f32x4::splat(right_weight);
+ let left = f32x4::splat(1.0 - right_weight);
+ let upper =
+ f32x4::from_i32x4((left * load(offset) + right * load(offset + 3)).trunc_int());
+ let lower = f32x4::from_i32x4(
+ (left * load(offset + stride) + right * load(offset + stride + 3)).trunc_int(),
+ );
+ let blended = (f32x4::splat(1.0 - bottom_weight) * upper
+ + f32x4::splat(bottom_weight) * lower)
+ .trunc_int()
+ .to_array();
+ for channel in 0..3 {
+ pixel[channel] = blended[channel] as u8;
+ }
+ }
+ }
+ output
+}
+
+#[cfg(test)]
+mod tests {
+ use image::Rgb;
+ use imageproc::geometric_transformations::{Interpolation, warp_into};
+
+ use super::*;
+
+ #[test]
+ fn matches_reference_pixels_for_projective_and_border_cases() {
+ let source = RgbImage::from_fn(83, 71, |x, y| {
+ Rgb([
+ (x * 17 + y * 3) as u8,
+ (x * 11 + y * 31) as u8,
+ (x * 7 + y * 19) as u8,
+ ])
+ });
+ for index in 0..200 {
+ let shift = index as f32 * 0.173 - 13.0;
+ let projection = Projection::from_matrix([
+ 0.78,
+ 0.13,
+ shift,
+ -0.07,
+ 1.14,
+ shift / 2.0,
+ index as f32 * 0.00001,
+ -0.0004,
+ 1.0,
+ ])
+ .expect("valid projection");
+ let actual = warp(&source, projection, 113, 97);
+ let mut expected = RgbImage::new(113, 97);
+ warp_into(
+ &source,
+ &projection,
+ Interpolation::Bilinear,
+ Rgb([0, 0, 0]),
+ &mut expected,
+ );
+ assert_eq!(actual, expected, "case {index}");
+ }
+ }
+}
diff --git a/rust/crates/ml/src/ocr/assets.rs b/rust/crates/ml/src/ocr/assets.rs
index 5698d696a7..d03b42acc3 100644
--- a/rust/crates/ml/src/ocr/assets.rs
+++ b/rust/crates/ml/src/ocr/assets.rs
@@ -17,27 +17,27 @@ struct OcrModelFile {
}
const DETECTION: OcrModelFile = OcrModelFile {
- key: "ppocrv5_det_gpu_v1",
+ key: "ppocrv5_det_fixed_v1",
name: "det.onnx",
- url: "https://entedevassets.priem.dev/det_opt.onnx",
- size: 4_750_896,
- sha256: "3e658f85236f1984e186070048d7cdc99ddbb84e12c91a6fd80922b85bd9487a",
+ url: "https://entedevassets.priem.dev/det_fixed_v1.onnx",
+ size: 5_548_458,
+ sha256: "f655f119225b579fa8c3cbf64f6bb7cf56a26c9dc211706a25234d70a543ec8c",
};
const CLASSIFICATION: OcrModelFile = OcrModelFile {
- key: "ppocrv5_cls_gpu_v1",
+ key: "ppocrv5_cls_fixed_v1",
name: "cls.onnx",
- url: "https://entedevassets.priem.dev/cls_opt.onnx",
- size: 574_875,
- sha256: "8675903197a1bab7060e830e9bcc63d29c5cf7d372735338bd9501f829e94ed1",
+ url: "https://entedevassets.priem.dev/cls_fixed_v1.onnx",
+ size: 590_475,
+ sha256: "378d52a73263828d08d4dda37f1d0aac2e36cbd486fdc6351bf309d515610654",
};
const RECOGNITION: OcrModelFile = OcrModelFile {
- key: "ppocrv5_rec_gpu_v1",
+ key: "ppocrv5_rec_fixed_v1",
name: "rec.onnx",
- url: "https://entedevassets.priem.dev/rec_opt.onnx",
- size: 16_686_404,
- sha256: "667691bb173a94106b595aa87c7d176b53b68cf831b6ffa09569018754265e57",
+ url: "https://entedevassets.priem.dev/rec_fixed_v1.onnx",
+ size: 17_031_601,
+ sha256: "6dda4c0891af5a70c5b0f618b62588df7f3140616d1c560be5ec6496747b54fd",
};
const DICTIONARY: OcrModelFile = OcrModelFile {
diff --git a/rust/crates/ml/src/ocr/classify.rs b/rust/crates/ml/src/ocr/classify.rs
index b68a3db4b4..c67372b6a0 100644
--- a/rust/crates/ml/src/ocr/classify.rs
+++ b/rust/crates/ml/src/ocr/classify.rs
@@ -47,14 +47,19 @@ impl AngleClassifier {
}
fn score_batch(&self, batch: &[ImageU8]) -> MlResult<Vec<AngleScores>> {
- let input = PreparedF32Input::new(prepare_crops(|| batch_tensor(batch))?);
- let count = batch.len() as i64;
+ let mut values = prepare_crops(|| batch_tensor(batch))?;
+ values.resize(
+ BATCH_SIZE * 3 * INPUT_HEIGHT as usize * INPUT_WIDTH as usize,
+ 0.0,
+ );
+ let input = PreparedF32Input::new(values);
+ let count = BATCH_SIZE as i64;
let expected_shape = [count, CLASS_COUNT as i64];
let mut session = self.session.lock().unwrap_or_else(PoisonError::into_inner);
let input_shape = [count, 3, i64::from(INPUT_HEIGHT), i64::from(INPUT_WIDTH)];
- let (values, _usage) = session.run(input_shape, |session| {
+ let (values, _usage) = session.run(|session| {
let (shape, values) = run_f32(session, &input, input_shape)?;
- if shape != expected_shape {
+ if shape != expected_shape || values.len() != BATCH_SIZE * CLASS_COUNT {
return Err(SessionRunError::from(MlError::CorruptModel(format!(
"angle classifier produced output shape {shape:?}, expected {expected_shape:?}"
))));
@@ -65,6 +70,7 @@ impl AngleClassifier {
.as_chunks::<CLASS_COUNT>()
.0
.iter()
+ .take(batch.len())
.map(|&[p0, p180]| AngleScores { p0, p180 })
.collect())
}
diff --git a/rust/crates/ml/src/ocr/context.rs b/rust/crates/ml/src/ocr/context.rs
new file mode 100644
index 0000000000..957fbc95c5
--- /dev/null
+++ b/rust/crates/ml/src/ocr/context.rs
@@ -0,0 +1,362 @@
+use ort::session::{Session, SessionInputValue};
+use ort::value::{Tensor, TensorRef};
+
+use crate::onnx::{SessionRunError, SessionRunResult};
+
+#[derive(Clone)]
+pub(super) struct SharedValues(std::sync::Arc<Vec<f32>>);
+
+impl std::ops::Deref for SharedValues {
+ type Target = Vec<f32>;
+ fn deref(&self) -> &Self::Target {
+ &self.0
+ }
+}
+
+impl std::ops::DerefMut for SharedValues {
+ fn deref_mut(&mut self) -> &mut Self::Target {
+ std::sync::Arc::make_mut(&mut self.0)
+ }
+}
+
+#[derive(Clone)]
+pub(super) struct Input {
+ pub name: String,
+ pub shape: Vec<i64>,
+ pub values: SharedValues,
+}
+
+impl Input {
+ pub fn zeros(name: impl Into<String>, shape: &[usize]) -> Self {
+ Self {
+ name: name.into(),
+ shape: shape.iter().map(|&v| v as i64).collect(),
+ values: SharedValues(std::sync::Arc::new(vec![0.0; shape.iter().product()])),
+ }
+ }
+}
+
+pub(super) fn infer(
+ session: &mut Session,
+ inputs: &[Input],
+) -> SessionRunResult<(Vec<i64>, Vec<f32>)> {
+ let tensors: Result<Vec<_>, _> = inputs
+ .iter()
+ .map(|input| {
+ if input.name.starts_with("path_") {
+ return Tensor::from_array((input.shape.clone(), vec![input.values[0] != 0.0]))
+ .map(|tensor| (input.name.as_str(), SessionInputValue::from(tensor)));
+ }
+ TensorRef::from_array_view((input.shape.as_slice(), input.values.as_slice()))
+ .map(|tensor| (input.name.as_str(), SessionInputValue::from(tensor)))
+ })
+ .collect();
+ let tensors = tensors?;
+ let result = session.run(tensors);
+ let outputs = result.map_err(SessionRunError::from_inference_error)?;
+ let (shape, values) = outputs[0].try_extract_tensor::<f32>()?;
+ Ok((shape.to_vec(), values.to_vec()))
+}
+
+pub(super) fn detector(raw: &[f32], height: usize, width: usize) -> Vec<Input> {
+ let side = 960;
+ let mut image = Input::zeros("x", &[1, 3, side, side]);
+ for channel in 0..3 {
+ for y in 0..height {
+ let source = (channel * height + y) * width;
+ let target = (channel * side + y) * side;
+ image.values[target..target + width].copy_from_slice(&raw[source..source + width]);
+ }
+ }
+ detector_paths(image, height, width)
+}
+
+pub(super) struct Line {
+ pub values: Vec<f32>,
+ pub width: usize,
+ pub content_width: i32,
+}
+
+pub(super) fn packed(lines: &[&Line], width: usize, slots: usize) -> (Vec<Input>, Vec<usize>) {
+ let mut image = Input::zeros("x", &[1, 3, 48, width]);
+ let mut positions = Vec::with_capacity(lines.len());
+ let mut cursor = 0;
+ for line in lines {
+ for row in 0..3 * 48 {
+ let source = row * line.width;
+ let target = row * width + cursor;
+ image.values[target..target + line.width]
+ .copy_from_slice(&line.values[source..source + line.width]);
+ }
+ positions.push(cursor);
+ cursor += line.width.div_ceil(8) * 8 + 16;
+ }
+ let mut inputs = vec![image];
+ for (height, scale) in [(48, 1), (24, 2), (12, 2), (12, 4), (6, 4), (3, 4), (1, 8)] {
+ let size = width / scale;
+ let mut input = Input::zeros(format!("mask_{height}_{size}"), &[1, 1, 1, size]);
+ for (line, &offset) in lines.iter().zip(&positions) {
+ let count = if scale == 8 {
+ line.width.div_ceil(4) / 2
+ } else {
+ line.width.div_ceil(scale)
+ };
+ input.values[offset / scale..offset / scale + count].fill(1.0);
+ }
+ inputs.push(input);
+ }
+ let size = width / 4;
+ let mut pool = Input::zeros(format!("pool_weights_{size}"), &[1, size, slots]);
+ let mut gate = Input::zeros(format!("gate_weights_{size}"), &[1, slots, size]);
+ for (slot, (line, &offset)) in lines.iter().zip(&positions).enumerate() {
+ let count = line.width.div_ceil(4);
+ for x in offset / 4..offset / 4 + count {
+ pool.values[x * slots + slot] = 1.0 / count as f32;
+ gate.values[slot * size + x] = 1.0;
+ }
+ }
+ inputs.extend([pool, gate]);
+ let steps = width / 8;
+ let mut attention = Input::zeros("attention_bias", &[1, 1, steps, steps]);
+ attention.values.fill(-10000.0);
+ for query in 0..steps {
+ attention.values[query * steps] = 0.0;
+ }
+ for (line, &offset) in lines.iter().zip(&positions) {
+ let count = line.width.div_ceil(4) / 2;
+ let start = offset / 8;
+ for query in start..start + count {
+ let row = &mut attention.values[query * steps..(query + 1) * steps];
+ row.fill(-10000.0);
+ row[start..start + count].fill(0.0);
+ }
+ }
+ inputs.push(attention);
+ (inputs, positions)
+}
+
+fn detector_auxiliary(index: usize, height: usize, width: usize, h: usize, w: usize) -> Vec<Input> {
+ let mut inputs = Vec::new();
+ for scale in [1, 2, 4, 8, 16, 32] {
+ let mh = h / scale;
+ let mw = w / scale;
+ let mut mask = Input::zeros(format!("p{index}_mask_{mh}_{mw}"), &[1, 1, mh, mw]);
+ for y in 0..height / scale {
+ mask.values[y * mw..y * mw + width / scale].fill(1.0);
+ }
+ inputs.push(mask);
+ if scale >= 4 {
+ let mut value = Input::zeros(format!("p{index}_pool_scale_{mh}_{mw}"), &[1, 1, 1, 1]);
+ value.values[0] = if height > 0 && width > 0 {
+ (h * w) as f32 / (height * width) as f32
+ } else {
+ 1.0
+ };
+ inputs.push(value);
+ }
+ }
+ inputs
+}
+
+fn detector_paths(image: Input, height: usize, width: usize) -> Vec<Input> {
+ static UNUSED: std::sync::OnceLock<Vec<Vec<Input>>> = std::sync::OnceLock::new();
+ let shapes = &[(960, 480), (480, 960), (960, 704), (704, 960), (960, 960)];
+ let unused = UNUSED.get_or_init(|| {
+ shapes
+ .iter()
+ .enumerate()
+ .map(|(i, &(h, w))| detector_auxiliary(i, 0, 0, h, w))
+ .collect()
+ });
+ let selected = shapes
+ .iter()
+ .position(|&(h, w)| height <= h && width <= w)
+ .unwrap_or(shapes.len() - 1);
+ let mut inputs = vec![image];
+ for (index, &(h, w)) in shapes.iter().enumerate() {
+ inputs.extend(if index == selected {
+ detector_auxiliary(index, height, width, h, w)
+ } else {
+ unused[index].clone()
+ });
+ if index < shapes.len() - 1 {
+ let mut flag = Input::zeros(format!("path_{index}"), &[]);
+ flag.values[0] = if selected == index { 1.0 } else { 0.0 };
+ inputs.push(flag);
+ }
+ }
+ inputs
+}
+
+pub(super) fn branch_widths() -> &'static [usize] {
+ &[2048, 7168]
+}
+
+pub(super) fn recognizer_paths(mut selected: Vec<Input>, width: usize) -> Vec<Input> {
+ let mut original = selected.remove(0);
+ let mut image = Input::zeros("x", &[1, 3, 48, 7168]);
+ if width == 7168 {
+ std::mem::swap(&mut image.values, &mut original.values);
+ } else {
+ for row in 0..3 * 48 {
+ image.values[row * 7168..row * 7168 + width]
+ .copy_from_slice(&original.values[row * width..(row + 1) * width]);
+ }
+ }
+ static UNUSED: std::sync::OnceLock<Vec<Vec<Input>>> = std::sync::OnceLock::new();
+ let widths = branch_widths();
+ let unused = UNUSED.get_or_init(|| {
+ widths
+ .iter()
+ .map(|&w| {
+ let (mut values, _) = packed(&[], w, w / 336);
+ values.remove(0);
+ values
+ })
+ .collect()
+ });
+ let mut inputs = vec![image];
+ for (index, &branch_width) in widths.iter().enumerate() {
+ let mut values = if branch_width == width {
+ std::mem::take(&mut selected)
+ } else {
+ unused[index].clone()
+ };
+ for input in &mut values {
+ input.name = format!("p{index}_{}", input.name);
+ }
+ inputs.extend(values);
+ }
+ for (index, &w) in widths.iter().enumerate().take(widths.len() - 1) {
+ let mut flag = Input::zeros(format!("path_{index}"), &[]);
+ flag.values[0] = if width == w { 1.0 } else { 0.0 };
+ inputs.push(flag);
+ }
+ inputs
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ fn input<'a>(inputs: &'a [Input], name: &str) -> &'a Input {
+ inputs.iter().find(|input| input.name == name).unwrap()
+ }
+
+ #[test]
+ fn detector_masks_preserve_area_at_every_scale_and_choose_the_smallest_path() {
+ for (height, width, path) in [
+ (32, 32, 0),
+ (960, 480, 0),
+ (448, 960, 1),
+ (960, 672, 2),
+ (704, 960, 3),
+ (960, 736, 4),
+ ] {
+ let raw = vec![0.5; 3 * height * width];
+ let inputs = detector(&raw, height, width);
+ let image = input(&inputs, "x");
+ assert_eq!(image.shape, [1, 3, 960, 960]);
+ for channel in 0..3 {
+ for row in 0..960 {
+ let pixels =
+ &image.values[(channel * 960 + row) * 960..(channel * 960 + row + 1) * 960];
+ if row < height {
+ assert!(pixels[..width].iter().all(|&v| v == 0.5));
+ assert!(pixels[width..].iter().all(|&v| v == 0.0));
+ } else {
+ assert!(pixels.iter().all(|&v| v == 0.0));
+ }
+ }
+ }
+ for index in 0..4 {
+ assert_eq!(
+ input(&inputs, &format!("path_{index}")).values[0],
+ f32::from(index == path)
+ );
+ }
+ let prefix = format!("p{path}_mask_");
+ for mask in inputs
+ .iter()
+ .filter(|input| input.name.starts_with(&prefix))
+ {
+ let scale = match path {
+ 1 => 480,
+ 3 => 704,
+ _ => 960,
+ } / mask.shape[2] as usize;
+ assert_eq!(
+ mask.values.iter().filter(|&&v| v == 1.0).count(),
+ height * width / (scale * scale)
+ );
+ let scale_name = mask.name.replace("mask", "pool_scale");
+ if let Some(pool_scale) = inputs.iter().find(|input| input.name == scale_name) {
+ let active = height * width / (scale * scale);
+ assert!(
+ (pool_scale.values[0] * active as f32 / mask.values.len() as f32 - 1.0)
+ .abs()
+ < 1e-6
+ );
+ }
+ }
+ }
+ }
+
+ #[test]
+ fn packing_isolates_odd_width_lines_in_pooling_and_attention() {
+ let lines: Vec<_> = [321, 327]
+ .into_iter()
+ .map(|width| Line {
+ values: vec![0.25; 3 * 48 * width],
+ width,
+ content_width: width as i32,
+ })
+ .collect();
+ let (inputs, offsets) = packed(&[&lines[0], &lines[1]], 2048, 6);
+ assert_eq!(offsets, [0, 344]);
+ let mask = input(&inputs, "mask_1_256");
+ assert_eq!(mask.values.iter().filter(|&&v| v == 1.0).count(), 81);
+ let attention = input(&inputs, "attention_bias");
+ for (line, &offset) in lines.iter().zip(&offsets) {
+ let start = offset / 8;
+ let count = line.width.div_ceil(4) / 2;
+ for query in start..start + count {
+ let row = &attention.values[query * 256..(query + 1) * 256];
+ assert!(row[start..start + count].iter().all(|&v| v == 0.0));
+ assert!(
+ row[..start]
+ .iter()
+ .chain(&row[start + count..])
+ .all(|&v| v == -10000.0)
+ );
+ }
+ }
+ let pool = input(&inputs, "pool_weights_512");
+ for slot in 0..2 {
+ let sum: f32 = pool.values.iter().skip(slot).step_by(6).sum();
+ assert!((sum - 1.0).abs() < 1e-6);
+ }
+ let lifted = recognizer_paths(inputs, 2048);
+ assert_eq!(input(&lifted, "x").shape, [1, 3, 48, 7168]);
+ assert_eq!(input(&lifted, "path_0").values[0], 1.0);
+ assert_eq!(input(&lifted, "p1_attention_bias").shape, [1, 1, 896, 896]);
+ }
+
+ #[test]
+ fn packing_preserves_the_last_step_at_the_maximum_width() {
+ for width in [7167, 7168] {
+ let line = Line {
+ values: vec![0.5; 3 * 48 * width],
+ width,
+ content_width: width as i32,
+ };
+ let (inputs, offsets) = packed(&[&line], 7168, 21);
+ assert_eq!(offsets, [0]);
+ assert_eq!(input(&inputs, "mask_1_896").values[895], 1.0);
+ assert_eq!(
+ input(&recognizer_paths(inputs, 7168), "path_0").values[0],
+ 0.0
+ );
+ }
+ }
+}
diff --git a/rust/crates/ml/src/ocr/detect.rs b/rust/crates/ml/src/ocr/detect.rs
index cf440933ea..0731629ca2 100644
--- a/rust/crates/ml/src/ocr/detect.rs
+++ b/rust/crates/ml/src/ocr/detect.rs
@@ -9,7 +9,7 @@ use super::tensor::{BgrNormalization, write_bgr_planes};
use crate::cv;
use crate::cv::image::{Contour, ImageU8};
use crate::error::{MlError, MlResult};
-use crate::onnx::{PreparedF32Input, SessionRunError, run_f32};
+use crate::onnx::SessionRunError;
const MAX_INPUT_SIDE: i32 = 960;
const INPUT_STRIDE: i32 = 32;
@@ -54,7 +54,7 @@ impl TextDetector {
let (input_width, input_height) = detector_input_size(working.width, working.height);
let resized = cv::resize_u8(working, input_width, input_height, cv::Interp::Bilinear)
.map_err(MlError::Preprocess)?;
- let input = PreparedF32Input::new(normalized_bgr_planes(&resized)?);
+ let input = normalized_bgr_planes(&resized)?;
let values = self.infer(&input, input_width, input_height)?;
let probability_map = ProbabilityMap {
width: input_width as usize,
@@ -75,18 +75,24 @@ impl TextDetector {
})
}
- fn infer(&self, input: &PreparedF32Input, width: i32, height: i32) -> MlResult<Vec<f32>> {
- let expected_shape = [1i64, 1, i64::from(height), i64::from(width)];
+ fn infer(&self, input: &[f32], width: i32, height: i32) -> MlResult<Vec<f32>> {
+ let expected_shape = [1i64, 1, 960, 960];
let mut session = self.session.lock().unwrap_or_else(PoisonError::into_inner);
- let input_shape = [1, 3, i64::from(height), i64::from(width)];
- let (values, _usage) = session.run(input_shape, |session| {
- let (shape, values) = run_f32(session, input, input_shape)?;
- if shape != expected_shape {
+ let inputs = super::context::detector(input, height as usize, width as usize);
+ let (values, _usage) = session.run(|session| {
+ let (shape, values) = super::context::infer(session, &inputs)?;
+ if shape != expected_shape || values.len() != 960 * 960 {
return Err(SessionRunError::from(MlError::CorruptModel(format!(
"text detector produced output shape {shape:?}, expected {expected_shape:?}"
))));
}
- Ok(values)
+ Ok(values
+ .as_chunks::<960>()
+ .0
+ .iter()
+ .take(height as usize)
+ .flat_map(|row| row[..width as usize].iter().copied())
+ .collect())
})?;
Ok(values)
}
diff --git a/rust/crates/ml/src/ocr/mod.rs b/rust/crates/ml/src/ocr/mod.rs
index ddaf78c382..10ab3a4514 100644
--- a/rust/crates/ml/src/ocr/mod.rs
+++ b/rust/crates/ml/src/ocr/mod.rs
@@ -2,6 +2,7 @@ pub mod assets;
mod cancel;
mod characters;
mod classify;
+mod context;
mod crop;
mod detect;
mod dictionary;
diff --git a/rust/crates/ml/src/ocr/recognize.rs b/rust/crates/ml/src/ocr/recognize.rs
index b0dec5eff8..40600dff6f 100644
--- a/rust/crates/ml/src/ocr/recognize.rs
+++ b/rust/crates/ml/src/ocr/recognize.rs
@@ -8,9 +8,7 @@ use super::tensor::{BgrNormalization, prepare_crops, write_bgr_planes};
use crate::cv;
use crate::cv::image::ImageU8;
use crate::error::{MlError, MlResult};
-use crate::onnx::{
- BorrowedFloatTensor, FloatTensorData, PreparedF32Input, with_prepared_float_output,
-};
+use crate::onnx::{FloatTensorData, SessionRunError};
use super::session::{OcrModel, OcrSession};
const REC_VOCABULARY_SIZE: usize = 18385;
@@ -81,37 +79,95 @@ impl TextRecognizer {
return Ok(Vec::new());
}
let dictionary = self.dictionary.entries()?;
- recognize_in_batches(crops, request, |batch, layout| {
- self.infer_batch(batch, layout, dictionary)
- })
- }
-
- fn infer_batch(
- &self,
- batch: &[&ImageU8],
- layout: &BatchLayout,
- dictionary: &[String],
- ) -> MlResult<Vec<Recognition>> {
- let input = PreparedF32Input::new(prepare_crops(|| layout.tensor(batch))?);
- let count = batch.len();
- let input_shape = [
- count as i64,
- 3,
- i64::from(REC_HEIGHT),
- i64::from(layout.target_width),
- ];
- let mut session = self.session.lock().unwrap_or_else(PoisonError::into_inner);
- let (recognized, _usage) = session.run(input_shape, |session| {
- with_prepared_float_output(session, &input, input_shape, |shape, values| match values {
- BorrowedFloatTensor::F32(values) => {
- decode_output(shape, values, layout, dictionary)
+ let mut lines = Vec::new();
+ let mut line_crops = Vec::new();
+ let ordered = recognize_in_batches(crops, request, |batch, layout| {
+ let mut indices = Vec::with_capacity(batch.len());
+ for (&crop, &content_width) in batch.iter().zip(&layout.content_widths) {
+ indices.push(lines.len());
+ line_crops.push(crop);
+ lines.push(super::context::Line {
+ values: Vec::new(),
+ width: layout.target_width as usize,
+ content_width,
+ });
+ }
+ Ok(indices)
+ })?;
+ let mut results = vec![Recognition::default(); lines.len()];
+ let mut pending: Vec<usize> = (0..lines.len()).collect();
+ pending.sort_by_key(|&i| std::cmp::Reverse(lines[i].width));
+ while !pending.is_empty() {
+ request.check()?;
+ let width = choose_packed_width(&pending, &lines);
+ let slots = width / 336;
+ let mut selected = Vec::new();
+ let mut used = 0;
+ pending.retain(|&index| {
+ let size = lines[index].width.div_ceil(8) * 8;
+ if selected.len() < slots && used + size <= width {
+ selected.push(index);
+ used += size + 16;
+ false
+ } else {
+ true
+ }
+ });
+ if selected.is_empty() {
+ return Err(
+ MlError::Preprocess("line exceeds fixed packed canvas".to_owned()).into(),
+ );
+ }
+ prepare_crops(|| {
+ for &index in &selected {
+ let line = &mut lines[index];
+ line.values = BatchLayout {
+ target_width: line.width as i32,
+ content_widths: vec![line.content_width],
+ }
+ .tensor(&[line_crops[index]])?;
}
- BorrowedFloatTensor::F16(values) => {
- decode_output(shape, values, layout, dictionary)
+ Ok(())
+ })?;
+ let batch: Vec<_> = selected.iter().map(|&index| &lines[index]).collect();
+ let (inputs, offsets) = super::context::packed(&batch, width, slots);
+ let inputs = super::context::recognizer_paths(inputs, width);
+ let mut session = self.session.lock().unwrap_or_else(PoisonError::into_inner);
+ let (values, _) = session.run(|session| {
+ let (shape, values) = super::context::infer(session, &inputs)?;
+ if shape != [1, 896, 2] || values.len() != 896 * 2 {
+ return Err(SessionRunError::from(MlError::CorruptModel(format!(
+ "text recognizer produced output shape {shape:?}, expected [1, 896, 2]"
+ ))));
}
+ Ok(values)
+ })?;
+ for (&index, &offset) in selected.iter().zip(&offsets) {
+ let line = &lines[index];
+ let start = offset / 8 * 2;
+ let count = line.width.div_ceil(4) / 2;
+ let layout = BatchLayout {
+ target_width: line.width as i32,
+ content_widths: vec![line.content_width],
+ };
+ let mut decoded = decode_output(
+ &[1, count as i64, 2],
+ &values[start..start + count * 2],
+ &layout,
+ dictionary,
+ )?;
+ results[index] = decoded.remove(0);
+ lines[index].values = Vec::new();
+ }
+ }
+ ordered
+ .into_iter()
+ .map(|index| {
+ results.get(index).cloned().ok_or_else(|| {
+ MlError::Preprocess("packed line index out of range".to_owned()).into()
+ })
})
- })?;
- Ok(recognized)
+ .collect()
}
}
@@ -179,12 +235,12 @@ where
.collect()
}
-fn recognize_in_batches(
- crops: &[&ImageU8],
+fn recognize_in_batches<'a, T>(
+ crops: &[&'a ImageU8],
request: &RequestGuard<'_>,
- mut infer_batch: impl FnMut(&[&ImageU8], &BatchLayout) -> MlResult<Vec<Recognition>>,
-) -> Result<Vec<Recognition>, OcrError> {
- let mut results: Vec<Option<Recognition>> = vec![None; crops.len()];
+ mut infer_batch: impl FnMut(&[&'a ImageU8], &BatchLayout) -> MlResult<Vec<T>>,
+) -> Result<Vec<T>, OcrError> {
+ let mut results: Vec<Option<T>> = std::iter::repeat_with(|| None).take(crops.len()).collect();
for indices in ascending_aspect_order(crops).chunks(REC_BATCH_SIZE) {
request.check()?;
let batch: Vec<&ImageU8> = indices.iter().map(|&index| crops[index]).collect();
@@ -855,7 +911,8 @@ mod tests {
let request = registry.begin(Some("cancelled"));
registry.cancel("cancelled");
- let error = recognize_in_batches(&[&crop], &request, |_, _| unreachable!()).unwrap_err();
+ let error = recognize_in_batches::<Recognition>(&[&crop], &request, |_, _| unreachable!())
+ .unwrap_err();
assert!(matches!(error, OcrError::Cancelled), "{error}");
}
@@ -1152,3 +1209,32 @@ mod tests {
assert!(matches!(error, OcrError::Cancelled));
}
}
+
+fn choose_packed_width(pending: &[usize], lines: &[super::context::Line]) -> usize {
+ let mut best_width = 7168;
+ let mut best_cost = f64::INFINITY;
+ for &width in super::context::branch_widths() {
+ if width < lines[pending[0]].width {
+ continue;
+ }
+ let mut used = 0;
+ let mut payload = 0;
+ let mut count = 0;
+ for &index in pending {
+ let size = lines[index].width.div_ceil(8) * 8;
+ if count < width / 336 && used + size <= width {
+ used += size + 16;
+ payload += lines[index].width;
+ count += 1;
+ }
+ }
+ if payload > 0 {
+ let cost = (width + 128) as f64 / payload as f64;
+ if cost < best_cost {
+ best_cost = cost;
+ best_width = width;
+ }
+ }
+ }
+ best_width
+}
diff --git a/rust/crates/ml/src/ocr/session.rs b/rust/crates/ml/src/ocr/session.rs
index 71c7c39cdc..4f82a1f44f 100644
--- a/rust/crates/ml/src/ocr/session.rs
+++ b/rust/crates/ml/src/ocr/session.rs
@@ -1,6 +1,4 @@
-use std::collections::VecDeque;
-
-use crate::error::{MlError, MlResult};
+use crate::error::MlResult;
use crate::onnx::{
ExecutionMode, GpuOptions, OnnxSession, ProviderUsage, SessionHandle, SessionRunResult,
};
@@ -12,157 +10,33 @@ pub(super) enum OcrModel {
Recognition,
}
-impl OcrModel {
- fn cache_capacity(self) -> usize {
- match self {
- Self::Detection => 2,
- Self::Classification => 6,
- Self::Recognition => 8,
- }
- }
-
- fn namespace(self) -> &'static str {
- match self {
- Self::Detection => "ocr-detection",
- Self::Classification => "ocr-classification",
- Self::Recognition => "ocr-recognition",
- }
- }
-
- fn mode(self) -> ExecutionMode {
- if matches!(self, Self::Classification)
- && !cfg!(any(target_os = "ios", target_os = "macos"))
- {
- ExecutionMode::CpuOnly
- } else {
- ExecutionMode::GpuPreferred
- }
- }
-
- fn gpu_options(self, [count, _, height, width]: [i64; 4]) -> GpuOptions {
- let dimensions = match self {
- Self::Detection => vec![
- ("DynamicDimension.0", count),
- ("DynamicDimension.1", height),
- ("DynamicDimension.2", width),
- ],
- Self::Classification => vec![
- ("p2o.DynamicDimension.0", count),
- ("p2o.DynamicDimension.1", height),
- ("p2o.DynamicDimension.2", width),
- ],
- Self::Recognition => vec![("DynamicDimension.0", count), ("DynamicDimension.1", width)],
- };
- GpuOptions {
- dimensions,
- #[cfg(any(target_os = "android", target_os = "linux", target_os = "windows"))]
- prefer_nhwc: matches!(self, Self::Detection),
- }
- }
-}
-
pub(super) struct OcrSession {
- model_path: String,
- model: OcrModel,
- sessions: VecDeque<([i64; 4], OnnxSession)>,
+ session: OnnxSession,
}
impl OcrSession {
pub(super) fn new(model_path: &str, model: OcrModel) -> Self {
+ let namespace = match model {
+ OcrModel::Detection => "ocr-detection-fixed-v1",
+ OcrModel::Classification => "ocr-classification-fixed-v1",
+ OcrModel::Recognition => "ocr-recognition-fixed-v1",
+ };
+ let options = GpuOptions {
+ subgraphs: !matches!(model, OcrModel::Classification),
+ #[cfg(any(target_os = "android", target_os = "linux", target_os = "windows"))]
+ prefer_nhwc: matches!(model, OcrModel::Detection),
+ };
Self {
- model_path: model_path.to_owned(),
- model,
- sessions: VecDeque::new(),
+ session: OnnxSession::new(model_path, namespace, ExecutionMode::GpuPreferred)
+ .with_gpu_options(options)
+ .with_unvalidated_acceleration(),
}
}
pub(super) fn run<T>(
&mut self,
- shape: [i64; 4],
operation: impl FnMut(&mut SessionHandle) -> SessionRunResult<T>,
) -> MlResult<(T, ProviderUsage)> {
- self.session_for(shape)?.run(operation)
- }
-
- fn session_for(&mut self, shape: [i64; 4]) -> MlResult<&mut OnnxSession> {
- let mode = self.model.mode();
- let key = if mode == ExecutionMode::CpuOnly {
- [0; 4]
- } else {
- shape
- };
- if let Some(index) = self.sessions.iter().position(|(cached, _)| *cached == key) {
- self.sessions.make_contiguous()[..=index].rotate_right(1);
- } else {
- if self.sessions.len() == self.model.cache_capacity() {
- self.sessions.pop_back();
- }
- let [n, c, h, w] = key;
- let namespace = format!("{}-{n}x{c}x{h}x{w}-gpu-f32-v1", self.model.namespace());
- let mut session = OnnxSession::new(&self.model_path, &namespace, mode);
- if mode == ExecutionMode::GpuPreferred {
- session = session
- .with_gpu_options(self.model.gpu_options(shape))
- .with_unvalidated_acceleration();
- }
- self.sessions.push_front((key, session));
- }
- self.sessions
- .front_mut()
- .map(|(_, session)| session)
- .ok_or_else(|| MlError::Runtime("OCR session cache is empty".to_owned()))
- }
-}
-
-#[cfg(test)]
-mod tests {
- use super::*;
-
- #[test]
- fn reuses_shapes_and_evicts_the_least_recently_used_session() {
- let mut cache = OcrSession::new("rec.onnx", OcrModel::Recognition);
- let first = [6, 3, 48, 320];
- cache.session_for(first).unwrap().initialize_load_state();
- for width in 321..328 {
- cache.session_for([6, 3, 48, width]).unwrap();
- }
- assert!(cache.session_for(first).unwrap().has_load_state());
- cache.session_for([1, 3, 48, 320]).unwrap();
- assert_eq!(cache.sessions.len(), OcrModel::Recognition.cache_capacity());
- assert!(cache.session_for(first).unwrap().has_load_state());
- assert!(
- !cache
- .sessions
- .iter()
- .any(|(shape, _)| *shape == [6, 3, 48, 321])
- );
- }
-
- #[test]
- fn models_override_their_actual_spatial_dimensions() {
- assert_eq!(
- OcrModel::Detection.gpu_options([1, 3, 960, 704]).dimensions,
- vec![
- ("DynamicDimension.0", 1),
- ("DynamicDimension.1", 960),
- ("DynamicDimension.2", 704)
- ]
- );
- assert_eq!(
- OcrModel::Recognition
- .gpu_options([3, 3, 48, 641])
- .dimensions,
- vec![("DynamicDimension.0", 3), ("DynamicDimension.1", 641)]
- );
- assert_eq!(
- OcrModel::Classification
- .gpu_options([3, 3, 48, 192])
- .dimensions,
- vec![
- ("p2o.DynamicDimension.0", 3),
- ("p2o.DynamicDimension.1", 48),
- ("p2o.DynamicDimension.2", 192)
- ]
- );
+ self.session.run(operation)
}
}
diff --git a/rust/crates/ml/src/ocr/tensor.rs b/rust/crates/ml/src/ocr/tensor.rs
index 0f8c976d7b..7c8f914480 100644
--- a/rust/crates/ml/src/ocr/tensor.rs
+++ b/rust/crates/ml/src/ocr/tensor.rs
@@ -71,6 +71,9 @@ pub(crate) fn write_bgr_planes(
planes.len()
)));
}
+ let lookup: [[f32; 256]; 3] = std::array::from_fn(|channel| {
+ std::array::from_fn(|value| normalization.apply(channel, value as u8))
+ });
let (blue, rest) = planes.split_at_mut(plane);
let (green, red) = rest.split_at_mut(plane);
let top = (plane_height - height) / 2 * plane_width;
@@ -82,9 +85,9 @@ pub(crate) fn write_bgr_planes(
for (y, row) in rgb.data.chunks_exact(width * 3).enumerate() {
let offset = top + y * plane_width;
for (x, px) in row.as_chunks::<3>().0.iter().enumerate() {
- blue[offset + x] = normalization.apply(0, px[2]);
- green[offset + x] = normalization.apply(1, px[1]);
- red[offset + x] = normalization.apply(2, px[0]);
+ blue[offset + x] = lookup[0][px[2] as usize];
+ green[offset + x] = lookup[1][px[1] as usize];
+ red[offset + x] = lookup[2][px[0] as usize];
}
let padding = offset + width..offset + plane_width;
blue[padding.clone()].fill(0.0);
diff --git a/rust/crates/ml/src/onnx.rs b/rust/crates/ml/src/onnx.rs
index 6e2255895e..fc57a26d1e 100644
--- a/rust/crates/ml/src/onnx.rs
+++ b/rust/crates/ml/src/onnx.rs
@@ -26,7 +26,7 @@ use providers::{ExecutionProvider, ProviderPlan};
#[derive(Clone, Debug)]
pub(crate) struct GpuOptions {
- pub(crate) dimensions: Vec<(&'static str, i64)>,
+ pub(crate) subgraphs: bool,
#[cfg(any(target_os = "android", target_os = "linux", target_os = "windows"))]
pub(crate) prefer_nhwc: bool,
}
@@ -237,7 +237,7 @@ impl SessionRunError {
Self::Retryable(error)
}
- fn from_inference_error(error: ort::Error) -> Self {
+ pub(crate) fn from_inference_error(error: ort::Error) -> Self {
match error.code() {
ort::ErrorCode::GenericFailure
| ort::ErrorCode::RuntimeException
diff --git a/rust/crates/ml/src/onnx/providers.rs b/rust/crates/ml/src/onnx/providers.rs
index 596cdf04d8..9187c029e3 100644
--- a/rust/crates/ml/src/onnx/providers.rs
+++ b/rust/crates/ml/src/onnx/providers.rs
@@ -157,7 +157,7 @@ pub(super) fn provider_attempt(
#[cfg(any(target_os = "ios", target_os = "macos"))]
ExecutionProvider::CoreMl => {
let (coreml_provider, coreml_cache_dir) =
- coreml_provider(_model_path, _model_namespace, _gpu_options.is_some());
+ coreml_provider(_model_path, _model_namespace, _gpu_options);
ProviderAttempt {
providers: if _gpu_options.is_some() {
vec![coreml_provider]
@@ -175,7 +175,7 @@ pub(super) fn provider_attempt(
#[cfg(any(target_os = "android", target_os = "linux", target_os = "windows"))]
ExecutionProvider::WebGpu => ProviderAttempt {
providers: if let Some(options) = _gpu_options {
- vec![webgpu_provider(options.prefer_nhwc)]
+ vec![ocr_webgpu_provider(options.prefer_nhwc)]
} else {
webgpu_attempt_providers()
},
@@ -200,15 +200,10 @@ pub(super) fn build_session(
.with_inter_threads(1)?;
if let Some(options) = gpu_options
- && matches!(
- attempt.execution_provider,
- ExecutionProvider::CoreMl | ExecutionProvider::WebGpu
- )
+ && (attempt.execution_provider == ExecutionProvider::WebGpu
+ || (attempt.execution_provider == ExecutionProvider::CoreMl && !options.subgraphs))
{
builder = builder.with_disable_cpu_fallback()?;
- for &(name, value) in &options.dimensions {
- builder = builder.with_dimension_override(name, value)?;
- }
}
if attempt.disable_intra_op_spinning {
@@ -310,18 +305,19 @@ fn accelerated_provider_allowed(
fn coreml_provider(
model_path: &str,
model_namespace: &str,
- full_gpu: bool,
+ options: Option<&GpuOptions>,
) -> (ExecutionProviderDispatch, Option<PathBuf>) {
let mut provider = CoreML::default()
.with_model_format(ModelFormat::MLProgram)
.with_compute_units(ComputeUnits::All)
.with_specialization_strategy(SpecializationStrategy::Default);
- if full_gpu {
+ if let Some(options) = options {
provider = provider
.with_compute_units(ComputeUnits::CPUAndGPU)
.with_low_precision_accumulation_on_gpu(false)
- .with_static_input_shapes(true);
+ .with_static_input_shapes(true)
+ .with_subgraphs(options.subgraphs);
}
let mut prepared_cache_dir = None;
@@ -360,6 +356,23 @@ fn webgpu_provider(prefer_nhwc: bool) -> ExecutionProviderDispatch {
provider.build().error_on_failure()
}
+#[cfg(any(target_os = "android", target_os = "linux", target_os = "windows"))]
+fn ocr_webgpu_provider(prefer_nhwc: bool) -> ExecutionProviderDispatch {
+ use ort::ep::ArbitrarilyConfigurableExecutionProvider;
+
+ let backend = if cfg!(target_os = "windows") {
+ "D3D12"
+ } else {
+ "Vulkan"
+ };
+ WebGPU::default()
+ .with_arbitrary_config("preferredLayout", if prefer_nhwc { "NHWC" } else { "NCHW" })
+ .with_arbitrary_config("dawnBackendType", backend)
+ .with_arbitrary_config("enableGraphCapture", "0")
+ .build()
+ .error_on_failure()
+}
+
#[cfg(target_os = "android")]
fn webgpu_attempt_providers() -> Vec<ExecutionProviderDispatch> {
vec![