Skip to content

A lightweight yet powerful audio-to-MIDI converter with pitch bend detection

Add CoreML serialized model

David Rubinstein5d575a38c38e
diff --git a/basic_pitch/__init__.py b/basic_pitch/__init__.py
index 64761f2..b72ba2e 100644
--- a/basic_pitch/__init__.py
+++ b/basic_pitch/__init__.py
@@ -38,6 +38,18 @@ except ImportError:
"reinstall basic-pitch with `pip install 'basic-pitch[tf]'`"
)
+try:
+ import coremltools as ct
+
+ CT_PRESENT = True
+except ImportError:
+ CT_PRESENT = False
+ logging.warning(
+ "Coremltools is not installed. "
+ "If you plan to use a CoreML Saved Model, "
+ "reinstall basic-pitch with `pip install 'basic-pitch[coreml]'`"
+ )
+
try:
import tflite_runtime.interpreter as tflite
@@ -65,6 +77,8 @@ except ImportError:
if TF_PRESENT:
_filename = "nmp"
+elif CT_PRESENT:
+ _filename = "nmp.mlpackage"
elif TFLITE_PRESENT:
_filename = "nmp.tflite"
elif ONNX_PRESENT:
diff --git a/basic_pitch/inference.py b/basic_pitch/inference.py
index 36c130e..679f1f2 100644
--- a/basic_pitch/inference.py
+++ b/basic_pitch/inference.py
@@ -54,7 +54,7 @@ from basic_pitch.constants import (
ANNOTATIONS_FPS,
FFT_HOP,
)
-from basic_pitch import ONNX_PRESENT, TF_PRESENT, TFLITE_PRESENT
+from basic_pitch import CT_PRESENT, ONNX_PRESENT, TF_PRESENT, TFLITE_PRESENT
from basic_pitch.commandline_printing import (
generating_file_message,
no_tf_warnings,
@@ -67,6 +67,7 @@ import basic_pitch.note_creation as infer
class Model:
class MODEL_TYPES(enum.Enum):
TENSORFLOW = enum.auto()
+ COREML = enum.auto()
TFLITE = enum.auto()
ONNX = enum.auto()
@@ -79,6 +80,13 @@ class Model:
except Exception:
pass
+ if CT_PRESENT:
+ try:
+ self.model_type = Model.MODEL_TYPES.COREML
+ self.model = ct.models.MLModel(str(model_path))
+ return
+ except Exception:
+ pass
if TFLITE_PRESENT:
try:
@@ -107,6 +115,8 @@ class Model:
def predict(self, x: npt.NDArray[np.float32]) -> Dict[str, npt.NDArray[np.float32]]:
if self.model_type == Model.MODEL_TYPES.TENSORFLOW:
return {k: v.numpy() for k, v in cast(tf.keras.Model, self.model(x)).items()}
+ elif self.model_type == Model.MODEL_TYPES.COREML:
+ return cast(ct.models.MLModel, self.model.predict({"input": x.tolist()}))
elif self.model_type == Model.MODEL_TYPES.TFLITE:
return cast(tflite.Interpreter, self.model)(x)
elif self.model_type == Model.MODEL_TYPES.ONNX:
diff --git a/basic_pitch/predict.py b/basic_pitch/predict.py
index dfec692..becaf51 100644
--- a/basic_pitch/predict.py
+++ b/basic_pitch/predict.py
@@ -38,7 +38,7 @@ def main() -> None:
default=ICASSP_2022_MODEL_PATH,
help="path to the saved model directory. Defaults to a ICASSP 2022 model. "
"The preferred model is determined by the first library available in "
- "[tensorflow, tensorflow-lite, onnx]",
+ "[tensorflow, coreml, tensorflow-lite, onnx]",
)
parser.add_argument(
"--save-midi",
diff --git a/basic_pitch/saved_models/icassp_2022/nmp.mlpackage/Data/com.apple.CoreML/model.mlmodel b/basic_pitch/saved_models/icassp_2022/nmp.mlpackage/Data/com.apple.CoreML/model.mlmodel
new file mode 100644
index 0000000..f70fef0
Binary files /dev/null and b/basic_pitch/saved_models/icassp_2022/nmp.mlpackage/Data/com.apple.CoreML/model.mlmodel differ
diff --git a/basic_pitch/saved_models/icassp_2022/nmp.mlpackage/Data/com.apple.CoreML/weights/weight.bin b/basic_pitch/saved_models/icassp_2022/nmp.mlpackage/Data/com.apple.CoreML/weights/weight.bin
new file mode 100644
index 0000000..2871b09
Binary files /dev/null and b/basic_pitch/saved_models/icassp_2022/nmp.mlpackage/Data/com.apple.CoreML/weights/weight.bin differ
diff --git a/basic_pitch/saved_models/icassp_2022/nmp.mlpackage/Manifest.json b/basic_pitch/saved_models/icassp_2022/nmp.mlpackage/Manifest.json
new file mode 100644
index 0000000..884803d
--- /dev/null
+++ b/basic_pitch/saved_models/icassp_2022/nmp.mlpackage/Manifest.json
@@ -0,0 +1,18 @@
+{
+ "fileFormatVersion": "1.0.0",
+ "itemInfoEntries": {
+ "2DFFAB47-B4DD-4386-B1A5-BF5E85C7EB2A": {
+ "author": "com.apple.CoreML",
+ "description": "CoreML Model Specification",
+ "name": "model.mlmodel",
+ "path": "com.apple.CoreML/model.mlmodel"
+ },
+ "403F4E6F-A404-4968-8017-279110B82EF3": {
+ "author": "com.apple.CoreML",
+ "description": "CoreML Model Weights",
+ "name": "weights",
+ "path": "com.apple.CoreML/weights"
+ }
+ },
+ "rootModelIdentifier": "2DFFAB47-B4DD-4386-B1A5-BF5E85C7EB2A"
+}
diff --git a/setup.cfg b/setup.cfg
index a06c8e4..6e5e449 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -33,6 +33,7 @@ zip_safe = False
packages = find:
include_package_data = True
install_requires =
+ coremltools>=2.4.1; platform_system == "Darwin"
librosa>=0.8.0
mir_eval>=0.6
numpy<1.24,>=1.18
@@ -55,12 +56,14 @@ test =
tf =
tensorflow>=2.4.1; platform_system != "Darwin"
tensorflow-macos>=2.4.1; platform_system == "Darwin"
+coreml =
+ coremltools
onnx =
onnxruntime
docs =
mkdocs>=1.0.4
dev =
- basic_pitch[test,tf,onnx,docs]
+ basic_pitch[test,tf,coreml,onnx,docs]
bump2version>=1.0.1
mypy
tox