Skip to content

A lightweight yet powerful audio-to-MIDI converter with pitch bend detection

Refactor inference so we can add more model types

David Rubinstein8d9daa924364
diff --git a/basic_pitch/__init__.py b/basic_pitch/__init__.py
index bde0d4a..e390847 100644
--- a/basic_pitch/__init__.py
+++ b/basic_pitch/__init__.py
@@ -15,6 +15,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+import logging
import pathlib
__author__ = "Spotify"
@@ -24,4 +25,20 @@ __demowebsite__ = "https://basicpitch.io"
__description__ = "Basic Pitch, a lightweight yet powerful audio-to-MIDI converter with pitch bend detection."
__url__ = "https://github.com/spotify/basic-pitch"
-ICASSP_2022_MODEL_PATH = pathlib.Path(__file__).parent / "saved_models/icassp_2022/nmp"
+
+try:
+ import tensorflow as tf
+
+ TF_PRESENT = True
+except ImportError:
+ TF_PRESENT = False
+ logging.warning(
+ "Tensorflow is not installed. "
+ "If you plan to use a TF Saved Model, "
+ "reinstall basic-pitch with `pip install 'basic-pitch[tf]'`"
+ )
+
+if TF_PRESENT:
+ _filename = "nmp"
+
+ICASSP_2022_MODEL_PATH = pathlib.Path(__file__).parent / "saved_models/icassp_2022" / _filename
diff --git a/basic_pitch/inference.py b/basic_pitch/inference.py
index be521a1..f770f6c 100644
--- a/basic_pitch/inference.py
+++ b/basic_pitch/inference.py
@@ -20,10 +20,31 @@ import enum
import json
import os
import pathlib
-from typing import Dict, List, Optional, Sequence, Tuple, Union
+from typing import Dict, List, Optional, Sequence, Tuple, Union, cast
+
+
+try:
+ import tensorflow as tf
+except ImportError:
+ pass
+
+try:
+ import coremltools as ct
+except ImportError:
+ pass
+
+try:
+ import tflite_runtime.interpreter as tflite
+except ImportError:
+ pass
+
+try:
+ import onnxruntime as ort
+except ImportError:
+ pass
-from tensorflow import Tensor, signal, keras, saved_model
import numpy as np
+import numpy.typing as npt
import librosa
import pretty_midi
@@ -33,16 +54,43 @@ from basic_pitch.constants import (
ANNOTATIONS_FPS,
FFT_HOP,
)
-from basic_pitch import ICASSP_2022_MODEL_PATH, note_creation as infer
+from basic_pitch import TF_PRESENT
from basic_pitch.commandline_printing import (
generating_file_message,
no_tf_warnings,
file_saved_confirmation,
failed_to_save,
)
+import basic_pitch.note_creation as infer
+
+
+class Model:
+ class MODEL_TYPES(enum.Enum):
+ TENSORFLOW = enum.auto()
+
+ def __init__(self, model_path: Union[pathlib.Path, str]):
+ if TF_PRESENT:
+ try:
+ self.model_type = Model.MODEL_TYPES.TENSORFLOW
+ self.model = tf.saved_model.load(model_path)
+ return
+ except Exception:
+ pass
+
+ raise ValueError(
+ f"File {model_path} cannot be loaded into either "
+ "TensorFlow, CoreML, TFLite or ONNX. "
+ "Please check if it is a supported and valid serialized model"
+ )
+
+ def predict(self, x: npt.NDArray[np.float32]) -> Dict[str, npt.NDArray[np.float32]]:
+ if self.model_type == Model.MODEL_TYPES.TENSORFLOW:
+ return {k: v.numpy() for k, v in cast(tf.keras.Model, self.model(x)).items()}
-def window_audio_file(audio_original: Tensor, hop_size: int) -> Tuple[Tensor, List[Dict[str, int]]]:
+def window_audio_file(
+ audio_original: npt.NDArray[np.float32], hop_size: int
+) -> Tuple[npt.NDArray[np.float32], List[Dict[str, int]]]:
"""
Pad appropriately an audio file, and return as
windowed signal, with window length = AUDIO_N_SAMPLES
@@ -53,10 +101,13 @@ def window_audio_file(audio_original: Tensor, hop_size: int) -> Tuple[Tensor, Li
window_times: list of {'start':.., 'end':...} objects (times in seconds)
"""
- from tensorflow import expand_dims # imporing this here so the module loads faster
-
- audio_windowed = expand_dims(
- signal.frame(audio_original, AUDIO_N_SAMPLES, hop_size, pad_end=True, pad_value=0),
+ audio_windowed = np.expand_dims(
+ librosa.util.frame(
+ librosa.util.fix_length(audio_original, size=(audio_original.shape[0] // hop_size + 2) * hop_size),
+ frame_length=AUDIO_N_SAMPLES,
+ hop_length=hop_size,
+ axis=0,
+ ),
axis=-1,
)
window_times = [
@@ -71,7 +122,7 @@ def window_audio_file(audio_original: Tensor, hop_size: int) -> Tuple[Tensor, Li
def get_audio_input(
audio_path: Union[pathlib.Path, str], overlap_len: int, hop_size: int
-) -> Tuple[Tensor, List[Dict[str, int]], int]:
+) -> Tuple[npt.NDArray[np.float32], List[Dict[str, int]], int]:
"""
Read wave file (as mono), pad appropriately, and return as
windowed signal, with window length = AUDIO_N_SAMPLES
@@ -91,10 +142,11 @@ def get_audio_input(
original_length = audio_original.shape[0]
audio_original = np.concatenate([np.zeros((int(overlap_len / 2),), dtype=np.float32), audio_original])
audio_windowed, window_times = window_audio_file(audio_original, hop_size)
- return audio_windowed, window_times, original_length
+ for window, window_time in zip(audio_windowed, window_times):
+ yield np.expand_dims(window, axis=0), window_time, original_length
-def unwrap_output(output: Tensor, audio_original_length: int, n_overlapping_frames: int) -> np.array:
+def unwrap_output(output: npt.NDArray[np.float32], audio_original_length: int, n_overlapping_frames: int) -> np.array:
"""Unwrap batched model predictions to a single matrix.
Args:
@@ -105,31 +157,30 @@ def unwrap_output(output: Tensor, audio_original_length: int, n_overlapping_fram
Returns:
array (n_times, n_freqs)
"""
- raw_output = output.numpy()
- if len(raw_output.shape) != 3:
+ if len(output.shape) != 3:
return None
n_olap = int(0.5 * n_overlapping_frames)
if n_olap > 0:
# remove half of the overlapping frames from beginning and end
- raw_output = raw_output[:, n_olap:-n_olap, :]
+ output = output[:, n_olap:-n_olap, :]
- output_shape = raw_output.shape
+ output_shape = output.shape
n_output_frames_original = int(np.floor(audio_original_length * (ANNOTATIONS_FPS / AUDIO_SAMPLE_RATE)))
- unwrapped_output = raw_output.reshape(output_shape[0] * output_shape[1], output_shape[2])
+ unwrapped_output = output.reshape(output_shape[0] * output_shape[1], output_shape[2])
return unwrapped_output[:n_output_frames_original, :] # trim to original audio length
def run_inference(
audio_path: Union[pathlib.Path, str],
- model: keras.Model,
+ model: Model,
debug_file: Optional[pathlib.Path] = None,
) -> Dict[str, np.array]:
"""Run the model on the input audio path.
Args:
audio_path: The audio to run inference on.
- model: A loaded keras model to run inference with.
+ model: A loaded model to run inference with.
debug_file: An optional path to output debug data to. Useful for testing/verification.
Returns:
@@ -140,10 +191,14 @@ def run_inference(
overlap_len = n_overlapping_frames * FFT_HOP
hop_size = AUDIO_N_SAMPLES - overlap_len
- audio_windowed, _, audio_original_length = get_audio_input(audio_path, overlap_len, hop_size)
+ output = {"note": [], "onset": [], "contour": []}
+ for audio_windowed, _, audio_original_length in get_audio_input(audio_path, overlap_len, hop_size):
+ for k, v in model.predict(audio_windowed).items():
+ output[k].append(v)
- output = model(audio_windowed)
- unwrapped_output = {k: unwrap_output(output[k], audio_original_length, n_overlapping_frames) for k in output}
+ unwrapped_output = {
+ k: unwrap_output(np.concatenate(output[k]), audio_original_length, n_overlapping_frames) for k in output
+ }
if debug_file:
with open(debug_file, "w") as f:
@@ -261,7 +316,7 @@ def save_note_events(
def predict(
audio_path: Union[pathlib.Path, str],
- model_or_model_path: Union[keras.Model, pathlib.Path, str] = ICASSP_2022_MODEL_PATH,
+ model: Model,
onset_threshold: float = 0.5,
frame_threshold: float = 0.3,
minimum_note_length: float = 127.70,
@@ -276,7 +331,7 @@ def predict(
Args:
audio_path: File path for the audio to run inference on.
- model_or_model_path: Path to load the Keras saved model from. Can be local or on GCS.
+ model: A loaded Model.
onset_threshold: Minimum energy required for an onset to be considered present.
frame_threshold: Minimum energy requirement for a frame to be considered present.
minimum_note_length: The minimum allowed note length in milliseconds.
@@ -290,14 +345,6 @@ def predict(
"""
with no_tf_warnings():
- # It's convenient to be able to pass in a keras saved model so if
- # someone wants to place this function in a loop,
- # the model doesn't have to be reloaded every function call
- if isinstance(model_or_model_path, (pathlib.Path, str)):
- model = saved_model.load(str(model_or_model_path))
- else:
- model = model_or_model_path
-
print(f"Predicting MIDI for {audio_path}...")
model_output = run_inference(audio_path, model, debug_file)
@@ -348,7 +395,7 @@ def predict_and_save(
sonify_midi: bool,
save_model_outputs: bool,
save_notes: bool,
- model_path: Union[pathlib.Path, str] = ICASSP_2022_MODEL_PATH,
+ model: Model,
onset_threshold: float = 0.5,
frame_threshold: float = 0.3,
minimum_note_length: float = 127.70,
@@ -369,7 +416,7 @@ def predict_and_save(
sonify_midi: Whether or not to render audio from the MIDI and output it to a file.
save_model_outputs: True to save contours, onsets and notes from the model prediction.
save_notes: True to save note events.
- model_path: Path to load the Keras saved model from. Can be local or on GCS.
+ model: A loaded model.
onset_threshold: Minimum energy required for an onset to be considered present.
frame_threshold: Minimum energy requirement for a frame to be considered present.
minimum_note_length: The minimum allowed note length in milliseconds.
@@ -380,8 +427,6 @@ def predict_and_save(
debug_file: An optional path to output debug data to. Useful for testing/verification.
sonification_samplerate: Sample rate for rendering audio from MIDI.
"""
- model = saved_model.load(str(model_path))
-
for audio_path in audio_path_list:
print("")
try:
diff --git a/basic_pitch/predict.py b/basic_pitch/predict.py
index 76bb886..1299e63 100644
--- a/basic_pitch/predict.py
+++ b/basic_pitch/predict.py
@@ -21,6 +21,7 @@ import pathlib
import traceback
from basic_pitch import ICASSP_2022_MODEL_PATH
+from basic_pitch.inference import Model
os.environ["TF_CPP_MIN_LOG_LEVEL"] = "3"
@@ -35,7 +36,9 @@ def main() -> None:
"--model_path",
type=str,
default=ICASSP_2022_MODEL_PATH,
- help="path to the saved model directory. Defaults to a ICASSP 2022 model",
+ help="path to the saved model directory. Defaults to a ICASSP 2022 model. "
+ "The preferred model is determined by the first library available in "
+ "[tensorflow]",
)
parser.add_argument(
"--save-midi",
@@ -136,7 +139,7 @@ def main() -> None:
args.sonify_midi,
args.save_model_outputs,
args.save_note_events,
- pathlib.Path(args.model_path),
+ Model(pathlib.Path(args.model_path)),
args.onset_threshold,
args.frame_threshold,
args.minimum_note_length,
diff --git a/setup.cfg b/setup.cfg
new file mode 100644
index 0000000..9b25c3b
--- /dev/null
+++ b/setup.cfg
@@ -0,0 +1,70 @@
+[bumpversion]
+current_version = 0.3.0
+commit = True
+tag = True
+
+[metadata]
+name = basic-pitch
+version = attr: basic_pitch.__version__
+description = Basic Pitch, a lightweight yet powerful audio-to-MIDI converter with pitch bend detection.
+long_description = Basic Pitch, a lightweight yet powerful audio-to-MIDI converter with pitch bend detection. See https://github.com/spotify/basic-pitch for more details.
+author = Spotify
+author_email = [email protected]
+maintainer = Spotify
+maintainer_email = [email protected]
+url = https://github.com/spotify/basic-pitch
+keywords =
+license = Apache 2.0
+classifiers =
+ Development Status :: 5 - Production/Stable
+ Natural Language :: English
+ Operating System :: POSIX :: Linux
+ Operating System :: MacOS :: MacOS X
+ Operating System :: Microsoft :: Windows
+ Programming Language :: Python
+ Programming Language :: Python :: 3.7
+ Programming Language :: Python :: 3.8
+ Programming Language :: Python :: 3.9
+ Programming Language :: Python :: 3.10
+ Programming Language :: Python :: Implementation :: CPython
+
+[options]
+zip_safe = False
+packages = find:
+include_package_data = True
+install_requires =
+ librosa>=0.8.0
+ mir_eval>=0.6
+ numpy<1.24,>=1.18
+ pretty_midi>=0.2.9
+ resampy>=0.2.2
+ scipy>=1.4.1
+ tensorflow>=2.4.1; platform_machine != 'arm64'
+ tensorflow-macos>=2.4.1; platform_machine == 'arm64'
+ typing_extensions
+
+[options.entry_points]
+console_scripts =
+ basic-pitch = basic_pitch.predict:main
+
+[options.extras_require]
+test =
+ coverage>=5.0.2
+ pytest>=6.1.1
+ pytest-mock
+tf =
+ tensorflow>=2.4.1; platform_system != "Darwin"
+ tensorflow-macos>=2.4.1; platform_system == "Darwin"
+docs =
+ mkdocs>=1.0.4
+dev =
+ basic_pitch[test,tf,docs]
+ bump2version>=1.0.1
+ mypy
+ tox
+
+[bumpversion:file:basic_pitch/__init__.py]
+
+[bdist_wheel]
+universal = 1
+
diff --git a/tests/test_inference.py b/tests/test_inference.py
index 2bc1607..0c9ae73 100644
--- a/tests/test_inference.py
+++ b/tests/test_inference.py
@@ -36,7 +36,7 @@ class TestPredict(unittest.TestCase):
test_audio_path = RESOURCES_PATH / "vocadito_10.wav"
model_output, midi_data, note_events = inference.predict(
test_audio_path,
- ICASSP_2022_MODEL_PATH,
+ inference.Model(ICASSP_2022_MODEL_PATH),
)
assert set(model_output.keys()) == set(["note", "onset", "contour"])
assert model_output["note"].shape == model_output["onset"].shape
@@ -64,12 +64,7 @@ class TestPredict(unittest.TestCase):
test_audio_path = RESOURCES_PATH / "vocadito_10.wav"
with tempfile.TemporaryDirectory() as tmpdir:
inference.predict_and_save(
- [test_audio_path],
- tmpdir,
- True,
- True,
- True,
- True,
+ [test_audio_path], tmpdir, True, True, True, True, model=inference.Model(ICASSP_2022_MODEL_PATH)
)
expected_midi_path = tmpdir / pathlib.Path("vocadito_10_basic_pitch.mid")
expected_csv_path = tmpdir / pathlib.Path("vocadito_10_basic_pitch.csv")
@@ -84,7 +79,7 @@ class TestPredict(unittest.TestCase):
for onset_threshold in [0, 0.3, 0.8, 1]:
inference.predict(
test_audio_path,
- ICASSP_2022_MODEL_PATH,
+ inference.Model(ICASSP_2022_MODEL_PATH),
onset_threshold=onset_threshold,
)
@@ -93,7 +88,7 @@ class TestPredict(unittest.TestCase):
for frame_threshold in [0, 0.3, 0.8, 1]:
inference.predict(
test_audio_path,
- ICASSP_2022_MODEL_PATH,
+ inference.Model(ICASSP_2022_MODEL_PATH),
frame_threshold=frame_threshold,
)
@@ -102,7 +97,7 @@ class TestPredict(unittest.TestCase):
for minimum_note_length in [10, 100, 1000]:
_, _, note_events = inference.predict(
test_audio_path,
- ICASSP_2022_MODEL_PATH,
+ inference.Model(ICASSP_2022_MODEL_PATH),
minimum_note_length=minimum_note_length,
)
min_len_s = minimum_note_length / 1000.0
@@ -114,7 +109,7 @@ class TestPredict(unittest.TestCase):
for minimum_frequency in [40, 80, 200, 2000]:
_, _, note_events = inference.predict(
test_audio_path,
- ICASSP_2022_MODEL_PATH,
+ inference.Model(ICASSP_2022_MODEL_PATH),
minimum_frequency=minimum_frequency,
)
min_freq_midi = np.round(librosa.hz_to_midi(minimum_frequency))
@@ -126,7 +121,7 @@ class TestPredict(unittest.TestCase):
for maximum_frequency in [40, 80, 200, 2000]:
_, _, note_events = inference.predict(
test_audio_path,
- ICASSP_2022_MODEL_PATH,
+ inference.Model(ICASSP_2022_MODEL_PATH),
maximum_frequency=maximum_frequency,
)
max_freq_midi = np.round(librosa.hz_to_midi(maximum_frequency))
diff --git a/tox.ini b/tox.ini
index b53a493..a33dad2 100644
--- a/tox.ini
+++ b/tox.ini
@@ -1,5 +1,5 @@
[tox]
-envlist = py38,py39,py310,py311,manifest,check-formatting,lint,mypy
+envlist = py38,py39,py310,py311,minimal,manifest,check-formatting,lint,mypy
skipsdist = True
usedevelop = True
requires =
@@ -16,6 +16,16 @@ setenv =
TEST_SOURCE = {toxinidir}/tests
LINE_LENGTH = "120"
+[testenv:minimal]
+deps = -e .[test]
+basepython=3.10
+commands =
+ pytest tests --ignore tests/test_nn.py {posargs}
+setenv =
+ SOURCE = {toxinidir}/basic_pitch
+ TEST_SOURCE = {toxinidir}/tests
+ LINE_LENGTH = "120"
+
[testenv:manifest]
deps = check-manifest
skip_install = true