A lightweight yet powerful audio-to-MIDI converter with pitch bend detection
Python audiolightweightmachine-learningmidimusicpitch-detectionpolyphonicpythontranscriptiontypescript
Refactor inference so we can add more model types
David Rubinstein8d9daa924364
diff --git a/basic_pitch/__init__.py b/basic_pitch/__init__.pyindex bde0d4a..e390847 100644--- a/basic_pitch/__init__.py+++ b/basic_pitch/__init__.py@@ -15,6 +15,7 @@# See the License for the specific language governing permissions and# limitations under the License.+import loggingimport pathlib__author__ = "Spotify"@@ -24,4 +25,20 @@ __demowebsite__ = "https://basicpitch.io"__description__ = "Basic Pitch, a lightweight yet powerful audio-to-MIDI converter with pitch bend detection."__url__ = "https://github.com/spotify/basic-pitch"-ICASSP_2022_MODEL_PATH = pathlib.Path(__file__).parent / "saved_models/icassp_2022/nmp"++try:+ import tensorflow as tf++ TF_PRESENT = True+except ImportError:+ TF_PRESENT = False+ logging.warning(+ "Tensorflow is not installed. "+ "If you plan to use a TF Saved Model, "+ "reinstall basic-pitch with `pip install 'basic-pitch[tf]'`"+ )++if TF_PRESENT:+ _filename = "nmp"++ICASSP_2022_MODEL_PATH = pathlib.Path(__file__).parent / "saved_models/icassp_2022" / _filenamediff --git a/basic_pitch/inference.py b/basic_pitch/inference.pyindex be521a1..f770f6c 100644--- a/basic_pitch/inference.py+++ b/basic_pitch/inference.py@@ -20,10 +20,31 @@ import enumimport jsonimport osimport pathlib-from typing import Dict, List, Optional, Sequence, Tuple, Union+from typing import Dict, List, Optional, Sequence, Tuple, Union, cast+++try:+ import tensorflow as tf+except ImportError:+ pass++try:+ import coremltools as ct+except ImportError:+ pass++try:+ import tflite_runtime.interpreter as tflite+except ImportError:+ pass++try:+ import onnxruntime as ort+except ImportError:+ pass-from tensorflow import Tensor, signal, keras, saved_modelimport numpy as np+import numpy.typing as nptimport librosaimport pretty_midi@@ -33,16 +54,43 @@ from basic_pitch.constants import (ANNOTATIONS_FPS,FFT_HOP,)-from basic_pitch import ICASSP_2022_MODEL_PATH, note_creation as infer+from basic_pitch import TF_PRESENTfrom basic_pitch.commandline_printing import (generating_file_message,no_tf_warnings,file_saved_confirmation,failed_to_save,)+import basic_pitch.note_creation as infer+++class Model:+ class MODEL_TYPES(enum.Enum):+ TENSORFLOW = enum.auto()++ def __init__(self, model_path: Union[pathlib.Path, str]):+ if TF_PRESENT:+ try:+ self.model_type = Model.MODEL_TYPES.TENSORFLOW+ self.model = tf.saved_model.load(model_path)+ return+ except Exception:+ pass++ raise ValueError(+ f"File {model_path} cannot be loaded into either "+ "TensorFlow, CoreML, TFLite or ONNX. "+ "Please check if it is a supported and valid serialized model"+ )++ def predict(self, x: npt.NDArray[np.float32]) -> Dict[str, npt.NDArray[np.float32]]:+ if self.model_type == Model.MODEL_TYPES.TENSORFLOW:+ return {k: v.numpy() for k, v in cast(tf.keras.Model, self.model(x)).items()}-def window_audio_file(audio_original: Tensor, hop_size: int) -> Tuple[Tensor, List[Dict[str, int]]]:+def window_audio_file(+ audio_original: npt.NDArray[np.float32], hop_size: int+) -> Tuple[npt.NDArray[np.float32], List[Dict[str, int]]]:"""Pad appropriately an audio file, and return aswindowed signal, with window length = AUDIO_N_SAMPLES@@ -53,10 +101,13 @@ def window_audio_file(audio_original: Tensor, hop_size: int) -> Tuple[Tensor, Liwindow_times: list of {'start':.., 'end':...} objects (times in seconds)"""- from tensorflow import expand_dims # imporing this here so the module loads faster-- audio_windowed = expand_dims(- signal.frame(audio_original, AUDIO_N_SAMPLES, hop_size, pad_end=True, pad_value=0),+ audio_windowed = np.expand_dims(+ librosa.util.frame(+ librosa.util.fix_length(audio_original, size=(audio_original.shape[0] // hop_size + 2) * hop_size),+ frame_length=AUDIO_N_SAMPLES,+ hop_length=hop_size,+ axis=0,+ ),axis=-1,)window_times = [@@ -71,7 +122,7 @@ def window_audio_file(audio_original: Tensor, hop_size: int) -> Tuple[Tensor, Lidef get_audio_input(audio_path: Union[pathlib.Path, str], overlap_len: int, hop_size: int-) -> Tuple[Tensor, List[Dict[str, int]], int]:+) -> Tuple[npt.NDArray[np.float32], List[Dict[str, int]], int]:"""Read wave file (as mono), pad appropriately, and return aswindowed signal, with window length = AUDIO_N_SAMPLES@@ -91,10 +142,11 @@ def get_audio_input(original_length = audio_original.shape[0]audio_original = np.concatenate([np.zeros((int(overlap_len / 2),), dtype=np.float32), audio_original])audio_windowed, window_times = window_audio_file(audio_original, hop_size)- return audio_windowed, window_times, original_length+ for window, window_time in zip(audio_windowed, window_times):+ yield np.expand_dims(window, axis=0), window_time, original_length-def unwrap_output(output: Tensor, audio_original_length: int, n_overlapping_frames: int) -> np.array:+def unwrap_output(output: npt.NDArray[np.float32], audio_original_length: int, n_overlapping_frames: int) -> np.array:"""Unwrap batched model predictions to a single matrix.Args:@@ -105,31 +157,30 @@ def unwrap_output(output: Tensor, audio_original_length: int, n_overlapping_framReturns:array (n_times, n_freqs)"""- raw_output = output.numpy()- if len(raw_output.shape) != 3:+ if len(output.shape) != 3:return Nonen_olap = int(0.5 * n_overlapping_frames)if n_olap > 0:# remove half of the overlapping frames from beginning and end- raw_output = raw_output[:, n_olap:-n_olap, :]+ output = output[:, n_olap:-n_olap, :]- output_shape = raw_output.shape+ output_shape = output.shapen_output_frames_original = int(np.floor(audio_original_length * (ANNOTATIONS_FPS / AUDIO_SAMPLE_RATE)))- unwrapped_output = raw_output.reshape(output_shape[0] * output_shape[1], output_shape[2])+ unwrapped_output = output.reshape(output_shape[0] * output_shape[1], output_shape[2])return unwrapped_output[:n_output_frames_original, :] # trim to original audio lengthdef run_inference(audio_path: Union[pathlib.Path, str],- model: keras.Model,+ model: Model,debug_file: Optional[pathlib.Path] = None,) -> Dict[str, np.array]:"""Run the model on the input audio path.Args:audio_path: The audio to run inference on.- model: A loaded keras model to run inference with.+ model: A loaded model to run inference with.debug_file: An optional path to output debug data to. Useful for testing/verification.Returns:@@ -140,10 +191,14 @@ def run_inference(overlap_len = n_overlapping_frames * FFT_HOPhop_size = AUDIO_N_SAMPLES - overlap_len- audio_windowed, _, audio_original_length = get_audio_input(audio_path, overlap_len, hop_size)+ output = {"note": [], "onset": [], "contour": []}+ for audio_windowed, _, audio_original_length in get_audio_input(audio_path, overlap_len, hop_size):+ for k, v in model.predict(audio_windowed).items():+ output[k].append(v)- output = model(audio_windowed)- unwrapped_output = {k: unwrap_output(output[k], audio_original_length, n_overlapping_frames) for k in output}+ unwrapped_output = {+ k: unwrap_output(np.concatenate(output[k]), audio_original_length, n_overlapping_frames) for k in output+ }if debug_file:with open(debug_file, "w") as f:@@ -261,7 +316,7 @@ def save_note_events(def predict(audio_path: Union[pathlib.Path, str],- model_or_model_path: Union[keras.Model, pathlib.Path, str] = ICASSP_2022_MODEL_PATH,+ model: Model,onset_threshold: float = 0.5,frame_threshold: float = 0.3,minimum_note_length: float = 127.70,@@ -276,7 +331,7 @@ def predict(Args:audio_path: File path for the audio to run inference on.- model_or_model_path: Path to load the Keras saved model from. Can be local or on GCS.+ model: A loaded Model.onset_threshold: Minimum energy required for an onset to be considered present.frame_threshold: Minimum energy requirement for a frame to be considered present.minimum_note_length: The minimum allowed note length in milliseconds.@@ -290,14 +345,6 @@ def predict("""with no_tf_warnings():- # It's convenient to be able to pass in a keras saved model so if- # someone wants to place this function in a loop,- # the model doesn't have to be reloaded every function call- if isinstance(model_or_model_path, (pathlib.Path, str)):- model = saved_model.load(str(model_or_model_path))- else:- model = model_or_model_path-print(f"Predicting MIDI for {audio_path}...")model_output = run_inference(audio_path, model, debug_file)@@ -348,7 +395,7 @@ def predict_and_save(sonify_midi: bool,save_model_outputs: bool,save_notes: bool,- model_path: Union[pathlib.Path, str] = ICASSP_2022_MODEL_PATH,+ model: Model,onset_threshold: float = 0.5,frame_threshold: float = 0.3,minimum_note_length: float = 127.70,@@ -369,7 +416,7 @@ def predict_and_save(sonify_midi: Whether or not to render audio from the MIDI and output it to a file.save_model_outputs: True to save contours, onsets and notes from the model prediction.save_notes: True to save note events.- model_path: Path to load the Keras saved model from. Can be local or on GCS.+ model: A loaded model.onset_threshold: Minimum energy required for an onset to be considered present.frame_threshold: Minimum energy requirement for a frame to be considered present.minimum_note_length: The minimum allowed note length in milliseconds.@@ -380,8 +427,6 @@ def predict_and_save(debug_file: An optional path to output debug data to. Useful for testing/verification.sonification_samplerate: Sample rate for rendering audio from MIDI."""- model = saved_model.load(str(model_path))-for audio_path in audio_path_list:print("")try:diff --git a/basic_pitch/predict.py b/basic_pitch/predict.pyindex 76bb886..1299e63 100644--- a/basic_pitch/predict.py+++ b/basic_pitch/predict.py@@ -21,6 +21,7 @@ import pathlibimport tracebackfrom basic_pitch import ICASSP_2022_MODEL_PATH+from basic_pitch.inference import Modelos.environ["TF_CPP_MIN_LOG_LEVEL"] = "3"@@ -35,7 +36,9 @@ def main() -> None:"--model_path",type=str,default=ICASSP_2022_MODEL_PATH,- help="path to the saved model directory. Defaults to a ICASSP 2022 model",+ help="path to the saved model directory. Defaults to a ICASSP 2022 model. "+ "The preferred model is determined by the first library available in "+ "[tensorflow]",)parser.add_argument("--save-midi",@@ -136,7 +139,7 @@ def main() -> None:args.sonify_midi,args.save_model_outputs,args.save_note_events,- pathlib.Path(args.model_path),+ Model(pathlib.Path(args.model_path)),args.onset_threshold,args.frame_threshold,args.minimum_note_length,diff --git a/setup.cfg b/setup.cfgnew file mode 100644index 0000000..9b25c3b--- /dev/null+++ b/setup.cfg@@ -0,0 +1,70 @@+[bumpversion]+current_version = 0.3.0+commit = True+tag = True++[metadata]+name = basic-pitch+version = attr: basic_pitch.__version__+description = Basic Pitch, a lightweight yet powerful audio-to-MIDI converter with pitch bend detection.+long_description = Basic Pitch, a lightweight yet powerful audio-to-MIDI converter with pitch bend detection. See https://github.com/spotify/basic-pitch for more details.+author = Spotify+author_email = [email protected]+maintainer = Spotify+maintainer_email = [email protected]+url = https://github.com/spotify/basic-pitch+keywords =+license = Apache 2.0+classifiers =+ Development Status :: 5 - Production/Stable+ Natural Language :: English+ Operating System :: POSIX :: Linux+ Operating System :: MacOS :: MacOS X+ Operating System :: Microsoft :: Windows+ Programming Language :: Python+ Programming Language :: Python :: 3.7+ Programming Language :: Python :: 3.8+ Programming Language :: Python :: 3.9+ Programming Language :: Python :: 3.10+ Programming Language :: Python :: Implementation :: CPython++[options]+zip_safe = False+packages = find:+include_package_data = True+install_requires =+ librosa>=0.8.0+ mir_eval>=0.6+ numpy<1.24,>=1.18+ pretty_midi>=0.2.9+ resampy>=0.2.2+ scipy>=1.4.1+ tensorflow>=2.4.1; platform_machine != 'arm64'+ tensorflow-macos>=2.4.1; platform_machine == 'arm64'+ typing_extensions++[options.entry_points]+console_scripts =+ basic-pitch = basic_pitch.predict:main++[options.extras_require]+test =+ coverage>=5.0.2+ pytest>=6.1.1+ pytest-mock+tf =+ tensorflow>=2.4.1; platform_system != "Darwin"+ tensorflow-macos>=2.4.1; platform_system == "Darwin"+docs =+ mkdocs>=1.0.4+dev =+ basic_pitch[test,tf,docs]+ bump2version>=1.0.1+ mypy+ tox++[bumpversion:file:basic_pitch/__init__.py]++[bdist_wheel]+universal = 1+diff --git a/tests/test_inference.py b/tests/test_inference.pyindex 2bc1607..0c9ae73 100644--- a/tests/test_inference.py+++ b/tests/test_inference.py@@ -36,7 +36,7 @@ class TestPredict(unittest.TestCase):test_audio_path = RESOURCES_PATH / "vocadito_10.wav"model_output, midi_data, note_events = inference.predict(test_audio_path,- ICASSP_2022_MODEL_PATH,+ inference.Model(ICASSP_2022_MODEL_PATH),)assert set(model_output.keys()) == set(["note", "onset", "contour"])assert model_output["note"].shape == model_output["onset"].shape@@ -64,12 +64,7 @@ class TestPredict(unittest.TestCase):test_audio_path = RESOURCES_PATH / "vocadito_10.wav"with tempfile.TemporaryDirectory() as tmpdir:inference.predict_and_save(- [test_audio_path],- tmpdir,- True,- True,- True,- True,+ [test_audio_path], tmpdir, True, True, True, True, model=inference.Model(ICASSP_2022_MODEL_PATH))expected_midi_path = tmpdir / pathlib.Path("vocadito_10_basic_pitch.mid")expected_csv_path = tmpdir / pathlib.Path("vocadito_10_basic_pitch.csv")@@ -84,7 +79,7 @@ class TestPredict(unittest.TestCase):for onset_threshold in [0, 0.3, 0.8, 1]:inference.predict(test_audio_path,- ICASSP_2022_MODEL_PATH,+ inference.Model(ICASSP_2022_MODEL_PATH),onset_threshold=onset_threshold,)@@ -93,7 +88,7 @@ class TestPredict(unittest.TestCase):for frame_threshold in [0, 0.3, 0.8, 1]:inference.predict(test_audio_path,- ICASSP_2022_MODEL_PATH,+ inference.Model(ICASSP_2022_MODEL_PATH),frame_threshold=frame_threshold,)@@ -102,7 +97,7 @@ class TestPredict(unittest.TestCase):for minimum_note_length in [10, 100, 1000]:_, _, note_events = inference.predict(test_audio_path,- ICASSP_2022_MODEL_PATH,+ inference.Model(ICASSP_2022_MODEL_PATH),minimum_note_length=minimum_note_length,)min_len_s = minimum_note_length / 1000.0@@ -114,7 +109,7 @@ class TestPredict(unittest.TestCase):for minimum_frequency in [40, 80, 200, 2000]:_, _, note_events = inference.predict(test_audio_path,- ICASSP_2022_MODEL_PATH,+ inference.Model(ICASSP_2022_MODEL_PATH),minimum_frequency=minimum_frequency,)min_freq_midi = np.round(librosa.hz_to_midi(minimum_frequency))@@ -126,7 +121,7 @@ class TestPredict(unittest.TestCase):for maximum_frequency in [40, 80, 200, 2000]:_, _, note_events = inference.predict(test_audio_path,- ICASSP_2022_MODEL_PATH,+ inference.Model(ICASSP_2022_MODEL_PATH),maximum_frequency=maximum_frequency,)max_freq_midi = np.round(librosa.hz_to_midi(maximum_frequency))diff --git a/tox.ini b/tox.iniindex b53a493..a33dad2 100644--- a/tox.ini+++ b/tox.ini@@ -1,5 +1,5 @@[tox]-envlist = py38,py39,py310,py311,manifest,check-formatting,lint,mypy+envlist = py38,py39,py310,py311,minimal,manifest,check-formatting,lint,mypyskipsdist = Trueusedevelop = Truerequires =@@ -16,6 +16,16 @@ setenv =TEST_SOURCE = {toxinidir}/testsLINE_LENGTH = "120"+[testenv:minimal]+deps = -e .[test]+basepython=3.10+commands =+ pytest tests --ignore tests/test_nn.py {posargs}+setenv =+ SOURCE = {toxinidir}/basic_pitch+ TEST_SOURCE = {toxinidir}/tests+ LINE_LENGTH = "120"+[testenv:manifest]deps = check-manifestskip_install = true