A lightweight yet powerful audio-to-MIDI converter with pitch bend detection
Python audiolightweightmachine-learningmidimusicpitch-detectionpolyphonicpythontranscriptiontypescript
ported over necessary files for file download from benjieg/wip-training-fork. implemented and tested a dataset only arg version of download.py. Need to reconsider the other arguments.
Benjamin Genchelf1eaa6502a8f
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.mdindex 31d1a60..80915df 100644--- a/CONTRIBUTING.md+++ b/CONTRIBUTING.md@@ -13,6 +13,7 @@ We recommend first installing the following non-python dependencies:- To install on Windows, run `choco install libsndfile` using [Chocolatey](https://chocolatey.org/)- To install on Ubuntu, run `sudo apt-get update && sudo apt-get install --no-install-recommends -y --fix-missing pkg-config libsndfile1`- [ffmpeg](https://ffmpeg.org/) is a complete, cross-platform solution to record, convert and stream audio in all `basic-pitch` supported formats+- soxTo compile a debug build of `basic-pitch` that allows using a debugger (like gdb or lldb), use the following command to build the package locally and install a symbolic link for debugging:```shell@@ -87,4 +88,4 @@ terms of the [LICENSE](https://github.com/spotify/basic-pitch/blob/main/LICENSE)# Code of Conduct-Read our [Code of Conduct](CODE_OF_CONDUCT.md) for the project.\ No newline at end of file+Read our [Code of Conduct](CODE_OF_CONDUCT.md) for the project.diff --git a/basic_pitch/data/__init__.py b/basic_pitch/data/__init__.pynew file mode 100644index 0000000..e69de29diff --git a/basic_pitch/data/commandline.py b/basic_pitch/data/commandline.pynew file mode 100644index 0000000..ab178f8--- /dev/null+++ b/basic_pitch/data/commandline.py@@ -0,0 +1,64 @@+#!/usr/bin/env python+# encoding: utf-8+#+# Copyright 2022 Spotify AB+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.++import argparse+import os+import os.path as op+++def add_default(parser: argparse.ArgumentParser, dataset_name: str):+ parser.add_argument("source", nargs='?', default=op.join(op.expanduser('~'), 'mir_datasets'),+ help="Source directory for mir data. Defaults to local mir_datasets folder.")+ parser.add_argument("destination", nargs='?',+ default=op.join(op.expanduser('~'), 'data', 'basic_pitch', dataset_name),+ help="Output directory to write results to. Defaults to local ~/data/basic_pitch/{dataset}/")+ parser.add_argument("--runner", choices=["DataflowRunner", "DirectRunner"], default="DirectRunner")+ parser.add_argument("--timestamped", default=False, action="store_true",+ help="If passed, the dataset will be put into a timestamp directory instead of 'splits'")+ parser.add_argument("--batch-size", default=5, type=int, help="Number of examples per tfrecord")+ parser.add_argument("--worker-harness-container-image", default="",+ help="Container image to run dataset generation job with. \+ Required due to non-python dependencies.")+++def resolve_destination(namespace: argparse.Namespace, dataset: str, time_created: int) -> str:+ return os.path.join(namespace.destination, str(time_created) if namespace.timestamped else "splits")+++def add_split(+ parser: argparse.ArgumentParser,+ train_percent: float = 0.8,+ validation_percent: float = 0.1,+ split_seed: int = None,+):+ parser.add_argument(+ "--train-percent",+ type=float,+ default=train_percent,+ help="Percentage of tracks to mark as train",+ )+ parser.add_argument( "--validation-percent",+ type=float,+ default=validation_percent,+ help="Percentage of tracks to mark as validation",+ )+ parser.add_argument(+ "--split-seed",+ type=int,+ default=split_seed,+ help="Seed for random number generator used in split generation",+ )diff --git a/basic_pitch/data/datasets/__init__.py b/basic_pitch/data/datasets/__init__.pynew file mode 100644index 0000000..e69de29diff --git a/basic_pitch/data/datasets/guitarset.py b/basic_pitch/data/datasets/guitarset.pynew file mode 100644index 0000000..c4b0803--- /dev/null+++ b/basic_pitch/data/datasets/guitarset.py@@ -0,0 +1,185 @@+#!/usr/bin/env python+# encoding: utf-8+#+# Copyright 2022 Spotify AB+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.++import argparse+import logging+import os+import os.path as op+import random+import time+from typing import List, Tuple, Optional++import apache_beam as beam+import mirdata++from basic_pitch.data import commandline, pipeline++DIRNAME = "guitarset"+++class GuitarSetInvalidTracks(beam.DoFn):+ def process(self, element: Tuple[str, str]):+ track_id, split = element+ yield beam.pvalue.TaggedOutput(split, track_id)+++class GuitarSetToTfExample(beam.DoFn):+ DOWNLOAD_ATTRIBUTES = ["audio_mic_path", "jams_path"]++ def __init__(self, source: str):+ print(f"source_dir: {source}")+ self.source = source++ def setup(self):+ import apache_beam as beam+ import mirdata++ self.guitarset_remote = mirdata.initialize("guitarset", data_home=os.path.join(self.source, DIRNAME))+ self.filesystem = beam.io.filesystems.FileSystems()++ def process(self, element: List[str]):+ import tempfile++ import mirdata+ import numpy as np+ import sox++ from basic_pitch.constants import (+ AUDIO_N_CHANNELS,+ AUDIO_SAMPLE_RATE,+ FREQ_BINS_CONTOURS,+ FREQ_BINS_NOTES,+ ANNOTATION_HOP,+ N_FREQ_BINS_NOTES,+ N_FREQ_BINS_CONTOURS,+ )+ from basic_pitch.data import tf_example_serialization++ logging.info(f"Processing {element}")+ batch = []++ for track_id in element:+ track_remote = self.guitarset_remote.track(track_id)+ with tempfile.TemporaryDirectory() as local_tmp_dir:+ guitarset_local = mirdata.initialize("guitarset", local_tmp_dir)+ track_local = guitarset_local.track(track_id)++ for attribute in self.DOWNLOAD_ATTRIBUTES:+ source = getattr(track_remote, attribute)+ print(f"source: {source}")+ destination = getattr(track_local, attribute)+ os.makedirs(os.path.dirname(destination), exist_ok=True)+ with self.filesystem.open(source) as s, open(destination, "wb") as d:+ d.write(s.read())++ local_wav_path = f"{track_local.audio_mic_path}_tmp.wav"++ tfm = sox.Transformer()+ tfm.rate(AUDIO_SAMPLE_RATE)+ tfm.channels(AUDIO_N_CHANNELS)+ tfm.build(track_local.audio_mic_path, local_wav_path)++ duration = sox.file_info.duration(local_wav_path)+ time_scale = np.arange(0, duration + ANNOTATION_HOP, ANNOTATION_HOP)+ n_time_frames = len(time_scale)++ note_indices, note_values = track_local.notes_all.to_sparse_index(+ time_scale, "s", FREQ_BINS_NOTES, "hz"+ )+ onset_indices, onset_values = track_local.notes_all.to_sparse_index(+ time_scale, "s", FREQ_BINS_NOTES, "hz", onsets_only=True+ )+ contour_indices, contour_values = track_local.multif0.to_sparse_index(+ time_scale, "s", FREQ_BINS_CONTOURS, "hz"+ )++ batch.append(+ tf_example_serialization.to_transcription_tfexample(+ track_local.track_id,+ "guitarset",+ local_wav_path,+ note_indices,+ note_values,+ onset_indices,+ onset_values,+ contour_indices,+ contour_values,+ (n_time_frames, N_FREQ_BINS_NOTES),+ (n_time_frames, N_FREQ_BINS_CONTOURS),+ )+ )+ return [batch]+++def create_input_data(+ train_percent: float, validation_percent: float, seed: Optional[int] = None+) -> List[Tuple[str, str]]:+ assert train_percent + validation_percent < 1.0, "Don't over allocate the data!"++ # Test percent is 1 - train - validation+ validation_bound = train_percent+ test_bound = validation_bound + validation_percent++ if seed:+ random.seed(seed)++ def determine_split() -> str:+ partition = random.uniform(0, 1)+ if partition < validation_bound:+ return "train"+ if partition < test_bound:+ return "validation"+ return "test"++ guitarset = mirdata.initialize("guitarset")+ guitarset.download()++ return [(track_id, determine_split()) for track_id in guitarset.track_ids]+++def main(known_args, pipeline_args):+ time_created = int(time.time())+ destination = commandline.resolve_destination(known_args, DIRNAME, time_created)+ input_data = create_input_data(known_args.train_percent, known_args.validation_percent, known_args.split_seed)++ pipeline_options = {+ "runner": known_args.runner,+ "job_name": f"guitarset-tfrecords-{time_created}",+ "machine_type": "e2-standard-4",+ "num_workers": 25,+ "disk_size_gb": 128,+ "experiments": ["use_runner_v2"],+ "save_main_session": True,+ "worker_harness_container_image": known_args.worker_harness_container_image,+ }+ pipeline.run(+ pipeline_options,+ input_data,+ GuitarSetToTfExample(known_args.source),+ GuitarSetInvalidTracks(),+ destination,+ known_args.batch_size,+ )+++if __name__ == "__main__":+ parser = argparse.ArgumentParser()+ commandline.add_default(parser, op.basename(op.splittext(__file__)[0]))+ commandline.add_split(parser)+ known_args, pipeline_args = parser.parse_known_args()++ main(known_args, pipeline_args)diff --git a/basic_pitch/data/download.py b/basic_pitch/data/download.pynew file mode 100644index 0000000..ffd7665--- /dev/null+++ b/basic_pitch/data/download.py@@ -0,0 +1,25 @@+import argparse++from basic_pitch.data import commandline+from basic_pitch.data.datasets.guitarset import main as guitarset_main++DATASET_DICT = {+ 'guitarset': guitarset_main,+}++def main():+ dataset_parser = argparse.ArgumentParser()+ dataset_parser.add_argument("dataset", choices=list(DATASET_DICT.keys()), help="The dataset to download / process.")+ args, remaining_args = dataset_parser.parse_known_args()+ dataset = args.dataset++ print(f'got the arg: {dataset}')+ cl_parser = argparse.ArgumentParser()+ commandline.add_default(cl_parser, dataset)+ commandline.add_split(cl_parser)+ known_args, pipeline_args = cl_parser.parse_known_args(remaining_args)+ DATASET_DICT[dataset](known_args, pipeline_args)+++if __name__ == '__main__':+ main()diff --git a/basic_pitch/data/pipeline.py b/basic_pitch/data/pipeline.pynew file mode 100644index 0000000..3522785--- /dev/null+++ b/basic_pitch/data/pipeline.py@@ -0,0 +1,95 @@+#!/usr/bin/env python+# encoding: utf-8+#+# Copyright 2022 Spotify AB+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.++import logging+import os+import uuid+from typing import Dict, List, Tuple++import apache_beam as beam+import tensorflow as tf+from apache_beam.options.pipeline_options import PipelineOptions+++# Beacase beam.GroupIntoBatches isn't supported as of 2.29+class Batch(beam.DoFn):+ def __init__(self, batch_size):+ self.batch_size = batch_size++ def process(self, element):+ for i in range(0, len(element), self.batch_size):+ yield element[i : i + self.batch_size]+++class WriteBatchToTfRecord(beam.DoFn):+ def __init__(self, destination):+ self.destination = destination++ def process(self, element):+ logging.info(f"Writing to file batch of length {len(element)}")+ # hopefully uuids are unique enough+ with tf.io.TFRecordWriter(os.path.join(self.destination, f"{uuid.uuid4()}.tfrecord")) as writer:+ for example in element:+ writer.write(example.SerializeToString())+++def transcription_dataset_writer(+ pcoll,+ input_data: List[Tuple[str, str]],+ to_tf_example: beam.DoFn,+ filter_invalid_tracks: beam.DoFn,+ destination: str,+ batch_size: int,+):+ valid_track_ids = (+ pcoll+ | "Create PCollection of track IDS" >> beam.Create(input_data)+ | "Remove invalid track IDs"+ >> beam.ParDo(filter_invalid_tracks).with_outputs(+ "train",+ "test",+ "validation",+ )+ )++ for split in ["train", "test", "validation"]:+ (+ getattr(valid_track_ids, split)+ | f"Combine {split} into giant list" >> beam.transforms.combiners.ToList()+ | f"Batch {split}" >> beam.ParDo(Batch(batch_size))+ | f"Reshuffle {split}" >> beam.Reshuffle() # To prevent fuses+ | f"Create tf.Example {split} batch" >> beam.ParDo(to_tf_example)+ | f"Write {split} batch to tfrecord" >> beam.ParDo(WriteBatchToTfRecord(os.path.join(destination, split)))+ )+ getattr(valid_track_ids, split) | f"Write {split} index file" >> beam.io.textio.WriteToText(+ os.path.join(destination, split, "index.csv"),+ num_shards=1,+ header="track_id",+ shard_name_template="",+ )+++def run(+ pipeline_options: Dict[str, str],+ input_data: List[Tuple[str, str]],+ to_tf_example: beam.DoFn,+ filter_invalid_tracks: beam.DoFn,+ destination: str,+ batch_size: int,+):+ with beam.Pipeline(options=PipelineOptions(**pipeline_options)) as p:+ transcription_dataset_writer(p, input_data, to_tf_example, filter_invalid_tracks, destination, batch_size)diff --git a/basic_pitch/data/tf_example_serialization.py b/basic_pitch/data/tf_example_serialization.pynew file mode 100644index 0000000..c4985c7--- /dev/null+++ b/basic_pitch/data/tf_example_serialization.py@@ -0,0 +1,119 @@+#!/usr/bin/env python+# encoding: utf-8+#+# Copyright 2022 Spotify AB+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.++from typing import List, Tuple, Union++import sox+import numpy as np+import tensorflow as tf+from basic_pitch.constants import AUDIO_N_CHANNELS, AUDIO_SAMPLE_RATE+++def int64_feature(value: Union[List[int], int]) -> tf.train.Feature:+ if not isinstance(value, list):+ value = [value]+ return tf.train.Feature(int64_list=tf.train.Int64List(value=value))+++def float_feature(value: Union[List[float], float]) -> tf.train.Feature:+ if not isinstance(value, list):+ value = [value]+ return tf.train.Feature(float_list=tf.train.FloatList(value=value))+++def bytes_feature(value: Union[tf.Tensor, List[bytes], bytes]) -> tf.train.Feature:+ if isinstance(value, type(tf.constant(0))):+ value = value.numpy()+ if not isinstance(value, list):+ value = [value]+ return tf.train.Feature(bytes_list=tf.train.BytesList(value=value))+++def _to_transcription_tfex(+ file_id: str,+ source: str,+ encoded_wav: bytes,+ notes_indices: List[Tuple[int, int]],+ notes_values: List[float],+ onsets_indices: List[float],+ onsets_values: List[float],+ contours_indices: List[float],+ contours_values: List[float],+ notes_onsets_shape: Tuple[int, int],+ contours_shape: Tuple[int, int],+) -> tf.train.Example:+ return tf.train.Example(+ features=tf.train.Features(+ feature={+ "file_id": bytes_feature(bytes(file_id, "utf-8")),+ "source": bytes_feature(bytes(source, "utf-8")),+ "audio_wav": bytes_feature(encoded_wav),+ "notes_indices": bytes_feature(tf.io.serialize_tensor(np.array(notes_indices, np.int64))),+ "notes_values": bytes_feature(tf.io.serialize_tensor(np.array(notes_values, np.float32))),+ "onsets_indices": bytes_feature(tf.io.serialize_tensor(np.array(onsets_indices, np.int64))),+ "onsets_values": bytes_feature(tf.io.serialize_tensor(np.array(onsets_values, np.float32))),+ "contours_indices": bytes_feature(tf.io.serialize_tensor(np.array(contours_indices, np.int64))),+ "contours_values": bytes_feature(tf.io.serialize_tensor(np.array(contours_values, np.float32))),+ "notes_onsets_shape": bytes_feature(tf.io.serialize_tensor(np.array(notes_onsets_shape, np.int64))),+ "contours_shape": bytes_feature(tf.io.serialize_tensor(np.array(contours_shape, np.int64))),+ }+ )+ )+++def to_transcription_tfexample(+ file_id: str,+ source: str,+ audio_wav_file_path: str,+ notes_indices: List[Tuple[int, int]],+ notes_values: List[float],+ onsets_indices: List[float],+ onsets_values: List[float],+ contours_indices: List[float],+ contours_values: List[float],+ notes_onsets_shape: Tuple[int, int],+ contours_shape: Tuple[int, int],+) -> tf.train.Example:+ """+ - `file_id` string+ - `source` string (e.g., "maestro")+ - `audio_file_path` path to a local WAV file (must be 22kHz stereo, checked)+ - `notes_indices` [(time, freq)], integers+ - `notes_values` [float]+ - `onsets_indices` same as above+ - `onsets_values` same as above+ - `contours_indices` same as above+ - `contours_values` same as above+ - `notes_onsets_shape` (time, freq), ints+ - `contours_onsets_shape` (time, freq), ints+ """+ assert sox.file_info.sample_rate(audio_wav_file_path) == AUDIO_SAMPLE_RATE+ assert sox.file_info.channels(audio_wav_file_path) == AUDIO_N_CHANNELS+ encoded_wav = open(audio_wav_file_path, "rb").read()+ return _to_transcription_tfex(+ file_id,+ source,+ encoded_wav,+ notes_indices,+ notes_values,+ onsets_indices,+ onsets_values,+ contours_indices,+ contours_values,+ notes_onsets_shape,+ contours_shape,+ )diff --git a/pyproject.toml b/pyproject.tomlindex 3c12bae..cc4a19d 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -51,6 +51,12 @@ namespaces = falsebasic-pitch = "basic_pitch.predict:main"[project.optional-dependencies]+train = [+ "apache_beam",+ "mirdata",+ "smart_open",+ "sox"+]test = ["coverage>=5.0.2","pytest>=6.1.1",