Skip to content

A lightweight yet powerful audio-to-MIDI converter with pitch bend detection

ported over necessary files for file download from benjieg/wip-training-fork. implemented and tested a dataset only arg version of download.py. Need to reconsider the other arguments.

Benjamin Genchelf1eaa6502a8f
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index 31d1a60..80915df 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -13,6 +13,7 @@ We recommend first installing the following non-python dependencies:
- To install on Windows, run `choco install libsndfile` using [Chocolatey](https://chocolatey.org/)
- To install on Ubuntu, run `sudo apt-get update && sudo apt-get install --no-install-recommends -y --fix-missing pkg-config libsndfile1`
- [ffmpeg](https://ffmpeg.org/) is a complete, cross-platform solution to record, convert and stream audio in all `basic-pitch` supported formats
+- sox
To compile a debug build of `basic-pitch` that allows using a debugger (like gdb or lldb), use the following command to build the package locally and install a symbolic link for debugging:
```shell
@@ -87,4 +88,4 @@ terms of the [LICENSE](https://github.com/spotify/basic-pitch/blob/main/LICENSE)
# Code of Conduct
-Read our [Code of Conduct](CODE_OF_CONDUCT.md) for the project.
\ No newline at end of file
+Read our [Code of Conduct](CODE_OF_CONDUCT.md) for the project.
diff --git a/basic_pitch/data/__init__.py b/basic_pitch/data/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/basic_pitch/data/commandline.py b/basic_pitch/data/commandline.py
new file mode 100644
index 0000000..ab178f8
--- /dev/null
+++ b/basic_pitch/data/commandline.py
@@ -0,0 +1,64 @@
+#!/usr/bin/env python
+# encoding: utf-8
+#
+# Copyright 2022 Spotify AB
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import argparse
+import os
+import os.path as op
+
+
+def add_default(parser: argparse.ArgumentParser, dataset_name: str):
+ parser.add_argument("source", nargs='?', default=op.join(op.expanduser('~'), 'mir_datasets'),
+ help="Source directory for mir data. Defaults to local mir_datasets folder.")
+ parser.add_argument("destination", nargs='?',
+ default=op.join(op.expanduser('~'), 'data', 'basic_pitch', dataset_name),
+ help="Output directory to write results to. Defaults to local ~/data/basic_pitch/{dataset}/")
+ parser.add_argument("--runner", choices=["DataflowRunner", "DirectRunner"], default="DirectRunner")
+ parser.add_argument("--timestamped", default=False, action="store_true",
+ help="If passed, the dataset will be put into a timestamp directory instead of 'splits'")
+ parser.add_argument("--batch-size", default=5, type=int, help="Number of examples per tfrecord")
+ parser.add_argument("--worker-harness-container-image", default="",
+ help="Container image to run dataset generation job with. \
+ Required due to non-python dependencies.")
+
+
+def resolve_destination(namespace: argparse.Namespace, dataset: str, time_created: int) -> str:
+ return os.path.join(namespace.destination, str(time_created) if namespace.timestamped else "splits")
+
+
+def add_split(
+ parser: argparse.ArgumentParser,
+ train_percent: float = 0.8,
+ validation_percent: float = 0.1,
+ split_seed: int = None,
+):
+ parser.add_argument(
+ "--train-percent",
+ type=float,
+ default=train_percent,
+ help="Percentage of tracks to mark as train",
+ )
+ parser.add_argument( "--validation-percent",
+ type=float,
+ default=validation_percent,
+ help="Percentage of tracks to mark as validation",
+ )
+ parser.add_argument(
+ "--split-seed",
+ type=int,
+ default=split_seed,
+ help="Seed for random number generator used in split generation",
+ )
diff --git a/basic_pitch/data/datasets/__init__.py b/basic_pitch/data/datasets/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/basic_pitch/data/datasets/guitarset.py b/basic_pitch/data/datasets/guitarset.py
new file mode 100644
index 0000000..c4b0803
--- /dev/null
+++ b/basic_pitch/data/datasets/guitarset.py
@@ -0,0 +1,185 @@
+#!/usr/bin/env python
+# encoding: utf-8
+#
+# Copyright 2022 Spotify AB
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import argparse
+import logging
+import os
+import os.path as op
+import random
+import time
+from typing import List, Tuple, Optional
+
+import apache_beam as beam
+import mirdata
+
+from basic_pitch.data import commandline, pipeline
+
+DIRNAME = "guitarset"
+
+
+class GuitarSetInvalidTracks(beam.DoFn):
+ def process(self, element: Tuple[str, str]):
+ track_id, split = element
+ yield beam.pvalue.TaggedOutput(split, track_id)
+
+
+class GuitarSetToTfExample(beam.DoFn):
+ DOWNLOAD_ATTRIBUTES = ["audio_mic_path", "jams_path"]
+
+ def __init__(self, source: str):
+ print(f"source_dir: {source}")
+ self.source = source
+
+ def setup(self):
+ import apache_beam as beam
+ import mirdata
+
+ self.guitarset_remote = mirdata.initialize("guitarset", data_home=os.path.join(self.source, DIRNAME))
+ self.filesystem = beam.io.filesystems.FileSystems()
+
+ def process(self, element: List[str]):
+ import tempfile
+
+ import mirdata
+ import numpy as np
+ import sox
+
+ from basic_pitch.constants import (
+ AUDIO_N_CHANNELS,
+ AUDIO_SAMPLE_RATE,
+ FREQ_BINS_CONTOURS,
+ FREQ_BINS_NOTES,
+ ANNOTATION_HOP,
+ N_FREQ_BINS_NOTES,
+ N_FREQ_BINS_CONTOURS,
+ )
+ from basic_pitch.data import tf_example_serialization
+
+ logging.info(f"Processing {element}")
+ batch = []
+
+ for track_id in element:
+ track_remote = self.guitarset_remote.track(track_id)
+ with tempfile.TemporaryDirectory() as local_tmp_dir:
+ guitarset_local = mirdata.initialize("guitarset", local_tmp_dir)
+ track_local = guitarset_local.track(track_id)
+
+ for attribute in self.DOWNLOAD_ATTRIBUTES:
+ source = getattr(track_remote, attribute)
+ print(f"source: {source}")
+ destination = getattr(track_local, attribute)
+ os.makedirs(os.path.dirname(destination), exist_ok=True)
+ with self.filesystem.open(source) as s, open(destination, "wb") as d:
+ d.write(s.read())
+
+ local_wav_path = f"{track_local.audio_mic_path}_tmp.wav"
+
+ tfm = sox.Transformer()
+ tfm.rate(AUDIO_SAMPLE_RATE)
+ tfm.channels(AUDIO_N_CHANNELS)
+ tfm.build(track_local.audio_mic_path, local_wav_path)
+
+ duration = sox.file_info.duration(local_wav_path)
+ time_scale = np.arange(0, duration + ANNOTATION_HOP, ANNOTATION_HOP)
+ n_time_frames = len(time_scale)
+
+ note_indices, note_values = track_local.notes_all.to_sparse_index(
+ time_scale, "s", FREQ_BINS_NOTES, "hz"
+ )
+ onset_indices, onset_values = track_local.notes_all.to_sparse_index(
+ time_scale, "s", FREQ_BINS_NOTES, "hz", onsets_only=True
+ )
+ contour_indices, contour_values = track_local.multif0.to_sparse_index(
+ time_scale, "s", FREQ_BINS_CONTOURS, "hz"
+ )
+
+ batch.append(
+ tf_example_serialization.to_transcription_tfexample(
+ track_local.track_id,
+ "guitarset",
+ local_wav_path,
+ note_indices,
+ note_values,
+ onset_indices,
+ onset_values,
+ contour_indices,
+ contour_values,
+ (n_time_frames, N_FREQ_BINS_NOTES),
+ (n_time_frames, N_FREQ_BINS_CONTOURS),
+ )
+ )
+ return [batch]
+
+
+def create_input_data(
+ train_percent: float, validation_percent: float, seed: Optional[int] = None
+) -> List[Tuple[str, str]]:
+ assert train_percent + validation_percent < 1.0, "Don't over allocate the data!"
+
+ # Test percent is 1 - train - validation
+ validation_bound = train_percent
+ test_bound = validation_bound + validation_percent
+
+ if seed:
+ random.seed(seed)
+
+ def determine_split() -> str:
+ partition = random.uniform(0, 1)
+ if partition < validation_bound:
+ return "train"
+ if partition < test_bound:
+ return "validation"
+ return "test"
+
+ guitarset = mirdata.initialize("guitarset")
+ guitarset.download()
+
+ return [(track_id, determine_split()) for track_id in guitarset.track_ids]
+
+
+def main(known_args, pipeline_args):
+ time_created = int(time.time())
+ destination = commandline.resolve_destination(known_args, DIRNAME, time_created)
+ input_data = create_input_data(known_args.train_percent, known_args.validation_percent, known_args.split_seed)
+
+ pipeline_options = {
+ "runner": known_args.runner,
+ "job_name": f"guitarset-tfrecords-{time_created}",
+ "machine_type": "e2-standard-4",
+ "num_workers": 25,
+ "disk_size_gb": 128,
+ "experiments": ["use_runner_v2"],
+ "save_main_session": True,
+ "worker_harness_container_image": known_args.worker_harness_container_image,
+ }
+ pipeline.run(
+ pipeline_options,
+ input_data,
+ GuitarSetToTfExample(known_args.source),
+ GuitarSetInvalidTracks(),
+ destination,
+ known_args.batch_size,
+ )
+
+
+if __name__ == "__main__":
+ parser = argparse.ArgumentParser()
+ commandline.add_default(parser, op.basename(op.splittext(__file__)[0]))
+ commandline.add_split(parser)
+ known_args, pipeline_args = parser.parse_known_args()
+
+ main(known_args, pipeline_args)
diff --git a/basic_pitch/data/download.py b/basic_pitch/data/download.py
new file mode 100644
index 0000000..ffd7665
--- /dev/null
+++ b/basic_pitch/data/download.py
@@ -0,0 +1,25 @@
+import argparse
+
+from basic_pitch.data import commandline
+from basic_pitch.data.datasets.guitarset import main as guitarset_main
+
+DATASET_DICT = {
+ 'guitarset': guitarset_main,
+}
+
+def main():
+ dataset_parser = argparse.ArgumentParser()
+ dataset_parser.add_argument("dataset", choices=list(DATASET_DICT.keys()), help="The dataset to download / process.")
+ args, remaining_args = dataset_parser.parse_known_args()
+ dataset = args.dataset
+
+ print(f'got the arg: {dataset}')
+ cl_parser = argparse.ArgumentParser()
+ commandline.add_default(cl_parser, dataset)
+ commandline.add_split(cl_parser)
+ known_args, pipeline_args = cl_parser.parse_known_args(remaining_args)
+ DATASET_DICT[dataset](known_args, pipeline_args)
+
+
+if __name__ == '__main__':
+ main()
diff --git a/basic_pitch/data/pipeline.py b/basic_pitch/data/pipeline.py
new file mode 100644
index 0000000..3522785
--- /dev/null
+++ b/basic_pitch/data/pipeline.py
@@ -0,0 +1,95 @@
+#!/usr/bin/env python
+# encoding: utf-8
+#
+# Copyright 2022 Spotify AB
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import logging
+import os
+import uuid
+from typing import Dict, List, Tuple
+
+import apache_beam as beam
+import tensorflow as tf
+from apache_beam.options.pipeline_options import PipelineOptions
+
+
+# Beacase beam.GroupIntoBatches isn't supported as of 2.29
+class Batch(beam.DoFn):
+ def __init__(self, batch_size):
+ self.batch_size = batch_size
+
+ def process(self, element):
+ for i in range(0, len(element), self.batch_size):
+ yield element[i : i + self.batch_size]
+
+
+class WriteBatchToTfRecord(beam.DoFn):
+ def __init__(self, destination):
+ self.destination = destination
+
+ def process(self, element):
+ logging.info(f"Writing to file batch of length {len(element)}")
+ # hopefully uuids are unique enough
+ with tf.io.TFRecordWriter(os.path.join(self.destination, f"{uuid.uuid4()}.tfrecord")) as writer:
+ for example in element:
+ writer.write(example.SerializeToString())
+
+
+def transcription_dataset_writer(
+ pcoll,
+ input_data: List[Tuple[str, str]],
+ to_tf_example: beam.DoFn,
+ filter_invalid_tracks: beam.DoFn,
+ destination: str,
+ batch_size: int,
+):
+ valid_track_ids = (
+ pcoll
+ | "Create PCollection of track IDS" >> beam.Create(input_data)
+ | "Remove invalid track IDs"
+ >> beam.ParDo(filter_invalid_tracks).with_outputs(
+ "train",
+ "test",
+ "validation",
+ )
+ )
+
+ for split in ["train", "test", "validation"]:
+ (
+ getattr(valid_track_ids, split)
+ | f"Combine {split} into giant list" >> beam.transforms.combiners.ToList()
+ | f"Batch {split}" >> beam.ParDo(Batch(batch_size))
+ | f"Reshuffle {split}" >> beam.Reshuffle() # To prevent fuses
+ | f"Create tf.Example {split} batch" >> beam.ParDo(to_tf_example)
+ | f"Write {split} batch to tfrecord" >> beam.ParDo(WriteBatchToTfRecord(os.path.join(destination, split)))
+ )
+ getattr(valid_track_ids, split) | f"Write {split} index file" >> beam.io.textio.WriteToText(
+ os.path.join(destination, split, "index.csv"),
+ num_shards=1,
+ header="track_id",
+ shard_name_template="",
+ )
+
+
+def run(
+ pipeline_options: Dict[str, str],
+ input_data: List[Tuple[str, str]],
+ to_tf_example: beam.DoFn,
+ filter_invalid_tracks: beam.DoFn,
+ destination: str,
+ batch_size: int,
+):
+ with beam.Pipeline(options=PipelineOptions(**pipeline_options)) as p:
+ transcription_dataset_writer(p, input_data, to_tf_example, filter_invalid_tracks, destination, batch_size)
diff --git a/basic_pitch/data/tf_example_serialization.py b/basic_pitch/data/tf_example_serialization.py
new file mode 100644
index 0000000..c4985c7
--- /dev/null
+++ b/basic_pitch/data/tf_example_serialization.py
@@ -0,0 +1,119 @@
+#!/usr/bin/env python
+# encoding: utf-8
+#
+# Copyright 2022 Spotify AB
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from typing import List, Tuple, Union
+
+import sox
+import numpy as np
+import tensorflow as tf
+from basic_pitch.constants import AUDIO_N_CHANNELS, AUDIO_SAMPLE_RATE
+
+
+def int64_feature(value: Union[List[int], int]) -> tf.train.Feature:
+ if not isinstance(value, list):
+ value = [value]
+ return tf.train.Feature(int64_list=tf.train.Int64List(value=value))
+
+
+def float_feature(value: Union[List[float], float]) -> tf.train.Feature:
+ if not isinstance(value, list):
+ value = [value]
+ return tf.train.Feature(float_list=tf.train.FloatList(value=value))
+
+
+def bytes_feature(value: Union[tf.Tensor, List[bytes], bytes]) -> tf.train.Feature:
+ if isinstance(value, type(tf.constant(0))):
+ value = value.numpy()
+ if not isinstance(value, list):
+ value = [value]
+ return tf.train.Feature(bytes_list=tf.train.BytesList(value=value))
+
+
+def _to_transcription_tfex(
+ file_id: str,
+ source: str,
+ encoded_wav: bytes,
+ notes_indices: List[Tuple[int, int]],
+ notes_values: List[float],
+ onsets_indices: List[float],
+ onsets_values: List[float],
+ contours_indices: List[float],
+ contours_values: List[float],
+ notes_onsets_shape: Tuple[int, int],
+ contours_shape: Tuple[int, int],
+) -> tf.train.Example:
+ return tf.train.Example(
+ features=tf.train.Features(
+ feature={
+ "file_id": bytes_feature(bytes(file_id, "utf-8")),
+ "source": bytes_feature(bytes(source, "utf-8")),
+ "audio_wav": bytes_feature(encoded_wav),
+ "notes_indices": bytes_feature(tf.io.serialize_tensor(np.array(notes_indices, np.int64))),
+ "notes_values": bytes_feature(tf.io.serialize_tensor(np.array(notes_values, np.float32))),
+ "onsets_indices": bytes_feature(tf.io.serialize_tensor(np.array(onsets_indices, np.int64))),
+ "onsets_values": bytes_feature(tf.io.serialize_tensor(np.array(onsets_values, np.float32))),
+ "contours_indices": bytes_feature(tf.io.serialize_tensor(np.array(contours_indices, np.int64))),
+ "contours_values": bytes_feature(tf.io.serialize_tensor(np.array(contours_values, np.float32))),
+ "notes_onsets_shape": bytes_feature(tf.io.serialize_tensor(np.array(notes_onsets_shape, np.int64))),
+ "contours_shape": bytes_feature(tf.io.serialize_tensor(np.array(contours_shape, np.int64))),
+ }
+ )
+ )
+
+
+def to_transcription_tfexample(
+ file_id: str,
+ source: str,
+ audio_wav_file_path: str,
+ notes_indices: List[Tuple[int, int]],
+ notes_values: List[float],
+ onsets_indices: List[float],
+ onsets_values: List[float],
+ contours_indices: List[float],
+ contours_values: List[float],
+ notes_onsets_shape: Tuple[int, int],
+ contours_shape: Tuple[int, int],
+) -> tf.train.Example:
+ """
+ - `file_id` string
+ - `source` string (e.g., "maestro")
+ - `audio_file_path` path to a local WAV file (must be 22kHz stereo, checked)
+ - `notes_indices` [(time, freq)], integers
+ - `notes_values` [float]
+ - `onsets_indices` same as above
+ - `onsets_values` same as above
+ - `contours_indices` same as above
+ - `contours_values` same as above
+ - `notes_onsets_shape` (time, freq), ints
+ - `contours_onsets_shape` (time, freq), ints
+ """
+ assert sox.file_info.sample_rate(audio_wav_file_path) == AUDIO_SAMPLE_RATE
+ assert sox.file_info.channels(audio_wav_file_path) == AUDIO_N_CHANNELS
+ encoded_wav = open(audio_wav_file_path, "rb").read()
+ return _to_transcription_tfex(
+ file_id,
+ source,
+ encoded_wav,
+ notes_indices,
+ notes_values,
+ onsets_indices,
+ onsets_values,
+ contours_indices,
+ contours_values,
+ notes_onsets_shape,
+ contours_shape,
+ )
diff --git a/pyproject.toml b/pyproject.toml
index 3c12bae..cc4a19d 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -51,6 +51,12 @@ namespaces = false
basic-pitch = "basic_pitch.predict:main"
[project.optional-dependencies]
+train = [
+ "apache_beam",
+ "mirdata",
+ "smart_open",
+ "sox"
+]
test = [
"coverage>=5.0.2",
"pytest>=6.1.1",