Skip to content

A lightweight yet powerful audio-to-MIDI converter with pitch bend detection

add training files, add method typing with minor changes to code to support tying.

Benjie Genchel7c0268124e02
diff --git a/basic_pitch/callbacks.py b/basic_pitch/callbacks.py
new file mode 100644
index 0000000..55b492d
--- /dev/null
+++ b/basic_pitch/callbacks.py
@@ -0,0 +1,86 @@
+#!/usr/bin/env python
+# encoding: utf-8
+#
+# Copyright 2022 Spotify AB
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import logging
+import os
+
+from typing import Any, Dict
+
+import tensorflow as tf
+
+from basic_pitch import visualize
+
+
+class SavedModelCallback(tf.keras.callbacks.Callback):
+ def __init__(self, output_path: str, monitor: str):
+ self.output_savemodel_path = output_path
+ self.monitor = monitor # 'val_loss' typically
+ self.best_loss_so_far = 1000000.0
+
+ def on_epoch_end(self, epoch: int, logs: Dict[Any, Any]) -> None:
+ loss = logs.get(self.monitor)
+ if loss is None:
+ logging.warning("SaveModelCallback: monitored variable %s is not defined in logs, skipping" % self.monitor)
+ else:
+ if loss < self.best_loss_so_far:
+ output_path = os.path.join(
+ self.output_savemodel_path,
+ "%d/model" % epoch,
+ )
+ logging.info("SaveModelCallback: saving model at iteration %d in %s" % (epoch, output_path))
+ tf.saved_model.save(self.model, output_path)
+ self.best_loss_so_far = loss
+
+
+class VisualizeCallback(tf.keras.callbacks.Callback):
+ # TODO RACHEL make this WAY faster
+ def __init__(
+ self,
+ train_ds: tf.data.Dataset,
+ validation_ds: tf.data.Dataset,
+ tensorboard_dir: str,
+ original_validation_ds: tf.data.Dataset,
+ contours: bool,
+ ):
+ super(VisualizeCallback, self).__init__()
+ self.train_iter = iter(train_ds)
+ self.validation_iter = iter(validation_ds)
+ self.validation_ds = original_validation_ds
+ self.tensorboard_dir = os.path.join(tensorboard_dir, "tensorboard_logs")
+ self.file_writer = tf.summary.create_file_writer(tensorboard_dir)
+ self.contours = contours
+
+ def on_epoch_end(self, epoch: int, logs: Dict[Any, Any]) -> None:
+ # the first two outputs of generator needs to be the input and the targets
+ train_inputs, train_targets = next(self.train_iter)[:2]
+ validation_inputs, validation_targets = next(self.validation_iter)[:2]
+ for stage, inputs, targets, loss in [
+ ("train", train_inputs, train_targets, logs["loss"]),
+ ("validation", validation_inputs, validation_targets, logs["val_loss"]),
+ ]:
+ outputs = self.model.predict(inputs)
+ visualize.visualize_transcription(
+ self.file_writer,
+ stage,
+ inputs,
+ targets,
+ outputs,
+ loss,
+ epoch,
+ self.validation_ds,
+ contours=self.contours,
+ )
diff --git a/basic_pitch/data/tf_example_deserialization.py b/basic_pitch/data/tf_example_deserialization.py
new file mode 100644
index 0000000..dc60b54
--- /dev/null
+++ b/basic_pitch/data/tf_example_deserialization.py
@@ -0,0 +1,697 @@
+#!/usr/bin/env python
+# encoding: utf-8
+#
+# Copyright 2024 Spotify AB
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import uuid
+from typing import Any, Callable, Dict, Iterator, List, Optional, Tuple
+
+import numpy as np
+import tensorflow as tf
+
+# import tensorflow_addons as tfa
+
+from basic_pitch.constants import (
+ ANNOTATIONS_FPS,
+ ANNOT_N_FRAMES,
+ AUDIO_N_CHANNELS,
+ AUDIO_N_SAMPLES,
+ AUDIO_SAMPLE_RATE,
+ AUDIO_WINDOW_LENGTH,
+ N_FREQ_BINS_NOTES,
+ N_FREQ_BINS_CONTOURS,
+)
+
+N_SAMPLES_PER_TRACK = 20
+
+
+def prepare_datasets(
+ datasets_base_path: str,
+ training_shuffle_buffer_size: int,
+ batch_size: int,
+ validation_steps: int,
+ datasets_to_use: List[str],
+ dataset_sampling_frequency: np.ndarray,
+) -> Tuple[tf.data.Dataset, tf.data.Dataset]:
+ """
+ Return a training and a testing dataset.
+
+ Args:
+ training_shuffle_buffer_size : size of shuffle buffer (only for training set)
+ batch_size : ..
+ """
+ assert batch_size > 0
+ assert validation_steps is not None and validation_steps > 0
+ assert training_shuffle_buffer_size is not None
+
+ # init both
+ ds_train = sample_datasets(
+ "train",
+ datasets_base_path,
+ datasets=datasets_to_use,
+ dataset_sampling_frequency=dataset_sampling_frequency,
+ )
+ ds_validation = sample_datasets(
+ "validation",
+ datasets_base_path,
+ datasets=datasets_to_use,
+ dataset_sampling_frequency=dataset_sampling_frequency,
+ )
+
+ # check that the base dataset returned by ds_function is FINITE
+ for ds in [ds_train, ds_validation]:
+ tf.debugging.assert_none_equal(
+ tf.cast(tf.data.experimental.cardinality(ds), tf.int32),
+ tf.data.experimental.INFINITE_CARDINALITY,
+ )
+
+ # training dataset
+ if training_shuffle_buffer_size > 0:
+ # Lets try to cache before the shuffle. This is the entire training dataset so we'll cache
+ # to memory
+ ds_train = (
+ ds_train.shuffle(training_shuffle_buffer_size, reshuffle_each_iteration=True)
+ .repeat()
+ .batch(batch_size)
+ .prefetch(tf.data.AUTOTUNE)
+ )
+
+ # validation dataset
+ ds_validation = (
+ ds_validation.repeat()
+ .batch(batch_size)
+ .take(validation_steps)
+ .cache(f"validation_set_cache_{str(uuid.uuid4())}")
+ .repeat()
+ .prefetch(tf.data.AUTOTUNE)
+ )
+
+ return ds_train, ds_validation
+
+
+def prepare_visualization_datasets(
+ datasets_base_path: str,
+ batch_size: int,
+ validation_steps: int,
+ datasets_to_use: List[str],
+ dataset_sampling_frequency: np.ndarray,
+) -> Tuple[tf.data.Dataset, tf.data.Dataset]:
+ """
+ Return a training and a testing dataset.
+
+ Args:
+ training_shuffle_buffer_size : size of shuffle buffer (only for training set)
+ batch_size : ..
+ """
+
+ assert batch_size > 0
+ assert validation_steps is not None and validation_steps > 0
+
+ ds_train = sample_datasets(
+ "train",
+ datasets_base_path,
+ datasets=datasets_to_use,
+ dataset_sampling_frequency=dataset_sampling_frequency,
+ n_samples_per_track=1,
+ )
+ ds_validation = sample_datasets(
+ "validation",
+ datasets_base_path,
+ datasets=datasets_to_use,
+ dataset_sampling_frequency=dataset_sampling_frequency,
+ n_samples_per_track=1,
+ )
+
+ # check that the base dataset returned by ds_function is FINITE
+ for ds in [ds_train, ds_validation]:
+ tf.debugging.assert_none_equal(
+ tf.cast(tf.data.experimental.cardinality(ds), tf.int32),
+ tf.data.experimental.INFINITE_CARDINALITY,
+ )
+
+ # training dataset
+ ds_train = ds_train.repeat().batch(batch_size).prefetch(tf.data.AUTOTUNE)
+
+ # validation dataset
+ ds_validation = (
+ ds_validation.repeat()
+ .batch(batch_size)
+ .take(validation_steps)
+ .cache(f"validation_set_cache_{str(uuid.uuid4())}")
+ .repeat()
+ .prefetch(tf.data.AUTOTUNE)
+ )
+
+ return ds_train, ds_validation
+
+
+def sample_datasets(
+ split: str,
+ datasets_base_path: str,
+ datasets: List[str],
+ dataset_sampling_frequency: np.ndarray,
+ n_shuffle: int = 1000,
+ n_samples_per_track: int = N_SAMPLES_PER_TRACK,
+ pairs: bool = False,
+) -> tf.data.Dataset:
+ assert split in ["train", "validation"]
+ if split == "validation":
+ n_shuffle = 0
+ pairs = False
+ if n_samples_per_track != 1:
+ n_samples_per_track = 5
+
+ ds_list = []
+
+ file_generator, random_seed = transcription_file_generator(
+ split,
+ datasets,
+ datasets_base_path,
+ dataset_sampling_frequency,
+ )
+
+ ds_dataset = transcription_dataset(file_generator, n_samples_per_track, random_seed)
+
+ if n_shuffle > 0:
+ ds_dataset = ds_dataset.shuffle(n_shuffle)
+ ds_list.append(ds_dataset)
+
+ if pairs:
+ pairs_generator, random_seed_pairs = transcription_file_generator(
+ split,
+ datasets,
+ datasets_base_path,
+ dataset_sampling_frequency,
+ )
+ pairs_ds = transcription_dataset(
+ pairs_generator,
+ n_samples_per_track,
+ random_seed_pairs,
+ )
+
+ pairs_ds = pairs_ds.shuffle(n_samples_per_track * 10) # shuffle so that different tracks get mixed together
+ pairs_ds = pairs_ds.batch(2)
+ pairs_ds = pairs_ds.map(combine_transcription_examples)
+ ds_list.append(pairs_ds)
+
+ n_datasets = len(ds_list)
+ choice_dataset = tf.data.Dataset.range(
+ n_datasets
+ ).repeat() # this repeat is critical! if not, only n_dataset points will be sampled!!
+ return tf.data.Datasets.choose_from_datasets(ds_list, choice_dataset)
+
+
+def transcription_file_generator(
+ split: str,
+ dataset_names: List[str],
+ datasets_base_path: str,
+ sample_weights: np.ndarray,
+) -> Tuple[Callable[[], Iterator[str]], bool]:
+ """
+ dataset_names: list of dataset dataset_names
+ """
+ file_dict = {
+ dataset_name: tf.data.Dataset.list_files(
+ os.path.join(datasets_base_path, dataset_name, "splits", split, "*tfrecord")
+ )
+ for dataset_name in dataset_names
+ }
+
+ if split == "train":
+ return lambda: _train_file_generator(file_dict, sample_weights), False
+ return lambda: _validation_file_generator(file_dict), True
+
+
+def _train_file_generator(x: Dict[str, List[str]], weights: np.ndarray) -> Iterator[str]:
+ x = {k: list(v) for (k, v) in x.items()}
+ keys = list(x.keys())
+ # shuffle each list
+ for k in keys:
+ np.random.shuffle(x[k])
+
+ while all(x.values()):
+ # choose a random dataset and yield the last file
+ fpath = x[np.random.choice(keys, p=weights)].pop()
+ yield fpath
+
+
+def _validation_file_generator(x: Dict[str, tf.data.Dataset]) -> Iterator[str]:
+ x = {k: list(v) for (k, v) in x.items()}
+ # loop until there are no more test files
+ while any(x.values()):
+ # alternate between datasets (dataset 1 elt 1, dataset 2, elt 1, ...)
+ # this is so test files in the tensorboard have 4 different datasets
+ # instead of 4 elements from 1
+ for k in x:
+ # if the list of files for this dataset is empty skip it
+ if x[k]:
+ yield x[k].pop()
+
+
+def combine_transcription_examples(
+ a: tf.Tensor, target: Dict[str, tf.Tensor], w: Dict[str, tf.Tensor]
+) -> Tuple[tf.Tensor, Dict[str, tf.Tensor], Dict[str, tf.Tensor]]:
+ return (
+ # mix the audio snippets
+ tf.math.reduce_mean(a, axis=0),
+ # annotations are the max per bin - active frames stay active
+ {
+ "onset": tf.math.reduce_max(target["onset"], axis=0),
+ "contour": tf.math.reduce_max(target["contour"], axis=0),
+ "note": tf.math.reduce_max(target["note"], axis=0),
+ },
+ # weights are the minimum - if an annotation is missing in one, we should set the weights to zero
+ {
+ "onset": tf.math.reduce_min(w["onset"], axis=0),
+ "contour": tf.math.reduce_min(w["contour"], axis=0),
+ "note": tf.math.reduce_min(w["note"], axis=0),
+ },
+ )
+
+
+def transcription_dataset(
+ file_generator: Callable[[], Iterator[str]], n_samples_per_track: int, random_seed: bool
+) -> tf.data.Dataset:
+ """
+ `fpaths_in` is a list of .tfrecords files
+ return a tf.Dataset with the following fields (as tuple):
+ - audio (shape AUDIO_N_SAMPLES, 1)
+ - {'contours': contours, 'notes': notes, 'onsets': onsets}
+ contours has shape (ANNOT_N_FRAMES, N_FREQ_BINS_CONTOURS)
+ notes and onsets have shape: (ANNOT_N_FRAMES, N_FREQ_BINS_NOTES)
+ """
+ ds = tf.data.Dataset.from_generator(file_generator, output_types=tf.string, output_shapes=())
+ ds = tf.data.TFRecordDataset(ds)
+ ds = ds.map(parse_transcription_tfexample, num_parallel_calls=tf.data.AUTOTUNE)
+ ds = ds.filter(is_not_bad_shape)
+ ds = ds.map(
+ lambda file_id, source, audio_wav, notes_indices, notes_values, onsets_indices, onsets_values, contours_indices, contours_values, notes_onsets_shape, contours_shape: ( # noqa: E501
+ file_id,
+ source,
+ tf.audio.decode_wav(
+ audio_wav,
+ desired_channels=AUDIO_N_CHANNELS,
+ desired_samples=-1,
+ name=None,
+ ),
+ sparse2dense(notes_values, notes_indices, notes_onsets_shape),
+ sparse2dense(onsets_values, onsets_indices, notes_onsets_shape),
+ sparse2dense(contours_values, contours_indices, contours_shape),
+ ),
+ num_parallel_calls=tf.data.AUTOTUNE,
+ )
+ ds = ds.map(reduce_transcription_inputs)
+ ds = ds.map(get_sample_weights, num_parallel_calls=tf.data.AUTOTUNE)
+ ds = ds.flat_map(
+ lambda a, o, c, n, ow, cw, nw, m: get_transcription_chunks(
+ a, o, c, n, ow, cw, nw, n_samples_per_track, random_seed
+ )
+ )
+ ds = ds.filter(is_not_all_silent_annotations) # remove examples where all annotations are zero
+ ds = ds.map(to_transcription_training_input)
+ ds = ds.apply(tf.data.experimental.ignore_errors(log_warning=True)) # failsafe so training doesn't stop
+ return ds
+
+
+def parse_transcription_tfexample(
+ serialized_example: tf.train.Example,
+) -> Tuple[
+ tf.Tensor,
+ tf.Tensor,
+ tf.Tensor,
+ tf.Tensor,
+ tf.Tensor,
+ tf.Tensor,
+ tf.Tensor,
+ tf.Tensor,
+ tf.Tensor,
+ tf.Tensor,
+ tf.Tensor,
+]:
+ """
+ return a tuple with the following tensors, in order:
+ - file_id
+ - source
+ - audio_wav
+ - notes_indices
+ - notes_values
+ - onsets_indices
+ - onsets_values
+ - contours_indices
+ - contours_values
+ - notes_onsets_shape
+ - contours_shape
+ NB.: notes, onsets and contours are represented as sparse matrices
+ (to be reconstructed using `tf.SparseTensor(...)`). They share the
+ time dimension, while contours have a frequency dimension that is
+ a multiple (`ANNOTATIONS_BINS_PER_SEMITONE`) of that of
+ notes/onsets.
+ """
+ schema = {
+ "file_id": tf.io.FixedLenFeature((), tf.string),
+ "source": tf.io.FixedLenFeature((), tf.string),
+ "audio_wav": tf.io.FixedLenFeature((), tf.string),
+ "notes_indices": tf.io.FixedLenFeature((), tf.string),
+ "notes_values": tf.io.FixedLenFeature((), tf.string),
+ "onsets_indices": tf.io.FixedLenFeature((), tf.string),
+ "onsets_values": tf.io.FixedLenFeature((), tf.string),
+ "contours_indices": tf.io.FixedLenFeature((), tf.string),
+ "contours_values": tf.io.FixedLenFeature((), tf.string),
+ "notes_onsets_shape": tf.io.FixedLenFeature((), tf.string),
+ "contours_shape": tf.io.FixedLenFeature((), tf.string),
+ }
+ example = tf.io.parse_single_example(serialized_example, schema)
+ return (
+ example["file_id"],
+ example["source"],
+ example["audio_wav"],
+ tf.io.parse_tensor(example["notes_indices"], out_type=tf.int64),
+ tf.io.parse_tensor(example["notes_values"], out_type=tf.float32),
+ tf.io.parse_tensor(example["onsets_indices"], out_type=tf.int64),
+ tf.io.parse_tensor(example["onsets_values"], out_type=tf.float32),
+ tf.io.parse_tensor(example["contours_indices"], out_type=tf.int64),
+ tf.io.parse_tensor(example["contours_values"], out_type=tf.float32),
+ tf.io.parse_tensor(example["notes_onsets_shape"], out_type=tf.int64),
+ tf.io.parse_tensor(example["contours_shape"], out_type=tf.int64),
+ )
+
+
+def is_not_bad_shape(
+ _file_id: tf.Tensor,
+ _source: tf.Tensor,
+ _audio_wav: tf.Tensor,
+ _notes_indices: tf.Tensor,
+ notes_values: tf.Tensor,
+ _onsets_indices: tf.Tensor,
+ _onsets_values: tf.Tensor,
+ _contours_indices: tf.Tensor,
+ _contours_values: tf.Tensor,
+ notes_onsets_shape: tf.Tensor,
+ _contours_shape: tf.Tensor,
+) -> tf.Tensor:
+ bad_shape = tf.logical_and(
+ tf.shape(notes_values)[0] == 0,
+ tf.shape(notes_onsets_shape)[0] == 2,
+ )
+ return tf.math.logical_not(bad_shape)
+
+
+def sparse2dense(values: tf.Tensor, indices: tf.Tensor, dense_shape: tf.Tensor) -> tf.Tensor:
+ if tf.rank(indices) != 2 and tf.size(indices) == 0:
+ indices = tf.zeros([0, 1], dtype=indices.dtype)
+ tf.assert_rank(indices, 2)
+ tf.assert_rank(values, 1)
+ tf.assert_rank(dense_shape, 1)
+ sp = tf.SparseTensor(indices=indices, values=values, dense_shape=dense_shape)
+ return tf.sparse.to_dense(sp, validate_indices=False)
+
+
+def reduce_transcription_inputs(
+ file_id: str,
+ src: str,
+ wav: Tuple[tf.Tensor, int],
+ notes: tf.Tensor,
+ onsets: tf.Tensor,
+ contour: tf.Tensor,
+) -> Tuple[tf.Tensor, tf.Tensor, tf.Tensor, tf.Tensor, Dict[str, str]]:
+ """Map tf records data to a tuple
+ If audio is stereo, it is mixed down to mono.
+ This will error if the sample rate of the wav file is different from
+ what we hard code.
+ Args:
+ fid : file id (string)
+ src : name of dataset (string)
+ wav : tensorflow wav obejct (tuple of audio and sample rate)
+ The whole audio file length
+ notes : matrix of note frames (n_frames x N_FREQ_BINS_NOTES)
+ possibly size 0
+ onsets : matrix of note onsets (n_frames x N_FREQ_BINS_NOTES)
+ possibly size 0
+ contours : matrix of contour frames (n_frames x N_FREQ_BINS_CONTOURS)
+ possibly size 0
+ """
+ audio, sample_rate = wav
+ tf.debugging.assert_equal(
+ sample_rate,
+ AUDIO_SAMPLE_RATE,
+ message="audio sample rate {} is inconsistent".format(sample_rate),
+ )
+ return (
+ tf.math.reduce_mean(audio, axis=1, keepdims=True), # manually mixdown to mono
+ onsets,
+ contour,
+ notes,
+ {"fid": file_id, "src": src},
+ )
+
+
+def _infer_time_size(onsets: tf.Tensor, contour: tf.Tensor, notes: tf.Tensor) -> tf.Tensor:
+ """Some of the targets might be empty, but we need to find out the
+ number of time frames of one of the non-empty ones.
+ Returns
+ number of time frames in the targets
+ """
+ onset_shape = tf.shape(onsets)[0]
+ contour_shape = tf.shape(contour)[0]
+ note_shape = tf.shape(notes)[0]
+ time_size = tf.cast(
+ tf.math.maximum(
+ tf.cast(tf.math.maximum(onset_shape, contour_shape), dtype=tf.int32),
+ note_shape,
+ ),
+ dtype=tf.int32,
+ )
+
+ return time_size
+
+
+def get_sample_weights(
+ audio: tf.Tensor, onsets: np.ndarray, contour: np.ndarray, notes: np.ndarray, metadata: Dict[Any, Any]
+) -> Tuple[tf.Tensor, np.ndarray, np.ndarray, np.ndarray, tf.cond, tf.cond, tf.cond, Dict[Any, Any]]:
+ """Add sample weights based on whether or not the target is empty
+ If it's empty, the weight is 0, otherwise it's 1. Empty targets get filled
+ with matricies of 0's
+ Args:
+ audio : audio signal (full length)
+ notes : matrix of note frames (n_frames x N_FREQ_BINS_NOTES)
+ possibly size 0
+ onsets : matrix of note onsets (n_frames x N_FREQ_BINS_NOTES)
+ possibly size 0
+ contours : matrix of contour frames (n_frames x N_FREQ_BINS_CONTOURS)
+ possibly size 0
+ metadata : dictionary of metadata
+ Returns:
+ audio : audio signal (full length)
+ notes : matrix of note frames (n_frames x N_FREQ_BINS_NOTES)
+ onsets : matrix of note onsets (n_frames x N_FREQ_BINS_NOTES)
+ contours : matrix of contour frames (n_frames x N_FREQ_BINS_CONTOURS)
+ onset_weight : int (0 or 1)
+ note_weight : int (0 or 1)
+ contour_weight : int (0 or 1)
+ """
+ time_size = _infer_time_size(onsets, contour, notes)
+
+ # TODO - if we dont want to worry about batches with no examples for a task
+ # we can add a tiny constant here, but training will be unstable
+ onsets_weight = tf.cast(tf.shape(onsets)[0] != 0, tf.float32)
+ contour_weight = tf.cast(tf.shape(contour)[0] != 0, tf.float32)
+ note_weight = tf.cast(tf.shape(notes)[0] != 0, tf.float32)
+
+ onsets = tf.cond(
+ tf.shape(onsets)[0] == 0,
+ lambda: tf.zeros(
+ tf.stack([time_size, tf.constant(N_FREQ_BINS_NOTES, dtype=tf.int32)], axis=0),
+ dtype=tf.float32,
+ ),
+ lambda: onsets,
+ )
+ contour = tf.cond(
+ tf.shape(contour)[0] == 0,
+ lambda: tf.zeros(
+ tf.stack([time_size, tf.constant(N_FREQ_BINS_CONTOURS, dtype=tf.int32)], axis=0),
+ dtype=tf.float32,
+ ),
+ lambda: contour,
+ )
+ notes = tf.cond(
+ tf.shape(notes)[0] == 0,
+ lambda: tf.zeros(
+ tf.stack([time_size, tf.constant(N_FREQ_BINS_NOTES, dtype=tf.int32)], axis=0),
+ dtype=tf.float32,
+ ),
+ lambda: notes,
+ )
+
+ return (
+ audio,
+ onsets,
+ contour,
+ notes,
+ onsets_weight,
+ contour_weight,
+ note_weight,
+ metadata,
+ )
+
+
+def trim_time(data: np.ndarray, start: int, duration: int, sr: int) -> tf.Tensor:
+ """
+ Slice a data file
+ Args:
+ data: 2D data as (n_time_samples, n_channels) array
+ can be audio or a time-frequency matrix
+ start: trim start time in seconds
+ duration: trim duration in seconds
+ sr: data sample rate
+ Returns:
+ sliced_data (tf.tensor): (trimmed_time, n_channels)
+ """
+ n_start = tf.cast(tf.math.round(sr * start), dtype=tf.int32)
+ n_duration = tf.cast(tf.math.ceil(tf.cast(sr * duration, tf.float32)), dtype=tf.int32)
+ begin = (n_start, 0)
+ size = (n_duration, -1)
+ return tf.slice(data, begin=begin, size=size)
+
+
+def extract_window(
+ audio: tf.Tensor, onsets: np.ndarray, contour: np.ndarray, notes: np.ndarray, t_start: int
+) -> Tuple[tf.Tensor, tf.Tensor, tf.Tensor, tf.Tensor]:
+ # needs a hop size extra of samples for good mel spectrogram alignment
+ audio_trim = trim_time(
+ audio,
+ t_start,
+ tf.cast(AUDIO_N_SAMPLES / AUDIO_SAMPLE_RATE, dtype=tf.dtypes.float32),
+ AUDIO_SAMPLE_RATE,
+ )
+ onset_trim = trim_time(onsets, t_start, AUDIO_WINDOW_LENGTH, ANNOTATIONS_FPS)
+ contour_trim = trim_time(contour, t_start, AUDIO_WINDOW_LENGTH, ANNOTATIONS_FPS)
+ note_trim = trim_time(notes, t_start, AUDIO_WINDOW_LENGTH, ANNOTATIONS_FPS)
+ return (audio_trim, onset_trim, contour_trim, note_trim)
+
+
+def extract_random_window(
+ audio: tf.Tensor, onsets: np.ndarray, contour: np.ndarray, notes: np.ndarray, seed: Optional[int]
+) -> Tuple[tf.Tensor, tf.Tensor, tf.Tensor, tf.Tensor]:
+ """Trim transcription data to a fixed length of time
+ starting from a random time index.
+ Args:
+ audio : audio signal (full length)
+ notes : matrix of note frames (n_frames x N_FREQ_BINS_NOTES)
+ onsets : matrix of note onsets (n_frames x N_FREQ_BINS_NOTES)
+ contours : matrix of contour frames (n_frames x N_FREQ_BINS_CONTOURS)
+ Returns:
+ audio : audio signal (AUDIO_WINDOW_LENGTH * AUDIO_SAMPLE_RATE, 1)
+ notes : matrix of note frames (AUDIO_SAMPLE_RATE * ANNOTATIONS_FPS, N_FREQ_BINS_NOTES)
+ onsets : matrix of note onsets (AUDIO_SAMPLE_RATE * ANNOTATIONS_FPS, N_FREQ_BINS_NOTES)
+ contours : matrix of contour frames (AUDIO_SAMPLE_RATE * ANNOTATIONS_FPS, N_FREQ_BINS_CONTOURS)
+ """
+ n_sec = tf.math.divide(
+ tf.cast(tf.shape(audio)[0], dtype=tf.float32),
+ tf.cast(AUDIO_SAMPLE_RATE, dtype=tf.float32),
+ )
+ t_start = tf.random.uniform(
+ (),
+ minval=0.0,
+ maxval=n_sec - (AUDIO_N_SAMPLES / AUDIO_SAMPLE_RATE),
+ dtype=tf.dtypes.float32,
+ seed=seed,
+ )
+ return extract_window(audio, onsets, contour, notes, t_start)
+
+
+def get_transcription_chunks(
+ audio: tf.Tensor,
+ onsets: np.ndarray,
+ contour: np.ndarray,
+ notes: np.ndarray,
+ onset_weight: int,
+ contour_weight: int,
+ note_weight: int,
+ n_samples_per_track: int,
+ seed: bool,
+) -> tf.data.Dataset:
+ """Randomly sample fixed-length time chunks for transcription data
+ Args:
+ audio : audio signal (full length)
+ notes : matrix of note frames (n_frames x N_FREQ_BINS_NOTES)
+ onsets : matrix of note onsets (n_frames x N_FREQ_BINS_NOTES)
+ contours : matrix of contour frames (n_frames x N_FREQ_BINS_CONTOURS)
+ onset_weight : int (0 or 1)
+ note_weight : int (0 or 1)
+ contour_weight : int (0 or 1)
+ n_samples_per_track : int - how many samples to yield per track
+ Returns:
+ batches of size n_samples_per_track of:
+ audio : audio signal (AUDIO_WINDOW_LENGTH * AUDIO_SAMPLE_RATE, 1)
+ notes : matrix of note frames (AUDIO_SAMPLE_RATE * ANNOTATIONS_FPS, N_FREQ_BINS_NOTES)
+ onsets : matrix of note onsets (AUDIO_SAMPLE_RATE * ANNOTATIONS_FPS, N_FREQ_BINS_NOTES)
+ contours : matrix of contour frames (AUDIO_SAMPLE_RATE * ANNOTATIONS_FPS, N_FREQ_BINS_CONTOURS)
+ onset_weight : int (0 or 1)
+ note_weight : int (0 or 1)
+ contour_weight : int (0 or 1)
+ """
+ a = []
+ o = []
+ c = []
+ n = []
+ ow = []
+ cw = []
+ nw = []
+ for i in range(n_samples_per_track):
+ s0, s1, s2, s3 = extract_random_window(audio, onsets, contour, notes, i if seed else None)
+ a.append(s0)
+ o.append(s1)
+ c.append(s2)
+ n.append(s3)
+ ow.append(onset_weight)
+ cw.append(contour_weight)
+ nw.append(note_weight)
+ return tf.data.Dataset.from_tensor_slices((a, o, c, n, ow, cw, nw))
+
+
+def is_not_all_silent_annotations(
+ audio: tf.Tensor,
+ onsets: np.ndarray,
+ contour: np.ndarray,
+ notes: np.ndarray,
+ onset_weight: int,
+ contour_weight: int,
+ note_weight: int,
+) -> tf.Tensor:
+ contours_nonsilent = tf.math.reduce_mean(contour) != 0
+ notes_nonsilent = tf.math.reduce_mean(notes) != 0
+ return tf.math.logical_or(contours_nonsilent, notes_nonsilent)
+
+
+def to_transcription_training_input(
+ audio: tf.Tensor,
+ onsets: np.ndarray,
+ contour: np.ndarray,
+ notes: np.ndarray,
+ onset_weight: int,
+ contour_weight: int,
+ note_weight: int,
+) -> Tuple[tf.Tensor, Dict[str, tf.Tensor], Dict[str, int]]:
+ """convert transcription data to the format expected by the model"""
+ return (
+ audio,
+ {
+ "onset": tf.ensure_shape(onsets, (ANNOT_N_FRAMES, N_FREQ_BINS_NOTES)),
+ "contour": tf.ensure_shape(contour, (ANNOT_N_FRAMES, N_FREQ_BINS_CONTOURS)),
+ "note": tf.ensure_shape(notes, (ANNOT_N_FRAMES, N_FREQ_BINS_NOTES)),
+ },
+ {"onset": onset_weight, "contour": contour_weight, "note": note_weight},
+ )
diff --git a/basic_pitch/train.py b/basic_pitch/train.py
new file mode 100644
index 0000000..e0191e5
--- /dev/null
+++ b/basic_pitch/train.py
@@ -0,0 +1,261 @@
+#!/usr/bin/env python
+# encoding: utf-8
+#
+# Copyright 2024 Spotify AB
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import argparse
+import os
+import logging
+from datetime import datetime, timezone
+from typing import List
+
+import numpy as np
+import tensorflow as tf
+
+from basic_pitch import models
+from basic_pitch.callbacks import VisualizeCallback
+from basic_pitch.constants import DATASET_SAMPLING_FREQUENCY
+from basic_pitch.data import tf_example_deserialization
+
+logging.basicConfig(level=logging.INFO)
+
+
+def main(
+ source: str,
+ output: str,
+ batch_size: int,
+ shuffle_size: int,
+ learning_rate: float,
+ epochs: int,
+ steps_per_epoch: int,
+ validation_steps: int,
+ size_evaluation_callback_datasets: int,
+ datasets_to_use: List[str],
+ dataset_sampling_frequency: np.ndarray,
+ no_contours: bool,
+ weighted_onset_loss: bool,
+ positive_onset_weight: float,
+) -> None:
+ """Parse config and run training or evaluation."""
+ # configuration.add_externals()
+ logging.info(f"source directory: {source}")
+ logging.info(f"output directory: {output}")
+ logging.info(f"tensorflow version: {tf.__version__}")
+ logging.info("parameters to train.main() function:")
+ logging.info(f"batch_size: {batch_size}")
+ logging.info(f"shuffle_size: {shuffle_size}")
+ logging.info(f"learning_rate: {learning_rate}")
+ logging.info(f"epochs: {epochs}")
+ logging.info(f"steps_per_epoch: {steps_per_epoch}")
+ logging.info(f"validation_steps: {validation_steps}")
+ logging.info(f"size_evaluation_callback_datasets: {size_evaluation_callback_datasets}")
+ logging.info(f"using datasets: {datasets_to_use} with frequencies {dataset_sampling_frequency}")
+ logging.info(f"no_contours: {no_contours}")
+ logging.info(f"weighted_onset_loss: {weighted_onset_loss}")
+ logging.info(f"positive_onset_weight: {positive_onset_weight}")
+
+ # model
+ model = models.model(no_contours=no_contours)
+ input_shape = list(model.input_shape)
+ if input_shape[0] is None:
+ input_shape[0] = batch_size
+ logging.info("input_shape" + str(input_shape))
+
+ output_shape = model.output_shape
+ for k, v in output_shape.items():
+ output_shape[k] = list(v)
+ if v[0] is None:
+ output_shape[k][0] = batch_size
+ logging.info("output_shape" + str(output_shape))
+ # data loaders
+ train_ds, validation_ds = tf_example_deserialization.prepare_datasets(
+ source,
+ shuffle_size,
+ batch_size,
+ validation_steps,
+ datasets_to_use,
+ dataset_sampling_frequency,
+ )
+
+ MAX_EVAL_CBF_BATCH_SIZE = 4
+ (
+ train_visualization_ds,
+ validation_visualization_ds,
+ ) = tf_example_deserialization.prepare_visualization_datasets(
+ source,
+ batch_size=min(size_evaluation_callback_datasets, MAX_EVAL_CBF_BATCH_SIZE),
+ validation_steps=max(1, size_evaluation_callback_datasets // MAX_EVAL_CBF_BATCH_SIZE),
+ datasets_to_use=datasets_to_use,
+ dataset_sampling_frequency=dataset_sampling_frequency,
+ )
+
+ timestamp = datetime.now(timezone.utc).strftime("%Y%m%d-%H%M")
+ tensorboard_log_dir = os.path.join(output, timestamp, "tensorboard")
+ callbacks = [
+ tf.keras.callbacks.TensorBoard(log_dir=tensorboard_log_dir, histogram_freq=1),
+ tf.keras.callbacks.EarlyStopping(patience=25, verbose=2),
+ tf.keras.callbacks.ReduceLROnPlateau(verbose=1, patience=10, factor=0.5),
+ tf.keras.callbacks.ModelCheckpoint(filepath=os.path.join(output, timestamp, "model.best"), save_best_only=True),
+ tf.keras.callbacks.ModelCheckpoint(
+ filepath=os.path.join(output, timestamp, "checkpoints", "model.{epoch:02d}")
+ ),
+ VisualizeCallback(
+ train_visualization_ds,
+ validation_visualization_ds,
+ tensorboard_log_dir,
+ validation_ds.take(validation_steps),
+ not no_contours,
+ ),
+ ]
+
+ # if no_contours:
+ # loss = models.loss_no_contour(weighted=weighted_onset_loss, positive_weight=positive_onset_weight)
+ # else:
+ # loss = models.loss(weighted=weighted_onset_loss, positive_weight=positive_onset_weight)
+ loss = models.loss(weighted=weighted_onset_loss, positive_weight=positive_onset_weight)
+
+ # train
+ model.compile(
+ loss=loss,
+ optimizer=tf.keras.optimizers.Adam(learning_rate),
+ sample_weight_mode={"contour": None, "note": None, "onset": None},
+ )
+
+ logging.info("--- Model Training specs ---")
+ logging.info(f" train_ds: {train_ds}")
+ logging.info(f" validation_ds: {validation_ds}")
+ model.summary()
+
+ model.fit(
+ train_ds,
+ epochs=epochs,
+ callbacks=callbacks,
+ steps_per_epoch=steps_per_epoch,
+ validation_data=validation_ds,
+ validation_steps=validation_steps,
+ )
+
+
+def console_entry_point() -> None:
+ """From pip installed script."""
+ parser = argparse.ArgumentParser(description="")
+ parser.add_argument("--source", help="Path to directory containing train/validation splits.")
+ parser.add_argument("--output", help="Directory to save the model in.")
+ parser.add_argument("-e", "--epochs", type=int, default=500, help="Number of training epochs.")
+ parser.add_argument(
+ "-b",
+ "--batch-size",
+ type=int,
+ default=16,
+ help="batch size of training. Unlike Estimator API, this specifies the batch size per-GPU.",
+ )
+ parser.add_argument(
+ "-l",
+ "--learning-rate",
+ type=float,
+ default=0.001,
+ help="ADAM optimizer learning rate",
+ )
+ parser.add_argument(
+ "-s",
+ "--steps-per-epoch",
+ type=int,
+ default=100,
+ help="steps_per_epoch (batch) of each training loop",
+ )
+ parser.add_argument(
+ "-v",
+ "--validation-steps",
+ type=int,
+ default=10,
+ help="validation steps (number of BATCHES) for each validation run. MUST be a positive integer",
+ )
+ parser.add_argument(
+ "-z",
+ "--training-shuffle-size",
+ type=int,
+ default=100,
+ help="training dataset shuffle size",
+ )
+ parser.add_argument(
+ "--size-evaluation-callback-datasets",
+ type=int,
+ default=4,
+ help="number of elements in the dataset used by the evaluation callback function",
+ )
+ for dataset in DATASET_SAMPLING_FREQUENCY.keys():
+ parser.add_argument(
+ f"--{dataset.lower()}",
+ action="store_true",
+ default=False,
+ help=f"Use {dataset} dataset in training",
+ )
+ parser.add_argument(
+ "--no-contours",
+ action="store_true",
+ default=False,
+ help="if given, trains without supervising the contour layer",
+ )
+ parser.add_argument(
+ "--weighted-onset-loss",
+ action="store_true",
+ default=False,
+ help="if given, trains onsets with a class-weighted loss",
+ )
+ parser.add_argument(
+ "--positive-onset-weight",
+ type=float,
+ default=0.5,
+ help="Positive class onset weight. Only applies when weignted onset loss is true.",
+ )
+
+ args = parser.parse_args()
+ datasets_to_use = [
+ dataset.lower()
+ for dataset in DATASET_SAMPLING_FREQUENCY.keys()
+ if getattr(args, dataset.lower().replace("-", "_"))
+ ]
+ dataset_sampling_frequency = np.array(
+ [
+ frequency
+ for dataset, frequency in DATASET_SAMPLING_FREQUENCY.items()
+ if getattr(args, dataset.lower().replace("-", "_"))
+ ]
+ )
+ dataset_sampling_frequency = dataset_sampling_frequency / np.sum(dataset_sampling_frequency)
+
+ assert args.steps_per_epoch is not None
+ assert args.validation_steps > 0
+
+ main(
+ args.source,
+ args.output,
+ args.training_shuffle_size,
+ args.batch_size,
+ args.learning_rate,
+ args.epochs,
+ args.steps_per_epoch,
+ args.validation_steps,
+ args.size_evaluation_callback_datasets,
+ datasets_to_use,
+ dataset_sampling_frequency,
+ args.no_contours,
+ args.weighted_onset_loss,
+ args.positive_onset_weight,
+ )
+
+
+if __name__ == "__main__":
+ console_entry_point()
diff --git a/basic_pitch/visualize.py b/basic_pitch/visualize.py
new file mode 100644
index 0000000..a9032ca
--- /dev/null
+++ b/basic_pitch/visualize.py
@@ -0,0 +1,211 @@
+#!/usr/bin/env python
+# encoding: utf-8
+#
+# Copyright 2022 Spotify AB
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import numpy as np
+import tensorflow as tf
+import mir_eval
+import librosa
+
+from typing import Dict
+
+from basic_pitch.constants import (
+ AUDIO_SAMPLE_RATE,
+ ANNOTATIONS_FPS,
+ ANNOTATIONS_N_SEMITONES,
+ ANNOTATIONS_BASE_FREQUENCY,
+ ANNOT_N_FRAMES,
+ NOTES_BINS_PER_SEMITONE,
+ AUDIO_N_SAMPLES,
+)
+from basic_pitch import models
+
+SONIFY_FS = 3000
+MAX_OUTPUTS = 4
+
+FREQS = librosa.core.cqt_frequencies(
+ n_bins=ANNOTATIONS_N_SEMITONES * NOTES_BINS_PER_SEMITONE,
+ fmin=ANNOTATIONS_BASE_FREQUENCY,
+ bins_per_octave=12 * NOTES_BINS_PER_SEMITONE,
+)
+# this function is slow - for speed, only sonify frequencies below
+# sonify_fs/2 Hz (e.g. 1000 Hz)
+MAX_FREQ_INDEX = np.where(FREQS > SONIFY_FS / 2)[0][0]
+TIMES = librosa.core.frames_to_time(
+ np.arange(ANNOT_N_FRAMES),
+ sr=AUDIO_SAMPLE_RATE,
+ hop_length=AUDIO_SAMPLE_RATE // ANNOTATIONS_FPS,
+)
+
+
+def get_input_model() -> tf.keras.Model:
+ inputs = tf.keras.Input(shape=(AUDIO_N_SAMPLES, 1)) # (batch, time, ch)
+ x = models.get_cqt(inputs, 1, False)
+ model = tf.keras.Model(inputs=inputs, outputs=x)
+ model.compile()
+ return model
+
+
+INPUT_MODEL = get_input_model()
+
+
+def visualize_transcription(
+ file_writer: tf.summary.SummaryWriter,
+ stage: str,
+ inputs: tf.Tensor,
+ targets: Dict[str, tf.Tensor],
+ outputs: Dict[str, tf.Tensor],
+ loss: float,
+ step: int,
+ sonify: bool = True,
+ contours: bool = True,
+) -> None:
+ """Create tf.summaries of transcription outputs to be plotted in tensorboard
+
+ Args:
+ file_writer: tensorboard filewriter object
+ stage: train or validation
+ inputs: batch of input data (audio)
+ targets: batch of target data (dictionary)
+ outputs: batch of output data (dictionary)
+ loss: loss value for epoch
+ step: which epoch this is
+
+ """
+ with file_writer.as_default():
+ # create audio player
+ tf.summary.audio(
+ f"{stage}/audio/inputs",
+ inputs,
+ sample_rate=AUDIO_SAMPLE_RATE,
+ step=step,
+ max_outputs=MAX_OUTPUTS,
+ )
+ # plot mel spectrograms
+ tf.summary.image(
+ f"{stage}/audio/input",
+ _audio_input(inputs),
+ step=step,
+ max_outputs=MAX_OUTPUTS,
+ )
+
+ # plot onsets
+ tf.summary.image(
+ f"{stage}/images/onsets/target",
+ _array_to_image(targets["onset"]),
+ step=step,
+ max_outputs=MAX_OUTPUTS,
+ )
+ tf.summary.image(
+ f"{stage}/images/onsets/output",
+ _array_to_image(outputs["onset"]),
+ step=step,
+ max_outputs=MAX_OUTPUTS,
+ )
+
+ if sonify:
+ tf.summary.audio(
+ f"{stage}/audio/onsets-output",
+ _array_to_sonification(outputs["onset"], MAX_OUTPUTS),
+ sample_rate=SONIFY_FS,
+ step=step,
+ max_outputs=MAX_OUTPUTS,
+ )
+
+ if contours:
+ # plot contours
+ tf.summary.image(
+ f"{stage}/images/contours/target",
+ _array_to_image(targets["contour"]),
+ step=step,
+ max_outputs=MAX_OUTPUTS,
+ )
+ tf.summary.image(
+ f"{stage}/images/contours/output",
+ _array_to_image(outputs["contour"]),
+ step=step,
+ max_outputs=MAX_OUTPUTS,
+ )
+
+ # plot notes
+ tf.summary.image(
+ f"{stage}/images/notes/target",
+ _array_to_image(targets["note"]),
+ step=step,
+ max_outputs=MAX_OUTPUTS,
+ )
+ tf.summary.image(
+ f"{stage}/images/notes/output",
+ _array_to_image(outputs["note"]),
+ step=step,
+ max_outputs=MAX_OUTPUTS,
+ )
+
+ if sonify:
+ # sonify notes
+ tf.summary.audio(
+ f"{stage}/audio/notes-output",
+ _array_to_sonification(outputs["note"], MAX_OUTPUTS),
+ sample_rate=SONIFY_FS,
+ step=step,
+ max_outputs=MAX_OUTPUTS,
+ )
+
+ # plot loss
+ tf.summary.scalar(f"{stage}/loss", loss, step=step)
+ # plot max
+ if contours:
+ tf.summary.scalar(f"{stage}/contour-max", np.max(outputs["contour"]), step=step)
+ tf.summary.scalar(f"{stage}/note-max", np.max(outputs["note"]), step=step)
+ tf.summary.scalar(f"{stage}/onset-max", np.max(outputs["onset"]), step=step)
+
+
+def _array_to_sonification(array: tf.Tensor, max_outputs: int, clip: float = 0.3) -> tf.Tensor:
+ gram_batch = tf.transpose(array, perm=[0, 2, 1]).numpy()
+ audio_list = []
+
+ for i, gram in enumerate(gram_batch):
+
+ gram[gram < clip] = 0.0
+ y = mir_eval.sonify.time_frequency(
+ gram[:MAX_FREQ_INDEX, :],
+ FREQS[:MAX_FREQ_INDEX],
+ TIMES,
+ fs=SONIFY_FS,
+ )
+ audio_list.append(y[:, np.newaxis])
+ if i + 1 >= max_outputs:
+ break
+
+ return tf.convert_to_tensor(np.array(audio_list), dtype=tf.float32)
+
+
+def _audio_input(audio: tf.Tensor) -> tf.Tensor:
+ audio_in = INPUT_MODEL(audio)
+ return tf.transpose(audio_in, perm=[0, 2, 1, 3])
+
+
+def _array_to_image(array: tf.Tensor) -> tf.Tensor:
+ """Convert a time-frequency array shape=(batch, time, frequency) to
+ the shape expected by tf.summary.image (batch, frequency, time, 1)
+
+ Args:
+ array: a (batch, time, frequency) array
+
+ Returns:
+ reshaped array
+ """
+ return tf.expand_dims(tf.transpose(array, perm=[0, 2, 1]), 3)