A lightweight yet powerful audio-to-MIDI converter with pitch bend detection
Python audiolightweightmachine-learningmidimusicpitch-detectionpolyphonicpythontranscriptiontypescript
add training files, add method typing with minor changes to code to support tying.
Benjie Genchel7c0268124e02
diff --git a/basic_pitch/callbacks.py b/basic_pitch/callbacks.pynew file mode 100644index 0000000..55b492d--- /dev/null+++ b/basic_pitch/callbacks.py@@ -0,0 +1,86 @@+#!/usr/bin/env python+# encoding: utf-8+#+# Copyright 2022 Spotify AB+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.++import logging+import os++from typing import Any, Dict++import tensorflow as tf++from basic_pitch import visualize+++class SavedModelCallback(tf.keras.callbacks.Callback):+ def __init__(self, output_path: str, monitor: str):+ self.output_savemodel_path = output_path+ self.monitor = monitor # 'val_loss' typically+ self.best_loss_so_far = 1000000.0++ def on_epoch_end(self, epoch: int, logs: Dict[Any, Any]) -> None:+ loss = logs.get(self.monitor)+ if loss is None:+ logging.warning("SaveModelCallback: monitored variable %s is not defined in logs, skipping" % self.monitor)+ else:+ if loss < self.best_loss_so_far:+ output_path = os.path.join(+ self.output_savemodel_path,+ "%d/model" % epoch,+ )+ logging.info("SaveModelCallback: saving model at iteration %d in %s" % (epoch, output_path))+ tf.saved_model.save(self.model, output_path)+ self.best_loss_so_far = loss+++class VisualizeCallback(tf.keras.callbacks.Callback):+ # TODO RACHEL make this WAY faster+ def __init__(+ self,+ train_ds: tf.data.Dataset,+ validation_ds: tf.data.Dataset,+ tensorboard_dir: str,+ original_validation_ds: tf.data.Dataset,+ contours: bool,+ ):+ super(VisualizeCallback, self).__init__()+ self.train_iter = iter(train_ds)+ self.validation_iter = iter(validation_ds)+ self.validation_ds = original_validation_ds+ self.tensorboard_dir = os.path.join(tensorboard_dir, "tensorboard_logs")+ self.file_writer = tf.summary.create_file_writer(tensorboard_dir)+ self.contours = contours++ def on_epoch_end(self, epoch: int, logs: Dict[Any, Any]) -> None:+ # the first two outputs of generator needs to be the input and the targets+ train_inputs, train_targets = next(self.train_iter)[:2]+ validation_inputs, validation_targets = next(self.validation_iter)[:2]+ for stage, inputs, targets, loss in [+ ("train", train_inputs, train_targets, logs["loss"]),+ ("validation", validation_inputs, validation_targets, logs["val_loss"]),+ ]:+ outputs = self.model.predict(inputs)+ visualize.visualize_transcription(+ self.file_writer,+ stage,+ inputs,+ targets,+ outputs,+ loss,+ epoch,+ self.validation_ds,+ contours=self.contours,+ )diff --git a/basic_pitch/data/tf_example_deserialization.py b/basic_pitch/data/tf_example_deserialization.pynew file mode 100644index 0000000..dc60b54--- /dev/null+++ b/basic_pitch/data/tf_example_deserialization.py@@ -0,0 +1,697 @@+#!/usr/bin/env python+# encoding: utf-8+#+# Copyright 2024 Spotify AB+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.++import os+import uuid+from typing import Any, Callable, Dict, Iterator, List, Optional, Tuple++import numpy as np+import tensorflow as tf++# import tensorflow_addons as tfa++from basic_pitch.constants import (+ ANNOTATIONS_FPS,+ ANNOT_N_FRAMES,+ AUDIO_N_CHANNELS,+ AUDIO_N_SAMPLES,+ AUDIO_SAMPLE_RATE,+ AUDIO_WINDOW_LENGTH,+ N_FREQ_BINS_NOTES,+ N_FREQ_BINS_CONTOURS,+)++N_SAMPLES_PER_TRACK = 20+++def prepare_datasets(+ datasets_base_path: str,+ training_shuffle_buffer_size: int,+ batch_size: int,+ validation_steps: int,+ datasets_to_use: List[str],+ dataset_sampling_frequency: np.ndarray,+) -> Tuple[tf.data.Dataset, tf.data.Dataset]:+ """+ Return a training and a testing dataset.++ Args:+ training_shuffle_buffer_size : size of shuffle buffer (only for training set)+ batch_size : ..+ """+ assert batch_size > 0+ assert validation_steps is not None and validation_steps > 0+ assert training_shuffle_buffer_size is not None++ # init both+ ds_train = sample_datasets(+ "train",+ datasets_base_path,+ datasets=datasets_to_use,+ dataset_sampling_frequency=dataset_sampling_frequency,+ )+ ds_validation = sample_datasets(+ "validation",+ datasets_base_path,+ datasets=datasets_to_use,+ dataset_sampling_frequency=dataset_sampling_frequency,+ )++ # check that the base dataset returned by ds_function is FINITE+ for ds in [ds_train, ds_validation]:+ tf.debugging.assert_none_equal(+ tf.cast(tf.data.experimental.cardinality(ds), tf.int32),+ tf.data.experimental.INFINITE_CARDINALITY,+ )++ # training dataset+ if training_shuffle_buffer_size > 0:+ # Lets try to cache before the shuffle. This is the entire training dataset so we'll cache+ # to memory+ ds_train = (+ ds_train.shuffle(training_shuffle_buffer_size, reshuffle_each_iteration=True)+ .repeat()+ .batch(batch_size)+ .prefetch(tf.data.AUTOTUNE)+ )++ # validation dataset+ ds_validation = (+ ds_validation.repeat()+ .batch(batch_size)+ .take(validation_steps)+ .cache(f"validation_set_cache_{str(uuid.uuid4())}")+ .repeat()+ .prefetch(tf.data.AUTOTUNE)+ )++ return ds_train, ds_validation+++def prepare_visualization_datasets(+ datasets_base_path: str,+ batch_size: int,+ validation_steps: int,+ datasets_to_use: List[str],+ dataset_sampling_frequency: np.ndarray,+) -> Tuple[tf.data.Dataset, tf.data.Dataset]:+ """+ Return a training and a testing dataset.++ Args:+ training_shuffle_buffer_size : size of shuffle buffer (only for training set)+ batch_size : ..+ """++ assert batch_size > 0+ assert validation_steps is not None and validation_steps > 0++ ds_train = sample_datasets(+ "train",+ datasets_base_path,+ datasets=datasets_to_use,+ dataset_sampling_frequency=dataset_sampling_frequency,+ n_samples_per_track=1,+ )+ ds_validation = sample_datasets(+ "validation",+ datasets_base_path,+ datasets=datasets_to_use,+ dataset_sampling_frequency=dataset_sampling_frequency,+ n_samples_per_track=1,+ )++ # check that the base dataset returned by ds_function is FINITE+ for ds in [ds_train, ds_validation]:+ tf.debugging.assert_none_equal(+ tf.cast(tf.data.experimental.cardinality(ds), tf.int32),+ tf.data.experimental.INFINITE_CARDINALITY,+ )++ # training dataset+ ds_train = ds_train.repeat().batch(batch_size).prefetch(tf.data.AUTOTUNE)++ # validation dataset+ ds_validation = (+ ds_validation.repeat()+ .batch(batch_size)+ .take(validation_steps)+ .cache(f"validation_set_cache_{str(uuid.uuid4())}")+ .repeat()+ .prefetch(tf.data.AUTOTUNE)+ )++ return ds_train, ds_validation+++def sample_datasets(+ split: str,+ datasets_base_path: str,+ datasets: List[str],+ dataset_sampling_frequency: np.ndarray,+ n_shuffle: int = 1000,+ n_samples_per_track: int = N_SAMPLES_PER_TRACK,+ pairs: bool = False,+) -> tf.data.Dataset:+ assert split in ["train", "validation"]+ if split == "validation":+ n_shuffle = 0+ pairs = False+ if n_samples_per_track != 1:+ n_samples_per_track = 5++ ds_list = []++ file_generator, random_seed = transcription_file_generator(+ split,+ datasets,+ datasets_base_path,+ dataset_sampling_frequency,+ )++ ds_dataset = transcription_dataset(file_generator, n_samples_per_track, random_seed)++ if n_shuffle > 0:+ ds_dataset = ds_dataset.shuffle(n_shuffle)+ ds_list.append(ds_dataset)++ if pairs:+ pairs_generator, random_seed_pairs = transcription_file_generator(+ split,+ datasets,+ datasets_base_path,+ dataset_sampling_frequency,+ )+ pairs_ds = transcription_dataset(+ pairs_generator,+ n_samples_per_track,+ random_seed_pairs,+ )++ pairs_ds = pairs_ds.shuffle(n_samples_per_track * 10) # shuffle so that different tracks get mixed together+ pairs_ds = pairs_ds.batch(2)+ pairs_ds = pairs_ds.map(combine_transcription_examples)+ ds_list.append(pairs_ds)++ n_datasets = len(ds_list)+ choice_dataset = tf.data.Dataset.range(+ n_datasets+ ).repeat() # this repeat is critical! if not, only n_dataset points will be sampled!!+ return tf.data.Datasets.choose_from_datasets(ds_list, choice_dataset)+++def transcription_file_generator(+ split: str,+ dataset_names: List[str],+ datasets_base_path: str,+ sample_weights: np.ndarray,+) -> Tuple[Callable[[], Iterator[str]], bool]:+ """+ dataset_names: list of dataset dataset_names+ """+ file_dict = {+ dataset_name: tf.data.Dataset.list_files(+ os.path.join(datasets_base_path, dataset_name, "splits", split, "*tfrecord")+ )+ for dataset_name in dataset_names+ }++ if split == "train":+ return lambda: _train_file_generator(file_dict, sample_weights), False+ return lambda: _validation_file_generator(file_dict), True+++def _train_file_generator(x: Dict[str, List[str]], weights: np.ndarray) -> Iterator[str]:+ x = {k: list(v) for (k, v) in x.items()}+ keys = list(x.keys())+ # shuffle each list+ for k in keys:+ np.random.shuffle(x[k])++ while all(x.values()):+ # choose a random dataset and yield the last file+ fpath = x[np.random.choice(keys, p=weights)].pop()+ yield fpath+++def _validation_file_generator(x: Dict[str, tf.data.Dataset]) -> Iterator[str]:+ x = {k: list(v) for (k, v) in x.items()}+ # loop until there are no more test files+ while any(x.values()):+ # alternate between datasets (dataset 1 elt 1, dataset 2, elt 1, ...)+ # this is so test files in the tensorboard have 4 different datasets+ # instead of 4 elements from 1+ for k in x:+ # if the list of files for this dataset is empty skip it+ if x[k]:+ yield x[k].pop()+++def combine_transcription_examples(+ a: tf.Tensor, target: Dict[str, tf.Tensor], w: Dict[str, tf.Tensor]+) -> Tuple[tf.Tensor, Dict[str, tf.Tensor], Dict[str, tf.Tensor]]:+ return (+ # mix the audio snippets+ tf.math.reduce_mean(a, axis=0),+ # annotations are the max per bin - active frames stay active+ {+ "onset": tf.math.reduce_max(target["onset"], axis=0),+ "contour": tf.math.reduce_max(target["contour"], axis=0),+ "note": tf.math.reduce_max(target["note"], axis=0),+ },+ # weights are the minimum - if an annotation is missing in one, we should set the weights to zero+ {+ "onset": tf.math.reduce_min(w["onset"], axis=0),+ "contour": tf.math.reduce_min(w["contour"], axis=0),+ "note": tf.math.reduce_min(w["note"], axis=0),+ },+ )+++def transcription_dataset(+ file_generator: Callable[[], Iterator[str]], n_samples_per_track: int, random_seed: bool+) -> tf.data.Dataset:+ """+ `fpaths_in` is a list of .tfrecords files+ return a tf.Dataset with the following fields (as tuple):+ - audio (shape AUDIO_N_SAMPLES, 1)+ - {'contours': contours, 'notes': notes, 'onsets': onsets}+ contours has shape (ANNOT_N_FRAMES, N_FREQ_BINS_CONTOURS)+ notes and onsets have shape: (ANNOT_N_FRAMES, N_FREQ_BINS_NOTES)+ """+ ds = tf.data.Dataset.from_generator(file_generator, output_types=tf.string, output_shapes=())+ ds = tf.data.TFRecordDataset(ds)+ ds = ds.map(parse_transcription_tfexample, num_parallel_calls=tf.data.AUTOTUNE)+ ds = ds.filter(is_not_bad_shape)+ ds = ds.map(+ lambda file_id, source, audio_wav, notes_indices, notes_values, onsets_indices, onsets_values, contours_indices, contours_values, notes_onsets_shape, contours_shape: ( # noqa: E501+ file_id,+ source,+ tf.audio.decode_wav(+ audio_wav,+ desired_channels=AUDIO_N_CHANNELS,+ desired_samples=-1,+ name=None,+ ),+ sparse2dense(notes_values, notes_indices, notes_onsets_shape),+ sparse2dense(onsets_values, onsets_indices, notes_onsets_shape),+ sparse2dense(contours_values, contours_indices, contours_shape),+ ),+ num_parallel_calls=tf.data.AUTOTUNE,+ )+ ds = ds.map(reduce_transcription_inputs)+ ds = ds.map(get_sample_weights, num_parallel_calls=tf.data.AUTOTUNE)+ ds = ds.flat_map(+ lambda a, o, c, n, ow, cw, nw, m: get_transcription_chunks(+ a, o, c, n, ow, cw, nw, n_samples_per_track, random_seed+ )+ )+ ds = ds.filter(is_not_all_silent_annotations) # remove examples where all annotations are zero+ ds = ds.map(to_transcription_training_input)+ ds = ds.apply(tf.data.experimental.ignore_errors(log_warning=True)) # failsafe so training doesn't stop+ return ds+++def parse_transcription_tfexample(+ serialized_example: tf.train.Example,+) -> Tuple[+ tf.Tensor,+ tf.Tensor,+ tf.Tensor,+ tf.Tensor,+ tf.Tensor,+ tf.Tensor,+ tf.Tensor,+ tf.Tensor,+ tf.Tensor,+ tf.Tensor,+ tf.Tensor,+]:+ """+ return a tuple with the following tensors, in order:+ - file_id+ - source+ - audio_wav+ - notes_indices+ - notes_values+ - onsets_indices+ - onsets_values+ - contours_indices+ - contours_values+ - notes_onsets_shape+ - contours_shape+ NB.: notes, onsets and contours are represented as sparse matrices+ (to be reconstructed using `tf.SparseTensor(...)`). They share the+ time dimension, while contours have a frequency dimension that is+ a multiple (`ANNOTATIONS_BINS_PER_SEMITONE`) of that of+ notes/onsets.+ """+ schema = {+ "file_id": tf.io.FixedLenFeature((), tf.string),+ "source": tf.io.FixedLenFeature((), tf.string),+ "audio_wav": tf.io.FixedLenFeature((), tf.string),+ "notes_indices": tf.io.FixedLenFeature((), tf.string),+ "notes_values": tf.io.FixedLenFeature((), tf.string),+ "onsets_indices": tf.io.FixedLenFeature((), tf.string),+ "onsets_values": tf.io.FixedLenFeature((), tf.string),+ "contours_indices": tf.io.FixedLenFeature((), tf.string),+ "contours_values": tf.io.FixedLenFeature((), tf.string),+ "notes_onsets_shape": tf.io.FixedLenFeature((), tf.string),+ "contours_shape": tf.io.FixedLenFeature((), tf.string),+ }+ example = tf.io.parse_single_example(serialized_example, schema)+ return (+ example["file_id"],+ example["source"],+ example["audio_wav"],+ tf.io.parse_tensor(example["notes_indices"], out_type=tf.int64),+ tf.io.parse_tensor(example["notes_values"], out_type=tf.float32),+ tf.io.parse_tensor(example["onsets_indices"], out_type=tf.int64),+ tf.io.parse_tensor(example["onsets_values"], out_type=tf.float32),+ tf.io.parse_tensor(example["contours_indices"], out_type=tf.int64),+ tf.io.parse_tensor(example["contours_values"], out_type=tf.float32),+ tf.io.parse_tensor(example["notes_onsets_shape"], out_type=tf.int64),+ tf.io.parse_tensor(example["contours_shape"], out_type=tf.int64),+ )+++def is_not_bad_shape(+ _file_id: tf.Tensor,+ _source: tf.Tensor,+ _audio_wav: tf.Tensor,+ _notes_indices: tf.Tensor,+ notes_values: tf.Tensor,+ _onsets_indices: tf.Tensor,+ _onsets_values: tf.Tensor,+ _contours_indices: tf.Tensor,+ _contours_values: tf.Tensor,+ notes_onsets_shape: tf.Tensor,+ _contours_shape: tf.Tensor,+) -> tf.Tensor:+ bad_shape = tf.logical_and(+ tf.shape(notes_values)[0] == 0,+ tf.shape(notes_onsets_shape)[0] == 2,+ )+ return tf.math.logical_not(bad_shape)+++def sparse2dense(values: tf.Tensor, indices: tf.Tensor, dense_shape: tf.Tensor) -> tf.Tensor:+ if tf.rank(indices) != 2 and tf.size(indices) == 0:+ indices = tf.zeros([0, 1], dtype=indices.dtype)+ tf.assert_rank(indices, 2)+ tf.assert_rank(values, 1)+ tf.assert_rank(dense_shape, 1)+ sp = tf.SparseTensor(indices=indices, values=values, dense_shape=dense_shape)+ return tf.sparse.to_dense(sp, validate_indices=False)+++def reduce_transcription_inputs(+ file_id: str,+ src: str,+ wav: Tuple[tf.Tensor, int],+ notes: tf.Tensor,+ onsets: tf.Tensor,+ contour: tf.Tensor,+) -> Tuple[tf.Tensor, tf.Tensor, tf.Tensor, tf.Tensor, Dict[str, str]]:+ """Map tf records data to a tuple+ If audio is stereo, it is mixed down to mono.+ This will error if the sample rate of the wav file is different from+ what we hard code.+ Args:+ fid : file id (string)+ src : name of dataset (string)+ wav : tensorflow wav obejct (tuple of audio and sample rate)+ The whole audio file length+ notes : matrix of note frames (n_frames x N_FREQ_BINS_NOTES)+ possibly size 0+ onsets : matrix of note onsets (n_frames x N_FREQ_BINS_NOTES)+ possibly size 0+ contours : matrix of contour frames (n_frames x N_FREQ_BINS_CONTOURS)+ possibly size 0+ """+ audio, sample_rate = wav+ tf.debugging.assert_equal(+ sample_rate,+ AUDIO_SAMPLE_RATE,+ message="audio sample rate {} is inconsistent".format(sample_rate),+ )+ return (+ tf.math.reduce_mean(audio, axis=1, keepdims=True), # manually mixdown to mono+ onsets,+ contour,+ notes,+ {"fid": file_id, "src": src},+ )+++def _infer_time_size(onsets: tf.Tensor, contour: tf.Tensor, notes: tf.Tensor) -> tf.Tensor:+ """Some of the targets might be empty, but we need to find out the+ number of time frames of one of the non-empty ones.+ Returns+ number of time frames in the targets+ """+ onset_shape = tf.shape(onsets)[0]+ contour_shape = tf.shape(contour)[0]+ note_shape = tf.shape(notes)[0]+ time_size = tf.cast(+ tf.math.maximum(+ tf.cast(tf.math.maximum(onset_shape, contour_shape), dtype=tf.int32),+ note_shape,+ ),+ dtype=tf.int32,+ )++ return time_size+++def get_sample_weights(+ audio: tf.Tensor, onsets: np.ndarray, contour: np.ndarray, notes: np.ndarray, metadata: Dict[Any, Any]+) -> Tuple[tf.Tensor, np.ndarray, np.ndarray, np.ndarray, tf.cond, tf.cond, tf.cond, Dict[Any, Any]]:+ """Add sample weights based on whether or not the target is empty+ If it's empty, the weight is 0, otherwise it's 1. Empty targets get filled+ with matricies of 0's+ Args:+ audio : audio signal (full length)+ notes : matrix of note frames (n_frames x N_FREQ_BINS_NOTES)+ possibly size 0+ onsets : matrix of note onsets (n_frames x N_FREQ_BINS_NOTES)+ possibly size 0+ contours : matrix of contour frames (n_frames x N_FREQ_BINS_CONTOURS)+ possibly size 0+ metadata : dictionary of metadata+ Returns:+ audio : audio signal (full length)+ notes : matrix of note frames (n_frames x N_FREQ_BINS_NOTES)+ onsets : matrix of note onsets (n_frames x N_FREQ_BINS_NOTES)+ contours : matrix of contour frames (n_frames x N_FREQ_BINS_CONTOURS)+ onset_weight : int (0 or 1)+ note_weight : int (0 or 1)+ contour_weight : int (0 or 1)+ """+ time_size = _infer_time_size(onsets, contour, notes)++ # TODO - if we dont want to worry about batches with no examples for a task+ # we can add a tiny constant here, but training will be unstable+ onsets_weight = tf.cast(tf.shape(onsets)[0] != 0, tf.float32)+ contour_weight = tf.cast(tf.shape(contour)[0] != 0, tf.float32)+ note_weight = tf.cast(tf.shape(notes)[0] != 0, tf.float32)++ onsets = tf.cond(+ tf.shape(onsets)[0] == 0,+ lambda: tf.zeros(+ tf.stack([time_size, tf.constant(N_FREQ_BINS_NOTES, dtype=tf.int32)], axis=0),+ dtype=tf.float32,+ ),+ lambda: onsets,+ )+ contour = tf.cond(+ tf.shape(contour)[0] == 0,+ lambda: tf.zeros(+ tf.stack([time_size, tf.constant(N_FREQ_BINS_CONTOURS, dtype=tf.int32)], axis=0),+ dtype=tf.float32,+ ),+ lambda: contour,+ )+ notes = tf.cond(+ tf.shape(notes)[0] == 0,+ lambda: tf.zeros(+ tf.stack([time_size, tf.constant(N_FREQ_BINS_NOTES, dtype=tf.int32)], axis=0),+ dtype=tf.float32,+ ),+ lambda: notes,+ )++ return (+ audio,+ onsets,+ contour,+ notes,+ onsets_weight,+ contour_weight,+ note_weight,+ metadata,+ )+++def trim_time(data: np.ndarray, start: int, duration: int, sr: int) -> tf.Tensor:+ """+ Slice a data file+ Args:+ data: 2D data as (n_time_samples, n_channels) array+ can be audio or a time-frequency matrix+ start: trim start time in seconds+ duration: trim duration in seconds+ sr: data sample rate+ Returns:+ sliced_data (tf.tensor): (trimmed_time, n_channels)+ """+ n_start = tf.cast(tf.math.round(sr * start), dtype=tf.int32)+ n_duration = tf.cast(tf.math.ceil(tf.cast(sr * duration, tf.float32)), dtype=tf.int32)+ begin = (n_start, 0)+ size = (n_duration, -1)+ return tf.slice(data, begin=begin, size=size)+++def extract_window(+ audio: tf.Tensor, onsets: np.ndarray, contour: np.ndarray, notes: np.ndarray, t_start: int+) -> Tuple[tf.Tensor, tf.Tensor, tf.Tensor, tf.Tensor]:+ # needs a hop size extra of samples for good mel spectrogram alignment+ audio_trim = trim_time(+ audio,+ t_start,+ tf.cast(AUDIO_N_SAMPLES / AUDIO_SAMPLE_RATE, dtype=tf.dtypes.float32),+ AUDIO_SAMPLE_RATE,+ )+ onset_trim = trim_time(onsets, t_start, AUDIO_WINDOW_LENGTH, ANNOTATIONS_FPS)+ contour_trim = trim_time(contour, t_start, AUDIO_WINDOW_LENGTH, ANNOTATIONS_FPS)+ note_trim = trim_time(notes, t_start, AUDIO_WINDOW_LENGTH, ANNOTATIONS_FPS)+ return (audio_trim, onset_trim, contour_trim, note_trim)+++def extract_random_window(+ audio: tf.Tensor, onsets: np.ndarray, contour: np.ndarray, notes: np.ndarray, seed: Optional[int]+) -> Tuple[tf.Tensor, tf.Tensor, tf.Tensor, tf.Tensor]:+ """Trim transcription data to a fixed length of time+ starting from a random time index.+ Args:+ audio : audio signal (full length)+ notes : matrix of note frames (n_frames x N_FREQ_BINS_NOTES)+ onsets : matrix of note onsets (n_frames x N_FREQ_BINS_NOTES)+ contours : matrix of contour frames (n_frames x N_FREQ_BINS_CONTOURS)+ Returns:+ audio : audio signal (AUDIO_WINDOW_LENGTH * AUDIO_SAMPLE_RATE, 1)+ notes : matrix of note frames (AUDIO_SAMPLE_RATE * ANNOTATIONS_FPS, N_FREQ_BINS_NOTES)+ onsets : matrix of note onsets (AUDIO_SAMPLE_RATE * ANNOTATIONS_FPS, N_FREQ_BINS_NOTES)+ contours : matrix of contour frames (AUDIO_SAMPLE_RATE * ANNOTATIONS_FPS, N_FREQ_BINS_CONTOURS)+ """+ n_sec = tf.math.divide(+ tf.cast(tf.shape(audio)[0], dtype=tf.float32),+ tf.cast(AUDIO_SAMPLE_RATE, dtype=tf.float32),+ )+ t_start = tf.random.uniform(+ (),+ minval=0.0,+ maxval=n_sec - (AUDIO_N_SAMPLES / AUDIO_SAMPLE_RATE),+ dtype=tf.dtypes.float32,+ seed=seed,+ )+ return extract_window(audio, onsets, contour, notes, t_start)+++def get_transcription_chunks(+ audio: tf.Tensor,+ onsets: np.ndarray,+ contour: np.ndarray,+ notes: np.ndarray,+ onset_weight: int,+ contour_weight: int,+ note_weight: int,+ n_samples_per_track: int,+ seed: bool,+) -> tf.data.Dataset:+ """Randomly sample fixed-length time chunks for transcription data+ Args:+ audio : audio signal (full length)+ notes : matrix of note frames (n_frames x N_FREQ_BINS_NOTES)+ onsets : matrix of note onsets (n_frames x N_FREQ_BINS_NOTES)+ contours : matrix of contour frames (n_frames x N_FREQ_BINS_CONTOURS)+ onset_weight : int (0 or 1)+ note_weight : int (0 or 1)+ contour_weight : int (0 or 1)+ n_samples_per_track : int - how many samples to yield per track+ Returns:+ batches of size n_samples_per_track of:+ audio : audio signal (AUDIO_WINDOW_LENGTH * AUDIO_SAMPLE_RATE, 1)+ notes : matrix of note frames (AUDIO_SAMPLE_RATE * ANNOTATIONS_FPS, N_FREQ_BINS_NOTES)+ onsets : matrix of note onsets (AUDIO_SAMPLE_RATE * ANNOTATIONS_FPS, N_FREQ_BINS_NOTES)+ contours : matrix of contour frames (AUDIO_SAMPLE_RATE * ANNOTATIONS_FPS, N_FREQ_BINS_CONTOURS)+ onset_weight : int (0 or 1)+ note_weight : int (0 or 1)+ contour_weight : int (0 or 1)+ """+ a = []+ o = []+ c = []+ n = []+ ow = []+ cw = []+ nw = []+ for i in range(n_samples_per_track):+ s0, s1, s2, s3 = extract_random_window(audio, onsets, contour, notes, i if seed else None)+ a.append(s0)+ o.append(s1)+ c.append(s2)+ n.append(s3)+ ow.append(onset_weight)+ cw.append(contour_weight)+ nw.append(note_weight)+ return tf.data.Dataset.from_tensor_slices((a, o, c, n, ow, cw, nw))+++def is_not_all_silent_annotations(+ audio: tf.Tensor,+ onsets: np.ndarray,+ contour: np.ndarray,+ notes: np.ndarray,+ onset_weight: int,+ contour_weight: int,+ note_weight: int,+) -> tf.Tensor:+ contours_nonsilent = tf.math.reduce_mean(contour) != 0+ notes_nonsilent = tf.math.reduce_mean(notes) != 0+ return tf.math.logical_or(contours_nonsilent, notes_nonsilent)+++def to_transcription_training_input(+ audio: tf.Tensor,+ onsets: np.ndarray,+ contour: np.ndarray,+ notes: np.ndarray,+ onset_weight: int,+ contour_weight: int,+ note_weight: int,+) -> Tuple[tf.Tensor, Dict[str, tf.Tensor], Dict[str, int]]:+ """convert transcription data to the format expected by the model"""+ return (+ audio,+ {+ "onset": tf.ensure_shape(onsets, (ANNOT_N_FRAMES, N_FREQ_BINS_NOTES)),+ "contour": tf.ensure_shape(contour, (ANNOT_N_FRAMES, N_FREQ_BINS_CONTOURS)),+ "note": tf.ensure_shape(notes, (ANNOT_N_FRAMES, N_FREQ_BINS_NOTES)),+ },+ {"onset": onset_weight, "contour": contour_weight, "note": note_weight},+ )diff --git a/basic_pitch/train.py b/basic_pitch/train.pynew file mode 100644index 0000000..e0191e5--- /dev/null+++ b/basic_pitch/train.py@@ -0,0 +1,261 @@+#!/usr/bin/env python+# encoding: utf-8+#+# Copyright 2024 Spotify AB+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.++import argparse+import os+import logging+from datetime import datetime, timezone+from typing import List++import numpy as np+import tensorflow as tf++from basic_pitch import models+from basic_pitch.callbacks import VisualizeCallback+from basic_pitch.constants import DATASET_SAMPLING_FREQUENCY+from basic_pitch.data import tf_example_deserialization++logging.basicConfig(level=logging.INFO)+++def main(+ source: str,+ output: str,+ batch_size: int,+ shuffle_size: int,+ learning_rate: float,+ epochs: int,+ steps_per_epoch: int,+ validation_steps: int,+ size_evaluation_callback_datasets: int,+ datasets_to_use: List[str],+ dataset_sampling_frequency: np.ndarray,+ no_contours: bool,+ weighted_onset_loss: bool,+ positive_onset_weight: float,+) -> None:+ """Parse config and run training or evaluation."""+ # configuration.add_externals()+ logging.info(f"source directory: {source}")+ logging.info(f"output directory: {output}")+ logging.info(f"tensorflow version: {tf.__version__}")+ logging.info("parameters to train.main() function:")+ logging.info(f"batch_size: {batch_size}")+ logging.info(f"shuffle_size: {shuffle_size}")+ logging.info(f"learning_rate: {learning_rate}")+ logging.info(f"epochs: {epochs}")+ logging.info(f"steps_per_epoch: {steps_per_epoch}")+ logging.info(f"validation_steps: {validation_steps}")+ logging.info(f"size_evaluation_callback_datasets: {size_evaluation_callback_datasets}")+ logging.info(f"using datasets: {datasets_to_use} with frequencies {dataset_sampling_frequency}")+ logging.info(f"no_contours: {no_contours}")+ logging.info(f"weighted_onset_loss: {weighted_onset_loss}")+ logging.info(f"positive_onset_weight: {positive_onset_weight}")++ # model+ model = models.model(no_contours=no_contours)+ input_shape = list(model.input_shape)+ if input_shape[0] is None:+ input_shape[0] = batch_size+ logging.info("input_shape" + str(input_shape))++ output_shape = model.output_shape+ for k, v in output_shape.items():+ output_shape[k] = list(v)+ if v[0] is None:+ output_shape[k][0] = batch_size+ logging.info("output_shape" + str(output_shape))+ # data loaders+ train_ds, validation_ds = tf_example_deserialization.prepare_datasets(+ source,+ shuffle_size,+ batch_size,+ validation_steps,+ datasets_to_use,+ dataset_sampling_frequency,+ )++ MAX_EVAL_CBF_BATCH_SIZE = 4+ (+ train_visualization_ds,+ validation_visualization_ds,+ ) = tf_example_deserialization.prepare_visualization_datasets(+ source,+ batch_size=min(size_evaluation_callback_datasets, MAX_EVAL_CBF_BATCH_SIZE),+ validation_steps=max(1, size_evaluation_callback_datasets // MAX_EVAL_CBF_BATCH_SIZE),+ datasets_to_use=datasets_to_use,+ dataset_sampling_frequency=dataset_sampling_frequency,+ )++ timestamp = datetime.now(timezone.utc).strftime("%Y%m%d-%H%M")+ tensorboard_log_dir = os.path.join(output, timestamp, "tensorboard")+ callbacks = [+ tf.keras.callbacks.TensorBoard(log_dir=tensorboard_log_dir, histogram_freq=1),+ tf.keras.callbacks.EarlyStopping(patience=25, verbose=2),+ tf.keras.callbacks.ReduceLROnPlateau(verbose=1, patience=10, factor=0.5),+ tf.keras.callbacks.ModelCheckpoint(filepath=os.path.join(output, timestamp, "model.best"), save_best_only=True),+ tf.keras.callbacks.ModelCheckpoint(+ filepath=os.path.join(output, timestamp, "checkpoints", "model.{epoch:02d}")+ ),+ VisualizeCallback(+ train_visualization_ds,+ validation_visualization_ds,+ tensorboard_log_dir,+ validation_ds.take(validation_steps),+ not no_contours,+ ),+ ]++ # if no_contours:+ # loss = models.loss_no_contour(weighted=weighted_onset_loss, positive_weight=positive_onset_weight)+ # else:+ # loss = models.loss(weighted=weighted_onset_loss, positive_weight=positive_onset_weight)+ loss = models.loss(weighted=weighted_onset_loss, positive_weight=positive_onset_weight)++ # train+ model.compile(+ loss=loss,+ optimizer=tf.keras.optimizers.Adam(learning_rate),+ sample_weight_mode={"contour": None, "note": None, "onset": None},+ )++ logging.info("--- Model Training specs ---")+ logging.info(f" train_ds: {train_ds}")+ logging.info(f" validation_ds: {validation_ds}")+ model.summary()++ model.fit(+ train_ds,+ epochs=epochs,+ callbacks=callbacks,+ steps_per_epoch=steps_per_epoch,+ validation_data=validation_ds,+ validation_steps=validation_steps,+ )+++def console_entry_point() -> None:+ """From pip installed script."""+ parser = argparse.ArgumentParser(description="")+ parser.add_argument("--source", help="Path to directory containing train/validation splits.")+ parser.add_argument("--output", help="Directory to save the model in.")+ parser.add_argument("-e", "--epochs", type=int, default=500, help="Number of training epochs.")+ parser.add_argument(+ "-b",+ "--batch-size",+ type=int,+ default=16,+ help="batch size of training. Unlike Estimator API, this specifies the batch size per-GPU.",+ )+ parser.add_argument(+ "-l",+ "--learning-rate",+ type=float,+ default=0.001,+ help="ADAM optimizer learning rate",+ )+ parser.add_argument(+ "-s",+ "--steps-per-epoch",+ type=int,+ default=100,+ help="steps_per_epoch (batch) of each training loop",+ )+ parser.add_argument(+ "-v",+ "--validation-steps",+ type=int,+ default=10,+ help="validation steps (number of BATCHES) for each validation run. MUST be a positive integer",+ )+ parser.add_argument(+ "-z",+ "--training-shuffle-size",+ type=int,+ default=100,+ help="training dataset shuffle size",+ )+ parser.add_argument(+ "--size-evaluation-callback-datasets",+ type=int,+ default=4,+ help="number of elements in the dataset used by the evaluation callback function",+ )+ for dataset in DATASET_SAMPLING_FREQUENCY.keys():+ parser.add_argument(+ f"--{dataset.lower()}",+ action="store_true",+ default=False,+ help=f"Use {dataset} dataset in training",+ )+ parser.add_argument(+ "--no-contours",+ action="store_true",+ default=False,+ help="if given, trains without supervising the contour layer",+ )+ parser.add_argument(+ "--weighted-onset-loss",+ action="store_true",+ default=False,+ help="if given, trains onsets with a class-weighted loss",+ )+ parser.add_argument(+ "--positive-onset-weight",+ type=float,+ default=0.5,+ help="Positive class onset weight. Only applies when weignted onset loss is true.",+ )++ args = parser.parse_args()+ datasets_to_use = [+ dataset.lower()+ for dataset in DATASET_SAMPLING_FREQUENCY.keys()+ if getattr(args, dataset.lower().replace("-", "_"))+ ]+ dataset_sampling_frequency = np.array(+ [+ frequency+ for dataset, frequency in DATASET_SAMPLING_FREQUENCY.items()+ if getattr(args, dataset.lower().replace("-", "_"))+ ]+ )+ dataset_sampling_frequency = dataset_sampling_frequency / np.sum(dataset_sampling_frequency)++ assert args.steps_per_epoch is not None+ assert args.validation_steps > 0++ main(+ args.source,+ args.output,+ args.training_shuffle_size,+ args.batch_size,+ args.learning_rate,+ args.epochs,+ args.steps_per_epoch,+ args.validation_steps,+ args.size_evaluation_callback_datasets,+ datasets_to_use,+ dataset_sampling_frequency,+ args.no_contours,+ args.weighted_onset_loss,+ args.positive_onset_weight,+ )+++if __name__ == "__main__":+ console_entry_point()diff --git a/basic_pitch/visualize.py b/basic_pitch/visualize.pynew file mode 100644index 0000000..a9032ca--- /dev/null+++ b/basic_pitch/visualize.py@@ -0,0 +1,211 @@+#!/usr/bin/env python+# encoding: utf-8+#+# Copyright 2022 Spotify AB+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.++import numpy as np+import tensorflow as tf+import mir_eval+import librosa++from typing import Dict++from basic_pitch.constants import (+ AUDIO_SAMPLE_RATE,+ ANNOTATIONS_FPS,+ ANNOTATIONS_N_SEMITONES,+ ANNOTATIONS_BASE_FREQUENCY,+ ANNOT_N_FRAMES,+ NOTES_BINS_PER_SEMITONE,+ AUDIO_N_SAMPLES,+)+from basic_pitch import models++SONIFY_FS = 3000+MAX_OUTPUTS = 4++FREQS = librosa.core.cqt_frequencies(+ n_bins=ANNOTATIONS_N_SEMITONES * NOTES_BINS_PER_SEMITONE,+ fmin=ANNOTATIONS_BASE_FREQUENCY,+ bins_per_octave=12 * NOTES_BINS_PER_SEMITONE,+)+# this function is slow - for speed, only sonify frequencies below+# sonify_fs/2 Hz (e.g. 1000 Hz)+MAX_FREQ_INDEX = np.where(FREQS > SONIFY_FS / 2)[0][0]+TIMES = librosa.core.frames_to_time(+ np.arange(ANNOT_N_FRAMES),+ sr=AUDIO_SAMPLE_RATE,+ hop_length=AUDIO_SAMPLE_RATE // ANNOTATIONS_FPS,+)+++def get_input_model() -> tf.keras.Model:+ inputs = tf.keras.Input(shape=(AUDIO_N_SAMPLES, 1)) # (batch, time, ch)+ x = models.get_cqt(inputs, 1, False)+ model = tf.keras.Model(inputs=inputs, outputs=x)+ model.compile()+ return model+++INPUT_MODEL = get_input_model()+++def visualize_transcription(+ file_writer: tf.summary.SummaryWriter,+ stage: str,+ inputs: tf.Tensor,+ targets: Dict[str, tf.Tensor],+ outputs: Dict[str, tf.Tensor],+ loss: float,+ step: int,+ sonify: bool = True,+ contours: bool = True,+) -> None:+ """Create tf.summaries of transcription outputs to be plotted in tensorboard++ Args:+ file_writer: tensorboard filewriter object+ stage: train or validation+ inputs: batch of input data (audio)+ targets: batch of target data (dictionary)+ outputs: batch of output data (dictionary)+ loss: loss value for epoch+ step: which epoch this is++ """+ with file_writer.as_default():+ # create audio player+ tf.summary.audio(+ f"{stage}/audio/inputs",+ inputs,+ sample_rate=AUDIO_SAMPLE_RATE,+ step=step,+ max_outputs=MAX_OUTPUTS,+ )+ # plot mel spectrograms+ tf.summary.image(+ f"{stage}/audio/input",+ _audio_input(inputs),+ step=step,+ max_outputs=MAX_OUTPUTS,+ )++ # plot onsets+ tf.summary.image(+ f"{stage}/images/onsets/target",+ _array_to_image(targets["onset"]),+ step=step,+ max_outputs=MAX_OUTPUTS,+ )+ tf.summary.image(+ f"{stage}/images/onsets/output",+ _array_to_image(outputs["onset"]),+ step=step,+ max_outputs=MAX_OUTPUTS,+ )++ if sonify:+ tf.summary.audio(+ f"{stage}/audio/onsets-output",+ _array_to_sonification(outputs["onset"], MAX_OUTPUTS),+ sample_rate=SONIFY_FS,+ step=step,+ max_outputs=MAX_OUTPUTS,+ )++ if contours:+ # plot contours+ tf.summary.image(+ f"{stage}/images/contours/target",+ _array_to_image(targets["contour"]),+ step=step,+ max_outputs=MAX_OUTPUTS,+ )+ tf.summary.image(+ f"{stage}/images/contours/output",+ _array_to_image(outputs["contour"]),+ step=step,+ max_outputs=MAX_OUTPUTS,+ )++ # plot notes+ tf.summary.image(+ f"{stage}/images/notes/target",+ _array_to_image(targets["note"]),+ step=step,+ max_outputs=MAX_OUTPUTS,+ )+ tf.summary.image(+ f"{stage}/images/notes/output",+ _array_to_image(outputs["note"]),+ step=step,+ max_outputs=MAX_OUTPUTS,+ )++ if sonify:+ # sonify notes+ tf.summary.audio(+ f"{stage}/audio/notes-output",+ _array_to_sonification(outputs["note"], MAX_OUTPUTS),+ sample_rate=SONIFY_FS,+ step=step,+ max_outputs=MAX_OUTPUTS,+ )++ # plot loss+ tf.summary.scalar(f"{stage}/loss", loss, step=step)+ # plot max+ if contours:+ tf.summary.scalar(f"{stage}/contour-max", np.max(outputs["contour"]), step=step)+ tf.summary.scalar(f"{stage}/note-max", np.max(outputs["note"]), step=step)+ tf.summary.scalar(f"{stage}/onset-max", np.max(outputs["onset"]), step=step)+++def _array_to_sonification(array: tf.Tensor, max_outputs: int, clip: float = 0.3) -> tf.Tensor:+ gram_batch = tf.transpose(array, perm=[0, 2, 1]).numpy()+ audio_list = []++ for i, gram in enumerate(gram_batch):++ gram[gram < clip] = 0.0+ y = mir_eval.sonify.time_frequency(+ gram[:MAX_FREQ_INDEX, :],+ FREQS[:MAX_FREQ_INDEX],+ TIMES,+ fs=SONIFY_FS,+ )+ audio_list.append(y[:, np.newaxis])+ if i + 1 >= max_outputs:+ break++ return tf.convert_to_tensor(np.array(audio_list), dtype=tf.float32)+++def _audio_input(audio: tf.Tensor) -> tf.Tensor:+ audio_in = INPUT_MODEL(audio)+ return tf.transpose(audio_in, perm=[0, 2, 1, 3])+++def _array_to_image(array: tf.Tensor) -> tf.Tensor:+ """Convert a time-frequency array shape=(batch, time, frequency) to+ the shape expected by tf.summary.image (batch, frequency, time, 1)++ Args:+ array: a (batch, time, frequency) array++ Returns:+ reshaped array+ """+ return tf.expand_dims(tf.transpose(array, perm=[0, 2, 1]), 3)