A lightweight yet powerful audio-to-MIDI converter with pitch bend detection
Python audiolightweightmachine-learningmidimusicpitch-detectionpolyphonicpythontranscriptiontypescript
fix note creation bug where melodia trick never created notes
Rachel Bittner755b8c38e438
diff --git a/basic_pitch/note_creation.py b/basic_pitch/note_creation.pyindex b66d233..c49da0c 100644--- a/basic_pitch/note_creation.py+++ b/basic_pitch/note_creation.py@@ -330,6 +330,163 @@ def output_to_notes_polyphonic() -> List[Tuple[int, int, int, float]]:"""Decode raw model output to polyphonic note events+ Args:+ frames: Frame activation matrix (n_times, n_freqs).+ onsets: Onset activation matrix (n_times, n_freqs).+ onset_thresh: Minimum amplitude of an onset activation to be considered an onset.+ frame_thresh: Minimum amplitude of a frame activation for a note to remain "on".+ min_note_len: Minimum allowed note length in frames.+ infer_onsets: If True, add additional onsets when there are large differences in frame amplitudes.+ max_freq: Maximum allowed output frequency, in Hz.+ min_freq: Minimum allowed output frequency, in Hz.+ melodia_trick : Whether to use the melodia trick to better detect notes.+ energy_tol: Drop notes below this energy.++ Returns:+ list of tuples [(start_time_frames, end_time_frames, pitch_midi, amplitude)]+ representing the note events, where amplitude is a number between 0 and 1+ """++ n_frames = frames.shape[0]++ onsets, frames = constrain_frequency(onsets, frames, max_freq, min_freq)+ # use onsets inferred from frames in addition to the predicted onsets+ if infer_onsets:+ onsets = get_infered_onsets(onsets, frames)++ peak_thresh_mat = np.zeros(onsets.shape)+ peaks = scipy.signal.argrelmax(onsets, axis=0)+ peak_thresh_mat[peaks] = onsets[peaks]++ onset_idx = np.where(peak_thresh_mat >= onset_thresh)+ onset_time_idx = onset_idx[0][::-1] # sort to go backwards in time+ onset_freq_idx = onset_idx[1][::-1] # sort to go backwards in time++ remaining_energy = np.zeros(frames.shape)+ remaining_energy[:, :] = frames[:, :]++ # loop over onsets+ note_events = []+ for note_start_idx, freq_idx in zip(onset_time_idx, onset_freq_idx):+ # if we're too close to the end of the audio, continue+ if note_start_idx >= n_frames - 1:+ continue++ # find time index at this frequency band where the frames drop below an energy threshold+ i = note_start_idx + 1+ k = 0 # number of frames since energy dropped below threshold+ while i < n_frames - 1 and k < energy_tol:+ if remaining_energy[i, freq_idx] < frame_thresh:+ k += 1+ else:+ k = 0+ i += 1++ i -= k # go back to frame above threshold++ # if the note is too short, skip it+ if i - note_start_idx <= min_note_len:+ continue++ remaining_energy[note_start_idx:i, freq_idx] = 0+ if freq_idx < MAX_FREQ_IDX:+ remaining_energy[note_start_idx:i, freq_idx + 1] = 0+ if freq_idx > 0:+ remaining_energy[note_start_idx:i, freq_idx - 1] = 0++ # add the note+ amplitude = np.mean(frames[note_start_idx:i, freq_idx])+ note_events.append(+ (+ note_start_idx,+ i,+ freq_idx + MIDI_OFFSET,+ amplitude,+ )+ )++ if melodia_trick:++ energy_shape = remaining_energy.shape++ while np.max(remaining_energy) > frame_thresh:+ i_mid, freq_idx = np.unravel_index(np.argmax(remaining_energy), energy_shape)+ remaining_energy[i_mid, freq_idx] = 0++ # forward pass+ i = i_mid + 1+ k = 0+ while i < n_frames - 1 and k < energy_tol:++ if remaining_energy[i, freq_idx] < frame_thresh:+ k += 1+ else:+ k = 0++ remaining_energy[i, freq_idx] = 0+ if freq_idx < MAX_FREQ_IDX:+ remaining_energy[i, freq_idx + 1] = 0+ if freq_idx > 0:+ remaining_energy[i, freq_idx - 1] = 0++ i += 1++ i_end = i - 1 - k # go back to frame above threshold++ # backward pass+ i = i_mid - 1+ k = 0+ while i > 0 and k < energy_tol:++ if remaining_energy[i, freq_idx] < frame_thresh:+ k += 1+ else:+ k = 0++ remaining_energy[i, freq_idx] = 0+ if freq_idx < MAX_FREQ_IDX:+ remaining_energy[i, freq_idx + 1] = 0+ if freq_idx > 0:+ remaining_energy[i, freq_idx - 1] = 0++ i -= 1++ i_start = i + 1 + k # go back to frame above threshold+ assert i_start >= 0, "{}".format(i_start)+ assert i_end < n_frames++ if i_end - i_start <= min_note_len:+ # note is too short, skip it+ continue++ # add the note+ amplitude = np.mean(frames[i_start:i_end, freq_idx])+ note_events.append(+ (+ i_start,+ i_end,+ freq_idx + MIDI_OFFSET,+ amplitude,+ )+ )++ return note_events+++def output_to_notes_polyphonic_icassp(+ frames: np.array,+ onsets: np.array,+ onset_thresh: float,+ frame_thresh: float,+ min_note_len: int,+ infer_onsets: bool,+ max_freq: Optional[float],+ min_freq: Optional[float],+ melodia_trick: bool = True,+ energy_tol: int = 11,+) -> List[Tuple[int, int, int, float]]:+ """Decode raw model output to polyphonic note events+Args:frames: Frame activation matrix (n_times, n_freqs).onsets: Onset activation matrix (n_times, n_freqs).