Skip to content

A lightweight yet powerful audio-to-MIDI converter with pitch bend detection

fix note creation bug where melodia trick never created notes

Rachel Bittner755b8c38e438
diff --git a/basic_pitch/note_creation.py b/basic_pitch/note_creation.py
index b66d233..c49da0c 100644
--- a/basic_pitch/note_creation.py
+++ b/basic_pitch/note_creation.py
@@ -330,6 +330,163 @@ def output_to_notes_polyphonic(
) -> List[Tuple[int, int, int, float]]:
"""Decode raw model output to polyphonic note events
+ Args:
+ frames: Frame activation matrix (n_times, n_freqs).
+ onsets: Onset activation matrix (n_times, n_freqs).
+ onset_thresh: Minimum amplitude of an onset activation to be considered an onset.
+ frame_thresh: Minimum amplitude of a frame activation for a note to remain "on".
+ min_note_len: Minimum allowed note length in frames.
+ infer_onsets: If True, add additional onsets when there are large differences in frame amplitudes.
+ max_freq: Maximum allowed output frequency, in Hz.
+ min_freq: Minimum allowed output frequency, in Hz.
+ melodia_trick : Whether to use the melodia trick to better detect notes.
+ energy_tol: Drop notes below this energy.
+
+ Returns:
+ list of tuples [(start_time_frames, end_time_frames, pitch_midi, amplitude)]
+ representing the note events, where amplitude is a number between 0 and 1
+ """
+
+ n_frames = frames.shape[0]
+
+ onsets, frames = constrain_frequency(onsets, frames, max_freq, min_freq)
+ # use onsets inferred from frames in addition to the predicted onsets
+ if infer_onsets:
+ onsets = get_infered_onsets(onsets, frames)
+
+ peak_thresh_mat = np.zeros(onsets.shape)
+ peaks = scipy.signal.argrelmax(onsets, axis=0)
+ peak_thresh_mat[peaks] = onsets[peaks]
+
+ onset_idx = np.where(peak_thresh_mat >= onset_thresh)
+ onset_time_idx = onset_idx[0][::-1] # sort to go backwards in time
+ onset_freq_idx = onset_idx[1][::-1] # sort to go backwards in time
+
+ remaining_energy = np.zeros(frames.shape)
+ remaining_energy[:, :] = frames[:, :]
+
+ # loop over onsets
+ note_events = []
+ for note_start_idx, freq_idx in zip(onset_time_idx, onset_freq_idx):
+ # if we're too close to the end of the audio, continue
+ if note_start_idx >= n_frames - 1:
+ continue
+
+ # find time index at this frequency band where the frames drop below an energy threshold
+ i = note_start_idx + 1
+ k = 0 # number of frames since energy dropped below threshold
+ while i < n_frames - 1 and k < energy_tol:
+ if remaining_energy[i, freq_idx] < frame_thresh:
+ k += 1
+ else:
+ k = 0
+ i += 1
+
+ i -= k # go back to frame above threshold
+
+ # if the note is too short, skip it
+ if i - note_start_idx <= min_note_len:
+ continue
+
+ remaining_energy[note_start_idx:i, freq_idx] = 0
+ if freq_idx < MAX_FREQ_IDX:
+ remaining_energy[note_start_idx:i, freq_idx + 1] = 0
+ if freq_idx > 0:
+ remaining_energy[note_start_idx:i, freq_idx - 1] = 0
+
+ # add the note
+ amplitude = np.mean(frames[note_start_idx:i, freq_idx])
+ note_events.append(
+ (
+ note_start_idx,
+ i,
+ freq_idx + MIDI_OFFSET,
+ amplitude,
+ )
+ )
+
+ if melodia_trick:
+
+ energy_shape = remaining_energy.shape
+
+ while np.max(remaining_energy) > frame_thresh:
+ i_mid, freq_idx = np.unravel_index(np.argmax(remaining_energy), energy_shape)
+ remaining_energy[i_mid, freq_idx] = 0
+
+ # forward pass
+ i = i_mid + 1
+ k = 0
+ while i < n_frames - 1 and k < energy_tol:
+
+ if remaining_energy[i, freq_idx] < frame_thresh:
+ k += 1
+ else:
+ k = 0
+
+ remaining_energy[i, freq_idx] = 0
+ if freq_idx < MAX_FREQ_IDX:
+ remaining_energy[i, freq_idx + 1] = 0
+ if freq_idx > 0:
+ remaining_energy[i, freq_idx - 1] = 0
+
+ i += 1
+
+ i_end = i - 1 - k # go back to frame above threshold
+
+ # backward pass
+ i = i_mid - 1
+ k = 0
+ while i > 0 and k < energy_tol:
+
+ if remaining_energy[i, freq_idx] < frame_thresh:
+ k += 1
+ else:
+ k = 0
+
+ remaining_energy[i, freq_idx] = 0
+ if freq_idx < MAX_FREQ_IDX:
+ remaining_energy[i, freq_idx + 1] = 0
+ if freq_idx > 0:
+ remaining_energy[i, freq_idx - 1] = 0
+
+ i -= 1
+
+ i_start = i + 1 + k # go back to frame above threshold
+ assert i_start >= 0, "{}".format(i_start)
+ assert i_end < n_frames
+
+ if i_end - i_start <= min_note_len:
+ # note is too short, skip it
+ continue
+
+ # add the note
+ amplitude = np.mean(frames[i_start:i_end, freq_idx])
+ note_events.append(
+ (
+ i_start,
+ i_end,
+ freq_idx + MIDI_OFFSET,
+ amplitude,
+ )
+ )
+
+ return note_events
+
+
+def output_to_notes_polyphonic_icassp(
+ frames: np.array,
+ onsets: np.array,
+ onset_thresh: float,
+ frame_thresh: float,
+ min_note_len: int,
+ infer_onsets: bool,
+ max_freq: Optional[float],
+ min_freq: Optional[float],
+ melodia_trick: bool = True,
+ energy_tol: int = 11,
+) -> List[Tuple[int, int, int, float]]:
+ """Decode raw model output to polyphonic note events
+
Args:
frames: Frame activation matrix (n_times, n_freqs).
onsets: Onset activation matrix (n_times, n_freqs).