Skip to content

armgit / nota

Public Imported from GitHub
0

Nota — a free DAW for macOS, Windows and Linux. No subscription, no account — just open it and make music.

15.7 KB
Raw Blame
// SPDX-License-Identifier: AGPL-3.0-only
// Copyright (c) 2026 Egor Khindikaynen (Nota). See LICENSES/ for license terms.
//
// Audio clip → MIDI conversion ("Convert / Slice to New MIDI Track"). Melody and Harmony use
// the basic-pitch AI model when it's installed (ConvertWithModelAsync), else the DSP below.
// Phase 1: Convert Drums + Slice, both onto a new Drum Rack. The clip's audio is pulled
// mono via GetClipAudioMono, analysed by AudioOnsets (pure permissive DSP), sliced to temp
// WAVs and loaded onto rack pads; a MIDI clip triggers them under the source clip.

using System;
using System.Collections.Generic;
using System.IO;
using System.Threading.Tasks;
using Microsoft.Extensions.DependencyInjection;
using Nota.Application;
using Nota.Infrastructure;
using Nota.Application.Localization;

namespace Nota.App;

public partial class MainWindow
{
    // Audio → MIDI blocks the UI (it mutates the graph), so it runs behind a modal progress
    // dialog. The heavy analysis (onset/pitch/harmony DSP) runs on a background thread; the
    // engine mutations resume on the UI thread once the awaited analysis returns.
    private async void OnConvertClip(int trackId, int clipIndex, ClipConvertMode mode)
    {
        if (_vm is null) return;
        float[] mono;
        double sr, startBeat = 0, clipBeats = 0;
        try
        {
            int n = Engine.GetClipAudioMono(trackId, clipIndex, null, out sr);
            if (n <= 0 || sr <= 0) { _vm.StatusText = Loc.T("main.convert.this_clip_has_no_audio"); return; }
            mono = new float[n];
            Engine.GetClipAudioMono(trackId, clipIndex, mono, out sr);

            // Map onsets onto the clip's *displayed* beat length, which already accounts for warp/
            // tempo (a re-warped clip spans a different number of beats). Linear source→beats within
            // the clip; falls back to raw source time at the project tempo when the length is unknown.
            if (Engine.TryGetClipInfo(trackId, clipIndex, out var ci)) { startBeat = ci.StartBeat; clipBeats = ci.LengthBeats; }
            if (clipBeats <= 0) clipBeats = n / sr * (double)_vm.Transport.Bpm / 60.0;
        }
        catch (Exception ex) { _vm.StatusText = Loc.F("main.convert.convert_failed", ex.Message); return; }

        try
        {
            await RunBlockingAsync(Loc.T("main.convert_to_midi"), Loc.T("main.convert.analysing"), async prog =>
            {
                switch (mode)
                {
                    case ClipConvertMode.Drums:   await ConvertDrumsAsync(mono, sr, startBeat, clipBeats, prog); break;
                    case ClipConvertMode.Slice:   await SliceToMidiAsync(mono, sr, startBeat, clipBeats, prog); break;
                    case ClipConvertMode.Melody when Ai.CanTranscribe:
                    case ClipConvertMode.Harmony when Ai.CanTranscribe:
                        await ConvertWithModelAsync(mono, sr, startBeat, clipBeats, mode == ClipConvertMode.Melody, prog); break;
                    case ClipConvertMode.Melody:  await ConvertMelodyAsync(mono, sr, startBeat, clipBeats, prog); break;
                    case ClipConvertMode.Harmony: await ConvertHarmonyAsync(mono, sr, startBeat, clipBeats, prog); break;
                    default: _vm.StatusText = Loc.T("main.convert.that_converter_isnt_available_yet"); break;
                }
            });
        }
        catch (Exception ex) { _vm.StatusText = Loc.F("main.convert.convert_failed", ex.Message); }
    }

    // Slice: chop the audio at onsets (or an even 16-slice grid), one pad per slice on ascending
    // notes from C1, and a MIDI clip that replays them in order.
    private async Task SliceToMidiAsync(float[] mono, double sr, double startBeat, double clipBeats, IProgress<ProgressReport> prog)
    {
        prog.Report(ProgressReport.Indeterminate(Loc.T("main.convert.detecting_transients")));
        var cuts = await Task.Run(() =>
        {
            var c = AudioOnsets.Detect(mono, sr);
            if (c.Count == 0 || c[0] > (int)(sr * 0.01)) c.Insert(0, 0);
            if (c.Count < 2)   // too few transients → even grid
            {
                c.Clear();
                for (int i = 0; i < 16; i++) c.Add((int)((long)mono.Length * i / 16));
            }
            return c;
        });

        const int firstNote = 36, maxPads = 64;
        int minSlice = (int)(sr * 0.005);
        double toBeat = mono.Length > 0 ? clipBeats / mono.Length : 0;   // source frame → clip beat
        int dr = Engine.AddDrumRackTrack();
        var notes = new List<NotaNote>();
        int pads = 0;
        for (int i = 0; i < cuts.Count && pads < maxPads; i++)
        {
            int a = cuts[i];
            int b = (i + 1 < cuts.Count) ? cuts[i + 1] : mono.Length;
            if (b - a < minSlice) continue;
            int note = firstNote + pads;
            if (LoadPad(dr, mono, sr, a, b, note) < 0) continue;
            notes.Add(new NotaNote(note, a * toBeat, Math.Max(0.05, (b - a) * toBeat), 0.9f));
            pads++;
            prog.Report(ProgressReport.At((double)(i + 1) / cuts.Count, Loc.F("main.convert.slicing_pads", pads)));
        }
        FinishConvert(dr, startBeat, clipBeats, notes, pads == 0 ? Loc.T("main.convert.nothing_to_slice") : Loc.F("main.convert.sliced_into_pads", pads));
    }

    // Convert Drums: classify onsets into kick/snare/hat, voice each class with its strongest hit as
    // a pad, and place a MIDI note per onset at the class's pad note.
    private async Task ConvertDrumsAsync(float[] mono, double sr, double startBeat, double clipBeats, IProgress<ProgressReport> prog)
    {
        prog.Report(ProgressReport.Indeterminate(Loc.T("main.convert.detecting_hits")));
        var (onsets, cls, vel) = await Task.Run(() =>
        {
            var on = AudioOnsets.Detect(mono, sr);
            var cl = new DrumClass[on.Count];
            var ve = new float[on.Count];
            for (int i = 0; i < on.Count; i++)
            {
                cl[i] = AudioOnsets.Classify(mono, sr, on[i]);
                ve[i] = AudioOnsets.Strength(mono, sr, on[i]);
            }
            return (on, cl, ve);
        });

        double toBeat = mono.Length > 0 ? clipBeats / mono.Length : 0;   // source frame → clip beat
        double noteLen = Math.Min(0.1, clipBeats);
        // Per class: its GM-ish pad note + the strongest onset index (to voice the pad).
        var padNote = new Dictionary<DrumClass, int> { [DrumClass.Kick] = 36, [DrumClass.Snare] = 38, [DrumClass.Hat] = 42 };
        var best = new Dictionary<DrumClass, (int onset, float str)>();
        for (int i = 0; i < onsets.Count; i++)
            if (!best.TryGetValue(cls[i], out var cur) || vel[i] > cur.str) best[cls[i]] = (onsets[i], vel[i]);

        prog.Report(ProgressReport.Indeterminate(Loc.T("main.convert.building_pads")));
        int dr = Engine.AddDrumRackTrack();
        // Build one pad per class that occurred, voiced by its strongest hit.
        var loaded = new HashSet<DrumClass>();
        foreach (var (c, info) in best)
        {
            int a = info.onset, b = Math.Min(mono.Length, a + (int)(sr * 0.20));
            if (LoadPad(dr, mono, sr, a, b, padNote[c]) >= 0) loaded.Add(c);
        }
        var notes = new List<NotaNote>();
        for (int i = 0; i < onsets.Count; i++)
        {
            if (!loaded.Contains(cls[i])) continue;
            notes.Add(new NotaNote(padNote[cls[i]], onsets[i] * toBeat, noteLen, Math.Clamp(0.3f + vel[i] * 0.7f, 0.2f, 1f)));
        }
        FinishConvert(dr, startBeat, clipBeats, notes, notes.Count == 0 ? Loc.T("main.convert.no_drum_hits_found") : Loc.F("main.convert.converted_hits", notes.Count));
    }

    // Convert Melody: YIN pitch per frame → median-smoothed MIDI pitch → note segmentation, onto a
    // new Nota Volt track. Monophonic MVP — best on clean single-note lines / vocals.
    private async Task ConvertMelodyAsync(float[] mono, double sr, double startBeat, double clipBeats, IProgress<ProgressReport> prog)
    {
        const double minHz = 65, maxHz = 1100;
        int hop = Math.Max(1, (int)(sr * 0.010));   // 10 ms
        int win = (int)(sr / minHz);                // one lowest-note period of comparison window
        int need = win + (int)(sr / minHz);
        int frames = mono.Length > need ? (mono.Length - need) / hop + 1 : 0;
        if (frames < 2) { Status(Loc.T("main.convert.no_melody_found")); return; }

        prog.Report(ProgressReport.At(0, Loc.T("main.convert.detecting_pitch")));
        var (pitch, rms) = await Task.Run(() =>
        {
            var midi = new int[frames];
            var r = new float[frames];
            const float floor = 0.008f;             // silence gate
            int lastPct = -1;
            for (int i = 0; i < frames; i++)
            {
                int p = i * hop;
                double e = 0; for (int j = 0; j < win; j++) { float s = mono[p + j]; e += (double)s * s; }
                r[i] = (float)Math.Sqrt(e / win);
                if (r[i] < floor) midi[i] = -1;
                else
                {
                    double f0 = AudioPitch.YinHz(mono, p, win, sr, 0.15, minHz, maxHz);
                    midi[i] = f0 > 0 ? (int)Math.Round(69 + 12 * Math.Log2(f0 / 440.0)) : -1;
                }
                int pct = (int)(100.0 * (i + 1) / frames);
                if (pct != lastPct) { lastPct = pct; prog.Report(ProgressReport.At((double)(i + 1) / frames, Loc.T("main.convert.detecting_pitch"))); }
            }
            return (MedianFilter(midi, 2), r);      // ±2-frame median kills octave/jitter blips
        });

        double toBeat = mono.Length > 0 ? clipBeats / mono.Length : 0;
        int minFrames = Math.Max(1, (int)(0.06 * sr / hop));   // drop notes shorter than ~60 ms
        var notes = new List<NotaNote>();
        int cur = -1, curStart = 0; float peak = 0;
        void Flush(int endFrame)
        {
            if (cur >= 12 && cur <= 108 && endFrame - curStart >= minFrames)
            {
                double a = (double)curStart * hop, b = (double)endFrame * hop;
                notes.Add(new NotaNote(cur, a * toBeat, Math.Max(0.05, (b - a) * toBeat),
                                       Math.Clamp(0.25f + peak * 4f, 0.25f, 1f)));
            }
        }
        for (int i = 0; i < frames; i++)
        {
            if (pitch[i] != cur) { Flush(i); cur = pitch[i]; curStart = i; peak = rms[i]; }
            else peak = Math.Max(peak, rms[i]);
        }
        Flush(frames);

        if (notes.Count == 0) { Status(Loc.T("main.convert.no_melody_found")); return; }
        int t = Engine.AddVoltSynthTrack();         // Nota Volt
        int mc = Engine.AddMidiClip(t, startBeat, Math.Max(0.25, clipBeats));
        Engine.SetClipNotes(t, mc, notes.ToArray());
        Timeline.Refresh(); Timeline.Select(t, mc); ShowDevices(t); _session?.Refresh();
        Status(Loc.F("main.convert.converted_melody_notes_new_track", notes.Count, t));
    }

    private static IClipAi Ai => App.Services.GetRequiredService<IClipAi>();

    // Convert Melody / Harmony with the basic-pitch model, when it's installed (Settings →
    // Downloads → AI Models): polyphonic notes with their own velocities; Melody keeps the
    // strongest note wherever notes overlap. Otherwise the DSP converters below run.
    private async Task ConvertWithModelAsync(float[] mono, double sr, double startBeat, double clipBeats, bool melody, IProgress<ProgressReport> prog)
    {
        string what = melody ? "melody" : "harmony";
        prog.Report(ProgressReport.At(0, Loc.T("main.convert.listening_for_notes")));
        var p = new Progress<double>(f => prog.Report(ProgressReport.At(f, Loc.T("main.convert.listening_for_notes"))));
        var heard = await Task.Run(() => Ai.Transcribe(mono, sr, p));
        if (melody) heard = Transcription.Monophonic(heard);
        var notes = Transcription.ToClipNotes(heard, mono.Length / sr, clipBeats);
        if (notes.Length == 0) { Status(Loc.F("main.convert.no_found", what)); return; }
        int t = Engine.AddVoltSynthTrack();         // Nota Volt
        int mc = Engine.AddMidiClip(t, startBeat, Math.Max(0.25, clipBeats));
        Engine.SetClipNotes(t, mc, notes);
        Timeline.Refresh(); Timeline.Select(t, mc); ShowDevices(t); _session?.Refresh();
        Status(Loc.F("main.convert.converted_notes_new_track", what, notes.Length, t));
    }

    // Convert Harmony: harmonic-sum pitch estimation (AudioHarmony) → polyphonic note spans on a new
    // Nota Volt track. Approximate — best on sustained chords / pads.
    private async Task ConvertHarmonyAsync(float[] mono, double sr, double startBeat, double clipBeats, IProgress<ProgressReport> prog)
    {
        prog.Report(ProgressReport.Indeterminate(Loc.T("main.convert.detecting_harmony")));
        // Shortest kept note: an eighth (half a beat) at the clip's own beats-per-second.
        double secPerBeat = clipBeats > 0 ? mono.Length / sr / clipBeats : 0.5;
        var spans = await Task.Run(() => AudioHarmony.Detect(mono, sr, minNoteSec: 0.5 * secPerBeat));
        if (spans.Count == 0) { Status(Loc.T("main.convert.no_harmony_found")); return; }
        double toBeat = mono.Length > 0 ? clipBeats / mono.Length : 0;
        var notes = new List<NotaNote>(spans.Count);
        foreach (var s in spans)
            notes.Add(new NotaNote(s.Pitch, s.StartSample * toBeat, Math.Max(0.05, (s.EndSample - s.StartSample) * toBeat), s.Velocity));
        int t = Engine.AddVoltSynthTrack();         // Nota Volt (polyphonic)
        int mc = Engine.AddMidiClip(t, startBeat, Math.Max(0.25, clipBeats));
        Engine.SetClipNotes(t, mc, notes.ToArray());
        Timeline.Refresh(); Timeline.Select(t, mc); ShowDevices(t); _session?.Refresh();
        Status(Loc.F("main.convert.converted_harmony_notes_new_track", notes.Count, t));
    }

    private void Status(string s) { if (_vm is not null) _vm.StatusText = s; }

    // Median filter with radius r (odd window). Preserves array length.
    private static int[] MedianFilter(int[] a, int r)
    {
        var outp = new int[a.Length];
        var w = new List<int>(2 * r + 1);
        for (int i = 0; i < a.Length; i++)
        {
            w.Clear();
            for (int k = Math.Max(0, i - r); k <= Math.Min(a.Length - 1, i + r); k++) w.Add(a[k]);
            w.Sort();
            outp[i] = w[w.Count / 2];
        }
        return outp;
    }

    // Writes a mono slice to a temp WAV and loads it onto a Drum Rack pad at `note`. Returns the
    // chain index or -1. The sampler decodes synchronously, so the temp file is deleted right away.
    private int LoadPad(int drumTrack, float[] mono, double sr, int start, int end, int note)
    {
        int len = Math.Max(1, end - start);
        var buf = new float[len];
        Array.Copy(mono, start, buf, 0, Math.Min(len, mono.Length - start));
        string dir = Path.Combine(Path.GetTempPath(), "nota-slices");
        Directory.CreateDirectory(dir);
        string path = Path.Combine(dir, "slice_" + Guid.NewGuid().ToString("N") + ".wav");
        try
        {
            using (var w = new WavWriter(path, (int)Math.Round(sr), 1, WavBitDepth.Float32)) w.WriteFrames(buf, len);
            return Engine.RackAddSamplerChain(drumTrack, path, note, false);
        }
        finally { try { File.Delete(path); } catch { /* best-effort */ } }
    }

    private void FinishConvert(int dr, double startBeat, double clipBeats, List<NotaNote> notes, string status)
    {
        if (notes.Count == 0) { Engine.RemoveTrack(dr); if (_vm is not null) _vm.StatusText = status; Timeline.Refresh(); return; }
        double lenBeats = Math.Max(0.25, clipBeats);
        int mc = Engine.AddMidiClip(dr, startBeat, lenBeats);
        Engine.SetClipNotes(dr, mc, notes.ToArray());
        Timeline.Refresh();
        Timeline.Select(dr, mc);
        ShowDevices(dr);
        _session?.Refresh();
        if (_vm is not null) _vm.StatusText = Loc.F("main.convert.new_drum_rack_track", status, dr);
    }
}