Nota — a free DAW for macOS, Windows and Linux. No subscription, no account — just open it and make music.
C# audioaudio-unitavaloniacpp20cross-platformdawdigital-audio-workstationdspjucelinux-audiomcp-servermidimusicmusic-productionpiano-rollplugin-hostsequencersynthesizervstvst3
(feat) MCP: list_ai_models, separate_stems and convert_audio_to_midi
Egor Khindikaynen042252f165de
diff --git a/src/managed/Nota.App/MainWindow.Convert.cs b/src/managed/Nota.App/MainWindow.Convert.csindex 39fc3e0d..700ff2ff 100644--- a/src/managed/Nota.App/MainWindow.Convert.cs+++ b/src/managed/Nota.App/MainWindow.Convert.cs@@ -217,22 +217,14 @@ public partial class MainWindowprog.Report(ProgressReport.At(0, "Listening for notes…"));var p = new Progress<double>(f => prog.Report(ProgressReport.At(f, "Listening for notes…")));var heard = await Task.Run(() => Ai.Transcribe(mono, sr, p));- if (melody) heard = PitchTranscriber.Monophonic(heard);- if (heard.Count == 0) { Status($"No {what} found."); return; }-- double toBeat = mono.Length > 0 ? clipBeats / (mono.Length / sr) : 0; // seconds → beats- var notes = new List<NotaNote>(heard.Count);- foreach (var n in heard)- {- double start = Math.Max(0, n.Start) * toBeat;- if (start >= clipBeats) continue;- notes.Add(new NotaNote(n.Pitch, start, Math.Max(0.05, Math.Min(n.End * toBeat, clipBeats) - start), n.Velocity / 127f));- }+ if (melody) heard = Transcription.Monophonic(heard);+ var notes = Transcription.ToClipNotes(heard, mono.Length / sr, clipBeats);+ if (notes.Length == 0) { Status($"No {what} found."); return; }int t = Engine.AddInstrumentTrack(); // Nota Synthint mc = Engine.AddMidiClip(t, startBeat, Math.Max(0.25, clipBeats));- Engine.SetClipNotes(t, mc, notes.ToArray());+ Engine.SetClipNotes(t, mc, notes);Timeline.Refresh(); Timeline.Select(t, mc); ShowDevices(t); _session?.Refresh();- Status($"Converted {what}: {notes.Count} notes → new track {t}");+ Status($"Converted {what}: {notes.Length} notes → new track {t}");}// Convert Harmony: harmonic-sum pitch estimation (AudioHarmony) → polyphonic note spans on a newdiff --git a/src/managed/Nota.App/Mcp/McpAppServices.cs b/src/managed/Nota.App/Mcp/McpAppServices.csindex 74faf04c..99cd228d 100644--- a/src/managed/Nota.App/Mcp/McpAppServices.cs+++ b/src/managed/Nota.App/Mcp/McpAppServices.cs@@ -85,7 +85,7 @@ public sealed class McpService(IAudioEngine engine, IEngineDispatch dispatch, IArrangementRefresh refresh, ISettingsService settings, ILogSink log,IPluginCatalog pluginCatalog, IFactoryPresets factoryPresets, IPresetStore presetStore, IPresetLibrary presetLibrary,IAudioExporter exporter, IMidiDeviceService midiDevices, MidiLearnService midiLearn, IDrumKits drumKits,- ProjectVersionsBridge projectVersions, IProjectHistory projectHistory)+ ProjectVersionsBridge projectVersions, IProjectHistory projectHistory, IModelStore models, IClipAi clipAi){private readonly NotaMcpServer _server = new();@@ -106,6 +106,8 @@ public sealed class McpService(s.AddSingleton<IMidiLearnAccess>(new MidiLearnAccess(midiLearn));s.AddSingleton<IProjectVersionsAccess>(projectVersions);s.AddSingleton(projectHistory);+ s.AddSingleton(models);+ s.AddSingleton(clipAi);}/// <summary>Reconcile the server with <see cref="Settings.McpEnabled"/> / <see cref="Settings.McpPort"/>.</summary>diff --git a/src/managed/Nota.Application/Ports/IClipAi.cs b/src/managed/Nota.Application/Ports/IClipAi.csindex 3f1abde9..e4884368 100644--- a/src/managed/Nota.Application/Ports/IClipAi.cs+++ b/src/managed/Nota.Application/Ports/IClipAi.cs@@ -10,6 +10,37 @@ public readonly record struct TranscribedNote(double Start, double End, int Pitcpublic int Velocity => Math.Clamp((int)Math.Round(127 * Amplitude), 1, 127);}+/// <summary>Turning transcribed notes into a clip's notes; shared by Convert and its MCP tool.</summary>+public static class Transcription+{+ /// <summary>One line out of a transcription: wherever notes overlap, the strongest one+ /// wins and the others are dropped (Convert Melody).</summary>+ public static List<TranscribedNote> Monophonic(IReadOnlyList<TranscribedNote> notes)+ {+ var kept = new List<TranscribedNote>();+ foreach (var n in notes.OrderByDescending(n => n.Amplitude * (n.End - n.Start)))+ if (!kept.Any(k => k.Start < n.End && n.Start < k.End)) kept.Add(n);+ kept.Sort((a, b) => a.Start.CompareTo(b.Start));+ return kept;+ }++ /// <summary>Transcribed notes as a MIDI clip's notes: <paramref name="audioSeconds"/> of+ /// audio spans <paramref name="clipBeats"/> (so a warped clip's notes follow its tempo);+ /// notes are kept inside the clip.</summary>+ public static NotaNote[] ToClipNotes(IReadOnlyList<TranscribedNote> notes, double audioSeconds, double clipBeats)+ {+ double toBeat = audioSeconds > 0 ? clipBeats / audioSeconds : 0;+ var outp = new List<NotaNote>(notes.Count);+ foreach (var n in notes)+ {+ double start = Math.Max(0, n.Start) * toBeat;+ if (start >= clipBeats) continue;+ outp.Add(new NotaNote(n.Pitch, start, Math.Max(0.05, Math.Min(n.End * toBeat, clipBeats) - start), n.Velocity / 127f));+ }+ return outp.ToArray();+ }+}+/// <summary>An audio clip read for Separate Stems (<see cref="IClipAi.ReadStemSource"/>).</summary>public abstract class StemJob{diff --git a/src/managed/Nota.Infrastructure/Ai/PitchTranscriber.cs b/src/managed/Nota.Infrastructure/Ai/PitchTranscriber.csindex 3649b150..01899b18 100644--- a/src/managed/Nota.Infrastructure/Ai/PitchTranscriber.cs+++ b/src/managed/Nota.Infrastructure/Ai/PitchTranscriber.cs@@ -51,17 +51,6 @@ public sealed class PitchTranscriber : IDisposablereturn notes;}- /// <summary>One line out of a transcription: wherever notes overlap, the strongest one- /// wins and the others are dropped (Convert Melody).</summary>- public static List<TranscribedNote> Monophonic(IReadOnlyList<TranscribedNote> notes)- {- var kept = new List<TranscribedNote>();- foreach (var n in notes.OrderByDescending(n => n.Amplitude * (n.End - n.Start)))- if (!kept.Any(k => k.Start < n.End && n.Start < k.End)) kept.Add(n);- kept.Sort((a, b) => a.Start.CompareTo(b.Start));- return kept;- }-// ---- inference (inference.run_inference) -----------------------------------------------private (float[] Note, float[] Onset, int Frames) Infer(float[] mono, IProgress<double>? progress, CancellationToken ct)diff --git a/src/managed/Nota.Mcp/Tools/AiTools.cs b/src/managed/Nota.Mcp/Tools/AiTools.csnew file mode 100644index 00000000..e17f0b6c--- /dev/null+++ b/src/managed/Nota.Mcp/Tools/AiTools.cs@@ -0,0 +1,95 @@+// SPDX-License-Identifier: AGPL-3.0-only+// Copyright (c) 2026 Egor Khindikaynen (Nota). See LICENSES/ for license terms.+//+// MCP tools — the AI models on audio clips: Separate Stems (htdemucs) and audio → MIDI with+// basic-pitch. The models run on this computer and are installed by the user in Settings →+// Downloads → AI Models; an AI can see what is installed but not download anything.++using System.ComponentModel;+using ModelContextProtocol.Server;+using Nota.Application;++namespace Nota.Mcp.Tools;++[McpServerToolType]+public sealed class AiTools(+ IAudioEngine engine, IEngineDispatch dispatch, IArrangementRefresh refresh, IModelStore models, IClipAi ai)+ : EngineTools(engine, dispatch, refresh)+{+ private const string NotInstalled = " isn't installed. Ask the user to install it in Nota: Settings → Downloads → AI Models.";++ public sealed record AiModelInfo(+ string Id, string Name,+ [property: Description("The Nota feature it powers")] string Feature,+ bool Installed,+ [property: Description("What installing it would download now, in MB (the AI runtime included the first time)")] double DownloadMb);++ [McpServerTool(Name = "list_ai_models"), Description(+ "List the AI models Nota can run on this computer (htdemucs → separate_stems, basic-pitch → convert_audio_to_midi) "+ + "and whether each is installed. Only the user can install them, in Settings → Downloads → AI Models.")]+ public AiModelInfo[] ListAiModels()+ => models.Models.Select(m => new AiModelInfo(m.Id, m.Name, m.Feature, models.IsInstalled(m.Id),+ Math.Round(models.DownloadSize(m.Id) / 1048576.0, 1))).ToArray();++ public sealed record StemsResult(+ [property: Description("The new group track holding the stems")] int GroupTrackId,+ [property: Description("Stem track ids in order: drums, bass, other, vocals")] int[] StemTrackIds);++ [McpServerTool(Name = "separate_stems"), Description(+ "Split an audio clip into drums, bass, other and vocals with the htdemucs model: a group of four audio tracks "+ + "appears under the clip's track, each with a clip lined up with the original, and the original clip is switched "+ + "off (one undo step). Takes about a third of the clip's length on a recent computer; clips up to 10 minutes.")]+ public async Task<StemsResult> SeparateStems(+ [Description("Track id of the audio clip")] int trackId,+ [Description("Clip index on that track")] int clipIndex)+ {+ if (!ai.CanSeparate) throw new InvalidOperationException("The htdemucs model" + NotInstalled);+ var job = await Read(() => ai.ReadStemSource(trackId, clipIndex));+ using var stems = await Task.Run(() => ai.Separate(job));+ int group = await Mutate(() => ai.ApplyStems(job, stems));+ var kids = await Read(() => Enumerable.Range(0, E.TrackCount)+ .Select(i => E.TryGetTrackInfo(i, out var ti) ? ti : default)+ .Where(ti => ti.GroupId == group).Select(ti => ti.Id).ToArray());+ return new StemsResult(group, kids);+ }++ public sealed record ConvertResult(int TrackId, int ClipIndex, int Notes);++ [McpServerTool(Name = "convert_audio_to_midi"), Description(+ "Transcribe an audio clip into a MIDI clip on a new Nota Synth track, lined up with it, using the basic-pitch model. "+ + "'harmony' keeps every note (chords, polyphony); 'melody' keeps one line, the strongest note wherever notes overlap. "+ + "Velocities follow how loud each note is.")]+ public async Task<ConvertResult> ConvertAudioToMidi(+ [Description("Track id of the audio clip")] int trackId,+ [Description("Clip index on that track")] int clipIndex,+ [Description("melody | harmony")] string mode = "harmony")+ {+ bool melody = mode.Equals("melody", StringComparison.OrdinalIgnoreCase);+ if (!melody && !mode.Equals("harmony", StringComparison.OrdinalIgnoreCase))+ throw new ArgumentException("mode must be 'melody' or 'harmony'.");+ if (!ai.CanTranscribe) throw new InvalidOperationException("The basic-pitch model" + NotInstalled);++ var (mono, sr, startBeat, clipBeats) = await Read(() =>+ {+ int n = E.GetClipAudioMono(trackId, clipIndex, null, out var rate);+ if (n <= 0 || rate <= 0) throw new InvalidOperationException("That clip has no audio to convert.");+ var buf = new float[n];+ E.GetClipAudioMono(trackId, clipIndex, buf, out rate);+ double start = 0, beats = 0;+ if (E.TryGetClipInfo(trackId, clipIndex, out var ci)) { start = ci.StartBeat; beats = ci.LengthBeats; }+ if (beats <= 0) beats = n / rate * E.Bpm / 60.0;+ return (buf, rate, start, beats);+ });+ var heard = await Task.Run(() => ai.Transcribe(mono, sr));+ if (melody) heard = Transcription.Monophonic(heard);+ var notes = Transcription.ToClipNotes(heard, mono.Length / sr, clipBeats);+ if (notes.Length == 0) throw new InvalidOperationException($"No {(melody ? "melody" : "notes")} found in that clip.");+ return await Mutate(() =>+ {+ int t = E.AddInstrumentTrack();+ int mc = E.AddMidiClip(t, startBeat, Math.Max(0.25, clipBeats));+ E.SetClipNotes(t, mc, notes);+ return new ConvertResult(t, mc, notes.Length);+ });+ }+}diff --git a/tests/Nota.SmokeTest/AiGoldenTests.cs b/tests/Nota.SmokeTest/AiGoldenTests.csindex 474b4985..d7e1aaf4 100644--- a/tests/Nota.SmokeTest/AiGoldenTests.cs+++ b/tests/Nota.SmokeTest/AiGoldenTests.cs@@ -97,6 +97,7 @@ internal static class AiGoldenTests$"basic-pitch end to end: {matched}/{refNotes.Count} reference notes matched to a frame ({notes.Count} found)");foreach (var r in ClipFlow(runtime, dir, left, right)) yield return r;+ foreach (var r in McpFlow(runtime, dir, mono)) yield return r;// ---- resampler -----------------------------------------------------------------------var tone = new float[48000];@@ -170,16 +171,45 @@ internal static class AiGoldenTestsyield return (sig > 0 && snr > 15, $"…and the stems add back up to the clip ({snr:0.0} dB)");}- private sealed class FixedModels(string runtime, string stems) : IModelStore+ // The MCP tools over the same pieces: what's installed, convert_audio_to_midi, and the+ // message an AI gets when a model is missing.+ private static IEnumerable<(bool, string)> McpFlow(string runtime, string dir, float[] mono22)+ {+ var wav = Path.Combine(Path.GetTempPath(), $"nota-ai-mcp-{Guid.NewGuid():N}.wav");+ using (var w = new Nota.Infrastructure.WavWriter(wav, 22050, 1, WavBitDepth.Float32)) w.WriteFrames(mono22[..(22050 * 8)], 22050 * 8);+ using var eng = new NotaEngine();+ eng.SetBpm(120);+ int t = eng.AddAudioTrack();+ int c = eng.AddAudioClipEx(t, wav, 0, 0, 0, 1f);+ File.Delete(wav);++ var store = new FixedModels(runtime, Path.Combine(dir, "htdemucs.onnx"), Path.Combine(dir, "nmp.onnx"));+ var tools = new Nota.Mcp.Tools.AiTools(eng, new SyncDispatch(), new NoRefresh(), store, new ClipAi(eng, store));+ var conv = tools.ConvertAudioToMidi(t, c, "harmony").GetAwaiter().GetResult();+ var notes = eng.GetClipNotes(conv.TrackId, conv.ClipIndex);+ yield return (conv.Notes > 10 && notes.Length == conv.Notes && notes.All(n => n.StartBeat >= 0 && n.StartBeat < 16 && n.Velocity > 0),+ $"convert_audio_to_midi puts {conv.Notes} notes on a new track, inside the clip's 16 beats");+ var mel = tools.ConvertAudioToMidi(t, c, "melody").GetAwaiter().GetResult();+ var line = eng.GetClipNotes(mel.TrackId, mel.ClipIndex).OrderBy(n => n.StartBeat).ToArray();+ bool mono = line.Zip(line.Skip(1)).All(p => p.First.StartBeat + p.First.LengthBeats <= p.Second.StartBeat + 1e-9);+ yield return (mel.Notes > 0 && mel.Notes < conv.Notes && mono, $"…'melody' keeps one line ({mel.Notes} notes, none overlapping)");++ var none = new Nota.Mcp.Tools.AiTools(eng, new SyncDispatch(), new NoRefresh(), new FixedModels(null, null, null), new ClipAi(eng, new FixedModels(null, null, null)));+ string? msg = null;+ try { none.SeparateStems(t, c).GetAwaiter().GetResult(); } catch (InvalidOperationException e) { msg = e.Message; }+ yield return (msg?.Contains("Settings → Downloads → AI Models") == true, "without the model, separate_stems tells the AI to ask the user to install it");+ }++ private sealed class FixedModels(string? runtime, string? stems, string? transcription = null) : IModelStore{public IReadOnlyList<StoreModel> Models => [];public StoreModel? Runtime => null;- public bool IsInstalled(string id) => true;+ public bool IsInstalled(string id) => ModelPath(id) is not null;public long DownloadSize(string id) => 0;public Task InstallAsync(string id, IProgress<StoreProgress>? progress = null, CancellationToken ct = default) => Task.CompletedTask;public void Uninstall(string id) { }public string? RuntimePath => runtime;- public string? ModelPath(string id) => id == AiModels.Stems ? stems : null;+ public string? ModelPath(string id) => id == AiModels.Stems ? stems : id == AiModels.Transcription ? transcription : null;public event Action? Changed { add { } remove { } }}