Skip to content

armgit / nota

Public Imported from GitHub
0

Nota — a free DAW for macOS, Windows and Linux. No subscription, no account — just open it and make music.

(feat) MCP: list_ai_models, separate_stems and convert_audio_to_midi

Egor Khindikaynen042252f165de
diff --git a/src/managed/Nota.App/MainWindow.Convert.cs b/src/managed/Nota.App/MainWindow.Convert.cs
index 39fc3e0d..700ff2ff 100644
--- a/src/managed/Nota.App/MainWindow.Convert.cs
+++ b/src/managed/Nota.App/MainWindow.Convert.cs
@@ -217,22 +217,14 @@ public partial class MainWindow
prog.Report(ProgressReport.At(0, "Listening for notes…"));
var p = new Progress<double>(f => prog.Report(ProgressReport.At(f, "Listening for notes…")));
var heard = await Task.Run(() => Ai.Transcribe(mono, sr, p));
- if (melody) heard = PitchTranscriber.Monophonic(heard);
- if (heard.Count == 0) { Status($"No {what} found."); return; }
-
- double toBeat = mono.Length > 0 ? clipBeats / (mono.Length / sr) : 0; // seconds → beats
- var notes = new List<NotaNote>(heard.Count);
- foreach (var n in heard)
- {
- double start = Math.Max(0, n.Start) * toBeat;
- if (start >= clipBeats) continue;
- notes.Add(new NotaNote(n.Pitch, start, Math.Max(0.05, Math.Min(n.End * toBeat, clipBeats) - start), n.Velocity / 127f));
- }
+ if (melody) heard = Transcription.Monophonic(heard);
+ var notes = Transcription.ToClipNotes(heard, mono.Length / sr, clipBeats);
+ if (notes.Length == 0) { Status($"No {what} found."); return; }
int t = Engine.AddInstrumentTrack(); // Nota Synth
int mc = Engine.AddMidiClip(t, startBeat, Math.Max(0.25, clipBeats));
- Engine.SetClipNotes(t, mc, notes.ToArray());
+ Engine.SetClipNotes(t, mc, notes);
Timeline.Refresh(); Timeline.Select(t, mc); ShowDevices(t); _session?.Refresh();
- Status($"Converted {what}: {notes.Count} notes → new track {t}");
+ Status($"Converted {what}: {notes.Length} notes → new track {t}");
}
// Convert Harmony: harmonic-sum pitch estimation (AudioHarmony) → polyphonic note spans on a new
diff --git a/src/managed/Nota.App/Mcp/McpAppServices.cs b/src/managed/Nota.App/Mcp/McpAppServices.cs
index 74faf04c..99cd228d 100644
--- a/src/managed/Nota.App/Mcp/McpAppServices.cs
+++ b/src/managed/Nota.App/Mcp/McpAppServices.cs
@@ -85,7 +85,7 @@ public sealed class McpService(
IAudioEngine engine, IEngineDispatch dispatch, IArrangementRefresh refresh, ISettingsService settings, ILogSink log,
IPluginCatalog pluginCatalog, IFactoryPresets factoryPresets, IPresetStore presetStore, IPresetLibrary presetLibrary,
IAudioExporter exporter, IMidiDeviceService midiDevices, MidiLearnService midiLearn, IDrumKits drumKits,
- ProjectVersionsBridge projectVersions, IProjectHistory projectHistory)
+ ProjectVersionsBridge projectVersions, IProjectHistory projectHistory, IModelStore models, IClipAi clipAi)
{
private readonly NotaMcpServer _server = new();
@@ -106,6 +106,8 @@ public sealed class McpService(
s.AddSingleton<IMidiLearnAccess>(new MidiLearnAccess(midiLearn));
s.AddSingleton<IProjectVersionsAccess>(projectVersions);
s.AddSingleton(projectHistory);
+ s.AddSingleton(models);
+ s.AddSingleton(clipAi);
}
/// <summary>Reconcile the server with <see cref="Settings.McpEnabled"/> / <see cref="Settings.McpPort"/>.</summary>
diff --git a/src/managed/Nota.Application/Ports/IClipAi.cs b/src/managed/Nota.Application/Ports/IClipAi.cs
index 3f1abde9..e4884368 100644
--- a/src/managed/Nota.Application/Ports/IClipAi.cs
+++ b/src/managed/Nota.Application/Ports/IClipAi.cs
@@ -10,6 +10,37 @@ public readonly record struct TranscribedNote(double Start, double End, int Pitc
public int Velocity => Math.Clamp((int)Math.Round(127 * Amplitude), 1, 127);
}
+/// <summary>Turning transcribed notes into a clip's notes; shared by Convert and its MCP tool.</summary>
+public static class Transcription
+{
+ /// <summary>One line out of a transcription: wherever notes overlap, the strongest one
+ /// wins and the others are dropped (Convert Melody).</summary>
+ public static List<TranscribedNote> Monophonic(IReadOnlyList<TranscribedNote> notes)
+ {
+ var kept = new List<TranscribedNote>();
+ foreach (var n in notes.OrderByDescending(n => n.Amplitude * (n.End - n.Start)))
+ if (!kept.Any(k => k.Start < n.End && n.Start < k.End)) kept.Add(n);
+ kept.Sort((a, b) => a.Start.CompareTo(b.Start));
+ return kept;
+ }
+
+ /// <summary>Transcribed notes as a MIDI clip's notes: <paramref name="audioSeconds"/> of
+ /// audio spans <paramref name="clipBeats"/> (so a warped clip's notes follow its tempo);
+ /// notes are kept inside the clip.</summary>
+ public static NotaNote[] ToClipNotes(IReadOnlyList<TranscribedNote> notes, double audioSeconds, double clipBeats)
+ {
+ double toBeat = audioSeconds > 0 ? clipBeats / audioSeconds : 0;
+ var outp = new List<NotaNote>(notes.Count);
+ foreach (var n in notes)
+ {
+ double start = Math.Max(0, n.Start) * toBeat;
+ if (start >= clipBeats) continue;
+ outp.Add(new NotaNote(n.Pitch, start, Math.Max(0.05, Math.Min(n.End * toBeat, clipBeats) - start), n.Velocity / 127f));
+ }
+ return outp.ToArray();
+ }
+}
+
/// <summary>An audio clip read for Separate Stems (<see cref="IClipAi.ReadStemSource"/>).</summary>
public abstract class StemJob
{
diff --git a/src/managed/Nota.Infrastructure/Ai/PitchTranscriber.cs b/src/managed/Nota.Infrastructure/Ai/PitchTranscriber.cs
index 3649b150..01899b18 100644
--- a/src/managed/Nota.Infrastructure/Ai/PitchTranscriber.cs
+++ b/src/managed/Nota.Infrastructure/Ai/PitchTranscriber.cs
@@ -51,17 +51,6 @@ public sealed class PitchTranscriber : IDisposable
return notes;
}
- /// <summary>One line out of a transcription: wherever notes overlap, the strongest one
- /// wins and the others are dropped (Convert Melody).</summary>
- public static List<TranscribedNote> Monophonic(IReadOnlyList<TranscribedNote> notes)
- {
- var kept = new List<TranscribedNote>();
- foreach (var n in notes.OrderByDescending(n => n.Amplitude * (n.End - n.Start)))
- if (!kept.Any(k => k.Start < n.End && n.Start < k.End)) kept.Add(n);
- kept.Sort((a, b) => a.Start.CompareTo(b.Start));
- return kept;
- }
-
// ---- inference (inference.run_inference) -----------------------------------------------
private (float[] Note, float[] Onset, int Frames) Infer(float[] mono, IProgress<double>? progress, CancellationToken ct)
diff --git a/src/managed/Nota.Mcp/Tools/AiTools.cs b/src/managed/Nota.Mcp/Tools/AiTools.cs
new file mode 100644
index 00000000..e17f0b6c
--- /dev/null
+++ b/src/managed/Nota.Mcp/Tools/AiTools.cs
@@ -0,0 +1,95 @@
+// SPDX-License-Identifier: AGPL-3.0-only
+// Copyright (c) 2026 Egor Khindikaynen (Nota). See LICENSES/ for license terms.
+//
+// MCP tools — the AI models on audio clips: Separate Stems (htdemucs) and audio → MIDI with
+// basic-pitch. The models run on this computer and are installed by the user in Settings →
+// Downloads → AI Models; an AI can see what is installed but not download anything.
+
+using System.ComponentModel;
+using ModelContextProtocol.Server;
+using Nota.Application;
+
+namespace Nota.Mcp.Tools;
+
+[McpServerToolType]
+public sealed class AiTools(
+ IAudioEngine engine, IEngineDispatch dispatch, IArrangementRefresh refresh, IModelStore models, IClipAi ai)
+ : EngineTools(engine, dispatch, refresh)
+{
+ private const string NotInstalled = " isn't installed. Ask the user to install it in Nota: Settings → Downloads → AI Models.";
+
+ public sealed record AiModelInfo(
+ string Id, string Name,
+ [property: Description("The Nota feature it powers")] string Feature,
+ bool Installed,
+ [property: Description("What installing it would download now, in MB (the AI runtime included the first time)")] double DownloadMb);
+
+ [McpServerTool(Name = "list_ai_models"), Description(
+ "List the AI models Nota can run on this computer (htdemucs → separate_stems, basic-pitch → convert_audio_to_midi) "
+ + "and whether each is installed. Only the user can install them, in Settings → Downloads → AI Models.")]
+ public AiModelInfo[] ListAiModels()
+ => models.Models.Select(m => new AiModelInfo(m.Id, m.Name, m.Feature, models.IsInstalled(m.Id),
+ Math.Round(models.DownloadSize(m.Id) / 1048576.0, 1))).ToArray();
+
+ public sealed record StemsResult(
+ [property: Description("The new group track holding the stems")] int GroupTrackId,
+ [property: Description("Stem track ids in order: drums, bass, other, vocals")] int[] StemTrackIds);
+
+ [McpServerTool(Name = "separate_stems"), Description(
+ "Split an audio clip into drums, bass, other and vocals with the htdemucs model: a group of four audio tracks "
+ + "appears under the clip's track, each with a clip lined up with the original, and the original clip is switched "
+ + "off (one undo step). Takes about a third of the clip's length on a recent computer; clips up to 10 minutes.")]
+ public async Task<StemsResult> SeparateStems(
+ [Description("Track id of the audio clip")] int trackId,
+ [Description("Clip index on that track")] int clipIndex)
+ {
+ if (!ai.CanSeparate) throw new InvalidOperationException("The htdemucs model" + NotInstalled);
+ var job = await Read(() => ai.ReadStemSource(trackId, clipIndex));
+ using var stems = await Task.Run(() => ai.Separate(job));
+ int group = await Mutate(() => ai.ApplyStems(job, stems));
+ var kids = await Read(() => Enumerable.Range(0, E.TrackCount)
+ .Select(i => E.TryGetTrackInfo(i, out var ti) ? ti : default)
+ .Where(ti => ti.GroupId == group).Select(ti => ti.Id).ToArray());
+ return new StemsResult(group, kids);
+ }
+
+ public sealed record ConvertResult(int TrackId, int ClipIndex, int Notes);
+
+ [McpServerTool(Name = "convert_audio_to_midi"), Description(
+ "Transcribe an audio clip into a MIDI clip on a new Nota Synth track, lined up with it, using the basic-pitch model. "
+ + "'harmony' keeps every note (chords, polyphony); 'melody' keeps one line, the strongest note wherever notes overlap. "
+ + "Velocities follow how loud each note is.")]
+ public async Task<ConvertResult> ConvertAudioToMidi(
+ [Description("Track id of the audio clip")] int trackId,
+ [Description("Clip index on that track")] int clipIndex,
+ [Description("melody | harmony")] string mode = "harmony")
+ {
+ bool melody = mode.Equals("melody", StringComparison.OrdinalIgnoreCase);
+ if (!melody && !mode.Equals("harmony", StringComparison.OrdinalIgnoreCase))
+ throw new ArgumentException("mode must be 'melody' or 'harmony'.");
+ if (!ai.CanTranscribe) throw new InvalidOperationException("The basic-pitch model" + NotInstalled);
+
+ var (mono, sr, startBeat, clipBeats) = await Read(() =>
+ {
+ int n = E.GetClipAudioMono(trackId, clipIndex, null, out var rate);
+ if (n <= 0 || rate <= 0) throw new InvalidOperationException("That clip has no audio to convert.");
+ var buf = new float[n];
+ E.GetClipAudioMono(trackId, clipIndex, buf, out rate);
+ double start = 0, beats = 0;
+ if (E.TryGetClipInfo(trackId, clipIndex, out var ci)) { start = ci.StartBeat; beats = ci.LengthBeats; }
+ if (beats <= 0) beats = n / rate * E.Bpm / 60.0;
+ return (buf, rate, start, beats);
+ });
+ var heard = await Task.Run(() => ai.Transcribe(mono, sr));
+ if (melody) heard = Transcription.Monophonic(heard);
+ var notes = Transcription.ToClipNotes(heard, mono.Length / sr, clipBeats);
+ if (notes.Length == 0) throw new InvalidOperationException($"No {(melody ? "melody" : "notes")} found in that clip.");
+ return await Mutate(() =>
+ {
+ int t = E.AddInstrumentTrack();
+ int mc = E.AddMidiClip(t, startBeat, Math.Max(0.25, clipBeats));
+ E.SetClipNotes(t, mc, notes);
+ return new ConvertResult(t, mc, notes.Length);
+ });
+ }
+}
diff --git a/tests/Nota.SmokeTest/AiGoldenTests.cs b/tests/Nota.SmokeTest/AiGoldenTests.cs
index 474b4985..d7e1aaf4 100644
--- a/tests/Nota.SmokeTest/AiGoldenTests.cs
+++ b/tests/Nota.SmokeTest/AiGoldenTests.cs
@@ -97,6 +97,7 @@ internal static class AiGoldenTests
$"basic-pitch end to end: {matched}/{refNotes.Count} reference notes matched to a frame ({notes.Count} found)");
foreach (var r in ClipFlow(runtime, dir, left, right)) yield return r;
+ foreach (var r in McpFlow(runtime, dir, mono)) yield return r;
// ---- resampler -----------------------------------------------------------------------
var tone = new float[48000];
@@ -170,16 +171,45 @@ internal static class AiGoldenTests
yield return (sig > 0 && snr > 15, $"…and the stems add back up to the clip ({snr:0.0} dB)");
}
- private sealed class FixedModels(string runtime, string stems) : IModelStore
+ // The MCP tools over the same pieces: what's installed, convert_audio_to_midi, and the
+ // message an AI gets when a model is missing.
+ private static IEnumerable<(bool, string)> McpFlow(string runtime, string dir, float[] mono22)
+ {
+ var wav = Path.Combine(Path.GetTempPath(), $"nota-ai-mcp-{Guid.NewGuid():N}.wav");
+ using (var w = new Nota.Infrastructure.WavWriter(wav, 22050, 1, WavBitDepth.Float32)) w.WriteFrames(mono22[..(22050 * 8)], 22050 * 8);
+ using var eng = new NotaEngine();
+ eng.SetBpm(120);
+ int t = eng.AddAudioTrack();
+ int c = eng.AddAudioClipEx(t, wav, 0, 0, 0, 1f);
+ File.Delete(wav);
+
+ var store = new FixedModels(runtime, Path.Combine(dir, "htdemucs.onnx"), Path.Combine(dir, "nmp.onnx"));
+ var tools = new Nota.Mcp.Tools.AiTools(eng, new SyncDispatch(), new NoRefresh(), store, new ClipAi(eng, store));
+ var conv = tools.ConvertAudioToMidi(t, c, "harmony").GetAwaiter().GetResult();
+ var notes = eng.GetClipNotes(conv.TrackId, conv.ClipIndex);
+ yield return (conv.Notes > 10 && notes.Length == conv.Notes && notes.All(n => n.StartBeat >= 0 && n.StartBeat < 16 && n.Velocity > 0),
+ $"convert_audio_to_midi puts {conv.Notes} notes on a new track, inside the clip's 16 beats");
+ var mel = tools.ConvertAudioToMidi(t, c, "melody").GetAwaiter().GetResult();
+ var line = eng.GetClipNotes(mel.TrackId, mel.ClipIndex).OrderBy(n => n.StartBeat).ToArray();
+ bool mono = line.Zip(line.Skip(1)).All(p => p.First.StartBeat + p.First.LengthBeats <= p.Second.StartBeat + 1e-9);
+ yield return (mel.Notes > 0 && mel.Notes < conv.Notes && mono, $"…'melody' keeps one line ({mel.Notes} notes, none overlapping)");
+
+ var none = new Nota.Mcp.Tools.AiTools(eng, new SyncDispatch(), new NoRefresh(), new FixedModels(null, null, null), new ClipAi(eng, new FixedModels(null, null, null)));
+ string? msg = null;
+ try { none.SeparateStems(t, c).GetAwaiter().GetResult(); } catch (InvalidOperationException e) { msg = e.Message; }
+ yield return (msg?.Contains("Settings → Downloads → AI Models") == true, "without the model, separate_stems tells the AI to ask the user to install it");
+ }
+
+ private sealed class FixedModels(string? runtime, string? stems, string? transcription = null) : IModelStore
{
public IReadOnlyList<StoreModel> Models => [];
public StoreModel? Runtime => null;
- public bool IsInstalled(string id) => true;
+ public bool IsInstalled(string id) => ModelPath(id) is not null;
public long DownloadSize(string id) => 0;
public Task InstallAsync(string id, IProgress<StoreProgress>? progress = null, CancellationToken ct = default) => Task.CompletedTask;
public void Uninstall(string id) { }
public string? RuntimePath => runtime;
- public string? ModelPath(string id) => id == AiModels.Stems ? stems : null;
+ public string? ModelPath(string id) => id == AiModels.Stems ? stems : id == AiModels.Transcription ? transcription : null;
public event Action? Changed { add { } remove { } }
}