Skip to content

armgit / nota

Public Imported from GitHub
0

Nota — a free DAW for macOS, Windows and Linux. No subscription, no account — just open it and make music.

(feat) Separate Stems on audio clips, and Convert Melody / Harmony on the basic-pitch model

Egor Khindikaynen1f5bfddfdf6b
diff --git a/src/managed/Nota.App/App.axaml.cs b/src/managed/Nota.App/App.axaml.cs
index eb426b6a..c34ce176 100644
--- a/src/managed/Nota.App/App.axaml.cs
+++ b/src/managed/Nota.App/App.axaml.cs
@@ -40,6 +40,7 @@ public partial class App : Avalonia.Application
services.AddSingleton<IPluginStore, PluginStore>();
services.AddSingleton<ISampleStore, SampleStore>();
services.AddSingleton<IModelStore>(_ => new ModelStore());
+ services.AddSingleton<IClipAi, ClipAi>();
services.AddSingleton<IPresetLibrary, PresetLibrary>();
services.AddSingleton<IProjectStore, ProjectStore>();
services.AddSingleton<IProjectHistory>(_ => new ProjectHistory(AppInfo.Version));
diff --git a/src/managed/Nota.App/ArrangementView.LaneControl.cs b/src/managed/Nota.App/ArrangementView.LaneControl.cs
index 76e57733..96eedf4a 100644
--- a/src/managed/Nota.App/ArrangementView.LaneControl.cs
+++ b/src/managed/Nota.App/ArrangementView.LaneControl.cs
@@ -1103,10 +1103,12 @@ public sealed partial class ArrangementView
flyout.Items.Add(new Separator());
flyout.Items.Add(loop);
- // Audio clip → MIDI: Convert / Slice to New MIDI Track. Drums + Slice
- // ship now; Melody + Harmony (pitch detection) are disabled until their DSP lands.
+ // Audio clip → stems (AI model) and → MIDI: Convert / Slice to New MIDI Track.
if (!clip.IsMidi)
{
+ var stems = new MenuItem { Header = "Separate Stems…", Icon = MenuKit.Icon(GlyphKind.Stems) };
+ stems.Click += (_, _) => _o.SeparateStemsRequested?.Invoke(trackId, idx);
+
var convert = new MenuItem { Header = "Convert", Icon = MenuKit.Icon(GlyphKind.Arrow) };
MenuItem ConvItem(string header, ClipConvertMode mode, bool enabled)
{
@@ -1118,7 +1120,15 @@ public sealed partial class ArrangementView
convert.Items.Add(ConvItem("Convert Harmony to New MIDI Track", ClipConvertMode.Harmony, true));
convert.Items.Add(ConvItem("Convert Drums to New MIDI Track", ClipConvertMode.Drums, true));
convert.Items.Add(ConvItem("Slice to New MIDI Track", ClipConvertMode.Slice, true));
+ if (_o.CanTranscribe?.Invoke() == false)
+ {
+ var better = new MenuItem { Header = "Get the AI Model for Melody & Harmony…" };
+ better.Click += (_, _) => _o.TranscriptionModelRequested?.Invoke();
+ convert.Items.Add(new Separator());
+ convert.Items.Add(better);
+ }
flyout.Items.Add(new Separator());
+ flyout.Items.Add(stems);
flyout.Items.Add(convert);
}
diff --git a/src/managed/Nota.App/ArrangementView.cs b/src/managed/Nota.App/ArrangementView.cs
index 63ed887c..49a32db0 100644
--- a/src/managed/Nota.App/ArrangementView.cs
+++ b/src/managed/Nota.App/ArrangementView.cs
@@ -217,6 +217,12 @@ public sealed partial class ArrangementView : UserControl
public event Action<BrowserItem, int, double>? ItemDropped;
/// <summary>Audio-clip context-menu "Convert / Slice to New MIDI Track" (track id, clip index, mode).</summary>
public event Action<int, int, ClipConvertMode>? ConvertClipRequested;
+ /// <summary>Audio-clip context-menu "Separate Stems" (track id, clip index).</summary>
+ public event Action<int, int>? SeparateStemsRequested;
+ /// <summary>The Convert submenu's offer to download the basic-pitch model.</summary>
+ public event Action? TranscriptionModelRequested;
+ /// <summary>Whether Convert Melody / Harmony run on the AI model (else the menu offers it).</summary>
+ public Func<bool>? CanTranscribe { get; set; }
/// <summary>A context menu asked for a new track. MainWindow owns creation — the seed
/// MIDI clip, the session / modular / device-chain refreshes and the status line — so the
/// menu only asks, and the toolbar's + buttons and these entries stay one behaviour.
diff --git a/src/managed/Nota.App/Controls/Glyph.cs b/src/managed/Nota.App/Controls/Glyph.cs
index c60617a2..a4a25260 100644
--- a/src/managed/Nota.App/Controls/Glyph.cs
+++ b/src/managed/Nota.App/Controls/Glyph.cs
@@ -25,7 +25,7 @@ internal enum GlyphKind
Edit, Freeze, PopOut, Cycle, Dot, Bypass, Plus, Grip, Minus,
Headphones, // input monitoring
// Context-menu icons
- Copy, Cut, Paste, Trash, Duplicate, Split, Consolidate, Reverse, Folder, Ungroup,
+ Copy, Cut, Paste, Trash, Duplicate, Split, Consolidate, Stems, Reverse, Folder, Ungroup,
Palette, Note, Input, Arrow, Grid, Select, Save, List, Compare, Flatten,
}
@@ -274,6 +274,16 @@ internal sealed class Glyph : Control
ctx.DrawRectangle(ink, null, new RoundedRect(new Rect(cx + s * 0.04, cy - s * 0.28, s * 0.4, s * 0.56), 1));
break;
}
+ case GlyphKind.Stems:
+ {
+ // Consolidate the other way: one block fanning out into three strokes.
+ ctx.DrawRectangle(ink, null, new RoundedRect(new Rect(cx - s * 0.44, cy - s * 0.28, s * 0.3, s * 0.56), 1));
+ var fork = new Point(cx - s * 0.04, cy);
+ ctx.DrawLine(pen, fork, new Point(cx + s * 0.44, cy - s * 0.34));
+ ctx.DrawLine(pen, fork, new Point(cx + s * 0.44, cy));
+ ctx.DrawLine(pen, fork, new Point(cx + s * 0.44, cy + s * 0.34));
+ break;
+ }
case GlyphKind.Reverse:
{
// ⇄: one arrow each way.
diff --git a/src/managed/Nota.App/MainWindow.Convert.cs b/src/managed/Nota.App/MainWindow.Convert.cs
index 3cf3ce69..39fc3e0d 100644
--- a/src/managed/Nota.App/MainWindow.Convert.cs
+++ b/src/managed/Nota.App/MainWindow.Convert.cs
@@ -1,7 +1,8 @@
// SPDX-License-Identifier: AGPL-3.0-only
// Copyright (c) 2026 Egor Khindikaynen (Nota). See LICENSES/ for license terms.
//
-// Audio clip → MIDI conversion ("Convert / Slice to New MIDI Track").
+// Audio clip → MIDI conversion ("Convert / Slice to New MIDI Track"). Melody and Harmony use
+// the basic-pitch AI model when it's installed (ConvertWithModelAsync), else the DSP below.
// Phase 1: Convert Drums + Slice, both onto a new Drum Rack. The clip's audio is pulled
// mono via GetClipAudioMono, analysed by AudioOnsets (pure permissive DSP), sliced to temp
// WAVs and loaded onto rack pads; a MIDI clip triggers them under the source clip.
@@ -10,6 +11,7 @@ using System;
using System.Collections.Generic;
using System.IO;
using System.Threading.Tasks;
+using Microsoft.Extensions.DependencyInjection;
using Nota.Application;
using Nota.Infrastructure;
@@ -48,6 +50,9 @@ public partial class MainWindow
{
case ClipConvertMode.Drums: await ConvertDrumsAsync(mono, sr, startBeat, clipBeats, prog); break;
case ClipConvertMode.Slice: await SliceToMidiAsync(mono, sr, startBeat, clipBeats, prog); break;
+ case ClipConvertMode.Melody when Ai.CanTranscribe:
+ case ClipConvertMode.Harmony when Ai.CanTranscribe:
+ await ConvertWithModelAsync(mono, sr, startBeat, clipBeats, mode == ClipConvertMode.Melody, prog); break;
case ClipConvertMode.Melody: await ConvertMelodyAsync(mono, sr, startBeat, clipBeats, prog); break;
case ClipConvertMode.Harmony: await ConvertHarmonyAsync(mono, sr, startBeat, clipBeats, prog); break;
default: _vm.StatusText = "That converter isn't available yet."; break;
@@ -201,6 +206,35 @@ public partial class MainWindow
Status($"Converted melody: {notes.Count} notes → new track {t}");
}
+ private static IClipAi Ai => App.Services.GetRequiredService<IClipAi>();
+
+ // Convert Melody / Harmony with the basic-pitch model, when it's installed (Settings →
+ // Downloads → AI Models): polyphonic notes with their own velocities; Melody keeps the
+ // strongest note wherever notes overlap. Otherwise the DSP converters below run.
+ private async Task ConvertWithModelAsync(float[] mono, double sr, double startBeat, double clipBeats, bool melody, IProgress<ProgressReport> prog)
+ {
+ string what = melody ? "melody" : "harmony";
+ prog.Report(ProgressReport.At(0, "Listening for notes…"));
+ var p = new Progress<double>(f => prog.Report(ProgressReport.At(f, "Listening for notes…")));
+ var heard = await Task.Run(() => Ai.Transcribe(mono, sr, p));
+ if (melody) heard = PitchTranscriber.Monophonic(heard);
+ if (heard.Count == 0) { Status($"No {what} found."); return; }
+
+ double toBeat = mono.Length > 0 ? clipBeats / (mono.Length / sr) : 0; // seconds → beats
+ var notes = new List<NotaNote>(heard.Count);
+ foreach (var n in heard)
+ {
+ double start = Math.Max(0, n.Start) * toBeat;
+ if (start >= clipBeats) continue;
+ notes.Add(new NotaNote(n.Pitch, start, Math.Max(0.05, Math.Min(n.End * toBeat, clipBeats) - start), n.Velocity / 127f));
+ }
+ int t = Engine.AddInstrumentTrack(); // Nota Synth
+ int mc = Engine.AddMidiClip(t, startBeat, Math.Max(0.25, clipBeats));
+ Engine.SetClipNotes(t, mc, notes.ToArray());
+ Timeline.Refresh(); Timeline.Select(t, mc); ShowDevices(t); _session?.Refresh();
+ Status($"Converted {what}: {notes.Count} notes → new track {t}");
+ }
+
// Convert Harmony: harmonic-sum pitch estimation (AudioHarmony) → polyphonic note spans on a new
// Nota Synth track. Approximate — best on sustained chords / pads.
private async Task ConvertHarmonyAsync(float[] mono, double sr, double startBeat, double clipBeats, IProgress<ProgressReport> prog)
diff --git a/src/managed/Nota.App/MainWindow.Progress.cs b/src/managed/Nota.App/MainWindow.Progress.cs
index 747a89cd..2c8e09d0 100644
--- a/src/managed/Nota.App/MainWindow.Progress.cs
+++ b/src/managed/Nota.App/MainWindow.Progress.cs
@@ -9,6 +9,7 @@
// regardless of which surface shows it.
using System;
+using System.Threading;
using System.Threading.Tasks;
namespace Nota.App;
@@ -19,6 +20,10 @@ public partial class MainWindow
private Task RunBlockingAsync(string title, string initial, Func<IProgress<ProgressReport>, Task> work)
=> ProgressWindow.RunAsync(this, title, initial, work);
+ /// <summary>The same with a Cancel button; false when the user cancelled.</summary>
+ private Task<bool> RunBlockingAsync(string title, string initial, Func<IProgress<ProgressReport>, CancellationToken, Task> work)
+ => ProgressWindow.RunAsync(this, title, initial, work);
+
/// <summary>Run <paramref name="work"/> with progress shown in the status bar's background
/// strip (non-modal). The strip appears for the duration and clears when the work ends.</summary>
private async Task RunBackgroundAsync(string label, Func<IProgress<ProgressReport>, Task> work)
diff --git a/src/managed/Nota.App/MainWindow.Stems.cs b/src/managed/Nota.App/MainWindow.Stems.cs
new file mode 100644
index 00000000..06063e14
--- /dev/null
+++ b/src/managed/Nota.App/MainWindow.Stems.cs
@@ -0,0 +1,100 @@
+// SPDX-License-Identifier: AGPL-3.0-only
+// Copyright (c) 2026 Egor Khindikaynen (Nota). See LICENSES/ for license terms.
+//
+// Separate Stems (audio clip menu): splits the clip into Drums / Bass / Other / Vocals tracks
+// with the htdemucs model (IClipAi). The first use offers to download the model, and the AI
+// runtime with it; the separation itself runs behind a modal progress dialog with Cancel.
+
+using System;
+using System.Linq;
+using System.Threading.Tasks;
+using Microsoft.Extensions.DependencyInjection;
+using Nota.Application;
+
+namespace Nota.App;
+
+public partial class MainWindow
+{
+ private async void OnSeparateStems(int trackId, int clipIndex)
+ {
+ if (_vm is null) return;
+ var ai = App.Services.GetRequiredService<IClipAi>();
+ if (!await EnsureModelAsync(AiModels.Stems, "Separate Stems")) return;
+
+ StemJob job;
+ try { job = ai.ReadStemSource(trackId, clipIndex); }
+ catch (InvalidOperationException ex) { _vm.StatusText = ex.Message; return; }
+
+ StemResult? stems = null;
+ try
+ {
+ bool done = await RunBlockingAsync("Separate Stems", "Preparing audio…", async (prog, ct) =>
+ {
+ var p = new Progress<(double Fraction, string Label)>(r => prog.Report(ProgressReport.At(r.Fraction, r.Label)));
+ stems = await Task.Run(() => ai.Separate(job, p, ct), ct);
+ });
+ if (!done || stems is null) { _vm.StatusText = "Separate Stems cancelled."; return; }
+
+ int group = ai.ApplyStems(job, stems);
+ Timeline.Refresh();
+ if (group > 0) Timeline.Select(group, -1);
+ _session?.Refresh();
+ _vm.StatusText = $"Separated \"{job.Name}\" into drums, bass, other and vocals.";
+ }
+ catch (Exception ex) when (ex is InvalidOperationException or StoreException)
+ {
+ _vm.StatusText = $"Separate Stems failed: {ex.Message}";
+ }
+ catch (Exception ex)
+ {
+ App.Services.GetRequiredService<ILogSink>().Error("Separate Stems failed", ex);
+ _vm.StatusText = $"Separate Stems failed: {ex.Message}";
+ }
+ finally
+ {
+ stems?.Dispose();
+ }
+ }
+
+ /// <summary>True when the model is installed, or the user agreed to download it and the
+ /// download finished.</summary>
+ private async Task<bool> EnsureModelAsync(string id, string feature)
+ {
+ var store = App.Services.GetRequiredService<IModelStore>();
+ if (store.IsInstalled(id) && store.RuntimePath is not null) return true;
+ if (_vm is null) return false;
+ if (store.Runtime is null)
+ {
+ _vm.StatusText = $"{feature} isn't available on this computer.";
+ return false;
+ }
+
+ var model = store.Models.First(m => m.Id == id);
+ bool withRuntime = store.RuntimePath is null;
+ string what = withRuntime ? $"the {model.Name} model and the AI runtime it runs on" : $"the {model.Name} model";
+ bool ok = await new ConfirmWindow(feature,
+ $"{feature} uses an AI model that runs on this computer. Download {what} ({Size(store.DownloadSize(id))}) once? " +
+ "You can remove it later in Settings → Downloads → AI Models.",
+ "Download", "Cancel").ShowDialog<bool>(this);
+ if (!ok) return false;
+
+ try
+ {
+ bool done = await RunBlockingAsync(feature, "Downloading…", async (prog, ct) =>
+ {
+ var p = new Progress<StoreProgress>(r => prog.Report(r.Fraction < 0 ? ProgressReport.Indeterminate(r.Message) : ProgressReport.At(r.Fraction, r.Message)));
+ await store.InstallAsync(id, p, ct);
+ });
+ if (!done) _vm.StatusText = "Download cancelled.";
+ return done && store.IsInstalled(id);
+ }
+ catch (StoreException ex)
+ {
+ _vm.StatusText = ex.Message;
+ return false;
+ }
+ }
+
+ private static string Size(long bytes)
+ => bytes >= 1L << 30 ? $"{bytes / (double)(1L << 30):0.0} GB" : $"{Math.Max(1, bytes / (double)(1 << 20)):0} MB";
+}
diff --git a/src/managed/Nota.App/MainWindow.axaml.cs b/src/managed/Nota.App/MainWindow.axaml.cs
index 68014315..61867573 100644
--- a/src/managed/Nota.App/MainWindow.axaml.cs
+++ b/src/managed/Nota.App/MainWindow.axaml.cs
@@ -180,6 +180,13 @@ public partial class MainWindow : Window
Timeline.ItemDropped += OnArrangementDrop; // browser drag & drop (M7-5)
Timeline.PasteBouncedRequested += (track, beat) => _ = PasteBouncedAsync(track, beat);
Timeline.ConvertClipRequested += OnConvertClip; // audio clip → MIDI (Convert / Slice)
+ Timeline.SeparateStemsRequested += OnSeparateStems; // audio clip → stem tracks (AI model)
+ Timeline.CanTranscribe = () => Ai.CanTranscribe;
+ Timeline.TranscriptionModelRequested += async () =>
+ {
+ if (await EnsureModelAsync(AiModels.Transcription, "Convert to MIDI") && _vm is not null)
+ _vm.StatusText = "Convert Melody and Convert Harmony now use the AI model.";
+ };
// Arrangement context menus add tracks through the toolbar's own path, so the two
// routes seed, refresh and report identically; the menu's row places the new track.
Timeline.AddTrackRequested += (kind, anchor) =>
diff --git a/src/managed/Nota.App/ProgressWindow.cs b/src/managed/Nota.App/ProgressWindow.cs
index 00f0bfb0..cfe564dc 100644
--- a/src/managed/Nota.App/ProgressWindow.cs
+++ b/src/managed/Nota.App/ProgressWindow.cs
@@ -9,6 +9,7 @@
// written once and can be surfaced either way.
using System;
+using System.Threading;
using System.Threading.Tasks;
using Avalonia;
using Avalonia.Controls;
@@ -34,9 +35,11 @@ public sealed class ProgressWindow : NotaWindow
{
private readonly TextBlock _label;
private readonly ProgressBar _bar;
+ private readonly CancellationTokenSource? _cancel;
- public ProgressWindow(string title, string initial)
+ public ProgressWindow(string title, string initial, CancellationTokenSource? cancel = null)
{
+ _cancel = cancel;
Title = title;
CanResize = false;
ShowInTaskbar = false;
@@ -63,6 +66,17 @@ public sealed class ProgressWindow : NotaWindow
var body = new StackPanel { Spacing = 14, Margin = new Thickness(20) };
body.Children.Add(_label);
body.Children.Add(_bar);
+ if (cancel is not null)
+ {
+ var stop = new Button { Content = "Cancel", Classes = { "ghost" }, HorizontalAlignment = HorizontalAlignment.Right };
+ stop.Click += (_, _) =>
+ {
+ stop.IsEnabled = false;
+ _label.Text = "Cancelling…";
+ cancel.Cancel();
+ };
+ body.Children.Add(stop);
+ }
SetBody(body);
}
@@ -70,6 +84,7 @@ public sealed class ProgressWindow : NotaWindow
public void Report(ProgressReport r)
{
if (!Dispatcher.UIThread.CheckAccess()) { Dispatcher.UIThread.Post(() => Report(r)); return; }
+ if (_cancel?.IsCancellationRequested == true) return; // keep "Cancelling…" up
if (!string.IsNullOrEmpty(r.Message)) _label.Text = r.Message;
if (r.Fraction < 0.0) _bar.IsIndeterminate = true;
else { _bar.IsIndeterminate = false; _bar.Value = Math.Clamp(r.Fraction, 0.0, 1.0) * 100.0; }
@@ -91,5 +106,24 @@ public sealed class ProgressWindow : NotaWindow
finally { win.Close(); await dialog; }
}
+ /// <summary><see cref="RunAsync(Window, string, string, Func{IProgress{ProgressReport}, Task})"/>
+ /// with a Cancel button: <paramref name="work"/> gets a token that Cancel trips. A cancelled
+ /// run returns false (the work's <see cref="OperationCanceledException"/> is swallowed);
+ /// other exceptions propagate.</summary>
+ public static async Task<bool> RunAsync(Window owner, string title, string initial,
+ Func<IProgress<ProgressReport>, CancellationToken, Task> work)
+ {
+ using var cts = new CancellationTokenSource();
+ var win = new ProgressWindow(title, initial, cts);
+ var progress = new Progress<ProgressReport>(win.Report);
+ var opened = new TaskCompletionSource();
+ win.Opened += (_, _) => opened.TrySetResult();
+ var dialog = win.ShowDialog(owner);
+ await opened.Task;
+ try { await work(progress, cts.Token); return true; }
+ catch (OperationCanceledException) when (cts.IsCancellationRequested) { return false; }
+ finally { win.Close(); await dialog; }
+ }
+
private static IBrush Brush(string key) => (IBrush?)NotaPalette.ByKey(key) ?? NotaPalette.TextTertiary;
}
diff --git a/src/managed/Nota.Application/Ports/IClipAi.cs b/src/managed/Nota.Application/Ports/IClipAi.cs
new file mode 100644
index 00000000..3f1abde9
--- /dev/null
+++ b/src/managed/Nota.Application/Ports/IClipAi.cs
@@ -0,0 +1,48 @@
+// SPDX-License-Identifier: AGPL-3.0-only
+// Copyright (c) 2026 Egor Khindikaynen (Nota). See LICENSES/ for license terms.
+
+namespace Nota.Application;
+
+/// <summary>A transcribed note: times in seconds from the start of the audio, MIDI pitch, and
+/// the model's mean activation over the note (0..1).</summary>
+public readonly record struct TranscribedNote(double Start, double End, int Pitch, float Amplitude)
+{
+ public int Velocity => Math.Clamp((int)Math.Round(127 * Amplitude), 1, 127);
+}
+
+/// <summary>An audio clip read for Separate Stems (<see cref="IClipAi.ReadStemSource"/>).</summary>
+public abstract class StemJob
+{
+ public required int TrackId { get; init; }
+ public required int ClipIndex { get; init; }
+ /// <summary>The clip's name, else its track's: names the stem clips and their group.</summary>
+ public required string Name { get; init; }
+}
+
+/// <summary>Separated stems waiting to be applied; disposing deletes their temp files.</summary>
+public abstract class StemResult : IDisposable
+{
+ public abstract void Dispose();
+}
+
+/// <summary>The AI features on audio clips (models from <see cref="IModelStore"/>). Separate
+/// Stems runs in three steps so the engine stays on its thread: read (engine thread), separate
+/// (any thread; tens of seconds), apply (engine thread, one undo step). Failures throw
+/// <see cref="InvalidOperationException"/> with a user-facing message.</summary>
+public interface IClipAi
+{
+ bool CanSeparate { get; }
+ bool CanTranscribe { get; }
+
+ StemJob ReadStemSource(int trackId, int clipIndex);
+
+ StemResult Separate(StemJob job, IProgress<(double Fraction, string Label)>? progress = null, CancellationToken ct = default);
+
+ /// <summary>Puts a group of stem tracks (Drums, Bass, Other, Vocals) under the clip's track,
+ /// each with a clip shaped like the original, and switches the original off. Returns the
+ /// group's track id.</summary>
+ int ApplyStems(StemJob job, StemResult stems);
+
+ /// <summary>Notes in <paramref name="mono"/> (any sample rate), by start time.</summary>
+ IReadOnlyList<TranscribedNote> Transcribe(float[] mono, double sampleRate, IProgress<double>? progress = null, CancellationToken ct = default);
+}
diff --git a/src/managed/Nota.Infrastructure/Ai/ClipAi.cs b/src/managed/Nota.Infrastructure/Ai/ClipAi.cs
new file mode 100644
index 00000000..56b2816d
--- /dev/null
+++ b/src/managed/Nota.Infrastructure/Ai/ClipAi.cs
@@ -0,0 +1,236 @@
+// SPDX-License-Identifier: AGPL-3.0-only
+// Copyright (c) 2026 Egor Khindikaynen (Nota). See LICENSES/ for license terms.
+//
+// The AI features on an audio clip, shared by the clip menu and the MCP tools. Each runs in
+// three steps so the caller can keep the engine on its own thread: read the clip (engine
+// thread), run the model (any thread — tens of seconds for stems), apply the result (engine
+// thread, one undo step).
+//
+// Separate Stems works on the clip's *source*: the stems are written as files as long as the
+// source, at its sample rate, silent outside the part the clip uses, so each stem clip can take
+// the original clip's exact shape (region, warp markers, reverse, envelopes, ADSR —
+// AudioClipState) and line up with it sample for sample. The original clip is switched off,
+// not deleted.
+
+namespace Nota.Infrastructure;
+
+/// <summary>An audio clip read for separation: its shape, and the source audio to separate.</summary>
+public sealed class StemSource : StemJob
+{
+ public required AudioClipDto Shape { get; init; } // the clip, for the stem clips to copy
+ public required double SampleRate { get; init; }
+ public required long SourceFrames { get; init; } // the whole source's length
+ public required long From { get; init; } // separated region of the source
+ public required float[] Left { get; init; } // the region, de-interleaved
+ public required float[] Right { get; init; }
+}
+
+/// <summary>Separated stems written to temp WAVs (one per <see cref="StemSeparator.Stems"/>).</summary>
+public sealed class StemFiles : StemResult
+{
+ public required string[] Paths { get; init; }
+
+ public override void Dispose()
+ {
+ foreach (var p in Paths)
+ try { File.Delete(p); } catch { /* temp; best-effort */ }
+ }
+}
+
+public sealed class ClipAi(IAudioEngine engine, IModelStore models) : IClipAi
+{
+ // Unwarped clips: separate only the region the clip plays, plus a second either side so the
+ // model hears some context at the edges.
+ private const double MarginSeconds = 1.0;
+
+ /// <summary>The longest audio Separate Stems takes: the four stereo stems are held in memory
+ /// (~1 GB at this length).</summary>
+ public const double MaxSeconds = 10 * 60;
+
+ // ---- Separate Stems ------------------------------------------------------------------
+
+ /// <summary>Step 1 (engine thread). Throws <see cref="InvalidOperationException"/> with a
+ /// user-facing message when the clip can't be separated.</summary>
+ public bool CanSeparate => models.IsInstalled(AiModels.Stems) && models.RuntimePath is not null;
+
+ public StemJob ReadStemSource(int trackId, int clipIndex)
+ {
+ if (!engine.TryGetAudioClipInfo(trackId, clipIndex, out var ac))
+ throw new InvalidOperationException("That isn't an audio clip.");
+ if (!engine.TryGetSampleInfo(ac.SampleId, out var si) || si.Frames <= 0 || si.Channels <= 0 || si.SampleRate <= 0)
+ throw new InvalidOperationException("This clip has no audio to separate.");
+
+ long from = 0, to = si.Frames;
+ if (ac.WarpEnabled == 0)
+ {
+ long margin = (long)(MarginSeconds * si.SampleRate);
+ long start = (long)Math.Floor(ac.SourceOffsetFrames);
+ long end = ac.LengthFrames > 0 ? start + ac.LengthFrames : si.Frames;
+ from = Math.Clamp(start - margin, 0, si.Frames);
+ to = Math.Clamp(end + margin, from, si.Frames);
+ }
+ if ((to - from) / si.SampleRate > MaxSeconds)
+ throw new InvalidOperationException($"This clip is longer than {MaxSeconds / 60:0} minutes — split it first, then separate the parts.");
+ if (to - from < si.SampleRate * 0.1)
+ throw new InvalidOperationException("This clip is too short to separate.");
+
+ var data = engine.ReadSample(ac.SampleId);
+ int ch = si.Channels, n = (int)(to - from);
+ var left = new float[n];
+ var right = new float[n];
+ for (int i = 0; i < n; i++)
+ {
+ long at = (from + i) * ch;
+ if ((at + ch) > data.Length) break;
+ left[i] = data[at];
+ right[i] = ch > 1 ? data[at + 1] : data[at];
+ }
+
+ string name = engine.GetClipName(trackId, clipIndex) is { Length: > 0 } cn ? cn
+ : engine.GetTrackName(trackId) is { Length: > 0 } tn ? tn : "Audio";
+ return new StemSource
+ {
+ TrackId = trackId, ClipIndex = clipIndex, Name = name,
+ Shape = AudioClipState.Capture(engine, trackId, clipIndex, ac),
+ SampleRate = si.SampleRate, SourceFrames = si.Frames, From = from, Left = left, Right = right,
+ };
+ }
+
+ /// <summary>Step 2 (any thread): runs htdemucs and writes the stems as temp WAVs.
+ /// <paramref name="progress"/> reports 0..1 with a stage label.</summary>
+ public StemResult Separate(StemJob job, IProgress<(double Fraction, string Label)>? progress = null, CancellationToken ct = default)
+ {
+ var src = (StemSource)job;
+ var (runtime, model) = Require(AiModels.Stems);
+ progress?.Report((0, "Preparing audio…"));
+ double sr = src.SampleRate;
+ var l = Resampler.Convert(src.Left, sr, StemSeparator.SampleRate, ct);
+ var r = Resampler.Convert(src.Right, sr, StemSeparator.SampleRate, ct);
+
+ float[][][] stems;
+ progress?.Report((0, "Loading the model…"));
+ using (var sep = new StemSeparator(runtime, model))
+ stems = sep.Separate(l, r, new Progress(p => progress?.Report((p, "Separating stems…"))), ct);
+
+ progress?.Report((1, "Writing stems…"));
+ var files = new StemFiles { Paths = new string[StemSeparator.Stems.Length] };
+ try
+ {
+ int rate = (int)Math.Round(sr);
+ int regionLen = src.Left.Length;
+ for (int s = 0; s < StemSeparator.Stems.Length; s++)
+ {
+ ct.ThrowIfCancellationRequested();
+ var sl = Fit(Resampler.Convert(stems[s][0], StemSeparator.SampleRate, sr, ct), regionLen);
+ var sr2 = Fit(Resampler.Convert(stems[s][1], StemSeparator.SampleRate, sr, ct), regionLen);
+ var path = Path.Combine(Path.GetTempPath(), $"nota-stem-{Guid.NewGuid():N}.wav");
+ files.Paths[s] = path;
+ WriteStem(path, rate, src.SourceFrames, src.From, sl, sr2);
+ }
+ return files;
+ }
+ catch
+ {
+ files.Dispose();
+ throw;
+ }
+ }
+
+ /// <summary>Step 3 (engine thread): a group of stem tracks under the source track, each with
+ /// a clip shaped like the original, which is switched off. One undo step. Returns the group's id.</summary>
+ public int ApplyStems(StemJob job, StemResult stems)
+ {
+ var src = (StemSource)job;
+ var files = (StemFiles)stems;
+ int srcIndex = -1, parent = -1;
+ for (int i = 0; i < engine.TrackCount; i++)
+ if (engine.TryGetTrackInfo(i, out var ti) && ti.Id == src.TrackId) { srcIndex = i; parent = ti.GroupId; break; }
+ if (srcIndex < 0) throw new InvalidOperationException("The clip's track is gone.");
+
+ engine.BeginUndoGroup();
+ try
+ {
+ var ids = new int[StemSeparator.Stems.Length];
+ for (int s = 0; s < ids.Length; s++)
+ {
+ int id = engine.AddAudioTrack();
+ ids[s] = id;
+ engine.SetTrackName(id, StemSeparator.Stems[s]);
+ if (parent >= 0) engine.SetTrackGroup(id, parent);
+ engine.MoveTrack(id, srcIndex + 1 + s);
+ var shape = Clone(src.Shape);
+ shape.Name = $"{src.Name} {StemSeparator.Stems[s]}";
+ shape.Active = true;
+ if (AudioClipState.Restore(engine, id, shape, files.Paths[s]) < 0)
+ throw new InvalidOperationException("Nota couldn't load the separated stems.");
+ }
+ int group = engine.CreateGroup(ids);
+ if (group > 0) engine.SetTrackName(group, $"{src.Name} Stems");
+ engine.SetClipActive(src.TrackId, src.ClipIndex, false);
+ return group;
+ }
+ finally
+ {
+ engine.EndUndoGroup();
+ }
+ }
+
+ // ---- Convert to MIDI ----------------------------------------------------------------
+
+ /// <summary>Whether Convert Melody / Harmony can use basic-pitch.</summary>
+ public bool CanTranscribe => models.IsInstalled(AiModels.Transcription) && models.RuntimePath is not null;
+
+ /// <summary>Notes in <paramref name="mono"/> (any rate) with basic-pitch; times in seconds.</summary>
+ public IReadOnlyList<TranscribedNote> Transcribe(float[] mono, double sampleRate, IProgress<double>? progress = null, CancellationToken ct = default)
+ {
+ var (runtime, model) = Require(AiModels.Transcription);
+ var x = Resampler.Convert(mono, sampleRate, PitchTranscriber.SampleRate, ct);
+ using var tr = new PitchTranscriber(runtime, model);
+ return tr.Transcribe(x, progress, ct);
+ }
+
+ // ---- helpers ------------------------------------------------------------------------
+
+ private (string Runtime, string Model) Require(string id)
+ => models.RuntimePath is { } rt && models.ModelPath(id) is { } m
+ ? (rt, m)
+ : throw new InvalidOperationException("The AI model isn't installed — get it in Settings → Downloads → AI Models.");
+
+ private static float[] Fit(float[] x, int n)
+ {
+ if (x.Length == n) return x;
+ var y = new float[n];
+ Array.Copy(x, y, Math.Min(n, x.Length));
+ return y;
+ }
+
+ // The stem as long as the whole source, silent outside the separated region (24-bit is
+ // plenty for a stem and keeps the project a third smaller than float).
+ private static void WriteStem(string path, int rate, long sourceFrames, long from, float[] l, float[] r)
+ {
+ using var w = new WavWriter(path, rate, 2, WavBitDepth.Pcm24);
+ const int block = 1 << 15;
+ var buf = new float[block * 2];
+ for (long pos = 0; pos < sourceFrames; pos += block)
+ {
+ int n = (int)Math.Min(block, sourceFrames - pos);
+ for (int i = 0; i < n; i++)
+ {
+ long k = pos + i - from;
+ bool inside = k >= 0 && k < l.Length;
+ buf[2 * i] = inside ? l[k] : 0f;
+ buf[2 * i + 1] = inside ? r[k] : 0f;
+ }
+ w.WriteFrames(buf, n);
+ }
+ }
+
+ private static AudioClipDto Clone(AudioClipDto d)
+ => System.Text.Json.JsonSerializer.Deserialize<AudioClipDto>(System.Text.Json.JsonSerializer.Serialize(d))!;
+
+ // IProgress that reports on the calling thread (the separator's), not via a context.
+ private sealed class Progress(Action<double> report) : IProgress<double>
+ {
+ public void Report(double value) => report(value);
+ }
+}
diff --git a/src/managed/Nota.Infrastructure/Ai/PitchTranscriber.cs b/src/managed/Nota.Infrastructure/Ai/PitchTranscriber.cs
index e9d76cf3..3649b150 100644
--- a/src/managed/Nota.Infrastructure/Ai/PitchTranscriber.cs
+++ b/src/managed/Nota.Infrastructure/Ai/PitchTranscriber.cs
@@ -11,13 +11,6 @@ using Microsoft.ML.OnnxRuntime;
namespace Nota.Infrastructure;
-/// <summary>A transcribed note: times in seconds from the start of the audio, MIDI pitch, and
-/// the model's mean frame activation (0..1, basic-pitch's "amplitude").</summary>
-public readonly record struct TranscribedNote(double Start, double End, int Pitch, float Amplitude)
-{
- public int Velocity => Math.Clamp((int)Math.Round(127 * Amplitude), 1, 127);
-}
-
public sealed class PitchTranscriber : IDisposable
{
public const int SampleRate = 22050;
diff --git a/src/managed/Nota.Infrastructure/Persistence/AudioClipState.cs b/src/managed/Nota.Infrastructure/Persistence/AudioClipState.cs
new file mode 100644
index 00000000..155759d7
--- /dev/null
+++ b/src/managed/Nota.Infrastructure/Persistence/AudioClipState.cs
@@ -0,0 +1,82 @@
+// SPDX-License-Identifier: AGPL-3.0-only
+// Copyright (c) 2026 Egor Khindikaynen (Nota). See LICENSES/ for license terms.
+//
+// Everything that makes an audio clip play the way it does — region, gain, pitch, reverse,
+// warp (mode, markers, trim), clip envelopes, ADSR — read off the engine and put back onto a
+// new clip. The project file saves and loads clips through it, and Separate Stems uses it to
+// give each stem a clip shaped exactly like the one it came from, over a different file.
+
+namespace Nota.Infrastructure;
+
+internal static class AudioClipState
+{
+ /// <summary>The clip's state; <see cref="AudioClipDto.Sample"/> is left for the caller.</summary>
+ public static AudioClipDto Capture(IAudioEngine engine, int trackId, int clipIndex, NotaAudioClipInfo ac)
+ {
+ var dto = new AudioClipDto
+ {
+ Name = engine.GetClipName(trackId, clipIndex) is { Length: > 0 } an ? an : null,
+ Active = !engine.TryGetClipInfo(trackId, clipIndex, out var gci) || gci.IsActive,
+ StartBeat = ac.StartBeat,
+ SourceOffsetFrames = ac.SourceOffsetFrames,
+ LengthFrames = ac.LengthFrames,
+ Gain = ac.Gain,
+ PitchSemitones = ac.PitchSemitones,
+ WarpEnabled = ac.WarpEnabled,
+ WarpMode = ac.WarpMode,
+ WarpBeats = ac.WarpBeats,
+ WarpPlayStart = ac.WarpPlayStart,
+ WarpPlayEnd = ac.WarpPlayEnd,
+ Reversed = ac.Reversed != 0,
+ };
+ if (ac.WarpEnabled != 0)
+ {
+ var ms = new double[128]; var mb = new double[128];
+ int nm = Math.Min(engine.GetClipWarpMarkers(trackId, clipIndex, ms, mb), 128);
+ if (nm >= 2)
+ {
+ dto.WarpMarkers = new List<WarpMarkerDto>(nm);
+ for (int k = 0; k < nm; k++) dto.WarpMarkers.Add(new WarpMarkerDto { Src = ms[k], Beat = mb[k] });
+ }
+ }
+ var env = engine.GetClipVolumeEnvelope(trackId, clipIndex); // clip volume envelope (v8)
+ if (env.Length > 0) dto.VolumeEnvelope = Array.ConvertAll(env, p => new AutomationPointDto(p));
+ var penv = engine.GetClipPanEnvelope(trackId, clipIndex); // clip pan envelope (v9)
+ if (penv.Length > 0) dto.PanEnvelope = Array.ConvertAll(penv, p => new AutomationPointDto(p));
+ var adsr = engine.GetClipAdsr(trackId, clipIndex); // ADSR (v20); identity is omitted
+ if (!adsr.IsIdentity) dto.Adsr = new ClipAdsrDto(adsr);
+ return dto;
+ }
+
+ /// <summary>Adds a clip playing <paramref name="path"/> shaped as <paramref name="ac"/> on
+ /// <paramref name="trackId"/>. Returns its index, or -1 when the file can't be loaded.</summary>
+ public static int Restore(IAudioEngine engine, int trackId, AudioClipDto ac, string path)
+ {
+ int ci = engine.AddAudioClipEx(trackId, path, ac.StartBeat, ac.SourceOffsetFrames, ac.LengthFrames, ac.Gain);
+ if (ci < 0) return -1;
+ if (ac.Name is { Length: > 0 } acn) engine.SetClipName(trackId, ci, acn);
+ if (!ac.Active) engine.SetClipActive(trackId, ci, false); // clip deactivate (v17)
+ if (ac.PitchSemitones != 0) engine.SetClipPitch(trackId, ci, ac.PitchSemitones);
+ if (ac.Reversed) engine.SetClipReverse(trackId, ci, true); // reverse (v19)
+ if (ac.WarpEnabled != 0)
+ {
+ engine.SetClipWarp(trackId, ci, true, ac.WarpMode);
+ if (ac.WarpMarkers is { Count: >= 2 } wm)
+ {
+ var ms = new double[wm.Count]; var mb = new double[wm.Count];
+ for (int k = 0; k < wm.Count; k++) { ms[k] = wm[k].Src; mb[k] = wm[k].Beat; }
+ engine.SetClipWarpMarkers(trackId, ci, ms, mb); // also sets warpBeats from the last marker
+ }
+ else if (ac.WarpBeats > 0) engine.SetClipWarpLength(trackId, ci, ac.WarpBeats);
+ if (ac.WarpPlayEnd > 0) // restore the trim window (v11); 0 = whole warp
+ engine.SetClipWarpTrim(trackId, ci, ac.WarpPlayStart, ac.WarpPlayEnd);
+ }
+ if (ac.VolumeEnvelope is { Length: > 0 } venv) // clip volume envelope (v8)
+ engine.SetClipVolumeEnvelope(trackId, ci, Array.ConvertAll(venv, p => p.ToPoint()));
+ if (ac.PanEnvelope is { Length: > 0 } penv) // clip pan envelope (v9)
+ engine.SetClipPanEnvelope(trackId, ci, Array.ConvertAll(penv, p => p.ToPoint()));
+ if (ac.Adsr is { } adsr) // ADSR (v20)
+ engine.SetClipAdsr(trackId, ci, adsr.ToAdsr());
+ return ci;
+ }
+}
diff --git a/src/managed/Nota.Infrastructure/Persistence/ProjectService.cs b/src/managed/Nota.Infrastructure/Persistence/ProjectService.cs
index 692e687c..bc32eca7 100644
--- a/src/managed/Nota.Infrastructure/Persistence/ProjectService.cs
+++ b/src/managed/Nota.Infrastructure/Persistence/ProjectService.cs
@@ -202,39 +202,8 @@ public sealed class ProjectService
if (!engine.TryGetAudioClipInfo(ti.Id, c, out var ac)) continue;
string? rel = RegisterSample(doc, engine, ac.SampleId, warnings);
if (rel == null) continue;
- var dto = new AudioClipDto
- {
- Sample = rel,
- Name = engine.GetClipName(ti.Id, c) is { Length: > 0 } an ? an : null,
- Active = !engine.TryGetClipInfo(ti.Id, c, out var gci) || gci.IsActive,
- StartBeat = ac.StartBeat,
- SourceOffsetFrames = ac.SourceOffsetFrames,
- LengthFrames = ac.LengthFrames,
- Gain = ac.Gain,
- PitchSemitones = ac.PitchSemitones,
- WarpEnabled = ac.WarpEnabled,
- WarpMode = ac.WarpMode,
- WarpBeats = ac.WarpBeats,
- WarpPlayStart = ac.WarpPlayStart,
- WarpPlayEnd = ac.WarpPlayEnd,
- Reversed = ac.Reversed != 0,
- };
- if (ac.WarpEnabled != 0)
- {
- var ms = new double[128]; var mb = new double[128];
- int nm = Math.Min(engine.GetClipWarpMarkers(ti.Id, c, ms, mb), 128);
- if (nm >= 2)
- {
- dto.WarpMarkers = new List<WarpMarkerDto>(nm);
- for (int k = 0; k < nm; k++) dto.WarpMarkers.Add(new WarpMarkerDto { Src = ms[k], Beat = mb[k] });
- }
- }
- var env = engine.GetClipVolumeEnvelope(ti.Id, c); // clip volume envelope (v8)
- if (env.Length > 0) dto.VolumeEnvelope = System.Array.ConvertAll(env, p => new AutomationPointDto(p));
- var penv = engine.GetClipPanEnvelope(ti.Id, c); // clip pan envelope (v9)
- if (penv.Length > 0) dto.PanEnvelope = System.Array.ConvertAll(penv, p => new AutomationPointDto(p));
- var adsr = engine.GetClipAdsr(ti.Id, c); // ADSR (v20); identity is omitted
- if (!adsr.IsIdentity) dto.Adsr = new ClipAdsrDto(adsr);
+ var dto = AudioClipState.Capture(engine, ti.Id, c, ac);
+ dto.Sample = rel;
t.AudioClips.Add(dto);
}
}
@@ -650,33 +619,9 @@ public sealed class ProjectService
foreach (var ac in t.AudioClips)
{
- int ci = engine.AddAudioClipEx(id, ResolveInBundle(bundleDir, ac.Sample),
- ac.StartBeat, ac.SourceOffsetFrames, ac.LengthFrames, ac.Gain);
+ int ci = AudioClipState.Restore(engine, id, ac, ResolveInBundle(bundleDir, ac.Sample));
if (ci < 0) { warnings.Add($"Couldn't reload audio clip \"{ac.Sample}\"."); continue; }
if (engine.TryGetAudioClipInfo(id, ci, out var lci)) BundleContent.SeedSample(engine, lci.SampleId, ac.Sample);
- if (ac.Name is { Length: > 0 } acn) engine.SetClipName(id, ci, acn);
- if (!ac.Active) engine.SetClipActive(id, ci, false); // clip deactivate (v17)
- if (ac.PitchSemitones != 0) engine.SetClipPitch(id, ci, ac.PitchSemitones);
- if (ac.Reversed) engine.SetClipReverse(id, ci, true); // reverse (v19)
- if (ac.WarpEnabled != 0)
- {
- engine.SetClipWarp(id, ci, true, ac.WarpMode);
- if (ac.WarpMarkers is { Count: >= 2 } wm)
- {
- var ms = new double[wm.Count]; var mb = new double[wm.Count];
- for (int k = 0; k < wm.Count; k++) { ms[k] = wm[k].Src; mb[k] = wm[k].Beat; }
- engine.SetClipWarpMarkers(id, ci, ms, mb); // also sets warpBeats from the last marker
- }
- else if (ac.WarpBeats > 0) engine.SetClipWarpLength(id, ci, ac.WarpBeats);
- if (ac.WarpPlayEnd > 0) // restore the trim window (v11); 0 = whole warp
- engine.SetClipWarpTrim(id, ci, ac.WarpPlayStart, ac.WarpPlayEnd);
- }
- if (ac.VolumeEnvelope is { Length: > 0 } venv) // clip volume envelope (v8)
- engine.SetClipVolumeEnvelope(id, ci, System.Array.ConvertAll(venv, p => p.ToPoint()));
- if (ac.PanEnvelope is { Length: > 0 } penv) // clip pan envelope (v9)
- engine.SetClipPanEnvelope(id, ci, System.Array.ConvertAll(penv, p => p.ToPoint()));
- if (ac.Adsr is { } adsr) // ADSR (v20)
- engine.SetClipAdsr(id, ci, adsr.ToAdsr());
}
// Freeze (M7): restore the frozen-audio buffer so the track re-opens frozen
diff --git a/tests/Nota.SmokeTest/AiGoldenTests.cs b/tests/Nota.SmokeTest/AiGoldenTests.cs
index f31c0ae7..474b4985 100644
--- a/tests/Nota.SmokeTest/AiGoldenTests.cs
+++ b/tests/Nota.SmokeTest/AiGoldenTests.cs
@@ -9,6 +9,7 @@
// where <dir> holds htdemucs.onnx, nmp.onnx and fx/ (raw little-endian float32 files).
using System.Text.Json;
+using Nota.Application;
using Nota.Infrastructure;
namespace Nota.SmokeTest;
@@ -95,6 +96,8 @@ internal static class AiGoldenTests
yield return (matched >= refNotes.Count * 0.98 && Math.Abs(notes.Count - refNotes.Count) <= Math.Max(2, refNotes.Count / 50),
$"basic-pitch end to end: {matched}/{refNotes.Count} reference notes matched to a frame ({notes.Count} found)");
+ foreach (var r in ClipFlow(runtime, dir, left, right)) yield return r;
+
// ---- resampler -----------------------------------------------------------------------
var tone = new float[48000];
for (int i = 0; i < tone.Length; i++) tone[i] = (float)Math.Sin(2 * Math.PI * 1000 * i / 48000.0);
@@ -104,6 +107,82 @@ internal static class AiGoldenTests
yield return (down.Length == 44100 && rErr < 1e-3, $"resampler 48 → 44.1 kHz keeps a 1 kHz tone (max err {rErr:g3})");
}
+ // Separate Stems end to end on a real engine: a 48 kHz clip using part of its file, read,
+ // separated, applied as a group of stem tracks shaped like the clip — one undo step — and
+ // the stems summed back sound like the clip.
+ private static IEnumerable<(bool, string)> ClipFlow(string runtime, string dir, float[] left44, float[] right44)
+ {
+ const int rate = 48000;
+ var l = Resampler.Convert(left44, 44100, rate);
+ var r = Resampler.Convert(right44, 44100, rate);
+ var wav = Path.Combine(Path.GetTempPath(), $"nota-ai-clip-{Guid.NewGuid():N}.wav");
+ var inter = new float[l.Length * 2];
+ for (int i = 0; i < l.Length; i++) { inter[2 * i] = l[i]; inter[2 * i + 1] = r[i]; }
+ using (var w = new Nota.Infrastructure.WavWriter(wav, rate, 2, WavBitDepth.Float32)) w.WriteFrames(inter, l.Length);
+
+ using var eng = new NotaEngine();
+ eng.SetBpm(120);
+ int t = eng.AddAudioTrack();
+ eng.SetTrackName(t, "Mix");
+ int c = eng.AddAudioClipEx(t, wav, 4, 2 * rate, 8 * rate, 1f); // beat 4 (2 s), 8 s from 2 s into the file
+ File.Delete(wav);
+ int before = eng.TrackCount;
+
+ var ai = new ClipAi(eng, new FixedModels(runtime, Path.Combine(dir, "htdemucs.onnx")));
+ var job = ai.ReadStemSource(t, c);
+ int group;
+ using (var stems = ai.Separate(job))
+ group = ai.ApplyStems(job, stems);
+
+ var tracks = Enumerable.Range(0, eng.TrackCount).Select(i => { eng.TryGetTrackInfo(i, out var ti); return ti; }).ToList();
+ var kids = tracks.Where(x => x.GroupId == group).ToList();
+ yield return (group > 0 && eng.TrackCount == before + 5 && tracks[1].Id == group
+ && kids.Select(k => eng.GetTrackName(k.Id)).SequenceEqual(StemSeparator.Stems),
+ "Separate Stems puts a group of Drums, Bass, Other, Vocals right under the clip's track");
+ eng.TryGetAudioClipInfo(t, c, out var orig);
+ bool shaped = kids.All(k => eng.TryGetAudioClipInfo(k.Id, 0, out var sc)
+ && sc.StartBeat == orig.StartBeat && sc.SourceOffsetFrames == orig.SourceOffsetFrames && sc.LengthFrames == orig.LengthFrames);
+ yield return (shaped, "…each stem clip shaped exactly like the original");
+ yield return (eng.TryGetClipInfo(t, c, out var oc) && !oc.IsActive && eng.GetClipName(kids[3].Id, 0) == "Mix Vocals",
+ "…the original clip switched off, the stem clips named after it");
+
+ eng.Undo();
+ bool undone = eng.TrackCount == before && eng.TryGetClipInfo(t, c, out var uc) && uc.IsActive;
+ eng.Redo();
+ yield return (undone && eng.TrackCount == before + 5, "…as one undo step");
+
+ // The stems summed against the clip itself, over the clip's span (2 s … 10 s).
+ float[] Render()
+ {
+ var buf = new float[11 * rate * 2];
+ eng.Seek(0); eng.Play();
+ eng.RenderOffline(buf, 11 * rate, rate);
+ eng.Stop();
+ return buf;
+ }
+ var sum = Render();
+ eng.SetClipActive(t, c, true);
+ eng.SetTrackMute(group, true);
+ var mix = Render();
+ double err = 0, sig = 0;
+ for (int i = 2 * rate * 2 + 2000; i < 10 * rate * 2 - 2000; i++) { double d = sum[i] - mix[i]; err += d * d; sig += mix[i] * (double)mix[i]; }
+ double snr = 10 * Math.Log10(sig / Math.Max(1e-20, err));
+ yield return (sig > 0 && snr > 15, $"…and the stems add back up to the clip ({snr:0.0} dB)");
+ }
+
+ private sealed class FixedModels(string runtime, string stems) : IModelStore
+ {
+ public IReadOnlyList<StoreModel> Models => [];
+ public StoreModel? Runtime => null;
+ public bool IsInstalled(string id) => true;
+ public long DownloadSize(string id) => 0;
+ public Task InstallAsync(string id, IProgress<StoreProgress>? progress = null, CancellationToken ct = default) => Task.CompletedTask;
+ public void Uninstall(string id) { }
+ public string? RuntimePath => runtime;
+ public string? ModelPath(string id) => id == AiModels.Stems ? stems : null;
+ public event Action? Changed { add { } remove { } }
+ }
+
private static float[] Floats(string path)
{
var bytes = File.ReadAllBytes(path);