Skip to content

armgit / nota

Public Imported from GitHub
0

Nota — a free DAW for macOS, Windows and Linux. No subscription, no account — just open it and make music.

(feat) AI models: htdemucs stem separation and basic-pitch transcription ported to C# on ONNX Runtime

Egor Khindikaynend54da77eab19
diff --git a/src/managed/Nota.Infrastructure/Ai/DemucsSpec.cs b/src/managed/Nota.Infrastructure/Ai/DemucsSpec.cs
new file mode 100644
index 00000000..fcf77ddb
--- /dev/null
+++ b/src/managed/Nota.Infrastructure/Ai/DemucsSpec.cs
@@ -0,0 +1,106 @@
+// SPDX-License-Identifier: AGPL-3.0-only
+// Copyright (c) 2026 Egor Khindikaynen (Nota). See LICENSES/ for license terms.
+//
+// The STFT / iSTFT that htdemucs runs around its network. The ONNX export leaves them out
+// (ONNX has no complex STFT), so they are reproduced here exactly as HTDemucs._spec / _ispec
+// and demucs/spec.py compute them: a 4096-point periodic Hann window, hop 1024, torch's
+// "normalized" scaling (1/√N each way), centre padding by reflection, plus htdemucs's own
+// reflect pad of 1.5 hops so the frame count is exactly ceil(length / hop). Spectra are
+// bin-major: [bin * frames + frame], 2048 bins (the Nyquist bin is dropped, as htdemucs does).
+
+namespace Nota.Infrastructure;
+
+internal static class DemucsSpec
+{
+ public const int NFft = 4096, Hop = 1024, Bins = NFft / 2;
+ private const int Pad = Hop / 2 * 3; // htdemucs's reflect pad around the signal
+ private static readonly double Norm = 1.0 / Math.Sqrt(NFft);
+ private static readonly double[] Window = MakeWindow();
+
+ private static double[] MakeWindow()
+ {
+ var w = new double[NFft];
+ for (int n = 0; n < NFft; n++) w[n] = 0.5 - 0.5 * Math.Cos(2 * Math.PI * n / NFft); // periodic, as torch.hann_window
+ return w;
+ }
+
+ /// <summary>Frames the spectrum of a <paramref name="length"/>-sample signal has.</summary>
+ public static int Frames(int length) => (length + Hop - 1) / Hop;
+
+ /// <summary>Spectrum of one channel into <paramref name="re"/> / <paramref name="im"/>
+ /// (each <see cref="Bins"/> × <see cref="Frames"/>).</summary>
+ public static void Forward(ReadOnlySpan<float> x, float[] re, float[] im)
+ {
+ int len = x.Length, le = Frames(len);
+ int p1 = le * Hop + 2 * Pad; // after htdemucs's pad (pad, pad + le*hop - len)
+ var fr = new double[NFft];
+ var fi = new double[NFft];
+ for (int t = 0; t < le; t++)
+ {
+ // Kept frame t is STFT frame t + 2; STFT frames start at hop multiples of the
+ // centre-padded signal (NFft/2 of reflection either side of the p1-long one).
+ int start = (t + 2) * Hop - NFft / 2;
+ for (int n = 0; n < NFft; n++)
+ {
+ int i = Reflect(start + n, p1) - Pad;
+ fr[n] = x[Reflect(i, len)] * Window[n];
+ fi[n] = 0;
+ }
+ AudioFft.Forward(fr, fi);
+ for (int k = 0; k < Bins; k++)
+ {
+ re[k * le + t] = (float)(fr[k] * Norm);
+ im[k * le + t] = (float)(fi[k] * Norm);
+ }
+ }
+ }
+
+ /// <summary>Signal of <paramref name="length"/> samples back from a spectrum of
+ /// <see cref="Frames"/>(<paramref name="length"/>) frames (its planes start at the given
+ /// offsets), into <paramref name="dst"/>.</summary>
+ public static void Inverse(float[] re, float[] im, int reOffset, int imOffset, int length, Span<float> dst)
+ {
+ int le = Frames(length);
+ int frames = le + 4; // two silent frames either side, as _ispec pads
+ int olaLen = NFft + Hop * (frames - 1);
+ var ola = new double[olaLen];
+ var env = new double[olaLen];
+ var fr = new double[NFft];
+ var fi = new double[NFft];
+ double scale = Math.Sqrt(NFft); // undoes the forward 1/√N ("normalized")
+ for (int f = 0; f < frames; f++)
+ {
+ int at = f * Hop;
+ for (int n = 0; n < NFft; n++) env[at + n] += Window[n] * Window[n];
+ int t = f - 2;
+ if (t < 0 || t >= le) continue; // a silent frame adds only to the envelope
+ fr[0] = re[reOffset + t]; fi[0] = 0; // irfft ignores the DC bin's imaginary part
+ for (int k = 1; k < Bins; k++)
+ {
+ double a = re[reOffset + k * le + t], b = im[imOffset + k * le + t];
+ fr[k] = a; fi[k] = b;
+ fr[NFft - k] = a; fi[NFft - k] = -b;
+ }
+ fr[Bins] = 0; fi[Bins] = 0; // the dropped Nyquist bin
+ AudioFft.Inverse(fr, fi);
+ for (int n = 0; n < NFft; n++) ola[at + n] += fr[n] * scale * Window[n];
+ }
+ // istft drops NFft/2 of centre padding; _ispec then drops its own 1.5-hop pad.
+ int off = NFft / 2 + Pad;
+ for (int i = 0; i < length; i++)
+ {
+ double e = env[off + i];
+ dst[i] = e > 1e-11 ? (float)(ola[off + i] / e) : 0f;
+ }
+ }
+
+ // numpy/torch "reflect" (edge sample not repeated) into [0, n).
+ private static int Reflect(int i, int n)
+ {
+ if (n == 1) return 0;
+ int period = 2 * (n - 1);
+ i %= period;
+ if (i < 0) i += period;
+ return i < n ? i : period - i;
+ }
+}
diff --git a/src/managed/Nota.Infrastructure/Ai/OnnxRuntimeLoader.cs b/src/managed/Nota.Infrastructure/Ai/OnnxRuntimeLoader.cs
new file mode 100644
index 00000000..76d4278c
--- /dev/null
+++ b/src/managed/Nota.Infrastructure/Ai/OnnxRuntimeLoader.cs
@@ -0,0 +1,56 @@
+// SPDX-License-Identifier: AGPL-3.0-only
+// Copyright (c) 2026 Egor Khindikaynen (Nota). See LICENSES/ for license terms.
+//
+// Nota ships only ONNX Runtime's managed half (Microsoft.ML.OnnxRuntime.Managed); the native
+// library is downloaded with the first AI model (ModelStore) to keep the installer small. The
+// managed assembly P/Invokes "onnxruntime", so a resolver points that name at the downloaded
+// file. It must be in place before the first InferenceSession, and once the library is loaded
+// the process keeps it.
+
+using System.Reflection;
+using System.Runtime.InteropServices;
+using Microsoft.ML.OnnxRuntime;
+
+namespace Nota.Infrastructure;
+
+internal static class OnnxRuntimeLoader
+{
+ private static readonly object Gate = new();
+ private static string? _path;
+ private static IntPtr _handle;
+
+ /// <summary>Points the ORT assembly at the native library at <paramref name="libraryPath"/>.
+ /// Throws <see cref="StoreException"/> when it can't be loaded.</summary>
+ public static void Use(string libraryPath)
+ {
+ lock (Gate)
+ {
+ if (_handle != IntPtr.Zero) return; // already loaded; a process can't swap it
+ if (!NativeLibrary.TryLoad(libraryPath, out var h))
+ throw new StoreException("Nota couldn't load the AI runtime. Remove it in Settings → Downloads → AI Models and install it again.");
+ _handle = h;
+ if (_path is null)
+ NativeLibrary.SetDllImportResolver(typeof(InferenceSession).Assembly, Resolve);
+ _path = libraryPath;
+ }
+ }
+
+ private static IntPtr Resolve(string name, Assembly assembly, DllImportSearchPath? search)
+ => name == "onnxruntime" ? _handle : IntPtr.Zero;
+
+ /// <summary>A CPU session for the model at <paramref name="modelPath"/>. ORT's own thread
+ /// count (one per performance core) beats every core: efficiency cores slow the whole step.</summary>
+ public static InferenceSession Session(string modelPath)
+ {
+ var opts = new SessionOptions
+ {
+ GraphOptimizationLevel = GraphOptimizationLevel.ORT_ENABLE_ALL,
+ InterOpNumThreads = 1,
+ };
+ try { return new InferenceSession(modelPath, opts); }
+ catch (OnnxRuntimeException e)
+ {
+ throw new StoreException("The AI model file is damaged. Remove it in Settings → Downloads → AI Models and install it again.", e);
+ }
+ }
+}
diff --git a/src/managed/Nota.Infrastructure/Ai/PitchTranscriber.cs b/src/managed/Nota.Infrastructure/Ai/PitchTranscriber.cs
new file mode 100644
index 00000000..e9d76cf3
--- /dev/null
+++ b/src/managed/Nota.Infrastructure/Ai/PitchTranscriber.cs
@@ -0,0 +1,238 @@
+// SPDX-License-Identifier: AGPL-3.0-only
+// Copyright (c) 2026 Egor Khindikaynen (Nota). See LICENSES/ for license terms.
+//
+// Polyphonic audio → notes with Spotify's basic-pitch (Apache-2.0). The model takes 2 s
+// windows of 22.05 kHz mono and returns, per 256-sample frame, note and onset activations for
+// the 88 piano keys. Windowing, unwrapping and note decoding are ported from
+// basic_pitch.inference / note_creation (output_to_notes_polyphonic with inferred onsets and the
+// "melodia trick"), with the library's default thresholds. Pitch bends are left out.
+
+using Microsoft.ML.OnnxRuntime;
+
+namespace Nota.Infrastructure;
+
+/// <summary>A transcribed note: times in seconds from the start of the audio, MIDI pitch, and
+/// the model's mean frame activation (0..1, basic-pitch's "amplitude").</summary>
+public readonly record struct TranscribedNote(double Start, double End, int Pitch, float Amplitude)
+{
+ public int Velocity => Math.Clamp((int)Math.Round(127 * Amplitude), 1, 127);
+}
+
+public sealed class PitchTranscriber : IDisposable
+{
+ public const int SampleRate = 22050;
+ private const int FftHop = 256;
+ private const int WindowSamples = SampleRate * 2 - FftHop; // AUDIO_N_SAMPLES = 43844
+ private const int FramesPerWindow = 172; // ANNOT_N_FRAMES
+ private const int Keys = 88, MidiOffset = 21;
+ private const int OverlapFrames = 30;
+ private const int OverlapLen = OverlapFrames * FftHop;
+ private const int HopSize = WindowSamples - OverlapLen;
+ private const int Batch = 8;
+ private const int Fps = SampleRate / FftHop; // ANNOTATIONS_FPS = 86
+
+ private const float OnsetThresh = 0.5f, FrameThresh = 0.3f;
+ private const int MinNoteLen = 11; // round(127.70 ms × 86.13 fps)
+ private const int EnergyTol = 11;
+
+ private readonly InferenceSession _session;
+ private readonly string _input;
+
+ public PitchTranscriber(string runtimePath, string modelPath)
+ {
+ OnnxRuntimeLoader.Use(runtimePath);
+ _session = OnnxRuntimeLoader.Session(modelPath);
+ _input = _session.InputMetadata.Keys.First();
+ }
+
+ public void Dispose() => _session.Dispose();
+
+ /// <summary>Notes in <paramref name="mono"/> (at <see cref="SampleRate"/>), by start time.</summary>
+ public List<TranscribedNote> Transcribe(float[] mono, IProgress<double>? progress = null, CancellationToken ct = default)
+ {
+ var (note, onset, n) = Infer(mono, progress, ct);
+ var events = Decode(note, onset, n);
+ var notes = new List<TranscribedNote>(events.Count);
+ foreach (var (s, e, p, a) in events) notes.Add(new TranscribedNote(FrameTime(s), FrameTime(e), p, a));
+ notes.Sort((a, b) => a.Start != b.Start ? a.Start.CompareTo(b.Start) : a.Pitch.CompareTo(b.Pitch));
+ return notes;
+ }
+
+ /// <summary>One line out of a transcription: wherever notes overlap, the strongest one
+ /// wins and the others are dropped (Convert Melody).</summary>
+ public static List<TranscribedNote> Monophonic(IReadOnlyList<TranscribedNote> notes)
+ {
+ var kept = new List<TranscribedNote>();
+ foreach (var n in notes.OrderByDescending(n => n.Amplitude * (n.End - n.Start)))
+ if (!kept.Any(k => k.Start < n.End && n.Start < k.End)) kept.Add(n);
+ kept.Sort((a, b) => a.Start.CompareTo(b.Start));
+ return kept;
+ }
+
+ // ---- inference (inference.run_inference) -----------------------------------------------
+
+ private (float[] Note, float[] Onset, int Frames) Infer(float[] mono, IProgress<double>? progress, CancellationToken ct)
+ {
+ int original = mono.Length;
+ int padded = OverlapLen / 2 + original; // half an overlap of silence in front
+ int windows = Math.Max(1, (padded + HopSize - 1) / HopSize);
+ int keep = FramesPerWindow - OverlapFrames; // unwrap drops half the overlap at each end
+ int total = (int)Math.Floor(original * ((double)Fps / SampleRate));
+ var note = new float[Math.Max(0, total) * Keys];
+ var onset = new float[note.Length];
+
+ var input = new float[Batch * WindowSamples];
+ for (int w0 = 0; w0 < windows; w0 += Batch)
+ {
+ ct.ThrowIfCancellationRequested();
+ int b = Math.Min(Batch, windows - w0);
+ Array.Clear(input);
+ for (int w = 0; w < b; w++)
+ {
+ int from = (w0 + w) * HopSize - OverlapLen / 2; // in the unpadded signal
+ for (int i = 0; i < WindowSamples; i++)
+ {
+ int k = from + i;
+ if (k >= 0 && k < original) input[w * WindowSamples + i] = mono[k];
+ }
+ }
+ using var x = OrtValue.CreateTensorValueFromMemory(input, [b, WindowSamples, 1]);
+ using var run = new RunOptions();
+ using var r = _session.Run(run, [_input], [x], ["StatefulPartitionedCall:1", "StatefulPartitionedCall:2"]);
+ var nt = r[0].GetTensorDataAsSpan<float>();
+ var on = r[1].GetTensorDataAsSpan<float>();
+ for (int w = 0; w < b; w++)
+ for (int f = 0; f < keep; f++)
+ {
+ int dst = (w0 + w) * keep + f;
+ if (dst >= total) break;
+ int src = (w * FramesPerWindow + OverlapFrames / 2 + f) * Keys;
+ nt.Slice(src, Keys).CopyTo(note.AsSpan(dst * Keys, Keys));
+ on.Slice(src, Keys).CopyTo(onset.AsSpan(dst * Keys, Keys));
+ }
+ progress?.Report(Math.Min(1, (w0 + b) / (double)windows));
+ }
+ return (note, onset, Math.Max(0, total));
+ }
+
+ // note_creation.model_frames_to_time: frame index → seconds, with basic-pitch's per-window
+ // drift correction.
+ private static double FrameTime(int frame)
+ {
+ double t = frame * (double)FftHop / SampleRate;
+ double windowOffset = (double)FftHop / SampleRate * (FramesPerWindow - (double)WindowSamples / FftHop) + 0.0018;
+ return t - windowOffset * Math.Floor((double)frame / FramesPerWindow);
+ }
+
+ // ---- decoding (note_creation.output_to_notes_polyphonic) --------------------------------
+
+ internal static List<(int Start, int End, int Pitch, float Amplitude)> Decode(float[] frames, float[] onsetsIn, int n)
+ {
+ var events = new List<(int, int, int, float)>();
+ if (n == 0) return events;
+ var onsets = InferOnsets(onsetsIn, frames, n);
+
+ // Onset peaks (strict local maxima in time) above the threshold, latest first.
+ var peaks = new List<(int T, int F)>();
+ for (int t = 1; t < n - 1; t++)
+ for (int f = 0; f < Keys; f++)
+ {
+ float v = onsets[t * Keys + f];
+ if (v >= OnsetThresh && v > onsets[(t - 1) * Keys + f] && v > onsets[(t + 1) * Keys + f]) peaks.Add((t, f));
+ }
+ peaks.Reverse();
+
+ var energy = (float[])frames.Clone();
+ foreach (var (start, f) in peaks)
+ {
+ if (start >= n - 1) continue;
+ int i = start + 1, k = 0;
+ while (i < n - 1 && k < EnergyTol)
+ {
+ k = energy[i * Keys + f] < FrameThresh ? k + 1 : 0;
+ i++;
+ }
+ i -= k;
+ if (i - start <= MinNoteLen) continue;
+ for (int t = start; t < i; t++)
+ {
+ energy[t * Keys + f] = 0;
+ if (f < Keys - 1) energy[t * Keys + f + 1] = 0;
+ if (f > 0) energy[t * Keys + f - 1] = 0;
+ }
+ events.Add((start, i, f + MidiOffset, Mean(frames, start, i, f)));
+ }
+
+ // Melodia trick: follow the strongest remaining energy both ways into further notes.
+ while (true)
+ {
+ int best = -1; float max = FrameThresh;
+ for (int j = 0; j < energy.Length; j++)
+ if (energy[j] > max) { max = energy[j]; best = j; }
+ if (best < 0) break;
+ int mid = best / Keys, f = best % Keys;
+ energy[best] = 0;
+
+ int i = mid + 1, k = 0;
+ while (i < n - 1 && k < EnergyTol)
+ {
+ k = energy[i * Keys + f] < FrameThresh ? k + 1 : 0;
+ Clear(energy, i, f);
+ i++;
+ }
+ int end = i - 1 - k;
+
+ i = mid - 1; k = 0;
+ while (i > 0 && k < EnergyTol)
+ {
+ k = energy[i * Keys + f] < FrameThresh ? k + 1 : 0;
+ Clear(energy, i, f);
+ i--;
+ }
+ int start = i + 1 + k;
+
+ if (end - start <= MinNoteLen) continue;
+ events.Add((start, end, f + MidiOffset, Mean(frames, start, end, f)));
+ }
+ return events;
+ }
+
+ private static void Clear(float[] energy, int t, int f)
+ {
+ energy[t * Keys + f] = 0;
+ if (f < Keys - 1) energy[t * Keys + f + 1] = 0;
+ if (f > 0) energy[t * Keys + f - 1] = 0;
+ }
+
+ private static float Mean(float[] frames, int start, int end, int f)
+ {
+ double s = 0;
+ for (int t = start; t < end; t++) s += frames[t * Keys + f];
+ return end > start ? (float)(s / (end - start)) : 0f;
+ }
+
+ // get_infered_onsets: onsets also where frame activations jump (the smaller of the 1- and
+ // 2-frame rises), rescaled to the predicted onsets' peak.
+ private static float[] InferOnsets(float[] onsets, float[] frames, int n)
+ {
+ const int nDiff = 2;
+ var diff = new float[onsets.Length];
+ float maxDiff = 0, maxOnset = 0;
+ for (int t = 0; t < n; t++)
+ for (int f = 0; f < Keys; f++)
+ {
+ int j = t * Keys + f;
+ maxOnset = Math.Max(maxOnset, onsets[j]);
+ if (t < nDiff) continue;
+ float d = float.MaxValue;
+ for (int m = 1; m <= nDiff; m++) d = Math.Min(d, frames[j] - frames[(t - m) * Keys + f]);
+ d = Math.Max(0, d);
+ diff[j] = d;
+ maxDiff = Math.Max(maxDiff, d);
+ }
+ var outp = new float[onsets.Length];
+ float scale = maxDiff > 0 ? maxOnset / maxDiff : 0;
+ for (int j = 0; j < outp.Length; j++) outp[j] = Math.Max(onsets[j], diff[j] * scale);
+ return outp;
+ }
+}
diff --git a/src/managed/Nota.Infrastructure/Ai/Resampler.cs b/src/managed/Nota.Infrastructure/Ai/Resampler.cs
new file mode 100644
index 00000000..ddfb7cec
--- /dev/null
+++ b/src/managed/Nota.Infrastructure/Ai/Resampler.cs
@@ -0,0 +1,81 @@
+// SPDX-License-Identifier: AGPL-3.0-only
+// Copyright (c) 2026 Egor Khindikaynen (Nota). See LICENSES/ for license terms.
+//
+// Offline sample-rate conversion for the AI models (htdemucs wants 44.1 kHz, basic-pitch
+// 22.05 kHz): a Kaiser-windowed sinc, 32 zero crossings each side, its cutoff lowered to the
+// target's Nyquist when downsampling. Not real-time — it runs once over a clip.
+
+namespace Nota.Infrastructure;
+
+internal static class Resampler
+{
+ private const int Zeros = 32;
+ private const double Beta = 8.6; // ~ -90 dB stop band
+ private const int TableRes = 512; // kernel samples per zero crossing
+ private static readonly double[] Kernel = MakeKernel(); // Kaiser × sinc over [0, Zeros)
+
+ /// <summary><paramref name="src"/> (one channel at <paramref name="fromRate"/>) at
+ /// <paramref name="toRate"/>. Returns the input itself when the rates match.</summary>
+ public static float[] Convert(float[] src, double fromRate, double toRate, CancellationToken ct = default)
+ {
+ if (Math.Abs(fromRate - toRate) < 1e-6 || src.Length == 0) return src;
+ double ratio = toRate / fromRate;
+ double cutoff = Math.Min(1.0, ratio) * 0.97; // a little below Nyquist, inside the transition band
+ int half = (int)Math.Ceiling(Zeros / cutoff);
+ int outLen = (int)Math.Round(src.Length * ratio);
+ var dst = new float[outLen];
+ for (int j = 0; j < outLen; j++)
+ {
+ if ((j & 0xFFFF) == 0) ct.ThrowIfCancellationRequested();
+ double pos = j / ratio;
+ int centre = (int)Math.Floor(pos);
+ double acc = 0, wsum = 0;
+ for (int k = centre - half + 1; k <= centre + half; k++)
+ {
+ double d = (pos - k) * cutoff; // distance in zero crossings of the filter
+ double w = Tap(Math.Abs(d));
+ if (w == 0) continue;
+ wsum += w;
+ if (k >= 0 && k < src.Length) acc += src[k] * w;
+ }
+ dst[j] = wsum != 0 ? (float)(acc / wsum) : 0f; // unity DC gain at every phase
+ }
+ return dst;
+ }
+
+ private static double Tap(double ad)
+ {
+ double x = ad * TableRes;
+ int i = (int)x;
+ if (i >= Kernel.Length - 1) return 0;
+ double f = x - i;
+ return Kernel[i] + (Kernel[i + 1] - Kernel[i]) * f;
+ }
+
+ private static double[] MakeKernel()
+ {
+ int n = Zeros * TableRes + 2;
+ var t = new double[n];
+ double i0b = BesselI0(Beta);
+ for (int i = 0; i < n; i++)
+ {
+ double d = (double)i / TableRes;
+ double r = Math.Min(1.0, d / Zeros);
+ double sinc = i == 0 ? 1.0 : Math.Sin(Math.PI * d) / (Math.PI * d);
+ t[i] = BesselI0(Beta * Math.Sqrt(1 - r * r)) / i0b * sinc;
+ }
+ return t;
+ }
+
+ private static double BesselI0(double x)
+ {
+ double sum = 1, term = 1, q = x * x / 4;
+ for (int k = 1; k < 50; k++)
+ {
+ term *= q / (k * k);
+ sum += term;
+ if (term < sum * 1e-12) break;
+ }
+ return sum;
+ }
+}
diff --git a/src/managed/Nota.Infrastructure/Ai/StemSeparator.cs b/src/managed/Nota.Infrastructure/Ai/StemSeparator.cs
new file mode 100644
index 00000000..951688a4
--- /dev/null
+++ b/src/managed/Nota.Infrastructure/Ai/StemSeparator.cs
@@ -0,0 +1,137 @@
+// SPDX-License-Identifier: AGPL-3.0-only
+// Copyright (c) 2026 Egor Khindikaynen (Nota). See LICENSES/ for license terms.
+//
+// Stem separation with htdemucs (Demucs v4, Meta, MIT). The ONNX file is the network alone
+// (nota-daw/nota-models: export_htdemucs.py); everything around it is here, as Demucs does it:
+// the whole signal is normalised by its mono mean / std (demucs.separate), cut into 7.8 s
+// segments overlapping by a quarter (demucs.apply.apply_model, one model, no random shifts),
+// each segment's spectrogram goes in beside the waveform, and the two branches that come out —
+// a complex spectrogram per stem (back through the iSTFT) and a waveform per stem — are summed.
+// Overlapping segments are cross-faded with Demucs's triangular weights.
+
+using Microsoft.ML.OnnxRuntime;
+
+namespace Nota.Infrastructure;
+
+public sealed class StemSeparator : IDisposable
+{
+ /// <summary>The model's stems, in output order.</summary>
+ public static readonly string[] Stems = ["Drums", "Bass", "Other", "Vocals"];
+ public const int SampleRate = 44100;
+ private const int Segment = 343980; // htdemucs's training length: 39/5 s at 44.1 kHz
+ private const double Overlap = 0.25;
+
+ private readonly InferenceSession _session;
+
+ /// <param name="runtimePath">The ONNX Runtime native library (ModelStore).</param>
+ /// <param name="modelPath">htdemucs.onnx.</param>
+ public StemSeparator(string runtimePath, string modelPath)
+ {
+ OnnxRuntimeLoader.Use(runtimePath);
+ _session = OnnxRuntimeLoader.Session(modelPath);
+ }
+
+ public void Dispose() => _session.Dispose();
+
+ /// <summary>Separates a stereo signal at <see cref="SampleRate"/>. Returns
+ /// [stem][channel][sample], same length as the input, stems in <see cref="Stems"/> order.</summary>
+ public float[][][] Separate(float[] left, float[] right, IProgress<double>? progress = null, CancellationToken ct = default)
+ {
+ int len = left.Length;
+ var outp = new float[Stems.Length][][];
+ for (int s = 0; s < Stems.Length; s++) outp[s] = [new float[len], new float[len]];
+ if (len == 0) return outp;
+
+ // demucs.separate: normalise by the mono mix's mean and (unbiased) std.
+ double mean = 0;
+ for (int i = 0; i < len; i++) mean += (left[i] + right[i]) * 0.5;
+ mean /= len;
+ double var = 0;
+ for (int i = 0; i < len; i++) { double d = (left[i] + right[i]) * 0.5 - mean; var += d * d; }
+ double std = Math.Sqrt(var / Math.Max(1, len - 1));
+ if (std < 1e-8) return outp; // silence separates into silence
+ float fm = (float)mean, inv = (float)(1 / std);
+
+ int stride = (int)((1 - Overlap) * Segment);
+ var weight = new float[Segment];
+ int halfSeg = Segment / 2;
+ for (int i = 0; i < Segment; i++)
+ weight[i] = (i < halfSeg ? i + 1 : Segment - i) / (float)halfSeg;
+ var sumWeight = new float[len];
+
+ var mix = new float[2 * Segment];
+ int frames = DemucsSpec.Frames(Segment);
+ int plane = DemucsSpec.Bins * frames;
+ var mag = new float[4 * plane];
+ var re = new float[plane];
+ var im = new float[plane];
+ var chunk = new float[Stems.Length * 2 * Segment];
+ int segments = (len + stride - 1) / stride;
+ int done = 0;
+ progress?.Report(0);
+ for (int offset = 0; offset < len; offset += stride)
+ {
+ ct.ThrowIfCancellationRequested();
+ // TensorChunk.padded: the chunk centred in a full segment, real neighbouring audio
+ // where there is some, zeros past either end.
+ int chunkLen = Math.Min(Segment, len - offset);
+ int delta = Segment - chunkLen;
+ int start = offset - delta / 2;
+ for (int i = 0; i < Segment; i++)
+ {
+ int k = start + i;
+ bool inside = k >= 0 && k < len;
+ mix[i] = inside ? (left[k] - fm) * inv : 0f;
+ mix[Segment + i] = inside ? (right[k] - fm) * inv : 0f;
+ }
+ for (int c = 0; c < 2; c++)
+ {
+ DemucsSpec.Forward(mix.AsSpan(c * Segment, Segment), re, im);
+ Array.Copy(re, 0, mag, (2 * c) * plane, plane); // "cac": real and imaginary
+ Array.Copy(im, 0, mag, (2 * c + 1) * plane, plane); // parts as channels
+ }
+
+ RunSegment(mix, mag, frames, plane, chunk);
+
+ // center_trim back to the chunk, then the weighted overlap-add.
+ int trim = delta / 2;
+ for (int s = 0; s < Stems.Length; s++)
+ for (int c = 0; c < 2; c++)
+ {
+ var dst = outp[s][c];
+ int src = (s * 2 + c) * Segment + trim;
+ for (int i = 0; i < chunkLen; i++) dst[offset + i] += weight[i] * chunk[src + i];
+ }
+ for (int i = 0; i < chunkLen; i++) sumWeight[offset + i] += weight[i];
+ progress?.Report(++done / (double)segments);
+ }
+
+ float fstd = (float)std;
+ for (int s = 0; s < Stems.Length; s++)
+ for (int c = 0; c < 2; c++)
+ {
+ var d = outp[s][c];
+ for (int i = 0; i < len; i++) d[i] = d[i] / sumWeight[i] * fstd + fm;
+ }
+ return outp;
+ }
+
+ // One segment through the network: chunk[stem][channel][sample] = iSTFT(spec branch) + wave branch.
+ private void RunSegment(float[] mix, float[] mag, int frames, int plane, float[] chunk)
+ {
+ using var mixValue = OrtValue.CreateTensorValueFromMemory(mix, [1, 2, Segment]);
+ using var magValue = OrtValue.CreateTensorValueFromMemory(mag, [1, 4, DemucsSpec.Bins, frames]);
+ using var run = new RunOptions();
+ using var results = _session.Run(run, ["mix", "mag"], [mixValue, magValue], ["spec", "wave"]);
+ var spec = results[0].GetTensorDataAsSpan<float>().ToArray(); // [1, stems, 4, bins, frames]
+ var wave = results[1].GetTensorDataAsSpan<float>().ToArray(); // [1, stems, 2, Segment]
+ Parallel.For(0, Stems.Length * 2, sc =>
+ {
+ int reAt = (2 * sc) * plane, imAt = reAt + plane; // stem s, channel c: planes 2c, 2c+1
+ int at = sc * Segment;
+ var dst = chunk.AsSpan(at, Segment);
+ DemucsSpec.Inverse(spec, spec, reAt, imAt, Segment, dst);
+ for (int i = 0; i < Segment; i++) dst[i] += wave[at + i];
+ });
+ }
+}
diff --git a/src/managed/Nota.Infrastructure/Analysis/AudioFft.cs b/src/managed/Nota.Infrastructure/Analysis/AudioFft.cs
index 3d60a28f..cd0af6c9 100644
--- a/src/managed/Nota.Infrastructure/Analysis/AudioFft.cs
+++ b/src/managed/Nota.Infrastructure/Analysis/AudioFft.cs
@@ -2,7 +2,8 @@
// Copyright (c) 2026 Egor Khindikaynen (Nota). See LICENSES/ for license terms.
//
// Minimal in-house radix-2 FFT (iterative Cooley–Tukey) for offline spectral
-// analysis (Convert Harmony). Pure, permissive — no external DSP dependency.
+// analysis (Convert Harmony, the stem separator's STFT). Pure, permissive — no external DSP
+// dependency.
using System;
@@ -46,4 +47,14 @@ public static class AudioFft
}
}
}
+
+ /// <summary>In-place inverse FFT, scaled by 1/n (so Inverse(Forward(x)) == x).</summary>
+ public static void Inverse(double[] re, double[] im)
+ {
+ int n = re.Length;
+ for (int i = 0; i < n; i++) im[i] = -im[i];
+ Forward(re, im);
+ double s = 1.0 / n;
+ for (int i = 0; i < n; i++) { re[i] *= s; im[i] = -im[i] * s; }
+ }
}
diff --git a/src/managed/Nota.Infrastructure/Nota.Infrastructure.csproj b/src/managed/Nota.Infrastructure/Nota.Infrastructure.csproj
index b1b37506..ca9c7811 100644
--- a/src/managed/Nota.Infrastructure/Nota.Infrastructure.csproj
+++ b/src/managed/Nota.Infrastructure/Nota.Infrastructure.csproj
@@ -23,4 +23,16 @@
<Using Include="Nota.Application" />
</ItemGroup>
+ <!-- The smoke test checks the AI models' DSP ports (DemucsSpec, Resampler, basic-pitch
+ decoding) against golden output from the Python originals. -->
+ <ItemGroup>
+ <InternalsVisibleTo Include="Nota.SmokeTest" />
+ </ItemGroup>
+
+ <!-- ONNX Runtime's managed half only: the native library is downloaded with the first AI
+ model (ModelStore / OnnxRuntimeLoader), so the installer doesn't carry it. -->
+ <ItemGroup>
+ <PackageReference Include="Microsoft.ML.OnnxRuntime.Managed" Version="1.23.2" />
+ </ItemGroup>
+
</Project>
diff --git a/tests/Nota.SmokeTest/AiGoldenTests.cs b/tests/Nota.SmokeTest/AiGoldenTests.cs
new file mode 100644
index 00000000..f31c0ae7
--- /dev/null
+++ b/tests/Nota.SmokeTest/AiGoldenTests.cs
@@ -0,0 +1,114 @@
+// SPDX-License-Identifier: AGPL-3.0-only
+// Copyright (c) 2026 Egor Khindikaynen (Nota). See LICENSES/ for license terms.
+//
+// The AI models' C# ports against golden output from the Python originals: htdemucs's STFT /
+// iSTFT and the whole separation (demucs.apply.apply_model), basic-pitch's inference and note
+// decoding. Needs the ONNX Runtime library, the models and fixtures written by
+// nota-models/scripts/fixtures.py, so it is opt-in:
+// dotnet run --project tests/Nota.SmokeTest -- --ai-golden <libonnxruntime> <dir>
+// where <dir> holds htdemucs.onnx, nmp.onnx and fx/ (raw little-endian float32 files).
+
+using System.Text.Json;
+using Nota.Infrastructure;
+
+namespace Nota.SmokeTest;
+
+internal static class AiGoldenTests
+{
+ public static IEnumerable<(bool Ok, string Label)> Run(string runtime, string dir)
+ {
+ string fx = Path.Combine(dir, "fx");
+ var mix = Floats(Path.Combine(fx, "mix44.f32"));
+ int n = mix.Length / 2;
+ var left = mix[..n];
+ var right = mix[n..];
+
+ // ---- STFT ----------------------------------------------------------------------------
+ const int seg = 343980;
+ int frames = DemucsSpec.Frames(seg);
+ int plane = DemucsSpec.Bins * frames;
+ var re = new float[plane];
+ var im = new float[plane];
+ var refMag = Floats(Path.Combine(fx, "mag.f32"));
+ double magErr = 0, magPeak = 0;
+ for (int c = 0; c < 2; c++)
+ {
+ DemucsSpec.Forward((c == 0 ? left : right).AsSpan(0, seg), re, im);
+ for (int j = 0; j < plane; j++)
+ {
+ magErr = Math.Max(magErr, Math.Abs(re[j] - refMag[(2 * c) * plane + j]));
+ magErr = Math.Max(magErr, Math.Abs(im[j] - refMag[(2 * c + 1) * plane + j]));
+ magPeak = Math.Max(magPeak, Math.Abs(refMag[(2 * c) * plane + j]));
+ }
+ }
+ yield return (magErr < 1e-4 * Math.Max(1, magPeak), $"htdemucs spectrogram matches _spec (max err {magErr:g3}, peak {magPeak:g3})");
+
+ var refIspec = Floats(Path.Combine(fx, "ispec.f32"));
+ var back = new float[seg];
+ double ispecErr = 0;
+ for (int c = 0; c < 2; c++)
+ {
+ DemucsSpec.Forward((c == 0 ? left : right).AsSpan(0, seg), re, im);
+ DemucsSpec.Inverse(re, im, 0, 0, seg, back);
+ for (int i = 0; i < seg; i++) ispecErr = Math.Max(ispecErr, Math.Abs(back[i] - refIspec[c * seg + i]));
+ }
+ yield return (ispecErr < 1e-4, $"…and the iSTFT matches _ispec (max err {ispecErr:g3})");
+
+ // ---- separation ----------------------------------------------------------------------
+ var refStems = Floats(Path.Combine(fx, "stems.f32"));
+ var sw = System.Diagnostics.Stopwatch.StartNew();
+ float[][][] stems;
+ using (var sep = new StemSeparator(runtime, Path.Combine(dir, "htdemucs.onnx")))
+ stems = sep.Separate(left, right);
+ sw.Stop();
+ for (int s = 0; s < StemSeparator.Stems.Length; s++)
+ {
+ double err = 0, sig = 0;
+ for (int c = 0; c < 2; c++)
+ for (int i = 0; i < n; i++)
+ {
+ double r = refStems[(s * 2 + c) * n + i], d = stems[s][c][i] - r;
+ err += d * d; sig += r * r;
+ }
+ double snr = 10 * Math.Log10(sig / Math.Max(1e-20, err));
+ yield return (snr > 50, $"{StemSeparator.Stems[s]} stem matches apply_model ({snr:0} dB SNR)");
+ }
+ Console.WriteLine($" separation of {n / 44100.0:0.0} s took {sw.Elapsed.TotalSeconds:0.0} s");
+
+ // ---- basic-pitch ---------------------------------------------------------------------
+ var mono = Floats(Path.Combine(fx, "mono22.f32"));
+ var refNote = Floats(Path.Combine(fx, "bp_note.f32"));
+ var refOnset = Floats(Path.Combine(fx, "bp_onset.f32"));
+ using var doc = JsonDocument.Parse(File.ReadAllText(Path.Combine(fx, "bp_notes.json")));
+ int bpFrames = doc.RootElement.GetProperty("frames").GetInt32();
+ var refNotes = doc.RootElement.GetProperty("notes").EnumerateArray()
+ .Select(e => (Start: e[0].GetDouble(), End: e[1].GetDouble(), Pitch: e[2].GetInt32())).ToList();
+
+ // Decoding alone, fed the reference activations: must reproduce the notes exactly.
+ var decoded = PitchTranscriber.Decode(refNote, refOnset, bpFrames);
+ yield return (decoded.Count == refNotes.Count, $"basic-pitch decoding finds the reference's {refNotes.Count} notes (got {decoded.Count})");
+
+ List<TranscribedNote> notes;
+ using (var tr = new PitchTranscriber(runtime, Path.Combine(dir, "nmp.onnx")))
+ notes = tr.Transcribe(mono);
+ int matched = refNotes.Count(r => notes.Any(nn => nn.Pitch == r.Pitch && Math.Abs(nn.Start - r.Start) < 0.012 && Math.Abs(nn.End - r.End) < 0.012));
+ yield return (matched >= refNotes.Count * 0.98 && Math.Abs(notes.Count - refNotes.Count) <= Math.Max(2, refNotes.Count / 50),
+ $"basic-pitch end to end: {matched}/{refNotes.Count} reference notes matched to a frame ({notes.Count} found)");
+
+ // ---- resampler -----------------------------------------------------------------------
+ var tone = new float[48000];
+ for (int i = 0; i < tone.Length; i++) tone[i] = (float)Math.Sin(2 * Math.PI * 1000 * i / 48000.0);
+ var down = Resampler.Convert(tone, 48000, 44100);
+ double rErr = 0;
+ for (int i = 200; i < down.Length - 200; i++) rErr = Math.Max(rErr, Math.Abs(down[i] - Math.Sin(2 * Math.PI * 1000 * i / 44100.0)));
+ yield return (down.Length == 44100 && rErr < 1e-3, $"resampler 48 → 44.1 kHz keeps a 1 kHz tone (max err {rErr:g3})");
+ }
+
+ private static float[] Floats(string path)
+ {
+ var bytes = File.ReadAllBytes(path);
+ var f = new float[bytes.Length / 4];
+ Buffer.BlockCopy(bytes, 0, f, 0, f.Length * 4);
+ return f;
+ }
+}
diff --git a/tests/Nota.SmokeTest/Program.cs b/tests/Nota.SmokeTest/Program.cs
index beb8c1d1..585ac452 100644
--- a/tests/Nota.SmokeTest/Program.cs
+++ b/tests/Nota.SmokeTest/Program.cs
@@ -116,6 +116,14 @@ if (args.Length >= 1 && args[0] == "--samples")
Console.WriteLine(failures == 0 ? "SAMPLES PASSED" : $"SAMPLES FAILED ({failures})");
return failures == 0 ? 0 : 1;
}
+// AI models' DSP ports against the Python originals: `--ai-golden <libonnxruntime> <dir>`.
+if (args.Length >= 3 && args[0] == "--ai-golden")
+{
+ Console.WriteLine("-- ai models: golden output --");
+ foreach (var (ok, label) in AiGoldenTests.Run(args[1], args[2])) Check(ok, label);
+ Console.WriteLine(failures == 0 ? "AI GOLDEN PASSED" : $"AI GOLDEN FAILED ({failures})");
+ return failures == 0 ? 0 : 1;
+}
if (args.Length >= 3 && args[0] == "--store-catalog")
{
Console.WriteLine("-- get plug-ins: catalog identifiers --");