Nota — a free DAW for macOS, Windows and Linux. No subscription, no account — just open it and make music.
C# audioaudio-unitavaloniacpp20cross-platformdawdigital-audio-workstationdspjucelinux-audiomcp-servermidimusicmusic-productionpiano-rollplugin-hostsequencersynthesizervstvst3
(feat) AI models: htdemucs stem separation and basic-pitch transcription ported to C# on ONNX Runtime
Egor Khindikaynend54da77eab19
diff --git a/src/managed/Nota.Infrastructure/Ai/DemucsSpec.cs b/src/managed/Nota.Infrastructure/Ai/DemucsSpec.csnew file mode 100644index 00000000..fcf77ddb--- /dev/null+++ b/src/managed/Nota.Infrastructure/Ai/DemucsSpec.cs@@ -0,0 +1,106 @@+// SPDX-License-Identifier: AGPL-3.0-only+// Copyright (c) 2026 Egor Khindikaynen (Nota). See LICENSES/ for license terms.+//+// The STFT / iSTFT that htdemucs runs around its network. The ONNX export leaves them out+// (ONNX has no complex STFT), so they are reproduced here exactly as HTDemucs._spec / _ispec+// and demucs/spec.py compute them: a 4096-point periodic Hann window, hop 1024, torch's+// "normalized" scaling (1/√N each way), centre padding by reflection, plus htdemucs's own+// reflect pad of 1.5 hops so the frame count is exactly ceil(length / hop). Spectra are+// bin-major: [bin * frames + frame], 2048 bins (the Nyquist bin is dropped, as htdemucs does).++namespace Nota.Infrastructure;++internal static class DemucsSpec+{+ public const int NFft = 4096, Hop = 1024, Bins = NFft / 2;+ private const int Pad = Hop / 2 * 3; // htdemucs's reflect pad around the signal+ private static readonly double Norm = 1.0 / Math.Sqrt(NFft);+ private static readonly double[] Window = MakeWindow();++ private static double[] MakeWindow()+ {+ var w = new double[NFft];+ for (int n = 0; n < NFft; n++) w[n] = 0.5 - 0.5 * Math.Cos(2 * Math.PI * n / NFft); // periodic, as torch.hann_window+ return w;+ }++ /// <summary>Frames the spectrum of a <paramref name="length"/>-sample signal has.</summary>+ public static int Frames(int length) => (length + Hop - 1) / Hop;++ /// <summary>Spectrum of one channel into <paramref name="re"/> / <paramref name="im"/>+ /// (each <see cref="Bins"/> × <see cref="Frames"/>).</summary>+ public static void Forward(ReadOnlySpan<float> x, float[] re, float[] im)+ {+ int len = x.Length, le = Frames(len);+ int p1 = le * Hop + 2 * Pad; // after htdemucs's pad (pad, pad + le*hop - len)+ var fr = new double[NFft];+ var fi = new double[NFft];+ for (int t = 0; t < le; t++)+ {+ // Kept frame t is STFT frame t + 2; STFT frames start at hop multiples of the+ // centre-padded signal (NFft/2 of reflection either side of the p1-long one).+ int start = (t + 2) * Hop - NFft / 2;+ for (int n = 0; n < NFft; n++)+ {+ int i = Reflect(start + n, p1) - Pad;+ fr[n] = x[Reflect(i, len)] * Window[n];+ fi[n] = 0;+ }+ AudioFft.Forward(fr, fi);+ for (int k = 0; k < Bins; k++)+ {+ re[k * le + t] = (float)(fr[k] * Norm);+ im[k * le + t] = (float)(fi[k] * Norm);+ }+ }+ }++ /// <summary>Signal of <paramref name="length"/> samples back from a spectrum of+ /// <see cref="Frames"/>(<paramref name="length"/>) frames (its planes start at the given+ /// offsets), into <paramref name="dst"/>.</summary>+ public static void Inverse(float[] re, float[] im, int reOffset, int imOffset, int length, Span<float> dst)+ {+ int le = Frames(length);+ int frames = le + 4; // two silent frames either side, as _ispec pads+ int olaLen = NFft + Hop * (frames - 1);+ var ola = new double[olaLen];+ var env = new double[olaLen];+ var fr = new double[NFft];+ var fi = new double[NFft];+ double scale = Math.Sqrt(NFft); // undoes the forward 1/√N ("normalized")+ for (int f = 0; f < frames; f++)+ {+ int at = f * Hop;+ for (int n = 0; n < NFft; n++) env[at + n] += Window[n] * Window[n];+ int t = f - 2;+ if (t < 0 || t >= le) continue; // a silent frame adds only to the envelope+ fr[0] = re[reOffset + t]; fi[0] = 0; // irfft ignores the DC bin's imaginary part+ for (int k = 1; k < Bins; k++)+ {+ double a = re[reOffset + k * le + t], b = im[imOffset + k * le + t];+ fr[k] = a; fi[k] = b;+ fr[NFft - k] = a; fi[NFft - k] = -b;+ }+ fr[Bins] = 0; fi[Bins] = 0; // the dropped Nyquist bin+ AudioFft.Inverse(fr, fi);+ for (int n = 0; n < NFft; n++) ola[at + n] += fr[n] * scale * Window[n];+ }+ // istft drops NFft/2 of centre padding; _ispec then drops its own 1.5-hop pad.+ int off = NFft / 2 + Pad;+ for (int i = 0; i < length; i++)+ {+ double e = env[off + i];+ dst[i] = e > 1e-11 ? (float)(ola[off + i] / e) : 0f;+ }+ }++ // numpy/torch "reflect" (edge sample not repeated) into [0, n).+ private static int Reflect(int i, int n)+ {+ if (n == 1) return 0;+ int period = 2 * (n - 1);+ i %= period;+ if (i < 0) i += period;+ return i < n ? i : period - i;+ }+}diff --git a/src/managed/Nota.Infrastructure/Ai/OnnxRuntimeLoader.cs b/src/managed/Nota.Infrastructure/Ai/OnnxRuntimeLoader.csnew file mode 100644index 00000000..76d4278c--- /dev/null+++ b/src/managed/Nota.Infrastructure/Ai/OnnxRuntimeLoader.cs@@ -0,0 +1,56 @@+// SPDX-License-Identifier: AGPL-3.0-only+// Copyright (c) 2026 Egor Khindikaynen (Nota). See LICENSES/ for license terms.+//+// Nota ships only ONNX Runtime's managed half (Microsoft.ML.OnnxRuntime.Managed); the native+// library is downloaded with the first AI model (ModelStore) to keep the installer small. The+// managed assembly P/Invokes "onnxruntime", so a resolver points that name at the downloaded+// file. It must be in place before the first InferenceSession, and once the library is loaded+// the process keeps it.++using System.Reflection;+using System.Runtime.InteropServices;+using Microsoft.ML.OnnxRuntime;++namespace Nota.Infrastructure;++internal static class OnnxRuntimeLoader+{+ private static readonly object Gate = new();+ private static string? _path;+ private static IntPtr _handle;++ /// <summary>Points the ORT assembly at the native library at <paramref name="libraryPath"/>.+ /// Throws <see cref="StoreException"/> when it can't be loaded.</summary>+ public static void Use(string libraryPath)+ {+ lock (Gate)+ {+ if (_handle != IntPtr.Zero) return; // already loaded; a process can't swap it+ if (!NativeLibrary.TryLoad(libraryPath, out var h))+ throw new StoreException("Nota couldn't load the AI runtime. Remove it in Settings → Downloads → AI Models and install it again.");+ _handle = h;+ if (_path is null)+ NativeLibrary.SetDllImportResolver(typeof(InferenceSession).Assembly, Resolve);+ _path = libraryPath;+ }+ }++ private static IntPtr Resolve(string name, Assembly assembly, DllImportSearchPath? search)+ => name == "onnxruntime" ? _handle : IntPtr.Zero;++ /// <summary>A CPU session for the model at <paramref name="modelPath"/>. ORT's own thread+ /// count (one per performance core) beats every core: efficiency cores slow the whole step.</summary>+ public static InferenceSession Session(string modelPath)+ {+ var opts = new SessionOptions+ {+ GraphOptimizationLevel = GraphOptimizationLevel.ORT_ENABLE_ALL,+ InterOpNumThreads = 1,+ };+ try { return new InferenceSession(modelPath, opts); }+ catch (OnnxRuntimeException e)+ {+ throw new StoreException("The AI model file is damaged. Remove it in Settings → Downloads → AI Models and install it again.", e);+ }+ }+}diff --git a/src/managed/Nota.Infrastructure/Ai/PitchTranscriber.cs b/src/managed/Nota.Infrastructure/Ai/PitchTranscriber.csnew file mode 100644index 00000000..e9d76cf3--- /dev/null+++ b/src/managed/Nota.Infrastructure/Ai/PitchTranscriber.cs@@ -0,0 +1,238 @@+// SPDX-License-Identifier: AGPL-3.0-only+// Copyright (c) 2026 Egor Khindikaynen (Nota). See LICENSES/ for license terms.+//+// Polyphonic audio → notes with Spotify's basic-pitch (Apache-2.0). The model takes 2 s+// windows of 22.05 kHz mono and returns, per 256-sample frame, note and onset activations for+// the 88 piano keys. Windowing, unwrapping and note decoding are ported from+// basic_pitch.inference / note_creation (output_to_notes_polyphonic with inferred onsets and the+// "melodia trick"), with the library's default thresholds. Pitch bends are left out.++using Microsoft.ML.OnnxRuntime;++namespace Nota.Infrastructure;++/// <summary>A transcribed note: times in seconds from the start of the audio, MIDI pitch, and+/// the model's mean frame activation (0..1, basic-pitch's "amplitude").</summary>+public readonly record struct TranscribedNote(double Start, double End, int Pitch, float Amplitude)+{+ public int Velocity => Math.Clamp((int)Math.Round(127 * Amplitude), 1, 127);+}++public sealed class PitchTranscriber : IDisposable+{+ public const int SampleRate = 22050;+ private const int FftHop = 256;+ private const int WindowSamples = SampleRate * 2 - FftHop; // AUDIO_N_SAMPLES = 43844+ private const int FramesPerWindow = 172; // ANNOT_N_FRAMES+ private const int Keys = 88, MidiOffset = 21;+ private const int OverlapFrames = 30;+ private const int OverlapLen = OverlapFrames * FftHop;+ private const int HopSize = WindowSamples - OverlapLen;+ private const int Batch = 8;+ private const int Fps = SampleRate / FftHop; // ANNOTATIONS_FPS = 86++ private const float OnsetThresh = 0.5f, FrameThresh = 0.3f;+ private const int MinNoteLen = 11; // round(127.70 ms × 86.13 fps)+ private const int EnergyTol = 11;++ private readonly InferenceSession _session;+ private readonly string _input;++ public PitchTranscriber(string runtimePath, string modelPath)+ {+ OnnxRuntimeLoader.Use(runtimePath);+ _session = OnnxRuntimeLoader.Session(modelPath);+ _input = _session.InputMetadata.Keys.First();+ }++ public void Dispose() => _session.Dispose();++ /// <summary>Notes in <paramref name="mono"/> (at <see cref="SampleRate"/>), by start time.</summary>+ public List<TranscribedNote> Transcribe(float[] mono, IProgress<double>? progress = null, CancellationToken ct = default)+ {+ var (note, onset, n) = Infer(mono, progress, ct);+ var events = Decode(note, onset, n);+ var notes = new List<TranscribedNote>(events.Count);+ foreach (var (s, e, p, a) in events) notes.Add(new TranscribedNote(FrameTime(s), FrameTime(e), p, a));+ notes.Sort((a, b) => a.Start != b.Start ? a.Start.CompareTo(b.Start) : a.Pitch.CompareTo(b.Pitch));+ return notes;+ }++ /// <summary>One line out of a transcription: wherever notes overlap, the strongest one+ /// wins and the others are dropped (Convert Melody).</summary>+ public static List<TranscribedNote> Monophonic(IReadOnlyList<TranscribedNote> notes)+ {+ var kept = new List<TranscribedNote>();+ foreach (var n in notes.OrderByDescending(n => n.Amplitude * (n.End - n.Start)))+ if (!kept.Any(k => k.Start < n.End && n.Start < k.End)) kept.Add(n);+ kept.Sort((a, b) => a.Start.CompareTo(b.Start));+ return kept;+ }++ // ---- inference (inference.run_inference) -----------------------------------------------++ private (float[] Note, float[] Onset, int Frames) Infer(float[] mono, IProgress<double>? progress, CancellationToken ct)+ {+ int original = mono.Length;+ int padded = OverlapLen / 2 + original; // half an overlap of silence in front+ int windows = Math.Max(1, (padded + HopSize - 1) / HopSize);+ int keep = FramesPerWindow - OverlapFrames; // unwrap drops half the overlap at each end+ int total = (int)Math.Floor(original * ((double)Fps / SampleRate));+ var note = new float[Math.Max(0, total) * Keys];+ var onset = new float[note.Length];++ var input = new float[Batch * WindowSamples];+ for (int w0 = 0; w0 < windows; w0 += Batch)+ {+ ct.ThrowIfCancellationRequested();+ int b = Math.Min(Batch, windows - w0);+ Array.Clear(input);+ for (int w = 0; w < b; w++)+ {+ int from = (w0 + w) * HopSize - OverlapLen / 2; // in the unpadded signal+ for (int i = 0; i < WindowSamples; i++)+ {+ int k = from + i;+ if (k >= 0 && k < original) input[w * WindowSamples + i] = mono[k];+ }+ }+ using var x = OrtValue.CreateTensorValueFromMemory(input, [b, WindowSamples, 1]);+ using var run = new RunOptions();+ using var r = _session.Run(run, [_input], [x], ["StatefulPartitionedCall:1", "StatefulPartitionedCall:2"]);+ var nt = r[0].GetTensorDataAsSpan<float>();+ var on = r[1].GetTensorDataAsSpan<float>();+ for (int w = 0; w < b; w++)+ for (int f = 0; f < keep; f++)+ {+ int dst = (w0 + w) * keep + f;+ if (dst >= total) break;+ int src = (w * FramesPerWindow + OverlapFrames / 2 + f) * Keys;+ nt.Slice(src, Keys).CopyTo(note.AsSpan(dst * Keys, Keys));+ on.Slice(src, Keys).CopyTo(onset.AsSpan(dst * Keys, Keys));+ }+ progress?.Report(Math.Min(1, (w0 + b) / (double)windows));+ }+ return (note, onset, Math.Max(0, total));+ }++ // note_creation.model_frames_to_time: frame index → seconds, with basic-pitch's per-window+ // drift correction.+ private static double FrameTime(int frame)+ {+ double t = frame * (double)FftHop / SampleRate;+ double windowOffset = (double)FftHop / SampleRate * (FramesPerWindow - (double)WindowSamples / FftHop) + 0.0018;+ return t - windowOffset * Math.Floor((double)frame / FramesPerWindow);+ }++ // ---- decoding (note_creation.output_to_notes_polyphonic) --------------------------------++ internal static List<(int Start, int End, int Pitch, float Amplitude)> Decode(float[] frames, float[] onsetsIn, int n)+ {+ var events = new List<(int, int, int, float)>();+ if (n == 0) return events;+ var onsets = InferOnsets(onsetsIn, frames, n);++ // Onset peaks (strict local maxima in time) above the threshold, latest first.+ var peaks = new List<(int T, int F)>();+ for (int t = 1; t < n - 1; t++)+ for (int f = 0; f < Keys; f++)+ {+ float v = onsets[t * Keys + f];+ if (v >= OnsetThresh && v > onsets[(t - 1) * Keys + f] && v > onsets[(t + 1) * Keys + f]) peaks.Add((t, f));+ }+ peaks.Reverse();++ var energy = (float[])frames.Clone();+ foreach (var (start, f) in peaks)+ {+ if (start >= n - 1) continue;+ int i = start + 1, k = 0;+ while (i < n - 1 && k < EnergyTol)+ {+ k = energy[i * Keys + f] < FrameThresh ? k + 1 : 0;+ i++;+ }+ i -= k;+ if (i - start <= MinNoteLen) continue;+ for (int t = start; t < i; t++)+ {+ energy[t * Keys + f] = 0;+ if (f < Keys - 1) energy[t * Keys + f + 1] = 0;+ if (f > 0) energy[t * Keys + f - 1] = 0;+ }+ events.Add((start, i, f + MidiOffset, Mean(frames, start, i, f)));+ }++ // Melodia trick: follow the strongest remaining energy both ways into further notes.+ while (true)+ {+ int best = -1; float max = FrameThresh;+ for (int j = 0; j < energy.Length; j++)+ if (energy[j] > max) { max = energy[j]; best = j; }+ if (best < 0) break;+ int mid = best / Keys, f = best % Keys;+ energy[best] = 0;++ int i = mid + 1, k = 0;+ while (i < n - 1 && k < EnergyTol)+ {+ k = energy[i * Keys + f] < FrameThresh ? k + 1 : 0;+ Clear(energy, i, f);+ i++;+ }+ int end = i - 1 - k;++ i = mid - 1; k = 0;+ while (i > 0 && k < EnergyTol)+ {+ k = energy[i * Keys + f] < FrameThresh ? k + 1 : 0;+ Clear(energy, i, f);+ i--;+ }+ int start = i + 1 + k;++ if (end - start <= MinNoteLen) continue;+ events.Add((start, end, f + MidiOffset, Mean(frames, start, end, f)));+ }+ return events;+ }++ private static void Clear(float[] energy, int t, int f)+ {+ energy[t * Keys + f] = 0;+ if (f < Keys - 1) energy[t * Keys + f + 1] = 0;+ if (f > 0) energy[t * Keys + f - 1] = 0;+ }++ private static float Mean(float[] frames, int start, int end, int f)+ {+ double s = 0;+ for (int t = start; t < end; t++) s += frames[t * Keys + f];+ return end > start ? (float)(s / (end - start)) : 0f;+ }++ // get_infered_onsets: onsets also where frame activations jump (the smaller of the 1- and+ // 2-frame rises), rescaled to the predicted onsets' peak.+ private static float[] InferOnsets(float[] onsets, float[] frames, int n)+ {+ const int nDiff = 2;+ var diff = new float[onsets.Length];+ float maxDiff = 0, maxOnset = 0;+ for (int t = 0; t < n; t++)+ for (int f = 0; f < Keys; f++)+ {+ int j = t * Keys + f;+ maxOnset = Math.Max(maxOnset, onsets[j]);+ if (t < nDiff) continue;+ float d = float.MaxValue;+ for (int m = 1; m <= nDiff; m++) d = Math.Min(d, frames[j] - frames[(t - m) * Keys + f]);+ d = Math.Max(0, d);+ diff[j] = d;+ maxDiff = Math.Max(maxDiff, d);+ }+ var outp = new float[onsets.Length];+ float scale = maxDiff > 0 ? maxOnset / maxDiff : 0;+ for (int j = 0; j < outp.Length; j++) outp[j] = Math.Max(onsets[j], diff[j] * scale);+ return outp;+ }+}diff --git a/src/managed/Nota.Infrastructure/Ai/Resampler.cs b/src/managed/Nota.Infrastructure/Ai/Resampler.csnew file mode 100644index 00000000..ddfb7cec--- /dev/null+++ b/src/managed/Nota.Infrastructure/Ai/Resampler.cs@@ -0,0 +1,81 @@+// SPDX-License-Identifier: AGPL-3.0-only+// Copyright (c) 2026 Egor Khindikaynen (Nota). See LICENSES/ for license terms.+//+// Offline sample-rate conversion for the AI models (htdemucs wants 44.1 kHz, basic-pitch+// 22.05 kHz): a Kaiser-windowed sinc, 32 zero crossings each side, its cutoff lowered to the+// target's Nyquist when downsampling. Not real-time — it runs once over a clip.++namespace Nota.Infrastructure;++internal static class Resampler+{+ private const int Zeros = 32;+ private const double Beta = 8.6; // ~ -90 dB stop band+ private const int TableRes = 512; // kernel samples per zero crossing+ private static readonly double[] Kernel = MakeKernel(); // Kaiser × sinc over [0, Zeros)++ /// <summary><paramref name="src"/> (one channel at <paramref name="fromRate"/>) at+ /// <paramref name="toRate"/>. Returns the input itself when the rates match.</summary>+ public static float[] Convert(float[] src, double fromRate, double toRate, CancellationToken ct = default)+ {+ if (Math.Abs(fromRate - toRate) < 1e-6 || src.Length == 0) return src;+ double ratio = toRate / fromRate;+ double cutoff = Math.Min(1.0, ratio) * 0.97; // a little below Nyquist, inside the transition band+ int half = (int)Math.Ceiling(Zeros / cutoff);+ int outLen = (int)Math.Round(src.Length * ratio);+ var dst = new float[outLen];+ for (int j = 0; j < outLen; j++)+ {+ if ((j & 0xFFFF) == 0) ct.ThrowIfCancellationRequested();+ double pos = j / ratio;+ int centre = (int)Math.Floor(pos);+ double acc = 0, wsum = 0;+ for (int k = centre - half + 1; k <= centre + half; k++)+ {+ double d = (pos - k) * cutoff; // distance in zero crossings of the filter+ double w = Tap(Math.Abs(d));+ if (w == 0) continue;+ wsum += w;+ if (k >= 0 && k < src.Length) acc += src[k] * w;+ }+ dst[j] = wsum != 0 ? (float)(acc / wsum) : 0f; // unity DC gain at every phase+ }+ return dst;+ }++ private static double Tap(double ad)+ {+ double x = ad * TableRes;+ int i = (int)x;+ if (i >= Kernel.Length - 1) return 0;+ double f = x - i;+ return Kernel[i] + (Kernel[i + 1] - Kernel[i]) * f;+ }++ private static double[] MakeKernel()+ {+ int n = Zeros * TableRes + 2;+ var t = new double[n];+ double i0b = BesselI0(Beta);+ for (int i = 0; i < n; i++)+ {+ double d = (double)i / TableRes;+ double r = Math.Min(1.0, d / Zeros);+ double sinc = i == 0 ? 1.0 : Math.Sin(Math.PI * d) / (Math.PI * d);+ t[i] = BesselI0(Beta * Math.Sqrt(1 - r * r)) / i0b * sinc;+ }+ return t;+ }++ private static double BesselI0(double x)+ {+ double sum = 1, term = 1, q = x * x / 4;+ for (int k = 1; k < 50; k++)+ {+ term *= q / (k * k);+ sum += term;+ if (term < sum * 1e-12) break;+ }+ return sum;+ }+}diff --git a/src/managed/Nota.Infrastructure/Ai/StemSeparator.cs b/src/managed/Nota.Infrastructure/Ai/StemSeparator.csnew file mode 100644index 00000000..951688a4--- /dev/null+++ b/src/managed/Nota.Infrastructure/Ai/StemSeparator.cs@@ -0,0 +1,137 @@+// SPDX-License-Identifier: AGPL-3.0-only+// Copyright (c) 2026 Egor Khindikaynen (Nota). See LICENSES/ for license terms.+//+// Stem separation with htdemucs (Demucs v4, Meta, MIT). The ONNX file is the network alone+// (nota-daw/nota-models: export_htdemucs.py); everything around it is here, as Demucs does it:+// the whole signal is normalised by its mono mean / std (demucs.separate), cut into 7.8 s+// segments overlapping by a quarter (demucs.apply.apply_model, one model, no random shifts),+// each segment's spectrogram goes in beside the waveform, and the two branches that come out —+// a complex spectrogram per stem (back through the iSTFT) and a waveform per stem — are summed.+// Overlapping segments are cross-faded with Demucs's triangular weights.++using Microsoft.ML.OnnxRuntime;++namespace Nota.Infrastructure;++public sealed class StemSeparator : IDisposable+{+ /// <summary>The model's stems, in output order.</summary>+ public static readonly string[] Stems = ["Drums", "Bass", "Other", "Vocals"];+ public const int SampleRate = 44100;+ private const int Segment = 343980; // htdemucs's training length: 39/5 s at 44.1 kHz+ private const double Overlap = 0.25;++ private readonly InferenceSession _session;++ /// <param name="runtimePath">The ONNX Runtime native library (ModelStore).</param>+ /// <param name="modelPath">htdemucs.onnx.</param>+ public StemSeparator(string runtimePath, string modelPath)+ {+ OnnxRuntimeLoader.Use(runtimePath);+ _session = OnnxRuntimeLoader.Session(modelPath);+ }++ public void Dispose() => _session.Dispose();++ /// <summary>Separates a stereo signal at <see cref="SampleRate"/>. Returns+ /// [stem][channel][sample], same length as the input, stems in <see cref="Stems"/> order.</summary>+ public float[][][] Separate(float[] left, float[] right, IProgress<double>? progress = null, CancellationToken ct = default)+ {+ int len = left.Length;+ var outp = new float[Stems.Length][][];+ for (int s = 0; s < Stems.Length; s++) outp[s] = [new float[len], new float[len]];+ if (len == 0) return outp;++ // demucs.separate: normalise by the mono mix's mean and (unbiased) std.+ double mean = 0;+ for (int i = 0; i < len; i++) mean += (left[i] + right[i]) * 0.5;+ mean /= len;+ double var = 0;+ for (int i = 0; i < len; i++) { double d = (left[i] + right[i]) * 0.5 - mean; var += d * d; }+ double std = Math.Sqrt(var / Math.Max(1, len - 1));+ if (std < 1e-8) return outp; // silence separates into silence+ float fm = (float)mean, inv = (float)(1 / std);++ int stride = (int)((1 - Overlap) * Segment);+ var weight = new float[Segment];+ int halfSeg = Segment / 2;+ for (int i = 0; i < Segment; i++)+ weight[i] = (i < halfSeg ? i + 1 : Segment - i) / (float)halfSeg;+ var sumWeight = new float[len];++ var mix = new float[2 * Segment];+ int frames = DemucsSpec.Frames(Segment);+ int plane = DemucsSpec.Bins * frames;+ var mag = new float[4 * plane];+ var re = new float[plane];+ var im = new float[plane];+ var chunk = new float[Stems.Length * 2 * Segment];+ int segments = (len + stride - 1) / stride;+ int done = 0;+ progress?.Report(0);+ for (int offset = 0; offset < len; offset += stride)+ {+ ct.ThrowIfCancellationRequested();+ // TensorChunk.padded: the chunk centred in a full segment, real neighbouring audio+ // where there is some, zeros past either end.+ int chunkLen = Math.Min(Segment, len - offset);+ int delta = Segment - chunkLen;+ int start = offset - delta / 2;+ for (int i = 0; i < Segment; i++)+ {+ int k = start + i;+ bool inside = k >= 0 && k < len;+ mix[i] = inside ? (left[k] - fm) * inv : 0f;+ mix[Segment + i] = inside ? (right[k] - fm) * inv : 0f;+ }+ for (int c = 0; c < 2; c++)+ {+ DemucsSpec.Forward(mix.AsSpan(c * Segment, Segment), re, im);+ Array.Copy(re, 0, mag, (2 * c) * plane, plane); // "cac": real and imaginary+ Array.Copy(im, 0, mag, (2 * c + 1) * plane, plane); // parts as channels+ }++ RunSegment(mix, mag, frames, plane, chunk);++ // center_trim back to the chunk, then the weighted overlap-add.+ int trim = delta / 2;+ for (int s = 0; s < Stems.Length; s++)+ for (int c = 0; c < 2; c++)+ {+ var dst = outp[s][c];+ int src = (s * 2 + c) * Segment + trim;+ for (int i = 0; i < chunkLen; i++) dst[offset + i] += weight[i] * chunk[src + i];+ }+ for (int i = 0; i < chunkLen; i++) sumWeight[offset + i] += weight[i];+ progress?.Report(++done / (double)segments);+ }++ float fstd = (float)std;+ for (int s = 0; s < Stems.Length; s++)+ for (int c = 0; c < 2; c++)+ {+ var d = outp[s][c];+ for (int i = 0; i < len; i++) d[i] = d[i] / sumWeight[i] * fstd + fm;+ }+ return outp;+ }++ // One segment through the network: chunk[stem][channel][sample] = iSTFT(spec branch) + wave branch.+ private void RunSegment(float[] mix, float[] mag, int frames, int plane, float[] chunk)+ {+ using var mixValue = OrtValue.CreateTensorValueFromMemory(mix, [1, 2, Segment]);+ using var magValue = OrtValue.CreateTensorValueFromMemory(mag, [1, 4, DemucsSpec.Bins, frames]);+ using var run = new RunOptions();+ using var results = _session.Run(run, ["mix", "mag"], [mixValue, magValue], ["spec", "wave"]);+ var spec = results[0].GetTensorDataAsSpan<float>().ToArray(); // [1, stems, 4, bins, frames]+ var wave = results[1].GetTensorDataAsSpan<float>().ToArray(); // [1, stems, 2, Segment]+ Parallel.For(0, Stems.Length * 2, sc =>+ {+ int reAt = (2 * sc) * plane, imAt = reAt + plane; // stem s, channel c: planes 2c, 2c+1+ int at = sc * Segment;+ var dst = chunk.AsSpan(at, Segment);+ DemucsSpec.Inverse(spec, spec, reAt, imAt, Segment, dst);+ for (int i = 0; i < Segment; i++) dst[i] += wave[at + i];+ });+ }+}diff --git a/src/managed/Nota.Infrastructure/Analysis/AudioFft.cs b/src/managed/Nota.Infrastructure/Analysis/AudioFft.csindex 3d60a28f..cd0af6c9 100644--- a/src/managed/Nota.Infrastructure/Analysis/AudioFft.cs+++ b/src/managed/Nota.Infrastructure/Analysis/AudioFft.cs@@ -2,7 +2,8 @@// Copyright (c) 2026 Egor Khindikaynen (Nota). See LICENSES/ for license terms.//// Minimal in-house radix-2 FFT (iterative Cooley–Tukey) for offline spectral-// analysis (Convert Harmony). Pure, permissive — no external DSP dependency.+// analysis (Convert Harmony, the stem separator's STFT). Pure, permissive — no external DSP+// dependency.using System;@@ -46,4 +47,14 @@ public static class AudioFft}}}++ /// <summary>In-place inverse FFT, scaled by 1/n (so Inverse(Forward(x)) == x).</summary>+ public static void Inverse(double[] re, double[] im)+ {+ int n = re.Length;+ for (int i = 0; i < n; i++) im[i] = -im[i];+ Forward(re, im);+ double s = 1.0 / n;+ for (int i = 0; i < n; i++) { re[i] *= s; im[i] = -im[i] * s; }+ }}diff --git a/src/managed/Nota.Infrastructure/Nota.Infrastructure.csproj b/src/managed/Nota.Infrastructure/Nota.Infrastructure.csprojindex b1b37506..ca9c7811 100644--- a/src/managed/Nota.Infrastructure/Nota.Infrastructure.csproj+++ b/src/managed/Nota.Infrastructure/Nota.Infrastructure.csproj@@ -23,4 +23,16 @@<Using Include="Nota.Application" /></ItemGroup>+ <!-- The smoke test checks the AI models' DSP ports (DemucsSpec, Resampler, basic-pitch+ decoding) against golden output from the Python originals. -->+ <ItemGroup>+ <InternalsVisibleTo Include="Nota.SmokeTest" />+ </ItemGroup>++ <!-- ONNX Runtime's managed half only: the native library is downloaded with the first AI+ model (ModelStore / OnnxRuntimeLoader), so the installer doesn't carry it. -->+ <ItemGroup>+ <PackageReference Include="Microsoft.ML.OnnxRuntime.Managed" Version="1.23.2" />+ </ItemGroup>+</Project>diff --git a/tests/Nota.SmokeTest/AiGoldenTests.cs b/tests/Nota.SmokeTest/AiGoldenTests.csnew file mode 100644index 00000000..f31c0ae7--- /dev/null+++ b/tests/Nota.SmokeTest/AiGoldenTests.cs@@ -0,0 +1,114 @@+// SPDX-License-Identifier: AGPL-3.0-only+// Copyright (c) 2026 Egor Khindikaynen (Nota). See LICENSES/ for license terms.+//+// The AI models' C# ports against golden output from the Python originals: htdemucs's STFT /+// iSTFT and the whole separation (demucs.apply.apply_model), basic-pitch's inference and note+// decoding. Needs the ONNX Runtime library, the models and fixtures written by+// nota-models/scripts/fixtures.py, so it is opt-in:+// dotnet run --project tests/Nota.SmokeTest -- --ai-golden <libonnxruntime> <dir>+// where <dir> holds htdemucs.onnx, nmp.onnx and fx/ (raw little-endian float32 files).++using System.Text.Json;+using Nota.Infrastructure;++namespace Nota.SmokeTest;++internal static class AiGoldenTests+{+ public static IEnumerable<(bool Ok, string Label)> Run(string runtime, string dir)+ {+ string fx = Path.Combine(dir, "fx");+ var mix = Floats(Path.Combine(fx, "mix44.f32"));+ int n = mix.Length / 2;+ var left = mix[..n];+ var right = mix[n..];++ // ---- STFT ----------------------------------------------------------------------------+ const int seg = 343980;+ int frames = DemucsSpec.Frames(seg);+ int plane = DemucsSpec.Bins * frames;+ var re = new float[plane];+ var im = new float[plane];+ var refMag = Floats(Path.Combine(fx, "mag.f32"));+ double magErr = 0, magPeak = 0;+ for (int c = 0; c < 2; c++)+ {+ DemucsSpec.Forward((c == 0 ? left : right).AsSpan(0, seg), re, im);+ for (int j = 0; j < plane; j++)+ {+ magErr = Math.Max(magErr, Math.Abs(re[j] - refMag[(2 * c) * plane + j]));+ magErr = Math.Max(magErr, Math.Abs(im[j] - refMag[(2 * c + 1) * plane + j]));+ magPeak = Math.Max(magPeak, Math.Abs(refMag[(2 * c) * plane + j]));+ }+ }+ yield return (magErr < 1e-4 * Math.Max(1, magPeak), $"htdemucs spectrogram matches _spec (max err {magErr:g3}, peak {magPeak:g3})");++ var refIspec = Floats(Path.Combine(fx, "ispec.f32"));+ var back = new float[seg];+ double ispecErr = 0;+ for (int c = 0; c < 2; c++)+ {+ DemucsSpec.Forward((c == 0 ? left : right).AsSpan(0, seg), re, im);+ DemucsSpec.Inverse(re, im, 0, 0, seg, back);+ for (int i = 0; i < seg; i++) ispecErr = Math.Max(ispecErr, Math.Abs(back[i] - refIspec[c * seg + i]));+ }+ yield return (ispecErr < 1e-4, $"…and the iSTFT matches _ispec (max err {ispecErr:g3})");++ // ---- separation ----------------------------------------------------------------------+ var refStems = Floats(Path.Combine(fx, "stems.f32"));+ var sw = System.Diagnostics.Stopwatch.StartNew();+ float[][][] stems;+ using (var sep = new StemSeparator(runtime, Path.Combine(dir, "htdemucs.onnx")))+ stems = sep.Separate(left, right);+ sw.Stop();+ for (int s = 0; s < StemSeparator.Stems.Length; s++)+ {+ double err = 0, sig = 0;+ for (int c = 0; c < 2; c++)+ for (int i = 0; i < n; i++)+ {+ double r = refStems[(s * 2 + c) * n + i], d = stems[s][c][i] - r;+ err += d * d; sig += r * r;+ }+ double snr = 10 * Math.Log10(sig / Math.Max(1e-20, err));+ yield return (snr > 50, $"{StemSeparator.Stems[s]} stem matches apply_model ({snr:0} dB SNR)");+ }+ Console.WriteLine($" separation of {n / 44100.0:0.0} s took {sw.Elapsed.TotalSeconds:0.0} s");++ // ---- basic-pitch ---------------------------------------------------------------------+ var mono = Floats(Path.Combine(fx, "mono22.f32"));+ var refNote = Floats(Path.Combine(fx, "bp_note.f32"));+ var refOnset = Floats(Path.Combine(fx, "bp_onset.f32"));+ using var doc = JsonDocument.Parse(File.ReadAllText(Path.Combine(fx, "bp_notes.json")));+ int bpFrames = doc.RootElement.GetProperty("frames").GetInt32();+ var refNotes = doc.RootElement.GetProperty("notes").EnumerateArray()+ .Select(e => (Start: e[0].GetDouble(), End: e[1].GetDouble(), Pitch: e[2].GetInt32())).ToList();++ // Decoding alone, fed the reference activations: must reproduce the notes exactly.+ var decoded = PitchTranscriber.Decode(refNote, refOnset, bpFrames);+ yield return (decoded.Count == refNotes.Count, $"basic-pitch decoding finds the reference's {refNotes.Count} notes (got {decoded.Count})");++ List<TranscribedNote> notes;+ using (var tr = new PitchTranscriber(runtime, Path.Combine(dir, "nmp.onnx")))+ notes = tr.Transcribe(mono);+ int matched = refNotes.Count(r => notes.Any(nn => nn.Pitch == r.Pitch && Math.Abs(nn.Start - r.Start) < 0.012 && Math.Abs(nn.End - r.End) < 0.012));+ yield return (matched >= refNotes.Count * 0.98 && Math.Abs(notes.Count - refNotes.Count) <= Math.Max(2, refNotes.Count / 50),+ $"basic-pitch end to end: {matched}/{refNotes.Count} reference notes matched to a frame ({notes.Count} found)");++ // ---- resampler -----------------------------------------------------------------------+ var tone = new float[48000];+ for (int i = 0; i < tone.Length; i++) tone[i] = (float)Math.Sin(2 * Math.PI * 1000 * i / 48000.0);+ var down = Resampler.Convert(tone, 48000, 44100);+ double rErr = 0;+ for (int i = 200; i < down.Length - 200; i++) rErr = Math.Max(rErr, Math.Abs(down[i] - Math.Sin(2 * Math.PI * 1000 * i / 44100.0)));+ yield return (down.Length == 44100 && rErr < 1e-3, $"resampler 48 → 44.1 kHz keeps a 1 kHz tone (max err {rErr:g3})");+ }++ private static float[] Floats(string path)+ {+ var bytes = File.ReadAllBytes(path);+ var f = new float[bytes.Length / 4];+ Buffer.BlockCopy(bytes, 0, f, 0, f.Length * 4);+ return f;+ }+}diff --git a/tests/Nota.SmokeTest/Program.cs b/tests/Nota.SmokeTest/Program.csindex beb8c1d1..585ac452 100644--- a/tests/Nota.SmokeTest/Program.cs+++ b/tests/Nota.SmokeTest/Program.cs@@ -116,6 +116,14 @@ if (args.Length >= 1 && args[0] == "--samples")Console.WriteLine(failures == 0 ? "SAMPLES PASSED" : $"SAMPLES FAILED ({failures})");return failures == 0 ? 0 : 1;}+// AI models' DSP ports against the Python originals: `--ai-golden <libonnxruntime> <dir>`.+if (args.Length >= 3 && args[0] == "--ai-golden")+{+ Console.WriteLine("-- ai models: golden output --");+ foreach (var (ok, label) in AiGoldenTests.Run(args[1], args[2])) Check(ok, label);+ Console.WriteLine(failures == 0 ? "AI GOLDEN PASSED" : $"AI GOLDEN FAILED ({failures})");+ return failures == 0 ? 0 : 1;+}if (args.Length >= 3 && args[0] == "--store-catalog"){Console.WriteLine("-- get plug-ins: catalog identifiers --");