Skip to content

armgit / nota

Public Imported from GitHub
0

Nota — a free DAW for macOS, Windows and Linux. No subscription, no account — just open it and make music.

10.9 KB
Raw Blame
// SPDX-License-Identifier: AGPL-3.0-only
// Copyright (c) 2026 Egor Khindikaynen (Nota). See LICENSES/ for license terms.
//
// The AI features on an audio clip, shared by the clip menu and the MCP tools. Each runs in
// three steps so the caller can keep the engine on its own thread: read the clip (engine
// thread), run the model (any thread — tens of seconds for stems), apply the result (engine
// thread, one undo step).
//
// Separate Stems works on the clip's *source*: the stems are written as files as long as the
// source, at its sample rate, silent outside the part the clip uses, so each stem clip can take
// the original clip's exact shape (region, warp markers, reverse, envelopes, ADSR —
// AudioClipState) and line up with it sample for sample. The original clip is switched off,
// not deleted.

using Nota.Application.Localization;

namespace Nota.Infrastructure;

/// <summary>An audio clip read for separation: its shape, and the source audio to separate.</summary>
public sealed class StemSource : StemJob
{
    public required AudioClipDto Shape { get; init; }        // the clip, for the stem clips to copy
    public required double SampleRate { get; init; }
    public required long SourceFrames { get; init; }          // the whole source's length
    public required long From { get; init; }                  // separated region of the source
    public required float[] Left { get; init; }               // the region, de-interleaved
    public required float[] Right { get; init; }
}

/// <summary>Separated stems written to temp WAVs (one per <see cref="StemSeparator.Stems"/>).</summary>
public sealed class StemFiles : StemResult
{
    public required string[] Paths { get; init; }

    public override void Dispose()
    {
        foreach (var p in Paths)
            try { File.Delete(p); } catch { /* temp; best-effort */ }
    }
}

public sealed class ClipAi(IAudioEngine engine, IModelStore models) : IClipAi
{
    // Unwarped clips: separate only the region the clip plays, plus a second either side so the
    // model hears some context at the edges.
    private const double MarginSeconds = 1.0;

    /// <summary>The longest audio Separate Stems takes: the four stereo stems are held in memory
    /// (~1 GB at this length).</summary>
    public const double MaxSeconds = 10 * 60;

    // ---- Separate Stems ------------------------------------------------------------------

    /// <summary>Step 1 (engine thread). Throws <see cref="InvalidOperationException"/> with a
    /// user-facing message when the clip can't be separated.</summary>
    public bool CanSeparate => models.IsInstalled(AiModels.Stems) && models.RuntimePath is not null;

    public StemJob ReadStemSource(int trackId, int clipIndex)
    {
        if (!engine.TryGetAudioClipInfo(trackId, clipIndex, out var ac))
            throw new InvalidOperationException("That isn't an audio clip.");
        if (!engine.TryGetSampleInfo(ac.SampleId, out var si) || si.Frames <= 0 || si.Channels <= 0 || si.SampleRate <= 0)
            throw new InvalidOperationException("This clip has no audio to separate.");

        long from = 0, to = si.Frames;
        if (ac.WarpEnabled == 0)
        {
            long margin = (long)(MarginSeconds * si.SampleRate);
            long start = (long)Math.Floor(ac.SourceOffsetFrames);
            long end = ac.LengthFrames > 0 ? start + ac.LengthFrames : si.Frames;
            from = Math.Clamp(start - margin, 0, si.Frames);
            to = Math.Clamp(end + margin, from, si.Frames);
        }
        if ((to - from) / si.SampleRate > MaxSeconds)
            throw new InvalidOperationException($"This clip is longer than {MaxSeconds / 60:0} minutes — split it first, then separate the parts.");
        if (to - from < si.SampleRate * 0.1)
            throw new InvalidOperationException("This clip is too short to separate.");

        var data = engine.ReadSample(ac.SampleId);
        int ch = si.Channels, n = (int)(to - from);
        var left = new float[n];
        var right = new float[n];
        for (int i = 0; i < n; i++)
        {
            long at = (from + i) * ch;
            if ((at + ch) > data.Length) break;
            left[i] = data[at];
            right[i] = ch > 1 ? data[at + 1] : data[at];
        }

        string name = engine.GetClipName(trackId, clipIndex) is { Length: > 0 } cn ? cn
                    : engine.GetTrackName(trackId) is { Length: > 0 } tn ? tn : "Audio";
        return new StemSource
        {
            TrackId = trackId, ClipIndex = clipIndex, Name = name,
            Shape = AudioClipState.Capture(engine, trackId, clipIndex, ac),
            SampleRate = si.SampleRate, SourceFrames = si.Frames, From = from, Left = left, Right = right,
        };
    }

    /// <summary>Step 2 (any thread): runs htdemucs and writes the stems as temp WAVs.
    /// <paramref name="progress"/> reports 0..1 with a stage label.</summary>
    public StemResult Separate(StemJob job, IProgress<(double Fraction, string Label)>? progress = null, CancellationToken ct = default)
    {
        var src = (StemSource)job;
        var (runtime, model) = Require(AiModels.Stems);
        progress?.Report((0, Loc.T("ai.clip_ai.preparing_audio")));
        double sr = src.SampleRate;
        var l = Resampler.Convert(src.Left, sr, StemSeparator.SampleRate, ct);
        var r = Resampler.Convert(src.Right, sr, StemSeparator.SampleRate, ct);

        float[][][] stems;
        progress?.Report((0, Loc.T("ai.clip_ai.loading_the_model")));
        using (var sep = new StemSeparator(runtime, model))
            stems = sep.Separate(l, r, new Progress(p => progress?.Report((p, Loc.T("ai.clip_ai.separating_stems")))), ct);

        progress?.Report((1, Loc.T("ai.clip_ai.writing_stems")));
        var files = new StemFiles { Paths = new string[StemSeparator.Stems.Length] };
        try
        {
            int rate = (int)Math.Round(sr);
            int regionLen = src.Left.Length;
            for (int s = 0; s < StemSeparator.Stems.Length; s++)
            {
                ct.ThrowIfCancellationRequested();
                var sl = Fit(Resampler.Convert(stems[s][0], StemSeparator.SampleRate, sr, ct), regionLen);
                var sr2 = Fit(Resampler.Convert(stems[s][1], StemSeparator.SampleRate, sr, ct), regionLen);
                var path = Path.Combine(Path.GetTempPath(), $"nota-stem-{Guid.NewGuid():N}.wav");
                files.Paths[s] = path;
                WriteStem(path, rate, src.SourceFrames, src.From, sl, sr2);
            }
            return files;
        }
        catch
        {
            files.Dispose();
            throw;
        }
    }

    /// <summary>Step 3 (engine thread): a group of stem tracks under the source track, each with
    /// a clip shaped like the original, which is switched off. One undo step. Returns the group's id.</summary>
    public int ApplyStems(StemJob job, StemResult stems)
    {
        var src = (StemSource)job;
        var files = (StemFiles)stems;
        int srcIndex = -1, parent = -1;
        for (int i = 0; i < engine.TrackCount; i++)
            if (engine.TryGetTrackInfo(i, out var ti) && ti.Id == src.TrackId) { srcIndex = i; parent = ti.GroupId; break; }
        if (srcIndex < 0) throw new InvalidOperationException("The clip's track is gone.");

        engine.BeginUndoGroup();
        try
        {
            var ids = new int[StemSeparator.Stems.Length];
            for (int s = 0; s < ids.Length; s++)
            {
                int id = engine.AddAudioTrack();
                ids[s] = id;
                engine.SetTrackName(id, StemSeparator.Stems[s]);
                if (parent >= 0) engine.SetTrackGroup(id, parent);
                engine.MoveTrack(id, srcIndex + 1 + s);
                var shape = Clone(src.Shape);
                shape.Name = $"{src.Name} {StemSeparator.Stems[s]}";
                shape.Active = true;
                if (AudioClipState.Restore(engine, id, shape, files.Paths[s]) < 0)
                    throw new InvalidOperationException("Nota couldn't load the separated stems.");
            }
            int group = engine.CreateGroup(ids);
            if (group > 0) engine.SetTrackName(group, $"{src.Name} Stems");
            engine.SetClipActive(src.TrackId, src.ClipIndex, false);
            return group;
        }
        finally
        {
            engine.EndUndoGroup();
        }
    }

    // ---- Convert to MIDI ----------------------------------------------------------------

    /// <summary>Whether Convert Melody / Harmony can use basic-pitch.</summary>
    public bool CanTranscribe => models.IsInstalled(AiModels.Transcription) && models.RuntimePath is not null;

    /// <summary>Notes in <paramref name="mono"/> (any rate) with basic-pitch; times in seconds.</summary>
    public IReadOnlyList<TranscribedNote> Transcribe(float[] mono, double sampleRate, IProgress<double>? progress = null, CancellationToken ct = default)
    {
        var (runtime, model) = Require(AiModels.Transcription);
        var x = Resampler.Convert(mono, sampleRate, PitchTranscriber.SampleRate, ct);
        using var tr = new PitchTranscriber(runtime, model);
        return tr.Transcribe(x, progress, ct);
    }

    // ---- helpers ------------------------------------------------------------------------

    private (string Runtime, string Model) Require(string id)
        => models.RuntimePath is { } rt && models.ModelPath(id) is { } m
            ? (rt, m)
            : throw new InvalidOperationException("The AI model isn't installed — get it in Settings → Downloads → AI Models.");

    private static float[] Fit(float[] x, int n)
    {
        if (x.Length == n) return x;
        var y = new float[n];
        Array.Copy(x, y, Math.Min(n, x.Length));
        return y;
    }

    // The stem as long as the whole source, silent outside the separated region (24-bit is
    // plenty for a stem and keeps the project a third smaller than float).
    private static void WriteStem(string path, int rate, long sourceFrames, long from, float[] l, float[] r)
    {
        using var w = new WavWriter(path, rate, 2, WavBitDepth.Pcm24);
        const int block = 1 << 15;
        var buf = new float[block * 2];
        for (long pos = 0; pos < sourceFrames; pos += block)
        {
            int n = (int)Math.Min(block, sourceFrames - pos);
            for (int i = 0; i < n; i++)
            {
                long k = pos + i - from;
                bool inside = k >= 0 && k < l.Length;
                buf[2 * i] = inside ? l[k] : 0f;
                buf[2 * i + 1] = inside ? r[k] : 0f;
            }
            w.WriteFrames(buf, n);
        }
    }

    private static AudioClipDto Clone(AudioClipDto d)
        => System.Text.Json.JsonSerializer.Deserialize<AudioClipDto>(System.Text.Json.JsonSerializer.Serialize(d))!;

    // IProgress that reports on the calling thread (the separator's), not via a context.
    private sealed class Progress(Action<double> report) : IProgress<double>
    {
        public void Report(double value) => report(value);
    }
}