Skip to content

Liquid Glass and Apple Music touches for the Spotify iOS app. A GPL-3.0 fork of spoti.pw 0.21.1.

C
13.1 KB
Raw Blame
#import <Accelerate/Accelerate.h>
#import <CoreML/CoreML.h>
#import <mach/mach.h>
#import <os/lock.h>
#import "Core/SGLog.h"
#import "SGSingSeparator.h"

enum {
    kPad = kSGSingFFT / 2,                            // the centering's reflect padding on each side
    kPadded = kSGSingWindowFrames + 2 * kPad,
    kHalf = kSGSingFFT / 2,
};

static NSString *const kInput = @"spectrum", *const kOutput = @"vocals_spectrum";
// This many windows on each copy are logged with their time.
static const unsigned long long kLoggedWindows = 5;

const char *SGSingThermalName(void) {
    switch (NSProcessInfo.processInfo.thermalState) {
        case NSProcessInfoThermalStateNominal: return "nominal";
        case NSProcessInfoThermalStateFair: return "fair";
        case NSProcessInfoThermalStateSerious: return "serious";
        case NSProcessInfoThermalStateCritical: return "critical";
    }
    return "?";
}

NSString *SGSingMemoryText(void) {
    task_vm_info_data_t info;
    mach_msg_type_number_t count = TASK_VM_INFO_COUNT;
    if (task_info(mach_task_self(), TASK_VM_INFO, (task_info_t)&info, &count) != KERN_SUCCESS) return @"footprint unknown";
    return [NSString stringWithFormat:@"footprint %.2f GB, resident %.2f GB", info.phys_footprint / 1e9, info.resident_size / 1e9];
}

// Where bin `bin` of channel `channel` at STFT frame `frame` starts in the model's layout (its real part;
// the imaginary part follows it).
static inline size_t at(int bin, int channel, int frame) {
    return ((size_t)(2 * bin + channel) * kSGSingSTFTFrames + frame) * 2;
}

@implementation SGSingSeparator {
    MLModel *_model;
    os_unfair_lock _lock;        // the faster copy and its name
    MLModel *_fast;
    NSString *_fastName;
    SGSingSeparatorStats _stats; // the worker's; read whole under the lock
    MLMultiArray *_input;
    vDSP_DFT_Setup _forward, _inverse;
    float _window[kSGSingFFT];
    float _envelope[kSGSingWindowFrames];   // 1 / the squared window's overlap-add, per output frame
    float _padded[kPadded];
    float _frame[kSGSingFFT];
    float _real[kHalf], _imag[kHalf], _outReal[kHalf], _outImag[kHalf];
    float _sum[kPadded];
    float *_spectrum, *_vocals;
}

- (instancetype)initWithModel:(MLModel *)model {
    if (!(self = [super init])) return nil;
    _model = model;
    _lock = OS_UNFAIR_LOCK_INIT;
    _forward = vDSP_DFT_zrop_CreateSetup(NULL, kSGSingFFT, vDSP_DFT_FORWARD);
    _inverse = vDSP_DFT_zrop_CreateSetup(_forward, kSGSingFFT, vDSP_DFT_INVERSE);
    _spectrum = calloc(kSGSingSpectrumFloats, sizeof(float));
    _vocals = calloc(kSGSingSpectrumFloats, sizeof(float));
    if (!_forward || !_inverse || !_spectrum || !_vocals) return nil;
    // torch.hann_window's default, periodic: the denominator is N, not N - 1.
    for (int n = 0; n < kSGSingFFT; n++) _window[n] = 0.5f * (1 - cosf(2 * (float)M_PI * n / kSGSingFFT));
    float squares[kSGSingFFT];
    vDSP_vsq(_window, 1, squares, 1, kSGSingFFT);
    memset(_sum, 0, sizeof _sum);
    for (int t = 0; t < kSGSingSTFTFrames; t++) vDSP_vadd(_sum + t * kSGSingHop, 1, squares, 1, _sum + t * kSGSingHop, 1, kSGSingFFT);
    for (int i = 0; i < kSGSingWindowFrames; i++) _envelope[i] = 1 / _sum[i + kPad];
    if (model) {
        NSError *error;
        _input = [[MLMultiArray alloc] initWithDataPointer:_spectrum shape:@[@1, @(2 * kSGSingBins), @(kSGSingSTFTFrames), @2]
                                                  dataType:MLMultiArrayDataTypeFloat32
                                                   strides:@[@(2 * kSGSingBins * kSGSingSTFTFrames * 2), @(kSGSingSTFTFrames * 2), @2, @1]
                                               deallocator:nil error:&error];
        if (!_input) return nil;
    }
    return self;
}

- (void)dealloc {
    if (_inverse) vDSP_DFT_DestroySetup(_inverse);
    if (_forward) vDSP_DFT_DestroySetup(_forward);
    free(_spectrum);
    free(_vocals);
}

#pragma mark - the STFT

- (void)analyzeChannel:(const float *)samples channel:(int)channel into:(float *)spectrum {
    // Reflect padding, the edge sample itself not repeated (torch's "reflect").
    memcpy(_padded + kPad, samples, kSGSingWindowFrames * sizeof(float));
    for (int i = 1; i <= kPad; i++) {
        _padded[kPad - i] = samples[i];
        _padded[kPad + kSGSingWindowFrames - 1 + i] = samples[kSGSingWindowFrames - 1 - i];
    }
    DSPSplitComplex split = {_real, _imag};
    for (int t = 0; t < kSGSingSTFTFrames; t++) {
        vDSP_vmul(_padded + t * kSGSingHop, 1, _window, 1, _frame, 1, kSGSingFFT);
        vDSP_ctoz((const DSPComplex *)_frame, 2, &split, 1, kHalf);
        vDSP_DFT_Execute(_forward, _real, _imag, _outReal, _outImag);
        // vDSP's real transform comes out twice the DFT, with the Nyquist bin's real part where the DC
        // bin's imaginary part would be.
        float *dc = spectrum + at(0, channel, t), *nyquist = spectrum + at(kHalf, channel, t);
        dc[0] = _outReal[0] * 0.5f;
        dc[1] = 0;
        nyquist[0] = _outImag[0] * 0.5f;
        nyquist[1] = 0;
        for (int k = 1; k < kHalf; k++) {
            float *bin = spectrum + at(k, channel, t);
            bin[0] = _outReal[k] * 0.5f;
            bin[1] = _outImag[k] * 0.5f;
        }
    }
}

- (void)analyzeLeft:(const float *)left right:(const float *)right into:(float *)spectrum {
    [self analyzeChannel:left channel:0 into:spectrum];
    [self analyzeChannel:right channel:1 into:spectrum];
}

- (void)synthesizeChannel:(int)channel from:(const float *)spectrum into:(float *)samples {
    memset(_sum, 0, sizeof _sum);
    DSPSplitComplex split = {_outReal, _outImag};
    // The inverse of a true DFT comes out N times the signal, so 1/N with the window applied again.
    float scale = 1.0f / kSGSingFFT;
    for (int t = 0; t < kSGSingSTFTFrames; t++) {
        _real[0] = spectrum[at(0, channel, t)];
        _imag[0] = spectrum[at(kHalf, channel, t)];
        for (int k = 1; k < kHalf; k++) {
            const float *bin = spectrum + at(k, channel, t);
            _real[k] = bin[0];
            _imag[k] = bin[1];
        }
        vDSP_DFT_Execute(_inverse, _real, _imag, _outReal, _outImag);
        vDSP_ztoc(&split, 1, (DSPComplex *)_frame, 2, kHalf);
        vDSP_vmul(_frame, 1, _window, 1, _frame, 1, kSGSingFFT);
        vDSP_vsma(_frame, 1, &scale, _sum + t * kSGSingHop, 1, _sum + t * kSGSingHop, 1, kSGSingFFT);
    }
    vDSP_vmul(_sum + kPad, 1, _envelope, 1, samples, 1, kSGSingWindowFrames);
}

- (void)synthesize:(const float *)spectrum left:(float *)left right:(float *)right {
    [self synthesizeChannel:0 from:spectrum into:left];
    [self synthesizeChannel:1 from:spectrum into:right];
}

#pragma mark - the model

// The model's output into `_vocals`, by its strides, whatever they are and whether it is float32 or 16.
static BOOL readOutput(MLMultiArray *array, float *into) {
    NSArray<NSNumber *> *shape = array.shape, *strides = array.strides;
    if (shape.count != 4 || shape[1].intValue != 2 * kSGSingBins || shape[2].intValue != kSGSingSTFTFrames || shape[3].intValue != 2) return NO;
    MLMultiArrayDataType type = array.dataType;
    if (type != MLMultiArrayDataTypeFloat32 && type != MLMultiArrayDataTypeFloat16) return NO;
    NSInteger s1 = strides[1].integerValue, s2 = strides[2].integerValue, s3 = strides[3].integerValue;
    __block BOOL ok = YES;
    [array getBytesWithHandler:^(const void *bytes, NSInteger size) {
        NSInteger elementSize = type == MLMultiArrayDataTypeFloat32 ? 4 : 2;
        NSInteger last = ((2 * kSGSingBins - 1) * s1 + (kSGSingSTFTFrames - 1) * s2 + s3) * elementSize;
        if (last >= size) {
            ok = NO;
            return;
        }
        BOOL contiguous = s3 == 1 && s2 == 2 && s1 == kSGSingSTFTFrames * 2;
        if (contiguous && type == MLMultiArrayDataTypeFloat32) {
            memcpy(into, bytes, kSGSingSpectrumFloats * sizeof(float));
            return;
        }
        for (NSInteger row = 0; row < 2 * kSGSingBins; row++) {
            for (NSInteger frame = 0; frame < kSGSingSTFTFrames; frame++) {
                for (NSInteger part = 0; part < 2; part++) {
                    NSInteger index = row * s1 + frame * s2 + part * s3;
                    float value = type == MLMultiArrayDataTypeFloat32 ? ((const float *)bytes)[index] : (float)((const _Float16 *)bytes)[index];
                    into[(row * kSGSingSTFTFrames + frame) * 2 + part] = value;
                }
            }
        }
    }];
    return ok;
}

- (void)setFastModel:(MLModel *)model named:(NSString *)name {
    os_unfair_lock_lock(&_lock);
    _fast = model;
    _fastName = name;
    _stats.fastFailed = false;
    _stats.windows[1] = 0;
    _stats.averageMS[1] = 0;
    os_unfair_lock_unlock(&_lock);
}

- (SGSingSeparatorStats)stats {
    os_unfair_lock_lock(&_lock);
    SGSingSeparatorStats stats = _stats;
    os_unfair_lock_unlock(&_lock);
    return stats;
}

// The vocals' spectrum of `features` from `model` into `into`, NO and `error` when it failed.
static BOOL predict(MLModel *model, MLDictionaryFeatureProvider *features, float *into, NSError **error) {
    id<MLFeatureProvider> result = [model predictionFromFeatures:features error:error];
    MLMultiArray *output = [result featureValueForName:kOutput].multiArrayValue;
    if (output && readOutput(output, into)) return YES;
    if (error && !*error) *error = [NSError errorWithDomain:@"SGSing" code:1 userInfo:@{NSLocalizedDescriptionKey: @"The model answered in a shape Karaoke does not read"}];
    return NO;
}

+ (double)warmUp:(MLModel *)model error:(NSError **)error {
    float *silence = calloc(kSGSingSpectrumFloats, sizeof(float)), *out = malloc(kSGSingSpectrumFloats * sizeof(float));
    double took = -1;
    MLMultiArray *input = silence && out ? [[MLMultiArray alloc] initWithDataPointer:silence shape:@[@1, @(2 * kSGSingBins), @(kSGSingSTFTFrames), @2]
                                                                           dataType:MLMultiArrayDataTypeFloat32
                                                                            strides:@[@(2 * kSGSingBins * kSGSingSTFTFrames * 2), @(kSGSingSTFTFrames * 2), @2, @1]
                                                                        deallocator:nil error:error] : nil;
    MLDictionaryFeatureProvider *features = input ? [[MLDictionaryFeatureProvider alloc] initWithDictionary:@{kInput: input} error:error] : nil;
    if (features) {
        CFAbsoluteTime began = CFAbsoluteTimeGetCurrent();
        @autoreleasepool {
            if (predict(model, features, out, error)) took = CFAbsoluteTimeGetCurrent() - began;
        }
        // Silence must come back as silence, not as the non-finite numbers a half-precision floor makes of it.
        for (int i = 0; took >= 0 && i < kSGSingSpectrumFloats; i++) {
            if (!isfinite(out[i])) {
                took = -1;
                if (error) *error = [NSError errorWithDomain:@"SGSing" code:2 userInfo:@{NSLocalizedDescriptionKey: @"The model turned silence into numbers that are not finite"}];
            }
        }
    }
    input = nil;   // before the memory it points into
    free(silence);
    free(out);
    return took;
}

// The window's time into the copy's running average, the first windows logged.
- (void)count:(int)copy took:(double)took named:(NSString *)name {
    os_unfair_lock_lock(&_lock);
    unsigned long long n = ++_stats.windows[copy];
    double ms = took * 1000;
    _stats.averageMS[copy] = n == 1 ? ms : _stats.averageMS[copy] * 0.8 + ms * 0.2;
    _stats.fast = copy == 1;
    os_unfair_lock_unlock(&_lock);
    if (n <= kLoggedWindows) SGLog(@"sing: window %llu on the %@ copy took %.0f ms (1500 ms keeps up), thermal state %s", n, name, ms, SGSingThermalName());
    if (n == kLoggedWindows) SGLog(@"sing: memory after %llu windows on the %@ copy: %@", n, name, SGSingMemoryText());
}

- (BOOL)separateLeft:(const float *)left right:(const float *)right vocalsLeft:(float *)vocalsLeft
         vocalsRight:(float *)vocalsRight error:(NSError **)error {
    if (!_model) return NO;
    [self analyzeLeft:left right:right into:_spectrum];
    MLDictionaryFeatureProvider *features = [[MLDictionaryFeatureProvider alloc] initWithDictionary:@{kInput: _input} error:error];
    if (!features) return NO;
    os_unfair_lock_lock(&_lock);
    MLModel *fast = _stats.fastFailed ? nil : _fast;
    NSString *fastName = _fastName;
    os_unfair_lock_unlock(&_lock);
    if (fast) {
        CFAbsoluteTime began = CFAbsoluteTimeGetCurrent();
        NSError *fastError;
        if (predict(fast, features, _vocals, &fastError)) {
            [self count:1 took:CFAbsoluteTimeGetCurrent() - began named:fastName];
            [self synthesize:_vocals left:vocalsLeft right:vocalsRight];
            return YES;
        }
        os_unfair_lock_lock(&_lock);
        if (_fast == fast) _stats.fastFailed = true;
        _stats.fallbacks++;
        os_unfair_lock_unlock(&_lock);
        SGLog(@"sing: the %@ copy failed a window (%@), so it is done on the CPU's, and the %@ copy is not used again",
              fastName, fastError.localizedDescription, fastName);
    }
    CFAbsoluteTime began = CFAbsoluteTimeGetCurrent();
    if (!predict(_model, features, _vocals, error)) return NO;
    [self count:0 took:CFAbsoluteTimeGetCurrent() - began named:@"CPU"];
    [self synthesize:_vocals left:vocalsLeft right:vocalsRight];
    return YES;
}

@end