Skip to content

Liquid Glass and Apple Music touches for the Spotify iOS app. A GPL-3.0 fork of spoti.pw 0.21.1.

C
15.0 KB
Raw Blame
#import <TargetConditionals.h>
#if TARGET_OS_IPHONE
#import <os/proc.h>
#endif
#import "Core/SGLog.h"
#import "SGSingLoader.h"
#import "SGSingSeparator.h"

// The CPU copy loaded in 0.9-5 s on an M4. The Neural Engine's first load compiles the model, which Core ML then
// keeps: 24-63 s on an M4 and 32-46 s on an iPhone 15 Pro, then 0.4-0.7 s; it has ten minutes.
double SGSingLoaderCPUDeadline = 120, SGSingLoaderNeuralDeadline = 600, SGSingLoaderKeepSeconds = 60;
// A CPU copy loads beside an abandoned load still out (or again after a memory warning) only with this much left: a
// second copy took 1.07-1.15 GB more of the process's footprint on an M4.
static const unsigned long long kCopyRoom = 1600ull * 1000 * 1000;
// The first CPU copy loads only with this much left, in place of a floor on the iPhone's memory. On an iPhone 15 Pro both
// copies warm added 0.1 GB of footprint (0.20 to 0.30 GB); the weights are mapped from disk and counted only as resident.
// 1 GB leaves the Neural Engine copy its 0.5 GB after the CPU's.
static const unsigned long long kFirstRoom = 1000ull * 1000 * 1000;
// The Neural Engine copy loads beside the CPU's only with this much left. On an iPhone 15 Pro it added 0.06-0.1 GB of
// footprint (resident 1.06 to 1.13 GB); 0.5 GB is that five times over, for its first compile, which was not measured
// apart, and for Spotify's own memory to grow without iOS closing it for a copy Sing can do without.
static const unsigned long long kNeuralRoom = 500ull * 1000 * 1000;
// The light model's (separator-light.mlmodelc, 31 MB) rooms. Measured in harness/sing on an M4 with both models, the
// copies' footprint barely differs (a warm CPU copy 0.13 GB light, 0.12 GB full) and resident memory follows the
// model's size: two CPU copies warm 0.29 GB light against 0.72 GB full, a ratio of 0.40, which scales the first and
// second copies' rooms: 0.4 GB and 0.65 GB. Its Neural Engine copy added 0.03-0.05 GB of footprint against the full
// one's 0.08 GB; 0.25 GB is five times that, as the full one's is. An earlier export took 0.3 GB of footprint on an
// iPhone 15 Pro with the CPU copy warm, so 0.4 GB still covers it, and leaves the Neural Engine copy its room after.
static const unsigned long long kLightFirstRoom = 400ull * 1000 * 1000, kLightCopyRoom = 650ull * 1000 * 1000,
                                kLightNeuralRoom = 250ull * 1000 * 1000;
static const int kStillEvery = 30;

// The rooms of the model at `url`: the light model's by its folder's name, which its dev copy shares.
static BOOL isLight(NSURL *url) {
    return [url.lastPathComponent isEqualToString:@"separator-light.mlmodelc"];
}

static unsigned long long firstRoom(NSURL *url) {
    return isLight(url) ? kLightFirstRoom : kFirstRoom;
}

static unsigned long long copyRoom(NSURL *url) {
    return isLight(url) ? kLightCopyRoom : kCopyRoom;
}

static unsigned long long neuralRoom(NSURL *url) {
    return isLight(url) ? kLightNeuralRoom : kNeuralRoom;
}

// One load of one copy.
@interface SGSingLoad : NSObject
@property (nonatomic) unsigned number;
@property (nonatomic, copy) NSString *name;   // the copy's, for the log
@property (nonatomic, copy) NSURL *url;
@property (nonatomic) CFAbsoluteTime began;
@property (nonatomic) BOOL finished, abandoned;
@end

@implementation SGSingLoad
@end

static void (^sg_changed)(void);
static SGSingLoaderState sg_state;
static NSString *sg_error;
static SGSingSeparator *sg_separator;
static SGSingLoad *sg_cpuLoad, *sg_fastLoad;   // the loads that count; abandoned ones are let go
static unsigned sg_outstanding;                 // loads not come back yet, abandoned ones with them
static unsigned sg_attempts;
static NSURL *sg_url;
static BOOL sg_neural;                          // a Neural Engine copy is asked for
static SGSingFastState sg_fastState;
static BOOL sg_foreground;
static NSUInteger sg_keepToken;                 // counts the mic's releases and wants, so a stale minute drops nothing
static BOOL sg_wanted;                          // the mic is on: the Neural Engine copy is worth loading

// Run after the caller returns, so a listener that wants or purges again does not run inside it.
static void changed(void) {
    dispatch_async(dispatch_get_main_queue(), ^{
        if (sg_changed) sg_changed();
    });
}

// The memory the process has left before iOS closes it; 0 where the OS does not say (the Mac).
static unsigned long long memoryLeft(void) {
#if TARGET_OS_IPHONE
    return os_proc_available_memory();
#else
    return 0;
#endif
}

static NSString *memoryText(void) {
    unsigned long long left = memoryLeft();
    return left ? [NSString stringWithFormat:@"%.2f GB left to the process", left / 1e9] : @"memory left unknown";
}

// Whether `room` is left for a copy beside what is already in memory or still loading.
static BOOL roomFor(unsigned long long room) {
    unsigned long long left = memoryLeft();
    return !left || left >= room;
}

static void still(SGSingLoad *load, int seconds) {
    dispatch_after(dispatch_time(DISPATCH_TIME_NOW, kStillEvery * NSEC_PER_SEC), dispatch_get_main_queue(), ^{
        if (load.finished || load.abandoned) return;
        SGLog(@"sing: still loading the %@ copy (load %u), %d s", load.name, load.number, seconds + kStillEvery);
        still(load, seconds + kStillEvery);
    });
}

// Loads and warms a copy on a queue of its own; `done` runs on the main thread unless the load was abandoned
// first, and `late` when its deadline passes before it comes back.
static SGSingLoad *startLoad(NSURL *url, BOOL neural, double deadline, void (^done)(MLModel *model, NSError *error), void (^late)(void)) {
    SGSingLoad *load = [SGSingLoad new];
    load.number = ++sg_attempts;
    load.name = neural ? @"Neural Engine" : @"CPU";
    load.url = url;
    load.began = CFAbsoluteTimeGetCurrent();
    sg_outstanding++;
    SGLog(@"sing: load %u, the %@ copy of %@, starts (Spotify %@, thermal state %s, %@, %@, %u loads out)", load.number, load.name,
          url.lastPathComponent, sg_foreground ? @"active" : @"not active", SGSingThermalName(), memoryText(), SGSingMemoryText(), sg_outstanding);
    NSString *name = [NSString stringWithFormat:@"spotifyglass.sing.load.%u", load.number];
    dispatch_queue_t queue = dispatch_queue_create(name.UTF8String, dispatch_queue_attr_make_with_qos_class(DISPATCH_QUEUE_SERIAL, QOS_CLASS_USER_INITIATED, 0));
    dispatch_async(queue, ^{
        MLModelConfiguration *configuration = [MLModelConfiguration new];
        configuration.computeUnits = neural ? MLComputeUnitsCPUAndNeuralEngine : MLComputeUnitsCPUOnly;
        NSError *error;
        MLModel *model = [MLModel modelWithContentsOfURL:url configuration:configuration error:&error];
        double loaded = CFAbsoluteTimeGetCurrent() - load.began;
        NSString *loadedMemory = SGSingMemoryText();
        double warm = model ? [SGSingSeparator warmUp:model error:&error] : -1;
        NSString *warmMemory = SGSingMemoryText();
        dispatch_async(dispatch_get_main_queue(), ^{
            sg_outstanding--;
            load.finished = YES;
            NSString *outcome = !model ? [NSString stringWithFormat:@"did not load after %.1f s: %@", loaded, error.localizedDescription]
                              : warm < 0 ? [NSString stringWithFormat:@"loaded in %.1f s, and its warm-up failed: %@", loaded, error.localizedDescription]
                                         : [NSString stringWithFormat:@"loaded in %.1f s, and its warm-up window of silence took %.1f s", loaded, warm];
            SGLog(@"sing: load %u, the %@ copy, %@ (memory loaded: %@; warm: %@)%@", load.number, load.name, outcome, loadedMemory, warmMemory,
                  load.abandoned ? @"; it was abandoned before, so it is let go" : @"");
            if (load.abandoned) return;
            done(warm >= 0 ? model : nil, error);
        });
    });
    still(load, 0);
    dispatch_after(dispatch_time(DISPATCH_TIME_NOW, (int64_t)(deadline * NSEC_PER_SEC)), dispatch_get_main_queue(), ^{
        if (load.finished || load.abandoned) return;
        load.abandoned = YES;
        SGLog(@"sing: load %u, the %@ copy, has not come back in %.0f s: abandoned", load.number, load.name, deadline);
        late();
    });
    return load;
}

static SGSingLoad *sg_prepareLoad;   // SGSingLoaderPrepareNeural's, until it comes back or is taken over

static void fastLoaded(MLModel *model) {
    sg_fastLoad = nil;
    sg_fastState = model ? SGSingFastReady : SGSingFastFailed;
    if (model) [sg_separator setFastModel:model named:@"Neural Engine"];
    changed();
}

static void fastLate(void) {
    sg_fastLoad = nil;
    sg_fastState = SGSingFastTimedOut;
    changed();
}

static void startFast(void) {
    if (!sg_wanted || sg_state != SGSingLoaderReady || !sg_foreground || !sg_neural || sg_fastState != SGSingFastNone) return;
    // The compile running ahead of the mic is this copy's: a second load would compile it again beside it.
    if (sg_prepareLoad && [sg_prepareLoad.url isEqual:sg_url] && !sg_prepareLoad.finished && !sg_prepareLoad.abandoned) {
        sg_fastState = SGSingFastLoading;
        sg_fastLoad = sg_prepareLoad;
        SGLog(@"sing: the Neural Engine copy is the one already compiling (load %u)", sg_fastLoad.number);
        changed();
        return;
    }
    if (!roomFor(neuralRoom(sg_url))) {
        sg_fastState = SGSingFastSkipped;
        SGLog(@"sing: no Neural Engine copy: %@, it wants %.2f GB, so Sing stays on the CPU", memoryText(), neuralRoom(sg_url) / 1e9);
        changed();
        return;
    }
    sg_fastState = SGSingFastLoading;
    sg_fastLoad = startLoad(sg_url, YES, SGSingLoaderNeuralDeadline, ^(MLModel *model, NSError *error) {
        fastLoaded(model);
    }, ^{
        fastLate();
    });
    changed();
}

BOOL SGSingLoaderPrepareNeural(NSURL *url, void (^done)(BOOL loaded)) {
    if (!url || sg_outstanding || sg_state != SGSingLoaderIdle || !sg_foreground || !roomFor(neuralRoom(url))) return NO;
    __block SGSingLoad *load = startLoad(url, YES, SGSingLoaderNeuralDeadline, ^(MLModel *model, NSError *error) {
        BOOL taken = sg_fastLoad == load;
        sg_prepareLoad = nil;
        if (taken) fastLoaded(model);
        else if (done) done(model != nil);
    }, ^{
        BOOL taken = sg_fastLoad == load;
        sg_prepareLoad = nil;
        if (taken) fastLate();
        else if (done) done(NO);
    });
    sg_prepareLoad = load;
    return YES;
}

static void startCPU(void) {
    if (!roomFor(sg_outstanding ? copyRoom(sg_url) : firstRoom(sg_url))) {
        sg_state = SGSingLoaderFailed;
        sg_error = sg_outstanding
            ? [NSString stringWithFormat:@"A load abandoned earlier still holds memory, and %@. Close Spotify and open it again to load the voice model.", memoryText()]
            : [NSString stringWithFormat:@"iOS leaves Spotify too little memory for the voice model: %@, and it needs %.1f GB. Closing Spotify and opening it again can free some.", memoryText(), firstRoom(sg_url) / 1e9];
        SGLog(@"sing: no new load: %@", sg_error);
        changed();
        return;
    }
    sg_state = SGSingLoaderLoading;
    sg_error = nil;
    sg_cpuLoad = startLoad(sg_url, NO, SGSingLoaderCPUDeadline, ^(MLModel *model, NSError *error) {
        sg_cpuLoad = nil;
        sg_separator = model ? [[SGSingSeparator alloc] initWithModel:model] : nil;
        if (sg_separator) {
            sg_state = SGSingLoaderReady;
        } else {
            sg_state = SGSingLoaderFailed;
            sg_error = model ? @"Karaoke could not set aside memory for the voice model."
                             : [NSString stringWithFormat:@"The voice model did not load: %@", error.localizedDescription ?: @"Core ML gave no reason."];
        }
        changed();
        startFast();
    }, ^{
        sg_cpuLoad = nil;
        sg_state = SGSingLoaderFailed;
        sg_error = [NSString stringWithFormat:@"The voice model did not load in %.0f s, so that load was given up.", SGSingLoaderCPUDeadline];
        changed();
    });
    changed();
}

static void abandon(SGSingLoad *load) {
    if (load && !load.finished) load.abandoned = YES;
}

void SGSingLoaderSetChanged(void (^block)(void)) {
    sg_changed = block;
}

void SGSingLoaderDropFast(NSString *why) {
    BOOL had = sg_fastState == SGSingFastLoading || sg_fastState == SGSingFastReady;
    abandon(sg_fastLoad);
    sg_fastLoad = nil;
    [sg_separator setFastModel:nil named:nil];
    sg_fastState = SGSingFastNone;
    if (!had) return;
    if (why) SGLog(@"sing: the Neural Engine copy is dropped: %@", why);
    changed();
}

void SGSingLoaderWant(NSURL *url, BOOL neural) {
    // Another model (its update in): the copies of the one before go, a failure with them, and this one loads.
    if (sg_url && url && ![url isEqual:sg_url]) {
        SGSingLoaderPurge([NSString stringWithFormat:@"the model is now %@", url.lastPathComponent]);
        sg_state = SGSingLoaderIdle;
        sg_error = nil;
    }
    sg_keepToken++;
    sg_wanted = YES;
    sg_url = url;
    sg_neural = neural;
    if (!neural) SGSingLoaderDropFast(@"Karaoke runs on the CPU alone now");
    if (sg_state == SGSingLoaderIdle) startCPU();
    else startFast();
}

void SGSingLoaderPurge(NSString *why) {
    sg_keepToken++;
    sg_wanted = NO;
    BOOL had = sg_separator || sg_cpuLoad || sg_fastLoad;
    abandon(sg_cpuLoad);
    sg_cpuLoad = nil;
    SGSingLoaderDropFast(nil);
    sg_separator = nil;
    if (!had) return;
    sg_state = SGSingLoaderIdle;
    SGLog(@"sing: the voice model is dropped: %@", why);
    changed();
}

void SGSingLoaderRelease(void) {
    NSUInteger token = ++sg_keepToken;
    sg_wanted = NO;
    if (sg_state == SGSingLoaderFailed) {
        sg_state = SGSingLoaderIdle;
        sg_error = nil;
    }
    dispatch_after(dispatch_time(DISPATCH_TIME_NOW, (int64_t)(SGSingLoaderKeepSeconds * NSEC_PER_SEC)), dispatch_get_main_queue(), ^{
        if (token == sg_keepToken) SGSingLoaderPurge([NSString stringWithFormat:@"the mic has been off for %.0f s", SGSingLoaderKeepSeconds]);
    });
}

void SGSingLoaderSetForeground(BOOL foreground) {
    sg_foreground = foreground;
    if (foreground) startFast();
}

BOOL SGSingLoaderHasRoom(void) {
    return roomFor(copyRoom(sg_url));
}

NSURL *SGSingLoaderURL(void) {
    return sg_url;
}

SGSingLoaderState SGSingLoaderCurrentState(void) {
    return sg_state;
}

SGSingSeparator *SGSingLoaderSeparator(void) {
    return sg_state == SGSingLoaderReady ? sg_separator : nil;
}

NSString *SGSingLoaderError(void) {
    return sg_state == SGSingLoaderFailed ? sg_error : nil;
}

NSTimeInterval SGSingLoaderSeconds(void) {
    return sg_cpuLoad ? CFAbsoluteTimeGetCurrent() - sg_cpuLoad.began : 0;
}

SGSingFastState SGSingLoaderFastState(void) {
    return sg_fastState;
}

NSTimeInterval SGSingLoaderFastSeconds(void) {
    return sg_fastLoad ? CFAbsoluteTimeGetCurrent() - sg_fastLoad.began : 0;
}

unsigned SGSingLoaderAttempts(void) {
    return sg_attempts;
}

unsigned SGSingLoaderOutstanding(void) {
    return sg_outstanding;
}

NSTimeInterval SGSingLoaderPreparingSeconds(void) {
    SGSingLoad *load = sg_prepareLoad;
    if (!load || load.finished || load.abandoned || sg_fastLoad == load) return -1;
    return CFAbsoluteTimeGetCurrent() - load.began;
}