Skip to content

Liquid Glass and Apple Music touches for the Spotify iOS app. A GPL-3.0 fork of spoti.pw 0.21.1.

C
19.2 KB
Raw Blame
// Spotify's audio chain (AudioUnitDriver2: a converter fed by a render callback, a mixer, RemoteIO, wired
// with MakeConnection and slices of 4096) rebuilt with real units in the simulator, with
// PlayerSpeedPitch.x's rebinding, render callback and SPTPlayerState hook running on it for real. The
// converter's callback stands in for Spotify's decoder and counts what it hands over, so the log shows
// how fast the song is drained at each speed, and how far a mock player state's position drifts from it.
//
//     THEOS=$HOME/theos ./build.sh && xcrun simctl install booted build/SpeedHarness.app
//     xcrun simctl launch --console-pty booted com.vojta.speedharness [rate]
//
// `rate` hands the running output a 48 kHz format halfway through the 1.5x step, with no new start, while its
// mixer stays at 44.1 kHz: the mod's format listener gives Spotify's own connection back rather than play the
// mixer fast through its callback.
//
// `two` runs a second chain at 48 kHz beside the first at 44.1, both started, the way Spotify runs a chain per
// sample rate, and checks that each decoder is drained at its own rate and only the music's output (the one
// started last, or the one with sound) takes the speed, through a stop, a start, a silent decoder, a disposed
// mixer and a disposed output. Each check prints PASS or FAIL.
//
// Last, the sleep timer's gain: 0.25, then 1 again. A notify of the harness's own, added after the mod's,
// reads the loudest sample the speaker unit played over the last second of each step: 0.05 at 1, 0.0125
// at 0.25.
#import <UIKit/UIKit.h>
#import <AudioToolbox/AudioToolbox.h>
#import <AVFoundation/AVFoundation.h>
#import <stdatomic.h>

double SGPlayerSpeed(void);
void SGSetPlayerSpeed(double speed);
void SGSetPlayerPitch(float semitones);
BOOL SGPlayerSpeedAllowed(void);
void SGPlayerSetGain(float gain);

static const double kRate = 44100;
static atomic_uint_fast64_t sg_decoded;
// The loudest float sample the output played since the main thread last zeroed it, as its bits.
static atomic_uint sg_peakBits;

static OSStatus metered(void *refCon, AudioUnitRenderActionFlags *flags, const AudioTimeStamp *timestamp, UInt32 bus,
                        UInt32 frames, AudioBufferList *data) {
    if (!(*flags & kAudioUnitRenderAction_PostRender) || bus != 0) return noErr;
    uint32_t bits = atomic_load(&sg_peakBits);
    float peak;
    memcpy(&peak, &bits, sizeof peak);
    for (UInt32 b = 0; b < data->mNumberBuffers; b++) {
        const float *samples = data->mBuffers[b].mData;
        for (UInt32 i = 0; samples && i < data->mBuffers[b].mDataByteSize / sizeof(float); i++) peak = fmaxf(peak, fabsf(samples[i]));
    }
    memcpy(&bits, &peak, sizeof bits);
    atomic_store(&sg_peakBits, bits);
    return noErr;
}

AudioUnit SGPlayerMusicOutput(void);

// A chain of its own, for `two`: its decoder's count, and whether it hands over silence.
typedef struct {
    double rate;
    atomic_uint_fast64_t decoded;
    atomic_bool mute;
    AudioUnit converter, mixer, output;
    atomic_uint_fast64_t played;   // frames its output rendered, at the hardware's rate
    double hardwareRate;
    uint64_t lastDecoded, lastPlayed;
} Chain;

// Spotify's player state as far as -position goes (disassembly of -[SPTPlayerState position]).
@interface SPTPlayerState : NSObject
@property (nonatomic) double positionAsOfTimestamp;
@property (nonatomic, strong) NSDate *timestamp;
@end

@implementation SPTPlayerState
// The hooked -position (SpeedPitch.x) also reads which track and whether paused: one track, playing.
- (id)track {
    return nil;
}
- (BOOL)isPaused {
    return NO;
}
- (double)playbackSpeed {
    return 1;
}
- (double)position {
    return MAX(0, self.positionAsOfTimestamp - self.timestamp.timeIntervalSinceNow * [self playbackSpeed]);
}
@end

// Shared/Player/PlayerState.x: no player state for SGSetPlayerSpeed to start the mod's clock from (the mock has
// no track), so the speed checks see the units alone.
SPTPlayerState *SGPlayerState(void) { return nil; }

static OSStatus decoder(void *refCon, AudioUnitRenderActionFlags *flags, const AudioTimeStamp *timestamp, UInt32 bus,
                        UInt32 frames, AudioBufferList *data) {
    uint64_t start = atomic_fetch_add(&sg_decoded, frames);
    for (UInt32 i = 0; i < frames; i++) {
        float value = 0.05f * sinf(2 * M_PI * 440 * (start + i) / kRate);
        for (UInt32 b = 0; b < data->mNumberBuffers; b++) ((float *)data->mBuffers[b].mData)[i] = value;
    }
    return noErr;
}

static OSStatus chainDecoder(void *refCon, AudioUnitRenderActionFlags *flags, const AudioTimeStamp *timestamp, UInt32 bus,
                             UInt32 frames, AudioBufferList *data) {
    Chain *chain = refCon;
    uint64_t start = atomic_fetch_add(&chain->decoded, frames);
    BOOL mute = atomic_load(&chain->mute);
    for (UInt32 i = 0; i < frames; i++) {
        float value = mute ? 0 : 0.05f * sinf(2 * M_PI * 440 * (start + i) / chain->rate);
        for (UInt32 b = 0; b < data->mNumberBuffers; b++) ((float *)data->mBuffers[b].mData)[i] = value;
    }
    return noErr;
}

static OSStatus counted(void *refCon, AudioUnitRenderActionFlags *flags, const AudioTimeStamp *timestamp, UInt32 bus,
                        UInt32 frames, AudioBufferList *data) {
    Chain *chain = refCon;
    if ((*flags & kAudioUnitRenderAction_PostRender) && bus == 0) atomic_fetch_add(&chain->played, frames);
    return noErr;
}

// Voice search's kind of unit: Spotify's own callback, silence here.
static OSStatus silentInput(void *refCon, AudioUnitRenderActionFlags *flags, const AudioTimeStamp *timestamp, UInt32 bus,
                            UInt32 frames, AudioBufferList *data) {
    for (UInt32 b = 0; b < data->mNumberBuffers; b++) memset(data->mBuffers[b].mData, 0, data->mBuffers[b].mDataByteSize);
    *flags |= kAudioUnitRenderAction_OutputIsSilence;
    return noErr;
}

static AudioUnit make(OSType type, OSType subType) {
    AudioComponentDescription description = {type, subType, kAudioUnitManufacturer_Apple, 0, 0};
    AudioUnit unit = NULL;
    AudioComponentInstanceNew(AudioComponentFindNext(NULL, &description), &unit);
    return unit;
}

static void check(OSStatus status, const char *what) {
    if (status) NSLog(@"[harness] %s failed: %d", what, (int)status);
}

@interface SGRHarnessDelegate : UIResponder <UIApplicationDelegate>
@property (nonatomic, strong) UIWindow *window;
@end

@implementation SGRHarnessDelegate {
    SPTPlayerState *_state;
    AudioUnit _output;
    uint64_t _lastDecoded;
    NSTimeInterval _lastAt, _startedAt;
}

- (void)startChain {
    [AVAudioSession.sharedInstance setCategory:AVAudioSessionCategoryPlayback error:nil];
    [AVAudioSession.sharedInstance setActive:YES error:nil];
    AudioUnit converter = make(kAudioUnitType_FormatConverter, kAudioUnitSubType_AUConverter);
    AudioUnit mixer = make(kAudioUnitType_Mixer, kAudioUnitSubType_MultiChannelMixer);
    AudioUnit output = make(kAudioUnitType_Output, kAudioUnitSubType_RemoteIO);
    AURenderCallbackStruct callback = {decoder, NULL};
    check(AudioUnitSetProperty(converter, kAudioUnitProperty_SetRenderCallback, kAudioUnitScope_Input, 0, &callback, sizeof callback), "callback");
    AudioUnitConnection toMixer = {converter, 0, 0}, toOutput = {mixer, 0, 0};
    check(AudioUnitSetProperty(mixer, kAudioUnitProperty_MakeConnection, kAudioUnitScope_Input, 0, &toMixer, sizeof toMixer), "connect mixer");
    check(AudioUnitSetProperty(output, kAudioUnitProperty_MakeConnection, kAudioUnitScope_Input, 0, &toOutput, sizeof toOutput), "connect output");
    AudioStreamBasicDescription format = {kRate, kAudioFormatLinearPCM, kAudioFormatFlagsNativeFloatPacked | kAudioFormatFlagIsNonInterleaved, 4, 1, 4, 2, 32, 0};
    check(AudioUnitSetProperty(converter, kAudioUnitProperty_StreamFormat, kAudioUnitScope_Input, 0, &format, sizeof format), "converter in");
    check(AudioUnitSetProperty(converter, kAudioUnitProperty_StreamFormat, kAudioUnitScope_Output, 0, &format, sizeof format), "converter out");
    check(AudioUnitSetProperty(mixer, kAudioUnitProperty_StreamFormat, kAudioUnitScope_Input, 0, &format, sizeof format), "mixer in");
    check(AudioUnitSetProperty(mixer, kAudioUnitProperty_StreamFormat, kAudioUnitScope_Output, 0, &format, sizeof format), "mixer out");
    check(AudioUnitSetProperty(output, kAudioUnitProperty_StreamFormat, kAudioUnitScope_Input, 0, &format, sizeof format), "output in");
    UInt32 slice = 4096;
    AudioUnit units[] = {converter, mixer, output};
    for (int i = 0; i < 3; i++) check(AudioUnitSetProperty(units[i], kAudioUnitProperty_MaximumFramesPerSlice, kAudioUnitScope_Global, 0, &slice, sizeof slice), "slice");
    for (int i = 0; i < 3; i++) check(AudioUnitInitialize(units[i]), "initialize");
    check(AudioOutputUnitStart(output), "start");
    check(AudioUnitAddRenderNotify(output, metered, NULL), "meter");
    _output = output;
}

// Spotify's side of the output at another rate while it runs, the way a file at another rate may bring one.
- (void)changeRate {
    AudioStreamBasicDescription format = {48000, kAudioFormatLinearPCM, kAudioFormatFlagsNativeFloatPacked | kAudioFormatFlagIsNonInterleaved, 4, 1, 4, 2, 32, 0};
    OSStatus status = AudioUnitSetProperty(_output, kAudioUnitProperty_StreamFormat, kAudioUnitScope_Input, 0, &format, sizeof format);
    NSLog(@"[harness] output's input set to 48000 Hz while running: status %d", (int)status);
}

// Spotify's chain at `rate`, wired and started the way AudioUnitDriver2 does it.
static void startChain(Chain *chain) {
    chain->converter = make(kAudioUnitType_FormatConverter, kAudioUnitSubType_AUConverter);
    chain->mixer = make(kAudioUnitType_Mixer, kAudioUnitSubType_MultiChannelMixer);
    chain->output = make(kAudioUnitType_Output, kAudioUnitSubType_RemoteIO);
    AURenderCallbackStruct callback = {chainDecoder, chain};
    check(AudioUnitSetProperty(chain->converter, kAudioUnitProperty_SetRenderCallback, kAudioUnitScope_Input, 0, &callback, sizeof callback), "callback");
    AudioStreamBasicDescription format = {chain->rate, kAudioFormatLinearPCM, kAudioFormatFlagsNativeFloatPacked | kAudioFormatFlagIsNonInterleaved, 4, 1, 4, 2, 32, 0};
    AudioUnit units[] = {chain->converter, chain->mixer, chain->output};
    for (int i = 0; i < 3; i++) {
        if (i < 2) check(AudioUnitSetProperty(units[i], kAudioUnitProperty_StreamFormat, kAudioUnitScope_Input, 0, &format, sizeof format), "format in");
        if (i < 2) check(AudioUnitSetProperty(units[i], kAudioUnitProperty_StreamFormat, kAudioUnitScope_Output, 0, &format, sizeof format), "format out");
    }
    check(AudioUnitSetProperty(chain->output, kAudioUnitProperty_StreamFormat, kAudioUnitScope_Input, 0, &format, sizeof format), "output in");
    AudioUnitConnection toMixer = {chain->converter, 0, 0}, toOutput = {chain->mixer, 0, 0};
    check(AudioUnitSetProperty(chain->mixer, kAudioUnitProperty_MakeConnection, kAudioUnitScope_Input, 0, &toMixer, sizeof toMixer), "connect mixer");
    check(AudioUnitSetProperty(chain->output, kAudioUnitProperty_MakeConnection, kAudioUnitScope_Input, 0, &toOutput, sizeof toOutput), "connect output");
    UInt32 slice = 4096;
    for (int i = 0; i < 3; i++) check(AudioUnitSetProperty(units[i], kAudioUnitProperty_MaximumFramesPerSlice, kAudioUnitScope_Global, 0, &slice, sizeof slice), "slice");
    for (int i = 0; i < 3; i++) check(AudioUnitInitialize(units[i]), "initialize");
    AudioStreamBasicDescription hardware = {0};
    UInt32 size = sizeof hardware;
    check(AudioUnitGetProperty(chain->output, kAudioUnitProperty_StreamFormat, kAudioUnitScope_Output, 0, &hardware, &size), "hardware format");
    chain->hardwareRate = hardware.mSampleRate;
    check(AudioUnitAddRenderNotify(chain->output, counted, chain), "count");
    check(AudioOutputUnitStart(chain->output), "start");
}

static Chain sg_a = {.rate = 44100}, sg_b = {.rate = 48000};
static int sg_failures;

// Each chain's decoder drained since the last call, in seconds of its own rate for each second its output
// played (the simulator's output stalls now and then, so the wall clock is no measure), against what is
// expected; 0 for a chain that should not be drained at all, or whose output does not play.
static void expect(NSString *what, double wantA, double wantB) {
    Chain *chains[] = {&sg_a, &sg_b};
    double want[] = {wantA, wantB}, got[2];
    for (int i = 0; i < 2; i++) {
        uint64_t decoded = atomic_load(&chains[i]->decoded), played = atomic_load(&chains[i]->played);
        double playedSeconds = chains[i]->hardwareRate > 0 ? (played - chains[i]->lastPlayed) / chains[i]->hardwareRate : 0;
        got[i] = playedSeconds > 0.5 ? (decoded - chains[i]->lastDecoded) / chains[i]->rate / playedSeconds : 0;
        chains[i]->lastDecoded = decoded;
        chains[i]->lastPlayed = played;
    }
    if (wantA < 0) return;
    BOOL pass = fabs(got[0] - wantA) < 0.06 && fabs(got[1] - wantB) < 0.06;
    if (!pass) sg_failures++;
    AudioUnit musicUnit = SGPlayerMusicOutput();
    NSLog(@"[harness] %@ %-44@ 44.1 kHz chain drained %.2fx (want %.2f), 48 kHz chain %.2fx (want %.2f); the processors on %@", pass ? @"PASS" : @"FAIL",
          what, got[0], want[0], got[1], want[1], musicUnit == sg_a.output ? @"44.1" : musicUnit == sg_b.output ? @"48" : @"neither");
}

- (void)report:(NSString *)what {
    NSTimeInterval now = CACurrentMediaTime();
    uint64_t decoded = atomic_load(&sg_decoded);
    double content = decoded / kRate;
    uint32_t bits = atomic_load(&sg_peakBits);
    float peak;
    memcpy(&peak, &bits, sizeof peak);
    NSLog(@"[harness] %-28@ decoder drained %.2fx over the last %.1f s; content %.2f s, state position %.2f s (off %+.0f ms), state speed %.2f, peak %.4f",
          what, (decoded - _lastDecoded) / kRate / (now - _lastAt), now - _lastAt, content, _state.position,
          (_state.position - content) * 1000, [_state playbackSpeed], peak);
    _lastDecoded = decoded;
    _lastAt = now;
}

// The player reporting: a state whose position is what the decoder has handed over, as of now.
- (void)playerReports {
    _state = [SPTPlayerState new];
    _state.positionAsOfTimestamp = atomic_load(&sg_decoded) / kRate;
    _state.timestamp = [NSDate date];
}

- (void)after:(double)seconds do:(void (^)(void))block {
    dispatch_after(dispatch_time(DISPATCH_TIME_NOW, (int64_t)(seconds * NSEC_PER_SEC)), dispatch_get_main_queue(), block);
}

- (BOOL)application:(UIApplication *)application didFinishLaunchingWithOptions:(NSDictionary *)options {
    self.window = [[UIWindow alloc] initWithFrame:UIScreen.mainScreen.bounds];
    self.window.rootViewController = [UIViewController new];
    [self.window makeKeyAndVisible];
    if ([NSProcessInfo.processInfo.arguments containsObject:@"two"]) {
        [self runTwo];
        return YES;
    }
    [self startChain];
    NSLog(@"[harness] speed allowed: %d", SGPlayerSpeedAllowed());
    [self after:1 do:^{ [self playerReports]; self->_lastDecoded = atomic_load(&sg_decoded); self->_lastAt = CACurrentMediaTime(); }];
    NSArray *script = @[
        @[@3, @"normal", @1, @0],
        @[@6, @"1.5x", @1.5, @0],
        @[@9, @"1.5x, +3 st", @1.5, @3],
        @[@12, @"0.75x", @0.75, @3],
        @[@15, @"back to normal", @1, @0],
        @[@18, @"normal, unit out", @1, @0],
        @[@21, @"gain 0.25", @1, @0, @0.25],
        @[@24, @"gain back to 1", @1, @0, @1],
    ];
    __block NSString *label = @"normal";
    for (NSArray *step in script) {
        [self after:[step[0] doubleValue] do:^{
            [self report:label];
            label = step[1];
            SGSetPlayerSpeed([step[2] doubleValue]);
            SGSetPlayerPitch([step[3] floatValue]);
            if (step.count > 4) SGPlayerSetGain([step[4] floatValue]);
        }];
        // The peak is read over the step's last second, past the gain's half second ramp.
        [self after:[step[0] doubleValue] + 2 do:^{ atomic_store(&sg_peakBits, 0); }];
        // A report mid step, the way Spotify's player reports now and then.
        [self after:[step[0] doubleValue] + 1.5 do:^{ [self playerReports]; }];
    }
    if ([NSProcessInfo.processInfo.arguments containsObject:@"rate"]) [self after:7.5 do:^{ [self changeRate]; }];
    [self after:27 do:^{
        [self report:label];
        exit(0);
    }];
    return YES;
}

// Two chains at once. Every step is measured over 2 s, from half a second after the change it makes; a silent
// output gives the processors up within 1.5 s, so that step waits 2 s first.
- (void)runTwo {
    [AVAudioSession.sharedInstance setCategory:AVAudioSessionCategoryPlayback error:nil];
    [AVAudioSession.sharedInstance setActive:YES error:nil];
    startChain(&sg_a);
    __block double at = 0.5;
    __block double settle = 0.5;
    void (^step)(NSString *, double, double, void (^)(void)) = ^(NSString *what, double wantA, double wantB, void (^change)(void)) {
        [self after:at do:^{
            if (change) change();
        }];
        [self after:at + settle do:^{
            expect(@"(settling)", -1, -1);
        }];
        [self after:at + settle + 2 do:^{
            expect(what, wantA, wantB);
        }];
        at += settle + 2;
        settle = 0.5;
    };
    // The second chain is the music's once started; the first plays on at its own rate.
    step(@"one chain, normal", 1, 0, nil);
    step(@"48 kHz chain started too, normal", 1, 1, ^{ startChain(&sg_b); });
    step(@"1.5x: only the 48 kHz chain", 1, 1.5, ^{ SGSetPlayerSpeed(1.5); });
    step(@"48 kHz stopped: 1.5x moves to 44.1", 1.5, 0, ^{ check(AudioOutputUnitStop(sg_b.output), "stop"); });
    step(@"48 kHz started again: back to it", 1, 1.5, ^{ check(AudioOutputUnitStart(sg_b.output), "start"); });
    // A unit Spotify feeds through a render callback of its own (voice search) never takes the processors from a
    // connected one.
    step(@"a callback-fed unit started: 1.5x stays", 1, 1.5, ^{
        AudioUnit voice = make(kAudioUnitType_Output, kAudioUnitSubType_RemoteIO);
        AURenderCallbackStruct callback = {silentInput, NULL};
        check(AudioUnitSetProperty(voice, kAudioUnitProperty_SetRenderCallback, kAudioUnitScope_Input, 0, &callback, sizeof callback), "voice callback");
        check(AudioUnitInitialize(voice), "voice initialize");
        check(AudioOutputUnitStart(voice), "voice start");
    });
    settle = 2;
    step(@"48 kHz silent for a while: 1.5x to 44.1", 1.5, 1, ^{ atomic_store(&sg_b.mute, true); });
    step(@"48 kHz sound again: stays on 44.1", 1.5, 1, ^{ atomic_store(&sg_b.mute, false); });
    step(@"48 kHz mixer disposed: 44.1 unchanged", 1.5, 0, ^{ check(AudioComponentInstanceDispose(sg_b.mixer), "dispose mixer"); });
    step(@"44.1 stopped, 48 output disposed", 0, 0, ^{
        check(AudioOutputUnitStop(sg_a.output), "stop");
        check(AudioComponentInstanceDispose(sg_b.output), "dispose output");
    });
    settle = 1;   // the simulator's IO takes a moment to come back once nothing runs
    step(@"44.1 started again, still 1.5x", 1.5, 0, ^{ check(AudioOutputUnitStart(sg_a.output), "start"); });
    step(@"back to normal", 1, 0, ^{ SGSetPlayerSpeed(1); });
    [self after:at + 0.5 do:^{
        NSLog(@"[harness] two chains: %d failures", sg_failures);
        exit(sg_failures ? 1 : 0);
    }];
}

@end

__attribute__((constructor(101))) static void sgr_harnessDefaults(void) {
    [NSUserDefaults.standardUserDefaults setBool:YES forKey:@"spotifyglass.redesign"];
}

int main(int argc, char *argv[]) {
    @autoreleasepool {
        return UIApplicationMain(argc, argv, nil, NSStringFromClass(SGRHarnessDelegate.class));
    }
}