Liquid Glass and Apple Music touches for the Spotify iOS app. A GPL-3.0 fork of spoti.pw 0.21.1.
C
// Spotify's audio chain rebuilt with real Core Audio units in the simulator (AudioUnitDriver2: a converter fed
// by a render callback, a mixer and RemoteIO, wired with MakeConnection, slices of 4096), with AudioEffects.x,
// its settings, its libraries and the engine compiled in. The harness is the main executable, so its
// AudioOutputUnitStart goes through the rebound import slot exactly as Spotify's does. A second render
// notify, added after the output started and so after the effects', reads what they left in the buffer, and
// a script of settings checks the level it measures.
//
// The generator plays a 440 Hz sine at -12 dBFS peak in the left channel and nothing in the right, so a
// channel swap shows as well as a gain.
//
// THEOS=$HOME/theos ../build-sim.sh
// xcrun simctl install <udid> build/sim/AudioEffectsHarness.app
// xcrun simctl launch --console-pty <udid> com.vojta.audioeffectsharness
#import <UIKit/UIKit.h>
#import <AudioToolbox/AudioToolbox.h>
#import <AVFoundation/AVFoundation.h>
#import <stdatomic.h>
#import "Shared/AudioEffects/AudioEffects.h"
static const float kAmplitude = 0.25f;
// SpeedPitch.x hooks Spotify's player state; this stands in for it.
@interface SPTPlayerState : NSObject
@end
@implementation SPTPlayerState
- (double)playbackSpeed {
return 1;
}
@end
// Shared/Player/PlayerState.x: no state for SGSetPlayerSpeed to start the mod's clock from.
SPTPlayerState *SGPlayerState(void) { return nil; }
static atomic_bool sg_mute;
static atomic_uint_fast64_t sg_phase;
static double sg_generatorRate;
static OSStatus generate(void *refCon, AudioUnitRenderActionFlags *flags, const AudioTimeStamp *timestamp, UInt32 bus,
UInt32 frames, AudioBufferList *data) {
uint64_t start = atomic_fetch_add(&sg_phase, frames);
BOOL mute = atomic_load(&sg_mute);
float *left = data->mBuffers[0].mData, *right = data->mNumberBuffers > 1 ? data->mBuffers[1].mData : NULL;
for (UInt32 i = 0; i < frames; i++) {
left[i] = mute ? 0 : kAmplitude * sinf(2 * M_PI * 440 * (start + i) / sg_generatorRate);
if (right) right[i] = 0;
}
if (mute) *flags |= kAudioUnitRenderAction_OutputIsSilence;
return noErr;
}
#pragma mark - the probe, after the effects' notify
enum { kProbeCalls = 64 };
typedef struct { float left, right; UInt32 frames; bool silent; } ProbeCall;
// One ring per output: [0] the chain the script starts with, [1] the second chain run beside it.
static ProbeCall sg_calls[2][kProbeCalls];
static atomic_uint sg_callCount[2];
static OSStatus probe(void *refCon, AudioUnitRenderActionFlags *flags, const AudioTimeStamp *timestamp, UInt32 bus,
UInt32 frames, AudioBufferList *data) {
if (!(*flags & kAudioUnitRenderAction_PostRender) || bus != 0 || !frames) return noErr;
// The notify's buffers are in the unit's output (hardware) format, float with a buffer per channel here,
// whatever the client format.
if (data->mNumberBuffers < 2 || data->mBuffers[0].mDataByteSize != frames * sizeof(float)) return noErr;
const float *l = data->mBuffers[0].mData, *r = data->mBuffers[1].mData;
double left = 0, right = 0;
for (UInt32 i = 0; i < frames; i++) {
left += l[i] * l[i];
right += r[i] * r[i];
}
int which = (int)(intptr_t)refCon;
unsigned n = atomic_load(&sg_callCount[which]);
sg_calls[which][n % kProbeCalls] = (ProbeCall){(float)(left / frames), (float)(right / frames), frames, (*flags & kAudioUnitRenderAction_OutputIsSilence) != 0};
atomic_store(&sg_callCount[which], n + 1);
return noErr;
}
// The level of output `which`'s last `calls` renders, in dB RMS, per channel.
static void levelOf(int which, unsigned calls, double *left, double *right, unsigned *frames, unsigned *silentCalls) {
unsigned n = atomic_load(&sg_callCount[which]);
double l = 0, r = 0;
unsigned count = 0;
*frames = 0;
*silentCalls = 0;
for (unsigned i = 0; i < calls && i < n && i < kProbeCalls; i++) {
ProbeCall call = sg_calls[which][(n - 1 - i) % kProbeCalls];
l += call.left;
r += call.right;
*frames = call.frames;
*silentCalls += call.silent;
count++;
}
*left = count ? 10 * log10(l / count + 1e-20) : -200;
*right = count ? 10 * log10(r / count + 1e-20) : -200;
}
static void level(unsigned calls, double *left, double *right, unsigned *frames, unsigned *silentCalls) {
levelOf(0, calls, left, right, frames, silentCalls);
}
static int sg_failures;
static void reportReader(double wantLeft, double wantRight);
static void reportOutput(int which, NSString *what, double wantLeft, double wantRight) {
double left, right;
unsigned frames, silent;
levelOf(which, 8, &left, &right, &frames, &silent);
BOOL ok = (isnan(wantLeft) || fabs(left - wantLeft) < 0.6) && (isnan(wantRight) || fabs(right - wantRight) < 0.6);
if (!ok) sg_failures++;
NSLog(@"[harness] %@ %-44@ left %6.1f dB, right %6.1f dB (want %@, %@), slices of %u, %u silent; status \"%@\"", ok ? @" ok " : @"FAILED",
what, left, right, isnan(wantLeft) ? @"any" : [NSString stringWithFormat:@"%.1f", wantLeft],
isnan(wantRight) ? @"any" : [NSString stringWithFormat:@"%.1f", wantRight], frames, silent, SGDSPStatus());
reportReader(wantLeft, wantRight);
}
#pragma mark - the output's reader (SGAudioSetOutputReader)
// What AudioEffects.x hands its one reader: the buffer after the effects, mixed to mono. The sine is in one
// channel only, so the mix is it at half, 6 dB down.
static float sg_readerLevels[kProbeCalls];
static atomic_uint sg_readerCalls;
static atomic_uint_fast64_t sg_readerRate;
static void reader(const float *samples, uint32_t count, double rate) {
double sum = 0;
for (uint32_t i = 0; i < count; i++) sum += samples[i] * samples[i];
unsigned n = atomic_load(&sg_readerCalls);
sg_readerLevels[n % kProbeCalls] = count ? (float)(sum / count) : 0;
atomic_store(&sg_readerCalls, n + 1);
atomic_store(&sg_readerRate, (uint64_t)rate);
}
static void reportReader(double wantLeft, double wantRight) {
if (isnan(wantLeft) || isnan(wantRight)) return;
unsigned n = atomic_load(&sg_readerCalls), count = 0;
double sum = 0;
for (unsigned i = 0; i < 8 && i < n; i++, count++) sum += sg_readerLevels[(n - 1 - i) % kProbeCalls];
double got = count ? 10 * log10(sum / count + 1e-20) : -200, want = MAX(wantLeft, wantRight) - 6.02;
BOOL ok = count && (want < -150 ? got < -150 : fabs(got - want) < 0.6);
if (!ok) sg_failures++;
NSLog(@"[harness] %@ the reader: %6.1f dB (want %.1f) at %llu Hz, %u calls", ok ? @" ok " : @"FAILED", got, want,
(unsigned long long)atomic_load(&sg_readerRate), n);
}
static void report(NSString *what, double wantLeft, double wantRight) {
reportOutput(0, what, wantLeft, wantRight);
}
#pragma mark - the chain
static AudioUnit make(OSType type, OSType subType) {
AudioComponentDescription description = {type, subType, kAudioUnitManufacturer_Apple, 0, 0};
AudioUnit unit = NULL;
AudioComponentInstanceNew(AudioComponentFindNext(NULL, &description), &unit);
return unit;
}
static void check(OSStatus status, const char *what) {
if (status) NSLog(@"[harness] %s failed: %d", what, (int)status);
}
@interface SGDSPHarnessDelegate : UIResponder <UIApplicationDelegate>
@property (nonatomic, strong) UIWindow *window;
@end
@implementation SGDSPHarnessDelegate {
AudioUnit _converter, _mixer, _output;
AudioUnit _secondConverter, _secondMixer, _secondOutput;
}
// Spotify's way: float, one buffer per channel, converter -> mixer -> RemoteIO, slices of 4096.
- (void)startSpotifyChainAt:(double)rate {
sg_generatorRate = rate;
_converter = make(kAudioUnitType_FormatConverter, kAudioUnitSubType_AUConverter);
_mixer = make(kAudioUnitType_Mixer, kAudioUnitSubType_MultiChannelMixer);
_output = make(kAudioUnitType_Output, kAudioUnitSubType_RemoteIO);
AURenderCallbackStruct callback = {generate, NULL};
check(AudioUnitSetProperty(_converter, kAudioUnitProperty_SetRenderCallback, kAudioUnitScope_Input, 0, &callback, sizeof callback), "callback");
AudioUnitConnection toMixer = {_converter, 0, 0}, toOutput = {_mixer, 0, 0};
check(AudioUnitSetProperty(_mixer, kAudioUnitProperty_MakeConnection, kAudioUnitScope_Input, 0, &toMixer, sizeof toMixer), "connect mixer");
check(AudioUnitSetProperty(_output, kAudioUnitProperty_MakeConnection, kAudioUnitScope_Input, 0, &toOutput, sizeof toOutput), "connect output");
AudioStreamBasicDescription format = {rate, kAudioFormatLinearPCM, kAudioFormatFlagsNativeFloatPacked | kAudioFormatFlagIsNonInterleaved, 4, 1, 4, 2, 32, 0};
check(AudioUnitSetProperty(_converter, kAudioUnitProperty_StreamFormat, kAudioUnitScope_Input, 0, &format, sizeof format), "converter in");
check(AudioUnitSetProperty(_converter, kAudioUnitProperty_StreamFormat, kAudioUnitScope_Output, 0, &format, sizeof format), "converter out");
check(AudioUnitSetProperty(_mixer, kAudioUnitProperty_StreamFormat, kAudioUnitScope_Input, 0, &format, sizeof format), "mixer in");
check(AudioUnitSetProperty(_mixer, kAudioUnitProperty_StreamFormat, kAudioUnitScope_Output, 0, &format, sizeof format), "mixer out");
check(AudioUnitSetProperty(_output, kAudioUnitProperty_StreamFormat, kAudioUnitScope_Input, 0, &format, sizeof format), "output in");
UInt32 slice = 4096;
AudioUnit units[] = {_converter, _mixer, _output};
for (int i = 0; i < 3; i++) check(AudioUnitSetProperty(units[i], kAudioUnitProperty_MaximumFramesPerSlice, kAudioUnitScope_Global, 0, &slice, sizeof slice), "slice");
for (int i = 0; i < 3; i++) check(AudioUnitInitialize(units[i]), "initialize");
check(AudioOutputUnitStart(_output), "start");
check(AudioUnitAddRenderNotify(_output, probe, (void *)0), "probe");
[self logScopes];
}
// A second chain, Spotify's way at `rate`, started beside the first, the way Spotify runs a chain per sample
// rate; its own probe is output 1's.
- (void)startSecondChainAt:(double)rate {
_secondConverter = make(kAudioUnitType_FormatConverter, kAudioUnitSubType_AUConverter);
_secondMixer = make(kAudioUnitType_Mixer, kAudioUnitSubType_MultiChannelMixer);
_secondOutput = make(kAudioUnitType_Output, kAudioUnitSubType_RemoteIO);
AURenderCallbackStruct callback = {generate, NULL};
check(AudioUnitSetProperty(_secondConverter, kAudioUnitProperty_SetRenderCallback, kAudioUnitScope_Input, 0, &callback, sizeof callback), "callback");
AudioStreamBasicDescription format = {rate, kAudioFormatLinearPCM, kAudioFormatFlagsNativeFloatPacked | kAudioFormatFlagIsNonInterleaved, 4, 1, 4, 2, 32, 0};
AudioUnit units[] = {_secondConverter, _secondMixer, _secondOutput};
for (int i = 0; i < 3; i++) {
if (i < 2) check(AudioUnitSetProperty(units[i], kAudioUnitProperty_StreamFormat, kAudioUnitScope_Output, 0, &format, sizeof format), "format out");
check(AudioUnitSetProperty(units[i], kAudioUnitProperty_StreamFormat, kAudioUnitScope_Input, 0, &format, sizeof format), "format in");
}
AudioUnitConnection toMixer = {_secondConverter, 0, 0}, toOutput = {_secondMixer, 0, 0};
check(AudioUnitSetProperty(_secondMixer, kAudioUnitProperty_MakeConnection, kAudioUnitScope_Input, 0, &toMixer, sizeof toMixer), "connect mixer");
check(AudioUnitSetProperty(_secondOutput, kAudioUnitProperty_MakeConnection, kAudioUnitScope_Input, 0, &toOutput, sizeof toOutput), "connect output");
UInt32 slice = 4096;
for (int i = 0; i < 3; i++) check(AudioUnitSetProperty(units[i], kAudioUnitProperty_MaximumFramesPerSlice, kAudioUnitScope_Global, 0, &slice, sizeof slice), "slice");
for (int i = 0; i < 3; i++) check(AudioUnitInitialize(units[i]), "initialize");
check(AudioOutputUnitStart(_secondOutput), "start");
check(AudioUnitAddRenderNotify(_secondOutput, probe, (void *)1), "probe");
}
// What the client format is and what the notify gets: the output scope, the hardware's.
- (void)logScopes {
AudioStreamBasicDescription client = {0}, hardware = {0};
UInt32 size = sizeof client;
AudioUnitGetProperty(_output, kAudioUnitProperty_StreamFormat, kAudioUnitScope_Input, 0, &client, &size);
size = sizeof hardware;
AudioUnitGetProperty(_output, kAudioUnitProperty_StreamFormat, kAudioUnitScope_Output, 0, &hardware, &size);
NSLog(@"[harness] RemoteIO's client format %.0f Hz %u-bit flags 0x%x; its output side, which the notifies get, %.0f Hz %u-bit flags 0x%x",
client.mSampleRate, (unsigned)client.mBitsPerChannel, (unsigned)client.mFormatFlags, hardware.mSampleRate,
(unsigned)hardware.mBitsPerChannel, (unsigned)hardware.mFormatFlags);
}
// Another client format: 16-bit interleaved at another rate, converter straight into RemoteIO. The notify
// still gets the output side's float, now with the engine following the rate change.
- (void)startInt16ChainAt:(double)rate {
AudioOutputUnitStop(_output);
AudioUnitRemoveRenderNotify(_output, probe, (void *)0);
AudioUnit old[] = {_output, _mixer, _converter};
for (int i = 0; i < 3; i++) {
if (!old[i]) continue;
AudioUnitUninitialize(old[i]);
AudioComponentInstanceDispose(old[i]);
}
sg_generatorRate = rate;
_converter = make(kAudioUnitType_FormatConverter, kAudioUnitSubType_AUConverter);
_output = make(kAudioUnitType_Output, kAudioUnitSubType_RemoteIO);
_mixer = NULL;
AURenderCallbackStruct callback = {generate, NULL};
check(AudioUnitSetProperty(_converter, kAudioUnitProperty_SetRenderCallback, kAudioUnitScope_Input, 0, &callback, sizeof callback), "callback");
AudioStreamBasicDescription floats = {rate, kAudioFormatLinearPCM, kAudioFormatFlagsNativeFloatPacked | kAudioFormatFlagIsNonInterleaved, 4, 1, 4, 2, 32, 0};
AudioStreamBasicDescription shorts = {rate, kAudioFormatLinearPCM, kAudioFormatFlagIsSignedInteger | kAudioFormatFlagIsPacked, 4, 1, 4, 2, 16, 0};
check(AudioUnitSetProperty(_converter, kAudioUnitProperty_StreamFormat, kAudioUnitScope_Input, 0, &floats, sizeof floats), "converter in");
check(AudioUnitSetProperty(_converter, kAudioUnitProperty_StreamFormat, kAudioUnitScope_Output, 0, &shorts, sizeof shorts), "converter out");
check(AudioUnitSetProperty(_output, kAudioUnitProperty_StreamFormat, kAudioUnitScope_Input, 0, &shorts, sizeof shorts), "output in");
AudioUnitConnection toOutput = {_converter, 0, 0};
check(AudioUnitSetProperty(_output, kAudioUnitProperty_MakeConnection, kAudioUnitScope_Input, 0, &toOutput, sizeof toOutput), "connect output");
check(AudioUnitInitialize(_converter), "initialize converter");
check(AudioUnitInitialize(_output), "initialize output");
check(AudioOutputUnitStart(_output), "start");
check(AudioUnitAddRenderNotify(_output, probe, (void *)0), "probe");
[self logScopes];
}
- (void)after:(double)seconds do:(void (^)(void))block {
dispatch_after(dispatch_time(DISPATCH_TIME_NOW, (int64_t)(seconds * NSEC_PER_SEC)), dispatch_get_main_queue(), block);
}
- (BOOL)application:(UIApplication *)application didFinishLaunchingWithOptions:(NSDictionary *)options {
self.window = [[UIWindow alloc] initWithFrame:UIScreen.mainScreen.bounds];
self.window.rootViewController = [UIViewController new];
[self.window makeKeyAndVisible];
[AVAudioSession.sharedInstance setCategory:AVAudioSessionCategoryPlayback error:nil];
[AVAudioSession.sharedInstance setActive:YES error:nil];
double sine = 20 * log10(kAmplitude / M_SQRT2), nothing = -200, any = NAN;
NSLog(@"[harness] the sine's level is %.1f dB RMS in the left channel", sine);
SGDSPSetSwitch(SGKeyDSP, NO);
SGAudioSetOutputReader(reader);
[self startSpotifyChainAt:44100];
NSArray *script = @[
@[@"master off", ^{}, @(sine), @(nothing)],
@[@"master off, post gain -12 dB (bypassed)", ^{ SGDSPSetNumber(SGKeyDSPPostGain, -12); }, @(sine), @(nothing)],
@[@"master on, post gain -12 dB", ^{ SGDSPSetSwitch(SGKeyDSP, YES); }, @(sine - 12), @(nothing)],
@[@"post gain back to 0 dB, every effect off", ^{ SGDSPSetNumber(SGKeyDSPPostGain, 0); }, @(sine), @(nothing)],
@[@"Liveprog swapChannels.eel from the library", ^{
SGDSPSetString(SGKeyDSPLiveprogFile, @"swapChannels.eel");
SGDSPSetSwitch(SGKeyDSPLiveprog, YES);
}, @(nothing), @(sine)],
@[@"Liveprog off, reverb on (plate)", ^{
SGDSPSetSwitch(SGKeyDSPLiveprog, NO);
SGDSPSetSwitch(SGKeyDSPReverb, YES);
}, @(any), @(any)],
@[@"the source goes silent, the reverb rings out", ^{ atomic_store(&sg_mute, true); }, @(any), @(any)],
@[@"reverb off, silent: silence", ^{ SGDSPSetSwitch(SGKeyDSPReverb, NO); }, @(nothing), @(nothing)],
@[@"a convolver file not in the library", ^{
atomic_store(&sg_mute, false);
SGDSPSetString(SGKeyDSPConvolverFile, @"missing.wav");
SGDSPSetSwitch(SGKeyDSPConvolver, YES);
}, @(sine), @(nothing)],
@[@"16-bit interleaved at 48 kHz, post gain -12 dB", ^{
SGDSPSetSwitch(SGKeyDSPConvolver, NO);
SGDSPSetNumber(SGKeyDSPPostGain, -12);
[self startInt16ChainAt:48000];
}, @(sine - 12), @(nothing)],
@[@"16-bit, Liveprog swapChannels.eel", ^{
SGDSPSetNumber(SGKeyDSPPostGain, 0);
SGDSPSetSwitch(SGKeyDSPLiveprog, YES);
}, @(nothing), @(sine)],
// Two outputs at once: the effects follow the one Spotify started last, and the other plays as it is.
@[@"a 44.1 kHz float chain started beside it", ^{ [self startSecondChainAt:44100]; }, @(sine), @(nothing), @(nothing), @(sine)],
@[@"44.1 kHz chain stopped: back on the 16-bit one", ^{
check(AudioOutputUnitStop(self->_secondOutput), "stop");
// The effects' notify went back on after the probe; the probe goes after it again, to read what it left.
AudioUnitRemoveRenderNotify(self->_output, probe, (void *)0);
check(AudioUnitAddRenderNotify(self->_output, probe, (void *)0), "probe");
}, @(nothing), @(sine)],
@[@"master off again", ^{ SGDSPSetSwitch(SGKeyDSP, NO); }, @(sine), @(nothing)],
];
__block NSString *label = nil;
__block double wantLeft = 0, wantRight = 0, wantSecondLeft = NAN, wantSecondRight = NAN;
double step = 2.5, at = 1;
for (NSArray *entry in script) {
[self after:at do:^{
if (label) report(label, wantLeft, wantRight);
if (label && !isnan(wantSecondLeft)) reportOutput(1, [label stringByAppendingString:@" (2nd)"], wantSecondLeft, wantSecondRight);
if ([label hasPrefix:@"the source goes silent"]) {
// The tail: the level over the renders since the source stopped, loud at first, then less.
double left, right;
unsigned frames, silent;
level(3, &left, &right, &frames, &silent);
NSLog(@"[harness] the tail's last renders: left %.1f dB, right %.1f dB, %u of them flagged silent", left, right, silent);
}
if ([label hasPrefix:@"a convolver file"]) {
NSString *error = SGDSPError(SGKeyDSPConvolver);
if (!error) sg_failures++;
NSLog(@"[harness] %@ its error: \"%@\"", error ? @" ok " : @"FAILED", error);
}
label = entry[0];
wantLeft = [entry[2] doubleValue];
wantRight = [entry[3] doubleValue];
wantSecondLeft = entry.count > 5 ? [entry[4] doubleValue] : NAN;
wantSecondRight = entry.count > 5 ? [entry[5] doubleValue] : NAN;
((void (^)(void))entry[1])();
}];
if ([entry[0] hasPrefix:@"the source goes silent"]) {
// Right after the source stops, the reverb's tail is still coming out.
[self after:at + 0.4 do:^{
double left, right;
unsigned frames, silent;
level(2, &left, &right, &frames, &silent);
BOOL ringing = left > -80 || right > -80;
if (!ringing) sg_failures++;
NSLog(@"[harness] %@ 0.4 s after the source stopped: left %.1f dB, right %.1f dB, %u of 2 renders flagged silent after the effects",
ringing ? @" ok " : @"FAILED", left, right, silent);
}];
}
at += step;
}
[self after:at do:^{
report(label, wantLeft, wantRight);
NSLog(@"[harness] %@: %d failed", sg_failures ? @"FAILED" : @"all passed", sg_failures);
exit(sg_failures ? 1 : 0);
}];
return YES;
}
@end
// Every audio effects key back to what it holds until set, and a script of the harness's own in the
// Liveprog library, before anything reads them.
__attribute__((constructor(101))) static void sgr_harnessDefaults(void) {
NSUserDefaults *defaults = NSUserDefaults.standardUserDefaults;
for (NSString *key in defaults.dictionaryRepresentation.allKeys) {
if ([key hasPrefix:@"spotifyglass.dsp"]) [defaults removeObjectForKey:key];
}
NSString *swap = @"desc: swap the channels\n@sample\nt = spl0;\nspl0 = spl1;\nspl1 = t;\n";
[swap writeToFile:[SGDSPLibraryDirectory(SGDSPFileLiveprog) stringByAppendingPathComponent:@"swapChannels.eel"] atomically:YES
encoding:NSUTF8StringEncoding error:nil];
}
int main(int argc, char *argv[]) {
@autoreleasepool {
return UIApplicationMain(argc, argv, nil, NSStringFromClass(SGDSPHarnessDelegate.class));
}
}