Skip to content

Liquid Glass and Apple Music touches for the Spotify iOS app. A GPL-3.0 fork of spoti.pw 0.21.1.

C
28.9 KB
Raw Blame
// Sing in the simulator, end to end: the voice model downloaded by SGSingModel.m from Hugging Face and checked,
// loaded by Sing.x, and Spotify's audio chain rebuilt with real Core Audio units (a converter fed by a render
// callback, a mixer and RemoteIO, wired with MakeConnection, slices of 4096) with SpeedPitch.x and Sing.x
// compiled in. The harness is the main executable, so its AudioUnitSetProperty and AudioOutputUnitStart go
// through the rebound import slots as Spotify's do, and Sing's stage runs in Speed and pitch's chain.
//
// The converter's callback stands in for Spotify's decoder: a voice (macOS's speech, bundled by build-sim.sh)
// over synthetic chords. A mock SPTPlayerState computes -position the way Spotify's does (positionAsOfTimestamp
// messaged and run on from its timestamp, disassembly at 0x1057735ec), reported as Spotify's player reports: when
// playing starts and on a seek, not as it plays. A mock SPTEsperantoPlayer moves the decoder on seekTo:. A render
// notify on the RemoteIO unit records what plays. The script: Sing on at the vocals' 0, play, seek, rest at As sung,
// separate again, the thermal state Serious and back to Fair, a memory warning's stop, a pause and a resume, Sing
// off; each check prints a line, and what played is scored against the chords alone.
//
//     THEOS=$HOME/theos VOICE=speech.aiff ./build-sim.sh
//     xcrun simctl install <udid> build/sim/SingHarness.app
//     xcrun simctl launch --console-pty <udid> com.vojta.singharness
#import <UIKit/UIKit.h>
#import <AudioToolbox/AudioToolbox.h>
#import <AVFoundation/AVFoundation.h>
#import <objc/runtime.h>
#import <stdatomic.h>
#import "Shared/Sing/Sing.h"
#import "Shared/Sing/SGSingLoader.h"

static const double kRate = 44100;
enum { kMixSeconds = 60, kRecordSeconds = 70 };

static int sg_failures;
// The vocals' checks need the model to keep up, which the simulator's Core ML (on the CPU) does not: there they
// are reported as skipped, with the time a window took.
static BOOL sg_modelKeepsUp = YES;
#define VOCALS_CHECK(ok, ...) do { if (sg_modelKeepsUp) CHECK(ok, __VA_ARGS__); else NSLog(@"[harness] skipped %@ (Sing is %@: the model is slower than the song here)", [NSString stringWithFormat:__VA_ARGS__], SGSingStatusText()); } while (0)
#define CHECK(ok, ...) do { bool _ok = (ok); if (!_ok) sg_failures++; NSLog(@"[harness] %@ %@", _ok ? @"  ok  " : @"FAILED", [NSString stringWithFormat:__VA_ARGS__]); } while (0)

#pragma mark - Spotify's player, as far as Sing reads it

// Shared/Player/PlayerState.x's: the harness's last report, told to Sing's track watcher when the script says.
NSString *SGURIString(id uri) {
    return [uri isKindOfClass:NSString.class] ? uri : nil;
}

@protocol SGHarnessWatcher
- (void)playerStateDidChange:(id)state;
@end

static id sg_watcher, sg_reported, sg_player;

// Which clock the mock player keeps (launch argument -clock): Spotify's last report run on by the time, read as it is
// (report); that report stamped again at every read of the player's state, the same line of the clock (restamp); or
// reported again every half second with what the decoder has handed over (decoded).
typedef NS_ENUM(NSInteger, SGHarnessClock) { SGHarnessClockReport, SGHarnessClockRestamp, SGHarnessClockDecoded };
static SGHarnessClock sg_clock;

void SGAddPlayerStateObserver(id observer) {
    sg_watcher = observer;
}

id SGPlayerState(void) {
    return sg_reported;
}

// Shared/Lyrics/KaraokeSource.x's: the player the lyrics read and seek, the mock below, sent the line's time in seconds.
@class SPTPlayerState;
@interface SPTEsperantoPlayer : NSObject
- (id)seekTo:(double)position;
- (SPTPlayerState *)state;
@end

id SGKaraokePlayer(void) {
    return sg_player;
}

void SGKaraokeSeek(NSInteger ms) {
    [sg_player seekTo:ms / 1000.0];
}

// Shared/HeadGestures' head motion, for spatial voice: none in the simulator, so the voice stays ahead.
void SGHeadMotionListen(NSString *name, void (^handler)(id motion)) {}
void SGHeadMotionAskPermission(void) {}
BOOL SGHeadGesturesAvailable(void) {
    return NO;
}

@interface SPTPlayerState : NSObject
@property (nonatomic) double positionAsOfTimestamp;
@property (nonatomic) double spotifyAsOf;   // the same, out of the hook's reach, for the mock to stamp again
@property (nonatomic, strong) NSDate *timestamp;
@property (nonatomic) BOOL isPaused, isPlaying;
@property (nonatomic) double duration;
@property (nonatomic, strong) id track;
@end

@implementation SPTPlayerState
- (double)playbackSpeed {
    return 1;
}
// As Spotify's: -1 kept, otherwise run on by the time since, never below 0.
- (double)position {
    double asOf = [self positionAsOfTimestamp];
    if (asOf == -1) return asOf;
    return MAX(0, asOf - self.timestamp.timeIntervalSinceNow * [self playbackSpeed]);
}
@end

// The thermal state Sing reads, set by the script.
static NSProcessInfoThermalState sg_thermal = NSProcessInfoThermalStateNominal;

static NSProcessInfoThermalState thermalState(id self, SEL _cmd) {
    return sg_thermal;
}

static void setThermal(NSProcessInfoThermalState state) {
    sg_thermal = state;
    [NSNotificationCenter.defaultCenter postNotificationName:NSProcessInfoThermalStateDidChangeNotification object:NSProcessInfo.processInfo];
}

#pragma mark - the decoder

static float *sg_left, *sg_right, *sg_chordsLeft, *sg_chordsRight;
static size_t sg_mixFrames;
static atomic_uint_fast64_t sg_decoded;   // frames handed over
static atomic_uint_fast64_t sg_readAt;    // where in the mix the next one comes from

static OSStatus decoder(void *refCon, AudioUnitRenderActionFlags *flags, const AudioTimeStamp *timestamp, UInt32 bus,
                        UInt32 frames, AudioBufferList *data) {
    uint64_t at = atomic_fetch_add(&sg_readAt, frames);
    atomic_fetch_add(&sg_decoded, frames);
    float *left = data->mBuffers[0].mData, *right = data->mNumberBuffers > 1 ? data->mBuffers[1].mData : left;
    for (UInt32 i = 0; i < frames; i++) {
        size_t index = (size_t)((at + i) % sg_mixFrames);
        left[i] = sg_left[index];
        right[i] = sg_right[index];
    }
    return noErr;
}

static id sg_seekResult;

// Each command returns an object of Spotify's, which the hook must hand back.
@implementation SPTEsperantoPlayer
// As Spotify's, into the last seek, which moves the decoder.
- (id)seekTo:(double)position {
    return [self seekTo:position relative:0 options:nil creatorTimestampPositionMs:-1];
}
- (SPTPlayerState *)state {
    SPTPlayerState *reported = sg_reported;
    if (sg_clock != SGHarnessClockRestamp || !reported || reported.isPaused) return reported;
    SPTPlayerState *again = [SPTPlayerState new];
    again.timestamp = [NSDate date];
    again.positionAsOfTimestamp = again.spotifyAsOf = reported.spotifyAsOf - reported.timestamp.timeIntervalSinceNow;
    again.isPlaying = YES;
    return again;
}
- (id)seekTo:(double)position relative:(long long)relative { return sg_seekResult; }
- (id)seekTo:(double)position options:(id)options { return [self seekTo:position]; }
- (id)seekTo:(double)position relative:(long long)relative options:(id)options { return sg_seekResult; }
- (id)seekTo:(double)position relative:(long long)relative options:(id)options creatorTimestampPositionMs:(double)creator {
    atomic_store(&sg_readAt, (uint64_t)(position * kRate));
    return sg_seekResult;
}
- (id)skipToNextTrackWithOptions:(id)options { return sg_seekResult; }
- (id)skipToPreviousTrackWithOptions:(id)options { return sg_seekResult; }
- (id)skipToNextTrackWithOptions:(id)options track:(id)track { return sg_seekResult; }
- (id)skipToPreviousTrackWithOptions:(id)options track:(id)track { return sg_seekResult; }
- (id)stop { return sg_seekResult; }
@end

#pragma mark - what plays

static float *sg_heardLeft, *sg_heardRight;
static size_t sg_heardCapacity;
static atomic_uint_fast64_t sg_heard;
static double sg_hardwareRate;

static OSStatus record(void *refCon, AudioUnitRenderActionFlags *flags, const AudioTimeStamp *timestamp, UInt32 bus,
                       UInt32 frames, AudioBufferList *data) {
    if (!(*flags & kAudioUnitRenderAction_PostRender) || bus != 0 || data->mNumberBuffers < 2) return noErr;
    uint64_t at = atomic_load(&sg_heard);
    for (UInt32 i = 0; i < frames && at + i < sg_heardCapacity; i++) {
        sg_heardLeft[at + i] = ((float *)data->mBuffers[0].mData)[i];
        sg_heardRight[at + i] = ((float *)data->mBuffers[1].mData)[i];
    }
    atomic_store(&sg_heard, at + frames);
    return noErr;
}

#pragma mark - the mix

static void makeMix(void) {
    sg_mixFrames = (size_t)(kMixSeconds * kRate);
    sg_left = calloc(sg_mixFrames, sizeof(float));
    sg_right = calloc(sg_mixFrames, sizeof(float));
    sg_chordsLeft = calloc(sg_mixFrames, sizeof(float));
    sg_chordsRight = calloc(sg_mixFrames, sizeof(float));
    float *voice = calloc(sg_mixFrames, sizeof(float));
    size_t voiceFrames = 0;
    NSURL *url = [NSBundle.mainBundle URLForResource:@"voice" withExtension:@"aiff"];
    ExtAudioFileRef file;
    if (url && !ExtAudioFileOpenURL((__bridge CFURLRef)url, &file)) {
        AudioStreamBasicDescription format = {kRate, kAudioFormatLinearPCM, kAudioFormatFlagsNativeFloatPacked, 4, 1, 4, 1, 32, 0};
        ExtAudioFileSetProperty(file, kExtAudioFileProperty_ClientDataFormat, sizeof format, &format);
        while (voiceFrames < sg_mixFrames) {
            UInt32 frames = (UInt32)MIN((size_t)8192, sg_mixFrames - voiceFrames);
            AudioBufferList list = {1, {{1, frames * 4, voice + voiceFrames}}};
            if (ExtAudioFileRead(file, &frames, &list) || !frames) break;
            voiceFrames += frames;
        }
        ExtAudioFileDispose(file);
    }
    NSLog(@"[harness] the voice is %.1f s long", voiceFrames / kRate);
    static const double roots[] = {130.81, 174.61, 196.00, 110.00}, ratios[] = {1, 1.26, 1.5, 2};
    size_t beat = (size_t)kRate / 2;
    for (size_t i = 0; i < sg_mixFrames; i++) {
        size_t bar = i / (beat * 4), within = i % beat;
        double root = roots[bar % 4], env = exp(-3.0 * within / beat), t = i / kRate, l = 0, r = 0;
        for (int n = 0; n < 4; n++) {
            double f = root * ratios[n], tone = sin(2 * M_PI * f * t) + 0.4 * sin(4 * M_PI * f * t) + 0.2 * sin(6 * M_PI * f * t);
            l += tone * (n % 2 ? 0.6 : 1.0);
            r += tone * (n % 2 ? 1.0 : 0.6);
        }
        sg_chordsLeft[i] = (float)(0.06 * env * l);
        sg_chordsRight[i] = (float)(0.06 * env * r);
        // The voice over and over, a second apart, from a second in.
        size_t from = i >= (size_t)kRate && voiceFrames ? (i - (size_t)kRate) % (voiceFrames + (size_t)kRate) : SIZE_MAX;
        float v = from < voiceFrames ? 0.5f * voice[from] : 0;
        sg_left[i] = sg_chordsLeft[i] + v;
        sg_right[i] = sg_chordsRight[i] + v;
    }
    free(voice);
}

// What played, at the hardware's rate, brought to 44.1 kHz.
static NSArray<NSData *> *heardAt44k(size_t *frames) {
    size_t count = MIN((size_t)atomic_load(&sg_heard), sg_heardCapacity);
    AVAudioFormat *from = [[AVAudioFormat alloc] initWithCommonFormat:AVAudioPCMFormatFloat32 sampleRate:sg_hardwareRate channels:2 interleaved:NO];
    AVAudioFormat *to = [[AVAudioFormat alloc] initWithCommonFormat:AVAudioPCMFormatFloat32 sampleRate:kRate channels:2 interleaved:NO];
    AVAudioPCMBuffer *input = [[AVAudioPCMBuffer alloc] initWithPCMFormat:from frameCapacity:(AVAudioFrameCount)count];
    memcpy(input.floatChannelData[0], sg_heardLeft, count * sizeof(float));
    memcpy(input.floatChannelData[1], sg_heardRight, count * sizeof(float));
    input.frameLength = (AVAudioFrameCount)count;
    AVAudioConverter *converter = [[AVAudioConverter alloc] initFromFormat:from toFormat:to];
    converter.sampleRateConverterQuality = AVAudioQualityMax;
    AVAudioPCMBuffer *output = [[AVAudioPCMBuffer alloc] initWithPCMFormat:to frameCapacity:(AVAudioFrameCount)(count * kRate / sg_hardwareRate + 4096)];
    __block BOOL given = NO;
    [converter convertToBuffer:output error:nil withInputFromBlock:^AVAudioBuffer *(AVAudioPacketCount packets, AVAudioConverterInputStatus *status) {
        if (given) {
            *status = AVAudioConverterInputStatus_EndOfStream;
            return nil;
        }
        given = YES;
        *status = AVAudioConverterInputStatus_HaveData;
        return input;
    }];
    *frames = output.frameLength;
    return @[[NSData dataWithBytes:output.floatChannelData[0] length:output.frameLength * sizeof(float)],
             [NSData dataWithBytes:output.floatChannelData[1] length:output.frameLength * sizeof(float)]];
}

static double snr(const float *truthL, const float *truthR, const float *left, const float *right, size_t from, size_t to) {
    double signal = 0, noise = 0;
    for (size_t i = from; i < to; i++) {
        signal += truthL[i] * truthL[i] + truthR[i] * truthR[i];
        double l = left[i] - truthL[i], r = right[i] - truthR[i];
        noise += l * l + r * r;
    }
    return 10 * log10(signal / fmax(noise, 1e-20));
}

#pragma mark - the app

static AudioUnit make(OSType type, OSType subType) {
    AudioComponentDescription description = {type, subType, kAudioUnitManufacturer_Apple, 0, 0};
    AudioUnit unit = NULL;
    AudioComponentInstanceNew(AudioComponentFindNext(NULL, &description), &unit);
    return unit;
}

static void check(OSStatus status, const char *what) {
    if (status) NSLog(@"[harness] %s failed: %d", what, (int)status);
}

@interface SGHarnessDelegate : UIResponder <UIApplicationDelegate>
@property (nonatomic, strong) UIWindow *window;
@end

@implementation SGHarnessDelegate {
    AudioUnit _output;
    CFTimeInterval _startedAt;
    uint64_t _lastDecoded;
    CFTimeInterval _lastAt;
    SPTPlayerState *_state;
    BOOL _paused;
    CFTimeInterval _tappedAt;
    double _seekLead;
}

- (void)startChain {
    [AVAudioSession.sharedInstance setCategory:AVAudioSessionCategoryPlayback error:nil];
    [AVAudioSession.sharedInstance setActive:YES error:nil];
    AudioUnit converter = make(kAudioUnitType_FormatConverter, kAudioUnitSubType_AUConverter);
    AudioUnit mixer = make(kAudioUnitType_Mixer, kAudioUnitSubType_MultiChannelMixer);
    _output = make(kAudioUnitType_Output, kAudioUnitSubType_RemoteIO);
    AURenderCallbackStruct callback = {decoder, NULL};
    check(AudioUnitSetProperty(converter, kAudioUnitProperty_SetRenderCallback, kAudioUnitScope_Input, 0, &callback, sizeof callback), "callback");
    AudioUnitConnection toMixer = {converter, 0, 0}, toOutput = {mixer, 0, 0};
    check(AudioUnitSetProperty(mixer, kAudioUnitProperty_MakeConnection, kAudioUnitScope_Input, 0, &toMixer, sizeof toMixer), "connect mixer");
    check(AudioUnitSetProperty(_output, kAudioUnitProperty_MakeConnection, kAudioUnitScope_Input, 0, &toOutput, sizeof toOutput), "connect output");
    AudioStreamBasicDescription format = {kRate, kAudioFormatLinearPCM, kAudioFormatFlagsNativeFloatPacked | kAudioFormatFlagIsNonInterleaved, 4, 1, 4, 2, 32, 0};
    check(AudioUnitSetProperty(converter, kAudioUnitProperty_StreamFormat, kAudioUnitScope_Input, 0, &format, sizeof format), "converter in");
    check(AudioUnitSetProperty(converter, kAudioUnitProperty_StreamFormat, kAudioUnitScope_Output, 0, &format, sizeof format), "converter out");
    check(AudioUnitSetProperty(mixer, kAudioUnitProperty_StreamFormat, kAudioUnitScope_Input, 0, &format, sizeof format), "mixer in");
    check(AudioUnitSetProperty(mixer, kAudioUnitProperty_StreamFormat, kAudioUnitScope_Output, 0, &format, sizeof format), "mixer out");
    check(AudioUnitSetProperty(_output, kAudioUnitProperty_StreamFormat, kAudioUnitScope_Input, 0, &format, sizeof format), "output in");
    UInt32 slice = 4096;
    AudioUnit units[] = {converter, mixer, _output};
    for (int i = 0; i < 3; i++) check(AudioUnitSetProperty(units[i], kAudioUnitProperty_MaximumFramesPerSlice, kAudioUnitScope_Global, 0, &slice, sizeof slice), "slice");
    for (int i = 0; i < 3; i++) check(AudioUnitInitialize(units[i]), "initialize");
    AudioStreamBasicDescription hardware = {0};
    UInt32 size = sizeof hardware;
    AudioUnitGetProperty(_output, kAudioUnitProperty_StreamFormat, kAudioUnitScope_Output, 0, &hardware, &size);
    sg_hardwareRate = hardware.mSampleRate;
    sg_heardCapacity = (size_t)(kRecordSeconds * sg_hardwareRate);
    sg_heardLeft = calloc(sg_heardCapacity, sizeof(float));
    sg_heardRight = calloc(sg_heardCapacity, sizeof(float));
    check(AudioOutputUnitStart(_output), "start");
    check(AudioUnitAddRenderNotify(_output, record, NULL), "record");
    _startedAt = CACurrentMediaTime();
    _lastAt = _startedAt;
    NSLog(@"[harness] the chain runs, the hardware at %.0f Hz", sg_hardwareRate);
}

- (void)after:(double)seconds do:(void (^)(void))block {
    dispatch_after(dispatch_time(DISPATCH_TIME_NOW, (int64_t)(seconds * NSEC_PER_SEC)), dispatch_get_main_queue(), block);
}

// The player reporting what the decoder has handed over, as Spotify's does on a change (playing, a seek), not as it plays.
- (void)playerReports {
    _state = [SPTPlayerState new];
    _state.positionAsOfTimestamp = _state.spotifyAsOf = atomic_load(&sg_readAt) / kRate;
    _state.timestamp = [NSDate date];
    _state.isPaused = _paused;
    _state.isPlaying = YES;
    sg_reported = _state;
}

// What is heard now, in seconds of the mix: what the output played, less what a seek skipped.
- (double)heardContent {
    return atomic_load(&sg_heard) / sg_hardwareRate;
}

// Where in the mix what is heard now is, by the decoder less what Sing holds (the Mac harness checks the lead to the frame).
- (double)heardInMix {
    return atomic_load(&sg_readAt) / kRate - SGSingHeldLead();
}

- (void)report {
    CFTimeInterval now = CACurrentMediaTime();
    uint64_t decoded = atomic_load(&sg_decoded);
    NSLog(@"[harness] %5.1f s  %-22@ decoder %.2fx  position %.2f s, heard %.2f s, decoded %.2f s, lead %.2f s",
          now - _startedAt, SGSingStatusText(), (decoded - _lastDecoded) / kRate / (now - _lastAt), _state.position,
          [self heardInMix], atomic_load(&sg_readAt) / kRate, SGSingHeldLead());
    _lastDecoded = decoded;
    _lastAt = now;
}

- (void)waitFor:(BOOL (^)(void))condition every:(double)seconds then:(void (^)(void))then {
    if (condition()) {
        then();
        return;
    }
    [self after:seconds do:^{ [self waitFor:condition every:seconds then:then]; }];
}

// The position Spotify shows against where in the mix what is heard is.
// The state the lyrics read: the player's.
- (SPTPlayerState *)lyricsState {
    return [(SPTEsperantoPlayer *)sg_player state];
}

- (void)checkPosition:(NSString *)when {
    double position = [self lyricsState].position, heard = [self heardInMix];
    CHECK(fabs(position - heard) < 0.15, @"%@ the position Spotify shows, %.2f s, is what is heard, %.2f s (decoded %.2f s, %.2f s held)", when, position, heard,
          atomic_load(&sg_readAt) / kRate, SGSingHeldLead());
}

- (void)play {
    // Spotify reports that it plays, once, and its clock runs on by itself from there; or, as the decoded clock,
    // again every half second while it plays.
    [self playerReports];
    if (sg_clock == SGHarnessClockDecoded) {
        [NSTimer scheduledTimerWithTimeInterval:0.5 repeats:YES block:^(NSTimer *timer) {
            if (!self->_paused) [self playerReports];
        }];
    }
    for (int s = 1; s <= 100; s++) [self after:s do:^{ [self report]; }];
    [self after:14 do:^{
        double heard = [self heardContent], position = [self lyricsState].position, decoded = atomic_load(&sg_decoded) / kRate;
        sg_modelKeepsUp = SGSingCurrentState() != SGSingStateBehind && SGSingCurrentState() != SGSingStateBuffering;
        VOCALS_CHECK(SGSingCurrentState() == SGSingStateSinging, @"Sing is on with the vocals down at 14 s (%@)", SGSingStatusText());
        CHECK(SGSingHeldLead() > 2 && fabs(decoded - heard - SGSingHeldLead()) < 0.1,
              @"the lead Sing reports, %.2f s, is what the decoder handed over less what was heard (%.2f s less %.2f s)", SGSingHeldLead(), decoded, heard);
        CHECK(fabs(position - heard) < 0.15, @"as the lead filled, the position the lyrics read, %.2f s, is what is heard, %.2f s, "
              @"not what was decoded, %.2f s, nor that less the lead now, %.2f s", position, heard, decoded, position - SGSingHeldLead());
    }];
    // A tap on the lyrics line at 5 s, with the lead held: the lyrics' seek, as the karaoke view sends it.
    [self after:20 do:^{
        self->_seekLead = SGSingHeldLead();
        CHECK(self->_seekLead > 2, @"before the tap a lead is held (%.2f s)", self->_seekLead);
        SGKaraokeSeek(5000);
        [self playerReports];
        self->_tappedAt = CACurrentMediaTime();
        CHECK(fabs(atomic_load(&sg_readAt) / kRate - 5) < 0.05, @"the tap sends Spotify to the line's 5.00 s (%.2f s)", atomic_load(&sg_readAt) / kRate);
    }];
    [self after:20.5 do:^{
        double ahead = atomic_load(&sg_readAt) / kRate - [self lyricsState].position;
        CHECK(ahead < 1.2, @"the seek dropped the %.1f s held ahead: %.2f s ahead half a second later", self->_seekLead, ahead);
    }];
    [self after:21 do:^{
        double since = CACurrentMediaTime() - self->_tappedAt, heard = [self heardInMix];
        CHECK(fabs(heard - (5 + since)) < 0.2, @"a second after the tap what is heard is the line's 5 s run on by %.2f s: %.2f s", since, heard);
        [self checkPosition:@"a second after the tap,"];
    }];
    [self after:27 do:^{
        CHECK(SGSingHeldLead() > 2, @"after the seek the lead is built again (%.2f s)", SGSingHeldLead());
        [self checkPosition:@"seven seconds after the seek, with no report since,"];
        // At As sung with Spatial voice off and no page reading the lines, Sing rests.
        SGSetSingLevel(1);
    }];
    [self after:29 do:^{
        CHECK(SGSingCurrentState() == SGSingStateWaiting && [SGSingStatusText() isEqualToString:@"As sung"] && SGSingHeldLead() > 2,
              @"resting at As sung, Sing reads %@ and plays on the %.2f s it holds as they are", SGSingStatusText(), SGSingHeldLead());
        [self checkPosition:@"resting,"];
        SGSetSingLevel(0);
    }];
    [self after:31 do:^{
        CHECK(SGSingHeldLead() > 2, @"separating again, the lead was kept (%.2f s)", SGSingHeldLead());
        [self checkPosition:@"separating again,"];
        setThermal(NSProcessInfoThermalStateSerious);
    }];
    [self after:32 do:^{
        CHECK(SGSingCurrentState() == SGSingStateHot && SGSingHeldLead() > 2 && !SGSingLoaderSeparator(),
              @"at the thermal state Serious Sing is held (%@), the model let go and the %.2f s held played on", SGSingStatusText(), SGSingHeldLead());
        [self checkPosition:@"held at Serious,"];
        setThermal(NSProcessInfoThermalStateFair);
        CFTimeInterval fair = CACurrentMediaTime();
        [self waitFor:^BOOL { return SGSingLoaderSeparator() || CACurrentMediaTime() - fair > 40; } every:0.25 then:^{
            CHECK(SGSingCurrentState() != SGSingStateHot && SGSingLoaderSeparator() && SGSingHeldLead() > 2,
                  @"at Fair it loads the model again in %.1f s (%@), the lead kept (%.2f s)", CACurrentMediaTime() - fair, SGSingStatusText(), SGSingHeldLead());
            [self checkPosition:@"running again,"];
            // A memory warning stops Sing with the mic still on.
            [NSNotificationCenter.defaultCenter postNotificationName:UIApplicationDidReceiveMemoryWarningNotification object:UIApplication.sharedApplication];
            [self after:1 do:^{
                CHECK(SGSingOn() && [SGSingStatusText() isEqualToString:@"Stopped"] && SGSingHeldLead() > 2,
                      @"stopped by a memory warning, Sing reads %@ and plays on the %.2f s it holds", SGSingStatusText(), SGSingHeldLead());
                [self checkPosition:@"stopped,"];
                [self pauseThenResume];
            }];
        }];
    }];
}

// Spotify pauses (its output stops and it reports), Sing seeks it back to what was heard, which drops the lead, and
// Spotify resumes from there: nothing heard is lost.
- (void)pauseThenResume {
    double heard = [self heardInMix], lead = SGSingHeldLead();
    AudioOutputUnitStop(_output);
    _paused = YES;
    [self playerReports];
    [sg_watcher playerStateDidChange:_state];
    [self playerReports];   // Spotify's report of the seek
    double seekedTo = atomic_load(&sg_readAt) / kRate;
    CHECK(fabs(seekedTo - heard) < 0.15 && SGSingHeldLead() == 0, @"paused, Spotify is seeked back from %.2f s to %.2f s, what was heard (%.2f s), and the %.2f s held "
          @"are let go (%.2f s held)", heard + lead, seekedTo, heard, lead, SGSingHeldLead());
    [self checkPosition:@"paused,"];
    _paused = NO;
    AudioOutputUnitStart(_output);
    [self playerReports];
    CFTimeInterval resumed = CACurrentMediaTime();
    [self after:2 do:^{
        double since = CACurrentMediaTime() - resumed;
        CHECK(SGSingHeldLead() < 0.001 && fabs([self heardInMix] - (heard + since)) < 0.2,
              @"resumed, the song plays on from %.2f s, where it was heard, nothing skipped: %.2f s heard %.2f s on (%.2f s held)", heard, [self heardInMix], since,
              SGSingHeldLead());
        [self checkPosition:@"resumed,"];
        SGSetSingOn(NO);
        [self after:1 do:^{
            CHECK(SGSingHeldLead() < 0.001 && SGSingCurrentState() == SGSingStateOff, @"switched off, nothing is held (%.3f s) and Sing is %@",
                  SGSingHeldLead(), SGSingStatusText());
            [self score];
            NSLog(@"[harness] %@", sg_failures ? @"FAILED" : @"all passed");
            exit(sg_failures ? 1 : 0);
        }];
    }];
}

// What played from 6 s to 19 s against the chords alone, beside the mix's own score. Before the seek what plays
// is the mix in order, so the two line up but for the resampler's delay, found on the first second (dry).
- (void)score {
    size_t frames;
    NSArray<NSData *> *heard = heardAt44k(&frames);
    const float *left = heard[0].bytes, *right = heard[1].bytes;
    size_t from = (size_t)(6 * kRate), to = (size_t)(19 * kRate);
    if (frames < to + 1000) {
        CHECK(NO, @"enough was heard to score (%.1f s)", frames / kRate);
        return;
    }
    int best = 0;
    double bestScore = -INFINITY;
    for (int lag = -64; lag <= 64; lag++) {
        double dot = 0;
        for (size_t i = (size_t)kRate / 4; i < (size_t)kRate; i++) dot += left[i + lag] * sg_left[i];
        if (dot > bestScore) bestScore = dot, best = lag;
    }
    double dry = snr(sg_left, sg_right, left + best, right + best, (size_t)(kRate / 4), (size_t)kRate);
    NSLog(@"[harness] what plays lags the mix by %d frames after resampling; the first second matches the mix to %.1f dB", best, dry);
    double karaoke = snr(sg_chordsLeft, sg_chordsRight, left + best, right + best, from, to);
    double mix = snr(sg_chordsLeft, sg_chordsRight, sg_left, sg_right, from, to);
    VOCALS_CHECK(karaoke > mix + 15, @"from 6 s to 19 s what plays scores %.1f dB against the chords alone, the mix %.1f dB", karaoke, mix);
}

- (BOOL)application:(UIApplication *)application didFinishLaunchingWithOptions:(NSDictionary *)options {
    self.window = [[UIWindow alloc] initWithFrame:UIScreen.mainScreen.bounds];
    self.window.rootViewController = [UIViewController new];
    sg_seekResult = [NSObject new];
    sg_player = [SPTEsperantoPlayer new];
    NSString *clock = [NSUserDefaults.standardUserDefaults stringForKey:@"clock"];
    sg_clock = [clock isEqualToString:@"restamp"] ? SGHarnessClockRestamp : [clock isEqualToString:@"decoded"] ? SGHarnessClockDecoded : SGHarnessClockReport;
    NSLog(@"[harness] Spotify's clock: %@", @[@"its last report run on", @"its last report stamped again at every read", @"reported again every 0.5 s with what was decoded"][sg_clock]);
    // On the class processInfo really is, which may override NSProcessInfo's.
    class_replaceMethod(object_getClass(NSProcessInfo.processInfo), @selector(thermalState), (IMP)thermalState, "q16@0:8");
    makeMix();
    NSLog(@"[harness] Sing: %@ (%@)", SGSingStatusText(), SGSingMissing() ?: @"nothing missing");
    CFTimeInterval began = CACurrentMediaTime();
    if (SGSingModelCurrentState() != SGSingModelReady) SGSingDownloadModel();
    __block CFTimeInterval logged = 0;
    [self waitFor:^BOOL {
        if (CACurrentMediaTime() - logged > 5) {
            logged = CACurrentMediaTime();
            NSLog(@"[harness] %@", SGSingStatusText());
        }
        return SGSingModelCurrentState() != SGSingModelDownloading;
    } every:0.5 then:^{
        CHECK(SGSingModelCurrentState() == SGSingModelReady, @"the voice model is on the phone, checked (%@, %.0f s)", SGSingModelError() ?: @"no error",
              CACurrentMediaTime() - began);
        if (SGSingModelCurrentState() != SGSingModelReady) exit(1);
        SGSetSingLevel(0);
        CFTimeInterval loading = CACurrentMediaTime();
        SGSetSingOn(YES);
        [self waitFor:^BOOL { return SGSingCurrentState() != SGSingStatePreparing; } every:0.25 then:^{
            CHECK(SGSingCurrentState() == SGSingStateWaiting, @"the model loads in %.1f s, and Sing waits for Spotify (%@)", CACurrentMediaTime() - loading,
                  SGSingStatusText());
            [self startChain];
            [self play];
        }];
    }];
    return YES;
}

@end

int main(int argc, char *argv[]) {
    @autoreleasepool {
        return UIApplicationMain(argc, argv, nil, NSStringFromClass(SGHarnessDelegate.class));
    }
}