Liquid Glass and Apple Music touches for the Spotify iOS app. A GPL-3.0 fork of spoti.pw 0.21.1.
C
// Runs the steps of moving artwork that need no phone (Shared/AnimatedArtwork/SGMotionClip.m) on the Mac:
// the cut's size, a tall clip cut to 3:4 and read back (its size, its frames, and that it was cut about its
// middle rather than squeezed), a clip stored on its side, Spotify's Canvas service's request and reply,
// and when a Canvas that turned up late is asked for. Exits 1 on any failure.
#import <AVFoundation/AVFoundation.h>
#import "Shared/AdBlock/Protobuf.h"
#import "Shared/AnimatedArtwork/SGMotionClip.h"
static int failures;
static void check(BOOL ok, NSString *what) {
printf(" %s %s\n", ok ? "ok " : "FAIL", what.UTF8String);
if (!ok) failures++;
}
// A stand-in Canvas, `size` as stored, of `count` frames at 30 fps, in three bands from the top as shown:
// red to a quarter, green to three quarters, blue below. `turn` stands it up when it is stored on its side.
static BOOL writeBands(NSURL *file, CGSize size, int count, CGAffineTransform turn, BOOL sideways) {
[NSFileManager.defaultManager removeItemAtURL:file error:nil];
AVAssetWriter *writer = [AVAssetWriter assetWriterWithURL:file fileType:AVFileTypeMPEG4 error:nil];
AVAssetWriterInput *input = [AVAssetWriterInput assetWriterInputWithMediaType:AVMediaTypeVideo outputSettings:@{
AVVideoCodecKey: AVVideoCodecTypeH264, AVVideoWidthKey: @(size.width), AVVideoHeightKey: @(size.height),
}];
input.transform = turn;
AVAssetWriterInputPixelBufferAdaptor *adaptor = [AVAssetWriterInputPixelBufferAdaptor assetWriterInputPixelBufferAdaptorWithAssetWriterInput:input
sourcePixelBufferAttributes:@{(id)kCVPixelBufferPixelFormatTypeKey: @(kCVPixelFormatType_32BGRA),
(id)kCVPixelBufferWidthKey: @(size.width), (id)kCVPixelBufferHeightKey: @(size.height)}];
[writer addInput:input];
[writer startWriting];
[writer startSessionAtSourceTime:kCMTimeZero];
for (int i = 0; i < count; i++) {
while (!input.readyForMoreMediaData) [NSThread sleepForTimeInterval:0.002];
CVPixelBufferRef buffer = NULL;
CVPixelBufferPoolCreatePixelBuffer(NULL, adaptor.pixelBufferPool, &buffer);
CVPixelBufferLockBaseAddress(buffer, 0);
uint8_t *base = CVPixelBufferGetBaseAddress(buffer);
size_t row = CVPixelBufferGetBytesPerRow(buffer);
for (size_t y = 0; y < size.height; y++) {
for (size_t x = 0; x < size.width; x++) {
// Along the shown picture's height: the stored rows, or, on its side, the stored columns.
double along = sideways ? (double)x / size.width : (double)y / size.height;
uint8_t *pixel = base + y * row + x * 4; // BGRA
pixel[0] = along >= 0.75 ? 255 : 0;
pixel[1] = along >= 0.25 && along < 0.75 ? 255 : 0;
pixel[2] = along < 0.25 ? 255 : 0;
pixel[3] = 255;
}
}
CVPixelBufferUnlockBaseAddress(buffer, 0);
[adaptor appendPixelBuffer:buffer withPresentationTime:CMTimeMake(i, 30)];
CVPixelBufferRelease(buffer);
}
[input markAsFinished];
dispatch_semaphore_t done = dispatch_semaphore_create(0);
[writer finishWritingWithCompletionHandler:^{ dispatch_semaphore_signal(done); }];
dispatch_semaphore_wait(done, DISPATCH_TIME_FOREVER);
return writer.status == AVAssetWriterStatusCompleted;
}
static AVAssetTrack *videoOf(NSURL *file) {
__block AVAssetTrack *track = nil;
dispatch_semaphore_t loaded = dispatch_semaphore_create(0);
[[AVURLAsset assetWithURL:file] loadTracksWithMediaType:AVMediaTypeVideo completionHandler:^(NSArray *tracks, NSError *error) {
track = tracks.firstObject;
dispatch_semaphore_signal(loaded);
}];
dispatch_semaphore_wait(loaded, DISPATCH_TIME_FOREVER);
return track;
}
static int framesIn(NSURL *file) {
AVAssetTrack *track = videoOf(file);
AVAssetReader *reader = [AVAssetReader assetReaderWithAsset:track.asset error:nil];
AVAssetReaderTrackOutput *output = [AVAssetReaderTrackOutput assetReaderTrackOutputWithTrack:track outputSettings:nil];
[reader addOutput:output];
[reader startReading];
int count = 0;
CMSampleBufferRef sample;
while ((sample = [output copyNextSampleBuffer])) {
if (CMSampleBufferGetNumSamples(sample)) count++;
CFRelease(sample);
}
return count;
}
// The color at `y`, a share of the shown picture's height, in its middle column: 'r', 'g', 'b' or '?'.
static char colourAt(NSURL *file, double y) {
AVAssetImageGenerator *generator = [AVAssetImageGenerator assetImageGeneratorWithAsset:[AVURLAsset assetWithURL:file]];
generator.appliesPreferredTrackTransform = YES;
generator.requestedTimeToleranceBefore = generator.requestedTimeToleranceAfter = kCMTimeZero;
__block CGImageRef image = NULL;
dispatch_semaphore_t made = dispatch_semaphore_create(0);
[generator generateCGImageAsynchronouslyForTime:kCMTimeZero completionHandler:^(CGImageRef frame, CMTime actual, NSError *error) {
image = frame ? CGImageRetain(frame) : NULL;
dispatch_semaphore_signal(made);
}];
dispatch_semaphore_wait(made, DISPATCH_TIME_FOREVER);
if (!image) return '?';
size_t width = CGImageGetWidth(image), height = CGImageGetHeight(image);
uint8_t pixel[4] = {0};
CGColorSpaceRef space = CGColorSpaceCreateWithName(kCGColorSpaceSRGB);
CGContextRef context = CGBitmapContextCreate(pixel, 1, 1, 8, 4, space, (CGBitmapInfo)kCGImageAlphaPremultipliedLast);
// The one pixel wanted lands on the 1x1 context: Core Graphics counts rows from the bottom.
CGContextDrawImage(context, CGRectMake(-(double)(width / 2), -(double)height * (1 - y), width, height), image);
CGContextRelease(context);
CGColorSpaceRelease(space);
CGImageRelease(image);
if (pixel[0] > 160 && pixel[1] < 90 && pixel[2] < 90) return 'r';
if (pixel[1] > 160 && pixel[0] < 90 && pixel[2] < 90) return 'g';
if (pixel[2] > 160 && pixel[0] < 90 && pixel[1] < 90) return 'b';
return '?';
}
static CGSize shownSize(AVAssetTrack *track) {
CGRect shown = CGRectApplyAffineTransform((CGRect){CGPointZero, track.naturalSize}, track.preferredTransform);
return CGSizeMake(fabs(shown.size.width), fabs(shown.size.height));
}
int main(void) {
@autoreleasepool {
printf("the cut\n");
check(CGSizeEqualToSize(SGMotionClipCut(CGSizeMake(720, 1280), 0.75), CGSizeMake(720, 960)), @"a 9:16 Canvas to 3:4 keeps its width");
check(CGSizeEqualToSize(SGMotionClipCut(CGSizeMake(1000, 1000), 0.75), CGSizeMake(750, 1000)), @"a square to 3:4 keeps its height");
check(CGSizeEqualToSize(SGMotionClipCut(CGSizeMake(720, 1280), 1), CGSizeMake(720, 720)), @"a 9:16 Canvas to 1:1");
CGSize odd = SGMotionClipCut(CGSizeMake(721, 1281), 0.75);
check((int)odd.width % 2 == 0 && (int)odd.height % 2 == 0 && odd.width <= 721 && odd.height <= 1281, @"odd sides come out even, inside the picture");
check(CGSizeEqualToSize(SGMotionClipCut(CGSizeZero, 0.75), CGSizeZero), @"no picture, no cut");
NSURL *folder = [NSURL fileURLWithPath:@"build/clips" isDirectory:YES];
[NSFileManager.defaultManager createDirectoryAtURL:folder withIntermediateDirectories:YES attributes:nil error:nil];
NSURL *tall = [folder URLByAppendingPathComponent:@"canvas-9x16.mp4"], *cut = [folder URLByAppendingPathComponent:@"canvas-3x4.mp4"];
printf("a 9:16 Canvas cut to 3:4\n");
check(writeBands(tall, CGSizeMake(720, 1280), 60, CGAffineTransformIdentity, NO), @"the stand-in Canvas written");
CFAbsoluteTime start = CFAbsoluteTimeGetCurrent();
BOOL shaped = SGMotionClipShape(tall, 0.75, cut);
printf(" (cut in %.0f ms)\n", (CFAbsoluteTimeGetCurrent() - start) * 1000);
check(shaped, @"cut and written");
AVAssetTrack *track = videoOf(cut);
check(CGSizeEqualToSize(shownSize(track), CGSizeMake(720, 960)), [NSString stringWithFormat:@"720 x 960 (%@)", NSStringFromSize(shownSize(track))]);
check(framesIn(cut) == 60 && framesIn(tall) == 60, [NSString stringWithFormat:@"every frame kept (%d of %d)", framesIn(cut), framesIn(tall)]);
check(fabs(CMTimeGetSeconds(track.timeRange.duration) - 2) < 0.05, [NSString stringWithFormat:@"2 s long (%.3f)", CMTimeGetSeconds(track.timeRange.duration)]);
// Cut about the middle, 160 rows off each end, the bands sit at 0-1/6, 1/6-5/6 and 5/6-1; squeezed
// they would sit at 0-1/4, 1/4-3/4 and 3/4-1.
NSString *seen = [NSString stringWithFormat:@"%c%c%c%c", colourAt(cut, 0.08), colourAt(cut, 0.21), colourAt(cut, 0.79), colourAt(cut, 0.92)];
check([seen isEqualToString:@"rggb"], [NSString stringWithFormat:@"cut about its middle, not squeezed (%@ at 8, 21, 79, 92 %%; rggb wanted)", seen]);
check(![NSFileManager.defaultManager fileExistsAtPath:[cut URLByAppendingPathExtension:@"partial"].path], @"no partial file left");
check(!SGMotionClipShape([folder URLByAppendingPathComponent:@"missing.mp4"], 0.75, [folder URLByAppendingPathComponent:@"none.mp4"]),
@"a file that is not there is not cut");
printf("a Canvas stored on its side\n");
NSURL *side = [folder URLByAppendingPathComponent:@"canvas-side.mp4"], *sideCut = [folder URLByAppendingPathComponent:@"canvas-side-3x4.mp4"];
// Stored 1280 x 720 and turned a quarter: shown 720 x 1280.
CGAffineTransform quarter = CGAffineTransformMake(0, 1, -1, 0, 720, 0);
check(writeBands(side, CGSizeMake(1280, 720), 30, quarter, YES), @"written");
check(CGSizeEqualToSize(shownSize(videoOf(side)), CGSizeMake(720, 1280)), @"shown 720 x 1280");
check(SGMotionClipShape(side, 0.75, sideCut), @"cut and written");
AVAssetTrack *sideTrack = videoOf(sideCut);
check(CGSizeEqualToSize(shownSize(sideTrack), CGSizeMake(720, 960)), [NSString stringWithFormat:@"shown 720 x 960 (%@, stored %@)",
NSStringFromSize(shownSize(sideTrack)), NSStringFromSize(sideTrack.naturalSize)]);
NSString *sideSeen = [NSString stringWithFormat:@"%c%c%c%c", colourAt(sideCut, 0.08), colourAt(sideCut, 0.21), colourAt(sideCut, 0.79), colourAt(sideCut, 0.92)];
check([sideSeen isEqualToString:@"rggb"], [NSString stringWithFormat:@"stood up and cut about its middle (%@)", sideSeen]);
printf("Spotify's Canvas service\n");
NSData *ask = SGMotionCanvasAsk(@"spotify:track:x");
const uint8_t wanted[] = {0x0a, 0x11, 0x0a, 0x0f, 's', 'p', 'o', 't', 'i', 'f', 'y', ':', 't', 'r', 'a', 'c', 'k', ':', 'x'};
check([ask isEqualToData:[NSData dataWithBytes:wanted length:sizeof wanted]], [NSString stringWithFormat:@"the request: an entity naming the track (%@)", ask]);
check(SGMotionCanvasAsk(nil) == nil, @"no track, no request");
NSData *(^canvas)(NSString *, int) = ^NSData *(NSString *address, int type) {
NSMutableArray *fields = [NSMutableArray arrayWithObjects:SGPBString(1, @"id"), SGPBString(2, address), SGPBString(3, @"file"), nil];
if (type) [fields addObject:SGPBVarint(4, type)];
return SGPBSerialize(@[SGPBBytes(1, SGPBSerialize(fields))]);
};
NSMutableData *both = [canvas(@"https://canvaz.scdn.co/still.jpg", 0) mutableCopy];
[both appendData:canvas(@"https://canvaz.scdn.co/loop.mp4", 2)];
check([SGMotionCanvasInReply(both) isEqualToString:@"https://canvaz.scdn.co/loop.mp4"], @"a still then a video: the video");
check([SGMotionCanvasInReply(canvas(@"https://canvaz.scdn.co/v.mp4", 1)) isEqualToString:@"https://canvaz.scdn.co/v.mp4"], @"type 1 is a video");
check([SGMotionCanvasInReply(canvas(@"https://canvaz.scdn.co/v.mp4", 3)) isEqualToString:@"https://canvaz.scdn.co/v.mp4"], @"type 3 is a video");
check(SGMotionCanvasInReply(canvas(@"https://canvaz.scdn.co/a.gif", 4)) == nil, @"a GIF is no clip");
check(SGMotionCanvasInReply(canvas(@"https://canvaz.scdn.co/still.jpg", 0)) == nil, @"a still alone is no clip");
check(SGMotionCanvasInReply(canvas(@"", 2)) == nil, @"a video with no address is no clip");
check(SGMotionCanvasInReply([NSData data]) == nil, @"an empty reply");
check(SGMotionCanvasInReply([@"<html>not found" dataUsingEncoding:NSUTF8StringEncoding]) == nil, @"a page that is not a reply");
check(SGMotionCanvasInReply(nil) == nil, @"no reply");
printf("a Canvas that turns up late\n");
NSArray *canvasFirst = @[@"canvas", @"applemusic"], *appleFirst = @[@"applemusic", @"canvas"];
check(SGMotionCanvasOutranks(canvasFirst, nil), @"nothing showing: asked");
check(SGMotionCanvasOutranks(canvasFirst, @"applemusic"), @"Apple Music's below Canvas showing: asked");
check(!SGMotionCanvasOutranks(canvasFirst, @"canvas"), @"the Canvas already showing: not asked");
check(!SGMotionCanvasOutranks(appleFirst, @"applemusic"), @"Apple Music's above Canvas showing: not asked");
check(SGMotionCanvasOutranks(appleFirst, nil), @"Apple Music first with nothing: asked");
check(!SGMotionCanvasOutranks(@[@"applemusic"], nil), @"Canvas switched off: not asked");
check(!SGMotionCanvasOutranks(@[], nil), @"no sources: not asked");
printf(failures ? "motion: %d FAILED\n" : "motion: all passed\n", failures);
return failures ? 1 : 0;
}
}