Liquid Glass and Apple Music touches for the Spotify iOS app. A GPL-3.0 fork of spoti.pw 0.21.1.
C
fix: Gemini fills partial translations and explains failures
My-Name-Is-Jeff353785fbb520
diff --git a/harness/lyrics-translation/README.md b/harness/lyrics-translation/README.mdnew file mode 100644index 00000000..36f8c959--- /dev/null+++ b/harness/lyrics-translation/README.md@@ -0,0 +1,11 @@+# Lyrics translation check++`./build.sh` checks the parts of lyrics translation that need no network: Musixmatch's community+translations matched onto a song's lines by their folded text, as copies that leave the lines kept+before untouched (`SGMusixmatchTranslatedLines`), and Gemini's reply read into one translation per line+or a reason to show (`SGGeminiTranslationsIn`): a blocked prompt, a recitation or safety stop, a line+count that does not match, the key, the limit. It runs on the Mac as a Mac Catalyst binary, with no+simulator and no network.++It does not cover the requests themselves, the lines taking the translations as they are kept+(`Shared/Lyrics/KaraokeSource.x`), or the lyrics view picking them up.diff --git a/harness/lyrics-translation/build.sh b/harness/lyrics-translation/build.shnew file mode 100755index 00000000..61599a53--- /dev/null+++ b/harness/lyrics-translation/build.sh@@ -0,0 +1,12 @@+#!/bin/sh+# build.sh: builds check.m against this checkout's Musixmatch.m, LyricsTranslation.m and KaraokeTiming.m+# as a Mac Catalyst binary and runs it, so no simulator is needed.+set -e+HERE=$(cd "$(dirname "$0")" && pwd)+SRC=$(cd "$HERE/../../tweak/Sources" && pwd)+mkdir -p "$HERE/build"+xcrun -sdk macosx clang -target arm64-apple-ios17.0-macabi -isysroot "$(xcrun --sdk macosx --show-sdk-path)" -iframework "$(xcrun --sdk macosx --show-sdk-path)/System/iOSSupport/System/Library/Frameworks" -fobjc-arc -g -Wall -I"$SRC" \+ "$HERE/check.m" "$SRC/Shared/LyricsSources/Musixmatch.m" "$SRC/Shared/LyricsTranslation/LyricsTranslation.m" \+ "$SRC/Shared/Lyrics/KaraokeTiming.m" "$SRC/Shared/Lyrics/Protobuf.m" \+ -framework UIKit -framework Security -framework Foundation -o "$HERE/build/check"+"$HERE/build/check"diff --git a/harness/lyrics-translation/check.m b/harness/lyrics-translation/check.mnew file mode 100644index 00000000..2a371d54--- /dev/null+++ b/harness/lyrics-translation/check.m@@ -0,0 +1,63 @@+// Checks the parts of lyrics translation that need no network: Musixmatch's community translations+// matched onto a song's lines as copies, and Gemini's reply read into lines or a reason. Built against+// this checkout's Musixmatch.m, LyricsTranslation.m and KaraokeTiming.m. Exits non-zero on the first+// wrong answer.+#import <UIKit/UIKit.h>+#import "Shared/Lyrics/Lyrics.h"+#import "Shared/LyricsSources/LyricsSources.h"+#import "Shared/LyricsTranslation/LyricsTranslation.h"++// What the files reach outside what is checked here; none of it is called.+@implementation SGLyricsResult+@end+void SGLyricsNoteReply(NSURLResponse *response, NSError *error) {}+NSString *SGLyricsTranslationLanguage(void) { return nil; }+SGModRow *SGStatActionRow(NSString *t, NSString *s, NSString *(^v)(void), void (^a)(void)) { return nil; }+UIViewController *SGTopController(void) { return nil; }++#define CHECK(cond) do { if (!(cond)) { fprintf(stderr, "FAILED line %d: %s\n", __LINE__, #cond); exit(1); } } while (0)++static NSArray<NSString *> *readReply(NSString *json, NSInteger status, NSUInteger count, NSString **problem) {+ *problem = nil;+ id root = [NSJSONSerialization JSONObjectWithData:[json dataUsingEncoding:NSUTF8StringEncoding] options:0 error:nil];+ return SGGeminiTranslationsIn(root, status, nil, count, problem);+}++int main(void) {+ @autoreleasepool {+ // Musixmatch's translations go on copies of the lines they match, by the folded text.+ NSArray<SGKaraokeLine *> *lines = SGKaraokeEstimatedLines(@[@0, @2000, @4000, @6000],+ @[@"Où est l'amour ?", @"Yeah", @"Rien", @""]);+ CHECK(lines.count >= 3);+ lines[2].translation = @"Nothing";+ NSDictionary *byLine = @{SGMusixmatchLineKey(@"ou est L'AMOUR"): @"Where is love?", SGMusixmatchLineKey(@"rien"): @"None"};+ NSArray<SGKaraokeLine *> *translated = SGMusixmatchTranslatedLines(lines, byLine);+ CHECK(translated && translated != lines && translated.count == lines.count);+ CHECK([translated[0].translation isEqualToString:@"Where is love?"] && translated[0] != lines[0]);+ CHECK(!lines[0].translation); // the lines kept before are left as they were+ CHECK(translated[0].start == lines[0].start && translated[0].end == lines[0].end+ && translated[0].words == lines[0].words && translated[0].timing == lines[0].timing);+ CHECK(translated[1] == lines[1] && !translated[1].translation); // no match: the same line+ CHECK([translated[2].translation isEqualToString:@"Nothing"]); // a translation of its own stays+ CHECK(!SGMusixmatchTranslatedLines(lines, @{@"nothing here": @"x"}));++ // Gemini's language by the whole tag, so the script survives.+ NSLocale *english = [NSLocale localeWithLocaleIdentifier:@"en"];+ CHECK(![[english localizedStringForLocaleIdentifier:@"zh-Hant"] isEqualToString:[english localizedStringForLocaleIdentifier:@"zh-Hans"]]);++ // Gemini's reply: lines, or why there are none.+ NSString *problem;+ NSArray<NSString *> *got = readReply(@"{\"candidates\":[{\"content\":{\"parts\":[{\"text\":\"[\\\"a\\\",\\\"\\\"]\"}]},\"finishReason\":\"STOP\"}]}", 200, 2, &problem);+ CHECK([got isEqualToArray:(@[@"a", @""])] && !problem);+ CHECK(!readReply(@"{\"promptFeedback\":{\"blockReason\":\"PROHIBITED_CONTENT\"}}", 200, 2, &problem) && [problem containsString:@"filters"]);+ CHECK(!readReply(@"{\"candidates\":[{\"finishReason\":\"RECITATION\"}]}", 200, 2, &problem) && [problem containsString:@"repeated"]);+ CHECK(!readReply(@"{\"candidates\":[{\"finishReason\":\"SAFETY\"}]}", 200, 2, &problem) && [problem containsString:@"filters"]);+ CHECK(!readReply(@"{\"candidates\":[{\"content\":{\"parts\":[{\"text\":\"[\\\"a\\\"]\"}]},\"finishReason\":\"STOP\"}]}", 200, 2, &problem)+ && [problem containsString:@"1 lines for the song's 2"]);+ CHECK(!readReply(@"{\"error\":{}}", 403, 2, &problem) && [problem containsString:@"key"]);+ CHECK(!readReply(@"{\"error\":{}}", 429, 2, &problem) && [problem containsString:@"limit"]);+ CHECK(!readReply(@"{}", 200, 2, &problem) && [problem containsString:@"could not be read"]);+ puts("lyrics translation: all checks passed");+ }+ return 0;+}diff --git a/tweak/Sources/Redesigned/Lyrics/SGRKaraokeView.m b/tweak/Sources/Redesigned/Lyrics/SGRKaraokeView.mindex 026fd7c5..81b4265d 100644--- a/tweak/Sources/Redesigned/Lyrics/SGRKaraokeView.m+++ b/tweak/Sources/Redesigned/Lyrics/SGRKaraokeView.m@@ -1096,6 +1096,7 @@ typedef struct {CFTimeInterval _stillSince; // when the position stopped moving, 0 while it movesSGRKaraokeStyle *_style; // how the lines were laid outBOOL _hasSpoken, _hasTranslation; // whether the song has any line with either+ BOOL _untranslated; // whether a line with words has no translation, which Gemini can fill inNSArray<SGKaraokeLine *> *_passedOver; // lines kept for the song that changed nothing here, not looked at againUIButton *_extras;CGFloat _builtWidth;@@ -1350,6 +1351,14 @@ static BOOL hasTranslations(NSArray<SGKaraokeLine *> *lines) {return NO;}+// A line with words to translate: not an empty one, nor a ♪ holding a break.+static BOOL hasWords(SGKaraokeLine *line) {+ for (SGKaraokeWord *word in line.words) {+ if ([word.text rangeOfCharacterFromSet:NSCharacterSet.letterCharacterSet].location != NSNotFound) return YES;+ }+ return NO;+}+// When each line is sung and where the breaks are, worked out once per song for the frames to read.- (void)timeLines {free(_spans);@@ -1358,7 +1367,7 @@ static BOOL hasTranslations(NSArray<SGKaraokeLine *> *lines) {_spans = calloc(count + 1, sizeof(SGRKaraokeSpan));_breaks = calloc(count + 1, sizeof(SGRKaraokeBreak));_breakCount = 0;- _hasSpoken = _hasTranslation = NO;+ _hasSpoken = _hasTranslation = _untranslated = NO;_plain = SGKaraokeLinesTiming(_lines) == SGKaraokeTimingNone;NSInteger sungTo = 0; // the top of the song counts as where the singing before the first line endsfor (NSUInteger i = 0; i < count; i++) {@@ -1368,6 +1377,7 @@ static BOOL hasTranslations(NSArray<SGKaraokeLine *> *lines) {sungTo = MAX(sungTo, _spans[i].end);_hasSpoken = _hasSpoken || line.pronunciation || line.backing.pronunciation;_hasTranslation = _hasTranslation || line.translation.length;+ _untranslated = _untranslated || (!line.translation.length && hasWords(line));}[self offerExtras];[self askMeanings];@@ -1488,7 +1498,8 @@ static BOOL hasTranslations(NSArray<SGKaraokeLine *> *lines) {image:[UIImage systemImageNamed:@"character.bubble"] identifier:nilhandler:^(UIAction *action) { SGRSetLyricsTextShown(SGRLyricsTextTranslation, !on); }]];}- if (gemini && !_hasTranslation) {+ // Also for a song a source translated in part, as Musixmatch's community often has: Gemini fills in the rest.+ if (gemini && _untranslated) {__weak SGRKaraokeView *weakSelf = self;[items addObject:[UIAction actionWithTitle:@"Translate with Gemini" image:[UIImage systemImageNamed:@"sparkles"]identifier:nil handler:^(UIAction *action) { [weakSelf translateWithGemini]; }]];@@ -1498,8 +1509,8 @@ static BOOL hasTranslations(NSArray<SGKaraokeLine *> *lines) {[self setNeedsLayout];}-// The song's lines into the Lyrics page's language, or the phone's: each line takes its translation, and-// translations are switched on so the answer shows.+// The song's lines into the Lyrics page's language, or the phone's: each line without a translation takes+// Gemini's, and translations are switched on so the answer shows. The whole song is sent, for the sense.- (void)translateWithGemini {NSArray<SGKaraokeLine *> *lines = _lines;NSString *track = _track;@@ -1514,9 +1525,10 @@ static BOOL hasTranslations(NSArray<SGKaraokeLine *> *lines) {}if (lines != self->_lines) return; // the song moved on meanwhile[lines enumerateObjectsUsingBlock:^(SGKaraokeLine *line, NSUInteger i, BOOL *stop) {- if (translations[i].length) line.translation = translations[i];+ if (translations[i].length && !line.translation.length) line.translation = translations[i];}];self->_hasTranslation = YES;+ self->_untranslated = NO;if (SGFlag(SGRKeyLyricsTranslation, NO)) [self restyle];else SGRSetLyricsTextShown(SGRLyricsTextTranslation, YES);});diff --git a/tweak/Sources/Shared/LyricsTranslation/LyricsTranslation.h b/tweak/Sources/Shared/LyricsTranslation/LyricsTranslation.hindex 3235c173..b5d923aa 100644--- a/tweak/Sources/Shared/LyricsTranslation/LyricsTranslation.h+++ b/tweak/Sources/Shared/LyricsTranslation/LyricsTranslation.h@@ -15,5 +15,9 @@ void SGLyricsTranslateWithGemini(NSString *trackID, NSArray<SGKaraokeLine *> *li// The language a translation is asked in: the Lyrics page's, else the phone's own.NSString *SGLyricsGeminiLanguage(void);+// Gemini's reply read: one translation per line when it has exactly `count`, else nil and in `problem`+// why not (the key, the limit, a filter, a recitation stop, a line count that does not match).+NSArray<NSString *> *SGGeminiTranslationsIn(id root, NSInteger status, NSError *error, NSUInteger count, NSString **problem);+// The Lyrics page's row: shows whether a key is set, and sets or removes it.SGModRow *SGGeminiKeyRow(void);diff --git a/tweak/Sources/Shared/LyricsTranslation/LyricsTranslation.m b/tweak/Sources/Shared/LyricsTranslation/LyricsTranslation.mindex 5501c7f4..3fd86c57 100644--- a/tweak/Sources/Shared/LyricsTranslation/LyricsTranslation.m+++ b/tweak/Sources/Shared/LyricsTranslation/LyricsTranslation.m@@ -44,7 +44,51 @@ BOOL SGGeminiKeySet(void) {#pragma mark - askingNSString *SGLyricsGeminiLanguage(void) {- return SGLyricsTranslationLanguage() ?: [NSLocale.preferredLanguages.firstObject componentsSeparatedByString:@"-"].firstObject ?: @"en";+ return SGLyricsTranslationLanguage() ?: NSLocale.preferredLanguages.firstObject ?: @"en";+}++// Why Gemini stopped short, as finishReason or promptFeedback.blockReason name it.+static NSString *blockedBecause(NSString *reason) {+ if ([reason isEqualToString:@"RECITATION"]) return @"Gemini stopped because its answer repeated published lyrics.";+ if ([@[@"SAFETY", @"PROHIBITED_CONTENT", @"BLOCKLIST", @"SPII"] containsObject:reason]) return @"Gemini's filters blocked this song's lyrics.";+ if ([reason isEqualToString:@"MAX_TOKENS"]) return @"The song is too long for one answer from Gemini.";+ return [NSString stringWithFormat:@"Gemini stopped before translating the song (%@).", reason];+}++NSArray<NSString *> *SGGeminiTranslationsIn(id root, NSInteger status, NSError *error, NSUInteger count, NSString **problem) {+ NSDictionary *reply = [root isKindOfClass:NSDictionary.class] ? root : nil;+ id candidates = reply[@"candidates"];+ id first = [candidates isKindOfClass:NSArray.class] ? [candidates firstObject] : nil;+ id parts = [first isKindOfClass:NSDictionary.class] && [first[@"content"] isKindOfClass:NSDictionary.class] ? first[@"content"][@"parts"] : nil;+ id text = [parts isKindOfClass:NSArray.class] && [[parts firstObject] isKindOfClass:NSDictionary.class] ? [parts firstObject][@"text"] : nil;+ id parsed = [text isKindOfClass:NSString.class] ? [NSJSONSerialization JSONObjectWithData:[text dataUsingEncoding:NSUTF8StringEncoding] options:0 error:nil] : nil;+ if ([parsed isKindOfClass:NSArray.class] && [parsed count] == count) {+ NSMutableArray<NSString *> *clean = [NSMutableArray arrayWithCapacity:count];+ for (id line in parsed) [clean addObject:[line isKindOfClass:NSString.class] ? line : @""];+ return clean;+ }+ id feedback = reply[@"promptFeedback"];+ id blocked = [feedback isKindOfClass:NSDictionary.class] ? feedback[@"blockReason"] : nil;+ id finished = [first isKindOfClass:NSDictionary.class] ? first[@"finishReason"] : nil;+ if (error) {+ *problem = @"Gemini could not be reached.";+ } else if (status == 400 || status == 403) {+ *problem = @"Gemini turned the key down. Check it on the Lyrics page.";+ } else if (status == 429) {+ *problem = @"Gemini's limit for this key is reached for now.";+ } else if (status != 200) {+ *problem = [NSString stringWithFormat:@"Gemini did not translate the song (%ld).", (long)status];+ } else if ([blocked isKindOfClass:NSString.class]) {+ *problem = blockedBecause(blocked);+ } else if ([finished isKindOfClass:NSString.class] && ![finished isEqualToString:@"STOP"]) {+ *problem = blockedBecause(finished);+ } else if ([parsed isKindOfClass:NSArray.class]) {+ *problem = [NSString stringWithFormat:@"Gemini answered %lu lines for the song's %lu. Try again.",+ (unsigned long)[parsed count], (unsigned long)count];+ } else {+ *problem = @"Gemini's answer could not be read. Try again.";+ }+ return nil;}static NSMutableDictionary<NSString *, NSArray<NSString *> *> *sg_done; // per launch, by track and language@@ -64,15 +108,24 @@ void SGLyricsTranslateWithGemini(NSString *trackID, NSArray<SGKaraokeLine *> *li}NSMutableArray<NSString *> *texts = [NSMutableArray arrayWithCapacity:lines.count];for (SGKaraokeLine *line in lines) [texts addObject:SGKaraokeLineText(line) ?: @""];- NSString *language = [[NSLocale localeWithLocaleIdentifier:@"en"] localizedStringForLanguageCode:languageTag] ?: languageTag;+ // By the whole tag, so the script stays: zh-Hant is "Chinese, Traditional" where its language alone is "Chinese".+ NSString *language = [[NSLocale localeWithLocaleIdentifier:@"en"] localizedStringForLocaleIdentifier:languageTag] ?: languageTag;NSData *input = [NSJSONSerialization dataWithJSONObject:texts options:0 error:nil];NSString *prompt = [NSString stringWithFormat:@"Translate the lines of this song into %@. Answer with a JSON array of exactly %lu strings, one "@"per input line and in the same order. Translate the meaning naturally, keep each line short like a "@"lyric, and keep an empty line empty. A line already in %@ stays as it is.\n\n%@",language, (unsigned long)texts.count, language, [[NSString alloc] initWithData:input encoding:NSUTF8StringEncoding]];+ // Lyrics are often explicit, and translating them is what the user asked for, so every filter that can+ // be lowered is. Recitation cannot be switched off, and is named when it stops an answer.+ NSMutableArray *safety = [NSMutableArray array];+ for (NSString *category in @[@"HARM_CATEGORY_HARASSMENT", @"HARM_CATEGORY_HATE_SPEECH",+ @"HARM_CATEGORY_SEXUALLY_EXPLICIT", @"HARM_CATEGORY_DANGEROUS_CONTENT"]) {+ [safety addObject:@{@"category": category, @"threshold": @"BLOCK_NONE"}];+ }NSDictionary *body = @{@"contents": @[@{@"parts": @[@{@"text": prompt}]}],+ @"safetySettings": safety,@"generationConfig": @{@"responseMimeType": @"application/json",@"responseSchema": @{@"type": @"ARRAY", @"items": @{@"type": @"STRING"}},@@ -87,26 +140,8 @@ void SGLyricsTranslateWithGemini(NSString *trackID, NSArray<SGKaraokeLine *> *li[[NSURLSession.sharedSession dataTaskWithRequest:request completionHandler:^(NSData *data, NSURLResponse *response, NSError *error) {NSInteger status = [response isKindOfClass:NSHTTPURLResponse.class] ? ((NSHTTPURLResponse *)response).statusCode : 0;id root = data ? [NSJSONSerialization JSONObjectWithData:data options:0 error:nil] : nil;- NSArray *translations = nil;NSString *problem = nil;- id candidates = [root isKindOfClass:NSDictionary.class] ? root[@"candidates"] : nil;- id first = [candidates isKindOfClass:NSArray.class] ? [candidates firstObject] : nil;- id parts = [first isKindOfClass:NSDictionary.class] && [first[@"content"] isKindOfClass:NSDictionary.class] ? first[@"content"][@"parts"] : nil;- id text = [parts isKindOfClass:NSArray.class] && [[parts firstObject] isKindOfClass:NSDictionary.class] ? [parts firstObject][@"text"] : nil;- id parsed = [text isKindOfClass:NSString.class] ? [NSJSONSerialization JSONObjectWithData:[text dataUsingEncoding:NSUTF8StringEncoding] options:0 error:nil] : nil;- if ([parsed isKindOfClass:NSArray.class] && [parsed count] == texts.count) {- NSMutableArray<NSString *> *clean = [NSMutableArray arrayWithCapacity:texts.count];- for (id line in parsed) [clean addObject:[line isKindOfClass:NSString.class] ? line : @""];- translations = clean;- } else if (error) {- problem = @"Gemini could not be reached.";- } else if (status == 400 || status == 403) {- problem = @"Gemini turned the key down. Check it on the Lyrics page.";- } else if (status == 429) {- problem = @"Gemini's limit for this key is reached for now.";- } else {- problem = [NSString stringWithFormat:@"Gemini did not translate the song (%ld).", (long)status];- }+ NSArray *translations = SGGeminiTranslationsIn(root, status, error, texts.count, &problem);SGLog(@"gemini: %@ lines into %@: %@", @(texts.count), languageTag, translations ? @"translated" : problem);dispatch_async(dispatch_get_main_queue(), ^{if (translations && trackID) sg_done[memo] = translations;