Skip to content

Liquid Glass and Apple Music touches for the Spotify iOS app. A GPL-3.0 fork of spoti.pw 0.21.1.

C

fix: Gemini fills partial translations and explains failures

My-Name-Is-Jeff353785fbb520
diff --git a/harness/lyrics-translation/README.md b/harness/lyrics-translation/README.md
new file mode 100644
index 00000000..36f8c959
--- /dev/null
+++ b/harness/lyrics-translation/README.md
@@ -0,0 +1,11 @@
+# Lyrics translation check
+
+`./build.sh` checks the parts of lyrics translation that need no network: Musixmatch's community
+translations matched onto a song's lines by their folded text, as copies that leave the lines kept
+before untouched (`SGMusixmatchTranslatedLines`), and Gemini's reply read into one translation per line
+or a reason to show (`SGGeminiTranslationsIn`): a blocked prompt, a recitation or safety stop, a line
+count that does not match, the key, the limit. It runs on the Mac as a Mac Catalyst binary, with no
+simulator and no network.
+
+It does not cover the requests themselves, the lines taking the translations as they are kept
+(`Shared/Lyrics/KaraokeSource.x`), or the lyrics view picking them up.
diff --git a/harness/lyrics-translation/build.sh b/harness/lyrics-translation/build.sh
new file mode 100755
index 00000000..61599a53
--- /dev/null
+++ b/harness/lyrics-translation/build.sh
@@ -0,0 +1,12 @@
+#!/bin/sh
+# build.sh: builds check.m against this checkout's Musixmatch.m, LyricsTranslation.m and KaraokeTiming.m
+# as a Mac Catalyst binary and runs it, so no simulator is needed.
+set -e
+HERE=$(cd "$(dirname "$0")" && pwd)
+SRC=$(cd "$HERE/../../tweak/Sources" && pwd)
+mkdir -p "$HERE/build"
+xcrun -sdk macosx clang -target arm64-apple-ios17.0-macabi -isysroot "$(xcrun --sdk macosx --show-sdk-path)" -iframework "$(xcrun --sdk macosx --show-sdk-path)/System/iOSSupport/System/Library/Frameworks" -fobjc-arc -g -Wall -I"$SRC" \
+ "$HERE/check.m" "$SRC/Shared/LyricsSources/Musixmatch.m" "$SRC/Shared/LyricsTranslation/LyricsTranslation.m" \
+ "$SRC/Shared/Lyrics/KaraokeTiming.m" "$SRC/Shared/Lyrics/Protobuf.m" \
+ -framework UIKit -framework Security -framework Foundation -o "$HERE/build/check"
+"$HERE/build/check"
diff --git a/harness/lyrics-translation/check.m b/harness/lyrics-translation/check.m
new file mode 100644
index 00000000..2a371d54
--- /dev/null
+++ b/harness/lyrics-translation/check.m
@@ -0,0 +1,63 @@
+// Checks the parts of lyrics translation that need no network: Musixmatch's community translations
+// matched onto a song's lines as copies, and Gemini's reply read into lines or a reason. Built against
+// this checkout's Musixmatch.m, LyricsTranslation.m and KaraokeTiming.m. Exits non-zero on the first
+// wrong answer.
+#import <UIKit/UIKit.h>
+#import "Shared/Lyrics/Lyrics.h"
+#import "Shared/LyricsSources/LyricsSources.h"
+#import "Shared/LyricsTranslation/LyricsTranslation.h"
+
+// What the files reach outside what is checked here; none of it is called.
+@implementation SGLyricsResult
+@end
+void SGLyricsNoteReply(NSURLResponse *response, NSError *error) {}
+NSString *SGLyricsTranslationLanguage(void) { return nil; }
+SGModRow *SGStatActionRow(NSString *t, NSString *s, NSString *(^v)(void), void (^a)(void)) { return nil; }
+UIViewController *SGTopController(void) { return nil; }
+
+#define CHECK(cond) do { if (!(cond)) { fprintf(stderr, "FAILED line %d: %s\n", __LINE__, #cond); exit(1); } } while (0)
+
+static NSArray<NSString *> *readReply(NSString *json, NSInteger status, NSUInteger count, NSString **problem) {
+ *problem = nil;
+ id root = [NSJSONSerialization JSONObjectWithData:[json dataUsingEncoding:NSUTF8StringEncoding] options:0 error:nil];
+ return SGGeminiTranslationsIn(root, status, nil, count, problem);
+}
+
+int main(void) {
+ @autoreleasepool {
+ // Musixmatch's translations go on copies of the lines they match, by the folded text.
+ NSArray<SGKaraokeLine *> *lines = SGKaraokeEstimatedLines(@[@0, @2000, @4000, @6000],
+ @[@"Où est l'amour ?", @"Yeah", @"Rien", @""]);
+ CHECK(lines.count >= 3);
+ lines[2].translation = @"Nothing";
+ NSDictionary *byLine = @{SGMusixmatchLineKey(@"ou est L'AMOUR"): @"Where is love?", SGMusixmatchLineKey(@"rien"): @"None"};
+ NSArray<SGKaraokeLine *> *translated = SGMusixmatchTranslatedLines(lines, byLine);
+ CHECK(translated && translated != lines && translated.count == lines.count);
+ CHECK([translated[0].translation isEqualToString:@"Where is love?"] && translated[0] != lines[0]);
+ CHECK(!lines[0].translation); // the lines kept before are left as they were
+ CHECK(translated[0].start == lines[0].start && translated[0].end == lines[0].end
+ && translated[0].words == lines[0].words && translated[0].timing == lines[0].timing);
+ CHECK(translated[1] == lines[1] && !translated[1].translation); // no match: the same line
+ CHECK([translated[2].translation isEqualToString:@"Nothing"]); // a translation of its own stays
+ CHECK(!SGMusixmatchTranslatedLines(lines, @{@"nothing here": @"x"}));
+
+ // Gemini's language by the whole tag, so the script survives.
+ NSLocale *english = [NSLocale localeWithLocaleIdentifier:@"en"];
+ CHECK(![[english localizedStringForLocaleIdentifier:@"zh-Hant"] isEqualToString:[english localizedStringForLocaleIdentifier:@"zh-Hans"]]);
+
+ // Gemini's reply: lines, or why there are none.
+ NSString *problem;
+ NSArray<NSString *> *got = readReply(@"{\"candidates\":[{\"content\":{\"parts\":[{\"text\":\"[\\\"a\\\",\\\"\\\"]\"}]},\"finishReason\":\"STOP\"}]}", 200, 2, &problem);
+ CHECK([got isEqualToArray:(@[@"a", @""])] && !problem);
+ CHECK(!readReply(@"{\"promptFeedback\":{\"blockReason\":\"PROHIBITED_CONTENT\"}}", 200, 2, &problem) && [problem containsString:@"filters"]);
+ CHECK(!readReply(@"{\"candidates\":[{\"finishReason\":\"RECITATION\"}]}", 200, 2, &problem) && [problem containsString:@"repeated"]);
+ CHECK(!readReply(@"{\"candidates\":[{\"finishReason\":\"SAFETY\"}]}", 200, 2, &problem) && [problem containsString:@"filters"]);
+ CHECK(!readReply(@"{\"candidates\":[{\"content\":{\"parts\":[{\"text\":\"[\\\"a\\\"]\"}]},\"finishReason\":\"STOP\"}]}", 200, 2, &problem)
+ && [problem containsString:@"1 lines for the song's 2"]);
+ CHECK(!readReply(@"{\"error\":{}}", 403, 2, &problem) && [problem containsString:@"key"]);
+ CHECK(!readReply(@"{\"error\":{}}", 429, 2, &problem) && [problem containsString:@"limit"]);
+ CHECK(!readReply(@"{}", 200, 2, &problem) && [problem containsString:@"could not be read"]);
+ puts("lyrics translation: all checks passed");
+ }
+ return 0;
+}
diff --git a/tweak/Sources/Redesigned/Lyrics/SGRKaraokeView.m b/tweak/Sources/Redesigned/Lyrics/SGRKaraokeView.m
index 026fd7c5..81b4265d 100644
--- a/tweak/Sources/Redesigned/Lyrics/SGRKaraokeView.m
+++ b/tweak/Sources/Redesigned/Lyrics/SGRKaraokeView.m
@@ -1096,6 +1096,7 @@ typedef struct {
CFTimeInterval _stillSince; // when the position stopped moving, 0 while it moves
SGRKaraokeStyle *_style; // how the lines were laid out
BOOL _hasSpoken, _hasTranslation; // whether the song has any line with either
+ BOOL _untranslated; // whether a line with words has no translation, which Gemini can fill in
NSArray<SGKaraokeLine *> *_passedOver; // lines kept for the song that changed nothing here, not looked at again
UIButton *_extras;
CGFloat _builtWidth;
@@ -1350,6 +1351,14 @@ static BOOL hasTranslations(NSArray<SGKaraokeLine *> *lines) {
return NO;
}
+// A line with words to translate: not an empty one, nor a ♪ holding a break.
+static BOOL hasWords(SGKaraokeLine *line) {
+ for (SGKaraokeWord *word in line.words) {
+ if ([word.text rangeOfCharacterFromSet:NSCharacterSet.letterCharacterSet].location != NSNotFound) return YES;
+ }
+ return NO;
+}
+
// When each line is sung and where the breaks are, worked out once per song for the frames to read.
- (void)timeLines {
free(_spans);
@@ -1358,7 +1367,7 @@ static BOOL hasTranslations(NSArray<SGKaraokeLine *> *lines) {
_spans = calloc(count + 1, sizeof(SGRKaraokeSpan));
_breaks = calloc(count + 1, sizeof(SGRKaraokeBreak));
_breakCount = 0;
- _hasSpoken = _hasTranslation = NO;
+ _hasSpoken = _hasTranslation = _untranslated = NO;
_plain = SGKaraokeLinesTiming(_lines) == SGKaraokeTimingNone;
NSInteger sungTo = 0; // the top of the song counts as where the singing before the first line ends
for (NSUInteger i = 0; i < count; i++) {
@@ -1368,6 +1377,7 @@ static BOOL hasTranslations(NSArray<SGKaraokeLine *> *lines) {
sungTo = MAX(sungTo, _spans[i].end);
_hasSpoken = _hasSpoken || line.pronunciation || line.backing.pronunciation;
_hasTranslation = _hasTranslation || line.translation.length;
+ _untranslated = _untranslated || (!line.translation.length && hasWords(line));
}
[self offerExtras];
[self askMeanings];
@@ -1488,7 +1498,8 @@ static BOOL hasTranslations(NSArray<SGKaraokeLine *> *lines) {
image:[UIImage systemImageNamed:@"character.bubble"] identifier:nil
handler:^(UIAction *action) { SGRSetLyricsTextShown(SGRLyricsTextTranslation, !on); }]];
}
- if (gemini && !_hasTranslation) {
+ // Also for a song a source translated in part, as Musixmatch's community often has: Gemini fills in the rest.
+ if (gemini && _untranslated) {
__weak SGRKaraokeView *weakSelf = self;
[items addObject:[UIAction actionWithTitle:@"Translate with Gemini" image:[UIImage systemImageNamed:@"sparkles"]
identifier:nil handler:^(UIAction *action) { [weakSelf translateWithGemini]; }]];
@@ -1498,8 +1509,8 @@ static BOOL hasTranslations(NSArray<SGKaraokeLine *> *lines) {
[self setNeedsLayout];
}
-// The song's lines into the Lyrics page's language, or the phone's: each line takes its translation, and
-// translations are switched on so the answer shows.
+// The song's lines into the Lyrics page's language, or the phone's: each line without a translation takes
+// Gemini's, and translations are switched on so the answer shows. The whole song is sent, for the sense.
- (void)translateWithGemini {
NSArray<SGKaraokeLine *> *lines = _lines;
NSString *track = _track;
@@ -1514,9 +1525,10 @@ static BOOL hasTranslations(NSArray<SGKaraokeLine *> *lines) {
}
if (lines != self->_lines) return; // the song moved on meanwhile
[lines enumerateObjectsUsingBlock:^(SGKaraokeLine *line, NSUInteger i, BOOL *stop) {
- if (translations[i].length) line.translation = translations[i];
+ if (translations[i].length && !line.translation.length) line.translation = translations[i];
}];
self->_hasTranslation = YES;
+ self->_untranslated = NO;
if (SGFlag(SGRKeyLyricsTranslation, NO)) [self restyle];
else SGRSetLyricsTextShown(SGRLyricsTextTranslation, YES);
});
diff --git a/tweak/Sources/Shared/LyricsTranslation/LyricsTranslation.h b/tweak/Sources/Shared/LyricsTranslation/LyricsTranslation.h
index 3235c173..b5d923aa 100644
--- a/tweak/Sources/Shared/LyricsTranslation/LyricsTranslation.h
+++ b/tweak/Sources/Shared/LyricsTranslation/LyricsTranslation.h
@@ -15,5 +15,9 @@ void SGLyricsTranslateWithGemini(NSString *trackID, NSArray<SGKaraokeLine *> *li
// The language a translation is asked in: the Lyrics page's, else the phone's own.
NSString *SGLyricsGeminiLanguage(void);
+// Gemini's reply read: one translation per line when it has exactly `count`, else nil and in `problem`
+// why not (the key, the limit, a filter, a recitation stop, a line count that does not match).
+NSArray<NSString *> *SGGeminiTranslationsIn(id root, NSInteger status, NSError *error, NSUInteger count, NSString **problem);
+
// The Lyrics page's row: shows whether a key is set, and sets or removes it.
SGModRow *SGGeminiKeyRow(void);
diff --git a/tweak/Sources/Shared/LyricsTranslation/LyricsTranslation.m b/tweak/Sources/Shared/LyricsTranslation/LyricsTranslation.m
index 5501c7f4..3fd86c57 100644
--- a/tweak/Sources/Shared/LyricsTranslation/LyricsTranslation.m
+++ b/tweak/Sources/Shared/LyricsTranslation/LyricsTranslation.m
@@ -44,7 +44,51 @@ BOOL SGGeminiKeySet(void) {
#pragma mark - asking
NSString *SGLyricsGeminiLanguage(void) {
- return SGLyricsTranslationLanguage() ?: [NSLocale.preferredLanguages.firstObject componentsSeparatedByString:@"-"].firstObject ?: @"en";
+ return SGLyricsTranslationLanguage() ?: NSLocale.preferredLanguages.firstObject ?: @"en";
+}
+
+// Why Gemini stopped short, as finishReason or promptFeedback.blockReason name it.
+static NSString *blockedBecause(NSString *reason) {
+ if ([reason isEqualToString:@"RECITATION"]) return @"Gemini stopped because its answer repeated published lyrics.";
+ if ([@[@"SAFETY", @"PROHIBITED_CONTENT", @"BLOCKLIST", @"SPII"] containsObject:reason]) return @"Gemini's filters blocked this song's lyrics.";
+ if ([reason isEqualToString:@"MAX_TOKENS"]) return @"The song is too long for one answer from Gemini.";
+ return [NSString stringWithFormat:@"Gemini stopped before translating the song (%@).", reason];
+}
+
+NSArray<NSString *> *SGGeminiTranslationsIn(id root, NSInteger status, NSError *error, NSUInteger count, NSString **problem) {
+ NSDictionary *reply = [root isKindOfClass:NSDictionary.class] ? root : nil;
+ id candidates = reply[@"candidates"];
+ id first = [candidates isKindOfClass:NSArray.class] ? [candidates firstObject] : nil;
+ id parts = [first isKindOfClass:NSDictionary.class] && [first[@"content"] isKindOfClass:NSDictionary.class] ? first[@"content"][@"parts"] : nil;
+ id text = [parts isKindOfClass:NSArray.class] && [[parts firstObject] isKindOfClass:NSDictionary.class] ? [parts firstObject][@"text"] : nil;
+ id parsed = [text isKindOfClass:NSString.class] ? [NSJSONSerialization JSONObjectWithData:[text dataUsingEncoding:NSUTF8StringEncoding] options:0 error:nil] : nil;
+ if ([parsed isKindOfClass:NSArray.class] && [parsed count] == count) {
+ NSMutableArray<NSString *> *clean = [NSMutableArray arrayWithCapacity:count];
+ for (id line in parsed) [clean addObject:[line isKindOfClass:NSString.class] ? line : @""];
+ return clean;
+ }
+ id feedback = reply[@"promptFeedback"];
+ id blocked = [feedback isKindOfClass:NSDictionary.class] ? feedback[@"blockReason"] : nil;
+ id finished = [first isKindOfClass:NSDictionary.class] ? first[@"finishReason"] : nil;
+ if (error) {
+ *problem = @"Gemini could not be reached.";
+ } else if (status == 400 || status == 403) {
+ *problem = @"Gemini turned the key down. Check it on the Lyrics page.";
+ } else if (status == 429) {
+ *problem = @"Gemini's limit for this key is reached for now.";
+ } else if (status != 200) {
+ *problem = [NSString stringWithFormat:@"Gemini did not translate the song (%ld).", (long)status];
+ } else if ([blocked isKindOfClass:NSString.class]) {
+ *problem = blockedBecause(blocked);
+ } else if ([finished isKindOfClass:NSString.class] && ![finished isEqualToString:@"STOP"]) {
+ *problem = blockedBecause(finished);
+ } else if ([parsed isKindOfClass:NSArray.class]) {
+ *problem = [NSString stringWithFormat:@"Gemini answered %lu lines for the song's %lu. Try again.",
+ (unsigned long)[parsed count], (unsigned long)count];
+ } else {
+ *problem = @"Gemini's answer could not be read. Try again.";
+ }
+ return nil;
}
static NSMutableDictionary<NSString *, NSArray<NSString *> *> *sg_done; // per launch, by track and language
@@ -64,15 +108,24 @@ void SGLyricsTranslateWithGemini(NSString *trackID, NSArray<SGKaraokeLine *> *li
}
NSMutableArray<NSString *> *texts = [NSMutableArray arrayWithCapacity:lines.count];
for (SGKaraokeLine *line in lines) [texts addObject:SGKaraokeLineText(line) ?: @""];
- NSString *language = [[NSLocale localeWithLocaleIdentifier:@"en"] localizedStringForLanguageCode:languageTag] ?: languageTag;
+ // By the whole tag, so the script stays: zh-Hant is "Chinese, Traditional" where its language alone is "Chinese".
+ NSString *language = [[NSLocale localeWithLocaleIdentifier:@"en"] localizedStringForLocaleIdentifier:languageTag] ?: languageTag;
NSData *input = [NSJSONSerialization dataWithJSONObject:texts options:0 error:nil];
NSString *prompt = [NSString stringWithFormat:
@"Translate the lines of this song into %@. Answer with a JSON array of exactly %lu strings, one "
@"per input line and in the same order. Translate the meaning naturally, keep each line short like a "
@"lyric, and keep an empty line empty. A line already in %@ stays as it is.\n\n%@",
language, (unsigned long)texts.count, language, [[NSString alloc] initWithData:input encoding:NSUTF8StringEncoding]];
+ // Lyrics are often explicit, and translating them is what the user asked for, so every filter that can
+ // be lowered is. Recitation cannot be switched off, and is named when it stops an answer.
+ NSMutableArray *safety = [NSMutableArray array];
+ for (NSString *category in @[@"HARM_CATEGORY_HARASSMENT", @"HARM_CATEGORY_HATE_SPEECH",
+ @"HARM_CATEGORY_SEXUALLY_EXPLICIT", @"HARM_CATEGORY_DANGEROUS_CONTENT"]) {
+ [safety addObject:@{@"category": category, @"threshold": @"BLOCK_NONE"}];
+ }
NSDictionary *body = @{
@"contents": @[@{@"parts": @[@{@"text": prompt}]}],
+ @"safetySettings": safety,
@"generationConfig": @{
@"responseMimeType": @"application/json",
@"responseSchema": @{@"type": @"ARRAY", @"items": @{@"type": @"STRING"}},
@@ -87,26 +140,8 @@ void SGLyricsTranslateWithGemini(NSString *trackID, NSArray<SGKaraokeLine *> *li
[[NSURLSession.sharedSession dataTaskWithRequest:request completionHandler:^(NSData *data, NSURLResponse *response, NSError *error) {
NSInteger status = [response isKindOfClass:NSHTTPURLResponse.class] ? ((NSHTTPURLResponse *)response).statusCode : 0;
id root = data ? [NSJSONSerialization JSONObjectWithData:data options:0 error:nil] : nil;
- NSArray *translations = nil;
NSString *problem = nil;
- id candidates = [root isKindOfClass:NSDictionary.class] ? root[@"candidates"] : nil;
- id first = [candidates isKindOfClass:NSArray.class] ? [candidates firstObject] : nil;
- id parts = [first isKindOfClass:NSDictionary.class] && [first[@"content"] isKindOfClass:NSDictionary.class] ? first[@"content"][@"parts"] : nil;
- id text = [parts isKindOfClass:NSArray.class] && [[parts firstObject] isKindOfClass:NSDictionary.class] ? [parts firstObject][@"text"] : nil;
- id parsed = [text isKindOfClass:NSString.class] ? [NSJSONSerialization JSONObjectWithData:[text dataUsingEncoding:NSUTF8StringEncoding] options:0 error:nil] : nil;
- if ([parsed isKindOfClass:NSArray.class] && [parsed count] == texts.count) {
- NSMutableArray<NSString *> *clean = [NSMutableArray arrayWithCapacity:texts.count];
- for (id line in parsed) [clean addObject:[line isKindOfClass:NSString.class] ? line : @""];
- translations = clean;
- } else if (error) {
- problem = @"Gemini could not be reached.";
- } else if (status == 400 || status == 403) {
- problem = @"Gemini turned the key down. Check it on the Lyrics page.";
- } else if (status == 429) {
- problem = @"Gemini's limit for this key is reached for now.";
- } else {
- problem = [NSString stringWithFormat:@"Gemini did not translate the song (%ld).", (long)status];
- }
+ NSArray *translations = SGGeminiTranslationsIn(root, status, error, texts.count, &problem);
SGLog(@"gemini: %@ lines into %@: %@", @(texts.count), languageTag, translations ? @"translated" : problem);
dispatch_async(dispatch_get_main_queue(), ^{
if (translations && trackID) sg_done[memo] = translations;