Skip to content

Unofficial YouTube Music / Spotify Premium client for Android, built in Liquid Glass. Audio effects, Spotify Connect, listening history, 8 languages. Rust core on librespot.

Cap concurrent track lookups and retry the dropped ones

Lelonio63c0952bb1ff
diff --git a/.claude/settings.local.json b/.claude/settings.local.json
index 35820b6..65b47c5 100644
--- a/.claude/settings.local.json
+++ b/.claude/settings.local.json
@@ -108,7 +108,9 @@
"Bash(git commit -q -m 'Credit the author in the settings *)",
"Bash(git commit -q -m 'Swap the player'\\\\''s heart for add-to-playlist *)",
"Bash(git commit -q -m 'Re-read a playlist when it is reopened *)",
- "Bash(git commit -q -m 'Load whole playlists, a batch at a time *)"
+ "Bash(git commit -q -m 'Load whole playlists, a batch at a time *)",
+ "Bash(PATH=/Users/emanuele/.cargo/bin:__TRACKED_VAR__ cargo check --quiet)",
+ "Bash(git commit -q -m 'Cap concurrent track lookups and retry the dropped ones *)"
]
}
}
diff --git a/app/.cxx/RelWithDebInfo/151m4dr6/arm64-v8a/configure_fingerprint.bin b/app/.cxx/RelWithDebInfo/151m4dr6/arm64-v8a/configure_fingerprint.bin
index 7dae8ba..82fbefe 100644
--- a/app/.cxx/RelWithDebInfo/151m4dr6/arm64-v8a/configure_fingerprint.bin
+++ b/app/.cxx/RelWithDebInfo/151m4dr6/arm64-v8a/configure_fingerprint.bin
@@ -2,27 +2,27 @@ C/C++ Structured Logl
j
h/Users/emanuele/Progetto Spotify/app/.cxx/RelWithDebInfo/151m4dr6/arm64-v8a/additional_project_files.txtC
A
-?com.android.build.gradle.internal.cxx.io.EncodedFileFingerPrint  �����3  ����3i
+?com.android.build.gradle.internal.cxx.io.EncodedFileFingerPrint  ����3  ����3i
g
-e/Users/emanuele/Progetto Spotify/app/.cxx/RelWithDebInfo/151m4dr6/arm64-v8a/android_gradle_build.json  �����3� ����3n
+e/Users/emanuele/Progetto Spotify/app/.cxx/RelWithDebInfo/151m4dr6/arm64-v8a/android_gradle_build.json  ����3� ����3n
l
-j/Users/emanuele/Progetto Spotify/app/.cxx/RelWithDebInfo/151m4dr6/arm64-v8a/android_gradle_build_mini.json  �����3� �����3[
+j/Users/emanuele/Progetto Spotify/app/.cxx/RelWithDebInfo/151m4dr6/arm64-v8a/android_gradle_build_mini.json  ����3� �����3[
Y
-W/Users/emanuele/Progetto Spotify/app/.cxx/RelWithDebInfo/151m4dr6/arm64-v8a/build.ninja  �����3�� ����3_
+W/Users/emanuele/Progetto Spotify/app/.cxx/RelWithDebInfo/151m4dr6/arm64-v8a/build.ninja  ����3�� ����3_
]
-[/Users/emanuele/Progetto Spotify/app/.cxx/RelWithDebInfo/151m4dr6/arm64-v8a/build.ninja.txt  �����3d
+[/Users/emanuele/Progetto Spotify/app/.cxx/RelWithDebInfo/151m4dr6/arm64-v8a/build.ninja.txt  ����3d
b
-`/Users/emanuele/Progetto Spotify/app/.cxx/RelWithDebInfo/151m4dr6/arm64-v8a/build_file_index.txt  �����3 @ ����3e
+`/Users/emanuele/Progetto Spotify/app/.cxx/RelWithDebInfo/151m4dr6/arm64-v8a/build_file_index.txt  ����3 @ ����3e
c
-a/Users/emanuele/Progetto Spotify/app/.cxx/RelWithDebInfo/151m4dr6/arm64-v8a/compile_commands.json  �����3�� ����3i
+a/Users/emanuele/Progetto Spotify/app/.cxx/RelWithDebInfo/151m4dr6/arm64-v8a/compile_commands.json  ����3�� ����3i
g
-e/Users/emanuele/Progetto Spotify/app/.cxx/RelWithDebInfo/151m4dr6/arm64-v8a/compile_commands.json.bin  �����3 �" ����3o
+e/Users/emanuele/Progetto Spotify/app/.cxx/RelWithDebInfo/151m4dr6/arm64-v8a/compile_commands.json.bin  ����3 �" ����3o
m
-k/Users/emanuele/Progetto Spotify/app/.cxx/RelWithDebInfo/151m4dr6/arm64-v8a/metadata_generation_command.txt  �����3
+k/Users/emanuele/Progetto Spotify/app/.cxx/RelWithDebInfo/151m4dr6/arm64-v8a/metadata_generation_command.txt  ����3
� ����3b
`
-^/Users/emanuele/Progetto Spotify/app/.cxx/RelWithDebInfo/151m4dr6/arm64-v8a/prefab_config.json  �����3  ( ����3g
+^/Users/emanuele/Progetto Spotify/app/.cxx/RelWithDebInfo/151m4dr6/arm64-v8a/prefab_config.json  ����3  ( ����3g
e
-c/Users/emanuele/Progetto Spotify/app/.cxx/RelWithDebInfo/151m4dr6/arm64-v8a/symbol_folder_index.txt  �����3  b ����3D
+c/Users/emanuele/Progetto Spotify/app/.cxx/RelWithDebInfo/151m4dr6/arm64-v8a/symbol_folder_index.txt  ����3  b ����3D
B
-@/Users/emanuele/Progetto Spotify/app/src/main/cpp/CMakeLists.txt  �����3 � �Ȅ��3
\ No newline at end of file
+@/Users/emanuele/Progetto Spotify/app/src/main/cpp/CMakeLists.txt  ����3 � �Ȅ��3
\ No newline at end of file
diff --git a/app/src/main/java/dev/emanuele/spot/ui/MainViewModel.kt b/app/src/main/java/dev/emanuele/spot/ui/MainViewModel.kt
index aea09ae..234cd51 100644
--- a/app/src/main/java/dev/emanuele/spot/ui/MainViewModel.kt
+++ b/app/src/main/java/dev/emanuele/spot/ui/MainViewModel.kt
@@ -656,12 +656,17 @@ class MainViewModel(app: Application) : AndroidViewModel(app) {
return
}
+ val batches = uris.chunked(METADATA_BATCH)
val loaded = mutableListOf<CatalogTrack>()
- uris.chunked(METADATA_BATCH).forEach { batch ->
+ batches.forEachIndexed { index, batch ->
loaded += Catalog.tracks(batch)
_playlist.value = base.copy(
tracks = loaded.toList(),
- loadingMore = loaded.size < uris.size,
+ // Counted in batches, not in tracks. Comparing the two totals
+ // left "more coming" on forever whenever anything was dropped —
+ // a delisted or region-locked track resolves to nothing, so the
+ // list is legitimately shorter than the URIs asked for.
+ loadingMore = index < batches.lastIndex,
)
}
}
diff --git a/native/src/catalog.rs b/native/src/catalog.rs
index 47be438..dfedf3d 100644
--- a/native/src/catalog.rs
+++ b/native/src/catalog.rs
@@ -11,7 +11,7 @@
//! protobufs into Java objects field by field.
use crate::engine::{with_session, EngineResult};
-use futures_util::future::join_all;
+use futures_util::stream::{self, StreamExt};
use http::Method;
use librespot_core::{session::Session, spotify_uri::SpotifyUri};
use librespot_metadata::{image::Images, Metadata, Track};
@@ -118,12 +118,22 @@ pub fn context_tracks(uri: &str) -> EngineResult<String> {
}
}
- // An empty page carries a link instead of its contents.
+ // An empty page carries a link instead of its contents, and a
+ // long playlist is a chain of those: each resolved page names the
+ // next one. Following only the first is how a 2000-track playlist
+ // used to arrive with a few hundred in it.
if page.tracks.is_empty() {
- if let Some(page_url) = page.page_url.as_ref().filter(|u| !u.is_empty()) {
- match resolve_page(&session, page_url).await {
- Ok(mut resolved) => uris.append(&mut resolved),
- Err(e) => log::warn!("could not resolve page {page_url}: {e}"),
+ let mut next = page.page_url.as_ref().filter(|u| !u.is_empty()).cloned();
+ while let Some(page_url) = next {
+ match resolve_page(&session, &page_url).await {
+ Ok((mut resolved, following)) => {
+ uris.append(&mut resolved);
+ next = following.filter(|u| !u.is_empty() && *u != page_url);
+ }
+ Err(e) => {
+ log::warn!("could not resolve page: {e}");
+ next = None;
+ }
}
}
}
@@ -134,8 +144,11 @@ pub fn context_tracks(uri: &str) -> EngineResult<String> {
})
}
-/// Fetch a lazy context page and pull the track URIs out of it.
-async fn resolve_page(session: &Session, page_url: &str) -> Result<Vec<String>, String> {
+/// Fetch a lazy context page: its track URIs, and the page after it if any.
+async fn resolve_page(
+ session: &Session,
+ page_url: &str,
+) -> Result<(Vec<String>, Option<String>), String> {
let bytes = session
.spclient()
.get_next_page(page_url)
@@ -150,17 +163,26 @@ async fn resolve_page(session: &Session, page_url: &str) -> Result<Vec<String>,
.and_then(Value::as_array)
.ok_or_else(|| {
// The exact shape is not documented; log enough to fix this quickly
- // if Spotify returns something else.
+ // if Spotify returns something else. Key names only — the values are
+ // the user's own library.
format!(
"no `tracks` array; keys were {:?}",
parsed.as_object().map(|o| o.keys().collect::<Vec<_>>())
)
})?;
- Ok(tracks
+ // Spelling varies by endpoint, and an unknown one simply ends the walk.
+ let next = ["next_page_url", "nextPageUrl", "next"]
+ .iter()
+ .find_map(|key| parsed.get(*key).and_then(Value::as_str))
+ .map(str::to_owned);
+
+ let uris = tracks
.iter()
.filter_map(|t| t.get("uri").and_then(Value::as_str).map(str::to_owned))
- .collect())
+ .collect();
+
+ Ok((uris, next))
}
/// Resolve display metadata for the given track URIs, in the order supplied.
@@ -169,28 +191,68 @@ async fn resolve_page(session: &Session, page_url: &str) -> Result<Vec<String>,
/// because each one is a separate access-point round trip and a playlist screen
/// needs dozens of them; doing it sequentially is the difference between a
/// screen that appears and one that fills in line by line.
+///
+/// Concurrency is capped, though, and that cap is the whole point of this
+/// function's shape. Firing a batch off at once made a long playlist load a
+/// different number of tracks every time it was opened — a few hundred, then
+/// thirty — because the access point drops requests once enough are in flight
+/// and a dropped lookup was silently skipped. A bounded window plus one retry
+/// turns that into a wait instead of a hole in the list.
pub fn tracks_metadata(uris_json: &str) -> EngineResult<String> {
let uris: Vec<String> =
serde_json::from_str(uris_json).map_err(|e| format!("invalid uri list: {e}"))?;
let session = with_session(|s| s.clone())?;
+ let requested = uris.len();
block_on(async move {
- let lookups = uris.iter().map(|uri| {
+ let lookups = stream::iter(uris.iter().map(|uri| {
let session = session.clone();
async move {
let parsed = SpotifyUri::from_uri(uri).ok()?;
- let track = Track::get(&session, &parsed).await.ok()?;
- Some(track_json(&track))
+ for attempt in 0..RESOLVE_ATTEMPTS {
+ match Track::get(&session, &parsed).await {
+ Ok(track) => return Some(track_json(&track)),
+ Err(e) => {
+ // Last attempt, or nothing to gain from waiting.
+ if attempt + 1 == RESOLVE_ATTEMPTS {
+ log::debug!("track lookup failed after retries: {e}");
+ } else {
+ tokio::time::sleep(RETRY_DELAY).await;
+ }
+ }
+ }
+ }
+ None
}
- });
+ }))
+ // `buffered`, not `buffer_unordered`: the caller relies on the order it
+ // asked in, which is the order the playlist is in.
+ .buffered(RESOLVE_CONCURRENCY);
// Unresolvable tracks are dropped rather than failing the whole batch:
// one region-locked or delisted item should not empty a playlist.
- let tracks: Vec<Value> = join_all(lookups).await.into_iter().flatten().collect();
+ let tracks: Vec<Value> = lookups.filter_map(|t| async move { t }).collect().await;
+
+ // Counts only — these are the user's own library.
+ if tracks.len() != requested {
+ log::warn!("resolved {} of {} tracks", tracks.len(), requested);
+ }
Ok(Value::from(tracks).to_string())
})
}
+/// How many track lookups may be in flight at once.
+///
+/// Empirical: the access point starts dropping them well before a hundred, and
+/// this is comfortably under whatever that threshold actually is while still
+/// being far quicker than resolving one at a time.
+const RESOLVE_CONCURRENCY: usize = 12;
+
+/// Attempts per track, the first one included.
+const RESOLVE_ATTEMPTS: usize = 3;
+
+const RETRY_DELAY: std::time::Duration = std::time::Duration::from_millis(220);
+
fn track_json(track: &Track) -> Value {
json!({
"uri": track.id.to_uri().unwrap_or_default(),