Skip to content

StreamVault is a TV-first free IPTV player for Android TV built with Kotlin, Jetpack Compose, Room, Hilt, and Media3.

Harden grouped cursor pagination

David Nashash0f352e2b282c
diff --git a/data/src/main/java/com/streamvault/data/local/dao/Daos.kt b/data/src/main/java/com/streamvault/data/local/dao/Daos.kt
index 2e18052e..b9bf0999 100644
--- a/data/src/main/java/com/streamvault/data/local/dao/Daos.kt
+++ b/data/src/main/java/com/streamvault/data/local/dao/Daos.kt
@@ -1394,7 +1394,7 @@ interface MovieDao {
limit: Int
): List<MovieBrowseEntity>
- @Query("SELECT * FROM movies WHERE provider_id = :providerId ORDER BY added_at DESC, name ASC, id ASC LIMIT :limit")
+ @Query("SELECT * FROM movies WHERE provider_id = :providerId AND added_at > 0 ORDER BY added_at DESC, name ASC, id ASC LIMIT :limit")
fun getFreshPreview(providerId: Long, limit: Int): Flow<List<MovieBrowseEntity>>
@Query(
@@ -1451,7 +1451,7 @@ interface MovieDao {
limit: Int
): List<MovieBrowseEntity>
- @Query("SELECT * FROM movies WHERE provider_id = :providerId AND category_id = :categoryId ORDER BY added_at DESC, name ASC, id ASC LIMIT :limit")
+ @Query("SELECT * FROM movies WHERE provider_id = :providerId AND category_id = :categoryId AND added_at > 0 ORDER BY added_at DESC, name ASC, id ASC LIMIT :limit")
fun getFreshByCategoryPreview(providerId: Long, categoryId: Long, limit: Int): Flow<List<MovieBrowseEntity>>
@Query(
@@ -2448,14 +2448,13 @@ interface SeriesDao {
)
fun getFreshCountByProvider(providerId: Long): Flow<Int>
- @Query("SELECT * FROM series WHERE provider_id = :providerId AND last_modified > 0 ORDER BY last_modified DESC, name ASC, id ASC LIMIT :limit")
+ @Query("SELECT * FROM series WHERE provider_id = :providerId ORDER BY last_modified DESC, name ASC, id ASC LIMIT :limit")
suspend fun getFreshCursorPage(providerId: Long, limit: Int): List<SeriesBrowseEntity>
@Query(
"""
SELECT * FROM series
WHERE provider_id = :providerId
- AND last_modified > 0
AND (
last_modified < :lastModified
OR (last_modified = :lastModified AND (name > :lastName OR (name = :lastName AND id > :lastId)))
@@ -2501,7 +2500,7 @@ interface SeriesDao {
)
fun getFreshCountByCategory(providerId: Long, categoryId: Long): Flow<Int>
- @Query("SELECT * FROM series WHERE provider_id = :providerId AND category_id = :categoryId AND last_modified > 0 ORDER BY last_modified DESC, name ASC, id ASC LIMIT :limit")
+ @Query("SELECT * FROM series WHERE provider_id = :providerId AND category_id = :categoryId ORDER BY last_modified DESC, name ASC, id ASC LIMIT :limit")
suspend fun getFreshByCategoryCursorPage(providerId: Long, categoryId: Long, limit: Int): List<SeriesBrowseEntity>
@Query(
@@ -2509,7 +2508,6 @@ interface SeriesDao {
SELECT * FROM series
WHERE provider_id = :providerId
AND category_id = :categoryId
- AND last_modified > 0
AND (
last_modified < :lastModified
OR (last_modified = :lastModified AND (name > :lastName OR (name = :lastName AND id > :lastId)))
diff --git a/data/src/main/java/com/streamvault/data/repository/MovieRepositoryImpl.kt b/data/src/main/java/com/streamvault/data/repository/MovieRepositoryImpl.kt
index 795bc17c..fd312fb2 100644
--- a/data/src/main/java/com/streamvault/data/repository/MovieRepositoryImpl.kt
+++ b/data/src/main/java/com/streamvault/data/repository/MovieRepositoryImpl.kt
@@ -140,6 +140,12 @@ class MovieRepositoryImpl @Inject constructor(
val id: Long
)
+ private data class CursorWindowResult<T>(
+ val items: List<T>,
+ val totalVisibleCount: Int,
+ val exhausted: Boolean
+ )
+
private val xtreamCategoryLoadLocks = KeyedMutexRegistry<String>()
private val backgroundRefreshes = BoundedKeySet<String>(MAX_BACKGROUND_CATEGORY_REFRESHES)
private val repositoryScope = CoroutineScope(
@@ -1219,10 +1225,12 @@ class MovieRepositoryImpl @Inject constructor(
.map { it.contentId }
.toSet()
- val canUseCursorWindow = supportsCursorBrowse(query)
- val items = if (canUseCursorWindow) {
- fetchMovieCursorWindow(query, favoriteIds)
+ val cursorWindow = if (supportsCursorBrowse(query)) {
+ fetchMovieCursorWindow(query, favoriteIds, presentationSettings)
} else {
+ null
+ }
+ val items = cursorWindow?.items ?: run {
val movies = movieBrowseSource(query).first()
val history = playbackHistoryDao.getByProvider(query.providerId).first()
val inProgressIds = history
@@ -1247,7 +1255,34 @@ class MovieRepositoryImpl @Inject constructor(
.take(query.limit)
}
- val totalCount = rawTotalCount
+ val totalCount = if (cursorWindow != null) {
+ if (cursorWindow.exhausted) cursorWindow.totalVisibleCount else rawTotalCount
+ } else if (presentationSettings.duplicateHandlingMode == VodDuplicateHandlingMode.SHOW_ALL) {
+ rawTotalCount
+ } else {
+ val movies = movieBrowseSource(query).first()
+ val history = playbackHistoryDao.getByProvider(query.providerId).first()
+ val inProgressIds = history
+ .asSequence()
+ .filter { it.contentType == ContentType.MOVIE }
+ .filter { it.resumePositionMs > 0L && (it.totalDurationMs <= 0L || !moviePlaybackComplete(it.resumePositionMs, it.totalDurationMs)) }
+ .map { it.contentId }
+ .toSet()
+ val watchCounts = history
+ .asSequence()
+ .filter { it.contentType == ContentType.MOVIE }
+ .associate { it.contentId to it.watchCount }
+ buildPresentedMovies(
+ applyMovieBrowseQuery(
+ movies = movies,
+ query = query,
+ favoriteIds = favoriteIds,
+ inProgressIds = inProgressIds,
+ watchCounts = watchCounts
+ ),
+ presentationSettings
+ ).size
+ }
val hasMoreRemote = query.categoryId?.let { categoryId ->
val provider = loadCompatibilityProvider(query.providerId)
@@ -1324,16 +1359,18 @@ class MovieRepositoryImpl @Inject constructor(
private suspend fun fetchMovieCursorWindow(
query: LibraryBrowseQuery,
- favoriteIds: Set<Long>
- ): List<Movie> {
+ favoriteIds: Set<Long>,
+ presentationSettings: MoviePresentationSettings
+ ): CursorWindowResult<Movie> {
val parentalLevel = preferencesRepository.parentalControlLevel.first()
val targetVisibleCount = (query.offset + query.limit).coerceAtLeast(query.limit)
val collected = ArrayList<Movie>(targetVisibleCount)
- when {
+ val exhausted = when {
query.filterBy.type == LibraryFilterType.ALL &&
query.sortBy == LibrarySortBy.LIBRARY -> {
collectMoviePages<FreshCursor>(query, parentalLevel, collected, favoriteIds,
+ presentationSettings,
extractCursor = { FreshCursor(it.addedAt, it.name, it.id) }
) { limit, cursor ->
loadMovieFreshPage(query, limit, cursor)
@@ -1342,6 +1379,7 @@ class MovieRepositoryImpl @Inject constructor(
query.filterBy.type == LibraryFilterType.ALL &&
query.sortBy == LibrarySortBy.TITLE -> {
collectMoviePages<NameCursor>(query, parentalLevel, collected, favoriteIds,
+ presentationSettings,
extractCursor = { NameCursor(it.name, it.id) }
) { limit, cursor ->
loadMovieNamePage(query, limit, cursor)
@@ -1349,6 +1387,7 @@ class MovieRepositoryImpl @Inject constructor(
}
(query.sortBy == LibrarySortBy.RATING || query.filterBy.type == LibraryFilterType.TOP_RATED) -> {
collectMoviePages<RatingCursor>(query, parentalLevel, collected, favoriteIds,
+ presentationSettings,
extractCursor = { RatingCursor(it.rating, it.name, it.id) }
) { limit, cursor ->
loadMovieRatingPage(query, limit, cursor)
@@ -1357,14 +1396,21 @@ class MovieRepositoryImpl @Inject constructor(
query.sortBy == LibrarySortBy.UPDATED ||
query.filterBy.type == LibraryFilterType.RECENTLY_UPDATED -> {
collectMoviePages<FreshCursor>(query, parentalLevel, collected, favoriteIds,
+ presentationSettings,
extractCursor = { FreshCursor(it.addedAt, it.name, it.id) }
) { limit, cursor ->
loadMovieFreshPage(query, limit, cursor)
}
}
+ else -> true
}
- return collected.drop(query.offset).take(query.limit)
+ val presented = buildPresentedMovies(collected, presentationSettings)
+ return CursorWindowResult(
+ items = presented.drop(query.offset).take(query.limit),
+ totalVisibleCount = presented.size,
+ exhausted = exhausted
+ )
}
private suspend fun <C> collectMoviePages(
@@ -1372,15 +1418,16 @@ class MovieRepositoryImpl @Inject constructor(
parentalLevel: Int,
collected: MutableList<Movie>,
favoriteIds: Set<Long>,
+ presentationSettings: MoviePresentationSettings,
extractCursor: (MovieBrowseEntity) -> C,
loadPage: suspend (limit: Int, cursor: C?) -> List<MovieBrowseEntity>
- ) {
+ ): Boolean {
var cursor: C? = null
- val targetVisibleCount = query.offset + query.limit
- while (collected.size < targetVisibleCount) {
+ val targetVisibleCount = (query.offset + query.limit).coerceAtLeast(query.limit)
+ while (presentedMovieCount(collected, presentationSettings) < targetVisibleCount) {
val batch = loadPage(CURSOR_BATCH_SIZE, cursor)
if (batch.isEmpty()) {
- return
+ return true
}
val visibleBatch = if (parentalLevel >= 3) {
batch.filterNot { it.isUserProtected }
@@ -1392,10 +1439,20 @@ class MovieRepositoryImpl @Inject constructor(
if (movie.id in favoriteIds) movie.copy(isFavorite = true) else movie
}
if (batch.size < CURSOR_BATCH_SIZE) {
- return
+ return true
}
cursor = extractCursor(batch.last())
}
+ return false
+ }
+
+ private fun presentedMovieCount(
+ movies: List<Movie>,
+ presentationSettings: MoviePresentationSettings
+ ): Int = if (presentationSettings.duplicateHandlingMode == VodDuplicateHandlingMode.SHOW_ALL) {
+ movies.size
+ } else {
+ buildPresentedMovies(movies, presentationSettings).size
}
private suspend fun loadMovieNamePage(query: LibraryBrowseQuery, limit: Int, cursor: NameCursor?): List<MovieBrowseEntity> {
diff --git a/data/src/main/java/com/streamvault/data/repository/SeriesRepositoryImpl.kt b/data/src/main/java/com/streamvault/data/repository/SeriesRepositoryImpl.kt
index 14362de2..0fc20632 100644
--- a/data/src/main/java/com/streamvault/data/repository/SeriesRepositoryImpl.kt
+++ b/data/src/main/java/com/streamvault/data/repository/SeriesRepositoryImpl.kt
@@ -115,6 +115,12 @@ class SeriesRepositoryImpl @Inject constructor(
val id: Long
)
+ private data class CursorWindowResult<T>(
+ val items: List<T>,
+ val totalVisibleCount: Int,
+ val exhausted: Boolean
+ )
+
private val xtreamCategoryLoadLocks = KeyedMutexRegistry<String>()
private val backgroundRefreshes = BoundedKeySet<String>(MAX_BACKGROUND_CATEGORY_REFRESHES)
private val repositoryScope = CoroutineScope(
@@ -1159,10 +1165,12 @@ class SeriesRepositoryImpl @Inject constructor(
.map { it.contentId }
.toSet()
- val canUseCursorWindow = supportsCursorBrowse(query)
- val items = if (canUseCursorWindow) {
- fetchSeriesCursorWindow(query, favoriteIds)
+ val cursorWindow = if (supportsCursorBrowse(query)) {
+ fetchSeriesCursorWindow(query, favoriteIds, presentationSettings)
} else {
+ null
+ }
+ val items = cursorWindow?.items ?: run {
val series = seriesBrowseSource(query).first()
val history = playbackHistoryDao.getByProvider(query.providerId).first()
val inProgressIds = history
@@ -1200,7 +1208,47 @@ class SeriesRepositoryImpl @Inject constructor(
.take(query.limit)
}
- val totalCount = rawTotalCount
+ val totalCount = if (cursorWindow != null) {
+ if (cursorWindow.exhausted) cursorWindow.totalVisibleCount else rawTotalCount
+ } else if (presentationSettings.duplicateHandlingMode == VodDuplicateHandlingMode.SHOW_ALL) {
+ rawTotalCount
+ } else {
+ val series = seriesBrowseSource(query).first()
+ val history = playbackHistoryDao.getByProvider(query.providerId).first()
+ val inProgressIds = history
+ .asSequence()
+ .filter { it.contentType == ContentType.SERIES || it.contentType == ContentType.SERIES_EPISODE }
+ .filter {
+ it.resumePositionMs > 0L && (
+ it.totalDurationMs <= 0L ||
+ it.resumePositionMs < (it.totalDurationMs * 0.95f).toLong()
+ )
+ }
+ .mapNotNull { it.seriesId ?: it.contentId }
+ .toSet()
+ val completedSeriesIds = history
+ .asSequence()
+ .filter { it.contentType == ContentType.SERIES_EPISODE }
+ .filter { it.totalDurationMs > 0L && it.resumePositionMs >= (it.totalDurationMs * 0.95f).toLong() }
+ .mapNotNull { it.seriesId }
+ .toSet()
+ val watchCounts = history
+ .asSequence()
+ .filter { it.contentType == ContentType.SERIES || it.contentType == ContentType.SERIES_EPISODE }
+ .groupBy { it.seriesId ?: it.contentId }
+ .mapValues { (_, entries) -> entries.maxOf { it.watchCount } }
+ buildPresentedSeries(
+ applySeriesBrowseQuery(
+ series = series,
+ query = query,
+ favoriteIds = favoriteIds,
+ inProgressIds = inProgressIds,
+ completedSeriesIds = completedSeriesIds,
+ watchCounts = watchCounts
+ ),
+ presentationSettings
+ ).size
+ }
val hasMoreRemote = query.categoryId?.let { categoryId ->
val provider = loadCompatibilityProvider(query.providerId)
@@ -1277,16 +1325,18 @@ class SeriesRepositoryImpl @Inject constructor(
private suspend fun fetchSeriesCursorWindow(
query: LibraryBrowseQuery,
- favoriteIds: Set<Long>
- ): List<Series> {
+ favoriteIds: Set<Long>,
+ presentationSettings: SeriesPresentationSettings
+ ): CursorWindowResult<Series> {
val parentalLevel = preferencesRepository.parentalControlLevel.first()
val targetVisibleCount = (query.offset + query.limit).coerceAtLeast(query.limit)
val collected = ArrayList<Series>(targetVisibleCount)
- when {
+ val exhausted = when {
query.filterBy.type == LibraryFilterType.ALL &&
query.sortBy == LibrarySortBy.LIBRARY -> {
collectSeriesPages<FreshCursor>(query, parentalLevel, collected, favoriteIds,
+ presentationSettings,
extractCursor = { FreshCursor(it.lastModified, it.name, it.id) }
) { limit, cursor ->
loadSeriesFreshPage(query, limit, cursor)
@@ -1295,6 +1345,7 @@ class SeriesRepositoryImpl @Inject constructor(
query.filterBy.type == LibraryFilterType.ALL &&
query.sortBy == LibrarySortBy.TITLE -> {
collectSeriesPages<NameCursor>(query, parentalLevel, collected, favoriteIds,
+ presentationSettings,
extractCursor = { NameCursor(it.name, it.id) }
) { limit, cursor ->
loadSeriesNamePage(query, limit, cursor)
@@ -1302,6 +1353,7 @@ class SeriesRepositoryImpl @Inject constructor(
}
(query.sortBy == LibrarySortBy.RATING || query.filterBy.type == LibraryFilterType.TOP_RATED) -> {
collectSeriesPages<RatingCursor>(query, parentalLevel, collected, favoriteIds,
+ presentationSettings,
extractCursor = { RatingCursor(it.rating, it.name, it.id) }
) { limit, cursor ->
loadSeriesRatingPage(query, limit, cursor)
@@ -1310,14 +1362,21 @@ class SeriesRepositoryImpl @Inject constructor(
query.sortBy == LibrarySortBy.UPDATED ||
query.filterBy.type == LibraryFilterType.RECENTLY_UPDATED -> {
collectSeriesPages<FreshCursor>(query, parentalLevel, collected, favoriteIds,
+ presentationSettings,
extractCursor = { FreshCursor(it.lastModified, it.name, it.id) }
) { limit, cursor ->
loadSeriesFreshPage(query, limit, cursor)
}
}
+ else -> true
}
- return collected.drop(query.offset).take(query.limit)
+ val presented = buildPresentedSeries(collected, presentationSettings)
+ return CursorWindowResult(
+ items = presented.drop(query.offset).take(query.limit),
+ totalVisibleCount = presented.size,
+ exhausted = exhausted
+ )
}
private suspend fun <C> collectSeriesPages(
@@ -1325,15 +1384,16 @@ class SeriesRepositoryImpl @Inject constructor(
parentalLevel: Int,
collected: MutableList<Series>,
favoriteIds: Set<Long>,
+ presentationSettings: SeriesPresentationSettings,
extractCursor: (SeriesBrowseEntity) -> C,
loadPage: suspend (limit: Int, cursor: C?) -> List<SeriesBrowseEntity>
- ) {
+ ): Boolean {
var cursor: C? = null
- val targetVisibleCount = query.offset + query.limit
- while (collected.size < targetVisibleCount) {
+ val targetVisibleCount = (query.offset + query.limit).coerceAtLeast(query.limit)
+ while (presentedSeriesCount(collected, presentationSettings) < targetVisibleCount) {
val batch = loadPage(CURSOR_BATCH_SIZE, cursor)
if (batch.isEmpty()) {
- return
+ return true
}
val visibleBatch = if (parentalLevel >= 3) {
batch.filterNot { it.isUserProtected }
@@ -1345,10 +1405,20 @@ class SeriesRepositoryImpl @Inject constructor(
if (item.id in favoriteIds) item.copy(isFavorite = true) else item
}
if (batch.size < CURSOR_BATCH_SIZE) {
- return
+ return true
}
cursor = extractCursor(batch.last())
}
+ return false
+ }
+
+ private fun presentedSeriesCount(
+ series: List<Series>,
+ presentationSettings: SeriesPresentationSettings
+ ): Int = if (presentationSettings.duplicateHandlingMode == VodDuplicateHandlingMode.SHOW_ALL) {
+ series.size
+ } else {
+ buildPresentedSeries(series, presentationSettings).size
}
private suspend fun loadSeriesNamePage(query: LibraryBrowseQuery, limit: Int, cursor: NameCursor?): List<SeriesBrowseEntity> {
diff --git a/data/src/test/java/com/streamvault/data/repository/MovieRepositoryImplTest.kt b/data/src/test/java/com/streamvault/data/repository/MovieRepositoryImplTest.kt
index 33063e8c..c6909418 100644
--- a/data/src/test/java/com/streamvault/data/repository/MovieRepositoryImplTest.kt
+++ b/data/src/test/java/com/streamvault/data/repository/MovieRepositoryImplTest.kt
@@ -1148,6 +1148,39 @@ class MovieRepositoryImplTest {
verify(movieDao, never()).getByProviderCursorPage(any(), any())
}
+ @Test
+ fun `browseMovies groups duplicates while using the cursor window`() = runTest {
+ whenever(preferencesRepository.parentalControlLevel).thenReturn(flowOf(0))
+ whenever(movieDao.getCount(7L)).thenReturn(flowOf(2))
+ whenever(movieDao.getFreshCursorPage(7L, 40)).thenReturn(
+ listOf(
+ movieEntity(id = 101L, name = "Arrival HD", genre = "Sci-Fi", categoryId = 42L, rating = 7.9f)
+ .copy(year = "2016", releaseDate = "2016-11-11", addedAt = 20L),
+ movieEntity(id = 102L, name = "Arrival 4K", genre = "Sci-Fi", categoryId = 42L, rating = 7.9f)
+ .copy(year = "2016", releaseDate = "2016-11-11", addedAt = 10L)
+ )
+ )
+ whenever(favoriteDao.getAllByType(7L, ContentType.MOVIE.name)).thenReturn(flowOf(emptyList()))
+
+ val result = createRepository(
+ duplicateHandlingMode = VodDuplicateHandlingMode.SMART,
+ variantPreferenceMode = VodVariantPreferenceMode.BEST_QUALITY
+ ).browseMovies(
+ LibraryBrowseQuery(
+ providerId = 7L,
+ sortBy = LibrarySortBy.LIBRARY,
+ offset = 0,
+ limit = 20
+ )
+ ).first()
+
+ assertThat(result.totalCount).isEqualTo(1)
+ assertThat(result.items).hasSize(1)
+ assertThat(result.items.single().selectedVariantId).isEqualTo(102L)
+ verify(movieDao).getFreshCursorPage(7L, 40)
+ verify(movieDao, never()).getByProviderPage(any(), any(), any())
+ }
+
private fun createRepository(
duplicateHandlingMode: VodDuplicateHandlingMode = VodDuplicateHandlingMode.SHOW_ALL,
variantPreferenceMode: VodVariantPreferenceMode = VodVariantPreferenceMode.BALANCED,
diff --git a/data/src/test/java/com/streamvault/data/repository/SeriesRepositoryImplTest.kt b/data/src/test/java/com/streamvault/data/repository/SeriesRepositoryImplTest.kt
index 5780a4f0..b4ca8d29 100644
--- a/data/src/test/java/com/streamvault/data/repository/SeriesRepositoryImplTest.kt
+++ b/data/src/test/java/com/streamvault/data/repository/SeriesRepositoryImplTest.kt
@@ -999,29 +999,27 @@ class SeriesRepositoryImplTest {
fun `browseSeries groups duplicate series when duplicate handling is enabled`() = runTest {
whenever(preferencesRepository.parentalControlLevel).thenReturn(flowOf(0))
whenever(seriesDao.getCount(7L)).thenReturn(flowOf(2))
- whenever(seriesDao.getByProviderPage(7L, 100, 0)).thenReturn(
- flowOf(
- listOf(
- SeriesBrowseEntity(
- id = 17L,
- seriesId = 1700L,
- providerSeriesId = "1700",
- name = "Drama HD",
- providerId = 7L,
- releaseDate = "2024-01-01",
- tmdbId = 5000L,
- lastModified = 10L
- ),
- SeriesBrowseEntity(
- id = 18L,
- seriesId = 1800L,
- providerSeriesId = "1800",
- name = "Drama 4K",
- providerId = 7L,
- releaseDate = "2024-02-01",
- tmdbId = 5000L,
- lastModified = 20L
- )
+ whenever(seriesDao.getFreshCursorPage(7L, 40)).thenReturn(
+ listOf(
+ SeriesBrowseEntity(
+ id = 17L,
+ seriesId = 1700L,
+ providerSeriesId = "1700",
+ name = "Drama HD",
+ providerId = 7L,
+ releaseDate = "2024-01-01",
+ tmdbId = 5000L,
+ lastModified = 10L
+ ),
+ SeriesBrowseEntity(
+ id = 18L,
+ seriesId = 1800L,
+ providerSeriesId = "1800",
+ name = "Drama 4K",
+ providerId = 7L,
+ releaseDate = "2024-02-01",
+ tmdbId = 5000L,
+ lastModified = 20L
)
)
)
@@ -1044,7 +1042,8 @@ class SeriesRepositoryImplTest {
assertThat(result.items).hasSize(1)
assertThat(result.items.single().selectedVariantId).isEqualTo(18L)
assertThat(result.items.single().variants.map { it.rawSeriesId }).containsExactly(18L, 17L).inOrder()
- verify(seriesDao, never()).getFreshCursorPage(any(), any())
+ verify(seriesDao).getFreshCursorPage(7L, 40)
+ verify(seriesDao, never()).getByProviderPage(any(), any(), any())
}
@Test