StreamVault is a TV-first free IPTV player for Android TV built with Kotlin, Jetpack Compose, Room, Hilt, and Media3.
Kotlin iptv-player
feat(sync): implement M3U staging with duplicate detection and paged reconciliation for channels and movies
David Nashashb1e3ee4fd4dc
diff --git a/data/src/main/java/com/streamvault/data/local/dao/CatalogSyncDao.kt b/data/src/main/java/com/streamvault/data/local/dao/CatalogSyncDao.ktindex 7dc39838..091881d1 100644--- a/data/src/main/java/com/streamvault/data/local/dao/CatalogSyncDao.kt+++ b/data/src/main/java/com/streamvault/data/local/dao/CatalogSyncDao.kt@@ -1,6 +1,7 @@package com.streamvault.data.local.daoimport androidx.room.Dao+import androidx.room.ColumnInfoimport androidx.room.Insertimport androidx.room.OnConflictStrategyimport androidx.room.Query@@ -15,14 +16,27 @@ data class ChannelStageCategorySummary(val isAdult: Boolean)+data class M3uChannelStageCursor(+ // The stage table's implicit rowid keeps paged reconciliation in playlist order.+ @ColumnInfo(name = "row_id") val rowId: Long,+ @ColumnInfo(name = "stream_id") val streamId: Long,+ @ColumnInfo(name = "category_id") val categoryId: Long?+)+@Daointerface CatalogSyncDao {@Insert(onConflict = OnConflictStrategy.REPLACE)suspend fun insertChannelStages(rows: List<ChannelImportStageEntity>)+ @Insert(onConflict = OnConflictStrategy.IGNORE)+ suspend fun insertM3uChannelStagesIgnoringConflicts(rows: List<ChannelImportStageEntity>): List<Long>+@Insert(onConflict = OnConflictStrategy.REPLACE)suspend fun insertMovieStages(rows: List<MovieImportStageEntity>)+ @Insert(onConflict = OnConflictStrategy.IGNORE)+ suspend fun insertM3uMovieStagesIgnoringConflicts(rows: List<MovieImportStageEntity>): List<Long>+@Insert(onConflict = OnConflictStrategy.REPLACE)suspend fun insertSeriesStages(rows: List<SeriesImportStageEntity>)@@ -64,6 +78,44 @@ interface CatalogSyncDao {@Query("SELECT * FROM channel_import_stage WHERE provider_id = :providerId AND session_id = :sessionId")suspend fun getChannelStages(providerId: Long, sessionId: Long): List<ChannelImportStageEntity>+ @Query(+ """+ SELECT rowid AS row_id, stream_id, category_id+ FROM channel_import_stage+ WHERE provider_id = :providerId+ AND session_id = :sessionId+ AND (:afterRowId IS NULL OR rowid > :afterRowId)+ ORDER BY rowid+ LIMIT :limit+ """+ )+ suspend fun getM3uChannelStagePage(+ providerId: Long,+ sessionId: Long,+ afterRowId: Long?,+ limit: Int+ ): List<M3uChannelStageCursor>++ @Query(+ """+ SELECT rowid AS row_id, stream_id, category_id+ FROM channel_import_stage+ WHERE provider_id = :providerId+ AND session_id = :sessionId+ AND category_id IN (:categoryIds)+ AND (:afterRowId IS NULL OR rowid > :afterRowId)+ ORDER BY rowid+ LIMIT :limit+ """+ )+ suspend fun getM3uChannelStagePageForCategories(+ providerId: Long,+ sessionId: Long,+ afterRowId: Long?,+ categoryIds: List<Long>,+ limit: Int+ ): List<M3uChannelStageCursor>+@Query("SELECT * FROM movie_import_stage WHERE provider_id = :providerId AND session_id = :sessionId")suspend fun getMovieStages(providerId: Long, sessionId: Long): List<MovieImportStageEntity>diff --git a/data/src/main/java/com/streamvault/data/sync/SyncCatalogStore.kt b/data/src/main/java/com/streamvault/data/sync/SyncCatalogStore.ktindex a42d5624..245d49d7 100644--- a/data/src/main/java/com/streamvault/data/sync/SyncCatalogStore.kt+++ b/data/src/main/java/com/streamvault/data/sync/SyncCatalogStore.kt@@ -5,6 +5,7 @@ import com.streamvault.data.local.DatabaseTransactionRunnerimport com.streamvault.data.local.dao.CatalogSyncDaoimport com.streamvault.data.local.dao.CategoryDaoimport com.streamvault.data.local.dao.ChannelDao+import com.streamvault.data.local.dao.M3uChannelStageCursorimport com.streamvault.data.local.dao.MovieDaoimport com.streamvault.data.local.dao.SeriesDaoimport com.streamvault.data.local.dao.TmdbIdentityDao@@ -404,6 +405,65 @@ internal class SyncCatalogStore(stageChannelBatchInternal(providerId, sessionId, channels, null)}+ /** M3U-only staging uses the stage-table key for bounded-memory duplicate detection. */+ suspend fun stageM3uChannelBatch(providerId: Long, sessionId: Long, channels: List<ChannelEntity>): Int =+ timed(providerId, "m3u-channel-staging-batch(${channels.size})") {+ val stages = buildChannelStages(providerId, sessionId, channels)+ providerTransaction(providerId) {+ val currentCount = catalogSyncDao.countChannelStages(providerId, sessionId)+ val insertedCount = insertStageRowsIgnoringConflicts(+ stages,+ catalogSyncDao::insertM3uChannelStagesIgnoringConflicts+ )+ requireStageCapacity(+ currentCount = currentCount,+ incomingCount = insertedCount,+ maximum = sizeLimits.maxChannelsPerProvider,+ label = "channel"+ )+ insertedCount+ }+ }++ /** M3U-only staging counterpart for VOD rows. Other providers retain replace semantics. */+ suspend fun stageM3uMovieBatch(providerId: Long, sessionId: Long, movies: List<MovieEntity>): Int =+ timed(providerId, "m3u-movie-staging-batch(${movies.size})") {+ val stages = buildMovieStages(providerId, sessionId, movies)+ providerTransaction(providerId) {+ val currentCount = catalogSyncDao.countMovieStages(providerId, sessionId)+ val insertedCount = insertStageRowsIgnoringConflicts(+ stages,+ catalogSyncDao::insertM3uMovieStagesIgnoringConflicts+ )+ requireStageCapacity(+ currentCount = currentCount,+ incomingCount = insertedCount,+ maximum = sizeLimits.maxMoviesPerProvider,+ label = "movie"+ )+ insertedCount+ }+ }++ suspend fun m3uChannelStagePage(+ providerId: Long,+ sessionId: Long,+ afterRowId: Long?,+ categoryIds: List<Long>?,+ limit: Int+ ): List<M3uChannelStageCursor> = if (categoryIds == null) {+ catalogSyncDao.getM3uChannelStagePage(providerId, sessionId, afterRowId, limit)+ } else {+ require(categoryIds.isNotEmpty())+ catalogSyncDao.getM3uChannelStagePageForCategories(+ providerId,+ sessionId,+ afterRowId,+ categoryIds,+ limit+ )+ }+/** Only for fresh, isolated attempts whose writer admits each ID once. */suspend fun stageAdmittedLiveBatch(providerId: Long, sessionId: Long, channels: List<ChannelEntity>, admittedCount: Int) {stageChannelBatchInternal(providerId, sessionId, channels, admittedCount)@@ -986,6 +1046,19 @@ internal class SyncCatalogStore(}}+ private suspend fun <T> insertStageRowsIgnoringConflicts(+ rows: List<T>,+ insert: suspend (List<T>) -> List<Long>+ ): Int {+ var insertedCount = 0+ rows.chunked(STAGE_BATCH_SIZE).forEach { chunk ->+ if (chunk.isNotEmpty()) {+ insertedCount += insert(chunk).count { rowId -> rowId != -1L }+ }+ }+ return insertedCount+ }+private fun movieStageEntity(providerId: Long,sessionId: Long,diff --git a/data/src/main/java/com/streamvault/data/sync/SyncManagerM3uImporter.kt b/data/src/main/java/com/streamvault/data/sync/SyncManagerM3uImporter.ktindex 699eb553..12f86c12 100644--- a/data/src/main/java/com/streamvault/data/sync/SyncManagerM3uImporter.kt+++ b/data/src/main/java/com/streamvault/data/sync/SyncManagerM3uImporter.kt@@ -35,6 +35,8 @@ import java.net.URIimport java.util.zip.GZIPInputStreamprivate const val M3U_PROGRESS_INTERVAL = 5_000+private const val M3U_SERIES_RECONCILIATION_PAGE_SIZE = 2_048+private const val M3U_MAX_SERIES_CATEGORY_QUERY_IDS = 900private const val M3U_IMPORTER_TAG = "SyncManagerM3u"internal class SyncManagerM3uImporter(@@ -80,9 +82,7 @@ internal class SyncManagerM3uImporter(val movieCategories = CategoryAccumulator(provider.id, ContentType.MOVIE, stableLongHasher)val channelBatch = ArrayList<ChannelEntity>(batchSize)val movieBatch = ArrayList<MovieEntity>(batchSize)- val seenLiveStreamIds = if (includeLive) mutableSetOf<Long>() else null- val seenMovieStreamIds = if (includeMovies) mutableSetOf<Long>() else null- val seriesToReconcile = mutableListOf<Pair<Long, M3uSeriesAssignment?>>()+ val seriesCategoryIds = mutableSetOf<Long>()val persistedOverrides = classificationDao?.getOverrides(provider.id).orEmpty()val itemOverrides = persistedOverrides.associateBy { it.sourceKey }// A movie that was automatically classified before the override tables existed may@@ -202,8 +202,8 @@ internal class SyncManagerM3uImporter(val groupTitle = entry.groupTitle.ifBlank { "Uncategorized" }val stableStreamId = override?.streamId?: sourceStableId- if (seenLiveStreamIds?.add(stableStreamId) != true) return@parseStreamingval categoryId = liveCategories.idFor(groupTitle)+ if (classificationRepository != null) seriesCategoryIds += categoryIdchannelBatch.add(ChannelEntity(streamId = stableStreamId,@@ -225,9 +225,9 @@ internal class SyncManagerM3uImporter(playbackMetadataJson = playbackMetadataJson))- seriesToReconcile += stableStreamId to override?.toSeriesAssignment()- liveCount++- if (channelBatch.size >= batchSize) flushChannelBatch(provider.id, sessionId, channelBatch)+ if (channelBatch.size >= batchSize) {+ liveCount += flushChannelBatch(provider.id, sessionId, channelBatch)+ }return@parseStreaming}@@ -249,8 +249,6 @@ internal class SyncManagerM3uImporter(}val stableStreamId = override?.streamId?: sourceStableId- if (seenMovieStreamIds?.add(stableStreamId) != true) return@parseStreaming- if (movieCount >= sizeLimits.maxMoviesPerProvider) throw CatalogAdmissionExceeded("M3U movie limit exceeded")val categoryId = if (manualTarget == M3uClassificationTarget.MOVIE) {manualCategoryId(provider.id, ContentType.MOVIE, stableLongHasher)} else {@@ -277,17 +275,14 @@ internal class SyncManagerM3uImporter(playbackMetadataJson = playbackMetadataJson))- movieCount++if (movieBatch.size >= batchSize) {- flushMovieBatch(provider.id, sessionId, movieBatch)+ movieCount += flushMovieBatch(provider.id, sessionId, movieBatch)}} else {if (!includeLive) return@parseStreamingval groupTitle = entry.groupTitle.ifBlank { "Uncategorized" }val stableStreamId = override?.streamId?: sourceStableId- if (seenLiveStreamIds?.add(stableStreamId) != true) return@parseStreaming- if (liveCount >= sizeLimits.maxChannelsPerProvider) throw CatalogAdmissionExceeded("M3U live-channel limit exceeded")val categoryId = liveCategories.idFor(groupTitle)if (liveCategories.count > sizeLimits.maxM3uCategoriesPerType) throw CatalogAdmissionExceeded("M3U live category limit exceeded")val isAdult = AdultContentClassifier.isAdultCategoryName(groupTitle)@@ -312,9 +307,8 @@ internal class SyncManagerM3uImporter(playbackMetadataJson = playbackMetadataJson))- liveCount++if (channelBatch.size >= batchSize) {- flushChannelBatch(provider.id, sessionId, channelBatch)+ liveCount += flushChannelBatch(provider.id, sessionId, channelBatch)}}},@@ -327,8 +321,8 @@ internal class SyncManagerM3uImporter(}}- flushChannelBatch(provider.id, sessionId, channelBatch)- flushMovieBatch(provider.id, sessionId, movieBatch)+ liveCount += flushChannelBatch(provider.id, sessionId, channelBatch)+ movieCount += flushMovieBatch(provider.id, sessionId, movieBatch)// Only commit a section if it produced at least one entry. Committing an// empty stage with includeLive=true runs stale deletion and wipes the entire// live-TV catalog — even though the absence of entries may reflect a server@@ -348,18 +342,40 @@ internal class SyncManagerM3uImporter(includeMovies = effectiveMovies,afterCatalogApply = afterCatalogApply)- if (classificationRepository != null) {- seriesToReconcile.forEach { (streamId, assignment) ->- when (val result = classificationRepository.classifyChannelByStream(- provider.id,- streamId,- M3uClassificationTarget.SERIES,- assignment- )) {- is com.streamvault.domain.model.Result.Error ->- warnings += result.message- else -> Unit+ if (classificationRepository != null && seriesCategoryIds.isNotEmpty()) {+ val categoryFilter = seriesCategoryIds.toList()+ .takeIf { it.size <= M3U_MAX_SERIES_CATEGORY_QUERY_IDS }+ var afterRowId: Long? = null+ while (true) {+ val stagedChannels = syncCatalogStore.m3uChannelStagePage(+ providerId = provider.id,+ sessionId = sessionId,+ afterRowId = afterRowId,+ categoryIds = categoryFilter,+ limit = M3U_SERIES_RECONCILIATION_PAGE_SIZE+ )+ if (stagedChannels.isEmpty()) break+ stagedChannels.forEach { stagedChannel ->+ val override = itemOverridesByStreamId[stagedChannel.streamId]+ val shouldReconcile = if (override != null) {+ override.targetType == M3uClassificationTarget.SERIES.name+ } else {+ stagedChannel.categoryId in seriesCategoryIds+ }+ if (shouldReconcile) {+ when (val result = classificationRepository.classifyChannelByStream(+ provider.id,+ stagedChannel.streamId,+ M3uClassificationTarget.SERIES,+ override?.toSeriesAssignment()+ )) {+ is com.streamvault.domain.model.Result.Error ->+ warnings += result.message+ else -> Unit+ }+ }}+ afterRowId = stagedChannels.last().rowId}}} finally {@@ -458,16 +474,18 @@ internal class SyncManagerM3uImporter(}}- private suspend fun flushChannelBatch(providerId: Long, sessionId: Long, batch: MutableList<ChannelEntity>) {- if (batch.isEmpty()) return- syncCatalogStore.stageChannelBatch(providerId, sessionId, batch)+ private suspend fun flushChannelBatch(providerId: Long, sessionId: Long, batch: MutableList<ChannelEntity>): Int {+ if (batch.isEmpty()) return 0+ val insertedCount = syncCatalogStore.stageM3uChannelBatch(providerId, sessionId, batch)batch.clear()+ return insertedCount}- private suspend fun flushMovieBatch(providerId: Long, sessionId: Long, batch: MutableList<MovieEntity>) {- if (batch.isEmpty()) return- syncCatalogStore.stageMovieBatch(providerId, sessionId, batch)+ private suspend fun flushMovieBatch(providerId: Long, sessionId: Long, batch: MutableList<MovieEntity>): Int {+ if (batch.isEmpty()) return 0+ val insertedCount = syncCatalogStore.stageM3uMovieBatch(providerId, sessionId, batch)batch.clear()+ return insertedCount}private fun stableId(diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.mdindex 41bdd84d..49d23328 100644--- a/docs/CHANGELOG.md+++ b/docs/CHANGELOG.md@@ -10,6 +10,7 @@ All notable product changes are recorded in this document.### Changed- Localized player recording notices and EPG recording messages across all supported locales.- Xtream series details now refresh every six hours and Stalker series details every 24 hours instead of every 14 days, so newly aired episodes appear promptly and incomplete episode caches repair themselves.+- Reduced peak memory during large M3U syncs by using staged database keys for duplicate detection and paged series reconciliation; Xtream and Stalker sync paths are unchanged.### Fixed- Fixed missing episodes from providers that return nested episode arrays or season-keyed episode maps, which were previously skipped and left series with no episodes.