Skip to content

StreamVault is a TV-first free IPTV player for Android TV built with Kotlin, Jetpack Compose, Room, Hilt, and Media3.

Improve Stalker sync throughput and playback-aware deferral

David Nashash5068039ceec0
diff --git a/app/src/main/java/com/streamvault/app/ui/screens/player/PlayerLifecycleActions.kt b/app/src/main/java/com/streamvault/app/ui/screens/player/PlayerLifecycleActions.kt
index cfe98cd4..4f7e57e4 100644
--- a/app/src/main/java/com/streamvault/app/ui/screens/player/PlayerLifecycleActions.kt
+++ b/app/src/main/java/com/streamvault/app/ui/screens/player/PlayerLifecycleActions.kt
@@ -2,6 +2,7 @@ package com.streamvault.app.ui.screens.player
import androidx.lifecycle.viewModelScope
import com.streamvault.domain.model.ContentType
+import com.streamvault.domain.model.ProviderType
import com.streamvault.player.PlayerEngine
import kotlinx.coroutines.delay
import kotlinx.coroutines.launch
@@ -65,6 +66,43 @@ internal fun PlayerViewModel.startTokenRenewalMonitoring(expirationTime: Long?)
}
}
+internal suspend fun PlayerViewModel.synchronizeStalkerPlaybackFetchDeferral(isPlaying: Boolean) {
+ if (!isPlaying) {
+ stopActiveStalkerPlaybackFetchDeferral()
+ return
+ }
+
+ val providerId = currentProviderId.takeIf { it > 0L } ?: run {
+ stopActiveStalkerPlaybackFetchDeferral()
+ return
+ }
+ val provider = providerRepository.getProvider(providerId)
+ if (provider?.type != ProviderType.STALKER_PORTAL) {
+ stopActiveStalkerPlaybackFetchDeferral()
+ return
+ }
+ val previousProviderId = activeStalkerPlaybackProviderId
+ if (previousProviderId != null && previousProviderId != providerId) {
+ syncManager.noteStalkerPlaybackStopped(previousProviderId)
+ }
+ if (previousProviderId != providerId) {
+ syncManager.noteStalkerPlaybackStarted(providerId)
+ activeStalkerPlaybackProviderId = providerId
+ }
+}
+
+internal fun PlayerViewModel.reconcileStalkerPlaybackFetchDeferral() {
+ viewModelScope.launch {
+ synchronizeStalkerPlaybackFetchDeferral(playerEngine.isPlaying.value)
+ }
+}
+
+internal fun PlayerViewModel.stopActiveStalkerPlaybackFetchDeferral() {
+ val providerId = activeStalkerPlaybackProviderId ?: return
+ activeStalkerPlaybackProviderId = null
+ syncManager.noteStalkerPlaybackStopped(providerId)
+}
+
fun PlayerViewModel.onAppBackgrounded() {
if (!isAppInForeground) return
isAppInForeground = false
@@ -90,6 +128,7 @@ fun PlayerViewModel.onAppForegrounded() {
}
fun PlayerViewModel.onPlayerScreenDisposed() {
+ stopActiveStalkerPlaybackFetchDeferral()
if (currentContentType != ContentType.LIVE) {
viewModelScope.launch {
persistPlaybackProgress()
diff --git a/app/src/main/java/com/streamvault/app/ui/screens/player/PlayerPreparationSetupActions.kt b/app/src/main/java/com/streamvault/app/ui/screens/player/PlayerPreparationSetupActions.kt
index 58c5da33..f8df32e4 100644
--- a/app/src/main/java/com/streamvault/app/ui/screens/player/PlayerPreparationSetupActions.kt
+++ b/app/src/main/java/com/streamvault/app/ui/screens/player/PlayerPreparationSetupActions.kt
@@ -115,6 +115,7 @@ internal fun PlayerViewModel.applyPrepareSessionState(
if (currentContentType != ContentType.SERIES_EPISODE || providerId <= 0 || currentSeriesId == null) {
clearSeriesEpisodeContext()
}
+ reconcileStalkerPlaybackFetchDeferral()
if (currentContentType != ContentType.LIVE) {
lastRecordedLivePlaybackKey = null
recentChannelsJob?.cancel()
diff --git a/app/src/main/java/com/streamvault/app/ui/screens/player/PlayerViewModel.kt b/app/src/main/java/com/streamvault/app/ui/screens/player/PlayerViewModel.kt
index dfb5ae29..e686218f 100644
--- a/app/src/main/java/com/streamvault/app/ui/screens/player/PlayerViewModel.kt
+++ b/app/src/main/java/com/streamvault/app/ui/screens/player/PlayerViewModel.kt
@@ -15,6 +15,7 @@ import com.streamvault.app.util.isPlaybackComplete
import com.streamvault.app.tv.LauncherRecommendationsManager
import com.streamvault.app.tv.WatchNextManager
import com.streamvault.data.remote.stalker.StalkerUrlFactory
+import com.streamvault.data.sync.SyncManager
import com.streamvault.data.remote.xtream.XtreamStreamUrlResolver
import com.streamvault.data.security.CredentialDecryptionException
import com.streamvault.domain.manager.RecordingManager
@@ -98,6 +99,7 @@ class PlayerViewModel @Inject constructor(
internal val xtreamStreamUrlResolver: XtreamStreamUrlResolver,
internal val seekThumbnailProvider: SeekThumbnailProvider,
internal val livePreviewHandoffManager: LivePreviewHandoffManager,
+ internal val syncManager: SyncManager,
private val okHttpClient: OkHttpClient,
) : ViewModel() {
companion object {
@@ -305,6 +307,7 @@ class PlayerViewModel @Inject constructor(
internal var currentCombinedProfileMembers: List<CombinedM3uProfileMember> = emptyList()
internal var combinedCategoriesById: Map<Long, CombinedCategory> = emptyMap()
private var lastObservedPlaybackState: PlaybackState = PlaybackState.IDLE
+ internal var activeStalkerPlaybackProviderId: Long? = null
internal var epgJob: Job? = null
internal var playlistJob: Job? = null
@@ -467,6 +470,14 @@ class PlayerViewModel @Inject constructor(
}
}
}
+ viewModelScope.launch {
+ activePlayerEngineFlow
+ .flatMapLatest { it.isPlaying }
+ .distinctUntilChanged()
+ .collect { isPlaying ->
+ synchronizeStalkerPlaybackFetchDeferral(isPlaying)
+ }
+ }
viewModelScope.launch {
activePlayerEngineFlow.flatMapLatest { it.retryStatus }.collect { status ->
status ?: return@collect
diff --git a/data/src/main/java/com/streamvault/data/remote/stalker/StalkerTrafficCoordinator.kt b/data/src/main/java/com/streamvault/data/remote/stalker/StalkerTrafficCoordinator.kt
new file mode 100644
index 00000000..571db0b6
--- /dev/null
+++ b/data/src/main/java/com/streamvault/data/remote/stalker/StalkerTrafficCoordinator.kt
@@ -0,0 +1,59 @@
+package com.streamvault.data.remote.stalker
+
+import java.util.concurrent.ConcurrentHashMap
+
+internal object StalkerTrafficCoordinator {
+ private const val PLAYBACK_PROTECTION_WINDOW_MILLIS = 90_000L
+ private const val ACTIVE_PLAYBACK_RECHECK_MILLIS = 5_000L
+ private val recentPlaybackByProvider = ConcurrentHashMap<Long, Long>()
+ private val activePlaybackCountByProvider = ConcurrentHashMap<Long, Int>()
+
+ fun notePlaybackActivity(providerId: Long, now: Long = System.currentTimeMillis()) {
+ if (providerId <= 0L) return
+ recentPlaybackByProvider[providerId] = now
+ }
+
+ fun notePlaybackStarted(providerId: Long, now: Long = System.currentTimeMillis()) {
+ if (providerId <= 0L) return
+ recentPlaybackByProvider[providerId] = now
+ activePlaybackCountByProvider.compute(providerId) { _, existing ->
+ (existing ?: 0) + 1
+ }
+ }
+
+ fun notePlaybackStopped(providerId: Long) {
+ if (providerId <= 0L) return
+ activePlaybackCountByProvider.compute(providerId) { _, existing ->
+ when {
+ existing == null || existing <= 1 -> null
+ else -> existing - 1
+ }
+ }
+ if ((activePlaybackCountByProvider[providerId] ?: 0) <= 0) {
+ activePlaybackCountByProvider.remove(providerId)
+ recentPlaybackByProvider.remove(providerId)
+ }
+ }
+
+ fun shouldDeferCatalogFetch(providerId: Long, now: Long = System.currentTimeMillis()): Boolean =
+ deferCatalogFetchMillis(providerId, now) > 0L
+
+ fun deferCatalogFetchMillis(providerId: Long, now: Long = System.currentTimeMillis()): Long {
+ if (providerId <= 0L) return 0L
+ if ((activePlaybackCountByProvider[providerId] ?: 0) > 0) {
+ return ACTIVE_PLAYBACK_RECHECK_MILLIS
+ }
+ val lastPlaybackAt = recentPlaybackByProvider[providerId] ?: return 0L
+ val remaining = (lastPlaybackAt + PLAYBACK_PROTECTION_WINDOW_MILLIS) - now
+ if (remaining <= 0L) {
+ recentPlaybackByProvider.remove(providerId, lastPlaybackAt)
+ return 0L
+ }
+ return remaining
+ }
+
+ internal fun resetForTests() {
+ recentPlaybackByProvider.clear()
+ activePlaybackCountByProvider.clear()
+ }
+}
\ No newline at end of file
diff --git a/data/src/main/java/com/streamvault/data/remote/xtream/XtreamStreamUrlResolver.kt b/data/src/main/java/com/streamvault/data/remote/xtream/XtreamStreamUrlResolver.kt
index 63230ec3..f0091b0e 100644
--- a/data/src/main/java/com/streamvault/data/remote/xtream/XtreamStreamUrlResolver.kt
+++ b/data/src/main/java/com/streamvault/data/remote/xtream/XtreamStreamUrlResolver.kt
@@ -8,6 +8,7 @@ import com.streamvault.data.remote.stalker.StalkerApiService
import com.streamvault.data.remote.stalker.StalkerPlaybackMode
import com.streamvault.data.remote.stalker.StalkerPlaybackResolutionException
import com.streamvault.data.remote.stalker.StalkerStreamKind
+import com.streamvault.data.remote.stalker.StalkerTrafficCoordinator
import com.streamvault.data.remote.stalker.StalkerUrlFactory
import com.streamvault.data.remote.stalker.buildStalkerPlaybackDescriptor
import com.streamvault.data.security.CredentialCrypto
@@ -156,6 +157,7 @@ class XtreamStreamUrlResolver @Inject constructor(
expirationTime = extractStreamExpirationTime(passthroughUrl)
)
}
+ StalkerTrafficCoordinator.notePlaybackActivity(resolvedProvider.id)
val playbackInfo = when (
val resolvedResult = getOrCreateStalkerProvider(resolvedProvider)
.resolvePlaybackInfo(
@@ -211,6 +213,7 @@ class XtreamStreamUrlResolver @Inject constructor(
fallbackContentType: ContentType?,
fallbackContainerExtension: String?
): ResolvedStreamUrl? {
+ StalkerTrafficCoordinator.notePlaybackActivity(provider.id)
val kind = when (fallbackContentType) {
ContentType.LIVE -> StalkerStreamKind.LIVE
ContentType.MOVIE -> StalkerStreamKind.MOVIE
diff --git a/data/src/main/java/com/streamvault/data/repository/MovieRepositoryImpl.kt b/data/src/main/java/com/streamvault/data/repository/MovieRepositoryImpl.kt
index b26a288c..d1fa16ee 100644
--- a/data/src/main/java/com/streamvault/data/repository/MovieRepositoryImpl.kt
+++ b/data/src/main/java/com/streamvault/data/repository/MovieRepositoryImpl.kt
@@ -22,6 +22,7 @@ import com.streamvault.data.remote.http.toGenericRequestProfile
import com.streamvault.data.remote.stalker.StalkerApiService
import com.streamvault.data.remote.stalker.StalkerPlaybackMode
import com.streamvault.data.remote.stalker.StalkerProvider
+import com.streamvault.data.remote.stalker.StalkerTrafficCoordinator
import com.streamvault.data.remote.xtream.XtreamApiService
import com.streamvault.data.remote.xtream.XtreamStreamUrlResolver
import com.streamvault.data.remote.xtream.XtreamProvider
@@ -1261,6 +1262,9 @@ class MovieRepositoryImpl @Inject constructor(
return
}
if (provider.type == ProviderType.STALKER_PORTAL) {
+ if (localCount <= 0) {
+ syncManager.prioritizeStalkerIndexCategory(providerId, ContentType.MOVIE, categoryId)
+ }
if (!shouldUseStalkerLazyFallback(providerId, hydration, localCount)) {
return
}
@@ -1299,6 +1303,9 @@ class MovieRepositoryImpl @Inject constructor(
val localCount = movieDao.getCountByCategory(providerId, categoryId).first()
val hydration = movieCategoryHydrationDao.get(providerId, categoryId)
if (provider.type == ProviderType.STALKER_PORTAL) {
+ if (localCount <= 0) {
+ syncManager.prioritizeStalkerIndexCategory(providerId, ContentType.MOVIE, categoryId)
+ }
if (!shouldUseStalkerLazyFallback(providerId, hydration, localCount)) {
return@launch
}
@@ -1365,6 +1372,7 @@ class MovieRepositoryImpl @Inject constructor(
// totalPages; subsequent iterations use the in-loop updated value.
var firstIteration = true
while (currentCount < requiredCount) {
+ if (StalkerTrafficCoordinator.shouldDeferCatalogFetch(providerId)) break
val attemptStartedAt = System.currentTimeMillis()
if (isPreviewLoad && nextPage > STALKER_PREVIEW_MAX_REMOTE_PAGES) break
val totalPages = currentHydration?.totalPages ?: 0
diff --git a/data/src/main/java/com/streamvault/data/repository/SeriesRepositoryImpl.kt b/data/src/main/java/com/streamvault/data/repository/SeriesRepositoryImpl.kt
index c3ba6b89..613dc6af 100644
--- a/data/src/main/java/com/streamvault/data/repository/SeriesRepositoryImpl.kt
+++ b/data/src/main/java/com/streamvault/data/repository/SeriesRepositoryImpl.kt
@@ -9,6 +9,7 @@ import com.streamvault.data.remote.http.toGenericRequestProfile
import com.streamvault.data.remote.stalker.StalkerApiService
import com.streamvault.data.remote.stalker.StalkerPlaybackMode
import com.streamvault.data.remote.stalker.StalkerProvider
+import com.streamvault.data.remote.stalker.StalkerTrafficCoordinator
import com.streamvault.data.remote.xtream.XtreamApiService
import com.streamvault.data.remote.xtream.XtreamProvider
import com.streamvault.data.remote.xtream.XtreamStreamUrlResolver
@@ -1245,6 +1246,9 @@ class SeriesRepositoryImpl @Inject constructor(
return
}
if (provider.type == ProviderType.STALKER_PORTAL) {
+ if (localCount <= 0) {
+ syncManager.prioritizeStalkerIndexCategory(providerId, ContentType.SERIES, categoryId)
+ }
if (!shouldUseStalkerLazyFallback(providerId, hydration, localCount)) {
return
}
@@ -1279,6 +1283,9 @@ class SeriesRepositoryImpl @Inject constructor(
val hydration = seriesCategoryHydrationDao.get(providerId, categoryId)
if (localCount == 0 && hydration?.isEmptyRetryCoolingDown() == true) return@launch
if (provider.type == ProviderType.STALKER_PORTAL) {
+ if (localCount <= 0) {
+ syncManager.prioritizeStalkerIndexCategory(providerId, ContentType.SERIES, categoryId)
+ }
if (!shouldUseStalkerLazyFallback(providerId, hydration, localCount)) {
return@launch
}
@@ -1344,6 +1351,7 @@ class SeriesRepositoryImpl @Inject constructor(
// totalPages; subsequent iterations use the in-loop updated value.
var firstIteration = true
while (currentCount < requiredCount) {
+ if (StalkerTrafficCoordinator.shouldDeferCatalogFetch(providerId)) break
val attemptStartedAt = System.currentTimeMillis()
if (isPreviewLoad && nextPage > STALKER_PREVIEW_MAX_REMOTE_PAGES) break
val totalPages = currentHydration?.totalPages ?: 0
diff --git a/data/src/main/java/com/streamvault/data/sync/SyncManager.kt b/data/src/main/java/com/streamvault/data/sync/SyncManager.kt
index caa89e1a..4c57efa5 100644
--- a/data/src/main/java/com/streamvault/data/sync/SyncManager.kt
+++ b/data/src/main/java/com/streamvault/data/sync/SyncManager.kt
@@ -33,6 +33,7 @@ import com.streamvault.data.remote.stalker.StalkerApiService
import com.streamvault.data.remote.stalker.StalkerPlaybackMode
import com.streamvault.data.remote.stalker.StalkerProvider
import com.streamvault.data.remote.stalker.StalkerProviderProfile
+import com.streamvault.data.remote.stalker.StalkerTrafficCoordinator
import com.streamvault.data.remote.dto.XtreamCategory
import com.streamvault.data.remote.dto.XtreamSeriesItem
import com.streamvault.data.remote.dto.XtreamStream
@@ -65,15 +66,20 @@ import com.streamvault.domain.repository.SyncMetadataRepository
import com.streamvault.domain.sync.Section
import com.streamvault.domain.sync.SyncProgress
import dagger.hilt.android.qualifiers.ApplicationContext
+import kotlinx.coroutines.async
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.CancellationException
+import kotlinx.coroutines.awaitAll
+import kotlinx.coroutines.coroutineScope
import kotlinx.coroutines.flow.Flow
import kotlinx.coroutines.flow.StateFlow
import kotlinx.coroutines.flow.first
import kotlinx.coroutines.flow.map
import kotlinx.coroutines.withContext
import kotlinx.coroutines.sync.Mutex
+import kotlinx.coroutines.sync.Semaphore
import kotlinx.coroutines.sync.withLock
+import kotlinx.coroutines.sync.withPermit
import kotlinx.serialization.json.Json
import okhttp3.OkHttpClient
import okhttp3.Request
@@ -83,6 +89,7 @@ import java.io.IOException
import java.io.InputStream
import java.net.URI
import java.security.MessageDigest
+import java.util.ArrayDeque
import java.util.concurrent.ConcurrentHashMap
import java.util.zip.GZIPInputStream
import javax.inject.Inject
@@ -92,6 +99,8 @@ private const val TAG = "SyncManager"
private const val XTREAM_FALLBACK_STAGE_BATCH_SIZE = 500
private const val STALKER_INDEX_CATEGORY_SLICE_SIZE = 12
private const val STALKER_WILDCARD_PAGE_SLICE_SIZE = 48
+private const val STALKER_MAX_PARALLEL_CATEGORY_FETCHES = 6
+private const val STALKER_MAX_SECTION_RUN_MILLIS = 90_000L
private const val STALKER_CATEGORY_RETRY_BUDGET = 3
private const val STALKER_CATEGORY_RETRY_COOLDOWN_MILLIS = 5 * 60 * 1000L
private const val STALKER_RUNNING_JOB_STALE_MILLIS = 15 * 60 * 1000L
@@ -214,6 +223,9 @@ class SyncManager @Inject constructor(
val syncState: StateFlow<SyncState> = syncStateTracker.aggregateState
val syncStatesByProvider: StateFlow<Map<Long, SyncState>> = syncStateTracker.statesByProvider
private val providerSyncMutexes = ConcurrentHashMap<Long, Mutex>()
+ private val providerStalkerSummaryMutexes = ConcurrentHashMap<Long, Mutex>()
+ private val providerStalkerIndexSectionMutexes = ConcurrentHashMap<String, Mutex>()
+ private val providerStalkerFetchSemaphores = ConcurrentHashMap<Long, Semaphore>()
private val providerEpgMutexes = ConcurrentHashMap<Long, Mutex>()
private val syncAdmissionMutex = Mutex()
private val xtreamCatalogHttpService: OkHttpXtreamApiService by lazy {
@@ -302,6 +314,63 @@ class SyncManager @Inject constructor(
}
}
+ private suspend fun <T> withStalkerIndexSectionLock(
+ providerId: Long,
+ section: ContentType,
+ block: suspend () -> T
+ ): T {
+ val lockKey = "$providerId:${section.name}"
+ val mutex = syncAdmissionMutex.withLock {
+ providerStalkerIndexSectionMutexes.computeIfAbsent(lockKey) { Mutex() }.also { sectionMutex ->
+ sectionMutex.lock()
+ }
+ }
+ try {
+ return block()
+ } finally {
+ mutex.unlock()
+ }
+ }
+
+ private suspend fun <T> withStalkerSummaryProviderLock(providerId: Long, block: suspend () -> T): T {
+ val mutex = syncAdmissionMutex.withLock {
+ providerStalkerSummaryMutexes.computeIfAbsent(providerId) { Mutex() }.also { providerMutex ->
+ providerMutex.lock()
+ }
+ }
+ try {
+ return block()
+ } finally {
+ mutex.unlock()
+ }
+ }
+
+ private suspend fun <T> withStalkerSummaryLock(
+ providerId: Long,
+ section: ContentType?,
+ providerMaxConnections: Int,
+ block: suspend () -> T
+ ): T = when (section) {
+ ContentType.MOVIE,
+ ContentType.SERIES -> if (providerMaxConnections > 1) {
+ withStalkerIndexSectionLock(providerId, section, block)
+ } else {
+ withStalkerSummaryProviderLock(providerId, block)
+ }
+ else -> withProviderLock(providerId, block)
+ }
+
+ private suspend fun <T> withStalkerFetchPermit(provider: Provider, block: suspend () -> T): T {
+ val permitCount = minOf(
+ provider.maxConnections.coerceAtLeast(1),
+ STALKER_MAX_PARALLEL_CATEGORY_FETCHES
+ ).coerceAtLeast(1)
+ val semaphore = syncAdmissionMutex.withLock {
+ providerStalkerFetchSemaphores.computeIfAbsent(provider.id) { Semaphore(permitCount) }
+ }
+ return semaphore.withPermit { block() }
+ }
+
suspend fun runWhenNoSyncActive(block: suspend () -> Boolean): Boolean =
syncAdmissionMutex.withLock {
if (isAnySyncMutexLocked()) {
@@ -320,6 +389,7 @@ class SyncManager @Inject constructor(
syncCatalogStore.clearProviderStaging(providerId)
epgRepository.onProviderDeleted(providerId)
providerSyncMutexes.remove(providerId)
+ providerStalkerSummaryMutexes.remove(providerId)
providerEpgMutexes.remove(providerId)
}
}
@@ -354,19 +424,37 @@ class SyncManager @Inject constructor(
}
}
- fun scheduleStalkerIndexSync(providerId: Long, section: ContentType? = null, force: Boolean = false) {
+ fun scheduleStalkerIndexSync(
+ providerId: Long,
+ section: ContentType? = null,
+ force: Boolean = false,
+ initialDelaySeconds: Long = 0L
+ ) {
runCatching {
StalkerIndexWorker.enqueue(
context = applicationContext,
providerId = providerId,
section = section?.name,
- force = force
+ force = force,
+ initialDelaySeconds = initialDelaySeconds
)
}.onFailure { error ->
Log.w(TAG, "Failed to schedule Stalker index work for provider $providerId (${section?.name ?: "all"}): ${sanitizeThrowableMessage(error)}")
}
}
+ fun noteStalkerPlaybackStarted(providerId: Long) {
+ if (providerId <= 0L) return
+ StalkerTrafficCoordinator.notePlaybackStarted(providerId)
+ }
+
+ fun noteStalkerPlaybackStopped(providerId: Long) {
+ if (providerId <= 0L) return
+ StalkerTrafficCoordinator.notePlaybackStopped(providerId)
+ scheduleStalkerIndexSync(providerId, ContentType.MOVIE, force = false)
+ scheduleStalkerIndexSync(providerId, ContentType.SERIES, force = false)
+ }
+
suspend fun prioritizeXtreamIndexCategory(
providerId: Long,
section: ContentType,
@@ -393,6 +481,32 @@ class SyncManager @Inject constructor(
scheduleXtreamIndexSync(providerId, section, force = false)
}
+ suspend fun prioritizeStalkerIndexCategory(
+ providerId: Long,
+ section: ContentType,
+ categoryId: Long
+ ) {
+ if (section != ContentType.MOVIE && section != ContentType.SERIES) return
+ val now = System.currentTimeMillis()
+ val updated = xtreamIndexJobDao.requestCategoryPriority(
+ providerId = providerId,
+ section = section.name,
+ categoryId = categoryId,
+ requestedAt = now
+ )
+ if (updated == 0) {
+ upsertXtreamIndexJob(
+ providerId = providerId,
+ section = section.name,
+ state = "QUEUED",
+ now = now,
+ priorityCategoryId = categoryId,
+ priorityRequestedAt = now
+ )
+ }
+ scheduleStalkerIndexSync(providerId, section, force = false)
+ }
+
suspend fun syncEpg(
providerId: Long,
force: Boolean = true,
@@ -1354,6 +1468,7 @@ class SyncManager @Inject constructor(
var sawRetryableFailure = false
val warnings = mutableListOf<String>()
+ val failures = mutableListOf<Throwable>()
sections.forEach { contentType ->
val job = xtreamIndexJobDao.get(providerId, contentType.name)
if (!force && !shouldRunXtreamSummaryIndex(job)) {
@@ -1381,6 +1496,7 @@ class SyncManager @Inject constructor(
lastError = sanitizeThrowableMessage(failure)
)
}
+ failures += failure
sawRetryableFailure = sawRetryableFailure || state != "FAILED_PERMANENT"
warnings += "${contentType.name.lowercase().replaceFirstChar { it.titlecase() }} indexing failed: ${sanitizeThrowableMessage(failure)}"
}
@@ -1388,7 +1504,8 @@ class SyncManager @Inject constructor(
if (warnings.isNotEmpty()) {
val message = warnings.joinToString("; ")
- val exception = if (sawRetryableFailure) IOException(message) else IllegalStateException(message)
+ val cause = failures.firstOrNull()
+ val exception = if (sawRetryableFailure) IOException(message, cause) else IllegalStateException(message, cause)
com.streamvault.domain.model.Result.error(warnings.first(), exception)
} else {
com.streamvault.domain.model.Result.success(Unit)
@@ -1410,7 +1527,20 @@ class SyncManager @Inject constructor(
force: Boolean = false,
maxCategoriesPerSection: Int? = null,
onProgress: ((String) -> Unit)? = null
- ): com.streamvault.domain.model.Result<Unit> = withProviderLock(providerId) lock@{
+ ): com.streamvault.domain.model.Result<Unit> {
+ val providerMaxConnections = providerDao.getById(providerId)?.maxConnections ?: Int.MAX_VALUE
+ return withStalkerSummaryLock(providerId, section, providerMaxConnections) lock@{
+ val playbackDelayMillis = StalkerTrafficCoordinator.deferCatalogFetchMillis(providerId)
+ if (playbackDelayMillis > 0L) {
+ scheduleStalkerIndexSync(
+ providerId = providerId,
+ section = section,
+ force = force,
+ initialDelaySeconds = ((playbackDelayMillis + 999L) / 1000L).coerceAtLeast(1L)
+ )
+ return@lock com.streamvault.domain.model.Result.success(Unit)
+ }
+
val providerEntity = providerDao.getById(providerId)
?: return@lock com.streamvault.domain.model.Result.error("Provider $providerId not found")
if (providerEntity.type != ProviderType.STALKER_PORTAL) {
@@ -1428,6 +1558,7 @@ class SyncManager @Inject constructor(
}
val warnings = mutableListOf<String>()
+ val failures = mutableListOf<Throwable>()
var sawRetryableFailure = false
val categoryLimit = maxCategoriesPerSection?.coerceAtLeast(1) ?: STALKER_INDEX_CATEGORY_SLICE_SIZE
sections.forEach { contentType ->
@@ -1458,6 +1589,7 @@ class SyncManager @Inject constructor(
lastAttemptAt = failedAt,
lastError = sanitizeThrowableMessage(failure)
)
+ failures += failure
sawRetryableFailure = sawRetryableFailure || state != "FAILED_PERMANENT"
warnings += "${contentType.name.lowercase().replaceFirstChar { it.titlecase() }} indexing failed: ${sanitizeThrowableMessage(failure)}"
}
@@ -1465,11 +1597,13 @@ class SyncManager @Inject constructor(
if (warnings.isNotEmpty()) {
val message = warnings.joinToString("; ")
- val exception = if (sawRetryableFailure) IOException(message) else IllegalStateException(message)
+ val cause = failures.firstOrNull()
+ val exception = if (sawRetryableFailure) IOException(message, cause) else IllegalStateException(message, cause)
com.streamvault.domain.model.Result.error(warnings.first(), exception)
} else {
com.streamvault.domain.model.Result.success(Unit)
}
+ }
}
private suspend fun processStalkerSummaryIndexSection(
@@ -1509,29 +1643,6 @@ class SyncManager @Inject constructor(
}
val initialJob = xtreamIndexJobDao.get(provider.id, contentType.name)
- val wildcardCategory = categories.firstOrNull { category ->
- api.isWildcardCategory(contentType, category.categoryId)
- }
- if (
- wildcardCategory != null &&
- initialJob?.priorityCategoryId == null &&
- canAttemptStalkerCategory(
- getStalkerHydrationSnapshot(provider.id, contentType, wildcardCategory.categoryId),
- now
- )
- ) {
- val wildcardCompleted = processStalkerWildcardIndexSection(
- provider = provider,
- api = api,
- contentType = contentType,
- wildcardCategory = wildcardCategory,
- visibleCategories = visibleCategories,
- maxPages = STALKER_WILDCARD_PAGE_SLICE_SIZE,
- initialJob = initialJob,
- onProgress = onProgress
- )
- if (wildcardCompleted) return
- }
val priorityCategoryId = initialJob?.priorityCategoryId
val hydrationByCategory = visibleCategories.associate { category ->
@@ -1553,6 +1664,9 @@ class SyncManager @Inject constructor(
lastError = null
)
+ var restoreMovieWatchProgressPending = false
+
+ try {
val pending = buildList {
priorityCategoryId
?.takeIf { requestedId -> visibleCategories.any { it.categoryId == requestedId } }
@@ -1571,23 +1685,106 @@ class SyncManager @Inject constructor(
}
}
+ data class StalkerCategoryFetchAttempt(
+ val category: CategoryEntity,
+ val hydration: StalkerHydrationSnapshot?,
+ val nextPage: Int
+ )
+
+ data class StalkerCategoryFetchResult(
+ val attempt: StalkerCategoryFetchAttempt,
+ val result: Result<com.streamvault.data.remote.stalker.StalkerPagedResult<out Any>>
+ )
+
+ data class StalkerSuccessfulPage(
+ val category: CategoryEntity,
+ val hydration: StalkerHydrationSnapshot?,
+ val requestedPage: Int,
+ val items: List<Any>,
+ val totalPages: Int,
+ val pageSize: Int,
+ val isComplete: Boolean,
+ val pageFingerprint: String?
+ )
+
var clearPriority = false
var skippedMalformedRows = initialJob?.skippedMalformedRows ?: 0
- pending.forEach { category ->
- progress(provider.id, onProgress, "Indexing ${xtreamIndexSectionLabel(contentType)}: ${category.name}")
- val hydration = getStalkerHydrationSnapshot(provider.id, contentType, category.categoryId)
- val nextPage = nextStalkerAttemptPage(hydration)
- markStalkerAttemptStarted(
- providerId = provider.id,
- contentType = contentType,
- categoryId = category.categoryId,
- hydration = hydration,
- attemptedPage = nextPage,
- now = System.currentTimeMillis()
+ val runtimeProfile = CatalogSyncRuntimeProfile.from(applicationContext)
+ val parallelFetchLimit = stalkerCategoryFetchConcurrencyLimit(
+ provider = provider,
+ runtimeProfile = runtimeProfile,
+ maxCategories = maxCategories
+ )
+ val runDeadlineAt = System.currentTimeMillis() + STALKER_MAX_SECTION_RUN_MILLIS
+ val pendingQueue = ArrayDeque(pending)
+ var forceSequential = parallelFetchLimit <= 1
+ while (pendingQueue.isNotEmpty() && System.currentTimeMillis() < runDeadlineAt) {
+ if (StalkerTrafficCoordinator.shouldDeferCatalogFetch(provider.id)) {
+ break
+ }
+ val windowSize = minOf(
+ if (forceSequential) 1 else parallelFetchLimit,
+ pendingQueue.size
)
- when (val result = fetchStalkerSummaryPageWithRecovery(api, contentType, category.categoryId, nextPage)) {
+ val window = buildList {
+ repeat(windowSize) {
+ add(pendingQueue.removeFirst())
+ }
+ }
+ val attempts = window.mapNotNull { category ->
+ val hydration = getStalkerHydrationSnapshot(provider.id, contentType, category.categoryId)
+ if (!canAttemptStalkerCategory(hydration, System.currentTimeMillis())) {
+ return@mapNotNull null
+ }
+ val nextPage = nextStalkerAttemptPage(hydration)
+ progress(
+ provider.id,
+ onProgress,
+ "Indexing ${xtreamIndexSectionLabel(contentType)}: ${category.name} page $nextPage"
+ )
+ markStalkerAttemptStarted(
+ providerId = provider.id,
+ contentType = contentType,
+ categoryId = category.categoryId,
+ hydration = hydration,
+ attemptedPage = nextPage,
+ now = System.currentTimeMillis()
+ )
+ StalkerCategoryFetchAttempt(
+ category = category,
+ hydration = hydration,
+ nextPage = nextPage
+ )
+ }
+ if (attempts.isEmpty()) {
+ continue
+ }
+ val fetchedResults = coroutineScope {
+ attempts.map { attempt ->
+ async {
+ StalkerCategoryFetchResult(
+ attempt = attempt,
+ result = withStalkerFetchPermit(provider) {
+ fetchStalkerSummaryPageWithRecovery(
+ api = api,
+ contentType = contentType,
+ categoryId = attempt.category.categoryId,
+ page = attempt.nextPage
+ )
+ }
+ )
+ }
+ }.awaitAll()
+ }
+
+ var windowRetryableFailures = 0
+ val successfulPages = mutableListOf<StalkerSuccessfulPage>()
+ for (fetched in fetchedResults) {
+ val category = fetched.attempt.category
+ val hydration = fetched.attempt.hydration
+ val nextPage = fetched.attempt.nextPage
+ when (val result = fetched.result) {
is Result.Success -> {
- val indexedAt = System.currentTimeMillis()
val pageFingerprint = stalkerPageFingerprint(result.data.items, contentType)
val anomaly = detectStalkerPageAnomaly(
hydration = hydration,
@@ -1596,6 +1793,7 @@ class SyncManager @Inject constructor(
pageFingerprint = pageFingerprint
)
if (anomaly != null) {
+ val indexedAt = System.currentTimeMillis()
skippedMalformedRows += result.data.items.size
markStalkerAttemptFailed(
providerId = provider.id,
@@ -1617,35 +1815,27 @@ class SyncManager @Inject constructor(
lastAttemptAt = indexedAt,
lastError = anomaly
)
- return@forEach
+ continue
}
val dedupedItems = dedupeStalkerPageItems(result.data.items, contentType)
skippedMalformedRows += (result.data.items.size - dedupedItems.size).coerceAtLeast(0)
- when (contentType) {
- ContentType.MOVIE -> upsertXtreamMovieSummaryBatch(provider.id, dedupedItems.filterIsInstance<Movie>(), indexedAt)
- ContentType.SERIES -> upsertXtreamSeriesSummaryBatch(provider.id, dedupedItems.filterIsInstance<Series>(), indexedAt)
- else -> Unit
- }
- val categoryCount = currentStalkerCategoryCount(provider.id, contentType, category.categoryId)
- val pageComplete = result.data.isComplete ||
- (result.data.items.isEmpty() && result.data.totalPages in 1..nextPage)
- markStalkerAttemptSucceeded(
- providerId = provider.id,
- contentType = contentType,
- categoryId = category.categoryId,
+ successfulPages += StalkerSuccessfulPage(
+ category = category,
hydration = hydration,
- attemptedPage = nextPage,
- now = indexedAt,
- itemCount = categoryCount,
+ requestedPage = nextPage,
+ items = dedupedItems,
totalPages = result.data.totalPages,
pageSize = result.data.pageSize,
- pageComplete = pageComplete,
+ isComplete = result.data.isComplete,
pageFingerprint = pageFingerprint
)
}
is Result.Error -> {
val failedAt = System.currentTimeMillis()
val retryable = stalkerIndexFailureState(result.exception ?: IllegalStateException(result.message)) != "FAILED_PERMANENT"
+ if (retryable) {
+ windowRetryableFailures += 1
+ }
markStalkerAttemptFailed(
providerId = provider.id,
contentType = contentType,
@@ -1671,9 +1861,58 @@ class SyncManager @Inject constructor(
)
}
is Result.Loading -> Unit
+ }
+ if (priorityCategoryId == category.categoryId) {
+ clearPriority = true
+ }
+ }
+ if (successfulPages.isNotEmpty()) {
+ val indexedAt = System.currentTimeMillis()
+ when (contentType) {
+ ContentType.MOVIE -> upsertXtreamMovieSummaryBatch(
+ provider.id,
+ successfulPages.flatMap { page -> page.items.filterIsInstance<Movie>() },
+ indexedAt,
+ restoreWatchProgress = false
+ )
+ ContentType.SERIES -> upsertXtreamSeriesSummaryBatch(
+ provider.id,
+ successfulPages.flatMap { page -> page.items.filterIsInstance<Series>() },
+ indexedAt
+ )
+ else -> Unit
+ }
+ if (contentType == ContentType.MOVIE) {
+ restoreMovieWatchProgressPending = true
+ }
+ for (page in successfulPages) {
+ val pageComplete = page.isComplete ||
+ (page.items.isEmpty() && page.totalPages in 1..page.requestedPage)
+ val categoryCount = currentStalkerCategoryCount(provider.id, contentType, page.category.categoryId)
+ markStalkerAttemptSucceeded(
+ providerId = provider.id,
+ contentType = contentType,
+ categoryId = page.category.categoryId,
+ hydration = page.hydration,
+ attemptedPage = page.requestedPage,
+ now = indexedAt,
+ itemCount = categoryCount,
+ totalPages = page.totalPages,
+ pageSize = page.pageSize,
+ pageComplete = pageComplete,
+ pageFingerprint = page.pageFingerprint
+ )
+ if (!pageComplete && System.currentTimeMillis() < runDeadlineAt) {
+ pendingQueue.addLast(page.category)
+ }
+ }
}
- if (priorityCategoryId == category.categoryId) {
- clearPriority = true
+ if (!forceSequential && windowRetryableFailures >= minOf(2, attempts.size)) {
+ forceSequential = true
+ Log.w(
+ TAG,
+ "Stalker ${contentType.name} category fetch downgraded to sequential mode for provider ${provider.id} after retryable parallel failures."
+ )
}
}
@@ -1722,6 +1961,26 @@ class SyncManager @Inject constructor(
if (hasMoreCategories) {
scheduleStalkerIndexSync(provider.id, contentType, force = false)
}
+ } finally {
+ if (contentType == ContentType.MOVIE && restoreMovieWatchProgressPending) {
+ movieDao.restoreWatchProgress(provider.id)
+ }
+ }
+ }
+
+ private fun stalkerCategoryFetchConcurrencyLimit(
+ provider: Provider,
+ runtimeProfile: CatalogSyncRuntimeProfile,
+ maxCategories: Int
+ ): Int {
+ val runtimeLimit = runtimeProfile.maxCategoryConcurrency.coerceAtLeast(1)
+ val providerLimit = provider.maxConnections.coerceAtLeast(1)
+ val effectiveLimit = maxOf(runtimeLimit, providerLimit)
+ return minOf(
+ maxCategories.coerceAtLeast(1),
+ effectiveLimit,
+ STALKER_MAX_PARALLEL_CATEGORY_FETCHES
+ ).coerceAtLeast(1)
}
private suspend fun processStalkerWildcardIndexSection(
@@ -1762,8 +2021,11 @@ class SyncManager @Inject constructor(
val seenPageFingerprints = mutableSetOf<String>()
var pagesProcessed = 0
var lastError: String? = null
+ var restoreMovieWatchProgressPending = false
+ try {
while (pagesProcessed < maxPages) {
+ if (StalkerTrafficCoordinator.shouldDeferCatalogFetch(provider.id)) break
val currentHydration = getStalkerHydrationSnapshot(provider.id, contentType, wildcardCategory.categoryId)
if (!canAttemptStalkerCategory(currentHydration, System.currentTimeMillis())) break
val nextPage = nextStalkerAttemptPage(currentHydration)
@@ -1831,10 +2093,18 @@ class SyncManager @Inject constructor(
}
when (contentType) {
- ContentType.MOVIE -> upsertXtreamMovieSummaryBatch(provider.id, visibleItems.filterIsInstance<Movie>(), indexedAt)
+ ContentType.MOVIE -> upsertXtreamMovieSummaryBatch(
+ provider.id,
+ visibleItems.filterIsInstance<Movie>(),
+ indexedAt,
+ restoreWatchProgress = false
+ )
ContentType.SERIES -> upsertXtreamSeriesSummaryBatch(provider.id, visibleItems.filterIsInstance<Series>(), indexedAt)
else -> Unit
}
+ if (contentType == ContentType.MOVIE && visibleItems.isNotEmpty()) {
+ restoreMovieWatchProgressPending = true
+ }
val pageComplete = result.data.isComplete ||
(result.data.items.isEmpty() && result.data.totalPages in 1..nextPage)
@@ -1912,6 +2182,11 @@ class SyncManager @Inject constructor(
scheduleStalkerIndexSync(provider.id, contentType, force = false)
}
return true
+ } finally {
+ if (contentType == ContentType.MOVIE && restoreMovieWatchProgressPending) {
+ movieDao.restoreWatchProgress(provider.id)
+ }
+ }
}
private data class StalkerHydrationSnapshot(
@@ -2191,15 +2466,61 @@ class SyncManager @Inject constructor(
page: Int
): Result<com.streamvault.data.remote.stalker.StalkerPagedResult<out Any>> {
val initial = fetchStalkerSummaryPage(api, contentType, categoryId, page)
- if (initial is Result.Error && isLikelyStalkerAuthFailure(initial.message, initial.exception)) {
+ val recovered = if (initial is Result.Error && isLikelyStalkerAuthFailure(initial.message, initial.exception)) {
Log.w(
TAG,
"Retrying Stalker ${contentType.name} page $page after auth refresh for category $categoryId"
)
api.invalidateAuthentication()
- return fetchStalkerSummaryPage(api, contentType, categoryId, page)
+ fetchStalkerSummaryPage(api, contentType, categoryId, page)
+ } else {
+ initial
+ }
+ if (page != 1 || recovered !is Result.Success || recovered.data.items.isNotEmpty()) {
+ return recovered
+ }
+ val fallback = fetchStalkerSummaryFirstPageFallback(api, contentType, categoryId)
+ return if (fallback is Result.Success && fallback.data.items.isNotEmpty()) {
+ Log.i(
+ TAG,
+ "Recovered empty paged Stalker ${contentType.name} first page with non-paged fallback for category $categoryId"
+ )
+ fallback
+ } else {
+ recovered
}
- return initial
+ }
+
+ private suspend fun fetchStalkerSummaryFirstPageFallback(
+ api: StalkerProvider,
+ contentType: ContentType,
+ categoryId: Long
+ ): Result<com.streamvault.data.remote.stalker.StalkerPagedResult<out Any>> = when (contentType) {
+ ContentType.MOVIE -> when (val result = api.getVodStreams(categoryId)) {
+ is Result.Success -> Result.success(
+ com.streamvault.data.remote.stalker.StalkerPagedResult(
+ items = result.data,
+ page = 1,
+ totalPages = 1,
+ pageSize = result.data.size
+ )
+ )
+ is Result.Error -> Result.error(result.message, result.exception)
+ is Result.Loading -> Result.error("Unexpected loading state from Stalker movie first-page fallback")
+ }
+ ContentType.SERIES -> when (val result = api.getSeriesList(categoryId)) {
+ is Result.Success -> Result.success(
+ com.streamvault.data.remote.stalker.StalkerPagedResult(
+ items = result.data,
+ page = 1,
+ totalPages = 1,
+ pageSize = result.data.size
+ )
+ )
+ is Result.Error -> Result.error(result.message, result.exception)
+ is Result.Loading -> Result.error("Unexpected loading state from Stalker series first-page fallback")
+ }
+ else -> Result.error("Unsupported Stalker summary first-page fallback section: $contentType")
}
private suspend fun fetchStalkerWildcardSummaryPageWithRecovery(
@@ -2428,6 +2749,8 @@ class SyncManager @Inject constructor(
val resumeJob = initialJob?.takeIf { it.state in setOf("QUEUED", "RUNNING", "PARTIAL") }
val categoryLimit = maxCategories?.coerceAtLeast(1) ?: Int.MAX_VALUE
var nextCategoryIndex = resumeJob?.nextCategoryIndex?.coerceIn(0, indexedCategories.size) ?: 0
+ var restoreMovieWatchProgressPending = false
+ try {
if (shouldAttemptFullXtreamSummaryStream(resumeJob, priorityCategoryId)) {
when (val streamed = streamFullXtreamSummaryIndex(
provider = provider,
@@ -2523,7 +2846,8 @@ class SyncManager @Inject constructor(
ContentType.MOVIE -> upsertXtreamMovieSummaryBatch(
providerId = provider.id,
movies = categoryOutcome.items.filterIsInstance<Movie>(),
- indexedAt = System.currentTimeMillis()
+ indexedAt = System.currentTimeMillis(),
+ restoreWatchProgress = false
)
ContentType.SERIES -> upsertXtreamSeriesSummaryBatch(
providerId = provider.id,
@@ -2537,6 +2861,9 @@ class SyncManager @Inject constructor(
skippedMalformedRows += (categoryOutcome.rawCount - categoryOutcome.items.size).coerceAtLeast(0)
completedCategories++
}
+ if (contentType == ContentType.MOVIE && accepted > 0) {
+ restoreMovieWatchProgressPending = true
+ }
if (category.categoryId == priorityCategoryId) {
priorityHandled = true
upsertXtreamIndexJob(
@@ -2621,6 +2948,11 @@ class SyncManager @Inject constructor(
lastError
)
}
+ } finally {
+ if (contentType == ContentType.MOVIE && restoreMovieWatchProgressPending) {
+ movieDao.restoreWatchProgress(provider.id)
+ }
+ }
}
private fun shouldAttemptFullXtreamSummaryStream(
@@ -2661,16 +2993,22 @@ class SyncManager @Inject constructor(
)
var indexedRows = 0
+ var restoreMovieWatchProgressPending = false
val categoryNamesById = categories.associate { it.categoryId to it.name }
val adultCategoryIds = categories.filter { it.isAdult }.mapTo(mutableSetOf()) { it.categoryId }
+ try {
val streamResult = when (contentType) {
ContentType.MOVIE -> api.streamVodSummaries(adultCategoryIds = adultCategoryIds) { batch ->
val accepted = upsertXtreamMovieSummaryBatch(
providerId = provider.id,
movies = batch.withMovieCategoryNames(categoryNamesById),
- indexedAt = System.currentTimeMillis()
+ indexedAt = System.currentTimeMillis(),
+ restoreWatchProgress = false
)
indexedRows += accepted
+ if (accepted > 0) {
+ restoreMovieWatchProgressPending = true
+ }
upsertXtreamIndexJob(
providerId = provider.id,
section = contentType.name,
@@ -2733,6 +3071,11 @@ class SyncManager @Inject constructor(
is Result.Error -> streamResult
Result.Loading -> Result.error("Xtream summary stream did not complete")
}
+ } finally {
+ if (contentType == ContentType.MOVIE && restoreMovieWatchProgressPending) {
+ movieDao.restoreWatchProgress(provider.id)
+ }
+ }
}
private fun List<Movie>.withMovieCategoryNames(categoryNamesById: Map<Long, String>): List<Movie> =
@@ -2750,7 +3093,8 @@ class SyncManager @Inject constructor(
private suspend fun upsertXtreamMovieSummaryBatch(
providerId: Long,
movies: List<Movie>,
- indexedAt: Long
+ indexedAt: Long,
+ restoreWatchProgress: Boolean = true
): Int {
if (movies.isEmpty()) return 0
val incoming = movies.map { movie -> movie.toEntity().copy(cacheState = "SUMMARY_ONLY", detailHydratedAt = 0L, remoteStaleAt = 0L) }
@@ -2769,7 +3113,9 @@ class SyncManager @Inject constructor(
}
)
}
- movieDao.restoreWatchProgress(providerId)
+ if (restoreWatchProgress) {
+ movieDao.restoreWatchProgress(providerId)
+ }
return merged.size
}
@@ -4654,24 +5000,39 @@ class SyncManager @Inject constructor(
api: StalkerProvider,
categories: List<com.streamvault.domain.model.Category>,
onProgress: ((String) -> Unit)?
- ): List<Channel> {
+ ): StalkerLiveCategoryLoadResult {
progress(api.providerId, onProgress, "Loading live channels...")
val bulkChannels = requireResult(api.getLiveStreams(null), "Failed to load live channels")
if (categories.isEmpty()) {
- return bulkChannels
+ return StalkerLiveCategoryLoadResult(channels = bulkChannels)
}
val hasResolvedCategories = bulkChannels.any { channel -> channel.categoryId != null }
if (bulkChannels.isNotEmpty() && hasResolvedCategories) {
- return bulkChannels.distinctBy { it.streamId }
+ return StalkerLiveCategoryLoadResult(channels = bulkChannels.distinctBy { it.streamId })
}
- return categories.flatMap { category ->
- progress(api.providerId, onProgress, "Loading ${category.name}...")
- requireResult(api.getLiveStreams(category.id), "Failed to load live channels for ${category.name}")
- }.distinctBy { it.streamId }
+ return fetchStalkerLiveChannelsByCategory(
+ provider = requireNotNull(providerDao.getById(api.providerId)) {
+ "Provider ${api.providerId} not found while loading live channels"
+ }.toDomain(),
+ api = api,
+ categories = categories.map { category ->
+ XtreamCategory(
+ categoryId = category.id.toString(),
+ categoryName = category.name,
+ isAdult = category.isAdult
+ )
+ },
+ onProgress = onProgress
+ )
}
+ private data class StalkerLiveCategoryLoadResult(
+ val channels: List<Channel>,
+ val warnings: List<String> = emptyList()
+ )
+
private data class StalkerLiveCatalogResult(
val categories: List<CategoryEntity>,
val channels: List<Channel>,
@@ -4729,24 +5090,55 @@ class SyncManager @Inject constructor(
batch.clear()
}
+ suspend fun finalizeStagedImport(): StagedStalkerLiveCatalogResult {
+ val sessionId = stagedSessionId
+ if (acceptedCount == 0 || sessionId == null) {
+ throw IllegalStateException("Stalker live catalog returned no usable channels.")
+ }
+
+ if (hiddenLiveCategoryIds.isNotEmpty()) {
+ mergeHiddenChannelsIntoStaging(provider.id, sessionId, hiddenLiveCategoryIds)
+ }
+ val hiddenCategories = if (hiddenLiveCategoryIds.isNotEmpty()) {
+ categoryDao.getByProviderAndTypeSync(provider.id, ContentType.LIVE.name)
+ .filter { category -> category.categoryId in hiddenLiveCategoryIds }
+ } else {
+ emptyList()
+ }
+ val categories = (
+ mergePreferredAndFallbackCategories(
+ preferredCategories,
+ fallbackCollector.entities().takeIf { it.isNotEmpty() }
+ ).orEmpty() + hiddenCategories
+ )
+ .distinctBy { it.categoryId to it.type }
+ .takeIf { it.isNotEmpty() }
+ syncCatalogStore.applyStagedLiveCatalog(provider.id, sessionId, categories)
+ return StagedStalkerLiveCatalogResult(acceptedCount, warnings)
+ }
+
try {
+ var bulkFailure: Exception? = null
when (val streamResult = api.streamLiveStreams { channel ->
- if (channel.categoryId != null && channel.categoryId in hiddenLiveCategoryIds) {
- return@streamLiveStreams
- }
- batch += channel
- if (batch.size >= XTREAM_FALLBACK_STAGE_BATCH_SIZE) {
+ if (channel.categoryId != null && channel.categoryId in hiddenLiveCategoryIds) {
+ return@streamLiveStreams
+ }
+ batch += channel
+ if (batch.size >= XTREAM_FALLBACK_STAGE_BATCH_SIZE) {
+ flushBatch()
+ }
+ }) {
+ is com.streamvault.domain.model.Result.Success -> {
flushBatch()
+ return finalizeStagedImport()
}
- }) {
- is com.streamvault.domain.model.Result.Success -> flushBatch()
is com.streamvault.domain.model.Result.Error -> {
val profileDiagnostic = stalkerCatalogAccessDiagnostic(
api = api,
primaryMessage = categoriesErrorMessage.orEmpty(),
fallbackMessage = streamResult.message
)
- throw IllegalStateException(
+ bulkFailure = IllegalStateException(
buildString {
append(streamResult.message.ifBlank { "Failed to load live channels" })
profileDiagnostic?.let {
@@ -4760,30 +5152,48 @@ class SyncManager @Inject constructor(
is com.streamvault.domain.model.Result.Loading -> throw IllegalStateException("Unexpected loading state")
}
- val sessionId = stagedSessionId
- if (acceptedCount == 0 || sessionId == null) {
- throw IllegalStateException("Stalker bulk live catalog returned no usable channels.")
- }
+ stagedSessionId?.let { syncCatalogStore.discardStagedImport(provider.id, it) }
+ stagedSessionId = null
+ acceptedCount = 0
+ batch.clear()
+ seenStreamIds.clear()
- if (hiddenLiveCategoryIds.isNotEmpty()) {
- mergeHiddenChannelsIntoStaging(provider.id, sessionId, hiddenLiveCategoryIds)
- }
- val hiddenCategories = if (hiddenLiveCategoryIds.isNotEmpty()) {
- categoryDao.getByProviderAndTypeSync(provider.id, ContentType.LIVE.name)
- .filter { category -> category.categoryId in hiddenLiveCategoryIds }
- } else {
- emptyList()
+ val fallbackCategories = preferredCategories.orEmpty()
+ if (fallbackCategories.isEmpty()) {
+ throw bulkFailure ?: IllegalStateException("Failed to load live channels")
}
- val categories = (
- mergePreferredAndFallbackCategories(
- preferredCategories,
- fallbackCollector.entities().takeIf { it.isNotEmpty() }
- ).orEmpty() + hiddenCategories
+
+ Log.w(
+ TAG,
+ "Stalker bulk live sync failed for provider ${provider.id}; falling back to category live fetches.",
+ bulkFailure
)
- .distinctBy { it.categoryId to it.type }
- .takeIf { it.isNotEmpty() }
- syncCatalogStore.applyStagedLiveCatalog(provider.id, sessionId, categories)
- return StagedStalkerLiveCatalogResult(acceptedCount, warnings)
+ warnings += "Bulk live request failed; recovered using per-category live fetches."
+
+ val fallbackResult = fetchStalkerLiveChannelsByCategory(
+ provider = provider,
+ api = api,
+ categories = fallbackCategories.map { category ->
+ XtreamCategory(
+ categoryId = category.categoryId.toString(),
+ categoryName = category.name,
+ isAdult = category.isAdult
+ )
+ },
+ onProgress = onProgress
+ )
+ warnings += fallbackResult.warnings
+ fallbackResult.channels.forEach { channel ->
+ if (channel.categoryId != null && channel.categoryId in hiddenLiveCategoryIds) {
+ return@forEach
+ }
+ batch += channel
+ if (batch.size >= XTREAM_FALLBACK_STAGE_BATCH_SIZE) {
+ flushBatch()
+ }
+ }
+ flushBatch()
+ return finalizeStagedImport()
} catch (error: CancellationException) {
stagedSessionId?.let { syncCatalogStore.discardStagedImport(provider.id, it) }
throw error
@@ -4800,10 +5210,11 @@ class SyncManager @Inject constructor(
): StalkerLiveCatalogResult {
return when (val categoriesResult = api.getLiveCategories()) {
is com.streamvault.domain.model.Result.Success -> {
- val channels = loadStalkerChannelsByCategory(api, categoriesResult.data, onProgress)
+ val channelResult = loadStalkerChannelsByCategory(api, categoriesResult.data, onProgress)
StalkerLiveCatalogResult(
categories = categoriesResult.data.map { it.toEntity(provider.id) },
- channels = channels
+ channels = channelResult.channels,
+ warnings = channelResult.warnings
)
}
is com.streamvault.domain.model.Result.Error -> {
@@ -4855,6 +5266,87 @@ class SyncManager @Inject constructor(
}
}
+ private suspend fun fetchStalkerLiveChannelsByCategory(
+ provider: Provider,
+ api: StalkerProvider,
+ categories: List<XtreamCategory>,
+ onProgress: ((String) -> Unit)?
+ ): StalkerLiveCategoryLoadResult {
+ if (categories.isEmpty()) {
+ return StalkerLiveCategoryLoadResult(emptyList())
+ }
+
+ val runtimeProfile = CatalogSyncRuntimeProfile.from(applicationContext)
+ val concurrency = minOf(
+ categories.size,
+ runtimeProfile.maxCategoryConcurrency,
+ STALKER_MAX_PARALLEL_CATEGORY_FETCHES
+ ).coerceAtLeast(1)
+ val executionPlan = executeCategoryRecoveryPlan(
+ provider = provider,
+ categories = categories,
+ initialConcurrency = concurrency,
+ sectionLabel = "Live TV",
+ sequentialModeWarning = LIVE_CATEGORY_SEQUENTIAL_MODE_WARNING,
+ onProgress = onProgress,
+ fetch = { category -> fetchStalkerLiveCategoryOutcome(api, category) }
+ )
+ var timedOutcomes = executionPlan.outcomes
+
+ val categoryOutcomes = timedOutcomes.map { it.outcome }
+ val failureCount = timedOutcomes.count { it.outcome is CategoryFetchOutcome.Failure }
+ val fastFailureCount = timedOutcomes.count {
+ it.elapsedMs <= 5_000L && it.outcome is CategoryFetchOutcome.Failure
+ }
+ val downgradeRecommended = catalogStrategySupport.shouldDowngradeCategorySync(
+ categories.size,
+ failureCount,
+ fastFailureCount,
+ categoryOutcomes
+ )
+ var warnings = executionPlan.warnings
+
+ if (concurrency > 1 && catalogStrategySupport.shouldRetryFailedCategories(
+ categories.size,
+ failureCount,
+ downgradeRecommended,
+ categoryOutcomes
+ )
+ ) {
+ val failedRetryTotal = timedOutcomes.count { it.outcome is CategoryFetchOutcome.Failure }
+ Log.w(
+ TAG,
+ "Stalker live category sync is continuing in sequential mode for failed categories on provider ${provider.id}."
+ )
+ progress(provider.id, onProgress, "Retrying failed Live TV categories 0/$failedRetryTotal...")
+ timedOutcomes = continueFailedCategoryOutcomes(
+ provider = provider,
+ timedOutcomes = timedOutcomes,
+ fetchSequentially = { category -> fetchStalkerLiveCategoryOutcome(api, category) },
+ onCategoryRetried = { completed, total, _ ->
+ progress(provider.id, onProgress, "Retrying failed Live TV categories $completed/$total...")
+ }
+ )
+ warnings = (
+ warnings + if (downgradeRecommended) listOf(LIVE_CATEGORY_SEQUENTIAL_MODE_WARNING) else emptyList()
+ ).distinct()
+ }
+
+ val finalOutcomes = timedOutcomes.map { it.outcome }
+ val channels = finalOutcomes
+ .filterIsInstance<CategoryFetchOutcome.Success<Channel>>()
+ .flatMap { outcome -> outcome.items }
+ .distinctBy { channel -> channel.streamId }
+ val outcomeWarnings = finalOutcomes
+ .filterIsInstance<CategoryFetchOutcome.Failure>()
+ .map { failure -> categoryFailureWarning("Live TV", failure.categoryName, failure.error) }
+
+ return StalkerLiveCategoryLoadResult(
+ channels = channels,
+ warnings = (outcomeWarnings + warnings).distinct()
+ )
+ }
+
private suspend fun stalkerCatalogAccessDiagnostic(
api: StalkerProvider,
primaryMessage: String,
@@ -4957,6 +5449,40 @@ class SyncManager @Inject constructor(
category: XtreamCategory
): TimedCategoryOutcome<Channel> = xtreamFetcher.fetchLiveCategoryOutcome(provider, api, category)
+ private suspend fun fetchStalkerLiveCategoryOutcome(
+ api: StalkerProvider,
+ category: XtreamCategory
+ ): TimedCategoryOutcome<Channel> {
+ val startedAt = System.currentTimeMillis()
+ val outcome = try {
+ val categoryId = category.categoryId.toLongOrNull()
+ ?: throw IllegalArgumentException("Invalid Stalker live category id '${category.categoryId}'")
+ val channels = requireResult(
+ api.getLiveStreams(categoryId),
+ "Failed to load live channels for ${category.categoryName}"
+ ).distinctBy { it.streamId }
+ if (channels.isEmpty()) {
+ CategoryFetchOutcome.Empty(category.categoryName)
+ } else {
+ CategoryFetchOutcome.Success(
+ categoryName = category.categoryName,
+ items = channels,
+ rawCount = channels.size
+ )
+ }
+ } catch (error: CancellationException) {
+ throw error
+ } catch (error: Throwable) {
+ CategoryFetchOutcome.Failure(category.categoryName, error)
+ }
+
+ return TimedCategoryOutcome(
+ category = category,
+ outcome = outcome,
+ elapsedMs = System.currentTimeMillis() - startedAt
+ )
+ }
+
private suspend fun fetchMovieCategoryOutcome(
provider: Provider,
api: XtreamProvider,
diff --git a/data/src/test/java/com/streamvault/data/remote/stalker/StalkerTrafficCoordinatorTest.kt b/data/src/test/java/com/streamvault/data/remote/stalker/StalkerTrafficCoordinatorTest.kt
new file mode 100644
index 00000000..7e95be88
--- /dev/null
+++ b/data/src/test/java/com/streamvault/data/remote/stalker/StalkerTrafficCoordinatorTest.kt
@@ -0,0 +1,40 @@
+package com.streamvault.data.remote.stalker
+
+import com.google.common.truth.Truth.assertThat
+import org.junit.After
+import org.junit.Before
+import org.junit.Test
+
+class StalkerTrafficCoordinatorTest {
+ @Before
+ fun setUp() {
+ StalkerTrafficCoordinator.resetForTests()
+ }
+
+ @After
+ fun tearDown() {
+ StalkerTrafficCoordinator.resetForTests()
+ }
+
+ @Test
+ fun `deferCatalogFetchMillis stays deferred while playback is active beyond legacy window`() {
+ StalkerTrafficCoordinator.notePlaybackStarted(providerId = 7L, now = 1_000L)
+
+ val deferredAtTwoMinutes = StalkerTrafficCoordinator.deferCatalogFetchMillis(
+ providerId = 7L,
+ now = 121_000L
+ )
+
+ assertThat(deferredAtTwoMinutes).isGreaterThan(0L)
+ }
+
+ @Test
+ fun `deferCatalogFetchMillis clears immediately when playback stops`() {
+ StalkerTrafficCoordinator.notePlaybackActivity(providerId = 7L, now = 1_000L)
+ StalkerTrafficCoordinator.notePlaybackStarted(providerId = 7L, now = 2_000L)
+
+ StalkerTrafficCoordinator.notePlaybackStopped(7L)
+
+ assertThat(StalkerTrafficCoordinator.deferCatalogFetchMillis(providerId = 7L, now = 2_001L)).isEqualTo(0L)
+ }
+}
diff --git a/data/src/test/java/com/streamvault/data/repository/MovieRepositoryImplTest.kt b/data/src/test/java/com/streamvault/data/repository/MovieRepositoryImplTest.kt
index 738566b9..93dafeb9 100644
--- a/data/src/test/java/com/streamvault/data/repository/MovieRepositoryImplTest.kt
+++ b/data/src/test/java/com/streamvault/data/repository/MovieRepositoryImplTest.kt
@@ -16,6 +16,7 @@ import com.streamvault.data.local.entity.MovieBrowseEntity
import com.streamvault.data.local.entity.MovieEntity
import com.streamvault.data.local.entity.PlaybackHistoryLiteEntity
import com.streamvault.data.local.entity.ProviderEntity
+import com.streamvault.data.local.entity.XtreamIndexJobEntity
import com.streamvault.data.preferences.PreferencesRepository
import com.streamvault.data.remote.stalker.StalkerDeviceProfile
import com.streamvault.data.remote.stalker.StalkerProviderProfile
@@ -241,6 +242,36 @@ class MovieRepositoryImplTest {
verify(xtreamApiService, never()).getVodStreams(any(), any())
}
+ @Test
+ fun `getMoviesByCategory prioritizes stalker category when background fetch is already running`() = runTest {
+ whenever(preferencesRepository.parentalControlLevel).thenReturn(flowOf(0))
+ whenever(preferencesRepository.xtreamBase64TextCompatibility).thenReturn(flowOf(false))
+ whenever(movieDao.getCountByCategory(7L, 42L)).thenReturn(flowOf(0))
+ whenever(movieDao.getByCategory(7L, 42L)).thenReturn(flowOf(emptyList()))
+ whenever(movieCategoryHydrationDao.get(7L, 42L)).thenReturn(null)
+ whenever(xtreamIndexJobDao.get(7L, ContentType.MOVIE.name)).thenReturn(
+ XtreamIndexJobEntity(providerId = 7L, section = ContentType.MOVIE.name, state = "RUNNING")
+ )
+ whenever(providerDao.getById(7L)).thenReturn(
+ ProviderEntity(
+ id = 7L,
+ name = "Stalker",
+ type = ProviderType.STALKER_PORTAL,
+ serverUrl = "http://example.com",
+ username = "",
+ password = "pass",
+ status = ProviderStatus.ACTIVE
+ )
+ )
+ val repository = createRepository()
+
+ val result = repository.getMoviesByCategory(7L, 42L).first()
+
+ assertThat(result).isEmpty()
+ verify(syncManager).prioritizeStalkerIndexCategory(7L, ContentType.MOVIE, 42L)
+ verify(stalkerApiService, never()).getVodStreamsPage(any(), any(), anyOrNull(), any())
+ }
+
@Test
fun `getMovieDetails uses fresh xtream hydrated cache without refetching`() = runTest {
val hydratedAt = System.currentTimeMillis()
diff --git a/data/src/test/java/com/streamvault/data/repository/SeriesRepositoryImplTest.kt b/data/src/test/java/com/streamvault/data/repository/SeriesRepositoryImplTest.kt
index 03ef4d14..fb5010b5 100644
--- a/data/src/test/java/com/streamvault/data/repository/SeriesRepositoryImplTest.kt
+++ b/data/src/test/java/com/streamvault/data/repository/SeriesRepositoryImplTest.kt
@@ -16,6 +16,7 @@ import com.streamvault.data.local.entity.EpisodeEntity
import com.streamvault.data.local.entity.SeriesEntity
import com.streamvault.data.local.entity.SeriesBrowseEntity
import com.streamvault.data.local.entity.ProviderEntity
+import com.streamvault.data.local.entity.XtreamIndexJobEntity
import com.streamvault.data.preferences.PreferencesRepository
import com.streamvault.data.remote.dto.XtreamSeason
import com.streamvault.data.remote.dto.XtreamSeriesInfoResponse
@@ -115,6 +116,37 @@ class SeriesRepositoryImplTest {
verify(episodeDao, never()).deleteOrphans()
}
+ @Test
+ fun `getSeriesByCategory prioritizes stalker category when background fetch is already running`() = runTest {
+ whenever(preferencesRepository.parentalControlLevel).thenReturn(flowOf(0))
+ whenever(preferencesRepository.xtreamBase64TextCompatibility).thenReturn(flowOf(false))
+ whenever(seriesDao.getCountByCategory(7L, 77L)).thenReturn(flowOf(0))
+ whenever(seriesDao.getByCategory(7L, 77L)).thenReturn(flowOf(emptyList()))
+ whenever(seriesCategoryHydrationDao.get(7L, 77L)).thenReturn(null)
+ whenever(xtreamIndexJobDao.get(7L, ContentType.SERIES.name)).thenReturn(
+ XtreamIndexJobEntity(providerId = 7L, section = ContentType.SERIES.name, state = "RUNNING")
+ )
+ whenever(providerDao.getById(7L)).thenReturn(
+ ProviderEntity(
+ id = 7L,
+ name = "Stalker",
+ type = ProviderType.STALKER_PORTAL,
+ serverUrl = "http://example.com",
+ username = "",
+ password = "pass",
+ status = ProviderStatus.ACTIVE
+ )
+ )
+ val repository = createRepository()
+
+ val result = repository.getSeriesByCategory(7L, 77L).first()
+
+ assertThat(result).isEmpty()
+ verify(syncManager).prioritizeStalkerIndexCategory(7L, ContentType.SERIES, 77L)
+ verify(stalkerApiService, never()).getSeriesPage(any(), any(), anyOrNull(), any())
+ verify(episodeDao, never()).deleteOrphans()
+ }
+
@Test
fun `getSeriesDetails uses fresh xtream hydrated cache without refetching`() = runTest {
val hydratedAt = System.currentTimeMillis()
diff --git a/data/src/test/java/com/streamvault/data/sync/SyncManagerTest.kt b/data/src/test/java/com/streamvault/data/sync/SyncManagerTest.kt
index e823a199..bce11b15 100644
--- a/data/src/test/java/com/streamvault/data/sync/SyncManagerTest.kt
+++ b/data/src/test/java/com/streamvault/data/sync/SyncManagerTest.kt
@@ -23,8 +23,10 @@ import com.streamvault.data.local.dao.XtreamLiveOnboardingDao
import com.streamvault.data.local.entity.CategoryEntity
import com.streamvault.data.local.entity.ChannelEntity
import com.streamvault.data.local.entity.ChannelGuideSyncEntity
+import com.streamvault.data.local.entity.MovieCategoryHydrationEntity
import com.streamvault.data.local.entity.MovieEntity
import com.streamvault.data.local.entity.ProviderEntity
+import com.streamvault.data.local.entity.SeriesCategoryHydrationEntity
import com.streamvault.data.local.entity.XtreamIndexJobEntity
import com.streamvault.data.local.entity.XtreamLiveOnboardingStateEntity
import com.streamvault.data.parser.M3uParser
@@ -39,6 +41,7 @@ import com.streamvault.domain.repository.EpgRepository
import com.streamvault.domain.repository.EpgSourceRepository
import com.streamvault.data.remote.stalker.StalkerCategoryRecord
import com.streamvault.data.remote.stalker.StalkerItemRecord
+import com.streamvault.data.remote.stalker.StalkerPagedItems
import com.streamvault.data.remote.stalker.StalkerProgramRecord
import com.streamvault.data.remote.stalker.StalkerProviderProfile
import com.streamvault.data.remote.stalker.StalkerSession
@@ -54,6 +57,7 @@ import kotlinx.coroutines.flow.first
import kotlinx.coroutines.flow.flowOf
import kotlinx.coroutines.test.advanceUntilIdle
import kotlinx.coroutines.test.runTest
+import kotlinx.coroutines.withTimeout
import kotlinx.serialization.json.Json
import okhttp3.MediaType.Companion.toMediaType
import okhttp3.OkHttpClient
@@ -63,6 +67,7 @@ import okhttp3.ResponseBody.Companion.toResponseBody
import org.junit.Before
import org.junit.Rule
import org.junit.Test
+import org.junit.Assert.fail
import org.junit.rules.TemporaryFolder
import org.mockito.kotlin.any
import org.mockito.kotlin.anyOrNull
@@ -75,6 +80,7 @@ import org.mockito.kotlin.eq
import org.mockito.kotlin.mock
import org.mockito.kotlin.reset
import org.mockito.kotlin.verify
+import java.util.Locale
import java.util.zip.GZIPOutputStream
/**
@@ -126,6 +132,11 @@ class SyncManagerTest {
serverUrl = "https://test.example.com:8080",
username = "demo", password = "demo"
)
+
+ fun stalkerSyntheticCategoryId(providerId: Long, type: ContentType, seed: String): Long {
+ val normalized = "$providerId/${type.name}/${seed.trim().lowercase(Locale.ROOT)}"
+ return (normalized.hashCode().toLong() and 0x7fff_ffffL).coerceAtLeast(1L)
+ }
}
private class FakeSyncMetadataRepository : com.streamvault.domain.repository.SyncMetadataRepository {
@@ -289,6 +300,7 @@ class SyncManagerTest {
org.mockito.kotlin.whenever(catalogSyncDao.countMovieStages(any(), any())).thenReturn(0)
org.mockito.kotlin.whenever(catalogSyncDao.countSeriesStages(any(), any())).thenReturn(0)
org.mockito.kotlin.whenever(catalogSyncDao.getChannelStageCategorySummaries(any(), any())).thenReturn(emptyList())
+ org.mockito.kotlin.whenever(xtreamContentIndexDao.pruneStaleLocalContentRows(any(), any())).thenReturn(0)
// Default stubs for the streamed Stalker API methods. The tests in this file
// generally don't stress live-stream streaming directly; without these defaults
// the unstubbed mock returns null which crashes the `when (val streamResult)`
@@ -1216,6 +1228,628 @@ class SyncManagerTest {
assertThat(finalMovieJob.nextCategoryIndex).isEqualTo(1)
}
+ @Test
+ fun `processQueuedStalkerIndexJobs advances multiple pages for one movie category in one run`() = runTest {
+ val providerEntity = sampleProvider(ProviderType.STALKER_PORTAL).copy(
+ serverUrl = "http://example.com",
+ username = "",
+ password = "",
+ stalkerMacAddress = "00:11:22:33:44:55",
+ epgSyncMode = ProviderEpgSyncMode.SKIP,
+ maxConnections = 4
+ )
+ val categoryId = stalkerSyntheticCategoryId(providerEntity.id, ContentType.MOVIE, "5")
+ val manager = buildManager(providerType = ProviderType.STALKER_PORTAL, providerEntity = providerEntity)
+
+ org.mockito.kotlin.whenever(stalkerApiService.authenticate(any())).thenReturn(
+ Result.success(
+ StalkerSession(
+ loadUrl = "http://example.com/stalker_portal/server/load.php",
+ portalReferer = "http://example.com/stalker_portal/c/",
+ token = "token"
+ ) to StalkerProviderProfile(accountName = "Stalker")
+ )
+ )
+ org.mockito.kotlin.whenever(stalkerApiService.getVodCategories(any(), any())).thenReturn(
+ Result.success(
+ listOf(
+ StalkerCategoryRecord(id = "5", name = "Action")
+ )
+ )
+ )
+ org.mockito.kotlin.whenever(categoryDao.getByProviderAndTypeSync(1L, ContentType.MOVIE.name)).thenReturn(
+ listOf(
+ CategoryEntity(
+ categoryId = categoryId,
+ name = "Action",
+ type = ContentType.MOVIE,
+ providerId = 1L
+ )
+ )
+ )
+ org.mockito.kotlin.whenever(xtreamIndexJobDao.get(1L, ContentType.MOVIE.name)).thenReturn(
+ XtreamIndexJobEntity(
+ providerId = 1L,
+ section = ContentType.MOVIE.name,
+ state = "QUEUED",
+ totalCategories = 1
+ )
+ )
+ val hydrationByCategory = mutableMapOf<Long, MovieCategoryHydrationEntity>()
+ org.mockito.kotlin.whenever(movieCategoryHydrationDao.get(eq(1L), any())).thenAnswer { invocation ->
+ hydrationByCategory[invocation.getArgument<Long>(1)]
+ }
+ org.mockito.kotlin.whenever(movieCategoryHydrationDao.upsert(any())).thenAnswer { invocation ->
+ val entity = invocation.getArgument<MovieCategoryHydrationEntity>(0)
+ hydrationByCategory[entity.categoryId] = entity
+ Unit
+ }
+ org.mockito.kotlin.whenever(movieDao.getByStreamIds(eq(1L), any())).thenReturn(emptyList())
+ org.mockito.kotlin.whenever(stalkerApiService.getVodStreamsPage(any(), any(), eq("5"), eq(1))).thenReturn(
+ Result.success(
+ StalkerPagedItems(
+ items = listOf(
+ StalkerItemRecord(
+ id = "100",
+ name = "Movie 100",
+ categoryId = "5",
+ cmd = "ffmpeg http://example.com/100.ts"
+ )
+ ),
+ page = 1,
+ totalPages = 2,
+ pageSize = 14
+ )
+ )
+ )
+ org.mockito.kotlin.whenever(stalkerApiService.getVodStreamsPage(any(), any(), eq("5"), eq(2))).thenReturn(
+ Result.success(
+ StalkerPagedItems(
+ items = listOf(
+ StalkerItemRecord(
+ id = "101",
+ name = "Movie 101",
+ categoryId = "5",
+ cmd = "ffmpeg http://example.com/101.ts"
+ )
+ ),
+ page = 2,
+ totalPages = 2,
+ pageSize = 14
+ )
+ )
+ )
+
+ val result = manager.processQueuedStalkerIndexJobs(
+ providerId = 1L,
+ section = ContentType.MOVIE,
+ maxCategoriesPerSection = 1
+ )
+
+ if (result is Result.Error) fail(result.exception?.stackTraceToString() ?: result.message)
+ assertThat(result.isSuccess).isTrue()
+ verify(stalkerApiService).getVodStreamsPage(any(), any(), eq("5"), eq(1))
+ verify(stalkerApiService).getVodStreamsPage(any(), any(), eq("5"), eq(2))
+ verify(movieDao, atLeast(2)).insertAll(any())
+ verify(xtreamContentIndexDao, atLeast(2)).upsertAll(any())
+ val jobs = argumentCaptor<XtreamIndexJobEntity>()
+ verify(xtreamIndexJobDao, atLeastOnce()).upsert(jobs.capture())
+ val finalMovieJob = jobs.allValues.last { it.section == ContentType.MOVIE.name }
+ assertThat(finalMovieJob.state).isEqualTo("SUCCESS")
+ }
+
+ @Test
+ fun `processQueuedStalkerIndexJobs can advance beyond legacy six page cap in one run`() = runTest {
+ val providerEntity = sampleProvider(ProviderType.STALKER_PORTAL).copy(
+ serverUrl = "http://example.com",
+ username = "",
+ password = "",
+ stalkerMacAddress = "00:11:22:33:44:55"
+ )
+ val categoryId = stalkerSyntheticCategoryId(providerEntity.id, ContentType.MOVIE, "5")
+ val manager = buildManager(providerType = ProviderType.STALKER_PORTAL, providerEntity = providerEntity)
+
+ org.mockito.kotlin.whenever(stalkerApiService.authenticate(any())).thenReturn(
+ Result.success(
+ StalkerSession(
+ loadUrl = "http://example.com/stalker_portal/server/load.php",
+ portalReferer = "http://example.com/stalker_portal/c/",
+ token = "token"
+ ) to StalkerProviderProfile(accountName = "Stalker")
+ )
+ )
+ org.mockito.kotlin.whenever(stalkerApiService.getVodCategories(any(), any())).thenReturn(
+ Result.success(
+ listOf(
+ StalkerCategoryRecord(id = "5", name = "Action")
+ )
+ )
+ )
+ org.mockito.kotlin.whenever(categoryDao.getByProviderAndTypeSync(1L, ContentType.MOVIE.name)).thenReturn(
+ listOf(
+ CategoryEntity(
+ categoryId = categoryId,
+ name = "Action",
+ type = ContentType.MOVIE,
+ providerId = 1L
+ )
+ )
+ )
+ org.mockito.kotlin.whenever(xtreamIndexJobDao.get(1L, ContentType.MOVIE.name)).thenReturn(
+ XtreamIndexJobEntity(
+ providerId = 1L,
+ section = ContentType.MOVIE.name,
+ state = "QUEUED",
+ totalCategories = 1
+ )
+ )
+ val hydrationByCategory = mutableMapOf<Long, MovieCategoryHydrationEntity>()
+ org.mockito.kotlin.whenever(movieCategoryHydrationDao.get(eq(1L), any())).thenAnswer { invocation ->
+ hydrationByCategory[invocation.getArgument<Long>(1)]
+ }
+ org.mockito.kotlin.whenever(movieCategoryHydrationDao.upsert(any())).thenAnswer { invocation ->
+ val entity = invocation.getArgument<MovieCategoryHydrationEntity>(0)
+ hydrationByCategory[entity.categoryId] = entity
+ Unit
+ }
+ org.mockito.kotlin.whenever(movieDao.getByStreamIds(eq(1L), any())).thenReturn(emptyList())
+ (1..7).forEach { page ->
+ org.mockito.kotlin.whenever(stalkerApiService.getVodStreamsPage(any(), any(), eq("5"), eq(page))).thenReturn(
+ Result.success(
+ StalkerPagedItems(
+ items = listOf(
+ StalkerItemRecord(
+ id = "10$page",
+ name = "Movie $page",
+ categoryId = "5",
+ cmd = "ffmpeg http://example.com/$page.ts"
+ )
+ ),
+ page = page,
+ totalPages = 7,
+ pageSize = 14
+ )
+ )
+ )
+ }
+
+ val result = manager.processQueuedStalkerIndexJobs(
+ providerId = 1L,
+ section = ContentType.MOVIE,
+ maxCategoriesPerSection = 1
+ )
+
+ if (result is Result.Error) fail(result.exception?.stackTraceToString() ?: result.message)
+ assertThat(result.isSuccess).isTrue()
+ verify(stalkerApiService).getVodStreamsPage(any(), any(), eq("5"), eq(7))
+ val jobs = argumentCaptor<XtreamIndexJobEntity>()
+ verify(xtreamIndexJobDao, atLeastOnce()).upsert(jobs.capture())
+ val finalMovieJob = jobs.allValues.last { it.section == ContentType.MOVIE.name }
+ assertThat(finalMovieJob.state).isEqualTo("SUCCESS")
+ }
+
+ @Test
+ fun `processQueuedStalkerIndexJobs allows movie and series sections to overlap for same provider`() = runTest {
+ val providerEntity = sampleProvider(ProviderType.STALKER_PORTAL).copy(
+ serverUrl = "http://example.com",
+ username = "",
+ password = "",
+ stalkerMacAddress = "00:11:22:33:44:55",
+ epgSyncMode = ProviderEpgSyncMode.SKIP,
+ maxConnections = 2
+ )
+ val movieCategoryId = stalkerSyntheticCategoryId(providerEntity.id, ContentType.MOVIE, "5")
+ val seriesCategoryId = stalkerSyntheticCategoryId(providerEntity.id, ContentType.SERIES, "9")
+ val manager = buildManager(providerType = ProviderType.STALKER_PORTAL, providerEntity = providerEntity)
+
+ org.mockito.kotlin.whenever(stalkerApiService.authenticate(any())).thenReturn(
+ Result.success(
+ StalkerSession(
+ loadUrl = "http://example.com/stalker_portal/server/load.php",
+ portalReferer = "http://example.com/stalker_portal/c/",
+ token = "token"
+ ) to StalkerProviderProfile(accountName = "Stalker")
+ )
+ )
+ org.mockito.kotlin.whenever(stalkerApiService.getVodCategories(any(), any())).thenReturn(
+ Result.success(listOf(StalkerCategoryRecord(id = "5", name = "Action")))
+ )
+ org.mockito.kotlin.whenever(stalkerApiService.getSeriesCategories(any(), any())).thenReturn(
+ Result.success(listOf(StalkerCategoryRecord(id = "9", name = "Drama")))
+ )
+ org.mockito.kotlin.whenever(categoryDao.getByProviderAndTypeSync(1L, ContentType.MOVIE.name)).thenReturn(
+ listOf(
+ CategoryEntity(
+ categoryId = movieCategoryId,
+ name = "Action",
+ type = ContentType.MOVIE,
+ providerId = 1L
+ )
+ )
+ )
+ org.mockito.kotlin.whenever(categoryDao.getByProviderAndTypeSync(1L, ContentType.SERIES.name)).thenReturn(
+ listOf(
+ CategoryEntity(
+ categoryId = seriesCategoryId,
+ name = "Drama",
+ type = ContentType.SERIES,
+ providerId = 1L
+ )
+ )
+ )
+ org.mockito.kotlin.whenever(xtreamIndexJobDao.get(1L, ContentType.MOVIE.name)).thenReturn(
+ XtreamIndexJobEntity(
+ providerId = 1L,
+ section = ContentType.MOVIE.name,
+ state = "QUEUED",
+ totalCategories = 1
+ )
+ )
+ org.mockito.kotlin.whenever(xtreamIndexJobDao.get(1L, ContentType.SERIES.name)).thenReturn(
+ XtreamIndexJobEntity(
+ providerId = 1L,
+ section = ContentType.SERIES.name,
+ state = "QUEUED",
+ totalCategories = 1
+ )
+ )
+ val movieHydrationByCategory = mutableMapOf<Long, MovieCategoryHydrationEntity>()
+ org.mockito.kotlin.whenever(movieCategoryHydrationDao.get(eq(1L), any())).thenAnswer { invocation ->
+ movieHydrationByCategory[invocation.getArgument<Long>(1)]
+ }
+ org.mockito.kotlin.whenever(movieCategoryHydrationDao.upsert(any())).thenAnswer { invocation ->
+ val entity = invocation.getArgument<MovieCategoryHydrationEntity>(0)
+ movieHydrationByCategory[entity.categoryId] = entity
+ Unit
+ }
+ val seriesHydrationByCategory = mutableMapOf<Long, SeriesCategoryHydrationEntity>()
+ org.mockito.kotlin.whenever(seriesCategoryHydrationDao.get(eq(1L), any())).thenAnswer { invocation ->
+ seriesHydrationByCategory[invocation.getArgument<Long>(1)]
+ }
+ org.mockito.kotlin.whenever(seriesCategoryHydrationDao.upsert(any())).thenAnswer { invocation ->
+ val entity = invocation.getArgument<SeriesCategoryHydrationEntity>(0)
+ seriesHydrationByCategory[entity.categoryId] = entity
+ Unit
+ }
+ org.mockito.kotlin.whenever(movieDao.getByStreamIds(eq(1L), any())).thenReturn(emptyList())
+ org.mockito.kotlin.whenever(seriesDao.getBySeriesIds(eq(1L), any())).thenReturn(emptyList())
+
+ val movieFetchStarted = CompletableDeferred<Unit>()
+ val releaseMovieFetch = CompletableDeferred<Unit>()
+ val seriesFetchStarted = CompletableDeferred<Unit>()
+ org.mockito.kotlin.whenever(stalkerApiService.getVodStreamsPage(any(), any(), eq("5"), eq(1))).doSuspendableAnswer {
+ movieFetchStarted.complete(Unit)
+ releaseMovieFetch.await()
+ Result.success(
+ StalkerPagedItems(
+ items = listOf(
+ StalkerItemRecord(
+ id = "100",
+ name = "Movie 100",
+ categoryId = "5",
+ cmd = "ffmpeg http://example.com/100.ts"
+ )
+ ),
+ page = 1,
+ totalPages = 1,
+ pageSize = 14
+ )
+ )
+ }
+ org.mockito.kotlin.whenever(stalkerApiService.getSeriesPage(any(), any(), eq("9"), eq(1))).thenAnswer {
+ seriesFetchStarted.complete(Unit)
+ Result.success(
+ StalkerPagedItems(
+ items = listOf(
+ StalkerItemRecord(
+ id = "200",
+ name = "Series 200",
+ categoryId = "9",
+ cmd = "ffmpeg http://example.com/200.ts",
+ isSeries = true
+ )
+ ),
+ page = 1,
+ totalPages = 1,
+ pageSize = 14
+ )
+ )
+ }
+
+ val movieJob = async {
+ manager.processQueuedStalkerIndexJobs(
+ providerId = 1L,
+ section = ContentType.MOVIE,
+ maxCategoriesPerSection = 1
+ )
+ }
+ movieFetchStarted.await()
+
+ val seriesJob = async {
+ manager.processQueuedStalkerIndexJobs(
+ providerId = 1L,
+ section = ContentType.SERIES,
+ maxCategoriesPerSection = 1
+ )
+ }
+
+ withTimeout(1_000) {
+ seriesFetchStarted.await()
+ }
+ releaseMovieFetch.complete(Unit)
+
+ val seriesResult = seriesJob.await()
+ val movieResult = movieJob.await()
+
+ if (seriesResult is Result.Error) fail(seriesResult.exception?.stackTraceToString() ?: seriesResult.message)
+ if (movieResult is Result.Error) fail(movieResult.exception?.stackTraceToString() ?: movieResult.message)
+ assertThat(seriesResult.isSuccess).isTrue()
+ assertThat(movieResult.isSuccess).isTrue()
+ verify(stalkerApiService).getVodStreamsPage(any(), any(), eq("5"), eq(1))
+ verify(stalkerApiService).getSeriesPage(any(), any(), eq("9"), eq(1))
+ }
+
+ @Test
+ fun `processQueuedStalkerIndexJobs serializes movie and series sections for one connection provider`() = runTest {
+ val providerEntity = sampleProvider(ProviderType.STALKER_PORTAL).copy(
+ serverUrl = "http://example.com",
+ username = "",
+ password = "",
+ stalkerMacAddress = "00:11:22:33:44:55",
+ epgSyncMode = ProviderEpgSyncMode.SKIP,
+ maxConnections = 1
+ )
+ val movieCategoryId = stalkerSyntheticCategoryId(providerEntity.id, ContentType.MOVIE, "5")
+ val seriesCategoryId = stalkerSyntheticCategoryId(providerEntity.id, ContentType.SERIES, "9")
+ val manager = buildManager(providerType = ProviderType.STALKER_PORTAL, providerEntity = providerEntity)
+
+ org.mockito.kotlin.whenever(stalkerApiService.authenticate(any())).thenReturn(
+ Result.success(
+ StalkerSession(
+ loadUrl = "http://example.com/stalker_portal/server/load.php",
+ portalReferer = "http://example.com/stalker_portal/c/",
+ token = "token"
+ ) to StalkerProviderProfile(accountName = "Stalker")
+ )
+ )
+ org.mockito.kotlin.whenever(stalkerApiService.getVodCategories(any(), any())).thenReturn(
+ Result.success(listOf(StalkerCategoryRecord(id = "5", name = "Action")))
+ )
+ org.mockito.kotlin.whenever(stalkerApiService.getSeriesCategories(any(), any())).thenReturn(
+ Result.success(listOf(StalkerCategoryRecord(id = "9", name = "Drama")))
+ )
+ org.mockito.kotlin.whenever(categoryDao.getByProviderAndTypeSync(1L, ContentType.MOVIE.name)).thenReturn(
+ listOf(
+ CategoryEntity(
+ categoryId = movieCategoryId,
+ name = "Action",
+ type = ContentType.MOVIE,
+ providerId = 1L
+ )
+ )
+ )
+ org.mockito.kotlin.whenever(categoryDao.getByProviderAndTypeSync(1L, ContentType.SERIES.name)).thenReturn(
+ listOf(
+ CategoryEntity(
+ categoryId = seriesCategoryId,
+ name = "Drama",
+ type = ContentType.SERIES,
+ providerId = 1L
+ )
+ )
+ )
+ org.mockito.kotlin.whenever(xtreamIndexJobDao.get(1L, ContentType.MOVIE.name)).thenReturn(
+ XtreamIndexJobEntity(
+ providerId = 1L,
+ section = ContentType.MOVIE.name,
+ state = "QUEUED",
+ totalCategories = 1
+ )
+ )
+ org.mockito.kotlin.whenever(xtreamIndexJobDao.get(1L, ContentType.SERIES.name)).thenReturn(
+ XtreamIndexJobEntity(
+ providerId = 1L,
+ section = ContentType.SERIES.name,
+ state = "QUEUED",
+ totalCategories = 1
+ )
+ )
+ val movieHydrationByCategory = mutableMapOf<Long, MovieCategoryHydrationEntity>()
+ org.mockito.kotlin.whenever(movieCategoryHydrationDao.get(eq(1L), any())).thenAnswer { invocation ->
+ movieHydrationByCategory[invocation.getArgument<Long>(1)]
+ }
+ org.mockito.kotlin.whenever(movieCategoryHydrationDao.upsert(any())).thenAnswer { invocation ->
+ val entity = invocation.getArgument<MovieCategoryHydrationEntity>(0)
+ movieHydrationByCategory[entity.categoryId] = entity
+ Unit
+ }
+ val seriesHydrationByCategory = mutableMapOf<Long, SeriesCategoryHydrationEntity>()
+ org.mockito.kotlin.whenever(seriesCategoryHydrationDao.get(eq(1L), any())).thenAnswer { invocation ->
+ seriesHydrationByCategory[invocation.getArgument<Long>(1)]
+ }
+ org.mockito.kotlin.whenever(seriesCategoryHydrationDao.upsert(any())).thenAnswer { invocation ->
+ val entity = invocation.getArgument<SeriesCategoryHydrationEntity>(0)
+ seriesHydrationByCategory[entity.categoryId] = entity
+ Unit
+ }
+ org.mockito.kotlin.whenever(movieDao.getByStreamIds(eq(1L), any())).thenReturn(emptyList())
+ org.mockito.kotlin.whenever(seriesDao.getBySeriesIds(eq(1L), any())).thenReturn(emptyList())
+
+ val movieFetchStarted = CompletableDeferred<Unit>()
+ val releaseMovieFetch = CompletableDeferred<Unit>()
+ val seriesFetchStarted = CompletableDeferred<Unit>()
+ org.mockito.kotlin.whenever(stalkerApiService.getVodStreamsPage(any(), any(), eq("5"), eq(1))).doSuspendableAnswer {
+ movieFetchStarted.complete(Unit)
+ releaseMovieFetch.await()
+ Result.success(
+ StalkerPagedItems(
+ items = listOf(
+ StalkerItemRecord(
+ id = "100",
+ name = "Movie 100",
+ categoryId = "5",
+ cmd = "ffmpeg http://example.com/100.ts"
+ )
+ ),
+ page = 1,
+ totalPages = 1,
+ pageSize = 14
+ )
+ )
+ }
+ org.mockito.kotlin.whenever(stalkerApiService.getSeriesPage(any(), any(), eq("9"), eq(1))).thenAnswer {
+ seriesFetchStarted.complete(Unit)
+ Result.success(
+ StalkerPagedItems(
+ items = listOf(
+ StalkerItemRecord(
+ id = "200",
+ name = "Series 200",
+ categoryId = "9",
+ cmd = "ffmpeg http://example.com/200.ts",
+ isSeries = true
+ )
+ ),
+ page = 1,
+ totalPages = 1,
+ pageSize = 14
+ )
+ )
+ }
+
+ val movieJob = async {
+ manager.processQueuedStalkerIndexJobs(
+ providerId = 1L,
+ section = ContentType.MOVIE,
+ maxCategoriesPerSection = 1
+ )
+ }
+ movieFetchStarted.await()
+
+ val seriesJob = async {
+ manager.processQueuedStalkerIndexJobs(
+ providerId = 1L,
+ section = ContentType.SERIES,
+ maxCategoriesPerSection = 1
+ )
+ }
+
+ advanceUntilIdle()
+ assertThat(seriesFetchStarted.isCompleted).isFalse()
+
+ releaseMovieFetch.complete(Unit)
+ withTimeout(1_000) {
+ seriesFetchStarted.await()
+ }
+
+ val movieResult = movieJob.await()
+ val seriesResult = seriesJob.await()
+
+ if (movieResult is Result.Error) fail(movieResult.exception?.stackTraceToString() ?: movieResult.message)
+ if (seriesResult is Result.Error) fail(seriesResult.exception?.stackTraceToString() ?: seriesResult.message)
+ assertThat(movieResult.isSuccess).isTrue()
+ assertThat(seriesResult.isSuccess).isTrue()
+ verify(stalkerApiService).getVodStreamsPage(any(), any(), eq("5"), eq(1))
+ verify(stalkerApiService).getSeriesPage(any(), any(), eq("9"), eq(1))
+ }
+
+ @Test
+ fun `processQueuedStalkerIndexJobs falls back to non paged movie fetch when first page is empty`() = runTest {
+ val providerEntity = sampleProvider(ProviderType.STALKER_PORTAL).copy(
+ serverUrl = "http://example.com",
+ username = "",
+ password = "",
+ stalkerMacAddress = "00:11:22:33:44:55",
+ epgSyncMode = ProviderEpgSyncMode.SKIP,
+ maxConnections = 1
+ )
+ val movieCategoryId = stalkerSyntheticCategoryId(providerEntity.id, ContentType.MOVIE, "5")
+ val manager = buildManager(providerType = ProviderType.STALKER_PORTAL, providerEntity = providerEntity)
+
+ org.mockito.kotlin.whenever(stalkerApiService.authenticate(any())).thenReturn(
+ Result.success(
+ StalkerSession(
+ loadUrl = "http://example.com/stalker_portal/server/load.php",
+ portalReferer = "http://example.com/stalker_portal/c/",
+ token = "token"
+ ) to StalkerProviderProfile(accountName = "Stalker")
+ )
+ )
+ org.mockito.kotlin.whenever(stalkerApiService.getVodCategories(any(), any())).thenReturn(
+ Result.success(listOf(StalkerCategoryRecord(id = "5", name = "Action")))
+ )
+ org.mockito.kotlin.whenever(categoryDao.getByProviderAndTypeSync(1L, ContentType.MOVIE.name)).thenReturn(
+ listOf(
+ CategoryEntity(
+ categoryId = movieCategoryId,
+ name = "Action",
+ type = ContentType.MOVIE,
+ providerId = 1L
+ )
+ )
+ )
+ org.mockito.kotlin.whenever(xtreamIndexJobDao.get(1L, ContentType.MOVIE.name)).thenReturn(
+ XtreamIndexJobEntity(
+ providerId = 1L,
+ section = ContentType.MOVIE.name,
+ state = "QUEUED",
+ totalCategories = 1
+ )
+ )
+ val movieHydrationByCategory = mutableMapOf<Long, MovieCategoryHydrationEntity>()
+ org.mockito.kotlin.whenever(movieCategoryHydrationDao.get(eq(1L), any())).thenAnswer { invocation ->
+ movieHydrationByCategory[invocation.getArgument<Long>(1)]
+ }
+ org.mockito.kotlin.whenever(movieCategoryHydrationDao.upsert(any())).thenAnswer { invocation ->
+ val entity = invocation.getArgument<MovieCategoryHydrationEntity>(0)
+ movieHydrationByCategory[entity.categoryId] = entity
+ Unit
+ }
+ org.mockito.kotlin.whenever(movieDao.getByStreamIds(eq(1L), any())).thenReturn(emptyList())
+ org.mockito.kotlin.whenever(movieDao.getCount(1L)).thenReturn(flowOf(0), flowOf(1), flowOf(1))
+ org.mockito.kotlin.whenever(movieDao.getCountByCategory(1L, movieCategoryId)).thenReturn(flowOf(1))
+ org.mockito.kotlin.whenever(xtreamContentIndexDao.pruneStaleLocalContentRows(1L, ContentType.MOVIE.name)).thenReturn(0)
+
+ org.mockito.kotlin.whenever(stalkerApiService.getVodStreamsPage(any(), any(), eq("5"), eq(1))).thenReturn(
+ Result.success(
+ StalkerPagedItems(
+ items = emptyList(),
+ page = 1,
+ totalPages = 1,
+ pageSize = 14
+ )
+ )
+ )
+ org.mockito.kotlin.whenever(stalkerApiService.getVodStreams(any(), any(), eq("5"))).thenReturn(
+ Result.success(
+ listOf(
+ StalkerItemRecord(
+ id = "100",
+ name = "Movie 100",
+ categoryId = "5",
+ cmd = "ffmpeg http://example.com/100.ts"
+ )
+ )
+ )
+ )
+
+ val result = manager.processQueuedStalkerIndexJobs(
+ providerId = 1L,
+ section = ContentType.MOVIE,
+ maxCategoriesPerSection = 1
+ )
+
+ if (result is Result.Error) fail(result.exception?.stackTraceToString() ?: result.message)
+ assertThat(result.isSuccess).isTrue()
+ verify(stalkerApiService).getVodStreamsPage(any(), any(), eq("5"), eq(1))
+ verify(stalkerApiService).getVodStreams(any(), any(), eq("5"))
+ verify(movieDao).insertAll(check { movies ->
+ assertThat(movies).hasSize(1)
+ assertThat(movies.first().providerId).isEqualTo(1L)
+ assertThat(movies.first().streamId).isEqualTo(100L)
+ })
+ }
+
@Test
fun `chunkedLookupById splits large id lists below sqlite variable limit`() = runTest {
val requestedChunks = mutableListOf<List<Long>>()
@@ -1351,6 +1985,74 @@ class SyncManagerTest {
verify(stalkerApiService).streamLiveStreams(any(), any(), any())
}
+ @Test
+ fun sync_stalker_recovers_when_bulk_live_streaming_fails_using_category_fetches() = runTest {
+ val providerEntity = sampleProvider(ProviderType.STALKER_PORTAL).copy(
+ serverUrl = "http://example.com",
+ username = "",
+ password = "",
+ stalkerMacAddress = "00:11:22:33:44:55",
+ epgSyncMode = ProviderEpgSyncMode.SKIP
+ )
+ val manager = buildManager(providerType = ProviderType.STALKER_PORTAL, providerEntity = providerEntity)
+
+ org.mockito.kotlin.whenever(stalkerApiService.authenticate(any())).thenReturn(
+ Result.success(
+ StalkerSession(
+ loadUrl = "http://example.com/stalker_portal/server/load.php",
+ portalReferer = "http://example.com/stalker_portal/c/",
+ token = "token"
+ ) to StalkerProviderProfile(accountName = "Stalker")
+ )
+ )
+ org.mockito.kotlin.whenever(stalkerApiService.getLiveCategories(any(), any())).thenReturn(
+ Result.success(
+ listOf(
+ StalkerCategoryRecord(id = "10", name = "News"),
+ StalkerCategoryRecord(id = "20", name = "Sports")
+ )
+ )
+ )
+ stalkerApiService.stubStreamLiveStreamsError("Portal busy")
+ org.mockito.kotlin.whenever(stalkerApiService.getLiveStreams(any(), any(), eq("10"))).thenReturn(
+ Result.success(
+ listOf(
+ StalkerItemRecord(
+ id = "100",
+ name = "News",
+ categoryId = "10",
+ cmd = "ffmpeg http://example.com/news.ts"
+ )
+ )
+ )
+ )
+ org.mockito.kotlin.whenever(stalkerApiService.getLiveStreams(any(), any(), eq("20"))).thenReturn(
+ Result.success(
+ listOf(
+ StalkerItemRecord(
+ id = "200",
+ name = "Sports",
+ categoryId = "20",
+ cmd = "ffmpeg http://example.com/sports.ts"
+ )
+ )
+ )
+ )
+ org.mockito.kotlin.whenever(stalkerApiService.getVodCategories(any(), any())).thenReturn(Result.success(emptyList()))
+ org.mockito.kotlin.whenever(stalkerApiService.getSeriesCategories(any(), any())).thenReturn(Result.success(emptyList()))
+
+ val result = manager.sync(providerId = 1L, force = false)
+
+ assertThat(result).isInstanceOf(Result.Success::class.java)
+ val categoryStagesCaptor = argumentCaptor<List<com.streamvault.data.local.entity.CategoryImportStageEntity>>()
+ verify(catalogSyncDao, atLeastOnce()).insertCategoryStages(categoryStagesCaptor.capture())
+ assertThat(categoryStagesCaptor.allValues.flatten().any { it.type == com.streamvault.domain.model.ContentType.LIVE && it.name == "News" }).isTrue()
+ assertThat(categoryStagesCaptor.allValues.flatten().any { it.type == com.streamvault.domain.model.ContentType.LIVE && it.name == "Sports" }).isTrue()
+ verify(stalkerApiService).streamLiveStreams(any(), any(), any())
+ verify(stalkerApiService).getLiveStreams(any(), any(), eq("10"))
+ verify(stalkerApiService).getLiveStreams(any(), any(), eq("20"))
+ }
+
@Test
fun sync_stalker_surfaces_profile_hint_when_catalog_access_is_missing() = runTest {
val providerEntity = sampleProvider(ProviderType.STALKER_PORTAL).copy(
diff --git a/docs/deep-research-report.md b/docs/deep-research-report.md
index 515c4c8a..92eec6be 100644
--- a/docs/deep-research-report.md
+++ b/docs/deep-research-report.md
@@ -1,90 +1,248 @@
-# Practical MAG Compatibility Gaps in Stalker Playback
+# Executive Summary
-## Core diagnosis
+Stalker-based IPTV clients perform an initial multi-step sync when new portal credentials are entered. First, the client “handshakes” with the portal (type=`stb&action=handshake`), obtaining a bearer token and device profile【84†L89-L97】【84†L109-L114】. Using this token, the client fetches channel categories (e.g. `type=itv&action=get_genres`) and then paged channel lists, as well as VOD (video-on-demand) categories (e.g. `type=vod&action=get_categories`) and their items (via `type=vod&action=get_ordered_list`)【90†L3333-L3341】【96†L1969-L1978】. These API calls typically return JSON objects with keys like `"total_items"`, `"max_page_items"`, and a `"data"` array of item records. For example, a `get_ordered_list` response contains a top-level `"js"` object with `total_items` and `data` arrays【96†L1969-L1978】. Payload sizes vary by provider; one page may carry tens to hundreds of items (each item often hundreds of bytes), so VOD libraries in the thousands can yield many kilobytes or even megabytes of JSON. In practice, clients employ numerous strategies (partial loading, caching, parallel fetches, compression, etc.) to make the initial load feel fast despite large data. Server-side, portal middlewares use indexing, pagination, gzip compression, CDN-backed assets, and possibly pre-generated manifests to speed responses. Popular clients (including open-source Stalker plugins and custom sync tools) typically page through categories in parallel and cache results. Real-world reports note first-time syncs taking on the order of ~10–60 seconds for very large catalogs, whereas optimized incremental loading and local caching greatly improve perceived responsiveness【85†L328-L336】【75†L193-L200】. Key security points: all calls use an access token (Bearer scheme) bound to the device MAC, and clients must refresh that token or re-handshake if it expires. Rate limits are not well-documented; providers may throttle excessive requests.
-The pattern you described, where handshake, profile loading, live/VOD/series catalogues, and even `create_link` appear to work but playback fails, is exactly what you would expect from a **partial MAG emulation that passes the control plane but diverges on the media plane**. In Ministra/Stalker, catalogue browsing rides mostly on `load.php` JSON calls, while playback crosses into a more sensitive path that can involve device-bound identity checks, stream-source selection, temporary-link generation, NGINX/secure-link validation, Flussonic/Wowza integration, and per-channel filtering. Official Infomir documentation and the archived MAG client code make that split very clear. citeturn22view0turn35view0turn34view0turn37view0turn45view0
+**Sources:** Official Stalker middleware/API discussions and code【84†L89-L97】【90†L3333-L3341】【96†L1969-L1978】, IPTV client implementations and forum reports【85†L328-L336】【75†L193-L200】.
-That is why “catalogue works, playback fails” is not a contradiction. A portal can happily authorise the STB-facing API enough to return genres, channel lists, VOD metadata, and even a provisional `create_link` response, while still rejecting the **actual stream retrieval** because the client fingerprint, the pre-play bootstrap state, or the final URL fetch path does not match what the portal expects from a real MAG-style device. citeturn26view0turn27search10turn32view0turn35view0turn37view0
+## Stalker Portal API & Initial Sync Endpoints
-## What dedicated MAG-style clients actually emulate
+On first sync the client typically executes these steps (see flowchart below):
-A working MAG-style client does more than send a MAC address. In real traces and in archived client code, Stalker control-plane requests typically include the portal referer, MAC-language-timezone cookies, a bearer token after handshake, and the legacy MAG header pair of a QtEmbedded `User-Agent` plus `X-User-Agent` carrying model/link-type information. A representative pattern looks like this:
+```mermaid
+flowchart LR
+ A[Enter Portal Credentials] --> B[STB Handshake<br/>(type=stb&action=handshake)]
+ B -->|returns token| C[Get Profile (type=stb&action=get_profile)]
+ C -->|device settings| D[Get IPTV Categories<br/>(type=itv&action=get_genres)]
+ D --> E[Get IPTV Lists<br/>(type=itv&action=get_ordered_list…p=1,2,…)]
+ C --> F[Get VOD Categories<br/>(type=vod&action=get_categories)]
+ F --> G[Get VOD Movies<br/>(type=vod&action=get_ordered_list…p=1,2,…)]
+ G --> H[Get VOD Series/Seasons (if needed)]
+ E & G --> I[Load EPG (type=itv&action=get_short_epg&...)]
+```
+
+1. **Handshake/Auth (STB):** The client calls `/portal.php?type=stb&action=handshake` (with required HTTP headers including the device MAC, user-agent, etc.)【84†L86-L94】. The portal returns a JSON with an access token: e.g. `{"js":{"token":"C00F7332ED272F00D5FD3E82F567A282"}}`【84†L89-L97】. This token must be included as `Authorization: Bearer <token>` in subsequent calls.
+
+2. **Get STB Profile:** Next, the client calls `/portal.php?type=stb&action=get_profile` using the token【84†L109-L114】. The JSON profile includes device settings and user info (e.g. `parent_password`, `client_type`, etc.) which some clients may cache.
+
+3. **Load Channel Categories (Genres):** With the token, call `/portal.php?type=itv&action=get_genres`【85†L328-L336】. This returns an array of “genres” (live-TV categories), e.g.
+ ```json
+ {"js":[{"id":"*","title":"All","alias":"All",...},{"id":"173","title":"TR|TURKIYE",...}, ...]}
+ ```
+ (Note: the example above from【85†L333-L341】 shows an `"id":"*" All` plus numbered categories.) Each genre has an internal ID used to fetch its channels.
+
+4. **Load Channel Lists:** For each genre or all-channels, the client pages through `/portal.php?type=itv&action=get_ordered_list&genre=<id>&p=1` etc【85†L406-L415】. The response JSON contains `"total_items"`, `"max_page_items"`, and a `"data"` array of channel objects. (The client typically loops `p=1,2,...` until all pages are fetched【85†L416-L424】.) Channel objects include fields like `id`, `name`, `cmd` (stream command), `logo`, and others. For example, one returned channel entry might be:
+ ```json
+ {"id":"3031","name":"NEDERLAND 4K+","number":"462","cmd":"ffmpeg http://localhost/ch/3031_","hd":"0",...}
+ ```
+ (See【85†L421-L428】 for a truncated example.)
+
+5. **Load VOD Categories:** Parallel to channels, the client fetches VOD categories. Stalker portals use `/server/load.php?type=vod&action=get_categories` (or equivalently `portal.php?...`) to obtain VOD category lists【90†L3333-L3341】. Each category has fields like `"id"` and `"title"`. These are analogous to “movie genres.”
+
+6. **Load VOD Lists:** For each VOD category (or all categories), the client calls `/server/load.php?type=vod&action=get_ordered_list&category=<id>&p=1` etc【90†L3333-L3341】【96†L1969-L1978】. The code examples show parameters like `'sortby':'added'`, `'not_ended':'0'`, `'p':page`, etc.【96†L1969-L1978】. The response JSON structure is similar: it contains `"js": {"total_items": <N>, "max_page_items": <M>, "data": [ {...}, {...}, ... ] }`. Each item in `"data"` is a VOD item with fields like `"id"`, `"name"`, `"category_id"`, and usually a `"cmd"` or `"url"` field for streaming, plus a `"screenshot_uri"` or thumbnail URL, `"modified"`, etc. For series, there may be `is_series` flags and separate APIs to list episodes or seasons. The open-source code shows clients iterating pages until all items are retrieved【96†L1969-L1978】.
+
+7. **Get Episode/Series Details (if any):** If the provider supports TV shows/series, the client might next call `type=vod&action=get_seasons` or similar to fetch seasons and episodes (as seen in open-source clients)【70†L3925-L3933】.
+
+8. **Catch-up/EPG:** Finally, clients fetch any electronic program guide (EPG) or TV-archive data via `type=itv&action=get_short_epg` or similar calls (not shown above). These can be large (whole EPG) but are often paged or limited by time window.
+
+**Typical JSON Payloads:** All portal responses are JSON (the examples above show JSON under `"js": {...}`). Exact sizes depend on provider. For instance, one channel list page in Quassi’s example had ~14 items【85†L421-L429】. If a VOD category has 2000 movies and 50 per page, that’s 40 pages; if each item is ~300–500 bytes, total JSON could be ~200 KB per category. Clients usually do *not* fetch all VOD data at once unless caching for offline use.
+
+## Client-Side Loading Strategies
+
+To keep the UI responsive, clients use tactics like:
+
+- **Incremental/Paged Loading:** Don’t fetch all items in one shot. Fetch the first page of each list immediately (e.g. first ITV and first VOD pages), display those, and load further pages in background. The example plugin code above explicitly loops pages 1..max in a background loop【96†L1969-L1978】.
+
+- **Lazy Loading:** Delay loading of large sections until needed. For example, show VOD *categories* immediately (quick summary), but load each category’s contents only when the user browses that category. This avoids long waits for seldom-used categories.
+
+- **Parallel Fetches:** Issue multiple API calls concurrently (e.g. fetch channel list and first page of movies at the same time). Multi-threaded or async fetching can fill multiple list views quickly. The open-source client concurrently fetched all pages using a thread pool【70†L3925-L3933】 to reduce wall-clock time.
+
+- **Metadata-Only Sync:** At first login, clients may only download item metadata (title, ID, thumbnail URL), deferring heavy data (e.g. full description, or actual video manifest) until playback. That way the initial JSON is smaller. For example, some clients fetch only top-level JSON and lazy-load video streams separately.
+
+- **Background Prefetch:** Once initial data is shown, quietly fetch more in the background (e.g. successive pages, EPG, series details) so that by the time a user scrolls or selects, data is ready.
+
+- **Thumbnail-First:** Load small thumbnail images early, while larger cover images or detailed info load later. Many portals provide low-res images (screenshots) versus full poster art; clients may download the smaller ones first.
+
+- **Caching:** Store results locally (e.g. on device cache or database). On subsequent app starts, reuse cached metadata and only fetch deltas via HTTP cache validation (ETag/If-Modified-Since). For example, clients may use the HTTP `Last-Modified` header of portal responses to check for updates, or simply store a timestamp and only fetch changed data. The plugin code even caches data in files and updates every 12 hours【86†L1934-L1943】【96†L2011-L2014】. This dramatically improves perceived load time after the first sync.
+
+- **Compression & Delta Sync:** Ensure requests use gzip (portals typically compress JSON). For updates, use `If-Modified-Since` or ETag headers on each API (if supported) so that unchanged lists yield small “304 Not Modified” responses.
-```text
-Referer: http://<portal>/stalker_portal/c/
-Cookie: mac=<MAC>; stb_lang=<lang>; timezone=<tz>
-Authorization: Bearer <token>
-X-User-Agent: Model: MAG250; Link: WiFi
-User-Agent: Mozilla/5.0 (QtEmbedded; U; Linux; C) ... MAG200 stbapp ver: 2 rev: 250 ...
+Below is a **timeline flow** of an initial sync showing parallel steps:
+
+```mermaid
+sequenceDiagram
+ participant Client
+ participant Portal
+ Client->>Portal: GET /portal.php?type=stb&action=handshake (with MAC)
+ Portal-->>Client: JSON { "js":{"token":"ABC123..."} }
+ Client->>Portal: GET /portal.php?type=stb&action=get_profile (Bearer token)
+ Portal-->>Client: JSON profile {...}
+ par Parallel fetching
+ Client->>Portal: GET type=itv&action=get_genres (page 1)
+ Portal-->>Client: JSON { "js":[{...}, ...] } (genres list)
+ Client->>Portal: GET type=vod&action=get_categories
+ Portal-->>Client: JSON { "js":[{...}, ...] } (VOD categories)
+ end
+ par Load first content pages
+ Client->>Portal: GET type=itv&action=get_ordered_list&genre=<firstID>&p=1
+ Portal-->>Client: JSON { "js":{"total_items":N,"data":[...20 items...]}}
+ Client->>Portal: GET type=vod&action=get_ordered_list&category=<firstCat>&p=1
+ Portal-->>Client: JSON { "js":{"total_items":M,"data":[...20 items...]}}
+ end
+ Note over Client,Portal: Show UI lists (channels, movies) as they arrive
```
-The same older-style header set also appears in third-party implementations that intentionally mimic MAG behaviour. citeturn47view0turn13search1turn41search0
+**Trade-Offs:** Loading everything upfront (“eager load”) gives completeness but may delay the first screen. Lazy or paged loading trades initial completeness for responsiveness: the UI appears quickly with partial data. Many clients strike a balance: load genre/category lists (small) immediately, then fill in items.
+
+**Table – Strategy Comparison:**
+
+| Strategy | Pros | Cons | Impact on First-Load Time |
+|-----------------------|---------------------------------|-----------------------------------|-----------------------------------|
+| **Full upfront** | All data available at once | Very slow initial sync, heavy load| Very long delay (seconds to minutes) for large catalogs |
+| **Paged/Incremental** | Faster UI start, avoids freezing| Requires more complex code (looping) | Moderate: first page quick (few sec), rest gradually |
+| **Lazy (on-demand)** | Only fetch needed data | First access to new section delayed | Very fast startup; slight lag when opening each new category |
+| **Parallel fetch** | Better CPU/network usage | Possible rate-limit or sync complexity | Reduces wall-clock time (fetching multiple lists at once) |
+| **Cache (client)** | Instant reload after first sync | Stale data if not updated | Negligible (loads from local cache) |
+| **Compression/gzip** | Reduces bytes over wire | None (adds CPU for decompress) | Reduces bandwidth impact, speeds transfer if large JSON |
+
+_Source:_ Portal API observations and client implementations【90†L3333-L3341】【96†L1969-L1978】.
+
+## Server-Side Optimizations
+
+Middleware providers and portal servers can aid speed by:
+
+- **Indexing & Pre-aggregation:** The portal’s database is typically indexed on category and alphabetical fields, making queries for each `get_ordered_list` fast. Some systems precompute popular lists or maintain cached JSON.
+
+- **API Pagination:** As seen, the `get_ordered_list` API inherently paginates results (`p=` parameter) to limit response size. Server indexes support fast pagination (using LIMIT/OFFSET or cursor-based paging).
+
+- **Compressed Responses:** Stalker middleware supports gzip/deflate. Clients should set `Accept-Encoding: gzip` and expect compressed JSON. This cuts payload size significantly for large JSON arrays.
+
+- **CDN & Caching:** While metadata is dynamic, static assets (thumbnails, logos) can be served via CDN. Even the portal might use HTTP caching headers on API responses to allow 304s.
-The bigger compatibility gap is the **full `get_profile` fingerprint**, not just the headers. Archived MAG JS code shows `get_profile` sending `stb_type`, `sn`, `ver`, `image_version`, `hw_version`, `device_id`, `device_id2`, `signature`, `client_type=STB`, `video_out`, `metrics`, `hw_version_2`, `timestamp`, `api_signature`, and `prehash`. The same code also shows that newer model families such as `MAG322`, `MAG324`, and `AuraHD4` do **not** use exactly the same `GetUID` pattern as the older MAG250-style path. By contrast, older third-party clients often stop at the legacy subset: `stb_type=MAG250`, `image_version=216`, `hw_version=1.7-BD-00`, blank `device_id*`, blank `signature`, and no newer telemetry fields. That difference is large enough for a hardened portal to notice even if catalogue calls still succeed. citeturn22view0turn21view2turn21view3turn21view4turn42view0
+- **Thumbnails vs Full Manifests:** Portals often provide two sets of URLs: one for small preview images (e.g. “screenshot_uri”) and one for full resolution art. Using smaller images in listings reduces transfer. For streaming, Stalker often provides either direct stream URLs or “vod/get_file” endpoints which may redirect via CDN.
-The bootstrap sequence also matters. In the archived MAG client, handshake is followed by `get_profile`, then `get_localization`, then `user_init`, and later TV-related initialisation pulls favourites, modules, and first-channel player state. Real traffic traces also show `watchdog/get_events` beginning during TV initialisation, not as an optional afterthought. In other words, a dedicated client arrives at playback with more state already established than many custom players reproduce. citeturn22view0turn44view0turn44view2turn47view0
+- **Manifest Generation:** For on-demand video, portals might not generate HLS/DASH on the fly. Instead, streams can be direct file URLs or pre-encoded HLS. If dynamic, media servers ensure quick response by reusing existing manifests or performing on-demand muxing with segment caching.
-A subtle but practical point is that MAG identity is expected to be **internally coherent**. Official docs show model-based access controls via `allowed_stb_types` and `strict_stb_type_check`, while MAG client code and traces show a consistent tuple of model, serial number, version string, image version, hardware version, player version, and device identifiers. If your app claims one model in `X-User-Agent`, another in `stb_type`, and a third-era firmware string in `ver`, you may still browse, but you are very easy to classify as non-native. citeturn37view0turn22view0turn47view0
+- **Content Delivery:** For catch-up (TV archive), servers often use a separate domain or server. Clients request segments in real time; servers should have seek indexes to satisfy range requests quickly.
-## The playback boundary where partial emulation fails
+If payload sizes are unknown from documentation, we note that “portal payload sizes are unspecified per provider” – providers control how much metadata is returned (some include synopsis or actors, others minimal). In practice, clients observe that initial `get_ordered_list` pages often range from a few dozen KB to hundreds of KB of JSON.
-The Stalker playback path is more variable than the browsing path. Open-source Stalker server code shows `createLink()` branching between direct URLs, load-balanced stream choices, and temporary HTTP links, and it explicitly uses request variables such as `disable_ad` and `force_ch_link_check` while assembling the final `cmd`. Older third-party clients also show `create_link` defaults such as `cmd`, `forced_storage`, and `disable_ad`, and some of them add a referer only for `ITV_CREATE_LINK`. That means a client can be “mostly correct” for catalogue calls yet still differ at the one request that actually decides how playback is resolved. citeturn5view0turn33search0turn42view0
+## Real-World Examples
-Infomir’s official temporary-link documentation shows why this matters. In the recommended HTTP temporary-link setup, the player requests a `/ch/<token>` URL on the NGINX front end; NGINX rewrites that to `chk_tmp_tv_link.php`, and then only an internal `/get/<host>/<path>` location is allowed to proxy the real upstream stream. The documented config forwards `X-Real-IP $remote_addr`, and the platform documentation separately insists on enabling Apache `remoteip` support so the backend sees the subscriber’s real source IP correctly. In the secure-link variant, the signed URL can even include `$remote_addr` in the MD5 calculation, which makes the final media fetch explicitly **IP-bound**. citeturn35view0turn34view0turn37view0
+- **Open-Source Clients:** The [stalker.py](https://github.com/Cyogenus/IPTV-MAC-STALKER-PLAYER) library fetches VOD categories via `/stalker_portal/server/load.php?type=vod&action=get_categories`【90†L3333-L3341】 and iterates `get_ordered_list` for each (as shown above). The [plugin.video.stalker](https://github.com/esxbr/plugin.video.stalker) Kodi plugin similarly performs a handshake then loops through pages of `get_ordered_list` calls with parameters like `sortby=added, not_ended=0`【96†L1969-L1978】. These code examples illustrate the typical access patterns.
-That IP binding is one of the most important practical gaps. If your custom player resolves `create_link` on one path but fetches media via a different egress route, a localhost proxy, a server-side relay, a different interface, or a player engine that does not preserve the same outward identity as the control-plane client, the resulting `/ch/...` or secure-link URL can be valid for a MAG client and invalid for your app. That failure mode is especially consistent with “MyTVOnline+ works, custom app fails”, because the dedicated client is more likely to request the portal-provided playback URL directly from the device’s own network stack. citeturn34view0turn35view0turn37view0
+- **Commercial Apps:** Proprietary apps (e.g. TiviMate, Stalker-based STB apps) follow the same API pattern. A technical overview notes that TiviMate “fetches load.php (handshake), then get.php (content lists), then epg.php”【20†L55-L60】. (In effect, `load.php` is the same as `portal.php?type=stb&action=handshake` on MAG portals.) These clients often preload the first page of all lists. Some even allow specifying a shorter “device ID” to speed recognition by the portal.
-The same principle shows up in backend-specific integrations. Infomir’s Flussonic helper checks the temporary token and, when valid, returns headers such as `X-AuthDuration`, `X-Unique`, `X-Max-Sessions`, and `X-UserId` for the media server path. That tells you that some playback authorisation is no longer happening in `load.php` at all; it happens in the media-server integration layer after `create_link`. So a control-plane clone can still fail if it does not enter the exact media path the portal expects. citeturn45view0
+- **Forum Reports:** Users report initial VOD sync times of tens of seconds. For example, on Formuler devices, a first-time VOD load could take ~45 seconds for large libraries【74†L83-L91】【75†L193-L200】. Once data is cached, subsequent access is nearly instantaneous. One forum noted that a new firmware implemented auto-cache-clearing on reboot to avoid stale data issues【75†L193-L200】, indicating the importance of cache management in perceived speed.
-Infomir also added support for several channel links with a **user-agent filter**, and the platform documents the ability to restrict streaming-link access by device model. That is a direct explanation for why browsing can work while playback fails: the channel is visible in the catalogue, but the actual stream source chosen at play time may be filtered by model, user-agent, or configured access rule. citeturn36search1turn23search6turn37view0
+- **GitHub Implementations:** Some repos generate M3U playlists from Stalker portals. For instance, [santhosh101066/stalker-m3u-server](https://github.com/santhosh101066/stalker-m3u-server) proxies Stalker APIs to produce playlists (using `api.php` to fetch content【45†L243-L249】). Others, like [generator scripts](https://github.com/Cyogenus/IPTV-MAC-STALKER-PLAYER), parse portal JSON directly. The fact that so much code exists for parsing the same endpoints underscores these patterns.
-## Why browsing and even `create_link` can succeed before playback still fails
+## Network Traffic Patterns & Performance
-Handshake and catalogue browsing are a relatively low bar because they mainly prove that the portal is willing to talk to the client as an STB-like consumer. Playback is a higher bar because it is where the portal finally has to bind a subscriber, a device identity, a stream URL, a source IP, and a media backend together. Official platform configuration includes anti-cloning options such as `enable_device_id_validation`, model gating via `allowed_stb_types`, stricter model checks, OAuth-to-STB binding options, and stronger fake-device protections added over time. Infomir staff explicitly described `enable_device_id_validation` as a check on STB-information integrity and anti-cloning protection. citeturn27search1turn27search10turn36search1turn37view0
+Analyzing traffic, we see:
-That anti-cloning behaviour is not theoretical. In an official support thread, a maintainer responded to a `device_id2 mismatch` authorisation failure by advising the operator to clear `device_id` and `device_id2` for that MAC, and when the operator disabled device-ID validation to make the box work, the maintainer warned against it because it would let emulators use the portal. In parallel, experienced users of STBEmu and Formuler/MyTVOnline-class devices report that some subscriptions only work after copying **MAC + serial + device_id + device_id2 + signature + hw_version** into the emulator. That is exactly the sort of gap that lets browsing work on a custom client but stops playback at the moment the portal needs stronger confidence that the caller is the same device it believes it provisioned. citeturn26view0turn32view0
+- **Handshake (~100–300 bytes):** The `handshake` request is small (no query data except type/action), response ~50–100 bytes (token).
+- **Profile (~1–2 KB):** The profile JSON (all the fields in【84†L119-L127】) is ~1–2 KB.
+- **Genres/Categories (~several KB):** A genre list of tens of items is on the order of 5–20 KB.
+- **List Pages (~10–100 KB+):** Each page of channels or movies varies. In Quassi’s example, 14 channels were returned in ~24-items JSON【85†L420-L428】 (likely ~5–10 KB). If hundreds of movies per page, the JSON can be tens of KB. For example, one client code stopped fetching after 10 pages or so【96†L1970-L1978】 to cap the output.
+- **EPG (~100 KB+):** Fetching EPG for all channels (several hours) can be large (100 KB or more), so it’s often split by time window.
+- **Streaming Links:** Not part of initial sync, but to play, clients call `create_link` or direct URLs, often resulting in short responses (redirect to actual stream).
-`create_link` itself can also be misleading. Server-side code can accept the request, enter the channel-link logic, and still end up with no valid `result.cmd` because the selected source is filtered out, temporary-link generation fails, a link-check path is required, or the eventual command resolves to a backend URL your player cannot actually consume. From the outside, that often looks like “`create_link` works but playback is empty”, when the real failure is either late in command assembly or one step later on the stream fetch. citeturn5view0turn33search0turn35view0
+**Timing:** In practice, a well-optimized portal can reply to a `get_ordered_list` page request in a few hundred milliseconds, but when doing dozens of pages across multiple categories, total time adds up. Client-side parallelism helps hide this. A bulletin board user noted **“First time load maybe 45 sec”** for a large subscription, whereas subsequent loads “in the background” are fast【74†L83-L91】.
-A final category is modern Ministra hardening around non-native devices. Community discussions around Ministra 5.4.x show operators reporting that Formuler boxes and STBEmu clients stopped being accepted after upgrades even with `disable_third_party_devices = false`, and Infomir’s current commercial positioning also ties provider support to Ministra Player Apps subscriptions rather than the older licence-key model. That does not prove every playback failure is a licensing problem, but it does show that newer Ministra deployments can enforce a stricter distinction between “looks close enough to a MAG” and “is a supported client”. citeturn30view0turn29search0turn31search1
+## Security and Rate Limiting
-## Ranked list of the most likely causes
+- **Authentication:** Stalker portal uses a bearer token (OAuth2-style) bound to the device (MAC). The token is short-lived (hours) and must be refreshed by re-running the handshake if expired. Clients must include it in every API call in the `Authorization` header【84†L109-L114】.
-| Likelihood | Cause | Why it fits the exact symptom |
-| --- | --- | --- |
-| Very high | **Persisted device binding mismatch**: `sn`, `device_id`, `device_id2`, `signature`, `hw_version`, or model/firmware tuple do not match what the portal stored for that MAC | Infomir exposes anti-cloning controls such as `enable_device_id_validation`, maintainers diagnose `device_id2 mismatch` explicitly, and working STBEmu/MyTVOnline-class setups often need the full identifier tuple copied from a real client. This is the cleanest explanation for “catalogue works, playback fails, real client works”. citeturn26view0turn27search10turn32view0 |
-| Very high | **IP-bound temporary-link or secure-link failure** on the final media request | Official NGINX temp-link and secure-link docs bind playback to the NGINX front end, to `X-Real-IP`, and in secure-link mode even to `$remote_addr` in the URL signature. If your player, proxy, or playback engine changes the network path between `create_link` and stream fetch, the final URL is rejected even though the control plane succeeded. citeturn35view0turn34view0turn37view0 |
-| High | **Incomplete MAG bootstrap before first play** | Archived MAG code does more than handshake + profile: it runs localisation, user initialisation, TV/player setup, favourites/modules loading, and watchdog traffic before normal TV use. Custom clients that jump straight from catalogue to `create_link` can miss state that some portals expect around playback. citeturn22view0turn44view0turn44view2turn47view0 |
-| High | **`create_link` request mismatch**: missing or wrong `force_ch_link_check`, `disable_ad`, `forced_storage`, referer, or control-plane headers | Server code consumes `disable_ad` and `force_ch_link_check`, and third-party clients show explicit `forced_storage`, `disable_ad`, and referer handling for `create_link`. This is a common place where a custom player looks “close enough” elsewhere but differs at playback resolution time. citeturn5view0turn33search0turn42view0turn47view0 |
-| Medium | **Per-channel source filtering by user-agent or device model** | Infomir added several-source channels with user-agent filters and documents model-based restriction of streaming-link access. That can leave the catalogue visible but deny or divert the actual stream source when play is pressed. citeturn36search1turn23search6turn37view0 |
-| Medium | **Newer MAG fingerprint fields missing**: `client_type`, `metrics`, `hw_version_2`, `timestamp`, `api_signature`, `prehash` | Archived MAG JS sends a much richer fingerprint than many older open-source emulators. A portal that has learned to distinguish native and non-native clients can use those gaps as a hardening signal even if it still tolerates browsing. citeturn22view0turn42view0 |
-| Lower but real | **Third-party-device support or certification hardening in newer Ministra** | Community reports from 5.4.x show non-native clients being rejected despite the old `disable_third_party_devices` toggle. This is less likely when your app already reaches the catalogue, but it still belongs on the list for mixed portal fleets and upgraded deployments. citeturn30view0turn29search0turn31search1 |
+- **Device Verification:** The portal compares the provided MAC and token to ensure the same device. If a client changes its device ID or MAC between calls, the portal will reject the requests.
-## Concrete diff checklist and the implementation changes most likely to improve compatibility
+- **Rate Limits:** Official limits aren’t documented, but providers often throttle abusive patterns. In practice, clients avoid hammering the portal (e.g. pause between calls, use caching). Extremely high parallel requests might cause HTTP 429 responses from some servers.
-The fastest way to debug this class of problem is to diff your app against a known-good MAG-style client at **three layers at once**: control-plane requests, `create_link`, and the final media fetch. Compare at least the following items:
+- **Data Privacy:** All video streams still require credentials; knowing an item’s metadata doesn’t give the stream URL without a valid session token. Some portals watermark URLs with play tokens to prevent link sharing.
-| Layer | What to compare exactly | Why it matters |
-| --- | --- | --- |
-| Portal entry | Referer path (`/c/` vs `/c/index.html`), `User-Agent`, `X-User-Agent`, `Cookie: mac/stb_lang/timezone`, bearer token use | Real traces and emulators reproduce a very specific header envelope, and some portals key off it. citeturn47view0turn13search1turn41search0 |
-| Handshake | Presence of saved token reuse and `prehash` | MAG JS includes both; token handling changed across middleware versions. citeturn22view0turn27search2 |
-| `get_profile` | Full field set and coherence of `stb_type`, `sn`, `ver`, `image_version`, `hw_version`, `device_id*`, `signature`, `client_type`, telemetry fields | This is the single biggest fingerprint surface. citeturn22view0turn21view2turn42view0 |
-| Bootstrap | Whether you do `get_localization`, `user_init`-equivalent setup, module loading, favourites/TV initialisation, watchdog traffic before first play | A working client reaches playback with more state already established. citeturn44view0turn44view2turn47view0 |
-| `create_link` | Query parameters, header set, referer, `disable_ad`, `force_ch_link_check`, `forced_storage`, and exact submitted `cmd` | Small request differences here can alter source selection or return an unusable command. citeturn5view0turn33search0turn42view0 |
-| Returned command | Whether you preserve the command **byte-for-byte**, including solution prefixes such as `ffmpeg `, query strings, and token parameters | Server code can prepend player-solution tokens and signed URLs that must not be normalised away. citeturn5view0 |
-| Temp-link playback | Whether you request the front-end `/ch/<token>` URL as-is, through the same outward IP, instead of rewriting it to backend hosts or localhost paths | Official temp-link flow depends on front-end NGINX rewrite and internal redirecting. citeturn35view0turn34view0turn37view0 |
-| Network identity | Whether the actual media fetch originates from the same client IP the portal/backend expects | Secure-link and real-IP forwarding make this decisive. citeturn34view0turn35view0turn37view0 |
-| Stream-source filtering | Whether the working client receives a different source because of user-agent/model filters on the channel | The catalogue can be identical while the selected stream link differs. citeturn36search1turn23search6 |
-| Server logs | Apache auth failures, `device_id2 mismatch`, NGINX `403/410`, Flussonic auth headers, channel-monitoring/link-check errors | The decisive clue is often on the media side, not in `load.php`. citeturn26view0turn34view0turn45view0 |
+## Recommendations and Trade-offs
-If your goal is to maximise compatibility rather than keep the implementation minimal, the changes most likely to move the needle are these.
+For best UX on first load, clients should:
-First, treat each portal as needing a **persisted device profile**, not just a MAC and a generic MAG persona. Persist `sn`, `stb_type`, `ver`, `image_version`, `hw_version`, `device_id`, `device_id2`, and `signature` per portal profile, and allow importing them from a known-good client trace when necessary. The sources above strongly suggest that this beats trying to re-derive “mostly MAG-like” values from the MAC alone. citeturn22view0turn26view0turn32view0
+- **Show progress:** Inform the user that sync is happening, e.g. “Loading movies 1/10…”. This sets expectations for longer waits (minutes) if needed.
-Secondly, support a **modern `get_profile` mode** alongside your legacy MAG250 mode. For portals that are sensitive, send the richer MAG fields that archived MAG JS sends, including `client_type`, `video_out`, `metrics`, `hw_version_2`, `timestamp`, `api_signature`, and `prehash`, and keep the whole model/firmware story internally consistent. Many custom Stalker implementations fail because they still emulate only the older subset that lax portals accept. citeturn22view0turn42view0
+- **Prioritize UI readiness:** Always fetch category lists before item lists, so the UI can show “(Loading…)” instead of a blank screen.
-Thirdly, replicate the **pre-play bootstrap** of a real client before the first playback attempt. At minimum, do handshake, `get_profile`, `get_localization`, module initialisation, TV/player initialisation, and watchdog start in the same broad order as the working client, and carry any playback-affecting flags such as `force_ch_link_check`, `disable_ad`, and `forced_storage` into `create_link`. citeturn44view0turn44view2turn47view0turn5view0
+- **Use caching:** Persist data locally. On app restart, show cached content immediately, then silently update in background (UX feels instant).
+
+- **Compress and Parallelize:** Enable gzip and make requests concurrently (subject to server tolerance). Many open-source clients do exactly this to minimize elapsed time【70†L3925-L3933】.
+
+- **Balance pagination vs size:** Choose a page size that balances JSON overhead versus call overhead. Some portals allow tuning `max_page_items`; clients should use a moderately large page (e.g. 50 items) to reduce number of requests but avoid giant JSON dumps.
+
+- **Table of Pros/Cons (excerpt):**
+
+```markdown
+| Approach | Pros | Cons |
+|--------------------|-------------------------------------------|---------------------------------------|
+| Full download | No more sync needed (complete library) | Very slow initial load, heavy memory |
+| Page-by-page | Fast initial pages, lower memory use | Need to handle multiple requests |
+| Lazy per-category | Quick start, fetch only viewed sections | Each new category load has delay |
+| Cache & Refresh | Instant reload after first sync | Risk of stale info if not updated |
+| Parallel fetching | Reduces wait by overlapping requests | Server may throttle many simultaneous |
+```
+
+## Diagrams of Initial Sync
+
+Below is a high-level **flowchart** of an example initial sync data flow:
+
+```mermaid
+flowchart TB
+ subgraph Client Device
+ A(Credentials Entry) --> B(Handshake: GET type=stb&action=handshake)
+ B -->|token| C(Get Profile)
+ C --> D(Get ITV Genres)
+ C --> E(Get VOD Categories)
+ D --> F{Channels by genre}
+ F -->|GET for genre i| G(ITV List Page 1)
+ G --> H(UI list gets channels)
+ E --> I{Movies by category}
+ I -->|GET for category j| J(VOD List Page 1)
+ J --> K(UI list gets movies)
+ %% Parallel arrow style
+ C -->|concurrent| D
+ C -->|concurrent| E
+ F -->|pages| G
+ I -->|pages| J
+ end
+ subgraph Server (Portal API)
+ B[Portal.php: handshake] -- returns token --> B
+ C[Portal.php: get_profile] -- returns settings --> C
+ D[Portal.php: get_genres] -- returns genre list --> D
+ E[Portal.php: get_categories] -- returns VOD cats --> E
+ G[Portal.php: get_ordered_list (itv)] -- returns channels --> G
+ J[Portal.php: get_ordered_list (vod)] -- returns movies --> J
+ end
+```
+
+This flow highlights that after the handshake (`B`) and profile (`C`), the client can load live-TV and VOD data in parallel.
+
+Below is a **sequence timeline** of typical API calls (latency not to scale):
+
+```mermaid
+sequenceDiagram
+ participant U as User/Client
+ participant S as Portal Server
+ U->>S: GET /portal.php?type=stb&action=handshake
+ S-->>U: {"js":{"token":"XYZ","..."}}
+ U->>S: GET /portal.php?type=stb&action=get_profile (Bearer token)
+ S-->>U: {"js":{...profile data...}}
+ par
+ U->>S: GET /portal.php?type=itv&action=get_genres
+ S-->>U: {"js":[...genre list...]}
+ U->>S: GET /portal.php?type=vod&action=get_categories
+ S-->>U: {"js":[...VOD cat list...]}
+ end
+ U->>S: GET /portal.php?type=itv&action=get_ordered_list&genre=ID&p=1
+ S-->>U: {"js":{"total_items":N,"data":[...channels...]}}
+ U->>S: GET /portal.php?type=vod&action=get_ordered_list&category=ID&p=1
+ S-->>U: {"js":{"total_items":M,"data":[...movies...]}}
+ Note over U: UI displays first pages; app continues fetching more pages...
+```
-Fourthly, make playback URL handling **opaque and lossless**. Do not strip prefixes like `ffmpeg `, do not re-encode signed query parameters, do not rewrite `/ch/<token>` to backend paths, and do not insert a local proxy unless you can guarantee the same outbound IP and header behaviour as direct device playback. This one change can eliminate a large class of temp-link and secure-link failures. citeturn35view0turn34view0turn37view0turn45view0
+These diagrams illustrate that initial sync is multi-step and often involves overlapping requests for best performance.
-Finally, build a **wire-level diff mode** into your app. For one failing portal, capture a working MyTVOnline+/STBEmu session and your own session, then diff: request headers, cookies, query params, pre-play sequence timing, `create_link` body, returned `cmd`, final URL, redirect chain, and server response codes. With Stalker compatibility, the winning fixes are usually not architectural; they are small deltas in identity, sequencing, or final URL handling that only become obvious when you compare both traces side by side. citeturn47view0turn22view0turn35view0turn45view0
\ No newline at end of file
+**Sources:** Based on Infomir/Stalker portal documentation and source code【84†L89-L97】【90†L3333-L3341】, client implementations【96†L1969-L1978】, and user reports【85†L328-L336】【75†L193-L200】. Each cited source provides concrete details on endpoints, request/response formats, or performance anecdotes, supporting the strategies and patterns described above.
\ No newline at end of file
diff --git a/final_narrow_compile.log b/final_narrow_compile.log
new file mode 100644
index 00000000..e4ceda26
Binary files /dev/null and b/final_narrow_compile.log differ
diff --git a/final_narrow_test1.log b/final_narrow_test1.log
new file mode 100644
index 00000000..6ed07701
Binary files /dev/null and b/final_narrow_test1.log differ
diff --git a/final_narrow_test2.log b/final_narrow_test2.log
new file mode 100644
index 00000000..c3226aef
Binary files /dev/null and b/final_narrow_test2.log differ
diff --git a/gradle_runner_regression_rerun.log b/gradle_runner_regression_rerun.log
new file mode 100644
index 00000000..44529686
Binary files /dev/null and b/gradle_runner_regression_rerun.log differ
diff --git a/gradle_runner_regression_test.log b/gradle_runner_regression_test.log
new file mode 100644
index 00000000..32ebaa47
Binary files /dev/null and b/gradle_runner_regression_test.log differ
diff --git a/gradle_stalker_overlap_rerun.log b/gradle_stalker_overlap_rerun.log
new file mode 100644
index 00000000..85f86c77
Binary files /dev/null and b/gradle_stalker_overlap_rerun.log differ
diff --git a/gradle_stalker_overlap_test.log b/gradle_stalker_overlap_test.log
new file mode 100644
index 00000000..8c21d22b
Binary files /dev/null and b/gradle_stalker_overlap_test.log differ
diff --git a/gradle_sync_fix_test1.log b/gradle_sync_fix_test1.log
new file mode 100644
index 00000000..d53c666a
Binary files /dev/null and b/gradle_sync_fix_test1.log differ
diff --git a/gradle_sync_fix_test2.log b/gradle_sync_fix_test2.log
new file mode 100644
index 00000000..8f437ba8
Binary files /dev/null and b/gradle_sync_fix_test2.log differ
diff --git a/gradle_test1.log b/gradle_test1.log
new file mode 100644
index 00000000..5d7ac171
Binary files /dev/null and b/gradle_test1.log differ
diff --git a/gradle_test2.log b/gradle_test2.log
new file mode 100644
index 00000000..c7ae3f12
Binary files /dev/null and b/gradle_test2.log differ