StreamVault is a TV-first free IPTV player for Android TV built with Kotlin, Jetpack Compose, Room, Hilt, and Media3.
Kotlin iptv-player
Pipeline Xtream live retrieval and staging with bounded buffers
David Nashash5b0d43c28bb1
diff --git a/data/src/main/java/com/streamvault/data/remote/xtream/OkHttpXtreamApiService.kt b/data/src/main/java/com/streamvault/data/remote/xtream/OkHttpXtreamApiService.ktindex b503a15c..c0227e58 100644--- a/data/src/main/java/com/streamvault/data/remote/xtream/OkHttpXtreamApiService.kt+++ b/data/src/main/java/com/streamvault/data/remote/xtream/OkHttpXtreamApiService.kt@@ -537,7 +537,10 @@ class OkHttpXtreamApiService()?.let { throw it }stream.unread(previewBytes)- val reader = JsonReader(InputStreamReader(stream, charset))+ val rowBudget = if (descriptor.action == "get_live_streams") {+ XtreamLiveRowBudgetReader(InputStreamReader(stream, charset))+ } else null+ val reader = JsonReader(rowBudget ?: InputStreamReader(stream, charset))reader.isLenient = truereturn when (reader.peek()) {JsonToken.BEGIN_ARRAY -> {@@ -545,16 +548,21 @@ class OkHttpXtreamApiService(reader.beginArray()while (reader.hasNext()) {currentCoroutineContext().ensureActive()+ rowBudget?.beginRow()val item = try {if (readItem != null) readItem(reader) else {val element = JsonParser.parseReader(reader)json.decodeFromString(deserializer, element.toString())}+ } catch (e: XtreamLiveRowTooLargeException) {+ throw e} catch (e: RuntimeException) {throw XtreamParsingException("Malformed JSON from ${descriptor.hint}${sanitizedPreview(preview)?.let { " (preview=$it)" } ?: ""}",e)+ } finally {+ rowBudget?.endRow()}onItem(item)emittedCount++diff --git a/data/src/main/java/com/streamvault/data/remote/xtream/XtreamLiveRowBudgetReader.kt b/data/src/main/java/com/streamvault/data/remote/xtream/XtreamLiveRowBudgetReader.ktnew file mode 100644index 00000000..b752aeb8--- /dev/null+++ b/data/src/main/java/com/streamvault/data/remote/xtream/XtreamLiveRowBudgetReader.kt@@ -0,0 +1,33 @@+package com.streamvault.data.remote.xtream++import java.io.Reader+import com.streamvault.domain.provider.ProviderSetupFailureKind++/** Bounds token allocation before JsonReader can build a very large string/tree. */+internal class XtreamLiveRowBudgetReader(private val delegate: Reader) : Reader() {+ private var remaining: Int? = null++ fun beginRow() { remaining = MAX_ROW_CHARS }+ fun endRow() { remaining = null }++ override fun read(buffer: CharArray, offset: Int, length: Int): Int {+ if (length == 0) return 0+ val budget = remaining+ if (budget != null && budget <= 0) throw XtreamLiveRowTooLargeException()+ val read = delegate.read(buffer, offset, minOf(length, budget ?: length))+ if (read > 0 && budget != null) remaining = budget - read+ return read+ }++ override fun close() = delegate.close()++ companion object {+ // A JSON row (including skipped metadata) may retain up to JsonReader's+ // 1024-character read-ahead in addition to this budget. No silent trim.+ const val MAX_ROW_CHARS = 1024 * 1024+ }+}++class XtreamLiveRowTooLargeException : XtreamApiException("Live JSON row exceeds token allocation budget") {+ override val kind = ProviderSetupFailureKind.RESPONSE_TOO_LARGE+}diff --git a/data/src/main/java/com/streamvault/data/sync/CatalogSyncRuntimeProfile.kt b/data/src/main/java/com/streamvault/data/sync/CatalogSyncRuntimeProfile.ktindex d3e623e4..0a98c025 100644--- a/data/src/main/java/com/streamvault/data/sync/CatalogSyncRuntimeProfile.kt+++ b/data/src/main/java/com/streamvault/data/sync/CatalogSyncRuntimeProfile.kt@@ -58,6 +58,15 @@ internal data class CatalogSyncRuntimeProfile(val deferBackgroundWorkOnLowMemory: Boolean,val snapshot: SyncDeviceSnapshot) {+ // Preserve existing row counts until on-device tuning. Wide rows flush by+ // estimated retained bytes first; LOW keeps only one queued batch.+ val liveQueuedBatches: Int get() = if (tier == DeviceSyncTier.LOW) 1 else 2+ val liveBatchBytes: Long get() = when (tier) {+ DeviceSyncTier.LOW -> 256L * 1024+ DeviceSyncTier.MID -> 512L * 1024+ DeviceSyncTier.HIGH -> 1024L * 1024+ }+val diagnosticsLabel: Stringget() = "${tier.name.lowercase()} batch=$stageBatchSize memory=${snapshot.memoryClassMb}MB"diff --git a/data/src/main/java/com/streamvault/data/sync/SyncManagerCatalogStrategySupport.kt b/data/src/main/java/com/streamvault/data/sync/SyncManagerCatalogStrategySupport.ktindex af604809..8d02681d 100644--- a/data/src/main/java/com/streamvault/data/sync/SyncManagerCatalogStrategySupport.kt+++ b/data/src/main/java/com/streamvault/data/sync/SyncManagerCatalogStrategySupport.kt@@ -81,6 +81,8 @@ internal class SyncManagerCatalogStrategySupport(return when (error) {is XtreamResponseTooLargeException,is LowMemoryCatalogAbortException,+ is XtreamLiveBufferLimitException,+ is com.streamvault.data.remote.xtream.XtreamLiveRowTooLargeException,is XtreamParsingException -> trueis java.net.SocketTimeoutException,is java.io.InterruptedIOException -> truediff --git a/data/src/main/java/com/streamvault/data/sync/SyncManagerXtreamLiveStrategy.kt b/data/src/main/java/com/streamvault/data/sync/SyncManagerXtreamLiveStrategy.ktindex ed041333..bd78c295 100644--- a/data/src/main/java/com/streamvault/data/sync/SyncManagerXtreamLiveStrategy.kt+++ b/data/src/main/java/com/streamvault/data/sync/SyncManagerXtreamLiveStrategy.kt@@ -18,6 +18,11 @@ import com.streamvault.domain.sync.Sectionimport com.streamvault.domain.sync.SyncProgressimport kotlinx.coroutines.sync.Muteximport kotlinx.coroutines.sync.withLock+import kotlinx.coroutines.CoroutineStart+import kotlinx.coroutines.async+import kotlinx.coroutines.coroutineScope+import kotlinx.coroutines.sync.Semaphore+import kotlinx.coroutines.sync.withPermitimport kotlin.system.measureTimeMillisprivate const val XTREAM_LIVE_STRATEGY_TAG = "SyncManager"@@ -56,38 +61,58 @@ internal class SyncManagerXtreamLiveStrategy(trackInitialLiveOnboarding: Boolean,effectiveLiveSyncMethod: EffectiveXtreamLiveSyncMethod = EffectiveXtreamLiveSyncMethod.STREAM_ALL,requiredHiddenLiveCategoryIds: Set<Long> = emptySet()- ): CatalogSyncPayload<Channel> {+ ): CatalogSyncPayload<Channel> = coroutineScope {Log.i(XTREAM_LIVE_STRATEGY_TAG, "Xtream live strategy start for provider ${provider.id}.")// Recovery runs before entering this strategy. Any other saved batches// are unsealed attempts and cannot be reused by this import.syncCatalogStore.clearProviderStaging(provider.id)- val rawLiveCategories = when (val attempt = xtreamSupport.attemptNonCancellation {- xtreamSupport.retryXtreamCatalogTransient(provider.id) {- xtreamSupport.executeXtreamRequest(provider.id, XtreamAdaptiveSyncPolicy.Stage.LIGHTWEIGHT) {- xtreamCatalogApiService.getLiveCategories(- XtreamUrlFactory.buildPlayerApiUrl(- serverUrl = provider.serverUrl,- username = provider.username,- password = provider.password,- action = "get_live_categories"+ val shouldAttemptFullCatalog = effectiveLiveSyncMethod == EffectiveXtreamLiveSyncMethod.STREAM_ALL &&+ hiddenLiveCategoryIds.isEmpty() &&+ runtimeProfile.shouldAttemptFullLiveCatalog(trackInitialLiveOnboarding)+ // A stressed provider gets one request at a time. Starting metadata first+ // also avoids a one-slot deadlock when the writer needs that context.+ val requestSlots = Semaphore(xtreamAdaptiveSyncPolicy.liveRequestConcurrencyFor(provider.id))+ val categoryContext = async(start = CoroutineStart.UNDISPATCHED) {+ val rawLiveCategories = requestSlots.withPermit {+ when (val attempt = xtreamSupport.attemptNonCancellation {+ xtreamSupport.retryXtreamCatalogTransient(provider.id) {+ xtreamSupport.executeXtreamRequest(provider.id, XtreamAdaptiveSyncPolicy.Stage.LIGHTWEIGHT) {+ xtreamCatalogApiService.getLiveCategories(+ XtreamUrlFactory.buildPlayerApiUrl(+ serverUrl = provider.serverUrl,+ username = provider.username,+ password = provider.password,+ action = "get_live_categories"+ )+ )+ }+ }+ }) {+ is Attempt.Success -> attempt.value+ is Attempt.Failure -> {+ Log.w(+ XTREAM_LIVE_STRATEGY_TAG,+ "Xtream live categories request failed for provider ${provider.id}: ${sanitizeThrowableMessage(attempt.error)}")- )+ null+ }}}- }) {- is Attempt.Success -> attempt.value- is Attempt.Failure -> {- Log.w(- XTREAM_LIVE_STRATEGY_TAG,- "Xtream live categories request failed for provider ${provider.id}: ${sanitizeThrowableMessage(attempt.error)}"- )- null+ // Cache even a failed (empty) result. Mapping must not retry category+ // metadata implicitly for every batch outside the request policy.+ val resolvedCategories = api.mapCategories(ContentType.LIVE, rawLiveCategories.orEmpty())+ .map { category -> category.toEntity(provider.id) }.takeIf { it.isNotEmpty() }+ rawLiveCategories to resolvedCategories+ }+ var fullPayload = CatalogSyncPayload<Channel>(+ catalogResult = CatalogStrategyResult.EmptyValid("full"), categories = null)+ if (shouldAttemptFullCatalog) {+ progress(provider.id, onProgress, "Downloading Live TV...")+ fullPayload = requestSlots.withPermit {+ loadXtreamLiveFull(provider, api, runtimeProfile, beforeMapping = { categoryContext.await() })}}- val resolvedCategories = rawLiveCategories- ?.let { categories -> api.mapCategories(ContentType.LIVE, categories) }- ?.map { category -> category.toEntity(provider.id) }- ?.takeIf { it.isNotEmpty() }+ val (rawLiveCategories, resolvedCategories) = categoryContext.await()val filteredRawLiveCategories = rawLiveCategories.orEmpty().filterNot { category ->val id = category.categoryId.toLongOrNull()id in hiddenLiveCategoryIds && id !in requiredHiddenLiveCategoryIds@@ -98,18 +123,9 @@ internal class SyncManagerXtreamLiveStrategy(// the hidden category itself.val resolvedCatalogCategories = resolvedCategories?.takeIf { it.isNotEmpty() }- var fullPayload = CatalogSyncPayload<Channel>(- catalogResult = CatalogStrategyResult.EmptyValid("full"),- categories = null- )- val shouldAttemptFullCatalog = effectiveLiveSyncMethod == EffectiveXtreamLiveSyncMethod.STREAM_ALL &&- hiddenLiveCategoryIds.isEmpty() &&- runtimeProfile.shouldAttemptFullLiveCatalog(trackInitialLiveOnboarding)if (shouldAttemptFullCatalog) {- progress(provider.id, onProgress, "Downloading Live TV...")- fullPayload = loadXtreamLiveFull(provider, api, runtimeProfile)when (val fullResult = fullPayload.catalogResult) {- is CatalogStrategyResult.Success -> return fullPayload.copy(+ is CatalogStrategyResult.Success -> return@coroutineScope fullPayload.copy(categories = catalogStrategySupport.mergePreferredAndFallbackCategories(resolvedCatalogCategories,fullPayload.categories ?: catalogStrategySupport.buildFallbackLiveCategories(provider.id, fullResult.items)@@ -120,7 +136,7 @@ internal class SyncManagerXtreamLiveStrategy(fullCatalogUnsafe = false))- is CatalogStrategyResult.Partial -> return fullPayload.copy(+ is CatalogStrategyResult.Partial -> return@coroutineScope fullPayload.copy(categories = catalogStrategySupport.mergePreferredAndFallbackCategories(resolvedCatalogCategories,fullPayload.categories ?: catalogStrategySupport.buildFallbackLiveCategories(provider.id, fullResult.items)@@ -149,7 +165,7 @@ internal class SyncManagerXtreamLiveStrategy(preferSequential = existingMetadata.liveSequentialFailuresRemembered || runtimeProfile.maxCategoryConcurrency <= 1,runtimeProfile = runtimeProfile)- return CatalogSyncPayload(+ CatalogSyncPayload(catalogResult = categoryPayload.catalogResult,categories = catalogStrategySupport.mergePreferredAndFallbackCategories(resolvedCatalogCategories,@@ -174,7 +190,17 @@ internal class SyncManagerXtreamLiveStrategy(suspend fun loadXtreamLiveFull(provider: Provider,api: XtreamProvider,- runtimeProfile: CatalogSyncRuntimeProfile+ runtimeProfile: CatalogSyncRuntimeProfile,+ beforeMapping: suspend () -> Unit = {}+ ): CatalogSyncPayload<Channel> = xtreamAdaptiveSyncPolicy.withLivePipelinePermit {+ loadXtreamLiveFullWithPermit(provider, api, runtimeProfile, beforeMapping)+ }++ private suspend fun loadXtreamLiveFullWithPermit(+ provider: Provider,+ api: XtreamProvider,+ runtimeProfile: CatalogSyncRuntimeProfile,+ beforeMapping: suspend () -> Unit): CatalogSyncPayload<Channel> {val endpoint = XtreamUrlFactory.buildPlayerApiUrl(serverUrl = provider.serverUrl,@@ -190,6 +216,8 @@ internal class SyncManagerXtreamLiveStrategy(failureNextStep = "category-bulk",streamItems = { item -> xtreamCatalogHttpService.streamLiveStreams(endpoint, onItem = item) },mapRawBatch = { batch -> api.mapLiveStreamsSequence(batch) },+ estimatedBytes = { it.estimatedBufferBytes() },+ beforeMapping = beforeMapping,runtimeProfile = runtimeProfile)}@@ -200,6 +228,8 @@ internal class SyncManagerXtreamLiveStrategy(failureNextStep = "legacy-full",streamItems = { item -> xtreamCatalogHttpService.streamLiveStreamRows(endpoint, onItem = item) },mapRawBatch = { batch -> api.mapLiveStreamRowsSequence(batch) },+ estimatedBytes = { it.estimatedBufferBytes() },+ beforeMapping = beforeMapping,runtimeProfile = runtimeProfile)if (!thinPayload.shouldRetryLegacyFullDecode()) {@@ -216,6 +246,8 @@ internal class SyncManagerXtreamLiveStrategy(failureNextStep = "category-bulk",streamItems = { item -> xtreamCatalogHttpService.streamLiveStreams(endpoint, onItem = item) },mapRawBatch = { batch -> api.mapLiveStreamsSequence(batch) },+ estimatedBytes = { it.estimatedBufferBytes() },+ beforeMapping = beforeMapping,runtimeProfile = runtimeProfile)}@@ -235,11 +267,12 @@ internal class SyncManagerXtreamLiveStrategy(failureNextStep: String,streamItems: suspend (suspend (RawItem) -> Unit) -> Int,mapRawBatch: suspend (Sequence<RawItem>) -> Sequence<Channel>,+ estimatedBytes: (RawItem) -> Long,+ beforeMapping: suspend () -> Unit,runtimeProfile: CatalogSyncRuntimeProfile): CatalogSyncPayload<Channel> {var fallbackCollector = FallbackCategoryCollector(provider.id, ContentType.LIVE)val seenStreamIds = HashSet<Long>()- val rawBatch = ArrayList<RawItem>(runtimeProfile.stageBatchSize)var stagedSessionId: Long? = nullvar acceptedCount = 0var streamedRawCount = 0@@ -251,7 +284,6 @@ internal class SyncManagerXtreamLiveStrategy(suspend fun discardAttempt() {stagedSessionId?.let { syncCatalogStore.discardStagedImport(provider.id, it) }stagedSessionId = null- rawBatch.clear()seenStreamIds.clear()fallbackCollector = FallbackCategoryCollector(provider.id, ContentType.LIVE)acceptedCount = 0@@ -279,7 +311,7 @@ internal class SyncManagerXtreamLiveStrategy(}}- suspend fun flushRawBatch() {+ suspend fun flushRawBatch(rawBatch: List<RawItem>) {if (rawBatch.isEmpty()) returnlateinit var mappedChannels: List<Channel>mappingElapsedMs += measureTimeMillis {@@ -301,7 +333,6 @@ internal class SyncManagerXtreamLiveStrategy(stagedSessionId = staged.sessionIdacceptedCount += staged.acceptedCountflushCount++- rawBatch.clear()// D10 — mode HIGH (full catalog) : pas de count par categorie disponible,// on emet en indetermine (`total = 0`) une fois par flush de batch (cadence// <= 1/s en pratique, jamais par item). Le label reste vide car aucune@@ -327,15 +358,16 @@ internal class SyncManagerXtreamLiveStrategy(stagingElapsedMs = 0Ltry {xtreamSupport.executeXtreamRequest(provider.id, XtreamAdaptiveSyncPolicy.Stage.HEAVY) {- streamItems { item ->- rawBatch += item- streamedRawCount++- if (rawBatch.size >= runtimeProfile.stageBatchSize) {- flushRawBatch()- }- }.also {- flushRawBatch()- }+ val stats = runXtreamLivePipeline(+ batchRows = runtimeProfile.stageBatchSize,+ batchBytes = runtimeProfile.liveBatchBytes,+ queuedBatches = runtimeProfile.liveQueuedBatches,+ estimatedBytes = estimatedBytes,+ prepare = beforeMapping,+ read = { emit -> streamItems { item -> streamedRawCount++; emit(item) } },+ consume = { batch -> flushRawBatch(batch) }+ )+ Log.i("XtreamSyncTiming", "phase=live-pipeline rows=${stats.rows} batches=${stats.batches} reader_ms=${stats.readerMs} drain_ms=${stats.drainMs} send_wait_ms=${stats.sendWaitMs} outstanding_batch_bytes=${stats.outstandingBatchBytes} queue_capacity=${runtimeProfile.liveQueuedBatches} batch_byte_limit=${runtimeProfile.liveBatchBytes}")}} catch (error: Throwable) {discardAttempt()diff --git a/data/src/main/java/com/streamvault/data/sync/XtreamAdaptiveSyncPolicy.kt b/data/src/main/java/com/streamvault/data/sync/XtreamAdaptiveSyncPolicy.ktindex dab9c298..f44850c4 100644--- a/data/src/main/java/com/streamvault/data/sync/XtreamAdaptiveSyncPolicy.kt+++ b/data/src/main/java/com/streamvault/data/sync/XtreamAdaptiveSyncPolicy.kt@@ -8,6 +8,8 @@ import java.util.concurrent.ConcurrentHashMapimport kotlinx.coroutines.delayimport kotlinx.coroutines.sync.Muteximport kotlinx.coroutines.sync.withLock+import kotlinx.coroutines.sync.Semaphore+import kotlinx.coroutines.sync.withPermitinternal class XtreamAdaptiveSyncPolicy {private companion object {@@ -36,6 +38,15 @@ internal class XtreamAdaptiveSyncPolicy {private val providerHealth = ConcurrentHashMap<Long, ProviderHealth>()private val providerPacing = ConcurrentHashMap<Long, ProviderPacingState>()+ // This policy is shared by SyncManager. Do not multiply queued bulk batches+ // when different providers import simultaneously.+ private val livePipelineSlot = Semaphore(1)++ suspend fun <T> withLivePipelinePermit(block: suspend () -> T): T =+ livePipelineSlot.withPermit { block() }++ fun liveRequestConcurrencyFor(providerId: Long): Int =+ if ((providerHealth[providerId]?.stressLevel ?: 0) > 0) 1 else 2suspend fun awaitTurn(providerId: Long, stage: Stage) {val waitMs = (providerHealth[providerId]?.backoffUntilMs ?: 0L) - System.currentTimeMillis()diff --git a/data/src/main/java/com/streamvault/data/sync/XtreamLivePipeline.kt b/data/src/main/java/com/streamvault/data/sync/XtreamLivePipeline.ktnew file mode 100644index 00000000..e926878c--- /dev/null+++ b/data/src/main/java/com/streamvault/data/sync/XtreamLivePipeline.kt@@ -0,0 +1,107 @@+package com.streamvault.data.sync++import com.streamvault.data.remote.dto.XtreamLiveStreamRow+import com.streamvault.data.remote.dto.XtreamStream+import java.util.concurrent.atomic.AtomicLong+import kotlinx.coroutines.CancellationException+import kotlinx.coroutines.Dispatchers+import kotlinx.coroutines.async+import kotlinx.coroutines.cancel+import kotlinx.coroutines.channels.Channel+import kotlinx.coroutines.coroutineScope++/** Only the consumer owns mapping, duplicate admission and staging state. */+internal suspend fun <T> runXtreamLivePipeline(+ batchRows: Int,+ batchBytes: Long,+ queuedBatches: Int,+ estimatedBytes: (T) -> Long,+ read: suspend (suspend (T) -> Unit) -> Int,+ prepare: suspend () -> Unit = {},+ consume: suspend (List<T>) -> Unit+): XtreamLivePipelineStats = coroutineScope {+ require(batchRows > 0 && batchBytes > 0 && queuedBatches > 0)+ val queue = Channel<BufferedLiveBatch<T>>(queuedBatches)+ val retainedBytes = AtomicLong()+ val highWaterBytes = AtomicLong()+ var sendWaitNanos = 0L+ var batches = 0+ val started = System.nanoTime()+ val writer = async(Dispatchers.Default) {+ try {+ prepare()+ for (batch in queue) {+ consume(batch.items)+ retainedBytes.addAndGet(-batch.bytes)+ }+ } catch (error: CancellationException) {+ // Child cancellation alone does not cancel its parent. Supersession+ // must also interrupt a reader blocked on HTTP or a full queue.+ [email protected](error)+ throw error+ }+ }+ try {+ var batch = ArrayList<T>(batchRows)+ var bytes = 0L+ suspend fun sendBatch() {+ if (batch.isEmpty()) return+ val sending = BufferedLiveBatch(batch, bytes)+ val outstanding = retainedBytes.addAndGet(bytes)+ highWaterBytes.updateAndGet { maxOf(it, outstanding) }+ val beforeSend = System.nanoTime()+ queue.send(sending)+ sendWaitNanos += System.nanoTime() - beforeSend+ batches+++ batch = ArrayList(batchRows)+ bytes = 0+ }+ val rows = read { row ->+ val size = estimatedBytes(row).coerceAtLeast(1)+ if (size > batchBytes) throw XtreamLiveBufferLimitException("Live row exceeds buffered batch budget")+ if (batch.isNotEmpty() && bytes + size > batchBytes) sendBatch()+ batch.add(row)+ bytes += size+ if (batch.size >= batchRows || bytes >= batchBytes) sendBatch()+ }+ // read only returns after the transport has validated END_DOCUMENT.+ sendBatch()+ val readerMs = (System.nanoTime() - started) / 1_000_000+ queue.close()+ val drainStarted = System.nanoTime()+ writer.await()+ XtreamLivePipelineStats(rows, batches, readerMs,+ (System.nanoTime() - drainStarted) / 1_000_000,+ sendWaitNanos / 1_000_000, highWaterBytes.get())+ } finally {+ queue.cancel()+ writer.cancel()+ // coroutineScope joins the writer before attempt cleanup/retry outside.+ }+}++internal data class XtreamLivePipelineStats(+ val rows: Int,+ val batches: Int,+ val readerMs: Long,+ val drainMs: Long,+ val sendWaitMs: Long,+ // Includes the writer batch and a producer suspended in send, not just queue.+ val outstandingBatchBytes: Long+)++internal class XtreamLiveBufferLimitException(message: String) : IllegalStateException(message)++private data class BufferedLiveBatch<T>(val items: List<T>, val bytes: Long)++private fun retainedTextBytes(fields: List<String?>, ids: List<String>?): Long =+ 256L + fields.sumOf { 40L + 2L * (it?.length ?: 0) } ++ ids.orEmpty().sumOf { 48L + 2L * it.length }++internal fun XtreamLiveStreamRow.estimatedBufferBytes(): Long = retainedTextBytes(+ listOf(name, streamIcon, epgChannelId, categoryId, categoryName, containerExtension), categoryIds)++internal fun XtreamStream.estimatedBufferBytes(): Long = retainedTextBytes(+ listOf(name, streamType, streamIcon, coverBig, epgChannelId, added, categoryId,+ categoryName, customSid, directSource, containerExtension, rating, rating5based,+ tmdb, trailer, youtubeTrailer), categoryIds)diff --git a/data/src/test/java/com/streamvault/data/remote/xtream/OkHttpXtreamApiServiceTest.kt b/data/src/test/java/com/streamvault/data/remote/xtream/OkHttpXtreamApiServiceTest.ktindex 2870b093..e1416d0a 100644--- a/data/src/test/java/com/streamvault/data/remote/xtream/OkHttpXtreamApiServiceTest.kt+++ b/data/src/test/java/com/streamvault/data/remote/xtream/OkHttpXtreamApiServiceTest.kt@@ -19,6 +19,9 @@ import kotlinx.coroutines.launchimport kotlinx.coroutines.withContextimport kotlinx.coroutines.withTimeoutimport kotlinx.coroutines.test.runTest+import kotlinx.coroutines.runBlocking+import com.streamvault.data.sync.runXtreamLivePipeline+import com.streamvault.data.sync.ProviderWorkflowSupersededExceptionimport kotlinx.serialization.json.Jsonimport okhttp3.Interceptorimport okhttp3.MediaType.Companion.toMediaType@@ -34,6 +37,72 @@ import org.junit.Testclass OkHttpXtreamApiServiceTest {+ @Test+ fun `pipeline writer supersession cancels a body blocked after its first row`() = runBlocking {+ withTimeout(5_000) {+ val bodyBlocked = CountDownLatch(1)+ val cancellationObserved = CountDownLatch(1)+ val prefix = ("""[{"stream_id":1,"name":"One"},""" + " ".repeat(16_384)).toByteArray()+ val client = OkHttpClient.Builder().addInterceptor { chain ->+ val source = object : InputStream() {+ var offset = 0+ override fun read(): Int {+ if (offset < prefix.size) return prefix[offset++].toInt()+ bodyBlocked.countDown()+ while (!chain.call().isCanceled()) Thread.sleep(10)+ cancellationObserved.countDown()+ throw IOException("Canceled body read")+ }+ }.source().buffer()+ Response.Builder().request(chain.request()).protocol(Protocol.HTTP_1_1).code(200).message("OK")+ .body(object : ResponseBody() {+ override fun contentType() = "application/json".toMediaType()+ override fun contentLength() = -1L+ override fun source() = source+ }).build()+ }.build()+ val service = OkHttpXtreamApiService(client, json)+ val failure = runCatching {+ runXtreamLivePipeline(1, 4096, 1, { _: com.streamvault.data.remote.dto.XtreamLiveStreamRow -> 256L },+ read = { emit -> service.streamLiveStreamRows("https://example.test/player_api.php?action=get_live_streams", onItem = emit) },+ consume = {+ check(bodyBlocked.await(4, TimeUnit.SECONDS))+ throw ProviderWorkflowSupersededException(1, 1)+ })+ }.exceptionOrNull()+ assertThat(failure).isInstanceOf(ProviderWorkflowSupersededException::class.java)+ assertThat(cancellationObserved.count).isEqualTo(0)+ }+ }++ @Test+ fun `streamed live rows reject huge tokens before full string allocation`() = runTest {+ val prefix = """[{"stream_id":1,"name":"""".toByteArray()+ val returned = AtomicInteger()+ val source = object : InputStream() {+ var offset = 0+ override fun read(): Int {+ returned.incrementAndGet()+ return if (offset < prefix.size) prefix[offset++].toInt() else 'x'.code+ }+ }.source().buffer()+ val client = OkHttpClient.Builder().addInterceptor { chain ->+ Response.Builder().request(chain.request()).protocol(Protocol.HTTP_1_1).code(200).message("OK")+ .body(object : ResponseBody() {+ override fun contentType() = "application/json".toMediaType()+ override fun contentLength() = -1L+ override fun source() = source+ }).build()+ }.build()+ val failure = runCatching {+ OkHttpXtreamApiService(client, json).streamLiveStreamRows(+ "https://example.test/player_api.php?action=get_live_streams") { error("oversized row emitted") }+ }.exceptionOrNull()+ assertThat(failure).isInstanceOf(XtreamLiveRowTooLargeException::class.java)+ // Includes Okio/InputStreamReader/JsonReader read-ahead, not a 96 MB response.+ assertThat(returned.get()).isAtMost(XtreamLiveRowBudgetReader.MAX_ROW_CHARS + 32 * 1024)+ }+private val json = Json {ignoreUnknownKeys = trueisLenient = truediff --git a/data/src/test/java/com/streamvault/data/sync/XtreamLiveAttemptIsolationTest.kt b/data/src/test/java/com/streamvault/data/sync/XtreamLiveAttemptIsolationTest.ktindex 5bd7fd52..6eb8e262 100644--- a/data/src/test/java/com/streamvault/data/sync/XtreamLiveAttemptIsolationTest.kt+++ b/data/src/test/java/com/streamvault/data/sync/XtreamLiveAttemptIsolationTest.kt@@ -13,6 +13,9 @@ import com.streamvault.data.remote.xtream.XtreamProviderimport com.streamvault.domain.model.ContentTypeimport com.streamvault.domain.model.LegacyProviderimport com.streamvault.domain.model.ProviderType+import com.streamvault.domain.model.SyncMetadata+import java.util.concurrent.CountDownLatch+import java.util.concurrent.TimeUnitimport java.util.concurrent.atomic.AtomicIntegerimport kotlinx.coroutines.CancellationExceptionimport kotlinx.coroutines.runBlocking@@ -68,6 +71,54 @@ class XtreamLiveAttemptIsolationTest {assertThat(totalStages()).isEqualTo(2)}+ @Test+ fun `cold metadata and bulk requests overlap and adult context is ready before staging`() = runBlocking<Unit> {+ val bulkStarted = CountDownLatch(1)+ val metadataRequests = AtomicInteger()+ val bulkRequests = AtomicInteger()+ val (strategy, api) = engine(primeCategories = false, categoryResponse = {+ metadataRequests.incrementAndGet()+ check(bulkStarted.await(5, TimeUnit.SECONDS)) { "bulk did not overlap metadata" }+ """[{"category_id":"1","category_name":"One","is_adult":1}]"""+ }, beforeStage = { channels -> assertThat(channels.all { it.isAdult }).isTrue() }) {+ bulkRequests.incrementAndGet(); bulkStarted.countDown(); array(row(101), row(102), row(103))+ }+ val result = strategy.syncXtreamLiveCatalog(provider, api, SyncMetadata(provider.id), emptySet(), null, profile(), true)+ assertThat(result.catalogResult).isInstanceOf(CatalogStrategyResult.Success::class.java)+ assertThat(result.stagedAcceptedCount).isEqualTo(3)+ assertThat(metadataRequests.get()).isEqualTo(1)+ assertThat(bulkRequests.get()).isEqualTo(1)+ assertThat(db.catalogSyncDao().getChannelStages(provider.id, requireNotNull(result.stagedSessionId)).all { it.isAdult }).isTrue()+ }++ @Test+ fun `metadata failure is resolved once instead of fetched again by every batch`() = runBlocking<Unit> {+ val metadataRequests = AtomicInteger()+ val (strategy, api) = engine(primeCategories = false, categoryStatus = 400,+ categoryResponse = { metadataRequests.incrementAndGet(); "{}" }) {+ array(row(101), row(102), row(103))+ }+ val result = strategy.syncXtreamLiveCatalog(provider, api, SyncMetadata(provider.id), emptySet(), null, profile(), true)+ assertThat(result.catalogResult).isInstanceOf(CatalogStrategyResult.Success::class.java)+ assertThat(result.stagedAcceptedCount).isEqualTo(3)+ assertThat(metadataRequests.get()).isEqualTo(1)+ assertThat(result.categories).isNotEmpty()+ }++ @Test+ fun `stressed provider resolves metadata before requesting bulk without deadlock`() = runBlocking<Unit> {+ val policy = XtreamAdaptiveSyncPolicy()+ policy.recordFailure(provider.id, java.io.IOException("connection reset"))+ val bulkRequests = AtomicInteger()+ val (strategy, api) = engine(policy = policy, primeCategories = false, categoryResponse = {+ assertThat(bulkRequests.get()).isEqualTo(0)+ """[{"category_id":"1","category_name":"One"}]"""+ }) { bulkRequests.incrementAndGet(); array(row(101), row(102)) }+ val result = strategy.syncXtreamLiveCatalog(provider, api, SyncMetadata(provider.id), emptySet(), null, profile(), true)+ assertThat(result.stagedAcceptedCount).isEqualTo(2)+ assertThat(bulkRequests.get()).isEqualTo(1)+ }+@Testfun `failed category decoder attempt cannot overwrite successful category or retain partial rows`() = runBlocking<Unit> {val attempts = AtomicInteger()@@ -190,14 +241,18 @@ class XtreamLiveAttemptIsolationTest {private fun categories() = listOf(XtreamCategory("1", "One"), XtreamCategory("2", "Two"))private suspend fun engine(failWriter: Boolean = false, failOnStreamId: Long? = null,+ primeCategories: Boolean = true, categoryStatus: Int = 200, categoryResponse: (() -> String)? = null,+ policy: XtreamAdaptiveSyncPolicy = XtreamAdaptiveSyncPolicy(),+ beforeStage: suspend (List<com.streamvault.domain.model.Channel>) -> Unit = {},response: (String?) -> String): Pair<SyncManagerXtreamLiveStrategy, XtreamProvider> {val http = OkHttpXtreamApiService(OkHttpClient.Builder().addInterceptor { chain ->- Response.Builder().request(chain.request()).protocol(Protocol.HTTP_1_1).code(200).message("OK")- .body(response(chain.request().url.queryParameter("category_id")).toResponseBody("application/json".toMediaType())).build()+ val isCategories = chain.request().url.queryParameter("action") == "get_live_categories"+ val body = if (isCategories && categoryResponse != null) categoryResponse() else response(chain.request().url.queryParameter("category_id"))+ Response.Builder().request(chain.request()).protocol(Protocol.HTTP_1_1).code(if (isCategories) categoryStatus else 200).message("OK")+ .body(body.toResponseBody("application/json".toMediaType())).build()}.build(), Json { ignoreUnknownKeys = true; isLenient = true; explicitNulls = false; coerceInputValues = true })val api = XtreamProvider(provider.id, http, provider.serverUrl, provider.username, provider.password)- api.mapCategories(ContentType.LIVE, categories())- val policy = XtreamAdaptiveSyncPolicy()+ if (primeCategories) api.mapCategories(ContentType.LIVE, categories())val progress: (Long, ((String) -> Unit)?, String) -> Unit = { _, callback, label -> callback?.invoke(label) }val sanitize: (Throwable?) -> String = { it?.javaClass?.simpleName.orEmpty() }val support = SyncManagerXtreamSupport(policy, { false }, sanitize, progress, 60_000, 60_000, "stopped")@@ -205,6 +260,7 @@ class XtreamLiveAttemptIsolationTest {return SyncManagerXtreamLiveStrategy(http, http, policy, support, SyncManagerXtreamFetcher(http, http, support, sanitize),SyncManagerCatalogStrategySupport({ false }, 60_000), store, progress, sanitize,{ _, _ -> "full failed" }, { _, _, _ -> "category failed" }, "sequential", stageChannelItems = { id, channels, seen, collector, session ->+ beforeStage(channels)if (failWriter || channels.any { it.streamId == failOnStreamId }) throw ProviderWorkflowSupersededException(id, 1)stager.stageChannelItems(id, channels, seen, collector, session)}, emitProgress = { _, _ -> }) to apidiff --git a/data/src/test/java/com/streamvault/data/sync/XtreamLivePipelineTest.kt b/data/src/test/java/com/streamvault/data/sync/XtreamLivePipelineTest.ktnew file mode 100644index 00000000..0bef1e20--- /dev/null+++ b/data/src/test/java/com/streamvault/data/sync/XtreamLivePipelineTest.kt@@ -0,0 +1,198 @@+package com.streamvault.data.sync++import com.google.common.truth.Truth.assertThat+import java.io.IOException+import java.util.concurrent.atomic.AtomicInteger+import kotlinx.coroutines.CancellationException+import kotlinx.coroutines.CompletableDeferred+import kotlinx.coroutines.NonCancellable+import kotlinx.coroutines.async+import kotlinx.coroutines.awaitCancellation+import kotlinx.coroutines.cancelAndJoin+import kotlinx.coroutines.runBlocking+import kotlinx.coroutines.withContext+import kotlinx.coroutines.withTimeout+import org.junit.Test++class XtreamLivePipelineTest {+ @Test+ fun `provider stress reduces metadata and bulk concurrency`() {+ val policy = XtreamAdaptiveSyncPolicy()+ assertThat(policy.liveRequestConcurrencyFor(1)).isEqualTo(2)+ policy.recordFailure(1, IOException("connection reset"))+ assertThat(policy.liveRequestConcurrencyFor(1)).isEqualTo(1)+ assertThat(policy.liveRequestConcurrencyFor(2)).isEqualTo(2)+ policy.forgetProvider(1)+ assertThat(policy.liveRequestConcurrencyFor(1)).isEqualTo(2)+ }++ @Test+ fun `different providers share one bulk pipeline permit`() = runBlocking {+ withTimeout(5_000) {+ val policy = XtreamAdaptiveSyncPolicy()+ val firstEntered = CompletableDeferred<Unit>()+ val release = CompletableDeferred<Unit>()+ val secondAttempting = CompletableDeferred<Unit>()+ val secondEntered = CompletableDeferred<Unit>()+ val first = async { policy.withLivePipelinePermit { firstEntered.complete(Unit); release.await() } }+ firstEntered.await()+ val second = async {+ secondAttempting.complete(Unit)+ policy.withLivePipelinePermit { secondEntered.complete(Unit) }+ }+ secondAttempting.await()+ assertThat(secondEntered.isCompleted).isFalse()+ release.complete(Unit)+ first.await(); second.await()+ assertThat(secondEntered.isCompleted).isTrue()+ }+ }++ @Test+ fun `reader overlaps writer but cannot run beyond queue capacity`() = runBlocking {+ withTimeout(5_000) {+ val writing = CompletableDeferred<Unit>()+ val thirdRead = CompletableDeferred<Unit>()+ val release = CompletableDeferred<Unit>()+ val emitted = AtomicInteger()+ val finished = AtomicInteger()+ val result = async {+ runXtreamLivePipeline(1, 10, 1, { 10L }, read = { emit ->+ repeat(6) { index ->+ emitted.incrementAndGet()+ if (index == 2) thirdRead.complete(Unit)+ emit(index)+ finished.incrementAndGet()+ }; 6+ }, consume = { writing.complete(Unit); release.await() })+ }+ writing.await()+ thirdRead.await()+ assertThat(emitted.get()).isAtMost(4) // writer + queue + suspended send + next row+ assertThat(finished.get()).isAtMost(2)+ release.complete(Unit)+ val stats = result.await()+ assertThat(stats.rows).isEqualTo(6)+ assertThat(stats.outstandingBatchBytes).isAtMost(30)+ }+ }++ @Test+ fun `metadata wait fills bounded queue and parent cancellation stops reader`() = runBlocking {+ withTimeout(5_000) {+ val thirdRead = CompletableDeferred<Unit>()+ val stopped = CompletableDeferred<Unit>()+ val context = CompletableDeferred<Unit>()+ val finished = AtomicInteger()+ val result = async {+ runXtreamLivePipeline(1, 10, 1, { 10L }, prepare = { context.await() }, read = { emit ->+ try {+ repeat(10) { index ->+ if (index == 2) thirdRead.complete(Unit)+ emit(index); finished.incrementAndGet()+ }; 10+ } finally { stopped.complete(Unit) }+ }, consume = { error("mapping before metadata") })+ }+ // With preparation blocked, even the second send must wait.+ while (finished.get() == 0) kotlinx.coroutines.yield()+ assertThat(finished.get()).isEqualTo(1)+ assertThat(thirdRead.isCompleted).isFalse()+ result.cancelAndJoin()+ stopped.await()+ }+ }++ @Test+ fun `completion waits for final batch writer drain`() = runBlocking {+ withTimeout(5_000) {+ val eof = CompletableDeferred<Unit>()+ val writing = CompletableDeferred<Unit>()+ val release = CompletableDeferred<Unit>()+ val result = async {+ runXtreamLivePipeline(10, 100, 1, { 1L }, read = { emit ->+ emit(1); eof.complete(Unit); 1+ }, consume = { writing.complete(Unit); release.await() })+ }+ eof.await(); writing.await()+ assertThat(result.isCompleted).isFalse()+ release.complete(Unit)+ assertThat(result.await().rows).isEqualTo(1)+ }+ }++ @Test+ fun `reader failure joins cancelled writer before returning for retry`() = runBlocking {+ withTimeout(5_000) {+ val writing = CompletableDeferred<Unit>()+ val cancelling = CompletableDeferred<Unit>()+ val releaseCleanup = CompletableDeferred<Unit>()+ val result = async {+ runCatching {+ runXtreamLivePipeline(1, 100, 1, { 1L }, read = { emit ->+ emit(1); writing.await(); throw IOException("truncated")+ }, consume = {+ writing.complete(Unit)+ try { awaitCancellation() } finally {+ withContext(NonCancellable) { cancelling.complete(Unit); releaseCleanup.await() }+ }+ })+ }+ }+ cancelling.await()+ assertThat(result.isCompleted).isFalse()+ releaseCleanup.complete(Unit)+ assertThat(result.await().exceptionOrNull()).isInstanceOf(IOException::class.java)+ }+ }++ @Test+ fun `superseded writer cancels reader even when reader is waiting on transport`() = runBlocking {+ withTimeout(5_000) {+ val reading = CompletableDeferred<Unit>()+ val stopped = CompletableDeferred<Unit>()+ val failure = runCatching {+ runXtreamLivePipeline(1, 100, 1, { 1L }, read = { emit ->+ try { emit(1); reading.complete(Unit); awaitCancellation() }+ finally { stopped.complete(Unit) }+ }, consume = { reading.await(); throw ProviderWorkflowSupersededException(1, 1) })+ }.exceptionOrNull()+ stopped.await()+ assertThat(failure).isInstanceOf(ProviderWorkflowSupersededException::class.java)+ }+ }++ @Test+ fun `wide rows flush by bytes and preserve order`() = runBlocking {+ val batches = mutableListOf<List<Int>>()+ val stats = runXtreamLivePipeline(100, 100, 2, { row: Int -> row.toLong() }, read = { emit ->+ listOf(30, 60, 70, 20, 10).forEach { emit(it) }; 5+ }, consume = { batches.add(it) })+ assertThat(batches).containsExactly(listOf(30, 60), listOf(70, 20, 10)).inOrder()+ assertThat(stats.batches).isEqualTo(2)+ assertThat(stats.outstandingBatchBytes).isAtMost(400)+ }++ @Test+ fun `oversized row is rejected without silent trimming`() = runBlocking {+ val failure = runCatching {+ runXtreamLivePipeline(100, 100, 1, { row: Int -> row.toLong() },+ read = { emit -> emit(101); 1 }, consume = { error("oversized write") })+ }.exceptionOrNull()+ assertThat(failure).isInstanceOf(XtreamLiveBufferLimitException::class.java)+ }++ @Test+ fun `ordinary writer failure stops producer`() = runBlocking {+ withTimeout(5_000) {+ val stopped = CompletableDeferred<Unit>()+ val failure = runCatching {+ runXtreamLivePipeline(1, 100, 1, { 1L }, read = { emit ->+ try { emit(1); awaitCancellation() } finally { stopped.complete(Unit) }+ }, consume = { throw IllegalStateException("write failed") })+ }.exceptionOrNull()+ stopped.await()+ assertThat(failure).hasMessageThat().isEqualTo("write failed")+ }+ }+}diff --git a/docs/XTREAM_SYNC_BENCHMARK.md b/docs/XTREAM_SYNC_BENCHMARK.mdindex 3cf1ab7b..c185b240 100644--- a/docs/XTREAM_SYNC_BENCHMARK.md+++ b/docs/XTREAM_SYNC_BENCHMARK.md@@ -62,6 +62,12 @@ rebuild, now using a monotonic clock. The runner retains only structuredbenchmark data and explicitly recognized numeric timing lines. It neverexports general provider logs, request URLs, response previews or passwords.+The P3 bulk pipeline adds numeric `live-pipeline` records for batch counts,+reader duration, final writer drain, send/backpressure time and estimated+outstanding raw batch bytes. See the+[pipeline report](XTREAM_SYNC_PIPELINE_IMPLEMENTATION.md) for resource accounting+and overlap semantics. These durations overlap and are not CPU measurements.+Java heap is sampled every 50 ms, with ART GC count deltas where available.These are debug-emulator measurements, not release-device performance, nativeRSS or an allocation profile. A short heap peak may fall between samples.@@ -103,8 +109,9 @@ This package is an initial backend baseline, not the full P0 gate. Follow-upwork still needs precise headers/first-row and workflow-wait timings, attemptand generation identifiers, row identity/field equivalence beyond counts,skipped/duplicate/hidden coverage, parser CPU/allocation profiling, auxiliary-index readiness, and a first-browsable-screen measurement. Queue occupancy and-writer-drain timings apply once the bounded pipeline exists. Run the full+index readiness, and a first-browsable-screen measurement. Pipeline byte+high-water and writer-drain timings now exist; actual heap/allocation attribution+and queue occupancy over time remain open. Run the fullfive-sample scenario matrix and representative release-device measurementsbefore freezing production tuning or claiming a speedup. Playback validationis separate and follows AGENTS.md's sustained capture/log requirements.diff --git a/docs/XTREAM_SYNC_IMPROVEMENT_PLAN.md b/docs/XTREAM_SYNC_IMPROVEMENT_PLAN.mdindex 6b6ed669..ba89e5df 100644--- a/docs/XTREAM_SYNC_IMPROVEMENT_PLAN.md+++ b/docs/XTREAM_SYNC_IMPROVEMENT_PLAN.md@@ -10,6 +10,7 @@ Supporting evidence:- [Initial backend baseline](XTREAM_SYNC_INITIAL_BASELINE.md)- [Attempt isolation and recovery implementation](XTREAM_SYNC_RECOVERY_IMPLEMENTATION.md)- [Live parsing and group identity implementation](XTREAM_SYNC_PARSING_IMPLEMENTATION.md)+- [Bounded live pipeline implementation](XTREAM_SYNC_PIPELINE_IMPLEMENTATION.md)The reference implementation provides hypotheses to investigate, not a designauthority or proof of correctness. Choose behavior for this app's measured@@ -279,14 +280,17 @@ P6 is the first live performance release milestone; P7 follows it. Keep actualproduction AUTO behavior unchanged until the P6 gate. Experiments before thenuse explicit STREAM_ALL or debug configuration, with identical data coverage.-Implementation progress on 2026-10-09: P0 has an initial backend baseline and+Implementation progress on 2026-10-10: P0 has an initial backend baseline andisolated fixture; its complete release-device/readiness comparison remains open.P1 attempt isolation and conservative receipt-based recovery are implemented andvalidated. P2 direct live decoding and cheaper group identity are implemented-with field-equivalence checks and measured component CPU reductions. P3–P8-remain pending. Repeated 10,000-row runs still show memory-driven category-fallback, so the default-strategy gate has not been met. See the implementation-reports linked above for exact evidence and limits.+with field-equivalence checks and measured component CPU reductions. P3 retrieval+overlap and a bounded reader/writer pipeline are implemented with cancellation,+resource-bound and real-graph setup/refresh evidence. P4–P8 remain pending.+The new measurements identify writer cost, slow fallback and extra completed+imports during some refresh intervals as remaining priorities. Earlier runs+also showed memory-driven category fallback; the default-strategy gate has not+been met. See the implementation reports linked above for evidence and limits.## P0 measurement and fixture deliverablesdiff --git a/docs/XTREAM_SYNC_PIPELINE_IMPLEMENTATION.md b/docs/XTREAM_SYNC_PIPELINE_IMPLEMENTATION.mdnew file mode 100644index 00000000..ec0588eb--- /dev/null+++ b/docs/XTREAM_SYNC_PIPELINE_IMPLEMENTATION.md@@ -0,0 +1,162 @@+# Xtream live import pipeline++P3 implementation on `codex/xtream-live-sync-performance`, following the+[improvement plan](XTREAM_SYNC_IMPROVEMENT_PLAN.md). This package changes bulk+live ingestion used by provider setup and Settings Live refresh. AUTO selection+remains behind the P6 gate. Category mode and automatic fallback remain available;+their worker scheduling and pacing improvements belong to P5.++## Retrieval and ownership++For a healthy provider, live category metadata and one bulk live response start+concurrently. Both use the existing provider backoff/retry policy. A provider+with remembered request stress gets one local request slot: metadata resolves+first, then bulk retrieval begins. There are no parallel downloads of the same+bulk endpoint.++The reader sends typed batches through a bounded coroutine channel. A single+consumer on `Dispatchers.Default` waits for category context once, then maps and+stages rows while the HTTP reader continues. Metadata failure explicitly primes+empty adult-category context and retains fallback category semantics. Mapping+does not issue another category request for each batch. Authoritative category+shells, adult flags, hidden/required scope and stream ordering retain their+existing semantics.++Only the consumer owns the staging session, duplicate set, fallback categories,+accepted count and write timings. Reader success requires valid END_DOCUMENT.+Import success also requires channel drain and completion of the final write;+the P1 completion receipt is sealed only after that return.++Reader failure cancels and joins the writer before discarding its attempt.+Writer failure cancels the reader and its HTTP call. Workflow supersession is a+CancellationException, which normally would only cancel the child; the consumer+explicitly cancels its enclosing pipeline scope too. Parent cancellation closes+the queue/response and joins children. Retry gets a fresh queue and the existing+fresh attempt state. No late writer can write into an already discarded session.++## Resource limits++| Tier | Existing row batch limit | Estimated raw batch bytes | Queued batches |+| --- | ---: | ---: | ---: |+| LOW | 100 | 256 KiB | 1 |+| MID | 300 | 512 KiB | 2 |+| HIGH | 500 | 1 MiB | 2 |++These preserve the current row limits; larger tuning candidates remain unselected.+A byte threshold can flush earlier. A row larger than its batch budget fails+explicitly instead of being trimmed or admitted into an oversized queue.+The estimate includes UTF-16 text, category-ID strings and conservative fixed+object/list overhead. It is an accounting limit, not an exact ART heap reading.++At most `queue capacity + 2` full batches are retained by the writer, queue and+suspended sender. While assembling a batch, the producer may also hold its next+decoded row: use `(queue capacity + 3) * batch bytes` as a conservative raw-row+budget. Mapped channels/entities, parser buffers, metadata, duplicate tracking,+Room and indexes are additional allocations.++Live streamed JSON rows also have a 1,048,576-character token-read budget, plus+up to JsonReader's 1,024-character read-ahead. This applies to thin and legacy+streamed live decoders, including category responses, and includes skipped+metadata. It rejects pathological fields before building an entire huge token+or JSON tree. This is a new conservative resource admission rule: large ignored+metadata can also exceed it. No string is silently truncated. Resource failures+do not trigger the more expensive legacy decoder.++The existing 100,000-channel store cap bounds duplicate tracking to that count+plus one incoming batch before rejection. Existing HTTP response caps bound+metadata/response input. Fallback category accumulation is bounded by admitted+rows. P4 will evaluate stage uniqueness instead of the in-memory duplicate set.++The shared policy in singleton SyncManager permits one bulk pipeline across all+providers, so simultaneous imports cannot multiply these queues. Category and+other section jobs still follow their existing concurrency rules; this is not+a general app-wide network or memory scheduler. Live low-memory admission and+runtime abort signals remain active.++## Evidence++Barrier-based tests demonstrate metadata/bulk overlap, adult-context readiness,+one metadata resolution after failure, byte flushing, bounded queue backpressure,+final writer drain and parent/reader/writer cancellation. A transport test+supersedes the writer while the HTTP body is blocked after the first row and+observes underlying Call cancellation. Room attempt-isolation, retry, partial+fallback, supersession and sealed snapshot tests remain part of the regression+suite. Request-stress serialization and shared bulk permits are tested too.++Final regression run: 232 data tests and 16 domain identity tests passed;+both isolated benchmark APKs built. The fixture/report suite passed 14 tests.+The benchmark package is `com.streamvault.app.syncbenchmark`, separate from+ordinary app storage. No user app data was cleared.++Android TV API 36 x86_64 emulator: memory class 192 MB, max Java heap+201,326,592 bytes. Production LOW row batches (100) and one queued batch were+used. Background work stayed enabled. All 24 setup/refresh samples below+(including two warmup samples) retained their full expected channel count and+passed account cleanup, instrumentation, completion/sample-count checks and+untruncated request history.++| Scenario | Initial | Settings Live refresh | Evidence |+| --- | ---: | ---: | --- |+| 2k rows, 10 categories, all modes | STREAM_ALL 1,018 ms | STREAM_ALL 937 ms | Six samples; AUTO/category about 2.5 s |+| 10k rows, 100 categories, STREAM_ALL | Median 4,246 ms; max 4,289 ms | Median 4,306 ms; max 8,290 ms | One warmup + five measured pairs |+| 2k rows, category metadata delayed 1.5 s | 2,725 ms | 2,509 ms | Single pair; bulk begins during metadata wait |+| 2k rows, bulk returns HTTP 503 | 10,523 ms | 16,805 ms | Single pair; complete category fallback |+| 50k rows, 200 categories, STREAM_ALL | 21,090 ms | 40,451 ms | Single pair; three completed bulk imports |++Reports under ignored `build/xtream-sync-performance/`:++- All-mode smoke: `20261010T063839794250Z/report.json`.+- Repeated 10k: `20261010T063927249264Z/report.json`.+- Delayed metadata: `20261010T064037889878Z/report.json`.+- Bulk rejection: `20261010T064106880909Z/report.json`.+- Large catalog: `20261010T064201982860Z/report.json`.++The delayed-metadata request log proves bulk started 2.994 ms after metadata+started, before its 1,500.719 ms response completed. Request high-water was two.+Each synthetic account in the healthy small run issued two bulk requests+(setup + refresh) and no category stream requests. AUTO and explicit category+mode still issued 20 category requests across each account's two phases.++For the repeated 10k run, maximum estimated outstanding batch bytes were+178,154; maximum final drain was 86 ms. Median send time was 3,797 ms: the+reader mostly waited for mapping/staging. Maximum measured sampled Java heap+was 12,964,864 bytes. Thirteen completed bulk responses were observed for+twelve setup/refresh samples; one measured account incurred an extra full+import during its refresh interval, explaining the approximately doubled tail.+The 50k run similarly had three completed responses for two samples, with+maximum outstanding batch bytes 179,184 and drain 85 ms. Peak sampled Java+heap was 59,484,816 bytes on its cold initial sample, falling to 14,574,624+on refresh. These samples do not isolate all sources of heap allocation.++The extra completed imports are not evidence of a partial-body transport retry.+Background scheduling/coalescing needs investigation under P6/P7 before making+latency guarantees. The bulk-rejection account also executed additional import+work: nine rejected bulk requests and 30 category requests across its lifecycle.+Fallback is correct but still slow; P5 pacing and duplicate-work handling remain+necessary.++The 10k payload checksum matches P0/P2. P2's nominal bulk-only initial/refresh+medians were 4,489/4,566 ms (only two samples per phase); its mixed-strategy tail+included low-memory category fallback. P3 shows a modest nominal reduction,+not a controlled speedup ratio. The emulator restarted before P3 and background+timing differed. No category fallback appeared in the healthy P3 bulk runs,+which does not prove that memory pressure was fixed. AUTO's release gate+remains unmet.++The evidence prioritizes P4 writer/mapping and database cost, followed by P5+category fallback. Include redundant completed live imports in the P6/P7+scheduling audit. The fixture supplies no playable media. Playback acceptance,+screen readiness and release-device results remain P8/P0 gates.++## Timing interpretation++`XtreamSyncTiming phase=live-pipeline` records rows, batches, reader elapsed time,+post-reader drain, time spent sending batches, maximum estimated outstanding+batch bytes, queue capacity and byte limit. Outstanding bytes include the+consumer and suspended sender, not just the queue. They exclude the next row+and a producer batch that has not reached send.++Send time includes scheduling/channel overhead as well as backpressure. Reader+wall time includes that waiting. Mapping/staging and response durations overlap;+do not add them together or interpret them as CPU time. The benchmark exporter+accepts only the exact numeric format and discards general logs.diff --git a/tools/tests/test_xtream_sync_benchmark.py b/tools/tests/test_xtream_sync_benchmark.pyindex 8bfb4b67..f98d8897 100644--- a/tools/tests/test_xtream_sync_benchmark.py+++ b/tools/tests/test_xtream_sync_benchmark.py@@ -11,6 +11,18 @@ SPEC.loader.exec_module(BENCHMARK)class BenchmarkReportTest(unittest.TestCase):+ def test_pipeline_metrics_are_numeric_and_exact_format_only(self):+ log = '\n'.join([+ 'I/XtreamSyncTiming( 123): phase=live-pipeline rows=100 batches=10 reader_ms=20 drain_ms=3 send_wait_ms=8 outstanding_batch_bytes=1024 queue_capacity=1 batch_byte_limit=4096',+ 'I/XtreamSyncTiming( 123): phase=live-pipeline rows=100 secret=private',+ ])+ records = BENCHMARK.parse_records(log)+ self.assertEqual(1, len(records))+ self.assertEqual(1024, records[0]["outstanding_batch_bytes"])+ self.assertEqual(8, records[0]["send_wait_ms"])+ self.assertEqual(records, BENCHMARK.summarize(records, [], {})["timings"])+ self.assertNotIn("private", json.dumps(records))+def test_only_benchmark_records_are_retained(self):log = '\n'.join(['I/OtherTag( 123): password=private',diff --git a/tools/xtream_sync_benchmark.py b/tools/xtream_sync_benchmark.pyindex a871ead3..04aa9603 100644--- a/tools/xtream_sync_benchmark.py+++ b/tools/xtream_sync_benchmark.py@@ -38,6 +38,12 @@ def parse_records(log: str) -> list[dict]:records.append({"event": "live-response", "elapsed_ms": int(timing[1]),"callback_ms": int(timing[2]), "rows": int(timing[3]),"completed": timing[4] == "true"})+ pipeline = re.search(r"\bXtreamSyncTiming\s*\([^)]*\): phase=live-pipeline rows=(\d+) batches=(\d+) reader_ms=(\d+) drain_ms=(\d+) send_wait_ms=(\d+) outstanding_batch_bytes=(\d+) queue_capacity=(\d+) batch_byte_limit=(\d+)$", line)+ if pipeline:+ records.append(dict(event="live-pipeline", **dict(zip(+ ["rows", "batches", "reader_ms", "drain_ms", "send_wait_ms",+ "outstanding_batch_bytes", "queue_capacity", "batch_byte_limit"],+ map(int, pipeline.groups())))))stage = re.search(r"\bSyncCatalogStore\s*\([^)]*\): catalog timing provider=(\d+) stage=([a-z-]+(?:\(\d+\))?) took=(\d+)ms$", line)if stage:records.append({"event": "catalog-stage", "provider_id": int(stage[1]),@@ -71,7 +77,7 @@ def summarize(records: list[dict], events: list[dict], metadata: dict) -> dict:"channel_counts": sorted({s["channels"] for s in selected})})return {"fixture": metadata, "environment": next((r for r in records if r["event"] == "environment"), None),"complete": any(r["event"] == "complete" for r in records), "groups": groups, "samples": samples,- "timings": [r for r in records if r["event"] in {"live-response", "catalog-stage"}],+ "timings": [r for r in records if r["event"] in {"live-response", "live-pipeline", "catalog-stage"}],"limits": ["Debug APK, synthetic local server, Java heap sampled every 50 ms.","Backend completion only; no UI readiness, native RSS, or playback measurement.","Refresh shares an account with setup; account request counts include both phases.",