Skip to content

StreamVault is a TV-first free IPTV player for Android TV built with Kotlin, Jetpack Compose, Room, Hilt, and Media3.

Pipeline Xtream live retrieval and staging with bounded buffers

David Nashash5b0d43c28bb1
diff --git a/data/src/main/java/com/streamvault/data/remote/xtream/OkHttpXtreamApiService.kt b/data/src/main/java/com/streamvault/data/remote/xtream/OkHttpXtreamApiService.kt
index b503a15c..c0227e58 100644
--- a/data/src/main/java/com/streamvault/data/remote/xtream/OkHttpXtreamApiService.kt
+++ b/data/src/main/java/com/streamvault/data/remote/xtream/OkHttpXtreamApiService.kt
@@ -537,7 +537,10 @@ class OkHttpXtreamApiService(
)?.let { throw it }
stream.unread(previewBytes)
- val reader = JsonReader(InputStreamReader(stream, charset))
+ val rowBudget = if (descriptor.action == "get_live_streams") {
+ XtreamLiveRowBudgetReader(InputStreamReader(stream, charset))
+ } else null
+ val reader = JsonReader(rowBudget ?: InputStreamReader(stream, charset))
reader.isLenient = true
return when (reader.peek()) {
JsonToken.BEGIN_ARRAY -> {
@@ -545,16 +548,21 @@ class OkHttpXtreamApiService(
reader.beginArray()
while (reader.hasNext()) {
currentCoroutineContext().ensureActive()
+ rowBudget?.beginRow()
val item = try {
if (readItem != null) readItem(reader) else {
val element = JsonParser.parseReader(reader)
json.decodeFromString(deserializer, element.toString())
}
+ } catch (e: XtreamLiveRowTooLargeException) {
+ throw e
} catch (e: RuntimeException) {
throw XtreamParsingException(
"Malformed JSON from ${descriptor.hint}${sanitizedPreview(preview)?.let { " (preview=$it)" } ?: ""}",
e
)
+ } finally {
+ rowBudget?.endRow()
}
onItem(item)
emittedCount++
diff --git a/data/src/main/java/com/streamvault/data/remote/xtream/XtreamLiveRowBudgetReader.kt b/data/src/main/java/com/streamvault/data/remote/xtream/XtreamLiveRowBudgetReader.kt
new file mode 100644
index 00000000..b752aeb8
--- /dev/null
+++ b/data/src/main/java/com/streamvault/data/remote/xtream/XtreamLiveRowBudgetReader.kt
@@ -0,0 +1,33 @@
+package com.streamvault.data.remote.xtream
+
+import java.io.Reader
+import com.streamvault.domain.provider.ProviderSetupFailureKind
+
+/** Bounds token allocation before JsonReader can build a very large string/tree. */
+internal class XtreamLiveRowBudgetReader(private val delegate: Reader) : Reader() {
+ private var remaining: Int? = null
+
+ fun beginRow() { remaining = MAX_ROW_CHARS }
+ fun endRow() { remaining = null }
+
+ override fun read(buffer: CharArray, offset: Int, length: Int): Int {
+ if (length == 0) return 0
+ val budget = remaining
+ if (budget != null && budget <= 0) throw XtreamLiveRowTooLargeException()
+ val read = delegate.read(buffer, offset, minOf(length, budget ?: length))
+ if (read > 0 && budget != null) remaining = budget - read
+ return read
+ }
+
+ override fun close() = delegate.close()
+
+ companion object {
+ // A JSON row (including skipped metadata) may retain up to JsonReader's
+ // 1024-character read-ahead in addition to this budget. No silent trim.
+ const val MAX_ROW_CHARS = 1024 * 1024
+ }
+}
+
+class XtreamLiveRowTooLargeException : XtreamApiException("Live JSON row exceeds token allocation budget") {
+ override val kind = ProviderSetupFailureKind.RESPONSE_TOO_LARGE
+}
diff --git a/data/src/main/java/com/streamvault/data/sync/CatalogSyncRuntimeProfile.kt b/data/src/main/java/com/streamvault/data/sync/CatalogSyncRuntimeProfile.kt
index d3e623e4..0a98c025 100644
--- a/data/src/main/java/com/streamvault/data/sync/CatalogSyncRuntimeProfile.kt
+++ b/data/src/main/java/com/streamvault/data/sync/CatalogSyncRuntimeProfile.kt
@@ -58,6 +58,15 @@ internal data class CatalogSyncRuntimeProfile(
val deferBackgroundWorkOnLowMemory: Boolean,
val snapshot: SyncDeviceSnapshot
) {
+ // Preserve existing row counts until on-device tuning. Wide rows flush by
+ // estimated retained bytes first; LOW keeps only one queued batch.
+ val liveQueuedBatches: Int get() = if (tier == DeviceSyncTier.LOW) 1 else 2
+ val liveBatchBytes: Long get() = when (tier) {
+ DeviceSyncTier.LOW -> 256L * 1024
+ DeviceSyncTier.MID -> 512L * 1024
+ DeviceSyncTier.HIGH -> 1024L * 1024
+ }
+
val diagnosticsLabel: String
get() = "${tier.name.lowercase()} batch=$stageBatchSize memory=${snapshot.memoryClassMb}MB"
diff --git a/data/src/main/java/com/streamvault/data/sync/SyncManagerCatalogStrategySupport.kt b/data/src/main/java/com/streamvault/data/sync/SyncManagerCatalogStrategySupport.kt
index af604809..8d02681d 100644
--- a/data/src/main/java/com/streamvault/data/sync/SyncManagerCatalogStrategySupport.kt
+++ b/data/src/main/java/com/streamvault/data/sync/SyncManagerCatalogStrategySupport.kt
@@ -81,6 +81,8 @@ internal class SyncManagerCatalogStrategySupport(
return when (error) {
is XtreamResponseTooLargeException,
is LowMemoryCatalogAbortException,
+ is XtreamLiveBufferLimitException,
+ is com.streamvault.data.remote.xtream.XtreamLiveRowTooLargeException,
is XtreamParsingException -> true
is java.net.SocketTimeoutException,
is java.io.InterruptedIOException -> true
diff --git a/data/src/main/java/com/streamvault/data/sync/SyncManagerXtreamLiveStrategy.kt b/data/src/main/java/com/streamvault/data/sync/SyncManagerXtreamLiveStrategy.kt
index ed041333..bd78c295 100644
--- a/data/src/main/java/com/streamvault/data/sync/SyncManagerXtreamLiveStrategy.kt
+++ b/data/src/main/java/com/streamvault/data/sync/SyncManagerXtreamLiveStrategy.kt
@@ -18,6 +18,11 @@ import com.streamvault.domain.sync.Section
import com.streamvault.domain.sync.SyncProgress
import kotlinx.coroutines.sync.Mutex
import kotlinx.coroutines.sync.withLock
+import kotlinx.coroutines.CoroutineStart
+import kotlinx.coroutines.async
+import kotlinx.coroutines.coroutineScope
+import kotlinx.coroutines.sync.Semaphore
+import kotlinx.coroutines.sync.withPermit
import kotlin.system.measureTimeMillis
private const val XTREAM_LIVE_STRATEGY_TAG = "SyncManager"
@@ -56,38 +61,58 @@ internal class SyncManagerXtreamLiveStrategy(
trackInitialLiveOnboarding: Boolean,
effectiveLiveSyncMethod: EffectiveXtreamLiveSyncMethod = EffectiveXtreamLiveSyncMethod.STREAM_ALL,
requiredHiddenLiveCategoryIds: Set<Long> = emptySet()
- ): CatalogSyncPayload<Channel> {
+ ): CatalogSyncPayload<Channel> = coroutineScope {
Log.i(XTREAM_LIVE_STRATEGY_TAG, "Xtream live strategy start for provider ${provider.id}.")
// Recovery runs before entering this strategy. Any other saved batches
// are unsealed attempts and cannot be reused by this import.
syncCatalogStore.clearProviderStaging(provider.id)
- val rawLiveCategories = when (val attempt = xtreamSupport.attemptNonCancellation {
- xtreamSupport.retryXtreamCatalogTransient(provider.id) {
- xtreamSupport.executeXtreamRequest(provider.id, XtreamAdaptiveSyncPolicy.Stage.LIGHTWEIGHT) {
- xtreamCatalogApiService.getLiveCategories(
- XtreamUrlFactory.buildPlayerApiUrl(
- serverUrl = provider.serverUrl,
- username = provider.username,
- password = provider.password,
- action = "get_live_categories"
+ val shouldAttemptFullCatalog = effectiveLiveSyncMethod == EffectiveXtreamLiveSyncMethod.STREAM_ALL &&
+ hiddenLiveCategoryIds.isEmpty() &&
+ runtimeProfile.shouldAttemptFullLiveCatalog(trackInitialLiveOnboarding)
+ // A stressed provider gets one request at a time. Starting metadata first
+ // also avoids a one-slot deadlock when the writer needs that context.
+ val requestSlots = Semaphore(xtreamAdaptiveSyncPolicy.liveRequestConcurrencyFor(provider.id))
+ val categoryContext = async(start = CoroutineStart.UNDISPATCHED) {
+ val rawLiveCategories = requestSlots.withPermit {
+ when (val attempt = xtreamSupport.attemptNonCancellation {
+ xtreamSupport.retryXtreamCatalogTransient(provider.id) {
+ xtreamSupport.executeXtreamRequest(provider.id, XtreamAdaptiveSyncPolicy.Stage.LIGHTWEIGHT) {
+ xtreamCatalogApiService.getLiveCategories(
+ XtreamUrlFactory.buildPlayerApiUrl(
+ serverUrl = provider.serverUrl,
+ username = provider.username,
+ password = provider.password,
+ action = "get_live_categories"
+ )
+ )
+ }
+ }
+ }) {
+ is Attempt.Success -> attempt.value
+ is Attempt.Failure -> {
+ Log.w(
+ XTREAM_LIVE_STRATEGY_TAG,
+ "Xtream live categories request failed for provider ${provider.id}: ${sanitizeThrowableMessage(attempt.error)}"
)
- )
+ null
+ }
}
}
- }) {
- is Attempt.Success -> attempt.value
- is Attempt.Failure -> {
- Log.w(
- XTREAM_LIVE_STRATEGY_TAG,
- "Xtream live categories request failed for provider ${provider.id}: ${sanitizeThrowableMessage(attempt.error)}"
- )
- null
+ // Cache even a failed (empty) result. Mapping must not retry category
+ // metadata implicitly for every batch outside the request policy.
+ val resolvedCategories = api.mapCategories(ContentType.LIVE, rawLiveCategories.orEmpty())
+ .map { category -> category.toEntity(provider.id) }.takeIf { it.isNotEmpty() }
+ rawLiveCategories to resolvedCategories
+ }
+ var fullPayload = CatalogSyncPayload<Channel>(
+ catalogResult = CatalogStrategyResult.EmptyValid("full"), categories = null)
+ if (shouldAttemptFullCatalog) {
+ progress(provider.id, onProgress, "Downloading Live TV...")
+ fullPayload = requestSlots.withPermit {
+ loadXtreamLiveFull(provider, api, runtimeProfile, beforeMapping = { categoryContext.await() })
}
}
- val resolvedCategories = rawLiveCategories
- ?.let { categories -> api.mapCategories(ContentType.LIVE, categories) }
- ?.map { category -> category.toEntity(provider.id) }
- ?.takeIf { it.isNotEmpty() }
+ val (rawLiveCategories, resolvedCategories) = categoryContext.await()
val filteredRawLiveCategories = rawLiveCategories.orEmpty().filterNot { category ->
val id = category.categoryId.toLongOrNull()
id in hiddenLiveCategoryIds && id !in requiredHiddenLiveCategoryIds
@@ -98,18 +123,9 @@ internal class SyncManagerXtreamLiveStrategy(
// the hidden category itself.
val resolvedCatalogCategories = resolvedCategories?.takeIf { it.isNotEmpty() }
- var fullPayload = CatalogSyncPayload<Channel>(
- catalogResult = CatalogStrategyResult.EmptyValid("full"),
- categories = null
- )
- val shouldAttemptFullCatalog = effectiveLiveSyncMethod == EffectiveXtreamLiveSyncMethod.STREAM_ALL &&
- hiddenLiveCategoryIds.isEmpty() &&
- runtimeProfile.shouldAttemptFullLiveCatalog(trackInitialLiveOnboarding)
if (shouldAttemptFullCatalog) {
- progress(provider.id, onProgress, "Downloading Live TV...")
- fullPayload = loadXtreamLiveFull(provider, api, runtimeProfile)
when (val fullResult = fullPayload.catalogResult) {
- is CatalogStrategyResult.Success -> return fullPayload.copy(
+ is CatalogStrategyResult.Success -> return@coroutineScope fullPayload.copy(
categories = catalogStrategySupport.mergePreferredAndFallbackCategories(
resolvedCatalogCategories,
fullPayload.categories ?: catalogStrategySupport.buildFallbackLiveCategories(provider.id, fullResult.items)
@@ -120,7 +136,7 @@ internal class SyncManagerXtreamLiveStrategy(
fullCatalogUnsafe = false
)
)
- is CatalogStrategyResult.Partial -> return fullPayload.copy(
+ is CatalogStrategyResult.Partial -> return@coroutineScope fullPayload.copy(
categories = catalogStrategySupport.mergePreferredAndFallbackCategories(
resolvedCatalogCategories,
fullPayload.categories ?: catalogStrategySupport.buildFallbackLiveCategories(provider.id, fullResult.items)
@@ -149,7 +165,7 @@ internal class SyncManagerXtreamLiveStrategy(
preferSequential = existingMetadata.liveSequentialFailuresRemembered || runtimeProfile.maxCategoryConcurrency <= 1,
runtimeProfile = runtimeProfile
)
- return CatalogSyncPayload(
+ CatalogSyncPayload(
catalogResult = categoryPayload.catalogResult,
categories = catalogStrategySupport.mergePreferredAndFallbackCategories(
resolvedCatalogCategories,
@@ -174,7 +190,17 @@ internal class SyncManagerXtreamLiveStrategy(
suspend fun loadXtreamLiveFull(
provider: Provider,
api: XtreamProvider,
- runtimeProfile: CatalogSyncRuntimeProfile
+ runtimeProfile: CatalogSyncRuntimeProfile,
+ beforeMapping: suspend () -> Unit = {}
+ ): CatalogSyncPayload<Channel> = xtreamAdaptiveSyncPolicy.withLivePipelinePermit {
+ loadXtreamLiveFullWithPermit(provider, api, runtimeProfile, beforeMapping)
+ }
+
+ private suspend fun loadXtreamLiveFullWithPermit(
+ provider: Provider,
+ api: XtreamProvider,
+ runtimeProfile: CatalogSyncRuntimeProfile,
+ beforeMapping: suspend () -> Unit
): CatalogSyncPayload<Channel> {
val endpoint = XtreamUrlFactory.buildPlayerApiUrl(
serverUrl = provider.serverUrl,
@@ -190,6 +216,8 @@ internal class SyncManagerXtreamLiveStrategy(
failureNextStep = "category-bulk",
streamItems = { item -> xtreamCatalogHttpService.streamLiveStreams(endpoint, onItem = item) },
mapRawBatch = { batch -> api.mapLiveStreamsSequence(batch) },
+ estimatedBytes = { it.estimatedBufferBytes() },
+ beforeMapping = beforeMapping,
runtimeProfile = runtimeProfile
)
}
@@ -200,6 +228,8 @@ internal class SyncManagerXtreamLiveStrategy(
failureNextStep = "legacy-full",
streamItems = { item -> xtreamCatalogHttpService.streamLiveStreamRows(endpoint, onItem = item) },
mapRawBatch = { batch -> api.mapLiveStreamRowsSequence(batch) },
+ estimatedBytes = { it.estimatedBufferBytes() },
+ beforeMapping = beforeMapping,
runtimeProfile = runtimeProfile
)
if (!thinPayload.shouldRetryLegacyFullDecode()) {
@@ -216,6 +246,8 @@ internal class SyncManagerXtreamLiveStrategy(
failureNextStep = "category-bulk",
streamItems = { item -> xtreamCatalogHttpService.streamLiveStreams(endpoint, onItem = item) },
mapRawBatch = { batch -> api.mapLiveStreamsSequence(batch) },
+ estimatedBytes = { it.estimatedBufferBytes() },
+ beforeMapping = beforeMapping,
runtimeProfile = runtimeProfile
)
}
@@ -235,11 +267,12 @@ internal class SyncManagerXtreamLiveStrategy(
failureNextStep: String,
streamItems: suspend (suspend (RawItem) -> Unit) -> Int,
mapRawBatch: suspend (Sequence<RawItem>) -> Sequence<Channel>,
+ estimatedBytes: (RawItem) -> Long,
+ beforeMapping: suspend () -> Unit,
runtimeProfile: CatalogSyncRuntimeProfile
): CatalogSyncPayload<Channel> {
var fallbackCollector = FallbackCategoryCollector(provider.id, ContentType.LIVE)
val seenStreamIds = HashSet<Long>()
- val rawBatch = ArrayList<RawItem>(runtimeProfile.stageBatchSize)
var stagedSessionId: Long? = null
var acceptedCount = 0
var streamedRawCount = 0
@@ -251,7 +284,6 @@ internal class SyncManagerXtreamLiveStrategy(
suspend fun discardAttempt() {
stagedSessionId?.let { syncCatalogStore.discardStagedImport(provider.id, it) }
stagedSessionId = null
- rawBatch.clear()
seenStreamIds.clear()
fallbackCollector = FallbackCategoryCollector(provider.id, ContentType.LIVE)
acceptedCount = 0
@@ -279,7 +311,7 @@ internal class SyncManagerXtreamLiveStrategy(
}
}
- suspend fun flushRawBatch() {
+ suspend fun flushRawBatch(rawBatch: List<RawItem>) {
if (rawBatch.isEmpty()) return
lateinit var mappedChannels: List<Channel>
mappingElapsedMs += measureTimeMillis {
@@ -301,7 +333,6 @@ internal class SyncManagerXtreamLiveStrategy(
stagedSessionId = staged.sessionId
acceptedCount += staged.acceptedCount
flushCount++
- rawBatch.clear()
// D10 — mode HIGH (full catalog) : pas de count par categorie disponible,
// on emet en indetermine (`total = 0`) une fois par flush de batch (cadence
// <= 1/s en pratique, jamais par item). Le label reste vide car aucune
@@ -327,15 +358,16 @@ internal class SyncManagerXtreamLiveStrategy(
stagingElapsedMs = 0L
try {
xtreamSupport.executeXtreamRequest(provider.id, XtreamAdaptiveSyncPolicy.Stage.HEAVY) {
- streamItems { item ->
- rawBatch += item
- streamedRawCount++
- if (rawBatch.size >= runtimeProfile.stageBatchSize) {
- flushRawBatch()
- }
- }.also {
- flushRawBatch()
- }
+ val stats = runXtreamLivePipeline(
+ batchRows = runtimeProfile.stageBatchSize,
+ batchBytes = runtimeProfile.liveBatchBytes,
+ queuedBatches = runtimeProfile.liveQueuedBatches,
+ estimatedBytes = estimatedBytes,
+ prepare = beforeMapping,
+ read = { emit -> streamItems { item -> streamedRawCount++; emit(item) } },
+ consume = { batch -> flushRawBatch(batch) }
+ )
+ Log.i("XtreamSyncTiming", "phase=live-pipeline rows=${stats.rows} batches=${stats.batches} reader_ms=${stats.readerMs} drain_ms=${stats.drainMs} send_wait_ms=${stats.sendWaitMs} outstanding_batch_bytes=${stats.outstandingBatchBytes} queue_capacity=${runtimeProfile.liveQueuedBatches} batch_byte_limit=${runtimeProfile.liveBatchBytes}")
}
} catch (error: Throwable) {
discardAttempt()
diff --git a/data/src/main/java/com/streamvault/data/sync/XtreamAdaptiveSyncPolicy.kt b/data/src/main/java/com/streamvault/data/sync/XtreamAdaptiveSyncPolicy.kt
index dab9c298..f44850c4 100644
--- a/data/src/main/java/com/streamvault/data/sync/XtreamAdaptiveSyncPolicy.kt
+++ b/data/src/main/java/com/streamvault/data/sync/XtreamAdaptiveSyncPolicy.kt
@@ -8,6 +8,8 @@ import java.util.concurrent.ConcurrentHashMap
import kotlinx.coroutines.delay
import kotlinx.coroutines.sync.Mutex
import kotlinx.coroutines.sync.withLock
+import kotlinx.coroutines.sync.Semaphore
+import kotlinx.coroutines.sync.withPermit
internal class XtreamAdaptiveSyncPolicy {
private companion object {
@@ -36,6 +38,15 @@ internal class XtreamAdaptiveSyncPolicy {
private val providerHealth = ConcurrentHashMap<Long, ProviderHealth>()
private val providerPacing = ConcurrentHashMap<Long, ProviderPacingState>()
+ // This policy is shared by SyncManager. Do not multiply queued bulk batches
+ // when different providers import simultaneously.
+ private val livePipelineSlot = Semaphore(1)
+
+ suspend fun <T> withLivePipelinePermit(block: suspend () -> T): T =
+ livePipelineSlot.withPermit { block() }
+
+ fun liveRequestConcurrencyFor(providerId: Long): Int =
+ if ((providerHealth[providerId]?.stressLevel ?: 0) > 0) 1 else 2
suspend fun awaitTurn(providerId: Long, stage: Stage) {
val waitMs = (providerHealth[providerId]?.backoffUntilMs ?: 0L) - System.currentTimeMillis()
diff --git a/data/src/main/java/com/streamvault/data/sync/XtreamLivePipeline.kt b/data/src/main/java/com/streamvault/data/sync/XtreamLivePipeline.kt
new file mode 100644
index 00000000..e926878c
--- /dev/null
+++ b/data/src/main/java/com/streamvault/data/sync/XtreamLivePipeline.kt
@@ -0,0 +1,107 @@
+package com.streamvault.data.sync
+
+import com.streamvault.data.remote.dto.XtreamLiveStreamRow
+import com.streamvault.data.remote.dto.XtreamStream
+import java.util.concurrent.atomic.AtomicLong
+import kotlinx.coroutines.CancellationException
+import kotlinx.coroutines.Dispatchers
+import kotlinx.coroutines.async
+import kotlinx.coroutines.cancel
+import kotlinx.coroutines.channels.Channel
+import kotlinx.coroutines.coroutineScope
+
+/** Only the consumer owns mapping, duplicate admission and staging state. */
+internal suspend fun <T> runXtreamLivePipeline(
+ batchRows: Int,
+ batchBytes: Long,
+ queuedBatches: Int,
+ estimatedBytes: (T) -> Long,
+ read: suspend (suspend (T) -> Unit) -> Int,
+ prepare: suspend () -> Unit = {},
+ consume: suspend (List<T>) -> Unit
+): XtreamLivePipelineStats = coroutineScope {
+ require(batchRows > 0 && batchBytes > 0 && queuedBatches > 0)
+ val queue = Channel<BufferedLiveBatch<T>>(queuedBatches)
+ val retainedBytes = AtomicLong()
+ val highWaterBytes = AtomicLong()
+ var sendWaitNanos = 0L
+ var batches = 0
+ val started = System.nanoTime()
+ val writer = async(Dispatchers.Default) {
+ try {
+ prepare()
+ for (batch in queue) {
+ consume(batch.items)
+ retainedBytes.addAndGet(-batch.bytes)
+ }
+ } catch (error: CancellationException) {
+ // Child cancellation alone does not cancel its parent. Supersession
+ // must also interrupt a reader blocked on HTTP or a full queue.
+ throw error
+ }
+ }
+ try {
+ var batch = ArrayList<T>(batchRows)
+ var bytes = 0L
+ suspend fun sendBatch() {
+ if (batch.isEmpty()) return
+ val sending = BufferedLiveBatch(batch, bytes)
+ val outstanding = retainedBytes.addAndGet(bytes)
+ highWaterBytes.updateAndGet { maxOf(it, outstanding) }
+ val beforeSend = System.nanoTime()
+ queue.send(sending)
+ sendWaitNanos += System.nanoTime() - beforeSend
+ batches++
+ batch = ArrayList(batchRows)
+ bytes = 0
+ }
+ val rows = read { row ->
+ val size = estimatedBytes(row).coerceAtLeast(1)
+ if (size > batchBytes) throw XtreamLiveBufferLimitException("Live row exceeds buffered batch budget")
+ if (batch.isNotEmpty() && bytes + size > batchBytes) sendBatch()
+ batch.add(row)
+ bytes += size
+ if (batch.size >= batchRows || bytes >= batchBytes) sendBatch()
+ }
+ // read only returns after the transport has validated END_DOCUMENT.
+ sendBatch()
+ val readerMs = (System.nanoTime() - started) / 1_000_000
+ queue.close()
+ val drainStarted = System.nanoTime()
+ writer.await()
+ XtreamLivePipelineStats(rows, batches, readerMs,
+ (System.nanoTime() - drainStarted) / 1_000_000,
+ sendWaitNanos / 1_000_000, highWaterBytes.get())
+ } finally {
+ queue.cancel()
+ writer.cancel()
+ // coroutineScope joins the writer before attempt cleanup/retry outside.
+ }
+}
+
+internal data class XtreamLivePipelineStats(
+ val rows: Int,
+ val batches: Int,
+ val readerMs: Long,
+ val drainMs: Long,
+ val sendWaitMs: Long,
+ // Includes the writer batch and a producer suspended in send, not just queue.
+ val outstandingBatchBytes: Long
+)
+
+internal class XtreamLiveBufferLimitException(message: String) : IllegalStateException(message)
+
+private data class BufferedLiveBatch<T>(val items: List<T>, val bytes: Long)
+
+private fun retainedTextBytes(fields: List<String?>, ids: List<String>?): Long =
+ 256L + fields.sumOf { 40L + 2L * (it?.length ?: 0) } +
+ ids.orEmpty().sumOf { 48L + 2L * it.length }
+
+internal fun XtreamLiveStreamRow.estimatedBufferBytes(): Long = retainedTextBytes(
+ listOf(name, streamIcon, epgChannelId, categoryId, categoryName, containerExtension), categoryIds)
+
+internal fun XtreamStream.estimatedBufferBytes(): Long = retainedTextBytes(
+ listOf(name, streamType, streamIcon, coverBig, epgChannelId, added, categoryId,
+ categoryName, customSid, directSource, containerExtension, rating, rating5based,
+ tmdb, trailer, youtubeTrailer), categoryIds)
diff --git a/data/src/test/java/com/streamvault/data/remote/xtream/OkHttpXtreamApiServiceTest.kt b/data/src/test/java/com/streamvault/data/remote/xtream/OkHttpXtreamApiServiceTest.kt
index 2870b093..e1416d0a 100644
--- a/data/src/test/java/com/streamvault/data/remote/xtream/OkHttpXtreamApiServiceTest.kt
+++ b/data/src/test/java/com/streamvault/data/remote/xtream/OkHttpXtreamApiServiceTest.kt
@@ -19,6 +19,9 @@ import kotlinx.coroutines.launch
import kotlinx.coroutines.withContext
import kotlinx.coroutines.withTimeout
import kotlinx.coroutines.test.runTest
+import kotlinx.coroutines.runBlocking
+import com.streamvault.data.sync.runXtreamLivePipeline
+import com.streamvault.data.sync.ProviderWorkflowSupersededException
import kotlinx.serialization.json.Json
import okhttp3.Interceptor
import okhttp3.MediaType.Companion.toMediaType
@@ -34,6 +37,72 @@ import org.junit.Test
class OkHttpXtreamApiServiceTest {
+ @Test
+ fun `pipeline writer supersession cancels a body blocked after its first row`() = runBlocking {
+ withTimeout(5_000) {
+ val bodyBlocked = CountDownLatch(1)
+ val cancellationObserved = CountDownLatch(1)
+ val prefix = ("""[{"stream_id":1,"name":"One"},""" + " ".repeat(16_384)).toByteArray()
+ val client = OkHttpClient.Builder().addInterceptor { chain ->
+ val source = object : InputStream() {
+ var offset = 0
+ override fun read(): Int {
+ if (offset < prefix.size) return prefix[offset++].toInt()
+ bodyBlocked.countDown()
+ while (!chain.call().isCanceled()) Thread.sleep(10)
+ cancellationObserved.countDown()
+ throw IOException("Canceled body read")
+ }
+ }.source().buffer()
+ Response.Builder().request(chain.request()).protocol(Protocol.HTTP_1_1).code(200).message("OK")
+ .body(object : ResponseBody() {
+ override fun contentType() = "application/json".toMediaType()
+ override fun contentLength() = -1L
+ override fun source() = source
+ }).build()
+ }.build()
+ val service = OkHttpXtreamApiService(client, json)
+ val failure = runCatching {
+ runXtreamLivePipeline(1, 4096, 1, { _: com.streamvault.data.remote.dto.XtreamLiveStreamRow -> 256L },
+ read = { emit -> service.streamLiveStreamRows("https://example.test/player_api.php?action=get_live_streams", onItem = emit) },
+ consume = {
+ check(bodyBlocked.await(4, TimeUnit.SECONDS))
+ throw ProviderWorkflowSupersededException(1, 1)
+ })
+ }.exceptionOrNull()
+ assertThat(failure).isInstanceOf(ProviderWorkflowSupersededException::class.java)
+ assertThat(cancellationObserved.count).isEqualTo(0)
+ }
+ }
+
+ @Test
+ fun `streamed live rows reject huge tokens before full string allocation`() = runTest {
+ val prefix = """[{"stream_id":1,"name":"""".toByteArray()
+ val returned = AtomicInteger()
+ val source = object : InputStream() {
+ var offset = 0
+ override fun read(): Int {
+ returned.incrementAndGet()
+ return if (offset < prefix.size) prefix[offset++].toInt() else 'x'.code
+ }
+ }.source().buffer()
+ val client = OkHttpClient.Builder().addInterceptor { chain ->
+ Response.Builder().request(chain.request()).protocol(Protocol.HTTP_1_1).code(200).message("OK")
+ .body(object : ResponseBody() {
+ override fun contentType() = "application/json".toMediaType()
+ override fun contentLength() = -1L
+ override fun source() = source
+ }).build()
+ }.build()
+ val failure = runCatching {
+ OkHttpXtreamApiService(client, json).streamLiveStreamRows(
+ "https://example.test/player_api.php?action=get_live_streams") { error("oversized row emitted") }
+ }.exceptionOrNull()
+ assertThat(failure).isInstanceOf(XtreamLiveRowTooLargeException::class.java)
+ // Includes Okio/InputStreamReader/JsonReader read-ahead, not a 96 MB response.
+ assertThat(returned.get()).isAtMost(XtreamLiveRowBudgetReader.MAX_ROW_CHARS + 32 * 1024)
+ }
+
private val json = Json {
ignoreUnknownKeys = true
isLenient = true
diff --git a/data/src/test/java/com/streamvault/data/sync/XtreamLiveAttemptIsolationTest.kt b/data/src/test/java/com/streamvault/data/sync/XtreamLiveAttemptIsolationTest.kt
index 5bd7fd52..6eb8e262 100644
--- a/data/src/test/java/com/streamvault/data/sync/XtreamLiveAttemptIsolationTest.kt
+++ b/data/src/test/java/com/streamvault/data/sync/XtreamLiveAttemptIsolationTest.kt
@@ -13,6 +13,9 @@ import com.streamvault.data.remote.xtream.XtreamProvider
import com.streamvault.domain.model.ContentType
import com.streamvault.domain.model.LegacyProvider
import com.streamvault.domain.model.ProviderType
+import com.streamvault.domain.model.SyncMetadata
+import java.util.concurrent.CountDownLatch
+import java.util.concurrent.TimeUnit
import java.util.concurrent.atomic.AtomicInteger
import kotlinx.coroutines.CancellationException
import kotlinx.coroutines.runBlocking
@@ -68,6 +71,54 @@ class XtreamLiveAttemptIsolationTest {
assertThat(totalStages()).isEqualTo(2)
}
+ @Test
+ fun `cold metadata and bulk requests overlap and adult context is ready before staging`() = runBlocking<Unit> {
+ val bulkStarted = CountDownLatch(1)
+ val metadataRequests = AtomicInteger()
+ val bulkRequests = AtomicInteger()
+ val (strategy, api) = engine(primeCategories = false, categoryResponse = {
+ metadataRequests.incrementAndGet()
+ check(bulkStarted.await(5, TimeUnit.SECONDS)) { "bulk did not overlap metadata" }
+ """[{"category_id":"1","category_name":"One","is_adult":1}]"""
+ }, beforeStage = { channels -> assertThat(channels.all { it.isAdult }).isTrue() }) {
+ bulkRequests.incrementAndGet(); bulkStarted.countDown(); array(row(101), row(102), row(103))
+ }
+ val result = strategy.syncXtreamLiveCatalog(provider, api, SyncMetadata(provider.id), emptySet(), null, profile(), true)
+ assertThat(result.catalogResult).isInstanceOf(CatalogStrategyResult.Success::class.java)
+ assertThat(result.stagedAcceptedCount).isEqualTo(3)
+ assertThat(metadataRequests.get()).isEqualTo(1)
+ assertThat(bulkRequests.get()).isEqualTo(1)
+ assertThat(db.catalogSyncDao().getChannelStages(provider.id, requireNotNull(result.stagedSessionId)).all { it.isAdult }).isTrue()
+ }
+
+ @Test
+ fun `metadata failure is resolved once instead of fetched again by every batch`() = runBlocking<Unit> {
+ val metadataRequests = AtomicInteger()
+ val (strategy, api) = engine(primeCategories = false, categoryStatus = 400,
+ categoryResponse = { metadataRequests.incrementAndGet(); "{}" }) {
+ array(row(101), row(102), row(103))
+ }
+ val result = strategy.syncXtreamLiveCatalog(provider, api, SyncMetadata(provider.id), emptySet(), null, profile(), true)
+ assertThat(result.catalogResult).isInstanceOf(CatalogStrategyResult.Success::class.java)
+ assertThat(result.stagedAcceptedCount).isEqualTo(3)
+ assertThat(metadataRequests.get()).isEqualTo(1)
+ assertThat(result.categories).isNotEmpty()
+ }
+
+ @Test
+ fun `stressed provider resolves metadata before requesting bulk without deadlock`() = runBlocking<Unit> {
+ val policy = XtreamAdaptiveSyncPolicy()
+ policy.recordFailure(provider.id, java.io.IOException("connection reset"))
+ val bulkRequests = AtomicInteger()
+ val (strategy, api) = engine(policy = policy, primeCategories = false, categoryResponse = {
+ assertThat(bulkRequests.get()).isEqualTo(0)
+ """[{"category_id":"1","category_name":"One"}]"""
+ }) { bulkRequests.incrementAndGet(); array(row(101), row(102)) }
+ val result = strategy.syncXtreamLiveCatalog(provider, api, SyncMetadata(provider.id), emptySet(), null, profile(), true)
+ assertThat(result.stagedAcceptedCount).isEqualTo(2)
+ assertThat(bulkRequests.get()).isEqualTo(1)
+ }
+
@Test
fun `failed category decoder attempt cannot overwrite successful category or retain partial rows`() = runBlocking<Unit> {
val attempts = AtomicInteger()
@@ -190,14 +241,18 @@ class XtreamLiveAttemptIsolationTest {
private fun categories() = listOf(XtreamCategory("1", "One"), XtreamCategory("2", "Two"))
private suspend fun engine(failWriter: Boolean = false, failOnStreamId: Long? = null,
+ primeCategories: Boolean = true, categoryStatus: Int = 200, categoryResponse: (() -> String)? = null,
+ policy: XtreamAdaptiveSyncPolicy = XtreamAdaptiveSyncPolicy(),
+ beforeStage: suspend (List<com.streamvault.domain.model.Channel>) -> Unit = {},
response: (String?) -> String): Pair<SyncManagerXtreamLiveStrategy, XtreamProvider> {
val http = OkHttpXtreamApiService(OkHttpClient.Builder().addInterceptor { chain ->
- Response.Builder().request(chain.request()).protocol(Protocol.HTTP_1_1).code(200).message("OK")
- .body(response(chain.request().url.queryParameter("category_id")).toResponseBody("application/json".toMediaType())).build()
+ val isCategories = chain.request().url.queryParameter("action") == "get_live_categories"
+ val body = if (isCategories && categoryResponse != null) categoryResponse() else response(chain.request().url.queryParameter("category_id"))
+ Response.Builder().request(chain.request()).protocol(Protocol.HTTP_1_1).code(if (isCategories) categoryStatus else 200).message("OK")
+ .body(body.toResponseBody("application/json".toMediaType())).build()
}.build(), Json { ignoreUnknownKeys = true; isLenient = true; explicitNulls = false; coerceInputValues = true })
val api = XtreamProvider(provider.id, http, provider.serverUrl, provider.username, provider.password)
- api.mapCategories(ContentType.LIVE, categories())
- val policy = XtreamAdaptiveSyncPolicy()
+ if (primeCategories) api.mapCategories(ContentType.LIVE, categories())
val progress: (Long, ((String) -> Unit)?, String) -> Unit = { _, callback, label -> callback?.invoke(label) }
val sanitize: (Throwable?) -> String = { it?.javaClass?.simpleName.orEmpty() }
val support = SyncManagerXtreamSupport(policy, { false }, sanitize, progress, 60_000, 60_000, "stopped")
@@ -205,6 +260,7 @@ class XtreamLiveAttemptIsolationTest {
return SyncManagerXtreamLiveStrategy(http, http, policy, support, SyncManagerXtreamFetcher(http, http, support, sanitize),
SyncManagerCatalogStrategySupport({ false }, 60_000), store, progress, sanitize,
{ _, _ -> "full failed" }, { _, _, _ -> "category failed" }, "sequential", stageChannelItems = { id, channels, seen, collector, session ->
+ beforeStage(channels)
if (failWriter || channels.any { it.streamId == failOnStreamId }) throw ProviderWorkflowSupersededException(id, 1)
stager.stageChannelItems(id, channels, seen, collector, session)
}, emitProgress = { _, _ -> }) to api
diff --git a/data/src/test/java/com/streamvault/data/sync/XtreamLivePipelineTest.kt b/data/src/test/java/com/streamvault/data/sync/XtreamLivePipelineTest.kt
new file mode 100644
index 00000000..0bef1e20
--- /dev/null
+++ b/data/src/test/java/com/streamvault/data/sync/XtreamLivePipelineTest.kt
@@ -0,0 +1,198 @@
+package com.streamvault.data.sync
+
+import com.google.common.truth.Truth.assertThat
+import java.io.IOException
+import java.util.concurrent.atomic.AtomicInteger
+import kotlinx.coroutines.CancellationException
+import kotlinx.coroutines.CompletableDeferred
+import kotlinx.coroutines.NonCancellable
+import kotlinx.coroutines.async
+import kotlinx.coroutines.awaitCancellation
+import kotlinx.coroutines.cancelAndJoin
+import kotlinx.coroutines.runBlocking
+import kotlinx.coroutines.withContext
+import kotlinx.coroutines.withTimeout
+import org.junit.Test
+
+class XtreamLivePipelineTest {
+ @Test
+ fun `provider stress reduces metadata and bulk concurrency`() {
+ val policy = XtreamAdaptiveSyncPolicy()
+ assertThat(policy.liveRequestConcurrencyFor(1)).isEqualTo(2)
+ policy.recordFailure(1, IOException("connection reset"))
+ assertThat(policy.liveRequestConcurrencyFor(1)).isEqualTo(1)
+ assertThat(policy.liveRequestConcurrencyFor(2)).isEqualTo(2)
+ policy.forgetProvider(1)
+ assertThat(policy.liveRequestConcurrencyFor(1)).isEqualTo(2)
+ }
+
+ @Test
+ fun `different providers share one bulk pipeline permit`() = runBlocking {
+ withTimeout(5_000) {
+ val policy = XtreamAdaptiveSyncPolicy()
+ val firstEntered = CompletableDeferred<Unit>()
+ val release = CompletableDeferred<Unit>()
+ val secondAttempting = CompletableDeferred<Unit>()
+ val secondEntered = CompletableDeferred<Unit>()
+ val first = async { policy.withLivePipelinePermit { firstEntered.complete(Unit); release.await() } }
+ firstEntered.await()
+ val second = async {
+ secondAttempting.complete(Unit)
+ policy.withLivePipelinePermit { secondEntered.complete(Unit) }
+ }
+ secondAttempting.await()
+ assertThat(secondEntered.isCompleted).isFalse()
+ release.complete(Unit)
+ first.await(); second.await()
+ assertThat(secondEntered.isCompleted).isTrue()
+ }
+ }
+
+ @Test
+ fun `reader overlaps writer but cannot run beyond queue capacity`() = runBlocking {
+ withTimeout(5_000) {
+ val writing = CompletableDeferred<Unit>()
+ val thirdRead = CompletableDeferred<Unit>()
+ val release = CompletableDeferred<Unit>()
+ val emitted = AtomicInteger()
+ val finished = AtomicInteger()
+ val result = async {
+ runXtreamLivePipeline(1, 10, 1, { 10L }, read = { emit ->
+ repeat(6) { index ->
+ emitted.incrementAndGet()
+ if (index == 2) thirdRead.complete(Unit)
+ emit(index)
+ finished.incrementAndGet()
+ }; 6
+ }, consume = { writing.complete(Unit); release.await() })
+ }
+ writing.await()
+ thirdRead.await()
+ assertThat(emitted.get()).isAtMost(4) // writer + queue + suspended send + next row
+ assertThat(finished.get()).isAtMost(2)
+ release.complete(Unit)
+ val stats = result.await()
+ assertThat(stats.rows).isEqualTo(6)
+ assertThat(stats.outstandingBatchBytes).isAtMost(30)
+ }
+ }
+
+ @Test
+ fun `metadata wait fills bounded queue and parent cancellation stops reader`() = runBlocking {
+ withTimeout(5_000) {
+ val thirdRead = CompletableDeferred<Unit>()
+ val stopped = CompletableDeferred<Unit>()
+ val context = CompletableDeferred<Unit>()
+ val finished = AtomicInteger()
+ val result = async {
+ runXtreamLivePipeline(1, 10, 1, { 10L }, prepare = { context.await() }, read = { emit ->
+ try {
+ repeat(10) { index ->
+ if (index == 2) thirdRead.complete(Unit)
+ emit(index); finished.incrementAndGet()
+ }; 10
+ } finally { stopped.complete(Unit) }
+ }, consume = { error("mapping before metadata") })
+ }
+ // With preparation blocked, even the second send must wait.
+ while (finished.get() == 0) kotlinx.coroutines.yield()
+ assertThat(finished.get()).isEqualTo(1)
+ assertThat(thirdRead.isCompleted).isFalse()
+ result.cancelAndJoin()
+ stopped.await()
+ }
+ }
+
+ @Test
+ fun `completion waits for final batch writer drain`() = runBlocking {
+ withTimeout(5_000) {
+ val eof = CompletableDeferred<Unit>()
+ val writing = CompletableDeferred<Unit>()
+ val release = CompletableDeferred<Unit>()
+ val result = async {
+ runXtreamLivePipeline(10, 100, 1, { 1L }, read = { emit ->
+ emit(1); eof.complete(Unit); 1
+ }, consume = { writing.complete(Unit); release.await() })
+ }
+ eof.await(); writing.await()
+ assertThat(result.isCompleted).isFalse()
+ release.complete(Unit)
+ assertThat(result.await().rows).isEqualTo(1)
+ }
+ }
+
+ @Test
+ fun `reader failure joins cancelled writer before returning for retry`() = runBlocking {
+ withTimeout(5_000) {
+ val writing = CompletableDeferred<Unit>()
+ val cancelling = CompletableDeferred<Unit>()
+ val releaseCleanup = CompletableDeferred<Unit>()
+ val result = async {
+ runCatching {
+ runXtreamLivePipeline(1, 100, 1, { 1L }, read = { emit ->
+ emit(1); writing.await(); throw IOException("truncated")
+ }, consume = {
+ writing.complete(Unit)
+ try { awaitCancellation() } finally {
+ withContext(NonCancellable) { cancelling.complete(Unit); releaseCleanup.await() }
+ }
+ })
+ }
+ }
+ cancelling.await()
+ assertThat(result.isCompleted).isFalse()
+ releaseCleanup.complete(Unit)
+ assertThat(result.await().exceptionOrNull()).isInstanceOf(IOException::class.java)
+ }
+ }
+
+ @Test
+ fun `superseded writer cancels reader even when reader is waiting on transport`() = runBlocking {
+ withTimeout(5_000) {
+ val reading = CompletableDeferred<Unit>()
+ val stopped = CompletableDeferred<Unit>()
+ val failure = runCatching {
+ runXtreamLivePipeline(1, 100, 1, { 1L }, read = { emit ->
+ try { emit(1); reading.complete(Unit); awaitCancellation() }
+ finally { stopped.complete(Unit) }
+ }, consume = { reading.await(); throw ProviderWorkflowSupersededException(1, 1) })
+ }.exceptionOrNull()
+ stopped.await()
+ assertThat(failure).isInstanceOf(ProviderWorkflowSupersededException::class.java)
+ }
+ }
+
+ @Test
+ fun `wide rows flush by bytes and preserve order`() = runBlocking {
+ val batches = mutableListOf<List<Int>>()
+ val stats = runXtreamLivePipeline(100, 100, 2, { row: Int -> row.toLong() }, read = { emit ->
+ listOf(30, 60, 70, 20, 10).forEach { emit(it) }; 5
+ }, consume = { batches.add(it) })
+ assertThat(batches).containsExactly(listOf(30, 60), listOf(70, 20, 10)).inOrder()
+ assertThat(stats.batches).isEqualTo(2)
+ assertThat(stats.outstandingBatchBytes).isAtMost(400)
+ }
+
+ @Test
+ fun `oversized row is rejected without silent trimming`() = runBlocking {
+ val failure = runCatching {
+ runXtreamLivePipeline(100, 100, 1, { row: Int -> row.toLong() },
+ read = { emit -> emit(101); 1 }, consume = { error("oversized write") })
+ }.exceptionOrNull()
+ assertThat(failure).isInstanceOf(XtreamLiveBufferLimitException::class.java)
+ }
+
+ @Test
+ fun `ordinary writer failure stops producer`() = runBlocking {
+ withTimeout(5_000) {
+ val stopped = CompletableDeferred<Unit>()
+ val failure = runCatching {
+ runXtreamLivePipeline(1, 100, 1, { 1L }, read = { emit ->
+ try { emit(1); awaitCancellation() } finally { stopped.complete(Unit) }
+ }, consume = { throw IllegalStateException("write failed") })
+ }.exceptionOrNull()
+ stopped.await()
+ assertThat(failure).hasMessageThat().isEqualTo("write failed")
+ }
+ }
+}
diff --git a/docs/XTREAM_SYNC_BENCHMARK.md b/docs/XTREAM_SYNC_BENCHMARK.md
index 3cf1ab7b..c185b240 100644
--- a/docs/XTREAM_SYNC_BENCHMARK.md
+++ b/docs/XTREAM_SYNC_BENCHMARK.md
@@ -62,6 +62,12 @@ rebuild, now using a monotonic clock. The runner retains only structured
benchmark data and explicitly recognized numeric timing lines. It never
exports general provider logs, request URLs, response previews or passwords.
+The P3 bulk pipeline adds numeric `live-pipeline` records for batch counts,
+reader duration, final writer drain, send/backpressure time and estimated
+outstanding raw batch bytes. See the
+[pipeline report](XTREAM_SYNC_PIPELINE_IMPLEMENTATION.md) for resource accounting
+and overlap semantics. These durations overlap and are not CPU measurements.
+
Java heap is sampled every 50 ms, with ART GC count deltas where available.
These are debug-emulator measurements, not release-device performance, native
RSS or an allocation profile. A short heap peak may fall between samples.
@@ -103,8 +109,9 @@ This package is an initial backend baseline, not the full P0 gate. Follow-up
work still needs precise headers/first-row and workflow-wait timings, attempt
and generation identifiers, row identity/field equivalence beyond counts,
skipped/duplicate/hidden coverage, parser CPU/allocation profiling, auxiliary
-index readiness, and a first-browsable-screen measurement. Queue occupancy and
-writer-drain timings apply once the bounded pipeline exists. Run the full
+index readiness, and a first-browsable-screen measurement. Pipeline byte
+high-water and writer-drain timings now exist; actual heap/allocation attribution
+and queue occupancy over time remain open. Run the full
five-sample scenario matrix and representative release-device measurements
before freezing production tuning or claiming a speedup. Playback validation
is separate and follows AGENTS.md's sustained capture/log requirements.
diff --git a/docs/XTREAM_SYNC_IMPROVEMENT_PLAN.md b/docs/XTREAM_SYNC_IMPROVEMENT_PLAN.md
index 6b6ed669..ba89e5df 100644
--- a/docs/XTREAM_SYNC_IMPROVEMENT_PLAN.md
+++ b/docs/XTREAM_SYNC_IMPROVEMENT_PLAN.md
@@ -10,6 +10,7 @@ Supporting evidence:
- [Initial backend baseline](XTREAM_SYNC_INITIAL_BASELINE.md)
- [Attempt isolation and recovery implementation](XTREAM_SYNC_RECOVERY_IMPLEMENTATION.md)
- [Live parsing and group identity implementation](XTREAM_SYNC_PARSING_IMPLEMENTATION.md)
+- [Bounded live pipeline implementation](XTREAM_SYNC_PIPELINE_IMPLEMENTATION.md)
The reference implementation provides hypotheses to investigate, not a design
authority or proof of correctness. Choose behavior for this app's measured
@@ -279,14 +280,17 @@ P6 is the first live performance release milestone; P7 follows it. Keep actual
production AUTO behavior unchanged until the P6 gate. Experiments before then
use explicit STREAM_ALL or debug configuration, with identical data coverage.
-Implementation progress on 2026-10-09: P0 has an initial backend baseline and
+Implementation progress on 2026-10-10: P0 has an initial backend baseline and
isolated fixture; its complete release-device/readiness comparison remains open.
P1 attempt isolation and conservative receipt-based recovery are implemented and
validated. P2 direct live decoding and cheaper group identity are implemented
-with field-equivalence checks and measured component CPU reductions. P3–P8
-remain pending. Repeated 10,000-row runs still show memory-driven category
-fallback, so the default-strategy gate has not been met. See the implementation
-reports linked above for exact evidence and limits.
+with field-equivalence checks and measured component CPU reductions. P3 retrieval
+overlap and a bounded reader/writer pipeline are implemented with cancellation,
+resource-bound and real-graph setup/refresh evidence. P4–P8 remain pending.
+The new measurements identify writer cost, slow fallback and extra completed
+imports during some refresh intervals as remaining priorities. Earlier runs
+also showed memory-driven category fallback; the default-strategy gate has not
+been met. See the implementation reports linked above for evidence and limits.
## P0 measurement and fixture deliverables
diff --git a/docs/XTREAM_SYNC_PIPELINE_IMPLEMENTATION.md b/docs/XTREAM_SYNC_PIPELINE_IMPLEMENTATION.md
new file mode 100644
index 00000000..ec0588eb
--- /dev/null
+++ b/docs/XTREAM_SYNC_PIPELINE_IMPLEMENTATION.md
@@ -0,0 +1,162 @@
+# Xtream live import pipeline
+
+P3 implementation on `codex/xtream-live-sync-performance`, following the
+[improvement plan](XTREAM_SYNC_IMPROVEMENT_PLAN.md). This package changes bulk
+live ingestion used by provider setup and Settings Live refresh. AUTO selection
+remains behind the P6 gate. Category mode and automatic fallback remain available;
+their worker scheduling and pacing improvements belong to P5.
+
+## Retrieval and ownership
+
+For a healthy provider, live category metadata and one bulk live response start
+concurrently. Both use the existing provider backoff/retry policy. A provider
+with remembered request stress gets one local request slot: metadata resolves
+first, then bulk retrieval begins. There are no parallel downloads of the same
+bulk endpoint.
+
+The reader sends typed batches through a bounded coroutine channel. A single
+consumer on `Dispatchers.Default` waits for category context once, then maps and
+stages rows while the HTTP reader continues. Metadata failure explicitly primes
+empty adult-category context and retains fallback category semantics. Mapping
+does not issue another category request for each batch. Authoritative category
+shells, adult flags, hidden/required scope and stream ordering retain their
+existing semantics.
+
+Only the consumer owns the staging session, duplicate set, fallback categories,
+accepted count and write timings. Reader success requires valid END_DOCUMENT.
+Import success also requires channel drain and completion of the final write;
+the P1 completion receipt is sealed only after that return.
+
+Reader failure cancels and joins the writer before discarding its attempt.
+Writer failure cancels the reader and its HTTP call. Workflow supersession is a
+CancellationException, which normally would only cancel the child; the consumer
+explicitly cancels its enclosing pipeline scope too. Parent cancellation closes
+the queue/response and joins children. Retry gets a fresh queue and the existing
+fresh attempt state. No late writer can write into an already discarded session.
+
+## Resource limits
+
+| Tier | Existing row batch limit | Estimated raw batch bytes | Queued batches |
+| --- | ---: | ---: | ---: |
+| LOW | 100 | 256 KiB | 1 |
+| MID | 300 | 512 KiB | 2 |
+| HIGH | 500 | 1 MiB | 2 |
+
+These preserve the current row limits; larger tuning candidates remain unselected.
+A byte threshold can flush earlier. A row larger than its batch budget fails
+explicitly instead of being trimmed or admitted into an oversized queue.
+The estimate includes UTF-16 text, category-ID strings and conservative fixed
+object/list overhead. It is an accounting limit, not an exact ART heap reading.
+
+At most `queue capacity + 2` full batches are retained by the writer, queue and
+suspended sender. While assembling a batch, the producer may also hold its next
+decoded row: use `(queue capacity + 3) * batch bytes` as a conservative raw-row
+budget. Mapped channels/entities, parser buffers, metadata, duplicate tracking,
+Room and indexes are additional allocations.
+
+Live streamed JSON rows also have a 1,048,576-character token-read budget, plus
+up to JsonReader's 1,024-character read-ahead. This applies to thin and legacy
+streamed live decoders, including category responses, and includes skipped
+metadata. It rejects pathological fields before building an entire huge token
+or JSON tree. This is a new conservative resource admission rule: large ignored
+metadata can also exceed it. No string is silently truncated. Resource failures
+do not trigger the more expensive legacy decoder.
+
+The existing 100,000-channel store cap bounds duplicate tracking to that count
+plus one incoming batch before rejection. Existing HTTP response caps bound
+metadata/response input. Fallback category accumulation is bounded by admitted
+rows. P4 will evaluate stage uniqueness instead of the in-memory duplicate set.
+
+The shared policy in singleton SyncManager permits one bulk pipeline across all
+providers, so simultaneous imports cannot multiply these queues. Category and
+other section jobs still follow their existing concurrency rules; this is not
+a general app-wide network or memory scheduler. Live low-memory admission and
+runtime abort signals remain active.
+
+## Evidence
+
+Barrier-based tests demonstrate metadata/bulk overlap, adult-context readiness,
+one metadata resolution after failure, byte flushing, bounded queue backpressure,
+final writer drain and parent/reader/writer cancellation. A transport test
+supersedes the writer while the HTTP body is blocked after the first row and
+observes underlying Call cancellation. Room attempt-isolation, retry, partial
+fallback, supersession and sealed snapshot tests remain part of the regression
+suite. Request-stress serialization and shared bulk permits are tested too.
+
+Final regression run: 232 data tests and 16 domain identity tests passed;
+both isolated benchmark APKs built. The fixture/report suite passed 14 tests.
+The benchmark package is `com.streamvault.app.syncbenchmark`, separate from
+ordinary app storage. No user app data was cleared.
+
+Android TV API 36 x86_64 emulator: memory class 192 MB, max Java heap
+201,326,592 bytes. Production LOW row batches (100) and one queued batch were
+used. Background work stayed enabled. All 24 setup/refresh samples below
+(including two warmup samples) retained their full expected channel count and
+passed account cleanup, instrumentation, completion/sample-count checks and
+untruncated request history.
+
+| Scenario | Initial | Settings Live refresh | Evidence |
+| --- | ---: | ---: | --- |
+| 2k rows, 10 categories, all modes | STREAM_ALL 1,018 ms | STREAM_ALL 937 ms | Six samples; AUTO/category about 2.5 s |
+| 10k rows, 100 categories, STREAM_ALL | Median 4,246 ms; max 4,289 ms | Median 4,306 ms; max 8,290 ms | One warmup + five measured pairs |
+| 2k rows, category metadata delayed 1.5 s | 2,725 ms | 2,509 ms | Single pair; bulk begins during metadata wait |
+| 2k rows, bulk returns HTTP 503 | 10,523 ms | 16,805 ms | Single pair; complete category fallback |
+| 50k rows, 200 categories, STREAM_ALL | 21,090 ms | 40,451 ms | Single pair; three completed bulk imports |
+
+Reports under ignored `build/xtream-sync-performance/`:
+
+- All-mode smoke: `20261010T063839794250Z/report.json`.
+- Repeated 10k: `20261010T063927249264Z/report.json`.
+- Delayed metadata: `20261010T064037889878Z/report.json`.
+- Bulk rejection: `20261010T064106880909Z/report.json`.
+- Large catalog: `20261010T064201982860Z/report.json`.
+
+The delayed-metadata request log proves bulk started 2.994 ms after metadata
+started, before its 1,500.719 ms response completed. Request high-water was two.
+Each synthetic account in the healthy small run issued two bulk requests
+(setup + refresh) and no category stream requests. AUTO and explicit category
+mode still issued 20 category requests across each account's two phases.
+
+For the repeated 10k run, maximum estimated outstanding batch bytes were
+178,154; maximum final drain was 86 ms. Median send time was 3,797 ms: the
+reader mostly waited for mapping/staging. Maximum measured sampled Java heap
+was 12,964,864 bytes. Thirteen completed bulk responses were observed for
+twelve setup/refresh samples; one measured account incurred an extra full
+import during its refresh interval, explaining the approximately doubled tail.
+The 50k run similarly had three completed responses for two samples, with
+maximum outstanding batch bytes 179,184 and drain 85 ms. Peak sampled Java
+heap was 59,484,816 bytes on its cold initial sample, falling to 14,574,624
+on refresh. These samples do not isolate all sources of heap allocation.
+
+The extra completed imports are not evidence of a partial-body transport retry.
+Background scheduling/coalescing needs investigation under P6/P7 before making
+latency guarantees. The bulk-rejection account also executed additional import
+work: nine rejected bulk requests and 30 category requests across its lifecycle.
+Fallback is correct but still slow; P5 pacing and duplicate-work handling remain
+necessary.
+
+The 10k payload checksum matches P0/P2. P2's nominal bulk-only initial/refresh
+medians were 4,489/4,566 ms (only two samples per phase); its mixed-strategy tail
+included low-memory category fallback. P3 shows a modest nominal reduction,
+not a controlled speedup ratio. The emulator restarted before P3 and background
+timing differed. No category fallback appeared in the healthy P3 bulk runs,
+which does not prove that memory pressure was fixed. AUTO's release gate
+remains unmet.
+
+The evidence prioritizes P4 writer/mapping and database cost, followed by P5
+category fallback. Include redundant completed live imports in the P6/P7
+scheduling audit. The fixture supplies no playable media. Playback acceptance,
+screen readiness and release-device results remain P8/P0 gates.
+
+## Timing interpretation
+
+`XtreamSyncTiming phase=live-pipeline` records rows, batches, reader elapsed time,
+post-reader drain, time spent sending batches, maximum estimated outstanding
+batch bytes, queue capacity and byte limit. Outstanding bytes include the
+consumer and suspended sender, not just the queue. They exclude the next row
+and a producer batch that has not reached send.
+
+Send time includes scheduling/channel overhead as well as backpressure. Reader
+wall time includes that waiting. Mapping/staging and response durations overlap;
+do not add them together or interpret them as CPU time. The benchmark exporter
+accepts only the exact numeric format and discards general logs.
diff --git a/tools/tests/test_xtream_sync_benchmark.py b/tools/tests/test_xtream_sync_benchmark.py
index 8bfb4b67..f98d8897 100644
--- a/tools/tests/test_xtream_sync_benchmark.py
+++ b/tools/tests/test_xtream_sync_benchmark.py
@@ -11,6 +11,18 @@ SPEC.loader.exec_module(BENCHMARK)
class BenchmarkReportTest(unittest.TestCase):
+ def test_pipeline_metrics_are_numeric_and_exact_format_only(self):
+ log = '\n'.join([
+ 'I/XtreamSyncTiming( 123): phase=live-pipeline rows=100 batches=10 reader_ms=20 drain_ms=3 send_wait_ms=8 outstanding_batch_bytes=1024 queue_capacity=1 batch_byte_limit=4096',
+ 'I/XtreamSyncTiming( 123): phase=live-pipeline rows=100 secret=private',
+ ])
+ records = BENCHMARK.parse_records(log)
+ self.assertEqual(1, len(records))
+ self.assertEqual(1024, records[0]["outstanding_batch_bytes"])
+ self.assertEqual(8, records[0]["send_wait_ms"])
+ self.assertEqual(records, BENCHMARK.summarize(records, [], {})["timings"])
+ self.assertNotIn("private", json.dumps(records))
+
def test_only_benchmark_records_are_retained(self):
log = '\n'.join([
'I/OtherTag( 123): password=private',
diff --git a/tools/xtream_sync_benchmark.py b/tools/xtream_sync_benchmark.py
index a871ead3..04aa9603 100644
--- a/tools/xtream_sync_benchmark.py
+++ b/tools/xtream_sync_benchmark.py
@@ -38,6 +38,12 @@ def parse_records(log: str) -> list[dict]:
records.append({"event": "live-response", "elapsed_ms": int(timing[1]),
"callback_ms": int(timing[2]), "rows": int(timing[3]),
"completed": timing[4] == "true"})
+ pipeline = re.search(r"\bXtreamSyncTiming\s*\([^)]*\): phase=live-pipeline rows=(\d+) batches=(\d+) reader_ms=(\d+) drain_ms=(\d+) send_wait_ms=(\d+) outstanding_batch_bytes=(\d+) queue_capacity=(\d+) batch_byte_limit=(\d+)$", line)
+ if pipeline:
+ records.append(dict(event="live-pipeline", **dict(zip(
+ ["rows", "batches", "reader_ms", "drain_ms", "send_wait_ms",
+ "outstanding_batch_bytes", "queue_capacity", "batch_byte_limit"],
+ map(int, pipeline.groups())))))
stage = re.search(r"\bSyncCatalogStore\s*\([^)]*\): catalog timing provider=(\d+) stage=([a-z-]+(?:\(\d+\))?) took=(\d+)ms$", line)
if stage:
records.append({"event": "catalog-stage", "provider_id": int(stage[1]),
@@ -71,7 +77,7 @@ def summarize(records: list[dict], events: list[dict], metadata: dict) -> dict:
"channel_counts": sorted({s["channels"] for s in selected})})
return {"fixture": metadata, "environment": next((r for r in records if r["event"] == "environment"), None),
"complete": any(r["event"] == "complete" for r in records), "groups": groups, "samples": samples,
- "timings": [r for r in records if r["event"] in {"live-response", "catalog-stage"}],
+ "timings": [r for r in records if r["event"] in {"live-response", "live-pipeline", "catalog-stage"}],
"limits": ["Debug APK, synthetic local server, Java heap sampled every 50 ms.",
"Backend completion only; no UI readiness, native RSS, or playback measurement.",
"Refresh shares an account with setup; account request counts include both phases.",