Skip to content

StreamVault is a TV-first free IPTV player for Android TV built with Kotlin, Jetpack Compose, Room, Hilt, and Media3.

Reduce Xtream live staging and index persistence overhead

David Nashash1cdbabd56b4c
diff --git a/data/build.gradle.kts b/data/build.gradle.kts
index 993eda21..1c609ecf 100644
--- a/data/build.gradle.kts
+++ b/data/build.gradle.kts
@@ -24,6 +24,8 @@ android {
}
testOptions {
+ // Library instrumentation runs as its own APK on current Android.
+ targetSdk = 36
unitTests.isReturnDefaultValues = true
}
diff --git a/data/src/androidTest/java/com/streamvault/data/local/CatalogSyncEquivalenceIntegrationTest.kt b/data/src/androidTest/java/com/streamvault/data/local/CatalogSyncEquivalenceIntegrationTest.kt
index 11aa0ea7..e05cb2aa 100644
--- a/data/src/androidTest/java/com/streamvault/data/local/CatalogSyncEquivalenceIntegrationTest.kt
+++ b/data/src/androidTest/java/com/streamvault/data/local/CatalogSyncEquivalenceIntegrationTest.kt
@@ -75,8 +75,40 @@ class CatalogSyncEquivalenceIntegrationTest {
assertThat(channelSnapshot(fullProviderId)).isEqualTo(channelSnapshot(repairProviderId))
assertThat(movieSnapshot(fullProviderId)).isEqualTo(movieSnapshot(repairProviderId))
assertThat(categorySnapshot(fullProviderId)).isEqualTo(categorySnapshot(repairProviderId))
- assertThat(channelSnapshot(fullProviderId).map { it.streamId }).doesNotContain(999L)
- assertThat(movieSnapshot(fullProviderId).map { it.streamId }).doesNotContain(999L)
+ assertThat(db.channelDao().getByProviderSync(fullProviderId).map { it.streamId }).doesNotContain(999L)
+ assertThat(db.movieDao().getByProviderSync(fullProviderId).map { it.streamId }).doesNotContain(999L)
+ }
+
+ @Test
+ fun liveIndexProjectionPreservesFieldsAndScopesReplacementToOneProvider() = runTest {
+ val providerId = insertProvider("Projection")
+ val otherId = insertProvider("Other")
+ store.replaceLiveCatalog(providerId, emptyList(), listOf(
+ ChannelEntity(streamId = 11, name = "News", providerId = providerId,
+ groupTitle = "Fallback", categoryId = 8, logoUrl = "https://example.test/logo", isAdult = true)
+ ))
+ store.replaceLiveCatalog(otherId, emptyList(), listOf(
+ ChannelEntity(streamId = 12, name = "Other", providerId = otherId)
+ ))
+ val dao = db.xtreamContentIndexDao()
+ assertThat(dao.replaceLiveIndexFromChannels(otherId, 100)).isEqualTo(1)
+ assertThat(dao.replaceLiveIndexFromChannels(providerId, 200)).isEqualTo(1)
+ val channel = db.channelDao().getByProviderSync(providerId).single()
+ val row = requireNotNull(dao.getByRemoteId(providerId, "LIVE", "11"))
+ assertThat(row.localContentId).isEqualTo(channel.id)
+ assertThat(row.name).isEqualTo(channel.name)
+ assertThat(row.categoryId).isEqualTo(8L)
+ assertThat(row.categoryName).isEqualTo("Fallback")
+ assertThat(row.imageUrl).isEqualTo(channel.logoUrl)
+ assertThat(row.isAdult).isTrue()
+ assertThat(row.indexedAt).isEqualTo(200L)
+ assertThat(row.detailHydratedAt).isEqualTo(200L)
+ assertThat(row.syncFingerprint).isEqualTo(channel.syncFingerprint)
+ assertThat(dao.getByRemoteId(otherId, "LIVE", "12")?.indexedAt).isEqualTo(100L)
+ db.channelDao().deleteByProvider(providerId)
+ assertThat(dao.replaceLiveIndexFromChannels(providerId, 300)).isEqualTo(0)
+ assertThat(dao.getByRemoteId(providerId, "LIVE", "11")).isNull()
+ assertThat(dao.getByRemoteId(otherId, "LIVE", "12")).isNotNull()
}
private suspend fun insertProvider(name: String): Long = db.providerDao().insert(
@@ -133,17 +165,18 @@ class CatalogSyncEquivalenceIntegrationTest {
)
private suspend fun channelSnapshot(providerId: Long) = db.channelDao().getByProviderSync(providerId)
- .sortedBy { it.streamId }
- .map { it.copy(id = 0L, providerId = 0L, syncFingerprint = "") }
+ .sortedBy { it.streamUrl }
+ // Synthetic M3U identities deliberately include the provider ID.
+ .map { it.copy(id = 0L, streamId = 0L, categoryId = null, providerId = 0L, syncFingerprint = "") }
private suspend fun movieSnapshot(providerId: Long) = db.movieDao().getByProviderSync(providerId)
- .sortedBy { it.streamId }
- .map { it.copy(id = 0L, providerId = 0L, syncFingerprint = "") }
+ .sortedBy { it.streamUrl }
+ .map { it.copy(id = 0L, streamId = 0L, categoryId = null, providerId = 0L, syncFingerprint = "") }
private suspend fun categorySnapshot(providerId: Long) = ContentType.entries
.flatMap { type -> db.categoryDao().getByProviderAndTypeSync(providerId, type.name) }
- .sortedWith(compareBy(CategoryEntity::type, CategoryEntity::categoryId))
- .map { it.copy(id = 0L, providerId = 0L, syncFingerprint = "") }
+ .sortedWith(compareBy(CategoryEntity::type, CategoryEntity::name))
+ .map { it.copy(id = 0L, categoryId = 0L, providerId = 0L, syncFingerprint = "") }
private companion object {
const val PLAYLIST_URL = "https://catalog.example.test/playlist.m3u"
diff --git a/data/src/androidTest/java/com/streamvault/data/local/XtreamLiveFtsIntegrationTest.kt b/data/src/androidTest/java/com/streamvault/data/local/XtreamLiveFtsIntegrationTest.kt
new file mode 100644
index 00000000..c0ab3487
--- /dev/null
+++ b/data/src/androidTest/java/com/streamvault/data/local/XtreamLiveFtsIntegrationTest.kt
@@ -0,0 +1,91 @@
+package com.streamvault.data.local
+
+import android.content.Context
+import androidx.room.Room
+import androidx.room.testing.MigrationTestHelper
+import androidx.sqlite.db.framework.FrameworkSQLiteOpenHelperFactory
+import androidx.test.core.app.ApplicationProvider
+import androidx.test.ext.junit.runners.AndroidJUnit4
+import androidx.test.platform.app.InstrumentationRegistry
+import com.google.common.truth.Truth.assertThat
+import com.streamvault.data.local.entity.ChannelEntity
+import com.streamvault.data.local.entity.ProviderEntity
+import com.streamvault.data.local.entity.FavoriteEntity
+import com.streamvault.data.sync.SyncCatalogStore
+import com.streamvault.domain.model.ProviderType
+import com.streamvault.domain.model.ContentType
+import kotlinx.coroutines.flow.first
+import kotlinx.coroutines.test.runTest
+import org.junit.Rule
+import org.junit.Test
+import org.junit.runner.RunWith
+
+/** Exercises trigger-maintained search through fresh and migrated production Room databases. */
+@RunWith(AndroidJUnit4::class)
+class XtreamLiveFtsIntegrationTest {
+ @get:Rule
+ val migrations = MigrationTestHelper(
+ InstrumentationRegistry.getInstrumentation(), StreamVaultDatabase::class.java,
+ emptyList(), FrameworkSQLiteOpenHelperFactory()
+ )
+
+ @Test
+ fun freshAndMigratedDatabasesMaintainLiveSearchThroughPromotion() = runTest {
+ val context = ApplicationProvider.getApplicationContext<Context>()
+ for (version in listOf<Int?>(null, 9, 78)) {
+ val name = "xtream-live-fts-${version ?: "fresh"}"
+ context.deleteDatabase(name)
+ if (version != null) migrations.createDatabase(name, version).close()
+ val db = Room.databaseBuilder(context, StreamVaultDatabase::class.java, name)
+ .addMigrations(*StreamVaultDatabaseMigrationRegistry.all.toTypedArray()).build()
+ try {
+ val providerId = db.providerDao().insert(ProviderEntity(name = "Live", type = ProviderType.XTREAM_CODES))
+ val otherId = db.providerDao().insert(ProviderEntity(name = "Other", type = ProviderType.XTREAM_CODES))
+ val store = SyncCatalogStore(db.channelDao(), db.movieDao(), db.seriesDao(),
+ db.categoryDao(), db.catalogSyncDao(), db.tmdbIdentityDao(), RoomDatabaseTransactionRunner(db))
+ suspend fun promote(provider: Long, rows: List<ChannelEntity>, partial: Boolean = false) {
+ val session = store.newSessionId()
+ store.stageAdmittedLiveBatch(provider, session, rows, rows.size)
+ if (partial) store.applyStagedLiveCatalogUpsertOnly(provider, session, null)
+ else store.applyStagedLiveCatalog(provider, session, null)
+ }
+ fun matches(term: String): Int = db.openHelper.writableDatabase.query(
+ "SELECT COUNT(*) FROM channels_fts WHERE channels_fts MATCH ?", arrayOf(term)
+ ).use { it.moveToFirst(); it.getInt(0) }
+ val first = ChannelEntity(streamId = 1, name = "Originaltoken", providerId = providerId)
+ promote(providerId, listOf(first))
+ promote(otherId, listOf(first.copy(providerId = otherId, name = "Othertoken")))
+ val originalId = db.channelDao().getByProviderSync(providerId).single().id
+ db.favoriteDao().insert(FavoriteEntity(providerId = providerId, contentId = originalId, contentType = ContentType.LIVE))
+ db.openHelper.writableDatabase.execSQL("UPDATE channels SET is_user_protected = 1 WHERE id = ?", arrayOf(originalId))
+ assertThat(matches("originaltoken")).isEqualTo(1)
+ promote(providerId, listOf(first)) // unchanged fingerprint
+ assertThat(matches("originaltoken")).isEqualTo(1)
+ promote(providerId, listOf(first.copy(name = "Renamedtoken")))
+ assertThat(matches("originaltoken")).isEqualTo(0)
+ assertThat(matches("renamedtoken")).isEqualTo(1)
+ assertThat(db.channelDao().getByProviderSync(providerId).single().id).isEqualTo(originalId)
+ assertThat(db.channelDao().getByProviderSync(providerId).single().isUserProtected).isTrue()
+ assertThat(db.favoriteDao().getAllGlobal(providerId).first().single().contentId).isEqualTo(originalId)
+ val failedSession = store.newSessionId()
+ store.stageAdmittedLiveBatch(providerId, failedSession, listOf(first.copy(name = "Rollbacktoken")), 1)
+ val failure = runCatching {
+ store.applyStagedLiveCatalog(providerId, failedSession, null) { error("rollback") }
+ }.exceptionOrNull()
+ assertThat(failure).isNotNull()
+ assertThat(matches("rollbacktoken")).isEqualTo(0)
+ assertThat(matches("renamedtoken")).isEqualTo(1)
+ promote(providerId, listOf(first.copy(streamId = 2, name = "Partialtoken")), partial = true)
+ assertThat(matches("renamedtoken")).isEqualTo(1)
+ assertThat(matches("partialtoken")).isEqualTo(1)
+ promote(providerId, listOf(first.copy(streamId = 2, name = "Partialtoken")))
+ assertThat(matches("renamedtoken")).isEqualTo(0)
+ assertThat(matches("partialtoken")).isEqualTo(1)
+ assertThat(matches("othertoken")).isEqualTo(1)
+ } finally {
+ db.close()
+ context.deleteDatabase(name)
+ }
+ }
+ }
+}
diff --git a/data/src/main/java/com/streamvault/data/local/dao/XtreamIndexDaos.kt b/data/src/main/java/com/streamvault/data/local/dao/XtreamIndexDaos.kt
index aca25924..a3bc2a92 100644
--- a/data/src/main/java/com/streamvault/data/local/dao/XtreamIndexDaos.kt
+++ b/data/src/main/java/com/streamvault/data/local/dao/XtreamIndexDaos.kt
@@ -109,6 +109,31 @@ abstract class XtreamContentIndexDao {
@Query("DELETE FROM xtream_content_index WHERE provider_id = :providerId AND content_type = :contentType")
abstract suspend fun deleteByProviderAndType(providerId: Long, contentType: String): Int
+ @Query("""
+ INSERT INTO xtream_content_index (
+ provider_id, content_type, remote_id, local_content_id, name,
+ category_id, category_name, image_url, container_extension, rating,
+ added_at, remote_updated_at, is_adult, indexed_at, detail_hydrated_at,
+ stale_state, error_state, sync_fingerprint
+ )
+ SELECT provider_id, 'LIVE', CAST(stream_id AS TEXT),
+ CASE WHEN id > 0 THEN id ELSE NULL END, name,
+ category_id, COALESCE(category_name, group_title), logo_url, NULL, 0,
+ 0, 0, is_adult, :indexedAt, :indexedAt, 'ACTIVE', NULL, sync_fingerprint
+ FROM channels WHERE provider_id = :providerId
+ """)
+ protected abstract suspend fun insertLiveIndexFromChannels(providerId: Long, indexedAt: Long)
+
+ @Query("SELECT COUNT(*) FROM xtream_content_index WHERE provider_id = :providerId AND content_type = 'LIVE'")
+ protected abstract suspend fun countLiveIndex(providerId: Long): Int
+
+ @Transaction
+ open suspend fun replaceLiveIndexFromChannels(providerId: Long, indexedAt: Long): Int {
+ deleteByProviderAndType(providerId, "LIVE")
+ insertLiveIndexFromChannels(providerId, indexedAt)
+ return countLiveIndex(providerId)
+ }
+
@Query(
"""
DELETE FROM movies
diff --git a/data/src/main/java/com/streamvault/data/sync/SyncCatalogStore.kt b/data/src/main/java/com/streamvault/data/sync/SyncCatalogStore.kt
index 753f99c3..4dd9f6c2 100644
--- a/data/src/main/java/com/streamvault/data/sync/SyncCatalogStore.kt
+++ b/data/src/main/java/com/streamvault/data/sync/SyncCatalogStore.kt
@@ -42,6 +42,8 @@ internal class SyncCatalogStore(
companion object {
private const val TAG = "SyncCatalogStore"
private const val STAGE_BATCH_SIZE = 500
+ private val TEXT_WHITESPACE = Regex("\\s+")
+ private const val HEX_DIGITS = "0123456789abcdef"
}
data class StagedLiveImportState(
@@ -163,7 +165,7 @@ internal class SyncCatalogStore(
categories?.let { timed(providerId, "category-staging") { stageCategories(providerId, sessionId, it) } }
categories?.let { timed(providerId, "category-commit") { applyCategories(providerId, sessionId, "LIVE") } }
timed(providerId, "channel-insert-update-delete") { applyChannels(providerId, sessionId) }
- timed(providerId, "channel-fts-rebuild") { catalogSyncDao.rebuildChannelFts() }
+ // Room's content-sync triggers update search for these changed rows.
afterCatalogApply()
}
} finally {
@@ -228,7 +230,7 @@ internal class SyncCatalogStore(
categories?.let { stageCategories(providerId, sessionId, it) }
categories?.let { applyCategories(providerId, sessionId, "LIVE", pruneStale = false) }
upsertChannels(providerId, sessionId)
- catalogSyncDao.rebuildChannelFts()
+ // Partial promotion uses the same trigger-maintained search index.
afterCatalogApply()
}
} finally {
@@ -369,15 +371,36 @@ internal class SyncCatalogStore(
}
suspend fun stageChannelBatch(providerId: Long, sessionId: Long, channels: List<ChannelEntity>) {
+ stageChannelBatchInternal(providerId, sessionId, channels, null)
+ }
+
+ /** Only for fresh, isolated attempts whose writer admits each ID once. */
+ suspend fun stageAdmittedLiveBatch(providerId: Long, sessionId: Long, channels: List<ChannelEntity>, admittedCount: Int) {
+ stageChannelBatchInternal(providerId, sessionId, channels, admittedCount)
+ }
+
+ private suspend fun stageChannelBatchInternal(providerId: Long, sessionId: Long, channels: List<ChannelEntity>, admittedCount: Int?) {
timed(providerId, "channel-staging-batch(${channels.size})") {
+ val stages = buildChannelStages(providerId, sessionId, channels)
providerTransaction(providerId) {
+ val previousCount = if (admittedCount == null) {
+ catalogSyncDao.countChannelStages(providerId, sessionId)
+ } else {
+ require(admittedCount >= stages.size)
+ if (admittedCount == stages.size) {
+ // Establish the fresh-session invariant once. Recovery and
+ // generic writers still use database accounting.
+ check(catalogSyncDao.countChannelStages(providerId, sessionId) == 0)
+ }
+ admittedCount - stages.size
+ }
requireStageCapacity(
- currentCount = catalogSyncDao.countChannelStages(providerId, sessionId),
- incomingCount = channels.distinctBy { it.streamId }.size,
+ currentCount = previousCount,
+ incomingCount = stages.size,
maximum = sizeLimits.maxChannelsPerProvider,
label = "channel"
)
- insertStageRows(buildChannelStages(providerId, sessionId, channels), catalogSyncDao::insertChannelStages)
+ insertStageRows(stages, catalogSyncDao::insertChannelStages)
}
}
}
@@ -1104,14 +1127,21 @@ internal class SyncCatalogStore(
messageDigest.update(value.toByteArray(Charsets.UTF_8))
messageDigest.update(0)
}
- return messageDigest.digest().joinToString("") { byte -> "%02x".format(byte.toInt() and 0xff) }
+ val bytes = messageDigest.digest()
+ val hex = CharArray(bytes.size * 2)
+ bytes.forEachIndexed { index, byte ->
+ val value = byte.toInt() and 0xff
+ hex[index * 2] = HEX_DIGITS[value ushr 4]
+ hex[index * 2 + 1] = HEX_DIGITS[value and 0xf]
+ }
+ return String(hex)
}
private fun normalizeText(value: String?): String {
return value
.orEmpty()
.trim()
- .replace(Regex("\\s+"), " ")
+ .replace(TEXT_WHITESPACE, " ")
.lowercase()
}
diff --git a/data/src/main/java/com/streamvault/data/sync/SyncManager.kt b/data/src/main/java/com/streamvault/data/sync/SyncManager.kt
index d614e906..ac47c739 100644
--- a/data/src/main/java/com/streamvault/data/sync/SyncManager.kt
+++ b/data/src/main/java/com/streamvault/data/sync/SyncManager.kt
@@ -756,7 +756,9 @@ class SyncManager @Inject constructor(
categoryFailureWarning = ::categoryFailureWarning,
liveCategorySequentialModeWarning = LIVE_CATEGORY_SEQUENTIAL_MODE_WARNING,
isCurrentlyLowOnMemory = applicationContext::isCurrentlyLowOnMemoryForSync,
- stageChannelItems = catalogStager::stageChannelItems,
+ stageChannelItems = { id, items, seen, categories, session ->
+ catalogStager.stageChannelItems(id, items, seen, categories, session, useLiveAdmissionCount = true)
+ },
emitProgress = ::emitProviderProgress
)
}
@@ -2118,29 +2120,9 @@ class SyncManager @Inject constructor(
}
private suspend fun backfillXtreamLiveIndex(providerId: Long, indexedAt: Long): Int {
- val channels = channelDao.getByProviderSync(providerId)
- if (channels.isEmpty()) return 0
- val indexRows = channels.map { channel ->
- XtreamContentIndexEntity(
- providerId = providerId,
- contentType = ContentType.LIVE,
- remoteId = channel.streamId.toString(),
- localContentId = channel.id.takeIf { it > 0 },
- name = channel.name,
- categoryId = channel.categoryId,
- categoryName = channel.categoryName ?: channel.groupTitle,
- imageUrl = channel.logoUrl,
- isAdult = channel.isAdult,
- indexedAt = indexedAt,
- detailHydratedAt = indexedAt,
- syncFingerprint = channel.syncFingerprint
- )
- }
- transactionRunner.inTransaction {
- xtreamContentIndexDao.deleteByProviderAndType(providerId, ContentType.LIVE.name)
- xtreamContentIndexDao.upsertAll(indexRows)
+ return syncCatalogStore.withLiveCatalogFence(providerId) {
+ xtreamContentIndexDao.replaceLiveIndexFromChannels(providerId, indexedAt)
}
- return channels.size
}
private suspend fun upsertXtreamIndexJob(
diff --git a/data/src/main/java/com/streamvault/data/sync/SyncManagerCatalogStager.kt b/data/src/main/java/com/streamvault/data/sync/SyncManagerCatalogStager.kt
index 01493022..d3304430 100644
--- a/data/src/main/java/com/streamvault/data/sync/SyncManagerCatalogStager.kt
+++ b/data/src/main/java/com/streamvault/data/sync/SyncManagerCatalogStager.kt
@@ -165,7 +165,8 @@ internal class SyncManagerCatalogStager(
items: List<Channel>,
seenStreamIds: MutableSet<Long>,
fallbackCollector: FallbackCategoryCollector,
- sessionId: Long?
+ sessionId: Long?,
+ useLiveAdmissionCount: Boolean = false
): StagedCatalogSnapshot {
var resolvedSessionId = sessionId
val acceptedEntities = ArrayList<ChannelEntity>(items.size)
@@ -182,12 +183,16 @@ internal class SyncManagerCatalogStager(
syncCatalogStore.clearProviderStaging(providerId)
resolvedSessionId = syncCatalogStore.newSessionId()
}
- syncCatalogStore.stageChannelBatch(providerId, requireNotNull(resolvedSessionId), acceptedEntities)
+ if (useLiveAdmissionCount) {
+ syncCatalogStore.stageAdmittedLiveBatch(providerId, requireNotNull(resolvedSessionId), acceptedEntities, seenStreamIds.size)
+ } else {
+ syncCatalogStore.stageChannelBatch(providerId, requireNotNull(resolvedSessionId), acceptedEntities)
+ }
}
return StagedCatalogSnapshot(
sessionId = resolvedSessionId,
acceptedCount = acceptedEntities.size,
- fallbackCategories = fallbackCollector.entities().takeIf { it.isNotEmpty() }
+ fallbackCategories = if (useLiveAdmissionCount) null else fallbackCollector.entities().takeIf { it.isNotEmpty() }
)
}
diff --git a/data/src/main/java/com/streamvault/data/sync/SyncManagerSupport.kt b/data/src/main/java/com/streamvault/data/sync/SyncManagerSupport.kt
index 1d5873c6..294754fa 100644
--- a/data/src/main/java/com/streamvault/data/sync/SyncManagerSupport.kt
+++ b/data/src/main/java/com/streamvault/data/sync/SyncManagerSupport.kt
@@ -173,6 +173,13 @@ internal class FallbackCategoryCollector(
val resolvedCategoryId = categoryId ?: return
val resolvedCategoryName = categoryName?.trim().takeUnless { it.isNullOrEmpty() }
?: "Category $resolvedCategoryId"
+ val existing = categories[resolvedCategoryId]
+ if (existing != null && existing.name == resolvedCategoryName) {
+ // This name was classified when first admitted. Explicit adult flags
+ // can still strengthen it on a later row.
+ if (isAdult && !existing.isAdult) categories[resolvedCategoryId] = existing.copy(isAdult = true)
+ return
+ }
val candidate = CategoryEntity(
categoryId = resolvedCategoryId,
name = resolvedCategoryName,
@@ -181,7 +188,6 @@ internal class FallbackCategoryCollector(
providerId = providerId,
isAdult = isAdult || AdultContentClassifier.isAdultCategoryName(resolvedCategoryName)
)
- val existing = categories[resolvedCategoryId]
categories[resolvedCategoryId] = if (existing == null) {
candidate
} else {
diff --git a/data/src/test/java/com/streamvault/data/sync/FallbackCategoryCollectorTest.kt b/data/src/test/java/com/streamvault/data/sync/FallbackCategoryCollectorTest.kt
new file mode 100644
index 00000000..63f4386a
--- /dev/null
+++ b/data/src/test/java/com/streamvault/data/sync/FallbackCategoryCollectorTest.kt
@@ -0,0 +1,30 @@
+package com.streamvault.data.sync
+
+import com.google.common.truth.Truth.assertThat
+import com.streamvault.domain.model.ContentType
+import org.junit.Test
+
+class FallbackCategoryCollectorTest {
+ @Test
+ fun `repeated names retain ordering and allow adult promotion`() {
+ val collector = FallbackCategoryCollector(7, ContentType.LIVE)
+ collector.record(12, "News", false)
+ collector.record(13, "Sports", false)
+ repeat(500) { collector.record(12, " News ", false) }
+ collector.record(12, "News", true)
+ collector.record(12, "News", false)
+ assertThat(collector.entities().map { it.categoryId }).containsExactly(12L, 13L).inOrder()
+ assertThat(collector.entities().first().isAdult).isTrue()
+ }
+
+ @Test
+ fun `placeholder can be replaced and alternate names still strengthen adult classification`() {
+ val collector = FallbackCategoryCollector(7, ContentType.LIVE)
+ collector.record(12, null, false)
+ collector.record(12, "News", false)
+ collector.record(12, "XXX", false)
+ val category = collector.entities().single()
+ assertThat(category.name).isEqualTo("News")
+ assertThat(category.isAdult).isTrue()
+ }
+}
diff --git a/data/src/test/java/com/streamvault/data/sync/SyncCatalogStoreTest.kt b/data/src/test/java/com/streamvault/data/sync/SyncCatalogStoreTest.kt
index 65691753..76cb28a5 100644
--- a/data/src/test/java/com/streamvault/data/sync/SyncCatalogStoreTest.kt
+++ b/data/src/test/java/com/streamvault/data/sync/SyncCatalogStoreTest.kt
@@ -67,6 +67,45 @@ class SyncCatalogStoreTest {
}
}
+ @Test
+ fun `live admission counts check fresh session once and enforce the limit`() = runTest {
+ val store = store(sizeLimits = CatalogSizeLimits(maxChannelsPerProvider = 2))
+ val first = ChannelEntity(streamId = 1, name = "One", providerId = 7)
+ store.stageAdmittedLiveBatch(7, 90, listOf(first), 1)
+ store.stageAdmittedLiveBatch(7, 90, listOf(first.copy(streamId = 2)), 2)
+ val failure = runCatching {
+ store.stageAdmittedLiveBatch(7, 90, listOf(first.copy(streamId = 3)), 3)
+ }.exceptionOrNull()
+ assertThat(failure).isNotNull()
+ verify(catalogSyncDao).countChannelStages(7, 90)
+ verify(catalogSyncDao, org.mockito.kotlin.times(2)).insertChannelStages(any())
+ }
+
+ @Test
+ fun `live admission refuses a nonempty initial session`() = runTest {
+ whenever(catalogSyncDao.countChannelStages(7, 90)).thenReturn(1)
+ val failure = runCatching {
+ store().stageAdmittedLiveBatch(7, 90, listOf(ChannelEntity(streamId = 1, name = "One", providerId = 7)), 1)
+ }.exceptionOrNull()
+ assertThat(failure).isInstanceOf(IllegalStateException::class.java)
+ verify(catalogSyncDao, never()).insertChannelStages(any())
+ }
+
+ @Test
+ fun `category fingerprints retain the existing persisted encoding`() = runTest {
+ val category = CategoryEntity(categoryId = 12, name = " NEWS\t世界 ", providerId = 7)
+ store().stageCategories(7, 90, listOf(category))
+ val rows = argumentCaptor<List<CategoryImportStageEntity>>()
+ verify(catalogSyncDao).insertCategoryStages(rows.capture())
+ val legacyDigest = java.security.MessageDigest.getInstance("SHA-256")
+ listOf("12", "news 世界", category.parentId?.toString().orEmpty(), "LIVE", category.providerOrder.toString(), "false").forEach {
+ legacyDigest.update(it.toByteArray(Charsets.UTF_8))
+ legacyDigest.update(0)
+ }
+ val legacyFingerprint = legacyDigest.digest().joinToString("") { "%02x".format(it.toInt() and 0xff) }
+ assertThat(rows.firstValue.single().syncFingerprint).isEqualTo(legacyFingerprint)
+ }
+
@Test
fun `replaceLiveCatalog batches changed category and channel updates`() = runTest {
val providerId = 7L
@@ -145,18 +184,18 @@ class SyncCatalogStoreTest {
}
@Test
- fun `applyStagedLiveCatalog rebuilds channel fts inside transaction`() = runTest {
+ fun `applyStagedLiveCatalog promotes inside transaction without rebuilding global fts`() = runTest {
val runner = TrackingTransactionRunner()
doAnswer {
assertThat(runner.isInTransaction).isTrue()
Unit
- }.whenever(catalogSyncDao).rebuildChannelFts()
+ }.whenever(catalogSyncDao).updateChangedChannelsFromStage(7L, 55L)
store(transactionRunner = runner).applyStagedLiveCatalog(providerId = 7L, sessionId = 55L, categories = null)
// One transaction applies the catalog and one atomically clears its staging session.
assertThat(runner.calls).isEqualTo(2)
- verify(catalogSyncDao).rebuildChannelFts()
+ verify(catalogSyncDao, never()).rebuildChannelFts()
}
@Test
diff --git a/data/src/test/java/com/streamvault/data/sync/XtreamLiveAttemptIsolationTest.kt b/data/src/test/java/com/streamvault/data/sync/XtreamLiveAttemptIsolationTest.kt
index 6eb8e262..aee5417f 100644
--- a/data/src/test/java/com/streamvault/data/sync/XtreamLiveAttemptIsolationTest.kt
+++ b/data/src/test/java/com/streamvault/data/sync/XtreamLiveAttemptIsolationTest.kt
@@ -262,7 +262,7 @@ class XtreamLiveAttemptIsolationTest {
{ _, _ -> "full failed" }, { _, _, _ -> "category failed" }, "sequential", stageChannelItems = { id, channels, seen, collector, session ->
beforeStage(channels)
if (failWriter || channels.any { it.streamId == failOnStreamId }) throw ProviderWorkflowSupersededException(id, 1)
- stager.stageChannelItems(id, channels, seen, collector, session)
+ stager.stageChannelItems(id, channels, seen, collector, session, useLiveAdmissionCount = true)
}, emitProgress = { _, _ -> }) to api
}
diff --git a/docs/XTREAM_SYNC_DATABASE_IMPLEMENTATION.md b/docs/XTREAM_SYNC_DATABASE_IMPLEMENTATION.md
new file mode 100644
index 00000000..d5ba7b62
--- /dev/null
+++ b/docs/XTREAM_SYNC_DATABASE_IMPLEMENTATION.md
@@ -0,0 +1,96 @@
+# Xtream live database write improvements
+
+P4 on `codex/xtream-live-sync-performance`, following the
+[improvement plan](XTREAM_SYNC_IMPROVEMENT_PLAN.md).
+
+## Changes and invariants
+
+The isolated Xtream live writer uses its unique-ID admission count for the
+per-provider staging limit. The first batch checks that its private session is
+empty. Later batches avoid a growing-table COUNT query. The writer advances its
+successful accepted count only after insertion returns; a failed write ends and
+discards the attempt under P1 rules. Completed-category merges, completion seals
+and recovery still reconcile actual database counts. General M3U/Stalker writers
+keep database accounting. The optimized entry point must not be used to append
+to a recovered session or with a duplicate/nonexclusive admission set.
+
+Stage entities and fingerprints are prepared before entering the fenced write
+transaction. SHA-256 inputs, delimiters and lowercase hexadecimal output remain
+compatible with existing persisted fingerprints. Hex encoding uses a lookup
+table, and text normalization reuses its whitespace expression. Unchanged-row
+promotion continues to compare fingerprints and preserves local channel IDs.
+
+Fallback categories classify repeated names once and still allow later adult
+flags or alternate adult names to strengthen classification. Only placeholders
+can be renamed. The live writer materializes the category snapshot at completion
+instead of copying the accumulated category list after every batch.
+
+Live index backfill uses a provider-scoped transactional DELETE plus INSERT
+SELECT from channels, under the workflow commit fence. It preserves the previous
+field mapping and defaults without allocating two full Kotlin lineup lists.
+An empty lineup clears obsolete live index rows; other providers and content
+types are untouched.
+
+Full and partial staged live promotion rely on Room's FTS content-sync triggers
+instead of rebuilding the global channel search index. Explicit rebuild paths
+and other content-type rebuilds remain. No schema migration is required.
+
+## Validation
+
+- 151 focused Kotlin unit tests passed, including attempt retry isolation,
+ supersession/recovery, fresh-session checks, size limits, fingerprint encoding,
+ adult category promotion and existing sync regression tests.
+- 14 Python fixture/benchmark tests passed.
+- Real emulator Room tests passed: SQL projection fields, provider isolation,
+ empty-lineup cleanup and full/section M3U catalog equivalence.
+- FTS tests passed on fresh Room and actual databases migrated from v9 and v78.
+ They exercise initial insert, unchanged refresh, rename, partial insertion,
+ stale deletion, another provider, transaction rollback, stable local IDs,
+ favorite linkage and user protection through production staged promotion.
+ Before removing rebuilds, the same search matrix also passed through the
+ production promotion DAO calls without a rebuild.
+- Data instrumentation now targets API 36. Its previously unspecified target
+ caused Android's old-app installation block. This affects the library test APK.
+- The existing M3U equivalence assertion was corrected to normalize generated
+ identities across providers; provider IDs intentionally affect those hashes.
+ Stale-row assertions still inspect real persisted remote IDs.
+
+## Measured results and limits
+
+The 10,000-row/100-category synthetic fixture has the same payload SHA-256 as P3:
+`4dfbcaf7e39ce168eb4cd58e34b5d8c78dddf4592d6638cb46c76b0fb31d63a3`.
+On the API 36 emulator with the isolated debug app, five measured initial/refresh
+pairs plus one warmup pair produced these medians:
+
+| Operation | P3 | P4 before FTS removal | Change |
+| --- | ---: | ---: | ---: |
+| Add Provider backend | 4,245.5 ms | 3,883.6 ms | 8.5% lower |
+| Settings Live refresh | 4,305.8 ms | 3,809.0 ms | 11.5% lower |
+
+Report: `build/xtream-sync-performance/20261010T074712205202Z/report.json`.
+All 12 samples retained exactly 10,000 channels and cleanup/count assertions
+passed. Thirteen complete bulk responses occurred: one account had a redundant
+third import, with a 7,831 ms refresh tail. This scheduling issue remains open.
+Aggregate staging time across those 13 imports decreased from 21,982 to 20,043
+ms. Peak sampled Java heap among measured pairs was 9,958,416 bytes; sampling
+does not capture exact peak RSS or all native SQLite memory.
+
+After FTS removal, all three modes passed initial and refresh coverage at 2,000
+rows/10 categories, with no routine `channel-fts-rebuild` timing event. Report:
+`build/xtream-sync-performance/20261010T075202281663Z/report.json`.
+Category mode still incurs its old pacing floor; one refresh had an extra import.
+These single samples are coverage checks, not comparable performance medians.
+
+The final 50,000-row/200-category coverage run also passed both paths:
+`build/xtream-sync-performance/20261010T075233892541Z/report.json`.
+Initial import took 24,672 ms and refresh took 49,078 ms, versus P3's single
+21,090/40,451 ms samples. Both runs performed three complete bulk imports for
+two requested operations. This larger case does not show a latency improvement;
+it retains the duplicate-refresh problem and requires more investigation.
+The final run contained no global channel FTS rebuild timing events.
+
+This is backend debug-emulator evidence. The emulator was restarted to resolve
+the test installation investigation, and repeated runs can differ in device
+state. Release-device end-to-end readiness, LOW/MID/HIGH acceptance and sustained
+real-channel playback remain open. AUTO remains unchanged until P6. These
+results do not establish a tenfold overall improvement.
diff --git a/docs/XTREAM_SYNC_IMPROVEMENT_PLAN.md b/docs/XTREAM_SYNC_IMPROVEMENT_PLAN.md
index ba89e5df..92529200 100644
--- a/docs/XTREAM_SYNC_IMPROVEMENT_PLAN.md
+++ b/docs/XTREAM_SYNC_IMPROVEMENT_PLAN.md
@@ -11,6 +11,7 @@ Supporting evidence:
- [Attempt isolation and recovery implementation](XTREAM_SYNC_RECOVERY_IMPLEMENTATION.md)
- [Live parsing and group identity implementation](XTREAM_SYNC_PARSING_IMPLEMENTATION.md)
- [Bounded live pipeline implementation](XTREAM_SYNC_PIPELINE_IMPLEMENTATION.md)
+- [Database write and search implementation](XTREAM_SYNC_DATABASE_IMPLEMENTATION.md)
The reference implementation provides hypotheses to investigate, not a design
authority or proof of correctness. Choose behavior for this app's measured
@@ -286,7 +287,10 @@ P1 attempt isolation and conservative receipt-based recovery are implemented and
validated. P2 direct live decoding and cheaper group identity are implemented
with field-equivalence checks and measured component CPU reductions. P3 retrieval
overlap and a bounded reader/writer pipeline are implemented with cancellation,
-resource-bound and real-graph setup/refresh evidence. P4–P8 remain pending.
+resource-bound and real-graph setup/refresh evidence. P4 stage accounting,
+fingerprint costs, SQL live-index projection and trigger-maintained search are
+implemented with fresh/migrated Room checks and measured backend improvements.
+P5–P8 remain pending.
The new measurements identify writer cost, slow fallback and extra completed
imports during some refresh intervals as remaining priorities. Earlier runs
also showed memory-driven category fallback; the default-strategy gate has not