Colota - Android GPS Location tracker
fix(sync): run one sync pass at a time (#843)
Max Dietrichc4c6ce8b572b
diff --git a/apps/docs/docs/development/architecture.md b/apps/docs/docs/development/architecture.mdindex 43aed2c4..d7f3fea9 100644--- a/apps/docs/docs/development/architecture.md+++ b/apps/docs/docs/development/architecture.md@@ -208,6 +208,7 @@ Orchestrates batch location uploads with:- Exponential backoff on failure- Periodic sync scheduling- Manual flush support+- One pass at a time: a periodic tick and a manual flush wait for the running pass. An instant send leaves its row queued while a pass runs, and a pass skips rows an instant send is still posting### NetworkManagerdiff --git a/apps/docs/docs/guides/data-management.md b/apps/docs/docs/guides/data-management.mdindex 4bd0e678..b0f19d22 100644--- a/apps/docs/docs/guides/data-management.md+++ b/apps/docs/docs/guides/data-management.md@@ -14,7 +14,7 @@ import ScreenshotGallery from '@site/src/components/ScreenshotGallery'| Action | Description || ---------------------------- | -------------------------------------------------------------------------- |-| **Sync Now** | Manually flush the queue and upload pending locations |+| **Sync Now** | Manually flush the queue and upload pending locations. A sync already running finishes first || **Clear Sent History** | Remove locations that have already been synced || **Clear Queue** | Remove unsent locations from the upload queue || **Delete Older Than X Days** | Clean up old data past a specified age |diff --git a/apps/mobile/android/app/src/main/java/com/colota/service/LocationForegroundService.kt b/apps/mobile/android/app/src/main/java/com/colota/service/LocationForegroundService.ktindex 2b048228..2927affe 100644--- a/apps/mobile/android/app/src/main/java/com/colota/service/LocationForegroundService.kt+++ b/apps/mobile/android/app/src/main/java/com/colota/service/LocationForegroundService.kt@@ -1301,6 +1301,7 @@ class LocationForegroundService : Service() {* concerns here - gating the save on a successful send loses the stay entirely whenever* the server is unreachable or sync is not allowed. The send skips the sync interval: on* Power Saver a queued stay reaches the endpoint 15 minutes after the user got home.+ * While a sync pass runs, the point waits in the queue for that pass or the next.*/private suspend fun recordHeartbeatLocation() {if (!::config.isInitialized) {diff --git a/apps/mobile/android/app/src/main/java/com/colota/sync/SyncManager.kt b/apps/mobile/android/app/src/main/java/com/colota/sync/SyncManager.ktindex d174309f..592a480e 100644--- a/apps/mobile/android/app/src/main/java/com/colota/sync/SyncManager.kt+++ b/apps/mobile/android/app/src/main/java/com/colota/sync/SyncManager.kt@@ -10,7 +10,10 @@ import com.Colota.bridge.LocationServiceModuleimport com.Colota.data.DatabaseHelperimport com.Colota.util.TimedCacheimport kotlinx.coroutines.*+import kotlinx.coroutines.sync.Mutex+import kotlinx.coroutines.sync.withLockimport org.json.JSONObject+import java.util.concurrent.ConcurrentHashMap/*** Two sync modes: instant (syncInterval=0) sends each location on arrival,@@ -46,6 +49,12 @@ class SyncManager(private val queueCountCache = TimedCache(5000L) { dbHelper.getQueuedCount() }+ /** One pass at a time: two passes fetch the same oldest rows and post each of them twice. */+ private val syncMutex = Mutex()++ /** Queue rows an instant send is posting; a pass skips them instead of posting them a second time. */+ private val instantInFlight: MutableSet<Long> = ConcurrentHashMap.newKeySet()+fun updateConfig(endpoint: String,syncIntervalSeconds: Int,@@ -88,21 +97,24 @@ class SyncManager(}if (endpoint.isNotBlank() && queued > 0) {- val errorMessage: String? = try {- val success = performSyncAndCheckSuccess()-- if (success) {- if (consecutiveFailures > 0) {- AppLogger.i(TAG, "Sync restored")+ if (syncMutex.isLocked) AppLogger.d(TAG, "Sync tick waiting for the running sync")+ val errorMessage: String? = syncMutex.withLock {+ try {+ val success = performSyncAndCheckSuccess()++ if (success) {+ if (consecutiveFailures > 0) {+ AppLogger.i(TAG, "Sync restored")+ }+ consecutiveFailures = 0+ null+ } else {+ "Sync failed"}- consecutiveFailures = 0- null- } else {- "Sync failed"+ } catch (e: Exception) {+ AppLogger.e(TAG, "Sync error", e)+ e.message ?: "Sync error"}- } catch (e: Exception) {- AppLogger.e(TAG, "Sync error", e)- e.message ?: "Sync error"}if (errorMessage != null) {@@ -128,9 +140,12 @@ class SyncManager(suspend fun manualFlush() {if (endpoint.isNotBlank()) {- val total = dbHelper.getQueuedCount()- AppLogger.d(TAG, "Manual flush started: $total items in queue")- syncQueue { sent, failed -> LocationServiceModule.sendSyncProgressEvent(sent, failed, total) }+ if (syncMutex.isLocked) AppLogger.d(TAG, "Manual flush waiting for the running sync")+ syncMutex.withLock {+ val total = dbHelper.getQueuedCount()+ AppLogger.d(TAG, "Manual flush started: $total items in queue")+ syncQueue { sent, failed -> LocationServiceModule.sendSyncProgressEvent(sent, failed, total) }+ }}}@@ -151,16 +166,26 @@ class SyncManager(// Immediate send mode (syncInterval = 0)if ((syncIntervalSeconds == 0 || bypassInterval) && isSyncAllowed()) {- AppLogger.d(TAG, "Instant send")- val success = networkManager.sendToEndpoint(payload, endpoint, authHeaders, httpMethod, apiFormat)-- if (success) {- dbHelper.markLocationsSent(listOf(locationId))- dbHelper.removeFromQueueByLocationId(locationId)- invalidateQueueCache()- lastSuccessfulSyncTime = System.currentTimeMillis()- } else {- dbHelper.incrementRetryCount(queueId, "Send failed")+ // Registered before the lock check: a later pass skips this row, a running one may already hold it+ instantInFlight.add(queueId)+ try {+ if (syncMutex.isLocked) {+ AppLogger.d(TAG, "Instant send deferred: a sync is running")+ return+ }+ AppLogger.d(TAG, "Instant send")+ val success = networkManager.sendToEndpoint(payload, endpoint, authHeaders, httpMethod, apiFormat)++ if (success) {+ dbHelper.markLocationsSent(listOf(locationId))+ dbHelper.removeFromQueueByLocationId(locationId)+ invalidateQueueCache()+ lastSuccessfulSyncTime = System.currentTimeMillis()+ } else {+ dbHelper.incrementRetryCount(queueId, "Send failed")+ }+ } finally {+ instantInFlight.remove(queueId)}}// Otherwise the periodic sync job will handle it@@ -186,12 +211,13 @@ class SyncManager(private suspend fun performSyncAndCheckSuccess(): Boolean {val countBefore = dbHelper.getQueuedCount()- syncQueue(onProgress = null)+ val pass = syncQueue(onProgress = null)val countAfter = dbHelper.getQueuedCount()invalidateQueueCache()- val success = countAfter < countBefore || countAfter == 0+ // A pass that only skipped rows an instant send was still posting did not fail+ val success = countAfter < countBefore || countAfter == 0 || (pass.sent + pass.failed == 0 && pass.skippedInFlight > 0)if (success && countAfter == 0) {lastSuccessfulSyncTime = System.currentTimeMillis()}@@ -213,7 +239,7 @@ class SyncManager(delay(backoffSeconds * 1000L)}- private suspend fun syncQueue(onProgress: ((sent: Int, failed: Int) -> Unit)? = null) = coroutineScope {+ private suspend fun syncQueue(onProgress: ((sent: Int, failed: Int) -> Unit)? = null): SyncPass = coroutineScope {// Snapshot volatile config so it stays consistent for the entire sync passval currentEndpoint = endpointval currentAuthHeaders = authHeaders@@ -223,11 +249,14 @@ class SyncManager(var totalProcessed = 0var totalSucceeded = 0var totalFailed = 0+ var skippedInFlight = 0var batchNumber = 1while (isActive && batchNumber <= MAX_BATCHES_PER_SYNC) {val fetchSize = if (currentApiFormat == ApiFormat.OVERLAND_BATCH) currentBatchSize else 50- val queued = dbHelper.getQueuedLocations(fetchSize)+ val fetched = dbHelper.getQueuedLocations(fetchSize)+ val queued = fetched.filterNot { it.queueId in instantInFlight }+ skippedInFlight += fetched.size - queued.sizeif (queued.isEmpty()) {if (totalProcessed > 0) {AppLogger.d(TAG, "Sync complete: $totalProcessed items in $batchNumber batches")@@ -314,8 +343,12 @@ class SyncManager(if (totalSucceeded > 0) {lastSuccessfulSyncTime = System.currentTimeMillis()}++ SyncPass(totalSucceeded, totalFailed, skippedInFlight)}+ data class SyncPass(val sent: Int, val failed: Int, val skippedInFlight: Int = 0)+private data class BatchSendResult(val processed: Int, val failed: Int, val stop: Boolean)/**diff --git a/apps/mobile/android/app/src/test/java/com/Colota/sync/SyncManagerTest.kt b/apps/mobile/android/app/src/test/java/com/Colota/sync/SyncManagerTest.ktindex 4f3e2150..7e14b165 100644--- a/apps/mobile/android/app/src/test/java/com/Colota/sync/SyncManagerTest.kt+++ b/apps/mobile/android/app/src/test/java/com/Colota/sync/SyncManagerTest.kt@@ -1250,6 +1250,180 @@ class SyncManagerTest {}}+ // ========================================================================+ // One sender at a time+ // ========================================================================++ @Test+ fun `a manual flush during another waits instead of posting the same rows again`() = scope.runTest {+ syncManager.updateConfig(+ endpoint = "https://example.com",+ syncIntervalSeconds = 0,+ retryIntervalSeconds = 30,+ isOfflineMode = false,+ syncCondition = "any",+ syncSsid = "",+ authHeaders = emptyMap()+ )+ fakeQueue(20)+ coEvery { networkManager.sendToEndpoint(any(), any(), any(), any(), any()) } coAnswers { delay(1_000); true }++ launch { syncManager.manualFlush() }+ launch { syncManager.manualFlush() }+ advanceUntilIdle()++ // Both passes fetch the oldest rows first, so a second pass beside the first posts every row twice+ coVerify(exactly = 20) { networkManager.sendToEndpoint(any(), any(), any(), any(), any()) }+ }++ @Test+ fun `a periodic tick during a manual flush waits and counts no failure`() = scope.runTest {+ syncManager.updateConfig(+ endpoint = "https://example.com",+ syncIntervalSeconds = 1,+ retryIntervalSeconds = 1,+ isOfflineMode = false,+ syncCondition = "any",+ syncSsid = "",+ authHeaders = emptyMap()+ )+ coEvery { networkManager.isNetworkAvailable() } returns true+ fakeQueue(20)+ coEvery { networkManager.sendToEndpoint(any(), any(), any(), any(), any()) } coAnswers { delay(1_500); true }++ launch { syncManager.manualFlush() }+ syncManager.startPeriodicSync()+ // The ticks at 1 s and 2 s land while the flush still holds its chunks+ advanceTimeBy(3_600)+ syncManager.stopPeriodicSync()++ coVerify(exactly = 20) { networkManager.sendToEndpoint(any(), any(), any(), any(), any()) }+ assertEquals(0, getField("consecutiveFailures"))+ }++ @Test+ fun `an instant send during a pass is posted once, by the pass`() = scope.runTest {+ syncManager.updateConfig(+ endpoint = "https://example.com",+ syncIntervalSeconds = 0,+ retryIntervalSeconds = 30,+ isOfflineMode = false,+ syncCondition = "any",+ syncSsid = "",+ authHeaders = emptyMap()+ )+ coEvery { networkManager.isNetworkAvailable() } returns true+ fakeQueue(10)+ coEvery { networkManager.sendToEndpoint(any(), any(), any(), any(), any()) } coAnswers { delay(1_000); true }++ launch { syncManager.manualFlush() }+ syncManager.queueAndSend(500L, JSONObject().put("lat", 53.0))+ advanceUntilIdle()++ coVerify(exactly = 1) {+ networkManager.sendToEndpoint(match { it.optDouble("lat") == 53.0 }, any(), any(), any(), any())+ }+ verify(exactly = 0) { dbHelper.removeFromQueueByLocationId(500L) }+ }++ @Test+ fun `a slow instant send does not hold back the next fix`() = scope.runTest {+ syncManager.updateConfig(+ endpoint = "https://example.com",+ syncIntervalSeconds = 0,+ retryIntervalSeconds = 30,+ isOfflineMode = false,+ syncCondition = "any",+ syncSsid = "",+ authHeaders = emptyMap()+ )+ coEvery { networkManager.isNetworkAvailable() } returns true+ fakeQueue(0)+ coEvery { networkManager.sendToEndpoint(any(), any(), any(), any(), any()) } coAnswers { delay(5_000); true }++ launch { syncManager.queueAndSend(1L, JSONObject().put("lat", 1.0)) }+ advanceTimeBy(1_000)+ launch { syncManager.queueAndSend(2L, JSONObject().put("lat", 2.0)) }+ advanceTimeBy(100)++ // Two instant sends never carry the same row, so serialising them would buy nothing+ coVerify(exactly = 1) {+ networkManager.sendToEndpoint(match { it.optDouble("lat") == 2.0 }, any(), any(), any(), any())+ }+ }++ @Test+ fun `a manual flush started during an instant send does not post that row again`() = scope.runTest {+ syncManager.updateConfig(+ endpoint = "https://example.com",+ syncIntervalSeconds = 0,+ retryIntervalSeconds = 30,+ isOfflineMode = false,+ syncCondition = "any",+ syncSsid = "",+ authHeaders = emptyMap()+ )+ coEvery { networkManager.isNetworkAvailable() } returns true+ fakeQueue(0)+ coEvery { networkManager.sendToEndpoint(any(), any(), any(), any(), any()) } coAnswers { delay(1_000); true }++ launch { syncManager.queueAndSend(500L, JSONObject().put("lat", 53.0), bypassInterval = true) }+ launch { syncManager.manualFlush() }+ advanceUntilIdle()++ // Zone entry starts both together, and the flush fetches the heartbeat row the instant send is posting+ coVerify(exactly = 1) {+ networkManager.sendToEndpoint(match { it.optDouble("lat") == 53.0 }, any(), any(), any(), any())+ }+ }++ @Test+ fun `a tick that finds only a row an instant send is posting counts no failure`() = scope.runTest {+ syncManager.updateConfig(+ endpoint = "https://example.com",+ syncIntervalSeconds = 0,+ retryIntervalSeconds = 1,+ isOfflineMode = false,+ syncCondition = "any",+ syncSsid = "",+ authHeaders = emptyMap()+ )+ coEvery { networkManager.isNetworkAvailable() } returns true+ fakeQueue(0)+ coEvery { networkManager.sendToEndpoint(any(), any(), any(), any(), any()) } coAnswers { delay(3_000); true }++ launch { syncManager.queueAndSend(500L, JSONObject().put("lat", 53.0)) }+ syncManager.startPeriodicSync()+ advanceTimeBy(1_500)+ syncManager.stopPeriodicSync()++ assertEquals(0, getField("consecutiveFailures"))+ coVerify(exactly = 1) { networkManager.sendToEndpoint(any(), any(), any(), any(), any()) }+ }++ /** Rows leave only when a pass or an instant send removes them, so two senders running together see the same rows. */+ private fun fakeQueue(size: Int) {+ val queue = (1L..size.toLong()).map { QueuedLocation(it, it + 100, """{"lat":52.0}""", 0) }.toMutableList()+ var nextId = size.toLong()+ every { dbHelper.getQueuedLocations(50) } answers { queue.toList() }+ every { dbHelper.getQueuedCount() } answers { queue.size }+ every { dbHelper.addToQueue(any(), any()) } answers {+ nextId+++ queue.add(QueuedLocation(nextId, firstArg(), secondArg(), 0))+ nextId+ }+ every { dbHelper.removeBatchFromQueue(any()) } answers {+ val ids = firstArg<List<Long>>()+ queue.removeAll { it.queueId in ids }+ }+ every { dbHelper.removeFromQueueByLocationId(any()) } answers {+ val locationId = firstArg<Long>()+ val before = queue.size+ queue.removeAll { it.locationId == locationId }+ before - queue.size+ }+ }+private fun setField(name: String, value: Any?) {val field = SyncManager::class.java.getDeclaredField(name)field.isAccessible = truediff --git a/fastlane/metadata/android/en-US/changelogs/49.txt b/fastlane/metadata/android/en-US/changelogs/49.txtindex 842c1f81..8370d814 100644--- a/fastlane/metadata/android/en-US/changelogs/49.txt+++ b/fastlane/metadata/android/en-US/changelogs/49.txt@@ -1,6 +1,7 @@-- A notification now alerts you when tracking stops on its own, instead of appearing silently+- A notification alerts you when tracking stops on its own- Auto-export now deletes old files on storage providers that rename them- Unplugging at a low battery no longer restarts tracking in a loop- Pause zones no longer take hours to engage when a tracking profile is active- The tracking notification shows the state and the sync queue, never your location, zone or profile name- A low-battery stop says tracking resumes when charging+- Sync now no longer uploads a location twice