Skip to content

Colota - Android GPS Location tracker

fix(sync): run one sync pass at a time (#843)

Max Dietrichc4c6ce8b572b
diff --git a/apps/docs/docs/development/architecture.md b/apps/docs/docs/development/architecture.md
index 43aed2c4..d7f3fea9 100644
--- a/apps/docs/docs/development/architecture.md
+++ b/apps/docs/docs/development/architecture.md
@@ -208,6 +208,7 @@ Orchestrates batch location uploads with:
- Exponential backoff on failure
- Periodic sync scheduling
- Manual flush support
+- One pass at a time: a periodic tick and a manual flush wait for the running pass. An instant send leaves its row queued while a pass runs, and a pass skips rows an instant send is still posting
### NetworkManager
diff --git a/apps/docs/docs/guides/data-management.md b/apps/docs/docs/guides/data-management.md
index 4bd0e678..b0f19d22 100644
--- a/apps/docs/docs/guides/data-management.md
+++ b/apps/docs/docs/guides/data-management.md
@@ -14,7 +14,7 @@ import ScreenshotGallery from '@site/src/components/ScreenshotGallery'
| Action | Description |
| ---------------------------- | -------------------------------------------------------------------------- |
-| **Sync Now** | Manually flush the queue and upload pending locations |
+| **Sync Now** | Manually flush the queue and upload pending locations. A sync already running finishes first |
| **Clear Sent History** | Remove locations that have already been synced |
| **Clear Queue** | Remove unsent locations from the upload queue |
| **Delete Older Than X Days** | Clean up old data past a specified age |
diff --git a/apps/mobile/android/app/src/main/java/com/colota/service/LocationForegroundService.kt b/apps/mobile/android/app/src/main/java/com/colota/service/LocationForegroundService.kt
index 2b048228..2927affe 100644
--- a/apps/mobile/android/app/src/main/java/com/colota/service/LocationForegroundService.kt
+++ b/apps/mobile/android/app/src/main/java/com/colota/service/LocationForegroundService.kt
@@ -1301,6 +1301,7 @@ class LocationForegroundService : Service() {
* concerns here - gating the save on a successful send loses the stay entirely whenever
* the server is unreachable or sync is not allowed. The send skips the sync interval: on
* Power Saver a queued stay reaches the endpoint 15 minutes after the user got home.
+ * While a sync pass runs, the point waits in the queue for that pass or the next.
*/
private suspend fun recordHeartbeatLocation() {
if (!::config.isInitialized) {
diff --git a/apps/mobile/android/app/src/main/java/com/colota/sync/SyncManager.kt b/apps/mobile/android/app/src/main/java/com/colota/sync/SyncManager.kt
index d174309f..592a480e 100644
--- a/apps/mobile/android/app/src/main/java/com/colota/sync/SyncManager.kt
+++ b/apps/mobile/android/app/src/main/java/com/colota/sync/SyncManager.kt
@@ -10,7 +10,10 @@ import com.Colota.bridge.LocationServiceModule
import com.Colota.data.DatabaseHelper
import com.Colota.util.TimedCache
import kotlinx.coroutines.*
+import kotlinx.coroutines.sync.Mutex
+import kotlinx.coroutines.sync.withLock
import org.json.JSONObject
+import java.util.concurrent.ConcurrentHashMap
/**
* Two sync modes: instant (syncInterval=0) sends each location on arrival,
@@ -46,6 +49,12 @@ class SyncManager(
private val queueCountCache = TimedCache(5000L) { dbHelper.getQueuedCount() }
+ /** One pass at a time: two passes fetch the same oldest rows and post each of them twice. */
+ private val syncMutex = Mutex()
+
+ /** Queue rows an instant send is posting; a pass skips them instead of posting them a second time. */
+ private val instantInFlight: MutableSet<Long> = ConcurrentHashMap.newKeySet()
+
fun updateConfig(
endpoint: String,
syncIntervalSeconds: Int,
@@ -88,21 +97,24 @@ class SyncManager(
}
if (endpoint.isNotBlank() && queued > 0) {
- val errorMessage: String? = try {
- val success = performSyncAndCheckSuccess()
-
- if (success) {
- if (consecutiveFailures > 0) {
- AppLogger.i(TAG, "Sync restored")
+ if (syncMutex.isLocked) AppLogger.d(TAG, "Sync tick waiting for the running sync")
+ val errorMessage: String? = syncMutex.withLock {
+ try {
+ val success = performSyncAndCheckSuccess()
+
+ if (success) {
+ if (consecutiveFailures > 0) {
+ AppLogger.i(TAG, "Sync restored")
+ }
+ consecutiveFailures = 0
+ null
+ } else {
+ "Sync failed"
}
- consecutiveFailures = 0
- null
- } else {
- "Sync failed"
+ } catch (e: Exception) {
+ AppLogger.e(TAG, "Sync error", e)
+ e.message ?: "Sync error"
}
- } catch (e: Exception) {
- AppLogger.e(TAG, "Sync error", e)
- e.message ?: "Sync error"
}
if (errorMessage != null) {
@@ -128,9 +140,12 @@ class SyncManager(
suspend fun manualFlush() {
if (endpoint.isNotBlank()) {
- val total = dbHelper.getQueuedCount()
- AppLogger.d(TAG, "Manual flush started: $total items in queue")
- syncQueue { sent, failed -> LocationServiceModule.sendSyncProgressEvent(sent, failed, total) }
+ if (syncMutex.isLocked) AppLogger.d(TAG, "Manual flush waiting for the running sync")
+ syncMutex.withLock {
+ val total = dbHelper.getQueuedCount()
+ AppLogger.d(TAG, "Manual flush started: $total items in queue")
+ syncQueue { sent, failed -> LocationServiceModule.sendSyncProgressEvent(sent, failed, total) }
+ }
}
}
@@ -151,16 +166,26 @@ class SyncManager(
// Immediate send mode (syncInterval = 0)
if ((syncIntervalSeconds == 0 || bypassInterval) && isSyncAllowed()) {
- AppLogger.d(TAG, "Instant send")
- val success = networkManager.sendToEndpoint(payload, endpoint, authHeaders, httpMethod, apiFormat)
-
- if (success) {
- dbHelper.markLocationsSent(listOf(locationId))
- dbHelper.removeFromQueueByLocationId(locationId)
- invalidateQueueCache()
- lastSuccessfulSyncTime = System.currentTimeMillis()
- } else {
- dbHelper.incrementRetryCount(queueId, "Send failed")
+ // Registered before the lock check: a later pass skips this row, a running one may already hold it
+ instantInFlight.add(queueId)
+ try {
+ if (syncMutex.isLocked) {
+ AppLogger.d(TAG, "Instant send deferred: a sync is running")
+ return
+ }
+ AppLogger.d(TAG, "Instant send")
+ val success = networkManager.sendToEndpoint(payload, endpoint, authHeaders, httpMethod, apiFormat)
+
+ if (success) {
+ dbHelper.markLocationsSent(listOf(locationId))
+ dbHelper.removeFromQueueByLocationId(locationId)
+ invalidateQueueCache()
+ lastSuccessfulSyncTime = System.currentTimeMillis()
+ } else {
+ dbHelper.incrementRetryCount(queueId, "Send failed")
+ }
+ } finally {
+ instantInFlight.remove(queueId)
}
}
// Otherwise the periodic sync job will handle it
@@ -186,12 +211,13 @@ class SyncManager(
private suspend fun performSyncAndCheckSuccess(): Boolean {
val countBefore = dbHelper.getQueuedCount()
- syncQueue(onProgress = null)
+ val pass = syncQueue(onProgress = null)
val countAfter = dbHelper.getQueuedCount()
invalidateQueueCache()
- val success = countAfter < countBefore || countAfter == 0
+ // A pass that only skipped rows an instant send was still posting did not fail
+ val success = countAfter < countBefore || countAfter == 0 || (pass.sent + pass.failed == 0 && pass.skippedInFlight > 0)
if (success && countAfter == 0) {
lastSuccessfulSyncTime = System.currentTimeMillis()
}
@@ -213,7 +239,7 @@ class SyncManager(
delay(backoffSeconds * 1000L)
}
- private suspend fun syncQueue(onProgress: ((sent: Int, failed: Int) -> Unit)? = null) = coroutineScope {
+ private suspend fun syncQueue(onProgress: ((sent: Int, failed: Int) -> Unit)? = null): SyncPass = coroutineScope {
// Snapshot volatile config so it stays consistent for the entire sync pass
val currentEndpoint = endpoint
val currentAuthHeaders = authHeaders
@@ -223,11 +249,14 @@ class SyncManager(
var totalProcessed = 0
var totalSucceeded = 0
var totalFailed = 0
+ var skippedInFlight = 0
var batchNumber = 1
while (isActive && batchNumber <= MAX_BATCHES_PER_SYNC) {
val fetchSize = if (currentApiFormat == ApiFormat.OVERLAND_BATCH) currentBatchSize else 50
- val queued = dbHelper.getQueuedLocations(fetchSize)
+ val fetched = dbHelper.getQueuedLocations(fetchSize)
+ val queued = fetched.filterNot { it.queueId in instantInFlight }
+ skippedInFlight += fetched.size - queued.size
if (queued.isEmpty()) {
if (totalProcessed > 0) {
AppLogger.d(TAG, "Sync complete: $totalProcessed items in $batchNumber batches")
@@ -314,8 +343,12 @@ class SyncManager(
if (totalSucceeded > 0) {
lastSuccessfulSyncTime = System.currentTimeMillis()
}
+
+ SyncPass(totalSucceeded, totalFailed, skippedInFlight)
}
+ data class SyncPass(val sent: Int, val failed: Int, val skippedInFlight: Int = 0)
+
private data class BatchSendResult(val processed: Int, val failed: Int, val stop: Boolean)
/**
diff --git a/apps/mobile/android/app/src/test/java/com/Colota/sync/SyncManagerTest.kt b/apps/mobile/android/app/src/test/java/com/Colota/sync/SyncManagerTest.kt
index 4f3e2150..7e14b165 100644
--- a/apps/mobile/android/app/src/test/java/com/Colota/sync/SyncManagerTest.kt
+++ b/apps/mobile/android/app/src/test/java/com/Colota/sync/SyncManagerTest.kt
@@ -1250,6 +1250,180 @@ class SyncManagerTest {
}
}
+ // ========================================================================
+ // One sender at a time
+ // ========================================================================
+
+ @Test
+ fun `a manual flush during another waits instead of posting the same rows again`() = scope.runTest {
+ syncManager.updateConfig(
+ endpoint = "https://example.com",
+ syncIntervalSeconds = 0,
+ retryIntervalSeconds = 30,
+ isOfflineMode = false,
+ syncCondition = "any",
+ syncSsid = "",
+ authHeaders = emptyMap()
+ )
+ fakeQueue(20)
+ coEvery { networkManager.sendToEndpoint(any(), any(), any(), any(), any()) } coAnswers { delay(1_000); true }
+
+ launch { syncManager.manualFlush() }
+ launch { syncManager.manualFlush() }
+ advanceUntilIdle()
+
+ // Both passes fetch the oldest rows first, so a second pass beside the first posts every row twice
+ coVerify(exactly = 20) { networkManager.sendToEndpoint(any(), any(), any(), any(), any()) }
+ }
+
+ @Test
+ fun `a periodic tick during a manual flush waits and counts no failure`() = scope.runTest {
+ syncManager.updateConfig(
+ endpoint = "https://example.com",
+ syncIntervalSeconds = 1,
+ retryIntervalSeconds = 1,
+ isOfflineMode = false,
+ syncCondition = "any",
+ syncSsid = "",
+ authHeaders = emptyMap()
+ )
+ coEvery { networkManager.isNetworkAvailable() } returns true
+ fakeQueue(20)
+ coEvery { networkManager.sendToEndpoint(any(), any(), any(), any(), any()) } coAnswers { delay(1_500); true }
+
+ launch { syncManager.manualFlush() }
+ syncManager.startPeriodicSync()
+ // The ticks at 1 s and 2 s land while the flush still holds its chunks
+ advanceTimeBy(3_600)
+ syncManager.stopPeriodicSync()
+
+ coVerify(exactly = 20) { networkManager.sendToEndpoint(any(), any(), any(), any(), any()) }
+ assertEquals(0, getField("consecutiveFailures"))
+ }
+
+ @Test
+ fun `an instant send during a pass is posted once, by the pass`() = scope.runTest {
+ syncManager.updateConfig(
+ endpoint = "https://example.com",
+ syncIntervalSeconds = 0,
+ retryIntervalSeconds = 30,
+ isOfflineMode = false,
+ syncCondition = "any",
+ syncSsid = "",
+ authHeaders = emptyMap()
+ )
+ coEvery { networkManager.isNetworkAvailable() } returns true
+ fakeQueue(10)
+ coEvery { networkManager.sendToEndpoint(any(), any(), any(), any(), any()) } coAnswers { delay(1_000); true }
+
+ launch { syncManager.manualFlush() }
+ syncManager.queueAndSend(500L, JSONObject().put("lat", 53.0))
+ advanceUntilIdle()
+
+ coVerify(exactly = 1) {
+ networkManager.sendToEndpoint(match { it.optDouble("lat") == 53.0 }, any(), any(), any(), any())
+ }
+ verify(exactly = 0) { dbHelper.removeFromQueueByLocationId(500L) }
+ }
+
+ @Test
+ fun `a slow instant send does not hold back the next fix`() = scope.runTest {
+ syncManager.updateConfig(
+ endpoint = "https://example.com",
+ syncIntervalSeconds = 0,
+ retryIntervalSeconds = 30,
+ isOfflineMode = false,
+ syncCondition = "any",
+ syncSsid = "",
+ authHeaders = emptyMap()
+ )
+ coEvery { networkManager.isNetworkAvailable() } returns true
+ fakeQueue(0)
+ coEvery { networkManager.sendToEndpoint(any(), any(), any(), any(), any()) } coAnswers { delay(5_000); true }
+
+ launch { syncManager.queueAndSend(1L, JSONObject().put("lat", 1.0)) }
+ advanceTimeBy(1_000)
+ launch { syncManager.queueAndSend(2L, JSONObject().put("lat", 2.0)) }
+ advanceTimeBy(100)
+
+ // Two instant sends never carry the same row, so serialising them would buy nothing
+ coVerify(exactly = 1) {
+ networkManager.sendToEndpoint(match { it.optDouble("lat") == 2.0 }, any(), any(), any(), any())
+ }
+ }
+
+ @Test
+ fun `a manual flush started during an instant send does not post that row again`() = scope.runTest {
+ syncManager.updateConfig(
+ endpoint = "https://example.com",
+ syncIntervalSeconds = 0,
+ retryIntervalSeconds = 30,
+ isOfflineMode = false,
+ syncCondition = "any",
+ syncSsid = "",
+ authHeaders = emptyMap()
+ )
+ coEvery { networkManager.isNetworkAvailable() } returns true
+ fakeQueue(0)
+ coEvery { networkManager.sendToEndpoint(any(), any(), any(), any(), any()) } coAnswers { delay(1_000); true }
+
+ launch { syncManager.queueAndSend(500L, JSONObject().put("lat", 53.0), bypassInterval = true) }
+ launch { syncManager.manualFlush() }
+ advanceUntilIdle()
+
+ // Zone entry starts both together, and the flush fetches the heartbeat row the instant send is posting
+ coVerify(exactly = 1) {
+ networkManager.sendToEndpoint(match { it.optDouble("lat") == 53.0 }, any(), any(), any(), any())
+ }
+ }
+
+ @Test
+ fun `a tick that finds only a row an instant send is posting counts no failure`() = scope.runTest {
+ syncManager.updateConfig(
+ endpoint = "https://example.com",
+ syncIntervalSeconds = 0,
+ retryIntervalSeconds = 1,
+ isOfflineMode = false,
+ syncCondition = "any",
+ syncSsid = "",
+ authHeaders = emptyMap()
+ )
+ coEvery { networkManager.isNetworkAvailable() } returns true
+ fakeQueue(0)
+ coEvery { networkManager.sendToEndpoint(any(), any(), any(), any(), any()) } coAnswers { delay(3_000); true }
+
+ launch { syncManager.queueAndSend(500L, JSONObject().put("lat", 53.0)) }
+ syncManager.startPeriodicSync()
+ advanceTimeBy(1_500)
+ syncManager.stopPeriodicSync()
+
+ assertEquals(0, getField("consecutiveFailures"))
+ coVerify(exactly = 1) { networkManager.sendToEndpoint(any(), any(), any(), any(), any()) }
+ }
+
+ /** Rows leave only when a pass or an instant send removes them, so two senders running together see the same rows. */
+ private fun fakeQueue(size: Int) {
+ val queue = (1L..size.toLong()).map { QueuedLocation(it, it + 100, """{"lat":52.0}""", 0) }.toMutableList()
+ var nextId = size.toLong()
+ every { dbHelper.getQueuedLocations(50) } answers { queue.toList() }
+ every { dbHelper.getQueuedCount() } answers { queue.size }
+ every { dbHelper.addToQueue(any(), any()) } answers {
+ nextId++
+ queue.add(QueuedLocation(nextId, firstArg(), secondArg(), 0))
+ nextId
+ }
+ every { dbHelper.removeBatchFromQueue(any()) } answers {
+ val ids = firstArg<List<Long>>()
+ queue.removeAll { it.queueId in ids }
+ }
+ every { dbHelper.removeFromQueueByLocationId(any()) } answers {
+ val locationId = firstArg<Long>()
+ val before = queue.size
+ queue.removeAll { it.locationId == locationId }
+ before - queue.size
+ }
+ }
+
private fun setField(name: String, value: Any?) {
val field = SyncManager::class.java.getDeclaredField(name)
field.isAccessible = true
diff --git a/fastlane/metadata/android/en-US/changelogs/49.txt b/fastlane/metadata/android/en-US/changelogs/49.txt
index 842c1f81..8370d814 100644
--- a/fastlane/metadata/android/en-US/changelogs/49.txt
+++ b/fastlane/metadata/android/en-US/changelogs/49.txt
@@ -1,6 +1,7 @@
-- A notification now alerts you when tracking stops on its own, instead of appearing silently
+- A notification alerts you when tracking stops on its own
- Auto-export now deletes old files on storage providers that rename them
- Unplugging at a low battery no longer restarts tracking in a loop
- Pause zones no longer take hours to engage when a tracking profile is active
- The tracking notification shows the state and the sync queue, never your location, zone or profile name
- A low-battery stop says tracking resumes when charging
+- Sync now no longer uploads a location twice