Colota - Android GPS Location tracker
fix(sync): report Overland bisect progress so Sync now does not give up mid-split (#989)
Max Dietrich517d386bee6f
diff --git a/apps/mobile/android/app/src/main/java/com/colota/sync/OverlandBatchSender.kt b/apps/mobile/android/app/src/main/java/com/colota/sync/OverlandBatchSender.ktindex 8b75a54f..aa2786b3 100644--- a/apps/mobile/android/app/src/main/java/com/colota/sync/OverlandBatchSender.kt+++ b/apps/mobile/android/app/src/main/java/com/colota/sync/OverlandBatchSender.kt@@ -25,10 +25,12 @@ class OverlandBatchSender(data class BatchSendResult(val processed: Int, val failed: Int, val stop: Boolean)+ /** [onProgress] hears each finished piece of the bisect: the rows it sent and the rows it failed. */suspend fun sendBatchRecursive(items: List<QueuedLocation>,endpoint: String,authHeaders: Map<String, String>,+ onProgress: (sent: Int, failed: Int) -> Unit = { _, _ -> },): BatchSendResult {if (items.isEmpty()) return BatchSendResult(0, 0, false)@@ -43,6 +45,7 @@ class OverlandBatchSender(}}val corruptedFailed = items.size - parsed.size+ if (corruptedFailed > 0) onProgress(0, corruptedFailed)if (parsed.isEmpty()) {return BatchSendResult(processed = 0, failed = corruptedFailed, stop = false)}@@ -62,18 +65,20 @@ class OverlandBatchSender(BatchResult.Success -> {dbHelper.markLocationsSent(parseableItems.map { it.locationId })dbHelper.removeBatchFromQueue(parseableItems.map { it.queueId })+ onProgress(parseableItems.size, 0)BatchSendResult(processed = parseableItems.size, failed = corruptedFailed, stop = false)}is BatchResult.ClientError -> {if (parseableItems.size == 1) {dbHelper.incrementRetryCount(parseableItems[0].queueId, "4xx: ${result.code}")+ onProgress(0, 1)BatchSendResult(processed = 0, failed = 1 + corruptedFailed, stop = false)} else {val mid = parseableItems.size / 2- val left = sendBatchRecursive(parseableItems.take(mid), endpoint, authHeaders)+ val left = sendBatchRecursive(parseableItems.take(mid), endpoint, authHeaders, onProgress)if (left.stop) return BatchSendResult(left.processed, left.failed + corruptedFailed, stop = true)yield()- val right = sendBatchRecursive(parseableItems.drop(mid), endpoint, authHeaders)+ val right = sendBatchRecursive(parseableItems.drop(mid), endpoint, authHeaders, onProgress)BatchSendResult(processed = left.processed + right.processed,failed = left.failed + right.failed + corruptedFailed,@@ -85,14 +90,18 @@ class OverlandBatchSender(retryReason(result)?.let { reason ->parseableItems.forEach { dbHelper.incrementRetryCount(it.queueId, reason) }}+ onProgress(0, parseableItems.size)BatchSendResult(processed = 0, failed = parseableItems.size + corruptedFailed, stop = true)}is BatchResult.RateLimited -> {onRateLimited(result.retryAfterSeconds)+ onProgress(0, parseableItems.size)BatchSendResult(processed = 0, failed = parseableItems.size + corruptedFailed, stop = true)}- BatchResult.NetworkError ->+ BatchResult.NetworkError -> {+ onProgress(0, parseableItems.size)BatchSendResult(processed = 0, failed = parseableItems.size + corruptedFailed, stop = true)+ }}}}diff --git a/apps/mobile/android/app/src/main/java/com/colota/sync/SyncManager.kt b/apps/mobile/android/app/src/main/java/com/colota/sync/SyncManager.ktindex 04a9f23a..33d9c742 100644--- a/apps/mobile/android/app/src/main/java/com/colota/sync/SyncManager.kt+++ b/apps/mobile/android/app/src/main/java/com/colota/sync/SyncManager.kt@@ -332,11 +332,12 @@ class SyncManager(val processedBefore = totalProcessedif (currentApiFormat == ApiFormat.OVERLAND_BATCH) {- val result = batchSender.sendBatchRecursive(queued, currentEndpoint, currentAuthHeaders)- totalProcessed += result.processed- totalSucceeded += result.processed- totalFailed += result.failed- reportProgress()+ val result = batchSender.sendBatchRecursive(queued, currentEndpoint, currentAuthHeaders) { sent, failed ->+ totalProcessed += sent+ totalSucceeded += sent+ totalFailed += failed+ reportProgress()+ }if (result.stop) {AppLogger.d(TAG, "Sync pass stopped: the server is unreachable, failing or rate limiting")breakdiff --git a/apps/mobile/android/app/src/test/java/com/Colota/sync/OverlandBatchSenderTest.kt b/apps/mobile/android/app/src/test/java/com/Colota/sync/OverlandBatchSenderTest.ktindex a327e80c..ba5ec56f 100644--- a/apps/mobile/android/app/src/test/java/com/Colota/sync/OverlandBatchSenderTest.kt+++ b/apps/mobile/android/app/src/test/java/com/Colota/sync/OverlandBatchSenderTest.kt@@ -100,6 +100,33 @@ class OverlandBatchSenderTest {assertEquals(OverlandBatchSender.BatchSendResult(processed = 3, failed = 1, stop = false), result)}+ /** A fully rejected batch costs about two requests per row, so the caller must hear from the split before it ends. */+ @Test+ fun `every finished piece of a bisect reports what it sent and failed`() = runTest {+ coEvery { networkManager.sendBatchToEndpoint(any(), any(), any(), any()) } answers {+ val containsPoison = arg<List<JSONObject>>(0).any { it.optInt("id", -1) == 4 }+ if (containsPoison) BatchResult.ClientError(400) else BatchResult.Success+ }+ val reports = mutableListOf<Pair<Int, Int>>()++ val result = sender.sendBatchRecursive(rows(4), endpoint, emptyMap()) { sent, failed -> reports.add(sent to failed) }++ assertEquals(listOf(2 to 0, 1 to 0, 0 to 1), reports)+ assertEquals(result.processed to result.failed, reports.sumOf { it.first } to reports.sumOf { it.second })+ }++ @Test+ fun `a corrupt row is reported once, beside the rows that went out`() = runTest {+ coEvery { networkManager.sendBatchToEndpoint(any(), any(), any(), any()) } returns BatchResult.Success+ val reports = mutableListOf<Pair<Int, Int>>()++ sender.sendBatchRecursive(listOf(QueuedLocation(9L, 109L, "NOT_VALID_JSON{{{", 0)) + rows(2), endpoint, emptyMap()) { sent, failed ->+ reports.add(sent to failed)+ }++ assertEquals(listOf(0 to 1, 2 to 0), reports)+ }+/** Overland carries custom fields once on the envelope, never as location keys. */@Testfun `custom fields lift to the envelope and location keys stay out of it`() = runTest {diff --git a/apps/mobile/android/app/src/test/java/com/Colota/sync/SyncManagerTest.kt b/apps/mobile/android/app/src/test/java/com/Colota/sync/SyncManagerTest.ktindex 648a29df..3a4d8a54 100644--- a/apps/mobile/android/app/src/test/java/com/Colota/sync/SyncManagerTest.kt+++ b/apps/mobile/android/app/src/test/java/com/Colota/sync/SyncManagerTest.kt@@ -1069,6 +1069,43 @@ class SyncManagerTest {verify { dbHelper.getQueuedLocations(200) }}+ /** Sync Now gives up after a silence, and a fully rejected batch of 50 takes about 100 requests to bisect. */+ @Test+ fun `a bisect reports each finished piece, so Sync Now hears from a long split`() = scope.runTest {+ mockkObject(LocationServiceModule.Companion)+ every { LocationServiceModule.sendSyncProgressEvent(any(), any(), any(), any(), any()) } returns true+ syncManager.updateConfig(+ endpoint = "https://dawarich.example/api/v1/overland/batches",+ syncIntervalSeconds = 300,+ retryIntervalSeconds = 30,+ isOfflineMode = false,+ syncCondition = "any",+ syncSsid = "",+ authHeaders = emptyMap(),+ apiFormat = ApiFormat.OVERLAND_BATCH,+ overlandBatchSize = 50+ )+ val items = (1L..4L).map { QueuedLocation(it, it + 100, """{"lat":52.0,"lon":13.0,"tst":1700000000}""", 0) }+ every { dbHelper.getQueuedLocations(50) } returnsMany listOf(items, emptyList())+ every { dbHelper.getQueuedCount() } returns 4+ coEvery { networkManager.sendBatchToEndpoint(any(), any(), any(), any()) } answers {+ if (firstArg<List<JSONObject>>().size > 1) BatchResult.ClientError(400) else BatchResult.Success+ }++ try {+ syncManager.manualFlush()++ verifyOrder {+ LocationServiceModule.sendSyncProgressEvent(1, 0, 4)+ LocationServiceModule.sendSyncProgressEvent(2, 0, 4)+ LocationServiceModule.sendSyncProgressEvent(3, 0, 4)+ LocationServiceModule.sendSyncProgressEvent(4, 0, 4)+ }+ } finally {+ unmockkObject(LocationServiceModule.Companion)+ }+ }+// --- One sender at a time ---@Testdiff --git a/fastlane/metadata/android/en-US/changelogs/53.txt b/fastlane/metadata/android/en-US/changelogs/53.txtindex a7abc5ac..021d8bab 100644--- a/fastlane/metadata/android/en-US/changelogs/53.txt+++ b/fastlane/metadata/android/en-US/changelogs/53.txt@@ -4,3 +4,4 @@- Positioning accuracy setting in Tracking & sync: High accuracy (default) or Lower accuracy to save battery- Fixed tracking recording nothing for hours after Android recreated it- Fixed GPS staying off when leaving a zone with motionless pause after the app was updated or restarted+- Fixed Sync now saying the app did not answer while the server was rejecting an Overland batch and Colota was retrying it in smaller parts