Skip to content

Colota - Android GPS Location tracker

fix(sync): report Overland bisect progress so Sync now does not give up mid-split (#989)

Max Dietrich517d386bee6f
diff --git a/apps/mobile/android/app/src/main/java/com/colota/sync/OverlandBatchSender.kt b/apps/mobile/android/app/src/main/java/com/colota/sync/OverlandBatchSender.kt
index 8b75a54f..aa2786b3 100644
--- a/apps/mobile/android/app/src/main/java/com/colota/sync/OverlandBatchSender.kt
+++ b/apps/mobile/android/app/src/main/java/com/colota/sync/OverlandBatchSender.kt
@@ -25,10 +25,12 @@ class OverlandBatchSender(
data class BatchSendResult(val processed: Int, val failed: Int, val stop: Boolean)
+ /** [onProgress] hears each finished piece of the bisect: the rows it sent and the rows it failed. */
suspend fun sendBatchRecursive(
items: List<QueuedLocation>,
endpoint: String,
authHeaders: Map<String, String>,
+ onProgress: (sent: Int, failed: Int) -> Unit = { _, _ -> },
): BatchSendResult {
if (items.isEmpty()) return BatchSendResult(0, 0, false)
@@ -43,6 +45,7 @@ class OverlandBatchSender(
}
}
val corruptedFailed = items.size - parsed.size
+ if (corruptedFailed > 0) onProgress(0, corruptedFailed)
if (parsed.isEmpty()) {
return BatchSendResult(processed = 0, failed = corruptedFailed, stop = false)
}
@@ -62,18 +65,20 @@ class OverlandBatchSender(
BatchResult.Success -> {
dbHelper.markLocationsSent(parseableItems.map { it.locationId })
dbHelper.removeBatchFromQueue(parseableItems.map { it.queueId })
+ onProgress(parseableItems.size, 0)
BatchSendResult(processed = parseableItems.size, failed = corruptedFailed, stop = false)
}
is BatchResult.ClientError -> {
if (parseableItems.size == 1) {
dbHelper.incrementRetryCount(parseableItems[0].queueId, "4xx: ${result.code}")
+ onProgress(0, 1)
BatchSendResult(processed = 0, failed = 1 + corruptedFailed, stop = false)
} else {
val mid = parseableItems.size / 2
- val left = sendBatchRecursive(parseableItems.take(mid), endpoint, authHeaders)
+ val left = sendBatchRecursive(parseableItems.take(mid), endpoint, authHeaders, onProgress)
if (left.stop) return BatchSendResult(left.processed, left.failed + corruptedFailed, stop = true)
yield()
- val right = sendBatchRecursive(parseableItems.drop(mid), endpoint, authHeaders)
+ val right = sendBatchRecursive(parseableItems.drop(mid), endpoint, authHeaders, onProgress)
BatchSendResult(
processed = left.processed + right.processed,
failed = left.failed + right.failed + corruptedFailed,
@@ -85,14 +90,18 @@ class OverlandBatchSender(
retryReason(result)?.let { reason ->
parseableItems.forEach { dbHelper.incrementRetryCount(it.queueId, reason) }
}
+ onProgress(0, parseableItems.size)
BatchSendResult(processed = 0, failed = parseableItems.size + corruptedFailed, stop = true)
}
is BatchResult.RateLimited -> {
onRateLimited(result.retryAfterSeconds)
+ onProgress(0, parseableItems.size)
BatchSendResult(processed = 0, failed = parseableItems.size + corruptedFailed, stop = true)
}
- BatchResult.NetworkError ->
+ BatchResult.NetworkError -> {
+ onProgress(0, parseableItems.size)
BatchSendResult(processed = 0, failed = parseableItems.size + corruptedFailed, stop = true)
+ }
}
}
}
diff --git a/apps/mobile/android/app/src/main/java/com/colota/sync/SyncManager.kt b/apps/mobile/android/app/src/main/java/com/colota/sync/SyncManager.kt
index 04a9f23a..33d9c742 100644
--- a/apps/mobile/android/app/src/main/java/com/colota/sync/SyncManager.kt
+++ b/apps/mobile/android/app/src/main/java/com/colota/sync/SyncManager.kt
@@ -332,11 +332,12 @@ class SyncManager(
val processedBefore = totalProcessed
if (currentApiFormat == ApiFormat.OVERLAND_BATCH) {
- val result = batchSender.sendBatchRecursive(queued, currentEndpoint, currentAuthHeaders)
- totalProcessed += result.processed
- totalSucceeded += result.processed
- totalFailed += result.failed
- reportProgress()
+ val result = batchSender.sendBatchRecursive(queued, currentEndpoint, currentAuthHeaders) { sent, failed ->
+ totalProcessed += sent
+ totalSucceeded += sent
+ totalFailed += failed
+ reportProgress()
+ }
if (result.stop) {
AppLogger.d(TAG, "Sync pass stopped: the server is unreachable, failing or rate limiting")
break
diff --git a/apps/mobile/android/app/src/test/java/com/Colota/sync/OverlandBatchSenderTest.kt b/apps/mobile/android/app/src/test/java/com/Colota/sync/OverlandBatchSenderTest.kt
index a327e80c..ba5ec56f 100644
--- a/apps/mobile/android/app/src/test/java/com/Colota/sync/OverlandBatchSenderTest.kt
+++ b/apps/mobile/android/app/src/test/java/com/Colota/sync/OverlandBatchSenderTest.kt
@@ -100,6 +100,33 @@ class OverlandBatchSenderTest {
assertEquals(OverlandBatchSender.BatchSendResult(processed = 3, failed = 1, stop = false), result)
}
+ /** A fully rejected batch costs about two requests per row, so the caller must hear from the split before it ends. */
+ @Test
+ fun `every finished piece of a bisect reports what it sent and failed`() = runTest {
+ coEvery { networkManager.sendBatchToEndpoint(any(), any(), any(), any()) } answers {
+ val containsPoison = arg<List<JSONObject>>(0).any { it.optInt("id", -1) == 4 }
+ if (containsPoison) BatchResult.ClientError(400) else BatchResult.Success
+ }
+ val reports = mutableListOf<Pair<Int, Int>>()
+
+ val result = sender.sendBatchRecursive(rows(4), endpoint, emptyMap()) { sent, failed -> reports.add(sent to failed) }
+
+ assertEquals(listOf(2 to 0, 1 to 0, 0 to 1), reports)
+ assertEquals(result.processed to result.failed, reports.sumOf { it.first } to reports.sumOf { it.second })
+ }
+
+ @Test
+ fun `a corrupt row is reported once, beside the rows that went out`() = runTest {
+ coEvery { networkManager.sendBatchToEndpoint(any(), any(), any(), any()) } returns BatchResult.Success
+ val reports = mutableListOf<Pair<Int, Int>>()
+
+ sender.sendBatchRecursive(listOf(QueuedLocation(9L, 109L, "NOT_VALID_JSON{{{", 0)) + rows(2), endpoint, emptyMap()) { sent, failed ->
+ reports.add(sent to failed)
+ }
+
+ assertEquals(listOf(0 to 1, 2 to 0), reports)
+ }
+
/** Overland carries custom fields once on the envelope, never as location keys. */
@Test
fun `custom fields lift to the envelope and location keys stay out of it`() = runTest {
diff --git a/apps/mobile/android/app/src/test/java/com/Colota/sync/SyncManagerTest.kt b/apps/mobile/android/app/src/test/java/com/Colota/sync/SyncManagerTest.kt
index 648a29df..3a4d8a54 100644
--- a/apps/mobile/android/app/src/test/java/com/Colota/sync/SyncManagerTest.kt
+++ b/apps/mobile/android/app/src/test/java/com/Colota/sync/SyncManagerTest.kt
@@ -1069,6 +1069,43 @@ class SyncManagerTest {
verify { dbHelper.getQueuedLocations(200) }
}
+ /** Sync Now gives up after a silence, and a fully rejected batch of 50 takes about 100 requests to bisect. */
+ @Test
+ fun `a bisect reports each finished piece, so Sync Now hears from a long split`() = scope.runTest {
+ mockkObject(LocationServiceModule.Companion)
+ every { LocationServiceModule.sendSyncProgressEvent(any(), any(), any(), any(), any()) } returns true
+ syncManager.updateConfig(
+ endpoint = "https://dawarich.example/api/v1/overland/batches",
+ syncIntervalSeconds = 300,
+ retryIntervalSeconds = 30,
+ isOfflineMode = false,
+ syncCondition = "any",
+ syncSsid = "",
+ authHeaders = emptyMap(),
+ apiFormat = ApiFormat.OVERLAND_BATCH,
+ overlandBatchSize = 50
+ )
+ val items = (1L..4L).map { QueuedLocation(it, it + 100, """{"lat":52.0,"lon":13.0,"tst":1700000000}""", 0) }
+ every { dbHelper.getQueuedLocations(50) } returnsMany listOf(items, emptyList())
+ every { dbHelper.getQueuedCount() } returns 4
+ coEvery { networkManager.sendBatchToEndpoint(any(), any(), any(), any()) } answers {
+ if (firstArg<List<JSONObject>>().size > 1) BatchResult.ClientError(400) else BatchResult.Success
+ }
+
+ try {
+ syncManager.manualFlush()
+
+ verifyOrder {
+ LocationServiceModule.sendSyncProgressEvent(1, 0, 4)
+ LocationServiceModule.sendSyncProgressEvent(2, 0, 4)
+ LocationServiceModule.sendSyncProgressEvent(3, 0, 4)
+ LocationServiceModule.sendSyncProgressEvent(4, 0, 4)
+ }
+ } finally {
+ unmockkObject(LocationServiceModule.Companion)
+ }
+ }
+
// --- One sender at a time ---
@Test
diff --git a/fastlane/metadata/android/en-US/changelogs/53.txt b/fastlane/metadata/android/en-US/changelogs/53.txt
index a7abc5ac..021d8bab 100644
--- a/fastlane/metadata/android/en-US/changelogs/53.txt
+++ b/fastlane/metadata/android/en-US/changelogs/53.txt
@@ -4,3 +4,4 @@
- Positioning accuracy setting in Tracking & sync: High accuracy (default) or Lower accuracy to save battery
- Fixed tracking recording nothing for hours after Android recreated it
- Fixed GPS staying off when leaving a zone with motionless pause after the app was updated or restarted
+- Fixed Sync now saying the app did not answer while the server was rejecting an Overland batch and Colota was retrying it in smaller parts