Drop GNSS batching, filter implausible and noisy fixes
Batching did not cause the jumps, and late batches break any rule that compares a fix with the previous one. - A GNSS speed below its speed accuracy counts as still. Indoors, a still phone reported up to 2 m/s, which kept GNSS on all day. - A vaguer fix counts only once the phone could have moved that far at 5 m/s, as in Organic Maps. This drops cell positions 750 m off. - Network fixes within 45 s after a GNSS fix are dropped. - Older fixes and fixes in the same second are dropped. - A minimum update interval stops faster fixes requested by other apps. - Fix an overflow that kept network fixes from switching to DWELL before the first GNSS fix. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Mandroid/README.md
@@ -30,7 +30,7 @@ Menu → Permissions shows what is missing. Many vendors add their own battery s
The app uses GNSS only while the phone moves. The phone's motion sensor wakes it again. When the phone is still, the app records one network position every 15 minutes.
The GNSS chip collects positions while the phone sleeps and delivers them in batches. Uploads also go out in batches. So the map can lag a few minutes behind.
Uploads go out in batches. So the map can lag a few minutes behind.
On Android 14 and newer, uploads use the system's HTTP stack with HTTP/2 and HTTP/3. Older versions use HTTP/1.1. HTTP/3 needs a server address with a host name, not an IP address.
Mandroid/app/src/main/java/org/opentracker/LocationSource.kt
@@ -27,26 +27,17 @@ class LocationSource(
}
private var current: Request? = null
/** Some GNSS chips, the emulator's among them, accept a batching request and then deliver nothing. */
var batching = true
set(value) {
field = value
current?.let {
current = null
apply(it)
}
}
/** LocationManager cannot change a running request, so a new one replaces it. Returns whether it changed. */
/** LocationManager cannot change a running request, so a new one replaces it. */
@SuppressLint("MissingPermission")
fun apply(req: Request): Boolean {
if (req == current) return false
fun apply(req: Request) {
if (req == current) return
stop()
current = req
val quality = if (req.gnss) LocationRequest.QUALITY_HIGH_ACCURACY else LocationRequest.QUALITY_LOW_POWER
// No minimum distance: the policy needs fixes from a still phone to notice that it stopped.
val lr = LocationRequest.Builder(req.intervalMs)
.setMaxUpdateDelayMillis(if (batching) req.maxDelayMs else 0)
// Without it, Android also passes on the faster fixes that other apps request, for example navigation.
.setMinUpdateIntervalMillis(req.intervalMs)
.setQuality(quality)
.build()
val providers = lm.allProviders
@@ -57,7 +48,6 @@ class LocationSource(
else if (!req.gnss) add(LocationManager.GPS_PROVIDER)
}
for (p in wanted.filter { it in providers }) lm.requestLocationUpdates(p, lr, executor, listener)
return true
}
fun stop() {
Mandroid/app/src/main/java/org/opentracker/SamplingPolicy.kt
@@ -12,6 +12,7 @@ data class Fix(
val accuracy: Float? = null,
/** m/s, from GNSS. Network fixes have none. */
val speed: Float? = null,
val speedAccuracy: Float? = null,
)
enum class Mode {
@@ -22,7 +23,7 @@ enum class Mode {
}
/** What to ask LocationManager for. */
data class Request(val gnss: Boolean, val intervalMs: Long, val maxDelayMs: Long)
data class Request(val gnss: Boolean, val intervalMs: Long)
/**
* Spends power in proportion to how fast the position changes.
@@ -37,18 +38,18 @@ class SamplingPolicy {
var mode = Mode.WALK
private set
/** Kept fixes, oldest first. GNSS batches arrive minutes late, after newer network fixes. */
private val kept = ArrayDeque<Fix>()
private val lastKept get() = kept.lastOrNull()
private var lastKept: Fix? = null
/** The last fix that passed [plausible]. */
private var lastFix: Fix? = null
private var lastGnssMs = Long.MIN_VALUE
private var lastGnssMs: Long? = null
private var stillSinceMs = 0L
private var slowSinceMs: Long? = null
/** Updates the mode and returns whether the fix is worth uploading. */
fun onFix(fix: Fix): Boolean {
tick(fix.timeMs)
if (fix.speed != null) lastGnssMs = max(lastGnssMs, fix.timeMs)
if (!plausible(fix)) return false
if (fix.speed != null) lastGnssMs = fix.timeMs
updateMode(fix, speedOf(fix))
lastFix = fix
return keep(fix)
@@ -68,19 +69,28 @@ class SamplingPolicy {
}
fun request(lowBattery: Boolean, charging: Boolean): Request {
val walk = Request(gnss = true, intervalMs = 30_000, maxDelayMs = 120_000)
val walk = Request(gnss = true, intervalMs = 30_000)
val base = when (mode) {
Mode.VEHICLE -> Request(gnss = true, intervalMs = 5_000, maxDelayMs = 60_000)
Mode.VEHICLE -> Request(gnss = true, intervalMs = 5_000)
Mode.WALK, Mode.DWELL -> walk
Mode.STATIONARY -> if (charging) walk else Request(gnss = false, intervalMs = HEARTBEAT_MS, maxDelayMs = 0)
Mode.STATIONARY -> if (charging) walk else Request(gnss = false, intervalMs = HEARTBEAT_MS)
}
return if (lowBattery && !charging) {
base.copy(intervalMs = base.intervalMs * 2, maxDelayMs = base.maxDelayMs * 2)
} else {
base
return if (lowBattery && !charging) base.copy(intervalMs = base.intervalMs * 2) else base
}
private fun plausible(fix: Fix): Boolean {
val last = lastFix ?: return true
return when {
// The network provider can repeat a position minutes later. The server keeps one point per second.
fix.timeMs / 1000 <= last.timeMs / 1000 -> false
fix.speed == null && gnssSilentMs(fix.timeMs) < NETWORK_GRACE_MS -> false
// A vaguer fix counts only once the phone could have moved that far. Cell positions often lie far outside their accuracy.
else -> (fix.accuracy ?: 0f) < (last.accuracy ?: 0f) + ASSUMED_SPEED * (fix.timeMs - last.timeMs) / 1000f
}
}
private fun gnssSilentMs(nowMs: Long) = lastGnssMs?.let { nowMs - it } ?: Long.MAX_VALUE
private fun updateMode(fix: Fix, speed: Float) {
val now = fix.timeMs
when {
@@ -104,7 +114,7 @@ class SamplingPolicy {
}
}
// Network fixes have no speed. While GNSS still reports, they cannot tell a stop.
mode != Mode.DWELL && (fix.speed != null || fix.timeMs - lastGnssMs >= LATE_MS) -> {
mode != Mode.DWELL && (fix.speed != null || gnssSilentMs(fix.timeMs) >= GNSS_LOST_MS) -> {
mode = Mode.DWELL
stillSinceMs = now
slowSinceMs = null
@@ -113,7 +123,13 @@ class SamplingPolicy {
}
/** The higher of the GNSS speed and a cautious estimate from the previous fix. Some chips report 0 while moving. */
private fun speedOf(fix: Fix): Float = max(fix.speed ?: 0f, estimatedSpeed(fix))
private fun speedOf(fix: Fix): Float = max(gnssSpeed(fix), estimatedSpeed(fix))
/** Indoors, a still phone reports up to 2 m/s, with a speed accuracy of about the same size. */
private fun gnssSpeed(fix: Fix): Float {
val speed = fix.speed ?: return 0f
return if (speed < (fix.speedAccuracy ?: 0f)) 0f else speed
}
private fun estimatedSpeed(fix: Fix): Float {
val prev = lastFix ?: return 0f
@@ -125,10 +141,7 @@ class SamplingPolicy {
}
private fun keep(fix: Fix): Boolean {
// The server keeps one point per second.
if (kept.any { it.timeMs / 1000 == fix.timeMs / 1000 }) return false
val i = kept.indexOfLast { it.timeMs < fix.timeMs }
val last = kept.getOrNull(i) ?: kept.firstOrNull()
val last = lastKept
val age = if (last == null) Long.MAX_VALUE else fix.timeMs - last.timeMs
val ok = when {
last == null -> true
@@ -139,18 +152,13 @@ class SamplingPolicy {
// Both positions can be off by their accuracy. A shorter step is noise.
else -> distance(last, fix) >= max(if (mode == Mode.VEHICLE) 20.0 else 10.0, ((last.accuracy ?: 0f) + (fix.accuracy ?: 0f)).toDouble())
}
if (ok) {
kept.add(i + 1, fix)
// Keeps one fix before the window, so a late fix still has a predecessor.
while (kept.size > 1 && kept[1].timeMs < kept.last().timeMs - LATE_MS) kept.removeFirst()
}
if (ok) lastKept = fix
return ok
}
companion object {
/** About 25 km/h. */
const val VEHICLE_SPEED = 7f
/** GNSS speed noise on a still phone stays below this. */
const val WALK_SPEED = 0.8f
const val DWELL_MS = 5 * 60_000L
const val SLOWDOWN_MS = 2 * 60_000L
@@ -158,8 +166,12 @@ class SamplingPolicy {
const val STALE_MS = 10 * 60_000L
const val MAX_ACCURACY = 50f
const val DEPARTURE_M = 200.0
/** Longer than the longest batch delay in [request]. */
const val LATE_MS = 5 * 60_000L
/** GNSS silent for this long is lost, for example indoors. */
const val GNSS_LOST_MS = 5 * 60_000L
/** While GNSS reports, network fixes only add jumps to a second track nearby. */
const val NETWORK_GRACE_MS = 45_000L
/** m/s. How fast a vague fix may plausibly have moved away from a better one. */
const val ASSUMED_SPEED = 5f
/** Metres. Equirectangular, which is exact enough at these distances. */
fun distance(a: Fix, b: Fix): Double {
Mandroid/app/src/main/java/org/opentracker/TrackerService.kt
@@ -42,11 +42,7 @@ class TrackerService : Service() {
private var backoffMs = 0L
private var uploadAtMs = Long.MAX_VALUE
private var maxUpload = MAX_UPLOAD
private var lastFixMs = 0L
/** Batching should deliver at least one fix in this time, else it is broken. 0 while not batching GNSS. */
private var batchWindowMs = 0L
private val upload = Runnable { upload() }
private val batchCheck = Runnable { checkBatching(System.currentTimeMillis()) }
private val network = object : ConnectivityManager.NetworkCallback() {
override fun onAvailable(n: Network) {
handler.post {
@@ -101,7 +97,6 @@ class TrackerService : Service() {
when (action) {
ACTION_UPLOAD -> schedule(urgent = true)
ACTION_TICK -> {
checkBatching(System.currentTimeMillis())
policy.tick(System.currentTimeMillis())
afterChange(urgent = false)
// Handler delays stop in deep sleep, so a due upload may still be waiting.
@@ -130,16 +125,18 @@ class TrackerService : Service() {
}
private fun onFixes(locations: List<Location>) {
lastFixMs = System.currentTimeMillis()
if (batchWindowMs > 0) {
handler.removeCallbacks(batchCheck)
handler.postDelayed(batchCheck, batchWindowMs)
}
val battery = getSystemService(BatteryManager::class.java).getIntProperty(BatteryManager.BATTERY_PROPERTY_CAPACITY)
val before = policy.mode
var kept = 0
for (l in locations) {
val fix = Fix(l.time, l.latitude, l.longitude, l.accuracyOrNull(), if (l.hasSpeed()) l.speed else null)
val fix = Fix(
l.time,
l.latitude,
l.longitude,
l.accuracyOrNull(),
if (l.hasSpeed()) l.speed else null,
if (l.hasSpeedAccuracy()) l.speedAccuracyMetersPerSecond else null,
)
if (policy.onFix(fix)) {
outbox.add(l.time / 1000, json(l, battery))
kept++
@@ -159,14 +156,7 @@ class TrackerService : Service() {
val bm = getSystemService(BatteryManager::class.java)
val low = bm.getIntProperty(BatteryManager.BATTERY_PROPERTY_CAPACITY) < LOW_BATTERY
val req = policy.request(lowBattery = low, charging = bm.isCharging)
if (source.apply(req)) {
batchWindowMs = if (req.gnss && req.maxDelayMs > 0 && source.batching) 2 * req.maxDelayMs + req.intervalMs else 0
handler.removeCallbacks(batchCheck)
if (batchWindowMs > 0) {
lastFixMs = System.currentTimeMillis()
handler.postDelayed(batchCheck, batchWindowMs)
}
}
source.apply(req)
if (policy.mode == Mode.STATIONARY) motion.arm() else motion.disarm()
if (prefs.mode != policy.mode.name) {
prefs.mode = policy.mode.name
@@ -176,15 +166,6 @@ class TrackerService : Service() {
schedule(urgent = urgent || policy.mode == Mode.STATIONARY)
}
/** Indoors, GNSS also stays silent. Then this turns batching off needlessly, but only until the service restarts. */
private fun checkBatching(now: Long) {
if (batchWindowMs > 0 && now - lastFixMs >= batchWindowMs) {
batchWindowMs = 0
Log.w(TAG, "no fixes with batching, turning it off")
source.batching = false
}
}
private fun schedule(urgent: Boolean) {
val count = outbox.count()
prefs.queued = count
Mandroid/app/src/test/java/org/opentracker/SamplingPolicyTest.kt
@@ -9,8 +9,8 @@ class SamplingPolicyTest {
private val min = 60_000L
/** About 1 m north of the start per unit of [north]. */
private fun fix(t: Long, north: Double = 0.0, speed: Float? = 0f, acc: Float = 5f) =
Fix(t, 48.0 + north / 111_320.0, 11.0, acc, speed)
private fun fix(t: Long, north: Double = 0.0, speed: Float? = 0f, acc: Float = 5f, sAcc: Float? = null) =
Fix(t, 48.0 + north / 111_320.0, 11.0, acc, speed, sAcc)
@Test
fun speeding_up_is_immediate() {
@@ -100,14 +100,37 @@ class SamplingPolicyTest {
}
@Test
fun a_late_gnss_batch_fills_the_gap_before_a_network_fix() {
fun older_fixes_are_dropped() {
val p = SamplingPolicy()
assertTrue(p.onFix(fix(60_000, speed = 1.5f)))
assertFalse(p.onFix(fix(60_500, 50.0, speed = 1.5f)))
assertFalse(p.onFix(fix(0, 50.0, speed = 1.5f)))
}
@Test
fun network_fixes_wait_while_gnss_reports() {
val p = SamplingPolicy()
assertTrue(p.onFix(fix(0, speed = 1.5f)))
assertTrue(p.onFix(fix(120_000, 160.0, speed = null, acc = 20f)))
assertTrue(p.onFix(fix(30_000, 40.0, speed = 1.5f)))
assertTrue(p.onFix(fix(60_000, 80.0, speed = 1.5f)))
assertFalse(p.onFix(fix(90_000, 84.0, speed = 1.5f)))
assertFalse(p.onFix(fix(120_500, 130.0, speed = 1.5f)))
assertFalse(p.onFix(fix(30_000, 100.0, speed = null, acc = 20f)))
assertTrue(p.onFix(fix(60_000, 100.0, speed = null, acc = 20f)))
}
@Test
fun a_vague_fix_right_after_a_better_one_is_dropped() {
val p = SamplingPolicy()
assertTrue(p.onFix(fix(0, speed = null, acc = 25f)))
assertFalse(p.onFix(fix(30_000, 750.0, speed = null, acc = 300f)))
assertTrue(p.onFix(fix(11 * min, 750.0, speed = null, acc = 300f)))
}
@Test
fun gnss_speed_inside_its_accuracy_is_still() {
val p = SamplingPolicy()
p.onFix(fix(0, speed = 0f))
p.onFix(fix(30_000, 5.0, speed = 1.9f, sAcc = 2.1f))
assertEquals(Mode.DWELL, p.mode)
p.onFix(fix(60_000, 10.0, speed = 1.4f, sAcc = 0.4f))
assertEquals(Mode.WALK, p.mode)
}
@Test