From 7684403491c186d10b17399c5826eae7bf3c7cbb Mon Sep 17 00:00:00 2001 From: James Babcock Date: Wed, 30 Sep 2026 16:01:39 -0700 Subject: [PATCH] Less-sensitive disconnect detection to avoid reconnect loops under BLE contention --- .../app/g2protocol/CfwMessageWindow.kt | 18 +++++-- .../faceclaw/app/g2protocol/MessageBuilder.kt | 2 +- .../g2protocol/session/GlassesSessionCore.kt | 18 ++++++- .../g2protocol/session/GlassesSessionSend.kt | 31 ++++++++++- .../faceclaw/app/GlassesSessionCoreTest.kt | 53 +++++++++++++++++++ .../kotlin/com/faceclaw/app/ProtocolTest.kt | 12 +++++ 6 files changed, 128 insertions(+), 6 deletions(-) diff --git a/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/g2protocol/CfwMessageWindow.kt b/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/g2protocol/CfwMessageWindow.kt index c75f3215..73b9e79e 100644 --- a/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/g2protocol/CfwMessageWindow.kt +++ b/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/g2protocol/CfwMessageWindow.kt @@ -5,11 +5,23 @@ import kotlin.jvm.JvmStatic /** Ordered completion and go-back-N recovery for the custom-message window. */ class CfwMessageWindow private constructor() { companion object { - // Flappy capture: p99 ACK 63ms, max 77ms. Allow full-frame processing - // while recovering a lost reply well before the stock timeout. - const val ACK_TIMEOUT_MS = 500 + // An unresolved message is presumed lost only once both kinds of progress on + // its arm have stopped: no write completion for DATA_STALL_MS, and no CFW ack + // for ACK_STALL_MS since it was sent. Android withholds write completions + // while its outgoing queue is congested, so completions mean data is still + // draining; acks mean the firmware is still working through the window. A + // single 500 ms deadline from send replayed large uploads whose acks were + // just slow (an 11 KB upload acked by one lens at 536 ms), and under + // bandwidth pressure (LE Audio streaming) it tore the session down. + const val DATA_STALL_MS = 500 + const val ACK_STALL_MS = 3_000 const val MAX_RETRIES = 3 + /** When a message sent at [sentAtMs] counts as lost, given its arm's latest write completion and CFW ack. */ + @JvmStatic + fun ackDeadline(sentAtMs: Long, lastArmWriteAtMs: Long, lastArmAckAtMs: Long): Long = + maxOf(lastArmWriteAtMs + DATA_STALL_MS, maxOf(sentAtMs, lastArmAckAtMs) + ACK_STALL_MS) + /** ID reuse must not invalidate an earlier command still eligible for replay. */ @JvmStatic fun canSend(messages: Iterable, candidate: OutboundMessage): Boolean = diff --git a/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/g2protocol/MessageBuilder.kt b/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/g2protocol/MessageBuilder.kt index 218c7d4c..7aea441e 100644 --- a/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/g2protocol/MessageBuilder.kt +++ b/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/g2protocol/MessageBuilder.kt @@ -97,7 +97,7 @@ class MessageBuilder { 0, streamId, payload, - CfwMessageWindow.ACK_TIMEOUT_MS, + CfwMessageWindow.ACK_STALL_MS, tileIndex, leftArm, ) diff --git a/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/g2protocol/session/GlassesSessionCore.kt b/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/g2protocol/session/GlassesSessionCore.kt index a8967265..a65cca61 100644 --- a/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/g2protocol/session/GlassesSessionCore.kt +++ b/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/g2protocol/session/GlassesSessionCore.kt @@ -314,6 +314,11 @@ class GlassesSessionCore( internal val pendingMessages = ArrayDeque() internal val cfwTransports = arrayOf(CfwTransport(platform), CfwTransport(platform)) + // Per arm (0 = left, 1 = right, like cfwTransports): when a write to it last + // completed and when a CFW ack last arrived from it. The CFW window's liveness + // signals (see CfwMessageWindow.ackDeadline). + internal val lastArmWriteAtMs = LongArray(2) + internal val lastArmCfwAckAtMs = LongArray(2) internal val inFlightMessages = ArrayDeque() internal var prewrittenMessage: OutboundMessage? = null internal var prewrittenFrames: List = emptyList() @@ -350,8 +355,10 @@ class GlassesSessionCore( val active = monitor.withLock { running } if (!active) return false return try { - link.writeFrames(address, BleProtocol.WRITE_CHAR_UUID, listOf(packet), ConnectionOptions.WRITE_MODE, + val result = link.writeFrames(address, BleProtocol.WRITE_CHAR_UUID, listOf(packet), ConnectionOptions.WRITE_MODE, ConnectionOptions.WRITE_TIMEOUT_MS) + if (result) monitor.withLock { noteWriteCompletedLocked(address, now()) } + result } catch (t: Throwable) { logLine("raw packet write failed: " + safeMessage(t)) false @@ -1348,6 +1355,8 @@ class GlassesSessionCore( val acks = CfwTransport.parseAcks(data) ?: return monitor.withLock { lastIncomingAtMs = now() + val arm = armIndex(address) + if (arm >= 0) lastArmCfwAckAtMs[arm] = lastIncomingAtMs for (ack in acks) { for (message in inFlightMessages) { val ingress = if (message.isLeftArmMessage) leftAddress else rightAddress @@ -2018,6 +2027,13 @@ class GlassesSessionCore( return id } + /** 0 for the left arm, 1 for the right (the cfwTransports order), -1 for anything else. */ + internal fun armIndex(address: String): Int = when { + address.equals(leftAddress, ignoreCase = true) -> 0 + address.equals(rightAddress, ignoreCase = true) -> 1 + else -> -1 + } + internal fun hasRingAddress(): Boolean { return ringAddress.trim().isNotEmpty() } diff --git a/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/g2protocol/session/GlassesSessionSend.kt b/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/g2protocol/session/GlassesSessionSend.kt index 3a151c19..a7548611 100644 --- a/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/g2protocol/session/GlassesSessionSend.kt +++ b/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/g2protocol/session/GlassesSessionSend.kt @@ -79,6 +79,7 @@ internal fun GlassesSessionCore.driveSession(): Long { ) } if (!inFlightMessages.isEmpty()) { + refreshCfwAckDeadlinesLocked() val oldest = inFlightMessages.firstOrNull() val replay = CfwMessageWindow.replayWindow(inFlightMessages, now) if (!replay.isEmpty()) { @@ -415,6 +416,7 @@ internal fun GlassesSessionCore.writeMessage(message: OutboundMessage): Boolean monitor.withLock { val sentAtMs = now() + if (result) noteWriteCompletedLocked(writeAddress, sentAtMs) message.sentAtMs = sentAtMs message.ackDeadlineAtMs = sentAtMs + message.ackTimeoutMs logImageUpdateSendLandmarkLocked(message) @@ -463,6 +465,7 @@ internal fun GlassesSessionCore.prewriteMessage(message: OutboundMessage): Boole if (!result) { return false } + monitor.withLock { noteWriteCompletedLocked(writeAddress, now()) } prewrittenMessage = message prewrittenFrames = frames.toList() @@ -486,13 +489,39 @@ internal fun GlassesSessionCore.spoilPrewrittenMessage(reason: String): Boolean val writeAddress = if (message.isLeftArmMessage) leftAddress else rightAddress logLine("spoiling prewritten " + message.label + ": " + reason) - return link.writeFrames( + val result = link.writeFrames( writeAddress, BleProtocol.WRITE_CHAR_UUID, listOf(finalFrame), ConnectionOptions.WRITE_MODE, ConnectionOptions.WRITE_TIMEOUT_MS ) + if (result) monitor.withLock { noteWriteCompletedLocked(writeAddress, now()) } + return result +} + +/** + * Record that writes to [address] just completed: every frame's onCharacteristicWrite + * fired. The drive loop only checks ack deadlines between writes, so noting it when + * writeFrames returns is as good as noting each callback. + */ +internal fun GlassesSessionCore.noteWriteCompletedLocked(address: String, atMs: Long) { + val arm = armIndex(address) + if (arm >= 0) lastArmWriteAtMs[arm] = atMs +} + +/** + * Re-derive each written, unresolved CFW message's ack deadline from its arm's latest + * write completion and CFW ack (CfwMessageWindow.ackDeadline), so either kind of + * progress keeps the window waiting. Deadlines only move later. + */ +internal fun GlassesSessionCore.refreshCfwAckDeadlinesLocked() { + for (message in inFlightMessages) { + if (message.sid != CfwTransport.SID || message.ackDeadlineAtMs <= 0) continue + val arm = if (message.isLeftArmMessage) 0 else 1 + message.ackDeadlineAtMs = CfwMessageWindow.ackDeadline( + message.sentAtMs, lastArmWriteAtMs[arm], lastArmCfwAckAtMs[arm]) + } } internal fun GlassesSessionCore.removePreparedMessageLocked(message: OutboundMessage?): Boolean { diff --git a/tests/kotlin/src/commonTest/kotlin/com/faceclaw/app/GlassesSessionCoreTest.kt b/tests/kotlin/src/commonTest/kotlin/com/faceclaw/app/GlassesSessionCoreTest.kt index 214a3092..3a41526e 100644 --- a/tests/kotlin/src/commonTest/kotlin/com/faceclaw/app/GlassesSessionCoreTest.kt +++ b/tests/kotlin/src/commonTest/kotlin/com/faceclaw/app/GlassesSessionCoreTest.kt @@ -464,6 +464,59 @@ class GlassesSessionCoreTest { s.core.disconnect() } + @Test + fun aCfwAckSlowerThanHalfASecondIsNotReplayed() { + val s = Session() + s.link.muted.add("image") + s.startAndAwaitLayout() + s.core.configureCompositorScreen(64, 32) + s.core.configureSurface("s", 0, 0, 64, 32, 0, SurfaceCompositor.TRANSPARENCY_OPAQUE) + try { + s.submitFrame(3, 0x20, 1) + assertTrue(waitUntil(5_000) { s.link.cfwMessages().size == 1 }) + // Well past the old 500 ms deadline, but inside the 3 s ack window. + s.platform.offsetMs += 1_500 + sleepMs(400) + assertFalse(s.host.hasLog("CFW recovery"), s.host.logs().takeLast(10).toString()) + assertEquals(1, s.link.cfwMessages().size) + // The late ack still completes the frame. + val first = s.link.writes.first { it.sid == CfwTransport.SID && it.kind == "image" } + val message = assertNotNull(s.core.monitor.withLock { s.core.inFlightMessages.firstOrNull { it.magic == first.magic } }) + for (lens in 1..2) { + s.core.onNotification(LEFT, BleProtocol.NOTIFY_CHAR_UUID, cfwAck(first.magic, lens, message.message.size, message.cfwChecksum)) + } + assertTrue(waitUntil(3_000) { s.displayedMatchesDesired() }) + } finally { + s.core.close() + } + } + + @Test + fun cfwAckDeadlinesFollowTheirOwnArmsWritesAndAcks() { + val core = Session().core + fun cfw(magic: Int, leftArm: Boolean) = OutboundMessage( + "image", "m$magic", CfwTransport.SID, 0, magic, byteArrayOf(1), CfwMessageWindow.ACK_STALL_MS, 0, leftArm) + val left = cfw(1, true) + val right = cfw(2, false) + core.monitor.withLock { + for (message in listOf(left, right)) { + message.sentAtMs = 1_000 + message.ackDeadlineAtMs = 4_000 + core.inFlightMessages.addLast(message) + } + core.lastArmCfwAckAtMs[0] = 2_000 + core.lastArmWriteAtMs[1] = 5_000 + core.refreshCfwAckDeadlinesLocked() + } + assertEquals(5_000L, left.ackDeadlineAtMs) + assertEquals(5_500L, right.ackDeadlineAtMs) + // Any CFW ack from an arm counts as that arm's progress, matched or not. + val before = core.now() + core.onNotification(LEFT, BleProtocol.NOTIFY_CHAR_UUID, cfwAck(77, 1, 1, 0)) + assertTrue(core.monitor.withLock { core.lastArmCfwAckAtMs[0] } >= before) + assertEquals(0L, core.monitor.withLock { core.lastArmCfwAckAtMs[1] }) + } + @Test fun reconnectBackoffGrowsAndRestartsAfterAStableSession() { val core = Session().core diff --git a/tests/kotlin/src/commonTest/kotlin/com/faceclaw/app/ProtocolTest.kt b/tests/kotlin/src/commonTest/kotlin/com/faceclaw/app/ProtocolTest.kt index 17db733f..9671f98f 100644 --- a/tests/kotlin/src/commonTest/kotlin/com/faceclaw/app/ProtocolTest.kt +++ b/tests/kotlin/src/commonTest/kotlin/com/faceclaw/app/ProtocolTest.kt @@ -212,6 +212,18 @@ class ProtocolTest { assertTrue(CfwMessageWindow.canSend(listOf(eviction), upload)) } + @Test + fun cfwAckDeadlineWaitsWhileEitherWritesOrAcksProgress() { + // Sent at 1000 into an idle arm: 3 s from the send. + assertEquals(4_000L, CfwMessageWindow.ackDeadline(1_000, 1_000, 0)) + // An ack from the arm restarts the 3 s ack window. + assertEquals(5_500L, CfwMessageWindow.ackDeadline(1_000, 1_000, 2_500)) + // Writes still completing keep it waiting past the ack window, 500 ms at a time. + assertEquals(6_700L, CfwMessageWindow.ackDeadline(1_000, 6_200, 2_500)) + // An ack from before the send doesn't shorten anything. + assertEquals(4_000L, CfwMessageWindow.ackDeadline(1_000, 1_000, 900)) + } + @Test fun orderedCompletionAndWholeWindowRecovery() { val head = message()