From f9724ddcabbebc7d497698c1e418ca01557d1caa Mon Sep 17 00:00:00 2001 From: James Babcock Date: Tue, 22 Sep 2026 17:26:09 -0700 Subject: [PATCH] Convert remaining Java to Kotlin --- App_Resources/Android/app.gradle | 2 +- .../com/faceclaw/app/AndroidNoiseEffects.java | 59 - .../com/faceclaw/app/AndroidNoiseEffects.kt | 67 + .../faceclaw/app/FaceclawAlarmActivity.java | 181 - .../com/faceclaw/app/FaceclawAlarmActivity.kt | 179 + .../faceclaw/app/FaceclawAlarmReceiver.java | 27 - .../com/faceclaw/app/FaceclawAlarmReceiver.kt | 26 + .../app/FaceclawAlarmRescheduleReceiver.java | 34 - .../app/FaceclawAlarmRescheduleReceiver.kt | 28 + .../faceclaw/app/FaceclawAlarmService.java | 649 --- .../com/faceclaw/app/FaceclawAlarmService.kt | 629 +++ .../java/com/faceclaw/app/FaceclawAlarms.java | 620 --- .../java/com/faceclaw/app/FaceclawAlarms.kt | 647 +++ .../faceclaw/app/FaceclawAudioTranscoder.java | 242 - .../faceclaw/app/FaceclawAudioTranscoder.kt | 236 + .../faceclaw/app/FaceclawBleCommunicator.java | 4100 ----------------- .../faceclaw/app/FaceclawBleCommunicator.kt | 4072 ++++++++++++++++ .../com/faceclaw/app/FaceclawBleManager.java | 558 --- .../com/faceclaw/app/FaceclawBleManager.kt | 539 +++ .../app/FaceclawCalendarProvider.java | 98 - .../faceclaw/app/FaceclawCalendarProvider.kt | 96 + .../faceclaw/app/FaceclawCaptionEngine.java | 446 -- .../com/faceclaw/app/FaceclawCaptionEngine.kt | 456 ++ .../app/FaceclawConversationStore.java | 778 ---- .../faceclaw/app/FaceclawConversationStore.kt | 790 ++++ .../faceclaw/app/FaceclawDeviceDiscovery.java | 386 -- .../faceclaw/app/FaceclawDeviceDiscovery.kt | 372 ++ .../faceclaw/app/FaceclawDeviceInfoProbe.java | 619 --- .../faceclaw/app/FaceclawDeviceInfoProbe.kt | 629 +++ .../com/faceclaw/app/FaceclawDiarizer.java | 262 -- .../java/com/faceclaw/app/FaceclawDiarizer.kt | 260 ++ .../faceclaw/app/FaceclawEvenAppDetector.java | 87 - .../faceclaw/app/FaceclawEvenAppDetector.kt | 90 + .../app/FaceclawEvenHubDocumentStart.java | 50 - .../app/FaceclawEvenHubDocumentStart.kt | 50 + .../faceclaw/app/FaceclawEvenHubJsBridge.java | 34 - .../faceclaw/app/FaceclawEvenHubJsBridge.kt | 26 + ...WebView.java => FaceclawEvenHubWebView.kt} | 19 +- .../app/FaceclawEvenHubWebViewClient.java | 172 - .../app/FaceclawEvenHubWebViewClient.kt | 158 + ...ost.java => FaceclawEvenHubWebViewHost.kt} | 149 +- .../faceclaw/app/FaceclawFirmwareFlasher.java | 695 --- .../faceclaw/app/FaceclawFirmwareFlasher.kt | 707 +++ .../app/FaceclawFlashPromptCommunicator.java | 546 --- .../app/FaceclawFlashPromptCommunicator.kt | 566 +++ .../app/FaceclawForegroundService.java | 142 - .../faceclaw/app/FaceclawForegroundService.kt | 141 + .../java/com/faceclaw/app/FaceclawHttp.java | 42 - .../java/com/faceclaw/app/FaceclawHttp.kt | 47 + .../com/faceclaw/app/FaceclawLc3Decoder.java | 117 - .../com/faceclaw/app/FaceclawLc3Decoder.kt | 130 + .../com/faceclaw/app/FaceclawLlamaRunner.java | 122 - .../com/faceclaw/app/FaceclawLlamaRunner.kt | 133 + .../app/FaceclawLocationProvider.java | 196 - .../faceclaw/app/FaceclawLocationProvider.kt | 188 + .../faceclaw/app/FaceclawLocationTracker.java | 188 - .../faceclaw/app/FaceclawLocationTracker.kt | 184 + .../faceclaw/app/FaceclawMediaBrowser.java | 321 -- .../com/faceclaw/app/FaceclawMediaBrowser.kt | 316 ++ .../faceclaw/app/FaceclawMediaController.java | 576 --- .../faceclaw/app/FaceclawMediaController.kt | 577 +++ ...eclawMediaNotificationListenerService.java | 657 --- ...aceclawMediaNotificationListenerService.kt | 662 +++ .../faceclaw/app/FaceclawModelDownloader.java | 187 - .../faceclaw/app/FaceclawModelDownloader.kt | 198 + .../app/FaceclawPreviewCompositor.java | 198 - .../faceclaw/app/FaceclawPreviewCompositor.kt | 199 + .../com/faceclaw/app/FaceclawQrScanner.java | 81 - .../com/faceclaw/app/FaceclawQrScanner.kt | 74 + .../com/faceclaw/app/FaceclawRemoteInput.java | 143 - .../com/faceclaw/app/FaceclawRemoteInput.kt | 152 + .../faceclaw/app/FaceclawResourceUsage.java | 68 - .../com/faceclaw/app/FaceclawResourceUsage.kt | 72 + .../com/faceclaw/app/FaceclawSettings.java | 121 - .../java/com/faceclaw/app/FaceclawSettings.kt | 112 + .../app/FaceclawSettingsPortReceiver.java | 141 - .../app/FaceclawSettingsPortReceiver.kt | 146 + .../com/faceclaw/app/FaceclawSpeakerId.java | 111 - .../com/faceclaw/app/FaceclawSpeakerId.kt | 116 + .../com/faceclaw/app/FaceclawSseRequest.java | 145 - .../com/faceclaw/app/FaceclawSseRequest.kt | 156 + .../app/FaceclawSystemStatusIconProvider.java | 193 - .../app/FaceclawSystemStatusIconProvider.kt | 193 + .../faceclaw/app/FaceclawVoiceController.java | 1162 ----- .../faceclaw/app/FaceclawVoiceController.kt | 1186 +++++ .../com/faceclaw/app/FaceclawWearBridge.java | 391 -- .../com/faceclaw/app/FaceclawWearBridge.kt | 386 ++ .../app/FaceclawWearListenerService.java | 26 - .../app/FaceclawWearListenerService.kt | 24 + .../com/faceclaw/app/FaceclawWebSocket.java | 151 - .../com/faceclaw/app/FaceclawWebSocket.kt | 156 + .../com/faceclaw/app/FontFileRenderer.java | 537 --- .../java/com/faceclaw/app/FontFileRenderer.kt | 574 +++ .../java/com/faceclaw/app/FrameTimings.java | 567 --- .../java/com/faceclaw/app/FrameTimings.kt | 554 +++ .../com/faceclaw/app/GifScreenRecorder.java | 26 - .../com/faceclaw/app/GifScreenRecorder.kt | 27 + .../java/com/faceclaw/app/IconRenderer.java | 185 - .../java/com/faceclaw/app/IconRenderer.kt | 196 + .../com/faceclaw/app/ImageFileLoader.java | 229 - .../java/com/faceclaw/app/ImageFileLoader.kt | 246 + .../com/faceclaw/app/PreviewBitmapUtil.java | 56 - .../com/faceclaw/app/PreviewBitmapUtil.kt | 61 + .../java/com/faceclaw/app/ScreenshotUtil.java | 45 - .../java/com/faceclaw/app/ScreenshotUtil.kt | 49 + .../faceclaw/app/util/InterruptibleSleep.java | 48 - .../faceclaw/app/util/InterruptibleSleep.kt | 48 + CLAUDE.md | 9 +- app/apps/timer/timer-engine.ts | 2 +- app/g2/dashboard-controller.ts | 7 +- app/native/asr-model.ts | 2 +- app/native/frame-timings.ts | 2 +- native/kotlin/README.md | 17 +- native/kotlin/android-java-boundaries.json | 114 - native/kotlin/migrated-java-sources.json | 314 +- tests/kotlin-build.test.cjs | 2 +- tests/music-settings.test.cjs | 118 +- wear/PROTOCOL.md | 2 +- 118 files changed, 18381 insertions(+), 18119 deletions(-) delete mode 100644 App_Resources/Android/src/main/java/com/faceclaw/app/AndroidNoiseEffects.java create mode 100644 App_Resources/Android/src/main/java/com/faceclaw/app/AndroidNoiseEffects.kt delete mode 100644 App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawAlarmActivity.java create mode 100644 App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawAlarmActivity.kt delete mode 100644 App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawAlarmReceiver.java create mode 100644 App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawAlarmReceiver.kt delete mode 100644 App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawAlarmRescheduleReceiver.java create mode 100644 App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawAlarmRescheduleReceiver.kt delete mode 100644 App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawAlarmService.java create mode 100644 App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawAlarmService.kt delete mode 100644 App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawAlarms.java create mode 100644 App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawAlarms.kt delete mode 100644 App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawAudioTranscoder.java create mode 100644 App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawAudioTranscoder.kt delete mode 100644 App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawBleCommunicator.java create mode 100644 App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawBleCommunicator.kt delete mode 100644 App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawBleManager.java create mode 100644 App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawBleManager.kt delete mode 100644 App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawCalendarProvider.java create mode 100644 App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawCalendarProvider.kt delete mode 100644 App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawCaptionEngine.java create mode 100644 App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawCaptionEngine.kt delete mode 100644 App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawConversationStore.java create mode 100644 App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawConversationStore.kt delete mode 100644 App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawDeviceDiscovery.java create mode 100644 App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawDeviceDiscovery.kt delete mode 100644 App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawDeviceInfoProbe.java create mode 100644 App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawDeviceInfoProbe.kt delete mode 100644 App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawDiarizer.java create mode 100644 App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawDiarizer.kt delete mode 100644 App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawEvenAppDetector.java create mode 100644 App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawEvenAppDetector.kt delete mode 100644 App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawEvenHubDocumentStart.java create mode 100644 App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawEvenHubDocumentStart.kt delete mode 100644 App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawEvenHubJsBridge.java create mode 100644 App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawEvenHubJsBridge.kt rename App_Resources/Android/src/main/java/com/faceclaw/app/{FaceclawEvenHubWebView.java => FaceclawEvenHubWebView.kt} (78%) delete mode 100644 App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawEvenHubWebViewClient.java create mode 100644 App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawEvenHubWebViewClient.kt rename App_Resources/Android/src/main/java/com/faceclaw/app/{FaceclawEvenHubWebViewHost.java => FaceclawEvenHubWebViewHost.kt} (56%) delete mode 100644 App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawFirmwareFlasher.java create mode 100644 App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawFirmwareFlasher.kt delete mode 100644 App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawFlashPromptCommunicator.java create mode 100644 App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawFlashPromptCommunicator.kt delete mode 100644 App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawForegroundService.java create mode 100644 App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawForegroundService.kt delete mode 100644 App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawHttp.java create mode 100644 App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawHttp.kt delete mode 100644 App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawLc3Decoder.java create mode 100644 App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawLc3Decoder.kt delete mode 100644 App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawLlamaRunner.java create mode 100644 App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawLlamaRunner.kt delete mode 100644 App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawLocationProvider.java create mode 100644 App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawLocationProvider.kt delete mode 100644 App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawLocationTracker.java create mode 100644 App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawLocationTracker.kt delete mode 100644 App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawMediaBrowser.java create mode 100644 App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawMediaBrowser.kt delete mode 100644 App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawMediaController.java create mode 100644 App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawMediaController.kt delete mode 100644 App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawMediaNotificationListenerService.java create mode 100644 App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawMediaNotificationListenerService.kt delete mode 100644 App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawModelDownloader.java create mode 100644 App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawModelDownloader.kt delete mode 100644 App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawPreviewCompositor.java create mode 100644 App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawPreviewCompositor.kt delete mode 100644 App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawQrScanner.java create mode 100644 App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawQrScanner.kt delete mode 100644 App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawRemoteInput.java create mode 100644 App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawRemoteInput.kt delete mode 100644 App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawResourceUsage.java create mode 100644 App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawResourceUsage.kt delete mode 100644 App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawSettings.java create mode 100644 App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawSettings.kt delete mode 100644 App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawSettingsPortReceiver.java create mode 100644 App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawSettingsPortReceiver.kt delete mode 100644 App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawSpeakerId.java create mode 100644 App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawSpeakerId.kt delete mode 100644 App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawSseRequest.java create mode 100644 App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawSseRequest.kt delete mode 100644 App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawSystemStatusIconProvider.java create mode 100644 App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawSystemStatusIconProvider.kt delete mode 100644 App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawVoiceController.java create mode 100644 App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawVoiceController.kt delete mode 100644 App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawWearBridge.java create mode 100644 App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawWearBridge.kt delete mode 100644 App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawWearListenerService.java create mode 100644 App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawWearListenerService.kt delete mode 100644 App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawWebSocket.java create mode 100644 App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawWebSocket.kt delete mode 100644 App_Resources/Android/src/main/java/com/faceclaw/app/FontFileRenderer.java create mode 100644 App_Resources/Android/src/main/java/com/faceclaw/app/FontFileRenderer.kt delete mode 100644 App_Resources/Android/src/main/java/com/faceclaw/app/FrameTimings.java create mode 100644 App_Resources/Android/src/main/java/com/faceclaw/app/FrameTimings.kt delete mode 100644 App_Resources/Android/src/main/java/com/faceclaw/app/GifScreenRecorder.java create mode 100644 App_Resources/Android/src/main/java/com/faceclaw/app/GifScreenRecorder.kt delete mode 100644 App_Resources/Android/src/main/java/com/faceclaw/app/IconRenderer.java create mode 100644 App_Resources/Android/src/main/java/com/faceclaw/app/IconRenderer.kt delete mode 100644 App_Resources/Android/src/main/java/com/faceclaw/app/ImageFileLoader.java create mode 100644 App_Resources/Android/src/main/java/com/faceclaw/app/ImageFileLoader.kt delete mode 100644 App_Resources/Android/src/main/java/com/faceclaw/app/PreviewBitmapUtil.java create mode 100644 App_Resources/Android/src/main/java/com/faceclaw/app/PreviewBitmapUtil.kt delete mode 100644 App_Resources/Android/src/main/java/com/faceclaw/app/ScreenshotUtil.java create mode 100644 App_Resources/Android/src/main/java/com/faceclaw/app/ScreenshotUtil.kt delete mode 100644 App_Resources/Android/src/main/java/com/faceclaw/app/util/InterruptibleSleep.java create mode 100644 App_Resources/Android/src/main/java/com/faceclaw/app/util/InterruptibleSleep.kt diff --git a/App_Resources/Android/app.gradle b/App_Resources/Android/app.gradle index 832823ad..65d42a3c 100644 --- a/App_Resources/Android/app.gradle +++ b/App_Resources/Android/app.gradle @@ -296,7 +296,7 @@ def removeLegacyKotlinBridgeSource = tasks.register("removeLegacyKotlinBridgeSou // Explicit, reviewed source moves: never infer deletions from class names. def migrations = new groovy.json.JsonSlurper().parse(file("$rootDir/../../native/kotlin/migrated-java-sources.json")) migrations.each { migration -> - if (!file("$rootDir/../../native/kotlin/${migration.kotlin}").isFile()) { + if (!file("$rootDir/../../${migration.kotlin}").isFile()) { throw new GradleException("Missing Kotlin replacement: ${migration.kotlin}") } delete file("$projectDir/src/main/java/${migration.java}") diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/AndroidNoiseEffects.java b/App_Resources/Android/src/main/java/com/faceclaw/app/AndroidNoiseEffects.java deleted file mode 100644 index 72757990..00000000 --- a/App_Resources/Android/src/main/java/com/faceclaw/app/AndroidNoiseEffects.java +++ /dev/null @@ -1,59 +0,0 @@ -package com.faceclaw.app; - -import android.media.audiofx.*; -import java.util.ArrayList; - -/** Android AudioRecord effect attachment; PCM processing lives in shared Kotlin. */ -public final class AndroidNoiseEffects { - /** Whether this Android build ships the built-in capture noise suppressor. */ - public static boolean platformSuppressorAvailable() { - try { - return NoiseSuppressor.isAvailable(); - } catch (Throwable t) { - return false; - } - } - - /** Short engine name for the UI. */ - public static String engineDescription() { - return "spectral (phone DSP)"; - } - - /** - * Attach the platform's built-in capture-effect chain (NoiseSuppressor, - * AcousticEchoCanceler, AutomaticGainControl — whichever this Android - * build provides) to a phone AudioRecord session. Only applicable when the - * phone itself captures the audio; the BLE glasses stream must go through - * {@link #process(byte[])} instead. The caller keeps the returned effects - * for the life of the recording and release()s them with it. - */ - public static AudioEffect[] attachPlatformEffects(int audioSessionId) { - ArrayList effects = new ArrayList<>(); - try { - if (NoiseSuppressor.isAvailable()) { - NoiseSuppressor ns = NoiseSuppressor.create(audioSessionId); - if (ns != null) { - ns.setEnabled(true); - effects.add(ns); - } - } - if (AcousticEchoCanceler.isAvailable()) { - AcousticEchoCanceler aec = AcousticEchoCanceler.create(audioSessionId); - if (aec != null) { - aec.setEnabled(true); - effects.add(aec); - } - } - if (AutomaticGainControl.isAvailable()) { - AutomaticGainControl agc = AutomaticGainControl.create(audioSessionId); - if (agc != null) { - agc.setEnabled(true); - effects.add(agc); - } - } - } catch (Throwable t) { - // A missing effect just means this tier is unavailable. - } - return effects.toArray(new AudioEffect[0]); - } -} diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/AndroidNoiseEffects.kt b/App_Resources/Android/src/main/java/com/faceclaw/app/AndroidNoiseEffects.kt new file mode 100644 index 00000000..a8db0cb5 --- /dev/null +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/AndroidNoiseEffects.kt @@ -0,0 +1,67 @@ +package com.faceclaw.app + +import android.media.audiofx.AcousticEchoCanceler +import android.media.audiofx.AudioEffect +import android.media.audiofx.AutomaticGainControl +import android.media.audiofx.NoiseSuppressor +import java.util.ArrayList + +/** Android AudioRecord effect attachment; PCM processing lives in shared Kotlin. */ +class AndroidNoiseEffects { + companion object { + /** Whether this Android build ships the built-in capture noise suppressor. */ + @JvmStatic + fun platformSuppressorAvailable(): Boolean { + return try { + NoiseSuppressor.isAvailable() + } catch (t: Throwable) { + false + } + } + + /** Short engine name for the UI. */ + @JvmStatic + fun engineDescription(): String { + return "spectral (phone DSP)" + } + + /** + * Attach the platform's built-in capture-effect chain (NoiseSuppressor, + * AcousticEchoCanceler, AutomaticGainControl — whichever this Android + * build provides) to a phone AudioRecord session. Only applicable when the + * phone itself captures the audio; the BLE glasses stream must go through + * {@link #process(byte[])} instead. The caller keeps the returned effects + * for the life of the recording and release()s them with it. + */ + @JvmStatic + fun attachPlatformEffects(audioSessionId: Int): Array { + val effects = ArrayList() + try { + if (NoiseSuppressor.isAvailable()) { + val ns = NoiseSuppressor.create(audioSessionId) + if (ns != null) { + ns.setEnabled(true) + effects.add(ns) + } + } + if (AcousticEchoCanceler.isAvailable()) { + val aec = AcousticEchoCanceler.create(audioSessionId) + if (aec != null) { + aec.setEnabled(true) + effects.add(aec) + } + } + if (AutomaticGainControl.isAvailable()) { + val agc = AutomaticGainControl.create(audioSessionId) + if (agc != null) { + agc.setEnabled(true) + effects.add(agc) + } + } + } catch (t: Throwable) { + // A missing effect just means this tier is unavailable. + } + return effects.toTypedArray() + } + } +} diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawAlarmActivity.java b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawAlarmActivity.java deleted file mode 100644 index d490ed5d..00000000 --- a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawAlarmActivity.java +++ /dev/null @@ -1,181 +0,0 @@ -package com.faceclaw.app; - -import android.app.Activity; -import android.app.KeyguardManager; -import android.content.Context; -import android.content.Intent; -import android.graphics.Color; -import android.graphics.Typeface; -import android.os.Build; -import android.os.Bundle; -import android.os.Handler; -import android.os.Looper; -import android.util.TypedValue; -import android.view.Gravity; -import android.view.View; -import android.view.WindowManager; -import android.widget.Button; -import android.widget.LinearLayout; -import android.widget.TextView; - -import java.text.DateFormat; -import java.util.Date; -import java.util.List; - -/** - * The phone's ringing screen: shown over the lock screen by the alarm - * notification's full-screen intent (or by tapping the notification). Pure - * Java, so it works with the JavaScript side asleep or gone. Snooze and - * Dismiss go to the service, which journals them for the engine. - */ -public class FaceclawAlarmActivity extends Activity { - private final Handler handler = new Handler(Looper.getMainLooper()); - private TextView titleView; - private TextView textView; - private TextView timeView; - private Button snoozeButton; - private long itemId; - - private final Runnable poll = new Runnable() { - @Override - public void run() { - List items = FaceclawAlarmService.snapshot(); - if (items.isEmpty()) { - finish(); - return; - } - FaceclawAlarmService.Ringing shown = null; - for (FaceclawAlarmService.Ringing item : items) { - if (item.id == itemId) { - shown = item; - } - } - if (shown == null) { - shown = items.get(0); - itemId = shown.id; - } - render(shown, items.size()); - handler.postDelayed(this, 1000); - } - }; - - @Override - protected void onCreate(Bundle savedInstanceState) { - super.onCreate(savedInstanceState); - showOverLockScreen(); - itemId = getIntent() == null ? 0 : getIntent().getLongExtra(FaceclawAlarms.EXTRA_ID, 0); - setContentView(buildLayout()); - } - - @Override - protected void onNewIntent(Intent intent) { - super.onNewIntent(intent); - setIntent(intent); - itemId = intent == null ? 0 : intent.getLongExtra(FaceclawAlarms.EXTRA_ID, 0); - } - - @Override - protected void onResume() { - super.onResume(); - handler.post(poll); - } - - @Override - protected void onPause() { - handler.removeCallbacks(poll); - super.onPause(); - } - - private void showOverLockScreen() { - if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O_MR1) { - setShowWhenLocked(true); - setTurnScreenOn(true); - KeyguardManager keyguard = (KeyguardManager) getSystemService(Context.KEYGUARD_SERVICE); - if (keyguard != null) { - keyguard.requestDismissKeyguard(this, null); - } - } else { - getWindow().addFlags(WindowManager.LayoutParams.FLAG_SHOW_WHEN_LOCKED - | WindowManager.LayoutParams.FLAG_TURN_SCREEN_ON - | WindowManager.LayoutParams.FLAG_DISMISS_KEYGUARD); - } - getWindow().addFlags(WindowManager.LayoutParams.FLAG_KEEP_SCREEN_ON); - } - - private View buildLayout() { - LinearLayout root = new LinearLayout(this); - root.setOrientation(LinearLayout.VERTICAL); - root.setGravity(Gravity.CENTER); - root.setBackgroundColor(Color.BLACK); - int pad = dp(32); - root.setPadding(pad, pad, pad, pad); - - timeView = new TextView(this); - timeView.setTextColor(Color.WHITE); - timeView.setTextSize(TypedValue.COMPLEX_UNIT_SP, 64); - timeView.setTypeface(Typeface.DEFAULT_BOLD); - timeView.setGravity(Gravity.CENTER); - root.addView(timeView, wrap()); - - titleView = new TextView(this); - titleView.setTextColor(Color.WHITE); - titleView.setTextSize(TypedValue.COMPLEX_UNIT_SP, 28); - titleView.setGravity(Gravity.CENTER); - titleView.setPadding(0, dp(16), 0, 0); - root.addView(titleView, wrap()); - - textView = new TextView(this); - textView.setTextColor(Color.LTGRAY); - textView.setTextSize(TypedValue.COMPLEX_UNIT_SP, 18); - textView.setGravity(Gravity.CENTER); - textView.setPadding(0, dp(8), 0, dp(40)); - root.addView(textView, wrap()); - - snoozeButton = new Button(this); - snoozeButton.setTextSize(TypedValue.COMPLEX_UNIT_SP, 20); - snoozeButton.setOnClickListener(view -> act(FaceclawAlarmService.ACTION_SNOOZE)); - root.addView(snoozeButton, button()); - - Button dismissButton = new Button(this); - dismissButton.setText("Dismiss"); - dismissButton.setTextSize(TypedValue.COMPLEX_UNIT_SP, 20); - dismissButton.setOnClickListener(view -> act(FaceclawAlarmService.ACTION_DISMISS)); - root.addView(dismissButton, button()); - return root; - } - - private void render(FaceclawAlarmService.Ringing item, int count) { - timeView.setText(DateFormat.getTimeInstance(DateFormat.SHORT).format(new Date())); - titleView.setText(count > 1 ? item.title + " (+" + (count - 1) + " more)" : item.title); - String status = item.silenced ? "Not answered" : item.escalated ? "" : "Ringing on the glasses"; - textView.setText(item.text.isEmpty() ? status : status.isEmpty() ? item.text : item.text + "\n" + status); - snoozeButton.setText(FaceclawAlarms.KIND_TIMER.equals(item.kind) - ? "+" + item.snoozeMinutes + " min" - : "Snooze " + item.snoozeMinutes + " min"); - } - - /** Apply the action to every ringing item (one tap silences the phone). */ - private void act(String action) { - for (FaceclawAlarmService.Ringing item : FaceclawAlarmService.snapshot()) { - Intent intent = new Intent(this, FaceclawAlarmService.class) - .setAction(action) - .putExtra(FaceclawAlarms.EXTRA_ID, item.id); - startService(intent); - } - finish(); - } - - private LinearLayout.LayoutParams wrap() { - return new LinearLayout.LayoutParams(LinearLayout.LayoutParams.MATCH_PARENT, LinearLayout.LayoutParams.WRAP_CONTENT); - } - - private LinearLayout.LayoutParams button() { - LinearLayout.LayoutParams params = new LinearLayout.LayoutParams(LinearLayout.LayoutParams.MATCH_PARENT, dp(64)); - params.topMargin = dp(12); - return params; - } - - private int dp(int value) { - return Math.round(TypedValue.applyDimension(TypedValue.COMPLEX_UNIT_DIP, value, getResources().getDisplayMetrics())); - } -} diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawAlarmActivity.kt b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawAlarmActivity.kt new file mode 100644 index 00000000..3aab1c35 --- /dev/null +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawAlarmActivity.kt @@ -0,0 +1,179 @@ +package com.faceclaw.app + +import android.app.Activity +import android.app.KeyguardManager +import android.content.Context +import android.content.Intent +import android.graphics.Color +import android.graphics.Typeface +import android.os.Build +import android.os.Bundle +import android.os.Handler +import android.os.Looper +import android.util.TypedValue +import android.view.Gravity +import android.view.View +import android.view.WindowManager +import android.widget.Button +import android.widget.LinearLayout +import android.widget.TextView + +import java.text.DateFormat +import java.util.Date + +/** + * The phone's ringing screen: shown over the lock screen by the alarm + * notification's full-screen intent (or by tapping the notification). Pure + * Java, so it works with the JavaScript side asleep or gone. Snooze and + * Dismiss go to the service, which journals them for the engine. + */ +class FaceclawAlarmActivity : Activity() { + private val handler = Handler(Looper.getMainLooper()) + private var titleView: TextView? = null + private var textView: TextView? = null + private var timeView: TextView? = null + private var snoozeButton: Button? = null + private var itemId: Long = 0 + + private val poll: Runnable = object : Runnable { + override fun run() { + val items = FaceclawAlarmService.snapshot() + if (items.isEmpty()) { + finish() + return + } + var shown: FaceclawAlarmService.Ringing? = null + for (item in items) { + if (item.id == itemId) { + shown = item + } + } + if (shown == null) { + shown = items[0] + itemId = shown.id + } + render(shown, items.size) + handler.postDelayed(this, 1000) + } + } + + override fun onCreate(savedInstanceState: Bundle?) { + super.onCreate(savedInstanceState) + showOverLockScreen() + itemId = if (intent == null) 0 else intent.getLongExtra(FaceclawAlarms.EXTRA_ID, 0) + setContentView(buildLayout()) + } + + override fun onNewIntent(intent: Intent?) { + super.onNewIntent(intent) + setIntent(intent) + itemId = if (intent == null) 0 else intent.getLongExtra(FaceclawAlarms.EXTRA_ID, 0) + } + + override fun onResume() { + super.onResume() + handler.post(poll) + } + + override fun onPause() { + handler.removeCallbacks(poll) + super.onPause() + } + + private fun showOverLockScreen() { + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O_MR1) { + setShowWhenLocked(true) + setTurnScreenOn(true) + val keyguard = getSystemService(Context.KEYGUARD_SERVICE) as KeyguardManager? + keyguard?.requestDismissKeyguard(this, null) + } else { + @Suppress("DEPRECATION") + window.addFlags(WindowManager.LayoutParams.FLAG_SHOW_WHEN_LOCKED + or WindowManager.LayoutParams.FLAG_TURN_SCREEN_ON + or WindowManager.LayoutParams.FLAG_DISMISS_KEYGUARD) + } + window.addFlags(WindowManager.LayoutParams.FLAG_KEEP_SCREEN_ON) + } + + private fun buildLayout(): View { + val root = LinearLayout(this) + root.orientation = LinearLayout.VERTICAL + root.gravity = Gravity.CENTER + root.setBackgroundColor(Color.BLACK) + val pad = dp(32) + root.setPadding(pad, pad, pad, pad) + + val time = TextView(this) + time.setTextColor(Color.WHITE) + time.setTextSize(TypedValue.COMPLEX_UNIT_SP, 64f) + time.typeface = Typeface.DEFAULT_BOLD + time.gravity = Gravity.CENTER + root.addView(time, wrap()) + timeView = time + + val title = TextView(this) + title.setTextColor(Color.WHITE) + title.setTextSize(TypedValue.COMPLEX_UNIT_SP, 28f) + title.gravity = Gravity.CENTER + title.setPadding(0, dp(16), 0, 0) + root.addView(title, wrap()) + titleView = title + + val text = TextView(this) + text.setTextColor(Color.LTGRAY) + text.setTextSize(TypedValue.COMPLEX_UNIT_SP, 18f) + text.gravity = Gravity.CENTER + text.setPadding(0, dp(8), 0, dp(40)) + root.addView(text, wrap()) + textView = text + + val snooze = Button(this) + snooze.setTextSize(TypedValue.COMPLEX_UNIT_SP, 20f) + snooze.setOnClickListener { act(FaceclawAlarmService.ACTION_SNOOZE) } + root.addView(snooze, button()) + snoozeButton = snooze + + val dismissButton = Button(this) + dismissButton.text = "Dismiss" + dismissButton.setTextSize(TypedValue.COMPLEX_UNIT_SP, 20f) + dismissButton.setOnClickListener { act(FaceclawAlarmService.ACTION_DISMISS) } + root.addView(dismissButton, button()) + return root + } + + private fun render(item: FaceclawAlarmService.Ringing, count: Int) { + timeView!!.text = DateFormat.getTimeInstance(DateFormat.SHORT).format(Date()) + titleView!!.text = if (count > 1) item.title + " (+" + (count - 1) + " more)" else item.title + val status = if (item.silenced) "Not answered" else if (item.escalated) "" else "Ringing on the glasses" + textView!!.text = if (item.text.isEmpty()) status else if (status.isEmpty()) item.text else item.text + "\n" + status + snoozeButton!!.text = if (FaceclawAlarms.KIND_TIMER == item.kind) + "+" + item.snoozeMinutes + " min" + else + "Snooze " + item.snoozeMinutes + " min" + } + + /** Apply the action to every ringing item (one tap silences the phone). */ + private fun act(action: String) { + for (item in FaceclawAlarmService.snapshot()) { + val intent = Intent(this, FaceclawAlarmService::class.java) + .setAction(action) + .putExtra(FaceclawAlarms.EXTRA_ID, item.id) + startService(intent) + } + finish() + } + + private fun wrap(): LinearLayout.LayoutParams { + return LinearLayout.LayoutParams(LinearLayout.LayoutParams.MATCH_PARENT, LinearLayout.LayoutParams.WRAP_CONTENT) + } + + private fun button(): LinearLayout.LayoutParams { + val params = LinearLayout.LayoutParams(LinearLayout.LayoutParams.MATCH_PARENT, dp(64)) + params.topMargin = dp(12) + return params + } + + private fun dp(value: Int): Int { + return Math.round(TypedValue.applyDimension(TypedValue.COMPLEX_UNIT_DIP, value.toFloat(), resources.displayMetrics)) + } +} diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawAlarmReceiver.java b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawAlarmReceiver.java deleted file mode 100644 index b8d44514..00000000 --- a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawAlarmReceiver.java +++ /dev/null @@ -1,27 +0,0 @@ -package com.faceclaw.app; - -import android.content.BroadcastReceiver; -import android.content.Context; -import android.content.Intent; - -/** The AlarmManager target: an item came due, start the ringing service. */ -public class FaceclawAlarmReceiver extends BroadcastReceiver { - @Override - public void onReceive(Context context, Intent intent) { - if (intent == null || !FaceclawAlarms.ACTION_EXPIRE.equals(intent.getAction())) { - return; - } - long id = intent.getLongExtra(FaceclawAlarms.EXTRA_ID, 0); - if (id == 0) { - return; - } - FaceclawAlarmService.ring( - context.getApplicationContext(), - id, - intent.getStringExtra(FaceclawAlarms.EXTRA_TITLE), - intent.getStringExtra(FaceclawAlarms.EXTRA_TEXT), - intent.getStringExtra(FaceclawAlarms.EXTRA_KIND), - intent.getIntExtra(FaceclawAlarms.EXTRA_SNOOZE_MINUTES, 10) - ); - } -} diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawAlarmReceiver.kt b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawAlarmReceiver.kt new file mode 100644 index 00000000..c884bbf1 --- /dev/null +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawAlarmReceiver.kt @@ -0,0 +1,26 @@ +package com.faceclaw.app + +import android.content.BroadcastReceiver +import android.content.Context +import android.content.Intent + +/** The AlarmManager target: an item came due, start the ringing service. */ +class FaceclawAlarmReceiver : BroadcastReceiver() { + override fun onReceive(context: Context, intent: Intent?) { + if (intent == null || FaceclawAlarms.ACTION_EXPIRE != intent.action) { + return + } + val id = intent.getLongExtra(FaceclawAlarms.EXTRA_ID, 0) + if (id == 0L) { + return + } + FaceclawAlarmService.ring( + context.applicationContext, + id, + intent.getStringExtra(FaceclawAlarms.EXTRA_TITLE), + intent.getStringExtra(FaceclawAlarms.EXTRA_TEXT), + intent.getStringExtra(FaceclawAlarms.EXTRA_KIND), + intent.getIntExtra(FaceclawAlarms.EXTRA_SNOOZE_MINUTES, 10) + ) + } +} diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawAlarmRescheduleReceiver.java b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawAlarmRescheduleReceiver.java deleted file mode 100644 index 4d7ed1b9..00000000 --- a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawAlarmRescheduleReceiver.java +++ /dev/null @@ -1,34 +0,0 @@ -package com.faceclaw.app; - -import android.content.BroadcastReceiver; -import android.content.Context; -import android.content.Intent; - -/** - * Re-arms the persisted alarm schedule when the system would otherwise have - * dropped it: after a reboot (AlarmManager alarms do not survive one), a - * package update, or a wall-clock / time-zone change. Runs without the - * JavaScript side; only the device-protected schedule file is needed. - */ -public class FaceclawAlarmRescheduleReceiver extends BroadcastReceiver { - @Override - public void onReceive(Context context, Intent intent) { - String action = intent == null ? null : intent.getAction(); - if (action == null) { - return; - } - switch (action) { - case Intent.ACTION_BOOT_COMPLETED: - case Intent.ACTION_LOCKED_BOOT_COMPLETED: - case Intent.ACTION_MY_PACKAGE_REPLACED: - case Intent.ACTION_TIME_CHANGED: - case Intent.ACTION_TIMEZONE_CHANGED: - case "android.intent.action.QUICKBOOT_POWERON": - case "com.htc.intent.action.QUICKBOOT_POWERON": - FaceclawAlarms.rescheduleAll(context, action); - break; - default: - break; - } - } -} diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawAlarmRescheduleReceiver.kt b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawAlarmRescheduleReceiver.kt new file mode 100644 index 00000000..d591704a --- /dev/null +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawAlarmRescheduleReceiver.kt @@ -0,0 +1,28 @@ +package com.faceclaw.app + +import android.content.BroadcastReceiver +import android.content.Context +import android.content.Intent + +/** + * Re-arms the persisted alarm schedule when the system would otherwise have + * dropped it: after a reboot (AlarmManager alarms do not survive one), a + * package update, or a wall-clock / time-zone change. Runs without the + * JavaScript side; only the device-protected schedule file is needed. + */ +class FaceclawAlarmRescheduleReceiver : BroadcastReceiver() { + override fun onReceive(context: Context, intent: Intent?) { + val action = intent?.action ?: return + when (action) { + Intent.ACTION_BOOT_COMPLETED, + Intent.ACTION_LOCKED_BOOT_COMPLETED, + Intent.ACTION_MY_PACKAGE_REPLACED, + Intent.ACTION_TIME_CHANGED, + Intent.ACTION_TIMEZONE_CHANGED, + "android.intent.action.QUICKBOOT_POWERON", + "com.htc.intent.action.QUICKBOOT_POWERON" -> + FaceclawAlarms.rescheduleAll(context, action) + else -> {} + } + } +} diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawAlarmService.java b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawAlarmService.java deleted file mode 100644 index a9cac015..00000000 --- a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawAlarmService.java +++ /dev/null @@ -1,649 +0,0 @@ -package com.faceclaw.app; - -import android.app.Notification; -import android.app.NotificationChannel; -import android.app.NotificationManager; -import android.app.PendingIntent; -import android.app.Service; -import android.content.Context; -import android.content.Intent; -import android.content.pm.ServiceInfo; -import android.media.AudioAttributes; -import android.media.AudioFocusRequest; -import android.media.AudioManager; -import android.media.MediaPlayer; -import android.media.RingtoneManager; -import android.net.Uri; -import android.os.Build; -import android.os.Handler; -import android.os.IBinder; -import android.os.Looper; -import android.os.PowerManager; -import android.os.VibrationEffect; -import android.os.Vibrator; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; - -/** - * The ringing foreground service. One instance carries every item currently - * ringing. Each item starts silent on the phone (a heads-up / lock-screen - * notification only, the glasses do the ringing) and escalates to phone - * sound and vibration when the glasses cannot carry it: not connected, not - * on a head, charging, no delivery confirmation from the JS side within a - * few seconds, or no acknowledgement within 30 seconds of delivery. Sound - * uses the alarm stream, so media volume, the ringer switch and (unless set - * to total silence) Do Not Disturb do not silence it. - */ -public class FaceclawAlarmService extends Service { - public static final String CHANNEL_ID = "faceclaw-alarms"; - static final String ACTION_RING = "com.faceclaw.app.action.ALARM_RING"; - static final String ACTION_DELIVERED = "com.faceclaw.app.action.ALARM_DELIVERED"; - static final String ACTION_DISMISS = "com.faceclaw.app.action.ALARM_DISMISS"; - static final String ACTION_SNOOZE = "com.faceclaw.app.action.ALARM_SNOOZE"; - /** Stop an item quietly (acknowledged on the glasses / cancelled by the engine). */ - static final String ACTION_STOP_ITEM = "com.faceclaw.app.action.ALARM_STOP_ITEM"; - - /** How long to wait for the JS side to confirm the glasses are showing the item. */ - static final long DELIVERY_WAIT_MS = 5_000L; - /** How long after delivery to the glasses before the phone joins in. */ - static final long ACK_WAIT_MS = 30_000L; - /** Sound and vibration stop after this; the notification stays. */ - static final long AUTO_SILENCE_MS = 10 * 60_000L; - /** Alarm-stream level (fraction of max) used when the wearer has it muted. */ - static final float MUTED_VOLUME_FRACTION = 0.6f; - - static final class Ringing { - final long id; - final String title; - final String text; - final String kind; - final int snoozeMinutes; - final long ringAtMs; - long deliveredAtMs; - boolean escalated; - boolean silenced; - - Ringing(long id, String title, String text, String kind, int snoozeMinutes) { - this.id = id; - this.title = title == null || title.trim().isEmpty() ? "Alarm" : title; - this.text = text == null ? "" : text; - this.kind = kind == null ? FaceclawAlarms.KIND_ALARM : kind; - this.snoozeMinutes = Math.max(1, snoozeMinutes); - this.ringAtMs = System.currentTimeMillis(); - } - } - - /** Ringing items, oldest first; static so the activity can read them. */ - private static final Map ringing = Collections.synchronizedMap(new LinkedHashMap<>()); - /** - * Signals that arrived before the ring intent was processed (the JS - * engine fires, launches its window and reports delivery on the same - * main thread the service's onStartCommand queues behind), keyed by id - * with their arrival time. Consumed by startRinging. - */ - private static final Map deliveredEarly = Collections.synchronizedMap(new LinkedHashMap<>()); - private static final Map stoppedEarly = Collections.synchronizedMap(new LinkedHashMap<>()); - private static final long EARLY_SIGNAL_MAX_AGE_MS = 15_000L; - - private final Handler handler = new Handler(Looper.getMainLooper()); - private MediaPlayer player; - private Vibrator vibrator; - private PowerManager.WakeLock wakeLock; - private AudioManager audioManager; - private AudioFocusRequest focusRequest; - private int restoreAlarmVolume = -1; - private long foregroundId; - - // ------------------------------------------------------------------ - // Static entry points - - public static void ring(Context context, long id, String title, String text, String kind, int snoozeMinutes) { - Context appContext = context.getApplicationContext(); - Intent intent = new Intent(appContext, FaceclawAlarmService.class) - .setAction(ACTION_RING) - .putExtra(FaceclawAlarms.EXTRA_ID, id) - .putExtra(FaceclawAlarms.EXTRA_TITLE, title) - .putExtra(FaceclawAlarms.EXTRA_TEXT, text) - .putExtra(FaceclawAlarms.EXTRA_KIND, kind) - .putExtra(FaceclawAlarms.EXTRA_SNOOZE_MINUTES, snoozeMinutes); - FaceclawAlarms.startServiceCompat(appContext, intent); - } - - public static void delivered(Context context, long id) { - if (!ringing.containsKey(id)) { - deliveredEarly.put(id, System.currentTimeMillis()); - return; - } - Context appContext = context.getApplicationContext(); - appContext.startService(new Intent(appContext, FaceclawAlarmService.class) - .setAction(ACTION_DELIVERED) - .putExtra(FaceclawAlarms.EXTRA_ID, id)); - } - - public static void stopItem(Context context, long id) { - if (!ringing.containsKey(id)) { - stoppedEarly.put(id, System.currentTimeMillis()); - return; - } - Context appContext = context.getApplicationContext(); - appContext.startService(new Intent(appContext, FaceclawAlarmService.class) - .setAction(ACTION_STOP_ITEM) - .putExtra(FaceclawAlarms.EXTRA_ID, id)); - } - - public static boolean isRinging(long id) { - return ringing.containsKey(id); - } - - public static boolean isAnythingRinging() { - return !ringing.isEmpty(); - } - - static List snapshot() { - synchronized (ringing) { - return new ArrayList<>(ringing.values()); - } - } - - static PendingIntent actionIntent(Context appContext, String action, long id) { - Intent intent = new Intent(appContext, FaceclawAlarmService.class) - .setAction(action) - .putExtra(FaceclawAlarms.EXTRA_ID, id); - int flags = PendingIntent.FLAG_UPDATE_CURRENT; - if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.M) { - flags |= PendingIntent.FLAG_IMMUTABLE; - } - int code = FaceclawAlarms.requestCode(id) ^ action.hashCode(); - if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) { - return PendingIntent.getForegroundService(appContext, code, intent, flags); - } - return PendingIntent.getService(appContext, code, intent, flags); - } - - // ------------------------------------------------------------------ - // Service lifecycle - - @Override - public IBinder onBind(Intent intent) { - return null; - } - - @Override - public int onStartCommand(Intent intent, int flags, int startId) { - String action = intent == null ? null : intent.getAction(); - long id = intent == null ? 0 : intent.getLongExtra(FaceclawAlarms.EXTRA_ID, 0); - if (action == null) { - // Restarted by the system with nothing to do. - finishIfIdle(); - return START_NOT_STICKY; - } - switch (action) { - case ACTION_RING: - startRinging(new Ringing( - id, - intent.getStringExtra(FaceclawAlarms.EXTRA_TITLE), - intent.getStringExtra(FaceclawAlarms.EXTRA_TEXT), - intent.getStringExtra(FaceclawAlarms.EXTRA_KIND), - intent.getIntExtra(FaceclawAlarms.EXTRA_SNOOZE_MINUTES, 10) - )); - break; - case ACTION_DELIVERED: - markDelivered(id); - break; - case ACTION_DISMISS: - dismiss(id); - break; - case ACTION_SNOOZE: - snooze(id); - break; - case ACTION_STOP_ITEM: - stopItemQuietly(id, "acknowledged"); - break; - default: - break; - } - finishIfIdle(); - return START_NOT_STICKY; - } - - @Override - public void onDestroy() { - handler.removeCallbacksAndMessages(null); - stopSound(); - releaseWakeLock(); - super.onDestroy(); - } - - // ------------------------------------------------------------------ - // Ringing - - private void startRinging(Ringing item) { - if (item.id == 0) { - return; - } - ensureChannel(); - Ringing existing = ringing.get(item.id); - if (existing != null) { - // The engine's JS timeout and the AlarmManager alarm both fired: one ring. - showNotification(existing); - return; - } - if (takeEarlySignal(stoppedEarly, item.id)) { - // Acknowledged or cancelled before this intent was processed. - FaceclawAlarms.log(this, "ring " + item.id + " already acknowledged"); - return; - } - ringing.put(item.id, item); - FaceclawAlarms.log(this, "ring " + item.kind + " " + item.id + " (" + FaceclawAlarms.glassesStatusDescription() + ")"); - acquireWakeLock(); - showNotification(item); - if (!FaceclawAlarms.glassesCanCarryAlarm()) { - escalate(item, FaceclawAlarms.glassesStatusDescription()); - return; - } - if (takeEarlySignal(deliveredEarly, item.id)) { - markDelivered(item.id); - return; - } - // The glasses could carry it: give the JS side a moment to confirm - // they are actually showing it, then hold for the acknowledgement. - handler.postDelayed(() -> { - Ringing current = ringing.get(item.id); - if (current != null && !current.escalated && current.deliveredAtMs == 0) { - escalate(current, "not delivered to the glasses"); - } - }, DELIVERY_WAIT_MS); - } - - /** Consume an early signal for the id; stale entries are dropped. */ - private static boolean takeEarlySignal(Map signals, long id) { - Long at = signals.remove(id); - long now = System.currentTimeMillis(); - synchronized (signals) { - signals.values().removeIf(stamp -> now - stamp > EARLY_SIGNAL_MAX_AGE_MS); - } - return at != null && now - at <= EARLY_SIGNAL_MAX_AGE_MS; - } - - private void markDelivered(long id) { - Ringing item = ringing.get(id); - if (item == null || item.deliveredAtMs != 0) { - return; - } - item.deliveredAtMs = System.currentTimeMillis(); - FaceclawAlarms.log(this, "delivered " + id + " to glasses"); - handler.postDelayed(() -> { - Ringing current = ringing.get(id); - if (current != null && !current.escalated) { - escalate(current, "not acknowledged on the glasses"); - } - }, ACK_WAIT_MS); - } - - private void escalate(Ringing item, String reason) { - if (item.escalated) { - return; - } - item.escalated = true; - FaceclawAlarms.log(this, "phone sound for " + item.id + ": " + reason); - showNotification(item); - startSound(); - handler.postDelayed(() -> { - Ringing current = ringing.get(item.id); - if (current != null && current.escalated && !current.silenced) { - current.silenced = true; - FaceclawAlarms.log(this, "auto-silenced " + current.id); - showNotification(current); - syncSound(); - } - }, AUTO_SILENCE_MS); - } - - private void dismiss(long id) { - Ringing item = ringing.get(id); - if (item == null) { - return; - } - FaceclawAlarms.recordPhoneAction(this, id, "dismiss", 0); - removeItem(item); - // Also drop the schedule entry (a one-off is over; the engine re-arms repeats). - FaceclawAlarms.cancel(this, id); - } - - private void snooze(long id) { - Ringing item = ringing.get(id); - if (item == null) { - return; - } - int minutes = item.snoozeMinutes; - FaceclawAlarms.recordPhoneAction(this, id, "snooze", minutes); - removeItem(item); - // Re-arm on the phone directly so the snooze holds even with the JS - // side gone; the engine replays the journal and lands on the same id. - FaceclawAlarms.schedule( - this, - id, - System.currentTimeMillis() + minutes * 60_000L, - item.title, - item.text, - item.kind, - minutes - ); - } - - private void stopItemQuietly(long id, String reason) { - Ringing item = ringing.get(id); - if (item == null) { - return; - } - FaceclawAlarms.log(this, reason + " " + id); - removeItem(item); - } - - private void removeItem(Ringing item) { - ringing.remove(item.id); - NotificationManager manager = getSystemService(NotificationManager.class); - if (manager != null) { - manager.cancel(notificationId(item.id)); - } - syncSound(); - if (item.id == foregroundId) { - promoteAnotherToForeground(); - } - } - - private void finishIfIdle() { - if (!ringing.isEmpty()) { - return; - } - handler.removeCallbacksAndMessages(null); - stopSound(); - releaseWakeLock(); - stopForeground(STOP_FOREGROUND_REMOVE); - stopSelf(); - } - - // ------------------------------------------------------------------ - // Notification (the phone's silent display of the alarm) - - private void showNotification(Ringing item) { - Notification notification = buildNotification(item); - if (foregroundId == 0 || foregroundId == item.id || !ringing.containsKey(foregroundId)) { - foregroundId = item.id; - startForegroundCompat(notificationId(item.id), notification); - } else { - NotificationManager manager = getSystemService(NotificationManager.class); - if (manager != null) { - manager.notify(notificationId(item.id), notification); - } - } - } - - private void promoteAnotherToForeground() { - List items = snapshot(); - if (items.isEmpty()) { - foregroundId = 0; - return; - } - Ringing next = items.get(0); - foregroundId = next.id; - startForegroundCompat(notificationId(next.id), buildNotification(next)); - } - - private void startForegroundCompat(int id, Notification notification) { - if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q) { - startForeground(id, notification, ServiceInfo.FOREGROUND_SERVICE_TYPE_MEDIA_PLAYBACK); - } else { - startForeground(id, notification); - } - } - - private Notification buildNotification(Ringing item) { - Intent activity = new Intent(this, FaceclawAlarmActivity.class) - .addFlags(Intent.FLAG_ACTIVITY_NEW_TASK | Intent.FLAG_ACTIVITY_CLEAR_TOP) - .putExtra(FaceclawAlarms.EXTRA_ID, item.id); - int flags = PendingIntent.FLAG_UPDATE_CURRENT; - if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.M) { - flags |= PendingIntent.FLAG_IMMUTABLE; - } - PendingIntent open = PendingIntent.getActivity(this, FaceclawAlarms.requestCode(item.id), activity, flags); - - String status; - if (item.silenced) { - status = "Not answered"; - } else if (item.escalated) { - status = "Ringing"; - } else { - status = "Ringing on the glasses"; - } - String text = item.text.isEmpty() ? status : item.text + " · " + status; - - Notification.Builder builder = Build.VERSION.SDK_INT >= Build.VERSION_CODES.O - ? new Notification.Builder(this, CHANNEL_ID) - : new Notification.Builder(this).setPriority(Notification.PRIORITY_MAX); - builder.setContentTitle(item.title) - .setContentText(text) - .setSmallIcon(getApplicationInfo().icon) - .setContentIntent(open) - .setFullScreenIntent(open, true) - .setCategory(Notification.CATEGORY_ALARM) - .setVisibility(Notification.VISIBILITY_PUBLIC) - .setOngoing(true) - .setAutoCancel(false) - .setOnlyAlertOnce(true) - .setWhen(item.ringAtMs) - .setShowWhen(true) - .addAction(new Notification.Action.Builder( - null, - FaceclawAlarms.KIND_TIMER.equals(item.kind) ? "+" + item.snoozeMinutes + " min" : "Snooze " + item.snoozeMinutes + " min", - actionIntent(this, ACTION_SNOOZE, item.id)).build()) - .addAction(new Notification.Action.Builder( - null, - "Dismiss", - actionIntent(this, ACTION_DISMISS, item.id)).build()); - if (Build.VERSION.SDK_INT < Build.VERSION_CODES.O) { - // Pre-channel: the builder controls sound, and we want none. - builder.setSound(null).setVibrate(null); - } - return builder.build(); - } - - private void ensureChannel() { - if (Build.VERSION.SDK_INT < Build.VERSION_CODES.O) { - return; - } - NotificationManager manager = getSystemService(NotificationManager.class); - if (manager == null) { - return; - } - NotificationChannel channel = new NotificationChannel(CHANNEL_ID, "Alarms", NotificationManager.IMPORTANCE_HIGH); - channel.setDescription("Timers and alarms going off. Silent here: the sound is played by the alarm itself."); - // The service plays the sound itself (on the alarm stream) so the - // notification stays silent and vibration-free on purpose. - channel.setSound(null, null); - channel.enableVibration(false); - channel.setBypassDnd(true); - channel.setLockscreenVisibility(Notification.VISIBILITY_PUBLIC); - manager.createNotificationChannel(channel); - } - - private static int notificationId(long id) { - return 0x41000000 | (FaceclawAlarms.requestCode(id) & 0x00ffffff); - } - - // ------------------------------------------------------------------ - // Sound and vibration (only while some item is escalated and not yet silenced) - - private void syncSound() { - boolean wanted = false; - for (Ringing item : snapshot()) { - if (item.escalated && !item.silenced) { - wanted = true; - break; - } - } - if (wanted) { - startSound(); - } else { - stopSound(); - } - } - - private void startSound() { - if (player != null) { - return; - } - audioManager = (AudioManager) getSystemService(Context.AUDIO_SERVICE); - AudioAttributes attributes = new AudioAttributes.Builder() - .setUsage(AudioAttributes.USAGE_ALARM) - .setContentType(AudioAttributes.CONTENT_TYPE_SONIFICATION) - .build(); - raiseMutedAlarmVolume(); - requestAudioFocus(attributes); - Uri tone = RingtoneManager.getDefaultUri(RingtoneManager.TYPE_ALARM); - if (tone == null) { - tone = RingtoneManager.getDefaultUri(RingtoneManager.TYPE_RINGTONE); - } - try { - MediaPlayer created = new MediaPlayer(); - created.setAudioAttributes(attributes); - created.setDataSource(this, tone); - created.setLooping(true); - created.prepare(); - created.start(); - player = created; - } catch (Exception error) { - FaceclawAlarms.log(this, "alarm sound failed: " + error.getMessage()); - player = null; - } - startVibration(); - } - - private void stopSound() { - if (player != null) { - try { - player.stop(); - } catch (RuntimeException ignored) { - // Already stopped. - } - player.release(); - player = null; - } - if (vibrator != null) { - vibrator.cancel(); - vibrator = null; - } - abandonAudioFocus(); - restoreAlarmVolume(); - } - - /** - * A muted alarm stream is the one thing that would keep an alarm clock - * silent; raise it for the duration of the ring and put it back after. - */ - private void raiseMutedAlarmVolume() { - if (audioManager == null) { - return; - } - try { - int current = audioManager.getStreamVolume(AudioManager.STREAM_ALARM); - int max = audioManager.getStreamMaxVolume(AudioManager.STREAM_ALARM); - if (current == 0 && max > 0) { - restoreAlarmVolume = current; - audioManager.setStreamVolume(AudioManager.STREAM_ALARM, Math.max(1, Math.round(max * MUTED_VOLUME_FRACTION)), 0); - } - } catch (RuntimeException error) { - FaceclawAlarms.log(this, "alarm volume change failed: " + error.getMessage()); - } - } - - private void restoreAlarmVolume() { - if (audioManager == null || restoreAlarmVolume < 0) { - return; - } - try { - audioManager.setStreamVolume(AudioManager.STREAM_ALARM, restoreAlarmVolume, 0); - } catch (RuntimeException ignored) { - // Best effort. - } - restoreAlarmVolume = -1; - } - - private void requestAudioFocus(AudioAttributes attributes) { - if (audioManager == null) { - return; - } - if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) { - focusRequest = new AudioFocusRequest.Builder(AudioManager.AUDIOFOCUS_GAIN_TRANSIENT) - .setAudioAttributes(attributes) - .build(); - audioManager.requestAudioFocus(focusRequest); - } else { - audioManager.requestAudioFocus(null, AudioManager.STREAM_ALARM, AudioManager.AUDIOFOCUS_GAIN_TRANSIENT); - } - } - - private void abandonAudioFocus() { - if (audioManager == null) { - return; - } - if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) { - if (focusRequest != null) { - audioManager.abandonAudioFocusRequest(focusRequest); - focusRequest = null; - } - } else { - audioManager.abandonAudioFocus(null); - } - } - - private void startVibration() { - Vibrator device = (Vibrator) getSystemService(Context.VIBRATOR_SERVICE); - if (device == null || !device.hasVibrator()) { - return; - } - vibrator = device; - long[] pattern = {0, 600, 400, 600, 1200}; - AudioAttributes attributes = new AudioAttributes.Builder() - .setUsage(AudioAttributes.USAGE_ALARM) - .setContentType(AudioAttributes.CONTENT_TYPE_SONIFICATION) - .build(); - try { - if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) { - device.vibrate(VibrationEffect.createWaveform(pattern, 0), attributes); - } else { - device.vibrate(pattern, 0, attributes); - } - } catch (RuntimeException error) { - FaceclawAlarms.log(this, "vibration failed: " + error.getMessage()); - } - } - - // ------------------------------------------------------------------ - // Wake lock - - private void acquireWakeLock() { - if (wakeLock != null && wakeLock.isHeld()) { - return; - } - PowerManager power = (PowerManager) getSystemService(Context.POWER_SERVICE); - if (power == null) { - return; - } - wakeLock = power.newWakeLock(PowerManager.PARTIAL_WAKE_LOCK, "faceclaw:alarm"); - // Bounded: sound auto-silences at AUTO_SILENCE_MS, and a stuck lock - // past that would only drain the battery. - wakeLock.acquire(AUTO_SILENCE_MS + ACK_WAIT_MS + DELIVERY_WAIT_MS + 10_000L); - } - - private void releaseWakeLock() { - if (wakeLock != null && wakeLock.isHeld()) { - wakeLock.release(); - } - wakeLock = null; - } -} diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawAlarmService.kt b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawAlarmService.kt new file mode 100644 index 00000000..65ea4dd2 --- /dev/null +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawAlarmService.kt @@ -0,0 +1,629 @@ +package com.faceclaw.app + +import android.app.Notification +import android.app.NotificationChannel +import android.app.NotificationManager +import android.app.PendingIntent +import android.app.Service +import android.content.Context +import android.content.Intent +import android.content.pm.ServiceInfo +import android.media.AudioAttributes +import android.media.AudioFocusRequest +import android.media.AudioManager +import android.media.MediaPlayer +import android.media.RingtoneManager +import android.net.Uri +import android.os.Build +import android.os.Handler +import android.os.IBinder +import android.os.Looper +import android.os.PowerManager +import android.os.VibrationEffect +import android.os.Vibrator + +import java.util.ArrayList +import java.util.Collections +import java.util.LinkedHashMap + +/** + * The ringing foreground service. One instance carries every item currently + * ringing. Each item starts silent on the phone (a heads-up / lock-screen + * notification only, the glasses do the ringing) and escalates to phone + * sound and vibration when the glasses cannot carry it: not connected, not + * on a head, charging, no delivery confirmation from the JS side within a + * few seconds, or no acknowledgement within 30 seconds of delivery. Sound + * uses the alarm stream, so media volume, the ringer switch and (unless set + * to total silence) Do Not Disturb do not silence it. + */ +class FaceclawAlarmService : Service() { + internal class Ringing( + @JvmField val id: Long, + title: String?, + text: String?, + kind: String?, + snoozeMinutes: Int + ) { + @JvmField val title: String = if (title == null || title.trim().isEmpty()) "Alarm" else title + @JvmField val text: String = text ?: "" + @JvmField val kind: String = kind ?: FaceclawAlarms.KIND_ALARM + @JvmField val snoozeMinutes: Int = Math.max(1, snoozeMinutes) + @JvmField val ringAtMs: Long = System.currentTimeMillis() + @JvmField var deliveredAtMs: Long = 0 + @JvmField var escalated: Boolean = false + @JvmField var silenced: Boolean = false + } + + companion object { + const val CHANNEL_ID = "faceclaw-alarms" + internal const val ACTION_RING = "com.faceclaw.app.action.ALARM_RING" + internal const val ACTION_DELIVERED = "com.faceclaw.app.action.ALARM_DELIVERED" + internal const val ACTION_DISMISS = "com.faceclaw.app.action.ALARM_DISMISS" + internal const val ACTION_SNOOZE = "com.faceclaw.app.action.ALARM_SNOOZE" + /** Stop an item quietly (acknowledged on the glasses / cancelled by the engine). */ + internal const val ACTION_STOP_ITEM = "com.faceclaw.app.action.ALARM_STOP_ITEM" + + /** How long to wait for the JS side to confirm the glasses are showing the item. */ + internal const val DELIVERY_WAIT_MS = 5_000L + /** How long after delivery to the glasses before the phone joins in. */ + internal const val ACK_WAIT_MS = 30_000L + /** Sound and vibration stop after this; the notification stays. */ + internal const val AUTO_SILENCE_MS = 10 * 60_000L + /** Alarm-stream level (fraction of max) used when the wearer has it muted. */ + internal const val MUTED_VOLUME_FRACTION = 0.6f + + /** Ringing items, oldest first; static so the activity can read them. */ + private val ringing: MutableMap = Collections.synchronizedMap(LinkedHashMap()) + /** + * Signals that arrived before the ring intent was processed (the JS + * engine fires, launches its window and reports delivery on the same + * main thread the service's onStartCommand queues behind), keyed by id + * with their arrival time. Consumed by startRinging. + */ + private val deliveredEarly: MutableMap = Collections.synchronizedMap(LinkedHashMap()) + private val stoppedEarly: MutableMap = Collections.synchronizedMap(LinkedHashMap()) + private const val EARLY_SIGNAL_MAX_AGE_MS = 15_000L + + // ------------------------------------------------------------------ + // Static entry points + + @JvmStatic + fun ring(context: Context, id: Long, title: String?, text: String?, kind: String?, snoozeMinutes: Int) { + val appContext = context.applicationContext + val intent = Intent(appContext, FaceclawAlarmService::class.java) + .setAction(ACTION_RING) + .putExtra(FaceclawAlarms.EXTRA_ID, id) + .putExtra(FaceclawAlarms.EXTRA_TITLE, title) + .putExtra(FaceclawAlarms.EXTRA_TEXT, text) + .putExtra(FaceclawAlarms.EXTRA_KIND, kind) + .putExtra(FaceclawAlarms.EXTRA_SNOOZE_MINUTES, snoozeMinutes) + FaceclawAlarms.startServiceCompat(appContext, intent) + } + + @JvmStatic + fun delivered(context: Context, id: Long) { + if (!ringing.containsKey(id)) { + deliveredEarly[id] = System.currentTimeMillis() + return + } + val appContext = context.applicationContext + appContext.startService(Intent(appContext, FaceclawAlarmService::class.java) + .setAction(ACTION_DELIVERED) + .putExtra(FaceclawAlarms.EXTRA_ID, id)) + } + + @JvmStatic + fun stopItem(context: Context, id: Long) { + if (!ringing.containsKey(id)) { + stoppedEarly[id] = System.currentTimeMillis() + return + } + val appContext = context.applicationContext + appContext.startService(Intent(appContext, FaceclawAlarmService::class.java) + .setAction(ACTION_STOP_ITEM) + .putExtra(FaceclawAlarms.EXTRA_ID, id)) + } + + @JvmStatic + fun isRinging(id: Long): Boolean { + return ringing.containsKey(id) + } + + @JvmStatic + fun isAnythingRinging(): Boolean { + return !ringing.isEmpty() + } + + @JvmStatic + internal fun snapshot(): List { + synchronized(ringing) { + return ArrayList(ringing.values) + } + } + + @JvmStatic + internal fun actionIntent(appContext: Context, action: String, id: Long): PendingIntent { + val intent = Intent(appContext, FaceclawAlarmService::class.java) + .setAction(action) + .putExtra(FaceclawAlarms.EXTRA_ID, id) + var flags = PendingIntent.FLAG_UPDATE_CURRENT + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.M) { + flags = flags or PendingIntent.FLAG_IMMUTABLE + } + val code = FaceclawAlarms.requestCode(id) xor action.hashCode() + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) { + return PendingIntent.getForegroundService(appContext, code, intent, flags) + } + return PendingIntent.getService(appContext, code, intent, flags) + } + + /** Consume an early signal for the id; stale entries are dropped. */ + private fun takeEarlySignal(signals: MutableMap, id: Long): Boolean { + val at = signals.remove(id) + val now = System.currentTimeMillis() + synchronized(signals) { + signals.values.removeIf { stamp -> now - stamp > EARLY_SIGNAL_MAX_AGE_MS } + } + return at != null && now - at <= EARLY_SIGNAL_MAX_AGE_MS + } + + private fun notificationId(id: Long): Int { + return 0x41000000 or (FaceclawAlarms.requestCode(id) and 0x00ffffff) + } + } + + private val handler = Handler(Looper.getMainLooper()) + private var player: MediaPlayer? = null + private var vibrator: Vibrator? = null + private var wakeLock: PowerManager.WakeLock? = null + private var audioManager: AudioManager? = null + private var focusRequest: AudioFocusRequest? = null + private var restoreAlarmVolume = -1 + private var foregroundId: Long = 0 + + // ------------------------------------------------------------------ + // Service lifecycle + + override fun onBind(intent: Intent?): IBinder? { + return null + } + + override fun onStartCommand(intent: Intent?, flags: Int, startId: Int): Int { + val action = intent?.action + val id = if (intent == null) 0L else intent.getLongExtra(FaceclawAlarms.EXTRA_ID, 0) + if (action == null) { + // Restarted by the system with nothing to do. + finishIfIdle() + return START_NOT_STICKY + } + when (action) { + ACTION_RING -> + startRinging(Ringing( + id, + intent!!.getStringExtra(FaceclawAlarms.EXTRA_TITLE), + intent.getStringExtra(FaceclawAlarms.EXTRA_TEXT), + intent.getStringExtra(FaceclawAlarms.EXTRA_KIND), + intent.getIntExtra(FaceclawAlarms.EXTRA_SNOOZE_MINUTES, 10) + )) + ACTION_DELIVERED -> + markDelivered(id) + ACTION_DISMISS -> + dismiss(id) + ACTION_SNOOZE -> + snooze(id) + ACTION_STOP_ITEM -> + stopItemQuietly(id, "acknowledged") + else -> {} + } + finishIfIdle() + return START_NOT_STICKY + } + + override fun onDestroy() { + handler.removeCallbacksAndMessages(null) + stopSound() + releaseWakeLock() + super.onDestroy() + } + + // ------------------------------------------------------------------ + // Ringing + + private fun startRinging(item: Ringing) { + if (item.id == 0L) { + return + } + ensureChannel() + val existing = ringing[item.id] + if (existing != null) { + // The engine's JS timeout and the AlarmManager alarm both fired: one ring. + showNotification(existing) + return + } + if (takeEarlySignal(stoppedEarly, item.id)) { + // Acknowledged or cancelled before this intent was processed. + FaceclawAlarms.log(this, "ring " + item.id + " already acknowledged") + return + } + ringing[item.id] = item + FaceclawAlarms.log(this, "ring " + item.kind + " " + item.id + " (" + FaceclawAlarms.glassesStatusDescription() + ")") + acquireWakeLock() + showNotification(item) + if (!FaceclawAlarms.glassesCanCarryAlarm()) { + escalate(item, FaceclawAlarms.glassesStatusDescription()) + return + } + if (takeEarlySignal(deliveredEarly, item.id)) { + markDelivered(item.id) + return + } + // The glasses could carry it: give the JS side a moment to confirm + // they are actually showing it, then hold for the acknowledgement. + handler.postDelayed({ + val current = ringing[item.id] + if (current != null && !current.escalated && current.deliveredAtMs == 0L) { + escalate(current, "not delivered to the glasses") + } + }, DELIVERY_WAIT_MS) + } + + private fun markDelivered(id: Long) { + val item = ringing[id] + if (item == null || item.deliveredAtMs != 0L) { + return + } + item.deliveredAtMs = System.currentTimeMillis() + FaceclawAlarms.log(this, "delivered $id to glasses") + handler.postDelayed({ + val current = ringing[id] + if (current != null && !current.escalated) { + escalate(current, "not acknowledged on the glasses") + } + }, ACK_WAIT_MS) + } + + private fun escalate(item: Ringing, reason: String) { + if (item.escalated) { + return + } + item.escalated = true + FaceclawAlarms.log(this, "phone sound for " + item.id + ": " + reason) + showNotification(item) + startSound() + handler.postDelayed({ + val current = ringing[item.id] + if (current != null && current.escalated && !current.silenced) { + current.silenced = true + FaceclawAlarms.log(this, "auto-silenced " + current.id) + showNotification(current) + syncSound() + } + }, AUTO_SILENCE_MS) + } + + private fun dismiss(id: Long) { + val item = ringing[id] ?: return + FaceclawAlarms.recordPhoneAction(this, id, "dismiss", 0) + removeItem(item) + // Also drop the schedule entry (a one-off is over; the engine re-arms repeats). + FaceclawAlarms.cancel(this, id) + } + + private fun snooze(id: Long) { + val item = ringing[id] ?: return + val minutes = item.snoozeMinutes + FaceclawAlarms.recordPhoneAction(this, id, "snooze", minutes) + removeItem(item) + // Re-arm on the phone directly so the snooze holds even with the JS + // side gone; the engine replays the journal and lands on the same id. + FaceclawAlarms.schedule( + this, + id, + System.currentTimeMillis() + minutes * 60_000L, + item.title, + item.text, + item.kind, + minutes + ) + } + + private fun stopItemQuietly(id: Long, reason: String) { + val item = ringing[id] ?: return + FaceclawAlarms.log(this, "$reason $id") + removeItem(item) + } + + private fun removeItem(item: Ringing) { + ringing.remove(item.id) + val manager = getSystemService(NotificationManager::class.java) + manager?.cancel(notificationId(item.id)) + syncSound() + if (item.id == foregroundId) { + promoteAnotherToForeground() + } + } + + private fun finishIfIdle() { + if (!ringing.isEmpty()) { + return + } + handler.removeCallbacksAndMessages(null) + stopSound() + releaseWakeLock() + stopForeground(STOP_FOREGROUND_REMOVE) + stopSelf() + } + + // ------------------------------------------------------------------ + // Notification (the phone's silent display of the alarm) + + private fun showNotification(item: Ringing) { + val notification = buildNotification(item) + if (foregroundId == 0L || foregroundId == item.id || !ringing.containsKey(foregroundId)) { + foregroundId = item.id + startForegroundCompat(notificationId(item.id), notification) + } else { + val manager = getSystemService(NotificationManager::class.java) + manager?.notify(notificationId(item.id), notification) + } + } + + private fun promoteAnotherToForeground() { + val items = snapshot() + if (items.isEmpty()) { + foregroundId = 0 + return + } + val next = items[0] + foregroundId = next.id + startForegroundCompat(notificationId(next.id), buildNotification(next)) + } + + private fun startForegroundCompat(id: Int, notification: Notification) { + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q) { + startForeground(id, notification, ServiceInfo.FOREGROUND_SERVICE_TYPE_MEDIA_PLAYBACK) + } else { + startForeground(id, notification) + } + } + + private fun buildNotification(item: Ringing): Notification { + val activity = Intent(this, FaceclawAlarmActivity::class.java) + .addFlags(Intent.FLAG_ACTIVITY_NEW_TASK or Intent.FLAG_ACTIVITY_CLEAR_TOP) + .putExtra(FaceclawAlarms.EXTRA_ID, item.id) + var flags = PendingIntent.FLAG_UPDATE_CURRENT + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.M) { + flags = flags or PendingIntent.FLAG_IMMUTABLE + } + val open = PendingIntent.getActivity(this, FaceclawAlarms.requestCode(item.id), activity, flags) + + val status: String = if (item.silenced) { + "Not answered" + } else if (item.escalated) { + "Ringing" + } else { + "Ringing on the glasses" + } + val text = if (item.text.isEmpty()) status else item.text + " · " + status + + val builder = if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) + Notification.Builder(this, CHANNEL_ID) + else + Notification.Builder(this).setPriority(Notification.PRIORITY_MAX) + builder.setContentTitle(item.title) + .setContentText(text) + .setSmallIcon(applicationInfo.icon) + .setContentIntent(open) + .setFullScreenIntent(open, true) + .setCategory(Notification.CATEGORY_ALARM) + .setVisibility(Notification.VISIBILITY_PUBLIC) + .setOngoing(true) + .setAutoCancel(false) + .setOnlyAlertOnce(true) + .setWhen(item.ringAtMs) + .setShowWhen(true) + .addAction(Notification.Action.Builder( + null, + if (FaceclawAlarms.KIND_TIMER == item.kind) "+" + item.snoozeMinutes + " min" else "Snooze " + item.snoozeMinutes + " min", + actionIntent(this, ACTION_SNOOZE, item.id)).build()) + .addAction(Notification.Action.Builder( + null, + "Dismiss", + actionIntent(this, ACTION_DISMISS, item.id)).build()) + if (Build.VERSION.SDK_INT < Build.VERSION_CODES.O) { + // Pre-channel: the builder controls sound, and we want none. + builder.setSound(null).setVibrate(null) + } + return builder.build() + } + + private fun ensureChannel() { + if (Build.VERSION.SDK_INT < Build.VERSION_CODES.O) { + return + } + val manager = getSystemService(NotificationManager::class.java) ?: return + val channel = NotificationChannel(CHANNEL_ID, "Alarms", NotificationManager.IMPORTANCE_HIGH) + channel.description = "Timers and alarms going off. Silent here: the sound is played by the alarm itself." + // The service plays the sound itself (on the alarm stream) so the + // notification stays silent and vibration-free on purpose. + channel.setSound(null, null) + channel.enableVibration(false) + channel.setBypassDnd(true) + channel.lockscreenVisibility = Notification.VISIBILITY_PUBLIC + manager.createNotificationChannel(channel) + } + + // ------------------------------------------------------------------ + // Sound and vibration (only while some item is escalated and not yet silenced) + + private fun syncSound() { + var wanted = false + for (item in snapshot()) { + if (item.escalated && !item.silenced) { + wanted = true + break + } + } + if (wanted) { + startSound() + } else { + stopSound() + } + } + + private fun startSound() { + if (player != null) { + return + } + audioManager = getSystemService(Context.AUDIO_SERVICE) as AudioManager? + val attributes = AudioAttributes.Builder() + .setUsage(AudioAttributes.USAGE_ALARM) + .setContentType(AudioAttributes.CONTENT_TYPE_SONIFICATION) + .build() + raiseMutedAlarmVolume() + requestAudioFocus(attributes) + var tone: Uri? = RingtoneManager.getDefaultUri(RingtoneManager.TYPE_ALARM) + if (tone == null) { + tone = RingtoneManager.getDefaultUri(RingtoneManager.TYPE_RINGTONE) + } + try { + val created = MediaPlayer() + created.setAudioAttributes(attributes) + created.setDataSource(this, tone!!) + created.isLooping = true + created.prepare() + created.start() + player = created + } catch (error: Exception) { + FaceclawAlarms.log(this, "alarm sound failed: " + error.message) + player = null + } + startVibration() + } + + private fun stopSound() { + val current = player + if (current != null) { + try { + current.stop() + } catch (ignored: RuntimeException) { + // Already stopped. + } + current.release() + player = null + } + val device = vibrator + if (device != null) { + device.cancel() + vibrator = null + } + abandonAudioFocus() + restoreAlarmVolume() + } + + /** + * A muted alarm stream is the one thing that would keep an alarm clock + * silent; raise it for the duration of the ring and put it back after. + */ + private fun raiseMutedAlarmVolume() { + val manager = audioManager ?: return + try { + val current = manager.getStreamVolume(AudioManager.STREAM_ALARM) + val max = manager.getStreamMaxVolume(AudioManager.STREAM_ALARM) + if (current == 0 && max > 0) { + restoreAlarmVolume = current + manager.setStreamVolume(AudioManager.STREAM_ALARM, Math.max(1, Math.round(max * MUTED_VOLUME_FRACTION)), 0) + } + } catch (error: RuntimeException) { + FaceclawAlarms.log(this, "alarm volume change failed: " + error.message) + } + } + + private fun restoreAlarmVolume() { + val manager = audioManager + if (manager == null || restoreAlarmVolume < 0) { + return + } + try { + manager.setStreamVolume(AudioManager.STREAM_ALARM, restoreAlarmVolume, 0) + } catch (ignored: RuntimeException) { + // Best effort. + } + restoreAlarmVolume = -1 + } + + private fun requestAudioFocus(attributes: AudioAttributes) { + val manager = audioManager ?: return + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) { + val request = AudioFocusRequest.Builder(AudioManager.AUDIOFOCUS_GAIN_TRANSIENT) + .setAudioAttributes(attributes) + .build() + focusRequest = request + manager.requestAudioFocus(request) + } else { + @Suppress("DEPRECATION") + manager.requestAudioFocus(null, AudioManager.STREAM_ALARM, AudioManager.AUDIOFOCUS_GAIN_TRANSIENT) + } + } + + private fun abandonAudioFocus() { + val manager = audioManager ?: return + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) { + val request = focusRequest + if (request != null) { + manager.abandonAudioFocusRequest(request) + focusRequest = null + } + } else { + @Suppress("DEPRECATION") + manager.abandonAudioFocus(null) + } + } + + private fun startVibration() { + val device = getSystemService(Context.VIBRATOR_SERVICE) as Vibrator? + if (device == null || !device.hasVibrator()) { + return + } + vibrator = device + val pattern = longArrayOf(0, 600, 400, 600, 1200) + val attributes = AudioAttributes.Builder() + .setUsage(AudioAttributes.USAGE_ALARM) + .setContentType(AudioAttributes.CONTENT_TYPE_SONIFICATION) + .build() + try { + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) { + device.vibrate(VibrationEffect.createWaveform(pattern, 0), attributes) + } else { + @Suppress("DEPRECATION") + device.vibrate(pattern, 0, attributes) + } + } catch (error: RuntimeException) { + FaceclawAlarms.log(this, "vibration failed: " + error.message) + } + } + + // ------------------------------------------------------------------ + // Wake lock + + private fun acquireWakeLock() { + val held = wakeLock + if (held != null && held.isHeld) { + return + } + val power = getSystemService(Context.POWER_SERVICE) as PowerManager? ?: return + val lock = power.newWakeLock(PowerManager.PARTIAL_WAKE_LOCK, "faceclaw:alarm") + wakeLock = lock + // Bounded: sound auto-silences at AUTO_SILENCE_MS, and a stuck lock + // past that would only drain the battery. + lock.acquire(AUTO_SILENCE_MS + ACK_WAIT_MS + DELIVERY_WAIT_MS + 10_000L) + } + + private fun releaseWakeLock() { + val lock = wakeLock + if (lock != null && lock.isHeld) { + lock.release() + } + wakeLock = null + } +} diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawAlarms.java b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawAlarms.java deleted file mode 100644 index 3dd2b0f1..00000000 --- a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawAlarms.java +++ /dev/null @@ -1,620 +0,0 @@ -package com.faceclaw.app; - -import android.app.ActivityManager; -import android.app.AlarmManager; -import android.app.NotificationChannel; -import android.app.NotificationManager; -import android.app.PendingIntent; -import android.content.Context; -import android.content.Intent; -import android.content.SharedPreferences; -import android.media.AudioManager; -import android.net.Uri; -import android.os.Build; -import android.os.Handler; -import android.os.Looper; -import android.os.PowerManager; -import android.provider.Settings; - -import androidx.core.content.ContextCompat; - -import com.tns.NativeScriptActivity; - -import org.json.JSONArray; -import org.json.JSONException; -import org.json.JSONObject; - -import java.util.Locale; - -/** - * The phone side of the Timers app's alarms and countdown timers, built to - * ring without the JavaScript side: scheduling through AlarmManager's alarm - * clock API, a persisted schedule (in device-protected storage) that the - * reschedule receiver replays after reboots and time changes, the hand-off - * to the ringing foreground service, and a journal of what the wearer did - * on the phone (dismiss / snooze) for the JS engine to replay when it is - * next awake. Also the reliability self-check the UI surfaces. - * - * Ids are the JS engine's item ids (a millisecond epoch times 100 plus a - * serial), shared between timers and alarms; `kind` tells the two apart for - * wording only. - */ -public final class FaceclawAlarms { - public static final String ACTION_EXPIRE = "com.faceclaw.app.action.ALARM_EXPIRE"; - static final String EXTRA_ID = "id"; - static final String EXTRA_TITLE = "title"; - static final String EXTRA_TEXT = "text"; - static final String EXTRA_KIND = "kind"; - static final String EXTRA_SNOOZE_MINUTES = "snoozeMinutes"; - - public static final String KIND_TIMER = "timer"; - public static final String KIND_ALARM = "alarm"; - - private static final String PREFS_NAME = "faceclaw-alarms"; - private static final String KEY_SCHEDULED = "scheduled"; - private static final String KEY_JOURNAL = "journal"; - private static final String KEY_LOG = "log"; - private static final int LOG_LINES = 60; - - /** - * A schedule entry found already past due (after a reboot, or when the - * process was asleep) rings if it is at most this late; older ones are - * dropped as missed. Matches the JS engine's grace window. - */ - static final long LATE_GRACE_MS = 5 * 60_000L; - /** A glasses status report older than this is not trusted (the JS side is probably gone). */ - static final long GLASSES_STATUS_MAX_AGE_MS = 3 * 60_000L; - - private static volatile boolean glassesConnected; - private static volatile boolean glassesWorn; - private static volatile boolean glassesCharging; - private static volatile long glassesStatusAtMs; - - private static volatile FaceclawAlarmListener listener; - private static final Handler mainHandler = new Handler(Looper.getMainLooper()); - - private FaceclawAlarms() {} - - // ------------------------------------------------------------------ - // Storage - - /** - * Device-protected storage, so the schedule is readable before the first - * unlock after a reboot (the reschedule receiver may run then on some - * devices) and survives credential-encrypted storage being unavailable. - */ - static SharedPreferences prefs(Context context) { - Context base = context.getApplicationContext(); - if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.N) { - base = base.createDeviceProtectedStorageContext(); - } - return base.getSharedPreferences(PREFS_NAME, Context.MODE_PRIVATE); - } - - private static synchronized JSONArray readArray(Context context, String key) { - String raw = prefs(context).getString(key, "[]"); - try { - return new JSONArray(raw); - } catch (JSONException error) { - return new JSONArray(); - } - } - - private static synchronized void writeArray(Context context, String key, JSONArray array) { - prefs(context).edit().putString(key, array.toString()).apply(); - } - - static JSONObject findScheduled(Context context, long id) { - JSONArray scheduled = readArray(context, KEY_SCHEDULED); - for (int index = 0; index < scheduled.length(); index++) { - JSONObject entry = scheduled.optJSONObject(index); - if (entry != null && entry.optLong("id") == id) { - return entry; - } - } - return null; - } - - private static synchronized void putScheduled(Context context, JSONObject entry) { - JSONArray scheduled = readArray(context, KEY_SCHEDULED); - JSONArray next = new JSONArray(); - long id = entry.optLong("id"); - for (int index = 0; index < scheduled.length(); index++) { - JSONObject existing = scheduled.optJSONObject(index); - if (existing != null && existing.optLong("id") != id) { - next.put(existing); - } - } - next.put(entry); - writeArray(context, KEY_SCHEDULED, next); - } - - private static synchronized void removeScheduled(Context context, long id) { - JSONArray scheduled = readArray(context, KEY_SCHEDULED); - JSONArray next = new JSONArray(); - for (int index = 0; index < scheduled.length(); index++) { - JSONObject existing = scheduled.optJSONObject(index); - if (existing != null && existing.optLong("id") != id) { - next.put(existing); - } - } - writeArray(context, KEY_SCHEDULED, next); - } - - // ------------------------------------------------------------------ - // Scheduling - - /** - * Schedule (or move) the phone alarm for an item. Idempotent: the same id - * always maps to the same PendingIntent, so a reschedule replaces rather - * than duplicates. setAlarmClock is exact, fires through Doze and battery - * savers, and shows the system's alarm indicator; the fallbacks only run - * where it is unavailable. - */ - public static void schedule( - Context context, - long id, - long triggerAtMs, - String title, - String text, - String kind, - int snoozeMinutes - ) { - Context appContext = context.getApplicationContext(); - JSONObject entry = new JSONObject(); - try { - entry.put("id", id); - entry.put("at", triggerAtMs); - entry.put("title", title == null ? "" : title); - entry.put("text", text == null ? "" : text); - entry.put("kind", kind == null ? KIND_ALARM : kind); - entry.put("snoozeMinutes", snoozeMinutes); - } catch (JSONException ignored) { - return; - } - putScheduled(appContext, entry); - armAlarm(appContext, entry); - } - - private static void armAlarm(Context appContext, JSONObject entry) { - AlarmManager manager = (AlarmManager) appContext.getSystemService(Context.ALARM_SERVICE); - if (manager == null) { - return; - } - long id = entry.optLong("id"); - long triggerAt = Math.max(System.currentTimeMillis(), entry.optLong("at")); - PendingIntent fire = expiryPendingIntent(appContext, entry); - try { - if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.LOLLIPOP) { - AlarmManager.AlarmClockInfo info = new AlarmManager.AlarmClockInfo(triggerAt, showIntent(appContext, id)); - manager.setAlarmClock(info, fire); - } else { - manager.setExact(AlarmManager.RTC_WAKEUP, triggerAt, fire); - } - } catch (SecurityException error) { - // Exact-alarm access revoked (Android 12/13 without USE_EXACT_ALARM - // honoured): the inexact alarm is still a wake-up, just a late one. - log(appContext, "exact alarm refused for " + id + ": " + error.getMessage()); - manager.setAndAllowWhileIdle(AlarmManager.RTC_WAKEUP, triggerAt, fire); - } - } - - /** Cancel the phone alarm for an item, stop it if ringing, and forget it. */ - public static void cancel(Context context, long id) { - Context appContext = context.getApplicationContext(); - AlarmManager manager = (AlarmManager) appContext.getSystemService(Context.ALARM_SERVICE); - if (manager != null) { - JSONObject entry = new JSONObject(); - try { - entry.put("id", id); - } catch (JSONException ignored) { - // unreachable for a numeric put - } - manager.cancel(expiryPendingIntent(appContext, entry)); - } - removeScheduled(appContext, id); - FaceclawAlarmService.stopItem(appContext, id); - } - - /** - * Replay the persisted schedule into AlarmManager: after a reboot (alarms - * are wiped), a package update, or a wall-clock / time-zone change. Entries - * already due ring now if inside the grace window, else are dropped. - */ - public static void rescheduleAll(Context context, String reason) { - Context appContext = context.getApplicationContext(); - JSONArray scheduled = readArray(appContext, KEY_SCHEDULED); - long now = System.currentTimeMillis(); - int armed = 0; - for (int index = 0; index < scheduled.length(); index++) { - JSONObject entry = scheduled.optJSONObject(index); - if (entry == null) { - continue; - } - long at = entry.optLong("at"); - if (at <= now) { - if (now - at <= LATE_GRACE_MS) { - ringEntry(appContext, entry); - } else { - log(appContext, "missed " + entry.optString("kind") + " " + entry.optLong("id") + " (" + reason + ")"); - removeScheduled(appContext, entry.optLong("id")); - } - continue; - } - armAlarm(appContext, entry); - armed++; - } - log(appContext, "rescheduled " + armed + " (" + reason + ")"); - } - - /** - * The schedule and AlarmManager can disagree (a force-stop clears alarms - * but not the file, an OEM killer drops them). True when the soonest - * stored entry is not what the system reports as the next alarm clock. - */ - public static boolean scheduleLooksStale(Context context) { - if (Build.VERSION.SDK_INT < Build.VERSION_CODES.LOLLIPOP) { - return false; - } - Context appContext = context.getApplicationContext(); - JSONArray scheduled = readArray(appContext, KEY_SCHEDULED); - long now = System.currentTimeMillis(); - long soonest = Long.MAX_VALUE; - for (int index = 0; index < scheduled.length(); index++) { - JSONObject entry = scheduled.optJSONObject(index); - if (entry != null && entry.optLong("at") > now) { - soonest = Math.min(soonest, entry.optLong("at")); - } - } - if (soonest == Long.MAX_VALUE) { - return false; - } - AlarmManager manager = (AlarmManager) appContext.getSystemService(Context.ALARM_SERVICE); - if (manager == null) { - return false; - } - AlarmManager.AlarmClockInfo next = manager.getNextAlarmClock(); - // Another app's alarm may legitimately be sooner; only a later (or - // absent) system alarm proves ours is missing. - return next == null || next.getTriggerTime() > soonest + 1000; - } - - // ------------------------------------------------------------------ - // Ringing - - /** An item came due: hand it to the ringing service (idempotent per id). */ - static void ringEntry(Context appContext, JSONObject entry) { - FaceclawAlarmService.ring( - appContext, - entry.optLong("id"), - entry.optString("title"), - entry.optString("text"), - entry.optString("kind", KIND_ALARM), - entry.optInt("snoozeMinutes", 10) - ); - } - - /** The JS engine's own expiry path; the service deduplicates against the alarm's. */ - public static void ring(Context context, long id, String title, String text, String kind, int snoozeMinutes) { - Context appContext = context.getApplicationContext(); - AlarmManager manager = (AlarmManager) appContext.getSystemService(Context.ALARM_SERVICE); - if (manager != null) { - JSONObject probe = new JSONObject(); - try { - probe.put("id", id); - } catch (JSONException ignored) { - // unreachable for a numeric put - } - manager.cancel(expiryPendingIntent(appContext, probe)); - } - FaceclawAlarmService.ring(appContext, id, title, text, kind, snoozeMinutes); - } - - /** The glasses are showing the ringing item; the acknowledgement clock starts now. */ - public static void deliveredToGlasses(Context context, long id) { - FaceclawAlarmService.delivered(context.getApplicationContext(), id); - } - - /** The wearer dismissed or snoozed it on the glasses: the phone side goes quiet. */ - public static void acknowledge(Context context, long id) { - Context appContext = context.getApplicationContext(); - removeScheduled(appContext, id); - FaceclawAlarmService.stopItem(appContext, id); - } - - // ------------------------------------------------------------------ - // Glasses status (pushed by the JS side; in-memory, so a dead process reads as "no glasses") - - public static void setGlassesStatus(boolean connected, boolean worn, boolean charging) { - glassesConnected = connected; - glassesWorn = worn; - glassesCharging = charging; - glassesStatusAtMs = System.currentTimeMillis(); - } - - /** True only when a fresh report says the glasses are connected, on a head, and not charging. */ - static boolean glassesCanCarryAlarm() { - long age = System.currentTimeMillis() - glassesStatusAtMs; - if (glassesStatusAtMs == 0 || age > GLASSES_STATUS_MAX_AGE_MS) { - return false; - } - return glassesConnected && glassesWorn && !glassesCharging; - } - - static String glassesStatusDescription() { - if (glassesStatusAtMs == 0) { - return "no glasses status"; - } - long age = System.currentTimeMillis() - glassesStatusAtMs; - if (age > GLASSES_STATUS_MAX_AGE_MS) { - return "glasses status stale"; - } - if (!glassesConnected) { - return "glasses not connected"; - } - if (glassesCharging) { - return "glasses charging"; - } - if (!glassesWorn) { - return "glasses not worn"; - } - return "glasses worn"; - } - - // ------------------------------------------------------------------ - // Phone actions -> JS - - public static void setListener(FaceclawAlarmListener newListener) { - listener = newListener; - } - - /** - * Record a dismiss / snooze done on the phone. Delivered live to the JS - * listener when one is registered, and journaled regardless so a JS side - * that was asleep or gone can catch up on its next boot. - */ - static void recordPhoneAction(Context context, long id, String action, int minutes) { - Context appContext = context.getApplicationContext(); - JSONObject event = new JSONObject(); - try { - event.put("id", id); - event.put("action", action); - event.put("minutes", minutes); - event.put("at", System.currentTimeMillis()); - } catch (JSONException ignored) { - return; - } - synchronized (FaceclawAlarms.class) { - JSONArray journal = readArray(appContext, KEY_JOURNAL); - journal.put(event); - writeArray(appContext, KEY_JOURNAL, journal); - } - log(appContext, "phone " + action + " " + id + (minutes > 0 ? " (" + minutes + " min)" : "")); - final FaceclawAlarmListener current = listener; - if (current != null) { - mainHandler.post(() -> { - try { - current.onPhoneAction(id, action, minutes); - } catch (RuntimeException error) { - log(appContext, "listener failed: " + error.getMessage()); - } - }); - } - } - - /** Hand the journal to the JS side (as a JSON array) and clear it. */ - public static synchronized String drainJournal(Context context) { - Context appContext = context.getApplicationContext(); - JSONArray journal = readArray(appContext, KEY_JOURNAL); - writeArray(appContext, KEY_JOURNAL, new JSONArray()); - return journal.toString(); - } - - // ------------------------------------------------------------------ - // Reliability self-check - - /** - * Conditions under which the phone may fail to ring, as a JSON array of - * {code, message, fixable}, most serious first. Empty when everything is - * in order. The UI shows these before the wearer relies on an alarm. - */ - public static String checkReliability(Context context) { - Context appContext = context.getApplicationContext(); - JSONArray issues = new JSONArray(); - NotificationManager notifications = (NotificationManager) appContext.getSystemService(Context.NOTIFICATION_SERVICE); - AlarmManager alarms = (AlarmManager) appContext.getSystemService(Context.ALARM_SERVICE); - - if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.S && alarms != null && !alarms.canScheduleExactAlarms()) { - addIssue(issues, "exact-alarm", "Exact alarms are not allowed for Faceclaw, so alarms may ring late.", true); - } - if (notifications != null && !notifications.areNotificationsEnabled()) { - addIssue(issues, "notifications", "Notifications are turned off for Faceclaw, so the phone cannot show or sound an alarm.", true); - } else if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O && notifications != null) { - NotificationChannel channel = notifications.getNotificationChannel(FaceclawAlarmService.CHANNEL_ID); - if (channel != null && channel.getImportance() == NotificationManager.IMPORTANCE_NONE) { - addIssue(issues, "alarm-channel", "The Alarms notification category is blocked, so the phone cannot show an alarm.", true); - } - } - if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.UPSIDE_DOWN_CAKE && notifications != null && !notifications.canUseFullScreenIntent()) { - addIssue(issues, "full-screen", "Full-screen alarms are not allowed, so a ringing alarm will not take over the lock screen.", true); - } - if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.M && notifications != null) { - int filter = notifications.getCurrentInterruptionFilter(); - if (filter == NotificationManager.INTERRUPTION_FILTER_NONE) { - addIssue(issues, "dnd-total", "Do Not Disturb is set to total silence, which silences alarms too.", true); - } else if (filter == NotificationManager.INTERRUPTION_FILTER_PRIORITY) { - try { - NotificationManager.Policy policy = notifications.getNotificationPolicy(); - if (policy != null && (policy.priorityCategories & NotificationManager.Policy.PRIORITY_CATEGORY_ALARMS) == 0) { - addIssue(issues, "dnd-alarms", "Do Not Disturb does not allow alarms, so the phone will stay silent.", true); - } - } catch (SecurityException ignored) { - // No notification-policy access: cannot inspect the exceptions. - } - } - } - if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.P) { - ActivityManager activityManager = (ActivityManager) appContext.getSystemService(Context.ACTIVITY_SERVICE); - if (activityManager != null && activityManager.isBackgroundRestricted()) { - addIssue(issues, "background-restricted", "Background usage is restricted for Faceclaw, which blocks alarms while the app is not open.", true); - } - } - PowerManager power = (PowerManager) appContext.getSystemService(Context.POWER_SERVICE); - if (power != null && !power.isIgnoringBatteryOptimizations(appContext.getPackageName())) { - addIssue(issues, "battery-optimized", "Battery optimization is on for Faceclaw; some phones delay alarms because of it.", true); - } - AudioManager audio = (AudioManager) appContext.getSystemService(Context.AUDIO_SERVICE); - if (audio != null && audio.getStreamVolume(AudioManager.STREAM_ALARM) == 0) { - addIssue(issues, "alarm-volume", "The phone's alarm volume is muted; Faceclaw raises it while ringing, but check it.", true); - } - String maker = Build.MANUFACTURER == null ? "" : Build.MANUFACTURER.toLowerCase(Locale.ROOT); - if (maker.contains("xiaomi") || maker.contains("huawei") || maker.contains("oppo") || maker.contains("vivo") - || maker.contains("oneplus") || maker.contains("realme") || maker.contains("meizu") || maker.contains("asus")) { - addIssue(issues, "oem-killer", "This phone maker is known to stop background apps; allow Faceclaw to auto-start and run unrestricted (see dontkillmyapp.com).", true); - } - if (scheduleLooksStale(appContext)) { - addIssue(issues, "schedule-stale", "The system lost Faceclaw's next alarm (the app may have been force-stopped); it was re-armed.", false); - rescheduleAll(appContext, "self-check"); - } - return issues.toString(); - } - - private static void addIssue(JSONArray issues, String code, String message, boolean fixable) { - JSONObject issue = new JSONObject(); - try { - issue.put("code", code); - issue.put("message", message); - issue.put("fixable", fixable); - } catch (JSONException ignored) { - return; - } - issues.put(issue); - } - - /** Open the system screen where the issue can be fixed. */ - public static void openReliabilityFix(Context context, String code) { - Context appContext = context.getApplicationContext(); - String pkg = appContext.getPackageName(); - Intent intent; - switch (code == null ? "" : code) { - case "exact-alarm": - intent = Build.VERSION.SDK_INT >= Build.VERSION_CODES.S - ? new Intent(Settings.ACTION_REQUEST_SCHEDULE_EXACT_ALARM, Uri.parse("package:" + pkg)) - : appDetails(pkg); - break; - case "notifications": - case "alarm-channel": - if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) { - intent = new Intent(Settings.ACTION_APP_NOTIFICATION_SETTINGS) - .putExtra(Settings.EXTRA_APP_PACKAGE, pkg); - if ("alarm-channel".equals(code)) { - intent = new Intent(Settings.ACTION_CHANNEL_NOTIFICATION_SETTINGS) - .putExtra(Settings.EXTRA_APP_PACKAGE, pkg) - .putExtra(Settings.EXTRA_CHANNEL_ID, FaceclawAlarmService.CHANNEL_ID); - } - } else { - intent = appDetails(pkg); - } - break; - case "full-screen": - intent = Build.VERSION.SDK_INT >= Build.VERSION_CODES.UPSIDE_DOWN_CAKE - ? new Intent(Settings.ACTION_MANAGE_APP_USE_FULL_SCREEN_INTENT, Uri.parse("package:" + pkg)) - : appDetails(pkg); - break; - case "dnd-total": - case "dnd-alarms": - intent = new Intent(Settings.ACTION_SOUND_SETTINGS); - break; - case "battery-optimized": - intent = new Intent(Settings.ACTION_REQUEST_IGNORE_BATTERY_OPTIMIZATIONS, Uri.parse("package:" + pkg)); - break; - case "alarm-volume": - intent = new Intent(Settings.ACTION_SOUND_SETTINGS); - break; - default: - intent = appDetails(pkg); - break; - } - intent.addFlags(Intent.FLAG_ACTIVITY_NEW_TASK); - try { - appContext.startActivity(intent); - } catch (RuntimeException error) { - Intent fallback = appDetails(pkg).addFlags(Intent.FLAG_ACTIVITY_NEW_TASK); - try { - appContext.startActivity(fallback); - } catch (RuntimeException ignored) { - // No settings activity at all; nothing more to do. - } - } - } - - private static Intent appDetails(String pkg) { - return new Intent(Settings.ACTION_APPLICATION_DETAILS_SETTINGS, Uri.parse("package:" + pkg)); - } - - // ------------------------------------------------------------------ - // Diagnostics log - - static synchronized void log(Context context, String line) { - Context appContext = context.getApplicationContext(); - JSONArray log = readArray(appContext, KEY_LOG); - JSONArray next = new JSONArray(); - int start = Math.max(0, log.length() - (LOG_LINES - 1)); - for (int index = start; index < log.length(); index++) { - next.put(log.optString(index)); - } - next.put(System.currentTimeMillis() + " " + line); - writeArray(appContext, KEY_LOG, next); - } - - /** The ring / miss / action log, newest last, one line per entry. */ - public static String readLog(Context context) { - JSONArray log = readArray(context.getApplicationContext(), KEY_LOG); - StringBuilder out = new StringBuilder(); - for (int index = 0; index < log.length(); index++) { - if (index > 0) { - out.append('\n'); - } - out.append(log.optString(index)); - } - return out.toString(); - } - - // ------------------------------------------------------------------ - // Intents - - private static PendingIntent expiryPendingIntent(Context appContext, JSONObject entry) { - Intent intent = new Intent(appContext, FaceclawAlarmReceiver.class); - intent.setAction(ACTION_EXPIRE); - long id = entry.optLong("id"); - intent.putExtra(EXTRA_ID, id); - intent.putExtra(EXTRA_TITLE, entry.optString("title")); - intent.putExtra(EXTRA_TEXT, entry.optString("text")); - intent.putExtra(EXTRA_KIND, entry.optString("kind", KIND_ALARM)); - intent.putExtra(EXTRA_SNOOZE_MINUTES, entry.optInt("snoozeMinutes", 10)); - int flags = PendingIntent.FLAG_UPDATE_CURRENT; - if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.M) { - flags |= PendingIntent.FLAG_IMMUTABLE; - } - return PendingIntent.getBroadcast(appContext, requestCode(id), intent, flags); - } - - /** What the system's alarm indicator opens: the app. */ - private static PendingIntent showIntent(Context appContext, long id) { - Intent intent = new Intent(appContext, NativeScriptActivity.class); - intent.addFlags(Intent.FLAG_ACTIVITY_SINGLE_TOP | Intent.FLAG_ACTIVITY_CLEAR_TOP); - int flags = PendingIntent.FLAG_UPDATE_CURRENT; - if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.M) { - flags |= PendingIntent.FLAG_IMMUTABLE; - } - return PendingIntent.getActivity(appContext, requestCode(id), intent, flags); - } - - static int requestCode(long id) { - return (int) (id ^ (id >>> 32)) & 0x7fffffff; - } - - static void startServiceCompat(Context appContext, Intent intent) { - ContextCompat.startForegroundService(appContext, intent); - } -} diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawAlarms.kt b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawAlarms.kt new file mode 100644 index 00000000..43ffcfc1 --- /dev/null +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawAlarms.kt @@ -0,0 +1,647 @@ +package com.faceclaw.app + +import android.app.ActivityManager +import android.app.AlarmManager +import android.app.NotificationManager +import android.app.PendingIntent +import android.content.Context +import android.content.Intent +import android.content.SharedPreferences +import android.media.AudioManager +import android.net.Uri +import android.os.Build +import android.os.Handler +import android.os.Looper +import android.os.PowerManager +import android.provider.Settings + +import androidx.core.content.ContextCompat + +import com.tns.NativeScriptActivity + +import org.json.JSONArray +import org.json.JSONException +import org.json.JSONObject + +import java.util.Locale + +/** + * The phone side of the Timers app's alarms and countdown timers, built to + * ring without the JavaScript side: scheduling through AlarmManager's alarm + * clock API, a persisted schedule (in device-protected storage) that the + * reschedule receiver replays after reboots and time changes, the hand-off + * to the ringing foreground service, and a journal of what the wearer did + * on the phone (dismiss / snooze) for the JS engine to replay when it is + * next awake. Also the reliability self-check the UI surfaces. + * + * Ids are the JS engine's item ids (a millisecond epoch times 100 plus a + * serial), shared between timers and alarms; `kind` tells the two apart for + * wording only. + */ +class FaceclawAlarms private constructor() { + companion object { + const val ACTION_EXPIRE = "com.faceclaw.app.action.ALARM_EXPIRE" + internal const val EXTRA_ID = "id" + internal const val EXTRA_TITLE = "title" + internal const val EXTRA_TEXT = "text" + internal const val EXTRA_KIND = "kind" + internal const val EXTRA_SNOOZE_MINUTES = "snoozeMinutes" + + const val KIND_TIMER = "timer" + const val KIND_ALARM = "alarm" + + private const val PREFS_NAME = "faceclaw-alarms" + private const val KEY_SCHEDULED = "scheduled" + private const val KEY_JOURNAL = "journal" + private const val KEY_LOG = "log" + private const val LOG_LINES = 60 + + /** + * A schedule entry found already past due (after a reboot, or when the + * process was asleep) rings if it is at most this late; older ones are + * dropped as missed. Matches the JS engine's grace window. + */ + internal const val LATE_GRACE_MS = 5 * 60_000L + /** A glasses status report older than this is not trusted (the JS side is probably gone). */ + internal const val GLASSES_STATUS_MAX_AGE_MS = 3 * 60_000L + + @Volatile + private var glassesConnected = false + @Volatile + private var glassesWorn = false + @Volatile + private var glassesCharging = false + @Volatile + private var glassesStatusAtMs = 0L + + @Volatile + private var listener: FaceclawAlarmListener? = null + private val mainHandler = Handler(Looper.getMainLooper()) + + // ------------------------------------------------------------------ + // Storage + + /** + * Device-protected storage, so the schedule is readable before the first + * unlock after a reboot (the reschedule receiver may run then on some + * devices) and survives credential-encrypted storage being unavailable. + */ + @JvmStatic + internal fun prefs(context: Context): SharedPreferences { + var base = context.applicationContext + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.N) { + base = base.createDeviceProtectedStorageContext() + } + return base.getSharedPreferences(PREFS_NAME, Context.MODE_PRIVATE) + } + + // The Java statics were `synchronized`, i.e. locked on the class object; + // the explicit blocks below keep that single monitor. + private fun readArray(context: Context, key: String): JSONArray { + synchronized(FaceclawAlarms::class.java) { + val raw = prefs(context).getString(key, "[]") + return try { + JSONArray(raw) + } catch (error: JSONException) { + JSONArray() + } + } + } + + private fun writeArray(context: Context, key: String, array: JSONArray) { + synchronized(FaceclawAlarms::class.java) { + prefs(context).edit().putString(key, array.toString()).apply() + } + } + + @JvmStatic + internal fun findScheduled(context: Context, id: Long): JSONObject? { + val scheduled = readArray(context, KEY_SCHEDULED) + for (index in 0 until scheduled.length()) { + val entry = scheduled.optJSONObject(index) + if (entry != null && entry.optLong("id") == id) { + return entry + } + } + return null + } + + private fun putScheduled(context: Context, entry: JSONObject) { + synchronized(FaceclawAlarms::class.java) { + val scheduled = readArray(context, KEY_SCHEDULED) + val next = JSONArray() + val id = entry.optLong("id") + for (index in 0 until scheduled.length()) { + val existing = scheduled.optJSONObject(index) + if (existing != null && existing.optLong("id") != id) { + next.put(existing) + } + } + next.put(entry) + writeArray(context, KEY_SCHEDULED, next) + } + } + + private fun removeScheduled(context: Context, id: Long) { + synchronized(FaceclawAlarms::class.java) { + val scheduled = readArray(context, KEY_SCHEDULED) + val next = JSONArray() + for (index in 0 until scheduled.length()) { + val existing = scheduled.optJSONObject(index) + if (existing != null && existing.optLong("id") != id) { + next.put(existing) + } + } + writeArray(context, KEY_SCHEDULED, next) + } + } + + // ------------------------------------------------------------------ + // Scheduling + + /** + * Schedule (or move) the phone alarm for an item. Idempotent: the same id + * always maps to the same PendingIntent, so a reschedule replaces rather + * than duplicates. setAlarmClock is exact, fires through Doze and battery + * savers, and shows the system's alarm indicator; the fallbacks only run + * where it is unavailable. + */ + @JvmStatic + fun schedule( + context: Context, + id: Long, + triggerAtMs: Long, + title: String?, + text: String?, + kind: String?, + snoozeMinutes: Int + ) { + val appContext = context.applicationContext + val entry = JSONObject() + try { + entry.put("id", id) + entry.put("at", triggerAtMs) + entry.put("title", title ?: "") + entry.put("text", text ?: "") + entry.put("kind", kind ?: KIND_ALARM) + entry.put("snoozeMinutes", snoozeMinutes) + } catch (ignored: JSONException) { + return + } + putScheduled(appContext, entry) + armAlarm(appContext, entry) + } + + private fun armAlarm(appContext: Context, entry: JSONObject) { + val manager = appContext.getSystemService(Context.ALARM_SERVICE) as AlarmManager? + ?: return + val id = entry.optLong("id") + val triggerAt = Math.max(System.currentTimeMillis(), entry.optLong("at")) + val fire = expiryPendingIntent(appContext, entry) + try { + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.LOLLIPOP) { + val info = AlarmManager.AlarmClockInfo(triggerAt, showIntent(appContext, id)) + manager.setAlarmClock(info, fire) + } else { + manager.setExact(AlarmManager.RTC_WAKEUP, triggerAt, fire) + } + } catch (error: SecurityException) { + // Exact-alarm access revoked (Android 12/13 without USE_EXACT_ALARM + // honoured): the inexact alarm is still a wake-up, just a late one. + log(appContext, "exact alarm refused for " + id + ": " + error.message) + manager.setAndAllowWhileIdle(AlarmManager.RTC_WAKEUP, triggerAt, fire) + } + } + + /** Cancel the phone alarm for an item, stop it if ringing, and forget it. */ + @JvmStatic + fun cancel(context: Context, id: Long) { + val appContext = context.applicationContext + val manager = appContext.getSystemService(Context.ALARM_SERVICE) as AlarmManager? + if (manager != null) { + val entry = JSONObject() + try { + entry.put("id", id) + } catch (ignored: JSONException) { + // unreachable for a numeric put + } + manager.cancel(expiryPendingIntent(appContext, entry)) + } + removeScheduled(appContext, id) + FaceclawAlarmService.stopItem(appContext, id) + } + + /** + * Replay the persisted schedule into AlarmManager: after a reboot (alarms + * are wiped), a package update, or a wall-clock / time-zone change. Entries + * already due ring now if inside the grace window, else are dropped. + */ + @JvmStatic + fun rescheduleAll(context: Context, reason: String?) { + val appContext = context.applicationContext + val scheduled = readArray(appContext, KEY_SCHEDULED) + val now = System.currentTimeMillis() + var armed = 0 + for (index in 0 until scheduled.length()) { + val entry = scheduled.optJSONObject(index) ?: continue + val at = entry.optLong("at") + if (at <= now) { + if (now - at <= LATE_GRACE_MS) { + ringEntry(appContext, entry) + } else { + log(appContext, "missed " + entry.optString("kind") + " " + entry.optLong("id") + " (" + reason + ")") + removeScheduled(appContext, entry.optLong("id")) + } + continue + } + armAlarm(appContext, entry) + armed++ + } + log(appContext, "rescheduled $armed ($reason)") + } + + /** + * The schedule and AlarmManager can disagree (a force-stop clears alarms + * but not the file, an OEM killer drops them). True when the soonest + * stored entry is not what the system reports as the next alarm clock. + */ + @JvmStatic + fun scheduleLooksStale(context: Context): Boolean { + if (Build.VERSION.SDK_INT < Build.VERSION_CODES.LOLLIPOP) { + return false + } + val appContext = context.applicationContext + val scheduled = readArray(appContext, KEY_SCHEDULED) + val now = System.currentTimeMillis() + var soonest = Long.MAX_VALUE + for (index in 0 until scheduled.length()) { + val entry = scheduled.optJSONObject(index) + if (entry != null && entry.optLong("at") > now) { + soonest = Math.min(soonest, entry.optLong("at")) + } + } + if (soonest == Long.MAX_VALUE) { + return false + } + val manager = appContext.getSystemService(Context.ALARM_SERVICE) as AlarmManager? + ?: return false + val next = manager.nextAlarmClock + // Another app's alarm may legitimately be sooner; only a later (or + // absent) system alarm proves ours is missing. + return next == null || next.triggerTime > soonest + 1000 + } + + // ------------------------------------------------------------------ + // Ringing + + /** An item came due: hand it to the ringing service (idempotent per id). */ + @JvmStatic + internal fun ringEntry(appContext: Context, entry: JSONObject) { + FaceclawAlarmService.ring( + appContext, + entry.optLong("id"), + entry.optString("title"), + entry.optString("text"), + entry.optString("kind", KIND_ALARM), + entry.optInt("snoozeMinutes", 10) + ) + } + + /** The JS engine's own expiry path; the service deduplicates against the alarm's. */ + @JvmStatic + fun ring(context: Context, id: Long, title: String?, text: String?, kind: String?, snoozeMinutes: Int) { + val appContext = context.applicationContext + val manager = appContext.getSystemService(Context.ALARM_SERVICE) as AlarmManager? + if (manager != null) { + val probe = JSONObject() + try { + probe.put("id", id) + } catch (ignored: JSONException) { + // unreachable for a numeric put + } + manager.cancel(expiryPendingIntent(appContext, probe)) + } + FaceclawAlarmService.ring(appContext, id, title, text, kind, snoozeMinutes) + } + + /** The glasses are showing the ringing item; the acknowledgement clock starts now. */ + @JvmStatic + fun deliveredToGlasses(context: Context, id: Long) { + FaceclawAlarmService.delivered(context.applicationContext, id) + } + + /** The wearer dismissed or snoozed it on the glasses: the phone side goes quiet. */ + @JvmStatic + fun acknowledge(context: Context, id: Long) { + val appContext = context.applicationContext + removeScheduled(appContext, id) + FaceclawAlarmService.stopItem(appContext, id) + } + + // ------------------------------------------------------------------ + // Glasses status (pushed by the JS side; in-memory, so a dead process reads as "no glasses") + + @JvmStatic + fun setGlassesStatus(connected: Boolean, worn: Boolean, charging: Boolean) { + glassesConnected = connected + glassesWorn = worn + glassesCharging = charging + glassesStatusAtMs = System.currentTimeMillis() + } + + /** True only when a fresh report says the glasses are connected, on a head, and not charging. */ + @JvmStatic + internal fun glassesCanCarryAlarm(): Boolean { + val age = System.currentTimeMillis() - glassesStatusAtMs + if (glassesStatusAtMs == 0L || age > GLASSES_STATUS_MAX_AGE_MS) { + return false + } + return glassesConnected && glassesWorn && !glassesCharging + } + + @JvmStatic + internal fun glassesStatusDescription(): String { + if (glassesStatusAtMs == 0L) { + return "no glasses status" + } + val age = System.currentTimeMillis() - glassesStatusAtMs + if (age > GLASSES_STATUS_MAX_AGE_MS) { + return "glasses status stale" + } + if (!glassesConnected) { + return "glasses not connected" + } + if (glassesCharging) { + return "glasses charging" + } + if (!glassesWorn) { + return "glasses not worn" + } + return "glasses worn" + } + + // ------------------------------------------------------------------ + // Phone actions -> JS + + @JvmStatic + fun setListener(newListener: FaceclawAlarmListener?) { + listener = newListener + } + + /** + * Record a dismiss / snooze done on the phone. Delivered live to the JS + * listener when one is registered, and journaled regardless so a JS side + * that was asleep or gone can catch up on its next boot. + */ + @JvmStatic + internal fun recordPhoneAction(context: Context, id: Long, action: String, minutes: Int) { + val appContext = context.applicationContext + val event = JSONObject() + try { + event.put("id", id) + event.put("action", action) + event.put("minutes", minutes) + event.put("at", System.currentTimeMillis()) + } catch (ignored: JSONException) { + return + } + synchronized(FaceclawAlarms::class.java) { + val journal = readArray(appContext, KEY_JOURNAL) + journal.put(event) + writeArray(appContext, KEY_JOURNAL, journal) + } + log(appContext, "phone " + action + " " + id + (if (minutes > 0) " ($minutes min)" else "")) + val current = listener + if (current != null) { + mainHandler.post { + try { + current.onPhoneAction(id, action, minutes) + } catch (error: RuntimeException) { + log(appContext, "listener failed: " + error.message) + } + } + } + } + + /** Hand the journal to the JS side (as a JSON array) and clear it. */ + @JvmStatic + fun drainJournal(context: Context): String { + synchronized(FaceclawAlarms::class.java) { + val appContext = context.applicationContext + val journal = readArray(appContext, KEY_JOURNAL) + writeArray(appContext, KEY_JOURNAL, JSONArray()) + return journal.toString() + } + } + + // ------------------------------------------------------------------ + // Reliability self-check + + /** + * Conditions under which the phone may fail to ring, as a JSON array of + * {code, message, fixable}, most serious first. Empty when everything is + * in order. The UI shows these before the wearer relies on an alarm. + */ + @JvmStatic + fun checkReliability(context: Context): String { + val appContext = context.applicationContext + val issues = JSONArray() + val notifications = appContext.getSystemService(Context.NOTIFICATION_SERVICE) as NotificationManager? + val alarms = appContext.getSystemService(Context.ALARM_SERVICE) as AlarmManager? + + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.S && alarms != null && !alarms.canScheduleExactAlarms()) { + addIssue(issues, "exact-alarm", "Exact alarms are not allowed for Faceclaw, so alarms may ring late.", true) + } + if (notifications != null && !notifications.areNotificationsEnabled()) { + addIssue(issues, "notifications", "Notifications are turned off for Faceclaw, so the phone cannot show or sound an alarm.", true) + } else if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O && notifications != null) { + val channel = notifications.getNotificationChannel(FaceclawAlarmService.CHANNEL_ID) + if (channel != null && channel.importance == NotificationManager.IMPORTANCE_NONE) { + addIssue(issues, "alarm-channel", "The Alarms notification category is blocked, so the phone cannot show an alarm.", true) + } + } + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.UPSIDE_DOWN_CAKE && notifications != null && !notifications.canUseFullScreenIntent()) { + addIssue(issues, "full-screen", "Full-screen alarms are not allowed, so a ringing alarm will not take over the lock screen.", true) + } + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.M && notifications != null) { + val filter = notifications.currentInterruptionFilter + if (filter == NotificationManager.INTERRUPTION_FILTER_NONE) { + addIssue(issues, "dnd-total", "Do Not Disturb is set to total silence, which silences alarms too.", true) + } else if (filter == NotificationManager.INTERRUPTION_FILTER_PRIORITY) { + try { + val policy = notifications.notificationPolicy + if (policy != null && (policy.priorityCategories and NotificationManager.Policy.PRIORITY_CATEGORY_ALARMS) == 0) { + addIssue(issues, "dnd-alarms", "Do Not Disturb does not allow alarms, so the phone will stay silent.", true) + } + } catch (ignored: SecurityException) { + // No notification-policy access: cannot inspect the exceptions. + } + } + } + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.P) { + val activityManager = appContext.getSystemService(Context.ACTIVITY_SERVICE) as ActivityManager? + if (activityManager != null && activityManager.isBackgroundRestricted) { + addIssue(issues, "background-restricted", "Background usage is restricted for Faceclaw, which blocks alarms while the app is not open.", true) + } + } + val power = appContext.getSystemService(Context.POWER_SERVICE) as PowerManager? + if (power != null && !power.isIgnoringBatteryOptimizations(appContext.packageName)) { + addIssue(issues, "battery-optimized", "Battery optimization is on for Faceclaw; some phones delay alarms because of it.", true) + } + val audio = appContext.getSystemService(Context.AUDIO_SERVICE) as AudioManager? + if (audio != null && audio.getStreamVolume(AudioManager.STREAM_ALARM) == 0) { + addIssue(issues, "alarm-volume", "The phone's alarm volume is muted; Faceclaw raises it while ringing, but check it.", true) + } + val maker = if (Build.MANUFACTURER == null) "" else Build.MANUFACTURER.lowercase(Locale.ROOT) + if (maker.contains("xiaomi") || maker.contains("huawei") || maker.contains("oppo") || maker.contains("vivo") + || maker.contains("oneplus") || maker.contains("realme") || maker.contains("meizu") || maker.contains("asus")) { + addIssue(issues, "oem-killer", "This phone maker is known to stop background apps; allow Faceclaw to auto-start and run unrestricted (see dontkillmyapp.com).", true) + } + if (scheduleLooksStale(appContext)) { + addIssue(issues, "schedule-stale", "The system lost Faceclaw's next alarm (the app may have been force-stopped); it was re-armed.", false) + rescheduleAll(appContext, "self-check") + } + return issues.toString() + } + + private fun addIssue(issues: JSONArray, code: String, message: String, fixable: Boolean) { + val issue = JSONObject() + try { + issue.put("code", code) + issue.put("message", message) + issue.put("fixable", fixable) + } catch (ignored: JSONException) { + return + } + issues.put(issue) + } + + /** Open the system screen where the issue can be fixed. */ + @JvmStatic + fun openReliabilityFix(context: Context, code: String?) { + val appContext = context.applicationContext + val pkg = appContext.packageName + var intent: Intent + when (code ?: "") { + "exact-alarm" -> + intent = if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.S) + Intent(Settings.ACTION_REQUEST_SCHEDULE_EXACT_ALARM, Uri.parse("package:$pkg")) + else + appDetails(pkg) + "notifications", "alarm-channel" -> { + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) { + intent = Intent(Settings.ACTION_APP_NOTIFICATION_SETTINGS) + .putExtra(Settings.EXTRA_APP_PACKAGE, pkg) + if ("alarm-channel" == code) { + intent = Intent(Settings.ACTION_CHANNEL_NOTIFICATION_SETTINGS) + .putExtra(Settings.EXTRA_APP_PACKAGE, pkg) + .putExtra(Settings.EXTRA_CHANNEL_ID, FaceclawAlarmService.CHANNEL_ID) + } + } else { + intent = appDetails(pkg) + } + } + "full-screen" -> + intent = if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.UPSIDE_DOWN_CAKE) + Intent(Settings.ACTION_MANAGE_APP_USE_FULL_SCREEN_INTENT, Uri.parse("package:$pkg")) + else + appDetails(pkg) + "dnd-total", "dnd-alarms" -> + intent = Intent(Settings.ACTION_SOUND_SETTINGS) + "battery-optimized" -> + intent = Intent(Settings.ACTION_REQUEST_IGNORE_BATTERY_OPTIMIZATIONS, Uri.parse("package:$pkg")) + "alarm-volume" -> + intent = Intent(Settings.ACTION_SOUND_SETTINGS) + else -> + intent = appDetails(pkg) + } + intent.addFlags(Intent.FLAG_ACTIVITY_NEW_TASK) + try { + appContext.startActivity(intent) + } catch (error: RuntimeException) { + val fallback = appDetails(pkg).addFlags(Intent.FLAG_ACTIVITY_NEW_TASK) + try { + appContext.startActivity(fallback) + } catch (ignored: RuntimeException) { + // No settings activity at all; nothing more to do. + } + } + } + + private fun appDetails(pkg: String): Intent { + return Intent(Settings.ACTION_APPLICATION_DETAILS_SETTINGS, Uri.parse("package:$pkg")) + } + + // ------------------------------------------------------------------ + // Diagnostics log + + @JvmStatic + internal fun log(context: Context, line: String) { + synchronized(FaceclawAlarms::class.java) { + val appContext = context.applicationContext + val log = readArray(appContext, KEY_LOG) + val next = JSONArray() + val start = Math.max(0, log.length() - (LOG_LINES - 1)) + for (index in start until log.length()) { + next.put(log.optString(index)) + } + next.put(System.currentTimeMillis().toString() + " " + line) + writeArray(appContext, KEY_LOG, next) + } + } + + /** The ring / miss / action log, newest last, one line per entry. */ + @JvmStatic + fun readLog(context: Context): String { + val log = readArray(context.applicationContext, KEY_LOG) + val out = StringBuilder() + for (index in 0 until log.length()) { + if (index > 0) { + out.append('\n') + } + out.append(log.optString(index)) + } + return out.toString() + } + + // ------------------------------------------------------------------ + // Intents + + private fun expiryPendingIntent(appContext: Context, entry: JSONObject): PendingIntent { + val intent = Intent(appContext, FaceclawAlarmReceiver::class.java) + intent.action = ACTION_EXPIRE + val id = entry.optLong("id") + intent.putExtra(EXTRA_ID, id) + intent.putExtra(EXTRA_TITLE, entry.optString("title")) + intent.putExtra(EXTRA_TEXT, entry.optString("text")) + intent.putExtra(EXTRA_KIND, entry.optString("kind", KIND_ALARM)) + intent.putExtra(EXTRA_SNOOZE_MINUTES, entry.optInt("snoozeMinutes", 10)) + var flags = PendingIntent.FLAG_UPDATE_CURRENT + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.M) { + flags = flags or PendingIntent.FLAG_IMMUTABLE + } + return PendingIntent.getBroadcast(appContext, requestCode(id), intent, flags) + } + + /** What the system's alarm indicator opens: the app. */ + private fun showIntent(appContext: Context, id: Long): PendingIntent { + val intent = Intent(appContext, NativeScriptActivity::class.java) + intent.addFlags(Intent.FLAG_ACTIVITY_SINGLE_TOP or Intent.FLAG_ACTIVITY_CLEAR_TOP) + var flags = PendingIntent.FLAG_UPDATE_CURRENT + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.M) { + flags = flags or PendingIntent.FLAG_IMMUTABLE + } + return PendingIntent.getActivity(appContext, requestCode(id), intent, flags) + } + + @JvmStatic + internal fun requestCode(id: Long): Int { + return (id xor (id ushr 32)).toInt() and 0x7fffffff + } + + @JvmStatic + internal fun startServiceCompat(appContext: Context, intent: Intent) { + ContextCompat.startForegroundService(appContext, intent) + } + } +} diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawAudioTranscoder.java b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawAudioTranscoder.java deleted file mode 100644 index 0aab2ff6..00000000 --- a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawAudioTranscoder.java +++ /dev/null @@ -1,242 +0,0 @@ -package com.faceclaw.app; - -import android.media.MediaCodec; -import android.media.MediaCodecInfo; -import android.media.MediaFormat; -import android.media.MediaMuxer; -import android.util.Log; - -import java.io.File; -import java.io.FileInputStream; -import java.io.IOException; -import java.nio.ByteBuffer; -import java.nio.ByteOrder; - -/** - * Transcodes a 16-bit PCM WAV conversation recording into AAC-LC in an M4A - * container, cutting speech recordings to roughly a tenth of their WAV size. - * Runs on its own thread; the source WAV is deleted on success when requested. - */ -public class FaceclawAudioTranscoder { - private static final String TAG = "FaceclawTranscode"; - private static final long CODEC_TIMEOUT_US = 10_000; - - public interface Listener { - void onDone(String outputPath, long outputBytes); - - void onError(String message); - } - - private final String wavPath; - private final String outputPath; - private final int bitrateBps; - private final boolean deleteSourceOnSuccess; - private final Listener listener; - - public FaceclawAudioTranscoder( - String wavPath, - String outputPath, - int bitrateBps, - boolean deleteSourceOnSuccess, - Listener listener) { - this.wavPath = wavPath; - this.outputPath = outputPath; - this.bitrateBps = bitrateBps > 0 ? bitrateBps : 32_000; - this.deleteSourceOnSuccess = deleteSourceOnSuccess; - this.listener = listener; - } - - public void start() { - Thread thread = new Thread(this::run, "FaceclawTranscode"); - thread.setPriority(Thread.MIN_PRIORITY); - thread.start(); - } - - private void run() { - try { - transcode(); - long size = new File(outputPath).length(); - if (deleteSourceOnSuccess) { - new File(wavPath).delete(); - } - if (listener != null) { - listener.onDone(outputPath, size); - } - } catch (Throwable t) { - Log.w(TAG, "transcode failed for " + wavPath, t); - new File(outputPath).delete(); - if (listener != null) { - listener.onError(t.getMessage() == null ? t.toString() : t.getMessage()); - } - } - } - - private void transcode() throws IOException { - WavInfo wav = readWavHeader(wavPath); - MediaFormat format = MediaFormat.createAudioFormat( - MediaFormat.MIMETYPE_AUDIO_AAC, wav.sampleRate, wav.channels); - format.setInteger(MediaFormat.KEY_AAC_PROFILE, MediaCodecInfo.CodecProfileLevel.AACObjectLC); - format.setInteger(MediaFormat.KEY_BIT_RATE, bitrateBps); - format.setInteger(MediaFormat.KEY_MAX_INPUT_SIZE, 16 * 1024); - - MediaCodec codec = MediaCodec.createEncoderByType(MediaFormat.MIMETYPE_AUDIO_AAC); - MediaMuxer muxer = null; - FileInputStream input = null; - try { - codec.configure(format, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE); - codec.start(); - muxer = new MediaMuxer(outputPath, MediaMuxer.OutputFormat.MUXER_OUTPUT_MPEG_4); - input = new FileInputStream(wavPath); - long skipped = 0; - while (skipped < wav.dataOffset) { - long step = input.skip(wav.dataOffset - skipped); - if (step <= 0) { - throw new IOException("could not seek past WAV header"); - } - skipped += step; - } - - MediaCodec.BufferInfo info = new MediaCodec.BufferInfo(); - byte[] chunk = new byte[8192]; - int track = -1; - boolean muxerStarted = false; - boolean inputDone = false; - boolean outputDone = false; - long remaining = wav.dataBytes; - long presentationUs = 0; - long bytesPerSecond = (long) wav.sampleRate * wav.channels * 2; - - while (!outputDone) { - if (!inputDone) { - int inIndex = codec.dequeueInputBuffer(CODEC_TIMEOUT_US); - if (inIndex >= 0) { - ByteBuffer inBuf = codec.getInputBuffer(inIndex); - int wanted = (int) Math.min(chunk.length, Math.min(remaining, inBuf.capacity())); - int read = wanted > 0 ? input.read(chunk, 0, wanted) : -1; - if (read <= 0) { - codec.queueInputBuffer(inIndex, 0, 0, presentationUs, - MediaCodec.BUFFER_FLAG_END_OF_STREAM); - inputDone = true; - } else { - inBuf.clear(); - inBuf.put(chunk, 0, read); - codec.queueInputBuffer(inIndex, 0, read, presentationUs, 0); - remaining -= read; - presentationUs += read * 1_000_000L / bytesPerSecond; - } - } - } - int outIndex = codec.dequeueOutputBuffer(info, CODEC_TIMEOUT_US); - if (outIndex == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) { - track = muxer.addTrack(codec.getOutputFormat()); - muxer.start(); - muxerStarted = true; - } else if (outIndex >= 0) { - if (info.size > 0 && (info.flags & MediaCodec.BUFFER_FLAG_CODEC_CONFIG) == 0) { - if (!muxerStarted) { - throw new IOException("encoder produced data before format"); - } - ByteBuffer outBuf = codec.getOutputBuffer(outIndex); - outBuf.position(info.offset); - outBuf.limit(info.offset + info.size); - muxer.writeSampleData(track, outBuf, info); - } - codec.releaseOutputBuffer(outIndex, false); - if ((info.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0) { - outputDone = true; - } - } - } - } finally { - if (input != null) { - try { - input.close(); - } catch (IOException ignored) { - } - } - try { - codec.stop(); - } catch (Throwable ignored) { - } - codec.release(); - if (muxer != null) { - try { - muxer.stop(); - } catch (Throwable ignored) { - } - muxer.release(); - } - } - } - - private static final class WavInfo { - int sampleRate; - int channels; - long dataOffset; - long dataBytes; - } - - /** Minimal RIFF walk: locates fmt and data chunks, 16-bit PCM only. */ - private static WavInfo readWavHeader(String path) throws IOException { - try (FileInputStream in = new FileInputStream(path)) { - byte[] head = new byte[12]; - if (in.read(head) != 12 || head[0] != 'R' || head[1] != 'I' || head[8] != 'W') { - throw new IOException("not a WAV file"); - } - WavInfo info = new WavInfo(); - long offset = 12; - byte[] chunkHead = new byte[8]; - while (in.read(chunkHead) == 8) { - ByteBuffer b = ByteBuffer.wrap(chunkHead).order(ByteOrder.LITTLE_ENDIAN); - int id = b.getInt(); - long size = b.getInt() & 0xffffffffL; - offset += 8; - if (id == 0x20746d66) { // "fmt " - byte[] fmt = new byte[(int) Math.min(size, 16)]; - if (in.read(fmt) != fmt.length) { - throw new IOException("truncated fmt chunk"); - } - ByteBuffer f = ByteBuffer.wrap(fmt).order(ByteOrder.LITTLE_ENDIAN); - int audioFormat = f.getShort(); - info.channels = f.getShort(); - info.sampleRate = f.getInt(); - f.getInt(); - f.getShort(); - int bits = f.getShort(); - if (audioFormat != 1 || bits != 16) { - throw new IOException("only 16-bit PCM WAV is supported"); - } - long skip = size - fmt.length; - while (skip > 0) { - long step = in.skip(skip); - if (step <= 0) { - break; - } - skip -= step; - } - offset += size; - } else if (id == 0x61746164) { // "data" - info.dataOffset = offset; - // A zero size means the header was never patched (crash - // mid-recording); recover the length from the file size. - info.dataBytes = size > 0 ? size : new File(path).length() - offset; - break; - } else { - long skip = size; - while (skip > 0) { - long step = in.skip(skip); - if (step <= 0) { - break; - } - skip -= step; - } - offset += size; - } - } - if (info.sampleRate == 0 || info.dataOffset == 0) { - throw new IOException("missing fmt or data chunk"); - } - return info; - } - } -} diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawAudioTranscoder.kt b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawAudioTranscoder.kt new file mode 100644 index 00000000..1ffe9d16 --- /dev/null +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawAudioTranscoder.kt @@ -0,0 +1,236 @@ +package com.faceclaw.app + +import android.media.MediaCodec +import android.media.MediaCodecInfo +import android.media.MediaFormat +import android.media.MediaMuxer +import android.util.Log + +import java.io.File +import java.io.FileInputStream +import java.io.IOException +import java.nio.ByteBuffer +import java.nio.ByteOrder + +/** + * Transcodes a 16-bit PCM WAV conversation recording into AAC-LC in an M4A + * container, cutting speech recordings to roughly a tenth of their WAV size. + * Runs on its own thread; the source WAV is deleted on success when requested. + */ +class FaceclawAudioTranscoder( + private val wavPath: String?, + private val outputPath: String?, + bitrateBps: Int, + private val deleteSourceOnSuccess: Boolean, + private val listener: Listener? +) { + companion object { + private const val TAG = "FaceclawTranscode" + private const val CODEC_TIMEOUT_US: Long = 10_000 + + /** Minimal RIFF walk: locates fmt and data chunks, 16-bit PCM only. */ + @JvmStatic + @Throws(IOException::class) + private fun readWavHeader(path: String?): WavInfo { + FileInputStream(path).use { `in` -> + val head = ByteArray(12) + if (`in`.read(head) != 12 || head[0] != 'R'.code.toByte() || head[1] != 'I'.code.toByte() || head[8] != 'W'.code.toByte()) { + throw IOException("not a WAV file") + } + val info = WavInfo() + var offset: Long = 12 + val chunkHead = ByteArray(8) + while (`in`.read(chunkHead) == 8) { + val b = ByteBuffer.wrap(chunkHead).order(ByteOrder.LITTLE_ENDIAN) + val id = b.getInt() + val size = b.getInt().toLong() and 0xffffffffL + offset += 8 + if (id == 0x20746d66) { // "fmt " + val fmt = ByteArray(Math.min(size, 16L).toInt()) + if (`in`.read(fmt) != fmt.size) { + throw IOException("truncated fmt chunk") + } + val f = ByteBuffer.wrap(fmt).order(ByteOrder.LITTLE_ENDIAN) + val audioFormat = f.getShort().toInt() + info.channels = f.getShort().toInt() + info.sampleRate = f.getInt() + f.getInt() + f.getShort() + val bits = f.getShort().toInt() + if (audioFormat != 1 || bits != 16) { + throw IOException("only 16-bit PCM WAV is supported") + } + var skip = size - fmt.size + while (skip > 0) { + val step = `in`.skip(skip) + if (step <= 0) { + break + } + skip -= step + } + offset += size + } else if (id == 0x61746164) { // "data" + info.dataOffset = offset + // A zero size means the header was never patched (crash + // mid-recording); recover the length from the file size. + info.dataBytes = if (size > 0) size else File(path).length() - offset + break + } else { + var skip = size + while (skip > 0) { + val step = `in`.skip(skip) + if (step <= 0) { + break + } + skip -= step + } + offset += size + } + } + if (info.sampleRate == 0 || info.dataOffset == 0L) { + throw IOException("missing fmt or data chunk") + } + return info + } + } + } + + interface Listener { + fun onDone(outputPath: String?, outputBytes: Long) + + fun onError(message: String?) + } + + private val bitrateBps: Int = if (bitrateBps > 0) bitrateBps else 32_000 + + fun start() { + val thread = Thread(Runnable { run() }, "FaceclawTranscode") + thread.priority = Thread.MIN_PRIORITY + thread.start() + } + + private fun run() { + try { + transcode() + val size = File(outputPath).length() + if (deleteSourceOnSuccess) { + File(wavPath).delete() + } + if (listener != null) { + listener.onDone(outputPath, size) + } + } catch (t: Throwable) { + Log.w(TAG, "transcode failed for $wavPath", t) + File(outputPath).delete() + if (listener != null) { + listener.onError(if (t.message == null) t.toString() else t.message) + } + } + } + + @Throws(IOException::class) + private fun transcode() { + val wav = readWavHeader(wavPath) + val format = MediaFormat.createAudioFormat( + MediaFormat.MIMETYPE_AUDIO_AAC, wav.sampleRate, wav.channels) + format.setInteger(MediaFormat.KEY_AAC_PROFILE, MediaCodecInfo.CodecProfileLevel.AACObjectLC) + format.setInteger(MediaFormat.KEY_BIT_RATE, bitrateBps) + format.setInteger(MediaFormat.KEY_MAX_INPUT_SIZE, 16 * 1024) + + val codec = MediaCodec.createEncoderByType(MediaFormat.MIMETYPE_AUDIO_AAC) + var muxer: MediaMuxer? = null + var input: FileInputStream? = null + try { + codec.configure(format, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE) + codec.start() + muxer = MediaMuxer(outputPath!!, MediaMuxer.OutputFormat.MUXER_OUTPUT_MPEG_4) + input = FileInputStream(wavPath) + var skipped: Long = 0 + while (skipped < wav.dataOffset) { + val step = input.skip(wav.dataOffset - skipped) + if (step <= 0) { + throw IOException("could not seek past WAV header") + } + skipped += step + } + + val info = MediaCodec.BufferInfo() + val chunk = ByteArray(8192) + var track = -1 + var muxerStarted = false + var inputDone = false + var outputDone = false + var remaining = wav.dataBytes + var presentationUs: Long = 0 + val bytesPerSecond = wav.sampleRate.toLong() * wav.channels * 2 + + while (!outputDone) { + if (!inputDone) { + val inIndex = codec.dequeueInputBuffer(CODEC_TIMEOUT_US) + if (inIndex >= 0) { + val inBuf = codec.getInputBuffer(inIndex)!! + val wanted = Math.min(chunk.size.toLong(), Math.min(remaining, inBuf.capacity().toLong())).toInt() + val read = if (wanted > 0) input.read(chunk, 0, wanted) else -1 + if (read <= 0) { + codec.queueInputBuffer(inIndex, 0, 0, presentationUs, + MediaCodec.BUFFER_FLAG_END_OF_STREAM) + inputDone = true + } else { + inBuf.clear() + inBuf.put(chunk, 0, read) + codec.queueInputBuffer(inIndex, 0, read, presentationUs, 0) + remaining -= read.toLong() + presentationUs += read * 1_000_000L / bytesPerSecond + } + } + } + val outIndex = codec.dequeueOutputBuffer(info, CODEC_TIMEOUT_US) + if (outIndex == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) { + track = muxer.addTrack(codec.outputFormat) + muxer.start() + muxerStarted = true + } else if (outIndex >= 0) { + if (info.size > 0 && (info.flags and MediaCodec.BUFFER_FLAG_CODEC_CONFIG) == 0) { + if (!muxerStarted) { + throw IOException("encoder produced data before format") + } + val outBuf = codec.getOutputBuffer(outIndex)!! + outBuf.position(info.offset) + outBuf.limit(info.offset + info.size) + muxer.writeSampleData(track, outBuf, info) + } + codec.releaseOutputBuffer(outIndex, false) + if ((info.flags and MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0) { + outputDone = true + } + } + } + } finally { + if (input != null) { + try { + input.close() + } catch (ignored: IOException) { + } + } + try { + codec.stop() + } catch (ignored: Throwable) { + } + codec.release() + if (muxer != null) { + try { + muxer.stop() + } catch (ignored: Throwable) { + } + muxer.release() + } + } + } + + private class WavInfo { + var sampleRate = 0 + var channels = 0 + var dataOffset: Long = 0 + var dataBytes: Long = 0 + } +} diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawBleCommunicator.java b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawBleCommunicator.java deleted file mode 100644 index c7aeef55..00000000 --- a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawBleCommunicator.java +++ /dev/null @@ -1,4100 +0,0 @@ -package com.faceclaw.app; - -import android.annotation.SuppressLint; -import android.bluetooth.BluetoothGatt; -import android.bluetooth.BluetoothGattCharacteristic; -import android.app.KeyguardManager; -import android.content.BroadcastReceiver; -import android.content.Context; -import android.content.Intent; -import android.content.IntentFilter; -import android.os.Handler; -import android.os.Looper; -import android.os.PowerManager; -import android.os.SystemClock; -import android.util.Log; - -import java.io.ByteArrayOutputStream; -import java.io.PrintWriter; -import java.io.StringWriter; -import java.nio.charset.StandardCharsets; -import java.util.ArrayDeque; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.HashMap; -import java.util.Iterator; -import java.util.List; -import java.util.Locale; -import java.util.Map; -import java.util.Random; - -@SuppressLint("MissingPermission") -public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable { - private static final String TAG = "FaceclawComm"; - - // Local metadata for custom-command bookkeeping, not an EvenHub container. - // Submitted frames supply pixel geometry; the stock layout only captures input. - private static final BleProtocol.ImageTileOptions DASHBOARD_TILE = - new BleProtocol.ImageTileOptions("img00", 10, 0, 0, 576, 288); - - private static final String G2_SCREEN_WAKE_LOCK_TAG = "Faceclaw:G2Screen"; - private static final long FACECLAW_WAKE_LEASE_RENEW_MS = 45_000; - private static final long FACECLAW_WAKE_CONTROL_WAIT_MS = 1_500; - private static final long CFW_CLEANUP_WAIT_MS = 4_000; - private static final int COMPASS_REPORT_INTERVAL_MS = 100; - private static final int COMPASS_MIN_CHANGE_DEGREES = 0; - - private final Context appContext; - private final PowerManager powerManager; - private final KeyguardManager keyguardManager; - private final FaceclawBleManager bleManager; - private final InterruptibleSleep interruptibleSleep = new InterruptibleSleep(); - private final Object lock = new Object(); - private final Handler mainHandler = new Handler(Looper.getMainLooper()); - private final String rightAddress; - private final String leftAddress; - private final String ringAddress; - - private volatile FaceclawBleCommunicatorListener listener; - private final java.util.List imuListeners = - new java.util.concurrent.CopyOnWriteArrayList<>(); - /** A compass subscriber plus the Looper it registered from (see addCompassListener). */ - private static final class CompassSubscription { - final FaceclawCompassListener listener; - final Handler handler; - - CompassSubscription(FaceclawCompassListener listener, Handler handler) { - this.listener = listener; - this.handler = handler; - } - } - - private final java.util.List compassSubscriptions = - new java.util.concurrent.CopyOnWriteArrayList<>(); - private final java.util.List ambientLightListeners = - new java.util.concurrent.CopyOnWriteArrayList<>(); - private final java.util.List micStatusListeners = - new java.util.concurrent.CopyOnWriteArrayList<>(); - private volatile Thread workerThread; - private volatile boolean running; - private volatile boolean userDisconnectRequested; - // Set when a connect attempt failed while an arm's Android bond is gone: - // retrying is pointless until the user re-pairs, so the worker loop parks - // instead of redialing. Cleared by start() (a fresh explicit connect). - private volatile boolean reconnectHalted; - - private String phase = "disconnected"; - private String status = "Disconnected."; - - private boolean rightConnected; - private boolean leftConnected; - private boolean ringConnected; - private boolean ringNotificationsReady; - private boolean sessionReady; - private boolean fixedLayoutCreated; - private boolean shutdownRequested; - // CFW firmware-debug-flags overlay (mode 7). Desired value pushed from TS; the - // sub-op last sent this session (-1 = not yet), reset on (re)connect so the - // overlay state is re-asserted on every reconnect and whenever the value changes. - private volatile boolean firmwareDebugFlagsEnabled; - private int firmwareDebugFlagsLastSent = -1; - // Desired CFW mode-10 compass state. It survives reconnects; lastSent is - // reset with each session so an open Compass window is re-asserted. - /** - * Who currently wants the stock compass running (the Compass window, the - * Navigate worker, ...). The magnetometer is one shared resource, so it - * stays on while any owner holds it and is released when the last lets go; - * this keeps one app's release from silently switching off another's feed. - */ - private final java.util.Set compassOwners = new java.util.HashSet<>(); - private int compassControlLastSent = -1; - // Whether the glasses-side compass may still be running: set when an enable - // is enqueued, cleared only when a disable is acked. Drives the forced - // disable sent ahead of an EvenHub shutdown/suspend, since a pending - // disable can be wiped by the shutdown's queue flush and the retry loop - // does not run while shutdownRequested (magnetometer left on = battery drain). - private boolean compassMaybeOn; - private boolean startupProbePending; - // Desired ownership of CFW's fail-open stock-wake lease (dashboard launch - // and Even AI foreground takeover). This survives a transport reconnect; - // the lease itself is volatile firmware state and is re-acquired once both - // arms are ready. - private boolean faceclawWakeLeaseEnabled; - private long lastFaceclawWakeLeaseQueuedAtMs; - private int faceclawWakeControlGeneration; - private int faceclawWakeControlSentCount; - private long lastFaceclawFramebufferLeaseQueuedAtMs; - private int faceclawFramebufferControlGeneration; - private int faceclawFramebufferControlSentCount; - private int faceclawWakePendingNonce = -1; - /** - * The last firmware-info read said the glasses run Faceclaw's custom - * firmware. Gates the private modes (cleanup, resource cache, ...) so stock - * or third-party firmware never sees them; the TS side checks the actual - * revision and disconnects on a mismatch, so no per-feature gating is - * needed here. - */ - private boolean customFirmwareDetected; - private boolean cfwCleanupDelivered; - private int lastCfwCleanupAckMagic; - - private long reconnectAfterMs; - private long ringReconnectAfterMs; - private long lastAckAtMs; - private long lastIncomingAtMs; - private long lastHeartbeatSentAtMs; - private long lastHeartbeatAckedAtMs; - private long lastConnectionOrInputAtMs; - private long lastBatteryRefreshAtMs; - private long imageRetryAfterMs; - private long lastSessionReadyAtMs; - private long lastEvenAppConflictAtMs; - private int consecutiveAckTimeouts; - private int lastAudioControlAckMagic = 0; - - private ConnectionOptions connectionOptions = new ConnectionOptions(); - private final BleMagicPool magicPool = new BleMagicPool(AndroidProtocolPlatform.INSTANCE); - private MessageBuilder messageBuilder = new MessageBuilder(magicPool); - private int nextTransportSeq = 0x40; - private int nextMapSessionId = 0; - private int nextImageUpdateId = 1; - // Wire frame id for mode-3 deltas (CFW reorder/skip/dup diagnostic). uint16, - // advanced by 1 per emitted delta; kept in [1, 0xfffe] to avoid the CFW's - // 0xffff "empty" sentinel. - private int nextImageFrameId = 1; - private int lastShutdownAckMagic = 0; - private long lastShutdownExitAtMs = 0; - private int headsetBattery = -1; - private int headsetCharging = -1; - private int ringBattery = -1; - private int ringCharging = -1; - // Silent mode: 1 = on, 0 = off, -1 = not yet known. See updateSilentModeLocked. - private int silentMode = -1; - private int wearState = -1; - private int phoneLockState = -1; - private long lastPhoneLockCheckAtMs; - private boolean phoneLockReceiverRegistered; - private boolean audioCaptureActive; - private boolean firmwareInfoQueried; - // Glasses are in the charging case: nobody is wearing them, so display - // communication pauses and only battery polls flow (see driveSession). - private boolean chargingMode; - private volatile FaceclawAudioPacketListener audioPacketListener; - private PowerManager.WakeLock g2ScreenWakeLock; - - // BLE bandwidth benchmark (Developer app). Streams no-op image payloads - // (CFW mode 7 with an unused sub-op: parsed, acked, and discarded — stock - // firmware likewise ignores unknown image modes) for a fixed duration with - // a selectable message size and pipeline window, then reports throughput. - // While active, desired-frame sends are held back and heartbeats are - // satisfied by the benchmark's own acks, so the stream is the only image - // traffic. All state below is guarded by `lock`; results are read with - // getBandwidthBenchmarkStatus() and survive until the next run starts. - private boolean benchmarkActive; - private boolean benchmarkAborted; - private final Random benchmarkRandom = new Random(); - private int benchmarkMessageSize; - private int benchmarkWindowSize; - private int benchmarkDurationMs; - private int benchmarkLinkMode; - private boolean benchmarkLinkPending; - private long benchmarkReadyAtMs; - private long benchmarkStartAtMs; // first benchmark write; 0 until then - private long benchmarkDeadlineAtMs; // start + duration; MAX_VALUE until first write - private long benchmarkLastAckAtMs; - private long benchmarkEndAtMs; // 0 while running; set when the run drains - private int benchmarkMessagesSent; - private int benchmarkMessagesAcked; - private int benchmarkTimeouts; - private long benchmarkPayloadBytesAcked; - private long benchmarkWireBytesAcked; - // A timed-out benchmark message aborts the run, but its already-in-flight - // peers still time out one by one; keep the window comfortably below - // MAX_CONSECUTIVE_ACK_TIMEOUTS so a dead run can't escalate into a - // transport-failure reconnect all by itself. - private static final int BENCHMARK_MAX_WINDOW = 6; - - private final BroadcastReceiver phoneLockReceiver = new BroadcastReceiver() { - @Override public void onReceive(Context context, Intent intent) { - emitPhoneLockStateIfChanged(true); - interruptibleSleep.interrupt(); - } - }; - - private String displayedFingerprint = ""; - // The frame the firmware shadow will hold once the current image pipeline - // drains: the most recently ENQUEUED image (headerless packed 4bpp, see - // BmpUtil.pack4bppFromGray8), which is the correct base for the next delta - // when frames are pipelined. Set at enqueue; cleared whenever the image - // pipeline is cleared (clearAllMessagesLocked / clearMessagesOfKindLocked - // "image"), so it is only ever read while it holds a valid current-session base. - private byte[] lastEnqueuedPacked = new byte[0]; - private int lastEnqueuedWidth; - private int lastEnqueuedHeight; - private String lastEnqueuedFingerprint = ""; - private final Map imageUpdateStats = new HashMap<>(); - - private final Object desiredTilesLock = new Object(); - private String desiredFingerprint = ""; - // Headerless packed 4bpp frame (see BmpUtil.pack4bppFromGray8) plus its - // pixel dimensions. - private byte[] desiredPacked = new byte[0]; - private int desiredWidth; - private int desiredHeight; - private int desiredPaintMs; - private int desiredFrameId; - // Screen-space deferred draws (glyphs + images) whose pixels are baked - // into desiredPacked; the resource-cache planner may replay them as - // on-glasses cached draws. - private SurfaceCompositor.ScreenDraw[] desiredDraws = new SurfaceCompositor.ScreenDraw[0]; - // (frame, reason) of the last "waiting to send" line, so a frame that - // stalls for seconds records one line per state change (see - // noteImageStallLocked). Send-loop thread only. - private int stallFrameId; - private String stallReason = ""; - // Highest compositor sequence stored as the desired frame; composites that - // lost a store race to a newer one are discarded (their content is already - // included in the newer composite). - private long lastStoredCompositeSeq; - - // Wire submissions need screenGray + the shell scene, never preview pixels. - // Preview/screenshot/recording callers render those explicitly on demand. - private final SurfaceCompositor compositor = new SurfaceCompositor(false); - - // Phone-side model of the CFW's 192 KiB resource cache. - // Reset whenever the image pipeline / EvenHub session is torn down: the - // firmware frees the cache with the fb lease, and after any resync the - // cheap safe assumption is an empty cache (glyphs re-upload lazily). - private final ResourceCacheState resourceCache = new ResourceCacheState(); - private final ScenePlanner scenePlanner = new ScenePlanner(resourceCache); - private ShellScene desiredShellScene = ShellScene.Companion.getEMPTY(); - - private final ArrayDeque pendingMessages = new ArrayDeque<>(); - private final CfwTransport[] cfwTransports = { new CfwTransport(AndroidProtocolPlatform.INSTANCE), new CfwTransport(AndroidProtocolPlatform.INSTANCE) }; - private final ArrayDeque inFlightMessages = new ArrayDeque<>(); - private OutboundMessage prewrittenMessage; - private List prewrittenFrames = Collections.emptyList(); - - public FaceclawBleCommunicator(Context context, String rightAddress, String leftAddress, String ringAddress) { - this.appContext = context.getApplicationContext(); - FrameTimings.getInstance().init(appContext); - this.powerManager = (PowerManager) appContext.getSystemService(Context.POWER_SERVICE); - this.keyguardManager = (KeyguardManager) appContext.getSystemService(Context.KEYGUARD_SERVICE); - this.bleManager = new FaceclawBleManager(appContext); - this.bleManager.setListener(this); - this.rightAddress = requireAddress("rightAddress", rightAddress); - this.leftAddress = requireAddress("leftAddress", leftAddress); - this.ringAddress = ringAddress == null ? "" : ringAddress.trim(); - IntentFilter phoneLockFilter = new IntentFilter(); - phoneLockFilter.addAction(Intent.ACTION_SCREEN_ON); - phoneLockFilter.addAction(Intent.ACTION_SCREEN_OFF); - phoneLockFilter.addAction(Intent.ACTION_USER_PRESENT); - appContext.registerReceiver(phoneLockReceiver, phoneLockFilter); - phoneLockReceiverRegistered = true; - } - - - public void setListener(FaceclawBleCommunicatorListener listener) { - this.listener = listener; - emitState(); - emitPhoneLockStateIfChanged(true); - } - - public void start() { - synchronized (lock) { - if (running) { - return; - } - running = true; - userDisconnectRequested = false; - reconnectHalted = false; - shutdownRequested = false; - activeInstance = this; - workerThread = new Thread(this, "FaceclawBleCommunicator"); - workerThread.start(); - } - } - - public void disconnect() { - /* On the normal path DashboardController already sent mode 11 after - * quiescing its producers. Also cover direct/early close callers here; - * a successful cleanup must remain the final BLE message. Older CFWs - * fall back to the standalone framebuffer-lease release. */ - if (!cfwCleanupDelivered && !sendCfwCleanup()) { - releaseFaceclawFramebufferLease(); - } - Thread threadToJoin; - synchronized (lock) { - userDisconnectRequested = true; - running = false; - audioCaptureActive = false; - audioPacketListener = null; - threadToJoin = workerThread; - } - setStateDisplay("disconnecting", "Disconnecting..."); - interruptibleSleep.interrupt(); - if (threadToJoin != null) { - threadToJoin.interrupt(); - try { - threadToJoin.join(5_000); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - } - } - synchronized (lock) { - workerThread = null; - resetSessionStateLocked(); - clearAllMessagesLocked("disconnect"); - // Unknown until the next connection's first push or settings poll. - silentMode = -1; - } - bleManager.disconnect(rightAddress); - bleManager.disconnect(leftAddress); - if (hasRingAddress()) { - bleManager.disconnect(ringAddress); - } - bleManager.close(); - releaseG2ScreenWakeLock(); - setStateDisplay("disconnected", "Disconnected."); - } - - public void close() { - if (activeInstance == this) { - activeInstance = null; - } - disconnect(); - for (CfwTransport transport : cfwTransports) transport.close(); - if (phoneLockReceiverRegistered) { - phoneLockReceiverRegistered = false; - appContext.unregisterReceiver(phoneLockReceiver); - } - } - - public void setG2ScreenOn(boolean screenOn) { - mainHandler.post(() -> updateG2ScreenWakeLock(screenOn)); - } - - public void setFirmwareDebugFlags(boolean enabled) { - // Just record it; the drive loop emits the mode-7 control message when the - // display path is ready and idle, and re-emits when this value changes. - firmwareDebugFlagsEnabled = enabled; - } - - public boolean startG2AudioCapture(FaceclawAudioPacketListener listener) { - if (listener == null) { - throw new IllegalArgumentException("listener is required"); - } - int magic; - synchronized (lock) { - if (!running || !sessionReady || shutdownRequested || !fixedLayoutCreated) { - logLine("skip G2 mic enable; EvenHub display path not ready"); - return false; - } - audioPacketListener = listener; - OutboundMessage message = createAudioControlMessageLocked(true); - magic = message.magic; - pendingMessages.addFirst(message); - logLine("queue G2 mic enable"); - } - interruptibleSleep.interrupt(); - return waitForAudioControlAck(magic, "enable"); - } - - public void stopG2AudioCapture() { - int magic = 0; - synchronized (lock) { - audioPacketListener = null; - audioCaptureActive = false; - clearMessagesOfKindLocked("audio-control"); - if (running && sessionReady) { - OutboundMessage message = createAudioControlMessageLocked(false); - magic = message.magic; - pendingMessages.addFirst(message); - logLine("queue G2 mic disable"); - } - } - interruptibleSleep.interrupt(); - if (magic != 0) { - waitForAudioControlAck(magic, "disable"); - } - } - - public boolean isSessionReady() { - synchronized (lock) { - return running && sessionReady; - } - } - - /** - * Whether the glasses mic is enabled right now. The enable lives in the - * current EvenHub session, so it dies with a transport drop, the charging - * case, or a suspend — silently, from the phone's point of view. Callers - * that track a capture across those events must check this rather than - * assume their earlier enable still holds. - */ - public boolean isAudioCaptureActive() { - synchronized (lock) { - return running && sessionReady && !shutdownRequested && audioCaptureActive; - } - } - - /** - * Acquire/renew or release CFW's volatile wake-takeover lease on both - * arms. Delivery (not a protocol ACK) is awaited so a caller can ensure - * the fail-open firmware policy is installed before relying on wakeword - * interception or suspending EvenHub. - */ - public boolean setFaceclawWakeLeaseEnabled(boolean enabled) { - int generation; - synchronized (lock) { - faceclawWakeLeaseEnabled = enabled; - if (!running || !sessionReady) { - return !enabled; - } - generation = enqueueFaceclawWakeControlLocked( - enabled ? BleProtocol.FACECLAW_WAKE_OP_ACQUIRE : BleProtocol.FACECLAW_WAKE_OP_RELEASE, - 0, - true - ); - if (!enabled) { - faceclawWakePendingNonce = -1; - } - } - interruptibleSleep.interrupt(); - return waitForFaceclawWakeControlDelivery(generation, FACECLAW_WAKE_CONTROL_WAIT_MS); - } - - /** - * Wait until the recreated layout and retained compositor frame have both - * landed. If this wake came from CFW's deferred double tap, READY - * is then sent to both arms to cancel their stock-dashboard fallback. - */ - public boolean awaitEvenHubSessionReady(int timeoutMs) { - long deadline = SystemClock.elapsedRealtime() + Math.max(0, timeoutMs); - int readyGeneration = 0; - synchronized (lock) { - while (running && sessionReady) { - boolean frameReady = false; - synchronized (desiredTilesLock) { - frameReady = !desiredFingerprint.isEmpty() - && desiredFingerprint.equals(displayedFingerprint); - } - if (!shutdownRequested && fixedLayoutCreated && frameReady) { - if (faceclawWakePendingNonce >= 0) { - readyGeneration = enqueueFaceclawWakeControlLocked( - BleProtocol.FACECLAW_WAKE_OP_READY, - faceclawWakePendingNonce, - true - ); - faceclawWakePendingNonce = -1; - } - break; - } - long remaining = deadline - SystemClock.elapsedRealtime(); - if (remaining <= 0) { - return false; - } - try { - lock.wait(Math.min(remaining, 100)); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - return false; - } - } - if (!running || !sessionReady) { - return false; - } - } - if (readyGeneration != 0) { - interruptibleSleep.interrupt(); - if (!waitForFaceclawWakeControlDelivery(readyGeneration, FACECLAW_WAKE_CONTROL_WAIT_MS)) { - logLine("wake READY delivery not confirmed before fallback deadline"); - } - } - return true; - } - - /** - * Enable or disable the IMU (accelerometer) report stream. Fire-and-forget: - * the control message is queued ahead of other traffic; readings arrive via - * registered FaceclawImuListeners. reportFrq is the requested sample rate - * (ignored on disable). - */ - public void setImuReportEnabled(boolean enable, int reportFrq) { - synchronized (lock) { - if (!running || !sessionReady) { - logLine("skip IMU " + (enable ? "enable" : "disable") + "; session not ready"); - return; - } - clearMessagesOfKindLocked("imu-control"); - OutboundMessage message = messageBuilder.enableOrDisableImu(enable, reportFrq); - message.onTimeout = () -> logLine("IMU control ack timeout"); - pendingMessages.addFirst(message); - logLine("queue IMU " + (enable ? "enable freq=" + reportFrq : "disable")); - } - interruptibleSleep.interrupt(); - } - - /** - * Enable/disable the stock compass through CFW image-handler mode 10. The - * desired state is retained across reconnects; headings arrive through - * stock sid-0x08 navigation notifications and FaceclawCompassListeners. - */ - public void setCompassEnabled(boolean enable) { - setCompassEnabled("compass", enable); - } - - /** - * As above, on behalf of a named owner. The compass runs while at least - * one owner has enabled it; an owner disabling it only takes effect once - * no other owner still wants it. - */ - public void setCompassEnabled(String owner, boolean enable) { - synchronized (lock) { - boolean before = !compassOwners.isEmpty(); - if (enable) { - compassOwners.add(owner); - } else { - compassOwners.remove(owner); - } - boolean wanted = !compassOwners.isEmpty(); - if (wanted == before && compassControlLastSent == (wanted ? 1 : 0)) { - logLine("compass " + (enable ? "enable" : "disable") + " by " + owner - + "; state unchanged (owners=" + compassOwners + ")"); - return; - } - compassControlLastSent = -1; - clearMessagesOfKindLocked("compass-control"); - if (running && sessionReady && !shutdownRequested && fixedLayoutCreated) { - enqueueCompassControlLocked(true, wanted); - } else { - logLine("defer compass " + (enable ? "enable" : "disable") + "; display path not ready"); - } - } - interruptibleSleep.interrupt(); - } - - /** - * Set the lens brightness. Fire-and-forget, like the IMU control: the - * message is queued ahead of other traffic and any not-yet-sent brightness - * message is superseded. When autoAdjust is true the ambient-light sensor - * drives brightness and brightnessLevel is ignored; otherwise - * brightnessLevel (0-100) is applied directly. - */ - public void setBrightness(boolean autoAdjust, int brightnessLevel) { - synchronized (lock) { - if (!running || !sessionReady) { - logLine("skip brightness set; session not ready"); - return; - } - clearMessagesOfKindLocked("brightness-control"); - OutboundMessage message = messageBuilder.setBrightness(autoAdjust, brightnessLevel); - message.onTimeout = () -> logLine("brightness control ack timeout"); - pendingMessages.addFirst(message); - logLine("queue brightness " + (autoAdjust ? "auto" : "level=" + brightnessLevel)); - } - interruptibleSleep.interrupt(); - } - - /** - * Enable the stock wear detector, then ask CFW to emit its current cached - * state. The queue order matters: the query must run after the setting is - * applied, including on a fresh install where wear detection was disabled. - */ - public void enableWearDetectionAndRequestState() { - synchronized (lock) { - if (!running || !sessionReady) { - logLine("skip wear detector setup; session not ready"); - return; - } - clearMessagesOfKindLocked("wear-detection-control"); - clearMessagesOfKindLocked("wear-query-control"); - OutboundMessage queryLeft = messageBuilder.faceclawWearQuery(true); - OutboundMessage queryRight = messageBuilder.faceclawWearQuery(false); - OutboundMessage enable = messageBuilder.setWearDetection(true); - enable.onTimeout = () -> logLine("wear detection enable ack timeout"); - pendingMessages.addFirst(queryLeft); - pendingMessages.addFirst(queryRight); - pendingMessages.addFirst(enable); - logLine("queue wear detection enable + current-state query"); - } - interruptibleSleep.interrupt(); - } - - /** - * Start the BLE bandwidth benchmark: stream messageSize-byte no-op image - * payloads for durationMs, keeping up to windowSize messages awaiting ack - * at once, then leave the results for getBandwidthBenchmarkStatus(). - * Returns false when a run is already active or the image path is not - * ready. The duration clock starts at the first benchmark write, so - * traffic already queued ahead of the run doesn't count against it. - */ - public boolean startBandwidthBenchmark(int messageSize, int windowSize, int durationMs) { - return startBandwidthBenchmarkWithLinkMode(messageSize, windowSize, durationMs, 0); - } - - // 0: current link; 1: re-request HIGH; 2: request 2M; 3: both. - public boolean startBandwidthBenchmarkWithLinkMode(int messageSize, int windowSize, - int durationMs, int linkMode) { - synchronized (lock) { - if (benchmarkActive || !running || !sessionReady || !fixedLayoutCreated - || shutdownRequested || chargingMode) { - logLine("skip bandwidth benchmark; already running or image path not ready"); - return false; - } - benchmarkMessageSize = Math.max(2, Math.min(messageSize, ConnectionOptions.IMAGE_FRAGMENT_SIZE)); - benchmarkWindowSize = Math.max(1, Math.min(windowSize, BENCHMARK_MAX_WINDOW)); - benchmarkDurationMs = Math.max(1_000, durationMs); - benchmarkLinkMode = linkMode & 3; - benchmarkLinkPending = true; - benchmarkReadyAtMs = Long.MAX_VALUE; - benchmarkStartAtMs = 0; - benchmarkDeadlineAtMs = Long.MAX_VALUE; - benchmarkLastAckAtMs = 0; - benchmarkEndAtMs = 0; - benchmarkMessagesSent = 0; - benchmarkMessagesAcked = 0; - benchmarkTimeouts = 0; - benchmarkPayloadBytesAcked = 0; - benchmarkWireBytesAcked = 0; - benchmarkAborted = false; - benchmarkActive = true; - logLine("bandwidth benchmark start: size=" + benchmarkMessageSize - + "B window=" + benchmarkWindowSize + " duration=" + benchmarkDurationMs - + "ms linkMode=" + benchmarkLinkMode); - } - interruptibleSleep.interrupt(); - return true; - } - - /** - * Cancel an in-progress benchmark (benchmark page closed). Queued no-op - * messages are dropped; in-flight ones drain through their normal acks. - */ - public void cancelBandwidthBenchmark() { - synchronized (lock) { - if (!benchmarkActive) { - return; - } - clearMessagesOfKindLocked("bandwidth"); - finishBenchmarkLocked(true, "cancelled"); - } - interruptibleSleep.interrupt(); - } - - /** Status/results of the current or most recent benchmark run, as JSON. */ - public String getBandwidthBenchmarkStatus() { - synchronized (lock) { - String state = benchmarkActive - ? (benchmarkStartAtMs == 0 ? "starting" : "running") - : (benchmarkEndAtMs != 0 ? "done" : "idle"); - long end = benchmarkActive ? SystemClock.elapsedRealtime() : benchmarkEndAtMs; - long elapsed = benchmarkStartAtMs == 0 ? 0 : Math.max(0, end - benchmarkStartAtMs); - try { - org.json.JSONObject status = new org.json.JSONObject(); - status.put("state", state); - status.put("messageSize", benchmarkMessageSize); - status.put("windowSize", benchmarkWindowSize); - status.put("linkMode", benchmarkLinkMode); - status.put("elapsedMs", elapsed); - status.put("messagesSent", benchmarkMessagesSent); - status.put("messagesAcked", benchmarkMessagesAcked); - status.put("timeouts", benchmarkTimeouts); - status.put("payloadBytesAcked", benchmarkPayloadBytesAcked); - status.put("wireBytesAcked", benchmarkWireBytesAcked); - status.put("aborted", benchmarkAborted); - return status.toString(); - } catch (org.json.JSONException e) { - return "{\"state\":\"idle\"}"; - } - } - } - - /** Pending + in-flight benchmark no-op messages. */ - private int benchmarkOutstandingLocked() { - int count = 0; - for (OutboundMessage message : pendingMessages) { - if ("bandwidth".equals(message.kind)) count++; - } - for (OutboundMessage message : inFlightMessages) { - if ("bandwidth".equals(message.kind)) count++; - } - return count; - } - - /** - * Keep the benchmark stream fed: top the pending queue up so the send - * window never starves, stop enqueueing once the run expires (or a message - * times out), and finish the run when the last outstanding message drains. - */ - private void maintainBenchmarkLocked(long now) { - if (!sessionReady || !fixedLayoutCreated || shutdownRequested) { - finishBenchmarkLocked(true, "session no longer ready"); - return; - } - if (benchmarkLinkPending || now < benchmarkReadyAtMs) { - return; - } - if (benchmarkAborted || now >= benchmarkDeadlineAtMs) { - // The run is over: drop queued-but-unsent no-ops (sending them - // would stretch the run past its deadline) and finish once the - // in-flight tail has acked or timed out. - Iterator pendingIterator = pendingMessages.iterator(); - while (pendingIterator.hasNext()) { - OutboundMessage queued = pendingIterator.next(); - if ("bandwidth".equals(queued.kind)) { - pendingIterator.remove(); - magicPool.release(queued.sid, queued.magic, queued.label, "benchmark over"); - } - } - if (benchmarkOutstandingLocked() == 0) { - finishBenchmarkLocked(false, "complete"); - } - return; - } - // One more than the window so a fresh message is always ready to write - // the moment an ack frees a slot. - for (int outstanding = benchmarkOutstandingLocked(); outstanding <= benchmarkWindowSize; outstanding++) { - enqueueBenchmarkMessageLocked(); - } - } - - private void finishBenchmarkLocked(boolean aborted, String reason) { - if (!benchmarkActive) { - return; - } - benchmarkActive = false; - benchmarkAborted |= aborted; - // Prefer the last ack as the end time so drain lag after the deadline - // doesn't dilute the throughput figure. - benchmarkEndAtMs = benchmarkLastAckAtMs != 0 ? benchmarkLastAckAtMs : SystemClock.elapsedRealtime(); - logLine("bandwidth benchmark " + (benchmarkAborted ? "aborted" : "finished") + " (" + reason + "): " - + benchmarkMessagesAcked + "/" + benchmarkMessagesSent + " acked, " - + benchmarkPayloadBytesAcked + "B payload, " + benchmarkTimeouts + " timeouts"); - } - - private void enqueueBenchmarkMessageLocked() { - // Fresh bytes per message: the transport's persistent compression history - // would compress even a random payload if we reused it across the run. - byte[] payload = new byte[benchmarkMessageSize]; - benchmarkRandom.nextBytes(payload); - payload[0] = CfwMessageTypeKt.CFW_MSG_DIAGNOSTICS; // CFW diagnostic-control mode... - payload[1] = (byte) 0x7f; // ...with an unused sub-op: acked, no effect - OutboundMessage message = messageBuilder.imagePayload( - "bandwidth", - DASHBOARD_TILE, - nextMapSessionId(), - payload, - "bandwidth no-op " + payload.length + "B", - connectionOptions.sendImagesToLeft); - final int payloadBytes = payload.length; - // Logical custom-message bytes; excludes length tag and packet framing. - final int wireBytes = message.message.length; - message.onSent = () -> { - benchmarkMessagesSent++; - if (benchmarkStartAtMs == 0) { - benchmarkStartAtMs = SystemClock.elapsedRealtime(); - benchmarkDeadlineAtMs = benchmarkStartAtMs + benchmarkDurationMs; - } - }; - message.onAck = () -> { - long ackedAtMs = SystemClock.elapsedRealtime(); - benchmarkMessagesAcked++; - benchmarkPayloadBytesAcked += payloadBytes; - benchmarkWireBytesAcked += wireBytes; - benchmarkLastAckAtMs = ackedAtMs; - // No-op payloads ride the image path, so the firmware resets its - // heartbeat timer on them just like real image messages. - lastHeartbeatAckedAtMs = ackedAtMs; - }; - message.onTimeout = () -> { - benchmarkTimeouts++; - benchmarkAborted = true; - logLine("bandwidth benchmark ack timeout"); - }; - pendingMessages.addLast(message); - } - - public void addImuListener(FaceclawImuListener listener) { - if (listener != null) { - imuListeners.add(listener); - } - } - - public void removeImuListener(FaceclawImuListener listener) { - if (listener != null) { - imuListeners.remove(listener); - } - } - - public void addAmbientLightListener(FaceclawAmbientLightListener listener) { - if (listener != null) { - ambientLightListeners.add(listener); - } - } - - public void removeAmbientLightListener(FaceclawAmbientLightListener listener) { - if (listener != null) { - ambientLightListeners.remove(listener); - } - } - - private void emitAmbientLight(byte[] body) { - if (ambientLightListeners.isEmpty()) { - return; - } - byte[] copy = java.util.Arrays.copyOf(body, body.length); - mainHandler.post(() -> { - for (FaceclawAmbientLightListener alsListener : ambientLightListeners) { - try { - alsListener.onAmbientLight(copy); - } catch (Throwable t) { - Log.w(TAG, "ambient light listener failed", t); - } - } - }); - } - - /** Request one CFW ambient-light report (image-handler mode 16 op 0). */ - public void queryAmbientLight() { - synchronized (lock) { - enqueueAmbientLightControlLocked(new byte[] { (byte) CfwMessageTypeKt.CFW_MSG_AMBIENT_LIGHT, (byte) 0 }, "als query", false); - } - interruptibleSleep.interrupt(); - } - - /** - * Start or stop CFW passive light-sensor polling (image-handler mode 16 - * ops 1/2). While polling, the firmware reads its OPT3001 every intervalMs - * (clamped 100..5000 by the firmware), never steps the panel brightness - * itself, and pushes a field-105 report when the reading moved by at least - * minDelta or heartbeatMs elapsed. bindToLease stops polling automatically - * when the Faceclaw framebuffer lease is released or lapses. Starting is - * idempotent and re-opens the sensor if the stock firmware closed it. - */ - public void setAmbientLightPolling(boolean enable, int intervalMs, int minDelta, - int heartbeatMs, boolean bindToLease) { - byte[] payload = enable - ? new byte[] { - (byte) CfwMessageTypeKt.CFW_MSG_AMBIENT_LIGHT, - (byte) 1, - (byte) (bindToLease ? 1 : 0), - (byte) (intervalMs & 0xff), - (byte) ((intervalMs >> 8) & 0xff), - (byte) (minDelta & 0xff), - (byte) ((minDelta >> 8) & 0xff), - (byte) (heartbeatMs & 0xff), - (byte) ((heartbeatMs >> 8) & 0xff), - } - : new byte[] { (byte) CfwMessageTypeKt.CFW_MSG_AMBIENT_LIGHT, (byte) 2 }; - synchronized (lock) { - clearMessagesOfKindLocked("als-control"); - enqueueAmbientLightControlLocked(payload, "als polling " + (enable ? "start" : "stop"), true); - } - interruptibleSleep.interrupt(); - } - - private void enqueueAmbientLightControlLocked(byte[] payload, String label, boolean priority) { - if (!running || !sessionReady || shutdownRequested || !fixedLayoutCreated) { - logLine("skip " + label + "; display path not ready"); - return; - } - OutboundMessage message = messageBuilder.imagePayload( - "als-control", - DASHBOARD_TILE, - nextMapSessionId(), - payload, - label, - connectionOptions.sendImagesToLeft); - message.onTimeout = () -> logLine(label + " ack timeout"); - if (priority) pendingMessages.addFirst(message); - else pendingMessages.addLast(message); - logLine("queue " + label); - } - - public void addMicStatusListener(FaceclawMicStatusListener listener) { - if (listener != null) { - micStatusListeners.add(listener); - } - } - - public void removeMicStatusListener(FaceclawMicStatusListener listener) { - if (listener != null) { - micStatusListeners.remove(listener); - } - } - - private void emitMicStatus(byte[] body, String address) { - if (micStatusListeners.isEmpty()) { - return; - } - String arm = address.equalsIgnoreCase(leftAddress) ? "L" - : address.equalsIgnoreCase(rightAddress) ? "R" : "?"; - byte[] copy = java.util.Arrays.copyOf(body, body.length); - mainHandler.post(() -> { - for (FaceclawMicStatusListener micListener : micStatusListeners) { - try { - micListener.onMicStatus(copy, arm); - } catch (Throwable t) { - Log.w(TAG, "mic status listener failed", t); - } - } - }); - } - - /** - * Queue a CFW mic_control record (['M','C',ver,op,...]) as a settings - * field-103 write to both temples, or to a single one. Fire-and-forget: - * the firmware answers with a field-104 status notify per temple, which - * arrives through addMicStatusListener. - */ - public void sendFaceclawMicControl(byte[] record, String label, boolean rightTemple, boolean leftTemple) { - if (record == null || record.length < 4) { - return; - } - synchronized (lock) { - if (!running || !sessionReady) { - logLine("skip mic control (" + label + "); session not ready"); - return; - } - if (rightTemple) { - pendingMessages.addLast(messageBuilder.faceclawMicControl(record, label, false)); - } - if (leftTemple) { - pendingMessages.addLast(messageBuilder.faceclawMicControl(record, label, true)); - } - logLine("queue mic control " + label); - } - interruptibleSleep.interrupt(); - } - - /** - * Forward render-characteristic audio packets to the listener WITHOUT - * sending the stock EvenHub audio-control enable. Used for the CFW - * mic_control streaming path, where capture is armed through settings - * field 103 and the temples emit 'SM' frames on the same characteristic - * that stock mono LC3 uses. Returns false when no session is up. - */ - public boolean startG2AudioForwarding(FaceclawAudioPacketListener listener) { - if (listener == null) { - throw new IllegalArgumentException("listener is required"); - } - synchronized (lock) { - if (!running || !sessionReady || shutdownRequested) { - logLine("skip G2 audio forwarding; session not ready"); - return false; - } - audioPacketListener = listener; - audioCaptureActive = true; - logLine("G2 audio forwarding enabled"); - } - return true; - } - - public void stopG2AudioForwarding() { - synchronized (lock) { - audioPacketListener = null; - audioCaptureActive = false; - logLine("G2 audio forwarding disabled"); - } - } - - /** - * Subscribe to compass events. Callbacks are delivered on the Looper of - * the thread that registered (falling back to the main thread), so app - * worker isolates can listen without a cross-thread hop into their JS. - */ - public void addCompassListener(FaceclawCompassListener listener) { - if (listener == null) { - return; - } - Looper looper = Looper.myLooper(); - Handler handler = looper != null ? new Handler(looper) : mainHandler; - compassSubscriptions.add(new CompassSubscription(listener, handler)); - } - - public void removeCompassListener(FaceclawCompassListener listener) { - if (listener == null) { - return; - } - for (CompassSubscription subscription : compassSubscriptions) { - if (subscription.listener == listener) { - compassSubscriptions.remove(subscription); - } - } - } - - - // The most recently started communicator; lets app worker threads submit - // surface frames without holding a cross-isolate reference to the bridge - // object (JS wrappers do not cross isolates, but the Java instance does). - private static volatile FaceclawBleCommunicator activeInstance; - - public static FaceclawBleCommunicator getActive() { - return activeInstance; - } - - /** Set the compositor's output frame size. Call before configuring surfaces. */ - public void configureCompositorScreen(int width, int height) { - compositor.configureScreen(width, height); - } - - /** - * The current composited screen as a phone-UI preview bitmap, or null - * before any surface has been configured. Built from the compositor so - * the preview reflects every surface (chrome + whichever app is - * foreground), including worker-app frames the TS side never sees. - */ - public android.graphics.Bitmap getCompositePreviewBitmap(double brightenGamma) { - return getCompositePreviewBitmap(brightenGamma, false); - } - - /** As above; `green` renders the preview green-on-black (Settings > Phone display > Preview color). */ - public android.graphics.Bitmap getCompositePreviewBitmap(double brightenGamma, boolean green) { - SurfaceCompositor.Composite composite = compositor.previewComposite(); - if (composite == null) { - return null; - } - return PreviewBitmapUtil.fromGray( - java.nio.ByteBuffer.wrap(composite.gray), composite.width, composite.height, brightenGamma, green); - } - - /** Save the current composite as a 4-bit grayscale PNG; returns the path or "". */ - public String saveCompositePngScreenshot() throws java.io.IOException { - SurfaceCompositor.Composite composite = compositor.previewComposite(); - if (composite == null) { - return ""; - } - return ScreenshotUtil.savePngScreenshot(appContext, composite.gray, composite.width, composite.height); - } - - /** - * Save the current composite cropped to the given screen rect (the region - * the shell says is actually occupied). The rect is clamped to the screen; - * a degenerate rect falls back to the full screen. - */ - public String saveCompositePngScreenshot(int cropX, int cropY, int cropWidth, int cropHeight) - throws java.io.IOException { - SurfaceCompositor.Composite composite = compositor.previewComposite(); - if (composite == null) { - return ""; - } - int x = Math.max(0, cropX); - int y = Math.max(0, cropY); - int width = Math.min(composite.width - x, cropWidth - (x - cropX)); - int height = Math.min(composite.height - y, cropHeight - (y - cropY)); - if (width <= 0 || height <= 0 || (x == 0 && y == 0 && width == composite.width && height == composite.height)) { - return ScreenshotUtil.savePngScreenshot(appContext, composite.gray, composite.width, composite.height); - } - byte[] cropped = new byte[width * height]; - for (int row = 0; row < height; row++) { - System.arraycopy(composite.gray, (y + row) * composite.width + x, cropped, row * width, width); - } - return ScreenshotUtil.savePngScreenshot(appContext, cropped, width, height); - } - - // Active animated-GIF screen recording, or null when idle. Frames are - // pushed by recordScreenFrame(), which the TS side calls at each - // phone-preview flush. - private volatile GifScreenRecorder screenRecorder; - - /** Begin collecting composite frames for an animated-GIF screen recording. */ - public void startScreenRecording() { - screenRecorder = new GifScreenRecorder(); - } - - /** Capture the current composite into the active recording; no-op when idle. */ - public void recordScreenFrame() { - GifScreenRecorder recorder = screenRecorder; - if (recorder == null) { - return; - } - SurfaceCompositor.Composite composite = compositor.previewComposite(); - if (composite == null) { - return; - } - recorder.addFrame(composite.gray, composite.width, composite.height, System.currentTimeMillis()); - } - - /** Finish the recording and save it as an animated GIF; returns the path or "". */ - public String stopScreenRecording() throws java.io.IOException { - GifScreenRecorder recorder = screenRecorder; - screenRecorder = null; - if (recorder == null) { - return ""; - } - if (recorder.isOverflowed()) { - logLine("screen recording hit its frame cap; the tail was dropped"); - } - return recorder.save(appContext); - } - - /** - * Show or hide a compositor surface, immediately submitting the resulting - * frame. Recompositing here (rather than waiting for the next surface - * update) is what makes a just-foregrounded window's retained frame - * actually appear — otherwise a static window (e.g. the terminal hub) whose - * frame landed while briefly hidden would stay blank until its next repaint. - */ - public void setSurfaceVisible(String id, boolean visible) { - // Its own frame: this recomposite is a real screen update with real - // latency, and without one it would show up in other frames' logs only - // as an anonymous "superseded by frame#0". - int frameId = FrameTimings.getInstance().startFrame( - "compositor:visible " + id + "=" + visible); - compositor.setSurfaceVisible(id, visible); - SurfaceCompositor.Composite composite = compositor.composite(); - byte[] packed = BmpUtil.pack4bppFromGray8(composite.screenGray, composite.width, composite.height); - storeDesiredComposite(composite, packed, 0, frameId); - } - - /** - * Blank (screen off) or unblank the composited output, immediately - * submitting the resulting frame. Retained surface state is untouched, so - * unblanking restores the previous screen content without repaints. - */ - public void setScreenBlanked(boolean blanked) { - int frameId = FrameTimings.getInstance().startFrame( - "compositor:" + (blanked ? "blank" : "unblank")); - compositor.setBlanked(blanked); - SurfaceCompositor.Composite composite = compositor.composite(); - byte[] packed = BmpUtil.pack4bppFromGray8(composite.screenGray, composite.width, composite.height); - storeDesiredComposite(composite, packed, 0, frameId); - } - - /** - * Create or reconfigure a compositor surface. transparency is one of the - * SurfaceCompositor.TRANSPARENCY_* constants. Geometry changes take effect - * when the next frame is submitted. - */ - public void configureSurface(String id, int x, int y, int width, int height, int zOrder, int transparency) { - compositor.configureSurface(id, x, y, width, height, zOrder, transparency); - } - - public void removeSurface(String id) { - compositor.removeSurface(id); - } - - /** Stage an immutable shell snapshot alongside the retained app screen. */ - public void submitShellScene(java.nio.ByteBuffer bytes, int paintMs, int frameId) { - compositor.setShellScene(new AndroidByteReader(bytes)); - SurfaceCompositor.Composite composite = compositor.composite(); - storeDesiredComposite(composite, BmpUtil.pack4bppFromGray8(composite.screenGray, composite.width, composite.height), paintMs, frameId); - } - - /** Legacy compositor dimming for callers without a shell scene. */ - public void setUnderlayDim(int belowZOrder, int factor256) { - compositor.setUnderlayDim(belowZOrder, factor256); - } - - /** - * Apply an update to one compositor surface and submit the recomposited - * screen as the desired frame. The update covers the rect (rectX, rectY, - * rectWidth, rectHeight) in surface-local coordinates; contentFingerprint - * identifies the surface's full content after the update. - * - * pixels8bpp arrives as a ByteBuffer because NativeScript marshals a JS - * ArrayBuffer to one without the per-element bridge copy that a byte[] - * parameter would need (~150ms for a full frame). - */ - public void submitSurfaceFrame( - java.nio.ByteBuffer pixels8bpp, - String surfaceId, - int rectX, - int rectY, - int rectWidth, - int rectHeight, - String contentFingerprint, - int paintMs, - int frameId - ) { - submitSurfaceFrame(pixels8bpp, surfaceId, rectX, rectY, rectWidth, rectHeight, - contentFingerprint, paintMs, frameId, null); - } - - /** - * As above, with the frame's glyph draws (see SurfaceCompositor's glyph - * overload for the buffer format): the surface's full text content as - * structured draws, letting the resource-cache planner ship glyphs as - * on-glasses cached draws instead of pixels. Null when the submitter has - * no glyph metadata; the pixels alone remain fully correct. - */ - public void submitSurfaceFrame( - java.nio.ByteBuffer pixels8bpp, - String surfaceId, - int rectX, - int rectY, - int rectWidth, - int rectHeight, - String contentFingerprint, - int paintMs, - int frameId, - java.nio.ByteBuffer glyphs - ) { - Log.i(TAG, "Received an updated frame for surface " + surfaceId); - FrameTimings.getInstance().log(frameId, "surface " + surfaceId + " updated rect=" - + rectWidth + "x" + rectHeight + "+" + rectX + "+" + rectY - + (glyphs == null ? " (no glyph draws)" : "")); - FrameTimings.getInstance().spanStart(frameId, "composite"); - SurfaceCompositor.Composite composite = compositor.applyAndComposite( - surfaceId, new AndroidByteReader(pixels8bpp), rectX, rectY, rectWidth, rectHeight, contentFingerprint, - glyphs == null ? null : new AndroidByteReader(glyphs)); - FrameTimings.getInstance().spanEnd(frameId, "composite"); - // Pack the composited 8bpp buffer down to the headerless 4bpp frame - // format the wire planners consume; BMP framing is added later only for - // the uncompressed fallback. - FrameTimings.getInstance().spanStart(frameId, "pack-4bpp"); - byte[] packed = BmpUtil.pack4bppFromGray8(composite.screenGray, composite.width, composite.height); - FrameTimings.getInstance().spanEnd(frameId, "pack-4bpp"); - storeDesiredComposite(composite, packed, paintMs, frameId); - } - - /** Store a composite as the desired frame unless a newer one won the race. */ - private void storeDesiredComposite(SurfaceCompositor.Composite composite, byte[] packed, int paintMs, int frameId) { - int supersededFrameId = 0; - boolean stale = false; - synchronized (desiredTilesLock) { - if (composite.seq <= lastStoredCompositeSeq) { - // A concurrent submission composited after us and stored first; - // its composite already includes this surface update. - stale = true; - } else { - lastStoredCompositeSeq = composite.seq; - supersededFrameId = desiredFrameId; - desiredPacked = packed; - desiredWidth = composite.width; - desiredHeight = composite.height; - desiredFingerprint = composite.fingerprint; - desiredPaintMs = paintMs; - desiredFrameId = frameId; - desiredDraws = composite.draws; - desiredShellScene = composite.shellScene; - } - } - if (stale) { - finishFrame(frameId, "discarded: composite superseded before store"); - return; - } - if (supersededFrameId != 0 && supersededFrameId != frameId) { - finishFrame(supersededFrameId, "discarded: superseded by frame#" + frameId + " before send"); - } - FrameTimings.getInstance().log(frameId, "image submitted as desired frame"); - interruptibleSleep.interrupt(); - } - - /** - * Play a tone sequence via CFW load_image_z mode 5 kind 4. The payload is - * the complete wire buffer ([5][4][nSteps][freqLo,freqHi,duty,msLo,msHi]*n, - * up to 48 steps), built on the TS side; it rides the arbitrary-payload - * image path like the other mode-5 controls. - */ - public void playBuzzerSequence(java.nio.ByteBuffer payload) { - synchronized (lock) { - if (!running || !sessionReady || !fixedLayoutCreated) { - logLine("skip buzzer sequence; session not ready"); - return; - } - byte[] bytes = new byte[payload == null ? 0 : payload.remaining()]; - if (payload != null) { - payload.get(bytes); - } - if (bytes.length < 3) { - logLine("skip buzzer sequence; empty payload"); - return; - } - OutboundMessage message = messageBuilder.imagePayload( - DASHBOARD_TILE, - nextMapSessionId(), - bytes, - "buzzer sequence " + bytes.length + "B", - connectionOptions.sendImagesToLeft - ); - message.onTimeout = () -> { - handleTransportFailure("buzzer sequence ack timeout"); - }; - pendingMessages.addLast(message); - logLine("queue " + message.label); - } - interruptibleSleep.interrupt(); - } - - public boolean sendShutdown(int exitMode) { - return sendShutdownInternal(exitMode, true); - } - - /** - * Send CFW image-handler mode 11 after quiescing normal traffic. A successful - * return means the cleanup was ACKed and no later Faceclaw message should be - * emitted before closing BLE. Unsupported/older CFWs return false so callers - * can use the legacy shutdown-and-lease-release path. - */ - public boolean sendCfwCleanup() { - int magic; - synchronized (lock) { - if (cfwCleanupDelivered) { - return true; - } - if (!customFirmwareDetected || !running || !sessionReady - || shutdownRequested || !fixedLayoutCreated) { - logLine("skip CFW cleanup; mode 11 unavailable or image path not ready"); - return false; - } - - /* Stop auto-renewals, heartbeats, image generation, and control - * retries, then discard everything that has not reached BLE yet. */ - shutdownRequested = true; - lastCfwCleanupAckMagic = 0; - clearPendingMessagesLocked("CFW cleanup requested"); - logLine("quiescing transport for CFW cleanup"); - } - interruptibleSleep.interrupt(); - - /* WINDOW_SIZE can exceed one, so merely appending cleanup would allow it - * to overlap previously-written image fragments. Wait until all of those - * ACK or time out; shutdownRequested prevents the drive loop from adding - * any fresh automatic traffic meanwhile. */ - long drainDeadline = SystemClock.elapsedRealtime() + CFW_CLEANUP_WAIT_MS; - synchronized (lock) { - while (running && sessionReady && !inFlightMessages.isEmpty()) { - long remaining = drainDeadline - SystemClock.elapsedRealtime(); - if (remaining <= 0) break; - try { - lock.wait(Math.min(remaining, 100)); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - break; - } - } - if (!running || !sessionReady || !inFlightMessages.isEmpty()) { - shutdownRequested = false; - logLine("CFW cleanup could not drain prior traffic"); - return false; - } - - /* External producers are expected to be stopped by the caller, but - * clear once more at the barrier so cleanup is definitely last. */ - clearPendingMessagesLocked("CFW cleanup barrier"); - OutboundMessage message = messageBuilder.cfwCleanup( - DASHBOARD_TILE, - nextMapSessionId(), - connectionOptions.sendImagesToLeft - ); - magic = message.magic; - message.onAck = () -> { - lastCfwCleanupAckMagic = message.magic; - cfwCleanupDelivered = true; - compassMaybeOn = false; - faceclawWakeLeaseEnabled = false; - logLine("CFW cleanup completed"); - }; - message.onTimeout = () -> logLine("CFW cleanup ack timeout"); - pendingMessages.addLast(message); - logLine("queue CFW cleanup"); - } - interruptibleSleep.interrupt(); - - long deadline = SystemClock.elapsedRealtime() + CFW_CLEANUP_WAIT_MS; - synchronized (lock) { - while (running - && sessionReady - && lastCfwCleanupAckMagic != magic - && hasPendingOrInflightMagicLocked(magic)) { - long remaining = deadline - SystemClock.elapsedRealtime(); - if (remaining <= 0) break; - try { - lock.wait(Math.min(remaining, 100)); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - break; - } - } - boolean acked = lastCfwCleanupAckMagic == magic; - if (!acked) shutdownRequested = false; - return acked; - } - } - - /** - * End the EvenHub page while retaining both arm GATT connections and all - * notification subscriptions. A missed ACK is intentionally non-fatal: - * reconnecting Bluetooth here would defeat the power-saving mode. - */ - public boolean suspendEvenHubSession() { - // Ending the plugin task releases the fb lease, which frees the - // on-glasses resource cache; forget it phone-side either way (a lost - // ack may still have taken effect). - synchronized (lock) { - resourceCache.reset(); - } - if (sendShutdownInternal(0, false)) { - return true; - } - synchronized (lock) { - // A shutdown ACK can be lost even though the command took effect. - // If the transport remains intentionally quiesced, callers still - // need to remember to run the resume path on the next wake. - return running && sessionReady && shutdownRequested; - } - } - - /** - * Start a fresh EvenHub plugin task on the existing BLE transport, then let - * the session driver create the layout, warm up the image path, and send - * the desired frame. - */ - public boolean resumeEvenHubSession() { - int claimGeneration = 0; - synchronized (lock) { - if (!running || !sessionReady || chargingMode) { - logLine("skip EvenHub resume; transport not ready"); - return false; - } - if (!shutdownRequested) { - return true; - } - if (faceclawWakePendingNonce >= 0 - && hasPendingOrInflightKindLocked("wake-lease-control")) { - claimGeneration = faceclawWakeControlGeneration; - } - logLine("replaying session prelude for EvenHub resume"); - } - - // A custom double-tap wake has only a short unclaimed fail-open - // deadline. Let the worker put CLAIM on both arms before the direct - // prelude write begins. - if (claimGeneration != 0) { - waitForFaceclawWakeControlDelivery(claimGeneration, 500); - } - - try { - // Empty-name Cmd=9 tears down the whole plugin task, not just its - // image container. Re-run the launch prelude before Cmd=0 CREATE. - sendPrelude(true); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - handleTransportFailure("EvenHub resume prelude interrupted"); - return false; - } catch (Throwable t) { - logLine("EvenHub resume prelude failed: " + safeMessage(t)); - handleTransportFailure("EvenHub resume prelude failed"); - return false; - } - - synchronized (lock) { - if (!running || !sessionReady || chargingMode) { - return false; - } - shutdownRequested = false; - fixedLayoutCreated = false; - startupProbePending = false; - audioCaptureActive = false; - clearAllMessagesPreservingWakeLeaseLocked("EvenHub resume"); - displayedFingerprint = ""; - imageRetryAfterMs = 0; - lastHeartbeatSentAtMs = 0; - lastHeartbeatAckedAtMs = 0; - logLine("EvenHub session resume requested"); - } - interruptibleSleep.interrupt(); - return true; - } - - private boolean sendShutdownInternal(int exitMode, boolean reconnectOnTimeout) { - int magic; - long startedAtMs = SystemClock.elapsedRealtime(); - synchronized (lock) { - if (!running || !sessionReady) { - logLine("skip shutdown; session not ready"); - return false; - } - if (shutdownRequested) { - return true; - } - shutdownRequested = true; - // Magic values wrap, so an ACK from a much older suspend must not - // satisfy this request after enough sleep/wake cycles. - lastShutdownAckMagic = 0; - clearPendingMessagesLocked("shutdown requested"); - OutboundMessage message = messageBuilder.shutdown(exitMode); - magic = message.magic; - message.onAck = () -> { - lastShutdownAckMagic = message.magic; - fixedLayoutCreated = false; - displayedFingerprint = ""; - }; - message.onTimeout = () -> { - if (reconnectOnTimeout) { - handleTransportFailure("shutdown ack timeout"); - } else { - logLine("EvenHub shutdown ack timeout; keeping BLE connected"); - } - }; - pendingMessages.addFirst(message); - logLine("queue shutdown"); - // The stock compass keeps the magnetometer sampling independently of - // the plugin task, so ending the page does not stop it. Force a - // disable ahead of the shutdown command whenever it may be running: - // this also covers a disable that was wiped by the queue flush above - // or whose ack was lost, and the charging-mode/exit paths where the - // Compass window never got a chance to release it. - if (compassMaybeOn && fixedLayoutCreated) { - enqueueCompassControlLocked(true, false); - } - } - interruptibleSleep.interrupt(); - - long ackDeadline = SystemClock.elapsedRealtime() + ConnectionOptions.ACK_TIMEOUT_MS + 500; - synchronized (lock) { - while (running - && sessionReady - && lastShutdownAckMagic != magic - && lastShutdownExitAtMs < startedAtMs - && hasPendingOrInflightMagicLocked(magic)) { - long remaining = ackDeadline - SystemClock.elapsedRealtime(); - if (remaining <= 0) { - break; - } - try { - lock.wait(Math.min(remaining, 100)); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - break; - } - } - boolean acked = lastShutdownAckMagic == magic; - if (acked) { - long exitDeadline = SystemClock.elapsedRealtime() + ConnectionOptions.ACK_TIMEOUT_MS + 500; - while (running && sessionReady && lastShutdownExitAtMs < startedAtMs) { - long remaining = exitDeadline - SystemClock.elapsedRealtime(); - if (remaining <= 0) { - break; - } - try { - lock.wait(Math.min(remaining, 100)); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - break; - } - } - } - return acked || lastShutdownExitAtMs >= startedAtMs; - } - } - - private boolean waitForAudioControlAck(int magic, String operation) { - long deadline = SystemClock.elapsedRealtime() + ConnectionOptions.ACK_TIMEOUT_MS + ConnectionOptions.WRITE_TIMEOUT_MS + 500; - synchronized (lock) { - while (running && sessionReady && lastAudioControlAckMagic != magic && hasPendingOrInflightMagicLocked(magic)) { - long remaining = deadline - SystemClock.elapsedRealtime(); - if (remaining <= 0) { - break; - } - try { - lock.wait(Math.min(remaining, 100)); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - break; - } - } - boolean acked = lastAudioControlAckMagic == magic; - if (!acked) { - if ("enable".equals(operation)) { - audioPacketListener = null; - audioCaptureActive = false; - } - logLine("G2 mic " + operation + " ack timeout"); - } - return acked; - } - } - - - @Override public void run() { - logLine(String.format(Locale.US, "communicator start R=%s L=%s ring=%s", rightAddress, leftAddress, ringAddress)); - while (true) { - try { - if (!running) { - Log.w(TAG, "Exiting event looop"); - break; - } - emitPhoneLockStateIfChanged(false); - if (!sessionReady) { - if (reconnectHalted) { - interruptibleSleep.sleep(ConnectionOptions.IDLE_SLEEP_MS); - continue; - } - long now = SystemClock.elapsedRealtime(); - if (now < reconnectAfterMs) { - interruptibleSleep.sleep(Math.min(ConnectionOptions.IDLE_SLEEP_MS, reconnectAfterMs - now)); - continue; - } - Log.w(TAG, "Attempting to connect"); - connectLoopOnce(); - continue; - } - - if (shouldAttemptRingConnect()) { - tryConnectRing("retry"); - continue; - } - - long sleepMs = driveSession(); - if (sleepMs > 0) { - interruptibleSleep.sleep(sleepMs); - } - } catch (Throwable t) { - logLine("communicator loop error: " + safeMessage(t)); - handleTransportFailure("loop error"); - } - } - logLine("communicator stop"); - } - - @Override public void onNotification(String address, String characteristicUuid, byte[] data) { - if (address == null || characteristicUuid == null || data == null) { - return; - } - String uuid = characteristicUuid.toLowerCase(Locale.US); - if (isDirectRingNotification(address, uuid)) { - handleDirectRingNotification(uuid, data); - return; - } - if (BleProtocol.RENDER_NOTIFY_UUID.equals(uuid)) { - handleRenderNotification(address, data); - return; - } - if (!BleProtocol.NOTIFY_CHAR_UUID.equals(uuid)) { - return; - } - if (data.length >= 7 && (data[6] & 255) == CfwTransport.SID) { - CfwTransport.Ack[] acks = CfwTransport.parseAcks(data); - if (acks == null) return; - synchronized (lock) { - lastIncomingAtMs = SystemClock.elapsedRealtime(); - for (CfwTransport.Ack ack : acks) { - for (OutboundMessage message : inFlightMessages) { - String ingress = message.isLeftArmMessage ? leftAddress : rightAddress; - if (message.sid == CfwTransport.SID && address.equalsIgnoreCase(ingress) && message.magic == ack.streamId) { - message.acceptCfwAck(ack); - Log.i(TAG, "CFW " + (ack.nack ? "NACK" : "ACK") + " id=" + ack.streamId - + " ordinal=" + ack.messageId + " lens=" + ack.lens + " txseq=" + (data[2] & 255) - + " redundant=" + (ack != acks[0]) - + " size=" + ack.size + " crc=" + ack.checksum - + " expected=" + message.message.length + "/" + message.cfwChecksum - + " ackedLenses=" + message.cfwAckLenses - + " ageMs=" + (lastIncomingAtMs - message.sentAtMs)); - message.ackPayload = Arrays.copyOf(data, data.length); - break; - } - } - } - drainCfwAcknowledgementsLocked(); - } - interruptibleSleep.interrupt(); - return; - } - Log.d(TAG, "onNotification: address=" + address + " characteristicUuid=" + characteristicUuid + " data.length=" + data.length); - BleProtocol.ParsedFrame frame = BleProtocol.parseFrame(data); - int decodedWearState = BleProtocol.parseWearState(frame); - BleProtocol.CompassEvent compassEvent = address.equalsIgnoreCase(rightAddress) - ? BleProtocol.parseCompassEvent(frame) - : null; - boolean emitWearState = false; - G2Event event = null; - synchronized (lock) { - lastIncomingAtMs = SystemClock.elapsedRealtime(); - if (decodedWearState >= 0 && decodedWearState != wearState) { - wearState = decodedWearState; - emitWearState = true; - } - boolean faceclawWakeNotification = false; - if (shutdownRequested - && frame.ok - && frame.sid == BleProtocol.SID_UI_SETTING - && address.equalsIgnoreCase(rightAddress)) { - int wakeNonce = BleProtocol.parseFaceclawWakeEvent(frame.pb); - if (wakeNonce >= 0) { - faceclawWakePendingNonce = wakeNonce; - enqueueFaceclawWakeControlLocked( - BleProtocol.FACECLAW_WAKE_OP_CLAIM, - wakeNonce, - true - ); - faceclawWakeNotification = true; - lastConnectionOrInputAtMs = lastIncomingAtMs; - // The event type says which gesture woke the glasses: TS - // gives a head-up the Glanceboard and a double tap the - // regular UI. The claim handshake is the same for both. - boolean headUp = BleProtocol.parseFaceclawWakeEventCode(frame.pb) - == BleProtocol.FACECLAW_WAKE_EVENT_HEAD_UP; - event = new G2Event( - "display-wake", - "", - headUp ? BleProtocol.EVENT_HEAD_UP : BleProtocol.EVENT_DOUBLE_CLICK, - 0, - 0 - ); - logLine("claimed deferred dashboard wake nonce=" + wakeNonce - + (headUp ? " (head-up)" : "")); - } - } - if (!faceclawWakeNotification - && shutdownRequested - && frame.ok - && frame.sid == BleProtocol.SID_UI_SETTING - && address.equalsIgnoreCase(rightAddress)) { - // CFW idle-gesture forwarding (firmware revision 2+): with no - // EvenHub page the stock display thread drops taps, long - // presses and releases; the CFW reports them on the settings - // sid while our wake lease is held. Deliver them as the - // sys-events a live page would have produced, so TS treats a - // sleep-time tap or hold exactly like one during soft sleep - // (the Glanceboard). - BleProtocol.FaceclawGestureEvent gesture = BleProtocol.parseFaceclawGestureEvent(frame.pb); - if (gesture != null) { - lastConnectionOrInputAtMs = lastIncomingAtMs; - event = new G2Event("sys-event", "", gesture.eventType, gesture.eventSource, 0); - logLine("idle gesture forwarded by CFW: type=" + gesture.eventType - + " source=" + gesture.eventSource); - } - } - if (!faceclawWakeNotification - && event == null - && shutdownRequested - && address.equalsIgnoreCase(rightAddress) - && BleProtocol.isDisplayWakeStateChange(frame)) { - // With no EvenHub page, ring/arm double-taps are handled by the - // stock display lifecycle and surface only as this state ping. - // Translate it back into an input event so TS can wake the shell - // and recreate the page. - event = new G2Event( - "display-wake", - "", - BleProtocol.EVENT_DOUBLE_CLICK, - 0, - 0 - ); - } - if (!faceclawWakeNotification - && frame.ok - && frame.sid == BleProtocol.SID_UI_SETTING) { - // Mode-17 query notifications; settings READs are handled by - // createBatteryQueryMessageLocked so headset/ring update together. - if (address.equalsIgnoreCase(rightAddress) - && (frame.flag == BleProtocol.FLAG_NOTIFY || frame.flag == BleProtocol.FLAG_NOTIFY_ALT)) { - BleProtocol.RingBatterySnapshot ring = BleProtocol.parseRingBattery(frame.pb); - if (ring != null) { - ringBattery = ring.battery; - ringCharging = ring.charging; - emitBatteryState(headsetBattery, headsetCharging); - } - } - // CFW mic status (field 104) rides both standalone pushes and - // settings read acks, from each temple on its own link. - byte[] micStatus = BleProtocol.parseFaceclawMicStatus(frame.pb); - if (micStatus != null) { - emitMicStatus(micStatus, address); - } - } - if (!faceclawWakeNotification - && frame.ok - && frame.sid == BleProtocol.SID_UI_SETTING) { - // CFW ambient-light report (field 105) from the master temple. - byte[] alsReport = BleProtocol.parseFaceclawAlsReport(frame.pb); - if (alsReport != null) { - emitAmbientLight(alsReport); - } - } - if (!faceclawWakeNotification - && decodedWearState < 0 - && frame.ok - && frame.sid == BleProtocol.SID_UI_SETTING) { - // Device-initiated settings push. It carries a magic the glasses - // chose, so it would otherwise fall through to resolveAckLocked, - // match nothing, and be logged as an unexpected ack. - int pushedSilentMode = BleProtocol.parseSilentModePush(frame.pb); - if (pushedSilentMode >= 0) { - updateSilentModeLocked(pushedSilentMode > 0); - return; - } - } - if (!faceclawWakeNotification - && decodedWearState < 0 - && frame.ok - && frame.msgSeq >= 0 - && frame.flag != BleProtocol.FLAG_NOTIFY - && frame.flag != BleProtocol.FLAG_NOTIFY_ALT) { - lastAckAtMs = lastIncomingAtMs; - resolveAckLocked(frame.sid, frame.msgSeq, frame.pb); - } - if (!faceclawWakeNotification - && event == null - && frame.ok - && address.equalsIgnoreCase(rightAddress) - && (frame.flag == BleProtocol.FLAG_NOTIFY || frame.flag == BleProtocol.FLAG_NOTIFY_ALT)) { - event = G2Event.decode(frame); - if (event != null - && "sys-event".equals(event.kind) - && event.eventType == BleProtocol.EVENT_HEAD_UP) { - // CFW forwards the IMU head-up while our page is on screen - // (soft sleep). Surface it as the same wake-only input the - // deferred head-up wake produces from a dark display. - event = new G2Event("display-wake", "", BleProtocol.EVENT_HEAD_UP, event.eventSource, 0); - logLine("head-up forwarded by CFW while page on screen"); - } - if (event != null) { - // Pure IMU samples arrive continuously; don't let them count - // as user input (which would starve battery polling). - boolean pureImuSample = "sys-event".equals(event.kind) - && event.eventType == BleProtocol.EVENT_IMU_DATA_REPORT; - if (!pureImuSample) { - lastConnectionOrInputAtMs = lastIncomingAtMs; - } - if ("list-click".equals(event.kind) || "text-click".equals(event.kind)) { - // Container-routed touchpad input reached us, so the - // firmware is dispatching input: silent mode is off, - // whether or not its end-of-silent push arrived. - updateSilentModeLocked(false); - } - if ("sys-event".equals(event.kind)) { - if (event.eventType == BleProtocol.EVENT_FOREGROUND_EXIT || event.eventType == BleProtocol.EVENT_ABNORMAL_EXIT || event.eventType == BleProtocol.EVENT_SYSTEM_EXIT) { - if (shutdownRequested) { - lastShutdownExitAtMs = SystemClock.elapsedRealtime(); - } - fixedLayoutCreated = false; - displayedFingerprint = ""; - clearAllMessagesLocked("firmware exit event"); - } - } - } - } - } - interruptibleSleep.interrupt(); - if (emitWearState) { - logLine(decodedWearState > 0 ? "wear state ON_HEAD" : "wear state OFF_HEAD"); - emitWearState(decodedWearState > 0); - } - if (compassEvent != null) { - emitCompassEvent(compassEvent); - } - if (event != null) { - if (event.hasImu) { - emitImuData(event.imuX, event.imuY, event.imuZ, event.eventSource); - } - // A standalone IMU_DATA_REPORT is a sensor sample, not a gesture: - // deliver it only to IMU listeners, skipping the input pipeline (and - // its per-frame latency bookkeeping) to avoid flooding it. - boolean pureImuSample = "sys-event".equals(event.kind) - && event.eventType == BleProtocol.EVENT_IMU_DATA_REPORT; - if (!pureImuSample) { - int frameId = FrameTimings.getInstance().startFrame( - "input:" + event.kind + " type=" + event.eventType + " src=" + event.eventSource); - FrameTimings.getInstance().log(frameId, "input event decoded from BLE notification"); - emitRingEvent(event, frameId); - } - } - } - - private void handleDirectRingNotification(String characteristicUuid, byte[] data) { - FaceclawRingEventDecoder.DirectRingEvent decoded = FaceclawRingEventDecoder.decode(data); - if (decoded == null) { - Log.d(TAG, "direct ring notify ignored: characteristicUuid=" + characteristicUuid + " raw=" + hex(data)); - return; - } - - G2Event event = decoded.event; - long arrivalMs = SystemClock.elapsedRealtime(); - synchronized (lock) { - lastIncomingAtMs = arrivalMs; - lastConnectionOrInputAtMs = arrivalMs; - } - logLine("direct ring " + decoded.label + " " + decoded.detail + " raw=" + hex(data)); - int frameId = FrameTimings.getInstance().startFrame("input:ring:" + decoded.label); - FrameTimings.getInstance().log(frameId, "input event decoded from direct ring notification"); - emitRingEvent(event, frameId); - interruptibleSleep.interrupt(); - } - - private void handleRenderNotification(String address, byte[] data) { - FaceclawAudioPacketListener listenerToCall; - long arrivalMs = SystemClock.elapsedRealtime(); - synchronized (lock) { - lastIncomingAtMs = arrivalMs; - listenerToCall = audioCaptureActive ? audioPacketListener : null; - } - if (listenerToCall == null) { - return; - } - String arm = address.equalsIgnoreCase(leftAddress) ? "L" : address.equalsIgnoreCase(rightAddress) ? "R" : "?"; - try { - listenerToCall.onAudioPacket(Arrays.copyOf(data, data.length), arm, arrivalMs); - } catch (Throwable t) { - logLine("G2 mic packet listener failed: " + safeMessage(t)); - } - } - - @Override public void onConnectionStateChange(String address, boolean connected) { - synchronized (lock) { - if (address == null) { - return; - } - if (isConfiguredRingAddress(address)) { - ringConnected = connected; - ringNotificationsReady = false; - if (!connected) { - ringReconnectAfterMs = SystemClock.elapsedRealtime() + ConnectionOptions.RING_RECONNECT_DELAY_MS; - } - logLine(connected ? "direct ring BLE connected" : "direct ring BLE disconnected"); - return; - } - if (address.equalsIgnoreCase(rightAddress)) { - rightConnected = connected; - } else if (address.equalsIgnoreCase(leftAddress)) { - leftConnected = connected; - } else { - return; - } - if (!connected) { - sessionReady = false; - fixedLayoutCreated = false; - startupProbePending = false; - chargingMode = false; - audioCaptureActive = false; - audioPacketListener = null; - clearAllMessagesLocked("connection lost"); - displayedFingerprint = ""; - if (!reconnectHalted) { - reconnectAfterMs = SystemClock.elapsedRealtime() + ConnectionOptions.RECONNECT_DELAY_MS; - } - } - } - interruptibleSleep.interrupt(); - if (connected) { - setStateDisplay("connected", "Connected."); - } else if (!reconnectHalted) { - // While parked on a missing bond, keep the "unpaired" display: this - // callback is just the teardown of the arm that did connect. - setStateDisplay("connecting", "Connecting to the glasses..."); - } - } - - private void connectLoopOnce() throws InterruptedException { - setStateDisplay("connecting", "Connecting to the glasses..."); - try { - connectArm(rightAddress, true); - connectArm(leftAddress, true); - if (!sleepDuringConnectSettling(800)) { - return; - } - authenticateArms(); - sendPrelude(); - - synchronized (lock) { - sessionReady = true; - // A fresh transport prelude always starts an active EvenHub - // lifecycle, even if the previous connection dropped while - // its page was intentionally suspended. - shutdownRequested = false; - fixedLayoutCreated = false; - clearAllMessagesLocked("session ready"); - displayedFingerprint = ""; - lastAckAtMs = SystemClock.elapsedRealtime(); - lastIncomingAtMs = lastAckAtMs; - lastConnectionOrInputAtMs = lastAckAtMs; - lastSessionReadyAtMs = lastAckAtMs; - lastBatteryRefreshAtMs = 0; - imageRetryAfterMs = 0; - lastHeartbeatSentAtMs = 0; - lastHeartbeatAckedAtMs = 0; - consecutiveAckTimeouts = 0; - lastAudioControlAckMagic = 0; - audioCaptureActive = false; - faceclawWakePendingNonce = -1; - cfwCleanupDelivered = false; - for (CfwTransport transport : cfwTransports) transport.reset(); - lastCfwCleanupAckMagic = 0; - lastFaceclawWakeLeaseQueuedAtMs = 0; - lastFaceclawFramebufferLeaseQueuedAtMs = 0; - enqueueFaceclawFramebufferControlLocked( - BleProtocol.FACECLAW_FB_OP_ACQUIRE, - true - ); - if (faceclawWakeLeaseEnabled) { - enqueueFaceclawWakeControlLocked( - BleProtocol.FACECLAW_WAKE_OP_ACQUIRE, - 0, - true - ); - } - } - setStateDisplay("connected", "Connected."); - logLine("session ready"); - synchronized (lock) { - // Query settings promptly on the first session so firmware - // version/extension (and battery) arrive without waiting for - // the input-quiet battery poll. The settings response doubles as - // the firmware-compatibility check surfaced during onboarding. - if (!firmwareInfoQueried) { - firmwareInfoQueried = true; - lastBatteryRefreshAtMs = SystemClock.elapsedRealtime(); - pendingMessages.addLast(createBatteryQueryMessageLocked()); - logLine("queue settings query for firmware info"); - } - } - tryConnectRing("initial"); - } catch (Throwable t) { - logLine("connect failed: " + safeMessage(t)); - String unpairedArm = firstUnpairedArm(); - if (unpairedArm != null) { - handleUnpairedFailure(unpairedArm); - } else { - handleTransportFailure("connect failed"); - } - } - } - - private boolean sleepDuringConnectSettling(long delayMs) throws InterruptedException { - long deadline = SystemClock.elapsedRealtime() + delayMs; - synchronized (lock) { - while (running && !userDisconnectRequested) { - long remaining = deadline - SystemClock.elapsedRealtime(); - if (remaining <= 0) { - return true; - } - lock.wait(Math.min(remaining, 100)); - } - return false; - } - } - - private void connectArm(String address, boolean enableRenderNotify) { - if (!bleManager.connect(address, ConnectionOptions.CONNECT_TIMEOUT_MS)) { - throw new IllegalStateException("connect failed: " + address); - } - // requestConnectionPriority has no callback in this Android compile target, so there is - // no reliable completion point to keep it in the global GATT operation pipeline. But it's - // important enough for performance that we call it anyways. - bleManager.requestConnectionPriority(address, BluetoothGatt.CONNECTION_PRIORITY_HIGH); - - bleManager.requestMtu(address, ConnectionOptions.DESIRED_MTU, ConnectionOptions.CONNECT_TIMEOUT_MS); - - if (!bleManager.discoverServices(address, ConnectionOptions.SERVICES_TIMEOUT_MS)) { - throw new IllegalStateException("discoverServices failed: " + address); - } - if (!bleManager.enableNotifications(address, BleProtocol.NOTIFY_CHAR_UUID, true, ConnectionOptions.DESCRIPTOR_TIMEOUT_MS)) { - throw new IllegalStateException("enableNotifications failed: " + address + " " + BleProtocol.NOTIFY_CHAR_UUID); - } - if (enableRenderNotify) { - bleManager.enableNotifications(address, BleProtocol.RENDER_NOTIFY_UUID, true, ConnectionOptions.DESCRIPTOR_TIMEOUT_MS); - } - synchronized (lock) { - if (address.equalsIgnoreCase(rightAddress)) { - rightConnected = true; - } else if (address.equalsIgnoreCase(leftAddress)) { - leftConnected = true; - } - } - } - - private boolean shouldAttemptRingConnect() { - if (!hasRingAddress()) { - return false; - } - long now = SystemClock.elapsedRealtime(); - synchronized (lock) { - return running - && sessionReady - && !ringNotificationsReady - && now >= ringReconnectAfterMs - && pendingMessages.isEmpty() - && inFlightMessages.isEmpty(); - } - } - - private void tryConnectRing(String reason) { - if (!hasRingAddress()) { - return; - } - try { - connectRing(); - } catch (Throwable t) { - synchronized (lock) { - ringConnected = false; - ringNotificationsReady = false; - ringReconnectAfterMs = SystemClock.elapsedRealtime() + ConnectionOptions.RING_RECONNECT_DELAY_MS; - } - logLine("direct ring connect failed (" + reason + "): " + safeMessage(t)); - } - } - - private void connectRing() { - logLine("connecting direct ring " + ringAddress); - if (!bleManager.connect(ringAddress, ConnectionOptions.CONNECT_TIMEOUT_MS)) { - throw new IllegalStateException("connect failed: " + ringAddress); - } - - bleManager.requestConnectionPriority(ringAddress, BluetoothGatt.CONNECTION_PRIORITY_HIGH); - bleManager.requestMtu(ringAddress, ConnectionOptions.RING_DESIRED_MTU, ConnectionOptions.CONNECT_TIMEOUT_MS); - - if (!bleManager.discoverServices(ringAddress, ConnectionOptions.SERVICES_TIMEOUT_MS)) { - throw new IllegalStateException("discoverServices failed: " + ringAddress); - } - - boolean phoneNotify = enableRingNotification(BleProtocol.R1_PHONE_NOTIFY_CHAR_UUID); - boolean dataNotify = enableRingNotification(BleProtocol.R1_NOTIFY_CHAR_UUID); - if (!phoneNotify && !dataNotify) { - throw new IllegalStateException("no R1 notify characteristic subscribed"); - } - - synchronized (lock) { - ringConnected = true; - ringNotificationsReady = true; - ringReconnectAfterMs = 0; - } - logLine("direct ring ready phoneNotify=" + phoneNotify + " dataNotify=" + dataNotify); - } - - private boolean enableRingNotification(String characteristicUuid) { - try { - return bleManager.enableNotifications( - ringAddress, - characteristicUuid, - true, - ConnectionOptions.DESCRIPTOR_TIMEOUT_MS - ); - } catch (Throwable t) { - Log.d(TAG, "direct ring notify subscribe skipped: " + characteristicUuid + " " + safeMessage(t)); - return false; - } - } - - /** - * Complete the sid-0x80 security-auth exchange on both freshly opened arm - * connections. Firmware 2.2.9 answers no queries until it completes over an - * encrypted link and closes unauthenticated links after ~30 s (see - * ../notes/ble-connections-2.2.9.md); on an unbonded phone the exchange is - * also what triggers SMP pairing. Deliberately soft: on timeout we log and - * continue rather than fail the connect — the custom firmware's response - * behavior is not yet hardware-verified, and on stock firmware an - * unanswered auth just means the prelude fails exactly as it did before. - * A pairing prompt accepted after our window still bonds at the OS level, - * so the next reconnect attempt authenticates promptly. - */ - private void authenticateArms() throws InterruptedException { - OutboundMessage right = messageBuilder.securityAuth(false); - OutboundMessage left = messageBuilder.securityAuth(true); - long now = SystemClock.elapsedRealtime(); - for (OutboundMessage message : new OutboundMessage[] {right, left}) { - message.onAck = () -> { - }; - message.onTimeout = () -> { - }; - message.sentAtMs = now; - writeMessage(message); - } - long deadline = SystemClock.elapsedRealtime() + ConnectionOptions.SECURITY_AUTH_SOFT_TIMEOUT_MS; - while (running && !userDisconnectRequested && !inFlightMessages.isEmpty()) { - synchronized (lock) { - if (!running || userDisconnectRequested || inFlightMessages.isEmpty()) { - break; - } - } - long remaining = deadline - SystemClock.elapsedRealtime(); - if (remaining <= 0) { - break; - } - interruptibleSleep.sleep(Math.min(remaining, 100)); - } - synchronized (lock) { - if (!inFlightMessages.isEmpty()) { - clearInFlightMessagesLocked("security auth timeout"); - logLine("security auth not acknowledged; continuing (2.2.9 stock requires it; older/custom firmware may not answer)"); - return; - } - } - boolean rightOk = BleProtocol.isAuthenticationSuccess(right.ackPayload, right.magic); - boolean leftOk = BleProtocol.isAuthenticationSuccess(left.ackPayload, left.magic); - logLine("security auth R=" + (rightOk ? "ok" : "unconfirmed") + " L=" + (leftOk ? "ok" : "unconfirmed")); - } - - private void sendPrelude() throws InterruptedException { - sendPrelude(false); - } - - private void sendPrelude(boolean preserveWakeLeaseControls) throws InterruptedException { - synchronized (lock) { - if (preserveWakeLeaseControls) { - clearAllMessagesPreservingWakeLeaseLocked("prelude"); - } else { - clearAllMessagesLocked("prelude"); - } - } - long now = SystemClock.elapsedRealtime(); - OutboundMessage prelude = messageBuilder.prelude(); - prelude.onAck = () -> { - }; - prelude.onTimeout = () -> { - handleTransportFailure("ack timeout"); - }; - prelude.sentAtMs = now; - writeMessage(prelude); - - long deadline = SystemClock.elapsedRealtime() + ConnectionOptions.PRELUDE_TIMEOUT_MS; - while (running && !userDisconnectRequested && !inFlightMessages.isEmpty()) { - synchronized (lock) { - if (!running || userDisconnectRequested || inFlightMessages.isEmpty()) { - break; - } - } - long remaining = deadline - SystemClock.elapsedRealtime(); - if (remaining <= 0) { - break; - } - interruptibleSleep.sleep(Math.min(remaining, 100)); - } - synchronized (lock) { - if (!inFlightMessages.isEmpty()) { - clearInFlightMessagesLocked("prelude timeout"); - throw new IllegalStateException("prelude ack timeout"); - } - } - } - - private long driveSession() { - //Log.d(TAG, "driveSession called (pendingMessages.size=" + pendingMessages.size() + " inFlightMessages.size=" + inFlightMessages.size() + ")"); - while (true) { - // Run on the sender thread, outside `lock`: Bluetooth API calls can - // wait on the global GATT lock, and callbacks need the session lock. - // Keep negotiation outside the timed stream. A fixed settling period - // is only an experiment boundary; HCI must confirm actual parameters. - int linkMode = -1; - String[] benchmarkAddresses = null; - synchronized (lock) { - if (benchmarkActive && benchmarkLinkPending) { - benchmarkLinkPending = false; - linkMode = benchmarkLinkMode; - benchmarkAddresses = new String[] {leftAddress, rightAddress}; - } - } - if (benchmarkAddresses != null) { - for (String address : benchmarkAddresses) { - try { - bleManager.prepareBenchmarkLink(address, linkMode); - } catch (RuntimeException error) { - logLine("benchmark link request failed: " + safeMessage(error)); - } - } - synchronized (lock) { - // Cancellation/new-run races leave the new run pending; its - // own preparation will replace this deadline before sending. - benchmarkReadyAtMs = SystemClock.elapsedRealtime() + 1_000; - } - } - OutboundMessage messageToWrite = null; - OutboundMessage messageToPrewrite = null; - long now = SystemClock.elapsedRealtime(); - - maybeFinishNoChangeDesiredFrame(); - - synchronized (lock) { - drainCfwAcknowledgementsLocked(); - if (cfwCleanupDelivered) { - /* Successful mode 11 must be the last Faceclaw write. Drop - * anything a late external producer attempted to enqueue - * while DashboardController was closing the transport. */ - clearPendingMessagesLocked("after CFW cleanup"); - return 250; - } - if (sessionReady - && now - lastFaceclawFramebufferLeaseQueuedAtMs >= FACECLAW_WAKE_LEASE_RENEW_MS - && !hasPendingOrInflightKindLocked("framebuffer-lease-control")) { - enqueueFaceclawFramebufferControlLocked( - BleProtocol.FACECLAW_FB_OP_ACQUIRE, - false - ); - } - if (faceclawWakeLeaseEnabled - && sessionReady - && now - lastFaceclawWakeLeaseQueuedAtMs >= FACECLAW_WAKE_LEASE_RENEW_MS - && !hasPendingOrInflightKindLocked("wake-lease-control")) { - enqueueFaceclawWakeControlLocked( - BleProtocol.FACECLAW_WAKE_OP_ACQUIRE, - 0, - false - ); - } - if (!inFlightMessages.isEmpty()) { - OutboundMessage oldest = inFlightMessages.peekFirst(); - List replay = CfwMessageWindow.replayWindow(inFlightMessages, now); - if (!replay.isEmpty()) { - logLine("CFW recovery: replay " + replay.size() - + " unresolved message(s) from id=" + replay.get(0).magic); - for (OutboundMessage candidate : replay) { - if (candidate.cfwRetries >= CfwMessageWindow.MAX_RETRIES) { - handleTransportFailure("CFW recovery retry limit"); - return 0; - } - logLine("CFW replay id=" + candidate.magic + " label=" + candidate.label - + " cause=" + (candidate.cfwRetryPending ? "NACK" - : candidate.cfwAckLenses == CfwTransport.BOTH ? "ordered tail" : "missing ACK") - + " ackedLenses=" + candidate.cfwAckLenses - + " ageMs=" + (now - candidate.sentAtMs)); - inFlightMessages.remove(candidate); - magicPool.release(candidate.sid, candidate.magic, candidate.label, "CFW replay"); - candidate.prepareCfwReplay(magicPool.allocate()); - } - for (int i = replay.size() - 1; i >= 0; --i) pendingMessages.addFirst(replay.get(i)); - return 0; - } - if (oldest != null && oldest.ackDeadlineAtMs <= now) { - Log.i(TAG, "message timed out: " + oldest.label); - inFlightMessages.removeFirst(); - logLine("message timed out: " + oldest.label + " sid=" + oldest.sid - + " id=" + oldest.magic + " ackedLenses=" + oldest.cfwAckLenses - + " ageMs=" + (now - oldest.sentAtMs)); - magicPool.release(oldest.sid, oldest.magic, oldest.label, "timeout"); - if (oldest.sid == CfwTransport.SID) - cfwTransports[oldest.isLeftArmMessage ? 0 : 1].reset(); - handleAckTimeoutLocked(oldest); - return 0; - } - if (connectionOptions.WINDOW_SIZE <= 1 - && sessionReady - && prewrittenMessage == null - && !pendingMessages.isEmpty() - && canPrewriteCandidate(pendingMessages.peekFirst()) - && !shouldBlockPrewriteForHeartbeatLocked(now)) { - // Only the serial (window==1) path pre-sends the all-but-last - // packet; with a real window we just send the next message fully. - messageToPrewrite = pendingMessages.peekFirst(); - } - } - - if (chargingMode) { - // Glasses are in the case: no display traffic, only battery - // polls (which also detect the end of charging). - finishDesiredFrameLocked("discarded: glasses charging"); - if (sessionReady && inFlightMessages.isEmpty() && !pendingMessages.isEmpty()) { - messageToWrite = pendingMessages.removeFirst(); - Log.i(TAG, "sending pending message (charging): " + messageToWrite.label); - } else if (sessionReady && pendingMessages.isEmpty() && inFlightMessages.isEmpty() - && now - lastBatteryRefreshAtMs >= ConnectionOptions.CHARGING_BATTERY_POLL_MS) { - Log.i(TAG, "Writing charging-mode battery poll"); - messageToWrite = createBatteryQueryMessageLocked(); - lastBatteryRefreshAtMs = now; - } else { - return 1_000; - } - } else { - if (messageToPrewrite == null - && !shutdownRequested - && !fixedLayoutCreated - && pendingMessages.isEmpty() - && inFlightMessages.isEmpty()) { - Log.i(TAG, "enqueueing create layout"); - enqueueCreateLayoutLocked(); - } else if (messageToPrewrite != null) { - // Prewrite outside the lock; the logical message remains pending until - // its final BLE frame is sent after the current protocol ACK. - } - if (messageToPrewrite == null && !shutdownRequested && fixedLayoutCreated - && (firmwareDebugFlagsEnabled ? 2 : 1) != firmwareDebugFlagsLastSent - && pendingMessages.isEmpty() && inFlightMessages.isEmpty()) { - Log.i(TAG, "enqueueing firmware debug flags " + (firmwareDebugFlagsEnabled ? "show" : "hide")); - enqueueFirmwareDebugFlagsLocked(); - } - - if (messageToPrewrite == null && !shutdownRequested && fixedLayoutCreated - && (compassOwners.isEmpty() ? 0 : 1) != compassControlLastSent - && pendingMessages.isEmpty() && inFlightMessages.isEmpty()) { - boolean wanted = !compassOwners.isEmpty(); - Log.i(TAG, "enqueueing compass " + (wanted ? "enable" : "disable")); - enqueueCompassControlLocked(false, wanted); - } - - if (benchmarkActive) { - maintainBenchmarkLocked(now); - } - - // Up to WINDOW_SIZE messages may be in flight at once (full - // pipelining); a slot frees when an ack arrives. An active - // bandwidth benchmark measures its own selected window instead. - boolean windowHasRoom = inFlightMessages.size() - < Math.max(1, benchmarkActive ? benchmarkWindowSize : connectionOptions.WINDOW_SIZE); - // A frame ready to send right now: don't inject a fresh - // heartbeat in front of it (the image's own ack resets the - // firmware heartbeat timer, so the heartbeat is redundant). - // "Ready" covers both a fresh composite still to be planned - // and a planned image already queued but not yet written -- - // enqueueing sets lastEnqueuedFingerprint, so testing only - // the desired fingerprint left the queued-but-unwritten - // window unprotected and a heartbeat that came due there - // cost the frame a full ack round trip (measured 124ms on - // frame#232 of the 2026-08-20 02:27 capture). - // A benchmark run counts as image traffic here for the same - // reason: its acks reset the firmware heartbeat timer, so a - // fresh heartbeat in front of it is redundant. - boolean imageWaiting = !shutdownRequested && fixedLayoutCreated - && !hasPendingOrInflightKindLocked("heartbeat") - && (benchmarkActive - || (now >= imageRetryAfterMs - && (hasPendingImageLocked() - || !getDesiredFingerprint().equals(lastEnqueuedFingerprint)))); - noteImageStallLocked(now, windowHasRoom); - if (messageToPrewrite == null && handleHeartbeat(imageWaiting)) { - return ConnectionOptions.IDLE_SLEEP_MS; - } - - if (messageToPrewrite == null && sessionReady && windowHasRoom && !pendingMessages.isEmpty() - && CfwMessageWindow.canSend(inFlightMessages, pendingMessages.peekFirst())) { - messageToWrite = pendingMessages.removeFirst(); - Log.i(TAG, "sending pending message: " + messageToWrite.label); - } else if (messageToPrewrite == null && !shutdownRequested && !benchmarkActive - && fixedLayoutCreated - && windowHasRoom && !hasPendingImageLocked() - && now >= imageRetryAfterMs - && !getDesiredFingerprint().equals(lastEnqueuedFingerprint)) { - // Enqueue the next frame's delta against lastEnqueuedPacked - // (what the shadow will be), so it can pipeline behind an - // image still awaiting its ack. - Log.i(TAG, "Enqueued image update"); - enqueueDesiredImageLocked(); - return 0; - } else if (messageToPrewrite == null && shouldPollBatteryLocked(now)) { - Log.i(TAG, "Writing battery query"); - messageToWrite = createBatteryQueryMessageLocked(); - lastBatteryRefreshAtMs = now; - } else if (messageToPrewrite == null && (!pendingMessages.isEmpty() || !inFlightMessages.isEmpty())) { - return ConnectionOptions.IDLE_SLEEP_MS; - } else if (messageToPrewrite == null) { - return 250; - } - } - } - - if (messageToPrewrite != null) { - if (prewriteMessage(messageToPrewrite)) { - return 0; - } - return ConnectionOptions.IDLE_SLEEP_MS; - } - - if (!writeMessage(messageToWrite)) { - synchronized (lock) { - if (removePreparedMessageLocked(messageToWrite) || messageToWrite.magic == 0) { - handleTransportFailure("write failed"); - } - } - return 0; - } - } - } - - /** - * If the desired image already matches what the glasses display, nothing will - * ever be enqueued for it, so finish its frame now (otherwise the TS side - * would block on it until its backpressure timeout). - */ - private void maybeFinishNoChangeDesiredFrame() { - int frameIdToFinish = 0; - synchronized (lock) { - synchronized (desiredTilesLock) { - if (desiredFrameId != 0 && !lastEnqueuedFingerprint.isEmpty() && desiredFingerprint.equals(lastEnqueuedFingerprint)) { - frameIdToFinish = desiredFrameId; - desiredFrameId = 0; - } - } - } - if (frameIdToFinish != 0) { - finishFrame(frameIdToFinish, "discarded: no change from displayed image"); - } - } - - private boolean shouldBlockPrewriteForHeartbeatLocked(long now) { - if (shutdownRequested || !fixedLayoutCreated) { - return false; - } - return hasPendingOrInflightKindLocked("heartbeat") - || now - lastHeartbeatAckedAtMs >= ConnectionOptions.HEARTBEAT_READY_MS; - } - - private boolean canPrewriteCandidate(OutboundMessage message) { - if (message == null || message.sid == CfwTransport.SID || !message.isLeftArmMessage) { - return false; - } - if (!"image".equals(message.kind)) { - return false; - } - return message.message.length + 2 > 232; - } - - private boolean handleHeartbeat(boolean imageWaiting) { - long now = SystemClock.elapsedRealtime(); - boolean heartbeatEligible = !shutdownRequested && fixedLayoutCreated; - boolean heartbeatPending = heartbeatEligible && hasPendingOrInflightKindLocked("heartbeat"); - long heartbeatElapsedMs = now - lastHeartbeatAckedAtMs; - boolean heartbeatReady = heartbeatEligible && heartbeatElapsedMs >= ConnectionOptions.HEARTBEAT_READY_MS; - boolean heartbeatUrgent = heartbeatEligible && heartbeatElapsedMs >= ConnectionOptions.HEARTBEAT_URGENT_MS; - boolean heartbeatBlocksLeftWrites = heartbeatReady || heartbeatPending; - - if (heartbeatReady && !heartbeatPending && inFlightMessages.isEmpty()) { - if (imageWaiting && !heartbeatUrgent) { - // Defer to the waiting frame: sending it now satisfies the - // firmware heartbeat deadline (its ack resets the timer), and - // the heartbeat still fires once we reach the URGENT threshold - // if rendering goes quiet again. Preserves the pending-heartbeat - // inter-lens-sync invariant below (that path is untouched). - return false; - } - Log.i(TAG, "Writing heartbeat"); - OutboundMessage heartbeatMessage = createHeartbeatMessage(); - lastHeartbeatSentAtMs = now; - writeMessage(heartbeatMessage); - return true; - } else if (heartbeatUrgent) { - return true; - } else if (heartbeatPending) { - // Don't send other message types while a heartbeat is pending because that - // can lead to inter-lens sync issues - return true; - } - - return false; - } - - private OutboundMessage createHeartbeatMessage() { - OutboundMessage message = messageBuilder.heartbeat(); - message.onAck = () -> { - synchronized (lock) { - lastHeartbeatAckedAtMs = SystemClock.elapsedRealtime(); - } - }; - message.onTimeout = () -> { - // If a heartbeat fails to ack and we're over the heartbeat deadline, assume the connection is failed and reconnect. - // Otherwise ignore it, which will cause a retransmission attempt. - boolean isPastDeadline; - synchronized (lock) { - isPastDeadline = SystemClock.elapsedRealtime() - lastHeartbeatSentAtMs >= ConnectionOptions.HEARTBEAT_FAILURE_DEADLINE_MS; - } - if (isPastDeadline) { - handleTransportFailure("heartbeat ack timeout"); - } - }; - return message; - } - - - private boolean writeMessage(OutboundMessage message) { - long now = SystemClock.elapsedRealtime(); - message.writeStartedAtMs = now; - message.sentAtMs = now; - message.ackDeadlineAtMs = now + message.ackTimeoutMs + ConnectionOptions.WRITE_TIMEOUT_MS; - if (message.magic != 0) { - synchronized (lock) { - inFlightMessages.addLast(message); - } - } - if (message.imageUpdateId > 0 && message.imageMessageNumber == 1) { - synchronized(lock) { - BleImageOptimizer.ImageUpdateStats stats = imageUpdateStats.get(message.imageUpdateId); - if (stats != null && stats.firstWriteStartedAtMs <= 0) { - stats.firstWriteStartedAtMs = now; - } - } - } - - String writeAddress = message.isLeftArmMessage ? leftAddress : rightAddress; - List frames; - if (prewrittenMessage != null && prewrittenMessage != message) { - if (!spoilPrewrittenMessage("before " + message.label)) { - return false; - } - } - if (prewrittenMessage == message) { - frames = Collections.singletonList(prewrittenFrames.get(prewrittenFrames.size() - 1)); - prewrittenMessage = null; - prewrittenFrames = Collections.emptyList(); - } else if (message.sid == CfwTransport.SID) { - if (message.cfwRetries > 0) cfwTransports[message.isLeftArmMessage ? 0 : 1].reset(); - frames = cfwTransports[message.isLeftArmMessage ? 0 : 1].encode( - message.message, message.magic, CfwTransport.BOTH, - bleManager.getNegotiatedMtu(writeAddress)); - } else { - frames = BleProtocol.framePb( - message.message, - message.sid, - message.flag, - nextTransportSeq++ - ); - } - boolean result = bleManager.writeFrames( - writeAddress, - BleProtocol.WRITE_CHAR_UUID, - frames, - AndroidProtocolPlatform.writeType(ConnectionOptions.WRITE_MODE), - ConnectionOptions.WRITE_TIMEOUT_MS - ); - - synchronized (lock) { - long sentAtMs = SystemClock.elapsedRealtime(); - message.sentAtMs = sentAtMs; - message.ackDeadlineAtMs = sentAtMs + message.ackTimeoutMs; - logImageUpdateSendLandmarkLocked(message); - if (result && message.onSent != null) { - message.onSent.run(); - lock.notifyAll(); - } - } - - if (!result && message.sid == CfwTransport.SID) - cfwTransports[message.isLeftArmMessage ? 0 : 1].reset(); - return result; - } - - private boolean prewriteMessage(OutboundMessage message) { - if (prewrittenMessage == message) { - return true; - } - if (prewrittenMessage != null && !spoilPrewrittenMessage("before prewrite " + message.label)) { - return false; - } - if (!canPrewriteCandidate(message)) { - return false; - } - - List frames = BleProtocol.framePb( - message.message, - message.sid, - message.flag, - nextTransportSeq++ - ); - if (frames.size() <= 1) { - return false; - } - - String writeAddress = message.isLeftArmMessage ? leftAddress : rightAddress; - List prefixFrames = frames.subList(0, frames.size() - 1); - boolean result = bleManager.writeFrames( - writeAddress, - BleProtocol.WRITE_CHAR_UUID, - prefixFrames, - AndroidProtocolPlatform.writeType(ConnectionOptions.WRITE_MODE), - ConnectionOptions.WRITE_TIMEOUT_MS - ); - if (!result) { - return false; - } - - prewrittenMessage = message; - prewrittenFrames = new ArrayList<>(frames); - logLine("prewrote " + message.label + " frames=" + prefixFrames.size() + "/" + frames.size()); - return true; - } - - private boolean spoilPrewrittenMessage(String reason) { - if (prewrittenMessage == null || prewrittenFrames.isEmpty()) { - prewrittenMessage = null; - prewrittenFrames = Collections.emptyList(); - return true; - } - OutboundMessage message = prewrittenMessage; - byte[] finalFrame = Arrays.copyOf( - prewrittenFrames.get(prewrittenFrames.size() - 1), - prewrittenFrames.get(prewrittenFrames.size() - 1).length - ); - if (finalFrame.length > 8) { - finalFrame[finalFrame.length - 1] ^= (byte) 0xff; - } - prewrittenMessage = null; - prewrittenFrames = Collections.emptyList(); - - String writeAddress = message.isLeftArmMessage ? leftAddress : rightAddress; - logLine("spoiling prewritten " + message.label + ": " + reason); - return bleManager.writeFrames( - writeAddress, - BleProtocol.WRITE_CHAR_UUID, - Collections.singletonList(finalFrame), - AndroidProtocolPlatform.writeType(ConnectionOptions.WRITE_MODE), - ConnectionOptions.WRITE_TIMEOUT_MS - ); - } - - private boolean removePreparedMessageLocked(OutboundMessage message) { - if (message == null || message.magic == 0) { - return false; - } - Iterator iterator = inFlightMessages.iterator(); - while (iterator.hasNext()) { - if (iterator.next() == message) { - iterator.remove(); - magicPool.release(message.sid, message.magic, message.label, "write failed"); - return true; - } - } - return false; - } - - - /** Also drain on the worker: a clear/timeout may have removed a blocking - * head since the last notification. Fully ACKed controls must never reach - * the generic timeout path just because no further BLE reply arrived. */ - private void drainCfwAcknowledgementsLocked() { - while (true) { - OutboundMessage first = CfwMessageWindow.acknowledgedHead(inFlightMessages); - if (first == null) return; - lastAckAtMs = SystemClock.elapsedRealtime(); - resolveAckLocked(first, first.ackPayload); - } - } - - private void resolveAckLocked(int sid, int magic, byte[] pb) { - Iterator iterator = inFlightMessages.iterator(); - while (iterator.hasNext()) { - OutboundMessage message = iterator.next(); - if (message.sid == sid && message.magic == magic) { - resolveAckLocked(message, pb); - return; - } - } - recordUnexpectedAckLocked(sid, magic); - } - - private void resolveAckLocked(OutboundMessage message, byte[] pb) { - Log.i(TAG, "Got ACK for " + message.label + "(sid=" + message.sid + ", id=" + message.magic + ")"); - inFlightMessages.remove(message); - message.ackPayload = pb == null ? new byte[0] : Arrays.copyOf(pb, pb.length); - magicPool.release(message.sid, message.magic, message.label, "ack"); - if (message.onAck != null) { - message.onAck.run(); - } - consecutiveAckTimeouts = 0; - // onAck may have just satisfied a waiter blocked on lock.wait() (e.g. - // awaitEvenHubSessionReady polling fixedLayoutCreated/displayedFingerprint - // after a create-layout or image ack). Without this, that waiter only - // notices on its own up-to-100ms poll tick, adding avoidable latency to - // every EvenHub wake. Always called with lock held (see call site). - lock.notifyAll(); - } - - private void logImageUpdateSendLandmarkLocked(OutboundMessage message) { - if (message.imageUpdateId <= 0) { - return; - } - BleImageOptimizer.ImageUpdateStats stats = imageUpdateStats.get(message.imageUpdateId); - int frameId = stats == null ? 0 : stats.frameId; - if (message.imageMessageNumber == 1) { - if (stats != null && stats.firstWriteStartedAtMs <= 0) { - stats.firstWriteStartedAtMs = message.writeStartedAtMs > 0 ? message.writeStartedAtMs : message.sentAtMs; - } - FrameTimings.getInstance().log(frameId, "first bluetooth packet sent"); - logImageUpdateLandmarkLocked("first bluetooth message sent", message, message.sentAtMs); - } - if (message.imageMessageNumber == message.imageMessageCount) { - FrameTimings.getInstance().log(frameId, - "last bluetooth packet sent (message " + message.imageMessageNumber + "/" + message.imageMessageCount + ")"); - logImageUpdateLandmarkLocked("last bluetooth message sent", message, message.sentAtMs); - } - } - - private void logImageUpdateAckLandmarkLocked(OutboundMessage message) { - if (message.imageUpdateId <= 0 || message.imageMessageNumber != message.imageMessageCount) { - return; - } - long ackedAtMs = SystemClock.elapsedRealtime(); - FaceclawBleManager.recordDisplayFrameSent(); - BleImageOptimizer.ImageUpdateStats stats = imageUpdateStats.remove(message.imageUpdateId); - if (stats != null && stats.firstWriteStartedAtMs > 0) { - emitFrameMetrics(stats.paintMs, (int) Math.max(0, ackedAtMs - stats.firstWriteStartedAtMs), stats.tileCount); - } - if (stats != null) { - finishFrame(stats.frameId, "sent"); - } - logImageUpdateLandmarkLocked("last bluetooth message acked", message, ackedAtMs); - } - - /** Remove the stats entry for an image update that will not complete, finishing its frame. */ - private void discardImageUpdateStatsLocked(int imageUpdateId, String reason) { - if (imageUpdateId <= 0) { - return; - } - BleImageOptimizer.ImageUpdateStats stats = imageUpdateStats.remove(imageUpdateId); - if (stats != null) { - finishFrame(stats.frameId, "discarded: " + reason); - } - } - - private void logImageUpdateLandmarkLocked(String event, OutboundMessage message, long elapsedMs) { - logLine("image update#" + message.imageUpdateId + " " + event - + " at " + timestamp(elapsedMs) - + " message=" + message.imageMessageNumber + "/" + message.imageMessageCount - + " label=" + message.label); - } - - private void enqueueCreateLayoutLocked() { - // New session: re-assert the firmware-debug-flags overlay once - // the layout is ready (the mode-7 send is gated on this having reset). - firmwareDebugFlagsLastSent = -1; - OutboundMessage message = messageBuilder.createLayout(); - message.onAck = () -> { - startupProbePending = false; - clearMessagesOfKindLocked("startup-text-probe"); - fixedLayoutCreated = true; - displayedFingerprint = ""; - }; - message.onTimeout = () -> { - if (startupProbePending) { - logLine("create layout timed out while startup text probe is pending"); - if (hasPendingOrInflightKindLocked("startup-text-probe")) { - return; - } - startupProbePending = false; - } - handleTransportFailure("ack timeout"); - }; - pendingMessages.addLast(message); - logLine("queue create layout"); - } - - private void enqueueStartupProbeLocked() { - enqueueCreateLayoutLocked(); - - OutboundMessage message = messageBuilder.startupTextProbe(); - message.onAck = () -> { - startupProbePending = false; - clearMessagesOfKindLocked("create-layout"); - fixedLayoutCreated = true; - displayedFingerprint = ""; - logLine("existing dashboard layout accepted text probe"); - }; - message.onTimeout = () -> { - startupProbePending = false; - if (hasPendingOrInflightKindLocked("create-layout")) { - return; - } - handleTransportFailure("ack timeout"); - }; - pendingMessages.addLast(message); - startupProbePending = true; - logLine("queue startup text probe"); - } - - /** - * Send the CFW mode-7 diagnostic-flag control op through the private stream: - * [7][2] to show the on-glasses debug-flag overlay, [7][1] to hide it. Uses the - * arbitrary-payload custom path (no bmp/dedup/frame-timing interaction). - */ - private void enqueueFirmwareDebugFlagsLocked() { - boolean show = firmwareDebugFlagsEnabled; - int sub = show ? 2 : 1; - byte[] payload = new byte[] { (byte) CfwMessageTypeKt.CFW_MSG_DIAGNOSTICS, (byte) sub }; - OutboundMessage message = messageBuilder.imagePayload( - DASHBOARD_TILE, nextMapSessionId(), payload, - "fw-debug-flags " + (show ? "show" : "hide"), - connectionOptions.sendImagesToLeft); - pendingMessages.addLast(message); - firmwareDebugFlagsLastSent = sub; - logLine("queue firmware debug flags " + (show ? "show" : "hide")); - } - - /** - * Send CFW image-handler mode 10. Enable uses the configurable form - * [10][2][interval-ms LE16][minimum-change-degrees LE16]; disable remains [10][0]. - */ - private void enqueueCompassControlLocked(boolean priority, boolean enable) { - int sentState = enable ? 1 : 0; - byte[] payload = enable - ? new byte[] { - (byte) CfwMessageTypeKt.CFW_MSG_COMPASS, - (byte) 2, - (byte) (COMPASS_REPORT_INTERVAL_MS & 0xff), - (byte) ((COMPASS_REPORT_INTERVAL_MS >> 8) & 0xff), - (byte) (COMPASS_MIN_CHANGE_DEGREES & 0xff), - (byte) ((COMPASS_MIN_CHANGE_DEGREES >> 8) & 0xff), - } - : new byte[] { (byte) CfwMessageTypeKt.CFW_MSG_COMPASS, (byte) 0 }; - OutboundMessage message = messageBuilder.imagePayload( - "compass-control", - DASHBOARD_TILE, - nextMapSessionId(), - payload, - "compass " + (enable ? "enable" : "disable"), - connectionOptions.sendImagesToLeft); - message.onTimeout = () -> { - compassControlLastSent = -1; - logLine("compass control ack timeout"); - }; - if (enable) { - compassMaybeOn = true; - } else { - message.onAck = () -> compassMaybeOn = false; - } - if (priority) pendingMessages.addFirst(message); - else pendingMessages.addLast(message); - compassControlLastSent = sentState; - logLine("queue " + message.label); - } - - private void enqueueDesiredImageLocked() { - String fingerprint = getDesiredFingerprint(); - byte[] packed; - int width; - int height; - int paintMs; - int frameId; - SurfaceCompositor.ScreenDraw[] draws; - ShellScene scene; - synchronized (desiredTilesLock) { - packed = desiredPacked; - width = desiredWidth; - height = desiredHeight; - paintMs = desiredPaintMs; - frameId = desiredFrameId; - draws = desiredDraws; - scene = desiredShellScene; - desiredFrameId = 0; - } - if (packed == null) { - packed = new byte[0]; - } - if (customFirmwareDetected && packed.length > 0) { - ScenePlanner.Plan rendered = scenePlanner.plan(packed, width, height, draws, scene, nextImageFrameId, - connectionOptions.TEXTURE_CACHE_FRAMES, connectionOptions.INCREMENTAL_FRAMES, - connectionOptions.MULTI_RECT_FRAMES, ConnectionOptions.MULTI_RECT_MAX_RECTS); - nextImageFrameId = rendered.getNextFid(); - List commands = rendered.getCommands(); - for (int i = 0; i < commands.size() - 1; i++) enqueueResourceCommandLocked(commands.get(i)); - BleImageOptimizer.TileImagePlan plan = new BleImageOptimizer.TileImagePlan( - 0, DASHBOARD_TILE, packed, width, height, nextMapSessionId(), commands.get(commands.size() - 1)); - finishEnqueueDesiredImageLocked(plan, fingerprint, paintMs, frameId); - return; - } - - FrameTimings.getInstance().spanStart(frameId, "compress-and-plan"); - // Incremental (mode 3 bounding box) update against the last ENQUEUED frame - // (the base the firmware shadow will hold when this update is applied). - // lastEnqueuedPacked is cleared whenever the image pipeline is cleared, so - // a non-empty value means the display base is trusted. - byte[] incrementalPayload = null; - String incrementalLog = null; - if (connectionOptions.INCREMENTAL_FRAMES && lastEnqueuedPacked.length > 0 - && lastEnqueuedWidth == width && lastEnqueuedHeight == height) { - int baseFid = nextImageFrameId; - BleImageOptimizer.IncrementalPlan single = - BleImageOptimizer.buildIncrementalImagePayload(lastEnqueuedPacked, packed, width, height, baseFid); - if (single != null) { - incrementalPayload = single.payload; - // advance only when a delta is actually emitted, so consecutive - // deltas carry consecutive ids (CFW skip/reorder detection) - nextImageFrameId = nextImageFrameId >= 0xfffe ? 1 : nextImageFrameId + 1; - incrementalLog = "incremental update bbox=" - + ((single.payload[3] & 0xff) * 4) + "x" + ((single.payload[4] & 0xff) * 2) - + "+" + ((single.payload[1] & 0xff) * 4) + "+" + ((single.payload[2] & 0xff) * 2) - + " changed=" + single.changedBytes + "/" + single.boxBytes + "B" - + " clusters=" + single.clusterCount; - - // When the bounding box spans multiple clusters or is sizeable, try - // splitting into tight rects (CFW mode-8). Only replace the single - // box if the multi-rect message is actually smaller on the wire. - if (connectionOptions.MULTI_RECT_FRAMES - && (single.clusterCount > 1 || single.payload.length > ConnectionOptions.MULTI_RECT_MIN_PAYLOAD)) { - BleImageOptimizer.MultiRectPlan multi = BleImageOptimizer.buildMultiRectImagePayload( - lastEnqueuedPacked, packed, width, height, baseFid, ConnectionOptions.MULTI_RECT_MAX_RECTS); - if (multi != null && multi.payload.length < single.payload.length) { - incrementalPayload = multi.payload; - nextImageFrameId = multi.nextFid; // rectCount fids consumed - incrementalLog = "multi-rect update n=" + multi.rectCount - + " covered=" + multi.coveredBytes + "B" - + " payload=" + multi.payload.length + "B (vs bbox " + single.payload.length + "B)"; - } - } - } - } - BleImageOptimizer.TileImagePlan plan = incrementalPayload != null - ? new BleImageOptimizer.TileImagePlan(0, DASHBOARD_TILE, packed, width, height, nextMapSessionId(), incrementalPayload) - : new BleImageOptimizer.TileImagePlan(0, DASHBOARD_TILE, packed, width, height, nextMapSessionId()); - FrameTimings.getInstance().spanEnd(frameId, "compress-and-plan"); - if (incrementalLog != null) { - FrameTimings.getInstance().log(frameId, incrementalLog); - } - finishEnqueueDesiredImageLocked(plan, fingerprint, paintMs, frameId); - } - - /** Queue complete private commands, advance the delta base, and retain frame/ACK bookkeeping. */ - private void finishEnqueueDesiredImageLocked( - BleImageOptimizer.TileImagePlan plan, String fingerprint, int paintMs, int frameId) { - int updateId = nextImageUpdateId++; - List commands = new ArrayList<>(); - if (plan.payload.length <= CfwTransport.MAX_MESSAGE) { - commands.add(plan.payload); - } else { - // A noisy full frame can exceed the stream record limit. Repaint it - // with independently decodable RLE bands below the decoded-message limit. - commands.addAll(BleImageOptimizer.encodeFullFrameBands( - plan.packed, plan.width, plan.height, nextImageFrameId)); - for (int i = 0; i < commands.size(); i++) { - nextImageFrameId = nextImageFrameId >= 0xfffe ? 1 : nextImageFrameId + 1; - } - } - int messageCount = commands.size(); - imageUpdateStats.put(updateId, new BleImageOptimizer.ImageUpdateStats(paintMs, 1, frameId)); - for (int i = 0; i < commands.size(); i++) { - enqueueCustomImageLocked(plan, commands.get(i), fingerprint, updateId, i + 1, messageCount); - } - // This frame is now the base for the next delta (it will be the firmware - // shadow once applied), even though it hasn't been acked yet — that is what - // lets the next frame pipeline behind it. plan.packed is the full frame; - // frames are immutable by convention, so referencing it is safe. - lastEnqueuedPacked = plan.packed; - lastEnqueuedWidth = plan.width; - lastEnqueuedHeight = plan.height; - lastEnqueuedFingerprint = fingerprint; - - FrameTimings.getInstance().log(frameId, "queued image update#" + updateId - + " messages=" + messageCount + " payload=" + plan.payload.length + "B"); - logLine("queue image update#" + updateId + " fingerprint=" + fingerprint - + " messages=" + messageCount); - } - - /** - * Enqueue one resource eviction/upload command ahead of the image message that - * references its glyphs (the transport is FIFO, so no ack round trip is - * needed before use). A timeout means the on-glasses cache state is - * unknown; forget everything phone-side (glyphs re-upload lazily) and let - * the accompanying image update's own timeout drive the frame resync. - */ - private void enqueueResourceCommandLocked(byte[] payload) { - OutboundMessage message = messageBuilder.imagePayload( - "resources", - DASHBOARD_TILE, - nextMapSessionId(), - payload, - "resource command " + payload.length + "B", - connectionOptions.sendImagesToLeft); - message.onTimeout = () -> { - resourceCache.reset(); - logLine("resource command ack timeout; resource cache state reset"); - }; - pendingMessages.addLast(message); - logLine("queue " + message.label); - } - - private void enqueueCustomImageLocked( - BleImageOptimizer.TileImagePlan plan, - byte[] payload, - String fingerprint, - int updateId, - int messageNumber, - int messageCount - ) { - OutboundMessage message = messageBuilder.customMessage("image", payload, - "image " + plan.tile.name + "#" + messageNumber, plan.tileIndex, connectionOptions.sendImagesToLeft); - message.setImageUpdatePosition(updateId, messageNumber, messageCount); - message.onAck = () -> { - imageRetryAfterMs = 0; - // Firmware >= 2.2.4.34 resets its heartbeat timer when it receives - // image messages (not just heartbeats), so an acked image fragment - // satisfies the heartbeat deadline and heartbeats stop contending - // with active rendering. - lastHeartbeatAckedAtMs = SystemClock.elapsedRealtime(); - logImageUpdateAckLandmarkLocked(message); - boolean imageStillInFlight = false; - for (OutboundMessage inFlight : inFlightMessages) { - if ("image".equals(inFlight.kind)) { - imageStillInFlight = true; - break; - } - } - if (!imageStillInFlight) { - boolean imageStillQueued = false; - for (OutboundMessage queued : pendingMessages) { - if ("image".equals(queued.kind)) { - imageStillQueued = true; - break; - } - } - if (!imageStillQueued) { - displayedFingerprint = fingerprint; - } - } - }; - message.onTimeout = () -> { - discardImageUpdateStatsLocked(message.imageUpdateId, "image ack timeout (will retry)"); - clearMessagesOfKindLocked("image"); - displayedFingerprint = ""; - imageRetryAfterMs = SystemClock.elapsedRealtime() + ConnectionOptions.IMAGE_RETRY_DELAY_MS; - }; - pendingMessages.addLast(message); - } - - private OutboundMessage createAudioControlMessageLocked(boolean enable) { - OutboundMessage message = messageBuilder.enableOrDisableMic(enable); - message.onAck = () -> { - lastAudioControlAckMagic = message.magic; - audioCaptureActive = message.label != null && message.label.contains("enable"); - logLine(audioCaptureActive ? "G2 mic enabled" : "G2 mic disabled"); - }; - message.onTimeout = () -> { - handleTransportFailure("audio control ack timeout"); - }; - return message; - } - - private boolean shouldPollBatteryLocked(long now) { - return !shutdownRequested - && sessionReady - && pendingMessages.isEmpty() - && inFlightMessages.isEmpty() - && now - lastConnectionOrInputAtMs >= ConnectionOptions.BATTERY_INPUT_QUIET_MS - && (lastBatteryRefreshAtMs == 0 || now - lastBatteryRefreshAtMs >= ConnectionOptions.BATTERY_REFRESH_INTERVAL_MS); - } - - private OutboundMessage createBatteryQueryMessageLocked() { - OutboundMessage message = messageBuilder.batteryQuery(); - message.onAck = () -> { - BleProtocol.RingBatterySnapshot ring = BleProtocol.parseRingBattery(message.ackPayload); - // Old firmware and missing/malformed extensions must clear any prior reading. - ringBattery = ring == null ? -1 : ring.battery; - ringCharging = ring == null ? -1 : ring.charging; - BleProtocol.BatterySnapshot snapshot = BleProtocol.parseSettingsBattery(message.ackPayload); - if (snapshot != null) { - headsetBattery = snapshot.battery; - headsetCharging = snapshot.charging; - emitBatteryState(headsetBattery, headsetCharging); - if (snapshot.silentMode >= 0) { - // Backstop for the push in onNotification: the firmware is - // confirmed to push silent-mode-on, but the off transition is - // not, so re-read the authoritative value on every poll. - updateSilentModeLocked(snapshot.silentMode > 0); - } - updateChargingModeLocked(snapshot.charging > 0, snapshot.battery); - } - BleProtocol.FirmwareInfo firmwareInfo = BleProtocol.parseSettingsFirmwareInfo(message.ackPayload); - if (firmwareInfo != null) { - customFirmwareDetected = firmwareInfo.isFaceclawFirmware(); - emitFirmwareInfo(firmwareInfo); - } - }; - message.onTimeout = () -> { - logLine("Battery query timed out"); - }; - return message; - } - - /** - * Track silent mode, which the wearer toggles by long-pressing both - * touchpads at once. While it is on the firmware refuses input events and - * app launches and powers the display down, so the glasses look dead even - * though the BLE session is healthy; the phone UI says so explicitly. - */ - private void updateSilentModeLocked(boolean silent) { - int next = silent ? 1 : 0; - if (silentMode == next) { - return; - } - silentMode = next; - logLine(silent ? "glasses entered silent mode" : "glasses left silent mode"); - emitSilentMode(silent); - } - - /** - * Track whether the glasses are in the charging case. Charging means nobody - * is wearing them: display communication pauses (no heartbeats, so the - * firmware tears down its EvenHub context on its own) and only battery polls - * continue. When charging stops, tear the transport down and let the normal - * reconnect loop rebuild the session, layout, and first frame. - */ - private void updateChargingModeLocked(boolean charging, int battery) { - if (charging == chargingMode) { - if (chargingMode) { - setStateDisplay("charging", chargingStatusText(battery)); - } - return; - } - if (charging) { - chargingMode = true; - clearAllMessagesLocked("glasses charging"); - fixedLayoutCreated = false; - startupProbePending = false; - displayedFingerprint = ""; - finishDesiredFrameLocked("discarded: glasses charging"); - logLine("glasses are charging; pausing display communication"); - setStateDisplay("charging", chargingStatusText(battery)); - } else { - chargingMode = false; - logLine("glasses removed from charger; reconnecting"); - handleTransportFailure("charging ended"); - } - } - - private static String chargingStatusText(int battery) { - return battery >= 0 - ? "Glasses charging. Battery " + battery + "%." - : "Glasses charging."; - } - - /** - * Record, into the frame that is waiting, why it did not go out on this - * pass of the send loop. Without this the export shows a bare multi-second - * jump between "image submitted as desired frame" and the first BLE - * packet, with no hint whether we were blocked on a heartbeat, the - * BLE window, or another message queued ahead. Deduped on (frame, reason), - * so a frame stalled for seconds gets one line per state change rather - * than one per loop pass. - */ - private void noteImageStallLocked(long now, boolean windowHasRoom) { - int frameId; - synchronized (desiredTilesLock) { - frameId = desiredFrameId; - } - String reason; - if (frameId != 0 && !getDesiredFingerprint().equals(lastEnqueuedFingerprint)) { - reason = describeEnqueueBlockerLocked(now, windowHasRoom); - } else { - // Nothing waiting to be planned; an already-planned image may still - // be queued behind other traffic. - OutboundMessage queuedImage = firstPendingImageLocked(); - frameId = queuedImage == null ? 0 : imageUpdateFrameIdLocked(queuedImage.imageUpdateId); - reason = queuedImage == null ? null : describeWriteBlockerLocked(now, windowHasRoom, queuedImage); - } - if (frameId == 0 || reason == null) { - stallFrameId = 0; - stallReason = ""; - return; - } - if (frameId == stallFrameId && reason.equals(stallReason)) { - return; - } - stallFrameId = frameId; - stallReason = reason; - FrameTimings.getInstance().log(frameId, "waiting to send: " + reason); - } - - /** Why the desired composite has not been turned into wire messages yet, or null. */ - private String describeEnqueueBlockerLocked(long now, boolean windowHasRoom) { - if (shutdownRequested) { - return "shutdown requested"; - } - if (!sessionReady) { - return "BLE session not ready"; - } - if (!fixedLayoutCreated) { - return "display layout not created yet"; - } - if (now < imageRetryAfterMs) { - return "image retry backoff (" + (imageRetryAfterMs - now) + "ms left)"; - } - if (hasPendingImageLocked()) { - return "an earlier image is still queued"; - } - if (!windowHasRoom) { - return "BLE window full (" + inFlightMessages.size() + " message(s) in flight)"; - } - if (hasPendingOrInflightKindLocked("heartbeat")) { - return "heartbeat in flight"; - } - if (!pendingMessages.isEmpty()) { - return pendingMessages.size() + " message(s) queued ahead, next " - + pendingMessages.peekFirst().label; - } - return null; - } - - /** Why a planned image message has not been written to BLE yet, or null. */ - private String describeWriteBlockerLocked(long now, boolean windowHasRoom, OutboundMessage queuedImage) { - if (!sessionReady) { - return "BLE session not ready"; - } - if (!windowHasRoom) { - return "BLE window full (" + inFlightMessages.size() + " message(s) in flight)"; - } - if (hasPendingOrInflightKindLocked("heartbeat")) { - return "heartbeat in flight"; - } - // handleHeartbeat is a barrier: while one is due it holds back every - // other write, so a frame queued at the wrong moment waits a heartbeat - // round trip. Reported explicitly because it is otherwise invisible -- - // heartbeats belong to no frame. - long heartbeatElapsedMs = now - lastHeartbeatAckedAtMs; - if (fixedLayoutCreated && !shutdownRequested - && heartbeatElapsedMs >= ConnectionOptions.HEARTBEAT_READY_MS) { - return "heartbeat due (" + heartbeatElapsedMs + "ms since the last one acked)"; - } - OutboundMessage head = pendingMessages.peekFirst(); - if (head != null && head != queuedImage) { - return "queued behind " + head.label; - } - return null; - } - - private OutboundMessage firstPendingImageLocked() { - for (OutboundMessage message : pendingMessages) { - if (message.imageUpdateId > 0) { - return message; - } - } - return null; - } - - private int imageUpdateFrameIdLocked(int imageUpdateId) { - BleImageOptimizer.ImageUpdateStats stats = imageUpdateStats.get(imageUpdateId); - return stats == null ? 0 : stats.frameId; - } - - private void finishDesiredFrameLocked(String outcome) { - int frameIdToFinish; - synchronized (desiredTilesLock) { - frameIdToFinish = desiredFrameId; - desiredFrameId = 0; - } - finishFrame(frameIdToFinish, outcome); - } - - private boolean hasPendingOrInflightKindLocked(String kind) { - for (OutboundMessage queued : pendingMessages) { - if (kind.equals(queued.kind)) { - return true; - } - } - for (OutboundMessage inFlight : inFlightMessages) { - if (kind.equals(inFlight.kind)) { - return true; - } - } - return false; - } - - private boolean hasPendingOrInflightMagicLocked(int magic) { - for (OutboundMessage queued : pendingMessages) { - if (queued.magic == magic) { - return true; - } - } - for (OutboundMessage inFlight : inFlightMessages) { - if (inFlight.magic == magic) { - return true; - } - } - return false; - } - - private boolean hasPendingMagicLocked(int sid, int magic) { - for (OutboundMessage queued : pendingMessages) { - if (queued.sid == sid && queued.magic == magic) { - return true; - } - } - return false; - } - - private void handleAckTimeoutLocked(OutboundMessage message) { - consecutiveAckTimeouts += 1; - - if (message.onTimeout != null) { - message.onTimeout.run(); - } - - if (consecutiveAckTimeouts > ConnectionOptions.MAX_CONSECUTIVE_ACK_TIMEOUTS) { - handleTransportFailure("too many ack timeouts"); - } - } - - /** - * Replace any stale lease control with one right-arm and one left-arm - * fire-and-forget write. With priority=true the right arm is sent first so - * CLAIM reaches the lens that originated the deferred wake immediately. - */ - private int enqueueFaceclawWakeControlLocked(int operation, int nonce, boolean priority) { - clearMessagesOfKindLocked("wake-lease-control"); - final int generation = ++faceclawWakeControlGeneration; - faceclawWakeControlSentCount = 0; - if (operation == BleProtocol.FACECLAW_WAKE_OP_ACQUIRE) { - lastFaceclawWakeLeaseQueuedAtMs = SystemClock.elapsedRealtime(); - } - MessageCallback onSent = () -> { - if (faceclawWakeControlGeneration == generation) { - faceclawWakeControlSentCount += 1; - } - }; - OutboundMessage right = messageBuilder.faceclawWakeControl(operation, nonce, false); - OutboundMessage left = messageBuilder.faceclawWakeControl(operation, nonce, true); - right.onSent = onSent; - left.onSent = onSent; - if (priority) { - pendingMessages.addFirst(left); - pendingMessages.addFirst(right); - } else { - pendingMessages.addLast(right); - pendingMessages.addLast(left); - } - logLine("queue " + right.label + " + L"); - return generation; - } - - private boolean waitForFaceclawWakeControlDelivery(int generation, long timeoutMs) { - long deadline = SystemClock.elapsedRealtime() + Math.max(0, timeoutMs); - synchronized (lock) { - while (running - && sessionReady - && faceclawWakeControlGeneration == generation - && faceclawWakeControlSentCount < 2) { - long remaining = deadline - SystemClock.elapsedRealtime(); - if (remaining <= 0) { - break; - } - try { - lock.wait(Math.min(remaining, 100)); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - break; - } - } - return faceclawWakeControlGeneration == generation - && faceclawWakeControlSentCount >= 2; - } - } - - /** - * Acquire/renew or release CFW's independent direct-framebuffer repaint - * guard on both arms. It is separate from the optional idle-wake lease: - * every Faceclaw display session needs this guard while its EvenHub layout - * contains swipe-capturing stock widgets. - */ - private int enqueueFaceclawFramebufferControlLocked(int operation, boolean priority) { - clearMessagesOfKindLocked("framebuffer-lease-control"); - final int generation = ++faceclawFramebufferControlGeneration; - faceclawFramebufferControlSentCount = 0; - if (operation == BleProtocol.FACECLAW_FB_OP_ACQUIRE) { - lastFaceclawFramebufferLeaseQueuedAtMs = SystemClock.elapsedRealtime(); - } - MessageCallback onSent = () -> { - if (faceclawFramebufferControlGeneration == generation) { - faceclawFramebufferControlSentCount += 1; - } - }; - OutboundMessage right = messageBuilder.faceclawFramebufferControl(operation, false); - OutboundMessage left = messageBuilder.faceclawFramebufferControl(operation, true); - right.onSent = onSent; - left.onSent = onSent; - if (priority) { - pendingMessages.addFirst(left); - pendingMessages.addFirst(right); - } else { - pendingMessages.addLast(right); - pendingMessages.addLast(left); - } - logLine("queue " + right.label + " + L"); - return generation; - } - - private boolean waitForFaceclawFramebufferControlDelivery(int generation, long timeoutMs) { - long deadline = SystemClock.elapsedRealtime() + Math.max(0, timeoutMs); - synchronized (lock) { - while (running - && sessionReady - && faceclawFramebufferControlGeneration == generation - && faceclawFramebufferControlSentCount < 2) { - long remaining = deadline - SystemClock.elapsedRealtime(); - if (remaining <= 0) { - break; - } - try { - lock.wait(Math.min(remaining, 100)); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - break; - } - } - return faceclawFramebufferControlGeneration == generation - && faceclawFramebufferControlSentCount >= 2; - } - } - - private boolean releaseFaceclawFramebufferLease() { - int generation; - synchronized (lock) { - if (!running || !sessionReady) { - return true; - } - generation = enqueueFaceclawFramebufferControlLocked( - BleProtocol.FACECLAW_FB_OP_RELEASE, - true - ); - } - interruptibleSleep.interrupt(); - return waitForFaceclawFramebufferControlDelivery( - generation, - FACECLAW_WAKE_CONTROL_WAIT_MS - ); - } - - private void clearMessagesOfKindLocked(String kind) { - Iterator pendingIterator = pendingMessages.iterator(); - while (pendingIterator.hasNext()) { - OutboundMessage message = pendingIterator.next(); - if (kind.equals(message.kind)) { - pendingIterator.remove(); - discardPrewriteIfMatchesLocked(message); - if ("image".equals(kind)) { - discardImageUpdateStatsLocked(message.imageUpdateId, "pending messages cleared (" + kind + ")"); - } - magicPool.release(message.sid, message.magic, message.label, "cleared pending " + kind); - } - } - Iterator inFlightIterator = inFlightMessages.iterator(); - while (inFlightIterator.hasNext()) { - OutboundMessage message = inFlightIterator.next(); - if (kind.equals(message.kind)) { - inFlightIterator.remove(); - if ("image".equals(kind)) { - discardImageUpdateStatsLocked(message.imageUpdateId, "inflight messages cleared (" + kind + ")"); - } - magicPool.release(message.sid, message.magic, message.label, "cleared inflight " + kind); - } - } - if ("image".equals(kind)) { - // The image pipeline was flushed (e.g. ack timeout -> keyframe resync): - // drop the pipelined delta base so the next image is a full keyframe. - lastEnqueuedPacked = new byte[0]; - lastEnqueuedFingerprint = ""; - } - } - - private void clearAllMessagesLocked(String reason) { - clearPendingMessagesLocked(reason); - clearInFlightMessagesLocked(reason); - // The image pipeline is gone: drop the pipelined delta base so the next - // image is a full keyframe rather than a delta onto a stale base. - lastEnqueuedPacked = new byte[0]; - lastEnqueuedFingerprint = ""; - // Queued resource commands (if any) were dropped with the rest, and the - // session churn behind a full clear may have freed the on-glasses - // cache; forget it phone-side so glyphs re-upload lazily. - resourceCache.reset(); - } - - /** - * A custom wake queues CLAIM before NativeScript asks for a resume. Keep - * that private control while flushing stale EvenHub traffic around the - * direct prelude write. - */ - private void clearAllMessagesPreservingWakeLeaseLocked(String reason) { - Iterator pendingIterator = pendingMessages.iterator(); - while (pendingIterator.hasNext()) { - OutboundMessage message = pendingIterator.next(); - if ("wake-lease-control".equals(message.kind)) { - continue; - } - pendingIterator.remove(); - discardPrewriteIfMatchesLocked(message); - discardImageUpdateStatsLocked( - message.imageUpdateId, - "pending messages cleared: " + reason - ); - magicPool.release( - message.sid, - message.magic, - message.label, - "cleared pending: " + reason - ); - } - clearInFlightMessagesLocked(reason); - lastEnqueuedPacked = new byte[0]; - lastEnqueuedFingerprint = ""; - resourceCache.reset(); - } - - /** Any image update whose fragments are still queued (not yet sent). */ - private boolean hasPendingImageLocked() { - for (OutboundMessage message : pendingMessages) { - if ("image".equals(message.kind)) { - return true; - } - } - return false; - } - - private void clearPendingMessagesLocked(String reason) { - while (!pendingMessages.isEmpty()) { - var message = pendingMessages.removeFirst(); - discardPrewriteIfMatchesLocked(message); - discardImageUpdateStatsLocked(message.imageUpdateId, "pending messages cleared: " + reason); - magicPool.release(message.sid, message.magic, message.label, "cleared pending: " + reason); - } - } - - private void clearInFlightMessagesLocked(String reason) { - while (!inFlightMessages.isEmpty()) { - var message = inFlightMessages.removeFirst(); - discardImageUpdateStatsLocked(message.imageUpdateId, "inflight messages cleared: " + reason); - magicPool.release(message.sid, message.magic, message.label, "cleared inflight: " + reason); - } - } - - private void discardPrewriteIfMatchesLocked(OutboundMessage message) { - if (message != null && message == prewrittenMessage) { - prewrittenMessage = null; - prewrittenFrames = Collections.emptyList(); - } - } - - private void recordUnexpectedAckLocked(int sid, int magic) { - if (magic < BleMagicPool.MIN_MAGIC || magic > BleMagicPool.MAX_MAGIC) { - return; - } - BleMagicPool.ReleaseRecord previous = magicPool.getReleaseRecord(sid, magic); - if (previous == null) { - String pendingNote = hasPendingMagicLocked(sid, magic) ? " while that magic is only pending locally" : ""; - logLine("unexpected ACK sid=" + sid + " magic=" + magic + pendingNote - + "; possible Even app BLE contention"); - return; - } - if ("timeout".equals(previous.reason)) { - logLine("late ACK after timeout sid=" + sid + " magic=" + magic - + " label=" + previous.label - + "; ACK timeout may be too short"); - return; - } - if ("ack".equals(previous.reason)) { - logLine("duplicate ACK for already-acked message sid=" + sid + " magic=" + magic - + " label=" + previous.label - + "; possible Even app BLE contention"); - return; - } - logLine("late ACK for released message sid=" + sid + " magic=" + magic - + " label=" + previous.label - + " release=" + previous.reason); - } - - /** The address of a configured arm Android no longer holds a bond for, or null. */ - private String firstUnpairedArm() { - if (!bleManager.isBonded(rightAddress)) return rightAddress; - if (!bleManager.isBonded(leftAddress)) return leftAddress; - return null; - } - - /** - * A connect failure while an arm's bond is missing (the pairing was - * forgotten in Android settings, or the address was typed in by hand and - * never paired) will repeat forever, so instead of scheduling a retry, - * park the worker loop and tell the user to re-pair. Only an explicit - * connect (which builds a fresh communicator) starts a new attempt. - */ - private void handleUnpairedFailure(String address) { - Log.e(TAG, "Connect failed and " + address + " is not paired; suspending reconnect"); - synchronized (lock) { - reconnectHalted = true; - sessionReady = false; - fixedLayoutCreated = false; - startupProbePending = false; - shutdownRequested = false; - chargingMode = false; - imageRetryAfterMs = 0; - displayedFingerprint = ""; - faceclawWakePendingNonce = -1; - lastFaceclawWakeLeaseQueuedAtMs = 0; - faceclawWakeControlSentCount = 0; - clearAllMessagesLocked("arm not paired: " + address); - reconnectAfterMs = Long.MAX_VALUE; - bleManager.disconnect(rightAddress); - bleManager.disconnect(leftAddress); - } - if (!userDisconnectRequested) { - setStateDisplay( - "unpaired", - "The glasses (" + address + ") are not paired with this phone." - + " Use \"Pair glasses\" to pair them again, then connect." - ); - } - interruptibleSleep.interrupt(); - } - - private void handleTransportFailure(String reason) { - Log.e(TAG, "Transport failure: "+reason); - synchronized (lock) { - maybeEmitEvenAppConflictLocked(reason); - finishBenchmarkLocked(true, "transport failure"); - sessionReady = false; - fixedLayoutCreated = false; - startupProbePending = false; - shutdownRequested = false; - chargingMode = false; - imageRetryAfterMs = 0; - displayedFingerprint = ""; - faceclawWakePendingNonce = -1; - lastFaceclawWakeLeaseQueuedAtMs = 0; - faceclawWakeControlSentCount = 0; - clearAllMessagesLocked("transport failure: " + reason); - reconnectAfterMs = SystemClock.elapsedRealtime() + ConnectionOptions.RECONNECT_DELAY_MS; - bleManager.disconnect(rightAddress); - bleManager.disconnect(leftAddress); - } - if (!userDisconnectRequested) { - setStateDisplay("retrying", reason == null || reason.isEmpty() ? "Reconnecting..." : "Reconnecting after " + reason); - } - interruptibleSleep.interrupt(); - } - - private void resetSessionStateLocked() { - ringBattery = -1; - ringCharging = -1; - sessionReady = false; - shutdownRequested = false; - fixedLayoutCreated = false; - chargingMode = false; - rightConnected = false; - leftConnected = false; - ringConnected = false; - ringNotificationsReady = false; - reconnectAfterMs = 0; - reconnectHalted = false; - ringReconnectAfterMs = 0; - lastAckAtMs = 0; - lastIncomingAtMs = 0; - lastHeartbeatSentAtMs = 0; - lastSessionReadyAtMs = 0; - consecutiveAckTimeouts = 0; - lastAudioControlAckMagic = 0; - audioCaptureActive = false; - audioPacketListener = null; - compassControlLastSent = -1; - // A dead transport orphans any glasses-side compass state; the fresh - // session re-asserts the desired state once its layout is ready. - compassMaybeOn = false; - faceclawWakePendingNonce = -1; - lastFaceclawWakeLeaseQueuedAtMs = 0; - faceclawWakeControlSentCount = 0; - cfwCleanupDelivered = false; - for (CfwTransport transport : cfwTransports) transport.reset(); - lastCfwCleanupAckMagic = 0; - wearState = -1; - displayedFingerprint = ""; - // Deliberately not clearing silentMode: it is a property of the glasses, - // not of our session, and silent mode blocks app launches, so it can be - // the very cause of the session teardown that got us here. - } - - private void emitRingEvent(G2Event event, int frameId) { - final FaceclawBleCommunicatorListener current = listener; - if (current == null) { - FrameTimings.getInstance().finishFrame(frameId, "discarded: no listener attached"); - return; - } - final String containerNameSnapshot = event.containerName == null ? "" : event.containerName; - mainHandler.post(() -> { - FrameTimings.getInstance().log(frameId, "dispatching input event on main thread"); - try { - current.onRingEvent(event.kind, containerNameSnapshot, event.eventType, event.eventSource, event.systemExitReasonCode, frameId, - event.ringTick, event.ringType, event.ringAux, event.ringSpeed); - } catch (Throwable t) { - Log.w(TAG, "listener onRingEvent failed", t); - FrameTimings.getInstance().finishFrame(frameId, "discarded: listener onRingEvent failed"); - } - }); - } - - /** Finish a frame owned by the communicator and tell the TS side, which may be awaiting it. */ - private void finishFrame(int frameId, String outcome) { - if (frameId <= 0) { - return; - } - FrameTimings.getInstance().finishFrame(frameId, outcome); - final FaceclawBleCommunicatorListener current = listener; - if (current == null) { - return; - } - mainHandler.post(() -> { - try { - current.onFrameFinished(frameId, outcome); - } catch (Throwable t) { - Log.w(TAG, "listener onFrameFinished failed", t); - } - }); - } - - private void emitImuData(double x, double y, double z, int eventSource) { - if (imuListeners.isEmpty()) { - return; - } - mainHandler.post(() -> { - for (FaceclawImuListener imuListener : imuListeners) { - try { - imuListener.onImuData(x, y, z, eventSource); - } catch (Throwable t) { - Log.w(TAG, "listener onImuData failed", t); - } - } - }); - } - - private void emitCompassEvent(BleProtocol.CompassEvent event) { - if (event.diagnosticFlags >= 0) { - String[] sources = { "unknown", "GRV", "GMRV", "RV" }; - Log.i("FaceclawCompass", "heading=" + event.headingDegrees - + " magneticAccuracy=" + event.magneticAccuracy - + " magneticAnomalies=" + event.magneticAnomalies - + " orientationSource=" + sources[event.orientationSource] - + " flags=0x" + Integer.toHexString(event.diagnosticFlags) - + " sampleTimeMs=" + event.sampleTimeMs); - } - for (CompassSubscription subscription : compassSubscriptions) { - subscription.handler.post(() -> { - try { - subscription.listener.onCompassEvent(event.command, event.headingDegrees, - event.magneticAccuracy, event.magneticAnomalies, event.orientationSource, - event.diagnosticFlags, event.sampleTimeMs); - } catch (Throwable t) { - Log.w(TAG, "listener onCompassEvent failed", t); - } - }); - } - } - - private void emitSilentMode(boolean silent) { - final FaceclawBleCommunicatorListener current = listener; - if (current == null) { - return; - } - mainHandler.post(() -> { - try { - current.onSilentMode(silent); - } catch (Throwable t) { - Log.w(TAG, "listener onSilentMode failed", t); - } - }); - } - - private void emitWearState(boolean wearing) { - final FaceclawBleCommunicatorListener current = listener; - if (current == null) return; - mainHandler.post(() -> { - try { - current.onWearState(wearing); - } catch (Throwable t) { - Log.w(TAG, "listener onWearState failed", t); - } - }); - } - - private void emitPhoneLockStateIfChanged(boolean force) { - final boolean locked; - synchronized (lock) { - long now = SystemClock.elapsedRealtime(); - if (!force && now - lastPhoneLockCheckAtMs < 1_000) return; - lastPhoneLockCheckAtMs = now; - locked = keyguardManager != null && keyguardManager.isDeviceLocked(); - int value = locked ? 1 : 0; - if (!force && value == phoneLockState) return; - phoneLockState = value; - } - final FaceclawBleCommunicatorListener current = listener; - if (current == null) return; - mainHandler.post(() -> { - try { - current.onPhoneLockState(locked); - } catch (Throwable t) { - Log.w(TAG, "listener onPhoneLockState failed", t); - } - }); - } - - private void emitBatteryState(int headsetBattery, int headsetCharging) { - final int reportedRingBattery = ringBattery; - final int reportedRingCharging = ringCharging; - final FaceclawBleCommunicatorListener current = listener; - if (current == null) { - return; - } - mainHandler.post(() -> { - try { - current.onBatteryState(headsetBattery, headsetCharging, reportedRingBattery, reportedRingCharging); - } catch (Throwable t) { - Log.w(TAG, "listener onBatteryState failed", t); - } - }); - } - - private void emitFirmwareInfo(BleProtocol.FirmwareInfo info) { - final FaceclawBleCommunicatorListener current = listener; - if (current == null) { - return; - } - mainHandler.post(() -> { - try { - current.onFirmwareInfo(info.leftVersion, info.rightVersion, info.extension); - } catch (Throwable t) { - Log.w(TAG, "listener onFirmwareInfo failed", t); - } - }); - } - - private void emitFrameMetrics(int paintMs, int transmitMs, int tileCount) { - final FaceclawBleCommunicatorListener current = listener; - if (current == null) { - return; - } - mainHandler.post(() -> { - try { - current.onFrameMetrics(paintMs, transmitMs, tileCount); - } catch (Throwable t) { - Log.w(TAG, "listener onFrameMetrics failed", t); - } - }); - } - - private void maybeEmitEvenAppConflictLocked(String reason) { - if (!"write failed".equals(reason)) { - return; - } - long now = SystemClock.elapsedRealtime(); - if (lastSessionReadyAtMs <= 0 || now - lastSessionReadyAtMs > ConnectionOptions.EVEN_APP_WRITE_FAILURE_WINDOW_MS) { - return; - } - if (lastEvenAppConflictAtMs > 0 && now - lastEvenAppConflictAtMs < 60_000) { - return; - } - if (!FaceclawEvenAppDetector.isEvenNotificationActive(appContext)) { - return; - } - lastEvenAppConflictAtMs = now; - emitEvenAppConflict("The Even Realities app still appears to be running. It can hold the glasses BLE link and cause Faceclaw write failures. Open its app settings and force stop it, then reconnect Faceclaw."); - } - - private void emitEvenAppConflict(String message) { - final FaceclawBleCommunicatorListener current = listener; - if (current == null) { - return; - } - final String messageSnapshot = message == null ? "" : message; - mainHandler.post(() -> { - try { - current.onEvenAppConflict(messageSnapshot); - } catch (Throwable t) { - Log.w(TAG, "listener onEvenAppConflict failed", t); - } - }); - } - - private void setStateDisplay(String nextPhase, String nextStatus) { - synchronized (lock) { - phase = nextPhase; - status = nextStatus; - } - emitState(); - } - - private void emitState() { - final FaceclawBleCommunicatorListener current = listener; - if (current == null) { - return; - } - final String phaseSnapshot; - final String statusSnapshot; - synchronized (lock) { - phaseSnapshot = phase; - statusSnapshot = status; - } - mainHandler.post(() -> { - try { - current.onStateChange(phaseSnapshot, statusSnapshot); - } catch (Throwable t) { - Log.w(TAG, "listener onStateChange failed", t); - } - }); - } - - private void updateG2ScreenWakeLock(boolean screenOn) { - if (screenOn) { - if (g2ScreenWakeLock == null) { - g2ScreenWakeLock = powerManager.newWakeLock(PowerManager.PARTIAL_WAKE_LOCK, G2_SCREEN_WAKE_LOCK_TAG); - g2ScreenWakeLock.setReferenceCounted(false); - } - if (!g2ScreenWakeLock.isHeld()) { - g2ScreenWakeLock.acquire(); - logLine("G2 screen wake lock acquired"); - } - return; - } - releaseG2ScreenWakeLock(); - } - - private void releaseG2ScreenWakeLock() { - if (g2ScreenWakeLock != null && g2ScreenWakeLock.isHeld()) { - g2ScreenWakeLock.release(); - logLine("G2 screen wake lock released"); - } - } - - private void logLine(String line) { - Log.i(TAG, line); - } - - private static String timestamp(long elapsedMs) { - long wallMs = System.currentTimeMillis(); - return String.format(Locale.US, "%tF %tT.%tL elapsed=%dms", wallMs, wallMs, wallMs, elapsedMs); - } - - private int nextMapSessionId() { - int id = nextMapSessionId; - int increment = connectionOptions.skipSessionIds ? 2 : 1; - nextMapSessionId = (nextMapSessionId + increment) & 0xff; - return id; - } - - private static String requireAddress(String name, String address) { - if (address == null || address.trim().isEmpty()) { - throw new IllegalArgumentException(name + " is required"); - } - return address.trim(); - } - - private boolean hasRingAddress() { - return ringAddress != null && !ringAddress.trim().isEmpty(); - } - - private boolean isConfiguredRingAddress(String address) { - return hasRingAddress() && address != null && address.equalsIgnoreCase(ringAddress); - } - - private boolean isDirectRingNotification(String address, String characteristicUuid) { - if (!isConfiguredRingAddress(address) || characteristicUuid == null) { - return false; - } - return BleProtocol.R1_PHONE_NOTIFY_CHAR_UUID.equals(characteristicUuid) - || BleProtocol.R1_NOTIFY_CHAR_UUID.equals(characteristicUuid); - } - - private static String hex(byte[] data) { - if (data == null || data.length == 0) { - return ""; - } - char[] out = new char[data.length * 2]; - char[] digits = "0123456789abcdef".toCharArray(); - for (int i = 0; i < data.length; i++) { - int value = data[i] & 0xff; - out[i * 2] = digits[value >>> 4]; - out[i * 2 + 1] = digits[value & 0x0f]; - } - return new String(out); - } - - private static String safeMessage(Throwable t) { - if (t == null) { - return "unknown"; - } - StringWriter writer = new StringWriter(); - t.printStackTrace(new PrintWriter(writer)); - String trace = writer.toString(); - if (!trace.trim().isEmpty()) { - return trace; - } - String message = t.getMessage(); - return message == null || message.trim().isEmpty() ? String.valueOf(t) : message; - } - - private String getDesiredFingerprint() { - synchronized (desiredTilesLock) { - return desiredFingerprint; - } - } -} diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawBleCommunicator.kt b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawBleCommunicator.kt new file mode 100644 index 00000000..97e8d343 --- /dev/null +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawBleCommunicator.kt @@ -0,0 +1,4072 @@ +package com.faceclaw.app + +import android.annotation.SuppressLint +import android.bluetooth.BluetoothGatt +import android.app.KeyguardManager +import android.content.BroadcastReceiver +import android.content.Context +import android.content.Intent +import android.content.IntentFilter +import android.os.Handler +import android.os.Looper +import android.os.PowerManager +import android.os.SystemClock +import android.util.Log + +import java.io.PrintWriter +import java.io.StringWriter +import java.util.ArrayDeque +import java.util.ArrayList +import java.util.Arrays +import java.util.Collections +import java.util.HashMap +import java.util.Locale +import java.util.Random + +@SuppressLint("MissingPermission") +class FaceclawBleCommunicator(context: Context, rightAddress: String?, leftAddress: String?, ringAddress: String?) : FaceclawBleListener, Runnable { + companion object { + private const val TAG = "FaceclawComm" + + // Local metadata for custom-command bookkeeping, not an EvenHub container. + // Submitted frames supply pixel geometry; the stock layout only captures input. + private val DASHBOARD_TILE: BleProtocol.ImageTileOptions = + BleProtocol.ImageTileOptions("img00", 10, 0, 0, 576, 288) + + private const val G2_SCREEN_WAKE_LOCK_TAG = "Faceclaw:G2Screen" + private const val FACECLAW_WAKE_LEASE_RENEW_MS = 45_000L + private const val FACECLAW_WAKE_CONTROL_WAIT_MS = 1_500L + private const val CFW_CLEANUP_WAIT_MS = 4_000L + private const val COMPASS_REPORT_INTERVAL_MS = 100 + private const val COMPASS_MIN_CHANGE_DEGREES = 0 + + // A timed-out benchmark message aborts the run, but its already-in-flight + // peers still time out one by one; keep the window comfortably below + // MAX_CONSECUTIVE_ACK_TIMEOUTS so a dead run can't escalate into a + // transport-failure reconnect all by itself. + private const val BENCHMARK_MAX_WINDOW = 6 + + // The most recently started communicator; lets app worker threads submit + // surface frames without holding a cross-isolate reference to the bridge + // object (JS wrappers do not cross isolates, but the Java instance does). + @Volatile private var activeInstance: FaceclawBleCommunicator? = null + + @JvmStatic + fun getActive(): FaceclawBleCommunicator? { + return activeInstance + } + + private fun chargingStatusText(battery: Int): String { + return if (battery >= 0) + "Glasses charging. Battery $battery%." + else + "Glasses charging." + } + + private fun timestamp(elapsedMs: Long): String { + val wallMs = System.currentTimeMillis() + return String.format(Locale.US, "%tF %tT.%tL elapsed=%dms", wallMs, wallMs, wallMs, elapsedMs) + } + + private fun requireAddress(name: String, address: String?): String { + if (address == null || address.trim().isEmpty()) { + throw IllegalArgumentException("$name is required") + } + return address.trim() + } + + private fun hex(data: ByteArray?): String { + if (data == null || data.isEmpty()) { + return "" + } + val out = CharArray(data.size * 2) + val digits = "0123456789abcdef".toCharArray() + for (i in data.indices) { + val value = data[i].toInt() and 0xff + out[i * 2] = digits[value ushr 4] + out[i * 2 + 1] = digits[value and 0x0f] + } + return String(out) + } + + private fun safeMessage(t: Throwable?): String { + if (t == null) { + return "unknown" + } + val writer = StringWriter() + t.printStackTrace(PrintWriter(writer)) + val trace = writer.toString() + if (trace.trim().isNotEmpty()) { + return trace + } + val message = t.message + return if (message == null || message.trim().isEmpty()) t.toString() else message + } + } + + private val appContext: Context = context.applicationContext + init { + FrameTimings.getInstance().init(appContext) + } + private val powerManager: PowerManager = appContext.getSystemService(Context.POWER_SERVICE) as PowerManager + private val keyguardManager: KeyguardManager? = appContext.getSystemService(Context.KEYGUARD_SERVICE) as KeyguardManager? + private val bleManager: FaceclawBleManager = FaceclawBleManager(appContext) + private val interruptibleSleep = InterruptibleSleep() + private val lock = java.lang.Object() + private val mainHandler = Handler(Looper.getMainLooper()) + private val rightAddress: String = requireAddress("rightAddress", rightAddress) + private val leftAddress: String = requireAddress("leftAddress", leftAddress) + private val ringAddress: String = ringAddress?.trim() ?: "" + + @Volatile private var listener: FaceclawBleCommunicatorListener? = null + private val imuListeners: MutableList = + java.util.concurrent.CopyOnWriteArrayList() + /** A compass subscriber plus the Looper it registered from (see addCompassListener). */ + private class CompassSubscription( + @JvmField val listener: FaceclawCompassListener, + @JvmField val handler: Handler, + ) + + private val compassSubscriptions: MutableList = + java.util.concurrent.CopyOnWriteArrayList() + private val ambientLightListeners: MutableList = + java.util.concurrent.CopyOnWriteArrayList() + private val micStatusListeners: MutableList = + java.util.concurrent.CopyOnWriteArrayList() + @Volatile private var workerThread: Thread? = null + @Volatile private var running = false + @Volatile private var userDisconnectRequested = false + // Set when a connect attempt failed while an arm's Android bond is gone: + // retrying is pointless until the user re-pairs, so the worker loop parks + // instead of redialing. Cleared by start() (a fresh explicit connect). + @Volatile private var reconnectHalted = false + + private var phase = "disconnected" + private var status = "Disconnected." + + private var rightConnected = false + private var leftConnected = false + private var ringConnected = false + private var ringNotificationsReady = false + private var sessionReady = false + private var fixedLayoutCreated = false + private var shutdownRequested = false + // CFW firmware-debug-flags overlay (mode 7). Desired value pushed from TS; the + // sub-op last sent this session (-1 = not yet), reset on (re)connect so the + // overlay state is re-asserted on every reconnect and whenever the value changes. + @Volatile private var firmwareDebugFlagsEnabled = false + private var firmwareDebugFlagsLastSent = -1 + // Desired CFW mode-10 compass state. It survives reconnects; lastSent is + // reset with each session so an open Compass window is re-asserted. + /** + * Who currently wants the stock compass running (the Compass window, the + * Navigate worker, ...). The magnetometer is one shared resource, so it + * stays on while any owner holds it and is released when the last lets go; + * this keeps one app's release from silently switching off another's feed. + */ + private val compassOwners: MutableSet = HashSet() + private var compassControlLastSent = -1 + // Whether the glasses-side compass may still be running: set when an enable + // is enqueued, cleared only when a disable is acked. Drives the forced + // disable sent ahead of an EvenHub shutdown/suspend, since a pending + // disable can be wiped by the shutdown's queue flush and the retry loop + // does not run while shutdownRequested (magnetometer left on = battery drain). + private var compassMaybeOn = false + private var startupProbePending = false + // Desired ownership of CFW's fail-open stock-wake lease (dashboard launch + // and Even AI foreground takeover). This survives a transport reconnect; + // the lease itself is volatile firmware state and is re-acquired once both + // arms are ready. + private var faceclawWakeLeaseEnabled = false + private var lastFaceclawWakeLeaseQueuedAtMs = 0L + private var faceclawWakeControlGeneration = 0 + private var faceclawWakeControlSentCount = 0 + private var lastFaceclawFramebufferLeaseQueuedAtMs = 0L + private var faceclawFramebufferControlGeneration = 0 + private var faceclawFramebufferControlSentCount = 0 + private var faceclawWakePendingNonce = -1 + /** + * The last firmware-info read said the glasses run Faceclaw's custom + * firmware. Gates the private modes (cleanup, resource cache, ...) so stock + * or third-party firmware never sees them; the TS side checks the actual + * revision and disconnects on a mismatch, so no per-feature gating is + * needed here. + */ + private var customFirmwareDetected = false + private var cfwCleanupDelivered = false + private var lastCfwCleanupAckMagic = 0 + + private var reconnectAfterMs = 0L + private var ringReconnectAfterMs = 0L + private var lastAckAtMs = 0L + private var lastIncomingAtMs = 0L + private var lastHeartbeatSentAtMs = 0L + private var lastHeartbeatAckedAtMs = 0L + private var lastConnectionOrInputAtMs = 0L + private var lastBatteryRefreshAtMs = 0L + private var imageRetryAfterMs = 0L + private var lastSessionReadyAtMs = 0L + private var lastEvenAppConflictAtMs = 0L + private var consecutiveAckTimeouts = 0 + private var lastAudioControlAckMagic = 0 + + private val connectionOptions = ConnectionOptions() + private val magicPool = BleMagicPool(AndroidProtocolPlatform) + private val messageBuilder = MessageBuilder(magicPool) + private var nextTransportSeq = 0x40 + private var nextMapSessionId = 0 + private var nextImageUpdateId = 1 + // Wire frame id for mode-3 deltas (CFW reorder/skip/dup diagnostic). uint16, + // advanced by 1 per emitted delta; kept in [1, 0xfffe] to avoid the CFW's + // 0xffff "empty" sentinel. + private var nextImageFrameId = 1 + private var lastShutdownAckMagic = 0 + private var lastShutdownExitAtMs = 0L + private var headsetBattery = -1 + private var headsetCharging = -1 + private var ringBattery = -1 + private var ringCharging = -1 + // Silent mode: 1 = on, 0 = off, -1 = not yet known. See updateSilentModeLocked. + private var silentMode = -1 + private var wearState = -1 + private var phoneLockState = -1 + private var lastPhoneLockCheckAtMs = 0L + private var phoneLockReceiverRegistered = false + private var audioCaptureActive = false + private var firmwareInfoQueried = false + // Glasses are in the charging case: nobody is wearing them, so display + // communication pauses and only battery polls flow (see driveSession). + private var chargingMode = false + @Volatile private var audioPacketListener: FaceclawAudioPacketListener? = null + private var g2ScreenWakeLock: PowerManager.WakeLock? = null + + // BLE bandwidth benchmark (Developer app). Streams no-op image payloads + // (CFW mode 7 with an unused sub-op: parsed, acked, and discarded — stock + // firmware likewise ignores unknown image modes) for a fixed duration with + // a selectable message size and pipeline window, then reports throughput. + // While active, desired-frame sends are held back and heartbeats are + // satisfied by the benchmark's own acks, so the stream is the only image + // traffic. All state below is guarded by `lock`; results are read with + // getBandwidthBenchmarkStatus() and survive until the next run starts. + private var benchmarkActive = false + private var benchmarkAborted = false + private val benchmarkRandom = Random() + private var benchmarkMessageSize = 0 + private var benchmarkWindowSize = 0 + private var benchmarkDurationMs = 0 + private var benchmarkLinkMode = 0 + private var benchmarkLinkPending = false + private var benchmarkReadyAtMs = 0L + private var benchmarkStartAtMs = 0L // first benchmark write; 0 until then + private var benchmarkDeadlineAtMs = 0L // start + duration; MAX_VALUE until first write + private var benchmarkLastAckAtMs = 0L + private var benchmarkEndAtMs = 0L // 0 while running; set when the run drains + private var benchmarkMessagesSent = 0 + private var benchmarkMessagesAcked = 0 + private var benchmarkTimeouts = 0 + private var benchmarkPayloadBytesAcked = 0L + private var benchmarkWireBytesAcked = 0L + + private val phoneLockReceiver: BroadcastReceiver = object : BroadcastReceiver() { + override fun onReceive(context: Context?, intent: Intent?) { + emitPhoneLockStateIfChanged(true) + interruptibleSleep.interrupt() + } + } + + private var displayedFingerprint = "" + // The frame the firmware shadow will hold once the current image pipeline + // drains: the most recently ENQUEUED image (headerless packed 4bpp, see + // BmpUtil.pack4bppFromGray8), which is the correct base for the next delta + // when frames are pipelined. Set at enqueue; cleared whenever the image + // pipeline is cleared (clearAllMessagesLocked / clearMessagesOfKindLocked + // "image"), so it is only ever read while it holds a valid current-session base. + private var lastEnqueuedPacked: ByteArray = ByteArray(0) + private var lastEnqueuedWidth = 0 + private var lastEnqueuedHeight = 0 + private var lastEnqueuedFingerprint = "" + private val imageUpdateStats: MutableMap = HashMap() + + private val desiredTilesLock = Any() + private var desiredFingerprint = "" + // Headerless packed 4bpp frame (see BmpUtil.pack4bppFromGray8) plus its + // pixel dimensions. + private var desiredPacked: ByteArray? = ByteArray(0) + private var desiredWidth = 0 + private var desiredHeight = 0 + private var desiredPaintMs = 0 + private var desiredFrameId = 0 + // Screen-space deferred draws (glyphs + images) whose pixels are baked + // into desiredPacked; the resource-cache planner may replay them as + // on-glasses cached draws. + private var desiredDraws: Array? = arrayOf() + // (frame, reason) of the last "waiting to send" line, so a frame that + // stalls for seconds records one line per state change (see + // noteImageStallLocked). Send-loop thread only. + private var stallFrameId = 0 + private var stallReason = "" + // Highest compositor sequence stored as the desired frame; composites that + // lost a store race to a newer one are discarded (their content is already + // included in the newer composite). + private var lastStoredCompositeSeq = 0L + + // Wire submissions need screenGray + the shell scene, never preview pixels. + // Preview/screenshot/recording callers render those explicitly on demand. + private val compositor = SurfaceCompositor(false) + + // Phone-side model of the CFW's 192 KiB resource cache. + // Reset whenever the image pipeline / EvenHub session is torn down: the + // firmware frees the cache with the fb lease, and after any resync the + // cheap safe assumption is an empty cache (glyphs re-upload lazily). + private val resourceCache = ResourceCacheState() + private val scenePlanner = ScenePlanner(resourceCache) + private var desiredShellScene: ShellScene = ShellScene.EMPTY + + private val pendingMessages = ArrayDeque() + private val cfwTransports = arrayOf(CfwTransport(AndroidProtocolPlatform), CfwTransport(AndroidProtocolPlatform)) + private val inFlightMessages = ArrayDeque() + private var prewrittenMessage: OutboundMessage? = null + private var prewrittenFrames: List = Collections.emptyList() + + // Active animated-GIF screen recording, or null when idle. Frames are + // pushed by recordScreenFrame(), which the TS side calls at each + // phone-preview flush. + @Volatile private var screenRecorder: GifScreenRecorder? = null + + init { + bleManager.setListener(this) + val phoneLockFilter = IntentFilter() + phoneLockFilter.addAction(Intent.ACTION_SCREEN_ON) + phoneLockFilter.addAction(Intent.ACTION_SCREEN_OFF) + phoneLockFilter.addAction(Intent.ACTION_USER_PRESENT) + appContext.registerReceiver(phoneLockReceiver, phoneLockFilter) + phoneLockReceiverRegistered = true + } + + + fun setListener(listener: FaceclawBleCommunicatorListener?) { + this.listener = listener + emitState() + emitPhoneLockStateIfChanged(true) + } + + fun start() { + synchronized(lock) { + if (running) { + return + } + running = true + userDisconnectRequested = false + reconnectHalted = false + shutdownRequested = false + activeInstance = this + val thread = Thread(this, "FaceclawBleCommunicator") + workerThread = thread + thread.start() + } + } + + fun disconnect() { + /* On the normal path DashboardController already sent mode 11 after + * quiescing its producers. Also cover direct/early close callers here; + * a successful cleanup must remain the final BLE message. Older CFWs + * fall back to the standalone framebuffer-lease release. */ + if (!cfwCleanupDelivered && !sendCfwCleanup()) { + releaseFaceclawFramebufferLease() + } + val threadToJoin: Thread? + synchronized(lock) { + userDisconnectRequested = true + running = false + audioCaptureActive = false + audioPacketListener = null + threadToJoin = workerThread + } + setStateDisplay("disconnecting", "Disconnecting...") + interruptibleSleep.interrupt() + if (threadToJoin != null) { + threadToJoin.interrupt() + try { + threadToJoin.join(5_000) + } catch (e: InterruptedException) { + Thread.currentThread().interrupt() + } + } + synchronized(lock) { + workerThread = null + resetSessionStateLocked() + clearAllMessagesLocked("disconnect") + // Unknown until the next connection's first push or settings poll. + silentMode = -1 + } + bleManager.disconnect(rightAddress) + bleManager.disconnect(leftAddress) + if (hasRingAddress()) { + bleManager.disconnect(ringAddress) + } + bleManager.close() + releaseG2ScreenWakeLock() + setStateDisplay("disconnected", "Disconnected.") + } + + fun close() { + if (activeInstance === this) { + activeInstance = null + } + disconnect() + for (transport in cfwTransports) transport.close() + if (phoneLockReceiverRegistered) { + phoneLockReceiverRegistered = false + appContext.unregisterReceiver(phoneLockReceiver) + } + } + + fun setG2ScreenOn(screenOn: Boolean) { + mainHandler.post { updateG2ScreenWakeLock(screenOn) } + } + + fun setFirmwareDebugFlags(enabled: Boolean) { + // Just record it; the drive loop emits the mode-7 control message when the + // display path is ready and idle, and re-emits when this value changes. + firmwareDebugFlagsEnabled = enabled + } + + fun startG2AudioCapture(listener: FaceclawAudioPacketListener?): Boolean { + if (listener == null) { + throw IllegalArgumentException("listener is required") + } + var magic = 0 + synchronized(lock) { + if (!running || !sessionReady || shutdownRequested || !fixedLayoutCreated) { + logLine("skip G2 mic enable; EvenHub display path not ready") + return false + } + audioPacketListener = listener + val message = createAudioControlMessageLocked(true) + magic = message.magic + pendingMessages.addFirst(message) + logLine("queue G2 mic enable") + } + interruptibleSleep.interrupt() + return waitForAudioControlAck(magic, "enable") + } + + fun stopG2AudioCapture() { + var magic = 0 + synchronized(lock) { + audioPacketListener = null + audioCaptureActive = false + clearMessagesOfKindLocked("audio-control") + if (running && sessionReady) { + val message = createAudioControlMessageLocked(false) + magic = message.magic + pendingMessages.addFirst(message) + logLine("queue G2 mic disable") + } + } + interruptibleSleep.interrupt() + if (magic != 0) { + waitForAudioControlAck(magic, "disable") + } + } + + fun isSessionReady(): Boolean { + synchronized(lock) { + return running && sessionReady + } + } + + /** + * Whether the glasses mic is enabled right now. The enable lives in the + * current EvenHub session, so it dies with a transport drop, the charging + * case, or a suspend — silently, from the phone's point of view. Callers + * that track a capture across those events must check this rather than + * assume their earlier enable still holds. + */ + fun isAudioCaptureActive(): Boolean { + synchronized(lock) { + return running && sessionReady && !shutdownRequested && audioCaptureActive + } + } + + /** + * Acquire/renew or release CFW's volatile wake-takeover lease on both + * arms. Delivery (not a protocol ACK) is awaited so a caller can ensure + * the fail-open firmware policy is installed before relying on wakeword + * interception or suspending EvenHub. + */ + fun setFaceclawWakeLeaseEnabled(enabled: Boolean): Boolean { + val generation: Int + synchronized(lock) { + faceclawWakeLeaseEnabled = enabled + if (!running || !sessionReady) { + return !enabled + } + generation = enqueueFaceclawWakeControlLocked( + if (enabled) BleProtocol.FACECLAW_WAKE_OP_ACQUIRE else BleProtocol.FACECLAW_WAKE_OP_RELEASE, + 0, + true + ) + if (!enabled) { + faceclawWakePendingNonce = -1 + } + } + interruptibleSleep.interrupt() + return waitForFaceclawWakeControlDelivery(generation, FACECLAW_WAKE_CONTROL_WAIT_MS) + } + + /** + * Wait until the recreated layout and retained compositor frame have both + * landed. If this wake came from CFW's deferred double tap, READY + * is then sent to both arms to cancel their stock-dashboard fallback. + */ + fun awaitEvenHubSessionReady(timeoutMs: Int): Boolean { + val deadline = SystemClock.elapsedRealtime() + Math.max(0, timeoutMs) + var readyGeneration = 0 + synchronized(lock) { + while (running && sessionReady) { + var frameReady = false + synchronized(desiredTilesLock) { + frameReady = desiredFingerprint.isNotEmpty() + && desiredFingerprint == displayedFingerprint + } + if (!shutdownRequested && fixedLayoutCreated && frameReady) { + if (faceclawWakePendingNonce >= 0) { + readyGeneration = enqueueFaceclawWakeControlLocked( + BleProtocol.FACECLAW_WAKE_OP_READY, + faceclawWakePendingNonce, + true + ) + faceclawWakePendingNonce = -1 + } + break + } + val remaining = deadline - SystemClock.elapsedRealtime() + if (remaining <= 0) { + return false + } + try { + lock.wait(Math.min(remaining, 100L)) + } catch (e: InterruptedException) { + Thread.currentThread().interrupt() + return false + } + } + if (!running || !sessionReady) { + return false + } + } + if (readyGeneration != 0) { + interruptibleSleep.interrupt() + if (!waitForFaceclawWakeControlDelivery(readyGeneration, FACECLAW_WAKE_CONTROL_WAIT_MS)) { + logLine("wake READY delivery not confirmed before fallback deadline") + } + } + return true + } + + /** + * Enable or disable the IMU (accelerometer) report stream. Fire-and-forget: + * the control message is queued ahead of other traffic; readings arrive via + * registered FaceclawImuListeners. reportFrq is the requested sample rate + * (ignored on disable). + */ + fun setImuReportEnabled(enable: Boolean, reportFrq: Int) { + synchronized(lock) { + if (!running || !sessionReady) { + logLine("skip IMU " + (if (enable) "enable" else "disable") + "; session not ready") + return + } + clearMessagesOfKindLocked("imu-control") + val message = messageBuilder.enableOrDisableImu(enable, reportFrq) + message.onTimeout = MessageCallback { logLine("IMU control ack timeout") } + pendingMessages.addFirst(message) + logLine("queue IMU " + (if (enable) "enable freq=$reportFrq" else "disable")) + } + interruptibleSleep.interrupt() + } + + /** + * Enable/disable the stock compass through CFW image-handler mode 10. The + * desired state is retained across reconnects; headings arrive through + * stock sid-0x08 navigation notifications and FaceclawCompassListeners. + */ + fun setCompassEnabled(enable: Boolean) { + setCompassEnabled("compass", enable) + } + + /** + * As above, on behalf of a named owner. The compass runs while at least + * one owner has enabled it; an owner disabling it only takes effect once + * no other owner still wants it. + */ + fun setCompassEnabled(owner: String?, enable: Boolean) { + synchronized(lock) { + val before = compassOwners.isNotEmpty() + if (enable) { + compassOwners.add(owner) + } else { + compassOwners.remove(owner) + } + val wanted = compassOwners.isNotEmpty() + if (wanted == before && compassControlLastSent == (if (wanted) 1 else 0)) { + logLine("compass " + (if (enable) "enable" else "disable") + " by " + owner + + "; state unchanged (owners=" + compassOwners + ")") + return + } + compassControlLastSent = -1 + clearMessagesOfKindLocked("compass-control") + if (running && sessionReady && !shutdownRequested && fixedLayoutCreated) { + enqueueCompassControlLocked(true, wanted) + } else { + logLine("defer compass " + (if (enable) "enable" else "disable") + "; display path not ready") + } + } + interruptibleSleep.interrupt() + } + + /** + * Set the lens brightness. Fire-and-forget, like the IMU control: the + * message is queued ahead of other traffic and any not-yet-sent brightness + * message is superseded. When autoAdjust is true the ambient-light sensor + * drives brightness and brightnessLevel is ignored; otherwise + * brightnessLevel (0-100) is applied directly. + */ + fun setBrightness(autoAdjust: Boolean, brightnessLevel: Int) { + synchronized(lock) { + if (!running || !sessionReady) { + logLine("skip brightness set; session not ready") + return + } + clearMessagesOfKindLocked("brightness-control") + val message = messageBuilder.setBrightness(autoAdjust, brightnessLevel) + message.onTimeout = MessageCallback { logLine("brightness control ack timeout") } + pendingMessages.addFirst(message) + logLine("queue brightness " + (if (autoAdjust) "auto" else "level=$brightnessLevel")) + } + interruptibleSleep.interrupt() + } + + /** + * Enable the stock wear detector, then ask CFW to emit its current cached + * state. The queue order matters: the query must run after the setting is + * applied, including on a fresh install where wear detection was disabled. + */ + fun enableWearDetectionAndRequestState() { + synchronized(lock) { + if (!running || !sessionReady) { + logLine("skip wear detector setup; session not ready") + return + } + clearMessagesOfKindLocked("wear-detection-control") + clearMessagesOfKindLocked("wear-query-control") + val queryLeft = messageBuilder.faceclawWearQuery(true) + val queryRight = messageBuilder.faceclawWearQuery(false) + val enable = messageBuilder.setWearDetection(true) + enable.onTimeout = MessageCallback { logLine("wear detection enable ack timeout") } + pendingMessages.addFirst(queryLeft) + pendingMessages.addFirst(queryRight) + pendingMessages.addFirst(enable) + logLine("queue wear detection enable + current-state query") + } + interruptibleSleep.interrupt() + } + + /** + * Start the BLE bandwidth benchmark: stream messageSize-byte no-op image + * payloads for durationMs, keeping up to windowSize messages awaiting ack + * at once, then leave the results for getBandwidthBenchmarkStatus(). + * Returns false when a run is already active or the image path is not + * ready. The duration clock starts at the first benchmark write, so + * traffic already queued ahead of the run doesn't count against it. + */ + fun startBandwidthBenchmark(messageSize: Int, windowSize: Int, durationMs: Int): Boolean { + return startBandwidthBenchmarkWithLinkMode(messageSize, windowSize, durationMs, 0) + } + + // 0: current link; 1: re-request HIGH; 2: request 2M; 3: both. + fun startBandwidthBenchmarkWithLinkMode(messageSize: Int, windowSize: Int, + durationMs: Int, linkMode: Int): Boolean { + synchronized(lock) { + if (benchmarkActive || !running || !sessionReady || !fixedLayoutCreated + || shutdownRequested || chargingMode) { + logLine("skip bandwidth benchmark; already running or image path not ready") + return false + } + benchmarkMessageSize = Math.max(2, Math.min(messageSize, ConnectionOptions.IMAGE_FRAGMENT_SIZE)) + benchmarkWindowSize = Math.max(1, Math.min(windowSize, BENCHMARK_MAX_WINDOW)) + benchmarkDurationMs = Math.max(1_000, durationMs) + benchmarkLinkMode = linkMode and 3 + benchmarkLinkPending = true + benchmarkReadyAtMs = Long.MAX_VALUE + benchmarkStartAtMs = 0 + benchmarkDeadlineAtMs = Long.MAX_VALUE + benchmarkLastAckAtMs = 0 + benchmarkEndAtMs = 0 + benchmarkMessagesSent = 0 + benchmarkMessagesAcked = 0 + benchmarkTimeouts = 0 + benchmarkPayloadBytesAcked = 0 + benchmarkWireBytesAcked = 0 + benchmarkAborted = false + benchmarkActive = true + logLine("bandwidth benchmark start: size=" + benchmarkMessageSize + + "B window=" + benchmarkWindowSize + " duration=" + benchmarkDurationMs + + "ms linkMode=" + benchmarkLinkMode) + } + interruptibleSleep.interrupt() + return true + } + + /** + * Cancel an in-progress benchmark (benchmark page closed). Queued no-op + * messages are dropped; in-flight ones drain through their normal acks. + */ + fun cancelBandwidthBenchmark() { + synchronized(lock) { + if (!benchmarkActive) { + return + } + clearMessagesOfKindLocked("bandwidth") + finishBenchmarkLocked(true, "cancelled") + } + interruptibleSleep.interrupt() + } + + /** Status/results of the current or most recent benchmark run, as JSON. */ + fun getBandwidthBenchmarkStatus(): String { + synchronized(lock) { + val state = if (benchmarkActive) + (if (benchmarkStartAtMs == 0L) "starting" else "running") + else + (if (benchmarkEndAtMs != 0L) "done" else "idle") + val end = if (benchmarkActive) SystemClock.elapsedRealtime() else benchmarkEndAtMs + val elapsed = if (benchmarkStartAtMs == 0L) 0L else Math.max(0L, end - benchmarkStartAtMs) + try { + val status = org.json.JSONObject() + status.put("state", state) + status.put("messageSize", benchmarkMessageSize) + status.put("windowSize", benchmarkWindowSize) + status.put("linkMode", benchmarkLinkMode) + status.put("elapsedMs", elapsed) + status.put("messagesSent", benchmarkMessagesSent) + status.put("messagesAcked", benchmarkMessagesAcked) + status.put("timeouts", benchmarkTimeouts) + status.put("payloadBytesAcked", benchmarkPayloadBytesAcked) + status.put("wireBytesAcked", benchmarkWireBytesAcked) + status.put("aborted", benchmarkAborted) + return status.toString() + } catch (e: org.json.JSONException) { + return "{\"state\":\"idle\"}" + } + } + } + + /** Pending + in-flight benchmark no-op messages. */ + private fun benchmarkOutstandingLocked(): Int { + var count = 0 + for (message in pendingMessages) { + if ("bandwidth" == message.kind) count++ + } + for (message in inFlightMessages) { + if ("bandwidth" == message.kind) count++ + } + return count + } + + /** + * Keep the benchmark stream fed: top the pending queue up so the send + * window never starves, stop enqueueing once the run expires (or a message + * times out), and finish the run when the last outstanding message drains. + */ + private fun maintainBenchmarkLocked(now: Long) { + if (!sessionReady || !fixedLayoutCreated || shutdownRequested) { + finishBenchmarkLocked(true, "session no longer ready") + return + } + if (benchmarkLinkPending || now < benchmarkReadyAtMs) { + return + } + if (benchmarkAborted || now >= benchmarkDeadlineAtMs) { + // The run is over: drop queued-but-unsent no-ops (sending them + // would stretch the run past its deadline) and finish once the + // in-flight tail has acked or timed out. + val pendingIterator = pendingMessages.iterator() + while (pendingIterator.hasNext()) { + val queued = pendingIterator.next() + if ("bandwidth" == queued.kind) { + pendingIterator.remove() + magicPool.release(queued.sid, queued.magic, queued.label, "benchmark over") + } + } + if (benchmarkOutstandingLocked() == 0) { + finishBenchmarkLocked(false, "complete") + } + return + } + // One more than the window so a fresh message is always ready to write + // the moment an ack frees a slot. + var outstanding = benchmarkOutstandingLocked() + while (outstanding <= benchmarkWindowSize) { + enqueueBenchmarkMessageLocked() + outstanding++ + } + } + + private fun finishBenchmarkLocked(aborted: Boolean, reason: String) { + if (!benchmarkActive) { + return + } + benchmarkActive = false + benchmarkAborted = benchmarkAborted or aborted + // Prefer the last ack as the end time so drain lag after the deadline + // doesn't dilute the throughput figure. + benchmarkEndAtMs = if (benchmarkLastAckAtMs != 0L) benchmarkLastAckAtMs else SystemClock.elapsedRealtime() + logLine("bandwidth benchmark " + (if (benchmarkAborted) "aborted" else "finished") + " (" + reason + "): " + + benchmarkMessagesAcked + "/" + benchmarkMessagesSent + " acked, " + + benchmarkPayloadBytesAcked + "B payload, " + benchmarkTimeouts + " timeouts") + } + + private fun enqueueBenchmarkMessageLocked() { + // Fresh bytes per message: the transport's persistent compression history + // would compress even a random payload if we reused it across the run. + val payload = ByteArray(benchmarkMessageSize) + benchmarkRandom.nextBytes(payload) + payload[0] = CFW_MSG_DIAGNOSTICS.toByte() // CFW diagnostic-control mode... + payload[1] = 0x7f.toByte() // ...with an unused sub-op: acked, no effect + val message = messageBuilder.imagePayload( + "bandwidth", + DASHBOARD_TILE, + nextMapSessionId(), + payload, + "bandwidth no-op " + payload.size + "B", + connectionOptions.sendImagesToLeft) + val payloadBytes = payload.size + // Logical custom-message bytes; excludes length tag and packet framing. + val wireBytes = message.message.size + message.onSent = MessageCallback { + benchmarkMessagesSent++ + if (benchmarkStartAtMs == 0L) { + benchmarkStartAtMs = SystemClock.elapsedRealtime() + benchmarkDeadlineAtMs = benchmarkStartAtMs + benchmarkDurationMs + } + } + message.onAck = MessageCallback { + val ackedAtMs = SystemClock.elapsedRealtime() + benchmarkMessagesAcked++ + benchmarkPayloadBytesAcked += payloadBytes + benchmarkWireBytesAcked += wireBytes + benchmarkLastAckAtMs = ackedAtMs + // No-op payloads ride the image path, so the firmware resets its + // heartbeat timer on them just like real image messages. + lastHeartbeatAckedAtMs = ackedAtMs + } + message.onTimeout = MessageCallback { + benchmarkTimeouts++ + benchmarkAborted = true + logLine("bandwidth benchmark ack timeout") + } + pendingMessages.addLast(message) + } + + fun addImuListener(listener: FaceclawImuListener?) { + if (listener != null) { + imuListeners.add(listener) + } + } + + fun removeImuListener(listener: FaceclawImuListener?) { + if (listener != null) { + imuListeners.remove(listener) + } + } + + fun addAmbientLightListener(listener: FaceclawAmbientLightListener?) { + if (listener != null) { + ambientLightListeners.add(listener) + } + } + + fun removeAmbientLightListener(listener: FaceclawAmbientLightListener?) { + if (listener != null) { + ambientLightListeners.remove(listener) + } + } + + private fun emitAmbientLight(body: ByteArray) { + if (ambientLightListeners.isEmpty()) { + return + } + val copy = Arrays.copyOf(body, body.size) + mainHandler.post { + for (alsListener in ambientLightListeners) { + try { + alsListener.onAmbientLight(copy) + } catch (t: Throwable) { + Log.w(TAG, "ambient light listener failed", t) + } + } + } + } + + /** Request one CFW ambient-light report (image-handler mode 16 op 0). */ + fun queryAmbientLight() { + synchronized(lock) { + enqueueAmbientLightControlLocked(byteArrayOf(CFW_MSG_AMBIENT_LIGHT.toByte(), 0.toByte()), "als query", false) + } + interruptibleSleep.interrupt() + } + + /** + * Start or stop CFW passive light-sensor polling (image-handler mode 16 + * ops 1/2). While polling, the firmware reads its OPT3001 every intervalMs + * (clamped 100..5000 by the firmware), never steps the panel brightness + * itself, and pushes a field-105 report when the reading moved by at least + * minDelta or heartbeatMs elapsed. bindToLease stops polling automatically + * when the Faceclaw framebuffer lease is released or lapses. Starting is + * idempotent and re-opens the sensor if the stock firmware closed it. + */ + fun setAmbientLightPolling(enable: Boolean, intervalMs: Int, minDelta: Int, + heartbeatMs: Int, bindToLease: Boolean) { + val payload = if (enable) + byteArrayOf( + CFW_MSG_AMBIENT_LIGHT.toByte(), + 1.toByte(), + (if (bindToLease) 1 else 0).toByte(), + (intervalMs and 0xff).toByte(), + ((intervalMs shr 8) and 0xff).toByte(), + (minDelta and 0xff).toByte(), + ((minDelta shr 8) and 0xff).toByte(), + (heartbeatMs and 0xff).toByte(), + ((heartbeatMs shr 8) and 0xff).toByte(), + ) + else + byteArrayOf(CFW_MSG_AMBIENT_LIGHT.toByte(), 2.toByte()) + synchronized(lock) { + clearMessagesOfKindLocked("als-control") + enqueueAmbientLightControlLocked(payload, "als polling " + (if (enable) "start" else "stop"), true) + } + interruptibleSleep.interrupt() + } + + private fun enqueueAmbientLightControlLocked(payload: ByteArray, label: String, priority: Boolean) { + if (!running || !sessionReady || shutdownRequested || !fixedLayoutCreated) { + logLine("skip $label; display path not ready") + return + } + val message = messageBuilder.imagePayload( + "als-control", + DASHBOARD_TILE, + nextMapSessionId(), + payload, + label, + connectionOptions.sendImagesToLeft) + message.onTimeout = MessageCallback { logLine("$label ack timeout") } + if (priority) pendingMessages.addFirst(message) + else pendingMessages.addLast(message) + logLine("queue $label") + } + + fun addMicStatusListener(listener: FaceclawMicStatusListener?) { + if (listener != null) { + micStatusListeners.add(listener) + } + } + + fun removeMicStatusListener(listener: FaceclawMicStatusListener?) { + if (listener != null) { + micStatusListeners.remove(listener) + } + } + + private fun emitMicStatus(body: ByteArray, address: String) { + if (micStatusListeners.isEmpty()) { + return + } + val arm = if (address.equals(leftAddress, ignoreCase = true)) "L" + else if (address.equals(rightAddress, ignoreCase = true)) "R" else "?" + val copy = Arrays.copyOf(body, body.size) + mainHandler.post { + for (micListener in micStatusListeners) { + try { + micListener.onMicStatus(copy, arm) + } catch (t: Throwable) { + Log.w(TAG, "mic status listener failed", t) + } + } + } + } + + /** + * Queue a CFW mic_control record (['M','C',ver,op,...]) as a settings + * field-103 write to both temples, or to a single one. Fire-and-forget: + * the firmware answers with a field-104 status notify per temple, which + * arrives through addMicStatusListener. + */ + fun sendFaceclawMicControl(record: ByteArray?, label: String, rightTemple: Boolean, leftTemple: Boolean) { + if (record == null || record.size < 4) { + return + } + synchronized(lock) { + if (!running || !sessionReady) { + logLine("skip mic control ($label); session not ready") + return + } + if (rightTemple) { + pendingMessages.addLast(messageBuilder.faceclawMicControl(record, label, false)) + } + if (leftTemple) { + pendingMessages.addLast(messageBuilder.faceclawMicControl(record, label, true)) + } + logLine("queue mic control $label") + } + interruptibleSleep.interrupt() + } + + /** + * Forward render-characteristic audio packets to the listener WITHOUT + * sending the stock EvenHub audio-control enable. Used for the CFW + * mic_control streaming path, where capture is armed through settings + * field 103 and the temples emit 'SM' frames on the same characteristic + * that stock mono LC3 uses. Returns false when no session is up. + */ + fun startG2AudioForwarding(listener: FaceclawAudioPacketListener?): Boolean { + if (listener == null) { + throw IllegalArgumentException("listener is required") + } + synchronized(lock) { + if (!running || !sessionReady || shutdownRequested) { + logLine("skip G2 audio forwarding; session not ready") + return false + } + audioPacketListener = listener + audioCaptureActive = true + logLine("G2 audio forwarding enabled") + } + return true + } + + fun stopG2AudioForwarding() { + synchronized(lock) { + audioPacketListener = null + audioCaptureActive = false + logLine("G2 audio forwarding disabled") + } + } + + /** + * Subscribe to compass events. Callbacks are delivered on the Looper of + * the thread that registered (falling back to the main thread), so app + * worker isolates can listen without a cross-thread hop into their JS. + */ + fun addCompassListener(listener: FaceclawCompassListener?) { + if (listener == null) { + return + } + val looper = Looper.myLooper() + val handler = if (looper != null) Handler(looper) else mainHandler + compassSubscriptions.add(CompassSubscription(listener, handler)) + } + + fun removeCompassListener(listener: FaceclawCompassListener?) { + if (listener == null) { + return + } + for (subscription in compassSubscriptions) { + if (subscription.listener === listener) { + compassSubscriptions.remove(subscription) + } + } + } + + /** Set the compositor's output frame size. Call before configuring surfaces. */ + fun configureCompositorScreen(width: Int, height: Int) { + compositor.configureScreen(width, height) + } + + /** + * The current composited screen as a phone-UI preview bitmap, or null + * before any surface has been configured. Built from the compositor so + * the preview reflects every surface (chrome + whichever app is + * foreground), including worker-app frames the TS side never sees. + */ + fun getCompositePreviewBitmap(brightenGamma: Double): android.graphics.Bitmap? { + return getCompositePreviewBitmap(brightenGamma, false) + } + + /** As above; `green` renders the preview green-on-black (Settings > Phone display > Preview color). */ + fun getCompositePreviewBitmap(brightenGamma: Double, green: Boolean): android.graphics.Bitmap? { + val composite = compositor.previewComposite() ?: return null + return PreviewBitmapUtil.fromGray( + java.nio.ByteBuffer.wrap(composite.gray), composite.width, composite.height, brightenGamma, green) + } + + /** Save the current composite as a 4-bit grayscale PNG; returns the path or "". */ + @Throws(java.io.IOException::class) + fun saveCompositePngScreenshot(): String { + val composite = compositor.previewComposite() ?: return "" + return ScreenshotUtil.savePngScreenshot(appContext, composite.gray, composite.width, composite.height) + } + + /** + * Save the current composite cropped to the given screen rect (the region + * the shell says is actually occupied). The rect is clamped to the screen; + * a degenerate rect falls back to the full screen. + */ + @Throws(java.io.IOException::class) + fun saveCompositePngScreenshot(cropX: Int, cropY: Int, cropWidth: Int, cropHeight: Int): String { + val composite = compositor.previewComposite() ?: return "" + val x = Math.max(0, cropX) + val y = Math.max(0, cropY) + val width = Math.min(composite.width - x, cropWidth - (x - cropX)) + val height = Math.min(composite.height - y, cropHeight - (y - cropY)) + if (width <= 0 || height <= 0 || (x == 0 && y == 0 && width == composite.width && height == composite.height)) { + return ScreenshotUtil.savePngScreenshot(appContext, composite.gray, composite.width, composite.height) + } + val cropped = ByteArray(width * height) + for (row in 0 until height) { + System.arraycopy(composite.gray, (y + row) * composite.width + x, cropped, row * width, width) + } + return ScreenshotUtil.savePngScreenshot(appContext, cropped, width, height) + } + + /** Begin collecting composite frames for an animated-GIF screen recording. */ + fun startScreenRecording() { + screenRecorder = GifScreenRecorder() + } + + /** Capture the current composite into the active recording; no-op when idle. */ + fun recordScreenFrame() { + val recorder = screenRecorder ?: return + val composite = compositor.previewComposite() ?: return + recorder.addFrame(composite.gray, composite.width, composite.height, System.currentTimeMillis()) + } + + /** Finish the recording and save it as an animated GIF; returns the path or "". */ + @Throws(java.io.IOException::class) + fun stopScreenRecording(): String { + val recorder = screenRecorder + screenRecorder = null + if (recorder == null) { + return "" + } + if (recorder.isOverflowed()) { + logLine("screen recording hit its frame cap; the tail was dropped") + } + return recorder.save(appContext) + } + + /** + * Show or hide a compositor surface, immediately submitting the resulting + * frame. Recompositing here (rather than waiting for the next surface + * update) is what makes a just-foregrounded window's retained frame + * actually appear — otherwise a static window (e.g. the terminal hub) whose + * frame landed while briefly hidden would stay blank until its next repaint. + */ + fun setSurfaceVisible(id: String, visible: Boolean) { + // Its own frame: this recomposite is a real screen update with real + // latency, and without one it would show up in other frames' logs only + // as an anonymous "superseded by frame#0". + val frameId = FrameTimings.getInstance().startFrame( + "compositor:visible $id=$visible") + compositor.setSurfaceVisible(id, visible) + val composite = compositor.composite() + val packed = BmpUtil.pack4bppFromGray8(composite.screenGray, composite.width, composite.height) + storeDesiredComposite(composite, packed, 0, frameId) + } + + /** + * Blank (screen off) or unblank the composited output, immediately + * submitting the resulting frame. Retained surface state is untouched, so + * unblanking restores the previous screen content without repaints. + */ + fun setScreenBlanked(blanked: Boolean) { + val frameId = FrameTimings.getInstance().startFrame( + "compositor:" + (if (blanked) "blank" else "unblank")) + compositor.setBlanked(blanked) + val composite = compositor.composite() + val packed = BmpUtil.pack4bppFromGray8(composite.screenGray, composite.width, composite.height) + storeDesiredComposite(composite, packed, 0, frameId) + } + + /** + * Create or reconfigure a compositor surface. transparency is one of the + * SurfaceCompositor.TRANSPARENCY_* constants. Geometry changes take effect + * when the next frame is submitted. + */ + fun configureSurface(id: String?, x: Int, y: Int, width: Int, height: Int, zOrder: Int, transparency: Int) { + compositor.configureSurface(id, x, y, width, height, zOrder, transparency) + } + + fun removeSurface(id: String) { + compositor.removeSurface(id) + } + + /** Stage an immutable shell snapshot alongside the retained app screen. */ + fun submitShellScene(bytes: java.nio.ByteBuffer, paintMs: Int, frameId: Int) { + compositor.setShellScene(AndroidByteReader(bytes)) + val composite = compositor.composite() + storeDesiredComposite(composite, BmpUtil.pack4bppFromGray8(composite.screenGray, composite.width, composite.height), paintMs, frameId) + } + + /** Legacy compositor dimming for callers without a shell scene. */ + fun setUnderlayDim(belowZOrder: Int, factor256: Int) { + compositor.setUnderlayDim(belowZOrder, factor256) + } + + /** + * Apply an update to one compositor surface and submit the recomposited + * screen as the desired frame. The update covers the rect (rectX, rectY, + * rectWidth, rectHeight) in surface-local coordinates; contentFingerprint + * identifies the surface's full content after the update. + * + * pixels8bpp arrives as a ByteBuffer because NativeScript marshals a JS + * ArrayBuffer to one without the per-element bridge copy that a byte[] + * parameter would need (~150ms for a full frame). + */ + fun submitSurfaceFrame( + pixels8bpp: java.nio.ByteBuffer, + surfaceId: String, + rectX: Int, + rectY: Int, + rectWidth: Int, + rectHeight: Int, + contentFingerprint: String?, + paintMs: Int, + frameId: Int + ) { + submitSurfaceFrame(pixels8bpp, surfaceId, rectX, rectY, rectWidth, rectHeight, + contentFingerprint, paintMs, frameId, null) + } + + /** + * As above, with the frame's glyph draws (see SurfaceCompositor's glyph + * overload for the buffer format): the surface's full text content as + * structured draws, letting the resource-cache planner ship glyphs as + * on-glasses cached draws instead of pixels. Null when the submitter has + * no glyph metadata; the pixels alone remain fully correct. + */ + fun submitSurfaceFrame( + pixels8bpp: java.nio.ByteBuffer, + surfaceId: String, + rectX: Int, + rectY: Int, + rectWidth: Int, + rectHeight: Int, + contentFingerprint: String?, + paintMs: Int, + frameId: Int, + glyphs: java.nio.ByteBuffer? + ) { + Log.i(TAG, "Received an updated frame for surface $surfaceId") + FrameTimings.getInstance().log(frameId, "surface " + surfaceId + " updated rect=" + + rectWidth + "x" + rectHeight + "+" + rectX + "+" + rectY + + (if (glyphs == null) " (no glyph draws)" else "")) + FrameTimings.getInstance().spanStart(frameId, "composite") + val composite = compositor.applyAndComposite( + surfaceId, AndroidByteReader(pixels8bpp), rectX, rectY, rectWidth, rectHeight, contentFingerprint, + if (glyphs == null) null else AndroidByteReader(glyphs)) + FrameTimings.getInstance().spanEnd(frameId, "composite") + // Pack the composited 8bpp buffer down to the headerless 4bpp frame + // format the wire planners consume; BMP framing is added later only for + // the uncompressed fallback. + FrameTimings.getInstance().spanStart(frameId, "pack-4bpp") + val packed = BmpUtil.pack4bppFromGray8(composite.screenGray, composite.width, composite.height) + FrameTimings.getInstance().spanEnd(frameId, "pack-4bpp") + storeDesiredComposite(composite, packed, paintMs, frameId) + } + + /** Store a composite as the desired frame unless a newer one won the race. */ + private fun storeDesiredComposite(composite: SurfaceCompositor.Composite, packed: ByteArray, paintMs: Int, frameId: Int) { + var supersededFrameId = 0 + var stale = false + synchronized(desiredTilesLock) { + if (composite.seq <= lastStoredCompositeSeq) { + // A concurrent submission composited after us and stored first; + // its composite already includes this surface update. + stale = true + } else { + lastStoredCompositeSeq = composite.seq + supersededFrameId = desiredFrameId + desiredPacked = packed + desiredWidth = composite.width + desiredHeight = composite.height + desiredFingerprint = composite.fingerprint + desiredPaintMs = paintMs + desiredFrameId = frameId + desiredDraws = composite.draws + desiredShellScene = composite.shellScene + } + } + if (stale) { + finishFrame(frameId, "discarded: composite superseded before store") + return + } + if (supersededFrameId != 0 && supersededFrameId != frameId) { + finishFrame(supersededFrameId, "discarded: superseded by frame#$frameId before send") + } + FrameTimings.getInstance().log(frameId, "image submitted as desired frame") + interruptibleSleep.interrupt() + } + + /** + * Play a tone sequence via CFW load_image_z mode 5 kind 4. The payload is + * the complete wire buffer ([5][4][nSteps][freqLo,freqHi,duty,msLo,msHi]*n, + * up to 48 steps), built on the TS side; it rides the arbitrary-payload + * image path like the other mode-5 controls. + */ + fun playBuzzerSequence(payload: java.nio.ByteBuffer?) { + synchronized(lock) { + if (!running || !sessionReady || !fixedLayoutCreated) { + logLine("skip buzzer sequence; session not ready") + return + } + val bytes = ByteArray(if (payload == null) 0 else payload.remaining()) + if (payload != null) { + payload.get(bytes) + } + if (bytes.size < 3) { + logLine("skip buzzer sequence; empty payload") + return + } + val message = messageBuilder.imagePayload( + DASHBOARD_TILE, + nextMapSessionId(), + bytes, + "buzzer sequence " + bytes.size + "B", + connectionOptions.sendImagesToLeft + ) + message.onTimeout = MessageCallback { + handleTransportFailure("buzzer sequence ack timeout") + } + pendingMessages.addLast(message) + logLine("queue " + message.label) + } + interruptibleSleep.interrupt() + } + + fun sendShutdown(exitMode: Int): Boolean { + return sendShutdownInternal(exitMode, true) + } + + /** + * Send CFW image-handler mode 11 after quiescing normal traffic. A successful + * return means the cleanup was ACKed and no later Faceclaw message should be + * emitted before closing BLE. Unsupported/older CFWs return false so callers + * can use the legacy shutdown-and-lease-release path. + */ + fun sendCfwCleanup(): Boolean { + val magic: Int + synchronized(lock) { + if (cfwCleanupDelivered) { + return true + } + if (!customFirmwareDetected || !running || !sessionReady + || shutdownRequested || !fixedLayoutCreated) { + logLine("skip CFW cleanup; mode 11 unavailable or image path not ready") + return false + } + + /* Stop auto-renewals, heartbeats, image generation, and control + * retries, then discard everything that has not reached BLE yet. */ + shutdownRequested = true + lastCfwCleanupAckMagic = 0 + clearPendingMessagesLocked("CFW cleanup requested") + logLine("quiescing transport for CFW cleanup") + } + interruptibleSleep.interrupt() + + /* WINDOW_SIZE can exceed one, so merely appending cleanup would allow it + * to overlap previously-written image fragments. Wait until all of those + * ACK or time out; shutdownRequested prevents the drive loop from adding + * any fresh automatic traffic meanwhile. */ + val drainDeadline = SystemClock.elapsedRealtime() + CFW_CLEANUP_WAIT_MS + synchronized(lock) { + while (running && sessionReady && !inFlightMessages.isEmpty()) { + val remaining = drainDeadline - SystemClock.elapsedRealtime() + if (remaining <= 0) break + try { + lock.wait(Math.min(remaining, 100L)) + } catch (e: InterruptedException) { + Thread.currentThread().interrupt() + break + } + } + if (!running || !sessionReady || !inFlightMessages.isEmpty()) { + shutdownRequested = false + logLine("CFW cleanup could not drain prior traffic") + return false + } + + /* External producers are expected to be stopped by the caller, but + * clear once more at the barrier so cleanup is definitely last. */ + clearPendingMessagesLocked("CFW cleanup barrier") + val message = messageBuilder.cfwCleanup( + DASHBOARD_TILE, + nextMapSessionId(), + connectionOptions.sendImagesToLeft + ) + magic = message.magic + message.onAck = MessageCallback { + lastCfwCleanupAckMagic = message.magic + cfwCleanupDelivered = true + compassMaybeOn = false + faceclawWakeLeaseEnabled = false + logLine("CFW cleanup completed") + } + message.onTimeout = MessageCallback { logLine("CFW cleanup ack timeout") } + pendingMessages.addLast(message) + logLine("queue CFW cleanup") + } + interruptibleSleep.interrupt() + + val deadline = SystemClock.elapsedRealtime() + CFW_CLEANUP_WAIT_MS + synchronized(lock) { + while (running + && sessionReady + && lastCfwCleanupAckMagic != magic + && hasPendingOrInflightMagicLocked(magic)) { + val remaining = deadline - SystemClock.elapsedRealtime() + if (remaining <= 0) break + try { + lock.wait(Math.min(remaining, 100L)) + } catch (e: InterruptedException) { + Thread.currentThread().interrupt() + break + } + } + val acked = lastCfwCleanupAckMagic == magic + if (!acked) shutdownRequested = false + return acked + } + } + + /** + * End the EvenHub page while retaining both arm GATT connections and all + * notification subscriptions. A missed ACK is intentionally non-fatal: + * reconnecting Bluetooth here would defeat the power-saving mode. + */ + fun suspendEvenHubSession(): Boolean { + // Ending the plugin task releases the fb lease, which frees the + // on-glasses resource cache; forget it phone-side either way (a lost + // ack may still have taken effect). + synchronized(lock) { + resourceCache.reset() + } + if (sendShutdownInternal(0, false)) { + return true + } + synchronized(lock) { + // A shutdown ACK can be lost even though the command took effect. + // If the transport remains intentionally quiesced, callers still + // need to remember to run the resume path on the next wake. + return running && sessionReady && shutdownRequested + } + } + + /** + * Start a fresh EvenHub plugin task on the existing BLE transport, then let + * the session driver create the layout, warm up the image path, and send + * the desired frame. + */ + fun resumeEvenHubSession(): Boolean { + var claimGeneration = 0 + synchronized(lock) { + if (!running || !sessionReady || chargingMode) { + logLine("skip EvenHub resume; transport not ready") + return false + } + if (!shutdownRequested) { + return true + } + if (faceclawWakePendingNonce >= 0 + && hasPendingOrInflightKindLocked("wake-lease-control")) { + claimGeneration = faceclawWakeControlGeneration + } + logLine("replaying session prelude for EvenHub resume") + } + + // A custom double-tap wake has only a short unclaimed fail-open + // deadline. Let the worker put CLAIM on both arms before the direct + // prelude write begins. + if (claimGeneration != 0) { + waitForFaceclawWakeControlDelivery(claimGeneration, 500) + } + + try { + // Empty-name Cmd=9 tears down the whole plugin task, not just its + // image container. Re-run the launch prelude before Cmd=0 CREATE. + sendPrelude(true) + } catch (e: InterruptedException) { + Thread.currentThread().interrupt() + handleTransportFailure("EvenHub resume prelude interrupted") + return false + } catch (t: Throwable) { + logLine("EvenHub resume prelude failed: " + safeMessage(t)) + handleTransportFailure("EvenHub resume prelude failed") + return false + } + + synchronized(lock) { + if (!running || !sessionReady || chargingMode) { + return false + } + shutdownRequested = false + fixedLayoutCreated = false + startupProbePending = false + audioCaptureActive = false + clearAllMessagesPreservingWakeLeaseLocked("EvenHub resume") + displayedFingerprint = "" + imageRetryAfterMs = 0 + lastHeartbeatSentAtMs = 0 + lastHeartbeatAckedAtMs = 0 + logLine("EvenHub session resume requested") + } + interruptibleSleep.interrupt() + return true + } + + private fun sendShutdownInternal(exitMode: Int, reconnectOnTimeout: Boolean): Boolean { + val magic: Int + val startedAtMs = SystemClock.elapsedRealtime() + synchronized(lock) { + if (!running || !sessionReady) { + logLine("skip shutdown; session not ready") + return false + } + if (shutdownRequested) { + return true + } + shutdownRequested = true + // Magic values wrap, so an ACK from a much older suspend must not + // satisfy this request after enough sleep/wake cycles. + lastShutdownAckMagic = 0 + clearPendingMessagesLocked("shutdown requested") + val message = messageBuilder.shutdown(exitMode) + magic = message.magic + message.onAck = MessageCallback { + lastShutdownAckMagic = message.magic + fixedLayoutCreated = false + displayedFingerprint = "" + } + message.onTimeout = MessageCallback { + if (reconnectOnTimeout) { + handleTransportFailure("shutdown ack timeout") + } else { + logLine("EvenHub shutdown ack timeout; keeping BLE connected") + } + } + pendingMessages.addFirst(message) + logLine("queue shutdown") + // The stock compass keeps the magnetometer sampling independently of + // the plugin task, so ending the page does not stop it. Force a + // disable ahead of the shutdown command whenever it may be running: + // this also covers a disable that was wiped by the queue flush above + // or whose ack was lost, and the charging-mode/exit paths where the + // Compass window never got a chance to release it. + if (compassMaybeOn && fixedLayoutCreated) { + enqueueCompassControlLocked(true, false) + } + } + interruptibleSleep.interrupt() + + val ackDeadline = SystemClock.elapsedRealtime() + ConnectionOptions.ACK_TIMEOUT_MS + 500 + synchronized(lock) { + while (running + && sessionReady + && lastShutdownAckMagic != magic + && lastShutdownExitAtMs < startedAtMs + && hasPendingOrInflightMagicLocked(magic)) { + val remaining = ackDeadline - SystemClock.elapsedRealtime() + if (remaining <= 0) { + break + } + try { + lock.wait(Math.min(remaining, 100L)) + } catch (e: InterruptedException) { + Thread.currentThread().interrupt() + break + } + } + val acked = lastShutdownAckMagic == magic + if (acked) { + val exitDeadline = SystemClock.elapsedRealtime() + ConnectionOptions.ACK_TIMEOUT_MS + 500 + while (running && sessionReady && lastShutdownExitAtMs < startedAtMs) { + val remaining = exitDeadline - SystemClock.elapsedRealtime() + if (remaining <= 0) { + break + } + try { + lock.wait(Math.min(remaining, 100L)) + } catch (e: InterruptedException) { + Thread.currentThread().interrupt() + break + } + } + } + return acked || lastShutdownExitAtMs >= startedAtMs + } + } + + private fun waitForAudioControlAck(magic: Int, operation: String): Boolean { + val deadline = SystemClock.elapsedRealtime() + ConnectionOptions.ACK_TIMEOUT_MS + ConnectionOptions.WRITE_TIMEOUT_MS + 500 + synchronized(lock) { + while (running && sessionReady && lastAudioControlAckMagic != magic && hasPendingOrInflightMagicLocked(magic)) { + val remaining = deadline - SystemClock.elapsedRealtime() + if (remaining <= 0) { + break + } + try { + lock.wait(Math.min(remaining, 100L)) + } catch (e: InterruptedException) { + Thread.currentThread().interrupt() + break + } + } + val acked = lastAudioControlAckMagic == magic + if (!acked) { + if ("enable" == operation) { + audioPacketListener = null + audioCaptureActive = false + } + logLine("G2 mic $operation ack timeout") + } + return acked + } + } + + + override fun run() { + logLine(String.format(Locale.US, "communicator start R=%s L=%s ring=%s", rightAddress, leftAddress, ringAddress)) + while (true) { + try { + if (!running) { + Log.w(TAG, "Exiting event looop") + break + } + emitPhoneLockStateIfChanged(false) + if (!sessionReady) { + if (reconnectHalted) { + interruptibleSleep.sleep(ConnectionOptions.IDLE_SLEEP_MS.toLong()) + continue + } + val now = SystemClock.elapsedRealtime() + if (now < reconnectAfterMs) { + interruptibleSleep.sleep(Math.min(ConnectionOptions.IDLE_SLEEP_MS.toLong(), reconnectAfterMs - now)) + continue + } + Log.w(TAG, "Attempting to connect") + connectLoopOnce() + continue + } + + if (shouldAttemptRingConnect()) { + tryConnectRing("retry") + continue + } + + val sleepMs = driveSession() + if (sleepMs > 0) { + interruptibleSleep.sleep(sleepMs) + } + } catch (t: Throwable) { + logLine("communicator loop error: " + safeMessage(t)) + handleTransportFailure("loop error") + } + } + logLine("communicator stop") + } + + override fun onNotification(address: String?, characteristicUuid: String?, data: ByteArray?) { + if (address == null || characteristicUuid == null || data == null) { + return + } + val uuid = characteristicUuid.lowercase(Locale.US) + if (isDirectRingNotification(address, uuid)) { + handleDirectRingNotification(uuid, data) + return + } + if (BleProtocol.RENDER_NOTIFY_UUID == uuid) { + handleRenderNotification(address, data) + return + } + if (BleProtocol.NOTIFY_CHAR_UUID != uuid) { + return + } + if (data.size >= 7 && (data[6].toInt() and 255) == CfwTransport.SID) { + val acks = CfwTransport.parseAcks(data) ?: return + synchronized(lock) { + lastIncomingAtMs = SystemClock.elapsedRealtime() + for (ack in acks) { + for (message in inFlightMessages) { + val ingress = if (message.isLeftArmMessage) leftAddress else rightAddress + if (message.sid == CfwTransport.SID && address.equals(ingress, ignoreCase = true) && message.magic == ack.streamId) { + message.acceptCfwAck(ack) + Log.i(TAG, "CFW " + (if (ack.nack) "NACK" else "ACK") + " id=" + ack.streamId + + " ordinal=" + ack.messageId + " lens=" + ack.lens + " txseq=" + (data[2].toInt() and 255) + + " redundant=" + (ack !== acks[0]) + + " size=" + ack.size + " crc=" + ack.checksum + + " expected=" + message.message.size + "/" + message.cfwChecksum + + " ackedLenses=" + message.cfwAckLenses + + " ageMs=" + (lastIncomingAtMs - message.sentAtMs)) + message.ackPayload = Arrays.copyOf(data, data.size) + break + } + } + } + drainCfwAcknowledgementsLocked() + } + interruptibleSleep.interrupt() + return + } + Log.d(TAG, "onNotification: address=" + address + " characteristicUuid=" + characteristicUuid + " data.length=" + data.size) + val frame = BleProtocol.parseFrame(data) + val decodedWearState = BleProtocol.parseWearState(frame) + val compassEvent = if (address.equals(rightAddress, ignoreCase = true)) + BleProtocol.parseCompassEvent(frame) + else + null + var emitWearState = false + var event: G2Event? = null + synchronized(lock) { + lastIncomingAtMs = SystemClock.elapsedRealtime() + if (decodedWearState >= 0 && decodedWearState != wearState) { + wearState = decodedWearState + emitWearState = true + } + var faceclawWakeNotification = false + if (shutdownRequested + && frame.ok + && frame.sid == BleProtocol.SID_UI_SETTING + && address.equals(rightAddress, ignoreCase = true)) { + val wakeNonce = BleProtocol.parseFaceclawWakeEvent(frame.pb) + if (wakeNonce >= 0) { + faceclawWakePendingNonce = wakeNonce + enqueueFaceclawWakeControlLocked( + BleProtocol.FACECLAW_WAKE_OP_CLAIM, + wakeNonce, + true + ) + faceclawWakeNotification = true + lastConnectionOrInputAtMs = lastIncomingAtMs + // The event type says which gesture woke the glasses: TS + // gives a head-up the Glanceboard and a double tap the + // regular UI. The claim handshake is the same for both. + val headUp = BleProtocol.parseFaceclawWakeEventCode(frame.pb) == + BleProtocol.FACECLAW_WAKE_EVENT_HEAD_UP + event = G2Event( + "display-wake", + "", + if (headUp) BleProtocol.EVENT_HEAD_UP else BleProtocol.EVENT_DOUBLE_CLICK, + 0, + 0 + ) + logLine("claimed deferred dashboard wake nonce=" + wakeNonce + + (if (headUp) " (head-up)" else "")) + } + } + if (!faceclawWakeNotification + && shutdownRequested + && frame.ok + && frame.sid == BleProtocol.SID_UI_SETTING + && address.equals(rightAddress, ignoreCase = true)) { + // CFW idle-gesture forwarding (firmware revision 2+): with no + // EvenHub page the stock display thread drops taps, long + // presses and releases; the CFW reports them on the settings + // sid while our wake lease is held. Deliver them as the + // sys-events a live page would have produced, so TS treats a + // sleep-time tap or hold exactly like one during soft sleep + // (the Glanceboard). + val gesture = BleProtocol.parseFaceclawGestureEvent(frame.pb) + if (gesture != null) { + lastConnectionOrInputAtMs = lastIncomingAtMs + event = G2Event("sys-event", "", gesture.eventType, gesture.eventSource, 0) + logLine("idle gesture forwarded by CFW: type=" + gesture.eventType + + " source=" + gesture.eventSource) + } + } + if (!faceclawWakeNotification + && event == null + && shutdownRequested + && address.equals(rightAddress, ignoreCase = true) + && BleProtocol.isDisplayWakeStateChange(frame)) { + // With no EvenHub page, ring/arm double-taps are handled by the + // stock display lifecycle and surface only as this state ping. + // Translate it back into an input event so TS can wake the shell + // and recreate the page. + event = G2Event( + "display-wake", + "", + BleProtocol.EVENT_DOUBLE_CLICK, + 0, + 0 + ) + } + if (!faceclawWakeNotification + && frame.ok + && frame.sid == BleProtocol.SID_UI_SETTING) { + // Mode-17 query notifications; settings READs are handled by + // createBatteryQueryMessageLocked so headset/ring update together. + if (address.equals(rightAddress, ignoreCase = true) + && (frame.flag == BleProtocol.FLAG_NOTIFY || frame.flag == BleProtocol.FLAG_NOTIFY_ALT)) { + val ring = BleProtocol.parseRingBattery(frame.pb) + if (ring != null) { + ringBattery = ring.battery + ringCharging = ring.charging + emitBatteryState(headsetBattery, headsetCharging) + } + } + // CFW mic status (field 104) rides both standalone pushes and + // settings read acks, from each temple on its own link. + val micStatus = BleProtocol.parseFaceclawMicStatus(frame.pb) + if (micStatus != null) { + emitMicStatus(micStatus, address) + } + } + if (!faceclawWakeNotification + && frame.ok + && frame.sid == BleProtocol.SID_UI_SETTING) { + // CFW ambient-light report (field 105) from the master temple. + val alsReport = BleProtocol.parseFaceclawAlsReport(frame.pb) + if (alsReport != null) { + emitAmbientLight(alsReport) + } + } + if (!faceclawWakeNotification + && decodedWearState < 0 + && frame.ok + && frame.sid == BleProtocol.SID_UI_SETTING) { + // Device-initiated settings push. It carries a magic the glasses + // chose, so it would otherwise fall through to resolveAckLocked, + // match nothing, and be logged as an unexpected ack. + val pushedSilentMode = BleProtocol.parseSilentModePush(frame.pb) + if (pushedSilentMode >= 0) { + updateSilentModeLocked(pushedSilentMode > 0) + return + } + } + if (!faceclawWakeNotification + && decodedWearState < 0 + && frame.ok + && frame.msgSeq >= 0 + && frame.flag != BleProtocol.FLAG_NOTIFY + && frame.flag != BleProtocol.FLAG_NOTIFY_ALT) { + lastAckAtMs = lastIncomingAtMs + resolveAckLocked(frame.sid, frame.msgSeq, frame.pb) + } + if (!faceclawWakeNotification + && event == null + && frame.ok + && address.equals(rightAddress, ignoreCase = true) + && (frame.flag == BleProtocol.FLAG_NOTIFY || frame.flag == BleProtocol.FLAG_NOTIFY_ALT)) { + var decoded = G2Event.decode(frame) + if (decoded != null + && "sys-event" == decoded.kind + && decoded.eventType == BleProtocol.EVENT_HEAD_UP) { + // CFW forwards the IMU head-up while our page is on screen + // (soft sleep). Surface it as the same wake-only input the + // deferred head-up wake produces from a dark display. + decoded = G2Event("display-wake", "", BleProtocol.EVENT_HEAD_UP, decoded.eventSource, 0) + logLine("head-up forwarded by CFW while page on screen") + } + event = decoded + if (decoded != null) { + // Pure IMU samples arrive continuously; don't let them count + // as user input (which would starve battery polling). + val pureImuSample = "sys-event" == decoded.kind + && decoded.eventType == BleProtocol.EVENT_IMU_DATA_REPORT + if (!pureImuSample) { + lastConnectionOrInputAtMs = lastIncomingAtMs + } + if ("list-click" == decoded.kind || "text-click" == decoded.kind) { + // Container-routed touchpad input reached us, so the + // firmware is dispatching input: silent mode is off, + // whether or not its end-of-silent push arrived. + updateSilentModeLocked(false) + } + if ("sys-event" == decoded.kind) { + if (decoded.eventType == BleProtocol.EVENT_FOREGROUND_EXIT || decoded.eventType == BleProtocol.EVENT_ABNORMAL_EXIT || decoded.eventType == BleProtocol.EVENT_SYSTEM_EXIT) { + if (shutdownRequested) { + lastShutdownExitAtMs = SystemClock.elapsedRealtime() + } + fixedLayoutCreated = false + displayedFingerprint = "" + clearAllMessagesLocked("firmware exit event") + } + } + } + } + } + interruptibleSleep.interrupt() + if (emitWearState) { + logLine(if (decodedWearState > 0) "wear state ON_HEAD" else "wear state OFF_HEAD") + emitWearState(decodedWearState > 0) + } + if (compassEvent != null) { + emitCompassEvent(compassEvent) + } + val finalEvent = event + if (finalEvent != null) { + if (finalEvent.hasImu) { + emitImuData(finalEvent.imuX, finalEvent.imuY, finalEvent.imuZ, finalEvent.eventSource) + } + // A standalone IMU_DATA_REPORT is a sensor sample, not a gesture: + // deliver it only to IMU listeners, skipping the input pipeline (and + // its per-frame latency bookkeeping) to avoid flooding it. + val pureImuSample = "sys-event" == finalEvent.kind + && finalEvent.eventType == BleProtocol.EVENT_IMU_DATA_REPORT + if (!pureImuSample) { + val frameId = FrameTimings.getInstance().startFrame( + "input:" + finalEvent.kind + " type=" + finalEvent.eventType + " src=" + finalEvent.eventSource) + FrameTimings.getInstance().log(frameId, "input event decoded from BLE notification") + emitRingEvent(finalEvent, frameId) + } + } + } + + private fun handleDirectRingNotification(characteristicUuid: String, data: ByteArray) { + val decoded = FaceclawRingEventDecoder.decode(data) + if (decoded == null) { + Log.d(TAG, "direct ring notify ignored: characteristicUuid=" + characteristicUuid + " raw=" + hex(data)) + return + } + + val event = decoded.event + val arrivalMs = SystemClock.elapsedRealtime() + synchronized(lock) { + lastIncomingAtMs = arrivalMs + lastConnectionOrInputAtMs = arrivalMs + } + logLine("direct ring " + decoded.label + " " + decoded.detail + " raw=" + hex(data)) + val frameId = FrameTimings.getInstance().startFrame("input:ring:" + decoded.label) + FrameTimings.getInstance().log(frameId, "input event decoded from direct ring notification") + emitRingEvent(event, frameId) + interruptibleSleep.interrupt() + } + + private fun handleRenderNotification(address: String, data: ByteArray) { + val listenerToCall: FaceclawAudioPacketListener? + val arrivalMs = SystemClock.elapsedRealtime() + synchronized(lock) { + lastIncomingAtMs = arrivalMs + listenerToCall = if (audioCaptureActive) audioPacketListener else null + } + if (listenerToCall == null) { + return + } + val arm = if (address.equals(leftAddress, ignoreCase = true)) "L" else if (address.equals(rightAddress, ignoreCase = true)) "R" else "?" + try { + listenerToCall.onAudioPacket(Arrays.copyOf(data, data.size), arm, arrivalMs) + } catch (t: Throwable) { + logLine("G2 mic packet listener failed: " + safeMessage(t)) + } + } + + override fun onConnectionStateChange(address: String?, connected: Boolean) { + synchronized(lock) { + if (address == null) { + return + } + if (isConfiguredRingAddress(address)) { + ringConnected = connected + ringNotificationsReady = false + if (!connected) { + ringReconnectAfterMs = SystemClock.elapsedRealtime() + ConnectionOptions.RING_RECONNECT_DELAY_MS + } + logLine(if (connected) "direct ring BLE connected" else "direct ring BLE disconnected") + return + } + if (address.equals(rightAddress, ignoreCase = true)) { + rightConnected = connected + } else if (address.equals(leftAddress, ignoreCase = true)) { + leftConnected = connected + } else { + return + } + if (!connected) { + sessionReady = false + fixedLayoutCreated = false + startupProbePending = false + chargingMode = false + audioCaptureActive = false + audioPacketListener = null + clearAllMessagesLocked("connection lost") + displayedFingerprint = "" + if (!reconnectHalted) { + reconnectAfterMs = SystemClock.elapsedRealtime() + ConnectionOptions.RECONNECT_DELAY_MS + } + } + } + interruptibleSleep.interrupt() + if (connected) { + setStateDisplay("connected", "Connected.") + } else if (!reconnectHalted) { + // While parked on a missing bond, keep the "unpaired" display: this + // callback is just the teardown of the arm that did connect. + setStateDisplay("connecting", "Connecting to the glasses...") + } + } + + @Throws(InterruptedException::class) + private fun connectLoopOnce() { + setStateDisplay("connecting", "Connecting to the glasses...") + try { + connectArm(rightAddress, true) + connectArm(leftAddress, true) + if (!sleepDuringConnectSettling(800)) { + return + } + authenticateArms() + sendPrelude() + + synchronized(lock) { + sessionReady = true + // A fresh transport prelude always starts an active EvenHub + // lifecycle, even if the previous connection dropped while + // its page was intentionally suspended. + shutdownRequested = false + fixedLayoutCreated = false + clearAllMessagesLocked("session ready") + displayedFingerprint = "" + lastAckAtMs = SystemClock.elapsedRealtime() + lastIncomingAtMs = lastAckAtMs + lastConnectionOrInputAtMs = lastAckAtMs + lastSessionReadyAtMs = lastAckAtMs + lastBatteryRefreshAtMs = 0 + imageRetryAfterMs = 0 + lastHeartbeatSentAtMs = 0 + lastHeartbeatAckedAtMs = 0 + consecutiveAckTimeouts = 0 + lastAudioControlAckMagic = 0 + audioCaptureActive = false + faceclawWakePendingNonce = -1 + cfwCleanupDelivered = false + for (transport in cfwTransports) transport.reset() + lastCfwCleanupAckMagic = 0 + lastFaceclawWakeLeaseQueuedAtMs = 0 + lastFaceclawFramebufferLeaseQueuedAtMs = 0 + enqueueFaceclawFramebufferControlLocked( + BleProtocol.FACECLAW_FB_OP_ACQUIRE, + true + ) + if (faceclawWakeLeaseEnabled) { + enqueueFaceclawWakeControlLocked( + BleProtocol.FACECLAW_WAKE_OP_ACQUIRE, + 0, + true + ) + } + } + setStateDisplay("connected", "Connected.") + logLine("session ready") + synchronized(lock) { + // Query settings promptly on the first session so firmware + // version/extension (and battery) arrive without waiting for + // the input-quiet battery poll. The settings response doubles as + // the firmware-compatibility check surfaced during onboarding. + if (!firmwareInfoQueried) { + firmwareInfoQueried = true + lastBatteryRefreshAtMs = SystemClock.elapsedRealtime() + pendingMessages.addLast(createBatteryQueryMessageLocked()) + logLine("queue settings query for firmware info") + } + } + tryConnectRing("initial") + } catch (t: Throwable) { + logLine("connect failed: " + safeMessage(t)) + val unpairedArm = firstUnpairedArm() + if (unpairedArm != null) { + handleUnpairedFailure(unpairedArm) + } else { + handleTransportFailure("connect failed") + } + } + } + + @Throws(InterruptedException::class) + private fun sleepDuringConnectSettling(delayMs: Long): Boolean { + val deadline = SystemClock.elapsedRealtime() + delayMs + synchronized(lock) { + while (running && !userDisconnectRequested) { + val remaining = deadline - SystemClock.elapsedRealtime() + if (remaining <= 0) { + return true + } + lock.wait(Math.min(remaining, 100L)) + } + return false + } + } + + private fun connectArm(address: String, enableRenderNotify: Boolean) { + if (!bleManager.connect(address, ConnectionOptions.CONNECT_TIMEOUT_MS)) { + throw IllegalStateException("connect failed: $address") + } + // requestConnectionPriority has no callback in this Android compile target, so there is + // no reliable completion point to keep it in the global GATT operation pipeline. But it's + // important enough for performance that we call it anyways. + bleManager.requestConnectionPriority(address, BluetoothGatt.CONNECTION_PRIORITY_HIGH) + + bleManager.requestMtu(address, ConnectionOptions.DESIRED_MTU, ConnectionOptions.CONNECT_TIMEOUT_MS) + + if (!bleManager.discoverServices(address, ConnectionOptions.SERVICES_TIMEOUT_MS)) { + throw IllegalStateException("discoverServices failed: $address") + } + if (!bleManager.enableNotifications(address, BleProtocol.NOTIFY_CHAR_UUID, true, ConnectionOptions.DESCRIPTOR_TIMEOUT_MS)) { + throw IllegalStateException("enableNotifications failed: " + address + " " + BleProtocol.NOTIFY_CHAR_UUID) + } + if (enableRenderNotify) { + bleManager.enableNotifications(address, BleProtocol.RENDER_NOTIFY_UUID, true, ConnectionOptions.DESCRIPTOR_TIMEOUT_MS) + } + synchronized(lock) { + if (address.equals(rightAddress, ignoreCase = true)) { + rightConnected = true + } else if (address.equals(leftAddress, ignoreCase = true)) { + leftConnected = true + } + } + } + + private fun shouldAttemptRingConnect(): Boolean { + if (!hasRingAddress()) { + return false + } + val now = SystemClock.elapsedRealtime() + synchronized(lock) { + return running + && sessionReady + && !ringNotificationsReady + && now >= ringReconnectAfterMs + && pendingMessages.isEmpty() + && inFlightMessages.isEmpty() + } + } + + private fun tryConnectRing(reason: String) { + if (!hasRingAddress()) { + return + } + try { + connectRing() + } catch (t: Throwable) { + synchronized(lock) { + ringConnected = false + ringNotificationsReady = false + ringReconnectAfterMs = SystemClock.elapsedRealtime() + ConnectionOptions.RING_RECONNECT_DELAY_MS + } + logLine("direct ring connect failed (" + reason + "): " + safeMessage(t)) + } + } + + private fun connectRing() { + logLine("connecting direct ring $ringAddress") + if (!bleManager.connect(ringAddress, ConnectionOptions.CONNECT_TIMEOUT_MS)) { + throw IllegalStateException("connect failed: $ringAddress") + } + + bleManager.requestConnectionPriority(ringAddress, BluetoothGatt.CONNECTION_PRIORITY_HIGH) + bleManager.requestMtu(ringAddress, ConnectionOptions.RING_DESIRED_MTU, ConnectionOptions.CONNECT_TIMEOUT_MS) + + if (!bleManager.discoverServices(ringAddress, ConnectionOptions.SERVICES_TIMEOUT_MS)) { + throw IllegalStateException("discoverServices failed: $ringAddress") + } + + val phoneNotify = enableRingNotification(BleProtocol.R1_PHONE_NOTIFY_CHAR_UUID) + val dataNotify = enableRingNotification(BleProtocol.R1_NOTIFY_CHAR_UUID) + if (!phoneNotify && !dataNotify) { + throw IllegalStateException("no R1 notify characteristic subscribed") + } + + synchronized(lock) { + ringConnected = true + ringNotificationsReady = true + ringReconnectAfterMs = 0 + } + logLine("direct ring ready phoneNotify=$phoneNotify dataNotify=$dataNotify") + } + + private fun enableRingNotification(characteristicUuid: String): Boolean { + try { + return bleManager.enableNotifications( + ringAddress, + characteristicUuid, + true, + ConnectionOptions.DESCRIPTOR_TIMEOUT_MS + ) + } catch (t: Throwable) { + Log.d(TAG, "direct ring notify subscribe skipped: " + characteristicUuid + " " + safeMessage(t)) + return false + } + } + + /** + * Complete the sid-0x80 security-auth exchange on both freshly opened arm + * connections. Firmware 2.2.9 answers no queries until it completes over an + * encrypted link and closes unauthenticated links after ~30 s (see + * ../notes/ble-connections-2.2.9.md); on an unbonded phone the exchange is + * also what triggers SMP pairing. Deliberately soft: on timeout we log and + * continue rather than fail the connect — the custom firmware's response + * behavior is not yet hardware-verified, and on stock firmware an + * unanswered auth just means the prelude fails exactly as it did before. + * A pairing prompt accepted after our window still bonds at the OS level, + * so the next reconnect attempt authenticates promptly. + */ + @Throws(InterruptedException::class) + private fun authenticateArms() { + val right = messageBuilder.securityAuth(false) + val left = messageBuilder.securityAuth(true) + val now = SystemClock.elapsedRealtime() + for (message in arrayOf(right, left)) { + message.onAck = MessageCallback { + } + message.onTimeout = MessageCallback { + } + message.sentAtMs = now + writeMessage(message) + } + val deadline = SystemClock.elapsedRealtime() + ConnectionOptions.SECURITY_AUTH_SOFT_TIMEOUT_MS + while (running && !userDisconnectRequested && !inFlightMessages.isEmpty()) { + synchronized(lock) { + if (!running || userDisconnectRequested || inFlightMessages.isEmpty()) { + break + } + } + val remaining = deadline - SystemClock.elapsedRealtime() + if (remaining <= 0) { + break + } + interruptibleSleep.sleep(Math.min(remaining, 100L)) + } + synchronized(lock) { + if (!inFlightMessages.isEmpty()) { + clearInFlightMessagesLocked("security auth timeout") + logLine("security auth not acknowledged; continuing (2.2.9 stock requires it; older/custom firmware may not answer)") + return + } + } + val rightOk = BleProtocol.isAuthenticationSuccess(right.ackPayload, right.magic) + val leftOk = BleProtocol.isAuthenticationSuccess(left.ackPayload, left.magic) + logLine("security auth R=" + (if (rightOk) "ok" else "unconfirmed") + " L=" + (if (leftOk) "ok" else "unconfirmed")) + } + + @Throws(InterruptedException::class) + private fun sendPrelude() { + sendPrelude(false) + } + + @Throws(InterruptedException::class) + private fun sendPrelude(preserveWakeLeaseControls: Boolean) { + synchronized(lock) { + if (preserveWakeLeaseControls) { + clearAllMessagesPreservingWakeLeaseLocked("prelude") + } else { + clearAllMessagesLocked("prelude") + } + } + val now = SystemClock.elapsedRealtime() + val prelude = messageBuilder.prelude() + prelude.onAck = MessageCallback { + } + prelude.onTimeout = MessageCallback { + handleTransportFailure("ack timeout") + } + prelude.sentAtMs = now + writeMessage(prelude) + + val deadline = SystemClock.elapsedRealtime() + ConnectionOptions.PRELUDE_TIMEOUT_MS + while (running && !userDisconnectRequested && !inFlightMessages.isEmpty()) { + synchronized(lock) { + if (!running || userDisconnectRequested || inFlightMessages.isEmpty()) { + break + } + } + val remaining = deadline - SystemClock.elapsedRealtime() + if (remaining <= 0) { + break + } + interruptibleSleep.sleep(Math.min(remaining, 100L)) + } + synchronized(lock) { + if (!inFlightMessages.isEmpty()) { + clearInFlightMessagesLocked("prelude timeout") + throw IllegalStateException("prelude ack timeout") + } + } + } + + private fun driveSession(): Long { + //Log.d(TAG, "driveSession called (pendingMessages.size=" + pendingMessages.size() + " inFlightMessages.size=" + inFlightMessages.size() + ")"); + while (true) { + // Run on the sender thread, outside `lock`: Bluetooth API calls can + // wait on the global GATT lock, and callbacks need the session lock. + // Keep negotiation outside the timed stream. A fixed settling period + // is only an experiment boundary; HCI must confirm actual parameters. + var linkMode = -1 + var benchmarkAddresses: Array? = null + synchronized(lock) { + if (benchmarkActive && benchmarkLinkPending) { + benchmarkLinkPending = false + linkMode = benchmarkLinkMode + benchmarkAddresses = arrayOf(leftAddress, rightAddress) + } + } + val addressesToPrepare = benchmarkAddresses + if (addressesToPrepare != null) { + for (address in addressesToPrepare) { + try { + bleManager.prepareBenchmarkLink(address, linkMode) + } catch (error: RuntimeException) { + logLine("benchmark link request failed: " + safeMessage(error)) + } + } + synchronized(lock) { + // Cancellation/new-run races leave the new run pending; its + // own preparation will replace this deadline before sending. + benchmarkReadyAtMs = SystemClock.elapsedRealtime() + 1_000 + } + } + var messageToWrite: OutboundMessage? = null + var messageToPrewrite: OutboundMessage? = null + val now = SystemClock.elapsedRealtime() + + maybeFinishNoChangeDesiredFrame() + + synchronized(lock) { + drainCfwAcknowledgementsLocked() + if (cfwCleanupDelivered) { + /* Successful mode 11 must be the last Faceclaw write. Drop + * anything a late external producer attempted to enqueue + * while DashboardController was closing the transport. */ + clearPendingMessagesLocked("after CFW cleanup") + return 250 + } + if (sessionReady + && now - lastFaceclawFramebufferLeaseQueuedAtMs >= FACECLAW_WAKE_LEASE_RENEW_MS + && !hasPendingOrInflightKindLocked("framebuffer-lease-control")) { + enqueueFaceclawFramebufferControlLocked( + BleProtocol.FACECLAW_FB_OP_ACQUIRE, + false + ) + } + if (faceclawWakeLeaseEnabled + && sessionReady + && now - lastFaceclawWakeLeaseQueuedAtMs >= FACECLAW_WAKE_LEASE_RENEW_MS + && !hasPendingOrInflightKindLocked("wake-lease-control")) { + enqueueFaceclawWakeControlLocked( + BleProtocol.FACECLAW_WAKE_OP_ACQUIRE, + 0, + false + ) + } + if (!inFlightMessages.isEmpty()) { + val oldest = inFlightMessages.peekFirst() + val replay = CfwMessageWindow.replayWindow(inFlightMessages, now) + if (!replay.isEmpty()) { + logLine("CFW recovery: replay " + replay.size + + " unresolved message(s) from id=" + replay[0].magic) + for (candidate in replay) { + if (candidate.cfwRetries >= CfwMessageWindow.MAX_RETRIES) { + handleTransportFailure("CFW recovery retry limit") + return 0 + } + logLine("CFW replay id=" + candidate.magic + " label=" + candidate.label + + " cause=" + (if (candidate.cfwRetryPending) "NACK" + else if (candidate.cfwAckLenses == CfwTransport.BOTH) "ordered tail" else "missing ACK") + + " ackedLenses=" + candidate.cfwAckLenses + + " ageMs=" + (now - candidate.sentAtMs)) + inFlightMessages.remove(candidate) + magicPool.release(candidate.sid, candidate.magic, candidate.label, "CFW replay") + candidate.prepareCfwReplay(magicPool.allocate()) + } + for (i in replay.indices.reversed()) pendingMessages.addFirst(replay[i]) + return 0 + } + if (oldest != null && oldest.ackDeadlineAtMs <= now) { + Log.i(TAG, "message timed out: " + oldest.label) + inFlightMessages.removeFirst() + logLine("message timed out: " + oldest.label + " sid=" + oldest.sid + + " id=" + oldest.magic + " ackedLenses=" + oldest.cfwAckLenses + + " ageMs=" + (now - oldest.sentAtMs)) + magicPool.release(oldest.sid, oldest.magic, oldest.label, "timeout") + if (oldest.sid == CfwTransport.SID) + cfwTransports[if (oldest.isLeftArmMessage) 0 else 1].reset() + handleAckTimeoutLocked(oldest) + return 0 + } + if (connectionOptions.WINDOW_SIZE <= 1 + && sessionReady + && prewrittenMessage == null + && !pendingMessages.isEmpty() + && canPrewriteCandidate(pendingMessages.peekFirst()) + && !shouldBlockPrewriteForHeartbeatLocked(now)) { + // Only the serial (window==1) path pre-sends the all-but-last + // packet; with a real window we just send the next message fully. + messageToPrewrite = pendingMessages.peekFirst() + } + } + + if (chargingMode) { + // Glasses are in the case: no display traffic, only battery + // polls (which also detect the end of charging). + finishDesiredFrameLocked("discarded: glasses charging") + if (sessionReady && inFlightMessages.isEmpty() && !pendingMessages.isEmpty()) { + messageToWrite = pendingMessages.removeFirst() + Log.i(TAG, "sending pending message (charging): " + messageToWrite!!.label) + } else if (sessionReady && pendingMessages.isEmpty() && inFlightMessages.isEmpty() + && now - lastBatteryRefreshAtMs >= ConnectionOptions.CHARGING_BATTERY_POLL_MS) { + Log.i(TAG, "Writing charging-mode battery poll") + messageToWrite = createBatteryQueryMessageLocked() + lastBatteryRefreshAtMs = now + } else { + return 1_000 + } + } else { + if (messageToPrewrite == null + && !shutdownRequested + && !fixedLayoutCreated + && pendingMessages.isEmpty() + && inFlightMessages.isEmpty()) { + Log.i(TAG, "enqueueing create layout") + enqueueCreateLayoutLocked() + } else if (messageToPrewrite != null) { + // Prewrite outside the lock; the logical message remains pending until + // its final BLE frame is sent after the current protocol ACK. + } + if (messageToPrewrite == null && !shutdownRequested && fixedLayoutCreated + && (if (firmwareDebugFlagsEnabled) 2 else 1) != firmwareDebugFlagsLastSent + && pendingMessages.isEmpty() && inFlightMessages.isEmpty()) { + Log.i(TAG, "enqueueing firmware debug flags " + (if (firmwareDebugFlagsEnabled) "show" else "hide")) + enqueueFirmwareDebugFlagsLocked() + } + + if (messageToPrewrite == null && !shutdownRequested && fixedLayoutCreated + && (if (compassOwners.isEmpty()) 0 else 1) != compassControlLastSent + && pendingMessages.isEmpty() && inFlightMessages.isEmpty()) { + val wanted = compassOwners.isNotEmpty() + Log.i(TAG, "enqueueing compass " + (if (wanted) "enable" else "disable")) + enqueueCompassControlLocked(false, wanted) + } + + if (benchmarkActive) { + maintainBenchmarkLocked(now) + } + + // Up to WINDOW_SIZE messages may be in flight at once (full + // pipelining); a slot frees when an ack arrives. An active + // bandwidth benchmark measures its own selected window instead. + val windowHasRoom = inFlightMessages.size < + Math.max(1, if (benchmarkActive) benchmarkWindowSize else connectionOptions.WINDOW_SIZE) + // A frame ready to send right now: don't inject a fresh + // heartbeat in front of it (the image's own ack resets the + // firmware heartbeat timer, so the heartbeat is redundant). + // "Ready" covers both a fresh composite still to be planned + // and a planned image already queued but not yet written -- + // enqueueing sets lastEnqueuedFingerprint, so testing only + // the desired fingerprint left the queued-but-unwritten + // window unprotected and a heartbeat that came due there + // cost the frame a full ack round trip (measured 124ms on + // frame#232 of the 2026-08-20 02:27 capture). + // A benchmark run counts as image traffic here for the same + // reason: its acks reset the firmware heartbeat timer, so a + // fresh heartbeat in front of it is redundant. + val imageWaiting = !shutdownRequested && fixedLayoutCreated + && !hasPendingOrInflightKindLocked("heartbeat") + && (benchmarkActive + || (now >= imageRetryAfterMs + && (hasPendingImageLocked() + || getDesiredFingerprint() != lastEnqueuedFingerprint))) + noteImageStallLocked(now, windowHasRoom) + if (messageToPrewrite == null && handleHeartbeat(imageWaiting)) { + return ConnectionOptions.IDLE_SLEEP_MS.toLong() + } + + if (messageToPrewrite == null && sessionReady && windowHasRoom && !pendingMessages.isEmpty() + && CfwMessageWindow.canSend(inFlightMessages, pendingMessages.peekFirst()!!)) { + messageToWrite = pendingMessages.removeFirst() + Log.i(TAG, "sending pending message: " + messageToWrite!!.label) + } else if (messageToPrewrite == null && !shutdownRequested && !benchmarkActive + && fixedLayoutCreated + && windowHasRoom && !hasPendingImageLocked() + && now >= imageRetryAfterMs + && getDesiredFingerprint() != lastEnqueuedFingerprint) { + // Enqueue the next frame's delta against lastEnqueuedPacked + // (what the shadow will be), so it can pipeline behind an + // image still awaiting its ack. + Log.i(TAG, "Enqueued image update") + enqueueDesiredImageLocked() + return 0 + } else if (messageToPrewrite == null && shouldPollBatteryLocked(now)) { + Log.i(TAG, "Writing battery query") + messageToWrite = createBatteryQueryMessageLocked() + lastBatteryRefreshAtMs = now + } else if (messageToPrewrite == null && (!pendingMessages.isEmpty() || !inFlightMessages.isEmpty())) { + return ConnectionOptions.IDLE_SLEEP_MS.toLong() + } else if (messageToPrewrite == null) { + return 250 + } + } + } + + val prewrite = messageToPrewrite + if (prewrite != null) { + if (prewriteMessage(prewrite)) { + return 0 + } + return ConnectionOptions.IDLE_SLEEP_MS.toLong() + } + + val write = messageToWrite!! + if (!writeMessage(write)) { + synchronized(lock) { + if (removePreparedMessageLocked(write) || write.magic == 0) { + handleTransportFailure("write failed") + } + } + return 0 + } + } + } + + /** + * If the desired image already matches what the glasses display, nothing will + * ever be enqueued for it, so finish its frame now (otherwise the TS side + * would block on it until its backpressure timeout). + */ + private fun maybeFinishNoChangeDesiredFrame() { + var frameIdToFinish = 0 + synchronized(lock) { + synchronized(desiredTilesLock) { + if (desiredFrameId != 0 && lastEnqueuedFingerprint.isNotEmpty() && desiredFingerprint == lastEnqueuedFingerprint) { + frameIdToFinish = desiredFrameId + desiredFrameId = 0 + } + } + } + if (frameIdToFinish != 0) { + finishFrame(frameIdToFinish, "discarded: no change from displayed image") + } + } + + private fun shouldBlockPrewriteForHeartbeatLocked(now: Long): Boolean { + if (shutdownRequested || !fixedLayoutCreated) { + return false + } + return hasPendingOrInflightKindLocked("heartbeat") + || now - lastHeartbeatAckedAtMs >= ConnectionOptions.HEARTBEAT_READY_MS + } + + private fun canPrewriteCandidate(message: OutboundMessage?): Boolean { + if (message == null || message.sid == CfwTransport.SID || !message.isLeftArmMessage) { + return false + } + if ("image" != message.kind) { + return false + } + return message.message.size + 2 > 232 + } + + private fun handleHeartbeat(imageWaiting: Boolean): Boolean { + val now = SystemClock.elapsedRealtime() + val heartbeatEligible = !shutdownRequested && fixedLayoutCreated + val heartbeatPending = heartbeatEligible && hasPendingOrInflightKindLocked("heartbeat") + val heartbeatElapsedMs = now - lastHeartbeatAckedAtMs + val heartbeatReady = heartbeatEligible && heartbeatElapsedMs >= ConnectionOptions.HEARTBEAT_READY_MS + val heartbeatUrgent = heartbeatEligible && heartbeatElapsedMs >= ConnectionOptions.HEARTBEAT_URGENT_MS + val heartbeatBlocksLeftWrites = heartbeatReady || heartbeatPending + + if (heartbeatReady && !heartbeatPending && inFlightMessages.isEmpty()) { + if (imageWaiting && !heartbeatUrgent) { + // Defer to the waiting frame: sending it now satisfies the + // firmware heartbeat deadline (its ack resets the timer), and + // the heartbeat still fires once we reach the URGENT threshold + // if rendering goes quiet again. Preserves the pending-heartbeat + // inter-lens-sync invariant below (that path is untouched). + return false + } + Log.i(TAG, "Writing heartbeat") + val heartbeatMessage = createHeartbeatMessage() + lastHeartbeatSentAtMs = now + writeMessage(heartbeatMessage) + return true + } else if (heartbeatUrgent) { + return true + } else if (heartbeatPending) { + // Don't send other message types while a heartbeat is pending because that + // can lead to inter-lens sync issues + return true + } + + return false + } + + private fun createHeartbeatMessage(): OutboundMessage { + val message = messageBuilder.heartbeat() + message.onAck = MessageCallback { + synchronized(lock) { + lastHeartbeatAckedAtMs = SystemClock.elapsedRealtime() + } + } + message.onTimeout = MessageCallback { + // If a heartbeat fails to ack and we're over the heartbeat deadline, assume the connection is failed and reconnect. + // Otherwise ignore it, which will cause a retransmission attempt. + val isPastDeadline: Boolean + synchronized(lock) { + isPastDeadline = SystemClock.elapsedRealtime() - lastHeartbeatSentAtMs >= ConnectionOptions.HEARTBEAT_FAILURE_DEADLINE_MS + } + if (isPastDeadline) { + handleTransportFailure("heartbeat ack timeout") + } + } + return message + } + + + private fun writeMessage(message: OutboundMessage): Boolean { + val now = SystemClock.elapsedRealtime() + message.writeStartedAtMs = now + message.sentAtMs = now + message.ackDeadlineAtMs = now + message.ackTimeoutMs + ConnectionOptions.WRITE_TIMEOUT_MS + if (message.magic != 0) { + synchronized(lock) { + inFlightMessages.addLast(message) + } + } + if (message.imageUpdateId > 0 && message.imageMessageNumber == 1) { + synchronized(lock) { + val stats = imageUpdateStats[message.imageUpdateId] + if (stats != null && stats.firstWriteStartedAtMs <= 0) { + stats.firstWriteStartedAtMs = now + } + } + } + + val writeAddress = if (message.isLeftArmMessage) leftAddress else rightAddress + val frames: List + if (prewrittenMessage != null && prewrittenMessage !== message) { + if (!spoilPrewrittenMessage("before " + message.label)) { + return false + } + } + if (prewrittenMessage === message) { + frames = Collections.singletonList(prewrittenFrames[prewrittenFrames.size - 1]) + prewrittenMessage = null + prewrittenFrames = Collections.emptyList() + } else if (message.sid == CfwTransport.SID) { + if (message.cfwRetries > 0) cfwTransports[if (message.isLeftArmMessage) 0 else 1].reset() + frames = cfwTransports[if (message.isLeftArmMessage) 0 else 1].encode( + message.message, message.magic, CfwTransport.BOTH, + bleManager.getNegotiatedMtu(writeAddress)) + } else { + frames = BleProtocol.framePb( + message.message, + message.sid, + message.flag, + nextTransportSeq++ + ) + } + val result = bleManager.writeFrames( + writeAddress, + BleProtocol.WRITE_CHAR_UUID, + frames, + AndroidProtocolPlatform.writeType(ConnectionOptions.WRITE_MODE), + ConnectionOptions.WRITE_TIMEOUT_MS + ) + + synchronized(lock) { + val sentAtMs = SystemClock.elapsedRealtime() + message.sentAtMs = sentAtMs + message.ackDeadlineAtMs = sentAtMs + message.ackTimeoutMs + logImageUpdateSendLandmarkLocked(message) + val onSent = message.onSent + if (result && onSent != null) { + onSent.run() + lock.notifyAll() + } + } + + if (!result && message.sid == CfwTransport.SID) + cfwTransports[if (message.isLeftArmMessage) 0 else 1].reset() + return result + } + + private fun prewriteMessage(message: OutboundMessage): Boolean { + if (prewrittenMessage === message) { + return true + } + if (prewrittenMessage != null && !spoilPrewrittenMessage("before prewrite " + message.label)) { + return false + } + if (!canPrewriteCandidate(message)) { + return false + } + + val frames: List = BleProtocol.framePb( + message.message, + message.sid, + message.flag, + nextTransportSeq++ + ) + if (frames.size <= 1) { + return false + } + + val writeAddress = if (message.isLeftArmMessage) leftAddress else rightAddress + val prefixFrames = frames.subList(0, frames.size - 1) + val result = bleManager.writeFrames( + writeAddress, + BleProtocol.WRITE_CHAR_UUID, + prefixFrames, + AndroidProtocolPlatform.writeType(ConnectionOptions.WRITE_MODE), + ConnectionOptions.WRITE_TIMEOUT_MS + ) + if (!result) { + return false + } + + prewrittenMessage = message + prewrittenFrames = ArrayList(frames) + logLine("prewrote " + message.label + " frames=" + prefixFrames.size + "/" + frames.size) + return true + } + + private fun spoilPrewrittenMessage(reason: String): Boolean { + val message = prewrittenMessage + if (message == null || prewrittenFrames.isEmpty()) { + prewrittenMessage = null + prewrittenFrames = Collections.emptyList() + return true + } + val finalFrame = Arrays.copyOf( + prewrittenFrames[prewrittenFrames.size - 1], + prewrittenFrames[prewrittenFrames.size - 1].size + ) + if (finalFrame.size > 8) { + finalFrame[finalFrame.size - 1] = (finalFrame[finalFrame.size - 1].toInt() xor 0xff).toByte() + } + prewrittenMessage = null + prewrittenFrames = Collections.emptyList() + + val writeAddress = if (message.isLeftArmMessage) leftAddress else rightAddress + logLine("spoiling prewritten " + message.label + ": " + reason) + return bleManager.writeFrames( + writeAddress, + BleProtocol.WRITE_CHAR_UUID, + Collections.singletonList(finalFrame), + AndroidProtocolPlatform.writeType(ConnectionOptions.WRITE_MODE), + ConnectionOptions.WRITE_TIMEOUT_MS + ) + } + + private fun removePreparedMessageLocked(message: OutboundMessage?): Boolean { + if (message == null || message.magic == 0) { + return false + } + val iterator = inFlightMessages.iterator() + while (iterator.hasNext()) { + if (iterator.next() === message) { + iterator.remove() + magicPool.release(message.sid, message.magic, message.label, "write failed") + return true + } + } + return false + } + + + /** Also drain on the worker: a clear/timeout may have removed a blocking + * head since the last notification. Fully ACKed controls must never reach + * the generic timeout path just because no further BLE reply arrived. */ + private fun drainCfwAcknowledgementsLocked() { + while (true) { + val first = CfwMessageWindow.acknowledgedHead(inFlightMessages) ?: return + lastAckAtMs = SystemClock.elapsedRealtime() + resolveAckLocked(first, first.ackPayload) + } + } + + private fun resolveAckLocked(sid: Int, magic: Int, pb: ByteArray?) { + val iterator = inFlightMessages.iterator() + while (iterator.hasNext()) { + val message = iterator.next() + if (message.sid == sid && message.magic == magic) { + resolveAckLocked(message, pb) + return + } + } + recordUnexpectedAckLocked(sid, magic) + } + + private fun resolveAckLocked(message: OutboundMessage, pb: ByteArray?) { + Log.i(TAG, "Got ACK for " + message.label + "(sid=" + message.sid + ", id=" + message.magic + ")") + inFlightMessages.remove(message) + message.ackPayload = if (pb == null) ByteArray(0) else Arrays.copyOf(pb, pb.size) + magicPool.release(message.sid, message.magic, message.label, "ack") + val onAck = message.onAck + if (onAck != null) { + onAck.run() + } + consecutiveAckTimeouts = 0 + // onAck may have just satisfied a waiter blocked on lock.wait() (e.g. + // awaitEvenHubSessionReady polling fixedLayoutCreated/displayedFingerprint + // after a create-layout or image ack). Without this, that waiter only + // notices on its own up-to-100ms poll tick, adding avoidable latency to + // every EvenHub wake. Always called with lock held (see call site). + lock.notifyAll() + } + + private fun logImageUpdateSendLandmarkLocked(message: OutboundMessage) { + if (message.imageUpdateId <= 0) { + return + } + val stats = imageUpdateStats[message.imageUpdateId] + val frameId = if (stats == null) 0 else stats.frameId + if (message.imageMessageNumber == 1) { + if (stats != null && stats.firstWriteStartedAtMs <= 0) { + stats.firstWriteStartedAtMs = if (message.writeStartedAtMs > 0) message.writeStartedAtMs else message.sentAtMs + } + FrameTimings.getInstance().log(frameId, "first bluetooth packet sent") + logImageUpdateLandmarkLocked("first bluetooth message sent", message, message.sentAtMs) + } + if (message.imageMessageNumber == message.imageMessageCount) { + FrameTimings.getInstance().log(frameId, + "last bluetooth packet sent (message " + message.imageMessageNumber + "/" + message.imageMessageCount + ")") + logImageUpdateLandmarkLocked("last bluetooth message sent", message, message.sentAtMs) + } + } + + private fun logImageUpdateAckLandmarkLocked(message: OutboundMessage) { + if (message.imageUpdateId <= 0 || message.imageMessageNumber != message.imageMessageCount) { + return + } + val ackedAtMs = SystemClock.elapsedRealtime() + FaceclawBleManager.recordDisplayFrameSent() + val stats = imageUpdateStats.remove(message.imageUpdateId) + if (stats != null && stats.firstWriteStartedAtMs > 0) { + emitFrameMetrics(stats.paintMs, Math.max(0L, ackedAtMs - stats.firstWriteStartedAtMs).toInt(), stats.tileCount) + } + if (stats != null) { + finishFrame(stats.frameId, "sent") + } + logImageUpdateLandmarkLocked("last bluetooth message acked", message, ackedAtMs) + } + + /** Remove the stats entry for an image update that will not complete, finishing its frame. */ + private fun discardImageUpdateStatsLocked(imageUpdateId: Int, reason: String) { + if (imageUpdateId <= 0) { + return + } + val stats = imageUpdateStats.remove(imageUpdateId) + if (stats != null) { + finishFrame(stats.frameId, "discarded: $reason") + } + } + + private fun logImageUpdateLandmarkLocked(event: String, message: OutboundMessage, elapsedMs: Long) { + logLine("image update#" + message.imageUpdateId + " " + event + + " at " + timestamp(elapsedMs) + + " message=" + message.imageMessageNumber + "/" + message.imageMessageCount + + " label=" + message.label) + } + + private fun enqueueCreateLayoutLocked() { + // New session: re-assert the firmware-debug-flags overlay once + // the layout is ready (the mode-7 send is gated on this having reset). + firmwareDebugFlagsLastSent = -1 + val message = messageBuilder.createLayout() + message.onAck = MessageCallback { + startupProbePending = false + clearMessagesOfKindLocked("startup-text-probe") + fixedLayoutCreated = true + displayedFingerprint = "" + } + message.onTimeout = MessageCallback { + if (startupProbePending) { + logLine("create layout timed out while startup text probe is pending") + if (hasPendingOrInflightKindLocked("startup-text-probe")) { + return@MessageCallback + } + startupProbePending = false + } + handleTransportFailure("ack timeout") + } + pendingMessages.addLast(message) + logLine("queue create layout") + } + + private fun enqueueStartupProbeLocked() { + enqueueCreateLayoutLocked() + + val message = messageBuilder.startupTextProbe() + message.onAck = MessageCallback { + startupProbePending = false + clearMessagesOfKindLocked("create-layout") + fixedLayoutCreated = true + displayedFingerprint = "" + logLine("existing dashboard layout accepted text probe") + } + message.onTimeout = MessageCallback { + startupProbePending = false + if (hasPendingOrInflightKindLocked("create-layout")) { + return@MessageCallback + } + handleTransportFailure("ack timeout") + } + pendingMessages.addLast(message) + startupProbePending = true + logLine("queue startup text probe") + } + + /** + * Send the CFW mode-7 diagnostic-flag control op through the private stream: + * [7][2] to show the on-glasses debug-flag overlay, [7][1] to hide it. Uses the + * arbitrary-payload custom path (no bmp/dedup/frame-timing interaction). + */ + private fun enqueueFirmwareDebugFlagsLocked() { + val show = firmwareDebugFlagsEnabled + val sub = if (show) 2 else 1 + val payload = byteArrayOf(CFW_MSG_DIAGNOSTICS.toByte(), sub.toByte()) + val message = messageBuilder.imagePayload( + DASHBOARD_TILE, nextMapSessionId(), payload, + "fw-debug-flags " + (if (show) "show" else "hide"), + connectionOptions.sendImagesToLeft) + pendingMessages.addLast(message) + firmwareDebugFlagsLastSent = sub + logLine("queue firmware debug flags " + (if (show) "show" else "hide")) + } + + /** + * Send CFW image-handler mode 10. Enable uses the configurable form + * [10][2][interval-ms LE16][minimum-change-degrees LE16]; disable remains [10][0]. + */ + private fun enqueueCompassControlLocked(priority: Boolean, enable: Boolean) { + val sentState = if (enable) 1 else 0 + val payload = if (enable) + byteArrayOf( + CFW_MSG_COMPASS.toByte(), + 2.toByte(), + (COMPASS_REPORT_INTERVAL_MS and 0xff).toByte(), + ((COMPASS_REPORT_INTERVAL_MS shr 8) and 0xff).toByte(), + (COMPASS_MIN_CHANGE_DEGREES and 0xff).toByte(), + ((COMPASS_MIN_CHANGE_DEGREES shr 8) and 0xff).toByte(), + ) + else + byteArrayOf(CFW_MSG_COMPASS.toByte(), 0.toByte()) + val message = messageBuilder.imagePayload( + "compass-control", + DASHBOARD_TILE, + nextMapSessionId(), + payload, + "compass " + (if (enable) "enable" else "disable"), + connectionOptions.sendImagesToLeft) + message.onTimeout = MessageCallback { + compassControlLastSent = -1 + logLine("compass control ack timeout") + } + if (enable) { + compassMaybeOn = true + } else { + message.onAck = MessageCallback { compassMaybeOn = false } + } + if (priority) pendingMessages.addFirst(message) + else pendingMessages.addLast(message) + compassControlLastSent = sentState + logLine("queue " + message.label) + } + + private fun enqueueDesiredImageLocked() { + val fingerprint = getDesiredFingerprint() + val packedSnapshot: ByteArray? + val width: Int + val height: Int + val paintMs: Int + val frameId: Int + val draws: Array? + val scene: ShellScene + synchronized(desiredTilesLock) { + packedSnapshot = desiredPacked + width = desiredWidth + height = desiredHeight + paintMs = desiredPaintMs + frameId = desiredFrameId + draws = desiredDraws + scene = desiredShellScene + desiredFrameId = 0 + } + val packedFrame: ByteArray = packedSnapshot ?: ByteArray(0) + if (customFirmwareDetected && packedFrame.size > 0) { + val rendered = scenePlanner.plan(packedFrame, width, height, draws, scene, nextImageFrameId, + connectionOptions.TEXTURE_CACHE_FRAMES, connectionOptions.INCREMENTAL_FRAMES, + connectionOptions.MULTI_RECT_FRAMES, ConnectionOptions.MULTI_RECT_MAX_RECTS) + nextImageFrameId = rendered.nextFid + val commands = rendered.commands + for (i in 0 until commands.size - 1) enqueueResourceCommandLocked(commands[i]) + val plan = BleImageOptimizer.TileImagePlan( + 0, DASHBOARD_TILE, packedFrame, width, height, nextMapSessionId(), commands[commands.size - 1]) + finishEnqueueDesiredImageLocked(plan, fingerprint, paintMs, frameId) + return + } + + FrameTimings.getInstance().spanStart(frameId, "compress-and-plan") + // Incremental (mode 3 bounding box) update against the last ENQUEUED frame + // (the base the firmware shadow will hold when this update is applied). + // lastEnqueuedPacked is cleared whenever the image pipeline is cleared, so + // a non-empty value means the display base is trusted. + var incrementalPayload: ByteArray? = null + var incrementalLog: String? = null + if (connectionOptions.INCREMENTAL_FRAMES && lastEnqueuedPacked.size > 0 + && lastEnqueuedWidth == width && lastEnqueuedHeight == height) { + val baseFid = nextImageFrameId + val single = + BleImageOptimizer.buildIncrementalImagePayload(lastEnqueuedPacked, packedFrame, width, height, baseFid) + if (single != null) { + incrementalPayload = single.payload + // advance only when a delta is actually emitted, so consecutive + // deltas carry consecutive ids (CFW skip/reorder detection) + nextImageFrameId = if (nextImageFrameId >= 0xfffe) 1 else nextImageFrameId + 1 + incrementalLog = "incremental update bbox=" + + ((single.payload[3].toInt() and 0xff) * 4) + "x" + ((single.payload[4].toInt() and 0xff) * 2) + + "+" + ((single.payload[1].toInt() and 0xff) * 4) + "+" + ((single.payload[2].toInt() and 0xff) * 2) + + " changed=" + single.changedBytes + "/" + single.boxBytes + "B" + + " clusters=" + single.clusterCount + + // When the bounding box spans multiple clusters or is sizeable, try + // splitting into tight rects (CFW mode-8). Only replace the single + // box if the multi-rect message is actually smaller on the wire. + if (connectionOptions.MULTI_RECT_FRAMES + && (single.clusterCount > 1 || single.payload.size > ConnectionOptions.MULTI_RECT_MIN_PAYLOAD)) { + val multi = BleImageOptimizer.buildMultiRectImagePayload( + lastEnqueuedPacked, packedFrame, width, height, baseFid, ConnectionOptions.MULTI_RECT_MAX_RECTS) + if (multi != null && multi.payload.size < single.payload.size) { + incrementalPayload = multi.payload + nextImageFrameId = multi.nextFid // rectCount fids consumed + incrementalLog = "multi-rect update n=" + multi.rectCount + + " covered=" + multi.coveredBytes + "B" + + " payload=" + multi.payload.size + "B (vs bbox " + single.payload.size + "B)" + } + } + } + } + val plan = if (incrementalPayload != null) + BleImageOptimizer.TileImagePlan(0, DASHBOARD_TILE, packedFrame, width, height, nextMapSessionId(), incrementalPayload) + else + BleImageOptimizer.TileImagePlan(0, DASHBOARD_TILE, packedFrame, width, height, nextMapSessionId()) + FrameTimings.getInstance().spanEnd(frameId, "compress-and-plan") + if (incrementalLog != null) { + FrameTimings.getInstance().log(frameId, incrementalLog) + } + finishEnqueueDesiredImageLocked(plan, fingerprint, paintMs, frameId) + } + + /** Queue complete private commands, advance the delta base, and retain frame/ACK bookkeeping. */ + private fun finishEnqueueDesiredImageLocked( + plan: BleImageOptimizer.TileImagePlan, fingerprint: String, paintMs: Int, frameId: Int) { + val updateId = nextImageUpdateId++ + val commands: MutableList = ArrayList() + if (plan.payload.size <= CfwTransport.MAX_MESSAGE) { + commands.add(plan.payload) + } else { + // A noisy full frame can exceed the stream record limit. Repaint it + // with independently decodable RLE bands below the decoded-message limit. + commands.addAll(BleImageOptimizer.encodeFullFrameBands( + plan.packed!!, plan.width, plan.height, nextImageFrameId)) + for (i in 0 until commands.size) { + nextImageFrameId = if (nextImageFrameId >= 0xfffe) 1 else nextImageFrameId + 1 + } + } + val messageCount = commands.size + imageUpdateStats[updateId] = BleImageOptimizer.ImageUpdateStats(paintMs, 1, frameId) + for (i in 0 until commands.size) { + enqueueCustomImageLocked(plan, commands[i], fingerprint, updateId, i + 1, messageCount) + } + // This frame is now the base for the next delta (it will be the firmware + // shadow once applied), even though it hasn't been acked yet — that is what + // lets the next frame pipeline behind it. plan.packed is the full frame; + // frames are immutable by convention, so referencing it is safe. + lastEnqueuedPacked = plan.packed!! + lastEnqueuedWidth = plan.width + lastEnqueuedHeight = plan.height + lastEnqueuedFingerprint = fingerprint + + FrameTimings.getInstance().log(frameId, "queued image update#" + updateId + + " messages=" + messageCount + " payload=" + plan.payload.size + "B") + logLine("queue image update#" + updateId + " fingerprint=" + fingerprint + + " messages=" + messageCount) + } + + /** + * Enqueue one resource eviction/upload command ahead of the image message that + * references its glyphs (the transport is FIFO, so no ack round trip is + * needed before use). A timeout means the on-glasses cache state is + * unknown; forget everything phone-side (glyphs re-upload lazily) and let + * the accompanying image update's own timeout drive the frame resync. + */ + private fun enqueueResourceCommandLocked(payload: ByteArray) { + val message = messageBuilder.imagePayload( + "resources", + DASHBOARD_TILE, + nextMapSessionId(), + payload, + "resource command " + payload.size + "B", + connectionOptions.sendImagesToLeft) + message.onTimeout = MessageCallback { + resourceCache.reset() + logLine("resource command ack timeout; resource cache state reset") + } + pendingMessages.addLast(message) + logLine("queue " + message.label) + } + + private fun enqueueCustomImageLocked( + plan: BleImageOptimizer.TileImagePlan, + payload: ByteArray, + fingerprint: String, + updateId: Int, + messageNumber: Int, + messageCount: Int + ) { + val message = messageBuilder.customMessage("image", payload, + "image " + plan.tile.name + "#" + messageNumber, plan.tileIndex, connectionOptions.sendImagesToLeft) + message.setImageUpdatePosition(updateId, messageNumber, messageCount) + message.onAck = MessageCallback { + imageRetryAfterMs = 0 + // Firmware >= 2.2.4.34 resets its heartbeat timer when it receives + // image messages (not just heartbeats), so an acked image fragment + // satisfies the heartbeat deadline and heartbeats stop contending + // with active rendering. + lastHeartbeatAckedAtMs = SystemClock.elapsedRealtime() + logImageUpdateAckLandmarkLocked(message) + var imageStillInFlight = false + for (inFlight in inFlightMessages) { + if ("image" == inFlight.kind) { + imageStillInFlight = true + break + } + } + if (!imageStillInFlight) { + var imageStillQueued = false + for (queued in pendingMessages) { + if ("image" == queued.kind) { + imageStillQueued = true + break + } + } + if (!imageStillQueued) { + displayedFingerprint = fingerprint + } + } + } + message.onTimeout = MessageCallback { + discardImageUpdateStatsLocked(message.imageUpdateId, "image ack timeout (will retry)") + clearMessagesOfKindLocked("image") + displayedFingerprint = "" + imageRetryAfterMs = SystemClock.elapsedRealtime() + ConnectionOptions.IMAGE_RETRY_DELAY_MS + } + pendingMessages.addLast(message) + } + + private fun createAudioControlMessageLocked(enable: Boolean): OutboundMessage { + val message = messageBuilder.enableOrDisableMic(enable) + message.onAck = MessageCallback { + lastAudioControlAckMagic = message.magic + audioCaptureActive = message.label?.contains("enable") == true + logLine(if (audioCaptureActive) "G2 mic enabled" else "G2 mic disabled") + } + message.onTimeout = MessageCallback { + handleTransportFailure("audio control ack timeout") + } + return message + } + + private fun shouldPollBatteryLocked(now: Long): Boolean { + return !shutdownRequested + && sessionReady + && pendingMessages.isEmpty() + && inFlightMessages.isEmpty() + && now - lastConnectionOrInputAtMs >= ConnectionOptions.BATTERY_INPUT_QUIET_MS + && (lastBatteryRefreshAtMs == 0L || now - lastBatteryRefreshAtMs >= ConnectionOptions.BATTERY_REFRESH_INTERVAL_MS) + } + + private fun createBatteryQueryMessageLocked(): OutboundMessage { + val message = messageBuilder.batteryQuery() + message.onAck = MessageCallback { + val ring = BleProtocol.parseRingBattery(message.ackPayload) + // Old firmware and missing/malformed extensions must clear any prior reading. + ringBattery = if (ring == null) -1 else ring.battery + ringCharging = if (ring == null) -1 else ring.charging + val snapshot = BleProtocol.parseSettingsBattery(message.ackPayload) + if (snapshot != null) { + headsetBattery = snapshot.battery + headsetCharging = snapshot.charging + emitBatteryState(headsetBattery, headsetCharging) + if (snapshot.silentMode >= 0) { + // Backstop for the push in onNotification: the firmware is + // confirmed to push silent-mode-on, but the off transition is + // not, so re-read the authoritative value on every poll. + updateSilentModeLocked(snapshot.silentMode > 0) + } + updateChargingModeLocked(snapshot.charging > 0, snapshot.battery) + } + val firmwareInfo = BleProtocol.parseSettingsFirmwareInfo(message.ackPayload) + if (firmwareInfo != null) { + customFirmwareDetected = firmwareInfo.isFaceclawFirmware() + emitFirmwareInfo(firmwareInfo) + } + } + message.onTimeout = MessageCallback { + logLine("Battery query timed out") + } + return message + } + + /** + * Track silent mode, which the wearer toggles by long-pressing both + * touchpads at once. While it is on the firmware refuses input events and + * app launches and powers the display down, so the glasses look dead even + * though the BLE session is healthy; the phone UI says so explicitly. + */ + private fun updateSilentModeLocked(silent: Boolean) { + val next = if (silent) 1 else 0 + if (silentMode == next) { + return + } + silentMode = next + logLine(if (silent) "glasses entered silent mode" else "glasses left silent mode") + emitSilentMode(silent) + } + + /** + * Track whether the glasses are in the charging case. Charging means nobody + * is wearing them: display communication pauses (no heartbeats, so the + * firmware tears down its EvenHub context on its own) and only battery polls + * continue. When charging stops, tear the transport down and let the normal + * reconnect loop rebuild the session, layout, and first frame. + */ + private fun updateChargingModeLocked(charging: Boolean, battery: Int) { + if (charging == chargingMode) { + if (chargingMode) { + setStateDisplay("charging", chargingStatusText(battery)) + } + return + } + if (charging) { + chargingMode = true + clearAllMessagesLocked("glasses charging") + fixedLayoutCreated = false + startupProbePending = false + displayedFingerprint = "" + finishDesiredFrameLocked("discarded: glasses charging") + logLine("glasses are charging; pausing display communication") + setStateDisplay("charging", chargingStatusText(battery)) + } else { + chargingMode = false + logLine("glasses removed from charger; reconnecting") + handleTransportFailure("charging ended") + } + } + + /** + * Record, into the frame that is waiting, why it did not go out on this + * pass of the send loop. Without this the export shows a bare multi-second + * jump between "image submitted as desired frame" and the first BLE + * packet, with no hint whether we were blocked on a heartbeat, the + * BLE window, or another message queued ahead. Deduped on (frame, reason), + * so a frame stalled for seconds gets one line per state change rather + * than one per loop pass. + */ + private fun noteImageStallLocked(now: Long, windowHasRoom: Boolean) { + var frameId: Int + synchronized(desiredTilesLock) { + frameId = desiredFrameId + } + val reason: String? + if (frameId != 0 && getDesiredFingerprint() != lastEnqueuedFingerprint) { + reason = describeEnqueueBlockerLocked(now, windowHasRoom) + } else { + // Nothing waiting to be planned; an already-planned image may still + // be queued behind other traffic. + val queuedImage = firstPendingImageLocked() + frameId = if (queuedImage == null) 0 else imageUpdateFrameIdLocked(queuedImage.imageUpdateId) + reason = if (queuedImage == null) null else describeWriteBlockerLocked(now, windowHasRoom, queuedImage) + } + if (frameId == 0 || reason == null) { + stallFrameId = 0 + stallReason = "" + return + } + if (frameId == stallFrameId && reason == stallReason) { + return + } + stallFrameId = frameId + stallReason = reason + FrameTimings.getInstance().log(frameId, "waiting to send: $reason") + } + + /** Why the desired composite has not been turned into wire messages yet, or null. */ + private fun describeEnqueueBlockerLocked(now: Long, windowHasRoom: Boolean): String? { + if (shutdownRequested) { + return "shutdown requested" + } + if (!sessionReady) { + return "BLE session not ready" + } + if (!fixedLayoutCreated) { + return "display layout not created yet" + } + if (now < imageRetryAfterMs) { + return "image retry backoff (" + (imageRetryAfterMs - now) + "ms left)" + } + if (hasPendingImageLocked()) { + return "an earlier image is still queued" + } + if (!windowHasRoom) { + return "BLE window full (" + inFlightMessages.size + " message(s) in flight)" + } + if (hasPendingOrInflightKindLocked("heartbeat")) { + return "heartbeat in flight" + } + if (!pendingMessages.isEmpty()) { + return pendingMessages.size.toString() + " message(s) queued ahead, next " + + pendingMessages.peekFirst().label + } + return null + } + + /** Why a planned image message has not been written to BLE yet, or null. */ + private fun describeWriteBlockerLocked(now: Long, windowHasRoom: Boolean, queuedImage: OutboundMessage): String? { + if (!sessionReady) { + return "BLE session not ready" + } + if (!windowHasRoom) { + return "BLE window full (" + inFlightMessages.size + " message(s) in flight)" + } + if (hasPendingOrInflightKindLocked("heartbeat")) { + return "heartbeat in flight" + } + // handleHeartbeat is a barrier: while one is due it holds back every + // other write, so a frame queued at the wrong moment waits a heartbeat + // round trip. Reported explicitly because it is otherwise invisible -- + // heartbeats belong to no frame. + val heartbeatElapsedMs = now - lastHeartbeatAckedAtMs + if (fixedLayoutCreated && !shutdownRequested + && heartbeatElapsedMs >= ConnectionOptions.HEARTBEAT_READY_MS) { + return "heartbeat due (" + heartbeatElapsedMs + "ms since the last one acked)" + } + val head = pendingMessages.peekFirst() + if (head != null && head !== queuedImage) { + return "queued behind " + head.label + } + return null + } + + private fun firstPendingImageLocked(): OutboundMessage? { + for (message in pendingMessages) { + if (message.imageUpdateId > 0) { + return message + } + } + return null + } + + private fun imageUpdateFrameIdLocked(imageUpdateId: Int): Int { + val stats = imageUpdateStats[imageUpdateId] + return if (stats == null) 0 else stats.frameId + } + + private fun finishDesiredFrameLocked(outcome: String) { + val frameIdToFinish: Int + synchronized(desiredTilesLock) { + frameIdToFinish = desiredFrameId + desiredFrameId = 0 + } + finishFrame(frameIdToFinish, outcome) + } + + private fun hasPendingOrInflightKindLocked(kind: String): Boolean { + for (queued in pendingMessages) { + if (kind == queued.kind) { + return true + } + } + for (inFlight in inFlightMessages) { + if (kind == inFlight.kind) { + return true + } + } + return false + } + + private fun hasPendingOrInflightMagicLocked(magic: Int): Boolean { + for (queued in pendingMessages) { + if (queued.magic == magic) { + return true + } + } + for (inFlight in inFlightMessages) { + if (inFlight.magic == magic) { + return true + } + } + return false + } + + private fun hasPendingMagicLocked(sid: Int, magic: Int): Boolean { + for (queued in pendingMessages) { + if (queued.sid == sid && queued.magic == magic) { + return true + } + } + return false + } + + private fun handleAckTimeoutLocked(message: OutboundMessage) { + consecutiveAckTimeouts += 1 + + val onTimeout = message.onTimeout + if (onTimeout != null) { + onTimeout.run() + } + + if (consecutiveAckTimeouts > ConnectionOptions.MAX_CONSECUTIVE_ACK_TIMEOUTS) { + handleTransportFailure("too many ack timeouts") + } + } + + /** + * Replace any stale lease control with one right-arm and one left-arm + * fire-and-forget write. With priority=true the right arm is sent first so + * CLAIM reaches the lens that originated the deferred wake immediately. + */ + private fun enqueueFaceclawWakeControlLocked(operation: Int, nonce: Int, priority: Boolean): Int { + clearMessagesOfKindLocked("wake-lease-control") + val generation = ++faceclawWakeControlGeneration + faceclawWakeControlSentCount = 0 + if (operation == BleProtocol.FACECLAW_WAKE_OP_ACQUIRE) { + lastFaceclawWakeLeaseQueuedAtMs = SystemClock.elapsedRealtime() + } + val onSent = MessageCallback { + if (faceclawWakeControlGeneration == generation) { + faceclawWakeControlSentCount += 1 + } + } + val right = messageBuilder.faceclawWakeControl(operation, nonce, false) + val left = messageBuilder.faceclawWakeControl(operation, nonce, true) + right.onSent = onSent + left.onSent = onSent + if (priority) { + pendingMessages.addFirst(left) + pendingMessages.addFirst(right) + } else { + pendingMessages.addLast(right) + pendingMessages.addLast(left) + } + logLine("queue " + right.label + " + L") + return generation + } + + private fun waitForFaceclawWakeControlDelivery(generation: Int, timeoutMs: Long): Boolean { + val deadline = SystemClock.elapsedRealtime() + Math.max(0L, timeoutMs) + synchronized(lock) { + while (running + && sessionReady + && faceclawWakeControlGeneration == generation + && faceclawWakeControlSentCount < 2) { + val remaining = deadline - SystemClock.elapsedRealtime() + if (remaining <= 0) { + break + } + try { + lock.wait(Math.min(remaining, 100L)) + } catch (e: InterruptedException) { + Thread.currentThread().interrupt() + break + } + } + return faceclawWakeControlGeneration == generation + && faceclawWakeControlSentCount >= 2 + } + } + + /** + * Acquire/renew or release CFW's independent direct-framebuffer repaint + * guard on both arms. It is separate from the optional idle-wake lease: + * every Faceclaw display session needs this guard while its EvenHub layout + * contains swipe-capturing stock widgets. + */ + private fun enqueueFaceclawFramebufferControlLocked(operation: Int, priority: Boolean): Int { + clearMessagesOfKindLocked("framebuffer-lease-control") + val generation = ++faceclawFramebufferControlGeneration + faceclawFramebufferControlSentCount = 0 + if (operation == BleProtocol.FACECLAW_FB_OP_ACQUIRE) { + lastFaceclawFramebufferLeaseQueuedAtMs = SystemClock.elapsedRealtime() + } + val onSent = MessageCallback { + if (faceclawFramebufferControlGeneration == generation) { + faceclawFramebufferControlSentCount += 1 + } + } + val right = messageBuilder.faceclawFramebufferControl(operation, false) + val left = messageBuilder.faceclawFramebufferControl(operation, true) + right.onSent = onSent + left.onSent = onSent + if (priority) { + pendingMessages.addFirst(left) + pendingMessages.addFirst(right) + } else { + pendingMessages.addLast(right) + pendingMessages.addLast(left) + } + logLine("queue " + right.label + " + L") + return generation + } + + private fun waitForFaceclawFramebufferControlDelivery(generation: Int, timeoutMs: Long): Boolean { + val deadline = SystemClock.elapsedRealtime() + Math.max(0L, timeoutMs) + synchronized(lock) { + while (running + && sessionReady + && faceclawFramebufferControlGeneration == generation + && faceclawFramebufferControlSentCount < 2) { + val remaining = deadline - SystemClock.elapsedRealtime() + if (remaining <= 0) { + break + } + try { + lock.wait(Math.min(remaining, 100L)) + } catch (e: InterruptedException) { + Thread.currentThread().interrupt() + break + } + } + return faceclawFramebufferControlGeneration == generation + && faceclawFramebufferControlSentCount >= 2 + } + } + + private fun releaseFaceclawFramebufferLease(): Boolean { + val generation: Int + synchronized(lock) { + if (!running || !sessionReady) { + return true + } + generation = enqueueFaceclawFramebufferControlLocked( + BleProtocol.FACECLAW_FB_OP_RELEASE, + true + ) + } + interruptibleSleep.interrupt() + return waitForFaceclawFramebufferControlDelivery( + generation, + FACECLAW_WAKE_CONTROL_WAIT_MS + ) + } + + private fun clearMessagesOfKindLocked(kind: String) { + val pendingIterator = pendingMessages.iterator() + while (pendingIterator.hasNext()) { + val message = pendingIterator.next() + if (kind == message.kind) { + pendingIterator.remove() + discardPrewriteIfMatchesLocked(message) + if ("image" == kind) { + discardImageUpdateStatsLocked(message.imageUpdateId, "pending messages cleared ($kind)") + } + magicPool.release(message.sid, message.magic, message.label, "cleared pending $kind") + } + } + val inFlightIterator = inFlightMessages.iterator() + while (inFlightIterator.hasNext()) { + val message = inFlightIterator.next() + if (kind == message.kind) { + inFlightIterator.remove() + if ("image" == kind) { + discardImageUpdateStatsLocked(message.imageUpdateId, "inflight messages cleared ($kind)") + } + magicPool.release(message.sid, message.magic, message.label, "cleared inflight $kind") + } + } + if ("image" == kind) { + // The image pipeline was flushed (e.g. ack timeout -> keyframe resync): + // drop the pipelined delta base so the next image is a full keyframe. + lastEnqueuedPacked = ByteArray(0) + lastEnqueuedFingerprint = "" + } + } + + private fun clearAllMessagesLocked(reason: String) { + clearPendingMessagesLocked(reason) + clearInFlightMessagesLocked(reason) + // The image pipeline is gone: drop the pipelined delta base so the next + // image is a full keyframe rather than a delta onto a stale base. + lastEnqueuedPacked = ByteArray(0) + lastEnqueuedFingerprint = "" + // Queued resource commands (if any) were dropped with the rest, and the + // session churn behind a full clear may have freed the on-glasses + // cache; forget it phone-side so glyphs re-upload lazily. + resourceCache.reset() + } + + /** + * A custom wake queues CLAIM before NativeScript asks for a resume. Keep + * that private control while flushing stale EvenHub traffic around the + * direct prelude write. + */ + private fun clearAllMessagesPreservingWakeLeaseLocked(reason: String) { + val pendingIterator = pendingMessages.iterator() + while (pendingIterator.hasNext()) { + val message = pendingIterator.next() + if ("wake-lease-control" == message.kind) { + continue + } + pendingIterator.remove() + discardPrewriteIfMatchesLocked(message) + discardImageUpdateStatsLocked( + message.imageUpdateId, + "pending messages cleared: $reason" + ) + magicPool.release( + message.sid, + message.magic, + message.label, + "cleared pending: $reason" + ) + } + clearInFlightMessagesLocked(reason) + lastEnqueuedPacked = ByteArray(0) + lastEnqueuedFingerprint = "" + resourceCache.reset() + } + + /** Any image update whose fragments are still queued (not yet sent). */ + private fun hasPendingImageLocked(): Boolean { + for (message in pendingMessages) { + if ("image" == message.kind) { + return true + } + } + return false + } + + private fun clearPendingMessagesLocked(reason: String) { + while (!pendingMessages.isEmpty()) { + val message = pendingMessages.removeFirst() + discardPrewriteIfMatchesLocked(message) + discardImageUpdateStatsLocked(message.imageUpdateId, "pending messages cleared: $reason") + magicPool.release(message.sid, message.magic, message.label, "cleared pending: $reason") + } + } + + private fun clearInFlightMessagesLocked(reason: String) { + while (!inFlightMessages.isEmpty()) { + val message = inFlightMessages.removeFirst() + discardImageUpdateStatsLocked(message.imageUpdateId, "inflight messages cleared: $reason") + magicPool.release(message.sid, message.magic, message.label, "cleared inflight: $reason") + } + } + + private fun discardPrewriteIfMatchesLocked(message: OutboundMessage?) { + if (message != null && message === prewrittenMessage) { + prewrittenMessage = null + prewrittenFrames = Collections.emptyList() + } + } + + private fun recordUnexpectedAckLocked(sid: Int, magic: Int) { + if (magic < BleMagicPool.MIN_MAGIC || magic > BleMagicPool.MAX_MAGIC) { + return + } + val previous = magicPool.getReleaseRecord(sid, magic) + if (previous == null) { + val pendingNote = if (hasPendingMagicLocked(sid, magic)) " while that magic is only pending locally" else "" + logLine("unexpected ACK sid=" + sid + " magic=" + magic + pendingNote + + "; possible Even app BLE contention") + return + } + if ("timeout" == previous.reason) { + logLine("late ACK after timeout sid=" + sid + " magic=" + magic + + " label=" + previous.label + + "; ACK timeout may be too short") + return + } + if ("ack" == previous.reason) { + logLine("duplicate ACK for already-acked message sid=" + sid + " magic=" + magic + + " label=" + previous.label + + "; possible Even app BLE contention") + return + } + logLine("late ACK for released message sid=" + sid + " magic=" + magic + + " label=" + previous.label + + " release=" + previous.reason) + } + + /** The address of a configured arm Android no longer holds a bond for, or null. */ + private fun firstUnpairedArm(): String? { + if (!bleManager.isBonded(rightAddress)) return rightAddress + if (!bleManager.isBonded(leftAddress)) return leftAddress + return null + } + + /** + * A connect failure while an arm's bond is missing (the pairing was + * forgotten in Android settings, or the address was typed in by hand and + * never paired) will repeat forever, so instead of scheduling a retry, + * park the worker loop and tell the user to re-pair. Only an explicit + * connect (which builds a fresh communicator) starts a new attempt. + */ + private fun handleUnpairedFailure(address: String) { + Log.e(TAG, "Connect failed and $address is not paired; suspending reconnect") + synchronized(lock) { + reconnectHalted = true + sessionReady = false + fixedLayoutCreated = false + startupProbePending = false + shutdownRequested = false + chargingMode = false + imageRetryAfterMs = 0 + displayedFingerprint = "" + faceclawWakePendingNonce = -1 + lastFaceclawWakeLeaseQueuedAtMs = 0 + faceclawWakeControlSentCount = 0 + clearAllMessagesLocked("arm not paired: $address") + reconnectAfterMs = Long.MAX_VALUE + bleManager.disconnect(rightAddress) + bleManager.disconnect(leftAddress) + } + if (!userDisconnectRequested) { + setStateDisplay( + "unpaired", + "The glasses (" + address + ") are not paired with this phone." + + " Use \"Pair glasses\" to pair them again, then connect." + ) + } + interruptibleSleep.interrupt() + } + + private fun handleTransportFailure(reason: String?) { + Log.e(TAG, "Transport failure: $reason") + synchronized(lock) { + maybeEmitEvenAppConflictLocked(reason) + finishBenchmarkLocked(true, "transport failure") + sessionReady = false + fixedLayoutCreated = false + startupProbePending = false + shutdownRequested = false + chargingMode = false + imageRetryAfterMs = 0 + displayedFingerprint = "" + faceclawWakePendingNonce = -1 + lastFaceclawWakeLeaseQueuedAtMs = 0 + faceclawWakeControlSentCount = 0 + clearAllMessagesLocked("transport failure: $reason") + reconnectAfterMs = SystemClock.elapsedRealtime() + ConnectionOptions.RECONNECT_DELAY_MS + bleManager.disconnect(rightAddress) + bleManager.disconnect(leftAddress) + } + if (!userDisconnectRequested) { + setStateDisplay("retrying", if (reason == null || reason.isEmpty()) "Reconnecting..." else "Reconnecting after $reason") + } + interruptibleSleep.interrupt() + } + + private fun resetSessionStateLocked() { + ringBattery = -1 + ringCharging = -1 + sessionReady = false + shutdownRequested = false + fixedLayoutCreated = false + chargingMode = false + rightConnected = false + leftConnected = false + ringConnected = false + ringNotificationsReady = false + reconnectAfterMs = 0 + reconnectHalted = false + ringReconnectAfterMs = 0 + lastAckAtMs = 0 + lastIncomingAtMs = 0 + lastHeartbeatSentAtMs = 0 + lastSessionReadyAtMs = 0 + consecutiveAckTimeouts = 0 + lastAudioControlAckMagic = 0 + audioCaptureActive = false + audioPacketListener = null + compassControlLastSent = -1 + // A dead transport orphans any glasses-side compass state; the fresh + // session re-asserts the desired state once its layout is ready. + compassMaybeOn = false + faceclawWakePendingNonce = -1 + lastFaceclawWakeLeaseQueuedAtMs = 0 + faceclawWakeControlSentCount = 0 + cfwCleanupDelivered = false + for (transport in cfwTransports) transport.reset() + lastCfwCleanupAckMagic = 0 + wearState = -1 + displayedFingerprint = "" + // Deliberately not clearing silentMode: it is a property of the glasses, + // not of our session, and silent mode blocks app launches, so it can be + // the very cause of the session teardown that got us here. + } + + private fun emitRingEvent(event: G2Event, frameId: Int) { + val current = listener + if (current == null) { + FrameTimings.getInstance().finishFrame(frameId, "discarded: no listener attached") + return + } + val containerNameSnapshot = event.containerName + mainHandler.post { + FrameTimings.getInstance().log(frameId, "dispatching input event on main thread") + try { + current.onRingEvent(event.kind, containerNameSnapshot, event.eventType, event.eventSource, event.systemExitReasonCode, frameId, + event.ringTick, event.ringType, event.ringAux, event.ringSpeed) + } catch (t: Throwable) { + Log.w(TAG, "listener onRingEvent failed", t) + FrameTimings.getInstance().finishFrame(frameId, "discarded: listener onRingEvent failed") + } + } + } + + /** Finish a frame owned by the communicator and tell the TS side, which may be awaiting it. */ + private fun finishFrame(frameId: Int, outcome: String) { + if (frameId <= 0) { + return + } + FrameTimings.getInstance().finishFrame(frameId, outcome) + val current = listener ?: return + mainHandler.post { + try { + current.onFrameFinished(frameId, outcome) + } catch (t: Throwable) { + Log.w(TAG, "listener onFrameFinished failed", t) + } + } + } + + private fun emitImuData(x: Double, y: Double, z: Double, eventSource: Int) { + if (imuListeners.isEmpty()) { + return + } + mainHandler.post { + for (imuListener in imuListeners) { + try { + imuListener.onImuData(x, y, z, eventSource) + } catch (t: Throwable) { + Log.w(TAG, "listener onImuData failed", t) + } + } + } + } + + private fun emitCompassEvent(event: BleProtocol.CompassEvent) { + if (event.diagnosticFlags >= 0) { + val sources = arrayOf("unknown", "GRV", "GMRV", "RV") + Log.i("FaceclawCompass", "heading=" + event.headingDegrees + + " magneticAccuracy=" + event.magneticAccuracy + + " magneticAnomalies=" + event.magneticAnomalies + + " orientationSource=" + sources[event.orientationSource] + + " flags=0x" + Integer.toHexString(event.diagnosticFlags) + + " sampleTimeMs=" + event.sampleTimeMs) + } + for (subscription in compassSubscriptions) { + subscription.handler.post { + try { + subscription.listener.onCompassEvent(event.command, event.headingDegrees, + event.magneticAccuracy, event.magneticAnomalies, event.orientationSource, + event.diagnosticFlags, event.sampleTimeMs) + } catch (t: Throwable) { + Log.w(TAG, "listener onCompassEvent failed", t) + } + } + } + } + + private fun emitSilentMode(silent: Boolean) { + val current = listener ?: return + mainHandler.post { + try { + current.onSilentMode(silent) + } catch (t: Throwable) { + Log.w(TAG, "listener onSilentMode failed", t) + } + } + } + + private fun emitWearState(wearing: Boolean) { + val current = listener ?: return + mainHandler.post { + try { + current.onWearState(wearing) + } catch (t: Throwable) { + Log.w(TAG, "listener onWearState failed", t) + } + } + } + + private fun emitPhoneLockStateIfChanged(force: Boolean) { + val locked: Boolean + synchronized(lock) { + val now = SystemClock.elapsedRealtime() + if (!force && now - lastPhoneLockCheckAtMs < 1_000) return + lastPhoneLockCheckAtMs = now + locked = keyguardManager != null && keyguardManager.isDeviceLocked + val value = if (locked) 1 else 0 + if (!force && value == phoneLockState) return + phoneLockState = value + } + val current = listener ?: return + mainHandler.post { + try { + current.onPhoneLockState(locked) + } catch (t: Throwable) { + Log.w(TAG, "listener onPhoneLockState failed", t) + } + } + } + + private fun emitBatteryState(headsetBattery: Int, headsetCharging: Int) { + val reportedRingBattery = ringBattery + val reportedRingCharging = ringCharging + val current = listener ?: return + mainHandler.post { + try { + current.onBatteryState(headsetBattery, headsetCharging, reportedRingBattery, reportedRingCharging) + } catch (t: Throwable) { + Log.w(TAG, "listener onBatteryState failed", t) + } + } + } + + private fun emitFirmwareInfo(info: BleProtocol.FirmwareInfo) { + val current = listener ?: return + mainHandler.post { + try { + current.onFirmwareInfo(info.leftVersion, info.rightVersion, info.extension) + } catch (t: Throwable) { + Log.w(TAG, "listener onFirmwareInfo failed", t) + } + } + } + + private fun emitFrameMetrics(paintMs: Int, transmitMs: Int, tileCount: Int) { + val current = listener ?: return + mainHandler.post { + try { + current.onFrameMetrics(paintMs, transmitMs, tileCount) + } catch (t: Throwable) { + Log.w(TAG, "listener onFrameMetrics failed", t) + } + } + } + + private fun maybeEmitEvenAppConflictLocked(reason: String?) { + if ("write failed" != reason) { + return + } + val now = SystemClock.elapsedRealtime() + if (lastSessionReadyAtMs <= 0 || now - lastSessionReadyAtMs > ConnectionOptions.EVEN_APP_WRITE_FAILURE_WINDOW_MS) { + return + } + if (lastEvenAppConflictAtMs > 0 && now - lastEvenAppConflictAtMs < 60_000) { + return + } + if (!FaceclawEvenAppDetector.isEvenNotificationActive(appContext)) { + return + } + lastEvenAppConflictAtMs = now + emitEvenAppConflict("The Even Realities app still appears to be running. It can hold the glasses BLE link and cause Faceclaw write failures. Open its app settings and force stop it, then reconnect Faceclaw.") + } + + private fun emitEvenAppConflict(message: String?) { + val current = listener ?: return + val messageSnapshot = message ?: "" + mainHandler.post { + try { + current.onEvenAppConflict(messageSnapshot) + } catch (t: Throwable) { + Log.w(TAG, "listener onEvenAppConflict failed", t) + } + } + } + + private fun setStateDisplay(nextPhase: String, nextStatus: String) { + synchronized(lock) { + phase = nextPhase + status = nextStatus + } + emitState() + } + + private fun emitState() { + val current = listener ?: return + val phaseSnapshot: String + val statusSnapshot: String + synchronized(lock) { + phaseSnapshot = phase + statusSnapshot = status + } + mainHandler.post { + try { + current.onStateChange(phaseSnapshot, statusSnapshot) + } catch (t: Throwable) { + Log.w(TAG, "listener onStateChange failed", t) + } + } + } + + private fun updateG2ScreenWakeLock(screenOn: Boolean) { + if (screenOn) { + var wakeLock = g2ScreenWakeLock + if (wakeLock == null) { + wakeLock = powerManager.newWakeLock(PowerManager.PARTIAL_WAKE_LOCK, G2_SCREEN_WAKE_LOCK_TAG) + wakeLock.setReferenceCounted(false) + g2ScreenWakeLock = wakeLock + } + if (!wakeLock.isHeld) { + wakeLock.acquire() + logLine("G2 screen wake lock acquired") + } + return + } + releaseG2ScreenWakeLock() + } + + private fun releaseG2ScreenWakeLock() { + val wakeLock = g2ScreenWakeLock + if (wakeLock != null && wakeLock.isHeld) { + wakeLock.release() + logLine("G2 screen wake lock released") + } + } + + private fun logLine(line: String) { + Log.i(TAG, line) + } + + private fun nextMapSessionId(): Int { + val id = nextMapSessionId + val increment = if (connectionOptions.skipSessionIds) 2 else 1 + nextMapSessionId = (nextMapSessionId + increment) and 0xff + return id + } + + private fun hasRingAddress(): Boolean { + return ringAddress.trim().isNotEmpty() + } + + private fun isConfiguredRingAddress(address: String?): Boolean { + return hasRingAddress() && address != null && address.equals(ringAddress, ignoreCase = true) + } + + private fun isDirectRingNotification(address: String?, characteristicUuid: String?): Boolean { + if (!isConfiguredRingAddress(address) || characteristicUuid == null) { + return false + } + return BleProtocol.R1_PHONE_NOTIFY_CHAR_UUID == characteristicUuid + || BleProtocol.R1_NOTIFY_CHAR_UUID == characteristicUuid + } + + private fun getDesiredFingerprint(): String { + synchronized(desiredTilesLock) { + return desiredFingerprint + } + } +} diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawBleManager.java b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawBleManager.java deleted file mode 100644 index 7531a50e..00000000 --- a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawBleManager.java +++ /dev/null @@ -1,558 +0,0 @@ -package com.faceclaw.app; - -import android.annotation.SuppressLint; -import android.bluetooth.BluetoothAdapter; -import android.bluetooth.BluetoothDevice; -import android.bluetooth.BluetoothGatt; -import android.bluetooth.BluetoothGattCallback; -import android.bluetooth.BluetoothGattCharacteristic; -import android.bluetooth.BluetoothGattDescriptor; -import android.bluetooth.BluetoothGattService; -import android.bluetooth.BluetoothManager; -import android.bluetooth.BluetoothProfile; -import android.content.Context; -import android.os.Handler; -import android.os.Looper; -import android.os.SystemClock; -import android.util.Log; - -import java.util.List; -import java.util.UUID; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicLong; - -@SuppressLint("MissingPermission") -public class FaceclawBleManager { - private static final String TAG = "FaceclawBle"; - private static final int[] WRITE_RETRY_DELAYS_MS = new int[] {1, 1, 1, 2, 4, 8, 12, 20, 35, 100, 200}; - - private final Context context; - private final BluetoothAdapter bluetoothAdapter; - private final Handler mainHandler = new Handler(Looper.getMainLooper()); - - private final ConcurrentHashMap gattClients = new ConcurrentHashMap<>(); - private final Object bluetoothApiLock = new Object(); - - private final ConcurrentHashMap connectLatches = new ConcurrentHashMap<>(); - private final ConcurrentHashMap connectResults = new ConcurrentHashMap<>(); - - private final ConcurrentHashMap servicesLatches = new ConcurrentHashMap<>(); - private final ConcurrentHashMap servicesStatuses = new ConcurrentHashMap<>(); - - private final ConcurrentHashMap mtuLatches = new ConcurrentHashMap<>(); - private final ConcurrentHashMap negotiatedMtus = new ConcurrentHashMap<>(); - private final ConcurrentHashMap mtuStatuses = new ConcurrentHashMap<>(); - - private final ConcurrentHashMap descriptorLatches = new ConcurrentHashMap<>(); - private final ConcurrentHashMap descriptorStatuses = new ConcurrentHashMap<>(); - - private final ConcurrentHashMap writeLatches = new ConcurrentHashMap<>(); - private final ConcurrentHashMap writeStatuses = new ConcurrentHashMap<>(); - - private volatile FaceclawBleListener listener; - - // Process-wide outbound-traffic totals, sampled by the phone UI's BLE - // bandwidth indicator. Static so counts from every manager instance and - // isolate land in the same totals. - private static final AtomicLong outboundMessages = new AtomicLong(); - private static final AtomicLong outboundBytes = new AtomicLong(); - private static final AtomicLong displayFramesSent = new AtomicLong(); - - /** Called by the communicator when a display frame's last message is acked. */ - public static void recordDisplayFrameSent() { - displayFramesSent.incrementAndGet(); - } - - /** Running totals of outbound BLE traffic since process start: [messages, bytes, display frames]. */ - public static long[] sampleOutboundTraffic() { - return new long[] { outboundMessages.get(), outboundBytes.get(), displayFramesSent.get() }; - } - - public FaceclawBleManager(Context context) { - this.context = context.getApplicationContext(); - BluetoothManager bluetoothManager = (BluetoothManager) this.context.getSystemService(Context.BLUETOOTH_SERVICE); - if (bluetoothManager == null || bluetoothManager.getAdapter() == null) { - throw new IllegalStateException("Bluetooth adapter unavailable"); - } - this.bluetoothAdapter = bluetoothManager.getAdapter(); - } - - public void setListener(FaceclawBleListener listener) { - this.listener = listener; - } - - /** - * Whether Android still holds a bond (pairing) for this address. Returns - * true when the state cannot be determined, so callers only act on a - * definite BOND_NONE. - */ - public boolean isBonded(String address) { - if (address == null || address.trim().isEmpty()) { - return true; - } - try { - BluetoothDevice device = bluetoothAdapter.getRemoteDevice(address); - return device == null || device.getBondState() != BluetoothDevice.BOND_NONE; - } catch (Throwable t) { - return true; - } - } - - /** - * Android's bond state for this address (BluetoothDevice.BOND_NONE / - * BOND_BONDING / BOND_BONDED), or -1 when it cannot be determined. - * BOND_BONDING covers the whole OS pairing flow, including a pairing - * dialog that is still waiting for the user. - */ - public int getBondState(String address) { - if (address == null || address.trim().isEmpty()) { - return -1; - } - try { - BluetoothDevice device = bluetoothAdapter.getRemoteDevice(address); - return device == null ? -1 : device.getBondState(); - } catch (Throwable t) { - return -1; - } - } - - /** Whether a GATT client for this address is open, i.e. no disconnect callback has arrived for it. */ - public boolean isConnected(String address) { - return address != null && gattClients.containsKey(address); - } - - public boolean connect(String address, int timeoutMs) { - if (address == null || address.trim().isEmpty()) { - throw new IllegalArgumentException("address is required"); - } - CountDownLatch latch = new CountDownLatch(1); - connectLatches.put(address, latch); - connectResults.remove(address); - - Object gattLock = gattLock(address); - BluetoothGatt gatt; - synchronized (gattLock) { - BluetoothGatt existing = gattClients.get(address); - if (existing != null) { - connectLatches.remove(address); - return true; - } - - final BluetoothDevice device = bluetoothAdapter.getRemoteDevice(address); - if (device == null) { - connectLatches.remove(address); - throw new IllegalArgumentException("remote device not found: " + address); - } - - gatt = device.connectGatt( - context, - false, - gattCallback, - BluetoothDevice.TRANSPORT_LE, - BluetoothDevice.PHY_LE_2M|BluetoothDevice.PHY_LE_1M - ); - if (gatt != null) { - gattClients.put(address, gatt); - } - - if (gatt == null) { - connectLatches.remove(address); - return false; - } - - if (!awaitLatch(latch, timeoutMs)) { - connectLatches.remove(address); - connectResults.remove(address); - gattClients.remove(address, gatt); - negotiatedMtus.remove(address); - gatt.disconnect(); - gatt.close(); - return false; - } - - Boolean connected = connectResults.remove(address); - connectLatches.remove(address); - return Boolean.TRUE.equals(connected); - } - } - - public boolean requestConnectionPriority(String address, int priority) { - synchronized (gattLock(address)) { - BluetoothGatt gatt = requireGatt(address); - return gatt.requestConnectionPriority(priority); - } - } - - /** One-shot benchmark preferences; success is established by callbacks/HCI, not submission. */ - public void prepareBenchmarkLink(String address, int mode) { - synchronized (gattLock(address)) { - BluetoothGatt gatt = requireGatt(address); - if ((mode & 1) != 0) { - boolean accepted = gatt.requestConnectionPriority(BluetoothGatt.CONNECTION_PRIORITY_HIGH); - Log.i(TAG, "benchmark HIGH address=" + address + " submitted=" + accepted); - } - if ((mode & 2) != 0) { - Log.i(TAG, "benchmark request 2M address=" + address); - gatt.setPreferredPhy(BluetoothDevice.PHY_LE_2M_MASK, - BluetoothDevice.PHY_LE_2M_MASK, BluetoothDevice.PHY_OPTION_NO_PREFERRED); - } - gatt.readPhy(); - } - } - - public int getNegotiatedMtu(String address) { - Integer mtu = negotiatedMtus.get(address); - return mtu == null ? 23 : mtu; - } - - public boolean requestMtu(String address, int mtu, int timeoutMs) { - synchronized (gattLock(address)) { - CountDownLatch latch = new CountDownLatch(1); - mtuLatches.put(address, latch); - mtuStatuses.remove(address); - - BluetoothGatt gatt = requireGatt(address); - boolean started = gatt.requestMtu(mtu); - if (!started) { - mtuLatches.remove(address); - return false; - } - if (!awaitLatch(latch, timeoutMs)) { - mtuLatches.remove(address); - mtuStatuses.remove(address); - return false; - } - Integer status = mtuStatuses.remove(address); - mtuLatches.remove(address); - return status != null && status == BluetoothGatt.GATT_SUCCESS; - } - } - - public boolean discoverServices(String address, int timeoutMs) { - synchronized (gattLock(address)) { - CountDownLatch latch = new CountDownLatch(1); - servicesLatches.put(address, latch); - servicesStatuses.remove(address); - - BluetoothGatt gatt = requireGatt(address); - boolean started = gatt.discoverServices(); - if (!started) { - servicesLatches.remove(address); - return false; - } - if (!awaitLatch(latch, timeoutMs)) { - servicesLatches.remove(address); - servicesStatuses.remove(address); - return false; - } - Integer status = servicesStatuses.remove(address); - servicesLatches.remove(address); - return status != null && status == BluetoothGatt.GATT_SUCCESS; - } - } - - public boolean enableNotifications(String address, String characteristicUuid, boolean enable, int timeoutMs) { - synchronized (gattLock(address)) { - CountDownLatch latch = new CountDownLatch(1); - descriptorLatches.put(address, latch); - descriptorStatuses.remove(address); - - BluetoothGatt gatt = requireGatt(address); - BluetoothGattCharacteristic characteristic = requireCharacteristic(gatt, characteristicUuid); - - boolean notificationSet = gatt.setCharacteristicNotification(characteristic, enable); - if (!notificationSet) { - descriptorLatches.remove(address); - return false; - } - - BluetoothGattDescriptor descriptor = characteristic.getDescriptor(java.util.UUID.fromString(BleProtocol.CCCD_UUID)); - if (descriptor == null) { - descriptorLatches.remove(address); - return true; - } - - descriptor.setValue(enable - ? BluetoothGattDescriptor.ENABLE_NOTIFICATION_VALUE - : BluetoothGattDescriptor.DISABLE_NOTIFICATION_VALUE); - boolean started = gatt.writeDescriptor(descriptor); - if (!started) { - descriptorLatches.remove(address); - descriptorStatuses.remove(address); - return false; - } - if (!awaitLatch(latch, timeoutMs)) { - descriptorLatches.remove(address); - descriptorStatuses.remove(address); - return false; - } - Integer status = descriptorStatuses.remove(address); - descriptorLatches.remove(address); - return status != null && status == BluetoothGatt.GATT_SUCCESS; - } - } - - public boolean writeFrames( - String address, - String characteristicUuid, - List frames, - int writeType, - int timeoutMs - ) { - if (frames == null || frames.isEmpty()) { - return true; - } - - long startMs = System.currentTimeMillis(); - synchronized (gattLock(address)) { - BluetoothGatt gatt = requireGatt(address); - BluetoothGattCharacteristic characteristic = requireCharacteristic(gatt, characteristicUuid); - - for (int i = 0; i < frames.size(); i++) { - byte[] frame = frames.get(i); - if (!startWrite(gatt, characteristic, address, frame, writeType, timeoutMs)) { - return false; - } - outboundBytes.addAndGet(frame != null ? frame.length : 0); - } - } - outboundMessages.incrementAndGet(); - int totalSize = frames.stream().mapToInt(frame -> frame != null ? frame.length : 0).sum(); - Log.i(TAG, "writeFrames wrote " + frames.size() + " frames totaling " + totalSize + " bytes in " + (System.currentTimeMillis() - startMs) + "ms"); - return true; - } - - private boolean startWrite( - BluetoothGatt gatt, - BluetoothGattCharacteristic characteristic, - String address, - byte[] data, - int writeType, - int timeoutMs - ) { - int retryCount = 0; - while (true) { - CountDownLatch latch = new CountDownLatch(1); - writeLatches.put(address, latch); - writeStatuses.remove(address); - - long currentTime = System.currentTimeMillis(); - int result = gatt.writeCharacteristic(characteristic, data, writeType); - long timeAsleep = System.currentTimeMillis() - currentTime; - //Log.i(TAG, "writeCharacteristic: spent " + timeAsleep + "ms"); - - if (result == android.bluetooth.BluetoothStatusCodes.ERROR_GATT_WRITE_REQUEST_BUSY) { - writeLatches.remove(address, latch); - writeStatuses.remove(address); - if (!sleepBeforeWriteRetry(address, "busy", retryCount++)) { - return false; - } - continue; - } else { - //Log.i(TAG, "writeCharacteristic with writeType=" + writeType + " result=" + result + " retryCount=" + retryCount); - if (result != android.bluetooth.BluetoothStatusCodes.SUCCESS) { - writeLatches.remove(address, latch); - writeStatuses.remove(address); - if (!sleepBeforeWriteRetry(address, "start result=" + result, retryCount++)) { - return false; - } - continue; - } - if (!awaitLatch(latch, timeoutMs)) { - writeLatches.remove(address, latch); - writeStatuses.remove(address); - return false; - } - Integer status = writeStatuses.remove(address); - writeLatches.remove(address); - if (status != null && status == BluetoothGatt.GATT_SUCCESS) { - return true; - } - if (!sleepBeforeWriteRetry(address, "callback status=" + status, retryCount++)) { - return false; - } - } - } - } - - private boolean sleepBeforeWriteRetry(String address, String reason, int retryIndex) { - if (retryIndex >= WRITE_RETRY_DELAYS_MS.length) { - Log.w(TAG, "writeCharacteristic retry exhausted: address=" + address + " reason=" + reason); - return false; - } - int delayMs = WRITE_RETRY_DELAYS_MS[retryIndex]; - Log.w(TAG, "writeCharacteristic retry: address=" + address + " reason=" + reason + " delayMs=" + delayMs); - try { - Thread.sleep(delayMs); - return true; - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - return false; - } - } - - public void disconnect(String address) { - Object gattLock = gattLock(address); - synchronized (gattLock) { - BluetoothGatt gatt = gattClients.remove(address); - negotiatedMtus.remove(address); - if (gatt == null) { - return; - } - gatt.disconnect(); - gatt.close(); - } - } - - public void close() { - for (String address : gattClients.keySet()) { - disconnect(address); - } - } - - private BluetoothGatt requireGatt(String address) { - BluetoothGatt gatt = gattClients.get(address); - if (gatt == null) { - throw new IllegalStateException("Not connected: " + address); - } - return gatt; - } - - private BluetoothGattCharacteristic requireCharacteristic(BluetoothGatt gatt, String characteristicUuid) { - UUID target = UUID.fromString(characteristicUuid); - for (BluetoothGattService service : gatt.getServices()) { - BluetoothGattCharacteristic characteristic = service.getCharacteristic(target); - if (characteristic != null) { - return characteristic; - } - } - throw new IllegalStateException("Characteristic not found: " + characteristicUuid); - } - - private boolean awaitLatch(CountDownLatch latch, int timeoutMs) { - try { - return latch.await(timeoutMs, TimeUnit.MILLISECONDS); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - return false; - } - } - - private Object gattLock(String address) { - // Android's Bluetooth stack has process-wide command-pipeline constraints on some devices. - // Keep every BluetoothGatt API call serialized globally, even for different MAC addresses. - return bluetoothApiLock; - } - - private final BluetoothGattCallback gattCallback = new BluetoothGattCallback() { - @Override - public void onConnectionStateChange(BluetoothGatt gatt, int status, int newState) { - Log.i(TAG, "onConnectionStateChange: status=" + status + " newState=" + newState); - String address = gatt.getDevice().getAddress(); - - if (newState == BluetoothProfile.STATE_CONNECTED && status == BluetoothGatt.GATT_SUCCESS) { - connectResults.put(address, true); - CountDownLatch latch = connectLatches.remove(address); - if (latch != null) { - latch.countDown(); - } - dispatchConnectionState(address, true); - return; - } - - if (newState == BluetoothProfile.STATE_DISCONNECTED) { - connectResults.put(address, false); - CountDownLatch latch = connectLatches.remove(address); - if (latch != null) { - latch.countDown(); - } - Object gattLock = gattLock(address); - synchronized (gattLock) { - gattClients.remove(address, gatt); - negotiatedMtus.remove(address); - gatt.close(); - } - dispatchConnectionState(address, false); - } - } - - @Override - public void onServicesDiscovered(BluetoothGatt gatt, int status) { - String address = gatt.getDevice().getAddress(); - servicesStatuses.put(address, status); - CountDownLatch latch = servicesLatches.remove(address); - if (latch != null) { - latch.countDown(); - } - } - - @Override - public void onMtuChanged(BluetoothGatt gatt, int mtu, int status) { - String address = gatt.getDevice().getAddress(); - if (status == BluetoothGatt.GATT_SUCCESS) negotiatedMtus.put(address, mtu); - mtuStatuses.put(address, status); - CountDownLatch latch = mtuLatches.remove(address); - if (latch != null) { - latch.countDown(); - } - } - - @Override - public void onPhyRead(BluetoothGatt gatt, int txPhy, int rxPhy, int status) { - Log.i(TAG, "onPhyRead: address=" + gatt.getDevice().getAddress() - + " txPhy=" + txPhy + " rxPhy=" + rxPhy + " status=" + status); - } - - @Override - public void onPhyUpdate(BluetoothGatt gatt, int txPhy, int rxPhy, int status) { - Log.i(TAG, "onPhyUpdate: address=" + gatt.getDevice().getAddress() - + " txPhy=" + txPhy + " rxPhy=" + rxPhy + " status=" + status); - } - - @Override - public void onDescriptorWrite(BluetoothGatt gatt, BluetoothGattDescriptor descriptor, int status) { - String address = gatt.getDevice().getAddress(); - descriptorStatuses.put(address, status); - CountDownLatch latch = descriptorLatches.remove(address); - if (latch != null) { - latch.countDown(); - } - } - - @Override - public void onCharacteristicWrite(BluetoothGatt gatt, BluetoothGattCharacteristic characteristic, int status) { - String address = gatt.getDevice().getAddress(); - writeStatuses.put(address, status); - CountDownLatch latch = writeLatches.remove(address); - if (latch != null) { - latch.countDown(); - } - } - - @Override - public void onCharacteristicChanged(BluetoothGatt gatt, BluetoothGattCharacteristic characteristic, byte[] value) { - dispatchNotification(gatt.getDevice().getAddress(), characteristic.getUuid().toString(), value); - } - - @Deprecated - @Override - public void onCharacteristicChanged(BluetoothGatt gatt, BluetoothGattCharacteristic characteristic) { - dispatchNotification(gatt.getDevice().getAddress(), characteristic.getUuid().toString(), characteristic.getValue()); - } - }; - - private void dispatchConnectionState(String address, boolean connected) { - FaceclawBleListener current = listener; - if (current == null) return; - current.onConnectionStateChange(address, connected); - } - - private void dispatchNotification(String address, String characteristicUuid, byte[] data) { - FaceclawBleListener current = listener; - if (current == null) return; - byte[] copy = data != null ? data.clone() : new byte[0]; - current.onNotification(address, characteristicUuid, copy); - } -} diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawBleManager.kt b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawBleManager.kt new file mode 100644 index 00000000..2335605f --- /dev/null +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawBleManager.kt @@ -0,0 +1,539 @@ +package com.faceclaw.app + +import android.annotation.SuppressLint +import android.bluetooth.BluetoothAdapter +import android.bluetooth.BluetoothDevice +import android.bluetooth.BluetoothGatt +import android.bluetooth.BluetoothGattCallback +import android.bluetooth.BluetoothGattCharacteristic +import android.bluetooth.BluetoothGattDescriptor +import android.bluetooth.BluetoothGattService +import android.bluetooth.BluetoothManager +import android.bluetooth.BluetoothProfile +import android.content.Context +import android.os.Handler +import android.os.Looper +import android.os.SystemClock +import android.util.Log + +import java.util.UUID +import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.CountDownLatch +import java.util.concurrent.TimeUnit +import java.util.concurrent.atomic.AtomicLong + +@SuppressLint("MissingPermission") +class FaceclawBleManager(context: Context) { + companion object { + private const val TAG = "FaceclawBle" + private val WRITE_RETRY_DELAYS_MS = intArrayOf(1, 1, 1, 2, 4, 8, 12, 20, 35, 100, 200) + + // Process-wide outbound-traffic totals, sampled by the phone UI's BLE + // bandwidth indicator. Static so counts from every manager instance and + // isolate land in the same totals. + private val outboundMessages = AtomicLong() + private val outboundBytes = AtomicLong() + private val displayFramesSent = AtomicLong() + + /** Called by the communicator when a display frame's last message is acked. */ + @JvmStatic + fun recordDisplayFrameSent() { + displayFramesSent.incrementAndGet() + } + + /** Running totals of outbound BLE traffic since process start: [messages, bytes, display frames]. */ + @JvmStatic + fun sampleOutboundTraffic(): LongArray { + return longArrayOf(outboundMessages.get(), outboundBytes.get(), displayFramesSent.get()) + } + } + + private val context: Context + private val bluetoothAdapter: BluetoothAdapter + private val mainHandler = Handler(Looper.getMainLooper()) + + private val gattClients = ConcurrentHashMap() + private val bluetoothApiLock = Any() + + private val connectLatches = ConcurrentHashMap() + private val connectResults = ConcurrentHashMap() + + private val servicesLatches = ConcurrentHashMap() + private val servicesStatuses = ConcurrentHashMap() + + private val mtuLatches = ConcurrentHashMap() + private val negotiatedMtus = ConcurrentHashMap() + private val mtuStatuses = ConcurrentHashMap() + + private val descriptorLatches = ConcurrentHashMap() + private val descriptorStatuses = ConcurrentHashMap() + + private val writeLatches = ConcurrentHashMap() + private val writeStatuses = ConcurrentHashMap() + + @Volatile + private var listener: FaceclawBleListener? = null + + init { + this.context = context.applicationContext + val bluetoothManager = this.context.getSystemService(Context.BLUETOOTH_SERVICE) as BluetoothManager? + if (bluetoothManager == null || bluetoothManager.adapter == null) { + throw IllegalStateException("Bluetooth adapter unavailable") + } + this.bluetoothAdapter = bluetoothManager.adapter + } + + fun setListener(listener: FaceclawBleListener?) { + this.listener = listener + } + + /** + * Whether Android still holds a bond (pairing) for this address. Returns + * true when the state cannot be determined, so callers only act on a + * definite BOND_NONE. + */ + fun isBonded(address: String?): Boolean { + if (address == null || address.trim().isEmpty()) { + return true + } + try { + val device = bluetoothAdapter.getRemoteDevice(address) + return device == null || device.bondState != BluetoothDevice.BOND_NONE + } catch (t: Throwable) { + return true + } + } + + /** + * Android's bond state for this address (BluetoothDevice.BOND_NONE / + * BOND_BONDING / BOND_BONDED), or -1 when it cannot be determined. + * BOND_BONDING covers the whole OS pairing flow, including a pairing + * dialog that is still waiting for the user. + */ + fun getBondState(address: String?): Int { + if (address == null || address.trim().isEmpty()) { + return -1 + } + try { + val device = bluetoothAdapter.getRemoteDevice(address) + return if (device == null) -1 else device.bondState + } catch (t: Throwable) { + return -1 + } + } + + /** Whether a GATT client for this address is open, i.e. no disconnect callback has arrived for it. */ + fun isConnected(address: String?): Boolean { + return address != null && gattClients.containsKey(address) + } + + fun connect(address: String?, timeoutMs: Int): Boolean { + if (address == null || address.trim().isEmpty()) { + throw IllegalArgumentException("address is required") + } + val latch = CountDownLatch(1) + connectLatches[address] = latch + connectResults.remove(address) + + val gattLock = gattLock(address) + synchronized(gattLock) { + val existing = gattClients[address] + if (existing != null) { + connectLatches.remove(address) + return true + } + + val device = bluetoothAdapter.getRemoteDevice(address) + if (device == null) { + connectLatches.remove(address) + throw IllegalArgumentException("remote device not found: $address") + } + + val gatt: BluetoothGatt? = device.connectGatt( + context, + false, + gattCallback, + BluetoothDevice.TRANSPORT_LE, + BluetoothDevice.PHY_LE_2M or BluetoothDevice.PHY_LE_1M + ) + if (gatt != null) { + gattClients[address] = gatt + } + + if (gatt == null) { + connectLatches.remove(address) + return false + } + + if (!awaitLatch(latch, timeoutMs)) { + connectLatches.remove(address) + connectResults.remove(address) + gattClients.remove(address, gatt) + negotiatedMtus.remove(address) + gatt.disconnect() + gatt.close() + return false + } + + val connected = connectResults.remove(address) + connectLatches.remove(address) + return connected == true + } + } + + fun requestConnectionPriority(address: String, priority: Int): Boolean { + synchronized(gattLock(address)) { + val gatt = requireGatt(address) + return gatt.requestConnectionPriority(priority) + } + } + + /** One-shot benchmark preferences; success is established by callbacks/HCI, not submission. */ + fun prepareBenchmarkLink(address: String, mode: Int) { + synchronized(gattLock(address)) { + val gatt = requireGatt(address) + if ((mode and 1) != 0) { + val accepted = gatt.requestConnectionPriority(BluetoothGatt.CONNECTION_PRIORITY_HIGH) + Log.i(TAG, "benchmark HIGH address=$address submitted=$accepted") + } + if ((mode and 2) != 0) { + Log.i(TAG, "benchmark request 2M address=$address") + gatt.setPreferredPhy(BluetoothDevice.PHY_LE_2M_MASK, + BluetoothDevice.PHY_LE_2M_MASK, BluetoothDevice.PHY_OPTION_NO_PREFERRED) + } + gatt.readPhy() + } + } + + fun getNegotiatedMtu(address: String): Int { + val mtu = negotiatedMtus[address] + return mtu ?: 23 + } + + fun requestMtu(address: String, mtu: Int, timeoutMs: Int): Boolean { + synchronized(gattLock(address)) { + val latch = CountDownLatch(1) + mtuLatches[address] = latch + mtuStatuses.remove(address) + + val gatt = requireGatt(address) + val started = gatt.requestMtu(mtu) + if (!started) { + mtuLatches.remove(address) + return false + } + if (!awaitLatch(latch, timeoutMs)) { + mtuLatches.remove(address) + mtuStatuses.remove(address) + return false + } + val status = mtuStatuses.remove(address) + mtuLatches.remove(address) + return status != null && status == BluetoothGatt.GATT_SUCCESS + } + } + + fun discoverServices(address: String, timeoutMs: Int): Boolean { + synchronized(gattLock(address)) { + val latch = CountDownLatch(1) + servicesLatches[address] = latch + servicesStatuses.remove(address) + + val gatt = requireGatt(address) + val started = gatt.discoverServices() + if (!started) { + servicesLatches.remove(address) + return false + } + if (!awaitLatch(latch, timeoutMs)) { + servicesLatches.remove(address) + servicesStatuses.remove(address) + return false + } + val status = servicesStatuses.remove(address) + servicesLatches.remove(address) + return status != null && status == BluetoothGatt.GATT_SUCCESS + } + } + + fun enableNotifications(address: String, characteristicUuid: String, enable: Boolean, timeoutMs: Int): Boolean { + synchronized(gattLock(address)) { + val latch = CountDownLatch(1) + descriptorLatches[address] = latch + descriptorStatuses.remove(address) + + val gatt = requireGatt(address) + val characteristic = requireCharacteristic(gatt, characteristicUuid) + + val notificationSet = gatt.setCharacteristicNotification(characteristic, enable) + if (!notificationSet) { + descriptorLatches.remove(address) + return false + } + + val descriptor = characteristic.getDescriptor(java.util.UUID.fromString(BleProtocol.CCCD_UUID)) + if (descriptor == null) { + descriptorLatches.remove(address) + return true + } + + descriptor.value = if (enable) + BluetoothGattDescriptor.ENABLE_NOTIFICATION_VALUE + else + BluetoothGattDescriptor.DISABLE_NOTIFICATION_VALUE + val started = gatt.writeDescriptor(descriptor) + if (!started) { + descriptorLatches.remove(address) + descriptorStatuses.remove(address) + return false + } + if (!awaitLatch(latch, timeoutMs)) { + descriptorLatches.remove(address) + descriptorStatuses.remove(address) + return false + } + val status = descriptorStatuses.remove(address) + descriptorLatches.remove(address) + return status != null && status == BluetoothGatt.GATT_SUCCESS + } + } + + fun writeFrames( + address: String, + characteristicUuid: String, + frames: List?, + writeType: Int, + timeoutMs: Int + ): Boolean { + if (frames == null || frames.isEmpty()) { + return true + } + + val startMs = System.currentTimeMillis() + synchronized(gattLock(address)) { + val gatt = requireGatt(address) + val characteristic = requireCharacteristic(gatt, characteristicUuid) + + for (i in frames.indices) { + val frame = frames[i] + if (!startWrite(gatt, characteristic, address, frame, writeType, timeoutMs)) { + return false + } + outboundBytes.addAndGet((frame?.size ?: 0).toLong()) + } + } + outboundMessages.incrementAndGet() + val totalSize = frames.sumOf { frame -> frame?.size ?: 0 } + Log.i(TAG, "writeFrames wrote " + frames.size + " frames totaling " + totalSize + " bytes in " + (System.currentTimeMillis() - startMs) + "ms") + return true + } + + private fun startWrite( + gatt: BluetoothGatt, + characteristic: BluetoothGattCharacteristic, + address: String, + data: ByteArray?, + writeType: Int, + timeoutMs: Int + ): Boolean { + var retryCount = 0 + while (true) { + val latch = CountDownLatch(1) + writeLatches[address] = latch + writeStatuses.remove(address) + + val currentTime = System.currentTimeMillis() + val result = gatt.writeCharacteristic(characteristic, data!!, writeType) + val timeAsleep = System.currentTimeMillis() - currentTime + //Log.i(TAG, "writeCharacteristic: spent " + timeAsleep + "ms"); + + if (result == android.bluetooth.BluetoothStatusCodes.ERROR_GATT_WRITE_REQUEST_BUSY) { + writeLatches.remove(address, latch) + writeStatuses.remove(address) + if (!sleepBeforeWriteRetry(address, "busy", retryCount++)) { + return false + } + continue + } else { + //Log.i(TAG, "writeCharacteristic with writeType=" + writeType + " result=" + result + " retryCount=" + retryCount); + if (result != android.bluetooth.BluetoothStatusCodes.SUCCESS) { + writeLatches.remove(address, latch) + writeStatuses.remove(address) + if (!sleepBeforeWriteRetry(address, "start result=$result", retryCount++)) { + return false + } + continue + } + if (!awaitLatch(latch, timeoutMs)) { + writeLatches.remove(address, latch) + writeStatuses.remove(address) + return false + } + val status = writeStatuses.remove(address) + writeLatches.remove(address) + if (status != null && status == BluetoothGatt.GATT_SUCCESS) { + return true + } + if (!sleepBeforeWriteRetry(address, "callback status=$status", retryCount++)) { + return false + } + } + } + } + + private fun sleepBeforeWriteRetry(address: String, reason: String, retryIndex: Int): Boolean { + if (retryIndex >= WRITE_RETRY_DELAYS_MS.size) { + Log.w(TAG, "writeCharacteristic retry exhausted: address=$address reason=$reason") + return false + } + val delayMs = WRITE_RETRY_DELAYS_MS[retryIndex] + Log.w(TAG, "writeCharacteristic retry: address=$address reason=$reason delayMs=$delayMs") + try { + Thread.sleep(delayMs.toLong()) + return true + } catch (e: InterruptedException) { + Thread.currentThread().interrupt() + return false + } + } + + fun disconnect(address: String?) { + val gattLock = gattLock(address) + synchronized(gattLock) { + val gatt = if (address != null) gattClients.remove(address) else null + if (address != null) negotiatedMtus.remove(address) + if (gatt == null) { + return + } + gatt.disconnect() + gatt.close() + } + } + + fun close() { + for (address in gattClients.keys) { + disconnect(address) + } + } + + private fun requireGatt(address: String): BluetoothGatt { + val gatt = gattClients[address] + if (gatt == null) { + throw IllegalStateException("Not connected: $address") + } + return gatt + } + + private fun requireCharacteristic(gatt: BluetoothGatt, characteristicUuid: String): BluetoothGattCharacteristic { + val target = UUID.fromString(characteristicUuid) + for (service: BluetoothGattService in gatt.services) { + val characteristic = service.getCharacteristic(target) + if (characteristic != null) { + return characteristic + } + } + throw IllegalStateException("Characteristic not found: $characteristicUuid") + } + + private fun awaitLatch(latch: CountDownLatch, timeoutMs: Int): Boolean { + try { + return latch.await(timeoutMs.toLong(), TimeUnit.MILLISECONDS) + } catch (e: InterruptedException) { + Thread.currentThread().interrupt() + return false + } + } + + private fun gattLock(address: String?): Any { + // Android's Bluetooth stack has process-wide command-pipeline constraints on some devices. + // Keep every BluetoothGatt API call serialized globally, even for different MAC addresses. + return bluetoothApiLock + } + + private val gattCallback: BluetoothGattCallback = object : BluetoothGattCallback() { + override fun onConnectionStateChange(gatt: BluetoothGatt, status: Int, newState: Int) { + Log.i(TAG, "onConnectionStateChange: status=$status newState=$newState") + val address = gatt.device.address + + if (newState == BluetoothProfile.STATE_CONNECTED && status == BluetoothGatt.GATT_SUCCESS) { + connectResults[address] = true + val latch = connectLatches.remove(address) + latch?.countDown() + dispatchConnectionState(address, true) + return + } + + if (newState == BluetoothProfile.STATE_DISCONNECTED) { + connectResults[address] = false + val latch = connectLatches.remove(address) + latch?.countDown() + val gattLock = gattLock(address) + synchronized(gattLock) { + gattClients.remove(address, gatt) + negotiatedMtus.remove(address) + gatt.close() + } + dispatchConnectionState(address, false) + } + } + + override fun onServicesDiscovered(gatt: BluetoothGatt, status: Int) { + val address = gatt.device.address + servicesStatuses[address] = status + val latch = servicesLatches.remove(address) + latch?.countDown() + } + + override fun onMtuChanged(gatt: BluetoothGatt, mtu: Int, status: Int) { + val address = gatt.device.address + if (status == BluetoothGatt.GATT_SUCCESS) negotiatedMtus[address] = mtu + mtuStatuses[address] = status + val latch = mtuLatches.remove(address) + latch?.countDown() + } + + override fun onPhyRead(gatt: BluetoothGatt, txPhy: Int, rxPhy: Int, status: Int) { + Log.i(TAG, "onPhyRead: address=" + gatt.device.address + + " txPhy=" + txPhy + " rxPhy=" + rxPhy + " status=" + status) + } + + override fun onPhyUpdate(gatt: BluetoothGatt, txPhy: Int, rxPhy: Int, status: Int) { + Log.i(TAG, "onPhyUpdate: address=" + gatt.device.address + + " txPhy=" + txPhy + " rxPhy=" + rxPhy + " status=" + status) + } + + override fun onDescriptorWrite(gatt: BluetoothGatt, descriptor: BluetoothGattDescriptor, status: Int) { + val address = gatt.device.address + descriptorStatuses[address] = status + val latch = descriptorLatches.remove(address) + latch?.countDown() + } + + override fun onCharacteristicWrite(gatt: BluetoothGatt, characteristic: BluetoothGattCharacteristic, status: Int) { + val address = gatt.device.address + writeStatuses[address] = status + val latch = writeLatches.remove(address) + latch?.countDown() + } + + override fun onCharacteristicChanged(gatt: BluetoothGatt, characteristic: BluetoothGattCharacteristic, value: ByteArray) { + dispatchNotification(gatt.device.address, characteristic.uuid.toString(), value) + } + + @Deprecated("Deprecated in Java") + override fun onCharacteristicChanged(gatt: BluetoothGatt, characteristic: BluetoothGattCharacteristic) { + dispatchNotification(gatt.device.address, characteristic.uuid.toString(), characteristic.value) + } + } + + private fun dispatchConnectionState(address: String, connected: Boolean) { + val current = listener ?: return + current.onConnectionStateChange(address, connected) + } + + private fun dispatchNotification(address: String, characteristicUuid: String, data: ByteArray?) { + val current = listener ?: return + val copy = data?.clone() ?: ByteArray(0) + current.onNotification(address, characteristicUuid, copy) + } +} diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawCalendarProvider.java b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawCalendarProvider.java deleted file mode 100644 index fbdb25af..00000000 --- a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawCalendarProvider.java +++ /dev/null @@ -1,98 +0,0 @@ -package com.faceclaw.app; - -import android.content.ContentUris; -import android.content.Context; -import android.database.Cursor; -import android.net.Uri; -import android.provider.CalendarContract; -import android.util.Log; - -import org.json.JSONArray; -import org.json.JSONException; -import org.json.JSONObject; - -/** - * Reads upcoming events from the Android Calendar provider for the Calendar - * app. Queries the Instances table (rather than Events) so that recurring - * events are expanded into concrete occurrences within the requested window. - * Requires the READ_CALENDAR runtime permission; without it the content - * resolver throws SecurityException and this returns an empty array. - */ -public final class FaceclawCalendarProvider { - private static final String TAG = "FaceclawCalendar"; - - private static final String[] PROJECTION = { - CalendarContract.Instances.EVENT_ID, - CalendarContract.Instances.TITLE, - CalendarContract.Instances.BEGIN, - CalendarContract.Instances.END, - CalendarContract.Instances.ALL_DAY, - CalendarContract.Instances.EVENT_LOCATION, - CalendarContract.Instances.CALENDAR_DISPLAY_NAME, - }; - - private FaceclawCalendarProvider() { - } - - /** - * JSON array of upcoming events starting from now through now+windowMs, - * ordered by start time, capped at maxEvents. Each element carries id, - * title, startMs, endMs, allDay, location, and calendarName. - */ - public static String getUpcomingEventsJson(Context context, int maxEvents, long windowMs) { - if (context == null || maxEvents <= 0) { - return "[]"; - } - long now = System.currentTimeMillis(); - long end = now + Math.max(0L, windowMs); - int limit = Math.min(200, maxEvents); - - Uri.Builder builder = CalendarContract.Instances.CONTENT_URI.buildUpon(); - ContentUris.appendId(builder, now); - ContentUris.appendId(builder, end); - Uri uri = builder.build(); - - JSONArray out = new JSONArray(); - Cursor cursor = null; - try { - cursor = context.getContentResolver().query( - uri, - PROJECTION, - null, - null, - CalendarContract.Instances.BEGIN + " ASC"); - if (cursor != null) { - while (cursor.moveToNext() && out.length() < limit) { - try { - out.put(buildEventJson(cursor)); - } catch (JSONException e) { - Log.w(TAG, "failed to serialize calendar event", e); - } - } - } - } catch (SecurityException e) { - Log.w(TAG, "calendar access denied while reading events", e); - return "[]"; - } catch (Throwable t) { - Log.w(TAG, "failed to read calendar events", t); - return "[]"; - } finally { - if (cursor != null) { - cursor.close(); - } - } - return out.toString(); - } - - private static JSONObject buildEventJson(Cursor cursor) throws JSONException { - JSONObject event = new JSONObject(); - event.put("id", cursor.getLong(0)); - event.put("title", cursor.isNull(1) ? "" : cursor.getString(1)); - event.put("startMs", cursor.getLong(2)); - event.put("endMs", cursor.getLong(3)); - event.put("allDay", cursor.getInt(4) != 0); - event.put("location", cursor.isNull(5) ? "" : cursor.getString(5)); - event.put("calendarName", cursor.isNull(6) ? "" : cursor.getString(6)); - return event; - } -} diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawCalendarProvider.kt b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawCalendarProvider.kt new file mode 100644 index 00000000..baa36749 --- /dev/null +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawCalendarProvider.kt @@ -0,0 +1,96 @@ +package com.faceclaw.app + +import android.content.ContentUris +import android.content.Context +import android.database.Cursor +import android.provider.CalendarContract +import android.util.Log + +import org.json.JSONArray +import org.json.JSONException +import org.json.JSONObject + +/** + * Reads upcoming events from the Android Calendar provider for the Calendar + * app. Queries the Instances table (rather than Events) so that recurring + * events are expanded into concrete occurrences within the requested window. + * Requires the READ_CALENDAR runtime permission; without it the content + * resolver throws SecurityException and this returns an empty array. + */ +class FaceclawCalendarProvider private constructor() { + companion object { + private const val TAG = "FaceclawCalendar" + + private val PROJECTION = arrayOf( + CalendarContract.Instances.EVENT_ID, + CalendarContract.Instances.TITLE, + CalendarContract.Instances.BEGIN, + CalendarContract.Instances.END, + CalendarContract.Instances.ALL_DAY, + CalendarContract.Instances.EVENT_LOCATION, + CalendarContract.Instances.CALENDAR_DISPLAY_NAME, + ) + + /** + * JSON array of upcoming events starting from now through now+windowMs, + * ordered by start time, capped at maxEvents. Each element carries id, + * title, startMs, endMs, allDay, location, and calendarName. + */ + @JvmStatic + fun getUpcomingEventsJson(context: Context?, maxEvents: Int, windowMs: Long): String { + if (context == null || maxEvents <= 0) { + return "[]" + } + val now = System.currentTimeMillis() + val end = now + Math.max(0L, windowMs) + val limit = Math.min(200, maxEvents) + + val builder = CalendarContract.Instances.CONTENT_URI.buildUpon() + ContentUris.appendId(builder, now) + ContentUris.appendId(builder, end) + val uri = builder.build() + + val out = JSONArray() + var cursor: Cursor? = null + try { + cursor = context.contentResolver.query( + uri, + PROJECTION, + null, + null, + CalendarContract.Instances.BEGIN + " ASC") + if (cursor != null) { + while (cursor.moveToNext() && out.length() < limit) { + try { + out.put(buildEventJson(cursor)) + } catch (e: JSONException) { + Log.w(TAG, "failed to serialize calendar event", e) + } + } + } + } catch (e: SecurityException) { + Log.w(TAG, "calendar access denied while reading events", e) + return "[]" + } catch (t: Throwable) { + Log.w(TAG, "failed to read calendar events", t) + return "[]" + } finally { + cursor?.close() + } + return out.toString() + } + + @Throws(JSONException::class) + private fun buildEventJson(cursor: Cursor): JSONObject { + val event = JSONObject() + event.put("id", cursor.getLong(0)) + event.put("title", if (cursor.isNull(1)) "" else cursor.getString(1)) + event.put("startMs", cursor.getLong(2)) + event.put("endMs", cursor.getLong(3)) + event.put("allDay", cursor.getInt(4) != 0) + event.put("location", if (cursor.isNull(5)) "" else cursor.getString(5)) + event.put("calendarName", if (cursor.isNull(6)) "" else cursor.getString(6)) + return event + } + } +} diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawCaptionEngine.java b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawCaptionEngine.java deleted file mode 100644 index 488d2bca..00000000 --- a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawCaptionEngine.java +++ /dev/null @@ -1,446 +0,0 @@ -package com.faceclaw.app; - -import android.os.Handler; -import android.os.Looper; -import android.util.Log; - -import com.k2fsa.sherpa.onnx.FeatureConfig; -import com.k2fsa.sherpa.onnx.OfflineModelConfig; -import com.k2fsa.sherpa.onnx.OfflineMoonshineModelConfig; -import com.k2fsa.sherpa.onnx.OfflineRecognizer; -import com.k2fsa.sherpa.onnx.OfflineRecognizerConfig; -import com.k2fsa.sherpa.onnx.OfflineRecognizerResult; -import com.k2fsa.sherpa.onnx.OfflineStream; - -import java.io.File; -import java.util.ArrayDeque; -import java.util.Arrays; - -/** - * Continuous captioning over the decoded mic PCM: segments speech into - * utterances with an adaptive energy gate, transcribes each utterance with - * the on-device Moonshine model, and attaches a speaker voice-print per - * utterance. PCM is pushed in from the TS side (which owns mic arbitration - * and any beam-direction gating), so this engine has no BLE dependencies. - * - * Utterance boundaries are measured on the sample clock, not the wall clock: - * BLE delivers mic packets in bursts, so elapsed real time overestimates the - * audio heard. The TS side converts startMs/endMs to wall-clock times using - * the engine start timestamp. - */ -public class FaceclawCaptionEngine { - private static final String TAG = "FaceclawCaptions"; - private static final int SAMPLE_RATE = 16000; - private static final int FEATURE_DIM = 80; - private static final int MAX_QUEUE_PACKETS = 200; - - // Segmentation: RMS thresholds relative to a rolling noise floor, in the - // same spirit as FaceclawVoiceController.EndpointDetector but recurring. - private static final double ONSET_FACTOR = 3.0; - private static final double RELEASE_FACTOR = 1.8; - private static final double MIN_RMS = 220.0; - private static final double NOISE_EMA_ALPHA = 0.05; - private static final int PRE_ROLL_MS = 400; - private static final int DEFAULT_SILENCE_MS = 800; - private static final int MIN_UTTERANCE_MS = 350; - private static final int MAX_UTTERANCE_MS = 15000; - // Moonshine v2 fails past ~9.1 s of input; decode long utterances in - // segments cut at the quietest window (mirrors FaceclawVoiceController). - private static final int DECODE_SEGMENT_MAX_SAMPLES = SAMPLE_RATE * 8; - private static final int CUT_SEARCH_SAMPLES = SAMPLE_RATE * 2; - private static final int CUT_WINDOW_SAMPLES = SAMPLE_RATE * 30 / 1000; - private static final float NORMALIZE_TARGET_PEAK = 0.9f; - private static final float NORMALIZE_MAX_GAIN = 30f; - // Voice-prints degrade on very long inputs; embed at most the first 10 s. - private static final int EMBED_MAX_SAMPLES = SAMPLE_RATE * 10; - - private final Handler mainHandler = new Handler(Looper.getMainLooper()); - private final Object lock = new Object(); - private final Object queueLock = new Object(); - private final ArrayDeque queue = new ArrayDeque<>(); - - private volatile FaceclawCaptionEngineListener listener; - private volatile String asrModelDir; - private volatile String speakerModelPath; - private volatile int silenceMs = DEFAULT_SILENCE_MS; - private Thread workerThread; - private volatile boolean started; - - private OfflineRecognizer recognizer; - private FaceclawSpeakerId speakerId; - - // Segmentation state (worker thread only). - private long totalSamples; - private double noiseFloor; - private boolean inUtterance; - private long utteranceStartSample; - private long silenceRunSamples; - private double utterancePeakRms; - private short[] utterance = new short[SAMPLE_RATE * (MAX_UTTERANCE_MS / 1000)]; - private int utteranceLength; - private final short[] preRoll = new short[SAMPLE_RATE * PRE_ROLL_MS / 1000]; - private int preRollLength; - - public void setListener(FaceclawCaptionEngineListener listener) { - this.listener = listener; - } - - /** Directory holding the Moonshine model files, or null to disable ASR. */ - public void setAsrModelDir(String dir) { - this.asrModelDir = dir; - } - - /** Speaker-embedding ONNX model path, or null to disable voice-prints. */ - public void setSpeakerModelPath(String path) { - this.speakerModelPath = path; - } - - public void setSilenceMs(int ms) { - this.silenceMs = Math.max(200, Math.min(3000, ms)); - } - - public void start() { - synchronized (lock) { - if (started) { - return; - } - started = true; - workerThread = new Thread(this::runLoop, "FaceclawCaptionEngine"); - workerThread.start(); - } - } - - public void stop() { - Thread threadToJoin; - synchronized (lock) { - if (!started) { - return; - } - started = false; - threadToJoin = workerThread; - workerThread = null; - } - synchronized (queueLock) { - queueLock.notifyAll(); - } - if (threadToJoin != null) { - threadToJoin.interrupt(); - if (Thread.currentThread() != threadToJoin) { - try { - threadToJoin.join(2000); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - } - } - } - } - - /** Push decoded 16 kHz mono S16LE PCM (any chunking). */ - public void acceptPcm(byte[] pcm16le) { - if (!started || pcm16le == null || pcm16le.length < 2) { - return; - } - synchronized (queueLock) { - if (queue.size() >= MAX_QUEUE_PACKETS) { - queue.removeFirst(); - } - queue.addLast(pcm16le); - queueLock.notifyAll(); - } - } - - private void runLoop() { - try { - loadModels(); - resetSegmentation(); - while (started && !Thread.currentThread().isInterrupted()) { - byte[] chunk = takeChunk(); - if (chunk == null) { - continue; - } - processChunk(chunk); - } - // Flush a trailing utterance so its text isn't lost on stop. - if (inUtterance && utteranceLength > 0) { - finalizeUtterance(); - } - } catch (Throwable t) { - Log.e(TAG, "caption engine failed", t); - emitStatus("Captions failed: " + t.getMessage()); - } finally { - releaseModels(); - } - } - - private void loadModels() { - String modelDir = asrModelDir; - if (modelDir != null && new File(modelDir, "tokens.txt").exists()) { - emitStatus("Loading caption model..."); - recognizer = new OfflineRecognizer(OfflineRecognizerConfig.builder() - .setFeatureConfig(FeatureConfig.builder() - .setSampleRate(SAMPLE_RATE) - .setFeatureDim(FEATURE_DIM) - .build()) - .setModelConfig(OfflineModelConfig.builder() - .setMoonshine(OfflineMoonshineModelConfig.builder() - .setEncoder(new File(modelDir, "encoder_model.ort").getAbsolutePath()) - .setMergedDecoder(new File(modelDir, "decoder_model_merged.ort").getAbsolutePath()) - .build()) - .setTokens(new File(modelDir, "tokens.txt").getAbsolutePath()) - .setNumThreads(1) - .build()) - .build()); - } else { - recognizer = null; - } - String embedModel = speakerModelPath; - if (embedModel != null && new File(embedModel).exists()) { - speakerId = new FaceclawSpeakerId(embedModel); - speakerId.ensureLoaded(); - } else { - speakerId = null; - } - emitStatus(recognizer != null ? "Captions listening..." : "Captions listening (no ASR model)..."); - } - - private void releaseModels() { - if (recognizer != null) { - recognizer.release(); - recognizer = null; - } - if (speakerId != null) { - speakerId.close(); - speakerId = null; - } - } - - private byte[] takeChunk() { - synchronized (queueLock) { - while (started && queue.isEmpty()) { - try { - queueLock.wait(250); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - return null; - } - } - return queue.pollFirst(); - } - } - - private void resetSegmentation() { - totalSamples = 0; - noiseFloor = 0; - inUtterance = false; - utteranceLength = 0; - preRollLength = 0; - silenceRunSamples = 0; - utterancePeakRms = 0; - } - - private void processChunk(byte[] chunk) { - int count = chunk.length / 2; - short[] pcm = new short[count]; - double sumSquares = 0; - for (int i = 0; i < count; i++) { - short s = (short) ((chunk[i * 2] & 0xff) | (chunk[i * 2 + 1] << 8)); - pcm[i] = s; - sumSquares += (double) s * s; - } - double rms = Math.sqrt(sumSquares / Math.max(1, count)); - totalSamples += count; - - if (!inUtterance) { - // Track the noise floor only while idle so speech doesn't raise it. - noiseFloor = noiseFloor == 0 ? rms : noiseFloor * (1 - NOISE_EMA_ALPHA) + rms * NOISE_EMA_ALPHA; - double threshold = Math.max(noiseFloor, MIN_RMS); - if (rms >= threshold * ONSET_FACTOR) { - inUtterance = true; - utteranceLength = 0; - silenceRunSamples = 0; - utterancePeakRms = rms; - appendUtterance(preRoll, preRollLength); - utteranceStartSample = Math.max(0, totalSamples - count - preRollLength); - appendUtterance(pcm, count); - emitSpeechStart(utteranceStartSample * 1000 / SAMPLE_RATE); - } else { - appendPreRoll(pcm, count); - } - return; - } - - appendUtterance(pcm, count); - utterancePeakRms = Math.max(utterancePeakRms, rms); - double threshold = Math.max(noiseFloor, MIN_RMS); - if (rms < threshold * RELEASE_FACTOR) { - silenceRunSamples += count; - } else { - silenceRunSamples = 0; - } - long utteranceMs = (long) utteranceLength * 1000 / SAMPLE_RATE; - boolean silenceEnded = silenceRunSamples * 1000L / SAMPLE_RATE >= silenceMs; - if (silenceEnded || utteranceMs >= MAX_UTTERANCE_MS) { - finalizeUtterance(); - inUtterance = false; - preRollLength = 0; - } - } - - private void appendPreRoll(short[] pcm, int count) { - // Keep the last PRE_ROLL_MS of idle audio so onset consonants survive. - if (count >= preRoll.length) { - System.arraycopy(pcm, count - preRoll.length, preRoll, 0, preRoll.length); - preRollLength = preRoll.length; - return; - } - int keep = Math.min(preRollLength, preRoll.length - count); - System.arraycopy(preRoll, preRollLength - keep, preRoll, 0, keep); - System.arraycopy(pcm, 0, preRoll, keep, count); - preRollLength = keep + count; - } - - private void appendUtterance(short[] pcm, int count) { - int room = utterance.length - utteranceLength; - int copied = Math.min(room, count); - if (copied > 0) { - System.arraycopy(pcm, 0, utterance, utteranceLength, copied); - utteranceLength += copied; - } - } - - private void finalizeUtterance() { - int length = utteranceLength; - long startMs = utteranceStartSample * 1000 / SAMPLE_RATE; - long endMs = (utteranceStartSample + length) * 1000 / SAMPLE_RATE; - if (endMs - startMs < MIN_UTTERANCE_MS) { - return; - } - String text = recognizeUtterance(length); - float[] embedding = embedUtterance(length); - emitUtterance(text, embedding, startMs, endMs, utterancePeakRms); - } - - private String recognizeUtterance(int length) { - if (recognizer == null || length <= 0) { - return ""; - } - StringBuilder joined = new StringBuilder(); - int offset = 0; - while (offset < length) { - int remaining = length - offset; - int segment = Math.min(remaining, DECODE_SEGMENT_MAX_SAMPLES); - if (remaining > DECODE_SEGMENT_MAX_SAMPLES) { - segment = findQuietCut(offset, segment); - } - String part = recognizeRange(offset, segment); - if (part.length() > 0) { - if (joined.length() > 0 && ".,!?;:%)]}".indexOf(part.charAt(0)) < 0) { - joined.append(' '); - } - joined.append(part); - } - offset += segment; - } - return joined.toString().trim(); - } - - /** - * End a decode segment at the center of the quietest window near its end - * so the cut lands between words (same approach as the PTT controller). - */ - private int findQuietCut(int offset, int segment) { - int searchStart = Math.max(0, segment - CUT_SEARCH_SAMPLES); - int win = CUT_WINDOW_SAMPLES; - if (segment - searchStart <= win) { - return segment; - } - double sum = 0; - for (int i = searchStart; i < searchStart + win; i++) { - double s = utterance[offset + i]; - sum += s * s; - } - double best = sum; - int bestStart = searchStart; - for (int start = searchStart + 1; start + win <= segment; start++) { - double dropped = utterance[offset + start - 1]; - double added = utterance[offset + start + win - 1]; - sum += added * added - dropped * dropped; - if (sum < best) { - best = sum; - bestStart = start; - } - } - return bestStart + win / 2; - } - - private String recognizeRange(int offset, int count) { - float[] samples = new float[count]; - float peak = 0f; - for (int i = 0; i < count; i++) { - float v = utterance[offset + i] / 32768.0f; - samples[i] = v; - float a = Math.abs(v); - if (a > peak) { - peak = a; - } - } - if (peak > 0f) { - float gain = Math.min(NORMALIZE_TARGET_PEAK / peak, NORMALIZE_MAX_GAIN); - if (gain > 1f) { - for (int i = 0; i < count; i++) { - samples[i] *= gain; - } - } - } - OfflineStream stream = recognizer.createStream(); - try { - stream.acceptWaveform(samples, SAMPLE_RATE); - recognizer.decode(stream); - OfflineRecognizerResult result = recognizer.getResult(stream); - String raw = result == null ? "" : result.getText(); - return raw == null ? "" : raw.trim(); - } finally { - stream.release(); - } - } - - private float[] embedUtterance(int length) { - FaceclawSpeakerId currentSpeakerId = speakerId; - if (currentSpeakerId == null || length <= 0) { - return null; - } - int count = Math.min(length, EMBED_MAX_SAMPLES); - byte[] le = new byte[count * 2]; - for (int i = 0; i < count; i++) { - short s = utterance[i]; - le[i * 2] = (byte) (s & 0xff); - le[i * 2 + 1] = (byte) ((s >> 8) & 0xff); - } - return currentSpeakerId.embed(le, SAMPLE_RATE); - } - - private void emitUtterance(String text, float[] embedding, long startMs, long endMs, double peakRms) { - FaceclawCaptionEngineListener currentListener = listener; - if (currentListener == null) { - return; - } - float[] embeddingCopy = embedding == null ? null : Arrays.copyOf(embedding, embedding.length); - mainHandler.post(() -> currentListener.onUtterance(text, embeddingCopy, startMs, endMs, peakRms)); - } - - private void emitSpeechStart(long startMs) { - FaceclawCaptionEngineListener currentListener = listener; - if (currentListener == null) { - return; - } - mainHandler.post(() -> currentListener.onSpeechStart(startMs)); - } - - private void emitStatus(String status) { - FaceclawCaptionEngineListener currentListener = listener; - if (currentListener == null) { - return; - } - mainHandler.post(() -> currentListener.onStatus(status)); - } -} diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawCaptionEngine.kt b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawCaptionEngine.kt new file mode 100644 index 00000000..c0cc2e04 --- /dev/null +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawCaptionEngine.kt @@ -0,0 +1,456 @@ +package com.faceclaw.app + +import android.os.Handler +import android.os.Looper +import android.util.Log + +import com.k2fsa.sherpa.onnx.FeatureConfig +import com.k2fsa.sherpa.onnx.OfflineModelConfig +import com.k2fsa.sherpa.onnx.OfflineMoonshineModelConfig +import com.k2fsa.sherpa.onnx.OfflineRecognizer +import com.k2fsa.sherpa.onnx.OfflineRecognizerConfig +import com.k2fsa.sherpa.onnx.OfflineRecognizerResult +import com.k2fsa.sherpa.onnx.OfflineStream + +import java.io.File +import java.util.ArrayDeque +import java.util.Arrays + +/** + * Continuous captioning over the decoded mic PCM: segments speech into + * utterances with an adaptive energy gate, transcribes each utterance with + * the on-device Moonshine model, and attaches a speaker voice-print per + * utterance. PCM is pushed in from the TS side (which owns mic arbitration + * and any beam-direction gating), so this engine has no BLE dependencies. + * + * Utterance boundaries are measured on the sample clock, not the wall clock: + * BLE delivers mic packets in bursts, so elapsed real time overestimates the + * audio heard. The TS side converts startMs/endMs to wall-clock times using + * the engine start timestamp. + */ +class FaceclawCaptionEngine { + companion object { + private const val TAG = "FaceclawCaptions" + private const val SAMPLE_RATE = 16000 + private const val FEATURE_DIM = 80 + private const val MAX_QUEUE_PACKETS = 200 + + // Segmentation: RMS thresholds relative to a rolling noise floor, in the + // same spirit as FaceclawVoiceController.EndpointDetector but recurring. + private const val ONSET_FACTOR = 3.0 + private const val RELEASE_FACTOR = 1.8 + private const val MIN_RMS = 220.0 + private const val NOISE_EMA_ALPHA = 0.05 + private const val PRE_ROLL_MS = 400 + private const val DEFAULT_SILENCE_MS = 800 + private const val MIN_UTTERANCE_MS = 350 + private const val MAX_UTTERANCE_MS = 15000 + // Moonshine v2 fails past ~9.1 s of input; decode long utterances in + // segments cut at the quietest window (mirrors FaceclawVoiceController). + private const val DECODE_SEGMENT_MAX_SAMPLES = SAMPLE_RATE * 8 + private const val CUT_SEARCH_SAMPLES = SAMPLE_RATE * 2 + private const val CUT_WINDOW_SAMPLES = SAMPLE_RATE * 30 / 1000 + private const val NORMALIZE_TARGET_PEAK = 0.9f + private const val NORMALIZE_MAX_GAIN = 30f + // Voice-prints degrade on very long inputs; embed at most the first 10 s. + private const val EMBED_MAX_SAMPLES = SAMPLE_RATE * 10 + } + + private val mainHandler = Handler(Looper.getMainLooper()) + private val lock = Any() + private val queueLock = Any() + private val queue = ArrayDeque() + + @Volatile + private var listener: FaceclawCaptionEngineListener? = null + @Volatile + private var asrModelDir: String? = null + @Volatile + private var speakerModelPath: String? = null + @Volatile + private var silenceMs = DEFAULT_SILENCE_MS + private var workerThread: Thread? = null + @Volatile + private var started = false + + private var recognizer: OfflineRecognizer? = null + private var speakerId: FaceclawSpeakerId? = null + + // Segmentation state (worker thread only). + private var totalSamples: Long = 0 + private var noiseFloor = 0.0 + private var inUtterance = false + private var utteranceStartSample: Long = 0 + private var silenceRunSamples: Long = 0 + private var utterancePeakRms = 0.0 + private var utterance = ShortArray(SAMPLE_RATE * (MAX_UTTERANCE_MS / 1000)) + private var utteranceLength = 0 + private val preRoll = ShortArray(SAMPLE_RATE * PRE_ROLL_MS / 1000) + private var preRollLength = 0 + + fun setListener(listener: FaceclawCaptionEngineListener?) { + this.listener = listener + } + + /** Directory holding the Moonshine model files, or null to disable ASR. */ + fun setAsrModelDir(dir: String?) { + this.asrModelDir = dir + } + + /** Speaker-embedding ONNX model path, or null to disable voice-prints. */ + fun setSpeakerModelPath(path: String?) { + this.speakerModelPath = path + } + + fun setSilenceMs(ms: Int) { + this.silenceMs = Math.max(200, Math.min(3000, ms)) + } + + fun start() { + synchronized(lock) { + if (started) { + return + } + started = true + workerThread = Thread(Runnable { runLoop() }, "FaceclawCaptionEngine") + workerThread!!.start() + } + } + + fun stop() { + val threadToJoin: Thread? + synchronized(lock) { + if (!started) { + return + } + started = false + threadToJoin = workerThread + workerThread = null + } + synchronized(queueLock) { + (queueLock as java.lang.Object).notifyAll() + } + if (threadToJoin != null) { + threadToJoin.interrupt() + if (Thread.currentThread() !== threadToJoin) { + try { + threadToJoin.join(2000) + } catch (e: InterruptedException) { + Thread.currentThread().interrupt() + } + } + } + } + + /** Push decoded 16 kHz mono S16LE PCM (any chunking). */ + fun acceptPcm(pcm16le: ByteArray?) { + if (!started || pcm16le == null || pcm16le.size < 2) { + return + } + synchronized(queueLock) { + if (queue.size >= MAX_QUEUE_PACKETS) { + queue.removeFirst() + } + queue.addLast(pcm16le) + (queueLock as java.lang.Object).notifyAll() + } + } + + private fun runLoop() { + try { + loadModels() + resetSegmentation() + while (started && !Thread.currentThread().isInterrupted) { + val chunk = takeChunk() + if (chunk == null) { + continue + } + processChunk(chunk) + } + // Flush a trailing utterance so its text isn't lost on stop. + if (inUtterance && utteranceLength > 0) { + finalizeUtterance() + } + } catch (t: Throwable) { + Log.e(TAG, "caption engine failed", t) + emitStatus("Captions failed: " + t.message) + } finally { + releaseModels() + } + } + + private fun loadModels() { + val modelDir = asrModelDir + if (modelDir != null && File(modelDir, "tokens.txt").exists()) { + emitStatus("Loading caption model...") + recognizer = OfflineRecognizer(OfflineRecognizerConfig.builder() + .setFeatureConfig(FeatureConfig.builder() + .setSampleRate(SAMPLE_RATE) + .setFeatureDim(FEATURE_DIM) + .build()) + .setModelConfig(OfflineModelConfig.builder() + .setMoonshine(OfflineMoonshineModelConfig.builder() + .setEncoder(File(modelDir, "encoder_model.ort").absolutePath) + .setMergedDecoder(File(modelDir, "decoder_model_merged.ort").absolutePath) + .build()) + .setTokens(File(modelDir, "tokens.txt").absolutePath) + .setNumThreads(1) + .build()) + .build()) + } else { + recognizer = null + } + val embedModel = speakerModelPath + if (embedModel != null && File(embedModel).exists()) { + speakerId = FaceclawSpeakerId(embedModel) + speakerId!!.ensureLoaded() + } else { + speakerId = null + } + emitStatus(if (recognizer != null) "Captions listening..." else "Captions listening (no ASR model)...") + } + + private fun releaseModels() { + if (recognizer != null) { + recognizer!!.release() + recognizer = null + } + if (speakerId != null) { + speakerId!!.close() + speakerId = null + } + } + + private fun takeChunk(): ByteArray? { + synchronized(queueLock) { + while (started && queue.isEmpty()) { + try { + (queueLock as java.lang.Object).wait(250) + } catch (e: InterruptedException) { + Thread.currentThread().interrupt() + return null + } + } + return queue.pollFirst() + } + } + + private fun resetSegmentation() { + totalSamples = 0 + noiseFloor = 0.0 + inUtterance = false + utteranceLength = 0 + preRollLength = 0 + silenceRunSamples = 0 + utterancePeakRms = 0.0 + } + + private fun processChunk(chunk: ByteArray) { + val count = chunk.size / 2 + val pcm = ShortArray(count) + var sumSquares = 0.0 + for (i in 0 until count) { + val s = ((chunk[i * 2].toInt() and 0xff) or (chunk[i * 2 + 1].toInt() shl 8)).toShort() + pcm[i] = s + sumSquares += s.toDouble() * s + } + val rms = Math.sqrt(sumSquares / Math.max(1, count)) + totalSamples += count.toLong() + + if (!inUtterance) { + // Track the noise floor only while idle so speech doesn't raise it. + noiseFloor = if (noiseFloor == 0.0) rms else noiseFloor * (1 - NOISE_EMA_ALPHA) + rms * NOISE_EMA_ALPHA + val threshold = Math.max(noiseFloor, MIN_RMS) + if (rms >= threshold * ONSET_FACTOR) { + inUtterance = true + utteranceLength = 0 + silenceRunSamples = 0 + utterancePeakRms = rms + appendUtterance(preRoll, preRollLength) + utteranceStartSample = Math.max(0L, totalSamples - count - preRollLength) + appendUtterance(pcm, count) + emitSpeechStart(utteranceStartSample * 1000 / SAMPLE_RATE) + } else { + appendPreRoll(pcm, count) + } + return + } + + appendUtterance(pcm, count) + utterancePeakRms = Math.max(utterancePeakRms, rms) + val threshold = Math.max(noiseFloor, MIN_RMS) + if (rms < threshold * RELEASE_FACTOR) { + silenceRunSamples += count.toLong() + } else { + silenceRunSamples = 0 + } + val utteranceMs = utteranceLength.toLong() * 1000 / SAMPLE_RATE + val silenceEnded = silenceRunSamples * 1000L / SAMPLE_RATE >= silenceMs + if (silenceEnded || utteranceMs >= MAX_UTTERANCE_MS) { + finalizeUtterance() + inUtterance = false + preRollLength = 0 + } + } + + private fun appendPreRoll(pcm: ShortArray, count: Int) { + // Keep the last PRE_ROLL_MS of idle audio so onset consonants survive. + if (count >= preRoll.size) { + System.arraycopy(pcm, count - preRoll.size, preRoll, 0, preRoll.size) + preRollLength = preRoll.size + return + } + val keep = Math.min(preRollLength, preRoll.size - count) + System.arraycopy(preRoll, preRollLength - keep, preRoll, 0, keep) + System.arraycopy(pcm, 0, preRoll, keep, count) + preRollLength = keep + count + } + + private fun appendUtterance(pcm: ShortArray, count: Int) { + val room = utterance.size - utteranceLength + val copied = Math.min(room, count) + if (copied > 0) { + System.arraycopy(pcm, 0, utterance, utteranceLength, copied) + utteranceLength += copied + } + } + + private fun finalizeUtterance() { + val length = utteranceLength + val startMs = utteranceStartSample * 1000 / SAMPLE_RATE + val endMs = (utteranceStartSample + length) * 1000 / SAMPLE_RATE + if (endMs - startMs < MIN_UTTERANCE_MS) { + return + } + val text = recognizeUtterance(length) + val embedding = embedUtterance(length) + emitUtterance(text, embedding, startMs, endMs, utterancePeakRms) + } + + private fun recognizeUtterance(length: Int): String { + if (recognizer == null || length <= 0) { + return "" + } + val joined = StringBuilder() + var offset = 0 + while (offset < length) { + val remaining = length - offset + var segment = Math.min(remaining, DECODE_SEGMENT_MAX_SAMPLES) + if (remaining > DECODE_SEGMENT_MAX_SAMPLES) { + segment = findQuietCut(offset, segment) + } + val part = recognizeRange(offset, segment) + if (part.length > 0) { + if (joined.length > 0 && ".,!?;:%)]}".indexOf(part[0]) < 0) { + joined.append(' ') + } + joined.append(part) + } + offset += segment + } + return joined.toString().trim() + } + + /** + * End a decode segment at the center of the quietest window near its end + * so the cut lands between words (same approach as the PTT controller). + */ + private fun findQuietCut(offset: Int, segment: Int): Int { + val searchStart = Math.max(0, segment - CUT_SEARCH_SAMPLES) + val win = CUT_WINDOW_SAMPLES + if (segment - searchStart <= win) { + return segment + } + var sum = 0.0 + for (i in searchStart until searchStart + win) { + val s = utterance[offset + i].toDouble() + sum += s * s + } + var best = sum + var bestStart = searchStart + var start = searchStart + 1 + while (start + win <= segment) { + val dropped = utterance[offset + start - 1].toDouble() + val added = utterance[offset + start + win - 1].toDouble() + sum += added * added - dropped * dropped + if (sum < best) { + best = sum + bestStart = start + } + start++ + } + return bestStart + win / 2 + } + + private fun recognizeRange(offset: Int, count: Int): String { + val samples = FloatArray(count) + var peak = 0f + for (i in 0 until count) { + val v = utterance[offset + i] / 32768.0f + samples[i] = v + val a = Math.abs(v) + if (a > peak) { + peak = a + } + } + if (peak > 0f) { + val gain = Math.min(NORMALIZE_TARGET_PEAK / peak, NORMALIZE_MAX_GAIN) + if (gain > 1f) { + for (i in 0 until count) { + samples[i] *= gain + } + } + } + val recognizer = this.recognizer!! + val stream: OfflineStream = recognizer.createStream() + try { + stream.acceptWaveform(samples, SAMPLE_RATE) + recognizer.decode(stream) + val result: OfflineRecognizerResult? = recognizer.getResult(stream) + val raw: String? = if (result == null) "" else result.text + return raw?.trim() ?: "" + } finally { + stream.release() + } + } + + private fun embedUtterance(length: Int): FloatArray? { + val currentSpeakerId = speakerId + if (currentSpeakerId == null || length <= 0) { + return null + } + val count = Math.min(length, EMBED_MAX_SAMPLES) + val le = ByteArray(count * 2) + for (i in 0 until count) { + val s = utterance[i].toInt() + le[i * 2] = (s and 0xff).toByte() + le[i * 2 + 1] = ((s shr 8) and 0xff).toByte() + } + return currentSpeakerId.embed(le, SAMPLE_RATE) + } + + private fun emitUtterance(text: String?, embedding: FloatArray?, startMs: Long, endMs: Long, peakRms: Double) { + val currentListener = listener + if (currentListener == null) { + return + } + val embeddingCopy = if (embedding == null) null else Arrays.copyOf(embedding, embedding.size) + mainHandler.post { currentListener.onUtterance(text, embeddingCopy, startMs, endMs, peakRms) } + } + + private fun emitSpeechStart(startMs: Long) { + val currentListener = listener + if (currentListener == null) { + return + } + mainHandler.post { currentListener.onSpeechStart(startMs) } + } + + private fun emitStatus(status: String?) { + val currentListener = listener + if (currentListener == null) { + return + } + mainHandler.post { currentListener.onStatus(status) } + } +} diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawConversationStore.java b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawConversationStore.java deleted file mode 100644 index 38f723fe..00000000 --- a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawConversationStore.java +++ /dev/null @@ -1,778 +0,0 @@ -package com.faceclaw.app; - -import android.content.ContentValues; -import android.content.Context; -import android.database.Cursor; -import android.database.sqlite.SQLiteDatabase; -import android.database.sqlite.SQLiteOpenHelper; -import android.util.Base64; -import android.util.Log; - -import org.json.JSONArray; -import org.json.JSONException; -import org.json.JSONObject; - -import java.io.File; -import java.nio.ByteBuffer; -import java.nio.ByteOrder; -import java.util.ArrayList; -import java.util.List; - -/** - * Persistent store for caption sessions, transcript segments, and speaker - * voice profiles, backing the Microphones app and its phone-side review UI. - * All complex values cross the JS bridge as JSON strings; voice-print - * embeddings are float32 LE blobs, exposed as base64. - */ -public class FaceclawConversationStore extends SQLiteOpenHelper { - private static final String TAG = "FaceclawConvStore"; - private static final String DB_NAME = "faceclaw-conversations.db"; - private static final int DB_VERSION = 2; - - private static FaceclawConversationStore instance; - - public static synchronized FaceclawConversationStore getInstance(Context context) { - if (instance == null) { - instance = new FaceclawConversationStore(context.getApplicationContext()); - } - return instance; - } - - private FaceclawConversationStore(Context context) { - super(context, DB_NAME, null, DB_VERSION); - } - - @Override - public void onCreate(SQLiteDatabase db) { - db.execSQL("CREATE TABLE speakers (" - + "id INTEGER PRIMARY KEY AUTOINCREMENT," - + "name TEXT NOT NULL," - + "color TEXT NOT NULL DEFAULT '#4FC3F7'," - + "is_wearer INTEGER NOT NULL DEFAULT 0," - + "embedding BLOB," - + "embedding_count INTEGER NOT NULL DEFAULT 0," - + "created_at INTEGER NOT NULL," - + "last_heard_at INTEGER," - + "tag TEXT," - + "last_recap TEXT," - + "action_items TEXT," - + "facts TEXT," - + "insights_updated_at INTEGER," - + "insights_session_id INTEGER)"); - db.execSQL("CREATE TABLE sessions (" - + "id INTEGER PRIMARY KEY AUTOINCREMENT," - + "started_at INTEGER NOT NULL," - + "ended_at INTEGER," - + "title TEXT," - + "audio_path TEXT," - + "audio_codec TEXT," - + "avg_sentiment REAL," - + "segment_count INTEGER NOT NULL DEFAULT 0)"); - db.execSQL("CREATE TABLE segments (" - + "id INTEGER PRIMARY KEY AUTOINCREMENT," - + "session_id INTEGER NOT NULL," - + "speaker_id INTEGER," - + "started_at INTEGER NOT NULL," - + "ended_at INTEGER NOT NULL," - + "audio_offset_ms INTEGER," - + "text TEXT NOT NULL," - + "lang TEXT," - + "translation TEXT," - + "translation_lang TEXT," - + "sentiment REAL," - + "emotion TEXT," - + "search_meta TEXT," - + "angle INTEGER," - + "embedding BLOB)"); - db.execSQL("CREATE INDEX idx_segments_session ON segments(session_id)"); - db.execSQL("CREATE INDEX idx_segments_speaker ON segments(speaker_id)"); - db.execSQL("CREATE INDEX idx_segments_time ON segments(started_at)"); - db.execSQL("CREATE INDEX idx_sessions_time ON sessions(started_at)"); - } - - @Override - public void onUpgrade(SQLiteDatabase db, int oldVersion, int newVersion) { - if (oldVersion < 2) { - // v2: per-speaker conversation insights (recap of the most recent - // conversation, its action items, and accumulated inferred facts). - db.execSQL("ALTER TABLE speakers ADD COLUMN last_recap TEXT"); - db.execSQL("ALTER TABLE speakers ADD COLUMN action_items TEXT"); - db.execSQL("ALTER TABLE speakers ADD COLUMN facts TEXT"); - db.execSQL("ALTER TABLE speakers ADD COLUMN insights_updated_at INTEGER"); - db.execSQL("ALTER TABLE speakers ADD COLUMN insights_session_id INTEGER"); - } - } - - // ---- sessions ---- - - public long startSession(long startedAtMs, String title) { - ContentValues values = new ContentValues(); - values.put("started_at", startedAtMs); - values.put("title", title == null || title.isEmpty() ? null : title); - return getWritableDatabase().insert("sessions", null, values); - } - - public void endSession(long sessionId, long endedAtMs, String audioPath, String audioCodec, double avgSentiment) { - ContentValues values = new ContentValues(); - values.put("ended_at", endedAtMs); - if (audioPath != null && !audioPath.isEmpty()) { - values.put("audio_path", audioPath); - values.put("audio_codec", audioCodec); - } - values.put("avg_sentiment", avgSentiment); - getWritableDatabase().update("sessions", values, "id=?", new String[]{String.valueOf(sessionId)}); - } - - public void setSessionAudio(long sessionId, String audioPath, String audioCodec) { - ContentValues values = new ContentValues(); - values.put("audio_path", audioPath); - values.put("audio_codec", audioCodec); - getWritableDatabase().update("sessions", values, "id=?", new String[]{String.valueOf(sessionId)}); - } - - public String getSessionAudioPath(long sessionId) { - try (Cursor c = getReadableDatabase().rawQuery( - "SELECT audio_path FROM sessions WHERE id=?", new String[]{String.valueOf(sessionId)})) { - if (c.moveToFirst()) { - return c.isNull(0) ? "" : c.getString(0); - } - } - return ""; - } - - /** - * Sessions, newest first, as JSON. filterJson supports: sinceMs, untilMs, - * speakerId, emotion (matches any segment emotion in the session), query - * (LIKE over title and segment text/search metadata), limit. - */ - public String querySessions(String filterJson) { - try { - JSONObject filter = filterJson == null || filterJson.isEmpty() - ? new JSONObject() : new JSONObject(filterJson); - StringBuilder sql = new StringBuilder( - "SELECT DISTINCT s.id, s.started_at, s.ended_at, s.title, s.audio_path, s.audio_codec," - + " s.avg_sentiment, s.segment_count FROM sessions s"); - List args = new ArrayList<>(); - List where = new ArrayList<>(); - boolean joinSegments = filter.has("speakerId") || filter.has("emotion") || filter.has("query"); - if (joinSegments) { - sql.append(" LEFT JOIN segments g ON g.session_id = s.id"); - } - if (filter.has("sinceMs")) { - where.add("s.started_at >= ?"); - args.add(String.valueOf(filter.getLong("sinceMs"))); - } - if (filter.has("untilMs")) { - where.add("s.started_at <= ?"); - args.add(String.valueOf(filter.getLong("untilMs"))); - } - if (filter.has("speakerId")) { - where.add("g.speaker_id = ?"); - args.add(String.valueOf(filter.getLong("speakerId"))); - } - if (filter.has("emotion")) { - where.add("g.emotion = ?"); - args.add(filter.getString("emotion")); - } - if (filter.has("query")) { - String like = "%" + filter.getString("query") + "%"; - where.add("(s.title LIKE ? OR g.text LIKE ? OR g.search_meta LIKE ?)"); - args.add(like); - args.add(like); - args.add(like); - } - if (!where.isEmpty()) { - sql.append(" WHERE ").append(String.join(" AND ", where)); - } - sql.append(" ORDER BY s.started_at DESC"); - int limit = filter.optInt("limit", 200); - sql.append(" LIMIT ").append(Math.max(1, Math.min(limit, 1000))); - - JSONArray out = new JSONArray(); - try (Cursor c = getReadableDatabase().rawQuery(sql.toString(), args.toArray(new String[0]))) { - while (c.moveToNext()) { - JSONObject row = new JSONObject(); - long id = c.getLong(0); - row.put("id", id); - row.put("startedAt", c.getLong(1)); - row.put("endedAt", c.isNull(2) ? JSONObject.NULL : c.getLong(2)); - row.put("title", c.isNull(3) ? "" : c.getString(3)); - row.put("audioPath", c.isNull(4) ? "" : c.getString(4)); - row.put("audioCodec", c.isNull(5) ? "" : c.getString(5)); - row.put("avgSentiment", c.isNull(6) ? 0 : c.getDouble(6)); - row.put("segmentCount", c.getLong(7)); - row.put("speakers", sessionSpeakersJson(id)); - out.put(row); - } - } - return out.toString(); - } catch (JSONException e) { - Log.w(TAG, "querySessions failed", e); - return "[]"; - } - } - - private JSONArray sessionSpeakersJson(long sessionId) throws JSONException { - JSONArray out = new JSONArray(); - try (Cursor c = getReadableDatabase().rawQuery( - "SELECT DISTINCT p.id, p.name, p.color FROM segments g" - + " JOIN speakers p ON p.id = g.speaker_id WHERE g.session_id=?", - new String[]{String.valueOf(sessionId)})) { - while (c.moveToNext()) { - JSONObject row = new JSONObject(); - row.put("id", c.getLong(0)); - row.put("name", c.getString(1)); - row.put("color", c.getString(2)); - out.put(row); - } - } - return out; - } - - // ---- segments ---- - - /** - * Insert one transcript segment. json fields: sessionId, speakerId?, - * startedAt, endedAt, audioOffsetMs?, text, lang?, translation?, - * translationLang?, sentiment?, emotion?, searchMeta?, angle?, - * embeddingBase64?. Returns the new row id, or -1. - */ - public long insertSegment(String json) { - try { - JSONObject s = new JSONObject(json); - ContentValues values = new ContentValues(); - long sessionId = s.getLong("sessionId"); - values.put("session_id", sessionId); - if (s.has("speakerId")) { - values.put("speaker_id", s.getLong("speakerId")); - } - values.put("started_at", s.getLong("startedAt")); - values.put("ended_at", s.getLong("endedAt")); - if (s.has("audioOffsetMs")) { - values.put("audio_offset_ms", s.getLong("audioOffsetMs")); - } - values.put("text", s.getString("text")); - putOptString(values, s, "lang", "lang"); - putOptString(values, s, "translation", "translation"); - putOptString(values, s, "translationLang", "translation_lang"); - if (s.has("sentiment")) { - values.put("sentiment", s.getDouble("sentiment")); - } - putOptString(values, s, "emotion", "emotion"); - putOptString(values, s, "searchMeta", "search_meta"); - if (s.has("angle")) { - values.put("angle", s.getInt("angle")); - } - if (s.has("embeddingBase64")) { - values.put("embedding", Base64.decode(s.getString("embeddingBase64"), Base64.NO_WRAP)); - } - SQLiteDatabase db = getWritableDatabase(); - long id = db.insert("segments", null, values); - db.execSQL("UPDATE sessions SET segment_count = segment_count + 1 WHERE id=?", - new Object[]{sessionId}); - return id; - } catch (Throwable t) { - Log.w(TAG, "insertSegment failed", t); - return -1; - } - } - - private static void putOptString(ContentValues values, JSONObject s, String jsonKey, String column) - throws JSONException { - if (s.has(jsonKey) && !s.isNull(jsonKey)) { - values.put(column, s.getString(jsonKey)); - } - } - - /** Update mutable fields of a segment (same keys as insertSegment). */ - public void updateSegment(long segmentId, String json) { - try { - JSONObject s = new JSONObject(json); - ContentValues values = new ContentValues(); - if (s.has("speakerId")) { - if (s.isNull("speakerId")) { - values.putNull("speaker_id"); - } else { - values.put("speaker_id", s.getLong("speakerId")); - } - } - if (s.has("text")) { - values.put("text", s.getString("text")); - } - putOptString(values, s, "lang", "lang"); - putOptString(values, s, "translation", "translation"); - putOptString(values, s, "translationLang", "translation_lang"); - if (s.has("sentiment")) { - values.put("sentiment", s.getDouble("sentiment")); - } - putOptString(values, s, "emotion", "emotion"); - putOptString(values, s, "searchMeta", "search_meta"); - if (values.size() > 0) { - getWritableDatabase().update("segments", values, "id=?", - new String[]{String.valueOf(segmentId)}); - } - } catch (Throwable t) { - Log.w(TAG, "updateSegment failed", t); - } - } - - public String querySegments(long sessionId) { - JSONArray out = new JSONArray(); - try (Cursor c = getReadableDatabase().rawQuery( - "SELECT id, speaker_id, started_at, ended_at, audio_offset_ms, text, lang," - + " translation, translation_lang, sentiment, emotion, angle" - + " FROM segments WHERE session_id=? ORDER BY started_at ASC", - new String[]{String.valueOf(sessionId)})) { - while (c.moveToNext()) { - out.put(segmentRowJson(c)); - } - } catch (JSONException e) { - Log.w(TAG, "querySegments failed", e); - } - return out.toString(); - } - - private JSONObject segmentRowJson(Cursor c) throws JSONException { - JSONObject row = new JSONObject(); - row.put("id", c.getLong(0)); - row.put("speakerId", c.isNull(1) ? JSONObject.NULL : c.getLong(1)); - row.put("startedAt", c.getLong(2)); - row.put("endedAt", c.getLong(3)); - row.put("audioOffsetMs", c.isNull(4) ? JSONObject.NULL : c.getLong(4)); - row.put("text", c.getString(5)); - row.put("lang", c.isNull(6) ? "" : c.getString(6)); - row.put("translation", c.isNull(7) ? "" : c.getString(7)); - row.put("translationLang", c.isNull(8) ? "" : c.getString(8)); - row.put("sentiment", c.isNull(9) ? 0 : c.getDouble(9)); - row.put("emotion", c.isNull(10) ? "" : c.getString(10)); - row.put("angle", c.isNull(11) ? JSONObject.NULL : c.getInt(11)); - return row; - } - - /** - * Search segments across sessions. filterJson: query?, emotion?, - * speakerId?, sinceMs?, untilMs?, limit?. Matches text, translation, and - * the sentiment search metadata. - */ - public String searchSegments(String filterJson) { - try { - JSONObject filter = filterJson == null || filterJson.isEmpty() - ? new JSONObject() : new JSONObject(filterJson); - StringBuilder sql = new StringBuilder( - "SELECT id, speaker_id, started_at, ended_at, audio_offset_ms, text, lang," - + " translation, translation_lang, sentiment, emotion, angle, session_id" - + " FROM segments"); - List args = new ArrayList<>(); - List where = new ArrayList<>(); - if (filter.has("query") && !filter.getString("query").isEmpty()) { - String like = "%" + filter.getString("query") + "%"; - where.add("(text LIKE ? OR translation LIKE ? OR search_meta LIKE ?)"); - args.add(like); - args.add(like); - args.add(like); - } - if (filter.has("emotion")) { - where.add("emotion = ?"); - args.add(filter.getString("emotion")); - } - if (filter.has("speakerId")) { - where.add("speaker_id = ?"); - args.add(String.valueOf(filter.getLong("speakerId"))); - } - if (filter.has("sinceMs")) { - where.add("started_at >= ?"); - args.add(String.valueOf(filter.getLong("sinceMs"))); - } - if (filter.has("untilMs")) { - where.add("started_at <= ?"); - args.add(String.valueOf(filter.getLong("untilMs"))); - } - if (!where.isEmpty()) { - sql.append(" WHERE ").append(String.join(" AND ", where)); - } - sql.append(" ORDER BY started_at DESC LIMIT ") - .append(Math.max(1, Math.min(filter.optInt("limit", 300), 1000))); - JSONArray out = new JSONArray(); - try (Cursor c = getReadableDatabase().rawQuery(sql.toString(), args.toArray(new String[0]))) { - while (c.moveToNext()) { - JSONObject row = segmentRowJson(c); - row.put("sessionId", c.getLong(12)); - out.put(row); - } - } - return out.toString(); - } catch (JSONException e) { - Log.w(TAG, "searchSegments failed", e); - return "[]"; - } - } - - /** Per-segment voice-print embeddings of a session (for re-diarization). */ - public String querySegmentEmbeddings(long sessionId) { - JSONArray out = new JSONArray(); - try (Cursor c = getReadableDatabase().rawQuery( - "SELECT id, embedding FROM segments WHERE session_id=? AND embedding IS NOT NULL", - new String[]{String.valueOf(sessionId)})) { - while (c.moveToNext()) { - JSONObject row = new JSONObject(); - row.put("id", c.getLong(0)); - row.put("embeddingBase64", Base64.encodeToString(c.getBlob(1), Base64.NO_WRAP)); - out.put(row); - } - } catch (JSONException e) { - Log.w(TAG, "querySegmentEmbeddings failed", e); - } - return out.toString(); - } - - // ---- speakers ---- - - public long createSpeaker(String name, String color, boolean isWearer, String embeddingBase64) { - ContentValues values = new ContentValues(); - values.put("name", name); - values.put("color", color == null || color.isEmpty() ? "#4FC3F7" : color); - values.put("is_wearer", isWearer ? 1 : 0); - values.put("created_at", System.currentTimeMillis()); - values.put("last_heard_at", System.currentTimeMillis()); - if (embeddingBase64 != null && !embeddingBase64.isEmpty()) { - values.put("embedding", Base64.decode(embeddingBase64, Base64.NO_WRAP)); - values.put("embedding_count", 1); - } - return getWritableDatabase().insert("speakers", null, values); - } - - public String querySpeakers() { - JSONArray out = new JSONArray(); - try (Cursor c = getReadableDatabase().rawQuery( - "SELECT p.id, p.name, p.color, p.is_wearer, p.embedding, p.embedding_count," - + " p.last_heard_at, p.tag," - + " p.last_recap, p.action_items, p.facts, p.insights_updated_at, p.insights_session_id," - + " (SELECT COUNT(*) FROM segments g WHERE g.speaker_id = p.id) AS segments" - + " FROM speakers p ORDER BY p.last_heard_at DESC", null)) { - while (c.moveToNext()) { - JSONObject row = new JSONObject(); - row.put("id", c.getLong(0)); - row.put("name", c.getString(1)); - row.put("color", c.getString(2)); - row.put("isWearer", c.getInt(3) != 0); - byte[] blob = c.isNull(4) ? null : c.getBlob(4); - row.put("embeddingBase64", blob == null ? "" : Base64.encodeToString(blob, Base64.NO_WRAP)); - row.put("embeddingCount", c.getInt(5)); - row.put("lastHeardAt", c.isNull(6) ? 0 : c.getLong(6)); - row.put("tag", c.isNull(7) ? "" : c.getString(7)); - row.put("lastRecap", c.isNull(8) ? "" : c.getString(8)); - row.put("actionItemsJson", c.isNull(9) ? "" : c.getString(9)); - row.put("factsJson", c.isNull(10) ? "" : c.getString(10)); - row.put("insightsUpdatedAt", c.isNull(11) ? 0 : c.getLong(11)); - row.put("insightsSessionId", c.isNull(12) ? 0 : c.getLong(12)); - row.put("segmentCount", c.getLong(13)); - out.put(row); - } - } catch (JSONException e) { - Log.w(TAG, "querySpeakers failed", e); - } - return out.toString(); - } - - public void renameSpeaker(long speakerId, String name) { - ContentValues values = new ContentValues(); - values.put("name", name); - getWritableDatabase().update("speakers", values, "id=?", new String[]{String.valueOf(speakerId)}); - } - - public void setSpeakerColor(long speakerId, String color) { - ContentValues values = new ContentValues(); - values.put("color", color); - getWritableDatabase().update("speakers", values, "id=?", new String[]{String.valueOf(speakerId)}); - } - - public void setSpeakerTag(long speakerId, String tag) { - ContentValues values = new ContentValues(); - values.put("tag", tag); - getWritableDatabase().update("speakers", values, "id=?", new String[]{String.valueOf(speakerId)}); - } - - /** - * Store LLM-derived conversation insights for a speaker: a recap of their - * most recent conversation, its open action items (JSON string array), and - * the accumulated inferred facts about them (JSON string array). - */ - public void setSpeakerInsights(long speakerId, String recap, String actionItemsJson, - String factsJson, long sessionId) { - ContentValues values = new ContentValues(); - values.put("last_recap", recap == null ? "" : recap); - values.put("action_items", actionItemsJson == null ? "" : actionItemsJson); - values.put("facts", factsJson == null ? "" : factsJson); - values.put("insights_updated_at", System.currentTimeMillis()); - values.put("insights_session_id", sessionId); - getWritableDatabase().update("speakers", values, "id=?", new String[]{String.valueOf(speakerId)}); - } - - public void setSpeakerWearer(long speakerId, boolean isWearer) { - SQLiteDatabase db = getWritableDatabase(); - if (isWearer) { - // Exactly one wearer profile at a time. - db.execSQL("UPDATE speakers SET is_wearer = 0"); - } - ContentValues values = new ContentValues(); - values.put("is_wearer", isWearer ? 1 : 0); - db.update("speakers", values, "id=?", new String[]{String.valueOf(speakerId)}); - } - - /** - * Running-mean centroid update for a speaker's voice-print, capped so a - * long session cannot drown the enrolled voice. - */ - public void updateSpeakerEmbedding(long speakerId, String embeddingBase64, int maxCount) { - try { - byte[] incoming = Base64.decode(embeddingBase64, Base64.NO_WRAP); - float[] update = blobToFloats(incoming); - if (update == null) { - return; - } - float[] current = null; - int count = 0; - try (Cursor c = getReadableDatabase().rawQuery( - "SELECT embedding, embedding_count FROM speakers WHERE id=?", - new String[]{String.valueOf(speakerId)})) { - if (c.moveToFirst()) { - current = c.isNull(0) ? null : blobToFloats(c.getBlob(0)); - count = c.getInt(1); - } - } - float[] merged; - if (current == null || current.length != update.length) { - merged = update; - count = 1; - } else { - int effective = Math.min(count, Math.max(1, maxCount)); - merged = new float[current.length]; - for (int i = 0; i < merged.length; i++) { - merged[i] = (current[i] * effective + update[i]) / (effective + 1); - } - l2NormalizeInPlace(merged); - count = count + 1; - } - ContentValues values = new ContentValues(); - values.put("embedding", floatsToBlob(merged)); - values.put("embedding_count", count); - values.put("last_heard_at", System.currentTimeMillis()); - getWritableDatabase().update("speakers", values, "id=?", - new String[]{String.valueOf(speakerId)}); - } catch (Throwable t) { - Log.w(TAG, "updateSpeakerEmbedding failed", t); - } - } - - /** Move every segment from one speaker onto another and drop the source. */ - public void mergeSpeakers(long fromId, long intoId) { - if (fromId == intoId) { - return; - } - SQLiteDatabase db = getWritableDatabase(); - db.beginTransaction(); - try { - db.execSQL("UPDATE segments SET speaker_id=? WHERE speaker_id=?", - new Object[]{intoId, fromId}); - // Blend the two centroids weighted by their sample counts. - float[] from = null; - float[] into = null; - int fromCount = 0; - int intoCount = 0; - boolean fromWearer = false; - try (Cursor c = db.rawQuery( - "SELECT id, embedding, embedding_count, is_wearer FROM speakers WHERE id IN (?,?)", - new String[]{String.valueOf(fromId), String.valueOf(intoId)})) { - while (c.moveToNext()) { - long id = c.getLong(0); - float[] embedding = c.isNull(1) ? null : blobToFloats(c.getBlob(1)); - int count = c.getInt(2); - if (id == fromId) { - from = embedding; - fromCount = count; - fromWearer = c.getInt(3) != 0; - } else { - into = embedding; - intoCount = count; - } - } - } - float[] merged = into; - if (from != null && into != null && from.length == into.length) { - merged = new float[into.length]; - int total = Math.max(1, fromCount + intoCount); - for (int i = 0; i < merged.length; i++) { - merged[i] = (into[i] * intoCount + from[i] * fromCount) / total; - } - l2NormalizeInPlace(merged); - } else if (into == null) { - merged = from; - } - ContentValues values = new ContentValues(); - if (merged != null) { - values.put("embedding", floatsToBlob(merged)); - values.put("embedding_count", fromCount + intoCount); - } - if (fromWearer) { - values.put("is_wearer", 1); - } - values.put("last_heard_at", System.currentTimeMillis()); - // Keep the target's conversation insights; adopt the source's only - // where the target has none (e.g. merging a rich profile into a - // freshly auto-created one). - try (Cursor c = db.rawQuery( - "SELECT last_recap, action_items, facts, insights_updated_at, insights_session_id" - + " FROM speakers WHERE id IN (?,?) ORDER BY CASE id WHEN ? THEN 0 ELSE 1 END", - new String[]{String.valueOf(fromId), String.valueOf(intoId), String.valueOf(intoId)})) { - boolean targetHasInsights = false; - while (c.moveToNext()) { - boolean hasInsights = !c.isNull(0) && !c.getString(0).isEmpty(); - if (c.getPosition() == 0) { - targetHasInsights = hasInsights; - } else if (!targetHasInsights && hasInsights) { - values.put("last_recap", c.getString(0)); - values.put("action_items", c.isNull(1) ? "" : c.getString(1)); - values.put("facts", c.isNull(2) ? "" : c.getString(2)); - values.put("insights_updated_at", c.isNull(3) ? 0 : c.getLong(3)); - values.put("insights_session_id", c.isNull(4) ? 0 : c.getLong(4)); - } - } - } - db.update("speakers", values, "id=?", new String[]{String.valueOf(intoId)}); - db.delete("speakers", "id=?", new String[]{String.valueOf(fromId)}); - db.setTransactionSuccessful(); - } finally { - db.endTransaction(); - } - } - - public void reassignSegmentSpeaker(long segmentId, long speakerId) { - ContentValues values = new ContentValues(); - values.put("speaker_id", speakerId); - getWritableDatabase().update("segments", values, "id=?", new String[]{String.valueOf(segmentId)}); - } - - /** Reassign every segment of a speaker within one session (split support). */ - public void reassignSessionSpeaker(long sessionId, long fromSpeakerId, long toSpeakerId) { - getWritableDatabase().execSQL( - "UPDATE segments SET speaker_id=? WHERE session_id=? AND speaker_id=?", - new Object[]{toSpeakerId, sessionId, fromSpeakerId}); - } - - public void deleteSpeaker(long speakerId) { - SQLiteDatabase db = getWritableDatabase(); - db.execSQL("UPDATE segments SET speaker_id=NULL WHERE speaker_id=?", new Object[]{speakerId}); - db.delete("speakers", "id=?", new String[]{String.valueOf(speakerId)}); - } - - // ---- retention ---- - - /** - * Delete caption sessions older than the cutoff (0 disables), and strip - * recordings older than their own cutoff while keeping the transcript. - * Returns a JSON summary of what was removed. - */ - public String applyRetention(long captionCutoffMs, long recordingCutoffMs) { - SQLiteDatabase db = getWritableDatabase(); - int sessionsDeleted = 0; - int recordingsDeleted = 0; - if (recordingCutoffMs > 0) { - List paths = new ArrayList<>(); - try (Cursor c = db.rawQuery( - "SELECT id, audio_path FROM sessions WHERE audio_path IS NOT NULL AND started_at < ?", - new String[]{String.valueOf(recordingCutoffMs)})) { - while (c.moveToNext()) { - if (!c.isNull(1)) { - paths.add(c.getString(1)); - } - } - } - for (String path : paths) { - if (new File(path).delete()) { - recordingsDeleted++; - } - } - db.execSQL("UPDATE sessions SET audio_path=NULL, audio_codec=NULL WHERE started_at < ?", - new Object[]{recordingCutoffMs}); - } - if (captionCutoffMs > 0) { - List paths = new ArrayList<>(); - try (Cursor c = db.rawQuery( - "SELECT audio_path FROM sessions WHERE audio_path IS NOT NULL AND started_at < ?", - new String[]{String.valueOf(captionCutoffMs)})) { - while (c.moveToNext()) { - paths.add(c.getString(0)); - } - } - for (String path : paths) { - if (path != null && new File(path).delete()) { - recordingsDeleted++; - } - } - db.execSQL("DELETE FROM segments WHERE session_id IN (SELECT id FROM sessions WHERE started_at < ?)", - new Object[]{captionCutoffMs}); - Cursor count = db.rawQuery("SELECT changes()", null); - count.close(); - db.execSQL("DELETE FROM sessions WHERE started_at < ?", new Object[]{captionCutoffMs}); - try (Cursor c = db.rawQuery("SELECT changes()", null)) { - if (c.moveToFirst()) { - sessionsDeleted = c.getInt(0); - } - } - } - try { - JSONObject summary = new JSONObject(); - summary.put("sessionsDeleted", sessionsDeleted); - summary.put("recordingsDeleted", recordingsDeleted); - return summary.toString(); - } catch (JSONException e) { - return "{}"; - } - } - - public void deleteSession(long sessionId) { - SQLiteDatabase db = getWritableDatabase(); - String path = getSessionAudioPath(sessionId); - if (!path.isEmpty()) { - new File(path).delete(); - } - db.delete("segments", "session_id=?", new String[]{String.valueOf(sessionId)}); - db.delete("sessions", "id=?", new String[]{String.valueOf(sessionId)}); - } - - // ---- blob helpers ---- - - private static float[] blobToFloats(byte[] blob) { - if (blob == null || blob.length < 4 || blob.length % 4 != 0) { - return null; - } - ByteBuffer buffer = ByteBuffer.wrap(blob).order(ByteOrder.LITTLE_ENDIAN); - float[] out = new float[blob.length / 4]; - for (int i = 0; i < out.length; i++) { - out[i] = buffer.getFloat(); - } - return out; - } - - private static byte[] floatsToBlob(float[] values) { - ByteBuffer buffer = ByteBuffer.allocate(values.length * 4).order(ByteOrder.LITTLE_ENDIAN); - for (float v : values) { - buffer.putFloat(v); - } - return buffer.array(); - } - - private static void l2NormalizeInPlace(float[] v) { - double sum = 0; - for (float x : v) { - sum += (double) x * x; - } - double norm = Math.sqrt(sum); - if (norm <= 0) { - return; - } - for (int i = 0; i < v.length; i++) { - v[i] = (float) (v[i] / norm); - } - } -} diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawConversationStore.kt b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawConversationStore.kt new file mode 100644 index 00000000..d9002ede --- /dev/null +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawConversationStore.kt @@ -0,0 +1,790 @@ +package com.faceclaw.app + +import android.content.ContentValues +import android.content.Context +import android.database.Cursor +import android.database.sqlite.SQLiteDatabase +import android.database.sqlite.SQLiteOpenHelper +import android.util.Base64 +import android.util.Log + +import org.json.JSONArray +import org.json.JSONException +import org.json.JSONObject + +import java.io.File +import java.nio.ByteBuffer +import java.nio.ByteOrder +import java.util.ArrayList + +/** + * Persistent store for caption sessions, transcript segments, and speaker + * voice profiles, backing the Microphones app and its phone-side review UI. + * All complex values cross the JS bridge as JSON strings; voice-print + * embeddings are float32 LE blobs, exposed as base64. + */ +class FaceclawConversationStore private constructor(context: Context) : SQLiteOpenHelper(context, DB_NAME, null, DB_VERSION) { + companion object { + private const val TAG = "FaceclawConvStore" + private const val DB_NAME = "faceclaw-conversations.db" + private const val DB_VERSION = 2 + + private var instance: FaceclawConversationStore? = null + + @JvmStatic + @Synchronized + fun getInstance(context: Context): FaceclawConversationStore { + if (instance == null) { + instance = FaceclawConversationStore(context.applicationContext) + } + return instance!! + } + + @Throws(JSONException::class) + private fun putOptString(values: ContentValues, s: JSONObject, jsonKey: String, column: String) { + if (s.has(jsonKey) && !s.isNull(jsonKey)) { + values.put(column, s.getString(jsonKey)) + } + } + + // ---- blob helpers ---- + + private fun blobToFloats(blob: ByteArray?): FloatArray? { + if (blob == null || blob.size < 4 || blob.size % 4 != 0) { + return null + } + val buffer = ByteBuffer.wrap(blob).order(ByteOrder.LITTLE_ENDIAN) + val out = FloatArray(blob.size / 4) + for (i in out.indices) { + out[i] = buffer.getFloat() + } + return out + } + + private fun floatsToBlob(values: FloatArray): ByteArray { + val buffer = ByteBuffer.allocate(values.size * 4).order(ByteOrder.LITTLE_ENDIAN) + for (v in values) { + buffer.putFloat(v) + } + return buffer.array() + } + + private fun l2NormalizeInPlace(v: FloatArray) { + var sum = 0.0 + for (x in v) { + sum += x.toDouble() * x + } + val norm = Math.sqrt(sum) + if (norm <= 0) { + return + } + for (i in v.indices) { + v[i] = (v[i] / norm).toFloat() + } + } + } + + override fun onCreate(db: SQLiteDatabase) { + db.execSQL("CREATE TABLE speakers (" + + "id INTEGER PRIMARY KEY AUTOINCREMENT," + + "name TEXT NOT NULL," + + "color TEXT NOT NULL DEFAULT '#4FC3F7'," + + "is_wearer INTEGER NOT NULL DEFAULT 0," + + "embedding BLOB," + + "embedding_count INTEGER NOT NULL DEFAULT 0," + + "created_at INTEGER NOT NULL," + + "last_heard_at INTEGER," + + "tag TEXT," + + "last_recap TEXT," + + "action_items TEXT," + + "facts TEXT," + + "insights_updated_at INTEGER," + + "insights_session_id INTEGER)") + db.execSQL("CREATE TABLE sessions (" + + "id INTEGER PRIMARY KEY AUTOINCREMENT," + + "started_at INTEGER NOT NULL," + + "ended_at INTEGER," + + "title TEXT," + + "audio_path TEXT," + + "audio_codec TEXT," + + "avg_sentiment REAL," + + "segment_count INTEGER NOT NULL DEFAULT 0)") + db.execSQL("CREATE TABLE segments (" + + "id INTEGER PRIMARY KEY AUTOINCREMENT," + + "session_id INTEGER NOT NULL," + + "speaker_id INTEGER," + + "started_at INTEGER NOT NULL," + + "ended_at INTEGER NOT NULL," + + "audio_offset_ms INTEGER," + + "text TEXT NOT NULL," + + "lang TEXT," + + "translation TEXT," + + "translation_lang TEXT," + + "sentiment REAL," + + "emotion TEXT," + + "search_meta TEXT," + + "angle INTEGER," + + "embedding BLOB)") + db.execSQL("CREATE INDEX idx_segments_session ON segments(session_id)") + db.execSQL("CREATE INDEX idx_segments_speaker ON segments(speaker_id)") + db.execSQL("CREATE INDEX idx_segments_time ON segments(started_at)") + db.execSQL("CREATE INDEX idx_sessions_time ON sessions(started_at)") + } + + override fun onUpgrade(db: SQLiteDatabase, oldVersion: Int, newVersion: Int) { + if (oldVersion < 2) { + // v2: per-speaker conversation insights (recap of the most recent + // conversation, its action items, and accumulated inferred facts). + db.execSQL("ALTER TABLE speakers ADD COLUMN last_recap TEXT") + db.execSQL("ALTER TABLE speakers ADD COLUMN action_items TEXT") + db.execSQL("ALTER TABLE speakers ADD COLUMN facts TEXT") + db.execSQL("ALTER TABLE speakers ADD COLUMN insights_updated_at INTEGER") + db.execSQL("ALTER TABLE speakers ADD COLUMN insights_session_id INTEGER") + } + } + + // ---- sessions ---- + + fun startSession(startedAtMs: Long, title: String?): Long { + val values = ContentValues() + values.put("started_at", startedAtMs) + values.put("title", if (title == null || title.isEmpty()) null else title) + return writableDatabase.insert("sessions", null, values) + } + + fun endSession(sessionId: Long, endedAtMs: Long, audioPath: String?, audioCodec: String?, avgSentiment: Double) { + val values = ContentValues() + values.put("ended_at", endedAtMs) + if (audioPath != null && !audioPath.isEmpty()) { + values.put("audio_path", audioPath) + values.put("audio_codec", audioCodec) + } + values.put("avg_sentiment", avgSentiment) + writableDatabase.update("sessions", values, "id=?", arrayOf(sessionId.toString())) + } + + fun setSessionAudio(sessionId: Long, audioPath: String?, audioCodec: String?) { + val values = ContentValues() + values.put("audio_path", audioPath) + values.put("audio_codec", audioCodec) + writableDatabase.update("sessions", values, "id=?", arrayOf(sessionId.toString())) + } + + fun getSessionAudioPath(sessionId: Long): String { + readableDatabase.rawQuery( + "SELECT audio_path FROM sessions WHERE id=?", arrayOf(sessionId.toString())).use { c -> + if (c.moveToFirst()) { + return if (c.isNull(0)) "" else c.getString(0) + } + } + return "" + } + + /** + * Sessions, newest first, as JSON. filterJson supports: sinceMs, untilMs, + * speakerId, emotion (matches any segment emotion in the session), query + * (LIKE over title and segment text/search metadata), limit. + */ + fun querySessions(filterJson: String?): String { + try { + val filter = if (filterJson == null || filterJson.isEmpty()) + JSONObject() else JSONObject(filterJson) + val sql = StringBuilder( + "SELECT DISTINCT s.id, s.started_at, s.ended_at, s.title, s.audio_path, s.audio_codec," + + " s.avg_sentiment, s.segment_count FROM sessions s") + val args: MutableList = ArrayList() + val where: MutableList = ArrayList() + val joinSegments = filter.has("speakerId") || filter.has("emotion") || filter.has("query") + if (joinSegments) { + sql.append(" LEFT JOIN segments g ON g.session_id = s.id") + } + if (filter.has("sinceMs")) { + where.add("s.started_at >= ?") + args.add(filter.getLong("sinceMs").toString()) + } + if (filter.has("untilMs")) { + where.add("s.started_at <= ?") + args.add(filter.getLong("untilMs").toString()) + } + if (filter.has("speakerId")) { + where.add("g.speaker_id = ?") + args.add(filter.getLong("speakerId").toString()) + } + if (filter.has("emotion")) { + where.add("g.emotion = ?") + args.add(filter.getString("emotion")) + } + if (filter.has("query")) { + val like = "%" + filter.getString("query") + "%" + where.add("(s.title LIKE ? OR g.text LIKE ? OR g.search_meta LIKE ?)") + args.add(like) + args.add(like) + args.add(like) + } + if (!where.isEmpty()) { + sql.append(" WHERE ").append(where.joinToString(" AND ")) + } + sql.append(" ORDER BY s.started_at DESC") + val limit = filter.optInt("limit", 200) + sql.append(" LIMIT ").append(Math.max(1, Math.min(limit, 1000))) + + val out = JSONArray() + readableDatabase.rawQuery(sql.toString(), args.toTypedArray()).use { c -> + while (c.moveToNext()) { + val row = JSONObject() + val id = c.getLong(0) + row.put("id", id) + row.put("startedAt", c.getLong(1)) + row.put("endedAt", if (c.isNull(2)) JSONObject.NULL else c.getLong(2)) + row.put("title", if (c.isNull(3)) "" else c.getString(3)) + row.put("audioPath", if (c.isNull(4)) "" else c.getString(4)) + row.put("audioCodec", if (c.isNull(5)) "" else c.getString(5)) + row.put("avgSentiment", if (c.isNull(6)) 0.0 else c.getDouble(6)) + row.put("segmentCount", c.getLong(7)) + row.put("speakers", sessionSpeakersJson(id)) + out.put(row) + } + } + return out.toString() + } catch (e: JSONException) { + Log.w(TAG, "querySessions failed", e) + return "[]" + } + } + + @Throws(JSONException::class) + private fun sessionSpeakersJson(sessionId: Long): JSONArray { + val out = JSONArray() + readableDatabase.rawQuery( + "SELECT DISTINCT p.id, p.name, p.color FROM segments g" + + " JOIN speakers p ON p.id = g.speaker_id WHERE g.session_id=?", + arrayOf(sessionId.toString())).use { c -> + while (c.moveToNext()) { + val row = JSONObject() + row.put("id", c.getLong(0)) + row.put("name", c.getString(1)) + row.put("color", c.getString(2)) + out.put(row) + } + } + return out + } + + // ---- segments ---- + + /** + * Insert one transcript segment. json fields: sessionId, speakerId?, + * startedAt, endedAt, audioOffsetMs?, text, lang?, translation?, + * translationLang?, sentiment?, emotion?, searchMeta?, angle?, + * embeddingBase64?. Returns the new row id, or -1. + */ + fun insertSegment(json: String?): Long { + try { + val s = JSONObject(json) + val values = ContentValues() + val sessionId = s.getLong("sessionId") + values.put("session_id", sessionId) + if (s.has("speakerId")) { + values.put("speaker_id", s.getLong("speakerId")) + } + values.put("started_at", s.getLong("startedAt")) + values.put("ended_at", s.getLong("endedAt")) + if (s.has("audioOffsetMs")) { + values.put("audio_offset_ms", s.getLong("audioOffsetMs")) + } + values.put("text", s.getString("text")) + putOptString(values, s, "lang", "lang") + putOptString(values, s, "translation", "translation") + putOptString(values, s, "translationLang", "translation_lang") + if (s.has("sentiment")) { + values.put("sentiment", s.getDouble("sentiment")) + } + putOptString(values, s, "emotion", "emotion") + putOptString(values, s, "searchMeta", "search_meta") + if (s.has("angle")) { + values.put("angle", s.getInt("angle")) + } + if (s.has("embeddingBase64")) { + values.put("embedding", Base64.decode(s.getString("embeddingBase64"), Base64.NO_WRAP)) + } + val db = writableDatabase + val id = db.insert("segments", null, values) + db.execSQL("UPDATE sessions SET segment_count = segment_count + 1 WHERE id=?", + arrayOf(sessionId)) + return id + } catch (t: Throwable) { + Log.w(TAG, "insertSegment failed", t) + return -1 + } + } + + /** Update mutable fields of a segment (same keys as insertSegment). */ + fun updateSegment(segmentId: Long, json: String?) { + try { + val s = JSONObject(json) + val values = ContentValues() + if (s.has("speakerId")) { + if (s.isNull("speakerId")) { + values.putNull("speaker_id") + } else { + values.put("speaker_id", s.getLong("speakerId")) + } + } + if (s.has("text")) { + values.put("text", s.getString("text")) + } + putOptString(values, s, "lang", "lang") + putOptString(values, s, "translation", "translation") + putOptString(values, s, "translationLang", "translation_lang") + if (s.has("sentiment")) { + values.put("sentiment", s.getDouble("sentiment")) + } + putOptString(values, s, "emotion", "emotion") + putOptString(values, s, "searchMeta", "search_meta") + if (values.size() > 0) { + writableDatabase.update("segments", values, "id=?", + arrayOf(segmentId.toString())) + } + } catch (t: Throwable) { + Log.w(TAG, "updateSegment failed", t) + } + } + + fun querySegments(sessionId: Long): String { + val out = JSONArray() + try { + readableDatabase.rawQuery( + "SELECT id, speaker_id, started_at, ended_at, audio_offset_ms, text, lang," + + " translation, translation_lang, sentiment, emotion, angle" + + " FROM segments WHERE session_id=? ORDER BY started_at ASC", + arrayOf(sessionId.toString())).use { c -> + while (c.moveToNext()) { + out.put(segmentRowJson(c)) + } + } + } catch (e: JSONException) { + Log.w(TAG, "querySegments failed", e) + } + return out.toString() + } + + @Throws(JSONException::class) + private fun segmentRowJson(c: Cursor): JSONObject { + val row = JSONObject() + row.put("id", c.getLong(0)) + row.put("speakerId", if (c.isNull(1)) JSONObject.NULL else c.getLong(1)) + row.put("startedAt", c.getLong(2)) + row.put("endedAt", c.getLong(3)) + row.put("audioOffsetMs", if (c.isNull(4)) JSONObject.NULL else c.getLong(4)) + row.put("text", c.getString(5)) + row.put("lang", if (c.isNull(6)) "" else c.getString(6)) + row.put("translation", if (c.isNull(7)) "" else c.getString(7)) + row.put("translationLang", if (c.isNull(8)) "" else c.getString(8)) + row.put("sentiment", if (c.isNull(9)) 0.0 else c.getDouble(9)) + row.put("emotion", if (c.isNull(10)) "" else c.getString(10)) + row.put("angle", if (c.isNull(11)) JSONObject.NULL else c.getInt(11)) + return row + } + + /** + * Search segments across sessions. filterJson: query?, emotion?, + * speakerId?, sinceMs?, untilMs?, limit?. Matches text, translation, and + * the sentiment search metadata. + */ + fun searchSegments(filterJson: String?): String { + try { + val filter = if (filterJson == null || filterJson.isEmpty()) + JSONObject() else JSONObject(filterJson) + val sql = StringBuilder( + "SELECT id, speaker_id, started_at, ended_at, audio_offset_ms, text, lang," + + " translation, translation_lang, sentiment, emotion, angle, session_id" + + " FROM segments") + val args: MutableList = ArrayList() + val where: MutableList = ArrayList() + if (filter.has("query") && !filter.getString("query").isEmpty()) { + val like = "%" + filter.getString("query") + "%" + where.add("(text LIKE ? OR translation LIKE ? OR search_meta LIKE ?)") + args.add(like) + args.add(like) + args.add(like) + } + if (filter.has("emotion")) { + where.add("emotion = ?") + args.add(filter.getString("emotion")) + } + if (filter.has("speakerId")) { + where.add("speaker_id = ?") + args.add(filter.getLong("speakerId").toString()) + } + if (filter.has("sinceMs")) { + where.add("started_at >= ?") + args.add(filter.getLong("sinceMs").toString()) + } + if (filter.has("untilMs")) { + where.add("started_at <= ?") + args.add(filter.getLong("untilMs").toString()) + } + if (!where.isEmpty()) { + sql.append(" WHERE ").append(where.joinToString(" AND ")) + } + sql.append(" ORDER BY started_at DESC LIMIT ") + .append(Math.max(1, Math.min(filter.optInt("limit", 300), 1000))) + val out = JSONArray() + readableDatabase.rawQuery(sql.toString(), args.toTypedArray()).use { c -> + while (c.moveToNext()) { + val row = segmentRowJson(c) + row.put("sessionId", c.getLong(12)) + out.put(row) + } + } + return out.toString() + } catch (e: JSONException) { + Log.w(TAG, "searchSegments failed", e) + return "[]" + } + } + + /** Per-segment voice-print embeddings of a session (for re-diarization). */ + fun querySegmentEmbeddings(sessionId: Long): String { + val out = JSONArray() + try { + readableDatabase.rawQuery( + "SELECT id, embedding FROM segments WHERE session_id=? AND embedding IS NOT NULL", + arrayOf(sessionId.toString())).use { c -> + while (c.moveToNext()) { + val row = JSONObject() + row.put("id", c.getLong(0)) + row.put("embeddingBase64", Base64.encodeToString(c.getBlob(1), Base64.NO_WRAP)) + out.put(row) + } + } + } catch (e: JSONException) { + Log.w(TAG, "querySegmentEmbeddings failed", e) + } + return out.toString() + } + + // ---- speakers ---- + + fun createSpeaker(name: String?, color: String?, isWearer: Boolean, embeddingBase64: String?): Long { + val values = ContentValues() + values.put("name", name) + values.put("color", if (color == null || color.isEmpty()) "#4FC3F7" else color) + values.put("is_wearer", if (isWearer) 1 else 0) + values.put("created_at", System.currentTimeMillis()) + values.put("last_heard_at", System.currentTimeMillis()) + if (embeddingBase64 != null && !embeddingBase64.isEmpty()) { + values.put("embedding", Base64.decode(embeddingBase64, Base64.NO_WRAP)) + values.put("embedding_count", 1) + } + return writableDatabase.insert("speakers", null, values) + } + + fun querySpeakers(): String { + val out = JSONArray() + try { + readableDatabase.rawQuery( + "SELECT p.id, p.name, p.color, p.is_wearer, p.embedding, p.embedding_count," + + " p.last_heard_at, p.tag," + + " p.last_recap, p.action_items, p.facts, p.insights_updated_at, p.insights_session_id," + + " (SELECT COUNT(*) FROM segments g WHERE g.speaker_id = p.id) AS segments" + + " FROM speakers p ORDER BY p.last_heard_at DESC", null).use { c -> + while (c.moveToNext()) { + val row = JSONObject() + row.put("id", c.getLong(0)) + row.put("name", c.getString(1)) + row.put("color", c.getString(2)) + row.put("isWearer", c.getInt(3) != 0) + val blob = if (c.isNull(4)) null else c.getBlob(4) + row.put("embeddingBase64", if (blob == null) "" else Base64.encodeToString(blob, Base64.NO_WRAP)) + row.put("embeddingCount", c.getInt(5)) + row.put("lastHeardAt", if (c.isNull(6)) 0L else c.getLong(6)) + row.put("tag", if (c.isNull(7)) "" else c.getString(7)) + row.put("lastRecap", if (c.isNull(8)) "" else c.getString(8)) + row.put("actionItemsJson", if (c.isNull(9)) "" else c.getString(9)) + row.put("factsJson", if (c.isNull(10)) "" else c.getString(10)) + row.put("insightsUpdatedAt", if (c.isNull(11)) 0L else c.getLong(11)) + row.put("insightsSessionId", if (c.isNull(12)) 0L else c.getLong(12)) + row.put("segmentCount", c.getLong(13)) + out.put(row) + } + } + } catch (e: JSONException) { + Log.w(TAG, "querySpeakers failed", e) + } + return out.toString() + } + + fun renameSpeaker(speakerId: Long, name: String?) { + val values = ContentValues() + values.put("name", name) + writableDatabase.update("speakers", values, "id=?", arrayOf(speakerId.toString())) + } + + fun setSpeakerColor(speakerId: Long, color: String?) { + val values = ContentValues() + values.put("color", color) + writableDatabase.update("speakers", values, "id=?", arrayOf(speakerId.toString())) + } + + fun setSpeakerTag(speakerId: Long, tag: String?) { + val values = ContentValues() + values.put("tag", tag) + writableDatabase.update("speakers", values, "id=?", arrayOf(speakerId.toString())) + } + + /** + * Store LLM-derived conversation insights for a speaker: a recap of their + * most recent conversation, its open action items (JSON string array), and + * the accumulated inferred facts about them (JSON string array). + */ + fun setSpeakerInsights(speakerId: Long, recap: String?, actionItemsJson: String?, + factsJson: String?, sessionId: Long) { + val values = ContentValues() + values.put("last_recap", recap ?: "") + values.put("action_items", actionItemsJson ?: "") + values.put("facts", factsJson ?: "") + values.put("insights_updated_at", System.currentTimeMillis()) + values.put("insights_session_id", sessionId) + writableDatabase.update("speakers", values, "id=?", arrayOf(speakerId.toString())) + } + + fun setSpeakerWearer(speakerId: Long, isWearer: Boolean) { + val db = writableDatabase + if (isWearer) { + // Exactly one wearer profile at a time. + db.execSQL("UPDATE speakers SET is_wearer = 0") + } + val values = ContentValues() + values.put("is_wearer", if (isWearer) 1 else 0) + db.update("speakers", values, "id=?", arrayOf(speakerId.toString())) + } + + /** + * Running-mean centroid update for a speaker's voice-print, capped so a + * long session cannot drown the enrolled voice. + */ + fun updateSpeakerEmbedding(speakerId: Long, embeddingBase64: String?, maxCount: Int) { + try { + val incoming = Base64.decode(embeddingBase64, Base64.NO_WRAP) + val update = blobToFloats(incoming) + if (update == null) { + return + } + var current: FloatArray? = null + var count = 0 + val c = readableDatabase.rawQuery( + "SELECT embedding, embedding_count FROM speakers WHERE id=?", + arrayOf(speakerId.toString())) + try { + if (c.moveToFirst()) { + current = if (c.isNull(0)) null else blobToFloats(c.getBlob(0)) + count = c.getInt(1) + } + } finally { + c.close() + } + val merged: FloatArray + if (current == null || current.size != update.size) { + merged = update + count = 1 + } else { + val effective = Math.min(count, Math.max(1, maxCount)) + merged = FloatArray(current.size) + for (i in merged.indices) { + merged[i] = (current[i] * effective + update[i]) / (effective + 1) + } + l2NormalizeInPlace(merged) + count = count + 1 + } + val values = ContentValues() + values.put("embedding", floatsToBlob(merged)) + values.put("embedding_count", count) + values.put("last_heard_at", System.currentTimeMillis()) + writableDatabase.update("speakers", values, "id=?", + arrayOf(speakerId.toString())) + } catch (t: Throwable) { + Log.w(TAG, "updateSpeakerEmbedding failed", t) + } + } + + /** Move every segment from one speaker onto another and drop the source. */ + fun mergeSpeakers(fromId: Long, intoId: Long) { + if (fromId == intoId) { + return + } + val db = writableDatabase + db.beginTransaction() + try { + db.execSQL("UPDATE segments SET speaker_id=? WHERE speaker_id=?", + arrayOf(intoId, fromId)) + // Blend the two centroids weighted by their sample counts. + var from: FloatArray? = null + var into: FloatArray? = null + var fromCount = 0 + var intoCount = 0 + var fromWearer = false + val c = db.rawQuery( + "SELECT id, embedding, embedding_count, is_wearer FROM speakers WHERE id IN (?,?)", + arrayOf(fromId.toString(), intoId.toString())) + try { + while (c.moveToNext()) { + val id = c.getLong(0) + val embedding = if (c.isNull(1)) null else blobToFloats(c.getBlob(1)) + val count = c.getInt(2) + if (id == fromId) { + from = embedding + fromCount = count + fromWearer = c.getInt(3) != 0 + } else { + into = embedding + intoCount = count + } + } + } finally { + c.close() + } + var merged: FloatArray? = into + if (from != null && into != null && from.size == into.size) { + val blended = FloatArray(into.size) + val total = Math.max(1, fromCount + intoCount) + for (i in blended.indices) { + blended[i] = (into[i] * intoCount + from[i] * fromCount) / total + } + l2NormalizeInPlace(blended) + merged = blended + } else if (into == null) { + merged = from + } + val values = ContentValues() + if (merged != null) { + values.put("embedding", floatsToBlob(merged)) + values.put("embedding_count", fromCount + intoCount) + } + if (fromWearer) { + values.put("is_wearer", 1) + } + values.put("last_heard_at", System.currentTimeMillis()) + // Keep the target's conversation insights; adopt the source's only + // where the target has none (e.g. merging a rich profile into a + // freshly auto-created one). + db.rawQuery( + "SELECT last_recap, action_items, facts, insights_updated_at, insights_session_id" + + " FROM speakers WHERE id IN (?,?) ORDER BY CASE id WHEN ? THEN 0 ELSE 1 END", + arrayOf(fromId.toString(), intoId.toString(), intoId.toString())).use { c2 -> + var targetHasInsights = false + while (c2.moveToNext()) { + val hasInsights = !c2.isNull(0) && !c2.getString(0).isEmpty() + if (c2.position == 0) { + targetHasInsights = hasInsights + } else if (!targetHasInsights && hasInsights) { + values.put("last_recap", c2.getString(0)) + values.put("action_items", if (c2.isNull(1)) "" else c2.getString(1)) + values.put("facts", if (c2.isNull(2)) "" else c2.getString(2)) + values.put("insights_updated_at", if (c2.isNull(3)) 0L else c2.getLong(3)) + values.put("insights_session_id", if (c2.isNull(4)) 0L else c2.getLong(4)) + } + } + } + db.update("speakers", values, "id=?", arrayOf(intoId.toString())) + db.delete("speakers", "id=?", arrayOf(fromId.toString())) + db.setTransactionSuccessful() + } finally { + db.endTransaction() + } + } + + fun reassignSegmentSpeaker(segmentId: Long, speakerId: Long) { + val values = ContentValues() + values.put("speaker_id", speakerId) + writableDatabase.update("segments", values, "id=?", arrayOf(segmentId.toString())) + } + + /** Reassign every segment of a speaker within one session (split support). */ + fun reassignSessionSpeaker(sessionId: Long, fromSpeakerId: Long, toSpeakerId: Long) { + writableDatabase.execSQL( + "UPDATE segments SET speaker_id=? WHERE session_id=? AND speaker_id=?", + arrayOf(toSpeakerId, sessionId, fromSpeakerId)) + } + + fun deleteSpeaker(speakerId: Long) { + val db = writableDatabase + db.execSQL("UPDATE segments SET speaker_id=NULL WHERE speaker_id=?", arrayOf(speakerId)) + db.delete("speakers", "id=?", arrayOf(speakerId.toString())) + } + + // ---- retention ---- + + /** + * Delete caption sessions older than the cutoff (0 disables), and strip + * recordings older than their own cutoff while keeping the transcript. + * Returns a JSON summary of what was removed. + */ + fun applyRetention(captionCutoffMs: Long, recordingCutoffMs: Long): String { + val db = writableDatabase + var sessionsDeleted = 0 + var recordingsDeleted = 0 + if (recordingCutoffMs > 0) { + val paths: MutableList = ArrayList() + db.rawQuery( + "SELECT id, audio_path FROM sessions WHERE audio_path IS NOT NULL AND started_at < ?", + arrayOf(recordingCutoffMs.toString())).use { c -> + while (c.moveToNext()) { + if (!c.isNull(1)) { + paths.add(c.getString(1)) + } + } + } + for (path in paths) { + if (File(path).delete()) { + recordingsDeleted++ + } + } + db.execSQL("UPDATE sessions SET audio_path=NULL, audio_codec=NULL WHERE started_at < ?", + arrayOf(recordingCutoffMs)) + } + if (captionCutoffMs > 0) { + val paths: MutableList = ArrayList() + db.rawQuery( + "SELECT audio_path FROM sessions WHERE audio_path IS NOT NULL AND started_at < ?", + arrayOf(captionCutoffMs.toString())).use { c -> + while (c.moveToNext()) { + paths.add(c.getString(0)) + } + } + for (path in paths) { + if (path != null && File(path).delete()) { + recordingsDeleted++ + } + } + db.execSQL("DELETE FROM segments WHERE session_id IN (SELECT id FROM sessions WHERE started_at < ?)", + arrayOf(captionCutoffMs)) + val count = db.rawQuery("SELECT changes()", null) + count.close() + db.execSQL("DELETE FROM sessions WHERE started_at < ?", arrayOf(captionCutoffMs)) + db.rawQuery("SELECT changes()", null).use { c -> + if (c.moveToFirst()) { + sessionsDeleted = c.getInt(0) + } + } + } + try { + val summary = JSONObject() + summary.put("sessionsDeleted", sessionsDeleted) + summary.put("recordingsDeleted", recordingsDeleted) + return summary.toString() + } catch (e: JSONException) { + return "{}" + } + } + + fun deleteSession(sessionId: Long) { + val db = writableDatabase + val path = getSessionAudioPath(sessionId) + if (!path.isEmpty()) { + File(path).delete() + } + db.delete("segments", "session_id=?", arrayOf(sessionId.toString())) + db.delete("sessions", "id=?", arrayOf(sessionId.toString())) + } +} diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawDeviceDiscovery.java b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawDeviceDiscovery.java deleted file mode 100644 index 53645f27..00000000 --- a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawDeviceDiscovery.java +++ /dev/null @@ -1,386 +0,0 @@ -package com.faceclaw.app; - -import android.annotation.SuppressLint; -import android.bluetooth.BluetoothAdapter; -import android.bluetooth.BluetoothDevice; -import android.bluetooth.BluetoothManager; -import android.bluetooth.le.BluetoothLeScanner; -import android.bluetooth.le.ScanCallback; -import android.bluetooth.le.ScanRecord; -import android.bluetooth.le.ScanResult; -import android.bluetooth.le.ScanSettings; -import android.content.Context; -import android.os.Build; -import android.os.Handler; -import android.os.Looper; -import android.util.SparseArray; - -import org.json.JSONArray; -import org.json.JSONException; -import org.json.JSONObject; - -import java.util.ArrayList; -import java.util.List; -import java.util.Set; - -/** - * Finds Even Realities hardware over BLE and hands every advertisement up to - * TypeScript with the raw fields the pairing screen needs: the LIVE local name - * from the scan record (BluetoothDevice.getName() is a cache that can hold a - * pre-reset value indefinitely), the full manufacturer-specific payload with - * its company identifier restored so the "ER"+serial+MAC layout survives - * intact, RSSI, advertised TX power, and connectability. - * - * Admission is deliberately loose — a name containing G2, the stock - * "EVEN R1…" ring prefix, or the Even "ER" manufacturer signature — because - * renamed custom firmware may drop the stock name. Side, serial, and pair - * matching are decided in TypeScript (app/g2/even-advertisement.ts, - * app/g2/pairing-candidates.ts) where they can be unit-tested; this filter - * must stay a superset of the TS classifier's admission. - */ -@SuppressLint("MissingPermission") -public class FaceclawDeviceDiscovery { - /** Even Realities' company identifier: the ASCII bytes "ER" read little-endian. */ - public static final int EVEN_COMPANY_ID = 0x5245; - - private final BluetoothAdapter bluetoothAdapter; - private final Handler mainHandler = new Handler(Looper.getMainLooper()); - - private final Object lock = new Object(); - private ScanCallback activeCallback; - private volatile FaceclawDeviceDiscoveryListener listener; - - public FaceclawDeviceDiscovery(Context context) { - Context appContext = context.getApplicationContext(); - BluetoothManager bluetoothManager = - (BluetoothManager) appContext.getSystemService(Context.BLUETOOTH_SERVICE); - if (bluetoothManager == null || bluetoothManager.getAdapter() == null) { - throw new IllegalStateException("Bluetooth adapter unavailable"); - } - this.bluetoothAdapter = bluetoothManager.getAdapter(); - } - - public boolean isBluetoothEnabled() { - return bluetoothAdapter.isEnabled(); - } - - public void setListener(FaceclawDeviceDiscoveryListener newListener) { - this.listener = newListener; - } - - // ------------------------------------------------------------------ - // Live scan - // ------------------------------------------------------------------ - - /** - * Start streaming advertisements to the listener. Returns false when the - * radio is off or the scanner is unavailable; a later platform refusal - * arrives through onScanFailed. Calling while a scan is running is a no-op. - */ - public boolean startScan() { - synchronized (lock) { - if (activeCallback != null) { - return true; - } - if (!bluetoothAdapter.isEnabled()) { - emitLog("scan not started: Bluetooth is off"); - return false; - } - BluetoothLeScanner scanner = bluetoothAdapter.getBluetoothLeScanner(); - if (scanner == null) { - emitLog("scan not started: no BLE scanner"); - return false; - } - final ScanCallback callback = new ScanCallback() { - @Override - public void onScanResult(int callbackType, ScanResult result) { - if (result != null) { - emitAdvertisement(result); - } - } - - @Override - public void onBatchScanResults(List results) { - if (results == null) { - return; - } - for (ScanResult result : results) { - if (result != null) { - emitAdvertisement(result); - } - } - } - - @Override - public void onScanFailed(int errorCode) { - synchronized (lock) { - if (activeCallback == this) { - activeCallback = null; - } - } - final FaceclawDeviceDiscoveryListener l = listener; - if (l != null) { - final String message = describeScanError(errorCode); - mainHandler.post(() -> l.onScanFailed(errorCode, message)); - } - } - }; - activeCallback = callback; - try { - scanner.startScan(null, buildScanSettings(), callback); - } catch (Throwable t) { - activeCallback = null; - emitLog("scan not started: " + t); - return false; - } - emitLog("scan started (unfiltered, low latency)"); - return true; - } - } - - public void stopScan() { - ScanCallback callback; - synchronized (lock) { - callback = activeCallback; - activeCallback = null; - } - if (callback == null) { - return; - } - BluetoothLeScanner scanner = bluetoothAdapter.getBluetoothLeScanner(); - if (scanner != null) { - try { - scanner.stopScan(callback); - } catch (Throwable ignored) { - } - } - emitLog("scan stopped"); - } - - public boolean isScanning() { - synchronized (lock) { - return activeCallback != null; - } - } - - /** - * Replay bonded Even devices to the listener. A bonded listing has no - * signal sample and no manufacturer data, so its serial stays unknown until - * the device is also heard advertising. - */ - public void emitBondedDevices() { - final FaceclawDeviceDiscoveryListener l = listener; - if (l == null) { - return; - } - for (String json : bondedCandidateJsonList()) { - l.onAdvertisement(json); - } - } - - /** Bonded Even devices as one JSON array, for callers without a listener. */ - public String getBondedCandidatesJson() { - JSONArray array = new JSONArray(); - for (String json : bondedCandidateJsonList()) { - try { - array.put(new JSONObject(json)); - } catch (JSONException ignored) { - } - } - return array.toString(); - } - - // ------------------------------------------------------------------ - // Internals - // ------------------------------------------------------------------ - - private static ScanSettings buildScanSettings() { - ScanSettings.Builder builder = new ScanSettings.Builder() - .setScanMode(ScanSettings.SCAN_MODE_LOW_LATENCY) - .setReportDelay(0); - if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.M) { - builder.setCallbackType(ScanSettings.CALLBACK_TYPE_ALL_MATCHES); - builder.setMatchMode(ScanSettings.MATCH_MODE_AGGRESSIVE); - builder.setNumOfMatches(ScanSettings.MATCH_NUM_MAX_ADVERTISEMENT); - } - if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) { - // Accept extended advertising too; setLegacy(true) would hide it. - builder.setLegacy(false); - } - return builder.build(); - } - - private void emitAdvertisement(ScanResult result) { - final JSONObject object = toJson(result); - if (object == null) { - return; - } - final FaceclawDeviceDiscoveryListener l = listener; - if (l == null) { - return; - } - final String json = object.toString(); - if (Looper.myLooper() == Looper.getMainLooper()) { - l.onAdvertisement(json); - } else { - mainHandler.post(() -> { - FaceclawDeviceDiscoveryListener current = listener; - if (current != null) { - current.onAdvertisement(json); - } - }); - } - } - - private List bondedCandidateJsonList() { - List out = new ArrayList<>(); - Set bonded = bluetoothAdapter.getBondedDevices(); - if (bonded == null) { - return out; - } - long now = System.currentTimeMillis(); - for (BluetoothDevice device : bonded) { - if (device == null) { - continue; - } - String name = device.getName(); - if (!isAdmissible(name, null)) { - continue; - } - JSONObject object = buildJson( - device.getAddress(), name, "", null, null, null, true, "paired", now); - if (object != null) { - out.add(object.toString()); - } - } - return out; - } - - /** Null when the result is not something we pair with. */ - private static JSONObject toJson(ScanResult result) { - BluetoothDevice device = result.getDevice(); - if (device == null) { - return null; - } - ScanRecord record = result.getScanRecord(); - String liveName = record == null ? null : record.getDeviceName(); - String name = liveName != null && !liveName.isEmpty() ? liveName : device.getName(); - byte[] manufacturerData = extractEvenManufacturerData(record); - if (!isAdmissible(name, manufacturerData)) { - return null; - } - Integer txPower = null; - if (record != null && record.getTxPowerLevel() != Integer.MIN_VALUE) { - txPower = record.getTxPowerLevel(); - } - if (txPower == null && Build.VERSION.SDK_INT >= Build.VERSION_CODES.O - && result.getTxPower() != ScanResult.TX_POWER_NOT_PRESENT) { - txPower = result.getTxPower(); - } - Boolean connectable = null; - if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) { - connectable = result.isConnectable(); - } - int rssi = result.getRssi(); - // 127 is the "unavailable" sentinel; 0 dBm never happens for a real sample. - Integer rssiValue = (rssi == 127 || rssi == 0) ? null : rssi; - boolean bonded = device.getBondState() == BluetoothDevice.BOND_BONDED; - return buildJson( - device.getAddress(), name, FaceclawFirmwareUtil.bytesToHex(manufacturerData), rssiValue, txPower, - connectable, bonded, "scan", System.currentTimeMillis()); - } - - /** - * Android strips the 2-byte company identifier from each manufacturer - * record and keys the SparseArray by it. Put the bytes back (little-endian) - * so TypeScript sees the same "ER"+… layout the HCI captures - * document. Prefers Even's own id; otherwise returns the first record so an - * unexpected company id still reaches the diagnostics log. - */ - private static byte[] extractEvenManufacturerData(ScanRecord record) { - if (record == null) { - return new byte[0]; - } - SparseArray all = record.getManufacturerSpecificData(); - if (all == null || all.size() == 0) { - return new byte[0]; - } - byte[] payload = all.get(EVEN_COMPANY_ID); - int companyId = EVEN_COMPANY_ID; - if (payload == null) { - companyId = all.keyAt(0); - payload = all.valueAt(0); - } - if (payload == null) { - return new byte[0]; - } - byte[] out = new byte[payload.length + 2]; - out[0] = (byte) (companyId & 0xff); - out[1] = (byte) ((companyId >> 8) & 0xff); - System.arraycopy(payload, 0, out, 2, payload.length); - return out; - } - - private static boolean isAdmissible(String name, byte[] manufacturerData) { - if (manufacturerData != null && manufacturerData.length >= 2 - && manufacturerData[0] == 0x45 && manufacturerData[1] == 0x52) { - return true; - } - if (name == null) { - return false; - } - String upper = name.toUpperCase(); - // A bare "R1" substring also matches unrelated hardware ("Oppo Enco - // R1"), and TS only classifies rings from the stock name prefix. - return upper.contains("G2") || upper.startsWith("EVEN R1"); - } - - private static JSONObject buildJson( - String address, String name, String manufacturerHex, Integer rssi, Integer txPower, - Boolean connectable, boolean bonded, String source, long seenAtMs) { - if (address == null || address.isEmpty()) { - return null; - } - try { - JSONObject object = new JSONObject(); - object.put("address", address); - object.put("name", name == null ? "" : name); - object.put("manufacturerData", manufacturerHex == null ? "" : manufacturerHex); - object.put("rssi", rssi == null ? JSONObject.NULL : rssi); - object.put("txPower", txPower == null ? JSONObject.NULL : txPower); - object.put("connectable", connectable == null ? JSONObject.NULL : connectable); - object.put("bonded", bonded); - object.put("source", source); - object.put("seenAtMs", seenAtMs); - return object; - } catch (JSONException e) { - return null; - } - } - - private static String describeScanError(int errorCode) { - switch (errorCode) { - case ScanCallback.SCAN_FAILED_ALREADY_STARTED: - return "a scan is already running"; - case ScanCallback.SCAN_FAILED_APPLICATION_REGISTRATION_FAILED: - return "the Bluetooth stack refused to register the scan (try toggling Bluetooth)"; - case ScanCallback.SCAN_FAILED_FEATURE_UNSUPPORTED: - return "BLE scanning is not supported on this device"; - case ScanCallback.SCAN_FAILED_INTERNAL_ERROR: - return "internal Bluetooth error"; - case 5: // SCAN_FAILED_OUT_OF_HARDWARE_RESOURCES (API 31) - return "out of Bluetooth hardware resources"; - case 6: // SCAN_FAILED_SCANNING_TOO_FREQUENTLY (API 31) - return "scanning too frequently; wait a moment and try again"; - default: - return "scan failed (code " + errorCode + ")"; - } - } - - private void emitLog(String line) { - FaceclawDeviceDiscoveryListener l = listener; - if (l != null) { - l.onLog(line); - } - } -} diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawDeviceDiscovery.kt b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawDeviceDiscovery.kt new file mode 100644 index 00000000..b43bcaae --- /dev/null +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawDeviceDiscovery.kt @@ -0,0 +1,372 @@ +package com.faceclaw.app + +import android.annotation.SuppressLint +import android.bluetooth.BluetoothAdapter +import android.bluetooth.BluetoothDevice +import android.bluetooth.BluetoothManager +import android.bluetooth.le.BluetoothLeScanner +import android.bluetooth.le.ScanCallback +import android.bluetooth.le.ScanRecord +import android.bluetooth.le.ScanResult +import android.bluetooth.le.ScanSettings +import android.content.Context +import android.os.Build +import android.os.Handler +import android.os.Looper +import android.util.SparseArray + +import org.json.JSONArray +import org.json.JSONException +import org.json.JSONObject + +import java.util.ArrayList + +/** + * Finds Even Realities hardware over BLE and hands every advertisement up to + * TypeScript with the raw fields the pairing screen needs: the LIVE local name + * from the scan record (BluetoothDevice.getName() is a cache that can hold a + * pre-reset value indefinitely), the full manufacturer-specific payload with + * its company identifier restored so the "ER"+serial+MAC layout survives + * intact, RSSI, advertised TX power, and connectability. + * + * Admission is deliberately loose — a name containing G2, the stock + * "EVEN R1…" ring prefix, or the Even "ER" manufacturer signature — because + * renamed custom firmware may drop the stock name. Side, serial, and pair + * matching are decided in TypeScript (app/g2/even-advertisement.ts, + * app/g2/pairing-candidates.ts) where they can be unit-tested; this filter + * must stay a superset of the TS classifier's admission. + */ +@SuppressLint("MissingPermission") +class FaceclawDeviceDiscovery(context: Context) { + companion object { + /** Even Realities' company identifier: the ASCII bytes "ER" read little-endian. */ + const val EVEN_COMPANY_ID = 0x5245 + + // ------------------------------------------------------------------ + // Internals + // ------------------------------------------------------------------ + + private fun buildScanSettings(): ScanSettings { + val builder = ScanSettings.Builder() + .setScanMode(ScanSettings.SCAN_MODE_LOW_LATENCY) + .setReportDelay(0) + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.M) { + builder.setCallbackType(ScanSettings.CALLBACK_TYPE_ALL_MATCHES) + builder.setMatchMode(ScanSettings.MATCH_MODE_AGGRESSIVE) + builder.setNumOfMatches(ScanSettings.MATCH_NUM_MAX_ADVERTISEMENT) + } + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) { + // Accept extended advertising too; setLegacy(true) would hide it. + builder.setLegacy(false) + } + return builder.build() + } + + /** Null when the result is not something we pair with. */ + private fun toJson(result: ScanResult): JSONObject? { + val device = result.device ?: return null + val record = result.scanRecord + val liveName = record?.deviceName + val name = if (liveName != null && !liveName.isEmpty()) liveName else device.name + val manufacturerData = extractEvenManufacturerData(record) + if (!isAdmissible(name, manufacturerData)) { + return null + } + var txPower: Int? = null + if (record != null && record.txPowerLevel != Int.MIN_VALUE) { + txPower = record.txPowerLevel + } + if (txPower == null && Build.VERSION.SDK_INT >= Build.VERSION_CODES.O + && result.txPower != ScanResult.TX_POWER_NOT_PRESENT) { + txPower = result.txPower + } + var connectable: Boolean? = null + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) { + connectable = result.isConnectable + } + val rssi = result.rssi + // 127 is the "unavailable" sentinel; 0 dBm never happens for a real sample. + val rssiValue: Int? = if (rssi == 127 || rssi == 0) null else rssi + val bonded = device.bondState == BluetoothDevice.BOND_BONDED + return buildJson( + device.address, name, FaceclawFirmwareUtil.bytesToHex(manufacturerData), rssiValue, txPower, + connectable, bonded, "scan", System.currentTimeMillis()) + } + + /** + * Android strips the 2-byte company identifier from each manufacturer + * record and keys the SparseArray by it. Put the bytes back (little-endian) + * so TypeScript sees the same "ER"+… layout the HCI captures + * document. Prefers Even's own id; otherwise returns the first record so an + * unexpected company id still reaches the diagnostics log. + */ + private fun extractEvenManufacturerData(record: ScanRecord?): ByteArray { + if (record == null) { + return ByteArray(0) + } + val all: SparseArray? = record.manufacturerSpecificData + if (all == null || all.size() == 0) { + return ByteArray(0) + } + var payload: ByteArray? = all.get(EVEN_COMPANY_ID) + var companyId = EVEN_COMPANY_ID + if (payload == null) { + companyId = all.keyAt(0) + payload = all.valueAt(0) + } + if (payload == null) { + return ByteArray(0) + } + val out = ByteArray(payload.size + 2) + out[0] = (companyId and 0xff).toByte() + out[1] = ((companyId shr 8) and 0xff).toByte() + System.arraycopy(payload, 0, out, 2, payload.size) + return out + } + + private fun isAdmissible(name: String?, manufacturerData: ByteArray?): Boolean { + if (manufacturerData != null && manufacturerData.size >= 2 + && manufacturerData[0] == 0x45.toByte() && manufacturerData[1] == 0x52.toByte()) { + return true + } + if (name == null) { + return false + } + val upper = name.uppercase() + // A bare "R1" substring also matches unrelated hardware ("Oppo Enco + // R1"), and TS only classifies rings from the stock name prefix. + return upper.contains("G2") || upper.startsWith("EVEN R1") + } + + private fun buildJson( + address: String?, name: String?, manufacturerHex: String?, rssi: Int?, txPower: Int?, + connectable: Boolean?, bonded: Boolean, source: String, seenAtMs: Long): JSONObject? { + if (address == null || address.isEmpty()) { + return null + } + try { + val obj = JSONObject() + obj.put("address", address) + obj.put("name", name ?: "") + obj.put("manufacturerData", manufacturerHex ?: "") + obj.put("rssi", rssi ?: JSONObject.NULL) + obj.put("txPower", txPower ?: JSONObject.NULL) + obj.put("connectable", connectable ?: JSONObject.NULL) + obj.put("bonded", bonded) + obj.put("source", source) + obj.put("seenAtMs", seenAtMs) + return obj + } catch (e: JSONException) { + return null + } + } + + private fun describeScanError(errorCode: Int): String { + return when (errorCode) { + ScanCallback.SCAN_FAILED_ALREADY_STARTED -> + "a scan is already running" + ScanCallback.SCAN_FAILED_APPLICATION_REGISTRATION_FAILED -> + "the Bluetooth stack refused to register the scan (try toggling Bluetooth)" + ScanCallback.SCAN_FAILED_FEATURE_UNSUPPORTED -> + "BLE scanning is not supported on this device" + ScanCallback.SCAN_FAILED_INTERNAL_ERROR -> + "internal Bluetooth error" + 5 -> // SCAN_FAILED_OUT_OF_HARDWARE_RESOURCES (API 31) + "out of Bluetooth hardware resources" + 6 -> // SCAN_FAILED_SCANNING_TOO_FREQUENTLY (API 31) + "scanning too frequently; wait a moment and try again" + else -> + "scan failed (code $errorCode)" + } + } + } + + private val bluetoothAdapter: BluetoothAdapter + private val mainHandler = Handler(Looper.getMainLooper()) + + private val lock = Any() + private var activeCallback: ScanCallback? = null + @Volatile + private var listener: FaceclawDeviceDiscoveryListener? = null + + init { + val appContext = context.applicationContext + val bluetoothManager = + appContext.getSystemService(Context.BLUETOOTH_SERVICE) as BluetoothManager? + if (bluetoothManager == null || bluetoothManager.adapter == null) { + throw IllegalStateException("Bluetooth adapter unavailable") + } + this.bluetoothAdapter = bluetoothManager.adapter + } + + fun isBluetoothEnabled(): Boolean { + return bluetoothAdapter.isEnabled + } + + fun setListener(newListener: FaceclawDeviceDiscoveryListener?) { + this.listener = newListener + } + + // ------------------------------------------------------------------ + // Live scan + // ------------------------------------------------------------------ + + /** + * Start streaming advertisements to the listener. Returns false when the + * radio is off or the scanner is unavailable; a later platform refusal + * arrives through onScanFailed. Calling while a scan is running is a no-op. + */ + fun startScan(): Boolean { + synchronized(lock) { + if (activeCallback != null) { + return true + } + if (!bluetoothAdapter.isEnabled) { + emitLog("scan not started: Bluetooth is off") + return false + } + val scanner: BluetoothLeScanner? = bluetoothAdapter.bluetoothLeScanner + if (scanner == null) { + emitLog("scan not started: no BLE scanner") + return false + } + val callback = object : ScanCallback() { + override fun onScanResult(callbackType: Int, result: ScanResult?) { + if (result != null) { + emitAdvertisement(result) + } + } + + override fun onBatchScanResults(results: List?) { + if (results == null) { + return + } + for (result in results) { + if (result != null) { + emitAdvertisement(result) + } + } + } + + override fun onScanFailed(errorCode: Int) { + synchronized(lock) { + if (activeCallback === this) { + activeCallback = null + } + } + val l = listener + if (l != null) { + val message = describeScanError(errorCode) + mainHandler.post { l.onScanFailed(errorCode, message) } + } + } + } + activeCallback = callback + try { + scanner.startScan(null, buildScanSettings(), callback) + } catch (t: Throwable) { + activeCallback = null + emitLog("scan not started: $t") + return false + } + emitLog("scan started (unfiltered, low latency)") + return true + } + } + + fun stopScan() { + val callback: ScanCallback? + synchronized(lock) { + callback = activeCallback + activeCallback = null + } + if (callback == null) { + return + } + val scanner = bluetoothAdapter.bluetoothLeScanner + if (scanner != null) { + try { + scanner.stopScan(callback) + } catch (ignored: Throwable) { + } + } + emitLog("scan stopped") + } + + fun isScanning(): Boolean { + synchronized(lock) { + return activeCallback != null + } + } + + /** + * Replay bonded Even devices to the listener. A bonded listing has no + * signal sample and no manufacturer data, so its serial stays unknown until + * the device is also heard advertising. + */ + fun emitBondedDevices() { + val l = listener ?: return + for (json in bondedCandidateJsonList()) { + l.onAdvertisement(json) + } + } + + /** Bonded Even devices as one JSON array, for callers without a listener. */ + fun getBondedCandidatesJson(): String { + val array = JSONArray() + for (json in bondedCandidateJsonList()) { + try { + array.put(JSONObject(json)) + } catch (ignored: JSONException) { + } + } + return array.toString() + } + + private fun emitAdvertisement(result: ScanResult) { + val obj = toJson(result) ?: return + val l = listener ?: return + val json = obj.toString() + if (Looper.myLooper() == Looper.getMainLooper()) { + l.onAdvertisement(json) + } else { + mainHandler.post { + val current = listener + if (current != null) { + current.onAdvertisement(json) + } + } + } + } + + private fun bondedCandidateJsonList(): List { + val out = ArrayList() + val bonded: Set? = bluetoothAdapter.bondedDevices + if (bonded == null) { + return out + } + val now = System.currentTimeMillis() + for (device in bonded) { + if (device == null) { + continue + } + val name = device.name + if (!isAdmissible(name, null)) { + continue + } + val obj = buildJson( + device.address, name, "", null, null, null, true, "paired", now) + if (obj != null) { + out.add(obj.toString()) + } + } + return out + } + + private fun emitLog(line: String) { + val l = listener + if (l != null) { + l.onLog(line) + } + } +} diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawDeviceInfoProbe.java b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawDeviceInfoProbe.java deleted file mode 100644 index 7e7e5991..00000000 --- a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawDeviceInfoProbe.java +++ /dev/null @@ -1,619 +0,0 @@ -package com.faceclaw.app; - -import android.bluetooth.BluetoothDevice; -import android.bluetooth.BluetoothGatt; -import android.content.Context; -import android.os.Handler; -import android.os.Looper; -import android.os.SystemClock; -import android.util.Log; - -import java.util.HashSet; -import java.util.List; -import java.util.Set; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.TimeUnit; - -/** - * Connects to the glasses (stock-firmware compatible), reads the device-info / - * settings response, and reports the firmware versions plus the firmware- - * extension string (empty on stock firmware). Used by onboarding to decide whether to - * flash. Reuses FaceclawBleManager + BleProtocol; owns its own connection and - * runs on a single worker thread. Shows nothing on the lens. - */ -public class FaceclawDeviceInfoProbe implements FaceclawBleListener { - private static final String TAG = "FaceclawDeviceInfo"; - private static final int QUERY_TIMEOUT_MS = 4_000; - /** Connect + auth attempts per arm; a link that drops mid-pairing is retried once the bond settles. */ - private static final int ARM_ATTEMPTS = 3; - private static final int ARM_RETRY_DELAY_MS = 1_000; - /** - * Upper bound on waiting for the auth success while Android reports the arm - * as BOND_BONDING — i.e. an OS pairing dialog may be sitting there waiting - * for the user, which can take a lot longer than the normal 30 s window. - */ - private static final int PAIRING_WAIT_CAP_MS = 90_000; - /** After a fresh bond, how long the firmware gets to notify success on its own before the request is re-sent. */ - private static final int POST_BOND_GRACE_MS = 2_000; - private static final int POLL_MS = 250; - - private final Context context; - private final String rightAddress; - private final String leftAddress; - private final FaceclawBleManager bleManager; - private final Handler mainHandler = new Handler(Looper.getMainLooper()); - - private final Object lock = new Object(); - private int nextSeq = 0x40; - private int nextMagic = 100; - - private int awaitSid = -1; - private int awaitMagic = -1; - private byte[] awaitPb = null; - private CountDownLatch awaitLatch = null; - - /** - * Security-auth wait, kept separate from the generic ack wait: only the - * firmware's SUCCESS result (empty result message) may satisfy it. The - * firmware answers the request as soon as it arrives, but with a - * non-success result until the link is encrypted; accepting that first - * reply as "the ack" is what let the probe move on to the other arm while - * Android's pairing dialog for this one was still up. - */ - private final Set authMagics = new HashSet<>(); - private CountDownLatch authLatch = null; - - /** - * A sid-0x09 frame that carried firmware versions without matching the - * awaited magic — some firmware pushes the settings snapshot on a magic it - * picked itself instead of (or in addition to) acking the read. - */ - private volatile byte[] unsolicitedSettingsPb; - - private volatile FaceclawDeviceInfoProbeListener listener; - private volatile Thread worker; - private volatile boolean cancelled = false; - - public FaceclawDeviceInfoProbe(Context context, String rightAddress, String leftAddress) { - this.context = context.getApplicationContext(); - this.rightAddress = rightAddress == null ? "" : rightAddress; - this.leftAddress = leftAddress == null ? "" : leftAddress; - this.bleManager = new FaceclawBleManager(this.context); - this.bleManager.setListener(this); - } - - public void setListener(FaceclawDeviceInfoProbeListener listener) { - this.listener = listener; - } - - public void start() { - synchronized (lock) { - if (worker != null) { - return; - } - worker = new Thread(this::run, "faceclaw-device-info"); - worker.start(); - } - } - - public void cancel() { - cancelled = true; - Thread w = worker; - if (w != null) { - w.interrupt(); - } - } - - public void close() { - cancel(); - try { - bleManager.close(); - } catch (Exception ignored) { - } - } - - private void run() { - try { - if (rightAddress.trim().isEmpty()) { - emitError("No right-arm address configured."); - return; - } - - boolean haveLeft = !leftAddress.trim().isEmpty() && !leftAddress.equalsIgnoreCase(rightAddress); - - // Bring up and authenticate BOTH arms before asking for versions. - // Each arm is its own peripheral with its own bond, so on a fresh - // phone each raises its own pairing prompt; completing both here - // means the flash step (and the app proper) find both bonds in - // place. Strictly one arm at a time: Android pairs with one device - // at a time, and a second arm connected while the first was still - // pairing has been seen to drop within a second and never pair - // (2026-09-11 logcat). - boolean rightAuthenticated = bringUpArm(rightAddress, "right"); - boolean leftAuthenticated = false; - if (haveLeft && !cancelled) { - leftAuthenticated = bringUpArm(leftAddress, "left"); - } - if (cancelled) { - emitError("Cancelled."); - return; - } - if (!bleManager.isConnected(rightAddress)) { - // The right lens may have dropped while the left one paired; - // it is bonded by now, so this comes back quickly. - emitLog("right lens disconnected while the left lens was set up; reconnecting"); - rightAuthenticated = bringUpArm(rightAddress, "right"); - if (cancelled) { - emitError("Cancelled."); - return; - } - } - - byte[] ack = null; - String rightFailure = null; - try { - ack = queryArm(rightAddress, "right", rightAuthenticated); - } catch (IllegalStateException e) { - rightFailure = e.getMessage() == null ? e.toString() : e.getMessage(); - emitLog("right-lens query failed: " + rightFailure); - } - // The right lens is the documented control endpoint, but a silent - // right lens has been observed on stock 2.2.9 even after a - // successful security auth — run the same probe against the left - // lens rather than giving up, and log which lens answered. - if (ack == null && !cancelled && haveLeft) { - emitLog("right lens did not answer the settings query; probing the left lens"); - try { - ack = queryArm(leftAddress, "left", leftAuthenticated); - } catch (Exception e) { - emitLog("left-lens probe failed: " + (e.getMessage() == null ? e.toString() : e.getMessage())); - } - } - if (cancelled) { - emitError("Cancelled."); - return; - } - if (ack == null) { - throw new IllegalStateException(rightFailure != null - ? rightFailure - : "no response to the device-info query on either lens — check `adb logcat -s " - + TAG + "` for the frame trace"); - } - - BleProtocol.FirmwareInfo info = BleProtocol.parseSettingsFirmwareInfo(ack); - String left = info == null ? "" : info.leftVersion; - String right = info == null ? "" : info.rightVersion; - String extension = info == null ? "" : info.extension; - emitLog("device-info: L=" + left + " R=" + right + " ext=[" + extension + "]"); - emitResult(left, right, extension); - } catch (Exception e) { - String message = cancelled ? "Cancelled." : (e.getMessage() == null ? e.toString() : e.getMessage()); - emitError(message); - } finally { - try { - bleManager.close(); - } catch (Exception ignored) { - } - } - } - - /** - * Connect to one arm and complete its security-auth exchange, retrying - * when the link drops mid-pairing. Returns whether the auth success was - * seen (false = connected but unconfirmed, tolerated because the custom - * firmware's response to the exchange is not yet hardware-verified); - * throws when the arm cannot be brought up at all. - */ - private boolean bringUpArm(String address, String label) throws InterruptedException { - String lastFailure = null; - for (int attempt = 1; attempt <= ARM_ATTEMPTS && !cancelled; attempt++) { - if (attempt > 1) { - // A link Android dropped while it was still pairing comes back - // once the bond settles; reconnecting sooner just fails again. - waitForBondToSettle(address, label); - if (cancelled) { - break; - } - emitLog("retrying the " + label + " lens (attempt " + attempt + "/" + ARM_ATTEMPTS + "): " + lastFailure); - Thread.sleep(ARM_RETRY_DELAY_MS); - } - emitState("connecting", label); - try { - connectArm(address); - } catch (IllegalStateException e) { - lastFailure = e.getMessage() == null ? e.toString() : e.getMessage(); - emitLog(label + " lens: " + lastFailure); - continue; - } - if (cancelled) { - break; - } - - // Firmware 2.2.9 answers no queries until the security-auth - // exchange completes over an encrypted link; on a phone with no - // existing bond this is also what triggers SMP pairing (and its OS - // prompt), so it must come before the prelude and query. - emitState("authenticating", label); - AuthResult result = authenticate(address, label); - if (result == AuthResult.SUCCESS) { - return true; - } - if (result == AuthResult.UNCONFIRMED) { - return false; - } - lastFailure = "link dropped during authentication"; - } - if (cancelled) { - return false; - } - throw new IllegalStateException("could not connect to the " + label + " lens (" + address + "): " - + lastFailure + " — if Android showed a Bluetooth pairing request, accept it and try again"); - } - - private enum AuthResult { SUCCESS, UNCONFIRMED, LINK_DROPPED } - - /** - * Send the sid-0x80 authentication request and wait for the firmware's - * SUCCESS notification, which only arrives once the link is encrypted. - * On an unbonded phone that means waiting through Android's pairing flow - * (BOND_BONDING, possibly with a dialog up), so the wait is extended while - * the OS reports pairing in progress and the request is re-sent once - * after a fresh bond in case the pre-pairing write was dropped. Bond - * state is polled rather than awaited via broadcast; 250 ms granularity - * is plenty here. - */ - private AuthResult authenticate(String address, String label) throws InterruptedException { - long start = SystemClock.elapsedRealtime(); - long deadline = start + ConnectionOptions.SECURITY_AUTH_TIMEOUT_MS; - long hardDeadline = start + PAIRING_WAIT_CAP_MS; - CountDownLatch latch = new CountDownLatch(1); - synchronized (lock) { - authMagics.clear(); - authLatch = latch; - } - int initialBond = bleManager.getBondState(address); - emitLog("security auth: " + label + " lens, Android bond state " + bondStateName(initialBond)); - try { - int sends = 0; - boolean lastSendFailed = false; - long nextSendAt = start; - long bondedAt = -1; - boolean resentAfterBond = false; - boolean loggedBonding = false; - while (!cancelled) { - long now = SystemClock.elapsedRealtime(); - int bond = bleManager.getBondState(address); - - if (bond == BluetoothDevice.BOND_BONDING) { - // OS pairing in progress — the firmware will answer once the - // link is encrypted. Don't write, don't give up, don't move - // on to the other arm. - if (!loggedBonding) { - emitLog("Android is pairing with the " + label + " lens; waiting for the user to accept"); - loggedBonding = true; - } - if (now >= hardDeadline) { - break; - } - } else { - if (bond == BluetoothDevice.BOND_BONDED && initialBond != BluetoothDevice.BOND_BONDED - && bondedAt < 0) { - bondedAt = now; - emitLog("Android bond established with the " + label + " lens; waiting for the auth success"); - deadline = Math.max(deadline, now + ConnectionOptions.SECURITY_AUTH_SOFT_TIMEOUT_MS); - } - boolean wantSend = sends == 0 - || (lastSendFailed && sends < 3 && now >= nextSendAt) - || (bondedAt >= 0 && !resentAfterBond && now - bondedAt >= POST_BOND_GRACE_MS); - if (wantSend) { - if (bondedAt >= 0) { - resentAfterBond = true; - if (sends > 0) { - emitLog("re-sending the auth request after pairing (" + label + " lens)"); - } - } - lastSendFailed = !sendAuthRequest(address); - sends++; - nextSendAt = SystemClock.elapsedRealtime() + ARM_RETRY_DELAY_MS; - if (lastSendFailed && !bleManager.isConnected(address)) { - return AuthResult.LINK_DROPPED; - } - } - if (now >= deadline) { - break; - } - } - - if (latch.await(POLL_MS, TimeUnit.MILLISECONDS)) { - emitLog("security auth complete: " + label + " lens (" + address + ")"); - return AuthResult.SUCCESS; - } - if (!bleManager.isConnected(address)) { - emitLog("security auth: " + label + " lens disconnected while waiting" - + (bond == BluetoothDevice.BOND_BONDING ? " (Android was still pairing)" : "")); - return AuthResult.LINK_DROPPED; - } - } - if (cancelled) { - return AuthResult.UNCONFIRMED; - } - emitLog("security auth unconfirmed: " + label + " lens (" + address + "), Android bond state " - + bondStateName(bleManager.getBondState(address))); - return AuthResult.UNCONFIRMED; - } finally { - synchronized (lock) { - authLatch = null; - authMagics.clear(); - } - } - } - - private boolean sendAuthRequest(String address) { - int magic = allocMagic(); - int seq; - synchronized (lock) { - authMagics.add(magic); - seq = nextSeq++ & 0xff; - } - byte[] payload = BleProtocol.buildAuthenticationRequest(magic); - emitLog(String.format("tx %s sid=0x%02x flag=0x%02x magic=%d seq=0x%02x len=%d", - address, BleProtocol.SID_SECURITY_AUTH, BleProtocol.FLAG_SECURITY_AUTH, magic, seq, payload.length)); - List frames = BleProtocol.framePb(payload, BleProtocol.SID_SECURITY_AUTH, BleProtocol.FLAG_SECURITY_AUTH, seq); - boolean written; - try { - written = bleManager.writeFrames( - address, BleProtocol.WRITE_CHAR_UUID, frames, AndroidProtocolPlatform.writeType(ConnectionOptions.WRITE_MODE), ConnectionOptions.WRITE_TIMEOUT_MS); - } catch (IllegalStateException e) { - // "Not connected" — the link went away underneath us. - written = false; - } - if (!written) { - emitLog("tx write FAILED sid=0x80 (" + address + ")"); - } - return written; - } - - /** Block while Android reports this arm as BOND_BONDING (pairing dialog up / SMP in flight). */ - private void waitForBondToSettle(String address, String label) throws InterruptedException { - long cap = SystemClock.elapsedRealtime() + PAIRING_WAIT_CAP_MS; - boolean logged = false; - while (!cancelled && bleManager.getBondState(address) == BluetoothDevice.BOND_BONDING - && SystemClock.elapsedRealtime() < cap) { - if (!logged) { - emitLog("waiting for Android to finish pairing with the " + label + " lens before reconnecting"); - logged = true; - } - Thread.sleep(POLL_MS); - } - } - - private static String bondStateName(int state) { - switch (state) { - case BluetoothDevice.BOND_NONE: - return "none"; - case BluetoothDevice.BOND_BONDING: - return "bonding"; - case BluetoothDevice.BOND_BONDED: - return "bonded"; - default: - return "unknown"; - } - } - - /** - * Prelude + settings read on an already connected/authenticated lens. - * Returns the settings ack protobuf, an unsolicited settings push that - * carried firmware versions, or null when the lens never answered the read. - * Throws on prelude failure; the caller wraps the fallback lens's attempt - * so its failure cannot mask the primary lens's outcome. - */ - private byte[] queryArm(String address, String label, boolean authenticated) throws InterruptedException { - if (cancelled) { - return null; - } - - emitState("querying", label); - // Session prelude, then a settings/device-info read (both arms' - // versions and the firmware-extension string ride back in one response). - if (writeAndAwaitAck(address, BleProtocol.PRELUDE_ACK_SID, BleProtocol.FLAG_REQUEST, - BleProtocol.PRELUDE_ACK_MAGIC, BleProtocol.PRELUDE_F5872_PAYLOAD, - ConnectionOptions.PRELUDE_TIMEOUT_MS) == null) { - throw new IllegalStateException("session prelude not acked (" + label + " lens)" - + (authenticated ? "" : "; authentication did not complete —" - + " if Android shows a Bluetooth pairing request, accept it and try again")); - } - - // Two attempts: the first read straight after a fresh pairing has been - // seen to go unanswered while a later one succeeds. - for (int attempt = 0; attempt < 2 && !cancelled; attempt++) { - int magic = allocMagic(); - byte[] ack = writeAndAwaitAck(address, BleProtocol.SID_UI_SETTING, BleProtocol.FLAG_REQUEST, - magic, BleProtocol.buildSettingsQuery(magic), QUERY_TIMEOUT_MS); - if (ack != null) { - return ack; - } - // A push with the firmware versions on the device's own magic is - // as good as the ack we asked for. - byte[] pushed = unsolicitedSettingsPb; - if (pushed != null) { - emitLog("using unsolicited settings push instead of the read ack (" + label + " lens)"); - return pushed; - } - emitLog("settings query attempt " + (attempt + 1) + " unanswered (" + label + " lens)"); - } - return null; - } - - private void connectArm(String address) { - if (!bleManager.connect(address, ConnectionOptions.CONNECT_TIMEOUT_MS)) { - throw new IllegalStateException("connect failed"); - } - bleManager.requestConnectionPriority(address, BluetoothGatt.CONNECTION_PRIORITY_HIGH); - bleManager.requestMtu(address, ConnectionOptions.DESIRED_MTU, ConnectionOptions.CONNECT_TIMEOUT_MS); - if (!bleManager.discoverServices(address, ConnectionOptions.SERVICES_TIMEOUT_MS)) { - throw new IllegalStateException("service discovery failed"); - } - if (!bleManager.enableNotifications(address, BleProtocol.NOTIFY_CHAR_UUID, true, ConnectionOptions.DESCRIPTOR_TIMEOUT_MS)) { - throw new IllegalStateException("could not subscribe to notifications"); - } - } - - private byte[] writeAndAwaitAck(String address, int sid, int flag, int magic, byte[] payload, int timeoutMs) - throws InterruptedException { - CountDownLatch latch = new CountDownLatch(1); - synchronized (lock) { - awaitSid = sid; - awaitMagic = magic; - awaitPb = null; - awaitLatch = latch; - } - int seq; - synchronized (lock) { - seq = nextSeq++ & 0xff; - } - emitLog(String.format("tx %s sid=0x%02x flag=0x%02x magic=%d seq=0x%02x len=%d", - address, sid, flag, magic, seq, payload.length)); - List frames = BleProtocol.framePb(payload, sid, flag, seq); - boolean written = bleManager.writeFrames( - address, BleProtocol.WRITE_CHAR_UUID, frames, AndroidProtocolPlatform.writeType(ConnectionOptions.WRITE_MODE), ConnectionOptions.WRITE_TIMEOUT_MS); - if (!written) { - emitLog("tx write FAILED sid=0x" + Integer.toHexString(sid)); - synchronized (lock) { - awaitLatch = null; - } - return null; - } - boolean acked = latch.await(timeoutMs, TimeUnit.MILLISECONDS); - synchronized (lock) { - awaitLatch = null; - return acked ? awaitPb : null; - } - } - - @Override - public void onNotification(String address, String characteristicUuid, byte[] data) { - if (!BleProtocol.NOTIFY_CHAR_UUID.equalsIgnoreCase(characteristicUuid)) { - return; - } - // One notification value can carry several envelope frames back to - // back; reading only the first would silently drop the rest. - List frames = BleProtocol.splitFrames(data); - if (frames.size() > 1) { - emitLog("rx " + address + " value carries " + frames.size() + " frames (raw " + data.length + " bytes)"); - } - for (byte[] buf : frames) { - handleFrame(address, buf, data.length); - } - } - - private void handleFrame(String address, byte[] buf, int rawValueLength) { - BleProtocol.ParsedFrame frame = BleProtocol.parseFrame(buf); - if (!frame.ok) { - emitLog("rx " + address + " unparseable frame len=" + buf.length + " (raw value " + rawValueLength + ")" - + " head=" + FaceclawFirmwareUtil.bytesToHex(java.util.Arrays.copyOf(buf, Math.min(16, buf.length)))); - return; - } - // Log every control frame while diagnosing 2.2.9: sid/flag/type/magic - // plus a payload prefix is enough to reconstruct what the lens said. - int declared = buf.length > 3 ? buf[3] & 0xff : 0; - String truncated = buf.length < 8 + declared ? " TRUNCATED(declared=" + declared + ")" : ""; - emitLog(String.format("rx %s sid=0x%02x flag=0x%02x type=%d magic=%d frag=%d/%d len=%d%s pb=%s", - address, frame.sid, frame.flag, frame.msgType, frame.msgSeq, - buf.length > 5 ? buf[5] & 0xff : 0, buf.length > 4 ? buf[4] & 0xff : 0, frame.pb.length, truncated, - FaceclawFirmwareUtil.bytesToHex(java.util.Arrays.copyOf(frame.pb, Math.min(48, frame.pb.length))))); - if (frame.sid == BleProtocol.SID_UI_SETTING - && BleProtocol.parseSettingsFirmwareInfo(frame.pb) != null) { - // Any settings frame carrying firmware versions answers the probe's - // question, whether or not it matches the magic we asked with. - unsolicitedSettingsPb = frame.pb; - } - if (frame.flag == BleProtocol.FLAG_NOTIFY || frame.flag == BleProtocol.FLAG_NOTIFY_ALT) { - // Async event, not an ack. This includes the lens's own periodic - // sid-0x80 notifies (device-chosen magic, non-empty result). - return; - } - if (frame.sid == BleProtocol.SID_SECURITY_AUTH) { - boolean success = false; - boolean ours = false; - synchronized (lock) { - if (authLatch != null) { - ours = authMagics.contains(frame.msgSeq); - for (int magic : authMagics) { - if (BleProtocol.isAuthenticationSuccess(frame.pb, magic)) { - success = true; - authLatch.countDown(); - break; - } - } - } - } - if (ours && !success) { - // Expected before the link is encrypted: the firmware answers - // the request straight away with a non-success result and - // sends the real success once pairing/encryption completes. - emitLog("auth reply for magic " + frame.msgSeq + " is not the success result; still waiting"); - } - return; - } - synchronized (lock) { - if (awaitLatch != null && frame.sid == awaitSid && frame.msgSeq == awaitMagic) { - awaitPb = frame.pb; - awaitLatch.countDown(); - } - } - } - - @Override - public void onConnectionStateChange(String address, boolean connected) { - if (!connected) { - Log.i(TAG, "disconnected: " + address); - } - } - - private int allocMagic() { - synchronized (lock) { - int magic = nextMagic; - nextMagic = nextMagic >= 255 ? 100 : nextMagic + 1; - return magic; - } - } - - private void emitLog(String line) { - Log.i(TAG, line); - mainHandler.post(() -> { - FaceclawDeviceInfoProbeListener current = listener; - if (current != null) { - current.onLog(line); - } - }); - } - - private void emitState(String state, String detail) { - final String safeDetail = detail == null ? "" : detail; - mainHandler.post(() -> { - FaceclawDeviceInfoProbeListener current = listener; - if (current != null) { - current.onState(state, safeDetail); - } - }); - } - - private void emitResult(String left, String right, String extension) { - mainHandler.post(() -> { - FaceclawDeviceInfoProbeListener current = listener; - if (current != null) { - current.onResult(left, right, extension); - } - }); - } - - private void emitError(String message) { - final String safeMessage = message == null ? "" : message; - mainHandler.post(() -> { - FaceclawDeviceInfoProbeListener current = listener; - if (current != null) { - current.onError(safeMessage); - } - }); - } -} diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawDeviceInfoProbe.kt b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawDeviceInfoProbe.kt new file mode 100644 index 00000000..9d734da2 --- /dev/null +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawDeviceInfoProbe.kt @@ -0,0 +1,629 @@ +package com.faceclaw.app + +import android.bluetooth.BluetoothDevice +import android.bluetooth.BluetoothGatt +import android.content.Context +import android.os.Handler +import android.os.Looper +import android.os.SystemClock +import android.util.Log + +import java.util.HashSet +import java.util.concurrent.CountDownLatch +import java.util.concurrent.TimeUnit + +/** + * Connects to the glasses (stock-firmware compatible), reads the device-info / + * settings response, and reports the firmware versions plus the firmware- + * extension string (empty on stock firmware). Used by onboarding to decide whether to + * flash. Reuses FaceclawBleManager + BleProtocol; owns its own connection and + * runs on a single worker thread. Shows nothing on the lens. + */ +class FaceclawDeviceInfoProbe(context: Context, rightAddress: String?, leftAddress: String?) : FaceclawBleListener { + companion object { + private const val TAG = "FaceclawDeviceInfo" + private const val QUERY_TIMEOUT_MS = 4_000 + /** Connect + auth attempts per arm; a link that drops mid-pairing is retried once the bond settles. */ + private const val ARM_ATTEMPTS = 3 + private const val ARM_RETRY_DELAY_MS = 1_000 + /** + * Upper bound on waiting for the auth success while Android reports the arm + * as BOND_BONDING — i.e. an OS pairing dialog may be sitting there waiting + * for the user, which can take a lot longer than the normal 30 s window. + */ + private const val PAIRING_WAIT_CAP_MS = 90_000 + /** After a fresh bond, how long the firmware gets to notify success on its own before the request is re-sent. */ + private const val POST_BOND_GRACE_MS = 2_000 + private const val POLL_MS = 250 + + private fun bondStateName(state: Int): String { + return when (state) { + BluetoothDevice.BOND_NONE -> "none" + BluetoothDevice.BOND_BONDING -> "bonding" + BluetoothDevice.BOND_BONDED -> "bonded" + else -> "unknown" + } + } + } + + private val context: Context + private val rightAddress: String + private val leftAddress: String + private val bleManager: FaceclawBleManager + private val mainHandler = Handler(Looper.getMainLooper()) + + private val lock = Any() + private var nextSeq = 0x40 + private var nextMagic = 100 + + private var awaitSid = -1 + private var awaitMagic = -1 + private var awaitPb: ByteArray? = null + private var awaitLatch: CountDownLatch? = null + + /** + * Security-auth wait, kept separate from the generic ack wait: only the + * firmware's SUCCESS result (empty result message) may satisfy it. The + * firmware answers the request as soon as it arrives, but with a + * non-success result until the link is encrypted; accepting that first + * reply as "the ack" is what let the probe move on to the other arm while + * Android's pairing dialog for this one was still up. + */ + private val authMagics: MutableSet = HashSet() + private var authLatch: CountDownLatch? = null + + /** + * A sid-0x09 frame that carried firmware versions without matching the + * awaited magic — some firmware pushes the settings snapshot on a magic it + * picked itself instead of (or in addition to) acking the read. + */ + @Volatile + private var unsolicitedSettingsPb: ByteArray? = null + + @Volatile + private var listener: FaceclawDeviceInfoProbeListener? = null + @Volatile + private var worker: Thread? = null + @Volatile + private var cancelled = false + + init { + this.context = context.applicationContext + this.rightAddress = rightAddress ?: "" + this.leftAddress = leftAddress ?: "" + this.bleManager = FaceclawBleManager(this.context) + this.bleManager.setListener(this) + } + + fun setListener(listener: FaceclawDeviceInfoProbeListener?) { + this.listener = listener + } + + fun start() { + synchronized(lock) { + if (worker != null) { + return + } + val w = Thread({ run() }, "faceclaw-device-info") + worker = w + w.start() + } + } + + fun cancel() { + cancelled = true + val w = worker + if (w != null) { + w.interrupt() + } + } + + fun close() { + cancel() + try { + bleManager.close() + } catch (ignored: Exception) { + } + } + + private fun run() { + try { + if (rightAddress.trim().isEmpty()) { + emitError("No right-arm address configured.") + return + } + + val haveLeft = !leftAddress.trim().isEmpty() && !leftAddress.equals(rightAddress, ignoreCase = true) + + // Bring up and authenticate BOTH arms before asking for versions. + // Each arm is its own peripheral with its own bond, so on a fresh + // phone each raises its own pairing prompt; completing both here + // means the flash step (and the app proper) find both bonds in + // place. Strictly one arm at a time: Android pairs with one device + // at a time, and a second arm connected while the first was still + // pairing has been seen to drop within a second and never pair + // (2026-09-11 logcat). + var rightAuthenticated = bringUpArm(rightAddress, "right") + var leftAuthenticated = false + if (haveLeft && !cancelled) { + leftAuthenticated = bringUpArm(leftAddress, "left") + } + if (cancelled) { + emitError("Cancelled.") + return + } + if (!bleManager.isConnected(rightAddress)) { + // The right lens may have dropped while the left one paired; + // it is bonded by now, so this comes back quickly. + emitLog("right lens disconnected while the left lens was set up; reconnecting") + rightAuthenticated = bringUpArm(rightAddress, "right") + if (cancelled) { + emitError("Cancelled.") + return + } + } + + var ack: ByteArray? = null + var rightFailure: String? = null + try { + ack = queryArm(rightAddress, "right", rightAuthenticated) + } catch (e: IllegalStateException) { + rightFailure = if (e.message == null) e.toString() else e.message + emitLog("right-lens query failed: $rightFailure") + } + // The right lens is the documented control endpoint, but a silent + // right lens has been observed on stock 2.2.9 even after a + // successful security auth — run the same probe against the left + // lens rather than giving up, and log which lens answered. + if (ack == null && !cancelled && haveLeft) { + emitLog("right lens did not answer the settings query; probing the left lens") + try { + ack = queryArm(leftAddress, "left", leftAuthenticated) + } catch (e: Exception) { + emitLog("left-lens probe failed: " + (if (e.message == null) e.toString() else e.message)) + } + } + if (cancelled) { + emitError("Cancelled.") + return + } + if (ack == null) { + throw IllegalStateException(if (rightFailure != null) + rightFailure + else "no response to the device-info query on either lens — check `adb logcat -s " + + TAG + "` for the frame trace") + } + + val info: BleProtocol.FirmwareInfo? = BleProtocol.parseSettingsFirmwareInfo(ack) + val left = if (info == null) "" else info.leftVersion + val right = if (info == null) "" else info.rightVersion + val extension = if (info == null) "" else info.extension + emitLog("device-info: L=$left R=$right ext=[$extension]") + emitResult(left, right, extension) + } catch (e: Exception) { + val message = if (cancelled) "Cancelled." else (if (e.message == null) e.toString() else e.message) + emitError(message) + } finally { + try { + bleManager.close() + } catch (ignored: Exception) { + } + } + } + + /** + * Connect to one arm and complete its security-auth exchange, retrying + * when the link drops mid-pairing. Returns whether the auth success was + * seen (false = connected but unconfirmed, tolerated because the custom + * firmware's response to the exchange is not yet hardware-verified); + * throws when the arm cannot be brought up at all. + */ + @Throws(InterruptedException::class) + private fun bringUpArm(address: String, label: String): Boolean { + var lastFailure: String? = null + var attempt = 1 + while (attempt <= ARM_ATTEMPTS && !cancelled) { + if (attempt > 1) { + // A link Android dropped while it was still pairing comes back + // once the bond settles; reconnecting sooner just fails again. + waitForBondToSettle(address, label) + if (cancelled) { + break + } + emitLog("retrying the $label lens (attempt $attempt/$ARM_ATTEMPTS): $lastFailure") + Thread.sleep(ARM_RETRY_DELAY_MS.toLong()) + } + emitState("connecting", label) + try { + connectArm(address) + } catch (e: IllegalStateException) { + lastFailure = if (e.message == null) e.toString() else e.message + emitLog("$label lens: $lastFailure") + attempt++ + continue + } + if (cancelled) { + break + } + + // Firmware 2.2.9 answers no queries until the security-auth + // exchange completes over an encrypted link; on a phone with no + // existing bond this is also what triggers SMP pairing (and its OS + // prompt), so it must come before the prelude and query. + emitState("authenticating", label) + val result = authenticate(address, label) + if (result == AuthResult.SUCCESS) { + return true + } + if (result == AuthResult.UNCONFIRMED) { + return false + } + lastFailure = "link dropped during authentication" + attempt++ + } + if (cancelled) { + return false + } + throw IllegalStateException("could not connect to the $label lens ($address): " + + lastFailure + " — if Android showed a Bluetooth pairing request, accept it and try again") + } + + private enum class AuthResult { SUCCESS, UNCONFIRMED, LINK_DROPPED } + + /** + * Send the sid-0x80 authentication request and wait for the firmware's + * SUCCESS notification, which only arrives once the link is encrypted. + * On an unbonded phone that means waiting through Android's pairing flow + * (BOND_BONDING, possibly with a dialog up), so the wait is extended while + * the OS reports pairing in progress and the request is re-sent once + * after a fresh bond in case the pre-pairing write was dropped. Bond + * state is polled rather than awaited via broadcast; 250 ms granularity + * is plenty here. + */ + @Throws(InterruptedException::class) + private fun authenticate(address: String, label: String): AuthResult { + val start = SystemClock.elapsedRealtime() + var deadline = start + ConnectionOptions.SECURITY_AUTH_TIMEOUT_MS + val hardDeadline = start + PAIRING_WAIT_CAP_MS + val latch = CountDownLatch(1) + synchronized(lock) { + authMagics.clear() + authLatch = latch + } + val initialBond = bleManager.getBondState(address) + emitLog("security auth: $label lens, Android bond state " + bondStateName(initialBond)) + try { + var sends = 0 + var lastSendFailed = false + var nextSendAt = start + var bondedAt = -1L + var resentAfterBond = false + var loggedBonding = false + while (!cancelled) { + val now = SystemClock.elapsedRealtime() + val bond = bleManager.getBondState(address) + + if (bond == BluetoothDevice.BOND_BONDING) { + // OS pairing in progress — the firmware will answer once the + // link is encrypted. Don't write, don't give up, don't move + // on to the other arm. + if (!loggedBonding) { + emitLog("Android is pairing with the $label lens; waiting for the user to accept") + loggedBonding = true + } + if (now >= hardDeadline) { + break + } + } else { + if (bond == BluetoothDevice.BOND_BONDED && initialBond != BluetoothDevice.BOND_BONDED + && bondedAt < 0) { + bondedAt = now + emitLog("Android bond established with the $label lens; waiting for the auth success") + deadline = Math.max(deadline, now + ConnectionOptions.SECURITY_AUTH_SOFT_TIMEOUT_MS) + } + val wantSend = sends == 0 + || (lastSendFailed && sends < 3 && now >= nextSendAt) + || (bondedAt >= 0 && !resentAfterBond && now - bondedAt >= POST_BOND_GRACE_MS) + if (wantSend) { + if (bondedAt >= 0) { + resentAfterBond = true + if (sends > 0) { + emitLog("re-sending the auth request after pairing ($label lens)") + } + } + lastSendFailed = !sendAuthRequest(address) + sends++ + nextSendAt = SystemClock.elapsedRealtime() + ARM_RETRY_DELAY_MS + if (lastSendFailed && !bleManager.isConnected(address)) { + return AuthResult.LINK_DROPPED + } + } + if (now >= deadline) { + break + } + } + + if (latch.await(POLL_MS.toLong(), TimeUnit.MILLISECONDS)) { + emitLog("security auth complete: $label lens ($address)") + return AuthResult.SUCCESS + } + if (!bleManager.isConnected(address)) { + emitLog("security auth: $label lens disconnected while waiting" + + (if (bond == BluetoothDevice.BOND_BONDING) " (Android was still pairing)" else "")) + return AuthResult.LINK_DROPPED + } + } + if (cancelled) { + return AuthResult.UNCONFIRMED + } + emitLog("security auth unconfirmed: $label lens ($address), Android bond state " + + bondStateName(bleManager.getBondState(address))) + return AuthResult.UNCONFIRMED + } finally { + synchronized(lock) { + authLatch = null + authMagics.clear() + } + } + } + + private fun sendAuthRequest(address: String): Boolean { + val magic = allocMagic() + val seq: Int + synchronized(lock) { + authMagics.add(magic) + seq = nextSeq++ and 0xff + } + val payload = BleProtocol.buildAuthenticationRequest(magic) + emitLog(String.format("tx %s sid=0x%02x flag=0x%02x magic=%d seq=0x%02x len=%d", + address, BleProtocol.SID_SECURITY_AUTH, BleProtocol.FLAG_SECURITY_AUTH, magic, seq, payload.size)) + val frames = BleProtocol.framePb(payload, BleProtocol.SID_SECURITY_AUTH, BleProtocol.FLAG_SECURITY_AUTH, seq) + var written: Boolean + try { + written = bleManager.writeFrames( + address, BleProtocol.WRITE_CHAR_UUID, frames, AndroidProtocolPlatform.writeType(ConnectionOptions.WRITE_MODE), ConnectionOptions.WRITE_TIMEOUT_MS) + } catch (e: IllegalStateException) { + // "Not connected" — the link went away underneath us. + written = false + } + if (!written) { + emitLog("tx write FAILED sid=0x80 ($address)") + } + return written + } + + /** Block while Android reports this arm as BOND_BONDING (pairing dialog up / SMP in flight). */ + @Throws(InterruptedException::class) + private fun waitForBondToSettle(address: String, label: String) { + val cap = SystemClock.elapsedRealtime() + PAIRING_WAIT_CAP_MS + var logged = false + while (!cancelled && bleManager.getBondState(address) == BluetoothDevice.BOND_BONDING + && SystemClock.elapsedRealtime() < cap) { + if (!logged) { + emitLog("waiting for Android to finish pairing with the $label lens before reconnecting") + logged = true + } + Thread.sleep(POLL_MS.toLong()) + } + } + + /** + * Prelude + settings read on an already connected/authenticated lens. + * Returns the settings ack protobuf, an unsolicited settings push that + * carried firmware versions, or null when the lens never answered the read. + * Throws on prelude failure; the caller wraps the fallback lens's attempt + * so its failure cannot mask the primary lens's outcome. + */ + @Throws(InterruptedException::class) + private fun queryArm(address: String, label: String, authenticated: Boolean): ByteArray? { + if (cancelled) { + return null + } + + emitState("querying", label) + // Session prelude, then a settings/device-info read (both arms' + // versions and the firmware-extension string ride back in one response). + if (writeAndAwaitAck(address, BleProtocol.PRELUDE_ACK_SID, BleProtocol.FLAG_REQUEST, + BleProtocol.PRELUDE_ACK_MAGIC, BleProtocol.PRELUDE_F5872_PAYLOAD, + ConnectionOptions.PRELUDE_TIMEOUT_MS) == null) { + throw IllegalStateException("session prelude not acked ($label lens)" + + (if (authenticated) "" else "; authentication did not complete —" + + " if Android shows a Bluetooth pairing request, accept it and try again")) + } + + // Two attempts: the first read straight after a fresh pairing has been + // seen to go unanswered while a later one succeeds. + var attempt = 0 + while (attempt < 2 && !cancelled) { + val magic = allocMagic() + val ack = writeAndAwaitAck(address, BleProtocol.SID_UI_SETTING, BleProtocol.FLAG_REQUEST, + magic, BleProtocol.buildSettingsQuery(magic), QUERY_TIMEOUT_MS) + if (ack != null) { + return ack + } + // A push with the firmware versions on the device's own magic is + // as good as the ack we asked for. + val pushed = unsolicitedSettingsPb + if (pushed != null) { + emitLog("using unsolicited settings push instead of the read ack ($label lens)") + return pushed + } + emitLog("settings query attempt " + (attempt + 1) + " unanswered (" + label + " lens)") + attempt++ + } + return null + } + + private fun connectArm(address: String) { + if (!bleManager.connect(address, ConnectionOptions.CONNECT_TIMEOUT_MS)) { + throw IllegalStateException("connect failed") + } + bleManager.requestConnectionPriority(address, BluetoothGatt.CONNECTION_PRIORITY_HIGH) + bleManager.requestMtu(address, ConnectionOptions.DESIRED_MTU, ConnectionOptions.CONNECT_TIMEOUT_MS) + if (!bleManager.discoverServices(address, ConnectionOptions.SERVICES_TIMEOUT_MS)) { + throw IllegalStateException("service discovery failed") + } + if (!bleManager.enableNotifications(address, BleProtocol.NOTIFY_CHAR_UUID, true, ConnectionOptions.DESCRIPTOR_TIMEOUT_MS)) { + throw IllegalStateException("could not subscribe to notifications") + } + } + + @Throws(InterruptedException::class) + private fun writeAndAwaitAck(address: String, sid: Int, flag: Int, magic: Int, payload: ByteArray, timeoutMs: Int): ByteArray? { + val latch = CountDownLatch(1) + synchronized(lock) { + awaitSid = sid + awaitMagic = magic + awaitPb = null + awaitLatch = latch + } + val seq: Int + synchronized(lock) { + seq = nextSeq++ and 0xff + } + emitLog(String.format("tx %s sid=0x%02x flag=0x%02x magic=%d seq=0x%02x len=%d", + address, sid, flag, magic, seq, payload.size)) + val frames = BleProtocol.framePb(payload, sid, flag, seq) + val written = bleManager.writeFrames( + address, BleProtocol.WRITE_CHAR_UUID, frames, AndroidProtocolPlatform.writeType(ConnectionOptions.WRITE_MODE), ConnectionOptions.WRITE_TIMEOUT_MS) + if (!written) { + emitLog("tx write FAILED sid=0x" + Integer.toHexString(sid)) + synchronized(lock) { + awaitLatch = null + } + return null + } + val acked = latch.await(timeoutMs.toLong(), TimeUnit.MILLISECONDS) + synchronized(lock) { + awaitLatch = null + return if (acked) awaitPb else null + } + } + + override fun onNotification(address: String?, characteristicUuid: String?, data: ByteArray?) { + if (!BleProtocol.NOTIFY_CHAR_UUID.equals(characteristicUuid, ignoreCase = true)) { + return + } + // One notification value can carry several envelope frames back to + // back; reading only the first would silently drop the rest. + val frames = BleProtocol.splitFrames(data) + if (frames.size > 1) { + emitLog("rx " + address + " value carries " + frames.size + " frames (raw " + data!!.size + " bytes)") + } + for (buf in frames) { + handleFrame(address, buf, data!!.size) + } + } + + private fun handleFrame(address: String?, buf: ByteArray, rawValueLength: Int) { + val frame = BleProtocol.parseFrame(buf) + if (!frame.ok) { + emitLog("rx " + address + " unparseable frame len=" + buf.size + " (raw value " + rawValueLength + ")" + + " head=" + FaceclawFirmwareUtil.bytesToHex(java.util.Arrays.copyOf(buf, Math.min(16, buf.size)))) + return + } + // Log every control frame while diagnosing 2.2.9: sid/flag/type/magic + // plus a payload prefix is enough to reconstruct what the lens said. + val declared = if (buf.size > 3) buf[3].toInt() and 0xff else 0 + val truncated = if (buf.size < 8 + declared) " TRUNCATED(declared=$declared)" else "" + emitLog(String.format("rx %s sid=0x%02x flag=0x%02x type=%d magic=%d frag=%d/%d len=%d%s pb=%s", + address, frame.sid, frame.flag, frame.msgType, frame.msgSeq, + if (buf.size > 5) buf[5].toInt() and 0xff else 0, if (buf.size > 4) buf[4].toInt() and 0xff else 0, frame.pb.size, truncated, + FaceclawFirmwareUtil.bytesToHex(java.util.Arrays.copyOf(frame.pb, Math.min(48, frame.pb.size))))) + if (frame.sid == BleProtocol.SID_UI_SETTING + && BleProtocol.parseSettingsFirmwareInfo(frame.pb) != null) { + // Any settings frame carrying firmware versions answers the probe's + // question, whether or not it matches the magic we asked with. + unsolicitedSettingsPb = frame.pb + } + if (frame.flag == BleProtocol.FLAG_NOTIFY || frame.flag == BleProtocol.FLAG_NOTIFY_ALT) { + // Async event, not an ack. This includes the lens's own periodic + // sid-0x80 notifies (device-chosen magic, non-empty result). + return + } + if (frame.sid == BleProtocol.SID_SECURITY_AUTH) { + var success = false + var ours = false + synchronized(lock) { + val currentLatch = authLatch + if (currentLatch != null) { + ours = authMagics.contains(frame.msgSeq) + for (magic in authMagics) { + if (BleProtocol.isAuthenticationSuccess(frame.pb, magic)) { + success = true + currentLatch.countDown() + break + } + } + } + } + if (ours && !success) { + // Expected before the link is encrypted: the firmware answers + // the request straight away with a non-success result and + // sends the real success once pairing/encryption completes. + emitLog("auth reply for magic " + frame.msgSeq + " is not the success result; still waiting") + } + return + } + synchronized(lock) { + val currentLatch = awaitLatch + if (currentLatch != null && frame.sid == awaitSid && frame.msgSeq == awaitMagic) { + awaitPb = frame.pb + currentLatch.countDown() + } + } + } + + override fun onConnectionStateChange(address: String?, connected: Boolean) { + if (!connected) { + Log.i(TAG, "disconnected: $address") + } + } + + private fun allocMagic(): Int { + synchronized(lock) { + val magic = nextMagic + nextMagic = if (nextMagic >= 255) 100 else nextMagic + 1 + return magic + } + } + + private fun emitLog(line: String) { + Log.i(TAG, line) + mainHandler.post { + val current = listener + if (current != null) { + current.onLog(line) + } + } + } + + private fun emitState(state: String, detail: String?) { + val safeDetail = detail ?: "" + mainHandler.post { + val current = listener + if (current != null) { + current.onState(state, safeDetail) + } + } + } + + private fun emitResult(left: String, right: String, extension: String) { + mainHandler.post { + val current = listener + if (current != null) { + current.onResult(left, right, extension) + } + } + } + + private fun emitError(message: String?) { + val safeMessage = message ?: "" + mainHandler.post { + val current = listener + if (current != null) { + current.onError(safeMessage) + } + } + } +} diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawDiarizer.java b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawDiarizer.java deleted file mode 100644 index ea9352aa..00000000 --- a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawDiarizer.java +++ /dev/null @@ -1,262 +0,0 @@ -package com.faceclaw.app; - -import android.os.Handler; -import android.os.Looper; -import android.util.Log; - -import com.k2fsa.sherpa.onnx.FastClusteringConfig; -import com.k2fsa.sherpa.onnx.OfflineSpeakerDiarization; -import com.k2fsa.sherpa.onnx.OfflineSpeakerDiarizationConfig; -import com.k2fsa.sherpa.onnx.OfflineSpeakerDiarizationSegment; -import com.k2fsa.sherpa.onnx.OfflineSpeakerSegmentationModelConfig; -import com.k2fsa.sherpa.onnx.OfflineSpeakerSegmentationPyannoteModelConfig; -import com.k2fsa.sherpa.onnx.SpeakerEmbeddingExtractorConfig; - -import org.json.JSONArray; -import org.json.JSONObject; - -import java.io.DataInputStream; -import java.io.File; -import java.io.FileInputStream; -import java.io.IOException; -import java.nio.ByteBuffer; - -/** - * Offline re-diarization of a stored conversation recording (16-bit mono - * WAV): pyannote segmentation plus the shared speaker-embedding model with - * clustering, via sherpa-onnx. Runs on its own thread; results are handed to - * TS as JSON turns for reconciliation against the transcript segments. - */ -public class FaceclawDiarizer { - private static final String TAG = "FaceclawDiarizer"; - - public interface Listener { - /** progress in [0, 1]. */ - void onProgress(double progress); - - /** turns: [{startMs, endMs, cluster}], sorted by start time. */ - void onDone(String turnsJson); - - void onError(String message); - } - - private final String segmentationModelPath; - private final String embeddingModelPath; - private final String wavPath; - private final int numClusters; - private final float threshold; - private final Listener listener; - private final Handler mainHandler = new Handler(Looper.getMainLooper()); - private volatile boolean cancelled; - - public FaceclawDiarizer( - String segmentationModelPath, - String embeddingModelPath, - String wavPath, - int numClusters, - float threshold, - Listener listener) { - this.segmentationModelPath = segmentationModelPath; - this.embeddingModelPath = embeddingModelPath; - this.wavPath = wavPath; - this.numClusters = numClusters; - this.threshold = threshold <= 0 ? 0.5f : threshold; - this.listener = listener; - } - - public void start() { - Thread thread = new Thread(this::run, "FaceclawDiarizer"); - thread.setPriority(Thread.MIN_PRIORITY); - thread.start(); - } - - public void cancel() { - cancelled = true; - } - - private void run() { - OfflineSpeakerDiarization diarization = null; - try { - float[] samples = wavPath.endsWith(".wav") - ? readWavMono16(wavPath) - : decodeCompressedMono(wavPath); - OfflineSpeakerDiarizationConfig config = OfflineSpeakerDiarizationConfig.builder() - .setSegmentation(OfflineSpeakerSegmentationModelConfig.builder() - .setPyannote(OfflineSpeakerSegmentationPyannoteModelConfig.builder() - .setModel(segmentationModelPath) - .build()) - .setNumThreads(1) - .build()) - .setEmbedding(SpeakerEmbeddingExtractorConfig.builder() - .setModel(embeddingModelPath) - .setNumThreads(1) - .build()) - .setClustering(FastClusteringConfig.builder() - .setNumClusters(numClusters > 0 ? numClusters : -1) - .setThreshold(threshold) - .build()) - .setMinDurationOn(0.3f) - .setMinDurationOff(0.5f) - .build(); - diarization = new OfflineSpeakerDiarization(config); - OfflineSpeakerDiarizationSegment[] segments = diarization.processWithCallback( - samples, - (numProcessedChunks, numTotalChunks, arg) -> { - emitProgress(numTotalChunks <= 0 ? 0 - : (double) numProcessedChunks / numTotalChunks); - return cancelled ? 1 : 0; - }); - JSONArray turns = new JSONArray(); - if (segments != null) { - for (OfflineSpeakerDiarizationSegment segment : segments) { - JSONObject turn = new JSONObject(); - turn.put("startMs", Math.round(segment.getStart() * 1000)); - turn.put("endMs", Math.round(segment.getEnd() * 1000)); - turn.put("cluster", segment.getSpeaker()); - turns.put(turn); - } - } - String result = turns.toString(); - mainHandler.post(() -> listener.onDone(result)); - } catch (Throwable t) { - Log.w(TAG, "diarization failed for " + wavPath, t); - String message = t.getMessage() == null ? t.toString() : t.getMessage(); - mainHandler.post(() -> listener.onError(message)); - } finally { - if (diarization != null) { - diarization.release(); - } - } - } - - private void emitProgress(double progress) { - mainHandler.post(() -> listener.onProgress(Math.max(0, Math.min(1, progress)))); - } - - /** - * Decode a compressed recording (M4A/AAC from the transcode sweep) into - * [-1, 1] floats via MediaExtractor + MediaCodec. Recordings are written - * as 16 kHz mono, so no resampling is needed. - */ - private static float[] decodeCompressedMono(String path) throws IOException { - android.media.MediaExtractor extractor = new android.media.MediaExtractor(); - android.media.MediaCodec codec = null; - java.io.ByteArrayOutputStream pcm = new java.io.ByteArrayOutputStream(); - try { - extractor.setDataSource(path); - int trackIndex = -1; - android.media.MediaFormat format = null; - for (int i = 0; i < extractor.getTrackCount(); i++) { - android.media.MediaFormat candidate = extractor.getTrackFormat(i); - String mime = candidate.getString(android.media.MediaFormat.KEY_MIME); - if (mime != null && mime.startsWith("audio/")) { - trackIndex = i; - format = candidate; - break; - } - } - if (trackIndex < 0 || format == null) { - throw new IOException("no audio track in " + path); - } - extractor.selectTrack(trackIndex); - codec = android.media.MediaCodec.createDecoderByType( - format.getString(android.media.MediaFormat.KEY_MIME)); - codec.configure(format, null, null, 0); - codec.start(); - android.media.MediaCodec.BufferInfo info = new android.media.MediaCodec.BufferInfo(); - boolean inputDone = false; - boolean outputDone = false; - while (!outputDone) { - if (!inputDone) { - int inIndex = codec.dequeueInputBuffer(10_000); - if (inIndex >= 0) { - ByteBuffer inBuf = codec.getInputBuffer(inIndex); - int size = extractor.readSampleData(inBuf, 0); - if (size < 0) { - codec.queueInputBuffer(inIndex, 0, 0, 0, - android.media.MediaCodec.BUFFER_FLAG_END_OF_STREAM); - inputDone = true; - } else { - codec.queueInputBuffer(inIndex, 0, size, extractor.getSampleTime(), 0); - extractor.advance(); - } - } - } - int outIndex = codec.dequeueOutputBuffer(info, 10_000); - if (outIndex >= 0) { - ByteBuffer outBuf = codec.getOutputBuffer(outIndex); - byte[] chunk = new byte[info.size]; - outBuf.position(info.offset); - outBuf.get(chunk); - pcm.write(chunk); - codec.releaseOutputBuffer(outIndex, false); - if ((info.flags & android.media.MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0) { - outputDone = true; - } - } - } - } finally { - extractor.release(); - if (codec != null) { - try { - codec.stop(); - } catch (Throwable ignored) { - } - codec.release(); - } - } - byte[] bytes = pcm.toByteArray(); - int count = bytes.length / 2; - float[] samples = new float[count]; - for (int i = 0; i < count; i++) { - samples[i] = ((short) ((bytes[i * 2] & 0xff) | (bytes[i * 2 + 1] << 8))) / 32768.0f; - } - return samples; - } - - /** Load a 16-bit mono PCM WAV into [-1, 1] floats. */ - private static float[] readWavMono16(String path) throws IOException { - File file = new File(path); - long fileLength = file.length(); - try (DataInputStream in = new DataInputStream(new FileInputStream(file))) { - byte[] header = new byte[44]; - in.readFully(header); - if (header[0] != 'R' || header[1] != 'I' || header[8] != 'W') { - throw new IOException("not a WAV file"); - } - int channels = (header[22] & 0xff) | (header[23] << 8); - int bits = (header[34] & 0xff) | (header[35] << 8); - if (channels != 1 || bits != 16) { - throw new IOException("only 16-bit mono WAV is supported"); - } - long dataBytes = fileLength - 44; - int count = (int) Math.min(dataBytes / 2, Integer.MAX_VALUE); - float[] samples = new float[count]; - byte[] chunk = new byte[65536]; - int sampleIndex = 0; - int carried = 0; - byte carriedByte = 0; - int read; - while (sampleIndex < count && (read = in.read(chunk)) > 0) { - int offset = 0; - if (carried == 1) { - samples[sampleIndex++] = - ((short) ((carriedByte & 0xff) | (chunk[0] << 8))) / 32768.0f; - offset = 1; - carried = 0; - } - int pairs = (read - offset) / 2; - for (int i = 0; i < pairs && sampleIndex < count; i++) { - int base = offset + i * 2; - samples[sampleIndex++] = - ((short) ((chunk[base] & 0xff) | (chunk[base + 1] << 8))) / 32768.0f; - } - if (((read - offset) & 1) == 1) { - carried = 1; - carriedByte = chunk[read - 1]; - } - } - return samples; - } - } -} diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawDiarizer.kt b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawDiarizer.kt new file mode 100644 index 00000000..95cd7e98 --- /dev/null +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawDiarizer.kt @@ -0,0 +1,260 @@ +package com.faceclaw.app + +import android.os.Handler +import android.os.Looper +import android.util.Log + +import com.k2fsa.sherpa.onnx.FastClusteringConfig +import com.k2fsa.sherpa.onnx.OfflineSpeakerDiarization +import com.k2fsa.sherpa.onnx.OfflineSpeakerDiarizationCallback +import com.k2fsa.sherpa.onnx.OfflineSpeakerDiarizationConfig +import com.k2fsa.sherpa.onnx.OfflineSpeakerDiarizationSegment +import com.k2fsa.sherpa.onnx.OfflineSpeakerSegmentationModelConfig +import com.k2fsa.sherpa.onnx.OfflineSpeakerSegmentationPyannoteModelConfig +import com.k2fsa.sherpa.onnx.SpeakerEmbeddingExtractorConfig + +import org.json.JSONArray +import org.json.JSONObject + +import java.io.DataInputStream +import java.io.File +import java.io.FileInputStream +import java.io.IOException +import java.nio.ByteBuffer + +/** + * Offline re-diarization of a stored conversation recording (16-bit mono + * WAV): pyannote segmentation plus the shared speaker-embedding model with + * clustering, via sherpa-onnx. Runs on its own thread; results are handed to + * TS as JSON turns for reconciliation against the transcript segments. + */ +class FaceclawDiarizer( + private val segmentationModelPath: String?, + private val embeddingModelPath: String?, + private val wavPath: String?, + private val numClusters: Int, + threshold: Float, + private val listener: Listener +) { + companion object { + private const val TAG = "FaceclawDiarizer" + + /** + * Decode a compressed recording (M4A/AAC from the transcode sweep) into + * [-1, 1] floats via MediaExtractor + MediaCodec. Recordings are written + * as 16 kHz mono, so no resampling is needed. + */ + @JvmStatic + @Throws(IOException::class) + private fun decodeCompressedMono(path: String?): FloatArray { + val extractor = android.media.MediaExtractor() + var codec: android.media.MediaCodec? = null + val pcm = java.io.ByteArrayOutputStream() + try { + extractor.setDataSource(path!!) + var trackIndex = -1 + var format: android.media.MediaFormat? = null + for (i in 0 until extractor.trackCount) { + val candidate = extractor.getTrackFormat(i) + val mime = candidate.getString(android.media.MediaFormat.KEY_MIME) + if (mime != null && mime.startsWith("audio/")) { + trackIndex = i + format = candidate + break + } + } + if (trackIndex < 0 || format == null) { + throw IOException("no audio track in $path") + } + extractor.selectTrack(trackIndex) + codec = android.media.MediaCodec.createDecoderByType( + format.getString(android.media.MediaFormat.KEY_MIME)!!) + codec.configure(format, null, null, 0) + codec.start() + val info = android.media.MediaCodec.BufferInfo() + var inputDone = false + var outputDone = false + while (!outputDone) { + if (!inputDone) { + val inIndex = codec.dequeueInputBuffer(10_000) + if (inIndex >= 0) { + val inBuf = codec.getInputBuffer(inIndex)!! + val size = extractor.readSampleData(inBuf, 0) + if (size < 0) { + codec.queueInputBuffer(inIndex, 0, 0, 0, + android.media.MediaCodec.BUFFER_FLAG_END_OF_STREAM) + inputDone = true + } else { + codec.queueInputBuffer(inIndex, 0, size, extractor.sampleTime, 0) + extractor.advance() + } + } + } + val outIndex = codec.dequeueOutputBuffer(info, 10_000) + if (outIndex >= 0) { + val outBuf = codec.getOutputBuffer(outIndex)!! + val chunk = ByteArray(info.size) + outBuf.position(info.offset) + outBuf.get(chunk) + pcm.write(chunk) + codec.releaseOutputBuffer(outIndex, false) + if ((info.flags and android.media.MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0) { + outputDone = true + } + } + } + } finally { + extractor.release() + if (codec != null) { + try { + codec.stop() + } catch (ignored: Throwable) { + } + codec.release() + } + } + val bytes = pcm.toByteArray() + val count = bytes.size / 2 + val samples = FloatArray(count) + for (i in 0 until count) { + samples[i] = ((bytes[i * 2].toInt() and 0xff) or (bytes[i * 2 + 1].toInt() shl 8)).toShort() / 32768.0f + } + return samples + } + + /** Load a 16-bit mono PCM WAV into [-1, 1] floats. */ + @JvmStatic + @Throws(IOException::class) + private fun readWavMono16(path: String?): FloatArray { + val file = File(path) + val fileLength = file.length() + DataInputStream(FileInputStream(file)).use { `in` -> + val header = ByteArray(44) + `in`.readFully(header) + if (header[0] != 'R'.code.toByte() || header[1] != 'I'.code.toByte() || header[8] != 'W'.code.toByte()) { + throw IOException("not a WAV file") + } + val channels = (header[22].toInt() and 0xff) or (header[23].toInt() shl 8) + val bits = (header[34].toInt() and 0xff) or (header[35].toInt() shl 8) + if (channels != 1 || bits != 16) { + throw IOException("only 16-bit mono WAV is supported") + } + val dataBytes = fileLength - 44 + val count = Math.min(dataBytes / 2, Int.MAX_VALUE.toLong()).toInt() + val samples = FloatArray(count) + val chunk = ByteArray(65536) + var sampleIndex = 0 + var carried = 0 + var carriedByte: Byte = 0 + var read = 0 + while (sampleIndex < count && `in`.read(chunk).also { read = it } > 0) { + var offset = 0 + if (carried == 1) { + samples[sampleIndex++] = + ((carriedByte.toInt() and 0xff) or (chunk[0].toInt() shl 8)).toShort() / 32768.0f + offset = 1 + carried = 0 + } + val pairs = (read - offset) / 2 + var i = 0 + while (i < pairs && sampleIndex < count) { + val base = offset + i * 2 + samples[sampleIndex++] = + ((chunk[base].toInt() and 0xff) or (chunk[base + 1].toInt() shl 8)).toShort() / 32768.0f + i++ + } + if (((read - offset) and 1) == 1) { + carried = 1 + carriedByte = chunk[read - 1] + } + } + return samples + } + } + } + + interface Listener { + /** progress in [0, 1]. */ + fun onProgress(progress: Double) + + /** turns: [{startMs, endMs, cluster}], sorted by start time. */ + fun onDone(turnsJson: String?) + + fun onError(message: String?) + } + + private val threshold: Float = if (threshold <= 0) 0.5f else threshold + private val mainHandler = Handler(Looper.getMainLooper()) + @Volatile + private var cancelled = false + + fun start() { + val thread = Thread(Runnable { run() }, "FaceclawDiarizer") + thread.priority = Thread.MIN_PRIORITY + thread.start() + } + + fun cancel() { + cancelled = true + } + + private fun run() { + var diarization: OfflineSpeakerDiarization? = null + try { + val samples = if (wavPath!!.endsWith(".wav")) + readWavMono16(wavPath) + else + decodeCompressedMono(wavPath) + val config = OfflineSpeakerDiarizationConfig.builder() + .setSegmentation(OfflineSpeakerSegmentationModelConfig.builder() + .setPyannote(OfflineSpeakerSegmentationPyannoteModelConfig.builder() + .setModel(segmentationModelPath) + .build()) + .setNumThreads(1) + .build()) + .setEmbedding(SpeakerEmbeddingExtractorConfig.builder() + .setModel(embeddingModelPath) + .setNumThreads(1) + .build()) + .setClustering(FastClusteringConfig.builder() + .setNumClusters(if (numClusters > 0) numClusters else -1) + .setThreshold(threshold) + .build()) + .setMinDurationOn(0.3f) + .setMinDurationOff(0.5f) + .build() + diarization = OfflineSpeakerDiarization(config) + val segments: Array? = diarization.processWithCallback( + samples, + OfflineSpeakerDiarizationCallback { numProcessedChunks, numTotalChunks, _ -> + emitProgress(if (numTotalChunks <= 0) 0.0 + else numProcessedChunks.toDouble() / numTotalChunks) + if (cancelled) 1 else 0 + }) + val turns = JSONArray() + if (segments != null) { + for (segment in segments) { + val turn = JSONObject() + turn.put("startMs", Math.round(segment.start * 1000)) + turn.put("endMs", Math.round(segment.end * 1000)) + turn.put("cluster", segment.speaker) + turns.put(turn) + } + } + val result = turns.toString() + mainHandler.post { listener.onDone(result) } + } catch (t: Throwable) { + Log.w(TAG, "diarization failed for $wavPath", t) + val message = if (t.message == null) t.toString() else t.message + mainHandler.post { listener.onError(message) } + } finally { + if (diarization != null) { + diarization.release() + } + } + } + + private fun emitProgress(progress: Double) { + mainHandler.post { listener.onProgress(Math.max(0.0, Math.min(1.0, progress))) } + } +} diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawEvenAppDetector.java b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawEvenAppDetector.java deleted file mode 100644 index bb3e952d..00000000 --- a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawEvenAppDetector.java +++ /dev/null @@ -1,87 +0,0 @@ -package com.faceclaw.app; - -import android.content.ComponentName; -import android.content.Context; -import android.content.Intent; -import android.content.pm.PackageManager; -import android.net.Uri; -import android.provider.Settings; -import android.util.Log; - -public final class FaceclawEvenAppDetector { - private static final String TAG = "FaceclawEvenApp"; - - public static final String EVEN_PACKAGE_NAME = "com.even.sg"; - public static final String EVEN_NOTIFICATION_TITLE = "Even Notification Service"; - - private FaceclawEvenAppDetector() { - } - - public static boolean isNotificationAccessEnabled(Context context) { - if (context == null) { - return false; - } - Context appContext = context.getApplicationContext(); - ComponentName listenerComponent = new ComponentName(appContext, FaceclawMediaNotificationListenerService.class); - String enabledListeners = Settings.Secure.getString( - appContext.getContentResolver(), - "enabled_notification_listeners" - ); - if (enabledListeners == null || enabledListeners.isEmpty()) { - return false; - } - String fullName = listenerComponent.flattenToString(); - String shortName = listenerComponent.flattenToShortString(); - return enabledListeners.contains(fullName) - || enabledListeners.contains(shortName) - || enabledListeners.contains(appContext.getPackageName()); - } - - public static boolean isEvenNotificationActive(Context context) { - boolean hasAccess = isNotificationAccessEnabled(context); - if (!hasAccess) { - Log.i(TAG, "Can't check for Even Realities app notification, permission is not enabled"); - return false; - } - boolean hasNotfication = FaceclawMediaNotificationListenerService.hasActiveNotificationTitle(EVEN_NOTIFICATION_TITLE); - Log.i(TAG, "isEvenNotificationActive: " + hasNotfication); - return hasNotfication; - } - - /** Returns "opened", "not-installed", or "failed" for the phone UI. */ - public static String openEvenAppSettings(Context context) { - if (context == null) { - return "failed"; - } - Context appContext = context.getApplicationContext(); - try { - // Settings may launch successfully and then immediately close for a - // missing package, so check before starting the activity. - appContext.getPackageManager().getApplicationInfo(EVEN_PACKAGE_NAME, 0); - Intent intent = new Intent(Settings.ACTION_APPLICATION_DETAILS_SETTINGS); - intent.setData(Uri.parse("package:" + EVEN_PACKAGE_NAME)); - intent.addFlags(Intent.FLAG_ACTIVITY_NEW_TASK); - appContext.startActivity(intent); - return "opened"; - } catch (PackageManager.NameNotFoundException e) { - return "not-installed"; - } catch (Exception t) { - Log.w(TAG, "failed to open Even app settings", t); - return "failed"; - } - } - - public static void openNotificationAccessSettings(Context context) { - if (context == null) { - return; - } - Context appContext = context.getApplicationContext(); - Intent intent = new Intent(Settings.ACTION_NOTIFICATION_LISTENER_SETTINGS); - intent.addFlags(Intent.FLAG_ACTIVITY_NEW_TASK); - try { - appContext.startActivity(intent); - } catch (Throwable t) { - Log.w(TAG, "failed to open notification access settings", t); - } - } -} diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawEvenAppDetector.kt b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawEvenAppDetector.kt new file mode 100644 index 00000000..4a0b64be --- /dev/null +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawEvenAppDetector.kt @@ -0,0 +1,90 @@ +package com.faceclaw.app + +import android.content.ComponentName +import android.content.Context +import android.content.Intent +import android.content.pm.PackageManager +import android.net.Uri +import android.provider.Settings +import android.util.Log + +class FaceclawEvenAppDetector private constructor() { + companion object { + private const val TAG = "FaceclawEvenApp" + + const val EVEN_PACKAGE_NAME = "com.even.sg" + const val EVEN_NOTIFICATION_TITLE = "Even Notification Service" + + @JvmStatic + fun isNotificationAccessEnabled(context: Context?): Boolean { + if (context == null) { + return false + } + val appContext = context.applicationContext + val listenerComponent = ComponentName(appContext, FaceclawMediaNotificationListenerService::class.java) + val enabledListeners = Settings.Secure.getString( + appContext.contentResolver, + "enabled_notification_listeners" + ) + if (enabledListeners == null || enabledListeners.isEmpty()) { + return false + } + val fullName = listenerComponent.flattenToString() + val shortName = listenerComponent.flattenToShortString() + return enabledListeners.contains(fullName) + || enabledListeners.contains(shortName) + || enabledListeners.contains(appContext.packageName) + } + + @JvmStatic + fun isEvenNotificationActive(context: Context?): Boolean { + val hasAccess = isNotificationAccessEnabled(context) + if (!hasAccess) { + Log.i(TAG, "Can't check for Even Realities app notification, permission is not enabled") + return false + } + val hasNotfication = FaceclawMediaNotificationListenerService.hasActiveNotificationTitle(EVEN_NOTIFICATION_TITLE) + Log.i(TAG, "isEvenNotificationActive: $hasNotfication") + return hasNotfication + } + + /** Returns "opened", "not-installed", or "failed" for the phone UI. */ + @JvmStatic + fun openEvenAppSettings(context: Context?): String { + if (context == null) { + return "failed" + } + val appContext = context.applicationContext + try { + // Settings may launch successfully and then immediately close for a + // missing package, so check before starting the activity. + appContext.packageManager.getApplicationInfo(EVEN_PACKAGE_NAME, 0) + val intent = Intent(Settings.ACTION_APPLICATION_DETAILS_SETTINGS) + intent.data = Uri.parse("package:$EVEN_PACKAGE_NAME") + intent.addFlags(Intent.FLAG_ACTIVITY_NEW_TASK) + appContext.startActivity(intent) + return "opened" + } catch (e: PackageManager.NameNotFoundException) { + return "not-installed" + } catch (t: Exception) { + Log.w(TAG, "failed to open Even app settings", t) + return "failed" + } + } + + @JvmStatic + fun openNotificationAccessSettings(context: Context?) { + if (context == null) { + return + } + val appContext = context.applicationContext + val intent = Intent(Settings.ACTION_NOTIFICATION_LISTENER_SETTINGS) + intent.addFlags(Intent.FLAG_ACTIVITY_NEW_TASK) + try { + appContext.startActivity(intent) + } catch (t: Throwable) { + Log.w(TAG, "failed to open notification access settings", t) + } + } + } +} diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawEvenHubDocumentStart.java b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawEvenHubDocumentStart.java deleted file mode 100644 index 2f9c0e90..00000000 --- a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawEvenHubDocumentStart.java +++ /dev/null @@ -1,50 +0,0 @@ -package com.faceclaw.app; - -import android.util.Log; -import android.webkit.WebView; - -import androidx.webkit.WebViewCompat; -import androidx.webkit.WebViewFeature; - -import java.util.Collections; -import java.util.Set; - -/** - * Injects the EvenHub bridge shim at document start into a WebView that loads - * a page over the real network (the developer "Load app from URL" flow). - * - * Packaged apps get the shim spliced into the HTML they are served - * (FaceclawEvenHubWebViewClient), which is not an option for a page fetched - * from a live server. WebViewCompat.addDocumentStartJavaScript is the - * supported way to run script before any page script, and unlike an - * onPageStarted evaluateJavascript it is guaranteed to win the race — apps - * poll for window.flutter_inappwebview with a few seconds' patience, but the - * timer shim must be in place before the app captures setTimeout. - * - * Requires WebView 83+; the caller falls back to onPageStarted injection when - * this returns false. - */ -public final class FaceclawEvenHubDocumentStart { - private static final String TAG = "FaceclawEvenHub"; - - private FaceclawEvenHubDocumentStart() {} - - /** - * @param originRule an allowed-origin rule, e.g. "https://example.com" or "*". - * @return whether the script was registered. - */ - public static boolean install(WebView webView, String script, String originRule) { - if (!WebViewFeature.isFeatureSupported(WebViewFeature.DOCUMENT_START_SCRIPT)) { - Log.w(TAG, "document-start scripts unsupported; falling back to onPageStarted"); - return false; - } - try { - Set rules = Collections.singleton(originRule); - WebViewCompat.addDocumentStartJavaScript(webView, script, rules); - return true; - } catch (Throwable error) { - Log.w(TAG, "addDocumentStartJavaScript failed for " + originRule, error); - return false; - } - } -} diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawEvenHubDocumentStart.kt b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawEvenHubDocumentStart.kt new file mode 100644 index 00000000..3f50eaec --- /dev/null +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawEvenHubDocumentStart.kt @@ -0,0 +1,50 @@ +package com.faceclaw.app + +import android.util.Log +import android.webkit.WebView + +import androidx.webkit.WebViewCompat +import androidx.webkit.WebViewFeature + +import java.util.Collections + +/** + * Injects the EvenHub bridge shim at document start into a WebView that loads + * a page over the real network (the developer "Load app from URL" flow). + * + * Packaged apps get the shim spliced into the HTML they are served + * (FaceclawEvenHubWebViewClient), which is not an option for a page fetched + * from a live server. WebViewCompat.addDocumentStartJavaScript is the + * supported way to run script before any page script, and unlike an + * onPageStarted evaluateJavascript it is guaranteed to win the race — apps + * poll for window.flutter_inappwebview with a few seconds' patience, but the + * timer shim must be in place before the app captures setTimeout. + * + * Requires WebView 83+; the caller falls back to onPageStarted injection when + * this returns false. + */ +class FaceclawEvenHubDocumentStart private constructor() { + companion object { + private const val TAG = "FaceclawEvenHub" + + /** + * @param originRule an allowed-origin rule, e.g. "https://example.com" or "*". + * @return whether the script was registered. + */ + @JvmStatic + fun install(webView: WebView, script: String, originRule: String): Boolean { + if (!WebViewFeature.isFeatureSupported(WebViewFeature.DOCUMENT_START_SCRIPT)) { + Log.w(TAG, "document-start scripts unsupported; falling back to onPageStarted") + return false + } + try { + val rules: Set = Collections.singleton(originRule) + WebViewCompat.addDocumentStartJavaScript(webView, script, rules) + return true + } catch (error: Throwable) { + Log.w(TAG, "addDocumentStartJavaScript failed for $originRule", error) + return false + } + } + } +} diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawEvenHubJsBridge.java b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawEvenHubJsBridge.java deleted file mode 100644 index 6e40bbef..00000000 --- a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawEvenHubJsBridge.java +++ /dev/null @@ -1,34 +0,0 @@ -package com.faceclaw.app; - -import android.os.Handler; -import android.os.Looper; -import android.webkit.JavascriptInterface; - -/** - * The object injected into a hosted EvenHub app's WebView as - * window.__faceclawEvenHub. The document-start shim (served inline by - * FaceclawEvenHubWebViewClient) wraps it in a promise-returning - * window.flutter_inappwebview.callHandler, which is what the EvenHub SDK - * actually talks to. - * - * JavascriptInterface methods run on a WebView-internal thread; everything - * is bounced to the main thread before touching the NativeScript listener. - */ -public class FaceclawEvenHubJsBridge { - private final FaceclawEvenHubListener listener; - private final Handler mainHandler = new Handler(Looper.getMainLooper()); - - public FaceclawEvenHubJsBridge(FaceclawEvenHubListener listener) { - this.listener = listener; - } - - @JavascriptInterface - public void postMessage(final String handlerName, final String argsJson, final int callId) { - mainHandler.post(new Runnable() { - @Override - public void run() { - listener.onEvenAppMessage(handlerName, argsJson, callId); - } - }); - } -} diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawEvenHubJsBridge.kt b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawEvenHubJsBridge.kt new file mode 100644 index 00000000..90c8c1e7 --- /dev/null +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawEvenHubJsBridge.kt @@ -0,0 +1,26 @@ +package com.faceclaw.app + +import android.os.Handler +import android.os.Looper +import android.webkit.JavascriptInterface + +/** + * The object injected into a hosted EvenHub app's WebView as + * window.__faceclawEvenHub. The document-start shim (served inline by + * FaceclawEvenHubWebViewClient) wraps it in a promise-returning + * window.flutter_inappwebview.callHandler, which is what the EvenHub SDK + * actually talks to. + * + * JavascriptInterface methods run on a WebView-internal thread; everything + * is bounced to the main thread before touching the NativeScript listener. + */ +class FaceclawEvenHubJsBridge(private val listener: FaceclawEvenHubListener) { + private val mainHandler = Handler(Looper.getMainLooper()) + + @JavascriptInterface + fun postMessage(handlerName: String?, argsJson: String?, callId: Int) { + mainHandler.post { + listener.onEvenAppMessage(handlerName, argsJson, callId) + } + } +} diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawEvenHubWebView.java b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawEvenHubWebView.kt similarity index 78% rename from App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawEvenHubWebView.java rename to App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawEvenHubWebView.kt index 5a590902..4f5024a6 100644 --- a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawEvenHubWebView.java +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawEvenHubWebView.kt @@ -1,8 +1,8 @@ -package com.faceclaw.app; +package com.faceclaw.app -import android.content.Context; -import android.view.View; -import android.webkit.WebView; +import android.content.Context +import android.view.View +import android.webkit.WebView /** * A WebView for hosting an EvenHub app that keeps rendering and running JS even @@ -25,16 +25,11 @@ import android.webkit.WebView; * service regardless of the phone's screen, so "always rendering" is what we * actually want; there is no on-phone cost while the app is unseen. */ -public class FaceclawEvenHubWebView extends WebView { +class FaceclawEvenHubWebView(context: Context) : WebView(context) { - public FaceclawEvenHubWebView(Context context) { - super(context); - } - - @Override - protected void onWindowVisibilityChanged(int visibility) { + override fun onWindowVisibilityChanged(visibility: Int) { // Never propagate a hidden window: keep Chromium's page-visibility // "visible" so the renderer isn't frozen when Faceclaw backgrounds. - super.onWindowVisibilityChanged(View.VISIBLE); + super.onWindowVisibilityChanged(View.VISIBLE) } } diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawEvenHubWebViewClient.java b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawEvenHubWebViewClient.java deleted file mode 100644 index f8c21348..00000000 --- a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawEvenHubWebViewClient.java +++ /dev/null @@ -1,172 +0,0 @@ -package com.faceclaw.app; - -import android.os.Handler; -import android.os.Looper; -import android.webkit.WebResourceRequest; -import android.webkit.WebResourceResponse; -import android.webkit.WebView; -import android.webkit.WebViewClient; -import android.net.Uri; -import android.util.Log; - -import java.io.ByteArrayInputStream; -import java.io.File; -import java.io.FileInputStream; -import java.io.IOException; -import java.io.InputStream; -import java.nio.charset.StandardCharsets; -import java.util.Locale; - -/** - * Serves an unpacked EvenHub app's files to its WebView from a fake per-app - * https origin, entirely offline. HTML responses get the host bridge shim - * injected at the top of the document, guaranteeing - * window.flutter_inappwebview exists before any app JS runs (apps race - * waitForEvenAppBridge against 4-6s timeouts and silently fall back to demo - * modes if the handler appears late). - * - * shouldInterceptRequest runs on a WebView-internal thread; it only touches - * files, never the NativeScript runtime. Requests to other hosts fall - * through to the network (the manifest whitelist is not yet enforced). - * - * A developer-loaded app ("Load app from URL") has no local root: host is - * empty, nothing is intercepted, and the page loads normally from its server. - * The shim is registered as a document-start script instead - * (FaceclawEvenHubDocumentStart); injectOnPageStarted is the fallback for - * WebView versions that lack that API. - */ -public class FaceclawEvenHubWebViewClient extends WebViewClient { - private static final String TAG = "FaceclawEvenHub"; - - private final String rootDir; - private final String host; - private final String injectScript; - private final boolean injectOnPageStarted; - private final FaceclawEvenHubListener listener; - private final Handler mainHandler = new Handler(Looper.getMainLooper()); - - public FaceclawEvenHubWebViewClient( - String rootDir, - String host, - String injectScript, - boolean injectOnPageStarted, - FaceclawEvenHubListener listener) { - this.rootDir = rootDir; - this.host = host; - this.injectScript = injectScript; - this.injectOnPageStarted = injectOnPageStarted; - this.listener = listener; - } - - @Override - public WebResourceResponse shouldInterceptRequest(WebView view, WebResourceRequest request) { - if (host == null || host.isEmpty()) { - return null; // Remote app: everything comes from the network. - } - Uri url = request.getUrl(); - if (url == null || !host.equals(url.getHost())) { - return null; // External hosts: normal network handling. - } - String path = url.getPath(); - if (path == null || path.isEmpty() || path.equals("/")) { - path = "/index.html"; - } - try { - File root = new File(rootDir).getCanonicalFile(); - File target = new File(root, path.substring(1)).getCanonicalFile(); - if (!target.getPath().startsWith(root.getPath()) || !target.isFile()) { - Log.w(TAG, "404 " + path); - return new WebResourceResponse( - "text/plain", "utf-8", - new ByteArrayInputStream("not found".getBytes(StandardCharsets.UTF_8))); - } - String mime = mimeTypeFor(target.getName()); - if (mime.equals("text/html")) { - return new WebResourceResponse(mime, "utf-8", new ByteArrayInputStream(injectIntoHtml(target))); - } - InputStream stream = new FileInputStream(target); - return new WebResourceResponse(mime, null, stream); - } catch (IOException e) { - Log.e(TAG, "serve failed for " + path, e); - return new WebResourceResponse( - "text/plain", "utf-8", - new ByteArrayInputStream("error".getBytes(StandardCharsets.UTF_8))); - } - } - - @Override - public void onPageStarted(WebView view, String url, android.graphics.Bitmap favicon) { - super.onPageStarted(view, url, favicon); - if (!injectOnPageStarted) return; - // Best-effort: the document has committed but the parser has not run - // the page's own scripts yet, so this usually lands first. Only used - // when document-start scripts are unavailable. - view.evaluateJavascript(injectScript, null); - } - - @Override - public void onPageFinished(WebView view, final String url) { - mainHandler.post(new Runnable() { - @Override - public void run() { - listener.onPageFinished(url); - } - }); - } - - private byte[] injectIntoHtml(File file) throws IOException { - byte[] raw = readAll(file); - String html = new String(raw, StandardCharsets.UTF_8); - String tag = ""; - // After if present, else before everything. - int headIndex = indexOfIgnoreCase(html, "= 0) { - int close = html.indexOf('>', headIndex); - if (close >= 0) { - return (html.substring(0, close + 1) + tag + html.substring(close + 1)) - .getBytes(StandardCharsets.UTF_8); - } - } - return (tag + html).getBytes(StandardCharsets.UTF_8); - } - - private static int indexOfIgnoreCase(String haystack, String needle) { - return haystack.toLowerCase(Locale.ROOT).indexOf(needle); - } - - private static byte[] readAll(File file) throws IOException { - FileInputStream in = new FileInputStream(file); - try { - byte[] data = new byte[(int) file.length()]; - int off = 0; - while (off < data.length) { - int n = in.read(data, off, data.length - off); - if (n < 0) throw new IOException("short read: " + file); - off += n; - } - return data; - } finally { - in.close(); - } - } - - private static String mimeTypeFor(String name) { - String lower = name.toLowerCase(Locale.ROOT); - if (lower.endsWith(".html") || lower.endsWith(".htm")) return "text/html"; - if (lower.endsWith(".js") || lower.endsWith(".mjs")) return "application/javascript"; - if (lower.endsWith(".css")) return "text/css"; - if (lower.endsWith(".json")) return "application/json"; - if (lower.endsWith(".png")) return "image/png"; - if (lower.endsWith(".jpg") || lower.endsWith(".jpeg")) return "image/jpeg"; - if (lower.endsWith(".gif")) return "image/gif"; - if (lower.endsWith(".svg")) return "image/svg+xml"; - if (lower.endsWith(".webp")) return "image/webp"; - if (lower.endsWith(".ico")) return "image/x-icon"; - if (lower.endsWith(".wasm")) return "application/wasm"; - if (lower.endsWith(".woff")) return "font/woff"; - if (lower.endsWith(".woff2")) return "font/woff2"; - if (lower.endsWith(".ttf")) return "font/ttf"; - if (lower.endsWith(".txt") || lower.endsWith(".map")) return "text/plain"; - return "application/octet-stream"; - } -} diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawEvenHubWebViewClient.kt b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawEvenHubWebViewClient.kt new file mode 100644 index 00000000..bec2b8a3 --- /dev/null +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawEvenHubWebViewClient.kt @@ -0,0 +1,158 @@ +package com.faceclaw.app + +import android.os.Handler +import android.os.Looper +import android.webkit.WebResourceRequest +import android.webkit.WebResourceResponse +import android.webkit.WebView +import android.webkit.WebViewClient +import android.net.Uri +import android.util.Log + +import java.io.ByteArrayInputStream +import java.io.File +import java.io.FileInputStream +import java.io.IOException +import java.io.InputStream +import java.nio.charset.StandardCharsets +import java.util.Locale + +/** + * Serves an unpacked EvenHub app's files to its WebView from a fake per-app + * https origin, entirely offline. HTML responses get the host bridge shim + * injected at the top of the document, guaranteeing + * window.flutter_inappwebview exists before any app JS runs (apps race + * waitForEvenAppBridge against 4-6s timeouts and silently fall back to demo + * modes if the handler appears late). + * + * shouldInterceptRequest runs on a WebView-internal thread; it only touches + * files, never the NativeScript runtime. Requests to other hosts fall + * through to the network (the manifest whitelist is not yet enforced). + * + * A developer-loaded app ("Load app from URL") has no local root: host is + * empty, nothing is intercepted, and the page loads normally from its server. + * The shim is registered as a document-start script instead + * (FaceclawEvenHubDocumentStart); injectOnPageStarted is the fallback for + * WebView versions that lack that API. + */ +class FaceclawEvenHubWebViewClient( + private val rootDir: String?, + private val host: String?, + private val injectScript: String, + private val injectOnPageStarted: Boolean, + private val listener: FaceclawEvenHubListener +) : WebViewClient() { + private val mainHandler = Handler(Looper.getMainLooper()) + + override fun shouldInterceptRequest(view: WebView, request: WebResourceRequest): WebResourceResponse? { + if (host == null || host.isEmpty()) { + return null // Remote app: everything comes from the network. + } + val url: Uri? = request.url + if (url == null || host != url.host) { + return null // External hosts: normal network handling. + } + var path = url.path + if (path == null || path.isEmpty() || path == "/") { + path = "/index.html" + } + try { + val root = File(rootDir).canonicalFile + val target = File(root, path.substring(1)).canonicalFile + if (!target.path.startsWith(root.path) || !target.isFile) { + Log.w(TAG, "404 $path") + return WebResourceResponse( + "text/plain", "utf-8", + ByteArrayInputStream("not found".toByteArray(StandardCharsets.UTF_8))) + } + val mime = mimeTypeFor(target.name) + if (mime == "text/html") { + return WebResourceResponse(mime, "utf-8", ByteArrayInputStream(injectIntoHtml(target))) + } + val stream: InputStream = FileInputStream(target) + return WebResourceResponse(mime, null, stream) + } catch (e: IOException) { + Log.e(TAG, "serve failed for $path", e) + return WebResourceResponse( + "text/plain", "utf-8", + ByteArrayInputStream("error".toByteArray(StandardCharsets.UTF_8))) + } + } + + override fun onPageStarted(view: WebView, url: String?, favicon: android.graphics.Bitmap?) { + super.onPageStarted(view, url, favicon) + if (!injectOnPageStarted) return + // Best-effort: the document has committed but the parser has not run + // the page's own scripts yet, so this usually lands first. Only used + // when document-start scripts are unavailable. + view.evaluateJavascript(injectScript, null) + } + + override fun onPageFinished(view: WebView, url: String?) { + mainHandler.post { + listener.onPageFinished(url) + } + } + + @Throws(IOException::class) + private fun injectIntoHtml(file: File): ByteArray { + val raw = readAll(file) + val html = String(raw, StandardCharsets.UTF_8) + val tag = "" + // After if present, else before everything. + val headIndex = indexOfIgnoreCase(html, "= 0) { + val close = html.indexOf('>', headIndex) + if (close >= 0) { + return (html.substring(0, close + 1) + tag + html.substring(close + 1)) + .toByteArray(StandardCharsets.UTF_8) + } + } + return (tag + html).toByteArray(StandardCharsets.UTF_8) + } + + companion object { + private const val TAG = "FaceclawEvenHub" + + private fun indexOfIgnoreCase(haystack: String, needle: String): Int { + return haystack.lowercase(Locale.ROOT).indexOf(needle) + } + + @Throws(IOException::class) + private fun readAll(file: File): ByteArray { + val input = FileInputStream(file) + try { + val data = ByteArray(file.length().toInt()) + var off = 0 + while (off < data.size) { + val n = input.read(data, off, data.size - off) + if (n < 0) throw IOException("short read: $file") + off += n + } + return data + } finally { + input.close() + } + } + + private fun mimeTypeFor(name: String): String { + val lower = name.lowercase(Locale.ROOT) + if (lower.endsWith(".html") || lower.endsWith(".htm")) return "text/html" + if (lower.endsWith(".js") || lower.endsWith(".mjs")) return "application/javascript" + if (lower.endsWith(".css")) return "text/css" + if (lower.endsWith(".json")) return "application/json" + if (lower.endsWith(".png")) return "image/png" + if (lower.endsWith(".jpg") || lower.endsWith(".jpeg")) return "image/jpeg" + if (lower.endsWith(".gif")) return "image/gif" + if (lower.endsWith(".svg")) return "image/svg+xml" + if (lower.endsWith(".webp")) return "image/webp" + if (lower.endsWith(".ico")) return "image/x-icon" + if (lower.endsWith(".wasm")) return "application/wasm" + if (lower.endsWith(".woff")) return "font/woff" + if (lower.endsWith(".woff2")) return "font/woff2" + if (lower.endsWith(".ttf")) return "font/ttf" + if (lower.endsWith(".txt") || lower.endsWith(".map")) return "text/plain" + return "application/octet-stream" + } + } +} diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawEvenHubWebViewHost.java b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawEvenHubWebViewHost.kt similarity index 56% rename from App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawEvenHubWebViewHost.java rename to App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawEvenHubWebViewHost.kt index d50d00ae..7e563d0e 100644 --- a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawEvenHubWebViewHost.java +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawEvenHubWebViewHost.kt @@ -1,16 +1,15 @@ -package com.faceclaw.app; +package com.faceclaw.app -import android.app.Activity; -import android.os.Build; -import android.os.Handler; -import android.os.Looper; -import android.view.View; -import android.view.ViewGroup; -import android.webkit.WebView; -import android.widget.FrameLayout; +import android.app.Activity +import android.os.Build +import android.os.Handler +import android.os.Looper +import android.view.View +import android.view.ViewGroup +import android.webkit.WebView +import android.widget.FrameLayout -import java.util.ArrayList; -import java.util.List; +import java.util.ArrayList /** * Keeps EvenHub app WebViews alive and rendering while the phone shows the @@ -49,109 +48,113 @@ import java.util.List; * activity is foreground. When Faceclaw itself is backgrounded, the window goes * invisible and Chromium freezes the renderer — the timer/rAF ticks and pushed * input events then queue up and only run once foregrounded again. Two things - * prevent that: the WebViews are {@link FaceclawEvenHubWebView}, which lies to + * prevent that: the WebViews are [FaceclawEvenHubWebView], which lies to * Chromium about window visibility so the page never goes hidden; and each is * pinned to IMPORTANT renderer priority (not waived when not visible) with * timers/onResume asserted on attach. The main Looper keeps ticking under the * foreground service, so apps keep driving their glasses windows in the * background. */ -public class FaceclawEvenHubWebViewHost { - private static FaceclawEvenHubWebViewHost instance; +class FaceclawEvenHubWebViewHost { + private var overlay: FrameLayout? = null + private var shown = false - /** How often the host fires the JS timer queue. 60Hz keeps games smooth. */ - private static final long TICK_MS = 16; - - private FrameLayout overlay; - private boolean shown = false; - - private final Handler mainHandler = new Handler(Looper.getMainLooper()); - private final List webViews = new ArrayList<>(); - private boolean ticking = false; - private final Runnable ticker = new Runnable() { - @Override - public void run() { - for (int i = 0; i < webViews.size(); i++) { - webViews.get(i).evaluateJavascript( - "window.__fcTimerTick&&__fcTimerTick();window.__fcRafTick&&__fcRafTick()", null); + private val mainHandler = Handler(Looper.getMainLooper()) + private val webViews: MutableList = ArrayList() + private var ticking = false + private val ticker: Runnable = object : Runnable { + override fun run() { + for (i in 0 until webViews.size) { + webViews[i].evaluateJavascript( + "window.__fcTimerTick&&__fcTimerTick();window.__fcRafTick&&__fcRafTick()", null) } - if (ticking) mainHandler.postDelayed(this, TICK_MS); + if (ticking) mainHandler.postDelayed(this, TICK_MS) } - }; - - public static synchronized FaceclawEvenHubWebViewHost getInstance() { - if (instance == null) instance = new FaceclawEvenHubWebViewHost(); - return instance; } - private FrameLayout ensureOverlay(Activity activity) { - if (overlay != null) return overlay; - ViewGroup content = activity.findViewById(android.R.id.content); - overlay = new FrameLayout(activity); + companion object { + private var instance: FaceclawEvenHubWebViewHost? = null + + /** How often the host fires the JS timer queue. 60Hz keeps games smooth. */ + private const val TICK_MS = 16L + + @JvmStatic + @Synchronized + fun getInstance(): FaceclawEvenHubWebViewHost { + if (instance == null) instance = FaceclawEvenHubWebViewHost() + return instance!! + } + } + + private fun ensureOverlay(activity: Activity): FrameLayout { + overlay?.let { return it } + val content = activity.findViewById(android.R.id.content) + val created = FrameLayout(activity) + overlay = created // index 0 = behind the NativeScript content view. - content.addView(overlay, 0, new FrameLayout.LayoutParams( - ViewGroup.LayoutParams.MATCH_PARENT, ViewGroup.LayoutParams.MATCH_PARENT)); - return overlay; + content.addView(created, 0, FrameLayout.LayoutParams( + ViewGroup.LayoutParams.MATCH_PARENT, ViewGroup.LayoutParams.MATCH_PARENT)) + return created } /** Add a WebView to the host, full-size and rendering, hidden behind the UI. */ - public void attach(Activity activity, WebView web) { - FrameLayout o = ensureOverlay(activity); - web.setVisibility(View.VISIBLE); - if (web.getParent() == null) { - o.addView(web, new FrameLayout.LayoutParams( - ViewGroup.LayoutParams.MATCH_PARENT, ViewGroup.LayoutParams.MATCH_PARENT)); + fun attach(activity: Activity, web: WebView) { + val o = ensureOverlay(activity) + web.visibility = View.VISIBLE + if (web.parent == null) { + o.addView(web, FrameLayout.LayoutParams( + ViewGroup.LayoutParams.MATCH_PARENT, ViewGroup.LayoutParams.MATCH_PARENT)) } // resumeTimers() is process-global; make sure nothing left timers paused. // onResume() undoes any per-WebView pause. Together with // FaceclawEvenHubWebView faking window visibility, this keeps the page // running JS while Faceclaw is backgrounded. - web.resumeTimers(); - web.onResume(); + web.resumeTimers() + web.onResume() // Keep the renderer process at IMPORTANT priority even when the WebView // isn't visible (waivedWhenNotVisible=false), so Android doesn't // deprioritize/kill it in the background. if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) { - web.setRendererPriorityPolicy(WebView.RENDERER_PRIORITY_IMPORTANT, false); + web.setRendererPriorityPolicy(WebView.RENDERER_PRIORITY_IMPORTANT, false) } - if (!webViews.contains(web)) webViews.add(web); + if (!webViews.contains(web)) webViews.add(web) if (!ticking) { - ticking = true; - mainHandler.postDelayed(ticker, TICK_MS); + ticking = true + mainHandler.postDelayed(ticker, TICK_MS) } } /** Bring an app's WebView to the front so it is visible on the phone. */ - public void showOnPhone(WebView web) { - if (overlay == null) return; - web.bringToFront(); - overlay.bringToFront(); - shown = true; + fun showOnPhone(web: WebView) { + val o = overlay ?: return + web.bringToFront() + o.bringToFront() + shown = true } /** Send the overlay back behind the NativeScript UI. */ - public void hideOnPhone() { - shown = false; - if (overlay == null) return; - ViewGroup content = (ViewGroup) overlay.getParent(); + fun hideOnPhone() { + shown = false + val o = overlay ?: return + val content = o.parent as ViewGroup? if (content != null) { - content.removeView(overlay); - content.addView(overlay, 0); + content.removeView(o) + content.addView(o, 0) } } - public boolean isShown() { - return shown; + fun isShown(): Boolean { + return shown } /** Remove and destroy a WebView (its app is closing). */ - public void detach(WebView web) { - webViews.remove(web); + fun detach(web: WebView) { + webViews.remove(web) if (webViews.isEmpty() && ticking) { - ticking = false; - mainHandler.removeCallbacks(ticker); + ticking = false + mainHandler.removeCallbacks(ticker) } - if (overlay != null) overlay.removeView(web); - web.destroy(); + overlay?.removeView(web) + web.destroy() } } diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawFirmwareFlasher.java b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawFirmwareFlasher.java deleted file mode 100644 index 3a941ac6..00000000 --- a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawFirmwareFlasher.java +++ /dev/null @@ -1,695 +0,0 @@ -package com.faceclaw.app; - -import android.bluetooth.BluetoothGatt; -import android.content.Context; -import android.os.Handler; -import android.os.Looper; -import android.os.SystemClock; -import android.util.Log; - -import java.io.ByteArrayOutputStream; -import java.io.FileInputStream; -import java.io.IOException; -import java.nio.charset.StandardCharsets; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.LinkedBlockingQueue; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.TimeoutException; - -/** - * Ports g2flash.py's OTA flash procedure to Android. Streams the EVENOTA - * firmware container to each lens over the firmware-data service using the - * aa21 envelope (BleProtocol.framePb) and the c0/c1 control/data protocol. - * - * Each lens connection completes the sid-0x80 security-auth exchange before - * OTA BEGIN — firmware 2.2.9 closes unauthenticated links ~30 s in, killing - * the transfer mid-component. No control-channel traffic is injected during - * the transfer itself: official OTA captures show none between BEGIN and the - * final END, heartbeats demonstrably do not satisfy the 2.2.9 deadline, and a - * concurrent control writer would interleave with the marker/data - * transactions. (See ../notes/ble-connections-2.2.9.md.) - * - * Lenses are flashed one at a time (left, then right). Finishing either lens - * reboots BOTH lenses, so the second lens is briefly unreachable — the reconnect - * for it uses a generous window rather than failing fast. - * - * Separate from FaceclawBleCommunicator on purpose: this speaks the stock OTA - * protocol and must run before the custom firmware exists. It reuses the GATT - * wrapper (FaceclawBleManager) and framing (BleProtocol). - */ -public class FaceclawFirmwareFlasher implements FaceclawBleListener { - private static final String TAG = "FaceclawFlasher"; - - // OTA message types (envelope sid byte) and control opcodes. - private static final int SID_CTRL = 0xc0; - private static final int SID_DATA = 0xc1; - private static final int FLAG_OTA = 0x00; - private static final int OP_BEGIN = 0x00; - private static final int OP_FILE_CHECK = 0x01; - private static final int OP_BLOCK = 0x02; - private static final int OP_END = 0x03; - - private static final int BLOCK_SIZE = 4096; - private static final int BLOCK_ACK_TIMEOUT_MS = 4_000; - private static final int CTRL_ACK_TIMEOUT_MS = 8_000; - private static final int BLOCK_NAK_RETRIES = 3; - private static final int COMPONENT_RETRIES = 3; - - // Reconnect windows. The first lens connects from an idle device; the second - // must ride out the post-first-lens reboot of both lenses. - private static final int FIRST_LENS_CONNECT_WINDOW_MS = 30_000; - private static final int SECOND_LENS_CONNECT_WINDOW_MS = 120_000; - private static final int REBOOT_SETTLE_MS = 5_000; - private static final int NOTIFY_SETTLE_MS = 2_500; - private static final int RETRY_DELAY_MS = 2_500; - - // END ack statuses that mean "component accepted": SUCCESS, UPDATING, SYS_RESTART. - private static final int[] END_OK = new int[] {0, 8, 9}; - - // Firmware layout expectations (mirrors g2flash.py). Firmware 2.2.4 has 5 - // components; 2.2.6 has 6. - private static final int MIN_SEGMENTS = 5; - private static final int MAX_SEGMENTS = 6; - private static final String REQUIRED_SEGMENT = "ota/s200_firmware_ota.bin"; - private static final long APP_LOAD_ADDR = 0x00438000L; - private static final long APP_MAX_END = 0x007F0000L; - private static final int APP_PREAMBLE = 0x20; - - private final Context context; - private final String leftAddress; - private final String rightAddress; - private final String firmwarePath; - private final FaceclawBleManager bleManager; - private final Handler mainHandler = new Handler(Looper.getMainLooper()); - - private final Object lock = new Object(); - private final LinkedBlockingQueue dataAcks = new LinkedBlockingQueue<>(); - - private volatile FaceclawFirmwareFlasherListener listener; - private volatile Thread worker; - private volatile boolean cancelled = false; - - private int nextSeq = 0; - private int nextAuthMagic = 0x60; - private int pendingAuthMagic = -1; - private CountDownLatch authLatch; - private byte[] authAckPb; - - public FaceclawFirmwareFlasher(Context context, String rightAddress, String leftAddress, String firmwarePath) { - this.context = context.getApplicationContext(); - this.rightAddress = rightAddress == null ? "" : rightAddress; - this.leftAddress = leftAddress == null ? "" : leftAddress; - this.firmwarePath = firmwarePath == null ? "" : firmwarePath; - this.bleManager = new FaceclawBleManager(this.context); - this.bleManager.setListener(this); - } - - public void setListener(FaceclawFirmwareFlasherListener listener) { - this.listener = listener; - } - - public void start() { - synchronized (lock) { - if (worker != null) { - return; - } - worker = new Thread(this::run, "faceclaw-flasher"); - worker.start(); - } - } - - public void cancel() { - cancelled = true; - Thread w = worker; - if (w != null) { - w.interrupt(); - } - } - - public void close() { - cancel(); - try { - bleManager.close(); - } catch (Exception ignored) { - } - } - - private void run() { - try { - if (leftAddress.trim().isEmpty() || rightAddress.trim().isEmpty()) { - throw new IllegalStateException("Both lens addresses are required to flash."); - } - - emitState("validating", ""); - byte[] img = readFile(firmwarePath); - List segs = validate(img); - emitLog("firmware validated: " + segs.size() + " components, " + img.length + " bytes"); - - flashLens("left", leftAddress, img, segs, FIRST_LENS_CONNECT_WINDOW_MS); - if (cancelled) { - emitState("error", "Cancelled."); - emitComplete(false, "Cancelled after the left lens."); - return; - } - - emitState("rebooting", "Left lens done. Both lenses reboot briefly; reconnecting for the right lens."); - sleepInterruptibly(REBOOT_SETTLE_MS); - flashLens("right", rightAddress, img, segs, SECOND_LENS_CONNECT_WINDOW_MS); - - emitState("done", ""); - emitComplete(true, "Both lenses flashed. The glasses are rebooting into the custom firmware."); - } catch (Exception e) { - String message = e.getMessage() == null ? e.toString() : e.getMessage(); - if (cancelled) { - message = "Cancelled."; - } - emitState("error", message); - emitComplete(false, message); - } finally { - teardown(); - } - } - - private void flashLens(String lens, String address, byte[] img, List segs, int connectWindowMs) - throws TimeoutException { - emitState("connecting", lens); - connectLensResilient(lens, address, connectWindowMs); - sleepInterruptibly(NOTIFY_SETTLE_MS); - drainAcks(); - // Fresh envelope counter for the OTA stream (the auth exchange during - // bring-up used its own); mirrors the stock sequence. - resetSeq(); - - emitState("flashing", lens); - int beginStatus = sendCtrlAndWait(address, OP_BEGIN, EMPTY, CTRL_ACK_TIMEOUT_MS); - if (!isEndOk(beginStatus)) { - emitLog("warning: unexpected begin status " + beginStatus + "; continuing"); - } - // Progress is reported in bytes across the whole image: the segments - // are one large firmware blob plus several small ones, so a bar - // stepped per segment would sit still through the big one and then - // sprint through the rest. - long totalBytes = 0; - for (Segment seg : segs) { - totalBytes += seg.ps; - } - long bytesBefore = 0; - for (int i = 0; i < segs.size(); i++) { - if (cancelled) { - throw new IllegalStateException("Cancelled."); - } - flashComponentWithRetry(lens, address, i, segs.size(), segs.get(i), img, bytesBefore, totalBytes); - bytesBefore += segs.get(i).ps; - } - emitLog(lens + " lens: all components verified"); - try { - bleManager.disconnect(address); - } catch (Exception ignored) { - } - } - - private void flashComponentWithRetry(String lens, String address, int index, int count, Segment seg, byte[] img, - long bytesBefore, long totalBytes) { - for (int attempt = 0; attempt < COMPONENT_RETRIES; attempt++) { - if (attempt > 0) { - emitLog(seg.name + ": re-flash attempt " + (attempt + 1) + "/" + COMPONENT_RETRIES); - } - int endStatus; - try { - endStatus = flashComponent(lens, address, index, count, seg, img, bytesBefore, totalBytes); - } catch (TimeoutException | RuntimeException e) { - emitLog(seg.name + ": block phase failed: " + e.getMessage()); - endStatus = -1; - } - if (isEndOk(endStatus)) { - emitLog(seg.name + ": END verify OK (status " + endStatus + ")"); - return; - } - if (endStatus >= 0) { - emitLog(seg.name + ": END verify FAILED (status " + endStatus + ")"); - } - drainAcks(); - sleepInterruptibly(1_500); - } - throw new IllegalStateException("component " + seg.name + " failed after " + COMPONENT_RETRIES + " attempts"); - } - - private int flashComponent(String lens, String address, int index, int count, Segment seg, byte[] img, - long bytesBefore, long totalBytes) throws TimeoutException { - byte[] sub = Arrays.copyOfRange(img, seg.off, seg.off + 128); - int ps = seg.ps; - int payloadStart = seg.off + 128; - - int checkStatus = sendCtrlAndWait(address, OP_FILE_CHECK, sub, CTRL_ACK_TIMEOUT_MS); - if (checkStatus != 0) { - throw new IllegalStateException("FILE_CHECK rejected status=" + checkStatus); - } - - int blockCount = (ps + BLOCK_SIZE - 1) / BLOCK_SIZE; - for (int b = 0; b < blockCount; b++) { - if (cancelled) { - throw new IllegalStateException("Cancelled."); - } - int start = payloadStart + b * BLOCK_SIZE; - int end = Math.min(start + BLOCK_SIZE, payloadStart + ps); - byte[] block = Arrays.copyOfRange(img, start, end); - - boolean accepted = false; - for (int tries = 0; tries < BLOCK_NAK_RETRIES; tries++) { - int status = sendBlock(address, block); // throws TimeoutException -> component re-flash - if (status == 0) { - accepted = true; - break; - } - emitLog(seg.name + ": block " + b + "/" + blockCount + " NAK=" + status - + " resend " + (tries + 1) + "/" + BLOCK_NAK_RETRIES); - } - if (!accepted) { - throw new IllegalStateException("block " + b + " NAK'd " + BLOCK_NAK_RETRIES + " times"); - } - if (b % 20 == 0 || b == blockCount - 1) { - long bytesSent = bytesBefore + Math.min((long) (b + 1) * BLOCK_SIZE, (long) ps); - emitProgress(lens, index + 1, count, b + 1, blockCount, bytesSent, totalBytes); - } - } - emitLog(seg.name + ": data phase done; sending END"); - return sendCtrlAndWait(address, OP_END, EMPTY, CTRL_ACK_TIMEOUT_MS); - } - - /** Send one 4 KB block as a marker + data pair sharing one envelope seq. */ - private int sendBlock(String address, byte[] block) throws TimeoutException { - int seq = nextSeq(); - drainAcks(); - writeOta(address, BleProtocol.OTA_DATA_WRITE_UUID, SID_CTRL, new byte[] {(byte) OP_BLOCK}, seq); - writeOta(address, BleProtocol.OTA_DATA_WRITE_UUID, SID_DATA, block, seq); - return waitAck(OP_BLOCK, BLOCK_ACK_TIMEOUT_MS); - } - - private int sendCtrlAndWait(String address, int op, byte[] data, int timeoutMs) throws TimeoutException { - int seq = nextSeq(); - drainAcks(); - byte[] payload = new byte[1 + data.length]; - payload[0] = (byte) op; - System.arraycopy(data, 0, payload, 1, data.length); - writeOta(address, BleProtocol.OTA_DATA_WRITE_UUID, SID_CTRL, payload, seq); - return waitAck(op, timeoutMs); - } - - private boolean writeOta(String address, String writeChar, int sid, byte[] payload, int seq) { - List frames = BleProtocol.framePb(payload, sid, FLAG_OTA, seq); - return bleManager.writeFrames( - address, writeChar, frames, AndroidProtocolPlatform.writeType(ConnectionOptions.WRITE_MODE), ConnectionOptions.WRITE_TIMEOUT_MS); - } - - private int waitAck(int wantOp, int timeoutMs) throws TimeoutException { - long deadline = SystemClock.elapsedRealtime() + timeoutMs; - while (true) { - long remaining = deadline - SystemClock.elapsedRealtime(); - if (remaining <= 0) { - break; - } - byte[] pb; - try { - pb = dataAcks.poll(remaining, TimeUnit.MILLISECONDS); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - throw new TimeoutException("interrupted waiting for ack op=" + wantOp); - } - if (pb == null) { - break; - } - if (pb.length >= 2 && (pb[0] & 0xff) == wantOp) { - return pb[1] & 0xff; - } - // Different opcode (e.g. a late ack from a prior stage) — ignore, keep waiting. - } - throw new TimeoutException("no ack op=0x" + Integer.toHexString(wantOp) + " within " + timeoutMs + "ms"); - } - - // ---- connection ---------------------------------------------------------- - - private void connectLensResilient(String lens, String address, int windowMs) { - long deadline = SystemClock.elapsedRealtime() + windowMs; - String lastError = "unknown"; - int attempt = 0; - while (SystemClock.elapsedRealtime() < deadline && !cancelled) { - attempt++; - try { - if (bringUpLens(address)) { - emitLog("connected " + lens + " lens (attempt " + attempt + ")"); - return; - } - lastError = "connect/discover incomplete"; - } catch (Exception e) { - lastError = e.getMessage() == null ? e.toString() : e.getMessage(); - } - emitLog(lens + " connect attempt " + attempt + " failed (" + lastError + "); retrying..."); - try { - bleManager.disconnect(address); - } catch (Exception ignored) { - } - sleepInterruptibly(RETRY_DELAY_MS); - } - if (cancelled) { - throw new IllegalStateException("Cancelled."); - } - throw new IllegalStateException("could not reach " + lens + " lens: " + lastError); - } - - private boolean bringUpLens(String address) { - if (!bleManager.connect(address, ConnectionOptions.CONNECT_TIMEOUT_MS)) { - return false; - } - bleManager.requestConnectionPriority(address, BluetoothGatt.CONNECTION_PRIORITY_HIGH); - bleManager.requestMtu(address, ConnectionOptions.DESIRED_MTU, ConnectionOptions.CONNECT_TIMEOUT_MS); - if (!bleManager.discoverServices(address, ConnectionOptions.SERVICES_TIMEOUT_MS)) { - return false; - } - // Control-channel notify carries the auth response; mandatory. - if (!bleManager.enableNotifications(address, BleProtocol.NOTIFY_CHAR_UUID, true, ConnectionOptions.DESCRIPTOR_TIMEOUT_MS)) { - return false; - } - if (!bleManager.enableNotifications(address, BleProtocol.OTA_DATA_NOTIFY_UUID, true, ConnectionOptions.DESCRIPTOR_TIMEOUT_MS)) { - return false; - } - if (!authenticate(address)) { - throw new IllegalStateException( - "security auth not acknowledged — if Android shows a Bluetooth pairing request, accept it"); - } - return true; - } - - /** - * Complete the sid-0x80 security-auth exchange on this connection. On an - * unbonded phone this triggers SMP pairing (possibly with an OS prompt), - * so the wait is generous. Must succeed before any OTA traffic. - */ - private boolean authenticate(String address) { - int magic; - CountDownLatch latch = new CountDownLatch(1); - synchronized (lock) { - magic = nextAuthMagic; - nextAuthMagic = nextAuthMagic >= 0x7f ? 0x60 : nextAuthMagic + 1; - pendingAuthMagic = magic; - authAckPb = null; - authLatch = latch; - } - try { - if (!writeOta(address, BleProtocol.WRITE_CHAR_UUID, BleProtocol.SID_SECURITY_AUTH, - BleProtocol.buildAuthenticationRequest(magic), nextSeq())) { - emitLog("auth write failed: " + address); - return false; - } - boolean signalled; - try { - signalled = latch.await(ConnectionOptions.SECURITY_AUTH_TIMEOUT_MS, TimeUnit.MILLISECONDS); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - return false; - } - byte[] pb; - synchronized (lock) { - pb = authAckPb; - } - if (!signalled || !BleProtocol.isAuthenticationSuccess(pb, magic)) { - emitLog("security auth not acknowledged (" + address - + ") — if Android shows a Bluetooth pairing request, accept it"); - return false; - } - emitLog("security auth complete: " + address); - return true; - } finally { - synchronized (lock) { - pendingAuthMagic = -1; - authLatch = null; - } - } - } - - // ---- firmware container parsing / validation ----------------------------- - - private static final byte[] EMPTY = new byte[0]; - - private static final class Segment { - final int off; - final int ps; - final long crc; - final String name; - - Segment(int off, int ps, long crc, String name) { - this.off = off; - this.ps = ps; - this.crc = crc; - this.name = name; - } - } - - private List parseSegments(byte[] img) { - if (img.length < 0x40) { - throw new IllegalStateException("file is too small to be a firmware image"); - } - long n = readU32(img, 8); - if (n <= 0 || n > 64) { - throw new IllegalStateException("implausible component count " + n + " (corrupt image?)"); - } - List segs = new ArrayList<>(); - for (int i = 0; i < n; i++) { - int base = 0x40 + i * 16; - long crc = readU32(img, base + 12); - int off = (int) readU32(img, base + 4); - if (off + 128 > img.length) { - throw new IllegalStateException("segment " + i + " subheader runs past end of file"); - } - int ps = (int) readU32(img, off + 8); - String name = readCString(img, off + 48, 80); - segs.add(new Segment(off, ps, crc, name)); - } - return segs; - } - - private List validate(byte[] img) { - List segs = parseSegments(img); - if (segs.size() < MIN_SEGMENTS || segs.size() > MAX_SEGMENTS) { - throw new IllegalStateException( - "expected " + MIN_SEGMENTS + "-" + MAX_SEGMENTS + " components, found " + segs.size()); - } - Segment main = null; - for (Segment s : segs) { - byte[] payload = Arrays.copyOfRange(img, s.off + 128, s.off + 128 + s.ps); - long calc = crc32cMsb(payload) & 0xffffffffL; - long subCrc = readU32(img, s.off + 12); - if (calc != s.crc || calc != subCrc) { - throw new IllegalStateException("component " + s.name + " CRC32C is stale (image not checksum-fixed)"); - } - if (REQUIRED_SEGMENT.equals(s.name)) { - main = s; - } - } - if (main == null) { - throw new IllegalStateException("required component " + REQUIRED_SEGMENT + " not found"); - } - checkMainAppFitsMram(img, main); - return segs; - } - - private void checkMainAppFitsMram(byte[] img, Segment main) { - if (main.ps < APP_PREAMBLE) { - throw new IllegalStateException("main-app payload is smaller than its preamble"); - } - long loadAddr = readU32(img, main.off + 128 + 0x14); - long preLen = readU32(img, main.off + 128) & 0xFFFFFFL; - if (loadAddr != APP_LOAD_ADDR) { - throw new IllegalStateException( - "main-app preamble load address is 0x" + Long.toHexString(loadAddr) - + ", expected 0x" + Long.toHexString(APP_LOAD_ADDR)); - } - if (preLen != main.ps) { - throw new IllegalStateException( - "main-app preamble length (" + preLen + ") != staged payload size (" + main.ps + ")"); - } - long progEnd = APP_LOAD_ADDR + main.ps - APP_PREAMBLE; - if (progEnd > APP_MAX_END) { - long over = progEnd - APP_MAX_END; - throw new IllegalStateException( - "main-app is too large: programmed region ends at 0x" + Long.toHexString(progEnd) - + ", " + over + " bytes past the safe MRAM ceiling — refusing to flash (brick risk)"); - } - } - - // ---- listener callbacks -------------------------------------------------- - - @Override - public void onNotification(String address, String characteristicUuid, byte[] data) { - if (BleProtocol.NOTIFY_CHAR_UUID.equalsIgnoreCase(characteristicUuid)) { - // Control channel: only the security-auth response is awaited here. - // A value can carry several envelope frames back to back (e.g. the - // response packed with one of the lens's own sid-0x80 notifies). - for (byte[] buf : BleProtocol.splitFrames(data)) { - BleProtocol.ParsedFrame frame = BleProtocol.parseFrame(buf); - if (frame.ok && frame.sid == BleProtocol.SID_SECURITY_AUTH) { - synchronized (lock) { - if (authLatch != null && frame.msgSeq == pendingAuthMagic) { - authAckPb = frame.pb; - authLatch.countDown(); - } - } - } - } - return; - } - if (!BleProtocol.OTA_DATA_NOTIFY_UUID.equalsIgnoreCase(characteristicUuid)) { - return; // OTA acks arrive on the data-notify char - } - BleProtocol.ParsedFrame frame = BleProtocol.parseFrame(data); - if (!frame.ok || frame.pb.length < 2) { - return; - } - dataAcks.add(Arrays.copyOf(frame.pb, Math.min(frame.pb.length, 2))); - } - - @Override - public void onConnectionStateChange(String address, boolean connected) { - if (!connected) { - Log.i(TAG, "disconnected: " + address); - } - } - - // ---- helpers ------------------------------------------------------------- - - private void teardown() { - try { - bleManager.close(); - } catch (Exception ignored) { - } - } - - private void resetSeq() { - synchronized (lock) { - nextSeq = 0; - } - } - - private int nextSeq() { - synchronized (lock) { - nextSeq = (nextSeq + 1) & 0xff; - return nextSeq; - } - } - - private void drainAcks() { - dataAcks.clear(); - } - - private static boolean isEndOk(int status) { - for (int ok : END_OK) { - if (ok == status) { - return true; - } - } - return false; - } - - private void sleepInterruptibly(int ms) { - try { - Thread.sleep(ms); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - } - } - - private static byte[] readFile(String path) { - try (FileInputStream fis = new FileInputStream(path)) { - ByteArrayOutputStream out = new ByteArrayOutputStream(); - byte[] buf = new byte[65536]; - int read; - while ((read = fis.read(buf)) >= 0) { - out.write(buf, 0, read); - } - return out.toByteArray(); - } catch (IOException e) { - throw new IllegalStateException("could not read firmware: " + e.getMessage(), e); - } - } - - private static long readU32(byte[] buf, int offset) { - return (buf[offset] & 0xffL) - | ((buf[offset + 1] & 0xffL) << 8) - | ((buf[offset + 2] & 0xffL) << 16) - | ((buf[offset + 3] & 0xffL) << 24); - } - - private static String readCString(byte[] buf, int offset, int maxLen) { - int end = offset; - int limit = Math.min(buf.length, offset + maxLen); - while (end < limit && buf[end] != 0) { - end++; - } - return new String(buf, offset, end - offset, StandardCharsets.ISO_8859_1); - } - - private static final int[] CRC32C_TABLE = buildCrc32cTable(); - - private static int[] buildCrc32cTable() { - int[] table = new int[256]; - for (int b = 0; b < 256; b++) { - int c = b << 24; - for (int i = 0; i < 8; i++) { - c = (c & 0x80000000) != 0 ? (c << 1) ^ 0x1edc6f41 : c << 1; - } - table[b] = c; - } - return table; - } - - /** CRC-32C, MSB-first, init 0, no final xor (matches g2flash.py crc32c_msb). */ - private static int crc32cMsb(byte[] data) { - int crc = 0; - for (byte value : data) { - crc = (crc << 8) ^ CRC32C_TABLE[((crc >>> 24) ^ (value & 0xff)) & 0xff]; - } - return crc; - } - - private void emitLog(String line) { - Log.i(TAG, line); - mainHandler.post(() -> { - FaceclawFirmwareFlasherListener current = listener; - if (current != null) { - current.onLog(line); - } - }); - } - - private void emitProgress(String lens, int componentIndex, int componentCount, int blockIndex, int blockCount, - long bytesSent, long bytesTotal) { - mainHandler.post(() -> { - FaceclawFirmwareFlasherListener current = listener; - if (current != null) { - current.onProgress(lens, componentIndex, componentCount, blockIndex, blockCount, bytesSent, bytesTotal); - } - }); - } - - private void emitState(String state, String detail) { - final String safeDetail = detail == null ? "" : detail; - mainHandler.post(() -> { - FaceclawFirmwareFlasherListener current = listener; - if (current != null) { - current.onState(state, safeDetail); - } - }); - } - - private void emitComplete(boolean success, String detail) { - final String safeDetail = detail == null ? "" : detail; - mainHandler.post(() -> { - FaceclawFirmwareFlasherListener current = listener; - if (current != null) { - current.onComplete(success, safeDetail); - } - }); - } -} diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawFirmwareFlasher.kt b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawFirmwareFlasher.kt new file mode 100644 index 00000000..714b3377 --- /dev/null +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawFirmwareFlasher.kt @@ -0,0 +1,707 @@ +package com.faceclaw.app + +import android.bluetooth.BluetoothGatt +import android.content.Context +import android.os.Handler +import android.os.Looper +import android.os.SystemClock +import android.util.Log + +import java.io.ByteArrayOutputStream +import java.io.FileInputStream +import java.io.IOException +import java.nio.charset.StandardCharsets +import java.util.ArrayList +import java.util.Arrays +import java.util.concurrent.CountDownLatch +import java.util.concurrent.LinkedBlockingQueue +import java.util.concurrent.TimeUnit +import java.util.concurrent.TimeoutException + +/** + * Ports g2flash.py's OTA flash procedure to Android. Streams the EVENOTA + * firmware container to each lens over the firmware-data service using the + * aa21 envelope (BleProtocol.framePb) and the c0/c1 control/data protocol. + * + * Each lens connection completes the sid-0x80 security-auth exchange before + * OTA BEGIN — firmware 2.2.9 closes unauthenticated links ~30 s in, killing + * the transfer mid-component. No control-channel traffic is injected during + * the transfer itself: official OTA captures show none between BEGIN and the + * final END, heartbeats demonstrably do not satisfy the 2.2.9 deadline, and a + * concurrent control writer would interleave with the marker/data + * transactions. (See ../notes/ble-connections-2.2.9.md.) + * + * Lenses are flashed one at a time (left, then right). Finishing either lens + * reboots BOTH lenses, so the second lens is briefly unreachable — the reconnect + * for it uses a generous window rather than failing fast. + * + * Separate from FaceclawBleCommunicator on purpose: this speaks the stock OTA + * protocol and must run before the custom firmware exists. It reuses the GATT + * wrapper (FaceclawBleManager) and framing (BleProtocol). + */ +class FaceclawFirmwareFlasher(context: Context, rightAddress: String?, leftAddress: String?, firmwarePath: String?) : FaceclawBleListener { + companion object { + private const val TAG = "FaceclawFlasher" + + // OTA message types (envelope sid byte) and control opcodes. + private const val SID_CTRL = 0xc0 + private const val SID_DATA = 0xc1 + private const val FLAG_OTA = 0x00 + private const val OP_BEGIN = 0x00 + private const val OP_FILE_CHECK = 0x01 + private const val OP_BLOCK = 0x02 + private const val OP_END = 0x03 + + private const val BLOCK_SIZE = 4096 + private const val BLOCK_ACK_TIMEOUT_MS = 4_000 + private const val CTRL_ACK_TIMEOUT_MS = 8_000 + private const val BLOCK_NAK_RETRIES = 3 + private const val COMPONENT_RETRIES = 3 + + // Reconnect windows. The first lens connects from an idle device; the second + // must ride out the post-first-lens reboot of both lenses. + private const val FIRST_LENS_CONNECT_WINDOW_MS = 30_000 + private const val SECOND_LENS_CONNECT_WINDOW_MS = 120_000 + private const val REBOOT_SETTLE_MS = 5_000 + private const val NOTIFY_SETTLE_MS = 2_500 + private const val RETRY_DELAY_MS = 2_500 + + // END ack statuses that mean "component accepted": SUCCESS, UPDATING, SYS_RESTART. + private val END_OK = intArrayOf(0, 8, 9) + + // Firmware layout expectations (mirrors g2flash.py). Firmware 2.2.4 has 5 + // components; 2.2.6 has 6. + private const val MIN_SEGMENTS = 5 + private const val MAX_SEGMENTS = 6 + private const val REQUIRED_SEGMENT = "ota/s200_firmware_ota.bin" + private const val APP_LOAD_ADDR = 0x00438000L + private const val APP_MAX_END = 0x007F0000L + private const val APP_PREAMBLE = 0x20 + + // ---- firmware container parsing / validation ----------------------------- + + private val EMPTY = ByteArray(0) + + // ---- helpers ------------------------------------------------------------- + + private fun isEndOk(status: Int): Boolean { + for (ok in END_OK) { + if (ok == status) { + return true + } + } + return false + } + + private fun readFile(path: String): ByteArray { + try { + FileInputStream(path).use { fis -> + val out = ByteArrayOutputStream() + val buf = ByteArray(65536) + var read: Int + while (fis.read(buf).also { read = it } >= 0) { + out.write(buf, 0, read) + } + return out.toByteArray() + } + } catch (e: IOException) { + throw IllegalStateException("could not read firmware: " + e.message, e) + } + } + + private fun readU32(buf: ByteArray, offset: Int): Long { + return (buf[offset].toLong() and 0xffL) or + ((buf[offset + 1].toLong() and 0xffL) shl 8) or + ((buf[offset + 2].toLong() and 0xffL) shl 16) or + ((buf[offset + 3].toLong() and 0xffL) shl 24) + } + + private fun readCString(buf: ByteArray, offset: Int, maxLen: Int): String { + var end = offset + val limit = Math.min(buf.size, offset + maxLen) + while (end < limit && buf[end].toInt() != 0) { + end++ + } + return String(buf, offset, end - offset, StandardCharsets.ISO_8859_1) + } + + private val CRC32C_TABLE: IntArray = buildCrc32cTable() + + private fun buildCrc32cTable(): IntArray { + val table = IntArray(256) + for (b in 0 until 256) { + var c = b shl 24 + for (i in 0 until 8) { + c = if ((c and 0x80000000.toInt()) != 0) (c shl 1) xor 0x1edc6f41 else c shl 1 + } + table[b] = c + } + return table + } + + /** CRC-32C, MSB-first, init 0, no final xor (matches g2flash.py crc32c_msb). */ + private fun crc32cMsb(data: ByteArray): Int { + var crc = 0 + for (value in data) { + crc = (crc shl 8) xor CRC32C_TABLE[((crc ushr 24) xor (value.toInt() and 0xff)) and 0xff] + } + return crc + } + } + + private val context: Context + private val leftAddress: String + private val rightAddress: String + private val firmwarePath: String + private val bleManager: FaceclawBleManager + private val mainHandler = Handler(Looper.getMainLooper()) + + private val lock = Any() + private val dataAcks = LinkedBlockingQueue() + + @Volatile + private var listener: FaceclawFirmwareFlasherListener? = null + @Volatile + private var worker: Thread? = null + @Volatile + private var cancelled = false + + private var nextSeq = 0 + private var nextAuthMagic = 0x60 + private var pendingAuthMagic = -1 + private var authLatch: CountDownLatch? = null + private var authAckPb: ByteArray? = null + + init { + this.context = context.applicationContext + this.rightAddress = rightAddress ?: "" + this.leftAddress = leftAddress ?: "" + this.firmwarePath = firmwarePath ?: "" + this.bleManager = FaceclawBleManager(this.context) + this.bleManager.setListener(this) + } + + fun setListener(listener: FaceclawFirmwareFlasherListener?) { + this.listener = listener + } + + fun start() { + synchronized(lock) { + if (worker != null) { + return + } + val w = Thread({ run() }, "faceclaw-flasher") + worker = w + w.start() + } + } + + fun cancel() { + cancelled = true + val w = worker + if (w != null) { + w.interrupt() + } + } + + fun close() { + cancel() + try { + bleManager.close() + } catch (ignored: Exception) { + } + } + + private fun run() { + try { + if (leftAddress.trim().isEmpty() || rightAddress.trim().isEmpty()) { + throw IllegalStateException("Both lens addresses are required to flash.") + } + + emitState("validating", "") + val img = readFile(firmwarePath) + val segs = validate(img) + emitLog("firmware validated: " + segs.size + " components, " + img.size + " bytes") + + flashLens("left", leftAddress, img, segs, FIRST_LENS_CONNECT_WINDOW_MS) + if (cancelled) { + emitState("error", "Cancelled.") + emitComplete(false, "Cancelled after the left lens.") + return + } + + emitState("rebooting", "Left lens done. Both lenses reboot briefly; reconnecting for the right lens.") + sleepInterruptibly(REBOOT_SETTLE_MS) + flashLens("right", rightAddress, img, segs, SECOND_LENS_CONNECT_WINDOW_MS) + + emitState("done", "") + emitComplete(true, "Both lenses flashed. The glasses are rebooting into the custom firmware.") + } catch (e: Exception) { + var message = if (e.message == null) e.toString() else e.message + if (cancelled) { + message = "Cancelled." + } + emitState("error", message) + emitComplete(false, message) + } finally { + teardown() + } + } + + @Throws(TimeoutException::class) + private fun flashLens(lens: String, address: String, img: ByteArray, segs: List, connectWindowMs: Int) { + emitState("connecting", lens) + connectLensResilient(lens, address, connectWindowMs) + sleepInterruptibly(NOTIFY_SETTLE_MS) + drainAcks() + // Fresh envelope counter for the OTA stream (the auth exchange during + // bring-up used its own); mirrors the stock sequence. + resetSeq() + + emitState("flashing", lens) + val beginStatus = sendCtrlAndWait(address, OP_BEGIN, EMPTY, CTRL_ACK_TIMEOUT_MS) + if (!isEndOk(beginStatus)) { + emitLog("warning: unexpected begin status $beginStatus; continuing") + } + // Progress is reported in bytes across the whole image: the segments + // are one large firmware blob plus several small ones, so a bar + // stepped per segment would sit still through the big one and then + // sprint through the rest. + var totalBytes = 0L + for (seg in segs) { + totalBytes += seg.ps + } + var bytesBefore = 0L + for (i in segs.indices) { + if (cancelled) { + throw IllegalStateException("Cancelled.") + } + flashComponentWithRetry(lens, address, i, segs.size, segs[i], img, bytesBefore, totalBytes) + bytesBefore += segs[i].ps + } + emitLog("$lens lens: all components verified") + try { + bleManager.disconnect(address) + } catch (ignored: Exception) { + } + } + + private fun flashComponentWithRetry(lens: String, address: String, index: Int, count: Int, seg: Segment, img: ByteArray, + bytesBefore: Long, totalBytes: Long) { + for (attempt in 0 until COMPONENT_RETRIES) { + if (attempt > 0) { + emitLog(seg.name + ": re-flash attempt " + (attempt + 1) + "/" + COMPONENT_RETRIES) + } + var endStatus: Int + try { + endStatus = flashComponent(lens, address, index, count, seg, img, bytesBefore, totalBytes) + } catch (e: TimeoutException) { + emitLog(seg.name + ": block phase failed: " + e.message) + endStatus = -1 + } catch (e: RuntimeException) { + emitLog(seg.name + ": block phase failed: " + e.message) + endStatus = -1 + } + if (isEndOk(endStatus)) { + emitLog(seg.name + ": END verify OK (status " + endStatus + ")") + return + } + if (endStatus >= 0) { + emitLog(seg.name + ": END verify FAILED (status " + endStatus + ")") + } + drainAcks() + sleepInterruptibly(1_500) + } + throw IllegalStateException("component " + seg.name + " failed after " + COMPONENT_RETRIES + " attempts") + } + + @Throws(TimeoutException::class) + private fun flashComponent(lens: String, address: String, index: Int, count: Int, seg: Segment, img: ByteArray, + bytesBefore: Long, totalBytes: Long): Int { + val sub = Arrays.copyOfRange(img, seg.off, seg.off + 128) + val ps = seg.ps + val payloadStart = seg.off + 128 + + val checkStatus = sendCtrlAndWait(address, OP_FILE_CHECK, sub, CTRL_ACK_TIMEOUT_MS) + if (checkStatus != 0) { + throw IllegalStateException("FILE_CHECK rejected status=$checkStatus") + } + + val blockCount = (ps + BLOCK_SIZE - 1) / BLOCK_SIZE + for (b in 0 until blockCount) { + if (cancelled) { + throw IllegalStateException("Cancelled.") + } + val start = payloadStart + b * BLOCK_SIZE + val end = Math.min(start + BLOCK_SIZE, payloadStart + ps) + val block = Arrays.copyOfRange(img, start, end) + + var accepted = false + for (tries in 0 until BLOCK_NAK_RETRIES) { + val status = sendBlock(address, block) // throws TimeoutException -> component re-flash + if (status == 0) { + accepted = true + break + } + emitLog(seg.name + ": block " + b + "/" + blockCount + " NAK=" + status + + " resend " + (tries + 1) + "/" + BLOCK_NAK_RETRIES) + } + if (!accepted) { + throw IllegalStateException("block " + b + " NAK'd " + BLOCK_NAK_RETRIES + " times") + } + if (b % 20 == 0 || b == blockCount - 1) { + val bytesSent = bytesBefore + Math.min((b + 1).toLong() * BLOCK_SIZE, ps.toLong()) + emitProgress(lens, index + 1, count, b + 1, blockCount, bytesSent, totalBytes) + } + } + emitLog(seg.name + ": data phase done; sending END") + return sendCtrlAndWait(address, OP_END, EMPTY, CTRL_ACK_TIMEOUT_MS) + } + + /** Send one 4 KB block as a marker + data pair sharing one envelope seq. */ + @Throws(TimeoutException::class) + private fun sendBlock(address: String, block: ByteArray): Int { + val seq = nextSeq() + drainAcks() + writeOta(address, BleProtocol.OTA_DATA_WRITE_UUID, SID_CTRL, byteArrayOf(OP_BLOCK.toByte()), seq) + writeOta(address, BleProtocol.OTA_DATA_WRITE_UUID, SID_DATA, block, seq) + return waitAck(OP_BLOCK, BLOCK_ACK_TIMEOUT_MS) + } + + @Throws(TimeoutException::class) + private fun sendCtrlAndWait(address: String, op: Int, data: ByteArray, timeoutMs: Int): Int { + val seq = nextSeq() + drainAcks() + val payload = ByteArray(1 + data.size) + payload[0] = op.toByte() + System.arraycopy(data, 0, payload, 1, data.size) + writeOta(address, BleProtocol.OTA_DATA_WRITE_UUID, SID_CTRL, payload, seq) + return waitAck(op, timeoutMs) + } + + private fun writeOta(address: String, writeChar: String, sid: Int, payload: ByteArray, seq: Int): Boolean { + val frames = BleProtocol.framePb(payload, sid, FLAG_OTA, seq) + return bleManager.writeFrames( + address, writeChar, frames, AndroidProtocolPlatform.writeType(ConnectionOptions.WRITE_MODE), ConnectionOptions.WRITE_TIMEOUT_MS) + } + + @Throws(TimeoutException::class) + private fun waitAck(wantOp: Int, timeoutMs: Int): Int { + val deadline = SystemClock.elapsedRealtime() + timeoutMs + while (true) { + val remaining = deadline - SystemClock.elapsedRealtime() + if (remaining <= 0) { + break + } + val pb: ByteArray? + try { + pb = dataAcks.poll(remaining, TimeUnit.MILLISECONDS) + } catch (e: InterruptedException) { + Thread.currentThread().interrupt() + throw TimeoutException("interrupted waiting for ack op=$wantOp") + } + if (pb == null) { + break + } + if (pb.size >= 2 && (pb[0].toInt() and 0xff) == wantOp) { + return pb[1].toInt() and 0xff + } + // Different opcode (e.g. a late ack from a prior stage) — ignore, keep waiting. + } + throw TimeoutException("no ack op=0x" + Integer.toHexString(wantOp) + " within " + timeoutMs + "ms") + } + + // ---- connection ---------------------------------------------------------- + + private fun connectLensResilient(lens: String, address: String, windowMs: Int) { + val deadline = SystemClock.elapsedRealtime() + windowMs + var lastError: String? = "unknown" + var attempt = 0 + while (SystemClock.elapsedRealtime() < deadline && !cancelled) { + attempt++ + try { + if (bringUpLens(address)) { + emitLog("connected $lens lens (attempt $attempt)") + return + } + lastError = "connect/discover incomplete" + } catch (e: Exception) { + lastError = if (e.message == null) e.toString() else e.message + } + emitLog("$lens connect attempt $attempt failed ($lastError); retrying...") + try { + bleManager.disconnect(address) + } catch (ignored: Exception) { + } + sleepInterruptibly(RETRY_DELAY_MS) + } + if (cancelled) { + throw IllegalStateException("Cancelled.") + } + throw IllegalStateException("could not reach $lens lens: $lastError") + } + + private fun bringUpLens(address: String): Boolean { + if (!bleManager.connect(address, ConnectionOptions.CONNECT_TIMEOUT_MS)) { + return false + } + bleManager.requestConnectionPriority(address, BluetoothGatt.CONNECTION_PRIORITY_HIGH) + bleManager.requestMtu(address, ConnectionOptions.DESIRED_MTU, ConnectionOptions.CONNECT_TIMEOUT_MS) + if (!bleManager.discoverServices(address, ConnectionOptions.SERVICES_TIMEOUT_MS)) { + return false + } + // Control-channel notify carries the auth response; mandatory. + if (!bleManager.enableNotifications(address, BleProtocol.NOTIFY_CHAR_UUID, true, ConnectionOptions.DESCRIPTOR_TIMEOUT_MS)) { + return false + } + if (!bleManager.enableNotifications(address, BleProtocol.OTA_DATA_NOTIFY_UUID, true, ConnectionOptions.DESCRIPTOR_TIMEOUT_MS)) { + return false + } + if (!authenticate(address)) { + throw IllegalStateException( + "security auth not acknowledged — if Android shows a Bluetooth pairing request, accept it") + } + return true + } + + /** + * Complete the sid-0x80 security-auth exchange on this connection. On an + * unbonded phone this triggers SMP pairing (possibly with an OS prompt), + * so the wait is generous. Must succeed before any OTA traffic. + */ + private fun authenticate(address: String): Boolean { + val magic: Int + val latch = CountDownLatch(1) + synchronized(lock) { + magic = nextAuthMagic + nextAuthMagic = if (nextAuthMagic >= 0x7f) 0x60 else nextAuthMagic + 1 + pendingAuthMagic = magic + authAckPb = null + authLatch = latch + } + try { + if (!writeOta(address, BleProtocol.WRITE_CHAR_UUID, BleProtocol.SID_SECURITY_AUTH, + BleProtocol.buildAuthenticationRequest(magic), nextSeq())) { + emitLog("auth write failed: $address") + return false + } + val signalled: Boolean + try { + signalled = latch.await(ConnectionOptions.SECURITY_AUTH_TIMEOUT_MS.toLong(), TimeUnit.MILLISECONDS) + } catch (e: InterruptedException) { + Thread.currentThread().interrupt() + return false + } + val pb: ByteArray? + synchronized(lock) { + pb = authAckPb + } + if (!signalled || !BleProtocol.isAuthenticationSuccess(pb, magic)) { + emitLog("security auth not acknowledged (" + address + + ") — if Android shows a Bluetooth pairing request, accept it") + return false + } + emitLog("security auth complete: $address") + return true + } finally { + synchronized(lock) { + pendingAuthMagic = -1 + authLatch = null + } + } + } + + // ---- firmware container parsing / validation ----------------------------- + + private class Segment( + @JvmField val off: Int, + @JvmField val ps: Int, + @JvmField val crc: Long, + @JvmField val name: String + ) + + private fun parseSegments(img: ByteArray): List { + if (img.size < 0x40) { + throw IllegalStateException("file is too small to be a firmware image") + } + val n = readU32(img, 8) + if (n <= 0 || n > 64) { + throw IllegalStateException("implausible component count $n (corrupt image?)") + } + val segs = ArrayList() + var i = 0 + while (i < n) { + val base = 0x40 + i * 16 + val crc = readU32(img, base + 12) + val off = readU32(img, base + 4).toInt() + if (off + 128 > img.size) { + throw IllegalStateException("segment $i subheader runs past end of file") + } + val ps = readU32(img, off + 8).toInt() + val name = readCString(img, off + 48, 80) + segs.add(Segment(off, ps, crc, name)) + i++ + } + return segs + } + + private fun validate(img: ByteArray): List { + val segs = parseSegments(img) + if (segs.size < MIN_SEGMENTS || segs.size > MAX_SEGMENTS) { + throw IllegalStateException( + "expected " + MIN_SEGMENTS + "-" + MAX_SEGMENTS + " components, found " + segs.size) + } + var main: Segment? = null + for (s in segs) { + val payload = Arrays.copyOfRange(img, s.off + 128, s.off + 128 + s.ps) + val calc = crc32cMsb(payload).toLong() and 0xffffffffL + val subCrc = readU32(img, s.off + 12) + if (calc != s.crc || calc != subCrc) { + throw IllegalStateException("component " + s.name + " CRC32C is stale (image not checksum-fixed)") + } + if (REQUIRED_SEGMENT == s.name) { + main = s + } + } + if (main == null) { + throw IllegalStateException("required component $REQUIRED_SEGMENT not found") + } + checkMainAppFitsMram(img, main) + return segs + } + + private fun checkMainAppFitsMram(img: ByteArray, main: Segment) { + if (main.ps < APP_PREAMBLE) { + throw IllegalStateException("main-app payload is smaller than its preamble") + } + val loadAddr = readU32(img, main.off + 128 + 0x14) + val preLen = readU32(img, main.off + 128) and 0xFFFFFFL + if (loadAddr != APP_LOAD_ADDR) { + throw IllegalStateException( + "main-app preamble load address is 0x" + java.lang.Long.toHexString(loadAddr) + + ", expected 0x" + java.lang.Long.toHexString(APP_LOAD_ADDR)) + } + if (preLen != main.ps.toLong()) { + throw IllegalStateException( + "main-app preamble length (" + preLen + ") != staged payload size (" + main.ps + ")") + } + val progEnd = APP_LOAD_ADDR + main.ps - APP_PREAMBLE + if (progEnd > APP_MAX_END) { + val over = progEnd - APP_MAX_END + throw IllegalStateException( + "main-app is too large: programmed region ends at 0x" + java.lang.Long.toHexString(progEnd) + + ", " + over + " bytes past the safe MRAM ceiling — refusing to flash (brick risk)") + } + } + + // ---- listener callbacks -------------------------------------------------- + + override fun onNotification(address: String?, characteristicUuid: String?, data: ByteArray?) { + if (BleProtocol.NOTIFY_CHAR_UUID.equals(characteristicUuid, ignoreCase = true)) { + // Control channel: only the security-auth response is awaited here. + // A value can carry several envelope frames back to back (e.g. the + // response packed with one of the lens's own sid-0x80 notifies). + for (buf in BleProtocol.splitFrames(data)) { + val frame = BleProtocol.parseFrame(buf) + if (frame.ok && frame.sid == BleProtocol.SID_SECURITY_AUTH) { + synchronized(lock) { + val currentLatch = authLatch + if (currentLatch != null && frame.msgSeq == pendingAuthMagic) { + authAckPb = frame.pb + currentLatch.countDown() + } + } + } + } + return + } + if (!BleProtocol.OTA_DATA_NOTIFY_UUID.equals(characteristicUuid, ignoreCase = true)) { + return // OTA acks arrive on the data-notify char + } + val frame = BleProtocol.parseFrame(data) + if (!frame.ok || frame.pb.size < 2) { + return + } + dataAcks.add(Arrays.copyOf(frame.pb, Math.min(frame.pb.size, 2))) + } + + override fun onConnectionStateChange(address: String?, connected: Boolean) { + if (!connected) { + Log.i(TAG, "disconnected: $address") + } + } + + // ---- helpers ------------------------------------------------------------- + + private fun teardown() { + try { + bleManager.close() + } catch (ignored: Exception) { + } + } + + private fun resetSeq() { + synchronized(lock) { + nextSeq = 0 + } + } + + private fun nextSeq(): Int { + synchronized(lock) { + nextSeq = (nextSeq + 1) and 0xff + return nextSeq + } + } + + private fun drainAcks() { + dataAcks.clear() + } + + private fun sleepInterruptibly(ms: Int) { + try { + Thread.sleep(ms.toLong()) + } catch (e: InterruptedException) { + Thread.currentThread().interrupt() + } + } + + private fun emitLog(line: String) { + Log.i(TAG, line) + mainHandler.post { + val current = listener + if (current != null) { + current.onLog(line) + } + } + } + + private fun emitProgress(lens: String, componentIndex: Int, componentCount: Int, blockIndex: Int, blockCount: Int, + bytesSent: Long, bytesTotal: Long) { + mainHandler.post { + val current = listener + if (current != null) { + current.onProgress(lens, componentIndex, componentCount, blockIndex, blockCount, bytesSent, bytesTotal) + } + } + } + + private fun emitState(state: String, detail: String?) { + val safeDetail = detail ?: "" + mainHandler.post { + val current = listener + if (current != null) { + current.onState(state, safeDetail) + } + } + } + + private fun emitComplete(success: Boolean, detail: String?) { + val safeDetail = detail ?: "" + mainHandler.post { + val current = listener + if (current != null) { + current.onComplete(success, safeDetail) + } + } + } +} diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawFlashPromptCommunicator.java b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawFlashPromptCommunicator.java deleted file mode 100644 index ef342706..00000000 --- a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawFlashPromptCommunicator.java +++ /dev/null @@ -1,546 +0,0 @@ -package com.faceclaw.app; - -import android.bluetooth.BluetoothGatt; -import android.content.Context; -import android.os.Handler; -import android.os.Looper; -import android.util.Log; - -import java.util.List; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.TimeUnit; - -/** - * A minimal, stock-firmware-compatible BLE communicator used only to show the - * pre-flash confirmation prompt on the glasses and read back the user's Yes/No - * choice. It is deliberately separate from FaceclawBleCommunicator: the main - * app assumes the custom firmware (image containers, compositor, etc.), whereas - * this must work on unmodified glasses, so it speaks only the stock subset — - * session prelude, a text container, a list container, and heartbeats. No - * images. - * - * It reuses the low-level GATT wrapper (FaceclawBleManager) and the framing / - * protobuf helpers (BleProtocol), but owns its own GATT connection and runs its - * whole lifecycle on a single worker thread. It expects the main app to be - * disconnected while it runs (onboarding, before flashing). - * - * Both arms are connected and security-authenticated up front, even though the - * prompt itself only needs the right arm (the lenses relay messages to each - * other, and acks/events only ever come from the right arm): on an unbonded - * phone the auth exchange is what raises the OS pairing prompt, and doing both - * here means both prompts appear at the start rather than one popping up half - * way through flashing the second lens. While connected, each arm's battery - * level is read (they are independent batteries) so the caller can refuse to - * flash on a low charge. - * - * With `skipPrompt` the on-glasses confirmation is not shown: the flow is just - * connect + auth + battery read + result(approved). Used to re-check the - * battery after the user has already confirmed once. - */ -public class FaceclawFlashPromptCommunicator implements FaceclawBleListener { - private static final String TAG = "FaceclawFlashPrompt"; - - private static final String TEXT_NAME = "flashwarn"; - private static final String LIST_NAME = "flashmenu"; - private static final int TEXT_CONTAINER_ID = 1; - private static final int LIST_CONTAINER_ID = 2; - // Index 0 = decline, index 1 = approve. Kept short for the ~50-col grid. - private static final String[] ITEMS = new String[] {"No, cancel", "Yes, flash"}; - - private static final int HEARTBEAT_INTERVAL_MS = 4_000; - private static final int SELECTION_TIMEOUT_MS = 120_000; - private static final int CREATE_ACK_TIMEOUT_MS = 3_000; - private static final int BATTERY_ACK_TIMEOUT_MS = 3_000; - - private final Context context; - private final String rightAddress; - private final String leftAddress; - private final String warningText; - private final boolean skipPrompt; - private final FaceclawBleManager bleManager; - private final Handler mainHandler = new Handler(Looper.getMainLooper()); - - private final Object lock = new Object(); - private final ConcurrentHashMap pendingAcks = new ConcurrentHashMap<>(); - private final ConcurrentHashMap ackPayloads = new ConcurrentHashMap<>(); - private final CountDownLatch selectionLatch = new CountDownLatch(1); - - private volatile FaceclawFlashPromptListener listener; - private volatile Thread worker; - private volatile boolean cancelled = false; - private volatile boolean finished = false; - private volatile boolean rightConnected = false; - private volatile boolean leftConnected = false; - private volatile boolean rightLost = false; - private volatile Boolean approved = null; - private volatile ScheduledExecutorService heartbeatExecutor; - - private int nextMagic = 100; - private int nextSeq = 0x40; - - public FaceclawFlashPromptCommunicator( - Context context, String rightAddress, String leftAddress, String warningText, boolean skipPrompt) { - this.context = context.getApplicationContext(); - this.rightAddress = rightAddress == null ? "" : rightAddress; - this.leftAddress = leftAddress == null ? "" : leftAddress; - this.warningText = warningText == null ? "" : warningText; - this.skipPrompt = skipPrompt; - this.bleManager = new FaceclawBleManager(this.context); - this.bleManager.setListener(this); - } - - public void setListener(FaceclawFlashPromptListener listener) { - this.listener = listener; - } - - public void start() { - synchronized (lock) { - if (worker != null) { - return; - } - worker = new Thread(this::run, "faceclaw-flash-prompt"); - worker.start(); - } - } - - /** Abort the prompt (e.g. the user backed out on the phone). */ - public void cancel() { - cancelled = true; - selectionLatch.countDown(); - Thread w = worker; - if (w != null) { - w.interrupt(); - } - } - - public void close() { - cancel(); - stopHeartbeat(); - try { - bleManager.close(); - } catch (Exception ignored) { - } - } - - private void run() { - try { - if (rightAddress.trim().isEmpty()) { - emitState("error", "No right-arm address configured."); - return; - } - - emitState("connecting", ""); - connectArm(rightAddress, "right"); - rightConnected = true; - if (cancelled) { - teardown(); - return; - } - if (!leftAddress.trim().isEmpty()) { - connectArm(leftAddress, "left"); - leftConnected = true; - if (cancelled) { - teardown(); - return; - } - } - - emitState("connected", ""); - // Auth both arms now (pairing prompts, if any, happen here) before - // anything else, so both bonds exist by the time flashing starts. - authenticateArm(rightAddress, "right"); - if (leftConnected) { - authenticateArm(leftAddress, "left"); - } - if (cancelled) { - teardown(); - return; - } - sendPrelude(rightAddress); - - if (!skipPrompt) { - showPrompt(rightAddress); - startHeartbeat(); - emitState("prompting", ""); - - selectionLatch.await(SELECTION_TIMEOUT_MS, TimeUnit.MILLISECONDS); - stopHeartbeat(); - - if (cancelled) { - emitState("cancelled", ""); - teardown(); - return; - } - if (approved == null) { - if (rightLost) { - emitState("disconnected", "Lost connection to the glasses."); - } else { - emitState("timeout", "No response from the glasses."); - } - teardown(); - return; - } - try { - sendShutdown(); - } catch (Exception ignored) { - } - if (!Boolean.TRUE.equals(approved)) { - emitResult(false); - emitState("result", "declined"); - teardown(); - return; - } - } - - // Approved (or prompt skipped): read each arm's battery while the - // links are still up, then report the result. - emitState("battery", ""); - readBatteries(); - if (cancelled) { - emitState("cancelled", ""); - teardown(); - return; - } - if (rightLost) { - emitState("disconnected", "Lost connection to the glasses."); - teardown(); - return; - } - finished = true; - emitResult(true); - emitState("result", "approved"); - teardown(); - } catch (Exception e) { - stopHeartbeat(); - if (!cancelled) { - String message = e.getMessage() == null ? e.toString() : e.getMessage(); - emitState("error", message); - } - teardown(); - } - } - - private void connectArm(String address, String arm) { - if (!bleManager.connect(address, ConnectionOptions.CONNECT_TIMEOUT_MS)) { - throw new IllegalStateException("could not connect to the " + arm + " arm (" + address + ")"); - } - bleManager.requestConnectionPriority(address, BluetoothGatt.CONNECTION_PRIORITY_HIGH); - bleManager.requestMtu(address, ConnectionOptions.DESIRED_MTU, ConnectionOptions.CONNECT_TIMEOUT_MS); - if (!bleManager.discoverServices(address, ConnectionOptions.SERVICES_TIMEOUT_MS)) { - throw new IllegalStateException("discoverServices failed: " + arm + " arm (" + address + ")"); - } - if (!bleManager.enableNotifications(address, BleProtocol.NOTIFY_CHAR_UUID, true, ConnectionOptions.DESCRIPTOR_TIMEOUT_MS)) { - throw new IllegalStateException("enableNotifications failed: " + arm + " arm (" + address + ")"); - } - emitLog("connected " + arm + " arm"); - } - - /** - * Security-auth exchange on sid=0x80: firmware 2.2.9 will not run a - * session (and closes the link after ~30 s) without it, and on an - * unbonded phone this is what triggers SMP pairing, so it may sit - * waiting on an OS pairing prompt. - */ - private void authenticateArm(String address, String arm) throws InterruptedException { - int authMagic = allocMagic(); - byte[] authAck = writeAndAwaitAck( - address, - BleProtocol.SID_SECURITY_AUTH, - BleProtocol.FLAG_SECURITY_AUTH, - authMagic, - BleProtocol.buildAuthenticationRequest(authMagic), - ConnectionOptions.SECURITY_AUTH_TIMEOUT_MS); - if (authAck == null || !BleProtocol.isAuthenticationSuccess(authAck, authMagic)) { - throw new IllegalStateException( - "could not authenticate with the " + arm + " arm (" + address - + ") — if Android shows a Bluetooth pairing request, accept it and try again"); - } - emitLog("security auth complete: " + arm + " arm"); - } - - /** Mandatory session prelude on sid=0x01 (app-launch). Right arm only; it relays. */ - private void sendPrelude(String address) throws InterruptedException { - if (writeAndAwaitAck( - address, - BleProtocol.PRELUDE_ACK_SID, - BleProtocol.FLAG_REQUEST, - BleProtocol.PRELUDE_ACK_MAGIC, - BleProtocol.PRELUDE_F5872_PAYLOAD, - ConnectionOptions.PRELUDE_TIMEOUT_MS) == null) { - throw new IllegalStateException("session prelude not acked: " + address); - } - } - - private void showPrompt(String address) throws InterruptedException { - // Create the prompt page (warning text + No/Yes list) on sid=0xe0 Cmd=0. - // Two attempts: on 2.2.9 the first request sent right after the - // prelude can be dropped while the lens emits its own sid-0x80 - // notifications (observed with the device-info settings read). - byte[] pageAck = null; - for (int attempt = 0; attempt < 2 && pageAck == null && !cancelled; attempt++) { - int magic = allocMagic(); - byte[] page = BleProtocol.buildCreatePromptPage( - magic, TEXT_NAME, TEXT_CONTAINER_ID, warningText, LIST_NAME, LIST_CONTAINER_ID, ITEMS); - pageAck = writeAndAwaitAck(address, BleProtocol.SID_EVENHUB, BleProtocol.FLAG_REQUEST, magic, page, CREATE_ACK_TIMEOUT_MS); - if (pageAck == null) { - emitLog("prompt page attempt " + (attempt + 1) + " unacked: " + address); - } - } - if (pageAck == null) { - throw new IllegalStateException("prompt page not acked: " + address); - } - emitLog("prompt page shown on " + address); - } - - /** - * Read each arm's battery via a sid-0x09 settings read on that arm's own - * link (the arms have independent batteries and each answers with its - * own). An arm that does not answer reports -1; the caller decides what - * to do with a partial reading. - */ - private void readBatteries() throws InterruptedException { - int right = rightConnected ? readBattery(rightAddress, "right") : -1; - int left = leftConnected ? readBattery(leftAddress, "left") : -1; - emitLog("battery R=" + (right < 0 ? "?" : right + "%") + " L=" + (left < 0 ? "?" : left + "%")); - emitBattery(right, left); - } - - private int readBattery(String address, String arm) throws InterruptedException { - for (int attempt = 0; attempt < 2 && !cancelled; attempt++) { - int magic = allocMagic(); - byte[] ack = writeAndAwaitAck( - address, - BleProtocol.SID_UI_SETTING, - BleProtocol.FLAG_REQUEST, - magic, - BleProtocol.buildSettingsQuery(magic), - BATTERY_ACK_TIMEOUT_MS); - if (ack == null) { - emitLog("battery read attempt " + (attempt + 1) + " unacked: " + arm + " arm"); - continue; - } - BleProtocol.BatterySnapshot snapshot = BleProtocol.parseSettingsBattery(ack); - if (snapshot != null) { - return snapshot.battery; - } - emitLog("battery read ack had no battery field: " + arm + " arm"); - } - return -1; - } - - /** Write and wait for the matching ack; returns the ack's protobuf (may be empty), or null on timeout/write failure. */ - private byte[] writeAndAwaitAck(String address, int sid, int flag, int magic, byte[] payload, int timeoutMs) - throws InterruptedException { - CountDownLatch latch = new CountDownLatch(1); - String key = ackKey(sid, magic); - pendingAcks.put(key, latch); - try { - if (!writeFrame(address, sid, flag, payload)) { - return null; - } - if (!latch.await(timeoutMs, TimeUnit.MILLISECONDS)) { - return null; - } - byte[] pb = ackPayloads.get(key); - return pb == null ? new byte[0] : pb; - } finally { - pendingAcks.remove(key, latch); - ackPayloads.remove(key); - } - } - - private boolean writeFrame(String address, int sid, int flag, byte[] payload) { - int seq; - synchronized (lock) { - seq = nextSeq++ & 0xff; - } - List frames = BleProtocol.framePb(payload, sid, flag, seq); - return bleManager.writeFrames( - address, - BleProtocol.WRITE_CHAR_UUID, - frames, - AndroidProtocolPlatform.writeType(ConnectionOptions.WRITE_MODE), - ConnectionOptions.WRITE_TIMEOUT_MS); - } - - private void startHeartbeat() { - stopHeartbeat(); - ScheduledExecutorService executor = Executors.newSingleThreadScheduledExecutor(runnable -> { - Thread t = new Thread(runnable, "faceclaw-flash-hb"); - t.setDaemon(true); - return t; - }); - heartbeatExecutor = executor; - executor.scheduleWithFixedDelay( - this::sendHeartbeat, HEARTBEAT_INTERVAL_MS, HEARTBEAT_INTERVAL_MS, TimeUnit.MILLISECONDS); - } - - private void sendHeartbeat() { - if (finished || cancelled) { - return; - } - try { - byte[] heartbeat = BleProtocol.buildHeartbeat(allocMagic()); - if (rightConnected) { - writeFrame(rightAddress, BleProtocol.SID_EVENHUB, BleProtocol.FLAG_REQUEST, heartbeat); - } - } catch (Exception e) { - Log.w(TAG, "heartbeat failed: " + e.getMessage()); - } - } - - private void stopHeartbeat() { - ScheduledExecutorService executor = heartbeatExecutor; - heartbeatExecutor = null; - if (executor != null) { - executor.shutdownNow(); - } - } - - private void sendShutdown() { - byte[] shutdown = BleProtocol.buildShutdown(allocMagic(), 0); - if (rightConnected) { - writeFrame(rightAddress, BleProtocol.SID_EVENHUB, BleProtocol.FLAG_REQUEST, shutdown); - } - } - - private void teardown() { - finished = true; - stopHeartbeat(); - try { - bleManager.close(); - } catch (Exception ignored) { - } - rightConnected = false; - leftConnected = false; - } - - @Override - public void onNotification(String address, String characteristicUuid, byte[] data) { - if (!BleProtocol.NOTIFY_CHAR_UUID.equalsIgnoreCase(characteristicUuid)) { - return; - } - // A value can carry several envelope frames back to back. - for (byte[] buf : BleProtocol.splitFrames(data)) { - handleFrame(address, buf); - } - } - - private void handleFrame(String address, byte[] buf) { - BleProtocol.ParsedFrame frame = BleProtocol.parseFrame(buf); - if (!frame.ok) { - return; - } - if (frame.flag == BleProtocol.FLAG_NOTIFY || frame.flag == BleProtocol.FLAG_NOTIFY_ALT) { - handleEvent(frame); - return; - } - if (frame.msgSeq >= 0) { - String key = ackKey(frame.sid, frame.msgSeq); - CountDownLatch latch = pendingAcks.get(key); - if (latch != null) { - ackPayloads.put(key, frame.pb); - latch.countDown(); - } - } - } - - private void handleEvent(BleProtocol.ParsedFrame frame) { - BleProtocol.ListSelection selection = BleProtocol.parseListSelection(frame); - if (selection == null || !LIST_NAME.equals(selection.containerName)) { - return; - } - // Only a confirmed click is a decision; scroll/highlight changes are ignored. - if (selection.eventType != BleProtocol.EVENT_CLICK) { - return; - } - boolean yes = selection.itemIndex == 1 - || (selection.itemName != null && selection.itemName.toLowerCase().startsWith("yes")); - synchronized (lock) { - if (finished) { - return; - } - finished = true; - approved = yes; - } - emitLog("selection: " + (yes ? "flash" : "cancel") + " (index " + selection.itemIndex + ")"); - selectionLatch.countDown(); - } - - @Override - public void onConnectionStateChange(String address, boolean connected) { - if (connected) { - return; - } - if (address.equalsIgnoreCase(leftAddress)) { - // The prompt itself only needs the right arm; a dropped left link - // just means no left battery reading. - leftConnected = false; - emitLog("left arm disconnected"); - return; - } - if (address.equalsIgnoreCase(rightAddress)) { - rightConnected = false; - synchronized (lock) { - if (!finished && !cancelled) { - rightLost = true; - finished = true; - selectionLatch.countDown(); - } - } - } - } - - private int allocMagic() { - synchronized (lock) { - int magic = nextMagic; - nextMagic = nextMagic >= 255 ? 100 : nextMagic + 1; - return magic; - } - } - - private static String ackKey(int sid, int magic) { - return sid + ":" + magic; - } - - private void emitLog(String line) { - Log.i(TAG, line); - mainHandler.post(() -> { - FaceclawFlashPromptListener current = listener; - if (current != null) { - current.onLog(line); - } - }); - } - - private void emitState(String state, String detail) { - final String safeDetail = detail == null ? "" : detail; - mainHandler.post(() -> { - FaceclawFlashPromptListener current = listener; - if (current != null) { - current.onState(state, safeDetail); - } - }); - } - - private void emitBattery(int rightPercent, int leftPercent) { - mainHandler.post(() -> { - FaceclawFlashPromptListener current = listener; - if (current != null) { - current.onBattery(rightPercent, leftPercent); - } - }); - } - - private void emitResult(boolean approvedResult) { - mainHandler.post(() -> { - FaceclawFlashPromptListener current = listener; - if (current != null) { - current.onResult(approvedResult); - } - }); - } -} diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawFlashPromptCommunicator.kt b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawFlashPromptCommunicator.kt new file mode 100644 index 00000000..b3cf1d83 --- /dev/null +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawFlashPromptCommunicator.kt @@ -0,0 +1,566 @@ +package com.faceclaw.app + +import android.bluetooth.BluetoothGatt +import android.content.Context +import android.os.Handler +import android.os.Looper +import android.util.Log + +import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.CountDownLatch +import java.util.concurrent.Executors +import java.util.concurrent.ScheduledExecutorService +import java.util.concurrent.TimeUnit + +/** + * A minimal, stock-firmware-compatible BLE communicator used only to show the + * pre-flash confirmation prompt on the glasses and read back the user's Yes/No + * choice. It is deliberately separate from FaceclawBleCommunicator: the main + * app assumes the custom firmware (image containers, compositor, etc.), whereas + * this must work on unmodified glasses, so it speaks only the stock subset — + * session prelude, a text container, a list container, and heartbeats. No + * images. + * + * It reuses the low-level GATT wrapper (FaceclawBleManager) and the framing / + * protobuf helpers (BleProtocol), but owns its own GATT connection and runs its + * whole lifecycle on a single worker thread. It expects the main app to be + * disconnected while it runs (onboarding, before flashing). + * + * Both arms are connected and security-authenticated up front, even though the + * prompt itself only needs the right arm (the lenses relay messages to each + * other, and acks/events only ever come from the right arm): on an unbonded + * phone the auth exchange is what raises the OS pairing prompt, and doing both + * here means both prompts appear at the start rather than one popping up half + * way through flashing the second lens. While connected, each arm's battery + * level is read (they are independent batteries) so the caller can refuse to + * flash on a low charge. + * + * With `skipPrompt` the on-glasses confirmation is not shown: the flow is just + * connect + auth + battery read + result(approved). Used to re-check the + * battery after the user has already confirmed once. + */ +class FaceclawFlashPromptCommunicator( + context: Context, rightAddress: String?, leftAddress: String?, warningText: String?, skipPrompt: Boolean) : FaceclawBleListener { + companion object { + private const val TAG = "FaceclawFlashPrompt" + + private const val TEXT_NAME = "flashwarn" + private const val LIST_NAME = "flashmenu" + private const val TEXT_CONTAINER_ID = 1 + private const val LIST_CONTAINER_ID = 2 + // Index 0 = decline, index 1 = approve. Kept short for the ~50-col grid. + private val ITEMS = arrayOf("No, cancel", "Yes, flash") + + private const val HEARTBEAT_INTERVAL_MS = 4_000 + private const val SELECTION_TIMEOUT_MS = 120_000 + private const val CREATE_ACK_TIMEOUT_MS = 3_000 + private const val BATTERY_ACK_TIMEOUT_MS = 3_000 + + private fun ackKey(sid: Int, magic: Int): String { + return "$sid:$magic" + } + } + + private val context: Context + private val rightAddress: String + private val leftAddress: String + private val warningText: String + private val skipPrompt: Boolean + private val bleManager: FaceclawBleManager + private val mainHandler = Handler(Looper.getMainLooper()) + + private val lock = Any() + private val pendingAcks = ConcurrentHashMap() + private val ackPayloads = ConcurrentHashMap() + private val selectionLatch = CountDownLatch(1) + + @Volatile + private var listener: FaceclawFlashPromptListener? = null + @Volatile + private var worker: Thread? = null + @Volatile + private var cancelled = false + @Volatile + private var finished = false + @Volatile + private var rightConnected = false + @Volatile + private var leftConnected = false + @Volatile + private var rightLost = false + @Volatile + private var approved: Boolean? = null + @Volatile + private var heartbeatExecutor: ScheduledExecutorService? = null + + private var nextMagic = 100 + private var nextSeq = 0x40 + + init { + this.context = context.applicationContext + this.rightAddress = rightAddress ?: "" + this.leftAddress = leftAddress ?: "" + this.warningText = warningText ?: "" + this.skipPrompt = skipPrompt + this.bleManager = FaceclawBleManager(this.context) + this.bleManager.setListener(this) + } + + fun setListener(listener: FaceclawFlashPromptListener?) { + this.listener = listener + } + + fun start() { + synchronized(lock) { + if (worker != null) { + return + } + val w = Thread({ run() }, "faceclaw-flash-prompt") + worker = w + w.start() + } + } + + /** Abort the prompt (e.g. the user backed out on the phone). */ + fun cancel() { + cancelled = true + selectionLatch.countDown() + val w = worker + if (w != null) { + w.interrupt() + } + } + + fun close() { + cancel() + stopHeartbeat() + try { + bleManager.close() + } catch (ignored: Exception) { + } + } + + private fun run() { + try { + if (rightAddress.trim().isEmpty()) { + emitState("error", "No right-arm address configured.") + return + } + + emitState("connecting", "") + connectArm(rightAddress, "right") + rightConnected = true + if (cancelled) { + teardown() + return + } + if (!leftAddress.trim().isEmpty()) { + connectArm(leftAddress, "left") + leftConnected = true + if (cancelled) { + teardown() + return + } + } + + emitState("connected", "") + // Auth both arms now (pairing prompts, if any, happen here) before + // anything else, so both bonds exist by the time flashing starts. + authenticateArm(rightAddress, "right") + if (leftConnected) { + authenticateArm(leftAddress, "left") + } + if (cancelled) { + teardown() + return + } + sendPrelude(rightAddress) + + if (!skipPrompt) { + showPrompt(rightAddress) + startHeartbeat() + emitState("prompting", "") + + selectionLatch.await(SELECTION_TIMEOUT_MS.toLong(), TimeUnit.MILLISECONDS) + stopHeartbeat() + + if (cancelled) { + emitState("cancelled", "") + teardown() + return + } + if (approved == null) { + if (rightLost) { + emitState("disconnected", "Lost connection to the glasses.") + } else { + emitState("timeout", "No response from the glasses.") + } + teardown() + return + } + try { + sendShutdown() + } catch (ignored: Exception) { + } + if (approved != true) { + emitResult(false) + emitState("result", "declined") + teardown() + return + } + } + + // Approved (or prompt skipped): read each arm's battery while the + // links are still up, then report the result. + emitState("battery", "") + readBatteries() + if (cancelled) { + emitState("cancelled", "") + teardown() + return + } + if (rightLost) { + emitState("disconnected", "Lost connection to the glasses.") + teardown() + return + } + finished = true + emitResult(true) + emitState("result", "approved") + teardown() + } catch (e: Exception) { + stopHeartbeat() + if (!cancelled) { + val message = if (e.message == null) e.toString() else e.message + emitState("error", message) + } + teardown() + } + } + + private fun connectArm(address: String, arm: String) { + if (!bleManager.connect(address, ConnectionOptions.CONNECT_TIMEOUT_MS)) { + throw IllegalStateException("could not connect to the $arm arm ($address)") + } + bleManager.requestConnectionPriority(address, BluetoothGatt.CONNECTION_PRIORITY_HIGH) + bleManager.requestMtu(address, ConnectionOptions.DESIRED_MTU, ConnectionOptions.CONNECT_TIMEOUT_MS) + if (!bleManager.discoverServices(address, ConnectionOptions.SERVICES_TIMEOUT_MS)) { + throw IllegalStateException("discoverServices failed: $arm arm ($address)") + } + if (!bleManager.enableNotifications(address, BleProtocol.NOTIFY_CHAR_UUID, true, ConnectionOptions.DESCRIPTOR_TIMEOUT_MS)) { + throw IllegalStateException("enableNotifications failed: $arm arm ($address)") + } + emitLog("connected $arm arm") + } + + /** + * Security-auth exchange on sid=0x80: firmware 2.2.9 will not run a + * session (and closes the link after ~30 s) without it, and on an + * unbonded phone this is what triggers SMP pairing, so it may sit + * waiting on an OS pairing prompt. + */ + @Throws(InterruptedException::class) + private fun authenticateArm(address: String, arm: String) { + val authMagic = allocMagic() + val authAck = writeAndAwaitAck( + address, + BleProtocol.SID_SECURITY_AUTH, + BleProtocol.FLAG_SECURITY_AUTH, + authMagic, + BleProtocol.buildAuthenticationRequest(authMagic), + ConnectionOptions.SECURITY_AUTH_TIMEOUT_MS) + if (authAck == null || !BleProtocol.isAuthenticationSuccess(authAck, authMagic)) { + throw IllegalStateException( + "could not authenticate with the $arm arm ($address" + + ") — if Android shows a Bluetooth pairing request, accept it and try again") + } + emitLog("security auth complete: $arm arm") + } + + /** Mandatory session prelude on sid=0x01 (app-launch). Right arm only; it relays. */ + @Throws(InterruptedException::class) + private fun sendPrelude(address: String) { + if (writeAndAwaitAck( + address, + BleProtocol.PRELUDE_ACK_SID, + BleProtocol.FLAG_REQUEST, + BleProtocol.PRELUDE_ACK_MAGIC, + BleProtocol.PRELUDE_F5872_PAYLOAD, + ConnectionOptions.PRELUDE_TIMEOUT_MS) == null) { + throw IllegalStateException("session prelude not acked: $address") + } + } + + @Throws(InterruptedException::class) + private fun showPrompt(address: String) { + // Create the prompt page (warning text + No/Yes list) on sid=0xe0 Cmd=0. + // Two attempts: on 2.2.9 the first request sent right after the + // prelude can be dropped while the lens emits its own sid-0x80 + // notifications (observed with the device-info settings read). + var pageAck: ByteArray? = null + var attempt = 0 + while (attempt < 2 && pageAck == null && !cancelled) { + val magic = allocMagic() + val page = BleProtocol.buildCreatePromptPage( + magic, TEXT_NAME, TEXT_CONTAINER_ID, warningText, LIST_NAME, LIST_CONTAINER_ID, ITEMS) + pageAck = writeAndAwaitAck(address, BleProtocol.SID_EVENHUB, BleProtocol.FLAG_REQUEST, magic, page, CREATE_ACK_TIMEOUT_MS) + if (pageAck == null) { + emitLog("prompt page attempt " + (attempt + 1) + " unacked: " + address) + } + attempt++ + } + if (pageAck == null) { + throw IllegalStateException("prompt page not acked: $address") + } + emitLog("prompt page shown on $address") + } + + /** + * Read each arm's battery via a sid-0x09 settings read on that arm's own + * link (the arms have independent batteries and each answers with its + * own). An arm that does not answer reports -1; the caller decides what + * to do with a partial reading. + */ + @Throws(InterruptedException::class) + private fun readBatteries() { + val right = if (rightConnected) readBattery(rightAddress, "right") else -1 + val left = if (leftConnected) readBattery(leftAddress, "left") else -1 + emitLog("battery R=" + (if (right < 0) "?" else "$right%") + " L=" + (if (left < 0) "?" else "$left%")) + emitBattery(right, left) + } + + @Throws(InterruptedException::class) + private fun readBattery(address: String, arm: String): Int { + var attempt = 0 + while (attempt < 2 && !cancelled) { + val magic = allocMagic() + val ack = writeAndAwaitAck( + address, + BleProtocol.SID_UI_SETTING, + BleProtocol.FLAG_REQUEST, + magic, + BleProtocol.buildSettingsQuery(magic), + BATTERY_ACK_TIMEOUT_MS) + if (ack == null) { + emitLog("battery read attempt " + (attempt + 1) + " unacked: " + arm + " arm") + attempt++ + continue + } + val snapshot: BleProtocol.BatterySnapshot? = BleProtocol.parseSettingsBattery(ack) + if (snapshot != null) { + return snapshot.battery + } + emitLog("battery read ack had no battery field: $arm arm") + attempt++ + } + return -1 + } + + /** Write and wait for the matching ack; returns the ack's protobuf (may be empty), or null on timeout/write failure. */ + @Throws(InterruptedException::class) + private fun writeAndAwaitAck(address: String, sid: Int, flag: Int, magic: Int, payload: ByteArray, timeoutMs: Int): ByteArray? { + val latch = CountDownLatch(1) + val key = ackKey(sid, magic) + pendingAcks[key] = latch + try { + if (!writeFrame(address, sid, flag, payload)) { + return null + } + if (!latch.await(timeoutMs.toLong(), TimeUnit.MILLISECONDS)) { + return null + } + val pb = ackPayloads[key] + return pb ?: ByteArray(0) + } finally { + pendingAcks.remove(key, latch) + ackPayloads.remove(key) + } + } + + private fun writeFrame(address: String, sid: Int, flag: Int, payload: ByteArray): Boolean { + val seq: Int + synchronized(lock) { + seq = nextSeq++ and 0xff + } + val frames = BleProtocol.framePb(payload, sid, flag, seq) + return bleManager.writeFrames( + address, + BleProtocol.WRITE_CHAR_UUID, + frames, + AndroidProtocolPlatform.writeType(ConnectionOptions.WRITE_MODE), + ConnectionOptions.WRITE_TIMEOUT_MS) + } + + private fun startHeartbeat() { + stopHeartbeat() + val executor = Executors.newSingleThreadScheduledExecutor { runnable -> + val t = Thread(runnable, "faceclaw-flash-hb") + t.isDaemon = true + t + } + heartbeatExecutor = executor + executor.scheduleWithFixedDelay( + { sendHeartbeat() }, HEARTBEAT_INTERVAL_MS.toLong(), HEARTBEAT_INTERVAL_MS.toLong(), TimeUnit.MILLISECONDS) + } + + private fun sendHeartbeat() { + if (finished || cancelled) { + return + } + try { + val heartbeat = BleProtocol.buildHeartbeat(allocMagic()) + if (rightConnected) { + writeFrame(rightAddress, BleProtocol.SID_EVENHUB, BleProtocol.FLAG_REQUEST, heartbeat) + } + } catch (e: Exception) { + Log.w(TAG, "heartbeat failed: " + e.message) + } + } + + private fun stopHeartbeat() { + val executor = heartbeatExecutor + heartbeatExecutor = null + if (executor != null) { + executor.shutdownNow() + } + } + + private fun sendShutdown() { + val shutdown = BleProtocol.buildShutdown(allocMagic(), 0) + if (rightConnected) { + writeFrame(rightAddress, BleProtocol.SID_EVENHUB, BleProtocol.FLAG_REQUEST, shutdown) + } + } + + private fun teardown() { + finished = true + stopHeartbeat() + try { + bleManager.close() + } catch (ignored: Exception) { + } + rightConnected = false + leftConnected = false + } + + override fun onNotification(address: String?, characteristicUuid: String?, data: ByteArray?) { + if (!BleProtocol.NOTIFY_CHAR_UUID.equals(characteristicUuid, ignoreCase = true)) { + return + } + // A value can carry several envelope frames back to back. + for (buf in BleProtocol.splitFrames(data)) { + handleFrame(address, buf) + } + } + + private fun handleFrame(address: String?, buf: ByteArray) { + val frame = BleProtocol.parseFrame(buf) + if (!frame.ok) { + return + } + if (frame.flag == BleProtocol.FLAG_NOTIFY || frame.flag == BleProtocol.FLAG_NOTIFY_ALT) { + handleEvent(frame) + return + } + if (frame.msgSeq >= 0) { + val key = ackKey(frame.sid, frame.msgSeq) + val latch = pendingAcks[key] + if (latch != null) { + ackPayloads[key] = frame.pb + latch.countDown() + } + } + } + + private fun handleEvent(frame: BleProtocol.ParsedFrame) { + val selection: BleProtocol.ListSelection? = BleProtocol.parseListSelection(frame) + if (selection == null || LIST_NAME != selection.containerName) { + return + } + // Only a confirmed click is a decision; scroll/highlight changes are ignored. + if (selection.eventType != BleProtocol.EVENT_CLICK) { + return + } + val itemName = selection.itemName + val yes = selection.itemIndex == 1 + || (itemName != null && itemName.lowercase().startsWith("yes")) + synchronized(lock) { + if (finished) { + return + } + finished = true + approved = yes + } + emitLog("selection: " + (if (yes) "flash" else "cancel") + " (index " + selection.itemIndex + ")") + selectionLatch.countDown() + } + + override fun onConnectionStateChange(address: String?, connected: Boolean) { + if (connected) { + return + } + if (address!!.equals(leftAddress, ignoreCase = true)) { + // The prompt itself only needs the right arm; a dropped left link + // just means no left battery reading. + leftConnected = false + emitLog("left arm disconnected") + return + } + if (address.equals(rightAddress, ignoreCase = true)) { + rightConnected = false + synchronized(lock) { + if (!finished && !cancelled) { + rightLost = true + finished = true + selectionLatch.countDown() + } + } + } + } + + private fun allocMagic(): Int { + synchronized(lock) { + val magic = nextMagic + nextMagic = if (nextMagic >= 255) 100 else nextMagic + 1 + return magic + } + } + + private fun emitLog(line: String) { + Log.i(TAG, line) + mainHandler.post { + val current = listener + if (current != null) { + current.onLog(line) + } + } + } + + private fun emitState(state: String, detail: String?) { + val safeDetail = detail ?: "" + mainHandler.post { + val current = listener + if (current != null) { + current.onState(state, safeDetail) + } + } + } + + private fun emitBattery(rightPercent: Int, leftPercent: Int) { + mainHandler.post { + val current = listener + if (current != null) { + current.onBattery(rightPercent, leftPercent) + } + } + } + + private fun emitResult(approvedResult: Boolean) { + mainHandler.post { + val current = listener + if (current != null) { + current.onResult(approvedResult) + } + } + } +} diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawForegroundService.java b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawForegroundService.java deleted file mode 100644 index 44e38a0b..00000000 --- a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawForegroundService.java +++ /dev/null @@ -1,142 +0,0 @@ -package com.faceclaw.app; - -import android.Manifest; -import android.app.Notification; -import android.app.NotificationChannel; -import android.app.NotificationManager; -import android.app.PendingIntent; -import android.app.Service; -import android.content.Intent; -import android.content.pm.PackageManager; -import android.content.pm.ServiceInfo; -import android.os.Build; -import android.os.IBinder; - -import androidx.core.content.ContextCompat; - -import com.tns.NativeScriptActivity; - -public class FaceclawForegroundService extends Service { - public static final String ACTION_START = "com.faceclaw.app.action.START"; - public static final String ACTION_UPDATE = "com.faceclaw.app.action.UPDATE"; - public static final String ACTION_STOP = "com.faceclaw.app.action.STOP"; - public static final String EXTRA_TEXT = "text"; - - // Channel id was bumped from "faceclaw-dashboard" when the badge setting - // changed: Android freezes a channel's showBadge flag at creation, so the - // old channel (which let Samsung's launcher count the pinned notification - // as a red "1" badge) is deleted on upgrade rather than reused. - private static final String LEGACY_CHANNEL_ID = "faceclaw-dashboard"; - private static final String CHANNEL_ID = "faceclaw-connection"; - private static final int NOTIFICATION_ID = 4201; - - @Override - public IBinder onBind(Intent intent) { - return null; - } - - @Override - public int onStartCommand(Intent intent, int flags, int startId) { - String action = intent != null ? intent.getAction() : ACTION_START; - String text = intent != null ? intent.getStringExtra(EXTRA_TEXT) : null; - - if (ACTION_STOP.equals(action)) { - stopForeground(STOP_FOREGROUND_REMOVE); - stopSelf(); - return START_NOT_STICKY; - } - - ensureNotificationChannel(); - Notification notification = buildNotification( - text != null && !text.trim().isEmpty() ? text : "Connected to glasses" - ); - - if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q) { - startForeground(NOTIFICATION_ID, notification, foregroundServiceType()); - } else { - startForeground(NOTIFICATION_ID, notification); - } - - if (ACTION_UPDATE.equals(action)) { - NotificationManager manager = getSystemService(NotificationManager.class); - if (manager != null) { - manager.notify(NOTIFICATION_ID, notification); - } - } - - return START_STICKY; - } - - private void ensureNotificationChannel() { - if (Build.VERSION.SDK_INT < Build.VERSION_CODES.O) { - return; - } - - NotificationChannel channel = new NotificationChannel( - CHANNEL_ID, - "Glasses connection", - NotificationManager.IMPORTANCE_LOW - ); - channel.setDescription("Keeps Faceclaw connected to the glasses."); - // The pinned status notification must not count toward the launcher - // icon's notification badge. - channel.setShowBadge(false); - - NotificationManager manager = getSystemService(NotificationManager.class); - if (manager != null) { - manager.createNotificationChannel(channel); - manager.deleteNotificationChannel(LEGACY_CHANNEL_ID); - } - } - - private Notification buildNotification(String text) { - Intent launchIntent = new Intent(this, NativeScriptActivity.class); - launchIntent.addFlags(Intent.FLAG_ACTIVITY_SINGLE_TOP | Intent.FLAG_ACTIVITY_CLEAR_TOP); - - int flags = PendingIntent.FLAG_UPDATE_CURRENT; - if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.M) { - flags |= PendingIntent.FLAG_IMMUTABLE; - } - - PendingIntent contentIntent = PendingIntent.getActivity(this, 0, launchIntent, flags); - - Notification.Builder builder = Build.VERSION.SDK_INT >= Build.VERSION_CODES.O - ? new Notification.Builder(this, CHANNEL_ID) - : new Notification.Builder(this); - - return builder - .setContentTitle("Faceclaw") - .setContentText(text) - .setSmallIcon(getApplicationInfo().icon) - .setContentIntent(contentIntent) - .setOngoing(true) - .setOnlyAlertOnce(true) - .build(); - } - - private int foregroundServiceType() { - int type = ServiceInfo.FOREGROUND_SERVICE_TYPE_CONNECTED_DEVICE; - // TODO: Make this depend on which audio path (G2 vs phone) is selected - if (hasRecordAudioPermission()) { - type |= ServiceInfo.FOREGROUND_SERVICE_TYPE_MICROPHONE; - } - // The location type keeps while-in-use location flowing to the - // Navigate app when the phone screen locks. Only claimed once the - // permission exists: on API 34+ claiming it without the permission - // makes startForeground throw. - if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q && hasFineLocationPermission()) { - type |= ServiceInfo.FOREGROUND_SERVICE_TYPE_LOCATION; - } - return type; - } - - private boolean hasRecordAudioPermission() { - return ContextCompat.checkSelfPermission(this, Manifest.permission.RECORD_AUDIO) - == PackageManager.PERMISSION_GRANTED; - } - - private boolean hasFineLocationPermission() { - return ContextCompat.checkSelfPermission(this, Manifest.permission.ACCESS_FINE_LOCATION) - == PackageManager.PERMISSION_GRANTED; - } -} diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawForegroundService.kt b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawForegroundService.kt new file mode 100644 index 00000000..9b50bd3e --- /dev/null +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawForegroundService.kt @@ -0,0 +1,141 @@ +package com.faceclaw.app + +import android.Manifest +import android.app.Notification +import android.app.NotificationChannel +import android.app.NotificationManager +import android.app.PendingIntent +import android.app.Service +import android.content.Intent +import android.content.pm.PackageManager +import android.content.pm.ServiceInfo +import android.os.Build +import android.os.IBinder + +import androidx.core.content.ContextCompat + +import com.tns.NativeScriptActivity + +class FaceclawForegroundService : Service() { + companion object { + const val ACTION_START = "com.faceclaw.app.action.START" + const val ACTION_UPDATE = "com.faceclaw.app.action.UPDATE" + const val ACTION_STOP = "com.faceclaw.app.action.STOP" + const val EXTRA_TEXT = "text" + + // Channel id was bumped from "faceclaw-dashboard" when the badge setting + // changed: Android freezes a channel's showBadge flag at creation, so the + // old channel (which let Samsung's launcher count the pinned notification + // as a red "1" badge) is deleted on upgrade rather than reused. + private const val LEGACY_CHANNEL_ID = "faceclaw-dashboard" + private const val CHANNEL_ID = "faceclaw-connection" + private const val NOTIFICATION_ID = 4201 + } + + override fun onBind(intent: Intent?): IBinder? { + return null + } + + override fun onStartCommand(intent: Intent?, flags: Int, startId: Int): Int { + val action = if (intent != null) intent.action else ACTION_START + val text = intent?.getStringExtra(EXTRA_TEXT) + + if (ACTION_STOP == action) { + stopForeground(STOP_FOREGROUND_REMOVE) + stopSelf() + return START_NOT_STICKY + } + + ensureNotificationChannel() + val notification = buildNotification( + if (text != null && !text.trim().isEmpty()) text else "Connected to glasses" + ) + + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q) { + startForeground(NOTIFICATION_ID, notification, foregroundServiceType()) + } else { + startForeground(NOTIFICATION_ID, notification) + } + + if (ACTION_UPDATE == action) { + val manager = getSystemService(NotificationManager::class.java) + manager?.notify(NOTIFICATION_ID, notification) + } + + return START_STICKY + } + + private fun ensureNotificationChannel() { + if (Build.VERSION.SDK_INT < Build.VERSION_CODES.O) { + return + } + + val channel = NotificationChannel( + CHANNEL_ID, + "Glasses connection", + NotificationManager.IMPORTANCE_LOW + ) + channel.description = "Keeps Faceclaw connected to the glasses." + // The pinned status notification must not count toward the launcher + // icon's notification badge. + channel.setShowBadge(false) + + val manager = getSystemService(NotificationManager::class.java) + if (manager != null) { + manager.createNotificationChannel(channel) + manager.deleteNotificationChannel(LEGACY_CHANNEL_ID) + } + } + + private fun buildNotification(text: String): Notification { + val launchIntent = Intent(this, NativeScriptActivity::class.java) + launchIntent.addFlags(Intent.FLAG_ACTIVITY_SINGLE_TOP or Intent.FLAG_ACTIVITY_CLEAR_TOP) + + var flags = PendingIntent.FLAG_UPDATE_CURRENT + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.M) { + flags = flags or PendingIntent.FLAG_IMMUTABLE + } + + val contentIntent = PendingIntent.getActivity(this, 0, launchIntent, flags) + + val builder = if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) + Notification.Builder(this, CHANNEL_ID) + else + Notification.Builder(this) + + return builder + .setContentTitle("Faceclaw") + .setContentText(text) + .setSmallIcon(applicationInfo.icon) + .setContentIntent(contentIntent) + .setOngoing(true) + .setOnlyAlertOnce(true) + .build() + } + + private fun foregroundServiceType(): Int { + var type = ServiceInfo.FOREGROUND_SERVICE_TYPE_CONNECTED_DEVICE + // TODO: Make this depend on which audio path (G2 vs phone) is selected + if (hasRecordAudioPermission()) { + type = type or ServiceInfo.FOREGROUND_SERVICE_TYPE_MICROPHONE + } + // The location type keeps while-in-use location flowing to the + // Navigate app when the phone screen locks. Only claimed once the + // permission exists: on API 34+ claiming it without the permission + // makes startForeground throw. + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q && hasFineLocationPermission()) { + type = type or ServiceInfo.FOREGROUND_SERVICE_TYPE_LOCATION + } + return type + } + + private fun hasRecordAudioPermission(): Boolean { + return ContextCompat.checkSelfPermission(this, Manifest.permission.RECORD_AUDIO) == + PackageManager.PERMISSION_GRANTED + } + + private fun hasFineLocationPermission(): Boolean { + return ContextCompat.checkSelfPermission(this, Manifest.permission.ACCESS_FINE_LOCATION) == + PackageManager.PERMISSION_GRANTED + } +} diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawHttp.java b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawHttp.java deleted file mode 100644 index 9f9287a9..00000000 --- a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawHttp.java +++ /dev/null @@ -1,42 +0,0 @@ -package com.faceclaw.app; - -import java.io.IOException; - -import okhttp3.Interceptor; -import okhttp3.Request; -import okhttp3.Response; - -/** - * Shared HTTP identity for the Java networking helpers. The version lives in - * app/version.ts, so the TypeScript side pushes the real string in at startup - * (see installNativeUserAgent in app/util/http.ts); the default below is only - * a fallback for requests that somehow beat that call. - */ -public final class FaceclawHttp { - private static volatile String userAgent = "Faceclaw"; - - private FaceclawHttp() {} - - public static void setUserAgent(String value) { - if (value != null && !value.trim().isEmpty()) { - userAgent = value.trim(); - } - } - - public static String getUserAgent() { - return userAgent; - } - - /** Stamps our User-Agent on any request that doesn't already carry one. */ - public static Interceptor userAgentInterceptor() { - return new Interceptor() { - @Override public Response intercept(Chain chain) throws IOException { - Request request = chain.request(); - if (request.header("User-Agent") != null) { - return chain.proceed(request); - } - return chain.proceed(request.newBuilder().header("User-Agent", userAgent).build()); - } - }; - } -} diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawHttp.kt b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawHttp.kt new file mode 100644 index 00000000..6e5a2855 --- /dev/null +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawHttp.kt @@ -0,0 +1,47 @@ +package com.faceclaw.app + +import java.io.IOException + +import okhttp3.Interceptor +import okhttp3.Request +import okhttp3.Response + +/** + * Shared HTTP identity for the Java networking helpers. The version lives in + * app/version.ts, so the TypeScript side pushes the real string in at startup + * (see installNativeUserAgent in app/util/http.ts); the default below is only + * a fallback for requests that somehow beat that call. + */ +class FaceclawHttp private constructor() { + companion object { + @Volatile + private var userAgent = "Faceclaw" + + @JvmStatic + fun setUserAgent(value: String?) { + if (value != null && !value.trim().isEmpty()) { + userAgent = value.trim() + } + } + + @JvmStatic + fun getUserAgent(): String { + return userAgent + } + + /** Stamps our User-Agent on any request that doesn't already carry one. */ + @JvmStatic + fun userAgentInterceptor(): Interceptor { + return object : Interceptor { + @Throws(IOException::class) + override fun intercept(chain: Interceptor.Chain): Response { + val request: Request = chain.request() + if (request.header("User-Agent") != null) { + return chain.proceed(request) + } + return chain.proceed(request.newBuilder().header("User-Agent", userAgent).build()) + } + } + } + } +} diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawLc3Decoder.java b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawLc3Decoder.java deleted file mode 100644 index 1ed6b950..00000000 --- a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawLc3Decoder.java +++ /dev/null @@ -1,117 +0,0 @@ -package com.faceclaw.app; - -import android.util.Log; - -public class FaceclawLc3Decoder { - private static final String TAG = "FaceclawLc3"; - public static final int SAMPLE_RATE = 16000; - public static final int FRAME_US = 10000; - public static final int FRAME_BYTES = 40; - public static final int FRAMES_PER_PACKET = 5; - public static final int PACKET_BYTES = 205; - public static final int COUNTER_OFFSET = 204; - public static final int SAMPLES_PER_FRAME = 160; - public static final int SAMPLES_PER_PACKET = FRAMES_PER_PACKET * SAMPLES_PER_FRAME; - // Trailer layout (firmware service_audio.c): after the 200 LC3 bytes the - // glasses DSP appends two signed 16-bit LE values computed on the raw - // stereo capture before the mono downmix — a signal-strength ratio and the - // asin-derived direction-of-arrival in degrees — then the packet counter. - public static final int SSR_OFFSET = 200; - public static final int ANGLE_OFFSET = 202; - - static { - System.loadLibrary("faceclaw_lc3"); - } - - private long nativeHandle; - private int lastCounter = -1; - private long realPackets; - private long duplicatePackets; - private long missingPackets; - private long decodeErrors; - private int lastSsr; - private int lastAngleDegrees; - - public FaceclawLc3Decoder() { - nativeHandle = nativeCreate(FRAME_US, SAMPLE_RATE); - if (nativeHandle == 0) { - throw new IllegalStateException("Could not create LC3 decoder"); - } - } - - public synchronized int decodePacket(byte[] packet, short[] pcmOut) { - if (nativeHandle == 0) { - throw new IllegalStateException("LC3 decoder is closed"); - } - if (packet == null || packet.length != PACKET_BYTES) { - decodeErrors++; - Log.w(TAG, "unexpected G2 audio packet length=" + (packet == null ? -1 : packet.length)); - return 0; - } - if (pcmOut == null || pcmOut.length < SAMPLES_PER_PACKET) { - throw new IllegalArgumentException("pcmOut must hold " + SAMPLES_PER_PACKET + " samples"); - } - - int counter = packet[COUNTER_OFFSET] & 0xff; - if (lastCounter >= 0) { - int gap = (counter - lastCounter) & 0xff; - if (gap == 0) { - duplicatePackets++; - return 0; - } - int missing = gap - 1; - if (missing > 0) { - missingPackets += missing; - } - } - - int decoded = nativeDecodePacket(nativeHandle, packet, pcmOut); - if (decoded <= 0) { - decodeErrors++; - return 0; - } - realPackets++; - lastCounter = counter; - lastSsr = (short) ((packet[SSR_OFFSET] & 0xff) | (packet[SSR_OFFSET + 1] << 8)); - lastAngleDegrees = (short) ((packet[ANGLE_OFFSET] & 0xff) | (packet[ANGLE_OFFSET + 1] << 8)); - return decoded; - } - - /** Firmware signal-strength ratio from the most recent decoded packet. */ - public synchronized int getLastSsr() { - return lastSsr; - } - - /** Firmware direction-of-arrival (signed degrees) from the most recent decoded packet. */ - public synchronized int getLastAngleDegrees() { - return lastAngleDegrees; - } - - public synchronized long getRealPackets() { - return realPackets; - } - - public synchronized long getDuplicatePackets() { - return duplicatePackets; - } - - public synchronized long getMissingPackets() { - return missingPackets; - } - - public synchronized long getDecodeErrors() { - return decodeErrors; - } - - public synchronized void close() { - long handle = nativeHandle; - nativeHandle = 0; - if (handle != 0) { - nativeDestroy(handle); - } - } - - private static native long nativeCreate(int frameUs, int sampleRate); - private static native int nativeDecodePacket(long handle, byte[] packet, short[] pcmOut); - private static native void nativeDestroy(long handle); -} diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawLc3Decoder.kt b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawLc3Decoder.kt new file mode 100644 index 00000000..ce53c010 --- /dev/null +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawLc3Decoder.kt @@ -0,0 +1,130 @@ +package com.faceclaw.app + +import android.util.Log + +class FaceclawLc3Decoder { + companion object { + private const val TAG = "FaceclawLc3" + const val SAMPLE_RATE = 16000 + const val FRAME_US = 10000 + const val FRAME_BYTES = 40 + const val FRAMES_PER_PACKET = 5 + const val PACKET_BYTES = 205 + const val COUNTER_OFFSET = 204 + const val SAMPLES_PER_FRAME = 160 + const val SAMPLES_PER_PACKET = FRAMES_PER_PACKET * SAMPLES_PER_FRAME + // Trailer layout (firmware service_audio.c): after the 200 LC3 bytes the + // glasses DSP appends two signed 16-bit LE values computed on the raw + // stereo capture before the mono downmix — a signal-strength ratio and the + // asin-derived direction-of-arrival in degrees — then the packet counter. + const val SSR_OFFSET = 200 + const val ANGLE_OFFSET = 202 + + init { + System.loadLibrary("faceclaw_lc3") + } + + @JvmStatic + private external fun nativeCreate(frameUs: Int, sampleRate: Int): Long + @JvmStatic + private external fun nativeDecodePacket(handle: Long, packet: ByteArray, pcmOut: ShortArray): Int + @JvmStatic + private external fun nativeDestroy(handle: Long) + } + + private var nativeHandle: Long + private var lastCounter = -1 + private var realPackets: Long = 0 + private var duplicatePackets: Long = 0 + private var missingPackets: Long = 0 + private var decodeErrors: Long = 0 + private var lastSsr = 0 + private var lastAngleDegrees = 0 + + init { + nativeHandle = nativeCreate(FRAME_US, SAMPLE_RATE) + if (nativeHandle == 0L) { + throw IllegalStateException("Could not create LC3 decoder") + } + } + + @Synchronized + fun decodePacket(packet: ByteArray?, pcmOut: ShortArray?): Int { + if (nativeHandle == 0L) { + throw IllegalStateException("LC3 decoder is closed") + } + if (packet == null || packet.size != PACKET_BYTES) { + decodeErrors++ + Log.w(TAG, "unexpected G2 audio packet length=" + (packet?.size ?: -1)) + return 0 + } + if (pcmOut == null || pcmOut.size < SAMPLES_PER_PACKET) { + throw IllegalArgumentException("pcmOut must hold $SAMPLES_PER_PACKET samples") + } + + val counter = packet[COUNTER_OFFSET].toInt() and 0xff + if (lastCounter >= 0) { + val gap = (counter - lastCounter) and 0xff + if (gap == 0) { + duplicatePackets++ + return 0 + } + val missing = gap - 1 + if (missing > 0) { + missingPackets += missing.toLong() + } + } + + val decoded = nativeDecodePacket(nativeHandle, packet, pcmOut) + if (decoded <= 0) { + decodeErrors++ + return 0 + } + realPackets++ + lastCounter = counter + lastSsr = ((packet[SSR_OFFSET].toInt() and 0xff) or (packet[SSR_OFFSET + 1].toInt() shl 8)).toShort().toInt() + lastAngleDegrees = ((packet[ANGLE_OFFSET].toInt() and 0xff) or (packet[ANGLE_OFFSET + 1].toInt() shl 8)).toShort().toInt() + return decoded + } + + /** Firmware signal-strength ratio from the most recent decoded packet. */ + @Synchronized + fun getLastSsr(): Int { + return lastSsr + } + + /** Firmware direction-of-arrival (signed degrees) from the most recent decoded packet. */ + @Synchronized + fun getLastAngleDegrees(): Int { + return lastAngleDegrees + } + + @Synchronized + fun getRealPackets(): Long { + return realPackets + } + + @Synchronized + fun getDuplicatePackets(): Long { + return duplicatePackets + } + + @Synchronized + fun getMissingPackets(): Long { + return missingPackets + } + + @Synchronized + fun getDecodeErrors(): Long { + return decodeErrors + } + + @Synchronized + fun close() { + val handle = nativeHandle + nativeHandle = 0 + if (handle != 0L) { + nativeDestroy(handle) + } + } +} diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawLlamaRunner.java b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawLlamaRunner.java deleted file mode 100644 index 17e98834..00000000 --- a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawLlamaRunner.java +++ /dev/null @@ -1,122 +0,0 @@ -package com.faceclaw.app; - -import android.os.Handler; -import android.os.Looper; -import android.util.Log; - -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; - -/** - * On-phone LLM inference for the voice assistant (llama.cpp over JNI). - * - * All model work (load, generate, free) runs on a single background executor - * thread so calls are naturally serialized; listener callbacks are posted to - * the Looper of the thread that constructed this object (the JS thread), the - * same contract as FaceclawSseRequest. cancel() may be called from any thread - * and interrupts the in-flight generation at the next token boundary. - * - * The model stays loaded between generations (turn iterations reuse the KV - * cache prefix); the TypeScript side calls unload() on an idle timer to give - * the ~3GB back. - */ -public class FaceclawLlamaRunner { - private static final String TAG = "FaceclawLlamaRunner"; - - static { - System.loadLibrary("faceclaw_llama"); - } - - private final Handler callbackHandler; - private final ExecutorService executor = Executors.newSingleThreadExecutor(runnable -> { - Thread thread = new Thread(runnable, "FaceclawLlama"); - thread.setPriority(Thread.NORM_PRIORITY); - return thread; - }); - - // Mutated only on the executor thread; volatile so cancel() can read it. - private volatile long handle; - private String loadedPath; - - public FaceclawLlamaRunner() { - Looper looper = Looper.myLooper(); - callbackHandler = new Handler(looper != null ? looper : Looper.getMainLooper()); - } - - public boolean isModelLoaded() { - return handle != 0; - } - - /** - * Generate a completion, loading (or swapping) the model first if needed. - * grammar may be null/empty for unconstrained generation. - */ - public void generate(final String modelPath, final int nCtx, final int nThreads, - final String prompt, final String grammar, final int maxTokens, - final float temperature, final float topP, final int topK, - final FaceclawLlamaListener listener) { - executor.execute(() -> { - try { - if (handle != 0 && !modelPath.equals(loadedPath)) { - nativeFree(handle); - handle = 0; - loadedPath = null; - } - if (handle == 0) { - long loaded = nativeLoadModel(modelPath, nCtx, nThreads); - if (loaded == 0) { - post(() -> listener.onError("Could not load the on-phone model")); - return; - } - handle = loaded; - loadedPath = modelPath; - } - nativeGenerate(handle, prompt, grammar == null ? "" : grammar, - maxTokens, temperature, topP, topK, new FaceclawLlamaListener() { - @Override public void onToken(final String piece) { - post(() -> listener.onToken(piece)); - } - @Override public void onDone(final String stopReason) { - post(() -> listener.onDone(stopReason)); - } - @Override public void onError(final String message) { - post(() -> listener.onError(message)); - } - }); - } catch (Throwable t) { - Log.e(TAG, "generation failed", t); - final String message = String.valueOf(t); - post(() -> listener.onError("On-phone model failed: " + message)); - } - }); - } - - /** Interrupt the current generation (its listener gets onDone("cancelled")). */ - public void cancel() { - long current = handle; - if (current != 0) { - nativeCancel(current); - } - } - - /** Free the model and its KV cache. Safe to call when nothing is loaded. */ - public void unload() { - executor.execute(() -> { - if (handle != 0) { - nativeFree(handle); - handle = 0; - loadedPath = null; - } - }); - } - - private void post(Runnable runnable) { - callbackHandler.post(runnable); - } - - private static native long nativeLoadModel(String path, int nCtx, int nThreads); - private static native void nativeGenerate(long handle, String prompt, String grammar, - int maxTokens, float temperature, float topP, int topK, FaceclawLlamaListener listener); - private static native void nativeCancel(long handle); - private static native void nativeFree(long handle); -} diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawLlamaRunner.kt b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawLlamaRunner.kt new file mode 100644 index 00000000..9cc3fa66 --- /dev/null +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawLlamaRunner.kt @@ -0,0 +1,133 @@ +package com.faceclaw.app + +import android.os.Handler +import android.os.Looper +import android.util.Log + +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors + +/** + * On-phone LLM inference for the voice assistant (llama.cpp over JNI). + * + * All model work (load, generate, free) runs on a single background executor + * thread so calls are naturally serialized; listener callbacks are posted to + * the Looper of the thread that constructed this object (the JS thread), the + * same contract as FaceclawSseRequest. cancel() may be called from any thread + * and interrupts the in-flight generation at the next token boundary. + * + * The model stays loaded between generations (turn iterations reuse the KV + * cache prefix); the TypeScript side calls unload() on an idle timer to give + * the ~3GB back. + */ +class FaceclawLlamaRunner { + companion object { + private const val TAG = "FaceclawLlamaRunner" + + init { + System.loadLibrary("faceclaw_llama") + } + + @JvmStatic + private external fun nativeLoadModel(path: String?, nCtx: Int, nThreads: Int): Long + @JvmStatic + private external fun nativeGenerate( + handle: Long, prompt: String?, grammar: String, + maxTokens: Int, temperature: Float, topP: Float, topK: Int, listener: FaceclawLlamaListener + ) + @JvmStatic + private external fun nativeCancel(handle: Long) + @JvmStatic + private external fun nativeFree(handle: Long) + } + + private val callbackHandler: Handler + private val executor: ExecutorService = Executors.newSingleThreadExecutor { runnable -> + val thread = Thread(runnable, "FaceclawLlama") + thread.priority = Thread.NORM_PRIORITY + thread + } + + // Mutated only on the executor thread; volatile so cancel() can read it. + @Volatile + private var handle: Long = 0 + private var loadedPath: String? = null + + init { + val looper = Looper.myLooper() + callbackHandler = Handler(looper ?: Looper.getMainLooper()) + } + + fun isModelLoaded(): Boolean { + return handle != 0L + } + + /** + * Generate a completion, loading (or swapping) the model first if needed. + * grammar may be null/empty for unconstrained generation. + */ + fun generate( + modelPath: String?, nCtx: Int, nThreads: Int, + prompt: String?, grammar: String?, maxTokens: Int, + temperature: Float, topP: Float, topK: Int, + listener: FaceclawLlamaListener + ) { + executor.execute { + try { + if (handle != 0L && modelPath != loadedPath) { + nativeFree(handle) + handle = 0 + loadedPath = null + } + if (handle == 0L) { + val loaded = nativeLoadModel(modelPath, nCtx, nThreads) + if (loaded == 0L) { + post { listener.onError("Could not load the on-phone model") } + return@execute + } + handle = loaded + loadedPath = modelPath + } + nativeGenerate(handle, prompt, grammar ?: "", + maxTokens, temperature, topP, topK, object : FaceclawLlamaListener { + override fun onToken(piece: String?) { + post { listener.onToken(piece) } + } + override fun onDone(stopReason: String?) { + post { listener.onDone(stopReason) } + } + override fun onError(message: String?) { + post { listener.onError(message) } + } + }) + } catch (t: Throwable) { + Log.e(TAG, "generation failed", t) + val message = t.toString() + post { listener.onError("On-phone model failed: $message") } + } + } + } + + /** Interrupt the current generation (its listener gets onDone("cancelled")). */ + fun cancel() { + val current = handle + if (current != 0L) { + nativeCancel(current) + } + } + + /** Free the model and its KV cache. Safe to call when nothing is loaded. */ + fun unload() { + executor.execute { + if (handle != 0L) { + nativeFree(handle) + handle = 0 + loadedPath = null + } + } + } + + private fun post(runnable: Runnable) { + callbackHandler.post(runnable) + } +} diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawLocationProvider.java b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawLocationProvider.java deleted file mode 100644 index 90e610c9..00000000 --- a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawLocationProvider.java +++ /dev/null @@ -1,196 +0,0 @@ -package com.faceclaw.app; - -import android.Manifest; -import android.content.Context; -import android.content.pm.PackageManager; -import android.location.Criteria; -import android.location.Location; -import android.location.LocationListener; -import android.location.LocationManager; -import android.os.Bundle; -import android.os.Handler; -import android.os.Looper; - -import androidx.core.content.ContextCompat; - -import java.util.List; - -/** - * One-shot, foreground-only location lookup for Weather. It prefers a fresh - * coarse network fix, but can fall back to the newest cached provider fix if - * Android cannot produce a new location before the timeout. - */ -public final class FaceclawLocationProvider implements LocationListener { - private static final long FRESH_CACHE_MS = 10L * 60L * 1000L; - private static final long MAX_CACHE_MS = 24L * 60L * 60L * 1000L; - private static final long TIMEOUT_MS = 15L * 1000L; - - private final Context context; - private final LocationManager locationManager; - private final Handler mainHandler = new Handler(Looper.getMainLooper()); - private final Runnable timeout = this::onTimeout; - - private FaceclawLocationListener listener; - private Location cachedLocation; - private boolean running; - - public FaceclawLocationProvider(Context context) { - this.context = context.getApplicationContext(); - this.locationManager = (LocationManager) this.context.getSystemService(Context.LOCATION_SERVICE); - } - - public void setListener(FaceclawLocationListener listener) { - this.listener = listener; - } - - public void start() { - mainHandler.post(this::startOnMainThread); - } - - public void cancel() { - mainHandler.post(() -> finish(null, null)); - } - - private void startOnMainThread() { - if (running) { - return; - } - if (locationManager == null) { - deliverError("Android location service is unavailable."); - return; - } - if (ContextCompat.checkSelfPermission(context, Manifest.permission.ACCESS_COARSE_LOCATION) - != PackageManager.PERMISSION_GRANTED - && ContextCompat.checkSelfPermission(context, Manifest.permission.ACCESS_FINE_LOCATION) - != PackageManager.PERMISSION_GRANTED) { - deliverError("Location permission is required."); - return; - } - - cachedLocation = newestCachedLocation(); - if (isRecent(cachedLocation, FRESH_CACHE_MS)) { - deliverLocation(cachedLocation); - return; - } - - String provider = chooseProvider(); - if (provider == null) { - if (isRecent(cachedLocation, MAX_CACHE_MS)) { - deliverLocation(cachedLocation); - } else { - deliverError("Turn on Location on your phone, then retry."); - } - return; - } - - try { - running = true; - locationManager.requestSingleUpdate(provider, this, Looper.getMainLooper()); - mainHandler.postDelayed(timeout, TIMEOUT_MS); - } catch (SecurityException error) { - finish(null, "Location permission is required."); - } catch (Throwable error) { - finish(null, "Unable to request the current location."); - } - } - - private String chooseProvider() { - try { - if (locationManager.isProviderEnabled(LocationManager.NETWORK_PROVIDER)) { - return LocationManager.NETWORK_PROVIDER; - } - Criteria criteria = new Criteria(); - criteria.setAccuracy(Criteria.ACCURACY_COARSE); - criteria.setPowerRequirement(Criteria.POWER_LOW); - return locationManager.getBestProvider(criteria, true); - } catch (Throwable ignored) { - return null; - } - } - - private Location newestCachedLocation() { - Location newest = null; - try { - List providers = locationManager.getProviders(true); - for (String provider : providers) { - Location candidate = locationManager.getLastKnownLocation(provider); - if (candidate != null && (newest == null || candidate.getTime() > newest.getTime())) { - newest = candidate; - } - } - } catch (SecurityException ignored) { - // The explicit permission check above owns the user-facing error. - } catch (Throwable ignored) { - // A fresh request may still succeed even if cached providers fail. - } - return newest; - } - - private static boolean isRecent(Location location, long maximumAgeMs) { - return location != null - && location.getTime() > 0L - && System.currentTimeMillis() - location.getTime() <= maximumAgeMs; - } - - private void onTimeout() { - if (isRecent(cachedLocation, MAX_CACHE_MS)) { - finish(cachedLocation, null); - } else { - finish(null, "Couldn't get your current location. Tap to retry."); - } - } - - @Override - public void onLocationChanged(Location location) { - finish(location, null); - } - - @Override - public void onProviderDisabled(String provider) { - // Keep waiting: Android may still deliver a queued fix or the timeout - // can fall back to another provider's cached location. - } - - @Override - public void onProviderEnabled(String provider) { - } - - @Override - @SuppressWarnings("deprecation") - public void onStatusChanged(String provider, int status, Bundle extras) { - } - - private void finish(Location location, String error) { - if (running) { - try { - locationManager.removeUpdates(this); - } catch (Throwable ignored) { - } - } - running = false; - mainHandler.removeCallbacks(timeout); - if (location != null) { - deliverLocation(location); - } else if (error != null) { - deliverError(error); - } - } - - private void deliverLocation(Location location) { - FaceclawLocationListener current = listener; - if (current != null && location != null) { - current.onLocation( - location.getLatitude(), - location.getLongitude(), - location.hasAccuracy() ? location.getAccuracy() : -1f, - location.getTime()); - } - } - - private void deliverError(String message) { - FaceclawLocationListener current = listener; - if (current != null) { - current.onError(message); - } - } -} diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawLocationProvider.kt b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawLocationProvider.kt new file mode 100644 index 00000000..ce3a7568 --- /dev/null +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawLocationProvider.kt @@ -0,0 +1,188 @@ +package com.faceclaw.app + +import android.Manifest +import android.content.Context +import android.content.pm.PackageManager +import android.location.Criteria +import android.location.Location +import android.location.LocationListener +import android.location.LocationManager +import android.os.Bundle +import android.os.Handler +import android.os.Looper + +import androidx.core.content.ContextCompat + +/** + * One-shot, foreground-only location lookup for Weather. It prefers a fresh + * coarse network fix, but can fall back to the newest cached provider fix if + * Android cannot produce a new location before the timeout. + */ +class FaceclawLocationProvider(context: Context) : LocationListener { + companion object { + private const val FRESH_CACHE_MS = 10L * 60L * 1000L + private const val MAX_CACHE_MS = 24L * 60L * 60L * 1000L + private const val TIMEOUT_MS = 15L * 1000L + + private fun isRecent(location: Location?, maximumAgeMs: Long): Boolean { + return location != null && + location.time > 0L && + System.currentTimeMillis() - location.time <= maximumAgeMs + } + } + + private val context: Context = context.applicationContext + private val locationManager: LocationManager? = + this.context.getSystemService(Context.LOCATION_SERVICE) as LocationManager? + private val mainHandler = Handler(Looper.getMainLooper()) + private val timeout = Runnable { onTimeout() } + + private var listener: FaceclawLocationListener? = null + private var cachedLocation: Location? = null + private var running = false + + fun setListener(listener: FaceclawLocationListener?) { + this.listener = listener + } + + fun start() { + mainHandler.post { startOnMainThread() } + } + + fun cancel() { + mainHandler.post { finish(null, null) } + } + + private fun startOnMainThread() { + if (running) { + return + } + val manager = locationManager + if (manager == null) { + deliverError("Android location service is unavailable.") + return + } + if (ContextCompat.checkSelfPermission(context, Manifest.permission.ACCESS_COARSE_LOCATION) != + PackageManager.PERMISSION_GRANTED && + ContextCompat.checkSelfPermission(context, Manifest.permission.ACCESS_FINE_LOCATION) != + PackageManager.PERMISSION_GRANTED) { + deliverError("Location permission is required.") + return + } + + cachedLocation = newestCachedLocation() + if (isRecent(cachedLocation, FRESH_CACHE_MS)) { + deliverLocation(cachedLocation) + return + } + + val provider = chooseProvider() + if (provider == null) { + if (isRecent(cachedLocation, MAX_CACHE_MS)) { + deliverLocation(cachedLocation) + } else { + deliverError("Turn on Location on your phone, then retry.") + } + return + } + + try { + running = true + @Suppress("DEPRECATION") + manager.requestSingleUpdate(provider, this, Looper.getMainLooper()) + mainHandler.postDelayed(timeout, TIMEOUT_MS) + } catch (error: SecurityException) { + finish(null, "Location permission is required.") + } catch (error: Throwable) { + finish(null, "Unable to request the current location.") + } + } + + private fun chooseProvider(): String? { + try { + if (locationManager!!.isProviderEnabled(LocationManager.NETWORK_PROVIDER)) { + return LocationManager.NETWORK_PROVIDER + } + val criteria = Criteria() + criteria.accuracy = Criteria.ACCURACY_COARSE + criteria.powerRequirement = Criteria.POWER_LOW + return locationManager.getBestProvider(criteria, true) + } catch (ignored: Throwable) { + return null + } + } + + private fun newestCachedLocation(): Location? { + var newest: Location? = null + try { + val providers = locationManager!!.getProviders(true) + for (provider in providers) { + val candidate = locationManager.getLastKnownLocation(provider) + if (candidate != null && (newest == null || candidate.time > newest.time)) { + newest = candidate + } + } + } catch (ignored: SecurityException) { + // The explicit permission check above owns the user-facing error. + } catch (ignored: Throwable) { + // A fresh request may still succeed even if cached providers fail. + } + return newest + } + + private fun onTimeout() { + if (isRecent(cachedLocation, MAX_CACHE_MS)) { + finish(cachedLocation, null) + } else { + finish(null, "Couldn't get your current location. Tap to retry.") + } + } + + override fun onLocationChanged(location: Location) { + finish(location, null) + } + + override fun onProviderDisabled(provider: String) { + // Keep waiting: Android may still deliver a queued fix or the timeout + // can fall back to another provider's cached location. + } + + override fun onProviderEnabled(provider: String) { + } + + @Deprecated("Deprecated in Java") + override fun onStatusChanged(provider: String?, status: Int, extras: Bundle?) { + } + + private fun finish(location: Location?, error: String?) { + if (running) { + try { + locationManager!!.removeUpdates(this) + } catch (ignored: Throwable) { + } + } + running = false + mainHandler.removeCallbacks(timeout) + if (location != null) { + deliverLocation(location) + } else if (error != null) { + deliverError(error) + } + } + + private fun deliverLocation(location: Location?) { + val current = listener + if (current != null && location != null) { + current.onLocation( + location.latitude, + location.longitude, + if (location.hasAccuracy()) location.accuracy else -1f, + location.time) + } + } + + private fun deliverError(message: String) { + val current = listener + current?.onError(message) + } +} diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawLocationTracker.java b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawLocationTracker.java deleted file mode 100644 index c39f3fa6..00000000 --- a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawLocationTracker.java +++ /dev/null @@ -1,188 +0,0 @@ -package com.faceclaw.app; - -import android.Manifest; -import android.content.Context; -import android.content.pm.PackageManager; -import android.location.Location; -import android.location.LocationListener; -import android.location.LocationManager; -import android.os.Bundle; -import android.os.Handler; -import android.os.Looper; -import android.os.SystemClock; - -import androidx.core.content.ContextCompat; - -/** - * Continuous location updates for turn-by-turn navigation. Unlike - * FaceclawLocationProvider (one-shot, coarse, for Weather), this streams GPS - * fixes with bearing and speed until stopped. - * - * Callbacks are delivered on the Looper of the thread that constructed the - * tracker, so a worker isolate receives them on its own thread (the same - * convention as FaceclawSseRequest). - */ -public final class FaceclawLocationTracker implements LocationListener { - private final Context context; - private final LocationManager locationManager; - private final Handler callbackHandler; - - /** - * Only seed from the platform's cached fix when it is this fresh. An older - * one is likely wherever the phone was last used (for example the previous - * navigation's destination), and guidance built on it is wrong from the - * first frame: the route starts there and arrival fires immediately. - */ - private static final long MAX_SEED_AGE_MS = 60L * 1000L; - - private FaceclawLocationTrackerListener listener; - private boolean running; - - public FaceclawLocationTracker(Context context) { - this.context = context.getApplicationContext(); - this.locationManager = (LocationManager) this.context.getSystemService(Context.LOCATION_SERVICE); - Looper looper = Looper.myLooper(); - this.callbackHandler = new Handler(looper != null ? looper : Looper.getMainLooper()); - } - - public void setListener(FaceclawLocationTrackerListener listener) { - this.listener = listener; - } - - /** Begin streaming fixes at roughly the requested interval. */ - public void start(long intervalMs) { - callbackHandler.post(() -> startOnCallbackThread(intervalMs)); - } - - public void stop() { - callbackHandler.post(() -> { - if (!running) { - return; - } - running = false; - try { - locationManager.removeUpdates(this); - } catch (Throwable ignored) { - } - }); - } - - private void startOnCallbackThread(long intervalMs) { - if (running) { - return; - } - if (locationManager == null) { - deliverError("Android location service is unavailable."); - return; - } - if (ContextCompat.checkSelfPermission(context, Manifest.permission.ACCESS_FINE_LOCATION) - != PackageManager.PERMISSION_GRANTED) { - deliverError("Precise location permission is required for navigation."); - return; - } - - String provider = chooseProvider(); - if (provider == null) { - deliverError("Turn on Location on your phone, then retry."); - return; - } - - try { - running = true; - locationManager.requestLocationUpdates(provider, Math.max(500L, intervalMs), 0f, this, - callbackHandler.getLooper()); - // Seed with the freshest cached fix so the UI has a position - // before the first live fix (GPS cold starts can take a while), - // but only if it's recent enough to still be where the user is. - Location cached = freshestCachedLocation(provider); - if (cached != null) { - deliverLocation(cached); - } - } catch (SecurityException error) { - running = false; - deliverError("Precise location permission is required for navigation."); - } catch (Throwable error) { - running = false; - deliverError("Unable to start location updates."); - } - } - - private Location freshestCachedLocation(String primaryProvider) { - Location best = null; - for (String provider : new String[] {primaryProvider, LocationManager.NETWORK_PROVIDER, - LocationManager.GPS_PROVIDER}) { - Location candidate; - try { - candidate = locationManager.getLastKnownLocation(provider); - } catch (Throwable ignored) { - continue; - } - if (candidate == null || ageMs(candidate) > MAX_SEED_AGE_MS) { - continue; - } - if (best == null || candidate.getElapsedRealtimeNanos() > best.getElapsedRealtimeNanos()) { - best = candidate; - } - } - return best; - } - - /** Age via the monotonic clock, so a wall-clock change can't make an old fix look fresh. */ - private static long ageMs(Location location) { - long ageNanos = SystemClock.elapsedRealtimeNanos() - location.getElapsedRealtimeNanos(); - return ageNanos / 1_000_000L; - } - - private String chooseProvider() { - try { - if (locationManager.isProviderEnabled(LocationManager.GPS_PROVIDER)) { - return LocationManager.GPS_PROVIDER; - } - if (locationManager.isProviderEnabled(LocationManager.NETWORK_PROVIDER)) { - return LocationManager.NETWORK_PROVIDER; - } - } catch (Throwable ignored) { - } - return null; - } - - @Override - public void onLocationChanged(Location location) { - deliverLocation(location); - } - - @Override - public void onProviderDisabled(String provider) { - deliverError("Location was turned off on the phone."); - } - - @Override - public void onProviderEnabled(String provider) { - } - - @Override - @SuppressWarnings("deprecation") - public void onStatusChanged(String provider, int status, Bundle extras) { - } - - private void deliverLocation(Location location) { - FaceclawLocationTrackerListener current = listener; - if (current == null || location == null) { - return; - } - current.onLocation( - location.getLatitude(), - location.getLongitude(), - location.hasAccuracy() ? location.getAccuracy() : -1f, - location.hasBearing() ? location.getBearing() : -1f, - location.hasSpeed() ? location.getSpeed() : -1f, - location.getTime()); - } - - private void deliverError(String message) { - FaceclawLocationTrackerListener current = listener; - if (current != null) { - current.onError(message); - } - } -} diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawLocationTracker.kt b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawLocationTracker.kt new file mode 100644 index 00000000..9eb4e255 --- /dev/null +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawLocationTracker.kt @@ -0,0 +1,184 @@ +package com.faceclaw.app + +import android.Manifest +import android.content.Context +import android.content.pm.PackageManager +import android.location.Location +import android.location.LocationListener +import android.location.LocationManager +import android.os.Bundle +import android.os.Handler +import android.os.Looper +import android.os.SystemClock + +import androidx.core.content.ContextCompat + +/** + * Continuous location updates for turn-by-turn navigation. Unlike + * FaceclawLocationProvider (one-shot, coarse, for Weather), this streams GPS + * fixes with bearing and speed until stopped. + * + * Callbacks are delivered on the Looper of the thread that constructed the + * tracker, so a worker isolate receives them on its own thread (the same + * convention as FaceclawSseRequest). + */ +class FaceclawLocationTracker(context: Context) : LocationListener { + companion object { + /** + * Only seed from the platform's cached fix when it is this fresh. An older + * one is likely wherever the phone was last used (for example the previous + * navigation's destination), and guidance built on it is wrong from the + * first frame: the route starts there and arrival fires immediately. + */ + private const val MAX_SEED_AGE_MS = 60L * 1000L + + /** Age via the monotonic clock, so a wall-clock change can't make an old fix look fresh. */ + private fun ageMs(location: Location): Long { + val ageNanos = SystemClock.elapsedRealtimeNanos() - location.elapsedRealtimeNanos + return ageNanos / 1_000_000L + } + } + + private val context: Context = context.applicationContext + private val locationManager: LocationManager? = + this.context.getSystemService(Context.LOCATION_SERVICE) as LocationManager? + private val callbackHandler: Handler + + private var listener: FaceclawLocationTrackerListener? = null + private var running = false + + init { + val looper = Looper.myLooper() + this.callbackHandler = Handler(looper ?: Looper.getMainLooper()) + } + + fun setListener(listener: FaceclawLocationTrackerListener?) { + this.listener = listener + } + + /** Begin streaming fixes at roughly the requested interval. */ + fun start(intervalMs: Long) { + callbackHandler.post { startOnCallbackThread(intervalMs) } + } + + fun stop() { + callbackHandler.post { + if (!running) { + return@post + } + running = false + try { + locationManager!!.removeUpdates(this) + } catch (ignored: Throwable) { + } + } + } + + private fun startOnCallbackThread(intervalMs: Long) { + if (running) { + return + } + val manager = locationManager + if (manager == null) { + deliverError("Android location service is unavailable.") + return + } + if (ContextCompat.checkSelfPermission(context, Manifest.permission.ACCESS_FINE_LOCATION) != + PackageManager.PERMISSION_GRANTED) { + deliverError("Precise location permission is required for navigation.") + return + } + + val provider = chooseProvider() + if (provider == null) { + deliverError("Turn on Location on your phone, then retry.") + return + } + + try { + running = true + manager.requestLocationUpdates(provider, Math.max(500L, intervalMs), 0f, this, + callbackHandler.looper) + // Seed with the freshest cached fix so the UI has a position + // before the first live fix (GPS cold starts can take a while), + // but only if it's recent enough to still be where the user is. + val cached = freshestCachedLocation(provider) + if (cached != null) { + deliverLocation(cached) + } + } catch (error: SecurityException) { + running = false + deliverError("Precise location permission is required for navigation.") + } catch (error: Throwable) { + running = false + deliverError("Unable to start location updates.") + } + } + + private fun freshestCachedLocation(primaryProvider: String): Location? { + var best: Location? = null + for (provider in arrayOf(primaryProvider, LocationManager.NETWORK_PROVIDER, + LocationManager.GPS_PROVIDER)) { + val candidate: Location? + try { + candidate = locationManager!!.getLastKnownLocation(provider) + } catch (ignored: Throwable) { + continue + } + if (candidate == null || ageMs(candidate) > MAX_SEED_AGE_MS) { + continue + } + if (best == null || candidate.elapsedRealtimeNanos > best.elapsedRealtimeNanos) { + best = candidate + } + } + return best + } + + private fun chooseProvider(): String? { + try { + if (locationManager!!.isProviderEnabled(LocationManager.GPS_PROVIDER)) { + return LocationManager.GPS_PROVIDER + } + if (locationManager.isProviderEnabled(LocationManager.NETWORK_PROVIDER)) { + return LocationManager.NETWORK_PROVIDER + } + } catch (ignored: Throwable) { + } + return null + } + + override fun onLocationChanged(location: Location) { + deliverLocation(location) + } + + override fun onProviderDisabled(provider: String) { + deliverError("Location was turned off on the phone.") + } + + override fun onProviderEnabled(provider: String) { + } + + @Deprecated("Deprecated in Java") + override fun onStatusChanged(provider: String?, status: Int, extras: Bundle?) { + } + + private fun deliverLocation(location: Location?) { + val current = listener + if (current == null || location == null) { + return + } + current.onLocation( + location.latitude, + location.longitude, + if (location.hasAccuracy()) location.accuracy else -1f, + if (location.hasBearing()) location.bearing else -1f, + if (location.hasSpeed()) location.speed else -1f, + location.time) + } + + private fun deliverError(message: String) { + val current = listener + current?.onError(message) + } +} diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawMediaBrowser.java b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawMediaBrowser.java deleted file mode 100644 index 07bef961..00000000 --- a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawMediaBrowser.java +++ /dev/null @@ -1,321 +0,0 @@ -package com.faceclaw.app; - -import android.content.ComponentName; -import android.content.Context; -import android.content.Intent; -import android.content.pm.PackageManager; -import android.content.pm.ResolveInfo; -import android.media.browse.MediaBrowser; -import android.media.session.MediaController; -import android.os.Handler; -import android.os.Looper; -import android.os.SystemClock; -import android.util.Log; - -import org.json.JSONArray; -import org.json.JSONObject; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; - -/** - * Client for other apps' MediaBrowserService implementations (the mechanism - * Android Auto uses). Binding starts the target player's process even when it - * is not running, so this can list a player's library (playlists, albums, ...) - * and start playback without the phone being unlocked. - * - * Holds at most one connection at a time. All MediaBrowser interaction happens - * on the main thread; results are reported through - * FaceclawMediaBrowserListener, also on the main thread. - */ -public class FaceclawMediaBrowser { - private static final String TAG = "FaceclawMediaBrowser"; - private static final String BROWSER_SERVICE_ACTION = "android.media.browse.MediaBrowserService"; - // Generous: connecting may cold-start the player's process. - private static final long CONNECT_TIMEOUT_MS = 15_000; - private static final long BROWSE_TIMEOUT_MS = 15_000; - // Unbinding immediately after a play command can race the player promoting - // itself to a foreground service, letting the system kill its process - // before playback starts; keep the binding alive briefly instead. - private static final long PLAY_DISCONNECT_GRACE_MS = 5_000; - - private final Context appContext; - private final Handler mainHandler = new Handler(Looper.getMainLooper()); - private volatile FaceclawMediaBrowserListener listener; - - // Main-thread only. - private MediaBrowser browser; - private int generation; - private long lastPlayCommandAtMs; - - public FaceclawMediaBrowser(Context context) { - this.appContext = context.getApplicationContext(); - } - - public void setListener(FaceclawMediaBrowserListener listener) { - this.listener = listener; - } - - /** - * Installed apps exposing a media browser service, as JSON sorted by app - * name: [{"packageName": ..., "serviceClass": ..., "appName": ...}, ...]. - * Synchronous (PackageManager lookup only). - */ - public String listBrowsableAppsJson() { - PackageManager packageManager = appContext.getPackageManager(); - List services; - try { - services = packageManager.queryIntentServices(new Intent(BROWSER_SERVICE_ACTION), 0); - } catch (Exception e) { - Log.w(TAG, "media browser service query failed", e); - return "[]"; - } - List apps = new ArrayList<>(); - for (ResolveInfo service : services) { - if (service.serviceInfo == null) { - continue; - } - try { - CharSequence label = service.serviceInfo.applicationInfo.loadLabel(packageManager); - JSONObject app = new JSONObject(); - app.put("packageName", service.serviceInfo.packageName); - app.put("serviceClass", service.serviceInfo.name); - app.put("appName", label == null ? service.serviceInfo.packageName : label.toString()); - apps.add(app); - } catch (Exception e) { - Log.w(TAG, "media browser service entry failed", e); - } - } - Collections.sort(apps, (a, b) -> a.optString("appName").compareToIgnoreCase(b.optString("appName"))); - return new JSONArray(apps).toString(); - } - - /** - * Connect to an app's media browser service, starting its process if - * needed; replaces any existing connection. Async; reports through - * onConnectResult with the browse-tree root id on success. - */ - public void connect(int requestId, String packageName, String serviceClass) { - mainHandler.post(() -> { - closeBrowser(); - final int myGeneration = generation; - final boolean[] settled = {false}; - final MediaBrowser[] browserRef = new MediaBrowser[1]; - MediaBrowser newBrowser = new MediaBrowser( - appContext, - new ComponentName(packageName, serviceClass), - new MediaBrowser.ConnectionCallback() { - @Override - public void onConnected() { - if (generation != myGeneration || settled[0]) { - return; - } - settled[0] = true; - String rootId = ""; - try { - rootId = browserRef[0].getRoot(); - } catch (Exception e) { - Log.w(TAG, "getRoot failed", e); - } - emitConnectResult(requestId, true, rootId, ""); - } - - @Override - public void onConnectionFailed() { - if (generation != myGeneration || settled[0]) { - return; - } - settled[0] = true; - closeBrowser(); - emitConnectResult(requestId, false, "", "Connection refused by " + packageName); - } - - @Override - public void onConnectionSuspended() { - if (generation != myGeneration) { - return; - } - FaceclawMediaBrowserListener currentListener = listener; - if (currentListener != null) { - currentListener.onDisconnected(); - } - } - }, - null - ); - browserRef[0] = newBrowser; - browser = newBrowser; - mainHandler.postDelayed(() -> { - if (generation != myGeneration || settled[0]) { - return; - } - settled[0] = true; - closeBrowser(); - emitConnectResult(requestId, false, "", "Connection to " + packageName + " timed out"); - }, CONNECT_TIMEOUT_MS); - try { - newBrowser.connect(); - } catch (Exception e) { - if (!settled[0]) { - settled[0] = true; - closeBrowser(); - emitConnectResult(requestId, false, "", "Connection failed: " + e.getMessage()); - } - } - }); - } - - /** - * Fetch one level of the connected service's content tree. Async; reports - * through onBrowseResult with children as JSON: - * [{"mediaId": ..., "title": ..., "subtitle": ..., "browsable": bool, "playable": bool}, ...]. - */ - public void browse(int requestId, String parentId) { - mainHandler.post(() -> { - final MediaBrowser currentBrowser = browser; - if (currentBrowser == null || !currentBrowser.isConnected()) { - emitBrowseResult(requestId, "", "Not connected"); - return; - } - final int myGeneration = generation; - final boolean[] settled = {false}; - mainHandler.postDelayed(() -> { - if (generation != myGeneration || settled[0]) { - return; - } - settled[0] = true; - try { - currentBrowser.unsubscribe(parentId); - } catch (Exception ignored) { - } - emitBrowseResult(requestId, "", "Browse timed out"); - }, BROWSE_TIMEOUT_MS); - try { - currentBrowser.subscribe(parentId, new MediaBrowser.SubscriptionCallback() { - @Override - public void onChildrenLoaded(String loadedParentId, List children) { - if (generation != myGeneration || settled[0]) { - return; - } - settled[0] = true; - // One snapshot is all we want; unsubscribe outside the callback. - mainHandler.post(() -> { - try { - currentBrowser.unsubscribe(loadedParentId); - } catch (Exception ignored) { - } - }); - emitBrowseResult(requestId, serializeChildren(children), ""); - } - - @Override - public void onError(String erroredParentId) { - if (generation != myGeneration || settled[0]) { - return; - } - settled[0] = true; - mainHandler.post(() -> { - try { - currentBrowser.unsubscribe(erroredParentId); - } catch (Exception ignored) { - } - }); - emitBrowseResult(requestId, "", "Browse failed"); - } - }); - } catch (Exception e) { - if (!settled[0]) { - settled[0] = true; - emitBrowseResult(requestId, "", "Browse failed: " + e.getMessage()); - } - } - }); - } - - /** Start playback of a browsed item through the connected service's session. */ - public void playFromMediaId(String mediaId) { - mainHandler.post(() -> { - if (browser == null || !browser.isConnected()) { - return; - } - try { - MediaController controller = new MediaController(appContext, browser.getSessionToken()); - controller.getTransportControls().playFromMediaId(mediaId, null); - lastPlayCommandAtMs = SystemClock.elapsedRealtime(); - } catch (Exception e) { - Log.w(TAG, "playFromMediaId failed", e); - } - }); - } - - public void disconnect() { - mainHandler.post(() -> { - long sincePlayMs = SystemClock.elapsedRealtime() - lastPlayCommandAtMs; - long graceMs = lastPlayCommandAtMs == 0 ? 0 : PLAY_DISCONNECT_GRACE_MS - sincePlayMs; - if (graceMs <= 0) { - closeBrowser(); - return; - } - final MediaBrowser lingering = browser; - browser = null; - generation++; - if (lingering != null) { - mainHandler.postDelayed(() -> { - try { - lingering.disconnect(); - } catch (Exception ignored) { - } - }, graceMs); - } - }); - } - - private void closeBrowser() { - generation++; - if (browser != null) { - try { - browser.disconnect(); - } catch (Exception ignored) { - } - browser = null; - } - } - - private String serializeChildren(List children) { - JSONArray out = new JSONArray(); - if (children == null) { - return out.toString(); - } - for (MediaBrowser.MediaItem item : children) { - try { - CharSequence title = item.getDescription() == null ? null : item.getDescription().getTitle(); - CharSequence subtitle = item.getDescription() == null ? null : item.getDescription().getSubtitle(); - JSONObject entry = new JSONObject(); - entry.put("mediaId", item.getMediaId() == null ? "" : item.getMediaId()); - entry.put("title", title == null ? "" : title.toString()); - entry.put("subtitle", subtitle == null ? "" : subtitle.toString()); - entry.put("browsable", item.isBrowsable()); - entry.put("playable", item.isPlayable()); - out.put(entry); - } catch (Exception e) { - Log.w(TAG, "media item serialization failed", e); - } - } - return out.toString(); - } - - private void emitConnectResult(int requestId, boolean connected, String rootId, String error) { - FaceclawMediaBrowserListener currentListener = listener; - if (currentListener != null) { - currentListener.onConnectResult(requestId, connected, rootId, error); - } - } - - private void emitBrowseResult(int requestId, String childrenJson, String error) { - FaceclawMediaBrowserListener currentListener = listener; - if (currentListener != null) { - currentListener.onBrowseResult(requestId, childrenJson, error); - } - } -} diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawMediaBrowser.kt b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawMediaBrowser.kt new file mode 100644 index 00000000..fb716c84 --- /dev/null +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawMediaBrowser.kt @@ -0,0 +1,316 @@ +package com.faceclaw.app + +import android.content.ComponentName +import android.content.Context +import android.content.Intent +import android.content.pm.PackageManager +import android.content.pm.ResolveInfo +import android.media.browse.MediaBrowser +import android.media.session.MediaController +import android.os.Handler +import android.os.Looper +import android.os.SystemClock +import android.util.Log + +import org.json.JSONArray +import org.json.JSONObject + +import java.util.ArrayList +import java.util.Collections + +/** + * Client for other apps' MediaBrowserService implementations (the mechanism + * Android Auto uses). Binding starts the target player's process even when it + * is not running, so this can list a player's library (playlists, albums, ...) + * and start playback without the phone being unlocked. + * + * Holds at most one connection at a time. All MediaBrowser interaction happens + * on the main thread; results are reported through + * FaceclawMediaBrowserListener, also on the main thread. + */ +class FaceclawMediaBrowser(context: Context) { + companion object { + private const val TAG = "FaceclawMediaBrowser" + private const val BROWSER_SERVICE_ACTION = "android.media.browse.MediaBrowserService" + // Generous: connecting may cold-start the player's process. + private const val CONNECT_TIMEOUT_MS = 15_000L + private const val BROWSE_TIMEOUT_MS = 15_000L + // Unbinding immediately after a play command can race the player promoting + // itself to a foreground service, letting the system kill its process + // before playback starts; keep the binding alive briefly instead. + private const val PLAY_DISCONNECT_GRACE_MS = 5_000L + } + + private val appContext: Context = context.applicationContext + private val mainHandler = Handler(Looper.getMainLooper()) + @Volatile + private var listener: FaceclawMediaBrowserListener? = null + + // Main-thread only. + private var browser: MediaBrowser? = null + private var generation = 0 + private var lastPlayCommandAtMs = 0L + + fun setListener(listener: FaceclawMediaBrowserListener?) { + this.listener = listener + } + + /** + * Installed apps exposing a media browser service, as JSON sorted by app + * name: [{"packageName": ..., "serviceClass": ..., "appName": ...}, ...]. + * Synchronous (PackageManager lookup only). + */ + fun listBrowsableAppsJson(): String { + val packageManager = appContext.packageManager + val services: List + try { + services = packageManager.queryIntentServices(Intent(BROWSER_SERVICE_ACTION), 0) + } catch (e: Exception) { + Log.w(TAG, "media browser service query failed", e) + return "[]" + } + val apps: MutableList = ArrayList() + for (service in services) { + if (service.serviceInfo == null) { + continue + } + try { + val label: CharSequence? = service.serviceInfo.applicationInfo.loadLabel(packageManager) + val app = JSONObject() + app.put("packageName", service.serviceInfo.packageName) + app.put("serviceClass", service.serviceInfo.name) + app.put("appName", if (label == null) service.serviceInfo.packageName else label.toString()) + apps.add(app) + } catch (e: Exception) { + Log.w(TAG, "media browser service entry failed", e) + } + } + Collections.sort(apps) { a, b -> a.optString("appName").compareTo(b.optString("appName"), ignoreCase = true) } + return JSONArray(apps).toString() + } + + /** + * Connect to an app's media browser service, starting its process if + * needed; replaces any existing connection. Async; reports through + * onConnectResult with the browse-tree root id on success. + */ + fun connect(requestId: Int, packageName: String?, serviceClass: String?) { + mainHandler.post { + closeBrowser() + val myGeneration = generation + val settled = booleanArrayOf(false) + val browserRef = arrayOfNulls(1) + val newBrowser = MediaBrowser( + appContext, + ComponentName(packageName!!, serviceClass!!), + object : MediaBrowser.ConnectionCallback() { + override fun onConnected() { + if (generation != myGeneration || settled[0]) { + return + } + settled[0] = true + var rootId = "" + try { + rootId = browserRef[0]!!.root + } catch (e: Exception) { + Log.w(TAG, "getRoot failed", e) + } + emitConnectResult(requestId, true, rootId, "") + } + + override fun onConnectionFailed() { + if (generation != myGeneration || settled[0]) { + return + } + settled[0] = true + closeBrowser() + emitConnectResult(requestId, false, "", "Connection refused by $packageName") + } + + override fun onConnectionSuspended() { + if (generation != myGeneration) { + return + } + val currentListener = listener + if (currentListener != null) { + currentListener.onDisconnected() + } + } + }, + null + ) + browserRef[0] = newBrowser + browser = newBrowser + mainHandler.postDelayed({ + if (generation != myGeneration || settled[0]) { + return@postDelayed + } + settled[0] = true + closeBrowser() + emitConnectResult(requestId, false, "", "Connection to $packageName timed out") + }, CONNECT_TIMEOUT_MS) + try { + newBrowser.connect() + } catch (e: Exception) { + if (!settled[0]) { + settled[0] = true + closeBrowser() + emitConnectResult(requestId, false, "", "Connection failed: " + e.message) + } + } + } + } + + /** + * Fetch one level of the connected service's content tree. Async; reports + * through onBrowseResult with children as JSON: + * [{"mediaId": ..., "title": ..., "subtitle": ..., "browsable": bool, "playable": bool}, ...]. + */ + fun browse(requestId: Int, parentId: String?) { + mainHandler.post { + val currentBrowser = browser + if (currentBrowser == null || !currentBrowser.isConnected) { + emitBrowseResult(requestId, "", "Not connected") + return@post + } + val myGeneration = generation + val settled = booleanArrayOf(false) + mainHandler.postDelayed({ + if (generation != myGeneration || settled[0]) { + return@postDelayed + } + settled[0] = true + try { + currentBrowser.unsubscribe(parentId!!) + } catch (ignored: Exception) { + } + emitBrowseResult(requestId, "", "Browse timed out") + }, BROWSE_TIMEOUT_MS) + try { + currentBrowser.subscribe(parentId!!, object : MediaBrowser.SubscriptionCallback() { + override fun onChildrenLoaded(loadedParentId: String, children: List) { + if (generation != myGeneration || settled[0]) { + return + } + settled[0] = true + // One snapshot is all we want; unsubscribe outside the callback. + mainHandler.post { + try { + currentBrowser.unsubscribe(loadedParentId) + } catch (ignored: Exception) { + } + } + emitBrowseResult(requestId, serializeChildren(children), "") + } + + override fun onError(erroredParentId: String) { + if (generation != myGeneration || settled[0]) { + return + } + settled[0] = true + mainHandler.post { + try { + currentBrowser.unsubscribe(erroredParentId) + } catch (ignored: Exception) { + } + } + emitBrowseResult(requestId, "", "Browse failed") + } + }) + } catch (e: Exception) { + if (!settled[0]) { + settled[0] = true + emitBrowseResult(requestId, "", "Browse failed: " + e.message) + } + } + } + } + + /** Start playback of a browsed item through the connected service's session. */ + fun playFromMediaId(mediaId: String?) { + mainHandler.post { + val currentBrowser = browser + if (currentBrowser == null || !currentBrowser.isConnected) { + return@post + } + try { + val controller = MediaController(appContext, currentBrowser.sessionToken) + controller.transportControls.playFromMediaId(mediaId, null) + lastPlayCommandAtMs = SystemClock.elapsedRealtime() + } catch (e: Exception) { + Log.w(TAG, "playFromMediaId failed", e) + } + } + } + + fun disconnect() { + mainHandler.post { + val sincePlayMs = SystemClock.elapsedRealtime() - lastPlayCommandAtMs + val graceMs = if (lastPlayCommandAtMs == 0L) 0 else PLAY_DISCONNECT_GRACE_MS - sincePlayMs + if (graceMs <= 0) { + closeBrowser() + return@post + } + val lingering = browser + browser = null + generation++ + if (lingering != null) { + mainHandler.postDelayed({ + try { + lingering.disconnect() + } catch (ignored: Exception) { + } + }, graceMs) + } + } + } + + private fun closeBrowser() { + generation++ + val currentBrowser = browser + if (currentBrowser != null) { + try { + currentBrowser.disconnect() + } catch (ignored: Exception) { + } + browser = null + } + } + + private fun serializeChildren(children: List?): String { + val out = JSONArray() + if (children == null) { + return out.toString() + } + for (item in children) { + try { + val title: CharSequence? = if (item.description == null) null else item.description.title + val subtitle: CharSequence? = if (item.description == null) null else item.description.subtitle + val entry = JSONObject() + entry.put("mediaId", if (item.mediaId == null) "" else item.mediaId) + entry.put("title", if (title == null) "" else title.toString()) + entry.put("subtitle", if (subtitle == null) "" else subtitle.toString()) + entry.put("browsable", item.isBrowsable) + entry.put("playable", item.isPlayable) + out.put(entry) + } catch (e: Exception) { + Log.w(TAG, "media item serialization failed", e) + } + } + return out.toString() + } + + private fun emitConnectResult(requestId: Int, connected: Boolean, rootId: String?, error: String?) { + val currentListener = listener + if (currentListener != null) { + currentListener.onConnectResult(requestId, connected, rootId, error) + } + } + + private fun emitBrowseResult(requestId: Int, childrenJson: String?, error: String?) { + val currentListener = listener + if (currentListener != null) { + currentListener.onBrowseResult(requestId, childrenJson, error) + } + } +} diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawMediaController.java b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawMediaController.java deleted file mode 100644 index c8775f40..00000000 --- a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawMediaController.java +++ /dev/null @@ -1,576 +0,0 @@ -package com.faceclaw.app; - -import android.content.ComponentName; -import android.content.Context; -import android.content.Intent; -import android.content.pm.ApplicationInfo; -import android.content.pm.PackageManager; -import android.database.ContentObserver; -import android.graphics.Bitmap; -import android.media.AudioManager; -import android.media.MediaDescription; -import android.media.MediaMetadata; -import android.media.session.MediaController; -import android.media.session.MediaSession; -import android.media.session.MediaSessionManager; -import android.media.session.PlaybackState; -import android.os.Handler; -import android.os.Looper; -import android.os.SystemClock; -import android.provider.Settings; -import android.util.Log; - -import org.json.JSONArray; -import org.json.JSONObject; - -import java.util.List; -import java.util.HashSet; -import java.util.Set; - -public class FaceclawMediaController { - private static final String ENABLED_NOTIFICATION_LISTENERS = "enabled_notification_listeners"; - - private final Context appContext; - private final Object lock = new Object(); - private final Handler mainHandler = new Handler(Looper.getMainLooper()); - private final ComponentName listenerComponent; - private final MediaSessionManager sessionManager; - private final AudioManager audioManager; - - private final MediaController.Callback controllerCallback = new MediaController.Callback() { - @Override - public void onPlaybackStateChanged(PlaybackState state) { - synchronized (lock) { - emitStateLocked(); - } - } - - @Override - public void onMetadataChanged(MediaMetadata metadata) { - synchronized (lock) { - emitStateLocked(); - } - } - - @Override - public void onQueueChanged(List queue) { - synchronized (lock) { - emitStateLocked(); - } - } - - @Override - public void onSessionDestroyed() { - synchronized (lock) { - refreshActiveControllerLocked(null); - } - } - }; - - private final MediaSessionManager.OnActiveSessionsChangedListener sessionsChangedListener = controllers -> { - synchronized (lock) { - refreshActiveControllerLocked(controllers); - } - }; - - /** - * Fires when the user grants or revokes notification-listener access in - * system settings. Session access depends on that grant, so the sessions - * listener is (re)registered and the state re-emitted whenever it changes; - * otherwise a grant made after start() would never be noticed. - */ - private final ContentObserver notificationAccessObserver = new ContentObserver(mainHandler) { - @Override - public void onChange(boolean selfChange) { - synchronized (lock) { - if (!started) return; - syncSessionsListenerLocked(); - refreshActiveControllerLocked(null); - } - } - }; - - private volatile FaceclawMediaControllerListener listener; - private MediaController activeController; - private boolean started; - private boolean sessionsListenerRegistered; - private Set ignoredPackages = new HashSet<>(); - - public FaceclawMediaController(Context context) { - this.appContext = context.getApplicationContext(); - this.listenerComponent = new ComponentName(appContext, FaceclawMediaNotificationListenerService.class); - this.sessionManager = (MediaSessionManager) appContext.getSystemService(Context.MEDIA_SESSION_SERVICE); - this.audioManager = (AudioManager) appContext.getSystemService(Context.AUDIO_SERVICE); - } - - public void setListener(FaceclawMediaControllerListener listener) { - synchronized (lock) { - this.listener = listener; - emitStateLocked(); - } - } - - public void start() { - synchronized (lock) { - if (started) { - emitStateLocked(); - return; - } - started = true; - try { - appContext.getContentResolver().registerContentObserver( - Settings.Secure.getUriFor(ENABLED_NOTIFICATION_LISTENERS), - false, - notificationAccessObserver - ); - } catch (Exception e) { - Log.w("FaceclawMedia", "notification access observer registration failed", e); - } - syncSessionsListenerLocked(); - refreshActiveControllerLocked(null); - } - } - - /** - * Keep the active-sessions listener registered exactly while notification - * access is granted. Registration throws SecurityException without the - * grant, so this is retried from the settings observer rather than only - * attempted once at start(). - */ - private void syncSessionsListenerLocked() { - if (sessionManager == null) return; - boolean accessEnabled = started && isNotificationAccessEnabled(); - if (accessEnabled && !sessionsListenerRegistered) { - try { - sessionManager.addOnActiveSessionsChangedListener( - sessionsChangedListener, - listenerComponent, - mainHandler - ); - sessionsListenerRegistered = true; - } catch (SecurityException ignored) { - } - } else if (!accessEnabled && sessionsListenerRegistered) { - try { - sessionManager.removeOnActiveSessionsChangedListener(sessionsChangedListener); - } catch (SecurityException ignored) { - } - sessionsListenerRegistered = false; - } - } - - /** Apply the complete preference set and immediately choose an allowed session. */ - public void setIgnoredPackagesJson(String packagesJson) { - Set packages = new HashSet<>(); - try { - JSONArray array = new JSONArray(packagesJson); - for (int i = 0; i < array.length(); i++) packages.add(array.getString(i)); - } catch (Exception e) { - Log.w("FaceclawMedia", "invalid ignored packages", e); - return; - } - synchronized (lock) { - if (ignoredPackages.equals(packages)) return; - ignoredPackages = packages; - refreshActiveControllerLocked(null); - } - } - - public void stop() { - synchronized (lock) { - if (!started) { - return; - } - started = false; - try { - appContext.getContentResolver().unregisterContentObserver(notificationAccessObserver); - } catch (Exception ignored) { - } - syncSessionsListenerLocked(); - setActiveControllerLocked(null); - emitStateLocked(); - } - } - - public void playPause() { - synchronized (lock) { - if (activeController == null) { - return; - } - PlaybackState playbackState = activeController.getPlaybackState(); - MediaController.TransportControls controls = activeController.getTransportControls(); - if (playbackState == null) { - controls.play(); - return; - } - switch (playbackState.getState()) { - case PlaybackState.STATE_PLAYING: - case PlaybackState.STATE_BUFFERING: - case PlaybackState.STATE_CONNECTING: - controls.pause(); - return; - default: - controls.play(); - } - } - } - - public void skipNext() { - synchronized (lock) { - if (activeController != null) { - activeController.getTransportControls().skipToNext(); - } - } - } - - public void skipPrevious() { - synchronized (lock) { - if (activeController != null) { - activeController.getTransportControls().skipToPrevious(); - } - } - } - - public void skipToQueueItem(long queueId) { - synchronized (lock) { - if (activeController != null) { - activeController.getTransportControls().skipToQueueItem(queueId); - } - } - } - - /** Current phone media-stream volume normalized to the range 0..100. */ - public int getMediaVolumePercent() { - if (audioManager == null) { - return -1; - } - int maxVolume = audioManager.getStreamMaxVolume(AudioManager.STREAM_MUSIC); - if (maxVolume <= 0) { - return 0; - } - int currentVolume = audioManager.getStreamVolume(AudioManager.STREAM_MUSIC); - return Math.max(0, Math.min(100, Math.round(currentVolume * 100f / maxVolume))); - } - - /** Set the phone media-stream volume from a normalized 0..100 value. */ - public void setMediaVolumePercent(int volumePercent) { - if (audioManager == null) { - return; - } - int clampedPercent = Math.max(0, Math.min(100, volumePercent)); - int maxVolume = audioManager.getStreamMaxVolume(AudioManager.STREAM_MUSIC); - int streamVolume = Math.round(clampedPercent * maxVolume / 100f); - audioManager.setStreamVolume(AudioManager.STREAM_MUSIC, streamVolume, 0); - } - - /** - * Album art for the active session's current item, grayscale, scaled to - * fit within maxSize x maxSize preserving aspect. Returns a gray packet - * (see ImageFileLoader.bitmapToGrayPacket) or an empty array when no art - * is available. gamma and dither are the photographic tone handling - * described on ImageFileLoader.bitmapToGrayPacket; the TS side passes - * its shared photo preset. - */ - public byte[] getAlbumArtGray(int maxSize, float gamma, boolean dither) { - Bitmap art; - synchronized (lock) { - if (activeController == null) { - return new byte[0]; - } - MediaMetadata metadata = activeController.getMetadata(); - if (metadata == null) { - return new byte[0]; - } - art = metadata.getBitmap(MediaMetadata.METADATA_KEY_ALBUM_ART); - if (art == null) { - art = metadata.getBitmap(MediaMetadata.METADATA_KEY_ART); - } - if (art == null) { - art = metadata.getBitmap(MediaMetadata.METADATA_KEY_DISPLAY_ICON); - } - } - return ImageFileLoader.bitmapToGrayPacket(art, maxSize, maxSize, gamma, dither); - } - - /** - * The active session's queue (playlist) as JSON: - * [{"id": long, "title": string, "active": bool}, ...]. Empty string when - * the player exposes no queue. - */ - public String getQueueJson() { - synchronized (lock) { - if (activeController == null) { - return ""; - } - List queue = activeController.getQueue(); - if (queue == null || queue.isEmpty()) { - return ""; - } - PlaybackState state = activeController.getPlaybackState(); - long activeId = state == null ? -1 : state.getActiveQueueItemId(); - try { - JSONArray out = new JSONArray(); - for (MediaSession.QueueItem item : queue) { - MediaDescription description = item.getDescription(); - CharSequence title = description == null ? null : description.getTitle(); - JSONObject entry = new JSONObject(); - entry.put("id", item.getQueueId()); - entry.put("title", title == null ? "" : title.toString()); - entry.put("active", item.getQueueId() == activeId); - out.put(entry); - } - return out.toString(); - } catch (Exception e) { - Log.w("FaceclawMedia", "queue serialization failed", e); - return ""; - } - } - } - - public void openNotificationAccessSettings() { - Intent intent = new Intent(Settings.ACTION_NOTIFICATION_LISTENER_SETTINGS); - intent.addFlags(Intent.FLAG_ACTIVITY_NEW_TASK); - appContext.startActivity(intent); - } - - private void refreshActiveControllerLocked(List controllers) { - if (!started) { - setActiveControllerLocked(null); - emitStateLocked(); - return; - } - if (!isNotificationAccessEnabled()) { - setActiveControllerLocked(null); - emitStateLocked(); - return; - } - if (controllers == null && sessionManager != null) { - try { - controllers = sessionManager.getActiveSessions(listenerComponent); - } catch (SecurityException ignored) { - controllers = null; - } - } - emitSessionAppsLocked(controllers); - setActiveControllerLocked(chooseController(controllers)); - emitStateLocked(); - } - - private MediaController chooseController(List controllers) { - if (controllers == null || controllers.isEmpty()) { - return null; - } - MediaController first = null; - for (MediaController controller : controllers) { - if (ignoredPackages.contains(controller.getPackageName())) continue; - if (first == null) first = controller; - PlaybackState playbackState = controller.getPlaybackState(); - if (playbackState != null && playbackState.getState() == PlaybackState.STATE_PLAYING) { - return controller; - } - } - return first; - } - - private void emitSessionAppsLocked(List controllers) { - FaceclawMediaControllerListener currentListener = listener; - if (currentListener == null || controllers == null) return; - JSONArray apps = new JSONArray(); - Set seen = new HashSet<>(); - for (MediaController controller : controllers) { - String packageName = safe(controller.getPackageName()); - if (packageName.isEmpty() || !seen.add(packageName)) continue; - try { - JSONObject app = new JSONObject(); - app.put("packageName", packageName); - app.put("appName", getApplicationLabel(packageName)); - apps.put(app); - } catch (Exception e) { - Log.w("FaceclawMedia", "session app serialization failed", e); - } - } - String json = apps.toString(); - mainHandler.post(() -> currentListener.onSessionAppsChanged(json)); - } - - private void setActiveControllerLocked(MediaController controller) { - if (sameController(activeController, controller)) { - return; - } - if (activeController != null) { - activeController.unregisterCallback(controllerCallback); - } - activeController = controller; - if (activeController != null) { - activeController.registerCallback(controllerCallback, mainHandler); - } - } - - private boolean sameController(MediaController a, MediaController b) { - if (a == b) { - return true; - } - if (a == null || b == null) { - return false; - } - return a.getSessionToken().equals(b.getSessionToken()); - } - - private boolean isNotificationAccessEnabled() { - String enabledListeners = Settings.Secure.getString( - appContext.getContentResolver(), - ENABLED_NOTIFICATION_LISTENERS - ); - if (enabledListeners == null || enabledListeners.isEmpty()) { - return false; - } - String fullName = listenerComponent.flattenToString(); - String shortName = listenerComponent.flattenToShortString(); - return enabledListeners.contains(fullName) - || enabledListeners.contains(shortName) - || enabledListeners.contains(appContext.getPackageName()); - } - - private void emitStateLocked() { - FaceclawMediaControllerListener currentListener = listener; - if (currentListener == null) { - return; - } - - String playbackState = started ? "idle" : "stopped"; - String packageName = ""; - String appName = ""; - String title = ""; - String artist = ""; - String album = ""; - long positionMs = -1L; - long durationMs = -1L; - float playbackSpeed = 0f; - boolean canPlayPause = false; - boolean canSkipNext = false; - boolean canSkipPrevious = false; - boolean accessEnabled = isNotificationAccessEnabled(); - String status; - - if (!accessEnabled) { - playbackState = "notification-access-required"; - status = "Notification access required."; - } else if (activeController == null) { - status = "No active media session."; - } else { - packageName = safe(activeController.getPackageName()); - appName = getApplicationLabel(packageName); - PlaybackState state = activeController.getPlaybackState(); - MediaMetadata metadata = activeController.getMetadata(); - playbackState = playbackStateName(state); - if (metadata != null) { - title = safe(metadata.getString(MediaMetadata.METADATA_KEY_TITLE)); - artist = safe(metadata.getString(MediaMetadata.METADATA_KEY_ARTIST)); - album = safe(metadata.getString(MediaMetadata.METADATA_KEY_ALBUM)); - durationMs = metadata.getLong(MediaMetadata.METADATA_KEY_DURATION); - if (durationMs <= 0L) { - durationMs = -1L; - } - } - if (state != null) { - positionMs = estimatedPositionMs(state, durationMs); - playbackSpeed = state.getPlaybackSpeed(); - } - long actions = state == null ? 0L : state.getActions(); - canPlayPause = (actions & PlaybackState.ACTION_PLAY_PAUSE) != 0 - || (actions & PlaybackState.ACTION_PLAY) != 0 - || (actions & PlaybackState.ACTION_PAUSE) != 0; - canSkipNext = (actions & PlaybackState.ACTION_SKIP_TO_NEXT) != 0; - canSkipPrevious = (actions & PlaybackState.ACTION_SKIP_TO_PREVIOUS) != 0; - status = packageName.isEmpty() ? "Active media session." : "Active session: " + packageName; - } - - final String finalPlaybackState = playbackState; - final String finalPackageName = packageName; - final String finalAppName = appName; - final String finalTitle = title; - final String finalArtist = artist; - final String finalAlbum = album; - final long finalPositionMs = positionMs; - final long finalDurationMs = durationMs; - final float finalPlaybackSpeed = playbackSpeed; - final boolean finalCanPlayPause = canPlayPause; - final boolean finalCanSkipNext = canSkipNext; - final boolean finalCanSkipPrevious = canSkipPrevious; - final boolean finalAccessEnabled = accessEnabled; - final String finalStatus = status; - - mainHandler.post(() -> currentListener.onStateChange( - finalPlaybackState, - finalPackageName, - finalAppName, - finalTitle, - finalArtist, - finalAlbum, - finalPositionMs, - finalDurationMs, - finalPlaybackSpeed, - finalCanPlayPause, - finalCanSkipNext, - finalCanSkipPrevious, - finalAccessEnabled, - finalStatus - )); - } - - private String playbackStateName(PlaybackState state) { - if (state == null) { - return "idle"; - } - switch (state.getState()) { - case PlaybackState.STATE_PLAYING: - return "playing"; - case PlaybackState.STATE_PAUSED: - return "paused"; - case PlaybackState.STATE_BUFFERING: - case PlaybackState.STATE_CONNECTING: - return "buffering"; - case PlaybackState.STATE_STOPPED: - return "stopped"; - default: - return "idle"; - } - } - - /** PlaybackState positions are snapshots; advance a playing snapshot to now. */ - private long estimatedPositionMs(PlaybackState state, long durationMs) { - long positionMs = state.getPosition(); - if (positionMs == PlaybackState.PLAYBACK_POSITION_UNKNOWN) { - return -1L; - } - if (state.getState() == PlaybackState.STATE_PLAYING) { - long updatedAtMs = state.getLastPositionUpdateTime(); - if (updatedAtMs > 0L) { - positionMs += (long) ((SystemClock.elapsedRealtime() - updatedAtMs) * state.getPlaybackSpeed()); - } - } - positionMs = Math.max(0L, positionMs); - return durationMs > 0L ? Math.min(positionMs, durationMs) : positionMs; - } - - /** Resolve a media session's package id to the user-facing installed app name. */ - private String getApplicationLabel(String packageName) { - if (packageName == null || packageName.isEmpty()) { - return ""; - } - PackageManager packageManager = appContext.getPackageManager(); - try { - ApplicationInfo applicationInfo = packageManager.getApplicationInfo(packageName, 0); - CharSequence label = packageManager.getApplicationLabel(applicationInfo); - if (label != null && label.length() > 0) { - return label.toString(); - } - } catch (PackageManager.NameNotFoundException ignored) { - } - return packageName; - } - - private String safe(String value) { - return value == null ? "" : value; - } -} diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawMediaController.kt b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawMediaController.kt new file mode 100644 index 00000000..b7f8ca5a --- /dev/null +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawMediaController.kt @@ -0,0 +1,577 @@ +package com.faceclaw.app + +import android.content.ComponentName +import android.content.Context +import android.content.Intent +import android.content.pm.ApplicationInfo +import android.content.pm.PackageManager +import android.database.ContentObserver +import android.graphics.Bitmap +import android.media.AudioManager +import android.media.MediaDescription +import android.media.MediaMetadata +import android.media.session.MediaController +import android.media.session.MediaSession +import android.media.session.MediaSessionManager +import android.media.session.PlaybackState +import android.os.Handler +import android.os.Looper +import android.os.SystemClock +import android.provider.Settings +import android.util.Log + +import org.json.JSONArray +import org.json.JSONObject + +import java.util.HashSet + +class FaceclawMediaController(context: Context) { + companion object { + private const val ENABLED_NOTIFICATION_LISTENERS = "enabled_notification_listeners" + } + + private val appContext: Context = context.applicationContext + private val lock = Any() + private val mainHandler = Handler(Looper.getMainLooper()) + private val listenerComponent: ComponentName + private val sessionManager: MediaSessionManager? + private val audioManager: AudioManager? + + private val controllerCallback: MediaController.Callback = object : MediaController.Callback() { + override fun onPlaybackStateChanged(state: PlaybackState?) { + synchronized(lock) { + emitStateLocked() + } + } + + override fun onMetadataChanged(metadata: MediaMetadata?) { + synchronized(lock) { + emitStateLocked() + } + } + + override fun onQueueChanged(queue: MutableList?) { + synchronized(lock) { + emitStateLocked() + } + } + + override fun onSessionDestroyed() { + synchronized(lock) { + refreshActiveControllerLocked(null) + } + } + } + + private val sessionsChangedListener = MediaSessionManager.OnActiveSessionsChangedListener { controllers -> + synchronized(lock) { + refreshActiveControllerLocked(controllers) + } + } + + /** + * Fires when the user grants or revokes notification-listener access in + * system settings. Session access depends on that grant, so the sessions + * listener is (re)registered and the state re-emitted whenever it changes; + * otherwise a grant made after start() would never be noticed. + */ + private val notificationAccessObserver: ContentObserver = object : ContentObserver(mainHandler) { + override fun onChange(selfChange: Boolean) { + synchronized(lock) { + if (!started) return + syncSessionsListenerLocked() + refreshActiveControllerLocked(null) + } + } + } + + @Volatile + private var listener: FaceclawMediaControllerListener? = null + private var activeController: MediaController? = null + private var started = false + private var sessionsListenerRegistered = false + private var ignoredPackages: Set = HashSet() + + init { + this.listenerComponent = ComponentName(appContext, FaceclawMediaNotificationListenerService::class.java) + this.sessionManager = appContext.getSystemService(Context.MEDIA_SESSION_SERVICE) as MediaSessionManager? + this.audioManager = appContext.getSystemService(Context.AUDIO_SERVICE) as AudioManager? + } + + fun setListener(listener: FaceclawMediaControllerListener?) { + synchronized(lock) { + this.listener = listener + emitStateLocked() + } + } + + fun start() { + synchronized(lock) { + if (started) { + emitStateLocked() + return + } + started = true + try { + appContext.contentResolver.registerContentObserver( + Settings.Secure.getUriFor(ENABLED_NOTIFICATION_LISTENERS), + false, + notificationAccessObserver + ) + } catch (e: Exception) { + Log.w("FaceclawMedia", "notification access observer registration failed", e) + } + syncSessionsListenerLocked() + refreshActiveControllerLocked(null) + } + } + + /** + * Keep the active-sessions listener registered exactly while notification + * access is granted. Registration throws SecurityException without the + * grant, so this is retried from the settings observer rather than only + * attempted once at start(). + */ + private fun syncSessionsListenerLocked() { + if (sessionManager == null) return + val accessEnabled = started && isNotificationAccessEnabled() + if (accessEnabled && !sessionsListenerRegistered) { + try { + sessionManager.addOnActiveSessionsChangedListener( + sessionsChangedListener, + listenerComponent, + mainHandler + ) + sessionsListenerRegistered = true + } catch (ignored: SecurityException) { + } + } else if (!accessEnabled && sessionsListenerRegistered) { + try { + sessionManager.removeOnActiveSessionsChangedListener(sessionsChangedListener) + } catch (ignored: SecurityException) { + } + sessionsListenerRegistered = false + } + } + + /** Apply the complete preference set and immediately choose an allowed session. */ + fun setIgnoredPackagesJson(packagesJson: String?) { + val packages: MutableSet = HashSet() + try { + val array = JSONArray(packagesJson) + for (i in 0 until array.length()) packages.add(array.getString(i)) + } catch (e: Exception) { + Log.w("FaceclawMedia", "invalid ignored packages", e) + return + } + synchronized(lock) { + if (ignoredPackages == packages) return + ignoredPackages = packages + refreshActiveControllerLocked(null) + } + } + + fun stop() { + synchronized(lock) { + if (!started) { + return + } + started = false + try { + appContext.contentResolver.unregisterContentObserver(notificationAccessObserver) + } catch (ignored: Exception) { + } + syncSessionsListenerLocked() + setActiveControllerLocked(null) + emitStateLocked() + } + } + + fun playPause() { + synchronized(lock) { + val controller = activeController + if (controller == null) { + return + } + val playbackState: PlaybackState? = controller.playbackState + val controls: MediaController.TransportControls = controller.transportControls + if (playbackState == null) { + controls.play() + return + } + when (playbackState.state) { + PlaybackState.STATE_PLAYING, + PlaybackState.STATE_BUFFERING, + PlaybackState.STATE_CONNECTING -> { + controls.pause() + return + } + else -> controls.play() + } + } + } + + fun skipNext() { + synchronized(lock) { + val controller = activeController + if (controller != null) { + controller.transportControls.skipToNext() + } + } + } + + fun skipPrevious() { + synchronized(lock) { + val controller = activeController + if (controller != null) { + controller.transportControls.skipToPrevious() + } + } + } + + fun skipToQueueItem(queueId: Long) { + synchronized(lock) { + val controller = activeController + if (controller != null) { + controller.transportControls.skipToQueueItem(queueId) + } + } + } + + /** Current phone media-stream volume normalized to the range 0..100. */ + fun getMediaVolumePercent(): Int { + if (audioManager == null) { + return -1 + } + val maxVolume = audioManager.getStreamMaxVolume(AudioManager.STREAM_MUSIC) + if (maxVolume <= 0) { + return 0 + } + val currentVolume = audioManager.getStreamVolume(AudioManager.STREAM_MUSIC) + return Math.max(0, Math.min(100, Math.round(currentVolume * 100f / maxVolume))) + } + + /** Set the phone media-stream volume from a normalized 0..100 value. */ + fun setMediaVolumePercent(volumePercent: Int) { + if (audioManager == null) { + return + } + val clampedPercent = Math.max(0, Math.min(100, volumePercent)) + val maxVolume = audioManager.getStreamMaxVolume(AudioManager.STREAM_MUSIC) + val streamVolume = Math.round(clampedPercent * maxVolume / 100f) + audioManager.setStreamVolume(AudioManager.STREAM_MUSIC, streamVolume, 0) + } + + /** + * Album art for the active session's current item, grayscale, scaled to + * fit within maxSize x maxSize preserving aspect. Returns a gray packet + * (see ImageFileLoader.bitmapToGrayPacket) or an empty array when no art + * is available. gamma and dither are the photographic tone handling + * described on ImageFileLoader.bitmapToGrayPacket; the TS side passes + * its shared photo preset. + */ + fun getAlbumArtGray(maxSize: Int, gamma: Float, dither: Boolean): ByteArray { + var art: Bitmap? + synchronized(lock) { + val controller = activeController + if (controller == null) { + return ByteArray(0) + } + val metadata: MediaMetadata? = controller.metadata + if (metadata == null) { + return ByteArray(0) + } + art = metadata.getBitmap(MediaMetadata.METADATA_KEY_ALBUM_ART) + if (art == null) { + art = metadata.getBitmap(MediaMetadata.METADATA_KEY_ART) + } + if (art == null) { + art = metadata.getBitmap(MediaMetadata.METADATA_KEY_DISPLAY_ICON) + } + } + return ImageFileLoader.bitmapToGrayPacket(art, maxSize, maxSize, gamma, dither) + } + + /** + * The active session's queue (playlist) as JSON: + * [{"id": long, "title": string, "active": bool}, ...]. Empty string when + * the player exposes no queue. + */ + fun getQueueJson(): String { + synchronized(lock) { + val controller = activeController + if (controller == null) { + return "" + } + val queue: List? = controller.queue + if (queue == null || queue.isEmpty()) { + return "" + } + val state: PlaybackState? = controller.playbackState + val activeId = if (state == null) -1L else state.activeQueueItemId + try { + val out = JSONArray() + for (item in queue) { + val description: MediaDescription? = item.description + val title: CharSequence? = if (description == null) null else description.title + val entry = JSONObject() + entry.put("id", item.queueId) + entry.put("title", if (title == null) "" else title.toString()) + entry.put("active", item.queueId == activeId) + out.put(entry) + } + return out.toString() + } catch (e: Exception) { + Log.w("FaceclawMedia", "queue serialization failed", e) + return "" + } + } + } + + fun openNotificationAccessSettings() { + val intent = Intent(Settings.ACTION_NOTIFICATION_LISTENER_SETTINGS) + intent.addFlags(Intent.FLAG_ACTIVITY_NEW_TASK) + appContext.startActivity(intent) + } + + private fun refreshActiveControllerLocked(controllersIn: List?) { + var controllers = controllersIn + if (!started) { + setActiveControllerLocked(null) + emitStateLocked() + return + } + if (!isNotificationAccessEnabled()) { + setActiveControllerLocked(null) + emitStateLocked() + return + } + if (controllers == null && sessionManager != null) { + try { + controllers = sessionManager.getActiveSessions(listenerComponent) + } catch (ignored: SecurityException) { + controllers = null + } + } + emitSessionAppsLocked(controllers) + setActiveControllerLocked(chooseController(controllers)) + emitStateLocked() + } + + private fun chooseController(controllers: List?): MediaController? { + if (controllers == null || controllers.isEmpty()) { + return null + } + var first: MediaController? = null + for (controller in controllers) { + if (ignoredPackages.contains(controller.packageName)) continue + if (first == null) first = controller + val playbackState: PlaybackState? = controller.playbackState + if (playbackState != null && playbackState.state == PlaybackState.STATE_PLAYING) { + return controller + } + } + return first + } + + private fun emitSessionAppsLocked(controllers: List?) { + val currentListener = listener + if (currentListener == null || controllers == null) return + val apps = JSONArray() + val seen: MutableSet = HashSet() + for (controller in controllers) { + val packageName = safe(controller.packageName) + if (packageName.isEmpty() || !seen.add(packageName)) continue + try { + val app = JSONObject() + app.put("packageName", packageName) + app.put("appName", getApplicationLabel(packageName)) + apps.put(app) + } catch (e: Exception) { + Log.w("FaceclawMedia", "session app serialization failed", e) + } + } + val json = apps.toString() + mainHandler.post { currentListener.onSessionAppsChanged(json) } + } + + private fun setActiveControllerLocked(controller: MediaController?) { + if (sameController(activeController, controller)) { + return + } + val previous = activeController + if (previous != null) { + previous.unregisterCallback(controllerCallback) + } + activeController = controller + if (controller != null) { + controller.registerCallback(controllerCallback, mainHandler) + } + } + + private fun sameController(a: MediaController?, b: MediaController?): Boolean { + if (a === b) { + return true + } + if (a == null || b == null) { + return false + } + return a.sessionToken == b.sessionToken + } + + private fun isNotificationAccessEnabled(): Boolean { + val enabledListeners: String? = Settings.Secure.getString( + appContext.contentResolver, + ENABLED_NOTIFICATION_LISTENERS + ) + if (enabledListeners == null || enabledListeners.isEmpty()) { + return false + } + val fullName = listenerComponent.flattenToString() + val shortName = listenerComponent.flattenToShortString() + return enabledListeners.contains(fullName) + || enabledListeners.contains(shortName) + || enabledListeners.contains(appContext.packageName) + } + + private fun emitStateLocked() { + val currentListener = listener + if (currentListener == null) { + return + } + + var playbackState = if (started) "idle" else "stopped" + var packageName = "" + var appName = "" + var title = "" + var artist = "" + var album = "" + var positionMs = -1L + var durationMs = -1L + var playbackSpeed = 0f + var canPlayPause = false + var canSkipNext = false + var canSkipPrevious = false + val accessEnabled = isNotificationAccessEnabled() + val status: String + + val controller = activeController + if (!accessEnabled) { + playbackState = "notification-access-required" + status = "Notification access required." + } else if (controller == null) { + status = "No active media session." + } else { + packageName = safe(controller.packageName) + appName = getApplicationLabel(packageName) + val state: PlaybackState? = controller.playbackState + val metadata: MediaMetadata? = controller.metadata + playbackState = playbackStateName(state) + if (metadata != null) { + title = safe(metadata.getString(MediaMetadata.METADATA_KEY_TITLE)) + artist = safe(metadata.getString(MediaMetadata.METADATA_KEY_ARTIST)) + album = safe(metadata.getString(MediaMetadata.METADATA_KEY_ALBUM)) + durationMs = metadata.getLong(MediaMetadata.METADATA_KEY_DURATION) + if (durationMs <= 0L) { + durationMs = -1L + } + } + if (state != null) { + positionMs = estimatedPositionMs(state, durationMs) + playbackSpeed = state.playbackSpeed + } + val actions = if (state == null) 0L else state.actions + canPlayPause = (actions and PlaybackState.ACTION_PLAY_PAUSE) != 0L + || (actions and PlaybackState.ACTION_PLAY) != 0L + || (actions and PlaybackState.ACTION_PAUSE) != 0L + canSkipNext = (actions and PlaybackState.ACTION_SKIP_TO_NEXT) != 0L + canSkipPrevious = (actions and PlaybackState.ACTION_SKIP_TO_PREVIOUS) != 0L + status = if (packageName.isEmpty()) "Active media session." else "Active session: $packageName" + } + + val finalPlaybackState = playbackState + val finalPackageName = packageName + val finalAppName = appName + val finalTitle = title + val finalArtist = artist + val finalAlbum = album + val finalPositionMs = positionMs + val finalDurationMs = durationMs + val finalPlaybackSpeed = playbackSpeed + val finalCanPlayPause = canPlayPause + val finalCanSkipNext = canSkipNext + val finalCanSkipPrevious = canSkipPrevious + val finalAccessEnabled = accessEnabled + val finalStatus = status + + mainHandler.post { + currentListener.onStateChange( + finalPlaybackState, + finalPackageName, + finalAppName, + finalTitle, + finalArtist, + finalAlbum, + finalPositionMs, + finalDurationMs, + finalPlaybackSpeed, + finalCanPlayPause, + finalCanSkipNext, + finalCanSkipPrevious, + finalAccessEnabled, + finalStatus + ) + } + } + + private fun playbackStateName(state: PlaybackState?): String { + if (state == null) { + return "idle" + } + return when (state.state) { + PlaybackState.STATE_PLAYING -> "playing" + PlaybackState.STATE_PAUSED -> "paused" + PlaybackState.STATE_BUFFERING, + PlaybackState.STATE_CONNECTING -> "buffering" + PlaybackState.STATE_STOPPED -> "stopped" + else -> "idle" + } + } + + /** PlaybackState positions are snapshots; advance a playing snapshot to now. */ + private fun estimatedPositionMs(state: PlaybackState, durationMs: Long): Long { + var positionMs = state.position + if (positionMs == PlaybackState.PLAYBACK_POSITION_UNKNOWN) { + return -1L + } + if (state.state == PlaybackState.STATE_PLAYING) { + val updatedAtMs = state.lastPositionUpdateTime + if (updatedAtMs > 0L) { + positionMs += ((SystemClock.elapsedRealtime() - updatedAtMs) * state.playbackSpeed).toLong() + } + } + positionMs = Math.max(0L, positionMs) + return if (durationMs > 0L) Math.min(positionMs, durationMs) else positionMs + } + + /** Resolve a media session's package id to the user-facing installed app name. */ + private fun getApplicationLabel(packageName: String?): String { + if (packageName == null || packageName.isEmpty()) { + return "" + } + val packageManager = appContext.packageManager + try { + val applicationInfo: ApplicationInfo = packageManager.getApplicationInfo(packageName, 0) + val label: CharSequence? = packageManager.getApplicationLabel(applicationInfo) + if (label != null && label.length > 0) { + return label.toString() + } + } catch (ignored: PackageManager.NameNotFoundException) { + } + return packageName + } + + private fun safe(value: String?): String { + return value ?: "" + } +} diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawMediaNotificationListenerService.java b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawMediaNotificationListenerService.java deleted file mode 100644 index 8ca97f00..00000000 --- a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawMediaNotificationListenerService.java +++ /dev/null @@ -1,657 +0,0 @@ -package com.faceclaw.app; - -import android.app.Notification; -import android.app.NotificationManager; -import android.app.PendingIntent; -import android.graphics.Bitmap; -import android.graphics.Canvas; -import android.graphics.Color; -import android.graphics.drawable.BitmapDrawable; -import android.graphics.drawable.Drawable; -import android.graphics.drawable.Icon; -import android.os.Bundle; -import android.os.Handler; -import android.os.Looper; -import android.service.notification.NotificationListenerService; -import android.service.notification.StatusBarNotification; -import android.util.Log; - -import java.io.ByteArrayOutputStream; -import java.util.Arrays; -import java.util.Comparator; -import java.util.HashSet; -import java.util.Set; -import java.util.concurrent.CopyOnWriteArraySet; - -import org.json.JSONArray; -import org.json.JSONException; -import org.json.JSONObject; - -public class FaceclawMediaNotificationListenerService extends NotificationListenerService { - private static final String TAG = "FaceclawNotify"; - private static final double NOTIFICATION_ICON_GAMMA = 1.6; - private static final String EXTRA_SUBSTITUTE_APP_NAME = "android.substName"; - - private static volatile FaceclawMediaNotificationListenerService activeService; - private static final Handler mainHandler = new Handler(Looper.getMainLooper()); - private static final Set notificationListeners = new CopyOnWriteArraySet<>(); - private static final Set activeNotificationWakeKeys = new HashSet<>(); - - @Override - public void onCreate() { - super.onCreate(); - activeService = this; - } - - @Override - public void onDestroy() { - if (activeService == this) { - activeService = null; - } - super.onDestroy(); - } - - @Override - public void onListenerConnected() { - activeService = this; - super.onListenerConnected(); - refreshActiveNotificationWakeKeys(this); - } - - @Override - public void onListenerDisconnected() { - if (activeService == this) { - activeService = null; - } - super.onListenerDisconnected(); - } - - @Override - public void onNotificationPosted(StatusBarNotification statusBarNotification) { - super.onNotificationPosted(statusBarNotification); - if (!shouldShowNotificationInList(this, statusBarNotification)) { - forgetActiveNotificationWakeKey(statusBarNotification); - return; - } - if (shouldEmitNotificationPosted(statusBarNotification)) { - emitNotificationPosted(statusBarNotification.getKey()); - } - } - - @Override - public void onNotificationRemoved(StatusBarNotification statusBarNotification) { - forgetActiveNotificationWakeKey(statusBarNotification); - super.onNotificationRemoved(statusBarNotification); - } - - public static void addNotificationListener(FaceclawNotificationListener listener) { - if (listener != null) { - notificationListeners.add(listener); - } - } - - public static void removeNotificationListener(FaceclawNotificationListener listener) { - if (listener != null) { - notificationListeners.remove(listener); - } - } - - public static boolean hasActiveNotificationTitle(String expectedTitle) { - FaceclawMediaNotificationListenerService service = activeService; - if (service == null || expectedTitle == null || expectedTitle.isEmpty()) { - return false; - } - StatusBarNotification[] notifications; - try { - notifications = service.getActiveNotifications(); - } catch (SecurityException e) { - Log.w(TAG, "notification access denied while checking active notifications", e); - return false; - } catch (Throwable t) { - Log.w(TAG, "failed to check active notifications", t); - return false; - } - if (notifications == null || notifications.length == 0) { - return false; - } - for (StatusBarNotification notification : notifications) { - if (notification == null || notification.getNotification() == null) { - continue; - } - Bundle extras = notification.getNotification().extras; - if (extras == null) { - continue; - } - CharSequence title = extras.getCharSequence(Notification.EXTRA_TITLE); - if (title != null && expectedTitle.contentEquals(title)) { - return true; - } - } - return false; - } - - public static byte[] getActiveNotificationIconGrays(int iconSize, int maxIcons) { - FaceclawMediaNotificationListenerService service = activeService; - int size = Math.max(1, Math.min(96, iconSize)); - int limit = Math.max(0, maxIcons); - if (service == null || limit == 0) { - return new byte[0]; - } - - StatusBarNotification[] notifications; - try { - notifications = service.getActiveNotifications(); - } catch (SecurityException e) { - Log.w(TAG, "notification access denied while reading icons", e); - return new byte[0]; - } catch (Throwable t) { - Log.w(TAG, "failed to read notification icons", t); - return new byte[0]; - } - if (notifications == null || notifications.length == 0) { - return new byte[0]; - } - - ByteArrayOutputStream out = new ByteArrayOutputStream(size * size * Math.min(limit, notifications.length)); - Set emittedGroupKeys = new HashSet<>(); - int emitted = 0; - for (StatusBarNotification statusBarNotification : notifications) { - if (!shouldShowNotificationIcon(service, statusBarNotification)) { - continue; - } - String dedupeGroupKey = getNotificationDedupeGroupKey(statusBarNotification); - if (dedupeGroupKey != null && emittedGroupKeys.contains(dedupeGroupKey)) { - continue; - } - Drawable drawable = loadNotificationIcon(service, statusBarNotification.getNotification()); - if (drawable == null) { - Log.i(TAG, "icon skipped (no drawable): " + statusBarNotification.getPackageName()); - continue; - } - Log.i(TAG, "icon[" + emitted + "] pkg=" + statusBarNotification.getPackageName() - + " drawable=" + drawable.getClass().getSimpleName() - + " intrinsic=" + drawable.getIntrinsicWidth() + "x" + drawable.getIntrinsicHeight()); - appendIconGrayBytes(drawable, size, out, service, emitted, statusBarNotification.getPackageName()); - if (dedupeGroupKey != null) { - emittedGroupKeys.add(dedupeGroupKey); - } - emitted += 1; - if (emitted >= limit) { - break; - } - } - return out.toByteArray(); - } - - /** - * Grayscale icon (iconSize*iconSize bytes) for one active notification, - * identified by its key; empty array when the notification or its icon is - * unavailable. Shares the extraction/scaling pipeline with the tray icon - * strip above. - */ - public static byte[] getNotificationIconGrayForKey(String key, int iconSize) { - FaceclawMediaNotificationListenerService service = activeService; - int size = Math.max(1, Math.min(96, iconSize)); - if (service == null) { - return new byte[0]; - } - StatusBarNotification statusBarNotification = findActiveNotificationByKey(service, key); - if (statusBarNotification == null || statusBarNotification.getNotification() == null) { - return new byte[0]; - } - Drawable drawable = loadNotificationIcon(service, statusBarNotification.getNotification()); - if (drawable == null) { - return new byte[0]; - } - ByteArrayOutputStream out = new ByteArrayOutputStream(size * size); - appendIconGrayBytes(drawable, size, out, service, -1, statusBarNotification.getPackageName()); - return out.toByteArray(); - } - - public static String getActiveNotificationsJson(int maxNotifications) { - FaceclawMediaNotificationListenerService service = activeService; - int limit = Math.max(0, maxNotifications); - if (service == null || limit == 0) { - return "[]"; - } - - StatusBarNotification[] notifications; - try { - notifications = service.getActiveNotifications(); - } catch (SecurityException e) { - Log.w(TAG, "notification access denied while reading notifications", e); - return "[]"; - } catch (Throwable t) { - Log.w(TAG, "failed to read notifications", t); - return "[]"; - } - if (notifications == null || notifications.length == 0) { - return "[]"; - } - - Arrays.sort(notifications, new Comparator() { - @Override - public int compare(StatusBarNotification a, StatusBarNotification b) { - long left = a == null ? 0 : a.getPostTime(); - long right = b == null ? 0 : b.getPostTime(); - return Long.compare(right, left); - } - }); - - JSONArray out = new JSONArray(); - for (StatusBarNotification statusBarNotification : notifications) { - if (out.length() >= limit) { - break; - } - if (!shouldShowNotificationInList(service, statusBarNotification)) { - continue; - } - try { - out.put(buildNotificationJson(service, statusBarNotification)); - } catch (Throwable t) { - Log.w(TAG, "failed to serialize notification", t); - } - } - return out.toString(); - } - - public static boolean invokeNotificationAction(String key, int actionIndex) { - FaceclawMediaNotificationListenerService service = activeService; - StatusBarNotification statusBarNotification = findActiveNotificationByKey(service, key); - if (statusBarNotification == null || statusBarNotification.getNotification() == null) { - return false; - } - Notification.Action[] actions = statusBarNotification.getNotification().actions; - if (actions == null || actionIndex < 0 || actionIndex >= actions.length) { - return false; - } - PendingIntent intent = actions[actionIndex].actionIntent; - if (intent == null) { - return false; - } - try { - intent.send(); - return true; - } catch (PendingIntent.CanceledException e) { - Log.w(TAG, "notification action pending intent was canceled", e); - return false; - } catch (Throwable t) { - Log.w(TAG, "failed to invoke notification action", t); - return false; - } - } - - public static boolean dismissNotification(String key) { - FaceclawMediaNotificationListenerService service = activeService; - if (service == null || key == null || key.isEmpty()) { - return false; - } - try { - service.cancelNotification(key); - return true; - } catch (SecurityException e) { - Log.w(TAG, "notification access denied while dismissing notification", e); - return false; - } catch (Throwable t) { - Log.w(TAG, "failed to dismiss notification", t); - return false; - } - } - - private static void emitNotificationPosted(String key) { - if (key == null || key.isEmpty() || notificationListeners.isEmpty()) { - return; - } - for (FaceclawNotificationListener listener : notificationListeners) { - mainHandler.post(() -> { - try { - listener.onNotificationPosted(key); - } catch (Throwable t) { - Log.w(TAG, "notification listener failed", t); - } - }); - } - } - - private static void refreshActiveNotificationWakeKeys(FaceclawMediaNotificationListenerService service) { - StatusBarNotification[] notifications; - try { - notifications = service.getActiveNotifications(); - } catch (Throwable t) { - Log.w(TAG, "failed to refresh active notification wake keys", t); - return; - } - synchronized (activeNotificationWakeKeys) { - activeNotificationWakeKeys.clear(); - if (notifications == null) { - return; - } - for (StatusBarNotification statusBarNotification : notifications) { - if (shouldShowNotificationInList(service, statusBarNotification)) { - String key = statusBarNotification.getKey(); - if (key != null && !key.isEmpty()) { - activeNotificationWakeKeys.add(key); - } - } - } - } - } - - private static boolean shouldEmitNotificationPosted(StatusBarNotification statusBarNotification) { - String key = statusBarNotification.getKey(); - if (key == null || key.isEmpty()) { - return true; - } - - boolean alreadyActive; - synchronized (activeNotificationWakeKeys) { - alreadyActive = activeNotificationWakeKeys.contains(key); - activeNotificationWakeKeys.add(key); - } - return !alreadyActive || !isPersistentNotification(statusBarNotification); - } - - private static void forgetActiveNotificationWakeKey(StatusBarNotification statusBarNotification) { - if (statusBarNotification == null) { - return; - } - String key = statusBarNotification.getKey(); - if (key == null || key.isEmpty()) { - return; - } - synchronized (activeNotificationWakeKeys) { - activeNotificationWakeKeys.remove(key); - } - } - - private static boolean isPersistentNotification(StatusBarNotification statusBarNotification) { - Notification notification = statusBarNotification.getNotification(); - if (notification == null) { - return false; - } - int persistentFlags = Notification.FLAG_ONGOING_EVENT | Notification.FLAG_NO_CLEAR; - return (notification.flags & persistentFlags) != 0; - } - - private static boolean shouldShowNotificationIcon(FaceclawMediaNotificationListenerService service, StatusBarNotification statusBarNotification) { - if (!shouldShowNotificationInList(service, statusBarNotification)) { - return false; - } - Notification notification = statusBarNotification.getNotification(); - if ((notification.flags & Notification.FLAG_GROUP_SUMMARY) != 0) { - return false; - } - return true; - } - - private static boolean shouldShowNotificationInList(FaceclawMediaNotificationListenerService service, StatusBarNotification statusBarNotification) { - if (statusBarNotification == null || statusBarNotification.getNotification() == null) { - return false; - } - // Our own notifications stay out of the mirror: the foreground-service - // one is noise, and the Timers app rings on the glasses itself (its - // phone notification is for the phone), so mirroring it would stack a - // notification modal over the ringing screen. - if (service.getPackageName().equals(statusBarNotification.getPackageName())) { - return false; - } - Notification notification = statusBarNotification.getNotification(); - if (Notification.CATEGORY_TRANSPORT.equals(notification.category)) { - return false; - } - Bundle extras = notification.extras; - if (extras != null && extras.containsKey("android.mediaSession")) { - return false; - } - - NotificationListenerService.RankingMap rankingMap = service.getCurrentRanking(); - if (rankingMap == null) { - return true; - } - NotificationListenerService.Ranking ranking = new NotificationListenerService.Ranking(); - if (!rankingMap.getRanking(statusBarNotification.getKey(), ranking)) { - return true; - } - int importance = ranking.getImportance(); - return importance > NotificationManager.IMPORTANCE_MIN; - } - - private static String getNotificationDedupeGroupKey(StatusBarNotification statusBarNotification) { - Notification notification = statusBarNotification.getNotification(); - if (notification.getGroup() == null && statusBarNotification.getOverrideGroupKey() == null) { - return null; - } - String groupKey = statusBarNotification.getGroupKey(); - if (groupKey == null || groupKey.isEmpty()) { - return null; - } - // Group children often share the same small icon. Emit only one icon for the group. - return groupKey; - } - - private static Drawable loadNotificationIcon(FaceclawMediaNotificationListenerService service, Notification notification) { - try { - Icon smallIcon = notification.getSmallIcon(); - if (smallIcon != null) { - Drawable drawable = smallIcon.loadDrawable(service); - if (drawable != null) { - // Status-bar small icons are alpha templates: the platform - // draws them tinted and ignores their color channels, which - // apps may fill with garbage (Discord ships noise there). - // Tint white so the shape comes from alpha alone, matching - // how the status bar renders them. - drawable.mutate(); - drawable.setTint(Color.WHITE); - if (drawable instanceof BitmapDrawable) { - ((BitmapDrawable) drawable).setFilterBitmap(true); - } - return drawable; - } - } - } catch (Throwable t) { - Log.w(TAG, "failed to load small notification icon", t); - } - try { - // Large icons are real color images (avatars, album art); keep color. - Icon largeIcon = notification.getLargeIcon(); - if (largeIcon != null) { - return largeIcon.loadDrawable(service); - } - } catch (Throwable t) { - Log.w(TAG, "failed to load large notification icon", t); - } - return null; - } - - private static void appendIconGrayBytes(Drawable drawable, int size, ByteArrayOutputStream out, - FaceclawMediaNotificationListenerService service, int index, String packageName) { - Bitmap bitmap = renderIconScaled(drawable, size); - dumpIconDebugPng(service, index, packageName, bitmap); - for (int y = 0; y < size; y++) { - for (int x = 0; x < size; x++) { - int color = bitmap.getPixel(x, y); - int alpha = Color.alpha(color); - double grayLinear = (0.2126 * Color.red(color) + 0.7152 * Color.green(color) + 0.0722 * Color.blue(color)) * alpha / (255.0 * 255.0); - int gray = (int) Math.round(255.0 * Math.pow(Math.max(0.0, Math.min(1.0, grayLinear)), NOTIFICATION_ICON_GAMMA)); - out.write(gray & 0xff); - } - } - bitmap.recycle(); - } - - /** - * Render a drawable at the target size with proper downscaling. Detailed - * sources (e.g. avatar bitmaps used as notification icons) are rendered at - * native resolution and reduced by repeated halving: a single filtered pass - * from, say, 126px to 24px samples too sparsely and turns fine detail into - * speckle that reads as a garbled icon. - */ - private static Bitmap renderIconScaled(Drawable drawable, int size) { - int renderW = Math.max(size, drawable.getIntrinsicWidth()); - int renderH = Math.max(size, drawable.getIntrinsicHeight()); - Bitmap bitmap = Bitmap.createBitmap(renderW, renderH, Bitmap.Config.ARGB_8888); - Canvas canvas = new Canvas(bitmap); - drawable.setBounds(0, 0, renderW, renderH); - drawable.draw(canvas); - while (bitmap.getWidth() >= size * 2 && bitmap.getHeight() >= size * 2) { - Bitmap halved = Bitmap.createScaledBitmap(bitmap, bitmap.getWidth() / 2, bitmap.getHeight() / 2, true); - bitmap.recycle(); - bitmap = halved; - } - if (bitmap.getWidth() != size || bitmap.getHeight() != size) { - Bitmap scaled = Bitmap.createScaledBitmap(bitmap, size, size, true); - bitmap.recycle(); - bitmap = scaled; - } - return bitmap; - } - - /** - * Debug aid for garbled-icon reports: saves each rendered icon to - * /debug-icons/ (adb-pullable) so extraction problems can - * be told apart from downstream compositing/transmission problems. Cheap: - * runs at most once per icon-cache refresh on tiny bitmaps. - */ - private static void dumpIconDebugPng(FaceclawMediaNotificationListenerService service, int index, String packageName, Bitmap bitmap) { - if (index < 0) { - return; - } - try { - java.io.File dir = new java.io.File(service.getExternalFilesDir(null), "debug-icons"); - if (!dir.exists() && !dir.mkdirs()) { - return; - } - String safeName = packageName == null ? "unknown" : packageName.replaceAll("[^A-Za-z0-9._-]", "_"); - java.io.File file = new java.io.File(dir, "icon-" + index + "-" + safeName + ".png"); - try (java.io.FileOutputStream stream = new java.io.FileOutputStream(file)) { - bitmap.compress(Bitmap.CompressFormat.PNG, 100, stream); - } - } catch (Throwable t) { - Log.w(TAG, "failed to dump debug icon", t); - } - } - - private static StatusBarNotification findActiveNotificationByKey(FaceclawMediaNotificationListenerService service, String key) { - if (service == null || key == null || key.isEmpty()) { - return null; - } - StatusBarNotification[] notifications; - try { - notifications = service.getActiveNotifications(); - } catch (Throwable t) { - Log.w(TAG, "failed to find active notification", t); - return null; - } - if (notifications == null) { - return null; - } - for (StatusBarNotification statusBarNotification : notifications) { - if (statusBarNotification != null && key.equals(statusBarNotification.getKey())) { - return statusBarNotification; - } - } - return null; - } - - private static JSONObject buildNotificationJson(FaceclawMediaNotificationListenerService service, StatusBarNotification statusBarNotification) - throws JSONException { - Notification notification = statusBarNotification.getNotification(); - Bundle extras = notification.extras; - JSONObject out = new JSONObject(); - out.put("key", statusBarNotification.getKey()); - out.put("packageName", statusBarNotification.getPackageName()); - out.put("appName", getNotificationAppName(service, statusBarNotification)); - out.put("postTime", statusBarNotification.getPostTime()); - out.put("when", notification.when); - putString(out, "category", notification.category); - if (extras != null) { - putCharSequence(out, "title", firstNonEmpty( - extras.getCharSequence(Notification.EXTRA_TITLE_BIG), - extras.getCharSequence(Notification.EXTRA_TITLE) - )); - putCharSequence(out, "text", extras.getCharSequence(Notification.EXTRA_TEXT)); - putCharSequence(out, "bigText", extras.getCharSequence(Notification.EXTRA_BIG_TEXT)); - putCharSequence(out, "subText", extras.getCharSequence(Notification.EXTRA_SUB_TEXT)); - putCharSequence(out, "infoText", extras.getCharSequence(Notification.EXTRA_INFO_TEXT)); - putCharSequence(out, "summaryText", extras.getCharSequence(Notification.EXTRA_SUMMARY_TEXT)); - CharSequence[] textLines = extras.getCharSequenceArray(Notification.EXTRA_TEXT_LINES); - JSONArray lines = new JSONArray(); - if (textLines != null) { - for (CharSequence line : textLines) { - String text = charSequenceToString(line); - if (!text.isEmpty()) { - lines.put(text); - } - } - } - out.put("lines", lines); - } else { - out.put("lines", new JSONArray()); - } - - JSONArray actionsJson = new JSONArray(); - Notification.Action[] actions = notification.actions; - if (actions != null) { - for (int index = 0; index < actions.length; index++) { - Notification.Action action = actions[index]; - if (action == null) { - continue; - } - String title = charSequenceToString(action.title); - if (title.isEmpty()) { - continue; - } - JSONObject actionJson = new JSONObject(); - actionJson.put("index", index); - actionJson.put("title", title); - actionJson.put("enabled", action.actionIntent != null); - actionsJson.put(actionJson); - } - } - out.put("actions", actionsJson); - return out; - } - - private static String getNotificationAppName(FaceclawMediaNotificationListenerService service, StatusBarNotification statusBarNotification) { - Notification notification = statusBarNotification.getNotification(); - Bundle extras = notification.extras; - if (extras != null) { - String substituteName = charSequenceToString(extras.getCharSequence(EXTRA_SUBSTITUTE_APP_NAME)); - if (!substituteName.isEmpty()) { - return substituteName; - } - } - return getAppLabel(service, statusBarNotification.getPackageName()); - } - - private static String getAppLabel(FaceclawMediaNotificationListenerService service, String packageName) { - if (packageName == null || packageName.isEmpty()) { - return ""; - } - try { - CharSequence label = service - .getPackageManager() - .getApplicationLabel(service.getPackageManager().getApplicationInfo(packageName, 0)); - String text = charSequenceToString(label); - return text.isEmpty() ? packageName : text; - } catch (Throwable t) { - return packageName; - } - } - - private static void putString(JSONObject out, String key, String value) throws JSONException { - out.put(key, value == null ? "" : value); - } - - private static void putCharSequence(JSONObject out, String key, CharSequence value) throws JSONException { - out.put(key, charSequenceToString(value)); - } - - private static CharSequence firstNonEmpty(CharSequence first, CharSequence second) { - return charSequenceToString(first).isEmpty() ? second : first; - } - - private static String charSequenceToString(CharSequence value) { - return value == null ? "" : value.toString(); - } -} diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawMediaNotificationListenerService.kt b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawMediaNotificationListenerService.kt new file mode 100644 index 00000000..7c40de56 --- /dev/null +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawMediaNotificationListenerService.kt @@ -0,0 +1,662 @@ +package com.faceclaw.app + +import android.app.Notification +import android.app.NotificationManager +import android.app.PendingIntent +import android.graphics.Bitmap +import android.graphics.Canvas +import android.graphics.Color +import android.graphics.drawable.BitmapDrawable +import android.graphics.drawable.Drawable +import android.graphics.drawable.Icon +import android.os.Bundle +import android.os.Handler +import android.os.Looper +import android.service.notification.NotificationListenerService +import android.service.notification.StatusBarNotification +import android.util.Log + +import java.io.ByteArrayOutputStream +import java.util.Arrays +import java.util.Comparator +import java.util.HashSet +import java.util.concurrent.CopyOnWriteArraySet + +import org.json.JSONArray +import org.json.JSONException +import org.json.JSONObject + +class FaceclawMediaNotificationListenerService : NotificationListenerService() { + companion object { + private const val TAG = "FaceclawNotify" + private const val NOTIFICATION_ICON_GAMMA = 1.6 + private const val EXTRA_SUBSTITUTE_APP_NAME = "android.substName" + + @Volatile + private var activeService: FaceclawMediaNotificationListenerService? = null + private val mainHandler = Handler(Looper.getMainLooper()) + private val notificationListeners: MutableSet = CopyOnWriteArraySet() + private val activeNotificationWakeKeys: MutableSet = HashSet() + + @JvmStatic + fun addNotificationListener(listener: FaceclawNotificationListener?) { + if (listener != null) { + notificationListeners.add(listener) + } + } + + @JvmStatic + fun removeNotificationListener(listener: FaceclawNotificationListener?) { + if (listener != null) { + notificationListeners.remove(listener) + } + } + + @JvmStatic + fun hasActiveNotificationTitle(expectedTitle: String?): Boolean { + val service = activeService + if (service == null || expectedTitle == null || expectedTitle.isEmpty()) { + return false + } + val notifications: Array? + try { + notifications = service.activeNotifications + } catch (e: SecurityException) { + Log.w(TAG, "notification access denied while checking active notifications", e) + return false + } catch (t: Throwable) { + Log.w(TAG, "failed to check active notifications", t) + return false + } + if (notifications == null || notifications.size == 0) { + return false + } + for (notification in notifications) { + if (notification == null || notification.notification == null) { + continue + } + val extras: Bundle? = notification.notification.extras + if (extras == null) { + continue + } + val title: CharSequence? = extras.getCharSequence(Notification.EXTRA_TITLE) + if (title != null && expectedTitle.contentEquals(title)) { + return true + } + } + return false + } + + @JvmStatic + fun getActiveNotificationIconGrays(iconSize: Int, maxIcons: Int): ByteArray { + val service = activeService + val size = Math.max(1, Math.min(96, iconSize)) + val limit = Math.max(0, maxIcons) + if (service == null || limit == 0) { + return ByteArray(0) + } + + val notifications: Array? + try { + notifications = service.activeNotifications + } catch (e: SecurityException) { + Log.w(TAG, "notification access denied while reading icons", e) + return ByteArray(0) + } catch (t: Throwable) { + Log.w(TAG, "failed to read notification icons", t) + return ByteArray(0) + } + if (notifications == null || notifications.size == 0) { + return ByteArray(0) + } + + val out = ByteArrayOutputStream(size * size * Math.min(limit, notifications.size)) + val emittedGroupKeys: MutableSet = HashSet() + var emitted = 0 + for (statusBarNotification in notifications) { + if (!shouldShowNotificationIcon(service, statusBarNotification)) { + continue + } + val dedupeGroupKey = getNotificationDedupeGroupKey(statusBarNotification!!) + if (dedupeGroupKey != null && emittedGroupKeys.contains(dedupeGroupKey)) { + continue + } + val drawable = loadNotificationIcon(service, statusBarNotification.notification) + if (drawable == null) { + Log.i(TAG, "icon skipped (no drawable): " + statusBarNotification.packageName) + continue + } + Log.i(TAG, "icon[" + emitted + "] pkg=" + statusBarNotification.packageName + + " drawable=" + drawable.javaClass.simpleName + + " intrinsic=" + drawable.intrinsicWidth + "x" + drawable.intrinsicHeight) + appendIconGrayBytes(drawable, size, out, service, emitted, statusBarNotification.packageName) + if (dedupeGroupKey != null) { + emittedGroupKeys.add(dedupeGroupKey) + } + emitted += 1 + if (emitted >= limit) { + break + } + } + return out.toByteArray() + } + + /** + * Grayscale icon (iconSize*iconSize bytes) for one active notification, + * identified by its key; empty array when the notification or its icon is + * unavailable. Shares the extraction/scaling pipeline with the tray icon + * strip above. + */ + @JvmStatic + fun getNotificationIconGrayForKey(key: String?, iconSize: Int): ByteArray { + val service = activeService + val size = Math.max(1, Math.min(96, iconSize)) + if (service == null) { + return ByteArray(0) + } + val statusBarNotification = findActiveNotificationByKey(service, key) + if (statusBarNotification == null || statusBarNotification.notification == null) { + return ByteArray(0) + } + val drawable = loadNotificationIcon(service, statusBarNotification.notification) + if (drawable == null) { + return ByteArray(0) + } + val out = ByteArrayOutputStream(size * size) + appendIconGrayBytes(drawable, size, out, service, -1, statusBarNotification.packageName) + return out.toByteArray() + } + + @JvmStatic + fun getActiveNotificationsJson(maxNotifications: Int): String { + val service = activeService + val limit = Math.max(0, maxNotifications) + if (service == null || limit == 0) { + return "[]" + } + + val notifications: Array? + try { + notifications = service.activeNotifications + } catch (e: SecurityException) { + Log.w(TAG, "notification access denied while reading notifications", e) + return "[]" + } catch (t: Throwable) { + Log.w(TAG, "failed to read notifications", t) + return "[]" + } + if (notifications == null || notifications.size == 0) { + return "[]" + } + + Arrays.sort(notifications, object : Comparator { + override fun compare(a: StatusBarNotification?, b: StatusBarNotification?): Int { + val left = if (a == null) 0L else a.postTime + val right = if (b == null) 0L else b.postTime + return java.lang.Long.compare(right, left) + } + }) + + val out = JSONArray() + for (statusBarNotification in notifications) { + if (out.length() >= limit) { + break + } + if (!shouldShowNotificationInList(service, statusBarNotification)) { + continue + } + try { + out.put(buildNotificationJson(service, statusBarNotification!!)) + } catch (t: Throwable) { + Log.w(TAG, "failed to serialize notification", t) + } + } + return out.toString() + } + + @JvmStatic + fun invokeNotificationAction(key: String?, actionIndex: Int): Boolean { + val service = activeService + val statusBarNotification = findActiveNotificationByKey(service, key) + if (statusBarNotification == null || statusBarNotification.notification == null) { + return false + } + val actions: Array? = statusBarNotification.notification.actions + if (actions == null || actionIndex < 0 || actionIndex >= actions.size) { + return false + } + val intent: PendingIntent? = actions[actionIndex].actionIntent + if (intent == null) { + return false + } + try { + intent.send() + return true + } catch (e: PendingIntent.CanceledException) { + Log.w(TAG, "notification action pending intent was canceled", e) + return false + } catch (t: Throwable) { + Log.w(TAG, "failed to invoke notification action", t) + return false + } + } + + @JvmStatic + fun dismissNotification(key: String?): Boolean { + val service = activeService + if (service == null || key == null || key.isEmpty()) { + return false + } + try { + service.cancelNotification(key) + return true + } catch (e: SecurityException) { + Log.w(TAG, "notification access denied while dismissing notification", e) + return false + } catch (t: Throwable) { + Log.w(TAG, "failed to dismiss notification", t) + return false + } + } + + private fun emitNotificationPosted(key: String?) { + if (key == null || key.isEmpty() || notificationListeners.isEmpty()) { + return + } + for (listener in notificationListeners) { + mainHandler.post { + try { + listener.onNotificationPosted(key) + } catch (t: Throwable) { + Log.w(TAG, "notification listener failed", t) + } + } + } + } + + private fun refreshActiveNotificationWakeKeys(service: FaceclawMediaNotificationListenerService) { + val notifications: Array? + try { + notifications = service.activeNotifications + } catch (t: Throwable) { + Log.w(TAG, "failed to refresh active notification wake keys", t) + return + } + synchronized(activeNotificationWakeKeys) { + activeNotificationWakeKeys.clear() + if (notifications == null) { + return + } + for (statusBarNotification in notifications) { + if (shouldShowNotificationInList(service, statusBarNotification)) { + val key: String? = statusBarNotification!!.key + if (key != null && !key.isEmpty()) { + activeNotificationWakeKeys.add(key) + } + } + } + } + } + + private fun shouldEmitNotificationPosted(statusBarNotification: StatusBarNotification): Boolean { + val key: String? = statusBarNotification.key + if (key == null || key.isEmpty()) { + return true + } + + val alreadyActive: Boolean + synchronized(activeNotificationWakeKeys) { + alreadyActive = activeNotificationWakeKeys.contains(key) + activeNotificationWakeKeys.add(key) + } + return !alreadyActive || !isPersistentNotification(statusBarNotification) + } + + private fun forgetActiveNotificationWakeKey(statusBarNotification: StatusBarNotification?) { + if (statusBarNotification == null) { + return + } + val key: String? = statusBarNotification.key + if (key == null || key.isEmpty()) { + return + } + synchronized(activeNotificationWakeKeys) { + activeNotificationWakeKeys.remove(key) + } + } + + private fun isPersistentNotification(statusBarNotification: StatusBarNotification): Boolean { + val notification: Notification? = statusBarNotification.notification + if (notification == null) { + return false + } + val persistentFlags = Notification.FLAG_ONGOING_EVENT or Notification.FLAG_NO_CLEAR + return (notification.flags and persistentFlags) != 0 + } + + private fun shouldShowNotificationIcon(service: FaceclawMediaNotificationListenerService, statusBarNotification: StatusBarNotification?): Boolean { + if (!shouldShowNotificationInList(service, statusBarNotification)) { + return false + } + val notification = statusBarNotification!!.notification + if ((notification.flags and Notification.FLAG_GROUP_SUMMARY) != 0) { + return false + } + return true + } + + private fun shouldShowNotificationInList(service: FaceclawMediaNotificationListenerService, statusBarNotification: StatusBarNotification?): Boolean { + if (statusBarNotification == null || statusBarNotification.notification == null) { + return false + } + // Our own notifications stay out of the mirror: the foreground-service + // one is noise, and the Timers app rings on the glasses itself (its + // phone notification is for the phone), so mirroring it would stack a + // notification modal over the ringing screen. + if (service.packageName == statusBarNotification.packageName) { + return false + } + val notification = statusBarNotification.notification + if (Notification.CATEGORY_TRANSPORT == notification.category) { + return false + } + val extras: Bundle? = notification.extras + if (extras != null && extras.containsKey("android.mediaSession")) { + return false + } + + val rankingMap: NotificationListenerService.RankingMap? = service.currentRanking + if (rankingMap == null) { + return true + } + val ranking = NotificationListenerService.Ranking() + if (!rankingMap.getRanking(statusBarNotification.key, ranking)) { + return true + } + val importance = ranking.importance + return importance > NotificationManager.IMPORTANCE_MIN + } + + private fun getNotificationDedupeGroupKey(statusBarNotification: StatusBarNotification): String? { + val notification = statusBarNotification.notification + if (notification.group == null && statusBarNotification.overrideGroupKey == null) { + return null + } + val groupKey: String? = statusBarNotification.groupKey + if (groupKey == null || groupKey.isEmpty()) { + return null + } + // Group children often share the same small icon. Emit only one icon for the group. + return groupKey + } + + private fun loadNotificationIcon(service: FaceclawMediaNotificationListenerService, notification: Notification): Drawable? { + try { + val smallIcon: Icon? = notification.smallIcon + if (smallIcon != null) { + val drawable: Drawable? = smallIcon.loadDrawable(service) + if (drawable != null) { + // Status-bar small icons are alpha templates: the platform + // draws them tinted and ignores their color channels, which + // apps may fill with garbage (Discord ships noise there). + // Tint white so the shape comes from alpha alone, matching + // how the status bar renders them. + drawable.mutate() + drawable.setTint(Color.WHITE) + if (drawable is BitmapDrawable) { + drawable.setFilterBitmap(true) + } + return drawable + } + } + } catch (t: Throwable) { + Log.w(TAG, "failed to load small notification icon", t) + } + try { + // Large icons are real color images (avatars, album art); keep color. + val largeIcon: Icon? = notification.getLargeIcon() + if (largeIcon != null) { + return largeIcon.loadDrawable(service) + } + } catch (t: Throwable) { + Log.w(TAG, "failed to load large notification icon", t) + } + return null + } + + private fun appendIconGrayBytes(drawable: Drawable, size: Int, out: ByteArrayOutputStream, + service: FaceclawMediaNotificationListenerService, index: Int, packageName: String?) { + val bitmap = renderIconScaled(drawable, size) + dumpIconDebugPng(service, index, packageName, bitmap) + for (y in 0 until size) { + for (x in 0 until size) { + val color = bitmap.getPixel(x, y) + val alpha = Color.alpha(color) + val grayLinear = (0.2126 * Color.red(color) + 0.7152 * Color.green(color) + 0.0722 * Color.blue(color)) * alpha / (255.0 * 255.0) + val gray = Math.round(255.0 * Math.pow(Math.max(0.0, Math.min(1.0, grayLinear)), NOTIFICATION_ICON_GAMMA)).toInt() + out.write(gray and 0xff) + } + } + bitmap.recycle() + } + + /** + * Render a drawable at the target size with proper downscaling. Detailed + * sources (e.g. avatar bitmaps used as notification icons) are rendered at + * native resolution and reduced by repeated halving: a single filtered pass + * from, say, 126px to 24px samples too sparsely and turns fine detail into + * speckle that reads as a garbled icon. + */ + private fun renderIconScaled(drawable: Drawable, size: Int): Bitmap { + val renderW = Math.max(size, drawable.intrinsicWidth) + val renderH = Math.max(size, drawable.intrinsicHeight) + var bitmap = Bitmap.createBitmap(renderW, renderH, Bitmap.Config.ARGB_8888) + val canvas = Canvas(bitmap) + drawable.setBounds(0, 0, renderW, renderH) + drawable.draw(canvas) + while (bitmap.width >= size * 2 && bitmap.height >= size * 2) { + val halved = Bitmap.createScaledBitmap(bitmap, bitmap.width / 2, bitmap.height / 2, true) + bitmap.recycle() + bitmap = halved + } + if (bitmap.width != size || bitmap.height != size) { + val scaled = Bitmap.createScaledBitmap(bitmap, size, size, true) + bitmap.recycle() + bitmap = scaled + } + return bitmap + } + + /** + * Debug aid for garbled-icon reports: saves each rendered icon to + * /debug-icons/ (adb-pullable) so extraction problems can + * be told apart from downstream compositing/transmission problems. Cheap: + * runs at most once per icon-cache refresh on tiny bitmaps. + */ + private fun dumpIconDebugPng(service: FaceclawMediaNotificationListenerService, index: Int, packageName: String?, bitmap: Bitmap) { + if (index < 0) { + return + } + try { + val dir = java.io.File(service.getExternalFilesDir(null), "debug-icons") + if (!dir.exists() && !dir.mkdirs()) { + return + } + val safeName = if (packageName == null) "unknown" else packageName.replace(Regex("[^A-Za-z0-9._-]"), "_") + val file = java.io.File(dir, "icon-$index-$safeName.png") + java.io.FileOutputStream(file).use { stream -> + bitmap.compress(Bitmap.CompressFormat.PNG, 100, stream) + } + } catch (t: Throwable) { + Log.w(TAG, "failed to dump debug icon", t) + } + } + + private fun findActiveNotificationByKey(service: FaceclawMediaNotificationListenerService?, key: String?): StatusBarNotification? { + if (service == null || key == null || key.isEmpty()) { + return null + } + val notifications: Array? + try { + notifications = service.activeNotifications + } catch (t: Throwable) { + Log.w(TAG, "failed to find active notification", t) + return null + } + if (notifications == null) { + return null + } + for (statusBarNotification in notifications) { + if (statusBarNotification != null && key == statusBarNotification.key) { + return statusBarNotification + } + } + return null + } + + @Throws(JSONException::class) + private fun buildNotificationJson(service: FaceclawMediaNotificationListenerService, statusBarNotification: StatusBarNotification): JSONObject { + val notification = statusBarNotification.notification + val extras: Bundle? = notification.extras + val out = JSONObject() + out.put("key", statusBarNotification.key) + out.put("packageName", statusBarNotification.packageName) + out.put("appName", getNotificationAppName(service, statusBarNotification)) + out.put("postTime", statusBarNotification.postTime) + out.put("when", notification.`when`) + putString(out, "category", notification.category) + if (extras != null) { + putCharSequence(out, "title", firstNonEmpty( + extras.getCharSequence(Notification.EXTRA_TITLE_BIG), + extras.getCharSequence(Notification.EXTRA_TITLE) + )) + putCharSequence(out, "text", extras.getCharSequence(Notification.EXTRA_TEXT)) + putCharSequence(out, "bigText", extras.getCharSequence(Notification.EXTRA_BIG_TEXT)) + putCharSequence(out, "subText", extras.getCharSequence(Notification.EXTRA_SUB_TEXT)) + putCharSequence(out, "infoText", extras.getCharSequence(Notification.EXTRA_INFO_TEXT)) + putCharSequence(out, "summaryText", extras.getCharSequence(Notification.EXTRA_SUMMARY_TEXT)) + val textLines: Array? = extras.getCharSequenceArray(Notification.EXTRA_TEXT_LINES) + val lines = JSONArray() + if (textLines != null) { + for (line in textLines) { + val text = charSequenceToString(line) + if (!text.isEmpty()) { + lines.put(text) + } + } + } + out.put("lines", lines) + } else { + out.put("lines", JSONArray()) + } + + val actionsJson = JSONArray() + val actions: Array? = notification.actions + if (actions != null) { + for (index in actions.indices) { + val action = actions[index] + if (action == null) { + continue + } + val title = charSequenceToString(action.title) + if (title.isEmpty()) { + continue + } + val actionJson = JSONObject() + actionJson.put("index", index) + actionJson.put("title", title) + actionJson.put("enabled", action.actionIntent != null) + actionsJson.put(actionJson) + } + } + out.put("actions", actionsJson) + return out + } + + private fun getNotificationAppName(service: FaceclawMediaNotificationListenerService, statusBarNotification: StatusBarNotification): String { + val notification = statusBarNotification.notification + val extras: Bundle? = notification.extras + if (extras != null) { + val substituteName = charSequenceToString(extras.getCharSequence(EXTRA_SUBSTITUTE_APP_NAME)) + if (!substituteName.isEmpty()) { + return substituteName + } + } + return getAppLabel(service, statusBarNotification.packageName) + } + + private fun getAppLabel(service: FaceclawMediaNotificationListenerService, packageName: String?): String { + if (packageName == null || packageName.isEmpty()) { + return "" + } + try { + val label: CharSequence? = service + .packageManager + .getApplicationLabel(service.packageManager.getApplicationInfo(packageName, 0)) + val text = charSequenceToString(label) + return if (text.isEmpty()) packageName else text + } catch (t: Throwable) { + return packageName + } + } + + @Throws(JSONException::class) + private fun putString(out: JSONObject, key: String, value: String?) { + out.put(key, value ?: "") + } + + @Throws(JSONException::class) + private fun putCharSequence(out: JSONObject, key: String, value: CharSequence?) { + out.put(key, charSequenceToString(value)) + } + + private fun firstNonEmpty(first: CharSequence?, second: CharSequence?): CharSequence? { + return if (charSequenceToString(first).isEmpty()) second else first + } + + private fun charSequenceToString(value: CharSequence?): String { + return value?.toString() ?: "" + } + } + + override fun onCreate() { + super.onCreate() + activeService = this + } + + override fun onDestroy() { + if (activeService === this) { + activeService = null + } + super.onDestroy() + } + + override fun onListenerConnected() { + activeService = this + super.onListenerConnected() + refreshActiveNotificationWakeKeys(this) + } + + override fun onListenerDisconnected() { + if (activeService === this) { + activeService = null + } + super.onListenerDisconnected() + } + + override fun onNotificationPosted(statusBarNotification: StatusBarNotification?) { + super.onNotificationPosted(statusBarNotification) + if (!shouldShowNotificationInList(this, statusBarNotification)) { + forgetActiveNotificationWakeKey(statusBarNotification) + return + } + if (shouldEmitNotificationPosted(statusBarNotification!!)) { + emitNotificationPosted(statusBarNotification.key) + } + } + + override fun onNotificationRemoved(statusBarNotification: StatusBarNotification?) { + forgetActiveNotificationWakeKey(statusBarNotification) + super.onNotificationRemoved(statusBarNotification) + } +} diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawModelDownloader.java b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawModelDownloader.java deleted file mode 100644 index c1764d6e..00000000 --- a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawModelDownloader.java +++ /dev/null @@ -1,187 +0,0 @@ -package com.faceclaw.app; - -import android.os.Handler; -import android.os.Looper; -import android.util.Log; - -import java.io.File; -import java.io.FileInputStream; -import java.io.FileOutputStream; -import java.io.IOException; -import java.io.InputStream; -import java.security.MessageDigest; -import java.util.concurrent.TimeUnit; - -import okhttp3.Call; -import okhttp3.OkHttpClient; -import okhttp3.Request; -import okhttp3.Response; -import okhttp3.ResponseBody; - -/** - * Large-file downloader for on-phone model weights (multi-GB, from Hugging - * Face). Downloads to ".part" with HTTP Range resume across app - * restarts, verifies a pinned sha256 (hashed incrementally, including the - * resumed prefix), then renames into place. Listener callbacks are posted to - * the constructing thread's Looper, like the other Faceclaw bridges. - */ -public class FaceclawModelDownloader { - private static final String TAG = "FaceclawModelDl"; - private static final long PROGRESS_INTERVAL_MS = 500; - - private static volatile OkHttpClient sharedClient; - - private final Handler callbackHandler; - private final String url; - private final File destFile; - private final File partFile; - private final String expectedSha256; - private final long expectedTotalBytes; - private final FaceclawModelDownloaderListener listener; - - private volatile boolean cancelled; - private volatile Call call; - private Thread worker; - - public FaceclawModelDownloader(String url, String destPath, String expectedSha256, - long expectedTotalBytes, FaceclawModelDownloaderListener listener) { - if (listener == null) throw new IllegalArgumentException("listener is required"); - Looper looper = Looper.myLooper(); - this.callbackHandler = new Handler(looper != null ? looper : Looper.getMainLooper()); - this.url = url; - this.destFile = new File(destPath); - this.partFile = new File(destPath + ".part"); - this.expectedSha256 = expectedSha256 == null ? "" : expectedSha256.toLowerCase(); - this.expectedTotalBytes = expectedTotalBytes; - this.listener = listener; - } - - private static OkHttpClient getClient() { - if (sharedClient == null) { - synchronized (FaceclawModelDownloader.class) { - if (sharedClient == null) { - sharedClient = new OkHttpClient.Builder() - .connectTimeout(30, TimeUnit.SECONDS) - .readTimeout(60, TimeUnit.SECONDS) - .addInterceptor(FaceclawHttp.userAgentInterceptor()) - .build(); - } - } - } - return sharedClient; - } - - public void start() { - worker = new Thread(this::run, "FaceclawModelDl"); - worker.start(); - } - - /** Stops the download but keeps the .part file so a later start() resumes. */ - public void cancel() { - cancelled = true; - Call current = call; - if (current != null) current.cancel(); - } - - private void run() { - try { - if (destFile.exists() && destFile.length() > 0) { - post(() -> listener.onDone(destFile.getAbsolutePath())); - return; - } - destFile.getParentFile().mkdirs(); - - MessageDigest digest = MessageDigest.getInstance("SHA-256"); - long offset = 0; - if (partFile.exists()) { - offset = hashExistingPrefix(digest); - if (cancelled) return; - } - - Request.Builder builder = new Request.Builder().url(url); - if (offset > 0) builder.header("Range", "bytes=" + offset + "-"); - call = getClient().newCall(builder.build()); - - try (Response response = call.execute()) { - ResponseBody body = response.body(); - if (!response.isSuccessful() || body == null) { - postError("Model download failed: HTTP " + response.code()); - return; - } - boolean resumed = response.code() == 206; - if (offset > 0 && !resumed) { - // Server ignored the Range header; start over. - offset = 0; - digest = MessageDigest.getInstance("SHA-256"); - } - long total = expectedTotalBytes > 0 ? expectedTotalBytes - : (body.contentLength() > 0 ? offset + body.contentLength() : -1); - - try (FileOutputStream out = new FileOutputStream(partFile, resumed); - InputStream in = body.byteStream()) { - byte[] buffer = new byte[1 << 16]; - long done = offset; - long lastProgressAt = 0; - int read; - while ((read = in.read(buffer)) != -1) { - if (cancelled) return; - out.write(buffer, 0, read); - digest.update(buffer, 0, read); - done += read; - long now = System.currentTimeMillis(); - if (now - lastProgressAt >= PROGRESS_INTERVAL_MS) { - lastProgressAt = now; - final long doneFinal = done; - final long totalFinal = total; - post(() -> listener.onProgress(doneFinal, totalFinal)); - } - } - } - } - - if (expectedTotalBytes > 0 && partFile.length() != expectedTotalBytes) { - postError("Model download ended early (" + partFile.length() + " of " - + expectedTotalBytes + " bytes); try again to resume"); - return; - } - String actualSha = FaceclawFirmwareUtil.bytesToHex(digest.digest()); - if (!expectedSha256.isEmpty() && !actualSha.equals(expectedSha256)) { - partFile.delete(); - postError("Model download was corrupted (checksum mismatch); download it again"); - return; - } - if (!partFile.renameTo(destFile)) { - postError("Could not move the downloaded model into place"); - return; - } - post(() -> listener.onDone(destFile.getAbsolutePath())); - } catch (Exception e) { - if (cancelled) return; - Log.w(TAG, "download failed", e); - postError("Model download failed: " + e); - } - } - - /** Hash the already-downloaded prefix so the final digest covers the whole file. */ - private long hashExistingPrefix(MessageDigest digest) throws IOException { - long hashed = 0; - try (FileInputStream in = new FileInputStream(partFile)) { - byte[] buffer = new byte[1 << 16]; - int read; - while ((read = in.read(buffer)) != -1) { - if (cancelled) return hashed; - digest.update(buffer, 0, read); - hashed += read; - } - } - return hashed; - } - - private void post(Runnable runnable) { - callbackHandler.post(runnable); - } - - private void postError(final String message) { - post(() -> listener.onError(message)); - } -} diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawModelDownloader.kt b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawModelDownloader.kt new file mode 100644 index 00000000..376175d8 --- /dev/null +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawModelDownloader.kt @@ -0,0 +1,198 @@ +package com.faceclaw.app + +import android.os.Handler +import android.os.Looper +import android.util.Log + +import java.io.File +import java.io.FileInputStream +import java.io.FileOutputStream +import java.io.IOException +import java.security.MessageDigest +import java.util.concurrent.TimeUnit + +import okhttp3.Call +import okhttp3.OkHttpClient +import okhttp3.Request + +/** + * Large-file downloader for on-phone model weights (multi-GB, from Hugging + * Face). Downloads to ".part" with HTTP Range resume across app + * restarts, verifies a pinned sha256 (hashed incrementally, including the + * resumed prefix), then renames into place. Listener callbacks are posted to + * the constructing thread's Looper, like the other Faceclaw bridges. + */ +class FaceclawModelDownloader( + url: String, + destPath: String, + expectedSha256: String?, + expectedTotalBytes: Long, + listener: FaceclawModelDownloaderListener? +) { + private val callbackHandler: Handler + private val url: String + private val destFile: File + private val partFile: File + private val expectedSha256: String + private val expectedTotalBytes: Long + private val listener: FaceclawModelDownloaderListener + + @Volatile + private var cancelled = false + @Volatile + private var call: Call? = null + private var worker: Thread? = null + + init { + if (listener == null) throw IllegalArgumentException("listener is required") + val looper = Looper.myLooper() + this.callbackHandler = Handler(looper ?: Looper.getMainLooper()) + this.url = url + this.destFile = File(destPath) + this.partFile = File("$destPath.part") + this.expectedSha256 = expectedSha256?.lowercase() ?: "" + this.expectedTotalBytes = expectedTotalBytes + this.listener = listener + } + + companion object { + private const val TAG = "FaceclawModelDl" + private const val PROGRESS_INTERVAL_MS = 500L + + @Volatile + private var sharedClient: OkHttpClient? = null + + private fun getClient(): OkHttpClient { + if (sharedClient == null) { + synchronized(FaceclawModelDownloader::class.java) { + if (sharedClient == null) { + sharedClient = OkHttpClient.Builder() + .connectTimeout(30, TimeUnit.SECONDS) + .readTimeout(60, TimeUnit.SECONDS) + .addInterceptor(FaceclawHttp.userAgentInterceptor()) + .build() + } + } + } + return sharedClient!! + } + } + + fun start() { + val thread = Thread({ run() }, "FaceclawModelDl") + worker = thread + thread.start() + } + + /** Stops the download but keeps the .part file so a later start() resumes. */ + fun cancel() { + cancelled = true + val current = call + current?.cancel() + } + + private fun run() { + try { + if (destFile.exists() && destFile.length() > 0) { + post { listener.onDone(destFile.absolutePath) } + return + } + destFile.parentFile.mkdirs() + + var digest = MessageDigest.getInstance("SHA-256") + var offset = 0L + if (partFile.exists()) { + offset = hashExistingPrefix(digest) + if (cancelled) return + } + + val builder = Request.Builder().url(url) + if (offset > 0) builder.header("Range", "bytes=$offset-") + val newCall = getClient().newCall(builder.build()) + call = newCall + + newCall.execute().use { response -> + val body = response.body + if (!response.isSuccessful || body == null) { + postError("Model download failed: HTTP " + response.code) + return + } + val resumed = response.code == 206 + if (offset > 0 && !resumed) { + // Server ignored the Range header; start over. + offset = 0L + digest = MessageDigest.getInstance("SHA-256") + } + val total = if (expectedTotalBytes > 0) expectedTotalBytes + else (if (body.contentLength() > 0) offset + body.contentLength() else -1L) + + FileOutputStream(partFile, resumed).use { out -> + body.byteStream().use { input -> + val buffer = ByteArray(1 shl 16) + var done = offset + var lastProgressAt = 0L + var read: Int + while (input.read(buffer).also { read = it } != -1) { + if (cancelled) return + out.write(buffer, 0, read) + digest.update(buffer, 0, read) + done += read + val now = System.currentTimeMillis() + if (now - lastProgressAt >= PROGRESS_INTERVAL_MS) { + lastProgressAt = now + val doneFinal = done + val totalFinal = total + post { listener.onProgress(doneFinal, totalFinal) } + } + } + } + } + } + + if (expectedTotalBytes > 0 && partFile.length() != expectedTotalBytes) { + postError("Model download ended early (" + partFile.length() + " of " + + expectedTotalBytes + " bytes); try again to resume") + return + } + val actualSha = FaceclawFirmwareUtil.bytesToHex(digest.digest()) + if (!expectedSha256.isEmpty() && actualSha != expectedSha256) { + partFile.delete() + postError("Model download was corrupted (checksum mismatch); download it again") + return + } + if (!partFile.renameTo(destFile)) { + postError("Could not move the downloaded model into place") + return + } + post { listener.onDone(destFile.absolutePath) } + } catch (e: Exception) { + if (cancelled) return + Log.w(TAG, "download failed", e) + postError("Model download failed: $e") + } + } + + /** Hash the already-downloaded prefix so the final digest covers the whole file. */ + @Throws(IOException::class) + private fun hashExistingPrefix(digest: MessageDigest): Long { + var hashed = 0L + FileInputStream(partFile).use { input -> + val buffer = ByteArray(1 shl 16) + var read: Int + while (input.read(buffer).also { read = it } != -1) { + if (cancelled) return hashed + digest.update(buffer, 0, read) + hashed += read + } + } + return hashed + } + + private fun post(runnable: Runnable) { + callbackHandler.post(runnable) + } + + private fun postError(message: String) { + post { listener.onError(message) } + } +} diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawPreviewCompositor.java b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawPreviewCompositor.java deleted file mode 100644 index e0351588..00000000 --- a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawPreviewCompositor.java +++ /dev/null @@ -1,198 +0,0 @@ -package com.faceclaw.app; - -import android.content.Context; - -/** - * Headless display target for preview-only mode (no glasses paired). Hosts - * the same SurfaceCompositor the BLE communicator uses, so the TS side can - * run its full surface/render pipeline unchanged; composited frames simply - * have nowhere to go, and the phone-mirror preview, screenshots, and GIF - * recordings read the retained composite back exactly as they do on a live - * connection. - * - * The method signatures mirror the display subset of FaceclawBleCommunicator - * (the TS DisplayTarget interface is typed against that subset). - */ -public final class FaceclawPreviewCompositor { - private final Context appContext; - private final SurfaceCompositor compositor = new SurfaceCompositor(); - private final android.os.Handler mainHandler = new android.os.Handler(android.os.Looper.getMainLooper()); - - // Notified (on the main looper, where the phone UI's JS runs) after each - // applied surface frame, standing in for the BLE path's per-frame metrics - // callback so the phone mirror refreshes promptly instead of waiting for - // its safety-net poll. - private volatile Runnable frameListener; - - // Active animated-GIF screen recording, or null when idle. Frames are - // pushed by recordScreenFrame(), which the TS side calls at each - // phone-preview flush. - private volatile GifScreenRecorder screenRecorder; - - // Worker-app isolates locate their frame target through a static, like - // FaceclawBleCommunicator.getActive(); Java statics are shared across - // isolates, unlike anything on the JS side. - private static volatile FaceclawPreviewCompositor activeInstance; - - public static FaceclawPreviewCompositor getActive() { - return activeInstance; - } - - public FaceclawPreviewCompositor(Context context) { - this.appContext = context.getApplicationContext(); - } - - /** Publish this compositor as the workers' frame target. */ - public void makeActive() { - activeInstance = this; - } - - /** Withdraw from workers (a real connection is taking over). */ - public void release() { - if (activeInstance == this) { - activeInstance = null; - } - frameListener = null; - } - - public void setFrameListener(Runnable listener) { - this.frameListener = listener; - } - - /** Set the compositor's output frame size. Call before configuring surfaces. */ - public void configureCompositorScreen(int width, int height) { - compositor.configureScreen(width, height); - } - - public void configureSurface(String id, int x, int y, int width, int height, int zOrder, int transparency) { - compositor.configureSurface(id, x, y, width, height, zOrder, transparency); - } - - public void removeSurface(String id) { - compositor.removeSurface(id); - } - - /** Dim every surface below zOrder belowZOrder to factor256/256 (see SurfaceCompositor). */ - public void submitShellScene(java.nio.ByteBuffer bytes, int paintMs, int frameId) { - compositor.setShellScene(new AndroidByteReader(bytes)); - FrameTimings.getInstance().finishFrame(frameId, "composited (preview-only, no glasses)"); - Runnable listener = frameListener; - if (listener != null) mainHandler.post(listener); - } - - public void setUnderlayDim(int belowZOrder, int factor256) { - compositor.setUnderlayDim(belowZOrder, factor256); - } - - public void setSurfaceVisible(String id, boolean visible) { - compositor.setSurfaceVisible(id, visible); - } - - /** Blank (screen off) or unblank the output; retained surface state survives. */ - public void setScreenBlanked(boolean blanked) { - compositor.setBlanked(blanked); - } - - /** - * Apply an update to one compositor surface. Unlike the connected path - * there is no desired-frame store or transmit pipeline downstream; the - * preview/screenshot calls recomposite the retained state on demand, so - * applying the update is the whole job and the frame finishes here. - */ - public void submitSurfaceFrame( - java.nio.ByteBuffer pixels8bpp, - String surfaceId, - int rectX, - int rectY, - int rectWidth, - int rectHeight, - String contentFingerprint, - int paintMs, - int frameId, - java.nio.ByteBuffer glyphs - ) { - compositor.submitSurface( - surfaceId, new AndroidByteReader(pixels8bpp), rectX, rectY, rectWidth, rectHeight, contentFingerprint, - glyphs == null ? null : new AndroidByteReader(glyphs)); - FrameTimings.getInstance().finishFrame(frameId, "composited (preview-only, no glasses)"); - Runnable listener = frameListener; - if (listener != null) { - mainHandler.post(listener); - } - } - - /** - * The current composited screen as a phone-UI preview bitmap, or null - * before any surface has been configured. - */ - public android.graphics.Bitmap getCompositePreviewBitmap(double brightenGamma, boolean green) { - SurfaceCompositor.Composite composite = compositor.previewComposite(); - if (composite == null) { - return null; - } - return PreviewBitmapUtil.fromGray( - java.nio.ByteBuffer.wrap(composite.gray), composite.width, composite.height, brightenGamma, green); - } - - /** Save the current composite as a 4-bit grayscale PNG; returns the path or "". */ - public String saveCompositePngScreenshot() throws java.io.IOException { - SurfaceCompositor.Composite composite = compositor.previewComposite(); - if (composite == null) { - return ""; - } - return ScreenshotUtil.savePngScreenshot(appContext, composite.gray, composite.width, composite.height); - } - - /** - * Save the current composite cropped to the given screen rect (the region - * the shell says is actually occupied). The rect is clamped to the screen; - * a degenerate rect falls back to the full screen. - */ - public String saveCompositePngScreenshot(int cropX, int cropY, int cropWidth, int cropHeight) - throws java.io.IOException { - SurfaceCompositor.Composite composite = compositor.previewComposite(); - if (composite == null) { - return ""; - } - int x = Math.max(0, cropX); - int y = Math.max(0, cropY); - int width = Math.min(composite.width - x, cropWidth - (x - cropX)); - int height = Math.min(composite.height - y, cropHeight - (y - cropY)); - if (width <= 0 || height <= 0 || (x == 0 && y == 0 && width == composite.width && height == composite.height)) { - return ScreenshotUtil.savePngScreenshot(appContext, composite.gray, composite.width, composite.height); - } - byte[] cropped = new byte[width * height]; - for (int row = 0; row < height; row++) { - System.arraycopy(composite.gray, (y + row) * composite.width + x, cropped, row * width, width); - } - return ScreenshotUtil.savePngScreenshot(appContext, cropped, width, height); - } - - /** Begin collecting composite frames for an animated-GIF screen recording. */ - public void startScreenRecording() { - screenRecorder = new GifScreenRecorder(); - } - - /** Capture the current composite into the active recording; no-op when idle. */ - public void recordScreenFrame() { - GifScreenRecorder recorder = screenRecorder; - if (recorder == null) { - return; - } - SurfaceCompositor.Composite composite = compositor.previewComposite(); - if (composite == null) { - return; - } - recorder.addFrame(composite.gray, composite.width, composite.height, System.currentTimeMillis()); - } - - /** Finish the recording and save it as an animated GIF; returns the path or "". */ - public String stopScreenRecording() throws java.io.IOException { - GifScreenRecorder recorder = screenRecorder; - screenRecorder = null; - if (recorder == null) { - return ""; - } - return recorder.save(appContext); - } -} diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawPreviewCompositor.kt b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawPreviewCompositor.kt new file mode 100644 index 00000000..570ad3f2 --- /dev/null +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawPreviewCompositor.kt @@ -0,0 +1,199 @@ +package com.faceclaw.app + +import android.content.Context + +/** + * Headless display target for preview-only mode (no glasses paired). Hosts + * the same SurfaceCompositor the BLE communicator uses, so the TS side can + * run its full surface/render pipeline unchanged; composited frames simply + * have nowhere to go, and the phone-mirror preview, screenshots, and GIF + * recordings read the retained composite back exactly as they do on a live + * connection. + * + * The method signatures mirror the display subset of FaceclawBleCommunicator + * (the TS DisplayTarget interface is typed against that subset). + */ +class FaceclawPreviewCompositor(context: Context) { + private val appContext: Context = context.applicationContext + private val compositor = SurfaceCompositor() + private val mainHandler = android.os.Handler(android.os.Looper.getMainLooper()) + + // Notified (on the main looper, where the phone UI's JS runs) after each + // applied surface frame, standing in for the BLE path's per-frame metrics + // callback so the phone mirror refreshes promptly instead of waiting for + // its safety-net poll. + @Volatile private var frameListener: Runnable? = null + + // Active animated-GIF screen recording, or null when idle. Frames are + // pushed by recordScreenFrame(), which the TS side calls at each + // phone-preview flush. + @Volatile private var screenRecorder: GifScreenRecorder? = null + + companion object { + // Worker-app isolates locate their frame target through a static, like + // FaceclawBleCommunicator.getActive(); Java statics are shared across + // isolates, unlike anything on the JS side. + @Volatile private var activeInstance: FaceclawPreviewCompositor? = null + + @JvmStatic + fun getActive(): FaceclawPreviewCompositor? { + return activeInstance + } + } + + /** Publish this compositor as the workers' frame target. */ + fun makeActive() { + activeInstance = this + } + + /** Withdraw from workers (a real connection is taking over). */ + fun release() { + if (activeInstance === this) { + activeInstance = null + } + frameListener = null + } + + fun setFrameListener(listener: Runnable?) { + this.frameListener = listener + } + + /** Set the compositor's output frame size. Call before configuring surfaces. */ + fun configureCompositorScreen(width: Int, height: Int) { + compositor.configureScreen(width, height) + } + + fun configureSurface(id: String?, x: Int, y: Int, width: Int, height: Int, zOrder: Int, transparency: Int) { + compositor.configureSurface(id, x, y, width, height, zOrder, transparency) + } + + fun removeSurface(id: String) { + compositor.removeSurface(id) + } + + /** Dim every surface below zOrder belowZOrder to factor256/256 (see SurfaceCompositor). */ + fun submitShellScene(bytes: java.nio.ByteBuffer, paintMs: Int, frameId: Int) { + compositor.setShellScene(AndroidByteReader(bytes)) + FrameTimings.getInstance().finishFrame(frameId, "composited (preview-only, no glasses)") + val listener = frameListener + if (listener != null) mainHandler.post(listener) + } + + fun setUnderlayDim(belowZOrder: Int, factor256: Int) { + compositor.setUnderlayDim(belowZOrder, factor256) + } + + fun setSurfaceVisible(id: String, visible: Boolean) { + compositor.setSurfaceVisible(id, visible) + } + + /** Blank (screen off) or unblank the output; retained surface state survives. */ + fun setScreenBlanked(blanked: Boolean) { + compositor.setBlanked(blanked) + } + + /** + * Apply an update to one compositor surface. Unlike the connected path + * there is no desired-frame store or transmit pipeline downstream; the + * preview/screenshot calls recomposite the retained state on demand, so + * applying the update is the whole job and the frame finishes here. + */ + fun submitSurfaceFrame( + pixels8bpp: java.nio.ByteBuffer, + surfaceId: String, + rectX: Int, + rectY: Int, + rectWidth: Int, + rectHeight: Int, + contentFingerprint: String, + paintMs: Int, + frameId: Int, + glyphs: java.nio.ByteBuffer? + ) { + compositor.submitSurface( + surfaceId, AndroidByteReader(pixels8bpp), rectX, rectY, rectWidth, rectHeight, contentFingerprint, + if (glyphs == null) null else AndroidByteReader(glyphs)) + FrameTimings.getInstance().finishFrame(frameId, "composited (preview-only, no glasses)") + val listener = frameListener + if (listener != null) { + mainHandler.post(listener) + } + } + + /** + * The current composited screen as a phone-UI preview bitmap, or null + * before any surface has been configured. + */ + fun getCompositePreviewBitmap(brightenGamma: Double, green: Boolean): android.graphics.Bitmap? { + val composite = compositor.previewComposite() + if (composite == null) { + return null + } + return PreviewBitmapUtil.fromGray( + java.nio.ByteBuffer.wrap(composite.gray), composite.width, composite.height, brightenGamma, green) + } + + /** Save the current composite as a 4-bit grayscale PNG; returns the path or "". */ + @Throws(java.io.IOException::class) + fun saveCompositePngScreenshot(): String { + val composite = compositor.previewComposite() + if (composite == null) { + return "" + } + return ScreenshotUtil.savePngScreenshot(appContext, composite.gray, composite.width, composite.height) + } + + /** + * Save the current composite cropped to the given screen rect (the region + * the shell says is actually occupied). The rect is clamped to the screen; + * a degenerate rect falls back to the full screen. + */ + @Throws(java.io.IOException::class) + fun saveCompositePngScreenshot(cropX: Int, cropY: Int, cropWidth: Int, cropHeight: Int): String { + val composite = compositor.previewComposite() + if (composite == null) { + return "" + } + val x = Math.max(0, cropX) + val y = Math.max(0, cropY) + val width = Math.min(composite.width - x, cropWidth - (x - cropX)) + val height = Math.min(composite.height - y, cropHeight - (y - cropY)) + if (width <= 0 || height <= 0 || (x == 0 && y == 0 && width == composite.width && height == composite.height)) { + return ScreenshotUtil.savePngScreenshot(appContext, composite.gray, composite.width, composite.height) + } + val cropped = ByteArray(width * height) + for (row in 0 until height) { + System.arraycopy(composite.gray, (y + row) * composite.width + x, cropped, row * width, width) + } + return ScreenshotUtil.savePngScreenshot(appContext, cropped, width, height) + } + + /** Begin collecting composite frames for an animated-GIF screen recording. */ + fun startScreenRecording() { + screenRecorder = GifScreenRecorder() + } + + /** Capture the current composite into the active recording; no-op when idle. */ + fun recordScreenFrame() { + val recorder = screenRecorder + if (recorder == null) { + return + } + val composite = compositor.previewComposite() + if (composite == null) { + return + } + recorder.addFrame(composite.gray, composite.width, composite.height, System.currentTimeMillis()) + } + + /** Finish the recording and save it as an animated GIF; returns the path or "". */ + @Throws(java.io.IOException::class) + fun stopScreenRecording(): String { + val recorder = screenRecorder + screenRecorder = null + if (recorder == null) { + return "" + } + return recorder.save(appContext) + } +} diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawQrScanner.java b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawQrScanner.java deleted file mode 100644 index 4b7cd5c4..00000000 --- a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawQrScanner.java +++ /dev/null @@ -1,81 +0,0 @@ -package com.faceclaw.app; - -import android.app.Activity; -import android.content.Context; -import android.util.Log; - -import com.google.android.gms.common.ConnectionResult; -import com.google.android.gms.common.GoogleApiAvailability; -import com.google.android.gms.tasks.OnCanceledListener; -import com.google.android.gms.tasks.OnFailureListener; -import com.google.android.gms.tasks.OnSuccessListener; -import com.google.mlkit.vision.barcode.common.Barcode; -import com.google.mlkit.vision.codescanner.GmsBarcodeScanner; -import com.google.mlkit.vision.codescanner.GmsBarcodeScannerOptions; -import com.google.mlkit.vision.codescanner.GmsBarcodeScanning; - -/** - * QR scanning through Play Services' ML Kit code scanner. The scanner UI and - * the camera both live inside Play Services, so Faceclaw needs no camera - * permission of its own and no third-party scanner app — which is what the - * earlier ZXing SCAN intent needed, and no app on a stock Pixel registers for - * it. - * - * The scanner module is downloaded on demand the first time it is used (the - * manifest's com.google.mlkit.vision.DEPENDENCIES meta-data asks Play Services - * to fetch it at install time instead); a scan attempted before that finishes - * fails with a module-unavailable error rather than blocking. - * - * Callbacks fire on the main thread. - */ -public final class FaceclawQrScanner { - private static final String TAG = "FaceclawQrScanner"; - - private FaceclawQrScanner() {} - - /** Whether Play Services is present and current enough to scan. */ - public static boolean isAvailable(Context context) { - try { - int status = GoogleApiAvailability.getInstance().isGooglePlayServicesAvailable(context); - return status == ConnectionResult.SUCCESS; - } catch (Throwable error) { - Log.w(TAG, "Play Services availability check failed", error); - return false; - } - } - - /** Open the scanner UI; the listener gets the decoded text, or a cancel/error. */ - public static void scan(Activity activity, final FaceclawQrScannerListener listener) { - GmsBarcodeScannerOptions options = new GmsBarcodeScannerOptions.Builder() - .setBarcodeFormats(Barcode.FORMAT_QR_CODE) - .enableAutoZoom() - .build(); - GmsBarcodeScanner scanner = GmsBarcodeScanning.getClient(activity, options); - scanner.startScan() - .addOnSuccessListener(new OnSuccessListener() { - @Override - public void onSuccess(Barcode barcode) { - String value = barcode == null ? null : barcode.getRawValue(); - if (value == null) { - listener.onError("The code held no text."); - } else { - listener.onResult(value); - } - } - }) - .addOnCanceledListener(new OnCanceledListener() { - @Override - public void onCanceled() { - listener.onCancelled(); - } - }) - .addOnFailureListener(new OnFailureListener() { - @Override - public void onFailure(Exception error) { - Log.w(TAG, "scan failed", error); - String message = error.getMessage(); - listener.onError(message == null ? error.toString() : message); - } - }); - } -} diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawQrScanner.kt b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawQrScanner.kt new file mode 100644 index 00000000..a3857f44 --- /dev/null +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawQrScanner.kt @@ -0,0 +1,74 @@ +package com.faceclaw.app + +import android.app.Activity +import android.content.Context +import android.util.Log + +import com.google.android.gms.common.ConnectionResult +import com.google.android.gms.common.GoogleApiAvailability +import com.google.android.gms.tasks.OnCanceledListener +import com.google.android.gms.tasks.OnFailureListener +import com.google.android.gms.tasks.OnSuccessListener +import com.google.mlkit.vision.barcode.common.Barcode +import com.google.mlkit.vision.codescanner.GmsBarcodeScanner +import com.google.mlkit.vision.codescanner.GmsBarcodeScannerOptions +import com.google.mlkit.vision.codescanner.GmsBarcodeScanning + +/** + * QR scanning through Play Services' ML Kit code scanner. The scanner UI and + * the camera both live inside Play Services, so Faceclaw needs no camera + * permission of its own and no third-party scanner app — which is what the + * earlier ZXing SCAN intent needed, and no app on a stock Pixel registers for + * it. + * + * The scanner module is downloaded on demand the first time it is used (the + * manifest's com.google.mlkit.vision.DEPENDENCIES meta-data asks Play Services + * to fetch it at install time instead); a scan attempted before that finishes + * fails with a module-unavailable error rather than blocking. + * + * Callbacks fire on the main thread. + */ +class FaceclawQrScanner private constructor() { + companion object { + private const val TAG = "FaceclawQrScanner" + + /** Whether Play Services is present and current enough to scan. */ + @JvmStatic + fun isAvailable(context: Context): Boolean { + try { + val status = GoogleApiAvailability.getInstance().isGooglePlayServicesAvailable(context) + return status == ConnectionResult.SUCCESS + } catch (error: Throwable) { + Log.w(TAG, "Play Services availability check failed", error) + return false + } + } + + /** Open the scanner UI; the listener gets the decoded text, or a cancel/error. */ + @JvmStatic + fun scan(activity: Activity, listener: FaceclawQrScannerListener) { + val options = GmsBarcodeScannerOptions.Builder() + .setBarcodeFormats(Barcode.FORMAT_QR_CODE) + .enableAutoZoom() + .build() + val scanner: GmsBarcodeScanner = GmsBarcodeScanning.getClient(activity, options) + scanner.startScan() + .addOnSuccessListener(OnSuccessListener { barcode -> + val value = barcode?.rawValue + if (value == null) { + listener.onError("The code held no text.") + } else { + listener.onResult(value) + } + }) + .addOnCanceledListener(OnCanceledListener { + listener.onCancelled() + }) + .addOnFailureListener(OnFailureListener { error -> + Log.w(TAG, "scan failed", error) + val message = error.message + listener.onError(message ?: error.toString()) + }) + } + } +} diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawRemoteInput.java b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawRemoteInput.java deleted file mode 100644 index a44f47c9..00000000 --- a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawRemoteInput.java +++ /dev/null @@ -1,143 +0,0 @@ -package com.faceclaw.app; - -import android.os.Handler; -import android.os.Looper; -import java.io.ByteArrayOutputStream; -import java.net.InetAddress; -import java.net.NetworkInterface; -import java.util.Collections; -import org.json.JSONArray; -import java.net.ServerSocket; -import java.net.Socket; -import java.nio.charset.StandardCharsets; -import java.nio.ByteBuffer; -import java.security.MessageDigest; -import java.security.SecureRandom; -import org.json.JSONObject; - -/** Explicit-address, bounded input port. Complete requests notify JS on the main thread. */ -public final class FaceclawRemoteInput { - private static final FaceclawRemoteInput INSTANCE = new FaceclawRemoteInput(); - public static FaceclawRemoteInput getInstance() { return INSTANCE; } - private ServerSocket server; - private Socket client; - private String pending, reply; - private long nextId, requestId; - private long deadline; - private final Handler mainHandler = new Handler(Looper.getMainLooper()); - private Runnable requestListener; - - public synchronized void setRequestListener(Runnable listener) { requestListener = listener; } - - private void notifyRequestReady(long id) { - // Called with this monitor held; posting avoids entering JS on the socket thread. - mainHandler.post(() -> { - Runnable callback; - synchronized (this) { - if (requestId != id || pending == null || System.currentTimeMillis() >= deadline) return; - callback = requestListener; - } - if (callback != null) callback.run(); - }); - } - - public synchronized String start(int port) { return startAddress(port, "127.0.0.1"); } - public synchronized String startAddress(int port, String address) { - if (server != null) return ""; - try { - // Numeric local addresses only; never resolve hostnames or bind a wildcard. - if (!address.matches("[0-9.]+|[0-9a-fA-F]*:[0-9a-fA-F:.]*(%[a-zA-Z0-9_.-]+)?")) return "Invalid local address."; - InetAddress local = InetAddress.getByName(address); - if (local.isAnyLocalAddress() || local.isMulticastAddress() || NetworkInterface.getByInetAddress(local) == null) - return "Address is not assigned to a local interface."; - ServerSocket listener = new ServerSocket(port, 8, local); - server = listener; - Thread thread = new Thread(() -> serve(listener), "FaceclawInput"); - thread.setDaemon(true); - thread.start(); - return ""; - } catch (Exception e) { return "Could not open local input port " + port + "."; } - } - public synchronized void stop() { - try { if (server != null) server.close(); } catch (Exception ignored) {} - try { if (client != null) client.close(); } catch (Exception ignored) {} - server = null; client = null; pending = null; requestId = 0; reply = null; - notifyAll(); - } - public synchronized String nextRequest() { - if (System.currentTimeMillis() >= deadline) { pending = null; return null; } - String result = pending; pending = null; return result; - } - public synchronized void complete(long id, String value) { - if (id == requestId && System.currentTimeMillis() < deadline) { reply = value; notifyAll(); } - } - private void serve(ServerSocket listener) { - while (!listener.isClosed()) { - try (Socket socket = listener.accept()) { - synchronized (this) { if (server != listener) return; client = socket; } - socket.setSoTimeout(5000); - socket.setTcpNoDelay(true); - long readDeadline = System.currentTimeMillis() + 5000; - ByteArrayOutputStream body = new ByteArrayOutputStream(); - int b; - for (;;) { - long remaining = readDeadline - System.currentTimeMillis(); - if (remaining <= 0) throw new Exception("Read timeout"); - socket.setSoTimeout((int) remaining); - b = socket.getInputStream().read(); - if (b == -1 || b == '\n') break; - if (body.size() >= 65536) throw new Exception("Invalid frame"); - body.write(b); - } - if (b != '\n') continue; - String text = StandardCharsets.UTF_8.newDecoder().decode(ByteBuffer.wrap(body.toByteArray())).toString(); - String response; - synchronized (this) { - if (server != listener) return; - requestId = ++nextId; - deadline = System.currentTimeMillis() + 5000; - reply = null; - pending = new JSONObject().put("id", requestId).put("expiresAt", deadline) - .put("body", text).toString(); - notifyRequestReady(requestId); - while (reply == null && server == listener && System.currentTimeMillis() < deadline) { - wait(Math.max(1, deadline - System.currentTimeMillis())); - } - response = reply != null ? reply : "{\"ok\":false,\"error\":\"timeout\",\"message\":\"Faceclaw did not respond in time.\"}"; - if (server == listener) { pending = null; reply = null; requestId = 0; } - } - socket.getOutputStream().write((response + "\n").getBytes(StandardCharsets.UTF_8)); - } catch (Exception ignored) { - // Never log request bodies: they contain credentials and possibly private text. - } finally { - synchronized (this) { if (server == listener) client = null; } - } - } - } - public static String interfaces() { - JSONArray result = new JSONArray(); - try { - for (NetworkInterface device : Collections.list(NetworkInterface.getNetworkInterfaces())) { - if (!device.isUp()) continue; - for (InetAddress address : Collections.list(device.getInetAddresses())) { - if (address.isLoopbackAddress() || address.isLinkLocalAddress() || address.isAnyLocalAddress()) continue; - result.put(new JSONObject().put("name", device.getName()).put("address", address.getHostAddress()) - .put("pointToPoint", device.isPointToPoint())); - } - } - } catch (Exception ignored) {} - return result.toString(); - } - public String randomSecret() { - byte[] bytes = new byte[32]; new SecureRandom().nextBytes(bytes); return hex(bytes); - } - public String hash(String value) { - try { return hex(MessageDigest.getInstance("SHA-256").digest(value.getBytes(StandardCharsets.UTF_8))); } - catch (Exception e) { throw new IllegalStateException("SHA-256 unavailable", e); } - } - private static String hex(byte[] bytes) { - StringBuilder result = new StringBuilder(bytes.length * 2); - for (byte b : bytes) result.append(String.format(java.util.Locale.ROOT, "%02x", b & 255)); - return result.toString(); - } -} diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawRemoteInput.kt b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawRemoteInput.kt new file mode 100644 index 00000000..358098ab --- /dev/null +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawRemoteInput.kt @@ -0,0 +1,152 @@ +package com.faceclaw.app + +import android.os.Handler +import android.os.Looper +import java.io.ByteArrayOutputStream +import java.net.InetAddress +import java.net.NetworkInterface +import java.util.Collections +import org.json.JSONArray +import java.net.ServerSocket +import java.net.Socket +import java.nio.charset.StandardCharsets +import java.nio.ByteBuffer +import java.security.MessageDigest +import java.security.SecureRandom +import org.json.JSONObject + +/** Explicit-address, bounded input port. Complete requests notify JS on the main thread. */ +class FaceclawRemoteInput { + companion object { + private val INSTANCE = FaceclawRemoteInput() + @JvmStatic fun getInstance(): FaceclawRemoteInput { return INSTANCE } + + @JvmStatic + fun interfaces(): String { + val result = JSONArray() + try { + for (device in Collections.list(NetworkInterface.getNetworkInterfaces())) { + if (!device.isUp) continue + for (address in Collections.list(device.inetAddresses)) { + if (address.isLoopbackAddress || address.isLinkLocalAddress || address.isAnyLocalAddress) continue + result.put(JSONObject().put("name", device.name).put("address", address.hostAddress) + .put("pointToPoint", device.isPointToPoint)) + } + } + } catch (ignored: Exception) {} + return result.toString() + } + + private fun hex(bytes: ByteArray): String { + val result = StringBuilder(bytes.size * 2) + for (b in bytes) result.append(String.format(java.util.Locale.ROOT, "%02x", b.toInt() and 255)) + return result.toString() + } + } + private var server: ServerSocket? = null + private var client: Socket? = null + private var pending: String? = null + private var reply: String? = null + private var nextId = 0L + private var requestId = 0L + private var deadline = 0L + private val mainHandler = Handler(Looper.getMainLooper()) + private var requestListener: Runnable? = null + + @Synchronized fun setRequestListener(listener: Runnable?) { requestListener = listener } + + private fun notifyRequestReady(id: Long) { + // Called with this monitor held; posting avoids entering JS on the socket thread. + mainHandler.post { + val callback: Runnable? + synchronized(this) { + if (requestId != id || pending == null || System.currentTimeMillis() >= deadline) return@post + callback = requestListener + } + callback?.run() + } + } + + @Synchronized fun start(port: Int): String { return startAddress(port, "127.0.0.1") } + @Synchronized fun startAddress(port: Int, address: String): String { + if (server != null) return "" + try { + // Numeric local addresses only; never resolve hostnames or bind a wildcard. + if (!address.matches(Regex("[0-9.]+|[0-9a-fA-F]*:[0-9a-fA-F:.]*(%[a-zA-Z0-9_.-]+)?"))) return "Invalid local address." + val local = InetAddress.getByName(address) + if (local.isAnyLocalAddress || local.isMulticastAddress || NetworkInterface.getByInetAddress(local) == null) + return "Address is not assigned to a local interface." + val listener = ServerSocket(port, 8, local) + server = listener + val thread = Thread({ serve(listener) }, "FaceclawInput") + thread.isDaemon = true + thread.start() + return "" + } catch (e: Exception) { return "Could not open local input port $port." } + } + @Synchronized fun stop() { + try { server?.close() } catch (ignored: Exception) {} + try { client?.close() } catch (ignored: Exception) {} + server = null; client = null; pending = null; requestId = 0; reply = null + (this as java.lang.Object).notifyAll() + } + @Synchronized fun nextRequest(): String? { + if (System.currentTimeMillis() >= deadline) { pending = null; return null } + val result = pending; pending = null; return result + } + @Synchronized fun complete(id: Long, value: String?) { + if (id == requestId && System.currentTimeMillis() < deadline) { reply = value; (this as java.lang.Object).notifyAll() } + } + private fun serve(listener: ServerSocket) { + while (!listener.isClosed) { + try { + listener.accept().use { socket -> + synchronized(this) { if (server !== listener) return; client = socket } + socket.soTimeout = 5000 + socket.tcpNoDelay = true + val readDeadline = System.currentTimeMillis() + 5000 + val body = ByteArrayOutputStream() + var b = -1 + while (true) { + val remaining = readDeadline - System.currentTimeMillis() + if (remaining <= 0) throw Exception("Read timeout") + socket.soTimeout = remaining.toInt() + b = socket.getInputStream().read() + if (b == -1 || b == '\n'.code) break + if (body.size() >= 65536) throw Exception("Invalid frame") + body.write(b) + } + if (b != '\n'.code) return@use + val text = StandardCharsets.UTF_8.newDecoder().decode(ByteBuffer.wrap(body.toByteArray())).toString() + val response: String + synchronized(this) { + if (server !== listener) return + requestId = ++nextId + deadline = System.currentTimeMillis() + 5000 + reply = null + pending = JSONObject().put("id", requestId).put("expiresAt", deadline) + .put("body", text).toString() + notifyRequestReady(requestId) + while (reply == null && server === listener && System.currentTimeMillis() < deadline) { + (this as java.lang.Object).wait(Math.max(1L, deadline - System.currentTimeMillis())) + } + response = reply ?: "{\"ok\":false,\"error\":\"timeout\",\"message\":\"Faceclaw did not respond in time.\"}" + if (server === listener) { pending = null; reply = null; requestId = 0 } + } + socket.getOutputStream().write((response + "\n").toByteArray(StandardCharsets.UTF_8)) + } + } catch (ignored: Exception) { + // Never log request bodies: they contain credentials and possibly private text. + } finally { + synchronized(this) { if (server === listener) client = null } + } + } + } + fun randomSecret(): String { + val bytes = ByteArray(32); SecureRandom().nextBytes(bytes); return hex(bytes) + } + fun hash(value: String): String { + try { return hex(MessageDigest.getInstance("SHA-256").digest(value.toByteArray(StandardCharsets.UTF_8))) } + catch (e: Exception) { throw IllegalStateException("SHA-256 unavailable", e) } + } +} diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawResourceUsage.java b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawResourceUsage.java deleted file mode 100644 index d2358878..00000000 --- a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawResourceUsage.java +++ /dev/null @@ -1,68 +0,0 @@ -package com.faceclaw.app; - -import android.os.Debug; -import android.os.Process; -import android.os.SystemClock; - -import java.io.BufferedReader; -import java.io.FileReader; -import java.io.IOException; - -/** Lightweight process counters for the on-glasses resource-usage monitor. */ -public final class FaceclawResourceUsage { - private FaceclawResourceUsage() {} - - /** - * Returns elapsedMs, processCpuMs, rssKb, nativeHeapKb, javaHeapKb, - * javaHeapCommittedKb, and threadCount. Reading /proc/self/status avoids - * the considerably heavier full Debug.MemoryInfo/PSS collection once per - * second while still reporting the RSS that Android records at exit. - */ - public static long[] sample() { - long[] status = readProcessStatus(); - Runtime runtime = Runtime.getRuntime(); - long javaUsedBytes = runtime.totalMemory() - runtime.freeMemory(); - return new long[] { - SystemClock.elapsedRealtime(), - Process.getElapsedCpuTime(), - status[0], - Debug.getNativeHeapAllocatedSize() / 1024L, - javaUsedBytes / 1024L, - runtime.totalMemory() / 1024L, - status[1], - }; - } - - /** Returns {VmRSS in KiB, thread count}; unavailable values are zero. */ - private static long[] readProcessStatus() { - long rssKb = 0; - long threadCount = 0; - try (BufferedReader reader = new BufferedReader(new FileReader("/proc/self/status"))) { - String line; - while ((line = reader.readLine()) != null) { - if (line.startsWith("VmRSS:")) { - rssKb = firstNumber(line); - } else if (line.startsWith("Threads:")) { - threadCount = firstNumber(line); - } - if (rssKb > 0 && threadCount > 0) break; - } - } catch (IOException ignored) { - // The graph remains useful with the runtime heap counters alone. - } - return new long[] { rssKb, threadCount }; - } - - private static long firstNumber(String line) { - int start = 0; - while (start < line.length() && !Character.isDigit(line.charAt(start))) start++; - int end = start; - while (end < line.length() && Character.isDigit(line.charAt(end))) end++; - if (start == end) return 0; - try { - return Long.parseLong(line.substring(start, end)); - } catch (NumberFormatException ignored) { - return 0; - } - } -} diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawResourceUsage.kt b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawResourceUsage.kt new file mode 100644 index 00000000..c5a94b4c --- /dev/null +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawResourceUsage.kt @@ -0,0 +1,72 @@ +package com.faceclaw.app + +import android.os.Debug +import android.os.Process +import android.os.SystemClock + +import java.io.BufferedReader +import java.io.FileReader +import java.io.IOException + +/** Lightweight process counters for the on-glasses resource-usage monitor. */ +class FaceclawResourceUsage private constructor() { + companion object { + /** + * Returns elapsedMs, processCpuMs, rssKb, nativeHeapKb, javaHeapKb, + * javaHeapCommittedKb, and threadCount. Reading /proc/self/status avoids + * the considerably heavier full Debug.MemoryInfo/PSS collection once per + * second while still reporting the RSS that Android records at exit. + */ + @JvmStatic + fun sample(): LongArray { + val status = readProcessStatus() + val runtime = Runtime.getRuntime() + val javaUsedBytes = runtime.totalMemory() - runtime.freeMemory() + return longArrayOf( + SystemClock.elapsedRealtime(), + Process.getElapsedCpuTime(), + status[0], + Debug.getNativeHeapAllocatedSize() / 1024L, + javaUsedBytes / 1024L, + runtime.totalMemory() / 1024L, + status[1], + ) + } + + /** Returns {VmRSS in KiB, thread count}; unavailable values are zero. */ + private fun readProcessStatus(): LongArray { + var rssKb = 0L + var threadCount = 0L + try { + BufferedReader(FileReader("/proc/self/status")).use { reader -> + var line: String? + while (reader.readLine().also { line = it } != null) { + val current = line!! + if (current.startsWith("VmRSS:")) { + rssKb = firstNumber(current) + } else if (current.startsWith("Threads:")) { + threadCount = firstNumber(current) + } + if (rssKb > 0 && threadCount > 0) break + } + } + } catch (ignored: IOException) { + // The graph remains useful with the runtime heap counters alone. + } + return longArrayOf(rssKb, threadCount) + } + + private fun firstNumber(line: String): Long { + var start = 0 + while (start < line.length && !Character.isDigit(line[start])) start++ + var end = start + while (end < line.length && Character.isDigit(line[end])) end++ + if (start == end) return 0 + return try { + line.substring(start, end).toLong() + } catch (ignored: NumberFormatException) { + 0 + } + } + } +} diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawSettings.java b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawSettings.java deleted file mode 100644 index bec8c494..00000000 --- a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawSettings.java +++ /dev/null @@ -1,121 +0,0 @@ -package com.faceclaw.app; - -import android.content.Context; -import android.content.SharedPreferences; -import android.os.Handler; -import android.os.Looper; -import android.util.Log; - -import java.util.concurrent.CopyOnWriteArrayList; - -/** - * App settings store shared by every JS isolate (main thread and app - * workers). Values live in a dedicated SharedPreferences file, distinct from - * NativeScript's ApplicationSettings file, so this store owns its keys - * outright (the old TS-side settings were deliberately abandoned, not - * migrated). - * - * Change notifications: each isolate registers one listener from its own - * thread. The registering thread's Looper is captured, and notifications are - * posted through it so the JS callback always runs on the isolate's own - * thread (calling into an isolate from a foreign thread is not allowed). - * NativeScript worker threads run a message loop, so both the main thread - * and workers have a Looper; a listener registered from a Looper-less thread - * is accepted but never notified (it can still read fresh values on demand). - */ -public final class FaceclawSettings { - private static final String TAG = "FaceclawSettings"; - private static final String PREFS_NAME = "faceclaw_settings"; - private static volatile FaceclawSettings instance; - - private final SharedPreferences prefs; - private final CopyOnWriteArrayList listeners = new CopyOnWriteArrayList<>(); - - private static final class ListenerEntry { - final FaceclawSettingsListener listener; - final Handler handler; - - ListenerEntry(FaceclawSettingsListener listener, Handler handler) { - this.listener = listener; - this.handler = handler; - } - } - - private FaceclawSettings(Context context) { - this.prefs = context.getApplicationContext() - .getSharedPreferences(PREFS_NAME, Context.MODE_PRIVATE); - } - - /** Initialize (idempotent) and return the singleton. */ - public static FaceclawSettings getInstance(Context context) { - if (instance == null) { - synchronized (FaceclawSettings.class) { - if (instance == null) { - instance = new FaceclawSettings(context); - } - } - } - return instance; - } - - /** Return the singleton; the main isolate must have initialized it first. */ - public static FaceclawSettings getInstance() { - FaceclawSettings result = instance; - if (result == null) { - throw new IllegalStateException("FaceclawSettings not initialized; call getInstance(context) first"); - } - return result; - } - - public String getString(String key, String defaultValue) { - return prefs.getString(key, defaultValue); - } - - public void setString(String key, String value) { - prefs.edit().putString(key, value).apply(); - notifyChanged(key); - } - - public boolean getBoolean(String key, boolean defaultValue) { - return prefs.getBoolean(key, defaultValue); - } - - public void setBoolean(String key, boolean value) { - prefs.edit().putBoolean(key, value).apply(); - notifyChanged(key); - } - - /** - * Register a change listener. Must be called from the thread whose - * isolate owns the listener; that thread's Looper is captured for - * dispatch. - */ - public void registerListener(FaceclawSettingsListener listener) { - Looper looper = Looper.myLooper(); - if (looper == null) { - Log.w(TAG, "settings listener registered from a Looper-less thread; it will never be notified"); - } - listeners.add(new ListenerEntry(listener, looper != null ? new Handler(looper) : null)); - } - - public void unregisterListener(FaceclawSettingsListener listener) { - for (ListenerEntry entry : listeners) { - if (entry.listener == listener) { - listeners.remove(entry); - } - } - } - - private void notifyChanged(String key) { - for (ListenerEntry entry : listeners) { - if (entry.handler == null) continue; - entry.handler.post(() -> { - try { - entry.listener.onSettingChanged(key); - } catch (Exception e) { - Log.w(TAG, "settings listener failed for key " + key, e); - } - }); - } - } -} diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawSettings.kt b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawSettings.kt new file mode 100644 index 00000000..abd6d7c1 --- /dev/null +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawSettings.kt @@ -0,0 +1,112 @@ +package com.faceclaw.app + +import android.content.Context +import android.content.SharedPreferences +import android.os.Handler +import android.os.Looper +import android.util.Log + +import java.util.concurrent.CopyOnWriteArrayList + +/** + * App settings store shared by every JS isolate (main thread and app + * workers). Values live in a dedicated SharedPreferences file, distinct from + * NativeScript's ApplicationSettings file, so this store owns its keys + * outright (the old TS-side settings were deliberately abandoned, not + * migrated). + * + * Change notifications: each isolate registers one listener from its own + * thread. The registering thread's Looper is captured, and notifications are + * posted through it so the JS callback always runs on the isolate's own + * thread (calling into an isolate from a foreign thread is not allowed). + * NativeScript worker threads run a message loop, so both the main thread + * and workers have a Looper; a listener registered from a Looper-less thread + * is accepted but never notified (it can still read fresh values on demand). + */ +class FaceclawSettings private constructor(context: Context) { + private val prefs: SharedPreferences = context.applicationContext + .getSharedPreferences(PREFS_NAME, Context.MODE_PRIVATE) + private val listeners = CopyOnWriteArrayList() + + private class ListenerEntry(val listener: FaceclawSettingsListener, val handler: Handler?) + + companion object { + private const val TAG = "FaceclawSettings" + private const val PREFS_NAME = "faceclaw_settings" + @Volatile + private var instance: FaceclawSettings? = null + + /** Initialize (idempotent) and return the singleton. */ + @JvmStatic + fun getInstance(context: Context): FaceclawSettings { + if (instance == null) { + synchronized(FaceclawSettings::class.java) { + if (instance == null) { + instance = FaceclawSettings(context) + } + } + } + return instance!! + } + + /** Return the singleton; the main isolate must have initialized it first. */ + @JvmStatic + fun getInstance(): FaceclawSettings { + val result = instance + ?: throw IllegalStateException("FaceclawSettings not initialized; call getInstance(context) first") + return result + } + } + + fun getString(key: String?, defaultValue: String?): String? { + return prefs.getString(key, defaultValue) + } + + fun setString(key: String?, value: String?) { + prefs.edit().putString(key, value).apply() + notifyChanged(key) + } + + fun getBoolean(key: String?, defaultValue: Boolean): Boolean { + return prefs.getBoolean(key, defaultValue) + } + + fun setBoolean(key: String?, value: Boolean) { + prefs.edit().putBoolean(key, value).apply() + notifyChanged(key) + } + + /** + * Register a change listener. Must be called from the thread whose + * isolate owns the listener; that thread's Looper is captured for + * dispatch. + */ + fun registerListener(listener: FaceclawSettingsListener) { + val looper = Looper.myLooper() + if (looper == null) { + Log.w(TAG, "settings listener registered from a Looper-less thread; it will never be notified") + } + listeners.add(ListenerEntry(listener, if (looper != null) Handler(looper) else null)) + } + + fun unregisterListener(listener: FaceclawSettingsListener?) { + for (entry in listeners) { + if (entry.listener === listener) { + listeners.remove(entry) + } + } + } + + private fun notifyChanged(key: String?) { + for (entry in listeners) { + val handler = entry.handler ?: continue + handler.post { + try { + entry.listener.onSettingChanged(key) + } catch (e: Exception) { + Log.w(TAG, "settings listener failed for key $key", e) + } + } + } + } +} diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawSettingsPortReceiver.java b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawSettingsPortReceiver.java deleted file mode 100644 index 90becbef..00000000 --- a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawSettingsPortReceiver.java +++ /dev/null @@ -1,141 +0,0 @@ -package com.faceclaw.app; - -import android.content.BroadcastReceiver; -import android.content.Context; -import android.content.Intent; -import android.os.Handler; -import android.os.Looper; -import android.util.Log; -import android.util.Xml; - -import org.xmlpull.v1.XmlPullParser; - -import java.io.File; -import java.io.FileInputStream; -import java.io.FileOutputStream; -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; - -/** - * adb-triggered export/import of the faceclaw_settings SharedPreferences - * file, so configuration (API keys, device addresses, ...) can be moved off a - * release build, where run-as is unavailable. See scripts/pull_config.sh and - * scripts/push_config.sh, which use run-as on debuggable builds and fall back - * to this receiver otherwise. - * - * adb shell am broadcast -n com.faceclaw.app/.FaceclawSettingsPortReceiver \ - * -a com.faceclaw.app.SETTINGS_EXPORT - * adb pull /sdcard/Android/data/com.faceclaw.app/files/faceclaw-settings-export.xml - * - * adb push settings.xml /sdcard/Android/data/com.faceclaw.app/files/faceclaw-settings-import.xml - * adb shell am broadcast -n com.faceclaw.app/.FaceclawSettingsPortReceiver \ - * -a com.faceclaw.app.SETTINGS_IMPORT - * - * Security: the settings file contains API tokens, so the manifest guards - * this receiver with android.permission.DUMP — a development permission the - * adb shell holds but ordinary apps cannot obtain without adb's help, keeping - * the trigger reachable from adb only. The transfer file lives in - * getExternalFilesDir(), which other apps cannot read on Android 11+ (and - * which the scripts delete as soon as the transfer completes); on the - * Android 7-10 devices minSdk still admits, apps holding READ_EXTERNAL_STORAGE - * could read it during that window. - */ -public class FaceclawSettingsPortReceiver extends BroadcastReceiver { - private static final String TAG = "FaceclawSettingsPort"; - public static final String ACTION_EXPORT = "com.faceclaw.app.SETTINGS_EXPORT"; - public static final String ACTION_IMPORT = "com.faceclaw.app.SETTINGS_IMPORT"; - private static final String PREFS_FILE = "faceclaw_settings.xml"; - private static final String EXPORT_NAME = "faceclaw-settings-export.xml"; - private static final String IMPORT_NAME = "faceclaw-settings-import.xml"; - - @Override - public void onReceive(Context context, Intent intent) { - String action = intent == null ? null : intent.getAction(); - try { - if (ACTION_EXPORT.equals(action)) { - setResultData(exportSettings(context)); - } else if (ACTION_IMPORT.equals(action)) { - setResultData(importSettings(context)); - // Exit once the broadcast result has been delivered: this - // process may hold stale in-memory SharedPreferences that a - // later apply() would flush over the imported file, and the - // next launch must re-read from disk. - new Handler(Looper.getMainLooper()).postDelayed(() -> { - Log.i(TAG, "Exiting after settings import"); - System.exit(0); - }, 500); - } - } catch (Exception e) { - Log.e(TAG, "Settings " + action + " failed", e); - setResultData("error: " + e); - } - } - - private File prefsFile(Context context) { - return new File(context.getApplicationInfo().dataDir, "shared_prefs/" + PREFS_FILE); - } - - private File externalFile(Context context, String name) throws IOException { - File dir = context.getExternalFilesDir(null); - if (dir == null) { - throw new IOException("external files dir unavailable"); - } - return new File(dir, name); - } - - private String exportSettings(Context context) throws IOException { - File prefs = prefsFile(context); - if (!prefs.exists()) { - return "error: no settings file yet (" + prefs + ")"; - } - File out = externalFile(context, EXPORT_NAME); - copy(prefs, out); - return "exported: " + out; - } - - private String importSettings(Context context) throws Exception { - File staged = externalFile(context, IMPORT_NAME); - if (!staged.exists()) { - return "error: nothing staged at " + staged; - } - validatePrefsXml(staged); - File prefs = prefsFile(context); - prefs.getParentFile().mkdirs(); - if (prefs.exists()) { - copy(prefs, new File(prefs.getPath() + ".bak")); - } - copy(staged, prefs); - staged.delete(); - return "imported: " + prefs + " (previous settings in " + PREFS_FILE + ".bak)"; - } - - /** Reject files that are not a SharedPreferences document. */ - private void validatePrefsXml(File file) throws Exception { - try (InputStream in = new FileInputStream(file)) { - XmlPullParser parser = Xml.newPullParser(); - parser.setInput(in, null); - int event = parser.next(); - while (event != XmlPullParser.START_TAG && event != XmlPullParser.END_DOCUMENT) { - event = parser.next(); - } - if (event != XmlPullParser.START_TAG || !"map".equals(parser.getName())) { - throw new IOException("not a SharedPreferences document"); - } - // Walk the rest so malformed XML is caught before it replaces - // the live settings file. - while (parser.next() != XmlPullParser.END_DOCUMENT) { } - } - } - - private void copy(File from, File to) throws IOException { - try (InputStream in = new FileInputStream(from); - OutputStream out = new FileOutputStream(to)) { - byte[] buffer = new byte[8192]; - int n; - while ((n = in.read(buffer)) > 0) { - out.write(buffer, 0, n); - } - } - } -} diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawSettingsPortReceiver.kt b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawSettingsPortReceiver.kt new file mode 100644 index 00000000..788c93a1 --- /dev/null +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawSettingsPortReceiver.kt @@ -0,0 +1,146 @@ +package com.faceclaw.app + +import android.content.BroadcastReceiver +import android.content.Context +import android.content.Intent +import android.os.Handler +import android.os.Looper +import android.util.Log +import android.util.Xml + +import org.xmlpull.v1.XmlPullParser + +import java.io.File +import java.io.FileInputStream +import java.io.FileOutputStream +import java.io.IOException +import java.io.InputStream +import java.io.OutputStream + +/** + * adb-triggered export/import of the faceclaw_settings SharedPreferences + * file, so configuration (API keys, device addresses, ...) can be moved off a + * release build, where run-as is unavailable. See scripts/pull_config.sh and + * scripts/push_config.sh, which use run-as on debuggable builds and fall back + * to this receiver otherwise. + * + * adb shell am broadcast -n com.faceclaw.app/.FaceclawSettingsPortReceiver \ + * -a com.faceclaw.app.SETTINGS_EXPORT + * adb pull /sdcard/Android/data/com.faceclaw.app/files/faceclaw-settings-export.xml + * + * adb push settings.xml /sdcard/Android/data/com.faceclaw.app/files/faceclaw-settings-import.xml + * adb shell am broadcast -n com.faceclaw.app/.FaceclawSettingsPortReceiver \ + * -a com.faceclaw.app.SETTINGS_IMPORT + * + * Security: the settings file contains API tokens, so the manifest guards + * this receiver with android.permission.DUMP — a development permission the + * adb shell holds but ordinary apps cannot obtain without adb's help, keeping + * the trigger reachable from adb only. The transfer file lives in + * getExternalFilesDir(), which other apps cannot read on Android 11+ (and + * which the scripts delete as soon as the transfer completes); on the + * Android 7-10 devices minSdk still admits, apps holding READ_EXTERNAL_STORAGE + * could read it during that window. + */ +class FaceclawSettingsPortReceiver : BroadcastReceiver() { + companion object { + private const val TAG = "FaceclawSettingsPort" + const val ACTION_EXPORT = "com.faceclaw.app.SETTINGS_EXPORT" + const val ACTION_IMPORT = "com.faceclaw.app.SETTINGS_IMPORT" + private const val PREFS_FILE = "faceclaw_settings.xml" + private const val EXPORT_NAME = "faceclaw-settings-export.xml" + private const val IMPORT_NAME = "faceclaw-settings-import.xml" + } + + override fun onReceive(context: Context, intent: Intent?) { + val action = intent?.action + try { + if (ACTION_EXPORT == action) { + resultData = exportSettings(context) + } else if (ACTION_IMPORT == action) { + resultData = importSettings(context) + // Exit once the broadcast result has been delivered: this + // process may hold stale in-memory SharedPreferences that a + // later apply() would flush over the imported file, and the + // next launch must re-read from disk. + Handler(Looper.getMainLooper()).postDelayed({ + Log.i(TAG, "Exiting after settings import") + System.exit(0) + }, 500) + } + } catch (e: Exception) { + Log.e(TAG, "Settings $action failed", e) + resultData = "error: $e" + } + } + + private fun prefsFile(context: Context): File { + return File(context.applicationInfo.dataDir, "shared_prefs/$PREFS_FILE") + } + + @Throws(IOException::class) + private fun externalFile(context: Context, name: String): File { + val dir = context.getExternalFilesDir(null) + ?: throw IOException("external files dir unavailable") + return File(dir, name) + } + + @Throws(IOException::class) + private fun exportSettings(context: Context): String { + val prefs = prefsFile(context) + if (!prefs.exists()) { + return "error: no settings file yet ($prefs)" + } + val out = externalFile(context, EXPORT_NAME) + copy(prefs, out) + return "exported: $out" + } + + @Throws(Exception::class) + private fun importSettings(context: Context): String { + val staged = externalFile(context, IMPORT_NAME) + if (!staged.exists()) { + return "error: nothing staged at $staged" + } + validatePrefsXml(staged) + val prefs = prefsFile(context) + prefs.parentFile?.mkdirs() + if (prefs.exists()) { + copy(prefs, File(prefs.path + ".bak")) + } + copy(staged, prefs) + staged.delete() + return "imported: $prefs (previous settings in $PREFS_FILE.bak)" + } + + /** Reject files that are not a SharedPreferences document. */ + @Throws(Exception::class) + private fun validatePrefsXml(file: File) { + FileInputStream(file).use { input -> + val parser = Xml.newPullParser() + parser.setInput(input, null) + var event = parser.next() + while (event != XmlPullParser.START_TAG && event != XmlPullParser.END_DOCUMENT) { + event = parser.next() + } + if (event != XmlPullParser.START_TAG || "map" != parser.name) { + throw IOException("not a SharedPreferences document") + } + // Walk the rest so malformed XML is caught before it replaces + // the live settings file. + while (parser.next() != XmlPullParser.END_DOCUMENT) { } + } + } + + @Throws(IOException::class) + private fun copy(from: File, to: File) { + FileInputStream(from).use { input: InputStream -> + FileOutputStream(to).use { out: OutputStream -> + val buffer = ByteArray(8192) + var n: Int + while (input.read(buffer).also { n = it } > 0) { + out.write(buffer, 0, n) + } + } + } + } +} diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawSpeakerId.java b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawSpeakerId.java deleted file mode 100644 index c258ad91..00000000 --- a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawSpeakerId.java +++ /dev/null @@ -1,111 +0,0 @@ -package com.faceclaw.app; - -import android.util.Log; - -import com.k2fsa.sherpa.onnx.OnlineStream; -import com.k2fsa.sherpa.onnx.SpeakerEmbeddingExtractor; -import com.k2fsa.sherpa.onnx.SpeakerEmbeddingExtractorConfig; - -/** - * Speaker voice-print extraction over sherpa-onnx. One embedding per - * utterance; matching/enrollment happens on the TS side (cosine similarity - * against the stored speaker profiles). - */ -public class FaceclawSpeakerId { - private static final String TAG = "FaceclawSpeakerId"; - - private SpeakerEmbeddingExtractor extractor; - private final String modelPath; - - public FaceclawSpeakerId(String modelPath) { - this.modelPath = modelPath; - } - - /** Lazily loads the model; returns false when it cannot be loaded. */ - public synchronized boolean ensureLoaded() { - if (extractor != null) { - return true; - } - try { - SpeakerEmbeddingExtractorConfig config = SpeakerEmbeddingExtractorConfig.builder() - .setModel(modelPath) - .setNumThreads(1) - .setDebug(false) - .setProvider("cpu") - .build(); - extractor = new SpeakerEmbeddingExtractor(config); - return true; - } catch (Throwable t) { - Log.w(TAG, "could not load speaker model at " + modelPath, t); - extractor = null; - return false; - } - } - - public synchronized int embeddingDim() { - return extractor == null ? 0 : extractor.getDim(); - } - - /** - * Compute an L2-normalized voice-print for one utterance of 16 kHz mono - * S16LE PCM. Returns null when the model is unavailable or the utterance - * is too short to embed (under ~0.5 s). - */ - public synchronized float[] embed(byte[] pcm16le, int sampleRate) { - if (pcm16le == null || pcm16le.length < sampleRate) { - return null; - } - if (!ensureLoaded()) { - return null; - } - int count = pcm16le.length / 2; - float[] samples = new float[count]; - for (int i = 0; i < count; i++) { - short s = (short) ((pcm16le[i * 2] & 0xff) | (pcm16le[i * 2 + 1] << 8)); - samples[i] = s / 32768.0f; - } - OnlineStream stream = null; - try { - stream = extractor.createStream(); - stream.acceptWaveform(samples, sampleRate); - stream.inputFinished(); - if (!extractor.isReady(stream)) { - return null; - } - float[] embedding = extractor.compute(stream); - return l2Normalize(embedding); - } catch (Throwable t) { - Log.w(TAG, "embedding failed", t); - return null; - } finally { - if (stream != null) { - stream.release(); - } - } - } - - private static float[] l2Normalize(float[] v) { - if (v == null) { - return null; - } - double sum = 0; - for (float x : v) { - sum += (double) x * x; - } - double norm = Math.sqrt(sum); - if (norm <= 0) { - return v; - } - for (int i = 0; i < v.length; i++) { - v[i] = (float) (v[i] / norm); - } - return v; - } - - public synchronized void close() { - if (extractor != null) { - extractor.release(); - extractor = null; - } - } -} diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawSpeakerId.kt b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawSpeakerId.kt new file mode 100644 index 00000000..5cbf7bd2 --- /dev/null +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawSpeakerId.kt @@ -0,0 +1,116 @@ +package com.faceclaw.app + +import android.util.Log + +import com.k2fsa.sherpa.onnx.OnlineStream +import com.k2fsa.sherpa.onnx.SpeakerEmbeddingExtractor +import com.k2fsa.sherpa.onnx.SpeakerEmbeddingExtractorConfig + +/** + * Speaker voice-print extraction over sherpa-onnx. One embedding per + * utterance; matching/enrollment happens on the TS side (cosine similarity + * against the stored speaker profiles). + */ +class FaceclawSpeakerId(private val modelPath: String?) { + companion object { + private const val TAG = "FaceclawSpeakerId" + + @JvmStatic + private fun l2Normalize(v: FloatArray?): FloatArray? { + if (v == null) { + return null + } + var sum = 0.0 + for (x in v) { + sum += x.toDouble() * x + } + val norm = Math.sqrt(sum) + if (norm <= 0) { + return v + } + for (i in v.indices) { + v[i] = (v[i] / norm).toFloat() + } + return v + } + } + + private var extractor: SpeakerEmbeddingExtractor? = null + + /** Lazily loads the model; returns false when it cannot be loaded. */ + @Synchronized + fun ensureLoaded(): Boolean { + if (extractor != null) { + return true + } + try { + val config = SpeakerEmbeddingExtractorConfig.builder() + .setModel(modelPath) + .setNumThreads(1) + .setDebug(false) + .setProvider("cpu") + .build() + extractor = SpeakerEmbeddingExtractor(config) + return true + } catch (t: Throwable) { + Log.w(TAG, "could not load speaker model at $modelPath", t) + extractor = null + return false + } + } + + @Synchronized + fun embeddingDim(): Int { + val extractor = this.extractor + return if (extractor == null) 0 else extractor.dim + } + + /** + * Compute an L2-normalized voice-print for one utterance of 16 kHz mono + * S16LE PCM. Returns null when the model is unavailable or the utterance + * is too short to embed (under ~0.5 s). + */ + @Synchronized + fun embed(pcm16le: ByteArray?, sampleRate: Int): FloatArray? { + if (pcm16le == null || pcm16le.size < sampleRate) { + return null + } + if (!ensureLoaded()) { + return null + } + val extractor = this.extractor!! + val count = pcm16le.size / 2 + val samples = FloatArray(count) + for (i in 0 until count) { + val s = ((pcm16le[i * 2].toInt() and 0xff) or (pcm16le[i * 2 + 1].toInt() shl 8)).toShort() + samples[i] = s / 32768.0f + } + var stream: OnlineStream? = null + try { + stream = extractor.createStream() + stream.acceptWaveform(samples, sampleRate) + stream.inputFinished() + if (!extractor.isReady(stream)) { + return null + } + val embedding = extractor.compute(stream) + return l2Normalize(embedding) + } catch (t: Throwable) { + Log.w(TAG, "embedding failed", t) + return null + } finally { + if (stream != null) { + stream.release() + } + } + } + + @Synchronized + fun close() { + val extractor = this.extractor + if (extractor != null) { + extractor.release() + this.extractor = null + } + } +} diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawSseRequest.java b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawSseRequest.java deleted file mode 100644 index 5ec20fe9..00000000 --- a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawSseRequest.java +++ /dev/null @@ -1,145 +0,0 @@ -package com.faceclaw.app; - -import android.os.Handler; -import android.os.Looper; -import android.util.Log; - -import java.io.BufferedReader; -import java.io.IOException; -import java.util.concurrent.TimeUnit; - -import okhttp3.Call; -import okhttp3.Callback; -import okhttp3.MediaType; -import okhttp3.OkHttpClient; -import okhttp3.Request; -import okhttp3.RequestBody; -import okhttp3.Response; -import okhttp3.ResponseBody; - -/** - * Streaming HTTP POST for the TypeScript side, used for server-sent-events - * APIs (e.g. the Anthropic Messages API with stream=true). The response body - * is delivered line by line as it arrives, so the JS side can parse SSE - * events incrementally. Like FaceclawWebSocket, listener callbacks are posted - * to the Looper of the thread that constructed this object. - */ -public class FaceclawSseRequest { - private static final String TAG = "FaceclawSseRequest"; - private static final MediaType JSON = MediaType.get("application/json; charset=utf-8"); - private static volatile OkHttpClient sharedClient; - - private final Handler callbackHandler; - private final Call call; - private volatile boolean cancelled; - - /** - * Starts the request immediately. headers is a flat alternating - * [name, value, name, value, ...] array (okhttp Headers can't cross the - * JS bridge, and parallel arrays are easy to build with Array.create). - */ - public FaceclawSseRequest(String url, String jsonBody, String[] headers, final FaceclawSseListener listener) { - if (url == null || url.trim().isEmpty()) { - throw new IllegalArgumentException("url is required"); - } - if (listener == null) { - throw new IllegalArgumentException("listener is required"); - } - Looper looper = Looper.myLooper(); - callbackHandler = new Handler(looper != null ? looper : Looper.getMainLooper()); - Request.Builder builder = new Request.Builder() - .url(url.trim()) - .post(RequestBody.create(JSON, jsonBody == null ? "" : jsonBody)); - if (headers != null) { - for (int i = 0; i + 1 < headers.length; i += 2) { - if (headers[i] != null && !headers[i].isEmpty() && headers[i + 1] != null) { - builder.addHeader(headers[i], headers[i + 1]); - } - } - } - call = getClient().newCall(builder.build()); - call.enqueue(new Callback() { - @Override public void onFailure(Call call, IOException e) { - postFailure(listener, String.valueOf(e)); - } - - @Override public void onResponse(Call call, Response response) { - try (ResponseBody body = response.body()) { - if (!response.isSuccessful()) { - final int code = response.code(); - String errorBody = ""; - try { - errorBody = body == null ? "" : body.string(); - } catch (IOException e) { - Log.w(TAG, "error body read failed", e); - } - final String finalBody = errorBody; - post(() -> listener.onHttpError(code, finalBody)); - return; - } - if (body == null) { - post(listener::onComplete); - return; - } - BufferedReader reader = new BufferedReader(body.charStream()); - String line; - while ((line = reader.readLine()) != null) { - final String finalLine = line; - post(() -> listener.onLine(finalLine)); - } - post(listener::onComplete); - } catch (IOException e) { - postFailure(listener, String.valueOf(e)); - } - } - }); - } - - public void cancel() { - cancelled = true; - call.cancel(); - } - - private void post(Runnable runnable) { - if (cancelled) { - return; - } - callbackHandler.post(() -> { - if (cancelled) { - return; - } - try { - runnable.run(); - } catch (Throwable t) { - Log.w(TAG, "listener callback failed", t); - } - }); - } - - private void postFailure(FaceclawSseListener listener, String message) { - if (cancelled) { - return; - } - post(() -> listener.onFailure(message)); - } - - private static OkHttpClient getClient() { - OkHttpClient client = sharedClient; - if (client == null) { - synchronized (FaceclawSseRequest.class) { - if (sharedClient == null) { - // Streaming responses can pause between events (e.g. while - // the model thinks), so the read timeout is generous. The - // Anthropic API sends periodic ping events well within it. - sharedClient = new OkHttpClient.Builder() - .connectTimeout(15, TimeUnit.SECONDS) - .readTimeout(180, TimeUnit.SECONDS) - .addInterceptor(FaceclawHttp.userAgentInterceptor()) - .build(); - } - client = sharedClient; - } - } - return client; - } -} diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawSseRequest.kt b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawSseRequest.kt new file mode 100644 index 00000000..cf7a9514 --- /dev/null +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawSseRequest.kt @@ -0,0 +1,156 @@ +package com.faceclaw.app + +import android.os.Handler +import android.os.Looper +import android.util.Log + +import java.io.BufferedReader +import java.io.IOException +import java.util.concurrent.TimeUnit + +import okhttp3.Call +import okhttp3.Callback +import okhttp3.MediaType +import okhttp3.MediaType.Companion.toMediaType +import okhttp3.OkHttpClient +import okhttp3.Request +import okhttp3.RequestBody.Companion.toRequestBody +import okhttp3.Response + +/** + * Streaming HTTP POST for the TypeScript side, used for server-sent-events + * APIs (e.g. the Anthropic Messages API with stream=true). The response body + * is delivered line by line as it arrives, so the JS side can parse SSE + * events incrementally. Like FaceclawWebSocket, listener callbacks are posted + * to the Looper of the thread that constructed this object. + */ +class FaceclawSseRequest +/** + * Starts the request immediately. headers is a flat alternating + * [name, value, name, value, ...] array (okhttp Headers can't cross the + * JS bridge, and parallel arrays are easy to build with Array.create). + */ +constructor(url: String?, jsonBody: String?, headers: Array?, listener: FaceclawSseListener?) { + private val callbackHandler: Handler + private val call: Call + @Volatile + private var cancelled = false + + init { + if (url == null || url.trim().isEmpty()) { + throw IllegalArgumentException("url is required") + } + if (listener == null) { + throw IllegalArgumentException("listener is required") + } + val looper = Looper.myLooper() + callbackHandler = Handler(looper ?: Looper.getMainLooper()) + val builder = Request.Builder() + .url(url.trim()) + .post((jsonBody ?: "").toRequestBody(JSON)) + if (headers != null) { + var i = 0 + while (i + 1 < headers.size) { + val name = headers[i] + val value = headers[i + 1] + if (name != null && !name.isEmpty() && value != null) { + builder.addHeader(name, value) + } + i += 2 + } + } + call = getClient().newCall(builder.build()) + call.enqueue(object : Callback { + override fun onFailure(call: Call, e: IOException) { + postFailure(listener, e.toString()) + } + + override fun onResponse(call: Call, response: Response) { + try { + response.body.use { body -> + if (!response.isSuccessful) { + val code = response.code + var errorBody = "" + try { + errorBody = body?.string() ?: "" + } catch (e: IOException) { + Log.w(TAG, "error body read failed", e) + } + val finalBody = errorBody + post { listener.onHttpError(code, finalBody) } + return + } + if (body == null) { + post { listener.onComplete() } + return + } + val reader = BufferedReader(body.charStream()) + var line: String? + while (reader.readLine().also { line = it } != null) { + val finalLine = line + post { listener.onLine(finalLine) } + } + post { listener.onComplete() } + } + } catch (e: IOException) { + postFailure(listener, e.toString()) + } + } + }) + } + + fun cancel() { + cancelled = true + call.cancel() + } + + private fun post(runnable: Runnable) { + if (cancelled) { + return + } + callbackHandler.post { + if (cancelled) { + return@post + } + try { + runnable.run() + } catch (t: Throwable) { + Log.w(TAG, "listener callback failed", t) + } + } + } + + private fun postFailure(listener: FaceclawSseListener, message: String) { + if (cancelled) { + return + } + post { listener.onFailure(message) } + } + + companion object { + private const val TAG = "FaceclawSseRequest" + private val JSON: MediaType = "application/json; charset=utf-8".toMediaType() + @Volatile + private var sharedClient: OkHttpClient? = null + + private fun getClient(): OkHttpClient { + var client = sharedClient + if (client == null) { + synchronized(FaceclawSseRequest::class.java) { + if (sharedClient == null) { + // Streaming responses can pause between events (e.g. while + // the model thinks), so the read timeout is generous. The + // Anthropic API sends periodic ping events well within it. + sharedClient = OkHttpClient.Builder() + .connectTimeout(15, TimeUnit.SECONDS) + .readTimeout(180, TimeUnit.SECONDS) + .addInterceptor(FaceclawHttp.userAgentInterceptor()) + .build() + } + client = sharedClient + } + } + return client!! + } + } +} diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawSystemStatusIconProvider.java b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawSystemStatusIconProvider.java deleted file mode 100644 index 34209e16..00000000 --- a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawSystemStatusIconProvider.java +++ /dev/null @@ -1,193 +0,0 @@ -package com.faceclaw.app; - -import android.content.Context; -import android.net.ConnectivityManager; -import android.net.Network; -import android.net.NetworkCapabilities; -import android.net.wifi.WifiInfo; -import android.net.wifi.WifiManager; -import android.os.Build; -import android.telephony.SignalStrength; -import android.telephony.TelephonyManager; -import android.util.Log; - -import java.io.ByteArrayOutputStream; -import java.lang.reflect.Method; - -public final class FaceclawSystemStatusIconProvider { - private static final String TAG = "FaceclawStatusIcons"; - private static final int ICON_SIZE = 24; - - private FaceclawSystemStatusIconProvider() { - } - - public static byte[] getSystemStatusIconGrays(Context context, int iconSize) { - if (context == null) { - return new byte[0]; - } - Context appContext = context.getApplicationContext(); - int size = Math.max(1, Math.min(96, iconSize)); - ByteArrayOutputStream out = new ByteArrayOutputStream(size * size * 3); - - int wifiLevel = getWifiLevel(appContext); - if (wifiLevel >= 0) { - append(out, scaleIcon(drawWifiIcon(wifiLevel), size)); - } - - int cellLevel = getCellLevel(appContext); - if (cellLevel >= 0) { - append(out, scaleIcon(drawCellIcon(cellLevel), size)); - } - - if (isHotspotEnabled(appContext)) { - append(out, scaleIcon(drawHotspotIcon(), size)); - } - - return out.toByteArray(); - } - - private static int getWifiLevel(Context context) { - try { - ConnectivityManager connectivityManager = (ConnectivityManager) context.getSystemService(Context.CONNECTIVITY_SERVICE); - Network activeNetwork = connectivityManager == null ? null : connectivityManager.getActiveNetwork(); - NetworkCapabilities capabilities = activeNetwork == null ? null : connectivityManager.getNetworkCapabilities(activeNetwork); - if (capabilities == null || !capabilities.hasTransport(NetworkCapabilities.TRANSPORT_WIFI)) { - return -1; - } - WifiManager wifiManager = (WifiManager) context.getApplicationContext().getSystemService(Context.WIFI_SERVICE); - WifiInfo info = wifiManager == null ? null : wifiManager.getConnectionInfo(); - if (info == null || !isValidRssi(info.getRssi())) { - return -1; - } - return Math.max(0, Math.min(4, WifiManager.calculateSignalLevel(info.getRssi(), 5))); - } catch (Throwable t) { - Log.w(TAG, "failed to read Wi-Fi status", t); - return -1; - } - } - - private static int getCellLevel(Context context) { - if (Build.VERSION.SDK_INT < Build.VERSION_CODES.P) { - return -1; - } - try { - TelephonyManager telephonyManager = (TelephonyManager) context.getSystemService(Context.TELEPHONY_SERVICE); - SignalStrength signalStrength = telephonyManager == null ? null : telephonyManager.getSignalStrength(); - if (signalStrength == null) { - return -1; - } - return Math.max(0, Math.min(4, signalStrength.getLevel())); - } catch (SecurityException e) { - // Most Android versions require READ_PHONE_STATE for this. We skip it - // rather than prompting for a broad phone permission just for an icon. - return -1; - } catch (Throwable t) { - Log.w(TAG, "failed to read cellular status", t); - return -1; - } - } - - private static boolean isHotspotEnabled(Context context) { - try { - WifiManager wifiManager = (WifiManager) context.getApplicationContext().getSystemService(Context.WIFI_SERVICE); - if (wifiManager == null) { - return false; - } - Method method = wifiManager.getClass().getDeclaredMethod("getWifiApState"); - method.setAccessible(true); - Object result = method.invoke(wifiManager); - int state = result instanceof Integer ? (Integer) result : -1; - return state == 13 || state == 12; // WIFI_AP_STATE_ENABLED / ENABLING - } catch (Throwable ignored) { - return false; - } - } - - private static boolean isValidRssi(int rssi) { - return rssi > -127 && rssi < 0; - } - - private static byte[] drawWifiIcon(int level) { - byte[] icon = new byte[ICON_SIZE * ICON_SIZE]; - int cx = 12; - int cy = 18; - fillRect(icon, cx - 1, cy - 1, 3, 3, 230); - int[] radii = new int[] {5, 9, 13, 17}; - for (int index = 0; index < radii.length; index++) { - if (level <= index) { - continue; - } - drawWifiArc(icon, cx, cy, radii[index], 210); - } - return icon; - } - - private static void drawWifiArc(byte[] icon, int cx, int cy, int radius, int value) { - int inner = (radius - 1) * (radius - 1); - int outer = (radius + 1) * (radius + 1); - for (int y = 0; y < ICON_SIZE; y++) { - for (int x = 0; x < ICON_SIZE; x++) { - int dx = x - cx; - int dy = y - cy; - int dist = dx * dx + dy * dy; - if (dy <= 0 && dist >= inner && dist <= outer && Math.abs(dx) <= radius) { - setPixel(icon, x, y, value); - } - } - } - } - - private static byte[] drawCellIcon(int level) { - byte[] icon = new byte[ICON_SIZE * ICON_SIZE]; - for (int bar = 0; bar < 4; bar++) { - int x = 5 + bar * 4; - int height = 5 + bar * 4; - int value = level > bar ? 220 : 55; - fillRect(icon, x, 20 - height, 3, height, value); - } - return icon; - } - - private static byte[] drawHotspotIcon() { - byte[] icon = new byte[ICON_SIZE * ICON_SIZE]; - fillRect(icon, 11, 17, 3, 3, 230); - drawWifiArc(icon, 12, 19, 7, 210); - drawWifiArc(icon, 12, 19, 12, 210); - fillRect(icon, 6, 5, 12, 2, 170); - return icon; - } - - private static byte[] scaleIcon(byte[] source, int size) { - if (size == ICON_SIZE) { - return source; - } - byte[] scaled = new byte[size * size]; - for (int y = 0; y < size; y++) { - int sy = Math.min(ICON_SIZE - 1, (y * ICON_SIZE) / size); - for (int x = 0; x < size; x++) { - int sx = Math.min(ICON_SIZE - 1, (x * ICON_SIZE) / size); - scaled[y * size + x] = source[sy * ICON_SIZE + sx]; - } - } - return scaled; - } - - private static void fillRect(byte[] icon, int x, int y, int width, int height, int value) { - for (int row = y; row < y + height; row++) { - for (int col = x; col < x + width; col++) { - setPixel(icon, col, row, value); - } - } - } - - private static void setPixel(byte[] icon, int x, int y, int value) { - if (x < 0 || y < 0 || x >= ICON_SIZE || y >= ICON_SIZE) { - return; - } - icon[y * ICON_SIZE + x] = (byte) Math.max(0, Math.min(255, value)); - } - - private static void append(ByteArrayOutputStream out, byte[] bytes) { - out.write(bytes, 0, bytes.length); - } -} diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawSystemStatusIconProvider.kt b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawSystemStatusIconProvider.kt new file mode 100644 index 00000000..7c8be12c --- /dev/null +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawSystemStatusIconProvider.kt @@ -0,0 +1,193 @@ +package com.faceclaw.app + +import android.content.Context +import android.net.ConnectivityManager +import android.net.Network +import android.net.NetworkCapabilities +import android.net.wifi.WifiInfo +import android.net.wifi.WifiManager +import android.os.Build +import android.telephony.SignalStrength +import android.telephony.TelephonyManager +import android.util.Log + +import java.io.ByteArrayOutputStream +import java.lang.reflect.Method + +class FaceclawSystemStatusIconProvider private constructor() { + companion object { + private const val TAG = "FaceclawStatusIcons" + private const val ICON_SIZE = 24 + + @JvmStatic + fun getSystemStatusIconGrays(context: Context?, iconSize: Int): ByteArray { + if (context == null) { + return ByteArray(0) + } + val appContext = context.applicationContext + val size = Math.max(1, Math.min(96, iconSize)) + val out = ByteArrayOutputStream(size * size * 3) + + val wifiLevel = getWifiLevel(appContext) + if (wifiLevel >= 0) { + append(out, scaleIcon(drawWifiIcon(wifiLevel), size)) + } + + val cellLevel = getCellLevel(appContext) + if (cellLevel >= 0) { + append(out, scaleIcon(drawCellIcon(cellLevel), size)) + } + + if (isHotspotEnabled(appContext)) { + append(out, scaleIcon(drawHotspotIcon(), size)) + } + + return out.toByteArray() + } + + private fun getWifiLevel(context: Context): Int { + try { + val connectivityManager = context.getSystemService(Context.CONNECTIVITY_SERVICE) as ConnectivityManager? + val activeNetwork: Network? = if (connectivityManager == null) null else connectivityManager.activeNetwork + val capabilities: NetworkCapabilities? = if (activeNetwork == null) null else connectivityManager!!.getNetworkCapabilities(activeNetwork) + if (capabilities == null || !capabilities.hasTransport(NetworkCapabilities.TRANSPORT_WIFI)) { + return -1 + } + val wifiManager = context.applicationContext.getSystemService(Context.WIFI_SERVICE) as WifiManager? + val info: WifiInfo? = if (wifiManager == null) null else wifiManager.connectionInfo + if (info == null || !isValidRssi(info.rssi)) { + return -1 + } + return Math.max(0, Math.min(4, WifiManager.calculateSignalLevel(info.rssi, 5))) + } catch (t: Throwable) { + Log.w(TAG, "failed to read Wi-Fi status", t) + return -1 + } + } + + private fun getCellLevel(context: Context): Int { + if (Build.VERSION.SDK_INT < Build.VERSION_CODES.P) { + return -1 + } + try { + val telephonyManager = context.getSystemService(Context.TELEPHONY_SERVICE) as TelephonyManager? + val signalStrength: SignalStrength? = if (telephonyManager == null) null else telephonyManager.signalStrength + if (signalStrength == null) { + return -1 + } + return Math.max(0, Math.min(4, signalStrength.level)) + } catch (e: SecurityException) { + // Most Android versions require READ_PHONE_STATE for this. We skip it + // rather than prompting for a broad phone permission just for an icon. + return -1 + } catch (t: Throwable) { + Log.w(TAG, "failed to read cellular status", t) + return -1 + } + } + + private fun isHotspotEnabled(context: Context): Boolean { + try { + val wifiManager = context.applicationContext.getSystemService(Context.WIFI_SERVICE) as WifiManager? + if (wifiManager == null) { + return false + } + val method: Method = wifiManager.javaClass.getDeclaredMethod("getWifiApState") + method.isAccessible = true + val result: Any? = method.invoke(wifiManager) + val state = if (result is Int) result else -1 + return state == 13 || state == 12 // WIFI_AP_STATE_ENABLED / ENABLING + } catch (ignored: Throwable) { + return false + } + } + + private fun isValidRssi(rssi: Int): Boolean { + return rssi > -127 && rssi < 0 + } + + private fun drawWifiIcon(level: Int): ByteArray { + val icon = ByteArray(ICON_SIZE * ICON_SIZE) + val cx = 12 + val cy = 18 + fillRect(icon, cx - 1, cy - 1, 3, 3, 230) + val radii = intArrayOf(5, 9, 13, 17) + for (index in radii.indices) { + if (level <= index) { + continue + } + drawWifiArc(icon, cx, cy, radii[index], 210) + } + return icon + } + + private fun drawWifiArc(icon: ByteArray, cx: Int, cy: Int, radius: Int, value: Int) { + val inner = (radius - 1) * (radius - 1) + val outer = (radius + 1) * (radius + 1) + for (y in 0 until ICON_SIZE) { + for (x in 0 until ICON_SIZE) { + val dx = x - cx + val dy = y - cy + val dist = dx * dx + dy * dy + if (dy <= 0 && dist >= inner && dist <= outer && Math.abs(dx) <= radius) { + setPixel(icon, x, y, value) + } + } + } + } + + private fun drawCellIcon(level: Int): ByteArray { + val icon = ByteArray(ICON_SIZE * ICON_SIZE) + for (bar in 0 until 4) { + val x = 5 + bar * 4 + val height = 5 + bar * 4 + val value = if (level > bar) 220 else 55 + fillRect(icon, x, 20 - height, 3, height, value) + } + return icon + } + + private fun drawHotspotIcon(): ByteArray { + val icon = ByteArray(ICON_SIZE * ICON_SIZE) + fillRect(icon, 11, 17, 3, 3, 230) + drawWifiArc(icon, 12, 19, 7, 210) + drawWifiArc(icon, 12, 19, 12, 210) + fillRect(icon, 6, 5, 12, 2, 170) + return icon + } + + private fun scaleIcon(source: ByteArray, size: Int): ByteArray { + if (size == ICON_SIZE) { + return source + } + val scaled = ByteArray(size * size) + for (y in 0 until size) { + val sy = Math.min(ICON_SIZE - 1, (y * ICON_SIZE) / size) + for (x in 0 until size) { + val sx = Math.min(ICON_SIZE - 1, (x * ICON_SIZE) / size) + scaled[y * size + x] = source[sy * ICON_SIZE + sx] + } + } + return scaled + } + + private fun fillRect(icon: ByteArray, x: Int, y: Int, width: Int, height: Int, value: Int) { + for (row in y until y + height) { + for (col in x until x + width) { + setPixel(icon, col, row, value) + } + } + } + + private fun setPixel(icon: ByteArray, x: Int, y: Int, value: Int) { + if (x < 0 || y < 0 || x >= ICON_SIZE || y >= ICON_SIZE) { + return + } + icon[y * ICON_SIZE + x] = Math.max(0, Math.min(255, value)).toByte() + } + + private fun append(out: ByteArrayOutputStream, bytes: ByteArray) { + out.write(bytes, 0, bytes.size) + } + } +} diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawVoiceController.java b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawVoiceController.java deleted file mode 100644 index de09ae61..00000000 --- a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawVoiceController.java +++ /dev/null @@ -1,1162 +0,0 @@ -package com.faceclaw.app; - -import android.content.Context; -import android.os.Handler; -import android.os.Looper; -import android.os.SystemClock; -import android.util.Log; - -import com.k2fsa.sherpa.onnx.FeatureConfig; -import com.k2fsa.sherpa.onnx.OfflineModelConfig; -import com.k2fsa.sherpa.onnx.OfflineMoonshineModelConfig; -import com.k2fsa.sherpa.onnx.OfflineRecognizer; -import com.k2fsa.sherpa.onnx.OfflineRecognizerConfig; -import com.k2fsa.sherpa.onnx.OfflineRecognizerResult; -import com.k2fsa.sherpa.onnx.OfflineStream; -import com.k2fsa.sherpa.onnx.OfflineWhisperModelConfig; - -import java.io.File; -import java.nio.charset.StandardCharsets; -import java.util.ArrayDeque; -import java.util.Arrays; - -public class FaceclawVoiceController { - private static final String TAG = "FaceclawVoice"; - private static final int SAMPLE_RATE = 16000; - private static final int FEATURE_DIM = 80; - private static final int MAX_AUDIO_QUEUE_PACKETS = 80; - private static final int EXPECTED_PACKET_INTERVAL_MS = 50; - private static final int LATE_PACKET_INTERVAL_MS = 90; - private static final int STATS_INTERVAL_MS = 5_000; - // Push-to-talk utterance boundaries come from the button. We re-decode the - // current audio segment in full for each live partial and emit the complete - // utterance text (REPLACE, never a delta). The sherpa Moonshine v2 decoder - // used here fails once a single input grows past roughly 9.1 seconds, so - // longer utterances are committed in model-safe segments. - private static final int TRANSCRIPT_DECODE_INTERVAL_MS = 700; - private static final int TRANSCRIPT_MIN_SAMPLES = SAMPLE_RATE / 3; - private static final int TRANSCRIPT_SEGMENT_MAX_SAMPLES = SAMPLE_RATE * 8; - // When a segment fills, cut at the quietest window within the last - // TRANSCRIPT_CUT_SEARCH_SAMPLES rather than mid-word at the 8s mark; the - // audio after the cut carries over into the next segment. - private static final int TRANSCRIPT_CUT_SEARCH_SAMPLES = SAMPLE_RATE * 2; - private static final int TRANSCRIPT_CUT_WINDOW_SAMPLES = SAMPLE_RATE * 30 / 1000; - // Glasses-mic PCM peaks around 0.1 full scale, and at that level the - // quantized Moonshine model often returns empty or garbled text. Boost - // each decode window toward this peak, with a gain cap so near-silent - // buffers aren't amplified into pure noise. - private static final float TRANSCRIPT_NORMALIZE_TARGET_PEAK = 0.9f; - private static final float TRANSCRIPT_NORMALIZE_MAX_GAIN = 30f; - private static final int TRANSCRIPT_LOG_PREVIEW_CHARS = 80; - // Model directory shared with the TS-side download flow (asr-model.ts), - // which fetches the Moonshine files here on demand; they are no longer - // bundled in the APK. - private static final String ASR_ROOT = "faceclaw-voice-asr"; - private static final String ASR_MODEL_DIR = "sherpa-onnx-moonshine-base-en-quantized-2026-02-27"; - // Model files for the retired on-phone wake-word spotter, copied to - // filesDir by earlier releases; deleted on sight to reclaim the space. - // (The wakeword is now detected by the glasses firmware itself.) - private static final String LEGACY_KWS_ROOT = "faceclaw-voice"; - private static final String[] ASR_MODEL_FILES = { - "encoder_model.ort", - "decoder_model_merged.ort", - "tokens.txt" - }; - // Second on-device model: sherpa-onnx's offline Whisper backend (base.en, - // int8-quantized -- see the model-choice note in asr-model.ts). Directory - // shared with the TS-side download flow, same convention as ASR_MODEL_DIR. - // - // On every call, sherpa-onnx (offline-recognizer-whisper-impl.h, v1.13.0) - // runs Whisper's encoder over the whole buffer plus 1000 frames (~10s) of - // zero tail padding, capped at 30s. Re-decoding on Moonshine's - // TRANSCRIPT_DECODE_INTERVAL_MS live-partial cadence would repeat that - // encode, growing with the utterance, roughly 1.4x/second while the user is - // still speaking, so onboardModelKind gates that loop off for Whisper -- - // see processRecognizer(). Whisper is also known to hallucinate text on - // near-silent input; recognizeTranscriptSegment() gates that too. - private static final String ASR_WHISPER_MODEL_DIR = "sherpa-onnx-whisper-base-en-int8"; - private static final String[] ASR_WHISPER_MODEL_FILES = { - "base.en-encoder.int8.onnx", - "base.en-decoder.int8.onnx", - "base.en-tokens.txt" - }; - // Below this peak amplitude (pre-normalization, of a full-scale +/-1.0f - // buffer) a segment is treated as silence and never reaches the Whisper - // recognizer at all, rather than risking a hallucinated non-answer. Picked - // conservatively low (well under typical mic noise floor already seen in - // this pipeline's normalization target) -- UNTESTED on real hardware, tune - // against real glasses captures rather than trusting this number. - private static final float WHISPER_SILENCE_PEAK_THRESHOLD = 0.01f; - - private enum VoiceInputMode { - ONBOARD, // on-phone transcription (Moonshine or Whisper; see onboardModelKind) - CLOUD // decode locally, emit PCM for a cloud recognizer on the TS side - } - - /** Which on-device model ONBOARD mode uses. Set via setOnboardModelKind() before start(). */ - private enum OnboardModelKind { - MOONSHINE, - WHISPER - } - - private final Context appContext; - private final Handler mainHandler = new Handler(Looper.getMainLooper()); - private final Object lock = new Object(); - private final Object audioQueueLock = new Object(); - private final ArrayDeque audioQueue = new ArrayDeque<>(); - private volatile FaceclawVoiceControllerListener listener; - private volatile FaceclawBleCommunicator communicator; - private Thread workerThread; - private volatile boolean started; - // Set once the worker has the glasses mic enabled for this session. - // Read and written under `lock`, so it flips with `started` atomically. - private boolean audioStarted; - // Capture from the phone's own microphone instead of the G2 over BLE - // (preview-only mode, where no glasses are connected). Latched into - // activePhoneMic at start() (under `lock`) so a mid-session setter call - // can't switch pipelines underneath the worker. - private volatile boolean usePhoneMic; - private boolean activePhoneMic; - private VoiceInputMode mode = VoiceInputMode.CLOUD; - private volatile OnboardModelKind onboardModelKind = OnboardModelKind.MOONSHINE; - private OfflineRecognizer recognizer; - private FaceclawLc3Decoder lc3Decoder; - private final float[] transcriptSamples = new float[TRANSCRIPT_SEGMENT_MAX_SAMPLES]; - private int transcriptSampleCount; - private long committedTranscriptSampleCount; - private String committedTranscript = ""; - private String currentSegmentTranscript = ""; - private long lastTranscriptDecodeAtMs; - private String lastTranscript = ""; - private volatile boolean saveRecordings; - private volatile boolean endpointing; - private final VoiceEndpointDetector endpointDetector = new VoiceEndpointDetector(); - private java.io.ByteArrayOutputStream recordingPcm; - // Speaker verification against the enrolled wearer voice-print ("my voice - // only" command gating). Configured before start(); the utterance PCM is - // buffered (capped) and verified once at session end. - private static final int VERIFY_MAX_SAMPLES = SAMPLE_RATE * 10; - private static final int VERIFY_MIN_SAMPLES = SAMPLE_RATE; - private volatile String verifySpeakerModelPath; - private volatile float[] verifyWearerEmbedding; - private volatile float verifyThreshold = 0.8f; - private short[] verifyBuffer; - private int verifyCount; - // Global mic processing (Microphones app config): spectral noise - // suppression and firmware-DoA beam gating, applied to every capture - // session that opts in (assistant push-to-talk, Transcribe, hands-free). - // The raw tap opts out — raw means raw, and the Microphones session does - // its own beam-compensated processing on that path. - private volatile boolean suppressionEnabled; - private volatile boolean beamFilterEnabled; - private volatile int beamCenterDeg; - private volatile int beamHalfWidthDeg = 180; - private FaceclawNoiseSuppressor suppressor; - private long queuedPackets; - private long queueDroppedPackets; - private long decodedSamples; - private long latePackets; - private long wrongArmPackets; - private long lastPacketArrivalMs; - private long maxInterPacketMs; - private long lastStatsAtMs; - - public FaceclawVoiceController(Context context) { - this.appContext = context.getApplicationContext(); - } - - public void setListener(FaceclawVoiceControllerListener listener) { - this.listener = listener; - } - - public void setCommunicator(FaceclawBleCommunicator communicator) { - this.communicator = communicator; - } - - /** Source the next capture from the phone microphone (no glasses paired). */ - public void setUsePhoneMic(boolean usePhoneMic) { - this.usePhoneMic = usePhoneMic; - } - - /** When true, the decoded mic PCM for each session is saved as a WAV. */ - public void setSaveRecordings(boolean saveRecordings) { - this.saveRecordings = saveRecordings; - } - - /** - * Which on-device model {@link #start}("onboard") should load: "whisper" - * selects the second on-device model (sherpa-onnx offline Whisper); any - * other value (including null/absent) keeps the existing Moonshine model, - * so callers that never call this see unchanged behavior. Must be set - * before {@link #start}; has no effect in CLOUD mode. - */ - public void setOnboardModelKind(String kind) { - this.onboardModelKind = "whisper".equals(kind) ? OnboardModelKind.WHISPER : OnboardModelKind.MOONSHINE; - } - - /** - * When true, watch the decoded PCM and fire {@code onSpeechEnd} once the - * speaker stops. Used by hands-free ("Hey Even") capture, which has no - * button release to end the utterance. Must be set before {@link #start}. - */ - public void setEndpointing(boolean endpointing) { - this.endpointing = endpointing; - } - - /** - * Verify this session's speaker against the enrolled wearer voice-print - * and report the result via onSpeakerVerified just before the final - * transcript. Must be set before {@link #start}; pass a null model path - * to disable. - */ - public void setSpeakerVerification(String speakerModelPath, float[] wearerEmbedding, float threshold) { - this.verifySpeakerModelPath = speakerModelPath; - this.verifyWearerEmbedding = wearerEmbedding; - this.verifyThreshold = threshold > 0 ? threshold : 0.8f; - } - - public void clearSpeakerVerification() { - this.verifySpeakerModelPath = null; - this.verifyWearerEmbedding = null; - } - - /** Spectral noise suppression on the decoded stream. Safe to flip mid-run. */ - public void setNoiseSuppression(boolean enabled) { - this.suppressionEnabled = enabled; - } - - /** - * Direction gating from the Sonic Radar beam: packets whose firmware - * direction-of-arrival falls outside centerDeg ± halfWidthDeg (device - * frame, 0 = straight ahead, positive right) are dropped before any - * consumer sees them. Safe to update mid-run. - */ - public void setBeamFilter(boolean enabled, int centerDeg, int halfWidthDeg) { - this.beamFilterEnabled = enabled; - this.beamCenterDeg = centerDeg; - this.beamHalfWidthDeg = Math.max(5, Math.min(180, halfWidthDeg)); - } - - private boolean withinBeam(int angleDegrees) { - int delta = angleDegrees - beamCenterDeg; - while (delta > 180) delta -= 360; - while (delta < -180) delta += 360; - return Math.abs(delta) <= beamHalfWidthDeg; - } - - private void applySuppression(short[] pcm, int count) { - try { - if (suppressor == null) { - suppressor = new FaceclawNoiseSuppressor(SAMPLE_RATE); - } - byte[] le = new byte[count * 2]; - for (int i = 0; i < count; i++) { - le[i * 2] = (byte) (pcm[i] & 0xff); - le[i * 2 + 1] = (byte) ((pcm[i] >> 8) & 0xff); - } - byte[] cleaned = suppressor.process(le); - int cleanedCount = Math.min(count, cleaned.length / 2); - for (int i = 0; i < cleanedCount; i++) { - pcm[i] = (short) ((cleaned[i * 2] & 0xff) | (cleaned[i * 2 + 1] << 8)); - } - } catch (Throwable t) { - Log.w(TAG, "noise suppression failed; passing audio through", t); - suppressionEnabled = false; - } - } - - public void start(String requestedMode) { - start(requestedMode, 0); - } - - public void start(String requestedMode, int captureId) { - synchronized (lock) { - if (started) { - emitStatus("Voice control is already listening."); - emitStopped(captureId); - return; - } - if (!usePhoneMic && (communicator == null || !communicator.isSessionReady())) { - emitStatus("Voice control needs an active G2 connection."); - emitStopped(captureId); - return; - } - mode = parseMode(requestedMode); - activePhoneMic = usePhoneMic; - started = true; - audioStarted = false; - workerThread = new Thread(() -> runLoop(captureId), "FaceclawVoiceController"); - workerThread.start(); - } - } - - /** - * Whether mic audio is actually flowing. {@link #start} only records - * intent: the enable lives in the glasses' EvenHub session, so a transport - * drop or a session suspend can leave this controller started with a - * worker that will never see another packet. Anything deciding whether to - * (re)start capture must ask this rather than assume its own bookkeeping. - */ - public boolean isCapturing() { - boolean audioUp; - boolean phoneMic; - synchronized (lock) { - if (!started) { - return false; - } - audioUp = audioStarted; - phoneMic = activePhoneMic; - } - if (!audioUp) { - // The worker is still bringing the mic up; report it as running so - // a concurrent request shares it instead of restarting it. - return true; - } - if (phoneMic) { - // AudioRecord has no session to lose the enable to; it runs until stop(). - return true; - } - FaceclawBleCommunicator currentCommunicator = communicator; - return currentCommunicator != null && currentCommunicator.isAudioCaptureActive(); - } - - public void stop() { - Thread threadToJoin; - synchronized (lock) { - if (!started) { - return; - } - started = false; - threadToJoin = workerThread; - } - stopG2Audio(); - synchronized (audioQueueLock) { - audioQueueLock.notifyAll(); - } - if (threadToJoin != null) { - threadToJoin.interrupt(); - if (Thread.currentThread() != threadToJoin) { - try { - threadToJoin.join(1500); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - } - } - } - } - - public void close() { - stop(); - } - - private VoiceInputMode parseMode(String requestedMode) { - if ("cloud".equals(requestedMode)) { - return VoiceInputMode.CLOUD; - } - return VoiceInputMode.ONBOARD; - } - - private void runLoop(int captureId) { - try { - deleteLegacyKwsFiles(); - VoiceInputMode currentMode = mode; - OnboardModelKind currentOnboardKind = onboardModelKind; - if (currentMode == VoiceInputMode.ONBOARD) { - File modelDir = findAsrModelDir(currentOnboardKind); - if (modelDir == null) { - emitStatus("Voice model not downloaded (see Settings > Voice)."); - return; - } - emitStatus("Loading transcription model..."); - recognizer = new OfflineRecognizer(buildRecognizerConfig(modelDir, currentOnboardKind)); - resetTranscriptState(); - lastTranscript = ""; - } - endpointDetector.reset(); - if (suppressor != null) { - suppressor.reset(); - } - recordingPcm = saveRecordings ? new java.io.ByteArrayOutputStream(SAMPLE_RATE * 2 * 4) : null; - boolean verifying = verifySpeakerModelPath != null && verifyWearerEmbedding != null; - verifyBuffer = verifying ? new short[VERIFY_MAX_SAMPLES] : null; - verifyCount = 0; - if (activePhoneMic) { - android.media.AudioRecord record = openPhoneMic(); - if (record == null) { - emitStatus("Could not start the phone microphone."); - return; - } - synchronized (lock) { - audioStarted = true; - } - emitStatus(currentMode == VoiceInputMode.CLOUD - ? "Listening (cloud)..." - : "Listening..."); - try { - processPhoneAudio(record); - } finally { - try { - record.stop(); - } catch (Throwable ignored) { - // Already stopped or never recording; release below either way. - } - record.release(); - } - } else { - lc3Decoder = new FaceclawLc3Decoder(); - if (!startG2Audio()) { - emitStatus("Could not start G2 microphone input."); - return; - } - synchronized (lock) { - audioStarted = true; - } - emitStatus(currentMode == VoiceInputMode.CLOUD - ? "Listening (cloud)..." - : "Listening..."); - processG2Audio(); - } - // Verification result must precede the final transcript so the - // TS bridge can suppress a non-wearer command before it is acted - // on (the callbacks are posted in order to the main handler). - runSpeakerVerification(); - // Button released / stop requested: emit one final full-utterance - // transcript so the UI can freeze it. - if (currentMode == VoiceInputMode.ONBOARD) { - decodeTranscript(true); - } - } catch (Throwable error) { - Log.e(TAG, "Voice control failed", error); - emitStatus("Voice control failed: " + error.getMessage()); - } finally { - stopG2Audio(); - writeRecordingIfAny(); - releaseSherpa(); - releaseLc3(); - synchronized (lock) { - started = false; - audioStarted = false; - workerThread = null; - } - emitStopped(captureId); - } - } - - private void appendRecording(short[] pcm, int count) { - java.io.ByteArrayOutputStream out = recordingPcm; - if (out == null) { - return; - } - for (int i = 0; i < count; i++) { - short s = pcm[i]; - out.write(s & 0xff); - out.write((s >> 8) & 0xff); - } - } - - /** Save the session's decoded mic PCM as a 16 kHz mono 16-bit WAV. */ - private void writeRecordingIfAny() { - java.io.ByteArrayOutputStream out = recordingPcm; - recordingPcm = null; - if (out == null || out.size() == 0) { - return; - } - try { - byte[] pcmBytes = out.toByteArray(); - java.io.File dir = new java.io.File(appContext.getExternalFilesDir(null), "voice-recordings"); - if (!dir.exists() && !dir.mkdirs()) { - Log.w(TAG, "could not create voice-recordings dir"); - return; - } - String stamp = new java.text.SimpleDateFormat("yyyyMMdd-HHmmss-SSS", java.util.Locale.US) - .format(new java.util.Date()); - java.io.File file = new java.io.File(dir, "voice-" + stamp + ".wav"); - try (java.io.FileOutputStream fos = new java.io.FileOutputStream(file)) { - fos.write(buildWavHeader(pcmBytes.length, SAMPLE_RATE, 1, 16)); - fos.write(pcmBytes); - } - Log.i(TAG, "saved voice recording " + file.getAbsolutePath() - + " samples=" + (pcmBytes.length / 2) - + " sec=" + String.format(java.util.Locale.US, "%.2f", pcmBytes.length / 2.0 / SAMPLE_RATE)); - } catch (Throwable t) { - Log.w(TAG, "failed to save voice recording", t); - } - } - - private static byte[] buildWavHeader(int pcmBytes, int sampleRate, int channels, int bitsPerSample) { - int byteRate = sampleRate * channels * bitsPerSample / 8; - int blockAlign = channels * bitsPerSample / 8; - int dataSize = pcmBytes; - int riffSize = 36 + dataSize; - java.nio.ByteBuffer b = java.nio.ByteBuffer.allocate(44).order(java.nio.ByteOrder.LITTLE_ENDIAN); - b.put("RIFF".getBytes(StandardCharsets.US_ASCII)); - b.putInt(riffSize); - b.put("WAVE".getBytes(StandardCharsets.US_ASCII)); - b.put("fmt ".getBytes(StandardCharsets.US_ASCII)); - b.putInt(16); // PCM fmt chunk size - b.putShort((short) 1); // PCM - b.putShort((short) channels); - b.putInt(sampleRate); - b.putInt(byteRate); - b.putShort((short) blockAlign); - b.putShort((short) bitsPerSample); - b.put("data".getBytes(StandardCharsets.US_ASCII)); - b.putInt(dataSize); - return b.array(); - } - - private OfflineRecognizerConfig buildRecognizerConfig(File modelDir, OnboardModelKind kind) { - OfflineModelConfig.Builder modelConfig = OfflineModelConfig.builder() - .setNumThreads(1); - if (kind == OnboardModelKind.WHISPER) { - modelConfig - .setWhisper(OfflineWhisperModelConfig.builder() - .setEncoder(new File(modelDir, "base.en-encoder.int8.onnx").getAbsolutePath()) - .setDecoder(new File(modelDir, "base.en-decoder.int8.onnx").getAbsolutePath()) - .setLanguage("en") - .setTask("transcribe") - .build()) - .setTokens(new File(modelDir, "base.en-tokens.txt").getAbsolutePath()); - } else { - modelConfig - .setMoonshine(OfflineMoonshineModelConfig.builder() - .setEncoder(new File(modelDir, "encoder_model.ort").getAbsolutePath()) - .setMergedDecoder(new File(modelDir, "decoder_model_merged.ort").getAbsolutePath()) - .build()) - .setTokens(new File(modelDir, "tokens.txt").getAbsolutePath()); - } - return OfflineRecognizerConfig.builder() - .setFeatureConfig(FeatureConfig.builder() - .setSampleRate(SAMPLE_RATE) - .setFeatureDim(FEATURE_DIM) - .build()) - .setModelConfig(modelConfig.build()) - .build(); - } - - /** - * The on-device model directory for the given kind, populated by the - * download flow in asr-model.ts (releases before 0.5.0 copied the - * Moonshine files out of the APK directly, so upgraded installs are - * already complete for that model). Null when any expected file is - * missing, i.e. the model still needs to be downloaded. - */ - private File findAsrModelDir(OnboardModelKind kind) { - boolean whisper = kind == OnboardModelKind.WHISPER; - String dirName = whisper ? ASR_WHISPER_MODEL_DIR : ASR_MODEL_DIR; - String[] fileNames = whisper ? ASR_WHISPER_MODEL_FILES : ASR_MODEL_FILES; - File modelDir = new File(appContext.getFilesDir(), ASR_ROOT + File.separator + dirName); - for (String fileName : fileNames) { - File file = new File(modelDir, fileName); - if (!file.exists() || file.length() == 0) { - return null; - } - } - return modelDir; - } - - private void deleteLegacyKwsFiles() { - try { - deleteRecursively(new File(appContext.getFilesDir(), LEGACY_KWS_ROOT)); - } catch (Throwable t) { - Log.w(TAG, "failed to delete legacy wake-word files", t); - } - } - - private static void deleteRecursively(File file) { - if (!file.exists()) { - return; - } - File[] children = file.listFiles(); - if (children != null) { - for (File child : children) { - deleteRecursively(child); - } - } - file.delete(); - } - - private boolean startG2Audio() { - FaceclawBleCommunicator currentCommunicator = communicator; - if (currentCommunicator == null) { - return false; - } - resetAudioStats(); - synchronized (audioQueueLock) { - audioQueue.clear(); - } - return currentCommunicator.startG2AudioCapture(this::queueAudioPacket); - } - - private void processG2Audio() { - short[] pcm = new short[FaceclawLc3Decoder.SAMPLES_PER_PACKET]; - while (started && !Thread.currentThread().isInterrupted()) { - FaceclawLc3Decoder currentDecoder = lc3Decoder; - if (currentDecoder == null) { - return; - } - - AudioPacket packet = takeAudioPacket(); - if (packet == null) { - continue; - } - - int count = currentDecoder.decodePacket(packet.data, pcm); - if (count <= 0) { - maybeEmitAudioStats(false); - continue; - } - decodedSamples += count; - // Global mic processing from the Microphones app config: the - // beam filter drops packets whose firmware direction-of-arrival - // falls outside the listening wedge (isolating the aimed talker - // for every consumer, recognition included), and the spectral - // noise suppressor cleans what remains before it reaches the - // recognizer, cloud PCM, endpointing, or speaker verification. - int angleDegrees = currentDecoder.getLastAngleDegrees(); - int ssr = currentDecoder.getLastSsr(); - if (beamFilterEnabled && ssr > 0 && !withinBeam(angleDegrees)) { - emitFrameMeta(angleDegrees, ssr); - maybeEmitAudioStats(false); - continue; - } - processPcmChunk(pcm, count, angleDegrees, ssr, true); - maybeEmitAudioStats(false); - } - } - - /** - * Per-chunk processing shared by the G2 and phone-mic paths, downstream of - * decode and the beam filter. hasFrameMeta is false for the phone mic, - * which has no firmware DSP metadata to report. - */ - private void processPcmChunk(short[] pcm, int count, int angleDegrees, int ssr, boolean hasFrameMeta) { - if (suppressionEnabled) { - applySuppression(pcm, count); - } - if (recordingPcm != null) { - appendRecording(pcm, count); - } - if (verifyBuffer != null && verifyCount < VERIFY_MAX_SAMPLES) { - int copied = Math.min(count, VERIFY_MAX_SAMPLES - verifyCount); - System.arraycopy(pcm, 0, verifyBuffer, verifyCount, copied); - verifyCount += copied; - } - if (endpointing && endpointDetector.accept(pcm, count)) { - emitSpeechEnd(); - } - // PCM and frame metadata flow in every mode so levels, recording, - // and the Microphones radar keep working alongside onboard ASR. - emitPcm(pcm, count); - if (hasFrameMeta) { - emitFrameMeta(angleDegrees, ssr); - } - if (mode != VoiceInputMode.CLOUD) { - float[] samples = new float[count]; - for (int i = 0; i < count; i++) { - samples[i] = pcm[i] / 32768.0f; - } - processRecognizer(samples); - } - } - - // 50 ms chunks match the G2 packet cadence the rest of the pipeline - // (endpointing, transcript pacing) is tuned for. - private static final int PHONE_MIC_CHUNK_SAMPLES = SAMPLE_RATE / 20; - - /** - * Open the phone's own microphone at the pipeline's native format - * (16 kHz mono PCM16), or null when it cannot start — the permission is - * missing (SecurityException) or the device refuses the configuration. - */ - private android.media.AudioRecord openPhoneMic() { - android.media.AudioRecord record = null; - try { - int minBytes = android.media.AudioRecord.getMinBufferSize( - SAMPLE_RATE, - android.media.AudioFormat.CHANNEL_IN_MONO, - android.media.AudioFormat.ENCODING_PCM_16BIT); - int bufferBytes = Math.max(minBytes, PHONE_MIC_CHUNK_SAMPLES * 2 * 4); - record = new android.media.AudioRecord( - android.media.MediaRecorder.AudioSource.VOICE_RECOGNITION, - SAMPLE_RATE, - android.media.AudioFormat.CHANNEL_IN_MONO, - android.media.AudioFormat.ENCODING_PCM_16BIT, - bufferBytes); - if (record.getState() != android.media.AudioRecord.STATE_INITIALIZED) { - record.release(); - return null; - } - record.startRecording(); - if (record.getRecordingState() != android.media.AudioRecord.RECORDSTATE_RECORDING) { - record.release(); - return null; - } - return record; - } catch (Throwable t) { - Log.w(TAG, "phone mic open failed", t); - if (record != null) { - record.release(); - } - return null; - } - } - - /** - * Phone-mic capture loop: no LC3 decode, no arm bookkeeping, no frame - * metadata — AudioRecord already delivers the pipeline's PCM format. The - * blocking read returns every chunk (50 ms), which bounds how long a - * stop() waits for the loop to notice `started` dropped. - */ - private void processPhoneAudio(android.media.AudioRecord record) { - short[] pcm = new short[PHONE_MIC_CHUNK_SAMPLES]; - while (started && !Thread.currentThread().isInterrupted()) { - int read = record.read(pcm, 0, pcm.length); - if (read < 0) { - Log.w(TAG, "phone mic read failed: " + read); - return; - } - if (read == 0) { - continue; - } - decodedSamples += read; - processPcmChunk(pcm, read, 0, 0, false); - } - } - - private void processRecognizer(float[] samples) { - appendTranscriptSamples(samples); - // Each Whisper call re-encodes the whole buffer plus ~10s of tail - // padding (see the ASR_WHISPER_* comment above), - // so it skips the live-partial redecode Moonshine does on this - // interval and only decodes when a segment commits (8s buffer fill) - // or the utterance ends (decodeTranscript(true) in runLoop()). This - // means no live preview text while speaking in Whisper mode -- status - // stays "Listening..." until release. This is a deliberate tradeoff. - if (onboardModelKind == OnboardModelKind.WHISPER) { - return; - } - long now = SystemClock.elapsedRealtime(); - if (transcriptSampleCount >= TRANSCRIPT_MIN_SAMPLES - && now - lastTranscriptDecodeAtMs >= TRANSCRIPT_DECODE_INTERVAL_MS) { - decodeTranscript(false); - lastTranscriptDecodeAtMs = now; - } - } - - private void appendTranscriptSamples(float[] samples) { - int sourceOffset = 0; - while (sourceOffset < samples.length) { - int available = TRANSCRIPT_SEGMENT_MAX_SAMPLES - transcriptSampleCount; - int count = Math.min(available, samples.length - sourceOffset); - System.arraycopy(samples, sourceOffset, transcriptSamples, transcriptSampleCount, count); - transcriptSampleCount += count; - sourceOffset += count; - - if (transcriptSampleCount == TRANSCRIPT_SEGMENT_MAX_SAMPLES) { - commitTranscriptSegment(); - } - } - } - - /** - * Decode the current model-safe segment and emit the best transcript of the - * complete utterance (REPLACE semantics — the caller displays it as-is). - */ - private void decodeTranscript(boolean isFinal) { - if (recognizer == null || transcriptSampleCount <= 0) { - if (isFinal) { - emitTranscript(lastTranscript, true); - } - return; - } - int segmentSampleCount = transcriptSampleCount; - String segmentText = recognizeTranscriptSegment(segmentSampleCount); - if (segmentText.length() > 0) { - currentSegmentTranscript = segmentText; - } else { - segmentText = currentSegmentTranscript; - } - String text = joinTranscript(committedTranscript, segmentText); - lastTranscript = text; - logTranscriptDecode(isFinal, segmentSampleCount, text); - emitTranscript(text, isFinal); - } - - /** - * Finalize a full segment before accepting more audio. This keeps every - * Moonshine invocation below its failing sequence length while retaining - * all earlier text in the replace-semantics preview. - */ - private void commitTranscriptSegment() { - int cut = findSegmentCutPoint(); - String segmentText = recognizeTranscriptSegment(cut); - if (segmentText.length() == 0) { - // Fallback text came from partial decodes of the full buffer, so it - // may include words from the carried-over tail; rare now that decode - // windows are peak-normalized. - segmentText = currentSegmentTranscript; - } - committedTranscript = joinTranscript(committedTranscript, segmentText); - currentSegmentTranscript = ""; - committedTranscriptSampleCount += cut; - int tail = transcriptSampleCount - cut; - System.arraycopy(transcriptSamples, cut, transcriptSamples, 0, tail); - transcriptSampleCount = tail; - lastTranscript = committedTranscript; - lastTranscriptDecodeAtMs = SystemClock.elapsedRealtime(); - logTranscriptDecode(false, cut, committedTranscript); - emitTranscript(committedTranscript, false); - } - - /** - * Pick where to end the committed segment: the center of the quietest - * window within the search region at the end of the buffer, so the cut - * lands between words instead of splitting one. - */ - private int findSegmentCutPoint() { - int count = transcriptSampleCount; - int searchStart = Math.max(0, count - TRANSCRIPT_CUT_SEARCH_SAMPLES); - int win = TRANSCRIPT_CUT_WINDOW_SAMPLES; - if (count - searchStart <= win) { - return count; - } - double sum = 0; - for (int i = searchStart; i < searchStart + win; i++) { - sum += (double) transcriptSamples[i] * transcriptSamples[i]; - } - double best = sum; - int bestStart = searchStart; - for (int start = searchStart + 1; start + win <= count; start++) { - float dropped = transcriptSamples[start - 1]; - float added = transcriptSamples[start + win - 1]; - sum += (double) added * added - (double) dropped * dropped; - if (sum < best) { - best = sum; - bestStart = start; - } - } - return bestStart + win / 2; - } - - private String recognizeTranscriptSegment(int sampleCount) { - OfflineRecognizer currentRecognizer = recognizer; - if (currentRecognizer == null || sampleCount <= 0) { - return ""; - } - float[] segment = Arrays.copyOf(transcriptSamples, sampleCount); - // Whisper hallucinates text on near-silent input (a known quirk of the - // model, not this pipeline); gate it on the PRE-normalization peak, since - // normalizePeak() below would otherwise amplify true silence right up to - // the target level and hide the very thing being checked for. Moonshine - // does not share this failure mode in practice, so it is left unchanged. - if (onboardModelKind == OnboardModelKind.WHISPER && peakAmplitude(segment) < WHISPER_SILENCE_PEAK_THRESHOLD) { - return ""; - } - normalizePeak(segment); - OfflineStream offlineStream = currentRecognizer.createStream(); - try { - offlineStream.acceptWaveform(segment, SAMPLE_RATE); - currentRecognizer.decode(offlineStream); - OfflineRecognizerResult result = currentRecognizer.getResult(offlineStream); - String raw = result == null ? "" : result.getText(); - return raw == null ? "" : raw.trim(); - } finally { - offlineStream.release(); - } - } - - private static float peakAmplitude(float[] samples) { - float peak = 0f; - for (float s : samples) { - float a = Math.abs(s); - if (a > peak) { - peak = a; - } - } - return peak; - } - - private static void normalizePeak(float[] samples) { - float peak = peakAmplitude(samples); - if (peak <= 0f) { - return; - } - float gain = Math.min(TRANSCRIPT_NORMALIZE_TARGET_PEAK / peak, TRANSCRIPT_NORMALIZE_MAX_GAIN); - if (gain <= 1f) { - return; - } - for (int i = 0; i < samples.length; i++) { - samples[i] *= gain; - } - } - - private void logTranscriptDecode(boolean isFinal, int segmentSampleCount, String text) { - double totalAudioSec = - (committedTranscriptSampleCount + transcriptSampleCount) / (double) SAMPLE_RATE; - String preview = text.length() <= TRANSCRIPT_LOG_PREVIEW_CHARS - ? text : text.substring(0, TRANSCRIPT_LOG_PREVIEW_CHARS) + "..."; - Log.i(TAG, (onboardModelKind == OnboardModelKind.WHISPER ? "Whisper" : "Moonshine") + " decode final=" + isFinal - + " audioSec=" + String.format(java.util.Locale.US, "%.2f", totalAudioSec) - + " segmentAudioSec=" + String.format(java.util.Locale.US, "%.2f", segmentSampleCount / (double) SAMPLE_RATE) - + " textLen=" + text.length() + " text=\"" + preview + "\""); - } - - private static String joinTranscript(String prefix, String suffix) { - if (prefix == null || prefix.length() == 0) { - return suffix == null ? "" : suffix; - } - if (suffix == null || suffix.length() == 0) { - return prefix; - } - char first = suffix.charAt(0); - boolean attachesToPrevious = ".,!?;:%)]}".indexOf(first) >= 0; - return prefix + (attachesToPrevious ? "" : " ") + suffix; - } - - private void resetTranscriptState() { - transcriptSampleCount = 0; - committedTranscriptSampleCount = 0; - committedTranscript = ""; - currentSegmentTranscript = ""; - lastTranscriptDecodeAtMs = 0; - } - - private void emitPcm(short[] pcm, int count) { - FaceclawVoiceControllerListener currentListener = listener; - if (currentListener == null || count <= 0) { - return; - } - byte[] le = new byte[count * 2]; - for (int i = 0; i < count; i++) { - short s = pcm[i]; - le[i * 2] = (byte) (s & 0xff); - le[i * 2 + 1] = (byte) ((s >> 8) & 0xff); - } - mainHandler.post(() -> currentListener.onPcm(le)); - } - - /** - * Embed the session's buffered utterance and compare it to the enrolled - * wearer voice-print. Fails open: a session too short to verify, or a - * model that will not load, counts as the wearer rather than silencing - * every command. - */ - private void runSpeakerVerification() { - short[] buffer = verifyBuffer; - float[] wearer = verifyWearerEmbedding; - String modelPath = verifySpeakerModelPath; - verifyBuffer = null; - if (buffer == null || wearer == null || modelPath == null) { - return; - } - if (verifyCount < VERIFY_MIN_SAMPLES) { - emitSpeakerVerified(true, 0f); - return; - } - FaceclawSpeakerId speakerId = cachedSpeakerId(modelPath); - try { - byte[] le = new byte[verifyCount * 2]; - for (int i = 0; i < verifyCount; i++) { - short s = buffer[i]; - le[i * 2] = (byte) (s & 0xff); - le[i * 2 + 1] = (byte) ((s >> 8) & 0xff); - } - float[] embedding = speakerId.embed(le, SAMPLE_RATE); - if (embedding == null || embedding.length != wearer.length) { - emitSpeakerVerified(true, 0f); - return; - } - double dot = 0; - for (int i = 0; i < embedding.length; i++) { - dot += (double) embedding[i] * wearer[i]; - } - boolean isWearer = dot >= verifyThreshold; - Log.i(TAG, "speaker verification similarity=" + String.format(java.util.Locale.US, "%.3f", dot) - + " threshold=" + verifyThreshold + " isWearer=" + isWearer); - emitSpeakerVerified(isWearer, (float) dot); - } catch (Throwable t) { - Log.w(TAG, "speaker verification failed", t); - emitSpeakerVerified(true, 0f); - } - } - - // The 28 MB embedding model takes seconds to load; keep one instance - // across capture sessions so verification adds only the embed time. - private static FaceclawSpeakerId sharedSpeakerId; - private static String sharedSpeakerIdPath; - - private static synchronized FaceclawSpeakerId cachedSpeakerId(String modelPath) { - if (sharedSpeakerId == null || !modelPath.equals(sharedSpeakerIdPath)) { - if (sharedSpeakerId != null) { - sharedSpeakerId.close(); - } - sharedSpeakerId = new FaceclawSpeakerId(modelPath); - sharedSpeakerIdPath = modelPath; - } - return sharedSpeakerId; - } - - private void emitSpeakerVerified(boolean isWearer, float similarity) { - FaceclawVoiceControllerListener currentListener = listener; - if (currentListener == null) { - return; - } - mainHandler.post(() -> currentListener.onSpeakerVerified(isWearer, similarity)); - } - - private void emitFrameMeta(int angleDegrees, int ssr) { - FaceclawVoiceControllerListener currentListener = listener; - if (currentListener == null) { - return; - } - mainHandler.post(() -> currentListener.onFrameMeta(angleDegrees, ssr)); - } - - private void emitSpeechEnd() { - FaceclawVoiceControllerListener currentListener = listener; - if (currentListener == null) { - return; - } - mainHandler.post(currentListener::onSpeechEnd); - } - - private void emitStopped(int captureId) { - FaceclawVoiceControllerListener currentListener = listener; - if (currentListener != null) { - mainHandler.post(() -> currentListener.onStopped(captureId)); - } - } - - private void stopG2Audio() { - FaceclawBleCommunicator currentCommunicator = communicator; - if (currentCommunicator != null) { - currentCommunicator.stopG2AudioCapture(); - } - maybeEmitAudioStats(true); - } - - private void releaseSherpa() { - if (recognizer != null) { - recognizer.release(); - recognizer = null; - } - } - - private void releaseLc3() { - if (lc3Decoder != null) { - lc3Decoder.close(); - lc3Decoder = null; - } - } - - private void queueAudioPacket(byte[] data, String arm, long arrivalMs) { - if (!started || data == null) { - return; - } - if (!"L".equals(arm)) { - wrongArmPackets++; - } - synchronized (audioQueueLock) { - if (audioQueue.size() >= MAX_AUDIO_QUEUE_PACKETS) { - audioQueue.removeFirst(); - queueDroppedPackets++; - } - audioQueue.addLast(new AudioPacket(data, arm, arrivalMs)); - queuedPackets++; - if (lastPacketArrivalMs > 0) { - long delta = arrivalMs - lastPacketArrivalMs; - if (delta > maxInterPacketMs) { - maxInterPacketMs = delta; - } - if (delta > LATE_PACKET_INTERVAL_MS) { - latePackets++; - } - } - lastPacketArrivalMs = arrivalMs; - audioQueueLock.notifyAll(); - } - } - - private AudioPacket takeAudioPacket() { - synchronized (audioQueueLock) { - while (started && audioQueue.isEmpty()) { - try { - audioQueueLock.wait(250); - maybeEmitAudioStats(false); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - return null; - } - } - return audioQueue.pollFirst(); - } - } - - private void resetAudioStats() { - queuedPackets = 0; - queueDroppedPackets = 0; - decodedSamples = 0; - latePackets = 0; - wrongArmPackets = 0; - lastPacketArrivalMs = 0; - maxInterPacketMs = 0; - lastStatsAtMs = SystemClock.elapsedRealtime(); - } - - private void maybeEmitAudioStats(boolean force) { - long now = SystemClock.elapsedRealtime(); - if (!force && now - lastStatsAtMs < STATS_INTERVAL_MS) { - return; - } - lastStatsAtMs = now; - FaceclawLc3Decoder currentDecoder = lc3Decoder; - long real = currentDecoder == null ? 0 : currentDecoder.getRealPackets(); - long duplicate = currentDecoder == null ? 0 : currentDecoder.getDuplicatePackets(); - long missing = currentDecoder == null ? 0 : currentDecoder.getMissingPackets(); - long decodeErrors = currentDecoder == null ? 0 : currentDecoder.getDecodeErrors(); - String status = "G2 mic packets=" + queuedPackets - + " decoded=" + real - + " missing=" + missing - + " duplicate=" + duplicate - + " late=" + latePackets - + " maxGapMs=" + maxInterPacketMs - + "\n" - + " queueDrop=" + queueDroppedPackets - + " decodeErrors=" + decodeErrors - + " wrongArm=" + wrongArmPackets - + " audioSec=" + String.format(java.util.Locale.US, "%.1f", decodedSamples / (double) SAMPLE_RATE); - // Audio-pipeline stats are diagnostic; keep them in logcat only, out of - // the on-glasses voice UI. - Log.i(TAG, status.replace('\n', ' ') + " expectedIntervalMs=" + EXPECTED_PACKET_INTERVAL_MS); - } - - private void emitStatus(String status) { - FaceclawVoiceControllerListener currentListener = listener; - if (currentListener == null) { - return; - } - mainHandler.post(() -> currentListener.onStatus(status)); - } - - private void emitTranscript(String text, boolean isFinal) { - FaceclawVoiceControllerListener currentListener = listener; - if (currentListener == null) { - return; - } - Log.i(TAG, "Emit transcript final=" + isFinal + " textLen=" + (text == null ? 0 : text.trim().length())); - mainHandler.post(() -> currentListener.onTranscript(text, isFinal)); - } - - private static final class AudioPacket { - final byte[] data; - final String arm; - final long arrivalMs; - - AudioPacket(byte[] data, String arm, long arrivalMs) { - this.data = data; - this.arm = arm == null ? "?" : arm; - this.arrivalMs = arrivalMs; - } - } -} diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawVoiceController.kt b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawVoiceController.kt new file mode 100644 index 00000000..0e59e435 --- /dev/null +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawVoiceController.kt @@ -0,0 +1,1186 @@ +package com.faceclaw.app + +import android.content.Context +import android.os.Handler +import android.os.Looper +import android.os.SystemClock +import android.util.Log + +import com.k2fsa.sherpa.onnx.FeatureConfig +import com.k2fsa.sherpa.onnx.OfflineModelConfig +import com.k2fsa.sherpa.onnx.OfflineMoonshineModelConfig +import com.k2fsa.sherpa.onnx.OfflineRecognizer +import com.k2fsa.sherpa.onnx.OfflineRecognizerConfig +import com.k2fsa.sherpa.onnx.OfflineRecognizerResult +import com.k2fsa.sherpa.onnx.OfflineStream +import com.k2fsa.sherpa.onnx.OfflineWhisperModelConfig + +import java.io.File +import java.nio.charset.StandardCharsets +import java.util.ArrayDeque +import java.util.Arrays + +class FaceclawVoiceController(context: Context) { + companion object { + private const val TAG = "FaceclawVoice" + private const val SAMPLE_RATE = 16000 + private const val FEATURE_DIM = 80 + private const val MAX_AUDIO_QUEUE_PACKETS = 80 + private const val EXPECTED_PACKET_INTERVAL_MS = 50 + private const val LATE_PACKET_INTERVAL_MS = 90 + private const val STATS_INTERVAL_MS = 5_000 + // Push-to-talk utterance boundaries come from the button. We re-decode the + // current audio segment in full for each live partial and emit the complete + // utterance text (REPLACE, never a delta). The sherpa Moonshine v2 decoder + // used here fails once a single input grows past roughly 9.1 seconds, so + // longer utterances are committed in model-safe segments. + private const val TRANSCRIPT_DECODE_INTERVAL_MS = 700 + private const val TRANSCRIPT_MIN_SAMPLES = SAMPLE_RATE / 3 + private const val TRANSCRIPT_SEGMENT_MAX_SAMPLES = SAMPLE_RATE * 8 + // When a segment fills, cut at the quietest window within the last + // TRANSCRIPT_CUT_SEARCH_SAMPLES rather than mid-word at the 8s mark; the + // audio after the cut carries over into the next segment. + private const val TRANSCRIPT_CUT_SEARCH_SAMPLES = SAMPLE_RATE * 2 + private const val TRANSCRIPT_CUT_WINDOW_SAMPLES = SAMPLE_RATE * 30 / 1000 + // Glasses-mic PCM peaks around 0.1 full scale, and at that level the + // quantized Moonshine model often returns empty or garbled text. Boost + // each decode window toward this peak, with a gain cap so near-silent + // buffers aren't amplified into pure noise. + private const val TRANSCRIPT_NORMALIZE_TARGET_PEAK = 0.9f + private const val TRANSCRIPT_NORMALIZE_MAX_GAIN = 30f + private const val TRANSCRIPT_LOG_PREVIEW_CHARS = 80 + // Model directory shared with the TS-side download flow (asr-model.ts), + // which fetches the Moonshine files here on demand; they are no longer + // bundled in the APK. + private const val ASR_ROOT = "faceclaw-voice-asr" + private const val ASR_MODEL_DIR = "sherpa-onnx-moonshine-base-en-quantized-2026-02-27" + // Model files for the retired on-phone wake-word spotter, copied to + // filesDir by earlier releases; deleted on sight to reclaim the space. + // (The wakeword is now detected by the glasses firmware itself.) + private const val LEGACY_KWS_ROOT = "faceclaw-voice" + private val ASR_MODEL_FILES = arrayOf( + "encoder_model.ort", + "decoder_model_merged.ort", + "tokens.txt" + ) + // Second on-device model: sherpa-onnx's offline Whisper backend (base.en, + // int8-quantized -- see the model-choice note in asr-model.ts). Directory + // shared with the TS-side download flow, same convention as ASR_MODEL_DIR. + // + // On every call, sherpa-onnx (offline-recognizer-whisper-impl.h, v1.13.0) + // runs Whisper's encoder over the whole buffer plus 1000 frames (~10s) of + // zero tail padding, capped at 30s. Re-decoding on Moonshine's + // TRANSCRIPT_DECODE_INTERVAL_MS live-partial cadence would repeat that + // encode, growing with the utterance, roughly 1.4x/second while the user is + // still speaking, so onboardModelKind gates that loop off for Whisper -- + // see processRecognizer(). Whisper is also known to hallucinate text on + // near-silent input; recognizeTranscriptSegment() gates that too. + private const val ASR_WHISPER_MODEL_DIR = "sherpa-onnx-whisper-base-en-int8" + private val ASR_WHISPER_MODEL_FILES = arrayOf( + "base.en-encoder.int8.onnx", + "base.en-decoder.int8.onnx", + "base.en-tokens.txt" + ) + // Below this peak amplitude (pre-normalization, of a full-scale +/-1.0f + // buffer) a segment is treated as silence and never reaches the Whisper + // recognizer at all, rather than risking a hallucinated non-answer. Picked + // conservatively low (well under typical mic noise floor already seen in + // this pipeline's normalization target) -- UNTESTED on real hardware, tune + // against real glasses captures rather than trusting this number. + private const val WHISPER_SILENCE_PEAK_THRESHOLD = 0.01f + + // Speaker verification against the enrolled wearer voice-print ("my voice + // only" command gating). Configured before start(); the utterance PCM is + // buffered (capped) and verified once at session end. + private const val VERIFY_MAX_SAMPLES = SAMPLE_RATE * 10 + private const val VERIFY_MIN_SAMPLES = SAMPLE_RATE + + // 50 ms chunks match the G2 packet cadence the rest of the pipeline + // (endpointing, transcript pacing) is tuned for. + private const val PHONE_MIC_CHUNK_SAMPLES = SAMPLE_RATE / 20 + + @JvmStatic + private fun buildWavHeader(pcmBytes: Int, sampleRate: Int, channels: Int, bitsPerSample: Int): ByteArray { + val byteRate = sampleRate * channels * bitsPerSample / 8 + val blockAlign = channels * bitsPerSample / 8 + val dataSize = pcmBytes + val riffSize = 36 + dataSize + val b = java.nio.ByteBuffer.allocate(44).order(java.nio.ByteOrder.LITTLE_ENDIAN) + b.put("RIFF".toByteArray(StandardCharsets.US_ASCII)) + b.putInt(riffSize) + b.put("WAVE".toByteArray(StandardCharsets.US_ASCII)) + b.put("fmt ".toByteArray(StandardCharsets.US_ASCII)) + b.putInt(16) // PCM fmt chunk size + b.putShort(1.toShort()) // PCM + b.putShort(channels.toShort()) + b.putInt(sampleRate) + b.putInt(byteRate) + b.putShort(blockAlign.toShort()) + b.putShort(bitsPerSample.toShort()) + b.put("data".toByteArray(StandardCharsets.US_ASCII)) + b.putInt(dataSize) + return b.array() + } + + @JvmStatic + private fun deleteRecursively(file: File) { + if (!file.exists()) { + return + } + val children = file.listFiles() + if (children != null) { + for (child in children) { + deleteRecursively(child) + } + } + file.delete() + } + + @JvmStatic + private fun peakAmplitude(samples: FloatArray): Float { + var peak = 0f + for (s in samples) { + val a = Math.abs(s) + if (a > peak) { + peak = a + } + } + return peak + } + + @JvmStatic + private fun normalizePeak(samples: FloatArray) { + val peak = peakAmplitude(samples) + if (peak <= 0f) { + return + } + val gain = Math.min(TRANSCRIPT_NORMALIZE_TARGET_PEAK / peak, TRANSCRIPT_NORMALIZE_MAX_GAIN) + if (gain <= 1f) { + return + } + for (i in samples.indices) { + samples[i] *= gain + } + } + + @JvmStatic + private fun joinTranscript(prefix: String?, suffix: String?): String { + if (prefix == null || prefix.length == 0) { + return suffix ?: "" + } + if (suffix == null || suffix.length == 0) { + return prefix + } + val first = suffix[0] + val attachesToPrevious = ".,!?;:%)]}".indexOf(first) >= 0 + return prefix + (if (attachesToPrevious) "" else " ") + suffix + } + + // The 28 MB embedding model takes seconds to load; keep one instance + // across capture sessions so verification adds only the embed time. + private var sharedSpeakerId: FaceclawSpeakerId? = null + private var sharedSpeakerIdPath: String? = null + + @JvmStatic + @Synchronized + private fun cachedSpeakerId(modelPath: String): FaceclawSpeakerId { + if (sharedSpeakerId == null || modelPath != sharedSpeakerIdPath) { + if (sharedSpeakerId != null) { + sharedSpeakerId!!.close() + } + sharedSpeakerId = FaceclawSpeakerId(modelPath) + sharedSpeakerIdPath = modelPath + } + return sharedSpeakerId!! + } + } + + private enum class VoiceInputMode { + ONBOARD, // on-phone transcription (Moonshine or Whisper; see onboardModelKind) + CLOUD // decode locally, emit PCM for a cloud recognizer on the TS side + } + + /** Which on-device model ONBOARD mode uses. Set via setOnboardModelKind() before start(). */ + private enum class OnboardModelKind { + MOONSHINE, + WHISPER + } + + private val appContext: Context = context.applicationContext + private val mainHandler = Handler(Looper.getMainLooper()) + private val lock = Any() + private val audioQueueLock = Any() + private val audioQueue = ArrayDeque() + @Volatile + private var listener: FaceclawVoiceControllerListener? = null + @Volatile + private var communicator: FaceclawBleCommunicator? = null + private var workerThread: Thread? = null + @Volatile + private var started = false + // Set once the worker has the glasses mic enabled for this session. + // Read and written under `lock`, so it flips with `started` atomically. + private var audioStarted = false + // Capture from the phone's own microphone instead of the G2 over BLE + // (preview-only mode, where no glasses are connected). Latched into + // activePhoneMic at start() (under `lock`) so a mid-session setter call + // can't switch pipelines underneath the worker. + @Volatile + private var usePhoneMic = false + private var activePhoneMic = false + private var mode = VoiceInputMode.CLOUD + @Volatile + private var onboardModelKind = OnboardModelKind.MOONSHINE + private var recognizer: OfflineRecognizer? = null + private var lc3Decoder: FaceclawLc3Decoder? = null + private val transcriptSamples = FloatArray(TRANSCRIPT_SEGMENT_MAX_SAMPLES) + private var transcriptSampleCount = 0 + private var committedTranscriptSampleCount: Long = 0 + private var committedTranscript = "" + private var currentSegmentTranscript = "" + private var lastTranscriptDecodeAtMs: Long = 0 + private var lastTranscript = "" + @Volatile + private var saveRecordings = false + @Volatile + private var endpointing = false + private val endpointDetector = VoiceEndpointDetector() + private var recordingPcm: java.io.ByteArrayOutputStream? = null + // Speaker verification against the enrolled wearer voice-print ("my voice + // only" command gating). Configured before start(); the utterance PCM is + // buffered (capped) and verified once at session end. + @Volatile + private var verifySpeakerModelPath: String? = null + @Volatile + private var verifyWearerEmbedding: FloatArray? = null + @Volatile + private var verifyThreshold = 0.8f + private var verifyBuffer: ShortArray? = null + private var verifyCount = 0 + // Global mic processing (Microphones app config): spectral noise + // suppression and firmware-DoA beam gating, applied to every capture + // session that opts in (assistant push-to-talk, Transcribe, hands-free). + // The raw tap opts out — raw means raw, and the Microphones session does + // its own beam-compensated processing on that path. + @Volatile + private var suppressionEnabled = false + @Volatile + private var beamFilterEnabled = false + @Volatile + private var beamCenterDeg = 0 + @Volatile + private var beamHalfWidthDeg = 180 + private var suppressor: FaceclawNoiseSuppressor? = null + private var queuedPackets: Long = 0 + private var queueDroppedPackets: Long = 0 + private var decodedSamples: Long = 0 + private var latePackets: Long = 0 + private var wrongArmPackets: Long = 0 + private var lastPacketArrivalMs: Long = 0 + private var maxInterPacketMs: Long = 0 + private var lastStatsAtMs: Long = 0 + + fun setListener(listener: FaceclawVoiceControllerListener?) { + this.listener = listener + } + + fun setCommunicator(communicator: FaceclawBleCommunicator?) { + this.communicator = communicator + } + + /** Source the next capture from the phone microphone (no glasses paired). */ + fun setUsePhoneMic(usePhoneMic: Boolean) { + this.usePhoneMic = usePhoneMic + } + + /** When true, the decoded mic PCM for each session is saved as a WAV. */ + fun setSaveRecordings(saveRecordings: Boolean) { + this.saveRecordings = saveRecordings + } + + /** + * Which on-device model {@link #start}("onboard") should load: "whisper" + * selects the second on-device model (sherpa-onnx offline Whisper); any + * other value (including null/absent) keeps the existing Moonshine model, + * so callers that never call this see unchanged behavior. Must be set + * before {@link #start}; has no effect in CLOUD mode. + */ + fun setOnboardModelKind(kind: String?) { + this.onboardModelKind = if ("whisper" == kind) OnboardModelKind.WHISPER else OnboardModelKind.MOONSHINE + } + + /** + * When true, watch the decoded PCM and fire {@code onSpeechEnd} once the + * speaker stops. Used by hands-free ("Hey Even") capture, which has no + * button release to end the utterance. Must be set before {@link #start}. + */ + fun setEndpointing(endpointing: Boolean) { + this.endpointing = endpointing + } + + /** + * Verify this session's speaker against the enrolled wearer voice-print + * and report the result via onSpeakerVerified just before the final + * transcript. Must be set before {@link #start}; pass a null model path + * to disable. + */ + fun setSpeakerVerification(speakerModelPath: String?, wearerEmbedding: FloatArray?, threshold: Float) { + this.verifySpeakerModelPath = speakerModelPath + this.verifyWearerEmbedding = wearerEmbedding + this.verifyThreshold = if (threshold > 0) threshold else 0.8f + } + + fun clearSpeakerVerification() { + this.verifySpeakerModelPath = null + this.verifyWearerEmbedding = null + } + + /** Spectral noise suppression on the decoded stream. Safe to flip mid-run. */ + fun setNoiseSuppression(enabled: Boolean) { + this.suppressionEnabled = enabled + } + + /** + * Direction gating from the Sonic Radar beam: packets whose firmware + * direction-of-arrival falls outside centerDeg ± halfWidthDeg (device + * frame, 0 = straight ahead, positive right) are dropped before any + * consumer sees them. Safe to update mid-run. + */ + fun setBeamFilter(enabled: Boolean, centerDeg: Int, halfWidthDeg: Int) { + this.beamFilterEnabled = enabled + this.beamCenterDeg = centerDeg + this.beamHalfWidthDeg = Math.max(5, Math.min(180, halfWidthDeg)) + } + + private fun withinBeam(angleDegrees: Int): Boolean { + var delta = angleDegrees - beamCenterDeg + while (delta > 180) delta -= 360 + while (delta < -180) delta += 360 + return Math.abs(delta) <= beamHalfWidthDeg + } + + private fun applySuppression(pcm: ShortArray, count: Int) { + try { + if (suppressor == null) { + suppressor = FaceclawNoiseSuppressor(SAMPLE_RATE) + } + val le = ByteArray(count * 2) + for (i in 0 until count) { + le[i * 2] = (pcm[i].toInt() and 0xff).toByte() + le[i * 2 + 1] = ((pcm[i].toInt() shr 8) and 0xff).toByte() + } + val cleaned = suppressor!!.process(le)!! + val cleanedCount = Math.min(count, cleaned.size / 2) + for (i in 0 until cleanedCount) { + pcm[i] = ((cleaned[i * 2].toInt() and 0xff) or (cleaned[i * 2 + 1].toInt() shl 8)).toShort() + } + } catch (t: Throwable) { + Log.w(TAG, "noise suppression failed; passing audio through", t) + suppressionEnabled = false + } + } + + fun start(requestedMode: String?) { + start(requestedMode, 0) + } + + fun start(requestedMode: String?, captureId: Int) { + synchronized(lock) { + if (started) { + emitStatus("Voice control is already listening.") + emitStopped(captureId) + return + } + if (!usePhoneMic && (communicator == null || !communicator!!.isSessionReady())) { + emitStatus("Voice control needs an active G2 connection.") + emitStopped(captureId) + return + } + mode = parseMode(requestedMode) + activePhoneMic = usePhoneMic + started = true + audioStarted = false + workerThread = Thread(Runnable { runLoop(captureId) }, "FaceclawVoiceController") + workerThread!!.start() + } + } + + /** + * Whether mic audio is actually flowing. {@link #start} only records + * intent: the enable lives in the glasses' EvenHub session, so a transport + * drop or a session suspend can leave this controller started with a + * worker that will never see another packet. Anything deciding whether to + * (re)start capture must ask this rather than assume its own bookkeeping. + */ + fun isCapturing(): Boolean { + val audioUp: Boolean + val phoneMic: Boolean + synchronized(lock) { + if (!started) { + return false + } + audioUp = audioStarted + phoneMic = activePhoneMic + } + if (!audioUp) { + // The worker is still bringing the mic up; report it as running so + // a concurrent request shares it instead of restarting it. + return true + } + if (phoneMic) { + // AudioRecord has no session to lose the enable to; it runs until stop(). + return true + } + val currentCommunicator = communicator + return currentCommunicator != null && currentCommunicator.isAudioCaptureActive() + } + + fun stop() { + val threadToJoin: Thread? + synchronized(lock) { + if (!started) { + return + } + started = false + threadToJoin = workerThread + } + stopG2Audio() + synchronized(audioQueueLock) { + (audioQueueLock as java.lang.Object).notifyAll() + } + if (threadToJoin != null) { + threadToJoin.interrupt() + if (Thread.currentThread() !== threadToJoin) { + try { + threadToJoin.join(1500) + } catch (e: InterruptedException) { + Thread.currentThread().interrupt() + } + } + } + } + + fun close() { + stop() + } + + private fun parseMode(requestedMode: String?): VoiceInputMode { + if ("cloud" == requestedMode) { + return VoiceInputMode.CLOUD + } + return VoiceInputMode.ONBOARD + } + + private fun runLoop(captureId: Int) { + try { + deleteLegacyKwsFiles() + val currentMode = mode + val currentOnboardKind = onboardModelKind + if (currentMode == VoiceInputMode.ONBOARD) { + val modelDir = findAsrModelDir(currentOnboardKind) + if (modelDir == null) { + emitStatus("Voice model not downloaded (see Settings > Voice).") + return + } + emitStatus("Loading transcription model...") + recognizer = OfflineRecognizer(buildRecognizerConfig(modelDir, currentOnboardKind)) + resetTranscriptState() + lastTranscript = "" + } + endpointDetector.reset() + if (suppressor != null) { + suppressor!!.reset() + } + recordingPcm = if (saveRecordings) java.io.ByteArrayOutputStream(SAMPLE_RATE * 2 * 4) else null + val verifying = verifySpeakerModelPath != null && verifyWearerEmbedding != null + verifyBuffer = if (verifying) ShortArray(VERIFY_MAX_SAMPLES) else null + verifyCount = 0 + if (activePhoneMic) { + val record = openPhoneMic() + if (record == null) { + emitStatus("Could not start the phone microphone.") + return + } + synchronized(lock) { + audioStarted = true + } + emitStatus(if (currentMode == VoiceInputMode.CLOUD) + "Listening (cloud)..." + else + "Listening...") + try { + processPhoneAudio(record) + } finally { + try { + record.stop() + } catch (ignored: Throwable) { + // Already stopped or never recording; release below either way. + } + record.release() + } + } else { + lc3Decoder = FaceclawLc3Decoder() + if (!startG2Audio()) { + emitStatus("Could not start G2 microphone input.") + return + } + synchronized(lock) { + audioStarted = true + } + emitStatus(if (currentMode == VoiceInputMode.CLOUD) + "Listening (cloud)..." + else + "Listening...") + processG2Audio() + } + // Verification result must precede the final transcript so the + // TS bridge can suppress a non-wearer command before it is acted + // on (the callbacks are posted in order to the main handler). + runSpeakerVerification() + // Button released / stop requested: emit one final full-utterance + // transcript so the UI can freeze it. + if (currentMode == VoiceInputMode.ONBOARD) { + decodeTranscript(true) + } + } catch (error: Throwable) { + Log.e(TAG, "Voice control failed", error) + emitStatus("Voice control failed: " + error.message) + } finally { + stopG2Audio() + writeRecordingIfAny() + releaseSherpa() + releaseLc3() + synchronized(lock) { + started = false + audioStarted = false + workerThread = null + } + emitStopped(captureId) + } + } + + private fun appendRecording(pcm: ShortArray, count: Int) { + val out = recordingPcm + if (out == null) { + return + } + for (i in 0 until count) { + val s = pcm[i].toInt() + out.write(s and 0xff) + out.write((s shr 8) and 0xff) + } + } + + /** Save the session's decoded mic PCM as a 16 kHz mono 16-bit WAV. */ + private fun writeRecordingIfAny() { + val out = recordingPcm + recordingPcm = null + if (out == null || out.size() == 0) { + return + } + try { + val pcmBytes = out.toByteArray() + val dir = java.io.File(appContext.getExternalFilesDir(null), "voice-recordings") + if (!dir.exists() && !dir.mkdirs()) { + Log.w(TAG, "could not create voice-recordings dir") + return + } + val stamp = java.text.SimpleDateFormat("yyyyMMdd-HHmmss-SSS", java.util.Locale.US) + .format(java.util.Date()) + val file = java.io.File(dir, "voice-$stamp.wav") + java.io.FileOutputStream(file).use { fos -> + fos.write(buildWavHeader(pcmBytes.size, SAMPLE_RATE, 1, 16)) + fos.write(pcmBytes) + } + Log.i(TAG, "saved voice recording " + file.absolutePath + + " samples=" + (pcmBytes.size / 2) + + " sec=" + String.format(java.util.Locale.US, "%.2f", pcmBytes.size / 2.0 / SAMPLE_RATE)) + } catch (t: Throwable) { + Log.w(TAG, "failed to save voice recording", t) + } + } + + private fun buildRecognizerConfig(modelDir: File, kind: OnboardModelKind): OfflineRecognizerConfig { + val modelConfig = OfflineModelConfig.builder() + .setNumThreads(1) + if (kind == OnboardModelKind.WHISPER) { + modelConfig + .setWhisper(OfflineWhisperModelConfig.builder() + .setEncoder(File(modelDir, "base.en-encoder.int8.onnx").absolutePath) + .setDecoder(File(modelDir, "base.en-decoder.int8.onnx").absolutePath) + .setLanguage("en") + .setTask("transcribe") + .build()) + .setTokens(File(modelDir, "base.en-tokens.txt").absolutePath) + } else { + modelConfig + .setMoonshine(OfflineMoonshineModelConfig.builder() + .setEncoder(File(modelDir, "encoder_model.ort").absolutePath) + .setMergedDecoder(File(modelDir, "decoder_model_merged.ort").absolutePath) + .build()) + .setTokens(File(modelDir, "tokens.txt").absolutePath) + } + return OfflineRecognizerConfig.builder() + .setFeatureConfig(FeatureConfig.builder() + .setSampleRate(SAMPLE_RATE) + .setFeatureDim(FEATURE_DIM) + .build()) + .setModelConfig(modelConfig.build()) + .build() + } + + /** + * The on-device model directory for the given kind, populated by the + * download flow in asr-model.ts (releases before 0.5.0 copied the + * Moonshine files out of the APK directly, so upgraded installs are + * already complete for that model). Null when any expected file is + * missing, i.e. the model still needs to be downloaded. + */ + private fun findAsrModelDir(kind: OnboardModelKind): File? { + val whisper = kind == OnboardModelKind.WHISPER + val dirName = if (whisper) ASR_WHISPER_MODEL_DIR else ASR_MODEL_DIR + val fileNames = if (whisper) ASR_WHISPER_MODEL_FILES else ASR_MODEL_FILES + val modelDir = File(appContext.filesDir, ASR_ROOT + File.separator + dirName) + for (fileName in fileNames) { + val file = File(modelDir, fileName) + if (!file.exists() || file.length() == 0L) { + return null + } + } + return modelDir + } + + private fun deleteLegacyKwsFiles() { + try { + deleteRecursively(File(appContext.filesDir, LEGACY_KWS_ROOT)) + } catch (t: Throwable) { + Log.w(TAG, "failed to delete legacy wake-word files", t) + } + } + + private fun startG2Audio(): Boolean { + val currentCommunicator = communicator + if (currentCommunicator == null) { + return false + } + resetAudioStats() + synchronized(audioQueueLock) { + audioQueue.clear() + } + return currentCommunicator.startG2AudioCapture(object : FaceclawAudioPacketListener { + override fun onAudioPacket(data: ByteArray?, arm: String?, arrivalMs: Long) { + queueAudioPacket(data, arm, arrivalMs) + } + }) + } + + private fun processG2Audio() { + val pcm = ShortArray(FaceclawLc3Decoder.SAMPLES_PER_PACKET) + while (started && !Thread.currentThread().isInterrupted) { + val currentDecoder = lc3Decoder + if (currentDecoder == null) { + return + } + + val packet = takeAudioPacket() + if (packet == null) { + continue + } + + val count = currentDecoder.decodePacket(packet.data, pcm) + if (count <= 0) { + maybeEmitAudioStats(false) + continue + } + decodedSamples += count.toLong() + // Global mic processing from the Microphones app config: the + // beam filter drops packets whose firmware direction-of-arrival + // falls outside the listening wedge (isolating the aimed talker + // for every consumer, recognition included), and the spectral + // noise suppressor cleans what remains before it reaches the + // recognizer, cloud PCM, endpointing, or speaker verification. + val angleDegrees = currentDecoder.getLastAngleDegrees() + val ssr = currentDecoder.getLastSsr() + if (beamFilterEnabled && ssr > 0 && !withinBeam(angleDegrees)) { + emitFrameMeta(angleDegrees, ssr) + maybeEmitAudioStats(false) + continue + } + processPcmChunk(pcm, count, angleDegrees, ssr, true) + maybeEmitAudioStats(false) + } + } + + /** + * Per-chunk processing shared by the G2 and phone-mic paths, downstream of + * decode and the beam filter. hasFrameMeta is false for the phone mic, + * which has no firmware DSP metadata to report. + */ + private fun processPcmChunk(pcm: ShortArray, count: Int, angleDegrees: Int, ssr: Int, hasFrameMeta: Boolean) { + if (suppressionEnabled) { + applySuppression(pcm, count) + } + if (recordingPcm != null) { + appendRecording(pcm, count) + } + val verifyBuffer = this.verifyBuffer + if (verifyBuffer != null && verifyCount < VERIFY_MAX_SAMPLES) { + val copied = Math.min(count, VERIFY_MAX_SAMPLES - verifyCount) + System.arraycopy(pcm, 0, verifyBuffer, verifyCount, copied) + verifyCount += copied + } + if (endpointing && endpointDetector.accept(pcm, count)) { + emitSpeechEnd() + } + // PCM and frame metadata flow in every mode so levels, recording, + // and the Microphones radar keep working alongside onboard ASR. + emitPcm(pcm, count) + if (hasFrameMeta) { + emitFrameMeta(angleDegrees, ssr) + } + if (mode != VoiceInputMode.CLOUD) { + val samples = FloatArray(count) + for (i in 0 until count) { + samples[i] = pcm[i] / 32768.0f + } + processRecognizer(samples) + } + } + + /** + * Open the phone's own microphone at the pipeline's native format + * (16 kHz mono PCM16), or null when it cannot start — the permission is + * missing (SecurityException) or the device refuses the configuration. + */ + private fun openPhoneMic(): android.media.AudioRecord? { + var record: android.media.AudioRecord? = null + try { + val minBytes = android.media.AudioRecord.getMinBufferSize( + SAMPLE_RATE, + android.media.AudioFormat.CHANNEL_IN_MONO, + android.media.AudioFormat.ENCODING_PCM_16BIT) + val bufferBytes = Math.max(minBytes, PHONE_MIC_CHUNK_SAMPLES * 2 * 4) + record = android.media.AudioRecord( + android.media.MediaRecorder.AudioSource.VOICE_RECOGNITION, + SAMPLE_RATE, + android.media.AudioFormat.CHANNEL_IN_MONO, + android.media.AudioFormat.ENCODING_PCM_16BIT, + bufferBytes) + if (record.state != android.media.AudioRecord.STATE_INITIALIZED) { + record.release() + return null + } + record.startRecording() + if (record.recordingState != android.media.AudioRecord.RECORDSTATE_RECORDING) { + record.release() + return null + } + return record + } catch (t: Throwable) { + Log.w(TAG, "phone mic open failed", t) + if (record != null) { + record.release() + } + return null + } + } + + /** + * Phone-mic capture loop: no LC3 decode, no arm bookkeeping, no frame + * metadata — AudioRecord already delivers the pipeline's PCM format. The + * blocking read returns every chunk (50 ms), which bounds how long a + * stop() waits for the loop to notice `started` dropped. + */ + private fun processPhoneAudio(record: android.media.AudioRecord) { + val pcm = ShortArray(PHONE_MIC_CHUNK_SAMPLES) + while (started && !Thread.currentThread().isInterrupted) { + val read = record.read(pcm, 0, pcm.size) + if (read < 0) { + Log.w(TAG, "phone mic read failed: $read") + return + } + if (read == 0) { + continue + } + decodedSamples += read.toLong() + processPcmChunk(pcm, read, 0, 0, false) + } + } + + private fun processRecognizer(samples: FloatArray) { + appendTranscriptSamples(samples) + // Each Whisper call re-encodes the whole buffer plus ~10s of tail + // padding (see the ASR_WHISPER_* comment above), + // so it skips the live-partial redecode Moonshine does on this + // interval and only decodes when a segment commits (8s buffer fill) + // or the utterance ends (decodeTranscript(true) in runLoop()). This + // means no live preview text while speaking in Whisper mode -- status + // stays "Listening..." until release. This is a deliberate tradeoff. + if (onboardModelKind == OnboardModelKind.WHISPER) { + return + } + val now = SystemClock.elapsedRealtime() + if (transcriptSampleCount >= TRANSCRIPT_MIN_SAMPLES + && now - lastTranscriptDecodeAtMs >= TRANSCRIPT_DECODE_INTERVAL_MS) { + decodeTranscript(false) + lastTranscriptDecodeAtMs = now + } + } + + private fun appendTranscriptSamples(samples: FloatArray) { + var sourceOffset = 0 + while (sourceOffset < samples.size) { + val available = TRANSCRIPT_SEGMENT_MAX_SAMPLES - transcriptSampleCount + val count = Math.min(available, samples.size - sourceOffset) + System.arraycopy(samples, sourceOffset, transcriptSamples, transcriptSampleCount, count) + transcriptSampleCount += count + sourceOffset += count + + if (transcriptSampleCount == TRANSCRIPT_SEGMENT_MAX_SAMPLES) { + commitTranscriptSegment() + } + } + } + + /** + * Decode the current model-safe segment and emit the best transcript of the + * complete utterance (REPLACE semantics — the caller displays it as-is). + */ + private fun decodeTranscript(isFinal: Boolean) { + if (recognizer == null || transcriptSampleCount <= 0) { + if (isFinal) { + emitTranscript(lastTranscript, true) + } + return + } + val segmentSampleCount = transcriptSampleCount + var segmentText = recognizeTranscriptSegment(segmentSampleCount) + if (segmentText.length > 0) { + currentSegmentTranscript = segmentText + } else { + segmentText = currentSegmentTranscript + } + val text = joinTranscript(committedTranscript, segmentText) + lastTranscript = text + logTranscriptDecode(isFinal, segmentSampleCount, text) + emitTranscript(text, isFinal) + } + + /** + * Finalize a full segment before accepting more audio. This keeps every + * Moonshine invocation below its failing sequence length while retaining + * all earlier text in the replace-semantics preview. + */ + private fun commitTranscriptSegment() { + val cut = findSegmentCutPoint() + var segmentText = recognizeTranscriptSegment(cut) + if (segmentText.length == 0) { + // Fallback text came from partial decodes of the full buffer, so it + // may include words from the carried-over tail; rare now that decode + // windows are peak-normalized. + segmentText = currentSegmentTranscript + } + committedTranscript = joinTranscript(committedTranscript, segmentText) + currentSegmentTranscript = "" + committedTranscriptSampleCount += cut.toLong() + val tail = transcriptSampleCount - cut + System.arraycopy(transcriptSamples, cut, transcriptSamples, 0, tail) + transcriptSampleCount = tail + lastTranscript = committedTranscript + lastTranscriptDecodeAtMs = SystemClock.elapsedRealtime() + logTranscriptDecode(false, cut, committedTranscript) + emitTranscript(committedTranscript, false) + } + + /** + * Pick where to end the committed segment: the center of the quietest + * window within the search region at the end of the buffer, so the cut + * lands between words instead of splitting one. + */ + private fun findSegmentCutPoint(): Int { + val count = transcriptSampleCount + val searchStart = Math.max(0, count - TRANSCRIPT_CUT_SEARCH_SAMPLES) + val win = TRANSCRIPT_CUT_WINDOW_SAMPLES + if (count - searchStart <= win) { + return count + } + var sum = 0.0 + for (i in searchStart until searchStart + win) { + sum += transcriptSamples[i].toDouble() * transcriptSamples[i] + } + var best = sum + var bestStart = searchStart + var start = searchStart + 1 + while (start + win <= count) { + val dropped = transcriptSamples[start - 1] + val added = transcriptSamples[start + win - 1] + sum += added.toDouble() * added - dropped.toDouble() * dropped + if (sum < best) { + best = sum + bestStart = start + } + start++ + } + return bestStart + win / 2 + } + + private fun recognizeTranscriptSegment(sampleCount: Int): String { + val currentRecognizer = recognizer + if (currentRecognizer == null || sampleCount <= 0) { + return "" + } + val segment = Arrays.copyOf(transcriptSamples, sampleCount) + // Whisper hallucinates text on near-silent input (a known quirk of the + // model, not this pipeline); gate it on the PRE-normalization peak, since + // normalizePeak() below would otherwise amplify true silence right up to + // the target level and hide the very thing being checked for. Moonshine + // does not share this failure mode in practice, so it is left unchanged. + if (onboardModelKind == OnboardModelKind.WHISPER && peakAmplitude(segment) < WHISPER_SILENCE_PEAK_THRESHOLD) { + return "" + } + normalizePeak(segment) + val offlineStream: OfflineStream = currentRecognizer.createStream() + try { + offlineStream.acceptWaveform(segment, SAMPLE_RATE) + currentRecognizer.decode(offlineStream) + val result: OfflineRecognizerResult? = currentRecognizer.getResult(offlineStream) + val raw: String? = if (result == null) "" else result.text + return raw?.trim() ?: "" + } finally { + offlineStream.release() + } + } + + private fun logTranscriptDecode(isFinal: Boolean, segmentSampleCount: Int, text: String) { + val totalAudioSec = + (committedTranscriptSampleCount + transcriptSampleCount) / SAMPLE_RATE.toDouble() + val preview = if (text.length <= TRANSCRIPT_LOG_PREVIEW_CHARS) + text else text.substring(0, TRANSCRIPT_LOG_PREVIEW_CHARS) + "..." + Log.i(TAG, (if (onboardModelKind == OnboardModelKind.WHISPER) "Whisper" else "Moonshine") + " decode final=" + isFinal + + " audioSec=" + String.format(java.util.Locale.US, "%.2f", totalAudioSec) + + " segmentAudioSec=" + String.format(java.util.Locale.US, "%.2f", segmentSampleCount / SAMPLE_RATE.toDouble()) + + " textLen=" + text.length + " text=\"" + preview + "\"") + } + + private fun resetTranscriptState() { + transcriptSampleCount = 0 + committedTranscriptSampleCount = 0 + committedTranscript = "" + currentSegmentTranscript = "" + lastTranscriptDecodeAtMs = 0 + } + + private fun emitPcm(pcm: ShortArray, count: Int) { + val currentListener = listener + if (currentListener == null || count <= 0) { + return + } + val le = ByteArray(count * 2) + for (i in 0 until count) { + val s = pcm[i].toInt() + le[i * 2] = (s and 0xff).toByte() + le[i * 2 + 1] = ((s shr 8) and 0xff).toByte() + } + mainHandler.post { currentListener.onPcm(le) } + } + + /** + * Embed the session's buffered utterance and compare it to the enrolled + * wearer voice-print. Fails open: a session too short to verify, or a + * model that will not load, counts as the wearer rather than silencing + * every command. + */ + private fun runSpeakerVerification() { + val buffer = verifyBuffer + val wearer = verifyWearerEmbedding + val modelPath = verifySpeakerModelPath + verifyBuffer = null + if (buffer == null || wearer == null || modelPath == null) { + return + } + if (verifyCount < VERIFY_MIN_SAMPLES) { + emitSpeakerVerified(true, 0f) + return + } + val speakerId = cachedSpeakerId(modelPath) + try { + val le = ByteArray(verifyCount * 2) + for (i in 0 until verifyCount) { + val s = buffer[i].toInt() + le[i * 2] = (s and 0xff).toByte() + le[i * 2 + 1] = ((s shr 8) and 0xff).toByte() + } + val embedding = speakerId.embed(le, SAMPLE_RATE) + if (embedding == null || embedding.size != wearer.size) { + emitSpeakerVerified(true, 0f) + return + } + var dot = 0.0 + for (i in embedding.indices) { + dot += embedding[i].toDouble() * wearer[i] + } + val isWearer = dot >= verifyThreshold + Log.i(TAG, "speaker verification similarity=" + String.format(java.util.Locale.US, "%.3f", dot) + + " threshold=" + verifyThreshold + " isWearer=" + isWearer) + emitSpeakerVerified(isWearer, dot.toFloat()) + } catch (t: Throwable) { + Log.w(TAG, "speaker verification failed", t) + emitSpeakerVerified(true, 0f) + } + } + + private fun emitSpeakerVerified(isWearer: Boolean, similarity: Float) { + val currentListener = listener + if (currentListener == null) { + return + } + mainHandler.post { currentListener.onSpeakerVerified(isWearer, similarity) } + } + + private fun emitFrameMeta(angleDegrees: Int, ssr: Int) { + val currentListener = listener + if (currentListener == null) { + return + } + mainHandler.post { currentListener.onFrameMeta(angleDegrees, ssr) } + } + + private fun emitSpeechEnd() { + val currentListener = listener + if (currentListener == null) { + return + } + mainHandler.post { currentListener.onSpeechEnd() } + } + + private fun emitStopped(captureId: Int) { + val currentListener = listener + if (currentListener != null) { + mainHandler.post { currentListener.onStopped(captureId) } + } + } + + private fun stopG2Audio() { + val currentCommunicator = communicator + if (currentCommunicator != null) { + currentCommunicator.stopG2AudioCapture() + } + maybeEmitAudioStats(true) + } + + private fun releaseSherpa() { + if (recognizer != null) { + recognizer!!.release() + recognizer = null + } + } + + private fun releaseLc3() { + if (lc3Decoder != null) { + lc3Decoder!!.close() + lc3Decoder = null + } + } + + private fun queueAudioPacket(data: ByteArray?, arm: String?, arrivalMs: Long) { + if (!started || data == null) { + return + } + if ("L" != arm) { + wrongArmPackets++ + } + synchronized(audioQueueLock) { + if (audioQueue.size >= MAX_AUDIO_QUEUE_PACKETS) { + audioQueue.removeFirst() + queueDroppedPackets++ + } + audioQueue.addLast(AudioPacket(data, arm, arrivalMs)) + queuedPackets++ + if (lastPacketArrivalMs > 0) { + val delta = arrivalMs - lastPacketArrivalMs + if (delta > maxInterPacketMs) { + maxInterPacketMs = delta + } + if (delta > LATE_PACKET_INTERVAL_MS) { + latePackets++ + } + } + lastPacketArrivalMs = arrivalMs + (audioQueueLock as java.lang.Object).notifyAll() + } + } + + private fun takeAudioPacket(): AudioPacket? { + synchronized(audioQueueLock) { + while (started && audioQueue.isEmpty()) { + try { + (audioQueueLock as java.lang.Object).wait(250) + maybeEmitAudioStats(false) + } catch (e: InterruptedException) { + Thread.currentThread().interrupt() + return null + } + } + return audioQueue.pollFirst() + } + } + + private fun resetAudioStats() { + queuedPackets = 0 + queueDroppedPackets = 0 + decodedSamples = 0 + latePackets = 0 + wrongArmPackets = 0 + lastPacketArrivalMs = 0 + maxInterPacketMs = 0 + lastStatsAtMs = SystemClock.elapsedRealtime() + } + + private fun maybeEmitAudioStats(force: Boolean) { + val now = SystemClock.elapsedRealtime() + if (!force && now - lastStatsAtMs < STATS_INTERVAL_MS) { + return + } + lastStatsAtMs = now + val currentDecoder = lc3Decoder + val real = if (currentDecoder == null) 0L else currentDecoder.getRealPackets() + val duplicate = if (currentDecoder == null) 0L else currentDecoder.getDuplicatePackets() + val missing = if (currentDecoder == null) 0L else currentDecoder.getMissingPackets() + val decodeErrors = if (currentDecoder == null) 0L else currentDecoder.getDecodeErrors() + val status = "G2 mic packets=" + queuedPackets + + " decoded=" + real + + " missing=" + missing + + " duplicate=" + duplicate + + " late=" + latePackets + + " maxGapMs=" + maxInterPacketMs + + "\n" + + " queueDrop=" + queueDroppedPackets + + " decodeErrors=" + decodeErrors + + " wrongArm=" + wrongArmPackets + + " audioSec=" + String.format(java.util.Locale.US, "%.1f", decodedSamples / SAMPLE_RATE.toDouble()) + // Audio-pipeline stats are diagnostic; keep them in logcat only, out of + // the on-glasses voice UI. + Log.i(TAG, status.replace('\n', ' ') + " expectedIntervalMs=" + EXPECTED_PACKET_INTERVAL_MS) + } + + private fun emitStatus(status: String?) { + val currentListener = listener + if (currentListener == null) { + return + } + mainHandler.post { currentListener.onStatus(status) } + } + + private fun emitTranscript(text: String?, isFinal: Boolean) { + val currentListener = listener + if (currentListener == null) { + return + } + Log.i(TAG, "Emit transcript final=" + isFinal + " textLen=" + (text?.trim()?.length ?: 0)) + mainHandler.post { currentListener.onTranscript(text, isFinal) } + } + + private class AudioPacket(val data: ByteArray?, arm: String?, val arrivalMs: Long) { + val arm: String = arm ?: "?" + } +} diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawWearBridge.java b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawWearBridge.java deleted file mode 100644 index cb0da66f..00000000 --- a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawWearBridge.java +++ /dev/null @@ -1,391 +0,0 @@ -package com.faceclaw.app; - -import android.content.Context; -import android.os.Handler; -import android.os.Looper; -import android.util.Log; - -import com.google.android.gms.common.ConnectionResult; -import com.google.android.gms.common.GoogleApiAvailability; -import com.google.android.gms.wearable.CapabilityClient; -import com.google.android.gms.wearable.CapabilityInfo; -import com.google.android.gms.wearable.DataMap; -import com.google.android.gms.wearable.MessageEvent; -import com.google.android.gms.wearable.Node; -import com.google.android.gms.wearable.PutDataMapRequest; -import com.google.android.gms.wearable.PutDataRequest; -import com.google.android.gms.wearable.Wearable; - -import org.json.JSONException; -import org.json.JSONObject; - -import java.nio.charset.StandardCharsets; -import java.util.ArrayDeque; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.Set; - -/** - * Phone side of the Wear OS integration: the single place that talks to the - * Wearable Data Layer (Google Play services), so the JS side only ever sees - * JSON strings and never touches Play services types. - * - * Inbound: FaceclawWearListenerService hands every watch message to - * {@link #handleMessage}; it is forwarded to the JS listener on the thread - * that registered it (the main isolate), mirroring FaceclawSettings. Messages - * that arrive before a listener exists (the watch poked a phone whose JS - * dashboard is not up yet) are acked with jsReady=false so the watch can say - * "open Faceclaw on your phone"; only idempotent state requests are kept for - * replay on registration — the watch already reported gestures and commands - * as failed, and executing them tens of seconds later would act on whatever - * the glasses happen to show by then. - * - * Outbound: {@link #publishState} mirrors the dashboard state into a Data - * Layer item (delivered even if the watch is out of range right now, and - * available to the watch app the moment it opens), while {@link #sendToWatch} - * sends fire-and-forget messages (acks, assistant text) to every reachable - * watch that advertises the Faceclaw watch-app capability. - * - * Everything is a no-op on phones without Google Play services. - */ -public final class FaceclawWearBridge { - private static final String TAG = "FaceclawWear"; - - /** Advertised by the watch app (wear/app/src/main/res/values/wear.xml). */ - public static final String CAPABILITY_WATCH = "faceclaw_watch"; - /** Advertised by this app (res/values/wear.xml) so the watch can find the phone. */ - public static final String CAPABILITY_PHONE = "faceclaw_phone"; - - public static final String PATH_PREFIX = "/faceclaw"; - public static final String PATH_STATE = "/faceclaw/state"; - public static final String PATH_STATE_REQUEST = "/faceclaw/state/request"; - /** Watch -> phone: the watch's own battery; carries no seq and is never acked. */ - public static final String PATH_WATCH_BATTERY = "/faceclaw/battery"; - /** Phone -> watch: answered with PATH_WATCH_BATTERY even while the watch app is closed. */ - public static final String PATH_WATCH_BATTERY_REQUEST = "/faceclaw/battery/request"; - public static final String PATH_ACK = "/faceclaw/ack"; - public static final String PATH_EVENT = "/faceclaw/event"; - - private static final String STATE_KEY_JSON = "json"; - private static final String STATE_KEY_UPDATED_AT = "updatedAt"; - - /** Messages kept for a listener that is not registered yet. */ - private static final int PENDING_LIMIT = 16; - private static final long PENDING_MAX_AGE_MS = 30_000; - - private static volatile FaceclawWearBridge instance; - - private final Context context; - private final boolean available; - private final Object lock = new Object(); - private FaceclawWearListener listener; - private Handler listenerHandler; - private final ArrayDeque pending = new ArrayDeque<>(); - private final List watchNodes = new ArrayList<>(); - private String lastPublishedState = null; - - private static final class PendingMessage { - final String path; - final String json; - final String nodeId; - final long receivedAt; - - PendingMessage(String path, String json, String nodeId, long receivedAt) { - this.path = path; - this.json = json; - this.nodeId = nodeId; - this.receivedAt = receivedAt; - } - } - - private FaceclawWearBridge(Context context) { - this.context = context.getApplicationContext(); - boolean playServices; - try { - playServices = GoogleApiAvailability.getInstance() - .isGooglePlayServicesAvailable(this.context) == ConnectionResult.SUCCESS; - } catch (Throwable error) { - Log.w(TAG, "Play services availability check failed", error); - playServices = false; - } - this.available = playServices; - if (!available) { - Log.i(TAG, "Google Play services unavailable; watch integration disabled"); - } - } - - /** Initialize (idempotent) and return the singleton. */ - public static FaceclawWearBridge getInstance(Context context) { - if (instance == null) { - synchronized (FaceclawWearBridge.class) { - if (instance == null) { - instance = new FaceclawWearBridge(context); - } - } - } - return instance; - } - - /** Whether the Wearable Data Layer can be used on this phone at all. */ - public boolean isAvailable() { - return available; - } - - /** - * Register the JS listener. Must be called from the thread whose isolate - * owns it; that thread's Looper is captured for dispatch. Replays any - * recently queued watch messages and refreshes the reachable-watch set. - */ - public void setListener(FaceclawWearListener newListener) { - Looper looper = Looper.myLooper(); - if (looper == null) { - Log.w(TAG, "wear listener registered from a Looper-less thread; it will never be notified"); - } - List replay; - synchronized (lock) { - listener = newListener; - listenerHandler = looper != null ? new Handler(looper) : null; - replay = new ArrayList<>(pending); - pending.clear(); - } - long now = System.currentTimeMillis(); - for (PendingMessage message : replay) { - if (now - message.receivedAt <= PENDING_MAX_AGE_MS) { - dispatchMessage(message.path, message.json, message.nodeId); - } - } - refreshWatchNodes(); - } - - public void clearListener() { - synchronized (lock) { - listener = null; - listenerHandler = null; - } - } - - public boolean isWatchReachable() { - synchronized (lock) { - return !watchNodes.isEmpty(); - } - } - - /** Display name of a reachable watch ("" if none). */ - public String getWatchName() { - synchronized (lock) { - return watchNodes.isEmpty() ? "" : watchNodes.get(0).getDisplayName(); - } - } - - /** Re-query which watches advertise the Faceclaw watch-app capability. */ - public void refreshWatchNodes() { - if (!available) return; - try { - Wearable.getCapabilityClient(context) - .getCapability(CAPABILITY_WATCH, CapabilityClient.FILTER_REACHABLE) - .addOnSuccessListener(this::updateWatchNodes) - .addOnFailureListener(error -> Log.w(TAG, "capability query failed", error)); - } catch (Throwable error) { - Log.w(TAG, "capability query failed", error); - } - } - - /** - * Mirror the dashboard state to the watch. Identical consecutive states - * are not re-sent (the Data Layer would drop them anyway). - */ - public void publishState(String json) { - publishState(json, false); - } - - /** As {@link #publishState(String)}; `force` re-sends an unchanged state. */ - public void publishState(String json, boolean force) { - if (!available || json == null) return; - synchronized (lock) { - if (!force && json.equals(lastPublishedState)) return; - lastPublishedState = json; - } - try { - PutDataMapRequest request = PutDataMapRequest.create(PATH_STATE); - DataMap map = request.getDataMap(); - map.putString(STATE_KEY_JSON, json); - map.putLong(STATE_KEY_UPDATED_AT, System.currentTimeMillis()); - PutDataRequest put = request.asPutDataRequest().setUrgent(); - Wearable.getDataClient(context) - .putDataItem(put) - .addOnFailureListener(error -> Log.w(TAG, "state publish failed", error)); - } catch (Throwable error) { - Log.w(TAG, "state publish failed", error); - } - } - - /** Send a message to every reachable watch running the Faceclaw watch app. */ - public void sendToWatch(String path, String json) { - if (!available) return; - List targets; - synchronized (lock) { - targets = new ArrayList<>(watchNodes); - } - if (targets.isEmpty()) { - // Nothing known yet (first message after boot): look the watch up, then send. - try { - Wearable.getCapabilityClient(context) - .getCapability(CAPABILITY_WATCH, CapabilityClient.FILTER_REACHABLE) - .addOnSuccessListener(info -> { - updateWatchNodes(info); - for (Node node : info.getNodes()) { - sendToNode(node.getId(), path, json); - } - }) - .addOnFailureListener(error -> Log.w(TAG, "capability query failed", error)); - } catch (Throwable error) { - Log.w(TAG, "capability query failed", error); - } - return; - } - for (Node node : targets) { - sendToNode(node.getId(), path, json); - } - } - - /** Send a message to one specific node (e.g. an ack back to the sender). */ - public void sendToNode(String nodeId, String path, String json) { - if (!available || nodeId == null || nodeId.isEmpty()) return; - try { - byte[] payload = (json != null ? json : "{}").getBytes(StandardCharsets.UTF_8); - Wearable.getMessageClient(context) - .sendMessage(nodeId, path, payload) - .addOnFailureListener(error -> Log.w(TAG, "send " + path + " to " + nodeId + " failed", error)); - } catch (Throwable error) { - Log.w(TAG, "send " + path + " failed", error); - } - } - - /** Reply to a watch message; `seq` echoes the watch's sequence number. */ - public void sendAck(String nodeId, long seq, boolean ok, boolean jsReady, String message) { - JSONObject ack = new JSONObject(); - try { - ack.put("seq", seq); - ack.put("ok", ok); - ack.put("jsReady", jsReady); - ack.put("message", message != null ? message : ""); - } catch (JSONException ignored) { - // The keys are constants; this cannot happen. - } - sendToNode(nodeId, PATH_ACK, ack.toString()); - } - - /** Entry point for FaceclawWearListenerService; may run on any thread. */ - void handleMessage(MessageEvent event) { - if (event == null) return; - String path = event.getPath(); - if (path == null || !path.startsWith(PATH_PREFIX)) return; - byte[] data = event.getData(); - String json = data != null && data.length > 0 ? new String(data, StandardCharsets.UTF_8) : "{}"; - String nodeId = event.getSourceNodeId(); - Log.d(TAG, "recv " + path + " seq=" + readSeq(json)); - - boolean delivered = dispatchMessage(path, json, nodeId); - if (delivered) return; - // A battery report answers a poll the dashboard sent; with no - // dashboard to receive it, it is simply dropped (the dashboard polls - // again when it starts), and the watch does not expect an ack. - if (PATH_WATCH_BATTERY.equals(path)) return; - - // No JS listener yet: tell the watch so it can explain why nothing is - // happening. Only state requests are kept for replay (see class doc); - // everything else is dropped after the failure ack. - if (PATH_STATE_REQUEST.equals(path)) { - synchronized (lock) { - long now = System.currentTimeMillis(); - while (!pending.isEmpty() && now - pending.peekFirst().receivedAt > PENDING_MAX_AGE_MS) { - pending.pollFirst(); - } - if (pending.size() >= PENDING_LIMIT) { - pending.pollFirst(); - } - pending.addLast(new PendingMessage(path, json, nodeId, now)); - } - } - sendAck(nodeId, readSeq(json), false, false, "Faceclaw is not running on the phone. Open it to connect the glasses."); - } - - /** Entry point for FaceclawWearListenerService's capability callback. */ - void handleCapabilityChanged(CapabilityInfo info) { - if (info == null || !CAPABILITY_WATCH.equals(info.getName())) return; - updateWatchNodes(info); - } - - private void updateWatchNodes(CapabilityInfo info) { - Set nodes = info != null ? info.getNodes() : Collections.emptySet(); - // Every reachable watch gets messages; a nearby (Bluetooth-linked) - // one is listed first so it is the name the phone UI shows. - List reachable = new ArrayList<>(); - for (Node node : nodes) { - if (node.isNearby()) reachable.add(0, node); else reachable.add(node); - } - boolean changed; - String name; - synchronized (lock) { - changed = !sameNodeIds(watchNodes, reachable); - watchNodes.clear(); - watchNodes.addAll(reachable); - name = watchNodes.isEmpty() ? "" : watchNodes.get(0).getDisplayName(); - } - if (changed) { - Log.i(TAG, reachable.isEmpty() ? "no Faceclaw watch reachable" : "watch reachable: " + name); - boolean reachableNow = !reachable.isEmpty(); - FaceclawWearListener target; - Handler handler; - synchronized (lock) { - target = listener; - handler = listenerHandler; - } - if (target != null && handler != null) { - handler.post(() -> { - try { - target.onWatchConnection(reachableNow, name); - } catch (Exception error) { - Log.w(TAG, "watch connection listener failed", error); - } - }); - } - } - } - - private static boolean sameNodeIds(List a, List b) { - if (a.size() != b.size()) return false; - for (int i = 0; i < a.size(); i++) { - if (!a.get(i).getId().equals(b.get(i).getId())) return false; - } - return true; - } - - /** Post to the JS listener; false when there is none to post to. */ - private boolean dispatchMessage(String path, String json, String nodeId) { - FaceclawWearListener target; - Handler handler; - synchronized (lock) { - target = listener; - handler = listenerHandler; - } - if (target == null || handler == null) return false; - handler.post(() -> { - try { - target.onMessage(path, json, nodeId); - } catch (Exception error) { - Log.w(TAG, "wear message listener failed for " + path, error); - } - }); - return true; - } - - private static long readSeq(String json) { - try { - return new JSONObject(json).optLong("seq", 0); - } catch (JSONException error) { - return 0; - } - } -} diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawWearBridge.kt b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawWearBridge.kt new file mode 100644 index 00000000..5be4a511 --- /dev/null +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawWearBridge.kt @@ -0,0 +1,386 @@ +package com.faceclaw.app + +import android.content.Context +import android.os.Handler +import android.os.Looper +import android.util.Log + +import com.google.android.gms.common.ConnectionResult +import com.google.android.gms.common.GoogleApiAvailability +import com.google.android.gms.wearable.CapabilityClient +import com.google.android.gms.wearable.CapabilityInfo +import com.google.android.gms.wearable.DataMap +import com.google.android.gms.wearable.MessageEvent +import com.google.android.gms.wearable.Node +import com.google.android.gms.wearable.PutDataMapRequest +import com.google.android.gms.wearable.PutDataRequest +import com.google.android.gms.wearable.Wearable + +import org.json.JSONException +import org.json.JSONObject + +import java.nio.charset.StandardCharsets +import java.util.ArrayDeque +import java.util.ArrayList +import java.util.Collections + +/** + * Phone side of the Wear OS integration: the single place that talks to the + * Wearable Data Layer (Google Play services), so the JS side only ever sees + * JSON strings and never touches Play services types. + * + * Inbound: FaceclawWearListenerService hands every watch message to + * [handleMessage]; it is forwarded to the JS listener on the thread + * that registered it (the main isolate), mirroring FaceclawSettings. Messages + * that arrive before a listener exists (the watch poked a phone whose JS + * dashboard is not up yet) are acked with jsReady=false so the watch can say + * "open Faceclaw on your phone"; only idempotent state requests are kept for + * replay on registration — the watch already reported gestures and commands + * as failed, and executing them tens of seconds later would act on whatever + * the glasses happen to show by then. + * + * Outbound: [publishState] mirrors the dashboard state into a Data + * Layer item (delivered even if the watch is out of range right now, and + * available to the watch app the moment it opens), while [sendToWatch] + * sends fire-and-forget messages (acks, assistant text) to every reachable + * watch that advertises the Faceclaw watch-app capability. + * + * Everything is a no-op on phones without Google Play services. + */ +class FaceclawWearBridge private constructor(context: Context) { + companion object { + private const val TAG = "FaceclawWear" + + /** Advertised by the watch app (wear/app/src/main/res/values/wear.xml). */ + const val CAPABILITY_WATCH = "faceclaw_watch" + /** Advertised by this app (res/values/wear.xml) so the watch can find the phone. */ + const val CAPABILITY_PHONE = "faceclaw_phone" + + const val PATH_PREFIX = "/faceclaw" + const val PATH_STATE = "/faceclaw/state" + const val PATH_STATE_REQUEST = "/faceclaw/state/request" + /** Watch -> phone: the watch's own battery; carries no seq and is never acked. */ + const val PATH_WATCH_BATTERY = "/faceclaw/battery" + /** Phone -> watch: answered with PATH_WATCH_BATTERY even while the watch app is closed. */ + const val PATH_WATCH_BATTERY_REQUEST = "/faceclaw/battery/request" + const val PATH_ACK = "/faceclaw/ack" + const val PATH_EVENT = "/faceclaw/event" + + private const val STATE_KEY_JSON = "json" + private const val STATE_KEY_UPDATED_AT = "updatedAt" + + /** Messages kept for a listener that is not registered yet. */ + private const val PENDING_LIMIT = 16 + private const val PENDING_MAX_AGE_MS = 30_000L + + @Volatile + private var instance: FaceclawWearBridge? = null + + /** Initialize (idempotent) and return the singleton. */ + @JvmStatic + fun getInstance(context: Context): FaceclawWearBridge { + if (instance == null) { + synchronized(FaceclawWearBridge::class.java) { + if (instance == null) { + instance = FaceclawWearBridge(context) + } + } + } + return instance!! + } + + private fun sameNodeIds(a: List, b: List): Boolean { + if (a.size != b.size) return false + for (i in a.indices) { + if (a[i].id != b[i].id) return false + } + return true + } + + private fun readSeq(json: String): Long { + try { + return JSONObject(json).optLong("seq", 0) + } catch (error: JSONException) { + return 0 + } + } + } + + private val context: Context + private val available: Boolean + private val lock = Any() + private var listener: FaceclawWearListener? = null + private var listenerHandler: Handler? = null + private val pending = ArrayDeque() + private val watchNodes: MutableList = ArrayList() + private var lastPublishedState: String? = null + + private class PendingMessage( + val path: String, + val json: String, + val nodeId: String?, + val receivedAt: Long, + ) + + init { + this.context = context.applicationContext + var playServices: Boolean + try { + playServices = GoogleApiAvailability.getInstance() + .isGooglePlayServicesAvailable(this.context) == ConnectionResult.SUCCESS + } catch (error: Throwable) { + Log.w(TAG, "Play services availability check failed", error) + playServices = false + } + this.available = playServices + if (!available) { + Log.i(TAG, "Google Play services unavailable; watch integration disabled") + } + } + + /** Whether the Wearable Data Layer can be used on this phone at all. */ + fun isAvailable(): Boolean { + return available + } + + /** + * Register the JS listener. Must be called from the thread whose isolate + * owns it; that thread's Looper is captured for dispatch. Replays any + * recently queued watch messages and refreshes the reachable-watch set. + */ + fun setListener(newListener: FaceclawWearListener?) { + val looper = Looper.myLooper() + if (looper == null) { + Log.w(TAG, "wear listener registered from a Looper-less thread; it will never be notified") + } + val replay: List + synchronized(lock) { + listener = newListener + listenerHandler = if (looper != null) Handler(looper) else null + replay = ArrayList(pending) + pending.clear() + } + val now = System.currentTimeMillis() + for (message in replay) { + if (now - message.receivedAt <= PENDING_MAX_AGE_MS) { + dispatchMessage(message.path, message.json, message.nodeId) + } + } + refreshWatchNodes() + } + + fun clearListener() { + synchronized(lock) { + listener = null + listenerHandler = null + } + } + + fun isWatchReachable(): Boolean { + synchronized(lock) { + return !watchNodes.isEmpty() + } + } + + /** Display name of a reachable watch ("" if none). */ + fun getWatchName(): String { + synchronized(lock) { + return if (watchNodes.isEmpty()) "" else watchNodes[0].displayName + } + } + + /** Re-query which watches advertise the Faceclaw watch-app capability. */ + fun refreshWatchNodes() { + if (!available) return + try { + Wearable.getCapabilityClient(context) + .getCapability(CAPABILITY_WATCH, CapabilityClient.FILTER_REACHABLE) + .addOnSuccessListener { info -> updateWatchNodes(info) } + .addOnFailureListener { error -> Log.w(TAG, "capability query failed", error) } + } catch (error: Throwable) { + Log.w(TAG, "capability query failed", error) + } + } + + /** + * Mirror the dashboard state to the watch. Identical consecutive states + * are not re-sent (the Data Layer would drop them anyway). + */ + fun publishState(json: String?) { + publishState(json, false) + } + + /** As [publishState]; `force` re-sends an unchanged state. */ + fun publishState(json: String?, force: Boolean) { + if (!available || json == null) return + synchronized(lock) { + if (!force && json == lastPublishedState) return + lastPublishedState = json + } + try { + val request = PutDataMapRequest.create(PATH_STATE) + val map: DataMap = request.dataMap + map.putString(STATE_KEY_JSON, json) + map.putLong(STATE_KEY_UPDATED_AT, System.currentTimeMillis()) + val put: PutDataRequest = request.asPutDataRequest().setUrgent() + Wearable.getDataClient(context) + .putDataItem(put) + .addOnFailureListener { error -> Log.w(TAG, "state publish failed", error) } + } catch (error: Throwable) { + Log.w(TAG, "state publish failed", error) + } + } + + /** Send a message to every reachable watch running the Faceclaw watch app. */ + fun sendToWatch(path: String, json: String?) { + if (!available) return + val targets: List + synchronized(lock) { + targets = ArrayList(watchNodes) + } + if (targets.isEmpty()) { + // Nothing known yet (first message after boot): look the watch up, then send. + try { + Wearable.getCapabilityClient(context) + .getCapability(CAPABILITY_WATCH, CapabilityClient.FILTER_REACHABLE) + .addOnSuccessListener { info -> + updateWatchNodes(info) + for (node in info.nodes) { + sendToNode(node.id, path, json) + } + } + .addOnFailureListener { error -> Log.w(TAG, "capability query failed", error) } + } catch (error: Throwable) { + Log.w(TAG, "capability query failed", error) + } + return + } + for (node in targets) { + sendToNode(node.id, path, json) + } + } + + /** Send a message to one specific node (e.g. an ack back to the sender). */ + fun sendToNode(nodeId: String?, path: String, json: String?) { + if (!available || nodeId == null || nodeId.isEmpty()) return + try { + val payload = (json ?: "{}").toByteArray(StandardCharsets.UTF_8) + Wearable.getMessageClient(context) + .sendMessage(nodeId, path, payload) + .addOnFailureListener { error -> Log.w(TAG, "send $path to $nodeId failed", error) } + } catch (error: Throwable) { + Log.w(TAG, "send $path failed", error) + } + } + + /** Reply to a watch message; `seq` echoes the watch's sequence number. */ + fun sendAck(nodeId: String?, seq: Long, ok: Boolean, jsReady: Boolean, message: String?) { + val ack = JSONObject() + try { + ack.put("seq", seq) + ack.put("ok", ok) + ack.put("jsReady", jsReady) + ack.put("message", message ?: "") + } catch (ignored: JSONException) { + // The keys are constants; this cannot happen. + } + sendToNode(nodeId, PATH_ACK, ack.toString()) + } + + /** Entry point for FaceclawWearListenerService; may run on any thread. */ + internal fun handleMessage(event: MessageEvent?) { + if (event == null) return + val path: String? = event.path + if (path == null || !path.startsWith(PATH_PREFIX)) return + val data: ByteArray? = event.data + val json = if (data != null && data.size > 0) String(data, StandardCharsets.UTF_8) else "{}" + val nodeId: String? = event.sourceNodeId + Log.d(TAG, "recv " + path + " seq=" + readSeq(json)) + + val delivered = dispatchMessage(path, json, nodeId) + if (delivered) return + // A battery report answers a poll the dashboard sent; with no + // dashboard to receive it, it is simply dropped (the dashboard polls + // again when it starts), and the watch does not expect an ack. + if (PATH_WATCH_BATTERY == path) return + + // No JS listener yet: tell the watch so it can explain why nothing is + // happening. Only state requests are kept for replay (see class doc); + // everything else is dropped after the failure ack. + if (PATH_STATE_REQUEST == path) { + synchronized(lock) { + val now = System.currentTimeMillis() + while (!pending.isEmpty() && now - pending.peekFirst().receivedAt > PENDING_MAX_AGE_MS) { + pending.pollFirst() + } + if (pending.size >= PENDING_LIMIT) { + pending.pollFirst() + } + pending.addLast(PendingMessage(path, json, nodeId, now)) + } + } + sendAck(nodeId, readSeq(json), false, false, "Faceclaw is not running on the phone. Open it to connect the glasses.") + } + + /** Entry point for FaceclawWearListenerService's capability callback. */ + internal fun handleCapabilityChanged(info: CapabilityInfo?) { + if (info == null || CAPABILITY_WATCH != info.name) return + updateWatchNodes(info) + } + + private fun updateWatchNodes(info: CapabilityInfo?) { + val nodes: Set = if (info != null) info.nodes else Collections.emptySet() + // Every reachable watch gets messages; a nearby (Bluetooth-linked) + // one is listed first so it is the name the phone UI shows. + val reachable: MutableList = ArrayList() + for (node in nodes) { + if (node.isNearby) reachable.add(0, node) else reachable.add(node) + } + val changed: Boolean + val name: String + synchronized(lock) { + changed = !sameNodeIds(watchNodes, reachable) + watchNodes.clear() + watchNodes.addAll(reachable) + name = if (watchNodes.isEmpty()) "" else watchNodes[0].displayName + } + if (changed) { + Log.i(TAG, if (reachable.isEmpty()) "no Faceclaw watch reachable" else "watch reachable: $name") + val reachableNow = !reachable.isEmpty() + val target: FaceclawWearListener? + val handler: Handler? + synchronized(lock) { + target = listener + handler = listenerHandler + } + if (target != null && handler != null) { + handler.post { + try { + target.onWatchConnection(reachableNow, name) + } catch (error: Exception) { + Log.w(TAG, "watch connection listener failed", error) + } + } + } + } + } + + /** Post to the JS listener; false when there is none to post to. */ + private fun dispatchMessage(path: String, json: String, nodeId: String?): Boolean { + val target: FaceclawWearListener? + val handler: Handler? + synchronized(lock) { + target = listener + handler = listenerHandler + } + if (target == null || handler == null) return false + handler.post { + try { + target.onMessage(path, json, nodeId) + } catch (error: Exception) { + Log.w(TAG, "wear message listener failed for $path", error) + } + } + return true + } +} diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawWearListenerService.java b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawWearListenerService.java deleted file mode 100644 index 7520d780..00000000 --- a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawWearListenerService.java +++ /dev/null @@ -1,26 +0,0 @@ -package com.faceclaw.app; - -import android.util.Log; - -import com.google.android.gms.wearable.CapabilityInfo; -import com.google.android.gms.wearable.MessageEvent; -import com.google.android.gms.wearable.WearableListenerService; - -/** - * Play services starts this service (in the app process) for every Data - * Layer message under /faceclaw and for capability changes, whether or not - * the app is running. It only forwards to FaceclawWearBridge, which owns the - * hand-off to the JS side. - */ -public class FaceclawWearListenerService extends WearableListenerService { - @Override - public void onMessageReceived(MessageEvent event) { - Log.d("FaceclawWear", "service onMessageReceived " + event.getPath() + " from " + event.getSourceNodeId()); - FaceclawWearBridge.getInstance(this).handleMessage(event); - } - - @Override - public void onCapabilityChanged(CapabilityInfo info) { - FaceclawWearBridge.getInstance(this).handleCapabilityChanged(info); - } -} diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawWearListenerService.kt b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawWearListenerService.kt new file mode 100644 index 00000000..ed0ce360 --- /dev/null +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawWearListenerService.kt @@ -0,0 +1,24 @@ +package com.faceclaw.app + +import android.util.Log + +import com.google.android.gms.wearable.CapabilityInfo +import com.google.android.gms.wearable.MessageEvent +import com.google.android.gms.wearable.WearableListenerService + +/** + * Play services starts this service (in the app process) for every Data + * Layer message under /faceclaw and for capability changes, whether or not + * the app is running. It only forwards to FaceclawWearBridge, which owns the + * hand-off to the JS side. + */ +class FaceclawWearListenerService : WearableListenerService() { + override fun onMessageReceived(event: MessageEvent) { + Log.d("FaceclawWear", "service onMessageReceived " + event.path + " from " + event.sourceNodeId) + FaceclawWearBridge.getInstance(this).handleMessage(event) + } + + override fun onCapabilityChanged(info: CapabilityInfo) { + FaceclawWearBridge.getInstance(this).handleCapabilityChanged(info) + } +} diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawWebSocket.java b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawWebSocket.java deleted file mode 100644 index f228ed7a..00000000 --- a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawWebSocket.java +++ /dev/null @@ -1,151 +0,0 @@ -package com.faceclaw.app; - -import android.os.Handler; -import android.os.Looper; -import android.util.Log; - -import java.util.concurrent.TimeUnit; - -import okhttp3.OkHttpClient; -import okhttp3.Request; -import okhttp3.Response; -import okhttp3.WebSocket; -import okhttp3.WebSocketListener; -import okio.ByteString; - -/** - * Thin okhttp WebSocket wrapper for the TypeScript side. Sends text frames - * (JSON protocols) and binary frames (raw audio for Soniox). Listener callbacks are - * posted to the Looper of the thread that constructed this object, so a JS - * isolate (main thread or app worker) always receives them on its own - * thread; a Looper-less constructing thread falls back to the main thread. - */ -public class FaceclawWebSocket { - private static final String TAG = "FaceclawWebSocket"; - private static volatile OkHttpClient sharedClient; - - private final Handler callbackHandler; - private final WebSocket socket; - private volatile boolean closeRequested; - - // Single constructor (no overloads) so NativeScript constructor resolution - // can never pick a variant that drops the auth header; callers with no - // header pass null/null. - public FaceclawWebSocket(String url, final FaceclawWebSocketListener listener, String headerName, String headerValue) { - if (url == null || url.trim().isEmpty()) { - throw new IllegalArgumentException("url is required"); - } - if (listener == null) { - throw new IllegalArgumentException("listener is required"); - } - Looper looper = Looper.myLooper(); - callbackHandler = new Handler(looper != null ? looper : Looper.getMainLooper()); - Request.Builder builder = new Request.Builder().url(url.trim()); - if (headerName != null && !headerName.isEmpty() && headerValue != null) { - builder.addHeader(headerName, headerValue); - } - Request request = builder.build(); - // Redacted diagnostics: confirm the auth header is actually present on - // the handshake and carries a plausible value (not empty/truncated). - StringBuilder headerLog = new StringBuilder(); - for (String name : request.headers().names()) { - String value = request.header(name); - headerLog.append(name).append('=').append(redact(value)).append(' '); - } - Log.i(TAG, "ws connect host=" + request.url().host() + request.url().encodedPath() - + " headers=[" + headerLog.toString().trim() + "]"); - socket = getClient().newWebSocket(request, new WebSocketListener() { - @Override public void onOpen(WebSocket webSocket, Response response) { - callbackHandler.post(() -> { - try { - listener.onOpen(); - } catch (Throwable t) { - Log.w(TAG, "listener onOpen failed", t); - } - }); - } - - @Override public void onMessage(WebSocket webSocket, String text) { - callbackHandler.post(() -> { - try { - listener.onTextMessage(text); - } catch (Throwable t) { - Log.w(TAG, "listener onTextMessage failed", t); - } - }); - } - - @Override public void onClosing(WebSocket webSocket, int code, String reason) { - webSocket.close(code, reason); - } - - @Override public void onClosed(WebSocket webSocket, int code, String reason) { - callbackHandler.post(() -> { - try { - listener.onClosed(code, reason == null ? "" : reason); - } catch (Throwable t) { - Log.w(TAG, "listener onClosed failed", t); - } - }); - } - - @Override public void onFailure(WebSocket webSocket, Throwable t, Response response) { - if (closeRequested) { - return; - } - final String message = t == null ? "unknown websocket failure" : String.valueOf(t); - callbackHandler.post(() -> { - try { - listener.onFailure(message); - } catch (Throwable inner) { - Log.w(TAG, "listener onFailure failed", inner); - } - }); - } - }); - } - - private static OkHttpClient getClient() { - OkHttpClient client = sharedClient; - if (client == null) { - synchronized (FaceclawWebSocket.class) { - if (sharedClient == null) { - sharedClient = new OkHttpClient.Builder() - .pingInterval(20, TimeUnit.SECONDS) - .addInterceptor(FaceclawHttp.userAgentInterceptor()) - .build(); - } - client = sharedClient; - } - } - return client; - } - - private static String redact(String value) { - if (value == null) { - return "null"; - } - int len = value.length(); - String prefix = value.substring(0, Math.min(4, len)); - return "len" + len + ":" + prefix + "..."; - } - - public boolean sendText(String message) { - return socket.send(message == null ? "" : message); - } - - public boolean sendBinary(byte[] bytes) { - return socket.send(ByteString.of(bytes == null ? new byte[0] : bytes)); - } - - public void close(int code, String reason) { - closeRequested = true; - try { - if (!socket.close(code, reason)) { - socket.cancel(); - } - } catch (Throwable t) { - socket.cancel(); - } - } -} diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawWebSocket.kt b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawWebSocket.kt new file mode 100644 index 00000000..f4a793bd --- /dev/null +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawWebSocket.kt @@ -0,0 +1,156 @@ +package com.faceclaw.app + +import android.os.Handler +import android.os.Looper +import android.util.Log + +import java.util.concurrent.TimeUnit + +import okhttp3.OkHttpClient +import okhttp3.Request +import okhttp3.Response +import okhttp3.WebSocket +import okhttp3.WebSocketListener +import okio.ByteString.Companion.toByteString + +/** + * Thin okhttp WebSocket wrapper for the TypeScript side. Sends text frames + * (JSON protocols) and binary frames (raw audio for Soniox). Listener callbacks are + * posted to the Looper of the thread that constructed this object, so a JS + * isolate (main thread or app worker) always receives them on its own + * thread; a Looper-less constructing thread falls back to the main thread. + */ +class FaceclawWebSocket +// Single constructor (no overloads) so NativeScript constructor resolution +// can never pick a variant that drops the auth header; callers with no +// header pass null/null. +constructor(url: String?, listener: FaceclawWebSocketListener?, headerName: String?, headerValue: String?) { + private val callbackHandler: Handler + private val socket: WebSocket + @Volatile + private var closeRequested = false + + init { + if (url == null || url.trim().isEmpty()) { + throw IllegalArgumentException("url is required") + } + if (listener == null) { + throw IllegalArgumentException("listener is required") + } + val looper = Looper.myLooper() + callbackHandler = Handler(looper ?: Looper.getMainLooper()) + val builder = Request.Builder().url(url.trim()) + if (headerName != null && !headerName.isEmpty() && headerValue != null) { + builder.addHeader(headerName, headerValue) + } + val request = builder.build() + // Redacted diagnostics: confirm the auth header is actually present on + // the handshake and carries a plausible value (not empty/truncated). + val headerLog = StringBuilder() + for (name in request.headers.names()) { + val value = request.header(name) + headerLog.append(name).append('=').append(redact(value)).append(' ') + } + Log.i(TAG, "ws connect host=" + request.url.host + request.url.encodedPath + + " headers=[" + headerLog.toString().trim() + "]") + socket = getClient().newWebSocket(request, object : WebSocketListener() { + override fun onOpen(webSocket: WebSocket, response: Response) { + callbackHandler.post { + try { + listener.onOpen() + } catch (t: Throwable) { + Log.w(TAG, "listener onOpen failed", t) + } + } + } + + override fun onMessage(webSocket: WebSocket, text: String) { + callbackHandler.post { + try { + listener.onTextMessage(text) + } catch (t: Throwable) { + Log.w(TAG, "listener onTextMessage failed", t) + } + } + } + + override fun onClosing(webSocket: WebSocket, code: Int, reason: String) { + webSocket.close(code, reason) + } + + override fun onClosed(webSocket: WebSocket, code: Int, reason: String) { + callbackHandler.post { + try { + listener.onClosed(code, reason) + } catch (t: Throwable) { + Log.w(TAG, "listener onClosed failed", t) + } + } + } + + override fun onFailure(webSocket: WebSocket, t: Throwable, response: Response?) { + if (closeRequested) { + return + } + val message = t.toString() + callbackHandler.post { + try { + listener.onFailure(message) + } catch (inner: Throwable) { + Log.w(TAG, "listener onFailure failed", inner) + } + } + } + }) + } + + companion object { + private const val TAG = "FaceclawWebSocket" + @Volatile + private var sharedClient: OkHttpClient? = null + + private fun getClient(): OkHttpClient { + var client = sharedClient + if (client == null) { + synchronized(FaceclawWebSocket::class.java) { + if (sharedClient == null) { + sharedClient = OkHttpClient.Builder() + .pingInterval(20, TimeUnit.SECONDS) + .addInterceptor(FaceclawHttp.userAgentInterceptor()) + .build() + } + client = sharedClient + } + } + return client!! + } + + private fun redact(value: String?): String { + if (value == null) { + return "null" + } + val len = value.length + val prefix = value.substring(0, Math.min(4, len)) + return "len$len:$prefix..." + } + } + + fun sendText(message: String?): Boolean { + return socket.send(message ?: "") + } + + fun sendBinary(bytes: ByteArray?): Boolean { + return socket.send((bytes ?: ByteArray(0)).toByteString()) + } + + fun close(code: Int, reason: String?) { + closeRequested = true + try { + if (!socket.close(code, reason)) { + socket.cancel() + } + } catch (t: Throwable) { + socket.cancel() + } + } +} diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FontFileRenderer.java b/App_Resources/Android/src/main/java/com/faceclaw/app/FontFileRenderer.java deleted file mode 100644 index 0eca05e5..00000000 --- a/App_Resources/Android/src/main/java/com/faceclaw/app/FontFileRenderer.java +++ /dev/null @@ -1,537 +0,0 @@ -package com.faceclaw.app; - -import android.graphics.Bitmap; -import android.graphics.Canvas; -import android.graphics.Color; -import android.graphics.Paint; -import android.graphics.Typeface; -import android.os.Build; -import android.util.Log; - -import java.io.File; -import java.io.RandomAccessFile; -import java.nio.ByteBuffer; -import java.nio.ByteOrder; -import java.util.LinkedHashMap; -import java.util.Map; - -/** - * Rasterizes text using font files (TTF/OTF/TTC) via the Android text stack, - * so shaping, kerning, and antialiasing are all delegated to - * minikin/HarfBuzz. Output uses the same grayscale packet format as - * ImageFileLoader ([widthLo, widthHi, heightLo, heightHi, pixels...], one - * byte per pixel, row-major), which the TS side turns into a GrayImage and - * the compositor later quantizes to the 4bpp the firmware wants. - * - * The gamma parameter maps antialiased coverage to output shade: - * out = 255 * (coverage/255)^gamma. The G2 display response looks roughly - * linear, so 1.0 is the expected default; the font previewer exposes it for - * on-hardware comparison. - */ -public final class FontFileRenderer { - private static final String TAG = "FontFileRenderer"; - private static final int TYPEFACE_CACHE_SIZE = 4; - - /** Small LRU of loaded typefaces keyed by file path. */ - private static final Map typefaceCache = - new LinkedHashMap(TYPEFACE_CACHE_SIZE, 0.75f, true) { - @Override - protected boolean removeEldestEntry(Map.Entry eldest) { - return size() > TYPEFACE_CACHE_SIZE; - } - }; - - private FontFileRenderer() {} - - /** - * Render one line of text with the given font file at the given pixel - * size. The returned image is the font's full vertical extent (top to - * bottom, taller than ascent+descent), plus any horizontal glyph - * overhang beyond the advance width. - * - * Returns an empty array if the font cannot be loaded or the text - * renders to nothing. - */ - public static byte[] renderText(String path, String text, float sizePx, double gamma) { - if (path == null || text == null || sizePx <= 0) { - return new byte[0]; - } - Typeface typeface = loadTypeface(path); - if (typeface == null) { - return new byte[0]; - } - try { - Paint paint = new Paint(Paint.ANTI_ALIAS_FLAG | Paint.SUBPIXEL_TEXT_FLAG); - paint.setTypeface(typeface); - paint.setTextSize(sizePx); - paint.setColor(Color.WHITE); - - // top/bottom (the font's maximal extent) rather than - // ascent/descent, so diacritics and swashes don't clip. - Paint.FontMetricsInt fm = paint.getFontMetricsInt(); - int ascent = -fm.top; - int height = -fm.top + fm.bottom; - // measureText gives advance width; italic/swash glyphs can paint - // outside it, so pad by half an em on each side and trim after. - int pad = (int) Math.ceil(sizePx / 2); - int advance = (int) Math.ceil(paint.measureText(text)); - int width = advance + 2 * pad; - if (width <= 0 || height <= 0 || width * height > 4_000_000) { - return new byte[0]; - } - - Bitmap bitmap = Bitmap.createBitmap(width, height, Bitmap.Config.ALPHA_8); - Canvas canvas = new Canvas(bitmap); - canvas.drawText(text, pad, ascent, paint); - byte[] pixels = alpha8Pixels(bitmap, width, height); - bitmap.recycle(); - - // Trim the horizontal padding down to the painted extent, but - // keep at least the advance width so spacing stays truthful. - int left = pad; - int right = pad + advance; - int[] painted = paintedColumnRange(pixels, width, height); - if (painted != null) { - left = Math.min(left, painted[0]); - right = Math.max(right, painted[1] + 1); - } - int outWidth = Math.max(1, right - left); - - byte[] lut = gammaLut(gamma); - byte[] out = new byte[4 + outWidth * height]; - out[0] = (byte) (outWidth & 0xff); - out[1] = (byte) ((outWidth >> 8) & 0xff); - out[2] = (byte) (height & 0xff); - out[3] = (byte) ((height >> 8) & 0xff); - for (int y = 0; y < height; y++) { - int srcRow = y * width; - int dstRow = 4 + y * outWidth; - for (int x = 0; x < outWidth; x++) { - out[dstRow + x] = lut[pixels[srcRow + left + x] & 0xff]; - } - } - return out; - } catch (Exception | OutOfMemoryError e) { - Log.w(TAG, "text render failed: " + path, e); - return new byte[0]; - } - } - - /** - * Render a multi-line block of text wrapped to maxWidth, with line - * breaking delegated to StaticLayout (so it matches Android text - * behavior, including breaking inside long words). Truncated with an - * ellipsis past maxLines. Same packet format as renderText. - */ - public static byte[] renderWrapped( - String path, String text, float sizePx, int maxWidth, double gamma, int maxLines) { - if (path == null || text == null || sizePx <= 0 || maxWidth <= 0 || maxLines <= 0) { - return new byte[0]; - } - Typeface typeface = loadTypeface(path); - if (typeface == null) { - return new byte[0]; - } - try { - android.text.TextPaint paint = - new android.text.TextPaint(Paint.ANTI_ALIAS_FLAG | Paint.SUBPIXEL_TEXT_FLAG); - paint.setTypeface(typeface); - paint.setTextSize(sizePx); - paint.setColor(Color.WHITE); - paint.setHinting(Paint.HINTING_ON); - - android.text.StaticLayout layout = android.text.StaticLayout.Builder - .obtain(text, 0, text.length(), paint, maxWidth) - .setAlignment(android.text.Layout.Alignment.ALIGN_NORMAL) - .setIncludePad(false) - .setMaxLines(maxLines) - .setEllipsize(android.text.TextUtils.TruncateAt.END) - .build(); - int height = Math.max(1, layout.getHeight()); - if ((long) maxWidth * height > 4_000_000L) { - return new byte[0]; - } - - Bitmap bitmap = Bitmap.createBitmap(maxWidth, height, Bitmap.Config.ALPHA_8); - Canvas canvas = new Canvas(bitmap); - layout.draw(canvas); - byte[] pixels = alpha8Pixels(bitmap, maxWidth, height); - bitmap.recycle(); - - byte[] lut = gammaLut(gamma); - byte[] out = new byte[4 + pixels.length]; - out[0] = (byte) (maxWidth & 0xff); - out[1] = (byte) ((maxWidth >> 8) & 0xff); - out[2] = (byte) (height & 0xff); - out[3] = (byte) ((height >> 8) & 0xff); - for (int i = 0; i < pixels.length; i++) { - out[4 + i] = lut[pixels[i] & 0xff]; - } - return out; - } catch (Exception | OutOfMemoryError e) { - Log.w(TAG, "wrapped render failed: " + path, e); - return new byte[0]; - } - } - - /** - * Render one codepoint as a tight-ink-box antialiased glyph cell for the - * texture-cache text pipeline (see app/graphics/ttf-font.ts). Ligatures - * are disabled so a per-codepoint raster plus pairwise kerning (measured - * via measureTextExact) fully describes a line. Little-endian packet: - * [advance u16, 26.6 fixed point] - * [bearingX s16] ink left relative to the pen position - * [inkTop s16] ink top relative to the line top (ascent above baseline) - * [width u16][height u16] - * [width*height coverage bytes, gamma-mapped] - * An ink-free glyph (space) has width = height = 0 but a valid advance. - * Empty array when the font cannot be loaded or the render fails. - */ - public static byte[] renderGlyphCell(String path, float sizePx, int codePoint, double gamma) { - Typeface typeface = loadTypeface(path); - if (typeface == null || sizePx <= 0 || codePoint < 0 || codePoint > 0x10ffff) { - return new byte[0]; - } - try { - Paint paint = glyphPaint(typeface, sizePx); - String text = new String(Character.toChars(codePoint)); - float advance = paint.measureText(text); - int advanceFixed = Math.max(0, Math.min(0xffff, Math.round(advance * 64f))); - - android.graphics.Rect bounds = new android.graphics.Rect(); - paint.getTextBounds(text, 0, text.length(), bounds); - if (bounds.isEmpty()) { - return glyphCellPacket(advanceFixed, 0, 0, 0, 0, null); - } - // getTextBounds can be off by a hair on AA edges; render with - // padding, then trim to the actually painted box. - int pad = 2; - int bw = bounds.width() + 2 * pad; - int bh = bounds.height() + 2 * pad; - if (bw <= 0 || bh <= 0 || bw * bh > 1_000_000) { - return new byte[0]; - } - int penX = pad - bounds.left; - int baselineY = pad - bounds.top; - Bitmap bitmap = Bitmap.createBitmap(bw, bh, Bitmap.Config.ALPHA_8); - Canvas canvas = new Canvas(bitmap); - canvas.drawText(text, penX, baselineY, paint); - byte[] pixels = alpha8Pixels(bitmap, bw, bh); - bitmap.recycle(); - - int x0 = bw, x1 = -1, y0 = bh, y1 = -1; - for (int y = 0; y < bh; y++) { - int row = y * bw; - for (int x = 0; x < bw; x++) { - if (pixels[row + x] != 0) { - if (x < x0) x0 = x; - if (x > x1) x1 = x; - if (y < y0) y0 = y; - if (y > y1) y1 = y; - } - } - } - if (x1 < x0) { - return glyphCellPacket(advanceFixed, 0, 0, 0, 0, null); - } - int width = x1 - x0 + 1; - int height = y1 - y0 + 1; - Paint.FontMetricsInt fm = paint.getFontMetricsInt(); - int bearingX = x0 - penX; - int inkTop = (-fm.ascent) + (y0 - baselineY); - - byte[] lut = gammaLut(gamma); - byte[] coverage = new byte[width * height]; - for (int y = 0; y < height; y++) { - int src = (y0 + y) * bw + x0; - int dst = y * width; - for (int x = 0; x < width; x++) { - coverage[dst + x] = lut[pixels[src + x] & 0xff]; - } - } - return glyphCellPacket(advanceFixed, bearingX, inkTop, width, height, coverage); - } catch (Exception | OutOfMemoryError e) { - Log.w(TAG, "glyph render failed: " + path + " U+" + Integer.toHexString(codePoint), e); - return new byte[0]; - } - } - - /** - * Exact (float) advance width of a text run, with the same paint setup as - * renderGlyphCell (subpixel advances, ligatures off), so pair kerning can - * be derived as measure(ab) - measure(a) - measure(b). - */ - public static double measureTextExact(String path, String text, float sizePx) { - Typeface typeface = loadTypeface(path); - if (typeface == null || text == null || sizePx <= 0) { - return 0; - } - return glyphPaint(typeface, sizePx).measureText(text); - } - - /** Paint used by both the glyph-cell renderer and its measurements. */ - private static Paint glyphPaint(Typeface typeface, float sizePx) { - Paint paint = new Paint(Paint.ANTI_ALIAS_FLAG | Paint.SUBPIXEL_TEXT_FLAG); - paint.setTypeface(typeface); - paint.setTextSize(sizePx); - paint.setColor(Color.WHITE); - // Ligatures merge codepoints into one glyph, which the per-codepoint - // cache model cannot represent; kerning still applies. - paint.setFontFeatureSettings("'liga' off, 'clig' off"); - return paint; - } - - private static byte[] glyphCellPacket( - int advanceFixed, int bearingX, int inkTop, int width, int height, byte[] coverage) { - byte[] out = new byte[10 + (coverage == null ? 0 : coverage.length)]; - out[0] = (byte) (advanceFixed & 0xff); - out[1] = (byte) ((advanceFixed >> 8) & 0xff); - out[2] = (byte) (bearingX & 0xff); - out[3] = (byte) ((bearingX >> 8) & 0xff); - out[4] = (byte) (inkTop & 0xff); - out[5] = (byte) ((inkTop >> 8) & 0xff); - out[6] = (byte) (width & 0xff); - out[7] = (byte) ((width >> 8) & 0xff); - out[8] = (byte) (height & 0xff); - out[9] = (byte) ((height >> 8) & 0xff); - if (coverage != null) { - System.arraycopy(coverage, 0, out, 10, coverage.length); - } - return out; - } - - /** - * Line metrics for a font at a pixel size, as "ascent descent lineGap" - * (integers, pixels). Empty string if the font cannot be loaded. - */ - public static String getFontMetrics(String path, float sizePx) { - Typeface typeface = loadTypeface(path); - if (typeface == null || sizePx <= 0) { - return ""; - } - Paint paint = new Paint(Paint.ANTI_ALIAS_FLAG); - paint.setTypeface(typeface); - paint.setTextSize(sizePx); - Paint.FontMetricsInt fm = paint.getFontMetricsInt(); - int lineGap = fm.leading; - return (-fm.ascent) + " " + fm.descent + " " + lineGap; - } - - /** Advance width in pixels of a line of text (rounded up). */ - public static int measureText(String path, String text, float sizePx) { - Typeface typeface = loadTypeface(path); - if (typeface == null || text == null || sizePx <= 0) { - return 0; - } - Paint paint = new Paint(Paint.ANTI_ALIAS_FLAG); - paint.setTypeface(typeface); - paint.setTextSize(sizePx); - return (int) Math.ceil(paint.measureText(text)); - } - - /** - * Whether the font file loads at all (strictly validated on API 29+; - * best-effort below that). - */ - public static boolean canLoadFont(String path) { - return loadTypeface(path) != null; - } - - /** - * The font's family and style names from its 'name' table, as - * "Family\nStyle" ("Style" may be empty). Falls back to "" when the - * table cannot be parsed; callers should then use the filename. - */ - public static String getFontName(String path) { - try { - return parseNameTable(path); - } catch (Exception e) { - Log.w(TAG, "name table parse failed: " + path, e); - return ""; - } - } - - private static synchronized Typeface loadTypeface(String path) { - if (path == null) { - return null; - } - Typeface cached = typefaceCache.get(path); - if (cached != null) { - return cached; - } - File file = new File(path); - if (!file.isFile() || !file.canRead()) { - return null; - } - Typeface typeface = null; - try { - if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q) { - // Strict path: Font.Builder throws on files that are not - // valid fonts, unlike createFromFile which silently falls - // back to the default typeface. - android.graphics.fonts.Font font = - new android.graphics.fonts.Font.Builder(file).build(); - typeface = new Typeface.CustomFallbackBuilder( - new android.graphics.fonts.FontFamily.Builder(font).build()) - .build(); - } else { - typeface = Typeface.createFromFile(file); - if (typeface == null || typeface.equals(Typeface.DEFAULT)) { - typeface = null; - } - } - } catch (Exception | LinkageError e) { - Log.w(TAG, "font load failed: " + path + ": " + e); - typeface = null; - } - if (typeface != null) { - typefaceCache.put(path, typeface); - } - return typeface; - } - - /** - * The coverage bytes of an ALPHA_8 bitmap as a tightly-packed - * width*height array (copyPixelsToBuffer copies rowBytes*height, and the - * row stride is not guaranteed to equal the width). - */ - private static byte[] alpha8Pixels(Bitmap bitmap, int width, int height) { - int rowBytes = bitmap.getRowBytes(); - byte[] raw = new byte[rowBytes * height]; - bitmap.copyPixelsToBuffer(ByteBuffer.wrap(raw)); - if (rowBytes == width) { - return raw; - } - byte[] pixels = new byte[width * height]; - for (int y = 0; y < height; y++) { - System.arraycopy(raw, y * rowBytes, pixels, y * width, width); - } - return pixels; - } - - /** [first, last] painted (nonzero) column indexes, or null if blank. */ - private static int[] paintedColumnRange(byte[] pixels, int width, int height) { - int first = width; - int last = -1; - for (int y = 0; y < height; y++) { - int row = y * width; - for (int x = 0; x < first; x++) { - if (pixels[row + x] != 0) { - first = x; - break; - } - } - for (int x = width - 1; x > last; x--) { - if (pixels[row + x] != 0) { - last = x; - break; - } - } - } - return last >= first ? new int[] { first, last } : null; - } - - private static byte[] gammaLut(double gamma) { - byte[] lut = new byte[256]; - if (gamma <= 0) { - gamma = 1.0; - } - for (int i = 0; i < 256; i++) { - lut[i] = (byte) Math.max(0, Math.min(255, - (int) Math.round(255.0 * Math.pow(i / 255.0, gamma)))); - } - return lut; - } - - // --- Minimal 'name' table reader (TTF/OTF/TTC) for preview titles --- - - private static String parseNameTable(String path) throws Exception { - try (RandomAccessFile raf = new RandomAccessFile(path, "r")) { - long fileSize = raf.length(); - if (fileSize < 12) { - return ""; - } - long offset = 0; - int tag = readU32(raf, 0); - if (tag == 0x74746366) { // 'ttcf': use the first face - if (readU32(raf, 8) < 1) { - return ""; - } - offset = readU32(raf, 12) & 0xffffffffL; - } - int numTables = readU16(raf, offset + 4); - long nameOffset = -1; - for (int i = 0; i < numTables; i++) { - long rec = offset + 12 + i * 16L; - if (rec + 16 > fileSize) { - return ""; - } - if (readU32(raf, rec) == 0x6e616d65) { // 'name' - nameOffset = readU32(raf, rec + 8) & 0xffffffffL; - break; - } - } - if (nameOffset < 0 || nameOffset + 6 > fileSize) { - return ""; - } - int count = readU16(raf, nameOffset + 2); - long stringStorage = nameOffset + readU16(raf, nameOffset + 4); - String family = null; - String style = null; - String preferredFamily = null; - String preferredStyle = null; - for (int i = 0; i < count; i++) { - long rec = nameOffset + 6 + i * 12L; - if (rec + 12 > fileSize) { - break; - } - int platform = readU16(raf, rec); - int nameId = readU16(raf, rec + 6); - if (nameId != 1 && nameId != 2 && nameId != 16 && nameId != 17) { - continue; - } - int length = readU16(raf, rec + 8); - long strOffset = stringStorage + readU16(raf, rec + 10); - if (strOffset + length > fileSize || length <= 0 || length > 512) { - continue; - } - byte[] data = new byte[length]; - raf.seek(strOffset); - raf.readFully(data); - // Platform 0 (Unicode) and 3 (Windows) store UTF-16BE; - // platform 1 (Mac) is close enough to Latin-1 for names. - String value = platform == 1 - ? new String(data, java.nio.charset.StandardCharsets.ISO_8859_1) - : new String(data, java.nio.charset.StandardCharsets.UTF_16BE); - value = value.trim(); - if (value.isEmpty()) { - continue; - } - if (nameId == 1 && family == null) family = value; - if (nameId == 2 && style == null) style = value; - if (nameId == 16 && preferredFamily == null) preferredFamily = value; - if (nameId == 17 && preferredStyle == null) preferredStyle = value; - } - String outFamily = preferredFamily != null ? preferredFamily : family; - String outStyle = preferredStyle != null ? preferredStyle : style; - if (outFamily == null) { - return ""; - } - return outFamily + "\n" + (outStyle == null ? "" : outStyle); - } - } - - private static int readU16(RandomAccessFile raf, long offset) throws Exception { - raf.seek(offset); - byte[] b = new byte[2]; - raf.readFully(b); - return ByteBuffer.wrap(b).order(ByteOrder.BIG_ENDIAN).getShort() & 0xffff; - } - - private static int readU32(RandomAccessFile raf, long offset) throws Exception { - raf.seek(offset); - byte[] b = new byte[4]; - raf.readFully(b); - return ByteBuffer.wrap(b).order(ByteOrder.BIG_ENDIAN).getInt(); - } -} diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FontFileRenderer.kt b/App_Resources/Android/src/main/java/com/faceclaw/app/FontFileRenderer.kt new file mode 100644 index 00000000..e6b717ec --- /dev/null +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/FontFileRenderer.kt @@ -0,0 +1,574 @@ +package com.faceclaw.app + +import android.graphics.Bitmap +import android.graphics.Canvas +import android.graphics.Color +import android.graphics.Paint +import android.graphics.Typeface +import android.os.Build +import android.util.Log + +import java.io.File +import java.io.RandomAccessFile +import java.nio.ByteBuffer +import java.nio.ByteOrder +import java.util.LinkedHashMap + +/** + * Rasterizes text using font files (TTF/OTF/TTC) via the Android text stack, + * so shaping, kerning, and antialiasing are all delegated to + * minikin/HarfBuzz. Output uses the same grayscale packet format as + * ImageFileLoader ([widthLo, widthHi, heightLo, heightHi, pixels...], one + * byte per pixel, row-major), which the TS side turns into a GrayImage and + * the compositor later quantizes to the 4bpp the firmware wants. + * + * The gamma parameter maps antialiased coverage to output shade: + * out = 255 * (coverage/255)^gamma. The G2 display response looks roughly + * linear, so 1.0 is the expected default; the font previewer exposes it for + * on-hardware comparison. + */ +class FontFileRenderer private constructor() { + companion object { + private const val TAG = "FontFileRenderer" + private const val TYPEFACE_CACHE_SIZE = 4 + + /** Small LRU of loaded typefaces keyed by file path. */ + private val typefaceCache: MutableMap = + object : LinkedHashMap(TYPEFACE_CACHE_SIZE, 0.75f, true) { + override fun removeEldestEntry(eldest: MutableMap.MutableEntry): Boolean { + return size > TYPEFACE_CACHE_SIZE + } + } + + /** + * Render one line of text with the given font file at the given pixel + * size. The returned image is the font's full vertical extent (top to + * bottom, taller than ascent+descent), plus any horizontal glyph + * overhang beyond the advance width. + * + * Returns an empty array if the font cannot be loaded or the text + * renders to nothing. + */ + @JvmStatic + fun renderText(path: String?, text: String?, sizePx: Float, gamma: Double): ByteArray { + if (path == null || text == null || sizePx <= 0) { + return ByteArray(0) + } + val typeface = loadTypeface(path) + if (typeface == null) { + return ByteArray(0) + } + try { + val paint = Paint(Paint.ANTI_ALIAS_FLAG or Paint.SUBPIXEL_TEXT_FLAG) + paint.typeface = typeface + paint.textSize = sizePx + paint.color = Color.WHITE + + // top/bottom (the font's maximal extent) rather than + // ascent/descent, so diacritics and swashes don't clip. + val fm = paint.fontMetricsInt + val ascent = -fm.top + val height = -fm.top + fm.bottom + // measureText gives advance width; italic/swash glyphs can paint + // outside it, so pad by half an em on each side and trim after. + val pad = Math.ceil((sizePx / 2).toDouble()).toInt() + val advance = Math.ceil(paint.measureText(text).toDouble()).toInt() + val width = advance + 2 * pad + if (width <= 0 || height <= 0 || width * height > 4_000_000) { + return ByteArray(0) + } + + val bitmap = Bitmap.createBitmap(width, height, Bitmap.Config.ALPHA_8) + val canvas = Canvas(bitmap) + canvas.drawText(text, pad.toFloat(), ascent.toFloat(), paint) + val pixels = alpha8Pixels(bitmap, width, height) + bitmap.recycle() + + // Trim the horizontal padding down to the painted extent, but + // keep at least the advance width so spacing stays truthful. + var left = pad + var right = pad + advance + val painted = paintedColumnRange(pixels, width, height) + if (painted != null) { + left = Math.min(left, painted[0]) + right = Math.max(right, painted[1] + 1) + } + val outWidth = Math.max(1, right - left) + + val lut = gammaLut(gamma) + val out = ByteArray(4 + outWidth * height) + out[0] = (outWidth and 0xff).toByte() + out[1] = ((outWidth shr 8) and 0xff).toByte() + out[2] = (height and 0xff).toByte() + out[3] = ((height shr 8) and 0xff).toByte() + for (y in 0 until height) { + val srcRow = y * width + val dstRow = 4 + y * outWidth + for (x in 0 until outWidth) { + out[dstRow + x] = lut[pixels[srcRow + left + x].toInt() and 0xff] + } + } + return out + } catch (e: Exception) { + Log.w(TAG, "text render failed: $path", e) + return ByteArray(0) + } catch (e: OutOfMemoryError) { + Log.w(TAG, "text render failed: $path", e) + return ByteArray(0) + } + } + + /** + * Render a multi-line block of text wrapped to maxWidth, with line + * breaking delegated to StaticLayout (so it matches Android text + * behavior, including breaking inside long words). Truncated with an + * ellipsis past maxLines. Same packet format as renderText. + */ + @JvmStatic + fun renderWrapped( + path: String?, text: String?, sizePx: Float, maxWidth: Int, gamma: Double, maxLines: Int): ByteArray { + if (path == null || text == null || sizePx <= 0 || maxWidth <= 0 || maxLines <= 0) { + return ByteArray(0) + } + val typeface = loadTypeface(path) + if (typeface == null) { + return ByteArray(0) + } + try { + val paint = android.text.TextPaint(Paint.ANTI_ALIAS_FLAG or Paint.SUBPIXEL_TEXT_FLAG) + paint.typeface = typeface + paint.textSize = sizePx + paint.color = Color.WHITE + paint.hinting = Paint.HINTING_ON + + val layout = android.text.StaticLayout.Builder + .obtain(text, 0, text.length, paint, maxWidth) + .setAlignment(android.text.Layout.Alignment.ALIGN_NORMAL) + .setIncludePad(false) + .setMaxLines(maxLines) + .setEllipsize(android.text.TextUtils.TruncateAt.END) + .build() + val height = Math.max(1, layout.height) + if (maxWidth.toLong() * height > 4_000_000L) { + return ByteArray(0) + } + + val bitmap = Bitmap.createBitmap(maxWidth, height, Bitmap.Config.ALPHA_8) + val canvas = Canvas(bitmap) + layout.draw(canvas) + val pixels = alpha8Pixels(bitmap, maxWidth, height) + bitmap.recycle() + + val lut = gammaLut(gamma) + val out = ByteArray(4 + pixels.size) + out[0] = (maxWidth and 0xff).toByte() + out[1] = ((maxWidth shr 8) and 0xff).toByte() + out[2] = (height and 0xff).toByte() + out[3] = ((height shr 8) and 0xff).toByte() + for (i in pixels.indices) { + out[4 + i] = lut[pixels[i].toInt() and 0xff] + } + return out + } catch (e: Exception) { + Log.w(TAG, "wrapped render failed: $path", e) + return ByteArray(0) + } catch (e: OutOfMemoryError) { + Log.w(TAG, "wrapped render failed: $path", e) + return ByteArray(0) + } + } + + /** + * Render one codepoint as a tight-ink-box antialiased glyph cell for the + * texture-cache text pipeline (see app/graphics/ttf-font.ts). Ligatures + * are disabled so a per-codepoint raster plus pairwise kerning (measured + * via measureTextExact) fully describes a line. Little-endian packet: + * [advance u16, 26.6 fixed point] + * [bearingX s16] ink left relative to the pen position + * [inkTop s16] ink top relative to the line top (ascent above baseline) + * [width u16][height u16] + * [width*height coverage bytes, gamma-mapped] + * An ink-free glyph (space) has width = height = 0 but a valid advance. + * Empty array when the font cannot be loaded or the render fails. + */ + @JvmStatic + fun renderGlyphCell(path: String?, sizePx: Float, codePoint: Int, gamma: Double): ByteArray { + val typeface = loadTypeface(path) + if (typeface == null || sizePx <= 0 || codePoint < 0 || codePoint > 0x10ffff) { + return ByteArray(0) + } + try { + val paint = glyphPaint(typeface, sizePx) + val text = String(Character.toChars(codePoint)) + val advance = paint.measureText(text) + val advanceFixed = Math.max(0, Math.min(0xffff, Math.round(advance * 64f))) + + val bounds = android.graphics.Rect() + paint.getTextBounds(text, 0, text.length, bounds) + if (bounds.isEmpty) { + return glyphCellPacket(advanceFixed, 0, 0, 0, 0, null) + } + // getTextBounds can be off by a hair on AA edges; render with + // padding, then trim to the actually painted box. + val pad = 2 + val bw = bounds.width() + 2 * pad + val bh = bounds.height() + 2 * pad + if (bw <= 0 || bh <= 0 || bw * bh > 1_000_000) { + return ByteArray(0) + } + val penX = pad - bounds.left + val baselineY = pad - bounds.top + val bitmap = Bitmap.createBitmap(bw, bh, Bitmap.Config.ALPHA_8) + val canvas = Canvas(bitmap) + canvas.drawText(text, penX.toFloat(), baselineY.toFloat(), paint) + val pixels = alpha8Pixels(bitmap, bw, bh) + bitmap.recycle() + + var x0 = bw + var x1 = -1 + var y0 = bh + var y1 = -1 + for (y in 0 until bh) { + val row = y * bw + for (x in 0 until bw) { + if (pixels[row + x].toInt() != 0) { + if (x < x0) x0 = x + if (x > x1) x1 = x + if (y < y0) y0 = y + if (y > y1) y1 = y + } + } + } + if (x1 < x0) { + return glyphCellPacket(advanceFixed, 0, 0, 0, 0, null) + } + val width = x1 - x0 + 1 + val height = y1 - y0 + 1 + val fm = paint.fontMetricsInt + val bearingX = x0 - penX + val inkTop = (-fm.ascent) + (y0 - baselineY) + + val lut = gammaLut(gamma) + val coverage = ByteArray(width * height) + for (y in 0 until height) { + val src = (y0 + y) * bw + x0 + val dst = y * width + for (x in 0 until width) { + coverage[dst + x] = lut[pixels[src + x].toInt() and 0xff] + } + } + return glyphCellPacket(advanceFixed, bearingX, inkTop, width, height, coverage) + } catch (e: Exception) { + Log.w(TAG, "glyph render failed: " + path + " U+" + Integer.toHexString(codePoint), e) + return ByteArray(0) + } catch (e: OutOfMemoryError) { + Log.w(TAG, "glyph render failed: " + path + " U+" + Integer.toHexString(codePoint), e) + return ByteArray(0) + } + } + + /** + * Exact (float) advance width of a text run, with the same paint setup as + * renderGlyphCell (subpixel advances, ligatures off), so pair kerning can + * be derived as measure(ab) - measure(a) - measure(b). + */ + @JvmStatic + fun measureTextExact(path: String?, text: String?, sizePx: Float): Double { + val typeface = loadTypeface(path) + if (typeface == null || text == null || sizePx <= 0) { + return 0.0 + } + return glyphPaint(typeface, sizePx).measureText(text).toDouble() + } + + /** Paint used by both the glyph-cell renderer and its measurements. */ + @JvmStatic + private fun glyphPaint(typeface: Typeface, sizePx: Float): Paint { + val paint = Paint(Paint.ANTI_ALIAS_FLAG or Paint.SUBPIXEL_TEXT_FLAG) + paint.typeface = typeface + paint.textSize = sizePx + paint.color = Color.WHITE + // Ligatures merge codepoints into one glyph, which the per-codepoint + // cache model cannot represent; kerning still applies. + paint.fontFeatureSettings = "'liga' off, 'clig' off" + return paint + } + + @JvmStatic + private fun glyphCellPacket( + advanceFixed: Int, bearingX: Int, inkTop: Int, width: Int, height: Int, coverage: ByteArray?): ByteArray { + val out = ByteArray(10 + (coverage?.size ?: 0)) + out[0] = (advanceFixed and 0xff).toByte() + out[1] = ((advanceFixed shr 8) and 0xff).toByte() + out[2] = (bearingX and 0xff).toByte() + out[3] = ((bearingX shr 8) and 0xff).toByte() + out[4] = (inkTop and 0xff).toByte() + out[5] = ((inkTop shr 8) and 0xff).toByte() + out[6] = (width and 0xff).toByte() + out[7] = ((width shr 8) and 0xff).toByte() + out[8] = (height and 0xff).toByte() + out[9] = ((height shr 8) and 0xff).toByte() + if (coverage != null) { + System.arraycopy(coverage, 0, out, 10, coverage.size) + } + return out + } + + /** + * Line metrics for a font at a pixel size, as "ascent descent lineGap" + * (integers, pixels). Empty string if the font cannot be loaded. + */ + @JvmStatic + fun getFontMetrics(path: String?, sizePx: Float): String { + val typeface = loadTypeface(path) + if (typeface == null || sizePx <= 0) { + return "" + } + val paint = Paint(Paint.ANTI_ALIAS_FLAG) + paint.typeface = typeface + paint.textSize = sizePx + val fm = paint.fontMetricsInt + val lineGap = fm.leading + return (-fm.ascent).toString() + " " + fm.descent + " " + lineGap + } + + /** Advance width in pixels of a line of text (rounded up). */ + @JvmStatic + fun measureText(path: String?, text: String?, sizePx: Float): Int { + val typeface = loadTypeface(path) + if (typeface == null || text == null || sizePx <= 0) { + return 0 + } + val paint = Paint(Paint.ANTI_ALIAS_FLAG) + paint.typeface = typeface + paint.textSize = sizePx + return Math.ceil(paint.measureText(text).toDouble()).toInt() + } + + /** + * Whether the font file loads at all (strictly validated on API 29+; + * best-effort below that). + */ + @JvmStatic + fun canLoadFont(path: String?): Boolean { + return loadTypeface(path) != null + } + + /** + * The font's family and style names from its 'name' table, as + * "Family\nStyle" ("Style" may be empty). Falls back to "" when the + * table cannot be parsed; callers should then use the filename. + */ + @JvmStatic + fun getFontName(path: String?): String { + try { + return parseNameTable(path) + } catch (e: Exception) { + Log.w(TAG, "name table parse failed: $path", e) + return "" + } + } + + @JvmStatic + @Synchronized + private fun loadTypeface(path: String?): Typeface? { + if (path == null) { + return null + } + val cached = typefaceCache[path] + if (cached != null) { + return cached + } + val file = File(path) + if (!file.isFile || !file.canRead()) { + return null + } + var typeface: Typeface? = null + try { + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q) { + // Strict path: Font.Builder throws on files that are not + // valid fonts, unlike createFromFile which silently falls + // back to the default typeface. + val font = android.graphics.fonts.Font.Builder(file).build() + typeface = Typeface.CustomFallbackBuilder( + android.graphics.fonts.FontFamily.Builder(font).build()) + .build() + } else { + typeface = Typeface.createFromFile(file) + if (typeface == null || typeface == Typeface.DEFAULT) { + typeface = null + } + } + } catch (e: Exception) { + Log.w(TAG, "font load failed: $path: $e") + typeface = null + } catch (e: LinkageError) { + Log.w(TAG, "font load failed: $path: $e") + typeface = null + } + if (typeface != null) { + typefaceCache[path] = typeface + } + return typeface + } + + /** + * The coverage bytes of an ALPHA_8 bitmap as a tightly-packed + * width*height array (copyPixelsToBuffer copies rowBytes*height, and the + * row stride is not guaranteed to equal the width). + */ + @JvmStatic + private fun alpha8Pixels(bitmap: Bitmap, width: Int, height: Int): ByteArray { + val rowBytes = bitmap.rowBytes + val raw = ByteArray(rowBytes * height) + bitmap.copyPixelsToBuffer(ByteBuffer.wrap(raw)) + if (rowBytes == width) { + return raw + } + val pixels = ByteArray(width * height) + for (y in 0 until height) { + System.arraycopy(raw, y * rowBytes, pixels, y * width, width) + } + return pixels + } + + /** [first, last] painted (nonzero) column indexes, or null if blank. */ + @JvmStatic + private fun paintedColumnRange(pixels: ByteArray, width: Int, height: Int): IntArray? { + var first = width + var last = -1 + for (y in 0 until height) { + val row = y * width + var x = 0 + while (x < first) { + if (pixels[row + x].toInt() != 0) { + first = x + break + } + x++ + } + x = width - 1 + while (x > last) { + if (pixels[row + x].toInt() != 0) { + last = x + break + } + x-- + } + } + return if (last >= first) intArrayOf(first, last) else null + } + + @JvmStatic + private fun gammaLut(gammaIn: Double): ByteArray { + val lut = ByteArray(256) + var gamma = gammaIn + if (gamma <= 0) { + gamma = 1.0 + } + for (i in 0 until 256) { + lut[i] = Math.max(0, Math.min(255, + Math.round(255.0 * Math.pow(i / 255.0, gamma)).toInt())).toByte() + } + return lut + } + + // --- Minimal 'name' table reader (TTF/OTF/TTC) for preview titles --- + + @JvmStatic + @Throws(Exception::class) + private fun parseNameTable(path: String?): String { + RandomAccessFile(path, "r").use { raf -> + val fileSize = raf.length() + if (fileSize < 12) { + return "" + } + var offset = 0L + val tag = readU32(raf, 0) + if (tag == 0x74746366) { // 'ttcf': use the first face + if (readU32(raf, 8) < 1) { + return "" + } + offset = readU32(raf, 12).toLong() and 0xffffffffL + } + val numTables = readU16(raf, offset + 4) + var nameOffset = -1L + for (i in 0 until numTables) { + val rec = offset + 12 + i * 16L + if (rec + 16 > fileSize) { + return "" + } + if (readU32(raf, rec) == 0x6e616d65) { // 'name' + nameOffset = readU32(raf, rec + 8).toLong() and 0xffffffffL + break + } + } + if (nameOffset < 0 || nameOffset + 6 > fileSize) { + return "" + } + val count = readU16(raf, nameOffset + 2) + val stringStorage = nameOffset + readU16(raf, nameOffset + 4) + var family: String? = null + var style: String? = null + var preferredFamily: String? = null + var preferredStyle: String? = null + for (i in 0 until count) { + val rec = nameOffset + 6 + i * 12L + if (rec + 12 > fileSize) { + break + } + val platform = readU16(raf, rec) + val nameId = readU16(raf, rec + 6) + if (nameId != 1 && nameId != 2 && nameId != 16 && nameId != 17) { + continue + } + val length = readU16(raf, rec + 8) + val strOffset = stringStorage + readU16(raf, rec + 10) + if (strOffset + length > fileSize || length <= 0 || length > 512) { + continue + } + val data = ByteArray(length) + raf.seek(strOffset) + raf.readFully(data) + // Platform 0 (Unicode) and 3 (Windows) store UTF-16BE; + // platform 1 (Mac) is close enough to Latin-1 for names. + var value = if (platform == 1) + String(data, java.nio.charset.StandardCharsets.ISO_8859_1) + else String(data, java.nio.charset.StandardCharsets.UTF_16BE) + value = value.trim() + if (value.isEmpty()) { + continue + } + if (nameId == 1 && family == null) family = value + if (nameId == 2 && style == null) style = value + if (nameId == 16 && preferredFamily == null) preferredFamily = value + if (nameId == 17 && preferredStyle == null) preferredStyle = value + } + val outFamily = preferredFamily ?: family + val outStyle = preferredStyle ?: style + if (outFamily == null) { + return "" + } + return outFamily + "\n" + (outStyle ?: "") + } + } + + @JvmStatic + @Throws(Exception::class) + private fun readU16(raf: RandomAccessFile, offset: Long): Int { + raf.seek(offset) + val b = ByteArray(2) + raf.readFully(b) + return ByteBuffer.wrap(b).order(ByteOrder.BIG_ENDIAN).getShort().toInt() and 0xffff + } + + @JvmStatic + @Throws(Exception::class) + private fun readU32(raf: RandomAccessFile, offset: Long): Int { + raf.seek(offset) + val b = ByteArray(4) + raf.readFully(b) + return ByteBuffer.wrap(b).order(ByteOrder.BIG_ENDIAN).getInt() + } + } +} diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FrameTimings.java b/App_Resources/Android/src/main/java/com/faceclaw/app/FrameTimings.java deleted file mode 100644 index 4dba5ac0..00000000 --- a/App_Resources/Android/src/main/java/com/faceclaw/app/FrameTimings.java +++ /dev/null @@ -1,567 +0,0 @@ -package com.faceclaw.app; - -import android.content.Context; -import android.os.SystemClock; -import android.util.Log; - -import java.io.File; -import java.io.FileOutputStream; -import java.io.OutputStreamWriter; -import java.io.Writer; -import java.nio.charset.StandardCharsets; -import java.text.SimpleDateFormat; -import java.util.ArrayDeque; -import java.util.ArrayList; -import java.util.Collections; -import java.util.Date; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Locale; -import java.util.Map; - -/** - * Per-frame timing instrumentation, shared between the Java BLE layer and the - * Typescript UI layer. A "frame" starts when we receive an input event (or a - * timer fires and we decide to redraw), and finishes when the resulting screen - * update has been fully transmitted to the glasses, is discarded unsent, or - * times out because nobody reported finishing it. - * - * Frames form a tree. One input event can fan out into several renders (the - * focused app's window plus the shell chrome, say), and each of those is its - * own frame linked back to the input frame that caused it via - * {@link #startFrame(String, int)}. The export nests descendants under their - * root and renders every line against the root's clock, so a single block - * shows the whole input-to-pixels story. Latency percentiles are measured over - * roots: root start to the first descendant that actually reached the glasses. - * - * All public methods are thread-safe and cheap enough to call from the BLE - * worker thread, the Android main thread, the JS thread, and app worker - * threads. Frame IDs are positive ints; 0 means "no frame" and is silently - * ignored everywhere, so callers do not need to null-check. - * - * Statistics and full log lines for recent and slowest frames are periodically - * exported to getExternalFilesDir()/frame-timings.txt, retrievable via - * adb pull /sdcard/Android/data//files/frame-timings.txt - */ -public final class FrameTimings { - private static final String TAG = "FrameTimings"; - private static final FrameTimings INSTANCE = new FrameTimings(); - - /** Frames still open after this long are finished as "timeout". */ - private static final long FRAME_TIMEOUT_MS = 30_000; - private static final long EXPORT_INTERVAL_MS = 15_000; - private static final long SWEEP_INTERVAL_MS = 5_000; - private static final int RECENT_ROOTS_KEPT = 40; - private static final int SLOWEST_ROOTS_KEPT = 20; - private static final int SENT_DURATIONS_WINDOW = 512; - private static final int MAX_LINES_PER_FRAME = 200; - /** - * Frames retained for ID lookup. Well above the number any export can - * reference, so a late log/span/finish call always finds its frame; frames - * older than this are unreachable and get collected. - */ - private static final int FRAMES_RETAINED = 1024; - private static final String EXPORT_FILE_NAME = "frame-timings.txt"; - - public static FrameTimings getInstance() { - return INSTANCE; - } - - private static final class Line { - final long atMs; // SystemClock.elapsedRealtime() - final String thread; - final String message; - - Line(long atMs, String thread, String message) { - this.atMs = atMs; - this.thread = thread; - this.message = message; - } - } - - private static final class Frame { - final int id; - final long startedAtMs; // SystemClock.elapsedRealtime() - final long startedWallClockMs; - final Frame parent; - final List children = new ArrayList<>(0); - final List lines = new ArrayList<>(); - final Map openSpans = new LinkedHashMap<>(); - String reason; - long finishedAtMs; - String outcome; // null while open; "sent", "discarded: ...", "timeout: ..." - /** Set on the root once some frame in its subtree reached the glasses. */ - boolean subtreeSentRecorded; - - Frame(int id, String reason, long startedAtMs, long startedWallClockMs, Frame parent) { - this.id = id; - this.reason = reason; - this.startedAtMs = startedAtMs; - this.startedWallClockMs = startedWallClockMs; - this.parent = parent; - } - - Frame root() { - Frame frame = this; - while (frame.parent != null) { - frame = frame.parent; - } - return frame; - } - - boolean isOpen() { - return outcome == null; - } - - long durationMs() { - return finishedAtMs - startedAtMs; - } - - boolean wasSent() { - return outcome != null && outcome.startsWith("sent"); - } - - /** Last moment anything in this subtree happened, for whole-tree duration. */ - long subtreeEndAtMs(long nowMs) { - long end = isOpen() ? nowMs : finishedAtMs; - for (Frame child : children) { - end = Math.max(end, child.subtreeEndAtMs(nowMs)); - } - return end; - } - - boolean subtreeHasOpenFrame() { - if (isOpen()) { - return true; - } - for (Frame child : children) { - if (child.subtreeHasOpenFrame()) { - return true; - } - } - return false; - } - } - - private final Object lock = new Object(); - /** - * Every frame we still retain, keyed by ID: open frames plus enough - * history that late calls and the export can still resolve one. - */ - private final LinkedHashMap framesById = - new LinkedHashMap(64, 0.75f, false) { - @Override - protected boolean removeEldestEntry(Map.Entry eldest) { - return size() > FRAMES_RETAINED && !eldest.getValue().isOpen(); - } - }; - private final ArrayDeque recentRoots = new ArrayDeque<>(); - private final List slowestRoots = new ArrayList<>(); - private final ArrayDeque inputLatencies = new ArrayDeque<>(); - private final ArrayDeque renderLatencies = new ArrayDeque<>(); - private int nextFrameId = 1; - private long framesStarted; - private long framesSent; - private long framesDiscarded; - private long framesTimedOut; - private boolean exportDirty; - private File exportDir; - private Thread exportThread; - - private FrameTimings() { - } - - /** Idempotent; enables filesystem export. Safe to call from any constructor path. */ - public void init(Context context) { - File dir = context.getApplicationContext().getExternalFilesDir(null); - if (dir == null) { - dir = context.getApplicationContext().getFilesDir(); - } - synchronized (lock) { - exportDir = dir; - if (exportThread == null) { - exportThread = new Thread(this::exportLoop, "FrameTimingsExport"); - exportThread.setDaemon(true); - exportThread.start(); - } - } - } - - /** Begin a root frame; reason is a short label like "input:sys-event type=0". */ - public int startFrame(String reason) { - return startFrame(reason, 0); - } - - /** - * Begin a frame caused by an existing one (parentFrameId; 0 for a root). - * The child is nested under its root in the export and its latency counts - * against the root, so an input event that fans out into an app render and - * a shell-chrome render reads as one timeline instead of three. - */ - public int startFrame(String reason, int parentFrameId) { - long now = SystemClock.elapsedRealtime(); - Frame frame; - synchronized (lock) { - Frame parent = framesById.get(parentFrameId); - frame = new Frame(nextFrameId++, reason == null ? "" : reason, now, - System.currentTimeMillis(), parent); - framesById.put(frame.id, frame); - framesStarted++; - if (parent != null) { - parent.children.add(frame); - addLineLocked(parent, now, "spawned frame#" + frame.id + " (" + frame.reason + ")"); - } - } - return frame.id; - } - - /** - * Append to a frame's label, for facts only known after it started (which - * app is in the foreground, which window a render is for). Shows up in the - * frame's header line, so the export is scannable without reading bodies. - */ - public void annotate(int frameId, String text) { - if (text == null || text.isEmpty()) { - return; - } - synchronized (lock) { - Frame frame = framesById.get(frameId); - if (frame == null) { - return; - } - frame.reason = frame.reason.isEmpty() ? text : frame.reason + " " + text; - } - } - - public void log(int frameId, String message) { - long now = SystemClock.elapsedRealtime(); - synchronized (lock) { - Frame frame = openFrameLocked(frameId); - if (frame == null) { - return; - } - addLineLocked(frame, now, message); - } - } - - public void spanStart(int frameId, String name) { - long now = SystemClock.elapsedRealtime(); - synchronized (lock) { - Frame frame = openFrameLocked(frameId); - if (frame == null) { - return; - } - frame.openSpans.put(name, now); - addLineLocked(frame, now, "span " + name + " start"); - } - } - - public void spanEnd(int frameId, String name) { - long now = SystemClock.elapsedRealtime(); - synchronized (lock) { - Frame frame = openFrameLocked(frameId); - if (frame == null) { - return; - } - Long startedAt = frame.openSpans.remove(name); - if (startedAt == null) { - addLineLocked(frame, now, "span " + name + " end (start not recorded)"); - } else { - addLineLocked(frame, now, "span " + name + " end (" + (now - startedAt) + "ms)"); - } - } - } - - /** - * Finish a frame. Outcomes: "sent" (counted in latency percentiles), anything - * starting with "discarded", or "timeout". First finish wins; later calls for - * the same frame are ignored, so racing completion paths are safe. - */ - public void finishFrame(int frameId, String outcome) { - long now = SystemClock.elapsedRealtime(); - Frame finished; - synchronized (lock) { - Frame frame = openFrameLocked(frameId); - if (frame == null) { - return; - } - finishFrameLocked(frame, now, outcome == null ? "discarded: no outcome given" : outcome); - finished = frame; - } - Log.i(TAG, "frame#" + finished.id + " [" + finished.reason + "] -> " + finished.outcome - + " in " + finished.durationMs() + "ms"); - } - - /** One-line stats summary, e.g. for showing in the phone UI. */ - public String statsSummary() { - synchronized (lock) { - long[] input = percentilesLocked(inputLatencies); - return "frames started=" + framesStarted + " sent=" + framesSent - + " discarded=" + framesDiscarded + " timeout=" + framesTimedOut - + (input == null - ? "" - : " | input-to-display p50=" + input[0] + "ms p90=" + input[1] - + "ms p99=" + input[2] + "ms max=" + input[3] + "ms"); - } - } - - // --------------------------------------------------------------------- - - /** The frame with this ID if it exists and is still open, else null. */ - private Frame openFrameLocked(int frameId) { - if (frameId <= 0) { - return null; - } - Frame frame = framesById.get(frameId); - return frame != null && frame.isOpen() ? frame : null; - } - - private void addLineLocked(Frame frame, long now, String message) { - if (frame.lines.size() >= MAX_LINES_PER_FRAME) { - if (frame.lines.size() == MAX_LINES_PER_FRAME) { - frame.lines.add(new Line(now, Thread.currentThread().getName(), "... line cap reached")); - } - return; - } - frame.lines.add(new Line(now, Thread.currentThread().getName(), message)); - } - - private void finishFrameLocked(Frame frame, long now, String outcome) { - frame.finishedAtMs = now; - frame.outcome = outcome; - for (Map.Entry open : frame.openSpans.entrySet()) { - frame.lines.add(new Line(now, Thread.currentThread().getName(), - "span " + open.getKey() + " never ended (started at +" - + (open.getValue() - frame.startedAtMs) + "ms)")); - } - frame.openSpans.clear(); - addLineLocked(frame, now, "finished: " + outcome); - - Frame root = frame.root(); - if (frame.wasSent()) { - framesSent++; - // The user-visible latency is input (or timer) to the first pixels - // that actually reached the glasses, wherever in the tree that was. - if (!root.subtreeSentRecorded) { - root.subtreeSentRecorded = true; - recordLatencyLocked(root, now - root.startedAtMs); - insertSlowestLocked(root); - } - } else if (outcome.startsWith("timeout")) { - framesTimedOut++; - } else { - framesDiscarded++; - } - - if (root == frame) { - recentRoots.addLast(frame); - while (recentRoots.size() > RECENT_ROOTS_KEPT) { - recentRoots.removeFirst(); - } - } - exportDirty = true; - } - - private void recordLatencyLocked(Frame root, long latencyMs) { - ArrayDeque bucket = root.reason.startsWith("input:") ? inputLatencies : renderLatencies; - bucket.addLast(latencyMs); - while (bucket.size() > SENT_DURATIONS_WINDOW) { - bucket.removeFirst(); - } - } - - private void insertSlowestLocked(Frame root) { - if (!slowestRoots.contains(root)) { - slowestRoots.add(root); - } - // Sorted and trimmed at export time, when every subtree duration is final. - } - - /** {p50, p90, p99, max} over a rolling latency window, or null if empty. */ - private long[] percentilesLocked(ArrayDeque window) { - if (window.isEmpty()) { - return null; - } - List sorted = new ArrayList<>(window); - Collections.sort(sorted); - return new long[] { - percentileOfSorted(sorted, 50), - percentileOfSorted(sorted, 90), - percentileOfSorted(sorted, 99), - sorted.get(sorted.size() - 1), - }; - } - - private static long percentileOfSorted(List sorted, int percentile) { - int index = (int) Math.ceil(percentile / 100.0 * sorted.size()) - 1; - return sorted.get(Math.max(0, Math.min(sorted.size() - 1, index))); - } - - private void sweepTimedOutFrames() { - long now = SystemClock.elapsedRealtime(); - List timedOut = new ArrayList<>(); - synchronized (lock) { - for (Frame frame : new ArrayList<>(framesById.values())) { - if (frame.isOpen() && now - frame.startedAtMs >= FRAME_TIMEOUT_MS) { - finishFrameLocked(frame, now, "timeout: never reported finished"); - timedOut.add(frame); - } - } - } - for (Frame frame : timedOut) { - Log.w(TAG, "frame#" + frame.id + " [" + frame.reason + "] timed out after " - + frame.durationMs() + "ms without being finished"); - } - } - - private void exportLoop() { - long lastExportAtMs = 0; - while (true) { - try { - Thread.sleep(SWEEP_INTERVAL_MS); - } catch (InterruptedException e) { - return; - } - try { - sweepTimedOutFrames(); - long now = SystemClock.elapsedRealtime(); - boolean shouldExport; - synchronized (lock) { - shouldExport = exportDirty && exportDir != null && now - lastExportAtMs >= EXPORT_INTERVAL_MS; - } - if (shouldExport) { - lastExportAtMs = now; - exportToFile(); - } - } catch (Throwable t) { - Log.w(TAG, "export loop error", t); - } - } - } - - private void exportToFile() { - String content; - File dir; - synchronized (lock) { - dir = exportDir; - if (dir == null) { - return; - } - content = buildExportLocked(); - exportDirty = false; - } - File target = new File(dir, EXPORT_FILE_NAME); - File temp = new File(dir, EXPORT_FILE_NAME + ".tmp"); - try { - try (Writer writer = new OutputStreamWriter(new FileOutputStream(temp), StandardCharsets.UTF_8)) { - writer.write(content); - } - if (!temp.renameTo(target)) { - Log.w(TAG, "failed to rename " + temp + " to " + target); - } - } catch (Throwable t) { - Log.w(TAG, "failed to export frame timings", t); - } - } - - private String buildExportLocked() { - long now = SystemClock.elapsedRealtime(); - SimpleDateFormat wallClockFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss.SSS", Locale.US); - StringBuilder out = new StringBuilder(64 * 1024); - out.append("FrameTimings export at ").append(wallClockFormat.format(new Date())).append('\n'); - out.append(statsSummaryLocked()).append('\n'); - out.append('\n'); - out.append("Frames are trees: an input event's own frame is the root, and the renders it\n"); - out.append("caused are indented under it with offsets measured from the root's start.\n"); - out.append('\n'); - - Collections.sort(slowestRoots, (a, b) -> - Long.compare(b.subtreeEndAtMs(now) - b.startedAtMs, a.subtreeEndAtMs(now) - a.startedAtMs)); - while (slowestRoots.size() > SLOWEST_ROOTS_KEPT) { - slowestRoots.remove(slowestRoots.size() - 1); - } - - out.append("=== slowest frames that reached the glasses ===\n"); - for (Frame frame : slowestRoots) { - appendTreeLocked(out, frame, frame, wallClockFormat, now); - } - - out.append("=== recent frames (oldest first) ===\n"); - for (Frame frame : recentRoots) { - appendTreeLocked(out, frame, frame, wallClockFormat, now); - } - return out.toString(); - } - - private String statsSummaryLocked() { - long[] input = percentilesLocked(inputLatencies); - long[] render = percentilesLocked(renderLatencies); - StringBuilder out = new StringBuilder(); - out.append("frames started=").append(framesStarted).append(" sent=").append(framesSent) - .append(" discarded=").append(framesDiscarded).append(" timeout=").append(framesTimedOut) - .append(" open=").append(openFrameCountLocked()); - appendPercentilesLocked(out, "input-to-display latency", inputLatencies.size(), input); - appendPercentilesLocked(out, "render-to-display latency", renderLatencies.size(), render); - return out.toString(); - } - - private void appendPercentilesLocked(StringBuilder out, String label, int count, long[] percentiles) { - if (percentiles == null) { - return; - } - out.append('\n').append(label).append(" (last ").append(count).append("): p50=") - .append(percentiles[0]).append("ms p90=").append(percentiles[1]) - .append("ms p99=").append(percentiles[2]).append("ms max=").append(percentiles[3]).append("ms"); - } - - private int openFrameCountLocked() { - int open = 0; - for (Frame frame : framesById.values()) { - if (frame.isOpen()) { - open++; - } - } - return open; - } - - /** Print a frame and its descendants, all timed against root's start. */ - private void appendTreeLocked( - StringBuilder out, Frame frame, Frame root, SimpleDateFormat wallClockFormat, long now) { - int depth = 0; - for (Frame walk = frame; walk != root; walk = walk.parent) { - depth++; - } - String indent = repeat(" ", depth); - long startOffsetMs = frame.startedAtMs - root.startedAtMs; - out.append(indent).append("frame#").append(frame.id) - .append(" [").append(frame.reason).append(']'); - if (frame == root) { - out.append(" started ").append(wallClockFormat.format(new Date(frame.startedWallClockMs))); - long totalMs = frame.subtreeEndAtMs(now) - frame.startedAtMs; - out.append(" duration ").append(frame.durationMs()).append("ms"); - if (!frame.children.isEmpty()) { - out.append(" (tree ").append(totalMs).append("ms") - .append(frame.subtreeHasOpenFrame() ? ", still open" : "").append(')'); - } - } else { - out.append(" started +").append(startOffsetMs).append("ms") - .append(" duration ").append(frame.isOpen() ? "open" : (frame.durationMs() + "ms")); - } - out.append(" outcome ").append(frame.outcome == null ? "(open)" : frame.outcome).append('\n'); - for (Line line : frame.lines) { - out.append(indent).append(String.format(Locale.US, " %+7dms [%s] %s", - line.atMs - root.startedAtMs, line.thread, line.message)).append('\n'); - } - for (Frame child : frame.children) { - appendTreeLocked(out, child, root, wallClockFormat, now); - } - } - - private static String repeat(String unit, int times) { - StringBuilder out = new StringBuilder(unit.length() * times); - for (int i = 0; i < times; i++) { - out.append(unit); - } - return out.toString(); - } -} diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FrameTimings.kt b/App_Resources/Android/src/main/java/com/faceclaw/app/FrameTimings.kt new file mode 100644 index 00000000..fdf2ce4d --- /dev/null +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/FrameTimings.kt @@ -0,0 +1,554 @@ +package com.faceclaw.app + +import android.content.Context +import android.os.SystemClock +import android.util.Log + +import java.io.File +import java.io.FileOutputStream +import java.io.OutputStreamWriter +import java.io.Writer +import java.nio.charset.StandardCharsets +import java.text.SimpleDateFormat +import java.util.ArrayDeque +import java.util.ArrayList +import java.util.Collections +import java.util.Date +import java.util.LinkedHashMap +import java.util.Locale + +/** + * Per-frame timing instrumentation, shared between the Java BLE layer and the + * Typescript UI layer. A "frame" starts when we receive an input event (or a + * timer fires and we decide to redraw), and finishes when the resulting screen + * update has been fully transmitted to the glasses, is discarded unsent, or + * times out because nobody reported finishing it. + * + * Frames form a tree. One input event can fan out into several renders (the + * focused app's window plus the shell chrome, say), and each of those is its + * own frame linked back to the input frame that caused it via + * [startFrame]. The export nests descendants under their + * root and renders every line against the root's clock, so a single block + * shows the whole input-to-pixels story. Latency percentiles are measured over + * roots: root start to the first descendant that actually reached the glasses. + * + * All public methods are thread-safe and cheap enough to call from the BLE + * worker thread, the Android main thread, the JS thread, and app worker + * threads. Frame IDs are positive ints; 0 means "no frame" and is silently + * ignored everywhere, so callers do not need to null-check. + * + * Statistics and full log lines for recent and slowest frames are periodically + * exported to getExternalFilesDir()/frame-timings.txt, retrievable via + * adb pull /sdcard/Android/data//files/frame-timings.txt + */ +class FrameTimings private constructor() { + companion object { + private const val TAG = "FrameTimings" + private val INSTANCE = FrameTimings() + + /** Frames still open after this long are finished as "timeout". */ + private const val FRAME_TIMEOUT_MS = 30_000L + private const val EXPORT_INTERVAL_MS = 15_000L + private const val SWEEP_INTERVAL_MS = 5_000L + private const val RECENT_ROOTS_KEPT = 40 + private const val SLOWEST_ROOTS_KEPT = 20 + private const val SENT_DURATIONS_WINDOW = 512 + private const val MAX_LINES_PER_FRAME = 200 + /** + * Frames retained for ID lookup. Well above the number any export can + * reference, so a late log/span/finish call always finds its frame; frames + * older than this are unreachable and get collected. + */ + private const val FRAMES_RETAINED = 1024 + private const val EXPORT_FILE_NAME = "frame-timings.txt" + + @JvmStatic + fun getInstance(): FrameTimings { + return INSTANCE + } + + @JvmStatic + private fun percentileOfSorted(sorted: List, percentile: Int): Long { + val index = Math.ceil(percentile / 100.0 * sorted.size).toInt() - 1 + return sorted[Math.max(0, Math.min(sorted.size - 1, index))] + } + + @JvmStatic + private fun repeat(unit: String, times: Int): String { + val out = StringBuilder(unit.length * times) + for (i in 0 until times) { + out.append(unit) + } + return out.toString() + } + } + + private class Line( + val atMs: Long, // SystemClock.elapsedRealtime() + val thread: String, + val message: String? + ) + + private class Frame( + val id: Int, + var reason: String, + val startedAtMs: Long, // SystemClock.elapsedRealtime() + val startedWallClockMs: Long, + val parent: Frame? + ) { + val children: MutableList = ArrayList(0) + val lines: MutableList = ArrayList() + val openSpans: MutableMap = LinkedHashMap() + var finishedAtMs: Long = 0 + var outcome: String? = null // null while open; "sent", "discarded: ...", "timeout: ..." + /** Set on the root once some frame in its subtree reached the glasses. */ + var subtreeSentRecorded = false + + fun root(): Frame { + var frame: Frame = this + while (frame.parent != null) { + frame = frame.parent!! + } + return frame + } + + fun isOpen(): Boolean { + return outcome == null + } + + fun durationMs(): Long { + return finishedAtMs - startedAtMs + } + + fun wasSent(): Boolean { + val outcome = this.outcome + return outcome != null && outcome.startsWith("sent") + } + + /** Last moment anything in this subtree happened, for whole-tree duration. */ + fun subtreeEndAtMs(nowMs: Long): Long { + var end = if (isOpen()) nowMs else finishedAtMs + for (child in children) { + end = Math.max(end, child.subtreeEndAtMs(nowMs)) + } + return end + } + + fun subtreeHasOpenFrame(): Boolean { + if (isOpen()) { + return true + } + for (child in children) { + if (child.subtreeHasOpenFrame()) { + return true + } + } + return false + } + } + + private val lock = Any() + /** + * Every frame we still retain, keyed by ID: open frames plus enough + * history that late calls and the export can still resolve one. + */ + private val framesById: LinkedHashMap = + object : LinkedHashMap(64, 0.75f, false) { + override fun removeEldestEntry(eldest: MutableMap.MutableEntry): Boolean { + return size > FRAMES_RETAINED && !eldest.value.isOpen() + } + } + private val recentRoots = ArrayDeque() + private val slowestRoots: MutableList = ArrayList() + private val inputLatencies = ArrayDeque() + private val renderLatencies = ArrayDeque() + private var nextFrameId = 1 + private var framesStarted: Long = 0 + private var framesSent: Long = 0 + private var framesDiscarded: Long = 0 + private var framesTimedOut: Long = 0 + private var exportDirty = false + private var exportDir: File? = null + private var exportThread: Thread? = null + + /** Idempotent; enables filesystem export. Safe to call from any constructor path. */ + fun init(context: Context) { + var dir = context.applicationContext.getExternalFilesDir(null) + if (dir == null) { + dir = context.applicationContext.filesDir + } + synchronized(lock) { + exportDir = dir + if (exportThread == null) { + val thread = Thread(Runnable { exportLoop() }, "FrameTimingsExport") + thread.isDaemon = true + thread.start() + exportThread = thread + } + } + } + + /** Begin a root frame; reason is a short label like "input:sys-event type=0". */ + fun startFrame(reason: String?): Int { + return startFrame(reason, 0) + } + + /** + * Begin a frame caused by an existing one (parentFrameId; 0 for a root). + * The child is nested under its root in the export and its latency counts + * against the root, so an input event that fans out into an app render and + * a shell-chrome render reads as one timeline instead of three. + */ + fun startFrame(reason: String?, parentFrameId: Int): Int { + val now = SystemClock.elapsedRealtime() + val frame: Frame + synchronized(lock) { + val parent = framesById[parentFrameId] + frame = Frame(nextFrameId++, reason ?: "", now, + System.currentTimeMillis(), parent) + framesById[frame.id] = frame + framesStarted++ + if (parent != null) { + parent.children.add(frame) + addLineLocked(parent, now, "spawned frame#" + frame.id + " (" + frame.reason + ")") + } + } + return frame.id + } + + /** + * Append to a frame's label, for facts only known after it started (which + * app is in the foreground, which window a render is for). Shows up in the + * frame's header line, so the export is scannable without reading bodies. + */ + fun annotate(frameId: Int, text: String?) { + if (text == null || text.isEmpty()) { + return + } + synchronized(lock) { + val frame = framesById[frameId] + if (frame == null) { + return + } + frame.reason = if (frame.reason.isEmpty()) text else frame.reason + " " + text + } + } + + fun log(frameId: Int, message: String?) { + val now = SystemClock.elapsedRealtime() + synchronized(lock) { + val frame = openFrameLocked(frameId) + if (frame == null) { + return + } + addLineLocked(frame, now, message) + } + } + + fun spanStart(frameId: Int, name: String) { + val now = SystemClock.elapsedRealtime() + synchronized(lock) { + val frame = openFrameLocked(frameId) + if (frame == null) { + return + } + frame.openSpans[name] = now + addLineLocked(frame, now, "span $name start") + } + } + + fun spanEnd(frameId: Int, name: String) { + val now = SystemClock.elapsedRealtime() + synchronized(lock) { + val frame = openFrameLocked(frameId) + if (frame == null) { + return + } + val startedAt = frame.openSpans.remove(name) + if (startedAt == null) { + addLineLocked(frame, now, "span $name end (start not recorded)") + } else { + addLineLocked(frame, now, "span " + name + " end (" + (now - startedAt) + "ms)") + } + } + } + + /** + * Finish a frame. Outcomes: "sent" (counted in latency percentiles), anything + * starting with "discarded", or "timeout". First finish wins; later calls for + * the same frame are ignored, so racing completion paths are safe. + */ + fun finishFrame(frameId: Int, outcome: String?) { + val now = SystemClock.elapsedRealtime() + val finished: Frame + synchronized(lock) { + val frame = openFrameLocked(frameId) + if (frame == null) { + return + } + finishFrameLocked(frame, now, outcome ?: "discarded: no outcome given") + finished = frame + } + Log.i(TAG, "frame#" + finished.id + " [" + finished.reason + "] -> " + finished.outcome + + " in " + finished.durationMs() + "ms") + } + + /** One-line stats summary, e.g. for showing in the phone UI. */ + fun statsSummary(): String { + synchronized(lock) { + val input = percentilesLocked(inputLatencies) + return "frames started=" + framesStarted + " sent=" + framesSent + + " discarded=" + framesDiscarded + " timeout=" + framesTimedOut + + (if (input == null) + "" + else " | input-to-display p50=" + input[0] + "ms p90=" + input[1] + + "ms p99=" + input[2] + "ms max=" + input[3] + "ms") + } + } + + // --------------------------------------------------------------------- + + /** The frame with this ID if it exists and is still open, else null. */ + private fun openFrameLocked(frameId: Int): Frame? { + if (frameId <= 0) { + return null + } + val frame = framesById[frameId] + return if (frame != null && frame.isOpen()) frame else null + } + + private fun addLineLocked(frame: Frame, now: Long, message: String?) { + if (frame.lines.size >= MAX_LINES_PER_FRAME) { + if (frame.lines.size == MAX_LINES_PER_FRAME) { + frame.lines.add(Line(now, Thread.currentThread().name, "... line cap reached")) + } + return + } + frame.lines.add(Line(now, Thread.currentThread().name, message)) + } + + private fun finishFrameLocked(frame: Frame, now: Long, outcome: String) { + frame.finishedAtMs = now + frame.outcome = outcome + for (open in frame.openSpans.entries) { + frame.lines.add(Line(now, Thread.currentThread().name, + "span " + open.key + " never ended (started at +" + + (open.value - frame.startedAtMs) + "ms)")) + } + frame.openSpans.clear() + addLineLocked(frame, now, "finished: $outcome") + + val root = frame.root() + if (frame.wasSent()) { + framesSent++ + // The user-visible latency is input (or timer) to the first pixels + // that actually reached the glasses, wherever in the tree that was. + if (!root.subtreeSentRecorded) { + root.subtreeSentRecorded = true + recordLatencyLocked(root, now - root.startedAtMs) + insertSlowestLocked(root) + } + } else if (outcome.startsWith("timeout")) { + framesTimedOut++ + } else { + framesDiscarded++ + } + + if (root === frame) { + recentRoots.addLast(frame) + while (recentRoots.size > RECENT_ROOTS_KEPT) { + recentRoots.removeFirst() + } + } + exportDirty = true + } + + private fun recordLatencyLocked(root: Frame, latencyMs: Long) { + val bucket = if (root.reason.startsWith("input:")) inputLatencies else renderLatencies + bucket.addLast(latencyMs) + while (bucket.size > SENT_DURATIONS_WINDOW) { + bucket.removeFirst() + } + } + + private fun insertSlowestLocked(root: Frame) { + if (!slowestRoots.contains(root)) { + slowestRoots.add(root) + } + // Sorted and trimmed at export time, when every subtree duration is final. + } + + /** {p50, p90, p99, max} over a rolling latency window, or null if empty. */ + private fun percentilesLocked(window: ArrayDeque): LongArray? { + if (window.isEmpty()) { + return null + } + val sorted: MutableList = ArrayList(window) + Collections.sort(sorted) + return longArrayOf( + percentileOfSorted(sorted, 50), + percentileOfSorted(sorted, 90), + percentileOfSorted(sorted, 99), + sorted[sorted.size - 1], + ) + } + + private fun sweepTimedOutFrames() { + val now = SystemClock.elapsedRealtime() + val timedOut: MutableList = ArrayList() + synchronized(lock) { + for (frame in ArrayList(framesById.values)) { + if (frame.isOpen() && now - frame.startedAtMs >= FRAME_TIMEOUT_MS) { + finishFrameLocked(frame, now, "timeout: never reported finished") + timedOut.add(frame) + } + } + } + for (frame in timedOut) { + Log.w(TAG, "frame#" + frame.id + " [" + frame.reason + "] timed out after " + + frame.durationMs() + "ms without being finished") + } + } + + private fun exportLoop() { + var lastExportAtMs = 0L + while (true) { + try { + Thread.sleep(SWEEP_INTERVAL_MS) + } catch (e: InterruptedException) { + return + } + try { + sweepTimedOutFrames() + val now = SystemClock.elapsedRealtime() + val shouldExport: Boolean + synchronized(lock) { + shouldExport = exportDirty && exportDir != null && now - lastExportAtMs >= EXPORT_INTERVAL_MS + } + if (shouldExport) { + lastExportAtMs = now + exportToFile() + } + } catch (t: Throwable) { + Log.w(TAG, "export loop error", t) + } + } + } + + private fun exportToFile() { + val content: String + val dir: File + synchronized(lock) { + dir = exportDir ?: return + content = buildExportLocked() + exportDirty = false + } + val target = File(dir, EXPORT_FILE_NAME) + val temp = File(dir, "$EXPORT_FILE_NAME.tmp") + try { + (OutputStreamWriter(FileOutputStream(temp), StandardCharsets.UTF_8) as Writer).use { writer -> + writer.write(content) + } + if (!temp.renameTo(target)) { + Log.w(TAG, "failed to rename $temp to $target") + } + } catch (t: Throwable) { + Log.w(TAG, "failed to export frame timings", t) + } + } + + private fun buildExportLocked(): String { + val now = SystemClock.elapsedRealtime() + val wallClockFormat = SimpleDateFormat("yyyy-MM-dd HH:mm:ss.SSS", Locale.US) + val out = StringBuilder(64 * 1024) + out.append("FrameTimings export at ").append(wallClockFormat.format(Date())).append('\n') + out.append(statsSummaryLocked()).append('\n') + out.append('\n') + out.append("Frames are trees: an input event's own frame is the root, and the renders it\n") + out.append("caused are indented under it with offsets measured from the root's start.\n") + out.append('\n') + + Collections.sort(slowestRoots, Comparator { a, b -> + java.lang.Long.compare(b.subtreeEndAtMs(now) - b.startedAtMs, a.subtreeEndAtMs(now) - a.startedAtMs) }) + while (slowestRoots.size > SLOWEST_ROOTS_KEPT) { + slowestRoots.removeAt(slowestRoots.size - 1) + } + + out.append("=== slowest frames that reached the glasses ===\n") + for (frame in slowestRoots) { + appendTreeLocked(out, frame, frame, wallClockFormat, now) + } + + out.append("=== recent frames (oldest first) ===\n") + for (frame in recentRoots) { + appendTreeLocked(out, frame, frame, wallClockFormat, now) + } + return out.toString() + } + + private fun statsSummaryLocked(): String { + val input = percentilesLocked(inputLatencies) + val render = percentilesLocked(renderLatencies) + val out = StringBuilder() + out.append("frames started=").append(framesStarted).append(" sent=").append(framesSent) + .append(" discarded=").append(framesDiscarded).append(" timeout=").append(framesTimedOut) + .append(" open=").append(openFrameCountLocked()) + appendPercentilesLocked(out, "input-to-display latency", inputLatencies.size, input) + appendPercentilesLocked(out, "render-to-display latency", renderLatencies.size, render) + return out.toString() + } + + private fun appendPercentilesLocked(out: StringBuilder, label: String, count: Int, percentiles: LongArray?) { + if (percentiles == null) { + return + } + out.append('\n').append(label).append(" (last ").append(count).append("): p50=") + .append(percentiles[0]).append("ms p90=").append(percentiles[1]) + .append("ms p99=").append(percentiles[2]).append("ms max=").append(percentiles[3]).append("ms") + } + + private fun openFrameCountLocked(): Int { + var open = 0 + for (frame in framesById.values) { + if (frame.isOpen()) { + open++ + } + } + return open + } + + /** Print a frame and its descendants, all timed against root's start. */ + private fun appendTreeLocked( + out: StringBuilder, frame: Frame, root: Frame, wallClockFormat: SimpleDateFormat, now: Long) { + var depth = 0 + var walk: Frame? = frame + while (walk !== root) { + depth++ + walk = walk!!.parent + } + val indent = repeat(" ", depth) + val startOffsetMs = frame.startedAtMs - root.startedAtMs + out.append(indent).append("frame#").append(frame.id) + .append(" [").append(frame.reason).append(']') + if (frame === root) { + out.append(" started ").append(wallClockFormat.format(Date(frame.startedWallClockMs))) + val totalMs = frame.subtreeEndAtMs(now) - frame.startedAtMs + out.append(" duration ").append(frame.durationMs()).append("ms") + if (!frame.children.isEmpty()) { + out.append(" (tree ").append(totalMs).append("ms") + .append(if (frame.subtreeHasOpenFrame()) ", still open" else "").append(')') + } + } else { + out.append(" started +").append(startOffsetMs).append("ms") + .append(" duration ").append(if (frame.isOpen()) "open" else (frame.durationMs().toString() + "ms")) + } + out.append(" outcome ").append(frame.outcome ?: "(open)").append('\n') + for (line in frame.lines) { + out.append(indent).append(String.format(Locale.US, " %+7dms [%s] %s", + line.atMs - root.startedAtMs, line.thread, line.message)).append('\n') + } + for (child in frame.children) { + appendTreeLocked(out, child, root, wallClockFormat, now) + } + } +} diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/GifScreenRecorder.java b/App_Resources/Android/src/main/java/com/faceclaw/app/GifScreenRecorder.java deleted file mode 100644 index 1346b07e..00000000 --- a/App_Resources/Android/src/main/java/com/faceclaw/app/GifScreenRecorder.java +++ /dev/null @@ -1,26 +0,0 @@ -package com.faceclaw.app; - -import android.content.Context; -import java.io.File; -import java.io.FileOutputStream; -import java.io.IOException; - -/** Android storage adapter for the shared GIF recorder. */ -public final class GifScreenRecorder { - private final SharedGifScreenRecorder recorder = new SharedGifScreenRecorder(); - - public void addFrame(byte[] gray, int width, int height, long timestampMs) { - recorder.addFrame(gray, width, height, timestampMs); - } - - public boolean isOverflowed() { return recorder.isOverflowed(); } - - public String save(Context context) throws IOException { - byte[] gif = recorder.encode(); - if (gif.length == 0) return ""; - File file = new File(ScreenshotUtil.ensureScreenshotsDir(context), - "recording-" + ScreenshotUtil.timestamp() + ".gif"); - try (FileOutputStream out = new FileOutputStream(file)) { out.write(gif); } - return file.getAbsolutePath(); - } -} diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/GifScreenRecorder.kt b/App_Resources/Android/src/main/java/com/faceclaw/app/GifScreenRecorder.kt new file mode 100644 index 00000000..e2fde296 --- /dev/null +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/GifScreenRecorder.kt @@ -0,0 +1,27 @@ +package com.faceclaw.app + +import android.content.Context +import java.io.File +import java.io.FileOutputStream +import java.io.IOException + +/** Android storage adapter for the shared GIF recorder. */ +class GifScreenRecorder { + private val recorder = SharedGifScreenRecorder() + + fun addFrame(gray: ByteArray?, width: Int, height: Int, timestampMs: Long) { + recorder.addFrame(gray, width, height, timestampMs) + } + + fun isOverflowed(): Boolean = recorder.isOverflowed() + + @Throws(IOException::class) + fun save(context: Context): String { + val gif = recorder.encode() + if (gif.isEmpty()) return "" + val file = File(ScreenshotUtil.ensureScreenshotsDir(context), + "recording-" + ScreenshotUtil.timestamp() + ".gif") + FileOutputStream(file).use { out -> out.write(gif) } + return file.absolutePath + } +} diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/IconRenderer.java b/App_Resources/Android/src/main/java/com/faceclaw/app/IconRenderer.java deleted file mode 100644 index 80589f47..00000000 --- a/App_Resources/Android/src/main/java/com/faceclaw/app/IconRenderer.java +++ /dev/null @@ -1,185 +0,0 @@ -package com.faceclaw.app; - -import android.graphics.Bitmap; -import android.graphics.Canvas; -import android.graphics.Paint; -import android.graphics.Path; -import android.util.Log; - -import androidx.core.graphics.PathParser; - -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -/** - * Renders a small subset of SVG (the shapes used by Lucide / simple Noun - * Project icons: path, circle, ellipse, rect, line, polyline, polygon) to a - * grayscale coverage buffer for use as an on-glasses icon. Icons are stroked - * (Lucide style: fill="none", 2px stroke, round caps/joins); the returned - * bytes are one coverage value per pixel (0=transparent .. 255=opaque white), - * row-major, size*size. Called once per icon and cached on the TS side. - */ -public final class IconRenderer { - private static final String TAG = "IconRenderer"; - private static final Pattern TAG_RE = Pattern.compile("<(path|circle|ellipse|rect|line|polyline|polygon)\\b([^>]*)>"); - private static final Pattern VIEWBOX_RE = Pattern.compile("viewBox\\s*=\\s*\"([^\"]*)\""); - - private IconRenderer() {} - - public static byte[] renderSvgGray(String svg, int size, float strokeWidth) { - if (svg == null || size <= 0) { - return new byte[0]; - } - try { - float minX = 0f, minY = 0f, viewW = 24f, viewH = 24f; - Matcher vb = VIEWBOX_RE.matcher(svg); - if (vb.find()) { - String[] parts = vb.group(1).trim().split("[\\s,]+"); - if (parts.length == 4) { - minX = Float.parseFloat(parts[0]); - minY = Float.parseFloat(parts[1]); - viewW = Float.parseFloat(parts[2]); - viewH = Float.parseFloat(parts[3]); - } - } - // Lucide-style icons are stroked outlines (fill="none"); everything - // else (Noun Project glyphs, brand logos) is a filled shape. - boolean stroked = svg.contains("fill=\"none\""); - Path path = new Path(); - Matcher m = TAG_RE.matcher(svg); - while (m.find()) { - appendElement(path, m.group(1), m.group(2)); - } - if (path.isEmpty()) { - return new byte[0]; - } - - Bitmap bitmap = Bitmap.createBitmap(size, size, Bitmap.Config.ARGB_8888); - Canvas canvas = new Canvas(bitmap); - float scale = Math.min(size / viewW, size / viewH); - // Center the (square) viewBox in the bitmap, then map its origin. - canvas.translate((size - viewW * scale) / 2f, (size - viewH * scale) / 2f); - canvas.scale(scale, scale); - canvas.translate(-minX, -minY); - Paint paint = new Paint(Paint.ANTI_ALIAS_FLAG); - paint.setColor(0xffffffff); - if (stroked) { - paint.setStyle(Paint.Style.STROKE); - paint.setStrokeWidth(strokeWidth); - paint.setStrokeCap(Paint.Cap.ROUND); - paint.setStrokeJoin(Paint.Join.ROUND); - } else { - paint.setStyle(Paint.Style.FILL); - } - canvas.drawPath(path, paint); - - int[] pixels = new int[size * size]; - bitmap.getPixels(pixels, 0, size, 0, 0, size, size); - byte[] gray = new byte[size * size]; - for (int i = 0; i < pixels.length; i++) { - gray[i] = (byte) quantizeCoverage((pixels[i] >>> 24) & 0xff); // alpha = coverage - } - return gray; - } catch (Exception e) { - Log.w(TAG, "icon render failed", e); - return new byte[0]; - } - } - - /** - * Snap antialiased coverage toward hard edges so the 4bpp frames sent - * over BLE deflate better: mostly-transparent and mostly-opaque pixels - * become exactly 0/255 (long runs, and 0 stays color-key transparent for - * bitBlt), and only genuinely half-covered edge pixels keep one mid gray. - * The packed frame then uses a 3-symbol alphabet instead of all 16 - * levels, at icon sizes this is visually near-identical. - */ - private static int quantizeCoverage(int alpha) { - if (alpha < 64) return 0; - if (alpha >= 192) return 255; - return 128; - } - - private static void appendElement(Path path, String tag, String attrs) { - switch (tag) { - case "path": { - String d = attr(attrs, "d"); - if (d != null) { - Path sub = PathParser.createPathFromPathData(d); - if (sub != null) { - path.addPath(sub); - } - } - break; - } - case "circle": { - float cx = num(attrs, "cx", 0), cy = num(attrs, "cy", 0), r = num(attrs, "r", 0); - if (r > 0) path.addCircle(cx, cy, r, Path.Direction.CW); - break; - } - case "ellipse": { - float cx = num(attrs, "cx", 0), cy = num(attrs, "cy", 0); - float rx = num(attrs, "rx", 0), ry = num(attrs, "ry", 0); - if (rx > 0 && ry > 0) path.addOval(cx - rx, cy - ry, cx + rx, cy + ry, Path.Direction.CW); - break; - } - case "rect": { - float x = num(attrs, "x", 0), y = num(attrs, "y", 0); - float w = num(attrs, "width", 0), h = num(attrs, "height", 0); - float rx = num(attrs, "rx", 0), ry = num(attrs, "ry", rx > 0 ? rx : 0); - if (rx <= 0) rx = ry; - if (ry <= 0) ry = rx; - if (w > 0 && h > 0) { - if (rx > 0 || ry > 0) { - path.addRoundRect(x, y, x + w, y + h, rx, ry, Path.Direction.CW); - } else { - path.addRect(x, y, x + w, y + h, Path.Direction.CW); - } - } - break; - } - case "line": { - path.moveTo(num(attrs, "x1", 0), num(attrs, "y1", 0)); - path.lineTo(num(attrs, "x2", 0), num(attrs, "y2", 0)); - break; - } - case "polyline": - case "polygon": { - String points = attr(attrs, "points"); - if (points != null) { - String[] nums = points.trim().split("[\\s,]+"); - boolean started = false; - for (int i = 0; i + 1 < nums.length; i += 2) { - float px = Float.parseFloat(nums[i]); - float py = Float.parseFloat(nums[i + 1]); - if (!started) { - path.moveTo(px, py); - started = true; - } else { - path.lineTo(px, py); - } - } - if (tag.equals("polygon")) path.close(); - } - break; - } - default: - break; - } - } - - private static String attr(String attrs, String name) { - Matcher m = Pattern.compile("\\b" + name + "\\s*=\\s*\"([^\"]*)\"").matcher(attrs); - return m.find() ? m.group(1) : null; - } - - private static float num(String attrs, String name, float fallback) { - String value = attr(attrs, name); - if (value == null) return fallback; - try { - return Float.parseFloat(value.trim()); - } catch (NumberFormatException e) { - return fallback; - } - } -} diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/IconRenderer.kt b/App_Resources/Android/src/main/java/com/faceclaw/app/IconRenderer.kt new file mode 100644 index 00000000..da181f0d --- /dev/null +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/IconRenderer.kt @@ -0,0 +1,196 @@ +package com.faceclaw.app + +import android.graphics.Bitmap +import android.graphics.Canvas +import android.graphics.Paint +import android.graphics.Path +import android.util.Log + +import androidx.core.graphics.PathParser + +import java.util.regex.Matcher +import java.util.regex.Pattern + +/** + * Renders a small subset of SVG (the shapes used by Lucide / simple Noun + * Project icons: path, circle, ellipse, rect, line, polyline, polygon) to a + * grayscale coverage buffer for use as an on-glasses icon. Icons are stroked + * (Lucide style: fill="none", 2px stroke, round caps/joins); the returned + * bytes are one coverage value per pixel (0=transparent .. 255=opaque white), + * row-major, size*size. Called once per icon and cached on the TS side. + */ +class IconRenderer private constructor() { + companion object { + private const val TAG = "IconRenderer" + private val TAG_RE: Pattern = Pattern.compile("<(path|circle|ellipse|rect|line|polyline|polygon)\\b([^>]*)>") + private val VIEWBOX_RE: Pattern = Pattern.compile("viewBox\\s*=\\s*\"([^\"]*)\"") + private val SEPARATOR_RE: Pattern = Pattern.compile("[\\s,]+") + + @JvmStatic + fun renderSvgGray(svg: String?, size: Int, strokeWidth: Float): ByteArray { + if (svg == null || size <= 0) { + return ByteArray(0) + } + try { + var minX = 0f + var minY = 0f + var viewW = 24f + var viewH = 24f + val vb = VIEWBOX_RE.matcher(svg) + if (vb.find()) { + val parts = SEPARATOR_RE.split(vb.group(1)!!.trim()) + if (parts.size == 4) { + minX = java.lang.Float.parseFloat(parts[0]) + minY = java.lang.Float.parseFloat(parts[1]) + viewW = java.lang.Float.parseFloat(parts[2]) + viewH = java.lang.Float.parseFloat(parts[3]) + } + } + // Lucide-style icons are stroked outlines (fill="none"); everything + // else (Noun Project glyphs, brand logos) is a filled shape. + val stroked = svg.contains("fill=\"none\"") + val path = Path() + val m = TAG_RE.matcher(svg) + while (m.find()) { + appendElement(path, m.group(1)!!, m.group(2)!!) + } + if (path.isEmpty) { + return ByteArray(0) + } + + val bitmap = Bitmap.createBitmap(size, size, Bitmap.Config.ARGB_8888) + val canvas = Canvas(bitmap) + val scale = Math.min(size / viewW, size / viewH) + // Center the (square) viewBox in the bitmap, then map its origin. + canvas.translate((size - viewW * scale) / 2f, (size - viewH * scale) / 2f) + canvas.scale(scale, scale) + canvas.translate(-minX, -minY) + val paint = Paint(Paint.ANTI_ALIAS_FLAG) + paint.color = 0xffffffff.toInt() + if (stroked) { + paint.style = Paint.Style.STROKE + paint.strokeWidth = strokeWidth + paint.strokeCap = Paint.Cap.ROUND + paint.strokeJoin = Paint.Join.ROUND + } else { + paint.style = Paint.Style.FILL + } + canvas.drawPath(path, paint) + + val pixels = IntArray(size * size) + bitmap.getPixels(pixels, 0, size, 0, 0, size, size) + val gray = ByteArray(size * size) + for (i in pixels.indices) { + gray[i] = quantizeCoverage((pixels[i] ushr 24) and 0xff).toByte() // alpha = coverage + } + return gray + } catch (e: Exception) { + Log.w(TAG, "icon render failed", e) + return ByteArray(0) + } + } + + /** + * Snap antialiased coverage toward hard edges so the 4bpp frames sent + * over BLE deflate better: mostly-transparent and mostly-opaque pixels + * become exactly 0/255 (long runs, and 0 stays color-key transparent for + * bitBlt), and only genuinely half-covered edge pixels keep one mid gray. + * The packed frame then uses a 3-symbol alphabet instead of all 16 + * levels, at icon sizes this is visually near-identical. + */ + @JvmStatic + private fun quantizeCoverage(alpha: Int): Int { + if (alpha < 64) return 0 + if (alpha >= 192) return 255 + return 128 + } + + @JvmStatic + private fun appendElement(path: Path, tag: String, attrs: String) { + when (tag) { + "path" -> { + val d = attr(attrs, "d") + if (d != null) { + val sub = PathParser.createPathFromPathData(d) + if (sub != null) { + path.addPath(sub) + } + } + } + "circle" -> { + val cx = num(attrs, "cx", 0f) + val cy = num(attrs, "cy", 0f) + val r = num(attrs, "r", 0f) + if (r > 0) path.addCircle(cx, cy, r, Path.Direction.CW) + } + "ellipse" -> { + val cx = num(attrs, "cx", 0f) + val cy = num(attrs, "cy", 0f) + val rx = num(attrs, "rx", 0f) + val ry = num(attrs, "ry", 0f) + if (rx > 0 && ry > 0) path.addOval(cx - rx, cy - ry, cx + rx, cy + ry, Path.Direction.CW) + } + "rect" -> { + val x = num(attrs, "x", 0f) + val y = num(attrs, "y", 0f) + val w = num(attrs, "width", 0f) + val h = num(attrs, "height", 0f) + var rx = num(attrs, "rx", 0f) + var ry = num(attrs, "ry", if (rx > 0) rx else 0f) + if (rx <= 0) rx = ry + if (ry <= 0) ry = rx + if (w > 0 && h > 0) { + if (rx > 0 || ry > 0) { + path.addRoundRect(x, y, x + w, y + h, rx, ry, Path.Direction.CW) + } else { + path.addRect(x, y, x + w, y + h, Path.Direction.CW) + } + } + } + "line" -> { + path.moveTo(num(attrs, "x1", 0f), num(attrs, "y1", 0f)) + path.lineTo(num(attrs, "x2", 0f), num(attrs, "y2", 0f)) + } + "polyline", "polygon" -> { + val points = attr(attrs, "points") + if (points != null) { + val nums = SEPARATOR_RE.split(points.trim()) + var started = false + var i = 0 + while (i + 1 < nums.size) { + val px = java.lang.Float.parseFloat(nums[i]) + val py = java.lang.Float.parseFloat(nums[i + 1]) + if (!started) { + path.moveTo(px, py) + started = true + } else { + path.lineTo(px, py) + } + i += 2 + } + if (tag == "polygon") path.close() + } + } + else -> { + } + } + } + + @JvmStatic + private fun attr(attrs: String, name: String): String? { + val m: Matcher = Pattern.compile("\\b" + name + "\\s*=\\s*\"([^\"]*)\"").matcher(attrs) + return if (m.find()) m.group(1) else null + } + + @JvmStatic + private fun num(attrs: String, name: String, fallback: Float): Float { + val value = attr(attrs, name) + if (value == null) return fallback + try { + return java.lang.Float.parseFloat(value.trim()) + } catch (e: NumberFormatException) { + return fallback + } + } + } +} diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/ImageFileLoader.java b/App_Resources/Android/src/main/java/com/faceclaw/app/ImageFileLoader.java deleted file mode 100644 index ef6debbb..00000000 --- a/App_Resources/Android/src/main/java/com/faceclaw/app/ImageFileLoader.java +++ /dev/null @@ -1,229 +0,0 @@ -package com.faceclaw.app; - -import android.graphics.Bitmap; -import android.graphics.BitmapFactory; -import android.util.Log; - -/** - * Decodes image files (and bitmaps generally) to the grayscale packet format - * used across the TS bridge: [widthLo, widthHi, heightLo, heightHi, - * pixels...] with one byte per pixel, row-major, or an empty array on - * failure. FaceclawMediaController's album art shares bitmapToGrayPacket; - * the TS side (app/native/image-files.ts) owns the shared photo-tone preset - * passed as (gamma, dither). - */ -public final class ImageFileLoader { - private static final String TAG = "ImageFileLoader"; - - private ImageFileLoader() {} - - /** - * Decode an image file and downscale it to fit within maxWidth x - * maxHeight, preserving aspect ratio and never upscaling. Flat - * conversion (no tone curve, per-pixel rounding): right for UI raster - * such as icons. - */ - public static byte[] loadGray(String path, int maxWidth, int maxHeight) { - return loadGray(path, maxWidth, maxHeight, 1f, false); - } - - /** - * As above, with the photographic tone handling described on - * bitmapToGrayPacket(Bitmap, int, int, float, boolean): use for - * continuous-tone images (photos) viewed on the glasses. - */ - public static byte[] loadGray( - String path, int maxWidth, int maxHeight, float gamma, boolean dither) { - if (path == null || maxWidth <= 0 || maxHeight <= 0) { - return new byte[0]; - } - try { - BitmapFactory.Options bounds = new BitmapFactory.Options(); - bounds.inJustDecodeBounds = true; - BitmapFactory.decodeFile(path, bounds); - if (bounds.outWidth <= 0 || bounds.outHeight <= 0) { - return new byte[0]; - } - // Power-of-two subsampling during decode keeps large photos from - // allocating full-size ARGB buffers; exact fitting happens in - // bitmapToGrayPacket. - BitmapFactory.Options opts = new BitmapFactory.Options(); - opts.inSampleSize = 1; - while (bounds.outWidth / (opts.inSampleSize * 2) >= maxWidth - && bounds.outHeight / (opts.inSampleSize * 2) >= maxHeight) { - opts.inSampleSize *= 2; - } - Bitmap bitmap = BitmapFactory.decodeFile(path, opts); - if (bitmap == null) { - return new byte[0]; - } - return bitmapToGrayPacket(bitmap, maxWidth, maxHeight, gamma, dither); - } catch (Exception | OutOfMemoryError e) { - Log.w(TAG, "image decode failed: " + path, e); - return new byte[0]; - } - } - - /** - * Decode an in-memory image file (PNG/JPEG/WebP bytes, e.g. a fetched - * map tile) and downscale to fit within maxWidth x maxHeight. Takes a - * ByteBuffer because NativeScript marshals a JS ArrayBuffer to one. - */ - public static byte[] loadGrayFromBytes(java.nio.ByteBuffer buffer, int maxWidth, int maxHeight) { - return loadGrayFromBytes(buffer, maxWidth, maxHeight, 1f, false); - } - - /** As above, with photographic tone handling (see bitmapToGrayPacket). */ - public static byte[] loadGrayFromBytes( - java.nio.ByteBuffer buffer, int maxWidth, int maxHeight, float gamma, boolean dither) { - if (buffer == null || maxWidth <= 0 || maxHeight <= 0) { - return new byte[0]; - } - try { - java.nio.ByteBuffer source = buffer.duplicate(); - source.rewind(); - byte[] data = new byte[source.remaining()]; - source.get(data); - if (data.length == 0) { - return new byte[0]; - } - Bitmap bitmap = BitmapFactory.decodeByteArray(data, 0, data.length); - if (bitmap == null) { - return new byte[0]; - } - return bitmapToGrayPacket(bitmap, maxWidth, maxHeight, gamma, dither); - } catch (Exception | OutOfMemoryError e) { - Log.w(TAG, "byte-array image decode failed", e); - return new byte[0]; - } - } - - /** - * Scale a bitmap to fit within maxWidth x maxHeight (preserving aspect, - * never upscaling) and convert it to the grayscale packet format. - * Transparent pixels darken toward black (the on-glasses background). - */ - public static byte[] bitmapToGrayPacket(Bitmap source, int maxWidth, int maxHeight) { - return bitmapToGrayPacket(source, maxWidth, maxHeight, 1f, false); - } - - /** - * As above, with photographic tone handling for continuous-tone images - * (album art, photos) rather than UI raster: - * - * gamma > 1 darkens midtones (out = 255 * (in/255)^gamma). Source pixels - * are sRGB-encoded but the G2 drives its 16 levels roughly linearly, so - * mid-gray otherwise displays far brighter than intended; 2.2 undoes the - * sRGB encoding outright. - * - * dither error-diffuses against the 16 levels the frame is eventually - * packed to (BmpUtil.GRAY_TO_NIBBLE) instead of leaving each pixel to - * round on its own, which turns smooth gradients into visible bands. - * Dithered output is already quantized: every byte is a level times 16, - * which BmpUtil re-quantizes back to exactly that level. - */ - public static byte[] bitmapToGrayPacket( - Bitmap source, int maxWidth, int maxHeight, float gamma, boolean dither) { - if (source == null || source.getWidth() <= 0 || source.getHeight() <= 0 - || maxWidth <= 0 || maxHeight <= 0) { - return new byte[0]; - } - try { - float scale = Math.min(1f, Math.min( - (float) maxWidth / source.getWidth(), - (float) maxHeight / source.getHeight())); - int width = Math.max(1, Math.round(source.getWidth() * scale)); - int height = Math.max(1, Math.round(source.getHeight() * scale)); - Bitmap scaled = Bitmap.createScaledBitmap(source, width, height, true); - if (scaled.getConfig() == Bitmap.Config.HARDWARE) { - scaled = scaled.copy(Bitmap.Config.ARGB_8888, false); - } - int[] pixels = new int[width * height]; - scaled.getPixels(pixels, 0, width, 0, 0, width, height); - byte[] out = new byte[4 + width * height]; - out[0] = (byte) (width & 0xff); - out[1] = (byte) ((width >> 8) & 0xff); - out[2] = (byte) (height & 0xff); - out[3] = (byte) ((height >> 8) & 0xff); - int[] tone = toneCurve(gamma); - for (int i = 0; i < pixels.length; i++) { - int p = pixels[i]; - int a = (p >>> 24) & 0xff; - int r = (p >> 16) & 0xff; - int g = (p >> 8) & 0xff; - int b = p & 0xff; - out[4 + i] = (byte) tone[((r * 299 + g * 587 + b * 114) / 1000) * a / 255]; - } - if (dither) { - ditherToDisplayLevels(out, 4, width, height); - } - return out; - } catch (Exception | OutOfMemoryError e) { - Log.w(TAG, "bitmap conversion failed", e); - return new byte[0]; - } - } - - /** 256-entry map of source gray to displayed gray for out = in^gamma. */ - private static int[] toneCurve(float gamma) { - int[] curve = new int[256]; - for (int v = 0; v < 256; v++) { - curve[v] = gamma == 1f - ? v - : Math.round(255f * (float) Math.pow(v / 255f, gamma)); - } - return curve; - } - - /** - * Floyd-Steinberg error diffusion onto the display's 16 gray levels, in - * place over an 8bpp plane. Serpentine scanning keeps the diffusion from - * building up a directional texture across wide flat areas. - * - * Level n is written back as n * 16 rather than the n * 17 it stands for - * so that BmpUtil's (v + 8) >> 4 reproduces n exactly; level 0 is written - * as 1, not 0, because 0 is the shell's color-key for transparent. - */ - private static void ditherToDisplayLevels(byte[] plane, int offset, int width, int height) { - float[] curr = new float[width]; - float[] next = new float[width]; - for (int x = 0; x < width; x++) { - curr[x] = plane[offset + x] & 0xff; - } - for (int y = 0; y < height; y++) { - int rowStart = offset + y * width; - int nextStart = rowStart + width; - boolean hasNext = y + 1 < height; - for (int x = 0; x < width; x++) { - next[x] = hasNext ? (plane[nextStart + x] & 0xff) : 0f; - } - boolean leftToRight = (y & 1) == 0; - int start = leftToRight ? 0 : width - 1; - int step = leftToRight ? 1 : -1; - for (int i = 0; i < width; i++) { - int x = start + i * step; - float wanted = curr[x]; - int level = Math.round(Math.min(255f, Math.max(0f, wanted)) / 17f); - float error = wanted - level * 17f; - plane[rowStart + x] = (byte) (level == 0 ? 1 : level * 16); - int ahead = x + step; - if (ahead >= 0 && ahead < width) { - curr[ahead] += error * (7f / 16f); - } - if (hasNext) { - if (ahead >= 0 && ahead < width) { - next[ahead] += error * (1f / 16f); - } - next[x] += error * (5f / 16f); - int behind = x - step; - if (behind >= 0 && behind < width) { - next[behind] += error * (3f / 16f); - } - } - } - float[] swap = curr; - curr = next; - next = swap; - } - } -} diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/ImageFileLoader.kt b/App_Resources/Android/src/main/java/com/faceclaw/app/ImageFileLoader.kt new file mode 100644 index 00000000..d1b6c300 --- /dev/null +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/ImageFileLoader.kt @@ -0,0 +1,246 @@ +package com.faceclaw.app + +import android.graphics.Bitmap +import android.graphics.BitmapFactory +import android.util.Log + +/** + * Decodes image files (and bitmaps generally) to the grayscale packet format + * used across the TS bridge: [widthLo, widthHi, heightLo, heightHi, + * pixels...] with one byte per pixel, row-major, or an empty array on + * failure. FaceclawMediaController's album art shares bitmapToGrayPacket; + * the TS side (app/native/image-files.ts) owns the shared photo-tone preset + * passed as (gamma, dither). + */ +class ImageFileLoader private constructor() { + companion object { + private const val TAG = "ImageFileLoader" + + /** + * Decode an image file and downscale it to fit within maxWidth x + * maxHeight, preserving aspect ratio and never upscaling. Flat + * conversion (no tone curve, per-pixel rounding): right for UI raster + * such as icons. + */ + @JvmStatic + fun loadGray(path: String?, maxWidth: Int, maxHeight: Int): ByteArray { + return loadGray(path, maxWidth, maxHeight, 1f, false) + } + + /** + * As above, with the photographic tone handling described on + * bitmapToGrayPacket(Bitmap, int, int, float, boolean): use for + * continuous-tone images (photos) viewed on the glasses. + */ + @JvmStatic + fun loadGray( + path: String?, maxWidth: Int, maxHeight: Int, gamma: Float, dither: Boolean): ByteArray { + if (path == null || maxWidth <= 0 || maxHeight <= 0) { + return ByteArray(0) + } + try { + val bounds = BitmapFactory.Options() + bounds.inJustDecodeBounds = true + BitmapFactory.decodeFile(path, bounds) + if (bounds.outWidth <= 0 || bounds.outHeight <= 0) { + return ByteArray(0) + } + // Power-of-two subsampling during decode keeps large photos from + // allocating full-size ARGB buffers; exact fitting happens in + // bitmapToGrayPacket. + val opts = BitmapFactory.Options() + opts.inSampleSize = 1 + while (bounds.outWidth / (opts.inSampleSize * 2) >= maxWidth + && bounds.outHeight / (opts.inSampleSize * 2) >= maxHeight) { + opts.inSampleSize *= 2 + } + val bitmap = BitmapFactory.decodeFile(path, opts) + if (bitmap == null) { + return ByteArray(0) + } + return bitmapToGrayPacket(bitmap, maxWidth, maxHeight, gamma, dither) + } catch (e: Exception) { + Log.w(TAG, "image decode failed: $path", e) + return ByteArray(0) + } catch (e: OutOfMemoryError) { + Log.w(TAG, "image decode failed: $path", e) + return ByteArray(0) + } + } + + /** + * Decode an in-memory image file (PNG/JPEG/WebP bytes, e.g. a fetched + * map tile) and downscale to fit within maxWidth x maxHeight. Takes a + * ByteBuffer because NativeScript marshals a JS ArrayBuffer to one. + */ + @JvmStatic + fun loadGrayFromBytes(buffer: java.nio.ByteBuffer?, maxWidth: Int, maxHeight: Int): ByteArray { + return loadGrayFromBytes(buffer, maxWidth, maxHeight, 1f, false) + } + + /** As above, with photographic tone handling (see bitmapToGrayPacket). */ + @JvmStatic + fun loadGrayFromBytes( + buffer: java.nio.ByteBuffer?, maxWidth: Int, maxHeight: Int, gamma: Float, dither: Boolean): ByteArray { + if (buffer == null || maxWidth <= 0 || maxHeight <= 0) { + return ByteArray(0) + } + try { + val source = buffer.duplicate() + source.rewind() + val data = ByteArray(source.remaining()) + source.get(data) + if (data.isEmpty()) { + return ByteArray(0) + } + val bitmap = BitmapFactory.decodeByteArray(data, 0, data.size) + if (bitmap == null) { + return ByteArray(0) + } + return bitmapToGrayPacket(bitmap, maxWidth, maxHeight, gamma, dither) + } catch (e: Exception) { + Log.w(TAG, "byte-array image decode failed", e) + return ByteArray(0) + } catch (e: OutOfMemoryError) { + Log.w(TAG, "byte-array image decode failed", e) + return ByteArray(0) + } + } + + /** + * Scale a bitmap to fit within maxWidth x maxHeight (preserving aspect, + * never upscaling) and convert it to the grayscale packet format. + * Transparent pixels darken toward black (the on-glasses background). + */ + @JvmStatic + fun bitmapToGrayPacket(source: Bitmap?, maxWidth: Int, maxHeight: Int): ByteArray { + return bitmapToGrayPacket(source, maxWidth, maxHeight, 1f, false) + } + + /** + * As above, with photographic tone handling for continuous-tone images + * (album art, photos) rather than UI raster: + * + * gamma > 1 darkens midtones (out = 255 * (in/255)^gamma). Source pixels + * are sRGB-encoded but the G2 drives its 16 levels roughly linearly, so + * mid-gray otherwise displays far brighter than intended; 2.2 undoes the + * sRGB encoding outright. + * + * dither error-diffuses against the 16 levels the frame is eventually + * packed to (BmpUtil.GRAY_TO_NIBBLE) instead of leaving each pixel to + * round on its own, which turns smooth gradients into visible bands. + * Dithered output is already quantized: every byte is a level times 16, + * which BmpUtil re-quantizes back to exactly that level. + */ + @JvmStatic + fun bitmapToGrayPacket( + source: Bitmap?, maxWidth: Int, maxHeight: Int, gamma: Float, dither: Boolean): ByteArray { + if (source == null || source.width <= 0 || source.height <= 0 + || maxWidth <= 0 || maxHeight <= 0) { + return ByteArray(0) + } + try { + val scale = Math.min(1f, Math.min( + maxWidth.toFloat() / source.width, + maxHeight.toFloat() / source.height)) + val width = Math.max(1, Math.round(source.width * scale)) + val height = Math.max(1, Math.round(source.height * scale)) + var scaled = Bitmap.createScaledBitmap(source, width, height, true) + if (scaled.config == Bitmap.Config.HARDWARE) { + scaled = scaled.copy(Bitmap.Config.ARGB_8888, false) + } + val pixels = IntArray(width * height) + scaled.getPixels(pixels, 0, width, 0, 0, width, height) + val out = ByteArray(4 + width * height) + out[0] = (width and 0xff).toByte() + out[1] = ((width shr 8) and 0xff).toByte() + out[2] = (height and 0xff).toByte() + out[3] = ((height shr 8) and 0xff).toByte() + val tone = toneCurve(gamma) + for (i in pixels.indices) { + val p = pixels[i] + val a = (p ushr 24) and 0xff + val r = (p shr 16) and 0xff + val g = (p shr 8) and 0xff + val b = p and 0xff + out[4 + i] = tone[((r * 299 + g * 587 + b * 114) / 1000) * a / 255].toByte() + } + if (dither) { + ditherToDisplayLevels(out, 4, width, height) + } + return out + } catch (e: Exception) { + Log.w(TAG, "bitmap conversion failed", e) + return ByteArray(0) + } catch (e: OutOfMemoryError) { + Log.w(TAG, "bitmap conversion failed", e) + return ByteArray(0) + } + } + + /** 256-entry map of source gray to displayed gray for out = in^gamma. */ + @JvmStatic + private fun toneCurve(gamma: Float): IntArray { + val curve = IntArray(256) + for (v in 0 until 256) { + curve[v] = if (gamma == 1f) + v + else Math.round(255f * Math.pow((v / 255f).toDouble(), gamma.toDouble()).toFloat()) + } + return curve + } + + /** + * Floyd-Steinberg error diffusion onto the display's 16 gray levels, in + * place over an 8bpp plane. Serpentine scanning keeps the diffusion from + * building up a directional texture across wide flat areas. + * + * Level n is written back as n * 16 rather than the n * 17 it stands for + * so that BmpUtil's (v + 8) >> 4 reproduces n exactly; level 0 is written + * as 1, not 0, because 0 is the shell's color-key for transparent. + */ + @JvmStatic + private fun ditherToDisplayLevels(plane: ByteArray, offset: Int, width: Int, height: Int) { + var curr = FloatArray(width) + var next = FloatArray(width) + for (x in 0 until width) { + curr[x] = (plane[offset + x].toInt() and 0xff).toFloat() + } + for (y in 0 until height) { + val rowStart = offset + y * width + val nextStart = rowStart + width + val hasNext = y + 1 < height + for (x in 0 until width) { + next[x] = if (hasNext) (plane[nextStart + x].toInt() and 0xff).toFloat() else 0f + } + val leftToRight = (y and 1) == 0 + val start = if (leftToRight) 0 else width - 1 + val step = if (leftToRight) 1 else -1 + for (i in 0 until width) { + val x = start + i * step + val wanted = curr[x] + val level = Math.round(Math.min(255f, Math.max(0f, wanted)) / 17f) + val error = wanted - level * 17f + plane[rowStart + x] = (if (level == 0) 1 else level * 16).toByte() + val ahead = x + step + if (ahead >= 0 && ahead < width) { + curr[ahead] += error * (7f / 16f) + } + if (hasNext) { + if (ahead >= 0 && ahead < width) { + next[ahead] += error * (1f / 16f) + } + next[x] += error * (5f / 16f) + val behind = x - step + if (behind >= 0 && behind < width) { + next[behind] += error * (3f / 16f) + } + } + } + val swap = curr + curr = next + next = swap + } + } + } +} diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/PreviewBitmapUtil.java b/App_Resources/Android/src/main/java/com/faceclaw/app/PreviewBitmapUtil.java deleted file mode 100644 index 2a37ad7d..00000000 --- a/App_Resources/Android/src/main/java/com/faceclaw/app/PreviewBitmapUtil.java +++ /dev/null @@ -1,56 +0,0 @@ -package com.faceclaw.app; - -import android.graphics.Bitmap; - -import java.nio.ByteBuffer; - -/** - * Builds the phone-UI preview bitmap from an 8bpp grayscale frame. The - * grayscale buffer arrives as a ByteBuffer (NativeScript marshals a JS - * ArrayBuffer to one without copying element-by-element); doing the - * gray-to-ARGB expansion here keeps the 165KB-per-frame loop out of the - * JS/Java bridge, where it used to cost ~150ms per preview. - */ -public final class PreviewBitmapUtil { - private static int[] cachedLut; - private static long cachedGammaBits; - private static boolean cachedGreen; - - private PreviewBitmapUtil() {} - - /** Reuse the current preview palette; rebuild only when gamma or color changes. */ - private static synchronized int[] lookupTable(double brightenGamma, boolean green) { - long gammaBits = Double.doubleToLongBits(brightenGamma); - if (cachedLut != null && cachedGammaBits == gammaBits && cachedGreen == green) { - return cachedLut; - } - int[] lut = new int[256]; - for (int g = 0; g < 256; g++) { - int v = (int) Math.max(0, Math.min(255, Math.round(255 * Math.pow(g / 255.0, brightenGamma)))); - lut[g] = green ? (0xff000000 | (v << 8)) : (0xff000000 | (v << 16) | (v << 8) | v); - } - cachedGammaBits = gammaBits; - cachedGreen = green; - cachedLut = lut; - return lut; - } - - public static Bitmap fromGray(ByteBuffer gray, int width, int height, double brightenGamma) { - return fromGray(gray, width, height, brightenGamma, false); - } - - /** As above; `green` renders green-on-black (matching the physical glasses) instead of grayscale. */ - public static Bitmap fromGray(ByteBuffer gray, int width, int height, double brightenGamma, boolean green) { - if (gray == null || width <= 0 || height <= 0 || gray.remaining() < width * height) { - throw new IllegalArgumentException("invalid gray preview buffer"); - } - int[] lut = lookupTable(brightenGamma, green); - byte[] bytes = new byte[width * height]; - gray.get(bytes); - int[] colors = new int[width * height]; - for (int i = 0; i < colors.length; i++) { - colors[i] = lut[bytes[i] & 0xff]; - } - return Bitmap.createBitmap(colors, width, height, Bitmap.Config.ARGB_8888); - } -} diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/PreviewBitmapUtil.kt b/App_Resources/Android/src/main/java/com/faceclaw/app/PreviewBitmapUtil.kt new file mode 100644 index 00000000..96f98bda --- /dev/null +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/PreviewBitmapUtil.kt @@ -0,0 +1,61 @@ +package com.faceclaw.app + +import android.graphics.Bitmap + +import java.nio.ByteBuffer + +/** + * Builds the phone-UI preview bitmap from an 8bpp grayscale frame. The + * grayscale buffer arrives as a ByteBuffer (NativeScript marshals a JS + * ArrayBuffer to one without copying element-by-element); doing the + * gray-to-ARGB expansion here keeps the 165KB-per-frame loop out of the + * JS/Java bridge, where it used to cost ~150ms per preview. + */ +class PreviewBitmapUtil private constructor() { + companion object { + private var cachedLut: IntArray? = null + private var cachedGammaBits: Long = 0 + private var cachedGreen: Boolean = false + + /** Reuse the current preview palette; rebuild only when gamma or color changes. */ + @JvmStatic + @Synchronized + private fun lookupTable(brightenGamma: Double, green: Boolean): IntArray { + val gammaBits = java.lang.Double.doubleToLongBits(brightenGamma) + val cached = cachedLut + if (cached != null && cachedGammaBits == gammaBits && cachedGreen == green) { + return cached + } + val lut = IntArray(256) + for (g in 0 until 256) { + val v = Math.max(0L, Math.min(255L, Math.round(255 * Math.pow(g / 255.0, brightenGamma)))).toInt() + lut[g] = if (green) (0xff000000.toInt() or (v shl 8)) else (0xff000000.toInt() or (v shl 16) or (v shl 8) or v) + } + cachedGammaBits = gammaBits + cachedGreen = green + cachedLut = lut + return lut + } + + @JvmStatic + fun fromGray(gray: ByteBuffer?, width: Int, height: Int, brightenGamma: Double): Bitmap { + return fromGray(gray, width, height, brightenGamma, false) + } + + /** As above; `green` renders green-on-black (matching the physical glasses) instead of grayscale. */ + @JvmStatic + fun fromGray(gray: ByteBuffer?, width: Int, height: Int, brightenGamma: Double, green: Boolean): Bitmap { + if (gray == null || width <= 0 || height <= 0 || gray.remaining() < width * height) { + throw IllegalArgumentException("invalid gray preview buffer") + } + val lut = lookupTable(brightenGamma, green) + val bytes = ByteArray(width * height) + gray.get(bytes) + val colors = IntArray(width * height) + for (i in colors.indices) { + colors[i] = lut[bytes[i].toInt() and 0xff] + } + return Bitmap.createBitmap(colors, width, height, Bitmap.Config.ARGB_8888) + } + } +} diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/ScreenshotUtil.java b/App_Resources/Android/src/main/java/com/faceclaw/app/ScreenshotUtil.java deleted file mode 100644 index 313abece..00000000 --- a/App_Resources/Android/src/main/java/com/faceclaw/app/ScreenshotUtil.java +++ /dev/null @@ -1,45 +0,0 @@ -package com.faceclaw.app; - -import android.content.Context; - -import java.io.File; -import java.io.FileOutputStream; -import java.io.IOException; -import java.text.SimpleDateFormat; -import java.util.Date; -import java.util.Locale; - -/** - * Saves screenshots of the composited screen as 4-bit grayscale PNGs (the - * display's native depth). Retrieve with - * adb pull /sdcard/Android/data/com.faceclaw.app/files/screenshots/ - */ -public final class ScreenshotUtil { - private ScreenshotUtil() {} - - /** Returns the absolute path of the written file. */ - public static String savePngScreenshot(Context context, byte[] gray, int width, int height) throws IOException { - if (gray == null || width <= 0 || height <= 0 || gray.length < width * height) { - throw new IllegalArgumentException("invalid screenshot buffer"); - } - byte[] png = SharedScreenshots.encode4BitGrayPng(gray, width, height); - File file = new File(ensureScreenshotsDir(context), "screen-" + timestamp() + ".png"); - try (FileOutputStream out = new FileOutputStream(file)) { - out.write(png); - } - return file.getAbsolutePath(); - } - - static File ensureScreenshotsDir(Context context) throws IOException { - File dir = new File(context.getExternalFilesDir(null), "screenshots"); - if (!dir.exists() && !dir.mkdirs()) { - throw new IOException("failed to create " + dir); - } - return dir; - } - - static String timestamp() { - return new SimpleDateFormat("yyyyMMdd-HHmmss-SSS", Locale.US).format(new Date()); - } - -} diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/ScreenshotUtil.kt b/App_Resources/Android/src/main/java/com/faceclaw/app/ScreenshotUtil.kt new file mode 100644 index 00000000..c0d33c9b --- /dev/null +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/ScreenshotUtil.kt @@ -0,0 +1,49 @@ +package com.faceclaw.app + +import android.content.Context + +import java.io.File +import java.io.FileOutputStream +import java.io.IOException +import java.text.SimpleDateFormat +import java.util.Date +import java.util.Locale + +/** + * Saves screenshots of the composited screen as 4-bit grayscale PNGs (the + * display's native depth). Retrieve with + * adb pull /sdcard/Android/data/com.faceclaw.app/files/screenshots/ + */ +class ScreenshotUtil private constructor() { + companion object { + /** Returns the absolute path of the written file. */ + @JvmStatic + @Throws(IOException::class) + fun savePngScreenshot(context: Context, gray: ByteArray?, width: Int, height: Int): String { + if (gray == null || width <= 0 || height <= 0 || gray.size < width * height) { + throw IllegalArgumentException("invalid screenshot buffer") + } + val png = SharedScreenshots.encode4BitGrayPng(gray, width, height) + val file = File(ensureScreenshotsDir(context), "screen-" + timestamp() + ".png") + FileOutputStream(file).use { out -> + out.write(png) + } + return file.absolutePath + } + + @JvmStatic + @Throws(IOException::class) + fun ensureScreenshotsDir(context: Context): File { + val dir = File(context.getExternalFilesDir(null), "screenshots") + if (!dir.exists() && !dir.mkdirs()) { + throw IOException("failed to create $dir") + } + return dir + } + + @JvmStatic + fun timestamp(): String { + return SimpleDateFormat("yyyyMMdd-HHmmss-SSS", Locale.US).format(Date()) + } + } +} diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/util/InterruptibleSleep.java b/App_Resources/Android/src/main/java/com/faceclaw/app/util/InterruptibleSleep.java deleted file mode 100644 index a77454a6..00000000 --- a/App_Resources/Android/src/main/java/com/faceclaw/app/util/InterruptibleSleep.java +++ /dev/null @@ -1,48 +0,0 @@ -package com.faceclaw.app; - -import android.os.SystemClock; - -final class InterruptibleSleep { - private final Object lock = new Object(); - private boolean interrupted; - - boolean sleep(long ms) { - long delayMs = Math.max(1, ms); - long deadline = SystemClock.elapsedRealtime() + delayMs; - synchronized (lock) { - // Consume (do not clear-and-ignore) an interrupt delivered before this - // sleep began. interrupt() runs on other threads with no lock held by - // the sleeper between driveSession() returning and sleep() starting, so - // clearing the flag at entry lost that wake and forced the worker to - // sleep the full interval — up to 250ms per idle frame. Honoring it - // costs at most one extra (cheap) driveSession pass. - if (interrupted) { - interrupted = false; - return false; - } - while (true) { - long remaining = deadline - SystemClock.elapsedRealtime(); - if (remaining <= 0) { - return true; - } - try { - lock.wait(remaining); - } catch (InterruptedException ignored) { - interrupted = false; - return false; - } - if (interrupted) { - interrupted = false; - return false; - } - } - } - } - - void interrupt() { - synchronized (lock) { - interrupted = true; - lock.notifyAll(); - } - } -} diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/util/InterruptibleSleep.kt b/App_Resources/Android/src/main/java/com/faceclaw/app/util/InterruptibleSleep.kt new file mode 100644 index 00000000..60153482 --- /dev/null +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/util/InterruptibleSleep.kt @@ -0,0 +1,48 @@ +package com.faceclaw.app + +import android.os.SystemClock + +class InterruptibleSleep { + private val lock = java.lang.Object() + private var interrupted = false + + fun sleep(ms: Long): Boolean { + val delayMs = Math.max(1L, ms) + val deadline = SystemClock.elapsedRealtime() + delayMs + synchronized(lock) { + // Consume (do not clear-and-ignore) an interrupt delivered before this + // sleep began. interrupt() runs on other threads with no lock held by + // the sleeper between driveSession() returning and sleep() starting, so + // clearing the flag at entry lost that wake and forced the worker to + // sleep the full interval — up to 250ms per idle frame. Honoring it + // costs at most one extra (cheap) driveSession pass. + if (interrupted) { + interrupted = false + return false + } + while (true) { + val remaining = deadline - SystemClock.elapsedRealtime() + if (remaining <= 0) { + return true + } + try { + lock.wait(remaining) + } catch (ignored: InterruptedException) { + interrupted = false + return false + } + if (interrupted) { + interrupted = false + return false + } + } + } + } + + fun interrupt() { + synchronized(lock) { + interrupted = true + lock.notifyAll() + } + } +} diff --git a/CLAUDE.md b/CLAUDE.md index 3ef2a977..7dd039ef 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -1,13 +1,16 @@ This is Faceclaw, an Android program that provides user interface on the Even Realities G2 smart glasses. It is written in a mix of Typescript/NativeScript -(for the user interface parts) and Java (for the low-level bluetooth parts and +(for the user interface parts) and Kotlin (for the low-level bluetooth parts and for interfacing with Android SDK). -Typescript parts are in app/. Java parts are in App_Resources/Android/src/main/java/com/faceclaw/app/. +Typescript parts are in app/. Android-specific Kotlin parts are in +App_Resources/Android/src/main/java/com/faceclaw/app/ (compiled by the NativeScript +Android build; only the vendored com.k2fsa.sherpa.onnx JNI classes there remain Java). +Shared Kotlin Multiplatform code is in native/kotlin/shared/ (see native/kotlin/README.md). wear/ is a separate Gradle project (Kotlin + Compose for Wear OS): the watch remote that drives Faceclaw over the Wearable Data Layer. Its phone-side -counterparts are FaceclawWearBridge.java, app/native/wear-bridge.ts and +counterparts are FaceclawWearBridge.kt, app/native/wear-bridge.ts and app/g2/wear-remote.ts; the message format is in wear/PROTOCOL.md. `build.sh` and `build_wear.sh` build the Android and Wear OS apps respectively. diff --git a/app/apps/timer/timer-engine.ts b/app/apps/timer/timer-engine.ts index 57bb5e88..34dbf3d7 100644 --- a/app/apps/timer/timer-engine.ts +++ b/app/apps/timer/timer-engine.ts @@ -8,7 +8,7 @@ * * Two expiry paths run side by side: a JS deadline for prompt on-glasses * ringing while the process is awake, and an AlarmManager alarm-clock alarm - * (FaceclawAlarms.java) that rings on the phone at the right moment even if + * (FaceclawAlarms.kt) that rings on the phone at the right moment even if * the process is asleep or dead. The phone shows the item silently first * and adds its own sound when the glasses cannot carry it (not connected, * not worn, charging, not shown within seconds, or not acknowledged within diff --git a/app/g2/dashboard-controller.ts b/app/g2/dashboard-controller.ts index a1d0bde9..ba201e83 100644 --- a/app/g2/dashboard-controller.ts +++ b/app/g2/dashboard-controller.ts @@ -1326,6 +1326,12 @@ class DashboardController { ring: ringAddress, }); this.communicator = communicator; + // Size the compositor before anything else is awaited: windows opened + // during the bridge start-ups below (the terminal hub, restored apps) + // configure their surfaces through this.communicator, and the Java call + // queue is FIFO, so enqueuing the screen size first guarantees it lands + // ahead of them. Mirrors ensurePreviewDisplay. + await communicator.configureCompositorScreen(G2_LENS_WIDTH, G2_LENS_HEIGHT); this.offState = communicator.onStateChange((state) => { if (state.phase !== "connected") resetRingInputFilter(); if (state.phase === "unpaired") { @@ -1493,7 +1499,6 @@ class DashboardController { await nightscoutBridge.start(); // Register the compositor surfaces: the shell chrome above all windows, // and a surface per live window (only the foreground one is composited). - await communicator.configureCompositorScreen(G2_LENS_WIDTH, G2_LENS_HEIGHT); await communicator.configureSurface(SHELL_SURFACE_ID, { x: 0, y: 0, diff --git a/app/native/asr-model.ts b/app/native/asr-model.ts index 591bdc73..97fc7e3b 100644 --- a/app/native/asr-model.ts +++ b/app/native/asr-model.ts @@ -18,7 +18,7 @@ declare const java: any; * - "whisper-base-en": sherpa-onnx's offline Whisper backend, base.en, * int8-quantized. Picked over tiny.en for materially better accuracy * (Whisper's own tiny/base WER gap is real and well documented) on the - * reasoning that FaceclawVoiceController.java no longer re-decodes a + * reasoning that FaceclawVoiceController.kt no longer re-decodes a * Whisper segment on every live-partial tick — see its ASR_WHISPER_* * comment — so base.en's extra compute is a one-time cost per utterance, * not a cost paid ~1.4x/second while the user is speaking. tiny.en is an diff --git a/app/native/frame-timings.ts b/app/native/frame-timings.ts index 56e2a5e1..294349e8 100644 --- a/app/native/frame-timings.ts +++ b/app/native/frame-timings.ts @@ -1,6 +1,6 @@ /** * TS-side wrapper for the Java FrameTimings singleton (per-frame latency - * instrumentation; see FrameTimings.java). Frame ID 0 means "no frame" and is + * instrumentation; see FrameTimings.kt). Frame ID 0 means "no frame" and is * ignored by every method, so callers never need to null-check. * * Frames form a tree: pass the causing frame's ID as `parentFrameId` when one diff --git a/native/kotlin/README.md b/native/kotlin/README.md index cdc76456..e2939a80 100644 --- a/native/kotlin/README.md +++ b/native/kotlin/README.md @@ -34,12 +34,17 @@ explicitly; iOS also cleans up abandoned native allocations. GATT write modes are mapped by platform adapters. `migrated-java-sources.json` lists each removed Java source and its Kotlin -replacement. Android's build removes precisely those stale generated copies -before compilation. `android-java-boundaries.json` inventories every retained -Java file and its Android API boundary: services/receivers, GATT, media, -WebView, SQLite, rendering, networking and JNI. The sherpa-onnx Java classes -remain the vendored Android JNI ABI. PNG/GIF adapters retain only Android -storage responsibilities; their codecs are shared. +replacement (paths relative to the repository root). Android's build removes +precisely those stale generated copies before compilation. The Android-only +classes (services/receivers, GATT, media, WebView, SQLite, rendering, +networking, JNI engines) are Kotlin in +`App_Resources/Android/src/main/java/com/faceclaw/app/`, compiled by +NativeScript's own `kotlin-android` plugin rather than this KMP module; they keep +their Java-era JVM API (`@JvmStatic`/`@JvmField`/`const val`) so TypeScript and +the manifest are unchanged. `android-java-boundaries.json` inventories the Java +that remains: the sherpa-onnx classes, which are the vendored Android JNI ABI. +PNG/GIF adapters retain only Android storage responsibilities; their codecs are +shared. On `ios-port`, platform adapters route the existing TypeScript session and UI logic through shared Kotlin for protocol builders, CRC/framing/reassembly, diff --git a/native/kotlin/android-java-boundaries.json b/native/kotlin/android-java-boundaries.json index cfceae10..5e5f418e 100644 --- a/native/kotlin/android-java-boundaries.json +++ b/native/kotlin/android-java-boundaries.json @@ -1,118 +1,4 @@ [ - { - "reason": "BluetoothGatt, BluetoothLeScanner, Android callback threads", - "files": [ - "com/faceclaw/app/FaceclawBleCommunicator.java", - "com/faceclaw/app/FaceclawBleManager.java", - "com/faceclaw/app/FaceclawDeviceDiscovery.java", - "com/faceclaw/app/FaceclawDeviceInfoProbe.java", - "com/faceclaw/app/FaceclawFirmwareFlasher.java", - "com/faceclaw/app/FaceclawFlashPromptCommunicator.java" - ] - }, - { - "reason": "AlarmManager, PendingIntent, Android activities/services/receivers", - "files": [ - "com/faceclaw/app/FaceclawAlarmActivity.java", - "com/faceclaw/app/FaceclawAlarmReceiver.java", - "com/faceclaw/app/FaceclawAlarmRescheduleReceiver.java", - "com/faceclaw/app/FaceclawAlarmService.java", - "com/faceclaw/app/FaceclawAlarms.java", - "com/faceclaw/app/FaceclawForegroundService.java" - ] - }, - { - "reason": "Android providers, sensors, permissions and preferences", - "files": [ - "com/faceclaw/app/FaceclawCalendarProvider.java", - "com/faceclaw/app/FaceclawLocationProvider.java", - "com/faceclaw/app/FaceclawLocationTracker.java", - "com/faceclaw/app/FaceclawSettings.java", - "com/faceclaw/app/FaceclawSettingsPortReceiver.java", - "com/faceclaw/app/FaceclawResourceUsage.java" - ] - }, - { - "reason": "Android MediaSession, notifications, status icons and wearable services", - "files": [ - "com/faceclaw/app/FaceclawMediaBrowser.java", - "com/faceclaw/app/FaceclawMediaController.java", - "com/faceclaw/app/FaceclawMediaNotificationListenerService.java", - "com/faceclaw/app/FaceclawSystemStatusIconProvider.java", - "com/faceclaw/app/FaceclawWearBridge.java", - "com/faceclaw/app/FaceclawWearListenerService.java" - ] - }, - { - "reason": "Android WebView/JavascriptInterface, package manager and intent integration", - "files": [ - "com/faceclaw/app/FaceclawEvenAppDetector.java", - "com/faceclaw/app/FaceclawEvenHubDocumentStart.java", - "com/faceclaw/app/FaceclawEvenHubJsBridge.java", - "com/faceclaw/app/FaceclawEvenHubWebView.java", - "com/faceclaw/app/FaceclawEvenHubWebViewClient.java", - "com/faceclaw/app/FaceclawEvenHubWebViewHost.java" - ] - }, - { - "reason": "Android AudioRecord/AudioTrack/MediaCodec, Handler delivery and JNI speech/model engines", - "files": [ - "com/faceclaw/app/AndroidNoiseEffects.java", - "com/faceclaw/app/FaceclawAudioTranscoder.java", - "com/faceclaw/app/FaceclawCaptionEngine.java", - "com/faceclaw/app/FaceclawDiarizer.java", - "com/faceclaw/app/FaceclawLc3Decoder.java", - "com/faceclaw/app/FaceclawLlamaRunner.java", - "com/faceclaw/app/FaceclawSpeakerId.java", - "com/faceclaw/app/FaceclawVoiceController.java" - ] - }, - { - "reason": "Android SQLite persistence and cursor/content APIs", - "files": [ - "com/faceclaw/app/FaceclawConversationStore.java" - ] - }, - { - "reason": "Android OkHttp backend, Handler/Log delivery, download lifecycle", - "files": [ - "com/faceclaw/app/FaceclawHttp.java", - "com/faceclaw/app/FaceclawModelDownloader.java", - "com/faceclaw/app/FaceclawSseRequest.java", - "com/faceclaw/app/FaceclawWebSocket.java" - ] - }, - { - "reason": "Android JVM loopback sockets, SecureRandom and SHA-256; iOS uses its native socket and Security implementations. Protocol and permissions stay in shared TypeScript.", - "files": [ - "com/faceclaw/app/FaceclawRemoteInput.java" - ] - }, - { - "reason": "Android Bitmap/Canvas/Paint/typeface, SVG rasterization and camera scanning", - "files": [ - "com/faceclaw/app/FontFileRenderer.java", - "com/faceclaw/app/IconRenderer.java", - "com/faceclaw/app/ImageFileLoader.java", - "com/faceclaw/app/PreviewBitmapUtil.java", - "com/faceclaw/app/FaceclawQrScanner.java" - ] - }, - { - "reason": "Android preview/UI delivery and app-scoped export paths; codecs/composition are shared", - "files": [ - "com/faceclaw/app/FaceclawPreviewCompositor.java", - "com/faceclaw/app/GifScreenRecorder.java", - "com/faceclaw/app/ScreenshotUtil.java" - ] - }, - { - "reason": "Android elapsedRealtime, worker wait/notify and Context-backed trace export", - "files": [ - "com/faceclaw/app/FrameTimings.java", - "com/faceclaw/app/util/InterruptibleSleep.java" - ] - }, { "reason": "Vendored sherpa-onnx JNI ABI: Java classes/fields are consumed by the Android native library; iOS uses its native engine APIs.", "files": [ diff --git a/native/kotlin/migrated-java-sources.json b/native/kotlin/migrated-java-sources.json index 9d676c85..dd32ffa3 100644 --- a/native/kotlin/migrated-java-sources.json +++ b/native/kotlin/migrated-java-sources.json @@ -1,198 +1,414 @@ [ { "java": "com/faceclaw/app/LvglFontFile.java", - "kotlin": "shared/src/commonMain/kotlin/com/faceclaw/app/graphics/LvglFontFile.kt" + "kotlin": "native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/graphics/LvglFontFile.kt" }, { "java": "com/faceclaw/app/FaceclawAlarmListener.java", - "kotlin": "shared/src/commonMain/kotlin/com/faceclaw/app/callbacks/FaceclawAlarmListener.kt" + "kotlin": "native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/callbacks/FaceclawAlarmListener.kt" }, { "java": "com/faceclaw/app/FaceclawAmbientLightListener.java", - "kotlin": "shared/src/commonMain/kotlin/com/faceclaw/app/callbacks/FaceclawAmbientLightListener.kt" + "kotlin": "native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/callbacks/FaceclawAmbientLightListener.kt" }, { "java": "com/faceclaw/app/FaceclawAudioPacketListener.java", - "kotlin": "shared/src/commonMain/kotlin/com/faceclaw/app/callbacks/FaceclawAudioPacketListener.kt" + "kotlin": "native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/callbacks/FaceclawAudioPacketListener.kt" }, { "java": "com/faceclaw/app/FaceclawBleCommunicatorListener.java", - "kotlin": "shared/src/commonMain/kotlin/com/faceclaw/app/callbacks/FaceclawBleCommunicatorListener.kt" + "kotlin": "native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/callbacks/FaceclawBleCommunicatorListener.kt" }, { "java": "com/faceclaw/app/FaceclawBleListener.java", - "kotlin": "shared/src/commonMain/kotlin/com/faceclaw/app/callbacks/FaceclawBleListener.kt" + "kotlin": "native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/callbacks/FaceclawBleListener.kt" }, { "java": "com/faceclaw/app/FaceclawCaptionEngineListener.java", - "kotlin": "shared/src/commonMain/kotlin/com/faceclaw/app/callbacks/FaceclawCaptionEngineListener.kt" + "kotlin": "native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/callbacks/FaceclawCaptionEngineListener.kt" }, { "java": "com/faceclaw/app/FaceclawCompassListener.java", - "kotlin": "shared/src/commonMain/kotlin/com/faceclaw/app/callbacks/FaceclawCompassListener.kt" + "kotlin": "native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/callbacks/FaceclawCompassListener.kt" }, { "java": "com/faceclaw/app/FaceclawDeviceDiscoveryListener.java", - "kotlin": "shared/src/commonMain/kotlin/com/faceclaw/app/callbacks/FaceclawDeviceDiscoveryListener.kt" + "kotlin": "native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/callbacks/FaceclawDeviceDiscoveryListener.kt" }, { "java": "com/faceclaw/app/FaceclawDeviceInfoProbeListener.java", - "kotlin": "shared/src/commonMain/kotlin/com/faceclaw/app/callbacks/FaceclawDeviceInfoProbeListener.kt" + "kotlin": "native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/callbacks/FaceclawDeviceInfoProbeListener.kt" }, { "java": "com/faceclaw/app/FaceclawEvenHubListener.java", - "kotlin": "shared/src/commonMain/kotlin/com/faceclaw/app/callbacks/FaceclawEvenHubListener.kt" + "kotlin": "native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/callbacks/FaceclawEvenHubListener.kt" }, { "java": "com/faceclaw/app/FaceclawFirmwareFlasherListener.java", - "kotlin": "shared/src/commonMain/kotlin/com/faceclaw/app/callbacks/FaceclawFirmwareFlasherListener.kt" + "kotlin": "native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/callbacks/FaceclawFirmwareFlasherListener.kt" }, { "java": "com/faceclaw/app/FaceclawFlashPromptListener.java", - "kotlin": "shared/src/commonMain/kotlin/com/faceclaw/app/callbacks/FaceclawFlashPromptListener.kt" + "kotlin": "native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/callbacks/FaceclawFlashPromptListener.kt" }, { "java": "com/faceclaw/app/FaceclawImuListener.java", - "kotlin": "shared/src/commonMain/kotlin/com/faceclaw/app/callbacks/FaceclawImuListener.kt" + "kotlin": "native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/callbacks/FaceclawImuListener.kt" }, { "java": "com/faceclaw/app/FaceclawLlamaListener.java", - "kotlin": "shared/src/commonMain/kotlin/com/faceclaw/app/callbacks/FaceclawLlamaListener.kt" + "kotlin": "native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/callbacks/FaceclawLlamaListener.kt" }, { "java": "com/faceclaw/app/FaceclawLocationListener.java", - "kotlin": "shared/src/commonMain/kotlin/com/faceclaw/app/callbacks/FaceclawLocationListener.kt" + "kotlin": "native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/callbacks/FaceclawLocationListener.kt" }, { "java": "com/faceclaw/app/FaceclawLocationTrackerListener.java", - "kotlin": "shared/src/commonMain/kotlin/com/faceclaw/app/callbacks/FaceclawLocationTrackerListener.kt" + "kotlin": "native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/callbacks/FaceclawLocationTrackerListener.kt" }, { "java": "com/faceclaw/app/FaceclawMediaBrowserListener.java", - "kotlin": "shared/src/commonMain/kotlin/com/faceclaw/app/callbacks/FaceclawMediaBrowserListener.kt" + "kotlin": "native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/callbacks/FaceclawMediaBrowserListener.kt" }, { "java": "com/faceclaw/app/FaceclawMediaControllerListener.java", - "kotlin": "shared/src/commonMain/kotlin/com/faceclaw/app/callbacks/FaceclawMediaControllerListener.kt" + "kotlin": "native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/callbacks/FaceclawMediaControllerListener.kt" }, { "java": "com/faceclaw/app/FaceclawMicStatusListener.java", - "kotlin": "shared/src/commonMain/kotlin/com/faceclaw/app/callbacks/FaceclawMicStatusListener.kt" + "kotlin": "native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/callbacks/FaceclawMicStatusListener.kt" }, { "java": "com/faceclaw/app/FaceclawModelDownloaderListener.java", - "kotlin": "shared/src/commonMain/kotlin/com/faceclaw/app/callbacks/FaceclawModelDownloaderListener.kt" + "kotlin": "native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/callbacks/FaceclawModelDownloaderListener.kt" }, { "java": "com/faceclaw/app/FaceclawNoiseSuppressor.java", - "kotlin": "shared/src/commonMain/kotlin/com/faceclaw/app/audio/FaceclawNoiseSuppressor.kt" + "kotlin": "native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/audio/FaceclawNoiseSuppressor.kt" }, { "java": "com/faceclaw/app/FaceclawNotificationListener.java", - "kotlin": "shared/src/commonMain/kotlin/com/faceclaw/app/callbacks/FaceclawNotificationListener.kt" + "kotlin": "native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/callbacks/FaceclawNotificationListener.kt" }, { "java": "com/faceclaw/app/FaceclawQrScannerListener.java", - "kotlin": "shared/src/commonMain/kotlin/com/faceclaw/app/callbacks/FaceclawQrScannerListener.kt" + "kotlin": "native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/callbacks/FaceclawQrScannerListener.kt" }, { "java": "com/faceclaw/app/FaceclawRingEventDecoder.java", - "kotlin": "shared/src/commonMain/kotlin/com/faceclaw/app/g2protocol/FaceclawRingEventDecoder.kt" + "kotlin": "native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/g2protocol/FaceclawRingEventDecoder.kt" }, { "java": "com/faceclaw/app/FaceclawSettingsListener.java", - "kotlin": "shared/src/commonMain/kotlin/com/faceclaw/app/callbacks/FaceclawSettingsListener.kt" + "kotlin": "native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/callbacks/FaceclawSettingsListener.kt" }, { "java": "com/faceclaw/app/FaceclawSseListener.java", - "kotlin": "shared/src/commonMain/kotlin/com/faceclaw/app/callbacks/FaceclawSseListener.kt" + "kotlin": "native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/callbacks/FaceclawSseListener.kt" }, { "java": "com/faceclaw/app/FaceclawVoiceControllerListener.java", - "kotlin": "shared/src/commonMain/kotlin/com/faceclaw/app/callbacks/FaceclawVoiceControllerListener.kt" + "kotlin": "native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/callbacks/FaceclawVoiceControllerListener.kt" }, { "java": "com/faceclaw/app/FaceclawWearListener.java", - "kotlin": "shared/src/commonMain/kotlin/com/faceclaw/app/callbacks/FaceclawWearListener.kt" + "kotlin": "native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/callbacks/FaceclawWearListener.kt" }, { "java": "com/faceclaw/app/FaceclawWebSocketListener.java", - "kotlin": "shared/src/commonMain/kotlin/com/faceclaw/app/callbacks/FaceclawWebSocketListener.kt" + "kotlin": "native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/callbacks/FaceclawWebSocketListener.kt" }, { "java": "com/faceclaw/app/FwGlyphAtlas.java", - "kotlin": "shared/src/commonMain/kotlin/com/faceclaw/app/graphics/FwGlyphAtlas.kt" + "kotlin": "native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/graphics/FwGlyphAtlas.kt" }, { "java": "com/faceclaw/app/g2protocol/BleImageOptimizer.java", - "kotlin": "shared/src/commonMain/kotlin/com/faceclaw/app/g2protocol/BleImageOptimizer.kt" + "kotlin": "native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/g2protocol/BleImageOptimizer.kt" }, { "java": "com/faceclaw/app/g2protocol/BleMagicPool.java", - "kotlin": "shared/src/commonMain/kotlin/com/faceclaw/app/g2protocol/BleMagicPool.kt" + "kotlin": "native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/g2protocol/BleMagicPool.kt" }, { "java": "com/faceclaw/app/g2protocol/BleProtocol.java", - "kotlin": "shared/src/commonMain/kotlin/com/faceclaw/app/g2protocol/BleProtocol.kt" + "kotlin": "native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/g2protocol/BleProtocol.kt" }, { "java": "com/faceclaw/app/g2protocol/CfwMessageWindow.java", - "kotlin": "shared/src/commonMain/kotlin/com/faceclaw/app/g2protocol/CfwMessageWindow.kt" + "kotlin": "native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/g2protocol/CfwMessageWindow.kt" }, { "java": "com/faceclaw/app/g2protocol/CfwTransport.java", - "kotlin": "shared/src/commonMain/kotlin/com/faceclaw/app/g2protocol/CfwTransport.kt" + "kotlin": "native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/g2protocol/CfwTransport.kt" }, { "java": "com/faceclaw/app/g2protocol/ConnectionOptions.java", - "kotlin": "shared/src/commonMain/kotlin/com/faceclaw/app/g2protocol/ConnectionOptions.kt" + "kotlin": "native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/g2protocol/ConnectionOptions.kt" }, { "java": "com/faceclaw/app/g2protocol/G2Event.java", - "kotlin": "shared/src/commonMain/kotlin/com/faceclaw/app/g2protocol/G2Event.kt" + "kotlin": "native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/g2protocol/G2Event.kt" }, { "java": "com/faceclaw/app/g2protocol/MessageBuilder.java", - "kotlin": "shared/src/commonMain/kotlin/com/faceclaw/app/g2protocol/MessageBuilder.kt" + "kotlin": "native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/g2protocol/MessageBuilder.kt" }, { "java": "com/faceclaw/app/g2protocol/OutboundMessage.java", - "kotlin": "shared/src/commonMain/kotlin/com/faceclaw/app/g2protocol/OutboundMessage.kt" + "kotlin": "native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/g2protocol/OutboundMessage.kt" }, { "java": "com/faceclaw/app/g2protocol/TextureCacheState.java", - "kotlin": "shared/src/commonMain/kotlin/com/faceclaw/app/g2protocol/ResourceCacheState.kt" + "kotlin": "native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/g2protocol/ResourceCacheState.kt" }, { "java": "com/faceclaw/app/g2protocol/TexturePlanner.java", - "kotlin": "shared/src/commonMain/kotlin/com/faceclaw/app/g2protocol/TexturePlanner.kt" + "kotlin": "native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/g2protocol/TexturePlanner.kt" }, { "java": "com/faceclaw/app/GlyphAtlas.java", - "kotlin": "shared/src/commonMain/kotlin/com/faceclaw/app/graphics/GlyphAtlas.kt" + "kotlin": "native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/graphics/GlyphAtlas.kt" }, { "java": "com/faceclaw/app/ImageAtlas.java", - "kotlin": "shared/src/commonMain/kotlin/com/faceclaw/app/graphics/ImageAtlas.kt" + "kotlin": "native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/graphics/ImageAtlas.kt" }, { "java": "com/faceclaw/app/SurfaceCompositor.java", - "kotlin": "shared/src/commonMain/kotlin/com/faceclaw/app/graphics/SurfaceCompositor.kt" + "kotlin": "native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/graphics/SurfaceCompositor.kt" }, { "java": "com/faceclaw/app/util/BmpUtil.java", - "kotlin": "shared/src/commonMain/kotlin/com/faceclaw/app/util/BmpUtil.kt" + "kotlin": "native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/util/BmpUtil.kt" }, { "java": "com/faceclaw/app/util/CollectionUtils.java", - "kotlin": "shared/src/commonMain/kotlin/com/faceclaw/app/util/CollectionUtils.kt" + "kotlin": "native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/util/CollectionUtils.kt" }, { "java": "com/faceclaw/app/FaceclawFirmwareUtil.java", - "kotlin": "shared/src/androidMain/kotlin/com/faceclaw/app/FaceclawFirmwareUtil.kt" + "kotlin": "native/kotlin/shared/src/androidMain/kotlin/com/faceclaw/app/FaceclawFirmwareUtil.kt" }, { "java": "com/faceclaw/app/FaceclawWavRecorder.java", - "kotlin": "shared/src/androidMain/kotlin/com/faceclaw/app/FaceclawWavRecorder.kt" + "kotlin": "native/kotlin/shared/src/androidMain/kotlin/com/faceclaw/app/FaceclawWavRecorder.kt" + }, + { + "java": "com/faceclaw/app/AndroidNoiseEffects.java", + "kotlin": "App_Resources/Android/src/main/java/com/faceclaw/app/AndroidNoiseEffects.kt" + }, + { + "java": "com/faceclaw/app/FaceclawAlarmActivity.java", + "kotlin": "App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawAlarmActivity.kt" + }, + { + "java": "com/faceclaw/app/FaceclawAlarmReceiver.java", + "kotlin": "App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawAlarmReceiver.kt" + }, + { + "java": "com/faceclaw/app/FaceclawAlarmRescheduleReceiver.java", + "kotlin": "App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawAlarmRescheduleReceiver.kt" + }, + { + "java": "com/faceclaw/app/FaceclawAlarmService.java", + "kotlin": "App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawAlarmService.kt" + }, + { + "java": "com/faceclaw/app/FaceclawAlarms.java", + "kotlin": "App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawAlarms.kt" + }, + { + "java": "com/faceclaw/app/FaceclawAudioTranscoder.java", + "kotlin": "App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawAudioTranscoder.kt" + }, + { + "java": "com/faceclaw/app/FaceclawBleCommunicator.java", + "kotlin": "App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawBleCommunicator.kt" + }, + { + "java": "com/faceclaw/app/FaceclawBleManager.java", + "kotlin": "App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawBleManager.kt" + }, + { + "java": "com/faceclaw/app/FaceclawCalendarProvider.java", + "kotlin": "App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawCalendarProvider.kt" + }, + { + "java": "com/faceclaw/app/FaceclawCaptionEngine.java", + "kotlin": "App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawCaptionEngine.kt" + }, + { + "java": "com/faceclaw/app/FaceclawConversationStore.java", + "kotlin": "App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawConversationStore.kt" + }, + { + "java": "com/faceclaw/app/FaceclawDeviceDiscovery.java", + "kotlin": "App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawDeviceDiscovery.kt" + }, + { + "java": "com/faceclaw/app/FaceclawDeviceInfoProbe.java", + "kotlin": "App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawDeviceInfoProbe.kt" + }, + { + "java": "com/faceclaw/app/FaceclawDiarizer.java", + "kotlin": "App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawDiarizer.kt" + }, + { + "java": "com/faceclaw/app/FaceclawEvenAppDetector.java", + "kotlin": "App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawEvenAppDetector.kt" + }, + { + "java": "com/faceclaw/app/FaceclawEvenHubDocumentStart.java", + "kotlin": "App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawEvenHubDocumentStart.kt" + }, + { + "java": "com/faceclaw/app/FaceclawEvenHubJsBridge.java", + "kotlin": "App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawEvenHubJsBridge.kt" + }, + { + "java": "com/faceclaw/app/FaceclawEvenHubWebView.java", + "kotlin": "App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawEvenHubWebView.kt" + }, + { + "java": "com/faceclaw/app/FaceclawEvenHubWebViewClient.java", + "kotlin": "App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawEvenHubWebViewClient.kt" + }, + { + "java": "com/faceclaw/app/FaceclawEvenHubWebViewHost.java", + "kotlin": "App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawEvenHubWebViewHost.kt" + }, + { + "java": "com/faceclaw/app/FaceclawFirmwareFlasher.java", + "kotlin": "App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawFirmwareFlasher.kt" + }, + { + "java": "com/faceclaw/app/FaceclawFlashPromptCommunicator.java", + "kotlin": "App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawFlashPromptCommunicator.kt" + }, + { + "java": "com/faceclaw/app/FaceclawForegroundService.java", + "kotlin": "App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawForegroundService.kt" + }, + { + "java": "com/faceclaw/app/FaceclawHttp.java", + "kotlin": "App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawHttp.kt" + }, + { + "java": "com/faceclaw/app/FaceclawLc3Decoder.java", + "kotlin": "App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawLc3Decoder.kt" + }, + { + "java": "com/faceclaw/app/FaceclawLlamaRunner.java", + "kotlin": "App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawLlamaRunner.kt" + }, + { + "java": "com/faceclaw/app/FaceclawLocationProvider.java", + "kotlin": "App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawLocationProvider.kt" + }, + { + "java": "com/faceclaw/app/FaceclawLocationTracker.java", + "kotlin": "App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawLocationTracker.kt" + }, + { + "java": "com/faceclaw/app/FaceclawMediaBrowser.java", + "kotlin": "App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawMediaBrowser.kt" + }, + { + "java": "com/faceclaw/app/FaceclawMediaController.java", + "kotlin": "App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawMediaController.kt" + }, + { + "java": "com/faceclaw/app/FaceclawMediaNotificationListenerService.java", + "kotlin": "App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawMediaNotificationListenerService.kt" + }, + { + "java": "com/faceclaw/app/FaceclawModelDownloader.java", + "kotlin": "App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawModelDownloader.kt" + }, + { + "java": "com/faceclaw/app/FaceclawPreviewCompositor.java", + "kotlin": "App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawPreviewCompositor.kt" + }, + { + "java": "com/faceclaw/app/FaceclawQrScanner.java", + "kotlin": "App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawQrScanner.kt" + }, + { + "java": "com/faceclaw/app/FaceclawRemoteInput.java", + "kotlin": "App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawRemoteInput.kt" + }, + { + "java": "com/faceclaw/app/FaceclawResourceUsage.java", + "kotlin": "App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawResourceUsage.kt" + }, + { + "java": "com/faceclaw/app/FaceclawSettings.java", + "kotlin": "App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawSettings.kt" + }, + { + "java": "com/faceclaw/app/FaceclawSettingsPortReceiver.java", + "kotlin": "App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawSettingsPortReceiver.kt" + }, + { + "java": "com/faceclaw/app/FaceclawSpeakerId.java", + "kotlin": "App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawSpeakerId.kt" + }, + { + "java": "com/faceclaw/app/FaceclawSseRequest.java", + "kotlin": "App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawSseRequest.kt" + }, + { + "java": "com/faceclaw/app/FaceclawSystemStatusIconProvider.java", + "kotlin": "App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawSystemStatusIconProvider.kt" + }, + { + "java": "com/faceclaw/app/FaceclawVoiceController.java", + "kotlin": "App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawVoiceController.kt" + }, + { + "java": "com/faceclaw/app/FaceclawWearBridge.java", + "kotlin": "App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawWearBridge.kt" + }, + { + "java": "com/faceclaw/app/FaceclawWearListenerService.java", + "kotlin": "App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawWearListenerService.kt" + }, + { + "java": "com/faceclaw/app/FaceclawWebSocket.java", + "kotlin": "App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawWebSocket.kt" + }, + { + "java": "com/faceclaw/app/FontFileRenderer.java", + "kotlin": "App_Resources/Android/src/main/java/com/faceclaw/app/FontFileRenderer.kt" + }, + { + "java": "com/faceclaw/app/FrameTimings.java", + "kotlin": "App_Resources/Android/src/main/java/com/faceclaw/app/FrameTimings.kt" + }, + { + "java": "com/faceclaw/app/GifScreenRecorder.java", + "kotlin": "App_Resources/Android/src/main/java/com/faceclaw/app/GifScreenRecorder.kt" + }, + { + "java": "com/faceclaw/app/IconRenderer.java", + "kotlin": "App_Resources/Android/src/main/java/com/faceclaw/app/IconRenderer.kt" + }, + { + "java": "com/faceclaw/app/ImageFileLoader.java", + "kotlin": "App_Resources/Android/src/main/java/com/faceclaw/app/ImageFileLoader.kt" + }, + { + "java": "com/faceclaw/app/PreviewBitmapUtil.java", + "kotlin": "App_Resources/Android/src/main/java/com/faceclaw/app/PreviewBitmapUtil.kt" + }, + { + "java": "com/faceclaw/app/ScreenshotUtil.java", + "kotlin": "App_Resources/Android/src/main/java/com/faceclaw/app/ScreenshotUtil.kt" + }, + { + "java": "com/faceclaw/app/util/InterruptibleSleep.java", + "kotlin": "App_Resources/Android/src/main/java/com/faceclaw/app/util/InterruptibleSleep.kt" } ] diff --git a/tests/kotlin-build.test.cjs b/tests/kotlin-build.test.cjs index 07d4f8cf..ba68d135 100644 --- a/tests/kotlin-build.test.cjs +++ b/tests/kotlin-build.test.cjs @@ -38,6 +38,6 @@ test('every remaining Java source has an explicit Android boundary and every mig assert.equal(new Set(migrated.map(entry => entry.java)).size, migrated.length); for (const entry of migrated) { assert.equal(fs.existsSync(path.join(java, entry.java)), false, entry.java); - assert.ok(fs.existsSync(path.join(root, 'native/kotlin', entry.kotlin)), entry.kotlin); + assert.ok(fs.existsSync(path.join(root, entry.kotlin)), entry.kotlin); } }); diff --git a/tests/music-settings.test.cjs b/tests/music-settings.test.cjs index 4b1159f2..ab843e61 100644 --- a/tests/music-settings.test.cjs +++ b/tests/music-settings.test.cjs @@ -186,49 +186,87 @@ test('Music settings keeps toggles and selection when a new app arrives and supp assert.equal(closes, 1); }); -test('native session selection skips ignored playing and idle apps, including the all-ignored case', () => { +// FaceclawMediaController is Kotlin compiled by the NativeScript Android build, so +// compile the extracted method with the same Kotlin compiler Gradle already cached. +function kotlinCompiler() { + const gradle = source('App_Resources/Android/before-plugins.gradle').match(/kotlinVersion\s*=\s*"([^"]+)"/); + const cache = path.join(os.homedir(), '.gradle/caches/modules-2/files-2.1'); + const jar = (group, artifact, version) => { + const directory = path.join(cache, group, artifact, version); + if (!fs.existsSync(directory)) return null; + for (const hash of fs.readdirSync(directory)) { + const file = path.join(directory, hash, `${artifact}-${version}.jar`); + if (fs.existsSync(file)) return file; + } + return null; + }; + const anyVersion = (group, artifact) => { + const directory = path.join(cache, group, artifact); + if (!fs.existsSync(directory)) return null; + for (const version of fs.readdirSync(directory).sort().reverse()) { + const file = jar(group, artifact, version); + if (file) return file; + } + return null; + }; + if (!gradle) return null; + const version = gradle[1]; + const stdlib = jar('org.jetbrains.kotlin', 'kotlin-stdlib', version); + const compiler = [ + jar('org.jetbrains.kotlin', 'kotlin-compiler-embeddable', version), + stdlib, + jar('org.jetbrains.kotlin', 'kotlin-script-runtime', version), + jar('org.jetbrains.kotlin', 'kotlin-daemon-embeddable', version), + anyVersion('org.jetbrains.kotlin', 'kotlin-reflect'), + anyVersion('org.jetbrains.intellij.deps', 'trove4j'), + anyVersion('org.jetbrains', 'annotations'), + anyVersion('org.jetbrains.kotlinx', 'kotlinx-coroutines-core-jvm'), + ]; + if (compiler.some(file => !file)) return null; + return { classpath: compiler.join(path.delimiter), stdlib }; +} + +test('native session selection skips ignored playing and idle apps, including the all-ignored case', (t) => { // Execute the production selection method with lightweight media-session doubles. - const java = source('App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawMediaController.java'); - const method = java.slice(java.indexOf(' private MediaController chooseController('), - java.indexOf(' private void emitSessionAppsLocked(')); - const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'faceclaw-media-java-')); - const javaBin = (name) => process.env.JAVA_HOME ? path.join(process.env.JAVA_HOME, 'bin', name) : name; + const compiler = kotlinCompiler(); + if (!compiler) return t.skip('Kotlin compiler not in the Gradle cache; run ./build.sh once'); + const kotlin = source('App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawMediaController.kt'); + const method = kotlin.slice(kotlin.indexOf(' private fun chooseController('), + kotlin.indexOf(' private fun emitSessionAppsLocked(')); + const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'faceclaw-media-kotlin-')); + const javaBin = process.env.JAVA_HOME ? path.join(process.env.JAVA_HOME, 'bin', 'java') : 'java'; try { - const file = path.join(directory, 'MediaSelectionTest.java'); - fs.writeFileSync(file, `import java.util.*; -public class MediaSelectionTest { - Set ignoredPackages = new HashSet<>(); - static class PlaybackState { - static final int STATE_PLAYING = 3; - int state; PlaybackState(int state) { this.state = state; } int getState() { return state; } + const file = path.join(directory, 'MediaSelectionTest.kt'); + fs.writeFileSync(file, ` +class PlaybackState(val state: Int) { companion object { const val STATE_PLAYING = 3 } } +class MediaController(val packageName: String, state: Int) { var playbackState: PlaybackState? = PlaybackState(state) } +class MediaSelectionTest { + val ignoredPackages: MutableSet = HashSet() +${method} + fun run() { + val ignored = MediaController("browser", 3) + val paused = MediaController("player", 2) + val playing = MediaController("music", 3) + ignoredPackages.add("browser") + check(chooseController(listOf(ignored, paused)) === paused) + check(chooseController(listOf(ignored, paused, playing)) === playing) + check(chooseController(listOf(ignored)) == null) + ignored.playbackState = null + check(chooseController(listOf(ignored, paused)) === paused) + ignoredPackages.add("music") + check(chooseController(listOf(ignored, paused, playing)) === paused) + ignoredPackages.remove("browser"); ignored.playbackState = PlaybackState(3) + check(chooseController(listOf(ignored, paused)) === ignored) + check(chooseController(null) == null) + check(chooseController(emptyList()) == null) } - static class MediaController { - String name; PlaybackState state; - MediaController(String name, int state) { this.name = name; this.state = new PlaybackState(state); } - String getPackageName() { return name; } PlaybackState getPlaybackState() { return state; } - } - ${method} - public static void main(String[] args) { - MediaSelectionTest test = new MediaSelectionTest(); - MediaController ignored = new MediaController("browser", 3); - MediaController paused = new MediaController("player", 2); - MediaController playing = new MediaController("music", 3); - test.ignoredPackages.add("browser"); - assert test.chooseController(Arrays.asList(ignored, paused)) == paused; - assert test.chooseController(Arrays.asList(ignored, paused, playing)) == playing; - assert test.chooseController(Arrays.asList(ignored)) == null; - ignored.state = null; - assert test.chooseController(Arrays.asList(ignored, paused)) == paused; - test.ignoredPackages.add("music"); - assert test.chooseController(Arrays.asList(ignored, paused, playing)) == paused; - test.ignoredPackages.remove("browser"); ignored.state = new PlaybackState(3); - assert test.chooseController(Arrays.asList(ignored, paused)) == ignored; - assert test.chooseController(null) == null; - assert test.chooseController(Collections.emptyList()) == null; - } -}`); - execFileSync(javaBin('javac'), ['-d', directory, file], { stdio: 'pipe' }); - execFileSync(javaBin('java'), ['-ea', '-cp', directory, 'MediaSelectionTest'], { stdio: 'pipe' }); +} +fun main() { MediaSelectionTest().run() } +`); + const out = path.join(directory, 'out'); + execFileSync(javaBin, ['-cp', compiler.classpath, 'org.jetbrains.kotlin.cli.jvm.K2JVMCompiler', + '-no-stdlib', '-cp', compiler.stdlib, '-d', out, file], { stdio: 'pipe' }); + execFileSync(javaBin, ['-cp', [out, compiler.stdlib].join(path.delimiter), 'MediaSelectionTestKt'], { stdio: 'pipe' }); } finally { fs.rmSync(directory, { recursive: true, force: true }); } diff --git a/wear/PROTOCOL.md b/wear/PROTOCOL.md index 90aaf2b0..fba306c9 100644 --- a/wear/PROTOCOL.md +++ b/wear/PROTOCOL.md @@ -3,7 +3,7 @@ Transport: the Wearable Data Layer (Google Play services). Both apps share the `com.faceclaw.app` application id and, for release builds, the signing key — the Data Layer routes only between matching apps. Payloads are UTF-8 JSON -objects. Phone side: `App_Resources/.../FaceclawWearBridge.java` (transport), +objects. Phone side: `App_Resources/.../FaceclawWearBridge.kt` (transport), `app/native/wear-bridge.ts` (JS wrapper), `app/g2/wear-remote.ts` (meaning). Watch side: `app/src/main/kotlin/com/faceclaw/wear/Protocol.kt`, `PhoneLink.kt`.