diff --git a/App_Resources/Android/app.gradle b/App_Resources/Android/app.gradle index 997e1f4d..99e36fe3 100644 --- a/App_Resources/Android/app.gradle +++ b/App_Resources/Android/app.gradle @@ -19,6 +19,9 @@ dependencies { // app's captions (translation models download on demand, per language). implementation 'com.google.mlkit:language-id:17.0.6' implementation 'com.google.mlkit:translate:17.0.3' + // Pure-Java Opus decoder for the RayNeo iO's microphone (16 kHz mono Opus over + // BLE); the same one Glassly uses. See AndroidSpeechEngines.OpusDecoderAdapter. + implementation 'io.github.jaredmdobson:concentus:1.0.1' } def faceclawSherpaOnnxVersion = "1.13.0" diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/AndroidSpeechEngines.kt b/App_Resources/Android/src/main/java/com/faceclaw/app/AndroidSpeechEngines.kt index 2b149f09..83357d08 100644 --- a/App_Resources/Android/src/main/java/com/faceclaw/app/AndroidSpeechEngines.kt +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/AndroidSpeechEngines.kt @@ -94,6 +94,44 @@ internal object AndroidSpeechEngines { override fun close() = decoder.close() } + /** + * The RayNeo iO's microphone: one 16 kHz mono Opus packet per BLE message (its remote + * audio stream, GlassesSessionIo), decoded with Concentus. No G2 DSP metadata; packets + * carry no counter, so "missing" stays 0 and a lost one is just a gap. + */ + class OpusDecoderAdapter : AudioPacketDecoder { + private val decoder = io.github.jaredmdobson.OpusDecoder(RayNeoIoProtocol.VOICE_SAMPLE_RATE, 1) + private var packets = 0L + private var errors = 0L + + override fun decodePacket(packet: ByteArray?, pcmOut: ShortArray?): Int { + if (packet == null || packet.isEmpty() || pcmOut == null) return 0 + return try { + val count = decoder.decode(packet, 0, packet.size, pcmOut, 0, pcmOut.size, false) + packets++ + count + } catch (e: io.github.jaredmdobson.OpusException) { + errors++ + if (errors == 1L || errors % 100 == 0L) { + android.util.Log.w("FaceclawVoice", "Opus decode error ($errors so far): ${e.message}") + } + 0 + } + } + + /** 120 ms, Opus's longest frame. */ + override val maxSamplesPerPacket: Int get() = RayNeoIoProtocol.VOICE_SAMPLE_RATE * 120 / 1000 + override val hasFrameMeta: Boolean get() = false + override val lastAngleDegrees: Int get() = 0 + override val lastSsr: Int get() = 0 + override val realPackets: Long get() = packets + override val duplicatePackets: Long get() = 0 + override val missingPackets: Long get() = 0 + override val decodeErrors: Long get() = errors + + override fun close() = decoder.resetState() + } + /** * Open the phone's own microphone at the pipeline's native format (16 kHz * mono PCM16), or null when it cannot start — the permission is missing diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawBleCommunicator.kt b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawBleCommunicator.kt index 5089a039..eb75263a 100644 --- a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawBleCommunicator.kt +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawBleCommunicator.kt @@ -254,6 +254,9 @@ class FaceclawBleCommunicator(context: Context, rightAddress: String?, leftAddre fun setFaceclawWakeLeaseEnabled(enabled: Boolean): Boolean = core.setFaceclawWakeLeaseEnabled(enabled) + /** See [GlassesSessionCore.setVoiceWakeEnabled]. */ + fun setVoiceWakeEnabled(enabled: Boolean) = core.setVoiceWakeEnabled(enabled) + fun awaitEvenHubSessionReady(timeoutMs: Int): Boolean = core.awaitEvenHubSessionReady(timeoutMs) fun setImuReportEnabled(enable: Boolean, reportFrq: Int) = core.setImuReportEnabled(enable, reportFrq) diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawVoiceController.kt b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawVoiceController.kt index ad0531bb..ec143d1a 100644 --- a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawVoiceController.kt +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawVoiceController.kt @@ -114,7 +114,9 @@ class FaceclawVoiceController(context: Context) { } } - override fun createPacketDecoder(): AudioPacketDecoder = AndroidSpeechEngines.Lc3DecoderAdapter() + override fun createPacketDecoder(): AudioPacketDecoder = + if (communicator?.isRayNeoIo() == true) AndroidSpeechEngines.OpusDecoderAdapter() + else AndroidSpeechEngines.Lc3DecoderAdapter() override fun openPhoneMic(): PcmAudioSource? = AndroidSpeechEngines.openPhoneMic(PHONE_MIC_CHUNK_SAMPLES) diff --git a/app/g2/dashboard-controller.ts b/app/g2/dashboard-controller.ts index a3d53c01..ca9fe7b5 100644 --- a/app/g2/dashboard-controller.ts +++ b/app/g2/dashboard-controller.ts @@ -468,6 +468,7 @@ class DashboardController { this.pushFirmwareDebugFlags(); this.pushBrightness(); this.syncEvenHubScreenOffSetting(); + this.syncVoiceWake(); // Display mode size first: a reflow repositions every surface too, so // the band-position check after it then has nothing left to do. this.applyDisplayModeIfChanged(); @@ -930,6 +931,24 @@ class DashboardController { return applied; } + /** What the communicator was last told about the glasses' own wake word (null: not yet). */ + private voiceWakeSent: boolean | null = null; + + /** + * The glasses' own wake word follows the wake action: on a RayNeo iO, its + * "Hey RayNeo" switch (G2 glasses use the wake lease, syncFaceclawWakeLease). + */ + private syncVoiceWake(communicator = this.communicator): void { + if (!communicator) return; + const enabled = wakeWordActionSetting.get() !== "off"; + if (this.voiceWakeSent === enabled) return; + this.voiceWakeSent = enabled; + void communicator.setVoiceWakeEnabled(enabled).catch((error) => { + this.voiceWakeSent = null; + this.appendLog(`wake word sync failed: ${this.formatError(error)}`); + }); + } + private syncEvenHubScreenOffSetting(): void { void this.syncFaceclawWakeLease().catch((error) => { this.appendLog(`wake takeover lease sync failed: ${this.formatError(error)}`); @@ -1537,6 +1556,8 @@ class DashboardController { // The paired glasses' screen: the G2's 640x480 or the RayNeo iO's 540x220. await communicator.configureCompositorScreen(screenWidth(), screenHeight()); await communicator.configureBrightness(getBrightnessPreferences()); + this.voiceWakeSent = null; + this.syncVoiceWake(communicator); this.offState = communicator.onStateChange((state) => { if (state.phase !== "connected") resetRingInputFilter(); if (isHaltedSessionPhase(state.phase)) { diff --git a/app/native/faceclaw-communicator.ios.ts b/app/native/faceclaw-communicator.ios.ts index 712a4da0..de99072a 100644 --- a/app/native/faceclaw-communicator.ios.ts +++ b/app/native/faceclaw-communicator.ios.ts @@ -679,6 +679,9 @@ export class FaceclawCommunicatorBridge { return this.enqueueNativeCall(() => Boolean(this.communicator.setFaceclawWakeLeaseEnabledEnabled(Boolean(enabled)))); } + /** The RayNeo iO's wake word; iOS doesn't drive an iO, so nothing to do. */ + async setVoiceWakeEnabled(_enabled: boolean): Promise {} + async awaitEvenHubSessionReady(timeoutMs: number): Promise { return this.enqueueNativeCall(() => Boolean(this.communicator.awaitEvenHubSessionReadyTimeoutMs(Math.round(nonNegativeNumber(timeoutMs))))); } diff --git a/app/native/faceclaw-communicator.ts b/app/native/faceclaw-communicator.ts index 80d13b24..8dc7d86f 100644 --- a/app/native/faceclaw-communicator.ts +++ b/app/native/faceclaw-communicator.ts @@ -734,6 +734,14 @@ export class FaceclawCommunicatorBridge { ); } + /** + * Whether the glasses' own wake word reaches Faceclaw: on a RayNeo iO, its + * "Hey RayNeo" switch (G2 glasses use the wake lease above instead). + */ + async setVoiceWakeEnabled(enabled: boolean): Promise { + await this.enqueueJavaCall(() => this.communicator.setVoiceWakeEnabled(enabled)); + } + /** * Resolve only once the recreated layout and retained frame * are visible. CFW's deferred-dashboard READY is emitted from this barrier. diff --git a/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/audio/VoiceCaptureSession.kt b/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/audio/VoiceCaptureSession.kt index 33c81cf1..3a68f866 100644 --- a/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/audio/VoiceCaptureSession.kt +++ b/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/audio/VoiceCaptureSession.kt @@ -449,9 +449,10 @@ class VoiceCaptureSession( } private fun processG2Audio() { - val pcm = ShortArray(PACKET_SAMPLES) + var pcm = ShortArray(PACKET_SAMPLES) while (started) { val currentDecoder = packetDecoder ?: return + if (pcm.size < currentDecoder.maxSamplesPerPacket) pcm = ShortArray(currentDecoder.maxSamplesPerPacket) val packet = takeAudioPacket() if (packet == null) { continue @@ -474,7 +475,7 @@ class VoiceCaptureSession( maybeEmitAudioStats(false) continue } - processPcmChunk(pcm, count, angleDegrees, ssr, true) + processPcmChunk(pcm, count, angleDegrees, ssr, currentDecoder.hasFrameMeta) maybeEmitAudioStats(false) } } diff --git a/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/audio/VoicePorts.kt b/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/audio/VoicePorts.kt index 148c68a4..1844459a 100644 --- a/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/audio/VoicePorts.kt +++ b/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/audio/VoicePorts.kt @@ -39,10 +39,19 @@ fun interface RecordingSink { fun save(pcm16le: ByteArray, sampleRate: Int) } -/** A G2 microphone packet decoder (the shared [Lc3PacketFramer] over a platform LC3 codec). */ +/** + * A glasses microphone packet decoder to 16 kHz mono PCM: the G2's LC3 (the shared + * [Lc3PacketFramer] over a platform LC3 codec) or the RayNeo iO's Opus. + */ interface AudioPacketDecoder { fun decodePacket(packet: ByteArray?, pcmOut: ShortArray?): Int + /** The most samples one packet decodes to: the size [decodePacket]'s pcmOut needs. */ + val maxSamplesPerPacket: Int get() = Lc3PacketFramer.SAMPLES_PER_PACKET + + /** Whether packets carry the G2 firmware's DSP metadata ([lastAngleDegrees], [lastSsr]). */ + val hasFrameMeta: Boolean get() = true + val lastAngleDegrees: Int val lastSsr: Int val realPackets: Long diff --git a/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/g2protocol/session/GlassesSessionCore.kt b/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/g2protocol/session/GlassesSessionCore.kt index 23832ed1..d15710d9 100644 --- a/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/g2protocol/session/GlassesSessionCore.kt +++ b/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/g2protocol/session/GlassesSessionCore.kt @@ -670,6 +670,8 @@ class GlassesSessionCore( if (listener == null) { throw IllegalArgumentException("listener is required") } + // The iO's packets are Opus from its remote audio stream (see ioStartAudioCapture). + if (isIo) return ioStartAudioCapture(listener) var magic = 0 monitor.withLock { if (!running || !sessionReady || shutdownRequested || !fixedLayoutCreated) { @@ -687,6 +689,7 @@ class GlassesSessionCore( } fun stopG2AudioCapture() { + if (isIo) return ioStopAudioCapture() var magic = 0 monitor.withLock { audioPacketListener = null @@ -794,6 +797,15 @@ class GlassesSessionCore( * that track a capture across those events must check this rather than * assume their earlier enable still holds. */ + /** + * Whether the glasses' own wake word should reach Faceclaw (its configured wake action). + * RayNeo iO: the launcher's "Hey RayNeo" switch, and the stock session it opens is + * ended in favor of Faceclaw's. G2 glasses use the wake lease instead (no-op here). + */ + fun setVoiceWakeEnabled(enabled: Boolean) { + if (isIo) ioSetVoiceWakeEnabled(enabled) + } + fun isAudioCaptureActive(): Boolean { monitor.withLock { return running && sessionReady && !shutdownRequested && audioCaptureActive @@ -810,6 +822,8 @@ class GlassesSessionCore( val generation: Int monitor.withLock { faceclawWakeLeaseEnabled = enabled + // A G2 firmware lease; the iO's wake word is setVoiceWakeEnabled. + if (isIo) return true if (!running || !sessionReady) { return !enabled } diff --git a/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/g2protocol/session/GlassesSessionIo.kt b/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/g2protocol/session/GlassesSessionIo.kt index 6f22bfc1..9dd54d91 100644 --- a/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/g2protocol/session/GlassesSessionIo.kt +++ b/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/g2protocol/session/GlassesSessionIo.kt @@ -52,6 +52,23 @@ internal class IoSessionState { /** Last `screen_status` push: 1 on, 0 off, -1 unknown. */ var screenStatus = -1 + /** Whether Faceclaw answers "Hey RayNeo" (null until the dashboard says); what this link was last told. */ + var voiceWakeWanted: Boolean? = null + var voiceWakeSent = -1 + /** The stock assistant opened a session (wake word): end it, it must not take the display. */ + var voiceAbortPending = false + /** Ask for the launcher's general settings (logged: the wake-word preferences among them). */ + var settingsRequestPending = true + /** When a wake was last reported: one phrase can wake twice (fast and second stage). */ + var lastVoiceWakeAtMs = 0L + + /** A capture wants the remote mic stream; whether this link was asked to open it, and when. */ + var micWanted = false + var micStreamOpen = false + var micStartSentAtMs = 0L + var micPackets = 0L + var lastMicPacketAtMs = 0L + fun resetLink() { reassembler.reset() versionReply = null @@ -65,6 +82,11 @@ internal class IoSessionState { brightnessSentLevel = -1 lastBrightnessAtMs = 0 screenStatus = -1 + voiceWakeSent = -1 + voiceAbortPending = false + settingsRequestPending = true + micStreamOpen = false + micStartSentAtMs = 0 } } @@ -83,6 +105,10 @@ internal object IoTimings { const val PAGE_REASON_USER = 4 /** While Faceclaw shows content, re-assert the screen this often (the stock auto-off is short). */ const val SCREEN_REFRESH_MS = 5_000L + /** A wanted mic stream that hasn't delivered a packet this long is asked for again. */ + const val MIC_RETRY_MS = 3_000L + /** Wakes this close to a reported one are the same phrase (fast stage, then second stage). */ + const val VOICE_WAKE_DEDUP_MS = 3_000L } /** The glasses hung up at the version request: bound to another phone id, or not in pairing mode. */ @@ -341,9 +367,38 @@ internal fun GlassesSessionCore.ioMaintain() { var ctrl = -1 var status = false var brightness = -1 + var wakeword = -1 + var micRc = -1 + var abortVoice = false + var settings = false monitor.withLock { if (!sessionReady) return val now = now() + val wake = io.voiceWakeWanted + if (wake != null && io.voiceWakeSent != (if (wake) 1 else 0)) { + wakeword = if (wake) 1 else 0 + io.voiceWakeSent = wakeword + // Read back what the switch did. + io.settingsRequestPending = true + } + if (io.settingsRequestPending) { + settings = true + io.settingsRequestPending = false + } + if (io.voiceAbortPending) { + abortVoice = true + io.voiceAbortPending = false + } + if (io.micWanted && (!io.micStreamOpen + || (io.lastMicPacketAtMs < io.micStartSentAtMs && now - io.micStartSentAtMs >= IoTimings.MIC_RETRY_MS))) { + // Not yet asked on this link, a voice session took the recorder, or the ask was lost. + micRc = RayNeoIoProtocol.VOICE_RC_START + io.micStreamOpen = true + io.micStartSentAtMs = now + } else if (!io.micWanted && io.micStreamOpen) { + micRc = RayNeoIoProtocol.VOICE_RC_STOP + io.micStreamOpen = false + } if (io.pageWanted && !fixedLayoutCreated && now - io.pageRequestSentAtMs >= IoTimings.PAGE_RETRY_MS) { ctrl = RayNeoIoProtocol.CTRL_PAGE_OPEN io.pageRequestSentAtMs = now @@ -374,6 +429,60 @@ internal fun GlassesSessionCore.ioMaintain() { } if (status) ioRequestStatus() if (brightness > 0) ioWriteLauncher(RayNeoIoProtocol.LAUNCHER_BRIGHTNESS, RayNeoIoProtocol.launcherBrightness(brightness)) + if (wakeword >= 0) { + logLine("RayNeo iO: wake word " + (if (wakeword == 1) "on" else "off")) + ioWriteLauncher(RayNeoIoProtocol.LAUNCHER_AI_VOICE_WAKEUP, RayNeoIoProtocol.launcherSetAiVoiceWakeup(wakeword == 1)) + } + if (settings) ioWriteLauncher(RayNeoIoProtocol.LAUNCHER_GENERAL_SETTINGS, RayNeoIoProtocol.GENERAL_SETTINGS_REQUEST) + if (abortVoice) { + // As Glassly does: decline the stock session, then abort it, so its page goes. + ioWriteFrame(RayNeoIoProtocol.BUSINESS_VOICE, RayNeoIoProtocol.voiceRc(RayNeoIoProtocol.VOICE_RC, RayNeoIoProtocol.VOICE_RC_STOP)) + ioWriteFrame(RayNeoIoProtocol.BUSINESS_VOICE, RayNeoIoProtocol.voiceRc(RayNeoIoProtocol.VOICE_ABORT, RayNeoIoProtocol.VOICE_RC_START)) + } + if (micRc > 0) { + logLine("RayNeo iO mic: " + (if (micRc == RayNeoIoProtocol.VOICE_RC_START) "open" else "close") + " the remote audio stream") + ioWriteFrame(RayNeoIoProtocol.BUSINESS_VOICE, RayNeoIoProtocol.voiceRc(RayNeoIoProtocol.VOICE_STREAM_CONTROL, micRc)) + } +} + +/** + * The glasses' mic for a voice capture: the remote audio stream (16 kHz mono Opus packets to + * [listener]), which needs IDENT_FLAG_MIC firmware. ioMaintain sends the open (and repeats it + * while no packet comes); a voice session or a call can take the recorder meanwhile. + */ +internal fun GlassesSessionCore.ioStartAudioCapture(listener: FaceclawAudioPacketListener): Boolean { + monitor.withLock { + if (!running || !sessionReady || shutdownRequested) { + logLine("skip RayNeo iO mic: session not ready") + return false + } + if (io.ident?.micStream != true) { + logLine("skip RayNeo iO mic: the firmware has no remote audio stream (IDENT flag 0x20)") + return false + } + audioPacketListener = listener + audioCaptureActive = true + io.micWanted = true + io.micStreamOpen = false + io.micPackets = 0 + } + interruptibleSleep.interrupt() + return true +} + +internal fun GlassesSessionCore.ioStopAudioCapture() { + monitor.withLock { + audioPacketListener = null + audioCaptureActive = false + io.micWanted = false + } + interruptibleSleep.interrupt() +} + +/** Whether "Hey RayNeo" should reach Faceclaw (the launcher's wake-word switch). */ +internal fun GlassesSessionCore.ioSetVoiceWakeEnabled(enabled: Boolean) { + monitor.withLock { io.voiceWakeWanted = enabled } + interruptibleSleep.interrupt() } /** Stop drawing and close the page, keeping the BLE link (the EvenHub suspend analogue). */ @@ -421,6 +530,7 @@ internal fun GlassesSessionCore.ioOnNotification(address: String, characteristic monitor.signalAll() } RayNeoIoProtocol.BUSINESS_LAUNCHER -> ioHandleLauncher(frame.payload) + RayNeoIoProtocol.BUSINESS_VOICE -> ioHandleVoice(frame.payload) RayNeoIoProtocol.BUSINESS_OTA -> { val envelope = RayNeoIoProtocol.decodeEnvelope(frame.payload) ?: continue if (envelope.type != RayNeoIoProtocol.OTA_VERSION) continue @@ -450,7 +560,11 @@ private fun GlassesSessionCore.ioHandleLauncher(payload: ByteArray) { return } val battery = RayNeoIoProtocol.batteryFromLauncherJson(envelope.json) - if (battery < 0) return + if (battery < 0) { + // Replies to our commands and settings (the wake word's among them): logged for now. + logLine("RayNeo iO launcher type " + envelope.type + ": " + envelope.json.take(400)) + return + } val charging = if (envelope.json.contains("\"isCharging\":true")) 1 else if (envelope.json.contains("\"isCharging\":false")) 0 else -1 var changed = false monitor.withLock { @@ -464,6 +578,73 @@ private fun GlassesSessionCore.ioHandleLauncher(payload: ByteArray) { if (changed) emitBatteryState(headsetBattery, headsetCharging) } +/** The voice channel (business 13): mic packets, and the stock assistant's wake word. */ +private fun GlassesSessionCore.ioHandleVoice(payload: ByteArray) { + val envelope = RayNeoIoProtocol.decodeEnvelope(payload) ?: return + when (envelope.type) { + RayNeoIoProtocol.VOICE_STREAM_AUDIO, RayNeoIoProtocol.VOICE_AUDIO -> { + val opus = envelope.binary ?: return + val arrivalMs = now() + var first = false + val listener = monitor.withLock { + lastIncomingAtMs = arrivalMs + io.micPackets++ + first = io.micPackets == 1L + io.lastMicPacketAtMs = arrivalMs + if (audioCaptureActive) audioPacketListener else null + } + if (first) logLine("RayNeo iO mic: first audio packet (type " + envelope.type + ", " + opus.size + " bytes)") + if (listener == null) return + try { + listener.onAudioPacket(opus, "R", arrivalMs) + } catch (t: Throwable) { + logLine("RayNeo iO mic packet listener failed: " + safeMessage(t)) + } + } + RayNeoIoProtocol.VOICE_WAKE -> { + // The stock assistant heard "Hey RayNeo" and opened its session. Faceclaw's own + // assistant answers instead: end the stock one, and report the wake word as the + // G2's (sid 0x07 EVEN_AI_WAKE_UP), which the dashboard routes to its wake action. + var repeat = false + val forward = monitor.withLock { + val now = now() + io.voiceAbortPending = true + lastIncomingAtMs = now + lastConnectionOrInputAtMs = now + repeat = io.lastVoiceWakeAtMs != 0L && now - io.lastVoiceWakeAtMs < IoTimings.VOICE_WAKE_DEDUP_MS + val wanted = io.voiceWakeWanted == true && sessionReady + if (wanted && !repeat) { + io.lastVoiceWakeAtMs = now + ioReclaimPageOnInputLocked() + } + wanted && !repeat + } + interruptibleSleep.interrupt() + logLine("RayNeo iO: voice wake " + envelope.json + when { + forward -> "" + repeat -> " (same phrase as the last wake)" + else -> " (not forwarded: wake word off)" + }) + if (!forward) return + val event = G2Event("even-ai", "", BleProtocol.EVEN_AI_STATUS_WAKE_UP, 0, 0) + val frameId = frameTimings.startFrame("input:even-ai wake") + frameTimings.log(frameId, "wake word from the RayNeo iO's voice channel") + emitRingEvent(event, frameId) + } + RayNeoIoProtocol.VOICE_EXITED -> { + // A voice session that held the recorder ended: ask for the stream again. + val reopen = monitor.withLock { + if (io.micWanted) io.micStreamOpen = false + io.micWanted + } + if (reopen) interruptibleSleep.interrupt() + logLine("RayNeo iO: voice session exited " + envelope.json + (if (reopen) "; reopening the mic stream" else "")) + } + RayNeoIoProtocol.VOICE_VAD -> logDebug("RayNeo iO: voice VAD " + envelope.json) + else -> logLine("RayNeo iO: voice message type " + envelope.type + " " + envelope.json.take(160)) + } +} + private fun GlassesSessionCore.ioHandleCfw(address: String, frame: RayNeoIoProtocol.Frame) { val payload = frame.payload if (payload.isEmpty()) return diff --git a/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/rayneo/RayNeoIoProtocol.kt b/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/rayneo/RayNeoIoProtocol.kt index c525bc15..d0cd42ae 100644 --- a/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/rayneo/RayNeoIoProtocol.kt +++ b/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/rayneo/RayNeoIoProtocol.kt @@ -40,6 +40,7 @@ object RayNeoIoBle { object RayNeoIoProtocol { const val BUSINESS_OTA = 9 + const val BUSINESS_VOICE = 13 const val BUSINESS_LAUNCHER = 15 const val BUSINESS_HANDSHAKE = 0x10 const val BUSINESS_CFW = 0xf0 @@ -248,8 +249,46 @@ object RayNeoIoProtocol { const val MODE_REPLY = 2 const val LAUNCHER_GENERAL_STATUS = 1 const val LAUNCHER_BRIGHTNESS = 2 + const val LAUNCHER_GENERAL_SETTINGS = 4 + const val LAUNCHER_AI_VOICE_WAKEUP = 16 const val OTA_VERSION = 1 const val GENERAL_STATUS_REQUEST = """{"cmd":"request_general_status","payload":{"data":"","mode":0,"value":0}}""" + /** The reply (type 4) carries `generalSettings`, among them the wake-word preferences. */ + const val GENERAL_SETTINGS_REQUEST = """{"cmd":"request_general_settings","payload":{"value":0,"mode":0,"data":""}}""" + + /** + * The launcher's wake-word switch. Every `set_ai_*` handler takes the new value from + * `mode` and ignores `value` (RnLinkMsgProcessor::handleAssistantSettingCommand, see + * notes/rayneo-io-wakeword-re.md): mode 1 on, 0 off. + */ + @JvmStatic + fun launcherSetAiVoiceWakeup(enabled: Boolean): String = + """{"cmd":"set_ai_voice_wakeup","payload":{"value":0,"mode":${if (enabled) 1 else 0},"data":""}}""" + + // ---- Voice assistant (business 13), as Glassly's RayNeoIOProtocol maps it ---- + // The stock assistant: the wake word (or a temple long/double press) opens a session + // and sends VOICE_WAKE; the phone answers VOICE_RC {"rc":1} for Opus as VOICE_AUDIO, + // or {"rc":2} plus VOICE_ABORT to end it; VOICE_EXITED reports it gone. The remote + // stream (VOICE_STREAM_CONTROL {"rc":1|2}, packets as VOICE_STREAM_AUDIO) needs + // IDENT_FLAG_MIC firmware and runs without a session or its page, until a session, + // a call, SCO or a recording takes the recorder. Packets are 16 kHz mono Opus in + // the envelope's bytes field. + + const val VOICE_WAKE = 1 + const val VOICE_RC = 2 + const val VOICE_AUDIO = 3 + const val VOICE_VAD = 4 + const val VOICE_ABORT = 7 + const val VOICE_EXITED = 8 + const val VOICE_STREAM_CONTROL = 58 + const val VOICE_STREAM_AUDIO = 59 + const val VOICE_RC_START = 1 + const val VOICE_RC_STOP = 2 + const val VOICE_SAMPLE_RATE = 16_000 + + /** A voice-channel message: `{"rc":n}` as [type] (VOICE_RC, VOICE_STREAM_CONTROL or VOICE_ABORT). */ + @JvmStatic + fun voiceRc(type: Int, rc: Int): ByteArray = encodeEnvelope(Envelope(type, """{"rc":$rc}""")) class Envelope( @JvmField val type: Int, @@ -415,6 +454,8 @@ object RayNeoIoProtocol { const val IDENT_FLAG_ZLIB = 0x01 const val IDENT_FLAG_STEREO = 0x08 + /** The assistant's remote audio stream (type 58/59 below) is allowed (CFW revision 10). */ + const val IDENT_FLAG_MIC = 0x20 const val IDENT_FLAG_FACECLAW = 0x80 class Ident( @@ -428,6 +469,7 @@ object RayNeoIoProtocol { @JvmField val faceclawRevision: Int, ) { val acceptsCompressed: Boolean get() = flags and IDENT_FLAG_ZLIB != 0 + val micStream: Boolean get() = flags and IDENT_FLAG_MIC != 0 val faceclaw: Boolean get() = flags and IDENT_FLAG_FACECLAW != 0 && faceclawRevision > 0 } diff --git a/tests/kotlin/src/commonTest/kotlin/com/faceclaw/app/RayNeoIoSessionTest.kt b/tests/kotlin/src/commonTest/kotlin/com/faceclaw/app/RayNeoIoSessionTest.kt index 01b087fc..9e140ac3 100644 --- a/tests/kotlin/src/commonTest/kotlin/com/faceclaw/app/RayNeoIoSessionTest.kt +++ b/tests/kotlin/src/commonTest/kotlin/com/faceclaw/app/RayNeoIoSessionTest.kt @@ -44,6 +44,13 @@ private class FakeIoGlasses(platform: ProtocolPlatform) : SessionLink { private val messagesList = ArrayList() private val launcherList = ArrayList() val launcher: List get() = lock.withLock { launcherList.toList() } + /** Voice-channel (business 13) messages from the phone: type to JSON. */ + private val voiceList = ArrayList>() + val voice: List> get() = lock.withLock { voiceList.toList() } + + /** A voice-channel message from the glasses. */ + fun voiceNotify(type: Int, json: String = "{}", binary: ByteArray? = null) = + notify(13, RayNeoIoProtocol.encodeEnvelope(RayNeoIoProtocol.Envelope(type, json, binary))) private val records = ArrayList() private var current = ByteArray(0) private val inbound = RayNeoIoProtocol.FrameReassembler() @@ -133,6 +140,10 @@ private class FakeIoGlasses(platform: ProtocolPlatform) : SessionLink { """{"generalStatus":{"battery":77,"isCharging":false}}""", sequence = RayNeoIoProtocol.decodeEnvelope(p)!!.sequence, mode = 2))) } 9 -> notify(9, RayNeoIoProtocol.encodeEnvelope(RayNeoIoProtocol.Envelope(1, """{"OsVersion":"Strix OS 1.0.4.12"}"""))) + 13 -> { + val envelope = RayNeoIoProtocol.decodeEnvelope(p)!! + lock.withLock { voiceList.add(envelope.type to envelope.json) } + } 0xf0 -> cfw(p) } } @@ -439,4 +450,70 @@ class RayNeoIoSessionTest { s.core.close() } } + + @Test + fun theWakeWordAndTheMicStream() { + val s = IoSession() + s.core.configureCompositorScreen(540, 220) + val lock = s.platform.createLock() + val packets = ArrayList() + val listener = object : FaceclawAudioPacketListener { + override fun onAudioPacket(data: ByteArray?, arm: String?, arrivalMs: Long) { + lock.withLock { packets.add(data!!) } + } + } + val count = { lock.withLock { packets.size } } + fun sent(type: Int, json: String) = s.glasses.voice.count { it == type to json } + val start = """{"rc":1}""" + val stop = """{"rc":2}""" + try { + assertTrue(s.core.start()) + assertTrue(awaitTrue(5_000) { s.pageUp() }, s.tail()) + + // "Hey RayNeo" goes to Faceclaw: the launcher's switch on, then each wake ends the + // stock session (declined, then aborted) and arrives as the G2's EVEN_AI_WAKE_UP. + s.core.setVoiceWakeEnabled(true) + // The firmware reads the switch from "mode", not "value". + assertTrue(awaitTrue(3_000) { s.glasses.launcher.any { "set_ai_voice_wakeup" in it && "\"mode\":1" in it } }, s.tail()) + s.glasses.voiceNotify(RayNeoIoProtocol.VOICE_WAKE, """{"rc":2,"mode":2,"linkType":0}""") + assertTrue(awaitTrue(2_000) { "even-ai:${BleProtocol.EVEN_AI_STATUS_WAKE_UP}:0" in s.listener.events }, s.listener.events.toString()) + // The same phrase's second-stage wake is not a second wake. + s.glasses.voiceNotify(RayNeoIoProtocol.VOICE_WAKE, """{"rc":2,"mode":3,"linkType":0}""") + sleepMs(200) + assertEquals(1, s.listener.events.count { it.startsWith("even-ai:") }) + assertTrue(awaitTrue(2_000) { + sent(RayNeoIoProtocol.VOICE_RC, stop) >= 1 && sent(RayNeoIoProtocol.VOICE_ABORT, start) >= 1 + }, s.glasses.voice.toString()) + + // The mic: capture opens the remote stream, its Opus packets go to the listener as + // they came, and stopping closes it. + assertTrue(s.core.startG2AudioCapture(listener)) + assertTrue(awaitTrue(2_000) { sent(RayNeoIoProtocol.VOICE_STREAM_CONTROL, start) == 1 }, s.tail()) + s.glasses.voiceNotify(RayNeoIoProtocol.VOICE_STREAM_AUDIO, binary = byteArrayOf(0x78, 1, 2, 3)) + assertTrue(awaitTrue(2_000) { count() == 1 }) + assertTrue(lock.withLock { packets[0] }.contentEquals(byteArrayOf(0x78, 1, 2, 3))) + assertTrue(s.core.isAudioCaptureActive()) + // A voice session that took the recorder ended: the stream is asked for again. + s.glasses.voiceNotify(RayNeoIoProtocol.VOICE_EXITED) + assertTrue(awaitTrue(2_000) { sent(RayNeoIoProtocol.VOICE_STREAM_CONTROL, start) == 2 }, s.tail()) + s.core.stopG2AudioCapture() + assertTrue(awaitTrue(2_000) { sent(RayNeoIoProtocol.VOICE_STREAM_CONTROL, stop) == 1 }, s.tail()) + assertFalse(s.core.isAudioCaptureActive()) + s.glasses.voiceNotify(RayNeoIoProtocol.VOICE_STREAM_AUDIO, binary = byteArrayOf(9)) + sleepMs(200) + assertEquals(1, count()) + + // With the wake action off: the switch follows, and a wake is ended but not reported. + s.core.setVoiceWakeEnabled(false) + assertTrue(awaitTrue(3_000) { s.glasses.launcher.any { "set_ai_voice_wakeup" in it && "\"mode\":0" in it } }, s.tail()) + val events = s.listener.events.size + val aborts = sent(RayNeoIoProtocol.VOICE_ABORT, start) + s.platform.offsetMs += 5_000 + s.glasses.voiceNotify(RayNeoIoProtocol.VOICE_WAKE) + assertTrue(awaitTrue(2_000) { sent(RayNeoIoProtocol.VOICE_ABORT, start) > aborts }, s.glasses.voice.toString()) + assertEquals(events, s.listener.events.size) + } finally { + s.core.close() + } + } }