From 53812609ed7fbb7d8042017fb7edf334e04031cf Mon Sep 17 00:00:00 2001 From: James Babcock Date: Sun, 25 Oct 2026 14:19:45 -0700 Subject: [PATCH] Fix EvenHub apps on iOS access to the mic --- CHANGELOG | 2 + app/apps/evenhub/mic-router.ts | 1 + app/apps/evenhub/session.ts | 1 - app/g2/ios-preview-controller.ts | 14 ++++ app/native/voice-control.ios.ts | 123 ++++++++++++++++++++++++++++++- 5 files changed, 138 insertions(+), 3 deletions(-) diff --git a/CHANGELOG b/CHANGELOG index 2badb3b3..2a6d8960 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -16,6 +16,8 @@ * Android notification icons that are full-color pictures rather than shapes no longer show as blank blocks * Fix missing R1 ring battery indicator on iOS * Fix a bug on iOS that would cause corrupted installs of EvenHub apps +* EvenHub apps can use the glasses microphone on iOS (such as Chess's voice moves, which said "Mic unavailable") +* EvenHub apps on iOS should now be able to use the mic ## 0.8.4 - 2026-10-17 diff --git a/app/apps/evenhub/mic-router.ts b/app/apps/evenhub/mic-router.ts index 76f7fe15..bf7d7d57 100644 --- a/app/apps/evenhub/mic-router.ts +++ b/app/apps/evenhub/mic-router.ts @@ -27,6 +27,7 @@ export type EvenHubMicClient = { deliverAudioPcm(pcm: Uint8Array): void; }; +/** Android's session for the raw tap; iOS binds its own to the bridge (setRawSession). */ function activeCommunicator(): any { if (!global.isAndroid) return null; try { diff --git a/app/apps/evenhub/session.ts b/app/apps/evenhub/session.ts index 53289885..bdbdd92d 100644 --- a/app/apps/evenhub/session.ts +++ b/app/apps/evenhub/session.ts @@ -726,7 +726,6 @@ export class EvenHubSession implements EvenHubMicClient, EvenHubImuClient, EvenH * flows (foreground + screen on + no assistant modal). */ private audioControl(data: Record): boolean { - if (global.isIOS) return false; // EvenHub PCM routing is not ported yet. if (!permissionsIncludeMicrophone(this.manifest.permissions)) { this.log("evenhub: audioControl denied (app declares no microphone permission)"); return false; diff --git a/app/g2/ios-preview-controller.ts b/app/g2/ios-preview-controller.ts index 84af7403..04ea57df 100644 --- a/app/g2/ios-preview-controller.ts +++ b/app/g2/ios-preview-controller.ts @@ -443,6 +443,7 @@ export class IosPreviewController { iosVoiceInput.handleSessionEnded('Glasses locked. Unlock your phone to start voice input again.', 'Glasses locked. Transcription resumes when you unlock your phone.') } else this.syncContinuousCapture() + this.syncRawMicSession() void (async () => { await this.displayReady const display = this.display @@ -745,6 +746,8 @@ export class IosPreviewController { this.pushBrightness() this.maybeLaunchOnboarding(this.state) if (this.state.phase !== 'connected' && this.state.phase !== 'connecting') this.glassesWorn = null + // Unbound first, so a speech capture ending below doesn't hand the mic back to the dead session. + this.syncRawMicSession() if (this.state.phase !== 'connected') iosVoiceInput.handleSessionEnded() this.syncContinuousSession() this.pushBatteryLevels() @@ -952,6 +955,8 @@ export class IosPreviewController { for (const off of this.sessionOffs.splice(0)) off() this.notifications = null this.communicator = null + // Releases an EvenHub app's mic before the session closes. + this.syncRawMicSession() this.glance.reset() await communicator.close().catch(error => this.logBluetooth(`Close: ${String(error)}`)) } @@ -1065,6 +1070,15 @@ export class IosPreviewController { } if (current()) this.syncContinuousClaim() } + /** + * EvenHub apps' mic (the bridge's raw tap) listens through a connected, + * unlocked glasses session, as speech capture does; the EvenHub mic router + * decides which app hears it and when. + */ + private syncRawMicSession(): void { + const ready = this.state.phase === 'connected' && !this.glassesLocked + iosVoiceInput.setRawSession(ready ? this.communicator : null, message => this.logBluetooth(message)) + } /** Other mic users stand down while continuous capture really holds the mic, not merely while a window wants it. */ private syncContinuousClaim(): void { voiceActivity.setActive(this.continuousClients.size > 0 && iosVoiceInput.continuousActive, CONTINUOUS_CAPTURE_HOLDER) diff --git a/app/native/voice-control.ios.ts b/app/native/voice-control.ios.ts index 4730ff81..2699515a 100644 --- a/app/native/voice-control.ios.ts +++ b/app/native/voice-control.ios.ts @@ -11,6 +11,9 @@ export type IosMicrophoneSession = { } /** How long a cloud provider gets, after the last audio, to send its final transcript. */ const CLOUD_FINAL_TIMEOUT_MS = 5000 +/** Retries of a failed raw-tap start (a new session's display path is usually still coming up). */ +const RAW_RETRY_MS = 3000 +const RAW_RETRY_LIMIT = 5 /** iOS implementation of the shared voice dialog's event bridge. Audio comes * from the glasses when connected, or the default phone input in preview mode, @@ -53,6 +56,21 @@ export class IosVoiceControlBridge { // and it is created by the first audio, so a failed start leaves no file. private recordingName = '' private recorder: WavRecorder | null = null + // Raw-PCM tap (EvenHub mic apps): decode-only glasses audio, broadcast to + // rawListeners while any owner wants it, from the session setRawSession + // bound. It shares the one glasses mic with speech capture, which preempts + // it; it resumes on its own once that capture has ended. + private rawNative: any = null + private readonly rawListeners = new Set<(pcm: Uint8Array) => void>() + private readonly rawOwners = new Set() + private rawSession: IosMicrophoneSession | null = null + // The session the tap enabled the mic on, while it is starting or live. + private rawHeld: IosMicrophoneSession | null = null + private rawLive = false + private rawGeneration = 0 + private rawFailures = 0 + private rawRetryTimer: ReturnType | null = null + private rawLog: (message: string) => void = () => {} onStatus(listener: (state: VoiceControlState) => void): () => void { this.statuses.add(listener); listener({ status: this.status, listening: this.capturing, detail: '' }); return () => { this.statuses.delete(listener) } @@ -62,7 +80,7 @@ export class IosVoiceControlBridge { } onSpeechEnd(listener: () => void): () => void { this.ends.add(listener); return () => { this.ends.delete(listener) } } onSpeechPause(_listener: () => void): () => void { return () => {} } - /** The mic level (see pcmLevel) of each PCM chunk; only cloud captures supply PCM. */ + /** The mic level (see pcmLevel) of each PCM chunk; only cloud captures and the raw tap supply PCM. */ onMicLevel(listener: (level: number) => void): () => void { this.levels.add(listener); return () => { this.levels.delete(listener) } } private setStatus(status: string): void { this.status = status; for (const fn of [...this.statuses]) fn({ status, listening: this.capturing, detail: '' }) } private ensureNative(): any { @@ -102,6 +120,8 @@ export class IosVoiceControlBridge { get statusText(): string { return this.status } async startGlassesCapture(session: IosMicrophoneSession, voice: VoiceProviderSelection, log: (message: string) => void, endpointing = false, continuous = false): Promise { if (this.joinsContinuous(continuous)) return + // Speech capture preempts the raw tap; its mic release is queued before this enable, which wins. + this.endRawTap() this.stop() const generation = ++this.generation this.continuous = continuous @@ -131,6 +151,7 @@ export class IosVoiceControlBridge { } async startPhoneCapture(voice: VoiceProviderSelection, log: (message: string) => void, endpointing = false, continuous = false): Promise { if (this.joinsContinuous(continuous)) return + this.endRawTap() this.stop() this.continuous = continuous this.log = log; this.source = 'phone'; this.capturing = true @@ -228,6 +249,100 @@ export class IosVoiceControlBridge { if (this.continuous) { if (this.capturing) void this.finalizeCapture(continuousMessage); return } this.stop(); this.setStatus(message); this.emitEnd() } + /** Subscribe to the raw tap's decoded glasses PCM (16 kHz mono S16LE). */ + onRawPcm(listener: (pcm: Uint8Array) => void): () => void { + this.rawListeners.add(listener); return () => { this.rawListeners.delete(listener) } + } + /** + * The glasses session the raw tap captures from (connected and unlocked), + * or null. Owners outlive a change: the tap moves to the next session. + */ + setRawSession(session: IosMicrophoneSession | null, log: (message: string) => void): void { + this.rawLog = log + if (session === this.rawSession) return + this.endRawTap() + this.rawSession = session; this.rawFailures = 0 + this.syncRawTap() + } + /** + * Start the decode-only raw tap on behalf of `owner`; the communicator is + * the one setRawSession bound, not `options.communicator`. False while + * speech capture holds the mic or no session is bound: the owner stays + * registered and the tap starts once it can. Idempotent while running. + */ + startRawCapture(options: { communicator: any; owner: string }): boolean { + if (!this.rawOwners.has(options.owner)) { this.rawOwners.add(options.owner); this.rawFailures = 0 } + this.syncRawTap() + return this.rawHeld !== null + } + /** `owner` no longer wants the raw tap; it stops once no owner does. */ + stopRawCapture(owner: string): void { + if (this.rawOwners.delete(owner)) this.syncRawTap() + } + /** Whether the raw tap's glasses mic enable went through and still holds. */ + isRawCaptureLive(): boolean { return this.rawLive } + /** Run the tap while an owner wants it, a session is bound, and no speech capture holds the mic. */ + private syncRawTap(): void { + const session = this.rawSession + if (!session || !this.rawOwners.size || this.capturing || this.finalizing) { this.endRawTap(); return } + if (this.rawHeld || this.rawRetryTimer !== null) return + void this.beginRawTap(session) + } + private async beginRawTap(session: IosMicrophoneSession): Promise { + const generation = ++this.rawGeneration + const native = this.ensureRawNative() + const error = String(native.startPcmCaptureWithEndpointingPhone(false, false) ?? '') + if (error) { this.failRawTap(error); return } + this.rawHeld = session + try { + await session.setMicrophone(true, packet => { + if (generation !== this.rawGeneration) return + const copy = new Uint8Array(packet) + native.acceptPacket(NSData.dataWithBytesLength(interop.handleof(copy.buffer), copy.byteLength)) + }) + if (generation !== this.rawGeneration) return + this.rawLive = true + this.rawLog('EvenHub mic: glasses microphone enabled') + } catch (failure) { + if (generation === this.rawGeneration) this.failRawTap(failure instanceof Error ? failure.message : String(failure)) + } + } + private ensureRawNative(): any { + if (!this.rawNative) { + this.rawNative = FaceclawSpeech.new() + // A PCM capture only ends on its own when decoding failed or fell behind. + this.rawNative.eventHandler = (json: string) => { + const event = JSON.parse(json) + if (event.kind === 'ended' && this.rawHeld) this.failRawTap(String(event.message ?? 'audio ended')) + } + this.rawNative.pcmHandler = (pcm: NSData) => { + // Copied: bufferFromData doesn't retain pcm. + const bytes = new Uint8Array(interop.bufferFromData(pcm)).slice() + // Audio is flowing, so a later failure is a new problem worth retrying. + this.rawFailures = 0 + for (const fn of [...this.rawListeners]) fn(bytes) + if (this.levels.size) { const level = pcmLevel(bytes); for (const fn of [...this.levels]) fn(level) } + } + } + return this.rawNative + } + /** Drop the tap and retry it shortly, a bounded number of times. */ + private failRawTap(message: string): void { + this.rawLog(`EvenHub mic: ${message}`) + this.endRawTap() + if (++this.rawFailures >= RAW_RETRY_LIMIT) return + this.rawRetryTimer = setTimeout(() => { this.rawRetryTimer = null; this.syncRawTap() }, RAW_RETRY_MS) + } + /** Stop the tap; owners stay. Releases the mic only if the tap enabled it. */ + private endRawTap(): void { + ++this.rawGeneration + if (this.rawRetryTimer !== null) clearTimeout(this.rawRetryTimer) + this.rawRetryTimer = null + this.rawLive = false + this.rawNative?.cancel() + const session = this.rawHeld; this.rawHeld = null + if (session) void session.setMicrophone(false).catch(error => this.rawLog(`EvenHub mic cleanup: ${error}`)) + } private clearAudioTimer(): void { if (this.audioTimer !== null) clearTimeout(this.audioTimer); this.audioTimer = null } private releaseMicrophone(): void { const session = this.session; this.session = null @@ -261,7 +376,11 @@ export class IosVoiceControlBridge { // In continuous capture, earlier pause segments (paragraphBreakAfter) may still arrive. if (event.isFinal && this.cloudEndStatus !== null && !(this.continuous && event.paragraphBreakAfter)) this.completeCloud() } - private emitEnd(): void { for (const fn of [...this.ends]) fn() } + private emitEnd(): void { + for (const fn of [...this.ends]) fn() + // After the listeners: one may start the next capture, which keeps the mic. + this.syncRawTap() + } private finishCompletion(): void { const resolve = this.resolveCompletion; this.resolveCompletion = null; resolve?.() }