mirror of
https://git.undefined-dev.fr/Blendman974/faceclaw.git
synced 2026-11-19 21:03:09 +00:00
Fix EvenHub apps on iOS access to the mic
This commit is contained in:
1 parent
247459bf99
commit
53812609ed
5 files changed
+138
-3
No files matched your search
@@ -16,6 +16,8 @@
|
||||
* Android notification icons that are full-color pictures rather than shapes no longer show as blank blocks
|
||||
* Fix missing R1 ring battery indicator on iOS
|
||||
* Fix a bug on iOS that would cause corrupted installs of EvenHub apps
|
||||
* EvenHub apps can use the glasses microphone on iOS (such as Chess's voice moves, which said "Mic unavailable")
|
||||
* EvenHub apps on iOS should now be able to use the mic
|
||||
|
||||
## 0.8.4 - 2026-10-17
|
||||
|
||||
|
||||
@@ -27,6 +27,7 @@ export type EvenHubMicClient = {
|
||||
deliverAudioPcm(pcm: Uint8Array): void;
|
||||
};
|
||||
|
||||
/** Android's session for the raw tap; iOS binds its own to the bridge (setRawSession). */
|
||||
function activeCommunicator(): any {
|
||||
if (!global.isAndroid) return null;
|
||||
try {
|
||||
|
||||
@@ -726,7 +726,6 @@ export class EvenHubSession implements EvenHubMicClient, EvenHubImuClient, EvenH
|
||||
* flows (foreground + screen on + no assistant modal).
|
||||
*/
|
||||
private audioControl(data: Record<string, unknown>): boolean {
|
||||
if (global.isIOS) return false; // EvenHub PCM routing is not ported yet.
|
||||
if (!permissionsIncludeMicrophone(this.manifest.permissions)) {
|
||||
this.log("evenhub: audioControl denied (app declares no microphone permission)");
|
||||
return false;
|
||||
|
||||
@@ -443,6 +443,7 @@ export class IosPreviewController {
|
||||
iosVoiceInput.handleSessionEnded('Glasses locked. Unlock your phone to start voice input again.',
|
||||
'Glasses locked. Transcription resumes when you unlock your phone.')
|
||||
} else this.syncContinuousCapture()
|
||||
this.syncRawMicSession()
|
||||
void (async () => {
|
||||
await this.displayReady
|
||||
const display = this.display
|
||||
@@ -745,6 +746,8 @@ export class IosPreviewController {
|
||||
this.pushBrightness()
|
||||
this.maybeLaunchOnboarding(this.state)
|
||||
if (this.state.phase !== 'connected' && this.state.phase !== 'connecting') this.glassesWorn = null
|
||||
// Unbound first, so a speech capture ending below doesn't hand the mic back to the dead session.
|
||||
this.syncRawMicSession()
|
||||
if (this.state.phase !== 'connected') iosVoiceInput.handleSessionEnded()
|
||||
this.syncContinuousSession()
|
||||
this.pushBatteryLevels()
|
||||
@@ -952,6 +955,8 @@ export class IosPreviewController {
|
||||
for (const off of this.sessionOffs.splice(0)) off()
|
||||
this.notifications = null
|
||||
this.communicator = null
|
||||
// Releases an EvenHub app's mic before the session closes.
|
||||
this.syncRawMicSession()
|
||||
this.glance.reset()
|
||||
await communicator.close().catch(error => this.logBluetooth(`Close: ${String(error)}`))
|
||||
}
|
||||
@@ -1065,6 +1070,15 @@ export class IosPreviewController {
|
||||
}
|
||||
if (current()) this.syncContinuousClaim()
|
||||
}
|
||||
/**
|
||||
* EvenHub apps' mic (the bridge's raw tap) listens through a connected,
|
||||
* unlocked glasses session, as speech capture does; the EvenHub mic router
|
||||
* decides which app hears it and when.
|
||||
*/
|
||||
private syncRawMicSession(): void {
|
||||
const ready = this.state.phase === 'connected' && !this.glassesLocked
|
||||
iosVoiceInput.setRawSession(ready ? this.communicator : null, message => this.logBluetooth(message))
|
||||
}
|
||||
/** Other mic users stand down while continuous capture really holds the mic, not merely while a window wants it. */
|
||||
private syncContinuousClaim(): void {
|
||||
voiceActivity.setActive(this.continuousClients.size > 0 && iosVoiceInput.continuousActive, CONTINUOUS_CAPTURE_HOLDER)
|
||||
|
||||
@@ -11,6 +11,9 @@ export type IosMicrophoneSession = {
|
||||
}
|
||||
/** How long a cloud provider gets, after the last audio, to send its final transcript. */
|
||||
const CLOUD_FINAL_TIMEOUT_MS = 5000
|
||||
/** Retries of a failed raw-tap start (a new session's display path is usually still coming up). */
|
||||
const RAW_RETRY_MS = 3000
|
||||
const RAW_RETRY_LIMIT = 5
|
||||
|
||||
/** iOS implementation of the shared voice dialog's event bridge. Audio comes
|
||||
* from the glasses when connected, or the default phone input in preview mode,
|
||||
@@ -53,6 +56,21 @@ export class IosVoiceControlBridge {
|
||||
// and it is created by the first audio, so a failed start leaves no file.
|
||||
private recordingName = ''
|
||||
private recorder: WavRecorder | null = null
|
||||
// Raw-PCM tap (EvenHub mic apps): decode-only glasses audio, broadcast to
|
||||
// rawListeners while any owner wants it, from the session setRawSession
|
||||
// bound. It shares the one glasses mic with speech capture, which preempts
|
||||
// it; it resumes on its own once that capture has ended.
|
||||
private rawNative: any = null
|
||||
private readonly rawListeners = new Set<(pcm: Uint8Array) => void>()
|
||||
private readonly rawOwners = new Set<string>()
|
||||
private rawSession: IosMicrophoneSession | null = null
|
||||
// The session the tap enabled the mic on, while it is starting or live.
|
||||
private rawHeld: IosMicrophoneSession | null = null
|
||||
private rawLive = false
|
||||
private rawGeneration = 0
|
||||
private rawFailures = 0
|
||||
private rawRetryTimer: ReturnType<typeof setTimeout> | null = null
|
||||
private rawLog: (message: string) => void = () => {}
|
||||
|
||||
onStatus(listener: (state: VoiceControlState) => void): () => void {
|
||||
this.statuses.add(listener); listener({ status: this.status, listening: this.capturing, detail: '' }); return () => { this.statuses.delete(listener) }
|
||||
@@ -62,7 +80,7 @@ export class IosVoiceControlBridge {
|
||||
}
|
||||
onSpeechEnd(listener: () => void): () => void { this.ends.add(listener); return () => { this.ends.delete(listener) } }
|
||||
onSpeechPause(_listener: () => void): () => void { return () => {} }
|
||||
/** The mic level (see pcmLevel) of each PCM chunk; only cloud captures supply PCM. */
|
||||
/** The mic level (see pcmLevel) of each PCM chunk; only cloud captures and the raw tap supply PCM. */
|
||||
onMicLevel(listener: (level: number) => void): () => void { this.levels.add(listener); return () => { this.levels.delete(listener) } }
|
||||
private setStatus(status: string): void { this.status = status; for (const fn of [...this.statuses]) fn({ status, listening: this.capturing, detail: '' }) }
|
||||
private ensureNative(): any {
|
||||
@@ -102,6 +120,8 @@ export class IosVoiceControlBridge {
|
||||
get statusText(): string { return this.status }
|
||||
async startGlassesCapture(session: IosMicrophoneSession, voice: VoiceProviderSelection, log: (message: string) => void, endpointing = false, continuous = false): Promise<void> {
|
||||
if (this.joinsContinuous(continuous)) return
|
||||
// Speech capture preempts the raw tap; its mic release is queued before this enable, which wins.
|
||||
this.endRawTap()
|
||||
this.stop()
|
||||
const generation = ++this.generation
|
||||
this.continuous = continuous
|
||||
@@ -131,6 +151,7 @@ export class IosVoiceControlBridge {
|
||||
}
|
||||
async startPhoneCapture(voice: VoiceProviderSelection, log: (message: string) => void, endpointing = false, continuous = false): Promise<void> {
|
||||
if (this.joinsContinuous(continuous)) return
|
||||
this.endRawTap()
|
||||
this.stop()
|
||||
this.continuous = continuous
|
||||
this.log = log; this.source = 'phone'; this.capturing = true
|
||||
@@ -228,6 +249,100 @@ export class IosVoiceControlBridge {
|
||||
if (this.continuous) { if (this.capturing) void this.finalizeCapture(continuousMessage); return }
|
||||
this.stop(); this.setStatus(message); this.emitEnd()
|
||||
}
|
||||
/** Subscribe to the raw tap's decoded glasses PCM (16 kHz mono S16LE). */
|
||||
onRawPcm(listener: (pcm: Uint8Array) => void): () => void {
|
||||
this.rawListeners.add(listener); return () => { this.rawListeners.delete(listener) }
|
||||
}
|
||||
/**
|
||||
* The glasses session the raw tap captures from (connected and unlocked),
|
||||
* or null. Owners outlive a change: the tap moves to the next session.
|
||||
*/
|
||||
setRawSession(session: IosMicrophoneSession | null, log: (message: string) => void): void {
|
||||
this.rawLog = log
|
||||
if (session === this.rawSession) return
|
||||
this.endRawTap()
|
||||
this.rawSession = session; this.rawFailures = 0
|
||||
this.syncRawTap()
|
||||
}
|
||||
/**
|
||||
* Start the decode-only raw tap on behalf of `owner`; the communicator is
|
||||
* the one setRawSession bound, not `options.communicator`. False while
|
||||
* speech capture holds the mic or no session is bound: the owner stays
|
||||
* registered and the tap starts once it can. Idempotent while running.
|
||||
*/
|
||||
startRawCapture(options: { communicator: any; owner: string }): boolean {
|
||||
if (!this.rawOwners.has(options.owner)) { this.rawOwners.add(options.owner); this.rawFailures = 0 }
|
||||
this.syncRawTap()
|
||||
return this.rawHeld !== null
|
||||
}
|
||||
/** `owner` no longer wants the raw tap; it stops once no owner does. */
|
||||
stopRawCapture(owner: string): void {
|
||||
if (this.rawOwners.delete(owner)) this.syncRawTap()
|
||||
}
|
||||
/** Whether the raw tap's glasses mic enable went through and still holds. */
|
||||
isRawCaptureLive(): boolean { return this.rawLive }
|
||||
/** Run the tap while an owner wants it, a session is bound, and no speech capture holds the mic. */
|
||||
private syncRawTap(): void {
|
||||
const session = this.rawSession
|
||||
if (!session || !this.rawOwners.size || this.capturing || this.finalizing) { this.endRawTap(); return }
|
||||
if (this.rawHeld || this.rawRetryTimer !== null) return
|
||||
void this.beginRawTap(session)
|
||||
}
|
||||
private async beginRawTap(session: IosMicrophoneSession): Promise<void> {
|
||||
const generation = ++this.rawGeneration
|
||||
const native = this.ensureRawNative()
|
||||
const error = String(native.startPcmCaptureWithEndpointingPhone(false, false) ?? '')
|
||||
if (error) { this.failRawTap(error); return }
|
||||
this.rawHeld = session
|
||||
try {
|
||||
await session.setMicrophone(true, packet => {
|
||||
if (generation !== this.rawGeneration) return
|
||||
const copy = new Uint8Array(packet)
|
||||
native.acceptPacket(NSData.dataWithBytesLength(interop.handleof(copy.buffer), copy.byteLength))
|
||||
})
|
||||
if (generation !== this.rawGeneration) return
|
||||
this.rawLive = true
|
||||
this.rawLog('EvenHub mic: glasses microphone enabled')
|
||||
} catch (failure) {
|
||||
if (generation === this.rawGeneration) this.failRawTap(failure instanceof Error ? failure.message : String(failure))
|
||||
}
|
||||
}
|
||||
private ensureRawNative(): any {
|
||||
if (!this.rawNative) {
|
||||
this.rawNative = FaceclawSpeech.new()
|
||||
// A PCM capture only ends on its own when decoding failed or fell behind.
|
||||
this.rawNative.eventHandler = (json: string) => {
|
||||
const event = JSON.parse(json)
|
||||
if (event.kind === 'ended' && this.rawHeld) this.failRawTap(String(event.message ?? 'audio ended'))
|
||||
}
|
||||
this.rawNative.pcmHandler = (pcm: NSData) => {
|
||||
// Copied: bufferFromData doesn't retain pcm.
|
||||
const bytes = new Uint8Array(interop.bufferFromData(pcm)).slice()
|
||||
// Audio is flowing, so a later failure is a new problem worth retrying.
|
||||
this.rawFailures = 0
|
||||
for (const fn of [...this.rawListeners]) fn(bytes)
|
||||
if (this.levels.size) { const level = pcmLevel(bytes); for (const fn of [...this.levels]) fn(level) }
|
||||
}
|
||||
}
|
||||
return this.rawNative
|
||||
}
|
||||
/** Drop the tap and retry it shortly, a bounded number of times. */
|
||||
private failRawTap(message: string): void {
|
||||
this.rawLog(`EvenHub mic: ${message}`)
|
||||
this.endRawTap()
|
||||
if (++this.rawFailures >= RAW_RETRY_LIMIT) return
|
||||
this.rawRetryTimer = setTimeout(() => { this.rawRetryTimer = null; this.syncRawTap() }, RAW_RETRY_MS)
|
||||
}
|
||||
/** Stop the tap; owners stay. Releases the mic only if the tap enabled it. */
|
||||
private endRawTap(): void {
|
||||
++this.rawGeneration
|
||||
if (this.rawRetryTimer !== null) clearTimeout(this.rawRetryTimer)
|
||||
this.rawRetryTimer = null
|
||||
this.rawLive = false
|
||||
this.rawNative?.cancel()
|
||||
const session = this.rawHeld; this.rawHeld = null
|
||||
if (session) void session.setMicrophone(false).catch(error => this.rawLog(`EvenHub mic cleanup: ${error}`))
|
||||
}
|
||||
private clearAudioTimer(): void { if (this.audioTimer !== null) clearTimeout(this.audioTimer); this.audioTimer = null }
|
||||
private releaseMicrophone(): void {
|
||||
const session = this.session; this.session = null
|
||||
@@ -261,7 +376,11 @@ export class IosVoiceControlBridge {
|
||||
// In continuous capture, earlier pause segments (paragraphBreakAfter) may still arrive.
|
||||
if (event.isFinal && this.cloudEndStatus !== null && !(this.continuous && event.paragraphBreakAfter)) this.completeCloud()
|
||||
}
|
||||
private emitEnd(): void { for (const fn of [...this.ends]) fn() }
|
||||
private emitEnd(): void {
|
||||
for (const fn of [...this.ends]) fn()
|
||||
// After the listeners: one may start the next capture, which keeps the mic.
|
||||
this.syncRawTap()
|
||||
}
|
||||
private finishCompletion(): void {
|
||||
const resolve = this.resolveCompletion; this.resolveCompletion = null; resolve?.()
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user