diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawVoiceController.java b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawVoiceController.java index 37a53d58..de09ae61 100644 --- a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawVoiceController.java +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawVoiceController.java @@ -130,7 +130,7 @@ public class FaceclawVoiceController { private String lastTranscript = ""; private volatile boolean saveRecordings; private volatile boolean endpointing; - private final EndpointDetector endpointDetector = new EndpointDetector(); + private final VoiceEndpointDetector endpointDetector = new VoiceEndpointDetector(); private java.io.ByteArrayOutputStream recordingPcm; // Speaker verification against the enrolled wearer voice-print ("my voice // only" command gating). Configured before start(); the utterance PCM is @@ -1028,111 +1028,6 @@ public class FaceclawVoiceController { } } - /** - * Decides when a hands-free utterance is over, so "Hey Even" capture can - * stop without a button release. - * - * Runs on the decoded 16 kHz PCM, so it works the same in every input mode - * (the cloud path never sees the samples on this side, and the onboard - * recognizer's own endpointing only covers ONBOARD). - * - * Timing is measured on the sample clock rather than the wall clock: BLE - * delivers mic packets in bursts, so elapsed real time badly overestimates - * how much audio has actually been heard. - * - * The threshold is relative to a noise floor measured over the first - * {@link #CALIBRATE_MS} of the session, which is roughly the interval where - * the user is reacting to the dialog appearing and not yet speaking. - */ - private static final class EndpointDetector { - /** Audio used to estimate the room's noise floor. */ - private static final int CALIBRATE_MS = 300; - /** Speech must exceed this multiple of the noise floor to count as onset. */ - private static final double ONSET_FACTOR = 3.0; - /** Below this multiple of the noise floor counts as silence again. */ - private static final double RELEASE_FACTOR = 1.8; - /** Absolute floor, so a silent room can't make the threshold ~0. */ - private static final double MIN_RMS = 220.0; - /** Trailing silence that ends an utterance. */ - private static final int SILENCE_MS = 900; - /** If the user never speaks, give up rather than record forever. */ - private static final int LEAD_IN_MS = 6000; - /** Hard cap on a single utterance. */ - private static final int MAX_UTTERANCE_MS = 30000; - - private long totalSamples; - private double noiseAccum; - private int noisePackets; - private double threshold; - private boolean speechStarted; - private long silenceSamples; - private boolean fired; - - void reset() { - totalSamples = 0; - noiseAccum = 0; - noisePackets = 0; - threshold = 0; - speechStarted = false; - silenceSamples = 0; - fired = false; - } - - /** Returns true exactly once, on the packet that ends the utterance. */ - boolean accept(short[] pcm, int count) { - if (fired || count <= 0) { - return false; - } - totalSamples += count; - long elapsedMs = totalSamples * 1000L / SAMPLE_RATE; - - double sumSquares = 0; - for (int i = 0; i < count; i++) { - double s = pcm[i]; - sumSquares += s * s; - } - double rms = Math.sqrt(sumSquares / count); - - if (elapsedMs <= CALIBRATE_MS) { - noiseAccum += rms; - noisePackets++; - return false; - } - if (threshold == 0) { - double noiseFloor = noisePackets > 0 ? noiseAccum / noisePackets : 0; - threshold = Math.max(noiseFloor, MIN_RMS); - } - - if (!speechStarted) { - if (rms >= threshold * ONSET_FACTOR) { - speechStarted = true; - silenceSamples = 0; - } else if (elapsedMs >= LEAD_IN_MS) { - // Never heard anything; close the dialog rather than hang. - fired = true; - return true; - } - return false; - } - - if (rms < threshold * RELEASE_FACTOR) { - silenceSamples += count; - if (silenceSamples * 1000L / SAMPLE_RATE >= SILENCE_MS) { - fired = true; - return true; - } - } else { - silenceSamples = 0; - } - - if (elapsedMs >= MAX_UTTERANCE_MS) { - fired = true; - return true; - } - return false; - } - } - private void stopG2Audio() { FaceclawBleCommunicator currentCommunicator = communicator; if (currentCommunicator != null) { diff --git a/App_Resources/iOS/src/FaceclawSpeech.h b/App_Resources/iOS/src/FaceclawSpeech.h index 7474d225..49e14f8c 100644 --- a/App_Resources/iOS/src/FaceclawSpeech.h +++ b/App_Resources/iOS/src/FaceclawSpeech.h @@ -5,7 +5,7 @@ + (NSInteger)authorizationStatus; + (void)requestAuthorization:(void (^)(NSInteger status))completion; /** Returns an actionable error, or an empty string when capture starts. */ -- (NSString *)start; +- (NSString *)startWithEndpointing:(BOOL)endpointing; - (void)acceptPacket:(NSData *)packet; - (void)finish; - (void)cancel; diff --git a/App_Resources/iOS/src/FaceclawSpeech.m b/App_Resources/iOS/src/FaceclawSpeech.m index 631d6748..8da4da55 100644 --- a/App_Resources/iOS/src/FaceclawSpeech.m +++ b/App_Resources/iOS/src/FaceclawSpeech.m @@ -1,6 +1,7 @@ #import "FaceclawSpeech.h" #import "FaceclawLc3Decoder.h" #import "FaceclawSpeechTranscript.h" +#import #import #import #import @@ -16,6 +17,7 @@ @property(nonatomic) BOOL accepting; @property(nonatomic, copy) NSString *bestText; @property(nonatomic, strong) FaceclawSpeechTranscript *transcript; +@property(nonatomic, strong) FaceclawKitVoiceEndpointDetector *endpointDetector; @end @implementation FaceclawSpeech - (instancetype)init { @@ -33,7 +35,7 @@ NSData *json = [NSJSONSerialization dataWithJSONObject:event options:0 error:nil]; if (json) self.eventHandler([[NSString alloc] initWithData:json encoding:NSUTF8StringEncoding]); } -- (NSString *)start { +- (NSString *)startWithEndpointing:(BOOL)endpointing { [self cancel]; if (SFSpeechRecognizer.authorizationStatus != SFSpeechRecognizerAuthorizationStatusAuthorized) return @"Allow Speech Recognition for Faceclaw in iPhone Settings."; @@ -52,6 +54,7 @@ if (@available(iOS 16.0, *)) self.request.addsPunctuation = YES; self.bestText = @""; self.accepting = YES; self.transcript = [FaceclawSpeechTranscript new]; + self.endpointDetector = endpointing ? [FaceclawKitVoiceEndpointDetector new] : nil; NSUInteger generation = self.generation; __weak FaceclawSpeech *weakSelf = self; self.task = [self.recognizer recognitionTaskWithRequest:self.request resultHandler:^(SFSpeechRecognitionResult *result, NSError *error) { @@ -89,11 +92,13 @@ NSUInteger generation = self.generation; FaceclawLc3Decoder *decoder = self.decoder; SFSpeechAudioBufferRecognitionRequest *request = self.request; + FaceclawKitVoiceEndpointDetector *endpointDetector = self.endpointDetector; NSData *copy = [packet copy]; dispatch_async(self.audioQueue, ^{ @autoreleasepool { if (self.generation != generation) return; NSData *pcm = [decoder decodePacket:copy]; double rms = 0; + BOOL speechEnded = NO; if (pcm.length) { AVAudioFormat *format = [[AVAudioFormat alloc] initWithCommonFormat:AVAudioPCMFormatFloat32 sampleRate:16000 channels:1 interleaved:NO]; AVAudioPCMBuffer *buffer = [[AVAudioPCMBuffer alloc] initWithPCMFormat:format frameCapacity:(AVAudioFrameCount)(pcm.length / 2)]; @@ -105,11 +110,13 @@ } rms = sqrt(rms / buffer.frameLength); if (self.generation == generation) [request appendAudioPCMBuffer:buffer]; + speechEnded = [endpointDetector acceptLevelRms:rms * 32768.0 sampleCount:(int32_t)buffer.frameLength]; } NSUInteger packets = decoder.packets, errors = decoder.errors, missing = decoder.missing; dispatch_async(dispatch_get_main_queue(), ^{ if (self.generation != generation) return; self.pendingPackets--; + if (speechEnded && self.accepting) [self finish]; if (pcm.length && packets % 20 == 0) [self emit:@{@"kind":@"audio", @"packets":@(packets), @"seconds":@(packets * 0.05), @"rms":@(rms), @"missing":@(missing), @"errors":@(errors)}]; if (errors > 10 && !packets) [self complete:@"Could not decode glasses audio. Reconnect the glasses and retry."]; }); @@ -140,6 +147,6 @@ self.generation++; self.accepting = NO; self.pendingPackets = 0; [self.task cancel]; self.task = nil; self.request = nil; self.decoder = nil; self.recognizer = nil; - self.bestText = @""; self.transcript = nil; + self.bestText = @""; self.transcript = nil; self.endpointDetector = nil; } @end diff --git a/app/g2/ble-protocol.ios.ts b/app/g2/ble-protocol.ios.ts index e2cfaa80..fbf9adc4 100644 --- a/app/g2/ble-protocol.ios.ts +++ b/app/g2/ble-protocol.ios.ts @@ -52,7 +52,7 @@ export const readString = (data: Uint8Array, field: number): string => protocol. export function authenticationSucceeded(message: ProtocolMessage, magic: number): boolean { return message.sid === SID.auth && message.command === 4 && message.magic === magic && readBytes(message.payload, 3)?.length === 0 } -export type GlassesInput = { kind: 'sys-event' | 'list-click' | 'text-click' | 'display-wake'; eventType: number; eventSource: number; systemExitReasonCode: number; containerName: string; frameId: number } +export type GlassesInput = { kind: 'sys-event' | 'list-click' | 'text-click' | 'display-wake' | 'even-ai'; eventType: number; eventSource: number; systemExitReasonCode: number; containerName: string; frameId: number } function input(event: any): GlassesInput | null { return event ? { kind: event.kind === 'sys-event' && event.eventType === OsEventTypeList.HEAD_UP_EVENT ? 'display-wake' : event.kind, eventType: event.eventType, eventSource: event.eventSource, systemExitReasonCode: event.systemExitReasonCode, diff --git a/app/g2/ble-protocol.ts b/app/g2/ble-protocol.ts index e8115773..603e36d2 100644 --- a/app/g2/ble-protocol.ts +++ b/app/g2/ble-protocol.ts @@ -127,7 +127,7 @@ export function readString(data: Uint8Array, number: number): string { return Ar export function authenticationSucceeded(message: ProtocolMessage, magic: number): boolean { return message.sid === SID.auth && message.command === 4 && message.magic === magic && readBytes(message.payload, 3)?.length === 0 } -export type GlassesInput = { kind: 'sys-event' | 'list-click' | 'text-click' | 'display-wake'; eventType: number; eventSource: number; systemExitReasonCode: number; containerName: string; frameId: number } +export type GlassesInput = { kind: 'sys-event' | 'list-click' | 'text-click' | 'display-wake' | 'even-ai'; eventType: number; eventSource: number; systemExitReasonCode: number; containerName: string; frameId: number } export function decodeGlassesInput(message: ProtocolMessage): GlassesInput | null { if (message.sid !== SID.hub || ![1, 6].includes(message.flag)) return null const events = readBytes(message.payload, 13); if (!events) return null diff --git a/app/g2/ios-preview-controller.ts b/app/g2/ios-preview-controller.ts index 8e211581..f61fed99 100644 --- a/app/g2/ios-preview-controller.ts +++ b/app/g2/ios-preview-controller.ts @@ -93,7 +93,7 @@ export class IosPreviewController { ...noopLayerActions, requestRender: () => this.requestShellRender(), disconnect: () => this.disconnect(), - startVoiceCapture: () => this.startVoiceCapture(), + startVoiceCapture: endpointing => this.startVoiceCapture(endpointing), stopVoiceCapture: () => iosVoiceInput.stopPushToTalk(), startContinuousVoiceCapture: () => this.onError("Voice capture is not available on iOS yet."), startTextSettingEdit: async setting => { await this.editSetting(setting) }, @@ -437,9 +437,9 @@ export class IosPreviewController { if (!ready && this.active) this.onError(iosVoiceInput.statusText) return ready && this.session?.state.phase === 'connected' } - private async startVoiceCapture(): Promise { + private async startVoiceCapture(endpointing = false): Promise { if (!this.session || this.session.state.phase !== 'connected') return - await iosVoiceInput.startGlassesCapture(this.session, message => this.logBluetooth(message)) + await iosVoiceInput.startGlassesCapture(this.session, message => this.logBluetooth(message), endpointing) } private logBluetooth(message: string): void { const line = `${new Date().toISOString()} [${this.active ? 'foreground' : 'background'}${UIApplication.sharedApplication.protectedDataAvailable ? '' : ',protected-data-unavailable'}] ${message}` diff --git a/app/native/voice-control.ios.ts b/app/native/voice-control.ios.ts index 9aad3654..62c9df51 100644 --- a/app/native/voice-control.ios.ts +++ b/app/native/voice-control.ios.ts @@ -18,6 +18,8 @@ export class IosVoiceControlBridge { private session: IosMicrophoneSession | null = null private capturing = false private finalizing = false + private completion: Promise = Promise.resolve() + private resolveCompletion: (() => void) | null = null private audioTimer: ReturnType | null = null private packets = 0 private log: (message: string) => void = () => {} @@ -48,12 +50,13 @@ export class IosVoiceControlBridge { return true } get statusText(): string { return this.status } - async startGlassesCapture(session: IosMicrophoneSession, log: (message: string) => void): Promise { + async startGlassesCapture(session: IosMicrophoneSession, log: (message: string) => void, endpointing = false): Promise { this.stop() const generation = ++this.generation this.log = log; this.session = session; this.packets = 0; this.capturing = true - const error = String(this.ensureNative().start() ?? '') - if (error) { this.capturing = false; this.session = null; this.setStatus(error); this.emitEnd(); return } + this.completion = new Promise(resolve => { this.resolveCompletion = resolve }) + const error = String(this.ensureNative().startWithEndpointing(endpointing) ?? '') + if (error) { this.stop(); this.setStatus(error); this.emitEnd(); return } try { await session.setMicrophone(true, packet => { if (generation !== this.generation || !this.capturing) return @@ -74,16 +77,19 @@ export class IosVoiceControlBridge { this.stop(); this.setStatus(error instanceof Error ? error.message : String(error)); this.emitEnd() } } - stopPushToTalk(): void { - if (!this.capturing) return + stopPushToTalk(): Promise { + const completion = this.completion + if (!this.capturing) return completion this.capturing = false; this.finalizing = true this.clearAudioTimer(); this.releaseMicrophone() this.setStatus('Recognizing…'); this.native?.finish() + return completion } stop(): void { ++this.generation this.capturing = this.finalizing = false this.clearAudioTimer(); this.releaseMicrophone(); this.native?.cancel() + this.finishCompletion() } handleSessionEnded(): void { if (!this.capturing && !this.finalizing) return @@ -95,6 +101,9 @@ export class IosVoiceControlBridge { if (session) void session.setMicrophone(false).catch(error => this.log(`Voice microphone cleanup: ${error}`)) } private emitEnd(): void { for (const fn of [...this.ends]) fn() } + private finishCompletion(): void { + const resolve = this.resolveCompletion; this.resolveCompletion = null; resolve?.() + } private receive(event: { kind: string; text?: string; final?: boolean; message?: string; seconds?: number; packets?: number; rms?: number; missing?: number; errors?: number }): void { if (!this.capturing && !this.finalizing) return if (event.kind === 'transcript') { @@ -109,6 +118,7 @@ export class IosVoiceControlBridge { } else if (event.kind === 'ended') { this.capturing = this.finalizing = false this.clearAudioTimer(); this.releaseMicrophone() + this.finishCompletion() this.setStatus(event.message ?? 'Ready to send'); this.emitEnd() } } diff --git a/app/ui/dashboard/settings-menus.ts b/app/ui/dashboard/settings-menus.ts index 064c3965..c06aef99 100644 --- a/app/ui/dashboard/settings-menus.ts +++ b/app/ui/dashboard/settings-menus.ts @@ -130,10 +130,10 @@ function settingsSections(): SettingsSection[] { // iOS offers cloud APIs; Android also supports local models and the bridge. enumSettingMenuItem(assistantBackendSetting), enumSettingMenuItem(assistantModelSetting), - // iOS has no local LLM, wakeword event delivery, or agent bridge yet. + ...(!global.isIOS ? [localModelMenuItem()] : []), + toggleSettingMenuItem(assistantSkipConfirmationSetting), + // iOS has no local LLM or agent bridge yet. ...(!global.isIOS ? [ - localModelMenuItem(), - toggleSettingMenuItem(assistantSkipConfirmationSetting), textSettingMenuItem(assistantBridgeHostSetting), textSettingMenuItem(assistantBridgePortSetting), textSettingMenuItem(assistantBridgeTokenSetting), @@ -245,9 +245,9 @@ function settingsSections(): SettingsSection[] { const deferred = new Set(["Navigate", "Watch"]); return sections.map(section => { if (section.label === "Developer") return { ...section, items: [toggleSettingMenuItem(showBleBandwidthSetting)] }; - if (section.label === "Voice") return { label: "Voice", items: [{ + if (section.label === "Voice") return { label: "Voice", items: [enumSettingMenuItem(wakeWordActionSetting), { label: "On-device dictation (Apple)", disabled: true, onSelect: () => {}, - description: "Uses the glasses microphone and your iPhone's speech language. No API key needed. Open Voice from the phone or glasses menu, speak, then click to finish and send the text into an app.", + description: "Uses the glasses microphone and your iPhone's speech language. No transcription API key needed. Say Hey Even for hands-free input, or open Voice from the menu and click when finished.", }] }; if (deferred.has(section.label)) return { label: section.label, items: [{ label: "Not available on iOS yet", disabled: true, onSelect: () => {}, diff --git a/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/VoiceEndpointDetector.kt b/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/VoiceEndpointDetector.kt new file mode 100644 index 00000000..c94a909f --- /dev/null +++ b/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/VoiceEndpointDetector.kt @@ -0,0 +1,66 @@ +package com.faceclaw.app + +import kotlin.math.max +import kotlin.math.sqrt + +/** Ends a hands-free utterance using the 16 kHz sample clock, not BLE arrival timing. + * Shared by Android's voice controller and iOS's native audio queue. + */ +class VoiceEndpointDetector { + private var totalSamples = 0L + private var noiseAccum = 0.0 + private var noisePackets = 0 + private var threshold = 0.0 + private var speechStarted = false + private var silenceSamples = 0L + private var fired = false + + fun reset() { + totalSamples = 0 + noiseAccum = 0.0 + noisePackets = 0 + threshold = 0.0 + speechStarted = false + silenceSamples = 0 + fired = false + } + + fun accept(pcm: ShortArray, count: Int): Boolean { + if (fired || count <= 0) return false + require(count <= pcm.size) + var squares = 0.0 + for (i in 0 until count) { + val sample = pcm[i].toDouble() + squares += sample * sample + } + return acceptLevel(sqrt(squares / count), count) + } + + /** RMS in PCM16 units. Allows native decoders to reuse their existing level calculation. */ + fun acceptLevel(rms: Double, sampleCount: Int): Boolean { + if (fired || sampleCount <= 0) return false + totalSamples += sampleCount + val elapsedMs = totalSamples * 1000 / 16000 + if (elapsedMs <= 300) { + noiseAccum += rms + noisePackets++ + return false + } + if (threshold == 0.0) threshold = max(if (noisePackets > 0) noiseAccum / noisePackets else 0.0, 220.0) + if (!speechStarted) { + if (rms >= threshold * 3.0) { + speechStarted = true + silenceSamples = 0 + } else if (elapsedMs >= 6000) { + fired = true + } + return fired + } + if (rms < threshold * 1.8) { + silenceSamples += sampleCount + if (silenceSamples * 1000 / 16000 >= 900) fired = true + } else silenceSamples = 0 + if (elapsedMs >= 30000) fired = true + return fired + } +} diff --git a/native/kotlin/shared/src/iosMain/kotlin/com/faceclaw/app/IosNativeApi.kt b/native/kotlin/shared/src/iosMain/kotlin/com/faceclaw/app/IosNativeApi.kt index 2c1b364f..2d3cb4ba 100644 --- a/native/kotlin/shared/src/iosMain/kotlin/com/faceclaw/app/IosNativeApi.kt +++ b/native/kotlin/shared/src/iosMain/kotlin/com/faceclaw/app/IosNativeApi.kt @@ -128,7 +128,7 @@ class IosProtocol { BleProtocol.readStringFieldValue(data.byteArray(), field) fun glassesInput(data: NSData, sid: Int, flag: Int): G2Event? = - if (sid == BleProtocol.SID_EVENHUB && flag in intArrayOf(1, 6)) + if (sid in intArrayOf(BleProtocol.SID_EVENHUB, BleProtocol.SID_EVEN_AI) && flag in intArrayOf(1, 6)) G2Event.decodePayload(sid, data.byteArray()) else null diff --git a/tests/ai-chat.test.cjs b/tests/ai-chat.test.cjs index 1da80634..db028775 100644 --- a/tests/ai-chat.test.cjs +++ b/tests/ai-chat.test.cjs @@ -228,7 +228,7 @@ test('empty capture, rejection and closing never submit stale speech', async () } }); -function shellEnv() { +function shellEnv({ wakeAction = 'voice-input', skipConfirmation = false } = {}) { const gestures = load('app/ui/gestures.ts', {}); class Image { constructor(width = 576, height = 260) { this.width = width; this.height = height; } } const images = { GrayImage: Image, G2_LENS_WIDTH: 640, G2_LENS_HEIGHT: 480 }; @@ -244,11 +244,17 @@ function shellEnv() { paint(_ctx, below) { return below(); } handleInput(event, ctx) { if (event.type === 'double-click') ctx.stack.pop(); } } - const settings = { wakeWordActionSetting: { get: () => 'voice-input' }, brightnessSetting: { get: () => 'auto' } }; - const { shell } = load('app/ui/shell/shell.ts', { - '../../graphics/image': images, '../../graphics/plane': {}, '../../graphics/ui-fonts': {}, '../../g2/events': {}, + const voiceDialogs = []; + const settings = { wakeWordActionSetting: { get: () => wakeAction }, brightnessSetting: { get: () => 'auto' }, + assistantSkipConfirmationSetting: { get: () => skipConfirmation } }; + const { shell, rawInputEventToInputEvent } = load('app/ui/shell/shell.ts', { + '../../graphics/image': images, '../../graphics/plane': {}, '../../graphics/ui-fonts': {}, '../../g2/events': load('app/g2/events.ts', {}), '../gestures': gestures, '../layers': layers, '../menu': { MenuLayer: Menu }, - './voice-input': {}, './keyboard-input': {}, './voice-activity': {}, './assistant': {}, + './voice-input': { VoiceInputLayer: class { + constructor(options) { this.options = options; voiceDialogs.push(options); } + startCapture() {} + onRemoved() { this.options.onClosed(); } + } }, './keyboard-input': {}, './voice-activity': { voiceActivity: { setActive() {} } }, './assistant': {}, '../../assistant/conversations': {}, '../../assistant/models': {}, '../../native/settings-store': {}, '../notifications': {}, '../dashboard-settings': settings, './ambient-cards': {}, './chrome-layer': { ShellChromeLayer: class {} }, './modal-layer': {}, './tool-debug-layer': {}, @@ -280,9 +286,32 @@ function shellEnv() { }); shell.registerWindow(app.window); shell.focusWindow('ai-chat'); - return { shell, app, input, menus, microphone: () => microphone, event: (type) => gestures.makeInputEvent({ type }) }; + return { shell, app, input, menus, voiceDialogs, rawInputEventToInputEvent, microphone: () => microphone, event: (type) => gestures.makeInputEvent({ type }) }; } +test('wakeword wakes the shared shell, respects its action, and opens only one hands-free assistant dialog', async () => { + for (const wakeAction of ['off', 'turn-screen-on', 'voice-input']) { + const env = shellEnv({ wakeAction, skipConfirmation: true }); + env.shell.isAssistantAvailable = () => true; + env.shell.sleep(); + const event = env.rawInputEventToInputEvent({ kind: 'even-ai', eventType: 1, eventSource: 0 }); + assert.equal(event.type, 'wakeword'); + await env.shell.receiveInput(event); + await new Promise(resolve => setImmediate(resolve)); + assert.equal(env.shell.isScreenOn(), wakeAction !== 'off'); + assert.equal(env.voiceDialogs.length, wakeAction === 'voice-input' ? 1 : 0); + if (wakeAction === 'voice-input') { + const options = env.voiceDialogs[0]; + assert.equal(options.handsFree, true); + assert.equal(options.autoSend, true); + assert.equal(options.sendTargets[options.defaultTargetIndex].id, 'assistant'); + await env.shell.receiveInput(event); + assert.equal(env.voiceDialogs.length, 1); + } + env.shell.stack.clearToBase(); + } +}); + test('chat holds bypass the escape timer; release, menus and double tap follow the requested routing', async () => { const env = shellEnv(); await env.shell.receiveInput(env.event('long-press')); diff --git a/tests/ios-app-host.test.cjs b/tests/ios-app-host.test.cjs index 44435ca6..f67fcdcc 100644 --- a/tests/ios-app-host.test.cjs +++ b/tests/ios-app-host.test.cjs @@ -42,6 +42,73 @@ test('iOS settings prompts honor password input and report cancellation', async assert.equal(prompts[0].inputType, 'password'); assert.equal(accepted, false); assert.deepEqual(writes, []); }); + +test('iOS host forwards hands-free capture mode to the voice bridge', async () => { + const captures = []; + const { IosPreviewController } = load('app/g2/ios-preview-controller.ts', { require: id => id === '../native/ios-voice-input' + ? { iosVoiceInput: { startGlassesCapture: (_session, _log, endpointing) => captures.push(endpointing) } } : {} }); + const host = Object.create(IosPreviewController.prototype); + host.session = { state: { phase: 'connected' } }; + await host.startVoiceCapture(true); + await host.startVoiceCapture(); + assert.deepEqual(captures, [true, false]); +}); + +for (const isIOS of [true, false]) { + test(`${isIOS ? 'iOS' : 'Android'} Assistant settings expose supported controls and usable model choices`, () => { + const store = new Map(); + let picker; + const models = load('app/assistant/models.ts', { global: { isIOS }, require: () => ({ + LOCAL_MODEL: { label: 'Qwen', id: 'qwen' }, isLocalModelReady: () => false, + }) }); + const settings = load('app/ui/dashboard-settings.ts', { global: { isIOS }, require: id => { + if (id.includes('settings-store')) return { + onSettingsStoreChanged() {}, + getStringSetting: (key, fallback) => store.get(key) ?? fallback, + setStringSetting: (key, value) => store.set(key, value), + getBooleanSetting: (key, fallback) => store.get(key) ?? fallback, + setBooleanSetting: (key, value) => store.set(key, value), + }; + if (id === '~/assistant/models') return models; + return { Layer: class {}, isLocalModelReady: () => false, + openModalMenu: (_ctx, _title, items) => { picker = items; } }; + } }); + const menus = load('app/ui/dashboard/settings-menus.ts', { global: { isIOS }, require: id => { + if (id === '../dashboard-settings') return settings; + if (id === './settings-panel') return { SettingsPanelLayer: class { constructor(sections) { this.sections = sections; } } }; + return { LOCAL_MODEL: { sizeBytes: 1000 }, ASR_MODELS: { moonshine: {}, 'whisper-base-en': {} }, + uiFontPickerMenuItem: () => ({ label: 'Font' }), terminalFontPickerMenuItem: () => ({ label: 'Terminal font' }) }; + } }); + const sections = menus.createSettingsPanelLayer().sections; + const assistant = sections.find(section => section.label === 'Assistant'); + assert.equal(assistant.items.some(item => item.disabled === true), false); + const ctx = { stack: { pop() {} }, actions: { requestRender() {} } }; + const row = setting => assistant.items.find(item => item.label === setting.label); + row(settings.assistantBackendSetting).onSelect(ctx); + assert.deepEqual(Array.from(picker, item => item.label), isIOS ? ['Cloud API'] : ['On-phone', 'My own agent (bridge)']); + assert.equal(assistant.items.some(item => item.label === 'On-phone model'), !isIOS); + assert.equal(!!row(settings.assistantBridgeHostSetting), !isIOS); + assert.equal(!!row(settings.assistantAllowProactiveSetting), !isIOS); + row(settings.assistantModelSetting).onSelect(ctx); + assert.equal(picker.some(item => item.label.includes('Qwen')), !isIOS); + const terra = picker.find(item => item.label === 'Terra'); + assert.equal(terra.disabled(), true); + const keys = sections.find(section => section.label === 'API Keys').items; + assert.ok(keys.some(item => item.label === settings.openAiApiKeySetting.label)); + assert.ok(keys.some(item => item.label === settings.anthropicApiKeySetting.label)); + settings.openAiApiKeySetting.set('fixture-key'); + assert.equal(terra.disabled(), false); + terra.onSelect(ctx); + assert.equal(settings.assistantModelSetting.get(), 'terra'); + assert.equal(models.resolveAssistantModel(settings.assistantModelSetting.get(), { + openai: settings.openAiApiKeySetting.get(), anthropic: '', + }).provider, 'openai'); + row(settings.assistantSkipConfirmationSetting).onSelect(ctx); + assert.equal(settings.assistantSkipConfirmationSetting.get(), true); + assert.ok(sections.find(section => section.label === 'Voice').items.some(item => item.label === settings.wakeWordActionSetting.label)); + }); +} + test('iOS phone battery distinguishes unknown, charging, full and unplugged readings', () => { const device = { batteryLevel: -1, batteryState: 0 }; const api = load('app/native/phone-battery.ts', { require: () => ({}), global: { isIOS: true }, diff --git a/tests/ios-svg-icons.test.cjs b/tests/ios-svg-icons.test.cjs new file mode 100644 index 00000000..65ee588a --- /dev/null +++ b/tests/ios-svg-icons.test.cjs @@ -0,0 +1,29 @@ +const test = require('node:test'); +const assert = require('node:assert/strict'); +const { loader } = require('./helpers/load-typescript.cjs'); + +function renderSource(svg) { + let source; + const { rasterizeSvg } = loader({}, { './ios-graphics': { + rasterizeSvg: (value, size) => { source = value; assert.equal(size, 32); return 'rendered'; }, + } })('app/native/svg-rasterizer.ios.ts'); + assert.equal(rasterizeSvg(svg, 32, 2), 'rendered'); + return source; +} + +test('EvenHub currentColor SVG fills are resolved before handing them to SVGKit', () => { + // Same document shape as the Snake and Weather store icons copied from the phone. + const source = renderSource(''); + assert.match(source, /fill="#ffffff"/); + assert.doesNotMatch(source, /currentColor/i); + assert.equal((source.match(/fill=/g) || []).length, 1); + assert.match(source, /viewBox="0 0 24 24"/); +}); + +test('currentColor strokes/styles resolve without filling open Lucide shapes', () => { + const source = renderSource(''); + assert.match(source, /fill="none"/); + assert.match(source, /stroke="#ffffff"/); + assert.match(source, /style="stroke:#ffffff"/); + assert.match(source, /stroke-linecap="round"/); +}); diff --git a/tests/ios-voice.test.cjs b/tests/ios-voice.test.cjs index ec29c011..5e3a924e 100644 --- a/tests/ios-voice.test.cjs +++ b/tests/ios-voice.test.cjs @@ -4,7 +4,7 @@ const fs = require('node:fs'); const vm = require('node:vm'); const ts = require('typescript'); function harness() { - const native = { start: () => '', cancel() { this.cancels++; }, cancels: 0, finish() { this.finished = true; }, acceptPacket: p => audio.push(p) }; + const native = { startWithEndpointing(endpointing) { this.endpointing = endpointing; return ''; }, cancel() { this.cancels++; }, cancels: 0, finish() { this.finished = true; }, acceptPacket: p => audio.push(p) }; const audio = [], timers = new Map(), commands = [], text = []; let authorization = 3, permissionRequests = 0, release; const sandbox = { exports: {}, FaceclawSpeech: { new: () => native, authorizationStatus: () => authorization, @@ -61,3 +61,23 @@ test('missing microphone data ends the session and disables the mic with an acti for (const fn of h.timers.values()) fn(); assert.deepEqual(h.commands, [true, false]); assert.match(h.bridge.statusText, /No microphone audio/); }); + +test('hands-free endpointing releases the microphone and waits for the final transcript before completing', async () => { + const h = harness(); let ends = 0, completed = false; + h.bridge.onSpeechEnd(() => ends++); + const start = h.bridge.startGlassesCapture(h.session, () => {}, true); h.release(); await start; + assert.equal(h.native.endpointing, true); + h.event({ kind: 'transcript', text: 'Set a timer', final: false }); + h.event({ kind: 'finishing' }); + const completion = h.bridge.stopPushToTalk().then(() => { completed = true; }); + await Promise.resolve(); + assert.equal(completed, false); assert.equal(ends, 1); + assert.deepEqual(h.commands, [true, false]); + h.event({ kind: 'transcript', text: 'Set a timer for five minutes.', final: true }); + h.event({ kind: 'ended' }); await completion; + assert.equal(completed, true); + assert.equal(h.text.at(-1).text, 'Set a timer for five minutes.'); + const manual = h.bridge.startGlassesCapture(h.session, () => {}); h.release(); await manual; + assert.equal(h.native.endpointing, false); + const cancelled = h.bridge.stopPushToTalk(); h.bridge.stop(); await cancelled; +}); diff --git a/tests/kotlin/src/commonTest/kotlin/com/faceclaw/app/VoiceEndpointDetectorTest.kt b/tests/kotlin/src/commonTest/kotlin/com/faceclaw/app/VoiceEndpointDetectorTest.kt new file mode 100644 index 00000000..845cf711 --- /dev/null +++ b/tests/kotlin/src/commonTest/kotlin/com/faceclaw/app/VoiceEndpointDetectorTest.kt @@ -0,0 +1,42 @@ +package com.faceclaw.app + +import kotlin.test.* + +class VoiceEndpointDetectorTest { + @Test fun silenceEndsOnlyAfterSpeechAndFiresOnce() { + val detector = VoiceEndpointDetector() + repeat(6) { assertFalse(detector.acceptLevel(100.0, 800)) } + repeat(10) { assertFalse(detector.accept(ShortArray(800) { 2000 }, 800)) } + repeat(17) { assertFalse(detector.acceptLevel(100.0, 800)) } + assertTrue(detector.acceptLevel(100.0, 800)) + assertFalse(detector.acceptLevel(100.0, 800)) + detector.reset() + assertFalse(detector.acceptLevel(100.0, 800)) + } + + @Test fun pausesResetWhenSpeechResumesAndContinuousSpeechIsBounded() { + val detector = VoiceEndpointDetector() + repeat(6) { assertFalse(detector.acceptLevel(0.0, 800)) } + assertFalse(detector.acceptLevel(2000.0, 800)) + repeat(17) { assertFalse(detector.acceptLevel(0.0, 800)) } + assertFalse(detector.acceptLevel(2000.0, 800)) + repeat(17) { assertFalse(detector.acceptLevel(0.0, 800)) } + assertTrue(detector.acceptLevel(0.0, 800)) + detector.reset() + repeat(6) { assertFalse(detector.acceptLevel(0.0, 800)) } + repeat(593) { assertFalse(detector.acceptLevel(2000.0, 800)) } + assertTrue(detector.acceptLevel(2000.0, 800)) + } + + @Test fun quietRoomTimesOutOnSampleClockAndNoiseCalibrationIsRelative() { + val detector = VoiceEndpointDetector() + repeat(119) { assertFalse(detector.acceptLevel(0.0, 800)) } + assertTrue(detector.acceptLevel(0.0, 800)) + detector.reset() + repeat(6) { assertFalse(detector.acceptLevel(1000.0, 800)) } + repeat(20) { assertFalse(detector.acceptLevel(2000.0, 800)) } + assertFalse(detector.acceptLevel(4000.0, 800)) + repeat(17) { assertFalse(detector.acceptLevel(1000.0, 800)) } + assertTrue(detector.acceptLevel(1000.0, 800)) + } +} diff --git a/tests/kotlin/src/iosTest/kotlin/com/faceclaw/app/IosProtocolTest.kt b/tests/kotlin/src/iosTest/kotlin/com/faceclaw/app/IosProtocolTest.kt index 0afc74d4..436058b0 100644 --- a/tests/kotlin/src/iosTest/kotlin/com/faceclaw/app/IosProtocolTest.kt +++ b/tests/kotlin/src/iosTest/kotlin/com/faceclaw/app/IosProtocolTest.kt @@ -47,6 +47,22 @@ internal actual fun inflateRecords(records: List): List = } class IosProtocolTest { + @Test + fun wakewordPacketsReachTheSharedEventDecoder() { + val api = IosProtocol() + val ctrl = BleProtocol.encodeVarintField(1, 1) // EVEN_AI_WAKE_UP + val payload = (BleProtocol.encodeVarintField(1, BleProtocol.EVEN_AI_CMD_CTRL) + + BleProtocol.encodeBytesField(3, ctrl)).data() + for (flag in listOf(1, 6)) { + val event = assertNotNull(api.glassesInput(payload, BleProtocol.SID_EVEN_AI, flag)) + assertEquals("even-ai", event.kind) + assertEquals(1, event.eventType) + } + assertNull(api.glassesInput(payload, BleProtocol.SID_EVEN_AI, 2)) + assertNull(api.glassesInput(payload, 0xff, 1)) + assertNull(api.glassesInput(byteArrayOf().data(), BleProtocol.SID_EVEN_AI, 1)) + } + @Test fun platformClockAndWriteModes() { val before = IosProtocolPlatform.elapsedRealtimeMs() diff --git a/tests/voice-finalization.test.cjs b/tests/voice-finalization.test.cjs index ce879820..c656a94f 100644 --- a/tests/voice-finalization.test.cjs +++ b/tests/voice-finalization.test.cjs @@ -13,27 +13,29 @@ function load(file, deps, globals = {}) { return exports; } -function dialog({ autoSend = false, deferred = true, synchronousFinal } = {}) { +function dialog({ autoSend = false, deferred = true, synchronousFinal, isIOS = false, handsFree = false } = {}) { const listeners = new Set(), timers = new Map(), sent = [], refinements = []; - let complete, timerId = 0, starts = 0, frame; + let complete, timerId = 0, starts = 0, frame, speechEnd, endpointing; let completion = new Promise(resolve => { complete = resolve; }); const { VoiceInputLayer } = load('app/ui/shell/voice-input.ts', { '../../native/voice-control': { voiceControlBridge: { onTranscript(cb) { listeners.add(cb); return () => listeners.delete(cb); }, onStatus() { return () => {}; }, + onSpeechEnd(cb) { speechEnd = cb; return () => { speechEnd = null; }; }, + stop() {}, } }, '../../native/anthropic': { refineDictation(options) { refinements.push(options); return { cancel() {} }; } }, '../dashboard-settings': { anthropicApiKeySetting: { get: () => 'key' } }, '../gestures': { gestureHints: () => '' }, './input-dialog': { paintInputDialog(_image, options) { frame = options; } }, }, { - global: { isIOS: false }, + global: { isIOS }, setTimeout(cb) { const id = ++timerId; timers.set(id, cb); return id; }, clearTimeout(id) { timers.delete(id); }, }); const transcript = (text, isFinal = false) => { for (const cb of [...listeners]) cb({ text, isFinal }); }; - const layer = new VoiceInputLayer({ autoSend, onClosed() {}, dismiss: () => layer.onRemoved(), - actions: { requestRender() {}, startVoiceCapture() { starts++; }, stopVoiceCapture() { + const layer = new VoiceInputLayer({ autoSend, handsFree, onClosed() {}, dismiss: () => layer.onRemoved(), + actions: { requestRender() {}, startVoiceCapture(value) { starts++; endpointing = value; }, stopVoiceCapture() { if (synchronousFinal) transcript(synchronousFinal, true); return deferred ? completion : undefined; } }, @@ -41,6 +43,7 @@ function dialog({ autoSend = false, deferred = true, synchronousFinal } = {}) { }); layer.startCapture(); return { layer, sent, refinements, transcript, starts: () => starts, + endpoint: () => speechEnd?.(), endpointing: () => endpointing, input: type => layer.handleInput({ type }, {}), frame() { layer.paint({}, () => ({})); return frame; }, timeout() { for (const [id, cb] of [...timers]) { timers.delete(id); cb(); } }, @@ -50,6 +53,26 @@ function dialog({ autoSend = false, deferred = true, synchronousFinal } = {}) { }; } +for (const autoSend of [true, false]) { + test(`iOS wakeword silence completion ${autoSend ? 'auto-sends' : 'asks for confirmation'} only after final recognition`, async () => { + const env = dialog({ isIOS: true, handsFree: true, autoSend }); + assert.equal(env.endpointing(), true); + env.transcript('Set a timer'); + env.endpoint(); env.timeout(); + env.input('click'); // Cannot send a partial while native recognition drains. + assert.deepEqual(env.sent, []); + env.transcript('Set a timer for five minutes.', true); + await env.complete(); env.timeout(); + if (!autoSend) { + assert.deepEqual(env.sent, []); + env.input('click'); + } + assert.deepEqual(env.sent, ['Set a timer for five minutes.']); + env.endpoint(); env.timeout(); + assert.equal(env.sent.length, 1); + }); +} + for (const partial of ['', 'Set a timer']) { test(`auto-send waits for slow native recognition with ${partial ? 'a committed segment' : 'no partial'}`, async () => { const env = dialog({ autoSend: true });