diff --git a/App_Resources/iOS/src/FaceclawSocket.h b/App_Resources/iOS/src/FaceclawSocket.h index 84a058c5..8c24e0a1 100644 --- a/App_Resources/iOS/src/FaceclawSocket.h +++ b/App_Resources/iOS/src/FaceclawSocket.h @@ -2,7 +2,10 @@ /** Thread-safe event queue; each NativeScript worker drains it on its own thread. */ @interface FaceclawSocket : NSObject - (instancetype)initWithURL:(NSString *)url; +/** headerName/headerValue add one handshake header (e.g. an API key); nil for none. */ +- (instancetype)initWithURL:(NSString *)url headerName:(NSString *)headerName headerValue:(NSString *)headerValue; - (NSString *)takeEvents; - (void)sendText:(NSString *)text; +- (void)sendData:(NSData *)data; - (void)close; @end diff --git a/App_Resources/iOS/src/FaceclawSocket.m b/App_Resources/iOS/src/FaceclawSocket.m index a53e6e6b..701e7a36 100644 --- a/App_Resources/iOS/src/FaceclawSocket.m +++ b/App_Resources/iOS/src/FaceclawSocket.m @@ -8,12 +8,17 @@ @end @implementation FaceclawSocket - (instancetype)initWithURL:(NSString *)url { + return [self initWithURL:url headerName:nil headerValue:nil]; +} +- (instancetype)initWithURL:(NSString *)url headerName:(NSString *)headerName headerValue:(NSString *)headerValue { if ((self = [super init])) { _events = [NSMutableArray array]; NSURLSessionConfiguration *config = [NSURLSessionConfiguration ephemeralSessionConfiguration]; config.timeoutIntervalForRequest = 20; _session = [NSURLSession sessionWithConfiguration:config delegate:self delegateQueue:nil]; - _task = [_session webSocketTaskWithURL:[NSURL URLWithString:url]]; + NSMutableURLRequest *request = [NSMutableURLRequest requestWithURL:[NSURL URLWithString:url]]; + if (headerName.length && headerValue) [request setValue:headerValue forHTTPHeaderField:headerName]; + _task = [_session webSocketTaskWithRequest:request]; _task.maximumMessageSize = 8 * 1024 * 1024; [_task resume]; } @@ -25,7 +30,7 @@ _queuedBytes += [event[@"text"] lengthOfBytesUsingEncoding:NSUTF8StringEncoding]; if (_events.count >= 256 || _queuedBytes > 16 * 1024 * 1024) { [_events removeAllObjects]; _queuedBytes = 0; - [_events addObject:@{@"kind": @"error", @"text": @"Terminal input queue overflow"}]; + [_events addObject:@{@"kind": @"error", @"text": @"WebSocket receive queue overflow"}]; [self close]; } else [_events addObject:event]; } @@ -56,7 +61,13 @@ [self close]; } - (void)URLSession:(NSURLSession *)session task:(NSURLSessionTask *)task didCompleteWithError:(NSError *)error { - if (error) [self enqueue:@{@"kind": @"error", @"text": error.localizedDescription}]; + if (error) { + // A refused handshake only says "bad response"; name the HTTP status + // so callers can tell a rejected key (401/403) from a network error. + NSInteger status = [task.response isKindOfClass:NSHTTPURLResponse.class] ? ((NSHTTPURLResponse *)task.response).statusCode : 0; + NSString *text = status >= 400 ? [NSString stringWithFormat:@"HTTP %ld: %@", (long)status, error.localizedDescription] : error.localizedDescription; + [self enqueue:@{@"kind": @"error", @"text": text}]; + } [self close]; } - (void)sendText:(NSString *)text { @@ -65,6 +76,12 @@ if (error) { [weakSelf enqueue:@{@"kind": @"error", @"text": error.localizedDescription}]; [weakSelf close]; } }]; } +- (void)sendData:(NSData *)data { + __weak FaceclawSocket *weakSelf = self; + [_task sendMessage:[[NSURLSessionWebSocketMessage alloc] initWithData:data] completionHandler:^(NSError *error) { + if (error) { [weakSelf enqueue:@{@"kind": @"error", @"text": error.localizedDescription}]; [weakSelf close]; } + }]; +} - (void)close { @synchronized(self) { if (_ended) return; _ended = YES; diff --git a/App_Resources/iOS/src/FaceclawSpeech.h b/App_Resources/iOS/src/FaceclawSpeech.h index 8f647292..2a29b17b 100644 --- a/App_Resources/iOS/src/FaceclawSpeech.h +++ b/App_Resources/iOS/src/FaceclawSpeech.h @@ -2,6 +2,8 @@ /** On-device speech recognition fed by G2 BLE audio or the default phone microphone. */ @interface FaceclawSpeech : NSObject @property(nonatomic, copy) void (^eventHandler)(NSString *json); +/** 16 kHz mono PCM16LE chunks from a PCM capture, on the main queue in capture order. */ +@property(nonatomic, copy) void (^pcmHandler)(NSData *pcm); + (NSInteger)authorizationStatus; + (NSInteger)microphoneAuthorizationStatus; + (void)requestMicrophoneAuthorization:(void (^)(NSInteger status))completion; @@ -9,6 +11,9 @@ /** Returns an actionable error, or an empty string when capture starts. */ - (NSString *)startWithEndpointing:(BOOL)endpointing; - (NSString *)startPhoneWithEndpointing:(BOOL)endpointing; +/** Capture without the recognizer (for a cloud provider): audio goes to + * pcmHandler, and "ended" follows the last chunk. No transcript events. */ +- (NSString *)startPcmCaptureWithEndpointing:(BOOL)endpointing phone:(BOOL)phone; - (void)acceptPacket:(NSData *)packet; - (void)finish; - (void)cancel; diff --git a/App_Resources/iOS/src/FaceclawSpeech.m b/App_Resources/iOS/src/FaceclawSpeech.m index 5487011e..a93b9676 100644 --- a/App_Resources/iOS/src/FaceclawSpeech.m +++ b/App_Resources/iOS/src/FaceclawSpeech.m @@ -15,6 +15,7 @@ @property(atomic) NSUInteger generation; @property(nonatomic) NSUInteger pendingPackets; @property(atomic) BOOL accepting; +@property(nonatomic) BOOL pcmOutput; @property(nonatomic, strong) AVAudioEngine *phoneEngine; @property(nonatomic) BOOL phoneTapInstalled; @property(nonatomic) BOOL phoneSessionActive; @@ -23,6 +24,23 @@ @property(nonatomic, strong) FaceclawSpeechTranscript *transcript; @property(nonatomic, strong) FaceclawKitVoiceEndpointDetector *endpointDetector; @end +/** Resamples one tap buffer to 16 kHz mono PCM16LE. Called on the serial + * audio queue; the converter keeps its filter state between buffers. */ +static NSData *resampleToPcm(AVAudioConverter *converter, AVAudioPCMBuffer *input) { + AVAudioFormat *format = converter.outputFormat; + AVAudioFrameCount capacity = (AVAudioFrameCount)ceil(input.frameLength * format.sampleRate / input.format.sampleRate) + 32; + AVAudioPCMBuffer *output = [[AVAudioPCMBuffer alloc] initWithPCMFormat:format frameCapacity:capacity]; + __block BOOL supplied = NO; + NSError *error = nil; + AVAudioConverterOutputStatus status = [converter convertToBuffer:output error:&error + withInputFromBlock:^AVAudioBuffer *(AVAudioPacketCount count, AVAudioConverterInputStatus *inputStatus) { + // NoDataNow (not EndOfStream) leaves the stream open for the next buffer. + if (supplied) { *inputStatus = AVAudioConverterInputStatus_NoDataNow; return nil; } + supplied = YES; *inputStatus = AVAudioConverterInputStatus_HaveData; return input; + }]; + if (status == AVAudioConverterOutputStatus_Error || !output.frameLength) return nil; + return [NSData dataWithBytes:output.int16ChannelData[0] length:output.frameLength * sizeof(int16_t)]; +} @implementation FaceclawSpeech - (instancetype)init { if ((self = [super init])) _audioQueue = dispatch_queue_create("com.faceclaw.glasses-speech", DISPATCH_QUEUE_SERIAL); @@ -46,37 +64,51 @@ if (json) self.eventHandler([[NSString alloc] initWithData:json encoding:NSUTF8StringEncoding]); } - (NSString *)startWithEndpointing:(BOOL)endpointing { - return [self startWithEndpointing:endpointing phone:NO]; + return [self startWithEndpointing:endpointing phone:NO pcm:NO]; } - (NSString *)startPhoneWithEndpointing:(BOOL)endpointing { - return [self startWithEndpointing:endpointing phone:YES]; + return [self startWithEndpointing:endpointing phone:YES pcm:NO]; } -- (NSString *)startWithEndpointing:(BOOL)endpointing phone:(BOOL)phone { +- (NSString *)startPcmCaptureWithEndpointing:(BOOL)endpointing phone:(BOOL)phone { + return [self startWithEndpointing:endpointing phone:phone pcm:YES]; +} +- (NSString *)startWithEndpointing:(BOOL)endpointing phone:(BOOL)phone pcm:(BOOL)pcm { [self cancel]; if (phone && [FaceclawSpeech microphoneAuthorizationStatus] != AVAuthorizationStatusAuthorized) return @"Allow Microphone access for Faceclaw in iPhone Settings."; - if (SFSpeechRecognizer.authorizationStatus != SFSpeechRecognizerAuthorizationStatusAuthorized) - return @"Allow Speech Recognition for Faceclaw in iPhone Settings."; NSString *language = NSLocale.preferredLanguages.firstObject ?: @"en-US"; - self.recognizer = [[SFSpeechRecognizer alloc] initWithLocale:[NSLocale localeWithLocaleIdentifier:language]]; - if (!self.recognizer.supportsOnDeviceRecognition) - return [NSString stringWithFormat:@"On-device speech is unavailable for %@. Enable Dictation for that language in iPhone Settings → General → Keyboard, then retry.", language]; - if (!self.recognizer.available) return @"Speech recognition is temporarily unavailable. Try again shortly."; + if (!pcm) { + if (SFSpeechRecognizer.authorizationStatus != SFSpeechRecognizerAuthorizationStatusAuthorized) + return @"Allow Speech Recognition for Faceclaw in iPhone Settings."; + self.recognizer = [[SFSpeechRecognizer alloc] initWithLocale:[NSLocale localeWithLocaleIdentifier:language]]; + if (!self.recognizer.supportsOnDeviceRecognition) + return [NSString stringWithFormat:@"On-device speech is unavailable for %@. Enable Dictation for that language in iPhone Settings → General → Keyboard, then retry.", language]; + if (!self.recognizer.available) return @"Speech recognition is temporarily unavailable. Try again shortly."; + } if (!phone) { self.decoder = [FaceclawLc3Decoder new]; if (!self.decoder) return @"Could not initialize the glasses audio decoder."; } + self.pcmOutput = pcm; + self.bestText = @""; self.accepting = YES; + self.endpointDetector = endpointing ? [FaceclawKitVoiceEndpointDetector new] : nil; + NSUInteger generation = self.generation; + __weak FaceclawSpeech *weakSelf = self; + if (pcm) { + if (phone) { + NSString *error = [self startPhoneAudio]; + if (error.length) { [self cancel]; return error; } + } + // The PCM consumer reports its own status and has no session limit. + return @""; + } self.recognizer.queue = NSOperationQueue.mainQueue; self.request = [SFSpeechAudioBufferRecognitionRequest new]; self.request.requiresOnDeviceRecognition = YES; self.request.shouldReportPartialResults = YES; self.request.taskHint = SFSpeechRecognitionTaskHintDictation; if (@available(iOS 16.0, *)) self.request.addsPunctuation = YES; - self.bestText = @""; self.accepting = YES; self.transcript = [FaceclawSpeechTranscript new]; - self.endpointDetector = endpointing ? [FaceclawKitVoiceEndpointDetector new] : nil; - NSUInteger generation = self.generation; - __weak FaceclawSpeech *weakSelf = self; self.task = [self.recognizer recognitionTaskWithRequest:self.request resultHandler:^(SFSpeechRecognitionResult *result, NSError *error) { dispatch_async(dispatch_get_main_queue(), ^{ FaceclawSpeech *owner = weakSelf; @@ -124,6 +156,13 @@ AVAudioInputNode *input = self.phoneEngine.inputNode; AVAudioFormat *format = [input outputFormatForBus:0]; if (format.sampleRate <= 0 || !format.channelCount) return @"No phone microphone is available. Check your audio input and try again."; + AVAudioConverter *converter = nil; + if (self.pcmOutput) { + AVAudioFormat *pcmFormat = [[AVAudioFormat alloc] initWithCommonFormat:AVAudioPCMFormatInt16 sampleRate:16000 channels:1 interleaved:YES]; + converter = [[AVAudioConverter alloc] initFromFormat:format toFormat:pcmFormat]; + if (!converter) return @"Could not convert phone microphone audio."; + converter.downmix = YES; + } NSUInteger generation = self.generation; SFSpeechAudioBufferRecognitionRequest *request = self.request; FaceclawKitVoiceEndpointDetector *detector = self.endpointDetector; @@ -141,7 +180,10 @@ memcpy(destination->mBuffers[i].mData, source->mBuffers[i].mData, source->mBuffers[i].mDataByteSize); dispatch_async(owner.audioQueue, ^{ @autoreleasepool { if (owner.generation != generation) return; - [request appendAudioPCMBuffer:copy]; + if (converter) { + NSData *pcm = resampleToPcm(converter, copy); + if (pcm.length) [owner deliverPcm:pcm generation:generation]; + } else [request appendAudioPCMBuffer:copy]; double sum = 0; if (copy.floatChannelData) { const float *samples = copy.floatChannelData[0]; @@ -188,6 +230,7 @@ FaceclawLc3Decoder *decoder = self.decoder; SFSpeechAudioBufferRecognitionRequest *request = self.request; FaceclawKitVoiceEndpointDetector *endpointDetector = self.endpointDetector; + BOOL pcmOutput = self.pcmOutput; NSData *copy = [packet copy]; dispatch_async(self.audioQueue, ^{ @autoreleasepool { if (self.generation != generation) return; @@ -195,17 +238,26 @@ double rms = 0; BOOL speechEnded = NO; if (pcm.length) { - AVAudioFormat *format = [[AVAudioFormat alloc] initWithCommonFormat:AVAudioPCMFormatFloat32 sampleRate:16000 channels:1 interleaved:NO]; - AVAudioPCMBuffer *buffer = [[AVAudioPCMBuffer alloc] initWithPCMFormat:format frameCapacity:(AVAudioFrameCount)(pcm.length / 2)]; - buffer.frameLength = buffer.frameCapacity; - const int16_t *samples = pcm.bytes; - for (NSUInteger i = 0; i < buffer.frameLength; i++) { - float value = samples[i] / 32768.0f; - buffer.floatChannelData[0][i] = value; rms += value * value; + AVAudioFrameCount frames = (AVAudioFrameCount)(pcm.length / 2); + AVAudioPCMBuffer *buffer = nil; + if (!pcmOutput) { + AVAudioFormat *format = [[AVAudioFormat alloc] initWithCommonFormat:AVAudioPCMFormatFloat32 sampleRate:16000 channels:1 interleaved:NO]; + buffer = [[AVAudioPCMBuffer alloc] initWithPCMFormat:format frameCapacity:frames]; + buffer.frameLength = frames; } - rms = sqrt(rms / buffer.frameLength); - if (self.generation == generation) [request appendAudioPCMBuffer:buffer]; - speechEnded = [endpointDetector acceptLevelRms:rms * 32768.0 sampleCount:(int32_t)buffer.frameLength]; + const int16_t *samples = pcm.bytes; + for (NSUInteger i = 0; i < frames; i++) { + float value = samples[i] / 32768.0f; + if (buffer) buffer.floatChannelData[0][i] = value; + rms += value * value; + } + rms = sqrt(rms / frames); + if (self.generation == generation) { + // The decoder's output is already the PCM consumer's format. + if (pcmOutput) [self deliverPcm:pcm generation:generation]; + else [request appendAudioPCMBuffer:buffer]; + } + speechEnded = [endpointDetector acceptLevelRms:rms * 32768.0 sampleCount:(int32_t)frames]; } NSUInteger packets = decoder.packets, errors = decoder.errors, missing = decoder.missing; dispatch_async(dispatch_get_main_queue(), ^{ @@ -223,9 +275,15 @@ [self stopPhoneAudio]; NSUInteger generation = self.generation; SFSpeechAudioBufferRecognitionRequest *request = self.request; + BOOL pcmOutput = self.pcmOutput; [self emit:@{@"kind":@"finishing"}]; // Drain already received audio before marking the utterance complete. - dispatch_async(self.audioQueue, ^{ if (self.generation == generation) [request endAudio]; }); + dispatch_async(self.audioQueue, ^{ + if (self.generation != generation) return; + if (!pcmOutput) { [request endAudio]; return; } + // Drained chunks are already queued for pcmHandler; "ended" follows them. + dispatch_async(dispatch_get_main_queue(), ^{ if (self.generation == generation) [self complete:@"Ready to send"]; }); + }); __weak FaceclawSpeech *weakSelf = self; dispatch_after(dispatch_time(DISPATCH_TIME_NOW, 8 * NSEC_PER_SEC), dispatch_get_main_queue(), ^{ FaceclawSpeech *owner = weakSelf; @@ -239,9 +297,14 @@ if (text.length) [self emit:@{@"kind":@"transcript", @"text":text, @"final":@YES}]; if (self.generation == generation) [self emit:@{@"kind":@"ended", @"message":status}]; } +- (void)deliverPcm:(NSData *)pcm generation:(NSUInteger)generation { + dispatch_async(dispatch_get_main_queue(), ^{ + if (self.generation == generation && self.pcmHandler) self.pcmHandler(pcm); + }); +} - (void)cancel { self.generation++; - self.accepting = NO; self.pendingPackets = 0; + self.accepting = NO; self.pendingPackets = 0; self.pcmOutput = NO; [self stopPhoneAudio]; [self.task cancel]; self.task = nil; self.request = nil; self.decoder = nil; self.recognizer = nil; self.bestText = @""; self.transcript = nil; self.endpointDetector = nil; diff --git a/CHANGELOG b/CHANGELOG index 9456acdb..ff3bcc1d 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -8,6 +8,7 @@ * Disclose Apple's separate TestFlight reporting in iOS onboarding and privacy settings * Add an on-glasses Onboarding app, which prompts you to configure some things * iOS Notifications have an icon based on ANCS category +* iOS voice input can use the ElevenLabs, OpenAI, and Soniox cloud transcription providers (Settings>Voice>Transcription Provider) * Weather app is more graphical, and can appear on the status bar and glanceboard Glanceboard system card ## 0.8.2 - 2026-10-03 diff --git a/app/g2/ios-preview-controller.ts b/app/g2/ios-preview-controller.ts index 699414e5..cd20065f 100644 --- a/app/g2/ios-preview-controller.ts +++ b/app/g2/ios-preview-controller.ts @@ -46,7 +46,9 @@ import { shell, rawInputEventToInputEvent, type ShellWindow } from '../ui/shell/ import { appViewportRect, isOnSwitcherEdge, sidebarStripVisible } from '../ui/shell/geometry' import { onAnySettingChanged, previewColorSetting, lockScreenEnabledSetting, getBrightnessPreferences, getStringSettingById, - nightscoutSiteUrlSetting, nightscoutApiTokenSetting } from '../ui/dashboard-settings' + nightscoutSiteUrlSetting, nightscoutApiTokenSetting, voiceProviderSetting, elevenLabsApiKeySetting, + openAiApiKeySetting, sonioxApiKeySetting } from '../ui/dashboard-settings' +import type { VoiceProviderSelection } from '../native/cloud-stt-provider' import type { PhoneGesture } from '../phone-ui/phone-gestures' import { claimGlassesOnboardingAutoLaunch, ONBOARDING_APP_ID } from '../apps/onboarding/onboarding-progress' @@ -885,7 +887,7 @@ export class IosPreviewController { if (this.glassesLocked) return false const usePhoneMic = this.state.phase !== 'connected' if (usePhoneMic && !this.active) return false - const ready = await iosVoiceInput.prepare(this.active, usePhoneMic) + const ready = await iosVoiceInput.prepare(this.active, usePhoneMic, this.voiceProvider()) if (!ready && this.active) this.onError(iosVoiceInput.statusText) return ready && !this.glassesLocked && (this.state.phase === 'connected' || this.active) } @@ -893,14 +895,20 @@ export class IosPreviewController { if (this.glassesLocked) return const log = (message: string) => this.logBluetooth(message) const communicator = this.communicator + const voice = this.voiceProvider() if (communicator && this.state.phase === 'connected') { - await iosVoiceInput.startGlassesCapture(communicator, log, endpointing) + await iosVoiceInput.startGlassesCapture(communicator, voice, log, endpointing) } else if (this.active) { // Recheck microphone permission if the glasses disconnected after prepare. - if (!await iosVoiceInput.prepare(this.active, true) || !this.active || this.glassesLocked) return - await iosVoiceInput.startPhoneCapture(log, endpointing) + if (!await iosVoiceInput.prepare(this.active, true, voice) || !this.active || this.glassesLocked) return + await iosVoiceInput.startPhoneCapture(voice, log, endpointing) } } + /** The transcription provider setting and the keys the cloud ones need. */ + private voiceProvider(): VoiceProviderSelection { + return { provider: voiceProviderSetting.get(), elevenLabsApiKey: elevenLabsApiKeySetting.get(), + openAiApiKey: openAiApiKeySetting.get(), sonioxApiKey: sonioxApiKeySetting.get() } + } private logBluetooth(message: string): void { const line = `${new Date().toISOString()} [${this.active ? 'foreground' : 'background'}${UIApplication.sharedApplication.protectedDataAvailable ? '' : ',protected-data-unavailable'}] ${message}` console.log(`[ios-ble] ${line}`); this.logLines.push(line) diff --git a/app/native/cloud-stt-provider.ts b/app/native/cloud-stt-provider.ts new file mode 100644 index 00000000..c844714b --- /dev/null +++ b/app/native/cloud-stt-provider.ts @@ -0,0 +1,68 @@ +import type { CloudSttClient, CloudSttOptions } from "./cloud-stt"; +import { ElevenLabsSttClient } from "./elevenlabs-stt"; +import { OpenAiRealtimeSttClient } from "./openai-stt"; +import { ReconnectingSttClient } from "./reconnecting-stt"; +import { SonioxSttClient } from "./soniox-stt"; +import type { VoiceProviderKind } from "./voice-control"; + +/** + * The transcription-provider setting plus the keys the cloud providers need. + * Shared by the Android and iOS voice bridges. + */ +export type VoiceProviderSelection = { + provider: VoiceProviderKind; + elevenLabsApiKey: string; + openAiApiKey: string; + sonioxApiKey: string; +}; + +type CloudProvider = { + name: string; + apiKey: (selection: VoiceProviderSelection) => string; + create: (options: CloudSttOptions) => CloudSttClient; +}; + +// Providers absent here ("onboard", "onboard-whisper") transcribe on-device. +const CLOUD_PROVIDERS: Partial> = { + elevenlabs: { + name: "ElevenLabs", + apiKey: (selection) => selection.elevenLabsApiKey, + create: (options) => new ElevenLabsSttClient(options), + }, + soniox: { + name: "Soniox", + apiKey: (selection) => selection.sonioxApiKey, + create: (options) => new SonioxSttClient(options), + }, + whisper: { + name: "OpenAI", + apiKey: (selection) => selection.openAiApiKey, + create: (options) => new OpenAiRealtimeSttClient(options), + }, +}; + +/** Whether createCloudSttClient would return a cloud client for this selection. */ +export function usesCloudStt(selection: VoiceProviderSelection): boolean { + return Boolean(CLOUD_PROVIDERS[selection.provider]?.apiKey(selection).trim()); +} + +/** + * The cloud provider for a capture, or null to transcribe on-device. A cloud + * provider whose API key is missing falls back to on-device (reported via + * onStatus) rather than failing the capture outright. `continuous` says + * whether the capture is open-ended (Transcribe), where pauses commit segments. + */ +export function createCloudSttClient( + selection: VoiceProviderSelection, + options: Omit, + continuous: () => boolean, +): CloudSttClient | null { + const provider = CLOUD_PROVIDERS[selection.provider]; + if (!provider) return null; + const apiKey = provider.apiKey(selection).trim(); + if (!apiKey) { + options.onStatus(`No ${provider.name} key set; using on-device voice.`); + return null; + } + return new ReconnectingSttClient(provider.create, { ...options, apiKey }, continuous); +} diff --git a/app/native/cloud-stt.ts b/app/native/cloud-stt.ts index f9a77167..2600f802 100644 --- a/app/native/cloud-stt.ts +++ b/app/native/cloud-stt.ts @@ -2,8 +2,9 @@ * Shared shape for the cloud speech-to-text providers (ElevenLabs, Whisper, * Soniox). * The voice bridge holds one of these while a cloud provider owns the - * transcript; the Java controller then only decodes LC3 to PCM and hands it - * over via acceptPcm. + * transcript; the native capture (FaceclawVoiceController on Android, + * FaceclawSpeech on iOS) then only decodes audio to PCM and hands it over + * via acceptPcm. */ export type CloudSttTranscriptEvent = { @@ -51,8 +52,13 @@ export function toJavaBytes(bytes: Uint8Array): any { return javaBytes; } -/** Base64 for audio payloads, via the Android SDK (no JS base64 in NS core). */ +/** Base64 for audio payloads, via the platform SDK (no JS base64 in NS core). */ export function encodeBase64(bytes: Uint8Array): string { + if (global.isIOS) { + const copy = bytes.slice(); + return NSData.dataWithBytesLength(interop.handleof(copy.buffer), copy.byteLength) + .base64EncodedStringWithOptions(0 as NSDataBase64EncodingOptions); + } if (!global.isAndroid) return ""; return String(android.util.Base64.encodeToString(toJavaBytes(bytes), android.util.Base64.NO_WRAP)); } diff --git a/app/native/elevenlabs-stt.ts b/app/native/elevenlabs-stt.ts index 6fa555fc..bc493fb9 100644 --- a/app/native/elevenlabs-stt.ts +++ b/app/native/elevenlabs-stt.ts @@ -1,6 +1,5 @@ import { type CloudSttOptions, type CloudSttTranscriptEvent, encodeBase64 } from "./cloud-stt"; - -declare const com: any; +import { openSocket, type SocketConnection } from "./socket"; /** * ElevenLabs realtime speech-to-text over WebSocket. @@ -22,8 +21,7 @@ export type ElevenLabsTranscriptEvent = CloudSttTranscriptEvent; export type ElevenLabsSttOptions = CloudSttOptions; export class ElevenLabsSttClient { - private ws: any = null; - private listenerProxy: any = null; + private ws: SocketConnection | null = null; private open = false; private closed = false; // PCM that arrived before the socket finished opening; flushed on open. @@ -36,7 +34,7 @@ export class ElevenLabsSttClient { start(): void { if (this.closed || this.ws) return; const url = `${WS_URL}?model_id=${MODEL_ID}&audio_format=pcm_${SAMPLE_RATE}&commit_strategy=manual`; - this.listenerProxy = new com.faceclaw.app.FaceclawWebSocketListener({ + const listener = { onOpen: () => { if (this.closed) return; this.open = true; @@ -56,12 +54,12 @@ export class ElevenLabsSttClient { if (this.closed) return; this.options.onDisconnected?.(`ElevenLabs connection failed: ${String(message)}`); }, - }); + }; try { - // The API key rides an xi-api-key header, added by FaceclawWebSocket. + // The API key rides an xi-api-key header on the handshake. const key = this.options.apiKey; console.log(`[elevenlabs] connecting; apiKey length=${key.length} prefix=${key.slice(0, 4)}`); - this.ws = new com.faceclaw.app.FaceclawWebSocket(url, this.listenerProxy, "xi-api-key", key); + this.ws = openSocket(url, listener, { name: "xi-api-key", value: key }); this.options.onStatus("Connecting to ElevenLabs..."); } catch (error) { this.options.onDisconnected?.(`ElevenLabs connection failed: ${String((error as Error)?.message ?? error)}`); @@ -106,7 +104,6 @@ export class ElevenLabsSttClient { } this.ws = null; } - this.listenerProxy = null; this.pendingChunks.length = 0; } diff --git a/app/native/openai-stt.ts b/app/native/openai-stt.ts index 2949be00..063e0a39 100644 --- a/app/native/openai-stt.ts +++ b/app/native/openai-stt.ts @@ -1,6 +1,5 @@ import { CLOUD_STT_SAMPLE_RATE, CloudSttClient, CloudSttOptions, encodeBase64 } from "./cloud-stt"; - -declare const com: any; +import { openSocket, type SocketConnection } from "./socket"; /** * OpenAI realtime speech-to-text over WebSocket (a transcription-only session @@ -23,8 +22,7 @@ const TARGET_SAMPLE_RATE = 24000; export type OpenAiSttOptions = CloudSttOptions; export class OpenAiRealtimeSttClient implements CloudSttClient { - private ws: any = null; - private listenerProxy: any = null; + private ws: SocketConnection | null = null; private open = false; private closed = false; // Base64 audio (or the commit sentinel) queued until the socket opens. @@ -38,7 +36,7 @@ export class OpenAiRealtimeSttClient implements CloudSttClient { start(): void { if (this.closed || this.ws) return; - this.listenerProxy = new com.faceclaw.app.FaceclawWebSocketListener({ + const listener = { onOpen: () => { if (this.closed) return; this.open = true; @@ -59,14 +57,9 @@ export class OpenAiRealtimeSttClient implements CloudSttClient { if (this.closed) return; this.options.onDisconnected?.(`OpenAI connection failed: ${String(message)}`); }, - }); + }; try { - this.ws = new com.faceclaw.app.FaceclawWebSocket( - WS_URL, - this.listenerProxy, - "Authorization", - `Bearer ${this.options.apiKey}`, - ); + this.ws = openSocket(WS_URL, listener, { name: "Authorization", value: `Bearer ${this.options.apiKey}` }); this.options.onStatus("Connecting to OpenAI..."); } catch (error) { this.options.onDisconnected?.(`OpenAI connection failed: ${String((error as Error)?.message ?? error)}`); @@ -117,7 +110,6 @@ export class OpenAiRealtimeSttClient implements CloudSttClient { } this.ws = null; } - this.listenerProxy = null; this.pendingChunks.length = 0; } diff --git a/app/native/socket.ios.ts b/app/native/socket.ios.ts index e0bb5897..dd9ba7a5 100644 --- a/app/native/socket.ios.ts +++ b/app/native/socket.ios.ts @@ -1,7 +1,7 @@ -export type SocketListener = { onOpen(): void; onTextMessage(message: string): void; onClosed(code: number, reason: string): void; onFailure(message: string): void } +import type { SocketConnection, SocketHeader, SocketListener } from './socket' declare const FaceclawSocket: any -export function openSocket(url: string, listener: SocketListener): { sendText(text: string): void; close(code: number, reason: string): void } { - const native = FaceclawSocket.alloc().initWithURL(url) +export function openSocket(url: string, listener: SocketListener, header?: SocketHeader): SocketConnection { + const native = FaceclawSocket.alloc().initWithURLHeaderNameHeaderValue(url, header?.name ?? null, header?.value ?? null) let ended = false const close = () => { if (ended) return; ended = true; clearInterval(timer); native.close() } const timer = setInterval(() => { @@ -12,5 +12,15 @@ export function openSocket(url: string, listener: SocketListener): { sendText(te else { close(); if (event.kind === 'closed') listener.onClosed(event.code, event.text); else listener.onFailure(event.text) } } }, 25) - return { sendText: text => { if (!ended) native.sendText(text) }, close } + // NSURLSession queues every send; a failed one surfaces as an error event. + return { + sendText: text => { if (!ended) native.sendText(text); return !ended }, + sendBinary: bytes => { + if (ended) return false + const copy = bytes.slice() + native.sendData(NSData.dataWithBytesLength(interop.handleof(copy.buffer), copy.byteLength)) + return true + }, + close, + } } diff --git a/app/native/socket.ts b/app/native/socket.ts index 7b952acc..8677b878 100644 --- a/app/native/socket.ts +++ b/app/native/socket.ts @@ -1,9 +1,20 @@ +import { toJavaBytes } from './cloud-stt' + declare const com: any export type SocketListener = { onOpen(): void; onTextMessage(message: string): void; onClosed(code: number, reason: string): void; onFailure(message: string): void } -export function openSocket(url: string, listener: SocketListener): { sendText(text: string): void; close(code: number, reason: string): void } { +/** One extra handshake header, e.g. an API key. */ +export type SocketHeader = { name: string; value: string } +export type SocketConnection = { + /** False when the frame was refused outright; later failures arrive via onFailure. */ + sendText(text: string): boolean + sendBinary(bytes: Uint8Array): boolean + close(code: number, reason: string): void +} +export function openSocket(url: string, listener: SocketListener, header?: SocketHeader): SocketConnection { const proxy = new com.faceclaw.app.FaceclawWebSocketListener(listener) - const native = new com.faceclaw.app.FaceclawWebSocket(url, proxy, null, null) - const connection = { listenerProxy: proxy, sendText: (text: string) => native.sendText(text), + const native = new com.faceclaw.app.FaceclawWebSocket(url, proxy, header?.name ?? null, header?.value ?? null) + const connection = { listenerProxy: proxy, sendText: (text: string) => native.sendText(text) !== false, + sendBinary: (bytes: Uint8Array) => native.sendBinary(toJavaBytes(bytes)) !== false, close: (code: number, reason: string) => native.close(code, reason) } return connection } diff --git a/app/native/soniox-stt.ts b/app/native/soniox-stt.ts index e52454e7..aa0d210c 100644 --- a/app/native/soniox-stt.ts +++ b/app/native/soniox-stt.ts @@ -1,8 +1,7 @@ -import { CLOUD_STT_SAMPLE_RATE, CloudSttClient, CloudSttOptions, toJavaBytes } from "./cloud-stt"; +import { CLOUD_STT_SAMPLE_RATE, CloudSttClient, CloudSttOptions } from "./cloud-stt"; +import { openSocket, type SocketConnection } from "./socket"; import { TimedTranscript } from "./transcript-format"; -declare const com: any; - /** * Soniox realtime speech-to-text over WebSocket. * https://soniox.com/docs/stt/rt/real-time-transcription @@ -24,8 +23,7 @@ const MODEL_ID = "stt-rt-v5"; export type SonioxSttOptions = CloudSttOptions; export class SonioxSttClient implements CloudSttClient { - private ws: any = null; - private listenerProxy: any = null; + private ws: SocketConnection | null = null; private open = false; private closed = false; // PCM queued until the socket opens and the config message is sent. @@ -40,7 +38,7 @@ export class SonioxSttClient implements CloudSttClient { start(): void { if (this.closed || this.ws) return; - this.listenerProxy = new com.faceclaw.app.FaceclawWebSocketListener({ + const listener = { onOpen: () => { if (this.closed) return; // The API key rides in the config message; there is no auth header. @@ -77,9 +75,9 @@ export class SonioxSttClient implements CloudSttClient { if (this.closed) return; this.options.onDisconnected?.(`Soniox connection failed: ${String(message)}`); }, - }); + }; try { - this.ws = new com.faceclaw.app.FaceclawWebSocket(WS_URL, this.listenerProxy, null, null); + this.ws = openSocket(WS_URL, listener); this.options.onStatus("Connecting to Soniox..."); } catch (error) { this.options.onDisconnected?.(`Soniox connection failed: ${String((error as Error)?.message ?? error)}`); @@ -122,13 +120,12 @@ export class SonioxSttClient implements CloudSttClient { } this.ws = null; } - this.listenerProxy = null; this.pendingPcm.length = 0; } private sendPcm(pcm: Uint8Array): void { try { - if (this.ws?.sendBinary(toJavaBytes(pcm)) === false) { + if (this.ws?.sendBinary(pcm) === false) { this.options.onDisconnected?.("Soniox audio send failed."); } } catch (error) { diff --git a/app/native/voice-control.ios.ts b/app/native/voice-control.ios.ts index 2896e21d..14f85cdd 100644 --- a/app/native/voice-control.ios.ts +++ b/app/native/voice-control.ios.ts @@ -1,12 +1,17 @@ import type { VoiceControlState, VoiceTranscriptEvent } from './voice-control' +import type { CloudSttClient, CloudSttTranscriptEvent } from './cloud-stt' +import { createCloudSttClient, usesCloudStt, type VoiceProviderSelection } from './cloud-stt-provider' declare const FaceclawSpeech: any export type IosMicrophoneSession = { setMicrophone(enabled: boolean, listener?: (packet: Uint8Array) => void): Promise } +/** How long a cloud provider gets, after the last audio, to send its final transcript. */ +const CLOUD_FINAL_TIMEOUT_MS = 5000 /** iOS implementation of the shared voice dialog's event bridge. Audio comes - * from the glasses when connected, or the default phone input in preview mode. + * from the glasses when connected, or the default phone input in preview mode, + * and is transcribed by Apple's on-device recognizer or a cloud provider. */ export class IosVoiceControlBridge { private native: any = null @@ -24,6 +29,12 @@ export class IosVoiceControlBridge { private audioTimer: ReturnType | null = null private packets = 0 private log: (message: string) => void = () => {} + // Non-null while a cloud provider owns the transcript; FaceclawSpeech then + // only supplies PCM. cloudEndStatus is set once the audio has ended and the + // provider was asked to finalize; it is the status shown when it does. + private cloud: CloudSttClient | null = null + private cloudEndStatus: string | null = null + private cloudTimer: ReturnType | null = null onStatus(listener: (state: VoiceControlState) => void): () => void { this.statuses.add(listener); listener({ status: this.status, listening: this.capturing, detail: '' }); return () => { this.statuses.delete(listener) } @@ -38,16 +49,20 @@ export class IosVoiceControlBridge { if (!this.native) { this.native = FaceclawSpeech.new() this.native.eventHandler = (json: string) => this.receive(JSON.parse(json)) + this.native.pcmHandler = (pcm: NSData) => this.cloud?.acceptPcm(new Uint8Array(interop.bufferFromData(pcm))) } return this.native } - async prepare(foreground: boolean, usePhoneMic = false): Promise { - let status = FaceclawSpeech.authorizationStatus() - if (status === 0) { - if (!foreground) { this.setStatus('Open Faceclaw on the phone once to allow Speech Recognition.'); return false } - status = await new Promise(resolve => FaceclawSpeech.requestAuthorization(resolve)) + async prepare(foreground: boolean, usePhoneMic: boolean, voice: VoiceProviderSelection): Promise { + // Only Apple's recognizer needs Speech Recognition; a cloud provider just needs audio. + if (!usesCloudStt(voice)) { + let status = FaceclawSpeech.authorizationStatus() + if (status === 0) { + if (!foreground) { this.setStatus('Open Faceclaw on the phone once to allow Speech Recognition.'); return false } + status = await new Promise(resolve => FaceclawSpeech.requestAuthorization(resolve)) + } + if (status !== 3) { this.setStatus('Allow Speech Recognition for Faceclaw in iPhone Settings.'); return false } } - if (status !== 3) { this.setStatus('Allow Speech Recognition for Faceclaw in iPhone Settings.'); return false } if (usePhoneMic) { let microphoneStatus = FaceclawSpeech.microphoneAuthorizationStatus() if (microphoneStatus === 0) { @@ -59,12 +74,12 @@ export class IosVoiceControlBridge { return true } get statusText(): string { return this.status } - async startGlassesCapture(session: IosMicrophoneSession, log: (message: string) => void, endpointing = false): Promise { + async startGlassesCapture(session: IosMicrophoneSession, voice: VoiceProviderSelection, log: (message: string) => void, endpointing = false): Promise { this.stop() const generation = ++this.generation this.log = log; this.source = 'glasses'; this.session = session; this.packets = 0; this.capturing = true this.completion = new Promise(resolve => { this.resolveCompletion = resolve }) - const startError = String(this.ensureNative().startWithEndpointing(endpointing) ?? '') + const startError = this.startNative(voice, endpointing, false) if (startError) { this.stop(); this.setStatus(startError); this.emitEnd(); return } try { await session.setMicrophone(true, packet => { @@ -74,7 +89,7 @@ export class IosVoiceControlBridge { this.native.acceptPacket(NSData.dataWithBytesLength(interop.handleof(copy.buffer), copy.byteLength)) }) if (generation !== this.generation || !this.capturing) return - log('Voice: glasses microphone enabled; using on-device recognition') + log(`Voice: glasses microphone enabled; using ${this.cloud ? 'cloud' : 'on-device'} recognition`) this.audioTimer = setTimeout(() => { this.audioTimer = null if (generation === this.generation && this.capturing && !this.packets) { @@ -86,13 +101,30 @@ export class IosVoiceControlBridge { this.stop(); this.setStatus(error instanceof Error ? error.message : String(error)); this.emitEnd() } } - async startPhoneCapture(log: (message: string) => void, endpointing = false): Promise { + async startPhoneCapture(voice: VoiceProviderSelection, log: (message: string) => void, endpointing = false): Promise { this.stop() this.log = log; this.source = 'phone'; this.capturing = true this.completion = new Promise(resolve => { this.resolveCompletion = resolve }) - const error = String(this.ensureNative().startPhoneWithEndpointing(endpointing) ?? '') + const error = this.startNative(voice, endpointing, true) if (error) { this.stop(); this.setStatus(error); this.emitEnd(); return } - log('Voice: phone default microphone enabled; using on-device recognition') + log(`Voice: phone default microphone enabled; using ${this.cloud ? 'cloud' : 'on-device'} recognition`) + } + /** Starts the native capture for the chosen provider; returns an actionable error, or ''. */ + private startNative(voice: VoiceProviderSelection, endpointing: boolean, phone: boolean): string { + const native = this.ensureNative() + const generation = this.generation + const current = () => generation === this.generation + const cloud = createCloudSttClient(voice, { + onTranscript: event => { if (current()) this.receiveCloudTranscript(event) }, + onStatus: status => { if (current()) this.setStatus(status) }, + onError: message => { if (current()) { this.stop(); this.setStatus(message); this.emitEnd() } }, + }, () => false) + if (!cloud) return String((phone ? native.startPhoneWithEndpointing(endpointing) : native.startWithEndpointing(endpointing)) ?? '') + const error = String(native.startPcmCaptureWithEndpointingPhone(endpointing, phone) ?? '') + if (error) return error + this.cloud = cloud + cloud.start() + return '' } stopPhoneCapture(): void { if (this.source !== 'phone') return @@ -111,6 +143,7 @@ export class IosVoiceControlBridge { this.capturing = this.finalizing = false this.source = null this.clearAudioTimer(); this.releaseMicrophone(); this.native?.cancel() + this.clearCloud() this.finishCompletion() } handleSessionEnded(message = 'Glasses disconnected. Start voice input again after reconnecting.'): void { @@ -122,6 +155,33 @@ export class IosVoiceControlBridge { const session = this.session; this.session = null if (session) void session.setMicrophone(false).catch(error => this.log(`Voice microphone cleanup: ${error}`)) } + private clearCloud(): void { + if (this.cloudTimer !== null) clearTimeout(this.cloudTimer) + this.cloudTimer = null + this.cloud?.stop(); this.cloud = null; this.cloudEndStatus = null + } + /** The native audio has ended; have the provider finalize what it heard. */ + private finishCloud(status: string): void { + this.capturing = false; this.finalizing = true + this.cloudEndStatus = status + this.setStatus('Recognizing…') + // Armed first: finish() can deliver the final synchronously. + this.cloudTimer = setTimeout(() => this.completeCloud(), CLOUD_FINAL_TIMEOUT_MS) + this.cloud?.finish() + } + private completeCloud(): void { + const status = this.cloudEndStatus ?? 'Ready to send' + this.clearCloud() + this.capturing = this.finalizing = false + this.finishCompletion() + this.setStatus(status); this.emitEnd() + } + private receiveCloudTranscript(event: CloudSttTranscriptEvent): void { + if (!this.capturing && !this.finalizing) return + for (const fn of [...this.transcripts]) fn(event) + // Push-to-talk commits once, so the first final after it is the last. + if (event.isFinal && this.cloudEndStatus !== null) this.completeCloud() + } private emitEnd(): void { for (const fn of [...this.ends]) fn() } private finishCompletion(): void { const resolve = this.resolveCompletion; this.resolveCompletion = null; resolve?.() @@ -134,13 +194,15 @@ export class IosVoiceControlBridge { } else if (event.kind === 'status') this.setStatus(event.message ?? '') else if (event.kind === 'audio') { this.log(`Voice audio: ${event.seconds?.toFixed(1)}s, packets=${event.packets}, RMS=${event.rms?.toFixed(4)}, missing=${event.missing}, errors=${event.errors}`) - if (this.capturing) this.setStatus(`Listening on ${this.source === 'phone' ? 'phone' : 'glasses'}… ${event.seconds?.toFixed(0)}s`) + // A cloud provider's own status (connecting, listening, reconnecting) stays up. + if (this.capturing && !this.cloud) this.setStatus(`Listening on ${this.source === 'phone' ? 'phone' : 'glasses'}… ${event.seconds?.toFixed(0)}s`) } else if (event.kind === 'finishing') { void this.stopPushToTalk(); this.emitEnd() } else if (event.kind === 'ended') { this.source = null - this.capturing = this.finalizing = false this.clearAudioTimer(); this.releaseMicrophone() + if (this.cloud) { this.finishCloud(event.message ?? 'Ready to send'); return } + this.capturing = this.finalizing = false this.finishCompletion() this.setStatus(event.message ?? 'Ready to send'); this.emitEnd() } diff --git a/app/native/voice-control.ts b/app/native/voice-control.ts index cdaedf74..7462b22c 100644 --- a/app/native/voice-control.ts +++ b/app/native/voice-control.ts @@ -1,11 +1,8 @@ import { Utils } from "@nativescript/core"; -import { type CloudSttTranscriptEvent, type CloudSttOptions, CloudSttClient } from "./cloud-stt"; -import { ReconnectingSttClient } from "./reconnecting-stt"; +import { type CloudSttTranscriptEvent, CloudSttClient } from "./cloud-stt"; +import { createCloudSttClient } from "./cloud-stt-provider"; import { SpeechPauseDetector } from "./speech-pause"; -import { ElevenLabsSttClient } from "./elevenlabs-stt"; -import { OpenAiRealtimeSttClient } from "./openai-stt"; -import { SonioxSttClient } from "./soniox-stt"; import { toUint8Array } from "../util/array-util"; declare const com: any; @@ -341,38 +338,12 @@ export class FaceclawVoiceControlBridge { * than failing the capture outright. */ private createCloudClient(options: PushToTalkOptions): CloudSttClient | null { - if (options.provider === "onboard" || options.provider === "onboard-whisper") return null; - const sttOptions = { - apiKey: "", + return createCloudSttClient(options, { onTranscript: (event: CloudSttTranscriptEvent) => this.emitTranscript(event.text, event.isFinal, event), onStatus: (status: string) => this.setStatus(status), onError: (message: string) => this.setStatus(message), - }; - const reconnecting = (create: (options: CloudSttOptions) => CloudSttClient, apiKey: string) => - new ReconnectingSttClient(create, { ...sttOptions, apiKey }, () => this.captureHolders.has("continuous")); - if (options.provider === "elevenlabs") { - const apiKey = options.elevenLabsApiKey.trim(); - if (!apiKey) { - this.setStatus("No ElevenLabs key set; using on-device voice."); - return null; - } - return reconnecting((config) => new ElevenLabsSttClient(config), apiKey); - } - if (options.provider === "soniox") { - const apiKey = options.sonioxApiKey.trim(); - if (!apiKey) { - this.setStatus("No Soniox key set; using on-device voice."); - return null; - } - return reconnecting((config) => new SonioxSttClient(config), apiKey); - } - const apiKey = options.openAiApiKey.trim(); - if (!apiKey) { - this.setStatus("No OpenAI key set; using on-device voice."); - return null; - } - return reconnecting((config) => new OpenAiRealtimeSttClient(config), apiKey); + }, () => this.captureHolders.has("continuous")); } private releaseCapture(holder: CaptureHolder, commit: boolean): void { diff --git a/app/ui/dashboard-settings.ts b/app/ui/dashboard-settings.ts index b88438fb..b0652865 100644 --- a/app/ui/dashboard-settings.ts +++ b/app/ui/dashboard-settings.ts @@ -803,8 +803,10 @@ export const ringConnectionModeSetting = new ConfigSettingEnum = { - onboard: "On-device (Moonshine)", + onboard: global.isIOS ? "On-device (Apple)" : "On-device (Moonshine)", "onboard-whisper": "On-device (Whisper)", elevenlabs: "ElevenLabs", whisper: "OpenAI (Whisper)", @@ -816,7 +818,7 @@ export const voiceProviderSetting = new ConfigSettingEnum({ label: "Transcription Provider", storageKey: "voice.provider", defaultValue: "onboard", - values: ["onboard", "onboard-whisper", "elevenlabs", "whisper", "soniox"], + values: global.isIOS ? ["onboard", "elevenlabs", "whisper", "soniox"] : ["onboard", "onboard-whisper", "elevenlabs", "whisper", "soniox"], formatValue: (value) => voiceProviderLabels[value] ?? value, isDisabled: (value) => { if (value === "elevenlabs") return elevenLabsApiKeySetting.get().trim().length === 0; @@ -824,7 +826,9 @@ export const voiceProviderSetting = new ConfigSettingEnum({ if (value === "soniox") return sonioxApiKeySetting.get().trim().length === 0; return false; }, - description: "Speech-to-text engine for voice input. ElevenLabs, OpenAI, and Soniox are cloud services that need an API key, with significantly better accuracy than on-device transcription. The two On-device options need their voice model downloaded (below) and never leave the phone.", + description: global.isIOS + ? "Speech-to-text engine for voice input. ElevenLabs, OpenAI, and Soniox are cloud services that need an API key (Settings > API Keys), with significantly better accuracy than on-device transcription. On-device uses your iPhone's speech language and never leaves the phone." + : "Speech-to-text engine for voice input. ElevenLabs, OpenAI, and Soniox are cloud services that need an API key, with significantly better accuracy than on-device transcription. The two On-device options need their voice model downloaded (below) and never leave the phone.", }); const wakeWordActionLabels: Record = { diff --git a/app/ui/dashboard/settings-menus.ts b/app/ui/dashboard/settings-menus.ts index 18ae195a..e8d8a7f7 100644 --- a/app/ui/dashboard/settings-menus.ts +++ b/app/ui/dashboard/settings-menus.ts @@ -233,10 +233,9 @@ function settingsSections(): SettingsSection[] { const deferred = new Set(["Watch"]); return sections.map(section => { if (section.label === "Developer") return { ...section, items: [toggleSettingMenuItem(showBleBandwidthSetting)] }; - if (section.label === "Voice") return { label: "Voice", items: [enumSettingMenuItem(wakeWordActionSetting), { - label: "On-device dictation (Apple)", disabled: true, onSelect: () => {}, - description: "Uses the glasses microphone and your iPhone's speech language. No transcription API key needed. Say Hey Even for hands-free input, or open Voice from the menu and click when finished.", - }] }; + // No downloadable on-device models: iOS's on-device option is Apple's recognizer. + if (section.label === "Voice") return { label: "Voice", items: [enumSettingMenuItem(wakeWordActionSetting), + enumSettingMenuItem(voiceProviderSetting)] }; if (deferred.has(section.label)) return { label: section.label, items: [{ label: "Not available on iOS yet", disabled: true, onSelect: () => {}, description: `${section.label} integration has not been ported to iOS.`, diff --git a/tests/app-layout.test.cjs b/tests/app-layout.test.cjs index 2266beec..cc31f474 100644 --- a/tests/app-layout.test.cjs +++ b/tests/app-layout.test.cjs @@ -690,7 +690,7 @@ test('resizing a terminal reconnects with the new grid and ignores callbacks fro }, } } }, }; - const socketContext = { exports: {}, com: context.com }; + const socketContext = { exports: {}, com: context.com, require: () => ({ toJavaBytes: bytes => bytes }) }; vm.runInNewContext(js(read('app/native/socket.ts')), socketContext); context.require = name => name === './socket' ? socketContext.exports : {}; vm.runInNewContext(js(read('app/native/g2mirror-client.ts')), context); diff --git a/tests/cloud-stt.test.cjs b/tests/cloud-stt.test.cjs index c6f0e78e..2e394e2e 100644 --- a/tests/cloud-stt.test.cjs +++ b/tests/cloud-stt.test.cjs @@ -193,3 +193,28 @@ test('an unsolicited Soniox session end finalizes visible text and reconnects', assert.equal(h.sockets.length, 2); h.client.stop(); }); + +test('provider selection falls back to on-device without a key and connects the chosen provider with one', () => { + const sockets = []; + const load = loader({ + setTimeout() {}, clearTimeout() {}, console: { log() {}, warn() {} }, + com: { faceclaw: { app: { + FaceclawWebSocketListener: function(callbacks) { return callbacks; }, + FaceclawWebSocket: class { constructor(url, _listener, header, value) { sockets.push({ url, header, value }); } }, + } } }, + }, { './cloud-stt': { CLOUD_STT_SAMPLE_RATE: 16000, encodeBase64: () => '', toJavaBytes: bytes => bytes } }); + const { createCloudSttClient, usesCloudStt } = load('app/native/cloud-stt-provider.ts'); + const statuses = []; + const options = { onTranscript() {}, onStatus: status => statuses.push(status), onError() {} }; + const selection = { provider: 'onboard', elevenLabsApiKey: '', openAiApiKey: ' ', sonioxApiKey: 'soniox-key' }; + assert.equal(usesCloudStt(selection), false); + assert.equal(createCloudSttClient(selection, options, () => false), null); + assert.equal(usesCloudStt({ ...selection, provider: 'whisper' }), false); + assert.equal(createCloudSttClient({ ...selection, provider: 'whisper' }, options, () => false), null); + assert.deepEqual(statuses, ['No OpenAI key set; using on-device voice.']); + assert.equal(usesCloudStt({ ...selection, provider: 'soniox' }), true); + createCloudSttClient({ ...selection, provider: 'soniox' }, options, () => false).start(); + createCloudSttClient({ ...selection, provider: 'elevenlabs', elevenLabsApiKey: ' xi ' }, options, () => false).start(); + assert.deepEqual(sockets.map(({ url, header, value }) => [new URL(url).host, header, value]), + [['stt-rt.soniox.com', null, null], ['api.elevenlabs.io', 'xi-api-key', 'xi']]); +}); diff --git a/tests/ios-app-host.test.cjs b/tests/ios-app-host.test.cjs index 444ed390..078c8951 100644 --- a/tests/ios-app-host.test.cjs +++ b/tests/ios-app-host.test.cjs @@ -4,6 +4,9 @@ const fs = require('node:fs'); const vm = require('node:vm'); const ts = require('typescript'); const path = require('node:path'); +// The provider settings ios-preview-controller hands the voice bridge. +const voiceSettings = { voiceProviderSetting: { get: () => 'onboard' }, elevenLabsApiKeySetting: { get: () => '' }, + openAiApiKeySetting: { get: () => '' }, sonioxApiKeySetting: { get: () => '' } }; function load(file, context) { const sandbox = { exports: {}, ...context }; const source = fs.readFileSync(path.join(__dirname, '..', file), 'utf8'); @@ -80,7 +83,8 @@ test('iOS settings prompts honor password input and report cancellation', async test('iOS host forwards hands-free capture mode to the voice bridge', async () => { const captures = []; const { IosPreviewController } = load('app/g2/ios-preview-controller.ts', { require: id => id === '../native/ios-voice-input' - ? { iosVoiceInput: { startGlassesCapture: (_session, _log, endpointing) => captures.push(endpointing) } } : {} }); + ? { iosVoiceInput: { startGlassesCapture: (_session, _voice, _log, endpointing) => captures.push(endpointing) } } + : id === '../ui/dashboard-settings' ? voiceSettings : {} }); const host = Object.create(IosPreviewController.prototype); host.communicator = {}; host.state = { phase: 'connected' }; await host.startVoiceCapture(true); @@ -304,20 +308,20 @@ test('iOS opens the on-glasses onboarding on a new acknowledged frame, once per test('iOS preview voice uses the phone microphone, while connected capture stays on glasses', async () => { const calls = []; - const bridge = { prepare: async (foreground, phone) => { calls.push(['prepare', foreground, phone]); return true; }, - startPhoneCapture: async (_log, endpointing) => calls.push(['phone', endpointing]), - startGlassesCapture: async (_session, _log, endpointing) => calls.push(['glasses', endpointing]) }; + const bridge = { prepare: async (foreground, phone, voice) => { calls.push(['prepare', foreground, phone, voice.provider]); return true; }, + startPhoneCapture: async (voice, _log, endpointing) => calls.push(['phone', voice.provider, endpointing]), + startGlassesCapture: async (_session, voice, _log, endpointing) => calls.push(['glasses', voice.provider, endpointing]) }; const { IosPreviewController } = load('app/g2/ios-preview-controller.ts', { - require: id => id === '../native/ios-voice-input' ? { iosVoiceInput: bridge } : {}, + require: id => id === '../native/ios-voice-input' ? { iosVoiceInput: bridge } : id === '../ui/dashboard-settings' ? voiceSettings : {}, }); const host = Object.create(IosPreviewController.prototype); host.active = true; host.glassesLocked = false; host.communicator = null; host.state = { phase: 'disconnected' }; assert.equal(await host.prepareVoiceCapture(), true); await host.startVoiceCapture(true); - assert.deepEqual(calls, [['prepare', true, true], ['prepare', true, true], ['phone', true]]); + assert.deepEqual(calls, [['prepare', true, true, 'onboard'], ['prepare', true, true, 'onboard'], ['phone', 'onboard', true]]); calls.length = 0; host.communicator = {}; host.state = { phase: 'connected' }; assert.equal(await host.prepareVoiceCapture(), true); await host.startVoiceCapture(false); - assert.deepEqual(calls, [['prepare', true, false], ['glasses', false]]); + assert.deepEqual(calls, [['prepare', true, false, 'onboard'], ['glasses', 'onboard', false]]); calls.length = 0; host.active = false; host.communicator = null; host.state = { phase: 'disconnected' }; assert.equal(await host.prepareVoiceCapture(), false); await host.startVoiceCapture(); assert.deepEqual(calls, []); diff --git a/tests/ios-socket-native.cjs b/tests/ios-socket-native.cjs index 6bfd8d93..29bd1b19 100644 --- a/tests/ios-socket-native.cjs +++ b/tests/ios-socket-native.cjs @@ -7,8 +7,13 @@ const { join, resolve } = require('node:path'); const root = resolve(__dirname, '..'), dir = mkdtempSync(join(tmpdir(), 'faceclaw-socket-')); execFileSync('xcrun', ['clang', '-fobjc-arc', '-framework', 'Foundation', '-I', 'App_Resources/iOS/src', 'tests/native/ios-socket-test.m', 'App_Resources/iOS/src/FaceclawSocket.m', '-o', join(dir, 'socket-test')], { cwd: root, stdio: 'inherit' }); -const server = new WebSocketServer({ host: '127.0.0.1', port: 0 }); -server.on('connection', socket => socket.on('message', data => socket.send(data.toString(), () => socket.close(1000, 'done')))); +// Requires the handshake header; answers binary frames with their hex, echoes text and closes. +const server = new WebSocketServer({ host: '127.0.0.1', port: 0, + verifyClient: (info, done) => done(info.req.headers['x-faceclaw-key'] === 'secret', 401, 'Unauthorized') }); +server.on('connection', socket => socket.on('message', data => { + if (Buffer.isBuffer(data)) socket.send(`binary:${data.toString('hex')}`); + else socket.send(data, () => socket.close(1000, 'done')); +})); server.on('listening', () => { const child = spawn(join(dir, 'socket-test'), [`ws://127.0.0.1:${server.address().port}`], { stdio: 'inherit' }); child.on('exit', code => { server.close(); rmSync(dir, { recursive: true, force: true }); process.exitCode = code ?? 1; }); diff --git a/tests/ios-voice.test.cjs b/tests/ios-voice.test.cjs index f9ad6181..9e0ba624 100644 --- a/tests/ios-voice.test.cjs +++ b/tests/ios-voice.test.cjs @@ -4,15 +4,29 @@ const fs = require('node:fs'); const vm = require('node:vm'); const ts = require('typescript'); function harness() { - const native = { startWithEndpointing(endpointing) { this.endpointing = endpointing; return ''; }, startPhoneWithEndpointing(endpointing) { this.endpointing = endpointing; this.phone = true; return this.error ?? ''; }, cancel() { this.cancels++; }, cancels: 0, finish() { this.finished = true; }, acceptPacket: p => audio.push(p) }; - const audio = [], timers = new Map(), commands = [], text = []; + const native = { startWithEndpointing(endpointing) { this.endpointing = endpointing; return ''; }, startPhoneWithEndpointing(endpointing) { this.endpointing = endpointing; this.phone = true; return this.error ?? ''; }, + startPcmCaptureWithEndpointingPhone(endpointing, phone) { this.pcmCapture = { endpointing, phone }; return this.error ?? ''; }, + cancel() { this.cancels++; }, cancels: 0, finish() { this.finished = true; }, acceptPacket: p => audio.push(p) }; + const audio = [], timers = new Map(), commands = [], text = [], clouds = []; + // Stands in for cloud-stt-provider: any provider but "onboard" is a keyed cloud one. + const provider = { + usesCloudStt: voice => voice.provider !== 'onboard', + createCloudSttClient(voice, options) { + if (voice.provider === 'onboard') return null; + const client = { options, pcm: [], started: false, finished: 0, stopped: false, + start() { this.started = true; }, acceptPcm(bytes) { this.pcm.push([...bytes]); }, + finish() { this.finished++; }, stop() { this.stopped = true; } }; + clouds.push(client); return client; + }, + }; let authorization = 3, microphoneAuthorization = 3, microphoneRequests = 0, permissionRequests = 0, release; const sandbox = { exports: {}, FaceclawSpeech: { new: () => native, authorizationStatus: () => authorization, requestAuthorization: done => { permissionRequests++; done(3); }, microphoneAuthorizationStatus: () => microphoneAuthorization, requestMicrophoneAuthorization: done => { microphoneRequests++; done(3); } }, setTimeout: fn => { timers.set(fn, fn); return fn; }, clearTimeout: fn => timers.delete(fn), - NSData: { dataWithBytesLength: (b, n) => Buffer.from(b, 0, n) }, interop: { handleof: b => b } }; + NSData: { dataWithBytesLength: (b, n) => Buffer.from(b, 0, n) }, interop: { handleof: b => b, bufferFromData: d => d }, + require: name => name === './cloud-stt-provider' ? provider : {} }; vm.runInNewContext(ts.transpileModule(fs.readFileSync('app/native/voice-control.ios.ts', 'utf8'), { compilerOptions: { module: ts.ModuleKind.CommonJS, target: ts.ScriptTarget.ES2020 } }).outputText, sandbox); const bridge = new sandbox.exports.IosVoiceControlBridge(); @@ -22,20 +36,22 @@ function harness() { if (enabled) { this.packet = fn; return new Promise(resolve => { release = resolve; }); } return Promise.resolve(); } }; - return { bridge, native, session, audio, commands, text, timers, release: () => release(), + const onboard = { provider: 'onboard', elevenLabsApiKey: '', openAiApiKey: '', sonioxApiKey: '' }; + return { bridge, native, session, audio, commands, text, timers, clouds, release: () => release(), + onboard, cloud: { ...onboard, provider: 'soniox', sonioxApiKey: 'key' }, microphoneAuthorization: value => { microphoneAuthorization = value; }, microphoneRequests: () => microphoneRequests, authorization: value => { authorization = value; }, permissionRequests: () => permissionRequests, event: event => native.eventHandler(JSON.stringify(event)) }; } test('iOS speech permission is requested only in foreground and denial stays actionable', async () => { const h = harness(); h.authorization(0); - assert.equal(await h.bridge.prepare(false), false); assert.equal(h.permissionRequests(), 0); - assert.equal(await h.bridge.prepare(true), true); assert.equal(h.permissionRequests(), 1); - h.authorization(2); assert.equal(await h.bridge.prepare(true), false); + assert.equal(await h.bridge.prepare(false, false, h.onboard), false); assert.equal(h.permissionRequests(), 0); + assert.equal(await h.bridge.prepare(true, false, h.onboard), true); assert.equal(h.permissionRequests(), 1); + h.authorization(2); assert.equal(await h.bridge.prepare(true, false, h.onboard), false); assert.match(h.bridge.statusText, /Settings/); }); test('glasses speech sends PCM-source packets, replaces partials, and drains final text after manual release', async () => { - const h = harness(); const start = h.bridge.startGlassesCapture(h.session, () => {}); + const h = harness(); const start = h.bridge.startGlassesCapture(h.session, h.onboard, () => {}); h.session.packet(new Uint8Array([1, 2, 3])); assert.deepEqual([...h.audio[0]], [1, 2, 3]); h.release(); await start; h.event({ kind: 'transcript', text: 'hello', final: false }); @@ -51,7 +67,7 @@ test('glasses speech sends PCM-source packets, replaces partials, and drains fin test('cancel and disconnect during enable cannot resurrect capture or leave its microphone listener active', async () => { for (const stop of ['stop', 'handleSessionEnded']) { const h = harness(); let ends = 0; h.bridge.onSpeechEnd(() => ends++); - const start = h.bridge.startGlassesCapture(h.session, () => {}); + const start = h.bridge.startGlassesCapture(h.session, h.onboard, () => {}); h.bridge[stop](); h.release(); await start; h.session.packet(new Uint8Array([9])); h.event({ kind: 'transcript', text: 'stale', final: true }); @@ -60,7 +76,7 @@ test('cancel and disconnect during enable cannot resurrect capture or leave its } }); test('missing microphone data ends the session and disables the mic with an actionable status', async () => { - const h = harness(); const start = h.bridge.startGlassesCapture(h.session, () => {}); h.release(); await start; + const h = harness(); const start = h.bridge.startGlassesCapture(h.session, h.onboard, () => {}); h.release(); await start; for (const fn of h.timers.values()) fn(); assert.deepEqual(h.commands, [true, false]); assert.match(h.bridge.statusText, /No microphone audio/); }); @@ -68,7 +84,7 @@ test('missing microphone data ends the session and disables the mic with an acti test('hands-free endpointing releases the microphone and waits for the final transcript before completing', async () => { const h = harness(); let ends = 0, completed = false; h.bridge.onSpeechEnd(() => ends++); - const start = h.bridge.startGlassesCapture(h.session, () => {}, true); h.release(); await start; + const start = h.bridge.startGlassesCapture(h.session, h.onboard, () => {}, true); h.release(); await start; assert.equal(h.native.endpointing, true); h.event({ kind: 'transcript', text: 'Set a timer', final: false }); h.event({ kind: 'finishing' }); @@ -80,27 +96,27 @@ test('hands-free endpointing releases the microphone and waits for the final tra h.event({ kind: 'ended' }); await completion; assert.equal(completed, true); assert.equal(h.text.at(-1).text, 'Set a timer for five minutes.'); - const manual = h.bridge.startGlassesCapture(h.session, () => {}); h.release(); await manual; + const manual = h.bridge.startGlassesCapture(h.session, h.onboard, () => {}); h.release(); await manual; assert.equal(h.native.endpointing, false); const cancelled = h.bridge.stopPushToTalk(); h.bridge.stop(); await cancelled; }); test('phone microphone permission is requested only for foreground preview capture', async () => { const h = harness(); h.microphoneAuthorization(0); - assert.equal(await h.bridge.prepare(true), true); + assert.equal(await h.bridge.prepare(true, false, h.onboard), true); assert.equal(h.microphoneRequests(), 0, 'glasses input does not need phone microphone permission'); - assert.equal(await h.bridge.prepare(false, true), false); + assert.equal(await h.bridge.prepare(false, true, h.onboard), false); assert.equal(h.microphoneRequests(), 0); - assert.equal(await h.bridge.prepare(true, true), true); + assert.equal(await h.bridge.prepare(true, true, h.onboard), true); assert.equal(h.microphoneRequests(), 1); h.microphoneAuthorization(2); - assert.equal(await h.bridge.prepare(true, true), false); + assert.equal(await h.bridge.prepare(true, true, h.onboard), false); assert.match(h.bridge.statusText, /Microphone.*Settings/); }); test('phone capture forwards endpointing and drains final text without touching Bluetooth', async () => { const h = harness(); - await h.bridge.startPhoneCapture(() => {}, true); + await h.bridge.startPhoneCapture(h.onboard, () => {}, true); assert.equal(h.native.phone, true); assert.equal(h.native.endpointing, true); h.bridge.handleSessionEnded(); h.event({ kind: 'transcript', text: 'First line\nSecond line', final: false }); @@ -118,11 +134,11 @@ test('phone capture forwards endpointing and drains final text without touching test('background cleanup stops phone capture but preserves glasses capture', async () => { const h = harness(); let ends = 0; h.bridge.onSpeechEnd(() => ends++); - await h.bridge.startPhoneCapture(() => {}); + await h.bridge.startPhoneCapture(h.onboard, () => {}); h.bridge.stopPhoneCapture(); assert.equal(ends, 1); h.event({ kind: 'transcript', text: 'stale', final: true }); assert.equal(h.text.length, 0); - const start = h.bridge.startGlassesCapture(h.session, () => {}); h.release(); await start; + const start = h.bridge.startGlassesCapture(h.session, h.onboard, () => {}); h.release(); await start; h.bridge.stopPhoneCapture(); h.event({ kind: 'transcript', text: 'on glasses', final: false }); assert.equal(h.text.at(-1).text, 'on glasses'); @@ -132,7 +148,68 @@ test('background cleanup stops phone capture but preserves glasses capture', asy test('phone microphone startup errors end capture with an actionable status', async () => { const h = harness(); let ends = 0; h.bridge.onSpeechEnd(() => ends++); h.native.error = 'No phone microphone is available.'; - await h.bridge.startPhoneCapture(() => {}); + await h.bridge.startPhoneCapture(h.onboard, () => {}); assert.equal(ends, 1); assert.match(h.bridge.statusText, /No phone microphone/); await h.bridge.stopPushToTalk(); }); + +test('a cloud provider needs no Speech Recognition permission, but phone capture still needs the microphone', async () => { + const h = harness(); h.authorization(0); h.microphoneAuthorization(0); + assert.equal(await h.bridge.prepare(false, false, h.cloud), true); + assert.equal(await h.bridge.prepare(false, true, h.cloud), false); + assert.equal(await h.bridge.prepare(true, true, h.cloud), true); + assert.equal(h.permissionRequests(), 0); assert.equal(h.microphoneRequests(), 1); +}); + +test('glasses cloud capture streams native PCM and completes on the final after the drained audio', async () => { + const h = harness(); + const start = h.bridge.startGlassesCapture(h.session, h.cloud, () => {}); + assert.deepEqual({ ...h.native.pcmCapture }, { endpointing: false, phone: false }); + const client = h.clouds[0]; assert.equal(client.started, true); + client.options.onStatus('Listening (Soniox)...'); + h.session.packet(new Uint8Array([1, 2])); assert.equal(h.audio.length, 1, 'LC3 still decodes natively'); + h.release(); await start; + h.native.pcmHandler(new Uint8Array([5, 6]).buffer); + h.event({ kind: 'audio', seconds: 1, packets: 20, rms: 0.1, missing: 0, errors: 0 }); + assert.equal(h.bridge.statusText, 'Listening (Soniox)...', 'the provider status stays up'); + client.options.onTranscript({ text: 'hello', isFinal: false }); + let done = false; + const completion = h.bridge.stopPushToTalk().then(() => { done = true; }); + assert.deepEqual(h.commands, [true, false]); assert.equal(h.native.finished, true); + h.event({ kind: 'finishing' }); + assert.equal(client.finished, 0, 'commit waits for the native audio to drain'); + h.native.pcmHandler(new Uint8Array([7]).buffer); + h.event({ kind: 'ended', message: 'Ready to send' }); + assert.deepEqual(client.pcm, [[5, 6], [7]]); assert.equal(client.finished, 1); + await Promise.resolve(); assert.equal(done, false); + client.options.onTranscript({ text: 'Hello there.', isFinal: true }); + await completion; + assert.equal(client.stopped, true); assert.equal(h.bridge.statusText, 'Ready to send'); + assert.deepEqual(h.text, [{ text: 'hello', isFinal: false }, { text: 'Hello there.', isFinal: true }]); + assert.equal(h.timers.size, 0); +}); + +test('cloud capture times out a missing final, keeps native errors, and ends on provider errors', async () => { + const h = harness(); + await h.bridge.startPhoneCapture(h.cloud, () => {}, true); + assert.deepEqual({ ...h.native.pcmCapture }, { endpointing: true, phone: true }); + const completion = h.bridge.stopPushToTalk(); + h.event({ kind: 'ended', message: 'Phone microphone interrupted.' }); + assert.equal(h.clouds[0].finished, 1); assert.equal(h.timers.size, 1); + for (const fn of [...h.timers.values()]) fn(); + await completion; + assert.equal(h.clouds[0].stopped, true); assert.equal(h.bridge.statusText, 'Phone microphone interrupted.'); + + let ends = 0; h.bridge.onSpeechEnd(() => ends++); + await h.bridge.startPhoneCapture(h.cloud, () => {}); + h.clouds[1].options.onError('Soniox: Invalid API key'); + assert.equal(h.clouds[1].stopped, true); assert.equal(ends, 1); + assert.match(h.bridge.statusText, /Invalid API key/); + h.clouds[1].options.onTranscript({ text: 'stale', isFinal: true }); + assert.equal(h.text.length, 0); + + h.native.error = 'No phone microphone is available.'; + await h.bridge.startPhoneCapture(h.cloud, () => {}); + assert.equal(h.clouds[2].started, false, 'no connection without audio'); + assert.match(h.bridge.statusText, /No phone microphone/); +}); diff --git a/tests/native/ios-socket-test.m b/tests/native/ios-socket-test.m index 19db162b..a26f9312 100644 --- a/tests/native/ios-socket-test.m +++ b/tests/native/ios-socket-test.m @@ -1,23 +1,39 @@ #import #import "FaceclawSocket.h" +/** Drains the socket's events until it closes; handler returns NO to stop early. */ +static void pump(FaceclawSocket *socket, BOOL (^handler)(NSDictionary *event)) { + NSDate *deadline = [NSDate dateWithTimeIntervalSinceNow:10]; + while (deadline.timeIntervalSinceNow > 0) { + NSArray *events = [NSJSONSerialization JSONObjectWithData:[[socket takeEvents] dataUsingEncoding:NSUTF8StringEncoding] options:0 error:nil]; + for (NSDictionary *event in events) if (!handler(event)) return; + [[NSRunLoop currentRunLoop] runUntilDate:[NSDate dateWithTimeIntervalSinceNow:0.01]]; + } +} int main(int argc, const char **argv) { @autoreleasepool { - FaceclawSocket *socket = [[FaceclawSocket alloc] initWithURL:[NSString stringWithUTF8String:argv[1]]]; - BOOL opened = NO, echoed = NO, closed = NO; - NSDate *deadline = [NSDate dateWithTimeIntervalSinceNow:10]; - while (!closed && deadline.timeIntervalSinceNow > 0) { - NSArray *events = [NSJSONSerialization JSONObjectWithData:[[socket takeEvents] dataUsingEncoding:NSUTF8StringEncoding] options:0 error:nil]; - for (NSDictionary *event in events) { - NSString *kind = event[@"kind"]; - if ([kind isEqual:@"open"]) { opened = YES; [socket sendText:@"Faceclaw echo: π 🔋"]; } - if ([kind isEqual:@"text"]) echoed = [event[@"text"] isEqual:@"Faceclaw echo: π 🔋"]; - if ([kind isEqual:@"closed"] || [kind isEqual:@"error"]) closed = YES; - } - [[NSRunLoop currentRunLoop] runUntilDate:[NSDate dateWithTimeIntervalSinceNow:0.01]]; - } + NSString *url = [NSString stringWithUTF8String:argv[1]]; + FaceclawSocket *socket = [[FaceclawSocket alloc] initWithURL:url headerName:@"X-Faceclaw-Key" headerValue:@"secret"]; + __block BOOL opened = NO, binary = NO, echoed = NO, closed = NO; + pump(socket, ^BOOL(NSDictionary *event) { + NSString *kind = event[@"kind"]; + if ([kind isEqual:@"open"]) { opened = YES; const uint8_t bytes[] = {1, 2, 255}; [socket sendData:[NSData dataWithBytes:bytes length:3]]; } + if ([kind isEqual:@"text"] && [event[@"text"] isEqual:@"binary:0102ff"]) { binary = YES; [socket sendText:@"Faceclaw echo: π 🔋"]; } + if ([kind isEqual:@"text"] && [event[@"text"] isEqual:@"Faceclaw echo: π 🔋"]) echoed = YES; + if ([kind isEqual:@"closed"] || [kind isEqual:@"error"]) { closed = YES; return NO; } + return YES; + }); [socket close]; - if (!opened || !echoed || !closed) { NSLog(@"FAIL open=%d echo=%d close=%d", opened, echoed, closed); return 1; } - puts("PASS: native WebSocket opens, echoes UTF-8 and reports close"); + if (!opened || !binary || !echoed || !closed) { NSLog(@"FAIL open=%d binary=%d echo=%d close=%d", opened, binary, echoed, closed); return 1; } + // Without the header the server refuses the handshake; the error names the status. + FaceclawSocket *refused = [[FaceclawSocket alloc] initWithURL:url]; + __block NSString *error = nil; + pump(refused, ^BOOL(NSDictionary *event) { + if ([event[@"kind"] isEqual:@"error"]) { error = event[@"text"]; return NO; } + return YES; + }); + [refused close]; + if (![error containsString:@"HTTP 401"]) { NSLog(@"FAIL refused handshake error=%@", error); return 1; } + puts("PASS: native WebSocket sends a handshake header, binary and UTF-8 frames, reports close and refused status"); } return 0; } diff --git a/tests/voice-finalization.test.cjs b/tests/voice-finalization.test.cjs index 2d4e638a..39d7b4e3 100644 --- a/tests/voice-finalization.test.cjs +++ b/tests/voice-finalization.test.cjs @@ -171,6 +171,7 @@ test('native completion is capture-specific, follows the final, and does not wai const { FaceclawVoiceControlBridge } = load('app/native/voice-control.ts', { '@nativescript/core': { Utils: { android: { getApplicationContext: () => ({}) } } }, './speech-pause': { SpeechPauseDetector: class { reset() {} } }, + './cloud-stt-provider': { createCloudSttClient: () => null }, }, { global: { isAndroid: true }, com: { faceclaw: { app: { FaceclawVoiceController: Controller, FaceclawVoiceControllerListener: function (listener) { return listener; },