mirror of
https://git.undefined-dev.fr/Blendman974/faceclaw.git
synced 2026-11-19 21:03:09 +00:00
Merge branch 'main' of https://github.com/jimrandomh/faceclaw
This commit is contained in:
commit
f6a4615953
24 files changed
+518
-176
No files matched your search
@@ -2,7 +2,10 @@
|
||||
/** Thread-safe event queue; each NativeScript worker drains it on its own thread. */
|
||||
@interface FaceclawSocket : NSObject <NSURLSessionWebSocketDelegate>
|
||||
- (instancetype)initWithURL:(NSString *)url;
|
||||
/** headerName/headerValue add one handshake header (e.g. an API key); nil for none. */
|
||||
- (instancetype)initWithURL:(NSString *)url headerName:(NSString *)headerName headerValue:(NSString *)headerValue;
|
||||
- (NSString *)takeEvents;
|
||||
- (void)sendText:(NSString *)text;
|
||||
- (void)sendData:(NSData *)data;
|
||||
- (void)close;
|
||||
@end
|
||||
@@ -8,12 +8,17 @@
|
||||
@end
|
||||
@implementation FaceclawSocket
|
||||
- (instancetype)initWithURL:(NSString *)url {
|
||||
return [self initWithURL:url headerName:nil headerValue:nil];
|
||||
}
|
||||
- (instancetype)initWithURL:(NSString *)url headerName:(NSString *)headerName headerValue:(NSString *)headerValue {
|
||||
if ((self = [super init])) {
|
||||
_events = [NSMutableArray array];
|
||||
NSURLSessionConfiguration *config = [NSURLSessionConfiguration ephemeralSessionConfiguration];
|
||||
config.timeoutIntervalForRequest = 20;
|
||||
_session = [NSURLSession sessionWithConfiguration:config delegate:self delegateQueue:nil];
|
||||
_task = [_session webSocketTaskWithURL:[NSURL URLWithString:url]];
|
||||
NSMutableURLRequest *request = [NSMutableURLRequest requestWithURL:[NSURL URLWithString:url]];
|
||||
if (headerName.length && headerValue) [request setValue:headerValue forHTTPHeaderField:headerName];
|
||||
_task = [_session webSocketTaskWithRequest:request];
|
||||
_task.maximumMessageSize = 8 * 1024 * 1024;
|
||||
[_task resume];
|
||||
}
|
||||
@@ -25,7 +30,7 @@
|
||||
_queuedBytes += [event[@"text"] lengthOfBytesUsingEncoding:NSUTF8StringEncoding];
|
||||
if (_events.count >= 256 || _queuedBytes > 16 * 1024 * 1024) {
|
||||
[_events removeAllObjects]; _queuedBytes = 0;
|
||||
[_events addObject:@{@"kind": @"error", @"text": @"Terminal input queue overflow"}];
|
||||
[_events addObject:@{@"kind": @"error", @"text": @"WebSocket receive queue overflow"}];
|
||||
[self close];
|
||||
} else [_events addObject:event];
|
||||
}
|
||||
@@ -56,7 +61,13 @@
|
||||
[self close];
|
||||
}
|
||||
- (void)URLSession:(NSURLSession *)session task:(NSURLSessionTask *)task didCompleteWithError:(NSError *)error {
|
||||
if (error) [self enqueue:@{@"kind": @"error", @"text": error.localizedDescription}];
|
||||
if (error) {
|
||||
// A refused handshake only says "bad response"; name the HTTP status
|
||||
// so callers can tell a rejected key (401/403) from a network error.
|
||||
NSInteger status = [task.response isKindOfClass:NSHTTPURLResponse.class] ? ((NSHTTPURLResponse *)task.response).statusCode : 0;
|
||||
NSString *text = status >= 400 ? [NSString stringWithFormat:@"HTTP %ld: %@", (long)status, error.localizedDescription] : error.localizedDescription;
|
||||
[self enqueue:@{@"kind": @"error", @"text": text}];
|
||||
}
|
||||
[self close];
|
||||
}
|
||||
- (void)sendText:(NSString *)text {
|
||||
@@ -65,6 +76,12 @@
|
||||
if (error) { [weakSelf enqueue:@{@"kind": @"error", @"text": error.localizedDescription}]; [weakSelf close]; }
|
||||
}];
|
||||
}
|
||||
- (void)sendData:(NSData *)data {
|
||||
__weak FaceclawSocket *weakSelf = self;
|
||||
[_task sendMessage:[[NSURLSessionWebSocketMessage alloc] initWithData:data] completionHandler:^(NSError *error) {
|
||||
if (error) { [weakSelf enqueue:@{@"kind": @"error", @"text": error.localizedDescription}]; [weakSelf close]; }
|
||||
}];
|
||||
}
|
||||
- (void)close {
|
||||
@synchronized(self) {
|
||||
if (_ended) return; _ended = YES;
|
||||
|
||||
@@ -2,6 +2,8 @@
|
||||
/** On-device speech recognition fed by G2 BLE audio or the default phone microphone. */
|
||||
@interface FaceclawSpeech : NSObject
|
||||
@property(nonatomic, copy) void (^eventHandler)(NSString *json);
|
||||
/** 16 kHz mono PCM16LE chunks from a PCM capture, on the main queue in capture order. */
|
||||
@property(nonatomic, copy) void (^pcmHandler)(NSData *pcm);
|
||||
+ (NSInteger)authorizationStatus;
|
||||
+ (NSInteger)microphoneAuthorizationStatus;
|
||||
+ (void)requestMicrophoneAuthorization:(void (^)(NSInteger status))completion;
|
||||
@@ -9,6 +11,9 @@
|
||||
/** Returns an actionable error, or an empty string when capture starts. */
|
||||
- (NSString *)startWithEndpointing:(BOOL)endpointing;
|
||||
- (NSString *)startPhoneWithEndpointing:(BOOL)endpointing;
|
||||
/** Capture without the recognizer (for a cloud provider): audio goes to
|
||||
* pcmHandler, and "ended" follows the last chunk. No transcript events. */
|
||||
- (NSString *)startPcmCaptureWithEndpointing:(BOOL)endpointing phone:(BOOL)phone;
|
||||
- (void)acceptPacket:(NSData *)packet;
|
||||
- (void)finish;
|
||||
- (void)cancel;
|
||||
|
||||
@@ -15,6 +15,7 @@
|
||||
@property(atomic) NSUInteger generation;
|
||||
@property(nonatomic) NSUInteger pendingPackets;
|
||||
@property(atomic) BOOL accepting;
|
||||
@property(nonatomic) BOOL pcmOutput;
|
||||
@property(nonatomic, strong) AVAudioEngine *phoneEngine;
|
||||
@property(nonatomic) BOOL phoneTapInstalled;
|
||||
@property(nonatomic) BOOL phoneSessionActive;
|
||||
@@ -23,6 +24,23 @@
|
||||
@property(nonatomic, strong) FaceclawSpeechTranscript *transcript;
|
||||
@property(nonatomic, strong) FaceclawKitVoiceEndpointDetector *endpointDetector;
|
||||
@end
|
||||
/** Resamples one tap buffer to 16 kHz mono PCM16LE. Called on the serial
|
||||
* audio queue; the converter keeps its filter state between buffers. */
|
||||
static NSData *resampleToPcm(AVAudioConverter *converter, AVAudioPCMBuffer *input) {
|
||||
AVAudioFormat *format = converter.outputFormat;
|
||||
AVAudioFrameCount capacity = (AVAudioFrameCount)ceil(input.frameLength * format.sampleRate / input.format.sampleRate) + 32;
|
||||
AVAudioPCMBuffer *output = [[AVAudioPCMBuffer alloc] initWithPCMFormat:format frameCapacity:capacity];
|
||||
__block BOOL supplied = NO;
|
||||
NSError *error = nil;
|
||||
AVAudioConverterOutputStatus status = [converter convertToBuffer:output error:&error
|
||||
withInputFromBlock:^AVAudioBuffer *(AVAudioPacketCount count, AVAudioConverterInputStatus *inputStatus) {
|
||||
// NoDataNow (not EndOfStream) leaves the stream open for the next buffer.
|
||||
if (supplied) { *inputStatus = AVAudioConverterInputStatus_NoDataNow; return nil; }
|
||||
supplied = YES; *inputStatus = AVAudioConverterInputStatus_HaveData; return input;
|
||||
}];
|
||||
if (status == AVAudioConverterOutputStatus_Error || !output.frameLength) return nil;
|
||||
return [NSData dataWithBytes:output.int16ChannelData[0] length:output.frameLength * sizeof(int16_t)];
|
||||
}
|
||||
@implementation FaceclawSpeech
|
||||
- (instancetype)init {
|
||||
if ((self = [super init])) _audioQueue = dispatch_queue_create("com.faceclaw.glasses-speech", DISPATCH_QUEUE_SERIAL);
|
||||
@@ -46,37 +64,51 @@
|
||||
if (json) self.eventHandler([[NSString alloc] initWithData:json encoding:NSUTF8StringEncoding]);
|
||||
}
|
||||
- (NSString *)startWithEndpointing:(BOOL)endpointing {
|
||||
return [self startWithEndpointing:endpointing phone:NO];
|
||||
return [self startWithEndpointing:endpointing phone:NO pcm:NO];
|
||||
}
|
||||
- (NSString *)startPhoneWithEndpointing:(BOOL)endpointing {
|
||||
return [self startWithEndpointing:endpointing phone:YES];
|
||||
return [self startWithEndpointing:endpointing phone:YES pcm:NO];
|
||||
}
|
||||
- (NSString *)startWithEndpointing:(BOOL)endpointing phone:(BOOL)phone {
|
||||
- (NSString *)startPcmCaptureWithEndpointing:(BOOL)endpointing phone:(BOOL)phone {
|
||||
return [self startWithEndpointing:endpointing phone:phone pcm:YES];
|
||||
}
|
||||
- (NSString *)startWithEndpointing:(BOOL)endpointing phone:(BOOL)phone pcm:(BOOL)pcm {
|
||||
[self cancel];
|
||||
if (phone && [FaceclawSpeech microphoneAuthorizationStatus] != AVAuthorizationStatusAuthorized)
|
||||
return @"Allow Microphone access for Faceclaw in iPhone Settings.";
|
||||
if (SFSpeechRecognizer.authorizationStatus != SFSpeechRecognizerAuthorizationStatusAuthorized)
|
||||
return @"Allow Speech Recognition for Faceclaw in iPhone Settings.";
|
||||
NSString *language = NSLocale.preferredLanguages.firstObject ?: @"en-US";
|
||||
self.recognizer = [[SFSpeechRecognizer alloc] initWithLocale:[NSLocale localeWithLocaleIdentifier:language]];
|
||||
if (!self.recognizer.supportsOnDeviceRecognition)
|
||||
return [NSString stringWithFormat:@"On-device speech is unavailable for %@. Enable Dictation for that language in iPhone Settings → General → Keyboard, then retry.", language];
|
||||
if (!self.recognizer.available) return @"Speech recognition is temporarily unavailable. Try again shortly.";
|
||||
if (!pcm) {
|
||||
if (SFSpeechRecognizer.authorizationStatus != SFSpeechRecognizerAuthorizationStatusAuthorized)
|
||||
return @"Allow Speech Recognition for Faceclaw in iPhone Settings.";
|
||||
self.recognizer = [[SFSpeechRecognizer alloc] initWithLocale:[NSLocale localeWithLocaleIdentifier:language]];
|
||||
if (!self.recognizer.supportsOnDeviceRecognition)
|
||||
return [NSString stringWithFormat:@"On-device speech is unavailable for %@. Enable Dictation for that language in iPhone Settings → General → Keyboard, then retry.", language];
|
||||
if (!self.recognizer.available) return @"Speech recognition is temporarily unavailable. Try again shortly.";
|
||||
}
|
||||
if (!phone) {
|
||||
self.decoder = [FaceclawLc3Decoder new];
|
||||
if (!self.decoder) return @"Could not initialize the glasses audio decoder.";
|
||||
}
|
||||
self.pcmOutput = pcm;
|
||||
self.bestText = @""; self.accepting = YES;
|
||||
self.endpointDetector = endpointing ? [FaceclawKitVoiceEndpointDetector new] : nil;
|
||||
NSUInteger generation = self.generation;
|
||||
__weak FaceclawSpeech *weakSelf = self;
|
||||
if (pcm) {
|
||||
if (phone) {
|
||||
NSString *error = [self startPhoneAudio];
|
||||
if (error.length) { [self cancel]; return error; }
|
||||
}
|
||||
// The PCM consumer reports its own status and has no session limit.
|
||||
return @"";
|
||||
}
|
||||
self.recognizer.queue = NSOperationQueue.mainQueue;
|
||||
self.request = [SFSpeechAudioBufferRecognitionRequest new];
|
||||
self.request.requiresOnDeviceRecognition = YES;
|
||||
self.request.shouldReportPartialResults = YES;
|
||||
self.request.taskHint = SFSpeechRecognitionTaskHintDictation;
|
||||
if (@available(iOS 16.0, *)) self.request.addsPunctuation = YES;
|
||||
self.bestText = @""; self.accepting = YES;
|
||||
self.transcript = [FaceclawSpeechTranscript new];
|
||||
self.endpointDetector = endpointing ? [FaceclawKitVoiceEndpointDetector new] : nil;
|
||||
NSUInteger generation = self.generation;
|
||||
__weak FaceclawSpeech *weakSelf = self;
|
||||
self.task = [self.recognizer recognitionTaskWithRequest:self.request resultHandler:^(SFSpeechRecognitionResult *result, NSError *error) {
|
||||
dispatch_async(dispatch_get_main_queue(), ^{
|
||||
FaceclawSpeech *owner = weakSelf;
|
||||
@@ -124,6 +156,13 @@
|
||||
AVAudioInputNode *input = self.phoneEngine.inputNode;
|
||||
AVAudioFormat *format = [input outputFormatForBus:0];
|
||||
if (format.sampleRate <= 0 || !format.channelCount) return @"No phone microphone is available. Check your audio input and try again.";
|
||||
AVAudioConverter *converter = nil;
|
||||
if (self.pcmOutput) {
|
||||
AVAudioFormat *pcmFormat = [[AVAudioFormat alloc] initWithCommonFormat:AVAudioPCMFormatInt16 sampleRate:16000 channels:1 interleaved:YES];
|
||||
converter = [[AVAudioConverter alloc] initFromFormat:format toFormat:pcmFormat];
|
||||
if (!converter) return @"Could not convert phone microphone audio.";
|
||||
converter.downmix = YES;
|
||||
}
|
||||
NSUInteger generation = self.generation;
|
||||
SFSpeechAudioBufferRecognitionRequest *request = self.request;
|
||||
FaceclawKitVoiceEndpointDetector *detector = self.endpointDetector;
|
||||
@@ -141,7 +180,10 @@
|
||||
memcpy(destination->mBuffers[i].mData, source->mBuffers[i].mData, source->mBuffers[i].mDataByteSize);
|
||||
dispatch_async(owner.audioQueue, ^{ @autoreleasepool {
|
||||
if (owner.generation != generation) return;
|
||||
[request appendAudioPCMBuffer:copy];
|
||||
if (converter) {
|
||||
NSData *pcm = resampleToPcm(converter, copy);
|
||||
if (pcm.length) [owner deliverPcm:pcm generation:generation];
|
||||
} else [request appendAudioPCMBuffer:copy];
|
||||
double sum = 0;
|
||||
if (copy.floatChannelData) {
|
||||
const float *samples = copy.floatChannelData[0];
|
||||
@@ -188,6 +230,7 @@
|
||||
FaceclawLc3Decoder *decoder = self.decoder;
|
||||
SFSpeechAudioBufferRecognitionRequest *request = self.request;
|
||||
FaceclawKitVoiceEndpointDetector *endpointDetector = self.endpointDetector;
|
||||
BOOL pcmOutput = self.pcmOutput;
|
||||
NSData *copy = [packet copy];
|
||||
dispatch_async(self.audioQueue, ^{ @autoreleasepool {
|
||||
if (self.generation != generation) return;
|
||||
@@ -195,17 +238,26 @@
|
||||
double rms = 0;
|
||||
BOOL speechEnded = NO;
|
||||
if (pcm.length) {
|
||||
AVAudioFormat *format = [[AVAudioFormat alloc] initWithCommonFormat:AVAudioPCMFormatFloat32 sampleRate:16000 channels:1 interleaved:NO];
|
||||
AVAudioPCMBuffer *buffer = [[AVAudioPCMBuffer alloc] initWithPCMFormat:format frameCapacity:(AVAudioFrameCount)(pcm.length / 2)];
|
||||
buffer.frameLength = buffer.frameCapacity;
|
||||
const int16_t *samples = pcm.bytes;
|
||||
for (NSUInteger i = 0; i < buffer.frameLength; i++) {
|
||||
float value = samples[i] / 32768.0f;
|
||||
buffer.floatChannelData[0][i] = value; rms += value * value;
|
||||
AVAudioFrameCount frames = (AVAudioFrameCount)(pcm.length / 2);
|
||||
AVAudioPCMBuffer *buffer = nil;
|
||||
if (!pcmOutput) {
|
||||
AVAudioFormat *format = [[AVAudioFormat alloc] initWithCommonFormat:AVAudioPCMFormatFloat32 sampleRate:16000 channels:1 interleaved:NO];
|
||||
buffer = [[AVAudioPCMBuffer alloc] initWithPCMFormat:format frameCapacity:frames];
|
||||
buffer.frameLength = frames;
|
||||
}
|
||||
rms = sqrt(rms / buffer.frameLength);
|
||||
if (self.generation == generation) [request appendAudioPCMBuffer:buffer];
|
||||
speechEnded = [endpointDetector acceptLevelRms:rms * 32768.0 sampleCount:(int32_t)buffer.frameLength];
|
||||
const int16_t *samples = pcm.bytes;
|
||||
for (NSUInteger i = 0; i < frames; i++) {
|
||||
float value = samples[i] / 32768.0f;
|
||||
if (buffer) buffer.floatChannelData[0][i] = value;
|
||||
rms += value * value;
|
||||
}
|
||||
rms = sqrt(rms / frames);
|
||||
if (self.generation == generation) {
|
||||
// The decoder's output is already the PCM consumer's format.
|
||||
if (pcmOutput) [self deliverPcm:pcm generation:generation];
|
||||
else [request appendAudioPCMBuffer:buffer];
|
||||
}
|
||||
speechEnded = [endpointDetector acceptLevelRms:rms * 32768.0 sampleCount:(int32_t)frames];
|
||||
}
|
||||
NSUInteger packets = decoder.packets, errors = decoder.errors, missing = decoder.missing;
|
||||
dispatch_async(dispatch_get_main_queue(), ^{
|
||||
@@ -223,9 +275,15 @@
|
||||
[self stopPhoneAudio];
|
||||
NSUInteger generation = self.generation;
|
||||
SFSpeechAudioBufferRecognitionRequest *request = self.request;
|
||||
BOOL pcmOutput = self.pcmOutput;
|
||||
[self emit:@{@"kind":@"finishing"}];
|
||||
// Drain already received audio before marking the utterance complete.
|
||||
dispatch_async(self.audioQueue, ^{ if (self.generation == generation) [request endAudio]; });
|
||||
dispatch_async(self.audioQueue, ^{
|
||||
if (self.generation != generation) return;
|
||||
if (!pcmOutput) { [request endAudio]; return; }
|
||||
// Drained chunks are already queued for pcmHandler; "ended" follows them.
|
||||
dispatch_async(dispatch_get_main_queue(), ^{ if (self.generation == generation) [self complete:@"Ready to send"]; });
|
||||
});
|
||||
__weak FaceclawSpeech *weakSelf = self;
|
||||
dispatch_after(dispatch_time(DISPATCH_TIME_NOW, 8 * NSEC_PER_SEC), dispatch_get_main_queue(), ^{
|
||||
FaceclawSpeech *owner = weakSelf;
|
||||
@@ -239,9 +297,14 @@
|
||||
if (text.length) [self emit:@{@"kind":@"transcript", @"text":text, @"final":@YES}];
|
||||
if (self.generation == generation) [self emit:@{@"kind":@"ended", @"message":status}];
|
||||
}
|
||||
- (void)deliverPcm:(NSData *)pcm generation:(NSUInteger)generation {
|
||||
dispatch_async(dispatch_get_main_queue(), ^{
|
||||
if (self.generation == generation && self.pcmHandler) self.pcmHandler(pcm);
|
||||
});
|
||||
}
|
||||
- (void)cancel {
|
||||
self.generation++;
|
||||
self.accepting = NO; self.pendingPackets = 0;
|
||||
self.accepting = NO; self.pendingPackets = 0; self.pcmOutput = NO;
|
||||
[self stopPhoneAudio];
|
||||
[self.task cancel]; self.task = nil; self.request = nil; self.decoder = nil; self.recognizer = nil;
|
||||
self.bestText = @""; self.transcript = nil; self.endpointDetector = nil;
|
||||
|
||||
@@ -8,6 +8,7 @@
|
||||
* Disclose Apple's separate TestFlight reporting in iOS onboarding and privacy settings
|
||||
* Add an on-glasses Onboarding app, which prompts you to configure some things
|
||||
* iOS Notifications have an icon based on ANCS category
|
||||
* iOS voice input can use the ElevenLabs, OpenAI, and Soniox cloud transcription providers (Settings>Voice>Transcription Provider)
|
||||
* Weather app is more graphical, and can appear on the status bar and glanceboard Glanceboard system card
|
||||
|
||||
## 0.8.2 - 2026-10-03
|
||||
|
||||
@@ -46,7 +46,9 @@ import { shell, rawInputEventToInputEvent, type ShellWindow } from '../ui/shell/
|
||||
import { appViewportRect, isOnSwitcherEdge, sidebarStripVisible } from '../ui/shell/geometry'
|
||||
import { onAnySettingChanged,
|
||||
previewColorSetting, lockScreenEnabledSetting, getBrightnessPreferences, getStringSettingById,
|
||||
nightscoutSiteUrlSetting, nightscoutApiTokenSetting } from '../ui/dashboard-settings'
|
||||
nightscoutSiteUrlSetting, nightscoutApiTokenSetting, voiceProviderSetting, elevenLabsApiKeySetting,
|
||||
openAiApiKeySetting, sonioxApiKeySetting } from '../ui/dashboard-settings'
|
||||
import type { VoiceProviderSelection } from '../native/cloud-stt-provider'
|
||||
import type { PhoneGesture } from '../phone-ui/phone-gestures'
|
||||
import { claimGlassesOnboardingAutoLaunch, ONBOARDING_APP_ID } from '../apps/onboarding/onboarding-progress'
|
||||
|
||||
@@ -885,7 +887,7 @@ export class IosPreviewController {
|
||||
if (this.glassesLocked) return false
|
||||
const usePhoneMic = this.state.phase !== 'connected'
|
||||
if (usePhoneMic && !this.active) return false
|
||||
const ready = await iosVoiceInput.prepare(this.active, usePhoneMic)
|
||||
const ready = await iosVoiceInput.prepare(this.active, usePhoneMic, this.voiceProvider())
|
||||
if (!ready && this.active) this.onError(iosVoiceInput.statusText)
|
||||
return ready && !this.glassesLocked && (this.state.phase === 'connected' || this.active)
|
||||
}
|
||||
@@ -893,14 +895,20 @@ export class IosPreviewController {
|
||||
if (this.glassesLocked) return
|
||||
const log = (message: string) => this.logBluetooth(message)
|
||||
const communicator = this.communicator
|
||||
const voice = this.voiceProvider()
|
||||
if (communicator && this.state.phase === 'connected') {
|
||||
await iosVoiceInput.startGlassesCapture(communicator, log, endpointing)
|
||||
await iosVoiceInput.startGlassesCapture(communicator, voice, log, endpointing)
|
||||
} else if (this.active) {
|
||||
// Recheck microphone permission if the glasses disconnected after prepare.
|
||||
if (!await iosVoiceInput.prepare(this.active, true) || !this.active || this.glassesLocked) return
|
||||
await iosVoiceInput.startPhoneCapture(log, endpointing)
|
||||
if (!await iosVoiceInput.prepare(this.active, true, voice) || !this.active || this.glassesLocked) return
|
||||
await iosVoiceInput.startPhoneCapture(voice, log, endpointing)
|
||||
}
|
||||
}
|
||||
/** The transcription provider setting and the keys the cloud ones need. */
|
||||
private voiceProvider(): VoiceProviderSelection {
|
||||
return { provider: voiceProviderSetting.get(), elevenLabsApiKey: elevenLabsApiKeySetting.get(),
|
||||
openAiApiKey: openAiApiKeySetting.get(), sonioxApiKey: sonioxApiKeySetting.get() }
|
||||
}
|
||||
private logBluetooth(message: string): void {
|
||||
const line = `${new Date().toISOString()} [${this.active ? 'foreground' : 'background'}${UIApplication.sharedApplication.protectedDataAvailable ? '' : ',protected-data-unavailable'}] ${message}`
|
||||
console.log(`[ios-ble] ${line}`); this.logLines.push(line)
|
||||
|
||||
@@ -0,0 +1,68 @@
|
||||
import type { CloudSttClient, CloudSttOptions } from "./cloud-stt";
|
||||
import { ElevenLabsSttClient } from "./elevenlabs-stt";
|
||||
import { OpenAiRealtimeSttClient } from "./openai-stt";
|
||||
import { ReconnectingSttClient } from "./reconnecting-stt";
|
||||
import { SonioxSttClient } from "./soniox-stt";
|
||||
import type { VoiceProviderKind } from "./voice-control";
|
||||
|
||||
/**
|
||||
* The transcription-provider setting plus the keys the cloud providers need.
|
||||
* Shared by the Android and iOS voice bridges.
|
||||
*/
|
||||
export type VoiceProviderSelection = {
|
||||
provider: VoiceProviderKind;
|
||||
elevenLabsApiKey: string;
|
||||
openAiApiKey: string;
|
||||
sonioxApiKey: string;
|
||||
};
|
||||
|
||||
type CloudProvider = {
|
||||
name: string;
|
||||
apiKey: (selection: VoiceProviderSelection) => string;
|
||||
create: (options: CloudSttOptions) => CloudSttClient;
|
||||
};
|
||||
|
||||
// Providers absent here ("onboard", "onboard-whisper") transcribe on-device.
|
||||
const CLOUD_PROVIDERS: Partial<Record<VoiceProviderKind, CloudProvider>> = {
|
||||
elevenlabs: {
|
||||
name: "ElevenLabs",
|
||||
apiKey: (selection) => selection.elevenLabsApiKey,
|
||||
create: (options) => new ElevenLabsSttClient(options),
|
||||
},
|
||||
soniox: {
|
||||
name: "Soniox",
|
||||
apiKey: (selection) => selection.sonioxApiKey,
|
||||
create: (options) => new SonioxSttClient(options),
|
||||
},
|
||||
whisper: {
|
||||
name: "OpenAI",
|
||||
apiKey: (selection) => selection.openAiApiKey,
|
||||
create: (options) => new OpenAiRealtimeSttClient(options),
|
||||
},
|
||||
};
|
||||
|
||||
/** Whether createCloudSttClient would return a cloud client for this selection. */
|
||||
export function usesCloudStt(selection: VoiceProviderSelection): boolean {
|
||||
return Boolean(CLOUD_PROVIDERS[selection.provider]?.apiKey(selection).trim());
|
||||
}
|
||||
|
||||
/**
|
||||
* The cloud provider for a capture, or null to transcribe on-device. A cloud
|
||||
* provider whose API key is missing falls back to on-device (reported via
|
||||
* onStatus) rather than failing the capture outright. `continuous` says
|
||||
* whether the capture is open-ended (Transcribe), where pauses commit segments.
|
||||
*/
|
||||
export function createCloudSttClient(
|
||||
selection: VoiceProviderSelection,
|
||||
options: Omit<CloudSttOptions, "apiKey">,
|
||||
continuous: () => boolean,
|
||||
): CloudSttClient | null {
|
||||
const provider = CLOUD_PROVIDERS[selection.provider];
|
||||
if (!provider) return null;
|
||||
const apiKey = provider.apiKey(selection).trim();
|
||||
if (!apiKey) {
|
||||
options.onStatus(`No ${provider.name} key set; using on-device voice.`);
|
||||
return null;
|
||||
}
|
||||
return new ReconnectingSttClient(provider.create, { ...options, apiKey }, continuous);
|
||||
}
|
||||
@@ -2,8 +2,9 @@
|
||||
* Shared shape for the cloud speech-to-text providers (ElevenLabs, Whisper,
|
||||
* Soniox).
|
||||
* The voice bridge holds one of these while a cloud provider owns the
|
||||
* transcript; the Java controller then only decodes LC3 to PCM and hands it
|
||||
* over via acceptPcm.
|
||||
* transcript; the native capture (FaceclawVoiceController on Android,
|
||||
* FaceclawSpeech on iOS) then only decodes audio to PCM and hands it over
|
||||
* via acceptPcm.
|
||||
*/
|
||||
|
||||
export type CloudSttTranscriptEvent = {
|
||||
@@ -51,8 +52,13 @@ export function toJavaBytes(bytes: Uint8Array): any {
|
||||
return javaBytes;
|
||||
}
|
||||
|
||||
/** Base64 for audio payloads, via the Android SDK (no JS base64 in NS core). */
|
||||
/** Base64 for audio payloads, via the platform SDK (no JS base64 in NS core). */
|
||||
export function encodeBase64(bytes: Uint8Array): string {
|
||||
if (global.isIOS) {
|
||||
const copy = bytes.slice();
|
||||
return NSData.dataWithBytesLength(interop.handleof(copy.buffer), copy.byteLength)
|
||||
.base64EncodedStringWithOptions(0 as NSDataBase64EncodingOptions);
|
||||
}
|
||||
if (!global.isAndroid) return "";
|
||||
return String(android.util.Base64.encodeToString(toJavaBytes(bytes), android.util.Base64.NO_WRAP));
|
||||
}
|
||||
@@ -1,6 +1,5 @@
|
||||
import { type CloudSttOptions, type CloudSttTranscriptEvent, encodeBase64 } from "./cloud-stt";
|
||||
|
||||
declare const com: any;
|
||||
import { openSocket, type SocketConnection } from "./socket";
|
||||
|
||||
/**
|
||||
* ElevenLabs realtime speech-to-text over WebSocket.
|
||||
@@ -22,8 +21,7 @@ export type ElevenLabsTranscriptEvent = CloudSttTranscriptEvent;
|
||||
export type ElevenLabsSttOptions = CloudSttOptions;
|
||||
|
||||
export class ElevenLabsSttClient {
|
||||
private ws: any = null;
|
||||
private listenerProxy: any = null;
|
||||
private ws: SocketConnection | null = null;
|
||||
private open = false;
|
||||
private closed = false;
|
||||
// PCM that arrived before the socket finished opening; flushed on open.
|
||||
@@ -36,7 +34,7 @@ export class ElevenLabsSttClient {
|
||||
start(): void {
|
||||
if (this.closed || this.ws) return;
|
||||
const url = `${WS_URL}?model_id=${MODEL_ID}&audio_format=pcm_${SAMPLE_RATE}&commit_strategy=manual`;
|
||||
this.listenerProxy = new com.faceclaw.app.FaceclawWebSocketListener({
|
||||
const listener = {
|
||||
onOpen: () => {
|
||||
if (this.closed) return;
|
||||
this.open = true;
|
||||
@@ -56,12 +54,12 @@ export class ElevenLabsSttClient {
|
||||
if (this.closed) return;
|
||||
this.options.onDisconnected?.(`ElevenLabs connection failed: ${String(message)}`);
|
||||
},
|
||||
});
|
||||
};
|
||||
try {
|
||||
// The API key rides an xi-api-key header, added by FaceclawWebSocket.
|
||||
// The API key rides an xi-api-key header on the handshake.
|
||||
const key = this.options.apiKey;
|
||||
console.log(`[elevenlabs] connecting; apiKey length=${key.length} prefix=${key.slice(0, 4)}`);
|
||||
this.ws = new com.faceclaw.app.FaceclawWebSocket(url, this.listenerProxy, "xi-api-key", key);
|
||||
this.ws = openSocket(url, listener, { name: "xi-api-key", value: key });
|
||||
this.options.onStatus("Connecting to ElevenLabs...");
|
||||
} catch (error) {
|
||||
this.options.onDisconnected?.(`ElevenLabs connection failed: ${String((error as Error)?.message ?? error)}`);
|
||||
@@ -106,7 +104,6 @@ export class ElevenLabsSttClient {
|
||||
}
|
||||
this.ws = null;
|
||||
}
|
||||
this.listenerProxy = null;
|
||||
this.pendingChunks.length = 0;
|
||||
}
|
||||
|
||||
|
||||
@@ -1,6 +1,5 @@
|
||||
import { CLOUD_STT_SAMPLE_RATE, CloudSttClient, CloudSttOptions, encodeBase64 } from "./cloud-stt";
|
||||
|
||||
declare const com: any;
|
||||
import { openSocket, type SocketConnection } from "./socket";
|
||||
|
||||
/**
|
||||
* OpenAI realtime speech-to-text over WebSocket (a transcription-only session
|
||||
@@ -23,8 +22,7 @@ const TARGET_SAMPLE_RATE = 24000;
|
||||
export type OpenAiSttOptions = CloudSttOptions;
|
||||
|
||||
export class OpenAiRealtimeSttClient implements CloudSttClient {
|
||||
private ws: any = null;
|
||||
private listenerProxy: any = null;
|
||||
private ws: SocketConnection | null = null;
|
||||
private open = false;
|
||||
private closed = false;
|
||||
// Base64 audio (or the commit sentinel) queued until the socket opens.
|
||||
@@ -38,7 +36,7 @@ export class OpenAiRealtimeSttClient implements CloudSttClient {
|
||||
|
||||
start(): void {
|
||||
if (this.closed || this.ws) return;
|
||||
this.listenerProxy = new com.faceclaw.app.FaceclawWebSocketListener({
|
||||
const listener = {
|
||||
onOpen: () => {
|
||||
if (this.closed) return;
|
||||
this.open = true;
|
||||
@@ -59,14 +57,9 @@ export class OpenAiRealtimeSttClient implements CloudSttClient {
|
||||
if (this.closed) return;
|
||||
this.options.onDisconnected?.(`OpenAI connection failed: ${String(message)}`);
|
||||
},
|
||||
});
|
||||
};
|
||||
try {
|
||||
this.ws = new com.faceclaw.app.FaceclawWebSocket(
|
||||
WS_URL,
|
||||
this.listenerProxy,
|
||||
"Authorization",
|
||||
`Bearer ${this.options.apiKey}`,
|
||||
);
|
||||
this.ws = openSocket(WS_URL, listener, { name: "Authorization", value: `Bearer ${this.options.apiKey}` });
|
||||
this.options.onStatus("Connecting to OpenAI...");
|
||||
} catch (error) {
|
||||
this.options.onDisconnected?.(`OpenAI connection failed: ${String((error as Error)?.message ?? error)}`);
|
||||
@@ -117,7 +110,6 @@ export class OpenAiRealtimeSttClient implements CloudSttClient {
|
||||
}
|
||||
this.ws = null;
|
||||
}
|
||||
this.listenerProxy = null;
|
||||
this.pendingChunks.length = 0;
|
||||
}
|
||||
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
export type SocketListener = { onOpen(): void; onTextMessage(message: string): void; onClosed(code: number, reason: string): void; onFailure(message: string): void }
|
||||
import type { SocketConnection, SocketHeader, SocketListener } from './socket'
|
||||
declare const FaceclawSocket: any
|
||||
export function openSocket(url: string, listener: SocketListener): { sendText(text: string): void; close(code: number, reason: string): void } {
|
||||
const native = FaceclawSocket.alloc().initWithURL(url)
|
||||
export function openSocket(url: string, listener: SocketListener, header?: SocketHeader): SocketConnection {
|
||||
const native = FaceclawSocket.alloc().initWithURLHeaderNameHeaderValue(url, header?.name ?? null, header?.value ?? null)
|
||||
let ended = false
|
||||
const close = () => { if (ended) return; ended = true; clearInterval(timer); native.close() }
|
||||
const timer = setInterval(() => {
|
||||
@@ -12,5 +12,15 @@ export function openSocket(url: string, listener: SocketListener): { sendText(te
|
||||
else { close(); if (event.kind === 'closed') listener.onClosed(event.code, event.text); else listener.onFailure(event.text) }
|
||||
}
|
||||
}, 25)
|
||||
return { sendText: text => { if (!ended) native.sendText(text) }, close }
|
||||
// NSURLSession queues every send; a failed one surfaces as an error event.
|
||||
return {
|
||||
sendText: text => { if (!ended) native.sendText(text); return !ended },
|
||||
sendBinary: bytes => {
|
||||
if (ended) return false
|
||||
const copy = bytes.slice()
|
||||
native.sendData(NSData.dataWithBytesLength(interop.handleof(copy.buffer), copy.byteLength))
|
||||
return true
|
||||
},
|
||||
close,
|
||||
}
|
||||
}
|
||||
+14
-3
@@ -1,9 +1,20 @@
|
||||
import { toJavaBytes } from './cloud-stt'
|
||||
|
||||
declare const com: any
|
||||
export type SocketListener = { onOpen(): void; onTextMessage(message: string): void; onClosed(code: number, reason: string): void; onFailure(message: string): void }
|
||||
export function openSocket(url: string, listener: SocketListener): { sendText(text: string): void; close(code: number, reason: string): void } {
|
||||
/** One extra handshake header, e.g. an API key. */
|
||||
export type SocketHeader = { name: string; value: string }
|
||||
export type SocketConnection = {
|
||||
/** False when the frame was refused outright; later failures arrive via onFailure. */
|
||||
sendText(text: string): boolean
|
||||
sendBinary(bytes: Uint8Array): boolean
|
||||
close(code: number, reason: string): void
|
||||
}
|
||||
export function openSocket(url: string, listener: SocketListener, header?: SocketHeader): SocketConnection {
|
||||
const proxy = new com.faceclaw.app.FaceclawWebSocketListener(listener)
|
||||
const native = new com.faceclaw.app.FaceclawWebSocket(url, proxy, null, null)
|
||||
const connection = { listenerProxy: proxy, sendText: (text: string) => native.sendText(text),
|
||||
const native = new com.faceclaw.app.FaceclawWebSocket(url, proxy, header?.name ?? null, header?.value ?? null)
|
||||
const connection = { listenerProxy: proxy, sendText: (text: string) => native.sendText(text) !== false,
|
||||
sendBinary: (bytes: Uint8Array) => native.sendBinary(toJavaBytes(bytes)) !== false,
|
||||
close: (code: number, reason: string) => native.close(code, reason) }
|
||||
return connection
|
||||
}
|
||||
@@ -1,8 +1,7 @@
|
||||
import { CLOUD_STT_SAMPLE_RATE, CloudSttClient, CloudSttOptions, toJavaBytes } from "./cloud-stt";
|
||||
import { CLOUD_STT_SAMPLE_RATE, CloudSttClient, CloudSttOptions } from "./cloud-stt";
|
||||
import { openSocket, type SocketConnection } from "./socket";
|
||||
import { TimedTranscript } from "./transcript-format";
|
||||
|
||||
declare const com: any;
|
||||
|
||||
/**
|
||||
* Soniox realtime speech-to-text over WebSocket.
|
||||
* https://soniox.com/docs/stt/rt/real-time-transcription
|
||||
@@ -24,8 +23,7 @@ const MODEL_ID = "stt-rt-v5";
|
||||
export type SonioxSttOptions = CloudSttOptions;
|
||||
|
||||
export class SonioxSttClient implements CloudSttClient {
|
||||
private ws: any = null;
|
||||
private listenerProxy: any = null;
|
||||
private ws: SocketConnection | null = null;
|
||||
private open = false;
|
||||
private closed = false;
|
||||
// PCM queued until the socket opens and the config message is sent.
|
||||
@@ -40,7 +38,7 @@ export class SonioxSttClient implements CloudSttClient {
|
||||
|
||||
start(): void {
|
||||
if (this.closed || this.ws) return;
|
||||
this.listenerProxy = new com.faceclaw.app.FaceclawWebSocketListener({
|
||||
const listener = {
|
||||
onOpen: () => {
|
||||
if (this.closed) return;
|
||||
// The API key rides in the config message; there is no auth header.
|
||||
@@ -77,9 +75,9 @@ export class SonioxSttClient implements CloudSttClient {
|
||||
if (this.closed) return;
|
||||
this.options.onDisconnected?.(`Soniox connection failed: ${String(message)}`);
|
||||
},
|
||||
});
|
||||
};
|
||||
try {
|
||||
this.ws = new com.faceclaw.app.FaceclawWebSocket(WS_URL, this.listenerProxy, null, null);
|
||||
this.ws = openSocket(WS_URL, listener);
|
||||
this.options.onStatus("Connecting to Soniox...");
|
||||
} catch (error) {
|
||||
this.options.onDisconnected?.(`Soniox connection failed: ${String((error as Error)?.message ?? error)}`);
|
||||
@@ -122,13 +120,12 @@ export class SonioxSttClient implements CloudSttClient {
|
||||
}
|
||||
this.ws = null;
|
||||
}
|
||||
this.listenerProxy = null;
|
||||
this.pendingPcm.length = 0;
|
||||
}
|
||||
|
||||
private sendPcm(pcm: Uint8Array): void {
|
||||
try {
|
||||
if (this.ws?.sendBinary(toJavaBytes(pcm)) === false) {
|
||||
if (this.ws?.sendBinary(pcm) === false) {
|
||||
this.options.onDisconnected?.("Soniox audio send failed.");
|
||||
}
|
||||
} catch (error) {
|
||||
|
||||
@@ -1,12 +1,17 @@
|
||||
import type { VoiceControlState, VoiceTranscriptEvent } from './voice-control'
|
||||
import type { CloudSttClient, CloudSttTranscriptEvent } from './cloud-stt'
|
||||
import { createCloudSttClient, usesCloudStt, type VoiceProviderSelection } from './cloud-stt-provider'
|
||||
|
||||
declare const FaceclawSpeech: any
|
||||
export type IosMicrophoneSession = {
|
||||
setMicrophone(enabled: boolean, listener?: (packet: Uint8Array) => void): Promise<void>
|
||||
}
|
||||
/** How long a cloud provider gets, after the last audio, to send its final transcript. */
|
||||
const CLOUD_FINAL_TIMEOUT_MS = 5000
|
||||
|
||||
/** iOS implementation of the shared voice dialog's event bridge. Audio comes
|
||||
* from the glasses when connected, or the default phone input in preview mode.
|
||||
* from the glasses when connected, or the default phone input in preview mode,
|
||||
* and is transcribed by Apple's on-device recognizer or a cloud provider.
|
||||
*/
|
||||
export class IosVoiceControlBridge {
|
||||
private native: any = null
|
||||
@@ -24,6 +29,12 @@ export class IosVoiceControlBridge {
|
||||
private audioTimer: ReturnType<typeof setTimeout> | null = null
|
||||
private packets = 0
|
||||
private log: (message: string) => void = () => {}
|
||||
// Non-null while a cloud provider owns the transcript; FaceclawSpeech then
|
||||
// only supplies PCM. cloudEndStatus is set once the audio has ended and the
|
||||
// provider was asked to finalize; it is the status shown when it does.
|
||||
private cloud: CloudSttClient | null = null
|
||||
private cloudEndStatus: string | null = null
|
||||
private cloudTimer: ReturnType<typeof setTimeout> | null = null
|
||||
|
||||
onStatus(listener: (state: VoiceControlState) => void): () => void {
|
||||
this.statuses.add(listener); listener({ status: this.status, listening: this.capturing, detail: '' }); return () => { this.statuses.delete(listener) }
|
||||
@@ -38,16 +49,20 @@ export class IosVoiceControlBridge {
|
||||
if (!this.native) {
|
||||
this.native = FaceclawSpeech.new()
|
||||
this.native.eventHandler = (json: string) => this.receive(JSON.parse(json))
|
||||
this.native.pcmHandler = (pcm: NSData) => this.cloud?.acceptPcm(new Uint8Array(interop.bufferFromData(pcm)))
|
||||
}
|
||||
return this.native
|
||||
}
|
||||
async prepare(foreground: boolean, usePhoneMic = false): Promise<boolean> {
|
||||
let status = FaceclawSpeech.authorizationStatus()
|
||||
if (status === 0) {
|
||||
if (!foreground) { this.setStatus('Open Faceclaw on the phone once to allow Speech Recognition.'); return false }
|
||||
status = await new Promise<number>(resolve => FaceclawSpeech.requestAuthorization(resolve))
|
||||
async prepare(foreground: boolean, usePhoneMic: boolean, voice: VoiceProviderSelection): Promise<boolean> {
|
||||
// Only Apple's recognizer needs Speech Recognition; a cloud provider just needs audio.
|
||||
if (!usesCloudStt(voice)) {
|
||||
let status = FaceclawSpeech.authorizationStatus()
|
||||
if (status === 0) {
|
||||
if (!foreground) { this.setStatus('Open Faceclaw on the phone once to allow Speech Recognition.'); return false }
|
||||
status = await new Promise<number>(resolve => FaceclawSpeech.requestAuthorization(resolve))
|
||||
}
|
||||
if (status !== 3) { this.setStatus('Allow Speech Recognition for Faceclaw in iPhone Settings.'); return false }
|
||||
}
|
||||
if (status !== 3) { this.setStatus('Allow Speech Recognition for Faceclaw in iPhone Settings.'); return false }
|
||||
if (usePhoneMic) {
|
||||
let microphoneStatus = FaceclawSpeech.microphoneAuthorizationStatus()
|
||||
if (microphoneStatus === 0) {
|
||||
@@ -59,12 +74,12 @@ export class IosVoiceControlBridge {
|
||||
return true
|
||||
}
|
||||
get statusText(): string { return this.status }
|
||||
async startGlassesCapture(session: IosMicrophoneSession, log: (message: string) => void, endpointing = false): Promise<void> {
|
||||
async startGlassesCapture(session: IosMicrophoneSession, voice: VoiceProviderSelection, log: (message: string) => void, endpointing = false): Promise<void> {
|
||||
this.stop()
|
||||
const generation = ++this.generation
|
||||
this.log = log; this.source = 'glasses'; this.session = session; this.packets = 0; this.capturing = true
|
||||
this.completion = new Promise(resolve => { this.resolveCompletion = resolve })
|
||||
const startError = String(this.ensureNative().startWithEndpointing(endpointing) ?? '')
|
||||
const startError = this.startNative(voice, endpointing, false)
|
||||
if (startError) { this.stop(); this.setStatus(startError); this.emitEnd(); return }
|
||||
try {
|
||||
await session.setMicrophone(true, packet => {
|
||||
@@ -74,7 +89,7 @@ export class IosVoiceControlBridge {
|
||||
this.native.acceptPacket(NSData.dataWithBytesLength(interop.handleof(copy.buffer), copy.byteLength))
|
||||
})
|
||||
if (generation !== this.generation || !this.capturing) return
|
||||
log('Voice: glasses microphone enabled; using on-device recognition')
|
||||
log(`Voice: glasses microphone enabled; using ${this.cloud ? 'cloud' : 'on-device'} recognition`)
|
||||
this.audioTimer = setTimeout(() => {
|
||||
this.audioTimer = null
|
||||
if (generation === this.generation && this.capturing && !this.packets) {
|
||||
@@ -86,13 +101,30 @@ export class IosVoiceControlBridge {
|
||||
this.stop(); this.setStatus(error instanceof Error ? error.message : String(error)); this.emitEnd()
|
||||
}
|
||||
}
|
||||
async startPhoneCapture(log: (message: string) => void, endpointing = false): Promise<void> {
|
||||
async startPhoneCapture(voice: VoiceProviderSelection, log: (message: string) => void, endpointing = false): Promise<void> {
|
||||
this.stop()
|
||||
this.log = log; this.source = 'phone'; this.capturing = true
|
||||
this.completion = new Promise(resolve => { this.resolveCompletion = resolve })
|
||||
const error = String(this.ensureNative().startPhoneWithEndpointing(endpointing) ?? '')
|
||||
const error = this.startNative(voice, endpointing, true)
|
||||
if (error) { this.stop(); this.setStatus(error); this.emitEnd(); return }
|
||||
log('Voice: phone default microphone enabled; using on-device recognition')
|
||||
log(`Voice: phone default microphone enabled; using ${this.cloud ? 'cloud' : 'on-device'} recognition`)
|
||||
}
|
||||
/** Starts the native capture for the chosen provider; returns an actionable error, or ''. */
|
||||
private startNative(voice: VoiceProviderSelection, endpointing: boolean, phone: boolean): string {
|
||||
const native = this.ensureNative()
|
||||
const generation = this.generation
|
||||
const current = () => generation === this.generation
|
||||
const cloud = createCloudSttClient(voice, {
|
||||
onTranscript: event => { if (current()) this.receiveCloudTranscript(event) },
|
||||
onStatus: status => { if (current()) this.setStatus(status) },
|
||||
onError: message => { if (current()) { this.stop(); this.setStatus(message); this.emitEnd() } },
|
||||
}, () => false)
|
||||
if (!cloud) return String((phone ? native.startPhoneWithEndpointing(endpointing) : native.startWithEndpointing(endpointing)) ?? '')
|
||||
const error = String(native.startPcmCaptureWithEndpointingPhone(endpointing, phone) ?? '')
|
||||
if (error) return error
|
||||
this.cloud = cloud
|
||||
cloud.start()
|
||||
return ''
|
||||
}
|
||||
stopPhoneCapture(): void {
|
||||
if (this.source !== 'phone') return
|
||||
@@ -111,6 +143,7 @@ export class IosVoiceControlBridge {
|
||||
this.capturing = this.finalizing = false
|
||||
this.source = null
|
||||
this.clearAudioTimer(); this.releaseMicrophone(); this.native?.cancel()
|
||||
this.clearCloud()
|
||||
this.finishCompletion()
|
||||
}
|
||||
handleSessionEnded(message = 'Glasses disconnected. Start voice input again after reconnecting.'): void {
|
||||
@@ -122,6 +155,33 @@ export class IosVoiceControlBridge {
|
||||
const session = this.session; this.session = null
|
||||
if (session) void session.setMicrophone(false).catch(error => this.log(`Voice microphone cleanup: ${error}`))
|
||||
}
|
||||
private clearCloud(): void {
|
||||
if (this.cloudTimer !== null) clearTimeout(this.cloudTimer)
|
||||
this.cloudTimer = null
|
||||
this.cloud?.stop(); this.cloud = null; this.cloudEndStatus = null
|
||||
}
|
||||
/** The native audio has ended; have the provider finalize what it heard. */
|
||||
private finishCloud(status: string): void {
|
||||
this.capturing = false; this.finalizing = true
|
||||
this.cloudEndStatus = status
|
||||
this.setStatus('Recognizing…')
|
||||
// Armed first: finish() can deliver the final synchronously.
|
||||
this.cloudTimer = setTimeout(() => this.completeCloud(), CLOUD_FINAL_TIMEOUT_MS)
|
||||
this.cloud?.finish()
|
||||
}
|
||||
private completeCloud(): void {
|
||||
const status = this.cloudEndStatus ?? 'Ready to send'
|
||||
this.clearCloud()
|
||||
this.capturing = this.finalizing = false
|
||||
this.finishCompletion()
|
||||
this.setStatus(status); this.emitEnd()
|
||||
}
|
||||
private receiveCloudTranscript(event: CloudSttTranscriptEvent): void {
|
||||
if (!this.capturing && !this.finalizing) return
|
||||
for (const fn of [...this.transcripts]) fn(event)
|
||||
// Push-to-talk commits once, so the first final after it is the last.
|
||||
if (event.isFinal && this.cloudEndStatus !== null) this.completeCloud()
|
||||
}
|
||||
private emitEnd(): void { for (const fn of [...this.ends]) fn() }
|
||||
private finishCompletion(): void {
|
||||
const resolve = this.resolveCompletion; this.resolveCompletion = null; resolve?.()
|
||||
@@ -134,13 +194,15 @@ export class IosVoiceControlBridge {
|
||||
} else if (event.kind === 'status') this.setStatus(event.message ?? '')
|
||||
else if (event.kind === 'audio') {
|
||||
this.log(`Voice audio: ${event.seconds?.toFixed(1)}s, packets=${event.packets}, RMS=${event.rms?.toFixed(4)}, missing=${event.missing}, errors=${event.errors}`)
|
||||
if (this.capturing) this.setStatus(`Listening on ${this.source === 'phone' ? 'phone' : 'glasses'}… ${event.seconds?.toFixed(0)}s`)
|
||||
// A cloud provider's own status (connecting, listening, reconnecting) stays up.
|
||||
if (this.capturing && !this.cloud) this.setStatus(`Listening on ${this.source === 'phone' ? 'phone' : 'glasses'}… ${event.seconds?.toFixed(0)}s`)
|
||||
} else if (event.kind === 'finishing') {
|
||||
void this.stopPushToTalk(); this.emitEnd()
|
||||
} else if (event.kind === 'ended') {
|
||||
this.source = null
|
||||
this.capturing = this.finalizing = false
|
||||
this.clearAudioTimer(); this.releaseMicrophone()
|
||||
if (this.cloud) { this.finishCloud(event.message ?? 'Ready to send'); return }
|
||||
this.capturing = this.finalizing = false
|
||||
this.finishCompletion()
|
||||
this.setStatus(event.message ?? 'Ready to send'); this.emitEnd()
|
||||
}
|
||||
|
||||
@@ -1,11 +1,8 @@
|
||||
import { Utils } from "@nativescript/core";
|
||||
|
||||
import { type CloudSttTranscriptEvent, type CloudSttOptions, CloudSttClient } from "./cloud-stt";
|
||||
import { ReconnectingSttClient } from "./reconnecting-stt";
|
||||
import { type CloudSttTranscriptEvent, CloudSttClient } from "./cloud-stt";
|
||||
import { createCloudSttClient } from "./cloud-stt-provider";
|
||||
import { SpeechPauseDetector } from "./speech-pause";
|
||||
import { ElevenLabsSttClient } from "./elevenlabs-stt";
|
||||
import { OpenAiRealtimeSttClient } from "./openai-stt";
|
||||
import { SonioxSttClient } from "./soniox-stt";
|
||||
import { toUint8Array } from "../util/array-util";
|
||||
|
||||
declare const com: any;
|
||||
@@ -341,38 +338,12 @@ export class FaceclawVoiceControlBridge {
|
||||
* than failing the capture outright.
|
||||
*/
|
||||
private createCloudClient(options: PushToTalkOptions): CloudSttClient | null {
|
||||
if (options.provider === "onboard" || options.provider === "onboard-whisper") return null;
|
||||
const sttOptions = {
|
||||
apiKey: "",
|
||||
return createCloudSttClient(options, {
|
||||
onTranscript: (event: CloudSttTranscriptEvent) =>
|
||||
this.emitTranscript(event.text, event.isFinal, event),
|
||||
onStatus: (status: string) => this.setStatus(status),
|
||||
onError: (message: string) => this.setStatus(message),
|
||||
};
|
||||
const reconnecting = (create: (options: CloudSttOptions) => CloudSttClient, apiKey: string) =>
|
||||
new ReconnectingSttClient(create, { ...sttOptions, apiKey }, () => this.captureHolders.has("continuous"));
|
||||
if (options.provider === "elevenlabs") {
|
||||
const apiKey = options.elevenLabsApiKey.trim();
|
||||
if (!apiKey) {
|
||||
this.setStatus("No ElevenLabs key set; using on-device voice.");
|
||||
return null;
|
||||
}
|
||||
return reconnecting((config) => new ElevenLabsSttClient(config), apiKey);
|
||||
}
|
||||
if (options.provider === "soniox") {
|
||||
const apiKey = options.sonioxApiKey.trim();
|
||||
if (!apiKey) {
|
||||
this.setStatus("No Soniox key set; using on-device voice.");
|
||||
return null;
|
||||
}
|
||||
return reconnecting((config) => new SonioxSttClient(config), apiKey);
|
||||
}
|
||||
const apiKey = options.openAiApiKey.trim();
|
||||
if (!apiKey) {
|
||||
this.setStatus("No OpenAI key set; using on-device voice.");
|
||||
return null;
|
||||
}
|
||||
return reconnecting((config) => new OpenAiRealtimeSttClient(config), apiKey);
|
||||
}, () => this.captureHolders.has("continuous"));
|
||||
}
|
||||
|
||||
private releaseCapture(holder: CaptureHolder, commit: boolean): void {
|
||||
|
||||
@@ -803,8 +803,10 @@ export const ringConnectionModeSetting = new ConfigSettingEnum<RingConnectionMod
|
||||
// below now says "OpenAI" to tell the two apart in the picker.
|
||||
export type VoiceProvider = "onboard" | "onboard-whisper" | "elevenlabs" | "whisper" | "soniox";
|
||||
|
||||
// On iOS "onboard" is Apple's on-device recognizer, and there is no
|
||||
// "onboard-whisper" (the sherpa-onnx models are Android-only).
|
||||
const voiceProviderLabels: Record<VoiceProvider, string> = {
|
||||
onboard: "On-device (Moonshine)",
|
||||
onboard: global.isIOS ? "On-device (Apple)" : "On-device (Moonshine)",
|
||||
"onboard-whisper": "On-device (Whisper)",
|
||||
elevenlabs: "ElevenLabs",
|
||||
whisper: "OpenAI (Whisper)",
|
||||
@@ -816,7 +818,7 @@ export const voiceProviderSetting = new ConfigSettingEnum<VoiceProvider>({
|
||||
label: "Transcription Provider",
|
||||
storageKey: "voice.provider",
|
||||
defaultValue: "onboard",
|
||||
values: ["onboard", "onboard-whisper", "elevenlabs", "whisper", "soniox"],
|
||||
values: global.isIOS ? ["onboard", "elevenlabs", "whisper", "soniox"] : ["onboard", "onboard-whisper", "elevenlabs", "whisper", "soniox"],
|
||||
formatValue: (value) => voiceProviderLabels[value] ?? value,
|
||||
isDisabled: (value) => {
|
||||
if (value === "elevenlabs") return elevenLabsApiKeySetting.get().trim().length === 0;
|
||||
@@ -824,7 +826,9 @@ export const voiceProviderSetting = new ConfigSettingEnum<VoiceProvider>({
|
||||
if (value === "soniox") return sonioxApiKeySetting.get().trim().length === 0;
|
||||
return false;
|
||||
},
|
||||
description: "Speech-to-text engine for voice input. ElevenLabs, OpenAI, and Soniox are cloud services that need an API key, with significantly better accuracy than on-device transcription. The two On-device options need their voice model downloaded (below) and never leave the phone.",
|
||||
description: global.isIOS
|
||||
? "Speech-to-text engine for voice input. ElevenLabs, OpenAI, and Soniox are cloud services that need an API key (Settings > API Keys), with significantly better accuracy than on-device transcription. On-device uses your iPhone's speech language and never leaves the phone."
|
||||
: "Speech-to-text engine for voice input. ElevenLabs, OpenAI, and Soniox are cloud services that need an API key, with significantly better accuracy than on-device transcription. The two On-device options need their voice model downloaded (below) and never leave the phone.",
|
||||
});
|
||||
|
||||
const wakeWordActionLabels: Record<WakeWordAction, string> = {
|
||||
|
||||
@@ -233,10 +233,9 @@ function settingsSections(): SettingsSection[] {
|
||||
const deferred = new Set(["Watch"]);
|
||||
return sections.map(section => {
|
||||
if (section.label === "Developer") return { ...section, items: [toggleSettingMenuItem(showBleBandwidthSetting)] };
|
||||
if (section.label === "Voice") return { label: "Voice", items: [enumSettingMenuItem(wakeWordActionSetting), {
|
||||
label: "On-device dictation (Apple)", disabled: true, onSelect: () => {},
|
||||
description: "Uses the glasses microphone and your iPhone's speech language. No transcription API key needed. Say Hey Even for hands-free input, or open Voice from the menu and click when finished.",
|
||||
}] };
|
||||
// No downloadable on-device models: iOS's on-device option is Apple's recognizer.
|
||||
if (section.label === "Voice") return { label: "Voice", items: [enumSettingMenuItem(wakeWordActionSetting),
|
||||
enumSettingMenuItem(voiceProviderSetting)] };
|
||||
if (deferred.has(section.label)) return { label: section.label, items: [{
|
||||
label: "Not available on iOS yet", disabled: true, onSelect: () => {},
|
||||
description: `${section.label} integration has not been ported to iOS.`,
|
||||
|
||||
@@ -690,7 +690,7 @@ test('resizing a terminal reconnects with the new grid and ignores callbacks fro
|
||||
},
|
||||
} } },
|
||||
};
|
||||
const socketContext = { exports: {}, com: context.com };
|
||||
const socketContext = { exports: {}, com: context.com, require: () => ({ toJavaBytes: bytes => bytes }) };
|
||||
vm.runInNewContext(js(read('app/native/socket.ts')), socketContext);
|
||||
context.require = name => name === './socket' ? socketContext.exports : {};
|
||||
vm.runInNewContext(js(read('app/native/g2mirror-client.ts')), context);
|
||||
|
||||
@@ -193,3 +193,28 @@ test('an unsolicited Soniox session end finalizes visible text and reconnects',
|
||||
assert.equal(h.sockets.length, 2);
|
||||
h.client.stop();
|
||||
});
|
||||
|
||||
test('provider selection falls back to on-device without a key and connects the chosen provider with one', () => {
|
||||
const sockets = [];
|
||||
const load = loader({
|
||||
setTimeout() {}, clearTimeout() {}, console: { log() {}, warn() {} },
|
||||
com: { faceclaw: { app: {
|
||||
FaceclawWebSocketListener: function(callbacks) { return callbacks; },
|
||||
FaceclawWebSocket: class { constructor(url, _listener, header, value) { sockets.push({ url, header, value }); } },
|
||||
} } },
|
||||
}, { './cloud-stt': { CLOUD_STT_SAMPLE_RATE: 16000, encodeBase64: () => '', toJavaBytes: bytes => bytes } });
|
||||
const { createCloudSttClient, usesCloudStt } = load('app/native/cloud-stt-provider.ts');
|
||||
const statuses = [];
|
||||
const options = { onTranscript() {}, onStatus: status => statuses.push(status), onError() {} };
|
||||
const selection = { provider: 'onboard', elevenLabsApiKey: '', openAiApiKey: ' ', sonioxApiKey: 'soniox-key' };
|
||||
assert.equal(usesCloudStt(selection), false);
|
||||
assert.equal(createCloudSttClient(selection, options, () => false), null);
|
||||
assert.equal(usesCloudStt({ ...selection, provider: 'whisper' }), false);
|
||||
assert.equal(createCloudSttClient({ ...selection, provider: 'whisper' }, options, () => false), null);
|
||||
assert.deepEqual(statuses, ['No OpenAI key set; using on-device voice.']);
|
||||
assert.equal(usesCloudStt({ ...selection, provider: 'soniox' }), true);
|
||||
createCloudSttClient({ ...selection, provider: 'soniox' }, options, () => false).start();
|
||||
createCloudSttClient({ ...selection, provider: 'elevenlabs', elevenLabsApiKey: ' xi ' }, options, () => false).start();
|
||||
assert.deepEqual(sockets.map(({ url, header, value }) => [new URL(url).host, header, value]),
|
||||
[['stt-rt.soniox.com', null, null], ['api.elevenlabs.io', 'xi-api-key', 'xi']]);
|
||||
});
|
||||
@@ -4,6 +4,9 @@ const fs = require('node:fs');
|
||||
const vm = require('node:vm');
|
||||
const ts = require('typescript');
|
||||
const path = require('node:path');
|
||||
// The provider settings ios-preview-controller hands the voice bridge.
|
||||
const voiceSettings = { voiceProviderSetting: { get: () => 'onboard' }, elevenLabsApiKeySetting: { get: () => '' },
|
||||
openAiApiKeySetting: { get: () => '' }, sonioxApiKeySetting: { get: () => '' } };
|
||||
function load(file, context) {
|
||||
const sandbox = { exports: {}, ...context };
|
||||
const source = fs.readFileSync(path.join(__dirname, '..', file), 'utf8');
|
||||
@@ -80,7 +83,8 @@ test('iOS settings prompts honor password input and report cancellation', async
|
||||
test('iOS host forwards hands-free capture mode to the voice bridge', async () => {
|
||||
const captures = [];
|
||||
const { IosPreviewController } = load('app/g2/ios-preview-controller.ts', { require: id => id === '../native/ios-voice-input'
|
||||
? { iosVoiceInput: { startGlassesCapture: (_session, _log, endpointing) => captures.push(endpointing) } } : {} });
|
||||
? { iosVoiceInput: { startGlassesCapture: (_session, _voice, _log, endpointing) => captures.push(endpointing) } }
|
||||
: id === '../ui/dashboard-settings' ? voiceSettings : {} });
|
||||
const host = Object.create(IosPreviewController.prototype);
|
||||
host.communicator = {}; host.state = { phase: 'connected' };
|
||||
await host.startVoiceCapture(true);
|
||||
@@ -304,20 +308,20 @@ test('iOS opens the on-glasses onboarding on a new acknowledged frame, once per
|
||||
|
||||
test('iOS preview voice uses the phone microphone, while connected capture stays on glasses', async () => {
|
||||
const calls = [];
|
||||
const bridge = { prepare: async (foreground, phone) => { calls.push(['prepare', foreground, phone]); return true; },
|
||||
startPhoneCapture: async (_log, endpointing) => calls.push(['phone', endpointing]),
|
||||
startGlassesCapture: async (_session, _log, endpointing) => calls.push(['glasses', endpointing]) };
|
||||
const bridge = { prepare: async (foreground, phone, voice) => { calls.push(['prepare', foreground, phone, voice.provider]); return true; },
|
||||
startPhoneCapture: async (voice, _log, endpointing) => calls.push(['phone', voice.provider, endpointing]),
|
||||
startGlassesCapture: async (_session, voice, _log, endpointing) => calls.push(['glasses', voice.provider, endpointing]) };
|
||||
const { IosPreviewController } = load('app/g2/ios-preview-controller.ts', {
|
||||
require: id => id === '../native/ios-voice-input' ? { iosVoiceInput: bridge } : {},
|
||||
require: id => id === '../native/ios-voice-input' ? { iosVoiceInput: bridge } : id === '../ui/dashboard-settings' ? voiceSettings : {},
|
||||
});
|
||||
const host = Object.create(IosPreviewController.prototype);
|
||||
host.active = true; host.glassesLocked = false; host.communicator = null; host.state = { phase: 'disconnected' };
|
||||
assert.equal(await host.prepareVoiceCapture(), true);
|
||||
await host.startVoiceCapture(true);
|
||||
assert.deepEqual(calls, [['prepare', true, true], ['prepare', true, true], ['phone', true]]);
|
||||
assert.deepEqual(calls, [['prepare', true, true, 'onboard'], ['prepare', true, true, 'onboard'], ['phone', 'onboard', true]]);
|
||||
calls.length = 0; host.communicator = {}; host.state = { phase: 'connected' };
|
||||
assert.equal(await host.prepareVoiceCapture(), true); await host.startVoiceCapture(false);
|
||||
assert.deepEqual(calls, [['prepare', true, false], ['glasses', false]]);
|
||||
assert.deepEqual(calls, [['prepare', true, false, 'onboard'], ['glasses', 'onboard', false]]);
|
||||
calls.length = 0; host.active = false; host.communicator = null; host.state = { phase: 'disconnected' };
|
||||
assert.equal(await host.prepareVoiceCapture(), false); await host.startVoiceCapture();
|
||||
assert.deepEqual(calls, []);
|
||||
|
||||
@@ -7,8 +7,13 @@ const { join, resolve } = require('node:path');
|
||||
const root = resolve(__dirname, '..'), dir = mkdtempSync(join(tmpdir(), 'faceclaw-socket-'));
|
||||
execFileSync('xcrun', ['clang', '-fobjc-arc', '-framework', 'Foundation', '-I', 'App_Resources/iOS/src',
|
||||
'tests/native/ios-socket-test.m', 'App_Resources/iOS/src/FaceclawSocket.m', '-o', join(dir, 'socket-test')], { cwd: root, stdio: 'inherit' });
|
||||
const server = new WebSocketServer({ host: '127.0.0.1', port: 0 });
|
||||
server.on('connection', socket => socket.on('message', data => socket.send(data.toString(), () => socket.close(1000, 'done'))));
|
||||
// Requires the handshake header; answers binary frames with their hex, echoes text and closes.
|
||||
const server = new WebSocketServer({ host: '127.0.0.1', port: 0,
|
||||
verifyClient: (info, done) => done(info.req.headers['x-faceclaw-key'] === 'secret', 401, 'Unauthorized') });
|
||||
server.on('connection', socket => socket.on('message', data => {
|
||||
if (Buffer.isBuffer(data)) socket.send(`binary:${data.toString('hex')}`);
|
||||
else socket.send(data, () => socket.close(1000, 'done'));
|
||||
}));
|
||||
server.on('listening', () => {
|
||||
const child = spawn(join(dir, 'socket-test'), [`ws://127.0.0.1:${server.address().port}`], { stdio: 'inherit' });
|
||||
child.on('exit', code => { server.close(); rmSync(dir, { recursive: true, force: true }); process.exitCode = code ?? 1; });
|
||||
|
||||
+97
-20
@@ -4,15 +4,29 @@ const fs = require('node:fs');
|
||||
const vm = require('node:vm');
|
||||
const ts = require('typescript');
|
||||
function harness() {
|
||||
const native = { startWithEndpointing(endpointing) { this.endpointing = endpointing; return ''; }, startPhoneWithEndpointing(endpointing) { this.endpointing = endpointing; this.phone = true; return this.error ?? ''; }, cancel() { this.cancels++; }, cancels: 0, finish() { this.finished = true; }, acceptPacket: p => audio.push(p) };
|
||||
const audio = [], timers = new Map(), commands = [], text = [];
|
||||
const native = { startWithEndpointing(endpointing) { this.endpointing = endpointing; return ''; }, startPhoneWithEndpointing(endpointing) { this.endpointing = endpointing; this.phone = true; return this.error ?? ''; },
|
||||
startPcmCaptureWithEndpointingPhone(endpointing, phone) { this.pcmCapture = { endpointing, phone }; return this.error ?? ''; },
|
||||
cancel() { this.cancels++; }, cancels: 0, finish() { this.finished = true; }, acceptPacket: p => audio.push(p) };
|
||||
const audio = [], timers = new Map(), commands = [], text = [], clouds = [];
|
||||
// Stands in for cloud-stt-provider: any provider but "onboard" is a keyed cloud one.
|
||||
const provider = {
|
||||
usesCloudStt: voice => voice.provider !== 'onboard',
|
||||
createCloudSttClient(voice, options) {
|
||||
if (voice.provider === 'onboard') return null;
|
||||
const client = { options, pcm: [], started: false, finished: 0, stopped: false,
|
||||
start() { this.started = true; }, acceptPcm(bytes) { this.pcm.push([...bytes]); },
|
||||
finish() { this.finished++; }, stop() { this.stopped = true; } };
|
||||
clouds.push(client); return client;
|
||||
},
|
||||
};
|
||||
let authorization = 3, microphoneAuthorization = 3, microphoneRequests = 0, permissionRequests = 0, release;
|
||||
const sandbox = { exports: {}, FaceclawSpeech: { new: () => native, authorizationStatus: () => authorization,
|
||||
requestAuthorization: done => { permissionRequests++; done(3); },
|
||||
microphoneAuthorizationStatus: () => microphoneAuthorization,
|
||||
requestMicrophoneAuthorization: done => { microphoneRequests++; done(3); } },
|
||||
setTimeout: fn => { timers.set(fn, fn); return fn; }, clearTimeout: fn => timers.delete(fn),
|
||||
NSData: { dataWithBytesLength: (b, n) => Buffer.from(b, 0, n) }, interop: { handleof: b => b } };
|
||||
NSData: { dataWithBytesLength: (b, n) => Buffer.from(b, 0, n) }, interop: { handleof: b => b, bufferFromData: d => d },
|
||||
require: name => name === './cloud-stt-provider' ? provider : {} };
|
||||
vm.runInNewContext(ts.transpileModule(fs.readFileSync('app/native/voice-control.ios.ts', 'utf8'), {
|
||||
compilerOptions: { module: ts.ModuleKind.CommonJS, target: ts.ScriptTarget.ES2020 } }).outputText, sandbox);
|
||||
const bridge = new sandbox.exports.IosVoiceControlBridge();
|
||||
@@ -22,20 +36,22 @@ function harness() {
|
||||
if (enabled) { this.packet = fn; return new Promise(resolve => { release = resolve; }); }
|
||||
return Promise.resolve();
|
||||
} };
|
||||
return { bridge, native, session, audio, commands, text, timers, release: () => release(),
|
||||
const onboard = { provider: 'onboard', elevenLabsApiKey: '', openAiApiKey: '', sonioxApiKey: '' };
|
||||
return { bridge, native, session, audio, commands, text, timers, clouds, release: () => release(),
|
||||
onboard, cloud: { ...onboard, provider: 'soniox', sonioxApiKey: 'key' },
|
||||
microphoneAuthorization: value => { microphoneAuthorization = value; }, microphoneRequests: () => microphoneRequests,
|
||||
authorization: value => { authorization = value; }, permissionRequests: () => permissionRequests,
|
||||
event: event => native.eventHandler(JSON.stringify(event)) };
|
||||
}
|
||||
test('iOS speech permission is requested only in foreground and denial stays actionable', async () => {
|
||||
const h = harness(); h.authorization(0);
|
||||
assert.equal(await h.bridge.prepare(false), false); assert.equal(h.permissionRequests(), 0);
|
||||
assert.equal(await h.bridge.prepare(true), true); assert.equal(h.permissionRequests(), 1);
|
||||
h.authorization(2); assert.equal(await h.bridge.prepare(true), false);
|
||||
assert.equal(await h.bridge.prepare(false, false, h.onboard), false); assert.equal(h.permissionRequests(), 0);
|
||||
assert.equal(await h.bridge.prepare(true, false, h.onboard), true); assert.equal(h.permissionRequests(), 1);
|
||||
h.authorization(2); assert.equal(await h.bridge.prepare(true, false, h.onboard), false);
|
||||
assert.match(h.bridge.statusText, /Settings/);
|
||||
});
|
||||
test('glasses speech sends PCM-source packets, replaces partials, and drains final text after manual release', async () => {
|
||||
const h = harness(); const start = h.bridge.startGlassesCapture(h.session, () => {});
|
||||
const h = harness(); const start = h.bridge.startGlassesCapture(h.session, h.onboard, () => {});
|
||||
h.session.packet(new Uint8Array([1, 2, 3])); assert.deepEqual([...h.audio[0]], [1, 2, 3]);
|
||||
h.release(); await start;
|
||||
h.event({ kind: 'transcript', text: 'hello', final: false });
|
||||
@@ -51,7 +67,7 @@ test('glasses speech sends PCM-source packets, replaces partials, and drains fin
|
||||
test('cancel and disconnect during enable cannot resurrect capture or leave its microphone listener active', async () => {
|
||||
for (const stop of ['stop', 'handleSessionEnded']) {
|
||||
const h = harness(); let ends = 0; h.bridge.onSpeechEnd(() => ends++);
|
||||
const start = h.bridge.startGlassesCapture(h.session, () => {});
|
||||
const start = h.bridge.startGlassesCapture(h.session, h.onboard, () => {});
|
||||
h.bridge[stop](); h.release(); await start;
|
||||
h.session.packet(new Uint8Array([9]));
|
||||
h.event({ kind: 'transcript', text: 'stale', final: true });
|
||||
@@ -60,7 +76,7 @@ test('cancel and disconnect during enable cannot resurrect capture or leave its
|
||||
}
|
||||
});
|
||||
test('missing microphone data ends the session and disables the mic with an actionable status', async () => {
|
||||
const h = harness(); const start = h.bridge.startGlassesCapture(h.session, () => {}); h.release(); await start;
|
||||
const h = harness(); const start = h.bridge.startGlassesCapture(h.session, h.onboard, () => {}); h.release(); await start;
|
||||
for (const fn of h.timers.values()) fn();
|
||||
assert.deepEqual(h.commands, [true, false]); assert.match(h.bridge.statusText, /No microphone audio/);
|
||||
});
|
||||
@@ -68,7 +84,7 @@ test('missing microphone data ends the session and disables the mic with an acti
|
||||
test('hands-free endpointing releases the microphone and waits for the final transcript before completing', async () => {
|
||||
const h = harness(); let ends = 0, completed = false;
|
||||
h.bridge.onSpeechEnd(() => ends++);
|
||||
const start = h.bridge.startGlassesCapture(h.session, () => {}, true); h.release(); await start;
|
||||
const start = h.bridge.startGlassesCapture(h.session, h.onboard, () => {}, true); h.release(); await start;
|
||||
assert.equal(h.native.endpointing, true);
|
||||
h.event({ kind: 'transcript', text: 'Set a timer', final: false });
|
||||
h.event({ kind: 'finishing' });
|
||||
@@ -80,27 +96,27 @@ test('hands-free endpointing releases the microphone and waits for the final tra
|
||||
h.event({ kind: 'ended' }); await completion;
|
||||
assert.equal(completed, true);
|
||||
assert.equal(h.text.at(-1).text, 'Set a timer for five minutes.');
|
||||
const manual = h.bridge.startGlassesCapture(h.session, () => {}); h.release(); await manual;
|
||||
const manual = h.bridge.startGlassesCapture(h.session, h.onboard, () => {}); h.release(); await manual;
|
||||
assert.equal(h.native.endpointing, false);
|
||||
const cancelled = h.bridge.stopPushToTalk(); h.bridge.stop(); await cancelled;
|
||||
});
|
||||
|
||||
test('phone microphone permission is requested only for foreground preview capture', async () => {
|
||||
const h = harness(); h.microphoneAuthorization(0);
|
||||
assert.equal(await h.bridge.prepare(true), true);
|
||||
assert.equal(await h.bridge.prepare(true, false, h.onboard), true);
|
||||
assert.equal(h.microphoneRequests(), 0, 'glasses input does not need phone microphone permission');
|
||||
assert.equal(await h.bridge.prepare(false, true), false);
|
||||
assert.equal(await h.bridge.prepare(false, true, h.onboard), false);
|
||||
assert.equal(h.microphoneRequests(), 0);
|
||||
assert.equal(await h.bridge.prepare(true, true), true);
|
||||
assert.equal(await h.bridge.prepare(true, true, h.onboard), true);
|
||||
assert.equal(h.microphoneRequests(), 1);
|
||||
h.microphoneAuthorization(2);
|
||||
assert.equal(await h.bridge.prepare(true, true), false);
|
||||
assert.equal(await h.bridge.prepare(true, true, h.onboard), false);
|
||||
assert.match(h.bridge.statusText, /Microphone.*Settings/);
|
||||
});
|
||||
|
||||
test('phone capture forwards endpointing and drains final text without touching Bluetooth', async () => {
|
||||
const h = harness();
|
||||
await h.bridge.startPhoneCapture(() => {}, true);
|
||||
await h.bridge.startPhoneCapture(h.onboard, () => {}, true);
|
||||
assert.equal(h.native.phone, true); assert.equal(h.native.endpointing, true);
|
||||
h.bridge.handleSessionEnded();
|
||||
h.event({ kind: 'transcript', text: 'First line\nSecond line', final: false });
|
||||
@@ -118,11 +134,11 @@ test('phone capture forwards endpointing and drains final text without touching
|
||||
|
||||
test('background cleanup stops phone capture but preserves glasses capture', async () => {
|
||||
const h = harness(); let ends = 0; h.bridge.onSpeechEnd(() => ends++);
|
||||
await h.bridge.startPhoneCapture(() => {});
|
||||
await h.bridge.startPhoneCapture(h.onboard, () => {});
|
||||
h.bridge.stopPhoneCapture();
|
||||
assert.equal(ends, 1);
|
||||
h.event({ kind: 'transcript', text: 'stale', final: true }); assert.equal(h.text.length, 0);
|
||||
const start = h.bridge.startGlassesCapture(h.session, () => {}); h.release(); await start;
|
||||
const start = h.bridge.startGlassesCapture(h.session, h.onboard, () => {}); h.release(); await start;
|
||||
h.bridge.stopPhoneCapture();
|
||||
h.event({ kind: 'transcript', text: 'on glasses', final: false });
|
||||
assert.equal(h.text.at(-1).text, 'on glasses');
|
||||
@@ -132,7 +148,68 @@ test('background cleanup stops phone capture but preserves glasses capture', asy
|
||||
test('phone microphone startup errors end capture with an actionable status', async () => {
|
||||
const h = harness(); let ends = 0; h.bridge.onSpeechEnd(() => ends++);
|
||||
h.native.error = 'No phone microphone is available.';
|
||||
await h.bridge.startPhoneCapture(() => {});
|
||||
await h.bridge.startPhoneCapture(h.onboard, () => {});
|
||||
assert.equal(ends, 1); assert.match(h.bridge.statusText, /No phone microphone/);
|
||||
await h.bridge.stopPushToTalk();
|
||||
});
|
||||
|
||||
test('a cloud provider needs no Speech Recognition permission, but phone capture still needs the microphone', async () => {
|
||||
const h = harness(); h.authorization(0); h.microphoneAuthorization(0);
|
||||
assert.equal(await h.bridge.prepare(false, false, h.cloud), true);
|
||||
assert.equal(await h.bridge.prepare(false, true, h.cloud), false);
|
||||
assert.equal(await h.bridge.prepare(true, true, h.cloud), true);
|
||||
assert.equal(h.permissionRequests(), 0); assert.equal(h.microphoneRequests(), 1);
|
||||
});
|
||||
|
||||
test('glasses cloud capture streams native PCM and completes on the final after the drained audio', async () => {
|
||||
const h = harness();
|
||||
const start = h.bridge.startGlassesCapture(h.session, h.cloud, () => {});
|
||||
assert.deepEqual({ ...h.native.pcmCapture }, { endpointing: false, phone: false });
|
||||
const client = h.clouds[0]; assert.equal(client.started, true);
|
||||
client.options.onStatus('Listening (Soniox)...');
|
||||
h.session.packet(new Uint8Array([1, 2])); assert.equal(h.audio.length, 1, 'LC3 still decodes natively');
|
||||
h.release(); await start;
|
||||
h.native.pcmHandler(new Uint8Array([5, 6]).buffer);
|
||||
h.event({ kind: 'audio', seconds: 1, packets: 20, rms: 0.1, missing: 0, errors: 0 });
|
||||
assert.equal(h.bridge.statusText, 'Listening (Soniox)...', 'the provider status stays up');
|
||||
client.options.onTranscript({ text: 'hello', isFinal: false });
|
||||
let done = false;
|
||||
const completion = h.bridge.stopPushToTalk().then(() => { done = true; });
|
||||
assert.deepEqual(h.commands, [true, false]); assert.equal(h.native.finished, true);
|
||||
h.event({ kind: 'finishing' });
|
||||
assert.equal(client.finished, 0, 'commit waits for the native audio to drain');
|
||||
h.native.pcmHandler(new Uint8Array([7]).buffer);
|
||||
h.event({ kind: 'ended', message: 'Ready to send' });
|
||||
assert.deepEqual(client.pcm, [[5, 6], [7]]); assert.equal(client.finished, 1);
|
||||
await Promise.resolve(); assert.equal(done, false);
|
||||
client.options.onTranscript({ text: 'Hello there.', isFinal: true });
|
||||
await completion;
|
||||
assert.equal(client.stopped, true); assert.equal(h.bridge.statusText, 'Ready to send');
|
||||
assert.deepEqual(h.text, [{ text: 'hello', isFinal: false }, { text: 'Hello there.', isFinal: true }]);
|
||||
assert.equal(h.timers.size, 0);
|
||||
});
|
||||
|
||||
test('cloud capture times out a missing final, keeps native errors, and ends on provider errors', async () => {
|
||||
const h = harness();
|
||||
await h.bridge.startPhoneCapture(h.cloud, () => {}, true);
|
||||
assert.deepEqual({ ...h.native.pcmCapture }, { endpointing: true, phone: true });
|
||||
const completion = h.bridge.stopPushToTalk();
|
||||
h.event({ kind: 'ended', message: 'Phone microphone interrupted.' });
|
||||
assert.equal(h.clouds[0].finished, 1); assert.equal(h.timers.size, 1);
|
||||
for (const fn of [...h.timers.values()]) fn();
|
||||
await completion;
|
||||
assert.equal(h.clouds[0].stopped, true); assert.equal(h.bridge.statusText, 'Phone microphone interrupted.');
|
||||
|
||||
let ends = 0; h.bridge.onSpeechEnd(() => ends++);
|
||||
await h.bridge.startPhoneCapture(h.cloud, () => {});
|
||||
h.clouds[1].options.onError('Soniox: Invalid API key');
|
||||
assert.equal(h.clouds[1].stopped, true); assert.equal(ends, 1);
|
||||
assert.match(h.bridge.statusText, /Invalid API key/);
|
||||
h.clouds[1].options.onTranscript({ text: 'stale', isFinal: true });
|
||||
assert.equal(h.text.length, 0);
|
||||
|
||||
h.native.error = 'No phone microphone is available.';
|
||||
await h.bridge.startPhoneCapture(h.cloud, () => {});
|
||||
assert.equal(h.clouds[2].started, false, 'no connection without audio');
|
||||
assert.match(h.bridge.statusText, /No phone microphone/);
|
||||
});
|
||||
@@ -1,23 +1,39 @@
|
||||
#import <Foundation/Foundation.h>
|
||||
#import "FaceclawSocket.h"
|
||||
/** Drains the socket's events until it closes; handler returns NO to stop early. */
|
||||
static void pump(FaceclawSocket *socket, BOOL (^handler)(NSDictionary *event)) {
|
||||
NSDate *deadline = [NSDate dateWithTimeIntervalSinceNow:10];
|
||||
while (deadline.timeIntervalSinceNow > 0) {
|
||||
NSArray *events = [NSJSONSerialization JSONObjectWithData:[[socket takeEvents] dataUsingEncoding:NSUTF8StringEncoding] options:0 error:nil];
|
||||
for (NSDictionary *event in events) if (!handler(event)) return;
|
||||
[[NSRunLoop currentRunLoop] runUntilDate:[NSDate dateWithTimeIntervalSinceNow:0.01]];
|
||||
}
|
||||
}
|
||||
int main(int argc, const char **argv) {
|
||||
@autoreleasepool {
|
||||
FaceclawSocket *socket = [[FaceclawSocket alloc] initWithURL:[NSString stringWithUTF8String:argv[1]]];
|
||||
BOOL opened = NO, echoed = NO, closed = NO;
|
||||
NSDate *deadline = [NSDate dateWithTimeIntervalSinceNow:10];
|
||||
while (!closed && deadline.timeIntervalSinceNow > 0) {
|
||||
NSArray *events = [NSJSONSerialization JSONObjectWithData:[[socket takeEvents] dataUsingEncoding:NSUTF8StringEncoding] options:0 error:nil];
|
||||
for (NSDictionary *event in events) {
|
||||
NSString *kind = event[@"kind"];
|
||||
if ([kind isEqual:@"open"]) { opened = YES; [socket sendText:@"Faceclaw echo: π 🔋"]; }
|
||||
if ([kind isEqual:@"text"]) echoed = [event[@"text"] isEqual:@"Faceclaw echo: π 🔋"];
|
||||
if ([kind isEqual:@"closed"] || [kind isEqual:@"error"]) closed = YES;
|
||||
}
|
||||
[[NSRunLoop currentRunLoop] runUntilDate:[NSDate dateWithTimeIntervalSinceNow:0.01]];
|
||||
}
|
||||
NSString *url = [NSString stringWithUTF8String:argv[1]];
|
||||
FaceclawSocket *socket = [[FaceclawSocket alloc] initWithURL:url headerName:@"X-Faceclaw-Key" headerValue:@"secret"];
|
||||
__block BOOL opened = NO, binary = NO, echoed = NO, closed = NO;
|
||||
pump(socket, ^BOOL(NSDictionary *event) {
|
||||
NSString *kind = event[@"kind"];
|
||||
if ([kind isEqual:@"open"]) { opened = YES; const uint8_t bytes[] = {1, 2, 255}; [socket sendData:[NSData dataWithBytes:bytes length:3]]; }
|
||||
if ([kind isEqual:@"text"] && [event[@"text"] isEqual:@"binary:0102ff"]) { binary = YES; [socket sendText:@"Faceclaw echo: π 🔋"]; }
|
||||
if ([kind isEqual:@"text"] && [event[@"text"] isEqual:@"Faceclaw echo: π 🔋"]) echoed = YES;
|
||||
if ([kind isEqual:@"closed"] || [kind isEqual:@"error"]) { closed = YES; return NO; }
|
||||
return YES;
|
||||
});
|
||||
[socket close];
|
||||
if (!opened || !echoed || !closed) { NSLog(@"FAIL open=%d echo=%d close=%d", opened, echoed, closed); return 1; }
|
||||
puts("PASS: native WebSocket opens, echoes UTF-8 and reports close");
|
||||
if (!opened || !binary || !echoed || !closed) { NSLog(@"FAIL open=%d binary=%d echo=%d close=%d", opened, binary, echoed, closed); return 1; }
|
||||
// Without the header the server refuses the handshake; the error names the status.
|
||||
FaceclawSocket *refused = [[FaceclawSocket alloc] initWithURL:url];
|
||||
__block NSString *error = nil;
|
||||
pump(refused, ^BOOL(NSDictionary *event) {
|
||||
if ([event[@"kind"] isEqual:@"error"]) { error = event[@"text"]; return NO; }
|
||||
return YES;
|
||||
});
|
||||
[refused close];
|
||||
if (![error containsString:@"HTTP 401"]) { NSLog(@"FAIL refused handshake error=%@", error); return 1; }
|
||||
puts("PASS: native WebSocket sends a handshake header, binary and UTF-8 frames, reports close and refused status");
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
@@ -171,6 +171,7 @@ test('native completion is capture-specific, follows the final, and does not wai
|
||||
const { FaceclawVoiceControlBridge } = load('app/native/voice-control.ts', {
|
||||
'@nativescript/core': { Utils: { android: { getApplicationContext: () => ({}) } } },
|
||||
'./speech-pause': { SpeechPauseDetector: class { reset() {} } },
|
||||
'./cloud-stt-provider': { createCloudSttClient: () => null },
|
||||
}, { global: { isAndroid: true }, com: { faceclaw: { app: {
|
||||
FaceclawVoiceController: Controller,
|
||||
FaceclawVoiceControllerListener: function (listener) { return listener; },
|
||||
|
||||
Reference in new issue
Block a user