diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawWebSocket.java b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawWebSocket.java index 0d4bb2c5..97f2a631 100644 --- a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawWebSocket.java +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawWebSocket.java @@ -11,10 +11,11 @@ import okhttp3.Request; import okhttp3.Response; import okhttp3.WebSocket; import okhttp3.WebSocketListener; +import okio.ByteString; /** - * Thin okhttp WebSocket wrapper for the TypeScript side. Text frames only - * (the g2mirror protocol is JSON-in-text-frames). Listener callbacks are + * Thin okhttp WebSocket wrapper for the TypeScript side. Sends text frames + * (JSON protocols) and binary frames (raw audio for Soniox). Listener callbacks are * posted to the Looper of the thread that constructed this object, so a JS * isolate (main thread or app worker) always receives them on its own * thread; a Looper-less constructing thread falls back to the main thread. @@ -132,6 +133,10 @@ public class FaceclawWebSocket { return socket.send(message == null ? "" : message); } + public boolean sendBinary(byte[] bytes) { + return socket.send(ByteString.of(bytes == null ? new byte[0] : bytes)); + } + public void close(int code, String reason) { closeRequested = true; try { diff --git a/PRIVACY b/PRIVACY index 7dd0c33a..e8b4089b 100644 --- a/PRIVACY +++ b/PRIVACY @@ -11,7 +11,8 @@ on-device model is selected, audio never leaves your device, and is not stored after it is processed. Users can also provide their own API key for a third-party transcription -service such as OpenAI Whisper or ElevenLabs, which may offer better accuracy. +service such as OpenAI Whisper, ElevenLabs, or Soniox, which may offer better +accuracy. Faceclaw never transmits audio to any party other than a user-selected transcription service. If a third-party transcription service is selected, audio will be sent to that third party, is linked to the account with which @@ -20,6 +21,7 @@ policies: OpenAI: https://openai.com/policies/row-privacy-policy/ ElevenLabs: https://elevenlabs.io/privacy-policy + Soniox: https://soniox.com/policies/privacy-policy Third party data retention and privacy policies may differ based on the API key provided. API keys are part of a relationship between the user and the API diff --git a/README.md b/README.md index 045b6b69..7e0656b9 100644 --- a/README.md +++ b/README.md @@ -38,7 +38,7 @@ To install, run ## Features * A voice assistant that wakes up when you say "Hey Even", transcribes text - with an on-device model, OpenAI Whisper, or ElevenLabs API, and responds + with an on-device model, OpenAI Whisper, ElevenLabs, or Soniox API, and responds to queries and commands using an Anthropic or OpenAI model (requires an API key), or using your own long-running OpenClaw agent (see "Connecting an external agent" below). diff --git a/app/g2/dashboard-controller.ts b/app/g2/dashboard-controller.ts index f19da239..b661d647 100644 --- a/app/g2/dashboard-controller.ts +++ b/app/g2/dashboard-controller.ts @@ -31,7 +31,7 @@ import { type InProcessAppOptions, type InProcessWindow } from "../ui/shell/in-p import { loadPersistedOpenApps, savePersistedOpenApps } from "../ui/shell/open-apps-persistence"; import { appViewportRect, type WindowHeightMode } from "../ui/shell/geometry"; import { type LayerActions } from "../ui/layers"; -import { assistantAllowProactiveSetting, assistantBackendSetting, assistantBridgeHostSetting, assistantBridgePortSetting, assistantBridgeTokenSetting, brightnessSetting, brightnessSettingToLevel, elevenLabsApiKeySetting, openAiApiKeySetting, nightscoutApiTokenSetting, firmwareDebugFlagsSetting, lockScreenEnabledSetting, nightscoutSiteUrlSetting, onAnySettingChanged, saveVoiceRecordingsSetting, screenTimeoutSetting, screenTimeoutSettingToMs, suspendEvenHubWhenScreenOffSetting, verticalPositionSetting, voiceProviderSetting, wakeWordActionSetting, type ConfigSettingString } from "../ui/dashboard-settings"; +import { assistantAllowProactiveSetting, assistantBackendSetting, assistantBridgeHostSetting, assistantBridgePortSetting, assistantBridgeTokenSetting, brightnessSetting, brightnessSettingToLevel, elevenLabsApiKeySetting, openAiApiKeySetting, nightscoutApiTokenSetting, firmwareDebugFlagsSetting, lockScreenEnabledSetting, nightscoutSiteUrlSetting, onAnySettingChanged, saveVoiceRecordingsSetting, sonioxApiKeySetting, screenTimeoutSetting, screenTimeoutSettingToMs, suspendEvenHubWhenScreenOffSetting, verticalPositionSetting, voiceProviderSetting, wakeWordActionSetting, type ConfigSettingString } from "../ui/dashboard-settings"; import { isIgnoringBatteryOptimizations, requestIgnoreBatteryOptimizations } from "../native/battery-optimization"; type ConnectionPhase = "disconnected" | "connecting" | "connected" | "charging" | "disconnecting"; @@ -1194,6 +1194,7 @@ class DashboardController { provider: voiceProviderSetting.get(), elevenLabsApiKey: elevenLabsApiKeySetting.get(), openAiApiKey: openAiApiKeySetting.get(), + sonioxApiKey: sonioxApiKeySetting.get(), saveRecording: saveVoiceRecordingsSetting.get(), endpointing, }; diff --git a/app/native/cloud-stt.ts b/app/native/cloud-stt.ts index 95c73a1d..83c9c76f 100644 --- a/app/native/cloud-stt.ts +++ b/app/native/cloud-stt.ts @@ -1,5 +1,6 @@ /** - * Shared shape for the cloud speech-to-text providers (ElevenLabs, Whisper). + * Shared shape for the cloud speech-to-text providers (ElevenLabs, Whisper, + * Soniox). * The voice bridge holds one of these while a cloud provider owns the * transcript; the Java controller then only decodes LC3 to PCM and hands it * over via acceptPcm. @@ -33,13 +34,18 @@ export const CLOUD_STT_SAMPLE_RATE = 16000; declare const android: any; -/** Base64 for audio payloads, via the Android SDK (no JS base64 in NS core). */ -export function encodeBase64(bytes: Uint8Array): string { - if (!global.isAndroid) return ""; +/** Copy a Uint8Array into a Java byte[] (for binary WebSocket frames etc.). */ +export function toJavaBytes(bytes: Uint8Array): any { const javaBytes = Array.create("byte", bytes.length); for (let i = 0; i < bytes.length; i++) { const value = bytes[i]!; javaBytes[i] = value > 127 ? value - 256 : value; } - return String(android.util.Base64.encodeToString(javaBytes, android.util.Base64.NO_WRAP)); + return javaBytes; +} + +/** Base64 for audio payloads, via the Android SDK (no JS base64 in NS core). */ +export function encodeBase64(bytes: Uint8Array): string { + if (!global.isAndroid) return ""; + return String(android.util.Base64.encodeToString(toJavaBytes(bytes), android.util.Base64.NO_WRAP)); } diff --git a/app/native/soniox-stt.ts b/app/native/soniox-stt.ts new file mode 100644 index 00000000..6c9a6f33 --- /dev/null +++ b/app/native/soniox-stt.ts @@ -0,0 +1,160 @@ +import { CLOUD_STT_SAMPLE_RATE, CloudSttClient, CloudSttOptions, toJavaBytes } from "./cloud-stt"; + +declare const com: any; + +/** + * Soniox realtime speech-to-text over WebSocket. + * https://soniox.com/docs/stt/rt/real-time-transcription + * + * Unlike the other cloud providers, Soniox authenticates with an api_key field + * in the first (JSON) message rather than an HTTP header, and audio is sent as + * raw binary frames rather than base64-in-JSON. Push-to-talk defines the + * utterance boundary: an empty text frame is the end-of-audio signal, after + * which the server finalizes all tokens and replies with finished=true. + * + * Transcripts arrive as TOKENS: final tokens are delivered exactly once and + * are accumulated here; non-final tokens repeat (revised) in every message and + * are appended for display only. Listeners get full replace-semantics text. + */ + +const WS_URL = "wss://stt-rt.soniox.com/transcribe-websocket"; +const MODEL_ID = "stt-rt-v5"; + +export type SonioxSttOptions = CloudSttOptions; + +export class SonioxSttClient implements CloudSttClient { + private ws: any = null; + private listenerProxy: any = null; + private open = false; + private closed = false; + // PCM queued until the socket opens and the config message is sent. + private readonly pendingPcm: Uint8Array[] = []; + private pendingFinish = false; + // Concatenation of all final tokens so far. + private finalText = ""; + + constructor(private readonly options: SonioxSttOptions) {} + + start(): void { + if (this.ws) return; + this.listenerProxy = new com.faceclaw.app.FaceclawWebSocketListener({ + onOpen: () => { + // The API key rides in the config message; there is no auth header. + this.trySendText( + JSON.stringify({ + api_key: this.options.apiKey, + model: MODEL_ID, + audio_format: "pcm_s16le", + sample_rate: CLOUD_STT_SAMPLE_RATE, + num_channels: 1, + }), + ); + this.open = true; + for (const chunk of this.pendingPcm.splice(0)) { + this.sendPcm(chunk); + } + if (this.pendingFinish) { + this.pendingFinish = false; + this.finish(); + } + this.options.onStatus("Listening (Soniox)..."); + }, + onTextMessage: (message: string) => this.handleMessage(String(message)), + onClosed: () => { + this.open = false; + }, + onFailure: (message: string) => { + if (this.closed) return; + this.options.onError(`Soniox connection failed: ${String(message)}`); + }, + }); + try { + this.ws = new com.faceclaw.app.FaceclawWebSocket(WS_URL, this.listenerProxy, null, null); + this.options.onStatus("Connecting to Soniox..."); + } catch (error) { + this.options.onError(`Soniox connection failed: ${String((error as Error)?.message ?? error)}`); + } + } + + /** Feed PCM (16 kHz signed-16-bit LE), as raw binary frames. */ + acceptPcm(pcm: Uint8Array): void { + if (this.closed || pcm.length === 0) return; + if (this.open) { + this.sendPcm(pcm); + } else { + this.pendingPcm.push(pcm); + } + } + + /** End of utterance: an empty text frame tells Soniox the audio is done. */ + finish(): void { + if (this.closed) return; + if (this.open) { + this.trySendText(""); + } else { + this.pendingFinish = true; + } + } + + stop(): void { + this.closed = true; + if (this.ws) { + try { + this.ws.close(1000, "bye"); + } catch { + // ignore + } + this.ws = null; + } + this.listenerProxy = null; + } + + private sendPcm(pcm: Uint8Array): void { + try { + this.ws?.sendBinary(toJavaBytes(pcm)); + } catch (error) { + console.warn("soniox send failed", error); + } + } + + private trySendText(message: string): void { + try { + this.ws?.sendText(message); + } catch (error) { + console.warn("soniox send failed", error); + } + } + + private handleMessage(text: string): void { + let message: any; + try { + message = JSON.parse(text); + } catch { + return; + } + if (message?.error_code != null) { + this.options.onError(`Soniox: ${String(message.error_message ?? message.error_code)}`); + return; + } + const tokens = Array.isArray(message?.tokens) ? message.tokens : []; + let nonFinal = ""; + for (const token of tokens) { + const tokenText = String(token?.text ?? ""); + // Markers emitted by endpoint detection / manual finalize; not speech. + if (tokenText === "" || tokenText === "") continue; + if (token?.is_final) { + this.finalText += tokenText; + } else { + nonFinal += tokenText; + } + } + if (message?.finished) { + this.options.onTranscript({ text: this.finalText, isFinal: true }); + this.stop(); + return; + } + if (tokens.length > 0) { + this.options.onTranscript({ text: this.finalText + nonFinal, isFinal: false }); + } + } +} diff --git a/app/native/voice-control.ts b/app/native/voice-control.ts index 21df7a1f..da5f10c9 100644 --- a/app/native/voice-control.ts +++ b/app/native/voice-control.ts @@ -3,6 +3,7 @@ import { Utils } from "@nativescript/core"; import { CloudSttClient } from "./cloud-stt"; import { ElevenLabsSttClient } from "./elevenlabs-stt"; import { OpenAiRealtimeSttClient } from "./openai-stt"; +import { SonioxSttClient } from "./soniox-stt"; import { toUint8Array } from "../util/array-util"; declare const com: any; @@ -11,7 +12,7 @@ export type VoiceControlState = { status: string; }; -export type VoiceProviderKind = "onboard" | "elevenlabs" | "whisper"; +export type VoiceProviderKind = "onboard" | "elevenlabs" | "whisper" | "soniox"; export type VoiceTranscriptEvent = { /** @@ -27,6 +28,7 @@ export type PushToTalkOptions = { provider: VoiceProviderKind; elevenLabsApiKey: string; openAiApiKey: string; + sonioxApiKey: string; saveRecording: boolean; /** * Watch the mic and fire onSpeechEnd when the speaker stops. For hands-free @@ -150,6 +152,14 @@ export class FaceclawVoiceControlBridge { } return new ElevenLabsSttClient({ ...sttOptions, apiKey }); } + if (options.provider === "soniox") { + const apiKey = options.sonioxApiKey.trim(); + if (!apiKey) { + this.setStatus("No Soniox key set; using on-device voice."); + return null; + } + return new SonioxSttClient({ ...sttOptions, apiKey }); + } const apiKey = options.openAiApiKey.trim(); if (!apiKey) { this.setStatus("No OpenAI key set; using on-device voice."); diff --git a/app/ui/dashboard-settings.ts b/app/ui/dashboard-settings.ts index 835b2099..258b000d 100644 --- a/app/ui/dashboard-settings.ts +++ b/app/ui/dashboard-settings.ts @@ -297,12 +297,13 @@ export const suspendEvenHubWhenScreenOffSetting = new ConfigSettingBoolean({ description: "Suspend the EvenHub session while the display is off. This significantly improves battery life, but increases the latency of waking the screen.", }); -export type VoiceProvider = "onboard" | "elevenlabs" | "whisper"; +export type VoiceProvider = "onboard" | "elevenlabs" | "whisper" | "soniox"; const voiceProviderLabels: Record = { onboard: "On-device", elevenlabs: "ElevenLabs", whisper: "Whisper", + soniox: "Soniox", }; export const voiceProviderSetting = new ConfigSettingEnum({ @@ -310,9 +311,15 @@ export const voiceProviderSetting = new ConfigSettingEnum({ label: "Transcription Provider", storageKey: "voice.provider", defaultValue: "onboard", - values: ["onboard", "elevenlabs", "whisper"], + values: ["onboard", "elevenlabs", "whisper", "soniox"], formatValue: (value) => voiceProviderLabels[value] ?? value, - description: "Speech-to-text engine for voice input. ElevenLabs and Whisper are cloud services that need an API key, with significantly better accuracy than on-device transcription.", + isDisabled: (value) => { + if (value === "elevenlabs") return elevenLabsApiKeySetting.get().trim().length === 0; + if (value === "whisper") return openAiApiKeySetting.get().trim().length === 0; + if (value === "soniox") return sonioxApiKeySetting.get().trim().length === 0; + return false; + }, + description: "Speech-to-text engine for voice input. ElevenLabs, Whisper, and Soniox are cloud services that need an API key, with significantly better accuracy than on-device transcription.", }); const wakeWordActionLabels: Record = { @@ -428,6 +435,17 @@ export const openAiApiKeySetting = new ConfigSettingString({ description: "OpenAI API key, used when Whisper is the transcription provider or an OpenAI model is selected for the voice assistant.", }); +export const sonioxApiKeySetting = new ConfigSettingString({ + id: "soniox-api-key", + label: "Soniox key", + storageKey: "voice.sonioxApiKey", + defaultValue: "", + editorTitle: "Soniox API key", + glassesEditTitle: "Edit Soniox key", + formatValue: (value) => (value ? `${value.slice(0, 6)}...` : "(not set)"), + description: "Soniox API key, used when Soniox is the transcription provider.", +}); + export const anthropicApiKeySetting = new ConfigSettingString({ id: "anthropic-api-key", label: "Anthropic key", diff --git a/app/ui/dashboard/settings-menus.ts b/app/ui/dashboard/settings-menus.ts index bd7fd50e..93fe7c0c 100644 --- a/app/ui/dashboard/settings-menus.ts +++ b/app/ui/dashboard/settings-menus.ts @@ -22,6 +22,7 @@ import { openAiApiKeySetting, roamApiTokenSetting, roamGraphNameSetting, + sonioxApiKeySetting, enumSettingMenuItem, firmwareDebugFlagsSetting, lockScreenEnabledSetting, @@ -100,6 +101,7 @@ function settingsSections(): SettingsSection[] { items: [ textSettingMenuItem(elevenLabsApiKeySetting), textSettingMenuItem(openAiApiKeySetting), + textSettingMenuItem(sonioxApiKeySetting), textSettingMenuItem(anthropicApiKeySetting), textSettingMenuItem(mapboxApiKeySetting), ],