Add Soniox as an API speech-to-text option

This commit is contained in:
James Babcock committed 2026-08-04 20:15:39 -07:00
1 parent 852626e950
commit ba81e46624
9 files changed
+218 -14

No files matched your search

@@ -11,10 +11,11 @@ import okhttp3.Request;
import okhttp3.Response;
import okhttp3.WebSocket;
import okhttp3.WebSocketListener;
import okio.ByteString;
/**
* Thin okhttp WebSocket wrapper for the TypeScript side. Text frames only
* (the g2mirror protocol is JSON-in-text-frames). Listener callbacks are
* Thin okhttp WebSocket wrapper for the TypeScript side. Sends text frames
* (JSON protocols) and binary frames (raw audio for Soniox). Listener callbacks are
* posted to the Looper of the thread that constructed this object, so a JS
* isolate (main thread or app worker) always receives them on its own
* thread; a Looper-less constructing thread falls back to the main thread.
@@ -132,6 +133,10 @@ public class FaceclawWebSocket {
return socket.send(message == null ? "" : message);
}
public boolean sendBinary(byte[] bytes) {
return socket.send(ByteString.of(bytes == null ? new byte[0] : bytes));
}
public void close(int code, String reason) {
closeRequested = true;
try {
+3 -1
View File
@@ -11,7 +11,8 @@ on-device model is selected, audio never leaves your device, and is not stored
after it is processed.
Users can also provide their own API key for a third-party transcription
service such as OpenAI Whisper or ElevenLabs, which may offer better accuracy.
service such as OpenAI Whisper, ElevenLabs, or Soniox, which may offer better
accuracy.
Faceclaw never transmits audio to any party other than a user-selected
transcription service. If a third-party transcription service is selected,
audio will be sent to that third party, is linked to the account with which
@@ -20,6 +21,7 @@ policies:
OpenAI: https://openai.com/policies/row-privacy-policy/
ElevenLabs: https://elevenlabs.io/privacy-policy
Soniox: https://soniox.com/policies/privacy-policy
Third party data retention and privacy policies may differ based on the API
key provided. API keys are part of a relationship between the user and the API
+1 -1
View File
@@ -38,7 +38,7 @@ To install, run
## Features
* A voice assistant that wakes up when you say "Hey Even", transcribes text
with an on-device model, OpenAI Whisper, or ElevenLabs API, and responds
with an on-device model, OpenAI Whisper, ElevenLabs, or Soniox API, and responds
to queries and commands using an Anthropic or OpenAI model (requires an API
key), or using your own long-running OpenClaw agent (see "Connecting an
external agent" below).
+2 -1
View File
@@ -31,7 +31,7 @@ import { type InProcessAppOptions, type InProcessWindow } from "../ui/shell/in-p
import { loadPersistedOpenApps, savePersistedOpenApps } from "../ui/shell/open-apps-persistence";
import { appViewportRect, type WindowHeightMode } from "../ui/shell/geometry";
import { type LayerActions } from "../ui/layers";
import { assistantAllowProactiveSetting, assistantBackendSetting, assistantBridgeHostSetting, assistantBridgePortSetting, assistantBridgeTokenSetting, brightnessSetting, brightnessSettingToLevel, elevenLabsApiKeySetting, openAiApiKeySetting, nightscoutApiTokenSetting, firmwareDebugFlagsSetting, lockScreenEnabledSetting, nightscoutSiteUrlSetting, onAnySettingChanged, saveVoiceRecordingsSetting, screenTimeoutSetting, screenTimeoutSettingToMs, suspendEvenHubWhenScreenOffSetting, verticalPositionSetting, voiceProviderSetting, wakeWordActionSetting, type ConfigSettingString } from "../ui/dashboard-settings";
import { assistantAllowProactiveSetting, assistantBackendSetting, assistantBridgeHostSetting, assistantBridgePortSetting, assistantBridgeTokenSetting, brightnessSetting, brightnessSettingToLevel, elevenLabsApiKeySetting, openAiApiKeySetting, nightscoutApiTokenSetting, firmwareDebugFlagsSetting, lockScreenEnabledSetting, nightscoutSiteUrlSetting, onAnySettingChanged, saveVoiceRecordingsSetting, sonioxApiKeySetting, screenTimeoutSetting, screenTimeoutSettingToMs, suspendEvenHubWhenScreenOffSetting, verticalPositionSetting, voiceProviderSetting, wakeWordActionSetting, type ConfigSettingString } from "../ui/dashboard-settings";
import { isIgnoringBatteryOptimizations, requestIgnoreBatteryOptimizations } from "../native/battery-optimization";
type ConnectionPhase = "disconnected" | "connecting" | "connected" | "charging" | "disconnecting";
@@ -1194,6 +1194,7 @@ class DashboardController {
provider: voiceProviderSetting.get(),
elevenLabsApiKey: elevenLabsApiKeySetting.get(),
openAiApiKey: openAiApiKeySetting.get(),
sonioxApiKey: sonioxApiKeySetting.get(),
saveRecording: saveVoiceRecordingsSetting.get(),
endpointing,
};
+11 -5
View File
@@ -1,5 +1,6 @@
/**
* Shared shape for the cloud speech-to-text providers (ElevenLabs, Whisper).
* Shared shape for the cloud speech-to-text providers (ElevenLabs, Whisper,
* Soniox).
* The voice bridge holds one of these while a cloud provider owns the
* transcript; the Java controller then only decodes LC3 to PCM and hands it
* over via acceptPcm.
@@ -33,13 +34,18 @@ export const CLOUD_STT_SAMPLE_RATE = 16000;
declare const android: any;
/** Base64 for audio payloads, via the Android SDK (no JS base64 in NS core). */
export function encodeBase64(bytes: Uint8Array): string {
if (!global.isAndroid) return "";
/** Copy a Uint8Array into a Java byte[] (for binary WebSocket frames etc.). */
export function toJavaBytes(bytes: Uint8Array): any {
const javaBytes = Array.create("byte", bytes.length);
for (let i = 0; i < bytes.length; i++) {
const value = bytes[i]!;
javaBytes[i] = value > 127 ? value - 256 : value;
}
return String(android.util.Base64.encodeToString(javaBytes, android.util.Base64.NO_WRAP));
return javaBytes;
}
/** Base64 for audio payloads, via the Android SDK (no JS base64 in NS core). */
export function encodeBase64(bytes: Uint8Array): string {
if (!global.isAndroid) return "";
return String(android.util.Base64.encodeToString(toJavaBytes(bytes), android.util.Base64.NO_WRAP));
}
+160
View File
@@ -0,0 +1,160 @@
import { CLOUD_STT_SAMPLE_RATE, CloudSttClient, CloudSttOptions, toJavaBytes } from "./cloud-stt";
declare const com: any;
/**
* Soniox realtime speech-to-text over WebSocket.
* https://soniox.com/docs/stt/rt/real-time-transcription
*
* Unlike the other cloud providers, Soniox authenticates with an api_key field
* in the first (JSON) message rather than an HTTP header, and audio is sent as
* raw binary frames rather than base64-in-JSON. Push-to-talk defines the
* utterance boundary: an empty text frame is the end-of-audio signal, after
* which the server finalizes all tokens and replies with finished=true.
*
* Transcripts arrive as TOKENS: final tokens are delivered exactly once and
* are accumulated here; non-final tokens repeat (revised) in every message and
* are appended for display only. Listeners get full replace-semantics text.
*/
const WS_URL = "wss://stt-rt.soniox.com/transcribe-websocket";
const MODEL_ID = "stt-rt-v5";
export type SonioxSttOptions = CloudSttOptions;
export class SonioxSttClient implements CloudSttClient {
private ws: any = null;
private listenerProxy: any = null;
private open = false;
private closed = false;
// PCM queued until the socket opens and the config message is sent.
private readonly pendingPcm: Uint8Array[] = [];
private pendingFinish = false;
// Concatenation of all final tokens so far.
private finalText = "";
constructor(private readonly options: SonioxSttOptions) {}
start(): void {
if (this.ws) return;
this.listenerProxy = new com.faceclaw.app.FaceclawWebSocketListener({
onOpen: () => {
// The API key rides in the config message; there is no auth header.
this.trySendText(
JSON.stringify({
api_key: this.options.apiKey,
model: MODEL_ID,
audio_format: "pcm_s16le",
sample_rate: CLOUD_STT_SAMPLE_RATE,
num_channels: 1,
}),
);
this.open = true;
for (const chunk of this.pendingPcm.splice(0)) {
this.sendPcm(chunk);
}
if (this.pendingFinish) {
this.pendingFinish = false;
this.finish();
}
this.options.onStatus("Listening (Soniox)...");
},
onTextMessage: (message: string) => this.handleMessage(String(message)),
onClosed: () => {
this.open = false;
},
onFailure: (message: string) => {
if (this.closed) return;
this.options.onError(`Soniox connection failed: ${String(message)}`);
},
});
try {
this.ws = new com.faceclaw.app.FaceclawWebSocket(WS_URL, this.listenerProxy, null, null);
this.options.onStatus("Connecting to Soniox...");
} catch (error) {
this.options.onError(`Soniox connection failed: ${String((error as Error)?.message ?? error)}`);
}
}
/** Feed PCM (16 kHz signed-16-bit LE), as raw binary frames. */
acceptPcm(pcm: Uint8Array): void {
if (this.closed || pcm.length === 0) return;
if (this.open) {
this.sendPcm(pcm);
} else {
this.pendingPcm.push(pcm);
}
}
/** End of utterance: an empty text frame tells Soniox the audio is done. */
finish(): void {
if (this.closed) return;
if (this.open) {
this.trySendText("");
} else {
this.pendingFinish = true;
}
}
stop(): void {
this.closed = true;
if (this.ws) {
try {
this.ws.close(1000, "bye");
} catch {
// ignore
}
this.ws = null;
}
this.listenerProxy = null;
}
private sendPcm(pcm: Uint8Array): void {
try {
this.ws?.sendBinary(toJavaBytes(pcm));
} catch (error) {
console.warn("soniox send failed", error);
}
}
private trySendText(message: string): void {
try {
this.ws?.sendText(message);
} catch (error) {
console.warn("soniox send failed", error);
}
}
private handleMessage(text: string): void {
let message: any;
try {
message = JSON.parse(text);
} catch {
return;
}
if (message?.error_code != null) {
this.options.onError(`Soniox: ${String(message.error_message ?? message.error_code)}`);
return;
}
const tokens = Array.isArray(message?.tokens) ? message.tokens : [];
let nonFinal = "";
for (const token of tokens) {
const tokenText = String(token?.text ?? "");
// Markers emitted by endpoint detection / manual finalize; not speech.
if (tokenText === "<end>" || tokenText === "<fin>") continue;
if (token?.is_final) {
this.finalText += tokenText;
} else {
nonFinal += tokenText;
}
}
if (message?.finished) {
this.options.onTranscript({ text: this.finalText, isFinal: true });
this.stop();
return;
}
if (tokens.length > 0) {
this.options.onTranscript({ text: this.finalText + nonFinal, isFinal: false });
}
}
}
+11 -1
View File
@@ -3,6 +3,7 @@ import { Utils } from "@nativescript/core";
import { CloudSttClient } from "./cloud-stt";
import { ElevenLabsSttClient } from "./elevenlabs-stt";
import { OpenAiRealtimeSttClient } from "./openai-stt";
import { SonioxSttClient } from "./soniox-stt";
import { toUint8Array } from "../util/array-util";
declare const com: any;
@@ -11,7 +12,7 @@ export type VoiceControlState = {
status: string;
};
export type VoiceProviderKind = "onboard" | "elevenlabs" | "whisper";
export type VoiceProviderKind = "onboard" | "elevenlabs" | "whisper" | "soniox";
export type VoiceTranscriptEvent = {
/**
@@ -27,6 +28,7 @@ export type PushToTalkOptions = {
provider: VoiceProviderKind;
elevenLabsApiKey: string;
openAiApiKey: string;
sonioxApiKey: string;
saveRecording: boolean;
/**
* Watch the mic and fire onSpeechEnd when the speaker stops. For hands-free
@@ -150,6 +152,14 @@ export class FaceclawVoiceControlBridge {
}
return new ElevenLabsSttClient({ ...sttOptions, apiKey });
}
if (options.provider === "soniox") {
const apiKey = options.sonioxApiKey.trim();
if (!apiKey) {
this.setStatus("No Soniox key set; using on-device voice.");
return null;
}
return new SonioxSttClient({ ...sttOptions, apiKey });
}
const apiKey = options.openAiApiKey.trim();
if (!apiKey) {
this.setStatus("No OpenAI key set; using on-device voice.");
+21 -3
View File
@@ -297,12 +297,13 @@ export const suspendEvenHubWhenScreenOffSetting = new ConfigSettingBoolean({
description: "Suspend the EvenHub session while the display is off. This significantly improves battery life, but increases the latency of waking the screen.",
});
export type VoiceProvider = "onboard" | "elevenlabs" | "whisper";
export type VoiceProvider = "onboard" | "elevenlabs" | "whisper" | "soniox";
const voiceProviderLabels: Record<VoiceProvider, string> = {
onboard: "On-device",
elevenlabs: "ElevenLabs",
whisper: "Whisper",
soniox: "Soniox",
};
export const voiceProviderSetting = new ConfigSettingEnum<VoiceProvider>({
@@ -310,9 +311,15 @@ export const voiceProviderSetting = new ConfigSettingEnum<VoiceProvider>({
label: "Transcription Provider",
storageKey: "voice.provider",
defaultValue: "onboard",
values: ["onboard", "elevenlabs", "whisper"],
values: ["onboard", "elevenlabs", "whisper", "soniox"],
formatValue: (value) => voiceProviderLabels[value] ?? value,
description: "Speech-to-text engine for voice input. ElevenLabs and Whisper are cloud services that need an API key, with significantly better accuracy than on-device transcription.",
isDisabled: (value) => {
if (value === "elevenlabs") return elevenLabsApiKeySetting.get().trim().length === 0;
if (value === "whisper") return openAiApiKeySetting.get().trim().length === 0;
if (value === "soniox") return sonioxApiKeySetting.get().trim().length === 0;
return false;
},
description: "Speech-to-text engine for voice input. ElevenLabs, Whisper, and Soniox are cloud services that need an API key, with significantly better accuracy than on-device transcription.",
});
const wakeWordActionLabels: Record<WakeWordAction, string> = {
@@ -428,6 +435,17 @@ export const openAiApiKeySetting = new ConfigSettingString({
description: "OpenAI API key, used when Whisper is the transcription provider or an OpenAI model is selected for the voice assistant.",
});
export const sonioxApiKeySetting = new ConfigSettingString({
id: "soniox-api-key",
label: "Soniox key",
storageKey: "voice.sonioxApiKey",
defaultValue: "",
editorTitle: "Soniox API key",
glassesEditTitle: "Edit Soniox key",
formatValue: (value) => (value ? `${value.slice(0, 6)}...` : "(not set)"),
description: "Soniox API key, used when Soniox is the transcription provider.",
});
export const anthropicApiKeySetting = new ConfigSettingString({
id: "anthropic-api-key",
label: "Anthropic key",
+2
View File
@@ -22,6 +22,7 @@ import {
openAiApiKeySetting,
roamApiTokenSetting,
roamGraphNameSetting,
sonioxApiKeySetting,
enumSettingMenuItem,
firmwareDebugFlagsSetting,
lockScreenEnabledSetting,
@@ -100,6 +101,7 @@ function settingsSections(): SettingsSection[] {
items: [
textSettingMenuItem(elevenLabsApiKeySetting),
textSettingMenuItem(openAiApiKeySetting),
textSettingMenuItem(sonioxApiKeySetting),
textSettingMenuItem(anthropicApiKeySetting),
textSettingMenuItem(mapboxApiKeySetting),
],