mirror of
https://git.undefined-dev.fr/Blendman974/faceclaw.git
synced 2026-11-19 21:03:09 +00:00
Add Soniox as an API speech-to-text option
This commit is contained in:
1 parent
852626e950
commit
ba81e46624
9 files changed
+218
-14
No files matched your search
@@ -11,10 +11,11 @@ import okhttp3.Request;
|
||||
import okhttp3.Response;
|
||||
import okhttp3.WebSocket;
|
||||
import okhttp3.WebSocketListener;
|
||||
import okio.ByteString;
|
||||
|
||||
/**
|
||||
* Thin okhttp WebSocket wrapper for the TypeScript side. Text frames only
|
||||
* (the g2mirror protocol is JSON-in-text-frames). Listener callbacks are
|
||||
* Thin okhttp WebSocket wrapper for the TypeScript side. Sends text frames
|
||||
* (JSON protocols) and binary frames (raw audio for Soniox). Listener callbacks are
|
||||
* posted to the Looper of the thread that constructed this object, so a JS
|
||||
* isolate (main thread or app worker) always receives them on its own
|
||||
* thread; a Looper-less constructing thread falls back to the main thread.
|
||||
@@ -132,6 +133,10 @@ public class FaceclawWebSocket {
|
||||
return socket.send(message == null ? "" : message);
|
||||
}
|
||||
|
||||
public boolean sendBinary(byte[] bytes) {
|
||||
return socket.send(ByteString.of(bytes == null ? new byte[0] : bytes));
|
||||
}
|
||||
|
||||
public void close(int code, String reason) {
|
||||
closeRequested = true;
|
||||
try {
|
||||
|
||||
@@ -11,7 +11,8 @@ on-device model is selected, audio never leaves your device, and is not stored
|
||||
after it is processed.
|
||||
|
||||
Users can also provide their own API key for a third-party transcription
|
||||
service such as OpenAI Whisper or ElevenLabs, which may offer better accuracy.
|
||||
service such as OpenAI Whisper, ElevenLabs, or Soniox, which may offer better
|
||||
accuracy.
|
||||
Faceclaw never transmits audio to any party other than a user-selected
|
||||
transcription service. If a third-party transcription service is selected,
|
||||
audio will be sent to that third party, is linked to the account with which
|
||||
@@ -20,6 +21,7 @@ policies:
|
||||
|
||||
OpenAI: https://openai.com/policies/row-privacy-policy/
|
||||
ElevenLabs: https://elevenlabs.io/privacy-policy
|
||||
Soniox: https://soniox.com/policies/privacy-policy
|
||||
|
||||
Third party data retention and privacy policies may differ based on the API
|
||||
key provided. API keys are part of a relationship between the user and the API
|
||||
|
||||
@@ -38,7 +38,7 @@ To install, run
|
||||
## Features
|
||||
|
||||
* A voice assistant that wakes up when you say "Hey Even", transcribes text
|
||||
with an on-device model, OpenAI Whisper, or ElevenLabs API, and responds
|
||||
with an on-device model, OpenAI Whisper, ElevenLabs, or Soniox API, and responds
|
||||
to queries and commands using an Anthropic or OpenAI model (requires an API
|
||||
key), or using your own long-running OpenClaw agent (see "Connecting an
|
||||
external agent" below).
|
||||
|
||||
@@ -31,7 +31,7 @@ import { type InProcessAppOptions, type InProcessWindow } from "../ui/shell/in-p
|
||||
import { loadPersistedOpenApps, savePersistedOpenApps } from "../ui/shell/open-apps-persistence";
|
||||
import { appViewportRect, type WindowHeightMode } from "../ui/shell/geometry";
|
||||
import { type LayerActions } from "../ui/layers";
|
||||
import { assistantAllowProactiveSetting, assistantBackendSetting, assistantBridgeHostSetting, assistantBridgePortSetting, assistantBridgeTokenSetting, brightnessSetting, brightnessSettingToLevel, elevenLabsApiKeySetting, openAiApiKeySetting, nightscoutApiTokenSetting, firmwareDebugFlagsSetting, lockScreenEnabledSetting, nightscoutSiteUrlSetting, onAnySettingChanged, saveVoiceRecordingsSetting, screenTimeoutSetting, screenTimeoutSettingToMs, suspendEvenHubWhenScreenOffSetting, verticalPositionSetting, voiceProviderSetting, wakeWordActionSetting, type ConfigSettingString } from "../ui/dashboard-settings";
|
||||
import { assistantAllowProactiveSetting, assistantBackendSetting, assistantBridgeHostSetting, assistantBridgePortSetting, assistantBridgeTokenSetting, brightnessSetting, brightnessSettingToLevel, elevenLabsApiKeySetting, openAiApiKeySetting, nightscoutApiTokenSetting, firmwareDebugFlagsSetting, lockScreenEnabledSetting, nightscoutSiteUrlSetting, onAnySettingChanged, saveVoiceRecordingsSetting, sonioxApiKeySetting, screenTimeoutSetting, screenTimeoutSettingToMs, suspendEvenHubWhenScreenOffSetting, verticalPositionSetting, voiceProviderSetting, wakeWordActionSetting, type ConfigSettingString } from "../ui/dashboard-settings";
|
||||
import { isIgnoringBatteryOptimizations, requestIgnoreBatteryOptimizations } from "../native/battery-optimization";
|
||||
|
||||
type ConnectionPhase = "disconnected" | "connecting" | "connected" | "charging" | "disconnecting";
|
||||
@@ -1194,6 +1194,7 @@ class DashboardController {
|
||||
provider: voiceProviderSetting.get(),
|
||||
elevenLabsApiKey: elevenLabsApiKeySetting.get(),
|
||||
openAiApiKey: openAiApiKeySetting.get(),
|
||||
sonioxApiKey: sonioxApiKeySetting.get(),
|
||||
saveRecording: saveVoiceRecordingsSetting.get(),
|
||||
endpointing,
|
||||
};
|
||||
|
||||
+11
-5
@@ -1,5 +1,6 @@
|
||||
/**
|
||||
* Shared shape for the cloud speech-to-text providers (ElevenLabs, Whisper).
|
||||
* Shared shape for the cloud speech-to-text providers (ElevenLabs, Whisper,
|
||||
* Soniox).
|
||||
* The voice bridge holds one of these while a cloud provider owns the
|
||||
* transcript; the Java controller then only decodes LC3 to PCM and hands it
|
||||
* over via acceptPcm.
|
||||
@@ -33,13 +34,18 @@ export const CLOUD_STT_SAMPLE_RATE = 16000;
|
||||
|
||||
declare const android: any;
|
||||
|
||||
/** Base64 for audio payloads, via the Android SDK (no JS base64 in NS core). */
|
||||
export function encodeBase64(bytes: Uint8Array): string {
|
||||
if (!global.isAndroid) return "";
|
||||
/** Copy a Uint8Array into a Java byte[] (for binary WebSocket frames etc.). */
|
||||
export function toJavaBytes(bytes: Uint8Array): any {
|
||||
const javaBytes = Array.create("byte", bytes.length);
|
||||
for (let i = 0; i < bytes.length; i++) {
|
||||
const value = bytes[i]!;
|
||||
javaBytes[i] = value > 127 ? value - 256 : value;
|
||||
}
|
||||
return String(android.util.Base64.encodeToString(javaBytes, android.util.Base64.NO_WRAP));
|
||||
return javaBytes;
|
||||
}
|
||||
|
||||
/** Base64 for audio payloads, via the Android SDK (no JS base64 in NS core). */
|
||||
export function encodeBase64(bytes: Uint8Array): string {
|
||||
if (!global.isAndroid) return "";
|
||||
return String(android.util.Base64.encodeToString(toJavaBytes(bytes), android.util.Base64.NO_WRAP));
|
||||
}
|
||||
@@ -0,0 +1,160 @@
|
||||
import { CLOUD_STT_SAMPLE_RATE, CloudSttClient, CloudSttOptions, toJavaBytes } from "./cloud-stt";
|
||||
|
||||
declare const com: any;
|
||||
|
||||
/**
|
||||
* Soniox realtime speech-to-text over WebSocket.
|
||||
* https://soniox.com/docs/stt/rt/real-time-transcription
|
||||
*
|
||||
* Unlike the other cloud providers, Soniox authenticates with an api_key field
|
||||
* in the first (JSON) message rather than an HTTP header, and audio is sent as
|
||||
* raw binary frames rather than base64-in-JSON. Push-to-talk defines the
|
||||
* utterance boundary: an empty text frame is the end-of-audio signal, after
|
||||
* which the server finalizes all tokens and replies with finished=true.
|
||||
*
|
||||
* Transcripts arrive as TOKENS: final tokens are delivered exactly once and
|
||||
* are accumulated here; non-final tokens repeat (revised) in every message and
|
||||
* are appended for display only. Listeners get full replace-semantics text.
|
||||
*/
|
||||
|
||||
const WS_URL = "wss://stt-rt.soniox.com/transcribe-websocket";
|
||||
const MODEL_ID = "stt-rt-v5";
|
||||
|
||||
export type SonioxSttOptions = CloudSttOptions;
|
||||
|
||||
export class SonioxSttClient implements CloudSttClient {
|
||||
private ws: any = null;
|
||||
private listenerProxy: any = null;
|
||||
private open = false;
|
||||
private closed = false;
|
||||
// PCM queued until the socket opens and the config message is sent.
|
||||
private readonly pendingPcm: Uint8Array[] = [];
|
||||
private pendingFinish = false;
|
||||
// Concatenation of all final tokens so far.
|
||||
private finalText = "";
|
||||
|
||||
constructor(private readonly options: SonioxSttOptions) {}
|
||||
|
||||
start(): void {
|
||||
if (this.ws) return;
|
||||
this.listenerProxy = new com.faceclaw.app.FaceclawWebSocketListener({
|
||||
onOpen: () => {
|
||||
// The API key rides in the config message; there is no auth header.
|
||||
this.trySendText(
|
||||
JSON.stringify({
|
||||
api_key: this.options.apiKey,
|
||||
model: MODEL_ID,
|
||||
audio_format: "pcm_s16le",
|
||||
sample_rate: CLOUD_STT_SAMPLE_RATE,
|
||||
num_channels: 1,
|
||||
}),
|
||||
);
|
||||
this.open = true;
|
||||
for (const chunk of this.pendingPcm.splice(0)) {
|
||||
this.sendPcm(chunk);
|
||||
}
|
||||
if (this.pendingFinish) {
|
||||
this.pendingFinish = false;
|
||||
this.finish();
|
||||
}
|
||||
this.options.onStatus("Listening (Soniox)...");
|
||||
},
|
||||
onTextMessage: (message: string) => this.handleMessage(String(message)),
|
||||
onClosed: () => {
|
||||
this.open = false;
|
||||
},
|
||||
onFailure: (message: string) => {
|
||||
if (this.closed) return;
|
||||
this.options.onError(`Soniox connection failed: ${String(message)}`);
|
||||
},
|
||||
});
|
||||
try {
|
||||
this.ws = new com.faceclaw.app.FaceclawWebSocket(WS_URL, this.listenerProxy, null, null);
|
||||
this.options.onStatus("Connecting to Soniox...");
|
||||
} catch (error) {
|
||||
this.options.onError(`Soniox connection failed: ${String((error as Error)?.message ?? error)}`);
|
||||
}
|
||||
}
|
||||
|
||||
/** Feed PCM (16 kHz signed-16-bit LE), as raw binary frames. */
|
||||
acceptPcm(pcm: Uint8Array): void {
|
||||
if (this.closed || pcm.length === 0) return;
|
||||
if (this.open) {
|
||||
this.sendPcm(pcm);
|
||||
} else {
|
||||
this.pendingPcm.push(pcm);
|
||||
}
|
||||
}
|
||||
|
||||
/** End of utterance: an empty text frame tells Soniox the audio is done. */
|
||||
finish(): void {
|
||||
if (this.closed) return;
|
||||
if (this.open) {
|
||||
this.trySendText("");
|
||||
} else {
|
||||
this.pendingFinish = true;
|
||||
}
|
||||
}
|
||||
|
||||
stop(): void {
|
||||
this.closed = true;
|
||||
if (this.ws) {
|
||||
try {
|
||||
this.ws.close(1000, "bye");
|
||||
} catch {
|
||||
// ignore
|
||||
}
|
||||
this.ws = null;
|
||||
}
|
||||
this.listenerProxy = null;
|
||||
}
|
||||
|
||||
private sendPcm(pcm: Uint8Array): void {
|
||||
try {
|
||||
this.ws?.sendBinary(toJavaBytes(pcm));
|
||||
} catch (error) {
|
||||
console.warn("soniox send failed", error);
|
||||
}
|
||||
}
|
||||
|
||||
private trySendText(message: string): void {
|
||||
try {
|
||||
this.ws?.sendText(message);
|
||||
} catch (error) {
|
||||
console.warn("soniox send failed", error);
|
||||
}
|
||||
}
|
||||
|
||||
private handleMessage(text: string): void {
|
||||
let message: any;
|
||||
try {
|
||||
message = JSON.parse(text);
|
||||
} catch {
|
||||
return;
|
||||
}
|
||||
if (message?.error_code != null) {
|
||||
this.options.onError(`Soniox: ${String(message.error_message ?? message.error_code)}`);
|
||||
return;
|
||||
}
|
||||
const tokens = Array.isArray(message?.tokens) ? message.tokens : [];
|
||||
let nonFinal = "";
|
||||
for (const token of tokens) {
|
||||
const tokenText = String(token?.text ?? "");
|
||||
// Markers emitted by endpoint detection / manual finalize; not speech.
|
||||
if (tokenText === "<end>" || tokenText === "<fin>") continue;
|
||||
if (token?.is_final) {
|
||||
this.finalText += tokenText;
|
||||
} else {
|
||||
nonFinal += tokenText;
|
||||
}
|
||||
}
|
||||
if (message?.finished) {
|
||||
this.options.onTranscript({ text: this.finalText, isFinal: true });
|
||||
this.stop();
|
||||
return;
|
||||
}
|
||||
if (tokens.length > 0) {
|
||||
this.options.onTranscript({ text: this.finalText + nonFinal, isFinal: false });
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -3,6 +3,7 @@ import { Utils } from "@nativescript/core";
|
||||
import { CloudSttClient } from "./cloud-stt";
|
||||
import { ElevenLabsSttClient } from "./elevenlabs-stt";
|
||||
import { OpenAiRealtimeSttClient } from "./openai-stt";
|
||||
import { SonioxSttClient } from "./soniox-stt";
|
||||
import { toUint8Array } from "../util/array-util";
|
||||
|
||||
declare const com: any;
|
||||
@@ -11,7 +12,7 @@ export type VoiceControlState = {
|
||||
status: string;
|
||||
};
|
||||
|
||||
export type VoiceProviderKind = "onboard" | "elevenlabs" | "whisper";
|
||||
export type VoiceProviderKind = "onboard" | "elevenlabs" | "whisper" | "soniox";
|
||||
|
||||
export type VoiceTranscriptEvent = {
|
||||
/**
|
||||
@@ -27,6 +28,7 @@ export type PushToTalkOptions = {
|
||||
provider: VoiceProviderKind;
|
||||
elevenLabsApiKey: string;
|
||||
openAiApiKey: string;
|
||||
sonioxApiKey: string;
|
||||
saveRecording: boolean;
|
||||
/**
|
||||
* Watch the mic and fire onSpeechEnd when the speaker stops. For hands-free
|
||||
@@ -150,6 +152,14 @@ export class FaceclawVoiceControlBridge {
|
||||
}
|
||||
return new ElevenLabsSttClient({ ...sttOptions, apiKey });
|
||||
}
|
||||
if (options.provider === "soniox") {
|
||||
const apiKey = options.sonioxApiKey.trim();
|
||||
if (!apiKey) {
|
||||
this.setStatus("No Soniox key set; using on-device voice.");
|
||||
return null;
|
||||
}
|
||||
return new SonioxSttClient({ ...sttOptions, apiKey });
|
||||
}
|
||||
const apiKey = options.openAiApiKey.trim();
|
||||
if (!apiKey) {
|
||||
this.setStatus("No OpenAI key set; using on-device voice.");
|
||||
|
||||
@@ -297,12 +297,13 @@ export const suspendEvenHubWhenScreenOffSetting = new ConfigSettingBoolean({
|
||||
description: "Suspend the EvenHub session while the display is off. This significantly improves battery life, but increases the latency of waking the screen.",
|
||||
});
|
||||
|
||||
export type VoiceProvider = "onboard" | "elevenlabs" | "whisper";
|
||||
export type VoiceProvider = "onboard" | "elevenlabs" | "whisper" | "soniox";
|
||||
|
||||
const voiceProviderLabels: Record<VoiceProvider, string> = {
|
||||
onboard: "On-device",
|
||||
elevenlabs: "ElevenLabs",
|
||||
whisper: "Whisper",
|
||||
soniox: "Soniox",
|
||||
};
|
||||
|
||||
export const voiceProviderSetting = new ConfigSettingEnum<VoiceProvider>({
|
||||
@@ -310,9 +311,15 @@ export const voiceProviderSetting = new ConfigSettingEnum<VoiceProvider>({
|
||||
label: "Transcription Provider",
|
||||
storageKey: "voice.provider",
|
||||
defaultValue: "onboard",
|
||||
values: ["onboard", "elevenlabs", "whisper"],
|
||||
values: ["onboard", "elevenlabs", "whisper", "soniox"],
|
||||
formatValue: (value) => voiceProviderLabels[value] ?? value,
|
||||
description: "Speech-to-text engine for voice input. ElevenLabs and Whisper are cloud services that need an API key, with significantly better accuracy than on-device transcription.",
|
||||
isDisabled: (value) => {
|
||||
if (value === "elevenlabs") return elevenLabsApiKeySetting.get().trim().length === 0;
|
||||
if (value === "whisper") return openAiApiKeySetting.get().trim().length === 0;
|
||||
if (value === "soniox") return sonioxApiKeySetting.get().trim().length === 0;
|
||||
return false;
|
||||
},
|
||||
description: "Speech-to-text engine for voice input. ElevenLabs, Whisper, and Soniox are cloud services that need an API key, with significantly better accuracy than on-device transcription.",
|
||||
});
|
||||
|
||||
const wakeWordActionLabels: Record<WakeWordAction, string> = {
|
||||
@@ -428,6 +435,17 @@ export const openAiApiKeySetting = new ConfigSettingString({
|
||||
description: "OpenAI API key, used when Whisper is the transcription provider or an OpenAI model is selected for the voice assistant.",
|
||||
});
|
||||
|
||||
export const sonioxApiKeySetting = new ConfigSettingString({
|
||||
id: "soniox-api-key",
|
||||
label: "Soniox key",
|
||||
storageKey: "voice.sonioxApiKey",
|
||||
defaultValue: "",
|
||||
editorTitle: "Soniox API key",
|
||||
glassesEditTitle: "Edit Soniox key",
|
||||
formatValue: (value) => (value ? `${value.slice(0, 6)}...` : "(not set)"),
|
||||
description: "Soniox API key, used when Soniox is the transcription provider.",
|
||||
});
|
||||
|
||||
export const anthropicApiKeySetting = new ConfigSettingString({
|
||||
id: "anthropic-api-key",
|
||||
label: "Anthropic key",
|
||||
|
||||
@@ -22,6 +22,7 @@ import {
|
||||
openAiApiKeySetting,
|
||||
roamApiTokenSetting,
|
||||
roamGraphNameSetting,
|
||||
sonioxApiKeySetting,
|
||||
enumSettingMenuItem,
|
||||
firmwareDebugFlagsSetting,
|
||||
lockScreenEnabledSetting,
|
||||
@@ -100,6 +101,7 @@ function settingsSections(): SettingsSection[] {
|
||||
items: [
|
||||
textSettingMenuItem(elevenLabsApiKeySetting),
|
||||
textSettingMenuItem(openAiApiKeySetting),
|
||||
textSettingMenuItem(sonioxApiKeySetting),
|
||||
textSettingMenuItem(anthropicApiKeySetting),
|
||||
textSettingMenuItem(mapboxApiKeySetting),
|
||||
],
|
||||
|
||||
Reference in new issue
Block a user