mirror of
https://git.undefined-dev.fr/Blendman974/faceclaw.git
synced 2026-11-19 21:03:09 +00:00
Merge commit 'e2caf28' into hermes-integration
This commit is contained in:
commit
33eef6538c
24 files changed
+1063
-43
No files matched your search
@@ -25,6 +25,7 @@ const API_KEY_SETTINGS: Record<string, string> = {
|
||||
openai: "voice.openAiApiKey",
|
||||
elevenlabs: "voice.elevenLabsApiKey",
|
||||
soniox: "voice.sonioxApiKey",
|
||||
"whisper-server": "voice.whisperServerApiKey",
|
||||
mapbox: "maps.mapboxApiKey",
|
||||
roam: "integrations.roam.apiToken",
|
||||
nightscout: "integrations.nightscout.apiToken",
|
||||
|
||||
@@ -116,6 +116,9 @@ export function createCalculatorAppWindow(options: InProcessAppOptions): InProce
|
||||
},
|
||||
removeSurface: options.removeSurface,
|
||||
onClosed: () => {
|
||||
// Closing keeps the base layer (clearToBase), so stop its continuous
|
||||
// listening explicitly; otherwise the mic capture outlives the window.
|
||||
layer.onRemoved();
|
||||
toolRegistry.removeAppTools(CALCULATOR_WINDOW_ID);
|
||||
options.onClosed();
|
||||
},
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
/** Native services still outside the current iOS app-screen milestone. */
|
||||
const unavailable: Record<string, string> = {
|
||||
music: 'Media-player integration is not available on iOS yet.',
|
||||
transcribe: 'Voice capture and transcription are not available on iOS yet.',
|
||||
microphones: 'Glasses microphone capture is not available on iOS yet.',
|
||||
}
|
||||
export function iosAppUnavailableReason(appId: string): string | null {
|
||||
|
||||
@@ -9,7 +9,7 @@ const transcribeApp: AppDefinition = {
|
||||
ctx.launchInProcessApp(TRANSCRIBE_WINDOW_ID, TRANSCRIBE_SURFACE_ID, (options) =>
|
||||
createTranscribeAppWindow({
|
||||
...options,
|
||||
startContinuousVoiceCapture: () => void options.actions.startContinuousVoiceCapture("transcribe"),
|
||||
startContinuousVoiceCapture: (capture) => void options.actions.startContinuousVoiceCapture("transcribe", capture),
|
||||
stopContinuousVoiceCapture: () => void options.actions.stopContinuousVoiceCapture("transcribe"),
|
||||
}),
|
||||
),
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import { imageFromAsciiArt } from "../../graphics/image";
|
||||
import { TranscribeLayer } from "./transcribe";
|
||||
import { TranscribeLayer, transcriptTimestamp } from "./transcribe";
|
||||
import {
|
||||
createInProcessWindow,
|
||||
YieldAtRootLayer,
|
||||
@@ -37,7 +37,7 @@ const MIC_ICON = imageFromAsciiArt(
|
||||
|
||||
export type TranscribeAppOptions = InProcessAppOptions & {
|
||||
/** Begin continuous mic capture (kept on while the window is open). */
|
||||
startContinuousVoiceCapture: () => void;
|
||||
startContinuousVoiceCapture: (options?: { recordAs?: string }) => void;
|
||||
stopContinuousVoiceCapture: () => void;
|
||||
};
|
||||
|
||||
@@ -47,7 +47,9 @@ export type TranscribeAppOptions = InProcessAppOptions & {
|
||||
* shows a microphone tray icon in the top bar.
|
||||
*/
|
||||
export function createTranscribeAppWindow(options: TranscribeAppOptions): InProcessWindow {
|
||||
const layer = new TranscribeLayer();
|
||||
// One timestamp per session names both the audio recording and a saved transcript.
|
||||
const stamp = transcriptTimestamp();
|
||||
const layer = new TranscribeLayer(stamp);
|
||||
const app = createInProcessWindow({
|
||||
appId: "transcribe",
|
||||
windowId: TRANSCRIBE_WINDOW_ID,
|
||||
@@ -77,7 +79,7 @@ export function createTranscribeAppWindow(options: TranscribeAppOptions): InProc
|
||||
// Wire async state changes to this window's renderer, rather than the
|
||||
// controller action from which the window-specific actions were derived.
|
||||
layer.start(app.requestRender);
|
||||
options.startContinuousVoiceCapture();
|
||||
options.startContinuousVoiceCapture({ recordAs: `transcribe-${stamp}` });
|
||||
shell.setTrayIcon(TRAY_ICON_ID, MIC_ICON);
|
||||
return app;
|
||||
}
|
||||
@@ -23,6 +23,9 @@ export class TranscribeLayer implements Layer {
|
||||
private unsubscribePause: (() => void) | null = null;
|
||||
private unsubscribeStatus: (() => void) | null = null;
|
||||
|
||||
/** `stamp` names the saved transcript, matching the session's audio recording. */
|
||||
constructor(private readonly stamp: string = transcriptTimestamp()) {}
|
||||
|
||||
start(requestRender: () => void): void {
|
||||
this.unsubscribeTranscript = voiceControlBridge.onTranscript((event) => {
|
||||
this.transcript.accept(event);
|
||||
@@ -93,13 +96,13 @@ export class TranscribeLayer implements Layer {
|
||||
this.saveNotice = "Nothing to save yet.";
|
||||
return;
|
||||
}
|
||||
const filename = `transcript-${transcriptTimestamp()}.txt`;
|
||||
const filename = `transcript-${this.stamp}.txt`;
|
||||
const path = writeTextToDownloads(filename, `${text}\n`);
|
||||
this.saveNotice = path ? `Saved ${filename}` : "Save failed (check file access).";
|
||||
}
|
||||
}
|
||||
|
||||
function transcriptTimestamp(): string {
|
||||
export function transcriptTimestamp(): string {
|
||||
const now = new Date();
|
||||
const pad = (value: number) => String(value).padStart(2, "0");
|
||||
return (
|
||||
|
||||
@@ -75,7 +75,7 @@ import { loadPersistedOpenApps, savePersistedOpenApps } from "../ui/shell/open-a
|
||||
import { appViewportRect, isOnSwitcherEdge, sidebarStripVisible, type ScreenRect, type WindowHeightMode } from "../ui/shell/geometry";
|
||||
import { type LayerActions, type TextSettingsEditToggle } from "../ui/layers";
|
||||
import { type KeyboardInputSession } from "../ui/shell/keyboard-input";
|
||||
import { appSwitcherPositionSetting, statusBarPositionSetting, statusBarVisibilitySetting, windowBorderSetting, assistantAllowProactiveSetting, assistantBackendSetting, assistantBridgeHostSetting, assistantBridgePortSetting, assistantBridgeTokenSetting, getBrightnessPreferences, displayLayoutSetting, navigateDisplayLayoutSetting, terminalDisplayLayoutSetting, elevenLabsApiKeySetting, getStringSettingById, openAiApiKeySetting, nightscoutApiTokenSetting, firmwareDebugFlagsSetting, lockScreenEnabledSetting, nightscoutSiteUrlSetting, onAnySettingChanged, previewColorSetting, ringConnectionModeSetting, saveVoiceRecordingsSetting, sonioxApiKeySetting, screenTimeoutSetting, screenTimeoutSettingToMs, suspendEvenHubWhenScreenOffSetting, voiceProviderSetting, wakeWordActionSetting, type ConfigSettingString } from "../ui/dashboard-settings";
|
||||
import { appSwitcherPositionSetting, statusBarPositionSetting, statusBarVisibilitySetting, windowBorderSetting, assistantAllowProactiveSetting, assistantBackendSetting, assistantBridgeHostSetting, assistantBridgePortSetting, assistantBridgeTokenSetting, getBrightnessPreferences, displayLayoutSetting, navigateDisplayLayoutSetting, terminalDisplayLayoutSetting, elevenLabsApiKeySetting, getStringSettingById, openAiApiKeySetting, nightscoutApiTokenSetting, firmwareDebugFlagsSetting, whisperServerApiKeySetting, whisperServerLanguageSetting, whisperServerModelSetting, whisperServerUrlSetting, lockScreenEnabledSetting, nightscoutSiteUrlSetting, onAnySettingChanged, previewColorSetting, ringConnectionModeSetting, saveVoiceRecordingsSetting, sonioxApiKeySetting, screenTimeoutSetting, screenTimeoutSettingToMs, suspendEvenHubWhenScreenOffSetting, voiceProviderSetting, wakeWordActionSetting, type ConfigSettingString } from "../ui/dashboard-settings";
|
||||
import { isIgnoringBatteryOptimizations, requestIgnoreBatteryOptimizations } from "../native/battery-optimization";
|
||||
import {
|
||||
getInstalledEvenHubAppById,
|
||||
@@ -2128,6 +2128,10 @@ class DashboardController {
|
||||
elevenLabsApiKey: elevenLabsApiKeySetting.get(),
|
||||
openAiApiKey: openAiApiKeySetting.get(),
|
||||
sonioxApiKey: sonioxApiKeySetting.get(),
|
||||
whisperServerUrl: whisperServerUrlSetting.get(),
|
||||
whisperServerModel: whisperServerModelSetting.get(),
|
||||
whisperServerLanguage: whisperServerLanguageSetting.get(),
|
||||
whisperServerApiKey: whisperServerApiKeySetting.get(),
|
||||
saveRecording: saveVoiceRecordingsSetting.get(),
|
||||
// "My voice only" (Microphones app): verify command utterances
|
||||
// against the enrolled wearer voice-print; non-matching speakers'
|
||||
|
||||
@@ -16,6 +16,7 @@ import { IosNavigationSensors } from '../native/ios-navigation-sensors'
|
||||
import { bindCompassSession, receiveCompassEvent } from '../native/compass.ios'
|
||||
import { Dialogs, File, knownFolders, path, Utils, type ImageSource } from '@nativescript/core'
|
||||
import { iosVoiceInput } from '../native/ios-voice-input'
|
||||
import { voiceActivity } from '../ui/shell/voice-activity'
|
||||
import { nightscoutBridge } from '../native/nightscout-bridge'
|
||||
import { weatherBridge } from '../native/weather'
|
||||
import { FaceclawCommunicatorBridge, resolveIosPeripherals, type CommunicatorState, type RawInputEvent } from '../native/faceclaw-communicator.ios'
|
||||
@@ -48,12 +49,15 @@ import { mirrorToPanelFraction, phonePreviewCrop } from './phone-preview-crop'
|
||||
import { onAnySettingChanged,
|
||||
previewColorSetting, lockScreenEnabledSetting, getBrightnessPreferences, getStringSettingById,
|
||||
nightscoutSiteUrlSetting, nightscoutApiTokenSetting, voiceProviderSetting, elevenLabsApiKeySetting,
|
||||
openAiApiKeySetting, sonioxApiKeySetting, ringConnectionModeSetting } from '../ui/dashboard-settings'
|
||||
openAiApiKeySetting, sonioxApiKeySetting, whisperServerApiKeySetting, whisperServerLanguageSetting,
|
||||
whisperServerModelSetting, whisperServerUrlSetting, ringConnectionModeSetting, saveTranscribeAudioSetting } from '../ui/dashboard-settings'
|
||||
import type { VoiceProviderSelection } from '../native/cloud-stt-provider'
|
||||
import type { PhoneGesture } from '../phone-ui/phone-gestures'
|
||||
import { claimGlassesOnboardingAutoLaunch, ONBOARDING_APP_ID } from '../apps/onboarding/onboarding-progress'
|
||||
|
||||
const SHELL_SURFACE_ID = 'shell'
|
||||
// voiceActivity holder for continuous capture (Transcribe), as on Android.
|
||||
const CONTINUOUS_CAPTURE_HOLDER = 'continuous-capture'
|
||||
const SHELL_SURFACE_Z_ORDER = 1
|
||||
const LOCK_SURFACE_Z_ORDER = 1000
|
||||
/** The shell render loop waits this long for the glasses to take a frame before moving on. */
|
||||
@@ -144,6 +148,7 @@ export class IosPreviewController {
|
||||
private acknowledgedFrames = 0
|
||||
private logLines: string[] = []
|
||||
private logTimer: ReturnType<typeof setTimeout> | null = null
|
||||
private continuousCaptureToken = 0
|
||||
private readonly actions: LayerActions = {
|
||||
...noopLayerActions,
|
||||
playBuzzerSequence: payload => this.communicator?.playBuzzerSequence(payload),
|
||||
@@ -151,7 +156,8 @@ export class IosPreviewController {
|
||||
disconnect: () => this.disconnect(),
|
||||
startVoiceCapture: endpointing => this.startVoiceCapture(endpointing),
|
||||
stopVoiceCapture: () => iosVoiceInput.stopPushToTalk(),
|
||||
startContinuousVoiceCapture: () => this.onError("Voice capture is not available on iOS yet."),
|
||||
startContinuousVoiceCapture: options => this.startContinuousVoiceCapture(options?.recordAs),
|
||||
stopContinuousVoiceCapture: () => this.stopContinuousVoiceCapture(),
|
||||
startTextSettingEdit: async setting => { await this.editSetting(setting) },
|
||||
startTextSettingsEdit: async (settings, title, finished, toggle, cancelled) => {
|
||||
for (const setting of settings) if (!await this.editSetting(setting)) { cancelled?.(); return }
|
||||
@@ -941,10 +947,40 @@ export class IosPreviewController {
|
||||
await iosVoiceInput.startPhoneCapture(voice, log, endpointing)
|
||||
}
|
||||
}
|
||||
/** Open-ended capture (Transcribe, Teleprompter); holds voiceActivity like Android. */
|
||||
private async startContinuousVoiceCapture(recordAs?: string): Promise<void> {
|
||||
if (this.glassesLocked) return
|
||||
const voice = this.voiceProvider()
|
||||
if (!iosVoiceInput.supportsContinuous(voice)) {
|
||||
this.onError('Transcribe on iOS needs a cloud or Whisper server transcription provider (Settings > Voice > Transcription Provider).')
|
||||
return
|
||||
}
|
||||
// A stop during an await below bumps the token; this start then backs off.
|
||||
const token = ++this.continuousCaptureToken
|
||||
const current = () => token === this.continuousCaptureToken
|
||||
const log = (message: string) => this.logBluetooth(message)
|
||||
const record = recordAs && saveTranscribeAudioSetting.get() ? recordAs : ''
|
||||
voiceActivity.setActive(true, CONTINUOUS_CAPTURE_HOLDER)
|
||||
const communicator = this.communicator
|
||||
if (communicator && this.state.phase === 'connected') {
|
||||
await iosVoiceInput.startGlassesCapture(communicator, voice, log, false, true, record)
|
||||
} else if (this.active && await iosVoiceInput.prepare(this.active, true, voice) && current() && this.active && !this.glassesLocked) {
|
||||
await iosVoiceInput.startPhoneCapture(voice, log, false, true, record)
|
||||
}
|
||||
if (!current()) return
|
||||
if (!iosVoiceInput.continuousActive) voiceActivity.setActive(false, CONTINUOUS_CAPTURE_HOLDER)
|
||||
}
|
||||
private stopContinuousVoiceCapture(): void {
|
||||
++this.continuousCaptureToken
|
||||
voiceActivity.setActive(false, CONTINUOUS_CAPTURE_HOLDER)
|
||||
void iosVoiceInput.stopContinuousCapture().catch(error => this.logBluetooth(`Stop continuous capture: ${String(error)}`))
|
||||
}
|
||||
/** The transcription provider setting and the keys the cloud ones need. */
|
||||
private voiceProvider(): VoiceProviderSelection {
|
||||
return { provider: voiceProviderSetting.get(), elevenLabsApiKey: elevenLabsApiKeySetting.get(),
|
||||
openAiApiKey: openAiApiKeySetting.get(), sonioxApiKey: sonioxApiKeySetting.get() }
|
||||
openAiApiKey: openAiApiKeySetting.get(), sonioxApiKey: sonioxApiKeySetting.get(),
|
||||
whisperServerUrl: whisperServerUrlSetting.get(), whisperServerModel: whisperServerModelSetting.get(),
|
||||
whisperServerLanguage: whisperServerLanguageSetting.get(), whisperServerApiKey: whisperServerApiKeySetting.get() }
|
||||
}
|
||||
private logBluetooth(message: string): void {
|
||||
const line = `${new Date().toISOString()} [${this.active ? 'foreground' : 'background'}${UIApplication.sharedApplication.protectedDataAvailable ? '' : ',protected-data-unavailable'}] ${message}`
|
||||
|
||||
@@ -4,6 +4,7 @@ import { OpenAiRealtimeSttClient } from "./openai-stt";
|
||||
import { ReconnectingSttClient } from "./reconnecting-stt";
|
||||
import { SonioxSttClient } from "./soniox-stt";
|
||||
import type { VoiceProviderKind } from "./voice-control";
|
||||
import { normalizeWhisperServerUrl, WHISPER_SERVER_FINAL_TIMEOUT_MS, WhisperServerSttClient } from "./whisper-server-stt";
|
||||
|
||||
/**
|
||||
* The transcription-provider setting plus the keys the cloud providers need.
|
||||
@@ -14,12 +15,22 @@ export type VoiceProviderSelection = {
|
||||
elevenLabsApiKey: string;
|
||||
openAiApiKey: string;
|
||||
sonioxApiKey: string;
|
||||
/** Self-hosted Whisper ("whisper-server"); the key is optional there. */
|
||||
whisperServerUrl?: string;
|
||||
whisperServerModel?: string;
|
||||
whisperServerLanguage?: string;
|
||||
whisperServerApiKey?: string;
|
||||
};
|
||||
|
||||
type CloudProvider = {
|
||||
name: string;
|
||||
apiKey: (selection: VoiceProviderSelection) => string;
|
||||
create: (options: CloudSttOptions) => CloudSttClient;
|
||||
/** Whether the provider can run; defaults to having a key. */
|
||||
configured?: (selection: VoiceProviderSelection) => boolean;
|
||||
/** What to tell the user when it can't. */
|
||||
missing?: string;
|
||||
create: (options: CloudSttOptions, selection: VoiceProviderSelection) => CloudSttClient;
|
||||
finalTimeoutMs?: number;
|
||||
};
|
||||
|
||||
// Providers absent here ("onboard", "onboard-whisper") transcribe on-device.
|
||||
@@ -39,11 +50,28 @@ const CLOUD_PROVIDERS: Partial<Record<VoiceProviderKind, CloudProvider>> = {
|
||||
apiKey: (selection) => selection.openAiApiKey,
|
||||
create: (options) => new OpenAiRealtimeSttClient(options),
|
||||
},
|
||||
"whisper-server": {
|
||||
name: "Whisper server",
|
||||
apiKey: (selection) => selection.whisperServerApiKey ?? "",
|
||||
configured: (selection) => Boolean(normalizeWhisperServerUrl(selection.whisperServerUrl ?? "")),
|
||||
missing: "No Whisper server URL set; using on-device voice.",
|
||||
create: (options, selection) => new WhisperServerSttClient(options, {
|
||||
baseUrl: selection.whisperServerUrl ?? "",
|
||||
model: selection.whisperServerModel ?? "",
|
||||
language: selection.whisperServerLanguage ?? "",
|
||||
}),
|
||||
finalTimeoutMs: WHISPER_SERVER_FINAL_TIMEOUT_MS,
|
||||
},
|
||||
};
|
||||
|
||||
function isConfigured(provider: CloudProvider, selection: VoiceProviderSelection): boolean {
|
||||
return provider.configured ? provider.configured(selection) : Boolean(provider.apiKey(selection).trim());
|
||||
}
|
||||
|
||||
/** Whether createCloudSttClient would return a cloud client for this selection. */
|
||||
export function usesCloudStt(selection: VoiceProviderSelection): boolean {
|
||||
return Boolean(CLOUD_PROVIDERS[selection.provider]?.apiKey(selection).trim());
|
||||
const provider = CLOUD_PROVIDERS[selection.provider];
|
||||
return Boolean(provider && isConfigured(provider, selection));
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -56,13 +84,18 @@ export function createCloudSttClient(
|
||||
selection: VoiceProviderSelection,
|
||||
options: Omit<CloudSttOptions, "apiKey">,
|
||||
continuous: () => boolean,
|
||||
): CloudSttClient | null {
|
||||
): ReconnectingSttClient | null {
|
||||
const provider = CLOUD_PROVIDERS[selection.provider];
|
||||
if (!provider) return null;
|
||||
const apiKey = provider.apiKey(selection).trim();
|
||||
if (!apiKey) {
|
||||
options.onStatus(`No ${provider.name} key set; using on-device voice.`);
|
||||
if (!isConfigured(provider, selection)) {
|
||||
options.onStatus(provider.missing ?? `No ${provider.name} key set; using on-device voice.`);
|
||||
return null;
|
||||
}
|
||||
return new ReconnectingSttClient(provider.create, { ...options, apiKey }, continuous);
|
||||
const apiKey = provider.apiKey(selection).trim();
|
||||
return new ReconnectingSttClient(
|
||||
(clientOptions) => provider.create(clientOptions, selection),
|
||||
{ ...options, apiKey },
|
||||
continuous,
|
||||
provider.finalTimeoutMs,
|
||||
);
|
||||
}
|
||||
@@ -24,6 +24,8 @@ export class ReconnectingSttClient implements CloudSttClient {
|
||||
private readonly create: (options: CloudSttOptions) => CloudSttClient,
|
||||
private readonly options: CloudSttOptions,
|
||||
private readonly continuous: () => boolean,
|
||||
/** How long finish() waits for the provider's final transcript. */
|
||||
readonly finalTimeoutMs = 5000,
|
||||
) {}
|
||||
|
||||
start(): void {
|
||||
@@ -90,7 +92,7 @@ export class ReconnectingSttClient implements CloudSttClient {
|
||||
this.stop();
|
||||
return;
|
||||
}
|
||||
this.finishTimeout = setTimeout(() => this.stop(), 5000);
|
||||
this.finishTimeout = setTimeout(() => this.stop(), this.finalTimeoutMs);
|
||||
if (this.ready) this.client?.finish();
|
||||
}
|
||||
|
||||
|
||||
@@ -1,6 +1,8 @@
|
||||
import type { VoiceControlState, VoiceTranscriptEvent } from './voice-control'
|
||||
import type { CloudSttClient, CloudSttTranscriptEvent } from './cloud-stt'
|
||||
import type { CloudSttTranscriptEvent } from './cloud-stt'
|
||||
import type { ReconnectingSttClient } from './reconnecting-stt'
|
||||
import { createCloudSttClient, usesCloudStt, type VoiceProviderSelection } from './cloud-stt-provider'
|
||||
import { WavRecorder } from './wav-recorder.ios'
|
||||
|
||||
declare const FaceclawSpeech: any
|
||||
export type IosMicrophoneSession = {
|
||||
@@ -12,6 +14,8 @@ const CLOUD_FINAL_TIMEOUT_MS = 5000
|
||||
/** iOS implementation of the shared voice dialog's event bridge. Audio comes
|
||||
* from the glasses when connected, or the default phone input in preview mode,
|
||||
* and is transcribed by Apple's on-device recognizer or a cloud provider.
|
||||
* Continuous capture (Transcribe, Teleprompter) needs a cloud provider: only
|
||||
* the PCM capture runs open-ended, and pauses commit its segments.
|
||||
*/
|
||||
export class IosVoiceControlBridge {
|
||||
private native: any = null
|
||||
@@ -32,9 +36,14 @@ export class IosVoiceControlBridge {
|
||||
// Non-null while a cloud provider owns the transcript; FaceclawSpeech then
|
||||
// only supplies PCM. cloudEndStatus is set once the audio has ended and the
|
||||
// provider was asked to finalize; it is the status shown when it does.
|
||||
private cloud: CloudSttClient | null = null
|
||||
private cloud: ReconnectingSttClient | null = null
|
||||
private cloudEndStatus: string | null = null
|
||||
private cloudTimer: ReturnType<typeof setTimeout> | null = null
|
||||
// Open-ended capture: runs until stopContinuousCapture, and push-to-talk
|
||||
// joins it instead of restarting the mic (transcripts reach every listener).
|
||||
private continuous = false
|
||||
// The capture's audio as a WAV in Downloads, while recordTo asked for one.
|
||||
private recorder: WavRecorder | null = null
|
||||
|
||||
onStatus(listener: (state: VoiceControlState) => void): () => void {
|
||||
this.statuses.add(listener); listener({ status: this.status, listening: this.capturing, detail: '' }); return () => { this.statuses.delete(listener) }
|
||||
@@ -49,7 +58,11 @@ export class IosVoiceControlBridge {
|
||||
if (!this.native) {
|
||||
this.native = FaceclawSpeech.new()
|
||||
this.native.eventHandler = (json: string) => this.receive(JSON.parse(json))
|
||||
this.native.pcmHandler = (pcm: NSData) => this.cloud?.acceptPcm(new Uint8Array(interop.bufferFromData(pcm)))
|
||||
this.native.pcmHandler = (pcm: NSData) => {
|
||||
const bytes = new Uint8Array(interop.bufferFromData(pcm))
|
||||
this.cloud?.acceptPcm(bytes)
|
||||
this.recorder?.append(bytes)
|
||||
}
|
||||
}
|
||||
return this.native
|
||||
}
|
||||
@@ -74,9 +87,12 @@ export class IosVoiceControlBridge {
|
||||
return true
|
||||
}
|
||||
get statusText(): string { return this.status }
|
||||
async startGlassesCapture(session: IosMicrophoneSession, voice: VoiceProviderSelection, log: (message: string) => void, endpointing = false): Promise<void> {
|
||||
async startGlassesCapture(session: IosMicrophoneSession, voice: VoiceProviderSelection, log: (message: string) => void, endpointing = false, continuous = false, recordTo = ''): Promise<void> {
|
||||
if (this.joinsContinuous(continuous)) return
|
||||
this.stop()
|
||||
const generation = ++this.generation
|
||||
this.continuous = continuous
|
||||
this.startRecording(recordTo, log)
|
||||
this.log = log; this.source = 'glasses'; this.session = session; this.packets = 0; this.capturing = true
|
||||
this.completion = new Promise(resolve => { this.resolveCompletion = resolve })
|
||||
const startError = this.startNative(voice, endpointing, false)
|
||||
@@ -101,8 +117,11 @@ export class IosVoiceControlBridge {
|
||||
this.stop(); this.setStatus(error instanceof Error ? error.message : String(error)); this.emitEnd()
|
||||
}
|
||||
}
|
||||
async startPhoneCapture(voice: VoiceProviderSelection, log: (message: string) => void, endpointing = false): Promise<void> {
|
||||
async startPhoneCapture(voice: VoiceProviderSelection, log: (message: string) => void, endpointing = false, continuous = false, recordTo = ''): Promise<void> {
|
||||
if (this.joinsContinuous(continuous)) return
|
||||
this.stop()
|
||||
this.continuous = continuous
|
||||
this.startRecording(recordTo, log)
|
||||
this.log = log; this.source = 'phone'; this.capturing = true
|
||||
this.completion = new Promise(resolve => { this.resolveCompletion = resolve })
|
||||
const error = this.startNative(voice, endpointing, true)
|
||||
@@ -118,7 +137,7 @@ export class IosVoiceControlBridge {
|
||||
onTranscript: event => { if (current()) this.receiveCloudTranscript(event) },
|
||||
onStatus: status => { if (current()) this.setStatus(status) },
|
||||
onError: message => { if (current()) { this.stop(); this.setStatus(message); this.emitEnd() } },
|
||||
}, () => false)
|
||||
}, () => this.continuous)
|
||||
if (!cloud) return String((phone ? native.startPhoneWithEndpointing(endpointing) : native.startWithEndpointing(endpointing)) ?? '')
|
||||
const error = String(native.startPcmCaptureWithEndpointingPhone(endpointing, phone) ?? '')
|
||||
if (error) return error
|
||||
@@ -130,7 +149,26 @@ export class IosVoiceControlBridge {
|
||||
if (this.source !== 'phone') return
|
||||
this.stop(); this.setStatus('Phone voice input stopped.'); this.emitEnd()
|
||||
}
|
||||
/** Whether continuous capture can run with this provider (Apple's recognizer can't). */
|
||||
supportsContinuous(voice: VoiceProviderSelection): boolean { return usesCloudStt(voice) }
|
||||
get continuousActive(): boolean { return this.continuous && (this.capturing || this.finalizing) }
|
||||
/** End continuous capture, transcribing the audio since the last pause. */
|
||||
stopContinuousCapture(): Promise<void> {
|
||||
if (!this.continuous) return Promise.resolve()
|
||||
return this.finalizeCapture()
|
||||
}
|
||||
/** Push-to-talk shares a listening continuous capture (not one that is only finalizing). */
|
||||
private joinsContinuous(continuous: boolean): boolean {
|
||||
if (continuous || !this.continuous || !this.capturing) return false
|
||||
this.log('Voice: push-to-talk joins the running continuous capture')
|
||||
return true
|
||||
}
|
||||
stopPushToTalk(): Promise<void> {
|
||||
// The continuous capture owns the mic and keeps it.
|
||||
if (this.continuous) return Promise.resolve()
|
||||
return this.finalizeCapture()
|
||||
}
|
||||
private finalizeCapture(): Promise<void> {
|
||||
const completion = this.completion
|
||||
if (!this.capturing) return completion
|
||||
this.capturing = false; this.finalizing = true
|
||||
@@ -140,7 +178,7 @@ export class IosVoiceControlBridge {
|
||||
}
|
||||
stop(): void {
|
||||
++this.generation
|
||||
this.capturing = this.finalizing = false
|
||||
this.capturing = this.finalizing = this.continuous = false
|
||||
this.source = null
|
||||
this.clearAudioTimer(); this.releaseMicrophone(); this.native?.cancel()
|
||||
this.clearCloud()
|
||||
@@ -155,7 +193,14 @@ export class IosVoiceControlBridge {
|
||||
const session = this.session; this.session = null
|
||||
if (session) void session.setMicrophone(false).catch(error => this.log(`Voice microphone cleanup: ${error}`))
|
||||
}
|
||||
/** Record this capture's audio as Downloads/<name>.wav (only cloud captures supply PCM). */
|
||||
private startRecording(name: string, log: (message: string) => void): void {
|
||||
if (!name) return
|
||||
this.recorder = WavRecorder.open(name)
|
||||
log(this.recorder ? `Voice: recording audio to ${this.recorder.path}` : 'Voice: could not create the audio recording')
|
||||
}
|
||||
private clearCloud(): void {
|
||||
this.recorder?.close(); this.recorder = null
|
||||
if (this.cloudTimer !== null) clearTimeout(this.cloudTimer)
|
||||
this.cloudTimer = null
|
||||
this.cloud?.stop(); this.cloud = null; this.cloudEndStatus = null
|
||||
@@ -166,13 +211,13 @@ export class IosVoiceControlBridge {
|
||||
this.cloudEndStatus = status
|
||||
this.setStatus('Recognizing…')
|
||||
// Armed first: finish() can deliver the final synchronously.
|
||||
this.cloudTimer = setTimeout(() => this.completeCloud(), CLOUD_FINAL_TIMEOUT_MS)
|
||||
this.cloudTimer = setTimeout(() => this.completeCloud(), this.cloud?.finalTimeoutMs ?? CLOUD_FINAL_TIMEOUT_MS)
|
||||
this.cloud?.finish()
|
||||
}
|
||||
private completeCloud(): void {
|
||||
const status = this.cloudEndStatus ?? 'Ready to send'
|
||||
this.clearCloud()
|
||||
this.capturing = this.finalizing = false
|
||||
this.capturing = this.finalizing = this.continuous = false
|
||||
this.finishCompletion()
|
||||
this.setStatus(status); this.emitEnd()
|
||||
}
|
||||
@@ -180,7 +225,8 @@ export class IosVoiceControlBridge {
|
||||
if (!this.capturing && !this.finalizing) return
|
||||
for (const fn of [...this.transcripts]) fn(event)
|
||||
// Push-to-talk commits once, so the first final after it is the last.
|
||||
if (event.isFinal && this.cloudEndStatus !== null) this.completeCloud()
|
||||
// In continuous capture, earlier pause segments (paragraphBreakAfter) may still arrive.
|
||||
if (event.isFinal && this.cloudEndStatus !== null && !(this.continuous && event.paragraphBreakAfter)) this.completeCloud()
|
||||
}
|
||||
private emitEnd(): void { for (const fn of [...this.ends]) fn() }
|
||||
private finishCompletion(): void {
|
||||
|
||||
@@ -28,7 +28,9 @@ export type VoiceControlState = {
|
||||
// family, not with "onboard-whisper" below, which is the on-device sherpa-onnx
|
||||
// Whisper backend added alongside "onboard" (Moonshine). Kept as-is rather than
|
||||
// renamed, since it's a persisted settings value on real installs already.
|
||||
export type VoiceProviderKind = "onboard" | "onboard-whisper" | "elevenlabs" | "whisper" | "soniox";
|
||||
// "whisper-server" is a self-hosted OpenAI-compatible Whisper over HTTP
|
||||
// (whisper-server-stt.ts).
|
||||
export type VoiceProviderKind = "onboard" | "onboard-whisper" | "elevenlabs" | "whisper" | "soniox" | "whisper-server";
|
||||
|
||||
export type VoiceTranscriptEvent = CloudSttTranscriptEvent & {
|
||||
/**
|
||||
@@ -52,6 +54,10 @@ export type PushToTalkOptions = {
|
||||
elevenLabsApiKey: string;
|
||||
openAiApiKey: string;
|
||||
sonioxApiKey: string;
|
||||
whisperServerUrl?: string;
|
||||
whisperServerModel?: string;
|
||||
whisperServerLanguage?: string;
|
||||
whisperServerApiKey?: string;
|
||||
saveRecording: boolean;
|
||||
/**
|
||||
* Watch the mic and fire onSpeechEnd when the speaker stops. For hands-free
|
||||
|
||||
@@ -0,0 +1,22 @@
|
||||
/** The 44-byte header of a 16 kHz mono PCM16 WAV file holding `dataBytes` of samples. */
|
||||
export function wavHeader(dataBytes: number): Uint8Array {
|
||||
const header = new Uint8Array(44);
|
||||
const view = new DataView(header.buffer);
|
||||
const ascii = (offset: number, text: string) => {
|
||||
for (let i = 0; i < text.length; i++) header[offset + i] = text.charCodeAt(i);
|
||||
};
|
||||
ascii(0, "RIFF");
|
||||
view.setUint32(4, 36 + dataBytes, true);
|
||||
ascii(8, "WAVE");
|
||||
ascii(12, "fmt ");
|
||||
view.setUint32(16, 16, true);
|
||||
view.setUint16(20, 1, true); // PCM
|
||||
view.setUint16(22, 1, true); // mono
|
||||
view.setUint32(24, 16000, true);
|
||||
view.setUint32(28, 32000, true); // byte rate
|
||||
view.setUint16(32, 2, true); // block align
|
||||
view.setUint16(34, 16, true); // bits per sample
|
||||
ascii(36, "data");
|
||||
view.setUint32(40, dataBytes, true);
|
||||
return header;
|
||||
}
|
||||
@@ -0,0 +1,75 @@
|
||||
import { knownFolders, path as paths } from '@nativescript/core'
|
||||
import { wavHeader } from './wav-header'
|
||||
|
||||
// About a second of 16 kHz mono PCM16: written in batches, not per 50 ms packet.
|
||||
const FLUSH_BYTES = 32000
|
||||
|
||||
/**
|
||||
* Appends 16 kHz mono PCM16 to a WAV file in Documents/Downloads while it is
|
||||
* captured. The header's sizes are rewritten on every flush, so a crash leaves a
|
||||
* playable file of everything up to the last second.
|
||||
*/
|
||||
export class WavRecorder {
|
||||
private pending: Uint8Array[] = []
|
||||
private pendingBytes = 0
|
||||
private dataBytes = 0
|
||||
|
||||
private constructor(private handle: NSFileHandle | null, readonly path: string) {}
|
||||
|
||||
/** A recorder for Downloads/<name>.wav (a -2, -3... suffix if taken), or null. */
|
||||
static open(name: string): WavRecorder | null {
|
||||
const fm = NSFileManager.defaultManager
|
||||
const dir = paths.join(knownFolders.documents().path, 'Downloads')
|
||||
try {
|
||||
fm.createDirectoryAtPathWithIntermediateDirectoriesAttributesError(dir, true, null)
|
||||
let path = paths.join(dir, `${name}.wav`)
|
||||
for (let n = 2; fm.fileExistsAtPath(path); n++) path = paths.join(dir, `${name}-${n}.wav`)
|
||||
if (!fm.createFileAtPathContentsAttributes(path, toData(wavHeader(0)), null)) return null
|
||||
const handle = NSFileHandle.fileHandleForWritingAtPath(path)
|
||||
if (!handle || !handle.seekToEndReturningOffsetError(null)) return null
|
||||
return new WavRecorder(handle, path)
|
||||
} catch {
|
||||
return null
|
||||
}
|
||||
}
|
||||
|
||||
append(pcm: Uint8Array): void {
|
||||
if (!this.handle || !pcm.length) return
|
||||
this.pending.push(pcm.slice())
|
||||
this.pendingBytes += pcm.length
|
||||
if (this.pendingBytes >= FLUSH_BYTES) this.flush()
|
||||
}
|
||||
|
||||
close(): void {
|
||||
this.flush()
|
||||
try { this.handle?.closeAndReturnError() } catch { /* already closed */ }
|
||||
this.handle = null
|
||||
}
|
||||
|
||||
private flush(): void {
|
||||
const handle = this.handle
|
||||
if (!handle || !this.pendingBytes) return
|
||||
const chunk = new Uint8Array(this.pendingBytes)
|
||||
let offset = 0
|
||||
for (const part of this.pending) { chunk.set(part, offset); offset += part.length }
|
||||
this.pending = []
|
||||
this.pendingBytes = 0
|
||||
try {
|
||||
// NativeScript throws a failed call's NSError (storage full, file removed).
|
||||
handle.writeDataError(toData(chunk))
|
||||
handle.seekToOffsetError(0)
|
||||
handle.writeDataError(toData(wavHeader(this.dataBytes + chunk.length)))
|
||||
handle.seekToEndReturningOffsetError(null)
|
||||
this.dataBytes += chunk.length
|
||||
return
|
||||
} catch {
|
||||
// Stop recording rather than retry.
|
||||
}
|
||||
try { handle.closeAndReturnError() } catch { /* ignore */ }
|
||||
this.handle = null
|
||||
}
|
||||
}
|
||||
|
||||
function toData(bytes: Uint8Array): NSData {
|
||||
return NSData.dataWithBytesLength(interop.handleof(bytes.buffer), bytes.byteLength)
|
||||
}
|
||||
@@ -0,0 +1,386 @@
|
||||
import { CLOUD_STT_SAMPLE_RATE, type CloudSttClient, type CloudSttOptions } from "./cloud-stt";
|
||||
|
||||
/**
|
||||
* Speech-to-text against a self-hosted OpenAI-compatible Whisper server (vLLM,
|
||||
* speaches, LocalAI): batch POST {baseUrl}/audio/transcriptions with a WAV file.
|
||||
* Each segment (a push-to-talk utterance, or a Transcribe span ended by a pause)
|
||||
* is one request, finalized in audio order; an open segment is re-sent every
|
||||
* INTERIM_STEP_BYTES for provisional text. start() lists {baseUrl}/models to check
|
||||
* reachability and pick a model when none is configured.
|
||||
*/
|
||||
|
||||
export type WhisperServerConfig = {
|
||||
/** OpenAI-style API root, e.g. http://host:8004/v1. */
|
||||
baseUrl: string;
|
||||
/** Model id to request; empty uses the first one the server lists. */
|
||||
model: string;
|
||||
/** ISO-639-1 language code; empty lets Whisper detect it. */
|
||||
language: string;
|
||||
};
|
||||
|
||||
export type WhisperHttpRequest = {
|
||||
url: string;
|
||||
method: "GET" | "POST";
|
||||
headers: Record<string, string>;
|
||||
body?: ArrayBuffer;
|
||||
timeoutMs: number;
|
||||
};
|
||||
export type WhisperHttpResponse = { status: number; text: string };
|
||||
export type WhisperHttp = (request: WhisperHttpRequest) => Promise<WhisperHttpResponse>;
|
||||
|
||||
const BYTES_PER_SECOND = CLOUD_STT_SAMPLE_RATE * 2;
|
||||
// Provisional transcripts: first after one second of audio, then every 1.5s,
|
||||
// and not for segments past Whisper's 30s window (each would re-send it all).
|
||||
const INTERIM_MIN_BYTES = BYTES_PER_SECOND;
|
||||
const INTERIM_STEP_BYTES = BYTES_PER_SECOND * 1.5;
|
||||
const INTERIM_MAX_BYTES = BYTES_PER_SECOND * 30;
|
||||
const MODELS_TIMEOUT_MS = 5000;
|
||||
const INTERIM_TIMEOUT_MS = 10000;
|
||||
const FINAL_TIMEOUT_MS = 12000;
|
||||
const FINAL_RETRY_DELAY_MS = 750;
|
||||
// Whisper invents text ("Thank you.") for silence, so a segment whose loudest
|
||||
// 50ms frame stays below this RMS is not sent at all. Normal speech on the G2
|
||||
// mic measures in the thousands; the pause detector's noise floor starts at 220.
|
||||
const SILENCE_RMS = 300;
|
||||
|
||||
/** Time the voice bridges should allow a final transcript before giving up. */
|
||||
export const WHISPER_SERVER_FINAL_TIMEOUT_MS = FINAL_TIMEOUT_MS * 2 + FINAL_RETRY_DELAY_MS + 2000;
|
||||
|
||||
type Segment = {
|
||||
chunks: Uint8Array[];
|
||||
bytes: number;
|
||||
peakRms: number;
|
||||
text: string;
|
||||
final: boolean;
|
||||
closed: boolean;
|
||||
paragraphBreakAfter: boolean;
|
||||
/** Emit a final even when empty: the end of push-to-talk needs one. */
|
||||
required: boolean;
|
||||
interimInFlight: boolean;
|
||||
interimBytes: number;
|
||||
};
|
||||
|
||||
/**
|
||||
* The API root from what a user typed: tolerates a trailing slash or the full
|
||||
* transcription endpoint, and adds /v1 to a bare host. Empty when unusable.
|
||||
*/
|
||||
export function normalizeWhisperServerUrl(input: string): string {
|
||||
let url = input.trim().replace(/\/+$/, "");
|
||||
if (!/^https?:\/\/[^/\s]+/i.test(url)) return "";
|
||||
url = url.replace(/\/audio\/transcriptions$/i, "");
|
||||
if (/^https?:\/\/[^/]+$/i.test(url)) url += "/v1";
|
||||
return url;
|
||||
}
|
||||
|
||||
export class WhisperServerSttClient implements CloudSttClient {
|
||||
private readonly baseUrl: string;
|
||||
private model: string;
|
||||
private ready = false;
|
||||
private closed = false;
|
||||
private readonly segments: Segment[] = [];
|
||||
|
||||
constructor(
|
||||
private readonly options: CloudSttOptions,
|
||||
private readonly config: WhisperServerConfig,
|
||||
private readonly http: WhisperHttp = nativeHttp,
|
||||
) {
|
||||
this.baseUrl = normalizeWhisperServerUrl(config.baseUrl);
|
||||
this.model = config.model.trim();
|
||||
}
|
||||
|
||||
start(): void {
|
||||
if (this.closed || this.ready) return;
|
||||
if (!this.baseUrl) {
|
||||
this.options.onError("Whisper server URL is not a valid http(s) address.");
|
||||
return;
|
||||
}
|
||||
this.options.onStatus("Connecting to Whisper server...");
|
||||
this.http({ url: `${this.baseUrl}/models`, method: "GET", headers: this.authHeaders(), timeoutMs: MODELS_TIMEOUT_MS })
|
||||
.then((response) => {
|
||||
if (this.closed) return;
|
||||
if (response.status < 200 || response.status >= 300) {
|
||||
this.options.onDisconnected?.(`Whisper server returned ${response.status} for /models.`);
|
||||
return;
|
||||
}
|
||||
if (!this.model) this.model = firstModelId(response.text);
|
||||
if (!this.model) {
|
||||
this.options.onError("Whisper server lists no models; set the model in Settings > Voice.");
|
||||
return;
|
||||
}
|
||||
this.ready = true;
|
||||
this.options.onStatus("Listening (Whisper server)...");
|
||||
this.options.onReady?.();
|
||||
})
|
||||
.catch((error) => {
|
||||
if (!this.closed) this.options.onDisconnected?.(`Whisper server unreachable: ${describeError(error)}`);
|
||||
});
|
||||
}
|
||||
|
||||
/** PCM16LE at 16 kHz, appended to the open segment. */
|
||||
acceptPcm(pcm: Uint8Array): void {
|
||||
if (this.closed || !pcm.length) return;
|
||||
const segment = this.openSegment();
|
||||
segment.chunks.push(pcm.slice());
|
||||
segment.bytes += pcm.length;
|
||||
segment.peakRms = Math.max(segment.peakRms, rms(pcm));
|
||||
this.maybeRequestInterim(segment);
|
||||
}
|
||||
|
||||
/** A pause in continuous capture: transcribe what came before it. */
|
||||
commitSegment(): void {
|
||||
this.closeSegment(true, false);
|
||||
}
|
||||
|
||||
/** End of the utterance: transcribe the rest and deliver a final. */
|
||||
finish(): void {
|
||||
this.closeSegment(false, true);
|
||||
}
|
||||
|
||||
stop(): void {
|
||||
this.closed = true;
|
||||
this.segments.length = 0;
|
||||
}
|
||||
|
||||
private openSegment(): Segment {
|
||||
const last = this.segments[this.segments.length - 1];
|
||||
if (last && !last.closed) return last;
|
||||
const segment: Segment = {
|
||||
chunks: [], bytes: 0, peakRms: 0, text: "", final: false, closed: false,
|
||||
paragraphBreakAfter: false, required: false, interimInFlight: false, interimBytes: 0,
|
||||
};
|
||||
this.segments.push(segment);
|
||||
return segment;
|
||||
}
|
||||
|
||||
private closeSegment(paragraphBreakAfter: boolean, required: boolean): void {
|
||||
if (this.closed) return;
|
||||
const last = this.segments[this.segments.length - 1];
|
||||
if (!last || last.closed) {
|
||||
// Nothing new since the last boundary. Push-to-talk still needs a final.
|
||||
if (required) this.finalize(this.openSegment(), "", paragraphBreakAfter, required);
|
||||
return;
|
||||
}
|
||||
if (last.peakRms < SILENCE_RMS) {
|
||||
this.finalize(last, "", paragraphBreakAfter, required);
|
||||
return;
|
||||
}
|
||||
last.closed = true;
|
||||
last.paragraphBreakAfter = paragraphBreakAfter;
|
||||
last.required = required;
|
||||
void this.requestFinal(last, 0);
|
||||
}
|
||||
|
||||
private async requestFinal(segment: Segment, attempt: number): Promise<void> {
|
||||
try {
|
||||
const text = await this.transcribe(segment, segment.bytes, FINAL_TIMEOUT_MS);
|
||||
if (!this.closed) this.finalize(segment, text, segment.paragraphBreakAfter, segment.required);
|
||||
} catch (error) {
|
||||
if (this.closed) return;
|
||||
if (error instanceof WhisperAuthError) {
|
||||
this.options.onError(error.message);
|
||||
return;
|
||||
}
|
||||
if (attempt === 0) {
|
||||
setTimeout(() => { void this.requestFinal(segment, 1); }, FINAL_RETRY_DELAY_MS);
|
||||
return;
|
||||
}
|
||||
// Keep the capture alive and keep whatever the interim pass heard.
|
||||
this.options.onStatus(`Whisper server request failed: ${describeError(error)}`);
|
||||
this.finalize(segment, segment.text, segment.paragraphBreakAfter, segment.required);
|
||||
}
|
||||
}
|
||||
|
||||
private maybeRequestInterim(segment: Segment): void {
|
||||
if (!this.ready || segment.interimInFlight || segment.closed) return;
|
||||
if (segment.bytes < INTERIM_MIN_BYTES || segment.bytes > INTERIM_MAX_BYTES) return;
|
||||
if (segment.bytes - segment.interimBytes < INTERIM_STEP_BYTES) return;
|
||||
if (segment.peakRms < SILENCE_RMS) return;
|
||||
segment.interimInFlight = true;
|
||||
segment.interimBytes = segment.bytes;
|
||||
this.transcribe(segment, segment.bytes, INTERIM_TIMEOUT_MS)
|
||||
.then((text) => {
|
||||
if (this.closed || segment.final) return;
|
||||
segment.text = text;
|
||||
this.emit();
|
||||
})
|
||||
.catch(() => {
|
||||
// Provisional only; the final request reports real failures.
|
||||
})
|
||||
.finally(() => {
|
||||
segment.interimInFlight = false;
|
||||
});
|
||||
}
|
||||
|
||||
private finalize(segment: Segment, text: string, paragraphBreakAfter: boolean, required: boolean): void {
|
||||
segment.closed = segment.final = true;
|
||||
segment.text = text;
|
||||
segment.paragraphBreakAfter = paragraphBreakAfter;
|
||||
segment.required = required;
|
||||
segment.chunks = [];
|
||||
this.emit();
|
||||
}
|
||||
|
||||
/**
|
||||
* Finals leave in audio order, even when a later request answers first;
|
||||
* whatever is not final yet is shown as one provisional transcript.
|
||||
*/
|
||||
private emit(): void {
|
||||
while (this.segments.length && this.segments[0]!.final) {
|
||||
const segment = this.segments.shift()!;
|
||||
if (!segment.text && !segment.required) continue;
|
||||
this.options.onTranscript({ text: segment.text, isFinal: true, paragraphBreakAfter: segment.paragraphBreakAfter });
|
||||
}
|
||||
const pending = this.segments.filter((segment) => segment.text.trim());
|
||||
if (!pending.length) return;
|
||||
const text = pending.map((segment) => segment.text.trim()).join(" ");
|
||||
const transcribeText = pending.map((segment, index) =>
|
||||
(index && pending[index - 1]!.paragraphBreakAfter ? "\n" : index ? " " : "") + segment.text.trim(),
|
||||
).join("");
|
||||
this.options.onTranscript({ text, transcribeText, isFinal: false });
|
||||
}
|
||||
|
||||
private async transcribe(segment: Segment, bytes: number, timeoutMs: number): Promise<string> {
|
||||
const fields: Record<string, string> = { model: this.model, response_format: "json", temperature: "0" };
|
||||
const language = this.config.language.trim().toLowerCase();
|
||||
if (language) fields.language = language;
|
||||
const boundary = `faceclaw-${Date.now().toString(36)}-${Math.random().toString(36).slice(2)}`;
|
||||
const body = multipartBody(boundary, fields, wavFile(segment.chunks, bytes));
|
||||
const response = await this.http({
|
||||
url: `${this.baseUrl}/audio/transcriptions`,
|
||||
method: "POST",
|
||||
headers: { ...this.authHeaders(), "Content-Type": `multipart/form-data; boundary=${boundary}` },
|
||||
body,
|
||||
timeoutMs,
|
||||
});
|
||||
if (response.status === 401 || response.status === 403) {
|
||||
throw new WhisperAuthError(`Whisper server rejected the key (${response.status}).`);
|
||||
}
|
||||
if (response.status < 200 || response.status >= 300) {
|
||||
throw new Error(`HTTP ${response.status}: ${response.text.slice(0, 200)}`);
|
||||
}
|
||||
return parseTranscript(response.text);
|
||||
}
|
||||
|
||||
private authHeaders(): Record<string, string> {
|
||||
const key = this.options.apiKey.trim();
|
||||
return key ? { Authorization: `Bearer ${key}` } : {};
|
||||
}
|
||||
}
|
||||
|
||||
class WhisperAuthError extends Error {}
|
||||
|
||||
/** The transcript from a JSON response; tolerates servers that send plain text. */
|
||||
export function parseTranscript(body: string): string {
|
||||
const raw = body.trim();
|
||||
try {
|
||||
const parsed = JSON.parse(raw);
|
||||
if (parsed && typeof parsed.text === "string") return parsed.text.trim();
|
||||
} catch {
|
||||
// Not JSON: a server that answered response_format=text anyway.
|
||||
}
|
||||
return raw;
|
||||
}
|
||||
|
||||
function firstModelId(body: string): string {
|
||||
try {
|
||||
const parsed = JSON.parse(body);
|
||||
const id = parsed?.data?.[0]?.id;
|
||||
return typeof id === "string" ? id : "";
|
||||
} catch {
|
||||
return "";
|
||||
}
|
||||
}
|
||||
|
||||
/** RMS of a PCM16LE chunk. */
|
||||
function rms(pcm: Uint8Array): number {
|
||||
const count = pcm.length >> 1;
|
||||
if (!count) return 0;
|
||||
let squares = 0;
|
||||
for (let i = 0; i < count; i++) {
|
||||
const unsigned = pcm[i * 2]! | (pcm[i * 2 + 1]! << 8);
|
||||
const sample = unsigned >= 32768 ? unsigned - 65536 : unsigned;
|
||||
squares += sample * sample;
|
||||
}
|
||||
return Math.sqrt(squares / count);
|
||||
}
|
||||
|
||||
/** The first `bytes` of the chunks as a 16 kHz mono 16-bit WAV file. */
|
||||
export function wavFile(chunks: Uint8Array[], bytes: number): Uint8Array {
|
||||
const length = bytes & ~1;
|
||||
const out = new Uint8Array(44 + length);
|
||||
const view = new DataView(out.buffer);
|
||||
const ascii = (offset: number, text: string) => {
|
||||
for (let i = 0; i < text.length; i++) out[offset + i] = text.charCodeAt(i);
|
||||
};
|
||||
ascii(0, "RIFF");
|
||||
view.setUint32(4, 36 + length, true);
|
||||
ascii(8, "WAVE");
|
||||
ascii(12, "fmt ");
|
||||
view.setUint32(16, 16, true);
|
||||
view.setUint16(20, 1, true); // PCM
|
||||
view.setUint16(22, 1, true); // mono
|
||||
view.setUint32(24, CLOUD_STT_SAMPLE_RATE, true);
|
||||
view.setUint32(28, BYTES_PER_SECOND, true);
|
||||
view.setUint16(32, 2, true);
|
||||
view.setUint16(34, 16, true);
|
||||
ascii(36, "data");
|
||||
view.setUint32(40, length, true);
|
||||
let offset = 44;
|
||||
for (const chunk of chunks) {
|
||||
const take = Math.min(chunk.length, out.length - offset);
|
||||
if (take <= 0) break;
|
||||
out.set(chunk.subarray(0, take), offset);
|
||||
offset += take;
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/** multipart/form-data with text fields and one WAV part named "file". */
|
||||
export function multipartBody(boundary: string, fields: Record<string, string>, wav: Uint8Array): ArrayBuffer {
|
||||
let head = "";
|
||||
for (const [name, value] of Object.entries(fields)) {
|
||||
head += `--${boundary}\r\nContent-Disposition: form-data; name="${name}"\r\n\r\n${value}\r\n`;
|
||||
}
|
||||
head += `--${boundary}\r\nContent-Disposition: form-data; name="file"; filename="audio.wav"\r\nContent-Type: audio/wav\r\n\r\n`;
|
||||
const headBytes = utf8(head);
|
||||
const tailBytes = utf8(`\r\n--${boundary}--\r\n`);
|
||||
const out = new Uint8Array(headBytes.length + wav.length + tailBytes.length);
|
||||
out.set(headBytes, 0);
|
||||
out.set(wav, headBytes.length);
|
||||
out.set(tailBytes, headBytes.length + wav.length);
|
||||
return out.buffer;
|
||||
}
|
||||
|
||||
function utf8(text: string): Uint8Array {
|
||||
const out: number[] = [];
|
||||
for (const char of text) {
|
||||
const code = char.codePointAt(0)!;
|
||||
if (code < 0x80) out.push(code);
|
||||
else if (code < 0x800) out.push(0xc0 | (code >> 6), 0x80 | (code & 63));
|
||||
else if (code < 0x10000) out.push(0xe0 | (code >> 12), 0x80 | ((code >> 6) & 63), 0x80 | (code & 63));
|
||||
else out.push(0xf0 | (code >> 18), 0x80 | ((code >> 12) & 63), 0x80 | ((code >> 6) & 63), 0x80 | (code & 63));
|
||||
}
|
||||
return Uint8Array.from(out);
|
||||
}
|
||||
|
||||
function describeError(error: unknown): string {
|
||||
return String((error as Error)?.message ?? error);
|
||||
}
|
||||
|
||||
/** NativeScript's HTTP module: sends an ArrayBuffer body as-is on both platforms. */
|
||||
const nativeHttp: WhisperHttp = async (request) => {
|
||||
// Required lazily so Node tests can load this module without NativeScript.
|
||||
const { withUserAgent } = require("../util/http") as typeof import("../util/http");
|
||||
const { request: send } = require("@nativescript/core/http") as typeof import("@nativescript/core/http");
|
||||
const response = await send({
|
||||
url: request.url,
|
||||
method: request.method,
|
||||
headers: withUserAgent(request.headers),
|
||||
content: request.body,
|
||||
timeout: request.timeoutMs,
|
||||
});
|
||||
const content = response.content as any;
|
||||
let text = "";
|
||||
if (content) text = global.isAndroid ? String(content.raw.toString("UTF-8")) : String(content.toString());
|
||||
return { status: response.statusCode, text };
|
||||
};
|
||||
@@ -17,6 +17,7 @@ import {
|
||||
type AssistantModel,
|
||||
} from "~/assistant/models";
|
||||
import { isLocalModelReady } from "../native/llama";
|
||||
import { normalizeWhisperServerUrl } from "../native/whisper-server-stt";
|
||||
import { drawRightValueMenuItem, drawToggleMenuItem, MenuItem, openModalMenu } from "./menu";
|
||||
import { ANIMATION_SPEEDS, animationSpeedLabel, scaleAnimationDuration, type AnimationSpeed } from "./animation-speed";
|
||||
import { MENU_ANIMATION_KEY } from "./menu-animation-pref";
|
||||
@@ -822,7 +823,8 @@ export const ringConnectionModeSetting = new ConfigSettingEnum<RingConnectionMod
|
||||
// runs -- see the same note in native/voice-control.ts. The "whisper" value
|
||||
// keeps its name (it's a persisted setting on real installs) but its label
|
||||
// below now says "OpenAI" to tell the two apart in the picker.
|
||||
export type VoiceProvider = "onboard" | "onboard-whisper" | "elevenlabs" | "whisper" | "soniox";
|
||||
// "whisper-server" is a self-hosted OpenAI-compatible Whisper (e.g. vLLM).
|
||||
export type VoiceProvider = "onboard" | "onboard-whisper" | "elevenlabs" | "whisper" | "soniox" | "whisper-server";
|
||||
|
||||
// On iOS "onboard" is Apple's on-device recognizer, and there is no
|
||||
// "onboard-whisper" (the sherpa-onnx models are Android-only).
|
||||
@@ -832,6 +834,7 @@ const voiceProviderLabels: Record<VoiceProvider, string> = {
|
||||
elevenlabs: "ElevenLabs",
|
||||
whisper: "OpenAI (Whisper)",
|
||||
soniox: "Soniox",
|
||||
"whisper-server": "Whisper server (self-hosted)",
|
||||
};
|
||||
|
||||
export const voiceProviderSetting = new ConfigSettingEnum<VoiceProvider>({
|
||||
@@ -839,17 +842,20 @@ export const voiceProviderSetting = new ConfigSettingEnum<VoiceProvider>({
|
||||
label: "Transcription Provider",
|
||||
storageKey: "voice.provider",
|
||||
defaultValue: "onboard",
|
||||
values: global.isIOS ? ["onboard", "elevenlabs", "whisper", "soniox"] : ["onboard", "onboard-whisper", "elevenlabs", "whisper", "soniox"],
|
||||
values: global.isIOS
|
||||
? ["onboard", "elevenlabs", "whisper", "soniox", "whisper-server"]
|
||||
: ["onboard", "onboard-whisper", "elevenlabs", "whisper", "soniox", "whisper-server"],
|
||||
formatValue: (value) => voiceProviderLabels[value] ?? value,
|
||||
isDisabled: (value) => {
|
||||
if (value === "elevenlabs") return elevenLabsApiKeySetting.get().trim().length === 0;
|
||||
if (value === "whisper") return openAiApiKeySetting.get().trim().length === 0;
|
||||
if (value === "soniox") return sonioxApiKeySetting.get().trim().length === 0;
|
||||
if (value === "whisper-server") return !normalizeWhisperServerUrl(whisperServerUrlSetting.get());
|
||||
return false;
|
||||
},
|
||||
description: global.isIOS
|
||||
? "Speech-to-text engine for voice input. ElevenLabs, OpenAI, and Soniox are cloud services that need an API key (Settings > API Keys), with significantly better accuracy than on-device transcription. On-device uses your iPhone's speech language and never leaves the phone."
|
||||
: "Speech-to-text engine for voice input. ElevenLabs, OpenAI, and Soniox are cloud services that need an API key, with significantly better accuracy than on-device transcription. The two On-device options need their voice model downloaded (below) and never leave the phone.",
|
||||
? "Speech-to-text engine for voice input. ElevenLabs, OpenAI, and Soniox are cloud services that need an API key (Settings > API Keys), with significantly better accuracy than on-device transcription. Whisper server is your own OpenAI-compatible Whisper (set its URL below). On-device uses your iPhone's speech language and never leaves the phone."
|
||||
: "Speech-to-text engine for voice input. ElevenLabs, OpenAI, and Soniox are cloud services that need an API key, with significantly better accuracy than on-device transcription. Whisper server is your own OpenAI-compatible Whisper (set its URL below). The two On-device options need their voice model downloaded (below) and never leave the phone.",
|
||||
});
|
||||
|
||||
const wakeWordActionLabels: Record<WakeWordAction, string> = {
|
||||
@@ -868,6 +874,14 @@ export const wakeWordActionSetting = new ConfigSettingEnum<WakeWordAction>({
|
||||
description: "What saying \"Hey Even\" does: start voice input, just turn the screen on, or nothing.",
|
||||
});
|
||||
|
||||
export const saveTranscribeAudioSetting = new ConfigSettingBoolean({
|
||||
id: "save-transcribe-audio",
|
||||
label: "Save Transcribe audio",
|
||||
storageKey: "voice.saveTranscribeAudio",
|
||||
defaultValue: true,
|
||||
description: "Record each Transcribe session's audio as a WAV file in Downloads (about 2 MB per minute), named like its transcript. Recordings are kept until you delete them.",
|
||||
});
|
||||
|
||||
export const saveVoiceRecordingsSetting = new ConfigSettingBoolean({
|
||||
id: "save-voice-recordings",
|
||||
label: "Save voice recordings",
|
||||
@@ -976,6 +990,54 @@ export const sonioxApiKeySetting = new ConfigSettingString({
|
||||
description: "Soniox API key, used when Soniox is the transcription provider.",
|
||||
});
|
||||
|
||||
export const whisperServerUrlSetting = new ConfigSettingString({
|
||||
id: "whisper-server-url",
|
||||
label: "Whisper server URL",
|
||||
storageKey: "voice.whisperServerUrl",
|
||||
defaultValue: "",
|
||||
editorTitle: "Whisper server URL (e.g. http://100.x.y.z:8004/v1)",
|
||||
glassesEditTitle: "Edit Whisper server URL",
|
||||
formatValue: (value) => value || "(not set)",
|
||||
validate: (value) => (!value.trim() || normalizeWhisperServerUrl(value) ? null : "Enter an http:// or https:// address."),
|
||||
description:
|
||||
"API root of an OpenAI-compatible Whisper server (vLLM, speaches, LocalAI...), used when Whisper server is the transcription provider. On iOS a plain-http address must also be allowed by the app's App Transport Security exceptions.",
|
||||
});
|
||||
|
||||
export const whisperServerModelSetting = new ConfigSettingString({
|
||||
id: "whisper-server-model",
|
||||
label: "Whisper server model",
|
||||
storageKey: "voice.whisperServerModel",
|
||||
defaultValue: "",
|
||||
editorTitle: "Whisper model id (empty: server's first)",
|
||||
glassesEditTitle: "Edit Whisper model",
|
||||
formatValue: (value) => value || "(server default)",
|
||||
description: "Model id sent to the Whisper server. Leave empty to use the first model the server lists.",
|
||||
});
|
||||
|
||||
export const whisperServerLanguageSetting = new ConfigSettingString({
|
||||
id: "whisper-server-language",
|
||||
label: "Whisper language",
|
||||
storageKey: "voice.whisperServerLanguage",
|
||||
defaultValue: "",
|
||||
editorTitle: "Language code, e.g. en or sk (empty: auto)",
|
||||
glassesEditTitle: "Edit Whisper language",
|
||||
formatValue: (value) => value || "Auto-detect",
|
||||
validate: (value) => (!value.trim() || /^[a-z]{2,3}$/i.test(value.trim()) ? null : "Use a language code like en, sk or cs."),
|
||||
description:
|
||||
"Spoken language for the Whisper server. Leave empty to auto-detect; a wrong fixed language makes Whisper translate into it instead of transcribing.",
|
||||
});
|
||||
|
||||
export const whisperServerApiKeySetting = new ConfigSettingString({
|
||||
id: "whisper-server-api-key",
|
||||
label: "Whisper server key",
|
||||
storageKey: "voice.whisperServerApiKey",
|
||||
defaultValue: "",
|
||||
editorTitle: "Whisper server API key (optional)",
|
||||
glassesEditTitle: "Edit Whisper server key",
|
||||
formatValue: (value) => (value ? `${value.slice(0, 6)}...` : "(none)"),
|
||||
description: "Bearer token for the Whisper server, if it requires one.",
|
||||
});
|
||||
|
||||
export const anthropicApiKeySetting = new ConfigSettingString({
|
||||
id: "anthropic-api-key",
|
||||
label: "Anthropic key",
|
||||
|
||||
@@ -61,6 +61,11 @@ import {
|
||||
phoneRotationSetting,
|
||||
ringConnectionModeSetting,
|
||||
sonioxApiKeySetting,
|
||||
saveTranscribeAudioSetting,
|
||||
whisperServerApiKeySetting,
|
||||
whisperServerLanguageSetting,
|
||||
whisperServerModelSetting,
|
||||
whisperServerUrlSetting,
|
||||
screenFadeSetting,
|
||||
enumSettingMenuItem,
|
||||
firmwareDebugFlagsSetting,
|
||||
@@ -126,6 +131,7 @@ function settingsSections(): SettingsSection[] {
|
||||
items: [
|
||||
enumSettingMenuItem(wakeWordActionSetting),
|
||||
enumSettingMenuItem(voiceProviderSetting),
|
||||
...whisperServerMenuItems(),
|
||||
asrModelMenuItem("moonshine"),
|
||||
asrModelMenuItem("whisper-base-en"),
|
||||
],
|
||||
@@ -240,7 +246,7 @@ function settingsSections(): SettingsSection[] {
|
||||
toggleSettingMenuItem(showBleBandwidthSetting)] };
|
||||
// No downloadable on-device models: iOS's on-device option is Apple's recognizer.
|
||||
if (section.label === "Voice") return { label: "Voice", items: [enumSettingMenuItem(wakeWordActionSetting),
|
||||
enumSettingMenuItem(voiceProviderSetting)] };
|
||||
enumSettingMenuItem(voiceProviderSetting), toggleSettingMenuItem(saveTranscribeAudioSetting), ...whisperServerMenuItems()] };
|
||||
if (deferred.has(section.label)) return { label: section.label, items: [{
|
||||
label: "Not available on iOS yet", disabled: true, onSelect: () => {},
|
||||
description: `${section.label} integration has not been ported to iOS.`,
|
||||
@@ -454,6 +460,19 @@ function asrModelStatusText(id: AsrModelId): string {
|
||||
return "not downloaded";
|
||||
}
|
||||
|
||||
/**
|
||||
* The self-hosted Whisper server's settings. Always listed: the provider
|
||||
* option stays disabled until the URL is set.
|
||||
*/
|
||||
function whisperServerMenuItems(): MenuItem[] {
|
||||
return [
|
||||
textSettingMenuItem(whisperServerUrlSetting),
|
||||
textSettingMenuItem(whisperServerModelSetting),
|
||||
textSettingMenuItem(whisperServerLanguageSetting),
|
||||
textSettingMenuItem(whisperServerApiKeySetting),
|
||||
];
|
||||
}
|
||||
|
||||
/** Download/cancel/delete management for one on-device transcription model. */
|
||||
function asrModelMenuItem(id: AsrModelId): MenuItem {
|
||||
const def = ASR_MODELS[id];
|
||||
|
||||
+3
-2
@@ -33,9 +33,10 @@ export type LayerActions = {
|
||||
stopVoiceCapture: () => Promise<void> | void;
|
||||
/**
|
||||
* Start continuous capture on behalf of `client` (one id per window); shares
|
||||
* the mic with push-to-talk, and runs while any client wants it.
|
||||
* the mic with push-to-talk, and runs while any client wants it. recordAs
|
||||
* names a WAV of the captured audio in Downloads (iOS, when enabled).
|
||||
*/
|
||||
startContinuousVoiceCapture: (client: string) => Promise<void> | void;
|
||||
startContinuousVoiceCapture: (client: string, options?: { recordAs?: string }) => Promise<void> | void;
|
||||
stopContinuousVoiceCapture: (client: string) => Promise<void> | void;
|
||||
/** Play a CFW tone-sequencer payload (see sound-effects.ts). */
|
||||
playBuzzerSequence: (payload: Uint8Array) => Promise<void> | void;
|
||||
|
||||
@@ -0,0 +1,44 @@
|
||||
# Self-hosted Whisper transcription
|
||||
|
||||
The "Whisper server (self-hosted)" transcription provider sends voice audio to
|
||||
any OpenAI-compatible Whisper server instead of a cloud service: vLLM, speaches
|
||||
(faster-whisper-server), LocalAI and similar. It uses the batch endpoint
|
||||
`POST {url}/audio/transcriptions` with a 16 kHz mono WAV. Push-to-talk
|
||||
utterances are one request each; in Transcribe, each span ended by a pause is
|
||||
one request. While a span is open its audio so far is re-sent every 1.5 s, so
|
||||
text appears while you are still talking.
|
||||
|
||||
## Server
|
||||
|
||||
Any server exposing `/v1/models` and `/v1/audio/transcriptions` works, e.g.
|
||||
|
||||
vllm serve openai/whisper-large-v3 --port 8004
|
||||
|
||||
## Settings (Settings > Voice)
|
||||
|
||||
- **Transcription Provider**: Whisper server (self-hosted). Disabled until the
|
||||
URL is set.
|
||||
- **Whisper server URL**: the API root, e.g. `http://host:8004/v1`. A bare
|
||||
`http://host:8004` gets `/v1` added.
|
||||
- **Whisper server model**: empty uses the first model the server lists.
|
||||
- **Whisper language**: an ISO code such as `en`, or empty to auto-detect.
|
||||
A wrong fixed language makes Whisper translate rather than transcribe.
|
||||
- **Whisper server key**: optional bearer token.
|
||||
|
||||
## iOS: plain http
|
||||
|
||||
iOS App Transport Security blocks plain `http://` to IP addresses (including
|
||||
Tailscale's 100.x addresses) unless the app's Info.plist has an exception for
|
||||
that host. Either serve the API over https (for example with `tailscale serve`)
|
||||
or add your server to `NSExceptionDomains` in `App_Resources/iOS/Info.plist`
|
||||
before building:
|
||||
|
||||
<key>192.0.2.10</key>
|
||||
<dict><key>NSExceptionAllowsInsecureHTTPLoads</key><true/></dict>
|
||||
|
||||
## Transcribe on iOS
|
||||
|
||||
Transcribe needs a cloud or Whisper-server provider on iOS (Apple's recognizer
|
||||
only handles short utterances). With "Save Transcribe audio" on (Voice
|
||||
settings), each session's audio is kept as `transcribe-<time>.wav` in Downloads,
|
||||
next to `transcript-<time>.txt` when the text is saved.
|
||||
@@ -6,7 +6,9 @@ const ts = require('typescript');
|
||||
const path = require('node:path');
|
||||
// The provider settings ios-preview-controller hands the voice bridge.
|
||||
const voiceSettings = { voiceProviderSetting: { get: () => 'onboard' }, elevenLabsApiKeySetting: { get: () => '' },
|
||||
openAiApiKeySetting: { get: () => '' }, sonioxApiKeySetting: { get: () => '' } };
|
||||
openAiApiKeySetting: { get: () => '' }, sonioxApiKeySetting: { get: () => '' },
|
||||
whisperServerUrlSetting: { get: () => '' }, whisperServerModelSetting: { get: () => '' },
|
||||
whisperServerLanguageSetting: { get: () => '' }, whisperServerApiKeySetting: { get: () => '' } };
|
||||
function load(file, context) {
|
||||
const sandbox = { exports: {}, ...context };
|
||||
const source = fs.readFileSync(path.join(__dirname, '..', file), 'utf8');
|
||||
|
||||
@@ -139,6 +139,7 @@ function fixture() {
|
||||
'@nativescript/core': { File: { fromPath: () => ({ writeTextSync() {} }) },
|
||||
knownFolders: { documents: () => ({ path: '/tmp' }) }, path },
|
||||
'../native/ios-voice-input': { iosVoiceInput: { handleSessionEnded() {}, stopPhoneCapture() {} } },
|
||||
'../ui/shell/voice-activity': { voiceActivity: { setActive() {}, isActive: () => false, subscribe: () => () => {} } },
|
||||
'../native/faceclaw-communicator.ios': { FaceclawCommunicatorBridge, resolveIosPeripherals: async addresses => addresses },
|
||||
'../native/preview-display.ios': { PreviewDisplayTarget },
|
||||
'./ancs-client': { ANCS_FIRMWARE_VERSION: 16, AncsClient: class { state = 'disconnected'; start() {} stop() {} stopCommand() { return new Uint8Array(); } receive() { return false; } } },
|
||||
|
||||
@@ -11,14 +11,20 @@ function harness() {
|
||||
// Stands in for cloud-stt-provider: any provider but "onboard" is a keyed cloud one.
|
||||
const provider = {
|
||||
usesCloudStt: voice => voice.provider !== 'onboard',
|
||||
createCloudSttClient(voice, options) {
|
||||
createCloudSttClient(voice, options, continuous) {
|
||||
if (voice.provider === 'onboard') return null;
|
||||
const client = { options, pcm: [], started: false, finished: 0, stopped: false,
|
||||
const client = { options, continuous, pcm: [], started: false, finished: 0, stopped: false,
|
||||
start() { this.started = true; }, acceptPcm(bytes) { this.pcm.push([...bytes]); },
|
||||
finish() { this.finished++; }, stop() { this.stopped = true; } };
|
||||
clouds.push(client); return client;
|
||||
},
|
||||
};
|
||||
const recordings = [];
|
||||
class FakeRecorder {
|
||||
static open(name) { const r = new FakeRecorder(); r.name = name; r.path = `/Downloads/${name}.wav`; r.pcm = []; r.closed = false; recordings.push(r); return r; }
|
||||
append(bytes) { this.pcm.push(...bytes); }
|
||||
close() { this.closed = true; }
|
||||
}
|
||||
let authorization = 3, microphoneAuthorization = 3, microphoneRequests = 0, permissionRequests = 0, release;
|
||||
const sandbox = { exports: {}, FaceclawSpeech: { new: () => native, authorizationStatus: () => authorization,
|
||||
requestAuthorization: done => { permissionRequests++; done(3); },
|
||||
@@ -26,7 +32,7 @@ function harness() {
|
||||
requestMicrophoneAuthorization: done => { microphoneRequests++; done(3); } },
|
||||
setTimeout: fn => { timers.set(fn, fn); return fn; }, clearTimeout: fn => timers.delete(fn),
|
||||
NSData: { dataWithBytesLength: (b, n) => Buffer.from(b, 0, n) }, interop: { handleof: b => b, bufferFromData: d => d },
|
||||
require: name => name === './cloud-stt-provider' ? provider : {} };
|
||||
require: name => name === './cloud-stt-provider' ? provider : name === './wav-recorder.ios' ? { WavRecorder: FakeRecorder } : {} };
|
||||
vm.runInNewContext(ts.transpileModule(fs.readFileSync('app/native/voice-control.ios.ts', 'utf8'), {
|
||||
compilerOptions: { module: ts.ModuleKind.CommonJS, target: ts.ScriptTarget.ES2020 } }).outputText, sandbox);
|
||||
const bridge = new sandbox.exports.IosVoiceControlBridge();
|
||||
@@ -37,7 +43,7 @@ function harness() {
|
||||
return Promise.resolve();
|
||||
} };
|
||||
const onboard = { provider: 'onboard', elevenLabsApiKey: '', openAiApiKey: '', sonioxApiKey: '' };
|
||||
return { bridge, native, session, audio, commands, text, timers, clouds, release: () => release(),
|
||||
return { bridge, native, session, audio, commands, text, timers, clouds, recordings, release: () => release(),
|
||||
onboard, cloud: { ...onboard, provider: 'soniox', sonioxApiKey: 'key' },
|
||||
microphoneAuthorization: value => { microphoneAuthorization = value; }, microphoneRequests: () => microphoneRequests,
|
||||
authorization: value => { authorization = value; }, permissionRequests: () => permissionRequests,
|
||||
@@ -213,3 +219,59 @@ test('cloud capture times out a missing final, keeps native errors, and ends on
|
||||
assert.equal(h.clouds[2].started, false, 'no connection without audio');
|
||||
assert.match(h.bridge.statusText, /No phone microphone/);
|
||||
});
|
||||
|
||||
test('continuous capture needs a cloud provider, shares its mic with push-to-talk, and drains pause segments on stop', async () => {
|
||||
const h = harness(); let ends = 0; h.bridge.onSpeechEnd(() => ends++);
|
||||
assert.equal(h.bridge.supportsContinuous(h.onboard), false);
|
||||
assert.equal(h.bridge.supportsContinuous(h.cloud), true);
|
||||
const start = h.bridge.startGlassesCapture(h.session, h.cloud, () => {}, false, true); h.release(); await start;
|
||||
assert.deepEqual(h.native.pcmCapture, { endpointing: false, phone: false });
|
||||
const [cloud] = h.clouds;
|
||||
assert.equal(cloud.continuous(), true, 'pauses commit segments while continuous');
|
||||
// Push-to-talk joins instead of restarting the mic, and releasing it keeps the capture.
|
||||
await h.bridge.startGlassesCapture(h.session, h.cloud, () => {});
|
||||
await h.bridge.stopPushToTalk();
|
||||
assert.equal(h.clouds.length, 1); assert.deepEqual(h.commands, [true]); assert.equal(h.native.finished, undefined);
|
||||
cloud.options.onTranscript({ text: 'First thought.', isFinal: true, paragraphBreakAfter: true });
|
||||
const stopped = h.bridge.stopContinuousCapture();
|
||||
assert.deepEqual(h.commands, [true, false]); assert.equal(h.native.finished, true);
|
||||
h.event({ kind: 'ended', message: 'Transcription stopped.' });
|
||||
assert.equal(cloud.finished, 1);
|
||||
// An earlier pause segment answering late must not end the drain.
|
||||
cloud.options.onTranscript({ text: 'Second thought.', isFinal: true, paragraphBreakAfter: true });
|
||||
assert.equal(cloud.stopped, false);
|
||||
cloud.options.onTranscript({ text: 'The tail.', isFinal: true, paragraphBreakAfter: false });
|
||||
await stopped;
|
||||
assert.equal(cloud.stopped, true);
|
||||
assert.deepEqual(h.text.map(event => event.text), ['First thought.', 'Second thought.', 'The tail.']);
|
||||
assert.equal(cloud.continuous(), false);
|
||||
assert.ok(ends >= 1);
|
||||
});
|
||||
|
||||
test('push-to-talk after Transcribe closed starts its own capture instead of joining the finalizing one', async () => {
|
||||
const h = harness();
|
||||
const start = h.bridge.startGlassesCapture(h.session, h.cloud, () => {}, false, true); h.release(); await start;
|
||||
void h.bridge.stopContinuousCapture();
|
||||
h.event({ kind: 'ended', message: 'Transcription stopped.' });
|
||||
assert.equal(h.clouds[0].finished, 1, 'the closed window is still finalizing');
|
||||
const ptt = h.bridge.startGlassesCapture(h.session, h.cloud, () => {}); h.release(); await ptt;
|
||||
assert.equal(h.clouds.length, 2, 'push-to-talk got a capture of its own');
|
||||
assert.equal(h.clouds[0].stopped, true);
|
||||
assert.equal(h.clouds[1].continuous(), false);
|
||||
});
|
||||
|
||||
test('a continuous capture asked to record writes every PCM chunk to its WAV and closes it when the capture ends', async () => {
|
||||
const h = harness();
|
||||
const start = h.bridge.startGlassesCapture(h.session, h.cloud, () => {}, false, true, 'transcribe-2026-10-17-120000'); h.release(); await start;
|
||||
assert.equal(h.recordings.length, 1);
|
||||
assert.equal(h.recordings[0].name, 'transcribe-2026-10-17-120000');
|
||||
h.native.pcmHandler(new Uint8Array([1, 2, 3, 4]));
|
||||
h.native.pcmHandler(new Uint8Array([5, 6]));
|
||||
assert.deepEqual(h.recordings[0].pcm, [1, 2, 3, 4, 5, 6]);
|
||||
assert.deepEqual(h.clouds[0].pcm, [[1, 2, 3, 4], [5, 6]], 'transcription still gets the audio');
|
||||
h.bridge.stop();
|
||||
assert.equal(h.recordings[0].closed, true);
|
||||
// Captures that don't ask (push-to-talk, Teleprompter) are not recorded.
|
||||
const ptt = h.bridge.startGlassesCapture(h.session, h.cloud, () => {}); h.release(); await ptt;
|
||||
assert.equal(h.recordings.length, 1);
|
||||
});
|
||||
@@ -0,0 +1,19 @@
|
||||
const test = require('node:test');
|
||||
const assert = require('node:assert/strict');
|
||||
const { loader } = require('./helpers/load-typescript.cjs');
|
||||
|
||||
test('the WAV header describes 16 kHz mono PCM16 and its data size', () => {
|
||||
const { wavHeader } = loader({ DataView }, {})('app/native/wav-header.ts');
|
||||
const header = Buffer.from(wavHeader(32000));
|
||||
assert.equal(header.length, 44);
|
||||
assert.equal(header.toString('ascii', 0, 4), 'RIFF');
|
||||
assert.equal(header.readUInt32LE(4), 36 + 32000);
|
||||
assert.equal(header.toString('ascii', 8, 16), 'WAVEfmt ');
|
||||
assert.equal(header.readUInt16LE(20), 1);
|
||||
assert.equal(header.readUInt16LE(22), 1);
|
||||
assert.equal(header.readUInt32LE(24), 16000);
|
||||
assert.equal(header.readUInt32LE(28), 32000);
|
||||
assert.equal(header.readUInt16LE(34), 16);
|
||||
assert.equal(header.toString('ascii', 36, 40), 'data');
|
||||
assert.equal(header.readUInt32LE(40), 32000);
|
||||
});
|
||||
@@ -0,0 +1,192 @@
|
||||
const test = require('node:test');
|
||||
const assert = require('node:assert/strict');
|
||||
const { loader } = require('./helpers/load-typescript.cjs');
|
||||
|
||||
function load() {
|
||||
const timers = [];
|
||||
const modules = loader({
|
||||
setTimeout(fn, delay) { timers.push({ fn, delay }); return timers.length; },
|
||||
clearTimeout() {},
|
||||
DataView, Math, Date, JSON, Object, String, Error, Promise,
|
||||
}, {});
|
||||
return { ...modules('app/native/whisper-server-stt.ts'), timers };
|
||||
}
|
||||
|
||||
/** An http stub whose requests resolve only when the test says so. */
|
||||
function server(models = '{"data":[{"id":"openai/whisper-large-v3"}]}') {
|
||||
const requests = [];
|
||||
const http = (request) => new Promise((resolve, reject) => {
|
||||
if (request.method === 'GET') { requests.push({ ...request, kind: 'models' }); resolve({ status: 200, text: models }); return; }
|
||||
requests.push({ ...request, kind: 'transcribe', resolve, reject, fields: fields(request.body) });
|
||||
});
|
||||
return { http, requests, posts: () => requests.filter(request => request.kind === 'transcribe') };
|
||||
}
|
||||
|
||||
function fields(body) {
|
||||
const text = Buffer.from(body).toString('latin1');
|
||||
const out = {};
|
||||
for (const match of text.matchAll(/name="([^"]+)"\r\n\r\n([^\r]*)\r\n/g)) out[match[1]] = match[2];
|
||||
return out;
|
||||
}
|
||||
|
||||
function client(config = {}, s = server()) {
|
||||
const { WhisperServerSttClient, timers } = load();
|
||||
const events = [], statuses = [], errors = [];
|
||||
let ready = 0;
|
||||
const c = new WhisperServerSttClient({
|
||||
apiKey: config.apiKey ?? '', onTranscript: event => events.push({ ...event }), onStatus: status => statuses.push(status),
|
||||
onError: error => errors.push(error), onReady: () => ready++, onDisconnected: message => errors.push(`disconnected: ${message}`),
|
||||
}, { baseUrl: 'http://whisper-host:8004/v1', model: '', language: '', ...config }, s.http);
|
||||
return { c, s, events, statuses, errors, timers, ready: () => ready };
|
||||
}
|
||||
|
||||
const flush = () => new Promise(resolve => setImmediate(resolve));
|
||||
|
||||
function speech(ms, amplitude = 3000) {
|
||||
const bytes = new Uint8Array(ms * 32);
|
||||
for (let i = 0; i < bytes.length; i += 2) {
|
||||
const sample = i % 4 ? amplitude : -amplitude;
|
||||
bytes[i] = sample & 255; bytes[i + 1] = (sample >> 8) & 255;
|
||||
}
|
||||
return bytes;
|
||||
}
|
||||
|
||||
test('URL normalization accepts a bare host, a /v1 root and the full endpoint', () => {
|
||||
const { normalizeWhisperServerUrl } = load();
|
||||
assert.equal(normalizeWhisperServerUrl('http://192.0.2.10:8004'), 'http://192.0.2.10:8004/v1');
|
||||
assert.equal(normalizeWhisperServerUrl(' http://192.0.2.10:8004/v1/ '), 'http://192.0.2.10:8004/v1');
|
||||
assert.equal(normalizeWhisperServerUrl('https://stt.example/v1/audio/transcriptions'), 'https://stt.example/v1');
|
||||
assert.equal(normalizeWhisperServerUrl('192.0.2.10:8004'), '');
|
||||
assert.equal(normalizeWhisperServerUrl(''), '');
|
||||
});
|
||||
|
||||
test('the WAV header describes 16 kHz mono PCM16 and the multipart body carries the fields', () => {
|
||||
const { wavFile, multipartBody } = load();
|
||||
const wav = wavFile([new Uint8Array([1, 2, 3, 4]), new Uint8Array([5, 6])], 6);
|
||||
const view = Buffer.from(wav);
|
||||
assert.equal(view.toString('ascii', 0, 4), 'RIFF');
|
||||
assert.equal(view.readUInt32LE(24), 16000);
|
||||
assert.equal(view.readUInt16LE(22), 1);
|
||||
assert.equal(view.readUInt32LE(40), 6);
|
||||
assert.deepEqual([...wav.subarray(44)], [1, 2, 3, 4, 5, 6]);
|
||||
const body = Buffer.from(multipartBody('B', { model: 'm', language: 'sk' }, wav)).toString('latin1');
|
||||
assert.match(body, /^--B\r\nContent-Disposition: form-data; name="model"\r\n\r\nm\r\n/);
|
||||
assert.match(body, /name="file"; filename="audio.wav"\r\nContent-Type: audio\/wav\r\n\r\nRIFF/);
|
||||
assert.ok(body.endsWith('\r\n--B--\r\n'));
|
||||
});
|
||||
|
||||
test('start lists models, picks the first id, and push-to-talk delivers one final', async () => {
|
||||
const h = client();
|
||||
h.c.start(); await flush();
|
||||
assert.equal(h.ready(), 1);
|
||||
assert.equal(h.s.requests[0].url, 'http://whisper-host:8004/v1/models');
|
||||
h.c.acceptPcm(speech(500));
|
||||
h.c.finish();
|
||||
const [post] = h.s.posts();
|
||||
assert.equal(post.url, 'http://whisper-host:8004/v1/audio/transcriptions');
|
||||
assert.equal(post.fields.model, 'openai/whisper-large-v3');
|
||||
assert.equal(post.fields.response_format, 'json');
|
||||
assert.equal('language' in post.fields, false, 'empty language lets Whisper auto-detect');
|
||||
assert.equal(post.headers.Authorization, undefined);
|
||||
post.resolve({ status: 200, text: '{"text":" Remind me to call the bank."}' }); await flush();
|
||||
assert.deepEqual(h.events, [{ text: 'Remind me to call the bank.', isFinal: true, paragraphBreakAfter: false }]);
|
||||
});
|
||||
|
||||
test('a configured model, language and key are sent as given', async () => {
|
||||
const h = client({ model: 'whisper-1', language: 'SK', apiKey: 'secret' });
|
||||
h.c.start(); await flush();
|
||||
h.c.acceptPcm(speech(300)); h.c.finish();
|
||||
const [post] = h.s.posts();
|
||||
assert.equal(post.fields.model, 'whisper-1');
|
||||
assert.equal(post.fields.language, 'sk');
|
||||
assert.equal(post.headers.Authorization, 'Bearer secret');
|
||||
});
|
||||
|
||||
test('silence is not sent (Whisper would invent text) but push-to-talk still gets an empty final', async () => {
|
||||
const h = client();
|
||||
h.c.start(); await flush();
|
||||
h.c.acceptPcm(speech(2000, 40));
|
||||
h.c.finish();
|
||||
assert.equal(h.s.posts().length, 0);
|
||||
assert.deepEqual(h.events, [{ text: '', isFinal: true, paragraphBreakAfter: false }]);
|
||||
});
|
||||
|
||||
test('interim requests show provisional text and never overwrite the final', async () => {
|
||||
const h = client();
|
||||
h.c.start(); await flush();
|
||||
h.c.acceptPcm(speech(1600));
|
||||
const [interim] = h.s.posts();
|
||||
assert.ok(interim, 'an interim request goes out once enough audio arrived');
|
||||
h.c.acceptPcm(speech(1600));
|
||||
assert.equal(h.s.posts().length, 1, 'one interim in flight at a time');
|
||||
interim.resolve({ status: 200, text: '{"text":"hello"}' }); await flush();
|
||||
assert.deepEqual(h.events.at(-1), { text: 'hello', transcribeText: 'hello', isFinal: false });
|
||||
h.c.finish();
|
||||
const final = h.s.posts().at(-1);
|
||||
final.resolve({ status: 200, text: '{"text":"hello world"}' }); await flush();
|
||||
assert.deepEqual(h.events.at(-1), { text: 'hello world', isFinal: true, paragraphBreakAfter: false });
|
||||
});
|
||||
|
||||
test('continuous segments finalize in audio order even when answered out of order', async () => {
|
||||
const h = client();
|
||||
h.c.start(); await flush();
|
||||
h.c.acceptPcm(speech(500)); h.c.commitSegment();
|
||||
h.c.acceptPcm(speech(500)); h.c.commitSegment();
|
||||
const [first, second] = h.s.posts();
|
||||
second.resolve({ status: 200, text: '{"text":"second"}' }); await flush();
|
||||
assert.deepEqual(h.events.map(event => event.isFinal), [false]);
|
||||
assert.equal(h.events[0].text, 'second');
|
||||
first.resolve({ status: 200, text: '{"text":"first"}' }); await flush();
|
||||
assert.deepEqual(h.events.filter(event => event.isFinal).map(event => [event.text, event.paragraphBreakAfter]),
|
||||
[['first', true], ['second', true]]);
|
||||
});
|
||||
|
||||
test('a failed final is retried once, then reported without ending the capture', async () => {
|
||||
const h = client();
|
||||
h.c.start(); await flush();
|
||||
h.c.acceptPcm(speech(500)); h.c.commitSegment();
|
||||
h.s.posts()[0].reject(new Error('timed out')); await flush();
|
||||
assert.equal(h.timers.at(-1).delay, 750);
|
||||
h.timers.at(-1).fn();
|
||||
h.s.posts()[1].resolve({ status: 502, text: 'bad gateway' }); await flush();
|
||||
assert.match(h.statuses.at(-1), /Whisper server request failed: HTTP 502/);
|
||||
assert.deepEqual(h.errors, []);
|
||||
h.c.acceptPcm(speech(500)); h.c.finish();
|
||||
h.s.posts()[2].resolve({ status: 200, text: '{"text":"still here"}' }); await flush();
|
||||
assert.equal(h.events.at(-1).text, 'still here');
|
||||
});
|
||||
|
||||
test('an auth rejection is an error, an unreachable server is a disconnect', async () => {
|
||||
const auth = client();
|
||||
auth.c.start(); await flush();
|
||||
auth.c.acceptPcm(speech(500)); auth.c.finish();
|
||||
auth.s.posts()[0].resolve({ status: 401, text: '' }); await flush();
|
||||
assert.deepEqual(auth.errors, ['Whisper server rejected the key (401).']);
|
||||
const down = client({}, { http: () => Promise.reject(new Error('connection refused')) });
|
||||
down.c.start(); await flush();
|
||||
assert.equal(down.ready(), 0);
|
||||
assert.deepEqual(down.errors, ['disconnected: Whisper server unreachable: connection refused']);
|
||||
});
|
||||
|
||||
test('provider selection needs a URL, not a key, and passes the server settings through', () => {
|
||||
const sockets = [];
|
||||
const modules = loader({
|
||||
setTimeout() {}, clearTimeout() {}, console: { log() {}, warn() {} }, DataView,
|
||||
com: { faceclaw: { app: {
|
||||
FaceclawWebSocketListener: function(callbacks) { return callbacks; },
|
||||
FaceclawWebSocket: class { constructor(url) { sockets.push(url); } },
|
||||
} } },
|
||||
}, { './cloud-stt': { CLOUD_STT_SAMPLE_RATE: 16000, encodeBase64: () => '', toJavaBytes: bytes => bytes } });
|
||||
const { createCloudSttClient, usesCloudStt } = modules('app/native/cloud-stt-provider.ts');
|
||||
const statuses = [];
|
||||
const options = { onTranscript() {}, onStatus: status => statuses.push(status), onError() {} };
|
||||
const selection = { provider: 'whisper-server', elevenLabsApiKey: '', openAiApiKey: '', sonioxApiKey: '' };
|
||||
assert.equal(usesCloudStt(selection), false);
|
||||
assert.equal(createCloudSttClient(selection, options, () => false), null);
|
||||
assert.deepEqual(statuses, ['No Whisper server URL set; using on-device voice.']);
|
||||
const configured = { ...selection, whisperServerUrl: 'http://192.0.2.10:8004' };
|
||||
assert.equal(usesCloudStt(configured), true);
|
||||
const client = createCloudSttClient(configured, options, () => false);
|
||||
assert.ok(client.finalTimeoutMs > 5000, 'batch requests get longer than the 5s realtime default');
|
||||
assert.equal(sockets.length, 0);
|
||||
});
|
||||
Reference in new issue
Block a user