diff --git a/app/analytics/analytics-sources.ts b/app/analytics/analytics-sources.ts index 65b026e7..f286d805 100644 --- a/app/analytics/analytics-sources.ts +++ b/app/analytics/analytics-sources.ts @@ -25,6 +25,7 @@ const API_KEY_SETTINGS: Record = { openai: "voice.openAiApiKey", elevenlabs: "voice.elevenLabsApiKey", soniox: "voice.sonioxApiKey", + "whisper-server": "voice.whisperServerApiKey", mapbox: "maps.mapboxApiKey", roam: "integrations.roam.apiToken", nightscout: "integrations.nightscout.apiToken", diff --git a/app/apps/calculator/calculator-app.ts b/app/apps/calculator/calculator-app.ts index e71b827a..483ad438 100644 --- a/app/apps/calculator/calculator-app.ts +++ b/app/apps/calculator/calculator-app.ts @@ -116,6 +116,9 @@ export function createCalculatorAppWindow(options: InProcessAppOptions): InProce }, removeSurface: options.removeSurface, onClosed: () => { + // Closing keeps the base layer (clearToBase), so stop its continuous + // listening explicitly; otherwise the mic capture outlives the window. + layer.onRemoved(); toolRegistry.removeAppTools(CALCULATOR_WINDOW_ID); options.onClosed(); }, diff --git a/app/apps/ios-availability.ts b/app/apps/ios-availability.ts index c2f062c3..8fd09e06 100644 --- a/app/apps/ios-availability.ts +++ b/app/apps/ios-availability.ts @@ -1,7 +1,6 @@ /** Native services still outside the current iOS app-screen milestone. */ const unavailable: Record = { music: 'Media-player integration is not available on iOS yet.', - transcribe: 'Voice capture and transcription are not available on iOS yet.', microphones: 'Glasses microphone capture is not available on iOS yet.', } export function iosAppUnavailableReason(appId: string): string | null { diff --git a/app/apps/transcribe/index.ts b/app/apps/transcribe/index.ts index ebd3c0f3..346d6f8d 100644 --- a/app/apps/transcribe/index.ts +++ b/app/apps/transcribe/index.ts @@ -9,7 +9,7 @@ const transcribeApp: AppDefinition = { ctx.launchInProcessApp(TRANSCRIBE_WINDOW_ID, TRANSCRIBE_SURFACE_ID, (options) => createTranscribeAppWindow({ ...options, - startContinuousVoiceCapture: () => void options.actions.startContinuousVoiceCapture(), + startContinuousVoiceCapture: (capture) => void options.actions.startContinuousVoiceCapture(capture), stopContinuousVoiceCapture: () => void options.actions.stopContinuousVoiceCapture(), }), ), diff --git a/app/apps/transcribe/transcribe-app.ts b/app/apps/transcribe/transcribe-app.ts index 21ac40d4..f3fe29c2 100644 --- a/app/apps/transcribe/transcribe-app.ts +++ b/app/apps/transcribe/transcribe-app.ts @@ -1,5 +1,5 @@ import { imageFromAsciiArt } from "../../graphics/image"; -import { TranscribeLayer } from "./transcribe"; +import { TranscribeLayer, transcriptTimestamp } from "./transcribe"; import { createInProcessWindow, YieldAtRootLayer, @@ -37,7 +37,7 @@ const MIC_ICON = imageFromAsciiArt( export type TranscribeAppOptions = InProcessAppOptions & { /** Begin continuous mic capture (kept on while the window is open). */ - startContinuousVoiceCapture: () => void; + startContinuousVoiceCapture: (options?: { recordAs?: string }) => void; stopContinuousVoiceCapture: () => void; }; @@ -47,7 +47,9 @@ export type TranscribeAppOptions = InProcessAppOptions & { * shows a microphone tray icon in the top bar. */ export function createTranscribeAppWindow(options: TranscribeAppOptions): InProcessWindow { - const layer = new TranscribeLayer(); + // One timestamp per session names both the audio recording and a saved transcript. + const stamp = transcriptTimestamp(); + const layer = new TranscribeLayer(stamp); const app = createInProcessWindow({ appId: "transcribe", windowId: TRANSCRIBE_WINDOW_ID, @@ -77,7 +79,7 @@ export function createTranscribeAppWindow(options: TranscribeAppOptions): InProc // Wire async state changes to this window's renderer, rather than the // controller action from which the window-specific actions were derived. layer.start(app.requestRender); - options.startContinuousVoiceCapture(); + options.startContinuousVoiceCapture({ recordAs: `transcribe-${stamp}` }); shell.setTrayIcon(TRAY_ICON_ID, MIC_ICON); return app; } diff --git a/app/apps/transcribe/transcribe.ts b/app/apps/transcribe/transcribe.ts index 3da56357..251b1807 100644 --- a/app/apps/transcribe/transcribe.ts +++ b/app/apps/transcribe/transcribe.ts @@ -23,6 +23,9 @@ export class TranscribeLayer implements Layer { private unsubscribePause: (() => void) | null = null; private unsubscribeStatus: (() => void) | null = null; + /** `stamp` names the saved transcript, matching the session's audio recording. */ + constructor(private readonly stamp: string = transcriptTimestamp()) {} + start(requestRender: () => void): void { this.unsubscribeTranscript = voiceControlBridge.onTranscript((event) => { this.transcript.accept(event); @@ -93,13 +96,13 @@ export class TranscribeLayer implements Layer { this.saveNotice = "Nothing to save yet."; return; } - const filename = `transcript-${transcriptTimestamp()}.txt`; + const filename = `transcript-${this.stamp}.txt`; const path = writeTextToDownloads(filename, `${text}\n`); this.saveNotice = path ? `Saved ${filename}` : "Save failed (check file access)."; } } -function transcriptTimestamp(): string { +export function transcriptTimestamp(): string { const now = new Date(); const pad = (value: number) => String(value).padStart(2, "0"); return ( diff --git a/app/g2/dashboard-controller.ts b/app/g2/dashboard-controller.ts index 1decb897..5f3b5035 100644 --- a/app/g2/dashboard-controller.ts +++ b/app/g2/dashboard-controller.ts @@ -75,7 +75,7 @@ import { loadPersistedOpenApps, savePersistedOpenApps } from "../ui/shell/open-a import { appViewportRect, isOnSwitcherEdge, sidebarStripVisible, type ScreenRect, type WindowHeightMode } from "../ui/shell/geometry"; import { type LayerActions, type TextSettingsEditToggle } from "../ui/layers"; import { type KeyboardInputSession } from "../ui/shell/keyboard-input"; -import { appSwitcherPositionSetting, statusBarPositionSetting, statusBarVisibilitySetting, windowBorderSetting, assistantAllowProactiveSetting, assistantBackendSetting, assistantBridgeHostSetting, assistantBridgePortSetting, assistantBridgeTokenSetting, getBrightnessPreferences, displayLayoutSetting, navigateDisplayLayoutSetting, terminalDisplayLayoutSetting, elevenLabsApiKeySetting, getStringSettingById, openAiApiKeySetting, nightscoutApiTokenSetting, firmwareDebugFlagsSetting, lockScreenEnabledSetting, nightscoutSiteUrlSetting, onAnySettingChanged, previewColorSetting, ringConnectionModeSetting, saveVoiceRecordingsSetting, sonioxApiKeySetting, screenTimeoutSetting, screenTimeoutSettingToMs, suspendEvenHubWhenScreenOffSetting, voiceProviderSetting, wakeWordActionSetting, type ConfigSettingString } from "../ui/dashboard-settings"; +import { appSwitcherPositionSetting, statusBarPositionSetting, statusBarVisibilitySetting, windowBorderSetting, assistantAllowProactiveSetting, assistantBackendSetting, assistantBridgeHostSetting, assistantBridgePortSetting, assistantBridgeTokenSetting, getBrightnessPreferences, displayLayoutSetting, navigateDisplayLayoutSetting, terminalDisplayLayoutSetting, elevenLabsApiKeySetting, getStringSettingById, openAiApiKeySetting, nightscoutApiTokenSetting, firmwareDebugFlagsSetting, whisperServerApiKeySetting, whisperServerLanguageSetting, whisperServerModelSetting, whisperServerUrlSetting, lockScreenEnabledSetting, nightscoutSiteUrlSetting, onAnySettingChanged, previewColorSetting, ringConnectionModeSetting, saveVoiceRecordingsSetting, sonioxApiKeySetting, screenTimeoutSetting, screenTimeoutSettingToMs, suspendEvenHubWhenScreenOffSetting, voiceProviderSetting, wakeWordActionSetting, type ConfigSettingString } from "../ui/dashboard-settings"; import { isIgnoringBatteryOptimizations, requestIgnoreBatteryOptimizations } from "../native/battery-optimization"; import { getInstalledEvenHubAppById, @@ -2087,6 +2087,10 @@ class DashboardController { elevenLabsApiKey: elevenLabsApiKeySetting.get(), openAiApiKey: openAiApiKeySetting.get(), sonioxApiKey: sonioxApiKeySetting.get(), + whisperServerUrl: whisperServerUrlSetting.get(), + whisperServerModel: whisperServerModelSetting.get(), + whisperServerLanguage: whisperServerLanguageSetting.get(), + whisperServerApiKey: whisperServerApiKeySetting.get(), saveRecording: saveVoiceRecordingsSetting.get(), // "My voice only" (Microphones app): verify command utterances // against the enrolled wearer voice-print; non-matching speakers' diff --git a/app/g2/ios-preview-controller.ts b/app/g2/ios-preview-controller.ts index 639f00dd..6a2c1d7c 100644 --- a/app/g2/ios-preview-controller.ts +++ b/app/g2/ios-preview-controller.ts @@ -16,6 +16,7 @@ import { IosNavigationSensors } from '../native/ios-navigation-sensors' import { bindCompassSession, receiveCompassEvent } from '../native/compass.ios' import { Dialogs, File, knownFolders, path, Utils, type ImageSource } from '@nativescript/core' import { iosVoiceInput } from '../native/ios-voice-input' +import { voiceActivity } from '../ui/shell/voice-activity' import { nightscoutBridge } from '../native/nightscout-bridge' import { weatherBridge } from '../native/weather' import { FaceclawCommunicatorBridge, resolveIosPeripherals, type CommunicatorState, type RawInputEvent } from '../native/faceclaw-communicator.ios' @@ -48,12 +49,15 @@ import { mirrorToPanelFraction, phonePreviewCrop } from './phone-preview-crop' import { onAnySettingChanged, previewColorSetting, lockScreenEnabledSetting, getBrightnessPreferences, getStringSettingById, nightscoutSiteUrlSetting, nightscoutApiTokenSetting, voiceProviderSetting, elevenLabsApiKeySetting, - openAiApiKeySetting, sonioxApiKeySetting, ringConnectionModeSetting } from '../ui/dashboard-settings' + openAiApiKeySetting, sonioxApiKeySetting, whisperServerApiKeySetting, whisperServerLanguageSetting, + whisperServerModelSetting, whisperServerUrlSetting, ringConnectionModeSetting, saveTranscribeAudioSetting } from '../ui/dashboard-settings' import type { VoiceProviderSelection } from '../native/cloud-stt-provider' import type { PhoneGesture } from '../phone-ui/phone-gestures' import { claimGlassesOnboardingAutoLaunch, ONBOARDING_APP_ID } from '../apps/onboarding/onboarding-progress' const SHELL_SURFACE_ID = 'shell' +// voiceActivity holder for continuous capture (Transcribe), as on Android. +const CONTINUOUS_CAPTURE_HOLDER = 'continuous-capture' const SHELL_SURFACE_Z_ORDER = 1 const LOCK_SURFACE_Z_ORDER = 1000 /** The shell render loop waits this long for the glasses to take a frame before moving on. */ @@ -144,6 +148,7 @@ export class IosPreviewController { private acknowledgedFrames = 0 private logLines: string[] = [] private logTimer: ReturnType | null = null + private continuousCaptureToken = 0 private readonly actions: LayerActions = { ...noopLayerActions, playBuzzerSequence: payload => this.communicator?.playBuzzerSequence(payload), @@ -151,7 +156,8 @@ export class IosPreviewController { disconnect: () => this.disconnect(), startVoiceCapture: endpointing => this.startVoiceCapture(endpointing), stopVoiceCapture: () => iosVoiceInput.stopPushToTalk(), - startContinuousVoiceCapture: () => this.onError("Voice capture is not available on iOS yet."), + startContinuousVoiceCapture: options => this.startContinuousVoiceCapture(options?.recordAs), + stopContinuousVoiceCapture: () => this.stopContinuousVoiceCapture(), startTextSettingEdit: async setting => { await this.editSetting(setting) }, startTextSettingsEdit: async (settings, title, finished, toggle, cancelled) => { for (const setting of settings) if (!await this.editSetting(setting)) { cancelled?.(); return } @@ -941,10 +947,40 @@ export class IosPreviewController { await iosVoiceInput.startPhoneCapture(voice, log, endpointing) } } + /** Open-ended capture (Transcribe, Teleprompter); holds voiceActivity like Android. */ + private async startContinuousVoiceCapture(recordAs?: string): Promise { + if (this.glassesLocked) return + const voice = this.voiceProvider() + if (!iosVoiceInput.supportsContinuous(voice)) { + this.onError('Transcribe on iOS needs a cloud or Whisper server transcription provider (Settings > Voice > Transcription Provider).') + return + } + // A stop during an await below bumps the token; this start then backs off. + const token = ++this.continuousCaptureToken + const current = () => token === this.continuousCaptureToken + const log = (message: string) => this.logBluetooth(message) + const record = recordAs && saveTranscribeAudioSetting.get() ? recordAs : '' + voiceActivity.setActive(true, CONTINUOUS_CAPTURE_HOLDER) + const communicator = this.communicator + if (communicator && this.state.phase === 'connected') { + await iosVoiceInput.startGlassesCapture(communicator, voice, log, false, true, record) + } else if (this.active && await iosVoiceInput.prepare(this.active, true, voice) && current() && this.active && !this.glassesLocked) { + await iosVoiceInput.startPhoneCapture(voice, log, false, true, record) + } + if (!current()) return + if (!iosVoiceInput.continuousActive) voiceActivity.setActive(false, CONTINUOUS_CAPTURE_HOLDER) + } + private stopContinuousVoiceCapture(): void { + ++this.continuousCaptureToken + voiceActivity.setActive(false, CONTINUOUS_CAPTURE_HOLDER) + void iosVoiceInput.stopContinuousCapture().catch(error => this.logBluetooth(`Stop continuous capture: ${String(error)}`)) + } /** The transcription provider setting and the keys the cloud ones need. */ private voiceProvider(): VoiceProviderSelection { return { provider: voiceProviderSetting.get(), elevenLabsApiKey: elevenLabsApiKeySetting.get(), - openAiApiKey: openAiApiKeySetting.get(), sonioxApiKey: sonioxApiKeySetting.get() } + openAiApiKey: openAiApiKeySetting.get(), sonioxApiKey: sonioxApiKeySetting.get(), + whisperServerUrl: whisperServerUrlSetting.get(), whisperServerModel: whisperServerModelSetting.get(), + whisperServerLanguage: whisperServerLanguageSetting.get(), whisperServerApiKey: whisperServerApiKeySetting.get() } } private logBluetooth(message: string): void { const line = `${new Date().toISOString()} [${this.active ? 'foreground' : 'background'}${UIApplication.sharedApplication.protectedDataAvailable ? '' : ',protected-data-unavailable'}] ${message}` diff --git a/app/native/cloud-stt-provider.ts b/app/native/cloud-stt-provider.ts index c844714b..73e5bdce 100644 --- a/app/native/cloud-stt-provider.ts +++ b/app/native/cloud-stt-provider.ts @@ -4,6 +4,7 @@ import { OpenAiRealtimeSttClient } from "./openai-stt"; import { ReconnectingSttClient } from "./reconnecting-stt"; import { SonioxSttClient } from "./soniox-stt"; import type { VoiceProviderKind } from "./voice-control"; +import { normalizeWhisperServerUrl, WHISPER_SERVER_FINAL_TIMEOUT_MS, WhisperServerSttClient } from "./whisper-server-stt"; /** * The transcription-provider setting plus the keys the cloud providers need. @@ -14,12 +15,22 @@ export type VoiceProviderSelection = { elevenLabsApiKey: string; openAiApiKey: string; sonioxApiKey: string; + /** Self-hosted Whisper ("whisper-server"); the key is optional there. */ + whisperServerUrl?: string; + whisperServerModel?: string; + whisperServerLanguage?: string; + whisperServerApiKey?: string; }; type CloudProvider = { name: string; apiKey: (selection: VoiceProviderSelection) => string; - create: (options: CloudSttOptions) => CloudSttClient; + /** Whether the provider can run; defaults to having a key. */ + configured?: (selection: VoiceProviderSelection) => boolean; + /** What to tell the user when it can't. */ + missing?: string; + create: (options: CloudSttOptions, selection: VoiceProviderSelection) => CloudSttClient; + finalTimeoutMs?: number; }; // Providers absent here ("onboard", "onboard-whisper") transcribe on-device. @@ -39,11 +50,28 @@ const CLOUD_PROVIDERS: Partial> = { apiKey: (selection) => selection.openAiApiKey, create: (options) => new OpenAiRealtimeSttClient(options), }, + "whisper-server": { + name: "Whisper server", + apiKey: (selection) => selection.whisperServerApiKey ?? "", + configured: (selection) => Boolean(normalizeWhisperServerUrl(selection.whisperServerUrl ?? "")), + missing: "No Whisper server URL set; using on-device voice.", + create: (options, selection) => new WhisperServerSttClient(options, { + baseUrl: selection.whisperServerUrl ?? "", + model: selection.whisperServerModel ?? "", + language: selection.whisperServerLanguage ?? "", + }), + finalTimeoutMs: WHISPER_SERVER_FINAL_TIMEOUT_MS, + }, }; +function isConfigured(provider: CloudProvider, selection: VoiceProviderSelection): boolean { + return provider.configured ? provider.configured(selection) : Boolean(provider.apiKey(selection).trim()); +} + /** Whether createCloudSttClient would return a cloud client for this selection. */ export function usesCloudStt(selection: VoiceProviderSelection): boolean { - return Boolean(CLOUD_PROVIDERS[selection.provider]?.apiKey(selection).trim()); + const provider = CLOUD_PROVIDERS[selection.provider]; + return Boolean(provider && isConfigured(provider, selection)); } /** @@ -56,13 +84,18 @@ export function createCloudSttClient( selection: VoiceProviderSelection, options: Omit, continuous: () => boolean, -): CloudSttClient | null { +): ReconnectingSttClient | null { const provider = CLOUD_PROVIDERS[selection.provider]; if (!provider) return null; - const apiKey = provider.apiKey(selection).trim(); - if (!apiKey) { - options.onStatus(`No ${provider.name} key set; using on-device voice.`); + if (!isConfigured(provider, selection)) { + options.onStatus(provider.missing ?? `No ${provider.name} key set; using on-device voice.`); return null; } - return new ReconnectingSttClient(provider.create, { ...options, apiKey }, continuous); + const apiKey = provider.apiKey(selection).trim(); + return new ReconnectingSttClient( + (clientOptions) => provider.create(clientOptions, selection), + { ...options, apiKey }, + continuous, + provider.finalTimeoutMs, + ); } diff --git a/app/native/reconnecting-stt.ts b/app/native/reconnecting-stt.ts index 1221bb8d..2b860e24 100644 --- a/app/native/reconnecting-stt.ts +++ b/app/native/reconnecting-stt.ts @@ -24,6 +24,8 @@ export class ReconnectingSttClient implements CloudSttClient { private readonly create: (options: CloudSttOptions) => CloudSttClient, private readonly options: CloudSttOptions, private readonly continuous: () => boolean, + /** How long finish() waits for the provider's final transcript. */ + readonly finalTimeoutMs = 5000, ) {} start(): void { @@ -90,7 +92,7 @@ export class ReconnectingSttClient implements CloudSttClient { this.stop(); return; } - this.finishTimeout = setTimeout(() => this.stop(), 5000); + this.finishTimeout = setTimeout(() => this.stop(), this.finalTimeoutMs); if (this.ready) this.client?.finish(); } diff --git a/app/native/voice-control.ios.ts b/app/native/voice-control.ios.ts index 14f85cdd..832543f1 100644 --- a/app/native/voice-control.ios.ts +++ b/app/native/voice-control.ios.ts @@ -1,6 +1,8 @@ import type { VoiceControlState, VoiceTranscriptEvent } from './voice-control' -import type { CloudSttClient, CloudSttTranscriptEvent } from './cloud-stt' +import type { CloudSttTranscriptEvent } from './cloud-stt' +import type { ReconnectingSttClient } from './reconnecting-stt' import { createCloudSttClient, usesCloudStt, type VoiceProviderSelection } from './cloud-stt-provider' +import { WavRecorder } from './wav-recorder.ios' declare const FaceclawSpeech: any export type IosMicrophoneSession = { @@ -12,6 +14,8 @@ const CLOUD_FINAL_TIMEOUT_MS = 5000 /** iOS implementation of the shared voice dialog's event bridge. Audio comes * from the glasses when connected, or the default phone input in preview mode, * and is transcribed by Apple's on-device recognizer or a cloud provider. + * Continuous capture (Transcribe, Teleprompter) needs a cloud provider: only + * the PCM capture runs open-ended, and pauses commit its segments. */ export class IosVoiceControlBridge { private native: any = null @@ -32,9 +36,14 @@ export class IosVoiceControlBridge { // Non-null while a cloud provider owns the transcript; FaceclawSpeech then // only supplies PCM. cloudEndStatus is set once the audio has ended and the // provider was asked to finalize; it is the status shown when it does. - private cloud: CloudSttClient | null = null + private cloud: ReconnectingSttClient | null = null private cloudEndStatus: string | null = null private cloudTimer: ReturnType | null = null + // Open-ended capture: runs until stopContinuousCapture, and push-to-talk + // joins it instead of restarting the mic (transcripts reach every listener). + private continuous = false + // The capture's audio as a WAV in Downloads, while recordTo asked for one. + private recorder: WavRecorder | null = null onStatus(listener: (state: VoiceControlState) => void): () => void { this.statuses.add(listener); listener({ status: this.status, listening: this.capturing, detail: '' }); return () => { this.statuses.delete(listener) } @@ -49,7 +58,11 @@ export class IosVoiceControlBridge { if (!this.native) { this.native = FaceclawSpeech.new() this.native.eventHandler = (json: string) => this.receive(JSON.parse(json)) - this.native.pcmHandler = (pcm: NSData) => this.cloud?.acceptPcm(new Uint8Array(interop.bufferFromData(pcm))) + this.native.pcmHandler = (pcm: NSData) => { + const bytes = new Uint8Array(interop.bufferFromData(pcm)) + this.cloud?.acceptPcm(bytes) + this.recorder?.append(bytes) + } } return this.native } @@ -74,9 +87,12 @@ export class IosVoiceControlBridge { return true } get statusText(): string { return this.status } - async startGlassesCapture(session: IosMicrophoneSession, voice: VoiceProviderSelection, log: (message: string) => void, endpointing = false): Promise { + async startGlassesCapture(session: IosMicrophoneSession, voice: VoiceProviderSelection, log: (message: string) => void, endpointing = false, continuous = false, recordTo = ''): Promise { + if (this.joinsContinuous(continuous)) return this.stop() const generation = ++this.generation + this.continuous = continuous + this.startRecording(recordTo, log) this.log = log; this.source = 'glasses'; this.session = session; this.packets = 0; this.capturing = true this.completion = new Promise(resolve => { this.resolveCompletion = resolve }) const startError = this.startNative(voice, endpointing, false) @@ -101,8 +117,11 @@ export class IosVoiceControlBridge { this.stop(); this.setStatus(error instanceof Error ? error.message : String(error)); this.emitEnd() } } - async startPhoneCapture(voice: VoiceProviderSelection, log: (message: string) => void, endpointing = false): Promise { + async startPhoneCapture(voice: VoiceProviderSelection, log: (message: string) => void, endpointing = false, continuous = false, recordTo = ''): Promise { + if (this.joinsContinuous(continuous)) return this.stop() + this.continuous = continuous + this.startRecording(recordTo, log) this.log = log; this.source = 'phone'; this.capturing = true this.completion = new Promise(resolve => { this.resolveCompletion = resolve }) const error = this.startNative(voice, endpointing, true) @@ -118,7 +137,7 @@ export class IosVoiceControlBridge { onTranscript: event => { if (current()) this.receiveCloudTranscript(event) }, onStatus: status => { if (current()) this.setStatus(status) }, onError: message => { if (current()) { this.stop(); this.setStatus(message); this.emitEnd() } }, - }, () => false) + }, () => this.continuous) if (!cloud) return String((phone ? native.startPhoneWithEndpointing(endpointing) : native.startWithEndpointing(endpointing)) ?? '') const error = String(native.startPcmCaptureWithEndpointingPhone(endpointing, phone) ?? '') if (error) return error @@ -130,7 +149,26 @@ export class IosVoiceControlBridge { if (this.source !== 'phone') return this.stop(); this.setStatus('Phone voice input stopped.'); this.emitEnd() } + /** Whether continuous capture can run with this provider (Apple's recognizer can't). */ + supportsContinuous(voice: VoiceProviderSelection): boolean { return usesCloudStt(voice) } + get continuousActive(): boolean { return this.continuous && (this.capturing || this.finalizing) } + /** End continuous capture, transcribing the audio since the last pause. */ + stopContinuousCapture(): Promise { + if (!this.continuous) return Promise.resolve() + return this.finalizeCapture() + } + /** Push-to-talk shares a listening continuous capture (not one that is only finalizing). */ + private joinsContinuous(continuous: boolean): boolean { + if (continuous || !this.continuous || !this.capturing) return false + this.log('Voice: push-to-talk joins the running continuous capture') + return true + } stopPushToTalk(): Promise { + // The continuous capture owns the mic and keeps it. + if (this.continuous) return Promise.resolve() + return this.finalizeCapture() + } + private finalizeCapture(): Promise { const completion = this.completion if (!this.capturing) return completion this.capturing = false; this.finalizing = true @@ -140,7 +178,7 @@ export class IosVoiceControlBridge { } stop(): void { ++this.generation - this.capturing = this.finalizing = false + this.capturing = this.finalizing = this.continuous = false this.source = null this.clearAudioTimer(); this.releaseMicrophone(); this.native?.cancel() this.clearCloud() @@ -155,7 +193,14 @@ export class IosVoiceControlBridge { const session = this.session; this.session = null if (session) void session.setMicrophone(false).catch(error => this.log(`Voice microphone cleanup: ${error}`)) } + /** Record this capture's audio as Downloads/.wav (only cloud captures supply PCM). */ + private startRecording(name: string, log: (message: string) => void): void { + if (!name) return + this.recorder = WavRecorder.open(name) + log(this.recorder ? `Voice: recording audio to ${this.recorder.path}` : 'Voice: could not create the audio recording') + } private clearCloud(): void { + this.recorder?.close(); this.recorder = null if (this.cloudTimer !== null) clearTimeout(this.cloudTimer) this.cloudTimer = null this.cloud?.stop(); this.cloud = null; this.cloudEndStatus = null @@ -166,13 +211,13 @@ export class IosVoiceControlBridge { this.cloudEndStatus = status this.setStatus('Recognizing…') // Armed first: finish() can deliver the final synchronously. - this.cloudTimer = setTimeout(() => this.completeCloud(), CLOUD_FINAL_TIMEOUT_MS) + this.cloudTimer = setTimeout(() => this.completeCloud(), this.cloud?.finalTimeoutMs ?? CLOUD_FINAL_TIMEOUT_MS) this.cloud?.finish() } private completeCloud(): void { const status = this.cloudEndStatus ?? 'Ready to send' this.clearCloud() - this.capturing = this.finalizing = false + this.capturing = this.finalizing = this.continuous = false this.finishCompletion() this.setStatus(status); this.emitEnd() } @@ -180,7 +225,8 @@ export class IosVoiceControlBridge { if (!this.capturing && !this.finalizing) return for (const fn of [...this.transcripts]) fn(event) // Push-to-talk commits once, so the first final after it is the last. - if (event.isFinal && this.cloudEndStatus !== null) this.completeCloud() + // In continuous capture, earlier pause segments (paragraphBreakAfter) may still arrive. + if (event.isFinal && this.cloudEndStatus !== null && !(this.continuous && event.paragraphBreakAfter)) this.completeCloud() } private emitEnd(): void { for (const fn of [...this.ends]) fn() } private finishCompletion(): void { diff --git a/app/native/voice-control.ts b/app/native/voice-control.ts index 91db6624..f7892157 100644 --- a/app/native/voice-control.ts +++ b/app/native/voice-control.ts @@ -28,7 +28,9 @@ export type VoiceControlState = { // family, not with "onboard-whisper" below, which is the on-device sherpa-onnx // Whisper backend added alongside "onboard" (Moonshine). Kept as-is rather than // renamed, since it's a persisted settings value on real installs already. -export type VoiceProviderKind = "onboard" | "onboard-whisper" | "elevenlabs" | "whisper" | "soniox"; +// "whisper-server" is a self-hosted OpenAI-compatible Whisper over HTTP +// (whisper-server-stt.ts). +export type VoiceProviderKind = "onboard" | "onboard-whisper" | "elevenlabs" | "whisper" | "soniox" | "whisper-server"; export type VoiceTranscriptEvent = CloudSttTranscriptEvent & { /** @@ -52,6 +54,10 @@ export type PushToTalkOptions = { elevenLabsApiKey: string; openAiApiKey: string; sonioxApiKey: string; + whisperServerUrl?: string; + whisperServerModel?: string; + whisperServerLanguage?: string; + whisperServerApiKey?: string; saveRecording: boolean; /** * Watch the mic and fire onSpeechEnd when the speaker stops. For hands-free diff --git a/app/native/wav-header.ts b/app/native/wav-header.ts new file mode 100644 index 00000000..a85ba2e0 --- /dev/null +++ b/app/native/wav-header.ts @@ -0,0 +1,22 @@ +/** The 44-byte header of a 16 kHz mono PCM16 WAV file holding `dataBytes` of samples. */ +export function wavHeader(dataBytes: number): Uint8Array { + const header = new Uint8Array(44); + const view = new DataView(header.buffer); + const ascii = (offset: number, text: string) => { + for (let i = 0; i < text.length; i++) header[offset + i] = text.charCodeAt(i); + }; + ascii(0, "RIFF"); + view.setUint32(4, 36 + dataBytes, true); + ascii(8, "WAVE"); + ascii(12, "fmt "); + view.setUint32(16, 16, true); + view.setUint16(20, 1, true); // PCM + view.setUint16(22, 1, true); // mono + view.setUint32(24, 16000, true); + view.setUint32(28, 32000, true); // byte rate + view.setUint16(32, 2, true); // block align + view.setUint16(34, 16, true); // bits per sample + ascii(36, "data"); + view.setUint32(40, dataBytes, true); + return header; +} diff --git a/app/native/wav-recorder.ios.ts b/app/native/wav-recorder.ios.ts new file mode 100644 index 00000000..ff5e815a --- /dev/null +++ b/app/native/wav-recorder.ios.ts @@ -0,0 +1,75 @@ +import { knownFolders, path as paths } from '@nativescript/core' +import { wavHeader } from './wav-header' + +// About a second of 16 kHz mono PCM16: written in batches, not per 50 ms packet. +const FLUSH_BYTES = 32000 + +/** + * Appends 16 kHz mono PCM16 to a WAV file in Documents/Downloads while it is + * captured. The header's sizes are rewritten on every flush, so a crash leaves a + * playable file of everything up to the last second. + */ +export class WavRecorder { + private pending: Uint8Array[] = [] + private pendingBytes = 0 + private dataBytes = 0 + + private constructor(private handle: NSFileHandle | null, readonly path: string) {} + + /** A recorder for Downloads/.wav (a -2, -3... suffix if taken), or null. */ + static open(name: string): WavRecorder | null { + const fm = NSFileManager.defaultManager + const dir = paths.join(knownFolders.documents().path, 'Downloads') + try { + fm.createDirectoryAtPathWithIntermediateDirectoriesAttributesError(dir, true, null) + let path = paths.join(dir, `${name}.wav`) + for (let n = 2; fm.fileExistsAtPath(path); n++) path = paths.join(dir, `${name}-${n}.wav`) + if (!fm.createFileAtPathContentsAttributes(path, toData(wavHeader(0)), null)) return null + const handle = NSFileHandle.fileHandleForWritingAtPath(path) + if (!handle || !handle.seekToEndReturningOffsetError(null)) return null + return new WavRecorder(handle, path) + } catch { + return null + } + } + + append(pcm: Uint8Array): void { + if (!this.handle || !pcm.length) return + this.pending.push(pcm.slice()) + this.pendingBytes += pcm.length + if (this.pendingBytes >= FLUSH_BYTES) this.flush() + } + + close(): void { + this.flush() + try { this.handle?.closeAndReturnError() } catch { /* already closed */ } + this.handle = null + } + + private flush(): void { + const handle = this.handle + if (!handle || !this.pendingBytes) return + const chunk = new Uint8Array(this.pendingBytes) + let offset = 0 + for (const part of this.pending) { chunk.set(part, offset); offset += part.length } + this.pending = [] + this.pendingBytes = 0 + try { + // NativeScript throws a failed call's NSError (storage full, file removed). + handle.writeDataError(toData(chunk)) + handle.seekToOffsetError(0) + handle.writeDataError(toData(wavHeader(this.dataBytes + chunk.length))) + handle.seekToEndReturningOffsetError(null) + this.dataBytes += chunk.length + return + } catch { + // Stop recording rather than retry. + } + try { handle.closeAndReturnError() } catch { /* ignore */ } + this.handle = null + } +} + +function toData(bytes: Uint8Array): NSData { + return NSData.dataWithBytesLength(interop.handleof(bytes.buffer), bytes.byteLength) +} diff --git a/app/native/whisper-server-stt.ts b/app/native/whisper-server-stt.ts new file mode 100644 index 00000000..f2caac29 --- /dev/null +++ b/app/native/whisper-server-stt.ts @@ -0,0 +1,386 @@ +import { CLOUD_STT_SAMPLE_RATE, type CloudSttClient, type CloudSttOptions } from "./cloud-stt"; + +/** + * Speech-to-text against a self-hosted OpenAI-compatible Whisper server (vLLM, + * speaches, LocalAI): batch POST {baseUrl}/audio/transcriptions with a WAV file. + * Each segment (a push-to-talk utterance, or a Transcribe span ended by a pause) + * is one request, finalized in audio order; an open segment is re-sent every + * INTERIM_STEP_BYTES for provisional text. start() lists {baseUrl}/models to check + * reachability and pick a model when none is configured. + */ + +export type WhisperServerConfig = { + /** OpenAI-style API root, e.g. http://host:8004/v1. */ + baseUrl: string; + /** Model id to request; empty uses the first one the server lists. */ + model: string; + /** ISO-639-1 language code; empty lets Whisper detect it. */ + language: string; +}; + +export type WhisperHttpRequest = { + url: string; + method: "GET" | "POST"; + headers: Record; + body?: ArrayBuffer; + timeoutMs: number; +}; +export type WhisperHttpResponse = { status: number; text: string }; +export type WhisperHttp = (request: WhisperHttpRequest) => Promise; + +const BYTES_PER_SECOND = CLOUD_STT_SAMPLE_RATE * 2; +// Provisional transcripts: first after one second of audio, then every 1.5s, +// and not for segments past Whisper's 30s window (each would re-send it all). +const INTERIM_MIN_BYTES = BYTES_PER_SECOND; +const INTERIM_STEP_BYTES = BYTES_PER_SECOND * 1.5; +const INTERIM_MAX_BYTES = BYTES_PER_SECOND * 30; +const MODELS_TIMEOUT_MS = 5000; +const INTERIM_TIMEOUT_MS = 10000; +const FINAL_TIMEOUT_MS = 12000; +const FINAL_RETRY_DELAY_MS = 750; +// Whisper invents text ("Thank you.") for silence, so a segment whose loudest +// 50ms frame stays below this RMS is not sent at all. Normal speech on the G2 +// mic measures in the thousands; the pause detector's noise floor starts at 220. +const SILENCE_RMS = 300; + +/** Time the voice bridges should allow a final transcript before giving up. */ +export const WHISPER_SERVER_FINAL_TIMEOUT_MS = FINAL_TIMEOUT_MS * 2 + FINAL_RETRY_DELAY_MS + 2000; + +type Segment = { + chunks: Uint8Array[]; + bytes: number; + peakRms: number; + text: string; + final: boolean; + closed: boolean; + paragraphBreakAfter: boolean; + /** Emit a final even when empty: the end of push-to-talk needs one. */ + required: boolean; + interimInFlight: boolean; + interimBytes: number; +}; + +/** + * The API root from what a user typed: tolerates a trailing slash or the full + * transcription endpoint, and adds /v1 to a bare host. Empty when unusable. + */ +export function normalizeWhisperServerUrl(input: string): string { + let url = input.trim().replace(/\/+$/, ""); + if (!/^https?:\/\/[^/\s]+/i.test(url)) return ""; + url = url.replace(/\/audio\/transcriptions$/i, ""); + if (/^https?:\/\/[^/]+$/i.test(url)) url += "/v1"; + return url; +} + +export class WhisperServerSttClient implements CloudSttClient { + private readonly baseUrl: string; + private model: string; + private ready = false; + private closed = false; + private readonly segments: Segment[] = []; + + constructor( + private readonly options: CloudSttOptions, + private readonly config: WhisperServerConfig, + private readonly http: WhisperHttp = nativeHttp, + ) { + this.baseUrl = normalizeWhisperServerUrl(config.baseUrl); + this.model = config.model.trim(); + } + + start(): void { + if (this.closed || this.ready) return; + if (!this.baseUrl) { + this.options.onError("Whisper server URL is not a valid http(s) address."); + return; + } + this.options.onStatus("Connecting to Whisper server..."); + this.http({ url: `${this.baseUrl}/models`, method: "GET", headers: this.authHeaders(), timeoutMs: MODELS_TIMEOUT_MS }) + .then((response) => { + if (this.closed) return; + if (response.status < 200 || response.status >= 300) { + this.options.onDisconnected?.(`Whisper server returned ${response.status} for /models.`); + return; + } + if (!this.model) this.model = firstModelId(response.text); + if (!this.model) { + this.options.onError("Whisper server lists no models; set the model in Settings > Voice."); + return; + } + this.ready = true; + this.options.onStatus("Listening (Whisper server)..."); + this.options.onReady?.(); + }) + .catch((error) => { + if (!this.closed) this.options.onDisconnected?.(`Whisper server unreachable: ${describeError(error)}`); + }); + } + + /** PCM16LE at 16 kHz, appended to the open segment. */ + acceptPcm(pcm: Uint8Array): void { + if (this.closed || !pcm.length) return; + const segment = this.openSegment(); + segment.chunks.push(pcm.slice()); + segment.bytes += pcm.length; + segment.peakRms = Math.max(segment.peakRms, rms(pcm)); + this.maybeRequestInterim(segment); + } + + /** A pause in continuous capture: transcribe what came before it. */ + commitSegment(): void { + this.closeSegment(true, false); + } + + /** End of the utterance: transcribe the rest and deliver a final. */ + finish(): void { + this.closeSegment(false, true); + } + + stop(): void { + this.closed = true; + this.segments.length = 0; + } + + private openSegment(): Segment { + const last = this.segments[this.segments.length - 1]; + if (last && !last.closed) return last; + const segment: Segment = { + chunks: [], bytes: 0, peakRms: 0, text: "", final: false, closed: false, + paragraphBreakAfter: false, required: false, interimInFlight: false, interimBytes: 0, + }; + this.segments.push(segment); + return segment; + } + + private closeSegment(paragraphBreakAfter: boolean, required: boolean): void { + if (this.closed) return; + const last = this.segments[this.segments.length - 1]; + if (!last || last.closed) { + // Nothing new since the last boundary. Push-to-talk still needs a final. + if (required) this.finalize(this.openSegment(), "", paragraphBreakAfter, required); + return; + } + if (last.peakRms < SILENCE_RMS) { + this.finalize(last, "", paragraphBreakAfter, required); + return; + } + last.closed = true; + last.paragraphBreakAfter = paragraphBreakAfter; + last.required = required; + void this.requestFinal(last, 0); + } + + private async requestFinal(segment: Segment, attempt: number): Promise { + try { + const text = await this.transcribe(segment, segment.bytes, FINAL_TIMEOUT_MS); + if (!this.closed) this.finalize(segment, text, segment.paragraphBreakAfter, segment.required); + } catch (error) { + if (this.closed) return; + if (error instanceof WhisperAuthError) { + this.options.onError(error.message); + return; + } + if (attempt === 0) { + setTimeout(() => { void this.requestFinal(segment, 1); }, FINAL_RETRY_DELAY_MS); + return; + } + // Keep the capture alive and keep whatever the interim pass heard. + this.options.onStatus(`Whisper server request failed: ${describeError(error)}`); + this.finalize(segment, segment.text, segment.paragraphBreakAfter, segment.required); + } + } + + private maybeRequestInterim(segment: Segment): void { + if (!this.ready || segment.interimInFlight || segment.closed) return; + if (segment.bytes < INTERIM_MIN_BYTES || segment.bytes > INTERIM_MAX_BYTES) return; + if (segment.bytes - segment.interimBytes < INTERIM_STEP_BYTES) return; + if (segment.peakRms < SILENCE_RMS) return; + segment.interimInFlight = true; + segment.interimBytes = segment.bytes; + this.transcribe(segment, segment.bytes, INTERIM_TIMEOUT_MS) + .then((text) => { + if (this.closed || segment.final) return; + segment.text = text; + this.emit(); + }) + .catch(() => { + // Provisional only; the final request reports real failures. + }) + .finally(() => { + segment.interimInFlight = false; + }); + } + + private finalize(segment: Segment, text: string, paragraphBreakAfter: boolean, required: boolean): void { + segment.closed = segment.final = true; + segment.text = text; + segment.paragraphBreakAfter = paragraphBreakAfter; + segment.required = required; + segment.chunks = []; + this.emit(); + } + + /** + * Finals leave in audio order, even when a later request answers first; + * whatever is not final yet is shown as one provisional transcript. + */ + private emit(): void { + while (this.segments.length && this.segments[0]!.final) { + const segment = this.segments.shift()!; + if (!segment.text && !segment.required) continue; + this.options.onTranscript({ text: segment.text, isFinal: true, paragraphBreakAfter: segment.paragraphBreakAfter }); + } + const pending = this.segments.filter((segment) => segment.text.trim()); + if (!pending.length) return; + const text = pending.map((segment) => segment.text.trim()).join(" "); + const transcribeText = pending.map((segment, index) => + (index && pending[index - 1]!.paragraphBreakAfter ? "\n" : index ? " " : "") + segment.text.trim(), + ).join(""); + this.options.onTranscript({ text, transcribeText, isFinal: false }); + } + + private async transcribe(segment: Segment, bytes: number, timeoutMs: number): Promise { + const fields: Record = { model: this.model, response_format: "json", temperature: "0" }; + const language = this.config.language.trim().toLowerCase(); + if (language) fields.language = language; + const boundary = `faceclaw-${Date.now().toString(36)}-${Math.random().toString(36).slice(2)}`; + const body = multipartBody(boundary, fields, wavFile(segment.chunks, bytes)); + const response = await this.http({ + url: `${this.baseUrl}/audio/transcriptions`, + method: "POST", + headers: { ...this.authHeaders(), "Content-Type": `multipart/form-data; boundary=${boundary}` }, + body, + timeoutMs, + }); + if (response.status === 401 || response.status === 403) { + throw new WhisperAuthError(`Whisper server rejected the key (${response.status}).`); + } + if (response.status < 200 || response.status >= 300) { + throw new Error(`HTTP ${response.status}: ${response.text.slice(0, 200)}`); + } + return parseTranscript(response.text); + } + + private authHeaders(): Record { + const key = this.options.apiKey.trim(); + return key ? { Authorization: `Bearer ${key}` } : {}; + } +} + +class WhisperAuthError extends Error {} + +/** The transcript from a JSON response; tolerates servers that send plain text. */ +export function parseTranscript(body: string): string { + const raw = body.trim(); + try { + const parsed = JSON.parse(raw); + if (parsed && typeof parsed.text === "string") return parsed.text.trim(); + } catch { + // Not JSON: a server that answered response_format=text anyway. + } + return raw; +} + +function firstModelId(body: string): string { + try { + const parsed = JSON.parse(body); + const id = parsed?.data?.[0]?.id; + return typeof id === "string" ? id : ""; + } catch { + return ""; + } +} + +/** RMS of a PCM16LE chunk. */ +function rms(pcm: Uint8Array): number { + const count = pcm.length >> 1; + if (!count) return 0; + let squares = 0; + for (let i = 0; i < count; i++) { + const unsigned = pcm[i * 2]! | (pcm[i * 2 + 1]! << 8); + const sample = unsigned >= 32768 ? unsigned - 65536 : unsigned; + squares += sample * sample; + } + return Math.sqrt(squares / count); +} + +/** The first `bytes` of the chunks as a 16 kHz mono 16-bit WAV file. */ +export function wavFile(chunks: Uint8Array[], bytes: number): Uint8Array { + const length = bytes & ~1; + const out = new Uint8Array(44 + length); + const view = new DataView(out.buffer); + const ascii = (offset: number, text: string) => { + for (let i = 0; i < text.length; i++) out[offset + i] = text.charCodeAt(i); + }; + ascii(0, "RIFF"); + view.setUint32(4, 36 + length, true); + ascii(8, "WAVE"); + ascii(12, "fmt "); + view.setUint32(16, 16, true); + view.setUint16(20, 1, true); // PCM + view.setUint16(22, 1, true); // mono + view.setUint32(24, CLOUD_STT_SAMPLE_RATE, true); + view.setUint32(28, BYTES_PER_SECOND, true); + view.setUint16(32, 2, true); + view.setUint16(34, 16, true); + ascii(36, "data"); + view.setUint32(40, length, true); + let offset = 44; + for (const chunk of chunks) { + const take = Math.min(chunk.length, out.length - offset); + if (take <= 0) break; + out.set(chunk.subarray(0, take), offset); + offset += take; + } + return out; +} + +/** multipart/form-data with text fields and one WAV part named "file". */ +export function multipartBody(boundary: string, fields: Record, wav: Uint8Array): ArrayBuffer { + let head = ""; + for (const [name, value] of Object.entries(fields)) { + head += `--${boundary}\r\nContent-Disposition: form-data; name="${name}"\r\n\r\n${value}\r\n`; + } + head += `--${boundary}\r\nContent-Disposition: form-data; name="file"; filename="audio.wav"\r\nContent-Type: audio/wav\r\n\r\n`; + const headBytes = utf8(head); + const tailBytes = utf8(`\r\n--${boundary}--\r\n`); + const out = new Uint8Array(headBytes.length + wav.length + tailBytes.length); + out.set(headBytes, 0); + out.set(wav, headBytes.length); + out.set(tailBytes, headBytes.length + wav.length); + return out.buffer; +} + +function utf8(text: string): Uint8Array { + const out: number[] = []; + for (const char of text) { + const code = char.codePointAt(0)!; + if (code < 0x80) out.push(code); + else if (code < 0x800) out.push(0xc0 | (code >> 6), 0x80 | (code & 63)); + else if (code < 0x10000) out.push(0xe0 | (code >> 12), 0x80 | ((code >> 6) & 63), 0x80 | (code & 63)); + else out.push(0xf0 | (code >> 18), 0x80 | ((code >> 12) & 63), 0x80 | ((code >> 6) & 63), 0x80 | (code & 63)); + } + return Uint8Array.from(out); +} + +function describeError(error: unknown): string { + return String((error as Error)?.message ?? error); +} + +/** NativeScript's HTTP module: sends an ArrayBuffer body as-is on both platforms. */ +const nativeHttp: WhisperHttp = async (request) => { + // Required lazily so Node tests can load this module without NativeScript. + const { withUserAgent } = require("../util/http") as typeof import("../util/http"); + const { request: send } = require("@nativescript/core/http") as typeof import("@nativescript/core/http"); + const response = await send({ + url: request.url, + method: request.method, + headers: withUserAgent(request.headers), + content: request.body, + timeout: request.timeoutMs, + }); + const content = response.content as any; + let text = ""; + if (content) text = global.isAndroid ? String(content.raw.toString("UTF-8")) : String(content.toString()); + return { status: response.statusCode, text }; +}; diff --git a/app/ui/dashboard-settings.ts b/app/ui/dashboard-settings.ts index 083291fe..6dddbc32 100644 --- a/app/ui/dashboard-settings.ts +++ b/app/ui/dashboard-settings.ts @@ -17,6 +17,7 @@ import { type AssistantModel, } from "~/assistant/models"; import { isLocalModelReady } from "../native/llama"; +import { normalizeWhisperServerUrl } from "../native/whisper-server-stt"; import { drawRightValueMenuItem, drawToggleMenuItem, MenuItem, openModalMenu } from "./menu"; import { ANIMATION_SPEEDS, animationSpeedLabel, scaleAnimationDuration, type AnimationSpeed } from "./animation-speed"; import { MENU_ANIMATION_KEY } from "./menu-animation-pref"; @@ -814,7 +815,8 @@ export const ringConnectionModeSetting = new ConfigSettingEnum = { elevenlabs: "ElevenLabs", whisper: "OpenAI (Whisper)", soniox: "Soniox", + "whisper-server": "Whisper server (self-hosted)", }; export const voiceProviderSetting = new ConfigSettingEnum({ @@ -831,17 +834,20 @@ export const voiceProviderSetting = new ConfigSettingEnum({ label: "Transcription Provider", storageKey: "voice.provider", defaultValue: "onboard", - values: global.isIOS ? ["onboard", "elevenlabs", "whisper", "soniox"] : ["onboard", "onboard-whisper", "elevenlabs", "whisper", "soniox"], + values: global.isIOS + ? ["onboard", "elevenlabs", "whisper", "soniox", "whisper-server"] + : ["onboard", "onboard-whisper", "elevenlabs", "whisper", "soniox", "whisper-server"], formatValue: (value) => voiceProviderLabels[value] ?? value, isDisabled: (value) => { if (value === "elevenlabs") return elevenLabsApiKeySetting.get().trim().length === 0; if (value === "whisper") return openAiApiKeySetting.get().trim().length === 0; if (value === "soniox") return sonioxApiKeySetting.get().trim().length === 0; + if (value === "whisper-server") return !normalizeWhisperServerUrl(whisperServerUrlSetting.get()); return false; }, description: global.isIOS - ? "Speech-to-text engine for voice input. ElevenLabs, OpenAI, and Soniox are cloud services that need an API key (Settings > API Keys), with significantly better accuracy than on-device transcription. On-device uses your iPhone's speech language and never leaves the phone." - : "Speech-to-text engine for voice input. ElevenLabs, OpenAI, and Soniox are cloud services that need an API key, with significantly better accuracy than on-device transcription. The two On-device options need their voice model downloaded (below) and never leave the phone.", + ? "Speech-to-text engine for voice input. ElevenLabs, OpenAI, and Soniox are cloud services that need an API key (Settings > API Keys), with significantly better accuracy than on-device transcription. Whisper server is your own OpenAI-compatible Whisper (set its URL below). On-device uses your iPhone's speech language and never leaves the phone." + : "Speech-to-text engine for voice input. ElevenLabs, OpenAI, and Soniox are cloud services that need an API key, with significantly better accuracy than on-device transcription. Whisper server is your own OpenAI-compatible Whisper (set its URL below). The two On-device options need their voice model downloaded (below) and never leave the phone.", }); const wakeWordActionLabels: Record = { @@ -860,6 +866,14 @@ export const wakeWordActionSetting = new ConfigSettingEnum({ description: "What saying \"Hey Even\" does: start voice input, just turn the screen on, or nothing.", }); +export const saveTranscribeAudioSetting = new ConfigSettingBoolean({ + id: "save-transcribe-audio", + label: "Save Transcribe audio", + storageKey: "voice.saveTranscribeAudio", + defaultValue: true, + description: "Record each Transcribe session's audio as a WAV file in Downloads (about 2 MB per minute), named like its transcript. Recordings are kept until you delete them.", +}); + export const saveVoiceRecordingsSetting = new ConfigSettingBoolean({ id: "save-voice-recordings", label: "Save voice recordings", @@ -968,6 +982,54 @@ export const sonioxApiKeySetting = new ConfigSettingString({ description: "Soniox API key, used when Soniox is the transcription provider.", }); +export const whisperServerUrlSetting = new ConfigSettingString({ + id: "whisper-server-url", + label: "Whisper server URL", + storageKey: "voice.whisperServerUrl", + defaultValue: "", + editorTitle: "Whisper server URL (e.g. http://100.x.y.z:8004/v1)", + glassesEditTitle: "Edit Whisper server URL", + formatValue: (value) => value || "(not set)", + validate: (value) => (!value.trim() || normalizeWhisperServerUrl(value) ? null : "Enter an http:// or https:// address."), + description: + "API root of an OpenAI-compatible Whisper server (vLLM, speaches, LocalAI...), used when Whisper server is the transcription provider. On iOS a plain-http address must also be allowed by the app's App Transport Security exceptions.", +}); + +export const whisperServerModelSetting = new ConfigSettingString({ + id: "whisper-server-model", + label: "Whisper server model", + storageKey: "voice.whisperServerModel", + defaultValue: "", + editorTitle: "Whisper model id (empty: server's first)", + glassesEditTitle: "Edit Whisper model", + formatValue: (value) => value || "(server default)", + description: "Model id sent to the Whisper server. Leave empty to use the first model the server lists.", +}); + +export const whisperServerLanguageSetting = new ConfigSettingString({ + id: "whisper-server-language", + label: "Whisper language", + storageKey: "voice.whisperServerLanguage", + defaultValue: "", + editorTitle: "Language code, e.g. en or sk (empty: auto)", + glassesEditTitle: "Edit Whisper language", + formatValue: (value) => value || "Auto-detect", + validate: (value) => (!value.trim() || /^[a-z]{2,3}$/i.test(value.trim()) ? null : "Use a language code like en, sk or cs."), + description: + "Spoken language for the Whisper server. Leave empty to auto-detect; a wrong fixed language makes Whisper translate into it instead of transcribing.", +}); + +export const whisperServerApiKeySetting = new ConfigSettingString({ + id: "whisper-server-api-key", + label: "Whisper server key", + storageKey: "voice.whisperServerApiKey", + defaultValue: "", + editorTitle: "Whisper server API key (optional)", + glassesEditTitle: "Edit Whisper server key", + formatValue: (value) => (value ? `${value.slice(0, 6)}...` : "(none)"), + description: "Bearer token for the Whisper server, if it requires one.", +}); + export const anthropicApiKeySetting = new ConfigSettingString({ id: "anthropic-api-key", label: "Anthropic key", diff --git a/app/ui/dashboard/settings-menus.ts b/app/ui/dashboard/settings-menus.ts index 92833593..bca76537 100644 --- a/app/ui/dashboard/settings-menus.ts +++ b/app/ui/dashboard/settings-menus.ts @@ -61,6 +61,11 @@ import { phoneRotationSetting, ringConnectionModeSetting, sonioxApiKeySetting, + saveTranscribeAudioSetting, + whisperServerApiKeySetting, + whisperServerLanguageSetting, + whisperServerModelSetting, + whisperServerUrlSetting, screenFadeSetting, enumSettingMenuItem, firmwareDebugFlagsSetting, @@ -124,6 +129,7 @@ function settingsSections(): SettingsSection[] { items: [ enumSettingMenuItem(wakeWordActionSetting), enumSettingMenuItem(voiceProviderSetting), + ...whisperServerMenuItems(), asrModelMenuItem("moonshine"), asrModelMenuItem("whisper-base-en"), ], @@ -237,7 +243,7 @@ function settingsSections(): SettingsSection[] { if (section.label === "Developer") return { ...section, items: [toggleSettingMenuItem(showBleBandwidthSetting)] }; // No downloadable on-device models: iOS's on-device option is Apple's recognizer. if (section.label === "Voice") return { label: "Voice", items: [enumSettingMenuItem(wakeWordActionSetting), - enumSettingMenuItem(voiceProviderSetting)] }; + enumSettingMenuItem(voiceProviderSetting), toggleSettingMenuItem(saveTranscribeAudioSetting), ...whisperServerMenuItems()] }; if (deferred.has(section.label)) return { label: section.label, items: [{ label: "Not available on iOS yet", disabled: true, onSelect: () => {}, description: `${section.label} integration has not been ported to iOS.`, @@ -451,6 +457,19 @@ function asrModelStatusText(id: AsrModelId): string { return "not downloaded"; } +/** + * The self-hosted Whisper server's settings. Always listed: the provider + * option stays disabled until the URL is set. + */ +function whisperServerMenuItems(): MenuItem[] { + return [ + textSettingMenuItem(whisperServerUrlSetting), + textSettingMenuItem(whisperServerModelSetting), + textSettingMenuItem(whisperServerLanguageSetting), + textSettingMenuItem(whisperServerApiKeySetting), + ]; +} + /** Download/cancel/delete management for one on-device transcription model. */ function asrModelMenuItem(id: AsrModelId): MenuItem { const def = ASR_MODELS[id]; diff --git a/app/ui/layers.ts b/app/ui/layers.ts index 4c2cb796..b3573b59 100644 --- a/app/ui/layers.ts +++ b/app/ui/layers.ts @@ -31,8 +31,11 @@ export type LayerActions = { startVoiceCapture: (endpointing?: boolean) => Promise | void; /** Stop push-to-talk; awaits native recognition, or commits a cloud provider for a final result. */ stopVoiceCapture: () => Promise | void; - /** Start continuous capture (Transcribe); shares the mic with push-to-talk. */ - startContinuousVoiceCapture: () => Promise | void; + /** + * Start continuous capture (Transcribe); shares the mic with push-to-talk. + * recordAs names a WAV of the captured audio in Downloads (iOS, when enabled). + */ + startContinuousVoiceCapture: (options?: { recordAs?: string }) => Promise | void; stopContinuousVoiceCapture: () => Promise | void; /** Play a CFW tone-sequencer payload (see sound-effects.ts). */ playBuzzerSequence: (payload: Uint8Array) => Promise | void; diff --git a/notes/whisper-server.md b/notes/whisper-server.md new file mode 100644 index 00000000..20ad4a5b --- /dev/null +++ b/notes/whisper-server.md @@ -0,0 +1,44 @@ +# Self-hosted Whisper transcription + +The "Whisper server (self-hosted)" transcription provider sends voice audio to +any OpenAI-compatible Whisper server instead of a cloud service: vLLM, speaches +(faster-whisper-server), LocalAI and similar. It uses the batch endpoint +`POST {url}/audio/transcriptions` with a 16 kHz mono WAV. Push-to-talk +utterances are one request each; in Transcribe, each span ended by a pause is +one request. While a span is open its audio so far is re-sent every 1.5 s, so +text appears while you are still talking. + +## Server + +Any server exposing `/v1/models` and `/v1/audio/transcriptions` works, e.g. + + vllm serve openai/whisper-large-v3 --port 8004 + +## Settings (Settings > Voice) + +- **Transcription Provider**: Whisper server (self-hosted). Disabled until the + URL is set. +- **Whisper server URL**: the API root, e.g. `http://host:8004/v1`. A bare + `http://host:8004` gets `/v1` added. +- **Whisper server model**: empty uses the first model the server lists. +- **Whisper language**: an ISO code such as `en`, or empty to auto-detect. + A wrong fixed language makes Whisper translate rather than transcribe. +- **Whisper server key**: optional bearer token. + +## iOS: plain http + +iOS App Transport Security blocks plain `http://` to IP addresses (including +Tailscale's 100.x addresses) unless the app's Info.plist has an exception for +that host. Either serve the API over https (for example with `tailscale serve`) +or add your server to `NSExceptionDomains` in `App_Resources/iOS/Info.plist` +before building: + + 192.0.2.10 + NSExceptionAllowsInsecureHTTPLoads + +## Transcribe on iOS + +Transcribe needs a cloud or Whisper-server provider on iOS (Apple's recognizer +only handles short utterances). With "Save Transcribe audio" on (Voice +settings), each session's audio is kept as `transcribe-