From a80d6697b5ebea8a73135c64b3746fa135afdf37 Mon Sep 17 00:00:00 2001 From: Guillaume ETHEVE Date: Sun, 8 Nov 2026 00:35:51 +0100 Subject: [PATCH] Allow choosing an OpenAI transcription model available to the project --- app/g2/dashboard-controller.ts | 4 +-- app/g2/ios-preview-controller.ts | 4 +-- app/native/cloud-stt-provider.ts | 10 ++++--- app/native/openai-stt.ts | 17 +++++++----- app/native/voice-control.ts | 4 ++- app/ui/dashboard-settings.ts | 10 +++++++ app/ui/dashboard/settings-menus.ts | 2 ++ docs/validation-whisper-languages.md | 22 +++++++++++++++ tests/cloud-stt.test.cjs | 40 ++++++++++++++++++++++++++++ tests/ios-app-host.test.cjs | 2 +- tests/whisper-language.test.cjs | 15 +++++++++++ 11 files changed, 113 insertions(+), 17 deletions(-) diff --git a/app/g2/dashboard-controller.ts b/app/g2/dashboard-controller.ts index 50d0d068..7fb8bf7a 100644 --- a/app/g2/dashboard-controller.ts +++ b/app/g2/dashboard-controller.ts @@ -78,7 +78,7 @@ import { loadPersistedOpenApps, savePersistedOpenApps } from "../ui/shell/open-a import { appViewportRect, isOnSwitcherEdge, sidebarStripVisible, type ScreenRect, type WindowHeightMode } from "../ui/shell/geometry"; import { type ContinuousCaptureOptions, type LayerActions, type TextSettingsEditToggle, type TranscriptionChoice } from "../ui/layers"; import { type KeyboardInputSession } from "../ui/shell/keyboard-input"; -import { appSwitcherPositionSetting, statusBarPositionSetting, statusBarVisibilitySetting, windowBorderSetting, assistantAllowProactiveSetting, assistantBackendSetting, assistantBridgeHostSetting, assistantBridgePortSetting, assistantBridgeTokenSetting, getBrightnessPreferences, displayLayoutSetting, navigateDisplayLayoutSetting, terminalDisplayLayoutSetting, elevenLabsApiKeySetting, getStringSettingById, openAiApiKeySetting, nightscoutApiTokenSetting, firmwareDebugFlagsSetting, whisperServerApiKeySetting, getVoiceTranscriptionLanguage, whisperServerModelSetting, whisperServerUrlSetting, lockScreenEnabledSetting, nightscoutSiteUrlSetting, onAnySettingChanged, previewColorSetting, ringConnectionModeSetting, type RingConnectionMode, saveVoiceRecordingsSetting, sonioxApiKeySetting, screenTimeoutSetting, screenTimeoutSettingToMs, suspendEvenHubWhenScreenOffSetting, voiceProviderSetting, wakeWordActionSetting, type ConfigSettingString } from "../ui/dashboard-settings"; +import { appSwitcherPositionSetting, statusBarPositionSetting, statusBarVisibilitySetting, windowBorderSetting, assistantAllowProactiveSetting, assistantBackendSetting, assistantBridgeHostSetting, assistantBridgePortSetting, assistantBridgeTokenSetting, getBrightnessPreferences, displayLayoutSetting, navigateDisplayLayoutSetting, terminalDisplayLayoutSetting, elevenLabsApiKeySetting, getStringSettingById, openAiApiKeySetting, openAiTranscriptionModelSetting, nightscoutApiTokenSetting, firmwareDebugFlagsSetting, whisperServerApiKeySetting, getVoiceTranscriptionLanguage, whisperServerModelSetting, whisperServerUrlSetting, lockScreenEnabledSetting, nightscoutSiteUrlSetting, onAnySettingChanged, previewColorSetting, ringConnectionModeSetting, type RingConnectionMode, saveVoiceRecordingsSetting, sonioxApiKeySetting, screenTimeoutSetting, screenTimeoutSettingToMs, suspendEvenHubWhenScreenOffSetting, voiceProviderSetting, wakeWordActionSetting, type ConfigSettingString } from "../ui/dashboard-settings"; import { isIgnoringBatteryOptimizations, requestIgnoreBatteryOptimizations } from "../native/battery-optimization"; import { getInstalledEvenHubAppById, @@ -2207,7 +2207,7 @@ class DashboardController { provider, language: choice ? choice.language : getVoiceTranscriptionLanguage(provider), elevenLabsApiKey: elevenLabsApiKeySetting.get(), - openAiApiKey: openAiApiKeySetting.get(), + openAiApiKey: openAiApiKeySetting.get(), openAiTranscriptionModel: openAiTranscriptionModelSetting.get(), sonioxApiKey: sonioxApiKeySetting.get(), whisperServerUrl: whisperServerUrlSetting.get(), whisperServerModel: whisperServerModelSetting.get(), diff --git a/app/g2/ios-preview-controller.ts b/app/g2/ios-preview-controller.ts index 036ec95c..8cb44b26 100644 --- a/app/g2/ios-preview-controller.ts +++ b/app/g2/ios-preview-controller.ts @@ -52,7 +52,7 @@ import { mirrorToPanelFraction, phonePreviewCrop } from './phone-preview-crop' import { onAnySettingChanged, previewColorSetting, lockScreenEnabledSetting, getBrightnessPreferences, getStringSettingById, nightscoutSiteUrlSetting, nightscoutApiTokenSetting, voiceProviderSetting, elevenLabsApiKeySetting, - openAiApiKeySetting, sonioxApiKeySetting, whisperServerApiKeySetting, getVoiceTranscriptionLanguage, + openAiApiKeySetting, openAiTranscriptionModelSetting, sonioxApiKeySetting, whisperServerApiKeySetting, getVoiceTranscriptionLanguage, whisperServerModelSetting, whisperServerUrlSetting, ringConnectionModeSetting, saveTranscribeAudioSetting } from '../ui/dashboard-settings' import { transcriptionKey, type VoiceProviderSelection } from '../native/cloud-stt-provider' import type { PhoneGesture } from '../phone-ui/phone-gestures' @@ -1205,7 +1205,7 @@ export class IosPreviewController { const choice = [...this.continuousClients.values()].find(options => options.transcription)?.transcription const provider = choice?.provider ?? voiceProviderSetting.get() return { provider, language: choice ? choice.language : getVoiceTranscriptionLanguage(provider), - elevenLabsApiKey: elevenLabsApiKeySetting.get(), openAiApiKey: openAiApiKeySetting.get(), sonioxApiKey: sonioxApiKeySetting.get(), + elevenLabsApiKey: elevenLabsApiKeySetting.get(), openAiApiKey: openAiApiKeySetting.get(), openAiTranscriptionModel: openAiTranscriptionModelSetting.get(), sonioxApiKey: sonioxApiKeySetting.get(), whisperServerUrl: whisperServerUrlSetting.get(), whisperServerModel: whisperServerModelSetting.get(), whisperServerApiKey: whisperServerApiKeySetting.get() } } diff --git a/app/native/cloud-stt-provider.ts b/app/native/cloud-stt-provider.ts index 51bde438..4dad2d70 100644 --- a/app/native/cloud-stt-provider.ts +++ b/app/native/cloud-stt-provider.ts @@ -1,6 +1,6 @@ import type { CloudSttClient, CloudSttOptions } from "./cloud-stt"; import { ElevenLabsSttClient } from "./elevenlabs-stt"; -import { OpenAiRealtimeSttClient } from "./openai-stt"; +import { DEFAULT_OPENAI_STT_MODEL, OpenAiRealtimeSttClient, type OpenAiSttModel } from "./openai-stt"; import { ReconnectingSttClient } from "./reconnecting-stt"; import { SonioxSttClient } from "./soniox-stt"; import type { VoiceProviderKind } from "./voice-control"; @@ -19,6 +19,7 @@ export type VoiceProviderSelection = { language?: string; elevenLabsApiKey: string; openAiApiKey: string; + openAiTranscriptionModel?: OpenAiSttModel; sonioxApiKey: string; /** Self-hosted Whisper ("whisper-server"); the key is optional there. */ whisperServerUrl?: string; @@ -52,7 +53,7 @@ const CLOUD_PROVIDERS: Partial> = { whisper: { name: "OpenAI", apiKey: (selection) => selection.openAiApiKey, - create: (options) => new OpenAiRealtimeSttClient(options), + create: (options, selection) => new OpenAiRealtimeSttClient({ ...options, model: selection.openAiTranscriptionModel }), }, "whisper-server": { name: "Whisper server", @@ -76,9 +77,10 @@ function isConfigured(provider: CloudProvider, selection: VoiceProviderSelection * What a capture transcribes with: the provider and the language it is told * to expect. A running capture is shared only by holders asking for the same. */ -export function transcriptionKey(selection: Pick): string { +export function transcriptionKey(selection: Pick): string { const fallback = selection.provider === "onboard-whisper" ? "en" : ""; - return `${selection.provider}:${(selection.language ?? fallback).trim().toLowerCase()}`; + const model = selection.provider === "whisper" ? `:${selection.openAiTranscriptionModel ?? DEFAULT_OPENAI_STT_MODEL}` : ""; + return `${selection.provider}:${(selection.language ?? fallback).trim().toLowerCase()}${model}`; } /** Whether createCloudSttClient would return a cloud client for this selection. */ diff --git a/app/native/openai-stt.ts b/app/native/openai-stt.ts index 24f109a1..d909cfc5 100644 --- a/app/native/openai-stt.ts +++ b/app/native/openai-stt.ts @@ -3,7 +3,7 @@ import { openSocket, type SocketConnection } from "./socket"; /** * OpenAI realtime speech-to-text over WebSocket (a transcription-only session - * on the Realtime API, with the streaming gpt-realtime-whisper model). + * on the Realtime API, with a configurable transcription model). * https://developers.openai.com/api/docs/guides/realtime-transcription * * Like the ElevenLabs client, push-to-talk defines the utterance boundary: @@ -16,10 +16,11 @@ import { openSocket, type SocketConnection } from "./socket"; */ const WS_URL = "wss://api.openai.com/v1/realtime?intent=transcription"; -const MODEL_ID = "gpt-realtime-whisper"; +export const DEFAULT_OPENAI_STT_MODEL = "gpt-realtime-whisper"; +export type OpenAiSttModel = "gpt-realtime-whisper" | "gpt-4o-mini-transcribe" | "gpt-4o-transcribe"; const TARGET_SAMPLE_RATE = 24000; -export type OpenAiSttOptions = CloudSttOptions; +export type OpenAiSttOptions = CloudSttOptions & { model?: OpenAiSttModel }; export class OpenAiRealtimeSttClient implements CloudSttClient { private ws: SocketConnection | null = null; @@ -40,7 +41,7 @@ export class OpenAiRealtimeSttClient implements CloudSttClient { onOpen: () => { if (this.closed) return; this.open = true; - this.trySend(JSON.stringify(sessionConfig(this.options.language ?? ""))); + this.trySend(JSON.stringify(sessionConfig(this.options.language ?? "", this.options.model))); for (const chunk of this.pendingChunks.splice(0)) { this.sendChunk(chunk); } @@ -165,7 +166,9 @@ export class OpenAiRealtimeSttClient implements CloudSttClient { this.options.onDisconnected?.(`OpenAI: ${String(message.error?.message)}`); return; } - this.options.onError(`OpenAI: ${String(message.error?.message ?? "unknown error")}`); + this.options.onError(message.error?.code === "model_not_found" + ? `OpenAI: ${this.options.model ?? DEFAULT_OPENAI_STT_MODEL} unavailable. Choose a model in Settings > Voice > OpenAI transcription model.` + : `OpenAI: ${String(message.error?.message ?? "unknown error")}`); return; default: return; @@ -201,7 +204,7 @@ export class OpenAiRealtimeSttClient implements CloudSttClient { } -function sessionConfig(language: string): object { +function sessionConfig(language: string, model: OpenAiSttModel = DEFAULT_OPENAI_STT_MODEL): object { return { type: "session.update", session: { @@ -209,7 +212,7 @@ function sessionConfig(language: string): object { audio: { input: { format: { type: "audio/pcm", rate: TARGET_SAMPLE_RATE }, - transcription: { model: MODEL_ID, ...(language ? { language } : {}) }, + transcription: { model, ...(language ? { language } : {}) }, // Push-to-talk owns the utterance boundary; no server VAD. turn_detection: null, }, diff --git a/app/native/voice-control.ts b/app/native/voice-control.ts index 4c845798..319a8673 100644 --- a/app/native/voice-control.ts +++ b/app/native/voice-control.ts @@ -1,3 +1,4 @@ +import type { OpenAiSttModel } from "./openai-stt"; import { Utils } from "@nativescript/core"; import { type CloudSttTranscriptEvent, CloudSttClient } from "./cloud-stt"; @@ -24,7 +25,7 @@ export type VoiceControlState = { detail: string; }; -// "whisper" (no "onboard-" prefix) is the OpenAI CLOUD provider (gpt-realtime-whisper, +// "whisper" (no "onboard-" prefix) is the OpenAI CLOUD provider (a configurable transcription model, // see openai-stt.ts) -- an unfortunate pre-existing name collision with the model // family, not with "onboard-whisper" below, which is the on-device sherpa-onnx // Whisper backend added alongside "onboard" (Moonshine). Kept as-is rather than @@ -54,6 +55,7 @@ export type PushToTalkOptions = { provider: VoiceProviderKind; elevenLabsApiKey: string; openAiApiKey: string; + openAiTranscriptionModel?: OpenAiSttModel; sonioxApiKey: string; whisperServerUrl?: string; whisperServerModel?: string; diff --git a/app/ui/dashboard-settings.ts b/app/ui/dashboard-settings.ts index e06b0c98..427edf9c 100644 --- a/app/ui/dashboard-settings.ts +++ b/app/ui/dashboard-settings.ts @@ -1,3 +1,4 @@ +import type { OpenAiSttModel } from "../native/openai-stt"; import { normalizeNightscoutThreshold, type NightscoutThresholds } from "../apps/nightscout/nightscout-alerts"; import { GESTURE_DOUBLE_CLICK, InputEvent } from "./gestures"; import { DEFAULT_BRIGHTNESS_CURVE, normalizeBrightnessCurve, brightnessCurveError } from "../g2/brightness-curve"; @@ -1137,6 +1138,15 @@ export const whisperServerModelSetting = new ConfigSettingString({ description: "Model id sent to the Whisper server. Leave empty to use the first model the server lists.", }); +export const openAiTranscriptionModelSetting = new ConfigSettingEnum({ + id: "openai-transcription-model", + label: "OpenAI transcription model", + storageKey: "voice.openAiTranscriptionModel", + defaultValue: "gpt-realtime-whisper", + values: ["gpt-realtime-whisper", "gpt-4o-mini-transcribe", "gpt-4o-transcribe"], + description: "Cloud speech-to-text for Translate, Transcribe and dictation. Choose a model available to your OpenAI project. This does not change the Assistant translation model. The transcribe models return text after each committed audio segment.", +}); + export const voiceLanguageSetting = new ConfigSettingEnum({ id: "voice-language", label: "Transcription language", diff --git a/app/ui/dashboard/settings-menus.ts b/app/ui/dashboard/settings-menus.ts index b7bff111..95ae991f 100644 --- a/app/ui/dashboard/settings-menus.ts +++ b/app/ui/dashboard/settings-menus.ts @@ -63,6 +63,7 @@ import { mapboxApiKeySetting, mirrorTouchSetting, openAiApiKeySetting, + openAiTranscriptionModelSetting, previewColorSetting, previewScalingSetting, phoneRotationSetting, @@ -530,6 +531,7 @@ function asrModelStatusText(id: AsrModelId): string { */ function whisperServerMenuItems(): MenuItem[] { return [ + enumSettingMenuItem(openAiTranscriptionModelSetting), textSettingMenuItem(whisperServerUrlSetting), textSettingMenuItem(whisperServerModelSetting), textSettingMenuItem(whisperServerLanguageSetting), diff --git a/docs/validation-whisper-languages.md b/docs/validation-whisper-languages.md index 00b67ba1..601a2d0a 100644 --- a/docs/validation-whisper-languages.md +++ b/docs/validation-whisper-languages.md @@ -44,3 +44,25 @@ software checks above; recognition accuracy and latency require device testing. - [OpenAI Realtime transcription: language configuration](https://developers.openai.com/api/docs/guides/realtime-transcription) - [sherpa-onnx Whisper configuration and automatic language detection](https://github.com/k2-fsa/sherpa-onnx/blob/master/sherpa-onnx/csrc/offline-whisper-model-config.cc) - [Multilingual base model, pinned revision](https://huggingface.co/csukuangfj/sherpa-onnx-whisper-base/tree/bb53ee204431c90d314c1cc08d28d23e5b7927cc) + +## OpenAI transcription model selection + +Settings > Voice > OpenAI transcription model selects gpt-realtime-whisper, +gpt-4o-mini-transcribe or gpt-4o-transcribe. For a project whose model list lacks +gpt-realtime-whisper but includes gpt-4o-mini-transcribe, select the latter. +The unset default retains gpt-realtime-whisper for existing installations; no +provider, language or Assistant model preference is overwritten. This setting +applies to Translate's speech recognition as well as Transcribe and dictation. +The two transcribe alternatives produce transcript deltas after a committed +audio segment; existing pause segmentation and push-to-talk release supply the +commits. Choosing a model retargets active continuous capture. + +Model access errors name the requested model before the long project error and +point to the setting. Factory/session-wire tests cover model, language, manual +commit and final transcript behavior; settings tests cover persistence and the +Android/iOS option plumbing. No real API inference was performed; actual project +access, recognition and latency remain to be checked on a device. The model list +provided by the operator establishes account visibility, not a successful audio +request. + +Reference: [OpenAI transcription session API](https://developers.openai.com/api/reference/resources/realtime/subresources/transcription_sessions/methods/create). diff --git a/tests/cloud-stt.test.cjs b/tests/cloud-stt.test.cjs index 5ca17214..82c7890b 100644 --- a/tests/cloud-stt.test.cjs +++ b/tests/cloud-stt.test.cjs @@ -242,3 +242,43 @@ test('a spoken-language hint reaches each provider on the wire, and is left out assert.equal(transcription.language, language || undefined); } }); + +test('OpenAI selected transcription models preserve French hints, manual commits and final transcripts', () => { + for (const model of ['gpt-realtime-whisper', 'gpt-4o-mini-transcribe', 'gpt-4o-transcribe']) { + const h = harness('openai', true, { model, language: 'fr' }); + h.open(); + const input = JSON.parse(h.sockets[0].sent[0]).session.audio.input; + assert.equal(input.transcription.model, model); + assert.equal(input.transcription.language, 'fr'); + assert.equal(input.turn_detection, null); + h.client.acceptPcm(pcm(1200, 150)); + h.client.acceptPcm(pcm(0, 1500)); + assert.ok(h.sockets[0].sent.some(value => JSON.parse(value).type === 'input_audio_buffer.commit')); + h.message({ type: 'input_audio_buffer.committed', item_id: 'one' }); + h.message({ type: 'conversation.item.input_audio_transcription.delta', item_id: 'one', delta: 'Bonjour' }); + h.message({ type: 'conversation.item.input_audio_transcription.completed', item_id: 'one', transcript: 'Bonjour à tous.' }); + assert.equal(h.events.at(-1).text, 'Bonjour à tous.'); + assert.equal(h.events.at(-1).isFinal, true); + h.message({ type: 'error', error: { code: 'model_not_found', message: 'Project private-id does not have access' } }); + assert.ok(h.errors[0].includes(model)); + assert.ok(h.errors[0].includes('OpenAI transcription model')); + } +}); + +test('OpenAI model selection reaches the factory and separates shared captures only for OpenAI', () => { + const created = []; + const load = loader({ setTimeout, clearTimeout }, { + './openai-stt': { + DEFAULT_OPENAI_STT_MODEL: 'gpt-realtime-whisper', + OpenAiRealtimeSttClient: class { constructor(options) { created.push(options); } start() {} stop() {} }, + }, + }); + const { createCloudSttClient, transcriptionKey } = load('app/native/cloud-stt-provider.ts'); + const selection = { provider: 'whisper', language: 'fr', openAiApiKey: 'test', openAiTranscriptionModel: 'gpt-4o-mini-transcribe' }; + createCloudSttClient(selection, { onStatus() {}, onError() {}, onTranscript() {} }, () => false).start(); + assert.equal(created[0].model, selection.openAiTranscriptionModel); + assert.equal(created[0].language, 'fr'); + assert.notEqual(transcriptionKey(selection), transcriptionKey({ ...selection, openAiTranscriptionModel: 'gpt-realtime-whisper' })); + assert.equal(transcriptionKey({ provider: 'whisper' }), transcriptionKey({ provider: 'whisper', openAiTranscriptionModel: 'gpt-realtime-whisper' })); + assert.equal(transcriptionKey({ ...selection, provider: 'soniox' }), transcriptionKey({ ...selection, provider: 'soniox', openAiTranscriptionModel: 'gpt-4o-transcribe' })); +}); diff --git a/tests/ios-app-host.test.cjs b/tests/ios-app-host.test.cjs index d087c394..d02d2063 100644 --- a/tests/ios-app-host.test.cjs +++ b/tests/ios-app-host.test.cjs @@ -7,7 +7,7 @@ const path = require('node:path'); // The provider settings ios-preview-controller hands the voice bridge. const voiceSettings = { voiceProviderSetting: { get: () => 'onboard' }, elevenLabsApiKeySetting: { get: () => '' }, getVoiceTranscriptionLanguage: () => '', - openAiApiKeySetting: { get: () => '' }, sonioxApiKeySetting: { get: () => '' }, + openAiTranscriptionModelSetting: { get: () => 'gpt-realtime-whisper' }, openAiApiKeySetting: { get: () => '' }, sonioxApiKeySetting: { get: () => '' }, whisperServerUrlSetting: { get: () => '' }, whisperServerModelSetting: { get: () => '' }, whisperServerLanguageSetting: { get: () => '' }, whisperServerApiKeySetting: { get: () => '' } }; function load(file, context) { diff --git a/tests/whisper-language.test.cjs b/tests/whisper-language.test.cjs index 6d2cc567..95435a47 100644 --- a/tests/whisper-language.test.cjs +++ b/tests/whisper-language.test.cjs @@ -62,6 +62,7 @@ test('Android and iOS capture options use the shared language and preserve Trans const s = settings(); s.voiceLanguageSetting.set('fr'); s.voiceProviderSetting.set('whisper'); + s.openAiTranscriptionModelSetting.set('gpt-4o-mini-transcribe'); for (const [file, name, method] of [ ['app/g2/dashboard-controller.ts', 'DashboardController', 'voiceCaptureOptions'], ['app/g2/ios-preview-controller.ts', 'IosPreviewController', 'voiceProvider'], @@ -75,11 +76,13 @@ test('Android and iOS capture options use the shared language and preserve Trans const member = declaration.members.find(n => n.name?.getText(tree) === method); const capture = loadBody(member.body.getText(tree), s, host); assert.equal(capture().language, 'fr'); + assert.equal(capture().openAiTranscriptionModel, 'gpt-4o-mini-transcribe'); host.continuousClients.set('translate', { transcription: { provider: 'soniox', language: 'ja' } }); assert.equal(capture().provider, 'soniox'); assert.equal(capture().language, 'ja'); host.continuousClients.clear(); assert.equal(capture().language, 'fr'); + assert.equal(capture().openAiTranscriptionModel, 'gpt-4o-mini-transcribe'); } }); @@ -129,3 +132,15 @@ test('multilingual downloads and deletion leave an existing English model intact assert.equal(asr.isAsrModelReady('whisper-base'), false); assert.equal(asr.isAsrModelReady('whisper-base-en'), true); }); + +test('OpenAI transcription model preserves legacy default and persists independently of language', () => { + const values = new Map(); + let s = settings(values); + assert.equal(s.openAiTranscriptionModelSetting.get(), 'gpt-realtime-whisper'); + s.openAiTranscriptionModelSetting.set('gpt-4o-mini-transcribe'); + s.voiceLanguageSetting.set('fr'); + s = settings(values); + assert.equal(s.openAiTranscriptionModelSetting.get(), 'gpt-4o-mini-transcribe'); + assert.equal(s.voiceLanguageSetting.get(), 'fr'); + assert.equal(settings(new Map([['voice.openAiTranscriptionModel', 'invalid']])).openAiTranscriptionModelSetting.get(), 'gpt-realtime-whisper'); +});