diff --git a/app/apps/microphones/mic-session.ts b/app/apps/microphones/mic-session.ts index 5139f09b..88aedc2d 100644 --- a/app/apps/microphones/mic-session.ts +++ b/app/apps/microphones/mic-session.ts @@ -64,6 +64,7 @@ import { analyzeLine, searchableMetadata } from "./sentiment"; import { generateSessionInsights } from "./speaker-insights"; import { speakerRegistry, type SpeakerMatch } from "./speakers"; import { postAmbientCard } from "../../ui/shell/ambient-cards"; +import { pcmLevel, rmsToLevel } from "../../util/audio-level"; import { formatRelativeTime } from "../../util/date-util"; import { deviceLanguage, identifyLanguage, translateText } from "./translate"; @@ -1669,31 +1670,12 @@ function wireConfig(config: MicConfig): MicConfig { return { ...config, channelMask: 0b11 }; } -function pcmLevel(pcm: Uint8Array): number { - let sumSquares = 0; - const count = pcm.length / 2; - if (count === 0) return 0; - for (let i = 0; i < count; i++) { - let value = pcm[i * 2]! | (pcm[i * 2 + 1]! << 8); - if (value >= 0x8000) value -= 0x10000; - sumSquares += value * value; - } - return rmsToLevel(Math.sqrt(sumSquares / count) / 32768); -} - function floatLevel(samples: Float32Array): number { let sumSquares = 0; for (let i = 0; i < samples.length; i++) sumSquares += samples[i]! * samples[i]!; return rmsToLevel(Math.sqrt(sumSquares / Math.max(1, samples.length))); } -/** Map RMS in [0,1] to a meter position with a log-ish curve (mic peaks ~0.1 FS). */ -function rmsToLevel(rms: number): number { - if (rms <= 0) return 0; - const db = 20 * Math.log10(rms); - return Math.max(0, Math.min(1, (db + 60) / 60)); -} - function floatToPcm16(samples: Float32Array): Uint8Array { const out = new Uint8Array(samples.length * 2); for (let i = 0; i < samples.length; i++) { diff --git a/app/apps/transcribe/transcribe-app.ts b/app/apps/transcribe/transcribe-app.ts index 71c1b4c6..900af357 100644 --- a/app/apps/transcribe/transcribe-app.ts +++ b/app/apps/transcribe/transcribe-app.ts @@ -78,7 +78,7 @@ export function createTranscribeAppWindow(options: TranscribeAppOptions): InProc }); // Wire async state changes to this window's renderer, rather than the // controller action from which the window-specific actions were derived. - layer.start(app.requestRender); + layer.start(app.requestRender, () => shell.isWindowVisible(TRANSCRIBE_WINDOW_ID)); options.startContinuousVoiceCapture({ recordAs: `transcribe-${stamp}` }); shell.setTrayIcon(TRAY_ICON_ID, MIC_ICON); return app; diff --git a/app/apps/transcribe/transcribe.ts b/app/apps/transcribe/transcribe.ts index 251b1807..2fd35a6c 100644 --- a/app/apps/transcribe/transcribe.ts +++ b/app/apps/transcribe/transcribe.ts @@ -1,6 +1,7 @@ import { GrayImage } from "../../graphics/image"; import { getDefaultSmallFont } from "../../graphics/ui-fonts"; import { lineStep } from "../../ui/metrics"; +import { MicLevelMeter, paintListeningHeader } from "../../ui/listening-header"; import { writeTextToDownloads } from "../../native/file-access"; import { voiceControlBridge } from "../../native/voice-control"; import { Layer, type LayerContext } from "../../ui/layers"; @@ -18,6 +19,7 @@ export class TranscribeLayer implements Layer { private detail = ""; private readonly transcript = new TranscriptModel(); private readonly scroll = new TranscriptScroll(); + private readonly meter = new MicLevelMeter(); private saveNotice = ""; private unsubscribeTranscript: (() => void) | null = null; private unsubscribePause: (() => void) | null = null; @@ -26,7 +28,9 @@ export class TranscribeLayer implements Layer { /** `stamp` names the saved transcript, matching the session's audio recording. */ constructor(private readonly stamp: string = transcriptTimestamp()) {} - start(requestRender: () => void): void { + /** `isVisible` says whether the window is on screen (the level meter repaints only then). */ + start(requestRender: () => void, isVisible: () => boolean): void { + this.meter.start(requestRender, isVisible); this.unsubscribeTranscript = voiceControlBridge.onTranscript((event) => { this.transcript.accept(event); requestRender(); @@ -48,11 +52,8 @@ export class TranscribeLayer implements Layer { const text = this.transcript.text || (this.listening ? "Listening..." : this.detail); const wrapped = wrapTranscribeText((line) => font.measureText(line), text, width - 64); - image.drawText(font, 24, 20, "Transcribe", 200); - image.drawText(font, 24, 40, this.saveNotice || this.status, 110); - + const bodyTop = paintListeningHeader(image, font, "Transcribe", this.saveNotice || this.status, this.listening ? this.meter : null); const step = lineStep(font) + 2; - const bodyTop = 62; const bodyLines = Math.max(1, Math.floor((height - 6 - bodyTop - font.lineHeight) / step) + 1); const firstLine = this.scroll.layout(wrapped.length, bodyLines); for (let index = firstLine; index < Math.min(wrapped.length, firstLine + bodyLines); index++) { @@ -82,6 +83,7 @@ export class TranscribeLayer implements Layer { } onRemoved(): void { + this.meter.stop(); this.unsubscribePause?.(); this.unsubscribePause = null; this.unsubscribeTranscript?.(); diff --git a/app/apps/translate/translate-app.ts b/app/apps/translate/translate-app.ts index 7cbd57e3..1cb11a4a 100644 --- a/app/apps/translate/translate-app.ts +++ b/app/apps/translate/translate-app.ts @@ -1,6 +1,12 @@ import { TranslateLayer } from "./translate"; import { resolveTranslationModel } from "./translate-llm"; -import { resolveTranslateProvider, translateProviderSetting, translateShowOriginalSetting } from "./translate-settings"; +import { + resolveTranslateLanguage, + resolveTranslateProvider, + translateLanguageSetting, + translateProviderSetting, + translateShowOriginalSetting, +} from "./translate-settings"; import { MIC_ICON } from "../transcribe/transcribe-app"; import { transcriptTimestamp } from "../transcribe/transcribe"; import { type ContinuousCaptureOptions } from "../../ui/layers"; @@ -38,8 +44,9 @@ export type TranslateAppOptions = InProcessAppOptions & { * provider (Translate settings), which a shared capture switches to. */ export function createTranslateAppWindow(options: TranslateAppOptions): InProcessWindow { - // Checks the setup and (re)starts capture with the chosen provider: at - // launch, when the provider setting changes, and on a tap after a problem. + // Checks the setup and (re)starts capture with the chosen provider and + // language: at launch, when either setting changes, and on a tap after a + // problem. const startCapture = () => { const provider = resolveTranslateProvider(); const problem = !provider ? NO_PROVIDER_PROBLEM : !resolveTranslationModel() ? NO_MODEL_PROBLEM : null; @@ -49,7 +56,7 @@ export function createTranslateAppWindow(options: TranslateAppOptions): InProces shell.setTrayIcon(TRAY_ICON_ID, null); return; } - options.startContinuousVoiceCapture({ multilingualProvider: provider }); + options.startContinuousVoiceCapture({ transcription: { provider, language: resolveTranslateLanguage() } }); shell.setTrayIcon(TRAY_ICON_ID, MIC_ICON); }; const layer = new TranslateLayer(transcriptTimestamp(), startCapture); @@ -75,6 +82,7 @@ export function createTranslateAppWindow(options: TranslateAppOptions): InProces onSelect: (ctx) => { ctx.stack.pop(); openSettingsSubMenu(ctx, "Translate settings", [ + enumSettingMenuItem(translateLanguageSetting, { onChange: startCapture }), enumSettingMenuItem(translateProviderSetting, { onChange: startCapture }), toggleSettingMenuItem(translateShowOriginalSetting), ]); @@ -93,7 +101,7 @@ export function createTranslateAppWindow(options: TranslateAppOptions): InProces }); // Wire async state changes to this window's renderer, rather than the // controller action from which the window-specific actions were derived. - layer.start(app.requestRender); + layer.start(app.requestRender, () => shell.isWindowVisible(TRANSLATE_WINDOW_ID)); startCapture(); return app; } diff --git a/app/apps/translate/translate-llm.ts b/app/apps/translate/translate-llm.ts index 26021899..70883340 100644 --- a/app/apps/translate/translate-llm.ts +++ b/app/apps/translate/translate-llm.ts @@ -29,6 +29,8 @@ export function resolveTranslationModel(): ResolvedAssistantModel | null { export type TranslationRequest = { speech: string; context: TranslationTurn[]; + /** English name of the spoken language when the user set one, else "". */ + language: string; onProgress: (textSoFar: string) => void; onDone: (translation: string) => void; onError: (message: string) => void; @@ -41,7 +43,7 @@ export function streamTranslation(model: ResolvedAssistantModel, request: Transl system: TRANSLATE_SYSTEM_PROMPT, effort: model.effort, maxTokens: 1024, - messages: [{ role: "user", content: buildTranslateUserMessage(request.speech, request.context) }], + messages: [{ role: "user", content: buildTranslateUserMessage(request.speech, request.context, request.language) }], onTextDelta: (_delta, textSoFar) => request.onProgress(cleanTranslation(textSoFar)), onDone: (result) => { if (result.stopReason === "refusal") request.onError("The model declined to translate this."); diff --git a/app/apps/translate/translate-settings.ts b/app/apps/translate/translate-settings.ts index c75d1ef3..802419e1 100644 --- a/app/apps/translate/translate-settings.ts +++ b/app/apps/translate/translate-settings.ts @@ -47,6 +47,58 @@ export const translateProviderSetting = new ConfigSettingEnum "Speech-to-text for Translate. Only cloud providers recognize any language (the on-device engines don't), so each needs its API key in Settings > API Keys, or a Whisper server URL in Settings > Voice. Auto uses the main Transcription Provider when it is one of these.", }); +/** + * Spoken languages Translate can be told to expect, by ISO 639-1 code (the + * form every multilingual provider accepts), in picker order. + */ +const SPOKEN_LANGUAGES = { + ar: "Arabic", + zh: "Chinese", + nl: "Dutch", + fr: "French", + de: "German", + el: "Greek", + he: "Hebrew", + hi: "Hindi", + id: "Indonesian", + it: "Italian", + ja: "Japanese", + ko: "Korean", + pl: "Polish", + pt: "Portuguese", + ru: "Russian", + es: "Spanish", + sv: "Swedish", + th: "Thai", + tr: "Turkish", + uk: "Ukrainian", + vi: "Vietnamese", +} as const; + +export type SpokenLanguage = keyof typeof SPOKEN_LANGUAGES; + +export const translateLanguageSetting = new ConfigSettingEnum<"auto" | SpokenLanguage>({ + id: "translate-language", + label: "Spoken language", + storageKey: "translate.language", + defaultValue: "auto", + values: ["auto", ...(Object.keys(SPOKEN_LANGUAGES) as SpokenLanguage[])], + formatValue: (value) => (value === "auto" ? "Auto-detect" : SPOKEN_LANGUAGES[value]), + description: + "The language being spoken. Auto-detect can mistake it, especially in short or noisy speech, and then transcribes nonsense in the wrong language; naming it makes transcription much more reliable. ElevenLabs then transcribes only that language.", +}); + +/** The language to tell the transcription provider (ISO 639-1), or "" to let it detect one. */ +export function resolveTranslateLanguage(): SpokenLanguage | "" { + const value = translateLanguageSetting.get(); + return value === "auto" ? "" : value; +} + +/** The English name of a spoken language, for the translation prompt. */ +export function spokenLanguageName(language: SpokenLanguage): string { + return SPOKEN_LANGUAGES[language]; +} + export const translateShowOriginalSetting = new ConfigSettingBoolean({ id: "translate-show-original", label: "Show original", diff --git a/app/apps/translate/translate.ts b/app/apps/translate/translate.ts index 252da0d4..e747be2d 100644 --- a/app/apps/translate/translate.ts +++ b/app/apps/translate/translate.ts @@ -1,6 +1,7 @@ import { GrayImage } from "../../graphics/image"; import { getDefaultSmallFont } from "../../graphics/ui-fonts"; import { lineStep } from "../../ui/metrics"; +import { MicLevelMeter, paintListeningHeader } from "../../ui/listening-header"; import { writeTextToDownloads } from "../../native/file-access"; import { voiceControlBridge } from "../../native/voice-control"; import { Layer, type LayerContext } from "../../ui/layers"; @@ -9,15 +10,16 @@ import { type LlmStreamHandle } from "../../assistant/llm-protocol"; import { TranscriptScroll, wrapTranscribeText } from "../transcribe/transcript-model"; import { TranslationModel, type TranslationJob, type TranslationLine } from "./translation-model"; import { resolveTranslationModel, streamTranslation } from "./translate-llm"; -import { translateShowOriginalSetting } from "./translate-settings"; +import { resolveTranslateLanguage, spokenLanguageName, translateShowOriginalSetting } from "./translate-settings"; const NO_MODEL_NOTICE = "No language model is set up."; /** * Live translation view. Continuous mic capture is owned by the app wrapper - * (createTranslateAppWindow); this layer feeds finalized speech to the - * assistant's model one stretch at a time, shows the English as it streams - * in, and saves the session to Downloads from the app menu. + * (createTranslateAppWindow); this layer feeds speech to the assistant's + * model one block at a time (provisionally while a block is still being + * spoken), shows the English as it streams in, and saves the session to + * Downloads from the app menu. */ export class TranslateLayer implements Layer { private status = "Starting microphone..."; @@ -30,7 +32,8 @@ export class TranslateLayer implements Layer { private problem: string | null = null; private readonly model = new TranslationModel(); private readonly scroll = new TranscriptScroll(); - private activeJob: TranslationJob | null = null; + private readonly meter = new MicLevelMeter(); + // The request in flight (the model tracks which job it is). private activeHandle: LlmStreamHandle | null = null; private requestRender: () => void = () => {}; private removed = false; @@ -40,10 +43,16 @@ export class TranslateLayer implements Layer { /** `stamp` names the saved transcript; `retry` re-runs the app's setup check. */ constructor(private readonly stamp: string, private readonly retry: () => void) {} - start(requestRender: () => void): void { + /** `isVisible` says whether the window is on screen (the level meter repaints only then). */ + start(requestRender: () => void, isVisible: () => boolean): void { this.requestRender = requestRender; + this.meter.start(requestRender, isVisible); this.unsubscribeTranscript = voiceControlBridge.onTranscript((event) => { - this.model.accept(event); + if (this.model.accept(event)) { + // The request in flight translates text the transcript has replaced. + this.activeHandle?.cancel(); + this.activeHandle = null; + } this.pump(); requestRender(); }); @@ -68,9 +77,9 @@ export class TranslateLayer implements Layer { const measure = (line: string) => font.measureText(line); const maxWidth = width - 64; - image.drawText(font, 24, 20, "Translate", 200); const status = this.problem ? "Setup needed" : this.translationError || this.status; - image.drawText(font, 24, 40, this.saveNotice || status, 110); + const meter = this.listening && !this.problem ? this.meter : null; + const bodyTop = paintListeningHeader(image, font, "Translate", this.saveNotice || status, meter); let lines: TranslationLine[]; if (this.problem) { @@ -84,7 +93,6 @@ export class TranslateLayer implements Layer { } const step = lineStep(font) + 2; - const bodyTop = 62; const bodyLines = Math.max(1, Math.floor((height - 6 - bodyTop - font.lineHeight) / step) + 1); const firstLine = this.scroll.layout(lines.length, bodyLines); for (let index = firstLine; index < Math.min(lines.length, firstLine + bodyLines); index++) { @@ -119,7 +127,7 @@ export class TranslateLayer implements Layer { onRemoved(): void { this.removed = true; - this.activeJob = null; + this.meter.stop(); this.activeHandle?.cancel(); this.activeHandle = null; this.unsubscribeTranscript?.(); @@ -139,7 +147,10 @@ export class TranslateLayer implements Layer { this.saveNotice = path ? `Saved ${filename}` : "Save failed (check file access)."; } - /** Start translating the next waiting stretch of speech, unless one is in flight. */ + /** + * Start the next translation request (a final block, else the live block's + * provisional one), unless one is in flight. + */ private pump(): void { if (this.removed) return; const job = this.model.nextJob(); @@ -147,36 +158,34 @@ export class TranslateLayer implements Layer { const llm = resolveTranslationModel(); if (!llm) { // The model went away mid-session (a key was cleared): keep the original. - this.model.fail(job.segment); + this.model.fail(job); this.translationError = NO_MODEL_NOTICE; this.pump(); return; } - this.activeJob = job; + const language = resolveTranslateLanguage(); const handle = streamTranslation(llm, { - speech: job.segment.source, + speech: job.source, context: job.context, + language: language ? spokenLanguageName(language) : "", onProgress: (textSoFar) => { - if (this.activeJob !== job) return; - this.model.progress(job.segment, textSoFar); - this.requestRender(); + if (this.model.progress(job, textSoFar)) this.requestRender(); }, onDone: (translation) => this.settle(job, () => { - this.model.complete(job.segment, translation); + this.model.complete(job, translation); this.translationError = ""; }), onError: (message) => this.settle(job, () => { - this.model.fail(job.segment); + this.model.fail(job); this.translationError = `Translation failed: ${message}`; }), }); // A provider that fails synchronously has already settled (and moved on). - if (this.activeJob === job) this.activeHandle = handle; + if (this.model.isActive(job)) this.activeHandle = handle; } private settle(job: TranslationJob, apply: () => void): void { - if (this.activeJob !== job) return; - this.activeJob = null; + if (!this.model.isActive(job)) return; this.activeHandle = null; apply(); this.pump(); diff --git a/app/apps/translate/translation-model.ts b/app/apps/translate/translation-model.ts index ea0e9204..2a1fc9da 100644 --- a/app/apps/translate/translation-model.ts +++ b/app/apps/translate/translation-model.ts @@ -2,21 +2,32 @@ import { wrapTranscribeText } from "../transcribe/transcript-model"; type TranscriptEvent = { text: string; isFinal: boolean }; -export type TranslationState = "pending" | "translating" | "done" | "failed"; - -/** One stretch of finalized speech and its English translation. */ +/** + * One block of speech (up to a pause) and its English translation. The last + * block may still be live, its transcript revising; it is translated + * provisionally as it grows, then again once final if its text changed. + */ export type TranslationSegment = { source: string; - /** The translation so far (streamed while translating). */ + /** Committed by the transcriber; a live block's source still revises. */ + final: boolean; + /** The latest finished translation, of `translatedSource`. */ translation: string; - state: TranslationState; + /** What `translation` translates: null until one finishes, behind `source` while provisional. */ + translatedSource: string | null; + /** The text streamed so far by the request in flight for this block, else null. */ + streaming: string | null; + /** A source whose translation failed; not retried until the source changes. */ + failedSource: string | null; }; export type TranslationTurn = { source: string; translation: string }; -/** A segment handed to the model, with the finished turns before it. */ +/** A request: a block's source as sent, with the finished turns before it. */ export type TranslationJob = { segment: TranslationSegment; + /** The source sent (a live block's moves on while the request is in flight). */ + source: string; /** Earlier translated turns, oldest first. */ context: TranslationTurn[]; }; @@ -27,59 +38,123 @@ export type TranslationLine = { text: string; gray: number }; // over between requests; bounded so a long session's requests stay small. const CONTEXT_TURNS = 4; const CONTEXT_CHARS = 800; +// A live block is translated again once its text has changed length by this +// much since the last request, so a trickle of revisions (punctuation, one +// corrected word) doesn't spend a request each. A final is always redone. +const PROVISIONAL_MIN_CHANGE = 3; const TRANSLATION_GRAY = 230; +const PROVISIONAL_GRAY = 170; const ORIGINAL_GRAY = 120; const LIVE_GRAY = 90; +function newSegment(source: string, final: boolean): TranslationSegment { + return { source, final, translation: "", translatedSource: null, streaming: null, failedSource: null }; +} + +/** Whether a block's current source still needs a translation request. */ +function wantsTranslation(segment: TranslationSegment): boolean { + const { source, translatedSource } = segment; + if (!source || source === translatedSource || source === segment.failedSource) return false; + return segment.final || translatedSource === null || Math.abs(source.length - translatedSource.length) >= PROVISIONAL_MIN_CHANGE; +} + /** - * The Translate app's conversation: finalized speech segments with their - * translations, plus the live partial of the utterance in progress. + * The Translate app's conversation: blocks of speech with their translations. * Translation is serial (one request at a time, in order, so each can carry - * the turns before it as context); finals that arrive while a request is in - * flight merge into one pending segment, so a slow model batches the backlog - * instead of falling further behind one request per pause. + * the turns before it as context): final blocks first, then the live one. + * Finals that arrive while a request is in flight merge into one waiting + * block, so a slow model batches the backlog instead of falling further + * behind one request per pause. */ export class TranslationModel { readonly segments: TranslationSegment[] = []; - /** The utterance in progress (replace semantics), not yet final. */ - live = ""; + private active: TranslationJob | null = null; - accept(event: TranscriptEvent): void { + /** + * Take a transcript event (replace semantics within the live block). + * Returns true when it made the request in flight moot (its block was + * finalized as other text, or dropped); that request is abandoned here, and + * the caller should cancel it. + */ + accept(event: TranscriptEvent): boolean { const text = event.text.trim(); - if (!event.isFinal) { - this.live = text; - return; - } - const finalText = text || this.live; - this.live = ""; - if (!finalText) return; const last = this.segments[this.segments.length - 1]; - if (last?.state === "pending") last.source = `${last.source} ${finalText}`; - else this.segments.push({ source: finalText, translation: "", state: "pending" }); + const live = last && !last.final ? last : null; + if (!event.isFinal) { + if (live) live.source = text; + else if (text) this.segments.push(newSegment(text, false)); + return false; + } + const finalText = text || live?.source || ""; + if (!live) { + if (finalText && !this.mergeIntoWaiting(finalText)) this.segments.push(newSegment(finalText, true)); + return false; + } + const moot = this.active?.segment === live && this.active.source !== finalText; + if (moot) this.abandonActive(); + if (!finalText || (this.active?.segment !== live && live.translatedSource === null && this.mergeIntoWaiting(finalText, live))) { + this.segments.pop(); + return moot; + } + live.source = finalText; + live.final = true; + return moot; } - /** Take the next segment to translate, or null while one is in flight or none waits. */ + /** Take the next request to make, or null while one is in flight or nothing waits. */ nextJob(): TranslationJob | null { - if (this.segments.some((segment) => segment.state === "translating")) return null; - const segment = this.segments.find((candidate) => candidate.state === "pending"); + if (this.active) return null; + const segment = this.segments.find((candidate) => wantsTranslation(candidate)); if (!segment) return null; - segment.state = "translating"; - return { segment, context: this.contextBefore(segment) }; + segment.streaming = ""; + this.active = { segment, source: segment.source, context: this.contextBefore(segment) }; + return this.active; } - progress(segment: TranslationSegment, textSoFar: string): void { - if (segment.state === "translating") segment.translation = textSoFar; + isActive(job: TranslationJob): boolean { + return this.active === job; } - complete(segment: TranslationSegment, translation: string): void { - segment.translation = translation; - segment.state = "done"; + /** Streamed text for the request in flight; false when `job` isn't it any more. */ + progress(job: TranslationJob, textSoFar: string): boolean { + if (this.active !== job) return false; + job.segment.streaming = textSoFar; + return true; } - fail(segment: TranslationSegment): void { - segment.translation = ""; - segment.state = "failed"; + complete(job: TranslationJob, translation: string): void { + if (this.active !== job) return; + this.active = null; + job.segment.streaming = null; + job.segment.translation = translation; + job.segment.translatedSource = job.source; + } + + fail(job: TranslationJob): void { + if (this.active !== job) return; + this.active = null; + job.segment.streaming = null; + job.segment.failedSource = job.source; + } + + private abandonActive(): void { + if (this.active) this.active.segment.streaming = null; + this.active = null; + } + + /** + * Append a final to the block before `before` (default: the end) when that + * block is final but still waiting for its first request. Returns whether + * it did. + */ + private mergeIntoWaiting(finalText: string, before?: TranslationSegment): boolean { + const index = (before ? this.segments.indexOf(before) : this.segments.length) - 1; + const waiting = this.segments[index]; + if (!waiting?.final || waiting.translatedSource !== null || waiting.failedSource !== null) return false; + if (this.active?.segment === waiting) return false; + waiting.source = `${waiting.source} ${finalText}`; + return true; } private contextBefore(segment: TranslationSegment): TranslationTurn[] { @@ -87,7 +162,7 @@ export class TranslationModel { let chars = 0; for (let index = this.segments.indexOf(segment) - 1; index >= 0 && turns.length < CONTEXT_TURNS; index--) { const earlier = this.segments[index]!; - if (earlier.state !== "done" || !earlier.translation) continue; + if (!earlier.final || earlier.translatedSource !== earlier.source || !earlier.translation) continue; chars += earlier.source.length + earlier.translation.length; if (turns.length && chars > CONTEXT_CHARS) break; turns.unshift({ source: earlier.source, translation: earlier.translation }); @@ -96,9 +171,11 @@ export class TranslationModel { } /** - * Display lines: each segment's translation, bright; its original, dim, - * above it when `showOriginal` is on and in its place until a translation - * arrives (or when it failed); then the live partial, dimmer. + * Display lines: each block's translation, bright once settled (final, and + * of its final text) and dimmer while provisional; its original, dim, above + * it when `showOriginal` is on and in its place until a translation + * arrives. A redo streams in over the translation it replaces only once it + * is as long, so the text doesn't shrink and regrow. */ lines(measure: (text: string) => number, maxWidth: number, showOriginal: boolean): TranslationLine[] { const lines: TranslationLine[] = []; @@ -106,21 +183,25 @@ export class TranslationModel { for (const line of wrapTranscribeText(measure, text, maxWidth)) lines.push({ text: line, gray }); }; for (const segment of this.segments) { - const translation = segment.translation.trim(); - // A finished, empty translation is noise the model chose to drop. - if (showOriginal || (!translation && segment.state !== "done")) push(segment.source, ORIGINAL_GRAY); - if (translation) push(translation, TRANSLATION_GRAY); + if (!segment.source) continue; + const settled = segment.final && segment.translatedSource === segment.source; + const finished = segment.translation.trim(); + const streaming = segment.streaming?.trim() ?? ""; + const translation = streaming.length >= finished.length ? streaming : finished; + // A settled, empty translation is noise the model chose to drop. + if (showOriginal || (!translation && !settled)) push(segment.source, segment.final ? ORIGINAL_GRAY : LIVE_GRAY); + if (translation) push(translation, settled ? TRANSLATION_GRAY : PROVISIONAL_GRAY); } - if (this.live) push(this.live, LIVE_GRAY); return lines; } - /** Plain-text transcript for saving: each original, then its translation. */ + /** Plain-text transcript for saving: each final original, then its translation. */ savedText(): string { return this.segments + .filter((segment) => segment.final) .map((segment) => { const translation = segment.translation.trim(); - if (segment.state === "failed") return `${segment.source}\n→ (not translated)`; + if (!translation && segment.failedSource === segment.source) return `${segment.source}\n→ (not translated)`; if (!translation || translation === segment.source.trim()) return segment.source; return `${segment.source}\n→ ${translation}`; }) diff --git a/app/g2/dashboard-controller.ts b/app/g2/dashboard-controller.ts index db83e5b2..a04781f6 100644 --- a/app/g2/dashboard-controller.ts +++ b/app/g2/dashboard-controller.ts @@ -74,9 +74,9 @@ import { type AppContext, type AppDefinition, type AppLaunchParams, type TextEdi import { type InProcessAppOptions, type InProcessWindow } from "../ui/shell/in-process-window"; import { loadPersistedOpenApps, savePersistedOpenApps } from "../ui/shell/open-apps-persistence"; import { appViewportRect, isOnSwitcherEdge, sidebarStripVisible, type ScreenRect, type WindowHeightMode } from "../ui/shell/geometry"; -import { type ContinuousCaptureOptions, type LayerActions, type TextSettingsEditToggle } from "../ui/layers"; +import { type ContinuousCaptureOptions, type LayerActions, type TextSettingsEditToggle, type TranscriptionChoice } from "../ui/layers"; import { type KeyboardInputSession } from "../ui/shell/keyboard-input"; -import { appSwitcherPositionSetting, statusBarPositionSetting, statusBarVisibilitySetting, windowBorderSetting, assistantAllowProactiveSetting, assistantBackendSetting, assistantBridgeHostSetting, assistantBridgePortSetting, assistantBridgeTokenSetting, getBrightnessPreferences, displayLayoutSetting, navigateDisplayLayoutSetting, terminalDisplayLayoutSetting, elevenLabsApiKeySetting, getStringSettingById, openAiApiKeySetting, nightscoutApiTokenSetting, firmwareDebugFlagsSetting, whisperServerApiKeySetting, whisperServerLanguageSetting, whisperServerModelSetting, whisperServerUrlSetting, lockScreenEnabledSetting, nightscoutSiteUrlSetting, onAnySettingChanged, previewColorSetting, ringConnectionModeSetting, saveVoiceRecordingsSetting, sonioxApiKeySetting, screenTimeoutSetting, screenTimeoutSettingToMs, suspendEvenHubWhenScreenOffSetting, voiceProviderSetting, wakeWordActionSetting, type ConfigSettingString, type VoiceProvider } from "../ui/dashboard-settings"; +import { appSwitcherPositionSetting, statusBarPositionSetting, statusBarVisibilitySetting, windowBorderSetting, assistantAllowProactiveSetting, assistantBackendSetting, assistantBridgeHostSetting, assistantBridgePortSetting, assistantBridgeTokenSetting, getBrightnessPreferences, displayLayoutSetting, navigateDisplayLayoutSetting, terminalDisplayLayoutSetting, elevenLabsApiKeySetting, getStringSettingById, openAiApiKeySetting, nightscoutApiTokenSetting, firmwareDebugFlagsSetting, whisperServerApiKeySetting, whisperServerLanguageSetting, whisperServerModelSetting, whisperServerUrlSetting, lockScreenEnabledSetting, nightscoutSiteUrlSetting, onAnySettingChanged, previewColorSetting, ringConnectionModeSetting, saveVoiceRecordingsSetting, sonioxApiKeySetting, screenTimeoutSetting, screenTimeoutSettingToMs, suspendEvenHubWhenScreenOffSetting, voiceProviderSetting, wakeWordActionSetting, type ConfigSettingString } from "../ui/dashboard-settings"; import { isIgnoringBatteryOptimizations, requestIgnoreBatteryOptimizations } from "../native/battery-optimization"; import { getInstalledEvenHubAppById, @@ -2116,10 +2116,10 @@ class DashboardController { voiceControlBridge.retargetContinuousCapture(this.voiceCaptureOptions(this.communicator)); } - /** The provider a continuous-capture window needs (Translate's multilingual one), if any. */ - private multilingualProvider(): VoiceProvider | undefined { + /** The provider and language a continuous-capture window needs (Translate's), if any. */ + private transcriptionChoice(): TranscriptionChoice | undefined { for (const options of this.continuousClients.values()) { - if (options.multilingualProvider) return options.multilingualProvider; + if (options.transcription) return options.transcription; } return undefined; } @@ -2144,20 +2144,21 @@ class DashboardController { * stands in for the G2 mic. */ private voiceCaptureOptions(communicator: FaceclawCommunicatorBridge | null) { - // A window that needs a multilingual provider sets it for every capture - // while it is open, so push-to-talk still shares its stream. - const multilingual = this.multilingualProvider(); + // A window that needs its own provider and language sets them for every + // capture while it is open, so push-to-talk still shares its stream. + // Otherwise only Whisper server is told a language (its own setting). + const choice = this.transcriptionChoice(); + const provider = choice?.provider ?? voiceProviderSetting.get(); return { communicator: communicator?.getNativeCommunicator() ?? null, usePhoneMic: communicator === null, - provider: multilingual ?? voiceProviderSetting.get(), + provider, + language: choice ? choice.language : provider === "whisper-server" ? whisperServerLanguageSetting.get() : "", elevenLabsApiKey: elevenLabsApiKeySetting.get(), openAiApiKey: openAiApiKeySetting.get(), sonioxApiKey: sonioxApiKeySetting.get(), whisperServerUrl: whisperServerUrlSetting.get(), whisperServerModel: whisperServerModelSetting.get(), - // Whisper detects the language itself when the speech may be in any. - whisperServerLanguage: multilingual ? "" : whisperServerLanguageSetting.get(), whisperServerApiKey: whisperServerApiKeySetting.get(), saveRecording: saveVoiceRecordingsSetting.get(), // "My voice only" (Microphones app): verify command utterances diff --git a/app/g2/ios-preview-controller.ts b/app/g2/ios-preview-controller.ts index 982f0904..b3fe04a5 100644 --- a/app/g2/ios-preview-controller.ts +++ b/app/g2/ios-preview-controller.ts @@ -149,11 +149,12 @@ export class IosPreviewController { private logLines: string[] = [] private logTimer: ReturnType | null = null // Windows that want continuous capture, and what each needs of it (a - // recording, a multilingual provider). They share one capture, which runs + // recording, a provider and language). They share one capture, which runs // while any of them wants it. private readonly continuousClients = new Map() - // transcriptionKey of the running continuous capture's provider; one that no - // longer matches what its windows need is drained and restarted. + // transcriptionKey of the running continuous capture (provider and + // language); one that no longer matches what its windows need is drained + // and restarted. private continuousTranscription = '' // Continuous capture should (re)start once audio is available: a window // asked, the glasses connected or disconnected, or the glasses lock or the @@ -1058,15 +1059,17 @@ export class IosPreviewController { } /** * The transcription provider setting and the keys the cloud ones need. A - * window that needs a multilingual provider (Translate) sets it for every - * capture while it is open, with Whisper left to detect the language. + * window that needs its own provider and language (Translate) sets them for + * every capture while it is open; otherwise only Whisper server is told a + * language (its own setting). */ private voiceProvider(): VoiceProviderSelection { - const multilingual = [...this.continuousClients.values()].find(options => options.multilingualProvider)?.multilingualProvider - return { provider: multilingual ?? voiceProviderSetting.get(), elevenLabsApiKey: elevenLabsApiKeySetting.get(), - openAiApiKey: openAiApiKeySetting.get(), sonioxApiKey: sonioxApiKeySetting.get(), + const choice = [...this.continuousClients.values()].find(options => options.transcription)?.transcription + const provider = choice?.provider ?? voiceProviderSetting.get() + return { provider, language: choice ? choice.language : provider === 'whisper-server' ? whisperServerLanguageSetting.get() : '', + elevenLabsApiKey: elevenLabsApiKeySetting.get(), openAiApiKey: openAiApiKeySetting.get(), sonioxApiKey: sonioxApiKeySetting.get(), whisperServerUrl: whisperServerUrlSetting.get(), whisperServerModel: whisperServerModelSetting.get(), - whisperServerLanguage: multilingual ? '' : whisperServerLanguageSetting.get(), whisperServerApiKey: whisperServerApiKeySetting.get() } + whisperServerApiKey: whisperServerApiKeySetting.get() } } private logBluetooth(message: string): void { const line = `${new Date().toISOString()} [${this.active ? 'foreground' : 'background'}${UIApplication.sharedApplication.protectedDataAvailable ? '' : ',protected-data-unavailable'}] ${message}` diff --git a/app/native/cloud-stt-provider.ts b/app/native/cloud-stt-provider.ts index 2ab6ea08..566cb1d7 100644 --- a/app/native/cloud-stt-provider.ts +++ b/app/native/cloud-stt-provider.ts @@ -12,13 +12,17 @@ import { normalizeWhisperServerUrl, WHISPER_SERVER_FINAL_TIMEOUT_MS, WhisperServ */ export type VoiceProviderSelection = { provider: VoiceProviderKind; + /** + * Spoken language the provider should expect (ISO 639-1); empty lets it + * detect the language. Cloud providers only. + */ + language?: string; elevenLabsApiKey: string; openAiApiKey: string; sonioxApiKey: string; /** Self-hosted Whisper ("whisper-server"); the key is optional there. */ whisperServerUrl?: string; whisperServerModel?: string; - whisperServerLanguage?: string; whisperServerApiKey?: string; }; @@ -58,7 +62,7 @@ const CLOUD_PROVIDERS: Partial> = { create: (options, selection) => new WhisperServerSttClient(options, { baseUrl: selection.whisperServerUrl ?? "", model: selection.whisperServerModel ?? "", - language: selection.whisperServerLanguage ?? "", + language: selection.language ?? "", }), finalTimeoutMs: WHISPER_SERVER_FINAL_TIMEOUT_MS, }, @@ -69,14 +73,11 @@ function isConfigured(provider: CloudProvider, selection: VoiceProviderSelection } /** - * What a capture transcribes with: the provider, and for Whisper server the - * language it is told to expect. A running capture is shared only by holders - * asking for the same. + * What a capture transcribes with: the provider and the language it is told + * to expect. A running capture is shared only by holders asking for the same. */ -export function transcriptionKey(selection: Pick): string { - return selection.provider === "whisper-server" - ? `whisper-server:${(selection.whisperServerLanguage ?? "").trim().toLowerCase()}` - : selection.provider; +export function transcriptionKey(selection: Pick): string { + return `${selection.provider}:${(selection.language ?? "").trim().toLowerCase()}`; } /** Whether createCloudSttClient would return a cloud client for this selection. */ @@ -105,7 +106,7 @@ export function createCloudSttClient( const apiKey = provider.apiKey(selection).trim(); return new ReconnectingSttClient( (clientOptions) => provider.create(clientOptions, selection), - { ...options, apiKey }, + { ...options, apiKey, language: (selection.language ?? "").trim() }, continuous, provider.finalTimeoutMs, ); diff --git a/app/native/cloud-stt.ts b/app/native/cloud-stt.ts index 2600f802..40d4629f 100644 --- a/app/native/cloud-stt.ts +++ b/app/native/cloud-stt.ts @@ -17,6 +17,8 @@ export type CloudSttTranscriptEvent = { export type CloudSttOptions = { apiKey: string; + /** Spoken language to expect (ISO 639-1); empty or absent lets the provider detect it. */ + language?: string; onTranscript: (event: CloudSttTranscriptEvent) => void; onStatus: (status: string) => void; onError: (message: string) => void; diff --git a/app/native/elevenlabs-stt.ts b/app/native/elevenlabs-stt.ts index bc493fb9..8b8ee545 100644 --- a/app/native/elevenlabs-stt.ts +++ b/app/native/elevenlabs-stt.ts @@ -33,7 +33,10 @@ export class ElevenLabsSttClient { start(): void { if (this.closed || this.ws) return; - const url = `${WS_URL}?model_id=${MODEL_ID}&audio_format=pcm_${SAMPLE_RATE}&commit_strategy=manual`; + // Without language_code the model detects the language, which can go + // badly wrong on short or noisy segments (Japanese heard as Russian). + const language = this.options.language ? `&language_code=${encodeURIComponent(this.options.language)}` : ""; + const url = `${WS_URL}?model_id=${MODEL_ID}&audio_format=pcm_${SAMPLE_RATE}&commit_strategy=manual${language}`; const listener = { onOpen: () => { if (this.closed) return; diff --git a/app/native/openai-stt.ts b/app/native/openai-stt.ts index 063e0a39..24f109a1 100644 --- a/app/native/openai-stt.ts +++ b/app/native/openai-stt.ts @@ -40,7 +40,7 @@ export class OpenAiRealtimeSttClient implements CloudSttClient { onOpen: () => { if (this.closed) return; this.open = true; - this.trySend(JSON.stringify(sessionConfig())); + this.trySend(JSON.stringify(sessionConfig(this.options.language ?? ""))); for (const chunk of this.pendingChunks.splice(0)) { this.sendChunk(chunk); } @@ -201,7 +201,7 @@ export class OpenAiRealtimeSttClient implements CloudSttClient { } -function sessionConfig(): object { +function sessionConfig(language: string): object { return { type: "session.update", session: { @@ -209,7 +209,7 @@ function sessionConfig(): object { audio: { input: { format: { type: "audio/pcm", rate: TARGET_SAMPLE_RATE }, - transcription: { model: MODEL_ID }, + transcription: { model: MODEL_ID, ...(language ? { language } : {}) }, // Push-to-talk owns the utterance boundary; no server VAD. turn_detection: null, }, diff --git a/app/native/soniox-stt.ts b/app/native/soniox-stt.ts index aa0d210c..fc24f2e0 100644 --- a/app/native/soniox-stt.ts +++ b/app/native/soniox-stt.ts @@ -50,6 +50,8 @@ export class SonioxSttClient implements CloudSttClient { sample_rate: CLOUD_STT_SAMPLE_RATE, num_channels: 1, enable_speaker_diarization: true, + // A hint, not a restriction: other languages are still recognized. + ...(this.options.language ? { language_hints: [this.options.language] } : {}), }), ); if (this.closed) return; diff --git a/app/native/voice-control.ios.ts b/app/native/voice-control.ios.ts index 38dd996f..f5a66124 100644 --- a/app/native/voice-control.ios.ts +++ b/app/native/voice-control.ios.ts @@ -3,6 +3,7 @@ import type { CloudSttTranscriptEvent } from './cloud-stt' import type { ReconnectingSttClient } from './reconnecting-stt' import { createCloudSttClient, usesCloudStt, type VoiceProviderSelection } from './cloud-stt-provider' import { WavRecorder } from './wav-recorder.ios' +import { pcmLevel } from '../util/audio-level' declare const FaceclawSpeech: any export type IosMicrophoneSession = { @@ -24,6 +25,7 @@ export class IosVoiceControlBridge { private readonly statuses = new Set<(state: VoiceControlState) => void>() private readonly transcripts = new Set<(event: VoiceTranscriptEvent) => void>() private readonly ends = new Set<() => void>() + private readonly levels = new Set<(level: number) => void>() private status = 'Voice input ready.' private generation = 0 private session: IosMicrophoneSession | null = null @@ -60,6 +62,8 @@ export class IosVoiceControlBridge { } onSpeechEnd(listener: () => void): () => void { this.ends.add(listener); return () => { this.ends.delete(listener) } } onSpeechPause(_listener: () => void): () => void { return () => {} } + /** The mic level (see pcmLevel) of each PCM chunk; only cloud captures supply PCM. */ + onMicLevel(listener: (level: number) => void): () => void { this.levels.add(listener); return () => { this.levels.delete(listener) } } private setStatus(status: string): void { this.status = status; for (const fn of [...this.statuses]) fn({ status, listening: this.capturing, detail: '' }) } private ensureNative(): any { if (!this.native) { @@ -69,6 +73,7 @@ export class IosVoiceControlBridge { const bytes = new Uint8Array(interop.bufferFromData(pcm)) this.cloud?.acceptPcm(bytes) if (this.continuous) this.record(bytes) + if (this.levels.size) { const level = pcmLevel(bytes); for (const fn of [...this.levels]) fn(level) } } } return this.native diff --git a/app/native/voice-control.ts b/app/native/voice-control.ts index c80d98ca..038c8ceb 100644 --- a/app/native/voice-control.ts +++ b/app/native/voice-control.ts @@ -4,6 +4,7 @@ import { type CloudSttTranscriptEvent, CloudSttClient } from "./cloud-stt"; import { createCloudSttClient, transcriptionKey } from "./cloud-stt-provider"; import { SpeechPauseDetector } from "./speech-pause"; import { toUint8Array } from "../util/array-util"; +import { pcmLevel } from "../util/audio-level"; declare const com: any; @@ -56,8 +57,9 @@ export type PushToTalkOptions = { sonioxApiKey: string; whisperServerUrl?: string; whisperServerModel?: string; - whisperServerLanguage?: string; whisperServerApiKey?: string; + /** Spoken language for cloud providers (see VoiceProviderSelection.language). */ + language?: string; saveRecording: boolean; /** * Watch the mic and fire onSpeechEnd when the speaker stops. For hands-free @@ -138,6 +140,7 @@ export class FaceclawVoiceControlBridge { // stopped for its duration. private readonly rawPcmListeners = new Set(); private readonly frameMetaListeners = new Set(); + private readonly micLevelListeners = new Set<(level: number) => void>(); private rawActive = false; // Who wants the raw tap. It runs while any owner does, so one owner letting // go (an EvenHub app backgrounding) doesn't silence another. @@ -237,6 +240,16 @@ export class FaceclawVoiceControlBridge { return () => this.rawPcmListeners.delete(listener); } + /** + * Subscribe to the mic level (meter position in [0,1], see pcmLevel) of + * each decoded audio chunk, about every 50 ms, from any capture: STT in + * every mode, or the raw tap. + */ + onMicLevel(listener: (level: number) => void): () => void { + this.micLevelListeners.add(listener); + return () => this.micLevelListeners.delete(listener); + } + /** Subscribe to per-packet firmware DSP metadata (DOA angle + SSR). */ onFrameMeta(listener: FrameMetaListener): () => void { this.frameMetaListeners.add(listener); @@ -539,6 +552,10 @@ export class FaceclawVoiceControlBridge { if (this.rawPcmListeners.size > 0) { this.rawPcmListeners.forEach((listener) => listener(bytes)); } + if (this.micLevelListeners.size > 0) { + const level = pcmLevel(bytes); + this.micLevelListeners.forEach((listener) => listener(level)); + } }, onFrameMeta: (angleDegrees: number, ssr: number) => { if (this.frameMetaListeners.size === 0) return; diff --git a/app/prompts.ts b/app/prompts.ts index 52b4de39..d5161cfd 100644 --- a/app/prompts.ts +++ b/app/prompts.ts @@ -85,15 +85,20 @@ export const TRANSLATE_SYSTEM_PROMPT = "Reply with the English translation only, with no preamble, quotes, notes, or transliteration. " + "If the speech is empty or meaningless noise, reply with nothing."; -/** User message for the Translate app: earlier turns for context, then the speech to translate. */ +/** + * User message for the Translate app: earlier turns for context, then the + * speech to translate. `language` names the spoken language when the user set + * one (the transcription was told to expect it), else "". + */ export function buildTranslateUserMessage( speech: string, context: ReadonlyArray<{ source: string; translation: string }>, + language = "", ): string { const earlier = context.length ? "Earlier in this conversation (already translated; context only):\n" + context.map((turn) => `${turn.source}\n${turn.translation}`).join("\n") + "\n\n" : ""; - return `${earlier}Translate this speech into English:\n${speech}`; + return `${earlier}Translate this ${language ? `${language} ` : ""}speech into English:\n${speech}`; } diff --git a/app/ui/layers.ts b/app/ui/layers.ts index 3b98c2d0..4101810c 100644 --- a/app/ui/layers.ts +++ b/app/ui/layers.ts @@ -14,14 +14,16 @@ export type ContinuousCaptureOptions = { /** Names a WAV of the captured audio in Downloads (iOS, when Save Transcribe audio is on). */ recordAs?: string; /** - * Transcribe with this provider instead of the Transcription Provider - * setting, letting it detect the spoken language (Translate). While any - * client asks for one, every capture uses it, and a shared capture running - * another provider restarts on it. + * Transcribe with this provider, expecting this spoken language (ISO 639-1, + * or "" for the provider to detect it), instead of the Transcription + * Provider setting (Translate). While any client asks for one, every + * capture uses it, and a shared capture running another restarts on it. */ - multilingualProvider?: VoiceProvider; + transcription?: TranscriptionChoice; }; +export type TranscriptionChoice = { provider: VoiceProvider; language: string }; + export type LayerActions = { /** Ask for a dashboard repaint+transmit, e.g. when async data arrives. */ requestRender: () => void; diff --git a/app/ui/listening-header.ts b/app/ui/listening-header.ts new file mode 100644 index 00000000..a93a2370 --- /dev/null +++ b/app/ui/listening-header.ts @@ -0,0 +1,92 @@ +import { type GrayImage, type UiFont } from "../graphics/image"; +import { truncateText } from "../graphics/textwrap"; +import { voiceControlBridge } from "../native/voice-control"; +import { lineStep } from "./metrics"; + +const HEADER_X = 24; +const HEADER_Y = 20; +const TITLE_GRAY = 200; +const STATUS_GRAY = 110; + +const SEGMENTS = 8; +const SEGMENT_WIDTH = 4; +const SEGMENT_GAP = 2; +const METER_GAP = 10; +const LIT_GRAY = 200; +const UNLIT_GRAY = 45; +/** How often the meter may ask for a repaint (each one is a frame over BLE). */ +const TICK_MS = 200; +/** Per tick, the shown level falls to this fraction without louder audio (~1 s to empty). */ +const DECAY = 0.5; + +/** + * A compact segmented mic-level meter for listening views. Levels arrive per + * 50 ms audio chunk; the meter holds their peak and, at most every 200 ms, + * asks for a repaint only when the number of lit segments changed and the + * window is on screen. + */ +export class MicLevelMeter { + readonly width = SEGMENTS * (SEGMENT_WIDTH + SEGMENT_GAP) - SEGMENT_GAP; + private peak = 0; + private shown = 0; + private lit = 0; + private timer: ReturnType | null = null; + private unsubscribe: (() => void) | null = null; + + start(requestRender: () => void, isVisible: () => boolean): void { + this.unsubscribe = voiceControlBridge.onMicLevel((level) => { + this.peak = Math.max(this.peak, level); + }); + this.timer = setInterval(() => { + this.shown = Math.max(this.peak, this.shown * DECAY); + this.peak = 0; + const lit = Math.min(SEGMENTS, Math.round(this.shown * SEGMENTS)); + if (lit === this.lit) return; + this.lit = lit; + if (isVisible()) requestRender(); + }, TICK_MS); + } + + stop(): void { + this.unsubscribe?.(); + this.unsubscribe = null; + if (this.timer !== null) clearInterval(this.timer); + this.timer = null; + } + + /** Draw the segments with their bottoms on `baseline`, `height` tall. */ + paint(image: GrayImage, x: number, baseline: number, height: number): void { + for (let index = 0; index < SEGMENTS; index++) { + const segmentX = x + index * (SEGMENT_WIDTH + SEGMENT_GAP); + image.fillRect(segmentX, baseline - height, SEGMENT_WIDTH, height, index < this.lit ? LIT_GRAY : UNLIT_GRAY); + } + } +} + +/** + * The one-line header of the listening views (Transcribe, Translate): + * "Title · status", the status truncated to fit, then the mic level meter at + * the line's end when one is given (while listening). Returns the y at which + * the view's body starts. + */ +export function paintListeningHeader( + image: GrayImage, + font: UiFont, + title: string, + status: string, + meter: MicLevelMeter | null, +): number { + let textRight = image.width - HEADER_X; + if (meter) { + const meterX = textRight - meter.width; + // Roughly cap height, sitting on the text's baseline. + meter.paint(image, meterX, HEADER_Y + font.ascent, Math.max(4, Math.round(font.ascent * 0.6))); + textRight = meterX - METER_GAP; + } + image.drawText(font, HEADER_X, HEADER_Y, title, TITLE_GRAY); + if (status) { + const statusX = HEADER_X + Math.ceil(font.measureText(title)); + image.drawText(font, statusX, HEADER_Y, truncateText(font, ` · ${status}`, textRight - statusX), STATUS_GRAY); + } + return HEADER_Y + lineStep(font) + 2; +} diff --git a/app/util/audio-level.ts b/app/util/audio-level.ts new file mode 100644 index 00000000..acfd9516 --- /dev/null +++ b/app/util/audio-level.ts @@ -0,0 +1,24 @@ +/** + * Mic level-meter math shared by the Microphones app's meters and the + * listening header (Transcribe, Translate): RMS mapped onto a 60 dB meter. + */ + +/** Meter position in [0,1] for a chunk of 16-bit little-endian PCM. */ +export function pcmLevel(pcm: Uint8Array): number { + let sumSquares = 0; + const count = pcm.length / 2; + if (count === 0) return 0; + for (let i = 0; i < count; i++) { + let value = pcm[i * 2]! | (pcm[i * 2 + 1]! << 8); + if (value >= 0x8000) value -= 0x10000; + sumSquares += value * value; + } + return rmsToLevel(Math.sqrt(sumSquares / count) / 32768); +} + +/** Map RMS in [0,1] to a meter position with a log-ish curve (mic peaks ~0.1 FS). */ +export function rmsToLevel(rms: number): number { + if (rms <= 0) return 0; + const db = 20 * Math.log10(rms); + return Math.max(0, Math.min(1, (db + 60) / 60)); +} diff --git a/tests/cloud-stt.test.cjs b/tests/cloud-stt.test.cjs index 2e394e2e..06abfb5f 100644 --- a/tests/cloud-stt.test.cjs +++ b/tests/cloud-stt.test.cjs @@ -2,7 +2,7 @@ const test = require('node:test'); const assert = require('node:assert/strict'); const { loader } = require('./helpers/load-typescript.cjs'); -function harness(provider = 'elevenlabs', continuous = true) { +function harness(provider = 'elevenlabs', continuous = true, extraOptions = {}) { const timers = new Map(); let timerId = 0; const sockets = []; @@ -30,7 +30,7 @@ function harness(provider = 'elevenlabs', continuous = true) { const Provider = load(`app/native/${provider}-stt.ts`)[names[provider]]; const { ReconnectingSttClient } = load('app/native/reconnecting-stt.ts'); const client = new ReconnectingSttClient(options => new Provider(options), { - apiKey: 'test', onTranscript: event => events.push(event), + apiKey: 'test', ...extraOptions, onTranscript: event => events.push(event), onStatus: status => statuses.push(status), onError: error => errors.push(error), }, () => continuous); client.start(); @@ -203,7 +203,10 @@ test('provider selection falls back to on-device without a key and connects the FaceclawWebSocket: class { constructor(url, _listener, header, value) { sockets.push({ url, header, value }); } }, } } }, }, { './cloud-stt': { CLOUD_STT_SAMPLE_RATE: 16000, encodeBase64: () => '', toJavaBytes: bytes => bytes } }); - const { createCloudSttClient, usesCloudStt } = load('app/native/cloud-stt-provider.ts'); + const { createCloudSttClient, usesCloudStt, transcriptionKey } = load('app/native/cloud-stt-provider.ts'); + // A running capture is shared only on the same provider and language. + assert.equal(transcriptionKey({ provider: 'soniox', language: ' JA ' }), 'soniox:ja'); + assert.notEqual(transcriptionKey({ provider: 'soniox' }), transcriptionKey({ provider: 'soniox', language: 'ja' })); const statuses = []; const options = { onTranscript() {}, onStatus: status => statuses.push(status), onError() {} }; const selection = { provider: 'onboard', elevenLabsApiKey: '', openAiApiKey: ' ', sonioxApiKey: 'soniox-key' }; @@ -217,4 +220,23 @@ test('provider selection falls back to on-device without a key and connects the createCloudSttClient({ ...selection, provider: 'elevenlabs', elevenLabsApiKey: ' xi ' }, options, () => false).start(); assert.deepEqual(sockets.map(({ url, header, value }) => [new URL(url).host, header, value]), [['stt-rt.soniox.com', null, null], ['api.elevenlabs.io', 'xi-api-key', 'xi']]); + createCloudSttClient({ ...selection, provider: 'elevenlabs', elevenLabsApiKey: 'xi', language: ' ja ' }, options, () => false).start(); + assert.equal(new URL(sockets.at(-1).url).searchParams.get('language_code'), 'ja'); +}); + +test('a spoken-language hint reaches each provider on the wire, and is left out to auto-detect', () => { + for (const language of ['ja', '']) { + const elevenlabs = harness('elevenlabs', true, { language }); + assert.equal(new URL(elevenlabs.sockets[0].url).searchParams.get('language_code'), language || null); + + const soniox = harness('soniox', true, { language }); + soniox.open(); + assert.deepEqual(JSON.parse(soniox.sockets[0].sent[0]).language_hints, language ? ['ja'] : undefined); + + const openai = harness('openai', true, { language }); + openai.open(); + const transcription = JSON.parse(openai.sockets[0].sent[0]).session.audio.input.transcription; + assert.equal(transcription.model, 'gpt-realtime-whisper'); + assert.equal(transcription.language, language || undefined); + } }); diff --git a/tests/ios-app-host.test.cjs b/tests/ios-app-host.test.cjs index 8631bb16..ef05adf7 100644 --- a/tests/ios-app-host.test.cjs +++ b/tests/ios-app-host.test.cjs @@ -348,7 +348,7 @@ test('iOS continuous capture is shared by windows and restarts after a disconnec get continuousActive() { return !!this.source || this.draining; }, prepare: async () => true, startGlassesCapture: async (_session, voice, _log, _endpointing, continuous) => { - calls.push(['glasses', continuous]); providers.push(`${voice.provider}:${voice.whisperServerLanguage}`); bridge.source = 'glasses'; }, + calls.push(['glasses', continuous]); providers.push(`${voice.provider}:${voice.language}`); bridge.source = 'glasses'; }, startPhoneCapture: async (_voice, _log, _endpointing, continuous) => { calls.push(['phone', continuous]); bridge.source = 'phone'; }, drain() { if (this.source) { this.source = null; this.draining = true; } }, stopContinuousCapture: async () => { calls.push(['stop']); bridge.drain(); }, @@ -361,7 +361,7 @@ test('iOS continuous capture is shared by windows and restarts after a disconnec const { IosPreviewController } = load('app/g2/ios-preview-controller.ts', { require: id => id === '../native/ios-voice-input' ? { iosVoiceInput: bridge } : id === '../ui/dashboard-settings' ? settings : id === '../ui/shell/voice-activity' ? { voiceActivity: { setActive: (active, holder) => claims.push([active, holder]) } } - : id === '../native/cloud-stt-provider' ? { transcriptionKey: voice => `${voice.provider}:${voice.whisperServerLanguage}` } : {}, + : id === '../native/cloud-stt-provider' ? { transcriptionKey: voice => `${voice.provider}:${voice.language}` } : {}, }); const host = Object.create(IosPreviewController.prototype); Object.assign(host, { active: true, glassesLocked: false, communicator: {}, state: { phase: 'connected' }, sessionConnected: true, @@ -381,14 +381,17 @@ test('iOS continuous capture is shared by windows and restarts after a disconnec host.stopContinuousVoiceCapture('calculator'); assert.deepEqual(calls, [['glasses', true]], 'closing one window keeps the other listening'); - // Translate needs a multilingual provider: the shared capture drains and - // restarts on it (Whisper left to detect the language), and moves back to - // the main provider once Translate closes. - calls.length = 0; host.startContinuousVoiceCapture('translate', { multilingualProvider: 'soniox' }); await settle(); + // Translate needs its own provider and language: the shared capture drains + // and restarts on them, again when only the language changes, and moves + // back to the main provider (and Whisper's own language) once Translate closes. + calls.length = 0; host.startContinuousVoiceCapture('translate', { transcription: { provider: 'soniox', language: '' } }); await settle(); assert.deepEqual(calls, [['stop']]); drained(); await settle(); assert.deepEqual(calls, [['stop'], ['glasses', true]]); assert.deepEqual(providers.slice(-2), ['whisper-server:en', 'soniox:']); + calls.length = 0; host.startContinuousVoiceCapture('translate', { transcription: { provider: 'soniox', language: 'ja' } }); drained(); await settle(); + assert.deepEqual(calls, [['stop'], ['glasses', true]]); + assert.equal(providers.at(-1), 'soniox:ja'); calls.length = 0; host.stopContinuousVoiceCapture('translate'); drained(); await settle(); assert.deepEqual(calls, [['stop'], ['glasses', true]]); assert.equal(providers.at(-1), 'whisper-server:en'); diff --git a/tests/listening-header.test.cjs b/tests/listening-header.test.cjs new file mode 100644 index 00000000..19ffc044 --- /dev/null +++ b/tests/listening-header.test.cjs @@ -0,0 +1,100 @@ +const test = require('node:test'); +const assert = require('node:assert/strict'); +const { loader } = require('./helpers/load-typescript.cjs'); + +const { pcmLevel } = loader()('app/util/audio-level.ts'); + +function harness() { + const levelListeners = new Set(); + let tick = null; + const { MicLevelMeter, paintListeningHeader } = loader({ + setInterval(fn) { tick = fn; return 1; }, + clearInterval() { tick = null; }, + }, { + '../native/voice-control': { voiceControlBridge: { + onMicLevel(listener) { levelListeners.add(listener); return () => levelListeners.delete(listener); }, + } }, + })('app/ui/listening-header.ts'); + const level = value => { for (const listener of levelListeners) listener(value); }; + return { MicLevelMeter, paintListeningHeader, level, tick: () => tick?.(), ticking: () => tick !== null, levelListeners }; +} + +/** Records draws; text is 6px per character, like a fixed-width bitmap face. */ +function fakeImage(width = 576) { + const texts = [], rects = []; + return { width, texts, rects, + drawText(_font, x, y, text, gray) { texts.push({ x, y, text, gray }); }, + fillRect(x, y, w, h, gray) { rects.push({ x, y, w, h, gray }); } }; +} +const font = { ascent: 12, lineHeight: 16, measureText: text => text.length * 6 }; + +function pcm(amplitude, samples = 800) { + const bytes = new Uint8Array(samples * 2); + for (let i = 0; i < samples; i++) { + const value = (i % 2 ? amplitude : -amplitude) & 0xffff; + bytes[i * 2] = value & 255; + bytes[i * 2 + 1] = value >> 8; + } + return bytes; +} + +test('pcm level maps RMS onto a 60 dB meter', () => { + assert.equal(pcmLevel(new Uint8Array(0)), 0); + assert.equal(pcmLevel(pcm(0)), 0); + assert.ok(Math.abs(pcmLevel(pcm(3277)) - 2 / 3) < 0.01, '-20 dBFS sits two thirds up'); + assert.ok(pcmLevel(pcm(32767)) > 0.99); +}); + +test('the meter repaints only when the lit segment count changes, and only while visible', () => { + const h = harness(); + const meter = new h.MicLevelMeter(); + let renders = 0, visible = true; + meter.start(() => renders++, () => visible); + h.level(0.3); h.level(0.5); h.level(0.2); + h.tick(); + assert.equal(renders, 1, 'the peak of the tick lights 4 of 8'); + h.level(0.5); h.tick(); + assert.equal(renders, 1, 'same segment count, no repaint'); + + // Off screen the level still tracks, so the next visible paint is current. + visible = false; + h.level(1); h.tick(); + assert.equal(renders, 1); + const image = fakeImage(); + meter.paint(image, 0, 10, 6); + assert.equal(image.rects.filter(rect => rect.gray === 200).length, 8); + + // Silence decays the meter to empty within about a second. + visible = true; + for (let i = 0; i < 5; i++) h.tick(); + const after = fakeImage(); + meter.paint(after, 0, 10, 6); + assert.equal(after.rects.filter(rect => rect.gray === 200).length, 0); + assert.ok(renders > 1); + + meter.stop(); + assert.equal(h.ticking(), false); + assert.equal(h.levelListeners.size, 0); +}); + +test('the header puts title and status on one line, truncated before the meter', () => { + const h = harness(); + const meter = new h.MicLevelMeter(); + const image = fakeImage(300); + const bodyTop = h.paintListeningHeader(image, font, 'Translate', 'Listening (ElevenLabs)...', meter); + assert.equal(bodyTop, 20 + 18 + 2); + assert.deepEqual(image.texts.map(({ x, y, gray }) => [x, y, gray]), [[24, 20, 200], [24 + 54, 20, 110]]); + assert.equal(image.texts[0].text, 'Translate'); + const status = image.texts[1].text; + assert.ok(status.startsWith(' · Listening') && status.endsWith('...'), status); + const meterX = Math.min(...image.rects.map(rect => rect.x)); + assert.equal(meterX, 300 - 24 - meter.width); + assert.ok(24 + 54 + status.length * 6 <= meterX - 10, 'status stops short of the meter'); + // Segments sit on the text baseline. + assert.ok(image.rects.every(rect => rect.y + rect.h === 20 + font.ascent)); + + const idle = fakeImage(300); + h.paintListeningHeader(idle, font, 'Transcribe', 'Starting microphone...', null); + assert.equal(idle.rects.length, 0); + assert.equal(idle.texts[1].text, ' · Starting microphone...'); +}); diff --git a/tests/translate.test.cjs b/tests/translate.test.cjs index a2bdf3ca..c9446b15 100644 --- a/tests/translate.test.cjs +++ b/tests/translate.test.cjs @@ -32,14 +32,13 @@ function settingsHarness({ main = 'onboard', setUp = [] } = {}) { })('app/apps/translate/translate-settings.ts'); } -test('finals become segments; finals arriving mid-translation merge into one pending segment', () => { +test('finals translate in order; finals arriving mid-request merge into one waiting block', () => { const model = new TranslationModel(); model.accept({ text: 'Hola, ¿qué', isFinal: false }); - assert.equal(model.live, 'Hola, ¿qué'); model.accept({ text: 'Hola, ¿qué tal?', isFinal: true }); - assert.equal(model.live, ''); + assert.equal(model.segments.length, 1); const first = model.nextJob(); - assert.equal(first.segment.source, 'Hola, ¿qué tal?'); + assert.equal(first.source, 'Hola, ¿qué tal?'); assert.deepEqual(plain(first.context), []); assert.equal(model.nextJob(), null, 'one request at a time'); @@ -49,30 +48,91 @@ test('finals become segments; finals arriving mid-translation merge into one pen assert.equal(model.segments.length, 2); assert.equal(model.segments[1].source, 'Muy bien. Gracias.'); - model.progress(first.segment, 'Hi, how'); - assert.equal(first.segment.translation, 'Hi, how'); - model.complete(first.segment, 'Hi, how are you?'); + assert.equal(model.progress(first, 'Hi, how'), true); + assert.equal(first.segment.streaming, 'Hi, how'); + model.complete(first, 'Hi, how are you?'); const second = model.nextJob(); - assert.equal(second.segment.source, 'Muy bien. Gracias.'); + assert.equal(second.source, 'Muy bien. Gracias.'); assert.deepEqual(plain(second.context), [{ source: 'Hola, ¿qué tal?', translation: 'Hi, how are you?' }]); - model.complete(second.segment, 'Very well. Thanks.'); + model.complete(second, 'Very well. Thanks.'); assert.equal(model.nextJob(), null); }); +test('the live block is translated provisionally, redone as it grows, and settles on a matching final', () => { + const model = new TranslationModel(); + model.accept({ text: '今日は', isFinal: false }); + const first = model.nextJob(); + assert.equal(first.source, '今日は'); + model.accept({ text: '今日はいい天気', isFinal: false }); // grows while the request is in flight + assert.equal(model.nextJob(), null); + model.complete(first, 'Today'); + assert.deepEqual(plain(model.lines(measure, 100, false)), [{ text: 'Today', gray: 170 }]); + + const second = model.nextJob(); + assert.equal(second.source, '今日はいい天気'); + // The redo streams over the old text only once it is as long. + model.progress(second, 'To'); + assert.deepEqual(plain(model.lines(measure, 100, false).map(line => line.text)), ['Today']); + model.progress(second, 'The weather'); + assert.deepEqual(plain(model.lines(measure, 100, false).map(line => line.text)), ['The weather']); + model.complete(second, 'The weather is nice today'); + + // A final matching what was translated needs no further request. + model.accept({ text: '今日はいい天気', isFinal: true }); + assert.equal(model.nextJob(), null); + assert.deepEqual(plain(model.lines(measure, 100, false)), [{ text: 'The weather is nice today', gray: 230 }]); +}); + +test('a final that differs from the request in flight makes it moot; a matching one keeps it', () => { + const model = new TranslationModel(); + model.accept({ text: 'Bonjour', isFinal: false }); + const stale = model.nextJob(); + assert.equal(model.accept({ text: 'Bonjour à tous', isFinal: true }), true); + assert.equal(model.isActive(stale), false); + assert.equal(model.progress(stale, 'Hello'), false); + const redo = model.nextJob(); + assert.equal(redo.source, 'Bonjour à tous'); + model.complete(stale, 'Hello'); // a late reply from the cancelled request is ignored + assert.equal(model.isActive(redo), true); + model.complete(redo, 'Hello everyone'); + + model.accept({ text: 'Merci', isFinal: false }); + const kept = model.nextJob(); + assert.equal(model.accept({ text: 'Merci', isFinal: true }), false); + model.complete(kept, 'Thank you'); + assert.equal(model.nextJob(), null); + assert.deepEqual(plain(model.lines(measure, 100, false).map(line => line.gray)), [230, 230]); +}); + +test('small revisions of a live block wait for a bigger change; its final is always redone', () => { + const model = new TranslationModel(); + model.accept({ text: 'abcdef', isFinal: false }); + model.complete(model.nextJob(), 'one'); + model.accept({ text: 'abcdeg', isFinal: false }); + assert.equal(model.nextJob(), null); + model.accept({ text: 'abcdefgh', isFinal: false }); + assert.equal(model.nextJob(), null); + model.accept({ text: 'abcdefghi', isFinal: false }); + model.complete(model.nextJob(), 'two'); + model.accept({ text: 'abcdefghj', isFinal: true }); + assert.equal(model.nextJob().source, 'abcdefghj'); +}); + test('an empty final keeps the live partial it replaces', () => { const model = new TranslationModel(); model.accept({ text: 'Bonjour', isFinal: false }); model.accept({ text: '', isFinal: true }); assert.equal(model.segments[0].source, 'Bonjour'); + assert.equal(model.segments[0].final, true); }); -test('context skips failed and dropped turns and stays bounded', () => { +test('context skips failed, dropped and provisional turns and stays bounded', () => { const model = new TranslationModel(); const translate = (source, translation) => { model.accept({ text: source, isFinal: true }); const job = model.nextJob(); - if (translation === null) model.fail(job.segment); - else model.complete(job.segment, translation); + if (translation === null) model.fail(job); + else model.complete(job, translation); }; translate('uno', 'one'); translate('ruido', null); @@ -87,18 +147,29 @@ test('context skips failed and dropped turns and stays bounded', () => { const long = new TranslationModel(); for (let i = 0; i < 3; i++) { long.accept({ text: 'x'.repeat(500), isFinal: true }); - long.complete(long.nextJob().segment, 'y'.repeat(500)); + long.complete(long.nextJob(), 'y'.repeat(500)); } long.accept({ text: 'next', isFinal: true }); assert.equal(long.nextJob().context.length, 1, 'the newest turn is kept even when it alone is over the cap'); + + // A final whose redo failed still shows its provisional translation, but + // that translates other text, so it isn't context. + const stale = new TranslationModel(); + stale.accept({ text: 'Ich gehe', isFinal: false }); + stale.complete(stale.nextJob(), 'I go'); + stale.accept({ text: 'Ich gehe nach Hause', isFinal: true }); + stale.fail(stale.nextJob()); + assert.deepEqual(plain(stale.lines(measure, 100, false)), [{ text: 'I go', gray: 170 }]); + stale.accept({ text: 'Tschüss', isFinal: true }); + assert.deepEqual(plain(stale.nextJob().context), []); }); -test('display lines: translations bright, originals dim until translated or when shown, live dimmest', () => { +test('display lines: settled bright, provisional dimmer, originals dim until translated or when shown', () => { const model = new TranslationModel(); model.accept({ text: 'Guten Morgen', isFinal: true }); - model.complete(model.nextJob().segment, 'Good morning'); + model.complete(model.nextJob(), 'Good morning'); model.accept({ text: 'Wie geht es?', isFinal: true }); - const asking = model.nextJob().segment; + const asking = model.nextJob(); model.accept({ text: 'Danke', isFinal: false }); assert.deepEqual(plain(model.lines(measure, 100, false)), [ { text: 'Good morning', gray: 230 }, @@ -107,28 +178,29 @@ test('display lines: translations bright, originals dim until translated or when ]); model.progress(asking, 'How are'); assert.deepEqual(plain(model.lines(measure, 100, false).map(line => line.text)), ['Good morning', 'How are', 'Danke']); - assert.deepEqual(plain(model.lines(measure, 100, true).map(line => line.gray)), [120, 230, 120, 230, 90]); + assert.deepEqual(plain(model.lines(measure, 100, true).map(line => line.gray)), [120, 230, 120, 170, 90]); assert.deepEqual(plain(model.lines(measure, 7, false).slice(0, 2)), [{ text: 'Good', gray: 230 }, { text: 'morning', gray: 230 }]); - // A failed segment keeps its original on screen. + // A failed block keeps its original on screen. model.fail(asking); assert.deepEqual(plain(model.lines(measure, 100, false)[1]), { text: 'Wie geht es?', gray: 120 }); - // A finished, empty translation is noise: hidden unless originals are shown. + // A settled, empty translation is noise: hidden unless originals are shown. model.accept({ text: 'mmm', isFinal: true }); - model.complete(model.nextJob().segment, ''); + model.complete(model.nextJob(), ''); assert.equal(model.lines(measure, 100, false).some(line => line.text === 'mmm'), false); assert.equal(model.lines(measure, 100, true).some(line => line.text === 'mmm'), true); }); -test('saved text pairs each original with its translation', () => { +test('saved text pairs each final original with its translation', () => { const model = new TranslationModel(); for (const [source, translation] of [['Ciao', 'Hello'], ['Okay.', 'Okay.'], ['boh', null]]) { model.accept({ text: source, isFinal: true }); const job = model.nextJob(); - if (translation === null) model.fail(job.segment); - else model.complete(job.segment, translation); + if (translation === null) model.fail(job); + else model.complete(job, translation); } + model.accept({ text: 'ancora', isFinal: false }); // still live: not saved assert.equal(model.savedText(), 'Ciao\n→ Hello\n\nOkay.\n\nboh\n→ (not translated)'); }); @@ -138,6 +210,8 @@ test('the user message carries earlier turns as context, then the speech', () => assert.match(message, /context only/); assert.match(message, /Bien<\/speech>\nFine<\/english>\n\nTranslate this speech/); assert.ok(message.endsWith('¿Y tú?')); + assert.equal(buildTranslateUserMessage('こんにちは', [], 'Japanese'), + 'Translate this Japanese speech into English:\nこんにちは'); }); test('Auto picks the main provider when it is multilingual, else the first one set up', () => { @@ -164,3 +238,15 @@ test('an explicit provider is used only while it is set up, and on-device ones a assert.equal(settings.translateProviderSetting.isDisabled('auto'), false); assert.deepEqual(plain(settings.translateProviderSetting.values), ['auto', 'elevenlabs', 'whisper', 'soniox', 'whisper-server']); }); + +test('the spoken language defaults to auto-detect and resolves to an ISO 639-1 code once chosen', () => { + const settings = settingsHarness(); + assert.equal(settings.resolveTranslateLanguage(), ''); + assert.equal(settings.translateLanguageSetting.displayValue('auto'), 'Auto-detect'); + settings.translateLanguageSetting.set('ja'); + assert.equal(settings.resolveTranslateLanguage(), 'ja'); + assert.equal(settings.spokenLanguageName('ja'), 'Japanese'); + assert.equal(settings.translateLanguageSetting.values[0], 'auto'); + assert.ok(settings.translateLanguageSetting.values.slice(1).every(code => /^[a-z]{2}$/.test(code))); + assert.equal(settings.translateLanguageSetting.values.includes('en'), false, 'English is the target, not a source'); +});