From 9b70880a5b5a2a2ba32400ab1aff8842483a6ce2 Mon Sep 17 00:00:00 2001 From: James Babcock Date: Sat, 5 Sep 2026 19:16:56 -0700 Subject: [PATCH] Improve the Transcribe app --- CHANGELOG | 1 + app/apps/transcribe/transcribe-app.ts | 8 ++ app/apps/transcribe/transcribe.ts | 87 +++++---------- app/apps/transcribe/transcript-model.ts | 83 ++++++++++++++ app/native/cloud-stt.ts | 7 ++ app/native/elevenlabs-stt.ts | 89 +++++++-------- app/native/openai-stt.ts | 90 ++++++++++++--- app/native/reconnecting-stt.ts | 141 ++++++++++++++++++++++++ app/native/soniox-stt.ts | 61 +++++++--- app/native/speech-pause.ts | 44 ++++++++ app/native/transcript-format.ts | 32 ++++++ app/native/voice-control.ts | 37 +++++-- 12 files changed, 540 insertions(+), 140 deletions(-) create mode 100644 app/apps/transcribe/transcript-model.ts create mode 100644 app/native/reconnecting-stt.ts create mode 100644 app/native/speech-pause.ts create mode 100644 app/native/transcript-format.ts diff --git a/CHANGELOG b/CHANGELOG index fc79aaa2..f0f292cc 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -2,6 +2,7 @@ ## 0.6.6 - Future * Navigate and terminal app: App-specific screen size/position setting +* Transcribe app: Scrolling; automatic reconnection; insert newlines ## 0.6.5 - 2026-09-04 * Changed context menu gestures: — for system, ●— for app context menu. diff --git a/app/apps/transcribe/transcribe-app.ts b/app/apps/transcribe/transcribe-app.ts index b51f0139..b0a17639 100644 --- a/app/apps/transcribe/transcribe-app.ts +++ b/app/apps/transcribe/transcribe-app.ts @@ -57,10 +57,18 @@ export function createTranscribeAppWindow(options: TranscribeAppOptions): InProc closeable: true, actions: options.actions, baseLayer: new YieldAtRootLayer(layer), + menuItems: () => [{ + label: "Save", + onSelect: (ctx) => { + ctx.stack.pop(); + layer.saveTranscript(); + }, + }], submitFrame: options.submitFrame, setSurfaceVisible: options.setSurfaceVisible, removeSurface: options.removeSurface, onClosed: () => { + layer.onRemoved(); options.stopContinuousVoiceCapture(); shell.setTrayIcon(TRAY_ICON_ID, null); options.onClosed(); diff --git a/app/apps/transcribe/transcribe.ts b/app/apps/transcribe/transcribe.ts index 0b6de439..8c056c52 100644 --- a/app/apps/transcribe/transcribe.ts +++ b/app/apps/transcribe/transcribe.ts @@ -1,33 +1,35 @@ -import { GrayImage, type UiFont } from "../../graphics/image"; +import { GrayImage } from "../../graphics/image"; import { getDefaultSmallFont } from "../../graphics/ui-fonts"; import { lineStep } from "../../ui/metrics"; import { writeTextToDownloads } from "../../native/file-access"; -import { voiceControlBridge, type VoiceTranscriptEvent } from "../../native/voice-control"; +import { voiceControlBridge } from "../../native/voice-control"; import { Layer, type LayerContext } from "../../ui/layers"; +import { type InputEvent } from "../../ui/gestures"; +import { TranscriptModel, TranscriptScroll, wrapTranscribeText } from "./transcript-model"; /** * Live transcription view. Continuous mic capture is owned by the app wrapper * (createTranscribeAppWindow); this layer accumulates the transcript, shows - * it, and saves it to Downloads on click. + * it, and saves it to Downloads from the app menu. */ -import { GESTURE_CLICK, GESTURE_DOUBLE_CLICK, type InputEvent } from "../../ui/gestures"; export class TranscribeLayer implements Layer { private status = "Listening..."; - // Finalized utterances, plus the live (replace-semantics) partial appended - // when painting. - private finalizedText = ""; - private liveText = ""; + private readonly transcript = new TranscriptModel(); + private readonly scroll = new TranscriptScroll(); private saveNotice = ""; private unsubscribeTranscript: (() => void) | null = null; + private unsubscribePause: (() => void) | null = null; private unsubscribeStatus: (() => void) | null = null; start(requestRender: () => void): void { this.unsubscribeTranscript = voiceControlBridge.onTranscript((event) => { - this.onTranscript(event); + this.transcript.accept(event); requestRender(); }); + this.unsubscribePause = voiceControlBridge.onSpeechPause(() => this.transcript.pause()); this.unsubscribeStatus = voiceControlBridge.onStatus((state) => { this.status = state.status; + this.saveNotice = ""; requestRender(); }); } @@ -36,29 +38,34 @@ export class TranscribeLayer implements Layer { const font = getDefaultSmallFont(); const { width, height } = ctx.stack.getBaseSize(); const image = new GrayImage(width, height, 0); - const text = this.displayText() || "Listening..."; - const wrapped = wrapTranscribeText(font, text, width - 64); + const text = this.transcript.text || "Listening..."; + const wrapped = wrapTranscribeText((text) => font.measureText(text), text, width - 64); image.drawText(font, 24, 20, "Transcribe", 200); image.drawText(font, 24, 40, this.saveNotice || this.status, 110); const step = lineStep(font) + 2; const bodyTop = 62; - const footerY = height - font.lineHeight - 6; - const bodyLines = Math.max(1, Math.floor((footerY - bodyTop) / step)); - const firstLine = Math.max(0, wrapped.length - bodyLines); - for (let index = firstLine; index < wrapped.length; index++) { + const bodyLines = Math.max(1, Math.floor((height - 6 - bodyTop - font.lineHeight) / step) + 1); + const firstLine = this.scroll.layout(wrapped.length, bodyLines); + for (let index = firstLine; index < Math.min(wrapped.length, firstLine + bodyLines); index++) { const y = bodyTop + (index - firstLine) * step; image.drawText(font, 32, y, wrapped[index]!, 230); } - image.drawText(font, 24, footerY, `${GESTURE_CLICK} save ${GESTURE_DOUBLE_CLICK} back`, 110); + if (wrapped.length > bodyLines) { + const trackHeight = height - bodyTop - 6; + const thumbHeight = Math.max(8, Math.round(trackHeight * bodyLines / wrapped.length)); + const thumbY = bodyTop + Math.round((trackHeight - thumbHeight) * firstLine / (wrapped.length - bodyLines)); + image.fillRect(width - 14, bodyTop, 2, trackHeight, 45); + image.fillRect(width - 14, thumbY, 2, thumbHeight, 140); + } return image; } handleInput(event: InputEvent, ctx: LayerContext): void { - if (event.type === "click") { - this.saveTranscript(); + if (event.type === "scroll-up" || event.type === "scroll-down") { + this.scroll.scroll(event.type === "scroll-up" ? -3 : 3); ctx.actions.requestRender(); return; } @@ -68,14 +75,16 @@ export class TranscribeLayer implements Layer { } onRemoved(): void { + this.unsubscribePause?.(); + this.unsubscribePause = null; this.unsubscribeTranscript?.(); this.unsubscribeTranscript = null; this.unsubscribeStatus?.(); this.unsubscribeStatus = null; } - private saveTranscript(): void { - const text = this.displayText().trim(); + saveTranscript(): void { + const text = this.transcript.text.trim(); if (!text) { this.saveNotice = "Nothing to save yet."; return; @@ -84,25 +93,6 @@ export class TranscribeLayer implements Layer { const path = writeTextToDownloads(filename, `${text}\n`); this.saveNotice = path ? `Saved ${filename}` : "Save failed (check file access)."; } - - private displayText(): string { - if (!this.finalizedText) return this.liveText; - if (!this.liveText) return this.finalizedText; - return `${this.finalizedText} ${this.liveText}`; - } - - private onTranscript(event: VoiceTranscriptEvent): void { - if (event.isFinal) { - const finalText = event.text.trim() || this.liveText.trim(); - if (finalText) { - this.finalizedText = this.finalizedText ? `${this.finalizedText} ${finalText}` : finalText; - } - this.liveText = ""; - } else { - // Replace semantics: the live partial is the whole current best transcript. - this.liveText = event.text.trim(); - } - } } function transcriptTimestamp(): string { @@ -113,22 +103,3 @@ function transcriptTimestamp(): string { `-${pad(now.getHours())}${pad(now.getMinutes())}${pad(now.getSeconds())}` ); } - -function wrapTranscribeText(font: UiFont, text: string, maxWidth: number): string[] { - const words = text.split(/\s+/).filter(Boolean); - const lines: string[] = []; - let line = ""; - for (const word of words) { - const candidate = line ? `${line} ${word}` : word; - if (line && font.measureText(candidate) > maxWidth) { - lines.push(line); - line = word; - } else { - line = candidate; - } - } - if (line) { - lines.push(line); - } - return lines.length ? lines : [""]; -} diff --git a/app/apps/transcribe/transcript-model.ts b/app/apps/transcribe/transcript-model.ts new file mode 100644 index 00000000..8b0e811a --- /dev/null +++ b/app/apps/transcribe/transcript-model.ts @@ -0,0 +1,83 @@ +type TranscriptEvent = { + text: string; + isFinal: boolean; + transcribeText?: string; + paragraphBreakAfter?: boolean; +}; + +/** Text accumulation belongs to this app, not the shared dictation stream. */ +export class TranscriptModel { + private finalized = ""; + private live = ""; + private separator = " "; + private breakAfterLive = false; + + get text(): string { + return this.finalized && this.live ? this.finalized + this.separator + this.live : this.finalized || this.live; + } + + accept(event: TranscriptEvent): void { + const text = (event.transcribeText ?? event.text).trim(); + if (!event.isFinal) { + this.live = text; + return; + } + const finalText = text || this.live; + if (finalText) this.finalized = this.finalized ? this.finalized + this.separator + finalText : finalText; + this.live = ""; + if (finalText) this.separator = event.paragraphBreakAfter || this.breakAfterLive ? "\n" : " "; + this.breakAfterLive = false; + } + + /** The onboard recognizer may finalize just before or just after the pause. */ + pause(): void { + if (this.live) this.breakAfterLive = true; + else if (this.finalized) this.separator = "\n"; + } +} + +/** An absolute history position, with an explicit follow-tail mode. */ +export class TranscriptScroll { + firstLine = 0; + private maximum = 0; + private following = true; + + layout(totalLines: number, visibleLines: number): number { + this.maximum = Math.max(0, totalLines - visibleLines); + this.firstLine = this.following ? this.maximum : Math.min(this.firstLine, this.maximum); + return this.firstLine; + } + + scroll(delta: number): void { + this.firstLine = Math.max(0, Math.min(this.maximum, this.firstLine + delta)); + this.following = this.firstLine === this.maximum; + } +} + +export function wrapTranscribeText(measure: (text: string) => number, text: string, maxWidth: number): string[] { + const lines: string[] = []; + for (const paragraph of text.split("\n")) { + let line = ""; + for (const word of paragraph.split(/\s+/).filter(Boolean)) { + const candidate = line ? `${line} ${word}` : word; + if (line && measure(candidate) > maxWidth) { + lines.push(line); + line = ""; + } + if (measure(word) <= maxWidth) { + line = line ? `${line} ${word}` : word; + continue; + } + // Long URLs and unspaced languages must not run off the display. + for (const char of (line ? " " : "") + word) { + if (line && measure(line + char) > maxWidth) { + lines.push(line); + line = ""; + } + line += char; + } + } + lines.push(line); + } + return lines; +} diff --git a/app/native/cloud-stt.ts b/app/native/cloud-stt.ts index 83c9c76f..f9a77167 100644 --- a/app/native/cloud-stt.ts +++ b/app/native/cloud-stt.ts @@ -9,6 +9,9 @@ export type CloudSttTranscriptEvent = { text: string; isFinal: boolean; + /** Optional presentation for Transcribe only; dictation uses plain text. */ + transcribeText?: string; + paragraphBreakAfter?: boolean; }; export type CloudSttOptions = { @@ -16,6 +19,8 @@ export type CloudSttOptions = { onTranscript: (event: CloudSttTranscriptEvent) => void; onStatus: (status: string) => void; onError: (message: string) => void; + onReady?: () => void; + onDisconnected?: (message: string) => void; }; export interface CloudSttClient { @@ -25,6 +30,8 @@ export interface CloudSttClient { acceptPcm(pcm: Uint8Array): void; /** End of utterance: produce a final transcript. */ finish(): void; + /** Commit a pause boundary while keeping the audio session open. */ + commitSegment?(): void; /** Abandon the session without finalizing. */ stop(): void; } diff --git a/app/native/elevenlabs-stt.ts b/app/native/elevenlabs-stt.ts index 8d602078..6fa555fc 100644 --- a/app/native/elevenlabs-stt.ts +++ b/app/native/elevenlabs-stt.ts @@ -1,4 +1,4 @@ -import { encodeBase64 } from "./cloud-stt"; +import { type CloudSttOptions, type CloudSttTranscriptEvent, encodeBase64 } from "./cloud-stt"; declare const com: any; @@ -18,17 +18,8 @@ const WS_URL = "wss://api.elevenlabs.io/v1/speech-to-text/realtime"; const MODEL_ID = "scribe_v2_realtime"; const SAMPLE_RATE = 16000; -export type ElevenLabsTranscriptEvent = { - text: string; - isFinal: boolean; -}; - -export type ElevenLabsSttOptions = { - apiKey: string; - onTranscript: (event: ElevenLabsTranscriptEvent) => void; - onStatus: (status: string) => void; - onError: (message: string) => void; -}; +export type ElevenLabsTranscriptEvent = CloudSttTranscriptEvent; +export type ElevenLabsSttOptions = CloudSttOptions; export class ElevenLabsSttClient { private ws: any = null; @@ -38,26 +29,32 @@ export class ElevenLabsSttClient { // PCM that arrived before the socket finished opening; flushed on open. private readonly pendingChunks: string[] = []; private latestText = ""; + private readonly commits: boolean[] = []; constructor(private readonly options: ElevenLabsSttOptions) {} start(): void { - if (this.ws) return; + if (this.closed || this.ws) return; const url = `${WS_URL}?model_id=${MODEL_ID}&audio_format=pcm_${SAMPLE_RATE}&commit_strategy=manual`; this.listenerProxy = new com.faceclaw.app.FaceclawWebSocketListener({ onOpen: () => { + if (this.closed) return; this.open = true; for (const chunk of this.pendingChunks.splice(0)) { this.sendChunk(chunk); } + if (!this.closed) this.options.onReady?.(); + }, + onTextMessage: (message: string) => { + if (!this.closed) this.handleMessage(String(message)); }, - onTextMessage: (message: string) => this.handleMessage(String(message)), onClosed: () => { this.open = false; + if (!this.closed) this.options.onDisconnected?.("ElevenLabs connection closed."); }, onFailure: (message: string) => { if (this.closed) return; - this.options.onError(`ElevenLabs connection failed: ${String(message)}`); + this.options.onDisconnected?.(`ElevenLabs connection failed: ${String(message)}`); }, }); try { @@ -67,7 +64,7 @@ export class ElevenLabsSttClient { this.ws = new com.faceclaw.app.FaceclawWebSocket(url, this.listenerProxy, "xi-api-key", key); this.options.onStatus("Connecting to ElevenLabs..."); } catch (error) { - this.options.onError(`ElevenLabs connection failed: ${String((error as Error)?.message ?? error)}`); + this.options.onDisconnected?.(`ElevenLabs connection failed: ${String((error as Error)?.message ?? error)}`); } } @@ -82,24 +79,25 @@ export class ElevenLabsSttClient { } } + commitSegment(): void { + this.commit(true); + } + /** End of utterance: flush and commit for a final transcript. */ finish(): void { + this.commit(false); + } + + private commit(paragraphBreakAfter: boolean): void { if (this.closed) return; - const commitMessage = JSON.stringify({ - message_type: "input_audio_chunk", - audio_base_64: "", - commit: true, - sample_rate: SAMPLE_RATE, - }); - if (this.open) { - this.trySend(commitMessage); - } else { - this.pendingChunks.push("__commit__"); - } + this.commits.push(paragraphBreakAfter); + if (this.open) this.sendChunk("__commit__"); + else this.pendingChunks.push("__commit__"); } stop(): void { this.closed = true; + this.open = false; if (this.ws) { try { this.ws.close(1000, "bye"); @@ -109,28 +107,26 @@ export class ElevenLabsSttClient { this.ws = null; } this.listenerProxy = null; + this.pendingChunks.length = 0; } private sendChunk(base64OrCommit: string): void { - if (base64OrCommit === "__commit__") { - this.finish(); - return; - } - this.trySend( - JSON.stringify({ - message_type: "input_audio_chunk", - audio_base_64: base64OrCommit, - commit: false, - sample_rate: SAMPLE_RATE, - }), - ); + const commit = base64OrCommit === "__commit__"; + this.trySend(JSON.stringify({ + message_type: "input_audio_chunk", + audio_base_64: commit ? "" : base64OrCommit, + commit, + sample_rate: SAMPLE_RATE, + })); } private trySend(message: string): void { try { - this.ws?.sendText(message); + if (this.ws?.sendText(message) === false) { + this.options.onDisconnected?.("ElevenLabs send failed."); + } } catch (error) { - console.warn("elevenlabs send failed", error); + this.options.onDisconnected?.(`ElevenLabs send failed: ${String(error)}`); } } @@ -150,14 +146,19 @@ export class ElevenLabsSttClient { this.options.onTranscript({ text: this.latestText, isFinal: false }); return; case "committed_transcript": - case "committed_transcript_with_timestamps": this.latestText = String(message.text ?? this.latestText); - this.options.onTranscript({ text: this.latestText, isFinal: true }); + this.options.onTranscript({ text: this.latestText, isFinal: true, paragraphBreakAfter: this.commits.shift() ?? false }); + this.latestText = ""; + return; + case "committed_transcript_with_timestamps": + // The timestamp event follows the plain commit; do not append twice. + return; + case "rate_limited": + this.options.onDisconnected?.("ElevenLabs rate limited."); return; case "error": case "auth_error": case "quota_exceeded": - case "rate_limited": case "input_error": this.options.onError(`ElevenLabs: ${String(message.error ?? message.message_type)}`); return; diff --git a/app/native/openai-stt.ts b/app/native/openai-stt.ts index e634f3cf..2949be00 100644 --- a/app/native/openai-stt.ts +++ b/app/native/openai-stt.ts @@ -30,28 +30,34 @@ export class OpenAiRealtimeSttClient implements CloudSttClient { // Base64 audio (or the commit sentinel) queued until the socket opens. private readonly pendingChunks: string[] = []; private readonly upsampler = new PcmUpsampler(); - private latestText = ""; + private readonly items = new Map(); + private readonly commits: boolean[] = []; private committed = false; constructor(private readonly options: OpenAiSttOptions) {} start(): void { - if (this.ws) return; + if (this.closed || this.ws) return; this.listenerProxy = new com.faceclaw.app.FaceclawWebSocketListener({ onOpen: () => { + if (this.closed) return; this.open = true; this.trySend(JSON.stringify(sessionConfig())); for (const chunk of this.pendingChunks.splice(0)) { this.sendChunk(chunk); } + if (!this.closed) this.options.onReady?.(); + }, + onTextMessage: (message: string) => { + if (!this.closed) this.handleMessage(String(message)); }, - onTextMessage: (message: string) => this.handleMessage(String(message)), onClosed: () => { this.open = false; + if (!this.closed) this.options.onDisconnected?.("OpenAI connection closed."); }, onFailure: (message: string) => { if (this.closed) return; - this.options.onError(`OpenAI connection failed: ${String(message)}`); + this.options.onDisconnected?.(`OpenAI connection failed: ${String(message)}`); }, }); try { @@ -63,7 +69,7 @@ export class OpenAiRealtimeSttClient implements CloudSttClient { ); this.options.onStatus("Connecting to OpenAI..."); } catch (error) { - this.options.onError(`OpenAI connection failed: ${String((error as Error)?.message ?? error)}`); + this.options.onDisconnected?.(`OpenAI connection failed: ${String((error as Error)?.message ?? error)}`); } } @@ -80,9 +86,19 @@ export class OpenAiRealtimeSttClient implements CloudSttClient { } } + commitSegment(): void { + this.commit(true); + } + /** End of utterance: commit the buffer for a final transcript. */ finish(): void { + this.committed = true; + this.commit(false); + } + + private commit(paragraphBreakAfter: boolean): void { if (this.closed) return; + this.commits.push(paragraphBreakAfter); if (this.open) { this.sendChunk("__commit__"); } else { @@ -92,6 +108,7 @@ export class OpenAiRealtimeSttClient implements CloudSttClient { stop(): void { this.closed = true; + this.open = false; if (this.ws) { try { this.ws.close(1000, "bye"); @@ -101,11 +118,11 @@ export class OpenAiRealtimeSttClient implements CloudSttClient { this.ws = null; } this.listenerProxy = null; + this.pendingChunks.length = 0; } private sendChunk(base64OrCommit: string): void { if (base64OrCommit === "__commit__") { - this.committed = true; this.trySend(JSON.stringify({ type: "input_audio_buffer.commit" })); return; } @@ -114,9 +131,11 @@ export class OpenAiRealtimeSttClient implements CloudSttClient { private trySend(message: string): void { try { - this.ws?.sendText(message); + if (this.ws?.sendText(message) === false) { + this.options.onDisconnected?.("OpenAI send failed."); + } } catch (error) { - console.warn("openai send failed", error); + this.options.onDisconnected?.(`OpenAI send failed: ${String(error)}`); } } @@ -132,23 +151,62 @@ export class OpenAiRealtimeSttClient implements CloudSttClient { case "session.updated": this.options.onStatus("Listening (OpenAI)..."); return; - case "conversation.item.input_audio_transcription.delta": - this.latestText += String(message.delta ?? ""); - this.options.onTranscript({ text: this.latestText, isFinal: false }); + case "input_audio_buffer.committed": { + this.item(String(message.item_id)).paragraphBreakAfter = this.commits.shift() ?? false; return; - case "conversation.item.input_audio_transcription.completed": - this.latestText = String(message.transcript ?? this.latestText); - this.options.onTranscript({ text: this.latestText, isFinal: true }); - // The utterance is done; nothing further is coming after the commit. - if (this.committed) this.stop(); + } + case "conversation.item.input_audio_transcription.delta": { + this.item(String(message.item_id)).text += String(message.delta ?? ""); + this.emitItems(); return; + } + case "conversation.item.input_audio_transcription.completed": { + const item = this.item(String(message.item_id)); + item.text = String(message.transcript ?? item.text); + item.final = true; + this.emitItems(); + if (this.committed && !this.items.size && !this.commits.length) this.stop(); + return; + } case "error": + if (message.error?.code === "rate_limit_exceeded" || message.error?.type === "server_error") { + this.options.onDisconnected?.(`OpenAI: ${String(message.error?.message)}`); + return; + } this.options.onError(`OpenAI: ${String(message.error?.message ?? "unknown error")}`); return; default: return; } } + + private item(id: string): { text: string; final: boolean; paragraphBreakAfter: boolean } { + let item = this.items.get(id); + if (!item) { + item = { text: "", final: false, paragraphBreakAfter: false }; + this.items.set(id, item); + } + return item; + } + + private emitItems(): void { + // Committed notifications precede transcription events, so Map insertion + // order is audio order even when completion events arrive out of order. + for (const [id, item] of this.items) { + if (!item.final) break; + this.items.delete(id); + this.options.onTranscript({ text: item.text, isFinal: true, paragraphBreakAfter: item.paragraphBreakAfter }); + } + if (this.items.size) { + const items = [...this.items.values()]; + const text = items.map((item) => item.text.trim()).filter(Boolean).join(" "); + const transcribeText = items.map((item, index) => + (index && items[index - 1]!.paragraphBreakAfter ? "\n" : index ? " " : "") + item.text.trim(), + ).join(""); + this.options.onTranscript({ text, transcribeText, isFinal: false }); + } + } + } function sessionConfig(): object { diff --git a/app/native/reconnecting-stt.ts b/app/native/reconnecting-stt.ts new file mode 100644 index 00000000..1221bb8d --- /dev/null +++ b/app/native/reconnecting-stt.ts @@ -0,0 +1,141 @@ +import type { CloudSttClient, CloudSttOptions, CloudSttTranscriptEvent } from "./cloud-stt"; +import { SpeechPauseDetector } from "./speech-pause"; + +// Ten seconds of unsent PCM. Never replay audio already sent: that would +// duplicate words already shown. An extended outage drops the oldest audio. +const MAX_PENDING_BYTES = 16000 * 2 * 10; + +/** Owns retries and audio buffering independently of the provider protocol. */ +export class ReconnectingSttClient implements CloudSttClient { + private client: CloudSttClient | null = null; + private generation = 0; + private stopped = false; + private finishing = false; + private ready = false; + private retry: ReturnType | null = null; + private finishTimeout: ReturnType | null = null; + private attempt = 0; + private pending: Uint8Array[] = []; + private pendingBytes = 0; + private latest: CloudSttTranscriptEvent | null = null; + private readonly pause = new SpeechPauseDetector(); + + constructor( + private readonly create: (options: CloudSttOptions) => CloudSttClient, + private readonly options: CloudSttOptions, + private readonly continuous: () => boolean, + ) {} + + start(): void { + if (this.stopped || this.client || this.retry !== null) return; + const generation = ++this.generation; + const current = () => !this.stopped && generation === this.generation; + this.client = this.create({ + ...this.options, + onReady: () => { + if (!current()) return; + this.ready = true; + const chunks = this.pending; + this.pending = []; + this.pendingBytes = 0; + for (const chunk of chunks) this.acceptPcm(chunk); + if (this.finishing) this.client?.finish(); + }, + onTranscript: (event) => { + if (!current()) return; + // A handshake alone can immediately be followed by another failure. + this.attempt = 0; + this.latest = event.isFinal ? null : event; + this.options.onTranscript(event); + }, + onStatus: (status) => { if (current()) this.options.onStatus(status); }, + onError: (message) => { + if (!current()) return; + this.stop(); + this.options.onError(message); + }, + onDisconnected: (message) => { + if (current()) this.disconnected(message); + }, + }); + try { + this.client.start(); + } catch (error) { + this.disconnected(`Transcription connection failed: ${String(error)}`); + } + } + + acceptPcm(pcm: Uint8Array): void { + if (this.stopped || !pcm.length) return; + if (!this.ready) { + const copy = pcm.slice(-MAX_PENDING_BYTES); + this.pending.push(copy); + this.pendingBytes += copy.length; + while (this.pendingBytes > MAX_PENDING_BYTES) this.pendingBytes -= this.pending.shift()!.length; + return; + } + this.client?.acceptPcm(pcm); + // A rejected send synchronously reports disconnection. Keep that packet + // with the unsent audio instead of losing it at the outage boundary. + if (!this.ready && !this.stopped) this.acceptPcm(pcm); + if (this.ready && !this.finishing && this.continuous() && this.pause.accept(pcm)) this.client?.commitSegment?.(); + } + + finish(): void { + if (this.stopped || this.finishing) return; + this.finishing = true; + // Releasing the last mic holder must not leave a reconnect loop alive. + if (this.retry !== null) { + this.preservePartial(); + this.stop(); + return; + } + this.finishTimeout = setTimeout(() => this.stop(), 5000); + if (this.ready) this.client?.finish(); + } + + stop(): void { + this.stopped = true; + ++this.generation; + if (this.retry !== null) clearTimeout(this.retry); + if (this.finishTimeout !== null) clearTimeout(this.finishTimeout); + this.retry = this.finishTimeout = null; + this.client?.stop(); + this.client = null; + this.ready = false; + this.pending = []; + this.pendingBytes = 0; + this.latest = null; + } + + private preservePartial(): void { + if (this.latest?.text.trim()) this.options.onTranscript({ ...this.latest, isFinal: true }); + this.latest = null; + } + + private disconnected(message: string): void { + if (this.stopped) return; + ++this.generation; // Ignore late finals / closes from the abandoned socket. + this.ready = false; + this.client?.stop(); + this.client = null; + this.pause.reset(); + this.preservePartial(); + if (this.stopped) return; // A transcript listener can release capture. + if (/\b(401|402|403)\b/.test(message)) { + this.stop(); + this.options.onError(message); + return; + } + if (this.finishing) { + this.stop(); + return; + } + const delay = Math.min(1000 * 2 ** Math.min(this.attempt++, 5), 30000); + this.options.onStatus(`Connection lost. Reconnecting in ${delay / 1000}s...`); + this.retry = setTimeout(() => { + this.retry = null; + this.start(); + }, delay); + } +} diff --git a/app/native/soniox-stt.ts b/app/native/soniox-stt.ts index 6c9a6f33..e52454e7 100644 --- a/app/native/soniox-stt.ts +++ b/app/native/soniox-stt.ts @@ -1,4 +1,5 @@ import { CLOUD_STT_SAMPLE_RATE, CloudSttClient, CloudSttOptions, toJavaBytes } from "./cloud-stt"; +import { TimedTranscript } from "./transcript-format"; declare const com: any; @@ -30,15 +31,18 @@ export class SonioxSttClient implements CloudSttClient { // PCM queued until the socket opens and the config message is sent. private readonly pendingPcm: Uint8Array[] = []; private pendingFinish = false; + private finishing = false; // Concatenation of all final tokens so far. private finalText = ""; + private formatted = new TimedTranscript(); constructor(private readonly options: SonioxSttOptions) {} start(): void { - if (this.ws) return; + if (this.closed || this.ws) return; this.listenerProxy = new com.faceclaw.app.FaceclawWebSocketListener({ onOpen: () => { + if (this.closed) return; // The API key rides in the config message; there is no auth header. this.trySendText( JSON.stringify({ @@ -47,8 +51,10 @@ export class SonioxSttClient implements CloudSttClient { audio_format: "pcm_s16le", sample_rate: CLOUD_STT_SAMPLE_RATE, num_channels: 1, + enable_speaker_diarization: true, }), ); + if (this.closed) return; this.open = true; for (const chunk of this.pendingPcm.splice(0)) { this.sendPcm(chunk); @@ -58,21 +64,25 @@ export class SonioxSttClient implements CloudSttClient { this.finish(); } this.options.onStatus("Listening (Soniox)..."); + this.options.onReady?.(); + }, + onTextMessage: (message: string) => { + if (!this.closed) this.handleMessage(String(message)); }, - onTextMessage: (message: string) => this.handleMessage(String(message)), onClosed: () => { this.open = false; + if (!this.closed) this.options.onDisconnected?.("Soniox connection closed."); }, onFailure: (message: string) => { if (this.closed) return; - this.options.onError(`Soniox connection failed: ${String(message)}`); + this.options.onDisconnected?.(`Soniox connection failed: ${String(message)}`); }, }); try { this.ws = new com.faceclaw.app.FaceclawWebSocket(WS_URL, this.listenerProxy, null, null); this.options.onStatus("Connecting to Soniox..."); } catch (error) { - this.options.onError(`Soniox connection failed: ${String((error as Error)?.message ?? error)}`); + this.options.onDisconnected?.(`Soniox connection failed: ${String((error as Error)?.message ?? error)}`); } } @@ -89,6 +99,7 @@ export class SonioxSttClient implements CloudSttClient { /** End of utterance: an empty text frame tells Soniox the audio is done. */ finish(): void { if (this.closed) return; + this.finishing = true; if (this.open) { this.trySendText(""); } else { @@ -96,8 +107,13 @@ export class SonioxSttClient implements CloudSttClient { } } + commitSegment(): void { + if (!this.closed && this.open) this.trySendText(JSON.stringify({ type: "finalize" })); + } + stop(): void { this.closed = true; + this.open = false; if (this.ws) { try { this.ws.close(1000, "bye"); @@ -107,21 +123,26 @@ export class SonioxSttClient implements CloudSttClient { this.ws = null; } this.listenerProxy = null; + this.pendingPcm.length = 0; } private sendPcm(pcm: Uint8Array): void { try { - this.ws?.sendBinary(toJavaBytes(pcm)); + if (this.ws?.sendBinary(toJavaBytes(pcm)) === false) { + this.options.onDisconnected?.("Soniox audio send failed."); + } } catch (error) { - console.warn("soniox send failed", error); + this.options.onDisconnected?.(`Soniox send failed: ${String(error)}`); } } private trySendText(message: string): void { try { - this.ws?.sendText(message); + if (this.ws?.sendText(message) === false) { + this.options.onDisconnected?.("Soniox send failed."); + } } catch (error) { - console.warn("soniox send failed", error); + this.options.onDisconnected?.(`Soniox send failed: ${String(error)}`); } } @@ -133,28 +154,42 @@ export class SonioxSttClient implements CloudSttClient { return; } if (message?.error_code != null) { - this.options.onError(`Soniox: ${String(message.error_message ?? message.error_code)}`); + const error = `Soniox: ${String(message.error_message ?? message.error_code)}`; + if (Number(message.error_code) === 429 || Number(message.error_code) >= 500) this.options.onDisconnected?.(error); + else this.options.onError(error); return; } const tokens = Array.isArray(message?.tokens) ? message.tokens : []; let nonFinal = ""; + let preview: TimedTranscript | null = null; for (const token of tokens) { const tokenText = String(token?.text ?? ""); // Markers emitted by endpoint detection / manual finalize; not speech. - if (tokenText === "" || tokenText === "") continue; + if (tokenText === "" || tokenText === "") { + if (token?.is_final && this.finalText) { + this.options.onTranscript({ text: this.finalText, transcribeText: this.formatted.text, isFinal: true, paragraphBreakAfter: true }); + this.finalText = ""; + this.formatted = new TimedTranscript(); + } + continue; + } if (token?.is_final) { this.finalText += tokenText; + this.formatted.append({ ...token, text: tokenText }); } else { nonFinal += tokenText; + preview ??= this.formatted.copy(); + preview.append({ ...token, text: tokenText }); } } if (message?.finished) { - this.options.onTranscript({ text: this.finalText, isFinal: true }); - this.stop(); + this.options.onTranscript({ text: this.finalText, transcribeText: this.formatted.text, isFinal: true }); + if (this.finishing) this.stop(); + else this.options.onDisconnected?.("Soniox session ended; restarting transcription."); return; } if (tokens.length > 0) { - this.options.onTranscript({ text: this.finalText + nonFinal, isFinal: false }); + this.options.onTranscript({ text: this.finalText + nonFinal, transcribeText: (preview ?? this.formatted).text, isFinal: false }); } } } diff --git a/app/native/speech-pause.ts b/app/native/speech-pause.ts new file mode 100644 index 00000000..08703ead --- /dev/null +++ b/app/native/speech-pause.ts @@ -0,0 +1,44 @@ +/** Sample-clock pause detection: BLE/network delivery timing is not speech timing. */ +export const TRANSCRIPT_PAUSE_MS = 1500; + +export class SpeechPauseDetector { + private noiseFloor = 220; + private speaking = false; + private quietSamples = 0; + + reset(): void { + this.noiseFloor = 220; + this.speaking = false; + this.quietSamples = 0; + } + + /** PCM16LE, 16 kHz. Returns true once per pause after speech. */ + accept(pcm: Uint8Array): boolean { + const count = pcm.length >> 1; + if (!count) return false; + let squares = 0; + for (let i = 0; i < count; i++) { + const unsigned = pcm[i * 2]! | (pcm[i * 2 + 1]! << 8); + const sample = unsigned >= 32768 ? unsigned - 65536 : unsigned; + squares += sample * sample; + } + const rms = Math.sqrt(squares / count); + const threshold = this.noiseFloor * (this.speaking ? 1.8 : 3); + if (rms >= threshold) { + this.speaking = true; + this.quietSamples = 0; + } else { + // Slowly follow background levels without learning speech as noise. + this.noiseFloor = Math.max(220, this.noiseFloor * 0.98 + rms * 0.02); + if (this.speaking) { + this.quietSamples += count; + if (this.quietSamples >= 16 * TRANSCRIPT_PAUSE_MS) { + this.speaking = false; + this.quietSamples = 0; + return true; + } + } + } + return false; + } +} diff --git a/app/native/transcript-format.ts b/app/native/transcript-format.ts new file mode 100644 index 00000000..23915030 --- /dev/null +++ b/app/native/transcript-format.ts @@ -0,0 +1,32 @@ +import { TRANSCRIPT_PAUSE_MS } from "./speech-pause"; + +export type TranscriptToken = { text: string; start_ms?: number; end_ms?: number; speaker?: string }; + +/** A separate presentation buffer keeps paragraph breaks out of dictation. */ +export class TimedTranscript { + text = ""; + private endMs: number | undefined; + private speaker: string | undefined; + + copy(): TimedTranscript { + const copy = new TimedTranscript(); + copy.text = this.text; + copy.endMs = this.endMs; + copy.speaker = this.speaker; + return copy; + } + + append(token: TranscriptToken): void { + const changedSpeaker = token.speaker != null && this.speaker != null && token.speaker !== this.speaker; + const paused = token.start_ms != null && this.endMs != null && token.start_ms - this.endMs >= TRANSCRIPT_PAUSE_MS; + if (token.text.trim() && this.text && (changedSpeaker || paused)) { + this.text = `${this.text.trimEnd()}\n${token.text.trimStart()}`; + } else { + this.text += token.text; + } + if (token.text.trim()) { + if (token.end_ms != null) this.endMs = token.end_ms; + if (token.speaker != null) this.speaker = token.speaker; + } + } +} diff --git a/app/native/voice-control.ts b/app/native/voice-control.ts index 63bf8f9c..f0e5669a 100644 --- a/app/native/voice-control.ts +++ b/app/native/voice-control.ts @@ -1,6 +1,8 @@ import { Utils } from "@nativescript/core"; -import { CloudSttClient } from "./cloud-stt"; +import { type CloudSttTranscriptEvent, type CloudSttOptions, CloudSttClient } from "./cloud-stt"; +import { ReconnectingSttClient } from "./reconnecting-stt"; +import { SpeechPauseDetector } from "./speech-pause"; import { ElevenLabsSttClient } from "./elevenlabs-stt"; import { OpenAiRealtimeSttClient } from "./openai-stt"; import { SonioxSttClient } from "./soniox-stt"; @@ -14,7 +16,7 @@ export type VoiceControlState = { export type VoiceProviderKind = "onboard" | "elevenlabs" | "whisper" | "soniox"; -export type VoiceTranscriptEvent = { +export type VoiceTranscriptEvent = CloudSttTranscriptEvent & { /** * Complete best transcript of the current utterance. REPLACE semantics — * render as-is, replacing any previous partial. Not a delta. @@ -80,6 +82,8 @@ export type FrameMetaListener = (meta: MicFrameMeta) => void; export class FaceclawVoiceControlBridge { private readonly statusListeners = new Set<(state: VoiceControlState) => void>(); private readonly transcriptListeners = new Set<(event: VoiceTranscriptEvent) => void>(); + private readonly speechPauseListeners = new Set<() => void>(); + private readonly speechPause = new SpeechPauseDetector(); private readonly speechEndListeners = new Set<() => void>(); private controller: any | null = null; private listenerProxy: any | null = null; @@ -132,6 +136,12 @@ export class FaceclawVoiceControlBridge { return () => this.speechEndListeners.delete(listener); } + /** Pauses in on-device recognition, for Transcribe's presentation only. */ + onSpeechPause(listener: () => void): () => void { + this.speechPauseListeners.add(listener); + return () => this.speechPauseListeners.delete(listener); + } + /** Begin push-to-talk capture (momentary; released with stopPushToTalk). */ startPushToTalk(options: PushToTalkOptions): void { this.acquireCapture("ptt", options); @@ -251,6 +261,9 @@ export class FaceclawVoiceControlBridge { this.controller?.clearSpeakerVerification(); } + this.speechPause.reset(); + // A previous push-to-talk commit may still be awaiting its final result. + this.cloudClient?.stop(); const cloudClient = this.createCloudClient(options); if (cloudClient) { this.cloudClient = cloudClient; @@ -274,18 +287,20 @@ export class FaceclawVoiceControlBridge { if (options.provider === "onboard") return null; const sttOptions = { apiKey: "", - onTranscript: (event: { text: string; isFinal: boolean }) => - this.emitTranscript(event.text, event.isFinal), + onTranscript: (event: CloudSttTranscriptEvent) => + this.emitTranscript(event.text, event.isFinal, event), onStatus: (status: string) => this.setStatus(status), onError: (message: string) => this.setStatus(message), }; + const reconnecting = (create: (options: CloudSttOptions) => CloudSttClient, apiKey: string) => + new ReconnectingSttClient(create, { ...sttOptions, apiKey }, () => this.captureHolders.has("continuous")); if (options.provider === "elevenlabs") { const apiKey = options.elevenLabsApiKey.trim(); if (!apiKey) { this.setStatus("No ElevenLabs key set; using on-device voice."); return null; } - return new ElevenLabsSttClient({ ...sttOptions, apiKey }); + return reconnecting((config) => new ElevenLabsSttClient(config), apiKey); } if (options.provider === "soniox") { const apiKey = options.sonioxApiKey.trim(); @@ -293,14 +308,14 @@ export class FaceclawVoiceControlBridge { this.setStatus("No Soniox key set; using on-device voice."); return null; } - return new SonioxSttClient({ ...sttOptions, apiKey }); + return reconnecting((config) => new SonioxSttClient(config), apiKey); } const apiKey = options.openAiApiKey.trim(); if (!apiKey) { this.setStatus("No OpenAI key set; using on-device voice."); return null; } - return new OpenAiRealtimeSttClient({ ...sttOptions, apiKey }); + return reconnecting((config) => new OpenAiRealtimeSttClient(config), apiKey); } private releaseCapture(holder: CaptureHolder, commit: boolean): void { @@ -432,6 +447,9 @@ export class FaceclawVoiceControlBridge { onPcm: (pcm: any) => { const bytes = toUint8Array(pcm); this.cloudClient?.acceptPcm(bytes); + if (!this.cloudClient && this.captureHolders.has("continuous") && this.speechPause.accept(bytes)) { + for (const listener of this.speechPauseListeners) listener(); + } if (this.rawPcmListeners.size > 0) { this.rawPcmListeners.forEach((listener) => listener(bytes)); } @@ -459,15 +477,16 @@ export class FaceclawVoiceControlBridge { this.controller.setListener(this.listenerProxy); } - private emitTranscript(text: string, isFinal: boolean): void { + private emitTranscript(text: string, isFinal: boolean, metadata?: CloudSttTranscriptEvent): void { // "My voice only": a rejected session's final transcript is emptied so // nothing downstream (assistant, dictation) acts on another speaker's // words. The verification verdict is posted before the final, and cloud // finals arrive over the network later still, so the flag is settled. if (isFinal && this.verificationActive && this.verificationRejected) { text = ""; + metadata = undefined; } - const event = { text, isFinal }; + const event = { ...metadata, text, isFinal }; for (const listener of this.transcriptListeners) { listener(event); }