diff --git a/App_Resources/iOS/src/FaceclawSpeechAnalyzer.swift b/App_Resources/iOS/src/FaceclawSpeechAnalyzer.swift new file mode 100644 index 00000000..d9624ed3 --- /dev/null +++ b/App_Resources/iOS/src/FaceclawSpeechAnalyzer.swift @@ -0,0 +1,273 @@ +@preconcurrency import AVFoundation +import CoreMedia +import Foundation +import Speech + +/// Long-form on-device transcription with SpeechAnalyzer and SpeechTranscriber +/// (iOS 26+), for continuous capture (Transcribe, Teleprompter, Calculator): +/// FaceclawSpeech's SFSpeechRecognizer only handles short dictations. It takes +/// the same 16 kHz mono PCM16LE the cloud providers get (see +/// app/native/speech-analyzer-stt.ios.ts) and reports JSON events on the main +/// queue through eventHandler: +/// +/// {"kind":"status","message":String} +/// {"kind":"ready","locale":String} audio is accepted from here on +/// {"kind":"transcript","text":String,"final":Bool, +/// "start":Double?,"end":Double?, the result's audio range +/// "speechEnd":Double?} when its last word ends +/// {"kind":"finished"} after finish(), once the last result is in +/// {"kind":"error","message":String} +/// +/// Times are seconds of audio since start. Volatile results (final: false) +/// cover all audio not finalized yet, replacing the previous volatile one; +/// their word times are only as fine as the result's range. A final's first +/// word reaches back into the silence before it, so only word ends are +/// reported. +/// +/// There is no way to settle text at a pause: on macOS 26.4 any mid-stream +/// SpeechAnalyzer.finalize(through:) left the next phrase's final as dots, +/// while the model splits its finals at pauses on its own. +/// Called from JavaScript, so on the main thread. +@MainActor +@objc(FaceclawSpeechAnalyzer) +@objcMembers +public class FaceclawSpeechAnalyzer: NSObject { + public var eventHandler: ((String) -> Void)? + // An AnalyzerEngine, typed loosely so the class itself loads before iOS 26. + private var engine: AnyObject? + + /// Whether this device can transcribe with SpeechTranscriber (iOS 26 and supported hardware). + public static func isSupported() -> Bool { + if #available(iOS 26.0, *) { return SpeechTranscriber.isAvailable } + return false + } + + /// `language` is a locale identifier ("en", "en-US"), or "" for the device's preferred language. + public func start(language: String) { + cancel() + guard #available(iOS 26.0, *) else { + emit(["kind": "error", "message": "On-device transcription needs iOS 26 or later."]) + return + } + let engine = AnalyzerEngine { [weak self] event in self?.emit(event) } + self.engine = engine + engine.start(language: language) + } + + /// 16 kHz mono PCM16LE; ignored until "ready". + public func acceptPcm(_ pcm: Data) { + if #available(iOS 26.0, *) { (engine as? AnalyzerEngine)?.accept(pcm) } + } + + /// End of audio: finalize the rest, then report "finished". + public func finish() { + if #available(iOS 26.0, *) { (engine as? AnalyzerEngine)?.finish() } + } + + /// Abandon the session without further events. + public func cancel() { + if #available(iOS 26.0, *) { (engine as? AnalyzerEngine)?.cancel() } + engine = nil + } + + private func emit(_ event: [String: Any]) { + // JSONSerialization raises (not throws) on NaN and other invalid values. + guard JSONSerialization.isValidJSONObject(event), + let data = try? JSONSerialization.data(withJSONObject: event), + let json = String(data: data, encoding: .utf8) else { return } + eventHandler?(json) + } +} + +@available(iOS 26.0, *) +@MainActor +private final class AnalyzerEngine { + private static let sampleRate = 16000.0 + private let emit: ([String: Any]) -> Void + private let pcmFormat = AVAudioFormat(commonFormat: .pcmFormatInt16, sampleRate: AnalyzerEngine.sampleRate, channels: 1, interleaved: true)! + private var analyzer: SpeechAnalyzer? + private var input: AsyncStream.Continuation? + private var analyzerFormat: AVAudioFormat? + // Nil when the analyzer takes the input format as it is. + private var converter: AVAudioConverter? + private var ready = false + private var finishing = false + private var closed = false + private var setup: Task? + private var results: Task? + + init(emit: @escaping ([String: Any]) -> Void) { self.emit = emit } + + func start(language: String) { + setup = Task { [weak self] in await self?.prepare(language: language) } + } + + /// Events stop once the session is cancelled, even from work still winding down. + private func send(_ event: [String: Any]) { + if !closed { emit(event) } + } + + private func prepare(language: String) async { + send(["kind": "status", "message": "Preparing on-device transcription…"]) + let identifier = language.isEmpty ? (Locale.preferredLanguages.first ?? "en-US") : language + guard let locale = await SpeechTranscriber.supportedLocale(equivalentTo: Locale(identifier: identifier)) else { + fail("On-device transcription doesn't support \(identifier). Choose a cloud or Whisper server transcription provider (Settings > Voice).") + return + } + if closed { return } + let transcriber = SpeechTranscriber(locale: locale, transcriptionOptions: [], + reportingOptions: [.volatileResults], attributeOptions: [.audioTimeRange]) + do { + // Keep the model installed between sessions (best effort). + if !(await AssetInventory.reservedLocales).contains(locale) { + _ = try? await AssetInventory.reserve(locale: locale) + } + if closed { return } + if let request = try await AssetInventory.assetInstallationRequest(supporting: [transcriber]) { + if closed { return } + try await download(request) + } + if closed { return } + guard let format = await SpeechAnalyzer.bestAvailableAudioFormat(compatibleWith: [transcriber]) else { + fail("On-device transcription has no usable audio format.") + return + } + if closed { return } + if format != pcmFormat { + guard let converter = AVAudioConverter(from: pcmFormat, to: format) else { + fail("On-device transcription can't convert the microphone audio.") + return + } + self.converter = converter + } + analyzerFormat = format + let analyzer = SpeechAnalyzer(modules: [transcriber]) + let (stream, input) = AsyncStream.makeStream() + self.analyzer = analyzer + self.input = input + results = Task { [weak self] in await self?.collect(transcriber) } + try await analyzer.prepareToAnalyze(in: format) + try await analyzer.start(inputSequence: stream) + if closed { return } + ready = true + send(["kind": "ready", "locale": locale.identifier]) + if finishing { await finishAnalysis() } + } catch { + fail("On-device transcription: \(error.localizedDescription)") + } + } + + /// Fetch the locale's model, reporting progress while it downloads. + private func download(_ request: AssetInstallationRequest) async throws { + let progress = request.progress + let reporter = Task { [weak self] in + while !Task.isCancelled { + let percent = Int((progress.fractionCompleted * 100).rounded()) + self?.send(["kind": "status", "message": "Downloading on-device speech model… \(percent)%"]) + try? await Task.sleep(nanoseconds: 1_000_000_000) + } + } + defer { reporter.cancel() } + try await request.downloadAndInstall() + } + + func accept(_ pcm: Data) { + guard ready, !finishing, !closed, let input, let analyzerFormat else { return } + let frames = AVAudioFrameCount(pcm.count / 2) + guard frames > 0, let buffer = AVAudioPCMBuffer(pcmFormat: pcmFormat, frameCapacity: frames), + let channel = buffer.int16ChannelData else { return } + buffer.frameLength = frames + pcm.withUnsafeBytes { raw in + if let base = raw.baseAddress { memcpy(channel[0], base, Int(frames) * 2) } + } + guard let converter else { + input.yield(AnalyzerInput(buffer: buffer)) + return + } + let capacity = AVAudioFrameCount((Double(frames) * analyzerFormat.sampleRate / AnalyzerEngine.sampleRate).rounded(.up)) + 32 + guard let converted = AVAudioPCMBuffer(pcmFormat: analyzerFormat, frameCapacity: capacity) else { return } + var consumed = false + var error: NSError? + // noDataNow, not endOfStream: the converter keeps its resampling state for the next chunk. + converter.convert(to: converted, error: &error) { _, status in + if consumed { + status.pointee = .noDataNow + return nil + } + consumed = true + status.pointee = .haveData + return buffer + } + if error == nil && converted.frameLength > 0 { input.yield(AnalyzerInput(buffer: converted)) } + } + + func finish() { + guard !finishing, !closed else { return } + finishing = true + // Before "ready", prepare() finishes once the analyzer is running. + if ready { Task { await finishAnalysis() } } + } + + private func finishAnalysis() async { + input?.finish() + input = nil + do { + try await analyzer?.finalizeAndFinishThroughEndOfInput() + } catch { + fail("On-device transcription: \(error.localizedDescription)") + } + // collect() reports "finished" once the results sequence ends. + } + + func cancel() { + guard !closed else { return } + closed = true + input?.finish() + input = nil + setup?.cancel() + results?.cancel() + if let analyzer { Task { await analyzer.cancelAndFinishNow() } } + analyzer = nil + } + + private func collect(_ transcriber: SpeechTranscriber) async { + do { + for try await result in transcriber.results { + if closed { return } + send(event(for: result)) + } + send(["kind": "finished"]) + } catch { + fail("On-device transcription: \(error.localizedDescription)") + } + } + + private func event(for result: SpeechTranscriber.Result) -> [String: Any] { + let text = result.text + var speechEnd: Double? + for run in text.runs { + guard let range = run.audioTimeRange, + !String(text[run.range].characters).trimmingCharacters(in: .whitespacesAndNewlines).isEmpty else { continue } + speechEnd = finite(range.end) ?? speechEnd + } + return [ + "kind": "transcript", + "text": String(text.characters), + "final": result.isFinal, + "start": finite(result.range.start) ?? NSNull(), + "end": finite(result.range.end) ?? NSNull(), + "speechEnd": speechEnd ?? NSNull(), + ] + } + + private func finite(_ time: CMTime) -> Double? { + let seconds = time.seconds + return seconds.isFinite ? seconds : nil + } + + private func fail(_ message: String) { + guard !closed else { return } + emit(["kind": "error", "message": message]) + cancel() + } +} diff --git a/app/g2/ios-preview-controller.ts b/app/g2/ios-preview-controller.ts index e9af4fb7..c10f984e 100644 --- a/app/g2/ios-preview-controller.ts +++ b/app/g2/ios-preview-controller.ts @@ -1054,7 +1054,7 @@ export class IosPreviewController { this.continuousClients.set(client, options) if (!iosVoiceInput.supportsContinuous(this.voiceProvider())) { this.continuousClients.delete(client) - this.onError('Transcribe on iOS needs a cloud or Whisper server transcription provider (Settings > Voice > Transcription Provider).') + this.onError('On-device transcription for Transcribe needs iOS 26 or later; before that, choose a cloud or Whisper server transcription provider (Settings > Voice > Transcription Provider).') return } this.syncContinuousRecording() diff --git a/app/native/speech-analyzer-stt.ios.ts b/app/native/speech-analyzer-stt.ios.ts new file mode 100644 index 00000000..a7afa7f1 --- /dev/null +++ b/app/native/speech-analyzer-stt.ios.ts @@ -0,0 +1,193 @@ +import { CLOUD_STT_SAMPLE_RATE, type CloudSttClient, type CloudSttOptions } from "./cloud-stt"; +import { ReconnectingSttClient } from "./reconnecting-stt"; +import { TRANSCRIPT_PAUSE_MS } from "./speech-pause"; + +declare const FaceclawSpeechAnalyzer: any; + +/** + * How long finish() waits for the last results: the model only has to settle + * the few seconds of audio it hadn't finalized yet. + */ +export const SPEECH_ANALYZER_FINAL_TIMEOUT_MS = 10000; + +/** + * A pause is detected TRANSCRIPT_PAUSE_MS after the last loud audio. A final + * whose last word ends within this much before that (the model's word ends + * against the detector's level threshold) leaves nothing before the pause + * unfinalized, so its paragraph is complete. + */ +const PARAGRAPH_TAIL_S = TRANSCRIPT_PAUSE_MS / 1000 + 0.25; + +type AnalyzerEvent = { + kind: string; + message?: string; + locale?: string; + text?: string; + final?: boolean; + start?: number | null; + end?: number | null; + speechEnd?: number | null; +}; + +/** Whether Apple's SpeechAnalyzer (iOS 26+, supported hardware) can run continuous capture. */ +export function speechAnalyzerSupported(): boolean { + try { + return typeof FaceclawSpeechAnalyzer !== "undefined" && Boolean(FaceclawSpeechAnalyzer.isSupported()); + } catch { + return false; + } +} + +/** + * On-device long-form transcription for continuous capture, wrapped like a + * cloud provider: it buffers audio until the model is ready (downloading it on + * first use) and gets pause commits. + */ +export function createSpeechAnalyzerClient( + options: Omit, + continuous: () => boolean, +): ReconnectingSttClient { + return new ReconnectingSttClient( + (clientOptions) => new SpeechAnalyzerSttClient(clientOptions), + { ...options, apiKey: "", language: (options.language ?? "").trim() }, + continuous, + SPEECH_ANALYZER_FINAL_TIMEOUT_MS, + ); +} + +/** + * Transcribes PCM with FaceclawSpeechAnalyzer (FaceclawSpeechAnalyzer.swift). + * The model finalizes text in its own pieces, a few seconds behind, and can't + * be made to settle at a pause; this gathers its pieces into paragraphs ended + * by pauses, like the cloud providers' segments. A paragraph is shown as + * provisional text and sent as one final (paragraphBreakAfter) once a final + * reaches its pause, and the last one as a plain final when the audio ends. + * + * Only finals place breaks, by when their last word ends (the model's + * word-end times are accurate; first words' starts reach back into the + * silence before them, and volatile results only carry the range of all audio + * not finalized yet). + */ +export class SpeechAnalyzerSttClient implements CloudSttClient { + private native: any = null; + private samples = 0; + private finishing = false; + // Audio times (s) of pauses whose paragraph hasn't ended yet. + private pauses: number[] = []; + // Finalized text since the last paragraph break, when its last word ends, + // and the model's provisional rest. + private paragraph: string[] = []; + private paragraphEnd: number | null = null; + private volatile = ""; + + constructor(private readonly options: CloudSttOptions) {} + + start(): void { + if (this.native) return; + const native = FaceclawSpeechAnalyzer.new(); + native.eventHandler = (json: string) => { + if (this.native === native) this.receive(JSON.parse(json)); + }; + this.native = native; + native.startWithLanguage(this.options.language ?? ""); + } + + acceptPcm(pcm: Uint8Array): void { + if (!this.native || this.finishing || !pcm.length) return; + this.samples += pcm.length >> 1; + const copy = pcm.slice(); + this.native.acceptPcm(NSData.dataWithBytesLength(interop.handleof(copy.buffer), copy.byteLength)); + } + + /** A pause: it ends the paragraph once the words before it are final. */ + commitSegment(): void { + if (!this.native || this.finishing) return; + this.pauses.push(this.samples / CLOUD_STT_SAMPLE_RATE); + if (this.endCompleteParagraph() && this.volatile) this.options.onTranscript({ text: this.liveText(), isFinal: false }); + } + + finish(): void { + if (!this.native || this.finishing) return; + this.finishing = true; + this.native.finish(); + } + + stop(): void { + const native = this.native; + this.native = null; + native?.cancel(); + } + + private receive(event: AnalyzerEvent): void { + switch (event.kind) { + case "status": + this.options.onStatus(event.message ?? ""); + break; + case "ready": + this.options.onStatus("Listening (on-device)..."); + this.options.onReady?.(); + break; + case "transcript": + this.transcript(event); + break; + case "finished": { + const text = this.liveText(); + this.paragraph = []; + this.volatile = ""; + this.stop(); + this.options.onTranscript({ text, isFinal: true }); + break; + } + case "error": + this.stop(); + this.options.onError(event.message ?? "On-device transcription failed."); + break; + } + } + + private transcript(event: AnalyzerEvent): void { + const text = (event.text ?? "").trim(); + let ended = false; + if (!event.final) { + this.volatile = text; + } else { + this.volatile = ""; + if (text) { + const end = event.speechEnd ?? event.end ?? null; + // Words ending after a pause come after it. (One whose words run + // across it, if the model didn't split there, breaks a little early.) + if (end !== null && this.pauses.some((pause) => end > pause)) { + this.pauses = this.pauses.filter((pause) => pause >= end); + this.endParagraph(); + } + this.paragraph.push(text); + this.paragraphEnd = end; + ended = this.endCompleteParagraph(); + } + } + // The paragraph's final already cleared the provisional text. + if (!ended || this.volatile) this.options.onTranscript({ text: this.liveText(), isFinal: false }); + } + + /** End the paragraph if its last word ends just before a pause. */ + private endCompleteParagraph(): boolean { + const end = this.paragraphEnd; + if (!this.paragraph.length || end === null) return false; + const reached = (pause: number) => end >= pause - PARAGRAPH_TAIL_S; + if (!this.pauses.some(reached)) return false; + this.pauses = this.pauses.filter((pause) => !reached(pause)); + this.endParagraph(); + return true; + } + + private endParagraph(): void { + const text = this.paragraph.join(" "); + this.paragraph = []; + this.paragraphEnd = null; + if (text) this.options.onTranscript({ text, isFinal: true, paragraphBreakAfter: true }); + } + + private liveText(): string { + return [...this.paragraph, this.volatile].filter(Boolean).join(" "); + } +} diff --git a/app/native/voice-control.ios.ts b/app/native/voice-control.ios.ts index 2699515a..407ab3cd 100644 --- a/app/native/voice-control.ios.ts +++ b/app/native/voice-control.ios.ts @@ -2,6 +2,7 @@ import type { VoiceControlState, VoiceTranscriptEvent } from './voice-control' import type { CloudSttTranscriptEvent } from './cloud-stt' import type { ReconnectingSttClient } from './reconnecting-stt' import { createCloudSttClient, usesCloudStt, type VoiceProviderSelection } from './cloud-stt-provider' +import { createSpeechAnalyzerClient, speechAnalyzerSupported } from './speech-analyzer-stt.ios' import { WavRecorder } from './wav-recorder.ios' import { pcmLevel } from '../util/audio-level' @@ -18,10 +19,12 @@ const RAW_RETRY_LIMIT = 5 /** iOS implementation of the shared voice dialog's event bridge. Audio comes * from the glasses when connected, or the default phone input in preview mode, * and is transcribed by Apple's on-device recognizer or a cloud provider. - * Continuous capture (Transcribe, Teleprompter) needs a cloud provider: only - * the PCM capture runs open-ended, and pauses commit its segments. When the - * session or the app lifecycle cuts one short it is drained, not discarded; - * restarting it is the controller's call. + * Continuous capture (Transcribe, Teleprompter) runs on the PCM capture, which + * is open-ended and has pauses commit its segments, so it needs a cloud + * provider or, on-device, SpeechAnalyzer (iOS 26+): FaceclawSpeech's + * recognizer only handles short dictations. When the session or the app + * lifecycle cuts one short it is drained, not discarded; restarting it is the + * controller's call. */ export class IosVoiceControlBridge { private native: any = null @@ -40,10 +43,13 @@ export class IosVoiceControlBridge { private audioTimer: ReturnType | null = null private packets = 0 private log: (message: string) => void = () => {} - // Non-null while a cloud provider owns the transcript; FaceclawSpeech then + // Non-null while a PCM transcriber (a cloud provider, or SpeechAnalyzer for + // on-device continuous capture) owns the transcript; FaceclawSpeech then // only supplies PCM. cloudEndStatus is set once the audio has ended and the // provider was asked to finalize; it is the status shown when it does. private cloud: ReconnectingSttClient | null = null + // What transcribes the current capture, for the log. + private recognition = 'on-device' private cloudEndStatus: string | null = null private cloudTimer: ReturnType | null = null // Open-ended capture: runs until stopContinuousCapture, and push-to-talk @@ -137,7 +143,7 @@ export class IosVoiceControlBridge { this.native.acceptPacket(NSData.dataWithBytesLength(interop.handleof(copy.buffer), copy.byteLength)) }) if (generation !== this.generation || !this.capturing) return - log(`Voice: glasses microphone enabled; using ${this.cloud ? 'cloud' : 'on-device'} recognition`) + log(`Voice: glasses microphone enabled; using ${this.recognition} recognition`) this.audioTimer = setTimeout(() => { this.audioTimer = null if (generation === this.generation && this.capturing && !this.packets) { @@ -158,18 +164,27 @@ export class IosVoiceControlBridge { this.completion = new Promise(resolve => { this.resolveCompletion = resolve }) const error = this.startNative(voice, endpointing, true) if (error) { this.stop(); this.setStatus(error); this.emitEnd(); return } - log(`Voice: phone default microphone enabled; using ${this.cloud ? 'cloud' : 'on-device'} recognition`) + log(`Voice: phone default microphone enabled; using ${this.recognition} recognition`) } /** Starts the native capture for the chosen provider; returns an actionable error, or ''. */ private startNative(voice: VoiceProviderSelection, endpointing: boolean, phone: boolean): string { const native = this.ensureNative() const generation = this.generation const current = () => generation === this.generation - const cloud = createCloudSttClient(voice, { - onTranscript: event => { if (current()) this.receiveCloudTranscript(event) }, - onStatus: status => { if (current()) this.setStatus(status) }, - onError: message => { if (current()) { this.stop(); this.setStatus(message); this.emitEnd() } }, - }, () => this.continuous) + const options = { + language: voice.language, + onTranscript: (event: CloudSttTranscriptEvent) => { if (current()) this.receiveCloudTranscript(event) }, + onStatus: (status: string) => { if (current()) this.setStatus(status) }, + onError: (message: string) => { if (current()) { this.stop(); this.setStatus(message); this.emitEnd() } }, + } + const continuous = () => this.continuous + let cloud = createCloudSttClient(voice, options, continuous) + this.recognition = cloud ? 'cloud' : 'on-device' + // Continuous capture outlasts FaceclawSpeech's recognizer; SpeechAnalyzer takes the PCM instead. + if (!cloud && this.continuous && speechAnalyzerSupported()) { + cloud = createSpeechAnalyzerClient(options, continuous) + this.recognition = 'on-device (SpeechAnalyzer)' + } if (!cloud) return String((phone ? native.startPhoneWithEndpointing(endpointing) : native.startWithEndpointing(endpointing)) ?? '') const error = String(native.startPcmCaptureWithEndpointingPhone(endpointing, phone) ?? '') if (error) return error @@ -182,8 +197,11 @@ export class IosVoiceControlBridge { if (this.continuous) { if (this.capturing) void this.finalizeCapture('Paused while Faceclaw is in the background.'); return } this.stop(); this.setStatus('Phone voice input stopped.'); this.emitEnd() } - /** Whether continuous capture can run with this provider (Apple's recognizer can't). */ - supportsContinuous(voice: VoiceProviderSelection): boolean { return usesCloudStt(voice) } + /** + * Whether continuous capture can run with this provider: a cloud one, or + * on-device with SpeechAnalyzer (FaceclawSpeech's recognizer can't). + */ + supportsContinuous(voice: VoiceProviderSelection): boolean { return usesCloudStt(voice) || speechAnalyzerSupported() } get continuousActive(): boolean { return this.continuous && (this.capturing || this.finalizing) } /** Where a listening continuous capture gets its audio; null when none is listening. */ get continuousSource(): 'glasses' | 'phone' | null { return this.continuous && this.capturing ? this.source : null } diff --git a/tests/ios-voice.test.cjs b/tests/ios-voice.test.cjs index 4624bedb..66c92440 100644 --- a/tests/ios-voice.test.cjs +++ b/tests/ios-voice.test.cjs @@ -7,7 +7,7 @@ function harness() { const native = { startWithEndpointing(endpointing) { this.endpointing = endpointing; return ''; }, startPhoneWithEndpointing(endpointing) { this.endpointing = endpointing; this.phone = true; return this.error ?? ''; }, startPcmCaptureWithEndpointingPhone(endpointing, phone) { this.pcmCapture = { endpointing, phone }; return this.error ?? ''; }, cancel() { this.cancels++; }, cancels: 0, finish() { this.finished = true; }, acceptPacket: p => audio.push(p) }; - const audio = [], timers = new Map(), commands = [], text = [], clouds = []; + const audio = [], timers = new Map(), commands = [], text = [], clouds = [], analyzers = []; // Stands in for cloud-stt-provider: any provider but "onboard" is a keyed cloud one. const provider = { usesCloudStt: voice => voice.provider !== 'onboard', @@ -19,6 +19,17 @@ function harness() { clouds.push(client); return client; }, }; + // Stands in for speech-analyzer-stt: SpeechAnalyzer (iOS 26+) when the test says the device has it. + let analyzerSupported = false; + const analyzer = { + speechAnalyzerSupported: () => analyzerSupported, + createSpeechAnalyzerClient(options, continuous) { + const client = { options, continuous, pcm: [], started: false, finished: 0, stopped: false, + start() { this.started = true; }, acceptPcm(bytes) { this.pcm.push([...bytes]); }, + finish() { this.finished++; }, stop() { this.stopped = true; } }; + analyzers.push(client); return client; + }, + }; const recordings = []; class FakeRecorder { static open(name) { const r = new FakeRecorder(); r.name = name; r.path = `/Downloads/${name}.wav`; r.pcm = []; r.closed = false; recordings.push(r); return r; } @@ -32,7 +43,8 @@ function harness() { requestMicrophoneAuthorization: done => { microphoneRequests++; done(3); } }, setTimeout: fn => { timers.set(fn, fn); return fn; }, clearTimeout: fn => timers.delete(fn), NSData: { dataWithBytesLength: (b, n) => Buffer.from(b, 0, n) }, interop: { handleof: b => b, bufferFromData: d => d }, - require: name => name === './cloud-stt-provider' ? provider : name === './wav-recorder.ios' ? { WavRecorder: FakeRecorder } : {} }; + require: name => name === './cloud-stt-provider' ? provider : name === './speech-analyzer-stt.ios' ? analyzer + : name === './wav-recorder.ios' ? { WavRecorder: FakeRecorder } : {} }; vm.runInNewContext(ts.transpileModule(fs.readFileSync('app/native/voice-control.ios.ts', 'utf8'), { compilerOptions: { module: ts.ModuleKind.CommonJS, target: ts.ScriptTarget.ES2020 } }).outputText, sandbox); const bridge = new sandbox.exports.IosVoiceControlBridge(); @@ -43,7 +55,8 @@ function harness() { return Promise.resolve(); } }; const onboard = { provider: 'onboard', elevenLabsApiKey: '', openAiApiKey: '', sonioxApiKey: '' }; - return { bridge, native, session, audio, commands, text, timers, clouds, recordings, release: () => release(), + return { bridge, native, session, audio, commands, text, timers, clouds, analyzers, recordings, release: () => release(), + analyzerSupported: value => { analyzerSupported = value; }, onboard, cloud: { ...onboard, provider: 'soniox', sonioxApiKey: 'key' }, microphoneAuthorization: value => { microphoneAuthorization = value; }, microphoneRequests: () => microphoneRequests, authorization: value => { authorization = value; }, permissionRequests: () => permissionRequests, @@ -248,6 +261,38 @@ test('continuous capture needs a cloud provider, shares its mic with push-to-tal assert.ok(ends >= 1); }); +test('on-device continuous capture runs on SpeechAnalyzer where supported; push-to-talk keeps the dictation recognizer', async () => { + const h = harness(); + assert.equal(h.bridge.supportsContinuous(h.onboard), false, 'before iOS 26 on-device needs a cloud provider'); + h.analyzerSupported(true); + assert.equal(h.bridge.supportsContinuous(h.onboard), true); + const start = h.bridge.startGlassesCapture(h.session, h.onboard, () => {}, false, true); h.release(); await start; + assert.deepEqual({ ...h.native.pcmCapture }, { endpointing: false, phone: false }, 'native only decodes PCM'); + assert.equal(h.native.endpointing, undefined, "Apple's dictation recognizer stays idle"); + const [analyzer] = h.analyzers; + assert.equal(analyzer.started, true); assert.equal(analyzer.continuous(), true); + assert.equal(h.clouds.length, 0); + h.native.pcmHandler(new Uint8Array([1, 2, 3, 4])); + assert.deepEqual(analyzer.pcm, [[1, 2, 3, 4]]); + analyzer.options.onTranscript({ text: 'First thought.', isFinal: true, paragraphBreakAfter: true }); + const stopped = h.bridge.stopContinuousCapture(); + h.event({ kind: 'ended', message: 'Transcription stopped.' }); + assert.equal(analyzer.finished, 1); + analyzer.options.onTranscript({ text: 'The tail.', isFinal: true }); + await stopped; + assert.equal(analyzer.stopped, true); + assert.deepEqual(h.text.map(event => event.text), ['First thought.', 'The tail.']); + // Push-to-talk (not continuous) still uses FaceclawSpeech's own recognizer. + const ptt = h.bridge.startGlassesCapture(h.session, h.onboard, () => {}); h.release(); await ptt; + assert.equal(h.analyzers.length, 1); assert.equal(h.native.endpointing, false); +}); + +test('a cloud provider keeps continuous capture even where SpeechAnalyzer is supported', async () => { + const h = harness(); h.analyzerSupported(true); + const start = h.bridge.startGlassesCapture(h.session, h.cloud, () => {}, false, true); h.release(); await start; + assert.equal(h.clouds.length, 1); assert.equal(h.analyzers.length, 0); +}); + test('push-to-talk after Transcribe closed starts its own capture instead of joining the finalizing one', async () => { const h = harness(); const start = h.bridge.startGlassesCapture(h.session, h.cloud, () => {}, false, true); h.release(); await start; diff --git a/tests/speech-analyzer-stt.test.cjs b/tests/speech-analyzer-stt.test.cjs new file mode 100644 index 00000000..df54ab06 --- /dev/null +++ b/tests/speech-analyzer-stt.test.cjs @@ -0,0 +1,162 @@ +const test = require('node:test'); +const assert = require('node:assert/strict'); +const { loader } = require('./helpers/load-typescript.cjs'); + +/** FaceclawSpeechAnalyzer.swift's surface; `emit` plays its JSON events. */ +class FakeAnalyzer { + static supported = true; + static instances = []; + static new() { const native = new FakeAnalyzer(); FakeAnalyzer.instances.push(native); return native; } + static isSupported() { return FakeAnalyzer.supported; } + constructor() { this.pcm = []; this.finished = 0; this.cancelled = 0; this.language = null; } + startWithLanguage(language) { this.language = language; } + acceptPcm(data) { this.pcm.push([...data]); } + finish() { this.finished++; } + cancel() { this.cancelled++; } + emit(event) { this.eventHandler(JSON.stringify(event)); } +} + +function load() { + FakeAnalyzer.instances = []; + FakeAnalyzer.supported = true; + const timers = new Set(); + const modules = loader({ + FaceclawSpeechAnalyzer: FakeAnalyzer, + NSData: { dataWithBytesLength: (buffer, length) => new Uint8Array(buffer, 0, length) }, + interop: { handleof: buffer => buffer }, + setTimeout(fn) { timers.add(fn); return fn; }, + clearTimeout(fn) { timers.delete(fn); }, + }, {}); + return { ...modules('app/native/speech-analyzer-stt.ios.ts'), timers }; +} + +function client(language = '') { + const { SpeechAnalyzerSttClient } = load(); + const events = [], statuses = [], errors = []; + let ready = 0; + const c = new SpeechAnalyzerSttClient({ + apiKey: '', language, onTranscript: event => events.push({ ...event }), onStatus: status => statuses.push(status), + onError: error => errors.push(error), onReady: () => ready++, + }); + c.start(); + const native = FakeAnalyzer.instances[0]; + return { c, native, events, statuses, errors, ready: () => ready }; +} + +/** `seconds` of 16 kHz PCM16LE silence. */ +const pcm = seconds => new Uint8Array(Math.round(seconds * 16000) * 2); +/** A result as FaceclawSpeechAnalyzer.swift reports it: its audio range and when its last word ends. */ +const result = (text, final, start, end, speechEnd = end) => ({ kind: 'transcript', text, final, start, end, speechEnd }); +const finals = h => h.events.filter(event => event.isFinal); + +test('starts in the requested language and reports readiness as a provider would', () => { + const h = client('es'); + assert.equal(h.native.language, 'es'); + h.native.emit({ kind: 'status', message: 'Downloading on-device speech model… 40%' }); + h.native.emit({ kind: 'ready', locale: 'es_ES' }); + assert.deepEqual(h.statuses, ['Downloading on-device speech model… 40%', 'Listening (on-device)...']); + assert.equal(h.ready(), 1); +}); + +// Recorded from SpeechAnalyzer on macOS 26.4 (scripted speech, 1.5 s quiet, a +// pause, more speech): volatile ranges start where the last final ended, and +// the second final's first word reaches back into the silence (5.04 s; the +// speech starts at 6.18 s). +test('real model output: the first paragraph ends as soon as its last words are final', () => { + const h = client(); + h.native.emit({ kind: 'ready', locale: 'en_US' }); + h.native.emit(result('Hello there.', false, 0, 4.1)); + h.native.emit(result('Hello there.', true, 0, 0.66, 0.6)); + h.native.emit(result(' This is the firstst paragraph of the test spoken ', false, 0.66, 4.1815)); + h.c.acceptPcm(pcm(5.6815)); + h.c.commitSegment(); + assert.equal(finals(h).length, 0, 'words before the pause are still provisional'); + h.native.emit(result(' This is the firstst paragraph of the test spoken before a pause.', false, 0.66, 7.7815)); + h.native.emit(result(' This is the first paragraph of the test, spoken before a pause.', true, 0.66, 4.38, 4.32)); + assert.deepEqual(finals(h), [{ text: 'Hello there. This is the first paragraph of the test, spoken before a pause.', isFinal: true, paragraphBreakAfter: true }]); + h.native.emit(result(' And this is a second', false, 4.38, 8.1815)); + assert.deepEqual(h.events.at(-1), { text: 'And this is a second', isFinal: false }, 'new speech shows in the new paragraph'); + h.native.emit(result(' And this is a second paragraph, which comes after the pause.', true, 5.04, 9.18, 9.12)); + h.c.finish(); + assert.equal(h.native.finished, 1); + h.native.emit({ kind: 'finished' }); + assert.deepEqual(h.events.at(-1), { text: 'And this is a second paragraph, which comes after the pause.', isFinal: true }); + assert.equal(finals(h).length, 2); + assert.equal(h.native.cancelled, 1, 'the finished session is released'); + h.native.emit(result('late', false, 9.2, 9.5)); + assert.equal(h.events.at(-1).text, 'And this is a second paragraph, which comes after the pause.', 'events after stop are ignored'); +}); + +test('a pause after words already final ends the paragraph at once, keeping later provisional text', () => { + const h = client(); + h.native.emit({ kind: 'ready', locale: 'en_US' }); + h.native.emit(result('Hello there.', true, 0, 1.0)); + h.native.emit(result('How are you?', true, 1.0, 1.8)); + h.c.acceptPcm(pcm(3.4)); + h.native.emit(result('Fine', false, 1.8, 3.4)); + h.c.commitSegment(); + assert.deepEqual(h.events.slice(-2), [ + { text: 'Hello there. How are you?', isFinal: true, paragraphBreakAfter: true }, + { text: 'Fine', isFinal: false }, + ]); +}); + +test('a final ending mid-sentence before the pause waits for the rest of the sentence', () => { + const h = client(); + h.native.emit({ kind: 'ready', locale: 'en_US' }); + h.c.acceptPcm(pcm(5.7)); + h.c.commitSegment(); + h.native.emit(result('This is the first paragraph, spoken', true, 0.5, 3.6)); + assert.equal(finals(h).length, 0); + h.native.emit(result('before a pause.', true, 3.6, 4.3)); + assert.deepEqual(finals(h), [{ text: 'This is the first paragraph, spoken before a pause.', isFinal: true, paragraphBreakAfter: true }]); +}); + +test('without a final near the pause, a final ending after it starts the next paragraph', () => { + const h = client(); + h.native.emit({ kind: 'ready', locale: 'en_US' }); + h.native.emit(result('Before.', true, 0, 2.0)); + // Background noise kept the pause detector from firing until well after the words. + h.c.acceptPcm(pcm(8)); + h.c.commitSegment(); + assert.equal(finals(h).length, 0); + h.native.emit(result('After.', true, 2.0, 9.5)); + assert.deepEqual(finals(h), [{ text: 'Before.', isFinal: true, paragraphBreakAfter: true }]); + assert.deepEqual(h.events.at(-1), { text: 'After.', isFinal: false }); +}); + +test('a pause with nothing before it adds no break', () => { + const h = client(); + h.native.emit({ kind: 'ready', locale: 'en_US' }); + h.c.acceptPcm(pcm(3)); + h.c.commitSegment(); + h.native.emit(result('First words.', true, 0, 4.2)); + h.native.emit(result('More.', true, 4.2, 5.0)); + assert.equal(finals(h).length, 0); + assert.deepEqual(h.events.at(-1), { text: 'First words. More.', isFinal: false }); +}); + +test('errors end the session and reach the provider\'s error path', () => { + const h = client(); + h.native.emit({ kind: 'error', message: "On-device transcription doesn't support xx." }); + assert.deepEqual(h.errors, ["On-device transcription doesn't support xx."]); + assert.equal(h.native.cancelled, 1); +}); + +test('the wrapped client holds audio until the model is ready, then streams it', () => { + const { createSpeechAnalyzerClient, speechAnalyzerSupported } = load(); + assert.equal(speechAnalyzerSupported(), true); + FakeAnalyzer.supported = false; + assert.equal(speechAnalyzerSupported(), false); + const events = []; + const wrapped = createSpeechAnalyzerClient({ onTranscript: event => events.push({ ...event }), onStatus() {}, onError() {} }, () => true); + wrapped.start(); + const native = FakeAnalyzer.instances[0]; + wrapped.acceptPcm(new Uint8Array([1, 0, 2, 0])); + assert.equal(native.pcm.length, 0, 'buffered while the model loads'); + native.emit({ kind: 'ready', locale: 'en_US' }); + wrapped.acceptPcm(new Uint8Array([3, 0])); + assert.deepEqual(native.pcm, [[1, 0, 2, 0], [3, 0]]); + wrapped.stop(); + assert.equal(native.cancelled, 1); +});