diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawAudioTranscoder.kt b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawAudioTranscoder.kt index 2bb9ca77..f53ba1ac 100644 --- a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawAudioTranscoder.kt +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawAudioTranscoder.kt @@ -78,6 +78,16 @@ class FaceclawAudioTranscoder( return } val size = File(outputPath).length() + if (size <= 0) { + Log.w(TAG, "transcode produced no output for $wavPath") + File(outputPath).delete() + try { + listener?.onError("transcode produced no output") + } catch (listenerError: Throwable) { + Log.w(TAG, "transcode error listener failed", listenerError) + } + return + } try { listener?.onDone(outputPath, size) } catch (t: Throwable) { @@ -103,6 +113,7 @@ class FaceclawAudioTranscoder( val codec = MediaCodec.createEncoderByType(MediaFormat.MIMETYPE_AUDIO_AAC) var muxer: MediaMuxer? = null + var muxerStopped = false var input: FileInputStream? = null try { codec.configure(format, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE) @@ -169,6 +180,11 @@ class FaceclawAudioTranscoder( } } } + // stop() writes the index (moov); without it the file won't play, so its failure + // (storage full, I/O error, a muxer that never started) fails the transcode rather + // than reporting a broken output as done. + muxerStopped = true + muxer.stop() } finally { if (input != null) { try { @@ -182,9 +198,12 @@ class FaceclawAudioTranscoder( } codec.release() if (muxer != null) { - try { - muxer.stop() - } catch (ignored: Throwable) { + if (!muxerStopped) { + // Already failing; the output is deleted either way. + try { + muxer.stop() + } catch (ignored: Throwable) { + } } muxer.release() } diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawConversationStore.kt b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawConversationStore.kt index dfce1746..2f309ad8 100644 --- a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawConversationStore.kt +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawConversationStore.kt @@ -760,6 +760,28 @@ class FaceclawConversationStore private constructor(context: Context) : SQLiteOp return closed.toString() } + /** + * Closed sessions whose audio is still the WAV: a transcode that never finished (the process + * died mid-transcode, or recovery failed partway on an earlier boot). Returns a JSON array of + * {id, audioPath}. + */ + fun listUntranscodedSessions(): String { + val sessions = JSONArray() + readableDatabase.rawQuery( + "SELECT id, audio_path FROM sessions" + + " WHERE ended_at IS NOT NULL AND audio_codec = 'wav' AND audio_path IS NOT NULL", + null, + ).use { c -> + while (c.moveToNext()) { + val row = JSONObject() + row.put("id", c.getLong(0)) + row.put("audioPath", c.getString(1)) + sessions.put(row) + } + } + return sessions.toString() + } + /** * Delete recordings in dirPath that no session's audio_path names: files a session row never * learned about, or partial transcoder output. Matched by file name, so a storage move that diff --git a/app/apps/calculator/calculator.ts b/app/apps/calculator/calculator.ts index 1a02dfed..6c7b3263 100644 --- a/app/apps/calculator/calculator.ts +++ b/app/apps/calculator/calculator.ts @@ -16,6 +16,7 @@ import { type Layer, type LayerActions, type LayerContext } from "../../ui/layer import { ConfigSettingBoolean, ConfigSettingEnum } from "../../ui/dashboard-settings"; import { shell } from "../../ui/shell/shell"; import { voiceControlBridge, type VoiceTranscriptEvent } from "../../native/voice-control"; +import { voiceActivity } from "../../ui/shell/voice-activity"; import { type MathBranchOutcome, MathCoordinator } from "./math/coordinator"; import { type LongRunOffer, type LongRunProgress, offerGlassesPrompt, progressText } from "./math/workload"; import { plain } from "./math/text-renderer"; @@ -56,6 +57,11 @@ export const calculatorDegreesSetting = new ConfigSettingBoolean({ description: "Evaluate sin/cos/tan in degrees instead of radians.", }); +// The calculator's id for shared continuous capture, and its claim on the mic +// while a dictation is armed. +const CONTINUOUS_CLIENT = "calculator"; +const DICTATION_HOLDER = "calculator-dictation"; + // --------------------------------------------------------------------------- // Layer @@ -131,22 +137,22 @@ export class CalculatorLayer implements Layer { if (this.listening === "continuous") { if (!this.continuousActive) { this.continuousActive = true; - void this.actions.startContinuousVoiceCapture(); + void this.actions.startContinuousVoiceCapture(CONTINUOUS_CLIENT); } } else if (this.continuousActive) { this.continuousActive = false; - void this.actions.stopContinuousVoiceCapture(); + void this.actions.stopContinuousVoiceCapture(CONTINUOUS_CLIENT); } } private stopListening(): void { if (this.dictating) { - this.dictating = false; void this.actions.stopVoiceCapture(); + this.setDictating(false); } if (this.continuousActive) { this.continuousActive = false; - void this.actions.stopContinuousVoiceCapture(); + void this.actions.stopContinuousVoiceCapture(CONTINUOUS_CLIENT); } } @@ -248,15 +254,25 @@ export class CalculatorLayer implements Layer { beginDictation(): void { if (this.textInput) { void this.textInput(); return; } if (this.dictating) return; - this.dictating = true; + this.setDictating(true); void this.actions.startVoiceCapture(true); this.requestRender(); } + /** + * Arm or disarm dictation, holding the mic claim for it as the voice dialog + * does: the Microphones app and EvenHub mic apps stand down meanwhile (in + * 4-mic mode their stream would otherwise reach the recognizer). + */ + private setDictating(dictating: boolean): void { + this.dictating = dictating; + voiceActivity.setActive(dictating, DICTATION_HOLDER); + } + cancelDictation(): void { if (!this.dictating) return; - this.dictating = false; void this.actions.stopVoiceCapture(); + this.setDictating(false); this.requestRender(); } @@ -277,8 +293,8 @@ export class CalculatorLayer implements Layer { const continuous = this.continuousActive && this.foreground; if (!armed && !continuous) return; if (armed) { - this.dictating = false; void this.actions.stopVoiceCapture(); + this.setDictating(false); } // A standing offer or a run in flight owns the next yes/no said in the // room ("keep going", "never mind", "stop"). diff --git a/app/apps/microphones/caption-clock.ts b/app/apps/microphones/caption-clock.ts new file mode 100644 index 00000000..ceb2e653 --- /dev/null +++ b/app/apps/microphones/caption-clock.ts @@ -0,0 +1,44 @@ +// Utterance times come from the caption engine's sample clock, which only +// advances with audio fed to it. Audio that never reaches the engine (beam +// filtered, paused for the voice dialog, lost to a stall) would otherwise +// shift every later line earlier, so the clock is re-anchored to wall time +// (arrival time, so ordinary BLE jitter is tolerated) whenever the two drift +// this far apart, in either direction: missing audio puts wall time ahead, +// and a backlog delivered late in a burst puts it behind once the stream +// catches up. +export const CAPTION_CLOCK_DRIFT_MS = 300; +const MAX_ANCHORS = 1000; + +/** Maps a caption engine's clock (ms of audio fed to it) to wall time. */ +export class CaptionClock { + private fedSamples = 0; + // Engine-clock times with the wall time each corresponds to, in order. + private readonly anchors: Array<{ engineMs: number; wallMs: number }> = []; + + constructor( + private readonly sampleRate: number, + private readonly startWallMs: number, + ) {} + + /** `bytes` of 16-bit mono PCM were fed to the engine, arriving at `nowMs`. */ + noteFed(bytes: number, nowMs: number): void { + const samples = Math.floor(bytes / 2); + const engineMs = (this.fedSamples * 1000) / this.sampleRate; + this.fedSamples += samples; + // The chunk arrived as it ended; its first sample is that much older. + const wallMs = nowMs - (samples * 1000) / this.sampleRate; + const last = this.anchors[this.anchors.length - 1]; + if (last && Math.abs(wallMs - (last.wallMs + engineMs - last.engineMs)) <= CAPTION_CLOCK_DRIFT_MS) return; + this.anchors.push({ engineMs, wallMs }); + if (this.anchors.length > MAX_ANCHORS) this.anchors.shift(); + } + + /** Wall time of a moment on the engine's clock. */ + wallMs(engineMs: number): number { + for (let i = this.anchors.length - 1; i >= 0; i--) { + const anchor = this.anchors[i]!; + if (anchor.engineMs <= engineMs) return anchor.wallMs + engineMs - anchor.engineMs; + } + return this.startWallMs + engineMs; + } +} diff --git a/app/apps/microphones/mic-session.ts b/app/apps/microphones/mic-session.ts index 3b3d7ef1..5139f09b 100644 --- a/app/apps/microphones/mic-session.ts +++ b/app/apps/microphones/mic-session.ts @@ -16,6 +16,7 @@ import { type HeadOrientation, } from "./dsp"; import { micArrayController, micControlSupported, loadMicConfig, saveMicConfig } from "./mic-control"; +import { CaptionClock } from "./caption-clock"; import { micModelPath } from "./mic-models"; import { MIC_CHANNEL_KEYS, @@ -183,16 +184,16 @@ type ConversationRecord = { rowId: number; sentimentSum: number; sentimentCount: /** * One caption-engine instance. Its utterance times are on its own sample - * clock, converted with the wall time and recording offset taken at its - * start; an engine restarted after a failure is a new run on the same - * conversation. + * clock (audio fed to it), converted to wall time by `clock` and to + * recording time with the recording offset taken at its start; an engine + * restarted after a failure is a new run on the same conversation. */ type CaptionRun = { engine: any; // The Java listener proxy must stay referenced or it gets GC'd. listener: any; - startWallMs: number; audioBaseMs: number; + clock: CaptionClock; conversation: ConversationRecord | null; /** The app session (start() count) it belongs to. */ generation: number; @@ -973,7 +974,8 @@ class MicSession { // ---- shared audio consumers ---- private feedPcm(pcm: Uint8Array): void { - const engine = this.captionRun?.engine; + const run = this.captionRun; + const engine = run?.engine; if (!engine && !this.recorder) return; let javaBytes = toJavaBytes(pcm); if (this.ancOn && this.suppressor) { @@ -983,9 +985,10 @@ class MicSession { console.warn(`noise suppression failed: ${error}`); } } - if (engine) { + if (run) { try { - engine.acceptPcm(javaBytes); + run.engine.acceptPcm(javaBytes); + run.clock.noteFed(Number(javaBytes.length), Date.now()); } catch (error) { console.warn(`caption feed failed: ${error}`); } @@ -1046,9 +1049,9 @@ class MicSession { const run: CaptionRun = { engine, listener: null, - startWallMs: Date.now(), // A restarted engine's clock starts at zero partway into the recording. audioBaseMs: this.recorder ? Number(this.recorder.getDurationMs()) : 0, + clock: new CaptionClock(SAMPLE_RATE, Date.now()), conversation: this.conversation, generation: this.sessionGeneration, stopTimer: null, @@ -1411,7 +1414,7 @@ class MicSession { lang: "", emotion: analyzed?.emotion ?? "", sentiment: analyzed?.score ?? 0, - atMs: run.startWallMs + startMs, + atMs: run.clock.wallMs(startMs), }; if (shown) { this.captionLines.push(line); @@ -1441,8 +1444,8 @@ class MicSession { JSON.stringify({ sessionId: conversation.rowId, speakerId: match?.profile.id, - startedAt: run.startWallMs + startMs, - endedAt: run.startWallMs + endMs, + startedAt: run.clock.wallMs(startMs), + endedAt: run.clock.wallMs(endMs), audioOffsetMs: run.audioBaseMs + startMs, text: line.text, sentiment: line.sentiment, @@ -1517,8 +1520,10 @@ class MicSession { private startRecording(conversation: ConversationRecord): void { try { + const dir = recordingsDir(); + if (!dir) throw new Error("external storage unavailable"); const stamp = new Date().toISOString().replace(/[:.]/g, "-"); - this.recordingPath = `${recordingsDir()}/conversation-${stamp}.wav`; + this.recordingPath = `${dir}/conversation-${stamp}.wav`; this.recorder = new com.faceclaw.app.FaceclawWavRecorder(this.recordingPath, SAMPLE_RATE, 1); } catch (error) { console.warn(`recording start failed: ${error}`); @@ -1582,32 +1587,56 @@ class MicSession { /** * Repair what a dead process left behind, once per process at boot: - * conversations it never closed, their unfinished recordings (header - * patched to the audio on disk, then transcoded), and recording files no - * session row knows about. + * conversations it never closed, recording files no session row knows + * about, and recordings never transcoded (header patched to the audio on + * disk, then transcoded). The steps fail independently, and the last one + * works from what the store still holds as WAV rather than from what this + * boot closed, so whatever a failure leaves is picked up by a later boot. */ recoverInterruptedConversations(): void { if (!global.isAndroid || recoveryDone || this.running) return; recoveryDone = true; + let store: any; try { - const store = this.store(); - const closed = JSON.parse( - String(store.closeInterruptedSessions(JSON.stringify(this.openConversationRows()))), - ) as Array<{ id: number; audioPath: string | null; audioCodec: string | null }>; - // Before any transcode below starts writing, so its output isn't swept. - const removed = Number(store.deleteUnreferencedRecordings(recordingsDir())); - for (const session of closed) { - const rowId = Number(session.id); - if (session.audioPath && session.audioCodec === "wav") { - recoverRecording(store, rowId, String(session.audioPath)); - } - generateSessionInsights(rowId); - } - if (closed.length > 0 || removed > 0) { - console.log(`mic recovery: closed ${closed.length} conversation(s), removed ${removed} orphaned recording(s)`); - } + store = this.store(); } catch (error) { console.warn(`mic recovery failed: ${error}`); + return; + } + let closed: number[] = []; + try { + const rows = JSON.parse( + String(store.closeInterruptedSessions(JSON.stringify(this.openConversationRows()))), + ) as Array<{ id: number }>; + closed = rows.map((row) => Number(row.id)); + } catch (error) { + console.warn(`mic recovery: closing conversations failed: ${error}`); + } + for (const rowId of closed) generateSessionInsights(rowId); + // Before any transcode below starts writing, so its output isn't swept. + let removed = 0; + try { + const dir = recordingsDir(); + if (dir) removed = Number(store.deleteUnreferencedRecordings(dir)); + } catch (error) { + console.warn(`mic recovery: orphan sweep failed: ${error}`); + } + // The ones just closed, plus any whose transcode died with an earlier + // process; patching an already-finished header changes nothing. + let untranscoded: Array<{ id: number; audioPath: string }> = []; + try { + untranscoded = JSON.parse(String(store.listUntranscodedSessions())) as typeof untranscoded; + } catch (error) { + console.warn(`mic recovery: listing recordings failed: ${error}`); + } + for (const session of untranscoded) { + recoverRecording(store, Number(session.id), String(session.audioPath)); + } + if (closed.length > 0 || removed > 0 || untranscoded.length > 0) { + console.log( + `mic recovery: closed ${closed.length} conversation(s), removed ${removed} orphaned recording(s), ` + + `finishing ${untranscoded.length} recording(s)`, + ); } } @@ -1694,9 +1723,11 @@ function moonshineModelDir(): string { return `${context.getFilesDir().getAbsolutePath()}/faceclaw-voice-asr/sherpa-onnx-moonshine-base-en-quantized-2026-02-27`; } -function recordingsDir(): string { +/** Where recordings live, or null while external storage is unavailable. */ +function recordingsDir(): string | null { const context = Utils.android.getApplicationContext(); - return `${context.getExternalFilesDir(null).getAbsolutePath()}/mic-recordings`; + const base = context.getExternalFilesDir(null); + return base ? `${base.getAbsolutePath()}/mic-recordings` : null; } function deleteFile(path: string): void { @@ -1746,9 +1777,9 @@ function transcodeRecording(store: any, rowId: number, wavPath: string): void { } /** - * Keep a recording its process died writing: patch the WAV header to the - * audio on disk, then transcode it as a normal stop would. Too short (or - * gone) and it is dropped from the session instead. + * Finish a recording left as WAV (its process died writing or transcoding + * it): patch the header to the audio on disk, then transcode it as a normal + * stop would. Too short (or gone) and it is dropped from the session instead. */ function recoverRecording(store: any, rowId: number, wavPath: string): void { try { diff --git a/app/apps/teleprompter/index.ts b/app/apps/teleprompter/index.ts index f2abbcf1..e382ea7b 100644 --- a/app/apps/teleprompter/index.ts +++ b/app/apps/teleprompter/index.ts @@ -9,8 +9,8 @@ const teleprompterApp: AppDefinition = { ctx.launchInProcessApp(TELEPROMPTER_WINDOW_ID, TELEPROMPTER_SURFACE_ID, (options) => createTeleprompterAppWindow({ ...options, - startContinuousVoiceCapture: () => void options.actions.startContinuousVoiceCapture(), - stopContinuousVoiceCapture: () => void options.actions.stopContinuousVoiceCapture(), + startContinuousVoiceCapture: () => void options.actions.startContinuousVoiceCapture("teleprompter"), + stopContinuousVoiceCapture: () => void options.actions.stopContinuousVoiceCapture("teleprompter"), appendLog: (message) => ctx.appendLog(message), }), ), diff --git a/app/apps/transcribe/index.ts b/app/apps/transcribe/index.ts index ebd3c0f3..d9057faa 100644 --- a/app/apps/transcribe/index.ts +++ b/app/apps/transcribe/index.ts @@ -9,8 +9,8 @@ const transcribeApp: AppDefinition = { ctx.launchInProcessApp(TRANSCRIBE_WINDOW_ID, TRANSCRIBE_SURFACE_ID, (options) => createTranscribeAppWindow({ ...options, - startContinuousVoiceCapture: () => void options.actions.startContinuousVoiceCapture(), - stopContinuousVoiceCapture: () => void options.actions.stopContinuousVoiceCapture(), + startContinuousVoiceCapture: () => void options.actions.startContinuousVoiceCapture("transcribe"), + stopContinuousVoiceCapture: () => void options.actions.stopContinuousVoiceCapture("transcribe"), }), ), }; diff --git a/app/g2/dashboard-controller.ts b/app/g2/dashboard-controller.ts index 1decb897..4ae294b0 100644 --- a/app/g2/dashboard-controller.ts +++ b/app/g2/dashboard-controller.ts @@ -165,7 +165,8 @@ const SHELL_REFRESH_INTERVAL_MS = 60_000; // schedules a refresh), so this just covers anything that slips through. const PREVIEW_INTERVAL_MS = 1_000; const SCREEN_TIMEOUT_CHECK_MS = 1_000; -// voiceActivity holder for Transcribe's continuous capture. +// voiceActivity holder for continuous capture (Transcribe, Calculator's +// continuous mode, the Teleprompter reader). const CONTINUOUS_CAPTURE_HOLDER = "continuous-capture"; // The lock screen is a short notice, not something to read at leisure; it // ignores the screen-timeout setting (including "never"). @@ -391,8 +392,8 @@ class DashboardController { endTextSettingEdit: () => this.endTextSettingEdit(), startVoiceCapture: (endpointing = false) => this.startVoiceCapture(endpointing), stopVoiceCapture: () => this.stopVoiceCapture(), - startContinuousVoiceCapture: () => this.startContinuousVoiceCapture(), - stopContinuousVoiceCapture: () => this.stopContinuousVoiceCapture(), + startContinuousVoiceCapture: (client: string) => this.startContinuousVoiceCapture(client), + stopContinuousVoiceCapture: (client: string) => this.stopContinuousVoiceCapture(client), playBuzzerSequence: (payload: Uint8Array) => this.playBuzzerSequence(payload), }; this.sharedActions = sharedActions; @@ -1737,6 +1738,10 @@ class DashboardController { await nightscoutBridge.stop().catch(() => {}); weatherBridge.setSessionActive(false); voiceControlBridge.stop(); + // The parked continuous capture went with it; the next session starts + // it afresh for any window still asking. + this.continuousAttemptedOn = undefined; + this.syncContinuousClaim(); if (communicator) { await communicator.setFaceclawWakeLeaseEnabled(false).catch(() => false); await communicator.close().catch(() => {}); @@ -2061,17 +2066,53 @@ class DashboardController { return voiceControlBridge.stopPushToTalk(); } - private startContinuousVoiceCapture(): void { - // Continuous capture (Transcribe) owns the glasses mic for as long as it - // runs. Claim it before the capture starts so the Microphones app and - // EvenHub mic apps stand down first, as they do for the voice dialog. - voiceActivity.setActive(true, CONTINUOUS_CAPTURE_HOLDER); - void this.beginVoiceCapture("continuous"); + // Windows that want continuous capture. They share one stream, which runs + // while any of them wants it: one window letting go must not cut off another. + private readonly continuousClients = new Set(); + // The connection a continuous-capture start was last tried on (undefined: + // none since the last teardown), so a window waiting on the glasses is + // retried once per new session rather than on every frame. + private continuousAttemptedOn: FaceclawCommunicatorBridge | null | undefined = undefined; + + private startContinuousVoiceCapture(client: string): void { + this.continuousClients.add(client); + this.ensureContinuousCapture(true); } - private stopContinuousVoiceCapture(): void { - voiceControlBridge.stopContinuousCapture(); - voiceActivity.setActive(false, CONTINUOUS_CAPTURE_HOLDER); + private stopContinuousVoiceCapture(client: string): void { + if (!this.continuousClients.delete(client) || this.continuousClients.size > 0) return; + try { + voiceControlBridge.stopContinuousCapture(); + } finally { + this.syncContinuousClaim(); + } + } + + /** + * Start continuous capture for the windows asking for it, unless it is + * running or parked across a session outage. A window restored at launch + * asks before the glasses connect, so this is retried on each new session + * (from resumeVoiceCapture); `force` retries on the same one. + */ + private ensureContinuousCapture(force = false): void { + if (this.continuousClients.size === 0 || voiceControlBridge.holdsCapture("continuous")) return; + if (!force && this.continuousAttemptedOn === this.communicator) return; + this.continuousAttemptedOn = this.communicator; + void this.beginVoiceCapture("continuous").finally(() => this.syncContinuousClaim()); + } + + /** + * Continuous capture's claim on the mic (the Microphones app and EvenHub + * mic apps stand down while it is held) follows whether the capture really + * holds the mic, not merely whether a window asked: a start that never + * happened (no glasses yet, permission refused) must not idle every other + * mic user. + */ + private syncContinuousClaim(): void { + voiceActivity.setActive( + this.continuousClients.size > 0 && voiceControlBridge.holdsCapture("continuous"), + CONTINUOUS_CAPTURE_HOLDER, + ); } /** @@ -2099,15 +2140,19 @@ class DashboardController { } /** - * Restart the mic for a capture that a previous session left holding. Cheap - * to call per frame: it does nothing unless one is actually parked. + * Restart the mic for a capture that a previous session left holding, and + * start a continuous capture asked for before this session. Cheap to call + * per frame: it does nothing unless one is actually waiting. */ private resumeVoiceCapture(): void { - if (!voiceControlBridge.hasSuspendedCapture()) return; const communicator = this.communicator; if (!communicator || this.phase !== "connected") return; - this.appendLog("resuming voice capture on the new glasses session"); - voiceControlBridge.resumeCapture(this.voiceCaptureOptions(communicator)); + if (voiceControlBridge.hasSuspendedCapture()) { + this.appendLog("resuming voice capture on the new glasses session"); + voiceControlBridge.resumeCapture(this.voiceCaptureOptions(communicator)); + } + // A continuous capture asked for before this session existed. + this.ensureContinuousCapture(); } /** @@ -2152,6 +2197,11 @@ class DashboardController { if (kind === "ptt") { voiceControlBridge.startPushToTalk(options); } else { + // Every window may have let go while permission was pending. + if (this.continuousClients.size === 0) return; + // Claimed before the capture starts so the Microphones app and + // EvenHub mic apps stand down first, as they do for the voice dialog. + voiceActivity.setActive(true, CONTINUOUS_CAPTURE_HOLDER); voiceControlBridge.startContinuousCapture(options); } }) diff --git a/app/native/voice-control.ts b/app/native/voice-control.ts index 91db6624..1cf97fa5 100644 --- a/app/native/voice-control.ts +++ b/app/native/voice-control.ts @@ -434,6 +434,11 @@ export class FaceclawVoiceControlBridge { return this.suspendedHolders.size > 0 || this.suspendedRaw; } + /** Whether `holder` holds the mic, including across a session outage. */ + holdsCapture(holder: CaptureHolder): boolean { + return this.captureHolders.has(holder) || this.suspendedHolders.has(holder); + } + /** Whether anyone holds the mic, including across a session outage. */ isCaptureHeld(): boolean { return this.captureHolders.size > 0 || this.suspendedHolders.size > 0; diff --git a/app/ui/layers.ts b/app/ui/layers.ts index 4c2cb796..ad226f62 100644 --- a/app/ui/layers.ts +++ b/app/ui/layers.ts @@ -31,9 +31,12 @@ export type LayerActions = { startVoiceCapture: (endpointing?: boolean) => Promise | void; /** Stop push-to-talk; awaits native recognition, or commits a cloud provider for a final result. */ stopVoiceCapture: () => Promise | void; - /** Start continuous capture (Transcribe); shares the mic with push-to-talk. */ - startContinuousVoiceCapture: () => Promise | void; - stopContinuousVoiceCapture: () => Promise | void; + /** + * Start continuous capture on behalf of `client` (one id per window); shares + * the mic with push-to-talk, and runs while any client wants it. + */ + startContinuousVoiceCapture: (client: string) => Promise | void; + stopContinuousVoiceCapture: (client: string) => Promise | void; /** Play a CFW tone-sequencer payload (see sound-effects.ts). */ playBuzzerSequence: (payload: Uint8Array) => Promise | void; }; diff --git a/tests/microphones-caption-clock.test.cjs b/tests/microphones-caption-clock.test.cjs new file mode 100644 index 00000000..70a801c7 --- /dev/null +++ b/tests/microphones-caption-clock.test.cjs @@ -0,0 +1,68 @@ +// The caption clock maps the engine's fed-audio clock to wall time, so a +// line's timestamp stays right after audio that never reached the engine +// (beam filter, voice-dialog pause, stall) or arrived late in a burst. +const test = require("node:test"); +const assert = require("node:assert/strict"); + +const { CaptionClock, CAPTION_CLOCK_DRIFT_MS } = require("../.test-build/app/apps/microphones/caption-clock.js"); + +const RATE = 16000; +// 50 ms of 16 kHz mono S16LE. +const CHUNK_BYTES = 1600; +const CHUNK_MS = 50; + +function feed(clock, startWallMs, chunks, intervalMs = CHUNK_MS) { + for (let i = 0; i < chunks; i++) clock.noteFed(CHUNK_BYTES, startWallMs + (i + 1) * intervalMs); +} + +test("real-time audio maps straight through", () => { + const clock = new CaptionClock(RATE, 1_000_000); + feed(clock, 1_000_000, 100); + assert.equal(clock.wallMs(0), 1_000_000); + assert.equal(clock.wallMs(2_500), 1_002_500); +}); + +test("before any audio, times count from the start", () => { + const clock = new CaptionClock(RATE, 5_000); + assert.equal(clock.wallMs(120), 5_120); +}); + +test("a gap in the audio moves later times past it", () => { + const clock = new CaptionClock(RATE, 0); + feed(clock, 0, 20); // engine 0..1000 ms at wall 0..1000 + feed(clock, 61_000, 20); // a minute paused, then 1000 ms more + assert.equal(clock.wallMs(500), 500); + assert.equal(clock.wallMs(1_000), 61_000); + assert.equal(clock.wallMs(1_500), 61_500); +}); + +test("many small dropped chunks don't accumulate past the drift bound", () => { + const clock = new CaptionClock(RATE, 0); + // Every other 50 ms chunk is dropped (beam filter): fed audio runs at half speed. + for (let i = 0; i < 400; i++) clock.noteFed(CHUNK_BYTES, (i + 1) * 2 * CHUNK_MS); + const engineMs = 399 * CHUNK_MS; + const actualWall = 400 * 2 * CHUNK_MS - CHUNK_MS; + assert.ok(Math.abs(clock.wallMs(engineMs) - actualWall) <= CAPTION_CLOCK_DRIFT_MS + CHUNK_MS); +}); + +test("jitter within the bound doesn't re-anchor", () => { + const clock = new CaptionClock(RATE, 0); + for (let i = 0; i < 100; i++) { + const jitter = i % 2 === 0 ? 120 : 0; + clock.noteFed(CHUNK_BYTES, (i + 1) * CHUNK_MS + jitter); + } + // Anchored once, by the first chunk (which arrived 120 ms late). + assert.equal(clock.wallMs(1_000), 1_120); +}); + +test("a late burst re-anchors once the stream is real-time again", () => { + const clock = new CaptionClock(RATE, 0); + feed(clock, 0, 20); // 0..1000 ms on time + // 2 s of audio held up, then delivered all at once at wall 3000. + for (let i = 0; i < 40; i++) clock.noteFed(CHUNK_BYTES, 3_000); + // Then real-time again. + feed(clock, 3_000, 40); + // Engine 3000 ms (fed during real-time after the burst) lands near wall 3000. + assert.ok(Math.abs(clock.wallMs(3_000) - 3_000) <= CAPTION_CLOCK_DRIFT_MS + CHUNK_MS); + assert.ok(Math.abs(clock.wallMs(4_500) - 4_500) <= CAPTION_CLOCK_DRIFT_MS + CHUNK_MS); +}); diff --git a/tests/tsconfig.json b/tests/tsconfig.json index 82c8b27b..4e18c26c 100644 --- a/tests/tsconfig.json +++ b/tests/tsconfig.json @@ -45,6 +45,7 @@ "../app/apps/microphones/sentiment.ts", "../app/apps/microphones/dsp.ts", "../app/apps/microphones/radar-dots.ts", + "../app/apps/microphones/caption-clock.ts", "../app/apps/microphones/name-guess.ts", "../app/apps/evenhub/containers.ts", "../app/apps/evenhub/list-menu.ts",