diff --git a/app/apps/developer/microphones-monitor.ts b/app/apps/developer/microphones-monitor.ts index 44100a34..513bc8db 100644 --- a/app/apps/developer/microphones-monitor.ts +++ b/app/apps/developer/microphones-monitor.ts @@ -65,6 +65,14 @@ type ArmStats = { statusAtMs: number | null; }; +/** Drop arrivals older than the rate window. */ +function trimRecent(stats: ArmStats, nowMs: number): void { + const cutoff = nowMs - RATE_WINDOW_MS; + let drop = 0; + while (drop < stats.recent.length && stats.recent[drop]!.wallMs < cutoff) drop++; + if (drop > 0) stats.recent.splice(0, drop); +} + function emptyStats(): ArmStats { return { packets: 0, @@ -169,6 +177,8 @@ export class MicrophonesMonitorLayer implements Layer { this.advanceBuckets(nowMs); stats.packets++; stats.buckets[BUCKETS - 1]!++; + // Trim here too: painting (the other trim) stops while the page is hidden. + trimRecent(stats, nowMs); stats.recent.push({ wallMs: nowMs, arrivalMs, bytes: data.length }); stats.lastArrivalWallMs = nowMs; @@ -268,8 +278,7 @@ export class MicrophonesMonitorLayer implements Layer { const font = getDefaultSmallFont(); const line = font.lineHeight; const stats = this.stats[arm]; - const cutoff = nowMs - RATE_WINDOW_MS; - while (stats.recent.length > 0 && stats.recent[0]!.wallMs < cutoff) stats.recent.shift(); + trimRecent(stats, nowMs); const rate = (stats.recent.length * 1000) / RATE_WINDOW_MS; const byteRate = stats.recent.reduce((sum, arrival) => sum + arrival.bytes, 0) / RATE_WINDOW_MS; let maxGap = 0; diff --git a/app/apps/microphones/captions-layer.ts b/app/apps/microphones/captions-layer.ts index c27807b9..9cfc6dec 100644 --- a/app/apps/microphones/captions-layer.ts +++ b/app/apps/microphones/captions-layer.ts @@ -15,12 +15,15 @@ export class CaptionsLayer implements Layer { private state: MicSessionState = micSession.getState(); private unsubscribe: (() => void) | null = null; private scrollback = 0; - private lastLineCount = 0; + private lastLine: CaptionLine | null = null; start(requestRender: () => void): void { this.unsubscribe = micSession.onState((state) => { - if (state.captionLines.length !== this.lastLineCount) { - this.lastLineCount = state.captionLines.length; + // Watch the newest line itself: once the session's line cap is reached + // the count stops changing even though new speech keeps arriving. + const last = state.captionLines[state.captionLines.length - 1] ?? null; + if (last !== this.lastLine) { + this.lastLine = last; this.scrollback = 0; } this.state = state; diff --git a/app/apps/microphones/conversation-qa.ts b/app/apps/microphones/conversation-qa.ts index 40f464ee..c3300c2e 100644 --- a/app/apps/microphones/conversation-qa.ts +++ b/app/apps/microphones/conversation-qa.ts @@ -16,6 +16,10 @@ declare const com: any; // Qwen runs with a 4096-token context; leave room for the answer. const MAX_CONTEXT_CHARS = 7000; +// The speaker roster grows with every voice ever heard; cap it and the +// conversation list so they can't crowd the transcript excerpts out. +const MAX_HEADER_CHARS = MAX_CONTEXT_CHARS / 2; +const EXCERPTS_HEADING = "# Transcript excerpts\n"; const MAX_KEYWORDS = 6; const SEGMENTS_PER_KEYWORD = 30; const RECENT_SEGMENT_LIMIT = 120; @@ -156,31 +160,57 @@ function buildContext(question: string): { context: string; note: string } { } } - const segments = [...segmentMap.values()].sort((a, b) => a.startedAt - b.startedAt); + let header = + `# Known speakers\n${speakerLines.join("\n") || "(none yet)"}\n\n` + + `# Conversations${filters.sinceMs !== null || filters.emotion !== null ? " (filtered by the question)" : ""}\n` + + `${sessionLines.join("\n") || "(none in range)"}\n\n`; + let truncated = false; + if (header.length > MAX_HEADER_CHARS) { + header = `${header.slice(0, MAX_HEADER_CHARS)}\n(...)\n\n`; + truncated = true; + } + + const sessionHeading = (sessionId: number) => + `\n## ${sessionTitles.get(sessionId) ?? `Conversation #${sessionId}`}`; + const segmentText = (segment: SegmentRow) => { + const speaker = + segment.speakerId !== null ? (speakerNames.get(segment.speakerId) ?? "Speaker") : "Speaker"; + const emotion = segment.emotion && segment.emotion !== "neutral" ? ` [${segment.emotion}]` : ""; + const line = `[${formatWhen(segment.startedAt)}] ${speaker}${emotion}: ${segment.text}`; + return segment.translation ? `${line}\n (translated: ${segment.translation})` : line; + }; + + // Fill the remaining budget newest-first, so an overflowing history drops + // its oldest excerpts rather than its most recent ones. + const sorted = [...segmentMap.values()].sort((a, b) => a.startedAt - b.startedAt); + let budget = MAX_CONTEXT_CHARS - header.length - EXCERPTS_HEADING.length; + const segments: SegmentRow[] = []; + let runSession = -1; + for (let index = sorted.length - 1; index >= 0; index--) { + const segment = sorted[index]!; + let cost = segmentText(segment).length + 1; + if (segment.sessionId !== runSession) cost += sessionHeading(segment.sessionId).length + 1; + if (cost > budget) { + truncated = true; + break; + } + budget -= cost; + runSession = segment.sessionId; + segments.push(segment); + } + segments.reverse(); + const excerptLines: string[] = []; let currentSession = -1; for (const segment of segments) { if (segment.sessionId !== currentSession) { currentSession = segment.sessionId; - excerptLines.push(`\n## ${sessionTitles.get(currentSession) ?? `Conversation #${currentSession}`}`); + excerptLines.push(sessionHeading(currentSession)); } - const speaker = - segment.speakerId !== null ? (speakerNames.get(segment.speakerId) ?? "Speaker") : "Speaker"; - const emotion = segment.emotion && segment.emotion !== "neutral" ? ` [${segment.emotion}]` : ""; - excerptLines.push(`[${formatWhen(segment.startedAt)}] ${speaker}${emotion}: ${segment.text}`); - if (segment.translation) excerptLines.push(` (translated: ${segment.translation})`); + excerptLines.push(segmentText(segment)); } - let context = - `# Known speakers\n${speakerLines.join("\n") || "(none yet)"}\n\n` + - `# Conversations${filters.sinceMs !== null || filters.emotion !== null ? " (filtered by the question)" : ""}\n` + - `${sessionLines.join("\n") || "(none in range)"}\n\n` + - `# Transcript excerpts\n${excerptLines.join("\n") || "(no matching lines)"}`; - let truncated = false; - if (context.length > MAX_CONTEXT_CHARS) { - context = context.slice(0, MAX_CONTEXT_CHARS); - truncated = true; - } + const context = `${header}${EXCERPTS_HEADING}${excerptLines.join("\n") || "(no matching lines)"}`; const note = `${sessions.length} conversation${sessions.length === 1 ? "" : "s"}, ` + `${segments.length} transcript line${segments.length === 1 ? "" : "s"} consulted` + diff --git a/app/apps/microphones/dsp.ts b/app/apps/microphones/dsp.ts index 9c683d28..34a58045 100644 --- a/app/apps/microphones/dsp.ts +++ b/app/apps/microphones/dsp.ts @@ -286,6 +286,12 @@ export function angleInArc(azimuth: number, arcCenter: number, arcHalfWidth: num export class TempleAudioPipeline { private readonly beamformer: DelaySumBeamformer; private readonly anc = new InterMicNoiseCanceller(); + // Steering delays reach back before the start of each frame. Carrying the + // previous frame's last samples keeps those real; zero-filling them (as a + // standalone beam() does) dips the output at every frame boundary. + private readonly carry: number; + private frontTail: Float32Array; + private rearTail: Float32Array; enableANC = true; constructor( @@ -294,19 +300,38 @@ export class TempleAudioPipeline { geometry: MicGeometry = G2_DEFAULT_GEOMETRY, ) { this.beamformer = new DelaySumBeamformer(sampleRate, geometry); + const { front, rear } = geometry.positions[side]; + const spacing = Math.hypot(front.x - rear.x, front.y - rear.y, front.z - rear.z); + this.carry = Math.ceil((spacing / geometry.speedOfSound) * sampleRate) + 1; + this.frontTail = new Float32Array(this.carry); + this.rearTail = new Float32Array(this.carry); } /** * Process one frame's deinterleaved front/rear channels toward a * device-frame target, returning the enhanced mono beam for this temple. + * Frames must be consecutive: each one continues the previous. */ process(front: Float32Array, rear: Float32Array, target: Bearing): Float32Array { - const beam = this.beamformer.beam(this.side, front, rear, target); + const n = Math.min(front.length, rear.length); + const extendedFront = this.withTail(this.frontTail, front, n); + const extendedRear = this.withTail(this.rearTail, rear, n); + this.frontTail = extendedFront.slice(n); + this.rearTail = extendedRear.slice(n); + const beam = this.beamformer + .beam(this.side, extendedFront, extendedRear, target) + .slice(this.carry); if (!this.enableANC) return beam; // Null-steer reference: the difference channel carries off-axis noise. - const n = Math.min(front.length, rear.length); const reference = new Float32Array(n); for (let i = 0; i < n; i++) reference[i] = front[i]! - rear[i]!; return this.anc.process(beam, reference); } + + private withTail(tail: Float32Array, frame: Float32Array, n: number): Float32Array { + const out = new Float32Array(tail.length + n); + out.set(tail); + out.set(frame.subarray(0, n), tail.length); + return out; + } } diff --git a/app/apps/microphones/mic-control.ts b/app/apps/microphones/mic-control.ts index fbb6a7b6..54a80d5d 100644 --- a/app/apps/microphones/mic-control.ts +++ b/app/apps/microphones/mic-control.ts @@ -110,6 +110,7 @@ class MicArrayController { * doesn't spend BLE bandwidth. */ apply(config: MicConfig): boolean { + if (config.armHardware && !micControlSupported()) return false; const leftActive = templeActive(config, "left"); const rightActive = templeActive(config, "right"); if (leftActive || rightActive) { @@ -148,8 +149,8 @@ class MicArrayController { ): boolean { const communicator = activeCommunicator(); if (!communicator) return false; - this.ensureStatusProxy(communicator); try { + this.ensureStatusProxy(communicator); const record = encodeMicControl(op, config); communicator.sendFaceclawMicControl(toJavaBytes(record), label, rightTemple, leftTemple); return true; diff --git a/app/apps/microphones/mic-models.ts b/app/apps/microphones/mic-models.ts index 0d2ff693..010cd2ff 100644 --- a/app/apps/microphones/mic-models.ts +++ b/app/apps/microphones/mic-models.ts @@ -142,6 +142,7 @@ function notifyStateChanged(id: MicModelId): void { export function startMicModelDownload(id: MicModelId): void { if (!global.isAndroid || downloaders.has(id) || isMicModelReady(id)) return; + progress.delete(id); downloadNextFile(id); notifyStateChanged(id); } @@ -171,15 +172,22 @@ function downloadNextFile(id: MicModelId): void { notifyStateChanged(id); }, }); - const downloader = new com.faceclaw.app.FaceclawModelDownloader( - `${definition.baseUrl}${nextFile.remoteName}`, - filePath(id, nextFile), - nextFile.sha256, - nextFile.sizeBytes, - listener, - ); - downloaders.set(id, downloader); - downloader.start(); + try { + const downloader = new com.faceclaw.app.FaceclawModelDownloader( + `${definition.baseUrl}${nextFile.remoteName}`, + filePath(id, nextFile), + nextFile.sha256, + nextFile.sizeBytes, + listener, + ); + downloaders.set(id, downloader); + downloader.start(); + } catch (error) { + // Without this, a dead entry would pin the model at "downloading". + console.error(`Mic model download failed to start (${id}/${nextFile.name}): ${String(error)}`); + downloaders.delete(id); + notifyStateChanged(id); + } } export function cancelMicModelDownload(id: MicModelId): void { diff --git a/app/apps/microphones/mic-session.ts b/app/apps/microphones/mic-session.ts index f6ee7a54..f8849dda 100644 --- a/app/apps/microphones/mic-session.ts +++ b/app/apps/microphones/mic-session.ts @@ -95,6 +95,13 @@ const RECORDING_AAC_BITRATE = 32_000; const ENCOUNTER_GAP_MS = 15 * 60 * 1000; const ENCOUNTER_CARD_TTL_MS = 12_000; const ENCOUNTER_CARD_MAX_ACTION_ITEMS = 2; +// A temple that has sent no frame (or no bearing) for this long no longer +// counts: its stream stopped, its mics were toggled off, or its arm dropped. +const TEMPLE_STALE_MS = 1000; +// How often the extended path checks that its forwarding survived the +// glasses session (an arm drop clears it). +const CAPTURE_CHECK_MS = 3000; +const COMPASS_OWNER = "microphones"; export type CaptionLine = { speakerId: number | null; @@ -181,6 +188,8 @@ class MicSession { private temples = new Map(); private templeAngles: { left: number | null; right: number | null } = { left: null, right: null }; private templeSsr: { left: number; right: number } = { left: 0, right: 0 }; + private templeAngleAtMs: Record = { left: 0, right: 0 }; + private templeFrameAtMs: Record = { left: 0, right: 0 }; private captionFeedSide: MicSide = "left"; // Cached at capture start so the ~80 Hz packet path never re-reads settings. private activeConfig: MicConfig = loadMicConfig(); @@ -202,6 +211,7 @@ class MicSession { private speechActive = false; private renderTimer: ReturnType | null = null; + private captureCheckTimer: ReturnType | null = null; onState(listener: StateListener): () => void { this.stateListeners.add(listener); @@ -262,33 +272,33 @@ class MicSession { this.activeConfig = loadMicConfig(); this.refreshCachedFlags(); + const extended = micControlSupported() && this.activeConfig.armHardware; + this.mode = extended ? "extended" : "stock"; + this.running = true; + // Capture first, so a failure has no captions session, recording, or + // subscriptions to unwind. + if (!this.startCapture()) { + this.statusText = "Microphone unavailable (glasses not connected?)"; + this.running = false; + this.mode = "off"; + this.notify(); + return; + } + this.statusText = this.listeningText(); this.offSettings = onSettingsStoreChanged(() => this.refreshCachedFlags()); this.offSpeakerChanges = speakerRegistry.onChanged(() => this.refreshSpeakerIdentities()); this.offVoiceActivity = voiceActivity.subscribe((active) => this.handleVoiceActivity(active)); - const extended = micControlSupported() && this.activeConfig.armHardware; - this.mode = extended ? "extended" : "stock"; this.startSuppressor(); this.startCompass(); if (captionsEnabledSetting.get()) { this.startCaptions(); } - if (saveRecordingsSetting.get() && recordingsRetentionSetting.get() !== "none") { + if (this.shouldRecord()) { this.startRecording(); } - const ok = extended ? this.startExtended() : this.startStock(); - if (!ok) { - this.statusText = "Microphone unavailable (glasses not connected?)"; - this.running = false; - this.stopCaptions(); - this.stopRecording(false); - this.stopCompass(); - this.mode = "off"; - this.notify(); - return; - } - this.statusText = extended ? "Listening (4-mic array)" : "Listening"; // Levels/DOA update at packet rate; coalesce UI renders to 5 Hz. this.renderTimer = setInterval(() => this.notify(), 200); + this.captureCheckTimer = setInterval(() => this.checkExtendedCapture(), CAPTURE_CHECK_MS); this.notify(); } @@ -299,11 +309,11 @@ class MicSession { clearInterval(this.renderTimer); this.renderTimer = null; } - if (this.mode === "extended") { - this.stopExtended(); - } else { - this.stopStock(); + if (this.captureCheckTimer) { + clearInterval(this.captureCheckTimer); + this.captureCheckTimer = null; } + this.stopCapture(); this.offSettings?.(); this.offSettings = null; this.offSpeakerChanges?.(); @@ -324,6 +334,55 @@ class MicSession { void this.applyRetentionSweep(); } + private listeningText(): string { + return this.mode === "extended" ? "Listening (4-mic array)" : "Listening"; + } + + /** Start the current mode's capture path, undoing any partial start on failure. */ + private startCapture(): boolean { + try { + if (this.mode === "extended" ? this.startExtended() : this.startStock()) return true; + } catch (error) { + console.warn(`mic capture start failed: ${error}`); + } + this.stopCapture(); + return false; + } + + private stopCapture(): void { + if (this.mode === "extended") { + this.stopExtended(); + } else { + this.stopStock(); + } + } + + /** + * Persist a Microphone setup change. While running it takes effect live: + * toggling 4-mic capture swaps the capture path, and any other change is + * re-sent to the streaming temples. + */ + updateConfig(mutate: (config: MicConfig) => MicConfig): void { + const config = mutate(loadMicConfig()); + saveMicConfig(config); + this.activeConfig = config; + if (this.running) { + const extended = micControlSupported() && config.armHardware; + if (extended !== (this.mode === "extended")) { + this.stopCapture(); + this.mode = extended ? "extended" : "stock"; + this.doaDeviceDeg = null; + this.ssr = 0; + this.statusText = this.startCapture() + ? this.listeningText() + : "Microphone unavailable (glasses not connected?)"; + } else if (extended && !voiceActivity.isActive()) { + micArrayController.apply(wireConfig(config)); + } + } + this.notify(); + } + private refreshCachedFlags(): void { this.beamFilterOn = beamFilterSetting.get(); const wasOn = this.ancOn; @@ -380,13 +439,34 @@ class MicSession { if (!communicator) return; if (this.mode === "extended") { if (this.forwardingProxy && communicator.startG2AudioForwarding(this.forwardingProxy)) { - micArrayController.apply({ ...this.activeConfig, channelMask: 0b11 }); + micArrayController.apply(wireConfig(this.activeConfig)); } } else { voiceControlBridge.startRawCapture({ communicator }); } } + /** + * The extended stream's forwarding lives in the glasses session: an arm + * drop clears it and nothing else re-arms it on the next session, so the + * app would sit at "Listening" hearing nothing. Re-establish forwarding + * and re-send the config whenever a ready session has lost it. (The stock + * path is resumed by the voice-control bridge.) + */ + private checkExtendedCapture(): void { + if (!this.running || this.mode !== "extended" || !this.forwardingProxy) return; + if (voiceActivity.isActive()) return; + try { + const communicator = this.nativeCommunicator(); + if (!communicator?.isSessionReady() || communicator.isAudioCaptureActive()) return; + if (communicator.startG2AudioForwarding(this.forwardingProxy)) { + micArrayController.apply(wireConfig(this.activeConfig)); + } + } catch (error) { + console.warn(`mic capture re-arm failed: ${error}`); + } + } + private startSuppressor(): void { try { this.suppressor = new com.faceclaw.app.FaceclawNoiseSuppressor(SAMPLE_RATE); @@ -459,7 +539,7 @@ class MicSession { private startCompass(): void { try { - setCompassEnabled(true); + setCompassEnabled(true, COMPASS_OWNER); this.offCompass = addCompassListener((event) => { if (event.headingDegrees >= 0) { this.headingDeg = event.headingDegrees; @@ -474,7 +554,7 @@ class MicSession { this.offCompass?.(); this.offCompass = null; try { - setCompassEnabled(false); + setCompassEnabled(false, COMPASS_OWNER); } catch { // Disconnected; the stream is already gone. } @@ -542,11 +622,7 @@ class MicSession { }, }); if (!communicator.startG2AudioForwarding(this.forwardingProxy)) return false; - if (!micArrayController.apply({ ...this.activeConfig, channelMask: 0b11 })) { - communicator.stopG2AudioForwarding(); - return false; - } - return true; + return micArrayController.apply(wireConfig(this.activeConfig)); } /** @@ -555,14 +631,7 @@ class MicSession { * captions; a temple with both mics off has its stream stopped entirely. */ setMicChannelEnabled(key: MicChannelKey, enabled: boolean): void { - const config = loadMicConfig(); - config.hostMics = { ...config.hostMics, [key]: enabled }; - saveMicConfig(config); - this.activeConfig = config; - if (this.running && this.mode === "extended") { - micArrayController.apply({ ...config, channelMask: 0b11 }); - } - this.notify(); + this.updateConfig((config) => ({ ...config, hostMics: { ...config.hostMics, [key]: enabled } })); } private stopExtended(): void { @@ -576,6 +645,14 @@ class MicSession { this.forwardingProxy = null; this.temples.clear(); this.templeAngles = { left: null, right: null }; + this.templeSsr = { left: 0, right: 0 }; + this.templeAngleAtMs = { left: 0, right: 0 }; + this.templeFrameAtMs = { left: 0, right: 0 }; + } + + /** A temple's last bearing, or null once it has gone quiet. */ + private freshTempleAngle(side: MicSide, nowMs: number): number | null { + return nowMs - this.templeAngleAtMs[side] < TEMPLE_STALE_MS ? this.templeAngles[side] : null; } private handleExtendedPacket(data: Uint8Array, arm: string): void { @@ -588,6 +665,8 @@ class MicSession { const side: MicSide = arm === "L" ? "left" : "right"; const channels = splitInterleavedPcm16(frame); if (!channels || channels.length === 0) return; + const now = Date.now(); + this.templeFrameAtMs[side] = now; // Meter every arriving channel — a host-disabled mic still shows its // live level (marked off in the UI), it just doesn't feed anything. @@ -606,8 +685,10 @@ class MicSession { if (frame.angleDegrees !== 0 || frame.ssr !== 0) { this.templeAngles[side] = frame.angleDegrees; this.templeSsr[side] = frame.ssr; - const left = this.templeAngles.left; - const right = this.templeAngles.right; + this.templeAngleAtMs[side] = now; + // A temple that stopped reporting must not keep steering the fusion. + const left = this.freshTempleAngle("left", now); + const right = this.freshTempleAngle("right", now); let fusedDeg: number; if (left !== null && right !== null) { const fused = fuseBearings( @@ -621,7 +702,7 @@ class MicSession { fusedDeg = (left ?? right)!; } this.doaDeviceDeg = smoothAngleDeg(this.doaDeviceDeg, fusedDeg, DOA_SMOOTHING); - this.ssr = Math.max(this.templeSsr.left, this.templeSsr.right); + this.ssr = Math.max(left === null ? 0 : this.templeSsr.left, right === null ? 0 : this.templeSsr.right); if (this.beamLocked && this.doaDeviceDeg !== null) { this.beamWorldDeg = this.toWorldDeg(this.doaDeviceDeg); } @@ -651,10 +732,16 @@ class MicSession { private deliverExtendedAudio(side: MicSide, samples: Float32Array): void { // Feed captions from one temple only; prefer the one hearing more - // signal, among the temples that still have an enabled microphone. - const leftUsable = templeActive(this.activeConfig, "left"); - const rightUsable = templeActive(this.activeConfig, "right"); - let stronger: MicSide = this.templeSsr.right > this.templeSsr.left * 1.5 ? "right" : "left"; + // signal, among the temples that still have an enabled microphone and + // are still streaming. + const now = Date.now(); + const leftUsable = + templeActive(this.activeConfig, "left") && now - this.templeFrameAtMs.left < TEMPLE_STALE_MS; + const rightUsable = + templeActive(this.activeConfig, "right") && now - this.templeFrameAtMs.right < TEMPLE_STALE_MS; + const leftSsr = this.freshTempleAngle("left", now) === null ? 0 : this.templeSsr.left; + const rightSsr = this.freshTempleAngle("right", now) === null ? 0 : this.templeSsr.right; + let stronger: MicSide = rightSsr > leftSsr * 1.5 ? "right" : "left"; if (stronger === "right" && !rightUsable) stronger = "left"; if (stronger === "left" && !leftUsable) stronger = "right"; if (strongerChanged(this.captionFeedSide, stronger)) { @@ -707,9 +794,12 @@ class MicSession { if (!this.running) return; if (enabled && !this.captionsActive) { this.startCaptions(); - if (saveRecordingsSetting.get() && !this.recorder) this.startRecording(); + if (!this.recorder && this.shouldRecord()) this.startRecording(); } else if (!enabled && this.captionsActive) { this.stopCaptions(); + // The recording belongs to this caption session (its segment offsets + // are relative to this engine's clock); close it out with the row. + this.stopRecording(true); this.finishSessionRow(); } this.notify(); @@ -717,33 +807,56 @@ class MicSession { private startCaptions(): void { if (this.captionsActive) return; - const engine = new com.faceclaw.app.FaceclawCaptionEngine(); - this.captionListenerProxy = new com.faceclaw.app.FaceclawCaptionEngineListener({ - onUtterance: (text: string, embedding: any, startMs: number, endMs: number, peakRms: number) => { - this.handleUtterance(String(text), embedding, Number(startMs), Number(endMs), Number(peakRms)); - }, - onSpeechStart: (_startMs: number) => { - this.speechActive = true; - }, - onStatus: (status: string) => { - this.statusText = String(status); - }, - }); - engine.setListener(this.captionListenerProxy); - // Live captions are a separate sherpa-onnx pipeline (FaceclawCaptionEngine) - // that only ever uses Moonshine; unaffected by the two-model on-device - // Whisper addition in native/voice-control.ts's push-to-talk path. - engine.setAsrModelDir(isAsrModelReady("moonshine") ? moonshineModelDir() : null); - engine.setSpeakerModelPath(micModelPath("speaker-embedding")); - engine.start(); + let engine: any = null; + try { + engine = new com.faceclaw.app.FaceclawCaptionEngine(); + this.captionListenerProxy = new com.faceclaw.app.FaceclawCaptionEngineListener({ + onUtterance: (text: string, embedding: any, startMs: number, endMs: number, peakRms: number) => { + // Never let an exception escape into the engine's Java callback. + try { + this.handleUtterance(String(text), embedding, Number(startMs), Number(endMs), Number(peakRms)); + } catch (error) { + console.warn(`caption utterance failed: ${error}`); + } + }, + onSpeechStart: (_startMs: number) => { + this.speechActive = true; + }, + onStatus: (status: string) => { + this.statusText = String(status); + }, + }); + engine.setListener(this.captionListenerProxy); + // Live captions are a separate sherpa-onnx pipeline (FaceclawCaptionEngine) + // that only ever uses Moonshine; unaffected by the two-model on-device + // Whisper addition in native/voice-control.ts's push-to-talk path. + engine.setAsrModelDir(isAsrModelReady("moonshine") ? moonshineModelDir() : null); + engine.setSpeakerModelPath(micModelPath("speaker-embedding")); + engine.start(); + } catch (error) { + console.warn(`caption engine start failed: ${error}`); + try { + engine?.stop(); + } catch { + // It never started. + } + this.captionListenerProxy = null; + this.statusText = "Captions unavailable"; + return; + } this.captionEngine = engine; this.captionsActive = true; this.engineStartWallMs = Date.now(); this.encounterShown.clear(); if (saveCaptionsSetting.get() && captionsRetentionSetting.get() !== "none") { - this.sessionRowId = Number( - this.store().startSession(Date.now(), `Conversation ${new Date().toLocaleString()}`), - ); + try { + this.sessionRowId = Number( + this.store().startSession(Date.now(), `Conversation ${new Date().toLocaleString()}`), + ); + } catch (error) { + console.warn(`caption session start failed: ${error}`); + this.sessionRowId = -1; + } } } @@ -892,10 +1005,23 @@ class MicSession { const embeddingFloats = embedding ? floatArrayToFloat32(embedding) : null; if (this.wearerEnrollmentArmed && embeddingFloats) { this.wearerEnrollmentArmed = false; - const wearer = speakerRegistry.enrollWearer(embeddingFloats); - this.statusText = `Enrolled your voice as "${wearer.name}"`; + try { + const wearer = speakerRegistry.enrollWearer(embeddingFloats); + this.statusText = `Enrolled your voice as "${wearer.name}"`; + } catch (error) { + console.warn(`wearer enrollment failed: ${error}`); + this.statusText = "Voice enrollment failed"; + } + } + let match: SpeakerMatch | null = null; + if (embeddingFloats) { + try { + match = speakerRegistry.assign(embeddingFloats); + } catch (error) { + // Still caption the line, just unattributed. + console.warn(`speaker assignment failed: ${error}`); + } } - let match = embeddingFloats ? speakerRegistry.assign(embeddingFloats) : null; if (match) this.maybeShowEncounterCard(match); // Spoken introductions name the auto-numbered speakers: "I'm Alice" // renames the speaker of the line; "this is Bob" pends the name for the @@ -957,23 +1083,27 @@ class MicSession { let segmentId = -1; if (this.sessionRowId >= 0 && line.text) { - segmentId = Number( - this.store().insertSegment( - JSON.stringify({ - sessionId: this.sessionRowId, - speakerId: match?.profile.id, - startedAt: this.engineStartWallMs + startMs, - endedAt: this.engineStartWallMs + endMs, - audioOffsetMs: startMs, - text: line.text, - sentiment: line.sentiment, - emotion: line.emotion, - searchMeta: analyzed ? searchableMetadata(analyzed.emotion, analyzed.score) : "", - angle: doaAtUtterance === null ? undefined : Math.round(doaAtUtterance), - embeddingBase64: embeddingFloats ? encodeEmbeddingBase64(embeddingFloats) : undefined, - }), - ), - ); + try { + segmentId = Number( + this.store().insertSegment( + JSON.stringify({ + sessionId: this.sessionRowId, + speakerId: match?.profile.id, + startedAt: this.engineStartWallMs + startMs, + endedAt: this.engineStartWallMs + endMs, + audioOffsetMs: startMs, + text: line.text, + sentiment: line.sentiment, + emotion: line.emotion, + searchMeta: analyzed ? searchableMetadata(analyzed.emotion, analyzed.score) : "", + angle: doaAtUtterance === null ? undefined : Math.round(doaAtUtterance), + embeddingBase64: embeddingFloats ? encodeEmbeddingBase64(embeddingFloats) : undefined, + }), + ), + ); + } catch (error) { + console.warn(`caption segment save failed: ${error}`); + } } this.notify(); @@ -1024,6 +1154,19 @@ class MicSession { // ---- recording ---- + /** + * Recordings belong to a saved caption session: retention and the phone UI + * find audio only through its session row, so a file recorded without one + * would never be shown or expired. + */ + private shouldRecord(): boolean { + return ( + this.sessionRowId >= 0 && + saveRecordingsSetting.get() && + recordingsRetentionSetting.get() !== "none" + ); + } + private startRecording(): void { try { const context = Utils.android.getApplicationContext(); @@ -1044,18 +1187,17 @@ class MicSession { if (!recorder) return; try { const bytes = Number(recorder.finish()); - if (!keep || bytes < SAMPLE_RATE) { - // Under half a second of audio isn't worth keeping. + const sessionRowId = this.sessionRowId; + if (!keep || bytes < SAMPLE_RATE || sessionRowId < 0) { + // Under half a second of audio isn't worth keeping, and audio with no + // session row to own it would be unreachable and never expire. recorder.discard(); return; } const wavPath = this.recordingPath; const m4aPath = wavPath.replace(/\.wav$/, ".m4a"); - const sessionRowId = this.sessionRowId; const store = this.store(); - if (sessionRowId >= 0) { - store.setSessionAudio(sessionRowId, wavPath, "wav"); - } + store.setSessionAudio(sessionRowId, wavPath, "wav"); // Transcode in the background; swap the session's audio to the M4A on // success (the transcoder deletes the WAV). const transcoder = new com.faceclaw.app.FaceclawAudioTranscoder( @@ -1065,8 +1207,10 @@ class MicSession { true, new com.faceclaw.app.FaceclawAudioTranscoder.Listener({ onDone: (outputPath: string, _outputBytes: number) => { - if (sessionRowId >= 0) { + try { store.setSessionAudio(sessionRowId, String(outputPath), "aac"); + } catch (error) { + console.warn(`recording audio update failed: ${error}`); } }, onError: (message: string) => { @@ -1129,6 +1273,14 @@ class MicSession { // ---- helpers ---- +/** + * The config as sent to the temples: each active temple always streams its + * full front+rear pair, and per-mic selection happens host-side. + */ +function wireConfig(config: MicConfig): MicConfig { + return { ...config, channelMask: 0b11 }; +} + function pcmLevel(pcm: Uint8Array): number { let sumSquares = 0; const count = pcm.length / 2; diff --git a/app/apps/microphones/microphones-app.ts b/app/apps/microphones/microphones-app.ts index c8f6a6ec..c7da0c86 100644 --- a/app/apps/microphones/microphones-app.ts +++ b/app/apps/microphones/microphones-app.ts @@ -22,7 +22,7 @@ import { isAsrModelReady, startAsrModelDownload, asrModelState } from "../../nat import { CaptionsLayer } from "./captions-layer"; import { LevelsLayer } from "./levels-layer"; import { RadarLayer } from "./radar-layer"; -import { loadMicConfig, micArrayController, micControlSupported, saveMicConfig } from "./mic-control"; +import { loadMicConfig, micArrayController, micControlSupported } from "./mic-control"; import { isMicModelReady, micModelState, startMicModelDownload } from "./mic-models"; import { micChannelLabel, type MicChannelKey, type MicConfig } from "./mic-protocol"; import { micSession } from "./mic-session"; @@ -72,13 +72,9 @@ const MENU_LAYOUT = { opaque: true, }; -/** Persist a config change and re-apply it live when the array is streaming. */ +/** Persist a config change; the running session applies it live. */ function updateMicConfig(mutate: (config: MicConfig) => MicConfig): void { - const updated = mutate(loadMicConfig()); - saveMicConfig(updated); - if (micSession.getState().mode === "extended") { - micArrayController.apply(updated); - } + micSession.updateConfig(mutate); } function micChannelToggleItem( diff --git a/app/apps/microphones/name-guess.ts b/app/apps/microphones/name-guess.ts index 0b38c76b..1db24241 100644 --- a/app/apps/microphones/name-guess.ts +++ b/app/apps/microphones/name-guess.ts @@ -55,7 +55,9 @@ const SELF_PATTERNS = [ new RegExp(`\\b[Mm]y name is ${NAME}`), new RegExp(`\\b[Ii](?:'| a)m ${NAME}`), new RegExp(`\\b[Cc]all me ${NAME}`), - new RegExp(`\\b[Tt]his is ${NAME} speaking\\b`), + // Phone-style: "this is Dana calling about..." is the speaker, not a + // third party being introduced. + new RegExp(`\\b[Tt]his is ${NAME} (?:[Ss]peaking|[Cc]alling)\\b`), ]; const THIRD_PARTY_PATTERNS = [ diff --git a/app/apps/microphones/rediarize.ts b/app/apps/microphones/rediarize.ts index 49423181..d5be9456 100644 --- a/app/apps/microphones/rediarize.ts +++ b/app/apps/microphones/rediarize.ts @@ -140,6 +140,8 @@ function applyTurns(sessionId: number, turns: Turn[]): RediarizeResult { } else { const id = Number(store().createSpeaker(nextAutoName(), "#90A4AE", false, encodeEmbedding(centroid))); speakerRegistry.reload(); + // A failed insert returns -1; leave that cluster's segments as they were. + if (id < 0) continue; clusterSpeaker.set(cluster, id); newSpeakers += 1; } diff --git a/app/apps/microphones/speakers.ts b/app/apps/microphones/speakers.ts index 7d4a7189..5975ea24 100644 --- a/app/apps/microphones/speakers.ts +++ b/app/apps/microphones/speakers.ts @@ -139,8 +139,9 @@ export class SpeakerRegistry { })); this.loaded = true; } catch (error) { + // Keep the last good cache: emptying it would make the next utterance + // enroll an already-known voice as a duplicate "Speaker N". console.error(`speaker reload failed: ${String(error)}`); - this.profiles = []; } } @@ -194,7 +195,8 @@ export class SpeakerRegistry { const id = Number(store().createSpeaker(name, color, false, encodeEmbedding(embedding))); this.reload(); this.notifyChanged(); - const profile = this.byId(id)!; + const profile = this.byId(id); + if (!profile) throw new Error(`speaker "${name}" was not created (id ${id})`); return { profile, similarity: best?.similarity ?? 0, classification: "new", previousLastHeardAt: 0 }; } @@ -213,12 +215,14 @@ export class SpeakerRegistry { store().updateSpeakerEmbedding(existing.id, encodeEmbedding(embedding), CENTROID_UPDATE_CAP); this.reload(); this.notifyChanged(); - return this.byId(existing.id)!; + return this.byId(existing.id) ?? existing; } const id = Number(store().createSpeaker("Me", "#64B5F6", true, encodeEmbedding(embedding))); this.reload(); this.notifyChanged(); - return this.byId(id)!; + const profile = this.byId(id); + if (!profile) throw new Error(`wearer profile was not created (id ${id})`); + return profile; } /** Is this utterance the wearer's own voice? Used to gate command recognition. */ diff --git a/app/apps/microphones/translate.ts b/app/apps/microphones/translate.ts index d27bbfa4..25fee989 100644 --- a/app/apps/microphones/translate.ts +++ b/app/apps/microphones/translate.ts @@ -18,7 +18,10 @@ const downloadedPairs = new Set(); export function deviceLanguage(): string { if (!global.isAndroid) return "en"; try { - return String(java.util.Locale.getDefault().getLanguage() || "en"); + // toLanguageTag() gives the modern code ("he", "id", "yi") that ML Kit + // uses; getLanguage() returns the legacy "iw", "in", "ji". + const tag = String(java.util.Locale.getDefault().toLanguageTag() || ""); + return tag.split("-")[0] || "en"; } catch { return "en"; } @@ -95,11 +98,18 @@ export function translateText(text: string, sourceLang: string, targetLang: stri if (!translator) return Promise.resolve(""); const key = `${sourceLang}>${targetLang}`; return new Promise((resolve) => { + // Also runs from the download listener, where a throw would escape into + // the Play Services callback and leave this promise unsettled. const runTranslate = () => { - translator - .translate(text) - .addOnSuccessListener(successListener((result: any) => resolve(String(result)))) - .addOnFailureListener(failureListener(() => resolve(""))); + try { + translator + .translate(text) + .addOnSuccessListener(successListener((result: any) => resolve(String(result)))) + .addOnFailureListener(failureListener(() => resolve(""))); + } catch (error) { + console.warn(`translation failed (${key}): ${error}`); + resolve(""); + } }; if (downloadedPairs.has(key)) { runTranslate(); diff --git a/tests/microphones-dsp.test.cjs b/tests/microphones-dsp.test.cjs index 625a2aa8..9e8c4254 100644 --- a/tests/microphones-dsp.test.cjs +++ b/tests/microphones-dsp.test.cjs @@ -172,3 +172,21 @@ test("temple pipeline matches the raw beam with ANC disabled", () => { assert.equal(viaPipeline.length, direct.length); for (let i = 0; i < direct.length; i++) assert.equal(viaPipeline[i], direct[i]); }); + +test("temple pipeline output is seamless across frame boundaries", () => { + // Streamed frame by frame, the steering delay must reach into the previous + // frame rather than zero-fill, so splitting the signal changes nothing. + const front = makeNoise(800, 3); + const rear = makeNoise(800, 4); + const target = { azimuth: deg(60) }; + const whole = new TempleAudioPipeline("left", 16000); + whole.enableANC = false; + const expected = whole.process(front, rear, target); + const framed = new TempleAudioPipeline("left", 16000); + framed.enableANC = false; + const first = framed.process(front.subarray(0, 400), rear.subarray(0, 400), target); + const second = framed.process(front.subarray(400), rear.subarray(400), target); + assert.equal(first.length + second.length, expected.length); + for (let i = 0; i < 400; i++) assert.ok(near(first[i], expected[i], 1e-6)); + for (let i = 0; i < 400; i++) assert.ok(near(second[i], expected[400 + i], 1e-6)); +}); diff --git a/tests/microphones-name-guess.test.cjs b/tests/microphones-name-guess.test.cjs index bb70b924..b8328eb6 100644 --- a/tests/microphones-name-guess.test.cjs +++ b/tests/microphones-name-guess.test.cjs @@ -22,6 +22,12 @@ test("self-introductions are recognized", () => { name: "Alice", kind: "self", }); + // Phone-style self-introductions, including ASR's stray capitalization. + assert.deepEqual(inferIntroducedName("Hey this is Dana calling about the order"), { + name: "Dana", + kind: "self", + }); + assert.deepEqual(inferIntroducedName("This is Bob Speaking."), { name: "Bob", kind: "self" }); }); test("third-party introductions are recognized", () => {