Fix a bunch of microphone-related bugs

> Most serious fixes
> - Recordings kept forever: a recording made with captions off, or with "Save captions"
>   off, was transcoded to .m4a and kept indefinitely. No conversation linked to it, so
>   the phone UI never showed it and the retention settings never deleted it. Recording
>   now only happens alongside a saved caption session, and turning captions off ends
>   that session's recording properly.
> - 4-mic mode went silent after a reconnect: a disconnect clears the phone's audio
>   forwarding, and nothing turned it back on, so the app kept showing "Listening (4-mic
>   array)" while hearing nothing. A check every 3 s now restores forwarding and re-sends
>   the config when it has been lost. The normal single-stream mode already recovered on
>   its own.
> - Turning off "4-mic capture" mid-session went silent: it stopped the glasses' stream
>   but never switched to the normal stream. Setup changes now take effect live,
>   including switching between the two modes.
> - Setup changes were undone: after the assistant closed, the app re-sent the config it
>   loaded at launch to the glasses, reverting any changes you'd made since.
> - Startup failures: if the caption engine failed to start, the session got stuck
>   half-running. If the glasses weren't connected, listeners piled up and an empty
>   conversation could be left open. Startup now brings up the microphone first and
>   cleans up properly on failure.
>
> Other fixes
> - Audio quality: the 4-mic beamformer put a small dropout at the start of every 25 ms
>   frame (an audible 40 Hz artifact in the audio feeding captions). Fixed, with a test
>   that fails on the old code.
> - Stale direction data: an arm that stopped sending kept steering the direction
>   estimate and could hold the caption feed on a silent arm.
> - Compass: closing Microphones switched off the Compass app's compass, because both
>   used the same owner name.
> - Speaker database failures:
>   - A failed speaker reload wiped the in-memory list, so the next voice became a
>     duplicate "Speaker N".
>   - A failed speaker insert crashed caption handling; in re-diarize it reassigned
>     transcript lines to a nonexistent speaker.
> - Translation:
>   - A crash path inside the model-download callback is now caught.
>   - Phones set to Hebrew, Indonesian or Yiddish never got translations, because of
>     old-style language codes.
> - Ask questions: once the speaker list grew past the size limit (every unknown voice
>   adds a "Speaker N"), it pushed all transcript lines out of the context sent to the
>   model. When trimming is needed, the oldest lines now go first.
> - Smaller fixes:
>   - The captions view stopped jumping back to live speech after 60 lines.
>   - "This is Dana calling…" was treated as introducing someone else.
>   - A model download that failed to start was stuck at "downloading" until restart.
>   - The developer mic monitor grew without limit while hidden.
This commit is contained in:
James Babcock committed 2026-10-14 00:56:54 -07:00
1 parent 22c31f4240
commit b50898ea02
14 files changed
+409 -143

No files matched your search

+11 -2
View File
@@ -65,6 +65,14 @@ type ArmStats = {
statusAtMs: number | null;
};
/** Drop arrivals older than the rate window. */
function trimRecent(stats: ArmStats, nowMs: number): void {
const cutoff = nowMs - RATE_WINDOW_MS;
let drop = 0;
while (drop < stats.recent.length && stats.recent[drop]!.wallMs < cutoff) drop++;
if (drop > 0) stats.recent.splice(0, drop);
}
function emptyStats(): ArmStats {
return {
packets: 0,
@@ -169,6 +177,8 @@ export class MicrophonesMonitorLayer implements Layer {
this.advanceBuckets(nowMs);
stats.packets++;
stats.buckets[BUCKETS - 1]!++;
// Trim here too: painting (the other trim) stops while the page is hidden.
trimRecent(stats, nowMs);
stats.recent.push({ wallMs: nowMs, arrivalMs, bytes: data.length });
stats.lastArrivalWallMs = nowMs;
@@ -268,8 +278,7 @@ export class MicrophonesMonitorLayer implements Layer {
const font = getDefaultSmallFont();
const line = font.lineHeight;
const stats = this.stats[arm];
const cutoff = nowMs - RATE_WINDOW_MS;
while (stats.recent.length > 0 && stats.recent[0]!.wallMs < cutoff) stats.recent.shift();
trimRecent(stats, nowMs);
const rate = (stats.recent.length * 1000) / RATE_WINDOW_MS;
const byteRate = stats.recent.reduce((sum, arrival) => sum + arrival.bytes, 0) / RATE_WINDOW_MS;
let maxGap = 0;
+6 -3
View File
@@ -15,12 +15,15 @@ export class CaptionsLayer implements Layer {
private state: MicSessionState = micSession.getState();
private unsubscribe: (() => void) | null = null;
private scrollback = 0;
private lastLineCount = 0;
private lastLine: CaptionLine | null = null;
start(requestRender: () => void): void {
this.unsubscribe = micSession.onState((state) => {
if (state.captionLines.length !== this.lastLineCount) {
this.lastLineCount = state.captionLines.length;
// Watch the newest line itself: once the session's line cap is reached
// the count stops changing even though new speech keeps arriving.
const last = state.captionLines[state.captionLines.length - 1] ?? null;
if (last !== this.lastLine) {
this.lastLine = last;
this.scrollback = 0;
}
this.state = state;
+47 -17
View File
@@ -16,6 +16,10 @@ declare const com: any;
// Qwen runs with a 4096-token context; leave room for the answer.
const MAX_CONTEXT_CHARS = 7000;
// The speaker roster grows with every voice ever heard; cap it and the
// conversation list so they can't crowd the transcript excerpts out.
const MAX_HEADER_CHARS = MAX_CONTEXT_CHARS / 2;
const EXCERPTS_HEADING = "# Transcript excerpts\n";
const MAX_KEYWORDS = 6;
const SEGMENTS_PER_KEYWORD = 30;
const RECENT_SEGMENT_LIMIT = 120;
@@ -156,31 +160,57 @@ function buildContext(question: string): { context: string; note: string } {
}
}
const segments = [...segmentMap.values()].sort((a, b) => a.startedAt - b.startedAt);
let header =
`# Known speakers\n${speakerLines.join("\n") || "(none yet)"}\n\n` +
`# Conversations${filters.sinceMs !== null || filters.emotion !== null ? " (filtered by the question)" : ""}\n` +
`${sessionLines.join("\n") || "(none in range)"}\n\n`;
let truncated = false;
if (header.length > MAX_HEADER_CHARS) {
header = `${header.slice(0, MAX_HEADER_CHARS)}\n(...)\n\n`;
truncated = true;
}
const sessionHeading = (sessionId: number) =>
`\n## ${sessionTitles.get(sessionId) ?? `Conversation #${sessionId}`}`;
const segmentText = (segment: SegmentRow) => {
const speaker =
segment.speakerId !== null ? (speakerNames.get(segment.speakerId) ?? "Speaker") : "Speaker";
const emotion = segment.emotion && segment.emotion !== "neutral" ? ` [${segment.emotion}]` : "";
const line = `[${formatWhen(segment.startedAt)}] ${speaker}${emotion}: ${segment.text}`;
return segment.translation ? `${line}\n (translated: ${segment.translation})` : line;
};
// Fill the remaining budget newest-first, so an overflowing history drops
// its oldest excerpts rather than its most recent ones.
const sorted = [...segmentMap.values()].sort((a, b) => a.startedAt - b.startedAt);
let budget = MAX_CONTEXT_CHARS - header.length - EXCERPTS_HEADING.length;
const segments: SegmentRow[] = [];
let runSession = -1;
for (let index = sorted.length - 1; index >= 0; index--) {
const segment = sorted[index]!;
let cost = segmentText(segment).length + 1;
if (segment.sessionId !== runSession) cost += sessionHeading(segment.sessionId).length + 1;
if (cost > budget) {
truncated = true;
break;
}
budget -= cost;
runSession = segment.sessionId;
segments.push(segment);
}
segments.reverse();
const excerptLines: string[] = [];
let currentSession = -1;
for (const segment of segments) {
if (segment.sessionId !== currentSession) {
currentSession = segment.sessionId;
excerptLines.push(`\n## ${sessionTitles.get(currentSession) ?? `Conversation #${currentSession}`}`);
excerptLines.push(sessionHeading(currentSession));
}
const speaker =
segment.speakerId !== null ? (speakerNames.get(segment.speakerId) ?? "Speaker") : "Speaker";
const emotion = segment.emotion && segment.emotion !== "neutral" ? ` [${segment.emotion}]` : "";
excerptLines.push(`[${formatWhen(segment.startedAt)}] ${speaker}${emotion}: ${segment.text}`);
if (segment.translation) excerptLines.push(` (translated: ${segment.translation})`);
excerptLines.push(segmentText(segment));
}
let context =
`# Known speakers\n${speakerLines.join("\n") || "(none yet)"}\n\n` +
`# Conversations${filters.sinceMs !== null || filters.emotion !== null ? " (filtered by the question)" : ""}\n` +
`${sessionLines.join("\n") || "(none in range)"}\n\n` +
`# Transcript excerpts\n${excerptLines.join("\n") || "(no matching lines)"}`;
let truncated = false;
if (context.length > MAX_CONTEXT_CHARS) {
context = context.slice(0, MAX_CONTEXT_CHARS);
truncated = true;
}
const context = `${header}${EXCERPTS_HEADING}${excerptLines.join("\n") || "(no matching lines)"}`;
const note =
`${sessions.length} conversation${sessions.length === 1 ? "" : "s"}, ` +
`${segments.length} transcript line${segments.length === 1 ? "" : "s"} consulted` +
+27 -2
View File
@@ -286,6 +286,12 @@ export function angleInArc(azimuth: number, arcCenter: number, arcHalfWidth: num
export class TempleAudioPipeline {
private readonly beamformer: DelaySumBeamformer;
private readonly anc = new InterMicNoiseCanceller();
// Steering delays reach back before the start of each frame. Carrying the
// previous frame's last samples keeps those real; zero-filling them (as a
// standalone beam() does) dips the output at every frame boundary.
private readonly carry: number;
private frontTail: Float32Array;
private rearTail: Float32Array;
enableANC = true;
constructor(
@@ -294,19 +300,38 @@ export class TempleAudioPipeline {
geometry: MicGeometry = G2_DEFAULT_GEOMETRY,
) {
this.beamformer = new DelaySumBeamformer(sampleRate, geometry);
const { front, rear } = geometry.positions[side];
const spacing = Math.hypot(front.x - rear.x, front.y - rear.y, front.z - rear.z);
this.carry = Math.ceil((spacing / geometry.speedOfSound) * sampleRate) + 1;
this.frontTail = new Float32Array(this.carry);
this.rearTail = new Float32Array(this.carry);
}
/**
* Process one frame's deinterleaved front/rear channels toward a
* device-frame target, returning the enhanced mono beam for this temple.
* Frames must be consecutive: each one continues the previous.
*/
process(front: Float32Array, rear: Float32Array, target: Bearing): Float32Array {
const beam = this.beamformer.beam(this.side, front, rear, target);
const n = Math.min(front.length, rear.length);
const extendedFront = this.withTail(this.frontTail, front, n);
const extendedRear = this.withTail(this.rearTail, rear, n);
this.frontTail = extendedFront.slice(n);
this.rearTail = extendedRear.slice(n);
const beam = this.beamformer
.beam(this.side, extendedFront, extendedRear, target)
.slice(this.carry);
if (!this.enableANC) return beam;
// Null-steer reference: the difference channel carries off-axis noise.
const n = Math.min(front.length, rear.length);
const reference = new Float32Array(n);
for (let i = 0; i < n; i++) reference[i] = front[i]! - rear[i]!;
return this.anc.process(beam, reference);
}
private withTail(tail: Float32Array, frame: Float32Array, n: number): Float32Array {
const out = new Float32Array(tail.length + n);
out.set(tail);
out.set(frame.subarray(0, n), tail.length);
return out;
}
}
+2 -1
View File
@@ -110,6 +110,7 @@ class MicArrayController {
* doesn't spend BLE bandwidth.
*/
apply(config: MicConfig): boolean {
if (config.armHardware && !micControlSupported()) return false;
const leftActive = templeActive(config, "left");
const rightActive = templeActive(config, "right");
if (leftActive || rightActive) {
@@ -148,8 +149,8 @@ class MicArrayController {
): boolean {
const communicator = activeCommunicator();
if (!communicator) return false;
this.ensureStatusProxy(communicator);
try {
this.ensureStatusProxy(communicator);
const record = encodeMicControl(op, config);
communicator.sendFaceclawMicControl(toJavaBytes(record), label, rightTemple, leftTemple);
return true;
+17 -9
View File
@@ -142,6 +142,7 @@ function notifyStateChanged(id: MicModelId): void {
export function startMicModelDownload(id: MicModelId): void {
if (!global.isAndroid || downloaders.has(id) || isMicModelReady(id)) return;
progress.delete(id);
downloadNextFile(id);
notifyStateChanged(id);
}
@@ -171,15 +172,22 @@ function downloadNextFile(id: MicModelId): void {
notifyStateChanged(id);
},
});
const downloader = new com.faceclaw.app.FaceclawModelDownloader(
`${definition.baseUrl}${nextFile.remoteName}`,
filePath(id, nextFile),
nextFile.sha256,
nextFile.sizeBytes,
listener,
);
downloaders.set(id, downloader);
downloader.start();
try {
const downloader = new com.faceclaw.app.FaceclawModelDownloader(
`${definition.baseUrl}${nextFile.remoteName}`,
filePath(id, nextFile),
nextFile.sha256,
nextFile.sizeBytes,
listener,
);
downloaders.set(id, downloader);
downloader.start();
} catch (error) {
// Without this, a dead entry would pin the model at "downloading".
console.error(`Mic model download failed to start (${id}/${nextFile.name}): ${String(error)}`);
downloaders.delete(id);
notifyStateChanged(id);
}
}
export function cancelMicModelDownload(id: MicModelId): void {
+244 -92
View File
@@ -95,6 +95,13 @@ const RECORDING_AAC_BITRATE = 32_000;
const ENCOUNTER_GAP_MS = 15 * 60 * 1000;
const ENCOUNTER_CARD_TTL_MS = 12_000;
const ENCOUNTER_CARD_MAX_ACTION_ITEMS = 2;
// A temple that has sent no frame (or no bearing) for this long no longer
// counts: its stream stopped, its mics were toggled off, or its arm dropped.
const TEMPLE_STALE_MS = 1000;
// How often the extended path checks that its forwarding survived the
// glasses session (an arm drop clears it).
const CAPTURE_CHECK_MS = 3000;
const COMPASS_OWNER = "microphones";
export type CaptionLine = {
speakerId: number | null;
@@ -181,6 +188,8 @@ class MicSession {
private temples = new Map<MicSide, TempleAudioPipeline>();
private templeAngles: { left: number | null; right: number | null } = { left: null, right: null };
private templeSsr: { left: number; right: number } = { left: 0, right: 0 };
private templeAngleAtMs: Record<MicSide, number> = { left: 0, right: 0 };
private templeFrameAtMs: Record<MicSide, number> = { left: 0, right: 0 };
private captionFeedSide: MicSide = "left";
// Cached at capture start so the ~80 Hz packet path never re-reads settings.
private activeConfig: MicConfig = loadMicConfig();
@@ -202,6 +211,7 @@ class MicSession {
private speechActive = false;
private renderTimer: ReturnType<typeof setInterval> | null = null;
private captureCheckTimer: ReturnType<typeof setInterval> | null = null;
onState(listener: StateListener): () => void {
this.stateListeners.add(listener);
@@ -262,33 +272,33 @@ class MicSession {
this.activeConfig = loadMicConfig();
this.refreshCachedFlags();
const extended = micControlSupported() && this.activeConfig.armHardware;
this.mode = extended ? "extended" : "stock";
this.running = true;
// Capture first, so a failure has no captions session, recording, or
// subscriptions to unwind.
if (!this.startCapture()) {
this.statusText = "Microphone unavailable (glasses not connected?)";
this.running = false;
this.mode = "off";
this.notify();
return;
}
this.statusText = this.listeningText();
this.offSettings = onSettingsStoreChanged(() => this.refreshCachedFlags());
this.offSpeakerChanges = speakerRegistry.onChanged(() => this.refreshSpeakerIdentities());
this.offVoiceActivity = voiceActivity.subscribe((active) => this.handleVoiceActivity(active));
const extended = micControlSupported() && this.activeConfig.armHardware;
this.mode = extended ? "extended" : "stock";
this.startSuppressor();
this.startCompass();
if (captionsEnabledSetting.get()) {
this.startCaptions();
}
if (saveRecordingsSetting.get() && recordingsRetentionSetting.get() !== "none") {
if (this.shouldRecord()) {
this.startRecording();
}
const ok = extended ? this.startExtended() : this.startStock();
if (!ok) {
this.statusText = "Microphone unavailable (glasses not connected?)";
this.running = false;
this.stopCaptions();
this.stopRecording(false);
this.stopCompass();
this.mode = "off";
this.notify();
return;
}
this.statusText = extended ? "Listening (4-mic array)" : "Listening";
// Levels/DOA update at packet rate; coalesce UI renders to 5 Hz.
this.renderTimer = setInterval(() => this.notify(), 200);
this.captureCheckTimer = setInterval(() => this.checkExtendedCapture(), CAPTURE_CHECK_MS);
this.notify();
}
@@ -299,11 +309,11 @@ class MicSession {
clearInterval(this.renderTimer);
this.renderTimer = null;
}
if (this.mode === "extended") {
this.stopExtended();
} else {
this.stopStock();
if (this.captureCheckTimer) {
clearInterval(this.captureCheckTimer);
this.captureCheckTimer = null;
}
this.stopCapture();
this.offSettings?.();
this.offSettings = null;
this.offSpeakerChanges?.();
@@ -324,6 +334,55 @@ class MicSession {
void this.applyRetentionSweep();
}
private listeningText(): string {
return this.mode === "extended" ? "Listening (4-mic array)" : "Listening";
}
/** Start the current mode's capture path, undoing any partial start on failure. */
private startCapture(): boolean {
try {
if (this.mode === "extended" ? this.startExtended() : this.startStock()) return true;
} catch (error) {
console.warn(`mic capture start failed: ${error}`);
}
this.stopCapture();
return false;
}
private stopCapture(): void {
if (this.mode === "extended") {
this.stopExtended();
} else {
this.stopStock();
}
}
/**
* Persist a Microphone setup change. While running it takes effect live:
* toggling 4-mic capture swaps the capture path, and any other change is
* re-sent to the streaming temples.
*/
updateConfig(mutate: (config: MicConfig) => MicConfig): void {
const config = mutate(loadMicConfig());
saveMicConfig(config);
this.activeConfig = config;
if (this.running) {
const extended = micControlSupported() && config.armHardware;
if (extended !== (this.mode === "extended")) {
this.stopCapture();
this.mode = extended ? "extended" : "stock";
this.doaDeviceDeg = null;
this.ssr = 0;
this.statusText = this.startCapture()
? this.listeningText()
: "Microphone unavailable (glasses not connected?)";
} else if (extended && !voiceActivity.isActive()) {
micArrayController.apply(wireConfig(config));
}
}
this.notify();
}
private refreshCachedFlags(): void {
this.beamFilterOn = beamFilterSetting.get();
const wasOn = this.ancOn;
@@ -380,13 +439,34 @@ class MicSession {
if (!communicator) return;
if (this.mode === "extended") {
if (this.forwardingProxy && communicator.startG2AudioForwarding(this.forwardingProxy)) {
micArrayController.apply({ ...this.activeConfig, channelMask: 0b11 });
micArrayController.apply(wireConfig(this.activeConfig));
}
} else {
voiceControlBridge.startRawCapture({ communicator });
}
}
/**
* The extended stream's forwarding lives in the glasses session: an arm
* drop clears it and nothing else re-arms it on the next session, so the
* app would sit at "Listening" hearing nothing. Re-establish forwarding
* and re-send the config whenever a ready session has lost it. (The stock
* path is resumed by the voice-control bridge.)
*/
private checkExtendedCapture(): void {
if (!this.running || this.mode !== "extended" || !this.forwardingProxy) return;
if (voiceActivity.isActive()) return;
try {
const communicator = this.nativeCommunicator();
if (!communicator?.isSessionReady() || communicator.isAudioCaptureActive()) return;
if (communicator.startG2AudioForwarding(this.forwardingProxy)) {
micArrayController.apply(wireConfig(this.activeConfig));
}
} catch (error) {
console.warn(`mic capture re-arm failed: ${error}`);
}
}
private startSuppressor(): void {
try {
this.suppressor = new com.faceclaw.app.FaceclawNoiseSuppressor(SAMPLE_RATE);
@@ -459,7 +539,7 @@ class MicSession {
private startCompass(): void {
try {
setCompassEnabled(true);
setCompassEnabled(true, COMPASS_OWNER);
this.offCompass = addCompassListener((event) => {
if (event.headingDegrees >= 0) {
this.headingDeg = event.headingDegrees;
@@ -474,7 +554,7 @@ class MicSession {
this.offCompass?.();
this.offCompass = null;
try {
setCompassEnabled(false);
setCompassEnabled(false, COMPASS_OWNER);
} catch {
// Disconnected; the stream is already gone.
}
@@ -542,11 +622,7 @@ class MicSession {
},
});
if (!communicator.startG2AudioForwarding(this.forwardingProxy)) return false;
if (!micArrayController.apply({ ...this.activeConfig, channelMask: 0b11 })) {
communicator.stopG2AudioForwarding();
return false;
}
return true;
return micArrayController.apply(wireConfig(this.activeConfig));
}
/**
@@ -555,14 +631,7 @@ class MicSession {
* captions; a temple with both mics off has its stream stopped entirely.
*/
setMicChannelEnabled(key: MicChannelKey, enabled: boolean): void {
const config = loadMicConfig();
config.hostMics = { ...config.hostMics, [key]: enabled };
saveMicConfig(config);
this.activeConfig = config;
if (this.running && this.mode === "extended") {
micArrayController.apply({ ...config, channelMask: 0b11 });
}
this.notify();
this.updateConfig((config) => ({ ...config, hostMics: { ...config.hostMics, [key]: enabled } }));
}
private stopExtended(): void {
@@ -576,6 +645,14 @@ class MicSession {
this.forwardingProxy = null;
this.temples.clear();
this.templeAngles = { left: null, right: null };
this.templeSsr = { left: 0, right: 0 };
this.templeAngleAtMs = { left: 0, right: 0 };
this.templeFrameAtMs = { left: 0, right: 0 };
}
/** A temple's last bearing, or null once it has gone quiet. */
private freshTempleAngle(side: MicSide, nowMs: number): number | null {
return nowMs - this.templeAngleAtMs[side] < TEMPLE_STALE_MS ? this.templeAngles[side] : null;
}
private handleExtendedPacket(data: Uint8Array, arm: string): void {
@@ -588,6 +665,8 @@ class MicSession {
const side: MicSide = arm === "L" ? "left" : "right";
const channels = splitInterleavedPcm16(frame);
if (!channels || channels.length === 0) return;
const now = Date.now();
this.templeFrameAtMs[side] = now;
// Meter every arriving channel — a host-disabled mic still shows its
// live level (marked off in the UI), it just doesn't feed anything.
@@ -606,8 +685,10 @@ class MicSession {
if (frame.angleDegrees !== 0 || frame.ssr !== 0) {
this.templeAngles[side] = frame.angleDegrees;
this.templeSsr[side] = frame.ssr;
const left = this.templeAngles.left;
const right = this.templeAngles.right;
this.templeAngleAtMs[side] = now;
// A temple that stopped reporting must not keep steering the fusion.
const left = this.freshTempleAngle("left", now);
const right = this.freshTempleAngle("right", now);
let fusedDeg: number;
if (left !== null && right !== null) {
const fused = fuseBearings(
@@ -621,7 +702,7 @@ class MicSession {
fusedDeg = (left ?? right)!;
}
this.doaDeviceDeg = smoothAngleDeg(this.doaDeviceDeg, fusedDeg, DOA_SMOOTHING);
this.ssr = Math.max(this.templeSsr.left, this.templeSsr.right);
this.ssr = Math.max(left === null ? 0 : this.templeSsr.left, right === null ? 0 : this.templeSsr.right);
if (this.beamLocked && this.doaDeviceDeg !== null) {
this.beamWorldDeg = this.toWorldDeg(this.doaDeviceDeg);
}
@@ -651,10 +732,16 @@ class MicSession {
private deliverExtendedAudio(side: MicSide, samples: Float32Array): void {
// Feed captions from one temple only; prefer the one hearing more
// signal, among the temples that still have an enabled microphone.
const leftUsable = templeActive(this.activeConfig, "left");
const rightUsable = templeActive(this.activeConfig, "right");
let stronger: MicSide = this.templeSsr.right > this.templeSsr.left * 1.5 ? "right" : "left";
// signal, among the temples that still have an enabled microphone and
// are still streaming.
const now = Date.now();
const leftUsable =
templeActive(this.activeConfig, "left") && now - this.templeFrameAtMs.left < TEMPLE_STALE_MS;
const rightUsable =
templeActive(this.activeConfig, "right") && now - this.templeFrameAtMs.right < TEMPLE_STALE_MS;
const leftSsr = this.freshTempleAngle("left", now) === null ? 0 : this.templeSsr.left;
const rightSsr = this.freshTempleAngle("right", now) === null ? 0 : this.templeSsr.right;
let stronger: MicSide = rightSsr > leftSsr * 1.5 ? "right" : "left";
if (stronger === "right" && !rightUsable) stronger = "left";
if (stronger === "left" && !leftUsable) stronger = "right";
if (strongerChanged(this.captionFeedSide, stronger)) {
@@ -707,9 +794,12 @@ class MicSession {
if (!this.running) return;
if (enabled && !this.captionsActive) {
this.startCaptions();
if (saveRecordingsSetting.get() && !this.recorder) this.startRecording();
if (!this.recorder && this.shouldRecord()) this.startRecording();
} else if (!enabled && this.captionsActive) {
this.stopCaptions();
// The recording belongs to this caption session (its segment offsets
// are relative to this engine's clock); close it out with the row.
this.stopRecording(true);
this.finishSessionRow();
}
this.notify();
@@ -717,33 +807,56 @@ class MicSession {
private startCaptions(): void {
if (this.captionsActive) return;
const engine = new com.faceclaw.app.FaceclawCaptionEngine();
this.captionListenerProxy = new com.faceclaw.app.FaceclawCaptionEngineListener({
onUtterance: (text: string, embedding: any, startMs: number, endMs: number, peakRms: number) => {
this.handleUtterance(String(text), embedding, Number(startMs), Number(endMs), Number(peakRms));
},
onSpeechStart: (_startMs: number) => {
this.speechActive = true;
},
onStatus: (status: string) => {
this.statusText = String(status);
},
});
engine.setListener(this.captionListenerProxy);
// Live captions are a separate sherpa-onnx pipeline (FaceclawCaptionEngine)
// that only ever uses Moonshine; unaffected by the two-model on-device
// Whisper addition in native/voice-control.ts's push-to-talk path.
engine.setAsrModelDir(isAsrModelReady("moonshine") ? moonshineModelDir() : null);
engine.setSpeakerModelPath(micModelPath("speaker-embedding"));
engine.start();
let engine: any = null;
try {
engine = new com.faceclaw.app.FaceclawCaptionEngine();
this.captionListenerProxy = new com.faceclaw.app.FaceclawCaptionEngineListener({
onUtterance: (text: string, embedding: any, startMs: number, endMs: number, peakRms: number) => {
// Never let an exception escape into the engine's Java callback.
try {
this.handleUtterance(String(text), embedding, Number(startMs), Number(endMs), Number(peakRms));
} catch (error) {
console.warn(`caption utterance failed: ${error}`);
}
},
onSpeechStart: (_startMs: number) => {
this.speechActive = true;
},
onStatus: (status: string) => {
this.statusText = String(status);
},
});
engine.setListener(this.captionListenerProxy);
// Live captions are a separate sherpa-onnx pipeline (FaceclawCaptionEngine)
// that only ever uses Moonshine; unaffected by the two-model on-device
// Whisper addition in native/voice-control.ts's push-to-talk path.
engine.setAsrModelDir(isAsrModelReady("moonshine") ? moonshineModelDir() : null);
engine.setSpeakerModelPath(micModelPath("speaker-embedding"));
engine.start();
} catch (error) {
console.warn(`caption engine start failed: ${error}`);
try {
engine?.stop();
} catch {
// It never started.
}
this.captionListenerProxy = null;
this.statusText = "Captions unavailable";
return;
}
this.captionEngine = engine;
this.captionsActive = true;
this.engineStartWallMs = Date.now();
this.encounterShown.clear();
if (saveCaptionsSetting.get() && captionsRetentionSetting.get() !== "none") {
this.sessionRowId = Number(
this.store().startSession(Date.now(), `Conversation ${new Date().toLocaleString()}`),
);
try {
this.sessionRowId = Number(
this.store().startSession(Date.now(), `Conversation ${new Date().toLocaleString()}`),
);
} catch (error) {
console.warn(`caption session start failed: ${error}`);
this.sessionRowId = -1;
}
}
}
@@ -892,10 +1005,23 @@ class MicSession {
const embeddingFloats = embedding ? floatArrayToFloat32(embedding) : null;
if (this.wearerEnrollmentArmed && embeddingFloats) {
this.wearerEnrollmentArmed = false;
const wearer = speakerRegistry.enrollWearer(embeddingFloats);
this.statusText = `Enrolled your voice as "${wearer.name}"`;
try {
const wearer = speakerRegistry.enrollWearer(embeddingFloats);
this.statusText = `Enrolled your voice as "${wearer.name}"`;
} catch (error) {
console.warn(`wearer enrollment failed: ${error}`);
this.statusText = "Voice enrollment failed";
}
}
let match: SpeakerMatch | null = null;
if (embeddingFloats) {
try {
match = speakerRegistry.assign(embeddingFloats);
} catch (error) {
// Still caption the line, just unattributed.
console.warn(`speaker assignment failed: ${error}`);
}
}
let match = embeddingFloats ? speakerRegistry.assign(embeddingFloats) : null;
if (match) this.maybeShowEncounterCard(match);
// Spoken introductions name the auto-numbered speakers: "I'm Alice"
// renames the speaker of the line; "this is Bob" pends the name for the
@@ -957,23 +1083,27 @@ class MicSession {
let segmentId = -1;
if (this.sessionRowId >= 0 && line.text) {
segmentId = Number(
this.store().insertSegment(
JSON.stringify({
sessionId: this.sessionRowId,
speakerId: match?.profile.id,
startedAt: this.engineStartWallMs + startMs,
endedAt: this.engineStartWallMs + endMs,
audioOffsetMs: startMs,
text: line.text,
sentiment: line.sentiment,
emotion: line.emotion,
searchMeta: analyzed ? searchableMetadata(analyzed.emotion, analyzed.score) : "",
angle: doaAtUtterance === null ? undefined : Math.round(doaAtUtterance),
embeddingBase64: embeddingFloats ? encodeEmbeddingBase64(embeddingFloats) : undefined,
}),
),
);
try {
segmentId = Number(
this.store().insertSegment(
JSON.stringify({
sessionId: this.sessionRowId,
speakerId: match?.profile.id,
startedAt: this.engineStartWallMs + startMs,
endedAt: this.engineStartWallMs + endMs,
audioOffsetMs: startMs,
text: line.text,
sentiment: line.sentiment,
emotion: line.emotion,
searchMeta: analyzed ? searchableMetadata(analyzed.emotion, analyzed.score) : "",
angle: doaAtUtterance === null ? undefined : Math.round(doaAtUtterance),
embeddingBase64: embeddingFloats ? encodeEmbeddingBase64(embeddingFloats) : undefined,
}),
),
);
} catch (error) {
console.warn(`caption segment save failed: ${error}`);
}
}
this.notify();
@@ -1024,6 +1154,19 @@ class MicSession {
// ---- recording ----
/**
* Recordings belong to a saved caption session: retention and the phone UI
* find audio only through its session row, so a file recorded without one
* would never be shown or expired.
*/
private shouldRecord(): boolean {
return (
this.sessionRowId >= 0 &&
saveRecordingsSetting.get() &&
recordingsRetentionSetting.get() !== "none"
);
}
private startRecording(): void {
try {
const context = Utils.android.getApplicationContext();
@@ -1044,18 +1187,17 @@ class MicSession {
if (!recorder) return;
try {
const bytes = Number(recorder.finish());
if (!keep || bytes < SAMPLE_RATE) {
// Under half a second of audio isn't worth keeping.
const sessionRowId = this.sessionRowId;
if (!keep || bytes < SAMPLE_RATE || sessionRowId < 0) {
// Under half a second of audio isn't worth keeping, and audio with no
// session row to own it would be unreachable and never expire.
recorder.discard();
return;
}
const wavPath = this.recordingPath;
const m4aPath = wavPath.replace(/\.wav$/, ".m4a");
const sessionRowId = this.sessionRowId;
const store = this.store();
if (sessionRowId >= 0) {
store.setSessionAudio(sessionRowId, wavPath, "wav");
}
store.setSessionAudio(sessionRowId, wavPath, "wav");
// Transcode in the background; swap the session's audio to the M4A on
// success (the transcoder deletes the WAV).
const transcoder = new com.faceclaw.app.FaceclawAudioTranscoder(
@@ -1065,8 +1207,10 @@ class MicSession {
true,
new com.faceclaw.app.FaceclawAudioTranscoder.Listener({
onDone: (outputPath: string, _outputBytes: number) => {
if (sessionRowId >= 0) {
try {
store.setSessionAudio(sessionRowId, String(outputPath), "aac");
} catch (error) {
console.warn(`recording audio update failed: ${error}`);
}
},
onError: (message: string) => {
@@ -1129,6 +1273,14 @@ class MicSession {
// ---- helpers ----
/**
* The config as sent to the temples: each active temple always streams its
* full front+rear pair, and per-mic selection happens host-side.
*/
function wireConfig(config: MicConfig): MicConfig {
return { ...config, channelMask: 0b11 };
}
function pcmLevel(pcm: Uint8Array): number {
let sumSquares = 0;
const count = pcm.length / 2;
+3 -7
View File
@@ -22,7 +22,7 @@ import { isAsrModelReady, startAsrModelDownload, asrModelState } from "../../nat
import { CaptionsLayer } from "./captions-layer";
import { LevelsLayer } from "./levels-layer";
import { RadarLayer } from "./radar-layer";
import { loadMicConfig, micArrayController, micControlSupported, saveMicConfig } from "./mic-control";
import { loadMicConfig, micArrayController, micControlSupported } from "./mic-control";
import { isMicModelReady, micModelState, startMicModelDownload } from "./mic-models";
import { micChannelLabel, type MicChannelKey, type MicConfig } from "./mic-protocol";
import { micSession } from "./mic-session";
@@ -72,13 +72,9 @@ const MENU_LAYOUT = {
opaque: true,
};
/** Persist a config change and re-apply it live when the array is streaming. */
/** Persist a config change; the running session applies it live. */
function updateMicConfig(mutate: (config: MicConfig) => MicConfig): void {
const updated = mutate(loadMicConfig());
saveMicConfig(updated);
if (micSession.getState().mode === "extended") {
micArrayController.apply(updated);
}
micSession.updateConfig(mutate);
}
function micChannelToggleItem(
+3 -1
View File
@@ -55,7 +55,9 @@ const SELF_PATTERNS = [
new RegExp(`\\b[Mm]y name is ${NAME}`),
new RegExp(`\\b[Ii](?:'| a)m ${NAME}`),
new RegExp(`\\b[Cc]all me ${NAME}`),
new RegExp(`\\b[Tt]his is ${NAME} speaking\\b`),
// Phone-style: "this is Dana calling about..." is the speaker, not a
// third party being introduced.
new RegExp(`\\b[Tt]his is ${NAME} (?:[Ss]peaking|[Cc]alling)\\b`),
];
const THIRD_PARTY_PATTERNS = [
+2
View File
@@ -140,6 +140,8 @@ function applyTurns(sessionId: number, turns: Turn[]): RediarizeResult {
} else {
const id = Number(store().createSpeaker(nextAutoName(), "#90A4AE", false, encodeEmbedding(centroid)));
speakerRegistry.reload();
// A failed insert returns -1; leave that cluster's segments as they were.
if (id < 0) continue;
clusterSpeaker.set(cluster, id);
newSpeakers += 1;
}
+8 -4
View File
@@ -139,8 +139,9 @@ export class SpeakerRegistry {
}));
this.loaded = true;
} catch (error) {
// Keep the last good cache: emptying it would make the next utterance
// enroll an already-known voice as a duplicate "Speaker N".
console.error(`speaker reload failed: ${String(error)}`);
this.profiles = [];
}
}
@@ -194,7 +195,8 @@ export class SpeakerRegistry {
const id = Number(store().createSpeaker(name, color, false, encodeEmbedding(embedding)));
this.reload();
this.notifyChanged();
const profile = this.byId(id)!;
const profile = this.byId(id);
if (!profile) throw new Error(`speaker "${name}" was not created (id ${id})`);
return { profile, similarity: best?.similarity ?? 0, classification: "new", previousLastHeardAt: 0 };
}
@@ -213,12 +215,14 @@ export class SpeakerRegistry {
store().updateSpeakerEmbedding(existing.id, encodeEmbedding(embedding), CENTROID_UPDATE_CAP);
this.reload();
this.notifyChanged();
return this.byId(existing.id)!;
return this.byId(existing.id) ?? existing;
}
const id = Number(store().createSpeaker("Me", "#64B5F6", true, encodeEmbedding(embedding)));
this.reload();
this.notifyChanged();
return this.byId(id)!;
const profile = this.byId(id);
if (!profile) throw new Error(`wearer profile was not created (id ${id})`);
return profile;
}
/** Is this utterance the wearer's own voice? Used to gate command recognition. */
+15 -5
View File
@@ -18,7 +18,10 @@ const downloadedPairs = new Set<string>();
export function deviceLanguage(): string {
if (!global.isAndroid) return "en";
try {
return String(java.util.Locale.getDefault().getLanguage() || "en");
// toLanguageTag() gives the modern code ("he", "id", "yi") that ML Kit
// uses; getLanguage() returns the legacy "iw", "in", "ji".
const tag = String(java.util.Locale.getDefault().toLanguageTag() || "");
return tag.split("-")[0] || "en";
} catch {
return "en";
}
@@ -95,11 +98,18 @@ export function translateText(text: string, sourceLang: string, targetLang: stri
if (!translator) return Promise.resolve("");
const key = `${sourceLang}>${targetLang}`;
return new Promise((resolve) => {
// Also runs from the download listener, where a throw would escape into
// the Play Services callback and leave this promise unsettled.
const runTranslate = () => {
translator
.translate(text)
.addOnSuccessListener(successListener((result: any) => resolve(String(result))))
.addOnFailureListener(failureListener(() => resolve("")));
try {
translator
.translate(text)
.addOnSuccessListener(successListener((result: any) => resolve(String(result))))
.addOnFailureListener(failureListener(() => resolve("")));
} catch (error) {
console.warn(`translation failed (${key}): ${error}`);
resolve("");
}
};
if (downloadedPairs.has(key)) {
runTranslate();
+18
View File
@@ -172,3 +172,21 @@ test("temple pipeline matches the raw beam with ANC disabled", () => {
assert.equal(viaPipeline.length, direct.length);
for (let i = 0; i < direct.length; i++) assert.equal(viaPipeline[i], direct[i]);
});
test("temple pipeline output is seamless across frame boundaries", () => {
// Streamed frame by frame, the steering delay must reach into the previous
// frame rather than zero-fill, so splitting the signal changes nothing.
const front = makeNoise(800, 3);
const rear = makeNoise(800, 4);
const target = { azimuth: deg(60) };
const whole = new TempleAudioPipeline("left", 16000);
whole.enableANC = false;
const expected = whole.process(front, rear, target);
const framed = new TempleAudioPipeline("left", 16000);
framed.enableANC = false;
const first = framed.process(front.subarray(0, 400), rear.subarray(0, 400), target);
const second = framed.process(front.subarray(400), rear.subarray(400), target);
assert.equal(first.length + second.length, expected.length);
for (let i = 0; i < 400; i++) assert.ok(near(first[i], expected[i], 1e-6));
for (let i = 0; i < 400; i++) assert.ok(near(second[i], expected[400 + i], 1e-6));
});
+6
View File
@@ -22,6 +22,12 @@ test("self-introductions are recognized", () => {
name: "Alice",
kind: "self",
});
// Phone-style self-introductions, including ASR's stray capitalization.
assert.deepEqual(inferIntroducedName("Hey this is Dana calling about the order"), {
name: "Dana",
kind: "self",
});
assert.deepEqual(inferIntroducedName("This is Bob Speaking."), { name: "Bob", kind: "self" });
});
test("third-party introductions are recognized", () => {