mirror of
https://git.undefined-dev.fr/Blendman974/faceclaw.git
synced 2026-11-19 21:03:09 +00:00
Fix a bunch of microphone-related bugs
> Most serious fixes > - Recordings kept forever: a recording made with captions off, or with "Save captions" > off, was transcoded to .m4a and kept indefinitely. No conversation linked to it, so > the phone UI never showed it and the retention settings never deleted it. Recording > now only happens alongside a saved caption session, and turning captions off ends > that session's recording properly. > - 4-mic mode went silent after a reconnect: a disconnect clears the phone's audio > forwarding, and nothing turned it back on, so the app kept showing "Listening (4-mic > array)" while hearing nothing. A check every 3 s now restores forwarding and re-sends > the config when it has been lost. The normal single-stream mode already recovered on > its own. > - Turning off "4-mic capture" mid-session went silent: it stopped the glasses' stream > but never switched to the normal stream. Setup changes now take effect live, > including switching between the two modes. > - Setup changes were undone: after the assistant closed, the app re-sent the config it > loaded at launch to the glasses, reverting any changes you'd made since. > - Startup failures: if the caption engine failed to start, the session got stuck > half-running. If the glasses weren't connected, listeners piled up and an empty > conversation could be left open. Startup now brings up the microphone first and > cleans up properly on failure. > > Other fixes > - Audio quality: the 4-mic beamformer put a small dropout at the start of every 25 ms > frame (an audible 40 Hz artifact in the audio feeding captions). Fixed, with a test > that fails on the old code. > - Stale direction data: an arm that stopped sending kept steering the direction > estimate and could hold the caption feed on a silent arm. > - Compass: closing Microphones switched off the Compass app's compass, because both > used the same owner name. > - Speaker database failures: > - A failed speaker reload wiped the in-memory list, so the next voice became a > duplicate "Speaker N". > - A failed speaker insert crashed caption handling; in re-diarize it reassigned > transcript lines to a nonexistent speaker. > - Translation: > - A crash path inside the model-download callback is now caught. > - Phones set to Hebrew, Indonesian or Yiddish never got translations, because of > old-style language codes. > - Ask questions: once the speaker list grew past the size limit (every unknown voice > adds a "Speaker N"), it pushed all transcript lines out of the context sent to the > model. When trimming is needed, the oldest lines now go first. > - Smaller fixes: > - The captions view stopped jumping back to live speech after 60 lines. > - "This is Dana calling…" was treated as introducing someone else. > - A model download that failed to start was stuck at "downloading" until restart. > - The developer mic monitor grew without limit while hidden.
This commit is contained in:
1 parent
22c31f4240
commit
b50898ea02
14 files changed
+409
-143
No files matched your search
@@ -65,6 +65,14 @@ type ArmStats = {
|
||||
statusAtMs: number | null;
|
||||
};
|
||||
|
||||
/** Drop arrivals older than the rate window. */
|
||||
function trimRecent(stats: ArmStats, nowMs: number): void {
|
||||
const cutoff = nowMs - RATE_WINDOW_MS;
|
||||
let drop = 0;
|
||||
while (drop < stats.recent.length && stats.recent[drop]!.wallMs < cutoff) drop++;
|
||||
if (drop > 0) stats.recent.splice(0, drop);
|
||||
}
|
||||
|
||||
function emptyStats(): ArmStats {
|
||||
return {
|
||||
packets: 0,
|
||||
@@ -169,6 +177,8 @@ export class MicrophonesMonitorLayer implements Layer {
|
||||
this.advanceBuckets(nowMs);
|
||||
stats.packets++;
|
||||
stats.buckets[BUCKETS - 1]!++;
|
||||
// Trim here too: painting (the other trim) stops while the page is hidden.
|
||||
trimRecent(stats, nowMs);
|
||||
stats.recent.push({ wallMs: nowMs, arrivalMs, bytes: data.length });
|
||||
stats.lastArrivalWallMs = nowMs;
|
||||
|
||||
@@ -268,8 +278,7 @@ export class MicrophonesMonitorLayer implements Layer {
|
||||
const font = getDefaultSmallFont();
|
||||
const line = font.lineHeight;
|
||||
const stats = this.stats[arm];
|
||||
const cutoff = nowMs - RATE_WINDOW_MS;
|
||||
while (stats.recent.length > 0 && stats.recent[0]!.wallMs < cutoff) stats.recent.shift();
|
||||
trimRecent(stats, nowMs);
|
||||
const rate = (stats.recent.length * 1000) / RATE_WINDOW_MS;
|
||||
const byteRate = stats.recent.reduce((sum, arrival) => sum + arrival.bytes, 0) / RATE_WINDOW_MS;
|
||||
let maxGap = 0;
|
||||
|
||||
@@ -15,12 +15,15 @@ export class CaptionsLayer implements Layer {
|
||||
private state: MicSessionState = micSession.getState();
|
||||
private unsubscribe: (() => void) | null = null;
|
||||
private scrollback = 0;
|
||||
private lastLineCount = 0;
|
||||
private lastLine: CaptionLine | null = null;
|
||||
|
||||
start(requestRender: () => void): void {
|
||||
this.unsubscribe = micSession.onState((state) => {
|
||||
if (state.captionLines.length !== this.lastLineCount) {
|
||||
this.lastLineCount = state.captionLines.length;
|
||||
// Watch the newest line itself: once the session's line cap is reached
|
||||
// the count stops changing even though new speech keeps arriving.
|
||||
const last = state.captionLines[state.captionLines.length - 1] ?? null;
|
||||
if (last !== this.lastLine) {
|
||||
this.lastLine = last;
|
||||
this.scrollback = 0;
|
||||
}
|
||||
this.state = state;
|
||||
|
||||
@@ -16,6 +16,10 @@ declare const com: any;
|
||||
|
||||
// Qwen runs with a 4096-token context; leave room for the answer.
|
||||
const MAX_CONTEXT_CHARS = 7000;
|
||||
// The speaker roster grows with every voice ever heard; cap it and the
|
||||
// conversation list so they can't crowd the transcript excerpts out.
|
||||
const MAX_HEADER_CHARS = MAX_CONTEXT_CHARS / 2;
|
||||
const EXCERPTS_HEADING = "# Transcript excerpts\n";
|
||||
const MAX_KEYWORDS = 6;
|
||||
const SEGMENTS_PER_KEYWORD = 30;
|
||||
const RECENT_SEGMENT_LIMIT = 120;
|
||||
@@ -156,31 +160,57 @@ function buildContext(question: string): { context: string; note: string } {
|
||||
}
|
||||
}
|
||||
|
||||
const segments = [...segmentMap.values()].sort((a, b) => a.startedAt - b.startedAt);
|
||||
let header =
|
||||
`# Known speakers\n${speakerLines.join("\n") || "(none yet)"}\n\n` +
|
||||
`# Conversations${filters.sinceMs !== null || filters.emotion !== null ? " (filtered by the question)" : ""}\n` +
|
||||
`${sessionLines.join("\n") || "(none in range)"}\n\n`;
|
||||
let truncated = false;
|
||||
if (header.length > MAX_HEADER_CHARS) {
|
||||
header = `${header.slice(0, MAX_HEADER_CHARS)}\n(...)\n\n`;
|
||||
truncated = true;
|
||||
}
|
||||
|
||||
const sessionHeading = (sessionId: number) =>
|
||||
`\n## ${sessionTitles.get(sessionId) ?? `Conversation #${sessionId}`}`;
|
||||
const segmentText = (segment: SegmentRow) => {
|
||||
const speaker =
|
||||
segment.speakerId !== null ? (speakerNames.get(segment.speakerId) ?? "Speaker") : "Speaker";
|
||||
const emotion = segment.emotion && segment.emotion !== "neutral" ? ` [${segment.emotion}]` : "";
|
||||
const line = `[${formatWhen(segment.startedAt)}] ${speaker}${emotion}: ${segment.text}`;
|
||||
return segment.translation ? `${line}\n (translated: ${segment.translation})` : line;
|
||||
};
|
||||
|
||||
// Fill the remaining budget newest-first, so an overflowing history drops
|
||||
// its oldest excerpts rather than its most recent ones.
|
||||
const sorted = [...segmentMap.values()].sort((a, b) => a.startedAt - b.startedAt);
|
||||
let budget = MAX_CONTEXT_CHARS - header.length - EXCERPTS_HEADING.length;
|
||||
const segments: SegmentRow[] = [];
|
||||
let runSession = -1;
|
||||
for (let index = sorted.length - 1; index >= 0; index--) {
|
||||
const segment = sorted[index]!;
|
||||
let cost = segmentText(segment).length + 1;
|
||||
if (segment.sessionId !== runSession) cost += sessionHeading(segment.sessionId).length + 1;
|
||||
if (cost > budget) {
|
||||
truncated = true;
|
||||
break;
|
||||
}
|
||||
budget -= cost;
|
||||
runSession = segment.sessionId;
|
||||
segments.push(segment);
|
||||
}
|
||||
segments.reverse();
|
||||
|
||||
const excerptLines: string[] = [];
|
||||
let currentSession = -1;
|
||||
for (const segment of segments) {
|
||||
if (segment.sessionId !== currentSession) {
|
||||
currentSession = segment.sessionId;
|
||||
excerptLines.push(`\n## ${sessionTitles.get(currentSession) ?? `Conversation #${currentSession}`}`);
|
||||
excerptLines.push(sessionHeading(currentSession));
|
||||
}
|
||||
const speaker =
|
||||
segment.speakerId !== null ? (speakerNames.get(segment.speakerId) ?? "Speaker") : "Speaker";
|
||||
const emotion = segment.emotion && segment.emotion !== "neutral" ? ` [${segment.emotion}]` : "";
|
||||
excerptLines.push(`[${formatWhen(segment.startedAt)}] ${speaker}${emotion}: ${segment.text}`);
|
||||
if (segment.translation) excerptLines.push(` (translated: ${segment.translation})`);
|
||||
excerptLines.push(segmentText(segment));
|
||||
}
|
||||
|
||||
let context =
|
||||
`# Known speakers\n${speakerLines.join("\n") || "(none yet)"}\n\n` +
|
||||
`# Conversations${filters.sinceMs !== null || filters.emotion !== null ? " (filtered by the question)" : ""}\n` +
|
||||
`${sessionLines.join("\n") || "(none in range)"}\n\n` +
|
||||
`# Transcript excerpts\n${excerptLines.join("\n") || "(no matching lines)"}`;
|
||||
let truncated = false;
|
||||
if (context.length > MAX_CONTEXT_CHARS) {
|
||||
context = context.slice(0, MAX_CONTEXT_CHARS);
|
||||
truncated = true;
|
||||
}
|
||||
const context = `${header}${EXCERPTS_HEADING}${excerptLines.join("\n") || "(no matching lines)"}`;
|
||||
const note =
|
||||
`${sessions.length} conversation${sessions.length === 1 ? "" : "s"}, ` +
|
||||
`${segments.length} transcript line${segments.length === 1 ? "" : "s"} consulted` +
|
||||
|
||||
@@ -286,6 +286,12 @@ export function angleInArc(azimuth: number, arcCenter: number, arcHalfWidth: num
|
||||
export class TempleAudioPipeline {
|
||||
private readonly beamformer: DelaySumBeamformer;
|
||||
private readonly anc = new InterMicNoiseCanceller();
|
||||
// Steering delays reach back before the start of each frame. Carrying the
|
||||
// previous frame's last samples keeps those real; zero-filling them (as a
|
||||
// standalone beam() does) dips the output at every frame boundary.
|
||||
private readonly carry: number;
|
||||
private frontTail: Float32Array;
|
||||
private rearTail: Float32Array;
|
||||
enableANC = true;
|
||||
|
||||
constructor(
|
||||
@@ -294,19 +300,38 @@ export class TempleAudioPipeline {
|
||||
geometry: MicGeometry = G2_DEFAULT_GEOMETRY,
|
||||
) {
|
||||
this.beamformer = new DelaySumBeamformer(sampleRate, geometry);
|
||||
const { front, rear } = geometry.positions[side];
|
||||
const spacing = Math.hypot(front.x - rear.x, front.y - rear.y, front.z - rear.z);
|
||||
this.carry = Math.ceil((spacing / geometry.speedOfSound) * sampleRate) + 1;
|
||||
this.frontTail = new Float32Array(this.carry);
|
||||
this.rearTail = new Float32Array(this.carry);
|
||||
}
|
||||
|
||||
/**
|
||||
* Process one frame's deinterleaved front/rear channels toward a
|
||||
* device-frame target, returning the enhanced mono beam for this temple.
|
||||
* Frames must be consecutive: each one continues the previous.
|
||||
*/
|
||||
process(front: Float32Array, rear: Float32Array, target: Bearing): Float32Array {
|
||||
const beam = this.beamformer.beam(this.side, front, rear, target);
|
||||
const n = Math.min(front.length, rear.length);
|
||||
const extendedFront = this.withTail(this.frontTail, front, n);
|
||||
const extendedRear = this.withTail(this.rearTail, rear, n);
|
||||
this.frontTail = extendedFront.slice(n);
|
||||
this.rearTail = extendedRear.slice(n);
|
||||
const beam = this.beamformer
|
||||
.beam(this.side, extendedFront, extendedRear, target)
|
||||
.slice(this.carry);
|
||||
if (!this.enableANC) return beam;
|
||||
// Null-steer reference: the difference channel carries off-axis noise.
|
||||
const n = Math.min(front.length, rear.length);
|
||||
const reference = new Float32Array(n);
|
||||
for (let i = 0; i < n; i++) reference[i] = front[i]! - rear[i]!;
|
||||
return this.anc.process(beam, reference);
|
||||
}
|
||||
|
||||
private withTail(tail: Float32Array, frame: Float32Array, n: number): Float32Array {
|
||||
const out = new Float32Array(tail.length + n);
|
||||
out.set(tail);
|
||||
out.set(frame.subarray(0, n), tail.length);
|
||||
return out;
|
||||
}
|
||||
}
|
||||
@@ -110,6 +110,7 @@ class MicArrayController {
|
||||
* doesn't spend BLE bandwidth.
|
||||
*/
|
||||
apply(config: MicConfig): boolean {
|
||||
if (config.armHardware && !micControlSupported()) return false;
|
||||
const leftActive = templeActive(config, "left");
|
||||
const rightActive = templeActive(config, "right");
|
||||
if (leftActive || rightActive) {
|
||||
@@ -148,8 +149,8 @@ class MicArrayController {
|
||||
): boolean {
|
||||
const communicator = activeCommunicator();
|
||||
if (!communicator) return false;
|
||||
this.ensureStatusProxy(communicator);
|
||||
try {
|
||||
this.ensureStatusProxy(communicator);
|
||||
const record = encodeMicControl(op, config);
|
||||
communicator.sendFaceclawMicControl(toJavaBytes(record), label, rightTemple, leftTemple);
|
||||
return true;
|
||||
|
||||
@@ -142,6 +142,7 @@ function notifyStateChanged(id: MicModelId): void {
|
||||
|
||||
export function startMicModelDownload(id: MicModelId): void {
|
||||
if (!global.isAndroid || downloaders.has(id) || isMicModelReady(id)) return;
|
||||
progress.delete(id);
|
||||
downloadNextFile(id);
|
||||
notifyStateChanged(id);
|
||||
}
|
||||
@@ -171,15 +172,22 @@ function downloadNextFile(id: MicModelId): void {
|
||||
notifyStateChanged(id);
|
||||
},
|
||||
});
|
||||
const downloader = new com.faceclaw.app.FaceclawModelDownloader(
|
||||
`${definition.baseUrl}${nextFile.remoteName}`,
|
||||
filePath(id, nextFile),
|
||||
nextFile.sha256,
|
||||
nextFile.sizeBytes,
|
||||
listener,
|
||||
);
|
||||
downloaders.set(id, downloader);
|
||||
downloader.start();
|
||||
try {
|
||||
const downloader = new com.faceclaw.app.FaceclawModelDownloader(
|
||||
`${definition.baseUrl}${nextFile.remoteName}`,
|
||||
filePath(id, nextFile),
|
||||
nextFile.sha256,
|
||||
nextFile.sizeBytes,
|
||||
listener,
|
||||
);
|
||||
downloaders.set(id, downloader);
|
||||
downloader.start();
|
||||
} catch (error) {
|
||||
// Without this, a dead entry would pin the model at "downloading".
|
||||
console.error(`Mic model download failed to start (${id}/${nextFile.name}): ${String(error)}`);
|
||||
downloaders.delete(id);
|
||||
notifyStateChanged(id);
|
||||
}
|
||||
}
|
||||
|
||||
export function cancelMicModelDownload(id: MicModelId): void {
|
||||
|
||||
@@ -95,6 +95,13 @@ const RECORDING_AAC_BITRATE = 32_000;
|
||||
const ENCOUNTER_GAP_MS = 15 * 60 * 1000;
|
||||
const ENCOUNTER_CARD_TTL_MS = 12_000;
|
||||
const ENCOUNTER_CARD_MAX_ACTION_ITEMS = 2;
|
||||
// A temple that has sent no frame (or no bearing) for this long no longer
|
||||
// counts: its stream stopped, its mics were toggled off, or its arm dropped.
|
||||
const TEMPLE_STALE_MS = 1000;
|
||||
// How often the extended path checks that its forwarding survived the
|
||||
// glasses session (an arm drop clears it).
|
||||
const CAPTURE_CHECK_MS = 3000;
|
||||
const COMPASS_OWNER = "microphones";
|
||||
|
||||
export type CaptionLine = {
|
||||
speakerId: number | null;
|
||||
@@ -181,6 +188,8 @@ class MicSession {
|
||||
private temples = new Map<MicSide, TempleAudioPipeline>();
|
||||
private templeAngles: { left: number | null; right: number | null } = { left: null, right: null };
|
||||
private templeSsr: { left: number; right: number } = { left: 0, right: 0 };
|
||||
private templeAngleAtMs: Record<MicSide, number> = { left: 0, right: 0 };
|
||||
private templeFrameAtMs: Record<MicSide, number> = { left: 0, right: 0 };
|
||||
private captionFeedSide: MicSide = "left";
|
||||
// Cached at capture start so the ~80 Hz packet path never re-reads settings.
|
||||
private activeConfig: MicConfig = loadMicConfig();
|
||||
@@ -202,6 +211,7 @@ class MicSession {
|
||||
private speechActive = false;
|
||||
|
||||
private renderTimer: ReturnType<typeof setInterval> | null = null;
|
||||
private captureCheckTimer: ReturnType<typeof setInterval> | null = null;
|
||||
|
||||
onState(listener: StateListener): () => void {
|
||||
this.stateListeners.add(listener);
|
||||
@@ -262,33 +272,33 @@ class MicSession {
|
||||
|
||||
this.activeConfig = loadMicConfig();
|
||||
this.refreshCachedFlags();
|
||||
const extended = micControlSupported() && this.activeConfig.armHardware;
|
||||
this.mode = extended ? "extended" : "stock";
|
||||
this.running = true;
|
||||
// Capture first, so a failure has no captions session, recording, or
|
||||
// subscriptions to unwind.
|
||||
if (!this.startCapture()) {
|
||||
this.statusText = "Microphone unavailable (glasses not connected?)";
|
||||
this.running = false;
|
||||
this.mode = "off";
|
||||
this.notify();
|
||||
return;
|
||||
}
|
||||
this.statusText = this.listeningText();
|
||||
this.offSettings = onSettingsStoreChanged(() => this.refreshCachedFlags());
|
||||
this.offSpeakerChanges = speakerRegistry.onChanged(() => this.refreshSpeakerIdentities());
|
||||
this.offVoiceActivity = voiceActivity.subscribe((active) => this.handleVoiceActivity(active));
|
||||
const extended = micControlSupported() && this.activeConfig.armHardware;
|
||||
this.mode = extended ? "extended" : "stock";
|
||||
this.startSuppressor();
|
||||
this.startCompass();
|
||||
if (captionsEnabledSetting.get()) {
|
||||
this.startCaptions();
|
||||
}
|
||||
if (saveRecordingsSetting.get() && recordingsRetentionSetting.get() !== "none") {
|
||||
if (this.shouldRecord()) {
|
||||
this.startRecording();
|
||||
}
|
||||
const ok = extended ? this.startExtended() : this.startStock();
|
||||
if (!ok) {
|
||||
this.statusText = "Microphone unavailable (glasses not connected?)";
|
||||
this.running = false;
|
||||
this.stopCaptions();
|
||||
this.stopRecording(false);
|
||||
this.stopCompass();
|
||||
this.mode = "off";
|
||||
this.notify();
|
||||
return;
|
||||
}
|
||||
this.statusText = extended ? "Listening (4-mic array)" : "Listening";
|
||||
// Levels/DOA update at packet rate; coalesce UI renders to 5 Hz.
|
||||
this.renderTimer = setInterval(() => this.notify(), 200);
|
||||
this.captureCheckTimer = setInterval(() => this.checkExtendedCapture(), CAPTURE_CHECK_MS);
|
||||
this.notify();
|
||||
}
|
||||
|
||||
@@ -299,11 +309,11 @@ class MicSession {
|
||||
clearInterval(this.renderTimer);
|
||||
this.renderTimer = null;
|
||||
}
|
||||
if (this.mode === "extended") {
|
||||
this.stopExtended();
|
||||
} else {
|
||||
this.stopStock();
|
||||
if (this.captureCheckTimer) {
|
||||
clearInterval(this.captureCheckTimer);
|
||||
this.captureCheckTimer = null;
|
||||
}
|
||||
this.stopCapture();
|
||||
this.offSettings?.();
|
||||
this.offSettings = null;
|
||||
this.offSpeakerChanges?.();
|
||||
@@ -324,6 +334,55 @@ class MicSession {
|
||||
void this.applyRetentionSweep();
|
||||
}
|
||||
|
||||
private listeningText(): string {
|
||||
return this.mode === "extended" ? "Listening (4-mic array)" : "Listening";
|
||||
}
|
||||
|
||||
/** Start the current mode's capture path, undoing any partial start on failure. */
|
||||
private startCapture(): boolean {
|
||||
try {
|
||||
if (this.mode === "extended" ? this.startExtended() : this.startStock()) return true;
|
||||
} catch (error) {
|
||||
console.warn(`mic capture start failed: ${error}`);
|
||||
}
|
||||
this.stopCapture();
|
||||
return false;
|
||||
}
|
||||
|
||||
private stopCapture(): void {
|
||||
if (this.mode === "extended") {
|
||||
this.stopExtended();
|
||||
} else {
|
||||
this.stopStock();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Persist a Microphone setup change. While running it takes effect live:
|
||||
* toggling 4-mic capture swaps the capture path, and any other change is
|
||||
* re-sent to the streaming temples.
|
||||
*/
|
||||
updateConfig(mutate: (config: MicConfig) => MicConfig): void {
|
||||
const config = mutate(loadMicConfig());
|
||||
saveMicConfig(config);
|
||||
this.activeConfig = config;
|
||||
if (this.running) {
|
||||
const extended = micControlSupported() && config.armHardware;
|
||||
if (extended !== (this.mode === "extended")) {
|
||||
this.stopCapture();
|
||||
this.mode = extended ? "extended" : "stock";
|
||||
this.doaDeviceDeg = null;
|
||||
this.ssr = 0;
|
||||
this.statusText = this.startCapture()
|
||||
? this.listeningText()
|
||||
: "Microphone unavailable (glasses not connected?)";
|
||||
} else if (extended && !voiceActivity.isActive()) {
|
||||
micArrayController.apply(wireConfig(config));
|
||||
}
|
||||
}
|
||||
this.notify();
|
||||
}
|
||||
|
||||
private refreshCachedFlags(): void {
|
||||
this.beamFilterOn = beamFilterSetting.get();
|
||||
const wasOn = this.ancOn;
|
||||
@@ -380,13 +439,34 @@ class MicSession {
|
||||
if (!communicator) return;
|
||||
if (this.mode === "extended") {
|
||||
if (this.forwardingProxy && communicator.startG2AudioForwarding(this.forwardingProxy)) {
|
||||
micArrayController.apply({ ...this.activeConfig, channelMask: 0b11 });
|
||||
micArrayController.apply(wireConfig(this.activeConfig));
|
||||
}
|
||||
} else {
|
||||
voiceControlBridge.startRawCapture({ communicator });
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* The extended stream's forwarding lives in the glasses session: an arm
|
||||
* drop clears it and nothing else re-arms it on the next session, so the
|
||||
* app would sit at "Listening" hearing nothing. Re-establish forwarding
|
||||
* and re-send the config whenever a ready session has lost it. (The stock
|
||||
* path is resumed by the voice-control bridge.)
|
||||
*/
|
||||
private checkExtendedCapture(): void {
|
||||
if (!this.running || this.mode !== "extended" || !this.forwardingProxy) return;
|
||||
if (voiceActivity.isActive()) return;
|
||||
try {
|
||||
const communicator = this.nativeCommunicator();
|
||||
if (!communicator?.isSessionReady() || communicator.isAudioCaptureActive()) return;
|
||||
if (communicator.startG2AudioForwarding(this.forwardingProxy)) {
|
||||
micArrayController.apply(wireConfig(this.activeConfig));
|
||||
}
|
||||
} catch (error) {
|
||||
console.warn(`mic capture re-arm failed: ${error}`);
|
||||
}
|
||||
}
|
||||
|
||||
private startSuppressor(): void {
|
||||
try {
|
||||
this.suppressor = new com.faceclaw.app.FaceclawNoiseSuppressor(SAMPLE_RATE);
|
||||
@@ -459,7 +539,7 @@ class MicSession {
|
||||
|
||||
private startCompass(): void {
|
||||
try {
|
||||
setCompassEnabled(true);
|
||||
setCompassEnabled(true, COMPASS_OWNER);
|
||||
this.offCompass = addCompassListener((event) => {
|
||||
if (event.headingDegrees >= 0) {
|
||||
this.headingDeg = event.headingDegrees;
|
||||
@@ -474,7 +554,7 @@ class MicSession {
|
||||
this.offCompass?.();
|
||||
this.offCompass = null;
|
||||
try {
|
||||
setCompassEnabled(false);
|
||||
setCompassEnabled(false, COMPASS_OWNER);
|
||||
} catch {
|
||||
// Disconnected; the stream is already gone.
|
||||
}
|
||||
@@ -542,11 +622,7 @@ class MicSession {
|
||||
},
|
||||
});
|
||||
if (!communicator.startG2AudioForwarding(this.forwardingProxy)) return false;
|
||||
if (!micArrayController.apply({ ...this.activeConfig, channelMask: 0b11 })) {
|
||||
communicator.stopG2AudioForwarding();
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
return micArrayController.apply(wireConfig(this.activeConfig));
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -555,14 +631,7 @@ class MicSession {
|
||||
* captions; a temple with both mics off has its stream stopped entirely.
|
||||
*/
|
||||
setMicChannelEnabled(key: MicChannelKey, enabled: boolean): void {
|
||||
const config = loadMicConfig();
|
||||
config.hostMics = { ...config.hostMics, [key]: enabled };
|
||||
saveMicConfig(config);
|
||||
this.activeConfig = config;
|
||||
if (this.running && this.mode === "extended") {
|
||||
micArrayController.apply({ ...config, channelMask: 0b11 });
|
||||
}
|
||||
this.notify();
|
||||
this.updateConfig((config) => ({ ...config, hostMics: { ...config.hostMics, [key]: enabled } }));
|
||||
}
|
||||
|
||||
private stopExtended(): void {
|
||||
@@ -576,6 +645,14 @@ class MicSession {
|
||||
this.forwardingProxy = null;
|
||||
this.temples.clear();
|
||||
this.templeAngles = { left: null, right: null };
|
||||
this.templeSsr = { left: 0, right: 0 };
|
||||
this.templeAngleAtMs = { left: 0, right: 0 };
|
||||
this.templeFrameAtMs = { left: 0, right: 0 };
|
||||
}
|
||||
|
||||
/** A temple's last bearing, or null once it has gone quiet. */
|
||||
private freshTempleAngle(side: MicSide, nowMs: number): number | null {
|
||||
return nowMs - this.templeAngleAtMs[side] < TEMPLE_STALE_MS ? this.templeAngles[side] : null;
|
||||
}
|
||||
|
||||
private handleExtendedPacket(data: Uint8Array, arm: string): void {
|
||||
@@ -588,6 +665,8 @@ class MicSession {
|
||||
const side: MicSide = arm === "L" ? "left" : "right";
|
||||
const channels = splitInterleavedPcm16(frame);
|
||||
if (!channels || channels.length === 0) return;
|
||||
const now = Date.now();
|
||||
this.templeFrameAtMs[side] = now;
|
||||
|
||||
// Meter every arriving channel — a host-disabled mic still shows its
|
||||
// live level (marked off in the UI), it just doesn't feed anything.
|
||||
@@ -606,8 +685,10 @@ class MicSession {
|
||||
if (frame.angleDegrees !== 0 || frame.ssr !== 0) {
|
||||
this.templeAngles[side] = frame.angleDegrees;
|
||||
this.templeSsr[side] = frame.ssr;
|
||||
const left = this.templeAngles.left;
|
||||
const right = this.templeAngles.right;
|
||||
this.templeAngleAtMs[side] = now;
|
||||
// A temple that stopped reporting must not keep steering the fusion.
|
||||
const left = this.freshTempleAngle("left", now);
|
||||
const right = this.freshTempleAngle("right", now);
|
||||
let fusedDeg: number;
|
||||
if (left !== null && right !== null) {
|
||||
const fused = fuseBearings(
|
||||
@@ -621,7 +702,7 @@ class MicSession {
|
||||
fusedDeg = (left ?? right)!;
|
||||
}
|
||||
this.doaDeviceDeg = smoothAngleDeg(this.doaDeviceDeg, fusedDeg, DOA_SMOOTHING);
|
||||
this.ssr = Math.max(this.templeSsr.left, this.templeSsr.right);
|
||||
this.ssr = Math.max(left === null ? 0 : this.templeSsr.left, right === null ? 0 : this.templeSsr.right);
|
||||
if (this.beamLocked && this.doaDeviceDeg !== null) {
|
||||
this.beamWorldDeg = this.toWorldDeg(this.doaDeviceDeg);
|
||||
}
|
||||
@@ -651,10 +732,16 @@ class MicSession {
|
||||
|
||||
private deliverExtendedAudio(side: MicSide, samples: Float32Array): void {
|
||||
// Feed captions from one temple only; prefer the one hearing more
|
||||
// signal, among the temples that still have an enabled microphone.
|
||||
const leftUsable = templeActive(this.activeConfig, "left");
|
||||
const rightUsable = templeActive(this.activeConfig, "right");
|
||||
let stronger: MicSide = this.templeSsr.right > this.templeSsr.left * 1.5 ? "right" : "left";
|
||||
// signal, among the temples that still have an enabled microphone and
|
||||
// are still streaming.
|
||||
const now = Date.now();
|
||||
const leftUsable =
|
||||
templeActive(this.activeConfig, "left") && now - this.templeFrameAtMs.left < TEMPLE_STALE_MS;
|
||||
const rightUsable =
|
||||
templeActive(this.activeConfig, "right") && now - this.templeFrameAtMs.right < TEMPLE_STALE_MS;
|
||||
const leftSsr = this.freshTempleAngle("left", now) === null ? 0 : this.templeSsr.left;
|
||||
const rightSsr = this.freshTempleAngle("right", now) === null ? 0 : this.templeSsr.right;
|
||||
let stronger: MicSide = rightSsr > leftSsr * 1.5 ? "right" : "left";
|
||||
if (stronger === "right" && !rightUsable) stronger = "left";
|
||||
if (stronger === "left" && !leftUsable) stronger = "right";
|
||||
if (strongerChanged(this.captionFeedSide, stronger)) {
|
||||
@@ -707,9 +794,12 @@ class MicSession {
|
||||
if (!this.running) return;
|
||||
if (enabled && !this.captionsActive) {
|
||||
this.startCaptions();
|
||||
if (saveRecordingsSetting.get() && !this.recorder) this.startRecording();
|
||||
if (!this.recorder && this.shouldRecord()) this.startRecording();
|
||||
} else if (!enabled && this.captionsActive) {
|
||||
this.stopCaptions();
|
||||
// The recording belongs to this caption session (its segment offsets
|
||||
// are relative to this engine's clock); close it out with the row.
|
||||
this.stopRecording(true);
|
||||
this.finishSessionRow();
|
||||
}
|
||||
this.notify();
|
||||
@@ -717,33 +807,56 @@ class MicSession {
|
||||
|
||||
private startCaptions(): void {
|
||||
if (this.captionsActive) return;
|
||||
const engine = new com.faceclaw.app.FaceclawCaptionEngine();
|
||||
this.captionListenerProxy = new com.faceclaw.app.FaceclawCaptionEngineListener({
|
||||
onUtterance: (text: string, embedding: any, startMs: number, endMs: number, peakRms: number) => {
|
||||
this.handleUtterance(String(text), embedding, Number(startMs), Number(endMs), Number(peakRms));
|
||||
},
|
||||
onSpeechStart: (_startMs: number) => {
|
||||
this.speechActive = true;
|
||||
},
|
||||
onStatus: (status: string) => {
|
||||
this.statusText = String(status);
|
||||
},
|
||||
});
|
||||
engine.setListener(this.captionListenerProxy);
|
||||
// Live captions are a separate sherpa-onnx pipeline (FaceclawCaptionEngine)
|
||||
// that only ever uses Moonshine; unaffected by the two-model on-device
|
||||
// Whisper addition in native/voice-control.ts's push-to-talk path.
|
||||
engine.setAsrModelDir(isAsrModelReady("moonshine") ? moonshineModelDir() : null);
|
||||
engine.setSpeakerModelPath(micModelPath("speaker-embedding"));
|
||||
engine.start();
|
||||
let engine: any = null;
|
||||
try {
|
||||
engine = new com.faceclaw.app.FaceclawCaptionEngine();
|
||||
this.captionListenerProxy = new com.faceclaw.app.FaceclawCaptionEngineListener({
|
||||
onUtterance: (text: string, embedding: any, startMs: number, endMs: number, peakRms: number) => {
|
||||
// Never let an exception escape into the engine's Java callback.
|
||||
try {
|
||||
this.handleUtterance(String(text), embedding, Number(startMs), Number(endMs), Number(peakRms));
|
||||
} catch (error) {
|
||||
console.warn(`caption utterance failed: ${error}`);
|
||||
}
|
||||
},
|
||||
onSpeechStart: (_startMs: number) => {
|
||||
this.speechActive = true;
|
||||
},
|
||||
onStatus: (status: string) => {
|
||||
this.statusText = String(status);
|
||||
},
|
||||
});
|
||||
engine.setListener(this.captionListenerProxy);
|
||||
// Live captions are a separate sherpa-onnx pipeline (FaceclawCaptionEngine)
|
||||
// that only ever uses Moonshine; unaffected by the two-model on-device
|
||||
// Whisper addition in native/voice-control.ts's push-to-talk path.
|
||||
engine.setAsrModelDir(isAsrModelReady("moonshine") ? moonshineModelDir() : null);
|
||||
engine.setSpeakerModelPath(micModelPath("speaker-embedding"));
|
||||
engine.start();
|
||||
} catch (error) {
|
||||
console.warn(`caption engine start failed: ${error}`);
|
||||
try {
|
||||
engine?.stop();
|
||||
} catch {
|
||||
// It never started.
|
||||
}
|
||||
this.captionListenerProxy = null;
|
||||
this.statusText = "Captions unavailable";
|
||||
return;
|
||||
}
|
||||
this.captionEngine = engine;
|
||||
this.captionsActive = true;
|
||||
this.engineStartWallMs = Date.now();
|
||||
this.encounterShown.clear();
|
||||
if (saveCaptionsSetting.get() && captionsRetentionSetting.get() !== "none") {
|
||||
this.sessionRowId = Number(
|
||||
this.store().startSession(Date.now(), `Conversation ${new Date().toLocaleString()}`),
|
||||
);
|
||||
try {
|
||||
this.sessionRowId = Number(
|
||||
this.store().startSession(Date.now(), `Conversation ${new Date().toLocaleString()}`),
|
||||
);
|
||||
} catch (error) {
|
||||
console.warn(`caption session start failed: ${error}`);
|
||||
this.sessionRowId = -1;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -892,10 +1005,23 @@ class MicSession {
|
||||
const embeddingFloats = embedding ? floatArrayToFloat32(embedding) : null;
|
||||
if (this.wearerEnrollmentArmed && embeddingFloats) {
|
||||
this.wearerEnrollmentArmed = false;
|
||||
const wearer = speakerRegistry.enrollWearer(embeddingFloats);
|
||||
this.statusText = `Enrolled your voice as "${wearer.name}"`;
|
||||
try {
|
||||
const wearer = speakerRegistry.enrollWearer(embeddingFloats);
|
||||
this.statusText = `Enrolled your voice as "${wearer.name}"`;
|
||||
} catch (error) {
|
||||
console.warn(`wearer enrollment failed: ${error}`);
|
||||
this.statusText = "Voice enrollment failed";
|
||||
}
|
||||
}
|
||||
let match: SpeakerMatch | null = null;
|
||||
if (embeddingFloats) {
|
||||
try {
|
||||
match = speakerRegistry.assign(embeddingFloats);
|
||||
} catch (error) {
|
||||
// Still caption the line, just unattributed.
|
||||
console.warn(`speaker assignment failed: ${error}`);
|
||||
}
|
||||
}
|
||||
let match = embeddingFloats ? speakerRegistry.assign(embeddingFloats) : null;
|
||||
if (match) this.maybeShowEncounterCard(match);
|
||||
// Spoken introductions name the auto-numbered speakers: "I'm Alice"
|
||||
// renames the speaker of the line; "this is Bob" pends the name for the
|
||||
@@ -957,23 +1083,27 @@ class MicSession {
|
||||
|
||||
let segmentId = -1;
|
||||
if (this.sessionRowId >= 0 && line.text) {
|
||||
segmentId = Number(
|
||||
this.store().insertSegment(
|
||||
JSON.stringify({
|
||||
sessionId: this.sessionRowId,
|
||||
speakerId: match?.profile.id,
|
||||
startedAt: this.engineStartWallMs + startMs,
|
||||
endedAt: this.engineStartWallMs + endMs,
|
||||
audioOffsetMs: startMs,
|
||||
text: line.text,
|
||||
sentiment: line.sentiment,
|
||||
emotion: line.emotion,
|
||||
searchMeta: analyzed ? searchableMetadata(analyzed.emotion, analyzed.score) : "",
|
||||
angle: doaAtUtterance === null ? undefined : Math.round(doaAtUtterance),
|
||||
embeddingBase64: embeddingFloats ? encodeEmbeddingBase64(embeddingFloats) : undefined,
|
||||
}),
|
||||
),
|
||||
);
|
||||
try {
|
||||
segmentId = Number(
|
||||
this.store().insertSegment(
|
||||
JSON.stringify({
|
||||
sessionId: this.sessionRowId,
|
||||
speakerId: match?.profile.id,
|
||||
startedAt: this.engineStartWallMs + startMs,
|
||||
endedAt: this.engineStartWallMs + endMs,
|
||||
audioOffsetMs: startMs,
|
||||
text: line.text,
|
||||
sentiment: line.sentiment,
|
||||
emotion: line.emotion,
|
||||
searchMeta: analyzed ? searchableMetadata(analyzed.emotion, analyzed.score) : "",
|
||||
angle: doaAtUtterance === null ? undefined : Math.round(doaAtUtterance),
|
||||
embeddingBase64: embeddingFloats ? encodeEmbeddingBase64(embeddingFloats) : undefined,
|
||||
}),
|
||||
),
|
||||
);
|
||||
} catch (error) {
|
||||
console.warn(`caption segment save failed: ${error}`);
|
||||
}
|
||||
}
|
||||
this.notify();
|
||||
|
||||
@@ -1024,6 +1154,19 @@ class MicSession {
|
||||
|
||||
// ---- recording ----
|
||||
|
||||
/**
|
||||
* Recordings belong to a saved caption session: retention and the phone UI
|
||||
* find audio only through its session row, so a file recorded without one
|
||||
* would never be shown or expired.
|
||||
*/
|
||||
private shouldRecord(): boolean {
|
||||
return (
|
||||
this.sessionRowId >= 0 &&
|
||||
saveRecordingsSetting.get() &&
|
||||
recordingsRetentionSetting.get() !== "none"
|
||||
);
|
||||
}
|
||||
|
||||
private startRecording(): void {
|
||||
try {
|
||||
const context = Utils.android.getApplicationContext();
|
||||
@@ -1044,18 +1187,17 @@ class MicSession {
|
||||
if (!recorder) return;
|
||||
try {
|
||||
const bytes = Number(recorder.finish());
|
||||
if (!keep || bytes < SAMPLE_RATE) {
|
||||
// Under half a second of audio isn't worth keeping.
|
||||
const sessionRowId = this.sessionRowId;
|
||||
if (!keep || bytes < SAMPLE_RATE || sessionRowId < 0) {
|
||||
// Under half a second of audio isn't worth keeping, and audio with no
|
||||
// session row to own it would be unreachable and never expire.
|
||||
recorder.discard();
|
||||
return;
|
||||
}
|
||||
const wavPath = this.recordingPath;
|
||||
const m4aPath = wavPath.replace(/\.wav$/, ".m4a");
|
||||
const sessionRowId = this.sessionRowId;
|
||||
const store = this.store();
|
||||
if (sessionRowId >= 0) {
|
||||
store.setSessionAudio(sessionRowId, wavPath, "wav");
|
||||
}
|
||||
store.setSessionAudio(sessionRowId, wavPath, "wav");
|
||||
// Transcode in the background; swap the session's audio to the M4A on
|
||||
// success (the transcoder deletes the WAV).
|
||||
const transcoder = new com.faceclaw.app.FaceclawAudioTranscoder(
|
||||
@@ -1065,8 +1207,10 @@ class MicSession {
|
||||
true,
|
||||
new com.faceclaw.app.FaceclawAudioTranscoder.Listener({
|
||||
onDone: (outputPath: string, _outputBytes: number) => {
|
||||
if (sessionRowId >= 0) {
|
||||
try {
|
||||
store.setSessionAudio(sessionRowId, String(outputPath), "aac");
|
||||
} catch (error) {
|
||||
console.warn(`recording audio update failed: ${error}`);
|
||||
}
|
||||
},
|
||||
onError: (message: string) => {
|
||||
@@ -1129,6 +1273,14 @@ class MicSession {
|
||||
|
||||
// ---- helpers ----
|
||||
|
||||
/**
|
||||
* The config as sent to the temples: each active temple always streams its
|
||||
* full front+rear pair, and per-mic selection happens host-side.
|
||||
*/
|
||||
function wireConfig(config: MicConfig): MicConfig {
|
||||
return { ...config, channelMask: 0b11 };
|
||||
}
|
||||
|
||||
function pcmLevel(pcm: Uint8Array): number {
|
||||
let sumSquares = 0;
|
||||
const count = pcm.length / 2;
|
||||
|
||||
@@ -22,7 +22,7 @@ import { isAsrModelReady, startAsrModelDownload, asrModelState } from "../../nat
|
||||
import { CaptionsLayer } from "./captions-layer";
|
||||
import { LevelsLayer } from "./levels-layer";
|
||||
import { RadarLayer } from "./radar-layer";
|
||||
import { loadMicConfig, micArrayController, micControlSupported, saveMicConfig } from "./mic-control";
|
||||
import { loadMicConfig, micArrayController, micControlSupported } from "./mic-control";
|
||||
import { isMicModelReady, micModelState, startMicModelDownload } from "./mic-models";
|
||||
import { micChannelLabel, type MicChannelKey, type MicConfig } from "./mic-protocol";
|
||||
import { micSession } from "./mic-session";
|
||||
@@ -72,13 +72,9 @@ const MENU_LAYOUT = {
|
||||
opaque: true,
|
||||
};
|
||||
|
||||
/** Persist a config change and re-apply it live when the array is streaming. */
|
||||
/** Persist a config change; the running session applies it live. */
|
||||
function updateMicConfig(mutate: (config: MicConfig) => MicConfig): void {
|
||||
const updated = mutate(loadMicConfig());
|
||||
saveMicConfig(updated);
|
||||
if (micSession.getState().mode === "extended") {
|
||||
micArrayController.apply(updated);
|
||||
}
|
||||
micSession.updateConfig(mutate);
|
||||
}
|
||||
|
||||
function micChannelToggleItem(
|
||||
|
||||
@@ -55,7 +55,9 @@ const SELF_PATTERNS = [
|
||||
new RegExp(`\\b[Mm]y name is ${NAME}`),
|
||||
new RegExp(`\\b[Ii](?:'| a)m ${NAME}`),
|
||||
new RegExp(`\\b[Cc]all me ${NAME}`),
|
||||
new RegExp(`\\b[Tt]his is ${NAME} speaking\\b`),
|
||||
// Phone-style: "this is Dana calling about..." is the speaker, not a
|
||||
// third party being introduced.
|
||||
new RegExp(`\\b[Tt]his is ${NAME} (?:[Ss]peaking|[Cc]alling)\\b`),
|
||||
];
|
||||
|
||||
const THIRD_PARTY_PATTERNS = [
|
||||
|
||||
@@ -140,6 +140,8 @@ function applyTurns(sessionId: number, turns: Turn[]): RediarizeResult {
|
||||
} else {
|
||||
const id = Number(store().createSpeaker(nextAutoName(), "#90A4AE", false, encodeEmbedding(centroid)));
|
||||
speakerRegistry.reload();
|
||||
// A failed insert returns -1; leave that cluster's segments as they were.
|
||||
if (id < 0) continue;
|
||||
clusterSpeaker.set(cluster, id);
|
||||
newSpeakers += 1;
|
||||
}
|
||||
|
||||
@@ -139,8 +139,9 @@ export class SpeakerRegistry {
|
||||
}));
|
||||
this.loaded = true;
|
||||
} catch (error) {
|
||||
// Keep the last good cache: emptying it would make the next utterance
|
||||
// enroll an already-known voice as a duplicate "Speaker N".
|
||||
console.error(`speaker reload failed: ${String(error)}`);
|
||||
this.profiles = [];
|
||||
}
|
||||
}
|
||||
|
||||
@@ -194,7 +195,8 @@ export class SpeakerRegistry {
|
||||
const id = Number(store().createSpeaker(name, color, false, encodeEmbedding(embedding)));
|
||||
this.reload();
|
||||
this.notifyChanged();
|
||||
const profile = this.byId(id)!;
|
||||
const profile = this.byId(id);
|
||||
if (!profile) throw new Error(`speaker "${name}" was not created (id ${id})`);
|
||||
return { profile, similarity: best?.similarity ?? 0, classification: "new", previousLastHeardAt: 0 };
|
||||
}
|
||||
|
||||
@@ -213,12 +215,14 @@ export class SpeakerRegistry {
|
||||
store().updateSpeakerEmbedding(existing.id, encodeEmbedding(embedding), CENTROID_UPDATE_CAP);
|
||||
this.reload();
|
||||
this.notifyChanged();
|
||||
return this.byId(existing.id)!;
|
||||
return this.byId(existing.id) ?? existing;
|
||||
}
|
||||
const id = Number(store().createSpeaker("Me", "#64B5F6", true, encodeEmbedding(embedding)));
|
||||
this.reload();
|
||||
this.notifyChanged();
|
||||
return this.byId(id)!;
|
||||
const profile = this.byId(id);
|
||||
if (!profile) throw new Error(`wearer profile was not created (id ${id})`);
|
||||
return profile;
|
||||
}
|
||||
|
||||
/** Is this utterance the wearer's own voice? Used to gate command recognition. */
|
||||
|
||||
@@ -18,7 +18,10 @@ const downloadedPairs = new Set<string>();
|
||||
export function deviceLanguage(): string {
|
||||
if (!global.isAndroid) return "en";
|
||||
try {
|
||||
return String(java.util.Locale.getDefault().getLanguage() || "en");
|
||||
// toLanguageTag() gives the modern code ("he", "id", "yi") that ML Kit
|
||||
// uses; getLanguage() returns the legacy "iw", "in", "ji".
|
||||
const tag = String(java.util.Locale.getDefault().toLanguageTag() || "");
|
||||
return tag.split("-")[0] || "en";
|
||||
} catch {
|
||||
return "en";
|
||||
}
|
||||
@@ -95,11 +98,18 @@ export function translateText(text: string, sourceLang: string, targetLang: stri
|
||||
if (!translator) return Promise.resolve("");
|
||||
const key = `${sourceLang}>${targetLang}`;
|
||||
return new Promise((resolve) => {
|
||||
// Also runs from the download listener, where a throw would escape into
|
||||
// the Play Services callback and leave this promise unsettled.
|
||||
const runTranslate = () => {
|
||||
translator
|
||||
.translate(text)
|
||||
.addOnSuccessListener(successListener((result: any) => resolve(String(result))))
|
||||
.addOnFailureListener(failureListener(() => resolve("")));
|
||||
try {
|
||||
translator
|
||||
.translate(text)
|
||||
.addOnSuccessListener(successListener((result: any) => resolve(String(result))))
|
||||
.addOnFailureListener(failureListener(() => resolve("")));
|
||||
} catch (error) {
|
||||
console.warn(`translation failed (${key}): ${error}`);
|
||||
resolve("");
|
||||
}
|
||||
};
|
||||
if (downloadedPairs.has(key)) {
|
||||
runTranslate();
|
||||
|
||||
@@ -172,3 +172,21 @@ test("temple pipeline matches the raw beam with ANC disabled", () => {
|
||||
assert.equal(viaPipeline.length, direct.length);
|
||||
for (let i = 0; i < direct.length; i++) assert.equal(viaPipeline[i], direct[i]);
|
||||
});
|
||||
|
||||
test("temple pipeline output is seamless across frame boundaries", () => {
|
||||
// Streamed frame by frame, the steering delay must reach into the previous
|
||||
// frame rather than zero-fill, so splitting the signal changes nothing.
|
||||
const front = makeNoise(800, 3);
|
||||
const rear = makeNoise(800, 4);
|
||||
const target = { azimuth: deg(60) };
|
||||
const whole = new TempleAudioPipeline("left", 16000);
|
||||
whole.enableANC = false;
|
||||
const expected = whole.process(front, rear, target);
|
||||
const framed = new TempleAudioPipeline("left", 16000);
|
||||
framed.enableANC = false;
|
||||
const first = framed.process(front.subarray(0, 400), rear.subarray(0, 400), target);
|
||||
const second = framed.process(front.subarray(400), rear.subarray(400), target);
|
||||
assert.equal(first.length + second.length, expected.length);
|
||||
for (let i = 0; i < 400; i++) assert.ok(near(first[i], expected[i], 1e-6));
|
||||
for (let i = 0; i < 400; i++) assert.ok(near(second[i], expected[400 + i], 1e-6));
|
||||
});
|
||||
@@ -22,6 +22,12 @@ test("self-introductions are recognized", () => {
|
||||
name: "Alice",
|
||||
kind: "self",
|
||||
});
|
||||
// Phone-style self-introductions, including ASR's stray capitalization.
|
||||
assert.deepEqual(inferIntroducedName("Hey this is Dana calling about the order"), {
|
||||
name: "Dana",
|
||||
kind: "self",
|
||||
});
|
||||
assert.deepEqual(inferIntroducedName("This is Bob Speaking."), { name: "Bob", kind: "self" });
|
||||
});
|
||||
|
||||
test("third-party introductions are recognized", () => {
|
||||
|
||||
Reference in new issue
Block a user