Files
faceclaw/tests/microphones-dsp.test.cjs
James Babcock b50898ea02 Fix a bunch of microphone-related bugs
> Most serious fixes
> - Recordings kept forever: a recording made with captions off, or with "Save captions"
>   off, was transcoded to .m4a and kept indefinitely. No conversation linked to it, so
>   the phone UI never showed it and the retention settings never deleted it. Recording
>   now only happens alongside a saved caption session, and turning captions off ends
>   that session's recording properly.
> - 4-mic mode went silent after a reconnect: a disconnect clears the phone's audio
>   forwarding, and nothing turned it back on, so the app kept showing "Listening (4-mic
>   array)" while hearing nothing. A check every 3 s now restores forwarding and re-sends
>   the config when it has been lost. The normal single-stream mode already recovered on
>   its own.
> - Turning off "4-mic capture" mid-session went silent: it stopped the glasses' stream
>   but never switched to the normal stream. Setup changes now take effect live,
>   including switching between the two modes.
> - Setup changes were undone: after the assistant closed, the app re-sent the config it
>   loaded at launch to the glasses, reverting any changes you'd made since.
> - Startup failures: if the caption engine failed to start, the session got stuck
>   half-running. If the glasses weren't connected, listeners piled up and an empty
>   conversation could be left open. Startup now brings up the microphone first and
>   cleans up properly on failure.
>
> Other fixes
> - Audio quality: the 4-mic beamformer put a small dropout at the start of every 25 ms
>   frame (an audible 40 Hz artifact in the audio feeding captions). Fixed, with a test
>   that fails on the old code.
> - Stale direction data: an arm that stopped sending kept steering the direction
>   estimate and could hold the caption feed on a silent arm.
> - Compass: closing Microphones switched off the Compass app's compass, because both
>   used the same owner name.
> - Speaker database failures:
>   - A failed speaker reload wiped the in-memory list, so the next voice became a
>     duplicate "Speaker N".
>   - A failed speaker insert crashed caption handling; in re-diarize it reassigned
>     transcript lines to a nonexistent speaker.
> - Translation:
>   - A crash path inside the model-download callback is now caught.
>   - Phones set to Hebrew, Indonesian or Yiddish never got translations, because of
>     old-style language codes.
> - Ask questions: once the speaker list grew past the size limit (every unknown voice
>   adds a "Speaker N"), it pushed all transcript lines out of the context sent to the
>   model. When trimming is needed, the oldest lines now go first.
> - Smaller fixes:
>   - The captions view stopped jumping back to live speech after 60 lines.
>   - "This is Dana calling…" was treated as introducing someone else.
>   - A model download that failed to start was stuck at "downloading" until restart.
>   - The developer mic monitor grew without limit while hidden.
2026-10-14 00:56:54 -07:00

193 lines
7.8 KiB
JavaScript

// These pin the DSP behaviours the microphones app depends on: fractional
// delays interpolate rather than round, the per-temple beamformer actually
// gains from coherent summation, the direct-correlation TDOA recovers known
// delays (including sub-sample ones), the NLMS canceller removes correlated
// noise without eating the signal, and the bearing math survives the ±π seam.
const test = require("node:test");
const assert = require("node:assert/strict");
const {
G2_DEFAULT_GEOMETRY,
bearingDegrees,
bearingUnitVector,
fractionalDelay,
DelaySumBeamformer,
InterMicNoiseCanceller,
gccPhatDelaySamples,
steerWorldToDevice,
deviceToWorld,
fuseBearings,
TempleAudioPipeline,
bearingFromTdoa,
angleInArc,
} = require("../.test-build/app/apps/microphones/dsp.js");
const near = (a, b, eps) => Math.abs(a - b) <= eps;
const deg = (d) => (d * Math.PI) / 180;
const rms = (x, from = 0, to = x.length) => {
let sum = 0;
for (let i = from; i < to; i++) sum += x[i] * x[i];
return Math.sqrt(sum / (to - from));
};
// Deterministic white-ish noise (LCG), so failures reproduce.
const makeNoise = (n, seed = 12345) => {
const out = new Float32Array(n);
let state = seed >>> 0;
for (let i = 0; i < n; i++) {
state = (state * 1664525 + 1013904223) >>> 0;
out[i] = state / 0x80000000 - 1; // uniform in [-1, 1)
}
return out;
};
test("fractional delay of a ramp by half a sample yields midpoint averages", () => {
const ramp = Float32Array.from({ length: 32 }, (_, i) => i);
const delayed = fractionalDelay(ramp, 0.5);
for (let i = 1; i < ramp.length; i++) {
assert.ok(near(delayed[i], (ramp[i] + ramp[i - 1]) / 2, 1e-5));
}
});
test("beamformer sums coherently when steered at the source", () => {
const sampleRate = 16000;
const beamformer = new DelaySumBeamformer(sampleRate);
const geometry = G2_DEFAULT_GEOMETRY;
const target = { azimuth: 0 };
const d = bearingUnitVector(target);
const c = geometry.speedOfSound;
const pF = geometry.positions.left.front;
const pR = geometry.positions.left.rear;
// A wavefront from straight ahead reaches each mic early by dot(p, d)/c.
const tauF = ((pF.x * d.x + pF.y * d.y) / c) * sampleRate;
const tauR = ((pR.x * d.x + pR.y * d.y) / c) * sampleRate;
// Frequency chosen so opposite-direction steering misaligns by half a
// period — the misaligned beam should nearly cancel.
const w = Math.PI / (2 * (tauF - tauR));
const n = 2048;
const front = new Float32Array(n);
const rear = new Float32Array(n);
for (let i = 0; i < n; i++) {
front[i] = Math.sin(w * (i + tauF));
rear[i] = Math.sin(w * (i + tauR));
}
const aligned = beamformer.beam("left", front, rear, target);
const misaligned = beamformer.beam("left", front, rear, { azimuth: Math.PI });
// Skip the edge the delay lines zero-fill.
const alignedRms = rms(aligned, 32, n);
const misalignedRms = rms(misaligned, 32, n);
assert.ok(near(alignedRms, Math.SQRT1_2, 0.05)); // full-amplitude sinusoid
assert.ok(alignedRms > 2 * misalignedRms);
});
test("direct-correlation TDOA recovers an integer delay", () => {
const a = makeNoise(2048);
const k = 5;
const b = new Float32Array(a.length);
for (let i = k; i < a.length; i++) b[i] = a[i - k];
assert.ok(near(gccPhatDelaySamples(a, b, 16), k, 0.15));
});
test("direct-correlation TDOA resolves a fractional delay via parabolic interpolation", () => {
const a = makeNoise(2048, 777);
const k = 3.4;
const b = fractionalDelay(a, k);
assert.ok(near(gccPhatDelaySamples(a, b, 16), k, 0.3));
});
test("NLMS canceller removes correlated noise and keeps the signal", () => {
const n = 8000;
const sampleRate = 16000;
const tone = new Float32Array(n);
for (let i = 0; i < n; i++) tone[i] = 0.5 * Math.sin((2 * Math.PI * 440 * i) / sampleRate);
const reference = makeNoise(n, 424242);
const primary = new Float32Array(n);
// The primary hears the noise through a short path (2-sample lag, 0.8 gain).
for (let i = 0; i < n; i++) {
primary[i] = tone[i] + 0.8 * (i >= 2 ? reference[i - 2] : 0);
}
const anc = new InterMicNoiseCanceller();
const out = anc.process(primary, reference);
// Judge the converged tail only.
const half = n / 2;
const primaryRms = rms(primary, half, n);
const outRms = rms(out, half, n);
const toneRms = rms(tone, half, n);
assert.ok(outRms < 0.8 * primaryRms); // noise reduced
assert.ok(outRms > 0.5 * toneRms); // signal survives
// Residual noise power (beyond the tone) drops substantially.
const noiseInPower = primaryRms * primaryRms - toneRms * toneRms;
const noiseOutPower = Math.max(0, outRms * outRms - toneRms * toneRms);
assert.ok(noiseOutPower < 0.5 * noiseInPower);
});
test("bearingFromTdoa is antisymmetric and clamps to ±90°", () => {
const sampleRate = 16000;
const spacing = 0.11;
const positive = bearingFromTdoa(2, sampleRate, spacing);
const negative = bearingFromTdoa(-2, sampleRate, spacing);
assert.ok(positive > 0);
assert.ok(near(positive, -negative, 1e-9));
assert.ok(near(bearingFromTdoa(100, sampleRate, spacing), Math.PI / 2, 1e-9));
assert.ok(near(bearingFromTdoa(-100, sampleRate, spacing), -Math.PI / 2, 1e-9));
assert.equal(bearingFromTdoa(0, sampleRate, spacing), 0);
});
test("angleInArc handles wraparound at ±180°", () => {
const center = deg(170);
const halfWidth = deg(30);
assert.ok(angleInArc(deg(-175), center, halfWidth)); // 15° away, across the seam
assert.ok(angleInArc(deg(150), center, halfWidth));
assert.ok(!angleInArc(deg(130), center, halfWidth)); // 40° away
assert.ok(!angleInArc(deg(-130), center, halfWidth)); // 60° away, across the seam
});
test("fuseBearings takes the circular mean across the ±180° seam", () => {
const fused = fuseBearings(deg(170), 1, deg(-170), 1);
assert.ok(near(Math.abs(fused), Math.PI, 1e-6));
assert.ok(near(fuseBearings(deg(10), 1, deg(20), 1), deg(15), 1e-6));
// Weights pull the mean toward the more confident temple.
const weighted = fuseBearings(0, 3, Math.PI / 2, 1);
assert.ok(near(weighted, Math.atan2(0.25, 0.75), 1e-6));
});
test("world and device frames round-trip through the head orientation", () => {
const head = { compassHeading: deg(40), imuYaw: deg(5) };
const world = deg(120);
const steered = steerWorldToDevice(world, head);
assert.ok(near(deviceToWorld(steered.azimuth, head), world, 1e-9));
assert.ok(near(bearingDegrees(steered), 120 - 45, 1e-9));
});
test("temple pipeline matches the raw beam with ANC disabled", () => {
const pipeline = new TempleAudioPipeline("right", 16000);
pipeline.enableANC = false;
const beamformer = new DelaySumBeamformer(16000);
const front = makeNoise(512, 1);
const rear = makeNoise(512, 2);
const target = { azimuth: deg(30) };
const viaPipeline = pipeline.process(front, rear, target);
const direct = beamformer.beam("right", front, rear, target);
assert.equal(viaPipeline.length, direct.length);
for (let i = 0; i < direct.length; i++) assert.equal(viaPipeline[i], direct[i]);
});
test("temple pipeline output is seamless across frame boundaries", () => {
// Streamed frame by frame, the steering delay must reach into the previous
// frame rather than zero-fill, so splitting the signal changes nothing.
const front = makeNoise(800, 3);
const rear = makeNoise(800, 4);
const target = { azimuth: deg(60) };
const whole = new TempleAudioPipeline("left", 16000);
whole.enableANC = false;
const expected = whole.process(front, rear, target);
const framed = new TempleAudioPipeline("left", 16000);
framed.enableANC = false;
const first = framed.process(front.subarray(0, 400), rear.subarray(0, 400), target);
const second = framed.process(front.subarray(400), rear.subarray(400), target);
assert.equal(first.length + second.length, expected.length);
for (let i = 0; i < 400; i++) assert.ok(near(first[i], expected[i], 1e-6));
for (let i = 0; i < 400; i++) assert.ok(near(second[i], expected[400 + i], 1e-6));
});