Files
faceclaw/app/apps/microphones/dsp.ts
T
James Babcock b50898ea02 Fix a bunch of microphone-related bugs
> Most serious fixes
> - Recordings kept forever: a recording made with captions off, or with "Save captions"
>   off, was transcoded to .m4a and kept indefinitely. No conversation linked to it, so
>   the phone UI never showed it and the retention settings never deleted it. Recording
>   now only happens alongside a saved caption session, and turning captions off ends
>   that session's recording properly.
> - 4-mic mode went silent after a reconnect: a disconnect clears the phone's audio
>   forwarding, and nothing turned it back on, so the app kept showing "Listening (4-mic
>   array)" while hearing nothing. A check every 3 s now restores forwarding and re-sends
>   the config when it has been lost. The normal single-stream mode already recovered on
>   its own.
> - Turning off "4-mic capture" mid-session went silent: it stopped the glasses' stream
>   but never switched to the normal stream. Setup changes now take effect live,
>   including switching between the two modes.
> - Setup changes were undone: after the assistant closed, the app re-sent the config it
>   loaded at launch to the glasses, reverting any changes you'd made since.
> - Startup failures: if the caption engine failed to start, the session got stuck
>   half-running. If the glasses weren't connected, listeners piled up and an empty
>   conversation could be left open. Startup now brings up the microphone first and
>   cleans up properly on failure.
>
> Other fixes
> - Audio quality: the 4-mic beamformer put a small dropout at the start of every 25 ms
>   frame (an audible 40 Hz artifact in the audio feeding captions). Fixed, with a test
>   that fails on the old code.
> - Stale direction data: an arm that stopped sending kept steering the direction
>   estimate and could hold the caption feed on a silent arm.
> - Compass: closing Microphones switched off the Compass app's compass, because both
>   used the same owner name.
> - Speaker database failures:
>   - A failed speaker reload wiped the in-memory list, so the next voice became a
>     duplicate "Speaker N".
>   - A failed speaker insert crashed caption handling; in re-diarize it reassigned
>     transcript lines to a nonexistent speaker.
> - Translation:
>   - A crash path inside the model-download callback is now caught.
>   - Phones set to Hebrew, Indonesian or Yiddish never got translations, because of
>     old-style language codes.
> - Ask questions: once the speaker list grew past the size limit (every unknown voice
>   adds a "Speaker N"), it pushed all transcript lines out of the context sent to the
>   model. When trimming is needed, the oldest lines now go first.
> - Smaller fixes:
>   - The captions view stopped jumping back to live speech after 60 lines.
>   - "This is Dana calling…" was treated as introducing someone else.
>   - A model download that failed to start was stuck at "downloading" until restart.
>   - The developer mic monitor grew without limit while hidden.
2026-10-14 00:56:54 -07:00

338 lines
12 KiB
TypeScript
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
// Reference DSP for the G2 4-mic head array: two 2-mic temples, front mic
// near each hinge and rear mic at each tip. Beamform WITHIN each temple (shared clock), then FUSE
// the two per-temple bearings — never raw cross-temple delay-and-sum, because
// the temples sample on independent clocks.
//
// One substitution from classic GCC-PHAT: the FFT cross-spectrum is
// replaced by a direct time-domain normalized cross-correlation over
// -maxLag..+maxLag. For the lags this array needs (|lag| <= 16 at 16 kHz) the
// direct form is cheaper than an FFT round-trip and carries no FFT dependency;
// the parabolic-interpolated peak still yields sub-sample delay.
//
// Pure TypeScript; no platform imports. Hot paths are Float32Array in and out.
export type MicSide = "left" | "right";
export type MicPosition = "front" | "rear";
// ---------------------------------------------------------------------------
// Array geometry (metres, head-centred; x=right, y=forward, z=up)
export type Vec3 = { x: number; y: number; z: number };
export type MicGeometry = {
positions: Record<MicSide, Record<MicPosition, Vec3>>;
speedOfSound: number;
};
export const G2_DEFAULT_GEOMETRY: MicGeometry = {
positions: {
left: {
front: { x: -0.07, y: 0.05, z: 0 },
rear: { x: -0.08, y: -0.06, z: 0 },
},
right: {
front: { x: 0.07, y: 0.05, z: 0 },
rear: { x: 0.08, y: -0.06, z: 0 },
},
},
speedOfSound: 343,
};
// ---------------------------------------------------------------------------
// Bearing helpers
/** Azimuth in radians, 0 = straight ahead (device +y), positive to the right. */
export type Bearing = { azimuth: number };
export function bearingDegrees(bearing: Bearing): number {
return (bearing.azimuth * 180) / Math.PI;
}
export function bearingUnitVector(bearing: Bearing): Vec3 {
return { x: Math.sin(bearing.azimuth), y: Math.cos(bearing.azimuth), z: 0 };
}
function dot(a: Vec3, b: Vec3): number {
return a.x * b.x + a.y * b.y + a.z * b.z;
}
// ---------------------------------------------------------------------------
// Fractional delay (shared by the beamformer and by tests)
/** Linear-interpolating fractional delay (positive = later). */
export function fractionalDelay(x: Float32Array, samples: number): Float32Array {
const n = x.length;
if (Math.abs(samples) < 1e-4) return x.slice();
const out = new Float32Array(n);
const i0 = Math.floor(samples);
const frac = samples - i0;
for (let i = 0; i < n; i++) {
const j = i - i0;
const a = j >= 0 && j < n ? x[j]! : 0;
const b = j - 1 >= 0 && j - 1 < n ? x[j - 1]! : 0;
out[i] = a * (1 - frac) + b * frac;
}
return out;
}
// ---------------------------------------------------------------------------
// Delay-and-sum beamformer (per temple, 2 mics, shared clock)
export class DelaySumBeamformer {
constructor(
public readonly sampleRate: number,
public readonly geometry: MicGeometry = G2_DEFAULT_GEOMETRY,
) {}
/**
* Steer this temple's front/rear pair toward `target` (device-frame bearing)
* and return the summed beam, halved to preserve amplitude.
*/
beam(side: MicSide, front: Float32Array, rear: Float32Array, target: Bearing): Float32Array {
const n = Math.min(front.length, rear.length);
if (n === 0) return new Float32Array(0);
const d = bearingUnitVector(target);
const pF = this.geometry.positions[side].front;
const pR = this.geometry.positions[side].rear;
// Delay (samples) that aligns each mic's copy of a wavefront from `d`.
const tauF = (dot(pF, d) / this.geometry.speedOfSound) * this.sampleRate;
const tauR = (dot(pR, d) / this.geometry.speedOfSound) * this.sampleRate;
const ref = Math.min(tauF, tauR);
const sF = fractionalDelay(front, tauF - ref);
const sR = fractionalDelay(rear, tauR - ref);
const out = new Float32Array(n);
for (let i = 0; i < n; i++) out[i] = (sF[i]! + sR[i]!) * 0.5;
return out;
}
}
// ---------------------------------------------------------------------------
// Two-mic adaptive noise canceller (NLMS) for inter-mic ANC
/**
* Griffiths–Jim style: `primary` carries the steered speech, `reference` is a
* null-steered noise estimate (front−rear after alignment). An NLMS filter
* subtracts the correlated noise from the primary.
*/
export class InterMicNoiseCanceller {
private readonly weights: Float32Array;
private readonly history: Float32Array;
constructor(
private readonly taps: number = 32,
private readonly stepSize: number = 0.05,
) {
this.weights = new Float32Array(taps);
this.history = new Float32Array(taps);
}
process(primary: Float32Array, reference: Float32Array): Float32Array {
const n = Math.min(primary.length, reference.length);
const out = new Float32Array(n);
const { weights, history, taps } = this;
const eps = 1e-6;
for (let i = 0; i < n; i++) {
for (let k = taps - 1; k > 0; k--) history[k] = history[k - 1]!;
history[0] = reference[i]!;
let yhat = 0;
let energy = 0;
for (let k = 0; k < taps; k++) {
yhat += weights[k]! * history[k]!;
energy += history[k]! * history[k]!;
}
const e = primary[i]! - yhat;
out[i] = e;
const step = (this.stepSize * e) / (energy + eps);
for (let k = 0; k < taps; k++) weights[k] += step * history[k]!;
}
return out;
}
}
// ---------------------------------------------------------------------------
// TDOA via direct normalized cross-correlation (GCC-PHAT substitution)
/**
* Estimate the delay of `b` relative to `a` in samples: if `b` is a copy of
* `a` delayed by k samples, this returns approximately +k.
*
* Substitution note: classic GCC-PHAT whitens an FFT cross-spectrum. Here
* the lag search is tiny (|lag| <= maxLag, typically <= 16),
* so a direct time-domain normalized cross-correlation over the lag range is
* cheaper and dependency-free. The per-lag normalization by the overlapped
* segments' energies plays the whitening role of making the peak
* amplitude-invariant. The peak is refined by parabolic interpolation for
* sub-sample precision.
*/
export function gccPhatDelaySamples(a: Float32Array, b: Float32Array, maxLag: number): number {
const n = Math.min(a.length, b.length);
if (n === 0 || maxLag < 0) return 0;
const lagCount = 2 * maxLag + 1;
const corr = new Float32Array(lagCount);
let bestLag = 0;
let best = -Infinity;
for (let lag = -maxLag; lag <= maxLag; lag++) {
// Correlate a[i] against b[i + lag] over the overlapping span.
const start = Math.max(0, -lag);
const end = Math.min(n, n - lag);
let sum = 0;
let ea = 0;
let eb = 0;
for (let i = start; i < end; i++) {
const av = a[i]!;
const bv = b[i + lag]!;
sum += av * bv;
ea += av * av;
eb += bv * bv;
}
const r = sum / (Math.sqrt(ea * eb) + 1e-12);
corr[lag + maxLag] = r;
if (r > best) {
best = r;
bestLag = lag;
}
}
// Parabolic interpolation around the integer peak for sub-sample precision.
const idx = bestLag + maxLag;
if (idx > 0 && idx < lagCount - 1) {
const y0 = corr[idx - 1]!;
const y1 = corr[idx]!;
const y2 = corr[idx + 1]!;
const denom = y0 - 2 * y1 + y2;
if (Math.abs(denom) > 1e-12) {
const offset = (0.5 * (y0 - y2)) / denom;
if (Math.abs(offset) <= 1) return bestLag + offset;
}
}
return bestLag;
}
// ---------------------------------------------------------------------------
// Compass / IMU fusion: world-frame arc <-> device-frame steering
/**
* `compassHeading` is the world heading of the device +y axis (radians,
* 0 = north); `imuYaw` is a supplementary IMU trim (radians).
*/
export type HeadOrientation = { compassHeading: number; imuYaw: number };
/** Combined world→device rotation about vertical (compass anchored, IMU trims). */
function worldToDevice(head: HeadOrientation): number {
return -(head.compassHeading + head.imuYaw);
}
/** Rotate a world-frame target bearing into the device frame for steering. */
export function steerWorldToDevice(worldAzimuth: number, head: HeadOrientation): Bearing {
return { azimuth: worldAzimuth + worldToDevice(head) };
}
/** Convert a device-frame estimate back to world frame for display/fusion. */
export function deviceToWorld(deviceAzimuth: number, head: HeadOrientation): number {
return deviceAzimuth - worldToDevice(head);
}
/**
* Triangulate the two temples' world-frame bearings into a single estimate:
* a circular-mean-safe blend, weighted by each temple's confidence.
*/
export function fuseBearings(
left: number,
leftWeight: number,
right: number,
rightWeight: number,
): number {
const w = Math.max(1e-3, leftWeight + rightWeight);
const x = (Math.sin(left) * leftWeight + Math.sin(right) * rightWeight) / w;
const y = (Math.cos(left) * leftWeight + Math.cos(right) * rightWeight) / w;
return Math.atan2(x, y);
}
// ---------------------------------------------------------------------------
// Faceclaw integration utilities
/**
* Azimuth from an inter-mic delay, mirroring the firmware's approach:
* asin of the delay normalized by the acoustic travel time across the pair,
* clamped to ±90°.
*/
export function bearingFromTdoa(
delaySamples: number,
sampleRate: number,
micSpacingMeters: number,
): number {
const maxDelay = (micSpacingMeters / 343) * sampleRate;
if (maxDelay <= 0) return 0;
const normalized = Math.max(-1, Math.min(1, delaySamples / maxDelay));
return Math.asin(normalized);
}
/** Wrap an angle to (-π, π]. */
function wrapAngle(angle: number): number {
const twoPi = 2 * Math.PI;
let a = angle % twoPi;
if (a <= -Math.PI) a += twoPi;
else if (a > Math.PI) a -= twoPi;
return a;
}
/** Whether `azimuth` falls inside the arc, handling wraparound at ±π. */
export function angleInArc(azimuth: number, arcCenter: number, arcHalfWidth: number): boolean {
return Math.abs(wrapAngle(azimuth - arcCenter)) <= arcHalfWidth;
}
// ---------------------------------------------------------------------------
// Top-level per-temple pipeline
export class TempleAudioPipeline {
private readonly beamformer: DelaySumBeamformer;
private readonly anc = new InterMicNoiseCanceller();
// Steering delays reach back before the start of each frame. Carrying the
// previous frame's last samples keeps those real; zero-filling them (as a
// standalone beam() does) dips the output at every frame boundary.
private readonly carry: number;
private frontTail: Float32Array;
private rearTail: Float32Array;
enableANC = true;
constructor(
public readonly side: MicSide,
sampleRate: number,
geometry: MicGeometry = G2_DEFAULT_GEOMETRY,
) {
this.beamformer = new DelaySumBeamformer(sampleRate, geometry);
const { front, rear } = geometry.positions[side];
const spacing = Math.hypot(front.x - rear.x, front.y - rear.y, front.z - rear.z);
this.carry = Math.ceil((spacing / geometry.speedOfSound) * sampleRate) + 1;
this.frontTail = new Float32Array(this.carry);
this.rearTail = new Float32Array(this.carry);
}
/**
* Process one frame's deinterleaved front/rear channels toward a
* device-frame target, returning the enhanced mono beam for this temple.
* Frames must be consecutive: each one continues the previous.
*/
process(front: Float32Array, rear: Float32Array, target: Bearing): Float32Array {
const n = Math.min(front.length, rear.length);
const extendedFront = this.withTail(this.frontTail, front, n);
const extendedRear = this.withTail(this.rearTail, rear, n);
this.frontTail = extendedFront.slice(n);
this.rearTail = extendedRear.slice(n);
const beam = this.beamformer
.beam(this.side, extendedFront, extendedRear, target)
.slice(this.carry);
if (!this.enableANC) return beam;
// Null-steer reference: the difference channel carries off-axis noise.
const reference = new Float32Array(n);
for (let i = 0; i < n; i++) reference[i] = front[i]! - rear[i]!;
return this.anc.process(beam, reference);
}
private withTail(tail: Float32Array, frame: Float32Array, n: number): Float32Array {
const out = new Float32Array(tail.length + n);
out.set(tail);
out.set(frame.subarray(0, n), tail.length);
return out;
}
}