Merge branch 'microphone-fixes'

This commit is contained in:
James Babcock committed 2026-09-29 20:34:09 -07:00
commit f7fa4addc0
9 files changed
+665 -51

No files matched your search

@@ -212,6 +212,10 @@ class FaceclawBleCommunicator(context: Context, rightAddress: String?, leftAddre
fun stopG2AudioForwarding() = core.stopG2AudioForwarding()
fun addAudioMonitorListener(listener: FaceclawAudioPacketListener?) = core.addAudioMonitorListener(listener)
fun removeAudioMonitorListener(listener: FaceclawAudioPacketListener?) = core.removeAudioMonitorListener(listener)
fun addCompassListener(listener: FaceclawCompassListener?) = core.addCompassListener(listener)
fun removeCompassListener(listener: FaceclawCompassListener?) = core.removeCompassListener(listener)
+17 -2
View File
@@ -6,6 +6,7 @@ import { BuzzerDemoLayer } from "./buzzer-demo";
import { AccelerometerDemoLayer } from "./accelerometer-demo";
import { BandwidthBenchmarkLayer } from "./bandwidth-benchmark";
import { LightSensorDemoLayer } from "./light-sensor-demo";
import { MicrophonesMonitorLayer } from "./microphones-monitor";
import { ResourceUsageLayer } from "./resource-usage";
import { LoadAppFromQrLayer, LoadAppFromUrlLayer } from "./load-app";
import { unicodeTestMenu } from "./unicode-test";
@@ -49,8 +50,10 @@ function submenuItem(label: string, onSelect: MenuItem["onSelect"]): MenuItem {
};
}
type OpenPage = (ctx: Parameters<MenuItem["onSelect"]>[0]) => void;
/** The diagnostic demos, one level down from the root menu. */
function debugTestsMenu(openInputEvents: (ctx: Parameters<MenuItem["onSelect"]>[0]) => void): MenuLayer {
function debugTestsMenu(openInputEvents: OpenPage, openMicrophones: OpenPage): MenuLayer {
return new MenuLayer(
"Debug tests",
[
@@ -85,6 +88,12 @@ function debugTestsMenu(openInputEvents: (ctx: Parameters<MenuItem["onSelect"]>[
ctx.stack.push(new LightSensorDemoLayer(DEVELOPER_WINDOW_ID, ctx.actions.requestRender));
},
},
{
label: "Microphones",
disabled: global.isIOS,
description: global.isIOS ? "Not available on iOS yet." : undefined,
onSelect: openMicrophones,
},
{
label: "BLE bandwidth",
onSelect: (ctx) => {
@@ -107,6 +116,7 @@ function debugTestsMenu(openInputEvents: (ctx: Parameters<MenuItem["onSelect"]>[
*/
export function createDeveloperAppWindow(appContext: AppContext, options: InProcessAppOptions): InProcessWindow {
let inputEvents: InputEventsLayer | null = null;
let microphones: MicrophonesMonitorLayer | null = null;
const menu = new MenuLayer(
"Developer",
[
@@ -141,6 +151,11 @@ export function createDeveloperAppWindow(appContext: AppContext, options: InProc
() => { if (inputEvents === page) inputEvents = null; });
inputEvents = page;
inputCtx.stack.push(page);
}, (micCtx) => {
const page = new MicrophonesMonitorLayer(DEVELOPER_WINDOW_ID, micCtx.actions.requestRender,
() => { if (microphones === page) microphones = null; });
microphones = page;
micCtx.stack.push(page);
}));
}),
],
@@ -155,7 +170,7 @@ export function createDeveloperAppWindow(appContext: AppContext, options: InProc
icon: "wrench",
closeable: true,
actions: options.actions,
menuItems: () => inputEvents?.menuItems() ?? [],
menuItems: () => inputEvents?.menuItems() ?? microphones?.menuItems() ?? [],
// Dictating a URL is the one thing worth speaking at in this app; the
// load pages take the text and every other page ignores it.
receiveTextInput: (text) => {
+426
View File
@@ -0,0 +1,426 @@
import { getDefaultSmallFont } from "../../graphics/ui-fonts";
import { GrayImage } from "../../graphics/image";
import { clamp } from "../../util/numeric-util";
import {
addAudioMonitorListeners,
audioMonitorCommunicator,
isFaceclawAudioCaptureActive,
type AudioArm,
} from "../../native/audio-monitor";
import { micArrayController } from "../microphones/mic-control";
import {
MIC_RET_NONE,
STOCK_PACKET_BYTES,
decodeMicStatus,
decodeMicStreamFrame,
splitInterleavedPcm16,
type MicStatus,
type MicStreamFrame,
} from "../microphones/mic-protocol";
import { GESTURE_DOUBLE_CLICK, type InputEvent } from "../../ui/gestures";
import { type MenuItem } from "../../ui/menu";
import { Layer, type LayerContext } from "../../ui/layers";
import { shell } from "../../ui/shell/shell";
// Packet rate / byte rate / largest gap are measured over this trailing window.
const RATE_WINDOW_MS = 2_000;
// Activity strip: packet counts per bucket over the last HISTORY_MS.
const HISTORY_MS = 20_000;
const BUCKET_MS = 500;
const BUCKETS = HISTORY_MS / BUCKET_MS;
// Repaint cadence; also how often a replaced communicator is re-subscribed.
const TICK_MS = 250;
// Stock packet trailer (see Lc3PacketFramer.kt): SSR, angle, counter.
const STOCK_SSR_OFFSET = 200;
const STOCK_ANGLE_OFFSET = 202;
const STOCK_COUNTER_OFFSET = 204;
// Counter gaps at or beyond this are late/duplicate copies, not losses.
const STOCK_LATE_GAP = 128;
const LEVEL_FLOOR_DB = -90;
// wallMs (JS receipt time) windows the rates; arrivalMs (the session core's
// elapsed-realtime stamp, taken on the BLE thread) measures gaps without the
// main-thread hop's jitter.
type Arrival = { wallMs: number; arrivalMs: number; bytes: number };
type ArmStats = {
packets: number;
stockPackets: number;
smPackets: number;
otherPackets: number;
lastOtherBytes: number;
lastArrivalWallMs: number | null;
recent: Arrival[];
buckets: number[];
stockCounter: number | null;
stockLost: number;
stockDuplicate: number;
stockAngle: number;
stockSsr: number;
frame: MicStreamFrame | null;
smSeq: number | null;
smSeqGaps: number;
channelDb: number[];
status: MicStatus | null;
statusAtMs: number | null;
};
function emptyStats(): ArmStats {
return {
packets: 0,
stockPackets: 0,
smPackets: 0,
otherPackets: 0,
lastOtherBytes: 0,
lastArrivalWallMs: null,
recent: [],
buckets: new Array<number>(BUCKETS).fill(0),
stockCounter: null,
stockLost: 0,
stockDuplicate: 0,
stockAngle: 0,
stockSsr: 0,
frame: null,
smSeq: null,
smSeqGaps: 0,
channelDb: [],
status: null,
statusAtMs: null,
};
}
/**
* Passive microphone monitor: shows what the glasses are sending on the
* audio characteristic, per temple, without enabling anything. Open it, then
* activate the mics from another app (Microphones, Transcribe, voice input)
* and come back: collection continues while this page is hidden, so the
* counters and the 20 s activity strip show whether that activation produced
* a stream, what kind (stock mono LC3 or CFW 'SM' frames), and how healthy it
* is. CFW mic_control status records are shown when they arrive: after a
* CONFIGURE, with the periodic battery read, or from this window's "Query mic
* status" menu item (a status request only; it touches no mic hardware).
* Only the right temple can send status; each lens's debug overlay shows its
* own counters.
*/
export class MicrophonesMonitorLayer implements Layer {
private readonly stats: Record<"L" | "R", ArmStats> = { L: emptyStats(), R: emptyStats() };
private unknownArmPackets = 0;
private bucketStartMs = Date.now();
private communicator: any = null;
private unsubscribe: (() => void) | null = null;
private timer: ReturnType<typeof setInterval> | null = null;
private removed = false;
constructor(
private readonly windowId: string,
private readonly requestRender: () => void,
private readonly onClosed: () => void = () => {},
) {
this.tick();
this.timer = setInterval(() => this.tick(), TICK_MS);
}
onRemoved(): void {
this.removed = true;
if (this.timer !== null) {
clearInterval(this.timer);
this.timer = null;
}
this.unsubscribe?.();
this.unsubscribe = null;
this.communicator = null;
this.onClosed();
}
private tick(): void {
if (this.removed) return;
const communicator = audioMonitorCommunicator();
if (communicator !== this.communicator) {
this.unsubscribe?.();
this.communicator = communicator;
this.unsubscribe = addAudioMonitorListeners(
communicator,
(data, arm, arrivalMs) => this.onPacket(data, arm, arrivalMs),
(body, arm) => this.onStatus(body, arm),
);
}
this.advanceBuckets(Date.now());
if (shell.isWindowVisible(this.windowId)) this.requestRender();
}
private advanceBuckets(nowMs: number): void {
while (nowMs - this.bucketStartMs >= BUCKET_MS) {
this.bucketStartMs += BUCKET_MS;
for (const stats of Object.values(this.stats)) {
stats.buckets.shift();
stats.buckets.push(0);
}
}
}
private onPacket(data: Uint8Array, arm: AudioArm, arrivalMs: number): void {
if (this.removed) return;
if (arm === "?") {
this.unknownArmPackets++;
return;
}
const stats = this.stats[arm];
const nowMs = Date.now();
this.advanceBuckets(nowMs);
stats.packets++;
stats.buckets[BUCKETS - 1]!++;
stats.recent.push({ wallMs: nowMs, arrivalMs, bytes: data.length });
stats.lastArrivalWallMs = nowMs;
const frame = decodeMicStreamFrame(data);
if (frame) {
this.onStreamFrame(stats, frame);
} else if (data.length === STOCK_PACKET_BYTES) {
this.onStockPacket(stats, data);
} else {
stats.otherPackets++;
stats.lastOtherBytes = data.length;
}
}
private onStockPacket(stats: ArmStats, data: Uint8Array): void {
stats.stockPackets++;
const counter = data[STOCK_COUNTER_OFFSET]!;
if (stats.stockCounter !== null) {
const gap = (counter - stats.stockCounter) & 0xff;
if (gap === 0 || gap >= STOCK_LATE_GAP) {
stats.stockDuplicate++;
return;
}
stats.stockLost += gap - 1;
}
stats.stockCounter = counter;
stats.stockSsr = readS16(data, STOCK_SSR_OFFSET);
stats.stockAngle = readS16(data, STOCK_ANGLE_OFFSET);
}
private onStreamFrame(stats: ArmStats, frame: MicStreamFrame): void {
stats.smPackets++;
if (stats.smSeq !== null && frame.sequence !== ((stats.smSeq + 1) & 0xffff)) stats.smSeqGaps++;
stats.smSeq = frame.sequence;
stats.frame = frame;
const channels = splitInterleavedPcm16(frame);
stats.channelDb = channels ? channels.map(rmsDb) : [];
}
private onStatus(body: Uint8Array, arm: AudioArm): void {
if (this.removed || arm === "?") return;
const status = decodeMicStatus(body);
if (!status) return;
this.stats[arm].status = status;
this.stats[arm].statusAtMs = Date.now();
}
paint(ctx: LayerContext): GrayImage {
const font = getDefaultSmallFont();
const line = font.lineHeight;
const { width, height } = ctx.stack.getBaseSize();
const image = new GrayImage(width, height, 0);
const nowMs = Date.now();
image.drawText(font, 16, 5, "Microphones (passive)", 230);
const capture = isFaceclawAudioCaptureActive() ? "Faceclaw capture: on" : "Faceclaw capture: off";
image.drawText(font, width - 16 - font.measureText(capture), 5, capture, 170);
if (!this.communicator) {
image.drawText(font, 16, 5 + line * 2, "Glasses not connected.", 190);
image.drawText(font, 16, height - 16, `${GESTURE_DOUBLE_CLICK} back`, 110);
return image;
}
const gutter = 16;
const columnWidth = Math.floor((width - gutter * 3) / 2);
const stripHeight = 22;
const top = 5 + line + 6;
const bottom = height - 16 - stripHeight - 8;
this.paintArm(image, "L", "Left", gutter, top, columnWidth, bottom, nowMs);
this.paintArm(image, "R", "Right", gutter * 2 + columnWidth, top, columnWidth, bottom, nowMs);
const stripY = height - 16 - stripHeight - 4;
const stripMax = Math.max(1, ...this.stats.L.buckets, ...this.stats.R.buckets);
drawStrip(image, this.stats.L.buckets, gutter, stripY, columnWidth, stripHeight, stripMax);
drawStrip(image, this.stats.R.buckets, gutter * 2 + columnWidth, stripY, columnWidth, stripHeight, stripMax);
const footer = this.unknownArmPackets > 0
? `last ${HISTORY_MS / 1000} s unattributed packets ${this.unknownArmPackets}`
: `packets per ${BUCKET_MS} ms, last ${HISTORY_MS / 1000} s`;
image.drawText(font, gutter, height - 16, footer, 110);
const back = `${GESTURE_DOUBLE_CLICK} back tap reset`;
image.drawText(font, width - gutter - font.measureText(back), height - 16, back, 110);
return image;
}
private paintArm(
image: GrayImage,
arm: "L" | "R",
label: string,
x: number,
top: number,
width: number,
bottom: number,
nowMs: number,
): void {
const font = getDefaultSmallFont();
const line = font.lineHeight;
const stats = this.stats[arm];
const cutoff = nowMs - RATE_WINDOW_MS;
while (stats.recent.length > 0 && stats.recent[0]!.wallMs < cutoff) stats.recent.shift();
const rate = (stats.recent.length * 1000) / RATE_WINDOW_MS;
const byteRate = stats.recent.reduce((sum, arrival) => sum + arrival.bytes, 0) / RATE_WINDOW_MS;
let maxGap = 0;
for (let i = 1; i < stats.recent.length; i++) {
maxGap = Math.max(maxGap, stats.recent[i]!.arrivalMs - stats.recent[i - 1]!.arrivalMs);
}
const rows: Array<[string, number]> = [];
const streaming = stats.recent.length > 0;
rows.push([`${label}: ${streaming ? `${rate.toFixed(1)} pkt/s ${byteRate.toFixed(1)} kB/s` : "silent"}`, streaming ? 235 : 150]);
const age = stats.lastArrivalWallMs === null ? "never" : `${formatAge(nowMs - stats.lastArrivalWallMs)} ago`;
rows.push([`last ${age}${streaming && stats.recent.length > 1 ? ` max gap ${maxGap} ms` : ""}`, 170]);
rows.push([`total ${stats.packets}: stock ${stats.stockPackets} SM ${stats.smPackets} other ${stats.otherPackets}${stats.otherPackets ? ` (${stats.lastOtherBytes} B)` : ""}`, 170]);
if (stats.stockPackets > 0) {
rows.push([`stock LC3: lost ${stats.stockLost} dup ${stats.stockDuplicate}`, 190]);
rows.push([` angle ${stats.stockAngle}° ssr ${stats.stockSsr}`, 170]);
}
const frame = stats.frame;
if (frame) {
const flags = [frame.truncated ? "trunc" : "", stats.smSeqGaps ? `seq gaps ${stats.smSeqGaps}` : ""].filter(Boolean).join(" ");
rows.push([`SM: ${frame.channelCount}ch ${frame.sampleRateHz / 1000} kHz ${frame.codec} ${frame.payload.length} B ${flags}`, 190]);
rows.push([` angle ${frame.angleDegrees}° ssr ${frame.ssr}`, 170]);
}
const status = stats.status;
if (status) {
const armed = status.hardwareArmed ? "armed" : status.active ? "configured" : "inactive";
rows.push([`CFW ${armed}: ${status.source} mask ${status.channelMask} frames ${status.framesEmitted}`, 190]);
rows.push([` ${status.codec} ${status.effectiveRateHz / 1000} kHz ${formatAge(nowMs - (stats.statusAtMs ?? nowMs))} ago`, 150]);
const diag = status.diagnostics;
if (diag) {
rows.push([` tap ${diag.tapCalls} calls last ${diag.tapLastBytes} B @slot ${diag.tapLastSlot}`, 190]);
rows.push([` skip idle ${diag.skipIdle} lease ${diag.skipLease} alloc ${diag.skipAlloc}`, 170]);
rows.push([
` reg ${formatRet(diag.registerResult)} unreg ${formatRet(diag.unregisterResult)} ntf ${formatRet(diag.notifyResult)} slots ${diag.slots.map(formatSlot).join("")} lease ${diag.leaseRemainingS} s`,
170,
]);
} else {
rows.push([" (no tap diagnostics: older firmware)", 120]);
}
} else {
rows.push(["CFW status: none seen", 120]);
}
let y = top;
for (const [text, shade] of rows) {
if (y + line > bottom) return;
image.drawText(font, x, y, fitText(font, text, width), shade);
y += line;
}
// Per-channel RMS level bars for raw PCM frames.
for (let channel = 0; channel < stats.channelDb.length; channel++) {
if (y + line > bottom) return;
const db = stats.channelDb[channel]!;
const tag = `ch${channel} ${db <= LEVEL_FLOOR_DB ? "-inf" : db.toFixed(0)} dB`;
image.drawText(font, x, y, tag, 190);
const barX = x + font.measureText("ch0 -100 dB") + 6;
const barWidth = Math.max(0, width - (barX - x));
const fill = Math.round(clamp((db - LEVEL_FLOOR_DB) / -LEVEL_FLOOR_DB, 0, 1) * barWidth);
image.drawRect(barX, y + 3, barWidth, line - 6, 60);
if (fill > 0) image.fillRect(barX, y + 3, fill, line - 6, db > -3 ? 255 : 170);
y += line;
}
}
/** Window context-menu entries while this page is open. */
menuItems(): MenuItem[] {
return [
{
label: "Query mic status",
onSelect: (ctx) => {
micArrayController.query();
ctx.stack.pop();
},
},
{
label: "Reset counters",
onSelect: (ctx) => {
this.resetCounters();
ctx.stack.pop();
},
},
];
}
private resetCounters(): void {
this.stats.L = emptyStats();
this.stats.R = emptyStats();
this.unknownArmPackets = 0;
this.requestRender();
}
async handleInput(event: InputEvent, ctx: LayerContext): Promise<void> {
if (event.type === "double-click") {
ctx.stack.pop();
} else if (event.type === "click") {
this.resetCounters();
}
}
}
function formatRet(value: number): string {
return value === MIC_RET_NONE ? "-" : String(value);
}
/** Same letters as the firmware overlay: - empty, S another callback, T our tap. */
function formatSlot(state: "empty" | "other" | "tap"): string {
return state === "tap" ? "T" : state === "other" ? "S" : "-";
}
function readS16(data: Uint8Array, offset: number): number {
const value = data[offset]! | (data[offset + 1]! << 8);
return value >= 0x8000 ? value - 0x10000 : value;
}
function rmsDb(samples: Float32Array): number {
let sumSquares = 0;
for (let i = 0; i < samples.length; i++) sumSquares += samples[i]! * samples[i]!;
const rms = Math.sqrt(sumSquares / Math.max(1, samples.length));
return rms > 0 ? Math.max(LEVEL_FLOOR_DB, 20 * Math.log10(rms)) : LEVEL_FLOOR_DB;
}
function formatAge(ms: number): string {
if (ms < 1_000) return `${Math.max(0, Math.round(ms))} ms`;
if (ms < 60_000) return `${(ms / 1000).toFixed(1)} s`;
return `${Math.round(ms / 60_000)} min`;
}
function fitText(font: ReturnType<typeof getDefaultSmallFont>, text: string, width: number): string {
if (font.measureText(text) <= width) return text;
let end = text.length;
while (end > 0 && font.measureText(`${text.slice(0, end)}…`) > width) end--;
return `${text.slice(0, end)}…`;
}
function drawStrip(
image: GrayImage,
buckets: readonly number[],
x: number,
y: number,
width: number,
height: number,
max: number,
): void {
image.drawRect(x, y, width, height, 45);
const barWidth = width / buckets.length;
for (let i = 0; i < buckets.length; i++) {
const count = buckets[i]!;
if (count <= 0) continue;
const barHeight = Math.max(1, Math.round((count / max) * (height - 2)));
const left = x + Math.floor(i * barWidth);
const right = x + Math.floor((i + 1) * barWidth) - 1;
image.fillRect(left + 1, y + height - 1 - barHeight, Math.max(1, right - left), barHeight, 200);
}
}
+68 -12
View File
@@ -12,8 +12,8 @@
*
* Per-temple: each arm is its own endpoint and its own 2-mic array (front
* mic near the hinge = chanMask bit0, rear mic near the touchpad = bit1);
* the same config goes to both temples. Stereo payloads arrive as two
* CONCATENATED channel blocks (front then rear), not interleaved. Sessions
* the same config goes to both temples. Stereo payloads are INTERLEAVED PCM16
* (ch0, ch1, ch0, ch1, ...), the capture dispatch buffer verbatim. Sessions
* hold a 90 s lease; renew with op RENEW while streaming.
*
* Pure module: byte codecs only, no platform imports (the BLE plumbing lives
@@ -54,7 +54,7 @@ export type MicConfig = {
/**
* Host-side per-microphone selection. Each temple streams its full
* front+rear stereo pair over BLE (wire channelMask stays 0b11); the phone
* splits the concatenated channels and this mask decides which of the four
* deinterleaves the channels and this mask decides which of the four
* mics feed levels-for-use, beamforming, captions, and recording. A temple
* with both mics off is sent STOP so its stream doesn't waste bandwidth.
*/
@@ -168,8 +168,42 @@ export type MicStatus = {
framesEmitted: number;
/** The rate actually in effect (16 kHz on current firmware). */
effectiveRateHz: number;
/** Tap diagnostics; absent from firmware that sends only the 21-byte body. */
diagnostics: MicStatusDiagnostics | null;
};
/** Result of a firmware call the temple has not made yet. */
export const MIC_RET_NONE = 127;
/** Occupant of one stock PCM app slot. */
export type MicSlotState = "empty" | "other" | "tap";
/**
* Counters from the status extension; they reset on each CONFIGURE. The
* firmware debug overlay shows the same values on each lens.
*/
export type MicStatusDiagnostics = {
/** Calls into the PCM tap (any outcome). */
tapCalls: number;
tapLastBytes: number;
tapLastSlot: number;
/** Tap calls dropped because the session was inactive or unarmed. */
skipIdle: number;
/** Tap calls dropped because the streaming lease had lapsed. */
skipLease: number;
/** Tap calls dropped because the frame buffer allocation failed. */
skipAlloc: number;
/** Signed firmware results, or MIC_RET_NONE when not called yet. */
registerResult: number;
unregisterResult: number;
notifyResult: number;
/** Stock PCM slot table, read live: [slot 0, slot 1]. */
slots: [MicSlotState, MicSlotState];
leaseRemainingS: number;
};
const MIC_STATUS_DIAGNOSTICS_BYTES = 40;
function le16(data: Uint8Array, offset: number): number {
return data[offset]! | (data[offset + 1]! << 8);
}
@@ -210,6 +244,27 @@ export function decodeMicStatus(body: Uint8Array): MicStatus | null {
side: body[14] === 2 ? "left" : "right",
framesEmitted: le32(body, 15),
effectiveRateHz: le16(body, 19) * 100,
diagnostics: body.length >= MIC_STATUS_DIAGNOSTICS_BYTES ? decodeMicDiagnostics(body) : null,
};
}
const SLOT_STATES: MicSlotState[] = ["empty", "other", "tap", "other"];
function decodeMicDiagnostics(body: Uint8Array): MicStatusDiagnostics {
const s8 = (offset: number) => (body[offset]! >= 0x80 ? body[offset]! - 0x100 : body[offset]!);
const table = body[37]!;
return {
tapCalls: le32(body, 21),
tapLastBytes: le16(body, 25),
tapLastSlot: body[27]!,
skipIdle: le16(body, 28),
skipLease: le16(body, 30),
skipAlloc: le16(body, 32),
registerResult: s8(34),
unregisterResult: s8(35),
notifyResult: s8(36),
slots: [SLOT_STATES[table & 3]!, SLOT_STATES[(table >> 2) & 3]!],
leaseRemainingS: le16(body, 38),
};
}
@@ -260,25 +315,26 @@ export function decodeMicStreamFrame(data: Uint8Array): MicStreamFrame | null {
/**
* Split a raw PCM16 SM payload into per-channel Float32Arrays in [-1, 1].
* Channels arrive as CONCATENATED blocks (front then rear), matching the
* firmware's stereo callback, which dispatches two back-to-back extraction
* results rather than interleaving.
* Channels arrive INTERLEAVED: the firmware forwards the stock capture
* dispatch buffer, which the stock algo reads as ch0 = pcm[2i], ch1 =
* pcm[2i+1] (in samples). Channel 0 is assumed to be the front mic; that
* mapping is not yet confirmed on hardware.
*/
export function splitConcatenatedPcm16(frame: MicStreamFrame): Float32Array[] | null {
export function splitInterleavedPcm16(frame: MicStreamFrame): Float32Array[] | null {
if (frame.codec !== "raw" || frame.format !== "pcm16" || frame.channelCount < 1) {
return null;
}
const channels = frame.channelCount;
const bytesPerChannel = Math.floor(frame.payload.length / channels) & ~1;
const samplesPerChannel = bytesPerChannel / 2;
const frameBytes = channels * 2;
const samplesPerChannel = Math.floor(frame.payload.length / frameBytes);
if (samplesPerChannel <= 0) return null;
const out: Float32Array[] = [];
for (let channel = 0; channel < channels; channel++) {
const base = channel * bytesPerChannel;
const samples = new Float32Array(samplesPerChannel);
for (let index = 0; index < samplesPerChannel; index++) {
const lo = frame.payload[base + index * 2]!;
const hi = frame.payload[base + index * 2 + 1]!;
const offset = index * frameBytes + channel * 2;
const lo = frame.payload[offset]!;
const hi = frame.payload[offset + 1]!;
let value = lo | (hi << 8);
if (value >= 0x8000) value -= 0x10000;
samples[index] = value / 32768;
+2 -2
View File
@@ -23,7 +23,7 @@ import {
isStockAudioPacket,
micChannelKey,
micChannelLabel,
splitConcatenatedPcm16,
splitInterleavedPcm16,
templeActive,
type MicChannelKey,
type MicConfig,
@@ -586,7 +586,7 @@ class MicSession {
return;
}
const side: MicSide = arm === "L" ? "left" : "right";
const channels = splitConcatenatedPcm16(frame);
const channels = splitInterleavedPcm16(frame);
if (!channels || channels.length === 0) return;
// Meter every arriving channel — a host-disabled mic still shows its
+34 -34
View File
@@ -19,12 +19,12 @@ export type FirmwarePatchSet = {
export const CFW_PATCH_SET: FirmwarePatchSet = {
"base": "g2_2.3.0.24.bin",
"baseSha256": "187ccf2bcc5c17a212106e8a376745511e8289c4232b634a7ea94b9bf25a0979",
"outputSha256": "d7971b68add0c5187fec81d1cb1ef8cc12322db426a98cb445f1b3a274272817",
"outputSha256": "354f23f88a4aaf9c56cf62078bf13e814faa25b1fbb4dc69a7754110d6502386",
"patches": [
{
"offset": 1045797,
"old": "38b584b0",
"new": "5df3b9b8",
"new": "5cf33bbb",
"desc": "ring receiver entry: timestamped unfiltered reports under framebuffer lease"
},
{
@@ -36,49 +36,49 @@ export const CFW_PATCH_SET: FirmwarePatchSet = {
{
"offset": 1437923,
"old": "fef788ff",
"new": "fdf2c8fd",
"new": "fdf256f8",
"desc": "cfw_ancs_event"
},
{
"offset": 1437951,
"old": "fef7e8fb",
"new": "fdf202fb",
"new": "fcf284fd",
"desc": "cfw_ancs_open"
},
{
"offset": 1437969,
"old": "fef7e7fb",
"new": "fdf219fb",
"new": "fcf29ffd",
"desc": "cfw_ancs_close"
},
{
"offset": 1388387,
"old": "39694d00",
"new": "79637d00",
"new": "8d587d00",
"desc": "connection-bound ANCS EUS write callback"
},
{
"offset": 971497,
"old": "fff717f8",
"new": "6ff3e9f9",
"new": "6ef36bfc",
"desc": "bl faceclaw_input_bridge_received (ring relay and ordered private transport before worker pool)"
},
{
"offset": 971845,
"old": "fef769ff",
"new": "6ff33bf9",
"new": "6ef3bdfb",
"desc": "bl faceclaw_input_bridge_received (ring relay and ordered private transport before worker pool)"
},
{
"offset": 979579,
"old": "fdf74ef8",
"new": "6df320fa",
"new": "6cf3a2fc",
"desc": "bl faceclaw_input_bridge_received (ring relay and ordered private transport before worker pool)"
},
{
"offset": 1428895,
"old": "f8f76cfd",
"new": "fcf238fa",
"new": "fbf23cfe",
"desc": "bl cfw_receive_packet (private SID-f0 probe before TPL reassembly)"
},
{
@@ -108,145 +108,145 @@ export const CFW_PATCH_SET: FirmwarePatchSet = {
{
"offset": 1248487,
"old": "d4f712fb",
"new": "2af302fb",
"new": "29f308fd",
"desc": "bl settings_send_wrapper (append caps field 100)"
},
{
"offset": 1246183,
"old": "f5f7e2f8",
"new": "2af386fc",
"new": "29f3acfe",
"desc": "bl settings_decode_wrapper (Faceclaw lease field 101)"
},
{
"offset": 975219,
"old": "02f083fc",
"new": "6cf3f2fc",
"new": "6bf30eff",
"desc": "bl faceclaw_display_start @ 0x467d68 (fail-open wake takeover)"
},
{
"offset": 975411,
"old": "02f023fc",
"new": "6cf356fd",
"new": "6bf37cff",
"desc": "bl faceclaw_display_start_headup @ 0x467e28 (fail-open wake takeover)"
},
{
"offset": 975103,
"old": "07f085fb",
"new": "6ef3f2f9",
"new": "6df370fc",
"desc": "bl headup_gate (head-up sensor event -> EvenHub sys event 12 under lease)"
},
{
"offset": 830549,
"old": "19f03efb",
"new": "91f3a1fb",
"new": "90f31ffe",
"desc": "bl gesture_ring_press_mode (observe ring touch-down, preserve stock dispatch)"
},
{
"offset": 830795,
"old": "1ff071ff",
"new": "91f35afb",
"new": "90f3d8fd",
"desc": "bl gesture_press (owned event 9 before stock menu gate)"
},
{
"offset": 830887,
"old": "1ff043ff",
"new": "91f354fb",
"new": "90f3d2fd",
"desc": "bl gesture_short_long (Faceclaw lease gates event 11 vs stock Menu path)"
},
{
"offset": 831763,
"old": "1ff08dfd",
"new": "91f38af9",
"new": "90f308fc",
"desc": "bl gesture_release (owned event 10 before stock menu gate)"
},
{
"offset": 1570209,
"old": "7fb50600",
"new": "dbf293be",
"new": "dbf299b8",
"desc": "even_ai_display_ctrl entry -> conditional Faceclaw lease trampoline"
},
{
"offset": 1051545,
"old": "f6f787f9",
"new": "55f321f9",
"new": "55f3b5f8",
"desc": "bl display_copy_hook @ 0x47a78e (640x480 direct framebuffer)"
},
{
"offset": 1051861,
"old": "f6f7e9f8",
"new": "55f383f8",
"new": "55f317f8",
"desc": "bl display_copy_hook @ 0x47a8ca (640x480 direct framebuffer)"
},
{
"offset": 1262237,
"old": "d9f7d0fd",
"new": "26f343fb",
"new": "25f383fd",
"desc": "bl faceclaw_send_wear_event @ 0x4ade92 (outside onboarding)"
},
{
"offset": 1262337,
"old": "d9f79efd",
"new": "26f311fb",
"new": "25f351fd",
"desc": "bl faceclaw_send_wear_event @ 0x4adef6 (outside onboarding)"
},
{
"offset": 1309405,
"old": "67f02dff",
"new": "1af31bfc",
"new": "19f347fe",
"desc": "bl compass_decode_capture (sample-matched GAF diagnostics)"
},
{
"offset": 1307865,
"old": "fff7c9fb",
"new": "1af399ff",
"new": "1af3c9f9",
"desc": "bl compass_report_event (stock UI + heading with diagnostics over BLE)"
},
{
"offset": 1300937,
"old": "00210170",
"new": "1cf37ffc",
"new": "1bf3a7fe",
"desc": "bl compass_preserve_accuracy (retain magnetic calibration under Faceclaw framebuffer lease)"
},
{
"offset": 4537963,
"old": "",
"new": "b0b581ea00039b0701d0034637e0c2b110f0030315d000f1010ca2f1010e0023cc5c9e45c45403f1010404d0634413f003032346f4d121440319121b102a14d302e00346102a10d3d1f800c0d1f804e0cc688d68103adc609d60c3f804e043f810cb0f2a01f11001eed8042a07d300bf51f8045b043a032a43f8045bf8d8002a08bfb0bd0139013b11f8015f013a03f8015ff9d1b0bd00bf884218bf002a00d1704710b50b1a38bf431a934207d288420ad281ea00039b0726d0034669e0fff79fffbde81040704781ea000300eb020c9b0701eb020e42d15fea8c732bd0a0f1010c00bf8b1813f801ec8418531e04f801ec04d0624412f003021a46f2d101eb030e00eb030c17e0830736d0a2f1010c00f1010e002300bfcc5c9c45c45403f1010404d0734413f003032346f4d1121b2144031922e01346042b0ed3aef10402acf1040152f80449043b032b41f80449f8d802f1040e01f1040c1a4653b1aef10101acf1010300bf11f80149013a03f80149f9d1bde8104070470346042a06d351f8044b043a032a43f8044bf8d8002af0d00139013b00bf11f8014f013a03f8014ff9d1e6e700bf80b5002a18bf10f0030301d186460ce000f1010c864600bf013a0ef8011b04d01cf003030cf1010cf6d1102a11d35ffa81fc4ff001330cfb03fc00bf103acef80cc0cef808c0cef804c04ef810cb0f2af4d8042a0bd35ffa81fc4ff001330cfb03fc00bf043a032a4ef804cbfad8002a08bf80bdaef10103013a03f8011ffbd180bd00bf2de9f04fc5b04ff62c484ef66c694ff29c5b1446cff6ff78ccf2ff09c2f2290b4ff6ff75002701260d93cde90e0106e048f2a351c0f24501bff32f8f8847dbf800a0bff35f8fdbf800000321cff680710140b1f1005fbff35f8f03d10168494500f0ba804ff4757007f0b6fa002800f0078241464218463182f8d47382f8d57382f8d67382f8d77382f8d87382f8d97382f8da7382f8db7382f8dc7382f8dd7382f8de7382f8df7382f8e07382f8e17382f8e27382f8e37382f8e47382f8e57382f8e67382f8e77382f8e87382f8e97382f8ea7382f8eb7382f8ec7382f8ed7382f8ee7382f8ef7382f8f07382f8f17382f8f27382f8f37382f8f47382f8f57382f8f67382f8f77382f8f87382f8f97382f8fa7382f8fb7382f8fc7382f8fd7382f8fe7382f8ff7382f8007482f8017482f8027482f8037482f8047482f8057482f8067482f8077482f8087482f8097482f80a7482f80b7482f80c7482f80d7482f80e7482f80f7482f8107482f8117482f8127482f8137482f8147482f8157482f8167482f8177482f8187482f819747ff470afc0f8009086728583c583058445848584c584058545858585c585058645868586c586058745875be8001f51457ff436afbff35f8f4be8000121b15be8001f5145f8d02be70f99012641ea04410024c0f8701206e048f2a351c0f24501bff32f8f8847dbf80070bff35f8fdbf800000321cff680710140b1f1005fbff35f8f03d10168494500f0ba804ff4757007f0daf9002800f02b8141464218463182f8d44382f8d54382f8d64382f8d74382f8d84382f8d94382f8da4382f8db4382f8dc4382f8dd4382f8de4382f8df4382f8e04382f8e14382f8e24382f8e34382f8e44382f8e54382f8e64382f8e74382f8e84382f8e94382f8ea4382f8eb4382f8ec4382f8ed4382f8ee4382f8ef4382f8f04382f8f14382f8f24382f8f34382f8f44382f8f54382f8f64382f8f74382f8f84382f8f94382f8fa4382f8fb4382f8fc4382f8fd4382f8fe4382f8ff4382f8004482f8014482f8024482f8034482f8044482f8054482f8064482f8074482f8084482f8094482f80a4482f80b4482f80c4482f80d4482f80e4482f80f4482f8104482f8114482f8124482f8134482f8144482f8154482f8164482f8174482f8184482f819447ff470afc0f8009086728583c583058445848584c584058545858585c585058645868586c586058745875be8001fb9427ff436afbff35f8f4be8000121b15be8001fb942f8d02be7d0f874a242f6f766c0f24406baf1000fbff35f8f4ad006f19a0250464ff0ff314ff0ff379047002840f09d810f98002861d00e980778a7f1170002285bd80f980c2840f086810e9e7078b178084300f08081f0780138022800f27b81b279f379747a42ea0322337a307942ea0342717942ea0463b27af67a192f42ea062200f01082172f40f02682002a40f06281fe2800f07786812800f06a86002840f07d8651ea030040f0558100f0c7be04460020b047e8b104f51d7151e800afbaf1000f0cd1bff35f8f00bf41e80002002a00f0f58051e800afbaf1000ff5d00021bff32f8f0029bff35f8f00f0e480824692e74ff0ff3737e147f6df40c0f24f008047002701241197adf8487006e048f2a351c0f24501bff32f8f8847dbf80060bff35f8fdbf800000321cff680710140b1f1005fbff35f8f03d10168494500f0c1804ff4757007f080f8002800f0b98041464218463182f8d47382f8d57382f8d67382f8d77382f8d87382f8d97382f8da7382f8db7382f8dc7382f8dd7382f8de7382f8df7382f8e07382f8e17382f8e27382f8e37382f8e47382f8e57382f8e67382f8e77382f8e87382f8e97382f8ea7382f8eb7382f8ec7382f8ed7382f8ee7382f8ef7382f8f07382f8f17382f8f27382f8f37382f8f47382f8f57382f8f67382f8f77382f8f87382f8f97382f8fa7382f8fb7382f8fc7382f8fd7382f8fe7382f8ff7382f8007482f8017482f8027482f8037482f8047482f8057482f8067482f8077482f8087482f8097482f80a7482f80b7482f80c7482f80d7482f80e7482f80f7482f8107482f8117482f8127482f8137482f8147482f8157482f8167482f8177482f8187482f819747ff470afc0f8009084728583c583058445848584c584058545858585c585058645868586c586058745875be8001fb1427ff436afbff35f8f4be8000159b15be8001fb142f8d02be706f5a9718847ace6012110e700200e9999b10f9a8ab1097801f07f02b2f10b0618bf0126b2f11c0718bf01278a2924dc0b2926d01c2924d03fe00d9010a804f0a7fa4ff0ff3701249df84900002810a818bf4ff0ff3704f0e3fb054644b99df8480028b94af21f30c0f247004c3080470d98002818bfc56013e09c2901d08b291bd160b14af21f31c0f247010446884794f8641119b994f80413204669b14ff0ff3742f6f760c0f2440000f5807150468847384645b0bde8f08f0d9010a806ea070404f065fa0e98017801f07f061e2e04d10f9908f086fb0746b5e71d2e01f07e0018bf1a2811d1dbf800100322cff680720a40b2f1005fbff35f8f0ad10a684a4504bf002281f8812303e0a6f115010129e9d90f2e0adc092e00f37381052e00f0ef82072e40f01c8300279ae0142e00f32182102e00f0eb82112e40f011830f98022841f0d5800e9908781Line truncated
"new": "b0b581ea00039b0701d0034637e0c2b110f0030315d000f1010ca2f1010e0023cc5c9e45c45403f1010404d0634413f003032346f4d121440319121b102a14d302e00346102a10d3d1f800c0d1f804e0cc688d68103adc609d60c3f804e043f810cb0f2a01f11001eed8042a07d300bf51f8045b043a032a43f8045bf8d8002a08bfb0bd0139013b11f8015f013a03f8015ff9d1b0bd00bf884218bf002a00d1704710b50b1a38bf431a934207d288420ad281ea00039b0726d0034669e0fff79fffbde81040704781ea000300eb020c9b0701eb020e42d15fea8c732bd0a0f1010c00bf8b1813f801ec8418531e04f801ec04d0624412f003021a46f2d101eb030e00eb030c17e0830736d0a2f1010c00f1010e002300bfcc5c9c45c45403f1010404d0734413f003032346f4d1121b2144031922e01346042b0ed3aef10402acf1040152f80449043b032b41f80449f8d802f1040e01f1040c1a4653b1aef10101acf1010300bf11f80149013a03f80149f9d1bde8104070470346042a06d351f8044b043a032a43f8044bf8d8002af0d00139013b00bf11f8014f013a03f8014ff9d1e6e700bf80b5002a18bf10f0030301d186460ce000f1010c864600bf013a0ef8011b04d01cf003030cf1010cf6d1102a11d35ffa81fc4ff001330cfb03fc00bf103acef80cc0cef808c0cef804c04ef810cb0f2af4d8042a0bd35ffa81fc4ff001330cfb03fc00bf043a032a4ef804cbfad8002a08bf80bdaef10103013a03f8011ffbd180bd00bf2de9f04fc7b04ff29c5bc2f2290b4ff6ff754ff47d7400274ff0010847f67f7a7f261191cde90f3007e000bf48f2a351c0f24501bff32f8f8847dbf80090bff35f8fdbf800000321cff680710140b1f1005fbff35f8f0bd101684ef66c62ccf2ff02914204d14068b0f57d7f00f096804ff47d7006f0c8fc002800f0c8814ff60c41cff6ff7100bf42182e3182f8f47382f8f57382f8f67382f8f77382f8f87382f8f97382f8fa7382f8fb7382f8fc7382f8fd7382f8fe7382f8ff7382f8007482f8017482f8027482f8037482f8047482f8057482f8067482f8077482f8087482f8097482f80a7482f80b7482f80c7482f80d7482f80e7482f80f7482f8107482f8117482f8127482f8137482f8147482f8157482f8167482f8177482f8187482f8197482f81a7482f81b7482f81c7482f81d7482f81e7482f81f7482f8207482f82174a0d14ef66c61ccf2ff010160446080f80e80a0f8c4a180f8c361058445848584c584058545858585c585058645868586c586058745878587c5875be8001f49457ff452afbff35f8f4be8000121b15be8001f4945f8d047e74ff47d7800274ff0010a47f67f797f2607e000bf48f2a351c0f24501bff32f8f8847dbf80040bff35f8fdbf800000321cff680710140b1f1005fbff35f8f0bd101684ef66c62ccf2ff02914204d14168b1f57d7f00f098804ff47d7006f006fc002800f006814ff60c41cff6ff7100bf42182e3182f8f47382f8f57382f8f67382f8f77382f8f87382f8f97382f8fa7382f8fb7382f8fc7382f8fd7382f8fe7382f8ff7382f8007482f8017482f8027482f8037482f8047482f8057482f8067482f8077482f8087482f8097482f80a7482f80b7482f80c7482f80d7482f80e7482f80f7482f8107482f8117482f8127482f8137482f8147482f8157482f8167482f8177482f8187482f8197482f81a7482f81b7482f81c7482f81d7482f81e7482f81f7482f8207482f82174a0d14ef66c61ccf2ff010160c0f8048080f80ea0a0f8c49180f8c361058445848584c584058545858585c585058645868586c586058745878587c5875be8001fa1427ff451afbff35f8f00bf4be8000121b15be8001fa142f8d045e7d0f8944242f6f76ac0f2440a002cbff35f8f4bd00af19a0220464ff0ff314ff0ff379047002840f09081119800285fd010980778a7f11700022859d811980c2840f07d81109b58789978084300f07781d8780138022800f272819a79d9795e7a42ea0122197a187942ea014242ea06619a7ade7a93f805c0192f42ea062200f0c981172f40f0df81002a40f05881fe2800f03e86812800f03086002840f045865cea010040f04b8100f092be04460020d047d0b104f5257151e8004f5cb9bff35f8f00bf41e80002002a00f0e38051e8004f002cf6d00021bff32f8f0029bff35f8f00f0d380044694e74ff0ff372ce147f6df40c0f24f000e94804700274ff47d784ff0010947f67f744ff07f0a1397adf8507007e000bf48f2a351c0f24501bff32f8f8847dbf80060bff35f8fdbf800000321cff680710140b1f1005fbff35f8f0bd101684ef66c62ccf2ff02914204d14168b1f57d7f00f09f804ff47d7006f0c0fa002800f097804ff60c41cff6ff7100bf42182e3182f8f47382f8f57382f8f67382f8f77382f8f87382f8f97382f8fa7382f8fb7382f8fc7382f8fd7382f8fe7382f8ff7382f8007482f8017482f8027482f8037482f8047482f8057482f8067482f8077482f8087482f8097482f80a7482f80b7482f80c7482f80d7482f80e7482f80f7482f8107482f8117482f8127482f8137482f8147482f8157482f8167482f8177482f8187482f8197482f81a7482f81b7482f81c7482f81d7482f81e7482f81f7482f8207482f82174a0d14ef66c61ccf2ff010160c0f8048080f80e90a0f8c44180f8c3a1058445848584c584058545858585c585058645868586c586058745878587c5875be8001fb1427ff451afbff35f8f00bf4be8000159b15be8001fb142f8d045e70af5a9718847bfe6012121e70020109f42f6f76ac0f2440a9fb1119989b1397801f07f02b2f10b0418bf0124b2f11c0618bf01268a2928dc0b292ad01c2928d041e00f9012a803f037ff4ff0ff374ff001080e9c9df85100002812a818bf4ff0ff3704f04ff8b8f1000f054608d19df8500028b94af21f30c0f247004c3080470f98002818bf056115e09c2901d08b2919d168b14af21f31c0f247010746884797f8681121b997f824133846109f51b10e9c4ff0ff370af5807120468847384647b0bde8f08f0f9012a804ea060803f0f3fe387800f07f041e2c05d11199384607f048fe0746b4e71d2c00f07e0018bf1a2819d1dbf800100322cff680720a40b2f1005fbff35f8f12d10a684ef66c63ccf2ff039a420bd14a68b2f57d7f04bf002281f8a12303e0a4f115010129e1d90f2c12dc092c00f31781052c00f0c982072c40f0c183cdf8348000274ff0010947f67f764ff07f0852e0142c00f3ea81102c00f08083112c40f0ae8311980e9c022806d13878112803d17878002800f09a864ff0ff3765e7bcf1000f4ff0ff3708bf00298fd11328bff48daf102a4ff0000138bf01211228b2fa82f218bf510901297ff480af00f0c5bc182f4ff0ff3708bfbcf1000f7ff476af00297ff473af352800f34284002800f09c84012800f0aLine truncated
"desc": "append injected blobs to main-app payload"
},
{
"offset": 779123,
"old": "805a3900",
"new": "34733a00",
"new": "186e3a00",
"desc": "main-app subheader payload size (ps)"
},
{
"offset": 152,
"old": "005b3900",
"new": "b4733a00",
"new": "986e3a00",
"desc": "main-app TOC entry size (ps + 128)"
},
{
"offset": 779243,
"old": "805a3904",
"new": "34733a04",
"new": "186e3a04",
"desc": "main-app preamble length (low 24 bits)"
},
{
"offset": 779247,
"old": "effdbcea",
"new": "85d22153",
"new": "cbb2ae81",
"desc": "[5] ota/s200_firmware_ota.bin preamble crc32"
},
{
"offset": 156,
"old": "c0c6322a",
"new": "817b5baa",
"new": "35dc5a11",
"desc": "[5] ota/s200_firmware_ota.bin component crc32c (TOC)"
},
{
"offset": 779127,
"old": "c0c6322a",
"new": "817b5baa",
"new": "35dc5a11",
"desc": "[5] ota/s200_firmware_ota.bin component crc32c (subheader)"
}
]
+77
View File
@@ -0,0 +1,77 @@
/**
* Passive view of the glasses' audio traffic, for diagnostics. Nothing here
* sends anything to the glasses: the listeners only observe packets and CFW
* mic_control status records that arrive because something else (a stock
* EvenHub mic enable, or a CFW mic_control CONFIGURE) turned the mics on.
*/
import { toUint8Array } from "../util/array-util";
declare const com: any;
declare const global: any;
export type AudioArm = "L" | "R" | "?";
function activeCommunicator(): any {
if (!global.isAndroid) return null;
try {
return com.faceclaw.app.FaceclawBleCommunicator.getActive();
} catch {
return null;
}
}
/** The communicator instance listeners attach to; changes across reconnects. */
export function audioMonitorCommunicator(): any {
return activeCommunicator();
}
/**
* True while Faceclaw itself is forwarding glasses audio to one of its apps
* (stock mic enable or CFW mic_control forwarding). Packets can also arrive
* with this false, e.g. when capture was armed and Faceclaw's side stopped.
*/
export function isFaceclawAudioCaptureActive(): boolean {
try {
return Boolean(activeCommunicator()?.isAudioCaptureActive());
} catch {
return false;
}
}
/**
* Subscribe `communicator` to every audio-characteristic packet and every
* field-104 mic status record body (undecoded). Delivered on the main thread.
* Returns an unsubscribe function.
*/
export function addAudioMonitorListeners(
communicator: any,
onPacket: (data: Uint8Array, arm: AudioArm, arrivalMs: number) => void,
onStatus: (body: Uint8Array, arm: AudioArm) => void,
): () => void {
if (!communicator) return () => {};
const packetProxy = new com.faceclaw.app.FaceclawAudioPacketListener({
onAudioPacket: (data: any, arm: string, arrivalMs: number) => {
onPacket(toUint8Array(data), toArm(arm), Number(arrivalMs));
},
});
const statusProxy = new com.faceclaw.app.FaceclawMicStatusListener({
onMicStatus: (body: any, arm: string) => {
onStatus(toUint8Array(body), toArm(arm));
},
});
communicator.addAudioMonitorListener(packetProxy);
communicator.addMicStatusListener(statusProxy);
return () => {
try {
communicator.removeAudioMonitorListener(packetProxy);
communicator.removeMicStatusListener(statusProxy);
} catch {
// The communicator may have been torn down during a disconnect.
}
};
}
function toArm(arm: string): AudioArm {
const value = String(arm);
return value === "L" || value === "R" ? value : "?";
}
@@ -517,6 +517,22 @@ internal fun GlassesSessionCore.emitMicStatus(body: ByteArray, address: String)
}
}
internal fun GlassesSessionCore.emitAudioMonitorPacket(data: ByteArray, arm: String, arrivalMs: Long) {
if (audioMonitorListeners.isEmpty()) {
return
}
val copy = data.copyOf()
host.postToMain {
for (monitorListener in audioMonitorListeners) {
try {
monitorListener.onAudioPacket(copy, arm, arrivalMs)
} catch (t: Throwable) {
logWarn("audio monitor listener failed", t)
}
}
}
}
internal fun GlassesSessionCore.emitRingEvent(event: G2Event, frameId: Int) {
val current = listener
if (current == null) {
@@ -124,6 +124,7 @@ class GlassesSessionCore(
internal val compassSubscriptions = CopyOnWriteList<CompassSubscription>(platform)
internal val ambientLightListeners = CopyOnWriteList<FaceclawAmbientLightListener>(platform)
internal val micStatusListeners = CopyOnWriteList<FaceclawMicStatusListener>(platform)
internal val audioMonitorListeners = CopyOnWriteList<FaceclawAudioPacketListener>(platform)
@Volatile internal var running = false
@Volatile internal var userDisconnectRequested = false
// Set when a connect attempt failed while an arm's OS bond is gone:
@@ -838,6 +839,24 @@ class GlassesSessionCore(
}
}
/**
* Observe every packet on the audio (render) characteristic, on the main
* thread, whether or not any capture is active. Purely passive: sends
* nothing to the glasses and does not affect the capture listener, so it
* shows what another app's mic activation actually produces.
*/
fun addAudioMonitorListener(listener: FaceclawAudioPacketListener?) {
if (listener != null) {
audioMonitorListeners.add(listener)
}
}
fun removeAudioMonitorListener(listener: FaceclawAudioPacketListener?) {
if (listener != null) {
audioMonitorListeners.remove(listener)
}
}
/**
* Queue a CFW mic_control record (['M','C',ver,op,...]) as a settings
* field-103 write to both temples, or to a single one. Fire-and-forget:
@@ -1601,10 +1620,11 @@ class GlassesSessionCore(
lastIncomingAtMs = arrivalMs
listenerToCall = if (audioCaptureActive) audioPacketListener else null
}
val arm = if (address.equals(leftAddress, ignoreCase = true)) "L" else if (address.equals(rightAddress, ignoreCase = true)) "R" else "?"
emitAudioMonitorPacket(data, arm, arrivalMs)
if (listenerToCall == null) {
return
}
val arm = if (address.equals(leftAddress, ignoreCase = true)) "L" else if (address.equals(rightAddress, ignoreCase = true)) "R" else "?"
try {
listenerToCall.onAudioPacket(data.copyOf(), arm, arrivalMs)
} catch (t: Throwable) {