First-pass G2 microphone streaming support

This commit is contained in:
James Babcock committed 2026-04-30 20:00:16 -07:00
1 parent c28b321705
commit 9996b09e9c
13 files changed
+493 -113

No files matched your search

+2
View File
@@ -5,3 +5,5 @@ to keep my room clean, so to speak.
Voice wake-word detection uses sherpa-onnx: https://github.com/k2-fsa/sherpa-onnx/
and the sherpa-onnx KWS Zipformer GigaSpeech model published by the k2-fsa/sherpa
community.
G2 microphone LC3 decoding uses Google's liblc3: https://github.com/google/liblc3
@@ -33,6 +33,7 @@ public class BleProtocol {
public static final int FLAG_REQUEST = 0x20;
public static final int FLAG_NOTIFY = 0x01;
public static final int FLAG_NOTIFY_ALT = 0x06;
public static final int CMD_AUDIO_CONTROL = 15;
public static final int EVENT_CLICK = 0;
public static final int EVENT_SCROLL_TOP = 1;
@@ -112,6 +113,10 @@ public class BleProtocol {
return wrapEvenHub(12, magic, 14, encodeVarintField(1, 0));
}
public static byte[] buildAudioControl(int magic, boolean enable) {
return wrapEvenHub(CMD_AUDIO_CONTROL, magic, 18, encodeVarintField(1, enable ? 1 : 0));
}
public static byte[] buildShutdown(int magic, int exitMode) {
return wrapEvenHub(9, magic, 11, encodeVarintField(1, exitMode));
}
@@ -0,0 +1,5 @@
package com.faceclaw.app;
public interface FaceclawAudioPacketListener {
void onAudioPacket(byte[] data, String arm, long arrivalMs);
}
@@ -102,6 +102,7 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable {
private long lastSessionReadyAtMs;
private long lastEvenAppConflictAtMs;
private int consecutiveAckTimeouts;
private int lastAudioControlAckMagic = 0;
private final BleMagicPool magicPool = new BleMagicPool();
private int nextTransportSeq = 0x40;
@@ -111,6 +112,8 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable {
private long lastShutdownExitAtMs = 0;
private int headsetBattery = -1;
private int headsetCharging = -1;
private boolean audioCaptureActive;
private volatile FaceclawAudioPacketListener audioPacketListener;
private String displayedFingerprint = "";
private byte[][] displayedTileBmps = emptyTileSet();
@@ -158,6 +161,8 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable {
synchronized (lock) {
userDisconnectRequested = true;
running = false;
audioCaptureActive = false;
audioPacketListener = null;
phase = "disconnecting";
status = "Disconnecting...";
lock.notifyAll();
@@ -190,6 +195,51 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable {
disconnect();
}
public boolean startG2AudioCapture(FaceclawAudioPacketListener listener) {
if (listener == null) {
throw new IllegalArgumentException("listener is required");
}
int magic;
synchronized (lock) {
if (!running || !sessionReady) {
logLine("skip G2 mic enable; session not ready");
return false;
}
audioPacketListener = listener;
magic = nextMagic();
pendingMessages.addFirst(createAudioControlMessageLocked(true, magic));
logLine("queue G2 mic enable");
lock.notifyAll();
}
interruptibleSleep.interrupt();
return waitForAudioControlAck(magic, "enable");
}
public void stopG2AudioCapture() {
int magic = 0;
synchronized (lock) {
audioPacketListener = null;
audioCaptureActive = false;
clearMessagesOfKindLocked("audio-control");
if (running && sessionReady) {
magic = nextMagic();
pendingMessages.addFirst(createAudioControlMessageLocked(false, magic));
logLine("queue G2 mic disable");
lock.notifyAll();
}
}
interruptibleSleep.interrupt();
if (magic != 0) {
waitForAudioControlAck(magic, "disable");
}
}
public boolean isSessionReady() {
synchronized (lock) {
return running && sessionReady;
}
}
public void submitDashboardImage(List<byte[]> tileBmps, String fingerprint) {
submitDashboardImage(tileBmps, fingerprint, false, -1);
@@ -365,6 +415,33 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable {
}
}
private boolean waitForAudioControlAck(int magic, String operation) {
long deadline = SystemClock.elapsedRealtime() + ACK_TIMEOUT_MS + WRITE_TIMEOUT_MS + 500;
synchronized (lock) {
while (running && sessionReady && lastAudioControlAckMagic != magic && hasPendingOrInflightMagicLocked(magic)) {
long remaining = deadline - SystemClock.elapsedRealtime();
if (remaining <= 0) {
break;
}
try {
lock.wait(Math.min(remaining, 100));
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
break;
}
}
boolean acked = lastAudioControlAckMagic == magic;
if (!acked) {
if ("enable".equals(operation)) {
audioPacketListener = null;
audioCaptureActive = false;
}
logLine("G2 mic " + operation + " ack timeout");
}
return acked;
}
}
@Override
public void run() {
@@ -404,6 +481,10 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable {
return;
}
String uuid = characteristicUuid.toLowerCase(Locale.US);
if (BleProtocol.RENDER_NOTIFY_UUID.equals(uuid)) {
handleRenderNotification(address, data);
return;
}
if (!BleProtocol.NOTIFY_CHAR_UUID.equals(uuid)) {
return;
}
@@ -430,6 +511,24 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable {
}
}
private void handleRenderNotification(String address, byte[] data) {
FaceclawAudioPacketListener listenerToCall;
long arrivalMs = SystemClock.elapsedRealtime();
synchronized (lock) {
lastIncomingAtMs = arrivalMs;
listenerToCall = audioCaptureActive ? audioPacketListener : null;
}
if (listenerToCall == null) {
return;
}
String arm = address.equalsIgnoreCase(leftAddress) ? "L" : address.equalsIgnoreCase(rightAddress) ? "R" : "?";
try {
listenerToCall.onAudioPacket(Arrays.copyOf(data, data.length), arm, arrivalMs);
} catch (Throwable t) {
logLine("G2 mic packet listener failed: " + safeMessage(t));
}
}
@Override
public void onConnectionStateChange(String address, boolean connected) {
synchronized (lock) {
@@ -446,6 +545,8 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable {
fixedLayoutCreated = false;
warmedUp = false;
startupProbePending = false;
audioCaptureActive = false;
audioPacketListener = null;
clearAllMessagesLocked("connection lost");
displayedFingerprint = "";
displayedTileBmps = emptyTileSet();
@@ -465,7 +566,7 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable {
setState("connecting", "Connecting to the glasses...");
try {
connectArm(rightAddress, true);
connectArm(leftAddress, false);
connectArm(leftAddress, true);
if (!sleepDuringConnectSettling(800)) {
return;
}
@@ -486,6 +587,8 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable {
imageRetryAfterMs = 0;
lastHeartbeatQueuedAtMs = 0;
consecutiveAckTimeouts = 0;
lastAudioControlAckMagic = 0;
audioCaptureActive = false;
}
setState("connected", "Connected.");
logLine("session ready");
@@ -785,6 +888,13 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable {
if ("heartbeat".equals(message.kind)) {
return;
}
if ("audio-control".equals(message.kind)) {
lastAudioControlAckMagic = message.magic;
audioCaptureActive = message.label != null && message.label.contains("enable");
logLine(audioCaptureActive ? "G2 mic enabled" : "G2 mic disabled");
lock.notifyAll();
return;
}
if ("battery".equals(message.kind)) {
BleProtocol.BatterySnapshot snapshot = BleProtocol.parseSettingsBattery(message.ackPayload);
if (snapshot != null) {
@@ -1035,6 +1145,25 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable {
);
}
private OutboundMessage createAudioControlMessageLocked(boolean enable, int magic) {
return new OutboundMessage(
"audio-control",
enable ? "G2 mic enable" : "G2 mic disable",
BleProtocol.SID_EVENHUB,
magic,
BleProtocol.framePb(
BleProtocol.buildAudioControl(magic, enable),
BleProtocol.SID_EVENHUB,
BleProtocol.FLAG_REQUEST,
nextTransportSeq++
),
ACK_TIMEOUT_MS,
displayedFingerprint,
-1,
null
);
}
private boolean shouldPollBatteryLocked(long now) {
return !shutdownRequested
&& sessionReady
@@ -1265,6 +1394,9 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable {
lastHeartbeatQueuedAtMs = 0;
lastSessionReadyAtMs = 0;
consecutiveAckTimeouts = 0;
lastAudioControlAckMagic = 0;
audioCaptureActive = false;
audioPacketListener = null;
displayedFingerprint = "";
displayedTileBmps = emptyTileSet();
}
@@ -107,6 +107,7 @@ public class FaceclawForegroundService extends Service {
private int foregroundServiceType() {
int type = ServiceInfo.FOREGROUND_SERVICE_TYPE_CONNECTED_DEVICE;
// TODO: Make this depend on which audio path (G2 vs phone) is selected
if (hasRecordAudioPermission()) {
type |= ServiceInfo.FOREGROUND_SERVICE_TYPE_MICROPHONE;
}
@@ -0,0 +1,97 @@
package com.faceclaw.app;
import android.util.Log;
public class FaceclawLc3Decoder {
private static final String TAG = "FaceclawLc3";
public static final int SAMPLE_RATE = 16000;
public static final int FRAME_US = 10000;
public static final int FRAME_BYTES = 40;
public static final int FRAMES_PER_PACKET = 5;
public static final int PACKET_BYTES = 205;
public static final int COUNTER_OFFSET = 204;
public static final int SAMPLES_PER_FRAME = 160;
public static final int SAMPLES_PER_PACKET = FRAMES_PER_PACKET * SAMPLES_PER_FRAME;
static {
System.loadLibrary("faceclaw_lc3");
}
private long nativeHandle;
private int lastCounter = -1;
private long realPackets;
private long duplicatePackets;
private long missingPackets;
private long decodeErrors;
public FaceclawLc3Decoder() {
nativeHandle = nativeCreate(FRAME_US, SAMPLE_RATE);
if (nativeHandle == 0) {
throw new IllegalStateException("Could not create LC3 decoder");
}
}
public synchronized int decodePacket(byte[] packet, short[] pcmOut) {
if (nativeHandle == 0) {
throw new IllegalStateException("LC3 decoder is closed");
}
if (packet == null || packet.length != PACKET_BYTES) {
decodeErrors++;
Log.w(TAG, "unexpected G2 audio packet length=" + (packet == null ? -1 : packet.length));
return 0;
}
if (pcmOut == null || pcmOut.length < SAMPLES_PER_PACKET) {
throw new IllegalArgumentException("pcmOut must hold " + SAMPLES_PER_PACKET + " samples");
}
int counter = packet[COUNTER_OFFSET] & 0xff;
if (lastCounter >= 0) {
int gap = (counter - lastCounter) & 0xff;
if (gap == 0) {
duplicatePackets++;
return 0;
}
int missing = gap - 1;
if (missing > 0) {
missingPackets += missing;
}
}
int decoded = nativeDecodePacket(nativeHandle, packet, pcmOut);
if (decoded <= 0) {
decodeErrors++;
return 0;
}
realPackets++;
lastCounter = counter;
return decoded;
}
public synchronized long getRealPackets() {
return realPackets;
}
public synchronized long getDuplicatePackets() {
return duplicatePackets;
}
public synchronized long getMissingPackets() {
return missingPackets;
}
public synchronized long getDecodeErrors() {
return decodeErrors;
}
public synchronized void close() {
long handle = nativeHandle;
nativeHandle = 0;
if (handle != 0) {
nativeDestroy(handle);
}
}
private static native long nativeCreate(int frameUs, int sampleRate);
private static native int nativeDecodePacket(long handle, byte[] packet, short[] pcmOut);
private static native void nativeDestroy(long handle);
}
@@ -1,25 +1,12 @@
package com.faceclaw.app;
import android.Manifest;
import android.bluetooth.BluetoothAdapter;
import android.bluetooth.BluetoothDevice;
import android.bluetooth.BluetoothManager;
import android.bluetooth.BluetoothProfile;
import android.content.Context;
import android.content.pm.PackageManager;
import android.content.res.AssetManager;
import android.media.AudioDeviceInfo;
import android.media.AudioFormat;
import android.media.AudioManager;
import android.media.AudioRecord;
import android.media.MediaRecorder;
import android.os.Build;
import android.os.Handler;
import android.os.Looper;
import android.os.SystemClock;
import android.util.Log;
import androidx.core.content.ContextCompat;
import com.k2fsa.sherpa.onnx.FeatureConfig;
import com.k2fsa.sherpa.onnx.KeywordSpotter;
import com.k2fsa.sherpa.onnx.KeywordSpotterConfig;
@@ -32,12 +19,16 @@ import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.util.Set;
import java.util.ArrayDeque;
public class FaceclawVoiceController {
private static final String TAG = "FaceclawVoice";
private static final int SAMPLE_RATE = 16000;
private static final int FEATURE_DIM = 80;
private static final int MAX_AUDIO_QUEUE_PACKETS = 80;
private static final int EXPECTED_PACKET_INTERVAL_MS = 50;
private static final int LATE_PACKET_INTERVAL_MS = 90;
private static final int STATS_INTERVAL_MS = 5_000;
private static final String ASSET_ROOT = "faceclaw-voice";
private static final String MODEL_DIR = "sherpa-onnx-kws-zipformer-gigaspeech-3.3M-2024-01-01";
private static final String[] MODEL_FILES = {
@@ -51,12 +42,23 @@ public class FaceclawVoiceController {
private final Context appContext;
private final Handler mainHandler = new Handler(Looper.getMainLooper());
private final Object lock = new Object();
private final Object audioQueueLock = new Object();
private final ArrayDeque<AudioPacket> audioQueue = new ArrayDeque<>();
private volatile FaceclawVoiceControllerListener listener;
private volatile FaceclawBleCommunicator communicator;
private Thread workerThread;
private volatile boolean started;
private AudioRecord audioRecord;
private KeywordSpotter keywordSpotter;
private OnlineStream stream;
private FaceclawLc3Decoder lc3Decoder;
private long queuedPackets;
private long queueDroppedPackets;
private long decodedSamples;
private long latePackets;
private long wrongArmPackets;
private long lastPacketArrivalMs;
private long maxInterPacketMs;
private long lastStatsAtMs;
public FaceclawVoiceController(Context context) {
this.appContext = context.getApplicationContext();
@@ -66,18 +68,18 @@ public class FaceclawVoiceController {
this.listener = listener;
}
public void setCommunicator(FaceclawBleCommunicator communicator) {
this.communicator = communicator;
}
public void start() {
synchronized (lock) {
if (started) {
emitStatus("Voice control is already listening.");
return;
}
if (!hasRecordAudioPermission()) {
emitStatus("Voice control needs microphone permission.");
return;
}
if (!hasBluetoothHeadset()) {
emitStatus("Voice control needs a paired Bluetooth headset.");
if (communicator == null || !communicator.isSessionReady()) {
emitStatus("Voice control needs an active G2 connection.");
return;
}
started = true;
@@ -95,7 +97,10 @@ public class FaceclawVoiceController {
started = false;
threadToJoin = workerThread;
}
stopAudio();
stopG2Audio();
synchronized (audioQueueLock) {
audioQueueLock.notifyAll();
}
if (threadToJoin != null) {
threadToJoin.interrupt();
}
@@ -111,19 +116,21 @@ public class FaceclawVoiceController {
File modelDir = installModelFiles();
keywordSpotter = new KeywordSpotter(buildConfig(modelDir));
stream = keywordSpotter.createStream();
if (!startAudio()) {
emitStatus("Voice control could not start microphone input.");
lc3Decoder = new FaceclawLc3Decoder();
if (!startG2Audio()) {
emitStatus("Voice control could not start G2 microphone input.");
return;
}
emitStatus("Voice control listening for \"screen on\".");
processAudio();
emitStatus("Voice control listening for \"screen on\" from the G2 mic.");
processG2Audio();
} catch (Throwable error) {
Log.e(TAG, "Voice control failed", error);
emitStatus("Voice control failed: " + error.getMessage());
} finally {
stopG2Audio();
releaseSherpa();
stopAudio();
releaseLc3();
synchronized (lock) {
started = false;
workerThread = null;
@@ -183,53 +190,42 @@ public class FaceclawVoiceController {
}
}
private boolean startAudio() {
routeToBluetoothHeadset();
int minBufferBytes = AudioRecord.getMinBufferSize(
SAMPLE_RATE,
AudioFormat.CHANNEL_IN_MONO,
AudioFormat.ENCODING_PCM_16BIT
);
if (minBufferBytes <= 0) {
private boolean startG2Audio() {
FaceclawBleCommunicator currentCommunicator = communicator;
if (currentCommunicator == null) {
return false;
}
if (!hasRecordAudioPermission()) {
return false;
resetAudioStats();
synchronized (audioQueueLock) {
audioQueue.clear();
}
audioRecord = new AudioRecord(
MediaRecorder.AudioSource.VOICE_RECOGNITION,
SAMPLE_RATE,
AudioFormat.CHANNEL_IN_MONO,
AudioFormat.ENCODING_PCM_16BIT,
minBufferBytes * 2
);
if (audioRecord.getState() != AudioRecord.STATE_INITIALIZED) {
audioRecord.release();
audioRecord = null;
return false;
}
audioRecord.startRecording();
return true;
return currentCommunicator.startG2AudioCapture(this::queueAudioPacket);
}
private void processAudio() {
short[] buffer = new short[SAMPLE_RATE / 10];
private void processG2Audio() {
short[] pcm = new short[FaceclawLc3Decoder.SAMPLES_PER_PACKET];
while (started && !Thread.currentThread().isInterrupted()) {
AudioRecord currentAudio = audioRecord;
OnlineStream currentStream = stream;
KeywordSpotter currentSpotter = keywordSpotter;
if (currentAudio == null || currentStream == null || currentSpotter == null) {
FaceclawLc3Decoder currentDecoder = lc3Decoder;
if (currentStream == null || currentSpotter == null || currentDecoder == null) {
return;
}
int count = currentAudio.read(buffer, 0, buffer.length);
if (count <= 0) {
AudioPacket packet = takeAudioPacket();
if (packet == null) {
continue;
}
int count = currentDecoder.decodePacket(packet.data, pcm);
if (count <= 0) {
maybeEmitAudioStats(false);
continue;
}
decodedSamples += count;
float[] samples = new float[count];
for (int i = 0; i < count; i++) {
samples[i] = buffer[i] / 32768.0f;
samples[i] = pcm[i] / 32768.0f;
}
currentStream.acceptWaveform(samples, SAMPLE_RATE);
@@ -242,27 +238,16 @@ public class FaceclawVoiceController {
emitWakeWord(keyword);
}
}
maybeEmitAudioStats(false);
}
}
private void stopAudio() {
AudioRecord record = audioRecord;
audioRecord = null;
if (record != null) {
try {
record.stop();
} catch (IllegalStateException ignored) {
}
record.release();
}
AudioManager audioManager = (AudioManager) appContext.getSystemService(Context.AUDIO_SERVICE);
if (audioManager != null) {
if (Build.VERSION.SDK_INT >= 31) {
audioManager.clearCommunicationDevice();
}
audioManager.stopBluetoothSco();
audioManager.setBluetoothScoOn(false);
private void stopG2Audio() {
FaceclawBleCommunicator currentCommunicator = communicator;
if (currentCommunicator != null) {
currentCommunicator.stopG2AudioCapture();
}
maybeEmitAudioStats(true);
}
private void releaseSherpa() {
@@ -276,51 +261,90 @@ public class FaceclawVoiceController {
}
}
private void routeToBluetoothHeadset() {
AudioManager audioManager = (AudioManager) appContext.getSystemService(Context.AUDIO_SERVICE);
if (audioManager == null) {
private void releaseLc3() {
if (lc3Decoder != null) {
lc3Decoder.close();
lc3Decoder = null;
}
}
private void queueAudioPacket(byte[] data, String arm, long arrivalMs) {
if (!started || data == null) {
return;
}
audioManager.setMode(AudioManager.MODE_IN_COMMUNICATION);
audioManager.startBluetoothSco();
audioManager.setBluetoothScoOn(true);
if (Build.VERSION.SDK_INT >= 31) {
for (AudioDeviceInfo device : audioManager.getAvailableCommunicationDevices()) {
if (device.getType() == AudioDeviceInfo.TYPE_BLUETOOTH_SCO
|| device.getType() == AudioDeviceInfo.TYPE_BLE_HEADSET) {
audioManager.setCommunicationDevice(device);
return;
if (!"L".equals(arm)) {
wrongArmPackets++;
}
synchronized (audioQueueLock) {
if (audioQueue.size() >= MAX_AUDIO_QUEUE_PACKETS) {
audioQueue.removeFirst();
queueDroppedPackets++;
}
audioQueue.addLast(new AudioPacket(data, arm, arrivalMs));
queuedPackets++;
if (lastPacketArrivalMs > 0) {
long delta = arrivalMs - lastPacketArrivalMs;
if (delta > maxInterPacketMs) {
maxInterPacketMs = delta;
}
if (delta > LATE_PACKET_INTERVAL_MS) {
latePackets++;
}
}
lastPacketArrivalMs = arrivalMs;
audioQueueLock.notifyAll();
}
}
private boolean hasBluetoothHeadset() {
if (Build.VERSION.SDK_INT >= 31
&& ContextCompat.checkSelfPermission(appContext, Manifest.permission.BLUETOOTH_CONNECT)
!= PackageManager.PERMISSION_GRANTED) {
return false;
}
BluetoothManager manager = (BluetoothManager) appContext.getSystemService(Context.BLUETOOTH_SERVICE);
BluetoothAdapter adapter = manager == null ? null : manager.getAdapter();
if (adapter == null) {
return false;
}
try {
int state = adapter.getProfileConnectionState(BluetoothProfile.HEADSET);
if (state == BluetoothProfile.STATE_CONNECTED || state == BluetoothProfile.STATE_CONNECTING) {
return true;
private AudioPacket takeAudioPacket() {
synchronized (audioQueueLock) {
while (started && audioQueue.isEmpty()) {
try {
audioQueueLock.wait(250);
maybeEmitAudioStats(false);
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
return null;
}
}
Set<BluetoothDevice> bondedDevices = adapter.getBondedDevices();
return bondedDevices != null && !bondedDevices.isEmpty();
} catch (SecurityException ignored) {
return false;
return audioQueue.pollFirst();
}
}
private boolean hasRecordAudioPermission() {
return ContextCompat.checkSelfPermission(appContext, Manifest.permission.RECORD_AUDIO)
== PackageManager.PERMISSION_GRANTED;
private void resetAudioStats() {
queuedPackets = 0;
queueDroppedPackets = 0;
decodedSamples = 0;
latePackets = 0;
wrongArmPackets = 0;
lastPacketArrivalMs = 0;
maxInterPacketMs = 0;
lastStatsAtMs = SystemClock.elapsedRealtime();
}
private void maybeEmitAudioStats(boolean force) {
long now = SystemClock.elapsedRealtime();
if (!force && now - lastStatsAtMs < STATS_INTERVAL_MS) {
return;
}
lastStatsAtMs = now;
FaceclawLc3Decoder currentDecoder = lc3Decoder;
long real = currentDecoder == null ? 0 : currentDecoder.getRealPackets();
long duplicate = currentDecoder == null ? 0 : currentDecoder.getDuplicatePackets();
long missing = currentDecoder == null ? 0 : currentDecoder.getMissingPackets();
long decodeErrors = currentDecoder == null ? 0 : currentDecoder.getDecodeErrors();
String status = "G2 mic packets=" + queuedPackets
+ " decoded=" + real
+ " missing=" + missing
+ " duplicate=" + duplicate
+ " late=" + latePackets
+ " maxGapMs=" + maxInterPacketMs
+ " queueDrop=" + queueDroppedPackets
+ " decodeErrors=" + decodeErrors
+ " wrongArm=" + wrongArmPackets
+ " audioSec=" + String.format(java.util.Locale.US, "%.1f", decodedSamples / (double) SAMPLE_RATE);
Log.i(TAG, status + " expectedIntervalMs=" + EXPECTED_PACKET_INTERVAL_MS);
emitStatus(status);
}
private void emitStatus(String status) {
@@ -338,4 +362,16 @@ public class FaceclawVoiceController {
}
mainHandler.post(() -> currentListener.onWakeWord(keyword));
}
private static final class AudioPacket {
final byte[] data;
final String arm;
final long arrivalMs;
AudioPacket(byte[] data, String arm, long arrivalMs) {
this.data = data;
this.arm = arm == null ? "?" : arm;
this.arrivalMs = arrivalMs;
}
}
}
Binary file not shown.
@@ -0,0 +1,92 @@
#include <jni.h>
#include <stdint.h>
#include <stdlib.h>
#include "lc3.h"
#define G2_FRAME_BYTES 40
#define G2_FRAMES_PER_PACKET 5
#define G2_SAMPLES_PER_PACKET 800
typedef struct {
void *mem;
lc3_decoder_t decoder;
} faceclaw_lc3_decoder_t;
JNIEXPORT jlong JNICALL
Java_com_faceclaw_app_FaceclawLc3Decoder_nativeCreate(JNIEnv *env, jclass clazz, jint frame_us, jint sample_rate) {
(void) env;
(void) clazz;
unsigned mem_size = lc3_decoder_size(frame_us, sample_rate);
if (mem_size == 0) {
return 0;
}
faceclaw_lc3_decoder_t *state = (faceclaw_lc3_decoder_t *) calloc(1, sizeof(faceclaw_lc3_decoder_t));
if (state == NULL) {
return 0;
}
state->mem = calloc(1, mem_size);
if (state->mem == NULL) {
free(state);
return 0;
}
state->decoder = lc3_setup_decoder(frame_us, sample_rate, 0, state->mem);
if (state->decoder == NULL) {
free(state->mem);
free(state);
return 0;
}
return (jlong) (uintptr_t) state;
}
JNIEXPORT jint JNICALL
Java_com_faceclaw_app_FaceclawLc3Decoder_nativeDecodePacket(JNIEnv *env, jclass clazz, jlong handle, jbyteArray packet, jshortArray pcm_out) {
(void) clazz;
faceclaw_lc3_decoder_t *state = (faceclaw_lc3_decoder_t *) (uintptr_t) handle;
if (state == NULL || state->decoder == NULL || packet == NULL || pcm_out == NULL) {
return 0;
}
if ((*env)->GetArrayLength(env, packet) < 205 || (*env)->GetArrayLength(env, pcm_out) < G2_SAMPLES_PER_PACKET) {
return 0;
}
jbyte *packet_bytes = (*env)->GetByteArrayElements(env, packet, NULL);
jshort *pcm = (*env)->GetShortArrayElements(env, pcm_out, NULL);
if (packet_bytes == NULL || pcm == NULL) {
if (packet_bytes != NULL) {
(*env)->ReleaseByteArrayElements(env, packet, packet_bytes, JNI_ABORT);
}
if (pcm != NULL) {
(*env)->ReleaseShortArrayElements(env, pcm_out, pcm, JNI_ABORT);
}
return 0;
}
int ok = 1;
for (int frame = 0; frame < G2_FRAMES_PER_PACKET; frame++) {
const uint8_t *lc3_frame = (const uint8_t *) packet_bytes + frame * G2_FRAME_BYTES;
int16_t *pcm_frame = (int16_t *) pcm + frame * 160;
int rc = lc3_decode(state->decoder, lc3_frame, G2_FRAME_BYTES, LC3_PCM_FORMAT_S16, pcm_frame, 1);
if (rc < 0) {
ok = 0;
break;
}
}
(*env)->ReleaseByteArrayElements(env, packet, packet_bytes, JNI_ABORT);
(*env)->ReleaseShortArrayElements(env, pcm_out, pcm, ok ? 0 : JNI_ABORT);
return ok ? G2_SAMPLES_PER_PACKET : 0;
}
JNIEXPORT void JNICALL
Java_com_faceclaw_app_FaceclawLc3Decoder_nativeDestroy(JNIEnv *env, jclass clazz, jlong handle) {
(void) env;
(void) clazz;
faceclaw_lc3_decoder_t *state = (faceclaw_lc3_decoder_t *) (uintptr_t) handle;
if (state == NULL) {
return;
}
free(state->mem);
free(state);
}
+1
View File
@@ -100,3 +100,4 @@ export async function ensureBlePermissions(): Promise<void> {
export async function ensureVoicePermissions(): Promise<void> {
await ensurePermissions([RECORD_AUDIO_PERMISSION], VOICE_PERMISSION_REQUEST_CODE, "Voice control");
}
+3 -1
View File
@@ -487,6 +487,8 @@ class DashboardController {
private async setVoiceControlEnabled(enabled: boolean): Promise<void> {
if (enabled) {
try {
// Treat Android mic permission as the consent gate for voice control,
// even when the current source is the G2 mic over BLE.
await ensureVoicePermissions();
} catch (error) {
this.appendLog(`voice control permission failed: ${this.formatError(error)}`);
@@ -671,7 +673,7 @@ class DashboardController {
if (!this.communicator) return;
if (!isDashboardVoiceControlEnabled()) return;
this.updateConnectedForegroundNotification();
voiceControlBridge.start();
voiceControlBridge.start(this.communicator.getNativeCommunicator());
}
private setStopwatchRenderActive(active: boolean): void {
+4
View File
@@ -216,6 +216,10 @@ export class FaceclawCommunicatorBridge {
return () => this.frameMetricsListeners.delete(listener);
}
getNativeCommunicator(): any {
return this.communicator;
}
waitForNextFrameMetrics(timeoutMs: number): Promise<FrameMetrics | null> {
const delayMs = Math.max(1, Math.round(nonNegativeNumber(timeoutMs)));
return new Promise((resolve) => {
+4 -1
View File
@@ -25,9 +25,12 @@ export class FaceclawVoiceControlBridge {
return () => this.wakeWordListeners.delete(listener);
}
start(): void {
start(communicator?: any): void {
if (this.started || !global.isAndroid) return;
this.ensureController();
if (communicator) {
this.controller?.setCommunicator(communicator);
}
this.started = true;
this.controller?.start();
}