mirror of
https://git.undefined-dev.fr/Blendman974/faceclaw.git
synced 2026-11-19 21:03:09 +00:00
First-pass G2 microphone streaming support
This commit is contained in:
1 parent
c28b321705
commit
9996b09e9c
13 files changed
+493
-113
No files matched your search
@@ -5,3 +5,5 @@ to keep my room clean, so to speak.
|
||||
Voice wake-word detection uses sherpa-onnx: https://github.com/k2-fsa/sherpa-onnx/
|
||||
and the sherpa-onnx KWS Zipformer GigaSpeech model published by the k2-fsa/sherpa
|
||||
community.
|
||||
|
||||
G2 microphone LC3 decoding uses Google's liblc3: https://github.com/google/liblc3
|
||||
@@ -33,6 +33,7 @@ public class BleProtocol {
|
||||
public static final int FLAG_REQUEST = 0x20;
|
||||
public static final int FLAG_NOTIFY = 0x01;
|
||||
public static final int FLAG_NOTIFY_ALT = 0x06;
|
||||
public static final int CMD_AUDIO_CONTROL = 15;
|
||||
|
||||
public static final int EVENT_CLICK = 0;
|
||||
public static final int EVENT_SCROLL_TOP = 1;
|
||||
@@ -112,6 +113,10 @@ public class BleProtocol {
|
||||
return wrapEvenHub(12, magic, 14, encodeVarintField(1, 0));
|
||||
}
|
||||
|
||||
public static byte[] buildAudioControl(int magic, boolean enable) {
|
||||
return wrapEvenHub(CMD_AUDIO_CONTROL, magic, 18, encodeVarintField(1, enable ? 1 : 0));
|
||||
}
|
||||
|
||||
public static byte[] buildShutdown(int magic, int exitMode) {
|
||||
return wrapEvenHub(9, magic, 11, encodeVarintField(1, exitMode));
|
||||
}
|
||||
|
||||
@@ -0,0 +1,5 @@
|
||||
package com.faceclaw.app;
|
||||
|
||||
public interface FaceclawAudioPacketListener {
|
||||
void onAudioPacket(byte[] data, String arm, long arrivalMs);
|
||||
}
|
||||
@@ -102,6 +102,7 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable {
|
||||
private long lastSessionReadyAtMs;
|
||||
private long lastEvenAppConflictAtMs;
|
||||
private int consecutiveAckTimeouts;
|
||||
private int lastAudioControlAckMagic = 0;
|
||||
|
||||
private final BleMagicPool magicPool = new BleMagicPool();
|
||||
private int nextTransportSeq = 0x40;
|
||||
@@ -111,6 +112,8 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable {
|
||||
private long lastShutdownExitAtMs = 0;
|
||||
private int headsetBattery = -1;
|
||||
private int headsetCharging = -1;
|
||||
private boolean audioCaptureActive;
|
||||
private volatile FaceclawAudioPacketListener audioPacketListener;
|
||||
|
||||
private String displayedFingerprint = "";
|
||||
private byte[][] displayedTileBmps = emptyTileSet();
|
||||
@@ -158,6 +161,8 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable {
|
||||
synchronized (lock) {
|
||||
userDisconnectRequested = true;
|
||||
running = false;
|
||||
audioCaptureActive = false;
|
||||
audioPacketListener = null;
|
||||
phase = "disconnecting";
|
||||
status = "Disconnecting...";
|
||||
lock.notifyAll();
|
||||
@@ -190,6 +195,51 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable {
|
||||
disconnect();
|
||||
}
|
||||
|
||||
public boolean startG2AudioCapture(FaceclawAudioPacketListener listener) {
|
||||
if (listener == null) {
|
||||
throw new IllegalArgumentException("listener is required");
|
||||
}
|
||||
int magic;
|
||||
synchronized (lock) {
|
||||
if (!running || !sessionReady) {
|
||||
logLine("skip G2 mic enable; session not ready");
|
||||
return false;
|
||||
}
|
||||
audioPacketListener = listener;
|
||||
magic = nextMagic();
|
||||
pendingMessages.addFirst(createAudioControlMessageLocked(true, magic));
|
||||
logLine("queue G2 mic enable");
|
||||
lock.notifyAll();
|
||||
}
|
||||
interruptibleSleep.interrupt();
|
||||
return waitForAudioControlAck(magic, "enable");
|
||||
}
|
||||
|
||||
public void stopG2AudioCapture() {
|
||||
int magic = 0;
|
||||
synchronized (lock) {
|
||||
audioPacketListener = null;
|
||||
audioCaptureActive = false;
|
||||
clearMessagesOfKindLocked("audio-control");
|
||||
if (running && sessionReady) {
|
||||
magic = nextMagic();
|
||||
pendingMessages.addFirst(createAudioControlMessageLocked(false, magic));
|
||||
logLine("queue G2 mic disable");
|
||||
lock.notifyAll();
|
||||
}
|
||||
}
|
||||
interruptibleSleep.interrupt();
|
||||
if (magic != 0) {
|
||||
waitForAudioControlAck(magic, "disable");
|
||||
}
|
||||
}
|
||||
|
||||
public boolean isSessionReady() {
|
||||
synchronized (lock) {
|
||||
return running && sessionReady;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
public void submitDashboardImage(List<byte[]> tileBmps, String fingerprint) {
|
||||
submitDashboardImage(tileBmps, fingerprint, false, -1);
|
||||
@@ -365,6 +415,33 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable {
|
||||
}
|
||||
}
|
||||
|
||||
private boolean waitForAudioControlAck(int magic, String operation) {
|
||||
long deadline = SystemClock.elapsedRealtime() + ACK_TIMEOUT_MS + WRITE_TIMEOUT_MS + 500;
|
||||
synchronized (lock) {
|
||||
while (running && sessionReady && lastAudioControlAckMagic != magic && hasPendingOrInflightMagicLocked(magic)) {
|
||||
long remaining = deadline - SystemClock.elapsedRealtime();
|
||||
if (remaining <= 0) {
|
||||
break;
|
||||
}
|
||||
try {
|
||||
lock.wait(Math.min(remaining, 100));
|
||||
} catch (InterruptedException e) {
|
||||
Thread.currentThread().interrupt();
|
||||
break;
|
||||
}
|
||||
}
|
||||
boolean acked = lastAudioControlAckMagic == magic;
|
||||
if (!acked) {
|
||||
if ("enable".equals(operation)) {
|
||||
audioPacketListener = null;
|
||||
audioCaptureActive = false;
|
||||
}
|
||||
logLine("G2 mic " + operation + " ack timeout");
|
||||
}
|
||||
return acked;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public void run() {
|
||||
@@ -404,6 +481,10 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable {
|
||||
return;
|
||||
}
|
||||
String uuid = characteristicUuid.toLowerCase(Locale.US);
|
||||
if (BleProtocol.RENDER_NOTIFY_UUID.equals(uuid)) {
|
||||
handleRenderNotification(address, data);
|
||||
return;
|
||||
}
|
||||
if (!BleProtocol.NOTIFY_CHAR_UUID.equals(uuid)) {
|
||||
return;
|
||||
}
|
||||
@@ -430,6 +511,24 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable {
|
||||
}
|
||||
}
|
||||
|
||||
private void handleRenderNotification(String address, byte[] data) {
|
||||
FaceclawAudioPacketListener listenerToCall;
|
||||
long arrivalMs = SystemClock.elapsedRealtime();
|
||||
synchronized (lock) {
|
||||
lastIncomingAtMs = arrivalMs;
|
||||
listenerToCall = audioCaptureActive ? audioPacketListener : null;
|
||||
}
|
||||
if (listenerToCall == null) {
|
||||
return;
|
||||
}
|
||||
String arm = address.equalsIgnoreCase(leftAddress) ? "L" : address.equalsIgnoreCase(rightAddress) ? "R" : "?";
|
||||
try {
|
||||
listenerToCall.onAudioPacket(Arrays.copyOf(data, data.length), arm, arrivalMs);
|
||||
} catch (Throwable t) {
|
||||
logLine("G2 mic packet listener failed: " + safeMessage(t));
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void onConnectionStateChange(String address, boolean connected) {
|
||||
synchronized (lock) {
|
||||
@@ -446,6 +545,8 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable {
|
||||
fixedLayoutCreated = false;
|
||||
warmedUp = false;
|
||||
startupProbePending = false;
|
||||
audioCaptureActive = false;
|
||||
audioPacketListener = null;
|
||||
clearAllMessagesLocked("connection lost");
|
||||
displayedFingerprint = "";
|
||||
displayedTileBmps = emptyTileSet();
|
||||
@@ -465,7 +566,7 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable {
|
||||
setState("connecting", "Connecting to the glasses...");
|
||||
try {
|
||||
connectArm(rightAddress, true);
|
||||
connectArm(leftAddress, false);
|
||||
connectArm(leftAddress, true);
|
||||
if (!sleepDuringConnectSettling(800)) {
|
||||
return;
|
||||
}
|
||||
@@ -486,6 +587,8 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable {
|
||||
imageRetryAfterMs = 0;
|
||||
lastHeartbeatQueuedAtMs = 0;
|
||||
consecutiveAckTimeouts = 0;
|
||||
lastAudioControlAckMagic = 0;
|
||||
audioCaptureActive = false;
|
||||
}
|
||||
setState("connected", "Connected.");
|
||||
logLine("session ready");
|
||||
@@ -785,6 +888,13 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable {
|
||||
if ("heartbeat".equals(message.kind)) {
|
||||
return;
|
||||
}
|
||||
if ("audio-control".equals(message.kind)) {
|
||||
lastAudioControlAckMagic = message.magic;
|
||||
audioCaptureActive = message.label != null && message.label.contains("enable");
|
||||
logLine(audioCaptureActive ? "G2 mic enabled" : "G2 mic disabled");
|
||||
lock.notifyAll();
|
||||
return;
|
||||
}
|
||||
if ("battery".equals(message.kind)) {
|
||||
BleProtocol.BatterySnapshot snapshot = BleProtocol.parseSettingsBattery(message.ackPayload);
|
||||
if (snapshot != null) {
|
||||
@@ -1035,6 +1145,25 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable {
|
||||
);
|
||||
}
|
||||
|
||||
private OutboundMessage createAudioControlMessageLocked(boolean enable, int magic) {
|
||||
return new OutboundMessage(
|
||||
"audio-control",
|
||||
enable ? "G2 mic enable" : "G2 mic disable",
|
||||
BleProtocol.SID_EVENHUB,
|
||||
magic,
|
||||
BleProtocol.framePb(
|
||||
BleProtocol.buildAudioControl(magic, enable),
|
||||
BleProtocol.SID_EVENHUB,
|
||||
BleProtocol.FLAG_REQUEST,
|
||||
nextTransportSeq++
|
||||
),
|
||||
ACK_TIMEOUT_MS,
|
||||
displayedFingerprint,
|
||||
-1,
|
||||
null
|
||||
);
|
||||
}
|
||||
|
||||
private boolean shouldPollBatteryLocked(long now) {
|
||||
return !shutdownRequested
|
||||
&& sessionReady
|
||||
@@ -1265,6 +1394,9 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable {
|
||||
lastHeartbeatQueuedAtMs = 0;
|
||||
lastSessionReadyAtMs = 0;
|
||||
consecutiveAckTimeouts = 0;
|
||||
lastAudioControlAckMagic = 0;
|
||||
audioCaptureActive = false;
|
||||
audioPacketListener = null;
|
||||
displayedFingerprint = "";
|
||||
displayedTileBmps = emptyTileSet();
|
||||
}
|
||||
|
||||
@@ -107,6 +107,7 @@ public class FaceclawForegroundService extends Service {
|
||||
|
||||
private int foregroundServiceType() {
|
||||
int type = ServiceInfo.FOREGROUND_SERVICE_TYPE_CONNECTED_DEVICE;
|
||||
// TODO: Make this depend on which audio path (G2 vs phone) is selected
|
||||
if (hasRecordAudioPermission()) {
|
||||
type |= ServiceInfo.FOREGROUND_SERVICE_TYPE_MICROPHONE;
|
||||
}
|
||||
|
||||
@@ -0,0 +1,97 @@
|
||||
package com.faceclaw.app;
|
||||
|
||||
import android.util.Log;
|
||||
|
||||
public class FaceclawLc3Decoder {
|
||||
private static final String TAG = "FaceclawLc3";
|
||||
public static final int SAMPLE_RATE = 16000;
|
||||
public static final int FRAME_US = 10000;
|
||||
public static final int FRAME_BYTES = 40;
|
||||
public static final int FRAMES_PER_PACKET = 5;
|
||||
public static final int PACKET_BYTES = 205;
|
||||
public static final int COUNTER_OFFSET = 204;
|
||||
public static final int SAMPLES_PER_FRAME = 160;
|
||||
public static final int SAMPLES_PER_PACKET = FRAMES_PER_PACKET * SAMPLES_PER_FRAME;
|
||||
|
||||
static {
|
||||
System.loadLibrary("faceclaw_lc3");
|
||||
}
|
||||
|
||||
private long nativeHandle;
|
||||
private int lastCounter = -1;
|
||||
private long realPackets;
|
||||
private long duplicatePackets;
|
||||
private long missingPackets;
|
||||
private long decodeErrors;
|
||||
|
||||
public FaceclawLc3Decoder() {
|
||||
nativeHandle = nativeCreate(FRAME_US, SAMPLE_RATE);
|
||||
if (nativeHandle == 0) {
|
||||
throw new IllegalStateException("Could not create LC3 decoder");
|
||||
}
|
||||
}
|
||||
|
||||
public synchronized int decodePacket(byte[] packet, short[] pcmOut) {
|
||||
if (nativeHandle == 0) {
|
||||
throw new IllegalStateException("LC3 decoder is closed");
|
||||
}
|
||||
if (packet == null || packet.length != PACKET_BYTES) {
|
||||
decodeErrors++;
|
||||
Log.w(TAG, "unexpected G2 audio packet length=" + (packet == null ? -1 : packet.length));
|
||||
return 0;
|
||||
}
|
||||
if (pcmOut == null || pcmOut.length < SAMPLES_PER_PACKET) {
|
||||
throw new IllegalArgumentException("pcmOut must hold " + SAMPLES_PER_PACKET + " samples");
|
||||
}
|
||||
|
||||
int counter = packet[COUNTER_OFFSET] & 0xff;
|
||||
if (lastCounter >= 0) {
|
||||
int gap = (counter - lastCounter) & 0xff;
|
||||
if (gap == 0) {
|
||||
duplicatePackets++;
|
||||
return 0;
|
||||
}
|
||||
int missing = gap - 1;
|
||||
if (missing > 0) {
|
||||
missingPackets += missing;
|
||||
}
|
||||
}
|
||||
|
||||
int decoded = nativeDecodePacket(nativeHandle, packet, pcmOut);
|
||||
if (decoded <= 0) {
|
||||
decodeErrors++;
|
||||
return 0;
|
||||
}
|
||||
realPackets++;
|
||||
lastCounter = counter;
|
||||
return decoded;
|
||||
}
|
||||
|
||||
public synchronized long getRealPackets() {
|
||||
return realPackets;
|
||||
}
|
||||
|
||||
public synchronized long getDuplicatePackets() {
|
||||
return duplicatePackets;
|
||||
}
|
||||
|
||||
public synchronized long getMissingPackets() {
|
||||
return missingPackets;
|
||||
}
|
||||
|
||||
public synchronized long getDecodeErrors() {
|
||||
return decodeErrors;
|
||||
}
|
||||
|
||||
public synchronized void close() {
|
||||
long handle = nativeHandle;
|
||||
nativeHandle = 0;
|
||||
if (handle != 0) {
|
||||
nativeDestroy(handle);
|
||||
}
|
||||
}
|
||||
|
||||
private static native long nativeCreate(int frameUs, int sampleRate);
|
||||
private static native int nativeDecodePacket(long handle, byte[] packet, short[] pcmOut);
|
||||
private static native void nativeDestroy(long handle);
|
||||
}
|
||||
+146
-110
@@ -1,25 +1,12 @@
|
||||
package com.faceclaw.app;
|
||||
|
||||
import android.Manifest;
|
||||
import android.bluetooth.BluetoothAdapter;
|
||||
import android.bluetooth.BluetoothDevice;
|
||||
import android.bluetooth.BluetoothManager;
|
||||
import android.bluetooth.BluetoothProfile;
|
||||
import android.content.Context;
|
||||
import android.content.pm.PackageManager;
|
||||
import android.content.res.AssetManager;
|
||||
import android.media.AudioDeviceInfo;
|
||||
import android.media.AudioFormat;
|
||||
import android.media.AudioManager;
|
||||
import android.media.AudioRecord;
|
||||
import android.media.MediaRecorder;
|
||||
import android.os.Build;
|
||||
import android.os.Handler;
|
||||
import android.os.Looper;
|
||||
import android.os.SystemClock;
|
||||
import android.util.Log;
|
||||
|
||||
import androidx.core.content.ContextCompat;
|
||||
|
||||
import com.k2fsa.sherpa.onnx.FeatureConfig;
|
||||
import com.k2fsa.sherpa.onnx.KeywordSpotter;
|
||||
import com.k2fsa.sherpa.onnx.KeywordSpotterConfig;
|
||||
@@ -32,12 +19,16 @@ import java.io.File;
|
||||
import java.io.FileOutputStream;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.util.Set;
|
||||
import java.util.ArrayDeque;
|
||||
|
||||
public class FaceclawVoiceController {
|
||||
private static final String TAG = "FaceclawVoice";
|
||||
private static final int SAMPLE_RATE = 16000;
|
||||
private static final int FEATURE_DIM = 80;
|
||||
private static final int MAX_AUDIO_QUEUE_PACKETS = 80;
|
||||
private static final int EXPECTED_PACKET_INTERVAL_MS = 50;
|
||||
private static final int LATE_PACKET_INTERVAL_MS = 90;
|
||||
private static final int STATS_INTERVAL_MS = 5_000;
|
||||
private static final String ASSET_ROOT = "faceclaw-voice";
|
||||
private static final String MODEL_DIR = "sherpa-onnx-kws-zipformer-gigaspeech-3.3M-2024-01-01";
|
||||
private static final String[] MODEL_FILES = {
|
||||
@@ -51,12 +42,23 @@ public class FaceclawVoiceController {
|
||||
private final Context appContext;
|
||||
private final Handler mainHandler = new Handler(Looper.getMainLooper());
|
||||
private final Object lock = new Object();
|
||||
private final Object audioQueueLock = new Object();
|
||||
private final ArrayDeque<AudioPacket> audioQueue = new ArrayDeque<>();
|
||||
private volatile FaceclawVoiceControllerListener listener;
|
||||
private volatile FaceclawBleCommunicator communicator;
|
||||
private Thread workerThread;
|
||||
private volatile boolean started;
|
||||
private AudioRecord audioRecord;
|
||||
private KeywordSpotter keywordSpotter;
|
||||
private OnlineStream stream;
|
||||
private FaceclawLc3Decoder lc3Decoder;
|
||||
private long queuedPackets;
|
||||
private long queueDroppedPackets;
|
||||
private long decodedSamples;
|
||||
private long latePackets;
|
||||
private long wrongArmPackets;
|
||||
private long lastPacketArrivalMs;
|
||||
private long maxInterPacketMs;
|
||||
private long lastStatsAtMs;
|
||||
|
||||
public FaceclawVoiceController(Context context) {
|
||||
this.appContext = context.getApplicationContext();
|
||||
@@ -66,18 +68,18 @@ public class FaceclawVoiceController {
|
||||
this.listener = listener;
|
||||
}
|
||||
|
||||
public void setCommunicator(FaceclawBleCommunicator communicator) {
|
||||
this.communicator = communicator;
|
||||
}
|
||||
|
||||
public void start() {
|
||||
synchronized (lock) {
|
||||
if (started) {
|
||||
emitStatus("Voice control is already listening.");
|
||||
return;
|
||||
}
|
||||
if (!hasRecordAudioPermission()) {
|
||||
emitStatus("Voice control needs microphone permission.");
|
||||
return;
|
||||
}
|
||||
if (!hasBluetoothHeadset()) {
|
||||
emitStatus("Voice control needs a paired Bluetooth headset.");
|
||||
if (communicator == null || !communicator.isSessionReady()) {
|
||||
emitStatus("Voice control needs an active G2 connection.");
|
||||
return;
|
||||
}
|
||||
started = true;
|
||||
@@ -95,7 +97,10 @@ public class FaceclawVoiceController {
|
||||
started = false;
|
||||
threadToJoin = workerThread;
|
||||
}
|
||||
stopAudio();
|
||||
stopG2Audio();
|
||||
synchronized (audioQueueLock) {
|
||||
audioQueueLock.notifyAll();
|
||||
}
|
||||
if (threadToJoin != null) {
|
||||
threadToJoin.interrupt();
|
||||
}
|
||||
@@ -111,19 +116,21 @@ public class FaceclawVoiceController {
|
||||
File modelDir = installModelFiles();
|
||||
keywordSpotter = new KeywordSpotter(buildConfig(modelDir));
|
||||
stream = keywordSpotter.createStream();
|
||||
if (!startAudio()) {
|
||||
emitStatus("Voice control could not start microphone input.");
|
||||
lc3Decoder = new FaceclawLc3Decoder();
|
||||
if (!startG2Audio()) {
|
||||
emitStatus("Voice control could not start G2 microphone input.");
|
||||
return;
|
||||
}
|
||||
|
||||
emitStatus("Voice control listening for \"screen on\".");
|
||||
processAudio();
|
||||
emitStatus("Voice control listening for \"screen on\" from the G2 mic.");
|
||||
processG2Audio();
|
||||
} catch (Throwable error) {
|
||||
Log.e(TAG, "Voice control failed", error);
|
||||
emitStatus("Voice control failed: " + error.getMessage());
|
||||
} finally {
|
||||
stopG2Audio();
|
||||
releaseSherpa();
|
||||
stopAudio();
|
||||
releaseLc3();
|
||||
synchronized (lock) {
|
||||
started = false;
|
||||
workerThread = null;
|
||||
@@ -183,53 +190,42 @@ public class FaceclawVoiceController {
|
||||
}
|
||||
}
|
||||
|
||||
private boolean startAudio() {
|
||||
routeToBluetoothHeadset();
|
||||
int minBufferBytes = AudioRecord.getMinBufferSize(
|
||||
SAMPLE_RATE,
|
||||
AudioFormat.CHANNEL_IN_MONO,
|
||||
AudioFormat.ENCODING_PCM_16BIT
|
||||
);
|
||||
if (minBufferBytes <= 0) {
|
||||
private boolean startG2Audio() {
|
||||
FaceclawBleCommunicator currentCommunicator = communicator;
|
||||
if (currentCommunicator == null) {
|
||||
return false;
|
||||
}
|
||||
if (!hasRecordAudioPermission()) {
|
||||
return false;
|
||||
resetAudioStats();
|
||||
synchronized (audioQueueLock) {
|
||||
audioQueue.clear();
|
||||
}
|
||||
audioRecord = new AudioRecord(
|
||||
MediaRecorder.AudioSource.VOICE_RECOGNITION,
|
||||
SAMPLE_RATE,
|
||||
AudioFormat.CHANNEL_IN_MONO,
|
||||
AudioFormat.ENCODING_PCM_16BIT,
|
||||
minBufferBytes * 2
|
||||
);
|
||||
if (audioRecord.getState() != AudioRecord.STATE_INITIALIZED) {
|
||||
audioRecord.release();
|
||||
audioRecord = null;
|
||||
return false;
|
||||
}
|
||||
audioRecord.startRecording();
|
||||
return true;
|
||||
return currentCommunicator.startG2AudioCapture(this::queueAudioPacket);
|
||||
}
|
||||
|
||||
private void processAudio() {
|
||||
short[] buffer = new short[SAMPLE_RATE / 10];
|
||||
private void processG2Audio() {
|
||||
short[] pcm = new short[FaceclawLc3Decoder.SAMPLES_PER_PACKET];
|
||||
while (started && !Thread.currentThread().isInterrupted()) {
|
||||
AudioRecord currentAudio = audioRecord;
|
||||
OnlineStream currentStream = stream;
|
||||
KeywordSpotter currentSpotter = keywordSpotter;
|
||||
if (currentAudio == null || currentStream == null || currentSpotter == null) {
|
||||
FaceclawLc3Decoder currentDecoder = lc3Decoder;
|
||||
if (currentStream == null || currentSpotter == null || currentDecoder == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
int count = currentAudio.read(buffer, 0, buffer.length);
|
||||
if (count <= 0) {
|
||||
AudioPacket packet = takeAudioPacket();
|
||||
if (packet == null) {
|
||||
continue;
|
||||
}
|
||||
|
||||
int count = currentDecoder.decodePacket(packet.data, pcm);
|
||||
if (count <= 0) {
|
||||
maybeEmitAudioStats(false);
|
||||
continue;
|
||||
}
|
||||
decodedSamples += count;
|
||||
float[] samples = new float[count];
|
||||
for (int i = 0; i < count; i++) {
|
||||
samples[i] = buffer[i] / 32768.0f;
|
||||
samples[i] = pcm[i] / 32768.0f;
|
||||
}
|
||||
currentStream.acceptWaveform(samples, SAMPLE_RATE);
|
||||
|
||||
@@ -242,27 +238,16 @@ public class FaceclawVoiceController {
|
||||
emitWakeWord(keyword);
|
||||
}
|
||||
}
|
||||
maybeEmitAudioStats(false);
|
||||
}
|
||||
}
|
||||
|
||||
private void stopAudio() {
|
||||
AudioRecord record = audioRecord;
|
||||
audioRecord = null;
|
||||
if (record != null) {
|
||||
try {
|
||||
record.stop();
|
||||
} catch (IllegalStateException ignored) {
|
||||
}
|
||||
record.release();
|
||||
}
|
||||
AudioManager audioManager = (AudioManager) appContext.getSystemService(Context.AUDIO_SERVICE);
|
||||
if (audioManager != null) {
|
||||
if (Build.VERSION.SDK_INT >= 31) {
|
||||
audioManager.clearCommunicationDevice();
|
||||
}
|
||||
audioManager.stopBluetoothSco();
|
||||
audioManager.setBluetoothScoOn(false);
|
||||
private void stopG2Audio() {
|
||||
FaceclawBleCommunicator currentCommunicator = communicator;
|
||||
if (currentCommunicator != null) {
|
||||
currentCommunicator.stopG2AudioCapture();
|
||||
}
|
||||
maybeEmitAudioStats(true);
|
||||
}
|
||||
|
||||
private void releaseSherpa() {
|
||||
@@ -276,51 +261,90 @@ public class FaceclawVoiceController {
|
||||
}
|
||||
}
|
||||
|
||||
private void routeToBluetoothHeadset() {
|
||||
AudioManager audioManager = (AudioManager) appContext.getSystemService(Context.AUDIO_SERVICE);
|
||||
if (audioManager == null) {
|
||||
private void releaseLc3() {
|
||||
if (lc3Decoder != null) {
|
||||
lc3Decoder.close();
|
||||
lc3Decoder = null;
|
||||
}
|
||||
}
|
||||
|
||||
private void queueAudioPacket(byte[] data, String arm, long arrivalMs) {
|
||||
if (!started || data == null) {
|
||||
return;
|
||||
}
|
||||
audioManager.setMode(AudioManager.MODE_IN_COMMUNICATION);
|
||||
audioManager.startBluetoothSco();
|
||||
audioManager.setBluetoothScoOn(true);
|
||||
if (Build.VERSION.SDK_INT >= 31) {
|
||||
for (AudioDeviceInfo device : audioManager.getAvailableCommunicationDevices()) {
|
||||
if (device.getType() == AudioDeviceInfo.TYPE_BLUETOOTH_SCO
|
||||
|| device.getType() == AudioDeviceInfo.TYPE_BLE_HEADSET) {
|
||||
audioManager.setCommunicationDevice(device);
|
||||
return;
|
||||
if (!"L".equals(arm)) {
|
||||
wrongArmPackets++;
|
||||
}
|
||||
synchronized (audioQueueLock) {
|
||||
if (audioQueue.size() >= MAX_AUDIO_QUEUE_PACKETS) {
|
||||
audioQueue.removeFirst();
|
||||
queueDroppedPackets++;
|
||||
}
|
||||
audioQueue.addLast(new AudioPacket(data, arm, arrivalMs));
|
||||
queuedPackets++;
|
||||
if (lastPacketArrivalMs > 0) {
|
||||
long delta = arrivalMs - lastPacketArrivalMs;
|
||||
if (delta > maxInterPacketMs) {
|
||||
maxInterPacketMs = delta;
|
||||
}
|
||||
if (delta > LATE_PACKET_INTERVAL_MS) {
|
||||
latePackets++;
|
||||
}
|
||||
}
|
||||
lastPacketArrivalMs = arrivalMs;
|
||||
audioQueueLock.notifyAll();
|
||||
}
|
||||
}
|
||||
|
||||
private boolean hasBluetoothHeadset() {
|
||||
if (Build.VERSION.SDK_INT >= 31
|
||||
&& ContextCompat.checkSelfPermission(appContext, Manifest.permission.BLUETOOTH_CONNECT)
|
||||
!= PackageManager.PERMISSION_GRANTED) {
|
||||
return false;
|
||||
}
|
||||
BluetoothManager manager = (BluetoothManager) appContext.getSystemService(Context.BLUETOOTH_SERVICE);
|
||||
BluetoothAdapter adapter = manager == null ? null : manager.getAdapter();
|
||||
if (adapter == null) {
|
||||
return false;
|
||||
}
|
||||
try {
|
||||
int state = adapter.getProfileConnectionState(BluetoothProfile.HEADSET);
|
||||
if (state == BluetoothProfile.STATE_CONNECTED || state == BluetoothProfile.STATE_CONNECTING) {
|
||||
return true;
|
||||
private AudioPacket takeAudioPacket() {
|
||||
synchronized (audioQueueLock) {
|
||||
while (started && audioQueue.isEmpty()) {
|
||||
try {
|
||||
audioQueueLock.wait(250);
|
||||
maybeEmitAudioStats(false);
|
||||
} catch (InterruptedException e) {
|
||||
Thread.currentThread().interrupt();
|
||||
return null;
|
||||
}
|
||||
}
|
||||
Set<BluetoothDevice> bondedDevices = adapter.getBondedDevices();
|
||||
return bondedDevices != null && !bondedDevices.isEmpty();
|
||||
} catch (SecurityException ignored) {
|
||||
return false;
|
||||
return audioQueue.pollFirst();
|
||||
}
|
||||
}
|
||||
|
||||
private boolean hasRecordAudioPermission() {
|
||||
return ContextCompat.checkSelfPermission(appContext, Manifest.permission.RECORD_AUDIO)
|
||||
== PackageManager.PERMISSION_GRANTED;
|
||||
private void resetAudioStats() {
|
||||
queuedPackets = 0;
|
||||
queueDroppedPackets = 0;
|
||||
decodedSamples = 0;
|
||||
latePackets = 0;
|
||||
wrongArmPackets = 0;
|
||||
lastPacketArrivalMs = 0;
|
||||
maxInterPacketMs = 0;
|
||||
lastStatsAtMs = SystemClock.elapsedRealtime();
|
||||
}
|
||||
|
||||
private void maybeEmitAudioStats(boolean force) {
|
||||
long now = SystemClock.elapsedRealtime();
|
||||
if (!force && now - lastStatsAtMs < STATS_INTERVAL_MS) {
|
||||
return;
|
||||
}
|
||||
lastStatsAtMs = now;
|
||||
FaceclawLc3Decoder currentDecoder = lc3Decoder;
|
||||
long real = currentDecoder == null ? 0 : currentDecoder.getRealPackets();
|
||||
long duplicate = currentDecoder == null ? 0 : currentDecoder.getDuplicatePackets();
|
||||
long missing = currentDecoder == null ? 0 : currentDecoder.getMissingPackets();
|
||||
long decodeErrors = currentDecoder == null ? 0 : currentDecoder.getDecodeErrors();
|
||||
String status = "G2 mic packets=" + queuedPackets
|
||||
+ " decoded=" + real
|
||||
+ " missing=" + missing
|
||||
+ " duplicate=" + duplicate
|
||||
+ " late=" + latePackets
|
||||
+ " maxGapMs=" + maxInterPacketMs
|
||||
+ " queueDrop=" + queueDroppedPackets
|
||||
+ " decodeErrors=" + decodeErrors
|
||||
+ " wrongArm=" + wrongArmPackets
|
||||
+ " audioSec=" + String.format(java.util.Locale.US, "%.1f", decodedSamples / (double) SAMPLE_RATE);
|
||||
Log.i(TAG, status + " expectedIntervalMs=" + EXPECTED_PACKET_INTERVAL_MS);
|
||||
emitStatus(status);
|
||||
}
|
||||
|
||||
private void emitStatus(String status) {
|
||||
@@ -338,4 +362,16 @@ public class FaceclawVoiceController {
|
||||
}
|
||||
mainHandler.post(() -> currentListener.onWakeWord(keyword));
|
||||
}
|
||||
|
||||
private static final class AudioPacket {
|
||||
final byte[] data;
|
||||
final String arm;
|
||||
final long arrivalMs;
|
||||
|
||||
AudioPacket(byte[] data, String arm, long arrivalMs) {
|
||||
this.data = data;
|
||||
this.arm = arm == null ? "?" : arm;
|
||||
this.arrivalMs = arrivalMs;
|
||||
}
|
||||
}
|
||||
}
|
||||
Binary file not shown.
@@ -0,0 +1,92 @@
|
||||
#include <jni.h>
|
||||
#include <stdint.h>
|
||||
#include <stdlib.h>
|
||||
|
||||
#include "lc3.h"
|
||||
|
||||
#define G2_FRAME_BYTES 40
|
||||
#define G2_FRAMES_PER_PACKET 5
|
||||
#define G2_SAMPLES_PER_PACKET 800
|
||||
|
||||
typedef struct {
|
||||
void *mem;
|
||||
lc3_decoder_t decoder;
|
||||
} faceclaw_lc3_decoder_t;
|
||||
|
||||
JNIEXPORT jlong JNICALL
|
||||
Java_com_faceclaw_app_FaceclawLc3Decoder_nativeCreate(JNIEnv *env, jclass clazz, jint frame_us, jint sample_rate) {
|
||||
(void) env;
|
||||
(void) clazz;
|
||||
unsigned mem_size = lc3_decoder_size(frame_us, sample_rate);
|
||||
if (mem_size == 0) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
faceclaw_lc3_decoder_t *state = (faceclaw_lc3_decoder_t *) calloc(1, sizeof(faceclaw_lc3_decoder_t));
|
||||
if (state == NULL) {
|
||||
return 0;
|
||||
}
|
||||
state->mem = calloc(1, mem_size);
|
||||
if (state->mem == NULL) {
|
||||
free(state);
|
||||
return 0;
|
||||
}
|
||||
state->decoder = lc3_setup_decoder(frame_us, sample_rate, 0, state->mem);
|
||||
if (state->decoder == NULL) {
|
||||
free(state->mem);
|
||||
free(state);
|
||||
return 0;
|
||||
}
|
||||
return (jlong) (uintptr_t) state;
|
||||
}
|
||||
|
||||
JNIEXPORT jint JNICALL
|
||||
Java_com_faceclaw_app_FaceclawLc3Decoder_nativeDecodePacket(JNIEnv *env, jclass clazz, jlong handle, jbyteArray packet, jshortArray pcm_out) {
|
||||
(void) clazz;
|
||||
faceclaw_lc3_decoder_t *state = (faceclaw_lc3_decoder_t *) (uintptr_t) handle;
|
||||
if (state == NULL || state->decoder == NULL || packet == NULL || pcm_out == NULL) {
|
||||
return 0;
|
||||
}
|
||||
if ((*env)->GetArrayLength(env, packet) < 205 || (*env)->GetArrayLength(env, pcm_out) < G2_SAMPLES_PER_PACKET) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
jbyte *packet_bytes = (*env)->GetByteArrayElements(env, packet, NULL);
|
||||
jshort *pcm = (*env)->GetShortArrayElements(env, pcm_out, NULL);
|
||||
if (packet_bytes == NULL || pcm == NULL) {
|
||||
if (packet_bytes != NULL) {
|
||||
(*env)->ReleaseByteArrayElements(env, packet, packet_bytes, JNI_ABORT);
|
||||
}
|
||||
if (pcm != NULL) {
|
||||
(*env)->ReleaseShortArrayElements(env, pcm_out, pcm, JNI_ABORT);
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
int ok = 1;
|
||||
for (int frame = 0; frame < G2_FRAMES_PER_PACKET; frame++) {
|
||||
const uint8_t *lc3_frame = (const uint8_t *) packet_bytes + frame * G2_FRAME_BYTES;
|
||||
int16_t *pcm_frame = (int16_t *) pcm + frame * 160;
|
||||
int rc = lc3_decode(state->decoder, lc3_frame, G2_FRAME_BYTES, LC3_PCM_FORMAT_S16, pcm_frame, 1);
|
||||
if (rc < 0) {
|
||||
ok = 0;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
(*env)->ReleaseByteArrayElements(env, packet, packet_bytes, JNI_ABORT);
|
||||
(*env)->ReleaseShortArrayElements(env, pcm_out, pcm, ok ? 0 : JNI_ABORT);
|
||||
return ok ? G2_SAMPLES_PER_PACKET : 0;
|
||||
}
|
||||
|
||||
JNIEXPORT void JNICALL
|
||||
Java_com_faceclaw_app_FaceclawLc3Decoder_nativeDestroy(JNIEnv *env, jclass clazz, jlong handle) {
|
||||
(void) env;
|
||||
(void) clazz;
|
||||
faceclaw_lc3_decoder_t *state = (faceclaw_lc3_decoder_t *) (uintptr_t) handle;
|
||||
if (state == NULL) {
|
||||
return;
|
||||
}
|
||||
free(state->mem);
|
||||
free(state);
|
||||
}
|
||||
@@ -100,3 +100,4 @@ export async function ensureBlePermissions(): Promise<void> {
|
||||
export async function ensureVoicePermissions(): Promise<void> {
|
||||
await ensurePermissions([RECORD_AUDIO_PERMISSION], VOICE_PERMISSION_REQUEST_CODE, "Voice control");
|
||||
}
|
||||
|
||||
@@ -487,6 +487,8 @@ class DashboardController {
|
||||
private async setVoiceControlEnabled(enabled: boolean): Promise<void> {
|
||||
if (enabled) {
|
||||
try {
|
||||
// Treat Android mic permission as the consent gate for voice control,
|
||||
// even when the current source is the G2 mic over BLE.
|
||||
await ensureVoicePermissions();
|
||||
} catch (error) {
|
||||
this.appendLog(`voice control permission failed: ${this.formatError(error)}`);
|
||||
@@ -671,7 +673,7 @@ class DashboardController {
|
||||
if (!this.communicator) return;
|
||||
if (!isDashboardVoiceControlEnabled()) return;
|
||||
this.updateConnectedForegroundNotification();
|
||||
voiceControlBridge.start();
|
||||
voiceControlBridge.start(this.communicator.getNativeCommunicator());
|
||||
}
|
||||
|
||||
private setStopwatchRenderActive(active: boolean): void {
|
||||
|
||||
@@ -216,6 +216,10 @@ export class FaceclawCommunicatorBridge {
|
||||
return () => this.frameMetricsListeners.delete(listener);
|
||||
}
|
||||
|
||||
getNativeCommunicator(): any {
|
||||
return this.communicator;
|
||||
}
|
||||
|
||||
waitForNextFrameMetrics(timeoutMs: number): Promise<FrameMetrics | null> {
|
||||
const delayMs = Math.max(1, Math.round(nonNegativeNumber(timeoutMs)));
|
||||
return new Promise((resolve) => {
|
||||
|
||||
@@ -25,9 +25,12 @@ export class FaceclawVoiceControlBridge {
|
||||
return () => this.wakeWordListeners.delete(listener);
|
||||
}
|
||||
|
||||
start(): void {
|
||||
start(communicator?: any): void {
|
||||
if (this.started || !global.isAndroid) return;
|
||||
this.ensureController();
|
||||
if (communicator) {
|
||||
this.controller?.setCommunicator(communicator);
|
||||
}
|
||||
this.started = true;
|
||||
this.controller?.start();
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user