mirror of
https://git.undefined-dev.fr/Blendman974/faceclaw.git
synced 2026-11-19 21:03:09 +00:00
Merge branch 'main' of https://github.com/jimrandomh/faceclaw
This commit is contained in:
commit
c162c8e633
24 files changed
+486
-154
No files matched your search
@@ -365,6 +365,7 @@ test('iOS preview voice uses the phone microphone, while connected capture stays
|
||||
|
||||
test('iOS continuous capture is shared by windows and restarts after a disconnect, a lock or backgrounding', async () => {
|
||||
const calls = [], claims = [], providers = [];
|
||||
let saveRecording = false;
|
||||
// Models the bridge's continuous state: listening on a source, or draining after a stop.
|
||||
const bridge = { source: null, draining: false, recording: '', rawSession: undefined,
|
||||
supportsContinuous: voice => voice.provider !== 'onboard',
|
||||
@@ -382,7 +383,7 @@ test('iOS continuous capture is shared by windows and restarts after a disconnec
|
||||
setRawSession: session => { bridge.rawSession = session; } };
|
||||
const settings = { ...voiceSettings, voiceProviderSetting: { get: () => 'whisper-server' },
|
||||
whisperServerUrlSetting: { get: () => 'http://whisper.local:8000' }, whisperServerLanguageSetting: { get: () => 'en' },
|
||||
saveTranscribeAudioSetting: { get: () => true } };
|
||||
saveRecordingSetting: { get: () => saveRecording } };
|
||||
const { IosPreviewController } = load('app/g2/ios-preview-controller.ts', {
|
||||
require: id => id === '../native/ios-voice-input' ? { iosVoiceInput: bridge } : id === '../ui/dashboard-settings' ? settings
|
||||
: id === '../ui/shell/voice-activity' ? { voiceActivity: { setActive: (active, holder) => claims.push([active, holder]) } }
|
||||
@@ -401,7 +402,15 @@ test('iOS continuous capture is shared by windows and restarts after a disconnec
|
||||
host.startContinuousVoiceCapture('transcribe', { recordAs: 'transcribe-1' }); await settle();
|
||||
host.startContinuousVoiceCapture('calculator'); await settle();
|
||||
assert.deepEqual(calls, [['glasses', true]], 'the second window shares the capture');
|
||||
// Save Recording (in the window's menu) starts and stops the recording while it is open.
|
||||
assert.equal(bridge.recording, '');
|
||||
saveRecording = true; host.syncContinuousRecording();
|
||||
assert.equal(bridge.recording, 'transcribe-1');
|
||||
saveRecording = false; host.syncContinuousRecording();
|
||||
assert.equal(bridge.recording, '');
|
||||
saveRecording = true; host.syncContinuousRecording();
|
||||
assert.equal(bridge.recording, 'transcribe-1');
|
||||
assert.deepEqual(calls, [['glasses', true]], 'without touching the capture');
|
||||
assert.deepEqual(claims.at(-1), [true, 'continuous-capture']);
|
||||
host.stopContinuousVoiceCapture('calculator');
|
||||
assert.deepEqual(calls, [['glasses', true]], 'closing one window keeps the other listening');
|
||||
|
||||
@@ -31,8 +31,13 @@ function harness() {
|
||||
},
|
||||
};
|
||||
const recordings = [];
|
||||
let failRecordings = false, recordingAttempts = 0;
|
||||
class FakeRecorder {
|
||||
static open(name) { const r = new FakeRecorder(); r.name = name; r.path = `/Downloads/${name}.wav`; r.pcm = []; r.closed = false; recordings.push(r); return r; }
|
||||
static open(name) {
|
||||
recordingAttempts++;
|
||||
if (failRecordings) return null;
|
||||
const r = new FakeRecorder(); r.name = name; r.path = `/Downloads/${name}.wav`; r.pcm = []; r.closed = false; recordings.push(r); return r;
|
||||
}
|
||||
append(bytes) { this.pcm.push(...bytes); }
|
||||
close() { this.closed = true; }
|
||||
}
|
||||
@@ -56,6 +61,7 @@ function harness() {
|
||||
} };
|
||||
const onboard = { provider: 'onboard', elevenLabsApiKey: '', openAiApiKey: '', sonioxApiKey: '' };
|
||||
return { bridge, native, session, audio, commands, text, timers, clouds, analyzers, recordings, release: () => release(),
|
||||
failRecordings: value => { failRecordings = value; }, recordingAttempts: () => recordingAttempts,
|
||||
analyzerSupported: value => { analyzerSupported = value; },
|
||||
onboard, cloud: { ...onboard, provider: 'soniox', sonioxApiKey: 'key' },
|
||||
microphoneAuthorization: value => { microphoneAuthorization = value; }, microphoneRequests: () => microphoneRequests,
|
||||
@@ -307,7 +313,10 @@ test('push-to-talk after Transcribe closed starts its own capture instead of joi
|
||||
|
||||
test('continuous captures record to the requested WAV, created by their first audio and kept across restarts', async () => {
|
||||
const h = harness();
|
||||
let changes = 0; h.bridge.onRecordingChange(() => changes++);
|
||||
assert.equal(h.bridge.recording, false);
|
||||
h.bridge.recordTo('transcribe-2026-10-17-120000');
|
||||
assert.deepEqual([h.bridge.recording, h.bridge.recordingFailed, changes], [true, false, 1]);
|
||||
// A start that fails before any audio leaves no file behind.
|
||||
h.native.error = 'No phone microphone is available.';
|
||||
await h.bridge.startPhoneCapture(h.cloud, () => {}, false, true);
|
||||
@@ -334,6 +343,30 @@ test('continuous captures record to the requested WAV, created by their first au
|
||||
assert.deepEqual(h.recordings[0].pcm, [1, 2, 3, 4, 5, 6, 7]);
|
||||
h.bridge.recordTo('');
|
||||
assert.equal(h.recordings[0].closed, true);
|
||||
assert.deepEqual([h.bridge.recording, changes], [false, 2]);
|
||||
});
|
||||
|
||||
test('a recording whose file cannot be created is reported, and not retried per packet', async () => {
|
||||
const h = harness();
|
||||
let changes = 0; h.bridge.onRecordingChange(() => changes++);
|
||||
h.failRecordings(true);
|
||||
h.bridge.recordTo('transcribe-1');
|
||||
const start = h.bridge.startGlassesCapture(h.session, h.cloud, () => {}, false, true); h.release(); await start;
|
||||
h.native.pcmHandler(new Uint8Array([1, 2]));
|
||||
h.native.pcmHandler(new Uint8Array([3, 4]));
|
||||
assert.deepEqual([h.bridge.recording, h.bridge.recordingFailed, changes, h.recordingAttempts()], [false, true, 2, 1]);
|
||||
// Asking for the same recording again changes nothing; another one (Save Recording
|
||||
// switched off and on) is a fresh attempt.
|
||||
h.bridge.recordTo('transcribe-1');
|
||||
h.native.pcmHandler(new Uint8Array([5]));
|
||||
assert.equal(h.recordingAttempts(), 1);
|
||||
h.failRecordings(false);
|
||||
h.bridge.recordTo('');
|
||||
assert.deepEqual([h.bridge.recording, h.bridge.recordingFailed, changes], [false, false, 3]);
|
||||
h.bridge.recordTo('transcribe-1');
|
||||
h.native.pcmHandler(new Uint8Array([6]));
|
||||
assert.deepEqual([h.bridge.recording, h.bridge.recordingFailed, changes], [true, false, 4]);
|
||||
assert.deepEqual(h.recordings[0].pcm, [6]);
|
||||
});
|
||||
|
||||
test('a glasses session ending drains continuous capture instead of discarding what it heard', async () => {
|
||||
|
||||
@@ -21,4 +21,31 @@ class WavRecorderRepairTest {
|
||||
file.delete()
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
fun keepsTheHeaderCurrentWhileRecording() {
|
||||
val file = File.createTempFile("recording", ".wav")
|
||||
try {
|
||||
val recorder = FaceclawWavRecorder(file.path, 16000, 1)
|
||||
// Under a second of audio: still the empty header from the start.
|
||||
recorder.append(ByteArray(16000))
|
||||
assertEquals(BinaryEncoding.wavHeader(0, 16000, 1).toList(), file.readBytes().copyOf(44).toList())
|
||||
// Past a second the header covers what is on disk, without waiting for finish(), and
|
||||
// later audio still lands at the end.
|
||||
recorder.append(ByteArray(16000) { 1 })
|
||||
recorder.append(ByteArray(100) { 2 })
|
||||
var bytes = file.readBytes()
|
||||
assertEquals(44 + 32100, bytes.size)
|
||||
assertEquals(BinaryEncoding.wavHeader(32000, 16000, 1).toList(), bytes.copyOf(44).toList())
|
||||
assertEquals(0.toByte(), bytes[44])
|
||||
assertEquals(1.toByte(), bytes[44 + 16000])
|
||||
assertEquals(2.toByte(), bytes.last())
|
||||
assertEquals(32100L, recorder.finish())
|
||||
bytes = file.readBytes()
|
||||
assertEquals(44 + 32100, bytes.size)
|
||||
assertEquals(BinaryEncoding.wavHeader(32100, 16000, 1).toList(), bytes.copyOf(44).toList())
|
||||
} finally {
|
||||
file.delete()
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -63,7 +63,6 @@ class VoiceCaptureSessionTest {
|
||||
override val dispatcher = CallbackDispatcher { it() }
|
||||
var packetListener: FaceclawAudioPacketListener? = null
|
||||
var g2Stops = 0
|
||||
val recordings = mutableListOf<ByteArray>()
|
||||
override fun onSessionStarting() {}
|
||||
override fun isG2SessionReady(): Boolean = g2Ready
|
||||
override fun isG2AudioCaptureActive(): Boolean = packetListener != null
|
||||
@@ -96,7 +95,6 @@ class VoiceCaptureSessionTest {
|
||||
override fun embed(pcm16le: ByteArray, sampleRate: Int): FloatArray? = embedding
|
||||
override fun release() {}
|
||||
}
|
||||
override fun recordingSink(): RecordingSink = RecordingSink { bytes, _ -> recordings.add(bytes) }
|
||||
}
|
||||
|
||||
private val logs = mutableListOf<String>()
|
||||
@@ -196,7 +194,6 @@ class VoiceCaptureSessionTest {
|
||||
val session = session(host, platform, "voice-test-onboard")
|
||||
session.setListener(events)
|
||||
session.setUsePhoneMic(true)
|
||||
session.setSaveRecordings(true)
|
||||
session.start("onboard", 1)
|
||||
assertTrue(events.stoppedLatch.await(10000))
|
||||
assertTrue(mic.closed)
|
||||
@@ -211,8 +208,6 @@ class VoiceCaptureSessionTest {
|
||||
// Every segment handed to the model is at most 8 s and was peak-normalized.
|
||||
assertTrue(transcriber.calls.all { it in 1..VoiceCaptureSession.TRANSCRIPT_SEGMENT_MAX_SAMPLES })
|
||||
assertTrue(logs.any { it.startsWith("I Moonshine decode final=true") }, logs.toString())
|
||||
assertEquals(1, host.recordings.size)
|
||||
assertEquals(170 * 800 * 2, host.recordings[0].size)
|
||||
assertEquals(170, events.pcmChunks.size)
|
||||
assertEquals("x x", VoiceCaptureSession.joinTranscript("x", "x"))
|
||||
assertEquals("x, y", VoiceCaptureSession.joinTranscript("x", ", y"))
|
||||
|
||||
@@ -7,7 +7,7 @@ const { pcmLevel } = loader()('app/util/audio-level.ts');
|
||||
function harness() {
|
||||
const levelListeners = new Set();
|
||||
let tick = null;
|
||||
const { MicLevelMeter, paintListeningHeader } = loader({
|
||||
const { MicLevelMeter, paintListeningHeader, RECORDING_ICON, withRecordingIcon } = loader({
|
||||
setInterval(fn) { tick = fn; return 1; },
|
||||
clearInterval() { tick = null; },
|
||||
}, {
|
||||
@@ -16,14 +16,15 @@ function harness() {
|
||||
} },
|
||||
})('app/ui/listening-header.ts');
|
||||
const level = value => { for (const listener of levelListeners) listener(value); };
|
||||
return { MicLevelMeter, paintListeningHeader, level, tick: () => tick?.(), ticking: () => tick !== null, levelListeners };
|
||||
return { MicLevelMeter, paintListeningHeader, RECORDING_ICON, withRecordingIcon, level, tick: () => tick?.(), ticking: () => tick !== null, levelListeners };
|
||||
}
|
||||
|
||||
/** Records draws; text is 6px per character, like a fixed-width bitmap face. */
|
||||
function fakeImage(width = 576) {
|
||||
const texts = [], rects = [];
|
||||
return { width, texts, rects,
|
||||
const texts = [], rects = [], images = [];
|
||||
return { width, texts, rects, images,
|
||||
drawText(_font, x, y, text, gray) { texts.push({ x, y, text, gray }); },
|
||||
drawImage(source, x, y) { images.push({ source, x, y }); },
|
||||
fillRect(x, y, w, h, gray) { rects.push({ x, y, w, h, gray }); } };
|
||||
}
|
||||
const font = { ascent: 12, lineHeight: 16, measureText: text => text.length * 6 };
|
||||
@@ -96,8 +97,48 @@ test('the header puts title and status on one line, truncated before the meter',
|
||||
// Segments sit on the text baseline.
|
||||
assert.ok(image.rects.every(rect => rect.y + rect.h === 8 + font.ascent));
|
||||
|
||||
assert.equal(image.images.length, 0, 'no recording circle unless the audio is being saved');
|
||||
|
||||
const idle = fakeImage(300);
|
||||
h.paintListeningHeader(idle, font, 'Transcribe', 'Starting microphone...', null, padding);
|
||||
assert.equal(idle.rects.length, 0);
|
||||
assert.equal(idle.texts[1].text, ' · Starting microphone...');
|
||||
});
|
||||
|
||||
test('while the audio is being saved, the recording circle sits beside the meter', () => {
|
||||
const h = harness();
|
||||
const meter = new h.MicLevelMeter();
|
||||
const image = fakeImage(300);
|
||||
const padding = { top: 8, right: 0, bottom: 8, left: 8 };
|
||||
h.paintListeningHeader(image, font, 'Translate', 'Listening (ElevenLabs)...', meter, padding, true);
|
||||
const meterX = Math.min(...image.rects.map(rect => rect.x));
|
||||
assert.equal(meterX, 300 - meter.width, 'the meter stays where it was');
|
||||
assert.equal(image.images.length, 1);
|
||||
const [circle] = image.images;
|
||||
assert.equal(circle.source, h.RECORDING_ICON);
|
||||
assert.equal(circle.x + circle.source.width + 4, meterX);
|
||||
// Centred on the meter, whose 7 rows sit on the text baseline.
|
||||
const meterTop = Math.min(...image.rects.map(rect => rect.y));
|
||||
assert.equal(circle.y + circle.source.height / 2, meterTop + 7 / 2);
|
||||
const status = image.texts[1].text;
|
||||
assert.ok(8 + 54 + status.length * 6 <= circle.x - 10, 'status stops short of the circle');
|
||||
|
||||
// The mic isn't live (starting, or waiting for the glasses): nothing is being saved to mark.
|
||||
const idle = fakeImage(300);
|
||||
h.paintListeningHeader(idle, font, 'Transcribe', 'Starting microphone...', null, padding, true);
|
||||
assert.equal(idle.images.length, 0);
|
||||
});
|
||||
|
||||
test('a tray icon gains the recording circle to its right, vertically centred', () => {
|
||||
const h = harness();
|
||||
const { GrayImage } = loader()('app/graphics/image.ts');
|
||||
const mic = new GrayImage(14, 15, 0);
|
||||
mic.setPixel(0, 0, 200); mic.setPixel(13, 14, 200);
|
||||
const icon = h.withRecordingIcon(mic);
|
||||
assert.deepEqual([icon.width, icon.height], [14 + 2 + 9, 15]);
|
||||
assert.deepEqual([icon.getPixel(0, 0), icon.getPixel(13, 14)], [200, 200]);
|
||||
assert.deepEqual([icon.getPixel(14, 7), icon.getPixel(15, 7)], [0, 0], 'a gap separates them');
|
||||
// The circle's widest row is the icon's middle row; its corners are clear.
|
||||
for (let x = 16; x < 25; x++) assert.equal(icon.getPixel(x, 7), 220);
|
||||
assert.deepEqual([icon.getPixel(16, 3), icon.getPixel(20, 3), icon.getPixel(20, 2), icon.getPixel(20, 12)], [0, 220, 0, 0]);
|
||||
});
|
||||
@@ -158,11 +158,11 @@ test('cloud fallback and a synchronous final still send exactly once', () => {
|
||||
});
|
||||
|
||||
/** The Android voice bridge over a fake native controller (on-device recognition only). */
|
||||
function androidBridge() {
|
||||
function androidBridge(modules = {}) {
|
||||
const controller = {
|
||||
starts: 0,
|
||||
setListener(listener) { this.listener = listener; },
|
||||
setCommunicator() {}, setUsePhoneMic() {}, setSaveRecordings() {}, setEndpointing() {},
|
||||
setCommunicator() {}, setUsePhoneMic() {}, setEndpointing() {},
|
||||
setNoiseSuppression() {}, setBeamFilter() {}, clearSpeakerVerification() {}, setOnboardModelKind() {},
|
||||
start(_mode, id) { this.captureId = id; this.active = true; this.starts++; },
|
||||
stop() { this.active = false; },
|
||||
@@ -172,6 +172,7 @@ function androidBridge() {
|
||||
'@nativescript/core': { Utils: { android: { getApplicationContext: () => ({}) } } },
|
||||
'./speech-pause': { SpeechPauseDetector: class { reset() {} } },
|
||||
'./cloud-stt-provider': { createCloudSttClient: () => null, transcriptionKey: options => options.provider },
|
||||
...modules,
|
||||
}, { global: { isAndroid: true }, com: { faceclaw: { app: {
|
||||
FaceclawVoiceController: function () { return controller; },
|
||||
FaceclawVoiceControllerListener: function (listener) { return listener; },
|
||||
@@ -215,3 +216,58 @@ test('a shared capture restarts for a holder that needs another provider, and is
|
||||
assert.equal(controller.active, true);
|
||||
assert.equal(bridge.holdsCapture('ptt'), true);
|
||||
});
|
||||
|
||||
test('Android continuous captures record to the requested WAV, and report one that cannot be created', () => {
|
||||
const recordings = [];
|
||||
let fail = false, attempts = 0;
|
||||
class FakeRecorder {
|
||||
static open(name) {
|
||||
attempts++;
|
||||
if (fail) return null;
|
||||
const r = new FakeRecorder(); r.name = name; r.path = `/Download/${name}.wav`; r.pcm = []; r.closed = false; recordings.push(r); return r;
|
||||
}
|
||||
append(pcm) { this.pcm.push(...pcm); }
|
||||
close() { this.closed = true; }
|
||||
}
|
||||
const { bridge, controller } = androidBridge({
|
||||
'./wav-recorder': { WavRecorder: FakeRecorder },
|
||||
'./speech-pause': { SpeechPauseDetector: class { reset() {} accept() { return false; } } },
|
||||
'../util/array-util': { toUint8Array: pcm => Uint8Array.from(pcm) },
|
||||
'../util/audio-level': { pcmLevel: () => 0 },
|
||||
});
|
||||
const options = { provider: 'onboard' };
|
||||
let changes = 0; bridge.onRecordingChange(() => changes++);
|
||||
bridge.recordTo('transcribe-1');
|
||||
assert.deepEqual([bridge.recording, bridge.recordingFailed, changes], [true, false, 1]);
|
||||
// Push-to-talk on a capture of its own is not recorded, and the file waits for audio to save.
|
||||
bridge.startPushToTalk(options);
|
||||
controller.listener.onPcm([9, 9]);
|
||||
assert.equal(attempts, 0);
|
||||
bridge.startContinuousCapture(options);
|
||||
controller.listener.onPcm([1, 2]);
|
||||
controller.listener.onPcm([3, 4]);
|
||||
assert.deepEqual(recordings.map(r => [r.name, r.pcm, r.closed]), [['transcribe-1', [1, 2, 3, 4], false]]);
|
||||
// The file outlives a capture: one restarted after the glasses reconnect keeps appending to it.
|
||||
bridge.handleSessionEnded();
|
||||
bridge.resumeCapture(options);
|
||||
controller.listener.onPcm([5]);
|
||||
assert.deepEqual(recordings.map(r => [r.pcm, r.closed]), [[[1, 2, 3, 4, 5], false]]);
|
||||
bridge.recordTo('');
|
||||
assert.deepEqual([recordings[0].closed, bridge.recording, changes], [true, false, 2]);
|
||||
controller.listener.onPcm([6]);
|
||||
assert.equal(recordings[0].pcm.length, 5);
|
||||
|
||||
// One attempt per recording, not one per packet; another recording is a fresh attempt.
|
||||
fail = true; attempts = 0;
|
||||
bridge.recordTo('transcribe-2');
|
||||
controller.listener.onPcm([7]); controller.listener.onPcm([8]);
|
||||
assert.deepEqual([bridge.recording, bridge.recordingFailed, changes, attempts], [false, true, 4, 1]);
|
||||
bridge.recordTo('transcribe-2');
|
||||
controller.listener.onPcm([9]);
|
||||
assert.equal(attempts, 1);
|
||||
fail = false;
|
||||
bridge.recordTo('');
|
||||
bridge.recordTo('transcribe-2');
|
||||
controller.listener.onPcm([10]);
|
||||
assert.deepEqual([bridge.recording, bridge.recordingFailed, recordings.at(-1).pcm], [true, false, [10]]);
|
||||
});
|
||||
Reference in new issue
Block a user