This commit is contained in:
James Babcock committed 2026-11-12 12:20:29 -08:00
commit c162c8e633
24 files changed
+486 -154

No files matched your search

+10 -1
View File
@@ -365,6 +365,7 @@ test('iOS preview voice uses the phone microphone, while connected capture stays
test('iOS continuous capture is shared by windows and restarts after a disconnect, a lock or backgrounding', async () => {
const calls = [], claims = [], providers = [];
let saveRecording = false;
// Models the bridge's continuous state: listening on a source, or draining after a stop.
const bridge = { source: null, draining: false, recording: '', rawSession: undefined,
supportsContinuous: voice => voice.provider !== 'onboard',
@@ -382,7 +383,7 @@ test('iOS continuous capture is shared by windows and restarts after a disconnec
setRawSession: session => { bridge.rawSession = session; } };
const settings = { ...voiceSettings, voiceProviderSetting: { get: () => 'whisper-server' },
whisperServerUrlSetting: { get: () => 'http://whisper.local:8000' }, whisperServerLanguageSetting: { get: () => 'en' },
saveTranscribeAudioSetting: { get: () => true } };
saveRecordingSetting: { get: () => saveRecording } };
const { IosPreviewController } = load('app/g2/ios-preview-controller.ts', {
require: id => id === '../native/ios-voice-input' ? { iosVoiceInput: bridge } : id === '../ui/dashboard-settings' ? settings
: id === '../ui/shell/voice-activity' ? { voiceActivity: { setActive: (active, holder) => claims.push([active, holder]) } }
@@ -401,7 +402,15 @@ test('iOS continuous capture is shared by windows and restarts after a disconnec
host.startContinuousVoiceCapture('transcribe', { recordAs: 'transcribe-1' }); await settle();
host.startContinuousVoiceCapture('calculator'); await settle();
assert.deepEqual(calls, [['glasses', true]], 'the second window shares the capture');
// Save Recording (in the window's menu) starts and stops the recording while it is open.
assert.equal(bridge.recording, '');
saveRecording = true; host.syncContinuousRecording();
assert.equal(bridge.recording, 'transcribe-1');
saveRecording = false; host.syncContinuousRecording();
assert.equal(bridge.recording, '');
saveRecording = true; host.syncContinuousRecording();
assert.equal(bridge.recording, 'transcribe-1');
assert.deepEqual(calls, [['glasses', true]], 'without touching the capture');
assert.deepEqual(claims.at(-1), [true, 'continuous-capture']);
host.stopContinuousVoiceCapture('calculator');
assert.deepEqual(calls, [['glasses', true]], 'closing one window keeps the other listening');
+34 -1
View File
@@ -31,8 +31,13 @@ function harness() {
},
};
const recordings = [];
let failRecordings = false, recordingAttempts = 0;
class FakeRecorder {
static open(name) { const r = new FakeRecorder(); r.name = name; r.path = `/Downloads/${name}.wav`; r.pcm = []; r.closed = false; recordings.push(r); return r; }
static open(name) {
recordingAttempts++;
if (failRecordings) return null;
const r = new FakeRecorder(); r.name = name; r.path = `/Downloads/${name}.wav`; r.pcm = []; r.closed = false; recordings.push(r); return r;
}
append(bytes) { this.pcm.push(...bytes); }
close() { this.closed = true; }
}
@@ -56,6 +61,7 @@ function harness() {
} };
const onboard = { provider: 'onboard', elevenLabsApiKey: '', openAiApiKey: '', sonioxApiKey: '' };
return { bridge, native, session, audio, commands, text, timers, clouds, analyzers, recordings, release: () => release(),
failRecordings: value => { failRecordings = value; }, recordingAttempts: () => recordingAttempts,
analyzerSupported: value => { analyzerSupported = value; },
onboard, cloud: { ...onboard, provider: 'soniox', sonioxApiKey: 'key' },
microphoneAuthorization: value => { microphoneAuthorization = value; }, microphoneRequests: () => microphoneRequests,
@@ -307,7 +313,10 @@ test('push-to-talk after Transcribe closed starts its own capture instead of joi
test('continuous captures record to the requested WAV, created by their first audio and kept across restarts', async () => {
const h = harness();
let changes = 0; h.bridge.onRecordingChange(() => changes++);
assert.equal(h.bridge.recording, false);
h.bridge.recordTo('transcribe-2026-10-17-120000');
assert.deepEqual([h.bridge.recording, h.bridge.recordingFailed, changes], [true, false, 1]);
// A start that fails before any audio leaves no file behind.
h.native.error = 'No phone microphone is available.';
await h.bridge.startPhoneCapture(h.cloud, () => {}, false, true);
@@ -334,6 +343,30 @@ test('continuous captures record to the requested WAV, created by their first au
assert.deepEqual(h.recordings[0].pcm, [1, 2, 3, 4, 5, 6, 7]);
h.bridge.recordTo('');
assert.equal(h.recordings[0].closed, true);
assert.deepEqual([h.bridge.recording, changes], [false, 2]);
});
test('a recording whose file cannot be created is reported, and not retried per packet', async () => {
const h = harness();
let changes = 0; h.bridge.onRecordingChange(() => changes++);
h.failRecordings(true);
h.bridge.recordTo('transcribe-1');
const start = h.bridge.startGlassesCapture(h.session, h.cloud, () => {}, false, true); h.release(); await start;
h.native.pcmHandler(new Uint8Array([1, 2]));
h.native.pcmHandler(new Uint8Array([3, 4]));
assert.deepEqual([h.bridge.recording, h.bridge.recordingFailed, changes, h.recordingAttempts()], [false, true, 2, 1]);
// Asking for the same recording again changes nothing; another one (Save Recording
// switched off and on) is a fresh attempt.
h.bridge.recordTo('transcribe-1');
h.native.pcmHandler(new Uint8Array([5]));
assert.equal(h.recordingAttempts(), 1);
h.failRecordings(false);
h.bridge.recordTo('');
assert.deepEqual([h.bridge.recording, h.bridge.recordingFailed, changes], [false, false, 3]);
h.bridge.recordTo('transcribe-1');
h.native.pcmHandler(new Uint8Array([6]));
assert.deepEqual([h.bridge.recording, h.bridge.recordingFailed, changes], [true, false, 4]);
assert.deepEqual(h.recordings[0].pcm, [6]);
});
test('a glasses session ending drains continuous capture instead of discarding what it heard', async () => {
@@ -21,4 +21,31 @@ class WavRecorderRepairTest {
file.delete()
}
}
@Test
fun keepsTheHeaderCurrentWhileRecording() {
val file = File.createTempFile("recording", ".wav")
try {
val recorder = FaceclawWavRecorder(file.path, 16000, 1)
// Under a second of audio: still the empty header from the start.
recorder.append(ByteArray(16000))
assertEquals(BinaryEncoding.wavHeader(0, 16000, 1).toList(), file.readBytes().copyOf(44).toList())
// Past a second the header covers what is on disk, without waiting for finish(), and
// later audio still lands at the end.
recorder.append(ByteArray(16000) { 1 })
recorder.append(ByteArray(100) { 2 })
var bytes = file.readBytes()
assertEquals(44 + 32100, bytes.size)
assertEquals(BinaryEncoding.wavHeader(32000, 16000, 1).toList(), bytes.copyOf(44).toList())
assertEquals(0.toByte(), bytes[44])
assertEquals(1.toByte(), bytes[44 + 16000])
assertEquals(2.toByte(), bytes.last())
assertEquals(32100L, recorder.finish())
bytes = file.readBytes()
assertEquals(44 + 32100, bytes.size)
assertEquals(BinaryEncoding.wavHeader(32100, 16000, 1).toList(), bytes.copyOf(44).toList())
} finally {
file.delete()
}
}
}
@@ -63,7 +63,6 @@ class VoiceCaptureSessionTest {
override val dispatcher = CallbackDispatcher { it() }
var packetListener: FaceclawAudioPacketListener? = null
var g2Stops = 0
val recordings = mutableListOf<ByteArray>()
override fun onSessionStarting() {}
override fun isG2SessionReady(): Boolean = g2Ready
override fun isG2AudioCaptureActive(): Boolean = packetListener != null
@@ -96,7 +95,6 @@ class VoiceCaptureSessionTest {
override fun embed(pcm16le: ByteArray, sampleRate: Int): FloatArray? = embedding
override fun release() {}
}
override fun recordingSink(): RecordingSink = RecordingSink { bytes, _ -> recordings.add(bytes) }
}
private val logs = mutableListOf<String>()
@@ -196,7 +194,6 @@ class VoiceCaptureSessionTest {
val session = session(host, platform, "voice-test-onboard")
session.setListener(events)
session.setUsePhoneMic(true)
session.setSaveRecordings(true)
session.start("onboard", 1)
assertTrue(events.stoppedLatch.await(10000))
assertTrue(mic.closed)
@@ -211,8 +208,6 @@ class VoiceCaptureSessionTest {
// Every segment handed to the model is at most 8 s and was peak-normalized.
assertTrue(transcriber.calls.all { it in 1..VoiceCaptureSession.TRANSCRIPT_SEGMENT_MAX_SAMPLES })
assertTrue(logs.any { it.startsWith("I Moonshine decode final=true") }, logs.toString())
assertEquals(1, host.recordings.size)
assertEquals(170 * 800 * 2, host.recordings[0].size)
assertEquals(170, events.pcmChunks.size)
assertEquals("x x", VoiceCaptureSession.joinTranscript("x", "x"))
assertEquals("x, y", VoiceCaptureSession.joinTranscript("x", ", y"))
+45 -4
View File
@@ -7,7 +7,7 @@ const { pcmLevel } = loader()('app/util/audio-level.ts');
function harness() {
const levelListeners = new Set();
let tick = null;
const { MicLevelMeter, paintListeningHeader } = loader({
const { MicLevelMeter, paintListeningHeader, RECORDING_ICON, withRecordingIcon } = loader({
setInterval(fn) { tick = fn; return 1; },
clearInterval() { tick = null; },
}, {
@@ -16,14 +16,15 @@ function harness() {
} },
})('app/ui/listening-header.ts');
const level = value => { for (const listener of levelListeners) listener(value); };
return { MicLevelMeter, paintListeningHeader, level, tick: () => tick?.(), ticking: () => tick !== null, levelListeners };
return { MicLevelMeter, paintListeningHeader, RECORDING_ICON, withRecordingIcon, level, tick: () => tick?.(), ticking: () => tick !== null, levelListeners };
}
/** Records draws; text is 6px per character, like a fixed-width bitmap face. */
function fakeImage(width = 576) {
const texts = [], rects = [];
return { width, texts, rects,
const texts = [], rects = [], images = [];
return { width, texts, rects, images,
drawText(_font, x, y, text, gray) { texts.push({ x, y, text, gray }); },
drawImage(source, x, y) { images.push({ source, x, y }); },
fillRect(x, y, w, h, gray) { rects.push({ x, y, w, h, gray }); } };
}
const font = { ascent: 12, lineHeight: 16, measureText: text => text.length * 6 };
@@ -96,8 +97,48 @@ test('the header puts title and status on one line, truncated before the meter',
// Segments sit on the text baseline.
assert.ok(image.rects.every(rect => rect.y + rect.h === 8 + font.ascent));
assert.equal(image.images.length, 0, 'no recording circle unless the audio is being saved');
const idle = fakeImage(300);
h.paintListeningHeader(idle, font, 'Transcribe', 'Starting microphone...', null, padding);
assert.equal(idle.rects.length, 0);
assert.equal(idle.texts[1].text, ' · Starting microphone...');
});
test('while the audio is being saved, the recording circle sits beside the meter', () => {
const h = harness();
const meter = new h.MicLevelMeter();
const image = fakeImage(300);
const padding = { top: 8, right: 0, bottom: 8, left: 8 };
h.paintListeningHeader(image, font, 'Translate', 'Listening (ElevenLabs)...', meter, padding, true);
const meterX = Math.min(...image.rects.map(rect => rect.x));
assert.equal(meterX, 300 - meter.width, 'the meter stays where it was');
assert.equal(image.images.length, 1);
const [circle] = image.images;
assert.equal(circle.source, h.RECORDING_ICON);
assert.equal(circle.x + circle.source.width + 4, meterX);
// Centred on the meter, whose 7 rows sit on the text baseline.
const meterTop = Math.min(...image.rects.map(rect => rect.y));
assert.equal(circle.y + circle.source.height / 2, meterTop + 7 / 2);
const status = image.texts[1].text;
assert.ok(8 + 54 + status.length * 6 <= circle.x - 10, 'status stops short of the circle');
// The mic isn't live (starting, or waiting for the glasses): nothing is being saved to mark.
const idle = fakeImage(300);
h.paintListeningHeader(idle, font, 'Transcribe', 'Starting microphone...', null, padding, true);
assert.equal(idle.images.length, 0);
});
test('a tray icon gains the recording circle to its right, vertically centred', () => {
const h = harness();
const { GrayImage } = loader()('app/graphics/image.ts');
const mic = new GrayImage(14, 15, 0);
mic.setPixel(0, 0, 200); mic.setPixel(13, 14, 200);
const icon = h.withRecordingIcon(mic);
assert.deepEqual([icon.width, icon.height], [14 + 2 + 9, 15]);
assert.deepEqual([icon.getPixel(0, 0), icon.getPixel(13, 14)], [200, 200]);
assert.deepEqual([icon.getPixel(14, 7), icon.getPixel(15, 7)], [0, 0], 'a gap separates them');
// The circle's widest row is the icon's middle row; its corners are clear.
for (let x = 16; x < 25; x++) assert.equal(icon.getPixel(x, 7), 220);
assert.deepEqual([icon.getPixel(16, 3), icon.getPixel(20, 3), icon.getPixel(20, 2), icon.getPixel(20, 12)], [0, 220, 0, 0]);
});
+58 -2
View File
@@ -158,11 +158,11 @@ test('cloud fallback and a synchronous final still send exactly once', () => {
});
/** The Android voice bridge over a fake native controller (on-device recognition only). */
function androidBridge() {
function androidBridge(modules = {}) {
const controller = {
starts: 0,
setListener(listener) { this.listener = listener; },
setCommunicator() {}, setUsePhoneMic() {}, setSaveRecordings() {}, setEndpointing() {},
setCommunicator() {}, setUsePhoneMic() {}, setEndpointing() {},
setNoiseSuppression() {}, setBeamFilter() {}, clearSpeakerVerification() {}, setOnboardModelKind() {},
start(_mode, id) { this.captureId = id; this.active = true; this.starts++; },
stop() { this.active = false; },
@@ -172,6 +172,7 @@ function androidBridge() {
'@nativescript/core': { Utils: { android: { getApplicationContext: () => ({}) } } },
'./speech-pause': { SpeechPauseDetector: class { reset() {} } },
'./cloud-stt-provider': { createCloudSttClient: () => null, transcriptionKey: options => options.provider },
...modules,
}, { global: { isAndroid: true }, com: { faceclaw: { app: {
FaceclawVoiceController: function () { return controller; },
FaceclawVoiceControllerListener: function (listener) { return listener; },
@@ -215,3 +216,58 @@ test('a shared capture restarts for a holder that needs another provider, and is
assert.equal(controller.active, true);
assert.equal(bridge.holdsCapture('ptt'), true);
});
test('Android continuous captures record to the requested WAV, and report one that cannot be created', () => {
const recordings = [];
let fail = false, attempts = 0;
class FakeRecorder {
static open(name) {
attempts++;
if (fail) return null;
const r = new FakeRecorder(); r.name = name; r.path = `/Download/${name}.wav`; r.pcm = []; r.closed = false; recordings.push(r); return r;
}
append(pcm) { this.pcm.push(...pcm); }
close() { this.closed = true; }
}
const { bridge, controller } = androidBridge({
'./wav-recorder': { WavRecorder: FakeRecorder },
'./speech-pause': { SpeechPauseDetector: class { reset() {} accept() { return false; } } },
'../util/array-util': { toUint8Array: pcm => Uint8Array.from(pcm) },
'../util/audio-level': { pcmLevel: () => 0 },
});
const options = { provider: 'onboard' };
let changes = 0; bridge.onRecordingChange(() => changes++);
bridge.recordTo('transcribe-1');
assert.deepEqual([bridge.recording, bridge.recordingFailed, changes], [true, false, 1]);
// Push-to-talk on a capture of its own is not recorded, and the file waits for audio to save.
bridge.startPushToTalk(options);
controller.listener.onPcm([9, 9]);
assert.equal(attempts, 0);
bridge.startContinuousCapture(options);
controller.listener.onPcm([1, 2]);
controller.listener.onPcm([3, 4]);
assert.deepEqual(recordings.map(r => [r.name, r.pcm, r.closed]), [['transcribe-1', [1, 2, 3, 4], false]]);
// The file outlives a capture: one restarted after the glasses reconnect keeps appending to it.
bridge.handleSessionEnded();
bridge.resumeCapture(options);
controller.listener.onPcm([5]);
assert.deepEqual(recordings.map(r => [r.pcm, r.closed]), [[[1, 2, 3, 4, 5], false]]);
bridge.recordTo('');
assert.deepEqual([recordings[0].closed, bridge.recording, changes], [true, false, 2]);
controller.listener.onPcm([6]);
assert.equal(recordings[0].pcm.length, 5);
// One attempt per recording, not one per packet; another recording is a fresh attempt.
fail = true; attempts = 0;
bridge.recordTo('transcribe-2');
controller.listener.onPcm([7]); controller.listener.onPcm([8]);
assert.deepEqual([bridge.recording, bridge.recordingFailed, changes, attempts], [false, true, 4, 1]);
bridge.recordTo('transcribe-2');
controller.listener.onPcm([9]);
assert.equal(attempts, 1);
fail = false;
bridge.recordTo('');
bridge.recordTo('transcribe-2');
controller.listener.onPcm([10]);
assert.deepEqual([bridge.recording, bridge.recordingFailed, recordings.at(-1).pcm], [true, false, [10]]);
});